mirror of
https://github.com/oxc-project/oxc.git
synced 2026-09-14 19:36:11 +08:00
overlookmotel
9a622c79b4
Performance improvement to tokens and comments APIs. ## The problem Previously, all tokens and comments methods would deserialize *all* tokens/comments into an array of `Token` / `Comment` / `Token | Comment` objects, and then binary search through those arrays to find the token(s) / comment(s) they're looking for. This has 2 major disadvantages: 1. Files typically contain *a lot* of tokens (even more than the number of AST nodes). Deserializing them all is very costly (up to 30% of total Oxlint runtime when run with only a JS rule which just calls a tokens-related method). 2. The binary searches these methods do are quite expensive. Even in TurboFan-optimized code, accessing `token.start` involves getting pointer to the `Token` object from the `tokens` array, an "is this object a `Token`?" safety check, then reading the `start` field from the `Token` - all just to access a single `u32`, and that happens over and over. ## This PR's solution Solve both these problems by making tokens and comments methods read `start` / `end` offsets directly from the buffers which contain the tokens/comments data. This data is tightly packed in memory, and strongly typed (read from `Uint32Array`s), so getting `start` / `end` of a token requires no indirection and no type checks. More importantly, it removes the need to deserialize all tokens / comments upfront. The desired token(s) are located, touching only the buffer, and then *only* the ones which need to be returned to rule code are deserialized into JS objects. If a rule accesses `ast.tokens`, `ast.comments`, or `sourceCode.tokensAndComments` then all tokens / comments need to be deserialized, as they're all returned to the rule as an array - but that's unavoidable. This PR doesn't make that any cheaper, but it doesn't make it measurably more costly either. But where no rule requires the full array of tokens / comments, and they only use token/comment search methods (e.g. `getFirstToken`, `getCommentsBefore`), a great deal of work will be saved. This covers the vast majority of rules. ## Implementation details The main complication is the `includeComments` option to tokens methods. When `true`, search needs to be over a combined set of both tokens and comments. When `includeComments: true` option is passed to a tokens method, a buffer is created containing data about all tokens and comments, interleaved in source code order. This buffer can then be used for binary search in tokens methods. Whether each token / comment has been deserialized already or not is tracked by a "deserialized" flag in the tokens/comments buffers. Each token / comment in the buffer is 16 bytes. This flag lives in byte 15. For tokens, this byte is always already 0 in the buffer when it arrives from Rust side. For comments, we manually set `comment.content = CommentContent::None;` for every comment on Rust side. `comment.content` is positioned at byte 15 in the `Comment` struct, and `CommentContent::None` is stored as 0. ## Possible future improvements ### SoA storage Binary search operates only on `start` field of tokens / comments, which are 16 bytes apart in the buffer. It would be more efficient if tokens were stored in struct-of-arrays (SoA) style so all `start` values were tightly packed together. This would reduce CPU cache misses in the hot loops of binary searches. ### Pre-compute tokens-and-comments buffer on Rust side The buffer containing tokens and comments, required to support `includeComments: true`, is currently generated on JS side (but lazily). We could move that to Rust side, which would be faster. However, it might be redundant work in many cases because the buffer is only required if a rule uses `includeComments: true`. We could alternatively keep the laziness optimization, by calling back into Rust to build the buffer on demand - but JS-Rust calls have a cost too. Maybe communicating via `Atomics` would be faster than an actual function call? If we had a way to share buffers with WASM, optimal solution might be to generate the buffer lazily (as now) but in WASM, which would be faster for this kind of pure number-crunching, but without the overhead of calling into Rust.
1715 lines
59 KiB
TypeScript
1715 lines
59 KiB
TypeScript
import { join as pathJoin } from "node:path";
|
|
import { beforeEach, describe, expect, it } from "vitest";
|
|
import {
|
|
getTokens,
|
|
getTokensBefore,
|
|
getTokenBefore,
|
|
getTokensAfter,
|
|
getTokenAfter,
|
|
getFirstTokens,
|
|
getFirstToken,
|
|
getLastTokens,
|
|
getLastToken,
|
|
getFirstTokensBetween,
|
|
getFirstTokenBetween,
|
|
getLastTokenBetween,
|
|
getLastTokensBetween,
|
|
getTokenByRangeStart,
|
|
getTokensBetween,
|
|
getTokenOrCommentBefore,
|
|
getTokenOrCommentAfter,
|
|
} from "../src-js/plugins/tokens_methods.ts";
|
|
import { setupFileContext, resetFileContext } from "../src-js/plugins/context.ts";
|
|
import { buffers } from "../src-js/plugins/lint.ts";
|
|
import {
|
|
initSourceText,
|
|
resetSourceAndAst,
|
|
setupSourceForFile,
|
|
} from "../src-js/plugins/source_code.ts";
|
|
import { getLoc } from "../src-js/plugins/location.ts";
|
|
import { parse as parseRaw } from "../src-js/package/parse.ts";
|
|
import { debugAssertIsNonNull } from "../src-js/utils/asserts.ts";
|
|
|
|
import type { Node } from "../src-js/plugins/types.ts";
|
|
import type { TokenOrComment } from "../src-js/plugins/tokens_and_comments.ts";
|
|
import type { BinaryExpression } from "../src-js/generated/types.d.ts";
|
|
|
|
// Source text used for most tests
|
|
const SOURCE_TEXT = "/*A*/var answer/*B*/=/*C*/a/*D*/* b/*E*///F\n call();\n/*Z*/";
|
|
|
|
/**
|
|
* Setup for a test case, using `sourceText` as source text.
|
|
*
|
|
* This function:
|
|
* - Parses source text with Oxc parser - which writes AST and source text into buffer.
|
|
* - Initializes global state, as if was linting the provided file.
|
|
* - Initializes source text.
|
|
* - Does *not* deserialize the AST - tokens methods should do that automatically where they require AST.
|
|
*
|
|
* It's not necessary to call this function before each test case, because `beforeEach` hook calls it already.
|
|
* Only use this function if test needs to set source text to a different value from default `SOURCE_TEXT`.
|
|
*
|
|
* @param sourceText - Source text to use for test
|
|
*/
|
|
function setup(sourceText: string) {
|
|
// Reset global state
|
|
resetFileContext();
|
|
resetSourceAndAst();
|
|
|
|
// Set file path
|
|
const filename = "dummy.js";
|
|
const path = pathJoin(import.meta.dirname, filename);
|
|
setupFileContext(path);
|
|
|
|
// Parse source text into buffer
|
|
parseRaw(path, sourceText);
|
|
|
|
// Set buffer (`parseRaw` adds buffer containing AST to `buffers` at index 0)
|
|
const buffer = buffers[0];
|
|
debugAssertIsNonNull(buffer);
|
|
setupSourceForFile(buffer, /* hasBOM */ false);
|
|
|
|
// Initialize source text (deserialize from buffer)
|
|
initSourceText();
|
|
}
|
|
|
|
// Reset global state and set source text to `SOURCE_TEXT` before each test.
|
|
// Individual tests can call `setup` to set a different source text if required, before calling token methods.
|
|
beforeEach(() => {
|
|
setup(SOURCE_TEXT);
|
|
});
|
|
|
|
// TODO: We are lying about `Program`'s range here.
|
|
// The range provided by TypeScript does not match the assertions for that of `espree`.
|
|
// The deviation is being corrected in upcoming releases of ESLint and TS-ESLint.
|
|
// https://eslint.org/blog/2025/10/whats-coming-in-eslint-10.0.0/#updates-to-program-ast-node-range-coverage
|
|
// https://github.com/typescript-eslint/typescript-eslint/issues/11026#issuecomment-3421887632
|
|
const Program = { range: [5, 55] } as Node;
|
|
const BinaryExpression = {
|
|
range: [26, 35],
|
|
left: { range: [26, 27] } as Node,
|
|
right: { range: [34, 35] } as Node,
|
|
} as BinaryExpression;
|
|
const VariableDeclaration = { range: [5, 35] } as Node;
|
|
const VariableDeclaratorIdentifier = { range: [9, 15] } as Node;
|
|
const CallExpression = { range: [48, 54] } as Node;
|
|
|
|
// https://github.com/eslint/eslint/blob/v9.39.1/tests/lib/languages/js/source-code/token-store.js#L62-L155
|
|
describe("when calling getTokens", () => {
|
|
it("should retrieve all tokens for root node", () => {
|
|
expect(getTokens(Program).map((token) => token.value)).toEqual([
|
|
"var",
|
|
"answer",
|
|
"=",
|
|
"a",
|
|
"*",
|
|
"b",
|
|
"call",
|
|
"(",
|
|
")",
|
|
";",
|
|
]);
|
|
});
|
|
|
|
it("should retrieve all tokens for binary expression", () => {
|
|
expect(getTokens(BinaryExpression).map((token) => token.value)).toEqual(["a", "*", "b"]);
|
|
});
|
|
|
|
it("should retrieve all tokens plus one before for binary expression", () => {
|
|
expect(getTokens(BinaryExpression, 1).map((token) => token.value)).toEqual([
|
|
"=",
|
|
"a",
|
|
"*",
|
|
"b",
|
|
]);
|
|
});
|
|
|
|
it("should retrieve all tokens plus one after for binary expression", () => {
|
|
expect(getTokens(BinaryExpression, 0, 1).map((token) => token.value)).toEqual([
|
|
"a",
|
|
"*",
|
|
"b",
|
|
"call",
|
|
]);
|
|
});
|
|
|
|
it("should retrieve all tokens plus two before and one after for binary expression", () => {
|
|
expect(getTokens(BinaryExpression, 2, 1).map((token) => token.value)).toEqual([
|
|
"answer",
|
|
"=",
|
|
"a",
|
|
"*",
|
|
"b",
|
|
"call",
|
|
]);
|
|
});
|
|
|
|
it("should retrieve all matched tokens for root node with filter", () => {
|
|
expect(getTokens(Program, (t) => t.type === "Identifier").map((token) => token.value)).toEqual([
|
|
"answer",
|
|
"a",
|
|
"b",
|
|
"call",
|
|
]);
|
|
expect(
|
|
getTokens(Program, {
|
|
filter: (t) => t.type === "Identifier",
|
|
}).map((token) => token.value),
|
|
).toEqual(["answer", "a", "b", "call"]);
|
|
});
|
|
|
|
it("should retrieve all tokens and comments in the node for root node with includeComments option", () => {
|
|
expect(getTokens(Program, { includeComments: true }).map((token) => token.value)).toEqual([
|
|
"var",
|
|
"answer",
|
|
"B",
|
|
"=",
|
|
"C",
|
|
"a",
|
|
"D",
|
|
"*",
|
|
"b",
|
|
"E",
|
|
"F",
|
|
"call",
|
|
"(",
|
|
")",
|
|
";",
|
|
]);
|
|
});
|
|
|
|
it("should retrieve matched tokens and comments in the node for root node with includeComments and filter options", () => {
|
|
expect(
|
|
getTokens(Program, {
|
|
includeComments: true,
|
|
filter: (t) => t.type.startsWith("Block"),
|
|
}).map((token) => token.value),
|
|
).toEqual(["B", "C", "D", "E"]);
|
|
});
|
|
|
|
it("should retrieve all tokens and comments in the node for binary expression with includeComments option", () => {
|
|
expect(
|
|
getTokens(BinaryExpression, { includeComments: true }).map((token) => token.value),
|
|
).toEqual(["a", "D", "*", "b"]);
|
|
});
|
|
|
|
it("should handle out of range nodes gracefully", () => {
|
|
expect(getTokens({ range: [1000, 1001] } as Node).map((token) => token.value)).toEqual([]);
|
|
});
|
|
});
|
|
|
|
// https://github.com/eslint/eslint/blob/v9.39.1/tests/lib/languages/js/source-code/token-store.js#L157-L236
|
|
describe("when calling getTokensBefore", () => {
|
|
it("should retrieve zero tokens before a node", () => {
|
|
expect(getTokensBefore(BinaryExpression, 0).map((token) => token.value)).toEqual([]);
|
|
});
|
|
|
|
it("should retrieve one token before a node", () => {
|
|
expect(getTokensBefore(BinaryExpression, 1).map((token) => token.value)).toEqual(["="]);
|
|
});
|
|
|
|
it("should retrieve more than one token before a node", () => {
|
|
expect(getTokensBefore(BinaryExpression, 2).map((token) => token.value)).toEqual([
|
|
"answer",
|
|
"=",
|
|
]);
|
|
});
|
|
|
|
it("should retrieve all tokens before a node", () => {
|
|
expect(getTokensBefore(BinaryExpression, 9e9).map((token) => token.value)).toEqual([
|
|
"var",
|
|
"answer",
|
|
"=",
|
|
]);
|
|
});
|
|
|
|
it("should retrieve more than one token before a node with count option", () => {
|
|
expect(getTokensBefore(BinaryExpression, { count: 2 }).map((token) => token.value)).toEqual([
|
|
"answer",
|
|
"=",
|
|
]);
|
|
});
|
|
|
|
it("should retrieve matched tokens before a node with count and filter options", () => {
|
|
expect(
|
|
getTokensBefore(BinaryExpression, {
|
|
count: 1,
|
|
filter: (t) => t.value !== "=",
|
|
}).map((token) => token.value),
|
|
).toEqual(["answer"]);
|
|
});
|
|
|
|
it("should retrieve all matched tokens before a node with filter option", () => {
|
|
expect(
|
|
getTokensBefore(BinaryExpression, {
|
|
filter: (t) => t.value !== "answer",
|
|
}).map((token) => token.value),
|
|
).toEqual(["var", "="]);
|
|
});
|
|
|
|
it("should retrieve zero tokens before a node with a filter when count is 0", () => {
|
|
expect(
|
|
getTokensBefore(BinaryExpression, {
|
|
count: 0,
|
|
filter: () => true,
|
|
}).map((token) => token.value),
|
|
).toEqual([]);
|
|
});
|
|
|
|
it("should retrieve no tokens before the root node", () => {
|
|
expect(getTokensBefore(Program, { count: 1 }).map((token) => token.value)).toEqual([]);
|
|
});
|
|
|
|
it("should retrieve tokens and comments before a node with count and includeComments option", () => {
|
|
expect(
|
|
getTokensBefore(BinaryExpression, {
|
|
count: 3,
|
|
includeComments: true,
|
|
}).map((token) => token.value),
|
|
).toEqual(["B", "=", "C"]);
|
|
});
|
|
|
|
it("should retrieve all tokens and comments before a node with includeComments option only", () => {
|
|
expect(
|
|
getTokensBefore(BinaryExpression, {
|
|
includeComments: true,
|
|
}).map((token) => token.value),
|
|
).toEqual(["A", "var", "answer", "B", "=", "C"]);
|
|
});
|
|
|
|
it("should retrieve all tokens and comments before a node with includeComments and filter options", () => {
|
|
expect(
|
|
getTokensBefore(BinaryExpression, {
|
|
includeComments: true,
|
|
filter: (t) => t.type.startsWith("Block"),
|
|
}).map((token) => token.value),
|
|
).toEqual(["A", "B", "C"]);
|
|
});
|
|
});
|
|
|
|
// https://github.com/eslint/eslint/blob/v9.39.1/tests/lib/languages/js/source-code/token-store.js#L238-L361
|
|
describe("when calling getTokenBefore", () => {
|
|
it("should retrieve one token before a node", () => {
|
|
expect(getTokenBefore(BinaryExpression)!.value).toBe("=");
|
|
});
|
|
|
|
it("should skip a given number of tokens", () => {
|
|
expect(getTokenBefore(BinaryExpression, 1)!.value).toBe("answer");
|
|
expect(getTokenBefore(BinaryExpression, 2)!.value).toBe("var");
|
|
});
|
|
|
|
it("should skip a given number of tokens with skip option", () => {
|
|
expect(getTokenBefore(BinaryExpression, { skip: 1 })!.value).toBe("answer");
|
|
expect(getTokenBefore(BinaryExpression, { skip: 2 })!.value).toBe("var");
|
|
});
|
|
|
|
it("should retrieve matched token with filter option", () => {
|
|
expect(getTokenBefore(BinaryExpression, (t) => t.value !== "=")!.value).toBe("answer");
|
|
});
|
|
|
|
it("should retrieve matched token with skip and filter options", () => {
|
|
expect(
|
|
getTokenBefore(BinaryExpression, {
|
|
skip: 1,
|
|
filter: (t) => t.value !== "=",
|
|
})!.value,
|
|
).toBe("var");
|
|
});
|
|
|
|
it("should retrieve one token or comment before a node with includeComments option", () => {
|
|
expect(
|
|
getTokenBefore(BinaryExpression, {
|
|
includeComments: true,
|
|
})!.value,
|
|
).toBe("C");
|
|
});
|
|
|
|
it("should retrieve one token or comment before a node with includeComments and skip options", () => {
|
|
expect(
|
|
getTokenBefore(BinaryExpression, {
|
|
includeComments: true,
|
|
skip: 1,
|
|
})!.value,
|
|
).toBe("=");
|
|
});
|
|
|
|
it("should retrieve one token or comment before a node with includeComments and skip and filter options", () => {
|
|
expect(
|
|
getTokenBefore(BinaryExpression, {
|
|
includeComments: true,
|
|
skip: 1,
|
|
filter: (t) => t.type.startsWith("Block"),
|
|
})!.value,
|
|
).toBe("B");
|
|
});
|
|
|
|
it("should retrieve the previous node if the comment at the end of source code is specified.", () => {
|
|
setup("a + b /*comment*/");
|
|
// TODO: this verbatim range should be replaced with `ast.comments[0]`
|
|
const token = getTokenBefore({ range: [6, 17] } as Node);
|
|
expect(token!.value).toBe("b");
|
|
});
|
|
|
|
it("should retrieve the previous comment if the first token is specified.", () => {
|
|
setup("/*comment*/ a + b");
|
|
// TODO: this verbatim range should be replaced with `ast.tokens[0]`
|
|
const token = getTokenBefore({ range: [12, 13] } as Node, { includeComments: true });
|
|
expect(token!.value).toBe("comment");
|
|
});
|
|
|
|
it("should retrieve null if the first comment is specified.", () => {
|
|
setup("/*comment*/ a + b");
|
|
// TODO: this verbatim range should be replaced with `ast.comments[0]`
|
|
const token = getTokenBefore({ range: [0, 11] } as Node, { includeComments: true });
|
|
expect(token).toBeNull();
|
|
});
|
|
});
|
|
|
|
// https://github.com/eslint/eslint/blob/v9.39.1/tests/lib/languages/js/source-code/token-store.js#L363-L459
|
|
describe("when calling getTokenAfter", () => {
|
|
it("should retrieve one token after a node", () => {
|
|
expect(getTokenAfter(VariableDeclaratorIdentifier)!.value).toBe("=");
|
|
});
|
|
|
|
it("should skip a given number of tokens", () => {
|
|
expect(getTokenAfter(VariableDeclaratorIdentifier, 1)!.value).toBe("a");
|
|
expect(getTokenAfter(VariableDeclaratorIdentifier, 2)!.value).toBe("*");
|
|
});
|
|
|
|
it("should skip a given number of tokens with skip option", () => {
|
|
expect(getTokenAfter(VariableDeclaratorIdentifier, { skip: 1 })!.value).toBe("a");
|
|
expect(getTokenAfter(VariableDeclaratorIdentifier, { skip: 2 })!.value).toBe("*");
|
|
});
|
|
|
|
it("should retrieve matched token with filter option", () => {
|
|
expect(getTokenAfter(VariableDeclaratorIdentifier, (t) => t.type === "Identifier")!.value).toBe(
|
|
"a",
|
|
);
|
|
expect(
|
|
getTokenAfter(VariableDeclaratorIdentifier, {
|
|
filter: (t) => t.type === "Identifier",
|
|
})!.value,
|
|
).toBe("a");
|
|
});
|
|
|
|
it("should retrieve matched token with filter and skip options", () => {
|
|
expect(
|
|
getTokenAfter(VariableDeclaratorIdentifier, {
|
|
skip: 1,
|
|
filter: (t) => t.type === "Identifier",
|
|
})!.value,
|
|
).toBe("b");
|
|
});
|
|
|
|
it("should retrieve one token or comment after a node with includeComments option", () => {
|
|
expect(
|
|
getTokenAfter(VariableDeclaratorIdentifier, {
|
|
includeComments: true,
|
|
})!.value,
|
|
).toBe("B");
|
|
});
|
|
|
|
it("should retrieve one token or comment after a node with includeComments and skip options", () => {
|
|
expect(
|
|
getTokenAfter(VariableDeclaratorIdentifier, {
|
|
includeComments: true,
|
|
skip: 2,
|
|
})!.value,
|
|
).toBe("C");
|
|
});
|
|
|
|
it("should retrieve one token or comment after a node with includeComments and skip and filter options", () => {
|
|
expect(
|
|
getTokenAfter(VariableDeclaratorIdentifier, {
|
|
includeComments: true,
|
|
skip: 2,
|
|
filter: (t) => t.type.startsWith("Block"),
|
|
})!.value,
|
|
).toBe("D");
|
|
});
|
|
|
|
it("should retrieve the next node if the comment at the first of source code is specified.", () => {
|
|
setup("/*comment*/ a + b");
|
|
// TODO: replace this verbatim range with `ast.comments[0]`
|
|
const token = getTokenAfter({ range: [0, 12] } as Node)!;
|
|
expect(token.value).toBe("a");
|
|
});
|
|
|
|
it("should retrieve the next comment if the last token is specified.", () => {
|
|
setup("a + b /*comment*/");
|
|
// TODO: replace this verbatim range with `ast.tokens[2]`
|
|
const token = getTokenAfter({ range: [4, 5] } as Node, {
|
|
includeComments: true,
|
|
});
|
|
expect(token!.value).toBe("comment");
|
|
});
|
|
|
|
it("should retrieve null if the last comment is specified.", () => {
|
|
setup("a + b /*comment*/");
|
|
// TODO: replace this verbatim range with `ast.comments[0]`
|
|
const token = getTokenAfter({ range: [6, 17] } as Node, {
|
|
includeComments: true,
|
|
});
|
|
expect(token).toBeNull();
|
|
});
|
|
});
|
|
|
|
// https://github.com/eslint/eslint/blob/v9.39.1/tests/lib/languages/js/source-code/token-store.js#L461-L592
|
|
describe("when calling getTokensAfter", () => {
|
|
it("should retrieve zero tokens after a node", () => {
|
|
expect(getTokensAfter(VariableDeclaratorIdentifier, 0).map((token) => token.value)).toEqual([]);
|
|
});
|
|
|
|
it("should retrieve one token after a node", () => {
|
|
expect(getTokensAfter(VariableDeclaratorIdentifier, 1).map((token) => token.value)).toEqual([
|
|
"=",
|
|
]);
|
|
});
|
|
|
|
it("should retrieve more than one token after a node", () => {
|
|
expect(getTokensAfter(VariableDeclaratorIdentifier, 2).map((token) => token.value)).toEqual([
|
|
"=",
|
|
"a",
|
|
]);
|
|
});
|
|
|
|
it("should retrieve all tokens after a node", () => {
|
|
expect(getTokensAfter(VariableDeclaratorIdentifier, 9e9).map((token) => token.value)).toEqual([
|
|
"=",
|
|
"a",
|
|
"*",
|
|
"b",
|
|
"call",
|
|
"(",
|
|
")",
|
|
";",
|
|
]);
|
|
});
|
|
|
|
it("should retrieve more than one token after a node with count option", () => {
|
|
expect(
|
|
getTokensAfter(VariableDeclaratorIdentifier, { count: 2 }).map((token) => token.value),
|
|
).toEqual(["=", "a"]);
|
|
});
|
|
|
|
it("should retrieve all matched tokens after a node with filter option", () => {
|
|
expect(
|
|
getTokensAfter(VariableDeclaratorIdentifier, {
|
|
filter: (t) => t.type === "Identifier",
|
|
}).map((token) => token.value),
|
|
).toEqual(["a", "b", "call"]);
|
|
});
|
|
|
|
it("should retrieve matched tokens after a node with count and filter options", () => {
|
|
expect(
|
|
getTokensAfter(VariableDeclaratorIdentifier, {
|
|
count: 2,
|
|
filter: (t) => t.type === "Identifier",
|
|
}).map((token) => token.value),
|
|
).toEqual(["a", "b"]);
|
|
});
|
|
|
|
it("should retrieve zero tokens after a node with a filter when count is 0", () => {
|
|
expect(
|
|
getTokensAfter(VariableDeclaratorIdentifier, {
|
|
count: 0,
|
|
filter: () => true,
|
|
}).map((token) => token.value),
|
|
).toEqual([]);
|
|
});
|
|
|
|
it("should retrieve all tokens and comments after a node with includeComments option", () => {
|
|
expect(
|
|
getTokensAfter(VariableDeclaratorIdentifier, {
|
|
includeComments: true,
|
|
}).map((token) => token.value),
|
|
).toEqual(["B", "=", "C", "a", "D", "*", "b", "E", "F", "call", "(", ")", ";", "Z"]);
|
|
});
|
|
|
|
it("should retrieve several tokens and comments after a node with includeComments and count options", () => {
|
|
expect(
|
|
getTokensAfter(VariableDeclaratorIdentifier, {
|
|
includeComments: true,
|
|
count: 3,
|
|
}).map((token) => token.value),
|
|
).toEqual(["B", "=", "C"]);
|
|
});
|
|
|
|
it("should retrieve matched tokens and comments after a node with includeComments and count and filter options", () => {
|
|
expect(
|
|
getTokensAfter(VariableDeclaratorIdentifier, {
|
|
includeComments: true,
|
|
count: 3,
|
|
filter: (t) => t.type.startsWith("Block"),
|
|
}).map((token) => token.value),
|
|
).toEqual(["B", "C", "D"]);
|
|
});
|
|
});
|
|
|
|
// https://github.com/eslint/eslint/blob/v9.39.1/tests/lib/languages/js/source-code/token-store.js#L594-L673
|
|
describe("when calling getFirstTokens", () => {
|
|
it("should retrieve zero tokens from a node's token stream", () => {
|
|
expect(getFirstTokens(BinaryExpression, 0).map((token) => token.value)).toEqual([]);
|
|
});
|
|
|
|
it("should retrieve one token from a node's token stream", () => {
|
|
expect(getFirstTokens(BinaryExpression, 1).map((token) => token.value)).toEqual(["a"]);
|
|
});
|
|
|
|
it("should retrieve more than one token from a node's token stream", () => {
|
|
expect(getFirstTokens(BinaryExpression, 2).map((token) => token.value)).toEqual(["a", "*"]);
|
|
});
|
|
|
|
it("should retrieve all tokens from a node's token stream", () => {
|
|
expect(getFirstTokens(BinaryExpression, 9e9).map((token) => token.value)).toEqual([
|
|
"a",
|
|
"*",
|
|
"b",
|
|
]);
|
|
});
|
|
|
|
it("should retrieve more than one token from a node's token stream with count option", () => {
|
|
expect(getFirstTokens(BinaryExpression, { count: 2 }).map((token) => token.value)).toEqual([
|
|
"a",
|
|
"*",
|
|
]);
|
|
});
|
|
|
|
it("should retrieve matched tokens from a node's token stream with filter option", () => {
|
|
expect(
|
|
getFirstTokens(BinaryExpression, (t) => t.type === "Identifier").map((token) => token.value),
|
|
).toEqual(["a", "b"]);
|
|
expect(
|
|
getFirstTokens(BinaryExpression, {
|
|
filter: (t) => t.type === "Identifier",
|
|
}).map((token) => token.value),
|
|
).toEqual(["a", "b"]);
|
|
});
|
|
|
|
it("should retrieve matched tokens from a node's token stream with filter and count options", () => {
|
|
expect(
|
|
getFirstTokens(BinaryExpression, {
|
|
count: 1,
|
|
filter: (t) => t.type === "Identifier",
|
|
}).map((token) => token.value),
|
|
).toEqual(["a"]);
|
|
});
|
|
|
|
it("should retrieve all tokens and comments from a node's token stream with includeComments option", () => {
|
|
expect(
|
|
getFirstTokens(BinaryExpression, {
|
|
includeComments: true,
|
|
}).map((token) => token.value),
|
|
).toEqual(["a", "D", "*", "b"]);
|
|
});
|
|
|
|
it("should retrieve zero tokens from a node's token stream with a filter when count is 0", () => {
|
|
expect(
|
|
getFirstTokens(BinaryExpression, {
|
|
count: 0,
|
|
filter: () => true,
|
|
}).map((token) => token.value),
|
|
).toEqual([]);
|
|
});
|
|
|
|
it("should retrieve several tokens and comments from a node's token stream with includeComments and count options", () => {
|
|
expect(
|
|
getFirstTokens(BinaryExpression, {
|
|
includeComments: true,
|
|
count: 3,
|
|
}).map((token) => token.value),
|
|
).toEqual(["a", "D", "*"]);
|
|
});
|
|
|
|
it("should retrieve several tokens and comments from a node's token stream with includeComments and count and filter options", () => {
|
|
expect(
|
|
getFirstTokens(BinaryExpression, {
|
|
includeComments: true,
|
|
count: 3,
|
|
filter: (t) => t.value !== "a",
|
|
}).map((token) => token.value),
|
|
).toEqual(["D", "*", "b"]);
|
|
});
|
|
});
|
|
|
|
// https://github.com/eslint/eslint/blob/v9.39.1/tests/lib/languages/js/source-code/token-store.js#L675-L849
|
|
describe("when calling getFirstToken", () => {
|
|
it("should retrieve the first token of a node's token stream", () => {
|
|
expect(getFirstToken(BinaryExpression)!.value).toBe("a");
|
|
});
|
|
|
|
it("should skip a given number of tokens", () => {
|
|
expect(getFirstToken(BinaryExpression, 1)!.value).toBe("*");
|
|
expect(getFirstToken(BinaryExpression, 2)!.value).toBe("b");
|
|
});
|
|
|
|
it("should skip a given number of tokens with skip option", () => {
|
|
expect(getFirstToken(BinaryExpression, { skip: 1 })!.value).toBe("*");
|
|
expect(getFirstToken(BinaryExpression, { skip: 2 })!.value).toBe("b");
|
|
});
|
|
|
|
it("should retrieve matched token with filter option", () => {
|
|
expect(getFirstToken(BinaryExpression, (t) => t.type === "Identifier")!.value).toBe("a");
|
|
expect(
|
|
getFirstToken(BinaryExpression, {
|
|
filter: (t) => t.type === "Identifier",
|
|
})!.value,
|
|
).toBe("a");
|
|
});
|
|
|
|
it("should retrieve matched token with filter and skip options", () => {
|
|
expect(
|
|
getFirstToken(BinaryExpression, {
|
|
skip: 1,
|
|
filter: (t) => t.type === "Identifier",
|
|
})!.value,
|
|
).toBe("b");
|
|
});
|
|
|
|
it("should retrieve the first token or comment of a node's token stream with includeComments option", () => {
|
|
expect(getFirstToken(BinaryExpression, { includeComments: true })!.value).toBe("a");
|
|
});
|
|
|
|
it("should retrieve the first matched token or comment of a node's token stream with includeComments and skip options", () => {
|
|
expect(
|
|
getFirstToken(BinaryExpression, {
|
|
includeComments: true,
|
|
skip: 1,
|
|
})!.value,
|
|
).toBe("D");
|
|
});
|
|
|
|
it("should retrieve the first matched token or comment of a node's token stream with includeComments and skip and filter options", () => {
|
|
expect(
|
|
getFirstToken(BinaryExpression, {
|
|
includeComments: true,
|
|
skip: 1,
|
|
filter: (t) => t.value !== "a",
|
|
})!.value,
|
|
).toBe("*");
|
|
});
|
|
|
|
it("should retrieve the first comment if the comment is at the last of nodes", () => {
|
|
setup("a + b\n/*comment*/ c + d");
|
|
/*
|
|
* A node must not start with a token: it can start with a comment or be empty.
|
|
* This test case is needed for completeness.
|
|
*/
|
|
expect(
|
|
getFirstToken(
|
|
// TODO: this verbatim range should be replaced with `[ast.comments[0].range[0], ast.tokens[5].range[1]]`
|
|
{ range: [6, 23] } as Node,
|
|
{ includeComments: true },
|
|
)!.value,
|
|
).toBe("comment");
|
|
});
|
|
|
|
it("should retrieve the first token (without includeComments option) if the comment is at the last of nodes", () => {
|
|
setup("a + b\n/*comment*/ c + d");
|
|
/*
|
|
* A node must not start with a token: it can start with a comment or be empty.
|
|
* This test case is needed for completeness.
|
|
*/
|
|
expect(
|
|
getFirstToken({
|
|
// TODO: this verbatim range should be replaced with `[ast.comments[0].range[0], ast.tokens[5].range[1]]`
|
|
range: [6, 23],
|
|
} as Node)!.value,
|
|
).toBe("c");
|
|
});
|
|
|
|
it("should retrieve the first token if the root node contains a trailing comment", () => {
|
|
setup("foo // comment");
|
|
// TODO: this verbatim range should be replaced with `ast`
|
|
expect(getFirstToken({ range: [0, 14] } as Node)!.value).toBe("foo");
|
|
});
|
|
|
|
it("should return null if the source contains only comments", () => {
|
|
setup("// comment");
|
|
// TODO: this verbatim range should be replaced with `ast`
|
|
expect(
|
|
getFirstToken({ range: [0, 11] } as Node, {
|
|
filter() {
|
|
expect.fail("Unexpected call to filter callback");
|
|
},
|
|
}),
|
|
).toBeNull();
|
|
});
|
|
|
|
it("should return null if the source is empty", () => {
|
|
setup("");
|
|
// TODO: this verbatim range should be replaced with `ast`
|
|
expect(getFirstToken({ range: [0, 0] } as Node)).toBeNull();
|
|
});
|
|
});
|
|
|
|
// https://github.com/eslint/eslint/blob/v9.39.1/tests/lib/languages/js/source-code/token-store.js#L851-L930
|
|
describe("when calling getLastTokens", () => {
|
|
it("should retrieve zero tokens from the end of a node's token stream", () => {
|
|
expect(getLastTokens(BinaryExpression, 0).map((token) => token.value)).toEqual([]);
|
|
});
|
|
|
|
it("should retrieve one token from the end of a node's token stream", () => {
|
|
expect(getLastTokens(BinaryExpression, 1).map((token) => token.value)).toEqual(["b"]);
|
|
});
|
|
|
|
it("should retrieve more than one token from the end of a node's token stream", () => {
|
|
expect(getLastTokens(BinaryExpression, 2).map((token) => token.value)).toEqual(["*", "b"]);
|
|
});
|
|
|
|
it("should retrieve all tokens from the end of a node's token stream", () => {
|
|
expect(getLastTokens(BinaryExpression, 9e9).map((token) => token.value)).toEqual([
|
|
"a",
|
|
"*",
|
|
"b",
|
|
]);
|
|
});
|
|
|
|
it("should retrieve more than one token from the end of a node's token stream with count option", () => {
|
|
expect(getLastTokens(BinaryExpression, { count: 2 }).map((token) => token.value)).toEqual([
|
|
"*",
|
|
"b",
|
|
]);
|
|
});
|
|
|
|
it("should retrieve matched tokens from the end of a node's token stream with filter option", () => {
|
|
expect(
|
|
getLastTokens(BinaryExpression, (t) => t.type === "Identifier").map((token) => token.value),
|
|
).toEqual(["a", "b"]);
|
|
expect(
|
|
getLastTokens(BinaryExpression, {
|
|
filter: (t) => t.type === "Identifier",
|
|
}).map((token) => token.value),
|
|
).toEqual(["a", "b"]);
|
|
});
|
|
|
|
it("should retrieve matched tokens from the end of a node's token stream with filter and count options", () => {
|
|
expect(
|
|
getLastTokens(BinaryExpression, {
|
|
count: 1,
|
|
filter: (t) => t.type === "Identifier",
|
|
}).map((token) => token.value),
|
|
).toEqual(["b"]);
|
|
});
|
|
|
|
it("should retrieve zero tokens from the end of a node's token stream with a filter when count is 0", () => {
|
|
expect(
|
|
getLastTokens(BinaryExpression, {
|
|
count: 0,
|
|
filter: () => true,
|
|
}).map((token) => token.value),
|
|
).toEqual([]);
|
|
});
|
|
|
|
it("should retrieve all tokens from the end of a node's token stream with includeComments option", () => {
|
|
expect(
|
|
getLastTokens(BinaryExpression, {
|
|
includeComments: true,
|
|
}).map((token) => token.value),
|
|
).toEqual(["a", "D", "*", "b"]);
|
|
});
|
|
|
|
it("should retrieve matched tokens from the end of a node's token stream with includeComments and count options", () => {
|
|
expect(
|
|
getLastTokens(BinaryExpression, {
|
|
includeComments: true,
|
|
count: 3,
|
|
}).map((token) => token.value),
|
|
).toEqual(["D", "*", "b"]);
|
|
});
|
|
|
|
it("should retrieve matched tokens from the end of a node's token stream with includeComments and count and filter options", () => {
|
|
expect(
|
|
getLastTokens(BinaryExpression, {
|
|
includeComments: true,
|
|
count: 3,
|
|
filter: (t) => t.type !== "Punctuator",
|
|
}).map((token) => token.value),
|
|
).toEqual(["a", "D", "b"]);
|
|
});
|
|
});
|
|
|
|
// https://github.com/eslint/eslint/blob/v9.39.1/tests/lib/languages/js/source-code/token-store.js#L932-L1105
|
|
describe("when calling getLastToken", () => {
|
|
it("should retrieve the last token of a node's token stream", () => {
|
|
expect(getLastToken(BinaryExpression)!.value).toBe("b");
|
|
expect(getLastToken(VariableDeclaration)!.value).toBe("b");
|
|
});
|
|
|
|
it("should skip a given number of tokens", () => {
|
|
expect(getLastToken(BinaryExpression, 1)!.value).toBe("*");
|
|
expect(getLastToken(BinaryExpression, 2)!.value).toBe("a");
|
|
});
|
|
|
|
it("should skip a given number of tokens with skip option", () => {
|
|
expect(getLastToken(BinaryExpression, { skip: 1 })!.value).toBe("*");
|
|
expect(getLastToken(BinaryExpression, { skip: 2 })!.value).toBe("a");
|
|
});
|
|
|
|
it("should retrieve the last matched token of a node's token stream with filter option", () => {
|
|
expect(getLastToken(BinaryExpression, (t) => t.value !== "b")!.value).toBe("*");
|
|
expect(
|
|
getLastToken(BinaryExpression, {
|
|
filter: (t) => t.value !== "b",
|
|
})!.value,
|
|
).toBe("*");
|
|
});
|
|
|
|
it("should retrieve the last matched token of a node's token stream with filter and skip options", () => {
|
|
expect(
|
|
getLastToken(BinaryExpression, {
|
|
skip: 1,
|
|
filter: (t) => t.type === "Identifier",
|
|
})!.value,
|
|
).toBe("a");
|
|
});
|
|
|
|
it("should retrieve the last token of a node's token stream with includeComments option", () => {
|
|
expect(getLastToken(BinaryExpression, { includeComments: true })!.value).toBe("b");
|
|
});
|
|
|
|
it("should retrieve the last token of a node's token stream with includeComments and skip options", () => {
|
|
expect(
|
|
getLastToken(BinaryExpression, {
|
|
includeComments: true,
|
|
skip: 2,
|
|
})!.value,
|
|
).toBe("D");
|
|
});
|
|
|
|
it("should retrieve the last token of a node's token stream with includeComments and skip and filter options", () => {
|
|
expect(
|
|
getLastToken(BinaryExpression, {
|
|
includeComments: true,
|
|
skip: 1,
|
|
filter: (t) => t.type !== "Identifier",
|
|
})!.value,
|
|
).toBe("D");
|
|
});
|
|
|
|
it("should retrieve the last comment if the comment is at the last of nodes", () => {
|
|
setup("a + b /*comment*/\nc + d");
|
|
// A node must not end with a token: it can end with a comment or be empty.
|
|
// This test case is needed for completeness.
|
|
expect(
|
|
getLastToken(
|
|
// TODO: this verbatim range should be replaced with `ast.tokens[0].range[0], ast.comments[0].range[1]`
|
|
{ range: [0, 17] } as Node,
|
|
{ includeComments: true },
|
|
)!.value,
|
|
).toBe("comment");
|
|
});
|
|
|
|
it("should retrieve the last token (without includeComments option) if the comment is at the last of nodes", () => {
|
|
setup("a + b /*comment*/\nc + d");
|
|
// A node must not end with a token: it can end with a comment or be empty.
|
|
// This test case is needed for completeness.
|
|
expect(
|
|
getLastToken(
|
|
// TODO: this verbatim range should be replaced with `ast.tokens[0].range[0], ast.comments[0].range[1]`
|
|
{ range: [0, 17] } as Node,
|
|
)!.value,
|
|
).toBe("b");
|
|
});
|
|
|
|
it("should retrieve the last token if the root node contains a trailing comment", () => {
|
|
setup("foo // comment");
|
|
// TODO: this verbatim range should be replaced with `ast`
|
|
expect(getLastToken({ range: [0, 3] } as Node)!.value).toBe("foo");
|
|
});
|
|
|
|
it("should return null if the source contains only comments", () => {
|
|
setup("// comment");
|
|
expect(
|
|
// TODO: this verbatim range should be replaced with `ast`
|
|
getLastToken({ range: [0, 10] } as Node, {
|
|
filter() {
|
|
expect.fail("Unexpected call to filter callback");
|
|
},
|
|
}),
|
|
).toBeNull();
|
|
});
|
|
|
|
it("should return null if the source is empty", () => {
|
|
setup("");
|
|
// TODO: this verbatim range should be replaced with `ast`
|
|
expect(getLastToken({ range: [0, 0] } as Node)).toBeNull();
|
|
});
|
|
});
|
|
|
|
// https://github.com/eslint/eslint/blob/v9.39.1/tests/lib/languages/js/source-code/token-store.js#L1107-L1191
|
|
describe("when calling getFirstTokensBetween", () => {
|
|
it("should retrieve zero tokens between adjacent nodes", () => {
|
|
expect(
|
|
getFirstTokensBetween(BinaryExpression, CallExpression).map((token) => token.value),
|
|
).toEqual([]);
|
|
});
|
|
|
|
it("should retrieve multiple tokens between non-adjacent nodes with count option", () => {
|
|
expect(
|
|
getFirstTokensBetween(VariableDeclaratorIdentifier, BinaryExpression.right, 2).map(
|
|
(token) => token.value,
|
|
),
|
|
).toEqual(["=", "a"]);
|
|
expect(
|
|
getFirstTokensBetween(VariableDeclaratorIdentifier, BinaryExpression.right, { count: 2 }).map(
|
|
(token) => token.value,
|
|
),
|
|
).toEqual(["=", "a"]);
|
|
});
|
|
|
|
it("should retrieve matched tokens between non-adjacent nodes with filter option", () => {
|
|
expect(
|
|
getFirstTokensBetween(VariableDeclaratorIdentifier, BinaryExpression.right, {
|
|
filter: (t) => t.type !== "Punctuator",
|
|
}).map((token) => token.value),
|
|
).toEqual(["a"]);
|
|
});
|
|
|
|
it("should retrieve all tokens between non-adjacent nodes with empty object option", () => {
|
|
expect(
|
|
getFirstTokensBetween(VariableDeclaratorIdentifier, BinaryExpression.right, {}).map(
|
|
(token) => token.value,
|
|
),
|
|
).toEqual(["=", "a", "*"]);
|
|
});
|
|
|
|
it("should retrieve multiple tokens between non-adjacent nodes with includeComments option", () => {
|
|
expect(
|
|
getFirstTokensBetween(VariableDeclaratorIdentifier, BinaryExpression.right, {
|
|
includeComments: true,
|
|
}).map((token) => token.value),
|
|
).toEqual(["B", "=", "C", "a", "D", "*"]);
|
|
});
|
|
|
|
it("should retrieve multiple tokens between non-adjacent nodes with includeComments and count options", () => {
|
|
expect(
|
|
getFirstTokensBetween(VariableDeclaratorIdentifier, BinaryExpression.right, {
|
|
includeComments: true,
|
|
count: 3,
|
|
}).map((token) => token.value),
|
|
).toEqual(["B", "=", "C"]);
|
|
});
|
|
|
|
it("should retrieve multiple tokens and comments between non-adjacent nodes with includeComments and filter options", () => {
|
|
expect(
|
|
getFirstTokensBetween(VariableDeclaratorIdentifier, BinaryExpression.right, {
|
|
includeComments: true,
|
|
filter: (t) => t.type !== "Punctuator",
|
|
}).map((token) => token.value),
|
|
).toEqual(["B", "C", "a", "D"]);
|
|
});
|
|
});
|
|
|
|
// https://github.com/eslint/eslint/blob/v9.39.1/tests/lib/languages/js/source-code/token-store.js#L1193-L1296
|
|
describe("when calling getFirstTokenBetween", () => {
|
|
it("should return null between adjacent nodes", () => {
|
|
expect(getFirstTokenBetween(BinaryExpression, CallExpression)).toBeNull();
|
|
});
|
|
|
|
it("should retrieve one token between non-adjacent nodes", () => {
|
|
expect(
|
|
getFirstTokenBetween(VariableDeclaratorIdentifier, BinaryExpression.right)!.value,
|
|
).toEqual("=");
|
|
});
|
|
|
|
it("should retrieve one token between non-adjacent nodes with skip option", () => {
|
|
expect(
|
|
getFirstTokenBetween(VariableDeclaratorIdentifier, BinaryExpression.right, 1)!.value,
|
|
).toEqual("a");
|
|
expect(
|
|
getFirstTokenBetween(VariableDeclaratorIdentifier, BinaryExpression.right, { skip: 2 })!
|
|
.value,
|
|
).toEqual("*");
|
|
});
|
|
|
|
it("should return null if it's skipped beyond the right token", () => {
|
|
expect(
|
|
getFirstTokenBetween(VariableDeclaratorIdentifier, BinaryExpression.right, { skip: 3 }),
|
|
).toBeNull();
|
|
expect(
|
|
getFirstTokenBetween(VariableDeclaratorIdentifier, BinaryExpression.right, { skip: 4 }),
|
|
).toBeNull();
|
|
});
|
|
|
|
it("should retrieve the first matched token between non-adjacent nodes with filter option", () => {
|
|
expect(
|
|
getFirstTokenBetween(VariableDeclaratorIdentifier, BinaryExpression.right, {
|
|
filter: (t) => t.type !== "Identifier",
|
|
})!.value,
|
|
).toEqual("=");
|
|
});
|
|
|
|
it("should retrieve first token or comment between non-adjacent nodes with includeComments option", () => {
|
|
expect(
|
|
getFirstTokenBetween(VariableDeclaratorIdentifier, BinaryExpression.right, {
|
|
includeComments: true,
|
|
})!.value,
|
|
).toEqual("B");
|
|
});
|
|
|
|
it("should retrieve first token or comment between non-adjacent nodes with includeComments and skip options", () => {
|
|
expect(
|
|
getFirstTokenBetween(VariableDeclaratorIdentifier, BinaryExpression.right, {
|
|
includeComments: true,
|
|
skip: 1,
|
|
})!.value,
|
|
).toEqual("=");
|
|
});
|
|
|
|
it("should retrieve first token or comment between non-adjacent nodes with includeComments and skip and filter options", () => {
|
|
expect(
|
|
getFirstTokenBetween(VariableDeclaratorIdentifier, BinaryExpression.right, {
|
|
includeComments: true,
|
|
skip: 1,
|
|
filter: (t) => t.type !== "Punctuator",
|
|
})!.value,
|
|
).toEqual("C");
|
|
});
|
|
});
|
|
|
|
// https://github.com/eslint/eslint/blob/v9.39.1/tests/lib/languages/js/source-code/token-store.js#L1298-L1382
|
|
describe("when calling getLastTokensBetween", () => {
|
|
it("should retrieve zero tokens between adjacent nodes", () => {
|
|
expect(
|
|
getLastTokensBetween(BinaryExpression, CallExpression).map((token) => token.value),
|
|
).toEqual([]);
|
|
});
|
|
|
|
it("should retrieve multiple tokens between non-adjacent nodes with count option", () => {
|
|
expect(
|
|
getLastTokensBetween(VariableDeclaratorIdentifier, BinaryExpression.right, 2).map(
|
|
(token) => token.value,
|
|
),
|
|
).toEqual(["a", "*"]);
|
|
expect(
|
|
getLastTokensBetween(VariableDeclaratorIdentifier, BinaryExpression.right, { count: 2 }).map(
|
|
(token) => token.value,
|
|
),
|
|
).toEqual(["a", "*"]);
|
|
});
|
|
|
|
it("should retrieve matched tokens between non-adjacent nodes with filter option", () => {
|
|
expect(
|
|
getLastTokensBetween(VariableDeclaratorIdentifier, BinaryExpression.right, {
|
|
filter: (t) => t.type !== "Punctuator",
|
|
}).map((token) => token.value),
|
|
).toEqual(["a"]);
|
|
});
|
|
|
|
it("should retrieve all tokens between non-adjacent nodes with empty object option", () => {
|
|
expect(
|
|
getLastTokensBetween(VariableDeclaratorIdentifier, BinaryExpression.right, {}).map(
|
|
(token) => token.value,
|
|
),
|
|
).toEqual(["=", "a", "*"]);
|
|
});
|
|
|
|
it("should retrieve all tokens and comments between non-adjacent nodes with includeComments option", () => {
|
|
expect(
|
|
getLastTokensBetween(VariableDeclaratorIdentifier, BinaryExpression.right, {
|
|
includeComments: true,
|
|
}).map((token) => token.value),
|
|
).toEqual(["B", "=", "C", "a", "D", "*"]);
|
|
});
|
|
|
|
it("should retrieve multiple tokens between non-adjacent nodes with includeComments and count options", () => {
|
|
expect(
|
|
getLastTokensBetween(VariableDeclaratorIdentifier, BinaryExpression.right, {
|
|
includeComments: true,
|
|
count: 3,
|
|
}).map((token) => token.value),
|
|
).toEqual(["a", "D", "*"]);
|
|
});
|
|
|
|
it("should retrieve multiple tokens and comments between non-adjacent nodes with includeComments and filter options", () => {
|
|
expect(
|
|
getLastTokensBetween(VariableDeclaratorIdentifier, BinaryExpression.right, {
|
|
includeComments: true,
|
|
filter: (t) => t.type !== "Punctuator",
|
|
}).map((token) => token.value),
|
|
).toEqual(["B", "C", "a", "D"]);
|
|
});
|
|
});
|
|
|
|
// https://github.com/eslint/eslint/blob/v9.39.1/tests/lib/languages/js/source-code/token-store.js#L1384-L1487
|
|
describe("when calling getLastTokenBetween", () => {
|
|
it("should return null between adjacent nodes", () => {
|
|
expect(getLastTokenBetween(BinaryExpression, CallExpression)).toBeNull();
|
|
});
|
|
|
|
it("should retrieve the last token between non-adjacent nodes with count option", () => {
|
|
expect(getLastTokenBetween(VariableDeclaratorIdentifier, BinaryExpression.right)!.value).toBe(
|
|
"*",
|
|
);
|
|
});
|
|
|
|
it("should retrieve one token between non-adjacent nodes with skip option", () => {
|
|
expect(
|
|
getLastTokenBetween(VariableDeclaratorIdentifier, BinaryExpression.right, 1)!.value,
|
|
).toBe("a");
|
|
expect(
|
|
getLastTokenBetween(VariableDeclaratorIdentifier, BinaryExpression.right, { skip: 2 })!.value,
|
|
).toBe("=");
|
|
});
|
|
|
|
it("should return null if it's skipped beyond the right token", () => {
|
|
expect(
|
|
getLastTokenBetween(VariableDeclaratorIdentifier, BinaryExpression.right, { skip: 3 }),
|
|
).toBeNull();
|
|
expect(
|
|
getLastTokenBetween(VariableDeclaratorIdentifier, BinaryExpression.right, { skip: 4 }),
|
|
).toBeNull();
|
|
});
|
|
|
|
it("should retrieve the last matched token between non-adjacent nodes with filter option", () => {
|
|
expect(
|
|
getLastTokenBetween(
|
|
VariableDeclaratorIdentifier,
|
|
BinaryExpression.right,
|
|
(t) => t.type !== "Identifier",
|
|
)!.value,
|
|
).toBe("*");
|
|
expect(
|
|
getLastTokenBetween(VariableDeclaratorIdentifier, BinaryExpression.right, {
|
|
filter: (t) => t.type !== "Identifier",
|
|
})!.value,
|
|
).toBe("*");
|
|
});
|
|
|
|
it("should retrieve last token or comment between non-adjacent nodes with includeComments option", () => {
|
|
expect(
|
|
getLastTokenBetween(VariableDeclaratorIdentifier, BinaryExpression.right, {
|
|
includeComments: true,
|
|
})!.value,
|
|
).toBe("*");
|
|
});
|
|
|
|
it("should retrieve last token or comment between non-adjacent nodes with includeComments and skip options", () => {
|
|
expect(
|
|
getLastTokenBetween(VariableDeclaratorIdentifier, BinaryExpression.right, {
|
|
includeComments: true,
|
|
skip: 1,
|
|
})!.value,
|
|
).toBe("D");
|
|
});
|
|
|
|
it("should retrieve last token or comment between non-adjacent nodes with includeComments and skip and filter options", () => {
|
|
expect(
|
|
getLastTokenBetween(VariableDeclaratorIdentifier, BinaryExpression.right, {
|
|
includeComments: true,
|
|
skip: 1,
|
|
filter: (t) => t.type !== "Punctuator",
|
|
})!.value,
|
|
).toBe("a");
|
|
});
|
|
});
|
|
|
|
// https://github.com/eslint/eslint/blob/v9.39.1/tests/lib/languages/js/source-code/token-store.js#L1489-L1524
|
|
describe("when calling getTokensBetween", () => {
|
|
it("should retrieve zero tokens between adjacent nodes", () => {
|
|
expect(getTokensBetween(BinaryExpression, CallExpression).map((token) => token.value)).toEqual(
|
|
[],
|
|
);
|
|
});
|
|
|
|
it("should retrieve one token between nodes", () => {
|
|
expect(
|
|
getTokensBetween(BinaryExpression.left, BinaryExpression.right).map((token) => token.value),
|
|
).toEqual(["*"]);
|
|
});
|
|
|
|
it("should retrieve multiple tokens between non-adjacent nodes", () => {
|
|
expect(
|
|
getTokensBetween(VariableDeclaratorIdentifier, BinaryExpression.right).map(
|
|
(token) => token.value,
|
|
),
|
|
).toEqual(["=", "a", "*"]);
|
|
});
|
|
|
|
it("should retrieve surrounding tokens when asked for padding", () => {
|
|
expect(
|
|
getTokensBetween(VariableDeclaratorIdentifier, BinaryExpression.left, 2).map(
|
|
(token) => token.value,
|
|
),
|
|
).toEqual(["var", "answer", "=", "a", "*"]);
|
|
});
|
|
});
|
|
|
|
// https://github.com/eslint/eslint/blob/v9.39.1/tests/lib/languages/js/source-code/token-store.js#L1526-L1562
|
|
describe("when calling getTokenByRangeStart", () => {
|
|
it("should return identifier token", () => {
|
|
const { type, value } = getTokenByRangeStart(9)!;
|
|
|
|
expect(type).toEqual("Identifier");
|
|
expect(value).toEqual("answer");
|
|
});
|
|
|
|
it("should return null when token doesn't exist", () => {
|
|
expect(getTokenByRangeStart(10)).toBeNull();
|
|
});
|
|
|
|
it("should return a comment token when includeComments is true", () => {
|
|
const { type, value } = getTokenByRangeStart(15, {
|
|
includeComments: true,
|
|
})!;
|
|
|
|
expect(type).toEqual("Block");
|
|
expect(value).toEqual("B");
|
|
});
|
|
|
|
it("should not return a comment token at the supplied index when includeComments is false", () => {
|
|
expect(
|
|
getTokenByRangeStart(15, {
|
|
includeComments: false,
|
|
}),
|
|
).toBeNull();
|
|
});
|
|
|
|
it("should not return comment tokens by default", () => {
|
|
expect(getTokenByRangeStart(15)).toBeNull();
|
|
});
|
|
});
|
|
|
|
// https://github.com/eslint/eslint/blob/v9.39.1/tests/lib/languages/js/source-code/token-store.js#L1564-L1582
|
|
describe("when calling getTokenOrCommentBefore", () => {
|
|
it("should retrieve one token or comment before a node", () => {
|
|
expect(getTokenOrCommentBefore(BinaryExpression)!.value).toBe("C");
|
|
});
|
|
|
|
it("should skip a given number of tokens", () => {
|
|
expect(getTokenOrCommentBefore(BinaryExpression, 1)!.value).toBe("=");
|
|
expect(getTokenOrCommentBefore(BinaryExpression, 2)!.value).toBe("B");
|
|
});
|
|
});
|
|
|
|
// https://github.com/eslint/eslint/blob/v9.39.1/tests/lib/languages/js/source-code/token-store.js#L1584-L1602
|
|
describe("when calling getTokenOrCommentAfter", () => {
|
|
it("should retrieve one token or comment after a node", () => {
|
|
expect(getTokenOrCommentAfter(VariableDeclaratorIdentifier)!.value).toBe("B");
|
|
});
|
|
|
|
it("should skip a given number of tokens", () => {
|
|
expect(getTokenOrCommentAfter(VariableDeclaratorIdentifier, 1)!.value).toBe("=");
|
|
expect(getTokenOrCommentAfter(VariableDeclaratorIdentifier, 2)!.value).toBe("C");
|
|
});
|
|
});
|
|
|
|
// https://github.com/eslint/eslint/blob/v9.39.1/tests/lib/languages/js/source-code/token-store.js#L1604-L1672
|
|
describe("when calling getFirstToken & getTokenAfter", () => {
|
|
it("should retrieve all tokens and comments in the node", () => {
|
|
setup("(function(a, /*b,*/ c){})");
|
|
const tokens = [];
|
|
// TODO: replace this verbatim range with `ast`
|
|
let token: TokenOrComment | null = getFirstToken({ range: [0, 25] } as Node);
|
|
|
|
while (token) {
|
|
tokens.push(token);
|
|
token = getTokenAfter(token, {
|
|
includeComments: true,
|
|
});
|
|
}
|
|
|
|
expect(tokens.map((token) => token.value)).toEqual([
|
|
"(",
|
|
"function",
|
|
"(",
|
|
"a",
|
|
",",
|
|
"b,",
|
|
"c",
|
|
")",
|
|
"{",
|
|
"}",
|
|
")",
|
|
]);
|
|
});
|
|
|
|
it("should retrieve all tokens and comments in the node (no spaces)", () => {
|
|
setup("(function(a,/*b,*/c){})");
|
|
const tokens = [];
|
|
// TODO: replace this verbatim range with `ast`
|
|
let token: TokenOrComment | null = getFirstToken({ range: [0, 23] } as Node);
|
|
|
|
while (token) {
|
|
tokens.push(token);
|
|
token = getTokenAfter(token, {
|
|
includeComments: true,
|
|
});
|
|
}
|
|
|
|
expect(tokens.map((token) => token.value)).toEqual([
|
|
"(",
|
|
"function",
|
|
"(",
|
|
"a",
|
|
",",
|
|
"b,",
|
|
"c",
|
|
")",
|
|
"{",
|
|
"}",
|
|
")",
|
|
]);
|
|
});
|
|
});
|
|
|
|
// https://github.com/eslint/eslint/blob/v9.39.1/tests/lib/languages/js/source-code/token-store.js#L1674-L1742
|
|
describe("when calling getLastToken & getTokenBefore", () => {
|
|
it("should retrieve all tokens and comments in the node", () => {
|
|
setup("(function(a, /*b,*/ c){})");
|
|
const tokens = [];
|
|
// TODO: replace this verbatim range with `ast`
|
|
let token: TokenOrComment | null = getLastToken({ range: [0, 25] } as Node);
|
|
|
|
while (token) {
|
|
tokens.push(token);
|
|
token = getTokenBefore(token, {
|
|
includeComments: true,
|
|
});
|
|
}
|
|
|
|
expect(tokens.reverse().map((token) => token.value)).toEqual([
|
|
"(",
|
|
"function",
|
|
"(",
|
|
"a",
|
|
",",
|
|
"b,",
|
|
"c",
|
|
")",
|
|
"{",
|
|
"}",
|
|
")",
|
|
]);
|
|
});
|
|
|
|
it("should retrieve all tokens and comments in the node (no spaces)", () => {
|
|
setup("(function(a,/*b,*/c){})");
|
|
const tokens = [];
|
|
// TODO: replace this verbatim range with `ast`
|
|
let token: TokenOrComment | null = getLastToken({ range: [0, 23] } as Node);
|
|
|
|
while (token) {
|
|
tokens.push(token);
|
|
token = getTokenBefore(token, {
|
|
includeComments: true,
|
|
});
|
|
}
|
|
|
|
expect(tokens.reverse().map((token) => token.value)).toEqual([
|
|
"(",
|
|
"function",
|
|
"(",
|
|
"a",
|
|
",",
|
|
"b,",
|
|
"c",
|
|
")",
|
|
"{",
|
|
"}",
|
|
")",
|
|
]);
|
|
});
|
|
});
|
|
|
|
// Regression tests for token `loc` correctness across pooled token reuse.
|
|
// Token objects are cached and reused across files. These tests verify that `loc` values
|
|
// are correct on the second file after token objects have been reused, and that stale
|
|
// cached `loc` from the first file does not leak into the second.
|
|
describe("token loc across sequential files", () => {
|
|
// Two source texts with different line structures.
|
|
// File 1 has everything on one line, file 2 spreads tokens across multiple lines.
|
|
// If stale `loc` leaks from file 1, the line/column values for file 2 would be wrong.
|
|
const FILE_1 = "var x = 1;";
|
|
const FILE_2 = "var\n y\n =\n 2;";
|
|
|
|
it("should compute correct `token.loc` on second file after accessing `loc` on first file", () => {
|
|
// File 1: access `loc` on all tokens to cache it
|
|
setup(FILE_1);
|
|
const tokens1 = getTokens({ range: [0, FILE_1.length] } as Node);
|
|
expect(tokens1.map((t) => t.value)).toEqual(["var", "x", "=", "1", ";"]);
|
|
for (const token of tokens1) {
|
|
// Access `loc` to trigger caching
|
|
expect(token.loc.start.line).toBeGreaterThan(0);
|
|
}
|
|
|
|
// File 2: verify `loc` is correct, not stale from file 1
|
|
setup(FILE_2);
|
|
const tokens2 = getTokens({ range: [0, FILE_2.length] } as Node);
|
|
expect(tokens2.map((t) => t.value)).toEqual(["var", "y", "=", "2", ";"]);
|
|
|
|
// `var` is at start of line 1
|
|
expect(tokens2[0].loc).toEqual({ start: { line: 1, column: 0 }, end: { line: 1, column: 3 } });
|
|
// `y` is at line 2, column 2
|
|
expect(tokens2[1].loc).toEqual({ start: { line: 2, column: 2 }, end: { line: 2, column: 3 } });
|
|
// `=` is at line 3, column 2
|
|
expect(tokens2[2].loc).toEqual({ start: { line: 3, column: 2 }, end: { line: 3, column: 3 } });
|
|
// `2` is at line 4, column 2
|
|
expect(tokens2[3].loc).toEqual({ start: { line: 4, column: 2 }, end: { line: 4, column: 3 } });
|
|
// `;` is at line 4, column 3
|
|
expect(tokens2[4].loc).toEqual({ start: { line: 4, column: 3 }, end: { line: 4, column: 4 } });
|
|
});
|
|
|
|
it("should compute correct `getLoc(token)` on second file after accessing `getLoc` on first file", () => {
|
|
// File 1: access `getLoc` on all tokens
|
|
setup(FILE_1);
|
|
const tokens1 = getTokens({ range: [0, FILE_1.length] } as Node);
|
|
for (const token of tokens1) {
|
|
expect(getLoc(token).start.line).toBeGreaterThan(0);
|
|
}
|
|
|
|
// File 2: verify `getLoc` returns correct values
|
|
setup(FILE_2);
|
|
const tokens2 = getTokens({ range: [0, FILE_2.length] } as Node);
|
|
|
|
expect(getLoc(tokens2[0])).toEqual({
|
|
start: { line: 1, column: 0 },
|
|
end: { line: 1, column: 3 },
|
|
});
|
|
expect(getLoc(tokens2[1])).toEqual({
|
|
start: { line: 2, column: 2 },
|
|
end: { line: 2, column: 3 },
|
|
});
|
|
expect(getLoc(tokens2[2])).toEqual({
|
|
start: { line: 3, column: 2 },
|
|
end: { line: 3, column: 3 },
|
|
});
|
|
});
|
|
|
|
it("should return same `loc` object from `token.loc` and `getLoc(token)`", () => {
|
|
setup(FILE_2);
|
|
const tokens2 = getTokens({ range: [0, FILE_2.length] } as Node);
|
|
|
|
for (const token of tokens2) {
|
|
// Both access paths should return the same cached object
|
|
expect(getLoc(token)).toBe(token.loc);
|
|
}
|
|
});
|
|
|
|
it("should compute correct `loc` when second file has fewer tokens than first", () => {
|
|
// File with many tokens, then file with few tokens.
|
|
// Tests the `previousTokens.length >= tokensLen` reuse path in `initTokens`.
|
|
const manyTokens = "a + b + c + d + e + f;";
|
|
const fewTokens = "x\n+\ny;";
|
|
|
|
setup(manyTokens);
|
|
const tokens1 = getTokens({ range: [0, manyTokens.length] } as Node);
|
|
for (const token of tokens1) {
|
|
expect(token.loc.start.line).toBe(1);
|
|
}
|
|
|
|
setup(fewTokens);
|
|
const tokens2 = getTokens({ range: [0, fewTokens.length] } as Node);
|
|
expect(tokens2.map((t) => t.value)).toEqual(["x", "+", "y", ";"]);
|
|
|
|
expect(tokens2[0].loc).toEqual({ start: { line: 1, column: 0 }, end: { line: 1, column: 1 } });
|
|
expect(tokens2[1].loc).toEqual({ start: { line: 2, column: 0 }, end: { line: 2, column: 1 } });
|
|
expect(tokens2[2].loc).toEqual({ start: { line: 3, column: 0 }, end: { line: 3, column: 1 } });
|
|
expect(tokens2[3].loc).toEqual({ start: { line: 3, column: 1 }, end: { line: 3, column: 2 } });
|
|
});
|
|
|
|
it("should compute correct `loc` across three sequential files", () => {
|
|
const file1 = "let a = 1;";
|
|
const file2 = "let\n b\n =\n 2;";
|
|
const file3 = "let c\n= 3;";
|
|
|
|
// File 1
|
|
setup(file1);
|
|
for (const token of getTokens({ range: [0, file1.length] } as Node)) {
|
|
expect(token.loc.start.line).toBe(1);
|
|
}
|
|
|
|
// File 2
|
|
setup(file2);
|
|
for (const token of getTokens({ range: [0, file2.length] } as Node)) {
|
|
expect(token.loc.start.line).toBeGreaterThan(0);
|
|
}
|
|
|
|
// File 3: verify locations are from file 3, not file 1 or 2
|
|
setup(file3);
|
|
const tokens3 = getTokens({ range: [0, file3.length] } as Node);
|
|
expect(tokens3.map((t) => t.value)).toEqual(["let", "c", "=", "3", ";"]);
|
|
|
|
// `let` on line 1
|
|
expect(tokens3[0].loc).toEqual({ start: { line: 1, column: 0 }, end: { line: 1, column: 3 } });
|
|
// `c` on line 1
|
|
expect(tokens3[1].loc).toEqual({ start: { line: 1, column: 4 }, end: { line: 1, column: 5 } });
|
|
// `=` on line 2
|
|
expect(tokens3[2].loc).toEqual({ start: { line: 2, column: 0 }, end: { line: 2, column: 1 } });
|
|
// `3` on line 2
|
|
expect(tokens3[3].loc).toEqual({ start: { line: 2, column: 2 }, end: { line: 2, column: 3 } });
|
|
// `;` on line 2
|
|
expect(tokens3[4].loc).toEqual({ start: { line: 2, column: 3 }, end: { line: 2, column: 4 } });
|
|
});
|
|
});
|
|
|
|
// Regression tests for token `regex` property correctness across pooled token reuse.
|
|
// Token objects are cached and reused across files. These tests verify that `regex` is correctly
|
|
// set for RegularExpression tokens and reset to `undefined` when the token object is reused
|
|
// for a non-regex token in a subsequent file.
|
|
describe("token regex across sequential files", () => {
|
|
it("should have `regex` undefined on second file when first file had regex tokens", () => {
|
|
// File 1: has a regex token
|
|
const withRegex = "var x = /abc/gi;";
|
|
setup(withRegex);
|
|
const tokens1 = getTokens({ range: [0, withRegex.length] } as Node);
|
|
const regexToken1 = tokens1.find((t) => t.type === "RegularExpression");
|
|
expect(regexToken1).toBeDefined();
|
|
expect(regexToken1!.regex).toEqual({ pattern: "abc", flags: "gi" });
|
|
|
|
// File 2: no regex tokens - reused token objects should have `regex: undefined`
|
|
const withoutRegex = "var y = 1;";
|
|
setup(withoutRegex);
|
|
const tokens2 = getTokens({ range: [0, withoutRegex.length] } as Node);
|
|
for (const token of tokens2) {
|
|
expect(token.regex).toBeUndefined();
|
|
}
|
|
});
|
|
|
|
it("should set `regex` correctly on second file when first file had no regex", () => {
|
|
// File 1: no regex
|
|
const withoutRegex = "var x = 1;";
|
|
setup(withoutRegex);
|
|
const tokens1 = getTokens({ range: [0, withoutRegex.length] } as Node);
|
|
for (const token of tokens1) {
|
|
expect(token.regex).toBeUndefined();
|
|
}
|
|
|
|
// File 2: has a regex token
|
|
const withRegex = "var y = /foo/m;";
|
|
setup(withRegex);
|
|
const tokens2 = getTokens({ range: [0, withRegex.length] } as Node);
|
|
const regexToken2 = tokens2.find((t) => t.type === "RegularExpression");
|
|
expect(regexToken2).toBeDefined();
|
|
expect(regexToken2!.regex).toEqual({ pattern: "foo", flags: "m" });
|
|
|
|
// Non-regex tokens should still have `regex: undefined`
|
|
for (const token of tokens2) {
|
|
if (token.type !== "RegularExpression") {
|
|
// oxlint-disable-next-line vitest/no-conditional-expect
|
|
expect(token.regex).toBeUndefined();
|
|
}
|
|
}
|
|
});
|
|
|
|
it("should update `regex` correctly across files that both have regex tokens", () => {
|
|
// File 1: regex with one pattern
|
|
const regex1 = "var x = /abc/g;";
|
|
setup(regex1);
|
|
const tokens1 = getTokens({ range: [0, regex1.length] } as Node);
|
|
const regexToken1 = tokens1.find((t) => t.type === "RegularExpression");
|
|
expect(regexToken1!.regex).toEqual({ pattern: "abc", flags: "g" });
|
|
|
|
// File 2: regex with a different pattern - should not carry over from file 1
|
|
const regex2 = "var y = /xyz/i;";
|
|
setup(regex2);
|
|
const tokens2 = getTokens({ range: [0, regex2.length] } as Node);
|
|
const regexToken2 = tokens2.find((t) => t.type === "RegularExpression");
|
|
expect(regexToken2!.regex).toEqual({ pattern: "xyz", flags: "i" });
|
|
});
|
|
|
|
it("should handle multiple regex tokens followed by a file with no regex", () => {
|
|
// File 1: multiple regex tokens
|
|
const multiRegex = "/a/g; /b/i; /c/m;";
|
|
setup(multiRegex);
|
|
const tokens1 = getTokens({ range: [0, multiRegex.length] } as Node);
|
|
const regexTokens1 = tokens1.filter((t) => t.type === "RegularExpression");
|
|
expect(regexTokens1).toHaveLength(3);
|
|
expect(regexTokens1[0].regex).toEqual({ pattern: "a", flags: "g" });
|
|
expect(regexTokens1[1].regex).toEqual({ pattern: "b", flags: "i" });
|
|
expect(regexTokens1[2].regex).toEqual({ pattern: "c", flags: "m" });
|
|
|
|
// File 2: no regex - all tokens should have `regex: undefined`
|
|
const noRegex = "x + y + z;";
|
|
setup(noRegex);
|
|
const tokens2 = getTokens({ range: [0, noRegex.length] } as Node);
|
|
for (const token of tokens2) {
|
|
expect(token.regex).toBeUndefined();
|
|
}
|
|
});
|
|
});
|
|
|
|
// Tests for `for (const key in token)` enumeration.
|
|
// All token properties (including the `loc` getter and `regex`) should be enumerable.
|
|
describe("token property enumeration", () => {
|
|
it("should enumerate all properties including `loc` for a non-regex token", () => {
|
|
const tokens = getTokens({ range: [0, SOURCE_TEXT.length] } as Node);
|
|
const token = tokens[0]; // `var` keyword
|
|
|
|
const keys: string[] = [];
|
|
for (const key in token) {
|
|
keys.push(key);
|
|
}
|
|
|
|
expect(keys).toContain("type");
|
|
expect(keys).toContain("value");
|
|
expect(keys).toContain("regex");
|
|
expect(keys).toContain("start");
|
|
expect(keys).toContain("end");
|
|
expect(keys).toContain("range");
|
|
expect(keys).toContain("loc");
|
|
});
|
|
|
|
it("should enumerate all properties after `token.loc` has been accessed", () => {
|
|
const tokens = getTokens({ range: [0, SOURCE_TEXT.length] } as Node);
|
|
const token = tokens[0]; // `var` keyword
|
|
|
|
// Access `loc` to trigger caching
|
|
expect(token.loc.start.line).toBe(1);
|
|
|
|
const keys: string[] = [];
|
|
for (const key in token) {
|
|
keys.push(key);
|
|
}
|
|
|
|
expect(keys).toContain("type");
|
|
expect(keys).toContain("value");
|
|
expect(keys).toContain("regex");
|
|
expect(keys).toContain("start");
|
|
expect(keys).toContain("end");
|
|
expect(keys).toContain("range");
|
|
expect(keys).toContain("loc");
|
|
});
|
|
|
|
it("should enumerate all properties after `getLoc(token)` has been called", () => {
|
|
const tokens = getTokens({ range: [0, SOURCE_TEXT.length] } as Node);
|
|
const token = tokens[0]; // `var` keyword
|
|
|
|
// Access `loc` via `getLoc` to trigger caching
|
|
expect(getLoc(token).start.line).toBe(1);
|
|
|
|
const keys: string[] = [];
|
|
for (const key in token) {
|
|
keys.push(key);
|
|
}
|
|
|
|
expect(keys).toContain("type");
|
|
expect(keys).toContain("value");
|
|
expect(keys).toContain("regex");
|
|
expect(keys).toContain("start");
|
|
expect(keys).toContain("end");
|
|
expect(keys).toContain("range");
|
|
expect(keys).toContain("loc");
|
|
});
|
|
|
|
it("should enumerate all properties for a regex token", () => {
|
|
setup("var x = /abc/g;");
|
|
const tokens = getTokens({ range: [0, 15] } as Node);
|
|
const regexToken = tokens.find((t) => t.type === "RegularExpression")!;
|
|
|
|
const keys: string[] = [];
|
|
for (const key in regexToken) {
|
|
keys.push(key);
|
|
}
|
|
|
|
expect(keys).toContain("type");
|
|
expect(keys).toContain("value");
|
|
expect(keys).toContain("regex");
|
|
expect(keys).toContain("start");
|
|
expect(keys).toContain("end");
|
|
expect(keys).toContain("range");
|
|
expect(keys).toContain("loc");
|
|
});
|
|
});
|