Files
oxc-project__oxc/justfile
overlookmotel 6365db964b test(lexer): enable oxc_lexer conformance tests on all platforms (#26263)
`oxc_lexer` has 2 implementations:

1\. SIMD-based on x86_64 with AVX2.
2\. Scalar/SWAR fallback which works on all platforms.

Both produce the same output.

This PR enables running the lexer conformance on either implementation. Which implementation is tested depends on what build flags are passed to `cargo coverage`.

It turns out that it's possible to test the SIMD implementation on Mac by compiling as x86_64, and running it under Rosetta 2.

This PR also adds `just` recipes to run tests and conformance on either implementation:

- `just test-lexer`
- `just test-lexer-simd`
- `just conformance-lexer`
- `just conformance-lexer-simd`

These commands should work correctly on any of Linux x86, Windows x86, Mac aarch64.
2026-09-03 10:59:05 +00:00

366 lines
12 KiB
Makefile
Executable File

#!/usr/bin/env -S just --justfile
set windows-shell := ["powershell.exe", "-NoLogo", "-Command"]
set shell := ["bash", "-cu"]
_default:
@just --list -u
# ==================== ALIASES ====================
alias r := ready
alias c := conformance
alias f := fix
# ==================== SETUP & INITIALIZATION ====================
# Initialize the project by installing all necessary tools
init:
# Rust related init
cargo binstall watchexec-cli cargo-insta typos-cli cargo-shear@1.13.1 -y
# Node.js related init
pnpm install
# Clone or update submodules
submodules:
node .github/scripts/clone-parallel.mjs
just update-transformer-fixtures
# Install git pre-commit hook to format files
install-hook:
echo -e "#!/bin/sh\njust fmt" > .git/hooks/pre-commit
chmod +x .git/hooks/pre-commit
# ==================== CORE DEVELOPMENT ====================
# When ready, run the same CI commands
ready:
git diff --exit-code --quiet
pnpm install
typos
cargo lintgen
just fmt
just check
just test
just lint
just doc
just ast
git status
# Run cargo check
check *args:
cargo ck {{args}}
# Run all the tests
test *args:
cargo test --all-features {{args}}
# Lint the whole project
[unix]
lint *args:
CARGO_BUILD_WARNINGS=deny cargo lint {{args}}
[windows]
lint *args:
$Env:CARGO_BUILD_WARNINGS='deny'; cargo lint {{args}}
# Format all files
fmt:
-cargo shear --fix --check-test-targets # remove all unused dependencies
cargo fmt
node --run fmt
[unix]
doc *args:
RUSTDOCFLAGS='-D warnings' cargo doc --no-deps --document-private-items --all-features {{args}}
[windows]
doc *args:
$Env:RUSTDOCFLAGS='-D warnings'; cargo doc --no-deps --document-private-items --all-features {{args}}
# Fix all auto-fixable format and lint issues
fix:
cargo clippy --fix --allow-staged --no-deps
just fmt
typos -w
git status
# ==================== DEVELOPMENT TOOLS ====================
watch *args='':
watchexec --no-vcs-ignore {{args}}
watch-check:
just watch "'cargo check; cargo clippy'"
watch-example *args='':
just watch 'just example {{args}}'
# Run examples in parser, formatter, linter
example tool *args='':
cargo run -p oxc_{{tool}} --example {{tool}} -- {{args}}
# Run the benchmarks
benchmark:
cargo benchmark
# Run benchmarks for a single component
benchmark-one *args:
cargo benchmark --bench {{args}} --no-default-features --features {{ if args == "linter" { "linter" } else { "compiler" } }}
# ==================== TESTING & CONFORMANCE ====================
# Run all conformance tests
coverage:
cargo coverage
cargo run -p oxc_transform_conformance -- --exec
# Run Test262, Babel and TypeScript conformance suite
conformance *args='':
cargo coverage -- {{args}}
# Test ESTree
test-estree *args='':
cargo run -p oxc_coverage --profile coverage -- estree {{args}}
test-estree-tokens *args='':
cargo run -p oxc_coverage --profile coverage -- estree_tokens {{args}}
# Get code coverage
codecov:
cargo codecov --html
# ==================== AST & CODEGEN ====================
# Generate AST related boilerplate code.
# If fails first time, run with JS generators disabled first, and then again with JS generators enabled.
# This is necessary because JS generators use `oxc_*` crates (e.g. `oxc_minifier`), and those crates may not compile
# unless Rust code is generated first.
# See: https://github.com/oxc-project/oxc/issues/15564
[unix]
ast:
cargo run -p oxc_ast_tools || { cargo run -p oxc_ast_tools --no-default-features && cargo run -p oxc_ast_tools; }
[windows]
ast:
try { cargo run -p oxc_ast_tools } catch { cargo run -p oxc_ast_tools --no-default-features; cargo run -p oxc_ast_tools }
# ==================== PARSER ====================
# Parser-specific commands will be added here as needed
# ==================== LEXER ====================
# `oxc_lexer` compiles to one of two implementations, chosen at build time:
#
# * SIMD core, on x86_64 with `avx2` + `bmi2` enabled
# * Scalar fallback, everywhere else
#
# Both need testing, and they should produce identical results. Neither `avx2` nor `bmi2` is in the x86_64 baseline
# on any platform, so reaching the SIMD core always means asking for them explicitly - even on an x86_64 host.
# The flags live in `.cargo/lexer-simd.toml`, passed with `cargo --config`, which avoids shell quoting entirely
# (this justfile runs PowerShell on Windows).
#
# `--target` is always passed, so that the flags land on the triple the config file names,
# and so that the two builds get separate target dirs instead of invalidating each other.
#
# On an ARM host the SIMD build is cross-compiled and run under emulation - macOS provides that via Rosetta 2.
# On ARM Linux or ARM Windows it will fail, loudly, when linking or running.
_lexer-simd-target := if os() == "macos" { "x86_64-apple-darwin" } else if os() == "windows" { "x86_64-pc-windows-msvc" } else { "x86_64-unknown-linux-gnu" }
_lexer-simd := "--target " + _lexer-simd-target + " --config .cargo/lexer-simd.toml"
# Run `oxc_lexer`'s tests against the scalar fallback
test-lexer *args='':
cargo test -p oxc_lexer {{args}}
# Run `oxc_lexer`'s tests against the SIMD core
test-lexer-simd *args='':
cargo test -p oxc_lexer {{_lexer-simd}} {{args}}
# Run lexer conformance against the scalar fallback
conformance-lexer *args='':
cargo run -p oxc_coverage --profile coverage --features lexer -- lexer {{args}}
# Run lexer conformance against the SIMD core
conformance-lexer-simd *args='':
cargo run -p oxc_coverage --profile coverage {{_lexer-simd}} --features lexer -- lexer {{args}}
# ==================== LINTER ====================
# oxlint release build
oxlint:
cargo build -p oxlint --release --features allocator
# watch oxlint, e.g. `just watch-oxlint test.js`
watch-oxlint *args='':
just watch 'cargo run -p oxlint -- --disable-nested-config {{args}}'
# oxlint release build for node.js
# After building, you can run oxlint with `node <oxc-root>/apps/oxlint/dist/cli.js`
oxlint-node:
pnpm -C apps/oxlint run build
# oxlint dev build, for testing with Node.js locally.
# This uses a non-release Rust build without the `allocator` feature (no mimalloc) and sets DEBUG options for the JS bundle,
# which mainly affects build time, performance, and debug assertions rather than available linting functionality.
# After building, you can run oxlint with `node <oxc-root>/apps/oxlint/dist/cli.js`
oxlint-node-dev:
pnpm -C apps/oxlint run build-dev
watch-oxlint-node *args='':
just watch 'pnpm run -C apps/oxlint build-dev && node apps/oxlint/dist/cli.js --disable-nested-config {{args}}'
# Create a new lint rule for any plugin
new-rule name plugin='eslint':
cargo run -p rulegen {{name}} {{plugin}}
just linter-schema-json
just linter-config-ts
just fmt
# Update test cases for an existing lint rule from upstream
update-rule-tests name plugin='eslint':
cargo run -p rulegen {{name}} {{plugin}} --update-tests
just fmt
# Legacy aliases for backward compatibility
new-eslint-rule name: (new-rule name "eslint")
new-jest-rule name: (new-rule name "jest")
new-ts-rule name: (new-rule name "typescript")
new-unicorn-rule name: (new-rule name "unicorn")
new-import-rule name: (new-rule name "import")
new-react-rule name: (new-rule name "react")
new-jsx-a11y-rule name: (new-rule name "jsx-a11y")
new-oxc-rule name: (new-rule name "oxc")
new-nextjs-rule name: (new-rule name "nextjs")
new-jsdoc-rule name: (new-rule name "jsdoc")
new-react-perf-rule name: (new-rule name "react-perf")
new-n-rule name: (new-rule name "n")
new-promise-rule name: (new-rule name "promise")
new-vitest-rule name: (new-rule name "vitest")
new-vue-rule name: (new-rule name "vue")
# Alias for backward compatibility
alias new-typescript-rule := new-ts-rule
# ==================== FORMATTER ====================
# oxfmt release build
oxfmt:
cargo build -p oxfmt --release --features allocator
# watch oxfmt, e.g. `just watch-oxfmt test.js`
watch-oxfmt *args='':
just watch 'cargo run -p oxfmt -- {{args}}'
# Build oxfmt in release build
# After building, you can run oxfmt with `node <oxc-root>/apps/oxfmt/dist/cli.js`
oxfmt-node:
pnpm -C apps/oxfmt run build
# oxfmt dev build, for testing with Node.js locally.
# This builds faster than the release build and may differ in performance or behavior.
# After building, you can run oxfmt with `node <oxc-root>/apps/oxfmt/dist/cli.js`
oxfmt-node-dev:
pnpm -C apps/oxfmt run build-dev
watch-oxfmt-node *args='':
just watch 'pnpm run -C apps/oxfmt build-dev && node apps/oxfmt/dist/cli.js {{args}}'
# ==================== TRANSFORMER ====================
# Test Transform
test-transform *args='':
cargo run -p oxc_transform_conformance -- --exec {{args}}
# Update transformer conformance test fixtures
update-transformer-fixtures:
cd tasks/coverage/babel; git reset --hard HEAD; git clean -f -q
node tasks/transform_conformance/update_fixtures.ts
# ==================== MINIFIER ====================
# Update minifier size snapshots
minsize:
cargo minsize
just allocs
# Update memory allocation snapshots
allocs:
cargo allocs
# Update linter timing snapshots
lint-timings:
cargo lint-timings
# Generate minifier size comparison
minifier-diff:
#!/usr/bin/env bash
cargo minsize --compress-only pr
git checkout main
cargo minsize --compress-only main
for file in antd bundle.min d3 echarts jquery lodash moment react.development three typescript victory vue
do
echo $file.js >> diff
diff target/minifier/main/$file.js target/minifier/pr/$file.js >> diff
done
git checkout -
# ==================== PLAYGROUND ====================
# Install wasm32-wasip1-threads for playground
install-wasm:
rustup target add wasm32-wasip1-threads
build-playground:
pnpm --filter oxc-playground build
watch-playground:
just watch 'pnpm --filter oxc-playground build-dev'
# ==================== UTILITIES & ADVANCED ====================
# Generate website documentation, intended for updating the oxc.rs website.
# Path should be the path to your clone of https://github.com/oxc-project/website
# When testing changes to the website documentation, you may also want to run `pnpm run fmt`
# in the website directory.
website path:
cargo run -p website_linter rules --rules-json {{path}}/.vitepress/data/rules.json --rule-docs {{path}}/src/docs/guide/usage/linter/rules --git-ref $(git rev-parse HEAD) --rule-count {{path}}/src/docs/guide/usage
cargo run -p website_linter cli > {{path}}/src/docs/guide/usage/linter/generated-cli.md
cargo run -p website_linter schema-markdown > {{path}}/src/docs/guide/usage/linter/generated-config.md
cargo run -p website_linter schema-markdown-lsp > {{path}}/src/docs/guide/usage/linter/generated-lsp-config.md
cargo run -p website_formatter cli > {{path}}/src/docs/guide/usage/formatter/generated-cli.md
cargo run -p website_formatter schema-markdown > {{path}}/src/docs/guide/usage/formatter/generated-config.md
# Generate linter schema json for `npm/oxlint/configuration_schema.json`
linter-schema-json:
cargo run -p website_linter schema-json > npm/oxlint/configuration_schema.json
# Generate linter config TypeScript types for `apps/oxlint/src-js/package/config.generated.ts`
linter-config-ts:
pnpm --filter oxlint-app generate-config-types
# Generate formatter schema json for `npm/oxfmt/configuration_schema.json`
formatter-schema-json:
cargo run -p website_formatter schema-json > npm/oxfmt/configuration_schema.json
# Generate formatter config TypeScript types for `apps/oxfmt/src-js/config.generated.ts`
formatter-config-ts:
pnpm --filter oxfmt-app generate-config-types
# Automatically DRY up Cargo.toml manifests in a workspace
autoinherit:
cargo binstall cargo-autoinherit
cargo autoinherit
# ==================== PLATFORM HELPERS ====================
[unix]
clone-submodule dir url sha:
cd {{dir}} || git init {{dir}}
cd {{dir}} && git remote add origin {{url}} || true
cd {{dir}} && git fetch --depth=1 origin {{sha}} && git reset --hard {{sha}} && git clean -f -q
[windows]
clone-submodule dir url sha:
if (-not (Test-Path {{dir}}/.git)) { git init {{dir}} }
cd {{dir}} ; if ((git remote) -notcontains 'origin') { git remote add origin {{url}} } else { git remote set-url origin {{url}} }
cd {{dir}} ; git fetch --depth=1 origin {{sha}} ; git reset --hard {{sha}} ; git clean -f -q