mirror of
https://github.com/oxc-project/oxc.git
synced 2026-09-14 19:36:11 +08:00
6365db964b
`oxc_lexer` has 2 implementations: 1\. SIMD-based on x86_64 with AVX2. 2\. Scalar/SWAR fallback which works on all platforms. Both produce the same output. This PR enables running the lexer conformance on either implementation. Which implementation is tested depends on what build flags are passed to `cargo coverage`. It turns out that it's possible to test the SIMD implementation on Mac by compiling as x86_64, and running it under Rosetta 2. This PR also adds `just` recipes to run tests and conformance on either implementation: - `just test-lexer` - `just test-lexer-simd` - `just conformance-lexer` - `just conformance-lexer-simd` These commands should work correctly on any of Linux x86, Windows x86, Mac aarch64.
366 lines
12 KiB
Makefile
Executable File
366 lines
12 KiB
Makefile
Executable File
#!/usr/bin/env -S just --justfile
|
|
|
|
set windows-shell := ["powershell.exe", "-NoLogo", "-Command"]
|
|
set shell := ["bash", "-cu"]
|
|
|
|
_default:
|
|
@just --list -u
|
|
|
|
# ==================== ALIASES ====================
|
|
alias r := ready
|
|
alias c := conformance
|
|
alias f := fix
|
|
|
|
# ==================== SETUP & INITIALIZATION ====================
|
|
|
|
# Initialize the project by installing all necessary tools
|
|
init:
|
|
# Rust related init
|
|
cargo binstall watchexec-cli cargo-insta typos-cli cargo-shear@1.13.1 -y
|
|
# Node.js related init
|
|
pnpm install
|
|
|
|
# Clone or update submodules
|
|
submodules:
|
|
node .github/scripts/clone-parallel.mjs
|
|
just update-transformer-fixtures
|
|
|
|
# Install git pre-commit hook to format files
|
|
install-hook:
|
|
echo -e "#!/bin/sh\njust fmt" > .git/hooks/pre-commit
|
|
chmod +x .git/hooks/pre-commit
|
|
|
|
# ==================== CORE DEVELOPMENT ====================
|
|
|
|
# When ready, run the same CI commands
|
|
ready:
|
|
git diff --exit-code --quiet
|
|
pnpm install
|
|
typos
|
|
cargo lintgen
|
|
just fmt
|
|
just check
|
|
just test
|
|
just lint
|
|
just doc
|
|
just ast
|
|
git status
|
|
|
|
# Run cargo check
|
|
check *args:
|
|
cargo ck {{args}}
|
|
|
|
# Run all the tests
|
|
test *args:
|
|
cargo test --all-features {{args}}
|
|
|
|
# Lint the whole project
|
|
[unix]
|
|
lint *args:
|
|
CARGO_BUILD_WARNINGS=deny cargo lint {{args}}
|
|
|
|
[windows]
|
|
lint *args:
|
|
$Env:CARGO_BUILD_WARNINGS='deny'; cargo lint {{args}}
|
|
|
|
# Format all files
|
|
fmt:
|
|
-cargo shear --fix --check-test-targets # remove all unused dependencies
|
|
cargo fmt
|
|
node --run fmt
|
|
|
|
[unix]
|
|
doc *args:
|
|
RUSTDOCFLAGS='-D warnings' cargo doc --no-deps --document-private-items --all-features {{args}}
|
|
|
|
[windows]
|
|
doc *args:
|
|
$Env:RUSTDOCFLAGS='-D warnings'; cargo doc --no-deps --document-private-items --all-features {{args}}
|
|
|
|
# Fix all auto-fixable format and lint issues
|
|
fix:
|
|
cargo clippy --fix --allow-staged --no-deps
|
|
just fmt
|
|
typos -w
|
|
git status
|
|
|
|
# ==================== DEVELOPMENT TOOLS ====================
|
|
|
|
watch *args='':
|
|
watchexec --no-vcs-ignore {{args}}
|
|
|
|
watch-check:
|
|
just watch "'cargo check; cargo clippy'"
|
|
|
|
watch-example *args='':
|
|
just watch 'just example {{args}}'
|
|
|
|
# Run examples in parser, formatter, linter
|
|
example tool *args='':
|
|
cargo run -p oxc_{{tool}} --example {{tool}} -- {{args}}
|
|
|
|
# Run the benchmarks
|
|
benchmark:
|
|
cargo benchmark
|
|
|
|
# Run benchmarks for a single component
|
|
benchmark-one *args:
|
|
cargo benchmark --bench {{args}} --no-default-features --features {{ if args == "linter" { "linter" } else { "compiler" } }}
|
|
|
|
# ==================== TESTING & CONFORMANCE ====================
|
|
|
|
# Run all conformance tests
|
|
coverage:
|
|
cargo coverage
|
|
cargo run -p oxc_transform_conformance -- --exec
|
|
|
|
# Run Test262, Babel and TypeScript conformance suite
|
|
conformance *args='':
|
|
cargo coverage -- {{args}}
|
|
|
|
# Test ESTree
|
|
test-estree *args='':
|
|
cargo run -p oxc_coverage --profile coverage -- estree {{args}}
|
|
|
|
test-estree-tokens *args='':
|
|
cargo run -p oxc_coverage --profile coverage -- estree_tokens {{args}}
|
|
|
|
# Get code coverage
|
|
codecov:
|
|
cargo codecov --html
|
|
|
|
# ==================== AST & CODEGEN ====================
|
|
|
|
# Generate AST related boilerplate code.
|
|
# If fails first time, run with JS generators disabled first, and then again with JS generators enabled.
|
|
# This is necessary because JS generators use `oxc_*` crates (e.g. `oxc_minifier`), and those crates may not compile
|
|
# unless Rust code is generated first.
|
|
# See: https://github.com/oxc-project/oxc/issues/15564
|
|
[unix]
|
|
ast:
|
|
cargo run -p oxc_ast_tools || { cargo run -p oxc_ast_tools --no-default-features && cargo run -p oxc_ast_tools; }
|
|
[windows]
|
|
ast:
|
|
try { cargo run -p oxc_ast_tools } catch { cargo run -p oxc_ast_tools --no-default-features; cargo run -p oxc_ast_tools }
|
|
|
|
# ==================== PARSER ====================
|
|
|
|
# Parser-specific commands will be added here as needed
|
|
|
|
# ==================== LEXER ====================
|
|
|
|
# `oxc_lexer` compiles to one of two implementations, chosen at build time:
|
|
#
|
|
# * SIMD core, on x86_64 with `avx2` + `bmi2` enabled
|
|
# * Scalar fallback, everywhere else
|
|
#
|
|
# Both need testing, and they should produce identical results. Neither `avx2` nor `bmi2` is in the x86_64 baseline
|
|
# on any platform, so reaching the SIMD core always means asking for them explicitly - even on an x86_64 host.
|
|
# The flags live in `.cargo/lexer-simd.toml`, passed with `cargo --config`, which avoids shell quoting entirely
|
|
# (this justfile runs PowerShell on Windows).
|
|
#
|
|
# `--target` is always passed, so that the flags land on the triple the config file names,
|
|
# and so that the two builds get separate target dirs instead of invalidating each other.
|
|
#
|
|
# On an ARM host the SIMD build is cross-compiled and run under emulation - macOS provides that via Rosetta 2.
|
|
# On ARM Linux or ARM Windows it will fail, loudly, when linking or running.
|
|
_lexer-simd-target := if os() == "macos" { "x86_64-apple-darwin" } else if os() == "windows" { "x86_64-pc-windows-msvc" } else { "x86_64-unknown-linux-gnu" }
|
|
_lexer-simd := "--target " + _lexer-simd-target + " --config .cargo/lexer-simd.toml"
|
|
|
|
# Run `oxc_lexer`'s tests against the scalar fallback
|
|
test-lexer *args='':
|
|
cargo test -p oxc_lexer {{args}}
|
|
|
|
# Run `oxc_lexer`'s tests against the SIMD core
|
|
test-lexer-simd *args='':
|
|
cargo test -p oxc_lexer {{_lexer-simd}} {{args}}
|
|
|
|
# Run lexer conformance against the scalar fallback
|
|
conformance-lexer *args='':
|
|
cargo run -p oxc_coverage --profile coverage --features lexer -- lexer {{args}}
|
|
|
|
# Run lexer conformance against the SIMD core
|
|
conformance-lexer-simd *args='':
|
|
cargo run -p oxc_coverage --profile coverage {{_lexer-simd}} --features lexer -- lexer {{args}}
|
|
|
|
# ==================== LINTER ====================
|
|
|
|
# oxlint release build
|
|
oxlint:
|
|
cargo build -p oxlint --release --features allocator
|
|
|
|
# watch oxlint, e.g. `just watch-oxlint test.js`
|
|
watch-oxlint *args='':
|
|
just watch 'cargo run -p oxlint -- --disable-nested-config {{args}}'
|
|
|
|
# oxlint release build for node.js
|
|
# After building, you can run oxlint with `node <oxc-root>/apps/oxlint/dist/cli.js`
|
|
oxlint-node:
|
|
pnpm -C apps/oxlint run build
|
|
|
|
# oxlint dev build, for testing with Node.js locally.
|
|
# This uses a non-release Rust build without the `allocator` feature (no mimalloc) and sets DEBUG options for the JS bundle,
|
|
# which mainly affects build time, performance, and debug assertions rather than available linting functionality.
|
|
# After building, you can run oxlint with `node <oxc-root>/apps/oxlint/dist/cli.js`
|
|
oxlint-node-dev:
|
|
pnpm -C apps/oxlint run build-dev
|
|
|
|
watch-oxlint-node *args='':
|
|
just watch 'pnpm run -C apps/oxlint build-dev && node apps/oxlint/dist/cli.js --disable-nested-config {{args}}'
|
|
|
|
# Create a new lint rule for any plugin
|
|
new-rule name plugin='eslint':
|
|
cargo run -p rulegen {{name}} {{plugin}}
|
|
just linter-schema-json
|
|
just linter-config-ts
|
|
just fmt
|
|
|
|
# Update test cases for an existing lint rule from upstream
|
|
update-rule-tests name plugin='eslint':
|
|
cargo run -p rulegen {{name}} {{plugin}} --update-tests
|
|
just fmt
|
|
|
|
# Legacy aliases for backward compatibility
|
|
new-eslint-rule name: (new-rule name "eslint")
|
|
new-jest-rule name: (new-rule name "jest")
|
|
new-ts-rule name: (new-rule name "typescript")
|
|
new-unicorn-rule name: (new-rule name "unicorn")
|
|
new-import-rule name: (new-rule name "import")
|
|
new-react-rule name: (new-rule name "react")
|
|
new-jsx-a11y-rule name: (new-rule name "jsx-a11y")
|
|
new-oxc-rule name: (new-rule name "oxc")
|
|
new-nextjs-rule name: (new-rule name "nextjs")
|
|
new-jsdoc-rule name: (new-rule name "jsdoc")
|
|
new-react-perf-rule name: (new-rule name "react-perf")
|
|
new-n-rule name: (new-rule name "n")
|
|
new-promise-rule name: (new-rule name "promise")
|
|
new-vitest-rule name: (new-rule name "vitest")
|
|
new-vue-rule name: (new-rule name "vue")
|
|
|
|
# Alias for backward compatibility
|
|
alias new-typescript-rule := new-ts-rule
|
|
|
|
# ==================== FORMATTER ====================
|
|
|
|
# oxfmt release build
|
|
oxfmt:
|
|
cargo build -p oxfmt --release --features allocator
|
|
|
|
# watch oxfmt, e.g. `just watch-oxfmt test.js`
|
|
watch-oxfmt *args='':
|
|
just watch 'cargo run -p oxfmt -- {{args}}'
|
|
|
|
# Build oxfmt in release build
|
|
# After building, you can run oxfmt with `node <oxc-root>/apps/oxfmt/dist/cli.js`
|
|
oxfmt-node:
|
|
pnpm -C apps/oxfmt run build
|
|
|
|
# oxfmt dev build, for testing with Node.js locally.
|
|
# This builds faster than the release build and may differ in performance or behavior.
|
|
# After building, you can run oxfmt with `node <oxc-root>/apps/oxfmt/dist/cli.js`
|
|
oxfmt-node-dev:
|
|
pnpm -C apps/oxfmt run build-dev
|
|
|
|
watch-oxfmt-node *args='':
|
|
just watch 'pnpm run -C apps/oxfmt build-dev && node apps/oxfmt/dist/cli.js {{args}}'
|
|
|
|
# ==================== TRANSFORMER ====================
|
|
|
|
# Test Transform
|
|
test-transform *args='':
|
|
cargo run -p oxc_transform_conformance -- --exec {{args}}
|
|
|
|
# Update transformer conformance test fixtures
|
|
update-transformer-fixtures:
|
|
cd tasks/coverage/babel; git reset --hard HEAD; git clean -f -q
|
|
node tasks/transform_conformance/update_fixtures.ts
|
|
|
|
# ==================== MINIFIER ====================
|
|
|
|
# Update minifier size snapshots
|
|
minsize:
|
|
cargo minsize
|
|
just allocs
|
|
|
|
# Update memory allocation snapshots
|
|
allocs:
|
|
cargo allocs
|
|
|
|
# Update linter timing snapshots
|
|
lint-timings:
|
|
cargo lint-timings
|
|
|
|
# Generate minifier size comparison
|
|
minifier-diff:
|
|
#!/usr/bin/env bash
|
|
cargo minsize --compress-only pr
|
|
git checkout main
|
|
cargo minsize --compress-only main
|
|
for file in antd bundle.min d3 echarts jquery lodash moment react.development three typescript victory vue
|
|
do
|
|
echo $file.js >> diff
|
|
diff target/minifier/main/$file.js target/minifier/pr/$file.js >> diff
|
|
done
|
|
git checkout -
|
|
|
|
# ==================== PLAYGROUND ====================
|
|
|
|
# Install wasm32-wasip1-threads for playground
|
|
install-wasm:
|
|
rustup target add wasm32-wasip1-threads
|
|
|
|
build-playground:
|
|
pnpm --filter oxc-playground build
|
|
|
|
watch-playground:
|
|
just watch 'pnpm --filter oxc-playground build-dev'
|
|
|
|
# ==================== UTILITIES & ADVANCED ====================
|
|
|
|
# Generate website documentation, intended for updating the oxc.rs website.
|
|
# Path should be the path to your clone of https://github.com/oxc-project/website
|
|
# When testing changes to the website documentation, you may also want to run `pnpm run fmt`
|
|
# in the website directory.
|
|
website path:
|
|
cargo run -p website_linter rules --rules-json {{path}}/.vitepress/data/rules.json --rule-docs {{path}}/src/docs/guide/usage/linter/rules --git-ref $(git rev-parse HEAD) --rule-count {{path}}/src/docs/guide/usage
|
|
cargo run -p website_linter cli > {{path}}/src/docs/guide/usage/linter/generated-cli.md
|
|
cargo run -p website_linter schema-markdown > {{path}}/src/docs/guide/usage/linter/generated-config.md
|
|
cargo run -p website_linter schema-markdown-lsp > {{path}}/src/docs/guide/usage/linter/generated-lsp-config.md
|
|
cargo run -p website_formatter cli > {{path}}/src/docs/guide/usage/formatter/generated-cli.md
|
|
cargo run -p website_formatter schema-markdown > {{path}}/src/docs/guide/usage/formatter/generated-config.md
|
|
|
|
# Generate linter schema json for `npm/oxlint/configuration_schema.json`
|
|
linter-schema-json:
|
|
cargo run -p website_linter schema-json > npm/oxlint/configuration_schema.json
|
|
|
|
# Generate linter config TypeScript types for `apps/oxlint/src-js/package/config.generated.ts`
|
|
linter-config-ts:
|
|
pnpm --filter oxlint-app generate-config-types
|
|
|
|
# Generate formatter schema json for `npm/oxfmt/configuration_schema.json`
|
|
formatter-schema-json:
|
|
cargo run -p website_formatter schema-json > npm/oxfmt/configuration_schema.json
|
|
|
|
# Generate formatter config TypeScript types for `apps/oxfmt/src-js/config.generated.ts`
|
|
formatter-config-ts:
|
|
pnpm --filter oxfmt-app generate-config-types
|
|
|
|
# Automatically DRY up Cargo.toml manifests in a workspace
|
|
autoinherit:
|
|
cargo binstall cargo-autoinherit
|
|
cargo autoinherit
|
|
|
|
# ==================== PLATFORM HELPERS ====================
|
|
|
|
[unix]
|
|
clone-submodule dir url sha:
|
|
cd {{dir}} || git init {{dir}}
|
|
cd {{dir}} && git remote add origin {{url}} || true
|
|
cd {{dir}} && git fetch --depth=1 origin {{sha}} && git reset --hard {{sha}} && git clean -f -q
|
|
|
|
[windows]
|
|
clone-submodule dir url sha:
|
|
if (-not (Test-Path {{dir}}/.git)) { git init {{dir}} }
|
|
cd {{dir}} ; if ((git remote) -notcontains 'origin') { git remote add origin {{url}} } else { git remote set-url origin {{url}} }
|
|
cd {{dir}} ; git fetch --depth=1 origin {{sha}} ; git reset --hard {{sha}} ; git clean -f -q
|