mirror of
https://github.com/oxc-project/oxc.git
synced 2026-09-14 19:36:11 +08:00
9ef028c2ad
Closes #17068. Supersedes #18095 / #18118. Adds `CodegenOptions::ascii_only` (and `codegen.asciiOnly` in `oxc_minify_napi`): escape every non-ASCII character in string literals, untagged template literals, regexp literals, directives and identifier names so the output is 7-bit clean — like esbuild `--charset=ascii` / terser `format.ascii_only`. Off by default; the default path is untouched (codegen bench and conformance unchanged). ```js // Input let café = "naïve 😀"; let re = /[–—]/u; // Output with ascii_only: true let caf\u00E9 = "na\u00EFve \u{1F600}"; let re = /[\u2013\u2014]/u; ``` Characters above U+FFFF use ES2015 code-point escapes. Regex patterns use escaped surrogate pairs instead; RegExp.prototype.source reflects the escaped spelling. Tagged template quasis, JSX names/text/attribute strings, hashbangs and preserved comments may still contain non-ASCII characters. JavaScript expressions inside tagged templates and JSX are escaped normally. Performance; with the option disabled, codegen codspeed benchmarks are withing noise (<1%). With the option on, performance is only affected for files that contain non ascii characters. --- Our use case is a large Vite/Rolldown app served through proxies that mangle non-ASCII JS (previously handled by terser's `ascii_only`); #17068 also lists deno_core snapshots and Chrome extension content scripts. AI-assisted (Claude Code); I reviewed the code, wrote the tests, and have been running this in production builds via a vendored `oxc_codegen`. --------- Co-authored-by: Samuel Attard <6634592+MarshallOfSound@users.noreply.github.com> Co-authored-by: Cameron Clark <cameron.clark@hey.com> Co-authored-by: Dunqing <dengqing0821@gmail.com>
177 lines
6.0 KiB
Rust
177 lines
6.0 KiB
Rust
use oxc_allocator::Allocator;
|
|
use oxc_ast::ast::Comment;
|
|
use oxc_codegen::{Codegen, CodegenOptions};
|
|
use oxc_parser::{ParseOptions, Parser};
|
|
use oxc_span::SourceType;
|
|
|
|
pub fn default_options() -> CodegenOptions {
|
|
// Ensure sourcemap do not crash.
|
|
CodegenOptions { source_map_path: Some("test.js".into()), ..CodegenOptions::default() }
|
|
}
|
|
|
|
#[track_caller]
|
|
pub fn test_with_parse_options(source_text: &str, expected: &str, parse_options: ParseOptions) {
|
|
let allocator = Allocator::default();
|
|
let ret =
|
|
Parser::new(&allocator, source_text, SourceType::tsx()).with_options(parse_options).parse();
|
|
assert!(ret.diagnostics.is_empty());
|
|
let result = Codegen::new().with_options(default_options()).build(&ret.program).code;
|
|
assert_eq!(result, expected, "\nfor source: {source_text}");
|
|
}
|
|
|
|
#[track_caller]
|
|
pub fn test(source_text: &str, expected: &str) {
|
|
test_options(source_text, expected, default_options());
|
|
}
|
|
|
|
#[track_caller]
|
|
pub fn test_ascii(source_text: &str, expected: &str) {
|
|
test_options(source_text, expected, CodegenOptions { ascii_only: true, ..default_options() });
|
|
}
|
|
|
|
#[track_caller]
|
|
pub fn test_minify_ascii(source_text: &str, expected: &str) {
|
|
test_options(
|
|
source_text,
|
|
expected,
|
|
CodegenOptions { ascii_only: true, minify: true, ..CodegenOptions::default() },
|
|
);
|
|
}
|
|
|
|
#[track_caller]
|
|
pub fn test_same(source_text: &str) {
|
|
test(source_text, source_text);
|
|
}
|
|
|
|
#[track_caller]
|
|
pub fn test_options(source_text: &str, expected: &str, options: CodegenOptions) {
|
|
test_options_with_source_type(source_text, expected, SourceType::tsx(), options);
|
|
}
|
|
|
|
#[track_caller]
|
|
pub fn test_tsx(source_text: &str, expected: &str) {
|
|
test_options_with_source_type(source_text, expected, SourceType::tsx(), default_options());
|
|
}
|
|
|
|
#[track_caller]
|
|
pub fn test_ts(source_text: &str, expected: &str) {
|
|
test_options_with_source_type(source_text, expected, SourceType::ts(), default_options());
|
|
}
|
|
|
|
#[track_caller]
|
|
pub fn test_options_with_source_type(
|
|
source_text: &str,
|
|
expected: &str,
|
|
source_type: SourceType,
|
|
options: CodegenOptions,
|
|
) {
|
|
let allocator = Allocator::default();
|
|
let ret = Parser::new(&allocator, source_text, source_type).parse();
|
|
assert!(ret.diagnostics.is_empty(), "Parse errors: {:?}", ret.diagnostics);
|
|
let result = Codegen::new().with_options(options).build(&ret.program).code;
|
|
assert_eq!(result, expected, "\nfor source: {source_text:?}");
|
|
}
|
|
|
|
/// Test with unambiguous source type (like .js files)
|
|
#[track_caller]
|
|
pub fn test_unambiguous(source_text: &str, expected: &str) {
|
|
test_options_with_source_type(
|
|
source_text,
|
|
expected,
|
|
SourceType::unambiguous(),
|
|
default_options(),
|
|
);
|
|
}
|
|
|
|
#[track_caller]
|
|
pub fn test_same_ignore_parse_errors(source_text: &str) {
|
|
let allocator = Allocator::default();
|
|
let ret = Parser::new(&allocator, source_text, SourceType::tsx()).parse();
|
|
let result = Codegen::new().with_options(default_options()).build(&ret.program).code;
|
|
assert_eq!(result, source_text, "\nfor source: {source_text:?}");
|
|
}
|
|
|
|
#[track_caller]
|
|
pub fn test_minify(source_text: &str, expected: &str) {
|
|
let source_type = SourceType::jsx();
|
|
let allocator = Allocator::default();
|
|
let ret = Parser::new(&allocator, source_text, source_type).parse();
|
|
assert!(ret.diagnostics.is_empty(), "Parse errors: {:?}", ret.diagnostics);
|
|
let result = Codegen::new()
|
|
.with_options(CodegenOptions { minify: true, ..CodegenOptions::default() })
|
|
.build(&ret.program)
|
|
.code;
|
|
assert_eq!(result, expected, "\nfor minify source: {source_text}");
|
|
}
|
|
|
|
#[track_caller]
|
|
pub fn test_minify_same(source_text: &str) {
|
|
test_minify(source_text, source_text);
|
|
}
|
|
|
|
#[track_caller]
|
|
pub fn codegen(source_text: &str) -> String {
|
|
codegen_options(source_text, &default_options()).code
|
|
}
|
|
|
|
#[track_caller]
|
|
pub fn codegen_options(source_text: &str, options: &CodegenOptions) -> CodegenTestReturn {
|
|
let allocator = Allocator::default();
|
|
let source_type = SourceType::ts();
|
|
let ret = Parser::new(&allocator, source_text, source_type).parse();
|
|
assert!(ret.diagnostics.is_empty(), "Parse errors: {:?}", ret.diagnostics);
|
|
let mut options = options.clone();
|
|
options.single_quote = true;
|
|
let ret = Codegen::new().with_options(options).build(&ret.program);
|
|
CodegenTestReturn { code: ret.code, legal_comments: ret.legal_comments }
|
|
}
|
|
|
|
pub struct CodegenTestReturn {
|
|
pub code: String,
|
|
pub legal_comments: Vec<Comment>,
|
|
}
|
|
|
|
#[track_caller]
|
|
pub fn test_idempotency(source_text: &str) {
|
|
test_idempotency_options(source_text, &default_options());
|
|
}
|
|
|
|
#[track_caller]
|
|
pub fn test_idempotency_options(source_text: &str, options: &CodegenOptions) {
|
|
let allocator = Allocator::default();
|
|
let source_type = SourceType::tsx();
|
|
let ret = Parser::new(&allocator, source_text, source_type).parse();
|
|
assert!(ret.diagnostics.is_empty(), "Parse errors: {:?}", ret.diagnostics);
|
|
let first = Codegen::new().with_options(options.clone()).build(&ret.program).code;
|
|
|
|
let allocator2 = Allocator::default();
|
|
let ret2 = Parser::new(&allocator2, &first, source_type).parse();
|
|
assert!(ret2.diagnostics.is_empty(), "Parse errors on second pass: {:?}", ret2.diagnostics);
|
|
let second = Codegen::new().with_options(options.clone()).build(&ret2.program).code;
|
|
|
|
assert_eq!(
|
|
first, second,
|
|
"\nIdempotency failed for source: {source_text}\nFirst pass:\n{first}\nSecond pass:\n{second}"
|
|
);
|
|
}
|
|
|
|
#[track_caller]
|
|
pub fn snapshot(name: &str, cases: &[&str]) {
|
|
snapshot_options(name, cases, &default_options());
|
|
}
|
|
|
|
#[track_caller]
|
|
pub fn snapshot_options(name: &str, cases: &[&str], options: &CodegenOptions) {
|
|
use std::fmt::Write;
|
|
|
|
let snapshot = cases.iter().enumerate().fold(String::new(), |mut w, (i, case)| {
|
|
let result = codegen_options(case, options).code;
|
|
write!(w, "########## {i}\n{case}\n----------\n{result}\n").unwrap();
|
|
w
|
|
});
|
|
|
|
insta::with_settings!({ prepend_module_to_snapshot => false, snapshot_suffix => "", omit_expression => true }, {
|
|
insta::assert_snapshot!(name, snapshot);
|
|
});
|
|
}
|