Files
oxc-project__oxc/crates/oxc_codegen/tests/integration/tester.rs
Samuel Attard 9ef028c2ad feat(codegen): add ascii_only option (#25994)
Closes #17068. Supersedes #18095 / #18118.

Adds `CodegenOptions::ascii_only` (and `codegen.asciiOnly` in
`oxc_minify_napi`): escape every non-ASCII character in string literals,
untagged template literals, regexp literals, directives and identifier
names so the output is 7-bit clean — like esbuild `--charset=ascii` /
terser `format.ascii_only`. Off by default; the default path is
untouched (codegen bench and conformance unchanged).

```js
// Input
let café = "naïve 😀";
let re = /[–—]/u;

// Output with ascii_only: true
let caf\u00E9 = "na\u00EFve \u{1F600}";
let re = /[\u2013\u2014]/u;
```

Characters above U+FFFF use ES2015 code-point escapes. Regex patterns
use escaped surrogate pairs instead; RegExp.prototype.source reflects
the escaped spelling.
Tagged template quasis, JSX names/text/attribute strings, hashbangs and
preserved comments may still contain non-ASCII characters. JavaScript
expressions inside tagged templates and JSX are escaped normally.

Performance; with the option disabled, codegen codspeed benchmarks are
withing noise (<1%). With the option on, performance is only affected
for files that contain non ascii characters.

---

Our use case is a large Vite/Rolldown app served through proxies that
mangle non-ASCII JS (previously handled by terser's `ascii_only`);
#17068 also lists deno_core snapshots and Chrome extension content
scripts.

AI-assisted (Claude Code); I reviewed the code, wrote the tests, and
have been running this in production builds via a vendored
`oxc_codegen`.

---------

Co-authored-by: Samuel Attard <6634592+MarshallOfSound@users.noreply.github.com>
Co-authored-by: Cameron Clark <cameron.clark@hey.com>
Co-authored-by: Dunqing <dengqing0821@gmail.com>
2026-09-10 17:10:02 +01:00

177 lines
6.0 KiB
Rust

use oxc_allocator::Allocator;
use oxc_ast::ast::Comment;
use oxc_codegen::{Codegen, CodegenOptions};
use oxc_parser::{ParseOptions, Parser};
use oxc_span::SourceType;
pub fn default_options() -> CodegenOptions {
// Ensure sourcemap do not crash.
CodegenOptions { source_map_path: Some("test.js".into()), ..CodegenOptions::default() }
}
#[track_caller]
pub fn test_with_parse_options(source_text: &str, expected: &str, parse_options: ParseOptions) {
let allocator = Allocator::default();
let ret =
Parser::new(&allocator, source_text, SourceType::tsx()).with_options(parse_options).parse();
assert!(ret.diagnostics.is_empty());
let result = Codegen::new().with_options(default_options()).build(&ret.program).code;
assert_eq!(result, expected, "\nfor source: {source_text}");
}
#[track_caller]
pub fn test(source_text: &str, expected: &str) {
test_options(source_text, expected, default_options());
}
#[track_caller]
pub fn test_ascii(source_text: &str, expected: &str) {
test_options(source_text, expected, CodegenOptions { ascii_only: true, ..default_options() });
}
#[track_caller]
pub fn test_minify_ascii(source_text: &str, expected: &str) {
test_options(
source_text,
expected,
CodegenOptions { ascii_only: true, minify: true, ..CodegenOptions::default() },
);
}
#[track_caller]
pub fn test_same(source_text: &str) {
test(source_text, source_text);
}
#[track_caller]
pub fn test_options(source_text: &str, expected: &str, options: CodegenOptions) {
test_options_with_source_type(source_text, expected, SourceType::tsx(), options);
}
#[track_caller]
pub fn test_tsx(source_text: &str, expected: &str) {
test_options_with_source_type(source_text, expected, SourceType::tsx(), default_options());
}
#[track_caller]
pub fn test_ts(source_text: &str, expected: &str) {
test_options_with_source_type(source_text, expected, SourceType::ts(), default_options());
}
#[track_caller]
pub fn test_options_with_source_type(
source_text: &str,
expected: &str,
source_type: SourceType,
options: CodegenOptions,
) {
let allocator = Allocator::default();
let ret = Parser::new(&allocator, source_text, source_type).parse();
assert!(ret.diagnostics.is_empty(), "Parse errors: {:?}", ret.diagnostics);
let result = Codegen::new().with_options(options).build(&ret.program).code;
assert_eq!(result, expected, "\nfor source: {source_text:?}");
}
/// Test with unambiguous source type (like .js files)
#[track_caller]
pub fn test_unambiguous(source_text: &str, expected: &str) {
test_options_with_source_type(
source_text,
expected,
SourceType::unambiguous(),
default_options(),
);
}
#[track_caller]
pub fn test_same_ignore_parse_errors(source_text: &str) {
let allocator = Allocator::default();
let ret = Parser::new(&allocator, source_text, SourceType::tsx()).parse();
let result = Codegen::new().with_options(default_options()).build(&ret.program).code;
assert_eq!(result, source_text, "\nfor source: {source_text:?}");
}
#[track_caller]
pub fn test_minify(source_text: &str, expected: &str) {
let source_type = SourceType::jsx();
let allocator = Allocator::default();
let ret = Parser::new(&allocator, source_text, source_type).parse();
assert!(ret.diagnostics.is_empty(), "Parse errors: {:?}", ret.diagnostics);
let result = Codegen::new()
.with_options(CodegenOptions { minify: true, ..CodegenOptions::default() })
.build(&ret.program)
.code;
assert_eq!(result, expected, "\nfor minify source: {source_text}");
}
#[track_caller]
pub fn test_minify_same(source_text: &str) {
test_minify(source_text, source_text);
}
#[track_caller]
pub fn codegen(source_text: &str) -> String {
codegen_options(source_text, &default_options()).code
}
#[track_caller]
pub fn codegen_options(source_text: &str, options: &CodegenOptions) -> CodegenTestReturn {
let allocator = Allocator::default();
let source_type = SourceType::ts();
let ret = Parser::new(&allocator, source_text, source_type).parse();
assert!(ret.diagnostics.is_empty(), "Parse errors: {:?}", ret.diagnostics);
let mut options = options.clone();
options.single_quote = true;
let ret = Codegen::new().with_options(options).build(&ret.program);
CodegenTestReturn { code: ret.code, legal_comments: ret.legal_comments }
}
pub struct CodegenTestReturn {
pub code: String,
pub legal_comments: Vec<Comment>,
}
#[track_caller]
pub fn test_idempotency(source_text: &str) {
test_idempotency_options(source_text, &default_options());
}
#[track_caller]
pub fn test_idempotency_options(source_text: &str, options: &CodegenOptions) {
let allocator = Allocator::default();
let source_type = SourceType::tsx();
let ret = Parser::new(&allocator, source_text, source_type).parse();
assert!(ret.diagnostics.is_empty(), "Parse errors: {:?}", ret.diagnostics);
let first = Codegen::new().with_options(options.clone()).build(&ret.program).code;
let allocator2 = Allocator::default();
let ret2 = Parser::new(&allocator2, &first, source_type).parse();
assert!(ret2.diagnostics.is_empty(), "Parse errors on second pass: {:?}", ret2.diagnostics);
let second = Codegen::new().with_options(options.clone()).build(&ret2.program).code;
assert_eq!(
first, second,
"\nIdempotency failed for source: {source_text}\nFirst pass:\n{first}\nSecond pass:\n{second}"
);
}
#[track_caller]
pub fn snapshot(name: &str, cases: &[&str]) {
snapshot_options(name, cases, &default_options());
}
#[track_caller]
pub fn snapshot_options(name: &str, cases: &[&str], options: &CodegenOptions) {
use std::fmt::Write;
let snapshot = cases.iter().enumerate().fold(String::new(), |mut w, (i, case)| {
let result = codegen_options(case, options).code;
write!(w, "########## {i}\n{case}\n----------\n{result}\n").unwrap();
w
});
insta::with_settings!({ prepend_module_to_snapshot => false, snapshot_suffix => "", omit_expression => true }, {
insta::assert_snapshot!(name, snapshot);
});
}