mirror of
https://github.com/vercel/workflow.git
synced 2026-09-14 19:59:43 +08:00
Pranay Prakash
bf6383d558
Switch the payload compression codec to zstd, which benchmarks 3–7× faster than gzip at an equal-or-better ratio on representative workloads (compression runs at every step boundary, so the write CPU is a per-step tax). zstd uses node:zlib (>= 22.15); gzip via the portable CompressionStream remains the fallback when zstd is unavailable, and WORKFLOW_COMPRESSION_CODEC=gzip forces it. Reads dispatch on the format prefix, so 'zstd' and 'gzip' payloads are both always decodable. zstd is Node-only (Web CompressionStream has no zstd), so the browser o11y read path registers a WASM-backed decoder (@tootallnate/zstd-wasm) via a new registerZstdDecoder hook; node:zlib handles Node-side reads (runtime replay, CLI, server o11y). A new workflow.serialization.codec span attribute reports which codec applied. gzip and zstd read support co-ship, so the existing specVersion-5 capability gate is unchanged. Verified end-to-end: spec-5 runs store zstd-prefixed payloads on disk and replay/complete correctly; the WASM decoder round-trips node:zlib zstd output. Benchmarks updated to compare zstd vs gzip. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
57 lines
1.9 KiB
TypeScript
57 lines
1.9 KiB
TypeScript
/**
|
|
* Compatibility test for the browser zstd decode path: payloads written by
|
|
* the SDK's `node:zlib` zstd codec must decode via the `@tootallnate/zstd-wasm`
|
|
* decoder the web o11y uses. If these ever disagree, the dashboard can't read
|
|
* compressed runs — so this locks the cross-codec contract in.
|
|
*/
|
|
import { readFileSync } from 'node:fs';
|
|
import { createRequire } from 'node:module';
|
|
import zlib from 'node:zlib';
|
|
import { decompressBytes } from '@tootallnate/zstd-wasm';
|
|
import { beforeAll, describe, expect, it } from 'vitest';
|
|
|
|
const require = createRequire(import.meta.url);
|
|
|
|
let wasmModule: WebAssembly.Module;
|
|
|
|
beforeAll(async () => {
|
|
const wasmPath = require.resolve('@tootallnate/zstd-wasm/zstd.wasm');
|
|
wasmModule = await WebAssembly.compile(readFileSync(wasmPath));
|
|
});
|
|
|
|
function zstd(bytes: Uint8Array): Uint8Array {
|
|
return new Uint8Array(
|
|
zlib.zstdCompressSync(bytes, {
|
|
params: { [zlib.constants.ZSTD_c_compressionLevel]: 3 },
|
|
})
|
|
);
|
|
}
|
|
|
|
describe('zstd WASM decoder ↔ node:zlib zstd compatibility', () => {
|
|
it('decodes a payload compressed by the SDK codec', async () => {
|
|
const original = new TextEncoder().encode(
|
|
JSON.stringify({
|
|
// Repetitive + varied content, like a real serialized payload.
|
|
users: Array.from({ length: 300 }, (_, i) => ({
|
|
id: `user_${i}`,
|
|
email: `user.${i}@example.com`,
|
|
role: i % 3 === 0 ? 'admin' : 'member',
|
|
})),
|
|
})
|
|
);
|
|
const compressed = zstd(original);
|
|
expect(compressed.length).toBeLessThan(original.length);
|
|
|
|
const decoded = await decompressBytes(wasmModule, compressed);
|
|
expect(new Uint8Array(decoded)).toEqual(original);
|
|
});
|
|
|
|
it('round-trips an empty and a tiny payload', async () => {
|
|
for (const s of ['', '{}', 'x']) {
|
|
const original = new TextEncoder().encode(s);
|
|
const decoded = await decompressBytes(wasmModule, zstd(original));
|
|
expect(new Uint8Array(decoded)).toEqual(original);
|
|
}
|
|
});
|
|
});
|