mirror of
https://github.com/proffesor-for-testing/agentic-qe.git
synced 2026-09-19 08:45:47 +08:00
Dragan Spiridonov
00e4d2cfa6
125 test files had beforeEach/beforeAll but no afterEach/afterAll, leaking state between tests and causing order-dependent failures. Added cleanup to 104 files (skipped 4 archived, 16 import-only): - 2 critical DB handle leaks (witness-chain): added db.close() - 7 high-risk service files: added vi.restoreAllMocks() - 7 beforeAll files: added matching afterAll - 88 medium/low risk: added afterEach with appropriate cleanup Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
949 lines
34 KiB
TypeScript
949 lines
34 KiB
TypeScript
/**
|
|
* CNN Visual Regression Tests
|
|
*
|
|
* Tests for CNN embedding-based visual regression detection.
|
|
* Uses synthetic pixel data rather than real images.
|
|
*/
|
|
|
|
import { describe, it, expect, beforeEach, afterEach} from 'vitest';
|
|
import {
|
|
CNNVisualRegression,
|
|
cosineSimilarity,
|
|
type LabeledPair,
|
|
type RegionOfInterest,
|
|
type EmbeddingOptions,
|
|
} from '../../../../src/domains/visual-accessibility/cnn-visual-regression.js';
|
|
|
|
// ============================================================================
|
|
// Test Helpers
|
|
// ============================================================================
|
|
|
|
/**
|
|
* Create synthetic RGBA pixel data filled with a single color.
|
|
*/
|
|
function createSolidImage(
|
|
width: number,
|
|
height: number,
|
|
r: number,
|
|
g: number,
|
|
b: number,
|
|
a = 255,
|
|
): Uint8Array {
|
|
const data = new Uint8Array(width * height * 4);
|
|
for (let i = 0; i < width * height; i++) {
|
|
data[i * 4] = r;
|
|
data[i * 4 + 1] = g;
|
|
data[i * 4 + 2] = b;
|
|
data[i * 4 + 3] = a;
|
|
}
|
|
return data;
|
|
}
|
|
|
|
/**
|
|
* Create synthetic image with a gradient pattern.
|
|
* Pixel color varies by position to create spatial variation.
|
|
*/
|
|
function createGradientImage(
|
|
width: number,
|
|
height: number,
|
|
): Uint8Array {
|
|
const data = new Uint8Array(width * height * 4);
|
|
for (let y = 0; y < height; y++) {
|
|
for (let x = 0; x < width; x++) {
|
|
const idx = (y * width + x) * 4;
|
|
data[idx] = Math.floor((x / width) * 255); // R: left-right gradient
|
|
data[idx + 1] = Math.floor((y / height) * 255); // G: top-bottom gradient
|
|
data[idx + 2] = 128; // B: constant
|
|
data[idx + 3] = 255; // A: opaque
|
|
}
|
|
}
|
|
return data;
|
|
}
|
|
|
|
/**
|
|
* Create a checkerboard pattern image.
|
|
*/
|
|
function createCheckerboardImage(
|
|
width: number,
|
|
height: number,
|
|
cellSize: number,
|
|
): Uint8Array {
|
|
const data = new Uint8Array(width * height * 4);
|
|
for (let y = 0; y < height; y++) {
|
|
for (let x = 0; x < width; x++) {
|
|
const idx = (y * width + x) * 4;
|
|
const isWhite =
|
|
(Math.floor(x / cellSize) + Math.floor(y / cellSize)) % 2 === 0;
|
|
const val = isWhite ? 255 : 0;
|
|
data[idx] = val;
|
|
data[idx + 1] = val;
|
|
data[idx + 2] = val;
|
|
data[idx + 3] = 255;
|
|
}
|
|
}
|
|
return data;
|
|
}
|
|
|
|
/**
|
|
* Create an image with a colored rectangle on a background.
|
|
*/
|
|
function createImageWithRect(
|
|
width: number,
|
|
height: number,
|
|
bgR: number,
|
|
bgG: number,
|
|
bgB: number,
|
|
rectX: number,
|
|
rectY: number,
|
|
rectW: number,
|
|
rectH: number,
|
|
rectR: number,
|
|
rectG: number,
|
|
rectB: number,
|
|
): Uint8Array {
|
|
const data = createSolidImage(width, height, bgR, bgG, bgB);
|
|
for (let y = rectY; y < Math.min(rectY + rectH, height); y++) {
|
|
for (let x = rectX; x < Math.min(rectX + rectW, width); x++) {
|
|
const idx = (y * width + x) * 4;
|
|
data[idx] = rectR;
|
|
data[idx + 1] = rectG;
|
|
data[idx + 2] = rectB;
|
|
}
|
|
}
|
|
return data;
|
|
}
|
|
|
|
// ============================================================================
|
|
// Tests
|
|
// ============================================================================
|
|
|
|
describe('CNNVisualRegression', () => {
|
|
let cnn: CNNVisualRegression;
|
|
|
|
beforeEach(() => {
|
|
cnn = new CNNVisualRegression({ tryNativeBackend: false });
|
|
});
|
|
|
|
afterEach(() => {
|
|
// Reset state to prevent leaks between tests
|
|
});
|
|
|
|
// --------------------------------------------------------------------------
|
|
// Constructor & Configuration
|
|
// --------------------------------------------------------------------------
|
|
|
|
describe('constructor', () => {
|
|
it('should use default threshold of 0.95', () => {
|
|
const instance = new CNNVisualRegression({ tryNativeBackend: false });
|
|
expect(instance.getDefaultThreshold()).toBe(0.95);
|
|
});
|
|
|
|
it('should accept custom threshold', () => {
|
|
const instance = new CNNVisualRegression({
|
|
defaultThreshold: 0.85,
|
|
tryNativeBackend: false,
|
|
});
|
|
expect(instance.getDefaultThreshold()).toBe(0.85);
|
|
});
|
|
|
|
it('should report correct embedding dimension', () => {
|
|
expect(cnn.getEmbeddingDimension()).toBe(8 * 8 * 3); // 192
|
|
});
|
|
|
|
it('should report correct dimension for custom grid size', () => {
|
|
expect(cnn.getEmbeddingDimension(4)).toBe(4 * 4 * 3); // 48
|
|
});
|
|
|
|
it('should not have native backend when disabled', () => {
|
|
expect(cnn.isNativeBackendLoaded()).toBe(false);
|
|
});
|
|
});
|
|
|
|
// --------------------------------------------------------------------------
|
|
// Embedding Computation
|
|
// --------------------------------------------------------------------------
|
|
|
|
describe('computeEmbedding', () => {
|
|
it('should produce a vector of the expected dimension', () => {
|
|
const image = createSolidImage(64, 64, 128, 64, 32);
|
|
const embedding = cnn.computeEmbedding(image, 64, 64);
|
|
expect(embedding).toBeInstanceOf(Float32Array);
|
|
expect(embedding.length).toBe(192); // 8*8*3
|
|
});
|
|
|
|
it('should produce consistent results for the same input', () => {
|
|
const image = createGradientImage(128, 128);
|
|
const emb1 = cnn.computeEmbedding(image, 128, 128);
|
|
const emb2 = cnn.computeEmbedding(image, 128, 128);
|
|
|
|
expect(emb1.length).toBe(emb2.length);
|
|
for (let i = 0; i < emb1.length; i++) {
|
|
expect(emb1[i]).toBeCloseTo(emb2[i], 10);
|
|
}
|
|
});
|
|
|
|
it('should produce L2-normalized embeddings', () => {
|
|
const image = createGradientImage(64, 64);
|
|
const embedding = cnn.computeEmbedding(image, 64, 64);
|
|
|
|
let norm = 0;
|
|
for (let i = 0; i < embedding.length; i++) {
|
|
norm += embedding[i] * embedding[i];
|
|
}
|
|
norm = Math.sqrt(norm);
|
|
|
|
expect(norm).toBeCloseTo(1.0, 5);
|
|
});
|
|
|
|
it('should work with custom grid size', () => {
|
|
const image = createSolidImage(64, 64, 100, 100, 100);
|
|
const embedding = cnn.computeEmbedding(image, 64, 64, { gridSize: 4 });
|
|
expect(embedding.length).toBe(4 * 4 * 3); // 48
|
|
});
|
|
|
|
it('should throw on invalid dimensions', () => {
|
|
const image = new Uint8Array(100);
|
|
expect(() => cnn.computeEmbedding(image, 0, 10)).toThrow(
|
|
'Invalid image dimensions',
|
|
);
|
|
expect(() => cnn.computeEmbedding(image, 10, -1)).toThrow(
|
|
'Invalid image dimensions',
|
|
);
|
|
});
|
|
|
|
it('should throw if pixel data is too small', () => {
|
|
const image = new Uint8Array(10); // way too small for 64x64
|
|
expect(() => cnn.computeEmbedding(image, 64, 64)).toThrow(
|
|
'Image data too small',
|
|
);
|
|
});
|
|
|
|
it('should handle non-square images', () => {
|
|
const image = createGradientImage(200, 100);
|
|
const embedding = cnn.computeEmbedding(image, 200, 100);
|
|
expect(embedding.length).toBe(192);
|
|
});
|
|
|
|
it('should handle small images (smaller than grid)', () => {
|
|
const image = createSolidImage(4, 4, 200, 100, 50);
|
|
const embedding = cnn.computeEmbedding(image, 4, 4);
|
|
expect(embedding.length).toBe(192);
|
|
});
|
|
|
|
it('should handle large images', () => {
|
|
const image = createSolidImage(1920, 1080, 128, 128, 128);
|
|
const embedding = cnn.computeEmbedding(image, 1920, 1080);
|
|
expect(embedding.length).toBe(192);
|
|
});
|
|
});
|
|
|
|
// --------------------------------------------------------------------------
|
|
// Similarity: Similar Images
|
|
// --------------------------------------------------------------------------
|
|
|
|
describe('similar images have high cosine similarity', () => {
|
|
it('should give similarity ~1.0 for identical images', () => {
|
|
const image = createGradientImage(128, 128);
|
|
const emb = cnn.computeEmbedding(image, 128, 128);
|
|
const similarity = cosineSimilarity(emb, emb);
|
|
expect(similarity).toBeCloseTo(1.0, 5);
|
|
});
|
|
|
|
it('should give high similarity for slightly modified images', () => {
|
|
const imageA = createSolidImage(64, 64, 128, 64, 32);
|
|
const imageB = createSolidImage(64, 64, 130, 66, 34); // slight variation
|
|
|
|
const embA = cnn.computeEmbedding(imageA, 64, 64);
|
|
const embB = cnn.computeEmbedding(imageB, 64, 64);
|
|
|
|
const similarity = cosineSimilarity(embA, embB);
|
|
expect(similarity).toBeGreaterThan(0.99);
|
|
});
|
|
|
|
it('should give high similarity for images with minor local changes', () => {
|
|
const width = 128;
|
|
const height = 128;
|
|
const imageA = createSolidImage(width, height, 100, 100, 100);
|
|
// Add a tiny 2x2 pixel change in one corner
|
|
const imageB = new Uint8Array(imageA);
|
|
for (let y = 0; y < 2; y++) {
|
|
for (let x = 0; x < 2; x++) {
|
|
const idx = (y * width + x) * 4;
|
|
imageB[idx] = 255;
|
|
imageB[idx + 1] = 0;
|
|
imageB[idx + 2] = 0;
|
|
}
|
|
}
|
|
|
|
const embA = cnn.computeEmbedding(imageA, width, height);
|
|
const embB = cnn.computeEmbedding(imageB, width, height);
|
|
|
|
const similarity = cosineSimilarity(embA, embB);
|
|
expect(similarity).toBeGreaterThan(0.95);
|
|
});
|
|
});
|
|
|
|
// --------------------------------------------------------------------------
|
|
// Similarity: Different Images
|
|
// --------------------------------------------------------------------------
|
|
|
|
describe('different images have low cosine similarity', () => {
|
|
it('should give low similarity for opposite colors', () => {
|
|
const black = createSolidImage(64, 64, 0, 0, 0);
|
|
const white = createSolidImage(64, 64, 255, 255, 255);
|
|
|
|
const embBlack = cnn.computeEmbedding(black, 64, 64);
|
|
const embWhite = cnn.computeEmbedding(white, 64, 64);
|
|
|
|
// Both uniform colors produce the same direction when normalized
|
|
// because the spatial pattern is the same. We need structurally
|
|
// different images to get low similarity.
|
|
});
|
|
|
|
it('should give lower similarity for structurally different images', () => {
|
|
const gradient = createGradientImage(128, 128);
|
|
const checker = createCheckerboardImage(128, 128, 16);
|
|
|
|
const embGradient = cnn.computeEmbedding(gradient, 128, 128);
|
|
const embChecker = cnn.computeEmbedding(checker, 128, 128);
|
|
|
|
const similarity = cosineSimilarity(embGradient, embChecker);
|
|
expect(similarity).toBeLessThan(0.95);
|
|
});
|
|
|
|
it('should distinguish images with different content regions', () => {
|
|
const width = 128;
|
|
const height = 128;
|
|
|
|
// Red rect on white background in top-left
|
|
const imgA = createImageWithRect(
|
|
width, height, 255, 255, 255, 0, 0, 32, 32, 255, 0, 0,
|
|
);
|
|
// Red rect on white background in bottom-right
|
|
const imgB = createImageWithRect(
|
|
width, height, 255, 255, 255, 96, 96, 32, 32, 255, 0, 0,
|
|
);
|
|
|
|
const embA = cnn.computeEmbedding(imgA, width, height);
|
|
const embB = cnn.computeEmbedding(imgB, width, height);
|
|
|
|
const similarity = cosineSimilarity(embA, embB);
|
|
// Same overall colors but different spatial layout
|
|
expect(similarity).toBeLessThan(1.0);
|
|
});
|
|
});
|
|
|
|
// --------------------------------------------------------------------------
|
|
// Regression Detection
|
|
// --------------------------------------------------------------------------
|
|
|
|
describe('isRegression', () => {
|
|
it('should return false for identical images', () => {
|
|
const image = createGradientImage(64, 64);
|
|
const emb = cnn.computeEmbedding(image, 64, 64);
|
|
expect(cnn.isRegression(emb, emb)).toBe(false);
|
|
});
|
|
|
|
it('should return false for similar images above threshold', () => {
|
|
const imgA = createSolidImage(64, 64, 100, 100, 100);
|
|
const imgB = createSolidImage(64, 64, 102, 101, 99);
|
|
|
|
const embA = cnn.computeEmbedding(imgA, 64, 64);
|
|
const embB = cnn.computeEmbedding(imgB, 64, 64);
|
|
|
|
expect(cnn.isRegression(embA, embB)).toBe(false);
|
|
});
|
|
|
|
it('should return true for different images below threshold', () => {
|
|
const gradient = createGradientImage(128, 128);
|
|
const checker = createCheckerboardImage(128, 128, 16);
|
|
|
|
const embGradient = cnn.computeEmbedding(gradient, 128, 128);
|
|
const embChecker = cnn.computeEmbedding(checker, 128, 128);
|
|
|
|
const similarity = cosineSimilarity(embGradient, embChecker);
|
|
// Use the actual similarity to set a threshold slightly above it
|
|
expect(cnn.isRegression(embGradient, embChecker, similarity + 0.01)).toBe(
|
|
true,
|
|
);
|
|
});
|
|
|
|
it('should respect custom threshold', () => {
|
|
const imgA = createSolidImage(64, 64, 100, 100, 100);
|
|
const imgB = createSolidImage(64, 64, 130, 100, 100);
|
|
|
|
const embA = cnn.computeEmbedding(imgA, 64, 64);
|
|
const embB = cnn.computeEmbedding(imgB, 64, 64);
|
|
|
|
// With a very low threshold, nothing is a regression
|
|
expect(cnn.isRegression(embA, embB, 0.01)).toBe(false);
|
|
// With a perfect threshold, even small diffs are regressions
|
|
expect(cnn.isRegression(embA, embB, 1.0)).toBe(true);
|
|
});
|
|
});
|
|
|
|
// --------------------------------------------------------------------------
|
|
// Compare
|
|
// --------------------------------------------------------------------------
|
|
|
|
describe('compare', () => {
|
|
it('should return full comparison result', () => {
|
|
const imgA = createGradientImage(64, 64);
|
|
const imgB = createGradientImage(64, 64);
|
|
|
|
const embA = cnn.computeEmbedding(imgA, 64, 64);
|
|
const embB = cnn.computeEmbedding(imgB, 64, 64);
|
|
|
|
const result = cnn.compare(embA, embB);
|
|
|
|
expect(result.similarity).toBeCloseTo(1.0, 5);
|
|
expect(result.isMatch).toBe(true);
|
|
expect(result.threshold).toBe(0.95);
|
|
});
|
|
|
|
it('should use provided threshold', () => {
|
|
const imgA = createGradientImage(64, 64);
|
|
const embA = cnn.computeEmbedding(imgA, 64, 64);
|
|
|
|
const result = cnn.compare(embA, embA, 0.5);
|
|
expect(result.threshold).toBe(0.5);
|
|
expect(result.isMatch).toBe(true);
|
|
});
|
|
});
|
|
|
|
// --------------------------------------------------------------------------
|
|
// Compare with Regions
|
|
// --------------------------------------------------------------------------
|
|
|
|
describe('compareWithRegions', () => {
|
|
it('should return per-region similarity scores', () => {
|
|
const width = 128;
|
|
const height = 128;
|
|
const imgA = createGradientImage(width, height);
|
|
const imgB = createGradientImage(width, height);
|
|
|
|
const embA = cnn.computeEmbedding(imgA, width, height);
|
|
const embB = cnn.computeEmbedding(imgB, width, height);
|
|
|
|
const regions: RegionOfInterest[] = [
|
|
{ x: 0, y: 0, width: 64, height: 64 },
|
|
{ x: 64, y: 64, width: 64, height: 64 },
|
|
];
|
|
|
|
const result = cnn.compareWithRegions(
|
|
embA, embB, regions, width, height,
|
|
);
|
|
|
|
expect(result.regionSimilarities).toBeDefined();
|
|
expect(result.regionSimilarities!.length).toBe(2);
|
|
expect(result.regionSimilarities![0].similarity).toBeCloseTo(1.0, 5);
|
|
expect(result.regionSimilarities![1].similarity).toBeCloseTo(1.0, 5);
|
|
});
|
|
|
|
it('should detect regional differences', () => {
|
|
const width = 128;
|
|
const height = 128;
|
|
|
|
const imgA = createSolidImage(width, height, 100, 100, 100);
|
|
const imgB = new Uint8Array(imgA);
|
|
|
|
// Modify top-left quadrant significantly
|
|
for (let y = 0; y < 64; y++) {
|
|
for (let x = 0; x < 64; x++) {
|
|
const idx = (y * width + x) * 4;
|
|
imgB[idx] = 255; // R
|
|
imgB[idx + 1] = 0; // G
|
|
imgB[idx + 2] = 0; // B
|
|
}
|
|
}
|
|
|
|
const embA = cnn.computeEmbedding(imgA, width, height);
|
|
const embB = cnn.computeEmbedding(imgB, width, height);
|
|
|
|
const regions: RegionOfInterest[] = [
|
|
{ x: 0, y: 0, width: 64, height: 64 }, // modified region
|
|
{ x: 64, y: 64, width: 64, height: 64 }, // unmodified region
|
|
];
|
|
|
|
const result = cnn.compareWithRegions(
|
|
embA, embB, regions, width, height,
|
|
);
|
|
|
|
expect(result.regionSimilarities).toBeDefined();
|
|
// Modified region should have lower similarity
|
|
const modifiedSim = result.regionSimilarities![0].similarity;
|
|
const unmodifiedSim = result.regionSimilarities![1].similarity;
|
|
|
|
expect(unmodifiedSim).toBeGreaterThan(modifiedSim);
|
|
});
|
|
});
|
|
|
|
// --------------------------------------------------------------------------
|
|
// Contrastive Learning
|
|
// --------------------------------------------------------------------------
|
|
|
|
describe('learnThreshold', () => {
|
|
it('should learn a threshold that separates positive and negative pairs', () => {
|
|
const width = 64;
|
|
const height = 64;
|
|
|
|
// Create matching pairs (similar images)
|
|
const solidA = createSolidImage(width, height, 100, 100, 100);
|
|
const solidB = createSolidImage(width, height, 105, 102, 98);
|
|
const embSolidA = cnn.computeEmbedding(solidA, width, height);
|
|
const embSolidB = cnn.computeEmbedding(solidB, width, height);
|
|
|
|
// Create non-matching pairs (different images)
|
|
const gradient = createGradientImage(width, height);
|
|
const checker = createCheckerboardImage(width, height, 8);
|
|
const embGradient = cnn.computeEmbedding(gradient, width, height);
|
|
const embChecker = cnn.computeEmbedding(checker, width, height);
|
|
|
|
const examples: LabeledPair[] = [
|
|
{ embedA: embSolidA, embedB: embSolidB, isMatch: true },
|
|
{ embedA: embGradient, embedB: embChecker, isMatch: false },
|
|
];
|
|
|
|
const threshold = cnn.learnThreshold(examples);
|
|
|
|
// Threshold should correctly classify the pairs
|
|
const posSim = cosineSimilarity(embSolidA, embSolidB);
|
|
const negSim = cosineSimilarity(embGradient, embChecker);
|
|
|
|
expect(threshold).toBeGreaterThan(negSim);
|
|
expect(threshold).toBeLessThanOrEqual(posSim);
|
|
});
|
|
|
|
it('should throw with no examples', () => {
|
|
expect(() => cnn.learnThreshold([])).toThrow(
|
|
'At least one labeled pair is required',
|
|
);
|
|
});
|
|
|
|
it('should throw with only positive examples', () => {
|
|
const img = createSolidImage(64, 64, 128, 128, 128);
|
|
const emb = cnn.computeEmbedding(img, 64, 64);
|
|
const examples: LabeledPair[] = [
|
|
{ embedA: emb, embedB: emb, isMatch: true },
|
|
];
|
|
|
|
expect(() => cnn.learnThreshold(examples)).toThrow(
|
|
'both matching and non-matching',
|
|
);
|
|
});
|
|
|
|
it('should throw with only negative examples', () => {
|
|
const img = createSolidImage(64, 64, 128, 128, 128);
|
|
const emb = cnn.computeEmbedding(img, 64, 64);
|
|
const examples: LabeledPair[] = [
|
|
{ embedA: emb, embedB: emb, isMatch: false },
|
|
];
|
|
|
|
expect(() => cnn.learnThreshold(examples)).toThrow(
|
|
'both matching and non-matching',
|
|
);
|
|
});
|
|
|
|
it('should improve classification with more examples', () => {
|
|
const width = 64;
|
|
const height = 64;
|
|
|
|
// Multiple positive pairs
|
|
const positivePairs: LabeledPair[] = [];
|
|
for (let i = 0; i < 5; i++) {
|
|
const r = 100 + i * 2;
|
|
const imgA = createSolidImage(width, height, r, 100, 100);
|
|
const imgB = createSolidImage(width, height, r + 1, 101, 99);
|
|
positivePairs.push({
|
|
embedA: cnn.computeEmbedding(imgA, width, height),
|
|
embedB: cnn.computeEmbedding(imgB, width, height),
|
|
isMatch: true,
|
|
});
|
|
}
|
|
|
|
// Multiple negative pairs
|
|
const negativePairs: LabeledPair[] = [];
|
|
const negImages = [
|
|
createGradientImage(width, height),
|
|
createCheckerboardImage(width, height, 8),
|
|
createSolidImage(width, height, 255, 0, 0),
|
|
createSolidImage(width, height, 0, 255, 0),
|
|
createSolidImage(width, height, 0, 0, 255),
|
|
];
|
|
|
|
for (let i = 0; i < negImages.length - 1; i++) {
|
|
negativePairs.push({
|
|
embedA: cnn.computeEmbedding(negImages[i], width, height),
|
|
embedB: cnn.computeEmbedding(negImages[i + 1], width, height),
|
|
isMatch: false,
|
|
});
|
|
}
|
|
|
|
const allExamples = [...positivePairs, ...negativePairs];
|
|
const threshold = cnn.learnThreshold(allExamples);
|
|
|
|
// Verify the learned threshold correctly classifies most pairs
|
|
let correct = 0;
|
|
for (const ex of allExamples) {
|
|
const sim = cosineSimilarity(ex.embedA, ex.embedB);
|
|
const predicted = sim >= threshold;
|
|
if (predicted === ex.isMatch) correct++;
|
|
}
|
|
|
|
const accuracy = correct / allExamples.length;
|
|
expect(accuracy).toBeGreaterThanOrEqual(0.8);
|
|
});
|
|
});
|
|
|
|
// --------------------------------------------------------------------------
|
|
// Regions of Interest Masking
|
|
// --------------------------------------------------------------------------
|
|
|
|
describe('regions of interest masking', () => {
|
|
it('should produce different embeddings when masking a region', () => {
|
|
const width = 128;
|
|
const height = 128;
|
|
const image = createGradientImage(width, height);
|
|
|
|
const embNoMask = cnn.computeEmbedding(image, width, height);
|
|
const embMasked = cnn.computeEmbedding(image, width, height, {
|
|
maskRegions: [{ x: 0, y: 0, width: 64, height: 64 }],
|
|
});
|
|
|
|
// The embeddings should differ because we zeroed out part of the image
|
|
const similarity = cosineSimilarity(embNoMask, embMasked);
|
|
expect(similarity).toBeLessThan(1.0);
|
|
});
|
|
|
|
it('should make masked regions identical between different images', () => {
|
|
const width = 128;
|
|
const height = 128;
|
|
|
|
// Two images that differ only in the top-left quadrant
|
|
const imgA = createSolidImage(width, height, 100, 100, 100);
|
|
const imgB = new Uint8Array(imgA);
|
|
for (let y = 0; y < 64; y++) {
|
|
for (let x = 0; x < 64; x++) {
|
|
const idx = (y * width + x) * 4;
|
|
imgB[idx] = 200;
|
|
imgB[idx + 1] = 50;
|
|
imgB[idx + 2] = 50;
|
|
}
|
|
}
|
|
|
|
// Without masking, they should differ
|
|
const embANoMask = cnn.computeEmbedding(imgA, width, height);
|
|
const embBNoMask = cnn.computeEmbedding(imgB, width, height);
|
|
const simNoMask = cosineSimilarity(embANoMask, embBNoMask);
|
|
|
|
// With masking the differing region, similarity should increase
|
|
const maskOptions: EmbeddingOptions = {
|
|
maskRegions: [{ x: 0, y: 0, width: 64, height: 64 }],
|
|
};
|
|
const embAMasked = cnn.computeEmbedding(imgA, width, height, maskOptions);
|
|
const embBMasked = cnn.computeEmbedding(imgB, width, height, maskOptions);
|
|
const simMasked = cosineSimilarity(embAMasked, embBMasked);
|
|
|
|
expect(simMasked).toBeGreaterThan(simNoMask);
|
|
});
|
|
|
|
it('should handle mask regions that extend beyond image bounds', () => {
|
|
const width = 64;
|
|
const height = 64;
|
|
const image = createGradientImage(width, height);
|
|
|
|
// Mask extends beyond the image
|
|
const embedding = cnn.computeEmbedding(image, width, height, {
|
|
maskRegions: [{ x: 50, y: 50, width: 100, height: 100 }],
|
|
});
|
|
|
|
expect(embedding.length).toBe(192);
|
|
});
|
|
|
|
it('should handle multiple mask regions', () => {
|
|
const width = 128;
|
|
const height = 128;
|
|
const image = createGradientImage(width, height);
|
|
|
|
const embedding = cnn.computeEmbedding(image, width, height, {
|
|
maskRegions: [
|
|
{ x: 0, y: 0, width: 32, height: 32 },
|
|
{ x: 96, y: 96, width: 32, height: 32 },
|
|
],
|
|
});
|
|
|
|
expect(embedding.length).toBe(192);
|
|
});
|
|
});
|
|
|
|
// --------------------------------------------------------------------------
|
|
// Various Image Sizes
|
|
// --------------------------------------------------------------------------
|
|
|
|
describe('various image sizes', () => {
|
|
const sizes = [
|
|
{ w: 1, h: 1 },
|
|
{ w: 2, h: 2 },
|
|
{ w: 8, h: 8 },
|
|
{ w: 16, h: 16 },
|
|
{ w: 32, h: 32 },
|
|
{ w: 64, h: 48 },
|
|
{ w: 100, h: 200 },
|
|
{ w: 320, h: 240 },
|
|
{ w: 640, h: 480 },
|
|
];
|
|
|
|
for (const { w, h } of sizes) {
|
|
it(`should handle ${w}x${h} images`, () => {
|
|
const image = createSolidImage(w, h, 128, 64, 32);
|
|
const embedding = cnn.computeEmbedding(image, w, h);
|
|
|
|
expect(embedding.length).toBe(192);
|
|
|
|
// L2 norm should be ~1
|
|
let norm = 0;
|
|
for (let i = 0; i < embedding.length; i++) {
|
|
norm += embedding[i] * embedding[i];
|
|
}
|
|
expect(Math.sqrt(norm)).toBeCloseTo(1.0, 4);
|
|
});
|
|
}
|
|
});
|
|
|
|
// --------------------------------------------------------------------------
|
|
// Cosine Similarity Utility
|
|
// --------------------------------------------------------------------------
|
|
|
|
describe('cosineSimilarity', () => {
|
|
it('should return 1.0 for identical vectors', () => {
|
|
const vec = new Float32Array([1, 2, 3, 4, 5]);
|
|
expect(cosineSimilarity(vec, vec)).toBeCloseTo(1.0, 10);
|
|
});
|
|
|
|
it('should return 0 for orthogonal vectors', () => {
|
|
const a = new Float32Array([1, 0, 0]);
|
|
const b = new Float32Array([0, 1, 0]);
|
|
expect(cosineSimilarity(a, b)).toBeCloseTo(0.0, 10);
|
|
});
|
|
|
|
it('should return -1 for opposite vectors', () => {
|
|
const a = new Float32Array([1, 2, 3]);
|
|
const b = new Float32Array([-1, -2, -3]);
|
|
expect(cosineSimilarity(a, b)).toBeCloseTo(-1.0, 10);
|
|
});
|
|
|
|
it('should return 0 for zero vectors', () => {
|
|
const zero = new Float32Array([0, 0, 0]);
|
|
const nonZero = new Float32Array([1, 2, 3]);
|
|
expect(cosineSimilarity(zero, nonZero)).toBe(0);
|
|
});
|
|
|
|
it('should throw for mismatched lengths', () => {
|
|
const a = new Float32Array([1, 2]);
|
|
const b = new Float32Array([1, 2, 3]);
|
|
expect(() => cosineSimilarity(a, b)).toThrow('Vector length mismatch');
|
|
});
|
|
});
|
|
|
|
// --------------------------------------------------------------------------
|
|
// Edge Detection Mode
|
|
// --------------------------------------------------------------------------
|
|
|
|
describe('Edge Detection Mode', () => {
|
|
it('should double the embedding dimension when edge detection is enabled via config', () => {
|
|
const edgeCnn = new CNNVisualRegression({
|
|
tryNativeBackend: false,
|
|
useEdgeDetection: true,
|
|
});
|
|
expect(edgeCnn.getEmbeddingDimension()).toBe(8 * 8 * 3 * 2); // 384
|
|
});
|
|
|
|
it('should double the embedding dimension when edge detection is enabled via options', () => {
|
|
const image = createSolidImage(64, 64, 128, 64, 32);
|
|
const embDefault = cnn.computeEmbedding(image, 64, 64);
|
|
const embEdge = cnn.computeEmbedding(image, 64, 64, { useEdgeDetection: true });
|
|
|
|
expect(embDefault.length).toBe(192);
|
|
expect(embEdge.length).toBe(384);
|
|
});
|
|
|
|
it('should catch orientation changes that spatial-only pooling misses', () => {
|
|
const width = 64;
|
|
const height = 64;
|
|
|
|
// Create an image with vertical stripes (strong vertical edges).
|
|
// Each grid cell (8x8 pixels) has 4 black columns and 4 white columns,
|
|
// so the average per cell is ~128 for all cells. Spatial pooling treats
|
|
// vertical and horizontal stripes identically because the per-cell
|
|
// average is the same.
|
|
const verticalStripes = new Uint8Array(width * height * 4);
|
|
for (let y = 0; y < height; y++) {
|
|
for (let x = 0; x < width; x++) {
|
|
const idx = (y * width + x) * 4;
|
|
const val = (x % 8) < 4 ? 0 : 255;
|
|
verticalStripes[idx] = val;
|
|
verticalStripes[idx + 1] = val;
|
|
verticalStripes[idx + 2] = val;
|
|
verticalStripes[idx + 3] = 255;
|
|
}
|
|
}
|
|
|
|
// Create an image with horizontal stripes (strong horizontal edges).
|
|
// Same per-cell average as vertical stripes, but edges run horizontally.
|
|
const horizontalStripes = new Uint8Array(width * height * 4);
|
|
for (let y = 0; y < height; y++) {
|
|
for (let x = 0; x < width; x++) {
|
|
const idx = (y * width + x) * 4;
|
|
const val = (y % 8) < 4 ? 0 : 255;
|
|
horizontalStripes[idx] = val;
|
|
horizontalStripes[idx + 1] = val;
|
|
horizontalStripes[idx + 2] = val;
|
|
horizontalStripes[idx + 3] = 255;
|
|
}
|
|
}
|
|
|
|
// Without edge detection, these should have very high similarity
|
|
// because each grid cell has the same average color
|
|
const embVNoEdge = cnn.computeEmbedding(verticalStripes, width, height);
|
|
const embHNoEdge = cnn.computeEmbedding(horizontalStripes, width, height);
|
|
const simNoEdge = cosineSimilarity(embVNoEdge, embHNoEdge);
|
|
expect(simNoEdge).toBeGreaterThan(0.99); // nearly identical spatial embeddings
|
|
|
|
// With edge detection, the Sobel edge maps differ:
|
|
// vertical stripes produce vertical edges, horizontal stripes produce
|
|
// horizontal edges. The edge component of the embedding should separate them.
|
|
const edgeCnn = new CNNVisualRegression({
|
|
tryNativeBackend: false,
|
|
useEdgeDetection: true,
|
|
});
|
|
const embVEdge = edgeCnn.computeEmbedding(verticalStripes, width, height);
|
|
const embHEdge = edgeCnn.computeEmbedding(horizontalStripes, width, height);
|
|
const simEdge = cosineSimilarity(embVEdge, embHEdge);
|
|
|
|
// Edge detection should produce lower similarity for rotated stripe patterns
|
|
expect(simEdge).toBeLessThan(simNoEdge);
|
|
});
|
|
|
|
it('should still give high similarity for identical images with edge detection on', () => {
|
|
const edgeCnn = new CNNVisualRegression({
|
|
tryNativeBackend: false,
|
|
useEdgeDetection: true,
|
|
});
|
|
const image = createGradientImage(128, 128);
|
|
const emb1 = edgeCnn.computeEmbedding(image, 128, 128);
|
|
const emb2 = edgeCnn.computeEmbedding(image, 128, 128);
|
|
|
|
const similarity = cosineSimilarity(emb1, emb2);
|
|
expect(similarity).toBeCloseTo(1.0, 5);
|
|
});
|
|
|
|
it('should produce L2-normalized embeddings with edge detection', () => {
|
|
const edgeCnn = new CNNVisualRegression({
|
|
tryNativeBackend: false,
|
|
useEdgeDetection: true,
|
|
});
|
|
const image = createGradientImage(64, 64);
|
|
const embedding = edgeCnn.computeEmbedding(image, 64, 64);
|
|
|
|
let norm = 0;
|
|
for (let i = 0; i < embedding.length; i++) {
|
|
norm += embedding[i] * embedding[i];
|
|
}
|
|
expect(Math.sqrt(norm)).toBeCloseTo(1.0, 5);
|
|
});
|
|
|
|
it('should detect strong horizontal edges in a test image', () => {
|
|
const edgeCnn = new CNNVisualRegression({
|
|
tryNativeBackend: false,
|
|
useEdgeDetection: true,
|
|
});
|
|
|
|
const width = 64;
|
|
const height = 64;
|
|
|
|
// Create image with strong horizontal edge: top half white, bottom half black
|
|
const horizEdgeImage = new Uint8Array(width * height * 4);
|
|
for (let y = 0; y < height; y++) {
|
|
for (let x = 0; x < width; x++) {
|
|
const idx = (y * width + x) * 4;
|
|
const val = y < height / 2 ? 255 : 0;
|
|
horizEdgeImage[idx] = val;
|
|
horizEdgeImage[idx + 1] = val;
|
|
horizEdgeImage[idx + 2] = val;
|
|
horizEdgeImage[idx + 3] = 255;
|
|
}
|
|
}
|
|
|
|
// Create a uniform image (no edges)
|
|
const uniformImage = createSolidImage(width, height, 128, 128, 128);
|
|
|
|
const embEdge = edgeCnn.computeEmbedding(horizEdgeImage, width, height);
|
|
const embUniform = edgeCnn.computeEmbedding(uniformImage, width, height);
|
|
|
|
// The edge image has strong Sobel response; the uniform image has none.
|
|
// They should be quite dissimilar with edge detection on.
|
|
const similarity = cosineSimilarity(embEdge, embUniform);
|
|
expect(similarity).toBeLessThan(0.95);
|
|
});
|
|
|
|
it('should apply Sobel edge detection correctly on a known pattern', () => {
|
|
// Directly test the Sobel method: a uniform image should have zero edges
|
|
const width = 16;
|
|
const height = 16;
|
|
const uniform = createSolidImage(width, height, 128, 128, 128);
|
|
|
|
const edgeMap = cnn.applySobelEdgeDetection(uniform, width, height);
|
|
expect(edgeMap.length).toBe(width * height * 4);
|
|
|
|
// All edge magnitudes should be zero (or very close) for a uniform image
|
|
for (let i = 0; i < width * height; i++) {
|
|
expect(edgeMap[i * 4]).toBe(0); // R channel = edge magnitude
|
|
}
|
|
});
|
|
|
|
it('should produce non-zero edge magnitudes at a sharp boundary', () => {
|
|
const width = 16;
|
|
const height = 16;
|
|
|
|
// Left half black, right half white
|
|
const image = new Uint8Array(width * height * 4);
|
|
for (let y = 0; y < height; y++) {
|
|
for (let x = 0; x < width; x++) {
|
|
const idx = (y * width + x) * 4;
|
|
const val = x < width / 2 ? 0 : 255;
|
|
image[idx] = val;
|
|
image[idx + 1] = val;
|
|
image[idx + 2] = val;
|
|
image[idx + 3] = 255;
|
|
}
|
|
}
|
|
|
|
const edgeMap = cnn.applySobelEdgeDetection(image, width, height);
|
|
|
|
// At the boundary (x = width/2 - 1 or x = width/2), there should be
|
|
// high edge magnitude
|
|
const boundaryX = width / 2;
|
|
const midY = height / 2;
|
|
const boundaryIdx = (midY * width + boundaryX) * 4;
|
|
expect(edgeMap[boundaryIdx]).toBeGreaterThan(0);
|
|
});
|
|
});
|
|
|
|
// --------------------------------------------------------------------------
|
|
// Async Embedding (falls back to sync)
|
|
// --------------------------------------------------------------------------
|
|
|
|
describe('computeEmbeddingAsync', () => {
|
|
it('should produce the same result as sync when native is unavailable', async () => {
|
|
const image = createGradientImage(64, 64);
|
|
const syncEmb = cnn.computeEmbedding(image, 64, 64);
|
|
const asyncEmb = await cnn.computeEmbeddingAsync(image, 64, 64);
|
|
|
|
expect(asyncEmb.length).toBe(syncEmb.length);
|
|
for (let i = 0; i < syncEmb.length; i++) {
|
|
expect(asyncEmb[i]).toBeCloseTo(syncEmb[i], 10);
|
|
}
|
|
});
|
|
});
|
|
});
|