Files
Jeffrey Emanuel 8c448bbd8e test: adapt the suite to fsqlite 0.3.x + frankensearch-head behavior (v0.6.25 gate)
- goldens: regenerate for additive upstream schema (frankensearch
  shard_count + search budget object; FAD antigravity/goose agent CSS;
  error-envelope kind→exit-code map incl. the new --track b path); pin
  budget.elapsed_ms to 0 in the search regression golden and scrub it in
  the test so wall-clock can't flake the comparison.
- storage-failure fixtures: stale-WAL/SHM and busy-lock classes now heal
  under 0.3.x (GH#333 recovery retries, GH#334 sidecar repair) — expect
  FailOpenTruthful and accept the healed storage states.
- stale-threshold policy: health's default deliberately collapsed to
  DEFAULT_STALE_THRESHOLD_SECS (1800); pin equality instead of divergence.
- fixture batches (upgrade/migration, upgrade/compatibility, legacy-v13,
  doctor-fts, stress): BEGIN/COMMIT-atomic with bounded busy retries so
  transient BusySnapshot aborts re-run wholesale; stress test routes
  begin/read conflicts through its own retry ledger.
- duplicate-ALTER fixtures tolerate the correct 0.3.x duplicate-column
  rejection; bakeoff empty-corpus expects the machine-readable
  "no-data: empty-corpus"; dyoj4 accepts the privacy-preserving
  archive-format rejection; FSVI fixture uses the canonical
  "little-endian" spelling; models fixture gains model.safetensors so
  --from-file reaches its SHA256 gate; permission tests probe-skip under
  root; api_coverage_pct asserts the metric-integrity number-or-
  null+status contract.
- receipted ignores (bead no0uw): six doctor_e2e_runner candidate
  scenarios + the status/doctor storage-state parity check — 0.3.x
  detects the corrupt-fixture class at open (0.2.x false-negatived it),
  which defers archive-wide collectors and leaves mirror-grounded
  candidate authority unreachable pending the redesign.
- docs: align the semantic-planning historical markers with the
  docs_safety_invariants exact strings.

Full suite on the dedicated builder: 247 binaries green, 0 failures
outside the receipted ignores and three load-sensitive SLO tests
adjudicated serially.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-15 05:48:04 -04:00

618 lines
21 KiB
Rust

//! Migration tests for database schema upgrades.
//!
//! These tests verify that:
//! - Database migrations work correctly
//! - Data is preserved during migration
//! - Failed migrations are handled gracefully
//! - Backup is created before destructive operations
use coding_agent_search::franken_sync::Connection as FrankenConnection;
use coding_agent_search::franken_sync::compat::{ConnectionExt, RowExt};
use coding_agent_search::storage::sqlite::{CURRENT_SCHEMA_VERSION, MigrationError, SqliteStorage};
use std::fs;
use std::path::Path;
use tempfile::TempDir;
fn open_fixture_db(path: &Path) -> FrankenConnection {
let path = path.to_string_lossy();
FrankenConnection::open(path.as_ref()).expect("open frankensqlite fixture database")
}
/// Run a fixture batch atomically (BEGIN/COMMIT) with a bounded retry: fsqlite
/// 0.3.x can abort the whole batch with a transient `BusySnapshot` while a
/// previously-dropped session's epoch unwinds (tempdir inode reuse), and the
/// atomic wrapper makes a wholesale re-run safe.
fn run_fixture_batch(conn: &FrankenConnection, sql: &str) {
let wrapped = format!("BEGIN;\n{sql}\nCOMMIT;");
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
let mut delay = std::time::Duration::from_millis(10);
loop {
match conn.execute_batch(&wrapped) {
Ok(()) => return,
Err(err)
if std::time::Instant::now() < deadline
&& err.to_string().to_ascii_lowercase().contains("busy") =>
{
std::thread::sleep(delay);
delay = (delay * 2).min(std::time::Duration::from_millis(200));
}
Err(err) => panic!("fixture batch failed: {err}"),
}
}
}
// =============================================================================
// Migration Flow Tests
// =============================================================================
/// Test that migration creates backup before modifying database.
#[test]
fn test_migration_creates_backup() {
let dir = TempDir::new().unwrap();
let db_path = dir.path().join("to_migrate.db");
// Create database with old schema
{
let conn = open_fixture_db(&db_path);
conn.execute_batch(
r#"
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT);
INSERT INTO meta (key, value) VALUES ('schema_version', '5');
CREATE TABLE agents (
id INTEGER PRIMARY KEY,
slug TEXT NOT NULL UNIQUE,
name TEXT,
version TEXT,
kind TEXT NOT NULL,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE workspaces (
id INTEGER PRIMARY KEY,
path TEXT NOT NULL UNIQUE,
display_name TEXT
);
CREATE TABLE conversations (
id INTEGER PRIMARY KEY,
agent_id INTEGER NOT NULL,
workspace_id INTEGER,
external_id TEXT UNIQUE,
title TEXT,
source_path TEXT NOT NULL,
started_at INTEGER,
ended_at INTEGER,
metadata_json TEXT
);
CREATE TABLE messages (
id INTEGER PRIMARY KEY,
conversation_id INTEGER NOT NULL,
idx INTEGER NOT NULL,
role TEXT NOT NULL,
author TEXT,
content TEXT NOT NULL,
created_at INTEGER,
extra_json TEXT
);
CREATE TABLE sources (
id INTEGER PRIMARY KEY,
kind TEXT NOT NULL,
path TEXT NOT NULL UNIQUE,
updated_at INTEGER NOT NULL
);
INSERT INTO sources (id, kind, path, updated_at) VALUES (0, 'local', 'default', 0);
"#,
)
.unwrap();
}
let _original_size = fs::metadata(&db_path).unwrap().len();
// Trigger migration
let result = SqliteStorage::open_or_rebuild(&db_path);
if result.is_ok() {
// Check for backup file (pattern: original_name.backup.timestamp)
let backup_exists = fs::read_dir(dir.path())
.unwrap()
.filter_map(|e| e.ok())
.any(|e| {
let name = e.file_name().to_string_lossy().to_string();
name.contains("backup") || name.contains(".bak")
});
// Note: Backup might not always be created if migration is in-place
// This test documents expected behavior
if backup_exists {
println!("Backup file was created");
} else {
println!("In-place migration (no backup created)");
}
}
}
/// Test migration preserves data.
#[test]
fn test_migration_preserves_data() {
let dir = TempDir::new().unwrap();
let db_path = dir.path().join("preserve.db");
// Create database with data
{
let conn = open_fixture_db(&db_path);
run_fixture_batch(
&conn,
&format!(
r#"
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT);
INSERT INTO meta (key, value) VALUES ('schema_version', '{}');
CREATE TABLE agents (
id INTEGER PRIMARY KEY,
slug TEXT NOT NULL UNIQUE,
name TEXT,
version TEXT,
kind TEXT NOT NULL,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE workspaces (
id INTEGER PRIMARY KEY,
path TEXT NOT NULL UNIQUE,
display_name TEXT
);
CREATE TABLE conversations (
id INTEGER PRIMARY KEY,
agent_id INTEGER NOT NULL,
workspace_id INTEGER,
source_id TEXT NOT NULL DEFAULT 'local',
external_id TEXT,
title TEXT,
source_path TEXT NOT NULL,
started_at INTEGER,
ended_at INTEGER,
approx_tokens INTEGER,
metadata_json TEXT,
origin_host TEXT,
metadata_bin BLOB
);
CREATE TABLE messages (
id INTEGER PRIMARY KEY,
conversation_id INTEGER NOT NULL,
idx INTEGER NOT NULL,
role TEXT NOT NULL,
author TEXT,
content TEXT NOT NULL,
created_at INTEGER,
extra_json TEXT,
extra_bin BLOB
);
CREATE TABLE sources (
id TEXT PRIMARY KEY,
kind TEXT NOT NULL,
path TEXT NOT NULL UNIQUE,
updated_at INTEGER NOT NULL
);
INSERT INTO sources (id, kind, path, updated_at) VALUES ('local', 'local', 'default', 0);
INSERT INTO agents (slug, name, kind, created_at, updated_at)
VALUES ('test-agent', 'Test Agent', 'cli', 1000000, 1000000);
INSERT INTO conversations (agent_id, source_id, source_path, title, started_at)
VALUES (1, 'local', '/path/to/source', 'Important Conversation', 1700000000000);
INSERT INTO messages (conversation_id, idx, role, content)
VALUES (1, 0, 'user', 'This is important data');
INSERT INTO messages (conversation_id, idx, role, content)
VALUES (1, 1, 'assistant', 'Acknowledged');
"#,
CURRENT_SCHEMA_VERSION
),
);
}
// Open and verify data
let storage = SqliteStorage::open(&db_path).unwrap();
// Verify agents
let agents = storage.list_agents().unwrap();
assert_eq!(agents.len(), 1);
assert_eq!(agents[0].slug, "test-agent");
assert_eq!(agents[0].name.as_str(), "Test Agent");
// Verify conversations
let convs = storage.list_conversations(10, 0).unwrap();
assert_eq!(convs.len(), 1);
assert_eq!(convs[0].title.as_deref(), Some("Important Conversation"));
// Verify messages
if let Some(conv) = convs.first()
&& let Some(id) = conv.id
{
let messages = storage.fetch_messages(id).unwrap();
assert_eq!(messages.len(), 2);
assert!(messages[0].content.contains("important data"));
}
}
/// Test handling of corrupted database during migration.
#[test]
fn test_migration_handles_corruption() {
let dir = TempDir::new().unwrap();
let db_path = dir.path().join("corrupted.db");
// Create a "corrupted" database (incomplete schema)
{
let conn = open_fixture_db(&db_path);
conn.execute("CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT)")
.unwrap();
conn.execute("INSERT INTO meta (key, value) VALUES ('schema_version', '5')")
.unwrap();
// Missing required tables - this simulates corruption
}
// Migration should either fix or signal rebuild
let result = SqliteStorage::open_or_rebuild(&db_path);
match result {
Ok(_) => {
// If it succeeds, verify it's usable
let storage = SqliteStorage::open(&db_path).unwrap();
assert!(storage.schema_version().unwrap() > 0);
}
Err(e) => {
// Should indicate need for rebuild
if let MigrationError::RebuildRequired { reason, .. } = e {
let reason_lower = reason.to_lowercase();
assert!(
reason_lower.contains("rebuild")
|| reason_lower.contains("corrupt")
|| reason_lower.contains("migration")
|| reason_lower.contains("schema"),
"Error should be migration-related: {}",
reason
);
}
}
}
}
// =============================================================================
// Schema Version Transition Tests
// =============================================================================
/// Test explicit schema version transitions.
#[test]
fn test_schema_version_5_to_current() {
let dir = TempDir::new().unwrap();
let db_path = dir.path().join("v5.db");
// Create v5 schema (basic structure)
create_v5_schema(&db_path);
// Migrate
let result = SqliteStorage::open_or_rebuild(&db_path);
match result {
Ok(_storage) => {
let storage = SqliteStorage::open(&db_path).unwrap();
let version = storage.schema_version().unwrap();
assert!(
version >= CURRENT_SCHEMA_VERSION,
"Version {} should be >= {}",
version,
CURRENT_SCHEMA_VERSION
);
}
Err(e) => {
// Rebuild is acceptable
match e {
MigrationError::RebuildRequired { reason, .. } => {
assert!(!reason.is_empty(), "Rebuild reason should be provided");
}
other => {
let unexpected = format!("{other:?}");
assert!(unexpected.is_empty(), "Unexpected error type: {unexpected}");
}
}
}
}
}
/// Create a v5 schema database for testing.
fn create_v5_schema(path: &Path) {
let conn = open_fixture_db(path);
conn.execute_batch(
r#"
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT);
INSERT INTO meta (key, value) VALUES ('schema_version', '5');
CREATE TABLE agents (
id INTEGER PRIMARY KEY,
slug TEXT NOT NULL UNIQUE,
name TEXT,
version TEXT,
kind TEXT NOT NULL,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE workspaces (
id INTEGER PRIMARY KEY,
path TEXT NOT NULL UNIQUE,
display_name TEXT
);
CREATE TABLE conversations (
id INTEGER PRIMARY KEY,
agent_id INTEGER NOT NULL,
workspace_id INTEGER,
external_id TEXT UNIQUE,
title TEXT,
source_path TEXT NOT NULL,
started_at INTEGER,
ended_at INTEGER,
metadata_json TEXT,
FOREIGN KEY(agent_id) REFERENCES agents(id)
);
CREATE TABLE messages (
id INTEGER PRIMARY KEY,
conversation_id INTEGER NOT NULL,
idx INTEGER NOT NULL,
role TEXT NOT NULL,
author TEXT,
content TEXT NOT NULL,
created_at INTEGER,
extra_json TEXT,
FOREIGN KEY(conversation_id) REFERENCES conversations(id)
);
CREATE TABLE sources (
id INTEGER PRIMARY KEY,
kind TEXT NOT NULL,
path TEXT NOT NULL UNIQUE,
updated_at INTEGER NOT NULL
);
INSERT INTO sources (id, kind, path, updated_at) VALUES (0, 'local', 'default', 0);
CREATE INDEX idx_conv_agent ON conversations(agent_id);
CREATE INDEX idx_conv_workspace ON conversations(workspace_id);
CREATE INDEX idx_msg_conv ON messages(conversation_id);
"#,
)
.unwrap();
}
// =============================================================================
// FTS Rebuild Tests
// =============================================================================
/// Test FTS rebuild functionality.
#[test]
fn test_fts_rebuild() {
let dir = TempDir::new().unwrap();
let db_path = dir.path().join("fts_rebuild.db");
// Create database with data but no FTS
{
let conn = open_fixture_db(&db_path);
run_fixture_batch(
&conn,
&format!(
r#"
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT);
INSERT INTO meta (key, value) VALUES ('schema_version', '{}');
CREATE TABLE agents (
id INTEGER PRIMARY KEY,
slug TEXT NOT NULL UNIQUE,
name TEXT,
version TEXT,
kind TEXT NOT NULL,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE workspaces (
id INTEGER PRIMARY KEY,
path TEXT NOT NULL UNIQUE,
display_name TEXT
);
CREATE TABLE conversations (
id INTEGER PRIMARY KEY,
agent_id INTEGER NOT NULL,
workspace_id INTEGER,
source_id TEXT NOT NULL DEFAULT 'local',
external_id TEXT,
title TEXT,
source_path TEXT NOT NULL,
started_at INTEGER,
ended_at INTEGER,
approx_tokens INTEGER,
metadata_json TEXT,
origin_host TEXT,
metadata_bin BLOB
);
CREATE TABLE messages (
id INTEGER PRIMARY KEY,
conversation_id INTEGER NOT NULL,
idx INTEGER NOT NULL,
role TEXT NOT NULL,
author TEXT,
content TEXT NOT NULL,
created_at INTEGER,
extra_json TEXT,
extra_bin BLOB
);
CREATE TABLE sources (
id TEXT PRIMARY KEY,
kind TEXT NOT NULL,
path TEXT NOT NULL UNIQUE,
updated_at INTEGER NOT NULL
);
INSERT INTO sources (id, kind, path, updated_at) VALUES ('local', 'local', 'default', 0);
-- Create FTS table for messages (matching actual schema)
CREATE VIRTUAL TABLE fts_messages USING fts5(
content,
title,
agent,
workspace,
source_path,
created_at UNINDEXED,
message_id UNINDEXED,
tokenize='porter'
);
-- Add test data
INSERT INTO agents (slug, name, kind, created_at, updated_at)
VALUES ('claude', 'Claude', 'cli', 0, 0);
INSERT INTO conversations (agent_id, source_id, source_path, title)
VALUES (1, 'local', '/test', 'Test');
INSERT INTO messages (conversation_id, idx, role, content)
VALUES (1, 0, 'user', 'Hello world from test');
INSERT INTO messages (conversation_id, idx, role, content)
VALUES (1, 1, 'assistant', 'Greetings user');
"#,
CURRENT_SCHEMA_VERSION
),
);
}
// Open and rebuild FTS
let storage = SqliteStorage::open(&db_path).unwrap();
let result = storage.rebuild_fts();
assert!(
result.is_ok(),
"FTS rebuild should succeed: {:?}",
result.err()
);
}
// =============================================================================
// Key Slot Migration Tests
// =============================================================================
/// Test that old encryption configs without recovery slots work.
#[test]
fn test_legacy_single_slot_config() {
use serde_json::json;
let legacy_config = json!({
"version": 1,
"export_id": "AAAAAAAAAAAAAAAAAAAAAA==",
"base_nonce": "AAAAAAAAAAAA",
"compression": "deflate",
"kdf_defaults": {
"memory_kb": 65536,
"iterations": 3,
"parallelism": 4
},
"payload": {
"chunk_size": 8388608,
"chunk_count": 1,
"total_compressed_size": 1024,
"total_plaintext_size": 2048,
"files": ["data.db"]
},
"key_slots": [{
"id": 0,
"slot_type": "password",
"kdf": "argon2id",
"salt": "c2FsdHNhbHRzYWx0c2FsdA==",
"wrapped_dek": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
"nonce": "AAAAAAAAAAAA",
"argon2_params": {
"memory_kb": 65536,
"iterations": 3,
"parallelism": 4
}
}]
});
// Should parse without recovery slot
let config: coding_agent_search::pages::encrypt::EncryptionConfig =
serde_json::from_value(legacy_config).unwrap();
assert_eq!(config.key_slots.len(), 1);
assert_eq!(
config.key_slots[0].slot_type,
coding_agent_search::pages::encrypt::SlotType::Password
);
}
// =============================================================================
// Rollback Tests
// =============================================================================
/// Test that failed migration doesn't corrupt data.
#[test]
fn test_failed_migration_preserves_original() {
let dir = TempDir::new().unwrap();
let db_path = dir.path().join("safe.db");
// Create a valid database
{
let conn = open_fixture_db(&db_path);
conn.execute_batch(&format!(
r#"
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT);
INSERT INTO meta (key, value) VALUES ('schema_version', '{}');
INSERT INTO meta (key, value) VALUES ('test_data', 'important');
CREATE TABLE agents (
id INTEGER PRIMARY KEY,
slug TEXT NOT NULL UNIQUE,
name TEXT,
version TEXT,
kind TEXT NOT NULL,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE workspaces (
id INTEGER PRIMARY KEY,
path TEXT NOT NULL UNIQUE,
display_name TEXT
);
CREATE TABLE conversations (
id INTEGER PRIMARY KEY,
agent_id INTEGER NOT NULL,
workspace_id INTEGER,
external_id TEXT UNIQUE,
title TEXT,
source_path TEXT NOT NULL,
started_at INTEGER,
ended_at INTEGER,
metadata_json TEXT
);
CREATE TABLE messages (
id INTEGER PRIMARY KEY,
conversation_id INTEGER NOT NULL,
idx INTEGER NOT NULL,
role TEXT NOT NULL,
author TEXT,
content TEXT NOT NULL,
created_at INTEGER,
extra_json TEXT
);
CREATE TABLE sources (
id INTEGER PRIMARY KEY,
kind TEXT NOT NULL,
path TEXT NOT NULL UNIQUE,
updated_at INTEGER NOT NULL
);
INSERT INTO sources (id, kind, path, updated_at) VALUES (0, 'local', 'default', 0);
"#,
CURRENT_SCHEMA_VERSION
))
.unwrap();
}
// Verify we can still read the test_data
let conn = open_fixture_db(&db_path);
let test_data: String = conn
.query_row_map(
"SELECT value FROM meta WHERE key = 'test_data'",
&[],
|row: &coding_agent_search::franken_sync::Row| row.get_typed(0),
)
.unwrap();
assert_eq!(test_data, "important");
}