Initial commit with translated description
This commit is contained in:
369
src/gep/assetStore.js
Normal file
369
src/gep/assetStore.js
Normal file
@@ -0,0 +1,369 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { getGepAssetsDir } = require('./paths');
|
||||
const { computeAssetId, SCHEMA_VERSION } = require('./contentHash');
|
||||
|
||||
function ensureDir(dir) {
|
||||
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
|
||||
function readJsonIfExists(filePath, fallback) {
|
||||
try {
|
||||
if (!fs.existsSync(filePath)) return fallback;
|
||||
const raw = fs.readFileSync(filePath, 'utf8');
|
||||
if (!raw.trim()) return fallback;
|
||||
return JSON.parse(raw);
|
||||
} catch (e) {
|
||||
console.warn('[AssetStore] Failed to read ' + filePath + ':', e && e.message || e);
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
function writeJsonAtomic(filePath, obj) {
|
||||
const dir = path.dirname(filePath);
|
||||
ensureDir(dir);
|
||||
const tmp = `${filePath}.tmp`;
|
||||
fs.writeFileSync(tmp, JSON.stringify(obj, null, 2) + '\n', 'utf8');
|
||||
fs.renameSync(tmp, filePath);
|
||||
}
|
||||
|
||||
// Build a validation command using repo-root-relative paths.
|
||||
// runValidations() executes with cwd=repoRoot, so require('./src/...')
|
||||
// resolves correctly without embedding machine-specific absolute paths.
|
||||
function buildValidationCmd(relModules) {
|
||||
const paths = relModules.map(m => `./${m}`);
|
||||
return `node scripts/validate-modules.js ${paths.join(' ')}`;
|
||||
}
|
||||
|
||||
function getDefaultGenes() {
|
||||
return {
|
||||
version: 1,
|
||||
genes: [
|
||||
{
|
||||
type: 'Gene', id: 'gene_gep_repair_from_errors', category: 'repair',
|
||||
signals_match: ['error', 'exception', 'failed', 'unstable'],
|
||||
preconditions: ['signals contains error-related indicators'],
|
||||
strategy: [
|
||||
'Extract structured signals from logs and user instructions',
|
||||
'Select an existing Gene by signals match (no improvisation)',
|
||||
'Estimate blast radius (files, lines) before editing',
|
||||
'Apply smallest reversible patch',
|
||||
'Validate using declared validation steps; rollback on failure',
|
||||
'Solidify knowledge: append EvolutionEvent, update Gene/Capsule store',
|
||||
],
|
||||
constraints: { max_files: 12, forbidden_paths: ['.git', 'node_modules'] },
|
||||
validation: [
|
||||
buildValidationCmd(['src/evolve', 'src/gep/solidify', 'src/gep/policyCheck', 'src/gep/selector', 'src/gep/memoryGraph', 'src/gep/assetStore']),
|
||||
'node scripts/validate-suite.js',
|
||||
],
|
||||
},
|
||||
{
|
||||
type: 'Gene', id: 'gene_gep_optimize_prompt_and_assets', category: 'optimize',
|
||||
signals_match: ['protocol', 'gep', 'prompt', 'audit', 'reusable'],
|
||||
preconditions: ['need stricter, auditable evolution protocol outputs'],
|
||||
strategy: [
|
||||
'Extract signals and determine selection rationale via Selector JSON',
|
||||
'Prefer reusing existing Gene/Capsule; only create if no match exists',
|
||||
'Refactor prompt assembly to embed assets (genes, capsules, parent event)',
|
||||
'Reduce noise and ambiguity; enforce strict output schema',
|
||||
'Validate by running node index.js run and ensuring no runtime errors',
|
||||
'Solidify: record EvolutionEvent, update Gene definitions, create Capsule on success',
|
||||
],
|
||||
constraints: { max_files: 20, forbidden_paths: ['.git', 'node_modules'] },
|
||||
validation: [
|
||||
buildValidationCmd(['src/evolve', 'src/gep/prompt', 'src/gep/contentHash', 'src/gep/skillDistiller']),
|
||||
'node scripts/validate-suite.js',
|
||||
],
|
||||
},
|
||||
{
|
||||
type: 'Gene', id: 'gene_tool_integrity', category: 'repair',
|
||||
signals_match: ['tool_bypass'],
|
||||
preconditions: ['agent used shell/exec to perform an action that a registered tool can handle'],
|
||||
strategy: [
|
||||
'Always prefer registered tools over ad-hoc scripts or shell workarounds',
|
||||
'If a registered tool fails, report the actual error honestly and attempt to fix the root cause',
|
||||
'Never fabricate explanations -- describe actual actions transparently',
|
||||
'Do not create temporary scripts in extension or project directories',
|
||||
],
|
||||
constraints: { max_files: 4, forbidden_paths: ['.git', 'node_modules'] },
|
||||
validation: [
|
||||
'node scripts/validate-suite.js',
|
||||
],
|
||||
anti_patterns: ['tool_bypass'],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function getDefaultCapsules() { return { version: 1, capsules: [] }; }
|
||||
function genesPath() { return path.join(getGepAssetsDir(), 'genes.json'); }
|
||||
function capsulesPath() { return path.join(getGepAssetsDir(), 'capsules.json'); }
|
||||
function capsulesJsonlPath() { return path.join(getGepAssetsDir(), 'capsules.jsonl'); }
|
||||
function eventsPath() { return path.join(getGepAssetsDir(), 'events.jsonl'); }
|
||||
function candidatesPath() { return path.join(getGepAssetsDir(), 'candidates.jsonl'); }
|
||||
function externalCandidatesPath() { return path.join(getGepAssetsDir(), 'external_candidates.jsonl'); }
|
||||
function failedCapsulesPath() { return path.join(getGepAssetsDir(), 'failed_capsules.json'); }
|
||||
|
||||
function loadGenes() {
|
||||
const jsonGenes = readJsonIfExists(genesPath(), getDefaultGenes()).genes || [];
|
||||
const jsonlGenes = [];
|
||||
try {
|
||||
const p = path.join(getGepAssetsDir(), 'genes.jsonl');
|
||||
if (fs.existsSync(p)) {
|
||||
const raw = fs.readFileSync(p, 'utf8');
|
||||
raw.split('\n').forEach(line => {
|
||||
if (line.trim()) {
|
||||
try {
|
||||
const parsed = JSON.parse(line);
|
||||
if (parsed && parsed.type === 'Gene') jsonlGenes.push(parsed);
|
||||
} catch(e) {}
|
||||
}
|
||||
});
|
||||
}
|
||||
} catch(e) {
|
||||
console.warn('[AssetStore] Failed to read genes.jsonl:', e && e.message || e);
|
||||
}
|
||||
|
||||
// Combine and deduplicate by ID (JSONL takes precedence if newer, but here we just merge)
|
||||
const combined = [...jsonGenes, ...jsonlGenes];
|
||||
const unique = new Map();
|
||||
combined.forEach(g => {
|
||||
if (g && g.id) unique.set(String(g.id), g);
|
||||
});
|
||||
return Array.from(unique.values());
|
||||
}
|
||||
|
||||
function loadCapsules() {
|
||||
const legacy = readJsonIfExists(capsulesPath(), getDefaultCapsules()).capsules || [];
|
||||
const jsonlCapsules = [];
|
||||
try {
|
||||
const p = capsulesJsonlPath();
|
||||
if (fs.existsSync(p)) {
|
||||
const raw = fs.readFileSync(p, 'utf8');
|
||||
raw.split('\n').forEach(line => {
|
||||
if (line.trim()) {
|
||||
try { jsonlCapsules.push(JSON.parse(line)); } catch(e) {}
|
||||
}
|
||||
});
|
||||
}
|
||||
} catch(e) {
|
||||
console.warn('[AssetStore] Failed to read capsules.jsonl:', e && e.message || e);
|
||||
}
|
||||
|
||||
// Combine and deduplicate by ID
|
||||
const combined = [...legacy, ...jsonlCapsules];
|
||||
const unique = new Map();
|
||||
combined.forEach(c => {
|
||||
if (c && c.id) unique.set(String(c.id), c);
|
||||
});
|
||||
return Array.from(unique.values());
|
||||
}
|
||||
|
||||
function getLastEventId() {
|
||||
try {
|
||||
const p = eventsPath();
|
||||
if (!fs.existsSync(p)) return null;
|
||||
const raw = fs.readFileSync(p, 'utf8');
|
||||
const lines = raw.split('\n').map(l => l.trim()).filter(Boolean);
|
||||
if (lines.length === 0) return null;
|
||||
const last = JSON.parse(lines[lines.length - 1]);
|
||||
return last && typeof last.id === 'string' ? last.id : null;
|
||||
} catch (e) {
|
||||
console.warn('[AssetStore] Failed to read last event ID:', e && e.message || e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function readAllEvents() {
|
||||
try {
|
||||
const p = eventsPath();
|
||||
if (!fs.existsSync(p)) return [];
|
||||
const raw = fs.readFileSync(p, 'utf8');
|
||||
return raw.split('\n').map(l => l.trim()).filter(Boolean).map(l => {
|
||||
try { return JSON.parse(l); } catch { return null; }
|
||||
}).filter(Boolean);
|
||||
} catch (e) {
|
||||
console.warn('[AssetStore] Failed to read events.jsonl:', e && e.message || e);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function appendEventJsonl(eventObj) {
|
||||
const dir = getGepAssetsDir(); ensureDir(dir);
|
||||
fs.appendFileSync(eventsPath(), JSON.stringify(eventObj) + '\n', 'utf8');
|
||||
}
|
||||
|
||||
function appendCandidateJsonl(candidateObj) {
|
||||
const dir = getGepAssetsDir(); ensureDir(dir);
|
||||
fs.appendFileSync(candidatesPath(), JSON.stringify(candidateObj) + '\n', 'utf8');
|
||||
}
|
||||
|
||||
function appendExternalCandidateJsonl(obj) {
|
||||
const dir = getGepAssetsDir(); ensureDir(dir);
|
||||
fs.appendFileSync(externalCandidatesPath(), JSON.stringify(obj) + '\n', 'utf8');
|
||||
}
|
||||
|
||||
function readRecentCandidates(limit = 20) {
|
||||
try {
|
||||
const p = candidatesPath();
|
||||
if (!fs.existsSync(p)) return [];
|
||||
const stat = fs.statSync(p);
|
||||
if (stat.size < 1024 * 1024) {
|
||||
const raw = fs.readFileSync(p, 'utf8');
|
||||
const lines = raw.split('\n').map(l => l.trim()).filter(Boolean);
|
||||
return lines.slice(-limit).map(l => {
|
||||
try { return JSON.parse(l); } catch { return null; }
|
||||
}).filter(Boolean);
|
||||
}
|
||||
// Large file (>1MB): only read the tail to avoid OOM.
|
||||
const fd = fs.openSync(p, 'r');
|
||||
try {
|
||||
const chunkSize = Math.min(stat.size, limit * 4096);
|
||||
const buf = Buffer.alloc(chunkSize);
|
||||
fs.readSync(fd, buf, 0, chunkSize, stat.size - chunkSize);
|
||||
const lines = buf.toString('utf8').split('\n').map(l => l.trim()).filter(Boolean);
|
||||
return lines.slice(-limit).map(l => {
|
||||
try { return JSON.parse(l); } catch { return null; }
|
||||
}).filter(Boolean);
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('[AssetStore] Failed to read candidates.jsonl:', e && e.message || e);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function readRecentExternalCandidates(limit = 50) {
|
||||
try {
|
||||
const p = externalCandidatesPath();
|
||||
if (!fs.existsSync(p)) return [];
|
||||
const stat = fs.statSync(p);
|
||||
if (stat.size < 1024 * 1024) {
|
||||
const raw = fs.readFileSync(p, 'utf8');
|
||||
const lines = raw.split('\n').map(l => l.trim()).filter(Boolean);
|
||||
return lines.slice(-limit).map(l => {
|
||||
try { return JSON.parse(l); } catch { return null; }
|
||||
}).filter(Boolean);
|
||||
}
|
||||
const fd = fs.openSync(p, 'r');
|
||||
try {
|
||||
const chunkSize = Math.min(stat.size, limit * 4096);
|
||||
const buf = Buffer.alloc(chunkSize);
|
||||
fs.readSync(fd, buf, 0, chunkSize, stat.size - chunkSize);
|
||||
const lines = buf.toString('utf8').split('\n').map(l => l.trim()).filter(Boolean);
|
||||
return lines.slice(-limit).map(l => {
|
||||
try { return JSON.parse(l); } catch { return null; }
|
||||
}).filter(Boolean);
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('[AssetStore] Failed to read external_candidates.jsonl:', e && e.message || e);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// Safety net: ensure schema_version and asset_id are present before writing.
|
||||
function ensureSchemaFields(obj) {
|
||||
if (!obj || typeof obj !== 'object') return obj;
|
||||
if (!obj.schema_version) obj.schema_version = SCHEMA_VERSION;
|
||||
if (!obj.asset_id) {
|
||||
try { obj.asset_id = computeAssetId(obj); } catch (e) {
|
||||
console.warn('[AssetStore] Failed to compute asset ID:', e && e.message || e);
|
||||
}
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
function upsertGene(geneObj) {
|
||||
ensureSchemaFields(geneObj);
|
||||
const current = readJsonIfExists(genesPath(), getDefaultGenes());
|
||||
const genes = Array.isArray(current.genes) ? current.genes : [];
|
||||
const idx = genes.findIndex(g => g && g.id === geneObj.id);
|
||||
if (idx >= 0) genes[idx] = geneObj; else genes.push(geneObj);
|
||||
writeJsonAtomic(genesPath(), { version: current.version || 1, genes });
|
||||
}
|
||||
|
||||
function appendCapsule(capsuleObj) {
|
||||
ensureSchemaFields(capsuleObj);
|
||||
const current = readJsonIfExists(capsulesPath(), getDefaultCapsules());
|
||||
const capsules = Array.isArray(current.capsules) ? current.capsules : [];
|
||||
capsules.push(capsuleObj);
|
||||
writeJsonAtomic(capsulesPath(), { version: current.version || 1, capsules });
|
||||
}
|
||||
|
||||
function upsertCapsule(capsuleObj) {
|
||||
if (!capsuleObj || capsuleObj.type !== 'Capsule' || !capsuleObj.id) return;
|
||||
ensureSchemaFields(capsuleObj);
|
||||
const current = readJsonIfExists(capsulesPath(), getDefaultCapsules());
|
||||
const capsules = Array.isArray(current.capsules) ? current.capsules : [];
|
||||
const idx = capsules.findIndex(c => c && c.type === 'Capsule' && String(c.id) === String(capsuleObj.id));
|
||||
if (idx >= 0) capsules[idx] = capsuleObj; else capsules.push(capsuleObj);
|
||||
writeJsonAtomic(capsulesPath(), { version: current.version || 1, capsules });
|
||||
}
|
||||
|
||||
const FAILED_CAPSULES_MAX = 200;
|
||||
const FAILED_CAPSULES_TRIM_TO = 100;
|
||||
|
||||
function getDefaultFailedCapsules() { return { version: 1, failed_capsules: [] }; }
|
||||
|
||||
function appendFailedCapsule(capsuleObj) {
|
||||
if (!capsuleObj || typeof capsuleObj !== 'object') return;
|
||||
ensureSchemaFields(capsuleObj);
|
||||
const current = readJsonIfExists(failedCapsulesPath(), getDefaultFailedCapsules());
|
||||
let list = Array.isArray(current.failed_capsules) ? current.failed_capsules : [];
|
||||
list.push(capsuleObj);
|
||||
if (list.length > FAILED_CAPSULES_MAX) {
|
||||
list = list.slice(list.length - FAILED_CAPSULES_TRIM_TO);
|
||||
}
|
||||
writeJsonAtomic(failedCapsulesPath(), { version: current.version || 1, failed_capsules: list });
|
||||
}
|
||||
|
||||
function readRecentFailedCapsules(limit) {
|
||||
const n = Number.isFinite(Number(limit)) && Number(limit) > 0 ? Number(limit) : 50;
|
||||
try {
|
||||
const current = readJsonIfExists(failedCapsulesPath(), getDefaultFailedCapsules());
|
||||
const list = Array.isArray(current.failed_capsules) ? current.failed_capsules : [];
|
||||
return list.slice(Math.max(0, list.length - n));
|
||||
} catch (e) {
|
||||
console.warn('[AssetStore] Failed to read failed_capsules.json:', e && e.message || e);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure all expected asset files exist on startup.
|
||||
// Creates empty files for optional append-only stores so that
|
||||
// external grep/read commands never fail with "No such file or directory".
|
||||
function ensureAssetFiles() {
|
||||
const dir = getGepAssetsDir();
|
||||
ensureDir(dir);
|
||||
const files = [
|
||||
{ path: genesPath(), defaultContent: JSON.stringify(getDefaultGenes(), null, 2) + '\n' },
|
||||
{ path: capsulesPath(), defaultContent: JSON.stringify(getDefaultCapsules(), null, 2) + '\n' },
|
||||
{ path: path.join(dir, 'genes.jsonl'), defaultContent: '' },
|
||||
{ path: eventsPath(), defaultContent: '' },
|
||||
{ path: candidatesPath(), defaultContent: '' },
|
||||
{ path: failedCapsulesPath(), defaultContent: JSON.stringify(getDefaultFailedCapsules(), null, 2) + '\n' },
|
||||
];
|
||||
for (const f of files) {
|
||||
if (!fs.existsSync(f.path)) {
|
||||
try {
|
||||
fs.writeFileSync(f.path, f.defaultContent, 'utf8');
|
||||
} catch (e) {
|
||||
// Non-fatal: log but continue
|
||||
console.error(`[AssetStore] Failed to create ${f.path}: ${e.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
loadGenes, loadCapsules, readAllEvents, getLastEventId,
|
||||
appendEventJsonl, appendCandidateJsonl, appendExternalCandidateJsonl,
|
||||
readRecentCandidates, readRecentExternalCandidates,
|
||||
upsertGene, appendCapsule, upsertCapsule,
|
||||
appendFailedCapsule, readRecentFailedCapsules,
|
||||
genesPath, capsulesPath, eventsPath, candidatesPath, externalCandidatesPath, failedCapsulesPath,
|
||||
ensureAssetFiles, buildValidationCmd,
|
||||
};
|
||||
Reference in New Issue
Block a user