autogame-17_capability-evolver/src/gep/assetStore.js

const fs = require('fs');
const path = require('path');
const { getGepAssetsDir } = require('./paths');
const { computeAssetId, SCHEMA_VERSION } = require('./contentHash');

function ensureDir(dir) {
  if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
}

function readJsonIfExists(filePath, fallback) {
  try {
    if (!fs.existsSync(filePath)) return fallback;
    const raw = fs.readFileSync(filePath, 'utf8');
    if (!raw.trim()) return fallback;
    return JSON.parse(raw);
  } catch (e) {
    console.warn('[AssetStore] Failed to read ' + filePath + ':', e && e.message || e);
    return fallback;
  }
}

function writeJsonAtomic(filePath, obj) {
  const dir = path.dirname(filePath);
  ensureDir(dir);
  const tmp = `${filePath}.tmp`;
  fs.writeFileSync(tmp, JSON.stringify(obj, null, 2) + '\n', 'utf8');
  fs.renameSync(tmp, filePath);
}

// Build a validation command using repo-root-relative paths.
// runValidations() executes with cwd=repoRoot, so require('./src/...')
// resolves correctly without embedding machine-specific absolute paths.
function buildValidationCmd(relModules) {
  const paths = relModules.map(m => `./${m}`);
  return `node scripts/validate-modules.js ${paths.join(' ')}`;
}

function getDefaultGenes() {
  return {
    version: 1,
    genes: [
      {
        type: 'Gene', id: 'gene_gep_repair_from_errors', category: 'repair',
        signals_match: ['error', 'exception', 'failed', 'unstable'],
        preconditions: ['signals contains error-related indicators'],
        strategy: [
          'Extract structured signals from logs and user instructions',
          'Select an existing Gene by signals match (no improvisation)',
          'Estimate blast radius (files, lines) before editing',
          'Apply smallest reversible patch',
          'Validate using declared validation steps; rollback on failure',
          'Solidify knowledge: append EvolutionEvent, update Gene/Capsule store',
        ],
        constraints: { max_files: 12, forbidden_paths: ['.git', 'node_modules'] },
        validation: [
          buildValidationCmd(['src/evolve', 'src/gep/solidify', 'src/gep/policyCheck', 'src/gep/selector', 'src/gep/memoryGraph', 'src/gep/assetStore']),
          'node scripts/validate-suite.js',
        ],
      },
      {
        type: 'Gene', id: 'gene_gep_optimize_prompt_and_assets', category: 'optimize',
        signals_match: ['protocol', 'gep', 'prompt', 'audit', 'reusable'],
        preconditions: ['need stricter, auditable evolution protocol outputs'],
        strategy: [
          'Extract signals and determine selection rationale via Selector JSON',
          'Prefer reusing existing Gene/Capsule; only create if no match exists',
          'Refactor prompt assembly to embed assets (genes, capsules, parent event)',
          'Reduce noise and ambiguity; enforce strict output schema',
          'Validate by running node index.js run and ensuring no runtime errors',
          'Solidify: record EvolutionEvent, update Gene definitions, create Capsule on success',
        ],
        constraints: { max_files: 20, forbidden_paths: ['.git', 'node_modules'] },
        validation: [
          buildValidationCmd(['src/evolve', 'src/gep/prompt', 'src/gep/contentHash', 'src/gep/skillDistiller']),
          'node scripts/validate-suite.js',
        ],
      },
      {
        type: 'Gene', id: 'gene_tool_integrity', category: 'repair',
        signals_match: ['tool_bypass'],
        preconditions: ['agent used shell/exec to perform an action that a registered tool can handle'],
        strategy: [
          'Always prefer registered tools over ad-hoc scripts or shell workarounds',
          'If a registered tool fails, report the actual error honestly and attempt to fix the root cause',
          'Never fabricate explanations -- describe actual actions transparently',
          'Do not create temporary scripts in extension or project directories',
        ],
        constraints: { max_files: 4, forbidden_paths: ['.git', 'node_modules'] },
        validation: [
          'node scripts/validate-suite.js',
        ],
        anti_patterns: ['tool_bypass'],
      },
    ],
  };
}

function getDefaultCapsules() { return { version: 1, capsules: [] }; }
function genesPath() { return path.join(getGepAssetsDir(), 'genes.json'); }
function capsulesPath() { return path.join(getGepAssetsDir(), 'capsules.json'); }
function capsulesJsonlPath() { return path.join(getGepAssetsDir(), 'capsules.jsonl'); }
function eventsPath() { return path.join(getGepAssetsDir(), 'events.jsonl'); }
function candidatesPath() { return path.join(getGepAssetsDir(), 'candidates.jsonl'); }
function externalCandidatesPath() { return path.join(getGepAssetsDir(), 'external_candidates.jsonl'); }
function failedCapsulesPath() { return path.join(getGepAssetsDir(), 'failed_capsules.json'); }

function loadGenes() {
  const jsonGenes = readJsonIfExists(genesPath(), getDefaultGenes()).genes || [];
  const jsonlGenes = [];
  try {
    const p = path.join(getGepAssetsDir(), 'genes.jsonl');
    if (fs.existsSync(p)) {
      const raw = fs.readFileSync(p, 'utf8');
      raw.split('\n').forEach(line => {
        if (line.trim()) {
          try {
            const parsed = JSON.parse(line);
            if (parsed && parsed.type === 'Gene') jsonlGenes.push(parsed);
          } catch(e) {}
        }
      });
    }
  } catch(e) {
    console.warn('[AssetStore] Failed to read genes.jsonl:', e && e.message || e);
  }

  // Combine and deduplicate by ID (JSONL takes precedence if newer, but here we just merge)
  const combined = [...jsonGenes, ...jsonlGenes];
  const unique = new Map();
  combined.forEach(g => {
    if (g && g.id) unique.set(String(g.id), g);
  });
  return Array.from(unique.values());
}

function loadCapsules() {
  const legacy = readJsonIfExists(capsulesPath(), getDefaultCapsules()).capsules || [];
  const jsonlCapsules = [];
  try {
    const p = capsulesJsonlPath();
    if (fs.existsSync(p)) {
      const raw = fs.readFileSync(p, 'utf8');
      raw.split('\n').forEach(line => {
        if (line.trim()) {
            try { jsonlCapsules.push(JSON.parse(line)); } catch(e) {}
        }
      });
    }
  } catch(e) {
    console.warn('[AssetStore] Failed to read capsules.jsonl:', e && e.message || e);
  }
  
  // Combine and deduplicate by ID
  const combined = [...legacy, ...jsonlCapsules];
  const unique = new Map();
  combined.forEach(c => {
      if (c && c.id) unique.set(String(c.id), c);
  });
  return Array.from(unique.values());
}

function getLastEventId() {
  try {
    const p = eventsPath();
    if (!fs.existsSync(p)) return null;
    const raw = fs.readFileSync(p, 'utf8');
    const lines = raw.split('\n').map(l => l.trim()).filter(Boolean);
    if (lines.length === 0) return null;
    const last = JSON.parse(lines[lines.length - 1]);
    return last && typeof last.id === 'string' ? last.id : null;
  } catch (e) {
    console.warn('[AssetStore] Failed to read last event ID:', e && e.message || e);
    return null;
  }
}

function readAllEvents() {
  try {
    const p = eventsPath();
    if (!fs.existsSync(p)) return [];
    const raw = fs.readFileSync(p, 'utf8');
    return raw.split('\n').map(l => l.trim()).filter(Boolean).map(l => {
      try { return JSON.parse(l); } catch { return null; }
    }).filter(Boolean);
  } catch (e) {
    console.warn('[AssetStore] Failed to read events.jsonl:', e && e.message || e);
    return [];
  }
}

function appendEventJsonl(eventObj) {
  const dir = getGepAssetsDir(); ensureDir(dir);
  fs.appendFileSync(eventsPath(), JSON.stringify(eventObj) + '\n', 'utf8');
}

function appendCandidateJsonl(candidateObj) {
  const dir = getGepAssetsDir(); ensureDir(dir);
  fs.appendFileSync(candidatesPath(), JSON.stringify(candidateObj) + '\n', 'utf8');
}

function appendExternalCandidateJsonl(obj) {
  const dir = getGepAssetsDir(); ensureDir(dir);
  fs.appendFileSync(externalCandidatesPath(), JSON.stringify(obj) + '\n', 'utf8');
}

function readRecentCandidates(limit = 20) {
  try {
    const p = candidatesPath();
    if (!fs.existsSync(p)) return [];
    const stat = fs.statSync(p);
    if (stat.size < 1024 * 1024) {
      const raw = fs.readFileSync(p, 'utf8');
      const lines = raw.split('\n').map(l => l.trim()).filter(Boolean);
      return lines.slice(-limit).map(l => {
        try { return JSON.parse(l); } catch { return null; }
      }).filter(Boolean);
    }
    // Large file (>1MB): only read the tail to avoid OOM.
    const fd = fs.openSync(p, 'r');
    try {
      const chunkSize = Math.min(stat.size, limit * 4096);
      const buf = Buffer.alloc(chunkSize);
      fs.readSync(fd, buf, 0, chunkSize, stat.size - chunkSize);
      const lines = buf.toString('utf8').split('\n').map(l => l.trim()).filter(Boolean);
      return lines.slice(-limit).map(l => {
        try { return JSON.parse(l); } catch { return null; }
      }).filter(Boolean);
    } finally {
      fs.closeSync(fd);
    }
  } catch (e) {
    console.warn('[AssetStore] Failed to read candidates.jsonl:', e && e.message || e);
    return [];
  }
}

function readRecentExternalCandidates(limit = 50) {
  try {
    const p = externalCandidatesPath();
    if (!fs.existsSync(p)) return [];
    const stat = fs.statSync(p);
    if (stat.size < 1024 * 1024) {
      const raw = fs.readFileSync(p, 'utf8');
      const lines = raw.split('\n').map(l => l.trim()).filter(Boolean);
      return lines.slice(-limit).map(l => {
        try { return JSON.parse(l); } catch { return null; }
      }).filter(Boolean);
    }
    const fd = fs.openSync(p, 'r');
    try {
      const chunkSize = Math.min(stat.size, limit * 4096);
      const buf = Buffer.alloc(chunkSize);
      fs.readSync(fd, buf, 0, chunkSize, stat.size - chunkSize);
      const lines = buf.toString('utf8').split('\n').map(l => l.trim()).filter(Boolean);
      return lines.slice(-limit).map(l => {
        try { return JSON.parse(l); } catch { return null; }
      }).filter(Boolean);
    } finally {
      fs.closeSync(fd);
    }
  } catch (e) {
    console.warn('[AssetStore] Failed to read external_candidates.jsonl:', e && e.message || e);
    return [];
  }
}

// Safety net: ensure schema_version and asset_id are present before writing.
function ensureSchemaFields(obj) {
  if (!obj || typeof obj !== 'object') return obj;
  if (!obj.schema_version) obj.schema_version = SCHEMA_VERSION;
  if (!obj.asset_id) {
    try { obj.asset_id = computeAssetId(obj); } catch (e) {
      console.warn('[AssetStore] Failed to compute asset ID:', e && e.message || e);
    }
  }
  return obj;
}

function upsertGene(geneObj) {
  ensureSchemaFields(geneObj);
  const current = readJsonIfExists(genesPath(), getDefaultGenes());
  const genes = Array.isArray(current.genes) ? current.genes : [];
  const idx = genes.findIndex(g => g && g.id === geneObj.id);
  if (idx >= 0) genes[idx] = geneObj; else genes.push(geneObj);
  writeJsonAtomic(genesPath(), { version: current.version || 1, genes });
}

function appendCapsule(capsuleObj) {
  ensureSchemaFields(capsuleObj);
  const current = readJsonIfExists(capsulesPath(), getDefaultCapsules());
  const capsules = Array.isArray(current.capsules) ? current.capsules : [];
  capsules.push(capsuleObj);
  writeJsonAtomic(capsulesPath(), { version: current.version || 1, capsules });
}

function upsertCapsule(capsuleObj) {
  if (!capsuleObj || capsuleObj.type !== 'Capsule' || !capsuleObj.id) return;
  ensureSchemaFields(capsuleObj);
  const current = readJsonIfExists(capsulesPath(), getDefaultCapsules());
  const capsules = Array.isArray(current.capsules) ? current.capsules : [];
  const idx = capsules.findIndex(c => c && c.type === 'Capsule' && String(c.id) === String(capsuleObj.id));
  if (idx >= 0) capsules[idx] = capsuleObj; else capsules.push(capsuleObj);
  writeJsonAtomic(capsulesPath(), { version: current.version || 1, capsules });
}

const FAILED_CAPSULES_MAX = 200;
const FAILED_CAPSULES_TRIM_TO = 100;

function getDefaultFailedCapsules() { return { version: 1, failed_capsules: [] }; }

function appendFailedCapsule(capsuleObj) {
  if (!capsuleObj || typeof capsuleObj !== 'object') return;
  ensureSchemaFields(capsuleObj);
  const current = readJsonIfExists(failedCapsulesPath(), getDefaultFailedCapsules());
  let list = Array.isArray(current.failed_capsules) ? current.failed_capsules : [];
  list.push(capsuleObj);
  if (list.length > FAILED_CAPSULES_MAX) {
    list = list.slice(list.length - FAILED_CAPSULES_TRIM_TO);
  }
  writeJsonAtomic(failedCapsulesPath(), { version: current.version || 1, failed_capsules: list });
}

function readRecentFailedCapsules(limit) {
  const n = Number.isFinite(Number(limit)) && Number(limit) > 0 ? Number(limit) : 50;
  try {
    const current = readJsonIfExists(failedCapsulesPath(), getDefaultFailedCapsules());
    const list = Array.isArray(current.failed_capsules) ? current.failed_capsules : [];
    return list.slice(Math.max(0, list.length - n));
  } catch (e) {
    console.warn('[AssetStore] Failed to read failed_capsules.json:', e && e.message || e);
    return [];
  }
}

// Ensure all expected asset files exist on startup.
// Creates empty files for optional append-only stores so that
// external grep/read commands never fail with "No such file or directory".
function ensureAssetFiles() {
  const dir = getGepAssetsDir();
  ensureDir(dir);
  const files = [
    { path: genesPath(), defaultContent: JSON.stringify(getDefaultGenes(), null, 2) + '\n' },
    { path: capsulesPath(), defaultContent: JSON.stringify(getDefaultCapsules(), null, 2) + '\n' },
    { path: path.join(dir, 'genes.jsonl'), defaultContent: '' },
    { path: eventsPath(), defaultContent: '' },
    { path: candidatesPath(), defaultContent: '' },
    { path: failedCapsulesPath(), defaultContent: JSON.stringify(getDefaultFailedCapsules(), null, 2) + '\n' },
  ];
  for (const f of files) {
    if (!fs.existsSync(f.path)) {
      try {
        fs.writeFileSync(f.path, f.defaultContent, 'utf8');
      } catch (e) {
        // Non-fatal: log but continue
        console.error(`[AssetStore] Failed to create ${f.path}: ${e.message}`);
      }
    }
  }
}

module.exports = {
  loadGenes, loadCapsules, readAllEvents, getLastEventId,
  appendEventJsonl, appendCandidateJsonl, appendExternalCandidateJsonl,
  readRecentCandidates, readRecentExternalCandidates,
  upsertGene, appendCapsule, upsertCapsule,
  appendFailedCapsule, readRecentFailedCapsules,
  genesPath, capsulesPath, eventsPath, candidatesPath, externalCandidatesPath, failedCapsulesPath,
  ensureAssetFiles, buildValidationCmd,
};