mirror of
https://github.com/proffesor-for-testing/agentic-qe.git
synced 2026-09-19 08:45:47 +08:00
Profa
fd2f37eaa0
Changes: - Fix README.md agent counts (48 QE agents = 41 main + 7 subagents) - Fix README.md skill counts (15 QE-specific skills) - Fix v3/README.md agent counts (47 → 48) - Fix init-wizard.ts hooks from 'npx @agentic-qe/v3' to 'npx agentic-qe' - Complete v2 → v2/ folder reorganization (54 items moved) - Update root package.json to point to v3 as main version Verified claims: - 48 QE agents (41 main + 7 TDD subagents) - 15 QE-specific skills - 12 DDD bounded contexts - ~5,600+ tests - HNSW, ReasoningBank, Queen Coordinator features exist Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1621 lines
54 KiB
TypeScript
1621 lines
54 KiB
TypeScript
/**
|
||
* LearningEngine - Phase 2 (Milestone 2.2)
|
||
*
|
||
* Implements reinforcement learning for agent performance improvement.
|
||
* Uses Q-learning algorithm to optimize task execution strategies.
|
||
*/
|
||
|
||
import { v4 as uuidv4 } from 'uuid';
|
||
import { Logger } from '../utils/Logger';
|
||
import { SwarmMemoryManager } from '../core/memory/SwarmMemoryManager';
|
||
import { QLearning, QLearningConfig } from './QLearning';
|
||
import { SARSALearner, SARSAConfig } from './algorithms/SARSALearner';
|
||
import { ActorCriticLearner, createDefaultActorCriticConfig } from './algorithms/ActorCriticLearner';
|
||
import { PPOLearner, createDefaultPPOConfig } from './algorithms/PPOLearner';
|
||
import { AbstractRLLearner } from './algorithms/AbstractRLLearner';
|
||
import { StateExtractor } from './StateExtractor';
|
||
import { RewardCalculator, TaskResult } from './RewardCalculator';
|
||
import packageJson from '../../package.json';
|
||
import {
|
||
recordLearningEpisode,
|
||
recordQValueUpdate,
|
||
recordPatternMatch,
|
||
recordAlgorithmSwitch,
|
||
recordExperienceSharing,
|
||
updateSharingConnections,
|
||
updateGaugeValues,
|
||
recordLearningSession,
|
||
withLearningSpan,
|
||
} from '../telemetry/LearningTelemetry';
|
||
|
||
const PACKAGE_VERSION = packageJson.version;
|
||
import {
|
||
LearningConfig,
|
||
TaskExperience,
|
||
TaskState,
|
||
AgentAction,
|
||
LearningFeedback,
|
||
LearningOutcome,
|
||
LearnedPattern,
|
||
LearningModelState,
|
||
FailurePattern,
|
||
StrategyRecommendation,
|
||
LearningEvent,
|
||
PerformanceMetrics
|
||
} from './types';
|
||
import { QETask } from '../types';
|
||
import { ExperienceSharingProtocol, SharedExperience } from './ExperienceSharingProtocol';
|
||
import { HNSWPatternAdapter, HNSWPatternAdapterConfig, PatternSearchResult } from './HNSWPatternAdapter';
|
||
|
||
/**
|
||
* RL Algorithm type selection
|
||
*/
|
||
export type RLAlgorithmType = 'q-learning' | 'sarsa' | 'actor-critic' | 'ppo' | 'maml' | 'legacy';
|
||
|
||
/**
|
||
* Extended learning configuration with algorithm selection
|
||
*/
|
||
export interface ExtendedLearningConfig extends LearningConfig {
|
||
algorithm?: RLAlgorithmType; // Which RL algorithm to use
|
||
enableExperienceSharing?: boolean; // Enable cross-agent experience sharing
|
||
experienceSharingPriority?: number; // Minimum priority for sharing (0-1)
|
||
/** Enable HNSW vector search for O(log n) pattern matching (Phase 0 M0.3) */
|
||
enableHNSW?: boolean;
|
||
/** HNSW adapter configuration */
|
||
hnswConfig?: HNSWPatternAdapterConfig;
|
||
}
|
||
|
||
/**
|
||
* Default learning configuration
|
||
*/
|
||
const DEFAULT_CONFIG: ExtendedLearningConfig = {
|
||
enabled: true,
|
||
learningRate: 0.1,
|
||
discountFactor: 0.95,
|
||
explorationRate: 0.3,
|
||
explorationDecay: 0.995,
|
||
minExplorationRate: 0.01,
|
||
maxMemorySize: 100 * 1024 * 1024, // 100MB
|
||
batchSize: 32,
|
||
updateFrequency: 10,
|
||
algorithm: 'q-learning', // Default to Q-Learning
|
||
enableExperienceSharing: false, // Disabled by default for backward compatibility
|
||
experienceSharingPriority: 0.5, // Only share moderately valuable experiences
|
||
enableHNSW: false, // Disabled by default for backward compatibility (Phase 0 M0.3)
|
||
};
|
||
|
||
/**
|
||
* LearningEngine - Reinforcement learning for agents
|
||
*/
|
||
export class LearningEngine {
|
||
private readonly logger: Logger;
|
||
private readonly memoryStore: SwarmMemoryManager;
|
||
private readonly agentId: string;
|
||
private config: ExtendedLearningConfig;
|
||
private qTable: Map<string, Map<string, number>>; // state-action values (legacy)
|
||
private rlAlgorithm?: AbstractRLLearner; // Current RL algorithm (Q-Learning, SARSA, etc.)
|
||
private qLearning?: QLearning; // Q-learning integration (backward compatibility)
|
||
private useQLearning: boolean;
|
||
private experiences: TaskExperience[];
|
||
// REMOVED: private patterns: Map<string, LearnedPattern>; (now persisted via memoryStore.storePattern)
|
||
private failurePatterns: Map<string, FailurePattern>;
|
||
private taskCount: number;
|
||
private readonly stateExtractor: StateExtractor;
|
||
private readonly rewardCalculator: RewardCalculator;
|
||
private experienceSharing?: ExperienceSharingProtocol;
|
||
// Phase 0 M0.3: HNSW vector search for O(log n) pattern matching
|
||
private hnswAdapter?: HNSWPatternAdapter;
|
||
|
||
constructor(
|
||
agentId: string,
|
||
memoryStore: SwarmMemoryManager,
|
||
config: Partial<ExtendedLearningConfig> = {}
|
||
) {
|
||
this.logger = Logger.getInstance();
|
||
this.agentId = agentId;
|
||
this.memoryStore = memoryStore;
|
||
this.config = { ...DEFAULT_CONFIG, ...config };
|
||
this.qTable = new Map();
|
||
this.useQLearning = false; // Default to legacy implementation
|
||
this.experiences = [];
|
||
// REMOVED: this.patterns = new Map(); (now using memoryStore.storePattern for persistence)
|
||
this.failurePatterns = new Map();
|
||
this.taskCount = 0;
|
||
this.stateExtractor = new StateExtractor();
|
||
this.rewardCalculator = new RewardCalculator();
|
||
|
||
// Initialize RL algorithm if specified
|
||
if (this.config.algorithm && this.config.algorithm !== 'legacy') {
|
||
this.setAlgorithm(this.config.algorithm);
|
||
}
|
||
|
||
// Phase 0 M0.3: Initialize HNSW adapter for O(log n) pattern matching
|
||
if (this.config.enableHNSW) {
|
||
this.hnswAdapter = new HNSWPatternAdapter(this.config.hnswConfig);
|
||
this.logger.info('HNSW pattern adapter created for O(log n) similarity search');
|
||
}
|
||
|
||
// UNIFIED PERSISTENCE ARCHITECTURE (v2.2.0):
|
||
// LearningEngine uses SwarmMemoryManager for ALL persistence.
|
||
// All data goes to the SINGLE database: .agentic-qe/memory.db
|
||
// This ensures CLI, MCP, and agents all share the same data.
|
||
//
|
||
// Key methods that persist to memory.db:
|
||
// - storeLearningExperience() -> learning_experiences table
|
||
// - storePattern() -> patterns table
|
||
// - upsertQValue() -> q_values table
|
||
this.logger.info(`LearningEngine initialized for agent ${agentId} with algorithm ${this.config.algorithm} - using ${memoryStore.constructor.name} for persistent storage`);
|
||
}
|
||
|
||
/**
|
||
* Initialize the learning engine
|
||
*
|
||
* Architecture: Uses shared memoryStore (SwarmMemoryManager) for all persistence.
|
||
* No database initialization needed - memoryStore handles the database connection.
|
||
*/
|
||
async initialize(): Promise<void> {
|
||
this.logger.info(`Initializing LearningEngine for agent ${this.agentId}`);
|
||
|
||
// Load previous learning state if exists
|
||
await this.loadState();
|
||
|
||
// Load Q-values from memoryStore if available
|
||
// Q-table loading done inline above
|
||
|
||
// Store config in memory via reasoningBank's memoryStore
|
||
if (this.memoryStore) {
|
||
// Serialize config to Record<string, unknown> for type safety
|
||
const serializableConfig: Record<string, unknown> = {
|
||
...this.config
|
||
};
|
||
await this.memoryStore.store(
|
||
`phase2/learning/${this.agentId}/config`,
|
||
serializableConfig,
|
||
{ partition: 'learning' }
|
||
);
|
||
}
|
||
|
||
// Phase 0 M0.3: Initialize HNSW adapter for vector similarity search
|
||
if (this.hnswAdapter) {
|
||
await this.hnswAdapter.initialize();
|
||
this.logger.info('HNSW pattern adapter initialized for O(log n) similarity search');
|
||
}
|
||
|
||
this.logger.info('LearningEngine initialized successfully');
|
||
}
|
||
|
||
/**
|
||
* @deprecated Use learnFromExecution() instead. This method is maintained for backward compatibility.
|
||
* Will be removed in v2.0.0
|
||
*
|
||
* Record experience from task execution (DEPRECATED)
|
||
* Redirects to learnFromExecution() for consolidated implementation
|
||
*/
|
||
async recordExperience(task: unknown, result: TaskResult, feedback?: LearningFeedback): Promise<void> {
|
||
this.logger.warn('[LearningEngine] recordExperience() is deprecated. Use learnFromExecution() instead. This method will be removed in v2.0.0');
|
||
|
||
// Redirect to unified method
|
||
await this.learnFromExecution(task, result, feedback);
|
||
}
|
||
|
||
/**
|
||
* Load patterns from memoryStore (SwarmMemoryManager -> memory.db)
|
||
*
|
||
* Architecture: Retrieves patterns from the unified memory.db database
|
||
* via SwarmMemoryManager.
|
||
*
|
||
* Note: This method is legacy and not currently used (patterns loaded on-demand).
|
||
*/
|
||
private async loadPatternsFromMemoryStore(): Promise<void> {
|
||
try {
|
||
// Query all patterns from memory.db (confidence >= 0 returns all)
|
||
const allPatterns = await this.memoryStore.queryPatternsByConfidence(0);
|
||
|
||
if (allPatterns.length === 0) {
|
||
this.logger.info('No existing patterns found in memory.db');
|
||
return;
|
||
}
|
||
|
||
this.logger.info(`Found ${allPatterns.length} patterns in memory.db for agent ${this.agentId}`);
|
||
} catch (error) {
|
||
this.logger.warn(`Failed to load patterns from memory.db:`, error);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Learn from task execution with automatic database persistence
|
||
* This is the primary method called by BaseAgent.onPostTask()
|
||
* Consolidates learnFromExecution() and recordExperience() functionality
|
||
*
|
||
* Architecture: Uses SwarmMemoryManager for all persistence (storeLearningExperience,
|
||
* upsertQValue, storeLearningSnapshot) instead of LearningPersistence adapter.
|
||
*/
|
||
async learnFromExecution(
|
||
task: unknown,
|
||
result: unknown,
|
||
feedback?: LearningFeedback
|
||
): Promise<LearningOutcome> {
|
||
if (!this.config.enabled) {
|
||
return this.createOutcome(false, 0, 0);
|
||
}
|
||
|
||
const episodeStartTime = Date.now();
|
||
|
||
try {
|
||
// Extract experience from task execution
|
||
const taskObj = (typeof task === 'object' && task !== null ? task : {}) as Record<string, unknown>;
|
||
const resultObj = (typeof result === 'object' && result !== null ? result : {}) as Record<string, unknown>;
|
||
const experience = this.extractExperience(taskObj, resultObj, feedback);
|
||
const reward = this.calculateReward(resultObj, feedback);
|
||
experience.reward = reward;
|
||
|
||
// Store in memory (fast)
|
||
this.experiences.push(experience);
|
||
|
||
// Update Q-table (in-memory)
|
||
await this.updateQTable(experience);
|
||
|
||
// Record telemetry: Q-value update occurred
|
||
recordQValueUpdate(this.agentId, 1, undefined, this.getAlgorithm());
|
||
|
||
// Persist to database via memoryStore (replaces persistence adapter)
|
||
// Only if memoryStore is SwarmMemoryManager
|
||
if (this.memoryStore && typeof (this.memoryStore as SwarmMemoryManager & { storeLearningExperience?: unknown }).storeLearningExperience === "function") {
|
||
try {
|
||
// Store experience
|
||
await this.memoryStore.storeLearningExperience({
|
||
agentId: this.agentId,
|
||
taskId: experience.taskId,
|
||
taskType: experience.taskType,
|
||
state: JSON.stringify(experience.state),
|
||
action: JSON.stringify(experience.action),
|
||
reward: experience.reward,
|
||
nextState: JSON.stringify(experience.nextState)
|
||
});
|
||
|
||
// Persist Q-value
|
||
const stateKey = this.stateExtractor.encodeState(experience.state);
|
||
const actionKey = this.stateExtractor.encodeAction(experience.action);
|
||
const stateActions = this.qTable.get(stateKey);
|
||
const qValue = stateActions?.get(actionKey) || 0;
|
||
|
||
await this.memoryStore.upsertQValue(
|
||
this.agentId,
|
||
stateKey,
|
||
actionKey,
|
||
qValue
|
||
);
|
||
} catch (persistError) {
|
||
this.logger.warn('Persistence failed, continuing with in-memory learning:', persistError);
|
||
// Don't fail learning due to persistence errors
|
||
}
|
||
}
|
||
|
||
// Update patterns
|
||
await this.updatePatterns(experience);
|
||
|
||
// Detect failure patterns
|
||
if (!resultObj.success) {
|
||
await this.detectFailurePattern(experience);
|
||
}
|
||
|
||
// Increment task count
|
||
this.taskCount++;
|
||
|
||
// Periodic batch update and snapshot
|
||
if (this.taskCount % this.config.updateFrequency === 0) {
|
||
await this.performBatchUpdate();
|
||
|
||
// Store learning snapshot via memoryStore
|
||
if (this.memoryStore && typeof (this.memoryStore as SwarmMemoryManager & { storeLearningExperience?: unknown }).storeLearningExperience === "function") {
|
||
const improvement = await this.calculateImprovement();
|
||
await this.memoryStore.storeLearningSnapshot({
|
||
agentId: this.agentId,
|
||
snapshotType: 'performance',
|
||
metrics: {
|
||
qTableSize: this.qTable.size
|
||
},
|
||
totalExperiences: this.experiences.length,
|
||
improvementRate: improvement.improvementRate,
|
||
explorationRate: this.config.explorationRate
|
||
});
|
||
}
|
||
}
|
||
|
||
// Decay exploration rate
|
||
this.decayExploration();
|
||
|
||
// Calculate improvement
|
||
const improvement = await this.calculateImprovement();
|
||
|
||
// Share experience with peers if enabled and valuable
|
||
if (this.config.enableExperienceSharing && this.experienceSharing) {
|
||
await this.shareExperienceWithPeers(experience);
|
||
}
|
||
|
||
// Emit learning event
|
||
await this.emitLearningEvent('training', {
|
||
experience,
|
||
reward,
|
||
improvement
|
||
});
|
||
|
||
// Save state periodically
|
||
if (this.taskCount % 50 === 0) {
|
||
await this.saveState();
|
||
}
|
||
|
||
return improvement;
|
||
} catch (error) {
|
||
this.logger.error(`Learning from execution failed:`, error);
|
||
return this.createOutcome(false, 0, 0);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Recommend best strategy for a given state
|
||
*/
|
||
async recommendStrategy(state: TaskState): Promise<StrategyRecommendation> {
|
||
const stateKey = this.encodeState(state);
|
||
|
||
// Get Q-values for all actions in this state
|
||
const actionValues = this.qTable.get(stateKey) || new Map();
|
||
|
||
if (actionValues.size === 0) {
|
||
// No learned strategies yet, return default
|
||
return {
|
||
strategy: 'default',
|
||
confidence: 0.5,
|
||
expectedImprovement: 0,
|
||
reasoning: 'No learned strategies available yet',
|
||
alternatives: []
|
||
};
|
||
}
|
||
|
||
// Find best action
|
||
let bestAction = '';
|
||
let bestValue = -Infinity;
|
||
const alternatives: { strategy: string; confidence: number }[] = [];
|
||
|
||
for (const [action, value] of actionValues.entries()) {
|
||
if (value > bestValue) {
|
||
if (bestAction) {
|
||
alternatives.push({
|
||
strategy: bestAction,
|
||
confidence: this.valueToConfidence(bestValue)
|
||
});
|
||
}
|
||
bestValue = value;
|
||
bestAction = action;
|
||
} else {
|
||
alternatives.push({
|
||
strategy: action,
|
||
confidence: this.valueToConfidence(value)
|
||
});
|
||
}
|
||
}
|
||
|
||
// Sort alternatives by confidence
|
||
alternatives.sort((a, b) => b.confidence - a.confidence);
|
||
|
||
return {
|
||
strategy: bestAction,
|
||
confidence: this.valueToConfidence(bestValue),
|
||
expectedImprovement: Math.max(0, bestValue * 100),
|
||
reasoning: `Learned from ${this.experiences.length} experiences`,
|
||
alternatives: alternatives.slice(0, 3)
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Get learned patterns for this agent
|
||
* Now queries from memoryStore (SwarmMemoryManager) with agent filtering
|
||
*/
|
||
async getPatterns(): Promise<LearnedPattern[]> {
|
||
try {
|
||
// Query patterns from memoryStore if available
|
||
if (!this.memoryStore || typeof this.memoryStore.queryPatternsByAgent !== 'function') {
|
||
return [];
|
||
}
|
||
|
||
// Use agent-specific query to avoid mixing patterns from different agents
|
||
const dbPatterns = await this.memoryStore.queryPatternsByAgent(this.agentId, 0);
|
||
|
||
return dbPatterns.map((p: { id?: string; pattern: string; confidence: number; metadata?: Record<string, unknown>; usageCount?: number }) => ({
|
||
id: p.id || uuidv4(), // Use existing ID or generate new one
|
||
pattern: p.pattern,
|
||
confidence: p.confidence,
|
||
successRate: (p.metadata?.success_rate as number) ?? 0.5,
|
||
usageCount: p.usageCount || 0,
|
||
contexts: (p.metadata?.contexts as string[]) ?? [],
|
||
createdAt: p.metadata?.created_at ? new Date(p.metadata.created_at as string | number) : new Date(),
|
||
lastUsedAt: p.metadata?.last_used_at ? new Date(p.metadata.last_used_at as string | number) : new Date()
|
||
}));
|
||
} catch (error) {
|
||
this.logger.warn('Failed to query patterns:', error);
|
||
return [];
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Search for similar patterns using HNSW vector similarity
|
||
* Phase 0 M0.3: O(log n) complexity with <1ms p95 latency
|
||
*
|
||
* @param query Search query (text to find similar patterns for)
|
||
* @param k Number of results to return (default: 5)
|
||
* @returns Similar patterns with similarity scores, or empty array if HNSW not enabled
|
||
*/
|
||
async searchSimilarPatterns(query: string, k: number = 5): Promise<PatternSearchResult[]> {
|
||
if (!this.hnswAdapter) {
|
||
this.logger.debug('HNSW not enabled, searchSimilarPatterns returning empty');
|
||
return [];
|
||
}
|
||
|
||
try {
|
||
return await this.hnswAdapter.searchSimilar(query, k);
|
||
} catch (error) {
|
||
this.logger.warn('HNSW similarity search failed:', error);
|
||
return [];
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Check if HNSW vector search is enabled
|
||
*/
|
||
isHNSWEnabled(): boolean {
|
||
return !!this.hnswAdapter;
|
||
}
|
||
|
||
/**
|
||
* Get HNSW adapter statistics
|
||
*/
|
||
async getHNSWStats(): Promise<{ patternCount: number; useRuvLLM: boolean; embeddingDimension: number } | null> {
|
||
if (!this.hnswAdapter) {
|
||
return null;
|
||
}
|
||
return this.hnswAdapter.getStats();
|
||
}
|
||
|
||
/**
|
||
* Get failure patterns
|
||
*/
|
||
getFailurePatterns(): FailurePattern[] {
|
||
return Array.from(this.failurePatterns.values())
|
||
.sort((a, b) => b.frequency - a.frequency);
|
||
}
|
||
|
||
/**
|
||
* Extract features from state for learning
|
||
*/
|
||
private extractFeatures(state: TaskState): number[] {
|
||
return [
|
||
state.taskComplexity,
|
||
state.requiredCapabilities.length / 10, // normalize
|
||
state.previousAttempts / 5, // normalize
|
||
state.availableResources,
|
||
state.timeConstraint ? Math.min(state.timeConstraint / 300000, 1) : 1 // normalize to 5 min
|
||
];
|
||
}
|
||
|
||
/**
|
||
* Encode state to string key for Q-table
|
||
*/
|
||
private encodeState(state: TaskState): string {
|
||
const features = this.extractFeatures(state);
|
||
return features.map(f => Math.round(f * 10) / 10).join(',');
|
||
}
|
||
|
||
/**
|
||
* Encode action to string key
|
||
*/
|
||
private encodeAction(action: AgentAction): string {
|
||
return `${action.strategy}:${action.parallelization.toFixed(1)}:${action.retryPolicy}`;
|
||
}
|
||
|
||
/**
|
||
* Extract experience from task execution
|
||
*/
|
||
private extractExperience(
|
||
task: Record<string, unknown>,
|
||
result: Record<string, unknown>,
|
||
feedback?: LearningFeedback
|
||
): TaskExperience {
|
||
const requirements = task.requirements as Record<string, unknown> | undefined;
|
||
const state: TaskState = {
|
||
taskComplexity: this.estimateComplexity(task),
|
||
requiredCapabilities: (requirements?.capabilities as string[]) || [],
|
||
contextFeatures: (task.context as Record<string, unknown>) || {},
|
||
previousAttempts: (task.previousAttempts as number) || 0,
|
||
availableResources: this.stateExtractor.extractState(task as unknown as QETask).availableResources,
|
||
timeConstraint: task.timeout as number | undefined
|
||
};
|
||
|
||
const action: AgentAction = {
|
||
strategy: (result.strategy as string) || 'default',
|
||
toolsUsed: (result.toolsUsed as string[]) || [],
|
||
parallelization: (result.parallelization as number) || 0.5,
|
||
retryPolicy: (result.retryPolicy as string) || 'exponential',
|
||
resourceAllocation: (result.resourceAllocation as number) || 0.5
|
||
};
|
||
|
||
// Next state (after execution)
|
||
const nextState: TaskState = {
|
||
...state,
|
||
previousAttempts: state.previousAttempts + 1,
|
||
availableResources: state.availableResources * 0.9 // resource consumption
|
||
};
|
||
|
||
return {
|
||
taskId: (task.id as string) || uuidv4(),
|
||
taskType: task.type as string,
|
||
state,
|
||
action,
|
||
reward: 0, // will be calculated
|
||
nextState,
|
||
timestamp: new Date(),
|
||
agentId: this.agentId
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Calculate reward from execution result and feedback
|
||
*/
|
||
private calculateReward(result: Record<string, unknown>, feedback?: LearningFeedback): number {
|
||
let reward = 0;
|
||
|
||
// Success/failure (primary component)
|
||
reward += result.success ? 1.0 : -1.0;
|
||
|
||
// Execution time (faster is better)
|
||
const executionTime = result.executionTime as number | undefined;
|
||
if (executionTime) {
|
||
const timeFactor = Math.max(0, 1 - executionTime / 30000); // 30 sec baseline
|
||
reward += timeFactor * 0.5;
|
||
}
|
||
|
||
// Error rate penalty
|
||
const errors = result.errors as unknown[] | undefined;
|
||
if (errors) {
|
||
reward -= errors.length * 0.1;
|
||
}
|
||
|
||
// User feedback
|
||
if (feedback) {
|
||
reward += (feedback.rating - 0.5) * 2; // -1 to +1
|
||
reward -= feedback.issues.length * 0.2;
|
||
}
|
||
|
||
// Coverage/quality bonus (for test generation)
|
||
const coverage = result.coverage as number | undefined;
|
||
if (coverage) {
|
||
reward += (coverage - 0.8) * 2; // bonus above 80%
|
||
}
|
||
|
||
return Math.max(-2, Math.min(2, reward)); // clamp to [-2, 2]
|
||
}
|
||
|
||
/**
|
||
* Update Q-table with new experience (Q-learning algorithm)
|
||
*/
|
||
private async updateQTable(experience: TaskExperience): Promise<void> {
|
||
const stateKey = this.encodeState(experience.state);
|
||
const actionKey = this.encodeAction(experience.action);
|
||
const nextStateKey = this.encodeState(experience.nextState);
|
||
|
||
// Get or create state-action map
|
||
if (!this.qTable.has(stateKey)) {
|
||
this.qTable.set(stateKey, new Map());
|
||
}
|
||
const stateActions = this.qTable.get(stateKey)!;
|
||
|
||
// Get current Q-value
|
||
const currentQ = stateActions.get(actionKey) || 0;
|
||
|
||
// Get max Q-value for next state
|
||
const nextStateActions = this.qTable.get(nextStateKey) || new Map();
|
||
const maxNextQ = nextStateActions.size > 0
|
||
? Math.max(...Array.from(nextStateActions.values()))
|
||
: 0;
|
||
|
||
// Q-learning update: Q(s,a) = Q(s,a) + α * [r + γ * max(Q(s',a')) - Q(s,a)]
|
||
const newQ = currentQ + this.config.learningRate * (
|
||
experience.reward + this.config.discountFactor * maxNextQ - currentQ
|
||
);
|
||
|
||
stateActions.set(actionKey, newQ);
|
||
}
|
||
|
||
/**
|
||
* Perform batch update on experiences
|
||
*/
|
||
private async performBatchUpdate(): Promise<void> {
|
||
if (this.experiences.length < this.config.batchSize) {
|
||
return;
|
||
}
|
||
|
||
// Sample recent experiences
|
||
const batch = this.experiences.slice(-this.config.batchSize);
|
||
|
||
// Re-train on batch
|
||
for (const experience of batch) {
|
||
await this.updateQTable(experience);
|
||
}
|
||
|
||
this.logger.info(`Performed batch update on ${batch.length} experiences`);
|
||
}
|
||
|
||
/**
|
||
* Update learned patterns
|
||
* Now uses memoryStore persistence with agent filtering instead of in-memory Map
|
||
*/
|
||
private async updatePatterns(experience: TaskExperience): Promise<void> {
|
||
const patternKey = `${experience.taskType}:${experience.action.strategy}`;
|
||
|
||
try {
|
||
// Try to query existing pattern from memoryStore (agent-specific)
|
||
let existingPattern: LearnedPattern | null = null;
|
||
|
||
if (this.memoryStore && typeof this.memoryStore.queryPatternsByAgent === 'function') {
|
||
const agentPatterns = await this.memoryStore.queryPatternsByAgent(this.agentId, 0);
|
||
const found = agentPatterns.find((p: { pattern?: string }) => p.pattern === patternKey);
|
||
|
||
if (found && found.metadata) {
|
||
const metadata = typeof found.metadata === 'string'
|
||
? JSON.parse(found.metadata)
|
||
: found.metadata;
|
||
|
||
existingPattern = {
|
||
id: found.id || uuidv4(),
|
||
pattern: found.pattern,
|
||
confidence: found.confidence,
|
||
successRate: metadata.success_rate || 0.5,
|
||
usageCount: found.usageCount,
|
||
contexts: metadata.contexts || [experience.taskType],
|
||
createdAt: metadata.created_at ? new Date(metadata.created_at) : new Date(),
|
||
lastUsedAt: metadata.last_used_at ? new Date(metadata.last_used_at) : new Date()
|
||
};
|
||
}
|
||
}
|
||
|
||
if (existingPattern) {
|
||
// Update existing pattern
|
||
existingPattern.usageCount++;
|
||
existingPattern.lastUsedAt = new Date();
|
||
|
||
// Update success rate
|
||
if (experience.reward > 0) {
|
||
existingPattern.successRate = (existingPattern.successRate * (existingPattern.usageCount - 1) + 1) / existingPattern.usageCount;
|
||
} else {
|
||
existingPattern.successRate = (existingPattern.successRate * (existingPattern.usageCount - 1)) / existingPattern.usageCount;
|
||
}
|
||
|
||
// Update confidence
|
||
existingPattern.confidence = Math.min(0.95, existingPattern.confidence + 0.01);
|
||
|
||
// Persist pattern update to database
|
||
await this.memoryStore.storePattern({
|
||
id: existingPattern.id,
|
||
pattern: existingPattern.pattern,
|
||
confidence: existingPattern.confidence,
|
||
usageCount: existingPattern.usageCount,
|
||
metadata: {
|
||
agent_id: this.agentId,
|
||
success_rate: existingPattern.successRate,
|
||
contexts: existingPattern.contexts,
|
||
created_at: existingPattern.createdAt,
|
||
last_used_at: existingPattern.lastUsedAt
|
||
}
|
||
});
|
||
|
||
// Phase 0 M0.3: Update in HNSW for vector similarity search
|
||
if (this.hnswAdapter) {
|
||
try {
|
||
await this.hnswAdapter.storePattern({
|
||
...existingPattern,
|
||
agentId: this.agentId,
|
||
averageReward: experience.reward,
|
||
});
|
||
} catch (hnswError) {
|
||
this.logger.warn('HNSW pattern update failed, continuing with SQLite only:', hnswError);
|
||
}
|
||
}
|
||
} else {
|
||
// Create new pattern
|
||
const pattern: LearnedPattern = {
|
||
id: uuidv4(),
|
||
pattern: patternKey,
|
||
confidence: 0.5,
|
||
successRate: experience.reward > 0 ? 1.0 : 0.0,
|
||
usageCount: 1,
|
||
contexts: [experience.taskType],
|
||
createdAt: new Date(),
|
||
lastUsedAt: new Date()
|
||
};
|
||
|
||
// Persist new pattern to database
|
||
await this.memoryStore.storePattern({
|
||
id: pattern.id,
|
||
pattern: pattern.pattern,
|
||
confidence: pattern.confidence,
|
||
usageCount: pattern.usageCount,
|
||
metadata: {
|
||
agent_id: this.agentId,
|
||
success_rate: pattern.successRate,
|
||
contexts: pattern.contexts,
|
||
created_at: pattern.createdAt,
|
||
last_used_at: pattern.lastUsedAt
|
||
}
|
||
});
|
||
|
||
// Phase 0 M0.3: Also store in HNSW for vector similarity search
|
||
if (this.hnswAdapter) {
|
||
try {
|
||
await this.hnswAdapter.storePattern({
|
||
...pattern,
|
||
agentId: this.agentId,
|
||
averageReward: experience.reward,
|
||
});
|
||
} catch (hnswError) {
|
||
this.logger.warn('HNSW pattern storage failed, continuing with SQLite only:', hnswError);
|
||
}
|
||
}
|
||
|
||
// Emit pattern discovered event
|
||
await this.emitLearningEvent('pattern_discovered', pattern);
|
||
}
|
||
} catch (error) {
|
||
this.logger.warn('Failed to update patterns:', error);
|
||
// Continue execution even if pattern update fails
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Detect failure patterns
|
||
*/
|
||
private async detectFailurePattern(experience: TaskExperience): Promise<void> {
|
||
const patternKey = `${experience.taskType}:failure`;
|
||
|
||
if (this.failurePatterns.has(patternKey)) {
|
||
const pattern = this.failurePatterns.get(patternKey)!;
|
||
pattern.frequency++;
|
||
pattern.confidence = Math.min(0.95, pattern.frequency / this.experiences.length);
|
||
} else {
|
||
const pattern: FailurePattern = {
|
||
id: uuidv4(),
|
||
pattern: patternKey,
|
||
frequency: 1,
|
||
contexts: [experience.taskType],
|
||
confidence: 0.1,
|
||
identifiedAt: new Date()
|
||
};
|
||
this.failurePatterns.set(patternKey, pattern);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Calculate improvement over time
|
||
*/
|
||
private async calculateImprovement(): Promise<LearningOutcome> {
|
||
if (this.experiences.length < 10) {
|
||
return this.createOutcome(false, 0, 0);
|
||
}
|
||
|
||
// Compare recent performance vs baseline
|
||
const recentExperiences = this.experiences.slice(-20);
|
||
const baselineExperiences = this.experiences.slice(0, 20);
|
||
|
||
const recentAvgReward = recentExperiences.reduce((sum, e) => sum + e.reward, 0) / recentExperiences.length;
|
||
const baselineAvgReward = baselineExperiences.reduce((sum, e) => sum + e.reward, 0) / baselineExperiences.length;
|
||
|
||
const improvementRate = baselineAvgReward !== 0
|
||
? ((recentAvgReward - baselineAvgReward) / Math.abs(baselineAvgReward)) * 100
|
||
: 0;
|
||
|
||
return this.createOutcome(
|
||
improvementRate > 0,
|
||
baselineAvgReward,
|
||
recentAvgReward,
|
||
improvementRate
|
||
);
|
||
}
|
||
|
||
/**
|
||
* Create learning outcome
|
||
*/
|
||
private async createOutcome(
|
||
improved: boolean,
|
||
previous: number,
|
||
current: number,
|
||
rate: number = 0
|
||
): Promise<LearningOutcome> {
|
||
const patterns = await this.getPatterns();
|
||
return {
|
||
improved,
|
||
previousPerformance: previous,
|
||
newPerformance: current,
|
||
improvementRate: rate,
|
||
confidence: Math.min(0.95, this.experiences.length / 100),
|
||
patterns: patterns.slice(0, 5),
|
||
timestamp: new Date()
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Estimate task complexity
|
||
*/
|
||
private estimateComplexity(task: Record<string, unknown>): number {
|
||
let complexity = 0.5; // baseline
|
||
|
||
const requirements = task.requirements as Record<string, unknown> | undefined;
|
||
const capabilities = requirements?.capabilities as unknown[] | undefined;
|
||
if (capabilities) {
|
||
complexity += capabilities.length * 0.1;
|
||
}
|
||
|
||
const previousAttempts = task.previousAttempts as number | undefined;
|
||
if (previousAttempts) {
|
||
complexity += previousAttempts * 0.1;
|
||
}
|
||
|
||
return Math.min(1.0, complexity);
|
||
}
|
||
|
||
/**
|
||
* Convert Q-value to confidence score
|
||
*/
|
||
private valueToConfidence(value: number): number {
|
||
return Math.max(0, Math.min(1, (value + 2) / 4)); // map [-2, 2] to [0, 1]
|
||
}
|
||
|
||
/**
|
||
* Decay exploration rate
|
||
*/
|
||
private decayExploration(): void {
|
||
this.config.explorationRate = Math.max(
|
||
this.config.minExplorationRate,
|
||
this.config.explorationRate * this.config.explorationDecay
|
||
);
|
||
}
|
||
|
||
/**
|
||
* Save learning state to memory
|
||
*/
|
||
private async saveState(): Promise<void> {
|
||
const patterns = await this.getPatterns();
|
||
const state: LearningModelState = {
|
||
agentId: this.agentId,
|
||
qTable: this.serializeQTable(),
|
||
experiences: this.experiences.slice(-1000), // keep last 1000
|
||
patterns,
|
||
config: this.config,
|
||
performance: await this.getCurrentPerformance(),
|
||
version: PACKAGE_VERSION,
|
||
lastUpdated: new Date(),
|
||
size: await this.calculateStateSize()
|
||
};
|
||
|
||
// Check size limit
|
||
if (state.size > this.config.maxMemorySize) {
|
||
this.logger.warn(`Learning state exceeds max size (${state.size} bytes), pruning...`);
|
||
// Fix: Prune in-memory array, not just saved state
|
||
this.experiences = this.experiences.slice(-500);
|
||
state.experiences = this.experiences;
|
||
state.size = await this.calculateStateSize();
|
||
}
|
||
|
||
// Serialize state to Record<string, unknown> for type safety
|
||
// Convert Date objects to ISO strings for JSON serialization
|
||
const serializableState: Record<string, unknown> = {
|
||
...state,
|
||
lastUpdated: state.lastUpdated.toISOString(),
|
||
experiences: state.experiences.map(exp => ({
|
||
...exp,
|
||
timestamp: exp.timestamp.toISOString()
|
||
})),
|
||
patterns: state.patterns.map(p => ({
|
||
...p,
|
||
createdAt: p.createdAt.toISOString(),
|
||
lastUsedAt: p.lastUsedAt.toISOString()
|
||
}))
|
||
};
|
||
|
||
await this.memoryStore.store(
|
||
`phase2/learning/${this.agentId}/state`,
|
||
serializableState,
|
||
{ partition: 'learning' }
|
||
);
|
||
|
||
this.logger.info(`Saved learning state (${state.size} bytes, ${state.experiences.length} experiences)`);
|
||
}
|
||
|
||
/**
|
||
* Load learning state from memory
|
||
* Pattern Map removed - patterns now persisted directly via memoryStore.storePattern()
|
||
*/
|
||
private async loadState(): Promise<void> {
|
||
try {
|
||
const state = await this.memoryStore.retrieve(
|
||
`phase2/learning/${this.agentId}/state`,
|
||
{ partition: 'learning' }
|
||
) as LearningModelState | null;
|
||
|
||
if (state) {
|
||
this.deserializeQTable(state.qTable);
|
||
this.experiences = state.experiences;
|
||
// REMOVED: this.patterns = new Map(state.patterns.map(p => [p.pattern, p]));
|
||
// Patterns are now persisted via memoryStore.storePattern() and queried via queryPatternsByConfidence()
|
||
this.taskCount = state.experiences.length;
|
||
this.logger.info(`Loaded learning state: ${state.experiences.length} experiences`);
|
||
}
|
||
} catch (error) {
|
||
this.logger.warn('No previous learning state found, starting fresh');
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Serialize Q-table for storage
|
||
*/
|
||
private serializeQTable(): Record<string, Record<string, number>> {
|
||
const serialized: Record<string, Record<string, number>> = {};
|
||
for (const [state, actions] of this.qTable.entries()) {
|
||
serialized[state] = Object.fromEntries(actions.entries());
|
||
}
|
||
return serialized;
|
||
}
|
||
|
||
/**
|
||
* Deserialize Q-table from storage
|
||
*/
|
||
private deserializeQTable(data: Record<string, Record<string, number>>): void {
|
||
this.qTable.clear();
|
||
for (const [state, actions] of Object.entries(data)) {
|
||
this.qTable.set(state, new Map(Object.entries(actions)));
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Serialize Q-table for QLearning import (converts to QValue format)
|
||
*/
|
||
private serializeQTableForQLearning(): Record<string, Record<string, { state: string; action: string; value: number; updateCount: number; lastUpdated: number }>> {
|
||
const serialized: Record<string, Record<string, { state: string; action: string; value: number; updateCount: number; lastUpdated: number }>> = {};
|
||
for (const [state, actions] of this.qTable.entries()) {
|
||
serialized[state] = {};
|
||
for (const [action, value] of actions.entries()) {
|
||
serialized[state][action] = {
|
||
state,
|
||
action,
|
||
value,
|
||
updateCount: 1,
|
||
lastUpdated: Date.now()
|
||
};
|
||
}
|
||
}
|
||
return serialized;
|
||
}
|
||
|
||
/**
|
||
* Deserialize Q-table from QLearning export (extracts values from QValue format)
|
||
*/
|
||
private deserializeQTableFromQLearning(data: Record<string, Record<string, { value: number }>>): void {
|
||
this.qTable.clear();
|
||
for (const [state, actions] of Object.entries(data)) {
|
||
const actionMap = new Map<string, number>();
|
||
for (const [action, qValue] of Object.entries(actions)) {
|
||
actionMap.set(action, qValue.value);
|
||
}
|
||
this.qTable.set(state, actionMap);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Calculate state size in bytes
|
||
*/
|
||
private async calculateStateSize(): Promise<number> {
|
||
const patterns = await this.getPatterns();
|
||
return JSON.stringify({
|
||
qTable: this.serializeQTable(),
|
||
experiences: this.experiences,
|
||
patterns
|
||
}).length;
|
||
}
|
||
|
||
/**
|
||
* Get current performance metrics
|
||
*/
|
||
private async getCurrentPerformance(): Promise<PerformanceMetrics> {
|
||
const recentExperiences = this.experiences.slice(-100);
|
||
const now = new Date();
|
||
const periodStart = new Date(now.getTime() - 24 * 60 * 60 * 1000); // last 24 hours
|
||
|
||
const avgReward = recentExperiences.length > 0
|
||
? recentExperiences.reduce((sum, e) => sum + e.reward, 0) / recentExperiences.length
|
||
: 0;
|
||
const successRate = recentExperiences.length > 0
|
||
? recentExperiences.filter(e => e.reward > 0).length / recentExperiences.length
|
||
: 0;
|
||
|
||
return {
|
||
agentId: this.agentId,
|
||
period: { start: periodStart, end: now },
|
||
metrics: {
|
||
tasksCompleted: recentExperiences.length,
|
||
successRate,
|
||
averageExecutionTime: 0, // Not tracked at this level
|
||
errorRate: 1 - successRate,
|
||
userSatisfaction: successRate, // Use success rate as proxy
|
||
resourceEfficiency: avgReward > 0 ? Math.min(1, avgReward) : 0
|
||
},
|
||
trends: []
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Emit learning event
|
||
*/
|
||
private async emitLearningEvent(type: string, data: unknown): Promise<void> {
|
||
const event: LearningEvent = {
|
||
id: uuidv4(),
|
||
type: type as LearningEvent['type'],
|
||
agentId: this.agentId,
|
||
data,
|
||
timestamp: new Date()
|
||
};
|
||
|
||
// Serialize event to Record<string, unknown> for type safety
|
||
const serializableEvent: Record<string, unknown> = {
|
||
...event,
|
||
timestamp: event.timestamp.toISOString()
|
||
};
|
||
|
||
await this.memoryStore.storeEvent({
|
||
type: `learning:${type}`,
|
||
payload: serializableEvent,
|
||
source: this.agentId,
|
||
timestamp: Date.now()
|
||
});
|
||
}
|
||
|
||
/**
|
||
* Get current exploration rate
|
||
*/
|
||
getExplorationRate(): number {
|
||
return this.config.explorationRate;
|
||
}
|
||
|
||
/**
|
||
* Get total experiences
|
||
*/
|
||
getTotalExperiences(): number {
|
||
return this.experiences.length;
|
||
}
|
||
|
||
/**
|
||
* Enable/disable learning
|
||
*/
|
||
setEnabled(enabled: boolean): void {
|
||
this.config.enabled = enabled;
|
||
}
|
||
|
||
/**
|
||
* Check if learning is enabled
|
||
*/
|
||
isEnabled(): boolean {
|
||
return this.config.enabled;
|
||
}
|
||
|
||
/**
|
||
* Enable Q-learning mode (Phase 2 Integration)
|
||
* Switches from basic Q-table to full QLearning algorithm with experience replay
|
||
*/
|
||
enableQLearning(config?: Partial<QLearningConfig>): void {
|
||
const qLearningConfig: Partial<QLearningConfig> = {
|
||
learningRate: this.config.learningRate,
|
||
discountFactor: this.config.discountFactor,
|
||
explorationRate: this.config.explorationRate,
|
||
explorationDecay: this.config.explorationDecay,
|
||
minExplorationRate: this.config.minExplorationRate,
|
||
useExperienceReplay: true,
|
||
replayBufferSize: 10000,
|
||
batchSize: this.config.batchSize,
|
||
...config
|
||
};
|
||
|
||
this.qLearning = new QLearning(qLearningConfig);
|
||
this.useQLearning = true;
|
||
|
||
// Import existing Q-table into QLearning if we have data
|
||
if (this.qTable.size > 0) {
|
||
const serialized = this.serializeQTableForQLearning();
|
||
this.qLearning.import({
|
||
qTable: serialized,
|
||
config: qLearningConfig as QLearningConfig,
|
||
stepCount: this.taskCount,
|
||
episodeCount: Math.floor(this.taskCount / 10)
|
||
});
|
||
}
|
||
|
||
this.logger.info(`Q-learning mode enabled for agent ${this.agentId}`, {
|
||
config: qLearningConfig
|
||
});
|
||
}
|
||
|
||
/**
|
||
* Disable Q-learning mode (revert to basic implementation)
|
||
*/
|
||
disableQLearning(): void {
|
||
if (this.qLearning && this.useQLearning) {
|
||
// Export Q-learning state to basic Q-table
|
||
const exported = this.qLearning.export();
|
||
this.deserializeQTableFromQLearning(exported.qTable);
|
||
}
|
||
|
||
this.qLearning = undefined;
|
||
this.useQLearning = false;
|
||
|
||
this.logger.info(`Q-learning mode disabled for agent ${this.agentId}`);
|
||
}
|
||
|
||
/**
|
||
* Learn from experience using Q-learning (when enabled)
|
||
* This method integrates with the QLearning algorithm
|
||
*/
|
||
async learnFromExperience(experience: TaskExperience): Promise<void> {
|
||
if (!this.config.enabled) {
|
||
return;
|
||
}
|
||
|
||
if (this.useQLearning && this.qLearning) {
|
||
// Use QLearning algorithm
|
||
this.qLearning.update(experience);
|
||
this.experiences.push(experience);
|
||
|
||
// Perform batch update periodically
|
||
if (this.taskCount % this.config.updateFrequency === 0) {
|
||
this.qLearning.batchUpdate();
|
||
}
|
||
|
||
// End episode periodically to trigger exploration decay
|
||
if (this.taskCount % 10 === 0) {
|
||
this.qLearning.endEpisode();
|
||
}
|
||
} else {
|
||
// Use legacy Q-table implementation
|
||
await this.updateQTable(experience);
|
||
this.experiences.push(experience);
|
||
}
|
||
|
||
this.taskCount++;
|
||
// Fix: Decay exploration rate after each experience
|
||
this.decayExploration();
|
||
}
|
||
|
||
/**
|
||
* Select action with policy (Q-learning integration)
|
||
* Uses epsilon-greedy policy when Q-learning is enabled
|
||
*/
|
||
async selectActionWithPolicy(state: TaskState, availableActions: AgentAction[]): Promise<AgentAction> {
|
||
if (this.useQLearning && this.qLearning) {
|
||
// Use Q-learning's epsilon-greedy policy
|
||
return this.qLearning.selectAction(state, availableActions);
|
||
}
|
||
|
||
// Fallback to recommendation-based selection
|
||
const recommendation = await this.recommendStrategy(state);
|
||
|
||
// Find the action matching the recommended strategy
|
||
const matchingAction = availableActions.find(
|
||
action => action.strategy === recommendation.strategy
|
||
);
|
||
|
||
return matchingAction || availableActions[0];
|
||
}
|
||
|
||
/**
|
||
* Get Q-learning statistics (when enabled)
|
||
*/
|
||
getQLearningStats(): {
|
||
enabled: boolean;
|
||
stats?: {
|
||
steps: number;
|
||
episodes: number;
|
||
tableSize: number;
|
||
explorationRate: number;
|
||
avgQValue: number;
|
||
maxQValue: number;
|
||
minQValue: number;
|
||
};
|
||
} {
|
||
if (!this.useQLearning || !this.qLearning) {
|
||
return { enabled: false };
|
||
}
|
||
|
||
return {
|
||
enabled: true,
|
||
stats: this.qLearning.getStatistics()
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Check if Q-learning mode is enabled
|
||
*/
|
||
isQLearningEnabled(): boolean {
|
||
return this.useQLearning;
|
||
}
|
||
|
||
/**
|
||
* Cleanup resources (timers, connections)
|
||
* Call this before destroying the LearningEngine instance
|
||
*
|
||
* Architecture: No cleanup needed - LearningEngine no longer manages Database instances.
|
||
* The shared SwarmMemoryManager handles database lifecycle management.
|
||
*/
|
||
dispose(): void {
|
||
// Clean up experience sharing listeners
|
||
if (this.experienceSharing) {
|
||
this.experienceSharing.removeAllListeners('experience_received');
|
||
this.experienceSharing = undefined;
|
||
}
|
||
|
||
this.logger.debug(`LearningEngine disposed for agent ${this.agentId}`);
|
||
}
|
||
|
||
/**
|
||
* Set the RL algorithm to use (Q-Learning, SARSA, etc.)
|
||
* Supports dynamic algorithm switching with state transfer
|
||
*/
|
||
setAlgorithm(algorithm: RLAlgorithmType): void {
|
||
if (algorithm === 'legacy') {
|
||
// Revert to legacy Q-table implementation
|
||
this.disableRLAlgorithm();
|
||
return;
|
||
}
|
||
|
||
const rlConfig = {
|
||
learningRate: this.config.learningRate,
|
||
discountFactor: this.config.discountFactor,
|
||
explorationRate: this.config.explorationRate,
|
||
explorationDecay: this.config.explorationDecay,
|
||
minExplorationRate: this.config.minExplorationRate,
|
||
useExperienceReplay: true,
|
||
replayBufferSize: 10000,
|
||
batchSize: this.config.batchSize
|
||
};
|
||
|
||
// Export current algorithm state if exists
|
||
let existingState;
|
||
if (this.rlAlgorithm) {
|
||
existingState = this.rlAlgorithm.export();
|
||
} else if (this.qLearning) {
|
||
// Migrate from legacy qLearning
|
||
existingState = this.qLearning.export();
|
||
}
|
||
|
||
// Create new algorithm instance
|
||
switch (algorithm) {
|
||
case 'q-learning':
|
||
this.rlAlgorithm = new QLearning(rlConfig);
|
||
this.qLearning = this.rlAlgorithm as QLearning; // Backward compatibility
|
||
this.useQLearning = true;
|
||
break;
|
||
case 'sarsa':
|
||
this.rlAlgorithm = new SARSALearner(rlConfig);
|
||
this.useQLearning = false; // SARSA is different from Q-Learning
|
||
break;
|
||
case 'actor-critic':
|
||
this.rlAlgorithm = new ActorCriticLearner({
|
||
...rlConfig,
|
||
actorLearningRate: rlConfig.learningRate * 0.1,
|
||
criticLearningRate: rlConfig.learningRate,
|
||
entropyCoefficient: 0.01,
|
||
temperature: 1.0,
|
||
normalizeAdvantage: true,
|
||
targetUpdateFrequency: 100
|
||
});
|
||
this.useQLearning = false;
|
||
break;
|
||
case 'ppo':
|
||
this.rlAlgorithm = new PPOLearner({
|
||
...rlConfig,
|
||
clipEpsilon: 0.2,
|
||
ppoEpochs: 4,
|
||
miniBatchSize: 32,
|
||
valueLossCoefficient: 0.5,
|
||
entropyCoefficient: 0.01,
|
||
gaeLambda: 0.95,
|
||
maxGradNorm: 0.5,
|
||
clipValueLoss: true,
|
||
policyLearningRate: rlConfig.learningRate,
|
||
valueLearningRate: rlConfig.learningRate * 3
|
||
});
|
||
this.useQLearning = false;
|
||
break;
|
||
default:
|
||
throw new Error(`Unknown RL algorithm: ${algorithm}`);
|
||
}
|
||
|
||
// Import existing state if available
|
||
if (existingState) {
|
||
this.rlAlgorithm.import(existingState);
|
||
this.logger.info(`Migrated existing Q-table to ${algorithm} (${this.rlAlgorithm.getTableSize()} state-action pairs)`);
|
||
} else if (this.qTable.size > 0) {
|
||
// Import from legacy Q-table
|
||
this.importLegacyQTable();
|
||
}
|
||
|
||
this.config.algorithm = algorithm;
|
||
this.logger.info(`Switched to ${algorithm} algorithm for agent ${this.agentId}`, { config: rlConfig });
|
||
}
|
||
|
||
/**
|
||
* Disable RL algorithm and revert to legacy Q-table
|
||
*/
|
||
private disableRLAlgorithm(): void {
|
||
if (this.rlAlgorithm) {
|
||
// Export RL algorithm state to legacy Q-table
|
||
const exported = this.rlAlgorithm.export();
|
||
this.deserializeQTableFromQLearning(exported.qTable);
|
||
} else if (this.qLearning && this.useQLearning) {
|
||
// Export from legacy qLearning
|
||
const exported = this.qLearning.export();
|
||
this.deserializeQTableFromQLearning(exported.qTable);
|
||
}
|
||
|
||
this.rlAlgorithm = undefined;
|
||
this.qLearning = undefined;
|
||
this.useQLearning = false;
|
||
this.config.algorithm = 'legacy';
|
||
|
||
this.logger.info(`Disabled RL algorithm for agent ${this.agentId}, reverted to legacy Q-table`);
|
||
}
|
||
|
||
/**
|
||
* Import legacy Q-table into current RL algorithm
|
||
*/
|
||
private importLegacyQTable(): void {
|
||
if (!this.rlAlgorithm) {
|
||
return;
|
||
}
|
||
|
||
const serialized = this.serializeQTableForQLearning();
|
||
this.rlAlgorithm.import({
|
||
qTable: serialized,
|
||
config: {
|
||
learningRate: this.config.learningRate,
|
||
discountFactor: this.config.discountFactor,
|
||
explorationRate: this.config.explorationRate,
|
||
explorationDecay: this.config.explorationDecay,
|
||
minExplorationRate: this.config.minExplorationRate,
|
||
useExperienceReplay: true,
|
||
replayBufferSize: 10000,
|
||
batchSize: this.config.batchSize
|
||
},
|
||
stepCount: this.taskCount,
|
||
episodeCount: Math.floor(this.taskCount / 10)
|
||
});
|
||
|
||
this.logger.info(`Imported legacy Q-table to ${this.config.algorithm} (${this.rlAlgorithm.getTableSize()} state-action pairs)`);
|
||
}
|
||
|
||
/**
|
||
* Get current RL algorithm type
|
||
*/
|
||
getAlgorithm(): RLAlgorithmType {
|
||
return this.config.algorithm ?? 'legacy';
|
||
}
|
||
|
||
/**
|
||
* Get RL algorithm instance (for advanced use)
|
||
*/
|
||
getRLAlgorithm(): AbstractRLLearner | undefined {
|
||
return this.rlAlgorithm;
|
||
}
|
||
|
||
/**
|
||
* Get algorithm-specific statistics
|
||
*/
|
||
getAlgorithmStats(): {
|
||
algorithm: RLAlgorithmType;
|
||
stats?: ReturnType<AbstractRLLearner['getStatistics']>;
|
||
} {
|
||
if (this.rlAlgorithm) {
|
||
return {
|
||
algorithm: this.config.algorithm ?? 'legacy',
|
||
stats: this.rlAlgorithm.getStatistics()
|
||
};
|
||
}
|
||
|
||
return {
|
||
algorithm: 'legacy'
|
||
};
|
||
}
|
||
|
||
// ============================================================================
|
||
// Experience Sharing Integration
|
||
// ============================================================================
|
||
|
||
/**
|
||
* Enable experience sharing with other agents
|
||
* Connects this LearningEngine to an ExperienceSharingProtocol
|
||
*
|
||
* @param protocol - The ExperienceSharingProtocol instance to use
|
||
*/
|
||
enableExperienceSharing(protocol: ExperienceSharingProtocol): void {
|
||
this.experienceSharing = protocol;
|
||
this.config.enableExperienceSharing = true;
|
||
|
||
// Subscribe to incoming experiences from other agents
|
||
protocol.on('experience_received', async (event: { experienceId: string; sourceAgentId: string }) => {
|
||
await this.handleReceivedExperience(event.experienceId);
|
||
});
|
||
|
||
this.logger.info(`Experience sharing enabled for agent ${this.agentId}`);
|
||
}
|
||
|
||
/**
|
||
* Disable experience sharing
|
||
*/
|
||
disableExperienceSharing(): void {
|
||
if (this.experienceSharing) {
|
||
this.experienceSharing.removeAllListeners('experience_received');
|
||
this.experienceSharing = undefined;
|
||
}
|
||
this.config.enableExperienceSharing = false;
|
||
|
||
this.logger.info(`Experience sharing disabled for agent ${this.agentId}`);
|
||
}
|
||
|
||
/**
|
||
* Check if experience sharing is enabled
|
||
*/
|
||
isExperienceSharingEnabled(): boolean {
|
||
return this.config.enableExperienceSharing === true && this.experienceSharing !== undefined;
|
||
}
|
||
|
||
/**
|
||
* Get the ExperienceSharingProtocol instance (if enabled)
|
||
*/
|
||
getExperienceSharingProtocol(): ExperienceSharingProtocol | undefined {
|
||
return this.experienceSharing;
|
||
}
|
||
|
||
/**
|
||
* Share an experience with other agents
|
||
* Called internally after learning from execution when sharing is enabled
|
||
*
|
||
* @param experience - The experience to share
|
||
* @param priority - Optional priority override (0-1, higher = more valuable)
|
||
*/
|
||
async shareExperienceWithPeers(experience: TaskExperience, priority?: number): Promise<string | undefined> {
|
||
if (!this.experienceSharing) {
|
||
return undefined;
|
||
}
|
||
|
||
// Calculate priority from reward if not provided
|
||
const calculatedPriority = priority ?? this.calculateExperiencePriority(experience);
|
||
|
||
// Only share if priority meets threshold
|
||
if (calculatedPriority < (this.config.experienceSharingPriority ?? 0.5)) {
|
||
this.logger.debug(`Experience below sharing threshold: ${calculatedPriority} < ${this.config.experienceSharingPriority}`);
|
||
return undefined;
|
||
}
|
||
|
||
try {
|
||
const experienceId = await this.experienceSharing.shareExperience(experience, calculatedPriority);
|
||
this.logger.debug(`Shared experience with peers: ${experienceId}`, { priority: calculatedPriority });
|
||
return experienceId;
|
||
} catch (error) {
|
||
this.logger.warn('Failed to share experience with peers:', error);
|
||
return undefined;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Calculate the priority/value of an experience for sharing
|
||
* Higher rewards and successful experiences get higher priority
|
||
*/
|
||
private calculateExperiencePriority(experience: TaskExperience): number {
|
||
// Base priority from reward (normalized from [-2, 2] to [0, 1])
|
||
const rewardPriority = (experience.reward + 2) / 4;
|
||
|
||
// Boost for successful experiences (positive reward)
|
||
const successBoost = experience.reward > 0 ? 0.2 : 0;
|
||
|
||
// Boost for unusual/interesting states (high complexity)
|
||
const complexityBoost = experience.state.taskComplexity > 0.7 ? 0.1 : 0;
|
||
|
||
return Math.min(1.0, Math.max(0, rewardPriority + successBoost + complexityBoost));
|
||
}
|
||
|
||
/**
|
||
* Handle an experience received from another agent
|
||
* Integrates the shared experience into local learning
|
||
*/
|
||
private async handleReceivedExperience(experienceId: string): Promise<void> {
|
||
if (!this.experienceSharing) {
|
||
return;
|
||
}
|
||
|
||
try {
|
||
// Get the shared experience from the protocol
|
||
const sharedExperiences = await this.experienceSharing.getRelevantExperiences(
|
||
{ taskComplexity: 0, requiredCapabilities: [], contextFeatures: {}, previousAttempts: 0, availableResources: 1 },
|
||
100
|
||
);
|
||
|
||
const sharedExp = sharedExperiences.find(e => e.id === experienceId);
|
||
if (!sharedExp) {
|
||
return; // Experience not found or already processed
|
||
}
|
||
|
||
// Learn from the shared experience (with reduced weight)
|
||
// Apply a transfer learning discount factor
|
||
const transferDiscount = 0.5; // Shared experiences weighted 50%
|
||
const adjustedExperience = {
|
||
...sharedExp.experience,
|
||
reward: sharedExp.experience.reward * transferDiscount
|
||
};
|
||
|
||
// Update Q-table with shared experience
|
||
await this.updateQTable(adjustedExperience);
|
||
|
||
// Store in local experiences buffer
|
||
this.experiences.push(adjustedExperience);
|
||
|
||
this.logger.debug(`Integrated shared experience: ${experienceId}`, {
|
||
from: sharedExp.sourceAgentId,
|
||
originalReward: sharedExp.experience.reward,
|
||
adjustedReward: adjustedExperience.reward
|
||
});
|
||
} catch (error) {
|
||
this.logger.warn(`Failed to handle received experience ${experienceId}:`, error);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Query relevant experiences from peers for a given state
|
||
* Useful for transfer learning before executing a task
|
||
*
|
||
* @param state - The current task state
|
||
* @param limit - Maximum number of experiences to retrieve
|
||
*/
|
||
async queryPeerExperiences(state: TaskState, limit: number = 10): Promise<TaskExperience[]> {
|
||
if (!this.experienceSharing) {
|
||
return [];
|
||
}
|
||
|
||
try {
|
||
const sharedExperiences = await this.experienceSharing.getRelevantExperiences(state, limit);
|
||
return sharedExperiences.map(se => se.experience);
|
||
} catch (error) {
|
||
this.logger.warn('Failed to query peer experiences:', error);
|
||
return [];
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Get experience sharing statistics
|
||
*/
|
||
getExperienceSharingStats(): {
|
||
enabled: boolean;
|
||
stats?: {
|
||
experiencesShared: number;
|
||
experiencesReceived: number;
|
||
activeConnections: number;
|
||
};
|
||
} {
|
||
if (!this.experienceSharing) {
|
||
return { enabled: false };
|
||
}
|
||
|
||
const stats = this.experienceSharing.getStats();
|
||
return {
|
||
enabled: true,
|
||
stats: {
|
||
experiencesShared: stats.experiencesShared,
|
||
experiencesReceived: stats.experiencesReceived,
|
||
activeConnections: stats.activeConnections
|
||
}
|
||
};
|
||
}
|
||
}
|