zCode-CLI-X/src/bot/memory-backend.js

/**
 * zCode Memory Backend — Enhanced with typed memory entries
 * Ported concept from Ruflo's MemoryBackend interface.
 *
 * Two backends:
 *   JSONBackend  — file-based, LRU (existing MemoryStore)
 *   InMemoryBackend — RAM-only, for ephemeral agent context
 *
 * Memory types: lesson, gotcha, pattern, preference, discovery, context
 */

import { logger } from '../utils/logger.js';
import fs from 'fs';
import path from 'path';

const MEMORY_TYPES = {
  LESSON: 'lesson',
  GOTCHA: 'gotcha',
  PATTERN: 'pattern',
  PREFERENCE: 'preference',
  DISCOVERY: 'discovery',
  CONTEXT: 'context',
  EPHEMERAL: 'ephemeral',
};

/** Priority for system prompt injection */
const TYPE_PRIORITY = {
  gotcha: 5,
  lesson: 4,
  pattern: 3,
  preference: 2,
  discovery: 1,
  context: 3,
  ephemeral: 0,
};

/**
 * JSONBackend — File-based persistent memory with LRU eviction
 */
export class JSONBackend {
  constructor(filePath, maxEntries = 500) {
    this.filePath = path.resolve(filePath);
    this.maxEntries = maxEntries;
    this._entries = new Map();
    this._loaded = false;
    this._dirty = false;
    this._saveTimer = null;
    this._debounceMs = 3000;
  }

  get loaded() { return this._loaded; }

  async initialize() {
    try {
      if (fs.existsSync(this.filePath)) {
        const data = JSON.parse(await fs.promises.readFile(this.filePath, 'utf-8'));
        if (Array.isArray(data)) {
          for (const entry of data) {
            this._entries.set(entry.id || entry.key, entry);
          }
        } else if (data.entries) {
          for (const entry of data.entries) {
            this._entries.set(entry.id || entry.key, entry);
          }
        }
      }
      this._loaded = true;
      logger.info(`✓ Memory: loaded ${this._entries.size} entries from ${path.basename(this.filePath)}`);
    } catch (err) {
      this._loaded = true;
      logger.warn(`⚠ Memory: could not load ${path.basename(this.filePath)}: ${err.message}`);
    }
  }

  async store(memory) {
    const key = memory.id || memory.key || `${memory.type}_${Date.now()}`;
    const entry = {
      ...memory,
      id: key,
      timestamp: memory.timestamp || Date.now(),
    };

    this._entries.set(key, entry);
    this._evictIfNeeded();
    this._markDirty();
    return entry;
  }

  async retrieve(id) {
    return this._entries.get(id) || null;
  }

  async query(filter) {
    let results = [...this._entries.values()];

    if (filter.type) {
      results = results.filter(e => e.type === filter.type);
    }
    if (filter.query) {
      const q = filter.query.toLowerCase();
      results = results.filter(e =>
        (e.content && e.content.toLowerCase().includes(q)) ||
        (e.key && e.key.toLowerCase().includes(q)) ||
        (e.tags && e.tags.some(t => t.toLowerCase().includes(q)))
      );
    }
    if (filter.agentId) {
      results = results.filter(e => e.agentId === filter.agentId);
    }
    if (filter.timeRange) {
      const { start, end } = filter.timeRange;
      results = results.filter(e => e.timestamp >= start && e.timestamp <= end);
    }

    // Sort by recency
    results.sort((a, b) => (b.timestamp || 0) - (a.timestamp || 0));

    if (filter.limit) results = results.slice(0, filter.limit);
    if (filter.offset) results = results.slice(filter.offset);

    return results;
  }

  /** Semantic-like search with BM25 scoring */
  async search(query, limit = 10) {
    const q = query.toLowerCase();
    const terms = q.split(/\s+/).filter(Boolean);
    if (terms.length === 0) return [];

    const scored = [...this._entries.values()].map(e => {
      let score = 0;
      const content = (e.content || '').toLowerCase();
      const key = (e.key || '').toLowerCase();
      const tags = (e.tags || []).join(' ').toLowerCase();

      for (const term of terms) {
        if (key.includes(term)) score += 10;
        if (content.includes(term)) score += 3;
        if (tags.includes(term)) score += 5;

        // TF-like scoring
        const tf = (content.match(new RegExp(term, 'g')) || []).length;
        score += Math.min(tf, 5);
      }

      // Priority boost
      score += TYPE_PRIORITY[e.type] || 0;

      // Recency boost
      const age = Date.now() - (e.timestamp || 0);
      score += Math.max(0, 1 - age / (30 * 24 * 60 * 60 * 1000)) * 5;

      return { entry: e, score };
    });

    scored.sort((a, b) => b.score - a.score);
    return scored.slice(0, limit).map(s => s.entry);
  }

async delete(id) {
    this._entries.delete(id);
    this._markDirty();
  }

  async clearType(type) {
    for (const [key, entry] of this._entries) {
      if (entry.type === type) this._entries.delete(key);
    }
    this._markDirty();
  }

  async flush() {
    clearTimeout(this._saveTimer);
    await this._save();
  }

  getAll() {
    // Group entries by type
    const grouped = {
      lesson: [], gotcha: [], pattern: [], preference: [], discovery: [], context: [],
      ephemeral: [], skill: [], conversation: [], error: []
    };
    for (const entry of this._entries.values()) {
      if (grouped[entry.type]) {
        grouped[entry.type].push(entry);
      }
    }
    return grouped;
  }

  async flush() {
    clearTimeout(this._saveTimer);
    await this._save();
  }

  getCount() { return this._entries.size; }
  getEntries() { return [...this._entries.values()]; }
  getStats() {
    const byType = {};
    for (const e of this._entries.values()) {
      byType[e.type] = (byType[e.type] || 0) + 1;
    }
    return { total: this._entries.size, byType };
  }

  _evictIfNeeded() {
    if (this._entries.size <= this.maxEntries) return;
    const sorted = [...this._entries.values()].sort((a, b) => (a.timestamp || 0) - (b.timestamp || 0));
    const toRemove = this._entries.size - this.maxEntries;
    for (let i = 0; i < toRemove && i < sorted.length; i++) {
      this._entries.delete(sorted[i].id || sorted[i].key);
    }
  }

  _markDirty() {
    this._dirty = true;
    if (this._saveTimer) clearTimeout(this._saveTimer);
    this._saveTimer = setTimeout(() => this._save(), this._debounceMs);
  }

  async _save() {
    try {
      const dir = path.dirname(this.filePath);
      if (!fs.existsSync(dir)) {
        fs.mkdirSync(dir, { recursive: true });
      }
      const data = [...this._entries.values()];
      await fs.promises.writeFile(this.filePath, JSON.stringify(data, null, 2), 'utf-8');
      this._dirty = false;
    } catch (err) {
      logger.error(`Memory save failed: ${err.message}`);
    }
  }
}

/**
 * InMemoryBackend — RAM-only, for ephemeral agent context
 * Auto-evicts after TTL or max entries
 */
export class InMemoryBackend {
  constructor(maxEntries = 200, ttlMs = 30 * 60 * 1000) {
    this._entries = new Map();
    this.maxEntries = maxEntries;
    this.ttlMs = ttlMs;
  }

  async store(memory) {
    const key = memory.id || memory.key || `mem_${Date.now()}`;
    const entry = {
      ...memory,
      id: key,
      timestamp: memory.timestamp || Date.now(),
      _ttl: Date.now() + this.ttlMs,
    };
    this._entries.set(key, entry);
    this._evictIfNeeded();
    return entry;
  }

  async retrieve(id) {
    const entry = this._entries.get(id);
    if (!entry) return null;
    if (Date.now() > entry._ttl) {
      this._entries.delete(id);
      return null;
    }
    return entry;
  }

  async query(filter) {
    this._purgeExpired();
    let results = [...this._entries.values()];
    if (filter.type) results = results.filter(e => e.type === filter.type);
    if (filter.query) {
      const q = filter.query.toLowerCase();
      results = results.filter(e => (e.content || '').toLowerCase().includes(q));
    }
    results.sort((a, b) => (b.timestamp || 0) - (a.timestamp || 0));
    if (filter.limit) results = results.slice(0, filter.limit);
    return results;
  }

  getCount() {
    this._purgeExpired();
    return this._entries.size;
  }

  _evictIfNeeded() {
    if (this._entries.size <= this.maxEntries) return;
    const sorted = [...this._entries.values()].sort((a, b) => (a.timestamp || 0) - (b.timestamp || 0));
    const toRemove = this._entries.size - this.maxEntries;
    for (let i = 0; i < toRemove; i++) {
      this._entries.delete(sorted[i].id);
    }
  }

  _purgeExpired() {
    const now = Date.now();
    for (const [key, entry] of this._entries) {
      if (now > entry._ttl) this._entries.delete(key);
    }
  }
}

export { MEMORY_TYPES };
export default JSONBackend;