package memory import ( "fmt" "os" "path/filepath" "time" ) // Result reports what a Reconcile pass changed: Indexed counts rows inserted or // updated (new or changed files), Pruned counts rows deleted because their file // no longer exists on disk. type Result struct { Indexed int Pruned int } // walkMemoryDir recursively collects every *.md file under root. A missing root // (ENOENT) yields an empty slice and no error, so reconcile is safe to run // before the memory directory has been created. func walkMemoryDir(root string) ([]string, error) { var out []string var recurse func(dir string) error recurse = func(dir string) error { entries, err := os.ReadDir(dir) if err != nil { if os.IsNotExist(err) { return nil } return err } for _, entry := range entries { full := filepath.Join(dir, entry.Name()) if entry.IsDir() { if err := recurse(full); err != nil { return err } } else if entry.Type().IsRegular() && isMarkdown(entry.Name()) { out = append(out, full) } } return nil } if err := recurse(root); err != nil { return nil, err } return out, nil } // walkCcRoot collects every //memory/**/*.md file. A missing base // (ENOENT) yields an empty slice; slugs without a memory subdirectory are // silently skipped. func walkCcRoot(base string) ([]string, error) { slugs, err := os.ReadDir(base) if err != nil { if os.IsNotExist(err) { return nil, nil } return nil, err } var out []string for _, entry := range slugs { if !entry.IsDir() { continue } memoryDir := filepath.Join(base, entry.Name(), "memory") info, err := os.Stat(memoryDir) if err != nil || !info.IsDir() { continue } files, err := walkMemoryDir(memoryDir) if err != nil { return nil, err } out = append(out, files...) } return out, nil } // Reconcile performs a lazy sync between the memory files on disk and the // memory_index table. It walks both the mimo root and (when configured) the cc // base, prunes rows whose file no longer exists, and indexes new or changed // files. Unchanged files are skipped via a size-mtime fingerprint. The FTS // index is kept consistent by the memory_ai/ad/au triggers, so only // memory_index is touched here. func (s *Store) Reconcile() (Result, error) { var res Result // Collect disk paths from BOTH roots BEFORE pruning. Pruning per-root would // wrongly wipe the other root's rows, because each walk's set is missing the // other root's paths. mimoFiles, err := walkMemoryDir(s.root) if err != nil { return res, fmt.Errorf("memory: walk mimo root %q: %w", s.root, err) } var ccFiles []string if s.ccBase != "" { ccFiles, err = walkCcRoot(s.ccBase) if err != nil { return res, fmt.Errorf("memory: walk cc base %q: %w", s.ccBase, err) } } diskPaths := make(map[string]struct{}, len(mimoFiles)+len(ccFiles)) for _, p := range mimoFiles { diskPaths[filepath.Clean(p)] = struct{}{} } for _, p := range ccFiles { diskPaths[filepath.Clean(p)] = struct{}{} } // Load existing {path -> fingerprint} from memory_index. existing, err := s.loadFingerprints() if err != nil { return res, err } // PRUNE: delete rows whose path is no longer on disk. for p := range existing { if _, ok := diskPaths[p]; ok { continue } if _, err := s.db.Exec(`DELETE FROM memory_index WHERE path = ?`, p); err != nil { return res, fmt.Errorf("memory: prune %q: %w", p, err) } res.Pruned++ } // INDEX: mimo files use parsePath and keep loc.Type. for _, p := range mimoFiles { loc, ok := parsePath(s.root, p) if !ok { continue } updated, err := s.indexFile(loc, loc.Type, false, existing[filepath.Clean(p)]) if err != nil { return res, err } if updated { res.Indexed++ } } // INDEX: cc files use parseCcPath; final type is derived from frontmatter. for _, p := range ccFiles { loc, ok := parseCcPath(s.ccBase, p) if !ok { continue } updated, err := s.indexFile(loc, loc.Type, true, existing[filepath.Clean(p)]) if err != nil { return res, err } if updated { res.Indexed++ } } return res, nil } // loadFingerprints returns the current {path -> fingerprint} map from // memory_index. func (s *Store) loadFingerprints() (map[string]string, error) { rows, err := s.db.Query(`SELECT path, fingerprint FROM memory_index`) if err != nil { return nil, fmt.Errorf("memory: load fingerprints: %w", err) } defer rows.Close() out := make(map[string]string) for rows.Next() { var path, fp string if err := rows.Scan(&path, &fp); err != nil { return nil, fmt.Errorf("memory: scan fingerprint: %w", err) } out[path] = fp } if err := rows.Err(); err != nil { return nil, fmt.Errorf("memory: iterate fingerprints: %w", err) } return out, nil } // indexFile stats loc.Path, computes its size-mtime fingerprint, and upserts the // row when the fingerprint differs from oldFingerprint. It returns true when a // row was inserted or updated. A file that vanished between the walk and the // stat (ENOENT) is silently skipped. For cc files (isCc) the semantic type is // derived from the file's YAML frontmatter, falling back to defaultType. func (s *Store) indexFile(loc *Locator, defaultType Type, isCc bool, oldFingerprint string) (bool, error) { info, err := os.Stat(loc.Path) if err != nil { if os.IsNotExist(err) { return false, nil } return false, fmt.Errorf("memory: stat %q: %w", loc.Path, err) } fingerprint := fmt.Sprintf("%d-%d", info.Size(), info.ModTime().UnixNano()) if oldFingerprint == fingerprint { return false, nil // hit: unchanged file } raw, err := os.ReadFile(loc.Path) if err != nil { if os.IsNotExist(err) { return false, nil } return false, fmt.Errorf("memory: read %q: %w", loc.Path, err) } body := string(raw) finalType := defaultType if isCc { finalType = parseCcFrontmatterType(body) } now := time.Now().UnixNano() const upsert = ` INSERT INTO memory_index (path, scope, scope_id, type, body, fingerprint, last_indexed_at) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(path) DO UPDATE SET scope = excluded.scope, scope_id = excluded.scope_id, type = excluded.type, body = excluded.body, fingerprint = excluded.fingerprint, last_indexed_at = excluded.last_indexed_at` if _, err := s.db.Exec(upsert, loc.Path, string(loc.Scope), loc.ScopeID, string(finalType), body, fingerprint, now, ); err != nil { return false, fmt.Errorf("memory: upsert %q: %w", loc.Path, err) } return true, nil }