first commit

2026-08-14 23:41:57 +08:00
commit 086803a8dd
471 changed files with 91938 additions and 0 deletions
@@ -0,0 +1,235 @@
+package memory
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+// Result reports what a Reconcile pass changed: Indexed counts rows inserted or
+// updated (new or changed files), Pruned counts rows deleted because their file
+// no longer exists on disk.
+type Result struct {
+	Indexed int
+	Pruned  int
+}
+
+// walkMemoryDir recursively collects every *.md file under root. A missing root
+// (ENOENT) yields an empty slice and no error, so reconcile is safe to run
+// before the memory directory has been created.
+func walkMemoryDir(root string) ([]string, error) {
+	var out []string
+	var recurse func(dir string) error
+	recurse = func(dir string) error {
+		entries, err := os.ReadDir(dir)
+		if err != nil {
+			if os.IsNotExist(err) {
+				return nil
+			}
+			return err
+		}
+		for _, entry := range entries {
+			full := filepath.Join(dir, entry.Name())
+			if entry.IsDir() {
+				if err := recurse(full); err != nil {
+					return err
+				}
+			} else if entry.Type().IsRegular() && isMarkdown(entry.Name()) {
+				out = append(out, full)
+			}
+		}
+		return nil
+	}
+	if err := recurse(root); err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// walkCcRoot collects every <base>/<slug>/memory/**/*.md file. A missing base
+// (ENOENT) yields an empty slice; slugs without a memory subdirectory are
+// silently skipped.
+func walkCcRoot(base string) ([]string, error) {
+	slugs, err := os.ReadDir(base)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	var out []string
+	for _, entry := range slugs {
+		if !entry.IsDir() {
+			continue
+		}
+		memoryDir := filepath.Join(base, entry.Name(), "memory")
+		info, err := os.Stat(memoryDir)
+		if err != nil || !info.IsDir() {
+			continue
+		}
+		files, err := walkMemoryDir(memoryDir)
+		if err != nil {
+			return nil, err
+		}
+		out = append(out, files...)
+	}
+	return out, nil
+}
+
+// Reconcile performs a lazy sync between the memory files on disk and the
+// memory_index table. It walks both the mimo root and (when configured) the cc
+// base, prunes rows whose file no longer exists, and indexes new or changed
+// files. Unchanged files are skipped via a size-mtime fingerprint. The FTS
+// index is kept consistent by the memory_ai/ad/au triggers, so only
+// memory_index is touched here.
+func (s *Store) Reconcile() (Result, error) {
+	var res Result
+
+	// Collect disk paths from BOTH roots BEFORE pruning. Pruning per-root would
+	// wrongly wipe the other root's rows, because each walk's set is missing the
+	// other root's paths.
+	mimoFiles, err := walkMemoryDir(s.root)
+	if err != nil {
+		return res, fmt.Errorf("memory: walk mimo root %q: %w", s.root, err)
+	}
+	var ccFiles []string
+	if s.ccBase != "" {
+		ccFiles, err = walkCcRoot(s.ccBase)
+		if err != nil {
+			return res, fmt.Errorf("memory: walk cc base %q: %w", s.ccBase, err)
+		}
+	}
+
+	diskPaths := make(map[string]struct{}, len(mimoFiles)+len(ccFiles))
+	for _, p := range mimoFiles {
+		diskPaths[filepath.Clean(p)] = struct{}{}
+	}
+	for _, p := range ccFiles {
+		diskPaths[filepath.Clean(p)] = struct{}{}
+	}
+
+	// Load existing {path -> fingerprint} from memory_index.
+	existing, err := s.loadFingerprints()
+	if err != nil {
+		return res, err
+	}
+
+	// PRUNE: delete rows whose path is no longer on disk.
+	for p := range existing {
+		if _, ok := diskPaths[p]; ok {
+			continue
+		}
+		if _, err := s.db.Exec(`DELETE FROM memory_index WHERE path = ?`, p); err != nil {
+			return res, fmt.Errorf("memory: prune %q: %w", p, err)
+		}
+		res.Pruned++
+	}
+
+	// INDEX: mimo files use parsePath and keep loc.Type.
+	for _, p := range mimoFiles {
+		loc, ok := parsePath(s.root, p)
+		if !ok {
+			continue
+		}
+		updated, err := s.indexFile(loc, loc.Type, false, existing[filepath.Clean(p)])
+		if err != nil {
+			return res, err
+		}
+		if updated {
+			res.Indexed++
+		}
+	}
+
+	// INDEX: cc files use parseCcPath; final type is derived from frontmatter.
+	for _, p := range ccFiles {
+		loc, ok := parseCcPath(s.ccBase, p)
+		if !ok {
+			continue
+		}
+		updated, err := s.indexFile(loc, loc.Type, true, existing[filepath.Clean(p)])
+		if err != nil {
+			return res, err
+		}
+		if updated {
+			res.Indexed++
+		}
+	}
+
+	return res, nil
+}
+
+// loadFingerprints returns the current {path -> fingerprint} map from
+// memory_index.
+func (s *Store) loadFingerprints() (map[string]string, error) {
+	rows, err := s.db.Query(`SELECT path, fingerprint FROM memory_index`)
+	if err != nil {
+		return nil, fmt.Errorf("memory: load fingerprints: %w", err)
+	}
+	defer rows.Close()
+
+	out := make(map[string]string)
+	for rows.Next() {
+		var path, fp string
+		if err := rows.Scan(&path, &fp); err != nil {
+			return nil, fmt.Errorf("memory: scan fingerprint: %w", err)
+		}
+		out[path] = fp
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("memory: iterate fingerprints: %w", err)
+	}
+	return out, nil
+}
+
+// indexFile stats loc.Path, computes its size-mtime fingerprint, and upserts the
+// row when the fingerprint differs from oldFingerprint. It returns true when a
+// row was inserted or updated. A file that vanished between the walk and the
+// stat (ENOENT) is silently skipped. For cc files (isCc) the semantic type is
+// derived from the file's YAML frontmatter, falling back to defaultType.
+func (s *Store) indexFile(loc *Locator, defaultType Type, isCc bool, oldFingerprint string) (bool, error) {
+	info, err := os.Stat(loc.Path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, fmt.Errorf("memory: stat %q: %w", loc.Path, err)
+	}
+
+	fingerprint := fmt.Sprintf("%d-%d", info.Size(), info.ModTime().UnixNano())
+	if oldFingerprint == fingerprint {
+		return false, nil // hit: unchanged file
+	}
+
+	raw, err := os.ReadFile(loc.Path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, fmt.Errorf("memory: read %q: %w", loc.Path, err)
+	}
+	body := string(raw)
+
+	finalType := defaultType
+	if isCc {
+		finalType = parseCcFrontmatterType(body)
+	}
+
+	now := time.Now().UnixNano()
+	const upsert = `
+INSERT INTO memory_index (path, scope, scope_id, type, body, fingerprint, last_indexed_at)
+VALUES (?, ?, ?, ?, ?, ?, ?)
+ON CONFLICT(path) DO UPDATE SET
+  scope = excluded.scope,
+  scope_id = excluded.scope_id,
+  type = excluded.type,
+  body = excluded.body,
+  fingerprint = excluded.fingerprint,
+  last_indexed_at = excluded.last_indexed_at`
+	if _, err := s.db.Exec(upsert,
+		loc.Path, string(loc.Scope), loc.ScopeID, string(finalType), body, fingerprint, now,
+	); err != nil {
+		return false, fmt.Errorf("memory: upsert %q: %w", loc.Path, err)
+	}
+	return true, nil
+}