first commit
This commit is contained in:
@@ -0,0 +1,247 @@
|
||||
package dream
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/smallnest/pigo/internal/agentcore"
|
||||
"github.com/smallnest/pigo/internal/provider"
|
||||
)
|
||||
|
||||
// This file wires the llmConsolidator to a real provider (the main-session
|
||||
// model, SPEC Q3) and holds the MEMORY.md index cleanup the Runner runs after
|
||||
// writeback. The provider plumbing mirrors internal/cli/run.SetupEnv: resolve a
|
||||
// Provider for the model/base-url/protocol/provider tuple, resolve the API key
|
||||
// through a CredentialStore (--api-key override → env → config), and run a
|
||||
// single StreamCompletion, draining the event stream to the final text.
|
||||
|
||||
// NewLLMConsolidator builds the production Consolidator backed by the given
|
||||
// model configuration — the same tuple cmd/pigo resolves for the main session
|
||||
// (CLI flags overlaid with config.toml). It resolves the Provider once so every
|
||||
// Consolidate call reuses it. A resolution failure (bad model / missing
|
||||
// provider) is returned so the caller can decide whether to fall back to the
|
||||
// no-op Consolidator or fail the run.
|
||||
func NewLLMConsolidator(model, baseURL, protocol, providerName, apiKey string, thinking agentcore.ThinkingLevel) (Consolidator, error) {
|
||||
complete, err := newModelCompleter(model, baseURL, protocol, providerName, apiKey, thinking)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &llmConsolidator{complete: complete}, nil
|
||||
}
|
||||
|
||||
// newModelCompleter resolves the provider and returns a completeFn that performs
|
||||
// one non-streaming-consuming completion: it sends the system+user prompt as a
|
||||
// single user turn (no tools — the dream agent only reasons and replies) and
|
||||
// returns the concatenated assistant text. A hard "cannot build the stream"
|
||||
// error is returned directly; a runtime failure rides the stream as a terminal
|
||||
// error event whose message we convert to an error (so the Runner marks the run
|
||||
// failed rather than silently deleting nothing, SPEC §5.5).
|
||||
func newModelCompleter(model, baseURL, protocol, providerName, apiKey string, thinking agentcore.ThinkingLevel) (completeFn, error) {
|
||||
prov, resolvedName, err := provider.ResolveProvider(model, baseURL, protocol, providerName, os.Getenv)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dream: resolve provider: %w", err)
|
||||
}
|
||||
creds := provider.NewCredentialStore(nil)
|
||||
creds.SetOverride(resolvedName, apiKey)
|
||||
|
||||
return func(ctx context.Context, systemPrompt, userPrompt string) (string, error) {
|
||||
key := creds.GetAPIKey(ctx, resolvedName)
|
||||
llm := provider.LlmContext{
|
||||
SystemPrompt: systemPrompt,
|
||||
Messages: agentcore.MessageList{
|
||||
agentcore.UserMessage{
|
||||
RoleField: agentcore.RoleUser,
|
||||
Content: agentcore.ContentList{agentcore.NewTextContent(userPrompt)},
|
||||
},
|
||||
},
|
||||
}
|
||||
stream, err := prov.StreamCompletion(ctx, provider.CompletionRequest{
|
||||
Model: model,
|
||||
Context: llm,
|
||||
Config: provider.StreamConfig{APIKey: key, ThinkingLevel: thinking},
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
final, err := drainToMessage(ctx, stream)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if final.StopReason == agentcore.StopReasonError {
|
||||
if final.ErrorMessage != "" {
|
||||
return "", fmt.Errorf("model error: %s", final.ErrorMessage)
|
||||
}
|
||||
return "", fmt.Errorf("model returned an error response")
|
||||
}
|
||||
return agentcore.ContentToText(final.Content), nil
|
||||
}, nil
|
||||
}
|
||||
|
||||
// drainToMessage consumes the provider event stream to completion and returns
|
||||
// the terminal assistant message. It mirrors the loop's stream-drain contract:
|
||||
// the done/error event carries the final message; if the stream closes without
|
||||
// one, it falls back to the stream Result. Draining is required because the
|
||||
// producer blocks on the event channel until consumed.
|
||||
func drainToMessage(ctx context.Context, stream *provider.AssistantMessageEventStream) (agentcore.AssistantMessage, error) {
|
||||
for ev := range stream.Events() {
|
||||
switch e := ev.(type) {
|
||||
case provider.StreamDoneEvent:
|
||||
return e.Message, nil
|
||||
case provider.StreamErrorEvent:
|
||||
return e.Message, nil
|
||||
}
|
||||
}
|
||||
final, err := stream.Result(ctx)
|
||||
if err != nil {
|
||||
return agentcore.AssistantMessage{}, err
|
||||
}
|
||||
return final, nil
|
||||
}
|
||||
|
||||
// updateScopeIndexes rewrites each affected scope's MEMORY.md to drop any line
|
||||
// that references a now-deleted memory file, keeping the index consistent with
|
||||
// the entries on disk and free of dangling links (PRD US-003). It is safe to
|
||||
// call when no MEMORY.md exists (no-op) and when deleted is empty. Each rewrite
|
||||
// is atomic (temp+rename) and guarded by withinScope, so it cannot escape the
|
||||
// memory store.
|
||||
func updateScopeIndexes(memoryRoot, projectDir string, deleted map[string]struct{}) error {
|
||||
if len(deleted) == 0 {
|
||||
return nil
|
||||
}
|
||||
scopes := []string{filepath.Join(memoryRoot, "global")}
|
||||
if projectDir != "" {
|
||||
scopes = append(scopes, filepath.Join(memoryRoot, "projects", projectID(projectDir)))
|
||||
}
|
||||
for _, scope := range scopes {
|
||||
index := filepath.Join(scope, "MEMORY.md")
|
||||
if _, err := os.Stat(index); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
if !withinScope(memoryRoot, projectDir, index) {
|
||||
continue
|
||||
}
|
||||
tokens := indexRefTokens(memoryRoot, scope, deleted)
|
||||
if len(tokens) == 0 {
|
||||
continue
|
||||
}
|
||||
raw, err := os.ReadFile(index)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newBody, changed := stripDanglingIndexLines(string(raw), tokens)
|
||||
if changed {
|
||||
if err := atomicWrite(index, []byte(newBody)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// indexRefTokens is the set of substrings that identify a deleted file inside a
|
||||
// MEMORY.md index line: its absolute path, its path relative to the memory root
|
||||
// and to the scope root, and its bare basename. A line containing any of these
|
||||
// is treated as a link/reference to the removed entry. MEMORY.md itself is never
|
||||
// a token (it is never a consolidation deletion target).
|
||||
func indexRefTokens(memoryRoot, scope string, deleted map[string]struct{}) map[string]struct{} {
|
||||
tokens := make(map[string]struct{})
|
||||
for p := range deleted {
|
||||
clean := filepath.Clean(p)
|
||||
add := func(s string) {
|
||||
if s != "" && s != "." {
|
||||
tokens[filepath.ToSlash(s)] = struct{}{}
|
||||
}
|
||||
}
|
||||
add(clean)
|
||||
if rel, err := filepath.Rel(memoryRoot, clean); err == nil && !strings.HasPrefix(rel, "..") {
|
||||
add(rel)
|
||||
}
|
||||
if rel, err := filepath.Rel(scope, clean); err == nil && !strings.HasPrefix(rel, "..") {
|
||||
add(rel)
|
||||
}
|
||||
add(filepath.Base(clean))
|
||||
}
|
||||
return tokens
|
||||
}
|
||||
|
||||
// stripDanglingIndexLines removes every line of body that references any of the
|
||||
// reference tokens, returning the rewritten body and whether anything changed.
|
||||
// It matches on the forward-slash form of each line so Windows-style separators
|
||||
// in the index still match the slash tokens. Matching is boundary-aware: a token
|
||||
// (e.g. the basename "b.md") only matches when it is not embedded inside a longer
|
||||
// filename token (so "club.md" or "b.mdx" is not mistaken for "b.md"), avoiding
|
||||
// dropping unrelated index lines.
|
||||
func stripDanglingIndexLines(body string, tokens map[string]struct{}) (string, bool) {
|
||||
lines := strings.Split(body, "\n")
|
||||
kept := make([]string, 0, len(lines))
|
||||
changed := false
|
||||
for _, line := range lines {
|
||||
probe := filepath.ToSlash(line)
|
||||
drop := false
|
||||
for tok := range tokens {
|
||||
if containsRefToken(probe, tok) {
|
||||
drop = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if drop {
|
||||
changed = true
|
||||
continue
|
||||
}
|
||||
kept = append(kept, line)
|
||||
}
|
||||
if !changed {
|
||||
return body, false
|
||||
}
|
||||
return strings.Join(kept, "\n"), true
|
||||
}
|
||||
|
||||
// containsRefToken reports whether tok occurs in line at a filename boundary:
|
||||
// the characters immediately before and after the match must not be filename
|
||||
// continuation characters ([A-Za-z0-9_-]). This lets "b.md" match "user/b.md",
|
||||
// "(b.md)" and "- b.md" while rejecting "club.md" and "b.mdx".
|
||||
func containsRefToken(line, tok string) bool {
|
||||
if tok == "" {
|
||||
return false
|
||||
}
|
||||
from := 0
|
||||
for {
|
||||
i := strings.Index(line[from:], tok)
|
||||
if i < 0 {
|
||||
return false
|
||||
}
|
||||
start := from + i
|
||||
end := start + len(tok)
|
||||
if !isFilenameChar(byteAt(line, start-1)) && !isFilenameChar(byteAt(line, end)) {
|
||||
return true
|
||||
}
|
||||
from = start + 1
|
||||
}
|
||||
}
|
||||
|
||||
// byteAt returns line[i], or 0 when i is out of range (treated as a boundary).
|
||||
func byteAt(line string, i int) byte {
|
||||
if i < 0 || i >= len(line) {
|
||||
return 0
|
||||
}
|
||||
return line[i]
|
||||
}
|
||||
|
||||
// isFilenameChar reports whether b can appear inside a bare filename token, used
|
||||
// to detect whether a reference-token match is embedded in a longer name.
|
||||
func isFilenameChar(b byte) bool {
|
||||
switch {
|
||||
case b >= 'a' && b <= 'z', b >= 'A' && b <= 'Z', b >= '0' && b <= '9':
|
||||
return true
|
||||
case b == '_' || b == '-':
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
package dream
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/smallnest/pigo/internal/memory"
|
||||
)
|
||||
|
||||
func TestUpdateScopeIndexesDropsDanglingLinks(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
// A global MEMORY.md linking to two entries; b.md will be deleted.
|
||||
writeMemFile(t, root, "global/user/a.md", "keep me")
|
||||
b := writeMemFile(t, root, "global/user/b.md", "remove me")
|
||||
idx := writeMemFile(t, root, "global/MEMORY.md",
|
||||
"# Index\n- [a](user/a.md)\n- [b](user/b.md)\n- freeform note\n")
|
||||
|
||||
deleted := map[string]struct{}{filepath.Clean(b): {}}
|
||||
if err := updateScopeIndexes(root, "", deleted); err != nil {
|
||||
t.Fatalf("updateScopeIndexes: %v", err)
|
||||
}
|
||||
raw, err := os.ReadFile(idx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := string(raw)
|
||||
if strings.Contains(got, "user/b.md") || strings.Contains(got, "b.md") {
|
||||
t.Fatalf("dangling link to b.md not removed:\n%s", got)
|
||||
}
|
||||
if !strings.Contains(got, "user/a.md") {
|
||||
t.Fatalf("live link to a.md wrongly removed:\n%s", got)
|
||||
}
|
||||
if !strings.Contains(got, "freeform note") {
|
||||
t.Fatalf("unrelated line wrongly removed:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestUpdateScopeIndexesBoundaryMatch: deleting b.md must not drop an index line
|
||||
// referencing a different entry whose name merely contains "b.md" as a
|
||||
// substring (e.g. club.md).
|
||||
func TestUpdateScopeIndexesBoundaryMatch(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeMemFile(t, root, "global/user/club.md", "keep me")
|
||||
b := writeMemFile(t, root, "global/user/b.md", "remove me")
|
||||
idx := writeMemFile(t, root, "global/MEMORY.md",
|
||||
"- [club](user/club.md)\n- [b](user/b.md)\n")
|
||||
|
||||
deleted := map[string]struct{}{filepath.Clean(b): {}}
|
||||
if err := updateScopeIndexes(root, "", deleted); err != nil {
|
||||
t.Fatalf("updateScopeIndexes: %v", err)
|
||||
}
|
||||
got, _ := os.ReadFile(idx)
|
||||
if !strings.Contains(string(got), "user/club.md") {
|
||||
t.Fatalf("club.md link wrongly removed by substring match:\n%s", got)
|
||||
}
|
||||
if strings.Contains(string(got), "user/b.md") {
|
||||
t.Fatalf("b.md link not removed:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateScopeIndexesNoIndexIsNoOp(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeMemFile(t, root, "global/user/a.md", "x")
|
||||
deleted := map[string]struct{}{filepath.Join(root, "global", "user", "a.md"): {}}
|
||||
if err := updateScopeIndexes(root, "", deleted); err != nil {
|
||||
t.Fatalf("updateScopeIndexes with no MEMORY.md should be a no-op, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunEndToEndWithMerge exercises the full non-dry-run write path with a stub
|
||||
// Consolidator that merges b.md into a.md and prunes c.md: files converge on
|
||||
// disk, the MEMORY.md index loses its dangling links, Reconcile runs, the Report
|
||||
// counters are correct, and a full-text search no longer hits the merged-away
|
||||
// fragment (US-003 / US-006 / US-009).
|
||||
func TestRunEndToEndWithMerge(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
a := writeMemFile(t, root, "global/user/a.md", "shared topic original a")
|
||||
b := writeMemFile(t, root, "global/user/b.md", "shared topic zebrafragment only in b")
|
||||
c := writeMemFile(t, root, "global/user/c.md", "outdated standalone note")
|
||||
idx := writeMemFile(t, root, "global/MEMORY.md",
|
||||
"# Index\n- [a](user/a.md)\n- [b](user/b.md)\n- [c](user/c.md)\n")
|
||||
|
||||
stub := &stubConsolidator{result: ConsolidateResult{
|
||||
MergedBodies: map[string]string{a: "shared topic merged and current"},
|
||||
Deletions: []string{b, c},
|
||||
Merged: 1,
|
||||
Pruned: 1,
|
||||
Notes: []string{"pruned c: outdated"},
|
||||
}}
|
||||
r := &Runner{MemoryRoot: root, Consolidator: stub}
|
||||
rep, err := r.Run(context.Background(), RunOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if !stub.called {
|
||||
t.Fatal("Consolidator not called")
|
||||
}
|
||||
if rep.Merged != 1 || rep.Pruned != 1 {
|
||||
t.Fatalf("counters wrong: %+v", rep)
|
||||
}
|
||||
// a.md rewritten, b.md + c.md gone.
|
||||
if got, _ := os.ReadFile(a); string(got) != "shared topic merged and current" {
|
||||
t.Fatalf("a.md not rewritten: %q", got)
|
||||
}
|
||||
if _, err := os.Stat(b); !os.IsNotExist(err) {
|
||||
t.Fatal("b.md should be deleted")
|
||||
}
|
||||
if _, err := os.Stat(c); !os.IsNotExist(err) {
|
||||
t.Fatal("c.md should be deleted")
|
||||
}
|
||||
// MEMORY.md no longer links to the removed entries.
|
||||
rawIdx, _ := os.ReadFile(idx)
|
||||
if strings.Contains(string(rawIdx), "b.md") || strings.Contains(string(rawIdx), "c.md") {
|
||||
t.Fatalf("MEMORY.md retains dangling links:\n%s", rawIdx)
|
||||
}
|
||||
if !strings.Contains(string(rawIdx), "a.md") {
|
||||
t.Fatalf("MEMORY.md lost the live a.md link:\n%s", rawIdx)
|
||||
}
|
||||
// Reconcile ran and indexed the surviving files.
|
||||
if rep.Reconciled.Indexed == 0 {
|
||||
t.Fatalf("Reconcile did not index: %+v", rep.Reconciled)
|
||||
}
|
||||
if rep.FilesAfter != 2 { // a.md + MEMORY.md
|
||||
t.Fatalf("FilesAfter = %d, want 2", rep.FilesAfter)
|
||||
}
|
||||
|
||||
// The merged-away fragment must no longer be searchable.
|
||||
store, err := memory.Open(filepath.Join(root, "index.db"), root, "")
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
hits, err := store.Search("zebrafragment", memory.SearchOptions{ReconcileFirst: true})
|
||||
if err != nil {
|
||||
t.Fatalf("search: %v", err)
|
||||
}
|
||||
if len(hits) != 0 {
|
||||
t.Fatalf("merged-away fragment still searchable: %+v", hits)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
// Package dream implements the /dream memory-consolidation feature's foundation
|
||||
// layer: the resolved [dream] configuration, on-disk run state, and the
|
||||
// deterministic due-check that decides whether an auto-trigger is warranted.
|
||||
// This layer has no LLM dependency; the runner, scheduler, lock, and plan/apply
|
||||
// logic live in later nodes. See tasks/spec-dream-memory-consolidation.md
|
||||
// §3.2/§3.3/§5.1/§5.4.
|
||||
package dream
|
||||
|
||||
// Built-in defaults for the [dream] table, exposed so config normalization and
|
||||
// tests share one source of truth.
|
||||
const (
|
||||
DefaultEnabled = true
|
||||
DefaultIntervalDays = 7
|
||||
DefaultRecentSessions = 20
|
||||
)
|
||||
|
||||
// Config is the resolved [dream] configuration with defaults applied. It is the
|
||||
// shape the scheduler and runner consume, distinct from the raw
|
||||
// config.DreamConfig (which uses *bool / zero to distinguish "unset").
|
||||
type Config struct {
|
||||
Enabled bool
|
||||
IntervalDays int
|
||||
RecentSessions int
|
||||
}
|
||||
|
||||
// NewConfig normalizes a raw [dream] table into a Config, applying defaults: a
|
||||
// nil enabled pointer means true (only an explicit false disables); a
|
||||
// non-positive interval_days falls back to 7; a non-positive recent_sessions
|
||||
// falls back to 20. A missing [dream] table is representable as the zero
|
||||
// arguments (nil, 0, 0) and yields all defaults, so parsing never errors on an
|
||||
// absent table.
|
||||
func NewConfig(enabled *bool, intervalDays, recentSessions int) Config {
|
||||
c := Config{
|
||||
Enabled: DefaultEnabled,
|
||||
IntervalDays: intervalDays,
|
||||
RecentSessions: recentSessions,
|
||||
}
|
||||
if enabled != nil {
|
||||
c.Enabled = *enabled
|
||||
}
|
||||
if c.IntervalDays <= 0 {
|
||||
c.IntervalDays = DefaultIntervalDays
|
||||
}
|
||||
if c.RecentSessions <= 0 {
|
||||
c.RecentSessions = DefaultRecentSessions
|
||||
}
|
||||
return c
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package dream
|
||||
|
||||
import "testing"
|
||||
|
||||
func boolPtr(b bool) *bool { return &b }
|
||||
|
||||
func TestNewConfigDefaults(t *testing.T) {
|
||||
// Missing [dream] table: nil enabled, zero ints → all defaults.
|
||||
c := NewConfig(nil, 0, 0)
|
||||
if !c.Enabled {
|
||||
t.Errorf("Enabled = false, want true (nil → true)")
|
||||
}
|
||||
if c.IntervalDays != DefaultIntervalDays {
|
||||
t.Errorf("IntervalDays = %d, want %d", c.IntervalDays, DefaultIntervalDays)
|
||||
}
|
||||
if c.RecentSessions != DefaultRecentSessions {
|
||||
t.Errorf("RecentSessions = %d, want %d", c.RecentSessions, DefaultRecentSessions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewConfigEnabledSemantics(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
enabled *bool
|
||||
want bool
|
||||
}{
|
||||
{"nil treated as true", nil, true},
|
||||
{"explicit true", boolPtr(true), true},
|
||||
{"explicit false", boolPtr(false), false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := NewConfig(tt.enabled, 0, 0).Enabled; got != tt.want {
|
||||
t.Errorf("Enabled = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewConfigNonPositiveFallback(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
interval, recent int
|
||||
wantInterval, wantRcnt int
|
||||
}{
|
||||
{"zero falls back", 0, 0, DefaultIntervalDays, DefaultRecentSessions},
|
||||
{"negative falls back", -3, -1, DefaultIntervalDays, DefaultRecentSessions},
|
||||
{"positive preserved", 14, 50, 14, 50},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
c := NewConfig(nil, tt.interval, tt.recent)
|
||||
if c.IntervalDays != tt.wantInterval {
|
||||
t.Errorf("IntervalDays = %d, want %d", c.IntervalDays, tt.wantInterval)
|
||||
}
|
||||
if c.RecentSessions != tt.wantRcnt {
|
||||
t.Errorf("RecentSessions = %d, want %d", c.RecentSessions, tt.wantRcnt)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,434 @@
|
||||
package dream
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// This file implements the real, LLM-backed Consolidator (SPEC §5.1 step 5,
|
||||
// §5.1.1). It turns the deterministic Plan into a prompt for the main-session
|
||||
// model, asks it to confirm semantic merges and conservative prunes, and parses
|
||||
// the strict-JSON response back into a ConsolidateResult the Runner applies.
|
||||
//
|
||||
// The deterministic half (exact dedup, dead-path cleanup, MEMORY.md rewrite,
|
||||
// Reconcile, counters) stays in the Runner; this file is purely the semantic
|
||||
// LLM step. It never touches disk — it only produces decisions the Runner
|
||||
// path-guards and applies.
|
||||
|
||||
// completeFn performs a single LLM completion: given the system and user
|
||||
// prompts it returns the model's text response, or an error for a hard
|
||||
// transport/provider failure. The provider-backed implementation lives in
|
||||
// apply.go (newModelCompleter); tests inject a canned function so no live model
|
||||
// is ever called.
|
||||
type completeFn func(ctx context.Context, systemPrompt, userPrompt string) (string, error)
|
||||
|
||||
// defaultBodyBudget caps how many bytes of each entry body are shown to the
|
||||
// model, bounding prompt size on large memory libraries (SPEC §8.2 token
|
||||
// budget). Bodies longer than this are truncated with a marker so the model
|
||||
// still sees the leading, usually most salient, content.
|
||||
const defaultBodyBudget = 6000
|
||||
|
||||
// llmConsolidator is the production Consolidator: it drives the main-session
|
||||
// model through complete and parses the response. bodyBudget (0 → default)
|
||||
// bounds per-entry body size in the prompt.
|
||||
type llmConsolidator struct {
|
||||
complete completeFn
|
||||
bodyBudget int
|
||||
}
|
||||
|
||||
// Consolidate builds the prompt from the plan, runs one model completion, and
|
||||
// parses the response into merge/prune decisions. A hard model/transport error
|
||||
// is returned (the Runner maps it to a failed run, SPEC §5.5). A well-formed
|
||||
// call whose text cannot be parsed is NOT an error: it yields an empty result
|
||||
// with an explanatory note, so an unparseable response conservatively KEEPs
|
||||
// everything (PRD FR-14) and the deterministic pass still applies.
|
||||
func (c *llmConsolidator) Consolidate(ctx context.Context, in ConsolidateInput) (ConsolidateResult, error) {
|
||||
if c.complete == nil {
|
||||
return ConsolidateResult{}, fmt.Errorf("dream: llmConsolidator has no completion function")
|
||||
}
|
||||
|
||||
var res ConsolidateResult
|
||||
eligible := eligibleFiles(in.Plan)
|
||||
if len(eligible) > 0 {
|
||||
budget := c.bodyBudget
|
||||
if budget <= 0 {
|
||||
budget = defaultBodyBudget
|
||||
}
|
||||
prompt := buildConsolidatePrompt(in, eligible, budget)
|
||||
|
||||
raw, err := c.complete(ctx, dreamSystemPrompt, prompt)
|
||||
if err != nil {
|
||||
return ConsolidateResult{}, fmt.Errorf("dream: model completion: %w", err)
|
||||
}
|
||||
|
||||
allowed := make(map[string]struct{}, len(eligible))
|
||||
for _, f := range eligible {
|
||||
allowed[filepath.Clean(f.Path)] = struct{}{}
|
||||
}
|
||||
res = parseConsolidateResponse(raw, allowed)
|
||||
}
|
||||
|
||||
// Distillation pass (SPEC §5.3, PRD US-005/FR-13): a SEPARATE model call over
|
||||
// the recent-session transcripts the Runner collected. It runs even when the
|
||||
// library is empty (nothing to merge/prune) so a first-time distill can seed
|
||||
// memory from sessions. A hard model failure aborts the run; a well-formed
|
||||
// call yielding nothing simply adds no entries (Runner records the no-op).
|
||||
if strings.TrimSpace(in.Transcripts) != "" {
|
||||
if err := c.distill(ctx, in, &res); err != nil {
|
||||
return ConsolidateResult{}, err
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// distill runs the JSONL distillation model call and folds the resulting new
|
||||
// entries into res: it prompts the distiller with the transcripts plus a summary
|
||||
// of the existing library (so the model avoids re-proposing known facts), parses
|
||||
// the response into path-guarded NewEntry writes deduped against the existing
|
||||
// memory, and bumps res.Distilled by the number added. A hard model/transport
|
||||
// error is returned so the Runner marks the run failed (SPEC §5.5); an
|
||||
// unparseable or empty response adds nothing and is not an error (conservative
|
||||
// KEEP, PRD FR-14).
|
||||
func (c *llmConsolidator) distill(ctx context.Context, in ConsolidateInput, res *ConsolidateResult) error {
|
||||
prompt := buildDistillPrompt(in)
|
||||
raw, err := c.complete(ctx, dreamDistillSystemPrompt, prompt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("dream: distill completion: %w", err)
|
||||
}
|
||||
entries, notes := parseDistillResponse(raw, in.Plan.Files, in.MemoryRoot, in.ProjectDir)
|
||||
res.NewEntries = append(res.NewEntries, entries...)
|
||||
res.Distilled += len(entries)
|
||||
res.Notes = append(res.Notes, notes...)
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildDistillPrompt renders the distiller's user prompt: the target scope, a
|
||||
// compact list of the titles/paths of existing memories (so the model does not
|
||||
// re-propose known facts), and the recent-session transcripts. Existing bodies
|
||||
// are summarized (path + leading text) rather than dumped in full to keep the
|
||||
// prompt bounded; the Go side still enforces near-duplicate rejection.
|
||||
func buildDistillPrompt(in ConsolidateInput) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("# Memory distillation request\n\n")
|
||||
if in.ProjectDir != "" {
|
||||
b.WriteString("Current project: ")
|
||||
b.WriteString(in.ProjectDir)
|
||||
b.WriteByte('\n')
|
||||
} else {
|
||||
b.WriteString("Scope: global only\n")
|
||||
}
|
||||
|
||||
existing := eligibleFiles(in.Plan)
|
||||
if len(existing) > 0 {
|
||||
b.WriteString(fmt.Sprintf("\n## Existing memories (%d) — do NOT re-propose these\n\n", len(existing)))
|
||||
for _, f := range existing {
|
||||
b.WriteString("- ")
|
||||
b.WriteString(clip(strings.TrimSpace(firstNonEmptyLine(f.Body)), 120))
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
}
|
||||
|
||||
b.WriteString("\n## Recent session transcripts\n\n")
|
||||
b.WriteString(in.Transcripts)
|
||||
b.WriteString("\n\nReturn the JSON object described in your instructions. Extract only genuinely new, durable facts. When in doubt, return no entries.\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// firstNonEmptyLine returns the first non-blank line of s (trimmed), or "" when
|
||||
// s is entirely blank. Used to label an existing memory in the distill prompt.
|
||||
func firstNonEmptyLine(s string) string {
|
||||
for _, line := range strings.Split(s, "\n") {
|
||||
if t := strings.TrimSpace(line); t != "" {
|
||||
return t
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// eligibleFiles is the subset of plan files the model may act on: it excludes
|
||||
// MEMORY.md index files (they are indexes, not entries — #521 NEW_WORK: we
|
||||
// special-case MEMORY.md out of merge/prune so the index is never folded into an
|
||||
// entry) and the redundant members of an exact-dedupe group (g.Paths[1:], which
|
||||
// the deterministic pass removes anyway — offering them would let the model
|
||||
// merge into a path about to be deleted). The representative g.Paths[0] stays.
|
||||
func eligibleFiles(plan Plan) []MemoryFile {
|
||||
drop := make(map[string]struct{})
|
||||
for _, g := range plan.DedupeGroups {
|
||||
for _, p := range g.Paths[1:] {
|
||||
drop[filepath.Clean(p)] = struct{}{}
|
||||
}
|
||||
}
|
||||
var out []MemoryFile
|
||||
for _, f := range plan.Files {
|
||||
if isMemoryIndex(f.Path) {
|
||||
continue
|
||||
}
|
||||
if _, dup := drop[filepath.Clean(f.Path)]; dup {
|
||||
continue
|
||||
}
|
||||
out = append(out, f)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// isMemoryIndex reports whether path is a scope MEMORY.md index file, which the
|
||||
// consolidation step must never merge, rewrite, or prune.
|
||||
func isMemoryIndex(path string) bool {
|
||||
return strings.EqualFold(filepath.Base(path), "MEMORY.md")
|
||||
}
|
||||
|
||||
// buildConsolidatePrompt renders the user prompt: the scope, the eligible
|
||||
// entries (path + scope/type + body, truncated to budget), and the deterministic
|
||||
// hints (near-dup candidate pairs, dead local-path references). It only lists
|
||||
// paths that are eligible, so the model is naturally steered away from MEMORY.md
|
||||
// and duplicate paths.
|
||||
func buildConsolidatePrompt(in ConsolidateInput, eligible []MemoryFile, budget int) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("# Memory consolidation request\n\n")
|
||||
b.WriteString("Memory root: ")
|
||||
b.WriteString(in.MemoryRoot)
|
||||
b.WriteByte('\n')
|
||||
if in.ProjectDir != "" {
|
||||
b.WriteString("Active project scope: ")
|
||||
b.WriteString(in.ProjectDir)
|
||||
b.WriteByte('\n')
|
||||
} else {
|
||||
b.WriteString("Scope: global only\n")
|
||||
}
|
||||
b.WriteString(fmt.Sprintf("\n## Entries (%d)\n\n", len(eligible)))
|
||||
for i, f := range eligible {
|
||||
typ := f.Type
|
||||
if typ == "" {
|
||||
typ = "(root)"
|
||||
}
|
||||
b.WriteString(fmt.Sprintf("### [%d] %s\n", i+1, f.Path))
|
||||
b.WriteString(fmt.Sprintf("scope=%s type=%s bytes=%d\n\n", f.Scope, typ, f.Size))
|
||||
b.WriteString("```\n")
|
||||
b.WriteString(truncateBody(f.Body, budget))
|
||||
b.WriteString("\n```\n\n")
|
||||
}
|
||||
|
||||
if pairs := eligiblePairs(in.Plan, eligible); len(pairs) > 0 {
|
||||
b.WriteString("## Near-duplicate candidate pairs (merge only if truly overlapping)\n\n")
|
||||
for _, p := range pairs {
|
||||
b.WriteString(fmt.Sprintf("- %s <-> %s (similarity %.2f)\n", p.A, p.B, p.Similarity))
|
||||
}
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
|
||||
if refs := eligibleInvalidRefs(in.Plan, eligible); len(refs) > 0 {
|
||||
b.WriteString("## Entries referencing local files that no longer exist\n")
|
||||
b.WriteString("(the dead reference text is cleaned automatically; only PRUNE an entry if losing that reference leaves it meaningless)\n\n")
|
||||
for _, r := range refs {
|
||||
b.WriteString(fmt.Sprintf("- %s references missing %s\n", r.File, r.Ref))
|
||||
}
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
|
||||
b.WriteString("Return the JSON object described in your instructions. When in doubt, KEEP.\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// truncateBody trims body to at most budget bytes on a rune boundary, appending
|
||||
// a marker when truncation occurred so the model knows content was elided.
|
||||
func truncateBody(body string, budget int) string {
|
||||
if budget <= 0 || len(body) <= budget {
|
||||
return body
|
||||
}
|
||||
cut := budget
|
||||
for cut > 0 && !isRuneStart(body[cut]) {
|
||||
cut--
|
||||
}
|
||||
return body[:cut] + "\n…[truncated]"
|
||||
}
|
||||
|
||||
// isRuneStart reports whether b is not a UTF-8 continuation byte, so a truncation
|
||||
// cut there does not split a multi-byte rune.
|
||||
func isRuneStart(b byte) bool { return b&0xC0 != 0x80 }
|
||||
|
||||
// eligiblePairs filters the plan's near-dup pairs to those whose BOTH members
|
||||
// are eligible (both are still offered to the model), so we never point the
|
||||
// model at a MEMORY.md or a to-be-deduped duplicate.
|
||||
func eligiblePairs(plan Plan, eligible []MemoryFile) []NearDupPair {
|
||||
ok := make(map[string]struct{}, len(eligible))
|
||||
for _, f := range eligible {
|
||||
ok[filepath.Clean(f.Path)] = struct{}{}
|
||||
}
|
||||
var out []NearDupPair
|
||||
for _, p := range plan.NearDupPairs {
|
||||
_, a := ok[filepath.Clean(p.A)]
|
||||
_, b := ok[filepath.Clean(p.B)]
|
||||
if a && b {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// eligibleInvalidRefs filters dead-path references to those on eligible files.
|
||||
func eligibleInvalidRefs(plan Plan, eligible []MemoryFile) []InvalidPathRef {
|
||||
ok := make(map[string]struct{}, len(eligible))
|
||||
for _, f := range eligible {
|
||||
ok[filepath.Clean(f.Path)] = struct{}{}
|
||||
}
|
||||
var out []InvalidPathRef
|
||||
for _, r := range plan.InvalidPathRefs {
|
||||
if _, found := ok[filepath.Clean(r.File)]; found {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// modelDecision mirrors the strict JSON output schema the dream prompt requires.
|
||||
type modelDecision struct {
|
||||
Merges []struct {
|
||||
Keep string `json:"keep"`
|
||||
Body string `json:"body"`
|
||||
Remove []string `json:"remove"`
|
||||
} `json:"merges"`
|
||||
Prunes []struct {
|
||||
Path string `json:"path"`
|
||||
Reason string `json:"reason"`
|
||||
} `json:"prunes"`
|
||||
Notes []string `json:"notes"`
|
||||
}
|
||||
|
||||
// parseConsolidateResponse turns the model's text into a ConsolidateResult,
|
||||
// validating every path against allowed (the eligible input paths). It is
|
||||
// conservative by construction: any decision that references an unknown path, a
|
||||
// MEMORY.md index, an empty merged body, or an empty prune reason is dropped
|
||||
// rather than obeyed, and an unparseable response yields an empty result with a
|
||||
// note (PRD FR-14 — default to KEEP on uncertainty). It never returns an error;
|
||||
// callers treat a hard model failure separately.
|
||||
func parseConsolidateResponse(raw string, allowed map[string]struct{}) ConsolidateResult {
|
||||
body := extractJSONObject(raw)
|
||||
if body == "" {
|
||||
return ConsolidateResult{Notes: []string{"dream: model response contained no JSON object; kept all entries"}}
|
||||
}
|
||||
var dec modelDecision
|
||||
if err := json.Unmarshal([]byte(body), &dec); err != nil {
|
||||
return ConsolidateResult{Notes: []string{"dream: model response was not valid JSON; kept all entries"}}
|
||||
}
|
||||
|
||||
var res ConsolidateResult
|
||||
res.MergedBodies = make(map[string]string)
|
||||
delSet := make(map[string]struct{})
|
||||
|
||||
valid := func(p string) (string, bool) {
|
||||
clean := filepath.Clean(strings.TrimSpace(p))
|
||||
if clean == "" || clean == "." {
|
||||
return "", false
|
||||
}
|
||||
if isMemoryIndex(clean) {
|
||||
return "", false
|
||||
}
|
||||
if _, ok := allowed[clean]; !ok {
|
||||
return "", false
|
||||
}
|
||||
return clean, true
|
||||
}
|
||||
|
||||
for _, m := range dec.Merges {
|
||||
keep, ok := valid(m.Keep)
|
||||
if !ok || strings.TrimSpace(m.Body) == "" {
|
||||
// Unknown/invalid target or an empty rewrite: skip, keep everything.
|
||||
continue
|
||||
}
|
||||
removed := 0
|
||||
for _, r := range m.Remove {
|
||||
rp, ok := valid(r)
|
||||
if !ok || rp == keep {
|
||||
continue
|
||||
}
|
||||
if _, dup := delSet[rp]; dup {
|
||||
continue
|
||||
}
|
||||
delSet[rp] = struct{}{}
|
||||
removed++
|
||||
}
|
||||
if removed == 0 {
|
||||
// A merge that removes nothing is a no-op rewrite; ignore it to avoid
|
||||
// gratuitously touching a file the model merely echoed back.
|
||||
continue
|
||||
}
|
||||
res.MergedBodies[keep] = m.Body
|
||||
res.Merged += removed
|
||||
}
|
||||
|
||||
for _, p := range dec.Prunes {
|
||||
pp, ok := valid(p.Path)
|
||||
if !ok || strings.TrimSpace(p.Reason) == "" {
|
||||
// No path or no stated reason → conservative KEEP.
|
||||
continue
|
||||
}
|
||||
if _, dup := delSet[pp]; dup {
|
||||
continue
|
||||
}
|
||||
// Never prune an entry we are simultaneously keeping as a merge target.
|
||||
if _, kept := res.MergedBodies[pp]; kept {
|
||||
continue
|
||||
}
|
||||
delSet[pp] = struct{}{}
|
||||
res.Pruned++
|
||||
res.Notes = append(res.Notes, fmt.Sprintf("pruned %s: %s", pp, strings.TrimSpace(p.Reason)))
|
||||
}
|
||||
|
||||
if len(res.MergedBodies) == 0 {
|
||||
res.MergedBodies = nil
|
||||
}
|
||||
for p := range delSet {
|
||||
res.Deletions = append(res.Deletions, p)
|
||||
}
|
||||
sort.Strings(res.Deletions)
|
||||
|
||||
for _, n := range dec.Notes {
|
||||
if s := strings.TrimSpace(n); s != "" {
|
||||
res.Notes = append(res.Notes, s)
|
||||
}
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// extractJSONObject returns the outermost {...} span of s, tolerating models
|
||||
// that wrap the object in prose or Markdown code fences. It returns "" when no
|
||||
// balanced object is found.
|
||||
func extractJSONObject(s string) string {
|
||||
start := strings.IndexByte(s, '{')
|
||||
if start < 0 {
|
||||
return ""
|
||||
}
|
||||
depth := 0
|
||||
inStr := false
|
||||
esc := false
|
||||
for i := start; i < len(s); i++ {
|
||||
ch := s[i]
|
||||
if inStr {
|
||||
switch {
|
||||
case esc:
|
||||
esc = false
|
||||
case ch == '\\':
|
||||
esc = true
|
||||
case ch == '"':
|
||||
inStr = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
switch ch {
|
||||
case '"':
|
||||
inStr = true
|
||||
case '{':
|
||||
depth++
|
||||
case '}':
|
||||
depth--
|
||||
if depth == 0 {
|
||||
return s[start : i+1]
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package dream
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// canned builds an allowed-set + Plan pair for parser tests from a list of
|
||||
// absolute paths.
|
||||
func allowedSet(paths ...string) map[string]struct{} {
|
||||
m := make(map[string]struct{}, len(paths))
|
||||
for _, p := range paths {
|
||||
m[filepath.Clean(p)] = struct{}{}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func TestBuildConsolidatePromptListsEntriesAndHints(t *testing.T) {
|
||||
root := "/mem"
|
||||
a := "/mem/global/user/a.md"
|
||||
b := "/mem/global/user/b.md"
|
||||
idx := "/mem/global/MEMORY.md"
|
||||
plan := Plan{
|
||||
Files: []MemoryFile{
|
||||
{Path: a, Scope: "global", Type: "user", Size: 5, Body: "alpha body"},
|
||||
{Path: b, Scope: "global", Type: "user", Size: 5, Body: "beta body"},
|
||||
{Path: idx, Scope: "global", Type: "", Size: 3, Body: "- [a](user/a.md)"},
|
||||
},
|
||||
NearDupPairs: []NearDupPair{{A: a, B: b, Similarity: 0.82}},
|
||||
InvalidPathRefs: []InvalidPathRef{{File: a, Ref: "./gone.go"}},
|
||||
}
|
||||
eligible := eligibleFiles(plan)
|
||||
if len(eligible) != 2 {
|
||||
t.Fatalf("eligibleFiles = %d, want 2 (MEMORY.md excluded)", len(eligible))
|
||||
}
|
||||
prompt := buildConsolidatePrompt(ConsolidateInput{Plan: plan, MemoryRoot: root, ProjectDir: ""}, eligible, defaultBodyBudget)
|
||||
|
||||
for _, want := range []string{a, b, "alpha body", "beta body", "similarity 0.82", "./gone.go", "global only"} {
|
||||
if !strings.Contains(prompt, want) {
|
||||
t.Errorf("prompt missing %q\n---\n%s", want, prompt)
|
||||
}
|
||||
}
|
||||
if strings.Contains(prompt, "MEMORY.md") {
|
||||
t.Errorf("prompt must not offer the MEMORY.md index as an entry:\n%s", prompt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTruncateBody(t *testing.T) {
|
||||
if got := truncateBody("short", 100); got != "short" {
|
||||
t.Fatalf("no truncation expected, got %q", got)
|
||||
}
|
||||
long := strings.Repeat("x", 50)
|
||||
got := truncateBody(long, 10)
|
||||
if !strings.HasPrefix(got, strings.Repeat("x", 10)) || !strings.Contains(got, "truncated") {
|
||||
t.Fatalf("truncateBody = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseConsolidateResponseMergeAndPrune(t *testing.T) {
|
||||
a := "/mem/global/user/a.md"
|
||||
b := "/mem/global/user/b.md"
|
||||
c := "/mem/global/user/c.md"
|
||||
allowed := allowedSet(a, b, c)
|
||||
|
||||
raw := "here you go:\n```json\n" + `{
|
||||
"merges": [{"keep": "` + a + `", "body": "merged body", "remove": ["` + b + `"]}],
|
||||
"prunes": [{"path": "` + c + `", "reason": "superseded by newer note"}],
|
||||
"notes": ["did the thing"]
|
||||
}` + "\n```\n"
|
||||
|
||||
res := parseConsolidateResponse(raw, allowed)
|
||||
if res.Merged != 1 {
|
||||
t.Errorf("Merged = %d, want 1", res.Merged)
|
||||
}
|
||||
if res.Pruned != 1 {
|
||||
t.Errorf("Pruned = %d, want 1", res.Pruned)
|
||||
}
|
||||
if got := res.MergedBodies[a]; got != "merged body" {
|
||||
t.Errorf("MergedBodies[a] = %q", got)
|
||||
}
|
||||
wantDel := map[string]bool{b: true, c: true}
|
||||
if len(res.Deletions) != 2 {
|
||||
t.Fatalf("Deletions = %v, want b and c", res.Deletions)
|
||||
}
|
||||
for _, d := range res.Deletions {
|
||||
if !wantDel[d] {
|
||||
t.Errorf("unexpected deletion %q", d)
|
||||
}
|
||||
}
|
||||
joined := strings.Join(res.Notes, "|")
|
||||
if !strings.Contains(joined, "superseded by newer note") || !strings.Contains(joined, "did the thing") {
|
||||
t.Errorf("notes missing content: %v", res.Notes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseConsolidateResponseRejectsUnknownAndUnsafePaths(t *testing.T) {
|
||||
a := "/mem/global/user/a.md"
|
||||
allowed := allowedSet(a)
|
||||
raw := `{
|
||||
"merges": [{"keep": "/etc/passwd", "body": "x", "remove": ["` + a + `"]}],
|
||||
"prunes": [{"path": "/mem/global/MEMORY.md", "reason": "index"}]
|
||||
}`
|
||||
res := parseConsolidateResponse(raw, allowed)
|
||||
if res.Merged != 0 || res.Pruned != 0 || len(res.Deletions) != 0 {
|
||||
t.Fatalf("unsafe/unknown paths must be ignored, got %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseConsolidateResponseConservativeOnEmptyReasonAndBadJSON(t *testing.T) {
|
||||
a := "/mem/global/user/a.md"
|
||||
allowed := allowedSet(a)
|
||||
|
||||
// Empty prune reason → KEEP.
|
||||
res := parseConsolidateResponse(`{"prunes":[{"path":"`+a+`","reason":""}]}`, allowed)
|
||||
if res.Pruned != 0 || len(res.Deletions) != 0 {
|
||||
t.Fatalf("empty reason must KEEP, got %+v", res)
|
||||
}
|
||||
|
||||
// Unparseable → empty result with a note, never a deletion.
|
||||
res = parseConsolidateResponse("the model rambled with no json", allowed)
|
||||
if res.Merged != 0 || res.Pruned != 0 || len(res.Deletions) != 0 {
|
||||
t.Fatalf("bad JSON must KEEP everything, got %+v", res)
|
||||
}
|
||||
if len(res.Notes) == 0 {
|
||||
t.Fatal("expected an explanatory note on unparseable response")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseConsolidateResponseNoOpMergeIgnored(t *testing.T) {
|
||||
a := "/mem/global/user/a.md"
|
||||
allowed := allowedSet(a)
|
||||
// A merge that removes nothing must not touch the file.
|
||||
res := parseConsolidateResponse(`{"merges":[{"keep":"`+a+`","body":"rewritten","remove":[]}]}`, allowed)
|
||||
if len(res.MergedBodies) != 0 || res.Merged != 0 {
|
||||
t.Fatalf("no-op merge must be ignored, got %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMConsolidatorUsesCompleter(t *testing.T) {
|
||||
root := "/mem"
|
||||
a := "/mem/global/user/a.md"
|
||||
b := "/mem/global/user/b.md"
|
||||
plan := Plan{Files: []MemoryFile{
|
||||
{Path: a, Scope: "global", Type: "user", Body: "one"},
|
||||
{Path: b, Scope: "global", Type: "user", Body: "two"},
|
||||
}}
|
||||
|
||||
var gotSystem, gotUser string
|
||||
c := &llmConsolidator{complete: func(_ context.Context, sys, user string) (string, error) {
|
||||
gotSystem, gotUser = sys, user
|
||||
return `{"merges":[{"keep":"` + a + `","body":"merged","remove":["` + b + `"]}]}`, nil
|
||||
}}
|
||||
res, err := c.Consolidate(context.Background(), ConsolidateInput{Plan: plan, MemoryRoot: root})
|
||||
if err != nil {
|
||||
t.Fatalf("Consolidate: %v", err)
|
||||
}
|
||||
if gotSystem != dreamSystemPrompt {
|
||||
t.Error("system prompt not passed through")
|
||||
}
|
||||
if !strings.Contains(gotUser, a) {
|
||||
t.Error("user prompt missing entry path")
|
||||
}
|
||||
if res.Merged != 1 || res.MergedBodies[a] != "merged" {
|
||||
t.Fatalf("merge not applied: %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMConsolidatorPropagatesHardError(t *testing.T) {
|
||||
plan := Plan{Files: []MemoryFile{{Path: "/mem/global/user/a.md", Scope: "global", Type: "user", Body: "x"}}}
|
||||
c := &llmConsolidator{complete: func(context.Context, string, string) (string, error) {
|
||||
return "", errors.New("upstream 500")
|
||||
}}
|
||||
if _, err := c.Consolidate(context.Background(), ConsolidateInput{Plan: plan, MemoryRoot: "/mem"}); err == nil {
|
||||
t.Fatal("expected hard completion error to propagate")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMConsolidatorSkipsWhenNoEligibleFiles(t *testing.T) {
|
||||
called := false
|
||||
c := &llmConsolidator{complete: func(context.Context, string, string) (string, error) {
|
||||
called = true
|
||||
return "{}", nil
|
||||
}}
|
||||
// Only a MEMORY.md index → nothing eligible → no model call.
|
||||
plan := Plan{Files: []MemoryFile{{Path: "/mem/global/MEMORY.md", Scope: "global", Body: "idx"}}}
|
||||
if _, err := c.Consolidate(context.Background(), ConsolidateInput{Plan: plan, MemoryRoot: "/mem"}); err != nil {
|
||||
t.Fatalf("Consolidate: %v", err)
|
||||
}
|
||||
if called {
|
||||
t.Fatal("model should not be called when no eligible files exist")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,411 @@
|
||||
package dream
|
||||
|
||||
// This file implements the JSONL distillation step of a /dream run (SPEC §5.3,
|
||||
// PRD US-005 / FR-13): it selects the current project's recent sessions, reads
|
||||
// their transcripts (truncated to a token/byte budget, SPEC §8.2), and — via the
|
||||
// LLM distiller in consolidator.go — turns durable facts into new memory
|
||||
// entries, deduped against the existing library so nothing already recorded is
|
||||
// re-added.
|
||||
//
|
||||
// The deterministic half lives here (session-window selection, project filter,
|
||||
// transcript rendering, dedup, path construction); the semantic half (which
|
||||
// facts are durable) is a separate LLM call driven by the llmConsolidator. This
|
||||
// keeps the merge/prune pass and its parser (consolidator.go) untouched: the
|
||||
// Runner gathers the transcripts, the consolidator runs a distinct distill
|
||||
// completion with its own prompt/schema, and the distilled NewEntries are folded
|
||||
// into the same ConsolidateResult the Runner already applies (wiring choice "b"
|
||||
// from #523: a separate step appended to the result, chosen over folding it into
|
||||
// the merge/prune prompt so each concern keeps its own input, prompt and schema
|
||||
// and the well-tested merge/prune parser is not perturbed).
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/smallnest/pigo/internal/agentcore"
|
||||
"github.com/smallnest/pigo/internal/session"
|
||||
)
|
||||
|
||||
// defaultTranscriptBudget bounds the total bytes of session transcript text fed
|
||||
// to the distiller, keeping the distill prompt within the model context even on
|
||||
// long or numerous recent sessions (SPEC §8.2 token budget). Sessions are added
|
||||
// most-recent-first until the budget is reached.
|
||||
const defaultTranscriptBudget = 24000
|
||||
|
||||
// distillNewEntryTypes is the closed set of durable memory types the distiller
|
||||
// may emit (PRD FR-13: user / feedback / project / reference). Ephemeral kinds
|
||||
// (checkpoint / progress) are intentionally excluded — one-shot task state is
|
||||
// not distilled into long-term memory.
|
||||
var distillNewEntryTypes = map[string]struct{}{
|
||||
"user": {},
|
||||
"feedback": {},
|
||||
"project": {},
|
||||
"reference": {},
|
||||
}
|
||||
|
||||
// SessionSource is the read-only view of the session store the distiller needs:
|
||||
// list session headers and load a session's messages. *session.Store satisfies
|
||||
// it; tests inject a stub so no real session files (or LLM) are required.
|
||||
type SessionSource interface {
|
||||
List() ([]session.SessionHeader, error)
|
||||
Load(id string) (session.SessionHeader, agentcore.MessageList, error)
|
||||
}
|
||||
|
||||
// resolveSessionStore opens the default session store rooted at
|
||||
// $PIGO_HOME/sessions (else ~/.pigo/sessions), mirroring
|
||||
// headless.SessionStore's resolution. It is duplicated here rather than imported
|
||||
// to keep internal/dream free of a dependency on the CLI assembly layer, exactly
|
||||
// as ResolveMemoryRoot duplicates the memory-root resolution. A resolution
|
||||
// failure yields a nil source and no error: distillation then degrades to a
|
||||
// no-op rather than failing the whole dream run.
|
||||
func resolveSessionStore() (SessionSource, error) {
|
||||
dir := os.Getenv("PIGO_HOME")
|
||||
if dir == "" {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
dir = filepath.Join(home, ".pigo")
|
||||
}
|
||||
store, err := session.NewStore(filepath.Join(dir, "sessions"))
|
||||
if err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
return store, nil
|
||||
}
|
||||
|
||||
// sessionMatchesProject reports whether a session belongs to the project rooted
|
||||
// at projectDir. Attribution is by the session's recorded Cwd: two directories
|
||||
// belong to the same project when their stable project ids match (the same id
|
||||
// internal/memory and BuildPlan derive). A session with an empty Cwd is
|
||||
// unattributed and never matches; an empty projectDir (a global-only run) has no
|
||||
// project to match, so nothing is selected.
|
||||
func sessionMatchesProject(h session.SessionHeader, projectDir string) bool {
|
||||
if projectDir == "" || h.Cwd == "" {
|
||||
return false
|
||||
}
|
||||
return projectID(h.Cwd) == projectID(projectDir)
|
||||
}
|
||||
|
||||
// collectRecentSessions selects the recent sessions to distill, applying the
|
||||
// SPEC §5.3 combined window over the project-filtered set:
|
||||
// - state.LastRunAt non-zero → every matching session updated strictly after
|
||||
// the last run (incremental distillation since the last dream).
|
||||
// - state.LastRunAt zero (never run) → the most-recent recentN matching
|
||||
// sessions by UpdatedAt descending (a bounded first-run window).
|
||||
//
|
||||
// recentN falls back to DefaultRecentSessions when non-positive. The result is
|
||||
// ordered most-recent-first so transcript budgeting keeps the freshest context.
|
||||
func collectRecentSessions(sessions []session.SessionHeader, state State, projectDir string, recentN int) []session.SessionHeader {
|
||||
if recentN <= 0 {
|
||||
recentN = DefaultRecentSessions
|
||||
}
|
||||
var matched []session.SessionHeader
|
||||
for _, h := range sessions {
|
||||
if sessionMatchesProject(h, projectDir) {
|
||||
matched = append(matched, h)
|
||||
}
|
||||
}
|
||||
// Most-recent-first regardless of the source ordering.
|
||||
sort.SliceStable(matched, func(i, j int) bool {
|
||||
return matched[i].UpdatedAt.After(matched[j].UpdatedAt)
|
||||
})
|
||||
|
||||
if !state.LastRunAt.IsZero() {
|
||||
var out []session.SessionHeader
|
||||
for _, h := range matched {
|
||||
if h.UpdatedAt.After(state.LastRunAt) {
|
||||
out = append(out, h)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
if len(matched) > recentN {
|
||||
matched = matched[:recentN]
|
||||
}
|
||||
return matched
|
||||
}
|
||||
|
||||
// collectTranscripts selects the recent project sessions (collectRecentSessions)
|
||||
// and renders their messages into a single transcript string truncated to
|
||||
// budget bytes (SPEC §8.2). It returns "" when the source is nil or no session
|
||||
// matches — the no-matching-sessions no-op (SPEC §5.5): the caller then records
|
||||
// Distilled=0 with a "无新增" note. A per-session load error is skipped rather
|
||||
// than failing the run (a single corrupt session must not abort distillation).
|
||||
func collectTranscripts(src SessionSource, state State, projectDir string, recentN, budget int) string {
|
||||
if src == nil {
|
||||
return ""
|
||||
}
|
||||
if budget <= 0 {
|
||||
budget = defaultTranscriptBudget
|
||||
}
|
||||
headers, err := src.List()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
selected := collectRecentSessions(headers, state, projectDir, recentN)
|
||||
if len(selected) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
for _, h := range selected {
|
||||
if b.Len() >= budget {
|
||||
break
|
||||
}
|
||||
_, msgs, err := src.Load(h.ID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
section := renderTranscript(h, msgs)
|
||||
if section == "" {
|
||||
continue
|
||||
}
|
||||
remaining := budget - b.Len()
|
||||
if len(section) > remaining {
|
||||
section = truncateBody(section, remaining)
|
||||
}
|
||||
b.WriteString(section)
|
||||
b.WriteString("\n")
|
||||
}
|
||||
return strings.TrimSpace(b.String())
|
||||
}
|
||||
|
||||
// renderTranscript renders one session's messages as a compact, role-tagged
|
||||
// transcript for the distiller. Tool-result bodies are included but bounded so a
|
||||
// single huge tool output cannot dominate the budget; empty messages are
|
||||
// skipped. The header line carries the session id and update time for context.
|
||||
func renderTranscript(h session.SessionHeader, msgs agentcore.MessageList) string {
|
||||
var b strings.Builder
|
||||
b.WriteString(fmt.Sprintf("## Session %s (updated %s)\n", h.ID, h.UpdatedAt.UTC().Format("2006-01-02")))
|
||||
wrote := false
|
||||
for _, m := range msgs {
|
||||
var role, text string
|
||||
switch mm := m.(type) {
|
||||
case agentcore.UserMessage:
|
||||
role, text = "user", agentcore.ContentToText(mm.Content)
|
||||
case agentcore.AssistantMessage:
|
||||
role, text = "assistant", agentcore.ContentToText(mm.Content)
|
||||
case agentcore.ToolResultMessage:
|
||||
role, text = "tool", clip(agentcore.ContentToText(mm.Content), 500)
|
||||
case agentcore.CompactionMessage:
|
||||
role, text = "summary", mm.Summary
|
||||
default:
|
||||
continue
|
||||
}
|
||||
text = strings.TrimSpace(text)
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
b.WriteString(role)
|
||||
b.WriteString(": ")
|
||||
b.WriteString(text)
|
||||
b.WriteString("\n")
|
||||
wrote = true
|
||||
}
|
||||
if !wrote {
|
||||
return ""
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// clip trims s to at most n bytes on a rune boundary, appending an ellipsis when
|
||||
// truncation occurred. Used to bound individual tool-result bodies inside a
|
||||
// transcript so one large output does not crowd out the rest.
|
||||
func clip(s string, n int) string {
|
||||
if n <= 0 || len(s) <= n {
|
||||
return s
|
||||
}
|
||||
cut := n
|
||||
for cut > 0 && !isRuneStart(s[cut]) {
|
||||
cut--
|
||||
}
|
||||
return s[:cut] + "…"
|
||||
}
|
||||
|
||||
// distilledEntry mirrors one element of the distiller's strict-JSON output: a
|
||||
// proposed new memory entry with its semantic type, target scope, a short title
|
||||
// (turned into a filename slug) and the Markdown body to persist.
|
||||
type distilledEntry struct {
|
||||
Type string `json:"type"`
|
||||
Scope string `json:"scope"`
|
||||
Title string `json:"title"`
|
||||
Body string `json:"body"`
|
||||
}
|
||||
|
||||
// distillResponse is the full strict-JSON schema the distill prompt requires.
|
||||
type distillResponse struct {
|
||||
Entries []distilledEntry `json:"entries"`
|
||||
Notes []string `json:"notes"`
|
||||
}
|
||||
|
||||
// parseDistillResponse turns the distiller's text into concrete NewEntry writes,
|
||||
// deduped against the existing memory bodies. It is conservative by
|
||||
// construction: an unparseable response, an unknown/ephemeral type, an empty
|
||||
// body, or an entry that is a near-duplicate of an existing memory (or of an
|
||||
// already-accepted new entry) is dropped rather than written (PRD FR-13/FR-14).
|
||||
// It never errors; a hard model failure is handled by the caller.
|
||||
//
|
||||
// memoryRoot + projectDir place each entry within the scope the Runner's
|
||||
// withinScope guard permits: global entries under <root>/global/<type>/, project
|
||||
// entries under <root>/projects/<id>/<type>/. A "project" entry on a global-only
|
||||
// run (empty projectDir) has nowhere valid to live and is skipped.
|
||||
func parseDistillResponse(raw string, existing []MemoryFile, memoryRoot, projectDir string) ([]NewEntry, []string) {
|
||||
body := extractJSONObject(raw)
|
||||
if body == "" {
|
||||
return nil, nil
|
||||
}
|
||||
var resp distillResponse
|
||||
if err := json.Unmarshal([]byte(body), &resp); err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Precompute token sets of existing bodies for near-duplicate rejection.
|
||||
existingTokens := make([]map[string]struct{}, 0, len(existing))
|
||||
for _, f := range existing {
|
||||
existingTokens = append(existingTokens, tokenize(f.Body))
|
||||
}
|
||||
|
||||
var out []NewEntry
|
||||
var notes []string
|
||||
acceptedTokens := make([]map[string]struct{}, 0)
|
||||
usedPaths := make(map[string]struct{})
|
||||
|
||||
for _, e := range resp.Entries {
|
||||
typ := strings.ToLower(strings.TrimSpace(e.Type))
|
||||
if _, ok := distillNewEntryTypes[typ]; !ok {
|
||||
continue
|
||||
}
|
||||
body := strings.TrimSpace(e.Body)
|
||||
if body == "" {
|
||||
continue
|
||||
}
|
||||
scopeDir, ok := distillScopeDir(e.Scope, projectDir)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
tok := tokenize(body)
|
||||
if isNearDup(tok, existingTokens) || isNearDup(tok, acceptedTokens) {
|
||||
notes = append(notes, fmt.Sprintf("distill: skipped near-duplicate of existing memory (%s)", firstLine(e.Title, body)))
|
||||
continue
|
||||
}
|
||||
path := distillPath(memoryRoot, scopeDir, typ, e.Title, body, usedPaths)
|
||||
usedPaths[filepath.Clean(path)] = struct{}{}
|
||||
acceptedTokens = append(acceptedTokens, tok)
|
||||
out = append(out, NewEntry{Path: path, Body: ensureTrailingNewline(body)})
|
||||
}
|
||||
|
||||
for _, n := range resp.Notes {
|
||||
if s := strings.TrimSpace(n); s != "" {
|
||||
notes = append(notes, s)
|
||||
}
|
||||
}
|
||||
return out, notes
|
||||
}
|
||||
|
||||
// distillScopeDir maps a requested scope ("global"/"project") to its on-disk
|
||||
// scope directory relative to the memory root. A "project" scope requires a
|
||||
// project dir; without one the entry cannot be placed and ok is false. An empty
|
||||
// or unrecognized scope defaults to global (the safe, always-valid target).
|
||||
func distillScopeDir(scope, projectDir string) (string, bool) {
|
||||
switch strings.ToLower(strings.TrimSpace(scope)) {
|
||||
case "project", "projects":
|
||||
if projectDir == "" {
|
||||
return "", false
|
||||
}
|
||||
return filepath.Join("projects", projectID(projectDir)), true
|
||||
default:
|
||||
return "global", true
|
||||
}
|
||||
}
|
||||
|
||||
// distillPath builds the absolute file path for a new distilled entry:
|
||||
// <memoryRoot>/<scopeDir>/<type>/<slug>.md. The slug derives from the title
|
||||
// (sanitized) or, when absent, a short content hash, so a re-run of the same
|
||||
// durable fact lands on a stable path (and is caught by the near-dup check
|
||||
// against the now-existing file, keeping distillation idempotent). If the path
|
||||
// is already taken within this run, a short body hash disambiguates it.
|
||||
func distillPath(memoryRoot, scopeDir, typ, title, body string, used map[string]struct{}) string {
|
||||
slug := slugify(title)
|
||||
if slug == "" {
|
||||
slug = shortHash(body)
|
||||
}
|
||||
name := slug + ".md"
|
||||
path := filepath.Join(memoryRoot, scopeDir, typ, name)
|
||||
if _, taken := used[filepath.Clean(path)]; taken {
|
||||
path = filepath.Join(memoryRoot, scopeDir, typ, slug+"-"+shortHash(body)+".md")
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
// isNearDup reports whether tok is at or above NearDupThreshold Jaccard
|
||||
// similarity with any set in others — the same conservative near-duplicate
|
||||
// signal the deterministic plan uses for merge candidates, reused here so a
|
||||
// distilled fact that already lives in memory is not re-added (PRD FR-13).
|
||||
func isNearDup(tok map[string]struct{}, others []map[string]struct{}) bool {
|
||||
for _, o := range others {
|
||||
if jaccard(tok, o) >= NearDupThreshold {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// slugify turns a title into a lowercase, dash-separated filename stem of ASCII
|
||||
// alphanumerics, bounding the length so paths stay reasonable. Non-alphanumeric
|
||||
// runs collapse to a single dash; leading/trailing dashes are trimmed. A title
|
||||
// with no usable characters yields "".
|
||||
func slugify(title string) string {
|
||||
var b strings.Builder
|
||||
lastDash := false
|
||||
for _, r := range strings.ToLower(strings.TrimSpace(title)) {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
lastDash = false
|
||||
default:
|
||||
if !lastDash && b.Len() > 0 {
|
||||
b.WriteByte('-')
|
||||
lastDash = true
|
||||
}
|
||||
}
|
||||
}
|
||||
slug := strings.Trim(b.String(), "-")
|
||||
const maxLen = 60
|
||||
if len(slug) > maxLen {
|
||||
slug = strings.Trim(slug[:maxLen], "-")
|
||||
}
|
||||
return slug
|
||||
}
|
||||
|
||||
// firstLine returns a short human-readable label for a note: the trimmed title
|
||||
// when present, else the first line of body, truncated.
|
||||
func firstLine(title, body string) string {
|
||||
s := strings.TrimSpace(title)
|
||||
if s == "" {
|
||||
s = strings.TrimSpace(body)
|
||||
}
|
||||
if i := strings.IndexByte(s, '\n'); i >= 0 {
|
||||
s = s[:i]
|
||||
}
|
||||
return clip(s, 60)
|
||||
}
|
||||
|
||||
// ensureTrailingNewline guarantees body ends with exactly one newline so written
|
||||
// memory files match the one-entry-per-file convention and diff cleanly.
|
||||
func ensureTrailingNewline(body string) string {
|
||||
return strings.TrimRight(body, "\n") + "\n"
|
||||
}
|
||||
|
||||
// shortHash returns the first 8 hex chars of sha256(s), a stable disambiguator
|
||||
// for slugs derived from identical/absent titles.
|
||||
func shortHash(s string) string {
|
||||
sum := sha256.Sum256([]byte(s))
|
||||
return hex.EncodeToString(sum[:])[:8]
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
package dream
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/smallnest/pigo/internal/agentcore"
|
||||
"github.com/smallnest/pigo/internal/session"
|
||||
)
|
||||
|
||||
// stubSessions is an in-memory SessionSource for distill tests: it lists the
|
||||
// headers it was given and returns canned messages per id, so no real session
|
||||
// files (or LLM) are touched.
|
||||
type stubSessions struct {
|
||||
headers []session.SessionHeader
|
||||
msgs map[string]agentcore.MessageList
|
||||
}
|
||||
|
||||
func (s *stubSessions) List() ([]session.SessionHeader, error) { return s.headers, nil }
|
||||
|
||||
func (s *stubSessions) Load(id string) (session.SessionHeader, agentcore.MessageList, error) {
|
||||
for _, h := range s.headers {
|
||||
if h.ID == id {
|
||||
return h, s.msgs[id], nil
|
||||
}
|
||||
}
|
||||
return session.SessionHeader{}, nil, os.ErrNotExist
|
||||
}
|
||||
|
||||
func userMsg(text string) agentcore.UserMessage {
|
||||
return agentcore.UserMessage{RoleField: agentcore.RoleUser, Content: agentcore.ContentList{agentcore.NewTextContent(text)}}
|
||||
}
|
||||
|
||||
func asstMsg(text string) agentcore.AssistantMessage {
|
||||
return agentcore.AssistantMessage{RoleField: agentcore.RoleAssistant, Content: agentcore.ContentList{agentcore.NewTextContent(text)}}
|
||||
}
|
||||
|
||||
// TestCollectRecentSessionsFirstRunWindow: never-run (zero LastRunAt) selects the
|
||||
// most-recent recentN matching sessions, ordered most-recent-first, filtered to
|
||||
// the active project.
|
||||
func TestCollectRecentSessionsFirstRunWindow(t *testing.T) {
|
||||
proj := t.TempDir()
|
||||
other := t.TempDir()
|
||||
base := time.Now().UTC()
|
||||
headers := []session.SessionHeader{
|
||||
{ID: "s1", Cwd: proj, UpdatedAt: base.Add(-3 * time.Hour)},
|
||||
{ID: "s2", Cwd: proj, UpdatedAt: base.Add(-1 * time.Hour)},
|
||||
{ID: "s3", Cwd: other, UpdatedAt: base}, // different project, excluded
|
||||
{ID: "s4", Cwd: proj, UpdatedAt: base.Add(-2 * time.Hour)},
|
||||
{ID: "s5", Cwd: "", UpdatedAt: base}, // unattributed, excluded
|
||||
}
|
||||
got := collectRecentSessions(headers, State{}, proj, 2)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("got %d sessions, want 2 (recentN cap)", len(got))
|
||||
}
|
||||
if got[0].ID != "s2" || got[1].ID != "s4" {
|
||||
t.Fatalf("wrong window/order: %s, %s (want s2, s4 most-recent-first)", got[0].ID, got[1].ID)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCollectRecentSessionsIncremental: a non-zero LastRunAt selects only
|
||||
// matching sessions updated strictly after it (incremental distillation).
|
||||
func TestCollectRecentSessionsIncremental(t *testing.T) {
|
||||
proj := t.TempDir()
|
||||
base := time.Now().UTC()
|
||||
last := base.Add(-2 * time.Hour)
|
||||
headers := []session.SessionHeader{
|
||||
{ID: "old", Cwd: proj, UpdatedAt: base.Add(-3 * time.Hour)}, // before last run
|
||||
{ID: "new1", Cwd: proj, UpdatedAt: base.Add(-1 * time.Hour)},
|
||||
{ID: "new2", Cwd: proj, UpdatedAt: base},
|
||||
}
|
||||
got := collectRecentSessions(headers, State{LastRunAt: last}, proj, 20)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("got %d, want 2 (only after LastRunAt)", len(got))
|
||||
}
|
||||
for _, h := range got {
|
||||
if !h.UpdatedAt.After(last) {
|
||||
t.Fatalf("session %s not after LastRunAt", h.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestCollectRecentSessionsGlobalOnlyNoMatch: an empty projectDir (global-only
|
||||
// run) matches nothing, so the window is empty (no-op path).
|
||||
func TestCollectRecentSessionsGlobalOnlyNoMatch(t *testing.T) {
|
||||
proj := t.TempDir()
|
||||
headers := []session.SessionHeader{{ID: "s1", Cwd: proj, UpdatedAt: time.Now().UTC()}}
|
||||
if got := collectRecentSessions(headers, State{}, "", 20); len(got) != 0 {
|
||||
t.Fatalf("global-only run must match no sessions, got %d", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
// TestCollectTranscriptsNoSourceOrNoMatch: nil source or no matching session
|
||||
// yields "" so the caller records Distilled=0 with a "无新增" note.
|
||||
func TestCollectTranscriptsNoSourceOrNoMatch(t *testing.T) {
|
||||
if got := collectTranscripts(nil, State{}, t.TempDir(), 20, 0); got != "" {
|
||||
t.Fatalf("nil source must yield empty transcript, got %q", got)
|
||||
}
|
||||
src := &stubSessions{headers: []session.SessionHeader{{ID: "s1", Cwd: t.TempDir(), UpdatedAt: time.Now().UTC()}}}
|
||||
// Ask for a different (empty) project → no match.
|
||||
if got := collectTranscripts(src, State{}, "", 20, 0); got != "" {
|
||||
t.Fatalf("no matching session must yield empty transcript, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCollectTranscriptsRendersRoleTagged: matching sessions are rendered into a
|
||||
// role-tagged transcript containing the session id and message text.
|
||||
func TestCollectTranscriptsRendersRoleTagged(t *testing.T) {
|
||||
proj := t.TempDir()
|
||||
src := &stubSessions{
|
||||
headers: []session.SessionHeader{{ID: "sess-abc", Cwd: proj, UpdatedAt: time.Now().UTC()}},
|
||||
msgs: map[string]agentcore.MessageList{
|
||||
"sess-abc": {userMsg("I always use tabs not spaces"), asstMsg("noted")},
|
||||
},
|
||||
}
|
||||
got := collectTranscripts(src, State{}, proj, 20, 0)
|
||||
for _, want := range []string{"sess-abc", "user: I always use tabs", "assistant: noted"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Fatalf("transcript missing %q\n---\n%s", want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseDistillResponseNewEntries: a canned JSON response yields NewEntry
|
||||
// writes under the right scope/type dirs, and ephemeral/unknown types are
|
||||
// dropped.
|
||||
func TestParseDistillResponseNewEntries(t *testing.T) {
|
||||
root := "/mem"
|
||||
proj := t.TempDir()
|
||||
pid := projectID(proj)
|
||||
raw := "```json\n" + `{
|
||||
"entries": [
|
||||
{"type": "user", "scope": "global", "title": "Tabs preference", "body": "Developer prefers tabs over spaces."},
|
||||
{"type": "project", "scope": "project", "title": "Arch", "body": "The runner is deterministic; the consolidator is the LLM half."},
|
||||
{"type": "todo", "scope": "global", "title": "bad", "body": "ephemeral one-shot task"},
|
||||
{"type": "user", "scope": "global", "title": "empty", "body": " "}
|
||||
],
|
||||
"notes": ["distilled 2"]
|
||||
}` + "\n```"
|
||||
|
||||
entries, notes := parseDistillResponse(raw, nil, root, proj)
|
||||
if len(entries) != 2 {
|
||||
t.Fatalf("got %d entries, want 2 (todo + empty dropped): %+v", len(entries), entries)
|
||||
}
|
||||
byPath := map[string]string{}
|
||||
for _, e := range entries {
|
||||
byPath[e.Path] = e.Body
|
||||
}
|
||||
wantGlobal := filepath.Join(root, "global", "user", "tabs-preference.md")
|
||||
wantProj := filepath.Join(root, "projects", pid, "project", "arch.md")
|
||||
if _, ok := byPath[wantGlobal]; !ok {
|
||||
t.Fatalf("missing global user entry at %s; got %v", wantGlobal, byPath)
|
||||
}
|
||||
if body, ok := byPath[wantProj]; !ok {
|
||||
t.Fatalf("missing project entry at %s; got %v", wantProj, byPath)
|
||||
} else if !strings.HasSuffix(body, "\n") {
|
||||
t.Fatalf("body must end with newline: %q", body)
|
||||
}
|
||||
if strings.Join(notes, "|") == "" {
|
||||
t.Fatal("expected distill notes surfaced")
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseDistillResponseProjectScopeNeedsProjectDir: a "project"-scoped entry on
|
||||
// a global-only run (empty projectDir) has nowhere valid to live and is skipped.
|
||||
func TestParseDistillResponseProjectScopeNeedsProjectDir(t *testing.T) {
|
||||
raw := `{"entries":[{"type":"project","scope":"project","title":"x","body":"y"}]}`
|
||||
entries, _ := parseDistillResponse(raw, nil, "/mem", "")
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("project entry on global-only run must be skipped, got %+v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseDistillResponseDedupAgainstExisting: an entry that is a near-duplicate
|
||||
// of an existing memory is dropped (FR-13).
|
||||
func TestParseDistillResponseDedupAgainstExisting(t *testing.T) {
|
||||
existing := []MemoryFile{{Body: "Developer prefers tabs over spaces in all files"}}
|
||||
raw := `{"entries":[{"type":"user","scope":"global","title":"tabs","body":"Developer prefers tabs over spaces in all files"}]}`
|
||||
entries, notes := parseDistillResponse(raw, existing, "/mem", "")
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("near-duplicate of existing memory must be dropped, got %+v", entries)
|
||||
}
|
||||
if len(notes) == 0 || !strings.Contains(strings.Join(notes, "|"), "near-duplicate") {
|
||||
t.Fatalf("expected a near-duplicate skip note, got %v", notes)
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseDistillResponseUnparseable: an unparseable response adds nothing and
|
||||
// is not an error (conservative KEEP).
|
||||
func TestParseDistillResponseUnparseable(t *testing.T) {
|
||||
entries, notes := parseDistillResponse("the model rambled with no json", nil, "/mem", "")
|
||||
if entries != nil || notes != nil {
|
||||
t.Fatalf("unparseable response must yield nothing, got %+v / %v", entries, notes)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunDistillsThroughRunner: an integration test with a stub session source
|
||||
// and a stub completer-backed llmConsolidator distills a durable fact into a new
|
||||
// on-disk memory entry, counts it, and reports it.
|
||||
func TestRunDistillsThroughRunner(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
proj := t.TempDir()
|
||||
|
||||
src := &stubSessions{
|
||||
headers: []session.SessionHeader{{ID: "s1", Cwd: proj, UpdatedAt: time.Now().UTC()}},
|
||||
msgs: map[string]agentcore.MessageList{"s1": {userMsg("always run tests with gotestsum")}},
|
||||
}
|
||||
// llmConsolidator whose distill completion returns one durable fact. The
|
||||
// merge/prune completion is not reached because there are no eligible files.
|
||||
cons := &llmConsolidator{complete: func(_ context.Context, _, _ string) (string, error) {
|
||||
return `{"entries":[{"type":"user","scope":"global","title":"Test runner","body":"Always run tests with gotestsum."}],"notes":["one fact"]}`, nil
|
||||
}}
|
||||
r := &Runner{MemoryRoot: root, Consolidator: cons, Sessions: src}
|
||||
|
||||
rep, err := r.Run(context.Background(), RunOptions{ProjectDir: proj})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if rep.Distilled != 1 {
|
||||
t.Fatalf("Distilled = %d, want 1", rep.Distilled)
|
||||
}
|
||||
newPath := filepath.Join(root, "global", "user", "test-runner.md")
|
||||
raw, err := os.ReadFile(newPath)
|
||||
if err != nil {
|
||||
t.Fatalf("distilled entry not written at %s: %v", newPath, err)
|
||||
}
|
||||
if !strings.Contains(string(raw), "gotestsum") {
|
||||
t.Fatalf("distilled body wrong: %q", raw)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunDryRunDistillsButWritesNothing: a dry-run still runs the distill pass
|
||||
// and reports the count, but writes no new memory files and updates no state.
|
||||
func TestRunDryRunDistillsButWritesNothing(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
proj := t.TempDir()
|
||||
|
||||
src := &stubSessions{
|
||||
headers: []session.SessionHeader{{ID: "s1", Cwd: proj, UpdatedAt: time.Now().UTC()}},
|
||||
msgs: map[string]agentcore.MessageList{"s1": {userMsg("my stack is Go plus SQLite")}},
|
||||
}
|
||||
cons := &llmConsolidator{complete: func(_ context.Context, _, _ string) (string, error) {
|
||||
return `{"entries":[{"type":"user","scope":"global","title":"Stack","body":"Stack is Go plus SQLite."}],"notes":[]}`, nil
|
||||
}}
|
||||
r := &Runner{MemoryRoot: root, Consolidator: cons, Sessions: src}
|
||||
|
||||
rep, err := r.Run(context.Background(), RunOptions{ProjectDir: proj, DryRun: true})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if rep.Distilled != 1 {
|
||||
t.Fatalf("dry-run Distilled = %d, want 1 (still reported)", rep.Distilled)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "global", "user", "stack.md")); !os.IsNotExist(err) {
|
||||
t.Fatalf("dry-run must not write the distilled entry (err=%v)", err)
|
||||
}
|
||||
st, _ := LoadState(root)
|
||||
if !st.LastRunAt.IsZero() || st.LastStatus != "" {
|
||||
t.Fatalf("dry-run must not update state: %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunNoDurableFactsNoOp: when distillation adds nothing, the report records
|
||||
// Distilled=0 with the "无新增" note (SPEC §5.5).
|
||||
func TestRunNoDurableFactsNoOp(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
proj := t.TempDir()
|
||||
src := &stubSessions{
|
||||
headers: []session.SessionHeader{{ID: "s1", Cwd: proj, UpdatedAt: time.Now().UTC()}},
|
||||
msgs: map[string]agentcore.MessageList{"s1": {userMsg("some transient chatter")}},
|
||||
}
|
||||
cons := &llmConsolidator{complete: func(_ context.Context, _, _ string) (string, error) {
|
||||
return `{"entries":[],"notes":[]}`, nil
|
||||
}}
|
||||
r := &Runner{MemoryRoot: root, Consolidator: cons, Sessions: src}
|
||||
rep, err := r.Run(context.Background(), RunOptions{ProjectDir: proj})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if rep.Distilled != 0 {
|
||||
t.Fatalf("Distilled = %d, want 0", rep.Distilled)
|
||||
}
|
||||
if !strings.Contains(strings.Join(rep.Notes, "|"), "无新增") {
|
||||
t.Fatalf("expected '无新增' no-op note, got %v", rep.Notes)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package dream
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrLocked is returned by AcquireLock when a live (non-stale) lock is already
|
||||
// held by another process. Callers detect it (via errors.Is) to exit "skipped"
|
||||
// rather than treating it as a real failure. It is deliberately distinct from
|
||||
// the I/O errors AcquireLock may also return.
|
||||
var ErrLocked = errors.New("dream: consolidation already running")
|
||||
|
||||
// DefaultStaleAfter is how long after a lock's started_at the lock is considered
|
||||
// abandoned (e.g. the holder crashed) and may be taken over. See spec §5.4.
|
||||
var DefaultStaleAfter = 30 * time.Minute
|
||||
|
||||
// lockInfo is the JSON body persisted in the lock file.
|
||||
type lockInfo struct {
|
||||
PID int `json:"pid"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
}
|
||||
|
||||
// Lock represents an acquired dream single-instance lock backed by the file
|
||||
// <memoryRoot>/global/dream/dream.lock. It guarantees at most one consolidation
|
||||
// runs at a time across processes, with crash-safe stale takeover.
|
||||
type Lock struct {
|
||||
path string
|
||||
released bool
|
||||
}
|
||||
|
||||
// lockPath is the lock file location under the memory root. It is a separate
|
||||
// file from state.json and never touches it.
|
||||
func lockPath(memoryRoot string) string {
|
||||
return filepath.Join(memoryRoot, "global", "dream", "dream.lock")
|
||||
}
|
||||
|
||||
// AcquireLock attempts to acquire the dream single-instance lock under
|
||||
// memoryRoot. On success it returns a *Lock the caller must Release (typically
|
||||
// via defer). If a live lock is already held it returns ErrLocked. If the
|
||||
// existing lock is stale (its started_at is older than now-DefaultStaleAfter) or
|
||||
// malformed/unparseable, it is treated as abandoned and taken over. Any other
|
||||
// error (permissions, unexpected I/O) is returned as-is so the caller can
|
||||
// distinguish it from the ErrLocked "skipped" case.
|
||||
func AcquireLock(memoryRoot string) (*Lock, error) {
|
||||
dir := filepath.Join(memoryRoot, "global", "dream")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
path := lockPath(memoryRoot)
|
||||
|
||||
// First attempt: atomic exclusive create.
|
||||
l, err := createLock(path)
|
||||
if err == nil {
|
||||
return l, nil
|
||||
}
|
||||
if !os.IsExist(err) {
|
||||
// Real I/O error (permissions, etc.), not a contention signal.
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// A lock file already exists. Decide whether it is stale and takeable.
|
||||
if !staleLock(path, time.Now()) {
|
||||
return nil, ErrLocked
|
||||
}
|
||||
|
||||
// Stale (or malformed) lock: take it over. Removing then re-creating with
|
||||
// O_EXCL keeps the create atomic. A racing process that recreates the file
|
||||
// between our Remove and create will cause our create to fail with EEXIST;
|
||||
// we surface that as ErrLocked (the other process won the race).
|
||||
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
||||
return nil, err
|
||||
}
|
||||
l, err = createLock(path)
|
||||
if err != nil {
|
||||
if os.IsExist(err) {
|
||||
return nil, ErrLocked
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return l, nil
|
||||
}
|
||||
|
||||
// createLock atomically creates the lock file with O_EXCL and writes the current
|
||||
// pid + start time as JSON. On EEXIST it returns an error for which os.IsExist
|
||||
// is true.
|
||||
func createLock(path string) (*Lock, error) {
|
||||
f, err := os.OpenFile(path, os.O_EXCL|os.O_CREATE|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data, err := json.Marshal(lockInfo{PID: os.Getpid(), StartedAt: time.Now().UTC()})
|
||||
if err != nil {
|
||||
f.Close()
|
||||
os.Remove(path)
|
||||
return nil, err
|
||||
}
|
||||
if _, err := f.Write(data); err != nil {
|
||||
f.Close()
|
||||
os.Remove(path)
|
||||
return nil, err
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
os.Remove(path)
|
||||
return nil, err
|
||||
}
|
||||
return &Lock{path: path}, nil
|
||||
}
|
||||
|
||||
// staleLock reports whether the lock file at path is stale (takeable) as of now.
|
||||
// A lock is stale when its started_at is older than now-DefaultStaleAfter. A
|
||||
// missing, unreadable, or malformed/unparseable lock file is also treated as
|
||||
// stale so a corrupt lock never wedges dream permanently.
|
||||
func staleLock(path string, now time.Time) bool {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
// Missing or unreadable: treat as takeable.
|
||||
return true
|
||||
}
|
||||
var info lockInfo
|
||||
if err := json.Unmarshal(data, &info); err != nil {
|
||||
// Malformed lock body: treat as stale.
|
||||
return true
|
||||
}
|
||||
if info.StartedAt.IsZero() {
|
||||
// No usable timestamp: treat as stale.
|
||||
return true
|
||||
}
|
||||
return now.Sub(info.StartedAt) > DefaultStaleAfter
|
||||
}
|
||||
|
||||
// Release removes the lock file. It is safe to call in a defer and safe to
|
||||
// double-call: a second call (or a call after the file was already removed) is a
|
||||
// no-op and never panics. A missing file is not an error.
|
||||
func (l *Lock) Release() error {
|
||||
if l == nil || l.released {
|
||||
return nil
|
||||
}
|
||||
l.released = true
|
||||
if err := os.Remove(l.path); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
package dream
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// writeLock writes a lock file with the given pid and started_at directly, for
|
||||
// tests that need to simulate a pre-existing (possibly stale) lock.
|
||||
func writeLock(t *testing.T, root string, pid int, startedAt time.Time) string {
|
||||
t.Helper()
|
||||
dir := filepath.Join(root, "global", "dream")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(dir, "dream.lock")
|
||||
data, err := json.Marshal(lockInfo{PID: pid, StartedAt: startedAt})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func TestAcquireLockMutualExclusion(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
l1, err := AcquireLock(root)
|
||||
if err != nil {
|
||||
t.Fatalf("first AcquireLock: %v", err)
|
||||
}
|
||||
defer l1.Release()
|
||||
|
||||
// A second acquire while the first is live must fail with ErrLocked.
|
||||
l2, err := AcquireLock(root)
|
||||
if !errors.Is(err, ErrLocked) {
|
||||
t.Fatalf("second AcquireLock err = %v, want ErrLocked", err)
|
||||
}
|
||||
if l2 != nil {
|
||||
t.Errorf("second AcquireLock returned non-nil Lock alongside error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcquireLockCreatesFileWithBody(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
l, err := AcquireLock(root)
|
||||
if err != nil {
|
||||
t.Fatalf("AcquireLock: %v", err)
|
||||
}
|
||||
defer l.Release()
|
||||
|
||||
path := filepath.Join(root, "global", "dream", "dream.lock")
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read lock file: %v", err)
|
||||
}
|
||||
var info lockInfo
|
||||
if err := json.Unmarshal(data, &info); err != nil {
|
||||
t.Fatalf("lock body not valid JSON: %v", err)
|
||||
}
|
||||
if info.PID != os.Getpid() {
|
||||
t.Errorf("lock pid = %d, want %d", info.PID, os.Getpid())
|
||||
}
|
||||
if info.StartedAt.IsZero() {
|
||||
t.Errorf("lock started_at is zero, want current time")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcquireLockDoesNotTouchState(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
l, err := AcquireLock(root)
|
||||
if err != nil {
|
||||
t.Fatalf("AcquireLock: %v", err)
|
||||
}
|
||||
defer l.Release()
|
||||
if _, err := os.Stat(filepath.Join(root, "global", "dream", "state.json")); !os.IsNotExist(err) {
|
||||
t.Errorf("AcquireLock created/left state.json (err=%v); lock must be a separate file", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcquireLockStaleTakeover(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
// Existing lock older than staleAfter → takeable.
|
||||
writeLock(t, root, 99999, time.Now().Add(-DefaultStaleAfter-time.Minute))
|
||||
|
||||
l, err := AcquireLock(root)
|
||||
if err != nil {
|
||||
t.Fatalf("AcquireLock over stale lock err = %v, want takeover success", err)
|
||||
}
|
||||
defer l.Release()
|
||||
|
||||
// The lock body should now reflect our pid.
|
||||
data, err := os.ReadFile(filepath.Join(root, "global", "dream", "dream.lock"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var info lockInfo
|
||||
if err := json.Unmarshal(data, &info); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.PID != os.Getpid() {
|
||||
t.Errorf("after takeover pid = %d, want %d (our pid)", info.PID, os.Getpid())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcquireLockFreshLockNotTakeable(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
// A recently-started lock is live, not stale.
|
||||
writeLock(t, root, 99999, time.Now().Add(-time.Minute))
|
||||
|
||||
if _, err := AcquireLock(root); !errors.Is(err, ErrLocked) {
|
||||
t.Fatalf("AcquireLock over fresh lock err = %v, want ErrLocked", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcquireLockMalformedTakeover(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
dir := filepath.Join(root, "global", "dream")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "dream.lock"), []byte("{garbage not json"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
l, err := AcquireLock(root)
|
||||
if err != nil {
|
||||
t.Fatalf("AcquireLock over malformed lock err = %v, want takeover success", err)
|
||||
}
|
||||
defer l.Release()
|
||||
}
|
||||
|
||||
func TestReleaseAndReacquire(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
l1, err := AcquireLock(root)
|
||||
if err != nil {
|
||||
t.Fatalf("first AcquireLock: %v", err)
|
||||
}
|
||||
if err := l1.Release(); err != nil {
|
||||
t.Fatalf("Release: %v", err)
|
||||
}
|
||||
// File must be gone after release.
|
||||
if _, err := os.Stat(filepath.Join(root, "global", "dream", "dream.lock")); !os.IsNotExist(err) {
|
||||
t.Errorf("lock file still present after Release (err=%v)", err)
|
||||
}
|
||||
// Re-acquire must succeed now that the lock is free.
|
||||
l2, err := AcquireLock(root)
|
||||
if err != nil {
|
||||
t.Fatalf("re-acquire after release: %v", err)
|
||||
}
|
||||
defer l2.Release()
|
||||
}
|
||||
|
||||
func TestReleaseDoubleCallSafe(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
l, err := AcquireLock(root)
|
||||
if err != nil {
|
||||
t.Fatalf("AcquireLock: %v", err)
|
||||
}
|
||||
if err := l.Release(); err != nil {
|
||||
t.Fatalf("first Release: %v", err)
|
||||
}
|
||||
// Second Release (and even after the file is gone) must not panic or error.
|
||||
if err := l.Release(); err != nil {
|
||||
t.Errorf("second Release err = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReleaseNilSafe(t *testing.T) {
|
||||
var l *Lock
|
||||
if err := l.Release(); err != nil {
|
||||
t.Errorf("nil Lock Release err = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStaleLockBoundary(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
path := writeLock(t, root, 1, time.Unix(0, 0).UTC())
|
||||
now := time.Unix(0, 0).UTC()
|
||||
if staleLock(path, now.Add(DefaultStaleAfter-time.Second)) {
|
||||
t.Errorf("lock within staleAfter reported stale")
|
||||
}
|
||||
if !staleLock(path, now.Add(DefaultStaleAfter+time.Second)) {
|
||||
t.Errorf("lock past staleAfter reported not stale")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,359 @@
|
||||
package dream
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// NearDupThreshold is the normalized-token Jaccard similarity at or above which
|
||||
// two memory files are emitted as a near-duplicate candidate pair for the later
|
||||
// LLM merge-decision step (spec §5.1.1). It is deliberately conservative: this
|
||||
// node only PAIRS candidates, it never merges, so a false-positive pair costs
|
||||
// only one extra LLM comparison while a missed pair silently loses a merge
|
||||
// opportunity.
|
||||
const NearDupThreshold = 0.7
|
||||
|
||||
// MemoryFile is one enumerated memory file under the global or the active
|
||||
// project scope, with the derived metadata the deterministic plan needs. Body
|
||||
// is retained so the near-duplicate pairing can tokenize it without a second
|
||||
// read.
|
||||
type MemoryFile struct {
|
||||
Path string // absolute path on disk
|
||||
Scope string // "global" | "projects"
|
||||
Type string // layout <type> segment ("" for a file directly under the scope root)
|
||||
Size int64 // byte size of the file content
|
||||
ContentHash string // sha256 hex of the raw file content
|
||||
Body string // full file content (used for path extraction + tokenization)
|
||||
}
|
||||
|
||||
// DedupeGroup is a set of two or more files whose content is byte-identical
|
||||
// (same ContentHash). The apply node keeps one representative and removes the
|
||||
// rest; Deduped in the Report counts len(Paths)-1 per group.
|
||||
type DedupeGroup struct {
|
||||
Hash string `json:"hash"`
|
||||
Paths []string `json:"paths"`
|
||||
}
|
||||
|
||||
// InvalidPathRef is a local filesystem path mentioned in a memory file's body
|
||||
// that no longer exists on disk. External references (URLs, mailto:, etc.) are
|
||||
// never recorded here (spec §5.2 / PRD US-004).
|
||||
type InvalidPathRef struct {
|
||||
File string `json:"file"` // memory file containing the reference
|
||||
Ref string `json:"ref"` // the original (unresolved) reference text
|
||||
}
|
||||
|
||||
// NearDupPair is a candidate pair of files whose token-set similarity is at or
|
||||
// above NearDupThreshold but whose content is not byte-identical. It is a
|
||||
// suggestion for the LLM to decide whether an actual merge is warranted; this
|
||||
// node performs no merge.
|
||||
type NearDupPair struct {
|
||||
A string `json:"a"`
|
||||
B string `json:"b"`
|
||||
Similarity float64 `json:"similarity"`
|
||||
}
|
||||
|
||||
// Plan is the plain-data output of the deterministic half of a /dream run. It
|
||||
// carries no behavior and makes no LLM calls; the later apply node consumes it
|
||||
// to drive merges/prunes and to compute the final Report counters.
|
||||
type Plan struct {
|
||||
Files []MemoryFile `json:"files"`
|
||||
DedupeGroups []DedupeGroup `json:"dedupe_groups"`
|
||||
InvalidPathRefs []InvalidPathRef `json:"invalid_path_refs"`
|
||||
NearDupPairs []NearDupPair `json:"near_dup_pairs"`
|
||||
BytesBefore int64 `json:"bytes_before"`
|
||||
FilesBefore int `json:"files_before"`
|
||||
}
|
||||
|
||||
// BuildPlan enumerates the consolidation-eligible memory files (global scope +
|
||||
// the given project's scope) under memoryRoot and computes the deterministic
|
||||
// consolidation plan: exact-dedup groups, invalid local path references, and
|
||||
// near-duplicate candidate pairs. It excludes the sessions scope entirely and
|
||||
// any file whose layout type is "checkpoint" (session-transient state, not
|
||||
// long-term memory — spec §1.3 / §5.1.1).
|
||||
//
|
||||
// projectDir is the working directory of the active project; its stable project
|
||||
// id (matching internal/memory's resolveProjectId) selects the projects
|
||||
// sub-scope. An empty projectDir restricts the plan to the global scope. A
|
||||
// missing memoryRoot or scope directory is not an error — it yields an empty
|
||||
// plan, mirroring memory.Reconcile's tolerance of an absent root.
|
||||
func BuildPlan(memoryRoot, projectDir string) (Plan, error) {
|
||||
var plan Plan
|
||||
|
||||
globalRoot := filepath.Join(memoryRoot, "global")
|
||||
globalFiles, err := enumerateScope(globalRoot, "global")
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
files := globalFiles
|
||||
|
||||
if projectDir != "" {
|
||||
projectRoot := filepath.Join(memoryRoot, "projects", projectID(projectDir))
|
||||
projFiles, err := enumerateScope(projectRoot, "projects")
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
files = append(files, projFiles...)
|
||||
}
|
||||
|
||||
plan.Files = files
|
||||
plan.FilesBefore = len(files)
|
||||
for _, f := range files {
|
||||
plan.BytesBefore += f.Size
|
||||
}
|
||||
|
||||
plan.DedupeGroups = dedupeGroups(files)
|
||||
plan.InvalidPathRefs = invalidPathRefs(files, projectDir)
|
||||
plan.NearDupPairs = nearDupPairs(files)
|
||||
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
// enumerateScope walks scopeRoot for *.md files, deriving each file's layout
|
||||
// type from its first path segment relative to scopeRoot. Files under a
|
||||
// "checkpoint" type directory are skipped. A missing scopeRoot yields no files
|
||||
// and no error.
|
||||
func enumerateScope(scopeRoot, scope string) ([]MemoryFile, error) {
|
||||
var out []MemoryFile
|
||||
err := filepath.WalkDir(scopeRoot, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if d.IsDir() || !strings.EqualFold(filepath.Ext(d.Name()), ".md") {
|
||||
return nil
|
||||
}
|
||||
rel, relErr := filepath.Rel(scopeRoot, path)
|
||||
if relErr != nil {
|
||||
return nil
|
||||
}
|
||||
segs := strings.Split(filepath.ToSlash(rel), "/")
|
||||
typ := ""
|
||||
if len(segs) >= 2 {
|
||||
typ = strings.ToLower(segs[0])
|
||||
}
|
||||
if typ == "checkpoint" {
|
||||
return nil
|
||||
}
|
||||
raw, readErr := os.ReadFile(path)
|
||||
if readErr != nil {
|
||||
if os.IsNotExist(readErr) {
|
||||
return nil
|
||||
}
|
||||
return readErr
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
out = append(out, MemoryFile{
|
||||
Path: filepath.Clean(path),
|
||||
Scope: scope,
|
||||
Type: typ,
|
||||
Size: int64(len(raw)),
|
||||
ContentHash: hex.EncodeToString(sum[:]),
|
||||
Body: string(raw),
|
||||
})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Deterministic order regardless of directory iteration order.
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Path < out[j].Path })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// dedupeGroups groups files by identical content hash, returning only groups
|
||||
// with two or more members (a file with a unique hash is not a duplicate). The
|
||||
// result is stable: groups are ordered by hash, paths within a group by path.
|
||||
func dedupeGroups(files []MemoryFile) []DedupeGroup {
|
||||
byHash := make(map[string][]string)
|
||||
for _, f := range files {
|
||||
byHash[f.ContentHash] = append(byHash[f.ContentHash], f.Path)
|
||||
}
|
||||
var groups []DedupeGroup
|
||||
for hash, paths := range byHash {
|
||||
if len(paths) < 2 {
|
||||
continue
|
||||
}
|
||||
sort.Strings(paths)
|
||||
groups = append(groups, DedupeGroup{Hash: hash, Paths: paths})
|
||||
}
|
||||
sort.Slice(groups, func(i, j int) bool { return groups[i].Hash < groups[j].Hash })
|
||||
return groups
|
||||
}
|
||||
|
||||
// invalidPathRefs scans each file's body for local filesystem path references
|
||||
// and records the ones that no longer exist. Relative references are resolved
|
||||
// against projectDir; "~/" against the user home dir. URLs and other external
|
||||
// references are never considered (see extractLocalPathRefs).
|
||||
func invalidPathRefs(files []MemoryFile, projectDir string) []InvalidPathRef {
|
||||
var out []InvalidPathRef
|
||||
for _, f := range files {
|
||||
seen := make(map[string]struct{})
|
||||
for _, ref := range extractLocalPathRefs(f.Body) {
|
||||
if _, dup := seen[ref]; dup {
|
||||
continue
|
||||
}
|
||||
seen[ref] = struct{}{}
|
||||
// A bare relative reference is only meaningful against a project
|
||||
// base. Without one (global-only plan) skip it rather than
|
||||
// resolving against an arbitrary cwd and flagging spuriously.
|
||||
isRelative := !filepath.IsAbs(ref) && !strings.HasPrefix(ref, "~/")
|
||||
if isRelative && projectDir == "" {
|
||||
continue
|
||||
}
|
||||
resolved := resolveRef(ref, projectDir)
|
||||
if _, err := os.Stat(resolved); err != nil && os.IsNotExist(err) {
|
||||
out = append(out, InvalidPathRef{File: f.Path, Ref: ref})
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// nearDupPairs emits candidate pairs whose normalized-token Jaccard similarity
|
||||
// is at or above NearDupThreshold. Pairs of byte-identical files are skipped:
|
||||
// those are exact duplicates handled by dedupeGroups, not near-duplicates.
|
||||
func nearDupPairs(files []MemoryFile) []NearDupPair {
|
||||
tokens := make([]map[string]struct{}, len(files))
|
||||
for i, f := range files {
|
||||
tokens[i] = tokenize(f.Body)
|
||||
}
|
||||
var out []NearDupPair
|
||||
for i := 0; i < len(files); i++ {
|
||||
for j := i + 1; j < len(files); j++ {
|
||||
if files[i].ContentHash == files[j].ContentHash {
|
||||
continue
|
||||
}
|
||||
sim := jaccard(tokens[i], tokens[j])
|
||||
if sim >= NearDupThreshold {
|
||||
out = append(out, NearDupPair{A: files[i].Path, B: files[j].Path, Similarity: sim})
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// projectID derives the stable projects-scope id from a project directory,
|
||||
// mirroring internal/memory.resolveProjectId: the first 12 hex chars of
|
||||
// sha256(absPath). The path is made absolute first so relative and absolute
|
||||
// forms of the same directory map to the same id.
|
||||
func projectID(projectDir string) string {
|
||||
abs, err := filepath.Abs(projectDir)
|
||||
if err != nil {
|
||||
abs = projectDir
|
||||
}
|
||||
sum := sha256.Sum256([]byte(abs))
|
||||
return hex.EncodeToString(sum[:])[:12]
|
||||
}
|
||||
|
||||
// extractLocalPathRefs returns the local filesystem path references found in
|
||||
// body. It splits on whitespace and common Markdown/code delimiters (so
|
||||
// `path`, [text](path) and bare tokens are all captured), then keeps only
|
||||
// tokens that look like a local path while rejecting URLs and other external
|
||||
// references.
|
||||
func extractLocalPathRefs(body string) []string {
|
||||
fields := strings.FieldsFunc(body, func(r rune) bool {
|
||||
switch r {
|
||||
case ' ', '\t', '\n', '\r', '`', '(', ')', '[', ']', '{', '}', '"', '\'', '<', '>', ',', ';', '|':
|
||||
return true
|
||||
}
|
||||
return false
|
||||
})
|
||||
var out []string
|
||||
for _, tok := range fields {
|
||||
// Trim trailing sentence punctuation that commonly abuts a path.
|
||||
tok = strings.TrimRight(tok, ".:!?")
|
||||
if isLocalPathRef(tok) {
|
||||
out = append(out, tok)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// isLocalPathRef reports whether tok looks like a local filesystem path rather
|
||||
// than a URL or other external reference. It accepts absolute ("/..."),
|
||||
// explicitly relative ("./", "../") and home-relative ("~/") tokens outright;
|
||||
// a bare relative token (no leading marker) must contain a separator AND a file
|
||||
// extension in its last segment, so ordinary prose like "TCP/IP", "read/write"
|
||||
// or "and/or" is not mistaken for a path. URL-schemed tokens are rejected.
|
||||
func isLocalPathRef(tok string) bool {
|
||||
if tok == "" {
|
||||
return false
|
||||
}
|
||||
if strings.Contains(tok, "://") {
|
||||
return false
|
||||
}
|
||||
lower := strings.ToLower(tok)
|
||||
for _, scheme := range []string{"http:", "https:", "ftp:", "ftps:", "file:", "mailto:", "ssh:", "git:", "www."} {
|
||||
if strings.HasPrefix(lower, scheme) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(tok, "/"),
|
||||
strings.HasPrefix(tok, "./"),
|
||||
strings.HasPrefix(tok, "../"),
|
||||
strings.HasPrefix(tok, "~/"):
|
||||
return true
|
||||
}
|
||||
// A bare relative token needs both a separator and an extension on its last
|
||||
// segment to be treated as a path (avoids flagging prose like "TCP/IP").
|
||||
if !strings.Contains(tok, "/") {
|
||||
return false
|
||||
}
|
||||
last := tok[strings.LastIndex(tok, "/")+1:]
|
||||
return strings.Contains(last, ".") && !strings.HasSuffix(last, ".")
|
||||
}
|
||||
|
||||
// resolveRef turns a reference into an absolute path for existence checking:
|
||||
// "~/" expands to the user home dir, absolute paths pass through, and relative
|
||||
// paths resolve against projectDir (or the current dir when projectDir is "").
|
||||
func resolveRef(ref, projectDir string) string {
|
||||
if strings.HasPrefix(ref, "~/") {
|
||||
if home, err := os.UserHomeDir(); err == nil {
|
||||
return filepath.Join(home, ref[2:])
|
||||
}
|
||||
}
|
||||
if filepath.IsAbs(ref) {
|
||||
return filepath.Clean(ref)
|
||||
}
|
||||
return filepath.Join(projectDir, ref)
|
||||
}
|
||||
|
||||
// tokenize lowercases body and splits it into the set of alphanumeric tokens
|
||||
// used for near-duplicate similarity. Punctuation and Markdown syntax are
|
||||
// discarded so formatting differences do not affect the score.
|
||||
func tokenize(body string) map[string]struct{} {
|
||||
set := make(map[string]struct{})
|
||||
for _, tok := range strings.FieldsFunc(strings.ToLower(body), func(r rune) bool {
|
||||
return !unicode.IsLetter(r) && !unicode.IsNumber(r)
|
||||
}) {
|
||||
set[tok] = struct{}{}
|
||||
}
|
||||
return set
|
||||
}
|
||||
|
||||
// jaccard returns |a∩b| / |a∪b|. Two empty sets are dissimilar (0) rather than
|
||||
// identical, so blank files are never paired as near-duplicates.
|
||||
func jaccard(a, b map[string]struct{}) float64 {
|
||||
if len(a) == 0 || len(b) == 0 {
|
||||
return 0
|
||||
}
|
||||
inter := 0
|
||||
for t := range a {
|
||||
if _, ok := b[t]; ok {
|
||||
inter++
|
||||
}
|
||||
}
|
||||
union := len(a) + len(b) - inter
|
||||
if union == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(inter) / float64(union)
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
package dream
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// writeMemFile writes body to <root>/<rel> creating parent dirs, returning the
|
||||
// absolute path.
|
||||
func writeMemFile(t *testing.T, root, rel, body string) string {
|
||||
t.Helper()
|
||||
p := filepath.Join(root, rel)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return filepath.Clean(p)
|
||||
}
|
||||
|
||||
func TestBuildPlanEnumeratesGlobalAndProjectExcludingSessionsAndCheckpoint(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
projectDir := t.TempDir()
|
||||
pid := projectID(projectDir)
|
||||
|
||||
gUser := writeMemFile(t, root, "global/user/prefs.md", "global user prefs\n")
|
||||
pProj := writeMemFile(t, root, filepath.Join("projects", pid, "project", "arch.md"), "project architecture notes\n")
|
||||
// Must be excluded:
|
||||
writeMemFile(t, root, "sessions/sess1/notes/x.md", "session scoped note\n")
|
||||
writeMemFile(t, root, "global/checkpoint/cp.md", "checkpoint transient\n")
|
||||
writeMemFile(t, root, filepath.Join("projects", pid, "checkpoint", "cp.md"), "project checkpoint\n")
|
||||
|
||||
plan, err := BuildPlan(root, projectDir)
|
||||
if err != nil {
|
||||
t.Fatalf("BuildPlan: %v", err)
|
||||
}
|
||||
|
||||
got := map[string]bool{}
|
||||
for _, f := range plan.Files {
|
||||
got[f.Path] = true
|
||||
}
|
||||
if !got[gUser] {
|
||||
t.Errorf("global user file not enumerated")
|
||||
}
|
||||
if !got[pProj] {
|
||||
t.Errorf("project file not enumerated")
|
||||
}
|
||||
if plan.FilesBefore != 2 {
|
||||
t.Errorf("FilesBefore = %d, want 2 (sessions + checkpoint excluded); files=%v", plan.FilesBefore, plan.Files)
|
||||
}
|
||||
if plan.BytesBefore == 0 {
|
||||
t.Errorf("BytesBefore = 0, want >0")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPlanEmptyRoot(t *testing.T) {
|
||||
plan, err := BuildPlan(filepath.Join(t.TempDir(), "does-not-exist"), "")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildPlan: %v", err)
|
||||
}
|
||||
if plan.FilesBefore != 0 || len(plan.Files) != 0 {
|
||||
t.Errorf("empty root should yield empty plan, got %+v", plan)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExactDedupGrouping(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
same := "identical memory body\nline two\n"
|
||||
a := writeMemFile(t, root, "global/user/a.md", same)
|
||||
b := writeMemFile(t, root, "global/reference/b.md", same)
|
||||
writeMemFile(t, root, "global/notes/c.md", "a totally different unique body\n")
|
||||
|
||||
plan, err := BuildPlan(root, "")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildPlan: %v", err)
|
||||
}
|
||||
if len(plan.DedupeGroups) != 1 {
|
||||
t.Fatalf("DedupeGroups = %d, want 1: %+v", len(plan.DedupeGroups), plan.DedupeGroups)
|
||||
}
|
||||
g := plan.DedupeGroups[0]
|
||||
if len(g.Paths) != 2 {
|
||||
t.Fatalf("group paths = %v, want [a b]", g.Paths)
|
||||
}
|
||||
want := map[string]bool{a: true, b: true}
|
||||
for _, p := range g.Paths {
|
||||
if !want[p] {
|
||||
t.Errorf("unexpected path in dedupe group: %q", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPathValidation(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
projectDir := t.TempDir()
|
||||
|
||||
// A real file the memory references (relative to projectDir).
|
||||
existingRel := "src/main.go"
|
||||
writeMemFile(t, projectDir, existingRel, "package main\n") // reuse helper; writes under projectDir
|
||||
|
||||
body := "See `src/main.go` for the entrypoint.\n" +
|
||||
"Old helper lived at `src/gone/removed.go` but was deleted.\n" +
|
||||
"Reference: https://example.com/docs and [site](https://pkg.go.dev/net/http).\n" +
|
||||
"Email me at mailto:dev@example.com.\n"
|
||||
writeMemFile(t, root, "global/project/notes.md", body)
|
||||
|
||||
plan, err := BuildPlan(root, projectDir)
|
||||
if err != nil {
|
||||
t.Fatalf("BuildPlan: %v", err)
|
||||
}
|
||||
|
||||
var flagged []string
|
||||
for _, r := range plan.InvalidPathRefs {
|
||||
flagged = append(flagged, r.Ref)
|
||||
}
|
||||
|
||||
// Missing local path must be flagged.
|
||||
if !containsRef(flagged, "src/gone/removed.go") {
|
||||
t.Errorf("missing path src/gone/removed.go not flagged; flagged=%v", flagged)
|
||||
}
|
||||
// Existing local path must NOT be flagged.
|
||||
if containsRef(flagged, "src/main.go") {
|
||||
t.Errorf("existing path src/main.go wrongly flagged; flagged=%v", flagged)
|
||||
}
|
||||
// URLs / external refs must NEVER be flagged.
|
||||
for _, r := range flagged {
|
||||
if wantsURLReject(r) {
|
||||
t.Errorf("external reference wrongly flagged as invalid local path: %q", r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func containsRef(refs []string, want string) bool {
|
||||
for _, r := range refs {
|
||||
if r == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func wantsURLReject(r string) bool {
|
||||
for _, bad := range []string{"http", "https", "mailto", "example.com", "pkg.go.dev"} {
|
||||
if strings.HasPrefix(r, bad) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestPathValidationIgnoresProseSlashes(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
projectDir := t.TempDir()
|
||||
// Prose tokens with slashes but no file extension must not be flagged as
|
||||
// missing local paths.
|
||||
body := "We support TCP/IP and read/write access; input/output is N/A here.\n"
|
||||
writeMemFile(t, root, "global/notes/prose.md", body)
|
||||
|
||||
plan, err := BuildPlan(root, projectDir)
|
||||
if err != nil {
|
||||
t.Fatalf("BuildPlan: %v", err)
|
||||
}
|
||||
if len(plan.InvalidPathRefs) != 0 {
|
||||
t.Errorf("prose slashes wrongly flagged as paths: %+v", plan.InvalidPathRefs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNearDupPairingThreshold(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
// Two highly-overlapping (but not identical) bodies -> should pair.
|
||||
writeMemFile(t, root, "global/user/a.md",
|
||||
"the quick brown fox jumps over the lazy dog near the river bank today\n")
|
||||
writeMemFile(t, root, "global/user/b.md",
|
||||
"the quick brown fox jumps over the lazy dog near the river bank tomorrow\n")
|
||||
// A dissimilar body -> should not pair with the others.
|
||||
writeMemFile(t, root, "global/notes/c.md",
|
||||
"completely unrelated content about database indexing and query planning\n")
|
||||
|
||||
plan, err := BuildPlan(root, "")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildPlan: %v", err)
|
||||
}
|
||||
if len(plan.NearDupPairs) != 1 {
|
||||
t.Fatalf("NearDupPairs = %d, want exactly 1: %+v", len(plan.NearDupPairs), plan.NearDupPairs)
|
||||
}
|
||||
p := plan.NearDupPairs[0]
|
||||
if p.Similarity < NearDupThreshold {
|
||||
t.Errorf("paired similarity %v below threshold %v", p.Similarity, NearDupThreshold)
|
||||
}
|
||||
// The dissimilar file must not appear in any pair.
|
||||
for _, pr := range plan.NearDupPairs {
|
||||
if filepath.Base(pr.A) == "c.md" || filepath.Base(pr.B) == "c.md" {
|
||||
t.Errorf("dissimilar file c.md wrongly paired: %+v", pr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNearDupSkipsExactDuplicates(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
same := "one two three four five six seven eight nine ten\n"
|
||||
writeMemFile(t, root, "global/user/a.md", same)
|
||||
writeMemFile(t, root, "global/user/b.md", same)
|
||||
|
||||
plan, err := BuildPlan(root, "")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildPlan: %v", err)
|
||||
}
|
||||
if len(plan.NearDupPairs) != 0 {
|
||||
t.Errorf("exact duplicates should be handled by dedupe, not near-dup pairs: %+v", plan.NearDupPairs)
|
||||
}
|
||||
if len(plan.DedupeGroups) != 1 {
|
||||
t.Errorf("DedupeGroups = %d, want 1", len(plan.DedupeGroups))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package dream
|
||||
|
||||
// This file holds the dream consolidation Agent's system prompt (SPEC §2.2) and
|
||||
// the strict JSON output schema the model must follow. The prompt scopes the
|
||||
// task to the LLM half of the mixed division of labor (SPEC §5.1.1): confirm
|
||||
// semantic merges of near-duplicate entries, and prune only clearly outdated or
|
||||
// contradicted entries — always conservatively (PRD FR-14: when uncertain,
|
||||
// KEEP). It never invents facts and only ever names paths that were provided in
|
||||
// the input, so the deterministic scope guard in the Runner cannot be tricked
|
||||
// into writing outside the memory store.
|
||||
|
||||
// dreamSystemPrompt is the fixed system instruction for the dream consolidation
|
||||
// pass. It is deliberately narrow: the Go side already handles exact dedup, path
|
||||
// validation, near-dup candidate pairing, MEMORY.md rewrite, and index rebuild;
|
||||
// the model only decides the semantic merges and prunes.
|
||||
const dreamSystemPrompt = `You are the memory-consolidation agent for pigo ("dream"). You run periodically over a developer's persistent memory library and produce a compact, non-redundant, current set of memory entries.
|
||||
|
||||
Each memory entry is a Markdown file. You are given the current entries (with their absolute file paths and bodies), plus deterministic hints: candidate near-duplicate pairs and references to local files that no longer exist. Exact byte-duplicates and dead-path cleanup are already handled mechanically — you do NOT need to act on those.
|
||||
|
||||
Your job, and ONLY your job:
|
||||
1. MERGE semantically-overlapping entries. When two or more entries cover the same fact/topic, combine them into a single entry that keeps the most recent and most informative content. Rewrite that surviving entry's full body to be self-contained and concise; the other entries are removed.
|
||||
2. PRUNE entries that are clearly outdated or directly contradicted by a newer entry.
|
||||
|
||||
Hard rules:
|
||||
- BE CONSERVATIVE. If you are unsure whether two entries truly overlap, do NOT merge them. If you are unsure whether an entry is outdated or contradicted, KEEP it. Losing a real memory is far worse than leaving a small redundancy.
|
||||
- NEVER invent facts. A merged body may only restate information already present in the entries you are combining. Do not add, infer, or embellish.
|
||||
- Only ever reference file paths that appear verbatim in the input. Never emit a path that was not given to you.
|
||||
- Never merge into, prune, or otherwise target a MEMORY.md index file. Those are indexes, not entries.
|
||||
- Preserve any Markdown frontmatter (the leading '---' block with name/description/metadata) on a surviving/merged entry, updating it only to reflect the merged content.
|
||||
- Do NOT create new entries. Distillation of new facts is handled by a separate step.
|
||||
|
||||
Output format:
|
||||
- Respond with a SINGLE JSON object and nothing else. No prose, no Markdown code fences.
|
||||
- Schema:
|
||||
{
|
||||
"merges": [
|
||||
{
|
||||
"keep": "<absolute path of the entry to keep and rewrite>",
|
||||
"body": "<the full rewritten body for the kept entry>",
|
||||
"remove": ["<absolute path merged away>", ...]
|
||||
}
|
||||
],
|
||||
"prunes": [
|
||||
{ "path": "<absolute path to remove>", "reason": "<why it is outdated or contradicted>" }
|
||||
],
|
||||
"notes": ["<short human-readable summary of a decision>", ...]
|
||||
}
|
||||
- Every "keep"/"remove"/"path" MUST be one of the input paths. "remove" must not contain the "keep" path.
|
||||
- If there is nothing to merge or prune, return {"merges": [], "prunes": [], "notes": []}. Returning an empty result is the correct, safe answer when in doubt.`
|
||||
|
||||
// dreamDistillSystemPrompt is the fixed system instruction for the JSONL
|
||||
// distillation pass (SPEC §5.3, PRD US-005 / FR-13). It is a SEPARATE model call
|
||||
// from the merge/prune pass above: its input is recent session transcripts plus
|
||||
// a list of memories that already exist, and its only job is to propose NEW
|
||||
// durable memory entries that are not already captured. The Go side then dedups
|
||||
// each proposal against the existing library and path-guards every write, so the
|
||||
// model only ever supplies type/scope/title/body — never a filesystem path.
|
||||
const dreamDistillSystemPrompt = `You are the memory-distillation agent for pigo ("dream"). You read recent session transcripts between a developer and an AI coding agent, and extract DURABLE facts worth remembering for future sessions.
|
||||
|
||||
You are also given a list of memories that ALREADY EXIST. Do NOT propose anything already covered by an existing memory — only genuinely new, not-yet-recorded facts.
|
||||
|
||||
What counts as a durable fact (extract these):
|
||||
- user: stable preferences, conventions, working style, environment the developer states ("I prefer X", "always run tests with Y", "my stack is Z").
|
||||
- feedback: corrections or standing instructions the developer gave the agent that should persist.
|
||||
- project: durable facts about the project's architecture, invariants, key decisions, or layout.
|
||||
- reference: stable pointers to important resources (a canonical doc, a command, an API) that will remain relevant.
|
||||
|
||||
What to IGNORE (never distill these):
|
||||
- One-shot task state, TODOs, "now do X" instructions, or anything tied to a single session's in-progress work.
|
||||
- Ephemeral context: transient errors already fixed, scratch reasoning, temporary file paths.
|
||||
- Anything you are not confident is durable. When unsure, SKIP it. Recording noise is worse than missing a fact.
|
||||
|
||||
Hard rules:
|
||||
- BE CONSERVATIVE and specific. Prefer zero entries over speculative ones. Only emit a fact you could justify keeping for months.
|
||||
- NEVER invent facts. Every entry must be grounded in the transcripts.
|
||||
- Each entry's body is a short, self-contained Markdown note (a sentence or a few bullet points). Do not include a filesystem path or a filename.
|
||||
- Classify each entry's "type" as exactly one of: user, feedback, project, reference.
|
||||
- Classify each entry's "scope" as "project" (specific to the current project) or "global" (applies across all the developer's work). When unsure, use "project".
|
||||
- Give each entry a short "title" (a few words) used only to name its file.
|
||||
|
||||
Output format:
|
||||
- Respond with a SINGLE JSON object and nothing else. No prose, no Markdown code fences.
|
||||
- Schema:
|
||||
{
|
||||
"entries": [
|
||||
{ "type": "user|feedback|project|reference", "scope": "project|global", "title": "<short title>", "body": "<self-contained markdown note>" }
|
||||
],
|
||||
"notes": ["<short human-readable summary of what was distilled>", ...]
|
||||
}
|
||||
- If there is nothing durable to add, return {"entries": [], "notes": []}. An empty result is the correct, safe answer when in doubt.`
|
||||
@@ -0,0 +1,105 @@
|
||||
package dream
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/smallnest/pigo/internal/memory"
|
||||
)
|
||||
|
||||
// searchHits reopens the memory store that Runner.Run built at
|
||||
// <root>/index.db and runs a full-text query, returning the set of hit paths.
|
||||
// The store is reconciled-on-open so the query reflects the exact on-disk state
|
||||
// left behind by the dream writeback (US-009 / FR-15). The score floor is
|
||||
// disabled so recall — not ranking — is what the assertion measures.
|
||||
func searchHits(t *testing.T, root, query string) map[string]bool {
|
||||
t.Helper()
|
||||
st, err := memory.Open(filepath.Join(root, "index.db"), root, "")
|
||||
if err != nil {
|
||||
t.Fatalf("reopen store: %v", err)
|
||||
}
|
||||
defer st.Close()
|
||||
res, err := st.Search(query, memory.SearchOptions{ScoreFloor: -1, Limit: 50, ReconcileFirst: true})
|
||||
if err != nil {
|
||||
t.Fatalf("Search(%q): %v", query, err)
|
||||
}
|
||||
hits := make(map[string]bool, len(res))
|
||||
for _, r := range res {
|
||||
hits[filepath.Clean(r.Path)] = true
|
||||
}
|
||||
return hits
|
||||
}
|
||||
|
||||
// TestReconcileConvergesAfterConsolidation is the US-009 / FR-15 acceptance
|
||||
// test: after a dream writeback that merges some entries and prunes others,
|
||||
// memory.Reconcile must rebuild the FTS index and memory_search must converge on
|
||||
// the compacted current state — never returning the merged-away or pruned
|
||||
// fragments, always returning the compacted entry.
|
||||
//
|
||||
// The scenario uses unique, made-up tokens per fragment so BM25 recall is
|
||||
// unambiguous: each token exists in exactly one fragment before the run, and the
|
||||
// assertions check that stale tokens vanish from the index while the compacted
|
||||
// token appears.
|
||||
func TestReconcileConvergesAfterConsolidation(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
|
||||
// Seed distinct (non-duplicate, path-ref-free) fragments so the deterministic
|
||||
// dedupe/path-clean passes are no-ops and the Consolidator drives the change.
|
||||
fragA := writeMemFile(t, root, "global/project/frag_a.md", "zorptholine legacy architecture fragment")
|
||||
fragB := writeMemFile(t, root, "global/project/frag_b.md", "wibblequux duplicate architecture note")
|
||||
fragC := writeMemFile(t, root, "global/project/frag_c.md", "frobnitz stale prunable outdated entry")
|
||||
keep := writeMemFile(t, root, "global/user/keep.md", "unrelated grocery shopping list")
|
||||
|
||||
// Stub Consolidator: rewrite frag_a into the compacted current state, merge
|
||||
// frag_b away into it (deletion), and prune the stale frag_c (deletion).
|
||||
stub := &stubConsolidator{result: ConsolidateResult{
|
||||
MergedBodies: map[string]string{
|
||||
fragA: "quombalter consolidated current architecture state",
|
||||
},
|
||||
Deletions: []string{fragB, fragC},
|
||||
Merged: 1,
|
||||
Pruned: 1,
|
||||
}}
|
||||
|
||||
r := &Runner{MemoryRoot: root, Consolidator: stub}
|
||||
rep, err := r.Run(context.Background(), RunOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if !stub.called {
|
||||
t.Fatal("Consolidator was not called")
|
||||
}
|
||||
|
||||
// (1) memory.Reconcile ran during writeback and indexed the surviving files.
|
||||
// The store's index.db is created fresh inside Run, so every kept file is a
|
||||
// new index row: Indexed must cover the compacted frag_a and the untouched
|
||||
// keep.md (>=2), proving the FTS index was rebuilt.
|
||||
if rep.Reconciled.Indexed < 2 {
|
||||
t.Fatalf("Reconciled.Indexed = %d, want >= 2 (rebuilt index over surviving files)", rep.Reconciled.Indexed)
|
||||
}
|
||||
if rep.Merged != 1 || rep.Pruned != 1 {
|
||||
t.Fatalf("counters not surfaced: Merged=%d Pruned=%d, want 1/1", rep.Merged, rep.Pruned)
|
||||
}
|
||||
|
||||
// (2) Stale fragments must be gone from the index: neither the merged-away
|
||||
// fragment (frag_b), the pruned fragment (frag_c), nor the overwritten body of
|
||||
// frag_a ("zorptholine") may still be searchable.
|
||||
for _, token := range []string{"zorptholine", "wibblequux", "frobnitz"} {
|
||||
if hits := searchHits(t, root, token); len(hits) != 0 {
|
||||
t.Fatalf("stale fragment token %q still searchable after consolidation: %v", token, hits)
|
||||
}
|
||||
}
|
||||
|
||||
// (3) The compacted current state IS searchable and resolves to the surviving
|
||||
// consolidated file (frag_a rewritten in place).
|
||||
hits := searchHits(t, root, "quombalter")
|
||||
if !hits[fragA] {
|
||||
t.Fatalf("compacted entry %q not returned by memory_search for its token, got %v", fragA, hits)
|
||||
}
|
||||
|
||||
// The unrelated memory must be untouched and still indexed.
|
||||
if hits := searchHits(t, root, "grocery"); !hits[keep] {
|
||||
t.Fatalf("untouched memory %q missing from index after consolidation, got %v", keep, hits)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package dream
|
||||
|
||||
// Report is the structured change summary produced by a /dream consolidation
|
||||
// run. It is persisted inside State.LastReport (state.go) and emitted as a
|
||||
// single line of JSON on the child process stdout (spec §4.2). Every count is a
|
||||
// deterministic tally the runner fills in after the plan (Go-deterministic) and
|
||||
// apply (LLM) phases; the zero value is a valid "nothing changed" report.
|
||||
//
|
||||
// The JSON tags match spec §3.2 exactly so the parent process (and any
|
||||
// scripted/headless caller) can decode the stdout contract without a shared Go
|
||||
// type.
|
||||
type Report struct {
|
||||
Merged int `json:"merged"` // entries merged away by the LLM apply step
|
||||
Deduped int `json:"deduped"` // exact (content-hash) duplicates removed
|
||||
PathsCleaned int `json:"paths_cleaned"` // stale local path references cleaned
|
||||
Pruned int `json:"pruned"` // stale/contradictory entries pruned
|
||||
Distilled int `json:"distilled"` // new memories distilled from session JSONL
|
||||
BytesBefore int64 `json:"bytes_before"`
|
||||
BytesAfter int64 `json:"bytes_after"`
|
||||
FilesBefore int `json:"files_before"`
|
||||
FilesAfter int `json:"files_after"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
Notes []string `json:"notes,omitempty"` // human-readable reasons (prune causes, etc.)
|
||||
Reconciled struct {
|
||||
Indexed int `json:"indexed"`
|
||||
Pruned int `json:"pruned"`
|
||||
} `json:"reconciled"`
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package dream
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestReportJSONTags(t *testing.T) {
|
||||
r := Report{
|
||||
Merged: 1,
|
||||
Deduped: 2,
|
||||
PathsCleaned: 3,
|
||||
Pruned: 4,
|
||||
Distilled: 5,
|
||||
BytesBefore: 100,
|
||||
BytesAfter: 80,
|
||||
FilesBefore: 10,
|
||||
FilesAfter: 9,
|
||||
DryRun: true,
|
||||
Notes: []string{"pruned stale entry"},
|
||||
}
|
||||
r.Reconciled.Indexed = 6
|
||||
r.Reconciled.Pruned = 7
|
||||
|
||||
data, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(data, &m); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
for _, key := range []string{
|
||||
"merged", "deduped", "paths_cleaned", "pruned", "distilled",
|
||||
"bytes_before", "bytes_after", "files_before", "files_after",
|
||||
"dry_run", "notes", "reconciled",
|
||||
} {
|
||||
if _, ok := m[key]; !ok {
|
||||
t.Errorf("missing JSON key %q in %s", key, data)
|
||||
}
|
||||
}
|
||||
rec, ok := m["reconciled"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("reconciled not an object: %v", m["reconciled"])
|
||||
}
|
||||
if _, ok := rec["indexed"]; !ok {
|
||||
t.Errorf("reconciled.indexed missing: %v", rec)
|
||||
}
|
||||
if _, ok := rec["pruned"]; !ok {
|
||||
t.Errorf("reconciled.pruned missing: %v", rec)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportZeroValueOmitsNotes(t *testing.T) {
|
||||
data, err := json.Marshal(Report{})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(data, &m); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
if _, ok := m["notes"]; ok {
|
||||
t.Errorf("empty notes should be omitted, got %s", data)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,527 @@
|
||||
package dream
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/smallnest/pigo/internal/memory"
|
||||
)
|
||||
|
||||
// Consolidator is the LLM-driven apply step, injected so the deterministic
|
||||
// Runner skeleton can run end-to-end without an LLM. It receives the plain-data
|
||||
// plan (dedupe groups + invalid-path refs + near-dup candidate pairs) and
|
||||
// returns the semantic decisions: which merged bodies to rewrite, which entries
|
||||
// to delete (merged-away / pruned), and which new entries to distill.
|
||||
//
|
||||
// #523 will implement a real Consolidator backed by the main-session model and
|
||||
// the dream system prompt (SPEC §5.1 step 5, §5.1.1). Until then the Runner uses
|
||||
// nopConsolidator, so the deterministic half (exact dedup + path clean +
|
||||
// Reconcile) is fully exercised and testable in isolation.
|
||||
type Consolidator interface {
|
||||
Consolidate(ctx context.Context, in ConsolidateInput) (ConsolidateResult, error)
|
||||
}
|
||||
|
||||
// ConsolidateInput is the plain-data view handed to the Consolidator. It carries
|
||||
// the deterministic Plan (which already embeds the dedupe groups, invalid-path
|
||||
// refs and near-dup pairs) plus the resolved roots so the implementation can
|
||||
// compute in-scope write targets. It intentionally carries no behavior and no
|
||||
// live handles (no *memory.Store, no *sql.DB) so it stays trivially serializable
|
||||
// if #523 chooses to marshal it across a subprocess/RPC boundary.
|
||||
type ConsolidateInput struct {
|
||||
Plan Plan `json:"plan"`
|
||||
MemoryRoot string `json:"memory_root"`
|
||||
ProjectDir string `json:"project_dir"`
|
||||
// Transcripts is the collected, budget-truncated text of the current
|
||||
// project's recent session JSONL, gathered deterministically by the Runner
|
||||
// (SPEC §5.3). It is the input to the separate distillation pass; an empty
|
||||
// string means there is nothing to distill (no matching sessions), so the
|
||||
// Consolidator skips the distill call entirely (SPEC §5.5 no-op).
|
||||
Transcripts string `json:"transcripts,omitempty"`
|
||||
}
|
||||
|
||||
// NewEntry is a distilled memory file the Consolidator wants created. Path must
|
||||
// resolve within the memory root's global/project scope; the Runner rejects any
|
||||
// out-of-scope target before writing (SPEC §5.2 / §7.1).
|
||||
type NewEntry struct {
|
||||
Path string `json:"path"`
|
||||
Body string `json:"body"`
|
||||
}
|
||||
|
||||
// ConsolidateResult is the Consolidator's decision set. All paths are absolute
|
||||
// and must lie within the memory root scope. Merged/Pruned/Distilled are the
|
||||
// report counters the Runner surfaces verbatim; the deterministic Deduped and
|
||||
// PathsCleaned counters are computed by the Runner itself, not here.
|
||||
type ConsolidateResult struct {
|
||||
// MergedBodies maps an existing memory file path to its rewritten (merged /
|
||||
// compacted) body. The Runner overwrites each file in place.
|
||||
MergedBodies map[string]string `json:"merged_bodies,omitempty"`
|
||||
// Deletions are memory files to remove (entries merged away or pruned as
|
||||
// stale/contradictory).
|
||||
Deletions []string `json:"deletions,omitempty"`
|
||||
// NewEntries are freshly distilled memory files to create.
|
||||
NewEntries []NewEntry `json:"new_entries,omitempty"`
|
||||
|
||||
Merged int `json:"merged"`
|
||||
Pruned int `json:"pruned"`
|
||||
Distilled int `json:"distilled"`
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
}
|
||||
|
||||
// nopConsolidator is the default no-op Consolidator used when none is injected.
|
||||
// It makes no decisions, so a Runner with it performs only the deterministic
|
||||
// dedup + path-clean + Reconcile pass — enough for the skeleton to run
|
||||
// end-to-end and be unit-tested without an LLM.
|
||||
type nopConsolidator struct{}
|
||||
|
||||
func (nopConsolidator) Consolidate(context.Context, ConsolidateInput) (ConsolidateResult, error) {
|
||||
return ConsolidateResult{}, nil
|
||||
}
|
||||
|
||||
// Runner is the subprocess-side consolidation entry point (SPEC §2.2 / §5.1). It
|
||||
// runs the deterministic plan, delegates semantic merge/prune/distill to an
|
||||
// injected Consolidator, applies the results within the memory-root scope, and
|
||||
// rebuilds the FTS index via memory.Reconcile. The zero value is usable: an
|
||||
// empty MemoryRoot is resolved from the environment and a nil Consolidator falls
|
||||
// back to nopConsolidator.
|
||||
type Runner struct {
|
||||
// Consolidator is the injected LLM apply step; nil selects nopConsolidator.
|
||||
Consolidator Consolidator
|
||||
// MemoryRoot overrides the environment-resolved memory root. Empty means
|
||||
// resolve via ResolveMemoryRoot (PIGO_HOME / ~/.pigo). Tests set it to a temp
|
||||
// dir; production leaves it empty.
|
||||
MemoryRoot string
|
||||
// Sessions is the source of recent session transcripts for the distillation
|
||||
// pass (SPEC §5.3). nil resolves the default store at $PIGO_HOME/sessions (or
|
||||
// ~/.pigo/sessions); tests inject a stub. If it cannot be resolved,
|
||||
// distillation degrades to a no-op rather than failing the run.
|
||||
Sessions SessionSource
|
||||
}
|
||||
|
||||
// RunOptions are the per-invocation parameters mirroring the CLI flags. ProjectDir
|
||||
// selects the projects sub-scope (empty → global-only); DryRun analyzes without
|
||||
// writing files or updating state (but still takes the lock — SPEC §5.5).
|
||||
type RunOptions struct {
|
||||
DryRun bool
|
||||
ProjectDir string
|
||||
// RecentSessions is the first-run distillation window: when dream has never
|
||||
// run, the most-recent RecentSessions project sessions are distilled (SPEC
|
||||
// §5.3). Non-positive falls back to DefaultRecentSessions (20).
|
||||
RecentSessions int
|
||||
}
|
||||
|
||||
// Run executes one consolidation pass and returns the change Report. The flow is
|
||||
// the deterministic algorithm of SPEC §5.1:
|
||||
//
|
||||
// open store → resolve memoryRoot → acquire lock (ErrLocked → skipped, no error)
|
||||
// → BuildPlan → Consolidate → if !DryRun: apply dedup + path-clean + consolidation
|
||||
// → Reconcile → recount → SaveState(ok); if DryRun: counts only, no writes/state.
|
||||
//
|
||||
// A held lock is not a failure: Run returns a zero-count Report and a nil error
|
||||
// so the caller exits 0 ("skipped"). Genuine errors (I/O, plan, apply) are
|
||||
// returned for the caller to map to exit 1 / status "failed".
|
||||
func (r *Runner) Run(ctx context.Context, opts RunOptions) (Report, error) {
|
||||
memoryRoot := r.MemoryRoot
|
||||
if memoryRoot == "" {
|
||||
memoryRoot = ResolveMemoryRoot()
|
||||
}
|
||||
if memoryRoot == "" {
|
||||
return Report{}, fmt.Errorf("dream: cannot resolve memory root")
|
||||
}
|
||||
|
||||
lock, err := AcquireLock(memoryRoot)
|
||||
if err != nil {
|
||||
if isLocked(err) {
|
||||
// Another dream is running: skip silently. Zero-count report, no error
|
||||
// → caller exits 0, leaves last_status unchanged (SPEC §5.5 / §6.1). We
|
||||
// have opened nothing and created no files, honoring the skip contract.
|
||||
return Report{DryRun: opts.DryRun}, nil
|
||||
}
|
||||
return Report{}, fmt.Errorf("dream: acquire lock: %w", err)
|
||||
}
|
||||
defer lock.Release()
|
||||
|
||||
plan, err := BuildPlan(memoryRoot, opts.ProjectDir)
|
||||
if err != nil {
|
||||
return Report{}, fmt.Errorf("dream: build plan: %w", err)
|
||||
}
|
||||
|
||||
rep := Report{
|
||||
DryRun: opts.DryRun,
|
||||
BytesBefore: plan.BytesBefore,
|
||||
FilesBefore: plan.FilesBefore,
|
||||
}
|
||||
|
||||
cons := r.Consolidator
|
||||
if cons == nil {
|
||||
cons = nopConsolidator{}
|
||||
}
|
||||
|
||||
// Gather the current project's recent session transcripts for the distill
|
||||
// pass (SPEC §5.3). This is deterministic Runner work, mirroring BuildPlan:
|
||||
// the Consolidator runs the semantic distill call over these transcripts. A
|
||||
// nil/unresolvable source or no matching session yields "" → the Consolidator
|
||||
// skips distillation and Distilled stays 0 (SPEC §5.5 no-op).
|
||||
src := r.Sessions
|
||||
if src == nil {
|
||||
if resolved, rerr := resolveSessionStore(); rerr == nil {
|
||||
src = resolved
|
||||
}
|
||||
}
|
||||
state, _ := LoadState(memoryRoot)
|
||||
transcripts := collectTranscripts(src, state, opts.ProjectDir, opts.RecentSessions, defaultTranscriptBudget)
|
||||
|
||||
cres, err := cons.Consolidate(ctx, ConsolidateInput{
|
||||
Plan: plan,
|
||||
MemoryRoot: memoryRoot,
|
||||
ProjectDir: opts.ProjectDir,
|
||||
Transcripts: transcripts,
|
||||
})
|
||||
if err != nil {
|
||||
// The runner surfaces the error; the parent/scheduler (node #8) maps a
|
||||
// non-zero exit to state.LastStatus="failed" (SPEC §4.2/§6.1). The runner
|
||||
// itself only ever persists "ok", so it never opens/creates the store on a
|
||||
// failing or dry-run path.
|
||||
return Report{}, fmt.Errorf("dream: consolidate: %w", err)
|
||||
}
|
||||
|
||||
// Surface the Consolidator's semantic counters verbatim (SPEC §5.1.1: merge /
|
||||
// prune / distill are LLM decisions).
|
||||
rep.Merged = cres.Merged
|
||||
rep.Pruned = cres.Pruned
|
||||
rep.Distilled = cres.Distilled
|
||||
rep.Notes = append(rep.Notes, cres.Notes...)
|
||||
|
||||
// Distillation no-op: no durable facts were added (no matching sessions or
|
||||
// nothing worth keeping). Record the "无新增" note so the report reflects the
|
||||
// step ran with no additions (SPEC §5.5, PRD FR-13).
|
||||
if rep.Distilled == 0 {
|
||||
rep.Notes = append(rep.Notes, "distill: 无新增")
|
||||
}
|
||||
|
||||
if opts.DryRun {
|
||||
// Predict the deterministic counters without touching disk or state, and
|
||||
// without opening the store (which would create index.db). The after-sizes
|
||||
// stay zero: nothing was written, so there is no post-state to measure
|
||||
// (SPEC §5.5 dry-run row).
|
||||
rep.Deduped = plannedDedupeCount(plan)
|
||||
rep.PathsCleaned = plannedPathCleanCount(plan)
|
||||
return rep, nil
|
||||
}
|
||||
|
||||
// --- write path (non dry-run) -------------------------------------------
|
||||
//
|
||||
// Open the store only here: a dry-run or a lock skip must not create index.db
|
||||
// (SPEC §5.5). The store is needed solely for the post-write Reconcile.
|
||||
store, err := memory.Open(filepath.Join(memoryRoot, "index.db"), memoryRoot, "")
|
||||
if err != nil {
|
||||
return Report{}, fmt.Errorf("dream: open memory store: %w", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
deleted, deduped, err := applyDedupe(memoryRoot, opts.ProjectDir, plan)
|
||||
if err != nil {
|
||||
return Report{}, fmt.Errorf("dream: apply dedupe: %w", err)
|
||||
}
|
||||
rep.Deduped = deduped
|
||||
|
||||
cleaned, err := applyPathClean(memoryRoot, opts.ProjectDir, plan, deleted)
|
||||
if err != nil {
|
||||
return Report{}, fmt.Errorf("dream: apply path-clean: %w", err)
|
||||
}
|
||||
rep.PathsCleaned = cleaned
|
||||
|
||||
if err := applyConsolidation(memoryRoot, opts.ProjectDir, cres, deleted); err != nil {
|
||||
return Report{}, fmt.Errorf("dream: apply consolidation: %w", err)
|
||||
}
|
||||
|
||||
// Keep each affected scope's MEMORY.md index consistent with the entries on
|
||||
// disk: drop any link to a file removed by dedupe or by the Consolidator so
|
||||
// no dangling references survive (PRD US-003). The full removed set is the
|
||||
// deterministic dedupe deletions plus the Consolidator's own deletions
|
||||
// (merged-away + pruned entries).
|
||||
for _, p := range cres.Deletions {
|
||||
deleted[filepath.Clean(p)] = struct{}{}
|
||||
}
|
||||
if err := updateScopeIndexes(memoryRoot, opts.ProjectDir, deleted); err != nil {
|
||||
return Report{}, fmt.Errorf("dream: update scope index: %w", err)
|
||||
}
|
||||
|
||||
res, err := store.Reconcile()
|
||||
if err != nil {
|
||||
return Report{}, fmt.Errorf("dream: reconcile: %w", err)
|
||||
}
|
||||
rep.Reconciled.Indexed = res.Indexed
|
||||
rep.Reconciled.Pruned = res.Pruned
|
||||
|
||||
// Recompute post-state sizes by re-enumerating the same scopes.
|
||||
after, err := BuildPlan(memoryRoot, opts.ProjectDir)
|
||||
if err != nil {
|
||||
return Report{}, fmt.Errorf("dream: recount: %w", err)
|
||||
}
|
||||
rep.BytesAfter = after.BytesBefore
|
||||
rep.FilesAfter = after.FilesBefore
|
||||
|
||||
repCopy := rep
|
||||
if err := SaveState(memoryRoot, State{
|
||||
LastRunAt: time.Now().UTC(),
|
||||
LastStatus: "ok",
|
||||
LastReport: &repCopy,
|
||||
}); err != nil {
|
||||
return Report{}, fmt.Errorf("dream: save state: %w", err)
|
||||
}
|
||||
|
||||
return rep, nil
|
||||
}
|
||||
|
||||
// isLocked reports whether err is the ErrLocked contention signal. Kept as a
|
||||
// helper so the skipped-vs-failed branch reads clearly.
|
||||
func isLocked(err error) bool {
|
||||
return errors.Is(err, ErrLocked)
|
||||
}
|
||||
|
||||
// ResolveMemoryRoot returns the persistent memory root directory the same way
|
||||
// the CLI does (internal/cli/run.MemoryDir): $PIGO_HOME/memory, else
|
||||
// ~/.pigo/memory. It is duplicated here rather than imported to keep the dream
|
||||
// package free of a dependency on the CLI assembly layer (and any import cycle
|
||||
// through it). Returns "" when neither PIGO_HOME nor the home dir is resolvable.
|
||||
func ResolveMemoryRoot() string {
|
||||
dir := os.Getenv("PIGO_HOME")
|
||||
if dir == "" {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
dir = filepath.Join(home, ".pigo")
|
||||
}
|
||||
return filepath.Join(dir, "memory")
|
||||
}
|
||||
|
||||
// withinScope reports whether target is a write target permitted by this run's
|
||||
// consolidation scope: it must live under <memoryRoot>/global or, when a
|
||||
// projectDir is set, under that project's own <memoryRoot>/projects/<id>
|
||||
// directory. Anything else — the sessions scope, an UNRELATED project's
|
||||
// directory, or a path outside memoryRoot (e.g. user source) — is rejected. This
|
||||
// is the SPEC §5.2 / §7.1 path-boundary guard that keeps an LLM-produced path
|
||||
// from escaping the memory store or clobbering other projects' memories, and it
|
||||
// mirrors BuildPlan, which only enumerates global + the active project.
|
||||
//
|
||||
// The check is symlink-aware: both the scope base and the target's longest
|
||||
// existing ancestor are passed through filepath.EvalSymlinks before comparison,
|
||||
// so a symlink planted inside an allowed scope cannot redirect a write/delete
|
||||
// outside the allowed directory. A prefix match is done on path boundaries (not
|
||||
// raw string prefixes) so "<root>/globalX" does not pass as "<root>/global".
|
||||
func withinScope(memoryRoot, projectDir, target string) bool {
|
||||
if memoryRoot == "" || target == "" {
|
||||
return false
|
||||
}
|
||||
absRoot, err := filepath.Abs(memoryRoot)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
absTarget, err := filepath.Abs(target)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
resolvedTarget := resolveExisting(filepath.Clean(absTarget))
|
||||
|
||||
bases := []string{filepath.Join(absRoot, "global")}
|
||||
if projectDir != "" {
|
||||
bases = append(bases, filepath.Join(absRoot, "projects", projectID(projectDir)))
|
||||
}
|
||||
for _, base := range bases {
|
||||
base = resolveExisting(base)
|
||||
if resolvedTarget == base {
|
||||
return true
|
||||
}
|
||||
if strings.HasPrefix(resolvedTarget, base+string(os.PathSeparator)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// resolveExisting returns path with symlinks resolved as far as the filesystem
|
||||
// allows: it walks up to the longest existing ancestor, resolves that with
|
||||
// filepath.EvalSymlinks, then rejoins the not-yet-existing tail. This lets the
|
||||
// scope guard defeat symlink redirection (an existing symlink component is
|
||||
// dereferenced to its real location) while still handling brand-new target
|
||||
// paths whose leaf files do not exist yet. On any resolution error it falls back
|
||||
// to the cleaned input so the guard fails closed via the lexical comparison.
|
||||
func resolveExisting(path string) string {
|
||||
path = filepath.Clean(path)
|
||||
tail := ""
|
||||
cur := path
|
||||
for {
|
||||
if resolved, err := filepath.EvalSymlinks(cur); err == nil {
|
||||
if tail == "" {
|
||||
return resolved
|
||||
}
|
||||
return filepath.Join(resolved, tail)
|
||||
}
|
||||
parent := filepath.Dir(cur)
|
||||
if parent == cur {
|
||||
// Reached the root without finding an existing component.
|
||||
return path
|
||||
}
|
||||
tail = filepath.Join(filepath.Base(cur), tail)
|
||||
cur = parent
|
||||
}
|
||||
}
|
||||
|
||||
// plannedDedupeCount is the number of files a dedupe pass would remove: one per
|
||||
// duplicate beyond the representative in each group (SPEC report.Deduped).
|
||||
func plannedDedupeCount(plan Plan) int {
|
||||
n := 0
|
||||
for _, g := range plan.DedupeGroups {
|
||||
if len(g.Paths) > 1 {
|
||||
n += len(g.Paths) - 1
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// plannedPathCleanCount is the number of distinct invalid local path references
|
||||
// a path-clean pass would strip (SPEC report.PathsCleaned).
|
||||
func plannedPathCleanCount(plan Plan) int {
|
||||
return len(plan.InvalidPathRefs)
|
||||
}
|
||||
|
||||
// applyDedupe removes exact-duplicate memory files, keeping the first path in
|
||||
// each group (paths are pre-sorted by BuildPlan) and deleting the rest. Every
|
||||
// deletion target is guarded by withinScope. It returns the set of deleted paths
|
||||
// (so later passes skip them) and the Deduped count.
|
||||
func applyDedupe(memoryRoot, projectDir string, plan Plan) (map[string]struct{}, int, error) {
|
||||
deleted := make(map[string]struct{})
|
||||
count := 0
|
||||
for _, g := range plan.DedupeGroups {
|
||||
if len(g.Paths) < 2 {
|
||||
continue
|
||||
}
|
||||
// Keep g.Paths[0] as the representative; remove the duplicates.
|
||||
for _, p := range g.Paths[1:] {
|
||||
if !withinScope(memoryRoot, projectDir, p) {
|
||||
return nil, 0, fmt.Errorf("refusing out-of-scope dedupe target %q", p)
|
||||
}
|
||||
if err := os.Remove(p); err != nil && !os.IsNotExist(err) {
|
||||
return nil, 0, err
|
||||
}
|
||||
deleted[filepath.Clean(p)] = struct{}{}
|
||||
count++
|
||||
}
|
||||
}
|
||||
return deleted, count, nil
|
||||
}
|
||||
|
||||
// applyPathClean strips invalid local path references from the bodies of the
|
||||
// files that still exist (skipping any removed by dedupe). Each distinct
|
||||
// (file, ref) is removed by deleting the exact reference substring and is
|
||||
// counted once. This is the deterministic half of FR-11; the Consolidator may
|
||||
// later decide whole-entry pruning for refs that leave an entry meaningless.
|
||||
// Every rewrite target is guarded by withinScope.
|
||||
func applyPathClean(memoryRoot, projectDir string, plan Plan, deleted map[string]struct{}) (int, error) {
|
||||
// Group refs by file so each file is read/written at most once.
|
||||
byFile := make(map[string][]string)
|
||||
for _, r := range plan.InvalidPathRefs {
|
||||
clean := filepath.Clean(r.File)
|
||||
if _, gone := deleted[clean]; gone {
|
||||
continue
|
||||
}
|
||||
byFile[clean] = append(byFile[clean], r.Ref)
|
||||
}
|
||||
|
||||
count := 0
|
||||
for file, refs := range byFile {
|
||||
if !withinScope(memoryRoot, projectDir, file) {
|
||||
return 0, fmt.Errorf("refusing out-of-scope path-clean target %q", file)
|
||||
}
|
||||
raw, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
body := string(raw)
|
||||
for _, ref := range refs {
|
||||
if strings.Contains(body, ref) {
|
||||
body = strings.ReplaceAll(body, ref, "")
|
||||
count++
|
||||
}
|
||||
}
|
||||
if body != string(raw) {
|
||||
if err := atomicWrite(file, []byte(body)); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// applyConsolidation writes the Consolidator's decisions: rewritten merged
|
||||
// bodies, new distilled entries, and deletions. Every write/delete target is
|
||||
// guarded by withinScope so an LLM cannot escape the memory store (SPEC §5.2 /
|
||||
// §7.1). Deletions already performed by dedupe are skipped.
|
||||
func applyConsolidation(memoryRoot, projectDir string, cres ConsolidateResult, deleted map[string]struct{}) error {
|
||||
for path, body := range cres.MergedBodies {
|
||||
if !withinScope(memoryRoot, projectDir, path) {
|
||||
return fmt.Errorf("refusing out-of-scope merged-body target %q", path)
|
||||
}
|
||||
if err := atomicWrite(path, []byte(body)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, e := range cres.NewEntries {
|
||||
if !withinScope(memoryRoot, projectDir, e.Path) {
|
||||
return fmt.Errorf("refusing out-of-scope new-entry target %q", e.Path)
|
||||
}
|
||||
if err := atomicWrite(e.Path, []byte(e.Body)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, path := range cres.Deletions {
|
||||
if _, gone := deleted[filepath.Clean(path)]; gone {
|
||||
continue
|
||||
}
|
||||
if !withinScope(memoryRoot, projectDir, path) {
|
||||
return fmt.Errorf("refusing out-of-scope deletion target %q", path)
|
||||
}
|
||||
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// atomicWrite writes data to path via a temp file + rename in the same directory
|
||||
// so a crash mid-write cannot leave a truncated memory file (SPEC §6.3). The
|
||||
// parent directory is created lazily.
|
||||
func atomicWrite(path string, data []byte) error {
|
||||
dir := filepath.Dir(path)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
tmp, err := os.CreateTemp(dir, ".dream-*.tmp")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmpName := tmp.Name()
|
||||
if _, err := tmp.Write(data); err != nil {
|
||||
tmp.Close()
|
||||
os.Remove(tmpName)
|
||||
return err
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
os.Remove(tmpName)
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tmpName, path); err != nil {
|
||||
os.Remove(tmpName)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
package dream
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// stubConsolidator returns a fixed result and records that it was called.
|
||||
type stubConsolidator struct {
|
||||
result ConsolidateResult
|
||||
called bool
|
||||
}
|
||||
|
||||
func (s *stubConsolidator) Consolidate(context.Context, ConsolidateInput) (ConsolidateResult, error) {
|
||||
s.called = true
|
||||
return s.result, nil
|
||||
}
|
||||
|
||||
// TestRunEmptyMemoryDir: an empty memory dir yields an all-zero Report with
|
||||
// status ok (no error). Reconcile tolerates the missing scopes.
|
||||
func TestRunEmptyMemoryDir(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
r := &Runner{MemoryRoot: root}
|
||||
rep, err := r.Run(context.Background(), RunOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if rep.FilesBefore != 0 || rep.BytesBefore != 0 || rep.FilesAfter != 0 || rep.BytesAfter != 0 {
|
||||
t.Fatalf("expected all-zero report, got %+v", rep)
|
||||
}
|
||||
if rep.Deduped != 0 || rep.Merged != 0 || rep.Pruned != 0 || rep.PathsCleaned != 0 {
|
||||
t.Fatalf("expected zero counters, got %+v", rep)
|
||||
}
|
||||
// State should be written with status ok for a non-dry-run.
|
||||
st, err := LoadState(root)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadState: %v", err)
|
||||
}
|
||||
if st.LastStatus != "ok" {
|
||||
t.Fatalf("LastStatus = %q, want ok", st.LastStatus)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunDryRunWritesNothing: dry-run computes counts, writes no files, does not
|
||||
// update state, but still acquires + releases the lock.
|
||||
func TestRunDryRunWritesNothing(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
// Two byte-identical files → one dedupe candidate.
|
||||
a := writeMemFile(t, root, "global/user/a.md", "same content")
|
||||
b := writeMemFile(t, root, "global/user/b.md", "same content")
|
||||
|
||||
r := &Runner{MemoryRoot: root}
|
||||
rep, err := r.Run(context.Background(), RunOptions{DryRun: true})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if !rep.DryRun {
|
||||
t.Fatal("Report.DryRun = false, want true")
|
||||
}
|
||||
if rep.Deduped != 1 {
|
||||
t.Fatalf("Deduped = %d, want 1 (predicted)", rep.Deduped)
|
||||
}
|
||||
// Both files must still exist — dry-run writes nothing.
|
||||
if _, err := os.Stat(a); err != nil {
|
||||
t.Fatalf("file a removed in dry-run: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(b); err != nil {
|
||||
t.Fatalf("file b removed in dry-run: %v", err)
|
||||
}
|
||||
// State must NOT be updated (never-run remains).
|
||||
st, err := LoadState(root)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadState: %v", err)
|
||||
}
|
||||
if !st.LastRunAt.IsZero() || st.LastStatus != "" {
|
||||
t.Fatalf("dry-run updated state: %+v", st)
|
||||
}
|
||||
// Lock must have been released (a fresh acquire succeeds).
|
||||
lk, err := AcquireLock(root)
|
||||
if err != nil {
|
||||
t.Fatalf("lock not released after dry-run: %v", err)
|
||||
}
|
||||
lk.Release()
|
||||
}
|
||||
|
||||
// TestRunLockedSkips: when a live lock is already held, Run returns a zero-count
|
||||
// report and NO error (exit-0 "skipped" semantics), and does not touch state.
|
||||
func TestRunLockedSkips(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeMemFile(t, root, "global/user/a.md", "content")
|
||||
|
||||
held, err := AcquireLock(root)
|
||||
if err != nil {
|
||||
t.Fatalf("pre-acquire lock: %v", err)
|
||||
}
|
||||
defer held.Release()
|
||||
|
||||
r := &Runner{MemoryRoot: root}
|
||||
rep, err := r.Run(context.Background(), RunOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Run under held lock should not error, got %v", err)
|
||||
}
|
||||
if rep.FilesBefore != 0 || rep.Deduped != 0 {
|
||||
t.Fatalf("skipped run should have zero report, got %+v", rep)
|
||||
}
|
||||
// State untouched (never ran).
|
||||
st, _ := LoadState(root)
|
||||
if !st.LastRunAt.IsZero() || st.LastStatus != "" {
|
||||
t.Fatalf("skipped run touched state: %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunAppliesDedupe: a non-dry-run removes exact duplicates, updates state,
|
||||
// and reflects counts in the Report.
|
||||
func TestRunAppliesDedupe(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
a := writeMemFile(t, root, "global/user/a.md", "same content")
|
||||
b := writeMemFile(t, root, "global/user/b.md", "same content")
|
||||
|
||||
r := &Runner{MemoryRoot: root}
|
||||
rep, err := r.Run(context.Background(), RunOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if rep.Deduped != 1 {
|
||||
t.Fatalf("Deduped = %d, want 1", rep.Deduped)
|
||||
}
|
||||
// Exactly one of the two duplicates must survive (the sorted-first: a.md).
|
||||
_, errA := os.Stat(a)
|
||||
_, errB := os.Stat(b)
|
||||
if errA != nil {
|
||||
t.Fatalf("representative a.md removed: %v", errA)
|
||||
}
|
||||
if errB == nil {
|
||||
t.Fatal("duplicate b.md should have been removed")
|
||||
}
|
||||
if rep.FilesAfter != 1 {
|
||||
t.Fatalf("FilesAfter = %d, want 1", rep.FilesAfter)
|
||||
}
|
||||
st, _ := LoadState(root)
|
||||
if st.LastStatus != "ok" || st.LastReport == nil {
|
||||
t.Fatalf("state not updated: %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWithinScope: the path-boundary guard accepts in-scope targets and rejects
|
||||
// everything outside <memoryRoot>/global and the active project's directory.
|
||||
func TestWithinScope(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
projectDir := t.TempDir()
|
||||
pid := projectID(projectDir)
|
||||
otherPID := "0123456789ab" // a different, unrelated project id
|
||||
cases := []struct {
|
||||
name string
|
||||
project string
|
||||
target string
|
||||
want bool
|
||||
}{
|
||||
{"global file", projectDir, filepath.Join(root, "global", "user", "x.md"), true},
|
||||
{"active project file", projectDir, filepath.Join(root, "projects", pid, "notes", "y.md"), true},
|
||||
{"global root itself", projectDir, filepath.Join(root, "global"), true},
|
||||
{"unrelated project rejected", projectDir, filepath.Join(root, "projects", otherPID, "z.md"), false},
|
||||
{"any project rejected when global-only", "", filepath.Join(root, "projects", pid, "y.md"), false},
|
||||
{"global still ok when global-only", "", filepath.Join(root, "global", "x.md"), true},
|
||||
{"sessions scope rejected", projectDir, filepath.Join(root, "sessions", "s1", "checkpoint.md"), false},
|
||||
{"outside root rejected", projectDir, filepath.Join(root, "..", "evil.md"), false},
|
||||
{"sibling prefix not confused", projectDir, filepath.Join(root, "globalX", "z.md"), false},
|
||||
{"absolute escape rejected", projectDir, "/etc/passwd", false},
|
||||
{"empty target rejected", projectDir, "", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := withinScope(root, tc.project, tc.target); got != tc.want {
|
||||
t.Fatalf("withinScope(%q, %q, %q) = %v, want %v", root, tc.project, tc.target, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
if withinScope("", projectDir, filepath.Join(root, "global", "x.md")) {
|
||||
t.Fatal("empty memoryRoot must reject")
|
||||
}
|
||||
}
|
||||
|
||||
// TestWithinScopeSymlinkEscape: a symlink planted inside an allowed scope must
|
||||
// not let a target escape memoryRoot. The guard resolves symlinks on existing
|
||||
// ancestors before the containment check (SPEC §7.1 defense-in-depth).
|
||||
func TestWithinScopeSymlinkEscape(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir() // a directory fully outside memoryRoot
|
||||
|
||||
globalDir := filepath.Join(root, "global")
|
||||
if err := os.MkdirAll(globalDir, 0o755); err != nil {
|
||||
t.Fatalf("mkdir global: %v", err)
|
||||
}
|
||||
// <root>/global/escape -> <outside>
|
||||
link := filepath.Join(globalDir, "escape")
|
||||
if err := os.Symlink(outside, link); err != nil {
|
||||
t.Skipf("symlink unsupported: %v", err)
|
||||
}
|
||||
|
||||
// Lexically this looks in-scope (<root>/global/escape/evil.md) but resolves
|
||||
// to <outside>/evil.md, which must be rejected.
|
||||
target := filepath.Join(link, "evil.md")
|
||||
if withinScope(root, "", target) {
|
||||
t.Fatalf("symlink escape target accepted: %q", target)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunPathClean: a memory file referencing a non-existent local path has that
|
||||
// reference stripped and counted.
|
||||
func TestRunPathClean(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
proj := t.TempDir()
|
||||
missing := filepath.Join(proj, "does", "not", "exist.go")
|
||||
body := "See `" + missing + "` for details."
|
||||
f := writeMemFile(t, root, "global/reference/r.md", body)
|
||||
|
||||
r := &Runner{MemoryRoot: root}
|
||||
rep, err := r.Run(context.Background(), RunOptions{ProjectDir: proj})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if rep.PathsCleaned != 1 {
|
||||
t.Fatalf("PathsCleaned = %d, want 1", rep.PathsCleaned)
|
||||
}
|
||||
raw, err := os.ReadFile(f)
|
||||
if err != nil {
|
||||
t.Fatalf("read cleaned file: %v", err)
|
||||
}
|
||||
if got := string(raw); got == body {
|
||||
t.Fatalf("body unchanged after path-clean: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunConsolidatorApplied: an injected Consolidator's new-entry write and
|
||||
// counters flow through, and the write lands within scope.
|
||||
func TestRunConsolidatorApplied(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeMemFile(t, root, "global/user/a.md", "hello")
|
||||
|
||||
newPath := filepath.Join(root, "global", "user", "distilled.md")
|
||||
stub := &stubConsolidator{result: ConsolidateResult{
|
||||
NewEntries: []NewEntry{{Path: newPath, Body: "distilled fact"}},
|
||||
Distilled: 1,
|
||||
Merged: 2,
|
||||
Pruned: 3,
|
||||
Notes: []string{"note"},
|
||||
}}
|
||||
r := &Runner{MemoryRoot: root, Consolidator: stub}
|
||||
rep, err := r.Run(context.Background(), RunOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if !stub.called {
|
||||
t.Fatal("Consolidator was not called")
|
||||
}
|
||||
if rep.Distilled != 1 || rep.Merged != 2 || rep.Pruned != 3 {
|
||||
t.Fatalf("counters not surfaced: %+v", rep)
|
||||
}
|
||||
if _, err := os.Stat(newPath); err != nil {
|
||||
t.Fatalf("new entry not written: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestApplyConsolidationRejectsOutOfScope: a Consolidator that tries to write
|
||||
// outside the memory root is rejected by the guard.
|
||||
func TestApplyConsolidationRejectsOutOfScope(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := filepath.Join(t.TempDir(), "escape.md")
|
||||
cres := ConsolidateResult{
|
||||
NewEntries: []NewEntry{{Path: outside, Body: "x"}},
|
||||
}
|
||||
if err := applyConsolidation(root, "", cres, nil); err == nil {
|
||||
t.Fatal("expected out-of-scope write to be rejected")
|
||||
}
|
||||
if _, err := os.Stat(outside); err == nil {
|
||||
t.Fatal("out-of-scope file must not be created")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunDryRunLeavesStaleLockTakeable is a small guard that dry-run's lock is
|
||||
// released promptly (no leftover live lock).
|
||||
func TestRunDryRunLockReleased(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
r := &Runner{MemoryRoot: root}
|
||||
if _, err := r.Run(context.Background(), RunOptions{DryRun: true}); err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
// Fresh acquire must succeed immediately (lock released, not merely stale).
|
||||
// Force a long stale window so a leftover *live* lock would block acquisition,
|
||||
// proving Run released it rather than leaving it to be reclaimed as stale.
|
||||
// Restore the package default afterward so this does not leak into other tests.
|
||||
orig := DefaultStaleAfter
|
||||
DefaultStaleAfter = time.Hour
|
||||
defer func() { DefaultStaleAfter = orig }()
|
||||
lk, err := AcquireLock(root)
|
||||
if err != nil {
|
||||
t.Fatalf("lock not released: %v", err)
|
||||
}
|
||||
lk.Release()
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package dream
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// BackgroundSpawn launches one dream consolidation subprocess for projectDir and
|
||||
// returns its decoded Report. Implementations live in the CLI layer (they shell
|
||||
// out to `pigo --dream -C <projectDir>` and parse the stdout Report), so this
|
||||
// package stays free of os/exec concerns and there is no import cycle back
|
||||
// through the CLI. A non-nil error means the run failed; background failures are
|
||||
// silent to the user (the subprocess itself records last_status="failed"), so
|
||||
// MaybeRunBackground swallows the error rather than surfacing it.
|
||||
type BackgroundSpawn func(ctx context.Context, projectDir string) (Report, error)
|
||||
|
||||
// BackgroundDeps carries everything MaybeRunBackground needs to decide on and run
|
||||
// a startup auto-consolidation without pulling process/exec or presentation
|
||||
// concerns into this package.
|
||||
type BackgroundDeps struct {
|
||||
// MemoryRoot is the dream state root read to decide whether a run is due. It
|
||||
// must match the root the subprocess consolidates (dream.ResolveMemoryRoot),
|
||||
// so the parent's Due check sees the same last_run_at the child updates.
|
||||
MemoryRoot string
|
||||
// ProjectDir is the working directory attributed to the run (project scope).
|
||||
ProjectDir string
|
||||
// Config is the resolved [dream] configuration (enabled / interval).
|
||||
Config Config
|
||||
// Now supplies the current time for the due check; nil uses time.Now. It is a
|
||||
// seam so tests can drive Due deterministically.
|
||||
Now func() time.Time
|
||||
// Spawn launches the subprocess. When nil, MaybeRunBackground does nothing.
|
||||
Spawn BackgroundSpawn
|
||||
// OnReport is invoked (from the background goroutine) with the completed
|
||||
// report only when the run produced actual changes — worth a one-line notice.
|
||||
// A skipped run (another dream held the lock → all-zero report) or a no-op run
|
||||
// yields no call, keeping the trigger non-intrusive. Nil disables the notice.
|
||||
OnReport func(Report)
|
||||
}
|
||||
|
||||
// Scheduler owns the startup auto-trigger decision (SPEC §2.1 Scheduler
|
||||
// component). It is stateless: Due reads state.json on demand and
|
||||
// MaybeRunBackground spawns at most one background run per call. The
|
||||
// single-instance guarantee is enforced by the subprocess's O_EXCL lock, not
|
||||
// here — a second trigger simply results in a skipped child.
|
||||
type Scheduler struct{}
|
||||
|
||||
// Due reports whether an auto-triggered consolidation is warranted now. It is
|
||||
// cheap by design (SPEC §8.2 zero-startup-overhead): when dream is disabled it
|
||||
// returns immediately without touching the filesystem; otherwise it reads
|
||||
// state.json once and defers to State.Due (which also returns false for a
|
||||
// never-run zero LastRunAt, so the first-ever run is never auto-triggered).
|
||||
func (Scheduler) Due(memoryRoot string, cfg Config, now time.Time) bool {
|
||||
if !cfg.Enabled {
|
||||
return false
|
||||
}
|
||||
st, _ := LoadState(memoryRoot)
|
||||
return st.Due(cfg, now)
|
||||
}
|
||||
|
||||
// MaybeRunBackground checks (cheaply) whether a consolidation is due and, if so,
|
||||
// spawns it in a detached goroutine and returns immediately — it never blocks
|
||||
// the caller, so the first interactive response is never delayed (SPEC FR-4 /
|
||||
// §8.2). It returns true when a background run was launched. When dream is
|
||||
// disabled or not due it returns false after at most a single state.json read
|
||||
// (no goroutine, no subprocess).
|
||||
//
|
||||
// On completion the goroutine surfaces a one-line notice via OnReport only for a
|
||||
// run that changed something; a skipped run (lock held elsewhere → zero report),
|
||||
// a no-op run, or a failed run is silent (SPEC §6.1 background row).
|
||||
func (s Scheduler) MaybeRunBackground(ctx context.Context, deps BackgroundDeps) bool {
|
||||
if deps.Spawn == nil {
|
||||
return false
|
||||
}
|
||||
now := time.Now
|
||||
if deps.Now != nil {
|
||||
now = deps.Now
|
||||
}
|
||||
if !s.Due(deps.MemoryRoot, deps.Config, now()) {
|
||||
return false
|
||||
}
|
||||
go func() {
|
||||
rep, err := deps.Spawn(ctx, deps.ProjectDir)
|
||||
if err != nil {
|
||||
// Background failure: silent. The subprocess already recorded
|
||||
// last_status="failed"; we do not interrupt the user with an error.
|
||||
return
|
||||
}
|
||||
if deps.OnReport != nil && reportHasChanges(rep) {
|
||||
deps.OnReport(rep)
|
||||
}
|
||||
}()
|
||||
return true
|
||||
}
|
||||
|
||||
// reportHasChanges reports whether r reflects any actual mutation. A background
|
||||
// run that skipped (lock contention) or found nothing to do produces an all-zero
|
||||
// report, which is not worth a startup notice.
|
||||
func reportHasChanges(r Report) bool {
|
||||
return r.Merged > 0 || r.Deduped > 0 || r.PathsCleaned > 0 ||
|
||||
r.Pruned > 0 || r.Distilled > 0 ||
|
||||
r.Reconciled.Indexed > 0 || r.Reconciled.Pruned > 0
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
package dream
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fixedNow returns a func yielding t, for the Now seam.
|
||||
func fixedNow(t time.Time) func() time.Time { return func() time.Time { return t } }
|
||||
|
||||
// writeDueState seeds a state.json under memoryRoot with a LastRunAt old enough
|
||||
// that Due(cfg, now) is true for the default interval.
|
||||
func writeDueState(t *testing.T, memoryRoot string, lastRun time.Time) {
|
||||
t.Helper()
|
||||
if err := SaveState(memoryRoot, State{LastRunAt: lastRun, LastStatus: "ok"}); err != nil {
|
||||
t.Fatalf("SaveState: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchedulerDue(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
now := time.Date(2026, 1, 20, 12, 0, 0, 0, time.UTC)
|
||||
cfg := NewConfig(nil, 7, 20) // enabled, 7-day interval
|
||||
|
||||
var s Scheduler
|
||||
|
||||
// Disabled → never due, and cheap (no state read needed).
|
||||
disabled := NewConfig(boolPtr(false), 7, 20)
|
||||
if s.Due(root, disabled, now) {
|
||||
t.Fatal("disabled config must not be due")
|
||||
}
|
||||
|
||||
// No state file (never run) → not due (first run is manual, spec §11.1).
|
||||
if s.Due(root, cfg, now) {
|
||||
t.Fatal("never-run state must not be due")
|
||||
}
|
||||
|
||||
// Last run within the interval → not due.
|
||||
writeDueState(t, root, now.Add(-3*24*time.Hour))
|
||||
if s.Due(root, cfg, now) {
|
||||
t.Fatal("run 3d ago with 7d interval must not be due")
|
||||
}
|
||||
|
||||
// Last run older than the interval → due.
|
||||
writeDueState(t, root, now.Add(-8*24*time.Hour))
|
||||
if !s.Due(root, cfg, now) {
|
||||
t.Fatal("run 8d ago with 7d interval must be due")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaybeRunBackground_NotSpawnedWhenDisabled(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeDueState(t, root, time.Now().Add(-30*24*time.Hour)) // would be due if enabled
|
||||
|
||||
var spawned bool
|
||||
launched := Scheduler{}.MaybeRunBackground(context.Background(), BackgroundDeps{
|
||||
MemoryRoot: root,
|
||||
Config: NewConfig(boolPtr(false), 7, 20),
|
||||
Now: fixedNow(time.Now()),
|
||||
Spawn: func(context.Context, string) (Report, error) {
|
||||
spawned = true
|
||||
return Report{}, nil
|
||||
},
|
||||
})
|
||||
if launched {
|
||||
t.Fatal("MaybeRunBackground returned true for disabled dream")
|
||||
}
|
||||
if spawned {
|
||||
t.Fatal("subprocess must not be spawned when disabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaybeRunBackground_NotSpawnedWhenNotDue(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
now := time.Date(2026, 2, 1, 9, 0, 0, 0, time.UTC)
|
||||
writeDueState(t, root, now.Add(-1*24*time.Hour)) // 1 day ago, interval 7d → not due
|
||||
|
||||
var spawned bool
|
||||
launched := Scheduler{}.MaybeRunBackground(context.Background(), BackgroundDeps{
|
||||
MemoryRoot: root,
|
||||
Config: NewConfig(nil, 7, 20),
|
||||
Now: fixedNow(now),
|
||||
Spawn: func(context.Context, string) (Report, error) {
|
||||
spawned = true
|
||||
return Report{}, nil
|
||||
},
|
||||
})
|
||||
if launched || spawned {
|
||||
t.Fatalf("not-due run must not launch/spawn (launched=%v spawned=%v)", launched, spawned)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaybeRunBackground_SpawnsAndNoticesOnChanges(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
now := time.Date(2026, 2, 10, 9, 0, 0, 0, time.UTC)
|
||||
writeDueState(t, root, now.Add(-10*24*time.Hour)) // due
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
gotDir string
|
||||
reported *Report
|
||||
done = make(chan struct{})
|
||||
)
|
||||
launched := Scheduler{}.MaybeRunBackground(context.Background(), BackgroundDeps{
|
||||
MemoryRoot: root,
|
||||
ProjectDir: "/proj/x",
|
||||
Config: NewConfig(nil, 7, 20),
|
||||
Now: fixedNow(now),
|
||||
Spawn: func(_ context.Context, dir string) (Report, error) {
|
||||
mu.Lock()
|
||||
gotDir = dir
|
||||
mu.Unlock()
|
||||
return Report{Merged: 2, Deduped: 1}, nil
|
||||
},
|
||||
OnReport: func(r Report) {
|
||||
mu.Lock()
|
||||
reported = &r
|
||||
mu.Unlock()
|
||||
close(done)
|
||||
},
|
||||
})
|
||||
if !launched {
|
||||
t.Fatal("due run must launch a background spawn")
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("OnReport not called within timeout")
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if gotDir != "/proj/x" {
|
||||
t.Fatalf("spawn got dir %q, want /proj/x", gotDir)
|
||||
}
|
||||
if reported == nil || reported.Merged != 2 {
|
||||
t.Fatalf("OnReport got %+v, want Merged=2", reported)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaybeRunBackground_SkippedRunIsSilent(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
now := time.Date(2026, 3, 1, 9, 0, 0, 0, time.UTC)
|
||||
writeDueState(t, root, now.Add(-10*24*time.Hour)) // due
|
||||
|
||||
spawnDone := make(chan struct{})
|
||||
var noticed bool
|
||||
var mu sync.Mutex
|
||||
Scheduler{}.MaybeRunBackground(context.Background(), BackgroundDeps{
|
||||
MemoryRoot: root,
|
||||
Config: NewConfig(nil, 7, 20),
|
||||
Now: fixedNow(now),
|
||||
Spawn: func(context.Context, string) (Report, error) {
|
||||
// A skipped/lock-held run emits an all-zero report with no error.
|
||||
defer close(spawnDone)
|
||||
return Report{}, nil
|
||||
},
|
||||
OnReport: func(Report) {
|
||||
mu.Lock()
|
||||
noticed = true
|
||||
mu.Unlock()
|
||||
},
|
||||
})
|
||||
<-spawnDone
|
||||
// Give the goroutine a moment past Spawn to (not) call OnReport.
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if noticed {
|
||||
t.Fatal("all-zero (skipped/no-op) report must not produce a notice")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaybeRunBackground_FailureIsSilent(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
now := time.Date(2026, 3, 5, 9, 0, 0, 0, time.UTC)
|
||||
writeDueState(t, root, now.Add(-10*24*time.Hour)) // due
|
||||
|
||||
spawnDone := make(chan struct{})
|
||||
var noticed bool
|
||||
var mu sync.Mutex
|
||||
Scheduler{}.MaybeRunBackground(context.Background(), BackgroundDeps{
|
||||
MemoryRoot: root,
|
||||
Config: NewConfig(nil, 7, 20),
|
||||
Now: fixedNow(now),
|
||||
Spawn: func(context.Context, string) (Report, error) {
|
||||
defer close(spawnDone)
|
||||
return Report{Merged: 5}, errors.New("boom")
|
||||
},
|
||||
OnReport: func(Report) {
|
||||
mu.Lock()
|
||||
noticed = true
|
||||
mu.Unlock()
|
||||
},
|
||||
})
|
||||
<-spawnDone
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if noticed {
|
||||
t.Fatal("a failed background run must be silent (no notice)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaybeRunBackground_NilSpawn(t *testing.T) {
|
||||
if (Scheduler{}).MaybeRunBackground(context.Background(), BackgroundDeps{}) {
|
||||
t.Fatal("nil Spawn must yield false (no-op)")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package dream
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
// State is the persisted /dream run state, stored as JSON at
|
||||
// <memoryRoot>/global/dream/state.json. A zero-value State (LastRunAt zero,
|
||||
// empty status, nil report) means dream has never run.
|
||||
type State struct {
|
||||
LastRunAt time.Time `json:"last_run_at"`
|
||||
LastStatus string `json:"last_status"` // "ok" | "failed" | "skipped"
|
||||
// LastReport holds the structured change report from the last run. It is
|
||||
// nil until dream has completed at least one non-dry-run pass.
|
||||
LastReport *Report `json:"last_report,omitempty"`
|
||||
}
|
||||
|
||||
// statePath is the state file location under the memory root.
|
||||
func statePath(memoryRoot string) string {
|
||||
return filepath.Join(memoryRoot, "global", "dream", "state.json")
|
||||
}
|
||||
|
||||
// LoadState reads the dream state from <memoryRoot>/global/dream/state.json. A
|
||||
// missing file returns a zero-value State (never run) with no error. Corrupt or
|
||||
// unreadable JSON is tolerated the same way: the caller gets a zero-value State
|
||||
// and no error, so a damaged state file degrades to "never run" rather than
|
||||
// breaking dream entirely.
|
||||
func LoadState(memoryRoot string) (State, error) {
|
||||
data, err := os.ReadFile(statePath(memoryRoot))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return State{}, nil
|
||||
}
|
||||
// Unreadable (permissions, transient IO): treat as never-run rather
|
||||
// than surfacing an error that would block dream.
|
||||
return State{}, nil
|
||||
}
|
||||
var s State
|
||||
if err := json.Unmarshal(data, &s); err != nil {
|
||||
// Corrupt JSON: degrade to never-run.
|
||||
return State{}, nil
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// SaveState writes the dream state to <memoryRoot>/global/dream/state.json,
|
||||
// creating the parent directory lazily. The file is written atomically via a
|
||||
// temp file + rename so a crash mid-write cannot leave a truncated state.json.
|
||||
func SaveState(memoryRoot string, s State) error {
|
||||
dir := filepath.Join(memoryRoot, "global", "dream")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := json.MarshalIndent(s, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
path := statePath(memoryRoot)
|
||||
tmp, err := os.CreateTemp(dir, "state-*.json.tmp")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmpName := tmp.Name()
|
||||
if _, err := tmp.Write(data); err != nil {
|
||||
tmp.Close()
|
||||
os.Remove(tmpName)
|
||||
return err
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
os.Remove(tmpName)
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tmpName, path); err != nil {
|
||||
os.Remove(tmpName)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Due reports whether an auto-triggered consolidation is warranted at now given
|
||||
// cfg. It returns true only when dream is enabled and the configured interval
|
||||
// has elapsed since the last run. Two cases deliberately return false:
|
||||
// - cfg.Enabled is false: auto-trigger is disabled entirely (US-001).
|
||||
// - a zero LastRunAt (dream has never run): the first-ever run is NOT
|
||||
// auto-triggered — the user is prompted to run /dream manually instead — to
|
||||
// avoid a cold-start token cost for new users. See spec §11.1.
|
||||
func (s State) Due(cfg Config, now time.Time) bool {
|
||||
if !cfg.Enabled {
|
||||
return false
|
||||
}
|
||||
if s.LastRunAt.IsZero() {
|
||||
return false
|
||||
}
|
||||
interval := time.Duration(cfg.IntervalDays) * 24 * time.Hour
|
||||
return now.Sub(s.LastRunAt) >= interval
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package dream
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestStateRoundTrip(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
report := &Report{Merged: 3, Deduped: 1, DryRun: false}
|
||||
want := State{
|
||||
LastRunAt: time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC),
|
||||
LastStatus: "ok",
|
||||
LastReport: report,
|
||||
}
|
||||
if err := SaveState(root, want); err != nil {
|
||||
t.Fatalf("SaveState: %v", err)
|
||||
}
|
||||
got, err := LoadState(root)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadState: %v", err)
|
||||
}
|
||||
if !got.LastRunAt.Equal(want.LastRunAt) {
|
||||
t.Errorf("LastRunAt = %v, want %v", got.LastRunAt, want.LastRunAt)
|
||||
}
|
||||
if got.LastStatus != want.LastStatus {
|
||||
t.Errorf("LastStatus = %q, want %q", got.LastStatus, want.LastStatus)
|
||||
}
|
||||
if got.LastReport == nil {
|
||||
t.Fatalf("LastReport = nil, want %+v", want.LastReport)
|
||||
}
|
||||
if got.LastReport.Merged != report.Merged || got.LastReport.Deduped != report.Deduped {
|
||||
t.Errorf("LastReport = %+v, want %+v", got.LastReport, report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSaveStateCreatesDir(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := SaveState(root, State{LastStatus: "ok"}); err != nil {
|
||||
t.Fatalf("SaveState: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "global", "dream", "state.json")); err != nil {
|
||||
t.Errorf("state.json not created: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadStateMissingIsZero(t *testing.T) {
|
||||
got, err := LoadState(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("LoadState: %v", err)
|
||||
}
|
||||
if !got.LastRunAt.IsZero() || got.LastStatus != "" || got.LastReport != nil {
|
||||
t.Errorf("missing state = %+v, want zero-value", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadStateCorruptTolerated(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
dir := filepath.Join(root, "global", "dream")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "state.json"), []byte("{not valid json"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := LoadState(root)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadState returned error on corrupt JSON, want tolerated: %v", err)
|
||||
}
|
||||
if !got.LastRunAt.IsZero() {
|
||||
t.Errorf("corrupt state = %+v, want zero-value (never run)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDue(t *testing.T) {
|
||||
now := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)
|
||||
cfg := Config{Enabled: true, IntervalDays: 7, RecentSessions: 20}
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg Config
|
||||
last time.Time
|
||||
want bool
|
||||
}{
|
||||
{"due: interval elapsed", cfg, now.Add(-8 * 24 * time.Hour), true},
|
||||
{"due: exactly at interval", cfg, now.Add(-7 * 24 * time.Hour), true},
|
||||
{"not due: within interval", cfg, now.Add(-3 * 24 * time.Hour), false},
|
||||
{"zero LastRunAt never due", cfg, time.Time{}, false},
|
||||
{"disabled never due", Config{Enabled: false, IntervalDays: 7}, now.Add(-30 * 24 * time.Hour), false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := State{LastRunAt: tt.last}
|
||||
if got := s.Due(tt.cfg, now); got != tt.want {
|
||||
t.Errorf("Due = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user