first commit
This commit is contained in:
@@ -0,0 +1,359 @@
|
||||
package dream
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// NearDupThreshold is the normalized-token Jaccard similarity at or above which
|
||||
// two memory files are emitted as a near-duplicate candidate pair for the later
|
||||
// LLM merge-decision step (spec §5.1.1). It is deliberately conservative: this
|
||||
// node only PAIRS candidates, it never merges, so a false-positive pair costs
|
||||
// only one extra LLM comparison while a missed pair silently loses a merge
|
||||
// opportunity.
|
||||
const NearDupThreshold = 0.7
|
||||
|
||||
// MemoryFile is one enumerated memory file under the global or the active
|
||||
// project scope, with the derived metadata the deterministic plan needs. Body
|
||||
// is retained so the near-duplicate pairing can tokenize it without a second
|
||||
// read.
|
||||
type MemoryFile struct {
|
||||
Path string // absolute path on disk
|
||||
Scope string // "global" | "projects"
|
||||
Type string // layout <type> segment ("" for a file directly under the scope root)
|
||||
Size int64 // byte size of the file content
|
||||
ContentHash string // sha256 hex of the raw file content
|
||||
Body string // full file content (used for path extraction + tokenization)
|
||||
}
|
||||
|
||||
// DedupeGroup is a set of two or more files whose content is byte-identical
|
||||
// (same ContentHash). The apply node keeps one representative and removes the
|
||||
// rest; Deduped in the Report counts len(Paths)-1 per group.
|
||||
type DedupeGroup struct {
|
||||
Hash string `json:"hash"`
|
||||
Paths []string `json:"paths"`
|
||||
}
|
||||
|
||||
// InvalidPathRef is a local filesystem path mentioned in a memory file's body
|
||||
// that no longer exists on disk. External references (URLs, mailto:, etc.) are
|
||||
// never recorded here (spec §5.2 / PRD US-004).
|
||||
type InvalidPathRef struct {
|
||||
File string `json:"file"` // memory file containing the reference
|
||||
Ref string `json:"ref"` // the original (unresolved) reference text
|
||||
}
|
||||
|
||||
// NearDupPair is a candidate pair of files whose token-set similarity is at or
|
||||
// above NearDupThreshold but whose content is not byte-identical. It is a
|
||||
// suggestion for the LLM to decide whether an actual merge is warranted; this
|
||||
// node performs no merge.
|
||||
type NearDupPair struct {
|
||||
A string `json:"a"`
|
||||
B string `json:"b"`
|
||||
Similarity float64 `json:"similarity"`
|
||||
}
|
||||
|
||||
// Plan is the plain-data output of the deterministic half of a /dream run. It
|
||||
// carries no behavior and makes no LLM calls; the later apply node consumes it
|
||||
// to drive merges/prunes and to compute the final Report counters.
|
||||
type Plan struct {
|
||||
Files []MemoryFile `json:"files"`
|
||||
DedupeGroups []DedupeGroup `json:"dedupe_groups"`
|
||||
InvalidPathRefs []InvalidPathRef `json:"invalid_path_refs"`
|
||||
NearDupPairs []NearDupPair `json:"near_dup_pairs"`
|
||||
BytesBefore int64 `json:"bytes_before"`
|
||||
FilesBefore int `json:"files_before"`
|
||||
}
|
||||
|
||||
// BuildPlan enumerates the consolidation-eligible memory files (global scope +
|
||||
// the given project's scope) under memoryRoot and computes the deterministic
|
||||
// consolidation plan: exact-dedup groups, invalid local path references, and
|
||||
// near-duplicate candidate pairs. It excludes the sessions scope entirely and
|
||||
// any file whose layout type is "checkpoint" (session-transient state, not
|
||||
// long-term memory — spec §1.3 / §5.1.1).
|
||||
//
|
||||
// projectDir is the working directory of the active project; its stable project
|
||||
// id (matching internal/memory's resolveProjectId) selects the projects
|
||||
// sub-scope. An empty projectDir restricts the plan to the global scope. A
|
||||
// missing memoryRoot or scope directory is not an error — it yields an empty
|
||||
// plan, mirroring memory.Reconcile's tolerance of an absent root.
|
||||
func BuildPlan(memoryRoot, projectDir string) (Plan, error) {
|
||||
var plan Plan
|
||||
|
||||
globalRoot := filepath.Join(memoryRoot, "global")
|
||||
globalFiles, err := enumerateScope(globalRoot, "global")
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
files := globalFiles
|
||||
|
||||
if projectDir != "" {
|
||||
projectRoot := filepath.Join(memoryRoot, "projects", projectID(projectDir))
|
||||
projFiles, err := enumerateScope(projectRoot, "projects")
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
files = append(files, projFiles...)
|
||||
}
|
||||
|
||||
plan.Files = files
|
||||
plan.FilesBefore = len(files)
|
||||
for _, f := range files {
|
||||
plan.BytesBefore += f.Size
|
||||
}
|
||||
|
||||
plan.DedupeGroups = dedupeGroups(files)
|
||||
plan.InvalidPathRefs = invalidPathRefs(files, projectDir)
|
||||
plan.NearDupPairs = nearDupPairs(files)
|
||||
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
// enumerateScope walks scopeRoot for *.md files, deriving each file's layout
|
||||
// type from its first path segment relative to scopeRoot. Files under a
|
||||
// "checkpoint" type directory are skipped. A missing scopeRoot yields no files
|
||||
// and no error.
|
||||
func enumerateScope(scopeRoot, scope string) ([]MemoryFile, error) {
|
||||
var out []MemoryFile
|
||||
err := filepath.WalkDir(scopeRoot, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if d.IsDir() || !strings.EqualFold(filepath.Ext(d.Name()), ".md") {
|
||||
return nil
|
||||
}
|
||||
rel, relErr := filepath.Rel(scopeRoot, path)
|
||||
if relErr != nil {
|
||||
return nil
|
||||
}
|
||||
segs := strings.Split(filepath.ToSlash(rel), "/")
|
||||
typ := ""
|
||||
if len(segs) >= 2 {
|
||||
typ = strings.ToLower(segs[0])
|
||||
}
|
||||
if typ == "checkpoint" {
|
||||
return nil
|
||||
}
|
||||
raw, readErr := os.ReadFile(path)
|
||||
if readErr != nil {
|
||||
if os.IsNotExist(readErr) {
|
||||
return nil
|
||||
}
|
||||
return readErr
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
out = append(out, MemoryFile{
|
||||
Path: filepath.Clean(path),
|
||||
Scope: scope,
|
||||
Type: typ,
|
||||
Size: int64(len(raw)),
|
||||
ContentHash: hex.EncodeToString(sum[:]),
|
||||
Body: string(raw),
|
||||
})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Deterministic order regardless of directory iteration order.
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Path < out[j].Path })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// dedupeGroups groups files by identical content hash, returning only groups
|
||||
// with two or more members (a file with a unique hash is not a duplicate). The
|
||||
// result is stable: groups are ordered by hash, paths within a group by path.
|
||||
func dedupeGroups(files []MemoryFile) []DedupeGroup {
|
||||
byHash := make(map[string][]string)
|
||||
for _, f := range files {
|
||||
byHash[f.ContentHash] = append(byHash[f.ContentHash], f.Path)
|
||||
}
|
||||
var groups []DedupeGroup
|
||||
for hash, paths := range byHash {
|
||||
if len(paths) < 2 {
|
||||
continue
|
||||
}
|
||||
sort.Strings(paths)
|
||||
groups = append(groups, DedupeGroup{Hash: hash, Paths: paths})
|
||||
}
|
||||
sort.Slice(groups, func(i, j int) bool { return groups[i].Hash < groups[j].Hash })
|
||||
return groups
|
||||
}
|
||||
|
||||
// invalidPathRefs scans each file's body for local filesystem path references
|
||||
// and records the ones that no longer exist. Relative references are resolved
|
||||
// against projectDir; "~/" against the user home dir. URLs and other external
|
||||
// references are never considered (see extractLocalPathRefs).
|
||||
func invalidPathRefs(files []MemoryFile, projectDir string) []InvalidPathRef {
|
||||
var out []InvalidPathRef
|
||||
for _, f := range files {
|
||||
seen := make(map[string]struct{})
|
||||
for _, ref := range extractLocalPathRefs(f.Body) {
|
||||
if _, dup := seen[ref]; dup {
|
||||
continue
|
||||
}
|
||||
seen[ref] = struct{}{}
|
||||
// A bare relative reference is only meaningful against a project
|
||||
// base. Without one (global-only plan) skip it rather than
|
||||
// resolving against an arbitrary cwd and flagging spuriously.
|
||||
isRelative := !filepath.IsAbs(ref) && !strings.HasPrefix(ref, "~/")
|
||||
if isRelative && projectDir == "" {
|
||||
continue
|
||||
}
|
||||
resolved := resolveRef(ref, projectDir)
|
||||
if _, err := os.Stat(resolved); err != nil && os.IsNotExist(err) {
|
||||
out = append(out, InvalidPathRef{File: f.Path, Ref: ref})
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// nearDupPairs emits candidate pairs whose normalized-token Jaccard similarity
|
||||
// is at or above NearDupThreshold. Pairs of byte-identical files are skipped:
|
||||
// those are exact duplicates handled by dedupeGroups, not near-duplicates.
|
||||
func nearDupPairs(files []MemoryFile) []NearDupPair {
|
||||
tokens := make([]map[string]struct{}, len(files))
|
||||
for i, f := range files {
|
||||
tokens[i] = tokenize(f.Body)
|
||||
}
|
||||
var out []NearDupPair
|
||||
for i := 0; i < len(files); i++ {
|
||||
for j := i + 1; j < len(files); j++ {
|
||||
if files[i].ContentHash == files[j].ContentHash {
|
||||
continue
|
||||
}
|
||||
sim := jaccard(tokens[i], tokens[j])
|
||||
if sim >= NearDupThreshold {
|
||||
out = append(out, NearDupPair{A: files[i].Path, B: files[j].Path, Similarity: sim})
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// projectID derives the stable projects-scope id from a project directory,
|
||||
// mirroring internal/memory.resolveProjectId: the first 12 hex chars of
|
||||
// sha256(absPath). The path is made absolute first so relative and absolute
|
||||
// forms of the same directory map to the same id.
|
||||
func projectID(projectDir string) string {
|
||||
abs, err := filepath.Abs(projectDir)
|
||||
if err != nil {
|
||||
abs = projectDir
|
||||
}
|
||||
sum := sha256.Sum256([]byte(abs))
|
||||
return hex.EncodeToString(sum[:])[:12]
|
||||
}
|
||||
|
||||
// extractLocalPathRefs returns the local filesystem path references found in
|
||||
// body. It splits on whitespace and common Markdown/code delimiters (so
|
||||
// `path`, [text](path) and bare tokens are all captured), then keeps only
|
||||
// tokens that look like a local path while rejecting URLs and other external
|
||||
// references.
|
||||
func extractLocalPathRefs(body string) []string {
|
||||
fields := strings.FieldsFunc(body, func(r rune) bool {
|
||||
switch r {
|
||||
case ' ', '\t', '\n', '\r', '`', '(', ')', '[', ']', '{', '}', '"', '\'', '<', '>', ',', ';', '|':
|
||||
return true
|
||||
}
|
||||
return false
|
||||
})
|
||||
var out []string
|
||||
for _, tok := range fields {
|
||||
// Trim trailing sentence punctuation that commonly abuts a path.
|
||||
tok = strings.TrimRight(tok, ".:!?")
|
||||
if isLocalPathRef(tok) {
|
||||
out = append(out, tok)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// isLocalPathRef reports whether tok looks like a local filesystem path rather
|
||||
// than a URL or other external reference. It accepts absolute ("/..."),
|
||||
// explicitly relative ("./", "../") and home-relative ("~/") tokens outright;
|
||||
// a bare relative token (no leading marker) must contain a separator AND a file
|
||||
// extension in its last segment, so ordinary prose like "TCP/IP", "read/write"
|
||||
// or "and/or" is not mistaken for a path. URL-schemed tokens are rejected.
|
||||
func isLocalPathRef(tok string) bool {
|
||||
if tok == "" {
|
||||
return false
|
||||
}
|
||||
if strings.Contains(tok, "://") {
|
||||
return false
|
||||
}
|
||||
lower := strings.ToLower(tok)
|
||||
for _, scheme := range []string{"http:", "https:", "ftp:", "ftps:", "file:", "mailto:", "ssh:", "git:", "www."} {
|
||||
if strings.HasPrefix(lower, scheme) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(tok, "/"),
|
||||
strings.HasPrefix(tok, "./"),
|
||||
strings.HasPrefix(tok, "../"),
|
||||
strings.HasPrefix(tok, "~/"):
|
||||
return true
|
||||
}
|
||||
// A bare relative token needs both a separator and an extension on its last
|
||||
// segment to be treated as a path (avoids flagging prose like "TCP/IP").
|
||||
if !strings.Contains(tok, "/") {
|
||||
return false
|
||||
}
|
||||
last := tok[strings.LastIndex(tok, "/")+1:]
|
||||
return strings.Contains(last, ".") && !strings.HasSuffix(last, ".")
|
||||
}
|
||||
|
||||
// resolveRef turns a reference into an absolute path for existence checking:
|
||||
// "~/" expands to the user home dir, absolute paths pass through, and relative
|
||||
// paths resolve against projectDir (or the current dir when projectDir is "").
|
||||
func resolveRef(ref, projectDir string) string {
|
||||
if strings.HasPrefix(ref, "~/") {
|
||||
if home, err := os.UserHomeDir(); err == nil {
|
||||
return filepath.Join(home, ref[2:])
|
||||
}
|
||||
}
|
||||
if filepath.IsAbs(ref) {
|
||||
return filepath.Clean(ref)
|
||||
}
|
||||
return filepath.Join(projectDir, ref)
|
||||
}
|
||||
|
||||
// tokenize lowercases body and splits it into the set of alphanumeric tokens
|
||||
// used for near-duplicate similarity. Punctuation and Markdown syntax are
|
||||
// discarded so formatting differences do not affect the score.
|
||||
func tokenize(body string) map[string]struct{} {
|
||||
set := make(map[string]struct{})
|
||||
for _, tok := range strings.FieldsFunc(strings.ToLower(body), func(r rune) bool {
|
||||
return !unicode.IsLetter(r) && !unicode.IsNumber(r)
|
||||
}) {
|
||||
set[tok] = struct{}{}
|
||||
}
|
||||
return set
|
||||
}
|
||||
|
||||
// jaccard returns |a∩b| / |a∪b|. Two empty sets are dissimilar (0) rather than
|
||||
// identical, so blank files are never paired as near-duplicates.
|
||||
func jaccard(a, b map[string]struct{}) float64 {
|
||||
if len(a) == 0 || len(b) == 0 {
|
||||
return 0
|
||||
}
|
||||
inter := 0
|
||||
for t := range a {
|
||||
if _, ok := b[t]; ok {
|
||||
inter++
|
||||
}
|
||||
}
|
||||
union := len(a) + len(b) - inter
|
||||
if union == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(inter) / float64(union)
|
||||
}
|
||||
Reference in New Issue
Block a user