first commit

2026-08-14 23:41:57 +08:00
commit 086803a8dd
471 changed files with 91938 additions and 0 deletions
@@ -0,0 +1,173 @@
+package memory
+
+import (
+	"database/sql"
+	"fmt"
+	"strings"
+)
+
+// SearchResult is a single ranked hit from Store.Search. Score is normalized so
+// that higher = better (the raw FTS5 bm25 value, where lower is better, is
+// negated).
+type SearchResult struct {
+	Path    string
+	Snippet string
+	Score   float64
+	Scope   Scope
+	ScopeID string
+	Type    Type
+}
+
+// SearchOptions tunes a Store.Search call. All fields are optional; the zero
+// value performs an unfiltered search with default limit and score floor.
+type SearchOptions struct {
+	// Scope, ScopeID and Type, when non-empty, restrict results to rows whose
+	// corresponding memory_index column matches exactly.
+	Scope   string
+	ScopeID string
+	Type    string
+	// Limit caps the number of returned results; <=0 means the default of 10.
+	Limit int
+	// ReconcileFirst runs a lazy Store.Reconcile before searching so that
+	// off-tool writes are picked up. Its counts are ignored; hard errors
+	// propagate.
+	ReconcileFirst bool
+	// ScoreFloor is the relative floor ratio: trailing rows scoring below
+	// topScore*ScoreFloor are dropped. <=0 keeps all matches when 0, but the
+	// default of 0.15 is used when the field is left at its zero value; pass a
+	// negative value to explicitly disable the floor.
+	ScoreFloor float64
+}
+
+// defaultSearchLimit is the result count used when SearchOptions.Limit <= 0.
+const defaultSearchLimit = 10
+
+// maxFetchLimit caps the over-fetch used to feed the relative score floor.
+const maxFetchLimit = 50
+
+// defaultScoreFloor is the relative floor applied when SearchOptions.ScoreFloor
+// is left at its zero value.
+const defaultScoreFloor = 0.15
+
+// Search runs a BM25 full-text query over the indexed memory bodies.
+//
+// The free-form query is tokenized and OR-joined by buildFtsQuery; an empty
+// token set returns (nil, nil) without touching SQL. Results are ranked by
+// BM25 (converted to higher = better), over-fetched 3x (capped at 50) so a
+// relative score floor can trim common-word-only noise, then sliced to the
+// requested limit. Optional scope/scope_id/type filters restrict the corpus.
+func (s *Store) Search(query string, opts SearchOptions) ([]SearchResult, error) {
+	if opts.ReconcileFirst {
+		if _, err := s.Reconcile(); err != nil {
+			return nil, fmt.Errorf("memory: reconcile before search: %w", err)
+		}
+	}
+
+	match := buildFtsQuery(query)
+	if match == "" {
+		// No usable tokens: treat as empty query with no results, send no SQL.
+		return nil, nil
+	}
+
+	limit := opts.Limit
+	if limit <= 0 {
+		limit = defaultSearchLimit
+	}
+	fetchLimit := limit * 3
+	if fetchLimit > maxFetchLimit {
+		fetchLimit = maxFetchLimit
+	}
+
+	// The FTS5 table `memory_fts` is external-content over `memory_index`, so
+	// filter columns (scope/scope_id/type) and the display path live on
+	// memory_index and the join is memory_index.id = memory_fts.rowid. snippet()
+	// and bm25() operate on the FTS table; body is FTS column 0.
+	var sb strings.Builder
+	sb.WriteString(`
+SELECT mi.path, mi.scope, mi.scope_id, mi.type,
+       snippet(memory_fts, 0, '<<', '>>', '...', 32) AS snippet,
+       bm25(memory_fts) AS score
+FROM memory_fts
+JOIN memory_index mi ON mi.id = memory_fts.rowid
+WHERE memory_fts MATCH ?`)
+
+	// MATCH parameter is always first; filter params follow in order.
+	args := []any{match}
+	if opts.Scope != "" {
+		sb.WriteString(" AND mi.scope = ?")
+		args = append(args, opts.Scope)
+	}
+	if opts.ScopeID != "" {
+		sb.WriteString(" AND mi.scope_id = ?")
+		args = append(args, opts.ScopeID)
+	}
+	if opts.Type != "" {
+		sb.WriteString(" AND mi.type = ?")
+		args = append(args, opts.Type)
+	}
+	// bm25(): lower = better, so ascending order puts the best hit first.
+	sb.WriteString(" ORDER BY score ASC LIMIT ?")
+	args = append(args, fetchLimit)
+
+	rows, err := s.db.Query(sb.String(), args...)
+	if err != nil {
+		return nil, fmt.Errorf("memory: search query: %w", err)
+	}
+	defer rows.Close()
+
+	var results []SearchResult
+	for rows.Next() {
+		var (
+			path, scope, scopeID, typ string
+			snippet                   sql.NullString
+			bm25                      float64
+		)
+		if err := rows.Scan(&path, &scope, &scopeID, &typ, &snippet, &bm25); err != nil {
+			return nil, fmt.Errorf("memory: scan search row: %w", err)
+		}
+		results = append(results, SearchResult{
+			Path:    path,
+			Snippet: snippet.String,
+			Score:   -bm25, // convert to higher = better
+			Scope:   Scope(scope),
+			ScopeID: scopeID,
+			Type:    Type(typ),
+		})
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("memory: iterate search rows: %w", err)
+	}
+	if len(results) == 0 {
+		return nil, nil
+	}
+
+	// Relative score floor. BM25 magnitudes are corpus-size dependent (in a
+	// tiny corpus every score collapses toward 0 due to low IDF), so an
+	// absolute floor would wrongly wipe real hits. We keep results scoring at
+	// least topScore*floor. The #1 result is ALWAYS kept — a match is a match
+	// even when BM25 can't discriminate. Default 0.15; a negative floor
+	// disables the trimming entirely.
+	floor := opts.ScoreFloor
+	if floor == 0 {
+		floor = defaultScoreFloor
+	}
+
+	// Rows come back ORDER BY score ASC (best first after negation), so
+	// results[0] is the top hit.
+	if floor > 0 {
+		topScore := results[0].Score
+		cutoff := topScore * floor
+		kept := results[:1]
+		for _, r := range results[1:] {
+			if r.Score >= cutoff {
+				kept = append(kept, r)
+			}
+		}
+		results = kept
+	}
+
+	if len(results) > limit {
+		results = results[:limit]
+	}
+	return results, nil
+}