fix(detect): honor .graphifyignore for the memory tree (fixes #2267)

graphify/detect.py

@@ -965,7 +965,15 @@ def _git_info_exclude(vcs_root: Path) -> Path | None:
     return exclude if exclude.is_file() else None
 
 
-def _load_dir_own_ignore(d: Path, *, gitignore: bool = True) -> list[tuple[Path, str]]:
+_IGNORE_SOURCE_GITIGNORE = "gitignore"
+_IGNORE_SOURCE_GRAPHIFYIGNORE = "graphifyignore"
+_IGNORE_SOURCE_INFO_EXCLUDE = "info_exclude"
+_IGNORE_SOURCE_CLI_EXCLUDE = "cli_exclude"
+
+
+def _load_dir_own_ignore(
+    d: Path, *, gitignore: bool = True
+) -> list[tuple[Path, str, str]]:
     """Read .gitignore/.graphifyignore directly inside *d* (not its ancestors).
 
     Merges .gitignore and .graphifyignore for this one directory (#1363):
@@ -975,25 +983,36 @@ def _load_dir_own_ignore(d: Path, *, gitignore: bool = True) -> list[tuple[Path,
     .gitignore-excluded file (#945 kept: a dir with only a .gitignore still
     gets sensible defaults).
 
+    Each pattern is tagged with its source so callers can apply only a subset
+    of origins — e.g. memory-tree files should respect ``.graphifyignore``
+    but NOT ``.gitignore`` (generated output like ``graphify-out/`` is
+    typically gitignored, and honouring that there would silently drop memory
+    files).
+
     Shared by `_load_graphifyignore` (ancestor chain, loaded once before the
     scan) and the live os.walk loop in `detect()` (called per-directory as
     each descendant is visited), so nested ignore files *below* the scan
     root are honored too — previously only the scan root and its ancestors
     were read, so e.g. `vendor/sub/.gitignore` was silently ignored (#1206).
     """
-    patterns: list[tuple[Path, str]] = []
-    for fname in ((".gitignore", ".graphifyignore") if gitignore else (".graphifyignore",)):
+    patterns: list[tuple[Path, str, str]] = []
+    for fname, source in (
+        ((".gitignore", _IGNORE_SOURCE_GITIGNORE),
+         (".graphifyignore", _IGNORE_SOURCE_GRAPHIFYIGNORE))
+        if gitignore
+        else ((".graphifyignore", _IGNORE_SOURCE_GRAPHIFYIGNORE),)
+    ):
         ignore_file = d / fname
         if ignore_file.exists():
             for raw in ignore_file.read_text(encoding="utf-8-sig", errors="ignore").splitlines():
                 line = _parse_gitignore_line(raw)
                 if line:
-                    patterns.append((d, line))
+                    patterns.append((d, line, source))
     return patterns
 
 
-def _load_graphifyignore(root: Path, *, gitignore: bool = True) -> list[tuple[Path, str]]:
-    """Read .graphifyignore files and return (anchor_dir, pattern) pairs.
+def _load_graphifyignore(root: Path, *, gitignore: bool = True) -> list[tuple[Path, str, str]]:
+    """Read .graphifyignore files and return (anchor_dir, pattern, source) triples.
 
     Patterns are returned outer-first so that inner (closer) rules are
     appended last and win via last-match-wins semantics — matching gitignore
@@ -1005,6 +1024,9 @@ def _load_graphifyignore(root: Path, *, gitignore: bool = True) -> list[tuple[Pa
     Covers the scan root and its ancestors only — directories *below* the
     scan root are picked up live during the os.walk in `detect()` instead,
     since they aren't known until the walk reaches them (#1206).
+
+    The third element of each tuple records provenance so callers can apply
+    only patterns from a specific origin (see ``_IGNORE_SOURCE_*`` constants).
     """
     root = root.resolve()
     ceiling = _find_vcs_root(root) or root
@@ -1019,7 +1041,7 @@ def _load_graphifyignore(root: Path, *, gitignore: bool = True) -> list[tuple[Pa
         current = current.parent
     dirs.reverse()  # ceiling first, scan root last
 
-    patterns: list[tuple[Path, str]] = []
+    patterns: list[tuple[Path, str, str]] = []
 
     # $GIT_DIR/info/exclude is repo-root-scoped and, per git, ranks below every
     # per-directory .gitignore/.graphifyignore — so load it first (lowest priority
@@ -1030,7 +1052,7 @@ def _load_graphifyignore(root: Path, *, gitignore: bool = True) -> list[tuple[Pa
         for raw in info_exclude.read_text(encoding="utf-8-sig", errors="ignore").splitlines():
             line = _parse_gitignore_line(raw)
             if line:
-                patterns.append((ceiling, line))
+                patterns.append((ceiling, line, _IGNORE_SOURCE_INFO_EXCLUDE))
 
     for d in dirs:
         patterns.extend(_load_dir_own_ignore(d, gitignore=gitignore))
@@ -1068,11 +1090,12 @@ def _matches(path_idx: int, pattern_idx: int) -> bool:
 def _is_ignored(
     path: Path,
     root: Path,
-    patterns: list[tuple[Path, str]],
+    patterns: list[tuple[Path, str, str]],
     *,
     _cache: dict[Path, bool] | None = None,
+    sources: set[str] | None = None,
 ) -> bool:
-    """Return True if the path should be ignored per .graphifyignore patterns.
+    """Return True if the path should be ignored per ignore patterns.
 
     Uses gitignore last-match-wins semantics: all patterns are evaluated in
     order; the final matching pattern determines the result. Negation patterns
@@ -1084,14 +1107,21 @@ def _is_ignored(
     _cache: optional dict shared across calls within the same scan. Ancestor
     directory results are memoised so files under the same subtree don't
     re-evaluate the same patterns repeatedly.
+
+    sources: optional set of provenance tags to filter by. When given, only
+    patterns whose third element is in ``sources`` are evaluated; all others
+    are skipped. Used to apply ONLY ``.graphifyignore``-sourced patterns to
+    the memory tree (so a gitignored ``graphify-out/`` doesn't silently drop
+    memory files) while keeping full ``.gitignore`` respect for the code tree.
     """
     if not patterns:
         return False
 
     def _eval(target: Path) -> bool:
         """Apply last-match-wins to a single target path."""
-        if _cache is not None and target in _cache:
-            return _cache[target]
+        cache_key = (target, frozenset(sources)) if sources else target
+        if _cache is not None and cache_key in _cache:
+            return _cache[cache_key]
         def _matches(rel: str, p: str, path_relative: bool) -> bool:
             if path_relative:
                 return _match_anchored_ignore_pattern(rel, p)
@@ -1108,7 +1138,12 @@ def _matches(rel: str, p: str, path_relative: bool) -> bool:
             return False
 
         result = False
-        for anchor, pattern in patterns:
+        for entry in patterns:
+            anchor, pattern = entry[0], entry[1]
+            if sources is not None:
+                source = entry[2] if len(entry) > 2 else None
+                if source not in sources:
+                    continue
             negated = pattern.startswith("!")
             raw = pattern[1:] if negated else pattern
             directory_only = raw.endswith("/")
@@ -1135,7 +1170,7 @@ def _matches(rel: str, p: str, path_relative: bool) -> bool:
             if matched:
                 result = not negated  # last match wins; ! flips to un-ignore
         if _cache is not None:
-            _cache[target] = result
+            _cache[cache_key] = result
         return result
 
     # Gitignore parent-exclusion rule: a ! re-include cannot rescue a file
@@ -1231,7 +1266,7 @@ def _wc(path: Path) -> int:
         for pat in extra_excludes:
             line = _parse_gitignore_line(pat)
             if line:
-                ignore_patterns.append((root, line))
+                ignore_patterns.append((root, line, _IGNORE_SOURCE_CLI_EXCLUDE))
 
     # Always include graphify-out/memory/ - query results filed back into the graph
     memory_dir = root / GRAPHIFY_OUT / "memory"
@@ -1334,9 +1369,21 @@ def _on_walk_error(err: OSError) -> None:
             # Skip files inside our own converted/ dir (avoid re-processing sidecars)
             if str(p).startswith(str(converted_dir)):
                 continue
-        if not in_memory and _is_ignored(p, root, ignore_patterns, _cache=ignore_cache):
-            ignored.append(str(p))
-            continue
+        # Memory-tree files respect ONLY .graphifyignore patterns — not
+        # .gitignore or $GIT_DIR/info/exclude. ``graphify-out/`` is generated
+        # output and is commonly gitignored; applying .gitignore to memory
+        # would silently drop every memory file (#2267).
+        if in_memory:
+            if _is_ignored(
+                p, root, ignore_patterns, _cache=ignore_cache,
+                sources={_IGNORE_SOURCE_GRAPHIFYIGNORE},
+            ):
+                ignored.append(str(p))
+                continue
+        else:
+            if _is_ignored(p, root, ignore_patterns, _cache=ignore_cache):
+                ignored.append(str(p))
+                continue
         if not _resolves_under_root(p, root):
             skipped_sensitive.append(str(p) + " [symlink target outside scan root]")
             continue

tests/test_detect.py

@@ -2569,3 +2569,121 @@ def test_sensitive_env_template_inside_secrets_dir_still_dropped(path):
     """Stage 1 dir guard runs before the Stage 2 template exemption: anything
     under a secrets/credentials dir stays excluded, template suffix or not."""
     assert _is_sensitive(Path(path)), f"{path} is under a secrets dir, must stay excluded (#2184)"
+
+def test_graphifyignore_excludes_memory_tree(tmp_path):
+    """#2267: .graphifyignore must reach graphify-out/memory/.
+
+    The memory tree (query results filed back into the graph) was exempt
+    from the ignore check, so a user could never exclude their own Q&A
+    output. After the fix, a pattern covering graphify-out/memory/ is
+    honored like every other path.
+    """
+    (tmp_path / "src").mkdir()
+    (tmp_path / "src" / "calc.py").write_text("def add(a, b):\n    return a + b\n")
+
+    # First detect seeds graphify-out/memory/ (the feedback loop).
+    detect(tmp_path)
+
+    memory = tmp_path / "graphify-out" / "memory"
+    memory.mkdir(parents=True, exist_ok=True)
+    (memory / "query_1.md").write_text("# Query\nThe calc module adds numbers.\n")
+
+    # User tries to exclude their own Q&A output.
+    (tmp_path / ".graphifyignore").write_text("graphify-out/memory/\n")
+
+    result = detect(tmp_path)
+    all_files = [f for files in result["files"].values() for f in files]
+    memory_hits = [f for f in all_files if "graphify-out" in f.replace("\\", "/")]
+    assert not memory_hits, \
+        "memory tree must be excludable via .graphifyignore, but got: %s" % memory_hits
+
+
+def test_gitignored_graphify_out_keeps_memory_included(tmp_path):
+    """#2267 regression: a gitignored graphify-out/ must NOT drop memory files.
+
+    ``graphify-out/`` is generated output and is commonly placed in
+    ``.gitignore``. The memory-tree ignore check is scoped to
+    ``.graphifyignore``-sourced patterns only, so a gitignored
+    ``graphify-out/`` still keeps memory files included — the default
+    memory round-trip stays intact.
+    """
+    (tmp_path / "src").mkdir()
+    (tmp_path / "src" / "calc.py").write_text("def add(a, b):\n    return a + b\n")
+
+    # First detect seeds graphify-out/memory/ (the feedback loop).
+    detect(tmp_path)
+
+    memory = tmp_path / "graphify-out" / "memory"
+    memory.mkdir(parents=True, exist_ok=True)
+    (memory / "query_1.md").write_text("# Query\nThe calc module adds numbers.\n")
+
+    # graphify-out/ is in .gitignore (common practice for generated output).
+    (tmp_path / ".gitignore").write_text("graphify-out/\n")
+
+    result = detect(tmp_path)
+    all_files = [f for files in result["files"].values() for f in files]
+    memory_hits = [f for f in all_files if "graphify-out" in f.replace("\\", "/")]
+    assert memory_hits, \
+        "memory files must stay included even when graphify-out/ is gitignored"
+
+
+def test_info_exclude_does_not_drop_memory(tmp_path):
+    """#2267 regression: $GIT_DIR/info/exclude must NOT drop memory files.
+
+    Same rationale as .gitignore: info/exclude is a git-sourced ignore,
+    not a user's explicit .graphifyignore. Memory files must survive.
+    """
+    (tmp_path / "src").mkdir()
+    (tmp_path / "src" / "calc.py").write_text("def add(a, b):\n    return a + b\n")
+
+    # First detect seeds graphify-out/memory/ (the feedback loop).
+    detect(tmp_path)
+
+    memory = tmp_path / "graphify-out" / "memory"
+    memory.mkdir(parents=True, exist_ok=True)
+    (memory / "query_1.md").write_text("# Query\nThe calc module adds numbers.\n")
+
+    # Stage a git repo so $GIT_DIR/info/exclude exists.
+    git_dir = tmp_path / ".git"
+    (git_dir / "info").mkdir(parents=True, exist_ok=True)
+    (git_dir / "info" / "exclude").write_text("graphify-out/\n")
+    # Minimal HEAD so _git_info_exclude recognises this as a git dir.
+    (git_dir / "HEAD").write_text("ref: refs/heads/main\n")
+
+    result = detect(tmp_path)
+    all_files = [f for files in result["files"].values() for f in files]
+    memory_hits = [f for f in all_files if "graphify-out" in f.replace("\\", "/")]
+    assert memory_hits, \
+        "memory files must stay included even when graphify-out/ is in info/exclude"
+
+
+def test_memory_cache_not_poisoned_by_non_memory_eval(tmp_path):
+    """#2267: _is_ignored cache must not collide across source filters.
+
+    A non-memory file under ``graphify-out/`` is evaluated with ALL ignore
+    sources (including ``.gitignore``) and cached as ignored. When a memory
+    file under the same ``graphify-out/`` ancestor is later evaluated with
+    ``sources={'graphifyignore'}``, the ancestor walk must NOT reuse the
+    cached non-graphifyignore result — otherwise memory files are silently
+    dropped. The cache key must differentiate by source filter.
+    """
+    (tmp_path / "src").mkdir()
+    (tmp_path / "src" / "calc.py").write_text("def add(a, b):\n    return a + b\n")
+
+    # graphify-out/ in .gitignore
+    (tmp_path / ".gitignore").write_text("graphify-out/\n")
+
+    # Seed memory + output
+    detect(tmp_path)
+
+    memory = tmp_path / "graphify-out" / "memory"
+    memory.mkdir(parents=True, exist_ok=True)
+    (memory / "query_1.md").write_text("# Query\nThe calc module adds numbers.\n")
+    # A non-memory file under graphify-out/ that .gitignore will catch
+    (tmp_path / "graphify-out" / "output.json").write_text('{"k": 1}\n')
+
+    result = detect(tmp_path)
+    all_files = [f for files in result["files"].values() for f in files]
+    memory_hits = [f for f in all_files if "graphify-out" in f.replace("\\", "/")]
+    assert memory_hits, \
+        "cache collision: memory files dropped because non-memory eval poisoned the cache"