From e47399d69e41a2aeed738b7e40821dd5b22dcc46 Mon Sep 17 00:00:00 2001 From: HerenderKumar Date: Wed, 15 Jul 2026 19:47:07 +0530 Subject: [PATCH] fix(detect): let .graphifyinclude rescue generic keyword skips (#1225) The secrets keyword heuristic (stem ends in token/secret/credential/...) silently dropped benign files like derive-locked-tokens.md, and the .graphifyinclude allowlist could not override it: the include machinery was loaded in detect() but never consulted after dot-dirs became indexed by default, so the allowlist literally could not rescue anything. Wire the allowlist into the sensitive check as an explicit escape hatch: - A wildcard-free .graphifyinclude entry now rescues a file from the keyword stage. Glob entries deliberately do not count, and the specific patterns (secrets dirs, .env, key files) are never overridden, so a broad * cannot start ingesting api_token.txt. - Keyword skips are annotated in skipped_sensitive and surfaced once per scan on stderr, naming the escape hatch. - GRAPH_REPORT.md lists every skipped file by name (previously the report did not mention skips at all), so a swallowed file is greppable. --- graphify/detect.py | 67 ++++++++++++++++++++++++++++++++++++-------- graphify/report.py | 11 ++++++++ tests/test_detect.py | 60 +++++++++++++++++++++++++++++++++++++++ tests/test_report.py | 20 +++++++++++++ 4 files changed, 147 insertions(+), 11 deletions(-) diff --git a/graphify/detect.py b/graphify/detect.py index bc73fbde7..9944ee4cf 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -182,17 +182,25 @@ def _generic_keyword_hit(name: str) -> bool: _PAPER_SIGNAL_THRESHOLD = 3 # need at least this many signals to call it a paper -def _is_sensitive(path: Path) -> bool: - """Return True if this file likely contains secrets and should be skipped.""" +# Reasons returned by _sensitive_reason. Stages 1-2 match specific, near-certain +# secret locations; Stage 3 is a heuristic guess on the name, so it is the only +# stage an explicit .graphifyinclude entry may override (#1225). +_SENSITIVE_DIR = "dir" +_SENSITIVE_PATTERN = "pattern" +_SENSITIVE_KEYWORD = "keyword" + + +def _sensitive_reason(path: Path) -> str | None: + """Return which stage flags this file as likely containing secrets, or None.""" # Stage 1: any PARENT directory is a known secrets dir (parts[:-1] excludes # the filename itself so a root-level file named "credentials" is not falsely # skipped — the name patterns in Stage 2 handle the filename). if any(part in _SENSITIVE_DIRS for part in path.parts[:-1]): - return True + return _SENSITIVE_DIR # Stage 2: filename pattern match name = path.name if any(p.search(name) for p in _SENSITIVE_PATTERNS): - return True + return _SENSITIVE_PATTERN # Stage 3: generic keywords, only when load-bearing in the name. Do NOT let a # bare name keyword silently drop a genuine programming-language source file: # a .rb/.py named device_token or passwords_controller is a module, not a secret @@ -204,8 +212,13 @@ def _is_sensitive(path: Path) -> bool: if _generic_keyword_hit(name): ext = path.suffix.lower() is_source_code = classify_file(path) == FileType.CODE and ext not in _SECRET_PRONE_DATA_EXTS - return not is_source_code - return False + return None if is_source_code else _SENSITIVE_KEYWORD + return None + + +def _is_sensitive(path: Path) -> bool: + """Return True if this file likely contains secrets and should be skipped.""" + return _sensitive_reason(path) is not None def _looks_like_paper(path: Path) -> bool: @@ -994,8 +1007,10 @@ def _matches(rel: str, p: str, anchored: bool) -> bool: def _load_graphifyinclude(root: Path) -> list[tuple[Path, str]]: """Read .graphifyinclude allowlist patterns from root and ancestors. - Include patterns opt matching hidden files/dirs into traversal. Sensitive - files and hard-skipped noise directories are still excluded later. + An include entry states explicit user intent to graph a file: wildcard-free + entries rescue files from the generic secrets keyword heuristic (#1225). + The specific sensitive patterns (secrets dirs, .env, key files) and + hard-skipped noise directories are still always excluded. Uses the same VCS-root ceiling logic as _load_graphifyignore. """ root = root.resolve() @@ -1150,6 +1165,7 @@ def _wc(path: Path) -> int: return _cache.cached_word_count(path, root, count_words, cache_root=cache_root) skipped_sensitive: list[str] = [] + keyword_skipped: list[str] = [] unclassified: list[str] = [] ignore_patterns = _load_graphifyignore(root) ignore_cache: dict[Path, bool] = {} # shared across all _is_ignored calls in this scan @@ -1160,7 +1176,15 @@ def _wc(path: Path) -> int: line = _parse_gitignore_line(pat) if line: ignore_patterns.append((root, line)) - include_patterns = _load_graphifyinclude(root) + # Only wildcard-free .graphifyinclude entries may rescue a file from the + # secrets keyword heuristic: a broad glob written to opt hidden files into + # traversal must not silently start ingesting keyword-named credential + # stores like api_token.txt (#1225). + explicit_includes = [ + (anchor, pat) + for anchor, pat in _load_graphifyinclude(root) + if not any(ch in pat for ch in "*?[") + ] # Always include graphify-out/memory/ - query results filed back into the graph memory_dir = root / GRAPHIFY_OUT / "memory" @@ -1260,8 +1284,18 @@ def _on_walk_error(err: OSError) -> None: if not _resolves_under_root(p, root): skipped_sensitive.append(str(p) + " [symlink target outside scan root]") continue - if _is_sensitive(p): - skipped_sensitive.append(str(p)) + sensitive = _sensitive_reason(p) + if sensitive == _SENSITIVE_KEYWORD and _is_included(p, root, explicit_includes): + sensitive = None # explicit allowlist entry beats the name heuristic (#1225) + if sensitive is not None: + if sensitive == _SENSITIVE_KEYWORD: + keyword_skipped.append(str(p)) + skipped_sensitive.append( + str(p) + " [name matches a secrets keyword - add the exact " + "path to .graphifyinclude to include it]" + ) + else: + skipped_sensitive.append(str(p)) continue ftype = classify_file(p) if not ftype: @@ -1309,6 +1343,17 @@ def _on_walk_error(err: OSError) -> None: if ftype != FileType.VIDEO: total_words += _wc(p) + if keyword_skipped: + import sys as _sys + _names = ", ".join(Path(f).name for f in keyword_skipped[:6]) + _more = f" (+{len(keyword_skipped) - 6} more)" if len(keyword_skipped) > 6 else "" + print( + f"[graphify] {len(keyword_skipped)} file(s) skipped because their " + f"names match secrets keywords: {_names}{_more}. False positive? " + f"Add the exact path to .graphifyinclude to include it.", + file=_sys.stderr, + ) + for ftype in files: files[ftype].sort() diff --git a/graphify/report.py b/graphify/report.py index 248bce9a1..8be590b39 100644 --- a/graphify/report.py +++ b/graphify/report.py @@ -113,6 +113,17 @@ def generate( "- Verdict: corpus is large enough that graph structure adds value.", ] + # Every skipped file by name, not a bare count: one benign doc swallowed by + # the secrets heuristic must stay findable by grepping the report (#1225). + skipped = detection_result.get("skipped_sensitive") or [] + if skipped: + lines += [ + "", + "## Skipped Files", + f"{len(skipped)} file(s) were skipped during the scan and are absent from the graph:", + ] + lines += [f"- `{entry}`" for entry in skipped] + from .analyze import _is_file_node as _ifn non_empty = {cid: nodes for cid, nodes in communities.items() if any(not _ifn(G, n) for n in nodes)} diff --git a/tests/test_detect.py b/tests/test_detect.py index 374b2552f..2b3949fd7 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -1099,6 +1099,66 @@ def test_sensitive_flags_plural_tokens_txt(): assert _is_sensitive(Path("tokens.txt")) +# ── Issue #1225: .graphifyinclude rescues generic keyword skips ─────────────── +# The keyword stage is the only heuristic guess; an exact .graphifyinclude entry +# states user intent and overrides it. The specific credential patterns and +# secrets dirs are never overridden, and glob entries don't count as explicit. + +def test_include_rescues_keyword_skipped_doc(tmp_path): + docs = tmp_path / "docs" + docs.mkdir() + doc = docs / "derive-locked-tokens.md" + doc.write_text("A prompt template about locking proper-noun tokens.\n" * 20) + (tmp_path / ".graphifyinclude").write_text("docs/derive-locked-tokens.md\n") + + result = detect(tmp_path) + + assert any("derive-locked-tokens.md" in f for f in result["files"]["document"]) + assert not any("derive-locked-tokens" in item for item in result["skipped_sensitive"]) + + +def test_keyword_skip_is_annotated_and_names_escape_hatch(tmp_path, capsys): + doc = tmp_path / "deploy-tokens.md" + doc.write_text("Runbook for the deploy token rotation schedule.\n" * 20) + + result = detect(tmp_path) + + assert not any("deploy-tokens.md" in f for f in result["files"]["document"]) + assert any( + "deploy-tokens.md" in item and ".graphifyinclude" in item + for item in result["skipped_sensitive"] + ) + err = capsys.readouterr().err + assert "deploy-tokens.md" in err + assert ".graphifyinclude" in err + + +def test_include_rescue_requires_exact_path_not_glob(tmp_path): + doc = tmp_path / "deploy-tokens.md" + doc.write_text("Runbook for the deploy token rotation schedule.\n" * 20) + (tmp_path / ".graphifyinclude").write_text("*\n") + + result = detect(tmp_path) + + assert not any("deploy-tokens.md" in f for f in result["files"]["document"]) + assert any("deploy-tokens.md" in item for item in result["skipped_sensitive"]) + + +def test_include_cannot_rescue_credential_stores(tmp_path): + secrets = tmp_path / "secrets" + secrets.mkdir() + (secrets / "db.json").write_text('{"password": "hunter2"}') + (tmp_path / "server.pem").write_text("-----BEGIN PRIVATE KEY-----\n") + (tmp_path / ".graphifyinclude").write_text("secrets/db.json\nserver.pem\n") + + result = detect(tmp_path) + + assert any("db.json" in item for item in result["skipped_sensitive"]) + assert any("server.pem" in item for item in result["skipped_sensitive"]) + for flist in result["files"].values(): + assert not any("db.json" in f or "server.pem" in f for f in flist) + + # ── Issue #933: failed-chunk files must not be frozen in manifest ───────────── def test_save_manifest_skips_semantic_hash_for_files_without_cache(tmp_path): diff --git a/tests/test_report.py b/tests/test_report.py index 767e2ba34..bfaddaf57 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -154,3 +154,23 @@ def test_report_hubs_use_wikilinks_when_obsidian(): labels = {cid: f"Widget {cid}" for cid in communities} report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project", min_community_size=1, obsidian=True) assert "[[_COMMUNITY_" in report + + +def test_report_lists_skipped_sensitive_files(): + # #1225: skipped files must be listed by name, not reduced to a count, so a + # silently dropped file is findable by grepping the report. + G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs() + detection["skipped_sensitive"] = [ + "/proj/references/prompts/derive-locked-tokens.md [name matches a secrets keyword]", + "/proj/.ssh/id_rsa", + ] + report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project") + assert "## Skipped Files" in report + assert "derive-locked-tokens.md" in report + assert "id_rsa" in report + + +def test_report_omits_skipped_section_when_nothing_skipped(): + G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs() + report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project") + assert "## Skipped Files" not in report