mirror of
https://github.com/Graphify-Labs/graphify.git
synced 2026-09-14 19:34:09 +08:00
krishnateja7
5777fec8c1
.rake files are plain Ruby (Rake's task DSL is ordinary method calls), but the extension was gated out everywhere, so rake tasks were classified as unsupported, skipped, and their calls invisible. Add `.rake` to all seven `.rb` gates the reporter mapped: - detect.CODE_EXTENSIONS (classification) - extract._DISPATCH (extractor dispatch) - extract._LANG_FAMILY_BY_EXT-adjacent language-name map (.rake -> ruby) - the ruby_member_calls LanguageResolver suffix set - both `.rb`-suffix filters in ruby_resolution.py (raw-call gather + class-def index) - analyze language-stats map - build repo-tag map The extractor already parsed the content; this is purely extension routing. Regression test: a `.rake` task's `Widget.tally` resolves cross-file to the `.rb` definition. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
4942 lines
214 KiB
Python
4942 lines
214 KiB
Python
"""Deterministic structural extraction from source code using tree-sitter. Outputs nodes+edges dicts."""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import importlib
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
from .cache import load_cached, save_cached
|
|
from .mcp_ingest import extract_mcp_config, is_mcp_config_path
|
|
from .manifest_ingest import extract_package_manifest, is_package_manifest_path
|
|
from .resolver_registry import (
|
|
LanguageResolver,
|
|
register as register_language_resolver,
|
|
run_language_resolvers,
|
|
)
|
|
from .ruby_resolution import resolve_ruby_member_calls
|
|
from .pascal_resolution import resolve_pascal_inherited_calls
|
|
|
|
# --- migrated to graphify/extractors/ (see graphify/extractors/MIGRATION.md) ---
|
|
from graphify.extractors.base import ( # noqa: F401
|
|
_LANGUAGE_BUILTIN_GLOBALS,
|
|
_file_stem,
|
|
_make_id,
|
|
_read_text,
|
|
)
|
|
from graphify.extractors.apex import extract_apex # noqa: F401
|
|
from graphify.extractors.bash import extract_bash # noqa: F401
|
|
from graphify.extractors.blade import extract_blade # noqa: F401
|
|
from graphify.extractors.csharp import (
|
|
_resolve_cross_file_csharp_imports,
|
|
_resolve_csharp_type_references,
|
|
)
|
|
from graphify.extractors.dart import extract_dart # noqa: F401
|
|
from graphify.extractors.dm import extract_dm, extract_dmf, extract_dmi, extract_dmm # noqa: F401
|
|
from graphify.extractors.elixir import extract_elixir # noqa: F401
|
|
from graphify.extractors.fortran import _cpp_preprocess, extract_fortran # noqa: F401
|
|
from graphify.extractors.go import extract_go # noqa: F401
|
|
from graphify.extractors.json_config import extract_json # noqa: F401
|
|
from graphify.extractors.markdown import extract_markdown # noqa: F401
|
|
from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form # noqa: F401
|
|
from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest # noqa: F401
|
|
from graphify.extractors.razor import extract_razor # noqa: F401
|
|
from graphify.extractors.rust import extract_rust # noqa: F401
|
|
from graphify.extractors.sln import extract_sln # noqa: F401
|
|
from graphify.extractors.sql import extract_sql # noqa: F401
|
|
from graphify.extractors.terraform import extract_terraform # noqa: F401
|
|
from graphify.extractors.verilog import extract_verilog # noqa: F401
|
|
from graphify.extractors.zig import extract_zig # noqa: F401
|
|
from graphify.security import sanitize_metadata
|
|
from graphify.paths import disambiguate_ambiguous_candidates
|
|
|
|
from graphify.extractors.models import LanguageConfig, _JS_CACHE_BYPASS_SUFFIXES, _NamespaceExportFact, _StarExportFact, _SymbolAliasFact, _SymbolDeclarationFact, _SymbolExportFact, _SymbolImportFact, _SymbolResolutionFacts, _SymbolUseFact, _WORKSPACE_PACKAGE_CACHE # noqa: E402,F401
|
|
|
|
from graphify.extractors.resolution import ( # noqa: E402,F401
|
|
_DECLDEF_HEADER_SUFFIXES,
|
|
_DECLDEF_IMPL_SUFFIXES,
|
|
_EXPORT_CONDITION_PRIORITY,
|
|
_JS_INDEX_FILES,
|
|
_JS_PRIMITIVE_TYPES,
|
|
_JS_RESOLVE_EXTS,
|
|
_TSCONFIG_ALIAS_CACHE,
|
|
_VUE_SCRIPT_LANG_RE,
|
|
_VUE_SCRIPT_RE,
|
|
_WORKSPACE_MANIFEST_NAMES,
|
|
_apply_symbol_resolution_facts,
|
|
_augment_symbol_resolution_edges,
|
|
_collect_js_symbol_resolution_facts,
|
|
_collect_python_symbol_resolution_facts,
|
|
_contained_in_package,
|
|
_decldef_class_stem,
|
|
_disambiguate_colliding_node_ids,
|
|
_find_workspace_root,
|
|
_is_type_like_definition,
|
|
_js_call_identifier,
|
|
_js_default_export_name,
|
|
_js_default_import_name,
|
|
_js_export_clause,
|
|
_js_export_statement_is_star,
|
|
_js_exported_declaration_names,
|
|
_js_lexical_aliases,
|
|
_js_module_specifier,
|
|
_js_named_specifiers,
|
|
_js_namespace_export_name,
|
|
_js_source_path,
|
|
_js_top_level_function_bodies,
|
|
_load_tsconfig_aliases,
|
|
_load_workspace_packages,
|
|
_match_tsconfig_alias,
|
|
_merge_decl_def_classes,
|
|
_node_disambiguation_source_key,
|
|
_package_entry_candidates,
|
|
_parse_js_tree,
|
|
_parse_python_tree,
|
|
_pascal_class_stem_cache,
|
|
_pascal_project_root,
|
|
_pascal_resolve_class,
|
|
_pascal_resolve_unit,
|
|
_pascal_unit_cache,
|
|
_pnpm_workspace_globs,
|
|
_python_call_identifier,
|
|
_python_import_from_module,
|
|
_python_imported_names,
|
|
_python_top_level_function_bodies,
|
|
_read_tsconfig_aliases,
|
|
_resolve_c_include_path,
|
|
_resolve_cross_file_imports,
|
|
_resolve_cross_file_java_imports,
|
|
_resolve_export_target,
|
|
_resolve_java_type_references,
|
|
_resolve_js_import_path,
|
|
_resolve_js_import_target,
|
|
_resolve_js_module_path,
|
|
_resolve_lua_import_target,
|
|
_resolve_python_module_path,
|
|
_resolve_tsconfig_alias,
|
|
_resolve_workspace_import,
|
|
_source_key,
|
|
_strip_jsonc,
|
|
_ts_collect_type_refs,
|
|
_ts_heritage_clause_entries,
|
|
_ts_walk_class_members,
|
|
_vue_mask_non_script,
|
|
_walk_js_tree,
|
|
_walk_python_tree,
|
|
_workspace_globs,
|
|
)
|
|
|
|
from graphify.extractors.engine import REFERENCE_CONTEXTS, _CSHARP_TYPE_PARAMETER_SCOPE_DECLARATIONS, _C_PRIMITIVE_TYPE_NODES, _JAVA_BUILTIN_TYPES, _JAVA_TYPE_PARAMETER_SCOPE_DECLARATIONS, _JS_FUNCTION_VALUE_TYPES, _JS_SCOPE_BOUNDARY, _PYTHON_ANNOTATION_NOISE, _PYTHON_TYPE_CONTAINERS, _RUBY_CLASS_FACTORIES, _c_collect_type_refs, _cpp_collect_type_refs, _cpp_declarator_name, _cpp_local_var_types, _csharp_attribute_names, _csharp_classify_base, _csharp_collect_type_refs, _csharp_extra_walk, _csharp_member_type_table, _csharp_namespace_id, _csharp_namespace_name, _csharp_pre_scan_interfaces, _csharp_type_parameters_in_scope, _dynamic_import_js, _extract_generic, _find_body, _find_require_call, _get_cpp_func_name, _java_annotation_names, _java_collect_type_refs, _java_extra_walk, _java_type_parameters_in_scope, _js_collect_pattern_idents, _js_dispatch_value_idents, _js_extra_walk, _js_local_bound_names, _js_member_assignment_target, _js_module_bound_names, _kotlin_collect_type_refs, _kotlin_function_return_type_node, _kotlin_property_type_node, _kotlin_user_type_name, _php_collect_type_refs, _php_method_return_type_node, _php_name_text, _python_collect_assignment_targets, _python_collect_param_refs, _python_collect_type_refs, _python_local_bound_names, _python_module_bound_names, _python_param_names, _read_csharp_type_name, _require_imports_js, _ruby_const_last_name, _ruby_extra_walk, _ruby_local_class_bindings, _ruby_new_class_name, _scala_collect_type_refs, _semantic_reference_edge, _source_location, _swift_classify_base, _swift_collect_type_refs, _swift_constructor_type, _swift_declaration_keyword, _swift_extra_walk, _swift_local_var_types, _swift_pre_scan, _swift_property_name, _swift_property_type_node, _swift_receiver_name, _swift_user_type_name, _ts_decorator_name, _ts_descendant_decorators, _ts_emit_decorator_edges, _ts_extra_walk, _ts_method_name, _ts_receiver_type_table # noqa: E402,F401
|
|
|
|
from graphify.extractors.pascal import _PAS_BEGIN_END_TOKEN_RE, _PAS_CALL_RE, _PAS_END_SEMI_RE, _PAS_IMPL_HEADER_RE, _PAS_KEYWORDS, _PAS_METHOD_DECL_RE, _PAS_MODULE_RE, _PAS_TOKEN_RE, _PAS_TYPE_HEADER_RE, _PAS_USES_RE, _extract_pascal_regex, _pascal_find_body, _pascal_split_bases, _pascal_split_sections, _pascal_split_uses, _pascal_strip_comments, extract_pascal # noqa: E402,F401
|
|
|
|
from graphify.extractors.objc import _objc_local_var_types, extract_objc # noqa: E402,F401
|
|
|
|
from graphify.extractors.julia import extract_julia # noqa: E402,F401
|
|
|
|
_RECURSION_LIMIT = 10_000
|
|
|
|
# Language built-in globals that AST may classify as call targets when used as
|
|
# constructors or coercion functions (e.g. String(x), Number(x), Boolean(x)).
|
|
# Without this filter they become god-nodes accumulating spurious edges from
|
|
# every call site. Filter applied at same-file and cross-file resolution.
|
|
# See issue #726.
|
|
|
|
|
|
def _raise_recursion_limit() -> None:
|
|
if sys.getrecursionlimit() < _RECURSION_LIMIT:
|
|
sys.setrecursionlimit(_RECURSION_LIMIT)
|
|
|
|
|
|
def _safe_extract(extractor: Callable, path: Path) -> dict:
|
|
try:
|
|
return extractor(path)
|
|
except RecursionError:
|
|
print(f" warning: skipped {path} (recursion limit exceeded)", file=sys.stderr, flush=True)
|
|
return {"nodes": [], "edges": [], "error": "recursion_limit_exceeded"}
|
|
except Exception as e:
|
|
if os.environ.get("GRAPHIFY_DEBUG"):
|
|
import traceback
|
|
traceback.print_exc(file=sys.stderr)
|
|
print(f" warning: skipped {path} ({type(e).__name__}: {e})", file=sys.stderr, flush=True)
|
|
return {"nodes": [], "edges": [], "error": f"{type(e).__name__}: {e}"}
|
|
|
|
|
|
def _file_node_id(rel_path: Path) -> str:
|
|
"""File-level node ID matching the skill.md spec: ``{parent_dir}_{stem}`` —
|
|
one parent directory level, no extension. ``rel_path`` MUST be relative to
|
|
the project root so top-level files collapse to a bare stem (``setup.py`` ->
|
|
``setup``) instead of picking up the root directory name. This must equal the
|
|
ID semantic subagents generate, or AST and semantic extraction split a file
|
|
into two disconnected ghost nodes (#1033)."""
|
|
return _make_id(_file_stem(rel_path))
|
|
|
|
|
|
SEMANTIC_RELATIONS = frozenset({
|
|
"inherits", "implements", "mixes_in", "embeds", "references",
|
|
"calls", "imports", "imports_from", "re_exports", "contains", "method",
|
|
})
|
|
|
|
|
|
# Condition keys consulted when resolving an `exports` target, in priority
|
|
# order. `default` is Node's catch-all and must be consulted LAST so a more
|
|
# specific condition (source/import/module/etc.) wins when several match.
|
|
|
|
|
|
# ── LanguageConfig dataclass ─────────────────────────────────────────────────
|
|
|
|
|
|
# ── Generic helpers ───────────────────────────────────────────────────────────
|
|
|
|
|
|
# Scalar builtins and test-mock names that appear as type annotations but carry
|
|
# no useful semantic meaning as graph nodes (#1147). Suppressed at the annotation
|
|
# walker level so they are never created as nodes or emitted as edges.
|
|
|
|
|
|
# java.lang (auto-imported) plus the ubiquitous java.util / java.io / java.time /
|
|
# java.util.{stream,function,concurrent} / java.math / java.nio.file types that
|
|
# appear as field, parameter, return, and generic-argument annotations. They never
|
|
# resolve to a project node, so emitting `references` edges to them is pure noise
|
|
# (mirrors _GO_PREDECLARED_TYPES / _PYTHON_ANNOTATION_NOISE). Suppressed at the
|
|
# type-ref walker so they are never created as nodes or emitted as edges. The
|
|
# boxed-scalar/`void` primitives are already dropped by grammar node type above;
|
|
# these are the class/interface names the grammar reports as identifiers.
|
|
|
|
|
|
# ── C / C++ type-ref helpers ─────────────────────────────────────────────────
|
|
|
|
|
|
# ── Scala type-ref helpers ───────────────────────────────────────────────────
|
|
|
|
|
|
def _resolve_name(node, source: bytes, config: LanguageConfig) -> str | None:
|
|
"""Get the name from a node using config.name_field, falling back to child types."""
|
|
if config.resolve_function_name_fn is not None:
|
|
# For C/C++ where the name is inside a declarator
|
|
return None # caller handles this separately
|
|
n = node.child_by_field_name(config.name_field)
|
|
if n:
|
|
return _read_text(n, source)
|
|
for child in node.children:
|
|
if child.type in config.name_fallback_child_types:
|
|
return _read_text(child, source)
|
|
return None
|
|
|
|
|
|
# ── Import handlers ───────────────────────────────────────────────────────────
|
|
|
|
def _import_python(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None:
|
|
t = node.type
|
|
if t == "import_statement":
|
|
for child in node.children:
|
|
if child.type in ("dotted_name", "aliased_import"):
|
|
raw = _read_text(child, source)
|
|
module_name = raw.split(" as ")[0].strip().lstrip(".")
|
|
tgt_nid = _make_id(module_name)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
elif t == "import_from_statement":
|
|
module_node = node.child_by_field_name("module_name")
|
|
if module_node:
|
|
raw = _read_text(module_node, source)
|
|
if raw.startswith("."):
|
|
# Relative import - resolve to full path so IDs match file node IDs
|
|
dots = len(raw) - len(raw.lstrip("."))
|
|
module_name = raw.lstrip(".")
|
|
base = Path(str_path).parent
|
|
for _ in range(dots - 1):
|
|
base = base.parent
|
|
rel = (module_name.replace(".", "/") + ".py") if module_name else "__init__.py"
|
|
tgt_nid = _make_id(str(base / rel))
|
|
else:
|
|
tgt_nid = _make_id(raw)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports_from",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
|
|
def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None:
|
|
is_reexport = node.type == "export_statement"
|
|
# Only handle export_statement if it has a `from` clause (re-export).
|
|
# Pure exports like `export const x = 1` or `export { localVar }` have no source module.
|
|
if is_reexport:
|
|
has_from = any(child.type == "from" or (_read_text(child, source) == "from") for child in node.children if child.type in ("from", "identifier"))
|
|
if not has_from:
|
|
# Check for string child (source path) as a more reliable indicator
|
|
has_from = any(child.type == "string" for child in node.children)
|
|
if not has_from:
|
|
return
|
|
|
|
resolved_path: "Path | None" = None
|
|
module_string = None
|
|
for child in node.children:
|
|
if child.type == "string":
|
|
module_string = child
|
|
break
|
|
if child.type == "import_require_clause":
|
|
# TS import-equals form: `import x = require("./m")`. The module
|
|
# string sits inside the clause, not on the import_statement
|
|
# itself, so the direct-child scan above never sees it.
|
|
module_string = next(
|
|
(sub for sub in child.children if sub.type == "string"), None
|
|
)
|
|
break
|
|
if module_string is not None:
|
|
raw = _read_text(module_string, source).strip("'\"` ")
|
|
resolved = _resolve_js_import_target(raw, str_path)
|
|
if resolved is not None:
|
|
tgt_nid, resolved_path = resolved
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports_from",
|
|
"context": "re-export" if is_reexport else "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
# Emit symbol-level edges for named imports/re-exports from local/aliased files.
|
|
# e.g. `import { Foo, type Bar } from './bar'` → file → Foo, file → Bar (EXTRACTED)
|
|
# e.g. `export { Foo } from './bar'` → file → Foo (re_exports edge)
|
|
# Uses the same _make_id(target_stem, name) key that _extract_generic emits when
|
|
# defining the symbol, so these edges wire importers directly to existing symbol nodes.
|
|
if resolved_path is not None:
|
|
target_stem = _file_stem(resolved_path)
|
|
line = node.start_point[0] + 1
|
|
|
|
if is_reexport:
|
|
# Handle: export { foo, bar } from './module'
|
|
# export { default as baz } from './module'
|
|
for child in node.children:
|
|
if child.type == "export_clause":
|
|
for spec in child.children:
|
|
if spec.type == "export_specifier":
|
|
# The exported name is the local name from the source module
|
|
name_node = spec.child_by_field_name("name")
|
|
if name_node:
|
|
sym = _read_text(name_node, source)
|
|
if sym == "default":
|
|
continue # skip default re-exports for ID matching
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": _make_id(target_stem, sym),
|
|
"relation": "re_exports",
|
|
"context": "re-export",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
else:
|
|
# Handle: import { Foo, type Bar } from './bar'
|
|
for child in node.children:
|
|
if child.type == "import_clause":
|
|
for sub in child.children:
|
|
if sub.type == "named_imports":
|
|
for spec in sub.children:
|
|
if spec.type == "import_specifier":
|
|
name_node = spec.child_by_field_name("name")
|
|
if name_node:
|
|
sym = _read_text(name_node, source)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": _make_id(target_stem, sym),
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
|
|
def _import_java(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None:
|
|
def _walk_scoped(n) -> str:
|
|
parts: list[str] = []
|
|
cur = n
|
|
while cur:
|
|
if cur.type == "scoped_identifier":
|
|
name_node = cur.child_by_field_name("name")
|
|
if name_node:
|
|
parts.append(_read_text(name_node, source))
|
|
cur = cur.child_by_field_name("scope")
|
|
elif cur.type == "identifier":
|
|
parts.append(_read_text(cur, source))
|
|
break
|
|
else:
|
|
break
|
|
parts.reverse()
|
|
return ".".join(parts)
|
|
|
|
for child in node.children:
|
|
if child.type in ("scoped_identifier", "identifier"):
|
|
path_str = _walk_scoped(child)
|
|
module_name = path_str.split(".")[-1].strip("*").strip(".") or (
|
|
path_str.split(".")[-2] if len(path_str.split(".")) > 1 else path_str
|
|
)
|
|
if module_name:
|
|
tgt_nid = _make_id(module_name)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
|
|
|
|
def _import_c(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None:
|
|
for child in node.children:
|
|
if child.type in ("string_literal", "system_lib_string", "string"):
|
|
raw = _read_text(child, source).strip('"<> ')
|
|
# Quoted includes: try to resolve to a real file so the target ID
|
|
# matches the node ID _extract_generic creates for that file.
|
|
if child.type != "system_lib_string":
|
|
resolved = _resolve_c_include_path(raw, str_path)
|
|
if resolved is not None:
|
|
tgt_nid = _make_id(str(resolved))
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
module_name = raw.split("/")[-1].split(".")[0]
|
|
if module_name:
|
|
tgt_nid = _make_id(module_name)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
|
|
|
|
def _import_csharp(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None:
|
|
text = _read_text(node, source).strip().rstrip(";")
|
|
if text.startswith("global "):
|
|
text = text[len("global "):].strip()
|
|
if not text.startswith("using"):
|
|
return
|
|
body = text[len("using"):].strip()
|
|
using_kind, alias, target_fqn = "namespace", None, body
|
|
if body.startswith("static "):
|
|
using_kind, target_fqn = "static", body[len("static "):].strip()
|
|
elif "=" in body:
|
|
lhs, rhs = body.split("=", 1)
|
|
using_kind, alias, target_fqn = "alias", lhs.strip(), rhs.strip()
|
|
if not target_fqn:
|
|
return
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": _make_id(target_fqn),
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
"metadata": sanitize_metadata({k: v for k, v in
|
|
{"using_kind": using_kind, "alias": alias, "target_fqn": target_fqn,
|
|
"scope_kind": "namespace" if scope_stack else "file",
|
|
"scope_id": scope_stack[-1] if scope_stack else None}.items() if v is not None}),
|
|
})
|
|
|
|
|
|
def _import_kotlin(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None:
|
|
path_node = node.child_by_field_name("path")
|
|
if path_node:
|
|
raw = _read_text(path_node, source)
|
|
module_name = raw.split(".")[-1].strip()
|
|
if module_name:
|
|
tgt_nid = _make_id(module_name)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
return
|
|
# Fallback: find identifier child
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
raw = _read_text(child, source)
|
|
tgt_nid = _make_id(raw)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
|
|
|
|
def _import_scala(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None:
|
|
for child in node.children:
|
|
if child.type in ("stable_id", "identifier"):
|
|
raw = _read_text(child, source)
|
|
module_name = raw.split(".")[-1].strip("{} ")
|
|
if module_name and module_name != "_":
|
|
tgt_nid = _make_id(module_name)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
|
|
|
|
def _import_php(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None:
|
|
for child in node.children:
|
|
if child.type in ("qualified_name", "name", "identifier"):
|
|
raw = _read_text(child, source)
|
|
module_name = raw.split("\\")[-1].strip()
|
|
if module_name:
|
|
tgt_nid = _make_id(module_name)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
|
|
|
|
# ── C/C++ function name helpers ───────────────────────────────────────────────
|
|
|
|
def _get_c_func_name(node, source: bytes) -> str | None:
|
|
"""Recursively unwrap declarator to find the innermost identifier (C)."""
|
|
if node.type == "identifier":
|
|
return _read_text(node, source)
|
|
decl = node.child_by_field_name("declarator")
|
|
if decl:
|
|
return _get_c_func_name(decl, source)
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
return _read_text(child, source)
|
|
return None
|
|
|
|
|
|
# ── JS/TS extra walk for arrow functions ──────────────────────────────────────
|
|
|
|
|
|
# Node types whose value is a callable, for the JS/TS assignment / class-field
|
|
# / function-expression forms below. Older tree-sitter-javascript grammars
|
|
# label a function expression `function`; current ones use `function_expression`.
|
|
|
|
|
|
# ── TS extra walk for namespace / module declarations ─────────────────────────
|
|
|
|
|
|
# ── C# extra walk for namespace declarations ──────────────────────────────────
|
|
|
|
|
|
# ── Swift extra walk for enum cases ──────────────────────────────────────────
|
|
|
|
|
|
# ── Java extra walk for enum constants ───────────────────────────────────────
|
|
|
|
|
|
# ── Language configs ──────────────────────────────────────────────────────────
|
|
|
|
_PYTHON_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_python",
|
|
class_types=frozenset({"class_definition"}),
|
|
function_types=frozenset({"function_definition"}),
|
|
import_types=frozenset({"import_statement", "import_from_statement"}),
|
|
call_types=frozenset({"call"}),
|
|
call_function_field="function",
|
|
call_accessor_node_types=frozenset({"attribute"}),
|
|
call_accessor_field="attribute",
|
|
call_accessor_object_field="object",
|
|
function_boundary_types=frozenset({"function_definition"}),
|
|
import_handler=_import_python,
|
|
)
|
|
|
|
_JS_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_javascript",
|
|
class_types=frozenset({"class_declaration"}),
|
|
function_types=frozenset({"function_declaration", "generator_function_declaration", "method_definition"}),
|
|
import_types=frozenset({"import_statement", "export_statement"}),
|
|
call_types=frozenset({"call_expression", "new_expression"}),
|
|
call_function_field="function",
|
|
call_accessor_node_types=frozenset({"member_expression"}),
|
|
call_accessor_field="property",
|
|
call_accessor_object_field="object",
|
|
function_boundary_types=frozenset({"function_declaration", "generator_function_declaration", "arrow_function", "method_definition"}),
|
|
import_handler=_import_js,
|
|
)
|
|
|
|
_TS_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_typescript",
|
|
ts_language_fn="language_typescript",
|
|
class_types=frozenset({
|
|
"class_declaration",
|
|
"abstract_class_declaration", # TS abstract class
|
|
"interface_declaration", # parity with Java/C#
|
|
"enum_declaration", # named enums
|
|
"type_alias_declaration", # named type aliases
|
|
}),
|
|
function_types=frozenset({"function_declaration", "generator_function_declaration", "method_definition", "method_signature"}),
|
|
import_types=frozenset({"import_statement", "export_statement"}),
|
|
call_types=frozenset({"call_expression", "new_expression"}),
|
|
call_function_field="function",
|
|
call_accessor_node_types=frozenset({"member_expression"}),
|
|
call_accessor_field="property",
|
|
call_accessor_object_field="object",
|
|
function_boundary_types=frozenset({"function_declaration", "generator_function_declaration", "arrow_function", "method_definition"}),
|
|
import_handler=_import_js,
|
|
)
|
|
|
|
# .tsx files must use the TSX grammar (JSX-aware), not the plain TypeScript grammar.
|
|
# tree-sitter-typescript ships two languages: language_typescript (for .ts) and
|
|
# language_tsx (for .tsx). Parsing .tsx with language_typescript silently fails on
|
|
# JSX expressions, dropping any call_expression nested inside JSX (e.g. {fmtDate(x)}).
|
|
_TSX_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_typescript",
|
|
ts_language_fn="language_tsx",
|
|
class_types=_TS_CONFIG.class_types,
|
|
function_types=_TS_CONFIG.function_types,
|
|
import_types=_TS_CONFIG.import_types,
|
|
call_types=_TS_CONFIG.call_types,
|
|
call_function_field=_TS_CONFIG.call_function_field,
|
|
call_accessor_node_types=_TS_CONFIG.call_accessor_node_types,
|
|
call_accessor_field=_TS_CONFIG.call_accessor_field,
|
|
call_accessor_object_field=_TS_CONFIG.call_accessor_object_field,
|
|
function_boundary_types=_TS_CONFIG.function_boundary_types,
|
|
import_handler=_TS_CONFIG.import_handler,
|
|
)
|
|
|
|
_JAVA_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_java",
|
|
# record_declaration shares class_declaration's name/body/interfaces fields,
|
|
# so it becomes a first-class type node instead of an isolated file (#1373).
|
|
# Enums and annotation declarations use the same name/body contract.
|
|
class_types=frozenset({
|
|
"class_declaration", "interface_declaration", "record_declaration",
|
|
"enum_declaration", "annotation_type_declaration",
|
|
}),
|
|
function_types=frozenset({"method_declaration", "constructor_declaration"}),
|
|
import_types=frozenset({"import_declaration"}),
|
|
# object_creation_expression (`new Foo(...)`) is handled by a dedicated Java
|
|
# branch in walk_calls below — its callee is in the `type` field, not `name`.
|
|
call_types=frozenset({"method_invocation", "object_creation_expression"}),
|
|
call_function_field="name",
|
|
call_accessor_node_types=frozenset(),
|
|
function_boundary_types=frozenset({"method_declaration", "constructor_declaration"}),
|
|
import_handler=_import_java,
|
|
)
|
|
|
|
_GROOVY_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_groovy",
|
|
class_types=frozenset({"class_declaration", "interface_declaration"}),
|
|
function_types=frozenset({"method_declaration", "constructor_declaration"}),
|
|
import_types=frozenset({"import_declaration"}),
|
|
call_types=frozenset({"method_invocation"}),
|
|
call_function_field="name",
|
|
call_accessor_node_types=frozenset(),
|
|
function_boundary_types=frozenset({"method_declaration", "constructor_declaration"}),
|
|
import_handler=_import_java,
|
|
)
|
|
|
|
_C_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_c",
|
|
class_types=frozenset(),
|
|
function_types=frozenset({"function_definition"}),
|
|
import_types=frozenset({"preproc_include"}),
|
|
call_types=frozenset({"call_expression"}),
|
|
call_function_field="function",
|
|
call_accessor_node_types=frozenset({"field_expression"}),
|
|
call_accessor_field="field",
|
|
function_boundary_types=frozenset({"function_definition"}),
|
|
import_handler=_import_c,
|
|
resolve_function_name_fn=_get_c_func_name,
|
|
)
|
|
|
|
_CPP_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_cpp",
|
|
class_types=frozenset({"class_specifier", "struct_specifier"}),
|
|
function_types=frozenset({"function_definition"}),
|
|
import_types=frozenset({"preproc_include"}),
|
|
call_types=frozenset({"call_expression"}),
|
|
call_function_field="function",
|
|
call_accessor_node_types=frozenset({"field_expression", "qualified_identifier"}),
|
|
call_accessor_field="field",
|
|
function_boundary_types=frozenset({"function_definition"}),
|
|
import_handler=_import_c,
|
|
resolve_function_name_fn=_get_cpp_func_name,
|
|
)
|
|
|
|
_RUBY_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_ruby",
|
|
# `module Foo` is a container node just like `class Foo` in tree-sitter's
|
|
# Ruby grammar (name in a `constant` child, body in `body_statement`), so it
|
|
# gets a node and its methods attach via `method` (#1640). Without it, plain
|
|
# utility/`module_function` modules produced no node and their methods hung
|
|
# off the file via `contains` with dot-less labels.
|
|
class_types=frozenset({"class", "module"}),
|
|
function_types=frozenset({"method", "singleton_method"}),
|
|
import_types=frozenset(),
|
|
call_types=frozenset({"call"}),
|
|
call_function_field="method",
|
|
call_accessor_node_types=frozenset(),
|
|
name_fallback_child_types=("constant", "scope_resolution", "identifier"),
|
|
body_fallback_child_types=("body_statement",),
|
|
function_boundary_types=frozenset({"method", "singleton_method"}),
|
|
)
|
|
|
|
_CSHARP_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_c_sharp",
|
|
class_types=frozenset({
|
|
"class_declaration",
|
|
"interface_declaration",
|
|
"enum_declaration",
|
|
"struct_declaration",
|
|
"record_declaration",
|
|
}),
|
|
function_types=frozenset({"method_declaration"}),
|
|
import_types=frozenset({"using_directive"}),
|
|
call_types=frozenset({"invocation_expression"}),
|
|
call_function_field="function",
|
|
call_accessor_node_types=frozenset({"member_access_expression"}),
|
|
call_accessor_field="name",
|
|
body_fallback_child_types=("declaration_list",),
|
|
function_boundary_types=frozenset({"method_declaration"}),
|
|
import_handler=_import_csharp,
|
|
)
|
|
|
|
_KOTLIN_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_kotlin",
|
|
class_types=frozenset({"class_declaration", "object_declaration"}),
|
|
function_types=frozenset({"function_declaration"}),
|
|
import_types=frozenset({"import_header"}),
|
|
call_types=frozenset({"call_expression"}),
|
|
call_function_field="",
|
|
call_accessor_node_types=frozenset({"navigation_expression"}),
|
|
call_accessor_field="",
|
|
# Different tree-sitter-kotlin grammar versions name plain identifier
|
|
# nodes differently: PyPI's `tree_sitter_kotlin` uses `identifier`,
|
|
# older forks use `simple_identifier`. Accept both so the extractor
|
|
# works across grammar generations.
|
|
name_fallback_child_types=("simple_identifier", "identifier"),
|
|
body_fallback_child_types=("function_body", "class_body", "enum_class_body"),
|
|
function_boundary_types=frozenset({"function_declaration"}),
|
|
import_handler=_import_kotlin,
|
|
)
|
|
|
|
_SCALA_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_scala",
|
|
class_types=frozenset({"class_definition", "object_definition"}),
|
|
function_types=frozenset({"function_definition"}),
|
|
import_types=frozenset({"import_declaration"}),
|
|
call_types=frozenset({"call_expression"}),
|
|
call_function_field="",
|
|
call_accessor_node_types=frozenset({"field_expression"}),
|
|
call_accessor_field="field",
|
|
name_fallback_child_types=("identifier",),
|
|
body_fallback_child_types=("template_body",),
|
|
function_boundary_types=frozenset({"function_definition"}),
|
|
import_handler=_import_scala,
|
|
)
|
|
|
|
_PHP_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_php",
|
|
ts_language_fn="language_php",
|
|
class_types=frozenset({"class_declaration"}),
|
|
function_types=frozenset({"function_definition", "method_declaration"}),
|
|
import_types=frozenset({"namespace_use_clause"}),
|
|
call_types=frozenset({"function_call_expression", "member_call_expression", "scoped_call_expression", "class_constant_access_expression"}),
|
|
static_prop_types=frozenset({"scoped_property_access_expression"}),
|
|
helper_fn_names=frozenset({"config"}),
|
|
container_bind_methods=frozenset({"bind", "singleton", "scoped", "instance"}),
|
|
event_listener_properties=frozenset({"listen", "subscribe"}),
|
|
call_function_field="function",
|
|
call_accessor_node_types=frozenset({"member_call_expression"}),
|
|
call_accessor_field="name",
|
|
name_fallback_child_types=("name",),
|
|
body_fallback_child_types=("declaration_list", "compound_statement"),
|
|
function_boundary_types=frozenset({"function_definition", "method_declaration"}),
|
|
import_handler=_import_php,
|
|
)
|
|
|
|
|
|
def _import_lua(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None:
|
|
"""Extract require('module') from Lua variable_declaration nodes."""
|
|
text = _read_text(node, source)
|
|
import re
|
|
m = re.search(r"""require\s*[\('"]\s*['"]?([^'")\s]+)""", text)
|
|
if m:
|
|
raw_module = m.group(1)
|
|
if raw_module:
|
|
tgt_nid = _resolve_lua_import_target(raw_module, str_path)
|
|
if tgt_nid:
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"confidence_score": 1.0,
|
|
"source_file": str_path,
|
|
"source_location": str(node.start_point[0] + 1),
|
|
"weight": 1.0,
|
|
})
|
|
|
|
|
|
_LUA_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_lua",
|
|
ts_language_fn="language",
|
|
class_types=frozenset(),
|
|
function_types=frozenset({"function_declaration"}),
|
|
import_types=frozenset({"variable_declaration"}),
|
|
call_types=frozenset({"function_call"}),
|
|
call_function_field="name",
|
|
call_accessor_node_types=frozenset({"method_index_expression"}),
|
|
call_accessor_field="name",
|
|
name_fallback_child_types=("identifier", "method_index_expression"),
|
|
body_fallback_child_types=("block",),
|
|
function_boundary_types=frozenset({"function_declaration"}),
|
|
import_handler=_import_lua,
|
|
)
|
|
|
|
|
|
def _import_swift(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> list[tuple[str, str]]:
|
|
"""Emit module-level ``imports`` edges and report the imported modules.
|
|
|
|
A Swift ``import CoreKit`` names a module, not a file path, so — unlike the
|
|
file-resolving JS/TS handlers — there is no existing node for the edge to
|
|
point at. The returned ``(id, label)`` pairs let the extractor materialize a
|
|
``type=module`` anchor node so the edge survives; without it ``build_from_json``
|
|
prunes every Swift import edge as a dangling/external reference (#1327).
|
|
"""
|
|
modules: list[tuple[str, str]] = []
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
raw = _read_text(child, source)
|
|
tgt_nid = _make_id(raw)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
modules.append((tgt_nid, raw))
|
|
break
|
|
return modules
|
|
|
|
|
|
_SWIFT_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_swift",
|
|
class_types=frozenset({"class_declaration", "protocol_declaration"}),
|
|
function_types=frozenset({"function_declaration", "init_declaration", "deinit_declaration", "subscript_declaration"}),
|
|
import_types=frozenset({"import_declaration"}),
|
|
call_types=frozenset({"call_expression"}),
|
|
call_function_field="",
|
|
call_accessor_node_types=frozenset({"navigation_expression"}),
|
|
call_accessor_field="",
|
|
name_fallback_child_types=("simple_identifier", "type_identifier", "user_type"),
|
|
body_fallback_child_types=("class_body", "protocol_body", "function_body", "enum_class_body"),
|
|
function_boundary_types=frozenset({"function_declaration", "init_declaration", "deinit_declaration", "subscript_declaration"}),
|
|
import_handler=_import_swift,
|
|
)
|
|
|
|
# ── Ruby local type inference (for member-call resolution) ─────────────────────
|
|
|
|
|
|
# `Const = <factory>(...)` shapes that define a lightweight class named after the
|
|
# constant. tree-sitter parses each as an `assignment`, not a `class`, so the
|
|
# generic class branch never saw them (#1640).
|
|
|
|
|
|
# ── Generic extractor ─────────────────────────────────────────────────────────
|
|
|
|
|
|
# ── Python rationale extraction ───────────────────────────────────────────────
|
|
|
|
_RATIONALE_PREFIXES = ("# NOTE:", "# IMPORTANT:", "# HACK:", "# WHY:", "# RATIONALE:", "# TODO:", "# FIXME:")
|
|
|
|
|
|
def _is_autogenerated_python(source: bytes) -> bool:
|
|
"""Return True if this Python file is auto-generated and its module docstring is noise.
|
|
|
|
Covers: Alembic/Flask-Migrate revisions, Django migrations, protobuf/gRPC/OpenAPI stubs.
|
|
Module docstrings in these files are change annotations or boilerplate, not rationale.
|
|
"""
|
|
head = source[:2048].decode("utf-8", errors="replace")
|
|
# Generic generated-file markers (protobuf, gRPC, OpenAPI codegen, etc.)
|
|
if any(m in head for m in ("DO NOT EDIT", "@generated", "Generated by the protocol buffer")):
|
|
return True
|
|
# Alembic / Flask-Migrate revision files
|
|
if (re.search(r"^revision\s*[:=]", head, re.MULTILINE)
|
|
and "def upgrade(" in head
|
|
and "down_revision" in head):
|
|
return True
|
|
# Django migrations
|
|
if "class Migration(migrations.Migration)" in head and "operations" in head:
|
|
return True
|
|
return False
|
|
|
|
|
|
def _extract_python_rationale(path: Path, result: dict) -> None:
|
|
"""Post-pass: extract docstrings and rationale comments from Python source.
|
|
Mutates result in-place by appending to result['nodes'] and result['edges'].
|
|
"""
|
|
try:
|
|
import tree_sitter_python as tspython
|
|
from tree_sitter import Language, Parser
|
|
language = Language(tspython.language())
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception:
|
|
return
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes = result["nodes"]
|
|
edges = result["edges"]
|
|
seen_ids = {n["id"] for n in nodes}
|
|
file_nid = _make_id(str(path))
|
|
|
|
def _get_docstring(body_node) -> tuple[str, int] | None:
|
|
if not body_node:
|
|
return None
|
|
for child in body_node.children:
|
|
if child.type == "expression_statement":
|
|
for sub in child.children:
|
|
if sub.type in ("string", "concatenated_string"):
|
|
text = source[sub.start_byte:sub.end_byte].decode("utf-8", errors="replace")
|
|
text = text.strip("\"'").strip('"""').strip("'''").strip()
|
|
if len(text) > 20:
|
|
return text, child.start_point[0] + 1
|
|
break
|
|
return None
|
|
|
|
def _add_rationale(text: str, line: int, parent_nid: str) -> None:
|
|
label = text[:80].replace("\r\n", " ").replace("\r", " ").replace("\n", " ").strip()
|
|
rid = _make_id(stem, "rationale", str(line))
|
|
if rid not in seen_ids:
|
|
seen_ids.add(rid)
|
|
nodes.append({
|
|
"id": rid,
|
|
"label": label,
|
|
"file_type": "rationale",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
})
|
|
edges.append({
|
|
"source": rid,
|
|
"target": parent_nid,
|
|
"relation": "rationale_for",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
# Module-level docstring — skip for auto-generated files (Alembic, Django
|
|
# migrations, protobuf stubs, etc.) whose module docstrings are revision
|
|
# annotations, not architectural rationale.
|
|
if not _is_autogenerated_python(source):
|
|
ds = _get_docstring(root)
|
|
if ds:
|
|
_add_rationale(ds[0], ds[1], file_nid)
|
|
|
|
# Class and function docstrings
|
|
def walk_docstrings(node, parent_nid: str) -> None:
|
|
t = node.type
|
|
if t == "class_definition":
|
|
name_node = node.child_by_field_name("name")
|
|
body = node.child_by_field_name("body")
|
|
if name_node and body:
|
|
class_name = source[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="replace")
|
|
nid = _make_id(stem, class_name)
|
|
ds = _get_docstring(body)
|
|
if ds:
|
|
_add_rationale(ds[0], ds[1], nid)
|
|
for child in body.children:
|
|
walk_docstrings(child, nid)
|
|
return
|
|
if t == "function_definition":
|
|
name_node = node.child_by_field_name("name")
|
|
body = node.child_by_field_name("body")
|
|
if name_node and body:
|
|
func_name = source[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="replace")
|
|
nid = _make_id(parent_nid, func_name) if parent_nid != file_nid else _make_id(stem, func_name)
|
|
ds = _get_docstring(body)
|
|
if ds:
|
|
_add_rationale(ds[0], ds[1], nid)
|
|
return
|
|
for child in node.children:
|
|
walk_docstrings(child, parent_nid)
|
|
|
|
walk_docstrings(root, file_nid)
|
|
|
|
# Rationale comments (# NOTE:, # IMPORTANT:, etc.)
|
|
source_text = source.decode("utf-8", errors="replace")
|
|
for lineno, line_text in enumerate(source_text.splitlines(), start=1):
|
|
stripped = line_text.strip()
|
|
if any(stripped.startswith(p) for p in _RATIONALE_PREFIXES):
|
|
_add_rationale(stripped, lineno, file_nid)
|
|
|
|
|
|
# ── Public API ────────────────────────────────────────────────────────────────
|
|
|
|
def extract_python(path: Path) -> dict:
|
|
"""Extract classes, functions, and imports from a .py file via tree-sitter AST."""
|
|
result = _extract_generic(path, _PYTHON_CONFIG)
|
|
if "error" not in result:
|
|
_extract_python_rationale(path, result)
|
|
return result
|
|
|
|
|
|
def extract_js(path: Path) -> dict:
|
|
"""Extract classes, functions, arrow functions, and imports from a .js/.ts/.tsx/.mts/.cts file."""
|
|
if path.suffix == ".tsx":
|
|
config = _TSX_CONFIG
|
|
elif path.suffix in (".ts", ".mts", ".cts"):
|
|
config = _TS_CONFIG
|
|
else:
|
|
config = _JS_CONFIG
|
|
result = _extract_generic(path, config)
|
|
if "error" not in result:
|
|
_extract_js_rationale(path, result)
|
|
return result
|
|
|
|
|
|
# ── JS/TS rationale + doc-reference extraction ────────────────────────────────
|
|
#
|
|
# Parity with _extract_python_rationale: Python files get rationale nodes from
|
|
# docstrings and `# NOTE:`-style comments, but JS/TS comments were discarded
|
|
# entirely. That silently drops two high-value signals in mixed corpora:
|
|
# 1. rationale comments (`// NOTE:`, `// WHY:`, ...) — same as Python;
|
|
# 2. architecture-decision references (`ADR-0011`, `RFC 793`) that teams
|
|
# conventionally cite in file/function headers. These are the natural
|
|
# join points between code and design docs in the same graph — without
|
|
# them, code<->ADR edges never form even when the code cites the ADR.
|
|
|
|
_JS_RATIONALE_PREFIXES = (
|
|
"// NOTE:", "// IMPORTANT:", "// HACK:", "// WHY:", "// RATIONALE:",
|
|
"// TODO:", "// FIXME:",
|
|
"* NOTE:", "* IMPORTANT:", "* HACK:", "* WHY:", "* RATIONALE:",
|
|
"* TODO:", "* FIXME:",
|
|
)
|
|
|
|
# Doc-reference tokens worth first-classing as graph nodes. Deliberately
|
|
# conservative: ADR-NNNN (Architecture Decision Records, any zero padding)
|
|
# and RFC NNNN / RFC-NNNN.
|
|
_JS_DOC_REF_RE = re.compile(r"\b(ADR[- ]?\d{1,5}|RFC[- ]?\d{1,5})\b", re.IGNORECASE)
|
|
|
|
# Only look for doc references inside comments, not string literals or code.
|
|
_JS_COMMENT_LINE_RE = re.compile(r"^\s*(//|/\*|\*)")
|
|
|
|
|
|
def _extract_js_rationale(path: Path, result: dict) -> None:
|
|
"""Post-pass: extract rationale comments and doc references from JS/TS source.
|
|
Mutates result in-place by appending to result['nodes'] and result['edges'].
|
|
"""
|
|
try:
|
|
source_text = path.read_text(encoding="utf-8", errors="replace")
|
|
except Exception:
|
|
return
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes = result["nodes"]
|
|
edges = result["edges"]
|
|
seen_ids = {n["id"] for n in nodes}
|
|
file_nid = _make_id(str(path))
|
|
seen_doc_refs: set[str] = set()
|
|
|
|
def _add_rationale(text: str, line: int) -> None:
|
|
label = text[:80].replace("\r\n", " ").replace("\r", " ").replace("\n", " ").strip()
|
|
rid = _make_id(stem, "rationale", str(line))
|
|
if rid not in seen_ids:
|
|
seen_ids.add(rid)
|
|
nodes.append({
|
|
"id": rid,
|
|
"label": label,
|
|
"file_type": "rationale",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
})
|
|
edges.append({
|
|
"source": rid,
|
|
"target": file_nid,
|
|
"relation": "rationale_for",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
def _add_doc_ref(token: str, line: int) -> None:
|
|
# Normalize "adr 11" / "ADR-0011" spellings to a canonical "ADR-0011"
|
|
# style label so references to the same document collapse to one node.
|
|
kind, num = re.match(r"([A-Za-z]+)[- ]?(\d+)", token).groups()
|
|
kind = kind.upper()
|
|
label = f"{kind}-{num.zfill(4)}" if kind == "ADR" else f"{kind}-{num}"
|
|
if label in seen_doc_refs:
|
|
return
|
|
seen_doc_refs.add(label)
|
|
rid = _make_id("docref", label)
|
|
if rid not in seen_ids:
|
|
seen_ids.add(rid)
|
|
nodes.append({
|
|
"id": rid,
|
|
"label": label,
|
|
"file_type": "doc_ref",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
})
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": rid,
|
|
"relation": "cites",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
for lineno, line_text in enumerate(source_text.splitlines(), start=1):
|
|
stripped = line_text.strip()
|
|
if any(stripped.startswith(p) for p in _JS_RATIONALE_PREFIXES):
|
|
_add_rationale(stripped.lstrip("/* "), lineno)
|
|
if _JS_COMMENT_LINE_RE.match(line_text):
|
|
for m in _JS_DOC_REF_RE.finditer(stripped):
|
|
_add_doc_ref(m.group(1), lineno)
|
|
|
|
|
|
def extract_svelte(path: Path) -> dict:
|
|
"""Extract imports from .svelte files: script-block via JS AST + template regex fallback.
|
|
|
|
Tree-sitter only sees the <script> block. Svelte template syntax like
|
|
{#await import('./X.svelte')} lives in the markup layer and is invisible
|
|
to the JS parser, so a regex pass covers those dynamic imports.
|
|
"""
|
|
result = _extract_generic(path, _JS_CONFIG)
|
|
try:
|
|
import re as _re
|
|
src = path.read_text(encoding="utf-8", errors="replace")
|
|
existing_ids = {n["id"] for n in result.get("nodes", [])}
|
|
# Source file node ID must match the one _extract_generic creates:
|
|
# _make_id(str(path)) - single arg, no stem prefix. Otherwise the source
|
|
# endpoint is a phantom node and build_from_json drops the edge (#701).
|
|
file_node_id = _make_id(str(path))
|
|
aliases = _load_tsconfig_aliases(path.parent)
|
|
for m in _re.finditer(r"""import\(\s*['"]([^'"]+)['"]\s*\)""", src):
|
|
raw = m.group(1)
|
|
if not raw:
|
|
continue
|
|
if raw.startswith("."):
|
|
# Relative import - resolve to full path so IDs match file node IDs.
|
|
resolved = Path(os.path.normpath(path.parent / raw))
|
|
# Apply same TS/Svelte resolver fixups as static imports so dynamic
|
|
# imports of bare paths and .svelte.ts rune files land on real
|
|
# file nodes instead of phantom ids (#716).
|
|
resolved = _resolve_js_module_path(resolved)
|
|
node_id = _make_id(str(resolved))
|
|
stub_source_file = str(resolved)
|
|
else:
|
|
# Check tsconfig.json path aliases (e.g. "$lib/" -> "src/lib/", "@/" -> "src/")
|
|
# before treating as external. Mirrors _import_js logic so SvelteKit alias
|
|
# imports resolve to the same file node IDs the extractor creates (#701).
|
|
resolved_alias = _resolve_tsconfig_alias(raw, aliases)
|
|
if resolved_alias is not None:
|
|
resolved_alias = _resolve_js_module_path(resolved_alias)
|
|
node_id = _make_id(str(resolved_alias))
|
|
stub_source_file = str(resolved_alias)
|
|
else:
|
|
# Bare/scoped import (node_modules) - use last segment;
|
|
# build_from_json drops as external if no matching node exists.
|
|
module_name = raw.split("/")[-1]
|
|
if not module_name:
|
|
continue
|
|
node_id = _make_id(module_name)
|
|
stub_source_file = raw
|
|
if node_id in existing_ids:
|
|
# Edge target already a real node - just add the edge, don't add a node.
|
|
result.setdefault("edges", []).append({
|
|
"source": file_node_id, "target": node_id,
|
|
"relation": "dynamic_import", "confidence": "EXTRACTED",
|
|
"source_file": str(path),
|
|
})
|
|
continue
|
|
result.setdefault("nodes", []).append({
|
|
"id": node_id, "label": raw,
|
|
"file_type": "code", "source_file": stub_source_file,
|
|
"confidence": "EXTRACTED",
|
|
})
|
|
result.setdefault("edges", []).append({
|
|
"source": file_node_id, "target": node_id,
|
|
"relation": "dynamic_import", "confidence": "EXTRACTED",
|
|
"source_file": str(path),
|
|
})
|
|
existing_ids.add(node_id)
|
|
# Static imports inside <script> blocks. The JS tree-sitter parser fed
|
|
# the full .svelte file produces a top-level ERROR node (HTML markup
|
|
# is not valid JS), so import_statement nodes are never reached and
|
|
# static imports are silently dropped (#713). Regex over each script
|
|
# body recovers them.
|
|
script_re = _re.compile(
|
|
r"<script\b[^>]*>([\s\S]*?)</script\s*>", _re.IGNORECASE
|
|
)
|
|
static_import_re = _re.compile(
|
|
r"""import\s+(?:[^'"`;]+?\s+from\s+)?['"]([^'"]+)['"]"""
|
|
)
|
|
for script_match in script_re.finditer(src):
|
|
script_body = script_match.group(1)
|
|
for m in static_import_re.finditer(script_body):
|
|
raw = m.group(1)
|
|
if not raw:
|
|
continue
|
|
if raw.startswith("."):
|
|
resolved = Path(os.path.normpath(path.parent / raw))
|
|
if resolved.suffix == ".js":
|
|
resolved = resolved.with_suffix(".ts")
|
|
elif resolved.suffix == ".jsx":
|
|
resolved = resolved.with_suffix(".tsx")
|
|
node_id = _make_id(str(resolved))
|
|
stub_source_file = str(resolved)
|
|
else:
|
|
resolved_alias = _resolve_tsconfig_alias(raw, aliases)
|
|
if resolved_alias is not None:
|
|
node_id = _make_id(str(resolved_alias))
|
|
stub_source_file = str(resolved_alias)
|
|
else:
|
|
module_name = raw.split("/")[-1]
|
|
if not module_name:
|
|
continue
|
|
node_id = _make_id(module_name)
|
|
stub_source_file = raw
|
|
if node_id in existing_ids:
|
|
result.setdefault("edges", []).append({
|
|
"source": file_node_id, "target": node_id,
|
|
"relation": "imports_from", "confidence": "EXTRACTED",
|
|
"source_file": str(path),
|
|
})
|
|
continue
|
|
result.setdefault("nodes", []).append({
|
|
"id": node_id, "label": raw,
|
|
"file_type": "code", "source_file": stub_source_file,
|
|
"confidence": "EXTRACTED",
|
|
})
|
|
result.setdefault("edges", []).append({
|
|
"source": file_node_id, "target": node_id,
|
|
"relation": "imports_from", "confidence": "EXTRACTED",
|
|
"source_file": str(path),
|
|
})
|
|
existing_ids.add(node_id)
|
|
except Exception:
|
|
pass
|
|
return result
|
|
|
|
|
|
def extract_astro(path: Path) -> dict:
|
|
"""Extract imports from .astro files: frontmatter (TS) + template regex fallback.
|
|
|
|
Astro files start with a ``---\\n...\\n---`` frontmatter block of TypeScript
|
|
setup code (where almost all imports live), followed by an HTML-with-expressions
|
|
template body, and optionally ``<script>`` blocks for client-side JS. Tree-sitter
|
|
only sees the file usefully through the frontmatter — feeding the whole file to
|
|
the JS parser produces a top-level ERROR node because the template is not valid
|
|
JS, so ``import_statement`` nodes are never reached and static imports are
|
|
silently dropped (#850). Mirrors :func:`extract_svelte` — same regex-rescue
|
|
approach, scanning the frontmatter block and any client-side ``<script>`` blocks
|
|
for static and dynamic imports.
|
|
"""
|
|
result = _extract_generic(path, _JS_CONFIG)
|
|
try:
|
|
import re as _re
|
|
src = path.read_text(encoding="utf-8", errors="replace")
|
|
existing_ids = {n["id"] for n in result.get("nodes", [])}
|
|
file_node_id = _make_id(str(path))
|
|
aliases = _load_tsconfig_aliases(path.parent)
|
|
# Dynamic imports anywhere in the file: `import('./X.astro')` is legal in
|
|
# frontmatter setup code and inside expression slots.
|
|
for m in _re.finditer(r"""import\(\s*['"]([^'"]+)['"]\s*\)""", src):
|
|
raw = m.group(1)
|
|
if not raw:
|
|
continue
|
|
if raw.startswith("."):
|
|
resolved = Path(os.path.normpath(path.parent / raw))
|
|
resolved = _resolve_js_module_path(resolved)
|
|
node_id = _make_id(str(resolved))
|
|
stub_source_file = str(resolved)
|
|
else:
|
|
resolved_alias = _resolve_tsconfig_alias(raw, aliases)
|
|
if resolved_alias is not None:
|
|
resolved_alias = _resolve_js_module_path(resolved_alias)
|
|
node_id = _make_id(str(resolved_alias))
|
|
stub_source_file = str(resolved_alias)
|
|
else:
|
|
module_name = raw.split("/")[-1]
|
|
if not module_name:
|
|
continue
|
|
node_id = _make_id(module_name)
|
|
stub_source_file = raw
|
|
if node_id in existing_ids:
|
|
result.setdefault("edges", []).append({
|
|
"source": file_node_id, "target": node_id,
|
|
"relation": "dynamic_import", "confidence": "EXTRACTED",
|
|
"source_file": str(path),
|
|
})
|
|
continue
|
|
result.setdefault("nodes", []).append({
|
|
"id": node_id, "label": raw,
|
|
"file_type": "code", "source_file": stub_source_file,
|
|
"confidence": "EXTRACTED",
|
|
})
|
|
result.setdefault("edges", []).append({
|
|
"source": file_node_id, "target": node_id,
|
|
"relation": "dynamic_import", "confidence": "EXTRACTED",
|
|
"source_file": str(path),
|
|
})
|
|
existing_ids.add(node_id)
|
|
# Static imports: scan the `---...---` frontmatter at the file head plus any
|
|
# client-side <script> blocks. Both are TS/JS regions but live inside a file
|
|
# the JS tree-sitter parser cannot validate as a whole.
|
|
frontmatter_re = _re.compile(
|
|
r"\A\s*---\s*\r?\n([\s\S]*?)\r?\n---\s*(?:\r?\n|\Z)"
|
|
)
|
|
script_re = _re.compile(
|
|
r"<script\b[^>]*>([\s\S]*?)</script\s*>", _re.IGNORECASE
|
|
)
|
|
static_import_re = _re.compile(
|
|
r"""import\s+(?:[^'"`;]+?\s+from\s+)?['"]([^'"]+)['"]"""
|
|
)
|
|
regions: list[str] = []
|
|
fm = frontmatter_re.search(src)
|
|
if fm:
|
|
regions.append(fm.group(1))
|
|
for script_match in script_re.finditer(src):
|
|
regions.append(script_match.group(1))
|
|
for region in regions:
|
|
for m in static_import_re.finditer(region):
|
|
raw = m.group(1)
|
|
if not raw:
|
|
continue
|
|
if raw.startswith("."):
|
|
resolved = Path(os.path.normpath(path.parent / raw))
|
|
if resolved.suffix == ".js":
|
|
resolved = resolved.with_suffix(".ts")
|
|
elif resolved.suffix == ".jsx":
|
|
resolved = resolved.with_suffix(".tsx")
|
|
node_id = _make_id(str(resolved))
|
|
stub_source_file = str(resolved)
|
|
else:
|
|
resolved_alias = _resolve_tsconfig_alias(raw, aliases)
|
|
if resolved_alias is not None:
|
|
node_id = _make_id(str(resolved_alias))
|
|
stub_source_file = str(resolved_alias)
|
|
else:
|
|
module_name = raw.split("/")[-1]
|
|
if not module_name:
|
|
continue
|
|
node_id = _make_id(module_name)
|
|
stub_source_file = raw
|
|
if node_id in existing_ids:
|
|
result.setdefault("edges", []).append({
|
|
"source": file_node_id, "target": node_id,
|
|
"relation": "imports_from", "confidence": "EXTRACTED",
|
|
"source_file": str(path),
|
|
})
|
|
continue
|
|
result.setdefault("nodes", []).append({
|
|
"id": node_id, "label": raw,
|
|
"file_type": "code", "source_file": stub_source_file,
|
|
"confidence": "EXTRACTED",
|
|
})
|
|
result.setdefault("edges", []).append({
|
|
"source": file_node_id, "target": node_id,
|
|
"relation": "imports_from", "confidence": "EXTRACTED",
|
|
"source_file": str(path),
|
|
})
|
|
existing_ids.add(node_id)
|
|
except Exception:
|
|
pass
|
|
return result
|
|
|
|
|
|
# The open-tag matcher skips over quoted attribute values so a `>` inside one
|
|
# (e.g. Vue 3.3+ generic components: `<script setup lang="ts"
|
|
# generic="T extends Record<string, unknown>">`) doesn't prematurely end the tag.
|
|
|
|
|
|
def extract_vue(path: Path) -> dict:
|
|
"""Extract imports, symbols, and type refs from a ``.vue`` SFC.
|
|
|
|
Masks the non-``<script>`` regions and parses the script with the grammar
|
|
its ``lang`` implies (``tsx``→TSX, ``js``/``jsx``→JS, ``ts`` or unset→TS;
|
|
TS is a superset of JS so it is a safe default). A regex pass then recovers
|
|
``import('…')`` dynamic imports the AST does not edge.
|
|
"""
|
|
try:
|
|
src = path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
return {"nodes": [], "edges": []}
|
|
|
|
masked, lang = _vue_mask_non_script(src)
|
|
if lang == "tsx":
|
|
config = _TSX_CONFIG
|
|
elif lang in ("js", "jsx"):
|
|
config = _JS_CONFIG
|
|
else: # "ts" or unspecified — default to the TS grammar (superset of JS)
|
|
config = _TS_CONFIG
|
|
|
|
result = _extract_generic(path, config, source_override=masked.encode("utf-8"))
|
|
|
|
# Dynamic `import('…')` calls aren't edged by the AST pass; recover by regex,
|
|
# mirroring extract_svelte/extract_astro.
|
|
try:
|
|
existing_ids = {n["id"] for n in result.get("nodes", [])}
|
|
file_node_id = _make_id(str(path))
|
|
aliases = _load_tsconfig_aliases(path.parent)
|
|
for m in re.finditer(r"""import\(\s*['"]([^'"]+)['"]\s*\)""", src):
|
|
raw = m.group(1)
|
|
if not raw:
|
|
continue
|
|
if raw.startswith("."):
|
|
resolved = Path(os.path.normpath(path.parent / raw))
|
|
resolved = _resolve_js_module_path(resolved)
|
|
node_id = _make_id(str(resolved))
|
|
stub_source_file = str(resolved)
|
|
else:
|
|
resolved_alias = _resolve_tsconfig_alias(raw, aliases)
|
|
if resolved_alias is not None:
|
|
resolved_alias = _resolve_js_module_path(resolved_alias)
|
|
node_id = _make_id(str(resolved_alias))
|
|
stub_source_file = str(resolved_alias)
|
|
else:
|
|
module_name = raw.split("/")[-1]
|
|
if not module_name:
|
|
continue
|
|
node_id = _make_id(module_name)
|
|
stub_source_file = raw
|
|
if node_id in existing_ids:
|
|
result.setdefault("edges", []).append({
|
|
"source": file_node_id, "target": node_id,
|
|
"relation": "dynamic_import", "confidence": "EXTRACTED",
|
|
"source_file": str(path),
|
|
})
|
|
continue
|
|
result.setdefault("nodes", []).append({
|
|
"id": node_id, "label": raw,
|
|
"file_type": "code", "source_file": stub_source_file,
|
|
"confidence": "EXTRACTED",
|
|
})
|
|
result.setdefault("edges", []).append({
|
|
"source": file_node_id, "target": node_id,
|
|
"relation": "dynamic_import", "confidence": "EXTRACTED",
|
|
"source_file": str(path),
|
|
})
|
|
existing_ids.add(node_id)
|
|
except Exception:
|
|
pass
|
|
return result
|
|
|
|
|
|
def extract_java(path: Path) -> dict:
|
|
"""Extract classes, interfaces, methods, constructors, and imports from a .java file."""
|
|
return _extract_generic(path, _JAVA_CONFIG)
|
|
|
|
|
|
def _is_spock_file(path: Path, ts_result: dict) -> bool:
|
|
"""Return True when the file contains Spock-style ``def "feature"()`` methods
|
|
that tree-sitter-groovy cannot parse, detected by checking the raw source."""
|
|
import re as _re
|
|
_SPOCK_FEATURE_RE = _re.compile(r"""^\s*def\s+[\"']""", _re.MULTILINE)
|
|
try:
|
|
return bool(_SPOCK_FEATURE_RE.search(path.read_text(errors="replace")))
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
def _extract_spock_fallback(path: Path, ts_result: dict) -> dict:
|
|
"""Regex-based fallback for Spock spec files where tree-sitter-groovy cannot parse
|
|
``def "feature name"()`` methods. Merges import edges from the tree-sitter pass
|
|
(which survive reliably) with class and feature-method nodes extracted via regex.
|
|
"""
|
|
import re as _re
|
|
source = path.read_text(errors="replace")
|
|
str_path = str(path)
|
|
stem = _file_stem(path)
|
|
|
|
# Only keep the file node from the tree-sitter pass (guaranteed present and
|
|
# correctly IDed) plus all import edges. All other ts nodes are discarded to
|
|
# avoid orphaned method/constructor nodes whose parent edges were dropped.
|
|
file_node = next((n for n in ts_result.get("nodes", []) if n.get("label") == path.name), None)
|
|
nodes: list[dict] = [file_node] if file_node else []
|
|
edges: list[dict] = [e for e in ts_result.get("edges", []) if e.get("context") == "import"]
|
|
seen_ids: set[str] = {n["id"] for n in nodes}
|
|
|
|
def _add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({
|
|
"id": nid,
|
|
"label": label,
|
|
"file_type": "code",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
})
|
|
|
|
def _add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED") -> None:
|
|
edges.append({
|
|
"source": src,
|
|
"target": tgt,
|
|
"relation": relation,
|
|
"confidence": confidence,
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
lines_text = source.splitlines()
|
|
|
|
# Extract class declarations
|
|
class_re = _re.compile(r"^\s*(?:[\w@]+\s+)*class\s+(\w+)")
|
|
# Extract Spock feature methods: def "..." () or def '...' ()
|
|
# Two separate capture groups per quote style so apostrophes inside
|
|
# double-quoted names (e.g. "shouldn't") are captured correctly.
|
|
feature_re = _re.compile(r"""^\s*def\s+(?:\"([^\"]+)\"|'([^']+)')\s*\(""")
|
|
# Extract plain def methods (non-string names) as well
|
|
plain_method_re = _re.compile(r"""^\s*def\s+(\w+)\s*\(""")
|
|
|
|
current_class_nid: str | None = None
|
|
file_nid = _make_id(str_path)
|
|
|
|
# Ensure the file node exists (tree-sitter pass may have emitted it)
|
|
if file_nid not in seen_ids:
|
|
_add_node(file_nid, path.name, 1)
|
|
|
|
for lineno, line_text in enumerate(lines_text, start=1):
|
|
cm = class_re.match(line_text)
|
|
if cm:
|
|
class_name = cm.group(1)
|
|
class_nid = _make_id(stem, class_name)
|
|
_add_node(class_nid, class_name, lineno)
|
|
_add_edge(file_nid, class_nid, "contains", lineno)
|
|
current_class_nid = class_nid
|
|
continue
|
|
|
|
if current_class_nid is None:
|
|
continue
|
|
|
|
fm = feature_re.match(line_text)
|
|
if fm:
|
|
method_name = fm.group(1) or fm.group(2)
|
|
method_label = f'"{method_name}"'
|
|
method_nid = _make_id(current_class_nid, method_name)
|
|
_add_node(method_nid, method_label, lineno)
|
|
_add_edge(current_class_nid, method_nid, "method", lineno)
|
|
continue
|
|
|
|
pm = plain_method_re.match(line_text)
|
|
if pm:
|
|
method_name = pm.group(1)
|
|
if method_name not in ("if", "while", "for", "switch", "catch"):
|
|
method_label = f".{method_name}()"
|
|
method_nid = _make_id(current_class_nid, method_name)
|
|
_add_node(method_nid, method_label, lineno)
|
|
_add_edge(current_class_nid, method_nid, "method", lineno)
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
def extract_groovy(path: Path) -> dict:
|
|
"""Extract classes, methods, constructors, and imports from a .groovy/.gradle file.
|
|
|
|
Falls back to a regex-based Spock extractor when tree-sitter-groovy cannot parse
|
|
``def "feature name"()`` methods (common in Spock specification classes).
|
|
"""
|
|
result = _extract_generic(path, _GROOVY_CONFIG)
|
|
if _is_spock_file(path, result):
|
|
result = _extract_spock_fallback(path, result)
|
|
return result
|
|
|
|
|
|
def extract_c(path: Path) -> dict:
|
|
"""Extract functions and includes from a .c/.h file."""
|
|
return _extract_generic(path, _C_CONFIG)
|
|
|
|
|
|
def extract_cpp(path: Path) -> dict:
|
|
"""Extract functions, classes, and includes from a .cpp/.cc/.cxx/.hpp file."""
|
|
return _extract_generic(path, _CPP_CONFIG)
|
|
|
|
|
|
def extract_ruby(path: Path) -> dict:
|
|
"""Extract classes, methods, singleton methods, and calls from a .rb file."""
|
|
return _extract_generic(path, _RUBY_CONFIG)
|
|
|
|
|
|
def extract_csharp(path: Path) -> dict:
|
|
"""Extract C# type declarations, methods, namespaces, and usings from a .cs file."""
|
|
return _extract_generic(path, _CSHARP_CONFIG)
|
|
|
|
|
|
def extract_kotlin(path: Path) -> dict:
|
|
"""Extract classes, objects, functions, and imports from a .kt/.kts file."""
|
|
return _extract_generic(path, _KOTLIN_CONFIG)
|
|
|
|
|
|
def extract_scala(path: Path) -> dict:
|
|
"""Extract classes, objects, functions, and imports from a .scala file."""
|
|
return _extract_generic(path, _SCALA_CONFIG)
|
|
|
|
|
|
def extract_php(path: Path) -> dict:
|
|
"""Extract classes, functions, methods, namespace uses, and calls from a .php file."""
|
|
return _extract_generic(path, _PHP_CONFIG)
|
|
|
|
|
|
# One level of balanced parens (e.g. `Foo #(Bar #(int))`) — bounded so malformed
|
|
# input cannot trigger pathological backtracking.
|
|
|
|
|
|
def extract_lua(path: Path) -> dict:
|
|
"""Extract functions, methods, require() imports, and calls from a .lua file."""
|
|
return _extract_generic(path, _LUA_CONFIG)
|
|
|
|
|
|
def extract_swift(path: Path) -> dict:
|
|
"""Extract classes, structs, protocols, functions, imports, and calls from a .swift file."""
|
|
return _extract_generic(path, _SWIFT_CONFIG)
|
|
|
|
|
|
# ── Julia extractor (custom walk) ────────────────────────────────────────────
|
|
|
|
|
|
# ── Go extractor (custom walk) ────────────────────────────────────────────────
|
|
|
|
|
|
# ── Rust extractor (custom walk) ──────────────────────────────────────────────
|
|
|
|
# Common Rust trait/stdlib method names that appear in virtually every codebase.
|
|
# Resolving these cross-file produces spurious INFERRED edges across crate
|
|
# boundaries (issue #908) — skip them from the unresolved-call queue entirely.
|
|
|
|
|
|
# ── Zig ───────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
# ── PowerShell ────────────────────────────────────────────────────────────────
|
|
|
|
|
|
# ── PowerShell manifest (.psd1) ──────────────────────────────────────────────
|
|
|
|
# Keys in a .psd1 whose values are module names/paths we treat as imports.
|
|
|
|
|
|
# ── Cross-file import resolution ──────────────────────────────────────────────
|
|
|
|
|
|
def _canonicalize_csharp_namespace_nodes(all_nodes: list[dict], all_edges: list[dict]) -> None:
|
|
"""Collapse duplicate C# namespace node entries to one canonical node per label."""
|
|
by_label: dict[str, list[dict]] = {}
|
|
for node in all_nodes:
|
|
if node.get("type") != "namespace":
|
|
continue
|
|
label = node.get("label")
|
|
if isinstance(label, str):
|
|
by_label.setdefault(label, []).append(node)
|
|
|
|
remap: dict[str, str] = {}
|
|
drop_node_ids: set[int] = set()
|
|
for group in by_label.values():
|
|
if len(group) < 2:
|
|
continue
|
|
canonical = sorted(
|
|
group,
|
|
key=lambda node: (
|
|
str(node.get("source_file") or ""),
|
|
str(node.get("source_location") or ""),
|
|
str(node.get("id") or ""),
|
|
),
|
|
)[0]
|
|
canonical_id = canonical.get("id")
|
|
for node in group:
|
|
if node is canonical:
|
|
continue
|
|
drop_node_ids.add(id(node))
|
|
dup_id = node.get("id")
|
|
if isinstance(dup_id, str) and isinstance(canonical_id, str):
|
|
remap[dup_id] = canonical_id
|
|
|
|
if remap:
|
|
for edge in all_edges:
|
|
if edge.get("source") in remap:
|
|
edge["source"] = remap[str(edge["source"])]
|
|
if edge.get("target") in remap:
|
|
edge["target"] = remap[str(edge["target"])]
|
|
|
|
if drop_node_ids:
|
|
all_nodes[:] = [node for node in all_nodes if id(node) not in drop_node_ids]
|
|
|
|
|
|
# Languages whose identifiers are case-insensitive, so cross-file name resolution
|
|
# may fold case. Everywhere else, case is semantic (`Path` the class vs `PATH` the
|
|
# env var are distinct) and folding manufactures false edges / super-hubs (#1581).
|
|
_CASE_INSENSITIVE_EXTS = frozenset({
|
|
".php", ".phtml", ".php3", ".php4", ".php5", ".php7", ".phps", # PHP fns/classes
|
|
".sql", # SQL identifiers
|
|
".nim", ".nims", ".nimble", # Nim (style-insensitive)
|
|
})
|
|
|
|
|
|
def _lang_is_case_insensitive(source_file: object) -> bool:
|
|
"""True when the file's language resolves identifiers case-insensitively (#1581)."""
|
|
if not source_file:
|
|
return False
|
|
return Path(str(source_file)).suffix.lower() in _CASE_INSENSITIVE_EXTS
|
|
|
|
|
|
# Language interop families for cross-file call resolution. A call in one language
|
|
# can never bind by name to a definition in another family — a TSX component does
|
|
# not invoke a Kotlin method, and a Python function does not invoke a Java one.
|
|
# Families are grouped by REAL interop so legitimate cross-language resolution
|
|
# keeps working: Kotlin/Java/Scala/Groovy share the JVM, C/C++/Objective-C/CUDA
|
|
# share headers and symbols (Swift bridges to Objective-C), and JS/TS variants
|
|
# (plus Vue/Svelte/Astro SFC script blocks) compile into one module graph.
|
|
# Extensions absent from this map (docs, configs, unknown languages) resolve to
|
|
# no family and are never filtered — same permissive default as before.
|
|
_LANG_FAMILY_BY_EXT: dict[str, str] = {
|
|
# JS/TS module graph (SFCs embed JS/TS)
|
|
".js": "jsts", ".jsx": "jsts", ".mjs": "jsts", ".cjs": "jsts",
|
|
".ts": "jsts", ".tsx": "jsts", ".mts": "jsts", ".cts": "jsts",
|
|
".vue": "jsts", ".svelte": "jsts", ".astro": "jsts",
|
|
# JVM interop
|
|
".java": "jvm", ".kt": "jvm", ".kts": "jvm",
|
|
".scala": "jvm", ".groovy": "jvm", ".gradle": "jvm",
|
|
# C-family: shared headers, Objective-C/C++ mix, Swift↔ObjC bridging
|
|
".c": "native", ".h": "native", ".cpp": "native", ".cc": "native",
|
|
".cxx": "native", ".hpp": "native", ".cu": "native", ".cuh": "native",
|
|
".metal": "native", ".m": "native", ".mm": "native", ".swift": "native",
|
|
# Single-language families
|
|
".py": "python",
|
|
".go": "go",
|
|
".rs": "rust",
|
|
".rb": "ruby", ".rake": "ruby",
|
|
".php": "php", ".phtml": "php", ".php3": "php", ".php4": "php",
|
|
".php5": "php", ".php7": "php", ".phps": "php",
|
|
".cs": "dotnet", ".razor": "dotnet", ".cshtml": "dotnet", ".xaml": "dotnet",
|
|
".lua": "lua", ".luau": "lua",
|
|
".zig": "zig",
|
|
".ex": "elixir", ".exs": "elixir",
|
|
".jl": "julia",
|
|
".dart": "dart",
|
|
".sh": "shell", ".bash": "shell",
|
|
".ps1": "powershell", ".psm1": "powershell", ".psd1": "powershell",
|
|
}
|
|
|
|
|
|
def _lang_family(source_file: object) -> str | None:
|
|
"""Interop family of the file's language, or None when unknown/not code."""
|
|
if not source_file:
|
|
return None
|
|
return _LANG_FAMILY_BY_EXT.get(Path(str(source_file)).suffix.lower())
|
|
|
|
|
|
def _node_label_key(node: dict, fold: bool = False) -> str:
|
|
label = str(node.get("label", "")).strip()
|
|
key = re.sub(r"[^a-zA-Z0-9]+", "", label)
|
|
return key.lower() if fold else key
|
|
|
|
|
|
def _is_top_level_function_definition(node: dict) -> bool:
|
|
"""A free/top-level function def (label ``name()``), not a method or type.
|
|
|
|
Methods carry a leading dot (``.foo()``) or a qualifier (``Class.foo()``);
|
|
excluding those keeps a bare-name reference from binding to a receiver-scoped
|
|
method, which the receiver-typed resolvers own (#1781).
|
|
"""
|
|
label = str(node.get("label", "")).strip()
|
|
return (
|
|
node.get("file_type") == "code"
|
|
and label.endswith(")")
|
|
and not label.startswith(".")
|
|
and "." not in label
|
|
)
|
|
|
|
|
|
def _rewire_unique_stub_nodes(nodes: list[dict], edges: list[dict]) -> None:
|
|
"""Map unresolved no-source stubs to a unique real definition with the same label."""
|
|
real_by_label: dict[str, list[dict]] = {} # exact-case type-like (all languages)
|
|
real_by_label_ci: dict[str, list[dict]] = {} # case-INSENSITIVE-language reals only
|
|
func_by_label: dict[str, list[dict]] = {} # top-level function defs (#1781)
|
|
stubs: list[dict] = []
|
|
|
|
for node in nodes:
|
|
key = _node_label_key(node)
|
|
if not key:
|
|
continue
|
|
if node.get("source_file"):
|
|
if _is_type_like_definition(node):
|
|
# Match stubs case-SENSITIVELY: a `Path` reference must not rewire to a
|
|
# `PATH` env var (#1581). Fold only for genuinely case-insensitive
|
|
# languages, where `foo` legitimately resolves to `Foo`.
|
|
real_by_label.setdefault(key, []).append(node)
|
|
if _lang_is_case_insensitive(node.get("source_file")):
|
|
real_by_label_ci.setdefault(
|
|
_node_label_key(node, fold=True), []).append(node)
|
|
elif _is_top_level_function_definition(node):
|
|
func_by_label.setdefault(key, []).append(node)
|
|
continue
|
|
stubs.append(node)
|
|
|
|
# Language families referencing each stub, for the function-merge guard (#1781):
|
|
# a cross-module `references` edge to a function used to dangle on a sourceless
|
|
# name-only stub because functions were excluded as rewire targets. We now allow
|
|
# a UNIQUE function definition to absorb it, but only when it shares a language
|
|
# family with the stub's referrers — so a Python `get_db` reference can't bind to
|
|
# a unique Go `get_db()` (mirrors the #1718/#1749 interop guard).
|
|
stub_ids = {str(s.get("id")) for s in stubs if s.get("id")}
|
|
stub_families: dict[str, set] = {}
|
|
supertype_stub_ids: set[str] = set() # stubs used as a base type — never a function
|
|
_SUPERTYPE_RELATIONS = {"inherits", "implements", "extends"}
|
|
for edge in edges:
|
|
rel = edge.get("relation")
|
|
for endpoint in ("source", "target"):
|
|
nid = edge.get(endpoint)
|
|
if nid in stub_ids:
|
|
fam = _lang_family(edge.get("source_file"))
|
|
if fam is not None:
|
|
stub_families.setdefault(str(nid), set()).add(fam)
|
|
# A stub referenced as a supertype must resolve to a class/type,
|
|
# not a same-named function (you don't inherit from a function).
|
|
if endpoint == "target" and rel in _SUPERTYPE_RELATIONS:
|
|
supertype_stub_ids.add(str(nid))
|
|
|
|
remap: dict[str, str] = {}
|
|
for stub in stubs:
|
|
stub_id = str(stub.get("id", ""))
|
|
if not stub_id:
|
|
continue
|
|
candidates = real_by_label.get(_node_label_key(stub), [])
|
|
if len(candidates) != 1:
|
|
# No unique exact type match — fall back to a case-insensitive match, but
|
|
# only against case-insensitive-language definitions (so a case-sensitive
|
|
# `PATH` can never absorb a `Path` reference).
|
|
candidates = real_by_label_ci.get(_node_label_key(stub, fold=True), [])
|
|
if len(candidates) != 1:
|
|
# #1781: no unique type — try a unique top-level FUNCTION definition,
|
|
# gated by (a) the stub not being used as a supertype and (b) a
|
|
# language-family match with the stub's referrers.
|
|
fcands = func_by_label.get(_node_label_key(stub), [])
|
|
if len(fcands) == 1 and stub_id not in supertype_stub_ids:
|
|
fams = stub_families.get(stub_id, set())
|
|
cand_fam = _lang_family(fcands[0].get("source_file"))
|
|
if not fams or cand_fam is None or cand_fam in fams:
|
|
candidates = fcands
|
|
if len(candidates) != 1:
|
|
continue
|
|
target_id = candidates[0].get("id")
|
|
if isinstance(target_id, str) and target_id and target_id != stub_id:
|
|
remap[stub_id] = target_id
|
|
|
|
if not remap:
|
|
return
|
|
|
|
by_id = {node.get("id"): node for node in nodes if node.get("id")}
|
|
csharp_scoped_relations = {"inherits", "implements", "references", "imports"}
|
|
for edge in edges:
|
|
is_csharp_scoped_edge = (
|
|
str(edge.get("source_file", "")).endswith(".cs")
|
|
and edge.get("relation") in csharp_scoped_relations
|
|
)
|
|
source = edge.get("source")
|
|
if source in remap:
|
|
remapped_source = remap[str(source)]
|
|
if not (
|
|
is_csharp_scoped_edge
|
|
and str(by_id.get(remapped_source, {}).get("source_file", "")).endswith(".cs")
|
|
):
|
|
edge["source"] = remapped_source
|
|
target = edge.get("target")
|
|
if target in remap:
|
|
remapped_target = remap[str(target)]
|
|
if not (
|
|
is_csharp_scoped_edge
|
|
and str(by_id.get(remapped_target, {}).get("source_file", "")).endswith(".cs")
|
|
):
|
|
edge["target"] = remapped_target
|
|
|
|
referenced = {x for e in edges for x in (e.get("source"), e.get("target"))}
|
|
drop_ids = {stub_id for stub_id in remap if stub_id not in referenced}
|
|
nodes[:] = [node for node in nodes if node.get("id") not in drop_ids]
|
|
|
|
|
|
def _augment_js_reexport_edges(
|
|
paths: list[Path],
|
|
nodes: list[dict],
|
|
edges: list[dict],
|
|
root: Path,
|
|
) -> None:
|
|
"""Compatibility wrapper for the JS/TS symbol-resolution post-pass."""
|
|
facts = _SymbolResolutionFacts()
|
|
_collect_js_symbol_resolution_facts(paths, facts)
|
|
_apply_symbol_resolution_facts(paths, nodes, edges, root, facts)
|
|
|
|
|
|
# Header / implementation file-extension pairing for the decl/def class merge.
|
|
|
|
|
|
def _merge_swift_extensions(
|
|
per_file: list[dict],
|
|
all_nodes: list[dict],
|
|
all_edges: list[dict],
|
|
) -> None:
|
|
"""Collapse cross-file Swift `extension Foo` nodes into the canonical `Foo`.
|
|
|
|
tree-sitter-swift reuses `class_declaration` for both `class Foo` and
|
|
`extension Foo`, and node ids carry the file stem, so each file that
|
|
extends `Foo` produces its own `Foo` node. The match is done by label:
|
|
when exactly one non-extension declaration shares the label, extension
|
|
nodes redirect onto it. Extensions of types outside the corpus (no match)
|
|
and ambiguous labels (more than one match) are left untouched — picking
|
|
arbitrarily would invent edges.
|
|
"""
|
|
extension_nids: set[str] = set()
|
|
extension_labels: dict[str, str] = {}
|
|
for result in per_file:
|
|
for ext in result.get("swift_extensions", []) or []:
|
|
extension_nids.add(ext["nid"])
|
|
extension_labels[ext["nid"]] = ext["label"]
|
|
|
|
if not extension_nids:
|
|
return
|
|
|
|
label_to_canonical: dict[str, list[str]] = {}
|
|
for n in all_nodes:
|
|
if n.get("id") in extension_nids:
|
|
continue
|
|
label = n.get("label")
|
|
if not label:
|
|
continue
|
|
label_to_canonical.setdefault(label, []).append(n["id"])
|
|
|
|
remap: dict[str, str] = {}
|
|
for ext_nid in extension_nids:
|
|
candidates = label_to_canonical.get(extension_labels[ext_nid], [])
|
|
if len(candidates) != 1:
|
|
continue
|
|
canonical_nid = candidates[0]
|
|
if canonical_nid != ext_nid:
|
|
remap[ext_nid] = canonical_nid
|
|
|
|
if not remap:
|
|
return
|
|
|
|
all_nodes[:] = [n for n in all_nodes if n.get("id") not in remap]
|
|
|
|
# Each extension file's `contains` edge ends up pointing at the canonical
|
|
# type — multiple files containing the same node is the intended shape:
|
|
# the type owns the methods, the files own their slice. Self-loops are
|
|
# dropped (e.g. an in-file extension method whose call already pointed at
|
|
# the canonical type).
|
|
rewritten: list[dict] = []
|
|
seen_keys: set[tuple] = set()
|
|
for e in all_edges:
|
|
src = remap.get(e.get("source"), e.get("source"))
|
|
tgt = remap.get(e.get("target"), e.get("target"))
|
|
if src == tgt:
|
|
continue
|
|
e["source"] = src
|
|
e["target"] = tgt
|
|
key = (src, tgt, e.get("relation"), e.get("source_file"), e.get("source_location"))
|
|
if key in seen_keys:
|
|
continue
|
|
seen_keys.add(key)
|
|
rewritten.append(e)
|
|
all_edges[:] = rewritten
|
|
|
|
|
|
def _resolve_swift_member_calls(
|
|
per_file: list[dict],
|
|
all_nodes: list[dict],
|
|
all_edges: list[dict],
|
|
) -> None:
|
|
"""Resolve cross-file Swift member calls (``recv.method()``) to the real
|
|
definition of the receiver's type (#1356).
|
|
|
|
The shared cross-file call pass drops every ``is_member_call`` because a bare
|
|
method name (``update``) collides across the corpus and inflates god-nodes
|
|
(#543/#1219). Swift extractors record the receiver of each member call and a
|
|
per-file ``name -> type`` table (``swift_type_table``); this pass uses them to
|
|
type the receiver, then emits an edge ONLY when that type name resolves to
|
|
exactly one definition. A type-qualified call (``Type.staticMethod()``) is
|
|
EXTRACTED (the type is named explicitly in source); an instance call typed via
|
|
local inference (``obj.method()``) is INFERRED. The shared-pass member-call drop
|
|
stays intact: this is purely additive and fires only on receiver-typed Swift calls.
|
|
|
|
Must run after id-disambiguation so node ids and caller_nids are final.
|
|
"""
|
|
type_table_by_file: dict[str, dict[str, str]] = {}
|
|
for result in per_file:
|
|
tt = result.get("swift_type_table")
|
|
if tt and tt.get("path"):
|
|
type_table_by_file[tt["path"]] = tt.get("table", {})
|
|
if not type_table_by_file:
|
|
return
|
|
|
|
def _key(label: str) -> str:
|
|
return re.sub(r"[^a-zA-Z0-9]+", "", str(label)).lower()
|
|
|
|
# A genuine Swift type is the target of a `contains` edge from its file node.
|
|
# Bare type references create a same-label shadow node (via ensure_named_node)
|
|
# that carries a source_file but is NOT contained; excluding non-contained
|
|
# nodes keeps that shadow from making a real type name look ambiguous.
|
|
contained = {e.get("target") for e in all_edges if e.get("relation") == "contains"}
|
|
|
|
# Type name -> definition node ids (real, source-backed, type-like defs only).
|
|
# len != 1 is the god-node guard: an ambiguous type name bails.
|
|
type_def_nids: dict[str, list[str]] = {}
|
|
node_by_id: dict[str, dict] = {}
|
|
for n in all_nodes:
|
|
node_by_id[n.get("id")] = n
|
|
if n.get("source_file") and n.get("id") in contained and _is_type_like_definition(n):
|
|
type_def_nids.setdefault(_key(n.get("label", "")), []).append(n["id"])
|
|
|
|
# (type_node_id, method_key) -> method_node_id, from `method` edges.
|
|
method_index: dict[tuple[str, str], str] = {}
|
|
for e in all_edges:
|
|
if e.get("relation") != "method":
|
|
continue
|
|
src, tgt = e.get("source"), e.get("target")
|
|
tnode = node_by_id.get(tgt)
|
|
if tnode is not None:
|
|
method_index[(src, _key(tnode.get("label", "")))] = tgt
|
|
|
|
all_raw_calls: list[dict] = []
|
|
for result in per_file:
|
|
all_raw_calls.extend(result.get("raw_calls", []))
|
|
|
|
existing_pairs = {(e.get("source"), e.get("target")) for e in all_edges}
|
|
for rc in all_raw_calls:
|
|
if not rc.get("is_member_call"):
|
|
continue
|
|
receiver = rc.get("receiver")
|
|
callee = rc.get("callee")
|
|
if not receiver or not callee:
|
|
continue
|
|
# Determine the receiver's type. An upper-cased receiver is itself a type
|
|
# (Type.staticMethod(), Singleton.shared.x()); otherwise look it up in the
|
|
# declaring file's local type table.
|
|
if receiver[:1].isupper():
|
|
type_name = receiver
|
|
type_qualified = True
|
|
else:
|
|
type_name = type_table_by_file.get(rc.get("source_file", ""), {}).get(receiver)
|
|
type_qualified = False
|
|
if not type_name:
|
|
continue
|
|
type_defs = type_def_nids.get(_key(type_name), [])
|
|
if len(type_defs) != 1: # ambiguous or absent -> bail (god-node guard)
|
|
continue
|
|
type_nid = type_defs[0]
|
|
caller = rc.get("caller_nid")
|
|
if not caller:
|
|
continue
|
|
method_nid = method_index.get((type_nid, _key(callee)))
|
|
target = method_nid or type_nid
|
|
relation = "calls" if method_nid else "references"
|
|
if target == caller or (caller, target) in existing_pairs:
|
|
continue
|
|
existing_pairs.add((caller, target))
|
|
# A type-qualified call (`Type.staticMethod()`) names the receiver type
|
|
# explicitly in source, so it is an exact reference — EXTRACTED, matching
|
|
# the Python qualified-class-method pass (#1533). An instance call whose
|
|
# receiver type came from local inference (`obj.method()`) stays INFERRED.
|
|
all_edges.append({
|
|
"source": caller,
|
|
"target": target,
|
|
"relation": relation,
|
|
"context": "call",
|
|
"confidence": "EXTRACTED" if type_qualified else "INFERRED",
|
|
"confidence_score": 1.0 if type_qualified else 0.8,
|
|
"source_file": rc.get("source_file", ""),
|
|
"source_location": rc.get("source_location"),
|
|
"weight": 1.0,
|
|
})
|
|
|
|
|
|
def _resolve_python_member_calls(
|
|
per_file: list[dict],
|
|
all_nodes: list[dict],
|
|
all_edges: list[dict],
|
|
) -> None:
|
|
"""Resolve cross-file Python qualified class-method calls (``ClassName.method()``)
|
|
to the class-qualified method node (#1446).
|
|
|
|
The shared cross-file call pass drops every ``is_member_call`` because a bare
|
|
method name (``log``) collides across the corpus and inflates god-nodes
|
|
(#543/#1219). That guard is right for *instance* calls (``obj.method()``) but
|
|
misses *class-qualified* calls (``ClassName.method()``), where the receiver is
|
|
an explicitly-named class — an exact, unambiguous reference. This pass uses the
|
|
receiver captured by the extractor, and when it is a capitalized name resolving
|
|
to exactly one class node that owns the called method, emits an EXTRACTED
|
|
``calls`` edge. Purely additive (only member calls the shared pass skipped),
|
|
with a single-definition god-node guard.
|
|
|
|
Must run after id-disambiguation so node ids and caller_nids are final.
|
|
"""
|
|
def _key(label: str) -> str:
|
|
return re.sub(r"[^a-zA-Z0-9]+", "", str(label)).lower()
|
|
|
|
node_by_id: dict[str, dict] = {n.get("id"): n for n in all_nodes}
|
|
|
|
# A class owns methods: it is the source of one or more `method` edges. Index
|
|
# class label -> owning class node ids (len != 1 is the god-node guard), and
|
|
# (class_node_id, method_key) -> method_node_id.
|
|
class_def_nids: dict[str, list[str]] = {}
|
|
method_index: dict[tuple[str, str], str] = {}
|
|
for e in all_edges:
|
|
if e.get("relation") != "method":
|
|
continue
|
|
src, tgt = e.get("source"), e.get("target")
|
|
cnode = node_by_id.get(src)
|
|
if cnode is not None:
|
|
class_def_nids.setdefault(_key(cnode.get("label", "")), []).append(src)
|
|
tnode = node_by_id.get(tgt)
|
|
if tnode is not None:
|
|
method_index[(src, _key(tnode.get("label", "")))] = tgt
|
|
if not class_def_nids:
|
|
return
|
|
# A class with N methods produced N entries; collapse to a unique set.
|
|
for k in list(class_def_nids):
|
|
class_def_nids[k] = sorted(set(class_def_nids[k]))
|
|
|
|
all_raw_calls: list[dict] = []
|
|
for result in per_file:
|
|
all_raw_calls.extend(result.get("raw_calls", []))
|
|
|
|
existing_pairs = {(e.get("source"), e.get("target")) for e in all_edges}
|
|
for rc in all_raw_calls:
|
|
if not rc.get("is_member_call"):
|
|
continue
|
|
receiver = rc.get("receiver")
|
|
callee = rc.get("callee")
|
|
caller = rc.get("caller_nid")
|
|
if not receiver or not callee or not caller:
|
|
continue
|
|
# Only a capitalized receiver is treated as a class reference, so an
|
|
# instance/module (`self`, `obj`, `config`) never collides with a
|
|
# same-spelled class via the case-folding key.
|
|
if not receiver[:1].isupper():
|
|
continue
|
|
class_nids = class_def_nids.get(_key(receiver), [])
|
|
if len(class_nids) != 1: # absent or ambiguous -> bail (god-node guard)
|
|
continue
|
|
method_nid = method_index.get((class_nids[0], _key(callee)))
|
|
if not method_nid or method_nid == caller:
|
|
continue
|
|
if (caller, method_nid) in existing_pairs:
|
|
continue
|
|
existing_pairs.add((caller, method_nid))
|
|
# EXTRACTED: a qualified `ClassName.method()` is an explicit, unambiguous
|
|
# static reference (unlike a bare instance member call), and the class
|
|
# resolved to exactly one definition that owns the method.
|
|
all_edges.append({
|
|
"source": caller,
|
|
"target": method_nid,
|
|
"relation": "calls",
|
|
"context": "call",
|
|
"confidence": "EXTRACTED",
|
|
"confidence_score": 1.0,
|
|
"source_file": rc.get("source_file", ""),
|
|
"source_location": rc.get("source_location"),
|
|
"weight": 1.0,
|
|
})
|
|
|
|
|
|
def _resolve_typescript_member_calls(
|
|
per_file: list[dict],
|
|
all_nodes: list[dict],
|
|
all_edges: list[dict],
|
|
) -> None:
|
|
"""Resolve cross-file TS/JS member calls via constructor-injection type tables (#1316).
|
|
|
|
``this.repo.findById()`` drops out in the shared cross-file pass because bare
|
|
``findById`` collides across the corpus (god-node guard). TS constructors with
|
|
parameter-property modifiers (``private repo: IUserRepository``) produce a
|
|
per-file type table mapping field names to their declared types. This pass
|
|
looks up the receiver field's type, finds a single-definition class/interface
|
|
owning a method with the callee name, and emits an EXTRACTED ``calls`` edge.
|
|
"""
|
|
type_table_by_file: dict[str, dict[str, str]] = {}
|
|
for result in per_file:
|
|
tt = result.get("ts_type_table")
|
|
if tt and tt.get("path"):
|
|
type_table_by_file[tt["path"]] = tt.get("table", {})
|
|
if not type_table_by_file:
|
|
return
|
|
|
|
def _key(label: str) -> str:
|
|
return re.sub(r"[^a-zA-Z0-9]+", "", str(label)).lower()
|
|
|
|
contained = {e.get("target") for e in all_edges if e.get("relation") == "contains"}
|
|
|
|
type_def_nids: dict[str, list[str]] = {}
|
|
node_by_id: dict[str, dict] = {}
|
|
for n in all_nodes:
|
|
node_by_id[n.get("id")] = n
|
|
if n.get("source_file") and n.get("id") in contained and _is_type_like_definition(n):
|
|
type_def_nids.setdefault(_key(n.get("label", "")), []).append(n["id"])
|
|
|
|
method_index: dict[tuple[str, str], str] = {}
|
|
for e in all_edges:
|
|
if e.get("relation") != "method":
|
|
continue
|
|
src, tgt = e.get("source"), e.get("target")
|
|
tnode = node_by_id.get(tgt)
|
|
if tnode is not None:
|
|
method_index[(src, _key(tnode.get("label", "")))] = tgt
|
|
|
|
all_raw_calls: list[dict] = []
|
|
for result in per_file:
|
|
all_raw_calls.extend(result.get("raw_calls", []))
|
|
|
|
existing_pairs = {(e.get("source"), e.get("target")) for e in all_edges}
|
|
for rc in all_raw_calls:
|
|
if not rc.get("is_member_call"):
|
|
continue
|
|
receiver = rc.get("receiver")
|
|
callee = rc.get("callee")
|
|
caller = rc.get("caller_nid")
|
|
if not receiver or not callee or not caller:
|
|
continue
|
|
if receiver[:1].isupper():
|
|
type_name = receiver
|
|
else:
|
|
type_name = type_table_by_file.get(rc.get("source_file", ""), {}).get(receiver)
|
|
if not type_name:
|
|
continue
|
|
# A builtin global receiver type (Date, Promise, Map, ...) must not resolve
|
|
# to a user symbol. _key() casefolds, so `x: Date; x.getTime()` would bind
|
|
# the caller to a same-named user `class DATE` in another file, inventing
|
|
# phantom `references[call]` edges and a false god node (#1726). The
|
|
# cross-file CALL resolver already skips these globals; do the same here.
|
|
if type_name in _LANGUAGE_BUILTIN_GLOBALS:
|
|
continue
|
|
type_defs = type_def_nids.get(_key(type_name), [])
|
|
if len(type_defs) != 1:
|
|
continue
|
|
type_nid = type_defs[0]
|
|
method_nid = method_index.get((type_nid, _key(callee)))
|
|
target = method_nid or type_nid
|
|
relation = "calls" if method_nid else "references"
|
|
if target == caller or (caller, target) in existing_pairs:
|
|
continue
|
|
existing_pairs.add((caller, target))
|
|
all_edges.append({
|
|
"source": caller,
|
|
"target": target,
|
|
"relation": relation,
|
|
"context": "call",
|
|
"confidence": "EXTRACTED",
|
|
"confidence_score": 1.0,
|
|
"source_file": rc.get("source_file", ""),
|
|
"source_location": rc.get("source_location"),
|
|
"weight": 1.0,
|
|
})
|
|
|
|
|
|
def _resolve_cpp_member_calls(
|
|
per_file: list[dict],
|
|
all_nodes: list[dict],
|
|
all_edges: list[dict],
|
|
) -> None:
|
|
"""Resolve cross-file C++ member calls (``f.bar()``, ``f->bar()``,
|
|
``Foo::bar()``, ``this->bar()``) to the real definition of the receiver's type
|
|
(#1547).
|
|
|
|
The shared cross-file pass drops every ``is_member_call`` because a bare method
|
|
name (``bar``) collides across the corpus and inflates god-nodes (#543/#1219).
|
|
The C++ extractor records each member call's receiver and a per-file
|
|
``var -> ClassName`` table (``cpp_type_table``) built from local declarations.
|
|
This pass types the receiver, then emits an edge ONLY when that type resolves
|
|
to exactly ONE definition (the god-node guard).
|
|
|
|
Receiver typing, by precision tier:
|
|
* ``Foo::bar()`` — the scope ``Foo`` names the type explicitly -> EXTRACTED.
|
|
* ``this->bar()`` — the receiver is the caller's own enclosing class -> EXTRACTED.
|
|
* ``f.bar()`` / ``f->bar()`` — ``f`` typed via the file's local table -> INFERRED.
|
|
A receiver whose type can't be inferred locally is SKIPPED (no guess): a false
|
|
call edge is worse than a missing one. The ``_merge_decl_def_classes`` pass has
|
|
already folded each header/impl class pair into one node, so a paired class is a
|
|
single definition and clears the single-definition guard.
|
|
|
|
Must run after id-disambiguation so node ids and caller_nids are final.
|
|
"""
|
|
type_table_by_file: dict[str, dict[str, str]] = {}
|
|
for result in per_file:
|
|
tt = result.get("cpp_type_table")
|
|
if tt and tt.get("path"):
|
|
type_table_by_file[tt["path"]] = tt.get("table", {})
|
|
|
|
def _key(label: str) -> str:
|
|
return re.sub(r"[^a-zA-Z0-9]+", "", str(label)).lower()
|
|
|
|
# A genuine C++ type is the target of a `contains` edge from its file node;
|
|
# bare-reference shadow nodes (ensure_named_node stubs) are not contained, so
|
|
# excluding non-contained nodes keeps them from making a real type ambiguous.
|
|
contained = {e.get("target") for e in all_edges if e.get("relation") == "contains"}
|
|
|
|
type_def_nids: dict[str, list[str]] = {}
|
|
node_by_id: dict[str, dict] = {}
|
|
for n in all_nodes:
|
|
node_by_id[n.get("id")] = n
|
|
if n.get("source_file") and n.get("id") in contained and _is_type_like_definition(n):
|
|
type_def_nids.setdefault(_key(n.get("label", "")), []).append(n["id"])
|
|
|
|
# (type_node_id, method_key) -> method_node_id, and caller -> enclosing type
|
|
# (the owning class) for `this->` calls. A C++ class owns its members via
|
|
# `method` edges (out-of-line definitions) AND `defines` edges (in-class
|
|
# declarations, which the extractor models as fields); index both so a header-
|
|
# declared `void bar();` resolves. `method` wins when a key has both.
|
|
method_index: dict[tuple[str, str], str] = {}
|
|
enclosing_type: dict[str, str] = {}
|
|
for rel in ("defines", "method"):
|
|
for e in all_edges:
|
|
if e.get("relation") != rel:
|
|
continue
|
|
src, tgt = e.get("source"), e.get("target")
|
|
tnode = node_by_id.get(tgt)
|
|
if tnode is None:
|
|
continue
|
|
enclosing_type.setdefault(tgt, src)
|
|
method_index[(src, _key(tnode.get("label", "")))] = tgt
|
|
|
|
all_raw_calls: list[dict] = []
|
|
for result in per_file:
|
|
all_raw_calls.extend(result.get("raw_calls", []))
|
|
|
|
existing_pairs = {(e.get("source"), e.get("target")) for e in all_edges}
|
|
for rc in all_raw_calls:
|
|
if not rc.get("is_member_call"):
|
|
continue
|
|
receiver = rc.get("receiver")
|
|
callee = rc.get("callee")
|
|
caller = rc.get("caller_nid")
|
|
if not receiver or not callee or not caller:
|
|
continue
|
|
src_file = rc.get("source_file", "")
|
|
# Only resolve C++ raw_calls (other languages share the raw_calls list;
|
|
# a `.h` may route to either extract_cpp or extract_objc by content, so the
|
|
# extractor-stamped `lang` tag — not the suffix — is the unambiguous gate).
|
|
if rc.get("lang") != "cpp":
|
|
continue
|
|
# Determine the receiver's type and the resulting confidence.
|
|
if receiver == "this":
|
|
# this->bar(): receiver is the caller's own enclosing class.
|
|
type_nid = enclosing_type.get(caller)
|
|
if not type_nid:
|
|
continue
|
|
type_qualified = True
|
|
elif receiver[:1].isupper():
|
|
# Foo::bar(): the type is named explicitly in source.
|
|
type_defs = type_def_nids.get(_key(receiver), [])
|
|
if len(type_defs) != 1: # ambiguous or absent -> bail (god-node guard)
|
|
continue
|
|
type_nid = type_defs[0]
|
|
type_qualified = True
|
|
else:
|
|
# f.bar() / f->bar(): type the receiver via the file's local table.
|
|
type_name = type_table_by_file.get(src_file, {}).get(receiver)
|
|
if not type_name:
|
|
continue
|
|
type_defs = type_def_nids.get(_key(type_name), [])
|
|
if len(type_defs) != 1: # ambiguous or absent -> bail (god-node guard)
|
|
continue
|
|
type_nid = type_defs[0]
|
|
type_qualified = False
|
|
method_nid = method_index.get((type_nid, _key(callee)))
|
|
target = method_nid or type_nid
|
|
relation = "calls" if method_nid else "references"
|
|
if target == caller or (caller, target) in existing_pairs:
|
|
continue
|
|
existing_pairs.add((caller, target))
|
|
all_edges.append({
|
|
"source": caller,
|
|
"target": target,
|
|
"relation": relation,
|
|
"context": "call",
|
|
"confidence": "EXTRACTED" if type_qualified else "INFERRED",
|
|
"confidence_score": 1.0 if type_qualified else 0.8,
|
|
"source_file": src_file,
|
|
"source_location": rc.get("source_location"),
|
|
"weight": 1.0,
|
|
})
|
|
|
|
|
|
def _resolve_csharp_member_calls(
|
|
per_file: list[dict],
|
|
all_nodes: list[dict],
|
|
all_edges: list[dict],
|
|
) -> None:
|
|
"""Resolve C# member calls (``recv.Method()``) to the receiver's declared type
|
|
(#1609).
|
|
|
|
The shared cross-file pass drops every ``is_member_call`` because a bare method
|
|
name collides across the corpus — and for C# an in-file bare match silently
|
|
mis-bound ``_server.Save()`` to an unrelated ``Cache.Save()``. The C# extractor
|
|
now records each member call's receiver plus a per-file ``name -> Type`` table
|
|
(``csharp_type_table``) of fields/properties/params/locals. This pass types the
|
|
receiver, then emits an edge ONLY when that type resolves to exactly ONE
|
|
definition (the god-node guard); an untypable receiver is skipped (no guess).
|
|
|
|
Receiver typing, by precision tier:
|
|
* ``this.M()`` — receiver is the caller's own enclosing class -> EXTRACTED.
|
|
* ``Type.M()`` (capitalized) — the type is named explicitly in source -> EXTRACTED.
|
|
* ``recv.M()`` — ``recv`` typed via the file's field/param/local table -> INFERRED.
|
|
|
|
Must run after id-disambiguation so node ids and caller_nids are final.
|
|
"""
|
|
type_table_by_file: dict[str, dict[str, str]] = {}
|
|
for result in per_file:
|
|
tt = result.get("csharp_type_table")
|
|
if tt and tt.get("path"):
|
|
type_table_by_file[tt["path"]] = tt.get("table", {})
|
|
|
|
def _key(label: str) -> str:
|
|
return re.sub(r"[^a-zA-Z0-9]+", "", str(label)).lower()
|
|
|
|
contained = {e.get("target") for e in all_edges if e.get("relation") == "contains"}
|
|
|
|
type_def_nids: dict[str, list[str]] = {}
|
|
node_by_id: dict[str, dict] = {}
|
|
for n in all_nodes:
|
|
node_by_id[n.get("id")] = n
|
|
if n.get("source_file") and n.get("id") in contained and _is_type_like_definition(n):
|
|
type_def_nids.setdefault(_key(n.get("label", "")), []).append(n["id"])
|
|
|
|
# (type_node_id, method_key) -> method_node_id, and caller -> enclosing type.
|
|
# C# owns its methods via `method` edges.
|
|
method_index: dict[tuple[str, str], str] = {}
|
|
enclosing_type: dict[str, str] = {}
|
|
for e in all_edges:
|
|
if e.get("relation") != "method":
|
|
continue
|
|
src, tgt = e.get("source"), e.get("target")
|
|
tnode = node_by_id.get(tgt)
|
|
if tnode is None:
|
|
continue
|
|
enclosing_type.setdefault(tgt, src)
|
|
method_index[(src, _key(tnode.get("label", "")))] = tgt
|
|
|
|
all_raw_calls: list[dict] = []
|
|
for result in per_file:
|
|
all_raw_calls.extend(result.get("raw_calls", []))
|
|
|
|
existing_pairs = {(e.get("source"), e.get("target")) for e in all_edges}
|
|
for rc in all_raw_calls:
|
|
if rc.get("lang") != "csharp" or not rc.get("is_member_call"):
|
|
continue
|
|
receiver = rc.get("receiver")
|
|
callee = rc.get("callee")
|
|
caller = rc.get("caller_nid")
|
|
if not receiver or not callee or not caller:
|
|
continue
|
|
src_file = rc.get("source_file", "")
|
|
if receiver == "this":
|
|
type_nid = enclosing_type.get(caller)
|
|
if not type_nid:
|
|
continue
|
|
type_qualified = True
|
|
elif receiver[:1].isupper():
|
|
# Type.M() — the type is named explicitly (also covers a Pascal-cased
|
|
# local whose name equals its type, resolved via the table below if the
|
|
# explicit-type lookup misses).
|
|
type_defs = type_def_nids.get(_key(receiver), [])
|
|
if len(type_defs) != 1:
|
|
type_name = type_table_by_file.get(src_file, {}).get(receiver)
|
|
type_defs = type_def_nids.get(_key(type_name), []) if type_name else []
|
|
if len(type_defs) != 1:
|
|
continue
|
|
type_nid = type_defs[0]
|
|
type_qualified = True
|
|
else:
|
|
type_name = type_table_by_file.get(src_file, {}).get(receiver)
|
|
if not type_name:
|
|
continue
|
|
type_defs = type_def_nids.get(_key(type_name), [])
|
|
if len(type_defs) != 1: # ambiguous or absent -> bail (god-node guard)
|
|
continue
|
|
type_nid = type_defs[0]
|
|
type_qualified = False
|
|
method_nid = method_index.get((type_nid, _key(callee)))
|
|
if not method_nid:
|
|
continue # receiver typed, but the type has no such method — skip
|
|
if method_nid == caller or (caller, method_nid) in existing_pairs:
|
|
continue
|
|
existing_pairs.add((caller, method_nid))
|
|
all_edges.append({
|
|
"source": caller,
|
|
"target": method_nid,
|
|
"relation": "calls",
|
|
"context": "call",
|
|
"confidence": "EXTRACTED" if type_qualified else "INFERRED",
|
|
"confidence_score": 1.0 if type_qualified else 0.8,
|
|
"source_file": src_file,
|
|
"source_location": rc.get("source_location"),
|
|
"weight": 1.0,
|
|
})
|
|
|
|
|
|
def _resolve_java_member_calls(
|
|
per_file: list[dict],
|
|
all_nodes: list[dict],
|
|
all_edges: list[dict],
|
|
) -> None:
|
|
"""Resolve Java member calls against the receiver's declared type.
|
|
|
|
Explicit type receivers and ``this`` are exact. Fields declared on the
|
|
caller's class plus method parameters and explicit locals are inferred from
|
|
the extractor's method-scoped type table. A missing or ambiguous receiver
|
|
type is skipped rather than falling back to a bare method-name match.
|
|
"""
|
|
def key(label: str) -> str:
|
|
return str(label).strip().removeprefix(".").removesuffix("()")
|
|
|
|
contained = {edge.get("target") for edge in all_edges
|
|
if edge.get("relation") == "contains"}
|
|
node_by_id = {node.get("id"): node for node in all_nodes}
|
|
|
|
type_def_nids: dict[str, list[str]] = {}
|
|
for node in all_nodes:
|
|
if (
|
|
node.get("source_file")
|
|
and node.get("id") in contained
|
|
and _is_type_like_definition(node)
|
|
):
|
|
type_def_nids.setdefault(key(node.get("label", "")), []).append(node["id"])
|
|
|
|
method_index: dict[tuple[str, str], set[str]] = {}
|
|
enclosing_type: dict[str, str] = {}
|
|
for edge in all_edges:
|
|
if edge.get("relation") != "method":
|
|
continue
|
|
owner, method = edge.get("source"), edge.get("target")
|
|
method_node = node_by_id.get(method)
|
|
if method_node is None:
|
|
continue
|
|
enclosing_type.setdefault(method, owner)
|
|
method_index.setdefault((owner, key(method_node.get("label", ""))), set()).add(method)
|
|
|
|
existing_pairs = {(edge.get("source"), edge.get("target")) for edge in all_edges}
|
|
for result in per_file:
|
|
for raw_call in result.get("raw_calls", []):
|
|
if raw_call.get("lang") != "java" or not raw_call.get("is_member_call"):
|
|
continue
|
|
receiver = raw_call.get("receiver")
|
|
callee = raw_call.get("callee")
|
|
caller = raw_call.get("caller_nid")
|
|
if not receiver or not callee or not caller:
|
|
continue
|
|
|
|
exact = False
|
|
if receiver == "this":
|
|
type_nid = enclosing_type.get(caller)
|
|
exact = True
|
|
if not type_nid:
|
|
continue
|
|
else:
|
|
type_name = raw_call.get("receiver_type")
|
|
if not type_name and receiver[:1].isupper():
|
|
type_name = receiver
|
|
exact = True
|
|
if not type_name:
|
|
continue
|
|
type_defs = type_def_nids.get(key(type_name), [])
|
|
if len(type_defs) != 1:
|
|
continue
|
|
type_nid = type_defs[0]
|
|
|
|
method_nids = method_index.get((type_nid, key(callee)), set())
|
|
if len(method_nids) != 1:
|
|
continue
|
|
method_nid = next(iter(method_nids))
|
|
if method_nid == caller or (caller, method_nid) in existing_pairs:
|
|
continue
|
|
existing_pairs.add((caller, method_nid))
|
|
all_edges.append({
|
|
"source": caller,
|
|
"target": method_nid,
|
|
"relation": "calls",
|
|
"context": "call",
|
|
"confidence": "EXTRACTED" if exact else "INFERRED",
|
|
"confidence_score": 1.0 if exact else 0.8,
|
|
"source_file": raw_call.get("source_file", ""),
|
|
"source_location": raw_call.get("source_location"),
|
|
"weight": 1.0,
|
|
})
|
|
|
|
|
|
def _resolve_objc_member_calls(
|
|
per_file: list[dict],
|
|
all_nodes: list[dict],
|
|
all_edges: list[dict],
|
|
) -> None:
|
|
"""Resolve cross-file Objective-C message sends (``[recv sel]``) to the real
|
|
definition of the receiver's type (#1556).
|
|
|
|
The ObjC extractor keeps its same-file selector matching (alloc/init refs,
|
|
dot-syntax accesses, @selector) and additionally emits ``raw_calls`` for every
|
|
message send, with the receiver and the reconstructed selector as the callee.
|
|
This pass types the receiver and emits a cross-file ``calls`` edge ONLY when the
|
|
type resolves to exactly ONE definition (the god-node guard).
|
|
|
|
Receiver typing:
|
|
* ``self`` / ``super`` — the caller's own enclosing class -> EXTRACTED.
|
|
* Capitalized receiver (``[Foo new]``) — the type named explicitly -> EXTRACTED.
|
|
* ``[f doThing]`` — ``f`` typed via the file's ``Foo *f`` local table -> INFERRED.
|
|
An uninferable receiver is SKIPPED (no guess), so an ambiguous selector across
|
|
classes never fans out. ``_merge_decl_def_classes`` folds each @interface/@impl
|
|
pair into one node, so a paired class clears the single-definition guard.
|
|
|
|
Must run after id-disambiguation so node ids and caller_nids are final.
|
|
"""
|
|
type_table_by_file: dict[str, dict[str, str]] = {}
|
|
for result in per_file:
|
|
tt = result.get("objc_type_table")
|
|
if tt and tt.get("path"):
|
|
type_table_by_file[tt["path"]] = tt.get("table", {})
|
|
|
|
def _key(label: str) -> str:
|
|
return re.sub(r"[^a-zA-Z0-9]+", "", str(label)).lower()
|
|
|
|
contained = {e.get("target") for e in all_edges if e.get("relation") == "contains"}
|
|
|
|
type_def_nids: dict[str, list[str]] = {}
|
|
node_by_id: dict[str, dict] = {}
|
|
for n in all_nodes:
|
|
node_by_id[n.get("id")] = n
|
|
if n.get("source_file") and n.get("id") in contained and _is_type_like_definition(n):
|
|
type_def_nids.setdefault(_key(n.get("label", "")), []).append(n["id"])
|
|
|
|
method_index: dict[tuple[str, str], str] = {}
|
|
enclosing_type: dict[str, str] = {}
|
|
for e in all_edges:
|
|
if e.get("relation") != "method":
|
|
continue
|
|
src, tgt = e.get("source"), e.get("target")
|
|
enclosing_type.setdefault(tgt, src)
|
|
tnode = node_by_id.get(tgt)
|
|
if tnode is not None:
|
|
# ObjC method labels carry a +/- sigil (`-doThing`); strip it so the
|
|
# selector `doThing` keys to the method.
|
|
method_index[(src, _key(tnode.get("label", "")))] = tgt
|
|
|
|
all_raw_calls: list[dict] = []
|
|
for result in per_file:
|
|
all_raw_calls.extend(result.get("raw_calls", []))
|
|
|
|
existing_pairs = {(e.get("source"), e.get("target")) for e in all_edges}
|
|
for rc in all_raw_calls:
|
|
if not rc.get("is_member_call"):
|
|
continue
|
|
receiver = rc.get("receiver")
|
|
callee = rc.get("callee")
|
|
caller = rc.get("caller_nid")
|
|
if not receiver or not callee or not caller:
|
|
continue
|
|
src_file = rc.get("source_file", "")
|
|
if rc.get("lang") != "objc":
|
|
continue
|
|
if receiver in ("self", "super"):
|
|
type_nid = enclosing_type.get(caller)
|
|
if not type_nid:
|
|
continue
|
|
type_qualified = True
|
|
elif receiver[:1].isupper():
|
|
type_defs = type_def_nids.get(_key(receiver), [])
|
|
if len(type_defs) != 1: # ambiguous or absent -> bail (god-node guard)
|
|
continue
|
|
type_nid = type_defs[0]
|
|
type_qualified = True
|
|
else:
|
|
type_name = type_table_by_file.get(src_file, {}).get(receiver)
|
|
if not type_name:
|
|
continue
|
|
type_defs = type_def_nids.get(_key(type_name), [])
|
|
if len(type_defs) != 1: # ambiguous or absent -> bail (god-node guard)
|
|
continue
|
|
type_nid = type_defs[0]
|
|
type_qualified = False
|
|
method_nid = method_index.get((type_nid, _key(callee)))
|
|
target = method_nid or type_nid
|
|
relation = "calls" if method_nid else "references"
|
|
if target == caller or (caller, target) in existing_pairs:
|
|
continue
|
|
existing_pairs.add((caller, target))
|
|
all_edges.append({
|
|
"source": caller,
|
|
"target": target,
|
|
"relation": relation,
|
|
"context": "call",
|
|
"confidence": "EXTRACTED" if type_qualified else "INFERRED",
|
|
"confidence_score": 1.0 if type_qualified else 0.8,
|
|
"source_file": src_file,
|
|
"source_location": rc.get("source_location"),
|
|
"weight": 1.0,
|
|
})
|
|
|
|
|
|
# Register the cross-file, language-specific member-call resolvers into the shared
|
|
# registry (framework lives in graphify.resolver_registry). A new language plugs in
|
|
# by adding one register() call below — no edits to extract()'s body. Order
|
|
# preserved from the prior inlined wiring: Swift (#1356) before Python (#1446).
|
|
register_language_resolver(
|
|
LanguageResolver("swift_member_calls", frozenset({".swift"}), _resolve_swift_member_calls)
|
|
)
|
|
register_language_resolver(
|
|
LanguageResolver("python_member_calls", frozenset({".py"}), _resolve_python_member_calls)
|
|
)
|
|
# Ruby type-aware member-call resolution (Class.new + typed var.method). Lives in
|
|
# graphify.ruby_resolution; registered here as a second consumer of the framework.
|
|
register_language_resolver(
|
|
LanguageResolver("ruby_member_calls", frozenset({".rb", ".rake"}), resolve_ruby_member_calls)
|
|
)
|
|
register_language_resolver(
|
|
LanguageResolver("typescript_member_calls", frozenset({".ts", ".tsx", ".mts", ".cts", ".js", ".jsx"}), _resolve_typescript_member_calls)
|
|
)
|
|
# C++ (#1547) and ObjC (#1556) receiver-typed member-call resolution. `.h` is in
|
|
# both suffix sets because it routes to extract_cpp or extract_objc by content; the
|
|
# resolvers each claim only their own raw_calls via the extractor-stamped `lang`.
|
|
register_language_resolver(
|
|
LanguageResolver(
|
|
"cpp_member_calls",
|
|
frozenset({".cpp", ".cc", ".cxx", ".hpp", ".cu", ".cuh", ".metal", ".h"}),
|
|
_resolve_cpp_member_calls,
|
|
)
|
|
)
|
|
register_language_resolver(
|
|
LanguageResolver(
|
|
"objc_member_calls",
|
|
frozenset({".m", ".mm", ".h"}),
|
|
_resolve_objc_member_calls,
|
|
)
|
|
)
|
|
# C# receiver-typed member-call resolution (#1609): `field/param/local.Method()`
|
|
# bound to the receiver's declared type instead of a bare same-named match.
|
|
register_language_resolver(
|
|
LanguageResolver("csharp_member_calls", frozenset({".cs"}), _resolve_csharp_member_calls)
|
|
)
|
|
register_language_resolver(
|
|
LanguageResolver("java_member_calls", frozenset({".java"}), _resolve_java_member_calls)
|
|
)
|
|
# Pascal/Delphi cross-file inherited-method-call resolution: a call from a
|
|
# manual descendant class to a method it inherits from an ancestor declared
|
|
# in a DIFFERENT file (the common generated-base/manual-descendant split,
|
|
# e.g. Sistec's Th0Xxx/Th5Xxx) falls outside the per-file extractor's own
|
|
# scope. Lives in graphify.pascal_resolution; registered here as a consumer
|
|
# of the framework, same as the Ruby resolver above.
|
|
register_language_resolver(
|
|
LanguageResolver(
|
|
"pascal_inherited_calls",
|
|
frozenset({".pas", ".pp", ".dpr", ".dpk", ".inc"}),
|
|
resolve_pascal_inherited_calls,
|
|
)
|
|
)
|
|
|
|
|
|
# Inline markdown link: [text](target "optional title"). The negative lookbehind
|
|
# excludes images (). The target stops at whitespace/closing paren so
|
|
# an optional "title" after the URL is dropped; an optional <...> wrapper is too.
|
|
# Reference-style link definition line: [label]: target "optional title"
|
|
# Obsidian-style wikilink: [[target]] / [[target|alias]] / [[target#anchor]].
|
|
|
|
# Extensions graphify creates document file nodes for. A link to one of these
|
|
# resolves to that file's node; links to code/assets are skipped (left to the
|
|
# language extractors).
|
|
|
|
|
|
# ── Pascal / Delphi extractor ─────────────────────────────────────────────────
|
|
|
|
|
|
# Size cap for project XML files we parse with stdlib ElementTree.
|
|
# Real .csproj/.fsproj/.vbproj/.lpk files are well under 2 MiB; anything
|
|
# larger is either malformed or hostile.
|
|
_PROJECT_XML_MAX_BYTES = 2 * 1024 * 1024
|
|
|
|
|
|
def _project_xml_is_safe(src: bytes) -> bool:
|
|
"""Reject XML that declares DTDs or entities.
|
|
|
|
Stdlib ``xml.etree.ElementTree`` does not cap entity expansion, so a
|
|
crafted project file could trigger a billion-laughs style DoS. External
|
|
entity resolution is already disabled by pyexpat defaults, but rejecting
|
|
``<!DOCTYPE`` / ``<!ENTITY`` outright is defense in depth.
|
|
|
|
Legitimate MSBuild and Lazarus package files never contain a DOCTYPE
|
|
or ENTITY declaration, so this is a zero-false-positive screen.
|
|
"""
|
|
# Only the prolog can hold a DTD/internal subset, but be conservative
|
|
# and scan the full byte range -- these formats use ASCII tags so a
|
|
# case-insensitive substring match is sufficient.
|
|
lowered = src.lower()
|
|
return b"<!doctype" not in lowered and b"<!entity" not in lowered
|
|
|
|
|
|
def extract_lazarus_package(path: Path) -> dict:
|
|
"""Extract package metadata from Lazarus .lpk package files (XML format).
|
|
|
|
.lpk is an XML file listing the package name, required dependencies,
|
|
and the Pascal units that belong to the package.
|
|
|
|
Produces nodes for:
|
|
- The package file itself
|
|
- The package (by name)
|
|
- Each required package (dependency)
|
|
- Each listed unit file (resolved to path-based IDs where possible)
|
|
|
|
Produces edges for:
|
|
- file --contains--> package
|
|
- package --imports--> required dependency (context: "import")
|
|
- package --contains--> listed unit
|
|
"""
|
|
try:
|
|
import xml.etree.ElementTree as ET
|
|
src = path.read_bytes()
|
|
except OSError as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
if len(src) > _PROJECT_XML_MAX_BYTES:
|
|
return {"nodes": [], "edges": [], "error": "package file too large"}
|
|
if not _project_xml_is_safe(src):
|
|
return {"nodes": [], "edges": [],
|
|
"error": "refusing XML with DOCTYPE/ENTITY declaration"}
|
|
|
|
try:
|
|
xml_root = ET.fromstring(src)
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
str_path = str(path)
|
|
stem = _file_stem(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
|
|
def add_node(nid: str, label: str) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({
|
|
"id": nid, "label": label, "file_type": "code",
|
|
"source_file": str_path, "source_location": "L1",
|
|
})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, context: str | None = None) -> None:
|
|
edge: dict[str, Any] = {
|
|
"source": src, "target": tgt, "relation": relation,
|
|
"confidence": "EXTRACTED", "source_file": str_path,
|
|
"source_location": "L1", "weight": 1.0,
|
|
}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name)
|
|
|
|
name_elem = xml_root.find(".//Package/Name")
|
|
pkg_name = name_elem.get("Value") if name_elem is not None else path.stem
|
|
pkg_nid = _make_id(stem, pkg_name)
|
|
add_node(pkg_nid, pkg_name)
|
|
add_edge(file_nid, pkg_nid, "contains")
|
|
|
|
# Required packages → imports edges
|
|
for item in xml_root.findall(".//RequiredPkgs/"):
|
|
dep_elem = item.find("PackageName")
|
|
if dep_elem is not None:
|
|
dep_name = dep_elem.get("Value", "")
|
|
if dep_name:
|
|
dep_nid = _make_id(dep_name)
|
|
add_node(dep_nid, dep_name)
|
|
add_edge(pkg_nid, dep_nid, "imports", context="import")
|
|
|
|
# Listed units → contains edges, resolved to path-based IDs where possible
|
|
for item in xml_root.findall(".//Files/"):
|
|
unit_elem = item.find("UnitName")
|
|
if unit_elem is not None:
|
|
unit_name = unit_elem.get("Value", "")
|
|
if unit_name:
|
|
unit_nid = _pascal_resolve_unit(path, unit_name)
|
|
add_node(unit_nid, unit_name)
|
|
add_edge(pkg_nid, unit_nid, "contains")
|
|
|
|
return {"nodes": nodes, "edges": edges, "input_tokens": 0, "output_tokens": 0}
|
|
|
|
|
|
# ── Main extract and collect_files ────────────────────────────────────────────
|
|
|
|
|
|
def _check_tree_sitter_version() -> None:
|
|
"""Raise a clear error if tree-sitter is too old for the new Language API."""
|
|
try:
|
|
from tree_sitter import LANGUAGE_VERSION
|
|
except ImportError:
|
|
raise ImportError(
|
|
"tree-sitter is not installed. Run: pip install 'tree-sitter>=0.23.0'"
|
|
)
|
|
# Language API v2 starts at LANGUAGE_VERSION 14
|
|
if LANGUAGE_VERSION < 14:
|
|
import tree_sitter as _ts
|
|
raise RuntimeError(
|
|
f"tree-sitter {getattr(_ts, '__version__', 'unknown')} is too old. "
|
|
f"graphify requires tree-sitter >= 0.23.0 (Language API v2). "
|
|
f"Run: pip install --upgrade tree-sitter"
|
|
)
|
|
|
|
|
|
# ── .NET project files (.sln, .slnx, .csproj, .razor) ───────────────────────
|
|
|
|
|
|
def extract_slnx(path: Path) -> dict:
|
|
"""Extract projects and inter-project dependencies from a .slnx file.
|
|
|
|
.slnx is the XML-based replacement for the legacy .sln format. Projects
|
|
are listed as ``<Project Path="..."/>`` elements (optionally nested inside
|
|
``<Folder>`` elements) and build-order dependencies as ``<BuildDependency
|
|
Project="..."/>`` children. Unlike .sln there are no GUIDs -- projects are
|
|
identified by their path.
|
|
"""
|
|
import xml.etree.ElementTree as ET
|
|
|
|
try:
|
|
src = path.read_bytes()
|
|
except OSError:
|
|
return {"nodes": [], "edges": [], "error": f"cannot read {path}"}
|
|
|
|
if len(src) > _PROJECT_XML_MAX_BYTES:
|
|
return {"nodes": [], "edges": [], "error": "project file too large"}
|
|
if not _project_xml_is_safe(src):
|
|
return {"nodes": [], "edges": [],
|
|
"error": "refusing XML with DOCTYPE/ENTITY declaration"}
|
|
|
|
try:
|
|
tree = ET.fromstring(src)
|
|
except ET.ParseError as e:
|
|
return {"nodes": [], "edges": [], "error": f"XML parse error: {e}"}
|
|
|
|
file_nid = _make_id(str(path))
|
|
str_path = str(path)
|
|
nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code",
|
|
"source_file": str_path, "source_location": None}]
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
seen_ids.add(file_nid)
|
|
|
|
ns = ""
|
|
if tree.tag.startswith("{"):
|
|
ns = tree.tag.split("}")[0] + "}"
|
|
|
|
def _resolve(proj_path: str) -> str:
|
|
proj_path = proj_path.replace("\\", "/")
|
|
try:
|
|
return str((path.parent / proj_path).resolve())
|
|
except Exception:
|
|
return proj_path
|
|
|
|
# First pass: collect projects (anywhere in the tree, incl. <Folder>).
|
|
project_nids: set[str] = set()
|
|
for proj in tree.iter(f"{ns}Project"):
|
|
proj_path = proj.get("Path")
|
|
if not proj_path:
|
|
continue
|
|
abs_proj = _resolve(proj_path)
|
|
proj_nid = _make_id(abs_proj)
|
|
if proj_nid and proj_nid not in seen_ids:
|
|
seen_ids.add(proj_nid)
|
|
label = Path(proj_path).stem
|
|
nodes.append({"id": proj_nid, "label": label,
|
|
"file_type": "code", "source_file": abs_proj,
|
|
"source_location": None})
|
|
edges.append({"source": file_nid, "target": proj_nid,
|
|
"relation": "contains", "confidence": "EXTRACTED",
|
|
"source_file": str_path, "weight": 1.0})
|
|
if proj_nid:
|
|
project_nids.add(proj_nid)
|
|
|
|
# Second pass: build-order dependencies between known projects.
|
|
for proj in tree.iter(f"{ns}Project"):
|
|
proj_path = proj.get("Path")
|
|
if not proj_path:
|
|
continue
|
|
from_nid = _make_id(_resolve(proj_path))
|
|
for dep in proj.iter(f"{ns}BuildDependency"):
|
|
dep_path = dep.get("Project")
|
|
if not dep_path:
|
|
continue
|
|
to_nid = _make_id(_resolve(dep_path))
|
|
if (from_nid and to_nid and from_nid != to_nid
|
|
and to_nid in project_nids):
|
|
edges.append({"source": from_nid, "target": to_nid,
|
|
"relation": "imports", "confidence": "EXTRACTED",
|
|
"source_file": str_path, "weight": 1.0})
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
def extract_csproj(path: Path) -> dict:
|
|
"""Extract packages, project refs, and target framework from a .csproj/.fsproj/.vbproj."""
|
|
import xml.etree.ElementTree as ET
|
|
|
|
try:
|
|
src = path.read_bytes()
|
|
except OSError:
|
|
return {"nodes": [], "edges": [], "error": f"cannot read {path}"}
|
|
|
|
if len(src) > _PROJECT_XML_MAX_BYTES:
|
|
return {"nodes": [], "edges": [], "error": "project file too large"}
|
|
if not _project_xml_is_safe(src):
|
|
return {"nodes": [], "edges": [],
|
|
"error": "refusing XML with DOCTYPE/ENTITY declaration"}
|
|
|
|
try:
|
|
tree = ET.fromstring(src)
|
|
except ET.ParseError as e:
|
|
return {"nodes": [], "edges": [], "error": f"XML parse error: {e}"}
|
|
|
|
file_nid = _make_id(str(path))
|
|
str_path = str(path)
|
|
nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code",
|
|
"source_file": str_path, "source_location": None}]
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
seen_ids.add(file_nid)
|
|
|
|
ns = ""
|
|
root_tag = tree.tag
|
|
if root_tag.startswith("{"):
|
|
ns = root_tag.split("}")[0] + "}"
|
|
|
|
def find_all(tag: str):
|
|
return tree.iter(f"{ns}{tag}")
|
|
|
|
for tf in find_all("TargetFramework"):
|
|
if tf.text:
|
|
fw_nid = _make_id("framework", tf.text.strip())
|
|
if fw_nid and fw_nid not in seen_ids:
|
|
seen_ids.add(fw_nid)
|
|
nodes.append({"id": fw_nid, "label": tf.text.strip(),
|
|
"file_type": "concept", "source_file": str_path,
|
|
"source_location": None})
|
|
edges.append({"source": file_nid, "target": fw_nid,
|
|
"relation": "references", "confidence": "EXTRACTED",
|
|
"source_file": str_path, "weight": 1.0})
|
|
|
|
for tf in find_all("TargetFrameworks"):
|
|
if tf.text:
|
|
for fw in tf.text.strip().split(";"):
|
|
fw = fw.strip()
|
|
if fw:
|
|
fw_nid = _make_id("framework", fw)
|
|
if fw_nid and fw_nid not in seen_ids:
|
|
seen_ids.add(fw_nid)
|
|
nodes.append({"id": fw_nid, "label": fw,
|
|
"file_type": "concept", "source_file": str_path,
|
|
"source_location": None})
|
|
edges.append({"source": file_nid, "target": fw_nid,
|
|
"relation": "references", "confidence": "EXTRACTED",
|
|
"source_file": str_path, "weight": 1.0})
|
|
|
|
for pkg in find_all("PackageReference"):
|
|
name = pkg.get("Include") or pkg.get("include") or ""
|
|
version = pkg.get("Version") or pkg.get("version") or ""
|
|
if not name:
|
|
continue
|
|
pkg_nid = _make_id("nuget", name)
|
|
label = f"{name} ({version})" if version else name
|
|
if pkg_nid and pkg_nid not in seen_ids:
|
|
seen_ids.add(pkg_nid)
|
|
nodes.append({"id": pkg_nid, "label": label,
|
|
"file_type": "code", "source_file": str_path,
|
|
"source_location": None})
|
|
edges.append({"source": file_nid, "target": pkg_nid,
|
|
"relation": "imports", "confidence": "EXTRACTED",
|
|
"source_file": str_path, "weight": 1.0})
|
|
|
|
for proj in find_all("ProjectReference"):
|
|
ref_path = proj.get("Include") or proj.get("include") or ""
|
|
if not ref_path:
|
|
continue
|
|
ref_path_norm = ref_path.replace("\\", "/")
|
|
try:
|
|
abs_ref = str((path.parent / ref_path_norm).resolve())
|
|
except Exception:
|
|
abs_ref = ref_path_norm
|
|
proj_nid = _make_id(abs_ref)
|
|
if proj_nid and proj_nid not in seen_ids:
|
|
seen_ids.add(proj_nid)
|
|
proj_label = Path(ref_path_norm).name
|
|
nodes.append({"id": proj_nid, "label": proj_label,
|
|
"file_type": "code", "source_file": abs_ref,
|
|
"source_location": None})
|
|
edges.append({"source": file_nid, "target": proj_nid,
|
|
"relation": "imports", "confidence": "EXTRACTED",
|
|
"source_file": str_path, "weight": 1.0})
|
|
|
|
sdk = tree.get("Sdk") or ""
|
|
if sdk:
|
|
sdk_nid = _make_id("sdk", sdk)
|
|
if sdk_nid and sdk_nid not in seen_ids:
|
|
seen_ids.add(sdk_nid)
|
|
nodes.append({"id": sdk_nid, "label": sdk,
|
|
"file_type": "concept", "source_file": str_path,
|
|
"source_location": None})
|
|
edges.append({"source": file_nid, "target": sdk_nid,
|
|
"relation": "references", "confidence": "EXTRACTED",
|
|
"source_file": str_path, "weight": 1.0})
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
def _xml_local_name(name: str) -> str:
|
|
return name.rsplit("}", 1)[-1] if name.startswith("{") else name
|
|
|
|
|
|
# A .NET event handler has the signature `(object sender, <T>EventArgs e)`. Used
|
|
# to tell a real event handler in the code-behind apart from an ordinary method
|
|
# whose name a XAML attribute value happens to match. Tolerates `object?`, a
|
|
# namespace-qualified args type, and a generic `EventArgs<T>`.
|
|
_EVENT_HANDLER_SIGNATURE_RE = re.compile(
|
|
r"\(\s*object\??\s+\w+\s*,\s*[\w.]*EventArgs(?:<[^>]*>)?\s+\w+\s*\)"
|
|
)
|
|
|
|
# XAML attribute names that carry free-form strings or identifiers and never name
|
|
# an event handler. They are skipped when matching attribute values to code-behind
|
|
# methods so e.g. Content="Save" or Tag="Refresh" can't fabricate an event edge.
|
|
_XAML_NON_EVENT_ATTRS = frozenset({
|
|
"Name", "Content", "Text", "Title", "Tag", "ToolTip", "Header",
|
|
"Class", "Key", "Uid", "DataContext", "Style", "Source",
|
|
})
|
|
|
|
# A handler attribute value is a bare method name (e.g. Click="Save_Click"), not
|
|
# markup, a path, or a sentence. Used to skip values like "{Binding ...}" or
|
|
# free-form content before looking them up as code-behind methods.
|
|
_XAML_IDENT_RE = re.compile(r"[A-Za-z_]\w*")
|
|
_XAML_DESIGN_INSTANCE_TYPE_RE = re.compile(
|
|
r"\bType\s*=\s*(?:\{x:Type\s+)?(?P<type>[\w.:+]+)"
|
|
)
|
|
|
|
|
|
def _xaml_markup_extension(value: str) -> tuple[str, str] | None:
|
|
value = value.strip()
|
|
if not (value.startswith("{") and value.endswith("}")):
|
|
return None
|
|
inner = value[1:-1].strip()
|
|
if not inner or inner.startswith("}"):
|
|
return None
|
|
name, _, args = inner.partition(" ")
|
|
return name, args.strip()
|
|
|
|
|
|
def _xaml_split_markup_args(args: str) -> list[str]:
|
|
parts: list[str] = []
|
|
start = 0
|
|
depth = 0
|
|
for idx, ch in enumerate(args):
|
|
if ch == "{":
|
|
depth += 1
|
|
elif ch == "}" and depth:
|
|
depth -= 1
|
|
elif ch == "," and depth == 0:
|
|
parts.append(args[start:idx].strip())
|
|
start = idx + 1
|
|
tail = args[start:].strip()
|
|
if tail:
|
|
parts.append(tail)
|
|
return parts
|
|
|
|
|
|
def _xaml_static_resource_key(value: str) -> str | None:
|
|
markup = _xaml_markup_extension(value)
|
|
if not markup:
|
|
return None
|
|
name, args = markup
|
|
if name != "StaticResource":
|
|
return None
|
|
for part in _xaml_split_markup_args(args):
|
|
if "=" not in part:
|
|
return part.strip() or None
|
|
key, resource = part.split("=", 1)
|
|
if key.strip() == "ResourceKey":
|
|
return resource.strip() or None
|
|
return None
|
|
|
|
|
|
def _xaml_binding_refs(value: str) -> tuple[str | None, str | None]:
|
|
markup = _xaml_markup_extension(value)
|
|
if not markup:
|
|
return None, None
|
|
name, args = markup
|
|
if name != "Binding":
|
|
return None, None
|
|
|
|
path_ref = None
|
|
converter_ref = None
|
|
for part in _xaml_split_markup_args(args):
|
|
if not part:
|
|
continue
|
|
if "=" not in part:
|
|
if path_ref is None:
|
|
path_ref = part.strip()
|
|
continue
|
|
key, raw_value = part.split("=", 1)
|
|
key = key.strip()
|
|
raw_value = raw_value.strip()
|
|
if key == "Path":
|
|
path_ref = raw_value
|
|
elif key == "Converter":
|
|
converter_ref = _xaml_static_resource_key(raw_value)
|
|
|
|
if path_ref and ("{" in path_ref or "}" in path_ref):
|
|
path_ref = None
|
|
return path_ref or None, converter_ref or None
|
|
|
|
|
|
def _xaml_codebehind_path(path: Path) -> Path | None:
|
|
expected = path.with_suffix(path.suffix + ".cs")
|
|
if expected.exists():
|
|
return expected
|
|
try:
|
|
for sibling in path.parent.iterdir():
|
|
if sibling.name.casefold() == expected.name.casefold():
|
|
return sibling
|
|
except OSError:
|
|
return None
|
|
return None
|
|
|
|
|
|
def _xaml_codebehind_symbols(
|
|
path: Path,
|
|
class_name: str | None,
|
|
) -> tuple[dict | None, dict[str, dict], list[dict]]:
|
|
codebehind = _xaml_codebehind_path(path)
|
|
if not codebehind:
|
|
return None, {}, []
|
|
result = extract_csharp(codebehind)
|
|
if result.get("error"):
|
|
return None, {}, []
|
|
|
|
class_simple = class_name.rsplit(".", 1)[-1] if class_name else None
|
|
class_node = None
|
|
if class_simple:
|
|
for node in result.get("nodes", []):
|
|
if node.get("label") == class_simple:
|
|
class_node = node
|
|
break
|
|
|
|
class_method_edges: list[dict] = []
|
|
if class_node:
|
|
class_id = class_node.get("id")
|
|
for edge in result.get("edges", []):
|
|
if edge.get("source") == class_id and edge.get("relation") == "method":
|
|
class_method_edges.append(edge)
|
|
method_ids = {edge.get("target") for edge in class_method_edges} if class_node else None
|
|
|
|
# Only methods with a .NET event-handler signature -- (object sender,
|
|
# <T>EventArgs e) -- are eligible to be wired to a XAML attribute as an
|
|
# event. Without this gate, any attribute whose value happens to match a
|
|
# method name (e.g. Content="Save" next to a business method Save()) would
|
|
# produce a spurious "event" edge. The C# extractor does not record the
|
|
# parameter list on method nodes, so we read it from the code-behind source
|
|
# at the method's recorded line.
|
|
try:
|
|
cb_lines = codebehind.read_text(encoding="utf-8", errors="replace").splitlines()
|
|
except OSError:
|
|
cb_lines = []
|
|
|
|
def _has_event_handler_signature(node: dict) -> bool:
|
|
loc = str(node.get("source_location") or "")
|
|
m = re.match(r"L(\d+)", loc)
|
|
if not m or not cb_lines:
|
|
return False
|
|
start = int(m.group(1)) - 1
|
|
# Join a few lines so a signature split across lines still matches.
|
|
snippet = " ".join(cb_lines[start:start + 3])
|
|
return _EVENT_HANDLER_SIGNATURE_RE.search(snippet) is not None
|
|
|
|
methods: dict[str, dict] = {}
|
|
for node in result.get("nodes", []):
|
|
if method_ids is not None and node.get("id") not in method_ids:
|
|
continue
|
|
label = str(node.get("label", ""))
|
|
if label.startswith(".") and label.endswith("()") and _has_event_handler_signature(node):
|
|
methods[label.strip("()").lstrip(".")] = node
|
|
return class_node, methods, class_method_edges
|
|
|
|
|
|
def _xaml_type_simple_name(type_ref: str) -> str | None:
|
|
type_ref = type_ref.strip().strip("{}")
|
|
if not type_ref:
|
|
return None
|
|
type_ref = type_ref.split(",", 1)[0].strip()
|
|
if type_ref.startswith("x:Type "):
|
|
type_ref = type_ref[len("x:Type "):].strip()
|
|
if ":" in type_ref:
|
|
type_ref = type_ref.rsplit(":", 1)[-1]
|
|
if "." in type_ref:
|
|
type_ref = type_ref.rsplit(".", 1)[-1]
|
|
if "+" in type_ref:
|
|
type_ref = type_ref.rsplit("+", 1)[-1]
|
|
return type_ref if _XAML_IDENT_RE.fullmatch(type_ref) else None
|
|
|
|
|
|
def _xaml_explicit_viewmodel_names(tree) -> tuple[bool, list[str]]:
|
|
has_data_context = False
|
|
names: list[str] = []
|
|
for elem in tree.iter():
|
|
elem_type = _xml_local_name(elem.tag)
|
|
if elem_type.endswith(".DataContext") or elem_type == "DataContext":
|
|
has_data_context = True
|
|
for child in list(elem):
|
|
vm_name = _xaml_type_simple_name(_xml_local_name(child.tag))
|
|
if vm_name and vm_name not in names:
|
|
names.append(vm_name)
|
|
for key, value in elem.attrib.items():
|
|
if _xml_local_name(key) != "DataContext" or not value:
|
|
continue
|
|
has_data_context = True
|
|
match = _XAML_DESIGN_INSTANCE_TYPE_RE.search(value)
|
|
if match:
|
|
vm_name = _xaml_type_simple_name(match.group("type"))
|
|
if vm_name and vm_name not in names:
|
|
names.append(vm_name)
|
|
return has_data_context, names
|
|
|
|
|
|
def _xaml_prism_autowire_viewmodel(tree) -> bool:
|
|
for elem in tree.iter():
|
|
for key, value in elem.attrib.items():
|
|
if (
|
|
_xml_local_name(key).endswith("ViewModelLocator.AutoWireViewModel")
|
|
and value.strip().lower() == "true"
|
|
):
|
|
return True
|
|
return False
|
|
|
|
|
|
def _xaml_inferred_viewmodel_names(view_name: str | None) -> list[str]:
|
|
if not view_name:
|
|
return []
|
|
names: list[str] = []
|
|
|
|
def add(name: str) -> None:
|
|
if name.endswith("ViewModel") and name not in names:
|
|
names.append(name)
|
|
|
|
if view_name == "MainWindow":
|
|
add("MainWindowViewModel")
|
|
add("MainViewModel")
|
|
for suffix in ("UserControl", "View", "Page", "Control"):
|
|
if view_name.endswith(suffix) and len(view_name) > len(suffix):
|
|
add(view_name[:-len(suffix)] + "ViewModel")
|
|
break
|
|
return names
|
|
|
|
|
|
def _xaml_project_root(path: Path) -> Path:
|
|
project_markers = (".csproj", ".fsproj", ".vbproj", ".sln", ".slnx")
|
|
root = path.parent
|
|
for directory in (path.parent, *path.parent.parents):
|
|
try:
|
|
if any(child.suffix in project_markers for child in directory.iterdir()):
|
|
root = directory
|
|
break
|
|
except OSError:
|
|
continue
|
|
if _XAML_ACTIVE_EXTRACT_ROOT is None:
|
|
return root
|
|
boundary = _XAML_ACTIVE_EXTRACT_ROOT.resolve()
|
|
try:
|
|
root.resolve().relative_to(boundary)
|
|
return root
|
|
except ValueError:
|
|
return boundary
|
|
|
|
|
|
def _xaml_csharp_class_nodes(path: Path) -> dict[str, list[dict]]:
|
|
from graphify.detect import _is_ignored, _is_noise_dir, _load_graphifyignore
|
|
root = _xaml_project_root(path)
|
|
cache_key = str(root.resolve()) if _XAML_ACTIVE_EXTRACT_ROOT is not None else None
|
|
if cache_key and cache_key in _XAML_CSHARP_CLASS_CACHE:
|
|
return _XAML_CSHARP_CLASS_CACHE[cache_key]
|
|
classes: dict[str, list[dict]] = {}
|
|
patterns = _load_graphifyignore(root)
|
|
ignore_cache: dict[Path, bool] = {}
|
|
try:
|
|
cs_files = sorted(root.rglob("*.cs"))
|
|
except OSError:
|
|
return classes
|
|
for cs_path in cs_files:
|
|
if any(_is_noise_dir(part) for part in cs_path.parts):
|
|
continue
|
|
if patterns and _is_ignored(cs_path, root, patterns, _cache=ignore_cache):
|
|
continue
|
|
result = extract_csharp(cs_path)
|
|
if result.get("error"):
|
|
continue
|
|
for node in result.get("nodes", []):
|
|
label = str(node.get("label", ""))
|
|
if not label.endswith("ViewModel") or not _XAML_IDENT_RE.fullmatch(label):
|
|
continue
|
|
if node.get("source_file"):
|
|
classes.setdefault(label, []).append(node)
|
|
if cache_key:
|
|
_XAML_CSHARP_CLASS_CACHE[cache_key] = classes
|
|
return classes
|
|
|
|
|
|
def _xaml_pascal_name(name: str) -> str | None:
|
|
name = name.strip().lstrip("_")
|
|
if name.startswith("m_"):
|
|
name = name[2:]
|
|
return name[:1].upper() + name[1:] if _XAML_IDENT_RE.fullmatch(name) else None
|
|
|
|
|
|
_XAML_TOOLKIT_FIELD_RE = re.compile(r"\b(?P<name>_?m?_?[A-Za-z_]\w*)\s*(?:=.*)?;")
|
|
_XAML_TOOLKIT_METHOD_RE = re.compile(r"\b(?P<name>[A-Za-z_]\w*)\s*\(")
|
|
_XAML_ACTIVE_EXTRACT_ROOT: Path | None = None
|
|
_XAML_CSHARP_CLASS_CACHE: dict[str, dict[str, list[dict]]] = {}
|
|
|
|
|
|
def _xaml_communitytoolkit_members(vm_node: dict) -> tuple[dict[str, dict], list[dict]]:
|
|
source_file = vm_node.get("source_file")
|
|
vm_id = vm_node.get("id")
|
|
if not source_file or not vm_id:
|
|
return {}, []
|
|
try:
|
|
# errors="replace" so a non-UTF8 code-behind can't raise UnicodeDecodeError
|
|
# and abort the whole extract_xaml (matches every other reader here).
|
|
lines = Path(source_file).read_text(encoding="utf-8", errors="replace").splitlines()
|
|
except OSError:
|
|
return {}, []
|
|
|
|
members: dict[str, dict] = {}
|
|
edges: list[dict] = []
|
|
|
|
def add_member(label: str, line_no: int, context: str) -> None:
|
|
nid = _make_id(vm_id, label)
|
|
members[label] = {
|
|
"id": nid,
|
|
"label": label,
|
|
"file_type": "code",
|
|
"source_file": source_file,
|
|
"source_location": f"L{line_no}",
|
|
}
|
|
edges.append({
|
|
"source": vm_id,
|
|
"target": nid,
|
|
"relation": "defines",
|
|
"confidence": "INFERRED",
|
|
"source_file": source_file,
|
|
"source_location": f"L{line_no}",
|
|
"weight": 1.0,
|
|
"context": context,
|
|
})
|
|
|
|
pending: tuple[str, int] | None = None
|
|
for line_no, line in enumerate(lines, 1):
|
|
remainder = line.split("]", 1)[1].strip() if "]" in line else ""
|
|
if "[" in line and "ObservableProperty" in line:
|
|
pending = ("property", line_no)
|
|
if not remainder:
|
|
continue
|
|
line = remainder
|
|
if "[" in line and "RelayCommand" in line:
|
|
pending = ("command", line_no)
|
|
if not remainder:
|
|
continue
|
|
line = remainder
|
|
if not pending or not line.strip() or line.lstrip().startswith("["):
|
|
continue
|
|
|
|
kind, attr_line = pending
|
|
pending = None
|
|
if kind == "property":
|
|
match = _XAML_TOOLKIT_FIELD_RE.search(line)
|
|
label = _xaml_pascal_name(match.group("name")) if match else None
|
|
if label:
|
|
add_member(label, attr_line, "communitytoolkit_observable_property")
|
|
else:
|
|
match = _XAML_TOOLKIT_METHOD_RE.search(line)
|
|
if match:
|
|
method = match.group("name").removesuffix("Async")
|
|
add_member(f"{method}Command", attr_line, "communitytoolkit_relay_command")
|
|
|
|
return members, edges
|
|
|
|
|
|
def extract_xaml(path: Path) -> dict:
|
|
"""Extract WPF/XAML structure, bindings, x:Class, and event handler references."""
|
|
import xml.etree.ElementTree as ET
|
|
|
|
try:
|
|
src = path.read_bytes()
|
|
except OSError:
|
|
return {"nodes": [], "edges": [], "error": f"cannot read {path}"}
|
|
|
|
if len(src) > _PROJECT_XML_MAX_BYTES:
|
|
return {"nodes": [], "edges": [], "error": "xaml file too large"}
|
|
if not _project_xml_is_safe(src):
|
|
return {"nodes": [], "edges": [],
|
|
"error": "refusing XML with DOCTYPE/ENTITY declaration"}
|
|
|
|
try:
|
|
tree = ET.fromstring(src)
|
|
except ET.ParseError as e:
|
|
return {"nodes": [], "edges": [], "error": f"XML parse error: {e}"}
|
|
|
|
text = src.decode("utf-8", errors="replace")
|
|
lines = text.splitlines()
|
|
str_path = str(path)
|
|
stem = _file_stem(path)
|
|
file_nid = _make_id(str(path))
|
|
root_type = _xml_local_name(tree.tag)
|
|
root_nid = _make_id(stem, root_type)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
seen_edges: set[tuple[str, str, str, str | None]] = set()
|
|
|
|
def line_for(value: str | None) -> int:
|
|
if value:
|
|
for idx, line in enumerate(lines, 1):
|
|
if value in line:
|
|
return idx
|
|
return 1
|
|
|
|
def add_node(
|
|
nid: str,
|
|
label: str,
|
|
line: int | None,
|
|
*,
|
|
file_type: str = "code",
|
|
source_file: str = str_path,
|
|
) -> None:
|
|
if nid in seen_ids:
|
|
return
|
|
seen_ids.add(nid)
|
|
nodes.append({
|
|
"id": nid, "label": label, "file_type": file_type,
|
|
"source_file": source_file,
|
|
"source_location": f"L{line}" if line else None,
|
|
})
|
|
|
|
def add_existing_node(node: dict | None) -> None:
|
|
if not node:
|
|
return
|
|
nid = node.get("id")
|
|
if not nid or nid in seen_ids:
|
|
return
|
|
seen_ids.add(nid)
|
|
nodes.append(dict(node))
|
|
|
|
def add_edge(
|
|
src_nid: str,
|
|
tgt_nid: str,
|
|
relation: str,
|
|
line: int,
|
|
*,
|
|
context: str | None = None,
|
|
source_file: str = str_path,
|
|
confidence: str = "EXTRACTED",
|
|
) -> None:
|
|
key = (src_nid, tgt_nid, relation, context)
|
|
if key in seen_edges:
|
|
return
|
|
seen_edges.add(key)
|
|
edge = {
|
|
"source": src_nid, "target": tgt_nid, "relation": relation,
|
|
"confidence": confidence, "source_file": source_file,
|
|
"source_location": f"L{line}", "weight": 1.0,
|
|
}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
def add_existing_edge(edge: dict) -> None:
|
|
key = (edge.get("source"), edge.get("target"), edge.get("relation"), edge.get("context"))
|
|
if key in seen_edges:
|
|
return
|
|
seen_edges.add(key)
|
|
edges.append(dict(edge))
|
|
|
|
add_node(file_nid, path.name, 1)
|
|
add_node(root_nid, root_type, 1)
|
|
add_edge(file_nid, root_nid, "contains", 1)
|
|
|
|
class_name = None
|
|
for key, value in tree.attrib.items():
|
|
if _xml_local_name(key) == "Class" and value:
|
|
class_name = value.strip()
|
|
break
|
|
|
|
class_node, codebehind_methods, class_method_edges = _xaml_codebehind_symbols(path, class_name)
|
|
if class_name:
|
|
if class_node:
|
|
class_nid = class_node["id"]
|
|
add_existing_node(class_node)
|
|
else:
|
|
class_label = class_name.rsplit(".", 1)[-1]
|
|
class_nid = _make_id(stem, class_label)
|
|
add_node(class_nid, class_label, line_for(class_name))
|
|
add_edge(root_nid, class_nid, "references", line_for(class_name), context="x_class")
|
|
|
|
has_data_context, vm_names = _xaml_explicit_viewmodel_names(tree)
|
|
prism_autowire = _xaml_prism_autowire_viewmodel(tree)
|
|
vm_confidence = "EXTRACTED"
|
|
if not has_data_context:
|
|
view_name = class_name.rsplit(".", 1)[-1] if class_name else None
|
|
view_name = view_name or (path.stem if prism_autowire else None)
|
|
vm_names = _xaml_inferred_viewmodel_names(view_name)
|
|
vm_confidence = "INFERRED"
|
|
generated_members: dict[str, dict] = {}
|
|
generated_member_edges: list[dict] = []
|
|
if vm_names:
|
|
csharp_classes = _xaml_csharp_class_nodes(path)
|
|
vm_candidates = []
|
|
for vm_name in vm_names:
|
|
vm_candidates.extend(csharp_classes.get(vm_name, []))
|
|
by_id = {node.get("id"): node for node in vm_candidates if node.get("id")}
|
|
if len(by_id) == 1:
|
|
vm_node = next(iter(by_id.values()))
|
|
add_existing_node(vm_node)
|
|
add_edge(
|
|
root_nid,
|
|
vm_node["id"],
|
|
"references",
|
|
line_for(vm_node["label"]),
|
|
context="view_model",
|
|
confidence=vm_confidence,
|
|
)
|
|
generated_members, generated_member_edges = _xaml_communitytoolkit_members(vm_node)
|
|
for member in generated_members.values():
|
|
add_existing_node(member)
|
|
for member_edge in generated_member_edges:
|
|
add_existing_edge(member_edge)
|
|
|
|
for elem in tree.iter():
|
|
elem_type = _xml_local_name(elem.tag)
|
|
elem_name = None
|
|
for key, value in elem.attrib.items():
|
|
if _xml_local_name(key) == "Name" and value:
|
|
elem_name = value.strip()
|
|
break
|
|
owner_nid = root_nid
|
|
if elem_name:
|
|
owner_nid = _make_id(stem, elem_name)
|
|
add_node(owner_nid, elem_name, line_for(elem_name))
|
|
add_edge(root_nid, owner_nid, "contains", line_for(elem_name))
|
|
type_nid = _make_id("xaml", elem_type)
|
|
add_node(type_nid, elem_type, line_for(elem_name), file_type="concept")
|
|
add_edge(owner_nid, type_nid, "references", line_for(elem_name), context="type")
|
|
|
|
for key, value in elem.attrib.items():
|
|
value = value or ""
|
|
# Event wiring: an attribute references a handler only when its local
|
|
# name isn't a known free-form/identity property, its value is a bare
|
|
# identifier (a method name, not markup or a sentence), and the matched
|
|
# code-behind method actually has an event-handler signature (the gate
|
|
# in _xaml_codebehind_symbols). This stops Content="Save" / Tag="..."
|
|
# from fabricating event edges against same-named ordinary methods.
|
|
attr_local = _xml_local_name(key)
|
|
if attr_local not in _XAML_NON_EVENT_ATTRS and _XAML_IDENT_RE.fullmatch(value):
|
|
method = codebehind_methods.get(value)
|
|
if method:
|
|
add_existing_node(method)
|
|
add_edge(owner_nid, method["id"], "references", line_for(value), context="event")
|
|
for method_edge in class_method_edges:
|
|
if method_edge.get("target") == method["id"]:
|
|
add_existing_node(class_node)
|
|
add_existing_edge(method_edge)
|
|
break
|
|
binding_path, binding_converter = _xaml_binding_refs(value)
|
|
if binding_path:
|
|
bind_nid = _make_id("binding", binding_path)
|
|
add_node(bind_nid, binding_path, line_for(value), file_type="concept")
|
|
binding_context = (
|
|
"binding_command"
|
|
if attr_local == "Command" or attr_local.endswith(".Command")
|
|
else "binding_path"
|
|
)
|
|
add_edge(owner_nid, bind_nid, "references", line_for(value), context=binding_context)
|
|
generated_member = generated_members.get(binding_path)
|
|
if generated_member:
|
|
add_existing_node(generated_member)
|
|
add_edge(
|
|
owner_nid,
|
|
generated_member["id"],
|
|
"references",
|
|
line_for(value),
|
|
context=binding_context,
|
|
confidence="INFERRED",
|
|
)
|
|
if binding_converter:
|
|
converter_nid = _make_id("binding_converter", binding_converter)
|
|
add_node(converter_nid, binding_converter, line_for(value), file_type="concept")
|
|
add_edge(owner_nid, converter_nid, "references", line_for(value), context="binding_converter")
|
|
if elem_type == "Binding" and attr_local == "Path":
|
|
direct_path = value.strip()
|
|
if direct_path and "{" not in direct_path and "}" not in direct_path:
|
|
bind_nid = _make_id("binding", direct_path)
|
|
add_node(bind_nid, direct_path, line_for(value), file_type="concept")
|
|
add_edge(owner_nid, bind_nid, "references", line_for(value), context="binding_path")
|
|
if elem_type == "Binding" and attr_local == "Converter":
|
|
direct_converter = _xaml_static_resource_key(value)
|
|
if direct_converter:
|
|
converter_nid = _make_id("binding_converter", direct_converter)
|
|
add_node(converter_nid, direct_converter, line_for(value), file_type="concept")
|
|
add_edge(owner_nid, converter_nid, "references", line_for(value), context="binding_converter")
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
# Config/manifest JSON filenames the structural extractor understands. Anything
|
|
# else (eval fixtures, datasets, GeoJSON, API dumps) is *data* and must NOT be
|
|
# AST-walked into per-key nodes — that floods the graph with orphan key-nodes
|
|
# and near-duplicate communities (#1224). Data JSON is left to the LLM semantic
|
|
# pass instead. Matched case-insensitively against the bare filename.
|
|
|
|
# Top-level keys that prove a JSON object is a config/manifest the extractor can
|
|
# draw *cross-file* edges from (deps, extends chains, schema refs).
|
|
|
|
|
|
# ── DM (BYOND DreamMaker) extractor ──────────────────────────────────────────
|
|
# DM identity is path-based (`/datum/object/proc/New()`), not block-based, so
|
|
# the generic class-body walker doesn't fit well.
|
|
|
|
|
|
# ── DMI (BYOND icon files) ────────────────────────────────────────────────────
|
|
# .dmi is a PNG with a tEXt/zTXt "Description" chunk containing BYOND state
|
|
# metadata. We want the icon state names (icon_state = "X" in DM code
|
|
# references them).
|
|
|
|
|
|
# ── DMM (BYOND map files) ─────────────────────────────────────────────────────
|
|
# A .dmm starts with a tile dictionary — each "key" = (type, type{var=val}, ...)
|
|
# names one or more types that compose a tile — then a grid. We only need the
|
|
# dictionary section: every type path referenced is a `uses` edge.
|
|
|
|
|
|
# ── DMF (BYOND interface forms) ───────────────────────────────────────────────
|
|
|
|
|
|
# Head tokens in an HCL traversal that are meta/builtins, not references to a
|
|
# block defined in the corpus (count.index, each.key, self.*, path.module, ...).
|
|
|
|
|
|
_DISPATCH: dict[str, Any] = {
|
|
".py": extract_python,
|
|
".js": extract_js,
|
|
".jsx": extract_js,
|
|
".mjs": extract_js,
|
|
".ts": extract_js,
|
|
".tsx": extract_js,
|
|
".mts": extract_js,
|
|
".cts": extract_js,
|
|
".go": extract_go,
|
|
".rs": extract_rust,
|
|
".java": extract_java,
|
|
".groovy": extract_groovy,
|
|
".gradle": extract_groovy,
|
|
".c": extract_c,
|
|
".h": extract_c,
|
|
".cpp": extract_cpp,
|
|
".cc": extract_cpp,
|
|
".cxx": extract_cpp,
|
|
".hpp": extract_cpp,
|
|
".cu": extract_cpp,
|
|
".cuh": extract_cpp,
|
|
".metal": extract_cpp,
|
|
".rb": extract_ruby, ".rake": extract_ruby,
|
|
".cs": extract_csharp,
|
|
".kt": extract_kotlin,
|
|
".kts": extract_kotlin,
|
|
".scala": extract_scala,
|
|
".php": extract_php,
|
|
".swift": extract_swift,
|
|
".lua": extract_lua,
|
|
".luau": extract_lua,
|
|
".toc": extract_lua,
|
|
".zig": extract_zig,
|
|
".ps1": extract_powershell,
|
|
".psm1": extract_powershell,
|
|
".psd1": extract_powershell_manifest,
|
|
".ex": extract_elixir,
|
|
".exs": extract_elixir,
|
|
".m": extract_objc,
|
|
".mm": extract_objc,
|
|
".jl": extract_julia,
|
|
".f": extract_fortran,
|
|
".F": extract_fortran,
|
|
".f90": extract_fortran,
|
|
".F90": extract_fortran,
|
|
".f95": extract_fortran,
|
|
".F95": extract_fortran,
|
|
".f03": extract_fortran,
|
|
".F03": extract_fortran,
|
|
".f08": extract_fortran,
|
|
".F08": extract_fortran,
|
|
".vue": extract_vue,
|
|
".svelte": extract_svelte,
|
|
".astro": extract_astro,
|
|
".dart": extract_dart,
|
|
".v": extract_verilog,
|
|
".sv": extract_verilog,
|
|
".svh": extract_verilog,
|
|
".sql": extract_sql,
|
|
".md": extract_markdown,
|
|
".mdx": extract_markdown,
|
|
".qmd": extract_markdown,
|
|
".pas": extract_pascal,
|
|
".pp": extract_pascal,
|
|
".dpr": extract_pascal,
|
|
".dpk": extract_pascal,
|
|
".lpr": extract_pascal,
|
|
".inc": extract_pascal,
|
|
".dfm": extract_delphi_form,
|
|
".lfm": extract_lazarus_form,
|
|
".lpk": extract_lazarus_package,
|
|
".sh": extract_bash,
|
|
".bash": extract_bash,
|
|
".json": extract_json,
|
|
".tf": extract_terraform,
|
|
".tfvars": extract_terraform,
|
|
".hcl": extract_terraform,
|
|
".dm": extract_dm,
|
|
".dme": extract_dm,
|
|
".dmi": extract_dmi,
|
|
".dmm": extract_dmm,
|
|
".dmf": extract_dmf,
|
|
".sln": extract_sln,
|
|
".slnx": extract_slnx,
|
|
".csproj": extract_csproj,
|
|
".fsproj": extract_csproj,
|
|
".vbproj": extract_csproj,
|
|
".xaml": extract_xaml,
|
|
".razor": extract_razor,
|
|
".cshtml": extract_razor,
|
|
".cls": extract_apex,
|
|
".trigger": extract_apex,
|
|
}
|
|
|
|
|
|
# Extensions whose extractor depends on an optional-dependency extra
|
|
# (pyproject [project.optional-dependencies]) and hard-fails without it,
|
|
# rather than falling back like Pascal does. Used by the #1745 warning in
|
|
# extract() to tell the user which extra restores the language.
|
|
_EXTRA_FOR_EXTENSION = {
|
|
".sql": "sql",
|
|
".tf": "terraform",
|
|
".tfvars": "terraform",
|
|
".hcl": "terraform",
|
|
".dm": "dm",
|
|
".dme": "dm",
|
|
}
|
|
|
|
|
|
# Extensionless executables (CLI entry points like `devctl` or `manage`) carry
|
|
# their language in the shebang, not the suffix. detect.classify_file already
|
|
# routes them to the CODE path via _shebang_interpreter; _get_extractor must
|
|
# honor the same signal or these files are classified as code and then silently
|
|
# dropped by extraction. Only interpreters with a real extractor are mapped —
|
|
# detect's wider set (perl, fish, tcsh, Rscript) stays unmapped and skipped.
|
|
_SHEBANG_DISPATCH: dict[str, Any] = {
|
|
"python": extract_python,
|
|
"python2": extract_python,
|
|
"python3": extract_python,
|
|
"bash": extract_bash,
|
|
"sh": extract_bash,
|
|
"dash": extract_bash,
|
|
"zsh": extract_bash,
|
|
"ksh": extract_bash,
|
|
"node": extract_js,
|
|
"nodejs": extract_js,
|
|
"ruby": extract_ruby,
|
|
"lua": extract_lua,
|
|
"php": extract_php,
|
|
"julia": extract_julia,
|
|
}
|
|
|
|
|
|
# ObjC-only directives. They are illegal in C and C++, so finding one in a `.h`
|
|
# file is a near-zero-false-positive signal that the header is Objective-C (and so
|
|
# belongs to extract_objc, not extract_c). `@property` is deliberately excluded: it
|
|
# doubles as a Doxygen comment command and ObjC properties only ever live inside an
|
|
# @interface/@protocol anyway, so the stronger directives already cover them.
|
|
#
|
|
# `#import` is included because an ObjC *bridging* header is often nothing but
|
|
# `#import "X.h"` lines with no @interface (#1556). Routed to extract_c it parses
|
|
# `#import` as a `preproc_call` (not `preproc_include`), so every import edge is
|
|
# dropped and the header is isolated. `#import` is an ObjC-only directive (illegal
|
|
# in C and C++), so this won't hijack genuine C/C++ headers, and extract_objc
|
|
# resolves quoted imports via _resolve_c_include_path.
|
|
_OBJC_HEADER_MARKERS = (b"@interface", b"@protocol", b"@implementation", b"@import", b"#import")
|
|
|
|
|
|
def _is_objc_header(path: Path) -> bool:
|
|
"""Whether a `.h` file is Objective-C rather than C/C++ (#1475).
|
|
|
|
`.h` is shared by C, C++, and ObjC; the suffix map routes it to extract_c,
|
|
which silently drops every @interface/@protocol/@property/method (1 node, 0
|
|
edges). Sniffing for an ObjC-only directive reroutes genuine ObjC headers to
|
|
extract_objc while leaving every C/C++ header on its existing extractor.
|
|
"""
|
|
try:
|
|
head = path.read_bytes()[:256 * 1024]
|
|
except OSError:
|
|
return False
|
|
return any(marker in head for marker in _OBJC_HEADER_MARKERS)
|
|
|
|
|
|
# C++-only signals. None of these are valid in a plain C header, so finding one
|
|
# in a `.h` is a high-confidence signal the header is C++ (#1547). The C grammar
|
|
# has no class_specifier, so a `class Foo { ... };` header routed to extract_c
|
|
# loses the class and its method prototypes (a junk `foo_foo` node + a sourceless
|
|
# `class` stub); routing to extract_cpp recovers the real type. Kept CONSERVATIVE:
|
|
# a plain C header with none of these stays on extract_c. ObjC sniffing keeps
|
|
# priority (an ObjC header can legitimately contain `::`/`class` inside an inline
|
|
# C++ block when compiled as Objective-C++).
|
|
_CPP_HEADER_MARKERS = (
|
|
b"class ", b"namespace ", b"template", b"::",
|
|
b"public:", b"private:", b"protected:",
|
|
)
|
|
|
|
|
|
def _is_objc_source(path: Path) -> bool:
|
|
"""Whether a `.m` file is Objective-C rather than MATLAB/Octave (#1702).
|
|
|
|
`.m` is shared by Objective-C implementation files and MATLAB (also Octave).
|
|
The suffix map routes `.m` to extract_objc unconditionally, which force-parses
|
|
MATLAB through the Objective-C tree-sitter grammar and emits garbage nodes/edges
|
|
(worse than skipping). A genuine ObjC `.m` always carries an ObjC directive
|
|
(@implementation/@interface/@import/#import); MATLAB has none of them. Reuses
|
|
the same marker set as the `.h` sniff. `.mm` is unambiguously Objective-C++ and
|
|
is not sniffed.
|
|
"""
|
|
return _is_objc_header(path)
|
|
|
|
|
|
def _is_cpp_header(path: Path) -> bool:
|
|
"""Whether a `.h` file is C++ rather than plain C (#1547).
|
|
|
|
Mirrors `_is_objc_header`: sniffs for a C++-only token. Used only to reroute
|
|
a `.h` from extract_c to extract_cpp when no ObjC marker is present (ObjC has
|
|
priority). Conservative by construction — a plain C header matches nothing
|
|
here and keeps its existing extract_c routing.
|
|
"""
|
|
try:
|
|
head = path.read_bytes()[:256 * 1024]
|
|
except OSError:
|
|
return False
|
|
return any(marker in head for marker in _CPP_HEADER_MARKERS)
|
|
|
|
|
|
def _get_extractor(path: Path) -> Any | None:
|
|
"""Return the correct extractor function for a file, or None if unsupported."""
|
|
if path.name.lower().endswith(".blade.php"):
|
|
return extract_blade
|
|
# MCP config files (.mcp.json, claude_desktop_config.json, ...) are routed
|
|
# by filename before generic .json dispatch so they get MCP-aware nodes
|
|
# (servers, commands, packages, env vars) instead of opaque JSON keys.
|
|
if is_mcp_config_path(path):
|
|
return extract_mcp_config
|
|
# Package manifests (apm.yml, pyproject.toml, go.mod, pom.xml) → a canonical
|
|
# package node + depends_on edges, by filename before generic suffix dispatch
|
|
# (#1377). apm.yml would otherwise be a .yml document handled by the LLM.
|
|
if is_package_manifest_path(path):
|
|
return extract_package_manifest
|
|
# `.h` is C/C++/ObjC-ambiguous; route Objective-C headers to extract_objc
|
|
# (the suffix map sends `.h` to extract_c, which can't read @interface etc.).
|
|
# ObjC sniffing has priority over the C++ sniff: an Objective-C++ header can
|
|
# contain both `@interface` and inline C++ (`::`), and it must parse as ObjC.
|
|
suffix = path.suffix
|
|
if suffix not in _DISPATCH and suffix.lower() in _DISPATCH:
|
|
suffix = suffix.lower()
|
|
if suffix == ".h":
|
|
if _is_objc_header(path):
|
|
return extract_objc
|
|
# A C++ class header routed to extract_c loses the class entirely (the C
|
|
# grammar has no class_specifier). Reroute to extract_cpp (#1547).
|
|
if _is_cpp_header(path):
|
|
return extract_cpp
|
|
# `.m` is Objective-C OR MATLAB. extract_objc unconditionally would force-parse
|
|
# MATLAB through the ObjC grammar into garbage (#1702). Route to extract_objc
|
|
# only when the file actually looks like Objective-C; otherwise leave it without
|
|
# an extractor (surfaced by the no-AST-extractor warning, #1689) rather than
|
|
# mis-parsed. `.mm` is unambiguously Objective-C++ and stays on extract_objc.
|
|
if suffix == ".m" and not _is_objc_source(path):
|
|
return None
|
|
# Extensionless files: resolve by shebang, mirroring detect.classify_file.
|
|
# Without this, detect labels e.g. `#!/usr/bin/env bash` CLIs as code but
|
|
# extraction returns no extractor and the file silently contributes nothing.
|
|
if not suffix:
|
|
from graphify.detect import _shebang_interpreter
|
|
interp = _shebang_interpreter(path)
|
|
if interp is not None:
|
|
return _SHEBANG_DISPATCH.get(interp)
|
|
return _DISPATCH.get(suffix)
|
|
|
|
|
|
def _safe_extract_with_xaml_root(extractor, path: Path, root: Path) -> dict:
|
|
global _XAML_ACTIVE_EXTRACT_ROOT
|
|
previous_root = _XAML_ACTIVE_EXTRACT_ROOT
|
|
_XAML_ACTIVE_EXTRACT_ROOT = root.resolve()
|
|
try:
|
|
return _safe_extract(extractor, path)
|
|
finally:
|
|
_XAML_ACTIVE_EXTRACT_ROOT = previous_root
|
|
|
|
|
|
def _extract_single_file(args: tuple) -> tuple[int, dict]:
|
|
"""Worker function for parallel extraction. Runs in a subprocess.
|
|
|
|
Must be at module level (not a closure) so it can be pickled by
|
|
ProcessPoolExecutor.
|
|
|
|
Args:
|
|
args: (index, path_str, cache_root_str) tuple
|
|
|
|
Returns:
|
|
(index, result_dict) so results can be placed back in order.
|
|
"""
|
|
idx, path_str, cache_root_str = args
|
|
path = Path(path_str)
|
|
cache_root = Path(cache_root_str)
|
|
_raise_recursion_limit()
|
|
bypass_cache = path.suffix in _JS_CACHE_BYPASS_SUFFIXES
|
|
|
|
# Check cache first (avoid re-extraction)
|
|
if not bypass_cache:
|
|
cached = load_cached(path, cache_root)
|
|
if cached is not None:
|
|
return idx, cached
|
|
|
|
extractor = _get_extractor(path)
|
|
if extractor is None:
|
|
return idx, {"nodes": [], "edges": []}
|
|
|
|
result = _safe_extract_with_xaml_root(extractor, path, cache_root)
|
|
# Never cache a zero-node result for an extractable file. Every supported
|
|
# source produces at least a file node, so an empty node list is anomalous
|
|
# (e.g. a transient batch/parallel hiccup). Caching it makes the empty
|
|
# byte-stable across runs and silently blinds affected/explain to and
|
|
# through the file (#1666); skipping the write lets a rerun self-heal.
|
|
if not bypass_cache and "error" not in result and result.get("nodes"):
|
|
save_cached(path, result, cache_root)
|
|
return idx, result
|
|
|
|
|
|
def _extract_parallel(
|
|
uncached_work: list[tuple[int, Path]],
|
|
per_file: list[dict | None],
|
|
effective_root: Path,
|
|
max_workers: int | None,
|
|
total_files: int,
|
|
) -> bool:
|
|
"""Extract uncached files in parallel using ProcessPoolExecutor.
|
|
|
|
Returns True if the pool ran to completion. Returns False if the pool
|
|
failed in a recoverable way (typically Windows-spawn without an
|
|
``if __name__ == "__main__"`` guard in the calling script, which causes
|
|
BrokenProcessPool); the caller should fall back to sequential extraction.
|
|
"""
|
|
import concurrent.futures
|
|
|
|
if max_workers is None:
|
|
# Honour GRAPHIFY_MAX_WORKERS env override; otherwise scale to the
|
|
# full CPU. The historical `, 8)` cap was a safety bound for laptops
|
|
# in 2023 — on a 32-thread workstation it costs a 4x slowdown
|
|
# (issue #792). Capping at len(uncached_work) keeps small jobs
|
|
# from spawning useless idle workers.
|
|
env_raw = os.environ.get("GRAPHIFY_MAX_WORKERS", "").strip()
|
|
env_cap = None
|
|
if env_raw:
|
|
try:
|
|
v = int(env_raw)
|
|
if v > 0:
|
|
env_cap = v
|
|
except ValueError:
|
|
pass
|
|
cpu_cap = env_cap if env_cap is not None else (os.cpu_count() or 4)
|
|
max_workers = min(cpu_cap, len(uncached_work))
|
|
|
|
# Windows ProcessPoolExecutor hard-caps at 61 workers (CPython limitation
|
|
# tied to WaitForMultipleObjects). Clamp here so every path — auto-compute,
|
|
# GRAPHIFY_MAX_WORKERS, and --max-workers — stays valid on >61-core boxes
|
|
# (issue #1298). Guard against 0 from an empty work list.
|
|
if sys.platform == "win32":
|
|
max_workers = min(max_workers, 61)
|
|
max_workers = max(max_workers, 1)
|
|
|
|
root_str = str(effective_root)
|
|
work_items = [(idx, str(path), root_str) for idx, path in uncached_work]
|
|
|
|
done_count = 0
|
|
_PROGRESS_INTERVAL = 100
|
|
try:
|
|
with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as pool:
|
|
futures = {
|
|
pool.submit(_extract_single_file, item): pos
|
|
for pos, item in enumerate(work_items)
|
|
}
|
|
for future in concurrent.futures.as_completed(futures):
|
|
try:
|
|
idx, result = future.result()
|
|
per_file[idx] = result
|
|
except Exception as exc:
|
|
pos = futures[future]
|
|
print(
|
|
f" warning: worker failed for {work_items[pos][1]}: {exc}",
|
|
file=sys.stderr, flush=True,
|
|
)
|
|
done_count += 1
|
|
if (
|
|
total_files >= _PROGRESS_INTERVAL
|
|
and done_count % _PROGRESS_INTERVAL == 0
|
|
):
|
|
print(
|
|
f" AST extraction: {done_count}/{len(uncached_work)} uncached files "
|
|
f"({done_count * 100 // len(uncached_work)}%) [{max_workers} workers]",
|
|
flush=True,
|
|
)
|
|
except concurrent.futures.process.BrokenProcessPool:
|
|
# On Windows (spawn start method) the worker subprocesses re-import the
|
|
# caller's __main__. Inline invocations like `python -c "..."` have no
|
|
# __main__ guard, so worker bootstrap raises and the pool dies before
|
|
# any work completes. Fall back to in-process sequential extraction —
|
|
# slower but correct.
|
|
print(
|
|
" warning: parallel extraction failed (BrokenProcessPool); "
|
|
"falling back to sequential. On Windows this usually means the "
|
|
'caller is missing an `if __name__ == "__main__":` guard. Pass '
|
|
"parallel=False to extract() to skip the pool entirely.",
|
|
flush=True,
|
|
)
|
|
return False
|
|
if total_files >= _PROGRESS_INTERVAL:
|
|
# Report the same denominator the intermediate lines used (uncached files
|
|
# actually processed this run), not total_files — switching to the full
|
|
# corpus made the count jump upward at the end (cached hits + files with no
|
|
# extractor never entered uncached_work), which read as inconsistent (#1693).
|
|
_done = len(uncached_work)
|
|
print(
|
|
f" AST extraction: {_done}/{_done} uncached files (100%) [{max_workers} workers]",
|
|
flush=True,
|
|
)
|
|
return True
|
|
|
|
|
|
def _extract_sequential(
|
|
uncached_work: list[tuple[int, Path]],
|
|
per_file: list[dict | None],
|
|
effective_root: Path,
|
|
total_files: int,
|
|
) -> None:
|
|
"""Extract uncached files sequentially (fallback for small batches)."""
|
|
_PROGRESS_INTERVAL = 100
|
|
for work_idx, (idx, path) in enumerate(uncached_work):
|
|
if (
|
|
total_files >= _PROGRESS_INTERVAL
|
|
and work_idx % _PROGRESS_INTERVAL == 0
|
|
and work_idx > 0
|
|
):
|
|
print(
|
|
f" AST extraction: {work_idx}/{len(uncached_work)} uncached files ({work_idx * 100 // len(uncached_work)}%)",
|
|
flush=True,
|
|
)
|
|
extractor = _get_extractor(path)
|
|
if extractor is None:
|
|
per_file[idx] = {"nodes": [], "edges": []}
|
|
continue
|
|
bypass_cache = path.suffix in _JS_CACHE_BYPASS_SUFFIXES
|
|
result = _safe_extract_with_xaml_root(extractor, path, effective_root)
|
|
# See _extract_single_file: don't cache an anomalous zero-node result (#1666).
|
|
if not bypass_cache and "error" not in result and result.get("nodes"):
|
|
save_cached(path, result, effective_root)
|
|
per_file[idx] = result
|
|
if total_files >= _PROGRESS_INTERVAL:
|
|
# Consistent denominator with the intermediate lines (#1693).
|
|
_done = len(uncached_work)
|
|
print(f" AST extraction: {_done}/{_done} uncached files (100%)", flush=True)
|
|
|
|
|
|
_PARALLEL_THRESHOLD = 20
|
|
|
|
|
|
def extract(
|
|
paths: list[Path],
|
|
cache_root: Path | None = None,
|
|
*,
|
|
parallel: bool = True,
|
|
max_workers: int | None = None,
|
|
) -> dict:
|
|
"""Extract AST nodes and edges from a list of code files.
|
|
|
|
Two-pass process:
|
|
1. Per-file structural extraction (classes, functions, imports)
|
|
2. Cross-file import resolution: turns file-level imports into
|
|
class-level INFERRED edges (DigestAuth --uses--> Response)
|
|
|
|
Args:
|
|
paths: files to extract from
|
|
cache_root: explicit root for graphify-out/cache/ (overrides the
|
|
inferred common path prefix). Pass Path('.') when running on a
|
|
subdirectory so the cache stays at ./graphify-out/cache/.
|
|
parallel: if True and there are >= _PARALLEL_THRESHOLD uncached files,
|
|
use ProcessPoolExecutor for multi-core extraction.
|
|
max_workers: max subprocess count. Defaults to cpu_count (or the
|
|
value of GRAPHIFY_MAX_WORKERS if set), bounded by len(uncached_work).
|
|
"""
|
|
paths = [Path(p) for p in paths]
|
|
_check_tree_sitter_version()
|
|
_raise_recursion_limit()
|
|
# Workspace package manifests/globs can change during watch or repeated extraction.
|
|
_WORKSPACE_PACKAGE_CACHE.clear()
|
|
_XAML_CSHARP_CLASS_CACHE.clear()
|
|
|
|
# Infer a common root for cache keys (use first diverging segment, not sum of all matches)
|
|
try:
|
|
if not paths:
|
|
root = Path(".")
|
|
elif len(paths) == 1:
|
|
root = paths[0].parent
|
|
else:
|
|
min_parts = min(len(p.parts) for p in paths)
|
|
common_len = 0
|
|
for i in range(min_parts):
|
|
if len({p.parts[i] for p in paths}) == 1:
|
|
common_len += 1
|
|
else:
|
|
break
|
|
root = Path(*paths[0].parts[:common_len]) if common_len else Path(".")
|
|
except Exception:
|
|
root = Path(".")
|
|
if cache_root is not None:
|
|
root = cache_root
|
|
root = root.resolve()
|
|
|
|
effective_root = cache_root or root
|
|
total = len(paths)
|
|
|
|
# Phase 1: separate cached hits from uncached work
|
|
per_file: list[dict | None] = [None] * total
|
|
uncached_work: list[tuple[int, Path]] = []
|
|
|
|
for i, path in enumerate(paths):
|
|
if _get_extractor(path) is None:
|
|
per_file[i] = {"nodes": [], "edges": []}
|
|
continue
|
|
bypass_cache = path.suffix in _JS_CACHE_BYPASS_SUFFIXES
|
|
if not bypass_cache:
|
|
cached = load_cached(path, effective_root)
|
|
if cached is not None:
|
|
per_file[i] = cached
|
|
continue
|
|
uncached_work.append((i, path))
|
|
|
|
# Phase 2: extract uncached files (parallel or sequential)
|
|
if uncached_work:
|
|
ran_parallel = False
|
|
if parallel and len(uncached_work) >= _PARALLEL_THRESHOLD:
|
|
ran_parallel = _extract_parallel(
|
|
uncached_work, per_file, effective_root, max_workers, total
|
|
)
|
|
if not ran_parallel:
|
|
_extract_sequential(uncached_work, per_file, effective_root, total)
|
|
|
|
# Fill any remaining None slots (shouldn't happen, but defensive)
|
|
for i in range(total):
|
|
if per_file[i] is None:
|
|
per_file[i] = {"nodes": [], "edges": []}
|
|
|
|
# #1666: surface any source file an extractor accepted but that produced zero
|
|
# nodes (not even a file node). Such a file is silently absent from the graph,
|
|
# so affected/explain are blind to and through it with no other signal.
|
|
_empty_sources: list[str] = []
|
|
for i, _p in enumerate(paths):
|
|
_res = per_file[i] or {}
|
|
if _res.get("nodes") or _res.get("error"):
|
|
continue
|
|
if _get_extractor(_p) is not None:
|
|
_empty_sources.append(str(_p))
|
|
if _empty_sources:
|
|
_shown = ", ".join(Path(x).name for x in _empty_sources[:5])
|
|
_more = f" (+{len(_empty_sources) - 5} more)" if len(_empty_sources) > 5 else ""
|
|
print(
|
|
f" warning: {len(_empty_sources)} source file(s) produced zero nodes and "
|
|
f"are absent from the graph: {_shown}{_more}. A re-run will retry them "
|
|
f"(empties are no longer cached); if it persists, please report the "
|
|
f"file(s) (#1666).",
|
|
file=sys.stderr, flush=True,
|
|
)
|
|
|
|
# #1689: a file counted as code (extension in CODE_EXTENSIONS) but with no AST
|
|
# extractor wired up (e.g. .r/.R — there is no tree-sitter-r dispatch) silently
|
|
# contributes zero nodes. The #1666 warning above deliberately skips these (it
|
|
# only fires when an extractor exists), so surface them explicitly, grouped by
|
|
# extension, rather than reporting success as if the language were mapped.
|
|
from graphify.detect import CODE_EXTENSIONS as _CODE_EXTS
|
|
_no_extractor: dict[str, int] = {}
|
|
for _p in paths:
|
|
_ext = _p.suffix.lower()
|
|
if _ext in _CODE_EXTS and _get_extractor(_p) is None:
|
|
_no_extractor[_ext] = _no_extractor.get(_ext, 0) + 1
|
|
if _no_extractor:
|
|
_by_count = ", ".join(
|
|
f"{ext} ({n})" for ext, n in sorted(_no_extractor.items(), key=lambda kv: (-kv[1], kv[0]))
|
|
)
|
|
_tot = sum(_no_extractor.values())
|
|
print(
|
|
f" warning: {_tot} file(s) are classified as code but graphify has no AST "
|
|
f"extractor for their language, so they contributed nothing to the graph: "
|
|
f"{_by_count}. Please open an issue to request support for these (#1689).",
|
|
file=sys.stderr, flush=True,
|
|
)
|
|
|
|
# #1745: an extractor IS wired up for these files but bailed out because its
|
|
# dependency is missing (e.g. .sql needs tree-sitter-sql from the [sql]
|
|
# extra). Neither warning above fires — #1666 skips results that carry an
|
|
# error, #1689 only covers files with no extractor — so the graph builds
|
|
# "successfully" while every such file silently contributes nothing.
|
|
# Surface them grouped by extension, naming the extra that provides the
|
|
# dependency when there is one.
|
|
_missing_dep_count: dict[str, int] = {}
|
|
_missing_dep_error: dict[str, str] = {}
|
|
for i, _p in enumerate(paths):
|
|
_err = (per_file[i] or {}).get("error") or ""
|
|
if "not installed" in _err:
|
|
_ext = _p.suffix.lower()
|
|
_missing_dep_count[_ext] = _missing_dep_count.get(_ext, 0) + 1
|
|
_missing_dep_error.setdefault(_ext, _err)
|
|
for _ext, _n in sorted(_missing_dep_count.items(), key=lambda kv: (-kv[1], kv[0])):
|
|
_extra = _EXTRA_FOR_EXTENSION.get(_ext)
|
|
if _extra:
|
|
_reason = _missing_dep_error[_ext].split(". ")[0]
|
|
_hint = f' Install it with: pip install "graphifyy[{_extra}]"'
|
|
else:
|
|
_reason = _missing_dep_error[_ext]
|
|
_hint = ""
|
|
print(
|
|
f" warning: {_n} {_ext} file(s) contributed nothing to the graph "
|
|
f"because a dependency is missing: {_reason}.{_hint} (#1745)",
|
|
file=sys.stderr, flush=True,
|
|
)
|
|
|
|
all_nodes: list[dict] = []
|
|
all_edges: list[dict] = []
|
|
all_raw_calls: list[dict] = []
|
|
for result in per_file:
|
|
all_nodes.extend(result.get("nodes", []))
|
|
all_edges.extend(result.get("edges", []))
|
|
all_raw_calls.extend(result.get("raw_calls", []))
|
|
# Function / method / class def ids for the cross-file indirect_call callable
|
|
# guard. Built from the `_callable` node marker AFTER the id-remap / disambiguation
|
|
# passes below (which rewrite node ids), so it can never go stale — see the
|
|
# marker set in the per-file extractor. Populated just before the pass that uses it.
|
|
callable_nids: set[str] = set()
|
|
|
|
_augment_symbol_resolution_edges(paths, all_nodes, all_edges, root)
|
|
|
|
# Merge a header-declared class (and its methods) with its sibling-impl
|
|
# definition into ONE node (C/C++/ObjC #1547/#1556). Runs BEFORE the id-remap
|
|
# below: a header symbol and its impl counterpart share an id only while both
|
|
# still carry the raw file-stem prefix; the per-file prefix remap then diverges
|
|
# them (foo_h vs foo_cpp), so the collapse must happen first. Collapsing here
|
|
# also means disambiguation sees one source_file per id and won't split them.
|
|
_merge_decl_def_classes(all_nodes, all_edges)
|
|
|
|
# Remap file node IDs from absolute-path-derived to the canonical
|
|
# {parent_dir}_{stem} spec form so (a) graph.json edge endpoints are stable
|
|
# across machines (#502) and (b) AST file nodes match the IDs semantic
|
|
# subagents generate (#1033). Resolve before relativizing so paths passed in
|
|
# relative form still anchor to the (resolved) root.
|
|
id_remap: dict[str, str] = {}
|
|
# Symbol node IDs embed the file stem as a prefix (_file_node_id of the path
|
|
# the extractor saw). For a root-level file that stem picks up the absolute
|
|
# parent directory name, so a symbol becomes <rootdir>_main_run while the
|
|
# file node is correctly relativized to main and the skill.md spec wants
|
|
# main_run -- splitting the symbol into AST/semantic ghosts (#1096). Relativize
|
|
# the symbol prefix the same way, gated by source_file so two files sharing a
|
|
# prefix can't cross-contaminate. Keyed by resolved path -> (old_pref, new_pref).
|
|
# Each file maps from up to TWO old prefixes — the input-form prefix
|
|
# _file_node_id(path) and the absolute-resolved-form prefix
|
|
# _file_node_id(path.resolve()). Alias/workspace imports resolve specifiers
|
|
# through .resolve(), so their edge targets are keyed off the ABSOLUTE form;
|
|
# when inputs are relative the two forms differ and absolute-derived targets
|
|
# would otherwise orphan (#1529). Stored as a list so the symbol-prefix remap
|
|
# below can try both (identical forms collapse to one — a no-op).
|
|
prefix_remap: dict[Path, list[tuple[str, str]]] = {}
|
|
for path in paths:
|
|
old_id = _make_id(str(path))
|
|
try:
|
|
rel = path.relative_to(root)
|
|
except ValueError:
|
|
try:
|
|
rel = path.resolve().relative_to(root)
|
|
except ValueError:
|
|
continue
|
|
new_id = _file_node_id(rel)
|
|
if old_id != new_id:
|
|
id_remap[old_id] = new_id
|
|
# Also register the absolute-resolved form of the file-level id so
|
|
# alias/workspace import targets (resolved via .resolve()) remap to
|
|
# canonical instead of orphaning (#1529).
|
|
old_id_abs = _make_id(str(path.resolve()))
|
|
if old_id_abs != new_id:
|
|
id_remap[old_id_abs] = new_id
|
|
old_prefs: list[tuple[str, str]] = []
|
|
old_pref = _file_node_id(path)
|
|
if old_pref != new_id:
|
|
old_prefs.append((old_pref, new_id))
|
|
old_pref_abs = _file_node_id(path.resolve())
|
|
if old_pref_abs != new_id and old_pref_abs != old_pref:
|
|
old_prefs.append((old_pref_abs, new_id))
|
|
if old_prefs:
|
|
prefix_remap[path.resolve()] = old_prefs
|
|
if id_remap:
|
|
for n in all_nodes:
|
|
if n.get("id") in id_remap:
|
|
n["id"] = id_remap[n["id"]]
|
|
for e in all_edges:
|
|
if e.get("source") in id_remap:
|
|
e["source"] = id_remap[e["source"]]
|
|
if e.get("target") in id_remap:
|
|
e["target"] = id_remap[e["target"]]
|
|
if prefix_remap:
|
|
sym_remap: dict[str, str] = {}
|
|
for n in all_nodes:
|
|
sf = n.get("source_file")
|
|
if not sf:
|
|
continue
|
|
# Package nodes carry a canonical name-keyed id (pkg_<name>) that must
|
|
# stay identical across every manifest that references the package, so
|
|
# they are exempt from the file-stem prefix remap (#1377), like the
|
|
# type=module anchors (#1327).
|
|
if n.get("type") == "package":
|
|
continue
|
|
try:
|
|
entry = prefix_remap.get(Path(sf).resolve())
|
|
except Exception:
|
|
continue
|
|
if entry is None:
|
|
continue
|
|
nid = n.get("id", "")
|
|
# Try both the input-form and absolute-form prefixes for this file
|
|
# (#1529). source_file gating above already prevents cross-file
|
|
# contamination, so the first matching prefix wins.
|
|
for old_pref, new_pref in entry:
|
|
if nid.startswith(old_pref + "_"):
|
|
new_nid = new_pref + nid[len(old_pref):]
|
|
if new_nid != nid:
|
|
sym_remap[nid] = new_nid
|
|
break
|
|
if sym_remap:
|
|
for n in all_nodes:
|
|
if n.get("id") in sym_remap:
|
|
n["id"] = sym_remap[n["id"]]
|
|
for e in all_edges:
|
|
if e.get("source") in sym_remap:
|
|
e["source"] = sym_remap[e["source"]]
|
|
if e.get("target") in sym_remap:
|
|
e["target"] = sym_remap[e["target"]]
|
|
# raw_calls carry caller_nid (a symbol id) consumed by the cross-file
|
|
# call pass below, after this remap — rewrite it too or those edges
|
|
# would dangle on their (stale) source.
|
|
for rc in all_raw_calls:
|
|
cn = rc.get("caller_nid")
|
|
if cn in sym_remap:
|
|
rc["caller_nid"] = sym_remap[cn]
|
|
|
|
_merge_swift_extensions(per_file, all_nodes, all_edges)
|
|
_disambiguate_colliding_node_ids(all_nodes, all_edges, all_raw_calls, root)
|
|
_canonicalize_csharp_namespace_nodes(all_nodes, all_edges)
|
|
_rewire_unique_stub_nodes(all_nodes, all_edges)
|
|
|
|
# Add cross-file class-level edges (Python only - uses Python parser internally)
|
|
py_paths = [p for p in paths if p.suffix == ".py"]
|
|
if py_paths:
|
|
py_results = [r for r, p in zip(per_file, paths) if p.suffix == ".py"]
|
|
try:
|
|
cross_file_edges = _resolve_cross_file_imports(py_results, py_paths)
|
|
all_edges.extend(cross_file_edges)
|
|
except Exception as exc:
|
|
import logging
|
|
logging.getLogger(__name__).warning("Cross-file import resolution failed, skipping: %s", exc)
|
|
|
|
# Cross-file Java import resolution
|
|
java_paths = [p for p in paths if p.suffix == ".java"]
|
|
if java_paths:
|
|
java_results = [r for r, p in zip(per_file, paths) if p.suffix == ".java"]
|
|
try:
|
|
all_edges.extend(_resolve_cross_file_java_imports(java_results, java_paths))
|
|
except Exception as exc:
|
|
import logging
|
|
logging.getLogger(__name__).warning("Java cross-file import resolution failed, skipping: %s", exc)
|
|
# Re-point dangling implements/inherits edges that bare-name resolution
|
|
# left on shadow stubs, using imports for exact-package disambiguation (#1318).
|
|
try:
|
|
_resolve_java_type_references(java_results, java_paths, all_nodes, all_edges)
|
|
except Exception as exc:
|
|
import logging
|
|
logging.getLogger(__name__).warning("Java type-reference resolution failed, skipping: %s", exc)
|
|
|
|
# Cross-file C# type-reference resolution: re-point dangling inherits/implements/
|
|
# references edges left on shadow stubs, disambiguating same-named types by the
|
|
# referencing file's `using` directives + enclosing namespace (mirrors Java #1318).
|
|
cs_paths = [p for p in paths if p.suffix == ".cs"]
|
|
if cs_paths:
|
|
cs_results = [r for r, p in zip(per_file, paths) if p.suffix == ".cs"]
|
|
try:
|
|
_resolve_csharp_type_references(cs_results, cs_paths, all_nodes, all_edges)
|
|
except Exception as exc:
|
|
import logging
|
|
logging.getLogger(__name__).warning("C# type-reference resolution failed, skipping: %s", exc)
|
|
try:
|
|
_resolve_cross_file_csharp_imports(cs_results, cs_paths, all_nodes, all_edges)
|
|
except Exception as exc:
|
|
import logging
|
|
logging.getLogger(__name__).warning("C# cross-file import resolution failed, skipping: %s", exc)
|
|
|
|
# Cross-file call resolution for all languages
|
|
# Each extractor saved unresolved calls in raw_calls. Now that we have all
|
|
# nodes from all files, resolve any callee that exists in another file.
|
|
# Build name → ALL matching node IDs so we can skip ambiguous common names
|
|
# (e.g. "log", "execute", "find") that appear in multiple files — resolving
|
|
# those inflates god_nodes ranking with spurious cross-file edges.
|
|
# Build label -> node_id index for cross-file call resolution.
|
|
# Skip rationale nodes (their labels are docstring text, not callable
|
|
# identifiers, and they were polluting matches for short names — #563).
|
|
global_label_to_nids: dict[str, list[str]] = {} # exact-case (all languages)
|
|
global_label_to_nids_ci: dict[str, list[str]] = {} # case-INSENSITIVE-language nodes
|
|
for n in all_nodes:
|
|
if n.get("file_type") == "rationale" or n.get("type") == "namespace":
|
|
continue
|
|
raw = n.get("label", "")
|
|
normalised = raw.strip("()").lstrip(".")
|
|
if normalised:
|
|
# Case is semantic in most languages, so index (and match, below) by exact
|
|
# case — folding collapses `Path` (class) into `PATH` (env var) and makes a
|
|
# single shell variable the #1 god-node (#1581). Only case-insensitive
|
|
# languages (PHP/SQL/Nim) also get a folded key for legitimate fold-matching.
|
|
global_label_to_nids.setdefault(normalised, []).append(n["id"])
|
|
if _lang_is_case_insensitive(n.get("source_file")):
|
|
global_label_to_nids_ci.setdefault(normalised.lower(), []).append(n["id"])
|
|
|
|
# Callable-def ids for the indirect_call callable guard, read from the `_callable`
|
|
# marker on the FINAL (post-remap) nodes — so a callback resolves only to a real
|
|
# function/method/class, never a same-named data symbol, and the guard never goes
|
|
# stale when node ids were relativized/disambiguated above (#1566).
|
|
callable_nids = {n["id"] for n in all_nodes if n.get("_callable")}
|
|
|
|
# Build evidence index from import edges so cross-file calls backed by an
|
|
# explicit import statement can be promoted from INFERRED to EXTRACTED.
|
|
# Direct symbol imports (`import { foo }` / `const { foo } = require()`) are
|
|
# the strongest evidence — caller's file_id has an `imports` edge directly to
|
|
# the callee's symbol id. Module imports (`imports_from`) are weaker but still
|
|
# confirm the caller pulled in the callee's source file.
|
|
file_to_symbol_imports: dict[str, set[str]] = {}
|
|
file_to_module_imports: dict[str, set[str]] = {}
|
|
for e in all_edges:
|
|
if e.get("relation") == "imports":
|
|
file_to_symbol_imports.setdefault(e["source"], set()).add(e["target"])
|
|
elif e.get("relation") == "imports_from":
|
|
file_to_module_imports.setdefault(e["source"], set()).add(e["target"])
|
|
|
|
# Map each node back to its containing file node id so we can ask
|
|
# "did the caller's file import the callee's file?"
|
|
# A node and its file node share the exact same ``source_file`` string, and a
|
|
# file node is the one whose label is the basename (``add_node(file_nid,
|
|
# path.name)``). Resolving file membership by that shared string is robust
|
|
# against the path-resolution/symlink mismatch that makes
|
|
# ``relative_to(root.resolve())`` throw and fall back to a non-matching
|
|
# absolute-derived id — which would spuriously fail import evidence and (with
|
|
# the #1659 JS/TS gate below) drop a legitimately-imported call.
|
|
sf_to_file_nid: dict[str, str] = {}
|
|
for n in all_nodes:
|
|
sf = n.get("source_file")
|
|
if sf and n.get("label") == Path(str(sf)).name:
|
|
sf_to_file_nid.setdefault(str(sf), n["id"])
|
|
nid_to_file_nid: dict[str, str] = {}
|
|
# nid -> raw source_file string, for the ambiguous-name tie-breakers below
|
|
# (test/non-test classification + path proximity). Kept separate from the
|
|
# file-node-id map because tie-breaking compares the actual file paths.
|
|
nid_to_source_file: dict[str, str] = {}
|
|
for n in all_nodes:
|
|
sf = n.get("source_file")
|
|
if not sf:
|
|
continue
|
|
nid_to_source_file[n["id"]] = str(sf)
|
|
fnid = sf_to_file_nid.get(str(sf))
|
|
if fnid is not None:
|
|
nid_to_file_nid[n["id"]] = fnid
|
|
continue
|
|
# Fallback (no file node found for this source_file): derive it the old
|
|
# way from the relativized path.
|
|
sf_path = Path(sf)
|
|
try:
|
|
sf_rel = sf_path.relative_to(root) if sf_path.is_absolute() else sf_path
|
|
except ValueError:
|
|
sf_rel = sf_path
|
|
nid_to_file_nid[n["id"]] = _file_node_id(sf_rel)
|
|
|
|
existing_pairs = {(e["source"], e["target"]) for e in all_edges}
|
|
# Call-like pairs only, for the indirect_call dedup: an `imports` edge from a
|
|
# file to the symbol it imports is EXPECTED and must not suppress an
|
|
# indirect_call to that same symbol (JS/TS named imports create such an edge).
|
|
call_like_pairs = {
|
|
(e["source"], e["target"]) for e in all_edges
|
|
if e.get("relation") in ("calls", "indirect_call")
|
|
}
|
|
# JS/TS/JSX modules have no implicit cross-module scope: a call into another
|
|
# file is real ONLY if the caller imported it. So a cross-file call from one
|
|
# of these files with no import evidence is gated below (#1659).
|
|
_JS_TS_CALL_SUFFIXES = (".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs")
|
|
for rc in all_raw_calls:
|
|
callee = rc.get("callee", "")
|
|
if not callee:
|
|
continue
|
|
if callee in _LANGUAGE_BUILTIN_GLOBALS:
|
|
continue
|
|
# Skip member-call callees: obj.log() → "log" has no import evidence
|
|
# and collides with any top-level function named "log" in the corpus.
|
|
if rc.get("is_member_call"):
|
|
continue
|
|
# Skip Ruby include/extend/prepend mixin markers: they carry a module
|
|
# name as `callee` but are not calls — the Ruby resolver turns them into
|
|
# `mixes_in` edges. Letting the shared pass emit a `calls` edge here would
|
|
# both mislabel the relation and block the mixes_in emit as a dup (#1668).
|
|
if rc.get("is_mixin"):
|
|
continue
|
|
# Exact-case match first (case is semantic). Fold only when the CALLING
|
|
# file's language is case-insensitive, and only against the folded index of
|
|
# case-insensitive-language definitions — so a Python `Path()` call can never
|
|
# resolve to a shell `PATH` node (#1581).
|
|
candidates = global_label_to_nids.get(callee, [])
|
|
if not candidates and _lang_is_case_insensitive(rc.get("source_file")):
|
|
candidates = global_label_to_nids_ci.get(callee.lower(), [])
|
|
if not candidates:
|
|
continue
|
|
# Cross-language guard: never bind a call to a definition in a different
|
|
# language family. Name-only matching was resolving a TSX callback passed
|
|
# by name to a same-named Kotlin method in the Android half of the repo
|
|
# (and a Python call to a Kotlin fun) — phantom edges the extraction spec
|
|
# explicitly forbids. Candidates whose family is unknown (no source_file,
|
|
# non-code nodes) are kept, preserving the previous permissive behavior;
|
|
# real interop pairs (Kotlin↔Java, C↔C++↔ObjC, JS↔TS) share a family and
|
|
# still resolve.
|
|
caller_family = _lang_family(rc.get("source_file"))
|
|
if caller_family is not None:
|
|
candidates = [
|
|
c for c in candidates
|
|
if (candidate_family := _lang_family(nid_to_source_file.get(c))) is None
|
|
or candidate_family == caller_family
|
|
]
|
|
if not candidates:
|
|
continue
|
|
caller = rc["caller_nid"]
|
|
# Resolve the caller's file via the raw_call's own source_file string,
|
|
# which is stable regardless of any caller_nid remap. An indirect
|
|
# callback's caller_nid is the file node, whose id may have been
|
|
# relativized after the raw_call was recorded, so a caller_nid lookup can
|
|
# miss and (with the #1659 gate) drop a legitimately-imported callback.
|
|
caller_file_nid = (
|
|
sf_to_file_nid.get(str(rc.get("source_file", "")))
|
|
or nid_to_file_nid.get(caller)
|
|
)
|
|
imported_symbols = file_to_symbol_imports.get(caller_file_nid, set())
|
|
imported_modules = file_to_module_imports.get(caller_file_nid, set())
|
|
|
|
def _has_import_evidence(candidate_id: str) -> bool:
|
|
# Direct symbol import (`import { foo }`) is the strongest evidence:
|
|
# the caller's file has an `imports` edge straight to this symbol.
|
|
# A module import (`import './helper.js'`) confirms the caller pulled
|
|
# in the file the candidate lives in.
|
|
candidate_file_nid = nid_to_file_nid.get(candidate_id)
|
|
return (
|
|
candidate_id in imported_symbols
|
|
or (candidate_file_nid is not None and candidate_file_nid in imported_modules)
|
|
)
|
|
|
|
if len(candidates) == 1:
|
|
tgt = candidates[0]
|
|
has_import_evidence = _has_import_evidence(tgt)
|
|
else:
|
|
# Ambiguous name (defined in 2+ files). Don't bail outright (#1219):
|
|
# if the caller has explicit import evidence pointing at exactly one
|
|
# of the candidates, that named import disambiguates unambiguously.
|
|
# Prefer direct symbol-import matches; fall back to module-import
|
|
# matches only when they too collapse to a single target. Without a
|
|
# unique evidence-backed pick we skip, preserving the #543 guard
|
|
# against over-connecting common short names (log, execute, find).
|
|
symbol_matches = [c for c in candidates if c in imported_symbols]
|
|
if len(symbol_matches) == 1:
|
|
tgt = symbol_matches[0]
|
|
has_import_evidence = True
|
|
else:
|
|
module_matches = [
|
|
c for c in candidates
|
|
if (cf := nid_to_file_nid.get(c)) is not None and cf in imported_modules
|
|
]
|
|
if len(module_matches) == 1:
|
|
tgt = module_matches[0]
|
|
has_import_evidence = True
|
|
else:
|
|
# No unique import evidence. Instead of dropping the edge
|
|
# outright (which let a single same-named test mock erase the
|
|
# real call graph, #1553), apply the shared god-node
|
|
# tie-breakers (non-test preference, then path proximity).
|
|
# Resolve only if exactly one candidate survives; otherwise
|
|
# the #543/#1219 guard still holds and we skip.
|
|
tgt = disambiguate_ambiguous_candidates(
|
|
candidates,
|
|
{c: nid_to_source_file.get(c, "") for c in candidates},
|
|
rc.get("source_file", ""),
|
|
)
|
|
if tgt is None:
|
|
continue
|
|
has_import_evidence = False
|
|
if rc.get("indirect"):
|
|
# Cross-file indirect dispatch: a callback passed BY NAME
|
|
# (`from .h import fn; pool.submit(fn)`, or listed in a dispatch
|
|
# table). Resolved through the same single-definition / import-evidence
|
|
# candidate logic as a direct call, but emitted as a distinct INFERRED
|
|
# `indirect_call` and ONLY when the target is a real callable def —
|
|
# never a same-named data symbol. Stays INFERRED even with import
|
|
# evidence: the name is referenced as a value here, not invoked. Dedup
|
|
# is call-aware (an existing direct `calls` edge pre-empts it; a benign
|
|
# `imports` edge to the same symbol does NOT suppress it).
|
|
if tgt != caller and (caller, tgt) not in call_like_pairs and tgt in callable_nids:
|
|
call_like_pairs.add((caller, tgt))
|
|
all_edges.append({
|
|
"source": caller,
|
|
"target": tgt,
|
|
"relation": "indirect_call",
|
|
"context": rc.get("context", "argument"),
|
|
"confidence": "INFERRED",
|
|
"confidence_score": 0.8,
|
|
"source_file": rc.get("source_file", ""),
|
|
"source_location": rc.get("source_location"),
|
|
"weight": 1.0,
|
|
})
|
|
continue
|
|
# #1659: a JS/TS DIRECT call with no import evidence is almost always an
|
|
# unrelated same-named export in a package that was never imported — a
|
|
# phantom cross-package edge (a 14-package monorepo had `platform` and
|
|
# `sidecar` shown as depending on `registry-protocol` purely because it
|
|
# exported generically-named symbols). JS/TS modules have no implicit
|
|
# cross-module scope, so leave it unresolved rather than binding by name
|
|
# alone. Other languages keep the #1553 single-candidate resolution:
|
|
# C/C++ headers, Ruby autoload, and same-package implicit scope
|
|
# legitimately call across files without an explicit import. Scoped to
|
|
# direct calls: the indirect_call path above is already conservative
|
|
# (INFERRED, callable-target-gated) and independent of import evidence.
|
|
if not has_import_evidence and str(rc.get("source_file", "")).endswith(_JS_TS_CALL_SUFFIXES):
|
|
continue
|
|
if tgt != caller and (caller, tgt) not in existing_pairs:
|
|
existing_pairs.add((caller, tgt))
|
|
# Promote to EXTRACTED when there's a direct import edge from the
|
|
# caller's file pointing at either the callee symbol itself or the
|
|
# file the callee lives in.
|
|
if has_import_evidence:
|
|
confidence = "EXTRACTED"
|
|
confidence_score = 1.0
|
|
else:
|
|
confidence = "INFERRED"
|
|
confidence_score = 0.8
|
|
all_edges.append({
|
|
"source": caller,
|
|
"target": tgt,
|
|
"relation": "calls",
|
|
"context": "call",
|
|
"confidence": confidence,
|
|
"confidence_score": confidence_score,
|
|
"source_file": rc.get("source_file", ""),
|
|
"source_location": rc.get("source_location"),
|
|
"weight": 1.0,
|
|
})
|
|
|
|
# Cross-file, language-specific member-call resolution. Runs after the shared
|
|
# call pass so node ids/caller_nids are final; each pass is additive (only the
|
|
# receiver-typed/qualified calls the shared pass skipped) with its own
|
|
# single-definition god-node guard. Registered in graphify.resolver_registry so
|
|
# a new language plugs in without editing this body (#1356 Swift, #1446 Python).
|
|
run_language_resolvers(paths, per_file, all_nodes, all_edges)
|
|
|
|
# Relativize source_file fields so paths are portable across machines (#555)
|
|
for item in all_nodes + all_edges:
|
|
sf = item.get("source_file")
|
|
if not sf:
|
|
continue
|
|
sf_path = Path(sf)
|
|
if not sf_path.is_absolute():
|
|
continue
|
|
try:
|
|
item["source_file"] = sf_path.relative_to(root).as_posix()
|
|
except ValueError:
|
|
pass
|
|
|
|
# origin_file is an internal disambiguation hint (#1462): the colliding-id pass
|
|
# above reads it to keep same-named cross-file stubs distinct, after which nothing
|
|
# consumes it. Drop it from the returned nodes so it never ships into graph.json as
|
|
# an absolute, machine-specific path — the same "no absolute paths in output"
|
|
# contract that relativizes source_file just above (#555, #932). The per-file AST
|
|
# cache keeps its own copy, which is what the colliding-id pass reads on a cache hit.
|
|
for n in all_nodes:
|
|
n.pop("origin_file", None)
|
|
n.pop("_callable", None) # internal indirect_call marker — never ships to graph.json
|
|
|
|
# Tag AST provenance so the incremental watch rebuild can distinguish
|
|
# AST-extracted nodes from semantic/LLM nodes. On a full re-extraction
|
|
# the watcher drops any AST-marked node missing from the fresh output
|
|
# even when its source file still exists (#1116).
|
|
for n in all_nodes:
|
|
n["_origin"] = "ast"
|
|
|
|
return {
|
|
"nodes": all_nodes,
|
|
"edges": all_edges,
|
|
"input_tokens": 0,
|
|
"output_tokens": 0,
|
|
}
|
|
|
|
|
|
def collect_files(target: Path, *, follow_symlinks: bool = False, root: Path | None = None) -> list[Path]:
|
|
containment_root = root if root is not None else target
|
|
from graphify.detect import _resolves_under_root
|
|
if target.is_file():
|
|
return [target] if _resolves_under_root(target, containment_root) else []
|
|
_EXTENSIONS = set(_DISPATCH.keys())
|
|
from graphify.detect import _is_ignored, _is_noise_dir, _load_graphifyignore
|
|
ignore_root = root if root is not None else target
|
|
patterns = _load_graphifyignore(ignore_root)
|
|
# Shared across all _is_ignored calls in this scan so ancestor-directory
|
|
# results are memoised instead of re-evaluated per file.
|
|
ignore_cache: dict[Path, bool] = {}
|
|
|
|
def _ignored(p: Path) -> bool:
|
|
return bool(patterns and _is_ignored(p, ignore_root, patterns, _cache=ignore_cache))
|
|
|
|
if not follow_symlinks:
|
|
# The old rglob filter rejected paths with a noise component anywhere,
|
|
# including components of target itself — preserve that.
|
|
if any(_is_noise_dir(part) for part in target.parts):
|
|
return []
|
|
# When negation (!) patterns exist, skip directory-level ignore pruning
|
|
# so negated files inside ignored dirs can still be reached (same
|
|
# conservatism as detect's scan walk).
|
|
has_negation = any(pat.startswith("!") for _, pat in patterns)
|
|
results: list[Path] = []
|
|
for dirpath, dirnames, filenames in os.walk(target):
|
|
dp = Path(dirpath)
|
|
dirnames[:] = [
|
|
d for d in dirnames
|
|
if not _is_noise_dir(d)
|
|
and (has_negation or not _ignored(dp / d))
|
|
]
|
|
for fname in filenames:
|
|
p = dp / fname
|
|
suffix = p.suffix
|
|
if (suffix in _EXTENSIONS or suffix.lower() in _EXTENSIONS) and not _ignored(p) and _resolves_under_root(p, containment_root):
|
|
results.append(p)
|
|
return sorted(results)
|
|
# Walk with symlink following + cycle detection
|
|
results = []
|
|
for dirpath, dirnames, filenames in os.walk(target, followlinks=True):
|
|
if os.path.islink(dirpath):
|
|
real = os.path.realpath(dirpath)
|
|
parent_real = os.path.realpath(os.path.dirname(dirpath))
|
|
if parent_real == real or parent_real.startswith(real + os.sep):
|
|
dirnames.clear()
|
|
continue
|
|
dp = Path(dirpath)
|
|
dirnames[:] = [
|
|
d for d in dirnames
|
|
if not _is_noise_dir(d)
|
|
and (not (dp / d).is_symlink() or _resolves_under_root(dp / d, containment_root))
|
|
]
|
|
for fname in filenames:
|
|
p = dp / fname
|
|
suffix = p.suffix
|
|
if (suffix in _EXTENSIONS or suffix.lower() in _EXTENSIONS) and not _ignored(p) and _resolves_under_root(p, containment_root):
|
|
results.append(p)
|
|
return sorted(results)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) < 2:
|
|
print("Usage: python -m graphify.extract <file_or_dir> ...", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
paths: list[Path] = []
|
|
for arg in sys.argv[1:]:
|
|
paths.extend(collect_files(Path(arg)))
|
|
|
|
result = extract(paths)
|
|
print(json.dumps(result, indent=2))
|