mirror of
https://github.com/Graphify-Labs/graphify.git
synced 2026-09-14 19:34:09 +08:00
6dc23db90f
JS/TS symbol resolution only handled named imports. A default-exported
symbol (`export default class Foo`) imported as `import Foo from './foo'`
produced only a file→file `imports_from` edge; the class node received no
incoming symbol edge. On codebases that default-export most classes
(NestJS services/helpers/models, etc.) those symbols looked like isolated
leaf nodes, and `graphify affected "<Class>"` / `explain` reported no
callers.
Record default imports with imported_name="default", register a "default"
export for `export default <class|function|identifier>`, and let the
existing exported-origin resolver wire the `imports` edge — which also
resolves calls made through the local binding, even when it is renamed
(`import Bar from './foo'; new Bar()`). `export { X as default }` was
already handled via the export-clause path; anonymous defaults
(`export default class {}`) have no resolvable symbol and stay file-level.
Adds regression tests for default-export class/function/identifier import
resolution and renamed-binding call resolution.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
12060 lines
509 KiB
Python
12060 lines
509 KiB
Python
"""Deterministic structural extraction from source code using tree-sitter. Outputs nodes+edges dicts."""
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import unicodedata
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
from .cache import load_cached, save_cached
|
|
from .mcp_ingest import extract_mcp_config, is_mcp_config_path
|
|
|
|
_RECURSION_LIMIT = 10_000
|
|
|
|
# Language built-in globals that AST may classify as call targets when used as
|
|
# constructors or coercion functions (e.g. String(x), Number(x), Boolean(x)).
|
|
# Without this filter they become god-nodes accumulating spurious edges from
|
|
# every call site. Filter applied at same-file and cross-file resolution.
|
|
# See issue #726.
|
|
_LANGUAGE_BUILTIN_GLOBALS: frozenset[str] = frozenset({
|
|
# JavaScript / TypeScript ECMAScript built-ins
|
|
"String", "Number", "Boolean", "Object", "Array", "Symbol", "BigInt",
|
|
"Date", "RegExp", "Error", "TypeError", "RangeError", "SyntaxError",
|
|
"ReferenceError", "EvalError", "URIError",
|
|
"Promise", "Map", "Set", "WeakMap", "WeakSet", "JSON", "Math",
|
|
"Reflect", "Proxy", "Intl",
|
|
"parseInt", "parseFloat", "isNaN", "isFinite",
|
|
"encodeURIComponent", "decodeURIComponent", "encodeURI", "decodeURI",
|
|
# Browser / Node common globals
|
|
"URL", "URLSearchParams", "FormData", "Blob", "File",
|
|
"Headers", "Request", "Response", "AbortController", "AbortSignal",
|
|
"TextEncoder", "TextDecoder", "console",
|
|
# Python built-in callables
|
|
"str", "int", "float", "bool", "list", "dict", "set", "tuple", "bytes",
|
|
"len", "range", "enumerate", "zip", "map", "filter", "sum", "min", "max",
|
|
"print", "open", "isinstance", "type", "super", "sorted", "reversed",
|
|
"any", "all", "abs", "round", "next", "iter", "hash", "id", "repr",
|
|
"callable", "getattr", "setattr", "hasattr", "delattr", "vars", "dir",
|
|
})
|
|
|
|
|
|
def _raise_recursion_limit() -> None:
|
|
if sys.getrecursionlimit() < _RECURSION_LIMIT:
|
|
sys.setrecursionlimit(_RECURSION_LIMIT)
|
|
|
|
|
|
def _safe_extract(extractor: Callable, path: Path) -> dict:
|
|
try:
|
|
return extractor(path)
|
|
except RecursionError:
|
|
print(f" warning: skipped {path} (recursion limit exceeded)", file=sys.stderr, flush=True)
|
|
return {"nodes": [], "edges": [], "error": "recursion_limit_exceeded"}
|
|
except Exception as e:
|
|
if os.environ.get("GRAPHIFY_DEBUG"):
|
|
import traceback
|
|
traceback.print_exc(file=sys.stderr)
|
|
print(f" warning: skipped {path} ({type(e).__name__}: {e})", file=sys.stderr, flush=True)
|
|
return {"nodes": [], "edges": [], "error": f"{type(e).__name__}: {e}"}
|
|
|
|
|
|
def _make_id(*parts: str) -> str:
|
|
r"""Build a stable node ID from one or more name parts.
|
|
|
|
Preserves Unicode letters/digits (CJK, Cyrillic, Arabic, accented Latin,
|
|
etc.) so non-ASCII identifiers produce distinct IDs and don't collapse to
|
|
a single per-file node (#811). NFKC normalization ensures composed and
|
|
decomposed forms of the same character (e.g. é vs e+combining-acute)
|
|
produce the same ID. Must stay in sync with build._normalize_id.
|
|
"""
|
|
combined = "_".join(p.strip("_.") for p in parts if p)
|
|
combined = unicodedata.normalize("NFKC", combined)
|
|
cleaned = re.sub(r"[^\w]+", "_", combined, flags=re.UNICODE)
|
|
cleaned = re.sub(r"_+", "_", cleaned)
|
|
return cleaned.strip("_").casefold()
|
|
|
|
|
|
def _file_stem(path: Path) -> str:
|
|
"""Return a stem qualified with the parent directory name to avoid ID collisions
|
|
when multiple files share the same filename in different directories (#550)."""
|
|
parent = path.parent.name
|
|
if parent and parent not in (".", ""):
|
|
return f"{parent}.{path.stem}"
|
|
return path.stem
|
|
|
|
|
|
def _file_node_id(rel_path: Path) -> str:
|
|
"""File-level node ID matching the skill.md spec: ``{parent_dir}_{stem}`` —
|
|
one parent directory level, no extension. ``rel_path`` MUST be relative to
|
|
the project root so top-level files collapse to a bare stem (``setup.py`` ->
|
|
``setup``) instead of picking up the root directory name. This must equal the
|
|
ID semantic subagents generate, or AST and semantic extraction split a file
|
|
into two disconnected ghost nodes (#1033)."""
|
|
return _make_id(_file_stem(rel_path))
|
|
|
|
|
|
_TSCONFIG_ALIAS_CACHE: dict[str, dict[str, str]] = {}
|
|
_WORKSPACE_PACKAGE_CACHE: dict[str, dict[str, Path]] = {}
|
|
_JS_CACHE_BYPASS_SUFFIXES = {".js", ".jsx", ".mjs", ".ts", ".tsx", ".vue", ".svelte"}
|
|
_JS_RESOLVE_EXTS = (".ts", ".tsx", ".svelte", ".js", ".jsx", ".mjs")
|
|
_JS_INDEX_FILES = ("index.ts", "index.tsx", "index.svelte", "index.js", "index.jsx", "index.mjs")
|
|
|
|
|
|
SEMANTIC_RELATIONS = frozenset({
|
|
"inherits", "implements", "mixes_in", "embeds", "references",
|
|
"calls", "imports", "imports_from", "re_exports", "contains", "method",
|
|
})
|
|
|
|
REFERENCE_CONTEXTS = frozenset({
|
|
"field", "parameter_type", "return_type", "generic_arg", "attribute", "value", "type",
|
|
})
|
|
|
|
|
|
def _source_location(line: int | str | None) -> str | None:
|
|
if line is None:
|
|
return None
|
|
if isinstance(line, str):
|
|
return line if line.startswith("L") else f"L{line}"
|
|
return f"L{line}"
|
|
|
|
|
|
def _semantic_reference_edge(
|
|
source: str,
|
|
target: str,
|
|
context: str,
|
|
source_file: str,
|
|
line: int | str | None,
|
|
) -> dict:
|
|
if context not in REFERENCE_CONTEXTS:
|
|
raise ValueError(f"unknown reference context: {context}")
|
|
return {
|
|
"source": source,
|
|
"target": target,
|
|
"relation": "references",
|
|
"context": context,
|
|
"confidence": "EXTRACTED",
|
|
"source_file": source_file,
|
|
"source_location": _source_location(line),
|
|
"weight": 1.0,
|
|
}
|
|
|
|
|
|
def _resolve_js_import_path(candidate: Path) -> Path:
|
|
"""Resolve a JS/TS/Svelte import target to a local file when it exists."""
|
|
candidate = Path(os.path.normpath(candidate))
|
|
if candidate.is_file():
|
|
return candidate
|
|
|
|
# TS ESM convention: imports often spell .js/.jsx while source is .ts/.tsx.
|
|
if candidate.suffix == ".js":
|
|
ts_candidate = candidate.with_suffix(".ts")
|
|
if ts_candidate.is_file():
|
|
return ts_candidate
|
|
elif candidate.suffix == ".jsx":
|
|
tsx_candidate = candidate.with_suffix(".tsx")
|
|
if tsx_candidate.is_file():
|
|
return tsx_candidate
|
|
|
|
# Append extensions to the full filename, which covers extensionless imports,
|
|
# multi-dot helpers, and Svelte 5 rune files like Foo.svelte.ts.
|
|
for ext in _JS_RESOLVE_EXTS:
|
|
with_ext = candidate.parent / f"{candidate.name}{ext}"
|
|
if with_ext.is_file():
|
|
return with_ext
|
|
|
|
# Only fall back to directory indexes after file candidates lose.
|
|
if candidate.is_dir():
|
|
for index_name in _JS_INDEX_FILES:
|
|
index_candidate = candidate / index_name
|
|
if index_candidate.is_file():
|
|
return index_candidate
|
|
|
|
return candidate
|
|
|
|
|
|
def _strip_jsonc(text: str) -> str:
|
|
"""Strip // line comments, /* */ block comments, and trailing commas from JSONC.
|
|
|
|
Preserves string contents (including // and /* inside strings) by skipping over
|
|
quoted spans first. Required for tsconfig.json files generated by SvelteKit,
|
|
NestJS, Vite, T3, Astro, etc., which use JSONC by default (#700).
|
|
"""
|
|
# Remove block and line comments while leaving string literals untouched.
|
|
pattern = re.compile(
|
|
r'"(?:\\.|[^"\\])*"' # double-quoted string (with escapes)
|
|
r"|/\*.*?\*/" # /* block comment */
|
|
r"|//[^\n]*", # // line comment
|
|
re.DOTALL,
|
|
)
|
|
|
|
def _replace(match: re.Match) -> str:
|
|
token = match.group(0)
|
|
if token.startswith('"'):
|
|
return token
|
|
return ""
|
|
|
|
stripped = pattern.sub(_replace, text)
|
|
# Remove trailing commas before } or ] (allowing whitespace between).
|
|
stripped = re.sub(r",(\s*[}\]])", r"\1", stripped)
|
|
return stripped
|
|
|
|
|
|
def _read_tsconfig_aliases(tsconfig: Path, base_dir: Path, seen: set) -> dict[str, str]:
|
|
"""Recursively read path aliases from a tsconfig, following extends chains.
|
|
|
|
Child config paths override parent. Circular extends are detected via seen set.
|
|
npm package configs (e.g. @tsconfig/svelte) are skipped since they're not on disk.
|
|
Handles JSONC (comments + trailing commas) which is the default tsconfig format
|
|
for SvelteKit, NestJS, Vite, T3, Astro, etc. (#700).
|
|
"""
|
|
if str(tsconfig) in seen:
|
|
return {}
|
|
seen.add(str(tsconfig))
|
|
try:
|
|
raw = tsconfig.read_text(encoding="utf-8")
|
|
except Exception as e:
|
|
print(f" warning: could not read {tsconfig} ({type(e).__name__}: {e})", file=sys.stderr, flush=True)
|
|
return {}
|
|
try:
|
|
data = json.loads(raw)
|
|
except json.JSONDecodeError:
|
|
try:
|
|
data = json.loads(_strip_jsonc(raw))
|
|
except json.JSONDecodeError as e:
|
|
print(f" warning: failed to parse {tsconfig} as JSON/JSONC ({e.msg} at line {e.lineno} col {e.colno})", file=sys.stderr, flush=True)
|
|
return {}
|
|
except Exception as e:
|
|
print(f" warning: failed to parse {tsconfig} ({type(e).__name__}: {e})", file=sys.stderr, flush=True)
|
|
return {}
|
|
|
|
aliases: dict[str, str] = {}
|
|
# `extends` may be a string or, since TypeScript 5.0, an array of paths.
|
|
# For an array, parents are processed in order with later entries
|
|
# overriding earlier ones; the extending config (paths below) overrides
|
|
# all parents. Without the list branch, an array `extends` raised
|
|
# `AttributeError: 'list' object has no attribute 'startswith'`, which
|
|
# _safe_extract turned into a skip of the whole file.
|
|
extends = data.get("extends")
|
|
if isinstance(extends, str):
|
|
extends_list = [extends]
|
|
elif isinstance(extends, list):
|
|
extends_list = [e for e in extends if isinstance(e, str)]
|
|
else:
|
|
extends_list = []
|
|
for ext in extends_list:
|
|
# Skip scoped npm package configs (e.g. @tsconfig/svelte) — not on disk.
|
|
if not ext or ext.startswith("@"):
|
|
continue
|
|
extended_path = (base_dir / ext).resolve()
|
|
if not extended_path.suffix:
|
|
extended_path = extended_path.with_suffix(".json")
|
|
if extended_path.exists():
|
|
aliases.update(_read_tsconfig_aliases(extended_path, extended_path.parent, seen))
|
|
|
|
# tsconfig `paths` are resolved relative to `baseUrl` (itself relative to
|
|
# the tsconfig's directory), not the tsconfig directory directly. Honoring
|
|
# baseUrl is required for the common monorepo / NestJS layout where
|
|
# baseUrl points at a subdirectory, e.g. baseUrl "./src" with
|
|
# "@services/*": ["services/*"] must resolve to <dir>/src/services rather
|
|
# than <dir>/services. Defaults to "." so configs without baseUrl (paths
|
|
# relative to the tsconfig dir, the TS 4.1+ behavior) keep working.
|
|
compiler_options = data.get("compilerOptions", {})
|
|
base_url = compiler_options.get("baseUrl") or "."
|
|
paths_base = base_dir / base_url
|
|
paths = compiler_options.get("paths", {})
|
|
for alias, targets in paths.items():
|
|
if not targets:
|
|
continue
|
|
alias_prefix = alias.rstrip("/*")
|
|
target_base = targets[0].rstrip("/*")
|
|
aliases[alias_prefix] = str(os.path.normpath(paths_base / target_base))
|
|
|
|
return aliases
|
|
|
|
|
|
def _load_tsconfig_aliases(start_dir: Path) -> dict[str, str]:
|
|
"""Walk up from start_dir to find tsconfig.json and return compilerOptions.paths aliases.
|
|
|
|
Follows extends chains so SvelteKit/Nuxt/NestJS inherited aliases are included.
|
|
Returns a dict mapping alias prefix (e.g. "@/") to resolved base dir (e.g. "src/").
|
|
Result is cached by tsconfig path string.
|
|
"""
|
|
current = start_dir.resolve()
|
|
for candidate in [current, *current.parents]:
|
|
tsconfig = candidate / "tsconfig.json"
|
|
if tsconfig.exists():
|
|
key = str(tsconfig)
|
|
if key not in _TSCONFIG_ALIAS_CACHE:
|
|
_TSCONFIG_ALIAS_CACHE[key] = _read_tsconfig_aliases(tsconfig, candidate, seen=set())
|
|
return _TSCONFIG_ALIAS_CACHE[key]
|
|
return {}
|
|
|
|
|
|
def _find_workspace_root(start_dir: Path) -> Path | None:
|
|
current = start_dir.resolve()
|
|
for candidate in [current, *current.parents]:
|
|
if (candidate / "pnpm-workspace.yaml").exists():
|
|
return candidate
|
|
return None
|
|
|
|
|
|
def _workspace_globs(workspace_file: Path) -> list[str]:
|
|
globs: list[str] = []
|
|
in_packages = False
|
|
for raw_line in workspace_file.read_text(encoding="utf-8", errors="replace").splitlines():
|
|
line = raw_line.strip()
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
if line.startswith("packages:"):
|
|
in_packages = True
|
|
continue
|
|
if in_packages and line.startswith("-"):
|
|
value = line[1:].strip().strip("'\"")
|
|
if value and not value.startswith("!"):
|
|
globs.append(value)
|
|
continue
|
|
if in_packages and not raw_line.startswith((" ", "\t")):
|
|
break
|
|
return globs
|
|
|
|
|
|
def _load_workspace_packages(start_dir: Path) -> dict[str, Path]:
|
|
root = _find_workspace_root(start_dir)
|
|
if root is None:
|
|
return {}
|
|
key = str(root)
|
|
if key in _WORKSPACE_PACKAGE_CACHE:
|
|
return _WORKSPACE_PACKAGE_CACHE[key]
|
|
|
|
packages: dict[str, Path] = {}
|
|
for pattern in _workspace_globs(root / "pnpm-workspace.yaml"):
|
|
package_dirs: list[Path] = [root] if pattern in (".", "./") else list(root.glob(pattern))
|
|
for package_dir in package_dirs:
|
|
manifest = package_dir / "package.json"
|
|
if not manifest.is_file():
|
|
continue
|
|
try:
|
|
data = json.loads(manifest.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
continue
|
|
name = data.get("name")
|
|
if isinstance(name, str) and name:
|
|
packages[name] = package_dir
|
|
_WORKSPACE_PACKAGE_CACHE[key] = packages
|
|
return packages
|
|
|
|
|
|
def _package_entry_candidates(package_dir: Path, subpath: str) -> list[Path]:
|
|
manifest = package_dir / "package.json"
|
|
manifest_data: dict[str, Any] = {}
|
|
try:
|
|
manifest_data = json.loads(manifest.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
pass
|
|
|
|
if subpath:
|
|
return [package_dir / subpath]
|
|
|
|
exports = manifest_data.get("exports")
|
|
if isinstance(exports, str):
|
|
return [package_dir / exports]
|
|
if isinstance(exports, dict):
|
|
dot_export = exports.get(".")
|
|
if isinstance(dot_export, str):
|
|
return [package_dir / dot_export]
|
|
if isinstance(dot_export, dict):
|
|
for key in ("types", "import", "default", "svelte"):
|
|
value = dot_export.get(key)
|
|
if isinstance(value, str):
|
|
return [package_dir / value]
|
|
|
|
candidates: list[Path] = []
|
|
for key in ("svelte", "module", "main", "types"):
|
|
value = manifest_data.get(key)
|
|
if isinstance(value, str):
|
|
candidates.append(package_dir / value)
|
|
candidates.append(package_dir / "src/index")
|
|
candidates.append(package_dir / "index")
|
|
return candidates
|
|
|
|
|
|
def _resolve_workspace_import(raw: str, start_dir: Path) -> Path | None:
|
|
packages = _load_workspace_packages(start_dir)
|
|
for package_name, package_dir in packages.items():
|
|
if raw == package_name:
|
|
subpath = ""
|
|
elif raw.startswith(package_name + "/"):
|
|
subpath = raw[len(package_name) + 1:]
|
|
else:
|
|
continue
|
|
for candidate in _package_entry_candidates(package_dir, subpath):
|
|
resolved = _resolve_js_import_path(candidate)
|
|
if resolved.is_file():
|
|
return resolved
|
|
return None
|
|
|
|
|
|
def _resolve_js_module_path(raw: str | Path, start_dir: Path | None = None) -> Path | None:
|
|
"""Resolve a JS/TS module path or specifier to a local source file.
|
|
|
|
With a Path argument this preserves the path-based helper API used by
|
|
import-extension tests. With a string plus start_dir it resolves JS/TS
|
|
module specifiers including relative paths, tsconfig aliases, and workspace
|
|
packages.
|
|
"""
|
|
if isinstance(raw, Path):
|
|
return _resolve_js_import_path(raw)
|
|
if start_dir is None:
|
|
return _resolve_js_import_path(Path(raw))
|
|
if raw.startswith("."):
|
|
return _resolve_js_import_path(start_dir / raw)
|
|
|
|
aliases = _load_tsconfig_aliases(start_dir)
|
|
for alias_prefix, alias_base in aliases.items():
|
|
if raw == alias_prefix or raw.startswith(alias_prefix + "/"):
|
|
rest = raw[len(alias_prefix):].lstrip("/")
|
|
return _resolve_js_import_path(Path(os.path.normpath(Path(alias_base) / rest)))
|
|
|
|
return _resolve_workspace_import(raw, start_dir)
|
|
|
|
|
|
# ── LanguageConfig dataclass ─────────────────────────────────────────────────
|
|
|
|
@dataclass
|
|
class LanguageConfig:
|
|
ts_module: str # e.g. "tree_sitter_python"
|
|
ts_language_fn: str = "language" # attr to call: e.g. tslang.language()
|
|
|
|
class_types: frozenset = frozenset()
|
|
function_types: frozenset = frozenset()
|
|
import_types: frozenset = frozenset()
|
|
call_types: frozenset = frozenset()
|
|
static_prop_types: frozenset = frozenset()
|
|
helper_fn_names: frozenset = frozenset()
|
|
container_bind_methods: frozenset = frozenset()
|
|
event_listener_properties: frozenset = frozenset()
|
|
|
|
# Name extraction
|
|
name_field: str = "name"
|
|
name_fallback_child_types: tuple = ()
|
|
|
|
# Body detection
|
|
body_field: str = "body"
|
|
body_fallback_child_types: tuple = () # e.g. ("declaration_list", "compound_statement")
|
|
|
|
# Call name extraction
|
|
call_function_field: str = "function" # field on call node for callee
|
|
call_accessor_node_types: frozenset = frozenset() # member/attribute nodes
|
|
call_accessor_field: str = "attribute" # field on accessor for method name
|
|
|
|
# Stop recursion at these types in walk_calls
|
|
function_boundary_types: frozenset = frozenset()
|
|
|
|
# Import handler: called for import nodes instead of generic handling
|
|
import_handler: Callable | None = None
|
|
|
|
# Optional custom name resolver for functions (C, C++ declarator unwrapping)
|
|
resolve_function_name_fn: Callable | None = None
|
|
|
|
# Extra label formatting for functions: if True, functions get "name()" label
|
|
function_label_parens: bool = True
|
|
|
|
# Extra walk hook called after generic dispatch (for JS arrow functions, C# namespaces, etc.)
|
|
extra_walk_fn: Callable | None = None
|
|
|
|
|
|
# ── Generic helpers ───────────────────────────────────────────────────────────
|
|
|
|
def _read_text(node, source: bytes) -> str:
|
|
return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
|
|
|
|
|
|
_PYTHON_TYPE_CONTAINERS = frozenset({
|
|
"list", "dict", "set", "tuple", "frozenset", "type",
|
|
"List", "Dict", "Set", "Tuple", "FrozenSet", "Type",
|
|
"Optional", "Union", "Sequence", "Iterable", "Mapping", "MutableMapping",
|
|
"Iterator", "Callable", "Awaitable", "AsyncIterable", "AsyncIterator", "Coroutine",
|
|
"Generator", "AsyncGenerator", "ContextManager", "AsyncContextManager",
|
|
"Annotated", "ClassVar", "Final", "Literal", "Concatenate", "ParamSpec", "TypeVar",
|
|
"None", "Ellipsis",
|
|
})
|
|
|
|
# Scalar builtins and test-mock names that appear as type annotations but carry
|
|
# no useful semantic meaning as graph nodes (#1147). Suppressed at the annotation
|
|
# walker level so they are never created as nodes or emitted as edges.
|
|
_PYTHON_ANNOTATION_NOISE = frozenset({
|
|
# scalar builtins
|
|
"str", "int", "float", "bool", "bytes", "bytearray", "complex", "object",
|
|
"True", "False",
|
|
# unittest.mock
|
|
"MagicMock", "Mock", "AsyncMock", "NonCallableMock",
|
|
"NonCallableMagicMock", "PropertyMock", "patch", "sentinel",
|
|
})
|
|
|
|
|
|
def _python_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None:
|
|
"""Walk a Python type annotation; append (name, role) where role is 'type' or 'generic_arg'.
|
|
|
|
Builtin/typing containers (list, dict, Optional, Union, …) are not emitted as refs themselves,
|
|
but their nested type arguments still count as generic_arg.
|
|
"""
|
|
if node is None:
|
|
return
|
|
t = node.type
|
|
if t == "type":
|
|
for c in node.children:
|
|
if c.is_named:
|
|
_python_collect_type_refs(c, source, generic, out)
|
|
return
|
|
if t == "identifier":
|
|
name = _read_text(node, source)
|
|
if name and name not in _PYTHON_TYPE_CONTAINERS and name not in _PYTHON_ANNOTATION_NOISE:
|
|
out.append((name, "generic_arg" if generic else "type"))
|
|
return
|
|
if t == "attribute":
|
|
tail = _read_text(node, source).rsplit(".", 1)[-1]
|
|
if tail and tail not in _PYTHON_TYPE_CONTAINERS and tail not in _PYTHON_ANNOTATION_NOISE:
|
|
out.append((tail, "generic_arg" if generic else "type"))
|
|
return
|
|
if t == "generic_type":
|
|
for c in node.children:
|
|
if c.type == "identifier":
|
|
container = _read_text(c, source)
|
|
if container and container not in _PYTHON_TYPE_CONTAINERS and container not in _PYTHON_ANNOTATION_NOISE:
|
|
out.append((container, "generic_arg" if generic else "type"))
|
|
elif c.type == "type_parameter":
|
|
for sub in c.children:
|
|
if sub.is_named:
|
|
_python_collect_type_refs(sub, source, True, out)
|
|
return
|
|
if t == "subscript":
|
|
value = node.child_by_field_name("value")
|
|
if value is not None:
|
|
_python_collect_type_refs(value, source, generic, out)
|
|
for c in node.children:
|
|
if c is value or not c.is_named:
|
|
continue
|
|
_python_collect_type_refs(c, source, True, out)
|
|
return
|
|
if node.is_named:
|
|
for c in node.children:
|
|
if c.is_named:
|
|
_python_collect_type_refs(c, source, generic, out)
|
|
|
|
|
|
def _csharp_pre_scan_interfaces(root_node, source: bytes) -> set[str]:
|
|
"""Return names declared as `interface` in this C# compilation unit."""
|
|
out: set[str] = set()
|
|
stack = [root_node]
|
|
while stack:
|
|
n = stack.pop()
|
|
if n.type == "interface_declaration":
|
|
name_node = n.child_by_field_name("name")
|
|
if name_node is not None:
|
|
text = _read_text(name_node, source)
|
|
if text:
|
|
out.add(text)
|
|
stack.extend(n.children)
|
|
return out
|
|
|
|
|
|
def _csharp_classify_base(name: str, interface_names: set[str]) -> str:
|
|
"""`implements` if the base name is an interface (declared or by I-prefix convention), else `inherits`."""
|
|
if name in interface_names:
|
|
return "implements"
|
|
if len(name) >= 2 and name[0] == "I" and name[1].isupper():
|
|
return "implements"
|
|
return "inherits"
|
|
|
|
|
|
def _csharp_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None:
|
|
"""Walk a C# type expression; append (name, role) tuples (role is 'type' or 'generic_arg')."""
|
|
if node is None:
|
|
return
|
|
t = node.type
|
|
if t == "predefined_type":
|
|
return
|
|
if t == "identifier":
|
|
name = _read_text(node, source)
|
|
if name:
|
|
out.append((name, "generic_arg" if generic else "type"))
|
|
return
|
|
if t == "qualified_name":
|
|
text = _read_text(node, source).rsplit(".", 1)[-1]
|
|
if text:
|
|
out.append((text, "generic_arg" if generic else "type"))
|
|
return
|
|
if t == "generic_name":
|
|
name_child = node.child_by_field_name("name")
|
|
if name_child is None:
|
|
for sub in node.children:
|
|
if sub.type == "identifier":
|
|
name_child = sub
|
|
break
|
|
if name_child is not None:
|
|
name = _read_text(name_child, source)
|
|
if name:
|
|
out.append((name, "generic_arg" if generic else "type"))
|
|
for sub in node.children:
|
|
if sub.type == "type_argument_list":
|
|
for arg in sub.children:
|
|
if arg.is_named:
|
|
_csharp_collect_type_refs(arg, source, True, out)
|
|
return
|
|
if t in ("nullable_type", "array_type", "pointer_type", "ref_type"):
|
|
for c in node.children:
|
|
if c.is_named:
|
|
_csharp_collect_type_refs(c, source, generic, out)
|
|
return
|
|
if node.is_named:
|
|
for c in node.children:
|
|
if c.is_named:
|
|
_csharp_collect_type_refs(c, source, generic, out)
|
|
|
|
|
|
def _csharp_attribute_names(method_node, source: bytes) -> list[str]:
|
|
"""Collect attribute names from a C# method/declaration's attribute_list children."""
|
|
names: list[str] = []
|
|
for child in method_node.children:
|
|
if child.type != "attribute_list":
|
|
continue
|
|
for attr in child.children:
|
|
if attr.type != "attribute":
|
|
continue
|
|
name_node = attr.child_by_field_name("name")
|
|
if name_node is None:
|
|
for sub in attr.children:
|
|
if sub.type in ("identifier", "qualified_name"):
|
|
name_node = sub
|
|
break
|
|
if name_node is not None:
|
|
text = _read_text(name_node, source).rsplit(".", 1)[-1]
|
|
if text:
|
|
names.append(text)
|
|
return names
|
|
|
|
|
|
def _java_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None:
|
|
"""Walk a Java type expression; append (name, role) tuples."""
|
|
if node is None:
|
|
return
|
|
t = node.type
|
|
if t in ("integral_type", "floating_point_type", "boolean_type", "void_type"):
|
|
return
|
|
if t == "type_identifier":
|
|
name = _read_text(node, source)
|
|
if name:
|
|
out.append((name, "generic_arg" if generic else "type"))
|
|
return
|
|
if t == "scoped_type_identifier":
|
|
text = _read_text(node, source).rsplit(".", 1)[-1]
|
|
if text:
|
|
out.append((text, "generic_arg" if generic else "type"))
|
|
return
|
|
if t == "generic_type":
|
|
for c in node.children:
|
|
if c.type in ("type_identifier", "scoped_type_identifier"):
|
|
text = _read_text(c, source).rsplit(".", 1)[-1]
|
|
if text:
|
|
out.append((text, "generic_arg" if generic else "type"))
|
|
break
|
|
for c in node.children:
|
|
if c.type == "type_arguments":
|
|
for arg in c.children:
|
|
if arg.is_named:
|
|
_java_collect_type_refs(arg, source, True, out)
|
|
return
|
|
if t == "array_type":
|
|
for c in node.children:
|
|
if c.is_named:
|
|
_java_collect_type_refs(c, source, generic, out)
|
|
return
|
|
if node.is_named:
|
|
for c in node.children:
|
|
if c.is_named:
|
|
_java_collect_type_refs(c, source, generic, out)
|
|
|
|
|
|
def _java_method_annotation_names(method_node, source: bytes) -> list[str]:
|
|
"""Collect annotation names from a Java method's `modifiers` child."""
|
|
names: list[str] = []
|
|
modifiers = None
|
|
for child in method_node.children:
|
|
if child.type == "modifiers":
|
|
modifiers = child
|
|
break
|
|
if modifiers is None:
|
|
return names
|
|
for anno in modifiers.children:
|
|
if anno.type not in ("marker_annotation", "annotation"):
|
|
continue
|
|
name_node = anno.child_by_field_name("name")
|
|
if name_node is None:
|
|
for sub in anno.children:
|
|
if sub.type in ("identifier", "scoped_identifier", "type_identifier"):
|
|
name_node = sub
|
|
break
|
|
if name_node is not None:
|
|
text = _read_text(name_node, source).rsplit(".", 1)[-1]
|
|
if text:
|
|
names.append(text)
|
|
return names
|
|
|
|
|
|
_GO_PREDECLARED_TYPES = frozenset({
|
|
"bool", "byte", "complex64", "complex128", "error", "float32", "float64",
|
|
"int", "int8", "int16", "int32", "int64", "rune", "string",
|
|
"uint", "uint8", "uint16", "uint32", "uint64", "uintptr", "any", "comparable",
|
|
})
|
|
|
|
|
|
def _go_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None:
|
|
"""Walk a Go type expression; append (name, role) tuples."""
|
|
if node is None:
|
|
return
|
|
t = node.type
|
|
if t == "type_identifier":
|
|
text = _read_text(node, source)
|
|
if text and text not in _GO_PREDECLARED_TYPES:
|
|
out.append((text, "generic_arg" if generic else "type"))
|
|
return
|
|
if t == "qualified_type":
|
|
text = _read_text(node, source).rsplit(".", 1)[-1]
|
|
if text and text not in _GO_PREDECLARED_TYPES:
|
|
out.append((text, "generic_arg" if generic else "type"))
|
|
return
|
|
if t == "generic_type":
|
|
type_field = node.child_by_field_name("type")
|
|
if type_field is not None:
|
|
sub: list[tuple[str, str]] = []
|
|
_go_collect_type_refs(type_field, source, generic, sub)
|
|
out.extend(sub)
|
|
for c in node.children:
|
|
if c.type == "type_arguments":
|
|
for arg in c.children:
|
|
if arg.is_named:
|
|
_go_collect_type_refs(arg, source, True, out)
|
|
return
|
|
if t in ("pointer_type", "slice_type", "array_type", "map_type",
|
|
"channel_type", "parenthesized_type"):
|
|
for c in node.children:
|
|
if c.is_named:
|
|
_go_collect_type_refs(c, source, generic, out)
|
|
return
|
|
if node.is_named:
|
|
for c in node.children:
|
|
if c.is_named:
|
|
_go_collect_type_refs(c, source, generic, out)
|
|
|
|
|
|
def _rust_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None:
|
|
"""Walk a Rust type expression; append (name, role) tuples."""
|
|
if node is None:
|
|
return
|
|
t = node.type
|
|
if t == "primitive_type":
|
|
return
|
|
if t == "type_identifier":
|
|
text = _read_text(node, source)
|
|
if text:
|
|
out.append((text, "generic_arg" if generic else "type"))
|
|
return
|
|
if t == "scoped_type_identifier":
|
|
text = _read_text(node, source).rsplit("::", 1)[-1]
|
|
if text:
|
|
out.append((text, "generic_arg" if generic else "type"))
|
|
return
|
|
if t == "generic_type":
|
|
name_node = node.child_by_field_name("type")
|
|
if name_node is None:
|
|
for c in node.children:
|
|
if c.type in ("type_identifier", "scoped_type_identifier"):
|
|
name_node = c
|
|
break
|
|
if name_node is not None:
|
|
text = _read_text(name_node, source).rsplit("::", 1)[-1]
|
|
if text:
|
|
out.append((text, "generic_arg" if generic else "type"))
|
|
for c in node.children:
|
|
if c.type == "type_arguments":
|
|
for arg in c.children:
|
|
if arg.is_named:
|
|
_rust_collect_type_refs(arg, source, True, out)
|
|
return
|
|
if t in ("reference_type", "pointer_type", "array_type", "tuple_type", "slice_type"):
|
|
for c in node.children:
|
|
if c.is_named:
|
|
_rust_collect_type_refs(c, source, generic, out)
|
|
return
|
|
if node.is_named:
|
|
for c in node.children:
|
|
if c.is_named:
|
|
_rust_collect_type_refs(c, source, generic, out)
|
|
|
|
|
|
def _php_name_text(node, source: bytes) -> str | None:
|
|
"""Return the unqualified name text from a PHP `name`/`qualified_name` node."""
|
|
if node is None:
|
|
return None
|
|
return _read_text(node, source).rsplit("\\", 1)[-1] or None
|
|
|
|
|
|
def _php_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None:
|
|
"""Walk a PHP type expression; append (name, role) tuples."""
|
|
if node is None:
|
|
return
|
|
t = node.type
|
|
if t == "primitive_type":
|
|
return
|
|
if t == "named_type":
|
|
for c in node.children:
|
|
if c.type in ("name", "qualified_name"):
|
|
text = _php_name_text(c, source)
|
|
if text:
|
|
out.append((text, "generic_arg" if generic else "type"))
|
|
return
|
|
return
|
|
if t in ("name", "qualified_name"):
|
|
text = _php_name_text(node, source)
|
|
if text:
|
|
out.append((text, "generic_arg" if generic else "type"))
|
|
return
|
|
if t in ("nullable_type", "union_type", "intersection_type", "optional_type"):
|
|
for c in node.children:
|
|
if c.is_named:
|
|
_php_collect_type_refs(c, source, generic, out)
|
|
return
|
|
if node.is_named:
|
|
for c in node.children:
|
|
if c.is_named:
|
|
_php_collect_type_refs(c, source, generic, out)
|
|
|
|
|
|
def _php_method_return_type_node(method_node):
|
|
"""Return the named_type/primitive_type node sitting after formal_parameters."""
|
|
saw_params = False
|
|
for c in method_node.children:
|
|
if c.type == "formal_parameters":
|
|
saw_params = True
|
|
continue
|
|
if saw_params and c.is_named and c.type not in ("compound_statement",):
|
|
if c.type in ("named_type", "primitive_type", "nullable_type",
|
|
"union_type", "intersection_type", "optional_type"):
|
|
return c
|
|
return None
|
|
|
|
|
|
def _kotlin_user_type_name(user_type_node, source: bytes) -> str | None:
|
|
"""Return the head identifier text from a Kotlin user_type node (without generics)."""
|
|
if user_type_node is None:
|
|
return None
|
|
for c in user_type_node.children:
|
|
if c.type == "type_identifier":
|
|
text = _read_text(c, source)
|
|
return text or None
|
|
if c.type == "identifier":
|
|
text = _read_text(c, source)
|
|
return text or None
|
|
if c.type == "simple_user_type":
|
|
for sub in c.children:
|
|
if sub.type in ("identifier", "type_identifier"):
|
|
text = _read_text(sub, source)
|
|
return text or None
|
|
return None
|
|
|
|
|
|
def _kotlin_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None:
|
|
"""Walk a Kotlin type expression; append (name, role) tuples."""
|
|
if node is None:
|
|
return
|
|
t = node.type
|
|
if t in ("integral_literal", "boolean_literal"):
|
|
return
|
|
if t == "user_type":
|
|
for c in node.children:
|
|
if c.type in ("identifier", "type_identifier"):
|
|
text = _read_text(c, source)
|
|
if text:
|
|
out.append((text, "generic_arg" if generic else "type"))
|
|
break
|
|
if c.type == "simple_user_type":
|
|
for sub in c.children:
|
|
if sub.type in ("identifier", "type_identifier"):
|
|
text = _read_text(sub, source)
|
|
if text:
|
|
out.append((text, "generic_arg" if generic else "type"))
|
|
break
|
|
break
|
|
for c in node.children:
|
|
if c.type == "type_arguments":
|
|
for arg in c.children:
|
|
if arg.type == "type_projection":
|
|
for sub in arg.children:
|
|
if sub.is_named:
|
|
_kotlin_collect_type_refs(sub, source, True, out)
|
|
elif arg.is_named:
|
|
_kotlin_collect_type_refs(arg, source, True, out)
|
|
return
|
|
if t in ("identifier", "type_identifier"):
|
|
text = _read_text(node, source)
|
|
if text:
|
|
out.append((text, "generic_arg" if generic else "type"))
|
|
return
|
|
if t in ("nullable_type", "parenthesized_type", "type_reference"):
|
|
for c in node.children:
|
|
if c.is_named:
|
|
_kotlin_collect_type_refs(c, source, generic, out)
|
|
return
|
|
if node.is_named:
|
|
for c in node.children:
|
|
if c.is_named:
|
|
_kotlin_collect_type_refs(c, source, generic, out)
|
|
|
|
|
|
def _kotlin_property_type_node(property_node):
|
|
"""Find the user_type node within a Kotlin property_declaration."""
|
|
for c in property_node.children:
|
|
if c.type == "variable_declaration":
|
|
for sub in c.children:
|
|
if sub.type in ("user_type", "nullable_type", "type_reference"):
|
|
return sub
|
|
if c.type in ("user_type", "nullable_type", "type_reference"):
|
|
return c
|
|
return None
|
|
|
|
|
|
def _kotlin_function_return_type_node(func_node):
|
|
"""Find the return-type node of a Kotlin function_declaration (the type after `: ` post-params)."""
|
|
saw_params = False
|
|
saw_colon = False
|
|
for c in func_node.children:
|
|
if c.type == "function_value_parameters":
|
|
saw_params = True
|
|
continue
|
|
if saw_params and c.type == ":":
|
|
saw_colon = True
|
|
continue
|
|
if saw_colon:
|
|
if c.is_named:
|
|
return c
|
|
return None
|
|
|
|
|
|
def _swift_declaration_keyword(node) -> str | None:
|
|
"""Return the leading kind token for a Swift class_declaration: class/struct/enum/extension/actor."""
|
|
for c in node.children:
|
|
if not c.is_named and c.type in ("class", "struct", "enum", "extension", "actor"):
|
|
return c.type
|
|
return None
|
|
|
|
|
|
def _swift_pre_scan(root_node, source: bytes) -> tuple[set[str], set[str]]:
|
|
"""Pre-scan a Swift compilation unit and return (protocol_names, class_like_names)."""
|
|
protocols: set[str] = set()
|
|
classes: set[str] = set()
|
|
stack = [root_node]
|
|
while stack:
|
|
n = stack.pop()
|
|
if n.type == "protocol_declaration":
|
|
name_node = n.child_by_field_name("name")
|
|
if name_node is None:
|
|
for c in n.children:
|
|
if c.type == "type_identifier":
|
|
name_node = c
|
|
break
|
|
if name_node is not None:
|
|
text = _read_text(name_node, source)
|
|
if text:
|
|
protocols.add(text)
|
|
elif n.type == "class_declaration":
|
|
kw = _swift_declaration_keyword(n)
|
|
if kw in ("class", "struct", "enum", "actor"):
|
|
name_node = n.child_by_field_name("name")
|
|
if name_node is not None:
|
|
text = _read_text(name_node, source)
|
|
if text:
|
|
classes.add(text)
|
|
stack.extend(n.children)
|
|
return protocols, classes
|
|
|
|
|
|
def _swift_classify_base(name: str, kind: str | None, is_first: bool,
|
|
protocols: set[str], classes: set[str]) -> str:
|
|
"""Classify a Swift inheritance_specifier entry as `inherits` or `implements`."""
|
|
if name in protocols:
|
|
return "implements"
|
|
if name in classes:
|
|
return "inherits"
|
|
# struct/enum/extension/actor cannot inherit a class — all conformances are protocols.
|
|
if kind in ("struct", "enum", "extension", "actor"):
|
|
return "implements"
|
|
# `class`: first entry is conventionally the base class; subsequent are protocols.
|
|
return "inherits" if is_first else "implements"
|
|
|
|
|
|
def _swift_user_type_name(user_type_node, source: bytes) -> str | None:
|
|
"""Return the head type_identifier text from a Swift user_type node (without generics)."""
|
|
if user_type_node is None:
|
|
return None
|
|
for c in user_type_node.children:
|
|
if c.type == "type_identifier":
|
|
text = _read_text(c, source)
|
|
return text or None
|
|
return None
|
|
|
|
|
|
def _swift_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None:
|
|
"""Walk a Swift type expression; append (name, role) tuples (role 'type' or 'generic_arg')."""
|
|
if node is None:
|
|
return
|
|
t = node.type
|
|
if t == "type_annotation":
|
|
for c in node.children:
|
|
if c.is_named:
|
|
_swift_collect_type_refs(c, source, generic, out)
|
|
return
|
|
if t == "user_type":
|
|
for c in node.children:
|
|
if c.type == "type_identifier":
|
|
text = _read_text(c, source)
|
|
if text:
|
|
out.append((text, "generic_arg" if generic else "type"))
|
|
break
|
|
for c in node.children:
|
|
if c.type == "type_arguments":
|
|
for arg in c.children:
|
|
if arg.is_named:
|
|
_swift_collect_type_refs(arg, source, True, out)
|
|
return
|
|
if t == "type_identifier":
|
|
text = _read_text(node, source)
|
|
if text:
|
|
out.append((text, "generic_arg" if generic else "type"))
|
|
return
|
|
if t in ("optional_type", "implicitly_unwrapped_optional_type", "array_type",
|
|
"dictionary_type", "tuple_type"):
|
|
for c in node.children:
|
|
if c.is_named:
|
|
_swift_collect_type_refs(c, source, generic, out)
|
|
return
|
|
if node.is_named:
|
|
for c in node.children:
|
|
if c.is_named:
|
|
_swift_collect_type_refs(c, source, generic, out)
|
|
|
|
|
|
def _swift_property_type_node(property_node):
|
|
"""Return the type_annotation child of a Swift property_declaration, if any."""
|
|
for c in property_node.children:
|
|
if c.type == "type_annotation":
|
|
return c
|
|
return None
|
|
|
|
|
|
# ── C / C++ type-ref helpers ─────────────────────────────────────────────────
|
|
|
|
_C_PRIMITIVE_TYPE_NODES = frozenset({
|
|
"primitive_type", "sized_type_specifier", "auto", "placeholder_type_specifier",
|
|
})
|
|
|
|
|
|
def _c_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None:
|
|
"""Walk a C type expression; append (name, role) tuples for user-defined types.
|
|
Skips primitive types and qualifiers; recognises type_identifier."""
|
|
if node is None or node.type in _C_PRIMITIVE_TYPE_NODES:
|
|
return
|
|
t = node.type
|
|
if t == "type_identifier":
|
|
text = _read_text(node, source)
|
|
if text:
|
|
out.append((text, "generic_arg" if generic else "type"))
|
|
return
|
|
if t in ("pointer_declarator", "reference_declarator", "array_declarator",
|
|
"type_qualifier", "type_descriptor", "abstract_pointer_declarator",
|
|
"abstract_reference_declarator", "abstract_array_declarator"):
|
|
for c in node.children:
|
|
if c.is_named:
|
|
_c_collect_type_refs(c, source, generic, out)
|
|
|
|
|
|
def _cpp_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None:
|
|
"""Walk a C++ type expression; append (name, role) tuples.
|
|
Resolves qualified_identifier tails (std::string → string) and template_type
|
|
base + arguments (std::vector<HttpClient> → vector + HttpClient as generic_arg)."""
|
|
if node is None or node.type in _C_PRIMITIVE_TYPE_NODES:
|
|
return
|
|
t = node.type
|
|
if t == "type_identifier":
|
|
text = _read_text(node, source)
|
|
if text:
|
|
out.append((text, "generic_arg" if generic else "type"))
|
|
return
|
|
if t == "qualified_identifier":
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node is not None:
|
|
_cpp_collect_type_refs(name_node, source, generic, out)
|
|
return
|
|
if t == "template_type":
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node is not None:
|
|
text = _read_text(name_node, source)
|
|
if text:
|
|
out.append((text, "generic_arg" if generic else "type"))
|
|
args_node = node.child_by_field_name("arguments")
|
|
if args_node is not None:
|
|
for c in args_node.children:
|
|
if c.is_named:
|
|
_cpp_collect_type_refs(c, source, True, out)
|
|
return
|
|
if t in ("type_descriptor", "pointer_declarator", "reference_declarator",
|
|
"array_declarator", "type_qualifier", "abstract_pointer_declarator",
|
|
"abstract_reference_declarator", "abstract_array_declarator"):
|
|
for c in node.children:
|
|
if c.is_named:
|
|
_cpp_collect_type_refs(c, source, generic, out)
|
|
|
|
|
|
# ── Scala type-ref helpers ───────────────────────────────────────────────────
|
|
|
|
def _scala_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None:
|
|
"""Walk a Scala type expression; append (name, role) tuples.
|
|
Handles type_identifier, generic_type (List[T]), and common type wrappers."""
|
|
if node is None:
|
|
return
|
|
t = node.type
|
|
if t == "type_identifier":
|
|
text = _read_text(node, source)
|
|
if text:
|
|
out.append((text, "generic_arg" if generic else "type"))
|
|
return
|
|
if t == "generic_type":
|
|
base = node.child_by_field_name("type")
|
|
if base is None:
|
|
for c in node.children:
|
|
if c.type == "type_identifier":
|
|
base = c
|
|
break
|
|
if base is not None and base.type == "type_identifier":
|
|
text = _read_text(base, source)
|
|
if text:
|
|
out.append((text, "generic_arg" if generic else "type"))
|
|
for c in node.children:
|
|
if c.type == "type_arguments":
|
|
for arg in c.children:
|
|
if arg.is_named:
|
|
_scala_collect_type_refs(arg, source, True, out)
|
|
return
|
|
if t in ("compound_type", "infix_type", "function_type", "tuple_type",
|
|
"annotated_type", "projected_type"):
|
|
for c in node.children:
|
|
if c.is_named:
|
|
_scala_collect_type_refs(c, source, generic, out)
|
|
|
|
|
|
def _python_collect_param_refs(params_node, source: bytes) -> list[tuple[str, str]]:
|
|
"""Collect type refs from each typed parameter under a `parameters` node."""
|
|
out: list[tuple[str, str]] = []
|
|
if params_node is None:
|
|
return out
|
|
for child in params_node.children:
|
|
if child.type in ("typed_parameter", "typed_default_parameter"):
|
|
type_node = child.child_by_field_name("type")
|
|
_python_collect_type_refs(type_node, source, False, out)
|
|
return out
|
|
|
|
|
|
def _resolve_name(node, source: bytes, config: LanguageConfig) -> str | None:
|
|
"""Get the name from a node using config.name_field, falling back to child types."""
|
|
if config.resolve_function_name_fn is not None:
|
|
# For C/C++ where the name is inside a declarator
|
|
return None # caller handles this separately
|
|
n = node.child_by_field_name(config.name_field)
|
|
if n:
|
|
return _read_text(n, source)
|
|
for child in node.children:
|
|
if child.type in config.name_fallback_child_types:
|
|
return _read_text(child, source)
|
|
return None
|
|
|
|
|
|
def _find_body(node, config: LanguageConfig):
|
|
"""Find the body node using config.body_field, falling back to child types."""
|
|
b = node.child_by_field_name(config.body_field)
|
|
if b:
|
|
return b
|
|
for child in node.children:
|
|
if child.type in config.body_fallback_child_types:
|
|
return child
|
|
return None
|
|
|
|
|
|
# ── Import handlers ───────────────────────────────────────────────────────────
|
|
|
|
def _import_python(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> None:
|
|
t = node.type
|
|
if t == "import_statement":
|
|
for child in node.children:
|
|
if child.type in ("dotted_name", "aliased_import"):
|
|
raw = _read_text(child, source)
|
|
module_name = raw.split(" as ")[0].strip().lstrip(".")
|
|
tgt_nid = _make_id(module_name)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
elif t == "import_from_statement":
|
|
module_node = node.child_by_field_name("module_name")
|
|
if module_node:
|
|
raw = _read_text(module_node, source)
|
|
if raw.startswith("."):
|
|
# Relative import - resolve to full path so IDs match file node IDs
|
|
dots = len(raw) - len(raw.lstrip("."))
|
|
module_name = raw.lstrip(".")
|
|
base = Path(str_path).parent
|
|
for _ in range(dots - 1):
|
|
base = base.parent
|
|
rel = (module_name.replace(".", "/") + ".py") if module_name else "__init__.py"
|
|
tgt_nid = _make_id(str(base / rel))
|
|
else:
|
|
tgt_nid = _make_id(raw)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports_from",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
|
|
def _resolve_js_import_target(raw: str, str_path: str) -> "tuple[str, Path | None] | None":
|
|
"""Resolve a JS/TS import path string to (target_nid, resolved_path).
|
|
|
|
Handles relative paths, tsconfig path aliases, workspace packages, and
|
|
bare/scoped imports.
|
|
Returns None if `raw` is empty.
|
|
"""
|
|
if not raw:
|
|
return None
|
|
resolved_path = _resolve_js_module_path(raw, Path(str_path).parent)
|
|
if resolved_path is not None:
|
|
return _make_id(str(resolved_path)), resolved_path
|
|
module_name = raw.split("/")[-1]
|
|
if not module_name:
|
|
return None
|
|
return _make_id(module_name), None
|
|
|
|
|
|
def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> None:
|
|
is_reexport = node.type == "export_statement"
|
|
# Only handle export_statement if it has a `from` clause (re-export).
|
|
# Pure exports like `export const x = 1` or `export { localVar }` have no source module.
|
|
if is_reexport:
|
|
has_from = any(child.type == "from" or (_read_text(child, source) == "from") for child in node.children if child.type in ("from", "identifier"))
|
|
if not has_from:
|
|
# Check for string child (source path) as a more reliable indicator
|
|
has_from = any(child.type == "string" for child in node.children)
|
|
if not has_from:
|
|
return
|
|
|
|
resolved_path: "Path | None" = None
|
|
for child in node.children:
|
|
if child.type == "string":
|
|
raw = _read_text(child, source).strip("'\"` ")
|
|
resolved = _resolve_js_import_target(raw, str_path)
|
|
if resolved is None:
|
|
break
|
|
tgt_nid, resolved_path = resolved
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports_from",
|
|
"context": "re-export" if is_reexport else "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
|
|
# Emit symbol-level edges for named imports/re-exports from local/aliased files.
|
|
# e.g. `import { Foo, type Bar } from './bar'` → file → Foo, file → Bar (EXTRACTED)
|
|
# e.g. `export { Foo } from './bar'` → file → Foo (re_exports edge)
|
|
# Uses the same _make_id(target_stem, name) key that _extract_generic emits when
|
|
# defining the symbol, so these edges wire importers directly to existing symbol nodes.
|
|
if resolved_path is not None:
|
|
target_stem = _file_stem(resolved_path)
|
|
line = node.start_point[0] + 1
|
|
|
|
if is_reexport:
|
|
# Handle: export { foo, bar } from './module'
|
|
# export { default as baz } from './module'
|
|
for child in node.children:
|
|
if child.type == "export_clause":
|
|
for spec in child.children:
|
|
if spec.type == "export_specifier":
|
|
# The exported name is the local name from the source module
|
|
name_node = spec.child_by_field_name("name")
|
|
if name_node:
|
|
sym = _read_text(name_node, source)
|
|
if sym == "default":
|
|
continue # skip default re-exports for ID matching
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": _make_id(target_stem, sym),
|
|
"relation": "re_exports",
|
|
"context": "re-export",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
else:
|
|
# Handle: import { Foo, type Bar } from './bar'
|
|
for child in node.children:
|
|
if child.type == "import_clause":
|
|
for sub in child.children:
|
|
if sub.type == "named_imports":
|
|
for spec in sub.children:
|
|
if spec.type == "import_specifier":
|
|
name_node = spec.child_by_field_name("name")
|
|
if name_node:
|
|
sym = _read_text(name_node, source)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": _make_id(target_stem, sym),
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
|
|
def _dynamic_import_js(node, source: bytes, caller_nid: str, str_path: str, edges: list,
|
|
seen_dyn_pairs: set) -> bool:
|
|
"""Detect dynamic import() calls in JS/TS and emit imports_from edges.
|
|
|
|
Handles patterns like:
|
|
await import('./foo.js')
|
|
import('./foo.js').then(...)
|
|
const m = await import(`./foo`)
|
|
|
|
Returns True if the node was a dynamic import (caller should skip normal call handling).
|
|
"""
|
|
# Dynamic import is a call_expression whose function child is the keyword "import".
|
|
# tree-sitter-typescript parses `import('...')` as call_expression with first child
|
|
# being an "import" token (type="import").
|
|
func_node = node.child_by_field_name("function")
|
|
if func_node is None:
|
|
# Fallback: check first child directly (some TS versions)
|
|
if node.children and _read_text(node.children[0], source) == "import":
|
|
func_node = node.children[0]
|
|
else:
|
|
return False
|
|
if _read_text(func_node, source) != "import":
|
|
return False
|
|
|
|
# Extract the module path from the arguments
|
|
args = node.child_by_field_name("arguments")
|
|
if args is None:
|
|
return True # It's an import() but no args — skip
|
|
for arg in args.children:
|
|
if arg.type == "template_string":
|
|
# Skip dynamic template literals — path can't be statically resolved
|
|
if any(c.type == "template_substitution" for c in arg.children):
|
|
break
|
|
raw = _read_text(arg, source).strip("`")
|
|
elif arg.type == "string":
|
|
raw = _read_text(arg, source).strip("'\" ")
|
|
else:
|
|
continue
|
|
if not raw:
|
|
break
|
|
# Resolve path using the same logic as static imports.
|
|
resolved = _resolve_js_import_target(raw, str_path)
|
|
if resolved is None:
|
|
break
|
|
tgt_nid, _ = resolved
|
|
pair = (caller_nid, tgt_nid)
|
|
if pair not in seen_dyn_pairs:
|
|
seen_dyn_pairs.add(pair)
|
|
edges.append({
|
|
"source": caller_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports_from",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
return True
|
|
|
|
|
|
def _import_java(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> None:
|
|
def _walk_scoped(n) -> str:
|
|
parts: list[str] = []
|
|
cur = n
|
|
while cur:
|
|
if cur.type == "scoped_identifier":
|
|
name_node = cur.child_by_field_name("name")
|
|
if name_node:
|
|
parts.append(_read_text(name_node, source))
|
|
cur = cur.child_by_field_name("scope")
|
|
elif cur.type == "identifier":
|
|
parts.append(_read_text(cur, source))
|
|
break
|
|
else:
|
|
break
|
|
parts.reverse()
|
|
return ".".join(parts)
|
|
|
|
for child in node.children:
|
|
if child.type in ("scoped_identifier", "identifier"):
|
|
path_str = _walk_scoped(child)
|
|
module_name = path_str.split(".")[-1].strip("*").strip(".") or (
|
|
path_str.split(".")[-2] if len(path_str.split(".")) > 1 else path_str
|
|
)
|
|
if module_name:
|
|
tgt_nid = _make_id(module_name)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
|
|
|
|
def _resolve_c_include_path(raw: str, str_path: str) -> "Path | None":
|
|
"""Resolve a quoted #include path to a real file on disk.
|
|
|
|
Searches relative to the including file's directory. Returns None for
|
|
system headers (<...>) or paths that don't exist on disk.
|
|
"""
|
|
if not raw:
|
|
return None
|
|
candidate = (Path(str_path).parent / raw).resolve()
|
|
if candidate.is_file():
|
|
return candidate
|
|
return None
|
|
|
|
|
|
def _import_c(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> None:
|
|
for child in node.children:
|
|
if child.type in ("string_literal", "system_lib_string", "string"):
|
|
raw = _read_text(child, source).strip('"<> ')
|
|
# Quoted includes: try to resolve to a real file so the target ID
|
|
# matches the node ID _extract_generic creates for that file.
|
|
if child.type != "system_lib_string":
|
|
resolved = _resolve_c_include_path(raw, str_path)
|
|
if resolved is not None:
|
|
tgt_nid = _make_id(str(resolved))
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
module_name = raw.split("/")[-1].split(".")[0]
|
|
if module_name:
|
|
tgt_nid = _make_id(module_name)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
|
|
|
|
def _import_csharp(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> None:
|
|
for child in node.children:
|
|
if child.type in ("qualified_name", "identifier", "name_equals"):
|
|
raw = _read_text(child, source)
|
|
module_name = raw.split(".")[-1].strip()
|
|
if module_name:
|
|
tgt_nid = _make_id(module_name)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
|
|
|
|
def _import_kotlin(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> None:
|
|
path_node = node.child_by_field_name("path")
|
|
if path_node:
|
|
raw = _read_text(path_node, source)
|
|
module_name = raw.split(".")[-1].strip()
|
|
if module_name:
|
|
tgt_nid = _make_id(module_name)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
return
|
|
# Fallback: find identifier child
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
raw = _read_text(child, source)
|
|
tgt_nid = _make_id(raw)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
|
|
|
|
def _import_scala(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> None:
|
|
for child in node.children:
|
|
if child.type in ("stable_id", "identifier"):
|
|
raw = _read_text(child, source)
|
|
module_name = raw.split(".")[-1].strip("{} ")
|
|
if module_name and module_name != "_":
|
|
tgt_nid = _make_id(module_name)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
|
|
|
|
def _import_php(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> None:
|
|
for child in node.children:
|
|
if child.type in ("qualified_name", "name", "identifier"):
|
|
raw = _read_text(child, source)
|
|
module_name = raw.split("\\")[-1].strip()
|
|
if module_name:
|
|
tgt_nid = _make_id(module_name)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
|
|
|
|
# ── C/C++ function name helpers ───────────────────────────────────────────────
|
|
|
|
def _get_c_func_name(node, source: bytes) -> str | None:
|
|
"""Recursively unwrap declarator to find the innermost identifier (C)."""
|
|
if node.type == "identifier":
|
|
return _read_text(node, source)
|
|
decl = node.child_by_field_name("declarator")
|
|
if decl:
|
|
return _get_c_func_name(decl, source)
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
return _read_text(child, source)
|
|
return None
|
|
|
|
|
|
def _get_cpp_func_name(node, source: bytes) -> str | None:
|
|
"""Recursively unwrap declarator to find the innermost identifier (C++)."""
|
|
if node.type == "identifier":
|
|
return _read_text(node, source)
|
|
if node.type in ("field_identifier", "destructor_name", "operator_name"):
|
|
return _read_text(node, source)
|
|
if node.type == "qualified_identifier":
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node:
|
|
return _read_text(name_node, source)
|
|
decl = node.child_by_field_name("declarator")
|
|
if decl:
|
|
return _get_cpp_func_name(decl, source)
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
return _read_text(child, source)
|
|
return None
|
|
|
|
|
|
# ── JS/TS extra walk for arrow functions ──────────────────────────────────────
|
|
|
|
def _find_require_call(value_node):
|
|
"""Return the call_expression node if `value_node` is a `require(...)` call
|
|
or `require(...).x` member access. Otherwise None."""
|
|
if value_node is None:
|
|
return None
|
|
if value_node.type == "call_expression":
|
|
fn = value_node.child_by_field_name("function")
|
|
if fn is not None and fn.type == "identifier":
|
|
return value_node
|
|
if value_node.type == "member_expression":
|
|
obj = value_node.child_by_field_name("object")
|
|
return _find_require_call(obj)
|
|
return None
|
|
|
|
|
|
def _require_imports_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> bool:
|
|
"""Detect CommonJS require imports inside lexical_declaration / variable_declaration.
|
|
|
|
Handles three patterns:
|
|
const { foo, bar } = require('./mod') → file → mod (imports_from), file → foo, file → bar
|
|
const mod = require('./mod') → file → mod (imports_from)
|
|
const x = require('./mod').y → file → mod (imports_from), file → y
|
|
|
|
Returns True if any require import was found.
|
|
"""
|
|
if node.type not in ("lexical_declaration", "variable_declaration"):
|
|
return False
|
|
found = False
|
|
for child in node.children:
|
|
if child.type != "variable_declarator":
|
|
continue
|
|
value = child.child_by_field_name("value")
|
|
call = _find_require_call(value)
|
|
if call is None:
|
|
continue
|
|
fn = call.child_by_field_name("function")
|
|
if fn is None or _read_text(fn, source) != "require":
|
|
continue
|
|
args = call.child_by_field_name("arguments")
|
|
if args is None:
|
|
continue
|
|
raw = None
|
|
for arg in args.children:
|
|
if arg.type == "string":
|
|
raw = _read_text(arg, source).strip("'\"` ")
|
|
break
|
|
if not raw:
|
|
continue
|
|
resolved = _resolve_js_import_target(raw, str_path)
|
|
if resolved is None:
|
|
continue
|
|
tgt_nid, resolved_path = resolved
|
|
line = node.start_point[0] + 1
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports_from",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
found = True
|
|
|
|
# Symbol-level edges for destructured / accessor binders.
|
|
target_stem = _file_stem(resolved_path) if resolved_path is not None else None
|
|
name_node = child.child_by_field_name("name")
|
|
sym_names: list[str] = []
|
|
if name_node is not None and name_node.type == "object_pattern":
|
|
# `const { a, b: alias } = require('./m')` — emit edges for each property key
|
|
for prop in name_node.children:
|
|
if prop.type == "shorthand_property_identifier_pattern":
|
|
sym_names.append(_read_text(prop, source))
|
|
elif prop.type == "pair_pattern":
|
|
key = prop.child_by_field_name("key")
|
|
if key is not None:
|
|
sym_names.append(_read_text(key, source))
|
|
elif value is not None and value.type == "member_expression":
|
|
# `const x = require('./m').y` — symbol is the property accessed
|
|
prop = value.child_by_field_name("property")
|
|
if prop is not None:
|
|
sym_names.append(_read_text(prop, source))
|
|
if target_stem is not None:
|
|
for sym in sym_names:
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": _make_id(target_stem, sym),
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
return found
|
|
|
|
|
|
def _js_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: str,
|
|
nodes: list, edges: list, seen_ids: set, function_bodies: list,
|
|
parent_class_nid: str | None, add_node_fn, add_edge_fn) -> bool:
|
|
"""Handle lexical_declaration (arrow functions, CJS requires, module-level const literals) for JS/TS. Returns True if handled."""
|
|
if node.type in ("lexical_declaration", "variable_declaration"):
|
|
# CJS require imports — emit edges, do not block other lexical_declaration handling
|
|
require_found = _require_imports_js(node, source, file_nid, stem, edges, str_path)
|
|
|
|
# Scope guard (#1077): only emit nodes for module-level declarations.
|
|
# Without this, `const x = ...` inside an arrow callback (e.g. inside
|
|
# `describe(() => { const set = new Set(...) })`) emits a bare-named
|
|
# node, and the same name collides across unrelated files producing
|
|
# phantom god-nodes. Bodies of arrow functions are walked separately
|
|
# via function_bodies, so we never need to emit nodes for locals here.
|
|
parent = node.parent
|
|
is_module_level = parent is not None and (
|
|
parent.type == "program"
|
|
or (parent.type == "export_statement"
|
|
and parent.parent is not None
|
|
and parent.parent.type == "program")
|
|
)
|
|
|
|
# Arrow function declarations and module-level const literals (lexical_declaration only)
|
|
arrow_found = False
|
|
const_found = False
|
|
if node.type == "lexical_declaration" and is_module_level:
|
|
for child in node.children:
|
|
if child.type == "variable_declarator":
|
|
value = child.child_by_field_name("value")
|
|
if value and value.type == "arrow_function":
|
|
name_node = child.child_by_field_name("name")
|
|
if name_node:
|
|
func_name = _read_text(name_node, source)
|
|
line = child.start_point[0] + 1
|
|
func_nid = _make_id(stem, func_name)
|
|
add_node_fn(func_nid, f"{func_name}()", line)
|
|
add_edge_fn(file_nid, func_nid, "contains", line)
|
|
body = value.child_by_field_name("body")
|
|
if body:
|
|
function_bodies.append((func_nid, body))
|
|
arrow_found = True
|
|
elif value and value.type in (
|
|
"object", "array", "as_expression", "call_expression", "new_expression",
|
|
):
|
|
# Module-level const with literal/object/array/factory value
|
|
name_node = child.child_by_field_name("name")
|
|
if name_node:
|
|
const_name = _read_text(name_node, source)
|
|
line = child.start_point[0] + 1
|
|
const_nid = _make_id(stem, const_name)
|
|
add_node_fn(const_nid, const_name, line)
|
|
add_edge_fn(file_nid, const_nid, "contains", line)
|
|
const_found = True
|
|
if arrow_found:
|
|
return True
|
|
if const_found:
|
|
return True
|
|
if require_found:
|
|
return True
|
|
return False
|
|
|
|
|
|
# ── C# extra walk for namespace declarations ──────────────────────────────────
|
|
|
|
def _csharp_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: str,
|
|
nodes: list, edges: list, seen_ids: set, function_bodies: list,
|
|
parent_class_nid: str | None, add_node_fn, add_edge_fn,
|
|
walk_fn) -> bool:
|
|
"""Handle namespace_declaration for C#. Returns True if handled."""
|
|
if node.type == "namespace_declaration":
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node:
|
|
ns_name = _read_text(name_node, source)
|
|
ns_nid = _make_id(stem, ns_name)
|
|
line = node.start_point[0] + 1
|
|
add_node_fn(ns_nid, ns_name, line)
|
|
add_edge_fn(file_nid, ns_nid, "contains", line)
|
|
body = node.child_by_field_name("body")
|
|
if body:
|
|
for child in body.children:
|
|
walk_fn(child, parent_class_nid)
|
|
return True
|
|
return False
|
|
|
|
|
|
# ── Swift extra walk for enum cases ──────────────────────────────────────────
|
|
|
|
def _swift_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: str,
|
|
nodes: list, edges: list, seen_ids: set, function_bodies: list,
|
|
parent_class_nid: str | None, add_node_fn, add_edge_fn) -> bool:
|
|
"""Handle enum_entry for Swift. Returns True if handled."""
|
|
if node.type == "enum_entry" and parent_class_nid:
|
|
for child in node.children:
|
|
if child.type == "simple_identifier":
|
|
case_name = _read_text(child, source)
|
|
case_nid = _make_id(parent_class_nid, case_name)
|
|
line = node.start_point[0] + 1
|
|
add_node_fn(case_nid, case_name, line)
|
|
add_edge_fn(parent_class_nid, case_nid, "case_of", line)
|
|
return True
|
|
return False
|
|
|
|
|
|
# ── Language configs ──────────────────────────────────────────────────────────
|
|
|
|
_PYTHON_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_python",
|
|
class_types=frozenset({"class_definition"}),
|
|
function_types=frozenset({"function_definition"}),
|
|
import_types=frozenset({"import_statement", "import_from_statement"}),
|
|
call_types=frozenset({"call"}),
|
|
call_function_field="function",
|
|
call_accessor_node_types=frozenset({"attribute"}),
|
|
call_accessor_field="attribute",
|
|
function_boundary_types=frozenset({"function_definition"}),
|
|
import_handler=_import_python,
|
|
)
|
|
|
|
_JS_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_javascript",
|
|
class_types=frozenset({"class_declaration"}),
|
|
function_types=frozenset({"function_declaration", "method_definition"}),
|
|
import_types=frozenset({"import_statement", "export_statement"}),
|
|
call_types=frozenset({"call_expression", "new_expression"}),
|
|
call_function_field="function",
|
|
call_accessor_node_types=frozenset({"member_expression"}),
|
|
call_accessor_field="property",
|
|
function_boundary_types=frozenset({"function_declaration", "arrow_function", "method_definition"}),
|
|
import_handler=_import_js,
|
|
)
|
|
|
|
_TS_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_typescript",
|
|
ts_language_fn="language_typescript",
|
|
class_types=frozenset({
|
|
"class_declaration",
|
|
"abstract_class_declaration", # TS abstract class
|
|
"interface_declaration", # parity with Java/C#
|
|
"enum_declaration", # named enums
|
|
"type_alias_declaration", # named type aliases
|
|
}),
|
|
function_types=frozenset({"function_declaration", "method_definition"}),
|
|
import_types=frozenset({"import_statement", "export_statement"}),
|
|
call_types=frozenset({"call_expression", "new_expression"}),
|
|
call_function_field="function",
|
|
call_accessor_node_types=frozenset({"member_expression"}),
|
|
call_accessor_field="property",
|
|
function_boundary_types=frozenset({"function_declaration", "arrow_function", "method_definition"}),
|
|
import_handler=_import_js,
|
|
)
|
|
|
|
# .tsx files must use the TSX grammar (JSX-aware), not the plain TypeScript grammar.
|
|
# tree-sitter-typescript ships two languages: language_typescript (for .ts) and
|
|
# language_tsx (for .tsx). Parsing .tsx with language_typescript silently fails on
|
|
# JSX expressions, dropping any call_expression nested inside JSX (e.g. {fmtDate(x)}).
|
|
_TSX_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_typescript",
|
|
ts_language_fn="language_tsx",
|
|
class_types=_TS_CONFIG.class_types,
|
|
function_types=_TS_CONFIG.function_types,
|
|
import_types=_TS_CONFIG.import_types,
|
|
call_types=_TS_CONFIG.call_types,
|
|
call_function_field=_TS_CONFIG.call_function_field,
|
|
call_accessor_node_types=_TS_CONFIG.call_accessor_node_types,
|
|
call_accessor_field=_TS_CONFIG.call_accessor_field,
|
|
function_boundary_types=_TS_CONFIG.function_boundary_types,
|
|
import_handler=_TS_CONFIG.import_handler,
|
|
)
|
|
|
|
_JAVA_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_java",
|
|
class_types=frozenset({"class_declaration", "interface_declaration"}),
|
|
function_types=frozenset({"method_declaration", "constructor_declaration"}),
|
|
import_types=frozenset({"import_declaration"}),
|
|
call_types=frozenset({"method_invocation"}),
|
|
call_function_field="name",
|
|
call_accessor_node_types=frozenset(),
|
|
function_boundary_types=frozenset({"method_declaration", "constructor_declaration"}),
|
|
import_handler=_import_java,
|
|
)
|
|
|
|
_GROOVY_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_groovy",
|
|
class_types=frozenset({"class_declaration", "interface_declaration"}),
|
|
function_types=frozenset({"method_declaration", "constructor_declaration"}),
|
|
import_types=frozenset({"import_declaration"}),
|
|
call_types=frozenset({"method_invocation"}),
|
|
call_function_field="name",
|
|
call_accessor_node_types=frozenset(),
|
|
function_boundary_types=frozenset({"method_declaration", "constructor_declaration"}),
|
|
import_handler=_import_java,
|
|
)
|
|
|
|
_C_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_c",
|
|
class_types=frozenset(),
|
|
function_types=frozenset({"function_definition"}),
|
|
import_types=frozenset({"preproc_include"}),
|
|
call_types=frozenset({"call_expression"}),
|
|
call_function_field="function",
|
|
call_accessor_node_types=frozenset({"field_expression"}),
|
|
call_accessor_field="field",
|
|
function_boundary_types=frozenset({"function_definition"}),
|
|
import_handler=_import_c,
|
|
resolve_function_name_fn=_get_c_func_name,
|
|
)
|
|
|
|
_CPP_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_cpp",
|
|
class_types=frozenset({"class_specifier", "struct_specifier"}),
|
|
function_types=frozenset({"function_definition"}),
|
|
import_types=frozenset({"preproc_include"}),
|
|
call_types=frozenset({"call_expression"}),
|
|
call_function_field="function",
|
|
call_accessor_node_types=frozenset({"field_expression", "qualified_identifier"}),
|
|
call_accessor_field="field",
|
|
function_boundary_types=frozenset({"function_definition"}),
|
|
import_handler=_import_c,
|
|
resolve_function_name_fn=_get_cpp_func_name,
|
|
)
|
|
|
|
_RUBY_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_ruby",
|
|
class_types=frozenset({"class"}),
|
|
function_types=frozenset({"method", "singleton_method"}),
|
|
import_types=frozenset(),
|
|
call_types=frozenset({"call"}),
|
|
call_function_field="method",
|
|
call_accessor_node_types=frozenset(),
|
|
name_fallback_child_types=("constant", "scope_resolution", "identifier"),
|
|
body_fallback_child_types=("body_statement",),
|
|
function_boundary_types=frozenset({"method", "singleton_method"}),
|
|
)
|
|
|
|
_CSHARP_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_c_sharp",
|
|
class_types=frozenset({"class_declaration", "interface_declaration"}),
|
|
function_types=frozenset({"method_declaration"}),
|
|
import_types=frozenset({"using_directive"}),
|
|
call_types=frozenset({"invocation_expression"}),
|
|
call_function_field="function",
|
|
call_accessor_node_types=frozenset({"member_access_expression"}),
|
|
call_accessor_field="name",
|
|
body_fallback_child_types=("declaration_list",),
|
|
function_boundary_types=frozenset({"method_declaration"}),
|
|
import_handler=_import_csharp,
|
|
)
|
|
|
|
_KOTLIN_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_kotlin",
|
|
class_types=frozenset({"class_declaration", "object_declaration"}),
|
|
function_types=frozenset({"function_declaration"}),
|
|
import_types=frozenset({"import_header"}),
|
|
call_types=frozenset({"call_expression"}),
|
|
call_function_field="",
|
|
call_accessor_node_types=frozenset({"navigation_expression"}),
|
|
call_accessor_field="",
|
|
# Different tree-sitter-kotlin grammar versions name plain identifier
|
|
# nodes differently: PyPI's `tree_sitter_kotlin` uses `identifier`,
|
|
# older forks use `simple_identifier`. Accept both so the extractor
|
|
# works across grammar generations.
|
|
name_fallback_child_types=("simple_identifier", "identifier"),
|
|
body_fallback_child_types=("function_body", "class_body"),
|
|
function_boundary_types=frozenset({"function_declaration"}),
|
|
import_handler=_import_kotlin,
|
|
)
|
|
|
|
_SCALA_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_scala",
|
|
class_types=frozenset({"class_definition", "object_definition"}),
|
|
function_types=frozenset({"function_definition"}),
|
|
import_types=frozenset({"import_declaration"}),
|
|
call_types=frozenset({"call_expression"}),
|
|
call_function_field="",
|
|
call_accessor_node_types=frozenset({"field_expression"}),
|
|
call_accessor_field="field",
|
|
name_fallback_child_types=("identifier",),
|
|
body_fallback_child_types=("template_body",),
|
|
function_boundary_types=frozenset({"function_definition"}),
|
|
import_handler=_import_scala,
|
|
)
|
|
|
|
_PHP_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_php",
|
|
ts_language_fn="language_php",
|
|
class_types=frozenset({"class_declaration"}),
|
|
function_types=frozenset({"function_definition", "method_declaration"}),
|
|
import_types=frozenset({"namespace_use_clause"}),
|
|
call_types=frozenset({"function_call_expression", "member_call_expression", "scoped_call_expression", "class_constant_access_expression"}),
|
|
static_prop_types=frozenset({"scoped_property_access_expression"}),
|
|
helper_fn_names=frozenset({"config"}),
|
|
container_bind_methods=frozenset({"bind", "singleton", "scoped", "instance"}),
|
|
event_listener_properties=frozenset({"listen", "subscribe"}),
|
|
call_function_field="function",
|
|
call_accessor_node_types=frozenset({"member_call_expression"}),
|
|
call_accessor_field="name",
|
|
name_fallback_child_types=("name",),
|
|
body_fallback_child_types=("declaration_list", "compound_statement"),
|
|
function_boundary_types=frozenset({"function_definition", "method_declaration"}),
|
|
import_handler=_import_php,
|
|
)
|
|
|
|
|
|
def _resolve_lua_import_target(raw_module: str, str_path: str) -> str:
|
|
"""Resolve a Lua require() module name to a node id.
|
|
|
|
Lua module names use dots as path separators: `require("pkg.b")` looks for
|
|
`pkg/b.lua` (or `pkg/b/init.lua`) relative to a package root. We probe the
|
|
importing file's directory and walk upward looking for a matching file on
|
|
disk; if found, the returned id matches the file node id `_extract_generic`
|
|
assigns to that file (`_make_id(str(path))`), so the edge lands on a real
|
|
node. When nothing matches, fall back to `_make_id` of the full dotted
|
|
module name so cross-file resolution can still complete via the symbol
|
|
resolution pass instead of dropping the edge entirely (#1075).
|
|
"""
|
|
if not raw_module:
|
|
return ""
|
|
rel = raw_module.replace(".", "/")
|
|
try:
|
|
start_dir = Path(str_path).parent
|
|
except Exception:
|
|
start_dir = None
|
|
if start_dir is not None:
|
|
probe = start_dir
|
|
# Walk up a few levels so requires from nested files still resolve when
|
|
# the package root is above the importing file.
|
|
for _ in range(6):
|
|
for suffix in (".lua", ".luau"):
|
|
cand = probe / f"{rel}{suffix}"
|
|
if cand.is_file():
|
|
return _make_id(str(cand))
|
|
for suffix in (".lua", ".luau"):
|
|
cand = probe / rel / f"init{suffix}"
|
|
if cand.is_file():
|
|
return _make_id(str(cand))
|
|
if probe.parent == probe:
|
|
break
|
|
probe = probe.parent
|
|
return _make_id(raw_module)
|
|
|
|
|
|
def _import_lua(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> None:
|
|
"""Extract require('module') from Lua variable_declaration nodes."""
|
|
text = _read_text(node, source)
|
|
import re
|
|
m = re.search(r"""require\s*[\('"]\s*['"]?([^'")\s]+)""", text)
|
|
if m:
|
|
raw_module = m.group(1)
|
|
if raw_module:
|
|
tgt_nid = _resolve_lua_import_target(raw_module, str_path)
|
|
if tgt_nid:
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"confidence_score": 1.0,
|
|
"source_file": str_path,
|
|
"source_location": str(node.start_point[0] + 1),
|
|
"weight": 1.0,
|
|
})
|
|
|
|
|
|
_LUA_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_lua",
|
|
ts_language_fn="language",
|
|
class_types=frozenset(),
|
|
function_types=frozenset({"function_declaration"}),
|
|
import_types=frozenset({"variable_declaration"}),
|
|
call_types=frozenset({"function_call"}),
|
|
call_function_field="name",
|
|
call_accessor_node_types=frozenset({"method_index_expression"}),
|
|
call_accessor_field="name",
|
|
name_fallback_child_types=("identifier", "method_index_expression"),
|
|
body_fallback_child_types=("block",),
|
|
function_boundary_types=frozenset({"function_declaration"}),
|
|
import_handler=_import_lua,
|
|
)
|
|
|
|
|
|
def _import_swift(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> None:
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
raw = _read_text(child, source)
|
|
tgt_nid = _make_id(raw)
|
|
edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
break
|
|
|
|
|
|
def _read_csharp_type_name(node, source: bytes) -> str | None:
|
|
"""Resolve a readable C# type name from a field/type node."""
|
|
if node is None:
|
|
return None
|
|
if node.type in ("identifier", "predefined_type"):
|
|
return _read_text(node, source)
|
|
if node.type == "qualified_name":
|
|
return _read_text(node, source).split(".")[-1]
|
|
if node.type == "generic_name":
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node is not None:
|
|
return _read_text(name_node, source)
|
|
for child in node.children:
|
|
if not child.is_named:
|
|
continue
|
|
name = _read_csharp_type_name(child, source)
|
|
if name:
|
|
return name
|
|
return None
|
|
|
|
|
|
_SWIFT_CONFIG = LanguageConfig(
|
|
ts_module="tree_sitter_swift",
|
|
class_types=frozenset({"class_declaration", "protocol_declaration"}),
|
|
function_types=frozenset({"function_declaration", "init_declaration", "deinit_declaration", "subscript_declaration"}),
|
|
import_types=frozenset({"import_declaration"}),
|
|
call_types=frozenset({"call_expression"}),
|
|
call_function_field="",
|
|
call_accessor_node_types=frozenset({"navigation_expression"}),
|
|
call_accessor_field="",
|
|
name_fallback_child_types=("simple_identifier", "type_identifier", "user_type"),
|
|
body_fallback_child_types=("class_body", "protocol_body", "function_body", "enum_class_body"),
|
|
function_boundary_types=frozenset({"function_declaration", "init_declaration", "deinit_declaration", "subscript_declaration"}),
|
|
import_handler=_import_swift,
|
|
)
|
|
|
|
# ── Generic extractor ─────────────────────────────────────────────────────────
|
|
|
|
def _extract_generic(path: Path, config: LanguageConfig) -> dict:
|
|
"""Generic AST extractor driven by LanguageConfig."""
|
|
try:
|
|
mod = importlib.import_module(config.ts_module)
|
|
from tree_sitter import Language, Parser
|
|
lang_fn = getattr(mod, config.ts_language_fn, None)
|
|
if lang_fn is None:
|
|
# Fallback for PHP: try "language_php" then "language"
|
|
lang_fn = getattr(mod, "language", None)
|
|
if lang_fn is None:
|
|
return {"nodes": [], "edges": [], "error": f"No language function in {config.ts_module}"}
|
|
language = Language(lang_fn())
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": f"{config.ts_module} not installed"}
|
|
except TypeError as e:
|
|
# tree-sitter version mismatch: old Language() expects (lib_path),
|
|
# new Language() expects (language_capsule, name). Surface a hint
|
|
# so users see the upgrade path instead of a bare TypeError.
|
|
hint = (
|
|
f"tree-sitter version mismatch for {config.ts_module}: {e}. "
|
|
"Try: pip install --upgrade tree-sitter tree-sitter-languages"
|
|
)
|
|
return {"nodes": [], "edges": [], "error": hint}
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
try:
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
function_bodies: list[tuple[str, object]] = []
|
|
pending_listen_edges: list[tuple[str, str, int]] = []
|
|
# tree-sitter-swift parses both `class Foo` and `extension Foo` as
|
|
# `class_declaration`. Same-file pairs collapse via seen_ids, but cross-file
|
|
# extensions don't (file stem is part of the id), so they're collected here
|
|
# for a corpus-level merge after every file has been parsed.
|
|
swift_extensions: list[dict] = []
|
|
|
|
csharp_interface_names: set[str] = set()
|
|
if config.ts_module == "tree_sitter_c_sharp":
|
|
csharp_interface_names = _csharp_pre_scan_interfaces(root, source)
|
|
|
|
swift_protocol_names: set[str] = set()
|
|
swift_class_names: set[str] = set()
|
|
if config.ts_module == "tree_sitter_swift":
|
|
swift_protocol_names, swift_class_names = _swift_pre_scan(root, source)
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({
|
|
"id": nid,
|
|
"label": label,
|
|
"file_type": "code",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", weight: float = 1.0,
|
|
context: str | None = None) -> None:
|
|
edge = {
|
|
"source": src,
|
|
"target": tgt,
|
|
"relation": relation,
|
|
"confidence": confidence,
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": weight,
|
|
}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
def ensure_named_node(name: str, line: int) -> str:
|
|
nid = _make_id(stem, name)
|
|
if nid in seen_ids:
|
|
return nid
|
|
nid = _make_id(name)
|
|
if nid not in seen_ids:
|
|
add_node(nid, name, line)
|
|
return nid
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
def walk(node, parent_class_nid: str | None = None) -> None:
|
|
t = node.type
|
|
|
|
# Import types
|
|
if t in config.import_types:
|
|
if config.import_handler:
|
|
config.import_handler(node, source, file_nid, stem, edges, str_path)
|
|
# For export_statement: only return (skip children) if it's a re-export
|
|
# (has a `from` source). Otherwise fall through to walk children which may
|
|
# contain function_declaration, class_declaration, etc.
|
|
if t == "export_statement":
|
|
has_source = any(c.type == "string" for c in node.children)
|
|
if not has_source:
|
|
for child in node.children:
|
|
walk(child, parent_class_nid)
|
|
return
|
|
|
|
# Class types
|
|
if t in config.class_types:
|
|
# Resolve class name
|
|
name_node = node.child_by_field_name(config.name_field)
|
|
if name_node is None:
|
|
for child in node.children:
|
|
if child.type in config.name_fallback_child_types:
|
|
name_node = child
|
|
break
|
|
if not name_node:
|
|
return
|
|
class_name = _read_text(name_node, source)
|
|
class_nid = _make_id(stem, class_name)
|
|
line = node.start_point[0] + 1
|
|
add_node(class_nid, class_name, line)
|
|
add_edge(file_nid, class_nid, "contains", line)
|
|
|
|
if config.ts_module == "tree_sitter_swift" and any(
|
|
c.type == "extension" for c in node.children
|
|
):
|
|
swift_extensions.append({"nid": class_nid, "label": class_name})
|
|
|
|
# Python-specific: inheritance
|
|
if config.ts_module == "tree_sitter_python":
|
|
args = node.child_by_field_name("superclasses")
|
|
if args:
|
|
for arg in args.children:
|
|
if arg.type == "identifier":
|
|
base = _read_text(arg, source)
|
|
base_nid = _make_id(stem, base)
|
|
if base_nid not in seen_ids:
|
|
base_nid = _make_id(base)
|
|
if base_nid not in seen_ids:
|
|
nodes.append({
|
|
"id": base_nid,
|
|
"label": base,
|
|
"file_type": "code",
|
|
"source_file": "",
|
|
"source_location": "",
|
|
})
|
|
seen_ids.add(base_nid)
|
|
add_edge(class_nid, base_nid, "inherits", line)
|
|
|
|
# Swift-specific: conformance / inheritance
|
|
if config.ts_module == "tree_sitter_swift":
|
|
swift_kind = _swift_declaration_keyword(node) if t == "class_declaration" else "protocol"
|
|
seen_swift_base = False
|
|
for child in node.children:
|
|
if child.type != "inheritance_specifier":
|
|
continue
|
|
base_name: str | None = None
|
|
user_type_node = None
|
|
for sub in child.children:
|
|
if sub.type == "user_type":
|
|
user_type_node = sub
|
|
base_name = _swift_user_type_name(sub, source)
|
|
break
|
|
if sub.type == "type_identifier":
|
|
base_name = _read_text(sub, source) or None
|
|
break
|
|
if not base_name:
|
|
continue
|
|
base_nid = _make_id(stem, base_name)
|
|
if base_nid not in seen_ids:
|
|
base_nid = _make_id(base_name)
|
|
if base_nid not in seen_ids:
|
|
nodes.append({
|
|
"id": base_nid,
|
|
"label": base_name,
|
|
"file_type": "code",
|
|
"source_file": "",
|
|
"source_location": "",
|
|
})
|
|
seen_ids.add(base_nid)
|
|
if t == "protocol_declaration":
|
|
relation = "inherits"
|
|
else:
|
|
relation = _swift_classify_base(
|
|
base_name, swift_kind, not seen_swift_base,
|
|
swift_protocol_names, swift_class_names,
|
|
)
|
|
seen_swift_base = True
|
|
add_edge(class_nid, base_nid, relation, line)
|
|
if user_type_node is not None:
|
|
for arg_child in user_type_node.children:
|
|
if arg_child.type != "type_arguments":
|
|
continue
|
|
for arg in arg_child.children:
|
|
if not arg.is_named:
|
|
continue
|
|
refs: list[tuple[str, str]] = []
|
|
_swift_collect_type_refs(arg, source, True, refs)
|
|
for ref_name, _role in refs:
|
|
target = ensure_named_node(ref_name, line)
|
|
add_edge(class_nid, target, "references", line,
|
|
context="generic_arg")
|
|
|
|
# PHP-specific: extends → inherits, implements → implements, use → mixes_in
|
|
if config.ts_module == "tree_sitter_php":
|
|
def _php_emit_base(base_name: str, rel: str, at_line: int) -> None:
|
|
if not base_name:
|
|
return
|
|
base_nid = _make_id(stem, base_name)
|
|
if base_nid not in seen_ids:
|
|
base_nid = _make_id(base_name)
|
|
if base_nid not in seen_ids:
|
|
nodes.append({
|
|
"id": base_nid,
|
|
"label": base_name,
|
|
"file_type": "code",
|
|
"source_file": "",
|
|
"source_location": "",
|
|
})
|
|
seen_ids.add(base_nid)
|
|
add_edge(class_nid, base_nid, rel, at_line)
|
|
|
|
for child in node.children:
|
|
if child.type == "base_clause":
|
|
for sub in child.children:
|
|
if sub.type in ("name", "qualified_name"):
|
|
_php_emit_base(_php_name_text(sub, source) or "",
|
|
"inherits", child.start_point[0] + 1)
|
|
elif child.type == "class_interface_clause":
|
|
for sub in child.children:
|
|
if sub.type in ("name", "qualified_name"):
|
|
_php_emit_base(_php_name_text(sub, source) or "",
|
|
"implements", child.start_point[0] + 1)
|
|
body = node.child_by_field_name("body")
|
|
if body is None:
|
|
for c in node.children:
|
|
if c.type == "declaration_list":
|
|
body = c
|
|
break
|
|
if body is not None:
|
|
for member in body.children:
|
|
if member.type != "use_declaration":
|
|
continue
|
|
for sub in member.children:
|
|
if sub.type in ("name", "qualified_name"):
|
|
_php_emit_base(_php_name_text(sub, source) or "",
|
|
"mixes_in", member.start_point[0] + 1)
|
|
|
|
# Kotlin-specific: delegation_specifiers → inherits (constructor_invocation) / implements (user_type)
|
|
if config.ts_module == "tree_sitter_kotlin":
|
|
for child in node.children:
|
|
if child.type != "delegation_specifiers":
|
|
continue
|
|
for spec in child.children:
|
|
if spec.type != "delegation_specifier":
|
|
continue
|
|
relation = "implements"
|
|
user_type_node = None
|
|
for sub in spec.children:
|
|
if sub.type == "constructor_invocation":
|
|
relation = "inherits"
|
|
for inner in sub.children:
|
|
if inner.type == "user_type":
|
|
user_type_node = inner
|
|
break
|
|
break
|
|
if sub.type == "user_type":
|
|
user_type_node = sub
|
|
break
|
|
if user_type_node is None:
|
|
continue
|
|
base = _kotlin_user_type_name(user_type_node, source)
|
|
if not base:
|
|
continue
|
|
base_nid = _make_id(stem, base)
|
|
if base_nid not in seen_ids:
|
|
base_nid = _make_id(base)
|
|
if base_nid not in seen_ids:
|
|
nodes.append({
|
|
"id": base_nid,
|
|
"label": base,
|
|
"file_type": "code",
|
|
"source_file": "",
|
|
"source_location": "",
|
|
})
|
|
seen_ids.add(base_nid)
|
|
add_edge(class_nid, base_nid, relation, line)
|
|
for arg_child in user_type_node.children:
|
|
if arg_child.type != "type_arguments":
|
|
continue
|
|
for arg in arg_child.children:
|
|
if arg.type == "type_projection":
|
|
for inner in arg.children:
|
|
if not inner.is_named:
|
|
continue
|
|
refs: list[tuple[str, str]] = []
|
|
_kotlin_collect_type_refs(inner, source, True, refs)
|
|
for ref_name, _role in refs:
|
|
target = ensure_named_node(ref_name, line)
|
|
add_edge(class_nid, target, "references", line,
|
|
context="generic_arg")
|
|
|
|
# C#-specific: inheritance / interface implementation via base_list
|
|
if config.ts_module == "tree_sitter_c_sharp":
|
|
for child in node.children:
|
|
if child.type != "base_list":
|
|
continue
|
|
for sub in child.children:
|
|
if sub.type not in ("identifier", "generic_name", "qualified_name"):
|
|
continue
|
|
if sub.type == "generic_name":
|
|
name_child = sub.child_by_field_name("name")
|
|
base = (
|
|
_read_text(name_child, source) if name_child
|
|
else _read_text(sub.children[0], source)
|
|
)
|
|
elif sub.type == "qualified_name":
|
|
base = _read_text(sub, source).rsplit(".", 1)[-1]
|
|
else:
|
|
base = _read_text(sub, source)
|
|
if not base:
|
|
continue
|
|
base_nid = _make_id(stem, base)
|
|
if base_nid not in seen_ids:
|
|
base_nid = _make_id(base)
|
|
if base_nid not in seen_ids:
|
|
nodes.append({
|
|
"id": base_nid,
|
|
"label": base,
|
|
"file_type": "code",
|
|
"source_file": "",
|
|
"source_location": "",
|
|
})
|
|
seen_ids.add(base_nid)
|
|
relation = _csharp_classify_base(base, csharp_interface_names)
|
|
add_edge(class_nid, base_nid, relation, line)
|
|
if sub.type == "generic_name":
|
|
for tal in sub.children:
|
|
if tal.type != "type_argument_list":
|
|
continue
|
|
for arg in tal.children:
|
|
if not arg.is_named:
|
|
continue
|
|
refs: list[tuple[str, str]] = []
|
|
_csharp_collect_type_refs(arg, source, True, refs)
|
|
for ref_name, _role in refs:
|
|
target = ensure_named_node(ref_name, line)
|
|
add_edge(class_nid, target, "references", line,
|
|
context="generic_arg")
|
|
|
|
# Java-specific: extends (superclass) / implements (interfaces) / interface-extends
|
|
if config.ts_module == "tree_sitter_java":
|
|
def _emit_java_parent(base_name: str, rel: str, at_line: int) -> None:
|
|
if not base_name:
|
|
return
|
|
base_nid = _make_id(stem, base_name)
|
|
if base_nid not in seen_ids:
|
|
base_nid = _make_id(base_name)
|
|
if base_nid not in seen_ids:
|
|
nodes.append({
|
|
"id": base_nid,
|
|
"label": base_name,
|
|
"file_type": "code",
|
|
"source_file": "",
|
|
"source_location": "",
|
|
})
|
|
seen_ids.add(base_nid)
|
|
add_edge(class_nid, base_nid, rel, at_line)
|
|
|
|
sup = node.child_by_field_name("superclass")
|
|
if sup is not None:
|
|
for sub in sup.children:
|
|
if sub.type == "type_identifier":
|
|
_emit_java_parent(_read_text(sub, source), "inherits", line)
|
|
break
|
|
|
|
ifs = node.child_by_field_name("interfaces")
|
|
if ifs is not None:
|
|
for sub in ifs.children:
|
|
if sub.type == "type_list":
|
|
for tid in sub.children:
|
|
if tid.type == "type_identifier":
|
|
_emit_java_parent(_read_text(tid, source), "implements", line)
|
|
|
|
if t == "interface_declaration":
|
|
for child in node.children:
|
|
if child.type == "extends_interfaces":
|
|
for sub in child.children:
|
|
if sub.type == "type_list":
|
|
for tid in sub.children:
|
|
if tid.type == "type_identifier":
|
|
_emit_java_parent(_read_text(tid, source), "inherits", line)
|
|
|
|
# Scala: extends_clause carries `extends Base with Trait1 with Trait2`.
|
|
# The first base after `extends` is `inherits`; each subsequent
|
|
# type after `with` is `mixes_in`. Also walk class_parameters for
|
|
# constructor-as-field type references.
|
|
if config.ts_module == "tree_sitter_scala":
|
|
extend = node.child_by_field_name("extend")
|
|
if extend is None:
|
|
for c in node.children:
|
|
if c.type == "extends_clause":
|
|
extend = c
|
|
break
|
|
if extend is not None:
|
|
bases: list[tuple[str, int]] = []
|
|
for c in extend.children:
|
|
if c.type == "type_identifier":
|
|
bases.append((_read_text(c, source), c.start_point[0] + 1))
|
|
elif c.type == "generic_type":
|
|
base = c.child_by_field_name("type")
|
|
if base is None:
|
|
for sc in c.children:
|
|
if sc.type == "type_identifier":
|
|
base = sc
|
|
break
|
|
if base is not None:
|
|
bases.append((_read_text(base, source), c.start_point[0] + 1))
|
|
for idx, (base_name, base_line) in enumerate(bases):
|
|
rel = "inherits" if idx == 0 else "mixes_in"
|
|
base_nid = ensure_named_node(base_name, base_line)
|
|
if base_nid != class_nid:
|
|
add_edge(class_nid, base_nid, rel, base_line)
|
|
for c in node.children:
|
|
if c.type != "class_parameters":
|
|
continue
|
|
for cp in c.children:
|
|
if cp.type != "class_parameter":
|
|
continue
|
|
ptype = cp.child_by_field_name("type")
|
|
if ptype is None:
|
|
continue
|
|
cp_line = cp.start_point[0] + 1
|
|
refs: list[tuple[str, str]] = []
|
|
_scala_collect_type_refs(ptype, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "field"
|
|
target_nid = ensure_named_node(ref_name, cp_line)
|
|
if target_nid != class_nid:
|
|
add_edge(class_nid, target_nid, "references",
|
|
cp_line, context=ctx)
|
|
|
|
# C++-specific: inheritance via base_class_clause (class and struct).
|
|
# tree-sitter-cpp shape:
|
|
# class_specifier / struct_specifier
|
|
# base_class_clause
|
|
# access_specifier? ("public"/"protected"/"private") -- skip
|
|
# "virtual"? -- skip
|
|
# type_identifier -- "Base"
|
|
# qualified_identifier -- "ns::Base"
|
|
# template_type -- "Vec<int>"
|
|
# Multiple bases are siblings separated by ',' tokens.
|
|
if config.ts_module == "tree_sitter_cpp":
|
|
for child in node.children:
|
|
if child.type != "base_class_clause":
|
|
continue
|
|
for sub in child.children:
|
|
base = ""
|
|
if sub.type == "type_identifier":
|
|
base = _read_text(sub, source)
|
|
elif sub.type == "qualified_identifier":
|
|
# Use the unqualified tail so "std::vector" matches
|
|
# a "vector" node id if one exists in the graph;
|
|
# fall back to the full qualified text otherwise.
|
|
tail = sub.child_by_field_name("name")
|
|
base = _read_text(tail, source) if tail else _read_text(sub, source)
|
|
elif sub.type == "template_type":
|
|
tname = sub.child_by_field_name("name")
|
|
base = _read_text(tname, source) if tname else _read_text(sub, source)
|
|
else:
|
|
continue
|
|
if not base:
|
|
continue
|
|
base_nid = _make_id(stem, base)
|
|
if base_nid not in seen_ids:
|
|
base_nid = _make_id(base)
|
|
if base_nid not in seen_ids:
|
|
nodes.append({
|
|
"id": base_nid,
|
|
"label": base,
|
|
"file_type": "code",
|
|
"source_file": "",
|
|
"source_location": "",
|
|
})
|
|
seen_ids.add(base_nid)
|
|
add_edge(class_nid, base_nid, "inherits", line)
|
|
|
|
# Find body and recurse
|
|
body = _find_body(node, config)
|
|
if body:
|
|
for child in body.children:
|
|
walk(child, parent_class_nid=class_nid)
|
|
return
|
|
|
|
# Event listener property arrays: $listen = [Event::class => [Listener::class]]
|
|
if (t == "property_declaration"
|
|
and parent_class_nid
|
|
and config.event_listener_properties):
|
|
handled_event_listener = False
|
|
for element in node.children:
|
|
if element.type != "property_element":
|
|
continue
|
|
prop_name: str | None = None
|
|
array_node = None
|
|
for c in element.children:
|
|
if c.type == "variable_name":
|
|
for sc in c.children:
|
|
if sc.type == "name":
|
|
prop_name = _read_text(sc, source)
|
|
break
|
|
elif c.type == "array_creation_expression":
|
|
array_node = c
|
|
if (prop_name is None
|
|
or prop_name not in config.event_listener_properties
|
|
or array_node is None):
|
|
continue
|
|
handled_event_listener = True
|
|
for entry in array_node.children:
|
|
if entry.type != "array_element_initializer":
|
|
continue
|
|
event_cls: str | None = None
|
|
listener_arr = None
|
|
for sub in entry.children:
|
|
if sub.type == "class_constant_access_expression" and event_cls is None:
|
|
for sc in sub.children:
|
|
if sc.is_named and sc.type in ("name", "qualified_name"):
|
|
event_cls = _read_text(sc, source)
|
|
break
|
|
elif sub.type == "array_creation_expression":
|
|
listener_arr = sub
|
|
if not event_cls or listener_arr is None:
|
|
continue
|
|
for listener_entry in listener_arr.children:
|
|
if listener_entry.type != "array_element_initializer":
|
|
continue
|
|
for item in listener_entry.children:
|
|
if item.type != "class_constant_access_expression":
|
|
continue
|
|
for sc in item.children:
|
|
if sc.is_named and sc.type in ("name", "qualified_name"):
|
|
listener_cls = _read_text(sc, source)
|
|
line_no = item.start_point[0] + 1
|
|
pending_listen_edges.append((event_cls, listener_cls, line_no))
|
|
break
|
|
break
|
|
if handled_event_listener:
|
|
return
|
|
|
|
if (config.ts_module == "tree_sitter_c_sharp"
|
|
and t == "field_declaration"
|
|
and parent_class_nid):
|
|
type_node = node.child_by_field_name("type")
|
|
if type_node is None:
|
|
for child in node.children:
|
|
if child.type == "variable_declaration":
|
|
type_node = child.child_by_field_name("type")
|
|
if type_node is not None:
|
|
break
|
|
type_name = _read_csharp_type_name(type_node, source)
|
|
if type_name:
|
|
line = node.start_point[0] + 1
|
|
add_edge(parent_class_nid, ensure_named_node(type_name, line),
|
|
"references", line, context="field")
|
|
return
|
|
|
|
if (config.ts_module == "tree_sitter_php"
|
|
and t == "property_declaration"
|
|
and parent_class_nid):
|
|
for c in node.children:
|
|
if c.type not in ("named_type", "primitive_type", "nullable_type",
|
|
"union_type", "intersection_type", "optional_type"):
|
|
continue
|
|
line = node.start_point[0] + 1
|
|
refs: list[tuple[str, str]] = []
|
|
_php_collect_type_refs(c, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "field"
|
|
target_nid = ensure_named_node(ref_name, line)
|
|
if target_nid != parent_class_nid:
|
|
add_edge(parent_class_nid, target_nid, "references", line, context=ctx)
|
|
break
|
|
return
|
|
|
|
if (config.ts_module == "tree_sitter_kotlin"
|
|
and t == "property_declaration"
|
|
and parent_class_nid):
|
|
type_node = _kotlin_property_type_node(node)
|
|
if type_node is not None:
|
|
line = node.start_point[0] + 1
|
|
refs: list[tuple[str, str]] = []
|
|
_kotlin_collect_type_refs(type_node, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "field"
|
|
target_nid = ensure_named_node(ref_name, line)
|
|
if target_nid != parent_class_nid:
|
|
add_edge(parent_class_nid, target_nid, "references", line, context=ctx)
|
|
return
|
|
|
|
if (config.ts_module == "tree_sitter_swift"
|
|
and t == "property_declaration"
|
|
and parent_class_nid):
|
|
type_anno = _swift_property_type_node(node)
|
|
if type_anno is not None:
|
|
line = node.start_point[0] + 1
|
|
refs: list[tuple[str, str]] = []
|
|
_swift_collect_type_refs(type_anno, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "field"
|
|
target_nid = ensure_named_node(ref_name, line)
|
|
if target_nid != parent_class_nid:
|
|
add_edge(parent_class_nid, target_nid, "references", line, context=ctx)
|
|
return
|
|
|
|
if (config.ts_module == "tree_sitter_scala"
|
|
and t == "val_definition"
|
|
and parent_class_nid):
|
|
type_node = node.child_by_field_name("type")
|
|
if type_node is not None:
|
|
line = node.start_point[0] + 1
|
|
refs: list[tuple[str, str]] = []
|
|
_scala_collect_type_refs(type_node, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "field"
|
|
target_nid = ensure_named_node(ref_name, line)
|
|
if target_nid != parent_class_nid:
|
|
add_edge(parent_class_nid, target_nid, "references",
|
|
line, context=ctx)
|
|
# fall through so any call expressions in the initializer get walked
|
|
|
|
if (config.ts_module == "tree_sitter_cpp"
|
|
and t == "field_declaration"
|
|
and parent_class_nid):
|
|
# Skip method prototypes (field_declaration with a function_declarator
|
|
# is a member-function declaration, not a data member).
|
|
decls = list(node.children_by_field_name("declarator"))
|
|
is_method = any(
|
|
d.type == "function_declarator"
|
|
or (d.type in ("pointer_declarator", "reference_declarator")
|
|
and any(c.type == "function_declarator" for c in d.children))
|
|
for d in decls
|
|
)
|
|
if not is_method:
|
|
type_node = node.child_by_field_name("type")
|
|
if type_node is not None:
|
|
line = node.start_point[0] + 1
|
|
refs: list[tuple[str, str]] = []
|
|
_cpp_collect_type_refs(type_node, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "field"
|
|
target_nid = ensure_named_node(ref_name, line)
|
|
if target_nid != parent_class_nid:
|
|
add_edge(parent_class_nid, target_nid, "references",
|
|
line, context=ctx)
|
|
# Emit a node for each data member. Use children_by_field_name so we
|
|
# only visit declarator children, not the type node (which would give
|
|
# us the type name, not the field name). Handles int x, y; via
|
|
# multiple declarator fields and static const int MAX = 100; via the
|
|
# init_declarator → field_identifier recursion in _get_cpp_func_name.
|
|
for decl in decls:
|
|
name = _get_cpp_func_name(decl, source)
|
|
if name:
|
|
line = decl.start_point[0] + 1
|
|
field_nid = _make_id(parent_class_nid, name)
|
|
add_node(field_nid, name, line)
|
|
add_edge(parent_class_nid, field_nid, "defines", line, context="field")
|
|
return
|
|
|
|
# Function types
|
|
if t in config.function_types:
|
|
# Swift deinit/subscript have no name field — resolve before generic fallback
|
|
if t == "deinit_declaration":
|
|
func_name: str | None = "deinit"
|
|
elif t == "subscript_declaration":
|
|
func_name = "subscript"
|
|
elif config.resolve_function_name_fn is not None:
|
|
# C/C++ style: use declarator
|
|
declarator = node.child_by_field_name("declarator")
|
|
func_name = None
|
|
if declarator:
|
|
func_name = config.resolve_function_name_fn(declarator, source)
|
|
else:
|
|
name_node = node.child_by_field_name(config.name_field)
|
|
if name_node is None:
|
|
for child in node.children:
|
|
if child.type in config.name_fallback_child_types:
|
|
name_node = child
|
|
break
|
|
func_name = _read_text(name_node, source) if name_node else None
|
|
|
|
if not func_name:
|
|
return
|
|
|
|
line = node.start_point[0] + 1
|
|
if parent_class_nid:
|
|
func_nid = _make_id(parent_class_nid, func_name)
|
|
add_node(func_nid, f".{func_name}()", line)
|
|
add_edge(parent_class_nid, func_nid, "method", line)
|
|
else:
|
|
func_nid = _make_id(stem, func_name)
|
|
add_node(func_nid, f"{func_name}()", line)
|
|
add_edge(file_nid, func_nid, "contains", line)
|
|
|
|
if config.ts_module == "tree_sitter_python":
|
|
params_node = node.child_by_field_name("parameters")
|
|
for ref_name, role in _python_collect_param_refs(params_node, source):
|
|
ctx = "generic_arg" if role == "generic_arg" else "parameter_type"
|
|
target_nid = ensure_named_node(ref_name, line)
|
|
if target_nid != func_nid:
|
|
edges.append(
|
|
_semantic_reference_edge(func_nid, target_nid, ctx, str_path, line)
|
|
)
|
|
return_type_node = node.child_by_field_name("return_type")
|
|
if return_type_node is not None:
|
|
return_refs: list[tuple[str, str]] = []
|
|
_python_collect_type_refs(return_type_node, source, False, return_refs)
|
|
for ref_name, role in return_refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "return_type"
|
|
target_nid = ensure_named_node(ref_name, line)
|
|
if target_nid != func_nid:
|
|
edges.append(
|
|
_semantic_reference_edge(func_nid, target_nid, ctx, str_path, line)
|
|
)
|
|
|
|
if config.ts_module == "tree_sitter_c_sharp":
|
|
params_node = node.child_by_field_name("parameters")
|
|
if params_node is not None:
|
|
for p in params_node.children:
|
|
if p.type != "parameter":
|
|
continue
|
|
type_node = p.child_by_field_name("type")
|
|
refs: list[tuple[str, str]] = []
|
|
_csharp_collect_type_refs(type_node, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "parameter_type"
|
|
target_nid = ensure_named_node(ref_name, line)
|
|
if target_nid != func_nid:
|
|
add_edge(func_nid, target_nid, "references", line, context=ctx)
|
|
return_node = node.child_by_field_name("returns")
|
|
if return_node is not None:
|
|
refs = []
|
|
_csharp_collect_type_refs(return_node, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "return_type"
|
|
target_nid = ensure_named_node(ref_name, line)
|
|
if target_nid != func_nid:
|
|
add_edge(func_nid, target_nid, "references", line, context=ctx)
|
|
for attr_name in _csharp_attribute_names(node, source):
|
|
target_nid = ensure_named_node(attr_name, line)
|
|
if target_nid != func_nid:
|
|
add_edge(func_nid, target_nid, "references", line, context="attribute")
|
|
|
|
if config.ts_module == "tree_sitter_java":
|
|
params_node = node.child_by_field_name("parameters")
|
|
if params_node is not None:
|
|
for p in params_node.children:
|
|
if p.type != "formal_parameter":
|
|
continue
|
|
type_node = p.child_by_field_name("type")
|
|
refs = []
|
|
_java_collect_type_refs(type_node, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "parameter_type"
|
|
target_nid = ensure_named_node(ref_name, line)
|
|
if target_nid != func_nid:
|
|
add_edge(func_nid, target_nid, "references", line, context=ctx)
|
|
return_node = node.child_by_field_name("type")
|
|
if return_node is not None:
|
|
refs = []
|
|
_java_collect_type_refs(return_node, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "return_type"
|
|
target_nid = ensure_named_node(ref_name, line)
|
|
if target_nid != func_nid:
|
|
add_edge(func_nid, target_nid, "references", line, context=ctx)
|
|
for anno_name in _java_method_annotation_names(node, source):
|
|
target_nid = ensure_named_node(anno_name, line)
|
|
if target_nid != func_nid:
|
|
add_edge(func_nid, target_nid, "references", line, context="attribute")
|
|
|
|
if config.ts_module == "tree_sitter_php":
|
|
params_container = None
|
|
for c in node.children:
|
|
if c.type == "formal_parameters":
|
|
params_container = c
|
|
break
|
|
if params_container is not None:
|
|
for p in params_container.children:
|
|
if p.type != "simple_parameter":
|
|
continue
|
|
type_node = None
|
|
for sub in p.children:
|
|
if sub.type in ("named_type", "primitive_type", "nullable_type",
|
|
"union_type", "intersection_type", "optional_type"):
|
|
type_node = sub
|
|
break
|
|
refs: list[tuple[str, str]] = []
|
|
_php_collect_type_refs(type_node, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "parameter_type"
|
|
target_nid = ensure_named_node(ref_name, line)
|
|
if target_nid != func_nid:
|
|
add_edge(func_nid, target_nid, "references", line, context=ctx)
|
|
return_node = _php_method_return_type_node(node)
|
|
if return_node is not None:
|
|
refs = []
|
|
_php_collect_type_refs(return_node, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "return_type"
|
|
target_nid = ensure_named_node(ref_name, line)
|
|
if target_nid != func_nid:
|
|
add_edge(func_nid, target_nid, "references", line, context=ctx)
|
|
|
|
if config.ts_module == "tree_sitter_kotlin":
|
|
params_container = None
|
|
for c in node.children:
|
|
if c.type == "function_value_parameters":
|
|
params_container = c
|
|
break
|
|
if params_container is not None:
|
|
for p in params_container.children:
|
|
if p.type != "parameter":
|
|
continue
|
|
param_type_node = None
|
|
for sub in p.children:
|
|
if sub.type in ("user_type", "nullable_type", "type_reference"):
|
|
param_type_node = sub
|
|
break
|
|
refs: list[tuple[str, str]] = []
|
|
_kotlin_collect_type_refs(param_type_node, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "parameter_type"
|
|
target_nid = ensure_named_node(ref_name, line)
|
|
if target_nid != func_nid:
|
|
add_edge(func_nid, target_nid, "references", line, context=ctx)
|
|
return_type_node = _kotlin_function_return_type_node(node)
|
|
if return_type_node is not None:
|
|
refs = []
|
|
_kotlin_collect_type_refs(return_type_node, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "return_type"
|
|
target_nid = ensure_named_node(ref_name, line)
|
|
if target_nid != func_nid:
|
|
add_edge(func_nid, target_nid, "references", line, context=ctx)
|
|
|
|
if config.ts_module == "tree_sitter_swift":
|
|
for p in node.children:
|
|
if p.type != "parameter":
|
|
continue
|
|
type_node = p.child_by_field_name("type")
|
|
refs: list[tuple[str, str]] = []
|
|
_swift_collect_type_refs(type_node, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "parameter_type"
|
|
target_nid = ensure_named_node(ref_name, line)
|
|
if target_nid != func_nid:
|
|
add_edge(func_nid, target_nid, "references", line, context=ctx)
|
|
return_node = node.child_by_field_name("return_type")
|
|
if return_node is not None:
|
|
refs = []
|
|
_swift_collect_type_refs(return_node, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "return_type"
|
|
target_nid = ensure_named_node(ref_name, line)
|
|
if target_nid != func_nid:
|
|
add_edge(func_nid, target_nid, "references", line, context=ctx)
|
|
|
|
if config.ts_module in ("tree_sitter_c", "tree_sitter_cpp"):
|
|
collect = (_cpp_collect_type_refs if config.ts_module == "tree_sitter_cpp"
|
|
else _c_collect_type_refs)
|
|
return_node = node.child_by_field_name("type")
|
|
if return_node is not None:
|
|
refs: list[tuple[str, str]] = []
|
|
collect(return_node, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "return_type"
|
|
target_nid = ensure_named_node(ref_name, line)
|
|
if target_nid != func_nid:
|
|
add_edge(func_nid, target_nid, "references", line, context=ctx)
|
|
# function_declarator may be wrapped in pointer/reference declarators
|
|
decl = node.child_by_field_name("declarator")
|
|
while decl is not None and decl.type in (
|
|
"pointer_declarator", "reference_declarator"):
|
|
decl = decl.child_by_field_name("declarator")
|
|
if decl is not None and decl.type == "function_declarator":
|
|
params_node = decl.child_by_field_name("parameters")
|
|
if params_node is not None:
|
|
for p in params_node.children:
|
|
if p.type != "parameter_declaration":
|
|
continue
|
|
ptype = p.child_by_field_name("type")
|
|
if ptype is None:
|
|
continue
|
|
refs = []
|
|
collect(ptype, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "parameter_type"
|
|
target_nid = ensure_named_node(ref_name, line)
|
|
if target_nid != func_nid:
|
|
add_edge(func_nid, target_nid, "references",
|
|
line, context=ctx)
|
|
|
|
if config.ts_module == "tree_sitter_scala":
|
|
params_node = None
|
|
for c in node.children:
|
|
if c.type == "parameters":
|
|
params_node = c
|
|
break
|
|
if params_node is not None:
|
|
for p in params_node.children:
|
|
if p.type != "parameter":
|
|
continue
|
|
ptype = p.child_by_field_name("type")
|
|
if ptype is None:
|
|
continue
|
|
refs: list[tuple[str, str]] = []
|
|
_scala_collect_type_refs(ptype, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "parameter_type"
|
|
target_nid = ensure_named_node(ref_name, line)
|
|
if target_nid != func_nid:
|
|
add_edge(func_nid, target_nid, "references",
|
|
line, context=ctx)
|
|
return_node = node.child_by_field_name("return_type")
|
|
if return_node is not None:
|
|
refs = []
|
|
_scala_collect_type_refs(return_node, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "return_type"
|
|
target_nid = ensure_named_node(ref_name, line)
|
|
if target_nid != func_nid:
|
|
add_edge(func_nid, target_nid, "references",
|
|
line, context=ctx)
|
|
|
|
body = _find_body(node, config)
|
|
if body:
|
|
function_bodies.append((func_nid, body))
|
|
return
|
|
|
|
# JS/TS arrow functions and C# namespaces — language-specific extra handling
|
|
if config.ts_module in ("tree_sitter_javascript", "tree_sitter_typescript"):
|
|
if _js_extra_walk(node, source, file_nid, stem, str_path,
|
|
nodes, edges, seen_ids, function_bodies,
|
|
parent_class_nid, add_node, add_edge):
|
|
return
|
|
|
|
if config.ts_module == "tree_sitter_c_sharp":
|
|
if _csharp_extra_walk(node, source, file_nid, stem, str_path,
|
|
nodes, edges, seen_ids, function_bodies,
|
|
parent_class_nid, add_node, add_edge, walk):
|
|
return
|
|
|
|
if config.ts_module == "tree_sitter_swift":
|
|
if _swift_extra_walk(node, source, file_nid, stem, str_path,
|
|
nodes, edges, seen_ids, function_bodies,
|
|
parent_class_nid, add_node, add_edge):
|
|
return
|
|
|
|
# Python's `@property` / `@staticmethod` / `@classmethod` wrap the
|
|
# inner function_definition in a `decorated_definition` node. The
|
|
# default recurse below clears parent_class_nid, which would cause the
|
|
# inner method to be emitted with a class-unqualified node id (e.g.
|
|
# `file_baz` instead of `file_bar_baz`). That diverges from the
|
|
# class-qualified id the rationale walker uses for the same method's
|
|
# docstring, leaving the rationale edge dangling and the docstring
|
|
# node orphaned (#1050). Treat decorated_definition as a transparent
|
|
# wrapper so parent_class_nid propagates to the real function node.
|
|
if t == "decorated_definition":
|
|
for child in node.children:
|
|
walk(child, parent_class_nid=parent_class_nid)
|
|
return
|
|
|
|
# Default: recurse
|
|
for child in node.children:
|
|
walk(child, parent_class_nid=None)
|
|
|
|
walk(root)
|
|
|
|
# ── Call-graph pass ───────────────────────────────────────────────────────
|
|
label_to_nid: dict[str, str] = {} # case-sensitive (Ruby, C#, Java, Kotlin, etc.)
|
|
label_to_nid_ci: dict[str, str] = {} # case-insensitive (PHP functions/classes)
|
|
for n in nodes:
|
|
raw = n["label"]
|
|
normalised = raw.strip("()").lstrip(".")
|
|
label_to_nid[normalised] = n["id"]
|
|
label_to_nid_ci[normalised.lower()] = n["id"]
|
|
|
|
seen_call_pairs: set[tuple[str, str]] = set()
|
|
seen_dyn_import_pairs: set[tuple[str, str]] = set()
|
|
seen_static_ref_pairs: set[tuple[str, str, str]] = set()
|
|
seen_helper_ref_pairs: set[tuple[str, str, str]] = set()
|
|
seen_bind_pairs: set[tuple[str, str, str]] = set()
|
|
raw_calls: list[dict] = [] # unresolved calls for cross-file resolution in extract()
|
|
|
|
def _php_class_const_scope(n) -> str | None:
|
|
scope = n.child_by_field_name("scope")
|
|
if scope is None:
|
|
for c in n.children:
|
|
if c.is_named and c.type in ("name", "qualified_name", "identifier"):
|
|
scope = c
|
|
break
|
|
if scope is None:
|
|
return None
|
|
return _read_text(scope, source)
|
|
|
|
def walk_calls(node, caller_nid: str) -> None:
|
|
if node.type in config.function_boundary_types:
|
|
return
|
|
|
|
if node.type in config.call_types:
|
|
# JS/TS dynamic imports: await import('./foo.js')
|
|
if config.ts_module in ("tree_sitter_javascript", "tree_sitter_typescript"):
|
|
if _dynamic_import_js(node, source, caller_nid, str_path,
|
|
edges, seen_dyn_import_pairs):
|
|
# Still recurse into children (import().then(...) may have calls)
|
|
for child in node.children:
|
|
walk_calls(child, caller_nid)
|
|
return
|
|
|
|
callee_name: str | None = None
|
|
is_member_call: bool = False
|
|
|
|
# Special handling per language
|
|
if config.ts_module == "tree_sitter_swift":
|
|
# Swift: first child may be simple_identifier or navigation_expression
|
|
first = node.children[0] if node.children else None
|
|
if first:
|
|
if first.type == "simple_identifier":
|
|
callee_name = _read_text(first, source)
|
|
elif first.type == "navigation_expression":
|
|
is_member_call = True
|
|
for child in first.children:
|
|
if child.type == "navigation_suffix":
|
|
for sc in child.children:
|
|
if sc.type == "simple_identifier":
|
|
callee_name = _read_text(sc, source)
|
|
elif config.ts_module == "tree_sitter_kotlin":
|
|
# Kotlin: first child may be simple_identifier/identifier or
|
|
# navigation_expression. PyPI's `tree_sitter_kotlin` produces
|
|
# `identifier` for plain identifier nodes; older grammar
|
|
# versions (including the JVM `io.github.bonede:tree-sitter-kotlin`
|
|
# binding) produce `simple_identifier`. Accept both.
|
|
first = node.children[0] if node.children else None
|
|
if first:
|
|
if first.type in ("simple_identifier", "identifier"):
|
|
callee_name = _read_text(first, source)
|
|
elif first.type == "navigation_expression":
|
|
is_member_call = True
|
|
for child in reversed(first.children):
|
|
if child.type in ("simple_identifier", "identifier"):
|
|
callee_name = _read_text(child, source)
|
|
break
|
|
elif config.ts_module == "tree_sitter_scala":
|
|
# Scala: first child
|
|
first = node.children[0] if node.children else None
|
|
if first:
|
|
if first.type == "identifier":
|
|
callee_name = _read_text(first, source)
|
|
elif first.type == "field_expression":
|
|
is_member_call = True
|
|
field = first.child_by_field_name("field")
|
|
if field:
|
|
callee_name = _read_text(field, source)
|
|
else:
|
|
for child in reversed(first.children):
|
|
if child.type == "identifier":
|
|
callee_name = _read_text(child, source)
|
|
break
|
|
elif config.ts_module == "tree_sitter_c_sharp" and node.type == "invocation_expression":
|
|
# C#: try name field, then first named child
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node:
|
|
callee_name = _read_text(name_node, source)
|
|
else:
|
|
for child in node.children:
|
|
if child.is_named:
|
|
raw = _read_text(child, source)
|
|
if "." in raw:
|
|
callee_name = raw.split(".")[-1]
|
|
is_member_call = True
|
|
else:
|
|
callee_name = raw
|
|
break
|
|
elif config.ts_module == "tree_sitter_php":
|
|
# PHP: distinguish call expression subtypes
|
|
if node.type == "function_call_expression":
|
|
func_node = node.child_by_field_name("function")
|
|
if func_node:
|
|
callee_name = _read_text(func_node, source)
|
|
elif node.type == "scoped_call_expression":
|
|
# Static method call: Helper::format() → callee = "Helper"
|
|
scope_node = node.child_by_field_name("scope")
|
|
if scope_node:
|
|
callee_name = _read_text(scope_node, source)
|
|
else:
|
|
# member_call_expression: $obj->method()
|
|
is_member_call = True
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node:
|
|
callee_name = _read_text(name_node, source)
|
|
elif config.ts_module == "tree_sitter_cpp":
|
|
# C++: function field, then field_expression/qualified_identifier
|
|
func_node = node.child_by_field_name(config.call_function_field) if config.call_function_field else None
|
|
if func_node:
|
|
if func_node.type == "identifier":
|
|
callee_name = _read_text(func_node, source)
|
|
elif func_node.type in ("field_expression", "qualified_identifier"):
|
|
is_member_call = True
|
|
name = func_node.child_by_field_name("field") or func_node.child_by_field_name("name")
|
|
if name:
|
|
callee_name = _read_text(name, source)
|
|
else:
|
|
# Generic: get callee from call_function_field
|
|
func_node = node.child_by_field_name(config.call_function_field) if config.call_function_field else None
|
|
if func_node:
|
|
if func_node.type == "identifier":
|
|
callee_name = _read_text(func_node, source)
|
|
elif func_node.type in config.call_accessor_node_types:
|
|
is_member_call = True
|
|
if config.call_accessor_field:
|
|
attr = func_node.child_by_field_name(config.call_accessor_field)
|
|
if attr:
|
|
callee_name = _read_text(attr, source)
|
|
else:
|
|
# Try reading the node directly (e.g. Java name field is the callee)
|
|
callee_name = _read_text(func_node, source)
|
|
|
|
if callee_name and callee_name not in _LANGUAGE_BUILTIN_GLOBALS:
|
|
tgt_nid = label_to_nid.get(callee_name)
|
|
if tgt_nid and tgt_nid != caller_nid:
|
|
pair = (caller_nid, tgt_nid)
|
|
if pair not in seen_call_pairs:
|
|
seen_call_pairs.add(pair)
|
|
line = node.start_point[0] + 1
|
|
edges.append({
|
|
"source": caller_nid,
|
|
"target": tgt_nid,
|
|
"relation": "calls",
|
|
"context": "call",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
elif callee_name and not tgt_nid:
|
|
# Callee not in this file — save for cross-file resolution in extract()
|
|
raw_calls.append({
|
|
"caller_nid": caller_nid,
|
|
"callee": callee_name,
|
|
"is_member_call": is_member_call,
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
})
|
|
|
|
# Helper function calls: config('foo.bar') → uses_config edge to "foo"
|
|
if (callee_name and callee_name in config.helper_fn_names):
|
|
args_node = node.child_by_field_name("arguments")
|
|
first_key: str | None = None
|
|
if args_node:
|
|
for arg in args_node.children:
|
|
if arg.type != "argument":
|
|
continue
|
|
for inner in arg.children:
|
|
if inner.type == "string":
|
|
for sc in inner.children:
|
|
if sc.type == "string_content":
|
|
first_key = _read_text(sc, source)
|
|
break
|
|
break
|
|
if first_key:
|
|
break
|
|
if first_key:
|
|
segment = first_key.split(".")[0]
|
|
tgt_nid = (label_to_nid_ci.get(segment.lower())
|
|
or label_to_nid_ci.get(f"{segment}.php".lower()))
|
|
if tgt_nid and tgt_nid != caller_nid:
|
|
relation = f"uses_{callee_name}"
|
|
pair3 = (caller_nid, tgt_nid, relation)
|
|
if pair3 not in seen_helper_ref_pairs:
|
|
seen_helper_ref_pairs.add(pair3)
|
|
line = node.start_point[0] + 1
|
|
edges.append({
|
|
"source": caller_nid,
|
|
"target": tgt_nid,
|
|
"relation": relation,
|
|
"confidence": "EXTRACTED",
|
|
"confidence_score": 1.0,
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
# Service container bindings: $this->app->bind(Foo::class, Bar::class)
|
|
if (node.type == "member_call_expression"
|
|
and callee_name
|
|
and callee_name in config.container_bind_methods):
|
|
args_node = node.child_by_field_name("arguments")
|
|
class_args: list[str] = []
|
|
if args_node:
|
|
for arg in args_node.children:
|
|
if arg.type != "argument":
|
|
continue
|
|
for inner in arg.children:
|
|
if inner.type == "class_constant_access_expression":
|
|
cls = _php_class_const_scope(inner)
|
|
if cls:
|
|
class_args.append(cls)
|
|
break
|
|
if len(class_args) >= 2:
|
|
break
|
|
if len(class_args) == 2:
|
|
contract_name, impl_name = class_args
|
|
contract_nid = label_to_nid_ci.get(contract_name.lower())
|
|
impl_nid = label_to_nid_ci.get(impl_name.lower())
|
|
if contract_nid and impl_nid and contract_nid != impl_nid:
|
|
pair3 = (contract_nid, impl_nid, "bound_to")
|
|
if pair3 not in seen_bind_pairs:
|
|
seen_bind_pairs.add(pair3)
|
|
line = node.start_point[0] + 1
|
|
edges.append({
|
|
"source": contract_nid,
|
|
"target": impl_nid,
|
|
"relation": "bound_to",
|
|
"confidence": "EXTRACTED",
|
|
"confidence_score": 1.0,
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
# Static property access: Foo::$bar → uses_static_prop edge
|
|
if node.type in config.static_prop_types:
|
|
scope_node = node.child_by_field_name("scope")
|
|
if scope_node is None:
|
|
for child in node.children:
|
|
if child.is_named and child.type in ("name", "qualified_name", "identifier"):
|
|
scope_node = child
|
|
break
|
|
if scope_node is not None:
|
|
class_name = _read_text(scope_node, source)
|
|
tgt_nid = label_to_nid_ci.get(class_name.lower())
|
|
if tgt_nid and tgt_nid != caller_nid:
|
|
pair3 = (caller_nid, tgt_nid, "uses_static_prop")
|
|
if pair3 not in seen_static_ref_pairs:
|
|
seen_static_ref_pairs.add(pair3)
|
|
line = node.start_point[0] + 1
|
|
edges.append({
|
|
"source": caller_nid,
|
|
"target": tgt_nid,
|
|
"relation": "uses_static_prop",
|
|
"confidence": "EXTRACTED",
|
|
"confidence_score": 1.0,
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
# PHP class constant access: Foo::BAR → references_constant edge
|
|
if config.ts_module == "tree_sitter_php" and node.type == "class_constant_access_expression":
|
|
class_name = _php_class_const_scope(node)
|
|
if class_name:
|
|
tgt_nid = label_to_nid_ci.get(class_name.lower())
|
|
if tgt_nid and tgt_nid != caller_nid:
|
|
pair3 = (caller_nid, tgt_nid, "references_constant")
|
|
if pair3 not in seen_static_ref_pairs:
|
|
seen_static_ref_pairs.add(pair3)
|
|
line = node.start_point[0] + 1
|
|
edges.append({
|
|
"source": caller_nid,
|
|
"target": tgt_nid,
|
|
"relation": "references_constant",
|
|
"confidence": "EXTRACTED",
|
|
"confidence_score": 1.0,
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
for child in node.children:
|
|
walk_calls(child, caller_nid)
|
|
|
|
for caller_nid, body_node in function_bodies:
|
|
walk_calls(body_node, caller_nid)
|
|
|
|
# ── Event listener pass ───────────────────────────────────────────────────
|
|
seen_listen_pairs: set[tuple[str, str]] = set()
|
|
for event_name, listener_name, line in pending_listen_edges:
|
|
event_nid = label_to_nid_ci.get(event_name.lower())
|
|
listener_nid = label_to_nid_ci.get(listener_name.lower())
|
|
if not event_nid or not listener_nid or event_nid == listener_nid:
|
|
continue
|
|
pair2 = (event_nid, listener_nid)
|
|
if pair2 in seen_listen_pairs:
|
|
continue
|
|
seen_listen_pairs.add(pair2)
|
|
edges.append({
|
|
"source": event_nid,
|
|
"target": listener_nid,
|
|
"relation": "listened_by",
|
|
"confidence": "EXTRACTED",
|
|
"confidence_score": 1.0,
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
# ── Clean edges ───────────────────────────────────────────────────────────
|
|
valid_ids = seen_ids
|
|
clean_edges = []
|
|
for edge in edges:
|
|
src, tgt = edge["source"], edge["target"]
|
|
if src in valid_ids and (tgt in valid_ids or edge["relation"] in ("imports", "imports_from", "re_exports")):
|
|
clean_edges.append(edge)
|
|
|
|
result = {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls}
|
|
if swift_extensions:
|
|
result["swift_extensions"] = swift_extensions
|
|
return result
|
|
|
|
|
|
# ── Python rationale extraction ───────────────────────────────────────────────
|
|
|
|
_RATIONALE_PREFIXES = ("# NOTE:", "# IMPORTANT:", "# HACK:", "# WHY:", "# RATIONALE:", "# TODO:", "# FIXME:")
|
|
|
|
|
|
def _is_autogenerated_python(source: bytes) -> bool:
|
|
"""Return True if this Python file is auto-generated and its module docstring is noise.
|
|
|
|
Covers: Alembic/Flask-Migrate revisions, Django migrations, protobuf/gRPC/OpenAPI stubs.
|
|
Module docstrings in these files are change annotations or boilerplate, not rationale.
|
|
"""
|
|
head = source[:2048].decode("utf-8", errors="replace")
|
|
# Generic generated-file markers (protobuf, gRPC, OpenAPI codegen, etc.)
|
|
if any(m in head for m in ("DO NOT EDIT", "@generated", "Generated by the protocol buffer")):
|
|
return True
|
|
# Alembic / Flask-Migrate revision files
|
|
if (re.search(r"^revision\s*[:=]", head, re.MULTILINE)
|
|
and "def upgrade(" in head
|
|
and "down_revision" in head):
|
|
return True
|
|
# Django migrations
|
|
if "class Migration(migrations.Migration)" in head and "operations" in head:
|
|
return True
|
|
return False
|
|
|
|
|
|
def _extract_python_rationale(path: Path, result: dict) -> None:
|
|
"""Post-pass: extract docstrings and rationale comments from Python source.
|
|
Mutates result in-place by appending to result['nodes'] and result['edges'].
|
|
"""
|
|
try:
|
|
import tree_sitter_python as tspython
|
|
from tree_sitter import Language, Parser
|
|
language = Language(tspython.language())
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception:
|
|
return
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes = result["nodes"]
|
|
edges = result["edges"]
|
|
seen_ids = {n["id"] for n in nodes}
|
|
file_nid = _make_id(str(path))
|
|
|
|
def _get_docstring(body_node) -> tuple[str, int] | None:
|
|
if not body_node:
|
|
return None
|
|
for child in body_node.children:
|
|
if child.type == "expression_statement":
|
|
for sub in child.children:
|
|
if sub.type in ("string", "concatenated_string"):
|
|
text = source[sub.start_byte:sub.end_byte].decode("utf-8", errors="replace")
|
|
text = text.strip("\"'").strip('"""').strip("'''").strip()
|
|
if len(text) > 20:
|
|
return text, child.start_point[0] + 1
|
|
break
|
|
return None
|
|
|
|
def _add_rationale(text: str, line: int, parent_nid: str) -> None:
|
|
label = text[:80].replace("\r\n", " ").replace("\r", " ").replace("\n", " ").strip()
|
|
rid = _make_id(stem, "rationale", str(line))
|
|
if rid not in seen_ids:
|
|
seen_ids.add(rid)
|
|
nodes.append({
|
|
"id": rid,
|
|
"label": label,
|
|
"file_type": "rationale",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
})
|
|
edges.append({
|
|
"source": rid,
|
|
"target": parent_nid,
|
|
"relation": "rationale_for",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
# Module-level docstring — skip for auto-generated files (Alembic, Django
|
|
# migrations, protobuf stubs, etc.) whose module docstrings are revision
|
|
# annotations, not architectural rationale.
|
|
if not _is_autogenerated_python(source):
|
|
ds = _get_docstring(root)
|
|
if ds:
|
|
_add_rationale(ds[0], ds[1], file_nid)
|
|
|
|
# Class and function docstrings
|
|
def walk_docstrings(node, parent_nid: str) -> None:
|
|
t = node.type
|
|
if t == "class_definition":
|
|
name_node = node.child_by_field_name("name")
|
|
body = node.child_by_field_name("body")
|
|
if name_node and body:
|
|
class_name = source[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="replace")
|
|
nid = _make_id(stem, class_name)
|
|
ds = _get_docstring(body)
|
|
if ds:
|
|
_add_rationale(ds[0], ds[1], nid)
|
|
for child in body.children:
|
|
walk_docstrings(child, nid)
|
|
return
|
|
if t == "function_definition":
|
|
name_node = node.child_by_field_name("name")
|
|
body = node.child_by_field_name("body")
|
|
if name_node and body:
|
|
func_name = source[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="replace")
|
|
nid = _make_id(parent_nid, func_name) if parent_nid != file_nid else _make_id(stem, func_name)
|
|
ds = _get_docstring(body)
|
|
if ds:
|
|
_add_rationale(ds[0], ds[1], nid)
|
|
return
|
|
for child in node.children:
|
|
walk_docstrings(child, parent_nid)
|
|
|
|
walk_docstrings(root, file_nid)
|
|
|
|
# Rationale comments (# NOTE:, # IMPORTANT:, etc.)
|
|
source_text = source.decode("utf-8", errors="replace")
|
|
for lineno, line_text in enumerate(source_text.splitlines(), start=1):
|
|
stripped = line_text.strip()
|
|
if any(stripped.startswith(p) for p in _RATIONALE_PREFIXES):
|
|
_add_rationale(stripped, lineno, file_nid)
|
|
|
|
|
|
# ── Public API ────────────────────────────────────────────────────────────────
|
|
|
|
def extract_python(path: Path) -> dict:
|
|
"""Extract classes, functions, and imports from a .py file via tree-sitter AST."""
|
|
result = _extract_generic(path, _PYTHON_CONFIG)
|
|
if "error" not in result:
|
|
_extract_python_rationale(path, result)
|
|
return result
|
|
|
|
|
|
def extract_js(path: Path) -> dict:
|
|
"""Extract classes, functions, arrow functions, and imports from a .js/.ts/.tsx file."""
|
|
if path.suffix == ".tsx":
|
|
config = _TSX_CONFIG
|
|
elif path.suffix == ".ts":
|
|
config = _TS_CONFIG
|
|
else:
|
|
config = _JS_CONFIG
|
|
return _extract_generic(path, config)
|
|
|
|
|
|
def extract_svelte(path: Path) -> dict:
|
|
"""Extract imports from .svelte files: script-block via JS AST + template regex fallback.
|
|
|
|
Tree-sitter only sees the <script> block. Svelte template syntax like
|
|
{#await import('./X.svelte')} lives in the markup layer and is invisible
|
|
to the JS parser, so a regex pass covers those dynamic imports.
|
|
"""
|
|
result = _extract_generic(path, _JS_CONFIG)
|
|
try:
|
|
import re as _re
|
|
src = path.read_text(encoding="utf-8", errors="replace")
|
|
existing_ids = {n["id"] for n in result.get("nodes", [])}
|
|
# Source file node ID must match the one _extract_generic creates:
|
|
# _make_id(str(path)) - single arg, no stem prefix. Otherwise the source
|
|
# endpoint is a phantom node and build_from_json drops the edge (#701).
|
|
file_node_id = _make_id(str(path))
|
|
aliases = _load_tsconfig_aliases(path.parent)
|
|
for m in _re.finditer(r"""import\(\s*['"]([^'"]+)['"]\s*\)""", src):
|
|
raw = m.group(1)
|
|
if not raw:
|
|
continue
|
|
if raw.startswith("."):
|
|
# Relative import - resolve to full path so IDs match file node IDs.
|
|
resolved = Path(os.path.normpath(path.parent / raw))
|
|
# Apply same TS/Svelte resolver fixups as static imports so dynamic
|
|
# imports of bare paths and .svelte.ts rune files land on real
|
|
# file nodes instead of phantom ids (#716).
|
|
resolved = _resolve_js_module_path(resolved)
|
|
node_id = _make_id(str(resolved))
|
|
stub_source_file = str(resolved)
|
|
else:
|
|
# Check tsconfig.json path aliases (e.g. "$lib/" -> "src/lib/", "@/" -> "src/")
|
|
# before treating as external. Mirrors _import_js logic so SvelteKit alias
|
|
# imports resolve to the same file node IDs the extractor creates (#701).
|
|
resolved_alias = None
|
|
for alias_prefix, alias_base in aliases.items():
|
|
if raw == alias_prefix or raw.startswith(alias_prefix + "/"):
|
|
rest = raw[len(alias_prefix):].lstrip("/")
|
|
resolved_alias = Path(os.path.normpath(Path(alias_base) / rest))
|
|
break
|
|
if resolved_alias is not None:
|
|
resolved_alias = _resolve_js_module_path(resolved_alias)
|
|
node_id = _make_id(str(resolved_alias))
|
|
stub_source_file = str(resolved_alias)
|
|
else:
|
|
# Bare/scoped import (node_modules) - use last segment;
|
|
# build_from_json drops as external if no matching node exists.
|
|
module_name = raw.split("/")[-1]
|
|
if not module_name:
|
|
continue
|
|
node_id = _make_id(module_name)
|
|
stub_source_file = raw
|
|
if node_id in existing_ids:
|
|
# Edge target already a real node - just add the edge, don't add a node.
|
|
result.setdefault("edges", []).append({
|
|
"source": file_node_id, "target": node_id,
|
|
"relation": "dynamic_import", "confidence": "EXTRACTED",
|
|
"source_file": str(path),
|
|
})
|
|
continue
|
|
result.setdefault("nodes", []).append({
|
|
"id": node_id, "label": raw,
|
|
"file_type": "code", "source_file": stub_source_file,
|
|
"confidence": "EXTRACTED",
|
|
})
|
|
result.setdefault("edges", []).append({
|
|
"source": file_node_id, "target": node_id,
|
|
"relation": "dynamic_import", "confidence": "EXTRACTED",
|
|
"source_file": str(path),
|
|
})
|
|
existing_ids.add(node_id)
|
|
# Static imports inside <script> blocks. The JS tree-sitter parser fed
|
|
# the full .svelte file produces a top-level ERROR node (HTML markup
|
|
# is not valid JS), so import_statement nodes are never reached and
|
|
# static imports are silently dropped (#713). Regex over each script
|
|
# body recovers them.
|
|
script_re = _re.compile(
|
|
r"<script\b[^>]*>([\s\S]*?)</script\s*>", _re.IGNORECASE
|
|
)
|
|
static_import_re = _re.compile(
|
|
r"""import\s+(?:[^'"`;]+?\s+from\s+)?['"]([^'"]+)['"]"""
|
|
)
|
|
for script_match in script_re.finditer(src):
|
|
script_body = script_match.group(1)
|
|
for m in static_import_re.finditer(script_body):
|
|
raw = m.group(1)
|
|
if not raw:
|
|
continue
|
|
if raw.startswith("."):
|
|
resolved = Path(os.path.normpath(path.parent / raw))
|
|
if resolved.suffix == ".js":
|
|
resolved = resolved.with_suffix(".ts")
|
|
elif resolved.suffix == ".jsx":
|
|
resolved = resolved.with_suffix(".tsx")
|
|
node_id = _make_id(str(resolved))
|
|
stub_source_file = str(resolved)
|
|
else:
|
|
resolved_alias = None
|
|
for alias_prefix, alias_base in aliases.items():
|
|
if raw == alias_prefix or raw.startswith(alias_prefix + "/"):
|
|
rest = raw[len(alias_prefix):].lstrip("/")
|
|
resolved_alias = Path(os.path.normpath(Path(alias_base) / rest))
|
|
break
|
|
if resolved_alias is not None:
|
|
node_id = _make_id(str(resolved_alias))
|
|
stub_source_file = str(resolved_alias)
|
|
else:
|
|
module_name = raw.split("/")[-1]
|
|
if not module_name:
|
|
continue
|
|
node_id = _make_id(module_name)
|
|
stub_source_file = raw
|
|
if node_id in existing_ids:
|
|
result.setdefault("edges", []).append({
|
|
"source": file_node_id, "target": node_id,
|
|
"relation": "imports_from", "confidence": "EXTRACTED",
|
|
"source_file": str(path),
|
|
})
|
|
continue
|
|
result.setdefault("nodes", []).append({
|
|
"id": node_id, "label": raw,
|
|
"file_type": "code", "source_file": stub_source_file,
|
|
"confidence": "EXTRACTED",
|
|
})
|
|
result.setdefault("edges", []).append({
|
|
"source": file_node_id, "target": node_id,
|
|
"relation": "imports_from", "confidence": "EXTRACTED",
|
|
"source_file": str(path),
|
|
})
|
|
existing_ids.add(node_id)
|
|
except Exception:
|
|
pass
|
|
return result
|
|
|
|
|
|
def extract_astro(path: Path) -> dict:
|
|
"""Extract imports from .astro files: frontmatter (TS) + template regex fallback.
|
|
|
|
Astro files start with a ``---\\n...\\n---`` frontmatter block of TypeScript
|
|
setup code (where almost all imports live), followed by an HTML-with-expressions
|
|
template body, and optionally ``<script>`` blocks for client-side JS. Tree-sitter
|
|
only sees the file usefully through the frontmatter — feeding the whole file to
|
|
the JS parser produces a top-level ERROR node because the template is not valid
|
|
JS, so ``import_statement`` nodes are never reached and static imports are
|
|
silently dropped (#850). Mirrors :func:`extract_svelte` — same regex-rescue
|
|
approach, scanning the frontmatter block and any client-side ``<script>`` blocks
|
|
for static and dynamic imports.
|
|
"""
|
|
result = _extract_generic(path, _JS_CONFIG)
|
|
try:
|
|
import re as _re
|
|
src = path.read_text(encoding="utf-8", errors="replace")
|
|
existing_ids = {n["id"] for n in result.get("nodes", [])}
|
|
file_node_id = _make_id(str(path))
|
|
aliases = _load_tsconfig_aliases(path.parent)
|
|
# Dynamic imports anywhere in the file: `import('./X.astro')` is legal in
|
|
# frontmatter setup code and inside expression slots.
|
|
for m in _re.finditer(r"""import\(\s*['"]([^'"]+)['"]\s*\)""", src):
|
|
raw = m.group(1)
|
|
if not raw:
|
|
continue
|
|
if raw.startswith("."):
|
|
resolved = Path(os.path.normpath(path.parent / raw))
|
|
resolved = _resolve_js_module_path(resolved)
|
|
node_id = _make_id(str(resolved))
|
|
stub_source_file = str(resolved)
|
|
else:
|
|
resolved_alias = None
|
|
for alias_prefix, alias_base in aliases.items():
|
|
if raw == alias_prefix or raw.startswith(alias_prefix + "/"):
|
|
rest = raw[len(alias_prefix):].lstrip("/")
|
|
resolved_alias = Path(os.path.normpath(Path(alias_base) / rest))
|
|
break
|
|
if resolved_alias is not None:
|
|
resolved_alias = _resolve_js_module_path(resolved_alias)
|
|
node_id = _make_id(str(resolved_alias))
|
|
stub_source_file = str(resolved_alias)
|
|
else:
|
|
module_name = raw.split("/")[-1]
|
|
if not module_name:
|
|
continue
|
|
node_id = _make_id(module_name)
|
|
stub_source_file = raw
|
|
if node_id in existing_ids:
|
|
result.setdefault("edges", []).append({
|
|
"source": file_node_id, "target": node_id,
|
|
"relation": "dynamic_import", "confidence": "EXTRACTED",
|
|
"source_file": str(path),
|
|
})
|
|
continue
|
|
result.setdefault("nodes", []).append({
|
|
"id": node_id, "label": raw,
|
|
"file_type": "code", "source_file": stub_source_file,
|
|
"confidence": "EXTRACTED",
|
|
})
|
|
result.setdefault("edges", []).append({
|
|
"source": file_node_id, "target": node_id,
|
|
"relation": "dynamic_import", "confidence": "EXTRACTED",
|
|
"source_file": str(path),
|
|
})
|
|
existing_ids.add(node_id)
|
|
# Static imports: scan the `---...---` frontmatter at the file head plus any
|
|
# client-side <script> blocks. Both are TS/JS regions but live inside a file
|
|
# the JS tree-sitter parser cannot validate as a whole.
|
|
frontmatter_re = _re.compile(
|
|
r"\A\s*---\s*\r?\n([\s\S]*?)\r?\n---\s*(?:\r?\n|\Z)"
|
|
)
|
|
script_re = _re.compile(
|
|
r"<script\b[^>]*>([\s\S]*?)</script\s*>", _re.IGNORECASE
|
|
)
|
|
static_import_re = _re.compile(
|
|
r"""import\s+(?:[^'"`;]+?\s+from\s+)?['"]([^'"]+)['"]"""
|
|
)
|
|
regions: list[str] = []
|
|
fm = frontmatter_re.search(src)
|
|
if fm:
|
|
regions.append(fm.group(1))
|
|
for script_match in script_re.finditer(src):
|
|
regions.append(script_match.group(1))
|
|
for region in regions:
|
|
for m in static_import_re.finditer(region):
|
|
raw = m.group(1)
|
|
if not raw:
|
|
continue
|
|
if raw.startswith("."):
|
|
resolved = Path(os.path.normpath(path.parent / raw))
|
|
if resolved.suffix == ".js":
|
|
resolved = resolved.with_suffix(".ts")
|
|
elif resolved.suffix == ".jsx":
|
|
resolved = resolved.with_suffix(".tsx")
|
|
node_id = _make_id(str(resolved))
|
|
stub_source_file = str(resolved)
|
|
else:
|
|
resolved_alias = None
|
|
for alias_prefix, alias_base in aliases.items():
|
|
if raw == alias_prefix or raw.startswith(alias_prefix + "/"):
|
|
rest = raw[len(alias_prefix):].lstrip("/")
|
|
resolved_alias = Path(os.path.normpath(Path(alias_base) / rest))
|
|
break
|
|
if resolved_alias is not None:
|
|
node_id = _make_id(str(resolved_alias))
|
|
stub_source_file = str(resolved_alias)
|
|
else:
|
|
module_name = raw.split("/")[-1]
|
|
if not module_name:
|
|
continue
|
|
node_id = _make_id(module_name)
|
|
stub_source_file = raw
|
|
if node_id in existing_ids:
|
|
result.setdefault("edges", []).append({
|
|
"source": file_node_id, "target": node_id,
|
|
"relation": "imports_from", "confidence": "EXTRACTED",
|
|
"source_file": str(path),
|
|
})
|
|
continue
|
|
result.setdefault("nodes", []).append({
|
|
"id": node_id, "label": raw,
|
|
"file_type": "code", "source_file": stub_source_file,
|
|
"confidence": "EXTRACTED",
|
|
})
|
|
result.setdefault("edges", []).append({
|
|
"source": file_node_id, "target": node_id,
|
|
"relation": "imports_from", "confidence": "EXTRACTED",
|
|
"source_file": str(path),
|
|
})
|
|
existing_ids.add(node_id)
|
|
except Exception:
|
|
pass
|
|
return result
|
|
|
|
|
|
def extract_java(path: Path) -> dict:
|
|
"""Extract classes, interfaces, methods, constructors, and imports from a .java file."""
|
|
return _extract_generic(path, _JAVA_CONFIG)
|
|
|
|
|
|
def _is_spock_file(path: Path, ts_result: dict) -> bool:
|
|
"""Return True when the file contains Spock-style ``def "feature"()`` methods
|
|
that tree-sitter-groovy cannot parse, detected by checking the raw source."""
|
|
import re as _re
|
|
_SPOCK_FEATURE_RE = _re.compile(r"""^\s*def\s+[\"']""", _re.MULTILINE)
|
|
try:
|
|
return bool(_SPOCK_FEATURE_RE.search(path.read_text(errors="replace")))
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
def _extract_spock_fallback(path: Path, ts_result: dict) -> dict:
|
|
"""Regex-based fallback for Spock spec files where tree-sitter-groovy cannot parse
|
|
``def "feature name"()`` methods. Merges import edges from the tree-sitter pass
|
|
(which survive reliably) with class and feature-method nodes extracted via regex.
|
|
"""
|
|
import re as _re
|
|
source = path.read_text(errors="replace")
|
|
str_path = str(path)
|
|
stem = _file_stem(path)
|
|
|
|
# Only keep the file node from the tree-sitter pass (guaranteed present and
|
|
# correctly IDed) plus all import edges. All other ts nodes are discarded to
|
|
# avoid orphaned method/constructor nodes whose parent edges were dropped.
|
|
file_node = next((n for n in ts_result.get("nodes", []) if n.get("label") == path.name), None)
|
|
nodes: list[dict] = [file_node] if file_node else []
|
|
edges: list[dict] = [e for e in ts_result.get("edges", []) if e.get("context") == "import"]
|
|
seen_ids: set[str] = {n["id"] for n in nodes}
|
|
|
|
def _add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({
|
|
"id": nid,
|
|
"label": label,
|
|
"file_type": "code",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
})
|
|
|
|
def _add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED") -> None:
|
|
edges.append({
|
|
"source": src,
|
|
"target": tgt,
|
|
"relation": relation,
|
|
"confidence": confidence,
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
lines_text = source.splitlines()
|
|
|
|
# Extract class declarations
|
|
class_re = _re.compile(r"^\s*(?:[\w@]+\s+)*class\s+(\w+)")
|
|
# Extract Spock feature methods: def "..." () or def '...' ()
|
|
# Two separate capture groups per quote style so apostrophes inside
|
|
# double-quoted names (e.g. "shouldn't") are captured correctly.
|
|
feature_re = _re.compile(r"""^\s*def\s+(?:\"([^\"]+)\"|'([^']+)')\s*\(""")
|
|
# Extract plain def methods (non-string names) as well
|
|
plain_method_re = _re.compile(r"""^\s*def\s+(\w+)\s*\(""")
|
|
|
|
current_class_nid: str | None = None
|
|
file_nid = _make_id(str_path)
|
|
|
|
# Ensure the file node exists (tree-sitter pass may have emitted it)
|
|
if file_nid not in seen_ids:
|
|
_add_node(file_nid, path.name, 1)
|
|
|
|
for lineno, line_text in enumerate(lines_text, start=1):
|
|
cm = class_re.match(line_text)
|
|
if cm:
|
|
class_name = cm.group(1)
|
|
class_nid = _make_id(stem, class_name)
|
|
_add_node(class_nid, class_name, lineno)
|
|
_add_edge(file_nid, class_nid, "contains", lineno)
|
|
current_class_nid = class_nid
|
|
continue
|
|
|
|
if current_class_nid is None:
|
|
continue
|
|
|
|
fm = feature_re.match(line_text)
|
|
if fm:
|
|
method_name = fm.group(1) or fm.group(2)
|
|
method_label = f'"{method_name}"'
|
|
method_nid = _make_id(current_class_nid, method_name)
|
|
_add_node(method_nid, method_label, lineno)
|
|
_add_edge(current_class_nid, method_nid, "method", lineno)
|
|
continue
|
|
|
|
pm = plain_method_re.match(line_text)
|
|
if pm:
|
|
method_name = pm.group(1)
|
|
if method_name not in ("if", "while", "for", "switch", "catch"):
|
|
method_label = f".{method_name}()"
|
|
method_nid = _make_id(current_class_nid, method_name)
|
|
_add_node(method_nid, method_label, lineno)
|
|
_add_edge(current_class_nid, method_nid, "method", lineno)
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
def extract_groovy(path: Path) -> dict:
|
|
"""Extract classes, methods, constructors, and imports from a .groovy/.gradle file.
|
|
|
|
Falls back to a regex-based Spock extractor when tree-sitter-groovy cannot parse
|
|
``def "feature name"()`` methods (common in Spock specification classes).
|
|
"""
|
|
result = _extract_generic(path, _GROOVY_CONFIG)
|
|
if _is_spock_file(path, result):
|
|
result = _extract_spock_fallback(path, result)
|
|
return result
|
|
|
|
|
|
def extract_c(path: Path) -> dict:
|
|
"""Extract functions and includes from a .c/.h file."""
|
|
return _extract_generic(path, _C_CONFIG)
|
|
|
|
|
|
def extract_cpp(path: Path) -> dict:
|
|
"""Extract functions, classes, and includes from a .cpp/.cc/.cxx/.hpp file."""
|
|
return _extract_generic(path, _CPP_CONFIG)
|
|
|
|
|
|
def extract_ruby(path: Path) -> dict:
|
|
"""Extract classes, methods, singleton methods, and calls from a .rb file."""
|
|
return _extract_generic(path, _RUBY_CONFIG)
|
|
|
|
|
|
def extract_csharp(path: Path) -> dict:
|
|
"""Extract classes, interfaces, methods, namespaces, and usings from a .cs file."""
|
|
return _extract_generic(path, _CSHARP_CONFIG)
|
|
|
|
|
|
def extract_apex(path: Path) -> dict:
|
|
"""Extract classes, interfaces, enums, methods, and Salesforce constructs from
|
|
Apex .cls and .trigger files using regex (no tree-sitter grammar on PyPI)."""
|
|
import re as _re
|
|
try:
|
|
source = path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
return {"nodes": [], "edges": []}
|
|
|
|
str_path = str(path)
|
|
stem = _file_stem(path)
|
|
file_nid = _make_id(str_path)
|
|
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({
|
|
"id": nid,
|
|
"label": label,
|
|
"file_type": "code",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED") -> None:
|
|
edges.append({
|
|
"source": src,
|
|
"target": tgt,
|
|
"relation": relation,
|
|
"confidence": confidence,
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
lines = source.splitlines()
|
|
|
|
_ACCESS = r"(?:public|private|protected|global|webService)?"
|
|
_SHARING = r"(?:\s+(?:with|without|inherited)\s+sharing)?"
|
|
_MOD = r"(?:\s+(?:abstract|virtual|override|static|final|transient|testMethod))?"
|
|
_ANNOTATION = r"(?:\s*@\w+(?:\s*\([^)]*\))?\s*)*"
|
|
|
|
cls_re = _re.compile(
|
|
rf"^{_ANNOTATION}\s*{_ACCESS}{_SHARING}{_MOD}\s*class\s+(\w+)"
|
|
rf"(?:\s+extends\s+(\w+))?(?:\s+implements\s+([\w,\s]+))?\s*\{{?",
|
|
_re.IGNORECASE,
|
|
)
|
|
iface_re = _re.compile(
|
|
rf"^{_ANNOTATION}\s*{_ACCESS}{_SHARING}{_MOD}\s*interface\s+(\w+)"
|
|
rf"(?:\s+extends\s+([\w,\s]+))?\s*\{{?",
|
|
_re.IGNORECASE,
|
|
)
|
|
enum_re = _re.compile(
|
|
rf"^{_ANNOTATION}\s*{_ACCESS}{_SHARING}{_MOD}\s*enum\s+(\w+)\s*\{{?",
|
|
_re.IGNORECASE,
|
|
)
|
|
trigger_re = _re.compile(
|
|
r"^\s*trigger\s+(\w+)\s+on\s+(\w+)\s*\(",
|
|
_re.IGNORECASE,
|
|
)
|
|
method_re = _re.compile(
|
|
rf"^{_ANNOTATION}\s*{_ACCESS}{_MOD}\s*(?:static\s+)?[\w<>\[\]]+\s+(\w+)\s*\([^)]*\)\s*(?:throws\s+\w+\s*)?\{{?",
|
|
_re.IGNORECASE,
|
|
)
|
|
annotation_re = _re.compile(r"@(\w+)", _re.IGNORECASE)
|
|
soql_re = _re.compile(r"\[\s*SELECT\b[^\]]+FROM\s+(\w+)", _re.IGNORECASE)
|
|
dml_re = _re.compile(r"\b(insert|update|delete|upsert|merge|undelete)\s+\w", _re.IGNORECASE)
|
|
|
|
_CONTROL_FLOW = frozenset({
|
|
"if", "else", "for", "while", "do", "switch", "try", "catch",
|
|
"finally", "return", "throw", "new", "void", "null",
|
|
"true", "false", "this", "super", "class", "interface", "enum",
|
|
"trigger", "on",
|
|
})
|
|
|
|
current_class_nid: str | None = None
|
|
pending_annotations: list[str] = []
|
|
|
|
for lineno, line_text in enumerate(lines, start=1):
|
|
stripped = line_text.strip()
|
|
|
|
if stripped.startswith("@"):
|
|
for m in annotation_re.finditer(stripped):
|
|
pending_annotations.append(m.group(1).lower())
|
|
continue
|
|
|
|
tm = trigger_re.match(stripped)
|
|
if tm:
|
|
trig_name, sobject = tm.group(1), tm.group(2)
|
|
trig_nid = _make_id(stem, trig_name)
|
|
add_node(trig_nid, trig_name, lineno)
|
|
add_edge(file_nid, trig_nid, "contains", lineno)
|
|
sob_nid = _make_id(sobject)
|
|
if sob_nid not in seen_ids:
|
|
add_node(sob_nid, sobject, lineno)
|
|
add_edge(trig_nid, sob_nid, "uses", lineno, confidence="INFERRED")
|
|
current_class_nid = trig_nid
|
|
pending_annotations = []
|
|
continue
|
|
|
|
cm = cls_re.match(stripped)
|
|
if cm:
|
|
class_name = cm.group(1)
|
|
if class_name.lower() in _CONTROL_FLOW:
|
|
pending_annotations = []
|
|
continue
|
|
class_nid = _make_id(stem, class_name)
|
|
add_node(class_nid, class_name, lineno)
|
|
add_edge(file_nid, class_nid, "contains", lineno)
|
|
if cm.group(2):
|
|
base = cm.group(2).strip()
|
|
base_nid = _make_id(stem, base)
|
|
if base_nid not in seen_ids:
|
|
base_nid = _make_id(base)
|
|
if base_nid not in seen_ids:
|
|
add_node(base_nid, base, lineno)
|
|
add_edge(class_nid, base_nid, "extends", lineno, confidence="INFERRED")
|
|
if cm.group(3):
|
|
for iface in cm.group(3).split(","):
|
|
iface = iface.strip()
|
|
if iface:
|
|
iface_nid = _make_id(stem, iface)
|
|
if iface_nid not in seen_ids:
|
|
iface_nid = _make_id(iface)
|
|
if iface_nid not in seen_ids:
|
|
add_node(iface_nid, iface, lineno)
|
|
add_edge(class_nid, iface_nid, "implements", lineno, confidence="INFERRED")
|
|
current_class_nid = class_nid
|
|
pending_annotations = []
|
|
continue
|
|
|
|
im = iface_re.match(stripped)
|
|
if im:
|
|
iface_name = im.group(1)
|
|
if iface_name.lower() in _CONTROL_FLOW:
|
|
pending_annotations = []
|
|
continue
|
|
iface_nid = _make_id(stem, iface_name)
|
|
add_node(iface_nid, iface_name, lineno)
|
|
add_edge(file_nid if current_class_nid is None else current_class_nid,
|
|
iface_nid, "contains", lineno)
|
|
pending_annotations = []
|
|
continue
|
|
|
|
em = enum_re.match(stripped)
|
|
if em:
|
|
enum_name = em.group(1)
|
|
if enum_name.lower() in _CONTROL_FLOW:
|
|
pending_annotations = []
|
|
continue
|
|
enum_nid = _make_id(stem, enum_name)
|
|
add_node(enum_nid, enum_name, lineno)
|
|
add_edge(file_nid if current_class_nid is None else current_class_nid,
|
|
enum_nid, "contains", lineno)
|
|
pending_annotations = []
|
|
continue
|
|
|
|
if current_class_nid is not None:
|
|
mm = method_re.match(stripped)
|
|
if mm:
|
|
method_name = mm.group(1)
|
|
if method_name.lower() not in _CONTROL_FLOW:
|
|
method_nid = _make_id(current_class_nid, method_name)
|
|
method_label = f".{method_name}()"
|
|
add_node(method_nid, method_label, lineno)
|
|
add_edge(current_class_nid, method_nid, "method", lineno)
|
|
if "auraenabled" in pending_annotations or "invocablemethod" in pending_annotations:
|
|
add_edge(file_nid, method_nid, "contains", lineno, confidence="INFERRED")
|
|
pending_annotations = []
|
|
continue
|
|
|
|
pending_annotations = []
|
|
|
|
for sm in soql_re.finditer(line_text):
|
|
sobject = sm.group(1)
|
|
sob_nid = _make_id(sobject)
|
|
if sob_nid not in seen_ids:
|
|
add_node(sob_nid, sobject, lineno)
|
|
src = current_class_nid or file_nid
|
|
add_edge(src, sob_nid, "uses", lineno, confidence="INFERRED")
|
|
|
|
for dm in dml_re.finditer(line_text):
|
|
dml_op = dm.group(1).lower()
|
|
dml_nid = _make_id(f"dml_{dml_op}")
|
|
if dml_nid not in seen_ids:
|
|
add_node(dml_nid, dml_op, lineno)
|
|
src = current_class_nid or file_nid
|
|
add_edge(src, dml_nid, "uses", lineno, confidence="INFERRED")
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
def extract_kotlin(path: Path) -> dict:
|
|
"""Extract classes, objects, functions, and imports from a .kt/.kts file."""
|
|
return _extract_generic(path, _KOTLIN_CONFIG)
|
|
|
|
|
|
def extract_scala(path: Path) -> dict:
|
|
"""Extract classes, objects, functions, and imports from a .scala file."""
|
|
return _extract_generic(path, _SCALA_CONFIG)
|
|
|
|
|
|
def extract_php(path: Path) -> dict:
|
|
"""Extract classes, functions, methods, namespace uses, and calls from a .php file."""
|
|
return _extract_generic(path, _PHP_CONFIG)
|
|
|
|
|
|
def extract_blade(path: Path) -> dict:
|
|
"""Extract @include, <livewire:> components, and wire:click bindings from Blade templates."""
|
|
import re
|
|
try:
|
|
src = path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
return {"error": f"cannot read {path}"}
|
|
|
|
file_nid = _make_id(str(path))
|
|
nodes = [{"id": file_nid, "label": path.name, "file_type": "code",
|
|
"source_file": str(path), "source_location": None}]
|
|
edges = []
|
|
|
|
# @include('path.to.partial') or @include("path.to.partial")
|
|
for m in re.finditer(r"@include\(['\"]([^'\"]+)['\"]", src):
|
|
tgt = m.group(1).replace(".", "/")
|
|
tgt_nid = _make_id(tgt)
|
|
if tgt_nid not in {n["id"] for n in nodes}:
|
|
nodes.append({"id": tgt_nid, "label": m.group(1), "file_type": "code",
|
|
"source_file": str(path), "source_location": None})
|
|
edges.append({"source": file_nid, "target": tgt_nid, "relation": "includes",
|
|
"confidence": "EXTRACTED", "confidence_score": 1.0,
|
|
"source_file": str(path), "source_location": None, "weight": 1.0})
|
|
|
|
# <livewire:component.name /> or <livewire:component.name>
|
|
for m in re.finditer(r"<livewire:([\w.\-]+)", src):
|
|
tgt_nid = _make_id(m.group(1))
|
|
if tgt_nid not in {n["id"] for n in nodes}:
|
|
nodes.append({"id": tgt_nid, "label": m.group(1), "file_type": "code",
|
|
"source_file": str(path), "source_location": None})
|
|
edges.append({"source": file_nid, "target": tgt_nid, "relation": "uses_component",
|
|
"confidence": "EXTRACTED", "confidence_score": 1.0,
|
|
"source_file": str(path), "source_location": None, "weight": 1.0})
|
|
|
|
# wire:click="methodName"
|
|
for m in re.finditer(r'wire:click=["\']([^"\']+)["\']', src):
|
|
tgt_nid = _make_id(m.group(1))
|
|
if tgt_nid not in {n["id"] for n in nodes}:
|
|
nodes.append({"id": tgt_nid, "label": m.group(1), "file_type": "code",
|
|
"source_file": str(path), "source_location": None})
|
|
edges.append({"source": file_nid, "target": tgt_nid, "relation": "binds_method",
|
|
"confidence": "EXTRACTED", "confidence_score": 1.0,
|
|
"source_file": str(path), "source_location": None, "weight": 1.0})
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
def extract_dart(path: Path) -> dict:
|
|
"""Extract classes, mixins, functions, imports, generic calls, and annotations from a .dart file using regex."""
|
|
try:
|
|
src = path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
return {"error": f"cannot read {path}"}
|
|
|
|
# Remove inline and multi-line comments while leaving string literals untouched to prevent stripping URLs/paths inside strings
|
|
comment_string_pattern = re.compile(
|
|
r'"""(?:\\.|[\s\S])*?"""'
|
|
r"|'''(?:\\.|[\s\S])*?'''"
|
|
r'|"(?:\\.|[^"\\])*"'
|
|
r"|'(?:\\.|[^'\\])*'"
|
|
r"|/\*[\s\S]*?\*/"
|
|
r"|//[^\n]*"
|
|
)
|
|
def _comment_replace(match: re.Match) -> str:
|
|
token = match.group(0)
|
|
if token.startswith("/"):
|
|
return ""
|
|
return token
|
|
src_clean = comment_string_pattern.sub(_comment_replace, src)
|
|
|
|
stem = _file_stem(path)
|
|
file_nid = _make_id(str(path))
|
|
|
|
# Check if this is a part-of file and redirect to parent
|
|
part_of_match = re.search(r"^\s*part\s+of\s+['\"]([^'\"]+)['\"]", src_clean, re.MULTILINE)
|
|
is_part = False
|
|
if part_of_match:
|
|
parent_ref = part_of_match.group(1)
|
|
if parent_ref.endswith(".dart"):
|
|
try:
|
|
parent_path = (path.parent / parent_ref).resolve()
|
|
if parent_path.exists():
|
|
stem = _file_stem(parent_path)
|
|
file_nid = _make_id(str(parent_path))
|
|
is_part = True
|
|
except Exception:
|
|
pass
|
|
|
|
nodes = []
|
|
if not is_part:
|
|
nodes.append({"id": file_nid, "label": path.name, "file_type": "code",
|
|
"source_file": str(path), "source_location": None})
|
|
edges = []
|
|
defined: set[str] = set()
|
|
|
|
def add_node(nid: str, label: str, ftype: str = "code", source_file: str | None = str(path)) -> None:
|
|
if nid not in defined:
|
|
nodes.append({"id": nid, "label": label, "file_type": ftype,
|
|
"source_file": source_file, "source_location": None})
|
|
defined.add(nid)
|
|
|
|
def add_edge(src_id: str, tgt_id: str, relation: str, weight: float = 1.0, context: str | None = None) -> None:
|
|
edge = {"source": src_id, "target": tgt_id, "relation": relation,
|
|
"confidence": "EXTRACTED", "confidence_score": 1.0,
|
|
"source_file": str(path), "source_location": None, "weight": weight}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
def _split_types(text: str) -> list[str]:
|
|
parts = []
|
|
current = []
|
|
depth = 0
|
|
for char in text:
|
|
if char == "<":
|
|
depth += 1
|
|
current.append(char)
|
|
elif char == ">":
|
|
depth -= 1
|
|
current.append(char)
|
|
elif char == "," and depth == 0:
|
|
parts.append("".join(current).strip())
|
|
current = []
|
|
else:
|
|
current.append(char)
|
|
if current:
|
|
parts.append("".join(current).strip())
|
|
return [p for p in parts if p]
|
|
|
|
def _find_matching_brace(text: str, start_pos: int) -> int:
|
|
brace_count = 0
|
|
in_double_quote = False
|
|
in_single_quote = False
|
|
escape = False
|
|
|
|
first_brace = text.find("{", start_pos)
|
|
if first_brace == -1:
|
|
return len(text)
|
|
|
|
brace_count = 1
|
|
i = first_brace + 1
|
|
n = len(text)
|
|
while i < n:
|
|
char = text[i]
|
|
if escape:
|
|
escape = False
|
|
i += 1
|
|
continue
|
|
if char == "\\":
|
|
escape = True
|
|
i += 1
|
|
continue
|
|
if text[i:i+3] == '"""' and not in_single_quote:
|
|
i += 3
|
|
end = text.find('"""', i)
|
|
i = end + 3 if end != -1 else n
|
|
continue
|
|
if text[i:i+3] == "'''" and not in_double_quote:
|
|
i += 3
|
|
end = text.find("'''", i)
|
|
i = end + 3 if end != -1 else n
|
|
continue
|
|
if char == '"' and not in_single_quote:
|
|
in_double_quote = not in_double_quote
|
|
elif char == "'" and not in_double_quote:
|
|
in_single_quote = not in_single_quote
|
|
elif not in_double_quote and not in_single_quote:
|
|
if char == "{":
|
|
brace_count += 1
|
|
elif char == "}":
|
|
brace_count -= 1
|
|
if brace_count == 0:
|
|
return i + 1
|
|
i += 1
|
|
return len(text)
|
|
|
|
# 1. Classes, mixins, and enums declarations (with inheritance, mixins, interfaces, and generics)
|
|
# Supports multiple combined modifiers (e.g., abstract base class, mixin class) without capturing "class" as a name
|
|
class_pattern = r"^\s*(?:(?:abstract|sealed|base|interface|final|mixin)\s+)*(?:class|mixin|enum|extension\s+type)\s+(\w+)"
|
|
for m in re.finditer(class_pattern, src_clean, re.MULTILINE):
|
|
class_name = m.group(1)
|
|
class_nid = _make_id(stem, class_name)
|
|
add_node(class_nid, class_name)
|
|
add_edge(file_nid, class_nid, "defines")
|
|
|
|
# Manually parse extends/on, with, and implements in header to handle nested generics brackets balanced
|
|
start_idx = m.end()
|
|
rest = src_clean[start_idx : start_idx + 500]
|
|
|
|
# Skip class generic parameters
|
|
if rest.lstrip().startswith("<"):
|
|
offset = rest.find("<")
|
|
depth = 1
|
|
i = offset + 1
|
|
while i < len(rest) and depth > 0:
|
|
if rest[i] == "<": depth += 1
|
|
elif rest[i] == ">": depth -= 1
|
|
i += 1
|
|
rest = rest[i:]
|
|
|
|
# Skip primary constructor (e.g. extension type MyExt(int id))
|
|
if rest.lstrip().startswith("("):
|
|
offset = rest.find("(")
|
|
depth = 1
|
|
i = offset + 1
|
|
while i < len(rest) and depth > 0:
|
|
if rest[i] == "(": depth += 1
|
|
elif rest[i] == ")": depth -= 1
|
|
i += 1
|
|
rest = rest[i:]
|
|
|
|
header_end = rest.find("{")
|
|
if header_end == -1:
|
|
header_end = rest.find(";")
|
|
if header_end == -1:
|
|
header_end = len(rest)
|
|
header = rest[:header_end]
|
|
|
|
base_class = None
|
|
generics = None
|
|
mixins_list = []
|
|
interfaces_list = []
|
|
|
|
# Parse extends or on
|
|
extends_m = re.search(r"^\s*(?:extends|on)\s+([a-zA-Z0-9_.]+)", header)
|
|
if extends_m:
|
|
base_class = extends_m.group(1)
|
|
rest_header = header[extends_m.end():]
|
|
if rest_header.strip().startswith("<"):
|
|
start_idx = rest_header.find("<")
|
|
depth = 1
|
|
i = start_idx + 1
|
|
while i < len(rest_header) and depth > 0:
|
|
if rest_header[i] == "<":
|
|
depth += 1
|
|
elif rest_header[i] == ">":
|
|
depth -= 1
|
|
if depth == 0:
|
|
generics = rest_header[start_idx + 1 : i]
|
|
break
|
|
i += 1
|
|
if generics is not None:
|
|
header = rest_header[i + 1:]
|
|
else:
|
|
header = rest_header
|
|
else:
|
|
header = rest_header
|
|
|
|
# Parse with
|
|
with_m = re.search(r"^\s*with\s+", header)
|
|
if with_m:
|
|
rest_header = header[with_m.end():]
|
|
impl_idx = rest_header.find("implements")
|
|
if impl_idx != -1:
|
|
mixins_str = rest_header[:impl_idx]
|
|
header = rest_header[impl_idx:]
|
|
else:
|
|
mixins_str = rest_header
|
|
header = ""
|
|
mixins_list = _split_types(mixins_str)
|
|
|
|
# Parse implements
|
|
impl_m = re.search(r"^\s*implements\s+", header)
|
|
if impl_m:
|
|
interfaces_list = _split_types(header[impl_m.end():])
|
|
|
|
# Map extends inheritance relation
|
|
if base_class:
|
|
base_nid = _make_id(base_class)
|
|
add_node(base_nid, base_class, source_file=None)
|
|
add_edge(class_nid, base_nid, "inherits")
|
|
|
|
# Map generic type arguments (e.g. MyBloc extends Bloc<MyEvent, MyState>)
|
|
if generics:
|
|
for gen in _split_types(generics):
|
|
gen_clean = gen.split("<")[0].strip()
|
|
if gen_clean not in {"String", "int", "double", "bool", "num", "dynamic", "Object", "void"}:
|
|
gen_nid = _make_id(gen_clean)
|
|
add_node(gen_nid, gen_clean, source_file=None)
|
|
add_edge(class_nid, gen_nid, "references")
|
|
|
|
# Map mixins
|
|
for mixin in mixins_list:
|
|
mixin_clean = mixin.split("<")[0].strip()
|
|
mixin_nid = _make_id(mixin_clean)
|
|
add_node(mixin_nid, mixin_clean, source_file=None)
|
|
add_edge(class_nid, mixin_nid, "mixes_in")
|
|
|
|
# Map interfaces
|
|
for interface in interfaces_list:
|
|
interface_clean = interface.split("<")[0].strip()
|
|
interface_nid = _make_id(interface_clean)
|
|
add_node(interface_nid, interface_clean, source_file=None)
|
|
add_edge(class_nid, interface_nid, "implements")
|
|
|
|
# Extract class body for precise framework dependencies and event handling
|
|
start_idx = m.start()
|
|
brace_pos = src_clean.find("{", start_idx)
|
|
semi_pos = src_clean.find(";", start_idx)
|
|
|
|
has_body = brace_pos != -1
|
|
if has_body and semi_pos != -1 and semi_pos < brace_pos:
|
|
has_body = False
|
|
|
|
if has_body:
|
|
end_pos = _find_matching_brace(src_clean, start_idx)
|
|
class_body = src_clean[brace_pos:end_pos]
|
|
|
|
# Bloc event registration: on<MyEvent>()
|
|
for em in re.finditer(r"\bon<(\w+)>\s*\(", class_body):
|
|
event_name = em.group(1)
|
|
event_nid = _make_id(event_name)
|
|
add_node(event_nid, event_name, source_file=None)
|
|
add_edge(class_nid, event_nid, "calls", context="bloc_event")
|
|
|
|
# Bloc state emissions: emit(MyState) or yield MyState
|
|
for sm in re.finditer(r"\b(?:emit|yield)\s*\(?\s*(?:const\s+)?([A-Z]\w*)\b", class_body):
|
|
state_name = sm.group(1)
|
|
if state_name not in {"String", "List", "Map", "Set", "Future", "Stream", "Object"}:
|
|
state_nid = _make_id(state_name)
|
|
add_node(state_nid, state_name, source_file=None)
|
|
add_edge(class_nid, state_nid, "calls", context="emit_state")
|
|
|
|
# Bloc event additions: widget.add(MyEvent()) or bloc.add(MyEvent())
|
|
for am in re.finditer(r"\b(?:\w*[Bb]loc\w*|context\.read<\w+>\(\))\.add\(\s*(?:const\s+)?([A-Z]\w*)\b", class_body):
|
|
event_name = am.group(1)
|
|
if event_name not in {"String", "List", "Map", "Set", "Future", "Stream", "Object"}:
|
|
event_nid = _make_id(event_name)
|
|
add_node(event_nid, event_name, source_file=None)
|
|
add_edge(class_nid, event_nid, "calls", context="bloc_add_event")
|
|
|
|
# Riverpod provider references: ref.watch(provider)
|
|
for rm in re.finditer(r"\bref\.(?:watch|read|listen)\s*\(\s*(\w+)\b", class_body):
|
|
provider_name = rm.group(1)
|
|
provider_nid = _make_id(provider_name)
|
|
add_node(provider_nid, provider_name, source_file=None)
|
|
add_edge(class_nid, provider_nid, "references", context="riverpod_reference")
|
|
|
|
# Widget to Bloc references: BlocBuilder<MyBloc, ...>
|
|
for bm in re.finditer(r"\bBloc(?:Builder|Listener|Consumer|Provider|Selector)\s*<\s*([a-zA-Z0-9_]+)\b", class_body):
|
|
bloc_name = bm.group(1)
|
|
if bloc_name not in {"String", "int", "double", "bool", "num", "dynamic", "Object", "void"}:
|
|
bloc_nid = _make_id(bloc_name)
|
|
add_node(bloc_nid, bloc_name, source_file=None)
|
|
add_edge(class_nid, bloc_nid, "references", context="bloc_widget_binding")
|
|
|
|
# context.read<MyBloc>() or BlocProvider.of<MyBloc>(context)
|
|
for lm in re.finditer(r"\b(?:read|watch|select|of)\s*<([a-zA-Z0-9_]+)>", class_body):
|
|
bloc_name = lm.group(1)
|
|
if bloc_name not in {"String", "int", "double", "bool", "num", "dynamic", "Object", "void"}:
|
|
bloc_nid = _make_id(bloc_name)
|
|
add_node(bloc_nid, bloc_name, source_file=None)
|
|
add_edge(class_nid, bloc_nid, "references", context="bloc_lookup")
|
|
|
|
# 2. Annotations mapping (class, mixin, enum, or function level annotations)
|
|
# Support: @riverpod, @Riverpod(...), @injectable, @singleton, @RoutePage(), @HiveType(typeId: 0), @RestApi()
|
|
# Matches `@annotation` and links it to the next class/mixin/enum/function declaration in the file
|
|
annotation_pattern = r"@(\w+)(?:\([^)]*\))?"
|
|
for am in re.finditer(annotation_pattern, src_clean):
|
|
annotation_name = am.group(1)
|
|
if annotation_name in {"override", "deprecated", "required", "protected", "mustCallSuper"}:
|
|
continue
|
|
annotation_pos = am.end()
|
|
intervening_text = src_clean[annotation_pos : annotation_pos + 300]
|
|
|
|
class_m = re.search(r"^\s*(?:(?:abstract|sealed|base|interface|final|mixin)\s+)*(?:class|mixin|enum|extension\s+type)\s+(\w+)", intervening_text, re.MULTILINE)
|
|
func_m = re.search(r"^\s*(?:factory\s+|static\s+|async\s+|external\s+|abstract\s+)?(?:\([^)]+\)|[a-zA-Z0-9_<>,.?]+)(?:\s+[a-zA-Z0-9_<>,.?]+){0,3}\s+(\w+)\s*\(", intervening_text, re.MULTILINE)
|
|
|
|
target_nid = None
|
|
target_name = None
|
|
target_type = None
|
|
|
|
if class_m and func_m:
|
|
if class_m.start() < func_m.start():
|
|
target_name = class_m.group(1)
|
|
target_type = "class"
|
|
target_nid = _make_id(stem, target_name)
|
|
else:
|
|
target_name = func_m.group(1)
|
|
target_type = "function"
|
|
target_nid = _make_id(stem, target_name)
|
|
elif class_m:
|
|
target_name = class_m.group(1)
|
|
target_type = "class"
|
|
target_nid = _make_id(stem, target_name)
|
|
elif func_m:
|
|
target_name = func_m.group(1)
|
|
target_type = "function"
|
|
target_nid = _make_id(stem, target_name)
|
|
|
|
if target_nid and target_name:
|
|
actual_intervening = intervening_text[:min(class_m.start() if class_m else 300, func_m.start() if func_m else 300)]
|
|
if ";" not in actual_intervening and "}" not in actual_intervening and "{" not in actual_intervening:
|
|
annotation_nid = _make_id("annotation", annotation_name.lower())
|
|
add_node(annotation_nid, f"@{annotation_name}", ftype="concept", source_file=None)
|
|
add_edge(target_nid, annotation_nid, "configures")
|
|
|
|
# Riverpod specific provider generation mapping (supports camelCase class and functional providers)
|
|
if annotation_name.lower() == "riverpod":
|
|
if target_type == "class":
|
|
provider_name = target_name[0].lower() + target_name[1:] + "Provider" if len(target_name) > 1 else target_name.lower() + "Provider"
|
|
else:
|
|
provider_name = target_name + "Provider"
|
|
provider_nid = _make_id(provider_name)
|
|
add_node(provider_nid, provider_name, ftype="concept", source_file=str(path))
|
|
add_edge(target_nid, provider_nid, "defines", context="riverpod_provider")
|
|
|
|
# 2.5 Typedefs (Type Aliases)
|
|
typedef_pattern = r"^\s*typedef\s+(\w+)\s*(?:<[^>]+>)?\s*=\s*([a-zA-Z0-9_<>,.?\s]+);"
|
|
for m in re.finditer(typedef_pattern, src_clean, re.MULTILINE):
|
|
typedef_name = m.group(1)
|
|
target_type = m.group(2).split("<")[0].split(".")[-1].strip()
|
|
if target_type not in {"String", "int", "double", "bool", "num", "dynamic", "Object", "List", "Map", "Set", "void", "Function"}:
|
|
typedef_nid = _make_id(stem, typedef_name)
|
|
add_node(typedef_nid, typedef_name)
|
|
add_edge(file_nid, typedef_nid, "defines")
|
|
target_nid = _make_id(target_type)
|
|
add_node(target_nid, target_type, source_file=None)
|
|
add_edge(typedef_nid, target_nid, "references", context="typedef")
|
|
|
|
# 3. Extensions (extension MyExt on MyClass)
|
|
ext_pattern = r"^\s{0,4}extension\s+(\w+)?(?:<[^>]+>)?\s+on\s+(\w+)"
|
|
for m in re.finditer(ext_pattern, src_clean, re.MULTILINE):
|
|
ext_name = m.group(1) or f"{stem}_anonymous_extension"
|
|
target_class = m.group(2)
|
|
|
|
ext_nid = _make_id(stem, ext_name)
|
|
label = m.group(1) or f"Extension on {target_class}"
|
|
add_node(ext_nid, label)
|
|
add_edge(file_nid, ext_nid, "defines")
|
|
|
|
target_nid = _make_id(target_class)
|
|
add_node(target_nid, target_class, source_file=None)
|
|
add_edge(ext_nid, target_nid, "extends")
|
|
|
|
# 4. Top-level and class-level variable declarations (generic variables, records, late, and destructuring)
|
|
# Restrict indentation to 0-2 spaces to avoid matching local variables inside functions or switch expressions
|
|
var_pattern = r"^\s{0,2}(?:late\s+)?(?:(?:final|const|var)\s+)?(?:\([^)]+\)\s+|([a-zA-Z0-9_<>,.?]+(?:\s+[a-zA-Z0-9_<>,.?]+){0,3})\s+)?(?:(\w+)|(?:\w+\s*)?\(([^)]+)\))\s*(?:=|$|;)"
|
|
for m in re.finditer(var_pattern, src_clean, re.MULTILINE):
|
|
var_type = m.group(1)
|
|
single_name = m.group(2)
|
|
destructured_names = m.group(3)
|
|
|
|
if not re.match(r"^\s*(?:late|final|const|var)\b", m.group(0)) and not var_type:
|
|
continue
|
|
|
|
if single_name:
|
|
if single_name not in {"if", "for", "while", "switch", "catch", "return"}:
|
|
var_nid = _make_id(stem, single_name)
|
|
add_node(var_nid, single_name)
|
|
add_edge(file_nid, var_nid, "defines")
|
|
|
|
if var_type and var_type not in {"String", "int", "double", "bool", "num", "dynamic", "Object", "List", "Map", "Set", "void"}:
|
|
clean_type = var_type.split("<")[0].split(".")[-1].strip()
|
|
type_nid = _make_id(clean_type)
|
|
add_node(type_nid, clean_type, source_file=None)
|
|
add_edge(file_nid, type_nid, "references", context="variable_type")
|
|
elif destructured_names:
|
|
for name in [n.strip() for n in destructured_names.split(",") if n.strip()]:
|
|
if ":" in name:
|
|
name = name.split(":")[-1].strip()
|
|
if re.match(r"^[a-zA-Z_]\w*$", name) and not re.match(r"^[A-Z]", name):
|
|
if name not in {"if", "for", "while", "switch", "catch", "return"}:
|
|
var_nid = _make_id(stem, name)
|
|
add_node(var_nid, name)
|
|
add_edge(file_nid, var_nid, "defines")
|
|
|
|
# 5. Top-level and member functions/methods (supports typed/generic/record return types and Riverpod/Bloc references)
|
|
# Restrict indentation to 0-2 spaces to avoid matching nested local functions or methods inside multiline switch statements
|
|
method_pattern = r"^\s{0,2}(?:factory\s+|static\s+|async\s+|external\s+|abstract\s+)?(?:\([^)]+\)|[a-zA-Z0-9_<>,.?]+)(?:\s+[a-zA-Z0-9_<>,.?]+){0,3}\s+(\w+(?:\.\w+)?)\s*\("
|
|
for m in re.finditer(method_pattern, src_clean, re.MULTILINE):
|
|
raw_name = m.group(1)
|
|
name = raw_name.split(".")[-1]
|
|
if name in {"if", "for", "while", "switch", "catch", "return", "void", "dynamic", "final", "const", "get", "set"}:
|
|
continue
|
|
if re.match(r"^[A-Z]", name):
|
|
continue
|
|
nid = _make_id(stem, name)
|
|
add_node(nid, name)
|
|
add_edge(file_nid, nid, "defines")
|
|
|
|
# Get function body using matching brace to extract Riverpod reference patterns
|
|
start_idx = m.start()
|
|
brace_pos = src_clean.find("{", start_idx)
|
|
semi_pos = src_clean.find(";", start_idx)
|
|
arrow_pos = src_clean.find("=>", start_idx)
|
|
|
|
has_body = brace_pos != -1
|
|
if has_body and semi_pos != -1 and semi_pos < brace_pos:
|
|
has_body = False
|
|
if has_body and arrow_pos != -1 and arrow_pos < brace_pos:
|
|
has_body = False
|
|
|
|
if has_body:
|
|
end_pos = _find_matching_brace(src_clean, start_idx)
|
|
func_body = src_clean[brace_pos:end_pos]
|
|
|
|
# Extract Riverpod provider references: ref.watch(provider)
|
|
for rm in re.finditer(r"\bref\.(?:watch|read|listen)\s*\(\s*(\w+)\b", func_body):
|
|
provider_name = rm.group(1)
|
|
provider_nid = _make_id(provider_name)
|
|
add_node(provider_nid, provider_name, source_file=None)
|
|
add_edge(nid, provider_nid, "references", context="riverpod_reference")
|
|
|
|
# Extract Bloc event additions: widget.add(MyEvent()) or bloc.add(MyEvent())
|
|
for am in re.finditer(r"\b(?:\w*[Bb]loc\w*|context\.read<\w+>\(\))\.add\(\s*(?:const\s+)?([A-Z]\w*)\b", func_body):
|
|
event_name = am.group(1)
|
|
if event_name not in {"String", "List", "Map", "Set", "Future", "Stream", "Object"}:
|
|
event_nid = _make_id(event_name)
|
|
add_node(event_nid, event_name, source_file=None)
|
|
add_edge(nid, event_nid, "calls", context="bloc_add_event")
|
|
|
|
# context.read<MyBloc>() or BlocProvider.of<MyBloc>(context)
|
|
for lm in re.finditer(r"\b(?:read|watch|select|of)\s*<([a-zA-Z0-9_]+)>", func_body):
|
|
bloc_name = lm.group(1)
|
|
if bloc_name not in {"String", "int", "double", "bool", "num", "dynamic", "Object", "void"}:
|
|
bloc_nid = _make_id(bloc_name)
|
|
add_node(bloc_nid, bloc_name, source_file=None)
|
|
add_edge(nid, bloc_nid, "references", context="bloc_lookup")
|
|
|
|
# Universal Navigation Patters (GoRouter, AutoRoute, Navigator)
|
|
for nm in re.finditer(r"\b(?:go|push|goNamed|pushNamed|replace|replaceNamed)\s*\(\s*(?:context\s*,\s*)?['\"]([a-zA-Z0-9_/?=&%-]+)['\"]", func_body):
|
|
route_path = nm.group(1)
|
|
route_nid = _make_id("route", route_path.replace("/", "_").replace("?", "_").replace("=", "_").replace("&", "_"))
|
|
add_node(route_nid, f"Route {route_path}", ftype="concept", source_file=None)
|
|
add_edge(nid, route_nid, "navigates", context="route_path")
|
|
|
|
for cm in re.finditer(r"\b(?:go|push|goNamed|pushNamed|replace|replaceNamed)\s*\(\s*(?:context\s*,\s*)?([A-Z][a-zA-Z0-9_]*\.[a-zA-Z0-9_]+)", func_body):
|
|
route_const = cm.group(1)
|
|
route_nid = _make_id("route", route_const.replace(".", "_"))
|
|
add_node(route_nid, route_const, ftype="concept", source_file=None)
|
|
add_edge(nid, route_nid, "navigates", context="route_const")
|
|
|
|
for om in re.finditer(r"\b(?:push|replace)\s*\(\s*(?:context\s*,\s*)?.*?\b([A-Z]\w*(?:Route|Screen|Page))\b", func_body):
|
|
route_class = om.group(1)
|
|
route_nid = _make_id(route_class)
|
|
add_node(route_nid, route_class, source_file=None)
|
|
add_edge(nid, route_nid, "navigates", context="route_object")
|
|
|
|
# 6. Imports and Exports
|
|
for m in re.finditer(r"""^\s*import\s+['"]([^'"]+)['"]""", src_clean, re.MULTILINE):
|
|
pkg = m.group(1)
|
|
tgt_nid = _make_id(pkg)
|
|
add_node(tgt_nid, pkg, source_file=None)
|
|
add_edge(file_nid, tgt_nid, "imports")
|
|
|
|
for m in re.finditer(r"""^\s*export\s+['"]([^'"]+)['"]""", src_clean, re.MULTILINE):
|
|
pkg = m.group(1)
|
|
tgt_nid = _make_id(pkg)
|
|
add_node(tgt_nid, pkg, source_file=None)
|
|
add_edge(file_nid, tgt_nid, "exports")
|
|
|
|
# 7. Generic Invocations / Type Lookups (Universal Dependency Lookup)
|
|
# Matches any method call with type parameters: methodName<Type>() or object.methodName<Type>()
|
|
# Automatically extracts GetIt, Injectable, Riverpod, Provider, BlocProvider, and InheritedWidget type lookups!
|
|
generic_call_pattern = r"\b\w+<([a-zA-Z0-9_.]+(?:<[a-zA-Z0-9_.,\s<>]+>)?)\s*>\s*\("
|
|
type_blacklist = {"String", "int", "double", "bool", "num", "dynamic", "Object", "List", "Map", "Set", "Future", "Stream", "void"}
|
|
for m in re.finditer(generic_call_pattern, src_clean):
|
|
type_name = m.group(1).split(".")[-1].strip()
|
|
clean_name = type_name.split("<")[0].strip()
|
|
if clean_name not in type_blacklist:
|
|
target_nid = _make_id(clean_name)
|
|
add_node(target_nid, clean_name, source_file=None)
|
|
add_edge(file_nid, target_nid, "references", context="type_lookup")
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
def _sv_first_identifier(node, source: bytes) -> str | None:
|
|
"""First `simple_identifier` under node in pre-order, or None.
|
|
|
|
tree-sitter-verilog 1.0.3 nests declaration names a few levels deep instead
|
|
of exposing a `name` field. Scope the search to the right child node (e.g.
|
|
`function_identifier`) or this returns the return-type instead of the name.
|
|
"""
|
|
if node is None:
|
|
return None
|
|
for child in node.children:
|
|
if child.type == "simple_identifier":
|
|
return _read_text(child, source)
|
|
found = _sv_first_identifier(child, source)
|
|
if found:
|
|
return found
|
|
return None
|
|
|
|
|
|
def _sv_child(node, type_name: str) -> object | None:
|
|
if node is None:
|
|
return None
|
|
for child in node.children:
|
|
if child.type == type_name:
|
|
return child
|
|
return None
|
|
|
|
|
|
_SV_BUILTIN_TYPES = frozenset({
|
|
"bit", "logic", "reg", "wire", "int", "integer", "shortint", "longint",
|
|
"byte", "time", "real", "shortreal", "void", "string", "type", "event",
|
|
"mailbox", "semaphore", "process", "chandle",
|
|
})
|
|
|
|
_SV_NON_TYPE_WORDS = frozenset({
|
|
"return", "if", "else", "for", "foreach", "while", "case", "begin", "end",
|
|
"function", "task", "class", "endclass", "endfunction", "endtask",
|
|
})
|
|
|
|
# One level of balanced parens (e.g. `Foo #(Bar #(int))`) — bounded so malformed
|
|
# input cannot trigger pathological backtracking.
|
|
_SV_PARENS_INNER = r"(?:[^()]|\([^()]*\))*"
|
|
_SV_PARENS = r"\(" + _SV_PARENS_INNER + r"\)"
|
|
|
|
_SV_FUNC_RE = re.compile(
|
|
r"\bfunction\s+([A-Za-z_]\w*(?:\s*#\s*" + _SV_PARENS + r")?)\s+(\w+)\s*"
|
|
r"\((" + _SV_PARENS_INNER + r")\)\s*;",
|
|
re.MULTILINE,
|
|
)
|
|
|
|
_SV_PARAM_RE = re.compile(
|
|
r"\s*(?:input|output|inout|ref|const\s+ref)?\s*"
|
|
r"([A-Za-z_]\w*(?:\s*#\s*" + _SV_PARENS + r")?)\s+\w+"
|
|
)
|
|
|
|
|
|
def _sv_strip_comments(text: str) -> str:
|
|
text = re.sub(r"/\*.*?\*/", "", text, flags=re.DOTALL)
|
|
return re.sub(r"//.*", "", text)
|
|
|
|
|
|
def _sv_split_type_list(text: str) -> list[str]:
|
|
parts: list[str] = []
|
|
depth = 0
|
|
start = 0
|
|
for idx, ch in enumerate(text):
|
|
if ch == "(":
|
|
depth += 1
|
|
elif ch == ")":
|
|
depth = max(0, depth - 1)
|
|
elif ch == "," and depth == 0:
|
|
item = text[start:idx].strip()
|
|
if item:
|
|
parts.append(item)
|
|
start = idx + 1
|
|
item = text[start:].strip()
|
|
if item:
|
|
parts.append(item)
|
|
return parts
|
|
|
|
|
|
def _sv_collect_type_refs(type_text: str, generic: bool = False,
|
|
skip: frozenset[str] = frozenset()) -> list[tuple[str, str]]:
|
|
refs: list[tuple[str, str]] = []
|
|
text = type_text.strip()
|
|
if not text:
|
|
return refs
|
|
head = re.match(r"([A-Za-z_]\w*)", text)
|
|
if head:
|
|
name = head.group(1)
|
|
# `skip` carries the enclosing class's `#(type T = ...)` parameters so
|
|
# they are not mistaken for referenced types.
|
|
if name not in _SV_BUILTIN_TYPES and name not in _SV_NON_TYPE_WORDS and name not in skip:
|
|
refs.append((name, "generic_arg" if generic else "type"))
|
|
params = re.search(r"#\s*\((" + _SV_PARENS_INNER + r")\)", text)
|
|
if params:
|
|
for arg in _sv_split_type_list(params.group(1)):
|
|
refs.extend(_sv_collect_type_refs(arg, generic=True, skip=skip))
|
|
return refs
|
|
|
|
|
|
def _augment_systemverilog_semantics(
|
|
raw: str,
|
|
stem: str,
|
|
str_path: str,
|
|
file_nid: str,
|
|
nodes: list[dict],
|
|
edges: list[dict],
|
|
seen_ids: set[str],
|
|
) -> None:
|
|
label_to_nid = {node["label"]: node["id"] for node in nodes}
|
|
|
|
def line_for(offset: int) -> int:
|
|
return raw.count("\n", 0, offset) + 1
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({"id": nid, "label": label, "file_type": "code",
|
|
"source_file": str_path, "source_location": f"L{line}",
|
|
"confidence_score": 1.0})
|
|
label_to_nid[label] = nid
|
|
|
|
def ensure_type(label: str, line: int) -> str:
|
|
if label in label_to_nid:
|
|
return label_to_nid[label]
|
|
nid = _make_id(stem, label)
|
|
add_node(nid, label, line)
|
|
return nid
|
|
|
|
def add_edge(src: str, target_label: str, relation: str, line: int, context: str | None = None) -> None:
|
|
tgt = ensure_type(target_label, line)
|
|
edge = {"source": src, "target": tgt, "relation": relation,
|
|
"confidence": "EXTRACTED", "confidence_score": 1.0,
|
|
"source_file": str_path, "source_location": f"L{line}", "weight": 1.0}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
text = _sv_strip_comments(raw)
|
|
# Consuming `endclass` (rather than a lookahead) makes each match own its
|
|
# terminator, so back-to-back or malformed classes cannot bleed bodies.
|
|
class_re = re.compile(
|
|
r"\b(?:(interface)\s+)?class\s+(\w+)([^;{]*)\s*;(.*?)\bendclass\b",
|
|
re.DOTALL,
|
|
)
|
|
for match in class_re.finditer(text):
|
|
class_name = match.group(2)
|
|
header = match.group(3) or ""
|
|
body = match.group(4) or ""
|
|
line = line_for(match.start())
|
|
# `#(type T = Payload)` declares `T` as a class type parameter, not a
|
|
# referenced type — collect these to skip below.
|
|
type_params = frozenset(re.findall(r"\btype\s+(\w+)", header))
|
|
class_nid = _make_id(stem, class_name)
|
|
add_node(class_nid, class_name, line)
|
|
edges.append({"source": file_nid, "target": class_nid, "relation": "defines",
|
|
"confidence": "EXTRACTED", "confidence_score": 1.0,
|
|
"source_file": str_path, "source_location": f"L{line}", "weight": 1.0})
|
|
|
|
ext = re.search(r"\bextends\s+(\w+)", header)
|
|
if ext:
|
|
add_edge(class_nid, ext.group(1), "inherits", line)
|
|
impl = re.search(r"\bimplements\s+([^;{]+)", header)
|
|
if impl:
|
|
for iface_name in _sv_split_type_list(impl.group(1)):
|
|
add_edge(class_nid, iface_name.split("#", 1)[0].strip(), "implements", line)
|
|
|
|
body_without_functions = re.sub(
|
|
r"\bfunction\b.*?\bendfunction\b",
|
|
lambda m: "\n" * m.group(0).count("\n"),
|
|
body,
|
|
flags=re.DOTALL,
|
|
)
|
|
for field in re.finditer(r"^\s*([A-Za-z_]\w*(?:\s*#\s*\([^;]+?\))?)\s+\w+\s*;", body_without_functions, re.MULTILINE):
|
|
# Count to the start of the type token (group 1), not the match
|
|
# start: `^\s*` consumes the leading newline(s), so field.start()
|
|
# would resolve to the class's line instead of the field's.
|
|
field_line = line + body_without_functions.count("\n", 0, field.start(1))
|
|
for ref_name, role in _sv_collect_type_refs(field.group(1), skip=type_params):
|
|
add_edge(class_nid, ref_name, "references", field_line, "generic_arg" if role == "generic_arg" else "field")
|
|
|
|
for fm in _SV_FUNC_RE.finditer(body):
|
|
return_type, func_name, params = fm.group(1), fm.group(2), fm.group(3)
|
|
func_line = line + body.count("\n", 0, fm.start())
|
|
func_nid = _make_id(class_nid, func_name)
|
|
add_node(func_nid, func_name, func_line)
|
|
edges.append({"source": class_nid, "target": func_nid, "relation": "method",
|
|
"confidence": "EXTRACTED", "confidence_score": 1.0,
|
|
"source_file": str_path, "source_location": f"L{func_line}", "weight": 1.0})
|
|
for ref_name, role in _sv_collect_type_refs(return_type, skip=type_params):
|
|
add_edge(func_nid, ref_name, "references", func_line, "generic_arg" if role == "generic_arg" else "return_type")
|
|
for param in _sv_split_type_list(params):
|
|
pm = _SV_PARAM_RE.match(param)
|
|
if not pm:
|
|
continue
|
|
for ref_name, role in _sv_collect_type_refs(pm.group(1), skip=type_params):
|
|
add_edge(func_nid, ref_name, "references", func_line, "generic_arg" if role == "generic_arg" else "parameter_type")
|
|
|
|
|
|
def extract_verilog(path: Path) -> dict:
|
|
"""Extract modules, functions, tasks, package imports, instantiations, and
|
|
SystemVerilog class semantics (inherits/implements edges, field/parameter/
|
|
return-type references) from .v/.sv files."""
|
|
try:
|
|
import tree_sitter_verilog as tsverilog
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": "tree_sitter_verilog not installed"}
|
|
|
|
try:
|
|
language = Language(tsverilog.language())
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({"id": nid, "label": label, "file_type": "code",
|
|
"source_file": str_path, "source_location": f"L{line}",
|
|
"confidence_score": 1.0})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", score: float = 1.0) -> None:
|
|
edges.append({"source": src, "target": tgt, "relation": relation,
|
|
"confidence": confidence, "confidence_score": score,
|
|
"source_file": str_path, "source_location": f"L{line}", "weight": 1.0})
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
def walk(node, module_nid: str | None = None) -> None:
|
|
t = node.type
|
|
|
|
# SystemVerilog class bodies are handled by _augment_systemverilog_semantics
|
|
# (regex over source text). Skip their subtrees so in-class methods are not
|
|
# double-emitted here — and with the wrong, return-type-derived name.
|
|
if t in ("class_declaration", "interface_class_declaration"):
|
|
return
|
|
|
|
if t == "module_declaration":
|
|
mod_name = _sv_first_identifier(_sv_child(node, "module_header"), source)
|
|
if mod_name:
|
|
line = node.start_point[0] + 1
|
|
nid = _make_id(stem, mod_name)
|
|
add_node(nid, mod_name, line)
|
|
add_edge(file_nid, nid, "defines", line)
|
|
for child in node.children:
|
|
walk(child, nid)
|
|
return
|
|
|
|
# `function_prototype` only appears inside class/interface-class bodies
|
|
# (skipped above) and nests its name differently; it is intentionally not
|
|
# handled here.
|
|
elif t == "function_declaration":
|
|
fn_body = _sv_child(node, "function_body_declaration")
|
|
func_name = _sv_first_identifier(_sv_child(fn_body, "function_identifier"), source)
|
|
if func_name:
|
|
line = node.start_point[0] + 1
|
|
parent = module_nid or file_nid
|
|
nid = _make_id(parent, func_name)
|
|
add_node(nid, f"{func_name}()", line)
|
|
add_edge(parent, nid, "contains", line)
|
|
|
|
elif t == "task_declaration":
|
|
tk_body = _sv_child(node, "task_body_declaration")
|
|
task_name = _sv_first_identifier(_sv_child(tk_body, "task_identifier"), source)
|
|
if task_name:
|
|
line = node.start_point[0] + 1
|
|
parent = module_nid or file_nid
|
|
nid = _make_id(parent, task_name)
|
|
add_node(nid, task_name, line)
|
|
add_edge(parent, nid, "contains", line)
|
|
|
|
elif t == "package_import_declaration":
|
|
for child in node.children:
|
|
if child.type == "package_import_item":
|
|
pkg_text = _read_text(child, source)
|
|
pkg_name = pkg_text.split("::")[0].strip()
|
|
if pkg_name:
|
|
line = node.start_point[0] + 1
|
|
tgt_nid = _make_id(pkg_name)
|
|
add_node(tgt_nid, pkg_name, line)
|
|
src_nid = module_nid or file_nid
|
|
add_edge(src_nid, tgt_nid, "imports_from", line)
|
|
|
|
elif t in ("module_instantiation", "checker_instantiation"):
|
|
# `leaf u_leaf();` parses as checker_instantiation in 1.0.3;
|
|
# module_instantiation (when it occurs) exposes a `module_type` field.
|
|
# Both reduce to the first identifier under the node — the instantiated
|
|
# type, not the instance name (which appears later).
|
|
if module_nid:
|
|
type_node = node.child_by_field_name("module_type")
|
|
inst_type = (_read_text(type_node, source).strip() if type_node
|
|
else _sv_first_identifier(node, source))
|
|
if inst_type:
|
|
line = node.start_point[0] + 1
|
|
tgt_nid = _make_id(inst_type)
|
|
add_node(tgt_nid, inst_type, line)
|
|
add_edge(module_nid, tgt_nid, "instantiates", line)
|
|
|
|
for child in node.children:
|
|
walk(child, module_nid)
|
|
|
|
walk(root)
|
|
_augment_systemverilog_semantics(
|
|
source.decode("utf-8", errors="replace"),
|
|
stem,
|
|
str_path,
|
|
file_nid,
|
|
nodes,
|
|
edges,
|
|
seen_ids,
|
|
)
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
def extract_sql(path: Path, content: str | bytes | None = None) -> dict:
|
|
"""Extract tables, views, functions, and relationships from .sql files via tree-sitter."""
|
|
try:
|
|
import tree_sitter_sql as tssql
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": "tree_sitter_sql not installed. Run: pip install tree-sitter-sql"}
|
|
|
|
try:
|
|
language = Language(tssql.language())
|
|
parser = Parser(language)
|
|
source = (
|
|
content.encode("utf-8") if isinstance(content, str)
|
|
else content if content is not None
|
|
else path.read_bytes()
|
|
)
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
file_nid = _make_id(str_path)
|
|
nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code",
|
|
"source_file": str_path, "source_location": None}]
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = {file_nid}
|
|
table_nids: dict[str, str] = {} # name → nid for reference resolution
|
|
|
|
def _read(n) -> str:
|
|
return source[n.start_byte:n.end_byte].decode("utf-8", errors="replace")
|
|
|
|
def _obj_name(n) -> str | None:
|
|
for c in n.children:
|
|
if c.type == "object_reference":
|
|
return _read(c)
|
|
return None
|
|
|
|
def _add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({"id": nid, "label": label, "file_type": "code",
|
|
"source_file": str_path, "source_location": f"L{line}"})
|
|
edges.append({"source": file_nid, "target": nid, "relation": "contains",
|
|
"confidence": "EXTRACTED", "source_file": str_path,
|
|
"source_location": f"L{line}", "weight": 1.0})
|
|
|
|
def _add_edge(src: str, tgt: str, relation: str, line: int) -> None:
|
|
edges.append({"source": src, "target": tgt, "relation": relation,
|
|
"confidence": "EXTRACTED", "source_file": str_path,
|
|
"source_location": f"L{line}", "weight": 1.0})
|
|
|
|
def walk(node) -> None:
|
|
t = node.type
|
|
line = node.start_point[0] + 1
|
|
|
|
if t == "create_table":
|
|
name = _obj_name(node)
|
|
if name:
|
|
nid = _make_id(stem, name)
|
|
_add_node(nid, name, line)
|
|
table_nids[name.lower()] = nid
|
|
# Foreign key REFERENCES
|
|
for col in node.children:
|
|
if col.type == "column_definitions":
|
|
has_error = any(cd.type == "ERROR" for cd in col.children)
|
|
seen_refs: set[str] = set()
|
|
for cd in col.children:
|
|
if cd.type == "column_definition":
|
|
# Inline column-level REFERENCES
|
|
ref_name: str | None = None
|
|
found_ref = False
|
|
for cc in cd.children:
|
|
if cc.type == "keyword_references":
|
|
found_ref = True
|
|
elif found_ref and cc.type == "object_reference":
|
|
ref_name = _read(cc)
|
|
break
|
|
if ref_name:
|
|
ref_nid = table_nids.get(ref_name.lower()) or _make_id(stem, ref_name)
|
|
_add_edge(nid, ref_nid, "references", line)
|
|
seen_refs.add(ref_name.lower())
|
|
elif cd.type == "constraints":
|
|
# Table-level FOREIGN KEY ... REFERENCES ... constraints
|
|
for constraint in cd.children:
|
|
if constraint.type != "constraint":
|
|
continue
|
|
ref_name = None
|
|
found_ref = False
|
|
for cc in constraint.children:
|
|
if cc.type == "keyword_references":
|
|
found_ref = True
|
|
elif found_ref and cc.type == "object_reference":
|
|
ref_name = _read(cc)
|
|
break
|
|
if ref_name:
|
|
ref_nid = table_nids.get(ref_name.lower()) or _make_id(stem, ref_name)
|
|
_add_edge(nid, ref_nid, "references", line)
|
|
seen_refs.add(ref_name.lower())
|
|
if has_error:
|
|
# Dialect-specific syntax (e.g. Firebird COMPUTED BY) causes ERROR
|
|
# nodes that make the parser drop the trailing constraints block.
|
|
# Regex-scan the raw column_definitions text as fallback.
|
|
col_text = _read(col)
|
|
for rm in re.finditer(r"\bREFERENCES\s+([\w$]+)", col_text, re.IGNORECASE):
|
|
ref_name = rm.group(1)
|
|
if ref_name.lower() not in seen_refs:
|
|
ref_nid = table_nids.get(ref_name.lower()) or _make_id(stem, ref_name)
|
|
_add_edge(nid, ref_nid, "references", line)
|
|
seen_refs.add(ref_name.lower())
|
|
|
|
elif t == "create_view":
|
|
name = _obj_name(node)
|
|
if name:
|
|
nid = _make_id(stem, name)
|
|
_add_node(nid, name, line)
|
|
table_nids[name.lower()] = nid
|
|
# FROM/JOIN table references inside view body
|
|
_walk_from_refs(node, nid, line)
|
|
|
|
elif t == "create_function":
|
|
name = _obj_name(node)
|
|
if name:
|
|
nid = _make_id(stem, name)
|
|
_add_node(nid, f"{name}()", line)
|
|
_walk_from_refs(node, nid, line)
|
|
|
|
elif t == "create_procedure":
|
|
name = _obj_name(node)
|
|
if name:
|
|
nid = _make_id(stem, name)
|
|
_add_node(nid, f"{name}()", line)
|
|
_walk_from_refs(node, nid, line)
|
|
|
|
elif t == "alter_table":
|
|
name = _obj_name(node)
|
|
if name:
|
|
src_nid = table_nids.get(name.lower())
|
|
if not src_nid:
|
|
src_nid = _make_id(stem, name)
|
|
_add_node(src_nid, name, line)
|
|
table_nids[name.lower()] = src_nid
|
|
for child in node.children:
|
|
if child.type == "add_constraint":
|
|
for cc in child.children:
|
|
if cc.type != "constraint":
|
|
continue
|
|
found_ref = False
|
|
ref_name: str | None = None
|
|
for ccc in cc.children:
|
|
if ccc.type == "keyword_references":
|
|
found_ref = True
|
|
elif found_ref and ccc.type == "object_reference":
|
|
ref_name = _read(ccc)
|
|
break
|
|
if ref_name:
|
|
ref_nid = table_nids.get(ref_name.lower())
|
|
if not ref_nid:
|
|
ref_nid = _make_id(stem, ref_name)
|
|
_add_edge(src_nid, ref_nid, "references", line)
|
|
|
|
elif t == "create_trigger":
|
|
trig_name: str | None = None
|
|
tbl_name: str | None = None
|
|
after_trigger = False
|
|
after_for = False
|
|
for c in node.children:
|
|
if c.type == "keyword_trigger":
|
|
after_trigger = True
|
|
elif after_trigger and not trig_name and c.type == "object_reference":
|
|
trig_name = _read(c)
|
|
elif c.type == "keyword_for":
|
|
after_for = True
|
|
elif after_for and not tbl_name and c.type == "object_reference":
|
|
tbl_name = _read(c)
|
|
if trig_name:
|
|
trig_nid = _make_id(stem, trig_name)
|
|
_add_node(trig_nid, trig_name, line)
|
|
if tbl_name:
|
|
tbl_nid = table_nids.get(tbl_name.lower()) or _make_id(stem, tbl_name)
|
|
_add_edge(trig_nid, tbl_nid, "triggers", line)
|
|
|
|
elif t == "fb_proc_or_trigger":
|
|
text = _read(node)
|
|
m = re.match(
|
|
r"CREATE\s+(?:OR\s+(?:REPLACE|ALTER)\s+)?"
|
|
r"(PROCEDURE|TRIGGER|FUNCTION)\s+([\w$]+)",
|
|
text, re.IGNORECASE,
|
|
)
|
|
if m:
|
|
obj_type = m.group(1).upper()
|
|
obj_name = m.group(2)
|
|
obj_nid = _make_id(stem, obj_name)
|
|
label = obj_name if obj_type == "TRIGGER" else f"{obj_name}()"
|
|
_add_node(obj_nid, label, line)
|
|
if obj_type == "TRIGGER":
|
|
fm = re.search(r"\bFOR\s+([\w$]+)", text, re.IGNORECASE)
|
|
if fm:
|
|
tbl = fm.group(1)
|
|
tbl_nid = table_nids.get(tbl.lower()) or _make_id(stem, tbl)
|
|
_add_edge(obj_nid, tbl_nid, "triggers", line)
|
|
_NON_TABLES = {
|
|
"select", "where", "set", "dual", "null", "true", "false",
|
|
"first", "skip", "rows", "next", "only", "lateral",
|
|
}
|
|
seen_tbls: set[str] = set()
|
|
for rm in re.finditer(r"\b(?:FROM|JOIN|INTO)\s+([\w$]+)", text, re.IGNORECASE):
|
|
tbl = rm.group(1)
|
|
if tbl.lower() not in _NON_TABLES and tbl.lower() not in seen_tbls:
|
|
seen_tbls.add(tbl.lower())
|
|
tbl_nid = table_nids.get(tbl.lower()) or _make_id(stem, tbl)
|
|
_add_edge(obj_nid, tbl_nid, "reads_from", line)
|
|
for rm in re.finditer(r"\bUPDATE\s+([\w$]+)", text, re.IGNORECASE):
|
|
tbl = rm.group(1)
|
|
if tbl.lower() not in _NON_TABLES and tbl.lower() not in seen_tbls:
|
|
seen_tbls.add(tbl.lower())
|
|
tbl_nid = table_nids.get(tbl.lower()) or _make_id(stem, tbl)
|
|
_add_edge(obj_nid, tbl_nid, "reads_from", line)
|
|
|
|
for child in node.children:
|
|
walk(child)
|
|
|
|
def _walk_from_refs(node, caller_nid: str, line: int) -> None:
|
|
"""Recursively find FROM/JOIN table references inside a node."""
|
|
if node.type in ("from", "join"):
|
|
for c in node.children:
|
|
if c.type == "relation":
|
|
for cc in c.children:
|
|
if cc.type == "object_reference":
|
|
tbl = _read(cc)
|
|
tbl_nid = _make_id(stem, tbl)
|
|
_add_edge(caller_nid, tbl_nid, "reads_from",
|
|
c.start_point[0] + 1)
|
|
for child in node.children:
|
|
_walk_from_refs(child, caller_nid, line)
|
|
|
|
for stmt in root.children:
|
|
if stmt.type == "statement":
|
|
for child in stmt.children:
|
|
walk(child)
|
|
elif stmt.type in ("fb_proc_or_trigger", "set_term", "declare_external_function"):
|
|
walk(stmt)
|
|
|
|
# Global regex fallback: catch any REFERENCES missed due to ERROR nodes in the parse tree
|
|
# (e.g. Firebird COMPUTED BY columns push constraints out of the tree entirely).
|
|
# Snapshot after tree walk so we don't re-emit edges already captured above.
|
|
emitted = {(e["source"], e["target"]) for e in edges if e["relation"] == "references"}
|
|
src_text = source.decode("utf-8", errors="replace")
|
|
for m in re.finditer(r"CREATE\s+TABLE\s+([\w$]+)\s*\(", src_text, re.IGNORECASE):
|
|
tbl_name = m.group(1)
|
|
tbl_nid = table_nids.get(tbl_name.lower())
|
|
if tbl_nid is None:
|
|
continue
|
|
tbl_line = src_text[: m.start()].count("\n") + 1
|
|
tail = src_text[m.start():]
|
|
end = re.search(r"(?:^|\n)(?:CREATE|SET\s+TERM|ALTER)\s", tail[1:], re.IGNORECASE)
|
|
block = tail[: end.start() + 1] if end else tail
|
|
for rm in re.finditer(r"\bREFERENCES\s+([\w$]+)", block, re.IGNORECASE):
|
|
ref_name = rm.group(1)
|
|
ref_nid = table_nids.get(ref_name.lower()) or _make_id(stem, ref_name)
|
|
if (tbl_nid, ref_nid) not in emitted:
|
|
_add_edge(tbl_nid, ref_nid, "references", tbl_line)
|
|
emitted.add((tbl_nid, ref_nid))
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
def extract_lua(path: Path) -> dict:
|
|
"""Extract functions, methods, require() imports, and calls from a .lua file."""
|
|
return _extract_generic(path, _LUA_CONFIG)
|
|
|
|
|
|
def extract_swift(path: Path) -> dict:
|
|
"""Extract classes, structs, protocols, functions, imports, and calls from a .swift file."""
|
|
return _extract_generic(path, _SWIFT_CONFIG)
|
|
|
|
|
|
# ── Julia extractor (custom walk) ────────────────────────────────────────────
|
|
|
|
def extract_julia(path: Path) -> dict:
|
|
"""Extract modules, structs, functions, imports, and calls from a .jl file."""
|
|
try:
|
|
import tree_sitter_julia as tsjulia
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": "tree-sitter-julia not installed"}
|
|
|
|
try:
|
|
language = Language(tsjulia.language())
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
function_bodies: list[tuple[str, object]] = []
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({
|
|
"id": nid,
|
|
"label": label,
|
|
"file_type": "code",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", weight: float = 1.0,
|
|
context: str | None = None) -> None:
|
|
edge = {
|
|
"source": src,
|
|
"target": tgt,
|
|
"relation": relation,
|
|
"confidence": confidence,
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": weight,
|
|
}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
def ensure_named_node(name: str, line: int) -> str:
|
|
nid = _make_id(stem, name)
|
|
if nid in seen_ids:
|
|
return nid
|
|
nid = _make_id(name)
|
|
if nid not in seen_ids:
|
|
add_node(nid, name, line)
|
|
return nid
|
|
|
|
def _func_name_from_signature(sig_node) -> str | None:
|
|
"""Extract function name from a Julia signature node (call_expression > identifier)."""
|
|
for child in sig_node.children:
|
|
if child.type == "call_expression":
|
|
callee = child.children[0] if child.children else None
|
|
if callee and callee.type == "identifier":
|
|
return _read_text(callee, source)
|
|
return None
|
|
|
|
def walk_calls(body_node, func_nid: str) -> None:
|
|
if body_node is None:
|
|
return
|
|
t = body_node.type
|
|
if t in ("function_definition", "short_function_definition"):
|
|
return
|
|
if t == "call_expression" and body_node.children:
|
|
callee = body_node.children[0]
|
|
# Direct call: foo(...)
|
|
if callee.type == "identifier":
|
|
callee_name = _read_text(callee, source)
|
|
target_nid = _make_id(stem, callee_name)
|
|
add_edge(func_nid, target_nid, "calls", body_node.start_point[0] + 1,
|
|
confidence="EXTRACTED", context="call")
|
|
# Method call: obj.method(...)
|
|
elif callee.type == "field_expression" and len(callee.children) >= 3:
|
|
method_node = callee.children[-1]
|
|
method_name = _read_text(method_node, source)
|
|
target_nid = _make_id(stem, method_name)
|
|
add_edge(func_nid, target_nid, "calls", body_node.start_point[0] + 1,
|
|
confidence="EXTRACTED", context="call")
|
|
for child in body_node.children:
|
|
walk_calls(child, func_nid)
|
|
|
|
def walk(node, scope_nid: str) -> None:
|
|
t = node.type
|
|
|
|
# Module
|
|
if t == "module_definition":
|
|
name_node = next((c for c in node.children if c.type == "identifier"), None)
|
|
if name_node:
|
|
mod_name = _read_text(name_node, source)
|
|
mod_nid = _make_id(stem, mod_name)
|
|
line = node.start_point[0] + 1
|
|
add_node(mod_nid, mod_name, line)
|
|
add_edge(file_nid, mod_nid, "defines", line)
|
|
for child in node.children:
|
|
walk(child, mod_nid)
|
|
return
|
|
|
|
# Struct (struct / mutable struct — both map to struct_definition in tree-sitter-julia)
|
|
if t == "struct_definition":
|
|
# type_head may contain: identifier (simple) or binary_expression (Foo <: Bar)
|
|
type_head = next((c for c in node.children if c.type == "type_head"), None)
|
|
if not type_head:
|
|
return
|
|
struct_name: str | None = None
|
|
super_name: str | None = None
|
|
bin_expr = next((c for c in type_head.children if c.type == "binary_expression"), None)
|
|
if bin_expr:
|
|
identifiers = [c for c in bin_expr.children if c.type == "identifier"]
|
|
if identifiers:
|
|
struct_name = _read_text(identifiers[0], source)
|
|
if len(identifiers) >= 2:
|
|
super_name = _read_text(identifiers[-1], source)
|
|
else:
|
|
name_node = next((c for c in type_head.children if c.type == "identifier"), None)
|
|
if name_node:
|
|
struct_name = _read_text(name_node, source)
|
|
if not struct_name:
|
|
return
|
|
struct_nid = _make_id(stem, struct_name)
|
|
line = node.start_point[0] + 1
|
|
add_node(struct_nid, struct_name, line)
|
|
add_edge(scope_nid, struct_nid, "defines", line)
|
|
if super_name:
|
|
add_edge(struct_nid, ensure_named_node(super_name, line),
|
|
"inherits", line, confidence="EXTRACTED")
|
|
# Field types: each `name::Type` lowers to a typed_expression child of struct_definition
|
|
for child in node.children:
|
|
if child.type == "typed_expression":
|
|
type_ids = [c for c in child.children if c.type == "identifier"]
|
|
if len(type_ids) >= 2:
|
|
field_line = child.start_point[0] + 1
|
|
type_name = _read_text(type_ids[-1], source)
|
|
type_nid = ensure_named_node(type_name, field_line)
|
|
edges.append(_semantic_reference_edge(
|
|
struct_nid, type_nid, "field", str_path, field_line))
|
|
return
|
|
|
|
# Abstract type
|
|
if t == "abstract_definition":
|
|
# type_head > identifier
|
|
type_head = next((c for c in node.children if c.type == "type_head"), None)
|
|
if type_head:
|
|
name_node = next((c for c in type_head.children if c.type == "identifier"), None)
|
|
if name_node:
|
|
abs_name = _read_text(name_node, source)
|
|
abs_nid = _make_id(stem, abs_name)
|
|
line = node.start_point[0] + 1
|
|
add_node(abs_nid, abs_name, line)
|
|
add_edge(scope_nid, abs_nid, "defines", line)
|
|
return
|
|
|
|
# Function: function foo(...) ... end
|
|
if t == "function_definition":
|
|
sig_node = next((c for c in node.children if c.type == "signature"), None)
|
|
if sig_node:
|
|
func_name = _func_name_from_signature(sig_node)
|
|
if func_name:
|
|
func_nid = _make_id(stem, func_name)
|
|
line = node.start_point[0] + 1
|
|
add_node(func_nid, f"{func_name}()", line)
|
|
add_edge(scope_nid, func_nid, "defines", line)
|
|
function_bodies.append((func_nid, node))
|
|
return
|
|
|
|
# Short function: foo(x) = expr
|
|
if t == "assignment":
|
|
lhs = node.children[0] if node.children else None
|
|
if lhs and lhs.type == "call_expression" and lhs.children:
|
|
callee = lhs.children[0]
|
|
if callee.type == "identifier":
|
|
func_name = _read_text(callee, source)
|
|
func_nid = _make_id(stem, func_name)
|
|
line = node.start_point[0] + 1
|
|
add_node(func_nid, f"{func_name}()", line)
|
|
add_edge(scope_nid, func_nid, "defines", line)
|
|
# Only walk the RHS (index 2 after lhs and operator) to avoid self-loops
|
|
rhs = node.children[-1] if len(node.children) >= 3 else None
|
|
if rhs:
|
|
function_bodies.append((func_nid, rhs))
|
|
return
|
|
|
|
# Using / Import
|
|
if t in ("using_statement", "import_statement"):
|
|
line = node.start_point[0] + 1
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
mod_name = _read_text(child, source)
|
|
imp_nid = _make_id(mod_name)
|
|
add_node(imp_nid, mod_name, line)
|
|
add_edge(scope_nid, imp_nid, "imports", line, context="import")
|
|
elif child.type == "selected_import":
|
|
identifiers = [c for c in child.children if c.type == "identifier"]
|
|
if identifiers:
|
|
pkg_name = _read_text(identifiers[0], source)
|
|
pkg_nid = _make_id(pkg_name)
|
|
add_node(pkg_nid, pkg_name, line)
|
|
add_edge(scope_nid, pkg_nid, "imports", line, context="import")
|
|
return
|
|
|
|
for child in node.children:
|
|
walk(child, scope_nid)
|
|
|
|
walk(root, file_nid)
|
|
|
|
for func_nid, body_node in function_bodies:
|
|
# For function_definition nodes, walk children directly to avoid
|
|
# the boundary check returning early on the top-level node itself.
|
|
# Skip the "signature" child — it contains the function's own call_expression
|
|
# which would create a self-loop.
|
|
if body_node.type == "function_definition":
|
|
for child in body_node.children:
|
|
if child.type != "signature":
|
|
walk_calls(child, func_nid)
|
|
else:
|
|
walk_calls(body_node, func_nid)
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
_FORTRAN_CPP_EXTS = {".F", ".F90", ".F95", ".F03", ".F08"}
|
|
|
|
|
|
def _cpp_preprocess(path: Path) -> bytes:
|
|
"""Run cpp -w -P on a capital-F Fortran file and return preprocessed bytes.
|
|
|
|
Falls back to raw file bytes if cpp is not available. Capital-F extensions
|
|
conventionally require C preprocessor expansion (#ifdef MPI, #define REAL8, etc.)
|
|
before parsing.
|
|
|
|
Security (F-007): we pass `-nostdinc` and `-I /dev/null` so a malicious
|
|
source file containing `#include "/home/victim/.ssh/id_rsa"` (or any other
|
|
include directive) cannot inline arbitrary host files into the output that
|
|
we then ship to an LLM. Without these flags `cpp` happily resolves any
|
|
relative or absolute include path it can read, which is a corpus-side
|
|
file-exfiltration vector.
|
|
"""
|
|
import shutil
|
|
import subprocess
|
|
if not shutil.which("cpp"):
|
|
return path.read_bytes()
|
|
try:
|
|
# Pass an absolute path so a corpus file named like "-I/etc/x.F90" cannot
|
|
# be parsed by cpp as an option (cpp does not accept a "--" end-of-options
|
|
# terminator). An absolute path always begins with "/".
|
|
result = subprocess.run(
|
|
["cpp", "-w", "-P", "-nostdinc", "-I", "/dev/null", str(path.resolve())],
|
|
capture_output=True,
|
|
timeout=30,
|
|
)
|
|
if result.returncode == 0 and result.stdout:
|
|
return result.stdout
|
|
except Exception:
|
|
pass
|
|
return path.read_bytes()
|
|
|
|
|
|
def extract_fortran(path: Path) -> dict:
|
|
"""Extract programs, modules, subroutines, functions, use statements, and calls from Fortran files.
|
|
|
|
Capital-F extensions (.F, .F90, etc.) are run through the C preprocessor before
|
|
parsing so #ifdef/#define macros are resolved.
|
|
"""
|
|
try:
|
|
import tree_sitter_fortran as tsfortran
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": "tree-sitter-fortran not installed"}
|
|
|
|
try:
|
|
language = Language(tsfortran.language())
|
|
parser = Parser(language)
|
|
source = _cpp_preprocess(path) if path.suffix in _FORTRAN_CPP_EXTS else path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
scope_bodies: list[tuple[str, object]] = []
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({
|
|
"id": nid,
|
|
"label": label,
|
|
"file_type": "code",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", weight: float = 1.0,
|
|
context: str | None = None) -> None:
|
|
edge = {
|
|
"source": src,
|
|
"target": tgt,
|
|
"relation": relation,
|
|
"confidence": confidence,
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": weight,
|
|
}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
def _fortran_name(stmt_node) -> str | None:
|
|
"""Extract name from a *_statement node. Fortran is case-insensitive; lowercase."""
|
|
for child in stmt_node.children:
|
|
if child.type in ("name", "identifier"):
|
|
return _read_text(child, source).lower()
|
|
return None
|
|
|
|
def ensure_named_node(name: str, line: int) -> str:
|
|
nid = _make_id(stem, name)
|
|
if nid in seen_ids:
|
|
return nid
|
|
nid = _make_id(name)
|
|
if nid not in seen_ids:
|
|
add_node(nid, name, line)
|
|
return nid
|
|
|
|
def emit_signature_refs(scope_node, fn_nid: str, is_function: bool) -> None:
|
|
"""Emit references[parameter_type] / references[return_type] edges for
|
|
a subroutine/function based on its variable_declaration siblings."""
|
|
stmt_type = "function_statement" if is_function else "subroutine_statement"
|
|
stmt = next((c for c in scope_node.children if c.type == stmt_type), None)
|
|
if stmt is None:
|
|
return
|
|
param_names: set[str] = set()
|
|
params_node = next((c for c in stmt.children if c.type == "parameters"), None)
|
|
if params_node is not None:
|
|
for c in params_node.children:
|
|
if c.type == "identifier":
|
|
param_names.add(_read_text(c, source).lower())
|
|
result_name: str | None = None
|
|
if is_function:
|
|
result_node = next((c for c in stmt.children if c.type == "function_result"), None)
|
|
if result_node is not None:
|
|
res_id = next((c for c in result_node.children if c.type == "identifier"), None)
|
|
if res_id is not None:
|
|
result_name = _read_text(res_id, source).lower()
|
|
else:
|
|
# implicit result variable: same name as the function
|
|
result_name = _fortran_name(stmt)
|
|
for child in scope_node.children:
|
|
if child.type != "variable_declaration":
|
|
continue
|
|
derived = next((c for c in child.children if c.type == "derived_type"), None)
|
|
if derived is None:
|
|
continue
|
|
type_name_node = next((c for c in derived.children if c.type == "type_name"), None)
|
|
if type_name_node is None:
|
|
continue
|
|
type_name = _read_text(type_name_node, source).lower()
|
|
for var in child.children:
|
|
if var.type != "identifier":
|
|
continue
|
|
var_name = _read_text(var, source).lower()
|
|
var_line = var.start_point[0] + 1
|
|
if var_name in param_names:
|
|
tgt = ensure_named_node(type_name, var_line)
|
|
if tgt != fn_nid:
|
|
add_edge(fn_nid, tgt, "references", var_line, context="parameter_type")
|
|
elif is_function and var_name == result_name:
|
|
tgt = ensure_named_node(type_name, var_line)
|
|
if tgt != fn_nid:
|
|
add_edge(fn_nid, tgt, "references", var_line, context="return_type")
|
|
|
|
def walk_calls(node, scope_nid: str) -> None:
|
|
if node is None:
|
|
return
|
|
t = node.type
|
|
if t in ("subroutine", "function", "module", "program", "internal_procedures"):
|
|
return
|
|
# call FOO(args) — tree-sitter-fortran uses subroutine_call
|
|
if t == "subroutine_call":
|
|
name_node = next((c for c in node.children if c.type == "identifier"), None)
|
|
if name_node:
|
|
callee = _read_text(name_node, source).lower()
|
|
target_nid = _make_id(stem, callee)
|
|
add_edge(scope_nid, target_nid, "calls", node.start_point[0] + 1,
|
|
confidence="EXTRACTED", context="call")
|
|
for child in node.children:
|
|
walk_calls(child, scope_nid)
|
|
|
|
def walk(node, scope_nid: str) -> None:
|
|
t = node.type
|
|
|
|
if t == "program":
|
|
stmt = next((c for c in node.children if c.type == "program_statement"), None)
|
|
name = _fortran_name(stmt) if stmt else None
|
|
if name:
|
|
nid = _make_id(stem, name)
|
|
line = node.start_point[0] + 1
|
|
add_node(nid, name, line)
|
|
add_edge(file_nid, nid, "defines", line)
|
|
scope_bodies.append((nid, node))
|
|
for child in node.children:
|
|
walk(child, nid)
|
|
return
|
|
|
|
if t == "module":
|
|
stmt = next((c for c in node.children if c.type == "module_statement"), None)
|
|
name = _fortran_name(stmt) if stmt else None
|
|
if name:
|
|
nid = _make_id(stem, name)
|
|
line = node.start_point[0] + 1
|
|
add_node(nid, name, line)
|
|
add_edge(file_nid, nid, "defines", line)
|
|
for child in node.children:
|
|
walk(child, nid)
|
|
return
|
|
|
|
# subroutines/functions inside a module live under internal_procedures
|
|
if t == "internal_procedures":
|
|
for child in node.children:
|
|
walk(child, scope_nid)
|
|
return
|
|
|
|
if t == "derived_type_definition":
|
|
stmt = next((c for c in node.children if c.type == "derived_type_statement"), None)
|
|
if stmt is not None:
|
|
name_node = next((c for c in stmt.children if c.type == "type_name"), None)
|
|
if name_node is not None:
|
|
type_name = _read_text(name_node, source).lower()
|
|
type_nid = _make_id(stem, type_name)
|
|
line = node.start_point[0] + 1
|
|
add_node(type_nid, type_name, line)
|
|
add_edge(scope_nid, type_nid, "defines", line)
|
|
return
|
|
|
|
if t == "subroutine":
|
|
stmt = next((c for c in node.children if c.type == "subroutine_statement"), None)
|
|
name = _fortran_name(stmt) if stmt else None
|
|
if name:
|
|
nid = _make_id(stem, name)
|
|
line = node.start_point[0] + 1
|
|
add_node(nid, f"{name}()", line)
|
|
add_edge(scope_nid, nid, "defines", line)
|
|
scope_bodies.append((nid, node))
|
|
emit_signature_refs(node, nid, is_function=False)
|
|
for child in node.children:
|
|
walk(child, nid)
|
|
return
|
|
|
|
if t == "function":
|
|
stmt = next((c for c in node.children if c.type == "function_statement"), None)
|
|
name = _fortran_name(stmt) if stmt else None
|
|
if name:
|
|
nid = _make_id(stem, name)
|
|
line = node.start_point[0] + 1
|
|
add_node(nid, f"{name}()", line)
|
|
add_edge(scope_nid, nid, "defines", line)
|
|
scope_bodies.append((nid, node))
|
|
emit_signature_refs(node, nid, is_function=True)
|
|
for child in node.children:
|
|
walk(child, nid)
|
|
return
|
|
|
|
if t == "use_statement":
|
|
line = node.start_point[0] + 1
|
|
# tree-sitter-fortran uses module_name node for the used module
|
|
name_node = next((c for c in node.children if c.type in ("module_name", "name", "identifier")), None)
|
|
if name_node:
|
|
mod_name = _read_text(name_node, source).lower()
|
|
imp_nid = _make_id(mod_name)
|
|
add_node(imp_nid, mod_name, line)
|
|
add_edge(scope_nid, imp_nid, "imports", line, context="use")
|
|
return
|
|
|
|
for child in node.children:
|
|
walk(child, scope_nid)
|
|
|
|
walk(root, file_nid)
|
|
|
|
_stmt_headers = {
|
|
"subroutine_statement", "function_statement",
|
|
"program_statement", "module_statement",
|
|
}
|
|
for scope_nid, body_node in scope_bodies:
|
|
for child in body_node.children:
|
|
if child.type not in _stmt_headers:
|
|
walk_calls(child, scope_nid)
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
# ── Go extractor (custom walk) ────────────────────────────────────────────────
|
|
|
|
def extract_go(path: Path) -> dict:
|
|
"""Extract functions, methods, type declarations, and imports from a .go file."""
|
|
try:
|
|
import tree_sitter_go as tsgo
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": "tree-sitter-go not installed"}
|
|
|
|
try:
|
|
language = Language(tsgo.language())
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
# Use directory name as package scope so methods on the same type across
|
|
# multiple files in a package share one canonical type node.
|
|
pkg_scope = path.parent.name or stem
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
function_bodies: list[tuple[str, object]] = []
|
|
go_imported_pkgs: set[str] = set() # local names of imported packages
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({
|
|
"id": nid,
|
|
"label": label,
|
|
"file_type": "code",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", weight: float = 1.0,
|
|
context: str | None = None) -> None:
|
|
edge = {
|
|
"source": src,
|
|
"target": tgt,
|
|
"relation": relation,
|
|
"confidence": confidence,
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": weight,
|
|
}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
def ensure_named_node(name: str, line: int) -> str:
|
|
nid = _make_id(pkg_scope, name)
|
|
if nid in seen_ids:
|
|
return nid
|
|
nid = _make_id(name)
|
|
if nid not in seen_ids:
|
|
add_node(nid, name, line)
|
|
return nid
|
|
|
|
def emit_go_method_refs(func_node, func_nid: str, line: int) -> None:
|
|
params = func_node.child_by_field_name("parameters")
|
|
if params is not None:
|
|
for p in params.children:
|
|
if p.type != "parameter_declaration":
|
|
continue
|
|
type_node = p.child_by_field_name("type")
|
|
refs: list[tuple[str, str]] = []
|
|
_go_collect_type_refs(type_node, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "parameter_type"
|
|
tgt = ensure_named_node(ref_name, line)
|
|
if tgt != func_nid:
|
|
add_edge(func_nid, tgt, "references", line, context=ctx)
|
|
result = func_node.child_by_field_name("result")
|
|
if result is not None:
|
|
if result.type == "parameter_list":
|
|
for p in result.children:
|
|
if p.type != "parameter_declaration":
|
|
continue
|
|
type_node = p.child_by_field_name("type")
|
|
if type_node is None:
|
|
for c in p.children:
|
|
if c.is_named:
|
|
type_node = c
|
|
break
|
|
refs = []
|
|
_go_collect_type_refs(type_node, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "return_type"
|
|
tgt = ensure_named_node(ref_name, line)
|
|
if tgt != func_nid:
|
|
add_edge(func_nid, tgt, "references", line, context=ctx)
|
|
else:
|
|
refs = []
|
|
_go_collect_type_refs(result, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "return_type"
|
|
tgt = ensure_named_node(ref_name, line)
|
|
if tgt != func_nid:
|
|
add_edge(func_nid, tgt, "references", line, context=ctx)
|
|
|
|
def walk(node) -> None:
|
|
t = node.type
|
|
|
|
if t == "function_declaration":
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node:
|
|
func_name = _read_text(name_node, source)
|
|
line = node.start_point[0] + 1
|
|
func_nid = _make_id(stem, func_name)
|
|
add_node(func_nid, f"{func_name}()", line)
|
|
add_edge(file_nid, func_nid, "contains", line)
|
|
emit_go_method_refs(node, func_nid, line)
|
|
body = node.child_by_field_name("body")
|
|
if body:
|
|
function_bodies.append((func_nid, body))
|
|
return
|
|
|
|
if t == "method_declaration":
|
|
receiver = node.child_by_field_name("receiver")
|
|
receiver_type: str | None = None
|
|
if receiver:
|
|
for param in receiver.children:
|
|
if param.type == "parameter_declaration":
|
|
type_node = param.child_by_field_name("type")
|
|
if type_node:
|
|
receiver_type = _read_text(type_node, source).lstrip("*").strip()
|
|
break
|
|
name_node = node.child_by_field_name("name")
|
|
if not name_node:
|
|
return
|
|
method_name = _read_text(name_node, source)
|
|
line = node.start_point[0] + 1
|
|
|
|
if receiver_type:
|
|
parent_nid = _make_id(pkg_scope, receiver_type)
|
|
add_node(parent_nid, receiver_type, line)
|
|
method_nid = _make_id(parent_nid, method_name)
|
|
add_node(method_nid, f".{method_name}()", line)
|
|
add_edge(parent_nid, method_nid, "method", line)
|
|
else:
|
|
method_nid = _make_id(stem, method_name)
|
|
add_node(method_nid, f"{method_name}()", line)
|
|
add_edge(file_nid, method_nid, "contains", line)
|
|
|
|
emit_go_method_refs(node, method_nid, line)
|
|
body = node.child_by_field_name("body")
|
|
if body:
|
|
function_bodies.append((method_nid, body))
|
|
return
|
|
|
|
if t == "type_declaration":
|
|
for child in node.children:
|
|
if child.type != "type_spec":
|
|
continue
|
|
name_node = child.child_by_field_name("name")
|
|
if not name_node:
|
|
continue
|
|
type_name = _read_text(name_node, source)
|
|
line = child.start_point[0] + 1
|
|
type_nid = _make_id(pkg_scope, type_name)
|
|
add_node(type_nid, type_name, line)
|
|
add_edge(file_nid, type_nid, "contains", line)
|
|
# Type body: struct fields (with embeds) or interface embedding.
|
|
type_body = None
|
|
for tc in child.children:
|
|
if tc.type in ("struct_type", "interface_type"):
|
|
type_body = tc
|
|
break
|
|
if type_body is None:
|
|
continue
|
|
if type_body.type == "struct_type":
|
|
for fdl in type_body.children:
|
|
if fdl.type != "field_declaration_list":
|
|
continue
|
|
for field in fdl.children:
|
|
if field.type != "field_declaration":
|
|
continue
|
|
has_name = any(
|
|
fc.type == "field_identifier" for fc in field.children
|
|
)
|
|
type_node = field.child_by_field_name("type")
|
|
if type_node is None:
|
|
for fc in field.children:
|
|
if fc.is_named and fc.type != "field_identifier":
|
|
type_node = fc
|
|
break
|
|
refs: list[tuple[str, str]] = []
|
|
_go_collect_type_refs(type_node, source, False, refs)
|
|
for ref_name, role in refs:
|
|
tgt = ensure_named_node(ref_name, field.start_point[0] + 1)
|
|
if tgt == type_nid:
|
|
continue
|
|
if not has_name and role == "type":
|
|
add_edge(type_nid, tgt, "embeds",
|
|
field.start_point[0] + 1)
|
|
else:
|
|
ctx = "generic_arg" if role == "generic_arg" else "field"
|
|
add_edge(type_nid, tgt, "references",
|
|
field.start_point[0] + 1, context=ctx)
|
|
elif type_body.type == "interface_type":
|
|
for elem in type_body.children:
|
|
if elem.type != "type_elem":
|
|
continue
|
|
refs = []
|
|
for sub in elem.children:
|
|
if sub.is_named:
|
|
_go_collect_type_refs(sub, source, False, refs)
|
|
for ref_name, role in refs:
|
|
tgt = ensure_named_node(ref_name, elem.start_point[0] + 1)
|
|
if tgt == type_nid:
|
|
continue
|
|
if role == "type":
|
|
add_edge(type_nid, tgt, "embeds",
|
|
elem.start_point[0] + 1)
|
|
else:
|
|
add_edge(type_nid, tgt, "references",
|
|
elem.start_point[0] + 1, context="generic_arg")
|
|
return
|
|
|
|
if t == "import_declaration":
|
|
for child in node.children:
|
|
if child.type == "import_spec_list":
|
|
for spec in child.children:
|
|
if spec.type == "import_spec":
|
|
path_node = spec.child_by_field_name("path")
|
|
if path_node:
|
|
raw = _read_text(path_node, source).strip('"')
|
|
# Prefix with go_pkg_ so stdlib names (e.g. "context")
|
|
# don't collide with local files of the same basename.
|
|
tgt_nid = _make_id("go", "pkg", raw)
|
|
add_edge(file_nid, tgt_nid, "imports_from", spec.start_point[0] + 1, context="import")
|
|
# Track local name (alias or last path segment)
|
|
alias = spec.child_by_field_name("name")
|
|
local_name = _read_text(alias, source) if alias else raw.split("/")[-1]
|
|
if local_name and local_name != "_" and local_name != ".":
|
|
go_imported_pkgs.add(local_name)
|
|
elif child.type == "import_spec":
|
|
path_node = child.child_by_field_name("path")
|
|
if path_node:
|
|
raw = _read_text(path_node, source).strip('"')
|
|
tgt_nid = _make_id("go", "pkg", raw)
|
|
add_edge(file_nid, tgt_nid, "imports_from", child.start_point[0] + 1, context="import")
|
|
alias = child.child_by_field_name("name")
|
|
local_name = _read_text(alias, source) if alias else raw.split("/")[-1]
|
|
if local_name and local_name != "_" and local_name != ".":
|
|
go_imported_pkgs.add(local_name)
|
|
return
|
|
|
|
for child in node.children:
|
|
walk(child)
|
|
|
|
walk(root)
|
|
|
|
label_to_nid: dict[str, str] = {}
|
|
for n in nodes:
|
|
raw = n["label"]
|
|
normalised = raw.strip("()").lstrip(".")
|
|
label_to_nid[normalised] = n["id"]
|
|
|
|
seen_call_pairs: set[tuple[str, str]] = set()
|
|
raw_calls: list[dict] = []
|
|
|
|
def walk_calls(node, caller_nid: str) -> None:
|
|
if node.type in ("function_declaration", "method_declaration"):
|
|
return
|
|
if node.type == "call_expression":
|
|
func_node = node.child_by_field_name("function")
|
|
callee_name: str | None = None
|
|
is_member_call: bool = False
|
|
if func_node:
|
|
if func_node.type == "identifier":
|
|
callee_name = _read_text(func_node, source)
|
|
elif func_node.type == "selector_expression":
|
|
field = func_node.child_by_field_name("field")
|
|
operand = func_node.child_by_field_name("operand")
|
|
receiver_name = _read_text(operand, source) if operand else ""
|
|
# Package-qualified call (e.g. fmt.Println) → allow cross-file resolution.
|
|
# Receiver method call (e.g. s.logger.Log) → skip, no import evidence.
|
|
is_member_call = receiver_name not in go_imported_pkgs
|
|
if field:
|
|
callee_name = _read_text(field, source)
|
|
if callee_name and callee_name not in _LANGUAGE_BUILTIN_GLOBALS:
|
|
tgt_nid = label_to_nid.get(callee_name)
|
|
if tgt_nid and tgt_nid != caller_nid:
|
|
pair = (caller_nid, tgt_nid)
|
|
if pair not in seen_call_pairs:
|
|
seen_call_pairs.add(pair)
|
|
line = node.start_point[0] + 1
|
|
edges.append({
|
|
"source": caller_nid,
|
|
"target": tgt_nid,
|
|
"relation": "calls",
|
|
"context": "call",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
elif callee_name:
|
|
raw_calls.append({
|
|
"caller_nid": caller_nid,
|
|
"callee": callee_name,
|
|
"is_member_call": is_member_call,
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
})
|
|
for child in node.children:
|
|
walk_calls(child, caller_nid)
|
|
|
|
for caller_nid, body_node in function_bodies:
|
|
walk_calls(body_node, caller_nid)
|
|
|
|
valid_ids = seen_ids
|
|
clean_edges = []
|
|
for edge in edges:
|
|
src, tgt = edge["source"], edge["target"]
|
|
if src in valid_ids and (tgt in valid_ids or edge["relation"] in ("imports", "imports_from")):
|
|
clean_edges.append(edge)
|
|
|
|
return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls}
|
|
|
|
|
|
# ── Rust extractor (custom walk) ──────────────────────────────────────────────
|
|
|
|
# Common Rust trait/stdlib method names that appear in virtually every codebase.
|
|
# Resolving these cross-file produces spurious INFERRED edges across crate
|
|
# boundaries (issue #908) — skip them from the unresolved-call queue entirely.
|
|
_RUST_TRAIT_METHOD_BLOCKLIST: frozenset[str] = frozenset({
|
|
"new", "default", "parse", "from_str", "now", "clone", "into", "from",
|
|
"to_string", "to_owned", "len", "is_empty", "iter", "next", "build",
|
|
"start", "run", "init", "app", "get", "set", "push", "pop", "insert",
|
|
"remove", "contains", "collect", "map", "filter", "unwrap", "expect",
|
|
"ok", "err", "some", "none", "send", "recv", "lock", "read", "write",
|
|
})
|
|
|
|
def extract_rust(path: Path) -> dict:
|
|
"""Extract functions, structs, enums, traits, impl methods, and use declarations from a .rs file."""
|
|
try:
|
|
import tree_sitter_rust as tsrust
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": "tree-sitter-rust not installed"}
|
|
|
|
try:
|
|
language = Language(tsrust.language())
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
function_bodies: list[tuple[str, object]] = []
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({
|
|
"id": nid,
|
|
"label": label,
|
|
"file_type": "code",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", weight: float = 1.0,
|
|
context: str | None = None) -> None:
|
|
edge = {
|
|
"source": src,
|
|
"target": tgt,
|
|
"relation": relation,
|
|
"confidence": confidence,
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": weight,
|
|
}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
def ensure_named_node(name: str, line: int) -> str:
|
|
nid = _make_id(stem, name)
|
|
if nid in seen_ids:
|
|
return nid
|
|
nid = _make_id(name)
|
|
if nid not in seen_ids:
|
|
add_node(nid, name, line)
|
|
return nid
|
|
|
|
def emit_param_return_refs(func_node, func_nid: str, line: int) -> None:
|
|
params = func_node.child_by_field_name("parameters")
|
|
if params is not None:
|
|
for p in params.children:
|
|
if p.type != "parameter":
|
|
continue
|
|
type_node = p.child_by_field_name("type")
|
|
refs: list[tuple[str, str]] = []
|
|
_rust_collect_type_refs(type_node, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "parameter_type"
|
|
tgt = ensure_named_node(ref_name, line)
|
|
if tgt != func_nid:
|
|
add_edge(func_nid, tgt, "references", line, context=ctx)
|
|
return_type = func_node.child_by_field_name("return_type")
|
|
if return_type is not None:
|
|
refs = []
|
|
_rust_collect_type_refs(return_type, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "return_type"
|
|
tgt = ensure_named_node(ref_name, line)
|
|
if tgt != func_nid:
|
|
add_edge(func_nid, tgt, "references", line, context=ctx)
|
|
|
|
def walk(node, parent_impl_nid: str | None = None) -> None:
|
|
t = node.type
|
|
|
|
if t == "function_item":
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node:
|
|
func_name = _read_text(name_node, source)
|
|
line = node.start_point[0] + 1
|
|
if parent_impl_nid:
|
|
func_nid = _make_id(parent_impl_nid, func_name)
|
|
add_node(func_nid, f".{func_name}()", line)
|
|
add_edge(parent_impl_nid, func_nid, "method", line)
|
|
else:
|
|
func_nid = _make_id(stem, func_name)
|
|
add_node(func_nid, f"{func_name}()", line)
|
|
add_edge(file_nid, func_nid, "contains", line)
|
|
emit_param_return_refs(node, func_nid, line)
|
|
body = node.child_by_field_name("body")
|
|
if body:
|
|
function_bodies.append((func_nid, body))
|
|
return
|
|
|
|
if t in ("struct_item", "enum_item", "trait_item"):
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node:
|
|
item_name = _read_text(name_node, source)
|
|
line = node.start_point[0] + 1
|
|
item_nid = _make_id(stem, item_name)
|
|
add_node(item_nid, item_name, line)
|
|
add_edge(file_nid, item_nid, "contains", line)
|
|
if t == "trait_item":
|
|
for c in node.children:
|
|
if c.type != "trait_bounds":
|
|
continue
|
|
for sub in c.children:
|
|
if not sub.is_named:
|
|
continue
|
|
refs: list[tuple[str, str]] = []
|
|
_rust_collect_type_refs(sub, source, False, refs)
|
|
for idx, (ref_name, _role) in enumerate(refs):
|
|
tgt = ensure_named_node(ref_name, line)
|
|
if tgt == item_nid:
|
|
continue
|
|
rel = "inherits" if idx == 0 else "references"
|
|
if rel == "inherits":
|
|
add_edge(item_nid, tgt, "inherits", line)
|
|
else:
|
|
add_edge(item_nid, tgt, "references", line,
|
|
context="generic_arg")
|
|
if t == "struct_item":
|
|
for c in node.children:
|
|
if c.type != "field_declaration_list":
|
|
continue
|
|
for field in c.children:
|
|
if field.type != "field_declaration":
|
|
continue
|
|
type_node = field.child_by_field_name("type")
|
|
if type_node is None:
|
|
for fc in field.children:
|
|
if fc.type in ("type_identifier", "generic_type",
|
|
"scoped_type_identifier",
|
|
"reference_type", "primitive_type"):
|
|
type_node = fc
|
|
break
|
|
refs = []
|
|
_rust_collect_type_refs(type_node, source, False, refs)
|
|
for ref_name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "field"
|
|
tgt = ensure_named_node(ref_name, field.start_point[0] + 1)
|
|
if tgt != item_nid:
|
|
add_edge(item_nid, tgt, "references",
|
|
field.start_point[0] + 1, context=ctx)
|
|
return
|
|
|
|
if t == "impl_item":
|
|
type_node = node.child_by_field_name("type")
|
|
trait_node = node.child_by_field_name("trait")
|
|
impl_nid: str | None = None
|
|
if type_node:
|
|
type_name = _read_text(type_node, source).strip()
|
|
impl_nid = _make_id(stem, type_name)
|
|
add_node(impl_nid, type_name, node.start_point[0] + 1)
|
|
if trait_node is not None and impl_nid is not None:
|
|
refs: list[tuple[str, str]] = []
|
|
_rust_collect_type_refs(trait_node, source, False, refs)
|
|
for idx, (ref_name, _role) in enumerate(refs):
|
|
tgt = ensure_named_node(ref_name, node.start_point[0] + 1)
|
|
if tgt == impl_nid:
|
|
continue
|
|
if idx == 0:
|
|
add_edge(impl_nid, tgt, "implements", node.start_point[0] + 1)
|
|
else:
|
|
add_edge(impl_nid, tgt, "references", node.start_point[0] + 1,
|
|
context="generic_arg")
|
|
body = node.child_by_field_name("body")
|
|
if body:
|
|
for child in body.children:
|
|
walk(child, parent_impl_nid=impl_nid)
|
|
return
|
|
|
|
if t == "use_declaration":
|
|
arg = node.child_by_field_name("argument")
|
|
if arg:
|
|
raw = _read_text(arg, source)
|
|
clean = raw.split("{")[0].rstrip(":").rstrip("*").rstrip(":")
|
|
module_name = clean.split("::")[-1].strip()
|
|
if module_name:
|
|
tgt_nid = _make_id(module_name)
|
|
add_edge(file_nid, tgt_nid, "imports_from", node.start_point[0] + 1, context="import")
|
|
return
|
|
|
|
for child in node.children:
|
|
walk(child, parent_impl_nid=None)
|
|
|
|
walk(root)
|
|
|
|
label_to_nid: dict[str, str] = {}
|
|
for n in nodes:
|
|
raw = n["label"]
|
|
normalised = raw.strip("()").lstrip(".")
|
|
label_to_nid[normalised] = n["id"]
|
|
|
|
seen_call_pairs: set[tuple[str, str]] = set()
|
|
raw_calls: list[dict] = []
|
|
|
|
def walk_calls(node, caller_nid: str) -> None:
|
|
if node.type == "function_item":
|
|
return
|
|
if node.type == "call_expression":
|
|
func_node = node.child_by_field_name("function")
|
|
callee_name: str | None = None
|
|
is_member_call: bool = False
|
|
is_scoped_call: bool = False
|
|
if func_node:
|
|
if func_node.type == "identifier":
|
|
callee_name = _read_text(func_node, source)
|
|
elif func_node.type == "field_expression":
|
|
is_member_call = True
|
|
field = func_node.child_by_field_name("field")
|
|
if field:
|
|
callee_name = _read_text(field, source)
|
|
elif func_node.type == "scoped_identifier":
|
|
# Type::method() — still allow in-file EXTRACTED match, but
|
|
# skip cross-file resolution: bare last-segment lookup ignores
|
|
# crate boundaries and produces spurious INFERRED edges (#908).
|
|
is_scoped_call = True
|
|
name = func_node.child_by_field_name("name")
|
|
if name:
|
|
callee_name = _read_text(name, source)
|
|
if callee_name and callee_name not in _LANGUAGE_BUILTIN_GLOBALS:
|
|
tgt_nid = label_to_nid.get(callee_name)
|
|
if tgt_nid and tgt_nid != caller_nid:
|
|
pair = (caller_nid, tgt_nid)
|
|
if pair not in seen_call_pairs:
|
|
seen_call_pairs.add(pair)
|
|
line = node.start_point[0] + 1
|
|
edges.append({
|
|
"source": caller_nid,
|
|
"target": tgt_nid,
|
|
"relation": "calls",
|
|
"context": "call",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
elif not is_scoped_call and callee_name.lower() not in _RUST_TRAIT_METHOD_BLOCKLIST:
|
|
raw_calls.append({
|
|
"caller_nid": caller_nid,
|
|
"callee": callee_name,
|
|
"is_member_call": is_member_call,
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
})
|
|
for child in node.children:
|
|
walk_calls(child, caller_nid)
|
|
|
|
for caller_nid, body_node in function_bodies:
|
|
walk_calls(body_node, caller_nid)
|
|
|
|
valid_ids = seen_ids
|
|
clean_edges = []
|
|
for edge in edges:
|
|
src, tgt = edge["source"], edge["target"]
|
|
if src in valid_ids and (tgt in valid_ids or edge["relation"] in ("imports", "imports_from")):
|
|
clean_edges.append(edge)
|
|
|
|
return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls}
|
|
|
|
|
|
# ── Zig ───────────────────────────────────────────────────────────────────────
|
|
|
|
def extract_zig(path: Path) -> dict:
|
|
"""Extract functions, structs, enums, unions, and imports from a .zig file."""
|
|
try:
|
|
import tree_sitter_zig as tszig
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": "tree_sitter_zig not installed"}
|
|
|
|
try:
|
|
language = Language(tszig.language())
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
function_bodies: list[tuple[str, Any]] = []
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({"id": nid, "label": label, "file_type": "code",
|
|
"source_file": str_path, "source_location": f"L{line}"})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", weight: float = 1.0,
|
|
context: str | None = None) -> None:
|
|
edge = {"source": src, "target": tgt, "relation": relation,
|
|
"confidence": confidence, "source_file": str_path,
|
|
"source_location": f"L{line}", "weight": weight}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
def _extract_import(node) -> None:
|
|
for child in node.children:
|
|
if child.type == "builtin_function":
|
|
bi = None
|
|
args = None
|
|
for c in child.children:
|
|
if c.type == "builtin_identifier":
|
|
bi = _read_text(c, source)
|
|
elif c.type == "arguments":
|
|
args = c
|
|
if bi in ("@import", "@cImport") and args:
|
|
for arg in args.children:
|
|
if arg.type in ("string_literal", "string"):
|
|
raw = _read_text(arg, source).strip('"')
|
|
module_name = raw.split("/")[-1].split(".")[0]
|
|
if module_name:
|
|
tgt_nid = _make_id(module_name)
|
|
add_edge(file_nid, tgt_nid, "imports_from",
|
|
node.start_point[0] + 1)
|
|
return
|
|
elif child.type == "field_expression":
|
|
_extract_import(child)
|
|
return
|
|
|
|
def walk(node, parent_struct_nid: str | None = None) -> None:
|
|
t = node.type
|
|
|
|
if t == "function_declaration":
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node:
|
|
func_name = _read_text(name_node, source)
|
|
line = node.start_point[0] + 1
|
|
if parent_struct_nid:
|
|
func_nid = _make_id(parent_struct_nid, func_name)
|
|
add_node(func_nid, f".{func_name}()", line)
|
|
add_edge(parent_struct_nid, func_nid, "method", line)
|
|
else:
|
|
func_nid = _make_id(stem, func_name)
|
|
add_node(func_nid, f"{func_name}()", line)
|
|
add_edge(file_nid, func_nid, "contains", line)
|
|
body = node.child_by_field_name("body")
|
|
if body:
|
|
function_bodies.append((func_nid, body))
|
|
return
|
|
|
|
if t == "variable_declaration":
|
|
name_node = None
|
|
value_node = None
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
name_node = child
|
|
elif child.type in ("struct_declaration", "enum_declaration",
|
|
"union_declaration", "builtin_function",
|
|
"field_expression"):
|
|
value_node = child
|
|
|
|
if value_node and value_node.type == "struct_declaration":
|
|
if name_node:
|
|
struct_name = _read_text(name_node, source)
|
|
line = node.start_point[0] + 1
|
|
struct_nid = _make_id(stem, struct_name)
|
|
add_node(struct_nid, struct_name, line)
|
|
add_edge(file_nid, struct_nid, "contains", line)
|
|
for child in value_node.children:
|
|
walk(child, parent_struct_nid=struct_nid)
|
|
return
|
|
|
|
if value_node and value_node.type in ("enum_declaration", "union_declaration"):
|
|
if name_node:
|
|
type_name = _read_text(name_node, source)
|
|
line = node.start_point[0] + 1
|
|
type_nid = _make_id(stem, type_name)
|
|
add_node(type_nid, type_name, line)
|
|
add_edge(file_nid, type_nid, "contains", line)
|
|
return
|
|
|
|
if value_node and value_node.type in ("builtin_function", "field_expression"):
|
|
_extract_import(node)
|
|
return
|
|
|
|
for child in node.children:
|
|
walk(child, parent_struct_nid)
|
|
|
|
walk(root)
|
|
|
|
seen_call_pairs: set[tuple[str, str]] = set()
|
|
raw_calls: list[dict] = []
|
|
|
|
def walk_calls(node, caller_nid: str) -> None:
|
|
if node.type == "function_declaration":
|
|
return
|
|
if node.type == "call_expression":
|
|
fn = node.child_by_field_name("function")
|
|
if fn:
|
|
fn_text = _read_text(fn, source)
|
|
callee = fn_text.split(".")[-1]
|
|
is_member_call = "." in fn_text
|
|
tgt_nid = next((n["id"] for n in nodes if n["label"] in
|
|
(f"{callee}()", f".{callee}()")), None)
|
|
if tgt_nid and tgt_nid != caller_nid:
|
|
pair = (caller_nid, tgt_nid)
|
|
if pair not in seen_call_pairs:
|
|
seen_call_pairs.add(pair)
|
|
add_edge(caller_nid, tgt_nid, "calls",
|
|
node.start_point[0] + 1,
|
|
confidence="EXTRACTED", weight=1.0)
|
|
elif callee:
|
|
raw_calls.append({
|
|
"caller_nid": caller_nid,
|
|
"callee": callee,
|
|
"is_member_call": is_member_call,
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
})
|
|
for child in node.children:
|
|
walk_calls(child, caller_nid)
|
|
|
|
for caller_nid, body_node in function_bodies:
|
|
walk_calls(body_node, caller_nid)
|
|
|
|
clean_edges = [e for e in edges if e["source"] in seen_ids and
|
|
(e["target"] in seen_ids or e["relation"] == "imports_from")]
|
|
return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls}
|
|
|
|
|
|
# ── PowerShell ────────────────────────────────────────────────────────────────
|
|
|
|
def extract_powershell(path: Path) -> dict:
|
|
"""Extract functions, classes, methods, and using statements from a .ps1 file."""
|
|
try:
|
|
import tree_sitter_powershell as tsps
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": "tree_sitter_powershell not installed"}
|
|
|
|
try:
|
|
language = Language(tsps.language())
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
function_bodies: list[tuple[str, Any]] = []
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({"id": nid, "label": label, "file_type": "code",
|
|
"source_file": str_path, "source_location": f"L{line}"})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", weight: float = 1.0,
|
|
context: str | None = None) -> None:
|
|
edge = {"source": src, "target": tgt, "relation": relation,
|
|
"confidence": confidence, "source_file": str_path,
|
|
"source_location": f"L{line}", "weight": weight}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
_PS_SKIP = frozenset({
|
|
"using", "return", "if", "else", "elseif", "foreach", "for",
|
|
"while", "do", "switch", "try", "catch", "finally", "throw",
|
|
"break", "continue", "exit", "param", "begin", "process", "end",
|
|
})
|
|
|
|
def _find_script_block_body(node):
|
|
for child in node.children:
|
|
if child.type == "script_block":
|
|
for sc in child.children:
|
|
if sc.type == "script_block_body":
|
|
return sc
|
|
return child
|
|
return None
|
|
|
|
def ensure_named_node(name: str, line: int) -> str:
|
|
nid = _make_id(stem, name)
|
|
if nid in seen_ids:
|
|
return nid
|
|
nid = _make_id(name)
|
|
if nid not in seen_ids:
|
|
add_node(nid, name, line)
|
|
return nid
|
|
|
|
def _ps_type_name(type_literal_node) -> str | None:
|
|
"""Drill into a type_literal node and return the inner type_identifier text."""
|
|
if type_literal_node is None:
|
|
return None
|
|
for spec in type_literal_node.children:
|
|
if spec.type != "type_spec":
|
|
continue
|
|
for tname in spec.children:
|
|
if tname.type != "type_name":
|
|
continue
|
|
for tid in tname.children:
|
|
if tid.type == "type_identifier":
|
|
return _read_text(tid, source)
|
|
return None
|
|
|
|
def walk(node, parent_class_nid: str | None = None) -> None:
|
|
t = node.type
|
|
|
|
if t == "function_statement":
|
|
name_node = next((c for c in node.children if c.type == "function_name"), None)
|
|
if name_node:
|
|
func_name = _read_text(name_node, source)
|
|
line = node.start_point[0] + 1
|
|
func_nid = _make_id(stem, func_name)
|
|
add_node(func_nid, f"{func_name}()", line)
|
|
add_edge(file_nid, func_nid, "contains", line)
|
|
body = _find_script_block_body(node)
|
|
if body:
|
|
function_bodies.append((func_nid, body))
|
|
return
|
|
|
|
if t == "class_statement":
|
|
name_node = next((c for c in node.children if c.type == "simple_name"), None)
|
|
if name_node:
|
|
class_name = _read_text(name_node, source)
|
|
line = node.start_point[0] + 1
|
|
class_nid = _make_id(stem, class_name)
|
|
add_node(class_nid, class_name, line)
|
|
add_edge(file_nid, class_nid, "contains", line)
|
|
for child in node.children:
|
|
walk(child, parent_class_nid=class_nid)
|
|
return
|
|
|
|
if t == "class_property_definition" and parent_class_nid:
|
|
type_literal = next((c for c in node.children if c.type == "type_literal"), None)
|
|
type_name = _ps_type_name(type_literal)
|
|
if type_name:
|
|
line = node.start_point[0] + 1
|
|
target_nid = ensure_named_node(type_name, line)
|
|
if target_nid != parent_class_nid:
|
|
add_edge(parent_class_nid, target_nid, "references",
|
|
line, context="field")
|
|
return
|
|
|
|
if t == "class_method_definition":
|
|
name_node = next((c for c in node.children if c.type == "simple_name"), None)
|
|
if name_node:
|
|
method_name = _read_text(name_node, source)
|
|
line = node.start_point[0] + 1
|
|
if parent_class_nid:
|
|
method_nid = _make_id(parent_class_nid, method_name)
|
|
add_node(method_nid, f".{method_name}()", line)
|
|
add_edge(parent_class_nid, method_nid, "method", line)
|
|
else:
|
|
method_nid = _make_id(stem, method_name)
|
|
add_node(method_nid, f"{method_name}()", line)
|
|
add_edge(file_nid, method_nid, "contains", line)
|
|
# Return type: type_literal sibling of simple_name
|
|
return_type_literal = next(
|
|
(c for c in node.children if c.type == "type_literal"), None)
|
|
return_type_name = _ps_type_name(return_type_literal)
|
|
if return_type_name:
|
|
target_nid = ensure_named_node(return_type_name, line)
|
|
if target_nid != method_nid:
|
|
add_edge(method_nid, target_nid, "references",
|
|
line, context="return_type")
|
|
# Parameter types: class_method_parameter_list
|
|
param_list = next(
|
|
(c for c in node.children if c.type == "class_method_parameter_list"), None)
|
|
if param_list is not None:
|
|
for p in param_list.children:
|
|
if p.type != "class_method_parameter":
|
|
continue
|
|
ptype_literal = next(
|
|
(c for c in p.children if c.type == "type_literal"), None)
|
|
ptype_name = _ps_type_name(ptype_literal)
|
|
if not ptype_name:
|
|
continue
|
|
p_line = p.start_point[0] + 1
|
|
target_nid = ensure_named_node(ptype_name, p_line)
|
|
if target_nid != method_nid:
|
|
add_edge(method_nid, target_nid, "references",
|
|
p_line, context="parameter_type")
|
|
body = _find_script_block_body(node)
|
|
if body:
|
|
function_bodies.append((method_nid, body))
|
|
return
|
|
|
|
if t == "command":
|
|
cmd_name_node = next((c for c in node.children if c.type == "command_name"), None)
|
|
if cmd_name_node:
|
|
cmd_text = _read_text(cmd_name_node, source).lower()
|
|
if cmd_text == "using":
|
|
tokens = []
|
|
for child in node.children:
|
|
if child.type == "command_elements":
|
|
for el in child.children:
|
|
if el.type == "generic_token":
|
|
tokens.append(_read_text(el, source))
|
|
module_tokens = [t for t in tokens
|
|
if t.lower() not in ("namespace", "module", "assembly")]
|
|
if module_tokens:
|
|
module_name = module_tokens[-1].split(".")[-1]
|
|
add_edge(file_nid, _make_id(module_name), "imports_from",
|
|
node.start_point[0] + 1)
|
|
return
|
|
|
|
for child in node.children:
|
|
walk(child, parent_class_nid)
|
|
|
|
walk(root)
|
|
|
|
label_to_nid = {n["label"].strip("()").lstrip(".").lower(): n["id"] for n in nodes}
|
|
seen_call_pairs: set[tuple[str, str]] = set()
|
|
raw_calls: list[dict] = []
|
|
|
|
def walk_calls(node, caller_nid: str) -> None:
|
|
if node.type in ("function_statement", "class_statement"):
|
|
return
|
|
if node.type == "command":
|
|
cmd_name_node = next((c for c in node.children if c.type == "command_name"), None)
|
|
if cmd_name_node:
|
|
cmd_text = _read_text(cmd_name_node, source)
|
|
if cmd_text.lower() not in _PS_SKIP:
|
|
tgt_nid = label_to_nid.get(cmd_text.lower())
|
|
if tgt_nid and tgt_nid != caller_nid:
|
|
pair = (caller_nid, tgt_nid)
|
|
if pair not in seen_call_pairs:
|
|
seen_call_pairs.add(pair)
|
|
add_edge(caller_nid, tgt_nid, "calls",
|
|
node.start_point[0] + 1,
|
|
confidence="EXTRACTED", weight=1.0)
|
|
elif cmd_text:
|
|
raw_calls.append({
|
|
"caller_nid": caller_nid,
|
|
"callee": cmd_text,
|
|
"is_member_call": False,
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
})
|
|
for child in node.children:
|
|
walk_calls(child, caller_nid)
|
|
|
|
for caller_nid, body_node in function_bodies:
|
|
walk_calls(body_node, caller_nid)
|
|
|
|
clean_edges = [e for e in edges if e["source"] in seen_ids and
|
|
(e["target"] in seen_ids or e["relation"] == "imports_from")]
|
|
return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls}
|
|
|
|
|
|
# ── Cross-file import resolution ──────────────────────────────────────────────
|
|
|
|
def _source_key(source_file: str, root: Path) -> str:
|
|
if not source_file:
|
|
return ""
|
|
source_path = Path(source_file)
|
|
try:
|
|
return str(source_path.resolve().relative_to(root))
|
|
except Exception:
|
|
return str(source_path)
|
|
|
|
|
|
def _disambiguate_colliding_node_ids(
|
|
nodes: list[dict],
|
|
edges: list[dict],
|
|
raw_calls: list[dict],
|
|
root: Path,
|
|
) -> None:
|
|
"""Rewrite only colliding node IDs, using source path as the disambiguator."""
|
|
by_id: dict[str, list[dict]] = {}
|
|
for node in nodes:
|
|
nid = node.get("id")
|
|
if isinstance(nid, str) and nid:
|
|
by_id.setdefault(nid, []).append(node)
|
|
|
|
remap: dict[tuple[str, str], str] = {}
|
|
ambiguous_ids: set[str] = set()
|
|
for old_id, group in by_id.items():
|
|
source_keys = {_source_key(str(node.get("source_file", "")), root) for node in group}
|
|
if len(group) < 2 or len(source_keys) < 2:
|
|
continue
|
|
ambiguous_ids.add(old_id)
|
|
for node in group:
|
|
source_key = _source_key(str(node.get("source_file", "")), root)
|
|
if not source_key:
|
|
continue
|
|
new_id = _make_id(source_key, old_id)
|
|
remap[(old_id, source_key)] = new_id
|
|
if new_id != old_id:
|
|
node["id"] = new_id
|
|
|
|
if not remap:
|
|
return
|
|
|
|
unambiguous_remaps: dict[str, str] = {}
|
|
for old_id, group in by_id.items():
|
|
if old_id in ambiguous_ids:
|
|
continue
|
|
candidates = {
|
|
node["id"] for node in group
|
|
if isinstance(node.get("id"), str) and node["id"] != old_id
|
|
}
|
|
if len(candidates) == 1:
|
|
unambiguous_remaps[old_id] = next(iter(candidates))
|
|
|
|
for edge in edges:
|
|
edge_source_key = _source_key(str(edge.get("source_file", "")), root)
|
|
source_key = (edge.get("source", ""), edge_source_key)
|
|
target_key = (edge.get("target", ""), edge_source_key)
|
|
if source_key in remap:
|
|
edge["source"] = remap[source_key]
|
|
elif edge.get("source") in unambiguous_remaps:
|
|
edge["source"] = unambiguous_remaps[str(edge["source"])]
|
|
if target_key in remap:
|
|
edge["target"] = remap[target_key]
|
|
elif edge.get("target") in unambiguous_remaps:
|
|
edge["target"] = unambiguous_remaps[str(edge["target"])]
|
|
|
|
for raw_call in raw_calls:
|
|
call_source_key = _source_key(str(raw_call.get("source_file", "")), root)
|
|
caller_key = (raw_call.get("caller_nid", ""), call_source_key)
|
|
if caller_key in remap:
|
|
raw_call["caller_nid"] = remap[caller_key]
|
|
elif raw_call.get("caller_nid") in unambiguous_remaps:
|
|
raw_call["caller_nid"] = unambiguous_remaps[str(raw_call["caller_nid"])]
|
|
|
|
|
|
def _node_label_key(node: dict) -> str:
|
|
label = str(node.get("label", "")).strip()
|
|
return re.sub(r"[^a-zA-Z0-9]+", "", label).lower()
|
|
|
|
|
|
def _is_type_like_definition(node: dict) -> bool:
|
|
label = str(node.get("label", "")).strip()
|
|
if not label:
|
|
return False
|
|
if label.endswith(")") or label.startswith("."):
|
|
return False
|
|
if "." in label:
|
|
return False
|
|
return node.get("file_type") == "code"
|
|
|
|
|
|
def _rewire_unique_stub_nodes(nodes: list[dict], edges: list[dict]) -> None:
|
|
"""Map unresolved no-source stubs to a unique real definition with the same label."""
|
|
real_by_label: dict[str, list[dict]] = {}
|
|
stubs: list[dict] = []
|
|
|
|
for node in nodes:
|
|
key = _node_label_key(node)
|
|
if not key:
|
|
continue
|
|
if node.get("source_file"):
|
|
if _is_type_like_definition(node):
|
|
real_by_label.setdefault(key, []).append(node)
|
|
continue
|
|
stubs.append(node)
|
|
|
|
remap: dict[str, str] = {}
|
|
drop_ids: set[str] = set()
|
|
for stub in stubs:
|
|
stub_id = str(stub.get("id", ""))
|
|
if not stub_id:
|
|
continue
|
|
candidates = real_by_label.get(_node_label_key(stub), [])
|
|
if len(candidates) != 1:
|
|
continue
|
|
target_id = candidates[0].get("id")
|
|
if isinstance(target_id, str) and target_id and target_id != stub_id:
|
|
remap[stub_id] = target_id
|
|
drop_ids.add(stub_id)
|
|
|
|
if not remap:
|
|
return
|
|
|
|
for edge in edges:
|
|
if edge.get("source") in remap:
|
|
edge["source"] = remap[str(edge["source"])]
|
|
if edge.get("target") in remap:
|
|
edge["target"] = remap[str(edge["target"])]
|
|
|
|
nodes[:] = [node for node in nodes if node.get("id") not in drop_ids]
|
|
|
|
|
|
def _js_source_path(source_file: str, root: Path) -> Path | None:
|
|
if not source_file:
|
|
return None
|
|
path = Path(source_file)
|
|
if not path.is_absolute():
|
|
path = root / path
|
|
try:
|
|
return path.resolve()
|
|
except Exception:
|
|
return path
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _SymbolDeclarationFact:
|
|
file_path: Path
|
|
name: str
|
|
line: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _SymbolImportFact:
|
|
file_path: Path
|
|
local_name: str
|
|
target_path: Path
|
|
imported_name: str
|
|
line: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _SymbolAliasFact:
|
|
file_path: Path
|
|
alias: str
|
|
target_name: str
|
|
line: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _SymbolExportFact:
|
|
file_path: Path
|
|
exported_name: str
|
|
line: int
|
|
local_name: str | None = None
|
|
target_path: Path | None = None
|
|
target_name: str | None = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _StarExportFact:
|
|
file_path: Path
|
|
target_path: Path
|
|
line: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _SymbolUseFact:
|
|
file_path: Path
|
|
source_id: str
|
|
local_name: str
|
|
relation: str
|
|
context: str
|
|
line: int
|
|
|
|
|
|
@dataclass
|
|
class _SymbolResolutionFacts:
|
|
declarations: list[_SymbolDeclarationFact] = field(default_factory=list)
|
|
imports: list[_SymbolImportFact] = field(default_factory=list)
|
|
aliases: list[_SymbolAliasFact] = field(default_factory=list)
|
|
exports: list[_SymbolExportFact] = field(default_factory=list)
|
|
star_exports: list[_StarExportFact] = field(default_factory=list)
|
|
uses: list[_SymbolUseFact] = field(default_factory=list)
|
|
# File-to-file submodule imports from `from pkg import submod` (#1146).
|
|
# Each entry is (importing_file, submodule_file, line).
|
|
module_imports: list[tuple[Path, Path, int]] = field(default_factory=list)
|
|
|
|
|
|
def _apply_symbol_resolution_facts(
|
|
paths: list[Path],
|
|
nodes: list[dict],
|
|
edges: list[dict],
|
|
root: Path,
|
|
facts: _SymbolResolutionFacts,
|
|
) -> None:
|
|
"""Apply language-provided import/export/use facts to graph edges."""
|
|
if not (
|
|
facts.declarations
|
|
or facts.imports
|
|
or facts.aliases
|
|
or facts.exports
|
|
or facts.star_exports
|
|
or facts.uses
|
|
or facts.module_imports
|
|
):
|
|
return
|
|
|
|
path_by_resolved = {path.resolve(): path for path in paths}
|
|
source_file_id = {path.resolve(): _make_id(str(path)) for path in paths}
|
|
symbol_nodes: dict[tuple[Path, str], str] = {}
|
|
for node in nodes:
|
|
source_path = _js_source_path(str(node.get("source_file", "")), root)
|
|
if source_path is None:
|
|
continue
|
|
label = str(node.get("label", "")).strip().strip("()").lstrip(".")
|
|
if label and node.get("id"):
|
|
symbol_nodes[(source_path, label)] = str(node["id"])
|
|
|
|
def ensure_symbol_node(path: Path, name: str, line: int) -> str:
|
|
resolved_path = path.resolve()
|
|
existing = symbol_nodes.get((resolved_path, name))
|
|
if existing is not None:
|
|
return existing
|
|
node_id = _make_id(_file_stem(path), name)
|
|
symbol_nodes[(resolved_path, name)] = node_id
|
|
nodes.append({
|
|
"id": node_id,
|
|
"label": name,
|
|
"file_type": "code",
|
|
"source_file": str(path),
|
|
"source_location": f"L{line}",
|
|
})
|
|
return node_id
|
|
|
|
existing_edges = {
|
|
(
|
|
str(edge.get("source")),
|
|
str(edge.get("target")),
|
|
str(edge.get("relation")),
|
|
str(edge.get("context") or ""),
|
|
)
|
|
for edge in edges
|
|
}
|
|
|
|
def add_edge(source: str, target: str, relation: str, context: str, line: int, source_path: Path) -> None:
|
|
key = (source, target, relation, context or "")
|
|
if key in existing_edges:
|
|
return
|
|
existing_edges.add(key)
|
|
edges.append({
|
|
"source": source,
|
|
"target": target,
|
|
"relation": relation,
|
|
"context": context,
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str(source_path),
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
})
|
|
|
|
for declaration in facts.declarations:
|
|
ensure_symbol_node(declaration.file_path, declaration.name, declaration.line)
|
|
|
|
local_aliases_by_file: dict[Path, dict[str, tuple[Path, str]]] = {}
|
|
for import_fact in facts.imports:
|
|
file_path = import_fact.file_path.resolve()
|
|
local_aliases_by_file.setdefault(file_path, {})[import_fact.local_name] = (
|
|
import_fact.target_path.resolve(),
|
|
import_fact.imported_name,
|
|
)
|
|
|
|
pending_aliases_by_file: dict[Path, list[_SymbolAliasFact]] = {}
|
|
for alias_fact in facts.aliases:
|
|
pending_aliases_by_file.setdefault(alias_fact.file_path.resolve(), []).append(alias_fact)
|
|
|
|
for file_path, aliases in pending_aliases_by_file.items():
|
|
local_aliases = local_aliases_by_file.setdefault(file_path, {})
|
|
changed = True
|
|
while changed:
|
|
changed = False
|
|
for alias_fact in aliases:
|
|
if alias_fact.alias in local_aliases:
|
|
continue
|
|
origin = local_aliases.get(alias_fact.target_name)
|
|
if origin is not None:
|
|
local_aliases[alias_fact.alias] = origin
|
|
changed = True
|
|
|
|
named_exports_by_file: dict[Path, dict[str, tuple[Path, str]]] = {}
|
|
star_exports_by_file: dict[Path, list[Path]] = {}
|
|
|
|
for star_fact in facts.star_exports:
|
|
source_path = star_fact.file_path.resolve()
|
|
target_path = star_fact.target_path.resolve()
|
|
star_exports_by_file.setdefault(source_path, []).append(target_path)
|
|
source_id = source_file_id.get(source_path)
|
|
if source_id is not None:
|
|
add_edge(
|
|
source_id,
|
|
_make_id(str(path_by_resolved.get(target_path, target_path))),
|
|
"re_exports",
|
|
"export",
|
|
star_fact.line,
|
|
star_fact.file_path,
|
|
)
|
|
|
|
for export_fact in facts.exports:
|
|
file_path = export_fact.file_path.resolve()
|
|
origin: tuple[Path, str] | None = None
|
|
if export_fact.target_path is not None and export_fact.target_name is not None:
|
|
origin = (export_fact.target_path.resolve(), export_fact.target_name)
|
|
elif export_fact.local_name is not None:
|
|
origin = local_aliases_by_file.get(file_path, {}).get(export_fact.local_name)
|
|
if origin is None and (file_path, export_fact.local_name) in symbol_nodes:
|
|
origin = (file_path, export_fact.local_name)
|
|
if origin is None:
|
|
continue
|
|
named_exports_by_file.setdefault(file_path, {})[export_fact.exported_name] = origin
|
|
if origin[0] != file_path:
|
|
source_id = source_file_id.get(file_path)
|
|
if source_id is not None:
|
|
add_edge(
|
|
source_id,
|
|
_make_id(str(path_by_resolved.get(origin[0], origin[0]))),
|
|
"re_exports",
|
|
"export",
|
|
export_fact.line,
|
|
export_fact.file_path,
|
|
)
|
|
|
|
def resolve_exported_origin(target_path: Path, imported_name: str, seen: set[tuple[Path, str]] | None = None) -> tuple[Path, str]:
|
|
target_path = target_path.resolve()
|
|
key = (target_path, imported_name)
|
|
if seen is None:
|
|
seen = set()
|
|
if key in seen:
|
|
return key
|
|
seen.add(key)
|
|
origin = named_exports_by_file.get(target_path, {}).get(imported_name)
|
|
if origin is not None:
|
|
return resolve_exported_origin(origin[0], origin[1], seen)
|
|
for star_target in star_exports_by_file.get(target_path, []):
|
|
star_key = (star_target, imported_name)
|
|
if star_key in symbol_nodes:
|
|
return star_key
|
|
resolved = resolve_exported_origin(star_target, imported_name, seen)
|
|
if resolved in symbol_nodes:
|
|
return resolved
|
|
return key
|
|
|
|
for import_fact in facts.imports:
|
|
source_id = source_file_id.get(import_fact.file_path.resolve())
|
|
if source_id is None:
|
|
continue
|
|
origin_path, origin_symbol = resolve_exported_origin(
|
|
import_fact.target_path,
|
|
import_fact.imported_name,
|
|
)
|
|
target_id = symbol_nodes.get((origin_path, origin_symbol))
|
|
if target_id is None:
|
|
continue
|
|
add_edge(
|
|
source_id,
|
|
target_id,
|
|
"imports",
|
|
"import",
|
|
import_fact.line,
|
|
import_fact.file_path,
|
|
)
|
|
|
|
# #1146: emit file-to-file imports_from edges for package-form submodule imports.
|
|
for from_path, to_path, line in facts.module_imports:
|
|
try:
|
|
from_rel = from_path.relative_to(root)
|
|
to_rel = to_path.relative_to(root)
|
|
except ValueError:
|
|
continue
|
|
source_id = _make_id(_file_stem(from_rel))
|
|
target_id = _make_id(_file_stem(to_rel))
|
|
add_edge(source_id, target_id, "imports_from", "submodule_import", line, from_path)
|
|
|
|
for use_fact in facts.uses:
|
|
file_path = use_fact.file_path.resolve()
|
|
target_id = None
|
|
unresolved_origin = local_aliases_by_file.get(file_path, {}).get(use_fact.local_name)
|
|
if unresolved_origin is not None:
|
|
origin_path, origin_symbol = resolve_exported_origin(*unresolved_origin)
|
|
target_id = symbol_nodes.get((origin_path, origin_symbol))
|
|
if target_id is None and use_fact.relation in ("inherits", "implements"):
|
|
# Same-file fallback for HERITAGE only: a base declared in the same
|
|
# file (`class X extends Y`, `interface A extends B`) has no import
|
|
# alias, so resolve it directly against the file's own symbol nodes.
|
|
# Scoped to heritage because same-file calls/uses already resolve via
|
|
# the dedicated call-graph pass; widening this would duplicate those
|
|
# edges. Import resolution still takes precedence (#1095).
|
|
target_id = symbol_nodes.get((file_path, use_fact.local_name))
|
|
if target_id is None:
|
|
continue
|
|
add_edge(
|
|
use_fact.source_id,
|
|
target_id,
|
|
use_fact.relation,
|
|
use_fact.context,
|
|
use_fact.line,
|
|
use_fact.file_path,
|
|
)
|
|
|
|
|
|
def _parse_js_tree(path: Path):
|
|
try:
|
|
from tree_sitter import Language, Parser
|
|
if path.suffix in (".ts", ".tsx"):
|
|
import tree_sitter_typescript as tstypescript
|
|
language = Language(tstypescript.language_typescript())
|
|
else:
|
|
import tree_sitter_javascript as tsjavascript
|
|
language = Language(tsjavascript.language())
|
|
source = path.read_bytes()
|
|
parser = Parser(language)
|
|
return source, parser.parse(source).root_node
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _walk_js_tree(node):
|
|
yield node
|
|
for child in node.children:
|
|
yield from _walk_js_tree(child)
|
|
|
|
|
|
def _js_module_specifier(node, source: bytes) -> str | None:
|
|
source_node = node.child_by_field_name("source")
|
|
if source_node is None:
|
|
for child in node.children:
|
|
if child.type == "string":
|
|
source_node = child
|
|
break
|
|
if source_node is None:
|
|
return None
|
|
raw = _read_text(source_node, source).strip()
|
|
return raw.strip("'\"`") or None
|
|
|
|
|
|
def _js_named_specifiers(node, source: bytes, specifier_type: str) -> list[tuple[str, str]]:
|
|
pairs: list[tuple[str, str]] = []
|
|
for child in _walk_js_tree(node):
|
|
if child.type != specifier_type:
|
|
continue
|
|
name_node = child.child_by_field_name("name")
|
|
if name_node is None:
|
|
continue
|
|
alias_node = child.child_by_field_name("alias")
|
|
name = _read_text(name_node, source)
|
|
exposed = _read_text(alias_node, source) if alias_node is not None else name
|
|
if name and exposed:
|
|
pairs.append((name, exposed))
|
|
return pairs
|
|
|
|
|
|
def _js_export_clause(node):
|
|
for child in node.children:
|
|
if child.type == "export_clause":
|
|
return child
|
|
return None
|
|
|
|
|
|
def _js_export_statement_is_star(node) -> bool:
|
|
return any(child.type == "*" for child in node.children)
|
|
|
|
|
|
def _js_lexical_aliases(node, source: bytes) -> list[tuple[str, str]]:
|
|
aliases: list[tuple[str, str]] = []
|
|
if node.type != "lexical_declaration":
|
|
return aliases
|
|
for child in node.children:
|
|
if child.type != "variable_declarator":
|
|
continue
|
|
name_node = child.child_by_field_name("name")
|
|
value_node = child.child_by_field_name("value")
|
|
if (
|
|
name_node is not None
|
|
and value_node is not None
|
|
and value_node.type in ("identifier", "type_identifier")
|
|
):
|
|
aliases.append((_read_text(name_node, source), _read_text(value_node, source)))
|
|
return aliases
|
|
|
|
|
|
def _js_exported_declaration_names(node, source: bytes) -> list[str]:
|
|
names: list[str] = []
|
|
declaration = node.child_by_field_name("declaration")
|
|
if declaration is None:
|
|
return names
|
|
|
|
if declaration.type == "lexical_declaration":
|
|
names.extend(alias for alias, _target in _js_lexical_aliases(declaration, source))
|
|
return names
|
|
|
|
if declaration.type in (
|
|
"class_declaration",
|
|
"abstract_class_declaration",
|
|
"interface_declaration",
|
|
"type_alias_declaration",
|
|
"function_declaration",
|
|
):
|
|
name_node = declaration.child_by_field_name("name")
|
|
if name_node is not None:
|
|
names.append(_read_text(name_node, source))
|
|
return names
|
|
|
|
|
|
def _js_default_import_name(node, source: bytes) -> str | None:
|
|
"""Local binding of a default import: the `Foo` in `import Foo from './x'`.
|
|
|
|
The default binding is a bare identifier child of the import_clause (named
|
|
imports live in a `named_imports` node, namespace imports in a
|
|
`namespace_import` node), so it is also picked up from the mixed form
|
|
`import Foo, { Bar } from './x'`.
|
|
"""
|
|
for child in node.children:
|
|
if child.type == "import_clause":
|
|
for sub in child.children:
|
|
if sub.type == "identifier":
|
|
return _read_text(sub, source)
|
|
return None
|
|
|
|
|
|
def _js_default_export_name(node, source: bytes) -> str | None:
|
|
"""Local name of a default export, or None for anonymous defaults.
|
|
|
|
Handles `export default class Foo {}`, `export default function foo() {}`,
|
|
`export default abstract class Foo {}` (name on the `declaration` field) and
|
|
`export default Foo` (an identifier on the `value` field). Anonymous defaults
|
|
(`export default class {}`, `export default {...}`) have no resolvable symbol
|
|
and return None.
|
|
"""
|
|
if not any(child.type == "default" for child in node.children):
|
|
return None
|
|
declaration = node.child_by_field_name("declaration")
|
|
if declaration is not None:
|
|
name_node = declaration.child_by_field_name("name")
|
|
return _read_text(name_node, source) if name_node is not None else None
|
|
value = node.child_by_field_name("value")
|
|
if value is not None and value.type == "identifier":
|
|
return _read_text(value, source)
|
|
return None
|
|
|
|
|
|
def _js_top_level_function_bodies(path: Path, root_node, source: bytes) -> list[tuple[str, object]]:
|
|
bodies: list[tuple[str, object]] = []
|
|
stem = _file_stem(path)
|
|
for node in root_node.children:
|
|
if node.type == "function_declaration":
|
|
name_node = node.child_by_field_name("name")
|
|
body = node.child_by_field_name("body")
|
|
if name_node is not None and body is not None:
|
|
bodies.append((_make_id(stem, _read_text(name_node, source)), body))
|
|
continue
|
|
if node.type != "lexical_declaration":
|
|
continue
|
|
for child in node.children:
|
|
if child.type != "variable_declarator":
|
|
continue
|
|
name_node = child.child_by_field_name("name")
|
|
value_node = child.child_by_field_name("value")
|
|
if (
|
|
name_node is not None
|
|
and value_node is not None
|
|
and value_node.type == "arrow_function"
|
|
):
|
|
bodies.append((_make_id(stem, _read_text(name_node, source)), value_node))
|
|
return bodies
|
|
|
|
|
|
def _js_call_identifier(node, source: bytes) -> str | None:
|
|
if node.type != "call_expression":
|
|
return None
|
|
function_node = node.child_by_field_name("function")
|
|
if function_node is None:
|
|
for child in node.children:
|
|
if child.is_named:
|
|
function_node = child
|
|
break
|
|
if function_node is not None and function_node.type in ("identifier", "type_identifier"):
|
|
return _read_text(function_node, source)
|
|
return None
|
|
|
|
|
|
_JS_PRIMITIVE_TYPES = frozenset({
|
|
"string", "number", "boolean", "any", "unknown", "void", "never",
|
|
"object", "null", "undefined", "bigint", "symbol", "this",
|
|
})
|
|
|
|
|
|
def _ts_heritage_clause_entries(clause_node, source: bytes) -> list[str]:
|
|
"""Return base/interface type names from an extends_clause or implements_clause."""
|
|
out: list[str] = []
|
|
for child in clause_node.children:
|
|
if not child.is_named:
|
|
continue
|
|
if child.type in ("identifier", "type_identifier"):
|
|
name = _read_text(child, source)
|
|
if name:
|
|
out.append(name)
|
|
elif child.type == "generic_type":
|
|
name_node = child.child_by_field_name("name")
|
|
if name_node is None:
|
|
for sub in child.children:
|
|
if sub.type in ("type_identifier", "nested_type_identifier", "identifier"):
|
|
name_node = sub
|
|
break
|
|
if name_node is not None:
|
|
text = _read_text(name_node, source).rsplit(".", 1)[-1]
|
|
if text:
|
|
out.append(text)
|
|
elif child.type == "nested_type_identifier":
|
|
text = _read_text(child, source).rsplit(".", 1)[-1]
|
|
if text:
|
|
out.append(text)
|
|
return out
|
|
|
|
|
|
def _ts_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None:
|
|
"""Walk a TS type annotation tree; append (name, role) tuples.
|
|
|
|
role is 'type' for the outermost type position and 'generic_arg' for entries
|
|
that appear inside `type_arguments`.
|
|
"""
|
|
if node is None:
|
|
return
|
|
t = node.type
|
|
if t == "type_annotation":
|
|
for c in node.children:
|
|
if c.is_named:
|
|
_ts_collect_type_refs(c, source, generic, out)
|
|
return
|
|
if t in ("type_identifier", "identifier"):
|
|
name = _read_text(node, source)
|
|
if name and name not in _JS_PRIMITIVE_TYPES:
|
|
out.append((name, "generic_arg" if generic else "type"))
|
|
return
|
|
if t == "nested_type_identifier":
|
|
tail = _read_text(node, source).rsplit(".", 1)[-1]
|
|
if tail and tail not in _JS_PRIMITIVE_TYPES:
|
|
out.append((tail, "generic_arg" if generic else "type"))
|
|
return
|
|
if t == "generic_type":
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node is not None:
|
|
text = _read_text(name_node, source).rsplit(".", 1)[-1]
|
|
if text and text not in _JS_PRIMITIVE_TYPES:
|
|
out.append((text, "generic_arg" if generic else "type"))
|
|
else:
|
|
for c in node.children:
|
|
if c.type in ("type_identifier", "nested_type_identifier"):
|
|
text = _read_text(c, source).rsplit(".", 1)[-1]
|
|
if text and text not in _JS_PRIMITIVE_TYPES:
|
|
out.append((text, "generic_arg" if generic else "type"))
|
|
break
|
|
for c in node.children:
|
|
if c.type == "type_arguments":
|
|
for sub in c.children:
|
|
if sub.is_named:
|
|
_ts_collect_type_refs(sub, source, True, out)
|
|
return
|
|
if node.is_named:
|
|
for c in node.children:
|
|
if c.is_named:
|
|
_ts_collect_type_refs(c, source, generic, out)
|
|
|
|
|
|
def _ts_walk_class_members(class_node, source: bytes, path: Path, class_nid: str,
|
|
facts: _SymbolResolutionFacts) -> None:
|
|
"""Emit type-relation and type-reference use facts for a class declaration node."""
|
|
line = class_node.start_point[0] + 1
|
|
for child in class_node.children:
|
|
if child.type == "class_heritage":
|
|
for clause in child.children:
|
|
if clause.type == "extends_clause":
|
|
for name in _ts_heritage_clause_entries(clause, source):
|
|
facts.uses.append(
|
|
_SymbolUseFact(path, class_nid, name, "inherits", "type",
|
|
clause.start_point[0] + 1)
|
|
)
|
|
elif clause.type == "implements_clause":
|
|
for name in _ts_heritage_clause_entries(clause, source):
|
|
facts.uses.append(
|
|
_SymbolUseFact(path, class_nid, name, "implements", "type",
|
|
clause.start_point[0] + 1)
|
|
)
|
|
elif child.type == "extends_type_clause":
|
|
# Interface heritage (`interface A extends B, C`) is an
|
|
# extends_type_clause node, NOT a class_heritage. Its base entries
|
|
# are the same node types extends_clause holds, so the helper is
|
|
# reusable. Without this branch interface inheritance is dropped (#1095).
|
|
for name in _ts_heritage_clause_entries(child, source):
|
|
facts.uses.append(
|
|
_SymbolUseFact(path, class_nid, name, "inherits", "type",
|
|
child.start_point[0] + 1)
|
|
)
|
|
|
|
body = class_node.child_by_field_name("body")
|
|
if body is None:
|
|
return
|
|
|
|
for member in body.children:
|
|
m_line = member.start_point[0] + 1
|
|
if member.type in ("method_definition", "method_signature", "abstract_method_signature"):
|
|
name_node = member.child_by_field_name("name")
|
|
if name_node is None:
|
|
continue
|
|
method_name = _read_text(name_node, source)
|
|
method_nid = _make_id(class_nid, method_name)
|
|
params = member.child_by_field_name("parameters")
|
|
if params is not None:
|
|
for p in params.children:
|
|
if p.type not in ("required_parameter", "optional_parameter"):
|
|
continue
|
|
type_anno = p.child_by_field_name("type")
|
|
if type_anno is None:
|
|
continue
|
|
refs: list[tuple[str, str]] = []
|
|
_ts_collect_type_refs(type_anno, source, False, refs)
|
|
for name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "parameter_type"
|
|
facts.uses.append(
|
|
_SymbolUseFact(path, method_nid, name, "references", ctx, m_line)
|
|
)
|
|
return_type = member.child_by_field_name("return_type")
|
|
if return_type is not None:
|
|
refs = []
|
|
_ts_collect_type_refs(return_type, source, False, refs)
|
|
for name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "return_type"
|
|
facts.uses.append(
|
|
_SymbolUseFact(path, method_nid, name, "references", ctx, m_line)
|
|
)
|
|
elif member.type in ("public_field_definition", "property_signature"):
|
|
type_anno = None
|
|
for c in member.children:
|
|
if c.type == "type_annotation":
|
|
type_anno = c
|
|
break
|
|
if type_anno is None:
|
|
continue
|
|
refs = []
|
|
_ts_collect_type_refs(type_anno, source, False, refs)
|
|
for name, role in refs:
|
|
ctx = "generic_arg" if role == "generic_arg" else "field"
|
|
facts.uses.append(
|
|
_SymbolUseFact(path, class_nid, name, "references", ctx, m_line)
|
|
)
|
|
|
|
|
|
def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolutionFacts) -> None:
|
|
js_paths = [
|
|
path for path in paths
|
|
if path.suffix in _JS_CACHE_BYPASS_SUFFIXES and path.suffix != ".vue"
|
|
]
|
|
if not js_paths:
|
|
return
|
|
|
|
trees: dict[Path, tuple[bytes, object]] = {}
|
|
|
|
for path in js_paths:
|
|
resolved_path = path.resolve()
|
|
parsed = _parse_js_tree(path)
|
|
if parsed is None:
|
|
continue
|
|
source, root_node = parsed
|
|
trees[resolved_path] = parsed
|
|
|
|
for node in _walk_js_tree(root_node):
|
|
if node.type == "export_statement":
|
|
for name in _js_exported_declaration_names(node, source):
|
|
facts.declarations.append(
|
|
_SymbolDeclarationFact(path, name, node.start_point[0] + 1)
|
|
)
|
|
|
|
if node.type != "import_statement":
|
|
continue
|
|
raw_module = _js_module_specifier(node, source)
|
|
if raw_module is None:
|
|
continue
|
|
target_path = _resolve_js_module_path(raw_module, path.parent)
|
|
if target_path is None:
|
|
continue
|
|
target_path = target_path.resolve()
|
|
for imported_name, local_name in _js_named_specifiers(node, source, "import_specifier"):
|
|
facts.imports.append(
|
|
_SymbolImportFact(
|
|
path,
|
|
local_name,
|
|
target_path,
|
|
imported_name,
|
|
node.start_point[0] + 1,
|
|
)
|
|
)
|
|
default_local = _js_default_import_name(node, source)
|
|
if default_local is not None:
|
|
facts.imports.append(
|
|
_SymbolImportFact(
|
|
path,
|
|
default_local,
|
|
target_path,
|
|
"default",
|
|
node.start_point[0] + 1,
|
|
)
|
|
)
|
|
|
|
for node in _walk_js_tree(root_node):
|
|
for alias, target in _js_lexical_aliases(node, source):
|
|
facts.aliases.append(
|
|
_SymbolAliasFact(path, alias, target, node.start_point[0] + 1)
|
|
)
|
|
|
|
for path in js_paths:
|
|
resolved_path = path.resolve()
|
|
parsed = trees.get(resolved_path)
|
|
if parsed is None:
|
|
continue
|
|
source, root_node = parsed
|
|
|
|
for node in _walk_js_tree(root_node):
|
|
if node.type != "export_statement":
|
|
continue
|
|
|
|
raw_module = _js_module_specifier(node, source)
|
|
export_clause = _js_export_clause(node)
|
|
if raw_module is not None:
|
|
target_path = _resolve_js_module_path(raw_module, path.parent)
|
|
if target_path is None:
|
|
continue
|
|
target_path = target_path.resolve()
|
|
if _js_export_statement_is_star(node):
|
|
facts.star_exports.append(
|
|
_StarExportFact(path, target_path, node.start_point[0] + 1)
|
|
)
|
|
if export_clause is not None:
|
|
for original_name, exported_name in _js_named_specifiers(
|
|
export_clause, source, "export_specifier"
|
|
):
|
|
facts.exports.append(
|
|
_SymbolExportFact(
|
|
path,
|
|
exported_name,
|
|
node.start_point[0] + 1,
|
|
target_path=target_path,
|
|
target_name=original_name,
|
|
)
|
|
)
|
|
continue
|
|
|
|
if export_clause is not None:
|
|
for local_name, exported_name in _js_named_specifiers(
|
|
export_clause, source, "export_specifier"
|
|
):
|
|
facts.exports.append(
|
|
_SymbolExportFact(
|
|
path,
|
|
exported_name,
|
|
node.start_point[0] + 1,
|
|
local_name=local_name,
|
|
)
|
|
)
|
|
continue
|
|
|
|
for exported_name in _js_exported_declaration_names(node, source):
|
|
facts.exports.append(
|
|
_SymbolExportFact(
|
|
path,
|
|
exported_name,
|
|
node.start_point[0] + 1,
|
|
local_name=exported_name,
|
|
)
|
|
)
|
|
|
|
# `export default class Foo {}` / `export default foo` exposes the
|
|
# symbol under the name "default"; record that so a default import
|
|
# (imported_name="default") resolves to it. `export { X as default }`
|
|
# is already handled via the export_clause path above.
|
|
default_name = _js_default_export_name(node, source)
|
|
if default_name is not None:
|
|
facts.exports.append(
|
|
_SymbolExportFact(
|
|
path,
|
|
"default",
|
|
node.start_point[0] + 1,
|
|
local_name=default_name,
|
|
)
|
|
)
|
|
|
|
for path in js_paths:
|
|
resolved_path = path.resolve()
|
|
parsed = trees.get(resolved_path)
|
|
if parsed is None:
|
|
continue
|
|
source, root_node = parsed
|
|
for source_id, body in _js_top_level_function_bodies(path, root_node, source):
|
|
for node in _walk_js_tree(body):
|
|
imported_name = _js_call_identifier(node, source)
|
|
if imported_name is None:
|
|
continue
|
|
facts.uses.append(
|
|
_SymbolUseFact(
|
|
path,
|
|
source_id,
|
|
imported_name,
|
|
"calls",
|
|
"call",
|
|
node.start_point[0] + 1,
|
|
)
|
|
)
|
|
|
|
for path in js_paths:
|
|
resolved_path = path.resolve()
|
|
parsed = trees.get(resolved_path)
|
|
if parsed is None:
|
|
continue
|
|
source, root_node = parsed
|
|
stem = _file_stem(path)
|
|
for node in _walk_js_tree(root_node):
|
|
if node.type not in (
|
|
"class_declaration",
|
|
"abstract_class_declaration",
|
|
"interface_declaration",
|
|
):
|
|
continue
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node is None:
|
|
continue
|
|
class_name = _read_text(name_node, source)
|
|
if not class_name:
|
|
continue
|
|
class_nid = _make_id(stem, class_name)
|
|
_ts_walk_class_members(node, source, path, class_nid, facts)
|
|
|
|
|
|
def _parse_python_tree(path: Path):
|
|
try:
|
|
import tree_sitter_python as tspython
|
|
from tree_sitter import Language, Parser
|
|
source = path.read_bytes()
|
|
parser = Parser(Language(tspython.language()))
|
|
return source, parser.parse(source).root_node
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _walk_python_tree(node):
|
|
yield node
|
|
for child in node.children:
|
|
yield from _walk_python_tree(child)
|
|
|
|
|
|
def _python_import_from_module(node, source: bytes) -> tuple[int, str] | None:
|
|
level = 0
|
|
module_name = ""
|
|
for child in node.children:
|
|
if child.type == "import":
|
|
break
|
|
if child.type == "relative_import":
|
|
raw = _read_text(child, source)
|
|
level = len(raw) - len(raw.lstrip("."))
|
|
remainder = raw.lstrip(".")
|
|
if remainder:
|
|
module_name = remainder
|
|
for sub in child.children:
|
|
if sub.type == "dotted_name":
|
|
module_name = _read_text(sub, source)
|
|
elif child.type == "dotted_name":
|
|
module_name = _read_text(child, source)
|
|
if level == 0 and not module_name:
|
|
return None
|
|
return level, module_name
|
|
|
|
|
|
def _python_imported_names(node, source: bytes) -> list[tuple[str, str]]:
|
|
names: list[tuple[str, str]] = []
|
|
past_import = False
|
|
for child in node.children:
|
|
if child.type == "import":
|
|
past_import = True
|
|
continue
|
|
if not past_import:
|
|
continue
|
|
if child.type == "dotted_name":
|
|
name = _read_text(child, source)
|
|
names.append((name, name.split(".")[-1]))
|
|
elif child.type == "aliased_import":
|
|
name_node = child.child_by_field_name("name")
|
|
alias_node = child.child_by_field_name("alias")
|
|
if name_node is None:
|
|
continue
|
|
name = _read_text(name_node, source)
|
|
local = _read_text(alias_node, source) if alias_node is not None else name.split(".")[-1]
|
|
names.append((name, local))
|
|
return names
|
|
|
|
|
|
def _resolve_python_module_path(module_name: str, current_path: Path, root: Path, level: int) -> Path | None:
|
|
if level > 0:
|
|
base = current_path.parent
|
|
for _ in range(level - 1):
|
|
base = base.parent
|
|
candidate = base / module_name.replace(".", "/") if module_name else base
|
|
else:
|
|
candidate = root / module_name.replace(".", "/")
|
|
|
|
if candidate.is_dir():
|
|
init_path = candidate / "__init__.py"
|
|
if init_path.is_file():
|
|
return init_path
|
|
if candidate.is_file():
|
|
return candidate
|
|
py_candidate = candidate.with_suffix(".py")
|
|
if py_candidate.is_file():
|
|
return py_candidate
|
|
return None
|
|
|
|
|
|
def _python_top_level_function_bodies(path: Path, root_node, source: bytes) -> list[tuple[str, object]]:
|
|
bodies: list[tuple[str, object]] = []
|
|
stem = _file_stem(path)
|
|
for node in root_node.children:
|
|
if node.type != "function_definition":
|
|
continue
|
|
name_node = node.child_by_field_name("name")
|
|
body = node.child_by_field_name("body")
|
|
if name_node is not None and body is not None:
|
|
bodies.append((_make_id(stem, _read_text(name_node, source)), body))
|
|
return bodies
|
|
|
|
|
|
def _python_call_identifier(node, source: bytes) -> str | None:
|
|
if node.type != "call":
|
|
return None
|
|
function_node = node.child_by_field_name("function")
|
|
if function_node is not None and function_node.type == "identifier":
|
|
return _read_text(function_node, source)
|
|
return None
|
|
|
|
|
|
def _collect_python_symbol_resolution_facts(
|
|
paths: list[Path],
|
|
root: Path,
|
|
facts: _SymbolResolutionFacts,
|
|
) -> None:
|
|
py_paths = [path for path in paths if path.suffix == ".py"]
|
|
if not py_paths:
|
|
return
|
|
|
|
trees: dict[Path, tuple[bytes, object]] = {}
|
|
for path in py_paths:
|
|
parsed = _parse_python_tree(path)
|
|
if parsed is None:
|
|
continue
|
|
source, root_node = parsed
|
|
trees[path.resolve()] = parsed
|
|
|
|
for node in _walk_python_tree(root_node):
|
|
if node.type != "import_from_statement":
|
|
continue
|
|
module = _python_import_from_module(node, source)
|
|
if module is None:
|
|
continue
|
|
level, module_name = module
|
|
target_path = _resolve_python_module_path(module_name, path, root, level)
|
|
if target_path is None:
|
|
continue
|
|
# #1146: `from pkg import submod` — if the target is a package
|
|
# (__init__.py) and an imported name matches a submodule file on
|
|
# disk, emit a file-level import edge to that submodule rather
|
|
# than only to the package.
|
|
pkg_dir = target_path.parent if target_path.name == "__init__.py" else None
|
|
for imported_name, local_name in _python_imported_names(node, source):
|
|
line = node.start_point[0] + 1
|
|
if pkg_dir is not None:
|
|
sub_py = pkg_dir / f"{imported_name}.py"
|
|
sub_pkg = pkg_dir / imported_name / "__init__.py"
|
|
submodule = sub_py if sub_py.is_file() else (sub_pkg if sub_pkg.is_file() else None)
|
|
if submodule is not None:
|
|
facts.module_imports.append((path, submodule, line))
|
|
continue
|
|
facts.imports.append(
|
|
_SymbolImportFact(path, local_name, target_path, imported_name, line)
|
|
)
|
|
if path.name == "__init__.py":
|
|
facts.exports.append(
|
|
_SymbolExportFact(
|
|
path,
|
|
local_name,
|
|
line,
|
|
target_path=target_path,
|
|
target_name=imported_name,
|
|
)
|
|
)
|
|
|
|
for path in py_paths:
|
|
parsed = trees.get(path.resolve())
|
|
if parsed is None:
|
|
continue
|
|
source, root_node = parsed
|
|
for source_id, body in _python_top_level_function_bodies(path, root_node, source):
|
|
for node in _walk_python_tree(body):
|
|
imported_name = _python_call_identifier(node, source)
|
|
if imported_name is None:
|
|
continue
|
|
facts.uses.append(
|
|
_SymbolUseFact(
|
|
path,
|
|
source_id,
|
|
imported_name,
|
|
"calls",
|
|
"call",
|
|
node.start_point[0] + 1,
|
|
)
|
|
)
|
|
|
|
|
|
def _augment_symbol_resolution_edges(
|
|
paths: list[Path],
|
|
nodes: list[dict],
|
|
edges: list[dict],
|
|
root: Path,
|
|
) -> None:
|
|
facts = _SymbolResolutionFacts()
|
|
_collect_js_symbol_resolution_facts(paths, facts)
|
|
_collect_python_symbol_resolution_facts(paths, root, facts)
|
|
_apply_symbol_resolution_facts(paths, nodes, edges, root, facts)
|
|
|
|
|
|
def _augment_js_reexport_edges(
|
|
paths: list[Path],
|
|
nodes: list[dict],
|
|
edges: list[dict],
|
|
root: Path,
|
|
) -> None:
|
|
"""Compatibility wrapper for the JS/TS symbol-resolution post-pass."""
|
|
facts = _SymbolResolutionFacts()
|
|
_collect_js_symbol_resolution_facts(paths, facts)
|
|
_apply_symbol_resolution_facts(paths, nodes, edges, root, facts)
|
|
|
|
|
|
def _resolve_cross_file_imports(
|
|
per_file: list[dict],
|
|
paths: list[Path],
|
|
) -> list[dict]:
|
|
"""
|
|
Two-pass import resolution: turn file-level imports into class-level edges.
|
|
|
|
Pass 1 - build a global map: class/function name → node_id, per stem.
|
|
Pass 2 - for each `from .module import Name`, look up Name in the global
|
|
map and add a direct INFERRED edge from each class in the
|
|
importing file to the imported entity.
|
|
|
|
This turns:
|
|
auth.py --imports_from--> models.py (obvious, filtered out)
|
|
Into:
|
|
DigestAuth --uses--> Response [INFERRED] (cross-file, interesting!)
|
|
BasicAuth --uses--> Request [INFERRED]
|
|
"""
|
|
try:
|
|
import tree_sitter_python as tspython
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return []
|
|
|
|
language = Language(tspython.language())
|
|
parser = Parser(language)
|
|
|
|
# Pass 1: _file_stem(path) → {ClassName: node_id}
|
|
# Keyed by directory-qualified stem (e.g. "auth_models") to avoid collisions
|
|
# when multiple files share the same filename in different directories.
|
|
# A secondary bare-stem index handles absolute imports where only the module
|
|
# name is known — first writer wins when names collide (inherently ambiguous).
|
|
stem_to_entities: dict[str, dict[str, str]] = {}
|
|
bare_to_qualified: dict[str, str] = {}
|
|
for file_result in per_file:
|
|
for node in file_result.get("nodes", []):
|
|
src = node.get("source_file", "")
|
|
if not src:
|
|
continue
|
|
src_path = Path(src)
|
|
fq_stem = _file_stem(src_path)
|
|
label = node.get("label", "")
|
|
nid = node.get("id", "")
|
|
# Index class-level entities only. Function/method labels end in "()"
|
|
# so are excluded by the `endswith(")")` filter; file nodes end in ".py";
|
|
# private/internal labels start with "_"; rationale nodes carry
|
|
# file_type=="rationale" and must never participate in cross-file
|
|
# import resolution (#563).
|
|
if (
|
|
label
|
|
and not label.endswith((")", ".py"))
|
|
and "_" not in label[:1]
|
|
and node.get("file_type") != "rationale"
|
|
):
|
|
stem_to_entities.setdefault(fq_stem, {})[label] = nid
|
|
if src_path.stem not in bare_to_qualified:
|
|
bare_to_qualified[src_path.stem] = fq_stem
|
|
|
|
# Pass 2: for each file, find `from .X import A, B, C` and resolve
|
|
new_edges: list[dict] = []
|
|
stem_to_path: dict[str, Path] = {_file_stem(p): p for p in paths}
|
|
|
|
for file_result, path in zip(per_file, paths):
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
|
|
# Find all classes defined in this file (the importers).
|
|
# Excludes rationale nodes whose labels happen not to end in ")" or ".py"
|
|
# but which must never be treated as importing entities (#563).
|
|
local_classes = [
|
|
n["id"] for n in file_result.get("nodes", [])
|
|
if n.get("source_file") == str_path
|
|
and not n["label"].endswith((")", ".py"))
|
|
and n["id"] != _make_id(stem) # exclude file-level node
|
|
and n.get("file_type") != "rationale"
|
|
]
|
|
if not local_classes:
|
|
continue
|
|
|
|
# Parse imports from this file
|
|
try:
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
except Exception:
|
|
continue
|
|
|
|
def walk_imports(node) -> None:
|
|
if node.type == "import_from_statement":
|
|
# Find the module name - handles both absolute and relative imports.
|
|
# Relative: `from .models import X` → relative_import → dotted_name
|
|
# Absolute: `from models import X` → module_name field
|
|
# target_fq is the directory-qualified stem used as the key in
|
|
# stem_to_entities. Relative imports are resolved exactly via the
|
|
# importing file's directory; absolute imports fall back to the
|
|
# bare-stem secondary index (first-writer-wins when names collide).
|
|
target_fq: str | None = None
|
|
for child in node.children:
|
|
if child.type == "relative_import":
|
|
for sub in child.children:
|
|
if sub.type == "dotted_name":
|
|
raw = source[sub.start_byte:sub.end_byte].decode("utf-8", errors="replace")
|
|
bare = raw.split(".")[-1]
|
|
# Resolve relative import to exact qualified stem.
|
|
candidate = path.parent / f"{bare}.py"
|
|
target_fq = _file_stem(candidate)
|
|
break
|
|
break
|
|
if child.type == "dotted_name" and target_fq is None:
|
|
raw = source[child.start_byte:child.end_byte].decode("utf-8", errors="replace")
|
|
bare = raw.split(".")[-1]
|
|
target_fq = bare_to_qualified.get(bare)
|
|
|
|
if not target_fq or target_fq not in stem_to_entities:
|
|
return
|
|
|
|
# Collect imported names: dotted_name children of import_from_statement
|
|
# that come AFTER the 'import' keyword token.
|
|
imported_names: list[str] = []
|
|
past_import_kw = False
|
|
for child in node.children:
|
|
if child.type == "import":
|
|
past_import_kw = True
|
|
continue
|
|
if not past_import_kw:
|
|
continue
|
|
if child.type == "dotted_name":
|
|
imported_names.append(
|
|
source[child.start_byte:child.end_byte].decode("utf-8", errors="replace")
|
|
)
|
|
elif child.type == "aliased_import":
|
|
# `import X as Y` - take the original name
|
|
name_node = child.child_by_field_name("name")
|
|
if name_node:
|
|
imported_names.append(
|
|
source[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="replace")
|
|
)
|
|
|
|
line = node.start_point[0] + 1
|
|
for name in imported_names:
|
|
tgt_nid = stem_to_entities[target_fq].get(name)
|
|
if tgt_nid:
|
|
for src_class_nid in local_classes:
|
|
new_edges.append({
|
|
"source": src_class_nid,
|
|
"target": tgt_nid,
|
|
"relation": "uses",
|
|
"confidence": "INFERRED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 0.8,
|
|
})
|
|
for child in node.children:
|
|
walk_imports(child)
|
|
|
|
walk_imports(tree.root_node)
|
|
|
|
return new_edges
|
|
|
|
|
|
def _merge_swift_extensions(
|
|
per_file: list[dict],
|
|
all_nodes: list[dict],
|
|
all_edges: list[dict],
|
|
) -> None:
|
|
"""Collapse cross-file Swift `extension Foo` nodes into the canonical `Foo`.
|
|
|
|
tree-sitter-swift reuses `class_declaration` for both `class Foo` and
|
|
`extension Foo`, and node ids carry the file stem, so each file that
|
|
extends `Foo` produces its own `Foo` node. The match is done by label:
|
|
when exactly one non-extension declaration shares the label, extension
|
|
nodes redirect onto it. Extensions of types outside the corpus (no match)
|
|
and ambiguous labels (more than one match) are left untouched — picking
|
|
arbitrarily would invent edges.
|
|
"""
|
|
extension_nids: set[str] = set()
|
|
extension_labels: dict[str, str] = {}
|
|
for result in per_file:
|
|
for ext in result.get("swift_extensions", []) or []:
|
|
extension_nids.add(ext["nid"])
|
|
extension_labels[ext["nid"]] = ext["label"]
|
|
|
|
if not extension_nids:
|
|
return
|
|
|
|
label_to_canonical: dict[str, list[str]] = {}
|
|
for n in all_nodes:
|
|
if n.get("id") in extension_nids:
|
|
continue
|
|
label = n.get("label")
|
|
if not label:
|
|
continue
|
|
label_to_canonical.setdefault(label, []).append(n["id"])
|
|
|
|
remap: dict[str, str] = {}
|
|
for ext_nid in extension_nids:
|
|
candidates = label_to_canonical.get(extension_labels[ext_nid], [])
|
|
if len(candidates) != 1:
|
|
continue
|
|
canonical_nid = candidates[0]
|
|
if canonical_nid != ext_nid:
|
|
remap[ext_nid] = canonical_nid
|
|
|
|
if not remap:
|
|
return
|
|
|
|
all_nodes[:] = [n for n in all_nodes if n.get("id") not in remap]
|
|
|
|
# Each extension file's `contains` edge ends up pointing at the canonical
|
|
# type — multiple files containing the same node is the intended shape:
|
|
# the type owns the methods, the files own their slice. Self-loops are
|
|
# dropped (e.g. an in-file extension method whose call already pointed at
|
|
# the canonical type).
|
|
rewritten: list[dict] = []
|
|
seen_keys: set[tuple] = set()
|
|
for e in all_edges:
|
|
src = remap.get(e.get("source"), e.get("source"))
|
|
tgt = remap.get(e.get("target"), e.get("target"))
|
|
if src == tgt:
|
|
continue
|
|
e["source"] = src
|
|
e["target"] = tgt
|
|
key = (src, tgt, e.get("relation"), e.get("source_file"), e.get("source_location"))
|
|
if key in seen_keys:
|
|
continue
|
|
seen_keys.add(key)
|
|
rewritten.append(e)
|
|
all_edges[:] = rewritten
|
|
|
|
|
|
def _resolve_cross_file_java_imports(
|
|
per_file: list[dict],
|
|
paths: list[Path],
|
|
) -> list[dict]:
|
|
"""Two-pass Java import resolution.
|
|
|
|
Pass 1: build a global index {ClassName: [node_id, ...]} across all Java nodes.
|
|
Pass 2: re-parse each Java file; for every `import a.b.C;`, resolve C against
|
|
the index. Wildcard and stdlib imports produce no edge.
|
|
"""
|
|
try:
|
|
import tree_sitter_java as tsjava
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return []
|
|
|
|
language = Language(tsjava.language())
|
|
parser = Parser(language)
|
|
|
|
# Pass 1: class-name → node_id index (only internal, uppercase-starting names)
|
|
name_to_ids: dict[str, list[str]] = {}
|
|
for file_result in per_file:
|
|
for node in file_result.get("nodes", []):
|
|
label = node.get("label", "")
|
|
nid = node.get("id", "")
|
|
src = node.get("source_file", "")
|
|
if not label or not nid or not src:
|
|
continue
|
|
if label.endswith(")") or label.endswith(".java"):
|
|
continue
|
|
if not label[0].isalpha() or not label[0].isupper():
|
|
continue
|
|
name_to_ids.setdefault(label, []).append(nid)
|
|
|
|
# Pass 2: resolve imports to real node IDs
|
|
new_edges: list[dict] = []
|
|
seen_pairs: set[tuple[str, str]] = set()
|
|
for path in paths:
|
|
file_nid = _make_id(str(path))
|
|
try:
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
except Exception:
|
|
continue
|
|
|
|
def walk(n) -> None:
|
|
if n.type == "import_declaration":
|
|
raw = _read_text(n, source).strip()
|
|
body = raw[len("import"):].strip().rstrip(";").strip()
|
|
if body.startswith("static "):
|
|
body = body[len("static "):].strip()
|
|
if body.endswith(".*"):
|
|
return
|
|
parts = body.split(".")
|
|
if not parts:
|
|
return
|
|
last = parts[-1]
|
|
if last and last[0].islower() and len(parts) >= 2:
|
|
last = parts[-2]
|
|
at_line = n.start_point[0] + 1
|
|
for tgt_nid in name_to_ids.get(last, []):
|
|
if tgt_nid == file_nid:
|
|
continue
|
|
key = (file_nid, tgt_nid)
|
|
if key in seen_pairs:
|
|
continue
|
|
seen_pairs.add(key)
|
|
new_edges.append({
|
|
"source": file_nid,
|
|
"target": tgt_nid,
|
|
"relation": "imports",
|
|
"confidence": "EXTRACTED",
|
|
"confidence_score": 1.0,
|
|
"source_file": str(path),
|
|
"source_location": f"L{at_line}",
|
|
"weight": 1.0,
|
|
})
|
|
for child in n.children:
|
|
walk(child)
|
|
|
|
walk(tree.root_node)
|
|
|
|
return new_edges
|
|
|
|
|
|
def extract_objc(path: Path) -> dict:
|
|
"""Extract interfaces, implementations, protocols, methods, and imports from .m/.mm/.h files."""
|
|
try:
|
|
import tree_sitter_objc as tsobjc
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": "tree_sitter_objc not installed"}
|
|
|
|
try:
|
|
language = Language(tsobjc.language())
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
method_bodies: list[tuple[str, Any]] = []
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({"id": nid, "label": label, "file_type": "code",
|
|
"source_file": str_path, "source_location": f"L{line}"})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", weight: float = 1.0,
|
|
context: str | None = None) -> None:
|
|
edge = {"source": src, "target": tgt, "relation": relation,
|
|
"confidence": confidence, "source_file": str_path,
|
|
"source_location": f"L{line}", "weight": weight}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
def _read(node) -> str:
|
|
return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
|
|
|
|
def _get_name(node, field: str) -> str | None:
|
|
n = node.child_by_field_name(field)
|
|
return _read(n) if n else None
|
|
|
|
def ensure_named_node(name: str, line: int) -> str:
|
|
nid = _make_id(stem, name)
|
|
if nid in seen_ids:
|
|
return nid
|
|
nid = _make_id(name)
|
|
if nid not in seen_ids:
|
|
add_node(nid, name, line)
|
|
return nid
|
|
|
|
def walk(node, parent_nid: str | None = None) -> None:
|
|
t = node.type
|
|
line = node.start_point[0] + 1
|
|
|
|
if t == "preproc_include":
|
|
# #import <Foundation/Foundation.h> or #import "MyClass.h"
|
|
for child in node.children:
|
|
if child.type == "system_lib_string":
|
|
raw = _read(child).strip("<>")
|
|
module = raw.split("/")[-1].replace(".h", "")
|
|
if module:
|
|
tgt_nid = _make_id(module)
|
|
add_edge(file_nid, tgt_nid, "imports", line, context="import")
|
|
elif child.type == "string_literal":
|
|
# recurse into string_literal to find string_content
|
|
for sub in child.children:
|
|
if sub.type == "string_content":
|
|
raw = _read(sub)
|
|
module = raw.split("/")[-1].replace(".h", "")
|
|
if module:
|
|
tgt_nid = _make_id(module)
|
|
add_edge(file_nid, tgt_nid, "imports", line, context="import")
|
|
return
|
|
|
|
if t == "class_interface":
|
|
# @interface ClassName : SuperClass <Protocols>
|
|
# children: @interface, identifier(name), ':', identifier(super), parameterized_arguments, ...
|
|
identifiers = [c for c in node.children if c.type == "identifier"]
|
|
if not identifiers:
|
|
for child in node.children:
|
|
walk(child, parent_nid)
|
|
return
|
|
name = _read(identifiers[0])
|
|
cls_nid = _make_id(stem, name)
|
|
add_node(cls_nid, name, line)
|
|
add_edge(file_nid, cls_nid, "contains", line)
|
|
# superclass is second identifier after ':'
|
|
colon_seen = False
|
|
for child in node.children:
|
|
if child.type == ":":
|
|
colon_seen = True
|
|
elif colon_seen and child.type == "identifier":
|
|
super_nid = ensure_named_node(_read(child), line)
|
|
add_edge(cls_nid, super_nid, "inherits", line)
|
|
colon_seen = False
|
|
elif child.type == "parameterized_arguments":
|
|
# protocols adopted: @interface Foo : Bar <Proto1, Proto2>
|
|
for sub in child.children:
|
|
if sub.type == "type_name":
|
|
for s in sub.children:
|
|
if s.type == "type_identifier":
|
|
proto_nid = ensure_named_node(_read(s), line)
|
|
add_edge(cls_nid, proto_nid, "implements", line)
|
|
elif child.type == "property_declaration":
|
|
prop_line = child.start_point[0] + 1
|
|
for sub in child.children:
|
|
if sub.type == "struct_declaration":
|
|
for s in sub.children:
|
|
if s.type == "type_identifier":
|
|
type_nid = ensure_named_node(_read(s), prop_line)
|
|
edges.append(_semantic_reference_edge(
|
|
cls_nid, type_nid, "field", str_path, prop_line))
|
|
break
|
|
elif child.type == "method_declaration":
|
|
walk(child, cls_nid)
|
|
return
|
|
|
|
if t == "class_implementation":
|
|
# @implementation ClassName
|
|
name = None
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
name = _read(child)
|
|
break
|
|
if not name:
|
|
for child in node.children:
|
|
walk(child, parent_nid)
|
|
return
|
|
impl_nid = _make_id(stem, name)
|
|
if impl_nid not in seen_ids:
|
|
add_node(impl_nid, name, line)
|
|
add_edge(file_nid, impl_nid, "contains", line)
|
|
for child in node.children:
|
|
if child.type == "implementation_definition":
|
|
for sub in child.children:
|
|
walk(sub, impl_nid)
|
|
return
|
|
|
|
if t == "protocol_declaration":
|
|
name = None
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
name = _read(child)
|
|
break
|
|
if name:
|
|
proto_nid = _make_id(stem, name)
|
|
add_node(proto_nid, f"<{name}>", line)
|
|
add_edge(file_nid, proto_nid, "contains", line)
|
|
for child in node.children:
|
|
walk(child, proto_nid)
|
|
return
|
|
|
|
if t in ("method_declaration", "method_definition"):
|
|
container = parent_nid or file_nid
|
|
# method name is the first identifier child (simple selector)
|
|
# for compound selectors: identifier + method_parameter pairs
|
|
parts = []
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
parts.append(_read(child))
|
|
elif child.type == "method_parameter":
|
|
for sub in child.children:
|
|
if sub.type == "identifier":
|
|
# selector keyword before ':'
|
|
pass
|
|
method_name = "".join(parts) if parts else None
|
|
if method_name:
|
|
method_nid = _make_id(container, method_name)
|
|
add_node(method_nid, f"-{method_name}", line)
|
|
add_edge(container, method_nid, "method", line)
|
|
if t == "method_definition":
|
|
method_bodies.append((method_nid, node))
|
|
return
|
|
|
|
for child in node.children:
|
|
walk(child, parent_nid)
|
|
|
|
walk(root)
|
|
|
|
# Second pass: resolve calls inside method bodies
|
|
all_method_nids = {n["id"] for n in nodes if n["id"] != file_nid}
|
|
seen_calls: set[tuple[str, str]] = set()
|
|
for caller_nid, body_node in method_bodies:
|
|
def walk_calls(n) -> None:
|
|
if n.type == "message_expression":
|
|
# [receiver selector]
|
|
for child in n.children:
|
|
if child.type in ("selector", "keyword_argument_list"):
|
|
sel = []
|
|
if child.type == "selector":
|
|
sel.append(_read(child))
|
|
else:
|
|
for sub in child.children:
|
|
if sub.type == "keyword_argument":
|
|
for s in sub.children:
|
|
if s.type == "selector":
|
|
sel.append(_read(s))
|
|
method_name = "".join(sel)
|
|
for candidate in all_method_nids:
|
|
if candidate.endswith(_make_id("", method_name).lstrip("_")):
|
|
pair = (caller_nid, candidate)
|
|
if pair not in seen_calls and caller_nid != candidate:
|
|
seen_calls.add(pair)
|
|
add_edge(caller_nid, candidate, "calls", body_node.start_point[0] + 1,
|
|
confidence="EXTRACTED", weight=1.0, context="call")
|
|
for child in n.children:
|
|
walk_calls(child)
|
|
walk_calls(body_node)
|
|
|
|
return {"nodes": nodes, "edges": edges, "input_tokens": 0, "output_tokens": 0}
|
|
|
|
|
|
def extract_elixir(path: Path) -> dict:
|
|
"""Extract modules, functions, imports, and calls from a .ex/.exs file."""
|
|
try:
|
|
import tree_sitter_elixir as tselixir
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": "tree_sitter_elixir not installed"}
|
|
|
|
try:
|
|
language = Language(tselixir.language())
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
function_bodies: list[tuple[str, Any]] = []
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({"id": nid, "label": label, "file_type": "code",
|
|
"source_file": str_path, "source_location": f"L{line}"})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", weight: float = 1.0,
|
|
context: str | None = None) -> None:
|
|
edge = {"source": src, "target": tgt, "relation": relation,
|
|
"confidence": confidence, "source_file": str_path,
|
|
"source_location": f"L{line}", "weight": weight}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
_IMPORT_KEYWORDS = frozenset({"alias", "import", "require", "use"})
|
|
|
|
def _get_alias_text(node) -> str | None:
|
|
for child in node.children:
|
|
if child.type == "alias":
|
|
return source[child.start_byte:child.end_byte].decode("utf-8", errors="replace")
|
|
return None
|
|
|
|
def walk(node, parent_module_nid: str | None = None) -> None:
|
|
if node.type != "call":
|
|
for child in node.children:
|
|
walk(child, parent_module_nid)
|
|
return
|
|
|
|
identifier_node = None
|
|
arguments_node = None
|
|
do_block_node = None
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
identifier_node = child
|
|
elif child.type == "arguments":
|
|
arguments_node = child
|
|
elif child.type == "do_block":
|
|
do_block_node = child
|
|
|
|
if identifier_node is None:
|
|
for child in node.children:
|
|
walk(child, parent_module_nid)
|
|
return
|
|
|
|
keyword = source[identifier_node.start_byte:identifier_node.end_byte].decode("utf-8", errors="replace")
|
|
line = node.start_point[0] + 1
|
|
|
|
if keyword == "defmodule":
|
|
module_name = _get_alias_text(arguments_node) if arguments_node else None
|
|
if not module_name:
|
|
return
|
|
module_nid = _make_id(stem, module_name)
|
|
add_node(module_nid, module_name, line)
|
|
add_edge(file_nid, module_nid, "contains", line)
|
|
if do_block_node:
|
|
for child in do_block_node.children:
|
|
walk(child, parent_module_nid=module_nid)
|
|
return
|
|
|
|
if keyword in ("def", "defp"):
|
|
func_name = None
|
|
if arguments_node:
|
|
for child in arguments_node.children:
|
|
if child.type == "call":
|
|
for sub in child.children:
|
|
if sub.type == "identifier":
|
|
func_name = source[sub.start_byte:sub.end_byte].decode("utf-8", errors="replace")
|
|
break
|
|
elif child.type == "identifier":
|
|
func_name = source[child.start_byte:child.end_byte].decode("utf-8", errors="replace")
|
|
break
|
|
if not func_name:
|
|
return
|
|
container = parent_module_nid or file_nid
|
|
func_nid = _make_id(container, func_name)
|
|
add_node(func_nid, f"{func_name}()", line)
|
|
if parent_module_nid:
|
|
add_edge(parent_module_nid, func_nid, "method", line)
|
|
else:
|
|
add_edge(file_nid, func_nid, "contains", line)
|
|
if do_block_node:
|
|
function_bodies.append((func_nid, do_block_node))
|
|
return
|
|
|
|
if keyword in _IMPORT_KEYWORDS and arguments_node:
|
|
module_name = _get_alias_text(arguments_node)
|
|
if module_name:
|
|
tgt_nid = _make_id(module_name)
|
|
add_edge(file_nid, tgt_nid, "imports", line, context="import")
|
|
return
|
|
|
|
for child in node.children:
|
|
walk(child, parent_module_nid)
|
|
|
|
walk(root)
|
|
|
|
label_to_nid: dict[str, str] = {}
|
|
for n in nodes:
|
|
normalised = n["label"].strip("()").lstrip(".")
|
|
label_to_nid[normalised] = n["id"]
|
|
|
|
seen_call_pairs: set[tuple[str, str]] = set()
|
|
raw_calls: list[dict] = []
|
|
_SKIP_KEYWORDS = frozenset({
|
|
"def", "defp", "defmodule", "defmacro", "defmacrop",
|
|
"defstruct", "defprotocol", "defimpl", "defguard",
|
|
"alias", "import", "require", "use",
|
|
"if", "unless", "case", "cond", "with", "for",
|
|
})
|
|
|
|
def walk_calls(node, caller_nid: str) -> None:
|
|
if node.type != "call":
|
|
for child in node.children:
|
|
walk_calls(child, caller_nid)
|
|
return
|
|
for child in node.children:
|
|
if child.type == "identifier":
|
|
kw = source[child.start_byte:child.end_byte].decode("utf-8", errors="replace")
|
|
if kw in _SKIP_KEYWORDS:
|
|
for c in node.children:
|
|
walk_calls(c, caller_nid)
|
|
return
|
|
break
|
|
callee_name: str | None = None
|
|
is_member_call: bool = False
|
|
for child in node.children:
|
|
if child.type == "dot":
|
|
is_member_call = True
|
|
dot_text = source[child.start_byte:child.end_byte].decode("utf-8", errors="replace")
|
|
parts = dot_text.rstrip(".").split(".")
|
|
if parts:
|
|
callee_name = parts[-1]
|
|
break
|
|
if child.type == "identifier":
|
|
callee_name = source[child.start_byte:child.end_byte].decode("utf-8", errors="replace")
|
|
break
|
|
if callee_name and callee_name not in _LANGUAGE_BUILTIN_GLOBALS:
|
|
tgt_nid = label_to_nid.get(callee_name)
|
|
if tgt_nid and tgt_nid != caller_nid:
|
|
pair = (caller_nid, tgt_nid)
|
|
if pair not in seen_call_pairs:
|
|
seen_call_pairs.add(pair)
|
|
add_edge(caller_nid, tgt_nid, "calls",
|
|
node.start_point[0] + 1, confidence="EXTRACTED", weight=1.0,
|
|
context="call")
|
|
else:
|
|
raw_calls.append({
|
|
"caller_nid": caller_nid,
|
|
"callee": callee_name,
|
|
"is_member_call": is_member_call,
|
|
"source_file": str_path,
|
|
"source_location": f"L{node.start_point[0] + 1}",
|
|
})
|
|
for child in node.children:
|
|
walk_calls(child, caller_nid)
|
|
|
|
for caller_nid, body in function_bodies:
|
|
walk_calls(body, caller_nid)
|
|
|
|
clean_edges = [e for e in edges if e["source"] in seen_ids and
|
|
(e["target"] in seen_ids or e["relation"] == "imports")]
|
|
return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls, "input_tokens": 0, "output_tokens": 0}
|
|
|
|
|
|
def extract_markdown(path: Path) -> dict:
|
|
"""Extract structural nodes and edges from a Markdown file.
|
|
|
|
Produces nodes for:
|
|
- The file itself
|
|
- Each heading (# / ## / ### etc.)
|
|
|
|
Produces edges for:
|
|
- file --contains--> heading
|
|
- parent heading --contains--> child heading (nesting by level)
|
|
- heading --references--> other node (when backtick `Name` matches a known pattern)
|
|
|
|
Fenced code blocks (``` ... ```) are skipped during parsing so their
|
|
contents don't get treated as headings, but no node is emitted for
|
|
them — they were always orphans (only a single contains edge to the
|
|
parent doc) and inflated the disconnected-component count (#1077).
|
|
|
|
No tree-sitter dependency — pure line-by-line parsing.
|
|
"""
|
|
try:
|
|
source = path.read_text(encoding="utf-8", errors="replace")
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
|
|
def add_node(nid: str, label: str, line: int, file_type: str = "document") -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({"id": nid, "label": label, "file_type": file_type,
|
|
"source_file": str_path, "source_location": f"L{line}"})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", weight: float = 1.0) -> None:
|
|
edges.append({"source": src, "target": tgt, "relation": relation,
|
|
"confidence": confidence, "source_file": str_path,
|
|
"source_location": f"L{line}", "weight": weight})
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
# Track heading stack for nesting: [(level, nid), ...]
|
|
heading_stack: list[tuple[int, str]] = []
|
|
in_code_block = False
|
|
|
|
lines = source.splitlines()
|
|
for line_num_0, line_text in enumerate(lines):
|
|
line_num = line_num_0 + 1
|
|
|
|
# Skip over fenced code blocks so their contents are not parsed as
|
|
# headings, but do not emit nodes/edges for them (#1077).
|
|
stripped = line_text.strip()
|
|
if stripped.startswith("```"):
|
|
in_code_block = not in_code_block
|
|
continue
|
|
|
|
if in_code_block:
|
|
continue
|
|
|
|
# Detect headings: # Heading, ## Heading, etc.
|
|
heading_match = re.match(r'^(#{1,6})\s+(.+)', line_text)
|
|
if heading_match:
|
|
level = len(heading_match.group(1))
|
|
title = heading_match.group(2).strip()
|
|
h_nid = _make_id(stem, title)
|
|
# Avoid duplicate heading IDs by appending line number
|
|
if h_nid in seen_ids:
|
|
h_nid = _make_id(stem, title, str(line_num))
|
|
add_node(h_nid, title, line_num)
|
|
|
|
# Pop headings at same or deeper level
|
|
while heading_stack and heading_stack[-1][0] >= level:
|
|
heading_stack.pop()
|
|
|
|
# Connect to parent heading or file
|
|
parent = heading_stack[-1][1] if heading_stack else file_nid
|
|
add_edge(parent, h_nid, "contains", line_num)
|
|
|
|
heading_stack.append((level, h_nid))
|
|
continue
|
|
|
|
return {"nodes": nodes, "edges": edges, "input_tokens": 0, "output_tokens": 0}
|
|
|
|
|
|
# ── Pascal / Delphi extractor ─────────────────────────────────────────────────
|
|
|
|
_pascal_unit_cache: dict[str, dict[str, str]] = {}
|
|
_pascal_class_stem_cache: dict[str, dict[str, str]] = {} # root_key → {stem_lower: _file_stem}
|
|
|
|
|
|
def _pascal_project_root(from_path: Path) -> Path:
|
|
"""Return the highest ancestor directory that looks like a Pascal project root.
|
|
|
|
Walks up the directory tree and tracks the topmost directory that:
|
|
- is NOT a filesystem root (e.g. D:/, C:/, /)
|
|
- has at least 2 .pas files OR at least 1 .dpr file as direct children
|
|
|
|
The minimum-2 threshold avoids treating a level as the root just because a
|
|
single stray .pas file was copied there. The filesystem-root exclusion
|
|
prevents overshoot on drives that have a stray file directly at D:/.
|
|
|
|
Falls back to from_path.parent if nothing better is found.
|
|
"""
|
|
best = from_path.parent
|
|
current = from_path.parent
|
|
for _ in range(12):
|
|
if len(current.parts) <= 1:
|
|
break # never use a filesystem root (D:/, C:/, /)
|
|
pas_count = sum(1 for _ in current.glob("*.pas"))
|
|
dpr_count = sum(1 for _ in current.glob("*.dpr"))
|
|
if pas_count >= 2 or dpr_count >= 1:
|
|
best = current
|
|
parent = current.parent
|
|
if parent == current:
|
|
break
|
|
current = parent
|
|
return best
|
|
|
|
|
|
def _pascal_resolve_unit(from_path: Path, unit_name: str) -> str:
|
|
"""Resolve a Pascal unit name to the graphify node ID of its source file.
|
|
|
|
Scans all Pascal files under the project root (the highest ancestor that
|
|
directly contains .pas/.dpr files) and returns _make_id(str(matched_path)).
|
|
Result is cached per project root so the rglob runs at most once per
|
|
project. Falls back to _make_id(unit_name) for units not found on disk
|
|
(e.g. standard RTL units like SysUtils, Windows).
|
|
"""
|
|
root = _pascal_project_root(from_path)
|
|
root_key = str(root)
|
|
if root_key not in _pascal_unit_cache:
|
|
unit_map: dict[str, str] = {}
|
|
for ext in (".pas", ".pp", ".dpr", ".dpk", ".inc"):
|
|
for f in root.rglob("*" + ext):
|
|
unit_map[f.stem.lower()] = _make_id(str(f))
|
|
_pascal_unit_cache[root_key] = unit_map
|
|
return _pascal_unit_cache[root_key].get(unit_name.lower(), _make_id(unit_name))
|
|
|
|
|
|
def _pascal_resolve_class(from_path: Path, class_name: str) -> str | None:
|
|
"""Resolve a Pascal class/interface name to the node ID of its defining file's class node.
|
|
|
|
Pascal convention: TFooBar is defined in FooBar.pas, IFooBar in FooBar.pas.
|
|
Strips the leading T/I prefix, finds the file, and returns
|
|
_make_id(_file_stem(found_file), class_name).
|
|
|
|
Returns None when no matching file is found on disk (RTL, stdlib, or
|
|
unconventionally-named class — caller should create a stub node).
|
|
"""
|
|
prefix = class_name[:1]
|
|
unit_name = class_name[1:] if prefix in ("T", "I") else class_name
|
|
|
|
root = _pascal_project_root(from_path)
|
|
root_key = str(root)
|
|
if root_key not in _pascal_class_stem_cache:
|
|
stem_map: dict[str, str] = {}
|
|
for ext in (".pas", ".pp", ".dpr", ".dpk"):
|
|
for f in root.rglob("*" + ext):
|
|
stem_map[f.stem.lower()] = _file_stem(f)
|
|
_pascal_class_stem_cache[root_key] = stem_map
|
|
|
|
file_stem = _pascal_class_stem_cache[root_key].get(unit_name.lower())
|
|
if file_stem:
|
|
return _make_id(file_stem, class_name)
|
|
return None
|
|
|
|
|
|
_PAS_TOKEN_RE = re.compile(
|
|
r"'(?:''|[^'])*'"
|
|
r"|\{[^}]*\}"
|
|
r"|\(\*.*?\*\)"
|
|
r"|//[^\n]*",
|
|
re.DOTALL,
|
|
)
|
|
_PAS_MODULE_RE = re.compile(
|
|
r"\b(unit|program|library)\s+([A-Za-z_][\w.]*)\s*;",
|
|
re.IGNORECASE,
|
|
)
|
|
_PAS_USES_RE = re.compile(
|
|
r"\buses\b\s*([^;]+);",
|
|
re.IGNORECASE | re.DOTALL,
|
|
)
|
|
_PAS_TYPE_HEADER_RE = re.compile(
|
|
r"\b(?P<name>[A-Za-z_]\w*)(?:\s*<[^>]+>)?\s*=\s*(?:packed\s+)?"
|
|
r"(?P<kind>class|interface)\b"
|
|
r"(?:\s*\(\s*(?P<bases>[^)]*)\s*\))?",
|
|
re.IGNORECASE,
|
|
)
|
|
_PAS_END_SEMI_RE = re.compile(r"\bend\s*;", re.IGNORECASE)
|
|
_PAS_METHOD_DECL_RE = re.compile(
|
|
r"\b(?:procedure|function|constructor|destructor)\s+"
|
|
r"(?P<name>[A-Za-z_]\w*)"
|
|
r"(?:\s*\([^)]*\))?"
|
|
r"(?:\s*:\s*[\w<>,\s.]+)?"
|
|
r"\s*;",
|
|
re.IGNORECASE,
|
|
)
|
|
_PAS_IMPL_HEADER_RE = re.compile(
|
|
r"\b(?:procedure|function|constructor|destructor)\s+"
|
|
r"(?P<qual>[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)?)"
|
|
r"(?:\s*<[^>]+>)?"
|
|
r"(?:\s*\([^)]*\))?"
|
|
r"(?:\s*:\s*[\w<>,\s.]+)?"
|
|
r"\s*;",
|
|
re.IGNORECASE,
|
|
)
|
|
_PAS_BEGIN_END_TOKEN_RE = re.compile(
|
|
r"\b(begin|end|case|try|asm|record)\b", re.IGNORECASE
|
|
)
|
|
_PAS_CALL_RE = re.compile(r"\b([A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)\s*[(;]")
|
|
_PAS_KEYWORDS = frozenset({
|
|
"begin", "end", "if", "then", "else", "while", "do", "for", "to",
|
|
"downto", "repeat", "until", "case", "of", "try", "finally", "except",
|
|
"with", "inherited", "result", "var", "const", "type", "nil", "true",
|
|
"false", "exit", "break", "continue", "uses", "unit", "program",
|
|
"library", "interface", "implementation", "initialization", "finalization",
|
|
"procedure", "function", "constructor", "destructor", "class", "record",
|
|
"object", "array", "string", "integer", "boolean", "real", "char",
|
|
"writeln", "write", "readln", "read", "assigned", "length", "high",
|
|
"low", "inc", "dec", "new", "dispose", "setlength", "copy", "pos",
|
|
"trim", "format", "inttostr", "strtoint", "ord", "chr", "sizeof",
|
|
"create", "free", "destroy",
|
|
})
|
|
|
|
|
|
def _pascal_strip_comments(text: str) -> str:
|
|
"""Strip Pascal comments ({}, (* *), //) while preserving newlines."""
|
|
def _sub(m: re.Match) -> str:
|
|
tok = m.group(0)
|
|
if tok.startswith("'"):
|
|
return tok
|
|
return "".join(c if c == "\n" else " " for c in tok)
|
|
return _PAS_TOKEN_RE.sub(_sub, text)
|
|
|
|
|
|
def _pascal_split_sections(text: str) -> tuple[str, int, str, int]:
|
|
"""Split into (iface_text, iface_offset, impl_text, impl_offset).
|
|
Files without interface/implementation sections (dpr/lpr/inc) return
|
|
the whole text as impl with offset 0.
|
|
"""
|
|
iface_m = re.search(r"\binterface\b", text, re.IGNORECASE)
|
|
impl_m = re.search(r"\bimplementation\b", text, re.IGNORECASE)
|
|
if iface_m and impl_m:
|
|
iface_off = iface_m.end()
|
|
impl_off = impl_m.end()
|
|
end_m = re.search(
|
|
r"\b(initialization|finalization)\b", text[impl_off:], re.IGNORECASE
|
|
)
|
|
impl_end = impl_off + end_m.start() if end_m else len(text)
|
|
return text[iface_off:impl_m.start()], iface_off, text[impl_off:impl_end], impl_off
|
|
return "", 0, text, 0
|
|
|
|
|
|
def _pascal_split_uses(s: str) -> list[str]:
|
|
"""Split a uses list string, handling 'Foo in ''bar.pas''' syntax."""
|
|
out = []
|
|
for chunk in s.split(","):
|
|
name = re.split(r"\s+in\s+", chunk.strip(), maxsplit=1, flags=re.IGNORECASE)[0]
|
|
name = name.strip().strip(";")
|
|
if name and re.match(r"[A-Za-z_][\w.]*$", name):
|
|
out.append(name)
|
|
return out
|
|
|
|
|
|
def _pascal_split_bases(s: str) -> list[str]:
|
|
"""Split inheritance list, handling generics like TList<T, U>."""
|
|
out, depth, buf = [], 0, []
|
|
for ch in s:
|
|
if ch == "<":
|
|
depth += 1
|
|
buf.append(ch)
|
|
elif ch == ">":
|
|
depth -= 1
|
|
buf.append(ch)
|
|
elif ch == "," and depth == 0:
|
|
name = re.sub(r"<.*$", "", "".join(buf).strip())
|
|
if name:
|
|
out.append(name)
|
|
buf = []
|
|
else:
|
|
buf.append(ch)
|
|
name = re.sub(r"<.*$", "", "".join(buf).strip())
|
|
if name:
|
|
out.append(name)
|
|
return [n for n in out if re.match(r"[A-Za-z_]\w*$", n)]
|
|
|
|
|
|
def _pascal_find_body(text: str, start: int) -> tuple[int, int]:
|
|
"""Find balanced begin..end after start. Returns (body_start, body_end).
|
|
Returns (0, 0) if no begin found.
|
|
"""
|
|
m = re.search(r"\bbegin\b", text[start:], re.IGNORECASE)
|
|
if not m:
|
|
return (0, 0)
|
|
body_start = start + m.end()
|
|
depth = 1
|
|
for tok in _PAS_BEGIN_END_TOKEN_RE.finditer(text, body_start):
|
|
kw = tok.group(1).lower()
|
|
if kw in ("begin", "case", "try", "asm", "record"):
|
|
depth += 1
|
|
elif kw == "end":
|
|
depth -= 1
|
|
if depth == 0:
|
|
return (body_start, tok.start())
|
|
return (body_start, len(text))
|
|
|
|
|
|
def _extract_pascal_regex(path: Path) -> dict:
|
|
"""Regex fallback for Pascal/Delphi extraction when tree-sitter-pascal
|
|
is unavailable. Produces the same node/edge schema as the tree-sitter pass.
|
|
"""
|
|
try:
|
|
raw = path.read_text(encoding="utf-8", errors="replace")
|
|
except Exception as exc:
|
|
return {"nodes": [], "edges": [], "error": str(exc)}
|
|
|
|
str_path = str(path)
|
|
stem = _file_stem(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
seen_call_pairs: set[tuple[str, str]] = set()
|
|
|
|
def _add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({
|
|
"id": nid,
|
|
"label": label,
|
|
"file_type": "code",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
})
|
|
|
|
def _add_edge(src: str, tgt: str, relation: str, line: int, context: str | None = None) -> None:
|
|
edge: dict = {
|
|
"source": src,
|
|
"target": tgt,
|
|
"relation": relation,
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
def _lineno(text: str, offset: int) -> int:
|
|
return text.count("\n", 0, offset) + 1
|
|
|
|
file_nid = _make_id(str_path)
|
|
_add_node(file_nid, path.name, 1)
|
|
|
|
stripped = _pascal_strip_comments(raw)
|
|
|
|
# Module header
|
|
module_nid = file_nid
|
|
mod_m = _PAS_MODULE_RE.search(stripped)
|
|
if mod_m:
|
|
mod_name = mod_m.group(2)
|
|
module_nid = _make_id(stem, mod_name)
|
|
_add_node(module_nid, mod_name, _lineno(stripped, mod_m.start()))
|
|
_add_edge(file_nid, module_nid, "contains", _lineno(stripped, mod_m.start()))
|
|
|
|
iface_text, iface_off, impl_text, impl_off = _pascal_split_sections(stripped)
|
|
|
|
# Uses clauses
|
|
for section_text, section_off in ((iface_text, iface_off), (impl_text, impl_off)):
|
|
for um in _PAS_USES_RE.finditer(section_text):
|
|
line = _lineno(stripped, section_off + um.start())
|
|
for unit_name in _pascal_split_uses(um.group(1)):
|
|
tgt_nid = _pascal_resolve_unit(path, unit_name)
|
|
_add_edge(module_nid, tgt_nid, "imports", line, context="import")
|
|
|
|
# Type declarations (classes / interfaces) in interface section
|
|
search_text = iface_text if iface_text else stripped
|
|
search_off = iface_off if iface_text else 0
|
|
pos = 0
|
|
while pos < len(search_text):
|
|
hm = _PAS_TYPE_HEADER_RE.search(search_text, pos)
|
|
if not hm:
|
|
break
|
|
type_name = hm.group("name")
|
|
bases_raw = hm.group("bases") or ""
|
|
line = _lineno(stripped, search_off + hm.start())
|
|
cls_nid = _make_id(stem, type_name)
|
|
_add_node(cls_nid, type_name, line)
|
|
_add_edge(module_nid, cls_nid, "contains", line)
|
|
|
|
for base_name in _pascal_split_bases(bases_raw):
|
|
resolved = _pascal_resolve_class(path, base_name)
|
|
base_nid = resolved if resolved else _make_id(base_name)
|
|
if base_nid not in seen_ids:
|
|
_add_node(base_nid, base_name, line)
|
|
_add_edge(cls_nid, base_nid, "inherits", line)
|
|
|
|
# Find class body (up to next end;)
|
|
end_m = _PAS_END_SEMI_RE.search(search_text, hm.end())
|
|
body_text = search_text[hm.end():end_m.start()] if end_m else ""
|
|
body_off = search_off + hm.end()
|
|
|
|
# Forward method declarations inside the class body
|
|
for mm in _PAS_METHOD_DECL_RE.finditer(body_text):
|
|
mname = mm.group("name")
|
|
mline = _lineno(stripped, body_off + mm.start())
|
|
method_nid = _make_id(cls_nid, mname)
|
|
_add_node(method_nid, f"{mname}()", mline)
|
|
_add_edge(cls_nid, method_nid, "method", mline)
|
|
|
|
pos = end_m.end() if end_m else len(search_text)
|
|
|
|
# Implementation headers (procedure/function/constructor/destructor)
|
|
impl_records: list[tuple[str, int, str]] = []
|
|
for fm in _PAS_IMPL_HEADER_RE.finditer(impl_text):
|
|
qualified = fm.group("qual")
|
|
line = _lineno(stripped, impl_off + fm.start())
|
|
if "." in qualified:
|
|
cls_part, method_part = qualified.split(".", 1)
|
|
cls_nid = _make_id(stem, cls_part)
|
|
container = cls_nid if cls_nid in seen_ids else module_nid
|
|
relation = "method" if cls_nid in seen_ids else "contains"
|
|
label = f"{method_part}()"
|
|
else:
|
|
container, relation = module_nid, "contains"
|
|
label = f"{qualified}()"
|
|
proc_nid = _make_id(stem, qualified)
|
|
_add_node(proc_nid, label, line)
|
|
_add_edge(container, proc_nid, relation, line)
|
|
|
|
body_start, body_end = _pascal_find_body(impl_text, fm.end())
|
|
body_text = impl_text[body_start:body_end] if body_start else ""
|
|
impl_records.append((proc_nid, line, body_text))
|
|
|
|
# Intra-file call edges
|
|
all_procs: dict[str, str] = {
|
|
n["label"].removesuffix("()").lower(): n["id"]
|
|
for n in nodes
|
|
if n["id"] != file_nid and n["label"].endswith("()")
|
|
}
|
|
for caller_nid, caller_line, body_text in impl_records:
|
|
for cm in _PAS_CALL_RE.finditer(body_text):
|
|
callee_name = cm.group(1).split(".")[-1].lower()
|
|
if callee_name in _PAS_KEYWORDS:
|
|
continue
|
|
callee_nid = all_procs.get(callee_name)
|
|
if not callee_nid or callee_nid == caller_nid:
|
|
continue
|
|
pair = (caller_nid, callee_nid)
|
|
if pair in seen_call_pairs:
|
|
continue
|
|
seen_call_pairs.add(pair)
|
|
call_line = caller_line + body_text.count("\n", 0, cm.start())
|
|
_add_edge(caller_nid, callee_nid, "calls", call_line, context="call")
|
|
|
|
return {"nodes": nodes, "edges": edges, "input_tokens": 0, "output_tokens": 0}
|
|
|
|
|
|
def extract_pascal(path: Path) -> dict:
|
|
"""Extract units, classes, procedures, uses-imports, and calls from Pascal/Delphi files.
|
|
|
|
Produces nodes for:
|
|
- The file itself
|
|
- unit / program / library declarations
|
|
- class and interface type declarations
|
|
- procedure / function implementations (including qualified TClass.Method names)
|
|
|
|
Produces edges for:
|
|
- file --contains--> module
|
|
- module --imports--> other file node (via uses clause, resolved to path-based IDs)
|
|
- class --inherits--> base class
|
|
- class/module --contains--> method forward declaration
|
|
- class/module --contains--> procedure/function implementation
|
|
- procedure --calls--> other procedure (within the same file)
|
|
|
|
Uses tree-sitter-pascal when available; falls back to a regex-based extractor
|
|
(_extract_pascal_regex) when it isn't installed or fails to parse, so Pascal
|
|
extraction works out of the box without an extra pip install.
|
|
"""
|
|
try:
|
|
import tree_sitter_pascal as tspascal
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return _extract_pascal_regex(path)
|
|
|
|
try:
|
|
language = Language(tspascal.language())
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception:
|
|
return _extract_pascal_regex(path)
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
proc_bodies: list[tuple[str, Any]] = []
|
|
|
|
def _read(node) -> str: # type: ignore[no-untyped-def]
|
|
return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({
|
|
"id": nid, "label": label, "file_type": "code",
|
|
"source_file": str_path, "source_location": f"L{line}",
|
|
})
|
|
|
|
def add_edge(
|
|
src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", weight: float = 1.0,
|
|
context: str | None = None,
|
|
) -> None:
|
|
edge: dict[str, Any] = {
|
|
"source": src, "target": tgt, "relation": relation,
|
|
"confidence": confidence, "source_file": str_path,
|
|
"source_location": f"L{line}", "weight": weight,
|
|
}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
module_nid = file_nid
|
|
|
|
def _proc_name(header_node) -> str | None: # type: ignore[no-untyped-def]
|
|
name_node = header_node.child_by_field_name("name")
|
|
if name_node:
|
|
return _read(name_node)
|
|
for child in header_node.children:
|
|
if child.type in ("identifier", "genericDot", "genericTpl"):
|
|
return _read(child)
|
|
return None
|
|
|
|
def walk(node, parent_nid: str) -> None: # type: ignore[no-untyped-def]
|
|
nonlocal module_nid
|
|
t = node.type
|
|
line = node.start_point[0] + 1
|
|
|
|
if t in ("unit", "program", "library"):
|
|
name_node = next((c for c in node.children if c.type == "moduleName"), None)
|
|
mod_name = _read(name_node) if name_node else path.stem
|
|
mod_nid = _make_id(stem, mod_name)
|
|
add_node(mod_nid, mod_name, line)
|
|
add_edge(file_nid, mod_nid, "contains", line)
|
|
module_nid = mod_nid
|
|
for child in node.children:
|
|
walk(child, mod_nid)
|
|
return
|
|
|
|
if t == "declUses":
|
|
for child in node.children:
|
|
if child.type == "moduleName":
|
|
mod_name = _read(child)
|
|
tgt_nid = _pascal_resolve_unit(path, mod_name)
|
|
add_edge(parent_nid, tgt_nid, "imports", line, context="import")
|
|
return
|
|
|
|
if t == "declType":
|
|
type_name = None
|
|
kind_node = None
|
|
for child in node.children:
|
|
if child.type == "identifier" and type_name is None:
|
|
type_name = _read(child)
|
|
elif child.type in ("declClass", "declIntf", "declHelper") and kind_node is None:
|
|
kind_node = child
|
|
if type_name and kind_node:
|
|
cls_nid = _make_id(stem, type_name)
|
|
add_node(cls_nid, type_name, line)
|
|
add_edge(parent_nid, cls_nid, "contains", line)
|
|
for child in kind_node.children:
|
|
if child.type == "typeref":
|
|
base_name = _read(child)
|
|
base_nid = _make_id(stem, base_name)
|
|
if base_nid not in seen_ids:
|
|
# Try cross-file resolution (TFooBar → FooBar.pas)
|
|
resolved = _pascal_resolve_class(path, base_name)
|
|
base_nid = resolved if resolved else _make_id(base_name)
|
|
if base_nid not in seen_ids:
|
|
# Stub for RTL/external/cross-file base classes
|
|
add_node(base_nid, base_name, line)
|
|
add_edge(cls_nid, base_nid, "inherits", line)
|
|
for child in kind_node.children:
|
|
walk(child, cls_nid)
|
|
return
|
|
for child in node.children:
|
|
walk(child, parent_nid)
|
|
return
|
|
|
|
if t == "declProcFwd":
|
|
header = next((c for c in node.children if c.type == "declProc"), None)
|
|
if header:
|
|
name = _proc_name(header)
|
|
if name and "." not in name:
|
|
method_nid = _make_id(parent_nid, name)
|
|
add_node(method_nid, f"{name}()", line)
|
|
add_edge(parent_nid, method_nid, "method", line)
|
|
return
|
|
|
|
if t == "defProc":
|
|
header = next((c for c in node.children if c.type == "declProc"), None)
|
|
body_node = next((c for c in node.children if c.type == "block"), None)
|
|
if not header:
|
|
for child in node.children:
|
|
walk(child, parent_nid)
|
|
return
|
|
name = _proc_name(header)
|
|
if not name:
|
|
for child in node.children:
|
|
walk(child, parent_nid)
|
|
return
|
|
container = parent_nid
|
|
if "." in name:
|
|
parts = name.split(".", 1)
|
|
cls_nid = _make_id(stem, parts[0])
|
|
if cls_nid in seen_ids:
|
|
container = cls_nid
|
|
label = f"{parts[-1]}()"
|
|
else:
|
|
label = f"{name}()"
|
|
proc_nid = _make_id(stem, name)
|
|
add_node(proc_nid, label, line)
|
|
add_edge(
|
|
container, proc_nid,
|
|
"method" if container != parent_nid else "contains",
|
|
line,
|
|
)
|
|
if body_node:
|
|
proc_bodies.append((proc_nid, body_node))
|
|
return
|
|
|
|
for child in node.children:
|
|
walk(child, parent_nid)
|
|
|
|
walk(root, file_nid)
|
|
|
|
# Second pass: resolve calls inside procedure/function bodies
|
|
all_procs: dict[str, str] = {
|
|
n["label"].removesuffix("()").lower(): n["id"]
|
|
for n in nodes if n["id"] != file_nid
|
|
}
|
|
seen_call_pairs: set[tuple[str, str]] = set()
|
|
|
|
def walk_calls(node, caller_nid: str) -> None: # type: ignore[no-untyped-def]
|
|
if node.type == "exprCall":
|
|
callee_text = None
|
|
for child in node.children:
|
|
if child.is_named and child.type not in ("exprArgs",):
|
|
callee_text = _read(child).split(".")[-1]
|
|
break
|
|
if callee_text:
|
|
callee_nid = all_procs.get(callee_text.lower())
|
|
if callee_nid and callee_nid != caller_nid:
|
|
pair = (caller_nid, callee_nid)
|
|
if pair not in seen_call_pairs:
|
|
seen_call_pairs.add(pair)
|
|
add_edge(
|
|
caller_nid, callee_nid, "calls",
|
|
node.start_point[0] + 1, context="call",
|
|
)
|
|
elif node.type == "statement":
|
|
# Pascal bare procedure calls with no args: `Reset;`
|
|
# tree-sitter represents these as statement → identifier (no exprCall wrapper)
|
|
named = [c for c in node.children if c.is_named]
|
|
if len(named) == 1 and named[0].type == "identifier":
|
|
callee_text = _read(named[0])
|
|
callee_nid = all_procs.get(callee_text.lower())
|
|
if callee_nid and callee_nid != caller_nid:
|
|
pair = (caller_nid, callee_nid)
|
|
if pair not in seen_call_pairs:
|
|
seen_call_pairs.add(pair)
|
|
add_edge(
|
|
caller_nid, callee_nid, "calls",
|
|
node.start_point[0] + 1, context="call",
|
|
)
|
|
for child in node.children:
|
|
walk_calls(child, caller_nid)
|
|
|
|
for proc_nid, body_node in proc_bodies:
|
|
walk_calls(body_node, proc_nid)
|
|
|
|
return {"nodes": nodes, "edges": edges, "input_tokens": 0, "output_tokens": 0}
|
|
|
|
|
|
def extract_lazarus_form(path: Path) -> dict:
|
|
"""Extract component hierarchy from Lazarus .lfm form files.
|
|
|
|
.lfm is a text-based declarative format for UI component trees, structured as:
|
|
object ComponentName: TClassName
|
|
PropertyName = Value
|
|
OnEvent = HandlerName
|
|
object ChildName: TChildClass
|
|
...
|
|
end
|
|
end
|
|
|
|
Produces nodes for:
|
|
- The form file itself
|
|
- Each component class encountered (TForm1, TButton, TPanel, ...)
|
|
- Event handler names referenced by OnXxx properties
|
|
|
|
Produces edges for:
|
|
- file --contains--> root form class
|
|
- parent component --contains--> child component class
|
|
- component --references--> event handler (context: "event")
|
|
"""
|
|
try:
|
|
text = path.read_text(encoding="utf-8", errors="replace")
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
import re
|
|
str_path = str(path)
|
|
stem = _file_stem(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
seen_edge_pairs: set[tuple[str, str, str]] = set()
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({
|
|
"id": nid, "label": label, "file_type": "code",
|
|
"source_file": str_path, "source_location": f"L{line}",
|
|
})
|
|
|
|
def add_edge(
|
|
src: str, tgt: str, relation: str, line: int,
|
|
context: str | None = None,
|
|
) -> None:
|
|
key = (src, tgt, relation)
|
|
if key in seen_edge_pairs:
|
|
return
|
|
seen_edge_pairs.add(key)
|
|
edge: dict[str, Any] = {
|
|
"source": src, "target": tgt, "relation": relation,
|
|
"confidence": "EXTRACTED", "source_file": str_path,
|
|
"source_location": f"L{line}", "weight": 1.0,
|
|
}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
obj_re = re.compile(r"^\s*object\s+\w+\s*:\s*(\w+)", re.IGNORECASE)
|
|
event_re = re.compile(r"^\s*On\w+\s*=\s*(\w+)", re.IGNORECASE)
|
|
end_re = re.compile(r"^\s*end\s*$", re.IGNORECASE)
|
|
|
|
# Stack of node IDs representing the nesting of object...end blocks
|
|
stack: list[str] = [file_nid]
|
|
|
|
for lineno, line in enumerate(text.splitlines(), 1):
|
|
m = obj_re.match(line)
|
|
if m:
|
|
class_name = m.group(1)
|
|
nid = _make_id(stem, class_name)
|
|
add_node(nid, class_name, lineno)
|
|
add_edge(stack[-1], nid, "contains", lineno)
|
|
stack.append(nid)
|
|
continue
|
|
|
|
m = event_re.match(line)
|
|
if m and len(stack) > 1:
|
|
handler = m.group(1)
|
|
handler_nid = _make_id(stem, handler)
|
|
add_node(handler_nid, f"{handler}()", lineno)
|
|
add_edge(stack[-1], handler_nid, "references", lineno, context="event")
|
|
continue
|
|
|
|
if end_re.match(line) and len(stack) > 1:
|
|
stack.pop()
|
|
|
|
return {"nodes": nodes, "edges": edges, "input_tokens": 0, "output_tokens": 0}
|
|
|
|
|
|
def extract_delphi_form(path: Path) -> dict:
|
|
"""Extract component hierarchy from Delphi .dfm form files.
|
|
|
|
.dfm files come in two formats:
|
|
- Text (same `object Name: TClassName ... end` syntax as .lfm)
|
|
- Binary (starts with a TPF0/FF0A magic header — unreadable as text)
|
|
|
|
Binary .dfm files are skipped gracefully: an empty result is returned
|
|
so the rest of the pipeline is unaffected. Convert binary forms to
|
|
text in the Delphi IDE via File → Save As (Text DFM) if you want them
|
|
indexed.
|
|
|
|
Text .dfm files are parsed identically to .lfm: component containment
|
|
(`contains`) and event handler references (`references`, context "event").
|
|
"""
|
|
try:
|
|
raw = path.read_bytes()
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
# Detect binary DFM: Delphi binary resource streams start with FF 0A
|
|
if raw[:2] == b"\xff\x0a":
|
|
return {
|
|
"nodes": [], "edges": [],
|
|
"error": f"binary DFM (convert to text in Delphi IDE to index): {path.name}",
|
|
}
|
|
|
|
# Text DFM — delegate to the shared form parser (same syntax as .lfm)
|
|
try:
|
|
text = raw.decode("utf-8", errors="replace")
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
import re
|
|
str_path = str(path)
|
|
stem = _file_stem(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
seen_edge_pairs: set[tuple[str, str, str]] = set()
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({
|
|
"id": nid, "label": label, "file_type": "code",
|
|
"source_file": str_path, "source_location": f"L{line}",
|
|
})
|
|
|
|
def add_edge(
|
|
src: str, tgt: str, relation: str, line: int,
|
|
context: str | None = None,
|
|
) -> None:
|
|
key = (src, tgt, relation)
|
|
if key in seen_edge_pairs:
|
|
return
|
|
seen_edge_pairs.add(key)
|
|
edge: dict[str, Any] = {
|
|
"source": src, "target": tgt, "relation": relation,
|
|
"confidence": "EXTRACTED", "source_file": str_path,
|
|
"source_location": f"L{line}", "weight": 1.0,
|
|
}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
obj_re = re.compile(r"^\s*object\s+\w+\s*:\s*(\w+)", re.IGNORECASE)
|
|
event_re = re.compile(r"^\s*On\w+\s*=\s*(\w+)", re.IGNORECASE)
|
|
end_re = re.compile(r"^\s*end\s*$", re.IGNORECASE)
|
|
stack: list[str] = [file_nid]
|
|
|
|
for lineno, line in enumerate(text.splitlines(), 1):
|
|
m = obj_re.match(line)
|
|
if m:
|
|
class_name = m.group(1)
|
|
nid = _make_id(stem, class_name)
|
|
add_node(nid, class_name, lineno)
|
|
add_edge(stack[-1], nid, "contains", lineno)
|
|
stack.append(nid)
|
|
continue
|
|
m = event_re.match(line)
|
|
if m and len(stack) > 1:
|
|
handler = m.group(1)
|
|
handler_nid = _make_id(stem, handler)
|
|
add_node(handler_nid, f"{handler}()", lineno)
|
|
add_edge(stack[-1], handler_nid, "references", lineno, context="event")
|
|
continue
|
|
if end_re.match(line) and len(stack) > 1:
|
|
stack.pop()
|
|
|
|
return {"nodes": nodes, "edges": edges, "input_tokens": 0, "output_tokens": 0}
|
|
|
|
|
|
# Size cap for project XML files we parse with stdlib ElementTree.
|
|
# Real .csproj/.fsproj/.vbproj/.lpk files are well under 2 MiB; anything
|
|
# larger is either malformed or hostile.
|
|
_PROJECT_XML_MAX_BYTES = 2 * 1024 * 1024
|
|
|
|
|
|
def _project_xml_is_safe(src: bytes) -> bool:
|
|
"""Reject XML that declares DTDs or entities.
|
|
|
|
Stdlib ``xml.etree.ElementTree`` does not cap entity expansion, so a
|
|
crafted project file could trigger a billion-laughs style DoS. External
|
|
entity resolution is already disabled by pyexpat defaults, but rejecting
|
|
``<!DOCTYPE`` / ``<!ENTITY`` outright is defense in depth.
|
|
|
|
Legitimate MSBuild and Lazarus package files never contain a DOCTYPE
|
|
or ENTITY declaration, so this is a zero-false-positive screen.
|
|
"""
|
|
# Only the prolog can hold a DTD/internal subset, but be conservative
|
|
# and scan the full byte range -- these formats use ASCII tags so a
|
|
# case-insensitive substring match is sufficient.
|
|
lowered = src.lower()
|
|
return b"<!doctype" not in lowered and b"<!entity" not in lowered
|
|
|
|
|
|
def extract_lazarus_package(path: Path) -> dict:
|
|
"""Extract package metadata from Lazarus .lpk package files (XML format).
|
|
|
|
.lpk is an XML file listing the package name, required dependencies,
|
|
and the Pascal units that belong to the package.
|
|
|
|
Produces nodes for:
|
|
- The package file itself
|
|
- The package (by name)
|
|
- Each required package (dependency)
|
|
- Each listed unit file (resolved to path-based IDs where possible)
|
|
|
|
Produces edges for:
|
|
- file --contains--> package
|
|
- package --imports--> required dependency (context: "import")
|
|
- package --contains--> listed unit
|
|
"""
|
|
try:
|
|
import xml.etree.ElementTree as ET
|
|
src = path.read_bytes()
|
|
except OSError as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
if len(src) > _PROJECT_XML_MAX_BYTES:
|
|
return {"nodes": [], "edges": [], "error": "package file too large"}
|
|
if not _project_xml_is_safe(src):
|
|
return {"nodes": [], "edges": [],
|
|
"error": "refusing XML with DOCTYPE/ENTITY declaration"}
|
|
|
|
try:
|
|
xml_root = ET.fromstring(src)
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
str_path = str(path)
|
|
stem = _file_stem(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
|
|
def add_node(nid: str, label: str) -> None:
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({
|
|
"id": nid, "label": label, "file_type": "code",
|
|
"source_file": str_path, "source_location": "L1",
|
|
})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, context: str | None = None) -> None:
|
|
edge: dict[str, Any] = {
|
|
"source": src, "target": tgt, "relation": relation,
|
|
"confidence": "EXTRACTED", "source_file": str_path,
|
|
"source_location": "L1", "weight": 1.0,
|
|
}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name)
|
|
|
|
name_elem = xml_root.find(".//Package/Name")
|
|
pkg_name = name_elem.get("Value") if name_elem is not None else path.stem
|
|
pkg_nid = _make_id(stem, pkg_name)
|
|
add_node(pkg_nid, pkg_name)
|
|
add_edge(file_nid, pkg_nid, "contains")
|
|
|
|
# Required packages → imports edges
|
|
for item in xml_root.findall(".//RequiredPkgs/"):
|
|
dep_elem = item.find("PackageName")
|
|
if dep_elem is not None:
|
|
dep_name = dep_elem.get("Value", "")
|
|
if dep_name:
|
|
dep_nid = _make_id(dep_name)
|
|
add_node(dep_nid, dep_name)
|
|
add_edge(pkg_nid, dep_nid, "imports", context="import")
|
|
|
|
# Listed units → contains edges, resolved to path-based IDs where possible
|
|
for item in xml_root.findall(".//Files/"):
|
|
unit_elem = item.find("UnitName")
|
|
if unit_elem is not None:
|
|
unit_name = unit_elem.get("Value", "")
|
|
if unit_name:
|
|
unit_nid = _pascal_resolve_unit(path, unit_name)
|
|
add_node(unit_nid, unit_name)
|
|
add_edge(pkg_nid, unit_nid, "contains")
|
|
|
|
return {"nodes": nodes, "edges": edges, "input_tokens": 0, "output_tokens": 0}
|
|
|
|
|
|
# ── Main extract and collect_files ────────────────────────────────────────────
|
|
|
|
|
|
def _check_tree_sitter_version() -> None:
|
|
"""Raise a clear error if tree-sitter is too old for the new Language API."""
|
|
try:
|
|
from tree_sitter import LANGUAGE_VERSION
|
|
except ImportError:
|
|
raise ImportError(
|
|
"tree-sitter is not installed. Run: pip install 'tree-sitter>=0.23.0'"
|
|
)
|
|
# Language API v2 starts at LANGUAGE_VERSION 14
|
|
if LANGUAGE_VERSION < 14:
|
|
import tree_sitter as _ts
|
|
raise RuntimeError(
|
|
f"tree-sitter {getattr(_ts, '__version__', 'unknown')} is too old. "
|
|
f"graphify requires tree-sitter >= 0.23.0 (Language API v2). "
|
|
f"Run: pip install --upgrade tree-sitter"
|
|
)
|
|
|
|
|
|
def extract_bash(path: Path) -> dict:
|
|
"""Extract functions, source imports, and cross-function calls from a .sh file."""
|
|
try:
|
|
import tree_sitter_bash as tsbash
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": "tree-sitter-bash not installed"}
|
|
|
|
try:
|
|
language = Language(tsbash.language())
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
function_bodies: list[tuple[str, Any]] = []
|
|
defined_functions: set[str] = set()
|
|
|
|
from graphify.security import sanitize_metadata # module-level cached import
|
|
|
|
def add_node(nid: str, label: str, line: int, kind: str = "code") -> None:
|
|
if nid and nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({"id": nid, "label": label, "file_type": "code",
|
|
"source_file": str_path, "source_location": f"L{line}",
|
|
"metadata": sanitize_metadata({"language": "bash", "kind": kind})}) # noqa: E501
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", weight: float = 1.0,
|
|
context: str | None = None) -> None:
|
|
if not src or not tgt or src == tgt:
|
|
return
|
|
edge = {"source": src, "target": tgt, "relation": relation,
|
|
"confidence": confidence, "source_file": str_path,
|
|
"source_location": f"L{line}", "weight": weight}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
file_nid = _make_id(str(path))
|
|
# file_nid is fully path-derived and never produced by _make_id(stem, func_name),
|
|
# so appending "__entry" guarantees a distinct ID from any function node.
|
|
entry_nid = file_nid + "__entry"
|
|
add_node(file_nid, path.name, 1, kind="file")
|
|
add_node(entry_nid, f"{path.name} script", 1, kind="bash_entrypoint")
|
|
add_edge(file_nid, entry_nid, "contains", 1)
|
|
|
|
_BASH_SOURCE_COMMANDS = frozenset({"source", "."})
|
|
# Parent node types that mean a contained command is part of a substitution
|
|
# or expansion, not a real function call. Token-level filtering misses
|
|
# these because `$(build)` exposes `build` as a child command whose name
|
|
# token has no metacharacters — only the parent does.
|
|
_BASH_EXPANSION_PARENTS = frozenset({
|
|
"command_substitution",
|
|
"process_substitution",
|
|
})
|
|
|
|
def text(node) -> str:
|
|
return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
|
|
|
|
def is_inside_expansion(node) -> bool:
|
|
parent = node.parent
|
|
while parent is not None:
|
|
if parent.type in _BASH_EXPANSION_PARENTS:
|
|
return True
|
|
parent = parent.parent
|
|
return False
|
|
|
|
def literal(node) -> str | None:
|
|
# Token-level filter: rejects names containing shell metacharacters.
|
|
# Combined with `is_inside_expansion` for parent-context rejection.
|
|
raw = text(node).strip()
|
|
if not raw:
|
|
return None
|
|
if raw[0:1] in {"'", '"'} and raw[-1:] == raw[0]:
|
|
raw = raw[1:-1]
|
|
if any(token in raw for token in ("$", "`", "$(", "<(", ">", "|", ";", "&")):
|
|
return None
|
|
return raw
|
|
|
|
def _bash_func_name(node) -> str | None:
|
|
"""Get the name from a function_definition node."""
|
|
# bash grammar: function_definition has a word child (the name)
|
|
for child in node.children:
|
|
if child.type == "word":
|
|
return literal(child)
|
|
return None
|
|
|
|
def walk_calls(body_node, func_nid: str, seen_calls: set) -> None:
|
|
if body_node is None:
|
|
return
|
|
for child in body_node.children:
|
|
if child.type == "function_definition":
|
|
# Skip nested function definitions — their bodies are walked
|
|
# separately, so we don't attribute their calls to the
|
|
# enclosing scope.
|
|
continue
|
|
if child.type == "command" and not is_inside_expansion(child):
|
|
cmd_name_node = child.child_by_field_name("name")
|
|
if cmd_name_node is None and child.children:
|
|
cmd_name_node = child.children[0]
|
|
if cmd_name_node:
|
|
name = literal(cmd_name_node)
|
|
# Defined-functions wins. Skip-lists for external commands
|
|
# would create false negatives when a user defines a
|
|
# function shadowing an external (`install`, `find`, etc.).
|
|
if name and name in defined_functions:
|
|
tgt = _make_id(stem, name)
|
|
key = (func_nid, tgt)
|
|
if tgt and key not in seen_calls:
|
|
seen_calls.add(key)
|
|
add_edge(func_nid, tgt, "calls",
|
|
child.start_point[0] + 1,
|
|
confidence="EXTRACTED", context="call")
|
|
walk_calls(child, func_nid, seen_calls)
|
|
|
|
def walk(node, parent_nid: str) -> None:
|
|
t = node.type
|
|
if t == "function_definition":
|
|
name = _bash_func_name(node)
|
|
if name:
|
|
fn_nid = _make_id(stem, name)
|
|
line = node.start_point[0] + 1
|
|
add_node(fn_nid, f"{name}()", line, kind="bash_function")
|
|
add_edge(parent_nid, fn_nid, "defines", line)
|
|
defined_functions.add(name)
|
|
# find the compound_statement body
|
|
body = None
|
|
for child in node.children:
|
|
if child.type == "compound_statement":
|
|
body = child
|
|
break
|
|
function_bodies.append((fn_nid, body))
|
|
# Recurse into the body so nested function definitions are discovered
|
|
# and added to function_bodies for the second-pass walk_calls.
|
|
if body is not None:
|
|
walk(body, fn_nid)
|
|
return
|
|
|
|
if t == "command":
|
|
if is_inside_expansion(node):
|
|
return
|
|
cmd_name_node = node.child_by_field_name("name")
|
|
if cmd_name_node is None and node.children:
|
|
cmd_name_node = node.children[0]
|
|
if cmd_name_node:
|
|
cmd = literal(cmd_name_node)
|
|
if cmd in _BASH_SOURCE_COMMANDS and cmd not in defined_functions:
|
|
# find the path argument (first word after command name)
|
|
args = [c for c in node.children
|
|
if c.type in ("word", "string", "concatenation")
|
|
and c != cmd_name_node]
|
|
if args:
|
|
raw = _read_text(args[0], source).strip().strip("'\"")
|
|
line = node.start_point[0] + 1
|
|
if raw.startswith((".", "/")):
|
|
resolved = (path.parent / raw).resolve()
|
|
# Only emit the edge if the target actually exists on
|
|
# disk — prevents graph pollution from crafted paths
|
|
# like `source ../../etc/passwd` that traverse outside
|
|
# the project tree (B-1).
|
|
if resolved.exists():
|
|
tgt_nid = _make_id(str(resolved))
|
|
add_edge(file_nid, tgt_nid, "imports_from", line,
|
|
context="import")
|
|
else:
|
|
tgt_nid = _make_id(raw)
|
|
if tgt_nid:
|
|
add_edge(file_nid, tgt_nid, "imports", line,
|
|
context="import")
|
|
return
|
|
|
|
if t == "declaration_command":
|
|
# export/declare/readonly VAR=value at program level
|
|
if node.parent and node.parent.type == "program":
|
|
for child in node.children:
|
|
if child.type == "variable_assignment":
|
|
var_node = child.child_by_field_name("name")
|
|
if var_node:
|
|
var = _read_text(var_node, source).strip()
|
|
if var:
|
|
var_nid = _make_id(stem, var)
|
|
line = child.start_point[0] + 1
|
|
add_node(var_nid, var, line)
|
|
add_edge(file_nid, var_nid, "defines", line)
|
|
return
|
|
|
|
for child in node.children:
|
|
walk(child, parent_nid)
|
|
|
|
# Pre-pass: collect all defined function names so the source-command handler
|
|
# in walk() can detect user-defined functions that shadow 'source' / '.'
|
|
# regardless of definition order in the file.
|
|
def _prescan_functions(node) -> None:
|
|
if node.type == "function_definition":
|
|
name = _bash_func_name(node)
|
|
if name:
|
|
defined_functions.add(name)
|
|
for child in node.children:
|
|
_prescan_functions(child)
|
|
else:
|
|
for child in node.children:
|
|
_prescan_functions(child)
|
|
|
|
_prescan_functions(root)
|
|
walk(root, file_nid)
|
|
|
|
# Second pass: cross-function calls
|
|
top_seen: set = set()
|
|
walk_calls(root, entry_nid, top_seen) # top-level calls attributed to the entrypoint
|
|
for fn_nid, body in function_bodies:
|
|
walk_calls(body, fn_nid, set())
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
# ── .NET project files (.sln, .slnx, .csproj, .razor) ───────────────────────
|
|
|
|
def extract_sln(path: Path) -> dict:
|
|
"""Extract projects and inter-project dependencies from a .sln file."""
|
|
try:
|
|
src = path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
return {"nodes": [], "edges": [], "error": f"cannot read {path}"}
|
|
|
|
file_nid = _make_id(str(path))
|
|
str_path = str(path)
|
|
nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code",
|
|
"source_file": str_path, "source_location": None}]
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
seen_ids.add(file_nid)
|
|
|
|
_PROJECT_RE = re.compile(
|
|
r'Project\("[^"]*"\)\s*=\s*"([^"]+)"\s*,\s*"([^"]+)"\s*,\s*"([^"]*)"'
|
|
)
|
|
_DEP_RE = re.compile(r'\{([0-9a-fA-F-]+)\}\s*=\s*\{([0-9a-fA-F-]+)\}')
|
|
|
|
guid_to_nid: dict[str, str] = {}
|
|
|
|
for m in _PROJECT_RE.finditer(src):
|
|
proj_name = m.group(1)
|
|
proj_path = m.group(2).replace("\\", "/")
|
|
proj_guid = m.group(3).strip("{}")
|
|
|
|
try:
|
|
abs_proj = str((path.parent / proj_path).resolve())
|
|
except Exception:
|
|
abs_proj = proj_path
|
|
proj_nid = _make_id(abs_proj)
|
|
if proj_nid and proj_nid not in seen_ids:
|
|
seen_ids.add(proj_nid)
|
|
nodes.append({"id": proj_nid, "label": proj_name,
|
|
"file_type": "code", "source_file": abs_proj,
|
|
"source_location": None})
|
|
edges.append({"source": file_nid, "target": proj_nid,
|
|
"relation": "contains", "confidence": "EXTRACTED",
|
|
"source_file": str_path, "weight": 1.0})
|
|
if proj_guid:
|
|
guid_to_nid[proj_guid.lower()] = proj_nid
|
|
|
|
in_dep_section = False
|
|
current_proj_guid: str | None = None
|
|
_PROJECT_LINE_RE = re.compile(r'Project\("[^"]*"\)\s*=\s*"[^"]+"\s*,\s*"[^"]+"\s*,\s*"\{([^}]+)\}"')
|
|
for line in src.splitlines():
|
|
proj_line_m = _PROJECT_LINE_RE.search(line)
|
|
if proj_line_m:
|
|
current_proj_guid = proj_line_m.group(1).lower()
|
|
continue
|
|
if line.strip() == "EndProject":
|
|
current_proj_guid = None
|
|
continue
|
|
if "ProjectSection(ProjectDependencies)" in line:
|
|
in_dep_section = True
|
|
continue
|
|
if in_dep_section and "EndProjectSection" in line:
|
|
in_dep_section = False
|
|
continue
|
|
if in_dep_section and current_proj_guid:
|
|
dep_m = _DEP_RE.search(line)
|
|
if dep_m:
|
|
to_guid = dep_m.group(1).lower()
|
|
from_nid = guid_to_nid.get(current_proj_guid)
|
|
to_nid = guid_to_nid.get(to_guid)
|
|
if from_nid and to_nid and from_nid != to_nid:
|
|
edges.append({"source": from_nid, "target": to_nid,
|
|
"relation": "imports", "confidence": "EXTRACTED",
|
|
"source_file": str_path, "weight": 1.0})
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
def extract_slnx(path: Path) -> dict:
|
|
"""Extract projects and inter-project dependencies from a .slnx file.
|
|
|
|
.slnx is the XML-based replacement for the legacy .sln format. Projects
|
|
are listed as ``<Project Path="..."/>`` elements (optionally nested inside
|
|
``<Folder>`` elements) and build-order dependencies as ``<BuildDependency
|
|
Project="..."/>`` children. Unlike .sln there are no GUIDs -- projects are
|
|
identified by their path.
|
|
"""
|
|
import xml.etree.ElementTree as ET
|
|
|
|
try:
|
|
src = path.read_bytes()
|
|
except OSError:
|
|
return {"nodes": [], "edges": [], "error": f"cannot read {path}"}
|
|
|
|
if len(src) > _PROJECT_XML_MAX_BYTES:
|
|
return {"nodes": [], "edges": [], "error": "project file too large"}
|
|
if not _project_xml_is_safe(src):
|
|
return {"nodes": [], "edges": [],
|
|
"error": "refusing XML with DOCTYPE/ENTITY declaration"}
|
|
|
|
try:
|
|
tree = ET.fromstring(src)
|
|
except ET.ParseError as e:
|
|
return {"nodes": [], "edges": [], "error": f"XML parse error: {e}"}
|
|
|
|
file_nid = _make_id(str(path))
|
|
str_path = str(path)
|
|
nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code",
|
|
"source_file": str_path, "source_location": None}]
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
seen_ids.add(file_nid)
|
|
|
|
ns = ""
|
|
if tree.tag.startswith("{"):
|
|
ns = tree.tag.split("}")[0] + "}"
|
|
|
|
def _resolve(proj_path: str) -> str:
|
|
proj_path = proj_path.replace("\\", "/")
|
|
try:
|
|
return str((path.parent / proj_path).resolve())
|
|
except Exception:
|
|
return proj_path
|
|
|
|
# First pass: collect projects (anywhere in the tree, incl. <Folder>).
|
|
project_nids: set[str] = set()
|
|
for proj in tree.iter(f"{ns}Project"):
|
|
proj_path = proj.get("Path")
|
|
if not proj_path:
|
|
continue
|
|
abs_proj = _resolve(proj_path)
|
|
proj_nid = _make_id(abs_proj)
|
|
if proj_nid and proj_nid not in seen_ids:
|
|
seen_ids.add(proj_nid)
|
|
label = Path(proj_path).stem
|
|
nodes.append({"id": proj_nid, "label": label,
|
|
"file_type": "code", "source_file": abs_proj,
|
|
"source_location": None})
|
|
edges.append({"source": file_nid, "target": proj_nid,
|
|
"relation": "contains", "confidence": "EXTRACTED",
|
|
"source_file": str_path, "weight": 1.0})
|
|
if proj_nid:
|
|
project_nids.add(proj_nid)
|
|
|
|
# Second pass: build-order dependencies between known projects.
|
|
for proj in tree.iter(f"{ns}Project"):
|
|
proj_path = proj.get("Path")
|
|
if not proj_path:
|
|
continue
|
|
from_nid = _make_id(_resolve(proj_path))
|
|
for dep in proj.iter(f"{ns}BuildDependency"):
|
|
dep_path = dep.get("Project")
|
|
if not dep_path:
|
|
continue
|
|
to_nid = _make_id(_resolve(dep_path))
|
|
if (from_nid and to_nid and from_nid != to_nid
|
|
and to_nid in project_nids):
|
|
edges.append({"source": from_nid, "target": to_nid,
|
|
"relation": "imports", "confidence": "EXTRACTED",
|
|
"source_file": str_path, "weight": 1.0})
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
def extract_csproj(path: Path) -> dict:
|
|
"""Extract packages, project refs, and target framework from a .csproj/.fsproj/.vbproj."""
|
|
import xml.etree.ElementTree as ET
|
|
|
|
try:
|
|
src = path.read_bytes()
|
|
except OSError:
|
|
return {"nodes": [], "edges": [], "error": f"cannot read {path}"}
|
|
|
|
if len(src) > _PROJECT_XML_MAX_BYTES:
|
|
return {"nodes": [], "edges": [], "error": "project file too large"}
|
|
if not _project_xml_is_safe(src):
|
|
return {"nodes": [], "edges": [],
|
|
"error": "refusing XML with DOCTYPE/ENTITY declaration"}
|
|
|
|
try:
|
|
tree = ET.fromstring(src)
|
|
except ET.ParseError as e:
|
|
return {"nodes": [], "edges": [], "error": f"XML parse error: {e}"}
|
|
|
|
file_nid = _make_id(str(path))
|
|
str_path = str(path)
|
|
nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code",
|
|
"source_file": str_path, "source_location": None}]
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
seen_ids.add(file_nid)
|
|
|
|
ns = ""
|
|
root_tag = tree.tag
|
|
if root_tag.startswith("{"):
|
|
ns = root_tag.split("}")[0] + "}"
|
|
|
|
def find_all(tag: str):
|
|
return tree.iter(f"{ns}{tag}")
|
|
|
|
for tf in find_all("TargetFramework"):
|
|
if tf.text:
|
|
fw_nid = _make_id("framework", tf.text.strip())
|
|
if fw_nid and fw_nid not in seen_ids:
|
|
seen_ids.add(fw_nid)
|
|
nodes.append({"id": fw_nid, "label": tf.text.strip(),
|
|
"file_type": "concept", "source_file": str_path,
|
|
"source_location": None})
|
|
edges.append({"source": file_nid, "target": fw_nid,
|
|
"relation": "references", "confidence": "EXTRACTED",
|
|
"source_file": str_path, "weight": 1.0})
|
|
|
|
for tf in find_all("TargetFrameworks"):
|
|
if tf.text:
|
|
for fw in tf.text.strip().split(";"):
|
|
fw = fw.strip()
|
|
if fw:
|
|
fw_nid = _make_id("framework", fw)
|
|
if fw_nid and fw_nid not in seen_ids:
|
|
seen_ids.add(fw_nid)
|
|
nodes.append({"id": fw_nid, "label": fw,
|
|
"file_type": "concept", "source_file": str_path,
|
|
"source_location": None})
|
|
edges.append({"source": file_nid, "target": fw_nid,
|
|
"relation": "references", "confidence": "EXTRACTED",
|
|
"source_file": str_path, "weight": 1.0})
|
|
|
|
for pkg in find_all("PackageReference"):
|
|
name = pkg.get("Include") or pkg.get("include") or ""
|
|
version = pkg.get("Version") or pkg.get("version") or ""
|
|
if not name:
|
|
continue
|
|
pkg_nid = _make_id("nuget", name)
|
|
label = f"{name} ({version})" if version else name
|
|
if pkg_nid and pkg_nid not in seen_ids:
|
|
seen_ids.add(pkg_nid)
|
|
nodes.append({"id": pkg_nid, "label": label,
|
|
"file_type": "code", "source_file": str_path,
|
|
"source_location": None})
|
|
edges.append({"source": file_nid, "target": pkg_nid,
|
|
"relation": "imports", "confidence": "EXTRACTED",
|
|
"source_file": str_path, "weight": 1.0})
|
|
|
|
for proj in find_all("ProjectReference"):
|
|
ref_path = proj.get("Include") or proj.get("include") or ""
|
|
if not ref_path:
|
|
continue
|
|
ref_path_norm = ref_path.replace("\\", "/")
|
|
try:
|
|
abs_ref = str((path.parent / ref_path_norm).resolve())
|
|
except Exception:
|
|
abs_ref = ref_path_norm
|
|
proj_nid = _make_id(abs_ref)
|
|
if proj_nid and proj_nid not in seen_ids:
|
|
seen_ids.add(proj_nid)
|
|
proj_label = Path(ref_path_norm).name
|
|
nodes.append({"id": proj_nid, "label": proj_label,
|
|
"file_type": "code", "source_file": abs_ref,
|
|
"source_location": None})
|
|
edges.append({"source": file_nid, "target": proj_nid,
|
|
"relation": "imports", "confidence": "EXTRACTED",
|
|
"source_file": str_path, "weight": 1.0})
|
|
|
|
sdk = tree.get("Sdk") or ""
|
|
if sdk:
|
|
sdk_nid = _make_id("sdk", sdk)
|
|
if sdk_nid and sdk_nid not in seen_ids:
|
|
seen_ids.add(sdk_nid)
|
|
nodes.append({"id": sdk_nid, "label": sdk,
|
|
"file_type": "concept", "source_file": str_path,
|
|
"source_location": None})
|
|
edges.append({"source": file_nid, "target": sdk_nid,
|
|
"relation": "references", "confidence": "EXTRACTED",
|
|
"source_file": str_path, "weight": 1.0})
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
def extract_razor(path: Path) -> dict:
|
|
"""Extract directives, component refs, and @code methods from .razor/.cshtml."""
|
|
try:
|
|
src = path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
return {"nodes": [], "edges": [], "error": f"cannot read {path}"}
|
|
|
|
file_nid = _make_id(str(path))
|
|
str_path = str(path)
|
|
nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code",
|
|
"source_file": str_path, "source_location": None}]
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
seen_ids.add(file_nid)
|
|
|
|
def _add_ref(target_name: str, relation: str, line: int) -> None:
|
|
tgt_nid = _make_id(target_name)
|
|
if not tgt_nid:
|
|
return
|
|
if tgt_nid not in seen_ids:
|
|
seen_ids.add(tgt_nid)
|
|
nodes.append({"id": tgt_nid, "label": target_name,
|
|
"file_type": "code", "source_file": str_path,
|
|
"source_location": f"L{line}"})
|
|
edges.append({"source": file_nid, "target": tgt_nid,
|
|
"relation": relation, "confidence": "EXTRACTED",
|
|
"source_file": str_path, "source_location": f"L{line}",
|
|
"weight": 1.0})
|
|
|
|
for i, line in enumerate(src.splitlines(), 1):
|
|
m = re.match(r'@using\s+([\w.]+)', line)
|
|
if m:
|
|
_add_ref(m.group(1), "imports", i)
|
|
continue
|
|
|
|
m = re.match(r'@inject\s+([\w.<>\[\]]+)\s+(\w+)', line)
|
|
if m:
|
|
_add_ref(m.group(1), "imports", i)
|
|
continue
|
|
|
|
m = re.match(r'@inherits\s+([\w.<>\[\]]+)', line)
|
|
if m:
|
|
_add_ref(m.group(1), "inherits", i)
|
|
continue
|
|
|
|
m = re.match(r'@model\s+([\w.<>\[\]]+)', line)
|
|
if m:
|
|
_add_ref(m.group(1), "references", i)
|
|
continue
|
|
|
|
m = re.match(r'@page\s+"([^"]+)"', line)
|
|
if m:
|
|
route = m.group(1)
|
|
route_nid = _make_id("route", route)
|
|
if route_nid and route_nid not in seen_ids:
|
|
seen_ids.add(route_nid)
|
|
nodes.append({"id": route_nid, "label": f"route:{route}",
|
|
"file_type": "concept", "source_file": str_path,
|
|
"source_location": f"L{i}"})
|
|
edges.append({"source": file_nid, "target": route_nid,
|
|
"relation": "references", "confidence": "EXTRACTED",
|
|
"source_file": str_path, "weight": 1.0})
|
|
continue
|
|
|
|
_COMPONENT_RE = re.compile(r'<([A-Z][A-Za-z0-9]+)[\s/>]')
|
|
_HTML_TAGS = frozenset({
|
|
"DOCTYPE", "Html", "Head", "Body", "Div", "Span", "Table", "Form",
|
|
"Input", "Button", "Select", "Option", "Label", "Textarea",
|
|
"Script", "Style", "Link", "Meta", "Title", "Header", "Footer",
|
|
"Nav", "Main", "Section", "Article", "Aside",
|
|
})
|
|
for m in _COMPONENT_RE.finditer(src):
|
|
comp_name = m.group(1)
|
|
if comp_name in _HTML_TAGS:
|
|
continue
|
|
line_num = src[:m.start()].count("\n") + 1
|
|
_add_ref(comp_name, "calls", line_num)
|
|
|
|
_CODE_BLOCK_RE = re.compile(r'@code\s*\{', re.MULTILINE)
|
|
for m in _CODE_BLOCK_RE.finditer(src):
|
|
block_start = m.end()
|
|
depth = 1
|
|
pos = block_start
|
|
while pos < len(src) and depth > 0:
|
|
if src[pos] == '{':
|
|
depth += 1
|
|
elif src[pos] == '}':
|
|
depth -= 1
|
|
pos += 1
|
|
code_block = src[block_start:pos - 1] if depth == 0 else ""
|
|
|
|
_METHOD_RE = re.compile(
|
|
r'(?:public|private|protected|internal|static|async|override|virtual|abstract)\s+'
|
|
r'[\w<>\[\],\s]+\s+(\w+)\s*\('
|
|
)
|
|
for mm in _METHOD_RE.finditer(code_block):
|
|
method_name = mm.group(1)
|
|
abs_pos = block_start + mm.start()
|
|
method_line = src[:abs_pos].count("\n") + 1
|
|
method_nid = _make_id(_file_stem(path), method_name)
|
|
if method_nid and method_nid not in seen_ids:
|
|
seen_ids.add(method_nid)
|
|
nodes.append({"id": method_nid, "label": method_name,
|
|
"file_type": "code", "source_file": str_path,
|
|
"source_location": f"L{method_line}"})
|
|
edges.append({"source": file_nid, "target": method_nid,
|
|
"relation": "contains", "confidence": "EXTRACTED",
|
|
"source_file": str_path, "weight": 1.0})
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
# Config/manifest JSON filenames the structural extractor understands. Anything
|
|
# else (eval fixtures, datasets, GeoJSON, API dumps) is *data* and must NOT be
|
|
# AST-walked into per-key nodes — that floods the graph with orphan key-nodes
|
|
# and near-duplicate communities (#1224). Data JSON is left to the LLM semantic
|
|
# pass instead. Matched case-insensitively against the bare filename.
|
|
_CONFIG_JSON_NAMES = frozenset({
|
|
"package.json", "tsconfig.json", "jsconfig.json", "composer.json",
|
|
"deno.json", "deno.jsonc", "bower.json", "manifest.json",
|
|
"app.json", "now.json", "vercel.json", "angular.json", "nest-cli.json",
|
|
"biome.json", "biome.jsonc", "renovate.json", ".babelrc", ".babelrc.json",
|
|
".eslintrc.json", ".prettierrc.json", ".prettierrc", "babel.config.json",
|
|
})
|
|
|
|
# Top-level keys that prove a JSON object is a config/manifest the extractor can
|
|
# draw *cross-file* edges from (deps, extends chains, schema refs).
|
|
_CONFIG_JSON_KEYS = frozenset({
|
|
"dependencies", "devDependencies", "peerDependencies",
|
|
"optionalDependencies", "bundleDependencies", "bundledDependencies",
|
|
"extends", "$ref", "$schema", "compilerOptions",
|
|
})
|
|
|
|
|
|
def _is_config_json(path: Path, obj_node, source: bytes) -> bool:
|
|
"""True if a .json file is a recognized config/manifest worth AST-extracting.
|
|
|
|
Matches by filename first (cheap), then falls back to a top-level key probe
|
|
so arbitrarily-named config files (e.g. ``api.tsconfig.json``,
|
|
``foo.eslintrc.json``) are still picked up. Returns False for data JSON so it
|
|
is skipped by the structural pass (#1224)."""
|
|
name = path.name.casefold()
|
|
if name in _CONFIG_JSON_NAMES:
|
|
return True
|
|
# Common compound config names: *.eslintrc.json, *.prettierrc.json, etc.
|
|
if name.endswith((".eslintrc.json", ".prettierrc.json", ".babelrc.json",
|
|
"tsconfig.json", "jsconfig.json")):
|
|
return True
|
|
# Top-level key probe: scan the root object's immediate keys (no deep walk).
|
|
for top_key in obj_node.children:
|
|
if top_key.type != "pair":
|
|
continue
|
|
key_node = top_key.child_by_field_name("key")
|
|
if key_node is None:
|
|
continue
|
|
kc = key_node.child_by_field_name("string_content")
|
|
text = _read_text(kc, source) if kc else _read_text(key_node, source).strip('"\'')
|
|
if text in _CONFIG_JSON_KEYS:
|
|
return True
|
|
return False
|
|
|
|
|
|
def extract_json(path: Path) -> dict:
|
|
"""Extract structure and dependency edges from a *config/manifest* .json file.
|
|
|
|
Data-shaped JSON (eval fixtures, datasets, GeoJSON, API response dumps) is
|
|
deliberately skipped — AST-walking it produced hundreds of orphan key-nodes
|
|
and duplicate communities that swamped real structure (#1224). Recognition
|
|
is by filename (package.json, tsconfig.json, …) or a top-level key probe
|
|
(dependencies / extends / $ref / $schema / compilerOptions)."""
|
|
_JSON_MAX_BYTES = 1_048_576 # 1 MiB — skip large fixture dumps / GeoJSON blobs
|
|
|
|
try:
|
|
import tree_sitter_json as tsjson
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": "tree-sitter-json not installed"}
|
|
|
|
try:
|
|
# Bounded read instead of stat()+read() to eliminate TOCTOU (J-1):
|
|
# read one byte beyond the limit so we can detect oversized files even
|
|
# if the file grows between stat and read.
|
|
with path.open("rb") as _f:
|
|
source = _f.read(_JSON_MAX_BYTES + 1)
|
|
if len(source) > _JSON_MAX_BYTES:
|
|
return {"nodes": [], "edges": [], "error": "json file too large to index"}
|
|
language = Language(tsjson.language())
|
|
parser = Parser(language)
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
|
|
# Keys whose string values become imports (package.json dep blocks)
|
|
_DEP_KEYS = frozenset({
|
|
"dependencies", "devDependencies", "peerDependencies",
|
|
"optionalDependencies", "bundleDependencies", "bundledDependencies",
|
|
})
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid and nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({"id": nid, "label": label, "file_type": "code",
|
|
"source_file": str_path, "source_location": f"L{line}"})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
context: str | None = None) -> None:
|
|
if not src or not tgt or src == tgt:
|
|
return
|
|
edge = {"source": src, "target": tgt, "relation": relation,
|
|
"confidence": "EXTRACTED", "source_file": str_path,
|
|
"source_location": f"L{line}", "weight": 1.0}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
def _key_text(pair_node) -> str | None:
|
|
"""Extract the string content of a pair's key."""
|
|
key_node = pair_node.child_by_field_name("key")
|
|
if key_node is None:
|
|
return None
|
|
if key_node.type == "string":
|
|
content = key_node.child_by_field_name("string_content")
|
|
if content:
|
|
return _read_text(content, source)
|
|
# fallback: strip surrounding quotes
|
|
raw = _read_text(key_node, source)
|
|
return raw.strip('"\'')
|
|
return _read_text(key_node, source)
|
|
|
|
def _val_node(pair_node):
|
|
return pair_node.child_by_field_name("value")
|
|
|
|
def walk_object(obj_node, parent_nid: str, parent_key: str | None,
|
|
depth: int, pair_count: list) -> None:
|
|
if depth > 6:
|
|
return
|
|
for child in obj_node.children:
|
|
if child.type != "pair":
|
|
continue
|
|
if pair_count[0] >= 500: # check per-pair so the cap is honoured exactly (J-3)
|
|
return
|
|
pair_count[0] += 1
|
|
key = _key_text(child)
|
|
if not key:
|
|
continue
|
|
key_nid = _make_id(stem, *(([parent_key] if parent_key else []) + [key]))
|
|
if not key_nid:
|
|
continue
|
|
line = child.start_point[0] + 1
|
|
add_node(key_nid, key, line)
|
|
add_edge(parent_nid, key_nid, "contains", line)
|
|
|
|
val = _val_node(child)
|
|
if val is None:
|
|
continue
|
|
|
|
if val.type == "object":
|
|
walk_object(val, key_nid, key, depth + 1, pair_count)
|
|
|
|
elif val.type == "array":
|
|
# For "extends" arrays (tsconfig, eslint): each string element.
|
|
# Prefix with "ref_" so external refs don't collide with real
|
|
# code/file node IDs that share the same collapsed _make_id (J-4).
|
|
for item in val.children:
|
|
if item.type == "string":
|
|
content = item.child_by_field_name("string_content")
|
|
ref = _read_text(content, source) if content else _read_text(item, source).strip('"\'')
|
|
if ref:
|
|
ref_nid = _make_id("ref", ref)
|
|
if ref_nid:
|
|
add_edge(key_nid, ref_nid, "extends", line, context="import")
|
|
|
|
elif val.type == "string":
|
|
content = val.child_by_field_name("string_content")
|
|
val_text = _read_text(content, source) if content else _read_text(val, source).strip('"\'')
|
|
|
|
if key == "extends" and val_text:
|
|
# Namespace external refs to avoid ID collision with file nodes (J-4)
|
|
ref_nid = _make_id("ref", val_text)
|
|
if ref_nid:
|
|
add_edge(file_nid, ref_nid, "extends", line, context="import")
|
|
|
|
elif key == "$ref" and val_text:
|
|
# Namespace $ref values to prevent edge hijacking into code nodes (J-4)
|
|
ref_nid = _make_id("ref", val_text)
|
|
if ref_nid:
|
|
add_edge(parent_nid, ref_nid, "references", line)
|
|
|
|
elif parent_key in _DEP_KEYS and val_text:
|
|
dep_nid = _make_id(key)
|
|
if dep_nid:
|
|
add_edge(key_nid, dep_nid, "imports", line, context="import")
|
|
|
|
# Entry: find root document → object
|
|
doc = root
|
|
if doc.type == "document" and doc.child_count > 0:
|
|
doc = doc.children[0]
|
|
if doc.type == "object":
|
|
# Only AST-extract recognized config/manifest JSON. Data JSON (fixtures,
|
|
# datasets, GeoJSON, API dumps) is skipped so it doesn't explode into
|
|
# orphan key-nodes (#1224); it's left to the LLM semantic pass.
|
|
if not _is_config_json(path, doc, source):
|
|
return {"nodes": [], "edges": [], "skipped": "data json (not a config/manifest)"}
|
|
walk_object(doc, file_nid, None, 0, [0])
|
|
else:
|
|
# Top-level array or scalar => data JSON, never a config/manifest.
|
|
return {"nodes": [], "edges": [], "skipped": "data json (non-object root)"}
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
# ── DM (BYOND DreamMaker) extractor ──────────────────────────────────────────
|
|
# DM identity is path-based (`/datum/object/proc/New()`), not block-based, so
|
|
# the generic class-body walker doesn't fit well.
|
|
|
|
def extract_dm(path: Path) -> dict:
|
|
"""Extract types, procs, includes, and calls from a .dm/.dme file."""
|
|
try:
|
|
import tree_sitter_dm as tsdm
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": "tree-sitter-dm not installed"}
|
|
try:
|
|
language = Language(tsdm.language())
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
stem = _file_stem(path)
|
|
str_path = str(path)
|
|
nodes: list[dict] = []
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = set()
|
|
function_bodies: list[tuple[str, Any, "str | None"]] = []
|
|
|
|
def add_node(nid: str, label: str, line: int) -> None:
|
|
if nid and nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({"id": nid, "label": label, "file_type": "code",
|
|
"source_file": str_path, "source_location": f"L{line}"})
|
|
|
|
def add_edge(src: str, tgt: str, relation: str, line: int,
|
|
confidence: str = "EXTRACTED", weight: float = 1.0,
|
|
context: str | None = None) -> None:
|
|
if not src or not tgt or src == tgt:
|
|
return
|
|
edge: dict = {"source": src, "target": tgt, "relation": relation,
|
|
"confidence": confidence, "source_file": str_path,
|
|
"source_location": f"L{line}", "weight": weight}
|
|
if context:
|
|
edge["context"] = context
|
|
edges.append(edge)
|
|
|
|
file_nid = _make_id(str(path))
|
|
add_node(file_nid, path.name, 1)
|
|
|
|
def _type_path_text(node) -> str:
|
|
return _read_text(node, source).strip()
|
|
|
|
def _ensure_type(path_text: str, line: int) -> str:
|
|
nid = _make_id(stem, path_text)
|
|
add_node(nid, path_text, line)
|
|
return nid
|
|
|
|
def _find_child(node, type_name: str):
|
|
for c in node.children:
|
|
if c.type == type_name:
|
|
return c
|
|
return None
|
|
|
|
def _read_include_path(file_node) -> str:
|
|
if file_node is None:
|
|
return ""
|
|
if file_node.type == "string_literal":
|
|
parts = []
|
|
for c in file_node.children:
|
|
if c.type == "string_content":
|
|
parts.append(_read_text(c, source))
|
|
return "".join(parts)
|
|
return _read_text(file_node, source).strip("'\"")
|
|
|
|
def walk(node, parent_type_path: "str | None" = None,
|
|
parent_type_nid: "str | None" = None) -> None:
|
|
t = node.type
|
|
line = node.start_point[0] + 1
|
|
|
|
if t == "preproc_include":
|
|
file_node = node.child_by_field_name("file")
|
|
raw = _read_include_path(file_node)
|
|
if raw:
|
|
norm = raw.replace("\\", "/").lstrip("./")
|
|
resolved = (path.parent / norm).resolve()
|
|
edge: dict = {
|
|
"source": file_nid,
|
|
"target": _make_id(str(resolved)) if resolved.exists() else _make_id(norm),
|
|
"relation": "imports_from" if resolved.exists() else "imports",
|
|
"context": "import",
|
|
"confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
"weight": 1.0,
|
|
}
|
|
if not resolved.exists():
|
|
edge["external"] = True
|
|
edges.append(edge)
|
|
return
|
|
|
|
if t == "type_definition":
|
|
tp_node = _find_child(node, "type_path")
|
|
if tp_node is None:
|
|
return
|
|
type_path_str = _type_path_text(tp_node)
|
|
type_nid = _ensure_type(type_path_str, line)
|
|
add_edge(file_nid, type_nid, "contains", line)
|
|
body = _find_child(node, "type_body")
|
|
if body is not None:
|
|
for c in body.children:
|
|
walk(c, parent_type_path=type_path_str, parent_type_nid=type_nid)
|
|
return
|
|
|
|
if t in ("type_body_intended", "type_body_braced"):
|
|
for c in node.children:
|
|
walk(c, parent_type_path, parent_type_nid)
|
|
return
|
|
|
|
if t in ("type_proc_definition", "type_proc_override"):
|
|
if parent_type_nid is None or parent_type_path is None:
|
|
return
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node is None:
|
|
return
|
|
proc_name = _read_text(name_node, source)
|
|
proc_nid = _make_id(stem, parent_type_path, proc_name)
|
|
add_node(proc_nid, f"{parent_type_path}/{proc_name}()", line)
|
|
add_edge(parent_type_nid, proc_nid, "method", line)
|
|
block = _find_child(node, "block")
|
|
if block is not None:
|
|
function_bodies.append((proc_nid, block, parent_type_path))
|
|
return
|
|
|
|
if t in ("proc_definition", "proc_override"):
|
|
tp_node = _find_child(node, "type_path")
|
|
owner_path: "str | None" = None
|
|
owner_nid: "str | None" = None
|
|
if tp_node is not None:
|
|
owner_path = _type_path_text(tp_node)
|
|
owner_nid = _ensure_type(owner_path, line)
|
|
add_edge(file_nid, owner_nid, "contains", line)
|
|
name_node = node.child_by_field_name("name")
|
|
if name_node is None:
|
|
return
|
|
proc_name = _read_text(name_node, source)
|
|
if owner_path and owner_nid:
|
|
proc_nid = _make_id(stem, owner_path, proc_name)
|
|
add_node(proc_nid, f"{owner_path}/{proc_name}()", line)
|
|
add_edge(owner_nid, proc_nid, "method", line)
|
|
else:
|
|
proc_nid = _make_id(stem, proc_name)
|
|
add_node(proc_nid, f"{proc_name}()", line)
|
|
add_edge(file_nid, proc_nid, "contains", line)
|
|
block = _find_child(node, "block")
|
|
if block is not None:
|
|
function_bodies.append((proc_nid, block, owner_path))
|
|
return
|
|
|
|
if t in ("operator_override", "type_operator_override"):
|
|
return
|
|
|
|
for child in node.children:
|
|
walk(child, parent_type_path, parent_type_nid)
|
|
|
|
walk(root)
|
|
|
|
label_to_nids: dict[str, list[str]] = {}
|
|
path_to_nids: dict[str, list[str]] = {}
|
|
for n in nodes:
|
|
label = n["label"].strip("()")
|
|
last = label.rsplit("/", 1)[-1] if "/" in label else label
|
|
if last:
|
|
label_to_nids.setdefault(last.lower(), []).append(n["id"])
|
|
if label.startswith("/"):
|
|
path_to_nids.setdefault(label.lower(), []).append(n["id"])
|
|
|
|
seen_call_pairs: set[tuple[str, str]] = set()
|
|
raw_calls: list[dict] = []
|
|
|
|
def _emit_call(caller_nid: str, callee: str, line: int, is_member: bool) -> None:
|
|
candidates = label_to_nids.get(callee.lower(), [])
|
|
tgt_nid = candidates[0] if len(candidates) == 1 else None
|
|
if tgt_nid and tgt_nid != caller_nid:
|
|
pair = (caller_nid, tgt_nid)
|
|
if pair in seen_call_pairs:
|
|
return
|
|
seen_call_pairs.add(pair)
|
|
edges.append({
|
|
"source": caller_nid, "target": tgt_nid, "relation": "calls",
|
|
"context": "call", "confidence": "EXTRACTED",
|
|
"source_file": str_path, "source_location": f"L{line}", "weight": 1.0,
|
|
})
|
|
else:
|
|
raw_calls.append({
|
|
"caller_nid": caller_nid, "callee": callee,
|
|
"is_member_call": is_member, "source_file": str_path,
|
|
"source_location": f"L{line}",
|
|
})
|
|
|
|
def walk_calls(body_node, caller_nid: str) -> None:
|
|
if body_node is None:
|
|
return
|
|
t = body_node.type
|
|
if t in ("proc_definition", "proc_override", "type_proc_definition",
|
|
"type_proc_override", "type_definition"):
|
|
return
|
|
if t == "call_expression":
|
|
name_node = body_node.child_by_field_name("name")
|
|
if name_node is not None:
|
|
callee = _read_text(name_node, source)
|
|
if callee and callee != "..":
|
|
_emit_call(caller_nid, callee, body_node.start_point[0] + 1,
|
|
is_member=False)
|
|
elif t == "field_proc_expression":
|
|
proc_field = body_node.child_by_field_name("proc")
|
|
if proc_field is not None:
|
|
callee = _read_text(proc_field, source)
|
|
if callee:
|
|
_emit_call(caller_nid, callee, body_node.start_point[0] + 1,
|
|
is_member=True)
|
|
elif t == "new_expression":
|
|
tp_node = _find_child(body_node, "type_path")
|
|
if tp_node is not None:
|
|
target_text = _type_path_text(tp_node)
|
|
candidates = path_to_nids.get(target_text.lower(), [])
|
|
tgt_nid = candidates[0] if len(candidates) == 1 else None
|
|
if tgt_nid and tgt_nid != caller_nid:
|
|
pair = (caller_nid, tgt_nid)
|
|
if pair not in seen_call_pairs:
|
|
seen_call_pairs.add(pair)
|
|
edges.append({
|
|
"source": caller_nid, "target": tgt_nid,
|
|
"relation": "instantiates", "context": "call",
|
|
"confidence": "EXTRACTED", "source_file": str_path,
|
|
"source_location": f"L{body_node.start_point[0] + 1}",
|
|
"weight": 1.0,
|
|
})
|
|
for child in body_node.children:
|
|
walk_calls(child, caller_nid)
|
|
|
|
for proc_nid, block, _owner_path in function_bodies:
|
|
walk_calls(block, proc_nid)
|
|
|
|
return {"nodes": nodes, "edges": edges, "raw_calls": raw_calls}
|
|
|
|
|
|
# ── DMI (BYOND icon files) ────────────────────────────────────────────────────
|
|
# .dmi is a PNG with a tEXt/zTXt "Description" chunk containing BYOND state
|
|
# metadata. We want the icon state names (icon_state = "X" in DM code
|
|
# references them).
|
|
|
|
def _read_dmi_description(data: bytes) -> str:
|
|
"""Pull the BYOND metadata text out of a .dmi PNG, or empty string on failure."""
|
|
import struct
|
|
import zlib as _zlib
|
|
if not data.startswith(b"\x89PNG\r\n\x1a\n"):
|
|
return ""
|
|
i = 8
|
|
while i + 8 <= len(data):
|
|
length = struct.unpack(">I", data[i:i + 4])[0]
|
|
chunk_type = data[i + 4:i + 8]
|
|
payload = data[i + 8:i + 8 + length]
|
|
if chunk_type in (b"tEXt", b"zTXt"):
|
|
try:
|
|
null = payload.index(b"\x00")
|
|
except ValueError:
|
|
return ""
|
|
keyword = payload[:null]
|
|
if keyword == b"Description":
|
|
if chunk_type == b"zTXt":
|
|
return _zlib.decompressobj().decompress(payload[null + 2:], max_length=1024 * 1024).decode("utf-8", errors="replace")
|
|
return payload[null + 1:].decode("utf-8", errors="replace")
|
|
i += 8 + length + 4
|
|
return ""
|
|
|
|
|
|
def extract_dmi(path: Path) -> dict:
|
|
"""Extract icon state names from a .dmi (BYOND PNG icon sheet)."""
|
|
try:
|
|
data = path.read_bytes()
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
str_path = str(path)
|
|
stem = _file_stem(path)
|
|
file_nid = _make_id(str(path))
|
|
nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code",
|
|
"source_file": str_path, "source_location": "L1"}]
|
|
edges: list[dict] = []
|
|
seen: set[str] = {file_nid}
|
|
|
|
description = _read_dmi_description(data)
|
|
if not description:
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
line_no = 0
|
|
for raw_line in description.splitlines():
|
|
line_no += 1
|
|
stripped = raw_line.strip()
|
|
if not stripped.startswith("state ="):
|
|
continue
|
|
value = stripped.split("=", 1)[1].strip()
|
|
if value.startswith('"') and value.endswith('"') and len(value) >= 2:
|
|
state_name = value[1:-1]
|
|
else:
|
|
state_name = value
|
|
if not state_name:
|
|
continue
|
|
nid = _make_id(stem, "state", state_name)
|
|
if nid in seen:
|
|
continue
|
|
seen.add(nid)
|
|
nodes.append({"id": nid, "label": f'"{state_name}"', "file_type": "code",
|
|
"source_file": str_path, "source_location": f"L{line_no}"})
|
|
edges.append({"source": file_nid, "target": nid, "relation": "contains",
|
|
"confidence": "EXTRACTED", "source_file": str_path,
|
|
"source_location": f"L{line_no}", "weight": 1.0})
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
# ── DMM (BYOND map files) ─────────────────────────────────────────────────────
|
|
# A .dmm starts with a tile dictionary — each "key" = (type, type{var=val}, ...)
|
|
# names one or more types that compose a tile — then a grid. We only need the
|
|
# dictionary section: every type path referenced is a `uses` edge.
|
|
|
|
_DMM_GRID_RE = re.compile(r"^\(\s*\d+\s*,\s*\d+\s*,\s*\d+\s*\)\s*=", re.MULTILINE)
|
|
|
|
|
|
def _split_dmm_tile(body: str) -> list[str]:
|
|
out: list[str] = []
|
|
buf: list[str] = []
|
|
depth = 0
|
|
in_string = False
|
|
escape = False
|
|
for ch in body:
|
|
if escape:
|
|
buf.append(ch)
|
|
escape = False
|
|
continue
|
|
if in_string:
|
|
buf.append(ch)
|
|
if ch == "\\":
|
|
escape = True
|
|
elif ch == '"':
|
|
in_string = False
|
|
continue
|
|
if ch == '"':
|
|
in_string = True
|
|
buf.append(ch)
|
|
elif ch in "({[":
|
|
depth += 1
|
|
buf.append(ch)
|
|
elif ch in ")}]":
|
|
depth -= 1
|
|
buf.append(ch)
|
|
elif ch == "," and depth == 0:
|
|
out.append("".join(buf).strip())
|
|
buf = []
|
|
else:
|
|
buf.append(ch)
|
|
tail = "".join(buf).strip()
|
|
if tail:
|
|
out.append(tail)
|
|
return out
|
|
|
|
|
|
def _dmm_type_path(entry: str) -> str:
|
|
brace = entry.find("{")
|
|
if brace != -1:
|
|
entry = entry[:brace]
|
|
return entry.strip()
|
|
|
|
|
|
def extract_dmm(path: Path) -> dict:
|
|
"""Extract type-path references from a .dmm map file's tile dictionary."""
|
|
try:
|
|
if path.stat().st_size > 50 * 1024 * 1024:
|
|
return {"nodes": [], "edges": [], "error": "file too large (>50 MB)"}
|
|
text = path.read_text(encoding="utf-8", errors="replace")
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
str_path = str(path)
|
|
file_nid = _make_id(str(path))
|
|
nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code",
|
|
"source_file": str_path, "source_location": "L1"}]
|
|
edges: list[dict] = []
|
|
|
|
grid_match = _DMM_GRID_RE.search(text)
|
|
dict_text = text[:grid_match.start()] if grid_match else text
|
|
|
|
seen_targets: set[str] = set()
|
|
buf: list[str] = []
|
|
open_line = 0
|
|
depth = 0
|
|
in_string = False
|
|
escape = False
|
|
for line_idx, line in enumerate(dict_text.splitlines(), start=1):
|
|
for ch in line:
|
|
if escape:
|
|
escape = False
|
|
elif in_string:
|
|
if ch == "\\":
|
|
escape = True
|
|
elif ch == '"':
|
|
in_string = False
|
|
elif ch == '"':
|
|
in_string = True
|
|
elif ch == "(":
|
|
if depth == 0:
|
|
open_line = line_idx
|
|
depth += 1
|
|
elif ch == ")":
|
|
depth -= 1
|
|
buf.append(ch)
|
|
buf.append("\n")
|
|
if depth == 0 and buf:
|
|
chunk = "".join(buf)
|
|
buf = []
|
|
lp = chunk.find("(")
|
|
rp = chunk.rfind(")")
|
|
if lp == -1 or rp == -1 or rp <= lp:
|
|
continue
|
|
inner = chunk[lp + 1:rp]
|
|
for entry in _split_dmm_tile(inner):
|
|
tpath = _dmm_type_path(entry)
|
|
if not tpath.startswith("/"):
|
|
continue
|
|
tgt = _make_id(tpath)
|
|
if tgt in seen_targets:
|
|
continue
|
|
seen_targets.add(tgt)
|
|
edges.append({"source": file_nid, "target": tgt, "relation": "uses",
|
|
"context": "map", "confidence": "EXTRACTED",
|
|
"source_file": str_path,
|
|
"source_location": f"L{open_line}", "weight": 1.0})
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
# ── DMF (BYOND interface forms) ───────────────────────────────────────────────
|
|
|
|
_DMF_WINDOW_RE = re.compile(r'^\s*window\s+"([^"]+)"\s*$')
|
|
_DMF_ELEM_RE = re.compile(r'^\s*elem\s+"([^"]+)"\s*$')
|
|
_DMF_TYPE_RE = re.compile(r'^\s*type\s*=\s*(\S+)\s*$')
|
|
|
|
|
|
def extract_dmf(path: Path) -> dict:
|
|
"""Extract windows and controls from a .dmf interface file."""
|
|
try:
|
|
text = path.read_text(encoding="utf-8", errors="replace")
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
str_path = str(path)
|
|
stem = _file_stem(path)
|
|
file_nid = _make_id(str(path))
|
|
nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code",
|
|
"source_file": str_path, "source_location": "L1"}]
|
|
edges: list[dict] = []
|
|
seen: set[str] = {file_nid}
|
|
|
|
current_window_nid: str | None = None
|
|
current_elem_nid: str | None = None
|
|
current_elem_name: str | None = None
|
|
|
|
for line_idx, line in enumerate(text.splitlines(), start=1):
|
|
m = _DMF_WINDOW_RE.match(line)
|
|
if m:
|
|
name = m.group(1)
|
|
nid = _make_id(stem, "window", name)
|
|
if nid not in seen:
|
|
seen.add(nid)
|
|
nodes.append({"id": nid, "label": f'window "{name}"', "file_type": "code",
|
|
"source_file": str_path, "source_location": f"L{line_idx}"})
|
|
edges.append({"source": file_nid, "target": nid, "relation": "contains",
|
|
"confidence": "EXTRACTED", "source_file": str_path,
|
|
"source_location": f"L{line_idx}", "weight": 1.0})
|
|
current_window_nid = nid
|
|
current_elem_nid = None
|
|
current_elem_name = None
|
|
continue
|
|
m = _DMF_ELEM_RE.match(line)
|
|
if m and current_window_nid is not None:
|
|
name = m.group(1)
|
|
nid = _make_id(stem, "elem", current_window_nid, name)
|
|
if nid not in seen:
|
|
seen.add(nid)
|
|
nodes.append({"id": nid, "label": f'elem "{name}"', "file_type": "code",
|
|
"source_file": str_path, "source_location": f"L{line_idx}"})
|
|
edges.append({"source": current_window_nid, "target": nid,
|
|
"relation": "contains", "confidence": "EXTRACTED",
|
|
"source_file": str_path, "source_location": f"L{line_idx}",
|
|
"weight": 1.0})
|
|
current_elem_nid = nid
|
|
current_elem_name = name
|
|
continue
|
|
m = _DMF_TYPE_RE.match(line)
|
|
if m and current_elem_nid is not None and current_elem_name is not None:
|
|
ctype = m.group(1)
|
|
for n in nodes:
|
|
if n["id"] == current_elem_nid and " [" not in n["label"]:
|
|
n["label"] = f'elem "{current_elem_name}" [{ctype}]'
|
|
break
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
# Head tokens in an HCL traversal that are meta/builtins, not references to a
|
|
# block defined in the corpus (count.index, each.key, self.*, path.module, ...).
|
|
_TF_META_HEADS = frozenset({"count", "each", "self", "path", "terraform"})
|
|
|
|
|
|
def extract_terraform(path: Path) -> dict:
|
|
"""Extract Terraform/HCL blocks and the references between them via tree-sitter.
|
|
|
|
Nodes: resources, data sources, modules, variables, outputs, providers, and
|
|
locals. Edges: `contains` (file -> block), `references` (block -> the blocks
|
|
it interpolates, e.g. `aws_instance.web` -> `var.region`), and `depends_on`
|
|
(explicit dependency edges).
|
|
|
|
Node IDs are scoped by the parent directory, not the file stem, because
|
|
Terraform resources are module(directory)-scoped: a resource defined in
|
|
main.tf is referenced from other .tf files in the same directory. Directory
|
|
scoping lets those cross-file references resolve when per-file extractions
|
|
are merged (stem scoping would split a definition from its references).
|
|
"""
|
|
try:
|
|
import tree_sitter_hcl as tshcl
|
|
from tree_sitter import Language, Parser
|
|
except ImportError:
|
|
return {"nodes": [], "edges": [], "error": "tree_sitter_hcl not installed. Run: pip install tree-sitter-hcl"}
|
|
|
|
try:
|
|
language = Language(tshcl.language())
|
|
parser = Parser(language)
|
|
source = path.read_bytes()
|
|
tree = parser.parse(source)
|
|
root = tree.root_node
|
|
except Exception as e:
|
|
return {"nodes": [], "edges": [], "error": str(e)}
|
|
|
|
str_path = str(path)
|
|
file_nid = _make_id(str_path)
|
|
scope = path.parent.name or "tf"
|
|
|
|
nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code",
|
|
"source_file": str_path, "source_location": None}]
|
|
edges: list[dict] = []
|
|
seen_ids: set[str] = {file_nid}
|
|
seen_edges: set[tuple[str, str, str]] = set()
|
|
|
|
def _read(n) -> str:
|
|
return source[n.start_byte:n.end_byte].decode("utf-8", errors="replace")
|
|
|
|
def _label_text(n) -> str:
|
|
return _read(n).strip().strip('"')
|
|
|
|
def _add_node(address: str, label: str, line: int) -> str:
|
|
nid = _make_id(scope, address)
|
|
if nid not in seen_ids:
|
|
seen_ids.add(nid)
|
|
nodes.append({"id": nid, "label": label, "file_type": "code",
|
|
"source_file": str_path, "source_location": f"L{line}"})
|
|
edges.append({"source": file_nid, "target": nid, "relation": "contains",
|
|
"confidence": "EXTRACTED", "source_file": str_path,
|
|
"source_location": f"L{line}", "weight": 1.0})
|
|
return nid
|
|
|
|
def _add_edge(src: str, address: str, relation: str, line: int) -> None:
|
|
tgt = _make_id(scope, address)
|
|
if src == tgt:
|
|
return
|
|
key = (src, tgt, relation)
|
|
if key in seen_edges:
|
|
return
|
|
seen_edges.add(key)
|
|
edges.append({"source": src, "target": tgt, "relation": relation,
|
|
"confidence": "EXTRACTED", "source_file": str_path,
|
|
"source_location": f"L{line}", "weight": 1.0})
|
|
|
|
def _block_parts(block) -> tuple:
|
|
btype = None
|
|
labels: list[str] = []
|
|
for c in block.children:
|
|
if c.type in ("block_start", "body", "block_end"):
|
|
break
|
|
if c.type == "identifier" and btype is None:
|
|
btype = _read(c)
|
|
elif c.type in ("string_lit", "identifier"):
|
|
labels.append(_label_text(c))
|
|
return btype, labels
|
|
|
|
def _ref_address(expr):
|
|
head = _read(expr)
|
|
parent = expr.parent
|
|
attrs: list[str] = []
|
|
if parent is not None:
|
|
seen_self = False
|
|
for c in parent.children:
|
|
if c.id == expr.id:
|
|
seen_self = True
|
|
continue
|
|
if seen_self and c.type == "get_attr":
|
|
name = None
|
|
for gc in c.children:
|
|
if gc.type == "identifier":
|
|
name = _read(gc)
|
|
break
|
|
if name is None:
|
|
break
|
|
attrs.append(name)
|
|
elif seen_self and c.type not in ("get_attr",):
|
|
break
|
|
if head in _TF_META_HEADS or not head:
|
|
return None
|
|
if head == "var":
|
|
return f"var.{attrs[0]}" if attrs else None
|
|
if head == "local":
|
|
return f"local.{attrs[0]}" if attrs else None
|
|
if head == "module":
|
|
return f"module.{attrs[0]}" if attrs else None
|
|
if head == "data":
|
|
return f"data.{attrs[0]}.{attrs[1]}" if len(attrs) >= 2 else None
|
|
return f"{head}.{attrs[0]}" if attrs else None
|
|
|
|
def _collect_refs(node, owner_nid: str, relation: str) -> None:
|
|
rel = relation
|
|
if node.type == "attribute":
|
|
key_node = node.child_by_field_name("key") or (
|
|
node.children[0] if node.children else None
|
|
)
|
|
if key_node is not None and _read(key_node) == "depends_on":
|
|
rel = "depends_on"
|
|
if node.type == "variable_expr":
|
|
addr = _ref_address(node)
|
|
if addr:
|
|
_add_edge(owner_nid, addr, rel, node.start_point[0] + 1)
|
|
for c in node.children:
|
|
if c.is_named:
|
|
_collect_refs(c, owner_nid, rel)
|
|
|
|
def _body_of(block):
|
|
for c in block.children:
|
|
if c.type == "body":
|
|
return c
|
|
return None
|
|
|
|
body = next((c for c in root.children if c.type == "body"), root)
|
|
for block in body.children:
|
|
if block.type != "block":
|
|
continue
|
|
btype, labels = _block_parts(block)
|
|
line = block.start_point[0] + 1
|
|
blk_body = _body_of(block)
|
|
if btype == "resource" and len(labels) >= 2:
|
|
owner = _add_node(f"{labels[0]}.{labels[1]}", f"{labels[0]}.{labels[1]}", line)
|
|
elif btype == "data" and len(labels) >= 2:
|
|
owner = _add_node(f"data.{labels[0]}.{labels[1]}", f"data.{labels[0]}.{labels[1]}", line)
|
|
elif btype == "module" and labels:
|
|
owner = _add_node(f"module.{labels[0]}", f"module.{labels[0]}", line)
|
|
elif btype == "variable" and labels:
|
|
owner = _add_node(f"var.{labels[0]}", f"var.{labels[0]}", line)
|
|
elif btype == "output" and labels:
|
|
owner = _add_node(f"output.{labels[0]}", f"output.{labels[0]}", line)
|
|
elif btype == "provider" and labels:
|
|
owner = _add_node(f"provider.{labels[0]}", f"provider.{labels[0]}", line)
|
|
elif btype == "locals" and blk_body is not None:
|
|
for attr in blk_body.children:
|
|
if attr.type != "attribute":
|
|
continue
|
|
key_node = attr.children[0] if attr.children else None
|
|
if key_node is None:
|
|
continue
|
|
key = _read(key_node)
|
|
lnid = _add_node(f"local.{key}", f"local.{key}", attr.start_point[0] + 1)
|
|
_collect_refs(attr, lnid, "references")
|
|
continue
|
|
else:
|
|
continue
|
|
if blk_body is not None:
|
|
_collect_refs(blk_body, owner, "references")
|
|
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
_DISPATCH: dict[str, Any] = {
|
|
".py": extract_python,
|
|
".js": extract_js,
|
|
".jsx": extract_js,
|
|
".mjs": extract_js,
|
|
".ts": extract_js,
|
|
".tsx": extract_js,
|
|
".go": extract_go,
|
|
".rs": extract_rust,
|
|
".java": extract_java,
|
|
".groovy": extract_groovy,
|
|
".gradle": extract_groovy,
|
|
".c": extract_c,
|
|
".h": extract_c,
|
|
".cpp": extract_cpp,
|
|
".cc": extract_cpp,
|
|
".cxx": extract_cpp,
|
|
".hpp": extract_cpp,
|
|
".rb": extract_ruby,
|
|
".cs": extract_csharp,
|
|
".kt": extract_kotlin,
|
|
".kts": extract_kotlin,
|
|
".scala": extract_scala,
|
|
".php": extract_php,
|
|
".swift": extract_swift,
|
|
".lua": extract_lua,
|
|
".luau": extract_lua,
|
|
".toc": extract_lua,
|
|
".zig": extract_zig,
|
|
".ps1": extract_powershell,
|
|
".ex": extract_elixir,
|
|
".exs": extract_elixir,
|
|
".m": extract_objc,
|
|
".mm": extract_objc,
|
|
".jl": extract_julia,
|
|
".f": extract_fortran,
|
|
".F": extract_fortran,
|
|
".f90": extract_fortran,
|
|
".F90": extract_fortran,
|
|
".f95": extract_fortran,
|
|
".F95": extract_fortran,
|
|
".f03": extract_fortran,
|
|
".F03": extract_fortran,
|
|
".f08": extract_fortran,
|
|
".F08": extract_fortran,
|
|
".vue": extract_js,
|
|
".svelte": extract_svelte,
|
|
".astro": extract_astro,
|
|
".dart": extract_dart,
|
|
".v": extract_verilog,
|
|
".sv": extract_verilog,
|
|
".svh": extract_verilog,
|
|
".sql": extract_sql,
|
|
".md": extract_markdown,
|
|
".mdx": extract_markdown,
|
|
".qmd": extract_markdown,
|
|
".pas": extract_pascal,
|
|
".pp": extract_pascal,
|
|
".dpr": extract_pascal,
|
|
".dpk": extract_pascal,
|
|
".lpr": extract_pascal,
|
|
".inc": extract_pascal,
|
|
".dfm": extract_delphi_form,
|
|
".lfm": extract_lazarus_form,
|
|
".lpk": extract_lazarus_package,
|
|
".sh": extract_bash,
|
|
".bash": extract_bash,
|
|
".json": extract_json,
|
|
".tf": extract_terraform,
|
|
".tfvars": extract_terraform,
|
|
".hcl": extract_terraform,
|
|
".dm": extract_dm,
|
|
".dme": extract_dm,
|
|
".dmi": extract_dmi,
|
|
".dmm": extract_dmm,
|
|
".dmf": extract_dmf,
|
|
".sln": extract_sln,
|
|
".slnx": extract_slnx,
|
|
".csproj": extract_csproj,
|
|
".fsproj": extract_csproj,
|
|
".vbproj": extract_csproj,
|
|
".razor": extract_razor,
|
|
".cshtml": extract_razor,
|
|
".cls": extract_apex,
|
|
".trigger": extract_apex,
|
|
}
|
|
|
|
|
|
def _get_extractor(path: Path) -> Any | None:
|
|
"""Return the correct extractor function for a file, or None if unsupported."""
|
|
if path.name.endswith(".blade.php"):
|
|
return extract_blade
|
|
# MCP config files (.mcp.json, claude_desktop_config.json, ...) are routed
|
|
# by filename before generic .json dispatch so they get MCP-aware nodes
|
|
# (servers, commands, packages, env vars) instead of opaque JSON keys.
|
|
if is_mcp_config_path(path):
|
|
return extract_mcp_config
|
|
return _DISPATCH.get(path.suffix)
|
|
|
|
|
|
def _extract_single_file(args: tuple) -> tuple[int, dict]:
|
|
"""Worker function for parallel extraction. Runs in a subprocess.
|
|
|
|
Must be at module level (not a closure) so it can be pickled by
|
|
ProcessPoolExecutor.
|
|
|
|
Args:
|
|
args: (index, path_str, cache_root_str) tuple
|
|
|
|
Returns:
|
|
(index, result_dict) so results can be placed back in order.
|
|
"""
|
|
idx, path_str, cache_root_str = args
|
|
path = Path(path_str)
|
|
cache_root = Path(cache_root_str)
|
|
_raise_recursion_limit()
|
|
bypass_cache = path.suffix in _JS_CACHE_BYPASS_SUFFIXES
|
|
|
|
# Check cache first (avoid re-extraction)
|
|
if not bypass_cache:
|
|
cached = load_cached(path, cache_root)
|
|
if cached is not None:
|
|
return idx, cached
|
|
|
|
extractor = _get_extractor(path)
|
|
if extractor is None:
|
|
return idx, {"nodes": [], "edges": []}
|
|
|
|
result = _safe_extract(extractor, path)
|
|
if not bypass_cache and "error" not in result:
|
|
save_cached(path, result, cache_root)
|
|
return idx, result
|
|
|
|
|
|
def _extract_parallel(
|
|
uncached_work: list[tuple[int, Path]],
|
|
per_file: list[dict | None],
|
|
effective_root: Path,
|
|
max_workers: int | None,
|
|
total_files: int,
|
|
) -> bool:
|
|
"""Extract uncached files in parallel using ProcessPoolExecutor.
|
|
|
|
Returns True if the pool ran to completion. Returns False if the pool
|
|
failed in a recoverable way (typically Windows-spawn without an
|
|
``if __name__ == "__main__"`` guard in the calling script, which causes
|
|
BrokenProcessPool); the caller should fall back to sequential extraction.
|
|
"""
|
|
import concurrent.futures
|
|
|
|
if max_workers is None:
|
|
# Honour GRAPHIFY_MAX_WORKERS env override; otherwise scale to the
|
|
# full CPU. The historical `, 8)` cap was a safety bound for laptops
|
|
# in 2023 — on a 32-thread workstation it costs a 4x slowdown
|
|
# (issue #792). Capping at len(uncached_work) keeps small jobs
|
|
# from spawning useless idle workers.
|
|
env_raw = os.environ.get("GRAPHIFY_MAX_WORKERS", "").strip()
|
|
env_cap = None
|
|
if env_raw:
|
|
try:
|
|
v = int(env_raw)
|
|
if v > 0:
|
|
env_cap = v
|
|
except ValueError:
|
|
pass
|
|
cpu_cap = env_cap if env_cap is not None else (os.cpu_count() or 4)
|
|
max_workers = min(cpu_cap, len(uncached_work))
|
|
|
|
root_str = str(effective_root)
|
|
work_items = [(idx, str(path), root_str) for idx, path in uncached_work]
|
|
|
|
done_count = 0
|
|
_PROGRESS_INTERVAL = 100
|
|
try:
|
|
with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as pool:
|
|
futures = {
|
|
pool.submit(_extract_single_file, item): item[0] for item in work_items
|
|
}
|
|
for future in concurrent.futures.as_completed(futures):
|
|
try:
|
|
idx, result = future.result()
|
|
per_file[idx] = result
|
|
except Exception as exc:
|
|
idx = futures[future]
|
|
print(
|
|
f" warning: worker failed for {work_items[idx][1]}: {exc}",
|
|
file=sys.stderr, flush=True,
|
|
)
|
|
done_count += 1
|
|
if (
|
|
total_files >= _PROGRESS_INTERVAL
|
|
and done_count % _PROGRESS_INTERVAL == 0
|
|
):
|
|
print(
|
|
f" AST extraction: {done_count}/{len(uncached_work)} uncached files "
|
|
f"({done_count * 100 // len(uncached_work)}%) [{max_workers} workers]",
|
|
flush=True,
|
|
)
|
|
except concurrent.futures.process.BrokenProcessPool:
|
|
# On Windows (spawn start method) the worker subprocesses re-import the
|
|
# caller's __main__. Inline invocations like `python -c "..."` have no
|
|
# __main__ guard, so worker bootstrap raises and the pool dies before
|
|
# any work completes. Fall back to in-process sequential extraction —
|
|
# slower but correct.
|
|
print(
|
|
" warning: parallel extraction failed (BrokenProcessPool); "
|
|
"falling back to sequential. On Windows this usually means the "
|
|
'caller is missing an `if __name__ == "__main__":` guard. Pass '
|
|
"parallel=False to extract() to skip the pool entirely.",
|
|
flush=True,
|
|
)
|
|
return False
|
|
if total_files >= _PROGRESS_INTERVAL:
|
|
print(
|
|
f" AST extraction: {total_files}/{total_files} files (100%) [{max_workers} workers]",
|
|
flush=True,
|
|
)
|
|
return True
|
|
|
|
|
|
def _extract_sequential(
|
|
uncached_work: list[tuple[int, Path]],
|
|
per_file: list[dict | None],
|
|
effective_root: Path,
|
|
total_files: int,
|
|
) -> None:
|
|
"""Extract uncached files sequentially (fallback for small batches)."""
|
|
_PROGRESS_INTERVAL = 100
|
|
for work_idx, (idx, path) in enumerate(uncached_work):
|
|
if (
|
|
total_files >= _PROGRESS_INTERVAL
|
|
and work_idx % _PROGRESS_INTERVAL == 0
|
|
and work_idx > 0
|
|
):
|
|
print(
|
|
f" AST extraction: {work_idx}/{len(uncached_work)} uncached files ({work_idx * 100 // len(uncached_work)}%)",
|
|
flush=True,
|
|
)
|
|
extractor = _get_extractor(path)
|
|
if extractor is None:
|
|
per_file[idx] = {"nodes": [], "edges": []}
|
|
continue
|
|
bypass_cache = path.suffix in _JS_CACHE_BYPASS_SUFFIXES
|
|
result = _safe_extract(extractor, path)
|
|
if not bypass_cache and "error" not in result:
|
|
save_cached(path, result, effective_root)
|
|
per_file[idx] = result
|
|
if total_files >= _PROGRESS_INTERVAL:
|
|
print(f" AST extraction: {total_files}/{total_files} files (100%)", flush=True)
|
|
|
|
|
|
_PARALLEL_THRESHOLD = 20
|
|
|
|
|
|
def extract(
|
|
paths: list[Path],
|
|
cache_root: Path | None = None,
|
|
*,
|
|
parallel: bool = True,
|
|
max_workers: int | None = None,
|
|
) -> dict:
|
|
"""Extract AST nodes and edges from a list of code files.
|
|
|
|
Two-pass process:
|
|
1. Per-file structural extraction (classes, functions, imports)
|
|
2. Cross-file import resolution: turns file-level imports into
|
|
class-level INFERRED edges (DigestAuth --uses--> Response)
|
|
|
|
Args:
|
|
paths: files to extract from
|
|
cache_root: explicit root for graphify-out/cache/ (overrides the
|
|
inferred common path prefix). Pass Path('.') when running on a
|
|
subdirectory so the cache stays at ./graphify-out/cache/.
|
|
parallel: if True and there are >= _PARALLEL_THRESHOLD uncached files,
|
|
use ProcessPoolExecutor for multi-core extraction.
|
|
max_workers: max subprocess count. Defaults to cpu_count (or the
|
|
value of GRAPHIFY_MAX_WORKERS if set), bounded by len(uncached_work).
|
|
"""
|
|
paths = [Path(p) for p in paths]
|
|
_check_tree_sitter_version()
|
|
_raise_recursion_limit()
|
|
# Workspace package manifests/globs can change during watch or repeated extraction.
|
|
_WORKSPACE_PACKAGE_CACHE.clear()
|
|
|
|
# Infer a common root for cache keys (use first diverging segment, not sum of all matches)
|
|
try:
|
|
if not paths:
|
|
root = Path(".")
|
|
elif len(paths) == 1:
|
|
root = paths[0].parent
|
|
else:
|
|
min_parts = min(len(p.parts) for p in paths)
|
|
common_len = 0
|
|
for i in range(min_parts):
|
|
if len({p.parts[i] for p in paths}) == 1:
|
|
common_len += 1
|
|
else:
|
|
break
|
|
root = Path(*paths[0].parts[:common_len]) if common_len else Path(".")
|
|
except Exception:
|
|
root = Path(".")
|
|
if cache_root is not None:
|
|
root = cache_root
|
|
root = root.resolve()
|
|
|
|
effective_root = cache_root or root
|
|
total = len(paths)
|
|
|
|
# Phase 1: separate cached hits from uncached work
|
|
per_file: list[dict | None] = [None] * total
|
|
uncached_work: list[tuple[int, Path]] = []
|
|
|
|
for i, path in enumerate(paths):
|
|
if _get_extractor(path) is None:
|
|
per_file[i] = {"nodes": [], "edges": []}
|
|
continue
|
|
bypass_cache = path.suffix in _JS_CACHE_BYPASS_SUFFIXES
|
|
if not bypass_cache:
|
|
cached = load_cached(path, effective_root)
|
|
if cached is not None:
|
|
per_file[i] = cached
|
|
continue
|
|
uncached_work.append((i, path))
|
|
|
|
# Phase 2: extract uncached files (parallel or sequential)
|
|
if uncached_work:
|
|
ran_parallel = False
|
|
if parallel and len(uncached_work) >= _PARALLEL_THRESHOLD:
|
|
ran_parallel = _extract_parallel(
|
|
uncached_work, per_file, effective_root, max_workers, total
|
|
)
|
|
if not ran_parallel:
|
|
_extract_sequential(uncached_work, per_file, effective_root, total)
|
|
|
|
# Fill any remaining None slots (shouldn't happen, but defensive)
|
|
for i in range(total):
|
|
if per_file[i] is None:
|
|
per_file[i] = {"nodes": [], "edges": []}
|
|
|
|
all_nodes: list[dict] = []
|
|
all_edges: list[dict] = []
|
|
all_raw_calls: list[dict] = []
|
|
for result in per_file:
|
|
all_nodes.extend(result.get("nodes", []))
|
|
all_edges.extend(result.get("edges", []))
|
|
all_raw_calls.extend(result.get("raw_calls", []))
|
|
|
|
_augment_symbol_resolution_edges(paths, all_nodes, all_edges, root)
|
|
|
|
# Remap file node IDs from absolute-path-derived to the canonical
|
|
# {parent_dir}_{stem} spec form so (a) graph.json edge endpoints are stable
|
|
# across machines (#502) and (b) AST file nodes match the IDs semantic
|
|
# subagents generate (#1033). Resolve before relativizing so paths passed in
|
|
# relative form still anchor to the (resolved) root.
|
|
id_remap: dict[str, str] = {}
|
|
# Symbol node IDs embed the file stem as a prefix (_file_node_id of the path
|
|
# the extractor saw). For a root-level file that stem picks up the absolute
|
|
# parent directory name, so a symbol becomes <rootdir>_main_run while the
|
|
# file node is correctly relativized to main and the skill.md spec wants
|
|
# main_run -- splitting the symbol into AST/semantic ghosts (#1096). Relativize
|
|
# the symbol prefix the same way, gated by source_file so two files sharing a
|
|
# prefix can't cross-contaminate. Keyed by resolved path -> (old_pref, new_pref).
|
|
prefix_remap: dict[Path, tuple[str, str]] = {}
|
|
for path in paths:
|
|
old_id = _make_id(str(path))
|
|
try:
|
|
rel = path.relative_to(root)
|
|
except ValueError:
|
|
try:
|
|
rel = path.resolve().relative_to(root)
|
|
except ValueError:
|
|
continue
|
|
new_id = _file_node_id(rel)
|
|
if old_id != new_id:
|
|
id_remap[old_id] = new_id
|
|
old_pref = _file_node_id(path)
|
|
if old_pref != new_id:
|
|
prefix_remap[path.resolve()] = (old_pref, new_id)
|
|
if id_remap:
|
|
for n in all_nodes:
|
|
if n.get("id") in id_remap:
|
|
n["id"] = id_remap[n["id"]]
|
|
for e in all_edges:
|
|
if e.get("source") in id_remap:
|
|
e["source"] = id_remap[e["source"]]
|
|
if e.get("target") in id_remap:
|
|
e["target"] = id_remap[e["target"]]
|
|
if prefix_remap:
|
|
sym_remap: dict[str, str] = {}
|
|
for n in all_nodes:
|
|
sf = n.get("source_file")
|
|
if not sf:
|
|
continue
|
|
try:
|
|
entry = prefix_remap.get(Path(sf).resolve())
|
|
except Exception:
|
|
continue
|
|
if entry is None:
|
|
continue
|
|
old_pref, new_pref = entry
|
|
nid = n.get("id", "")
|
|
if nid.startswith(old_pref + "_"):
|
|
new_nid = new_pref + nid[len(old_pref):]
|
|
if new_nid != nid:
|
|
sym_remap[nid] = new_nid
|
|
if sym_remap:
|
|
for n in all_nodes:
|
|
if n.get("id") in sym_remap:
|
|
n["id"] = sym_remap[n["id"]]
|
|
for e in all_edges:
|
|
if e.get("source") in sym_remap:
|
|
e["source"] = sym_remap[e["source"]]
|
|
if e.get("target") in sym_remap:
|
|
e["target"] = sym_remap[e["target"]]
|
|
# raw_calls carry caller_nid (a symbol id) consumed by the cross-file
|
|
# call pass below, after this remap — rewrite it too or those edges
|
|
# would dangle on their (stale) source.
|
|
for rc in all_raw_calls:
|
|
cn = rc.get("caller_nid")
|
|
if cn in sym_remap:
|
|
rc["caller_nid"] = sym_remap[cn]
|
|
|
|
_merge_swift_extensions(per_file, all_nodes, all_edges)
|
|
_disambiguate_colliding_node_ids(all_nodes, all_edges, all_raw_calls, root)
|
|
_rewire_unique_stub_nodes(all_nodes, all_edges)
|
|
|
|
# Add cross-file class-level edges (Python only - uses Python parser internally)
|
|
py_paths = [p for p in paths if p.suffix == ".py"]
|
|
if py_paths:
|
|
py_results = [r for r, p in zip(per_file, paths) if p.suffix == ".py"]
|
|
try:
|
|
cross_file_edges = _resolve_cross_file_imports(py_results, py_paths)
|
|
all_edges.extend(cross_file_edges)
|
|
except Exception as exc:
|
|
import logging
|
|
logging.getLogger(__name__).warning("Cross-file import resolution failed, skipping: %s", exc)
|
|
|
|
# Cross-file Java import resolution
|
|
java_paths = [p for p in paths if p.suffix == ".java"]
|
|
if java_paths:
|
|
java_results = [r for r, p in zip(per_file, paths) if p.suffix == ".java"]
|
|
try:
|
|
all_edges.extend(_resolve_cross_file_java_imports(java_results, java_paths))
|
|
except Exception as exc:
|
|
import logging
|
|
logging.getLogger(__name__).warning("Java cross-file import resolution failed, skipping: %s", exc)
|
|
|
|
# Cross-file call resolution for all languages
|
|
# Each extractor saved unresolved calls in raw_calls. Now that we have all
|
|
# nodes from all files, resolve any callee that exists in another file.
|
|
# Build name → ALL matching node IDs so we can skip ambiguous common names
|
|
# (e.g. "log", "execute", "find") that appear in multiple files — resolving
|
|
# those inflates god_nodes ranking with spurious cross-file edges.
|
|
# Build label -> node_id index for cross-file call resolution.
|
|
# Skip rationale nodes (their labels are docstring text, not callable
|
|
# identifiers, and they were polluting matches for short names — #563).
|
|
global_label_to_nids: dict[str, list[str]] = {}
|
|
for n in all_nodes:
|
|
if n.get("file_type") == "rationale":
|
|
continue
|
|
raw = n.get("label", "")
|
|
normalised = raw.strip("()").lstrip(".")
|
|
if normalised:
|
|
key = normalised.lower()
|
|
global_label_to_nids.setdefault(key, []).append(n["id"])
|
|
|
|
# Build evidence index from import edges so cross-file calls backed by an
|
|
# explicit import statement can be promoted from INFERRED to EXTRACTED.
|
|
# Direct symbol imports (`import { foo }` / `const { foo } = require()`) are
|
|
# the strongest evidence — caller's file_id has an `imports` edge directly to
|
|
# the callee's symbol id. Module imports (`imports_from`) are weaker but still
|
|
# confirm the caller pulled in the callee's source file.
|
|
file_to_symbol_imports: dict[str, set[str]] = {}
|
|
file_to_module_imports: dict[str, set[str]] = {}
|
|
for e in all_edges:
|
|
if e.get("relation") == "imports":
|
|
file_to_symbol_imports.setdefault(e["source"], set()).add(e["target"])
|
|
elif e.get("relation") == "imports_from":
|
|
file_to_module_imports.setdefault(e["source"], set()).add(e["target"])
|
|
|
|
# Map each node back to its containing file_id so we can ask
|
|
# "did the caller's file import the callee's file?"
|
|
# Use relativized paths to match how file node IDs were remapped above (#502).
|
|
nid_to_file_nid: dict[str, str] = {}
|
|
for n in all_nodes:
|
|
sf = n.get("source_file")
|
|
if not sf:
|
|
continue
|
|
sf_path = Path(sf)
|
|
try:
|
|
sf_rel = sf_path.relative_to(root) if sf_path.is_absolute() else sf_path
|
|
except ValueError:
|
|
sf_rel = sf_path
|
|
nid_to_file_nid[n["id"]] = _file_node_id(sf_rel)
|
|
|
|
existing_pairs = {(e["source"], e["target"]) for e in all_edges}
|
|
for rc in all_raw_calls:
|
|
callee = rc.get("callee", "")
|
|
if not callee:
|
|
continue
|
|
if callee in _LANGUAGE_BUILTIN_GLOBALS:
|
|
continue
|
|
# Skip member-call callees: obj.log() → "log" has no import evidence
|
|
# and collides with any top-level function named "log" in the corpus.
|
|
if rc.get("is_member_call"):
|
|
continue
|
|
candidates = global_label_to_nids.get(callee.lower(), [])
|
|
if not candidates:
|
|
continue
|
|
caller = rc["caller_nid"]
|
|
caller_file_nid = nid_to_file_nid.get(caller)
|
|
imported_symbols = file_to_symbol_imports.get(caller_file_nid, set())
|
|
imported_modules = file_to_module_imports.get(caller_file_nid, set())
|
|
|
|
def _has_import_evidence(candidate_id: str) -> bool:
|
|
# Direct symbol import (`import { foo }`) is the strongest evidence:
|
|
# the caller's file has an `imports` edge straight to this symbol.
|
|
# A module import (`import './helper.js'`) confirms the caller pulled
|
|
# in the file the candidate lives in.
|
|
candidate_file_nid = nid_to_file_nid.get(candidate_id)
|
|
return (
|
|
candidate_id in imported_symbols
|
|
or (candidate_file_nid is not None and candidate_file_nid in imported_modules)
|
|
)
|
|
|
|
if len(candidates) == 1:
|
|
tgt = candidates[0]
|
|
has_import_evidence = _has_import_evidence(tgt)
|
|
else:
|
|
# Ambiguous name (defined in 2+ files). Don't bail outright (#1219):
|
|
# if the caller has explicit import evidence pointing at exactly one
|
|
# of the candidates, that named import disambiguates unambiguously.
|
|
# Prefer direct symbol-import matches; fall back to module-import
|
|
# matches only when they too collapse to a single target. Without a
|
|
# unique evidence-backed pick we skip, preserving the #543 guard
|
|
# against over-connecting common short names (log, execute, find).
|
|
symbol_matches = [c for c in candidates if c in imported_symbols]
|
|
if len(symbol_matches) == 1:
|
|
tgt = symbol_matches[0]
|
|
else:
|
|
module_matches = [
|
|
c for c in candidates
|
|
if (cf := nid_to_file_nid.get(c)) is not None and cf in imported_modules
|
|
]
|
|
if len(module_matches) == 1:
|
|
tgt = module_matches[0]
|
|
else:
|
|
continue
|
|
has_import_evidence = True
|
|
if tgt != caller and (caller, tgt) not in existing_pairs:
|
|
existing_pairs.add((caller, tgt))
|
|
# Promote to EXTRACTED when there's a direct import edge from the
|
|
# caller's file pointing at either the callee symbol itself or the
|
|
# file the callee lives in.
|
|
if has_import_evidence:
|
|
confidence = "EXTRACTED"
|
|
confidence_score = 1.0
|
|
else:
|
|
confidence = "INFERRED"
|
|
confidence_score = 0.8
|
|
all_edges.append({
|
|
"source": caller,
|
|
"target": tgt,
|
|
"relation": "calls",
|
|
"context": "call",
|
|
"confidence": confidence,
|
|
"confidence_score": confidence_score,
|
|
"source_file": rc.get("source_file", ""),
|
|
"source_location": rc.get("source_location"),
|
|
"weight": 1.0,
|
|
})
|
|
|
|
# Relativize source_file fields so paths are portable across machines (#555)
|
|
for item in all_nodes + all_edges:
|
|
sf = item.get("source_file")
|
|
if not sf:
|
|
continue
|
|
sf_path = Path(sf)
|
|
if not sf_path.is_absolute():
|
|
continue
|
|
try:
|
|
item["source_file"] = sf_path.relative_to(root).as_posix()
|
|
except ValueError:
|
|
pass
|
|
|
|
# Tag AST provenance so the incremental watch rebuild can distinguish
|
|
# AST-extracted nodes from semantic/LLM nodes. On a full re-extraction
|
|
# the watcher drops any AST-marked node missing from the fresh output
|
|
# even when its source file still exists (#1116).
|
|
for n in all_nodes:
|
|
n["_origin"] = "ast"
|
|
|
|
return {
|
|
"nodes": all_nodes,
|
|
"edges": all_edges,
|
|
"input_tokens": 0,
|
|
"output_tokens": 0,
|
|
}
|
|
|
|
|
|
def collect_files(target: Path, *, follow_symlinks: bool = False, root: Path | None = None) -> list[Path]:
|
|
if target.is_file():
|
|
return [target]
|
|
_EXTENSIONS = set(_DISPATCH.keys())
|
|
from graphify.detect import _is_ignored, _is_noise_dir, _load_graphifyignore
|
|
ignore_root = root if root is not None else target
|
|
patterns = _load_graphifyignore(ignore_root)
|
|
# Shared across all _is_ignored calls in this scan so ancestor-directory
|
|
# results are memoised instead of re-evaluated per file.
|
|
ignore_cache: dict[Path, bool] = {}
|
|
|
|
def _ignored(p: Path) -> bool:
|
|
return bool(patterns and _is_ignored(p, ignore_root, patterns, _cache=ignore_cache))
|
|
|
|
if not follow_symlinks:
|
|
# The old rglob filter rejected paths with a noise component anywhere,
|
|
# including components of target itself — preserve that.
|
|
if any(_is_noise_dir(part) for part in target.parts):
|
|
return []
|
|
# When negation (!) patterns exist, skip directory-level ignore pruning
|
|
# so negated files inside ignored dirs can still be reached (same
|
|
# conservatism as detect's scan walk).
|
|
has_negation = any(pat.startswith("!") for _, pat in patterns)
|
|
results: list[Path] = []
|
|
for dirpath, dirnames, filenames in os.walk(target):
|
|
dp = Path(dirpath)
|
|
dirnames[:] = [
|
|
d for d in dirnames
|
|
if not _is_noise_dir(d)
|
|
and (has_negation or not _ignored(dp / d))
|
|
]
|
|
for fname in filenames:
|
|
p = dp / fname
|
|
if p.suffix in _EXTENSIONS and not _ignored(p):
|
|
results.append(p)
|
|
return sorted(results)
|
|
# Walk with symlink following + cycle detection
|
|
results = []
|
|
for dirpath, dirnames, filenames in os.walk(target, followlinks=True):
|
|
if os.path.islink(dirpath):
|
|
real = os.path.realpath(dirpath)
|
|
parent_real = os.path.realpath(os.path.dirname(dirpath))
|
|
if parent_real == real or parent_real.startswith(real + os.sep):
|
|
dirnames.clear()
|
|
continue
|
|
dp = Path(dirpath)
|
|
dirnames[:] = [d for d in dirnames if not _is_noise_dir(d)]
|
|
for fname in filenames:
|
|
p = dp / fname
|
|
if p.suffix in _EXTENSIONS and not _ignored(p):
|
|
results.append(p)
|
|
return sorted(results)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) < 2:
|
|
print("Usage: python -m graphify.extract <file_or_dir> ...", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
paths: list[Path] = []
|
|
for arg in sys.argv[1:]:
|
|
paths.extend(collect_files(Path(arg)))
|
|
|
|
result = extract(paths)
|
|
print(json.dumps(result, indent=2))
|