#!/usr/bin/env python3 """Turn a C++ codebase into `code`-module facts. The `code` reasoning module supplies the reasoning. This supplies the facts. It reads your source and writes a DKE Python program that records what it found — functions, classes, calls, includes — which you then submit to your store with the `compile` (or `run`) tool you already have: dke-ingest-cpp ./src ./include > facts.dpy # then submit facts.dpy through your DKE client Nothing here talks to the network, and nothing here needs your key. The output is a program you can read before you run it: every fact this tool would record is a line you can see. WHY THIS IS HAND-WRITTEN. Python's ingest parses with CPython's own `ast`, so what counts as a function there is what counts as a function to Python. C++ ships no such thing, and the discipline this ecology holds to (brickmason's per-language spec-first rule) puts external parsers off-limits: no libclang, no tree-sitter, nothing vendored. So this reads C++ the way the language's own grammar describes it — scrub comments and literal bodies, tokenize, then walk the token stream tracking scope and brace depth. That has a consequence worth stating before any number below is trusted: a hand-written reader sees LESS than a compiler. It does not instantiate templates, does not expand macros, does not evaluate `#if`, and does not do overload resolution by argument TYPE. Every one of those limits costs recorded edges, never invents them — the rule throughout is that a missing edge is a known gap and a guessed edge is a wrong answer the queries cannot tell from a fact. ON CALL RESOLUTION — read this before trusting `uncalled()`. The call graph is a LOWER BOUND. An edge is recorded only when the source makes the target plain: a bare or qualified name that resolves against the enclosing scopes, `this->m()`, or a call through a variable whose type was DECLARED in the source this reader could see. A call through a function pointer, a `std::function`, a virtual dispatch, a template parameter, or a macro is not visible here and is not recorded. Virtual dispatch is the one worth naming twice: a call through a base-class reference records nothing, so an override can look uncalled while running on every request. The tool prints its resolution rate to stderr and writes it into the generated program's header. Treat `uncalled()` as a list of candidates to look at, never as a list of dead code to delete. """ from __future__ import annotations import re import sys from dataclasses import dataclass from pathlib import Path, PurePosixPath from dke.ingest import core from dke.ingest.core import ( # noqa: F401 (re-exported: this module IS the C++ ingest) MODULE, Call, Dependency, Facts, Function, Ids, Import, Klass, Language, UnresolvedCall, reached_by_tests, sanitize, ) # What a C++ translation unit or header looks like from the outside. A tree # with none of these in it is not a C++ tree, and the core CLI says so # rather than writing an empty program and exiting 0. EXTENSIONS = (".cpp", ".cc", ".cxx", ".c++", ".hpp", ".hh", ".hxx", ".h", ".ipp", ".tpp") # An unbounded parameter list — `...` — recorded as a number so arity # comparison stays one expression rather than a special case everywhere. UNBOUNDED = 1 << 20 # ── Scrubbing ──────────────────────────────────────────────────────────── # # Comments and the BODIES of literals are replaced by spaces, newlines kept, # so every later offset still maps to its original line. Preprocessor lines # are lifted out whole (they are their own grammar, not C++ expressions) and # blanked, which is also what keeps a `#define` containing braces from # throwing off the brace tracking that everything else depends on. _DIRECTIVE = re.compile(r"^\s*#\s*(\w+)\s*(.*)$") def scrub(src: str, comments: list[tuple[int, str]] | None = None ) -> tuple[str, list[tuple[int, str, str]]]: """(code with comments/literals blanked, [(line, directive, rest)]). `comments` is an optional SINK: when given, each comment is appended as (line, body-without-delimiters). CODE-MODULE-RATIONALE-FACTS needs the comments this function exists to throw away, and collecting them here rather than in a second pass is what keeps one definition of where a comment starts — a naive `//` scan would find one inside a string literal, which this walk already gets right. """ out: list[str] = [] directives: list[tuple[int, str, str]] = [] i, n, line = 0, len(src), 1 at_line_start = True def blank(text: str) -> str: """Same length, same newlines, no content.""" return "".join("\n" if ch == "\n" else " " for ch in text) def literal(text: str) -> str: """A literal reduced to ONE placeholder token, same length and lines. Not blanked away. A literal is an ARGUMENT, and arguments are counted to tell overloads apart — blanking `fb.symbol('W')` to `fb.symbol( )` makes it a zero-argument call that matches no overload of `symbol`, so the edge is silently lost. That cost 31 call edges in a real C++ tree before it was measured. """ return "0" + blank(text[1:]) while i < n: ch = src[i] if ch == "\n": out.append("\n") line += 1 i += 1 at_line_start = True continue if at_line_start and ch in " \t": out.append(ch) i += 1 continue # Preprocessor directive: to end of line, following backslash-newline. if at_line_start and ch == "#": j = i start_line = line while j < n: if src[j] == "\\" and j + 1 < n and src[j + 1] == "\n": j += 2 continue if src[j] == "\n": break j += 1 raw = src[i:j].replace("\\\n", " ") m = _DIRECTIVE.match(raw) if m: directives.append((start_line, m.group(1), m.group(2).strip())) seg = src[i:j] line += seg.count("\n") out.append(blank(seg)) i = j continue at_line_start = False if ch == "/" and i + 1 < n and src[i + 1] == "/": j = src.find("\n", i) j = n if j < 0 else j if comments is not None: comments.append((line, src[i + 2:j])) out.append(blank(src[i:j])) i = j continue if ch == "/" and i + 1 < n and src[i + 1] == "*": j = src.find("*/", i + 2) j = n if j < 0 else j + 2 seg = src[i:j] if comments is not None: # Each LINE of a block comment separately, so a marker on the # third line of a banner is found and carries its own line # number. A leading `*` is the conventional continuation # gutter, not part of what the author wrote. body = seg[2:-2] if seg.endswith("*/") else seg[2:] for k, ln in enumerate(body.split("\n")): comments.append((line + k, ln.lstrip().lstrip("*"))) line += seg.count("\n") out.append(blank(seg)) i = j continue # Raw string: R"delim( … )delim" if ch == "R" and i + 1 < n and src[i + 1] == '"': k = src.find("(", i + 2) if k >= 0: delim = src[i + 2:k] close = ")" + delim + '"' j = src.find(close, k + 1) j = n if j < 0 else j + len(close) seg = src[i:j] line += seg.count("\n") out.append(literal(seg)) i = j continue if ch == '"' or ch == "'": # A single quote between two alphanumerics is C++14's digit # separator (1'000'000), not a character literal. Reading it as a # literal swallows the rest of the line and silently loses code. if (ch == "'" and i > 0 and src[i - 1].isalnum() and i + 1 < n and src[i + 1].isalnum()): out.append(" ") i += 1 continue j = i + 1 while j < n: if src[j] == "\\": j += 2 continue if src[j] == ch: j += 1 break if src[j] == "\n": break # unterminated: stop at the line end j += 1 seg = src[i:j] line += seg.count("\n") out.append(literal(seg)) i = j continue out.append(ch) i += 1 return "".join(out), directives # ── Tokenizing ─────────────────────────────────────────────────────────── _TOKEN = re.compile(r""" (?P[A-Za-z_][A-Za-z0-9_]*) | (?P\.?[0-9][0-9A-Za-z_.]*) | (?P::|->\*|->|\+\+|--|<<=|>>=|&&|\|\||==|!=|<=|>=|\+=|-=|\*=|/=|%=|\^=|&=|\|=|\.\.\.|.) """, re.VERBOSE) def tokenize(code: str) -> tuple[list[str], list[int]]: """(token texts, 1-based line of each token). `<` and `>` stay single characters even when adjacent: `>>` closing two template argument lists and `>>` shifting are the same two bytes, and nothing downstream needs to tell them apart — only `template <…>` counts angle depth, and a shift cannot appear there. """ toks: list[str] = [] lines: list[int] = [] line = 1 pos = 0 for m in _TOKEN.finditer(code): line += code.count("\n", pos, m.start()) pos = m.start() text = m.group(0) if text.isspace(): continue toks.append(text) lines.append(line) return toks, lines KEYWORDS_NOT_CALLS = { "if", "while", "for", "switch", "catch", "return", "sizeof", "alignof", "alignas", "static_cast", "dynamic_cast", "reinterpret_cast", "const_cast", "decltype", "typeid", "noexcept", "throw", "new", "delete", "operator", "and", "or", "not", "case", "do", "else", "using", "namespace", "template", "requires", "constexpr", "consteval", "constinit", "explicit", "friend", "static_assert", "asm", "co_await", "co_yield", "co_return", "goto", } # Names that decorate a declarator without being part of it. They may sit # between the closing `)` and the body, and they are not the function's name. TRAILING_QUALIFIERS = { "const", "volatile", "noexcept", "override", "final", "mutable", "throw", "requires", "auto", "decltype", "->", "&", "&&", "*", "::", } STORAGE_WORDS = { "static", "inline", "extern", "constexpr", "consteval", "constinit", "explicit", "virtual", "friend", "typedef", "register", "thread_local", "const", "volatile", "mutable", "template", "signed", "unsigned", } # A type name that never names one of THIS project's classes. Filtering them # out of variable-type tracking is not required for correctness — an # unresolvable type simply yields no edge — but it keeps the map small. BUILTIN_TYPES = { "void", "bool", "char", "short", "int", "long", "float", "double", "signed", "unsigned", "size_t", "auto", "wchar_t", "char8_t", "char16_t", "char32_t", "nullptr_t", } def is_ident(t: str) -> bool: return bool(t) and (t[0].isalpha() or t[0] == "_") def find_matching(toks: list[str], i: int, open_t: str, close_t: str) -> int: """Index of the token closing the group that opens at `i`, or len(toks).""" depth = 0 n = len(toks) while i < n: if toks[i] == open_t: depth += 1 elif toks[i] == close_t: depth -= 1 if depth == 0: return i i += 1 return n def _split_top_level(inner: list[str]) -> list[list[str]]: depth = 0 parts: list[list[str]] = [[]] for t in inner: if t in "([{": depth += 1 elif t in ")]}": depth -= 1 if t == "," and depth == 0: parts.append([]) continue parts[-1].append(t) return parts def arity_range(toks: list[str], lp: int, rp: int) -> tuple[int, int]: """(min, max) argument counts a parameter list `( lp … rp )` accepts. Counted lexically: top-level commas, minus the ones whose parameter has a default. This is the whole of what a reader without a type system can do about overloads, and it is enough for the common case — two overloads of one name almost always take different numbers of arguments. """ inner = toks[lp + 1:rp] if not inner or inner == ["void"]: return (0, 0) parts = _split_top_level(inner) total = len(parts) defaults = sum(1 for p in parts if "=" in p) if any("..." in p for p in parts): return (max(0, total - defaults - 1), UNBOUNDED) return (total - defaults, total) def call_arity(toks: list[str], lp: int, rp: int) -> int: """How many arguments a call site passes, counted the same lexical way.""" inner = toks[lp + 1:rp] return len(_split_top_level(inner)) if inner else 0 # ── What one file yields ───────────────────────────────────────────────── @dataclass class Definition: """One function DEFINITION, and the arities it accepts. Kept beside the `Function` fact rather than on it because a qualified name is not unique in C++: overloads share one. `Function.qual` is what becomes a store subject, so it has to be made unique before anything is emitted, and this record is what the disambiguation and the resolver both read. """ fn: Function base_qual: str pmin: int pmax: int module: str line: int class Pending: """A call site, kept until every definition in the ingest is known. Resolution cannot happen during the first pass for the same reason it cannot in the Python reader: a call into a file read later must still resolve, so no edge may be decided before every definition exists. """ __slots__ = ("caller", "segs", "seps", "argc", "line", "scopes", "class_qual", "locals_", "path") def __init__(self, caller: Function, segs: list[str], seps: list[str], argc: int, line: int, scopes: list[str], class_qual: str | None, locals_: dict[str, str], path: str) -> None: self.caller = caller self.segs = segs self.seps = seps self.argc = argc self.line = line self.scopes = scopes self.class_qual = class_qual self.locals_ = locals_ self.path = path class FileScanner: """One translation unit or header. Walks tokens; records what it sees.""" def __init__(self, module: str, path: str, facts: Facts, defs: list[Definition], members: dict[str, dict[str, str]]) -> None: self.module = module self.path = path self.facts = facts self.defs = defs self.in_test_file = is_test_file(path) self.toks: list[str] = [] self.lines: list[int] = [] # (qualified name, first line, last line) per definition, for # CODE-MODULE-RATIONALE-FACTS. Comments are blanked before the walk # sees anything, so attribution meets the walk on line numbers. self.spans: list[tuple[str, int, int]] = [] # Enclosing namespaces and classes, outermost first. self.scopes: list[tuple[str, str]] = [] # (kind, name) self.class_stack: list[Klass] = [] self.using_namespaces: list[str] = [] # Qualified prefixes of the anonymous namespaces this file opens. An # anonymous namespace's members are visible to the WHOLE file, not # just to code textually inside it, so without these a static helper # is unreachable from the named namespace that calls it two lines # later. That is not a stylistic gap — it is most of a C++ file's # internal call graph. self.anon_prefixes: list[str] = [] # class qualified name → {member name: core type name}. Shared across # every file in the ingest: a class declares its fields in a header and # uses them in a .cpp, so a per-file map cannot see its own members. self.members = members self.pending: list[Pending] = [] # ── helpers ────────────────────────────────────────────────────────── def scope_qual(self) -> str: return "::".join(n for _k, n in self.scopes) def enclosing_class(self) -> str | None: parts = [n for _k, n in self.scopes] for idx in range(len(self.scopes) - 1, -1, -1): if self.scopes[idx][0] == "class": return "::".join(parts[:idx + 1]) return None def namespace_prefixes(self) -> list[str]: """Enclosing scopes as resolution prefixes, innermost first, then ``.""" parts = [n for _k, n in self.scopes] out = ["::".join(parts[:k]) for k in range(len(parts), 0, -1)] out.append("") return out # ── the walk ───────────────────────────────────────────────────────── def scan(self, code: str, directives: list[tuple[int, str, str]]) -> None: for _line, name, rest in directives: if name == "include": target = self._include_target(rest) if target: self.facts.imports.append(Import(self.module, target)) self.toks, self.lines = tokenize(code) self._walk(0, len(self.toks)) @staticmethod def _include_target(rest: str) -> str | None: rest = rest.strip() if rest.startswith('"'): end = rest.find('"', 1) return rest[1:end] if end > 0 else None if rest.startswith("<"): end = rest.find(">", 1) return rest[1:end] if end > 0 else None return None # a macro-computed include names nothing we can read def _walk(self, i: int, end: int) -> None: """Declaration level: namespaces, classes, and function declarators.""" toks = self.toks stmt_start = i while i < end: t = toks[i] if t == ";": self._maybe_member_variable(stmt_start, i) i += 1 stmt_start = i continue if t == "}": return if t == "{": # A block we did not attribute to anything — an initializer # list at namespace scope, an `extern "C"` body, a bare scope. # Skipped whole rather than half-read. i = find_matching(toks, i, "{", "}") + 1 stmt_start = i continue if t == "namespace": i = self._namespace(i, end) stmt_start = i continue if t in ("class", "struct", "union"): nxt = self._class_head(i, end) if nxt is not None: i = nxt stmt_start = i continue if t == "enum": i = self._skip_enum(i, end) stmt_start = i continue if t == "template": j = i + 1 i = find_matching(toks, j, "<", ">") + 1 if j < end and toks[j] == "<" else i + 1 continue if t in ("using", "typedef", "static_assert", "friend"): i = self._skip_to_semicolon_or_block(i, end) stmt_start = i continue if t in ("public", "private", "protected") and i + 1 < end and toks[i + 1] == ":": i += 2 stmt_start = i continue if t == "extern": # `extern "C" { … }` is a linkage specification, not a scope: # what it contains belongs to the enclosing namespace. Walk # INTO it, or the generic unattributed-block case above skips # a whole C-linkage header without saying it did. j = i + 1 while j < end and toks[j] not in ("{", ";"): j += 1 if j < end and toks[j] == "{": close = find_matching(toks, j, "{", "}") self._walk(j + 1, close) i = close + 1 stmt_start = i continue if t == "(": nxt = self._declarator(stmt_start, i, end) if nxt is not None: i = nxt stmt_start = i continue i = find_matching(toks, i, "(", ")") + 1 continue i += 1 def _namespace(self, i: int, end: int) -> int: toks = self.toks j = i + 1 parts: list[str] = [] while j < end and (is_ident(toks[j]) or toks[j] == "::"): if toks[j] == "::" or toks[j] == "inline": j += 1 continue parts.append(toks[j]) j += 1 if j < end and toks[j] == "=": # namespace alias — not a scope return self._skip_to_semicolon_or_block(i, end) if j >= end or toks[j] != "{": return j + 1 # An anonymous namespace is internal linkage; its name is the file, so # two files' statics cannot collide on one qualified name. anonymous = not parts if anonymous: parts = [f"({self.module})"] close = find_matching(toks, j, "{", "}") for p in parts: self.scopes.append(("namespace", p)) if anonymous: self.anon_prefixes.append(self.scope_qual()) self._walk(j + 1, close) for _p in parts: self.scopes.pop() return close + 1 def _class_head(self, i: int, end: int) -> int | None: """`class X {`, `struct A::B {`, or a forward declaration.""" toks = self.toks j = i + 1 while j < end and toks[j] == "alignas": j = find_matching(toks, j + 1, "(", ")") + 1 parts: list[str] = [] while j < end and (is_ident(toks[j]) or toks[j] == "::"): if toks[j] == "::": j += 1 continue if toks[j] == "final": j += 1 break parts.append(toks[j]) j += 1 if not parts: # An anonymous struct — a member with no type name of its own. return find_matching(toks, j, "{", "}") + 1 if j < end and toks[j] == "{" else None if j < end and toks[j] == ":": # base-clause while j < end and toks[j] not in ("{", ";"): j += 1 if j >= end or toks[j] != "{": return None # forward declaration, or an elaborated type qual = "::".join([p for p in [self.scope_qual()] if p] + parts) kl = Klass(qual=qual, name=parts[-1], module=self.module, methods=0) self.facts.classes.append(kl) close = find_matching(toks, j, "{", "}") # A span for the class too, not only for its methods. Rationale is # attributed to the INNERMOST definition containing the comment, so a # class with no span does not merely lose class-level notes — a note # written between two methods falls outward to whatever encloses the # class, or to nothing. The Python walker records both; not doing so # here would make the same comment mean different things in a polyglot # store depending on the language it was written in. self.spans.append((qual, self.lines[j] if j < len(self.lines) else 0, self.lines[close] if close < len(self.lines) else 0)) # A qualified head (`struct Context::Impl`) enters every part it names. for p in parts: self.scopes.append(("class", p)) self.class_stack.append(kl) self.members.setdefault(qual, {}) self._walk(j + 1, close) self.class_stack.pop() for _p in parts: self.scopes.pop() # Past the closing brace there may be a declarator (`} instance_;`). k = close + 1 while k < end and toks[k] != ";": k += 1 return k + 1 def _skip_enum(self, i: int, end: int) -> int: toks = self.toks j = i while j < end and toks[j] not in ("{", ";"): j += 1 if j < end and toks[j] == "{": j = find_matching(toks, j, "{", "}") + 1 while j < end and toks[j] != ";": j += 1 return j + 1 def _skip_to_semicolon_or_block(self, i: int, end: int) -> int: """To the next `;`, or past a `{ … }` if a body comes first.""" toks = self.toks j = i while j < end and toks[j] not in (";", "{"): j += 1 if j < end and toks[j] == "{": return find_matching(toks, j, "{", "}") + 1 return j + 1 # ── declarators ────────────────────────────────────────────────────── def _declarator(self, stmt_start: int, lp: int, end: int) -> int | None: """A `(` at declaration level: a function, or not. Returns the index to continue from when it WAS one, else None so the caller can treat the parentheses as an expression and move on. """ toks, lines = self.toks, self.lines name_parts, name_start = self._declarator_name(stmt_start, lp) if name_parts is None: return None # `= f(1)` is an initializer, not a declarator. if name_start > stmt_start and toks[name_start - 1] in ("=", "return"): return None rp = find_matching(toks, lp, "(", ")") if rp >= end: return None after = self._after_parameter_list(rp + 1, end) if after is None: return None is_definition, at = after # A member DECLARED here bumps the class's method count. An out-of-line # definition (`void Context::feed(…)` at namespace scope) does not, or # every class that separates header from implementation counts double. if self.class_stack and len(name_parts) == 1: self.class_stack[-1].methods += 1 if not is_definition: return at + 1 close = find_matching(toks, at, "{", "}") start_line = lines[stmt_start] if stmt_start < len(lines) else lines[lp] end_line = lines[close] if close < len(lines) else start_line branches, loops, locals_ = self._scan_body(at, close, lp, rp) qual = "::".join([p for p in [self.scope_qual()] if p] + name_parts) # `std::optional Context::session_call(…)` is written at # NAMESPACE scope, so the lexical enclosing class is None and every # `impl_->…` inside it would resolve against nothing. The declarator's # own qualifier is what says which class the body belongs to. owner = ("::".join(qual.split("::")[:-1]) if len(name_parts) > 1 else self.enclosing_class()) # A `static` function at file scope has internal linkage: its name is # not visible to another translation unit, so two files may each hold # one. Qualifying by the file is what keeps them separate subjects. if (len(name_parts) == 1 and self.enclosing_class() is None and any(toks[k] == "static" for k in range(stmt_start, name_start))): qual = f"{self.module}::{name_parts[0]}" fn = Function( qual=qual, name=name_parts[-1], module=self.module, lines=max(1, end_line - start_line + 1), branches=branches, loops=loops, is_test=self.in_test_file and is_test_func(name_parts[-1])) self.facts.functions.append(fn) self.spans.append((qual, start_line, end_line)) pmin, pmax = arity_range(toks, lp, rp) self.defs.append(Definition(fn=fn, base_qual=qual, pmin=pmin, pmax=pmax, module=self.module, line=start_line)) self._record_calls(fn, at, close, locals_, owner) return close + 1 def _declarator_name(self, stmt_start: int, lp: int) -> tuple[list[str] | None, int]: """The declared name for a parameter list opening at `lp`. Walks LEFT from the `(`, which is what makes a return type of any shape irrelevant: `std::optional> wire::encode(` ends in `encode`, and the walk stops the moment it leaves the `name (:: name)*` chain. """ toks = self.toks k = lp - 1 if k < stmt_start: return None, lp # An operator function names itself with punctuation, so the walk-left # cannot find it — look right from the `operator` keyword instead. for j in range(stmt_start, lp): if toks[j] == "operator": spelling = "".join(toks[j + 1:lp]) if not spelling: return None, lp # this `(` is `operator()`'s NAME sep = " " if spelling[0].isalpha() else "" return ["operator" + sep + spelling], j if not is_ident(toks[k]) or toks[k] in KEYWORDS_NOT_CALLS: return None, lp parts = [toks[k]] start = k while k - 2 >= stmt_start and toks[k - 1] == "::" and is_ident(toks[k - 2]): parts.insert(0, toks[k - 2]) start = k - 2 k -= 2 if start - 1 >= stmt_start and toks[start - 1] == "~": parts[-1] = "~" + parts[-1] start -= 1 return parts, start def _after_parameter_list(self, j: int, end: int) -> tuple[bool, int] | None: """(is_definition, index of the `{` or `;`), or None if not a function. Everything a declarator may carry between `)` and its body is skipped here: cv-qualifiers, ref-qualifiers, `noexcept(…)`, `override`, `final`, a trailing return type, and a constructor's member-init list. """ toks = self.toks while j < end: t = toks[j] if t == "{": return (True, j) if t == ";": return (False, j) if t == ":": # Member-init list: run to the body, stepping over the parens # and braces each initializer brings with it. j += 1 while j < end and toks[j] != "{": if toks[j] in "([": closer = ")" if toks[j] == "(" else "]" j = find_matching(toks, j, toks[j], closer) + 1 continue if toks[j] == ";": return None j += 1 return (True, j) if j < end else None if t == "(": j = find_matching(toks, j, "(", ")") + 1 continue if t == "=": # `= default;` / `= delete;` / `= 0;` — a declaration that # still counts as one of the class's methods. j += 1 continue if t in TRAILING_QUALIFIERS or is_ident(t) or t in ("<", ">", ",", "0"): j += 1 continue return None return None # ── bodies ─────────────────────────────────────────────────────────── def _scan_body(self, lb: int, rb: int, lp: int, rp: int ) -> tuple[int, int, dict[str, str]]: """(branches, loops, locals) for one function body. Cyclomatic counting, the ordinary convention: one per `if`, per `case` label, per `catch`, per ternary, per `&&`, per `||`, and one per loop. `else` and `default` add nothing — neither is a decision, they are where control lands when the decisions above them did not fire. A lambda inside the body counts toward THIS function, unlike a nested `def` in the Python reader. That is not an inconsistency: a nested `def` is separately callable and gets its own Function, while a lambda is part of the text of the function that contains it. """ toks = self.toks branches = loops = 0 locals_: dict[str, str] = {} do_depths: list[int] = [] depth = 0 expect_do_while = False stmt_start = lb + 1 i = lb + 1 while i < rb: t = toks[i] if t == "{": depth += 1 elif t == "}": depth -= 1 if do_depths and do_depths[-1] == depth: do_depths.pop() expect_do_while = True i += 1 stmt_start = i continue elif t in ("if", "case", "catch", "?", "&&", "||"): branches += 1 elif t == "for": loops += 1 elif t == "do": loops += 1 do_depths.append(depth) elif t == "while": if expect_do_while: expect_do_while = False # the tail of a do-while else: loops += 1 if t not in ("}", "while"): expect_do_while = False if t == ";": self._read_variable(stmt_start, i, locals_) stmt_start = i + 1 elif t in ("{", "}"): stmt_start = i + 1 i += 1 # Parameters are locals too, and in a pimpl codebase they are how most # cross-object calls are spelled. for part in _split_top_level(toks[lp + 1:rp]): if len(part) >= 2 and is_ident(part[-1]): core_type = self._core_type(part[:-1]) if core_type: locals_[part[-1]] = core_type return branches, loops, locals_ @staticmethod def _core_type(type_toks: list[str]) -> str | None: """The class name a declared type ultimately refers to. The LAST identifier wins, which is what unwraps the wrappers a C++ codebase actually uses: `std::unique_ptr` yields `Impl`, `const std::vector&` yields `Frame`. A wrong guess costs nothing — an identifier that names no class in the ingest resolves to no edge at all. """ idents = [t for t in type_toks if is_ident(t) and t not in STORAGE_WORDS and t not in BUILTIN_TYPES] return idents[-1] if idents else None def _maybe_member_variable(self, stmt_start: int, semi: int) -> None: if self.class_stack: self._read_variable(stmt_start, semi, self.members.setdefault(self.class_stack[-1].qual, {})) def _read_variable(self, stmt_start: int, semi: int, into: dict[str, str]) -> None: """`Type name;` / `Type name = …;` / `Type name(…);` → name: Type. Deliberately narrow, for the reason the Python reader's binding rule is narrow: this is what the source STATES. `auto x = f()` states a type only to a compiler that can see through `f`, so it is not read here and calls through `x` go unrecorded rather than guessed. """ seg = self.toks[stmt_start:semi] if not seg: return for stop in ("=", "(", "{", "["): if stop in seg: seg = seg[:seg.index(stop)] break if len(seg) < 2 or not is_ident(seg[-1]) or seg[-1] in KEYWORDS_NOT_CALLS: return name = seg[-1] core_type = self._core_type(seg[:-1]) if core_type and core_type != name: into[name] = core_type # ── call sites ─────────────────────────────────────────────────────── def _record_calls(self, caller: Function, lb: int, rb: int, locals_: dict[str, str], owner: str | None) -> None: toks, lines = self.toks, self.lines i = lb + 1 while i < rb: if toks[i] != "(": i += 1 continue chain = self._callee_chain(i, lb) if chain is None: i += 1 continue rp = find_matching(toks, i, "(", ")") segs, seps = chain self.facts.calls_seen += 1 self.pending.append(Pending( caller=caller, segs=segs, seps=seps, argc=call_arity(toks, i, rp), line=lines[i] if i < len(lines) else 0, scopes=self.namespace_prefixes(), class_qual=owner, locals_=locals_, path=self.path)) i += 1 def _callee_chain(self, lp: int, floor: int) -> tuple[list[str], list[str]] | None: toks = self.toks k = lp - 1 if k <= floor or not is_ident(toks[k]) or toks[k] in KEYWORDS_NOT_CALLS: return None segs = [toks[k]] seps: list[str] = [] while (k - 2 > floor and toks[k - 1] in ("::", ".", "->") and is_ident(toks[k - 2])): seps.insert(0, toks[k - 1]) segs.insert(0, toks[k - 2]) k -= 2 # A declaration inside a body (`Frame f(a, b);`) is not a call. It is # told apart the way the declaration level tells them apart: a type # name sitting directly before the callee. if k - 1 > floor and is_ident(toks[k - 1]) and toks[k - 1] not in KEYWORDS_NOT_CALLS: return None return segs, seps # ── disambiguation + resolution ────────────────────────────────────────── def disambiguate(defs: list[Definition]) -> None: """Give every definition its own subject, and say how they differ. C++ overloads share a qualified name. `Function.qual` becomes a store subject, so leaving them equal would let one function's facts silently overwrite another's — the exact failure `Ids` exists to prevent, arriving one layer earlier where `Ids` cannot see it. An overload set is therefore keyed by parameter count, which is both the thing that tells them apart at a call site and something a reader of the subject can interpret. Two overloads with the SAME count (a copy and a move constructor, say) take a further index — deterministic by file and line, and calls into such a set resolve to nothing, because argument count cannot choose between them and guessing is the one thing this reader does not do. """ by_name: dict[str, list[Definition]] = {} for d in defs: by_name.setdefault(d.base_qual, []).append(d) for base, group in by_name.items(): if len(group) == 1: continue group.sort(key=lambda d: (d.module, d.line)) by_count: dict[int, list[Definition]] = {} for d in group: by_count.setdefault(d.pmax, []).append(d) for count, same in by_count.items(): for n, d in enumerate(same): suffix = f"/{count}" if len(same) == 1 else f"/{count}#{n + 1}" d.fn.qual = base + suffix def resolve_all(scanners: list[FileScanner], facts: Facts, defs: list[Definition]) -> None: by_base: dict[str, list[Definition]] = {} for d in defs: by_base.setdefault(d.base_qual, []).append(d) classes: dict[str, list[str]] = {} for kl in facts.classes: classes.setdefault(kl.name, []).append(kl.qual) for sc in scanners: for p in sc.pending: target, ambiguous, nameable = _resolve(p, by_base, classes, sc) if target is None: if ambiguous is None and not nameable: # CODE-INGEST-CPP-DROPS-IN-CORPUS-CONSTRUCTOR-CALLS. The # target could not be named — the receiver's type was not # inferable — so this site could be why any ingested # function looks uncalled. Recorded under the SOURCE text, # since there is no resolved name to record. facts.unresolved.append(UnresolvedCall( caller_qual=p.caller.qual, site=f"{p.path}:{p.line}", text=f"{'::'.join(p.segs)}/{p.argc}")) elif ambiguous is not None: # CODE-MODULE-UNRESOLVED-CALL-SITES — no edge is invented, # but the site is recorded: the callee is in the corpus and # the overload could not be picked, so one of them now looks # uncalled. An `ambiguous is None` miss is an external call # and is not recorded — it cannot suppress an ingested # function, so it says nothing about `uncalled()`. facts.unresolved.append(UnresolvedCall( caller_qual=p.caller.qual, site=f"{p.path}:{p.line}", text=f"{ambiguous}/{p.argc}")) continue facts.calls_resolved += 1 facts.calls.append(Call(caller_qual=p.caller.qual, callee_qual=target, site=f"{p.path}:{p.line}")) def _with_constructors(out: list[str], short: str, classes: dict[str, list[str]]) -> list[str]: """Append the constructor candidates for `T(...)` where `T` names a class. A constructor is stored under `{class_qual}::{short}` — the same shape as any method — while `T(...)` reads as a call to a FUNCTION named `T`. So a candidate list built for the function reading finds nothing, and without this the edge is lost: the miss returns no ambiguous name, `resolve_all` reads that as an external call, and the constructor is left looking uncalled with nothing recorded to say why. The candidates already enumerate every class qual `T` could denote from here, so the constructor candidates are those same entries plus `::short`. No second scope search, and nothing that scope lookup would not have reached. Appended LAST, and only when `short` really names a class, so a call that resolves to a function today still resolves to that function. """ if short not in classes: return out return out + [f"{c}::{short}" for c in out] def _candidates(p: Pending, classes: dict[str, list[str]], sc: FileScanner) -> list[str]: if not p.seps: name = p.segs[0] out = [f"{p.class_qual}::{name}"] if p.class_qual else [] out += [f"{prefix}::{name}" if prefix else name for prefix in p.scopes] out += [f"{a}::{name}" for a in sc.anon_prefixes] out += [f"{u}::{name}" for u in sc.using_namespaces] out.append(f"{sc.module}::{name}") return _with_constructors(out, name, classes) if all(s == "::" for s in p.seps): qualified = "::".join(p.segs) out = [qualified] out += [f"{prefix}::{qualified}" for prefix in p.scopes if prefix] out += [f"{a}::{qualified}" for a in sc.anon_prefixes] out += [f"{u}::{qualified}" for u in sc.using_namespaces] return _with_constructors(out, p.segs[-1], classes) base, member = p.segs[0], p.segs[-1] if base == "this" and p.class_qual: return [f"{p.class_qual}::{member}"] core_type = p.locals_.get(base) if core_type is None and p.class_qual: core_type = sc.members.get(p.class_qual, {}).get(base) if not core_type: return [] return [f"{cls_qual}::{member}" for cls_qual in classes.get(core_type, [])] def _resolve(p: Pending, by_base: dict[str, list[Definition]], classes: dict[str, list[str]], sc: FileScanner ) -> tuple[str | None, str | None, bool]: """(resolved target, ambiguous name, nameable) — at most one of the first two is set. CODE-MODULE-UNRESOLVED-CALL-SITES: the second element separates the two reasons this used to answer `None` for, which are not the same fact. no candidate matched → the callee is OUTSIDE the ingested set (a `std::` or third-party call). It can never make an ingested function look uncalled, so it is not a dispatch site and nothing is recorded. a candidate MATCHED, → the callee IS in the corpus and we still drop the arity did not pick edge. One of those overloads now looks uncalled, one overload which is exactly the hole `uncalled()` warns about — so the SITE is recorded, with the name. This is the C++ analogue of Python's unnameable target, and it is the sharper example of the two: here the walker knows the callee is in the corpus and drops the edge anyway. """ cands = _candidates(p, classes, sc) for cand in cands: group = by_base.get(cand) if not group: continue fits = [d for d in group if d.pmin <= p.argc <= d.pmax] # A name that exists but whose overloads cannot be told apart by # argument count resolves to nothing and stops looking. Falling # through to an outer scope here would find a DIFFERENT function of # the same name, which C++ name lookup would never have reached. if len(fits) == 1: return fits[0].fn.qual, None, True return None, cand, True # `nameable` is the third answer, and it is the one UnresolvedCall's # taxonomy turns on: an EMPTY candidate list means the target could not be # named from here at all — a call through a variable whose type this reader # could not infer. A non-empty list that matched nothing means the target # WAS named and simply lies outside the ingested set. Only the first can # make an ingested function look uncalled, so only the first is recorded. return None, None, bool(cands) # ── tests, dependencies, walking ───────────────────────────────────────── def is_test_file(path: str) -> bool: """A file a C++ test build would compile as a test, by name or directory.""" p = PurePosixPath(path.replace("\\", "/")) if p.name.startswith("test_") or p.stem.endswith("_test"): return True return core.path_in_test_dir(path) def is_test_func(name: str) -> bool: """A test callable, by the conventions C++ test code actually uses. Narrow on purpose. C++ has no collection protocol to read — no `test_*` rule a runner enforces — so anything looser starts inferring, and a wrong test edge is worse than a missing one for the same reason a wrong call edge is. """ return name.startswith("test_") or name == "test" or name.endswith("_test") _CMAKE_FIND = re.compile(r"^[ \t]*find_package[ \t]*\(\s*([A-Za-z0-9_.+-]+)([^)]*)\)", re.IGNORECASE | re.MULTILINE) _CMAKE_FETCH = re.compile(r"^[ \t]*FetchContent_Declare[ \t]*\(\s*([A-Za-z0-9_.+-]+)([^)]*)\)", re.IGNORECASE | re.MULTILINE) _CMAKE_PKGCONFIG = re.compile( r"^[ \t]*pkg_check_modules[ \t]*\(\s*[A-Za-z0-9_]+\s+([^)]*)\)", re.IGNORECASE | re.MULTILINE) _GIT_TAG = re.compile(r"GIT_TAG\s+([A-Za-z0-9_.+-]+)") def read_dependencies(roots: list[Path], facts: Facts) -> None: """What the project DECLARED it builds against. Offline, always. C++ has no single manifest, so this reads the one that is a file in the tree: CMake. `find_package` and `pkg_check_modules` name a dependency and usually no exact version, so they are recorded unpinned — a version RANGE is not a version, and writing one would be a guess the store cannot tell from a fact. `FetchContent_Declare` carries a `GIT_TAG`, which IS exact, so it is recorded pinned. A tree built by hand — a Makefile, a shell script — declares nothing this can read, and then nothing is recorded. That is the honest answer, and it is why `unused_dependencies()` over such a tree is empty rather than wrong. """ for root in roots: base = root if root.is_dir() else root.parent for path in sorted(base.rglob("CMakeLists.txt")) + sorted(base.rglob("*.cmake")): try: text = path.read_text(encoding="utf-8") except (OSError, UnicodeDecodeError): facts.files_failed.append((str(path), "unreadable-cmake")) continue got = 0 for m in _CMAKE_FIND.finditer(text): got += _add_dep(facts, m.group(1), "", False) for m in _CMAKE_FETCH.finditer(text): tag = _GIT_TAG.search(m.group(2)) got += _add_dep(facts, m.group(1), tag.group(1) if tag else "", bool(tag)) for m in _CMAKE_PKGCONFIG.finditer(text): for word in m.group(1).split(): if word.upper() in ("REQUIRED", "QUIET", "IMPORTED_TARGET"): continue got += _add_dep(facts, re.split(r"[<>=!]", word)[0], "", False) if got: facts.dep_files.append(str(path)) def _add_dep(facts: Facts, name: str, version: str, pinned: bool) -> int: key = name.lower() if not name or key in facts.seen_deps: return 0 facts.seen_deps.add(key) facts.deps.append(Dependency(name, version, True, pinned)) return 1 def module_name_for(path: Path, root: Path) -> str: """The file, POSIX-relative to the tree it was found in. A C++ file IS the unit an include names, so the path is what makes `code.Import.imported` join with `code.Function.module` and `importers_of("src/wire.cpp")` answer. """ try: rel = path.relative_to(root) except ValueError: rel = path return PurePosixPath(rel).as_posix() _USING_NS = re.compile(r"\busing\s+namespace\s+([A-Za-z_][A-Za-z0-9_:]*)\s*;") def collect(roots: list[Path], excludes: list[str]) -> Facts: facts = Facts() scanners: list[FileScanner] = [] defs: list[Definition] = [] members: dict[str, dict[str, str]] = {} for root in roots: base = root if root.is_dir() else root.parent if root.is_dir(): files = sorted(p for p in root.rglob("*") if p.suffix in EXTENSIONS) else: files = [root] for path in files: rel = str(path) if any(x in rel for x in excludes): facts.files_excluded += 1 continue facts.files_found += 1 try: src = path.read_text(encoding="utf-8") except (OSError, UnicodeDecodeError) as exc: facts.files_failed.append((rel, type(exc).__name__)) continue comments: list[tuple[int, str]] = [] code, directives = scrub(src, comments) sc = FileScanner(module_name_for(path, base), rel, facts, defs, members) # `using namespace antheos::wire;` widens what a bare name may # mean. Read per FILE rather than per scope: coarser than C++ — # a using inside one function is treated as file-wide — and it can # only ever ADD a candidate, which is kept only if a definition of # that exact qualified name exists. So the coarseness costs # precision in principle and cannot manufacture an edge. sc.using_namespaces = [m.group(1) for m in _USING_NS.finditer(code)] sc.scan(code, directives) for cline, body in comments: hit = core.rationale_marker(body) if hit is None: continue owner = core.enclosing_definition(cline, sc.spans) if owner is None: continue # above a definition, or file-level: not ours facts.rationale.append(core.Rationale( subject=owner, marker=hit[0], text=hit[1], site=f"{rel}:{cline}")) facts.files_parsed += 1 scanners.append(sc) # Overloads share a qualified name and a subject must not be shared, so # this happens BEFORE resolution — the resolver returns final quals. disambiguate(defs) # Resolution is a SECOND pass, for the same reason it is in the Python # reader: a call into a file read later must still resolve. resolve_all(scanners, facts, defs) # THIRD, over the resolved graph. facts.tested = reached_by_tests(facts) facts.test_files = sum(1 for sc in scanners if sc.in_test_file) _resolve_include_targets(facts, scanners) read_dependencies(roots, facts) return facts def _resolve_include_targets(facts: Facts, scanners: list[FileScanner]) -> None: """Point a quoted include at the file in this tree that it names. `#include "antheos.hpp"` from `src/wire.cpp` becomes an edge to `include/antheos.hpp`, so `importers_of` answers with the module names `code.Function.module` already uses. An include this tree does not contain — ``, a system header, a dependency — keeps the text as written, which is what makes it match a `Dependency` by name instead. A suffix that two files in the tree both end with is left alone: two candidates means the source did not say which, and naming one would be a guess in a file whose whole point is that it does not make them. """ by_suffix: dict[str, set[str]] = {} for sc in scanners: parts = sc.module.split("/") for k in range(len(parts)): by_suffix.setdefault("/".join(parts[k:]), set()).add(sc.module) for im in facts.imports: hit = by_suffix.get(im.imported.replace("\\", "/")) if hit and len(hit) == 1: im.imported = next(iter(hit)) LANG = Language( tag="cpp", name="C++", prog="dke-ingest-cpp", extensions=EXTENSIONS, caveats=[ "", "For C++ that means: a call through a function pointer, a", "std::function, a VIRTUAL dispatch, a template parameter or a macro", "is not visible in source and is not recorded. Virtual dispatch is", "the one to keep in mind — an override reached only through a base", "reference looks uncalled here while running on every request.", "", "This reader is hand-written against the C++ grammar rather than", "built on a compiler front end, so it also does not expand macros,", "instantiate templates, or evaluate #if. Code inside a disabled #if", "IS read, since deciding otherwise would mean evaluating the", "preprocessor. Overloads are told apart by ARGUMENT COUNT only, and", "carry a /N suffix on their subject; when two of them could take the", "count written at a call site, the edge is left unrecorded.", "", "`branches` counts one per if, per CASE LABEL, per catch, per", "ternary, and per && or ||. `else` and `default` count nothing.", "A lambda's branches belong to the function containing it.", ], collect=collect, ) def emit(facts: Facts, ids: Ids, *, stamp: int | None = None, load_key: str | None = None) -> str: """Render C++ facts as a DKE Python program.""" return core.emit(facts, ids, LANG, stamp=stamp, load_key=load_key) def main(argv: list[str] | None = None) -> int: return core.run(LANG, argv) if __name__ == "__main__": sys.exit(main())