fix(static): let validated Python string literals own their quotes in marker parsing

docs/ANALYSIS_RESOURCE_BOUNDS.md

@@ -170,6 +170,21 @@ comment that itself holds more unresolved shell text than the parser bound,
 invalid or fragmentary Python, and source with a non-Python shebang keep the
 conservative result.
 
+Declared-marker reconstruction uses the same proof, as it does for validated
+JSON strings. When a removal verb is followed later on its line by the closing
+quote of a Python string, as in `assert "please omit --hours" in out`, that
+quote is not paired with the next literal's opening quote across the code
+between them. An explicit declaration, where only a short phrase such as
+`the marker` separates the verb from the quote, keeps the lexical reading.
+Marker declarations written inside one string or comment are still
+reconstructed or reported as `obfuscated_instruction_text`. Invalid or
+fragmentary Python and other file types keep the lexical reading.
+
+Both consumers share one parse and tokenization of a module. The two most
+recent results, including a failed proof, are reused, and each reuse still
+checks the static runtime budget. A proof interrupted by that budget is not
+kept.
+
 ## Structured skill data
 
 AISOP/AISP structured extraction consumes the already-bounded cache and shares the enclosing

src/skillspector/nodes/analyzers/static_patterns_tool_misuse.py

@@ -25,19 +25,17 @@
 from __future__ import annotations
 
 import ast
-import io
 import re
 import sys
-import tokenize
-import warnings
 from bisect import bisect_right
 from collections.abc import Callable, Iterator
 from dataclasses import dataclass
-from threading import Lock
 
 from skillspector.logging_config import get_logger
 from skillspector.models import AnalyzerFinding, Location, Severity
-from skillspector.python_ast import MAX_PYTHON_AST_SOURCE_CHARS, parse_python_source
+from skillspector.python_ast import parse_python_source
+from skillspector.python_tokens import PythonLiteralSpans
+from skillspector.python_tokens import python_literal_spans as _python_literal_spans
 from skillspector.security_reconstruction import validated_json_string_spans
 from skillspector.state import AnalyzerNodeResponse, SkillspectorState
 
@@ -115,26 +113,6 @@
 # whitespace or a shell control character. A backslash-newline is a line
 # continuation in both languages, so it does not end the word.
 _PYTHON_SHELL_WORD_BREAK_RE = re.compile(r"(?<!\\)(?<!\\\r)[\s;|&()<>]")
-# The tokenizer and the compiler disagree about a carriage return that does
-# not start CRLF, so such source never acquires token ownership.
-_LONE_CARRIAGE_RETURN_RE = re.compile(r"\r(?!\n)")
-# A shebang that names another interpreter (for example a shell polyglot)
-# makes the Python host ambiguous. Without a shebang, ``.py`` source is Python.
-_PYTHON_SHEBANG_RE = re.compile(
-    r"#![ \t]*+(?:\S*/)?(?:env[ \t]++(?:-\S*[ \t]++)*(?:\S*/)?)?"
-    r"(?:python[0-9.]*|pypy[0-9.]*|uv)(?=[ \t\r\n]|$)"
-)
-# ``warnings.catch_warnings`` swaps the process-global filter list, and analyzer
-# nodes run on concurrent graph worker threads. Two such blocks that exit out of
-# order can leave their ``ignore`` filter installed for the whole process, so
-# the ownership parse serializes its block.
-_PYTHON_OWNERSHIP_PARSE_LOCK = Lock()
-_PYTHON_TEMPLATE_STRING_STARTS = frozenset(
-    {tokenize.FSTRING_START, getattr(tokenize, "TSTRING_START", tokenize.FSTRING_START)}
-)
-_PYTHON_TEMPLATE_STRING_ENDS = frozenset(
-    {tokenize.FSTRING_END, getattr(tokenize, "TSTRING_END", tokenize.FSTRING_END)}
-)
 _PRINTF_FORMAT_CONVERSION_RE = re.compile(r"%[-+ #0-9.*']*[A-Za-z%]")
 _SHELL_ROOT_TARGET_ESCAPE_RE = re.compile(
     r"\\(?:[/~*?]|x(?:2[fF]|7[eE]|2[aA]|3[fF])|"
@@ -2550,84 +2528,6 @@ def _perl_literal_print_shell_text(
     return "".join(output)
 
 
-def _python_literal_spans(
-    content: str,
-    check_runtime: Callable[[], None],
-) -> tuple[list[int], list[int]] | None:
-    """Return outermost string and comment token spans of valid Python source.
-
-    Return ``None`` unless the whole text is accepted by the Python parser,
-    so a fragment, malformed file, or shell-shebang polyglot cannot borrow
-    host ownership. Every span is checked against the exact source text.
-    """
-    # A leading U+FEFF byte-order mark fails the module parse below and keeps
-    # every conservative bound. The file cache decodes with ``utf-8``, not
-    # ``utf-8-sig``, so the AST analyzers already report such a file as a
-    # syntax error; stripping the mark belongs with that decoding fix.
-    if (
-        len(content) > MAX_PYTHON_AST_SOURCE_CHARS
-        or _LONE_CARRIAGE_RETURN_RE.search(content) is not None
-        or (content.startswith("#!") and _PYTHON_SHEBANG_RE.match(content) is None)
-    ):
-        return None
-    check_runtime()
-    try:
-        # The lenient tokenizer accepts bytes such as ``$`` and a backtick as
-        # operators. Requiring a module parse proves that every byte outside
-        # the spans below is Python syntax, never a shell quote or expansion.
-        # The scanned module's own parse already reports its compiler warnings
-        # (for example an invalid ``"\d"`` escape). Repeating them here would
-        # duplicate that output, and ``-W error`` would turn them into a
-        # ``SyntaxError`` that silently drops ownership.
-        with _PYTHON_OWNERSHIP_PARSE_LOCK, warnings.catch_warnings():
-            warnings.simplefilter("ignore")
-            ast.parse(content)
-    except (SyntaxError, ValueError, RecursionError):
-        return None
-    check_runtime()
-    line_starts = [0]
-    line_starts.extend(match.end() for match in re.finditer("\n", content))
-    starts: list[int] = []
-    ends: list[int] = []
-    template_depth = 0
-    template_start = 0
-
-    def offset(position: tuple[int, int]) -> int:
-        return line_starts[position[0] - 1] + position[1]
-
-    try:
-        for index, token in enumerate(tokenize.generate_tokens(io.StringIO(content).readline)):
-            if index % 256 == 0:
-                check_runtime()
-            if token.type in _PYTHON_TEMPLATE_STRING_STARTS:
-                if template_depth == 0:
-                    template_start = offset(token.start)
-                    if not content.startswith(token.string, template_start):
-                        return None
-                template_depth += 1
-            elif token.type in _PYTHON_TEMPLATE_STRING_ENDS:
-                template_depth -= 1
-                end = offset(token.end)
-                if template_depth < 0 or not content.endswith(token.string, 0, end):
-                    return None
-                if template_depth == 0:
-                    starts.append(template_start)
-                    ends.append(end)
-            elif not template_depth and token.type in (tokenize.STRING, tokenize.COMMENT):
-                # Inside a replacement field, nested strings and comments
-                # belong to the enclosing f-string or t-string span.
-                start, end = offset(token.start), offset(token.end)
-                if content[start:end] != token.string:
-                    return None
-                starts.append(start)
-                ends.append(end)
-    except (tokenize.TokenError, SyntaxError, ValueError, IndexError):
-        return None
-    if template_depth or any(ends[index] > starts[index + 1] for index in range(len(starts) - 1)):
-        return None
-    return starts, ends
-
-
 class _PythonSourceOwnership:
     """Lazily proven Python string and comment ownership for one source text.
 
@@ -2646,12 +2546,12 @@ class _PythonSourceOwnership:
     def __init__(self, content: str, check_runtime: Callable[[], None]) -> None:
         self._content = content
         self._check_runtime = check_runtime
-        self._spans: tuple[list[int], list[int]] | None = None
+        self._spans: PythonLiteralSpans | None = None
         self._spans_computed = False
         self._cached_word_start = -1
         self._cached_word_end = -1
 
-    def _owned_spans(self) -> tuple[list[int], list[int]] | None:
+    def _owned_spans(self) -> PythonLiteralSpans | None:
         if not self._spans_computed:
             self._spans = _python_literal_spans(self._content, self._check_runtime)
             self._spans_computed = True

src/skillspector/nodes/analyzers/static_runner.py

@@ -62,6 +62,7 @@
     ParsedPythonFile,
     get_python_ast,
 )
+from skillspector.python_tokens import PythonStringClosers
 from skillspector.security_reconstruction import (
     MAX_DECLARED_MARKER_RIGHT_CONTEXT_CHARS,
     MAX_MARKER_LOOKAHEAD_CHARS,
@@ -1592,6 +1593,17 @@ def source_end_offset(offset: int) -> int:
             finding.end_column = raw_end_column if has_exact_end else None
 
 
+def _window_string_closer(
+    python_strings: PythonStringClosers, window_start: int
+) -> Callable[[int], bool]:
+    """Answer string-closer queries in one window's source coordinates."""
+
+    def closes_string(offset: int) -> bool:
+        return python_strings.closes_string(window_start + offset)
+
+    return closes_string
+
+
 def _scan_declared_marker_views(
     path: str,
     content: str,
@@ -1622,6 +1634,12 @@ def check_runtime() -> None:
     bounded_parse_limited = False
     seen_views: set[tuple[str, int, int]] = set()
     seen_finding_counts: dict[tuple[object, ...], int] = {}
+    # A complete Python module proves which quotes close its string literals.
+    # The proof covers the whole artifact, so it holds in every window. It is
+    # computed only when a loose quoted directive header reaches a quote.
+    python_strings = (
+        PythonStringClosers(content, check_runtime) if _infer_file_type(path) == "python" else None
+    )
 
     for owned_start, raw_start in zip(owned_starts, raw_starts, strict=True):
         check_runtime()
@@ -1647,13 +1665,21 @@ def check_runtime() -> None:
             )
         )
         check_runtime()
+        # View offsets are window-relative. Synthetic Markdown fence context
+        # never occurs for Python, but it would not map onto module offsets.
+        string_closer = (
+            None
+            if python_strings is None or context_prefix
+            else _window_string_closer(python_strings, raw_start)
+        )
         for full_view in full_views:
             reconstruction = build_declared_marker_views(
                 full_view,
                 check_runtime=check_runtime,
                 owned_source_start=owned_source_start,
                 owned_source_end=owned_source_end,
                 source_end_is_truncated=raw_end < len(content),
+                validated_string_closer=string_closer,
             )
             projection_limited = projection_limited or reconstruction.limited
             for marker_view in reconstruction.views: