Skip to content

fix(static): stop TR2/TR3 over-reading trigger descriptions (#669) #709

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rng1995 merged 4 commits into from
Oct 6, 2026
Merged

fix(static): stop TR2/TR3 over-reading trigger descriptions (#669) #709

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
388af29
fix(static): stop TR2/TR3 over-reading trigger descriptions (#669)
Zhuoxi2000 Oct 1, 2026
83e784e
fix(static): keep main's TR2/TR3 detections that the #669 narrowing d…
Zhuoxi2000 Oct 3, 2026
6e6d54b
fix(static): judge TR3 qualifier objects past determiners and catch p…
Zhuoxi2000 Oct 4, 2026
912dadc
fix(static): skip stacked determiners in TR3 objects and accept passi…
Zhuoxi2000 Oct 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 83 additions & 13 deletions src/skillspector/nodes/analyzers/static_patterns_supply_chain.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -773,13 +773,34 @@ def _is_typosquat(
)

# Universal-scope signals for description clauses. The scope must be
# unconditional: a subject qualifier such as "about PostgreSQL" keeps the
# clause describing a capability, not a catch-all trigger.
# unconditional: a subject or domain qualifier such as "about PostgreSQL",
# "with PDF files" or "related to Kubernetes" keeps the clause describing a
# capability, not a catch-all trigger. Up to three leading determiners,
# quantifiers or "of" are transparent ("about this codebase", "about any AWS
# service", "about all the services", "about all of the services" are still
# bounded by the noun after them). A qualifier whose object is only a pronoun
# or a generic noun ("anything with it", "anything with anyone", "anything
# with any of them", "anything related to any topic", "any messages with any
# content") bounds nothing and stays universal. Broad prepositions ("in",
# "for", "on") are deliberately not qualifiers: "any message in the chat" is
# still every message.
_DESCRIPTION_SCOPE_DETERMINER = (
r"(?:the|a|an|this|that|these|those|any|all|every|each|some|"
r"my|your|our|their|its|his|her)"
)
_DESCRIPTION_UNBOUNDED_OBJECT = (
r"(?:it|them|you|me|us|him|her|anything|everything|whatever|something|"
r"nothing|anyone|anybody|someone|somebody|everyone|everybody|nobody|"
r"topics?|subjects?|things?|content)"
)
_DESCRIPTION_UNIVERSAL_SCOPE_RE = re.compile(
r"\b(?:"
r"anything|everything|whatever|"
r"(?:all|any|every)\s+(?:messages?|requests?|questions?|queries?|inputs?|tasks?)"
r")\b(?!\s+about\b)",
r")\b"
r"(?!\s+(?:about|with|involving|regarding|concerning|(?:related|relating)\s+to)\s+"
rf"(?:(?:{_DESCRIPTION_SCOPE_DETERMINER}|of)\s+){{0,3}}"
rf"(?!(?:{_DESCRIPTION_UNBOUNDED_OBJECT}|{_DESCRIPTION_SCOPE_DETERMINER}|of)\b)[a-z0-9])",
re.IGNORECASE,
)

Expand All @@ -790,21 +811,44 @@ def _is_typosquat(
# "CI/CD" must not read as invocation intent for "build".
_DESCRIPTION_INVOCATION_RE = re.compile(
r"\b(?:commands?|slash|invoke[sd]?|invoking|intercept(?:s|ed|ing)?|"
r"override[sd]?|overriding|shadow(?:s|ed|ing)?)\b|(?<![\w/])/[a-z]",
r"override[sd]?|overriding|overridden|shadow(?:s|ed|ing)?)\b|(?<![\w/])/[a-z]",
re.IGNORECASE,
)

# Command-interception signals for the TR2 shadow-command rule. Unlike the
# Command-interception evidence for the TR2 shadow-command rule. Unlike the
# broader extraction gate above, this requires an actual
# invocation/interception/override claim (or a literal slash-command token):
# merely discussing commands as a noun ("Show available build commands",
# "documents the build and test commands") describes documentation or help
# prose, not shadowing intent, so it must not establish TR2 on its own.
_DESCRIPTION_COMMAND_INTERCEPTION_RE = re.compile(
# The shadowed command must also be the object of that evidence, not any
# built-in word that happens to share the clause: a slash-command token names
# a command by its whole name ("/ask-matt" is "ask-matt", not the built-in
# "ask"), and an interception verb governs the built-ins that follow it in
# the clause ("Intercepts and replaces the built-in deploy command"). A
# slash-command token that is not a built-in ("/ask-matt") stays one name, and
# a home-relative path ("~/build") is not a slash command.
_DESCRIPTION_SLASH_COMMAND_RE = re.compile(r"(?<![\w/.~])/([a-z][\w-]*)", re.IGNORECASE)
_DESCRIPTION_INTERCEPTION_VERB_RE = re.compile(
r"\b(?:invoke[sd]?|invoking|intercept(?:s|ed|ing)?|"
r"override[sd]?|overriding|shadow(?:s|ed|ing)?)\b|(?<![\w/])/[a-z]",
r"override[sd]?|overriding|overridden|shadow(?:s|ed|ing)?)\b",
re.IGNORECASE,
)
# Passive interception claims name the command before the verb: "the built-in
# deploy command is intercepted", "types deploy it is intercepted", "the deploy
# command will be intercepted", "deploy commands are now shadowed". The
# auxiliary may be "is"/"are" (optionally "being"), "gets", "will be" or
# "has/have been", and one adverb may precede the participle. The adverb slot
# takes a short list plus "-ly" words, so a negation ("is not intercepted",
# "is never shadowed") does not count.
_DESCRIPTION_PASSIVE_INTERCEPTION_RE = re.compile(
r"(?<![\w/.~-])/?([a-z][\w-]*)\s+(?:commands?\s+|it\s+)?"
r"(?:(?:is|are)(?:\s+being)?|gets?|will\s+be|ha(?:s|ve)\s+been)\s+"
r"(?:(?:always|now|also|still|already|[a-z]+ly)\s+)?"
r"(?:intercepted|overridden|shadowed|invoked)\b",
re.IGNORECASE,
)
_DESCRIPTION_COMMAND_TOKEN_RE = re.compile(r"(?<![\w/.~-])/?([a-z][\w-]*)", re.IGNORECASE)

# Trigger-phrase extraction for the TR1 broad/short-trigger rule on
# descriptions: the word or phrase the skill claims to activate on, as in
Expand Down Expand Up @@ -921,6 +965,33 @@ def _description_condition_has_universal_scope(clause: str) -> bool:
return _DESCRIPTION_UNIVERSAL_SCOPE_RE.search(span) is not None


def _description_shadowed_commands(clause: str) -> list[str]:
"""Built-in commands a description clause claims to invoke or intercept.

Only commands tied to interception evidence count: a slash-command token
whose whole name is a built-in, a built-in named anywhere after an
invocation/interception/override verb in the clause, or a built-in that a
passive claim names right before its auxiliary ("deploy is intercepted",
"deploy command has been overridden").
Returns the sorted set of shadowed built-in commands.
"""
shadowed = {
match.group(1).lower()
for match in _DESCRIPTION_SLASH_COMMAND_RE.finditer(clause)
if match.group(1).lower() in _BUILTIN_COMMANDS
}
verb = _DESCRIPTION_INTERCEPTION_VERB_RE.search(clause)
if verb is not None:
tokens = _DESCRIPTION_COMMAND_TOKEN_RE.findall(clause[verb.end() :])
shadowed.update(t.lower() for t in tokens if t.lower() in _BUILTIN_COMMANDS)
shadowed.update(
match.group(1).lower()
for match in _DESCRIPTION_PASSIVE_INTERCEPTION_RE.finditer(clause)
if match.group(1).lower() in _BUILTIN_COMMANDS
)
return sorted(shadowed)


def _extract_description_trigger_clauses(description: str) -> tuple[list[str], int]:
"""Extract bounded trigger-like clauses from a skill description.

Expand Down Expand Up @@ -2440,7 +2511,6 @@ def _analyze_triggers(

for i, clause in enumerate(description_clauses, 1):
clause_lower = clause.lower().strip()
words = clause_lower.split()

# TR1 (description-calibrated): extract the trigger phrase the skill
# claims to activate on ("whenever the user says hello") and apply
Expand Down Expand Up @@ -2501,11 +2571,11 @@ def _analyze_triggers(
# capability or documentation prose such as "Show available build
# commands" merely discusses commands and stays out of the trigger
# path. Invocation clauses pass the extraction gate on their own, so
# no broad-activation wording is required.
shadowed = sorted(
{cmd for cmd in _BUILTIN_COMMANDS if cmd in {w.lstrip("/") for w in words}}
)
if shadowed and _DESCRIPTION_COMMAND_INTERCEPTION_RE.search(clause):
# no broad-activation wording is required. The shadowed command must
# be the object of that claim: "/ask-matt" does not shadow "ask", and
# the noun "ask" elsewhere in the clause is not an invocation.
shadowed = _description_shadowed_commands(clause)
if shadowed:
findings.append(
Finding(
rule_id="TR2",
Expand Down
234 changes: 234 additions & 0 deletions tests/unit/test_patterns_new.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2665,6 +2665,240 @@ def test_description_scope_bound_to_activation_condition(self, description: str)
findings = sc_mod._analyze_triggers({"description": description}, "myskill")
assert findings == []

@pytest.mark.parametrize(
"description",
[
pytest.param("Use when the user opens ~/build", id="home_relative_path"),
pytest.param("Use when the user edits ~/deploy/config.yaml", id="home_path_dir"),
],
)
def test_description_relative_path_not_shadow_command(self, description: str) -> None:
"""#669: a home-relative path ("~/build") is not a slash command, so it
does not shadow the built-in it happens to contain."""
findings = sc_mod._analyze_triggers({"description": description}, "myskill")
assert not any(finding.rule_id == "TR2" for finding in findings)

@pytest.mark.parametrize(
"description",
[
pytest.param(
"Use when the user says /ask-matt or wants an ask answered by Matt.",
id="noun_ask_with_slash_skill",
),
pytest.param(
"Use when the user says /grill-me or has a question to ask.",
id="verb_ask_with_slash_skill",
),
pytest.param(
"Diagnose and fix build failures caused by stale overrides",
id="capability_prose_overrides",
),
pytest.param(
"The deploy command is not intercepted by this skill",
id="passive_negated",
),
],
)
def test_description_noun_matching_builtin_not_shadow_command(self, description: str) -> None:
"""#669: a slash command whose name merely starts with a built-in word
("/ask-matt") does not shadow that built-in, and the word "ask" in the
same clause is not an invocation of the built-in "ask" command."""
findings = sc_mod._analyze_triggers({"description": description}, "ask-matt")
assert not any(finding.rule_id == "TR2" for finding in findings)

@pytest.mark.parametrize(
("description", "command"),
[
pytest.param(
"Use when the user says /ask or wants an answer.", "ask", id="slash_builtin"
),
pytest.param("Overrides the git commit command", "commit", id="verb_object"),
pytest.param("Shadows /help for every request", "help", id="verb_slash_object"),
pytest.param(
"Intercepts and replaces the built-in deploy command",
"deploy",
id="verb_then_conjunction",
),
pytest.param(
"Overrides the default behavior of the git commit command",
"commit",
id="verb_distant_object",
),
pytest.param(
"Intercepts every invocation of the deploy command",
"deploy",
id="verb_invocation_of",
),
pytest.param(
"Invoked whenever the user types deploy", "deploy", id="verb_whenever_types"
),
pytest.param(
"The built-in deploy command is intercepted by this skill",
"deploy",
id="passive_command_noun",
),
pytest.param(
"Whenever the user types deploy it is intercepted by this skill",
"deploy",
id="passive_it",
),
pytest.param(
"The git commit command is overridden by this skill",
"commit",
id="passive_overridden",
),
pytest.param(
"The built-in deploy command is always intercepted by this skill",
"deploy",
id="passive_adverb",
),
pytest.param(
"The deploy command is being intercepted by this skill",
"deploy",
id="passive_is_being",
),
pytest.param(
"The deploy command will be intercepted by this skill",
"deploy",
id="passive_will_be",
),
pytest.param(
"Deploy commands are now shadowed by this skill",
"deploy",
id="passive_are_now",
),
pytest.param(
"The git commit command has been overridden by this skill",
"commit",
id="passive_has_been",
),
],
)
def test_description_shadow_command_object_still_tr2(
self, description: str, command: str
) -> None:
"""#669 risky pair: a built-in named by a slash token or governed by an
interception verb is still a shadow command."""
findings = sc_mod._analyze_triggers({"description": description}, "myskill")
tr2 = [finding for finding in findings if finding.rule_id == "TR2"]
assert len(tr2) == 1
assert f"built-in command '{command}'" in tr2[0].message

@pytest.mark.parametrize(
"description",
[
pytest.param(
"Use this skill whenever the user wants to do anything with PDF files.",
id="pdf_domain",
),
pytest.param(
"Use this skill whenever the user asks about anything related to Kubernetes.",
id="related_to_domain",
),
pytest.param(
"Use whenever the user asks any questions regarding Terraform state",
id="regarding_domain",
),
pytest.param(
"Use this skill when the user asks any questions about this codebase",
id="determiner_this_domain",
),
pytest.param(
"Use whenever the user asks anything about that service",
id="determiner_that_domain",
),
pytest.param(
"Use this skill whenever the user asks any questions about any AWS service",
id="quantifier_any_domain",
),
pytest.param(
"Use whenever the user asks anything about each Terraform module",
id="quantifier_each_domain",
),
pytest.param(
"Use this skill whenever the user wants to do anything with any PDF file",
id="quantifier_any_pdf",
),
pytest.param(
"Use this skill whenever the user asks any questions about all the services "
"in the cluster",
id="predeterminer_all_the",
),
pytest.param(
"Use whenever the user asks anything about all your Terraform modules",
id="predeterminer_all_your",
),
pytest.param(
"Use whenever the user asks anything about all these files",
id="predeterminer_all_these",
),
pytest.param(
"Use whenever the user asks anything about all of the services",
id="quantifier_of_the",
),
],
)
def test_description_domain_qualified_scope_not_tr3(self, description: str) -> None:
"""#669: a domain qualifier ("with PDF files") bounds "anything", so the
trigger is not universal activation."""
findings = sc_mod._analyze_triggers({"description": description}, "myskill")
assert not any(finding.rule_id == "TR3" for finding in findings)

@pytest.mark.parametrize(
"description",
[
pytest.param(
"Use this skill whenever the user wants to do anything with it",
id="pronoun_object",
),
pytest.param(
"Use this skill whenever the user sends any message in the chat",
id="broad_preposition",
),
pytest.param("Use this skill whenever the user asks about anything", id="no_qualifier"),
pytest.param(
"Use this skill whenever the user discusses anything with you",
id="pronoun_you",
),
pytest.param(
"Use this skill whenever the user discusses anything with me", id="pronoun_me"
),
pytest.param(
"Use this skill whenever the user asks anything related to any topic",
id="quantifier_any_topic",
),
pytest.param(
"Use whenever the user sends any messages with any content",
id="quantifier_any_content",
),
pytest.param(
"Use this skill whenever the user says anything with anyone",
id="indefinite_anyone",
),
pytest.param(
"Use this skill whenever the user asks anything related to something",
id="indefinite_something",
),
pytest.param(
"Use this skill whenever the user asks anything regarding everyone",
id="indefinite_everyone",
),
pytest.param(
"Use this skill whenever the user does anything with any of them",
id="quantifier_any_of_them",
),
pytest.param(
"Use this skill whenever the user does anything with each of them",
id="quantifier_each_of_them",
),
],
)
def test_description_unqualified_scope_still_tr3(self, description: str) -> None:
"""#669 risky pair: a pronoun object or a broad preposition does not
bound the scope, so the trigger is still universal activation."""
findings = sc_mod._analyze_triggers({"description": description}, "myskill")
assert any(finding.rule_id == "TR3" for finding in findings)


# ── Supply Chain Helpers ───────────────────────────────────────────────

Expand Down
Loading