fix(evidence): make the gate's checks, triggers and exit codes mean something

domains/pr-workflow/skills/attest/skill.md

@@ -1,6 +1,7 @@
 ---
 name: attest
-description: The gate an evidence artifact passes before it is published to a pull request, issue or shared tracker. Two halves that do not substitute for each other — a mechanical pass that greps for the properties a reader needs (marker pair, pinned environment, a captured artifact rather than typed prose, a destination that is still open) and a dispatched pass sent to fresh instances that contest the framing, the coverage, and how it reads to a stranger. The author is the wrong checker: they remember running the check, and the memory supplies the provenance the text lacks. Verdicts are attested, attested with named caveats, blocked, or not a run — the last being common and legitimate, because a run that could not execute has produced nothing to publish. Triggers on mms-attest, or before posting any evidence, validation or diligence output to a public surface.
+description: >-
+  The gate an evidence artifact passes before it is published to a pull request, issue or shared tracker. Two halves that do not substitute for each other — a mechanical pass that greps for the properties a reader needs (marker pair, pinned environment, a captured artifact rather than typed prose, a destination that is still open) and a dispatched pass sent to fresh instances that contest the framing, the coverage, and how it reads to a stranger. The author is the wrong checker: they remember running the check, and the memory supplies the provenance the text lacks. Verdicts are attested, attested with named caveats, blocked, or not a run — the last being common and legitimate, because a run that could not execute has produced nothing to publish. Triggers on mms-attest, or before posting any evidence, validation or diligence output to a public surface.
 maturity: experimental
 ---
 

domains/pr-workflow/skills/evidence/assets/evidence-run.yml

@@ -224,7 +224,12 @@ jobs:
         # Contention produced numbers that were published and then retracted. Running the
         # head arm twice and diffing costs one repeat and turns that into a pre-publish
         # signal rather than a correction.
-        continue-on-error: true
+        #
+        # Deliberately NOT continue-on-error, unlike the two runner steps above. There the
+        # exit code is the verdict and a finding is not a failure. Here a difference is not
+        # a finding about the code — it says the instrument did not return the same answer
+        # twice, so neither answer can be published. With continue-on-error the step's own
+        # `exit 1` was swallowed and the run went green anyway.
         env:
           RUNNER: ${{ inputs.runner }}
           ARGS: ${{ inputs.args }}
@@ -246,8 +251,14 @@ jobs:
                     <(jq -S 'del(.env, .label, .log, .logs)' "$B") > determinism.diff; then
               echo "deterministic across two runs" | tee -a "$GITHUB_STEP_SUMMARY"
             else
-              echo "::warning::runner is NOT deterministic at this ref — do not publish these numbers"
+              # Hard fail, not a warning. The instruction this step emits is "do not
+              # publish these numbers", and a warning cannot enforce it — the run goes
+              # green, the artifact is produced, and the numbers publish anyway. A
+              # control whose only effect is advisory text is the failure this whole
+              # package is about, restated one level up.
+              echo "::error::runner is NOT deterministic at this ref — these numbers are not publishable"
               cat determinism.diff >> "$GITHUB_STEP_SUMMARY"
+              exit 1
             fi
           fi
 

domains/pr-workflow/skills/evidence/hooks/pr-evidence-gate.py

@@ -135,9 +135,10 @@ def main():
     "observation": "an OBSERVATION artifact (screenshot/recording/log/JSON/permalink) — "
                    "a /blob/ code link witnesses code, not runtime behavior",
     "deferral": "a co-located TRACKER (#issue, issues/pull URL, 'triage', 'tracked in')",
-    "ci-restatement": "removal — a validation surface carries zero CI references. "
-                      "The Checks tab already shows them; cite CI only as the revert "
-                      "lane's outcome, never as 'green at head'",
+    "ci-restatement": "removal of the CLAIM, not of the link — 'green at head' hands the "
+                      "reviewer their own Checks tab back. Citing a specific run and job "
+                      "whose log holds the figure you are reporting is evidence and is "
+                      "fine; asserting a status the Checks tab already shows is not",
     "inflated-verdict": "a downgraded verdict — 'live-proven' co-located with "
                         "'not exercised' is inflated; borrowed evidence never "
                         "upgrades an uncaptured lane",
@@ -235,10 +236,22 @@ def _extract_body(cmd):
     r"(?i)(?:#\d+|https?://\S*(?:issues|pull)/\d+|\btriage\b|follow-?up|tracked\s+in)"
 )
 # ── item 11: CI restatement — unconditional in validation scope ────────────
+# Restating a status is not the same as citing a measurement, and the rule is about the
+# first. "Tests are green at head <sha>" hands the reviewer their own Checks tab back and
+# carries no information. A link to a specific run and job whose log holds the figure
+# being reported carries the whole measurement, and is what evidence-run.yml exists to
+# produce — "move the measurement to CI, where the run URL is the capture".
+#
+# Matching a bare `actions/runs/N` conflated the two, so the package forbade its own
+# flagship output: five of the six branches below describe a CLAIM about CI, and one
+# described a URL. A run link is now a violation only when it carries restatement
+# language with it.
 CI_RESTATEMENT = re.compile(
-    r"(?i)(?:actions/runs/\d+|\bchecks?\s+tab\b|\bgreen\s+(?:at\s+head|in\s+)"
+    r"(?i)(?:\bchecks?\s+tab\b|\bgreen\s+(?:at\s+head|in\s+)"
     r"|\ball\s+(?:tests|checks|jobs)\s+(?:pass\w*|green)\b|\bCI\s+(?:is\s+)?green\b"
-    r"|\b\d+\s+pass(?:ing|ed)?\s*/\s*\d+\s+fail\w*)"
+    r"|\b\d+\s+pass(?:ing|ed)?\s*/\s*\d+\s+fail\w*"
+    r"|actions/runs/\d+[^.\n]{0,80}?\b(?:green|passing|all\s+checks|succeeded)\b"
+    r"|\b(?:green|passing|all\s+checks)\b[^.\n]{0,80}?actions/runs/\d+)"
 )
 # ── item 11: inflated verdict — proof language beside a non-exercise ───────
 NOT_EXERCISED = re.compile(
@@ -323,10 +336,14 @@ def _repo_pr_from_cmd(cmd):
 def _find_gate():
     here = os.path.dirname(os.path.abspath(__file__))
     for cand in (
+        # An explicit override that loses to a default is not an override. This
+        # ranked last, so a control run pointing ATTEST_GATE at a stand-in gate
+        # silently exercised the installed one instead and reported on it — the
+        # test looked like it passed and measured the wrong binary.
+        os.environ.get("ATTEST_GATE", ""),
         os.path.join(here, "..", "scripts", "attest-gate.sh"),
         os.path.join(here, "attest-gate.sh"),
         os.path.expanduser("~/.claude/skills/mms-evidence/scripts/attest-gate.sh"),
-        os.environ.get("ATTEST_GATE", ""),
     ):
         if cand and os.path.isfile(cand):
             return os.path.abspath(cand)
@@ -345,10 +362,27 @@ def _find_gate():
 )
 
 
+# Where the bold stops is not a fact about the claim. `**Verdict:** proven`
+# ran all thirteen checks and `**Verdict: proven**` ran none — the same
+# sentence, rendered the same way, one of them silently unenforced. Match the
+# bolded Verdict lead however the emphasis falls, while staying anchored to a
+# line-leading bold run so that mentioning the word in prose still does not
+# drag a normal comment into the gate.
+_VERDICT_LEAD = re.compile(r"^\s*\*\*\s*Verdict\b[^*\n]*\*\*", re.M | re.I)
+
+
+_FENCE = re.compile(r"^```.*?^```", re.M | re.S)
+
+
 def _is_evidence_artifact(body):
     if any(m in body for m in ARTIFACT_MARKERS):
         return True
-    return bool(re.search(r"^\*\*Verdict:\*\*", body, re.M))
+    # A verdict line inside a fenced block is an example of one, not one. Writing about
+    # this gate — a PR that quotes `**Verdict:** proven` to show what triggers it — was
+    # otherwise classified as a validation run and asked for the whole envelope. The
+    # trigger has to be able to tell a claim from a quotation of a claim, or documenting
+    # the rule becomes a violation of it.
+    return bool(_VERDICT_LEAD.search(_FENCE.sub("", body)))
 
 
 def _run_attest_gate(body, cmd):
@@ -473,6 +507,81 @@ def _add(violations, kind, token, unit):
     })
 
 
+# ── artifact credibility ───────────────────────────────────────────────────
+# Matching a URL shape only proves someone typed a URL, and the same generator
+# writes the claim and the string that satisfies the check — so presence alone
+# carries no information. An artifact counts only if the author could not have
+# authored its contents: a namespace where the bytes are written by CI, by the
+# upload endpoint, or by an observability backend; or a local path that is
+# actually on disk. Presence stays necessary and stops being sufficient.
+ARTIFACT_HOST_ALLOWLIST = (
+    "github.com/",                      # narrowed by ARTIFACT_PATH_ALLOWLIST below
+    "user-images.githubusercontent.com/",
+    "gist.github.com/",
+    "sentry.io/",
+    "grafana.net/",
+    "grafana.com/",
+)
+# github.com is author-writable in general (a branch, a wiki, a comment anchor),
+# so only the sub-namespaces whose bytes CI or the upload endpoint produce count.
+ARTIFACT_PATH_ALLOWLIST = (
+    "/actions/runs/",
+    "/user-attachments/",
+    "/blob/",
+    "/commit/",
+    "/pull/",
+)
+# Escape hatch for hosting the author does control — an artifact bucket, an
+# internal dashboard. Registering one is a deliberate, visible downgrade: the
+# artifact becomes fetchable rather than independent, and a reader who trusts
+# it is trusting the author. Comma-separated substrings.
+#   EVIDENCE_GATE_ARTIFACT_HOSTS=my-bucket.s3.amazonaws.com,dash.internal
+_EXTRA_HOSTS = tuple(
+    h.strip().lower()
+    for h in (os.environ.get("EVIDENCE_GATE_ARTIFACT_HOSTS") or "").split(",")
+    if h.strip()
+)
+_URL_RE = re.compile(r"https?://[^\s)>\]\"']+")
+_LOCAL_REF_RE = re.compile(
+    r"`?([\w./-]+\.(?:test|spec)\.[tj]sx?)(?::\d+)?`?"
+    r"|`?([\w./-]+\.(?:png|jpe?g|gif|mp4|webm|har|log|json))`?"
+)
+
+
+def _url_is_credible(url):
+    low = url.lower()
+    if _EXTRA_HOSTS and any(h in low for h in _EXTRA_HOSTS):
+        return True
+    if not any(h in low for h in ARTIFACT_HOST_ALLOWLIST):
+        return False
+    if "github.com/" in low and "githubusercontent" not in low and "gist." not in low:
+        return any(p in low for p in ARTIFACT_PATH_ALLOWLIST)
+    return True
+
+
+def _local_ref_exists(ref):
+    if os.path.isabs(ref):
+        return os.path.exists(ref)
+    for root in (os.getcwd(), os.environ.get("CLAUDE_PROJECT_DIR") or ""):
+        if root and os.path.exists(os.path.join(root, ref)):
+            return True
+    return False
+
+
+def _has_credible_artifact(unit, pattern):
+    """True only if this unit carries an artifact the author could not fabricate."""
+    if not pattern.search(unit):
+        return False
+    for url in _URL_RE.findall(unit):
+        if _url_is_credible(url):
+            return True
+    for m in _LOCAL_REF_RE.finditer(unit):
+        ref = m.group(1) or m.group(2)
+        if ref and _local_ref_exists(ref):
+            return True
+    return False
+
+
 def _positive_verdict(unit):
     """A non-negated verdict token in this unit, or None."""
     for m in VERDICT.finditer(unit):
@@ -483,14 +592,14 @@ def _positive_verdict(unit):
 
 def _scan_unit(unit, violations):
     # ── VERDICT: excused by a co-located inspectable artifact.
-    if not ARTIFACT.search(unit):
+    if not _has_credible_artifact(unit, ARTIFACT):
         tok = _positive_verdict(unit)
         if tok:
             _add(violations, "verdict", tok, unit)
 
     # ── OBSERVATION: needs an observation-class artifact. A /blob/ code
     #    permalink does NOT excuse it.
-    if not OBS_ARTIFACT.search(unit):
+    if not _has_credible_artifact(unit, OBS_ARTIFACT):
         for m in OBSERVATION.finditer(unit):
             if _negated(unit, m.start()):
                 continue
@@ -533,7 +642,7 @@ def _scan_unit(unit, violations):
 
     # ── BARE IDENTIFIER (item 12): an id with no resolving link and no
     #    re-hosted capture is a digging assignment.
-    if not RESOLVER.search(unit) and not OBS_ARTIFACT.search(unit):
+    if not RESOLVER.search(unit) and not _has_credible_artifact(unit, OBS_ARTIFACT):
         bm = BARE_ID.search(unit)
         if bm:
             _add(violations, "bare-identifier", bm.group(0), unit)

domains/pr-workflow/skills/evidence/hooks/session-audit.mjs

@@ -0,0 +1,74 @@
+#!/usr/bin/env node
+//
+// Stop hook: say, at the end of a session, whether anything published without a gate.
+//
+// The corpus's own diagnostic for a rule that keeps being violated is "count repeats within
+// a session — three fires means the rule is inert". Nothing was counting. The canonical file
+// for the largest failure cluster failed to maintain its own recurrence count, which is the
+// same class of failure it documents: a number that depends on someone remembering to
+// increment it is not a measurement. A script does not forget.
+//
+// It reports UNGATED publishes — an outward-facing write with no gate invocation anywhere
+// earlier in the session. That is the real signal and it is deterministic.
+//
+// It deliberately does NOT report "unchained" publishes, which skill-audit also emits. A
+// publish is unchained when the gate is not part of the same shell command. Since the gate
+// is wired as a PreToolUse hook it fires out-of-band on every write by construction, so it
+// is never in the command, so every publish is unchained. Counting those produces a large
+// number that measures the enforcement mechanism rather than any defect — which is exactly
+// how an audit comes to report thousands of violations of a rule that is being enforced.
+//
+// Contract: reads Stop-hook JSON on stdin, writes a note to stderr, always exits 0. It is a
+// report, not a gate; blocking the end of a session teaches nothing the note does not.
+import { spawnSync } from 'node:child_process';
+import { existsSync } from 'node:fs';
+
+const AUDIT_CANDIDATES = [
+  process.env.SKILL_AUDIT,
+  `${process.env.HOME}/Code/metamask/skills/tools/skill-audit.mjs`,
+  `${process.env.HOME}/.claude/skills/mms-evidence/hooks/skill-audit.mjs`,
+].filter(Boolean);
+
+let stdin = '';
+process.stdin.setEncoding('utf8');
+for await (const chunk of process.stdin) stdin += chunk;
+
+let transcript;
+try {
+  transcript = JSON.parse(stdin || '{}').transcript_path;
+} catch {
+  process.exit(0);
+}
+if (!transcript || !existsSync(transcript)) process.exit(0);
+
+const audit = AUDIT_CANDIDATES.find((p) => existsSync(p));
+if (!audit) process.exit(0);
+
+const proc = spawnSync(process.execPath, [audit, transcript, '--json'], {
+  encoding: 'utf8',
+  timeout: 30_000,
+});
+// skill-audit exits 1 when it FINDS something — the exit code is the verdict, not an
+// error. Treating non-zero as failure made this hook bail on precisely the sessions it
+// exists to report on, and report nothing on all the others, so it would have read as
+// "clean" forever. Parse whatever it printed and let the payload decide.
+if (!proc.stdout) process.exit(0);
+
+let report;
+try {
+  report = JSON.parse(proc.stdout);
+} catch {
+  process.exit(0);
+}
+
+const ungated = report.ungatedPublishLines ?? [];
+if (ungated.length === 0) process.exit(0);
+
+process.stderr.write(
+  `evidence audit: ${ungated.length} outward-facing publish(es) ran with no gate ` +
+    `anywhere earlier in this session.\n` +
+    `  transcript lines: ${ungated.slice(0, 12).join(', ')}` +
+    `${ungated.length > 12 ? `, … +${ungated.length - 12} more` : ''}\n` +
+    `  ${report.publishes} publish(es), ${report.gateRuns} gate run(s) total.\n`,
+);
+process.exit(0);

domains/pr-workflow/skills/evidence/scripts/ci-citation-controls.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""Does the CI rule separate restating a status from citing a measurement?
+
+RESTATEMENT asserts something the Checks tab already shows. It is the reviewer's own data
+read back to them, carries no information, and should be caught.
+
+CITATION points at a specific run and job whose log or artifact IS the capture — a figure
+the Checks tab does not show. `evidence-run.yml` exists to produce exactly this ("move the
+measurement to CI, where the run URL is the capture"), so catching it means the package
+forbids its own flagship output.
+
+The rule originally matched a bare `actions/runs/N`, which conflated the two: five of its
+six branches described a CLAIM about CI and one described a URL. This is the control that
+keeps them apart.
+
+It imports CI_RESTATEMENT from the hook rather than restating it, because a control that
+tests its own copy of a pattern passes forever while the real one drifts.
+"""
+import importlib.util
+import os
+import sys
+
+HOOK = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+                    "hooks", "pr-evidence-gate.py")
+spec = importlib.util.spec_from_file_location("_gate", HOOK)
+gate = importlib.util.module_from_spec(spec)
+try:
+    spec.loader.exec_module(gate)
+except SystemExit:
+    pass
+CI = gate.CI_RESTATEMENT
+
+RESTATEMENT = {
+    "green at head + run link": "Tests are green at head `7bfc16c` — https://github.com/o/r/actions/runs/123.",
+    "checks tab": "See the Checks tab; everything passes.",
+    "all jobs green": "All jobs green on this branch.",
+    "counts": "Unit tests: 412 passing / 0 failing.",
+    "CI is green": "CI is green, so the change is safe.",
+}
+
+CITATION = {
+    "run+job is the capture":
+        "The probe ran in CI: https://github.com/o/r/actions/runs/123/job/456 printed "
+        "`identical=3 unrelated=1 inputChanged=6`.",
+    "run link + artifact":
+        "Measured in CI — https://github.com/o/r/actions/runs/123 — artifact "
+        "`evidence-artifacts/recompute.json` attached there.",
+    "two-arm result from a run":
+        "Base arm failed and head arm passed in https://github.com/o/r/actions/runs/123/job/456; "
+        "both logs are on that job.",
+}
+
+ok = True
+print(f"  {'':<26}{'':<28}verdict")
+for kind, cases, want_caught in (("restatement (want CAUGHT)", RESTATEMENT, True),
+                                 ("citation (want ALLOWED)", CITATION, False)):
+    for label, text in cases.items():
+        caught = bool(CI.search(text))
+        good = caught == want_caught
+        ok &= good
+        print(f"  {'ok ' if good else 'FAIL'} {kind:<26}{label:<28}"
+              f"{'CAUGHT' if caught else 'allowed'}")
+
+print("\nall arms behave" if ok else "\nCONTROL FAILED")
+sys.exit(0 if ok else 1)

domains/pr-workflow/skills/evidence/scripts/principle-coverage-baseline.md

@@ -0,0 +1,13 @@
+<!-- VALIDATION_RUN_START -->
+## 🧪 Validation Run
+
+> Trial run of an experimental evidence skill — feedback via MetaMask/skills.
+
+**Verdict:** proven — **Claim:** the selector stops recomputing on unrelated writes.
+Measured 3 recomputations before and 1 after: ![capture](https://github.com/user-attachments/assets/1f2e3d4c-aaaa-bbbb-cccc-ddddeeeeffff) — `evidence-artifacts/recompute.json`
+
+Environment: head `7bfc16c`, node `v20.11.0`.
+
+Not covered by this run: one fixture, one perturbed key; a selector unmoved here can still
+recompute under state this fixture does not reach.
+<!-- VALIDATION_RUN_END -->