Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 35 additions & 2 deletions scripts/format_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -137,18 +137,51 @@ def deterministic_precheck(
}


def qualitative_review_instructions(kind: str, title_type: str | None) -> str:
if kind == "pr":
body_instruction = (
"- PR body completeness: expect a summary, linked issue or context, "
"validation evidence, and rollout or risk notes when relevant. Do not "
"require issue-template headings."
)
else:
body_instruction = (
"- Issue body completeness: expect explicit Context, Scope, Out of scope, "
"Acceptance criteria, and Validation sections."
)

if kind == "issue" and title_type == "epic":
tracking_instruction = (
"- EPIC child tracking: warn when the body mirrors child issue references "
"as Markdown tracking entries such as `- [ ] #N`; prefer GitHub native "
"sub-issues. Acceptance-criteria checklists that do not track child issues "
"are allowed."
)
else:
tracking_instruction = (
"- Sub-issue tracking is not applicable to this artifact. Do not request "
"GitHub native sub-issues or a Markdown child-tracking checklist. "
"Acceptance-criteria checklists are allowed."
)

return f"{body_instruction}\n{tracking_instruction}"


def build_prompts(governance: dict[str, object], artifact: dict, kind: str, repo: str) -> tuple[str, str]:
title_type_map = governance["title_type_map"]
if not isinstance(title_type_map, dict):
raise ValueError("governance title_type_map is not an object")
precheck = deterministic_precheck(artifact, kind, title_type_map)
qualitative_instructions = qualitative_review_instructions(
kind,
precheck["title_type"] if isinstance(precheck["title_type"], str) else None,
)
system = (
"You are Crystal Governance Validator, a qualitative reviewer for GitHub artifacts in the Crystal team. "
"You receive (a) the canonical conventions, (b) the artifact, and (c) a deterministic mechanical pre-check that has ALREADY validated title format, label presence, type matching, subject length, and kernel-label provenance. "
"TRUST THE PRE-CHECK as ground truth: do not re-validate items the pre-check has marked valid. Do not contradict the pre-check.\n\n"
"Your job is to add QUALITATIVE findings that the deterministic check cannot catch:\n"
"- Body completeness: issues should have Context, Acceptance criteria, Out of scope sections.\n"
"- Markdown checklist for sub-issues (`- [ ] #N`) — warn, prefer GitHub native sub-issues.\n"
f"{qualitative_instructions}\n"
"- Subject style: imperative present mood, ambiguous wording, jargon mismatch with body.\n"
"- Body inconsistencies (acceptance criteria vague, no measurable conditions, contradictions).\n"
"- Wrong scope choice when a more specific scope exists.\n\n"
Expand Down
21 changes: 20 additions & 1 deletion tests/test_governance.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,10 @@
CONVENTIONAL_TITLE_RE,
load_title_type_map,
)
from format_check import deterministic_precheck # noqa: E402
from format_check import ( # noqa: E402
deterministic_precheck,
qualitative_review_instructions,
)
from validate_labels import validate_label_names # noqa: E402


Expand Down Expand Up @@ -196,6 +199,22 @@ def test_kernel_label_provenance_is_deterministic(self) -> None:
self.assertEqual(human["manual_kernel_labels"], ["crystal:stage:implementation"])
self.assertEqual(bot["manual_kernel_labels"], [])

def test_qualitative_review_is_scoped_by_artifact_kind(self) -> None:
pr_instructions = qualitative_review_instructions("pr", "fix")
self.assertIn("Do not require issue-template headings", pr_instructions)
self.assertIn("Sub-issue tracking is not applicable", pr_instructions)
self.assertIn("Acceptance-criteria checklists are allowed", pr_instructions)

issue_instructions = qualitative_review_instructions("issue", "bug")
self.assertIn("explicit Context, Scope, Out of scope", issue_instructions)
self.assertIn("Sub-issue tracking is not applicable", issue_instructions)

epic_instructions = qualitative_review_instructions("issue", "epic")
self.assertIn("EPIC child tracking", epic_instructions)
self.assertIn("`- [ ] #N`", epic_instructions)
self.assertIn("prefer GitHub native sub-issues", epic_instructions)
self.assertNotIn("Sub-issue tracking is not applicable", epic_instructions)


class WorkflowRuntimeTests(unittest.TestCase):
def test_workflows_use_node_24_action_majors(self) -> None:
Expand Down