From 8f4756a318304475265b8daf29302a9aa5d16611 Mon Sep 17 00:00:00 2001 From: leynos Date: Tue, 4 Aug 2026 12:13:57 +0200 Subject: [PATCH 1/3] Check inline code with the shared spelling policy (#90) Remove the blanket inline-code ignore pattern while preserving fenced code blocks. Allow local overlays to withdraw exact shared patterns without failing when an upstream removal makes the withdrawal a no-op. --- data/typos-oxendict-base.toml | 1 - docs/developers-guide.md | 6 ++++ docs/users-guide.md | 7 ++++ scripts/typos_rollout.py | 24 ++++++++++++-- tests/test_typos_rollout.py | 16 +++++++++ tests/test_typos_rollout_semantics.py | 48 +++++++++++++++++++++++++++ tests/test_typos_spelling_gate.py | 6 +++- typos.local.toml | 4 +++ typos.toml | 5 ++- 9 files changed, 112 insertions(+), 5 deletions(-) diff --git a/data/typos-oxendict-base.toml b/data/typos-oxendict-base.toml index c69c1e7..52e653c 100644 --- a/data/typos-oxendict-base.toml +++ b/data/typos-oxendict-base.toml @@ -177,7 +177,6 @@ organisational = "organizational" [patterns] ignore = [ - '`[^`\n]+`', '(?s)```.*?```', '\brust-analyzer\b', ] diff --git a/docs/developers-guide.md b/docs/developers-guide.md index 5fcdb35..037ba7e 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -292,6 +292,12 @@ there only when it is valid across repositories. Product names, quoted upstream terms and fixture-specific vocabulary belong in the consumer repository's tracked `typos.local.toml` overlay. +Local pattern additions merge with the shared ignore list. A local +`[patterns] remove` list then withdraws exact shared entries, allowing a +consumer to narrow an overly broad authority pattern without forking the +generator. A pattern cannot appear in both the local `ignore` and `remove` +lists; removals that no longer exist upstream remain valid no-ops. + The executable `scripts/typos_rollout_cli.py` provides three commands. `harvest` emits JSON Lines evidence for both plain-British `-ise` and Oxford `-ize` forms found in Git-tracked UTF-8 text. `generate` conditionally diff --git a/docs/users-guide.md b/docs/users-guide.md index 9997f7c..a880d66 100644 --- a/docs/users-guide.md +++ b/docs/users-guide.md @@ -88,6 +88,13 @@ policy. `harvest` emits JSON Lines evidence for Oxford `-ize` and plain-British shared base; product names, quoted upstream terms, and deliberate fixtures belong in a consumer's `typos.local.toml`. +Inline code is checked by default so misspelled identifiers, flags, module +paths and file names remain visible. Add exact identifier patterns to the +local `[patterns] ignore` list when an upstream name is intentionally spelled +differently. A local `[patterns] remove` list can withdraw an exact shared +ignore pattern when a repository needs stricter checking; removing a pattern +that the shared base no longer contains is a harmless no-op. + Ignore expressions are validated before scanning. Malformed expressions, backreferences, and nested or adjacent repetitions are rejected, including Python's `{,n}` upper-bound form; separated bounded repetitions remain valid. diff --git a/scripts/typos_rollout.py b/scripts/typos_rollout.py index 1829a50..baadc4f 100644 --- a/scripts/typos_rollout.py +++ b/scripts/typos_rollout.py @@ -72,6 +72,8 @@ class Dictionary: Punctuation-separated phrase corrections checked outside Typos. ignore_patterns Bounded regular expressions used to mask upstream text. + removed_patterns + Shared ignore patterns withdrawn by a local overlay. excluded_files Repository-relative components and globs omitted from spelling scans. """ @@ -81,6 +83,7 @@ class Dictionary: corrections: tuple[tuple[str, str], ...] = () phrase_corrections: tuple[tuple[str, str], ...] = () ignore_patterns: tuple[str, ...] = () + removed_patterns: tuple[str, ...] = () excluded_files: tuple[str, ...] = () @@ -146,6 +149,7 @@ def _dictionary_from_text(text: str, *, sparse: bool = False) -> Dictionary: corrections=tuple(sorted(corrections.items())), phrase_corrections=tuple(sorted(phrase_corrections.items())), ignore_patterns=ignore_patterns, + removed_patterns=_string_list(patterns, "remove"), excluded_files=_string_list(files, "exclude"), ) @@ -193,6 +197,21 @@ def _merge_correction_items( return tuple(sorted(merged.items())) +def _merge_ignore_patterns(base: Dictionary, local: Dictionary) -> tuple[str, ...]: + """Merge ignore patterns, then apply explicit local withdrawals.""" + removed = set(local.removed_patterns) + contradictory = removed & set(local.ignore_patterns) + if contradictory: + message = ( + "local overlay both ignores and removes patterns: " + f"{', '.join(sorted(contradictory))}" + ) + raise ValueError(message) + return tuple( + sorted((set(base.ignore_patterns) | set(local.ignore_patterns)) - removed) + ) + + def merge_dictionaries(base: Dictionary, local: Dictionary) -> Dictionary: """Merge a shared dictionary with a non-conflicting local overlay. @@ -230,8 +249,9 @@ def merge_dictionaries(base: Dictionary, local: Dictionary) -> Dictionary: local.phrase_corrections, label="phrase correction", ), - ignore_patterns=tuple( - sorted(set(base.ignore_patterns) | set(local.ignore_patterns)) + ignore_patterns=_merge_ignore_patterns(base, local), + removed_patterns=tuple( + sorted(set(base.removed_patterns) | set(local.removed_patterns)) ), excluded_files=tuple( sorted(set(base.excluded_files) | set(local.excluded_files)) diff --git a/tests/test_typos_rollout.py b/tests/test_typos_rollout.py index c22287a..72f2bbb 100644 --- a/tests/test_typos_rollout.py +++ b/tests/test_typos_rollout.py @@ -477,3 +477,19 @@ def test_shared_dictionary_protects_exact_rust_analyzer_name( assert not matcher.search("analyzer"), "ordinary analyzer prose was ignored" assert not matcher.search("analyser"), "ordinary analyser prose was ignored" assert not matcher.search("rust analyzer"), "unhyphenated prose was ignored" + + +def test_shared_dictionary_checks_inline_code_but_ignores_fenced_code( + rollout: types.ModuleType, +) -> None: + """The shared policy masks code blocks without masking inline identifiers.""" + inline_pattern = r"`[^`\n]+`" + fenced_pattern = r"(?s)```.*?```" + dictionary = rollout.load_dictionary(SHARED_DICTIONARY_PATH) + generated = tomllib.loads(rollout.render_typos_config(dictionary))["default"] + generated_patterns = generated["extend-ignore-re"] + + assert inline_pattern not in dictionary.ignore_patterns + assert inline_pattern not in generated_patterns + assert fenced_pattern in dictionary.ignore_patterns + assert fenced_pattern in generated_patterns diff --git a/tests/test_typos_rollout_semantics.py b/tests/test_typos_rollout_semantics.py index e3534de..0927367 100644 --- a/tests/test_typos_rollout_semantics.py +++ b/tests/test_typos_rollout_semantics.py @@ -165,6 +165,54 @@ def test_sparse_overlay_validates_regex_and_accepts_separated_repetitions( rollout.load_dictionary(overlay, local_overlay=True) +def test_local_overlay_withdraws_a_shared_ignore_pattern( + rollout: types.ModuleType, + tmp_path: Path, +) -> None: + """A local removal narrows shared masking without losing other patterns.""" + broad_inline_pattern = r"`[^`\n]+`" + overlay = tmp_path / "typos.local.toml" + overlay.write_text( + "schema = 1\n\n[patterns]\n" + f"remove = [{json.dumps(broad_inline_pattern)}]\n", + encoding="utf-8", + ) + base = rollout.Dictionary( + ignore_patterns=(broad_inline_pattern, r"(?s)```.*?```"), + ) + local = rollout.load_dictionary(overlay, local_overlay=True) + + merged = rollout.merge_dictionaries(base, local) + + assert merged.ignore_patterns == (r"(?s)```.*?```",) + assert merged.removed_patterns == (broad_inline_pattern,) + + +def test_local_overlay_may_remove_an_absent_shared_pattern( + rollout: types.ModuleType, +) -> None: + """An upstream policy improvement does not break an existing withdrawal.""" + local = rollout.Dictionary(removed_patterns=(r"`[^`\n]+`",)) + + merged = rollout.merge_dictionaries(rollout.Dictionary(), local) + + assert merged.ignore_patterns == () + + +def test_local_overlay_cannot_add_and_remove_the_same_pattern( + rollout: types.ModuleType, +) -> None: + """Contradictory pattern instructions fail with a useful diagnostic.""" + pattern = r"`formal_api_name`" + local = rollout.Dictionary( + ignore_patterns=(pattern,), + removed_patterns=(pattern,), + ) + + with pytest.raises(ValueError, match="both ignores and removes patterns"): + rollout.merge_dictionaries(rollout.Dictionary(), local) + + def test_merge_accepts_existing_local_exceptions( rollout: types.ModuleType, ) -> None: diff --git a/tests/test_typos_spelling_gate.py b/tests/test_typos_spelling_gate.py index 8f6044f..a2fbb4b 100644 --- a/tests/test_typos_spelling_gate.py +++ b/tests/test_typos_spelling_gate.py @@ -266,9 +266,11 @@ def test_generated_config_loads_in_pinned_typos( ), ) sample = tmp_path / "sample.md" + misspelled_article = "t" + "eh" + misspelled_receive = "rec" + "ieve" sample.write_text( f"We {PLAIN_BRITISH_ORGANIZE} {AMERICAN_COLOUR} output but analyse " - "valid results.\n", + f"valid results. `{misspelled_article} {misspelled_receive}`\n", encoding="utf-8", ) @@ -303,3 +305,5 @@ def test_generated_config_loads_in_pinned_typos( "British colour spelling was not enforced" ) assert "analyse" not in corrections, "valid -yse spelling was rejected" + assert misspelled_article in corrections, "inline-code typo was not reported" + assert misspelled_receive in corrections, "inline-code typo was not reported" diff --git a/typos.local.toml b/typos.local.toml index 2c3a836..d6f50e6 100644 --- a/typos.local.toml +++ b/typos.local.toml @@ -28,6 +28,10 @@ polymerising = "polymerizing" [patterns] ignore = [ + '`EXPLAIN ANALYZE`', + '`colorScheme`', + '`colors`', + '`hand-written`', 'RUST_ANALYZER(?:_PRECHECK)?', '\bMOLD\b', '\bmold\b', diff --git a/typos.toml b/typos.toml index 28edcb7..e4b569d 100644 --- a/typos.toml +++ b/typos.toml @@ -37,7 +37,10 @@ extend-ignore-re = [ "\\bcolor:", "\\bmold\\b", "\\brust-analyzer\\b", - "`[^`\\n]+`", + "`EXPLAIN ANALYZE`", + "`colorScheme`", + "`colors`", + "`hand-written`", ] [default.extend-words] From 6df3233e0e7b05822e03a7d8da53c71be1064a54 Mon Sep 17 00:00:00 2001 From: leynos Date: Tue, 4 Aug 2026 21:09:40 +0200 Subject: [PATCH 2/3] Document conflicting pattern overlays (#90) State that configuration generation rejects an identical pattern in local `ignore` and `remove` lists. --- docs/users-guide.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/users-guide.md b/docs/users-guide.md index a880d66..5b29ce5 100644 --- a/docs/users-guide.md +++ b/docs/users-guide.md @@ -93,7 +93,8 @@ paths and file names remain visible. Add exact identifier patterns to the local `[patterns] ignore` list when an upstream name is intentionally spelled differently. A local `[patterns] remove` list can withdraw an exact shared ignore pattern when a repository needs stricter checking; removing a pattern -that the shared base no longer contains is a harmless no-op. +that the shared base no longer contains is a harmless no-op. Configuration +generation rejects an identical pattern in both local lists. Ignore expressions are validated before scanning. Malformed expressions, backreferences, and nested or adjacent repetitions are rejected, including From 679ab87aa8936603df14542dc73a439758016742 Mon Sep 17 00:00:00 2001 From: leynos Date: Tue, 4 Aug 2026 21:18:09 +0200 Subject: [PATCH 3/3] Prove ignore-pattern merge invariants (#90) Exercise arbitrary finite pattern memberships and orderings against the set-difference oracle. Pin deterministic output, harmless absent removals, and rejection of every generated local add-remove overlap. --- tests/test_typos_rollout_policy_properties.py | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/tests/test_typos_rollout_policy_properties.py b/tests/test_typos_rollout_policy_properties.py index 5c5c182..8757f7a 100644 --- a/tests/test_typos_rollout_policy_properties.py +++ b/tests/test_typos_rollout_policy_properties.py @@ -4,6 +4,7 @@ from hypothesis import given from hypothesis import strategies as st +import pytest def _exact_repetition(count: int) -> str: @@ -34,6 +35,31 @@ def _upper_bounded_repetition(count: int) -> str: st.tuples(COUNTS, COUNTS).map(_bounded_repetition), COUNTS.map(_upper_bounded_repetition), ) +PATTERN_NAMES = st.text( + alphabet="abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", + min_size=1, + max_size=12, +) +VALID_PATTERN_MEMBERSHIPS = st.dictionaries( + PATTERN_NAMES, + st.sampled_from( + ( + (True, False, False), + (False, True, False), + (False, False, True), + (True, True, False), + (True, False, True), + ) + ), + max_size=12, +) +PATTERN_LISTS = st.lists(PATTERN_NAMES, unique=True, max_size=12).map(tuple) +NON_EMPTY_PATTERN_LISTS = st.lists( + PATTERN_NAMES, + unique=True, + min_size=1, + max_size=12, +).map(tuple) @given(repetition=REPETITIONS) @@ -71,3 +97,91 @@ def test_separated_generated_repetitions_remain_safe( compiled = rollout._compile_ignore_patterns((pattern,)) assert compiled[0].pattern == pattern, "safe separated repetition changed" + + +@given(memberships=VALID_PATTERN_MEMBERSHIPS) +def test_ignore_pattern_merge_obeys_set_difference_for_every_valid_membership( + rollout: types.ModuleType, + memberships: dict[str, tuple[bool, bool, bool]], +) -> None: + """Every valid finite membership map follows the set-theoretic contract.""" + base_patterns = tuple( + pattern for pattern, membership in memberships.items() if membership[0] + ) + local_patterns = tuple( + pattern for pattern, membership in memberships.items() if membership[1] + ) + removed_patterns = tuple( + pattern for pattern, membership in memberships.items() if membership[2] + ) + expected = tuple( + sorted((set(base_patterns) | set(local_patterns)) - set(removed_patterns)) + ) + + merged = rollout._merge_ignore_patterns( + rollout.Dictionary(ignore_patterns=base_patterns), + rollout.Dictionary( + ignore_patterns=local_patterns, + removed_patterns=removed_patterns, + ), + ) + reordered = rollout._merge_ignore_patterns( + rollout.Dictionary(ignore_patterns=tuple(reversed(base_patterns))), + rollout.Dictionary( + ignore_patterns=tuple(reversed(local_patterns)), + removed_patterns=tuple(reversed(removed_patterns)), + ), + ) + + assert merged == expected + assert reordered == expected, "input ordering changed the merged policy" + + +@given(base=PATTERN_LISTS, local=PATTERN_LISTS, absent=PATTERN_LISTS) +def test_absent_removals_are_always_no_ops( + rollout: types.ModuleType, + base: tuple[str, ...], + local: tuple[str, ...], + absent: tuple[str, ...], +) -> None: + """Removing arbitrary patterns outside both inputs cannot change their union.""" + base_patterns = tuple(f"base:{pattern}" for pattern in base) + local_patterns = tuple(f"local:{pattern}" for pattern in local) + absent_patterns = tuple(f"absent:{pattern}" for pattern in absent) + + merged = rollout._merge_ignore_patterns( + rollout.Dictionary(ignore_patterns=base_patterns), + rollout.Dictionary( + ignore_patterns=local_patterns, + removed_patterns=absent_patterns, + ), + ) + + assert merged == tuple(sorted(set(base_patterns) | set(local_patterns))) + + +@given( + overlap=NON_EMPTY_PATTERN_LISTS, + base=PATTERN_LISTS, + local=PATTERN_LISTS, + removed=PATTERN_LISTS, +) +def test_every_local_add_remove_overlap_is_rejected( + rollout: types.ModuleType, + overlap: tuple[str, ...], + base: tuple[str, ...], + local: tuple[str, ...], + removed: tuple[str, ...], +) -> None: + """Every non-empty exact overlap fails regardless of other inputs or order.""" + local_patterns = (*local, *overlap) + removed_patterns = (*reversed(removed), *reversed(overlap)) + + with pytest.raises(ValueError, match="both ignores and removes patterns"): + rollout._merge_ignore_patterns( + rollout.Dictionary(ignore_patterns=tuple(reversed(base))), + rollout.Dictionary( + ignore_patterns=local_patterns, + removed_patterns=removed_patterns, + ), + )