diff --git a/data/typos-oxendict-base.toml b/data/typos-oxendict-base.toml index c69c1e7..52e653c 100644 --- a/data/typos-oxendict-base.toml +++ b/data/typos-oxendict-base.toml @@ -177,7 +177,6 @@ organisational = "organizational" [patterns] ignore = [ - '`[^`\n]+`', '(?s)```.*?```', '\brust-analyzer\b', ] diff --git a/docs/developers-guide.md b/docs/developers-guide.md index 5fcdb35..037ba7e 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -292,6 +292,12 @@ there only when it is valid across repositories. Product names, quoted upstream terms and fixture-specific vocabulary belong in the consumer repository's tracked `typos.local.toml` overlay. +Local pattern additions merge with the shared ignore list. A local +`[patterns] remove` list then withdraws exact shared entries, allowing a +consumer to narrow an overly broad authority pattern without forking the +generator. A pattern cannot appear in both the local `ignore` and `remove` +lists; removals that no longer exist upstream remain valid no-ops. + The executable `scripts/typos_rollout_cli.py` provides three commands. `harvest` emits JSON Lines evidence for both plain-British `-ise` and Oxford `-ize` forms found in Git-tracked UTF-8 text. `generate` conditionally diff --git a/docs/users-guide.md b/docs/users-guide.md index 9997f7c..5b29ce5 100644 --- a/docs/users-guide.md +++ b/docs/users-guide.md @@ -88,6 +88,14 @@ policy. `harvest` emits JSON Lines evidence for Oxford `-ize` and plain-British shared base; product names, quoted upstream terms, and deliberate fixtures belong in a consumer's `typos.local.toml`. +Inline code is checked by default so misspelled identifiers, flags, module +paths and file names remain visible. Add exact identifier patterns to the +local `[patterns] ignore` list when an upstream name is intentionally spelled +differently. A local `[patterns] remove` list can withdraw an exact shared +ignore pattern when a repository needs stricter checking; removing a pattern +that the shared base no longer contains is a harmless no-op. Configuration +generation rejects an identical pattern in both local lists. + Ignore expressions are validated before scanning. Malformed expressions, backreferences, and nested or adjacent repetitions are rejected, including Python's `{,n}` upper-bound form; separated bounded repetitions remain valid. diff --git a/scripts/typos_rollout.py b/scripts/typos_rollout.py index 1829a50..baadc4f 100644 --- a/scripts/typos_rollout.py +++ b/scripts/typos_rollout.py @@ -72,6 +72,8 @@ class Dictionary: Punctuation-separated phrase corrections checked outside Typos. ignore_patterns Bounded regular expressions used to mask upstream text. + removed_patterns + Shared ignore patterns withdrawn by a local overlay. excluded_files Repository-relative components and globs omitted from spelling scans. """ @@ -81,6 +83,7 @@ class Dictionary: corrections: tuple[tuple[str, str], ...] = () phrase_corrections: tuple[tuple[str, str], ...] = () ignore_patterns: tuple[str, ...] = () + removed_patterns: tuple[str, ...] = () excluded_files: tuple[str, ...] = () @@ -146,6 +149,7 @@ def _dictionary_from_text(text: str, *, sparse: bool = False) -> Dictionary: corrections=tuple(sorted(corrections.items())), phrase_corrections=tuple(sorted(phrase_corrections.items())), ignore_patterns=ignore_patterns, + removed_patterns=_string_list(patterns, "remove"), excluded_files=_string_list(files, "exclude"), ) @@ -193,6 +197,21 @@ def _merge_correction_items( return tuple(sorted(merged.items())) +def _merge_ignore_patterns(base: Dictionary, local: Dictionary) -> tuple[str, ...]: + """Merge ignore patterns, then apply explicit local withdrawals.""" + removed = set(local.removed_patterns) + contradictory = removed & set(local.ignore_patterns) + if contradictory: + message = ( + "local overlay both ignores and removes patterns: " + f"{', '.join(sorted(contradictory))}" + ) + raise ValueError(message) + return tuple( + sorted((set(base.ignore_patterns) | set(local.ignore_patterns)) - removed) + ) + + def merge_dictionaries(base: Dictionary, local: Dictionary) -> Dictionary: """Merge a shared dictionary with a non-conflicting local overlay. @@ -230,8 +249,9 @@ def merge_dictionaries(base: Dictionary, local: Dictionary) -> Dictionary: local.phrase_corrections, label="phrase correction", ), - ignore_patterns=tuple( - sorted(set(base.ignore_patterns) | set(local.ignore_patterns)) + ignore_patterns=_merge_ignore_patterns(base, local), + removed_patterns=tuple( + sorted(set(base.removed_patterns) | set(local.removed_patterns)) ), excluded_files=tuple( sorted(set(base.excluded_files) | set(local.excluded_files)) diff --git a/tests/test_typos_rollout.py b/tests/test_typos_rollout.py index c22287a..72f2bbb 100644 --- a/tests/test_typos_rollout.py +++ b/tests/test_typos_rollout.py @@ -477,3 +477,19 @@ def test_shared_dictionary_protects_exact_rust_analyzer_name( assert not matcher.search("analyzer"), "ordinary analyzer prose was ignored" assert not matcher.search("analyser"), "ordinary analyser prose was ignored" assert not matcher.search("rust analyzer"), "unhyphenated prose was ignored" + + +def test_shared_dictionary_checks_inline_code_but_ignores_fenced_code( + rollout: types.ModuleType, +) -> None: + """The shared policy masks code blocks without masking inline identifiers.""" + inline_pattern = r"`[^`\n]+`" + fenced_pattern = r"(?s)```.*?```" + dictionary = rollout.load_dictionary(SHARED_DICTIONARY_PATH) + generated = tomllib.loads(rollout.render_typos_config(dictionary))["default"] + generated_patterns = generated["extend-ignore-re"] + + assert inline_pattern not in dictionary.ignore_patterns + assert inline_pattern not in generated_patterns + assert fenced_pattern in dictionary.ignore_patterns + assert fenced_pattern in generated_patterns diff --git a/tests/test_typos_rollout_policy_properties.py b/tests/test_typos_rollout_policy_properties.py index 5c5c182..8757f7a 100644 --- a/tests/test_typos_rollout_policy_properties.py +++ b/tests/test_typos_rollout_policy_properties.py @@ -4,6 +4,7 @@ from hypothesis import given from hypothesis import strategies as st +import pytest def _exact_repetition(count: int) -> str: @@ -34,6 +35,31 @@ def _upper_bounded_repetition(count: int) -> str: st.tuples(COUNTS, COUNTS).map(_bounded_repetition), COUNTS.map(_upper_bounded_repetition), ) +PATTERN_NAMES = st.text( + alphabet="abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", + min_size=1, + max_size=12, +) +VALID_PATTERN_MEMBERSHIPS = st.dictionaries( + PATTERN_NAMES, + st.sampled_from( + ( + (True, False, False), + (False, True, False), + (False, False, True), + (True, True, False), + (True, False, True), + ) + ), + max_size=12, +) +PATTERN_LISTS = st.lists(PATTERN_NAMES, unique=True, max_size=12).map(tuple) +NON_EMPTY_PATTERN_LISTS = st.lists( + PATTERN_NAMES, + unique=True, + min_size=1, + max_size=12, +).map(tuple) @given(repetition=REPETITIONS) @@ -71,3 +97,91 @@ def test_separated_generated_repetitions_remain_safe( compiled = rollout._compile_ignore_patterns((pattern,)) assert compiled[0].pattern == pattern, "safe separated repetition changed" + + +@given(memberships=VALID_PATTERN_MEMBERSHIPS) +def test_ignore_pattern_merge_obeys_set_difference_for_every_valid_membership( + rollout: types.ModuleType, + memberships: dict[str, tuple[bool, bool, bool]], +) -> None: + """Every valid finite membership map follows the set-theoretic contract.""" + base_patterns = tuple( + pattern for pattern, membership in memberships.items() if membership[0] + ) + local_patterns = tuple( + pattern for pattern, membership in memberships.items() if membership[1] + ) + removed_patterns = tuple( + pattern for pattern, membership in memberships.items() if membership[2] + ) + expected = tuple( + sorted((set(base_patterns) | set(local_patterns)) - set(removed_patterns)) + ) + + merged = rollout._merge_ignore_patterns( + rollout.Dictionary(ignore_patterns=base_patterns), + rollout.Dictionary( + ignore_patterns=local_patterns, + removed_patterns=removed_patterns, + ), + ) + reordered = rollout._merge_ignore_patterns( + rollout.Dictionary(ignore_patterns=tuple(reversed(base_patterns))), + rollout.Dictionary( + ignore_patterns=tuple(reversed(local_patterns)), + removed_patterns=tuple(reversed(removed_patterns)), + ), + ) + + assert merged == expected + assert reordered == expected, "input ordering changed the merged policy" + + +@given(base=PATTERN_LISTS, local=PATTERN_LISTS, absent=PATTERN_LISTS) +def test_absent_removals_are_always_no_ops( + rollout: types.ModuleType, + base: tuple[str, ...], + local: tuple[str, ...], + absent: tuple[str, ...], +) -> None: + """Removing arbitrary patterns outside both inputs cannot change their union.""" + base_patterns = tuple(f"base:{pattern}" for pattern in base) + local_patterns = tuple(f"local:{pattern}" for pattern in local) + absent_patterns = tuple(f"absent:{pattern}" for pattern in absent) + + merged = rollout._merge_ignore_patterns( + rollout.Dictionary(ignore_patterns=base_patterns), + rollout.Dictionary( + ignore_patterns=local_patterns, + removed_patterns=absent_patterns, + ), + ) + + assert merged == tuple(sorted(set(base_patterns) | set(local_patterns))) + + +@given( + overlap=NON_EMPTY_PATTERN_LISTS, + base=PATTERN_LISTS, + local=PATTERN_LISTS, + removed=PATTERN_LISTS, +) +def test_every_local_add_remove_overlap_is_rejected( + rollout: types.ModuleType, + overlap: tuple[str, ...], + base: tuple[str, ...], + local: tuple[str, ...], + removed: tuple[str, ...], +) -> None: + """Every non-empty exact overlap fails regardless of other inputs or order.""" + local_patterns = (*local, *overlap) + removed_patterns = (*reversed(removed), *reversed(overlap)) + + with pytest.raises(ValueError, match="both ignores and removes patterns"): + rollout._merge_ignore_patterns( + rollout.Dictionary(ignore_patterns=tuple(reversed(base))), + rollout.Dictionary( + ignore_patterns=local_patterns, + removed_patterns=removed_patterns, + ), + ) diff --git a/tests/test_typos_rollout_semantics.py b/tests/test_typos_rollout_semantics.py index e3534de..0927367 100644 --- a/tests/test_typos_rollout_semantics.py +++ b/tests/test_typos_rollout_semantics.py @@ -165,6 +165,54 @@ def test_sparse_overlay_validates_regex_and_accepts_separated_repetitions( rollout.load_dictionary(overlay, local_overlay=True) +def test_local_overlay_withdraws_a_shared_ignore_pattern( + rollout: types.ModuleType, + tmp_path: Path, +) -> None: + """A local removal narrows shared masking without losing other patterns.""" + broad_inline_pattern = r"`[^`\n]+`" + overlay = tmp_path / "typos.local.toml" + overlay.write_text( + "schema = 1\n\n[patterns]\n" + f"remove = [{json.dumps(broad_inline_pattern)}]\n", + encoding="utf-8", + ) + base = rollout.Dictionary( + ignore_patterns=(broad_inline_pattern, r"(?s)```.*?```"), + ) + local = rollout.load_dictionary(overlay, local_overlay=True) + + merged = rollout.merge_dictionaries(base, local) + + assert merged.ignore_patterns == (r"(?s)```.*?```",) + assert merged.removed_patterns == (broad_inline_pattern,) + + +def test_local_overlay_may_remove_an_absent_shared_pattern( + rollout: types.ModuleType, +) -> None: + """An upstream policy improvement does not break an existing withdrawal.""" + local = rollout.Dictionary(removed_patterns=(r"`[^`\n]+`",)) + + merged = rollout.merge_dictionaries(rollout.Dictionary(), local) + + assert merged.ignore_patterns == () + + +def test_local_overlay_cannot_add_and_remove_the_same_pattern( + rollout: types.ModuleType, +) -> None: + """Contradictory pattern instructions fail with a useful diagnostic.""" + pattern = r"`formal_api_name`" + local = rollout.Dictionary( + ignore_patterns=(pattern,), + removed_patterns=(pattern,), + ) + + with pytest.raises(ValueError, match="both ignores and removes patterns"): + rollout.merge_dictionaries(rollout.Dictionary(), local) + + def test_merge_accepts_existing_local_exceptions( rollout: types.ModuleType, ) -> None: diff --git a/tests/test_typos_spelling_gate.py b/tests/test_typos_spelling_gate.py index 8f6044f..a2fbb4b 100644 --- a/tests/test_typos_spelling_gate.py +++ b/tests/test_typos_spelling_gate.py @@ -266,9 +266,11 @@ def test_generated_config_loads_in_pinned_typos( ), ) sample = tmp_path / "sample.md" + misspelled_article = "t" + "eh" + misspelled_receive = "rec" + "ieve" sample.write_text( f"We {PLAIN_BRITISH_ORGANIZE} {AMERICAN_COLOUR} output but analyse " - "valid results.\n", + f"valid results. `{misspelled_article} {misspelled_receive}`\n", encoding="utf-8", ) @@ -303,3 +305,5 @@ def test_generated_config_loads_in_pinned_typos( "British colour spelling was not enforced" ) assert "analyse" not in corrections, "valid -yse spelling was rejected" + assert misspelled_article in corrections, "inline-code typo was not reported" + assert misspelled_receive in corrections, "inline-code typo was not reported" diff --git a/typos.local.toml b/typos.local.toml index 2c3a836..d6f50e6 100644 --- a/typos.local.toml +++ b/typos.local.toml @@ -28,6 +28,10 @@ polymerising = "polymerizing" [patterns] ignore = [ + '`EXPLAIN ANALYZE`', + '`colorScheme`', + '`colors`', + '`hand-written`', 'RUST_ANALYZER(?:_PRECHECK)?', '\bMOLD\b', '\bmold\b', diff --git a/typos.toml b/typos.toml index 28edcb7..e4b569d 100644 --- a/typos.toml +++ b/typos.toml @@ -37,7 +37,10 @@ extend-ignore-re = [ "\\bcolor:", "\\bmold\\b", "\\brust-analyzer\\b", - "`[^`\\n]+`", + "`EXPLAIN ANALYZE`", + "`colorScheme`", + "`colors`", + "`hand-written`", ] [default.extend-words]