Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion data/typos-oxendict-base.toml
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,6 @@ organisational = "organizational"

[patterns]
ignore = [
'`[^`\n]+`',
'(?s)```.*?```',
'\brust-analyzer\b',
]
Expand Down
6 changes: 6 additions & 0 deletions docs/developers-guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,12 @@ there only when it is valid across repositories. Product names, quoted upstream
terms and fixture-specific vocabulary belong in the consumer repository's
tracked `typos.local.toml` overlay.

Local pattern additions merge with the shared ignore list. A local
`[patterns] remove` list then withdraws exact shared entries, allowing a
consumer to narrow an overly broad authority pattern without forking the
generator. A pattern cannot appear in both the local `ignore` and `remove`
lists; removals that no longer exist upstream remain valid no-ops.

The executable `scripts/typos_rollout_cli.py` provides three commands.
`harvest` emits JSON Lines evidence for both plain-British `-ise` and Oxford
`-ize` forms found in Git-tracked UTF-8 text. `generate` conditionally
Expand Down
8 changes: 8 additions & 0 deletions docs/users-guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,14 @@ policy. `harvest` emits JSON Lines evidence for Oxford `-ize` and plain-British
shared base; product names, quoted upstream terms, and deliberate fixtures
belong in a consumer's `typos.local.toml`.

Inline code is checked by default so misspelled identifiers, flags, module
paths and file names remain visible. Add exact identifier patterns to the
local `[patterns] ignore` list when an upstream name is intentionally spelled
differently. A local `[patterns] remove` list can withdraw an exact shared
ignore pattern when a repository needs stricter checking; removing a pattern
that the shared base no longer contains is a harmless no-op. Configuration
generation rejects an identical pattern in both local lists.

Ignore expressions are validated before scanning. Malformed expressions,
backreferences, and nested or adjacent repetitions are rejected, including
Python's `{,n}` upper-bound form; separated bounded repetitions remain valid.
Expand Down
24 changes: 22 additions & 2 deletions scripts/typos_rollout.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ class Dictionary:
Punctuation-separated phrase corrections checked outside Typos.
ignore_patterns
Bounded regular expressions used to mask upstream text.
removed_patterns
Shared ignore patterns withdrawn by a local overlay.
excluded_files
Repository-relative components and globs omitted from spelling scans.
"""
Expand All @@ -81,6 +83,7 @@ class Dictionary:
corrections: tuple[tuple[str, str], ...] = ()
phrase_corrections: tuple[tuple[str, str], ...] = ()
ignore_patterns: tuple[str, ...] = ()
removed_patterns: tuple[str, ...] = ()
excluded_files: tuple[str, ...] = ()


Expand Down Expand Up @@ -146,6 +149,7 @@ def _dictionary_from_text(text: str, *, sparse: bool = False) -> Dictionary:
corrections=tuple(sorted(corrections.items())),
phrase_corrections=tuple(sorted(phrase_corrections.items())),
ignore_patterns=ignore_patterns,
removed_patterns=_string_list(patterns, "remove"),
excluded_files=_string_list(files, "exclude"),
)

Expand Down Expand Up @@ -193,6 +197,21 @@ def _merge_correction_items(
return tuple(sorted(merged.items()))


def _merge_ignore_patterns(base: Dictionary, local: Dictionary) -> tuple[str, ...]:
"""Merge ignore patterns, then apply explicit local withdrawals."""
removed = set(local.removed_patterns)
contradictory = removed & set(local.ignore_patterns)
if contradictory:
message = (
"local overlay both ignores and removes patterns: "
f"{', '.join(sorted(contradictory))}"
)
raise ValueError(message)
return tuple(
sorted((set(base.ignore_patterns) | set(local.ignore_patterns)) - removed)
)


def merge_dictionaries(base: Dictionary, local: Dictionary) -> Dictionary:
"""Merge a shared dictionary with a non-conflicting local overlay.

Expand Down Expand Up @@ -230,8 +249,9 @@ def merge_dictionaries(base: Dictionary, local: Dictionary) -> Dictionary:
local.phrase_corrections,
label="phrase correction",
),
ignore_patterns=tuple(
sorted(set(base.ignore_patterns) | set(local.ignore_patterns))
ignore_patterns=_merge_ignore_patterns(base, local),
removed_patterns=tuple(
sorted(set(base.removed_patterns) | set(local.removed_patterns))
),
excluded_files=tuple(
sorted(set(base.excluded_files) | set(local.excluded_files))
Expand Down
16 changes: 16 additions & 0 deletions tests/test_typos_rollout.py
Original file line number Diff line number Diff line change
Expand Up @@ -477,3 +477,19 @@ def test_shared_dictionary_protects_exact_rust_analyzer_name(
assert not matcher.search("analyzer"), "ordinary analyzer prose was ignored"
assert not matcher.search("analyser"), "ordinary analyser prose was ignored"
assert not matcher.search("rust analyzer"), "unhyphenated prose was ignored"


def test_shared_dictionary_checks_inline_code_but_ignores_fenced_code(
rollout: types.ModuleType,
) -> None:
"""The shared policy masks code blocks without masking inline identifiers."""
inline_pattern = r"`[^`\n]+`"
fenced_pattern = r"(?s)```.*?```"
dictionary = rollout.load_dictionary(SHARED_DICTIONARY_PATH)
generated = tomllib.loads(rollout.render_typos_config(dictionary))["default"]
generated_patterns = generated["extend-ignore-re"]

assert inline_pattern not in dictionary.ignore_patterns
assert inline_pattern not in generated_patterns
assert fenced_pattern in dictionary.ignore_patterns
assert fenced_pattern in generated_patterns
114 changes: 114 additions & 0 deletions tests/test_typos_rollout_policy_properties.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

from hypothesis import given
from hypothesis import strategies as st
import pytest


def _exact_repetition(count: int) -> str:
Expand Down Expand Up @@ -34,6 +35,31 @@ def _upper_bounded_repetition(count: int) -> str:
st.tuples(COUNTS, COUNTS).map(_bounded_repetition),
COUNTS.map(_upper_bounded_repetition),
)
PATTERN_NAMES = st.text(
alphabet="abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789",
min_size=1,
max_size=12,
)
VALID_PATTERN_MEMBERSHIPS = st.dictionaries(
PATTERN_NAMES,
st.sampled_from(
(
(True, False, False),
(False, True, False),
(False, False, True),
(True, True, False),
(True, False, True),
)
),
max_size=12,
)
PATTERN_LISTS = st.lists(PATTERN_NAMES, unique=True, max_size=12).map(tuple)
NON_EMPTY_PATTERN_LISTS = st.lists(
PATTERN_NAMES,
unique=True,
min_size=1,
max_size=12,
).map(tuple)


@given(repetition=REPETITIONS)
Expand Down Expand Up @@ -71,3 +97,91 @@ def test_separated_generated_repetitions_remain_safe(
compiled = rollout._compile_ignore_patterns((pattern,))

assert compiled[0].pattern == pattern, "safe separated repetition changed"


@given(memberships=VALID_PATTERN_MEMBERSHIPS)
def test_ignore_pattern_merge_obeys_set_difference_for_every_valid_membership(
rollout: types.ModuleType,
memberships: dict[str, tuple[bool, bool, bool]],
) -> None:
"""Every valid finite membership map follows the set-theoretic contract."""
base_patterns = tuple(
pattern for pattern, membership in memberships.items() if membership[0]
)
local_patterns = tuple(
pattern for pattern, membership in memberships.items() if membership[1]
)
removed_patterns = tuple(
pattern for pattern, membership in memberships.items() if membership[2]
)
expected = tuple(
sorted((set(base_patterns) | set(local_patterns)) - set(removed_patterns))
)

merged = rollout._merge_ignore_patterns(
rollout.Dictionary(ignore_patterns=base_patterns),
rollout.Dictionary(
ignore_patterns=local_patterns,
removed_patterns=removed_patterns,
),
)
reordered = rollout._merge_ignore_patterns(
rollout.Dictionary(ignore_patterns=tuple(reversed(base_patterns))),
rollout.Dictionary(
ignore_patterns=tuple(reversed(local_patterns)),
removed_patterns=tuple(reversed(removed_patterns)),
),
)

assert merged == expected
assert reordered == expected, "input ordering changed the merged policy"


@given(base=PATTERN_LISTS, local=PATTERN_LISTS, absent=PATTERN_LISTS)
def test_absent_removals_are_always_no_ops(
rollout: types.ModuleType,
base: tuple[str, ...],
local: tuple[str, ...],
absent: tuple[str, ...],
) -> None:
"""Removing arbitrary patterns outside both inputs cannot change their union."""
base_patterns = tuple(f"base:{pattern}" for pattern in base)
local_patterns = tuple(f"local:{pattern}" for pattern in local)
absent_patterns = tuple(f"absent:{pattern}" for pattern in absent)

merged = rollout._merge_ignore_patterns(
rollout.Dictionary(ignore_patterns=base_patterns),
rollout.Dictionary(
ignore_patterns=local_patterns,
removed_patterns=absent_patterns,
),
)

assert merged == tuple(sorted(set(base_patterns) | set(local_patterns)))


@given(
overlap=NON_EMPTY_PATTERN_LISTS,
base=PATTERN_LISTS,
local=PATTERN_LISTS,
removed=PATTERN_LISTS,
)
def test_every_local_add_remove_overlap_is_rejected(
rollout: types.ModuleType,
overlap: tuple[str, ...],
base: tuple[str, ...],
local: tuple[str, ...],
removed: tuple[str, ...],
) -> None:
"""Every non-empty exact overlap fails regardless of other inputs or order."""
local_patterns = (*local, *overlap)
removed_patterns = (*reversed(removed), *reversed(overlap))

with pytest.raises(ValueError, match="both ignores and removes patterns"):
rollout._merge_ignore_patterns(
rollout.Dictionary(ignore_patterns=tuple(reversed(base))),
rollout.Dictionary(
ignore_patterns=local_patterns,
removed_patterns=removed_patterns,
),
)
48 changes: 48 additions & 0 deletions tests/test_typos_rollout_semantics.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,54 @@ def test_sparse_overlay_validates_regex_and_accepts_separated_repetitions(
rollout.load_dictionary(overlay, local_overlay=True)


def test_local_overlay_withdraws_a_shared_ignore_pattern(
rollout: types.ModuleType,
tmp_path: Path,
) -> None:
"""A local removal narrows shared masking without losing other patterns."""
broad_inline_pattern = r"`[^`\n]+`"
overlay = tmp_path / "typos.local.toml"
overlay.write_text(
"schema = 1\n\n[patterns]\n"
f"remove = [{json.dumps(broad_inline_pattern)}]\n",
encoding="utf-8",
)
base = rollout.Dictionary(
ignore_patterns=(broad_inline_pattern, r"(?s)```.*?```"),
)
local = rollout.load_dictionary(overlay, local_overlay=True)

merged = rollout.merge_dictionaries(base, local)

assert merged.ignore_patterns == (r"(?s)```.*?```",)
assert merged.removed_patterns == (broad_inline_pattern,)


def test_local_overlay_may_remove_an_absent_shared_pattern(
rollout: types.ModuleType,
) -> None:
"""An upstream policy improvement does not break an existing withdrawal."""
local = rollout.Dictionary(removed_patterns=(r"`[^`\n]+`",))

merged = rollout.merge_dictionaries(rollout.Dictionary(), local)

assert merged.ignore_patterns == ()


def test_local_overlay_cannot_add_and_remove_the_same_pattern(
rollout: types.ModuleType,
) -> None:
"""Contradictory pattern instructions fail with a useful diagnostic."""
pattern = r"`formal_api_name`"
local = rollout.Dictionary(
ignore_patterns=(pattern,),
removed_patterns=(pattern,),
)

with pytest.raises(ValueError, match="both ignores and removes patterns"):
rollout.merge_dictionaries(rollout.Dictionary(), local)


def test_merge_accepts_existing_local_exceptions(
rollout: types.ModuleType,
) -> None:
Expand Down
6 changes: 5 additions & 1 deletion tests/test_typos_spelling_gate.py
Original file line number Diff line number Diff line change
Expand Up @@ -266,9 +266,11 @@ def test_generated_config_loads_in_pinned_typos(
),
)
sample = tmp_path / "sample.md"
misspelled_article = "t" + "eh"
misspelled_receive = "rec" + "ieve"
sample.write_text(
f"We {PLAIN_BRITISH_ORGANIZE} {AMERICAN_COLOUR} output but analyse "
"valid results.\n",
f"valid results. `{misspelled_article} {misspelled_receive}`\n",
encoding="utf-8",
)

Expand Down Expand Up @@ -303,3 +305,5 @@ def test_generated_config_loads_in_pinned_typos(
"British colour spelling was not enforced"
)
assert "analyse" not in corrections, "valid -yse spelling was rejected"
assert misspelled_article in corrections, "inline-code typo was not reported"
assert misspelled_receive in corrections, "inline-code typo was not reported"
4 changes: 4 additions & 0 deletions typos.local.toml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@ polymerising = "polymerizing"

[patterns]
ignore = [
'`EXPLAIN ANALYZE`',
'`colorScheme`',
'`colors`',
'`hand-written`',
'RUST_ANALYZER(?:_PRECHECK)?',
'\bMOLD\b',
'\bmold\b',
Expand Down
5 changes: 4 additions & 1 deletion typos.toml
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,10 @@ extend-ignore-re = [
"\\bcolor:",
"\\bmold\\b",
"\\brust-analyzer\\b",
"`[^`\\n]+`",
"`EXPLAIN ANALYZE`",
"`colorScheme`",
"`colors`",
"`hand-written`",
]

[default.extend-words]
Expand Down