From 458dbf697bb0d854b43f65b09b20d256d5f57aef Mon Sep 17 00:00:00 2001 From: Andrew Jordan Date: Fri, 24 Jul 2026 13:36:24 -0500 Subject: [PATCH] Add UPPERCASE_CODE_NORMALIZE and EMAIL_TO_TITLE_NAME recipes (0.2.0) Two expression-first recipes plus the atoms they needed: - UPPERCASE_CODE_NORMALIZE: NFKC -> whitespace-canonicalize -> uppercase -> nullify the '?' placeholder. The canonical form for identifier codes (VIN); agrees byte-for-byte with SQL UPPER(TRIM(vin)) on clean input. - EMAIL_TO_TITLE_NAME: strict local-part extract -> [._-]+ to space -> titlecase. A title-cased display name from an email, modeled on the BigQuery report; the docstring documents the two cosmetic divergences from INITCAP (mcdonald and digit-adjacency) rather than claiming byte parity. New atoms: to_uppercase (case), separators_to_space (substitution), extract_email_local_part_strict (names), DEFAULT_PLACEHOLDER_SENTINELS (missing). Existing atoms reused where they fit. ruff + mypy + 945 tests green. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NGY3PA5SKv8f9mKWAo91xF --- pyproject.toml | 2 +- src/framesmith/__init__.py | 4 + src/framesmith/recipes.py | 55 ++++++ src/framesmith/transforms/__init__.py | 8 + src/framesmith/transforms/case.py | 13 ++ src/framesmith/transforms/missing.py | 11 ++ src/framesmith/transforms/names.py | 19 ++ src/framesmith/transforms/substitution.py | 26 ++- tests/test_recipes.py | 227 +++++++++++++++++++++- tests/transforms/test_case.py | 24 +++ tests/transforms/test_missing.py | 31 ++- tests/transforms/test_names.py | 65 +++++++ tests/transforms/test_substitution.py | 40 ++++ 13 files changed, 510 insertions(+), 15 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 39ef8a6..4baff8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta" ############################################################################### [project] name = "framesmith" -version = "0.1.0" +version = "0.2.0" description = "Composable, expression-first preprocessing for polars DataFrames: small atomic transforms, declarative recipes, and one function that composes them onto a column." readme = "README.md" authors = [{ name = "Andrew Jordan", email = "andrewjordan3@gmail.com" }] diff --git a/src/framesmith/__init__.py b/src/framesmith/__init__.py index e0556b0..7f3b089 100644 --- a/src/framesmith/__init__.py +++ b/src/framesmith/__init__.py @@ -15,6 +15,7 @@ CATEGORY_CANONICALIZE, CATEGORY_CANONICALIZE_TO_SNAKE_CASE, EMAIL_TO_DISPLAY_NAME, + EMAIL_TO_TITLE_NAME, NUMERIC_STRING_NORMALIZE, NUMERIC_STRING_TO_FLOAT, PERCENT_STRING_TO_FRACTION, @@ -27,6 +28,7 @@ TEXT_NORMALIZE, TEXT_NORMALIZE_TO_SNAKE_CASE, UNICODE_TO_ASCII, + UPPERCASE_CODE_NORMALIZE, WHITESPACE_CANONICALIZE, ) from framesmith.types import ExpressionTransform @@ -35,6 +37,7 @@ 'CATEGORY_CANONICALIZE', 'CATEGORY_CANONICALIZE_TO_SNAKE_CASE', 'EMAIL_TO_DISPLAY_NAME', + 'EMAIL_TO_TITLE_NAME', 'NUMERIC_STRING_NORMALIZE', 'NUMERIC_STRING_TO_FLOAT', 'PERCENT_STRING_TO_FRACTION', @@ -47,6 +50,7 @@ 'TEXT_NORMALIZE', 'TEXT_NORMALIZE_TO_SNAKE_CASE', 'UNICODE_TO_ASCII', + 'UPPERCASE_CODE_NORMALIZE', 'WHITESPACE_CANONICALIZE', 'ExpressionTransform', 'compose_column', diff --git a/src/framesmith/recipes.py b/src/framesmith/recipes.py index 577ea42..9189055 100644 --- a/src/framesmith/recipes.py +++ b/src/framesmith/recipes.py @@ -29,13 +29,16 @@ """ from framesmith.transforms import ( + DEFAULT_PLACEHOLDER_SENTINELS, accounting_parens_to_negative, cast_to_float64, collapse_whitespace, extract_email_local_part, + extract_email_local_part_strict, fold_to_ascii, normalize_unicode_nfkc, nullify_blank_strings, + nullify_sentinels, percent_to_fraction, periods_to_spaces, remove_apostrophes, @@ -43,6 +46,7 @@ remove_periods, remove_thousands_separators, replace_ampersand_with_and, + separators_to_space, standardize_directionals, standardize_initials, standardize_street_suffixes, @@ -53,6 +57,7 @@ to_lowercase, to_snake_case, to_titlecase, + to_uppercase, trailing_minus_to_prefix, underscores_to_spaces, ) @@ -62,6 +67,7 @@ 'CATEGORY_CANONICALIZE', 'CATEGORY_CANONICALIZE_TO_SNAKE_CASE', 'EMAIL_TO_DISPLAY_NAME', + 'EMAIL_TO_TITLE_NAME', 'NUMERIC_STRING_NORMALIZE', 'NUMERIC_STRING_TO_FLOAT', 'PERCENT_STRING_TO_FRACTION', @@ -74,6 +80,7 @@ 'TEXT_NORMALIZE', 'TEXT_NORMALIZE_TO_SNAKE_CASE', 'UNICODE_TO_ASCII', + 'UPPERCASE_CODE_NORMALIZE', 'WHITESPACE_CANONICALIZE', ] @@ -197,6 +204,29 @@ ) +# --- Identifier codes (domain normalization) --- + +# Canonicalize an identifier code (VIN, license plate, asset tag) to the one +# form a pipeline joins on: NFKC-normalize (fullwidth and other compatibility +# look-alikes fold to ASCII, so a fullwidth-digit VIN normalizes to plain +# ASCII), tidy whitespace and drop blanks to null via +# WHITESPACE_CANONICALIZE, UPPERCASE, then null the '?' +# placeholder that stands in for an unknown code. On a clean code (no interior +# whitespace, no Unicode oddity, not the placeholder) the result is exactly +# UPPER(TRIM(code)) — the collapse and NFKC steps are no-ops there — so the +# output joins byte-for-byte against a SQL UPPER(TRIM(...)) key while still +# repairing case, exotic whitespace, and look-alike Unicode on messy input. +# Blank-to-null runs inside the WHITESPACE_CANONICALIZE splice; ordering it +# ahead of the uppercase step is equivalent since neither creates or clears a +# blank the other would have caught. +UPPERCASE_CODE_NORMALIZE: tuple[ExpressionTransform, ...] = ( + normalize_unicode_nfkc, + *WHITESPACE_CANONICALIZE, + to_uppercase, + nullify_sentinels(DEFAULT_PLACEHOLDER_SENTINELS), +) + + # --- Numeric strings --- # Clean a messy numeric/currency string into a bare numeric string ready to @@ -240,6 +270,31 @@ collapse_whitespace, ) +# Derive a Title Case display name from an email, modeled on the BigQuery report +# expression +# INITCAP(REGEXP_REPLACE(REGEXP_EXTRACT(LOWER(email), '^([^@]+)@'), +# '[._-]+', ' ')) +# : strict-extract the local part before the first '@' (null for a value with no +# '@' or an empty local part — REGEXP_EXTRACT semantics), replace each +# '.'/'_'/'-' run with a single space (separators_to_space == the '[._-]+' -> ' ' +# regex), then title-case. This differs from EMAIL_TO_DISPLAY_NAME, whose lenient +# extractor keeps a non-conforming value and whose periods_to_spaces ignores '_' +# and '-'. The LOWER in the source is redundant here because to_titlecase re-cases +# every word, so it is omitted. A null or non-conforming email yields null. +# +# Title casing is polars `to_titlecase`, which is NOT byte-identical to BigQuery +# INITCAP on two rare, purely cosmetic edge cases (a single letter's case in a +# display name): (1) an intra-word capital's tail is lowercased, so 'mcdonald' -> +# 'Mcdonald' not 'McDonald'; (2) a letter immediately after a digit is +# capitalized, '2pac' -> '2Pac' where INITCAP (treating digits as word +# characters, not delimiters) yields '2pac'. Neither occurs for first.last@ +# corporate mailboxes. Do not rely on this recipe for byte-exact INITCAP parity. +EMAIL_TO_TITLE_NAME: tuple[ExpressionTransform, ...] = ( + extract_email_local_part_strict, + separators_to_space, + to_titlecase, +) + # Turn a snake_case identifier into a Title Case label: underscores to spaces, # then title-case each word. Title casing lowercases the tail of every word, so # it mangles acronyms ('nasa_program' -> 'Nasa Program'); splice diff --git a/src/framesmith/transforms/__init__.py b/src/framesmith/transforms/__init__.py index c41253c..e76efe5 100644 --- a/src/framesmith/transforms/__init__.py +++ b/src/framesmith/transforms/__init__.py @@ -27,6 +27,7 @@ to_lowercase, to_snake_case, to_titlecase, + to_uppercase, ) from framesmith.transforms.categorical import ( collapse_keep_top_n, @@ -41,6 +42,7 @@ ) from framesmith.transforms.missing import ( DEFAULT_MISSING_SENTINELS, + DEFAULT_PLACEHOLDER_SENTINELS, nullify_sentinels, ) from framesmith.transforms.names import ( @@ -48,6 +50,7 @@ DEFAULT_NAME_PREFIXES, DEFAULT_NAME_SUFFIXES, extract_email_local_part, + extract_email_local_part_strict, remove_credentials, remove_jr_suffix, standardize_initials, @@ -77,6 +80,7 @@ remove_apostrophes, remove_periods, replace_ampersand_with_and, + separators_to_space, underscores_to_spaces, ) from framesmith.transforms.unicode import ( @@ -96,6 +100,7 @@ 'DEFAULT_MISSING_SENTINELS', 'DEFAULT_NAME_PREFIXES', 'DEFAULT_NAME_SUFFIXES', + 'DEFAULT_PLACEHOLDER_SENTINELS', 'DEFAULT_SPLIT_DELIMITERS', 'DEFAULT_STREET_SUFFIX_MAP', 'DEFAULT_UNIT_MARKER_MAP', @@ -108,6 +113,7 @@ 'collapse_rare_by_count', 'collapse_whitespace', 'extract_email_local_part', + 'extract_email_local_part_strict', 'extract_zip_code', 'flag_dates_outside_range', 'flag_iqr_outliers', @@ -130,6 +136,7 @@ 'remove_thousands_separators', 'replace_ampersand_with_and', 'replace_whitespace_with', + 'separators_to_space', 'split_on_delimiters', 'standardize_directionals', 'standardize_initials', @@ -144,6 +151,7 @@ 'to_lowercase', 'to_snake_case', 'to_titlecase', + 'to_uppercase', 'trailing_minus_to_prefix', 'underscores_to_spaces', 'winsorize_numeric', diff --git a/src/framesmith/transforms/case.py b/src/framesmith/transforms/case.py index 9d15bec..935724f 100644 --- a/src/framesmith/transforms/case.py +++ b/src/framesmith/transforms/case.py @@ -16,6 +16,7 @@ 'to_lowercase', 'to_snake_case', 'to_titlecase', + 'to_uppercase', ] @@ -33,6 +34,18 @@ def to_lowercase(expr: pl.Expr) -> pl.Expr: return expr.str.to_lowercase() +def to_uppercase(expr: pl.Expr) -> pl.Expr: + """Uppercase all characters. + + Atomic: casing only. The mirror of :func:`to_lowercase`. Does not + strip, collapse, or Unicode-fold. On plain-ASCII input this agrees + with SQL ``UPPER(...)``, so it is the case step for canonicalizing + identifier codes (VINs, plates, asset tags) to a single join form. + Nulls pass through unchanged. + """ + return expr.str.to_uppercase() + + def to_titlecase(expr: pl.Expr) -> pl.Expr: """Title-case the string: first letter of each word upper, rest lower. diff --git a/src/framesmith/transforms/missing.py b/src/framesmith/transforms/missing.py index 99142f8..116030d 100644 --- a/src/framesmith/transforms/missing.py +++ b/src/framesmith/transforms/missing.py @@ -24,6 +24,7 @@ __all__: list[str] = [ 'DEFAULT_MISSING_SENTINELS', + 'DEFAULT_PLACEHOLDER_SENTINELS', 'nullify_sentinels', ] @@ -52,6 +53,16 @@ ) +# Placeholder tokens that stand in for an unknown identifier or code in +# source exports. Unlike ``DEFAULT_MISSING_SENTINELS`` — whose word-forms +# ('NA', 'NONE') can be valid data — ``'?'`` is never a valid VIN, plate, +# or asset code, so nulling it is safe on an identifier column even though +# it would be noise in free text. This is the set the identifier recipe +# (``UPPERCASE_CODE_NORMALIZE``) nulls; a source with other placeholders +# ('???', 'UNKNOWN') passes its own set to :func:`nullify_sentinels`. +DEFAULT_PLACEHOLDER_SENTINELS: frozenset[str] = frozenset({'?'}) + + def nullify_sentinels( sentinels: Collection[str], *, diff --git a/src/framesmith/transforms/names.py b/src/framesmith/transforms/names.py index 28b019e..5d645e8 100644 --- a/src/framesmith/transforms/names.py +++ b/src/framesmith/transforms/names.py @@ -20,6 +20,7 @@ 'DEFAULT_NAME_PREFIXES', 'DEFAULT_NAME_SUFFIXES', 'extract_email_local_part', + 'extract_email_local_part_strict', 'remove_credentials', 'remove_jr_suffix', 'standardize_initials', @@ -64,6 +65,24 @@ def extract_email_local_part(expr: pl.Expr) -> pl.Expr: return expr.str.split('@').list.first() +def extract_email_local_part_strict(expr: pl.Expr) -> pl.Expr: + """Take the local part of a well-formed address; else null. + + Strict counterpart to :func:`extract_email_local_part`. Captures the + run before the first ``'@'`` via ``^([^@]+)@`` — reproducing SQL + ``REGEXP_EXTRACT(email, '^([^@]+)@')`` — so a value with no ``'@'``, + or a leading ``'@'`` (empty local part), yields null instead of the + lenient extractor's pass-through or empty string. Use this when a + non-conforming address must become null; use the lenient + :func:`extract_email_local_part` when a bare token should survive. + + Atomic: does NOT strip surrounding whitespace, lowercase, or further + normalize — a leading space is part of ``[^@]+`` and is captured + verbatim. Nulls pass through as null. + """ + return expr.str.extract(r'^([^@]+)@', 1) + + def _name_token_alternation(tokens: Sequence[str]) -> str: """Build a regex alternation from affix tokens, longest-first and escaped. diff --git a/src/framesmith/transforms/substitution.py b/src/framesmith/transforms/substitution.py index 42a8e4b..f568e79 100644 --- a/src/framesmith/transforms/substitution.py +++ b/src/framesmith/transforms/substitution.py @@ -2,9 +2,10 @@ """Character- and substring-substitution transforms. Single-character substitutions (``remove_apostrophes``, ``remove_periods``, -``periods_to_spaces``, ``underscores_to_spaces``, ``replace_ampersand_with_and``) -and the configurable literal-substring ``apply_replacements`` factory. All -follow the ``ExpressionTransform`` contract. +``periods_to_spaces``, ``underscores_to_spaces``, ``replace_ampersand_with_and``), +the run-collapsing ``separators_to_space``, and the configurable +literal-substring ``apply_replacements`` factory. All follow the +``ExpressionTransform`` contract. """ import polars as pl @@ -17,6 +18,7 @@ 'remove_apostrophes', 'remove_periods', 'replace_ampersand_with_and', + 'separators_to_space', 'underscores_to_spaces', ] @@ -61,6 +63,24 @@ def underscores_to_spaces(expr: pl.Expr) -> pl.Expr: return expr.str.replace_all('_', ' ', literal=True) +def separators_to_space(expr: pl.Expr) -> pl.Expr: + """Replace each run of ``.`` ``_`` ``-`` separators with one space. + + Reproduces SQL ``REGEXP_REPLACE(text, '[._-]+', ' ')``: a maximal run + of periods, underscores, and/or hyphens — mixed or repeated — collapses + to a single space. ``'jane..doe'`` and ``'a._-b'`` both yield one space + per run (``'jane doe'``, ``'a b'``). + + Distinct from :func:`periods_to_spaces` and :func:`underscores_to_spaces`, + which are single-character and non-collapsing; this transform handles all + three separators at once and collapses runs, which is what the ``+`` in + the SQL pattern does. Whitespace already present is left untouched (only + ``[._-]`` runs are rewritten), so it does not double as + :func:`collapse_whitespace`. Nulls pass through unchanged. + """ + return expr.str.replace_all(r'[._-]+', ' ') + + def apply_replacements(replacements: dict[str, str]) -> ExpressionTransform: """Build a transform applying literal substring replacements. diff --git a/tests/test_recipes.py b/tests/test_recipes.py index 381e90b..dc92247 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -15,11 +15,13 @@ from framesmith import ( EMAIL_TO_DISPLAY_NAME, + EMAIL_TO_TITLE_NAME, NUMERIC_STRING_NORMALIZE, NUMERIC_STRING_TO_FLOAT, PERCENT_STRING_TO_FRACTION, TEXT_NORMALIZE, UNICODE_TO_ASCII, + UPPERCASE_CODE_NORMALIZE, ExpressionTransform, compose_column, recipes, @@ -37,11 +39,13 @@ WHITESPACE_CANONICALIZE, ) from framesmith.transforms import ( + DEFAULT_PLACEHOLDER_SENTINELS, accounting_parens_to_negative, apply_replacements, cast_to_float64, collapse_whitespace, extract_email_local_part, + extract_email_local_part_strict, fold_to_ascii, normalize_unicode_nfkc, nullify_blank_strings, @@ -52,11 +56,13 @@ remove_periods, remove_thousands_separators, replace_ampersand_with_and, + separators_to_space, standardize_initials, strip_whitespace, to_lowercase, to_snake_case, to_titlecase, + to_uppercase, trailing_minus_to_prefix, underscores_to_spaces, ) @@ -558,27 +564,35 @@ class TestNoSentinelNullificationInDefaultRecipes: """Regression guard for the opt-in property of ``nullify_sentinels``. Sentinel handling depends on the data source — defaulting it on would - silently null valid values (e.g. ``'NA'`` as Namibia). These tests pin - the exact transforms in each shipped recipe so a future edit that slips - ``nullify_sentinels`` (or any unexpected transform) into a default - recipe fires immediately. The recipes use ``nullify_blank_strings``, - which is a different transform. + silently null valid values (e.g. ``'NA'`` as Namibia). The general-purpose + text/category/numeric/name/address recipes therefore stay sentinel-free, + pinned by the exact-tuple and structural tests below so a future edit that + slips sentinel nullification (or any unexpected transform) into one fires + immediately; those recipes use ``nullify_blank_strings``, a different + transform. ``UPPERCASE_CODE_NORMALIZE`` is the deliberate exception: on an + identifier column ``'?'`` is never valid data, so it nulls that placeholder + via a *called* ``nullify_sentinels`` closure — locked positively by + ``TestUppercaseCodeNormalize`` rather than pinned here. """ - def test_no_recipe_contains_nullify_sentinels(self) -> None: - # Sweep all 16 published recipes, including the factory-closure - # ones that the exact-tuple pins below cannot reconstruct. + def test_no_recipe_contains_the_uncalled_factory(self) -> None: + # Tripwire for the specific mistake of splicing the *uncalled* + # nullify_sentinels factory into a recipe (a bug — the factory must be + # called to yield a transform). It holds for every published recipe, + # including UPPERCASE_CODE_NORMALIZE, whose intentional sentinel step is + # the called closure — a different object than the factory checked here. for name in recipes.__all__: recipe = getattr(recipes, name) assert nullify_sentinels not in recipe, name def test_published_recipe_names_pinned(self) -> None: - # The 16 recipes the module publishes, locked so a rename or - # accidental drop fires here. + # The recipes the module publishes, locked so a rename or accidental + # drop fires here. assert recipes.__all__ == [ 'CATEGORY_CANONICALIZE', 'CATEGORY_CANONICALIZE_TO_SNAKE_CASE', 'EMAIL_TO_DISPLAY_NAME', + 'EMAIL_TO_TITLE_NAME', 'NUMERIC_STRING_NORMALIZE', 'NUMERIC_STRING_TO_FLOAT', 'PERCENT_STRING_TO_FRACTION', @@ -591,6 +605,7 @@ def test_published_recipe_names_pinned(self) -> None: 'TEXT_NORMALIZE', 'TEXT_NORMALIZE_TO_SNAKE_CASE', 'UNICODE_TO_ASCII', + 'UPPERCASE_CODE_NORMALIZE', 'WHITESPACE_CANONICALIZE', ] @@ -697,6 +712,13 @@ def test_email_to_display_name_contents_pinned(self) -> None: def test_snake_case_to_title_contents_pinned(self) -> None: assert (underscores_to_spaces, to_titlecase) == SNAKE_CASE_TO_TITLE + def test_email_to_title_name_contents_pinned(self) -> None: + assert ( + extract_email_local_part_strict, + separators_to_space, + to_titlecase, + ) == EMAIL_TO_TITLE_NAME + # --- Structural pins for the factory-closure recipes --- # standardize_*() and remove_credentials()/strip_name_*() return fresh # closures that no independent construction can equal, so these pin the @@ -717,6 +739,15 @@ def test_person_name_normalize_structure_pinned(self) -> None: assert PERSON_NAME_NORMALIZE[6] is standardize_initials assert PERSON_NAME_NORMALIZE[7] is strip_whitespace + def test_uppercase_code_normalize_structure_pinned(self) -> None: + # NFKC, then the WHITESPACE_CANONICALIZE splice, then uppercase, then a + # nullify_sentinels closure that no independent construction can equal. + assert len(UPPERCASE_CODE_NORMALIZE) == 6 + assert UPPERCASE_CODE_NORMALIZE[0] is normalize_unicode_nfkc + assert UPPERCASE_CODE_NORMALIZE[1:4] == WHITESPACE_CANONICALIZE + assert UPPERCASE_CODE_NORMALIZE[4] is to_uppercase + assert callable(UPPERCASE_CODE_NORMALIZE[5]) + # --------------------------------------------------------------------- # PERCENT_STRING_TO_FRACTION end-to-end @@ -920,3 +951,179 @@ def test_splice_with_apply_replacements_fixes_acronym(self) -> None: recipe = (*SNAKE_CASE_TO_TITLE, apply_replacements({'Nasa': 'NASA'})) result = _apply(['nasa_program', 'visit_nasa'], recipe) assert result.to_list() == ['NASA Program', 'Visit NASA'] + + +# --------------------------------------------------------------------- +# UPPERCASE_CODE_NORMALIZE end-to-end +# --------------------------------------------------------------------- + + +class TestUppercaseCodeNormalize: + @pytest.mark.parametrize( + ('value', 'expected'), + [ + # Lowercase VIN uppercased. + ('1hgcm82633a004352', '1HGCM82633A004352'), + # Surrounding whitespace stripped. + (' JH4KA2650MC000000 ', 'JH4KA2650MC000000'), + # Interior whitespace run collapsed to a single space, then upper. + ('ab cd', 'AB CD'), + # Fullwidth Unicode look-alikes fold to ASCII via NFKC. + ('ＡＢＣ１２３', 'ABC123'), # noqa: RUF001 + # Fullwidth AND lowercase: NFKC folds, then uppercase. + ('ａｂｃ１２３', 'ABC123'), # noqa: RUF001 + ], + ) + def test_canonicalizes_identifier_code( + self, value: str, expected: str + ) -> None: + result = _apply([value], UPPERCASE_CODE_NORMALIZE) + assert result.to_list() == [expected] + + @pytest.mark.parametrize( + 'value', + ['', ' ', '\t', '?', ' ? ', None], + ) + def test_blank_and_placeholder_become_null( + self, value: str | None + ) -> None: + # Blank/whitespace-only via nullify_blank_strings; the '?' placeholder + # (even whitespace-padded) via the nullify_sentinels step. + result = _apply([value], UPPERCASE_CODE_NORMALIZE) + assert result.to_list() == [None] + + def test_agrees_with_sql_upper_trim_on_clean_input(self) -> None: + # The core join guarantee: on a clean code (no interior whitespace, no + # Unicode oddity, not the '?' placeholder) the recipe output is exactly + # UPPER(TRIM(code)), so it joins byte-for-byte against a SQL key. + clean = [ + '1hgcm82633a004352', + ' JH4KA2650MC000000 ', + 'wba3a5c50cf256949', + 'ABCdef123', + ] + df = pl.DataFrame({'x': clean}, schema={'x': pl.String}) + sql_upper_trim = df.select( + pl.col('x').str.strip_chars().str.to_uppercase().alias('x') + ) + recipe_out = df.with_columns( + compose_column('x', UPPERCASE_CODE_NORMALIZE) + ) + assert_frame_equal(recipe_out, sql_upper_trim) + + def test_nulls_the_default_placeholder_sentinel(self) -> None: + # Positive lock for the deliberate sentinel exception: the recipe nulls + # the '?' placeholder, and DEFAULT_PLACEHOLDER_SENTINELS is that token. + assert frozenset({'?'}) == DEFAULT_PLACEHOLDER_SENTINELS + assert _apply(['?'], UPPERCASE_CODE_NORMALIZE).to_list() == [None] + + def test_is_tuple_not_list(self) -> None: + assert isinstance(UPPERCASE_CODE_NORMALIZE, tuple) + assert not isinstance(UPPERCASE_CODE_NORMALIZE, list) + + def test_output_dtype_is_string(self) -> None: + result = _apply(['1hgcm82633a004352'], UPPERCASE_CODE_NORMALIZE) + assert result.dtype == pl.String + + def test_lazy_matches_eager(self) -> None: + df = pl.DataFrame( + { + 'x': [ + '1hgcm82633a004352', + ' JH4KA2650MC000000 ', + 'ab cd', + 'ＡＢＣ１２３', # noqa: RUF001 + '?', + ' ', + None, + ] + }, + schema={'x': pl.String}, + ) + expr = compose_column('x', UPPERCASE_CODE_NORMALIZE) + eager = df.with_columns(expr) + lazy = df.lazy().with_columns(expr).collect() + assert_frame_equal(eager, lazy) + + +# --------------------------------------------------------------------- +# EMAIL_TO_TITLE_NAME end-to-end +# --------------------------------------------------------------------- + + +class TestEmailToTitleName: + @pytest.mark.parametrize( + ('value', 'expected'), + [ + ('john.doe@example.com', 'John Doe'), + # Underscores and hyphens are separators too (unlike + # EMAIL_TO_DISPLAY_NAME, which handles only periods). + ('jane_q_smith@example.com', 'Jane Q Smith'), + ('a-b-c@example.com', 'A B C'), + ('first.middle_last-jr@example.com', 'First Middle Last Jr'), + # A mixed run of separators collapses to one space. + ('a._-b@example.com', 'A B'), + # Single-token local part. + ('john@example.com', 'John'), + # Known INITCAP limitation preserved to match the report. + ('mcdonald@example.com', 'Mcdonald'), + # Documented cosmetic divergence from BigQuery INITCAP: polars + # to_titlecase capitalizes a letter after a digit (INITCAP would + # leave '2pac' / '1st'). We pin the polars behavior. + ('2pac@example.com', '2Pac'), + ('1st.class@example.com', '1St Class'), + # Uppercase input still title-cases (the source LOWER is redundant). + ('JOHN.DOE@X.COM', 'John Doe'), + # Only the part before the first '@' is used. + ('john.doe@host@sub.com', 'John Doe'), + ], + ) + def test_produces_title_name(self, value: str, expected: str) -> None: + result = _apply([value], EMAIL_TO_TITLE_NAME) + assert result.to_list() == [expected] + + @pytest.mark.parametrize( + 'value', + [ + 'noatsign', # no '@' at all + '@example.com', # empty local part + '', + None, + ], + ) + def test_null_or_non_conforming_yields_null( + self, value: str | None + ) -> None: + # REGEXP_EXTRACT('^([^@]+)@') semantics: anything without a non-empty + # local part before an '@' is null, matching the BigQuery report. + result = _apply([value], EMAIL_TO_TITLE_NAME) + assert result.to_list() == [None] + + def test_is_tuple_not_list(self) -> None: + assert isinstance(EMAIL_TO_TITLE_NAME, tuple) + assert not isinstance(EMAIL_TO_TITLE_NAME, list) + + def test_output_dtype_is_string(self) -> None: + result = _apply(['john.doe@example.com'], EMAIL_TO_TITLE_NAME) + assert result.dtype == pl.String + + def test_lazy_matches_eager(self) -> None: + df = pl.DataFrame( + { + 'x': [ + 'john.doe@example.com', + 'jane_q_smith@example.com', + 'a-b-c@example.com', + 'mcdonald@example.com', + 'noatsign', + '@example.com', + '', + None, + ] + }, + schema={'x': pl.String}, + ) + expr = compose_column('x', EMAIL_TO_TITLE_NAME) + eager = df.with_columns(expr) + lazy = df.lazy().with_columns(expr).collect() + assert_frame_equal(eager, lazy) diff --git a/tests/transforms/test_case.py b/tests/transforms/test_case.py index eca7772..c3e143c 100644 --- a/tests/transforms/test_case.py +++ b/tests/transforms/test_case.py @@ -14,6 +14,7 @@ to_lowercase, to_snake_case, to_titlecase, + to_uppercase, ) @@ -47,6 +48,28 @@ def test_null_propagates(self) -> None: assert result.to_list() == [None] +class TestToUppercase: + def test_mixed_case_becomes_upper(self) -> None: + result = _apply(['Hello World'], to_uppercase) + assert result.to_list() == ['HELLO WORLD'] + + def test_already_uppercase_unchanged(self) -> None: + result = _apply(['ALREADY UPPER'], to_uppercase) + assert result.to_list() == ['ALREADY UPPER'] + + def test_digits_and_punctuation_unaffected(self) -> None: + result = _apply(['abc-123'], to_uppercase) + assert result.to_list() == ['ABC-123'] + + def test_null_propagates(self) -> None: + result = _apply([None], to_uppercase) + assert result.to_list() == [None] + + def test_output_dtype_is_string(self) -> None: + result = _apply(['hello'], to_uppercase) + assert result.dtype == pl.String + + class TestToSnakeCase: def test_single_space_becomes_underscore(self) -> None: result = _apply(['hello world'], to_snake_case) @@ -136,6 +159,7 @@ def test_null_propagates_for_module_transforms(self) -> None: to_lowercase, to_snake_case, to_titlecase, + to_uppercase, ] expected = pl.Series('x', [None], dtype=pl.String) for transform in transforms: diff --git a/tests/transforms/test_missing.py b/tests/transforms/test_missing.py index db538f8..a99511c 100644 --- a/tests/transforms/test_missing.py +++ b/tests/transforms/test_missing.py @@ -6,7 +6,11 @@ from polars.testing import assert_frame_equal from framesmith import TEXT_NORMALIZE, ExpressionTransform, compose_column -from framesmith.transforms import DEFAULT_MISSING_SENTINELS, nullify_sentinels +from framesmith.transforms import ( + DEFAULT_MISSING_SENTINELS, + DEFAULT_PLACEHOLDER_SENTINELS, + nullify_sentinels, +) def _apply( @@ -83,6 +87,31 @@ def test_null_propagates(self, transform: ExpressionTransform) -> None: assert result.to_list() == [None] +# --------------------------------------------------------------------- +# DEFAULT_PLACEHOLDER_SENTINELS behavior +# --------------------------------------------------------------------- + + +class TestDefaultPlaceholderSentinels: + def test_contains_the_question_mark_placeholder(self) -> None: + assert frozenset({'?'}) == DEFAULT_PLACEHOLDER_SENTINELS + + def test_nulls_question_mark(self) -> None: + transform = nullify_sentinels(DEFAULT_PLACEHOLDER_SENTINELS) + assert _apply(['?'], transform).to_list() == [None] + + def test_whitespace_padded_placeholder_nulled(self) -> None: + # The factory strips before comparison, so a padded '?' still matches. + transform = nullify_sentinels(DEFAULT_PLACEHOLDER_SENTINELS) + assert _apply([' ? '], transform).to_list() == [None] + + def test_real_code_unchanged(self) -> None: + transform = nullify_sentinels(DEFAULT_PLACEHOLDER_SENTINELS) + assert _apply(['1HGCM82633A004352'], transform).to_list() == [ + '1HGCM82633A004352' + ] + + # --------------------------------------------------------------------- # Custom sentinel sets # --------------------------------------------------------------------- diff --git a/tests/transforms/test_names.py b/tests/transforms/test_names.py index 2ff112f..3e3a039 100644 --- a/tests/transforms/test_names.py +++ b/tests/transforms/test_names.py @@ -8,6 +8,7 @@ from framesmith import ExpressionTransform, compose_column from framesmith.transforms import ( extract_email_local_part, + extract_email_local_part_strict, remove_credentials, remove_jr_suffix, standardize_initials, @@ -112,6 +113,70 @@ def test_output_dtype_is_string(self) -> None: assert result.dtype == pl.String +class TestExtractEmailLocalPartStrict: + @pytest.mark.parametrize( + ('value', 'expected'), + [ + ('john@example.com', 'john'), + # Separators preserved at this stage — that's separators_to_space's + # job downstream. + ('john.doe@example.com', 'john.doe'), + ('jane_q_smith@example.com', 'jane_q_smith'), + # Only the part before the FIRST '@' is captured. + ('john@host@sub', 'john'), + ], + ) + def test_takes_part_before_first_at( + self, value: str, expected: str + ) -> None: + result = _apply([value], extract_email_local_part_strict) + assert result.to_list() == [expected] + + def test_no_at_sign_yields_null(self) -> None: + # Strict divergence from the lenient extractor, which passes 'noatsign' + # through unchanged. Here a missing '@' means null. + result = _apply(['noatsign'], extract_email_local_part_strict) + assert result.to_list() == [None] + + def test_leading_at_empty_local_part_yields_null(self) -> None: + # The lenient extractor yields '' here; the strict pattern requires a + # non-empty run before '@', so this is null. + result = _apply(['@example.com'], extract_email_local_part_strict) + assert result.to_list() == [None] + + def test_empty_string_yields_null(self) -> None: + result = _apply([''], extract_email_local_part_strict) + assert result.to_list() == [None] + + def test_null_propagates(self) -> None: + result = _apply([None], extract_email_local_part_strict) + assert result.to_list() == [None] + + def test_does_not_strip_surrounding_whitespace(self) -> None: + # Atomic-contract proof: a leading space is part of the '[^@]+' run and + # is captured verbatim. Compose strip_whitespace upstream if needed. + result = _apply( + [' padded@example.com'], extract_email_local_part_strict + ) + assert result.to_list() == [' padded'] + + def test_output_dtype_is_string(self) -> None: + result = _apply( + ['john@example.com'], extract_email_local_part_strict + ) + assert result.dtype == pl.String + + def test_lazy_and_eager_produce_identical_results(self) -> None: + df = pl.DataFrame( + {'x': ['john.doe@x.com', 'noatsign', '@x.com', '', None]}, + schema={'x': pl.String}, + ) + expr = compose_column('x', [extract_email_local_part_strict]) + eager = df.with_columns(expr) + lazy = df.lazy().with_columns(expr).collect() + assert_frame_equal(eager, lazy) + + class TestStripNameSuffixes: @pytest.mark.parametrize( ('value', 'expected'), diff --git a/tests/transforms/test_substitution.py b/tests/transforms/test_substitution.py index 5e74d3a..ddf3361 100644 --- a/tests/transforms/test_substitution.py +++ b/tests/transforms/test_substitution.py @@ -16,6 +16,7 @@ remove_apostrophes, remove_periods, replace_ampersand_with_and, + separators_to_space, underscores_to_spaces, ) @@ -123,6 +124,44 @@ def test_output_dtype_is_string(self) -> None: assert result.dtype == pl.String +class TestSeparatorsToSpace: + @pytest.mark.parametrize( + ('value', 'expected'), + [ + ('john.doe', 'john doe'), + ('jane_q_smith', 'jane q smith'), + ('a-b-c', 'a b c'), + # Mixed and repeated runs collapse to a single space (the '+'). + ('john..doe', 'john doe'), + ('a._-b', 'a b'), + ('first.middle_last-jr', 'first middle last jr'), + # No separators — unchanged. + ('plain', 'plain'), + ('', ''), + ], + ) + def test_replaces_separator_runs_with_one_space( + self, value: str, expected: str + ) -> None: + result = _apply([value], separators_to_space) + assert result.to_list() == [expected] + + def test_existing_whitespace_left_untouched(self) -> None: + # Only [._-] runs are rewritten; pre-existing whitespace runs are not + # collapsed (that is collapse_whitespace's job). This is what keeps the + # transform faithful to SQL REGEXP_REPLACE(text, '[._-]+', ' '). + result = _apply(['a b.c'], separators_to_space) + assert result.to_list() == ['a b c'] + + def test_null_propagates(self) -> None: + result = _apply([None], separators_to_space) + assert result.to_list() == [None] + + def test_output_dtype_is_string(self) -> None: + result = _apply(['john.doe'], separators_to_space) + assert result.dtype == pl.String + + class TestApplyReplacements: def test_single_replacement(self) -> None: transform = apply_replacements({'Nasa': 'NASA'}) @@ -176,6 +215,7 @@ def test_null_propagates_for_module_transforms(self) -> None: remove_periods, periods_to_spaces, underscores_to_spaces, + separators_to_space, apply_replacements({'x': 'y'}), ] expected = pl.Series('x', [None], dtype=pl.String)