Add VIN + email-to-title recipes (0.2.0)

pyproject.toml

@@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta"
 ###############################################################################
 [project]
 name = "framesmith"
-version = "0.1.0"
+version = "0.2.0"
 description = "Composable, expression-first preprocessing for polars DataFrames: small atomic transforms, declarative recipes, and one function that composes them onto a column."
 readme = "README.md"
 authors = [{ name = "Andrew Jordan", email = "andrewjordan3@gmail.com" }]

src/framesmith/__init__.py

@@ -15,6 +15,7 @@
     CATEGORY_CANONICALIZE,
     CATEGORY_CANONICALIZE_TO_SNAKE_CASE,
     EMAIL_TO_DISPLAY_NAME,
+    EMAIL_TO_TITLE_NAME,
     NUMERIC_STRING_NORMALIZE,
     NUMERIC_STRING_TO_FLOAT,
     PERCENT_STRING_TO_FRACTION,
@@ -27,6 +28,7 @@
     TEXT_NORMALIZE,
     TEXT_NORMALIZE_TO_SNAKE_CASE,
     UNICODE_TO_ASCII,
+    UPPERCASE_CODE_NORMALIZE,
     WHITESPACE_CANONICALIZE,
 )
 from framesmith.types import ExpressionTransform
@@ -35,6 +37,7 @@
     'CATEGORY_CANONICALIZE',
     'CATEGORY_CANONICALIZE_TO_SNAKE_CASE',
     'EMAIL_TO_DISPLAY_NAME',
+    'EMAIL_TO_TITLE_NAME',
     'NUMERIC_STRING_NORMALIZE',
     'NUMERIC_STRING_TO_FLOAT',
     'PERCENT_STRING_TO_FRACTION',
@@ -47,6 +50,7 @@
     'TEXT_NORMALIZE',
     'TEXT_NORMALIZE_TO_SNAKE_CASE',
     'UNICODE_TO_ASCII',
+    'UPPERCASE_CODE_NORMALIZE',
     'WHITESPACE_CANONICALIZE',
     'ExpressionTransform',
     'compose_column',

src/framesmith/recipes.py

@@ -29,20 +29,24 @@
 """
 
 from framesmith.transforms import (
+    DEFAULT_PLACEHOLDER_SENTINELS,
     accounting_parens_to_negative,
     cast_to_float64,
     collapse_whitespace,
     extract_email_local_part,
+    extract_email_local_part_strict,
     fold_to_ascii,
     normalize_unicode_nfkc,
     nullify_blank_strings,
+    nullify_sentinels,
     percent_to_fraction,
     periods_to_spaces,
     remove_apostrophes,
     remove_credentials,
     remove_periods,
     remove_thousands_separators,
     replace_ampersand_with_and,
+    separators_to_space,
     standardize_directionals,
     standardize_initials,
     standardize_street_suffixes,
@@ -53,6 +57,7 @@
     to_lowercase,
     to_snake_case,
     to_titlecase,
+    to_uppercase,
     trailing_minus_to_prefix,
     underscores_to_spaces,
 )
@@ -62,6 +67,7 @@
     'CATEGORY_CANONICALIZE',
     'CATEGORY_CANONICALIZE_TO_SNAKE_CASE',
     'EMAIL_TO_DISPLAY_NAME',
+    'EMAIL_TO_TITLE_NAME',
     'NUMERIC_STRING_NORMALIZE',
     'NUMERIC_STRING_TO_FLOAT',
     'PERCENT_STRING_TO_FRACTION',
@@ -74,6 +80,7 @@
     'TEXT_NORMALIZE',
     'TEXT_NORMALIZE_TO_SNAKE_CASE',
     'UNICODE_TO_ASCII',
+    'UPPERCASE_CODE_NORMALIZE',
     'WHITESPACE_CANONICALIZE',
 ]
 
@@ -197,6 +204,29 @@
 )
 
 
+# --- Identifier codes (domain normalization) ---
+
+# Canonicalize an identifier code (VIN, license plate, asset tag) to the one
+# form a pipeline joins on: NFKC-normalize (fullwidth and other compatibility
+# look-alikes fold to ASCII, so a fullwidth-digit VIN normalizes to plain
+# ASCII), tidy whitespace and drop blanks to null via
+# WHITESPACE_CANONICALIZE, UPPERCASE, then null the '?'
+# placeholder that stands in for an unknown code. On a clean code (no interior
+# whitespace, no Unicode oddity, not the placeholder) the result is exactly
+# UPPER(TRIM(code)) — the collapse and NFKC steps are no-ops there — so the
+# output joins byte-for-byte against a SQL UPPER(TRIM(...)) key while still
+# repairing case, exotic whitespace, and look-alike Unicode on messy input.
+# Blank-to-null runs inside the WHITESPACE_CANONICALIZE splice; ordering it
+# ahead of the uppercase step is equivalent since neither creates or clears a
+# blank the other would have caught.
+UPPERCASE_CODE_NORMALIZE: tuple[ExpressionTransform, ...] = (
+    normalize_unicode_nfkc,
+    *WHITESPACE_CANONICALIZE,
+    to_uppercase,
+    nullify_sentinels(DEFAULT_PLACEHOLDER_SENTINELS),
+)
+
+
 # --- Numeric strings ---
 
 # Clean a messy numeric/currency string into a bare numeric string ready to
@@ -240,6 +270,31 @@
     collapse_whitespace,
 )
 
+# Derive a Title Case display name from an email, modeled on the BigQuery report
+# expression
+#   INITCAP(REGEXP_REPLACE(REGEXP_EXTRACT(LOWER(email), '^([^@]+)@'),
+#                          '[._-]+', ' '))
+# : strict-extract the local part before the first '@' (null for a value with no
+# '@' or an empty local part — REGEXP_EXTRACT semantics), replace each
+# '.'/'_'/'-' run with a single space (separators_to_space == the '[._-]+' -> ' '
+# regex), then title-case. This differs from EMAIL_TO_DISPLAY_NAME, whose lenient
+# extractor keeps a non-conforming value and whose periods_to_spaces ignores '_'
+# and '-'. The LOWER in the source is redundant here because to_titlecase re-cases
+# every word, so it is omitted. A null or non-conforming email yields null.
+#
+# Title casing is polars `to_titlecase`, which is NOT byte-identical to BigQuery
+# INITCAP on two rare, purely cosmetic edge cases (a single letter's case in a
+# display name): (1) an intra-word capital's tail is lowercased, so 'mcdonald' ->
+# 'Mcdonald' not 'McDonald'; (2) a letter immediately after a digit is
+# capitalized, '2pac' -> '2Pac' where INITCAP (treating digits as word
+# characters, not delimiters) yields '2pac'. Neither occurs for first.last@
+# corporate mailboxes. Do not rely on this recipe for byte-exact INITCAP parity.
+EMAIL_TO_TITLE_NAME: tuple[ExpressionTransform, ...] = (
+    extract_email_local_part_strict,
+    separators_to_space,
+    to_titlecase,
+)
+
 # Turn a snake_case identifier into a Title Case label: underscores to spaces,
 # then title-case each word. Title casing lowercases the tail of every word, so
 # it mangles acronyms ('nasa_program' -> 'Nasa Program'); splice

src/framesmith/transforms/__init__.py

@@ -27,6 +27,7 @@
     to_lowercase,
     to_snake_case,
     to_titlecase,
+    to_uppercase,
 )
 from framesmith.transforms.categorical import (
     collapse_keep_top_n,
@@ -41,13 +42,15 @@
 )
 from framesmith.transforms.missing import (
     DEFAULT_MISSING_SENTINELS,
+    DEFAULT_PLACEHOLDER_SENTINELS,
     nullify_sentinels,
 )
 from framesmith.transforms.names import (
     DEFAULT_CREDENTIALS,
     DEFAULT_NAME_PREFIXES,
     DEFAULT_NAME_SUFFIXES,
     extract_email_local_part,
+    extract_email_local_part_strict,
     remove_credentials,
     remove_jr_suffix,
     standardize_initials,
@@ -77,6 +80,7 @@
     remove_apostrophes,
     remove_periods,
     replace_ampersand_with_and,
+    separators_to_space,
     underscores_to_spaces,
 )
 from framesmith.transforms.unicode import (
@@ -96,6 +100,7 @@
     'DEFAULT_MISSING_SENTINELS',
     'DEFAULT_NAME_PREFIXES',
     'DEFAULT_NAME_SUFFIXES',
+    'DEFAULT_PLACEHOLDER_SENTINELS',
     'DEFAULT_SPLIT_DELIMITERS',
     'DEFAULT_STREET_SUFFIX_MAP',
     'DEFAULT_UNIT_MARKER_MAP',
@@ -108,6 +113,7 @@
     'collapse_rare_by_count',
     'collapse_whitespace',
     'extract_email_local_part',
+    'extract_email_local_part_strict',
     'extract_zip_code',
     'flag_dates_outside_range',
     'flag_iqr_outliers',
@@ -130,6 +136,7 @@
     'remove_thousands_separators',
     'replace_ampersand_with_and',
     'replace_whitespace_with',
+    'separators_to_space',
     'split_on_delimiters',
     'standardize_directionals',
     'standardize_initials',
@@ -144,6 +151,7 @@
     'to_lowercase',
     'to_snake_case',
     'to_titlecase',
+    'to_uppercase',
     'trailing_minus_to_prefix',
     'underscores_to_spaces',
     'winsorize_numeric',

src/framesmith/transforms/case.py

@@ -16,6 +16,7 @@
     'to_lowercase',
     'to_snake_case',
     'to_titlecase',
+    'to_uppercase',
 ]
 
 
@@ -33,6 +34,18 @@ def to_lowercase(expr: pl.Expr) -> pl.Expr:
     return expr.str.to_lowercase()
 
 
+def to_uppercase(expr: pl.Expr) -> pl.Expr:
+    """Uppercase all characters.
+
+    Atomic: casing only. The mirror of :func:`to_lowercase`. Does not
+    strip, collapse, or Unicode-fold. On plain-ASCII input this agrees
+    with SQL ``UPPER(...)``, so it is the case step for canonicalizing
+    identifier codes (VINs, plates, asset tags) to a single join form.
+    Nulls pass through unchanged.
+    """
+    return expr.str.to_uppercase()
+
+
 def to_titlecase(expr: pl.Expr) -> pl.Expr:
     """Title-case the string: first letter of each word upper, rest lower.
 

src/framesmith/transforms/missing.py

@@ -24,6 +24,7 @@
 
 __all__: list[str] = [
     'DEFAULT_MISSING_SENTINELS',
+    'DEFAULT_PLACEHOLDER_SENTINELS',
     'nullify_sentinels',
 ]
 
@@ -52,6 +53,16 @@
 )
 
 
+# Placeholder tokens that stand in for an unknown identifier or code in
+# source exports. Unlike ``DEFAULT_MISSING_SENTINELS`` — whose word-forms
+# ('NA', 'NONE') can be valid data — ``'?'`` is never a valid VIN, plate,
+# or asset code, so nulling it is safe on an identifier column even though
+# it would be noise in free text. This is the set the identifier recipe
+# (``UPPERCASE_CODE_NORMALIZE``) nulls; a source with other placeholders
+# ('???', 'UNKNOWN') passes its own set to :func:`nullify_sentinels`.
+DEFAULT_PLACEHOLDER_SENTINELS: frozenset[str] = frozenset({'?'})
+
+
 def nullify_sentinels(
     sentinels: Collection[str],
     *,

src/framesmith/transforms/names.py

@@ -20,6 +20,7 @@
     'DEFAULT_NAME_PREFIXES',
     'DEFAULT_NAME_SUFFIXES',
     'extract_email_local_part',
+    'extract_email_local_part_strict',
     'remove_credentials',
     'remove_jr_suffix',
     'standardize_initials',
@@ -64,6 +65,24 @@ def extract_email_local_part(expr: pl.Expr) -> pl.Expr:
     return expr.str.split('@').list.first()
 
 
+def extract_email_local_part_strict(expr: pl.Expr) -> pl.Expr:
+    """Take the local part of a well-formed address; else null.
+
+    Strict counterpart to :func:`extract_email_local_part`. Captures the
+    run before the first ``'@'`` via ``^([^@]+)@`` — reproducing SQL
+    ``REGEXP_EXTRACT(email, '^([^@]+)@')`` — so a value with no ``'@'``,
+    or a leading ``'@'`` (empty local part), yields null instead of the
+    lenient extractor's pass-through or empty string. Use this when a
+    non-conforming address must become null; use the lenient
+    :func:`extract_email_local_part` when a bare token should survive.
+
+    Atomic: does NOT strip surrounding whitespace, lowercase, or further
+    normalize — a leading space is part of ``[^@]+`` and is captured
+    verbatim. Nulls pass through as null.
+    """
+    return expr.str.extract(r'^([^@]+)@', 1)
+
+
 def _name_token_alternation(tokens: Sequence[str]) -> str:
     """Build a regex alternation from affix tokens, longest-first and escaped.
 

src/framesmith/transforms/substitution.py

@@ -2,9 +2,10 @@
 """Character- and substring-substitution transforms.
 
 Single-character substitutions (``remove_apostrophes``, ``remove_periods``,
-``periods_to_spaces``, ``underscores_to_spaces``, ``replace_ampersand_with_and``)
-and the configurable literal-substring ``apply_replacements`` factory. All
-follow the ``ExpressionTransform`` contract.
+``periods_to_spaces``, ``underscores_to_spaces``, ``replace_ampersand_with_and``),
+the run-collapsing ``separators_to_space``, and the configurable
+literal-substring ``apply_replacements`` factory. All follow the
+``ExpressionTransform`` contract.
 """
 
 import polars as pl
@@ -17,6 +18,7 @@
     'remove_apostrophes',
     'remove_periods',
     'replace_ampersand_with_and',
+    'separators_to_space',
     'underscores_to_spaces',
 ]
 
@@ -61,6 +63,24 @@ def underscores_to_spaces(expr: pl.Expr) -> pl.Expr:
     return expr.str.replace_all('_', ' ', literal=True)
 
 
+def separators_to_space(expr: pl.Expr) -> pl.Expr:
+    """Replace each run of ``.`` ``_`` ``-`` separators with one space.
+
+    Reproduces SQL ``REGEXP_REPLACE(text, '[._-]+', ' ')``: a maximal run
+    of periods, underscores, and/or hyphens — mixed or repeated — collapses
+    to a single space. ``'jane..doe'`` and ``'a._-b'`` both yield one space
+    per run (``'jane doe'``, ``'a b'``).
+
+    Distinct from :func:`periods_to_spaces` and :func:`underscores_to_spaces`,
+    which are single-character and non-collapsing; this transform handles all
+    three separators at once and collapses runs, which is what the ``+`` in
+    the SQL pattern does. Whitespace already present is left untouched (only
+    ``[._-]`` runs are rewritten), so it does not double as
+    :func:`collapse_whitespace`. Nulls pass through unchanged.
+    """
+    return expr.str.replace_all(r'[._-]+', ' ')
+
+
 def apply_replacements(replacements: dict[str, str]) -> ExpressionTransform:
     """Build a transform applying literal substring replacements.