From cb2f24fe3a16ab18a38c566563a25ffd1f364181 Mon Sep 17 00:00:00 2001
From: Eric Hills <53243273+ebhills@users.noreply.github.com>
Date: Sat, 15 Aug 2026 10:48:29 -0500
Subject: [PATCH 1/3] Add standardize.clean and namespace support for
standardize
Introduces a local text-cleaning wrangle (`standardize.clean`) powered by `ftfy`, with options for encoding fixes, Unicode normalization, control/whitespace cleanup, and shape-preserving behavior for strings/lists. Adds explicit `standardize.custom` while keeping legacy `standardize` callable, wires callable namespace behavior through recipe/dataframe/schema discovery, documents the new usage in README, and adds comprehensive recipe/core tests for compatibility and edge cases.
---
README.md | 35 +++
requirements.txt | 3 +
schema/generate_recipe_schema.py | 26 +-
tests/recipes/wrangles/test_standardize.py | 349 +++++++++++++++++++++
wrangles/dataframe.py | 28 +-
wrangles/recipe_wrangles/__init__.py | 8 +
wrangles/recipe_wrangles/standardize.py | 224 +++++++++++++
wrangles/standardize.py | 119 +++++++
8 files changed, 782 insertions(+), 10 deletions(-)
create mode 100644 tests/recipes/wrangles/test_standardize.py
create mode 100644 wrangles/recipe_wrangles/standardize.py
diff --git a/README.md b/README.md
index d071f7c1b..4d6a9f392 100644
--- a/README.md
+++ b/README.md
@@ -127,6 +127,41 @@ Wrangles broadly accept a single input string, or a list of strings. If a list i
]
```
+#### Integrated text cleaning
+
+`standardize.clean` repairs common mojibake, HTML character references,
+Unicode inconsistencies, control characters, and whitespace locally. It accepts
+a string or list and preserves that shape:
+
+```python
+>>> wrangles.standardize.clean(["Français", " AT&T "])
+['Français', 'AT&T']
+```
+
+In a recipe, one input maps to one output and an omitted output overwrites the
+input. Multiple inputs map positionally to equally many outputs, or concatenate
+row values when a single output is given. Wildcard-expanded inputs follow the
+same rules.
+
+```yaml
+wrangles:
+ - standardize.clean:
+ input: Description *
+ output: Clean Description
+ separator: " | "
+ normalization: NFKC
+ preserve_line_breaks: true
+```
+
+Common options include `fix_encoding`, `unescape_html`, `normalization`,
+`fix_character_width`, `uncurl_quotes`, `remove_control_chars`,
+`collapse_whitespace`, `preserve_line_breaks`, and `trim`. Less-common
+`ftfy.fix_text` options are forwarded by name. This wrangle does not detect raw
+byte encodings or remove HTML tags.
+
+The existing model-backed `standardize` name remains supported and is also
+available explicitly as `standardize.custom`.
+
### Recipes
Recipes are written in YAML and allow a series of Wrangles to be run as an automated sequence.
diff --git a/requirements.txt b/requirements.txt
index a1a978b61..546d3c9e3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -14,6 +14,9 @@ lorem
pydantic>=2.12.0
pyyaml
+# Text cleanup
+ftfy>=6.3.1,<7
+
# API & auth
requests
PyJWT
diff --git a/schema/generate_recipe_schema.py b/schema/generate_recipe_schema.py
index 01380ead6..ad929f09f 100644
--- a/schema/generate_recipe_schema.py
+++ b/schema/generate_recipe_schema.py
@@ -136,23 +136,31 @@ def getMethodDocs(schema_wrangles, obj, path):
Recursively loop through all non-hidden function and
look for appropriately formatted docstrings
"""
- non_hidden_methods = [conn for conn in dir(obj) if not conn.startswith('_')]
-
- if len(non_hidden_methods) > 0:
- for method in non_hidden_methods:
- # Prevent including methods twice that are referenced at the root
- if method not in ('main', 'pandas'):
- getMethodDocs(schema_wrangles, getattr(obj, method), '.'.join([path, method]))
- else:
+ # A wrangle can be both callable itself and a namespace for dotted child
+ # wrangles (for example standardize, standardize.custom, and
+ # standardize.clean). Capture the callable's own schema before walking its
+ # children so the legacy root entry remains available.
+ if callable(obj):
try:
schema_wrangle = yaml.safe_load(obj.__doc__)
- if 'type' in schema_wrangle.keys() or 'anyOf' in schema_wrangle.keys():
+ if (
+ isinstance(schema_wrangle, dict)
+ and ('type' in schema_wrangle or 'anyOf' in schema_wrangle)
+ ):
schema_wrangles[
reserved_word_replacements.get(path[1:], path[1:])
] = schema_wrangle
except Exception as e:
logging.warning(f'{obj} description={e}')
+ non_hidden_methods = [conn for conn in dir(obj) if not conn.startswith('_')]
+ for method in non_hidden_methods:
+ # Prevent including methods twice that are referenced at the root
+ if method not in ('main', 'pandas'):
+ child = getattr(obj, method)
+ if child is not obj:
+ getMethodDocs(schema_wrangles, child, '.'.join([path, method]))
+
getMethodDocs(schema['wrangles'], wrangles.recipe._recipe_wrangles, '')
diff --git a/tests/recipes/wrangles/test_standardize.py b/tests/recipes/wrangles/test_standardize.py
new file mode 100644
index 000000000..db02120b6
--- /dev/null
+++ b/tests/recipes/wrangles/test_standardize.py
@@ -0,0 +1,349 @@
+import importlib
+import logging
+
+import pandas as pd
+import pytest
+import yaml
+
+import wrangles
+
+
+class TestStandardizeNamespace:
+ def test_core_legacy_function_remains_callable_namespace(self, monkeypatch):
+ standardize_module = importlib.import_module('wrangles.standardize')
+ calls = []
+
+ def fake_standardize(input, model_id, case_sensitive=False, **kwargs):
+ calls.append((input, model_id, case_sensitive, kwargs))
+ return 'standardized'
+
+ monkeypatch.setattr(
+ standardize_module,
+ 'standardize',
+ fake_standardize
+ )
+
+ result = wrangles.standardize.custom(
+ 'value',
+ '12345678-1234-1234',
+ case_sensitive=True,
+ custom_option='kept'
+ )
+
+ assert result == 'standardized'
+ assert calls == [(
+ 'value',
+ '12345678-1234-1234',
+ True,
+ {'custom_option': 'kept'}
+ )]
+
+ def test_legacy_and_custom_recipe_names_share_implementation(self, monkeypatch):
+ recipe_main = importlib.import_module('wrangles.recipe_wrangles.main')
+ calls = []
+
+ def fake_standardize(values, model_id, case_sensitive=False, **kwargs):
+ calls.append((values, model_id, case_sensitive, kwargs))
+ return [f'clean:{value}' for value in values]
+
+ monkeypatch.setattr(recipe_main, '_standardize', fake_standardize)
+ source = pd.DataFrame({'raw': ['one', 'two']})
+
+ legacy = wrangles.recipe.run(
+ {
+ 'wrangles': [{
+ 'standardize': {
+ 'input': 'raw',
+ 'output': 'result',
+ 'model_id': '12345678-1234-1234',
+ 'case_sensitive': True,
+ 'custom_option': 'kept'
+ }
+ }]
+ },
+ dataframe=source.copy()
+ )
+ explicit = wrangles.recipe.run(
+ {
+ 'wrangles': [{
+ 'standardize.custom': {
+ 'input': 'raw',
+ 'output': 'result',
+ 'model_id': '12345678-1234-1234',
+ 'case_sensitive': True,
+ 'custom_option': 'kept'
+ }
+ }]
+ },
+ dataframe=source.copy()
+ )
+
+ assert legacy.equals(explicit)
+ assert legacy['result'].to_list() == ['clean:one', 'clean:two']
+ assert calls == [
+ (
+ ['one', 'two'],
+ '12345678-1234-1234',
+ True,
+ {'custom_option': 'kept'}
+ ),
+ (
+ ['one', 'two'],
+ '12345678-1234-1234',
+ True,
+ {'custom_option': 'kept'}
+ )
+ ]
+
+ def test_recipe_namespace_exposes_parseable_schemas(self):
+ namespace = wrangles.recipe._recipe_wrangles.standardize
+
+ assert callable(namespace)
+ assert callable(namespace.custom)
+ assert callable(namespace.clean)
+ for function in (namespace, namespace.custom, namespace.clean):
+ schema = yaml.safe_load(function.__doc__)
+ assert schema['type'] == 'object'
+
+ def test_dataframe_accessor_preserves_legacy_call_and_dotted_names(
+ self,
+ monkeypatch
+ ):
+ recipe_main = importlib.import_module('wrangles.recipe_wrangles.main')
+
+ monkeypatch.setattr(
+ recipe_main,
+ '_standardize',
+ lambda values, model_id, case_sensitive=False, **kwargs: [
+ f'model:{value}' for value in values
+ ]
+ )
+ source = wrangles.DataFrame({'raw': [' one ']})
+
+ legacy = source.wrangles.standardize(
+ input='raw',
+ output='result',
+ model_id='12345678-1234-1234'
+ )
+ explicit = source.wrangles.standardize.custom(
+ input='raw',
+ output='result',
+ model_id='12345678-1234-1234'
+ )
+ cleaned = source.wrangles.standardize.clean(
+ input='raw',
+ output='result'
+ )
+
+ assert legacy['result'].to_list() == ['model: one ']
+ assert explicit.equals(legacy)
+ assert cleaned['result'].to_list() == ['one']
+ assert source.columns.to_list() == ['raw']
+
+
+class TestStandardizeCleanCore:
+ def test_scalar_and_list_shapes_are_preserved(self):
+ assert wrangles.standardize.clean('') == ''
+ assert wrangles.standardize.clean(' clean ') == 'clean'
+ assert wrangles.standardize.clean([' one ', 'two', 3, None]) == [
+ 'one',
+ 'two',
+ 3,
+ None
+ ]
+ assert wrangles.standardize.clean([]) == []
+
+ def test_repairs_mojibake_and_html_character_references(self):
+ assert wrangles.standardize.clean('Français') == 'Français'
+ assert wrangles.standardize.clean('é') == 'é'
+ assert wrangles.standardize.clean('AT&T ') == 'AT&T'
+ assert (
+ wrangles.standardize.clean('AT&T')
+ == 'AT&T'
+ )
+
+ def test_normalization_controls_and_advanced_ftfy_kwargs(self):
+ assert wrangles.standardize.clean('e\u0301') == 'é'
+ assert wrangles.standardize.clean('①', normalization='NFC') == '①'
+ assert wrangles.standardize.clean('①', normalization='NFKC') == '1'
+ assert wrangles.standardize.clean('ffi') == 'ffi'
+ assert (
+ wrangles.standardize.clean('ffi', fix_latin_ligatures=False)
+ == 'ffi'
+ )
+ assert wrangles.standardize.clean('Ａ') == 'A'
+ assert (
+ wrangles.standardize.clean('Ａ', fix_character_width=False)
+ == 'Ａ'
+ )
+
+ def test_controls_quotes_surrogates_and_idempotence(self):
+ dirty = '\x00\x81“Hello”\x7f\ud800'
+ cleaned = wrangles.standardize.clean(dirty)
+
+ assert cleaned == '"Hello"�'
+ assert wrangles.standardize.clean(cleaned) == cleaned
+ assert wrangles.standardize.clean('\x1b[31mred\x1b[0m') == 'red'
+ assert (
+ wrangles.standardize.clean('“Hello”', uncurl_quotes=False)
+ == '“Hello”'
+ )
+
+ def test_whitespace_modes(self):
+ dirty = ' one\u00a0\t two\r\n three \n\n four '
+
+ assert wrangles.standardize.clean(dirty) == 'one two three four'
+ assert wrangles.standardize.clean(
+ dirty,
+ preserve_line_breaks=True
+ ) == 'one two\nthree\n\nfour'
+ assert wrangles.standardize.clean(
+ ' one two ',
+ trim=False
+ ) == ' one two '
+ assert wrangles.standardize.clean(
+ ' one two ',
+ collapse_whitespace=False,
+ trim=False
+ ) == ' one two '
+
+ def test_invalid_inputs_and_kwargs_fail_clearly(self):
+ with pytest.raises(TypeError, match='string or a list'):
+ wrangles.standardize.clean({'value': 'text'})
+
+ with pytest.raises(TypeError, match='unexpected field names'):
+ wrangles.standardize.clean('text', unknown_ftfy_option=True)
+
+
+class TestStandardizeCleanRecipe:
+ def test_scalar_output_and_overwrite_in_place(self):
+ source = pd.DataFrame({
+ 'raw': [' Français ', ' already clean ']
+ })
+
+ output = wrangles.recipe.run(
+ {
+ 'wrangles': [{
+ 'standardize.clean': {
+ 'input': 'raw',
+ 'output': 'clean'
+ }
+ }]
+ },
+ dataframe=source.copy()
+ )
+ overwritten = wrangles.recipe.run(
+ {
+ 'wrangles': [{
+ 'standardize.clean': {'input': 'raw'}
+ }]
+ },
+ dataframe=source.copy()
+ )
+
+ assert output['clean'].to_list() == ['Français', 'already clean']
+ assert overwritten['raw'].to_list() == ['Français', 'already clean']
+
+ def test_multiple_inputs_map_to_equal_outputs_and_warn_once(self, caplog):
+ wrapper = wrangles.recipe._recipe_wrangles.standardize.clean
+ source = pd.DataFrame({
+ 'first': [' café ', 7],
+ 'second': [None, ' Ａ ']
+ })
+
+ with caplog.at_level(logging.WARNING):
+ result = wrapper(
+ source,
+ input=['first', 'second'],
+ output=['first clean', 'second clean']
+ )
+
+ assert result['first clean'].to_list() == ['café', 7]
+ assert result['second clean'].to_list() == [None, 'A']
+ assert sum(
+ 'preserved non-string values' in message
+ for message in caplog.messages
+ ) == 1
+
+ def test_multiple_inputs_concatenate_to_one_output(self):
+ result = wrangles.recipe.run(
+ {
+ 'wrangles': [{
+ 'standardize.clean': {
+ 'input': ['first', 'second'],
+ 'output': 'clean',
+ 'separator': ' | '
+ }
+ }]
+ },
+ dataframe=pd.DataFrame({
+ 'first': [' A ', None, pd.NA],
+ 'second': [2, ' B ', ' C ']
+ })
+ )
+
+ assert result['clean'].to_list() == ['A | 2', 'B', 'C']
+
+ def test_mismatched_multiple_outputs_raise(self):
+ wrapper = wrangles.recipe._recipe_wrangles.standardize.clean
+ source = pd.DataFrame({'one': ['a'], 'two': ['b']})
+
+ with pytest.raises(ValueError, match='same number of columns'):
+ wrapper(
+ source,
+ input=['one', 'two'],
+ output=['one clean', 'two clean', 'extra']
+ )
+
+ def test_wildcard_inputs_follow_concatenation_rule(self):
+ result = wrangles.recipe.run(
+ """
+ wrangles:
+ - standardize.clean:
+ input: Raw *
+ output: Clean
+ separator: " / "
+ """,
+ dataframe=pd.DataFrame({
+ 'Raw A': [' one '],
+ 'Raw B': [' two '],
+ 'Other': ['kept']
+ })
+ )
+
+ assert result.iloc[0].to_dict() == {
+ 'Raw A': ' one ',
+ 'Raw B': ' two ',
+ 'Other': 'kept',
+ 'Clean': 'one / two'
+ }
+
+ def test_where_and_empty_dataframe_behavior(self):
+ filtered = wrangles.recipe.run(
+ """
+ wrangles:
+ - standardize.clean:
+ input: Raw
+ output: Clean
+ where: Apply = true
+ """,
+ dataframe=pd.DataFrame({
+ 'Raw': [' one ', ' two '],
+ 'Apply': [True, False]
+ })
+ )
+ empty = wrangles.recipe.run(
+ {
+ 'wrangles': [{
+ 'standardize.clean': {
+ 'input': 'Raw',
+ 'output': 'Clean'
+ }
+ }]
+ },
+ dataframe=pd.DataFrame({'Raw': []})
+ )
+
+ assert filtered['Clean'].to_list() == ['one', '']
+ assert empty.empty
+ assert empty.columns.to_list() == ['Raw', 'Clean']
diff --git a/wrangles/dataframe.py b/wrangles/dataframe.py
index e4cba92d6..0e181a3de 100644
--- a/wrangles/dataframe.py
+++ b/wrangles/dataframe.py
@@ -41,6 +41,18 @@ def method(self, *args, **kwargs):
# **kwargs
# )[output]
+
+class _callable_wrangles_accessor(_wrangles_accessor):
+ """Expose a wrangle that is both callable and a dotted namespace."""
+
+ def __init__(self, df, wrangle):
+ self._wrangle = wrangle
+ super().__init__(df, wrangle)
+
+ def __call__(self, *args, **kwargs):
+ return self._wrangle(self._df.copy(), *args, **kwargs)
+
+
class _wrangles:
"""
A class to hold wrangles-related methods and properties.
@@ -61,7 +73,21 @@ def method(self, *args, **kwargs):
method.__doc__ = target_func.__doc__
return method
- setattr(self, name, make_method(name).__get__(self))
+ target = getattr(_recipe_wrangles.main, name)
+ child_wrangles = [
+ child
+ for child in dir(target)
+ if not child.startswith('_')
+ and callable(getattr(target, child))
+ ]
+ if child_wrangles:
+ setattr(
+ self,
+ name,
+ _callable_wrangles_accessor(self._df, target)
+ )
+ else:
+ setattr(self, name, make_method(name).__get__(self))
@property
def compare(self):
diff --git a/wrangles/recipe_wrangles/__init__.py b/wrangles/recipe_wrangles/__init__.py
index 0d9e1908e..46f1e5e25 100644
--- a/wrangles/recipe_wrangles/__init__.py
+++ b/wrangles/recipe_wrangles/__init__.py
@@ -15,6 +15,8 @@
- convert.case:
"""
+from .standardize import clean as _standardize_clean
+from .standardize import custom as _standardize_custom
from .main import *
from .pandas import *
from . import convert
@@ -28,3 +30,9 @@
from . import generate
from . import compute
from . import search
+
+
+# The legacy recipe function stays callable while also exposing the new dotted
+# standardize namespace to recipe resolution and schema discovery.
+standardize.clean = _standardize_clean
+standardize.custom = _standardize_custom
diff --git a/wrangles/recipe_wrangles/standardize.py b/wrangles/recipe_wrangles/standardize.py
new file mode 100644
index 000000000..feedacc1d
--- /dev/null
+++ b/wrangles/recipe_wrangles/standardize.py
@@ -0,0 +1,224 @@
+"""Dataframe wrappers for standardization wrangles."""
+
+import logging as _logging
+from typing import Union as _Union
+
+import pandas as _pd
+
+from ..standardize import clean as _clean
+
+
+def _is_missing(value) -> bool:
+ """Return whether a scalar value should be skipped during concatenation."""
+ try:
+ result = _pd.isna(value)
+ except (TypeError, ValueError):
+ return False
+
+ # pandas returns an array for list-like cells; those are values, not a
+ # scalar missing marker.
+ if hasattr(result, '__len__'):
+ return False
+ return bool(result)
+
+
+def _concatenate_row(values, separator: str) -> str:
+ """Join non-missing row values as text."""
+ return separator.join(
+ str(value)
+ for value in values
+ if (
+ not _is_missing(value)
+ and (not isinstance(value, str) or value != '')
+ )
+ )
+
+
+def clean(
+ df: _pd.DataFrame,
+ input: _Union[str, int, list],
+ output: _Union[str, int, list] = None,
+ fix_encoding: bool = True,
+ unescape_html: _Union[str, bool] = 'auto',
+ normalization: str = 'NFC',
+ fix_character_width: bool = True,
+ uncurl_quotes: bool = True,
+ remove_control_chars: bool = True,
+ collapse_whitespace: bool = True,
+ preserve_line_breaks: bool = False,
+ trim: bool = True,
+ separator: str = ' ',
+ **kwargs
+) -> _pd.DataFrame:
+ """
+ type: object
+ description: Repair common encoding, Unicode, HTML character reference, control character, and whitespace problems locally.
+ required:
+ - input
+ properties:
+ input:
+ type:
+ - string
+ - integer
+ - array
+ description: Name or list of input columns.
+ output:
+ type:
+ - string
+ - integer
+ - array
+ description: Name or list of output columns. Defaults to overwriting input.
+ fix_encoding:
+ type: boolean
+ default: true
+ description: Repair mojibake and other reversible encoding errors.
+ unescape_html:
+ anyOf:
+ - type: boolean
+ - type: string
+ enum:
+ - auto
+ default: auto
+ description: Decode HTML character references. Auto avoids decoding text that appears to contain HTML markup.
+ normalization:
+ type:
+ - string
+ - "null"
+ enum:
+ - NFC
+ - NFKC
+ - NFD
+ - NFKD
+ - null
+ default: NFC
+ description: Unicode normalization form.
+ fix_character_width:
+ type: boolean
+ default: true
+ description: Normalize fullwidth and halfwidth characters.
+ uncurl_quotes:
+ type: boolean
+ default: true
+ description: Replace typographic quotes with straight quotes.
+ remove_control_chars:
+ type: boolean
+ default: true
+ description: Remove C0 and C1 control characters.
+ collapse_whitespace:
+ type: boolean
+ default: true
+ description: Collapse runs of Unicode whitespace.
+ preserve_line_breaks:
+ type: boolean
+ default: false
+ description: Preserve line breaks while collapsing other whitespace.
+ trim:
+ type: boolean
+ default: true
+ description: Remove leading and trailing whitespace.
+ separator:
+ type: string
+ default: " "
+ description: Text used to join multiple input columns into one output.
+ """
+ if output is None:
+ output = input
+
+ if not isinstance(input, list):
+ input = [input]
+ if not isinstance(output, list):
+ output = [output]
+
+ if not isinstance(separator, str):
+ raise TypeError('separator must be a string.')
+
+ clean_options = {
+ 'fix_encoding': fix_encoding,
+ 'unescape_html': unescape_html,
+ 'normalization': normalization,
+ 'fix_character_width': fix_character_width,
+ 'uncurl_quotes': uncurl_quotes,
+ 'remove_control_chars': remove_control_chars,
+ 'collapse_whitespace': collapse_whitespace,
+ 'preserve_line_breaks': preserve_line_breaks,
+ 'trim': trim,
+ **kwargs
+ }
+
+ if len(input) > 1 and len(output) == 1:
+ combined = [
+ _concatenate_row(values, separator)
+ for values in df[input].itertuples(index=False, name=None)
+ ]
+ df[output[0]] = _clean(combined, **clean_options)
+ return df
+
+ if len(input) != len(output):
+ raise ValueError(
+ 'standardize.clean must output to a single column or the same '
+ 'number of columns as input.'
+ )
+
+ warned_for_non_strings = False
+ for input_column, output_column in zip(input, output):
+ values = df[input_column].tolist()
+ if not warned_for_non_strings and any(
+ not isinstance(value, str)
+ for value in values
+ ):
+ _logging.warning(
+ ': standardize.clean preserved non-string values in mapped input columns.'
+ )
+ warned_for_non_strings = True
+
+ df[output_column] = _clean(values, **clean_options)
+
+ return df
+
+
+def custom(
+ df: _pd.DataFrame,
+ input: _Union[str, int, list],
+ model_id: _Union[str, list],
+ output: _Union[str, list] = None,
+ case_sensitive: bool = False,
+ **kwargs
+) -> _pd.DataFrame:
+ """
+ type: object
+ description: Standardize data using a DIY or bespoke standardization wrangle. Requires WrangleWorks Account and Subscription.
+ required:
+ - input
+ properties:
+ input:
+ type:
+ - string
+ - integer
+ - array
+ description: Name or list of input columns.
+ output:
+ type:
+ - string
+ - array
+ description: Name or list of output columns
+ model_id:
+ type:
+ - string
+ - array
+ description: The ID of the wrangle to use (do not include 'find' and 'replace')
+ case_sensitive:
+ type: boolean
+ description: Allows the wrangle to be case sensitive if set to True, default is False.
+ """
+ # Import lazily to avoid a package initialization cycle. Delegating keeps
+ # the existing recipe implementation as the single compatibility path.
+ from .main import standardize as _legacy_standardize
+
+ return _legacy_standardize(
+ df=df,
+ input=input,
+ model_id=model_id,
+ output=output,
+ case_sensitive=case_sensitive,
+ **kwargs
+ )
diff --git a/wrangles/standardize.py b/wrangles/standardize.py
index 37447c578..9da3dbf7f 100644
--- a/wrangles/standardize.py
+++ b/wrangles/standardize.py
@@ -1,5 +1,8 @@
from typing import Union as _Union
import logging as _logging
+import re as _re
+import unicodedata as _unicodedata
+from ftfy import fix_text as _fix_text
from . import config as _config
from . import data as _data
from . import batching as _batching
@@ -59,3 +62,119 @@ def standardize(
if isinstance(input, str): results = results[0]
return results
+
+
+def clean(
+ input: _Union[str, list],
+ fix_encoding: bool = True,
+ unescape_html: _Union[str, bool] = 'auto',
+ normalization: str = 'NFC',
+ fix_character_width: bool = True,
+ uncurl_quotes: bool = True,
+ remove_control_chars: bool = True,
+ collapse_whitespace: bool = True,
+ preserve_line_breaks: bool = False,
+ trim: bool = True,
+ **kwargs
+) -> _Union[str, list]:
+ """
+ Repair common Unicode and encoding problems, then normalize whitespace.
+
+ Strings are cleaned directly. Lists retain their length and order, and
+ non-string list elements are returned unchanged so dataframe wrappers can
+ preserve mixed-type cells.
+
+ :param input: A string or list of values to clean.
+ :param fix_encoding: Repair mojibake and other reversible encoding errors.
+ :param unescape_html: Decode HTML character references. ``'auto'`` avoids
+ decoding references in text that appears to contain HTML markup.
+ :param normalization: Unicode normalization form, such as NFC or NFKC.
+ :param fix_character_width: Normalize fullwidth and halfwidth characters.
+ :param uncurl_quotes: Replace typographic quotes with straight quotes.
+ :param remove_control_chars: Remove C0 and C1 control characters.
+ :param collapse_whitespace: Collapse runs of Unicode whitespace.
+ :param preserve_line_breaks: Preserve line breaks while collapsing other
+ whitespace.
+ :param trim: Remove leading and trailing whitespace.
+ :param kwargs: Additional options forwarded to ``ftfy.fix_text``.
+ :return: A cleaned string or shape-preserving list.
+ """
+ if isinstance(input, str):
+ values = [input]
+ scalar_input = True
+ elif isinstance(input, list):
+ values = input
+ scalar_input = False
+ else:
+ raise TypeError(
+ 'Invalid input data provided. The input must be either a string or a list.'
+ )
+
+ results = []
+ for value in values:
+ if not isinstance(value, str):
+ results.append(value)
+ continue
+
+ cleaned = _fix_text(
+ value,
+ fix_encoding=fix_encoding,
+ unescape_html=unescape_html,
+ normalization=normalization,
+ fix_character_width=fix_character_width,
+ uncurl_quotes=uncurl_quotes,
+ remove_control_chars=remove_control_chars,
+ **kwargs
+ )
+
+ if remove_control_chars:
+ cleaned = ''.join(
+ char
+ for char in cleaned
+ if char in '\t\r\n' or _unicodedata.category(char) != 'Cc'
+ )
+
+ if collapse_whitespace:
+ if preserve_line_breaks:
+ cleaned = _re.sub(r'[^\S\r\n]+', ' ', cleaned)
+ cleaned = _re.sub(r' *(\r\n|\r|\n) *', r'\1', cleaned)
+ else:
+ cleaned = _re.sub(r'\s+', ' ', cleaned)
+
+ if trim:
+ cleaned = cleaned.strip()
+
+ results.append(cleaned)
+
+ return results[0] if scalar_input else results
+
+
+def custom(
+ input: _Union[str, list],
+ model_id: str,
+ case_sensitive: bool = False,
+ **kwargs
+) -> _Union[str, list]:
+ """
+ Explicit name for the model-backed standardization wrangle.
+
+ This delegates to :func:`standardize` so existing Python callers and the
+ new ``standardize.custom`` entry point share exactly the same behavior.
+
+ :param input: A string or list of strings to be standardized.
+ :param model_id: The model to be used.
+ :param case_sensitive: Allows setting the model to be case sensitive.
+ :return: A string or list with the updated text.
+ """
+ return standardize(
+ input=input,
+ model_id=model_id,
+ case_sensitive=case_sensitive,
+ **kwargs
+ )
+
+
+# ``standardize`` remains callable for backwards compatibility while also
+# acting as the namespace used by dotted core calls.
+standardize.clean = clean
+standardize.custom = custom
From 6fb331f01be4aaa27c0680c4e038a9ca5258fcf3 Mon Sep 17 00:00:00 2001
From: Eric Hills <53243273+ebhills@users.noreply.github.com>
Date: Sat, 15 Aug 2026 11:00:24 -0500
Subject: [PATCH 2/3] Address output column name overlap with input names
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
---
wrangles/recipe_wrangles/standardize.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/wrangles/recipe_wrangles/standardize.py b/wrangles/recipe_wrangles/standardize.py
index feedacc1d..353ad8f07 100644
--- a/wrangles/recipe_wrangles/standardize.py
+++ b/wrangles/recipe_wrangles/standardize.py
@@ -160,8 +160,8 @@ def clean(
)
warned_for_non_strings = False
- for input_column, output_column in zip(input, output):
- values = df[input_column].tolist()
+ source_values = [df[input_column].tolist() for input_column in input]
+ for values, output_column in zip(source_values, output):
if not warned_for_non_strings and any(
not isinstance(value, str)
for value in values
From 6a2a118c06f2dfcc4170dd5e66d49ff2b41287c6 Mon Sep 17 00:00:00 2001
From: Eric Hills <53243273+ebhills@users.noreply.github.com>
Date: Sat, 15 Aug 2026 12:11:08 -0500
Subject: [PATCH 3/3] Normalize invalid ftfy kwargs errors
---
tests/recipes/wrangles/test_standardize.py | 10 +++++++++-
wrangles/standardize.py | 10 ++++++++++
2 files changed, 19 insertions(+), 1 deletion(-)
diff --git a/tests/recipes/wrangles/test_standardize.py b/tests/recipes/wrangles/test_standardize.py
index db02120b6..0a565868f 100644
--- a/tests/recipes/wrangles/test_standardize.py
+++ b/tests/recipes/wrangles/test_standardize.py
@@ -207,10 +207,18 @@ def test_whitespace_modes(self):
trim=False
) == ' one two '
- def test_invalid_inputs_and_kwargs_fail_clearly(self):
+ def test_invalid_inputs_and_kwargs_fail_clearly(self, monkeypatch):
with pytest.raises(TypeError, match='string or a list'):
wrangles.standardize.clean({'value': 'text'})
+ standardize_module = importlib.import_module('wrangles.standardize')
+ monkeypatch.setattr(
+ standardize_module,
+ '_fix_text',
+ lambda *args, **kwargs: pytest.fail(
+ 'unknown kwargs must be rejected before calling ftfy'
+ )
+ )
with pytest.raises(TypeError, match='unexpected field names'):
wrangles.standardize.clean('text', unknown_ftfy_option=True)
diff --git a/wrangles/standardize.py b/wrangles/standardize.py
index 9da3dbf7f..913b40f28 100644
--- a/wrangles/standardize.py
+++ b/wrangles/standardize.py
@@ -2,12 +2,18 @@
import logging as _logging
import re as _re
import unicodedata as _unicodedata
+from ftfy import TextFixerConfig as _TextFixerConfig
from ftfy import fix_text as _fix_text
from . import config as _config
from . import data as _data
from . import batching as _batching
+_SUPPORTED_FTFY_KWARGS = frozenset(_TextFixerConfig._fields) | {
+ 'fix_entities'
+}
+
+
def standardize(
input: _Union[str, list],
model_id: str,
@@ -99,6 +105,10 @@ def clean(
:param kwargs: Additional options forwarded to ``ftfy.fix_text``.
:return: A cleaned string or shape-preserving list.
"""
+ unexpected_kwargs = sorted(set(kwargs) - _SUPPORTED_FTFY_KWARGS)
+ if unexpected_kwargs:
+ raise TypeError(f'Got unexpected field names: {unexpected_kwargs}')
+
if isinstance(input, str):
values = [input]
scalar_input = True