Add integrated text cleaning and standardize namespace

README.md

@@ -127,6 +127,41 @@ Wrangles broadly accept a single input string, or a list of strings. If a list i
 ]
 ```
 
+#### Integrated text cleaning
+
+`standardize.clean` repairs common mojibake, HTML character references,
+Unicode inconsistencies, control characters, and whitespace locally. It accepts
+a string or list and preserves that shape:
+
+```python
+>>> wrangles.standardize.clean(["Français", " AT&T "])
+['Français', 'AT&T']
+```
+
+In a recipe, one input maps to one output and an omitted output overwrites the
+input. Multiple inputs map positionally to equally many outputs, or concatenate
+row values when a single output is given. Wildcard-expanded inputs follow the
+same rules.
+
+```yaml
+wrangles:
+  - standardize.clean:
+      input: Description *
+      output: Clean Description
+      separator: " | "
+      normalization: NFKC
+      preserve_line_breaks: true
+```
+
+Common options include `fix_encoding`, `unescape_html`, `normalization`,
+`fix_character_width`, `uncurl_quotes`, `remove_control_chars`,
+`collapse_whitespace`, `preserve_line_breaks`, and `trim`. Less-common
+`ftfy.fix_text` options are forwarded by name. This wrangle does not detect raw
+byte encodings or remove HTML tags.
+
+The existing model-backed `standardize` name remains supported and is also
+available explicitly as `standardize.custom`.
+
 ### Recipes
 
 Recipes are written in YAML and allow a series of Wrangles to be run as an automated sequence.

requirements.txt

@@ -14,6 +14,9 @@ lorem
 pydantic>=2.12.0
 pyyaml
 
+# Text cleanup
+ftfy>=6.3.1,<7
+
 # API & auth
 requests
 PyJWT

schema/generate_recipe_schema.py

@@ -136,23 +136,31 @@ def getMethodDocs(schema_wrangles, obj, path):
     Recursively loop through all non-hidden function and
     look for appropriately formatted docstrings
     """
-    non_hidden_methods = [conn for conn in dir(obj) if not conn.startswith('_')]
-
-    if len(non_hidden_methods) > 0:
-        for method in non_hidden_methods:
-            # Prevent including methods twice that are referenced at the root
-            if method not in ('main', 'pandas'):
-                getMethodDocs(schema_wrangles, getattr(obj, method), '.'.join([path, method]))
-    else:
+    # A wrangle can be both callable itself and a namespace for dotted child
+    # wrangles (for example standardize, standardize.custom, and
+    # standardize.clean). Capture the callable's own schema before walking its
+    # children so the legacy root entry remains available.
+    if callable(obj):
         try:
             schema_wrangle = yaml.safe_load(obj.__doc__)
-            if 'type' in schema_wrangle.keys() or 'anyOf' in schema_wrangle.keys():
+            if (
+                isinstance(schema_wrangle, dict)
+                and ('type' in schema_wrangle or 'anyOf' in schema_wrangle)
+            ):
                 schema_wrangles[
                     reserved_word_replacements.get(path[1:], path[1:])
                 ] = schema_wrangle
         except Exception as e:
             logging.warning(f'{obj} description={e}')
 
+    non_hidden_methods = [conn for conn in dir(obj) if not conn.startswith('_')]
+    for method in non_hidden_methods:
+        # Prevent including methods twice that are referenced at the root
+        if method not in ('main', 'pandas'):
+            child = getattr(obj, method)
+            if child is not obj:
+                getMethodDocs(schema_wrangles, child, '.'.join([path, method]))
+
 getMethodDocs(schema['wrangles'], wrangles.recipe._recipe_wrangles, '')