From de1565610fe0e72d7d39ba0a09146110edc6efda Mon Sep 17 00:00:00 2001 From: Scott Kleinman Date: Mon, 8 Jun 2020 20:16:35 -0700 Subject: [PATCH 1/4] Replace consolidation with regex pattern replacement --- lexos/helpers/constants.py | 11 +-- lexos/helpers/error_messages.py | 2 + lexos/processors/prepare/scrubber.py | 109 ++++++++++++++++++++------- lexos/static/help/scrub-help.html | 52 +++++++------ lexos/static/js/scrub.js | 18 ++--- lexos/templates/scrub.html | 28 +++---- 6 files changed, 142 insertions(+), 78 deletions(-) diff --git a/lexos/helpers/constants.py b/lexos/helpers/constants.py index 706919f28..b58e8989d 100644 --- a/lexos/helpers/constants.py +++ b/lexos/helpers/constants.py @@ -31,8 +31,9 @@ MUFI_3_FILENAME = "MUFI_3_DICT.tsv" MUFI_4_FILENAME = "MUFI_4_DICT.tsv" STOPWORD_FILENAME = "stopwords.p" +# CONSOLIDATION_FILENAME = "consolidations.p" +PATTERN_REPLACEMENTS_FILENAME = "pattern_replacements.p" LEMMA_FILENAME = "lemmas.p" -CONSOLIDATION_FILENAME = "consolidations.p" SPECIAL_CHAR_FILENAME = "specialchars.p" DIGIT_MAP_FILENAME = "digitmap.p" PUNCTUATION_MAP_FILENAME = "punctuationmap.p" @@ -84,14 +85,14 @@ SCRUBINPUTS = ( "stop_words", "special_characters", - "consolidations", + "pattern_replacements", "lemmas" ) OPTUPLOADNAMES = ( "stop_words_file[]", "lemmas_file[]", - "consolidations_file[]", + "pattern_replacements_file[]", "special_characters_file[]" ) @@ -209,13 +210,13 @@ "stop_words": "", "stop_words_method": "Off", "special_characters": "", - "consolidations": "", + "pattern_replacements": "", "lemmas": "", "special_characters_preset": "None", "file_uploads": { "stop_words_file[]": "", "lemmas_file[]": "", - "consolidations_file[]": "", + "pattern_replacements[]": "", "special_characters_file[]": ""}} DEFAULT_CUT_OPTIONS = { diff --git a/lexos/helpers/error_messages.py b/lexos/helpers/error_messages.py index 731f530e2..78e9f9954 100644 --- a/lexos/helpers/error_messages.py +++ b/lexos/helpers/error_messages.py @@ -33,6 +33,8 @@ "Too many values on right side of replacement string." REPLACEMENT_NO_LEFT_HAND_MESSAGE = \ "Missing value on the left side of replacement string." +UNESCAPED_GREATER_THAN_SIGN_MESSAGE = \ + "Please place a backslash before all examples of `>` in the pattern you wish to replace." # ---------------------------------------------------------------------------- diff --git a/lexos/processors/prepare/scrubber.py b/lexos/processors/prepare/scrubber.py index 39ed55cae..b010a697f 100644 --- a/lexos/processors/prepare/scrubber.py +++ b/lexos/processors/prepare/scrubber.py @@ -12,7 +12,8 @@ from lexos.helpers import constants as constants, \ general_functions as general_functions from lexos.helpers.error_messages import NOT_ONE_REPLACEMENT_COLON_MESSAGE, \ - REPLACEMENT_RIGHT_OPERAND_MESSAGE, REPLACEMENT_NO_LEFT_HAND_MESSAGE + REPLACEMENT_RIGHT_OPERAND_MESSAGE, REPLACEMENT_NO_LEFT_HAND_MESSAGE, \ + UNESCAPED_GREATER_THAN_SIGN_MESSAGE from lexos.helpers.exceptions import LexosException @@ -186,6 +187,62 @@ def replacement_handler(text: str, return text +def pattern_replacement_handler(text: str, + replacer_string: str) -> str: + """Handles pattern replacement lines found in the scrub-alteration-upload files. + + :param text: A unicode string with the whole text to be altered. + :param replacer_string: A formatted string input with newline-separated + "replacement lines", where each line is formatted to replace the + majority of the words with one word. + :returns: The input string with replacements made. + """ + + # Convert HTML character entities to Unicode if HTML is selected *and* there + # are further entities entered in the form field + if request.form['special_characters_preset'] == 'HTML': + text = html.unescape(text) + + # Remove spaces in replacement string for consistent format, then split the + # individual replacements to be made + # Not sure if this is needed + no_space_replacer = replacer_string.translate({ord(" "): None}).strip('\n') + + # Handle excess blank lines in file, etc. + replacement_lines = [token for token in no_space_replacer.split('\n') + if token != ""] + + replacement_jobs = [] + # Search for all examples of > not preceded by a backslash + pat_for_sep = re.compile(r'(?') + for replacement_line in replacement_lines: + # There is more than one potential separator, raise an error + if len(re.findall(pat_for_sep, replacement_line)) > 1: + raise LexosException( + UNESCAPED_GREATER_THAN_SIGN_MESSAGE + replacement_line) + # Otherwise, define a replacement tuple + else: + # Remove whitespace around the separator and then split + replacement_line = re.sub(r'\s+>\s+', '>', replacement_line) + pattern, substitution = re.split(pat_for_sep, replacement_line) + print(substitution) + substitution = substitution.replace('\\>', '>') + # If the pattern has the prefix REGEX:, remove it and set regex=True + if pattern.lower().startswith('regex:'): + regex = True + pattern = re.sub(r'^REGEX:', '', pattern, flags=re.IGNORECASE) + else: + regex = False + replacement_jobs.append((regex, pattern, substitution)) + print(replacement_jobs) + # Do the replacement + if regex == True: + text = re.sub(pattern, substitution, text) + else: + text = text.replace(pattern, substitution) + return text + + def replace_with_dict(text: str, replacement_dict: Dict[str, str], edge1: str, edge2: str) -> str: """Alters text according to the replacements dictionary. @@ -807,9 +864,9 @@ def prepare_additional_options(opt_uploads: Dict[str, FileStorage], option text fields and files. """ - file_strings = {'consolidations_file[]': '', 'lemmas_file[]': '', + file_strings = {'pattern_replacements_file[]': '', 'lemmas_file[]': '', 'special_characters_file[]': '', 'stop_words_file[]': '', - 'consolidations': '', 'lemmas': '', + 'pattern_replacements': '', 'lemmas': '', 'special_characters': '', 'stop_words': ''} for index, key in enumerate(sorted(opt_uploads)): @@ -825,14 +882,14 @@ def prepare_additional_options(opt_uploads: Dict[str, FileStorage], file_strings[key] = "" # Create an array of option strings: - # cons_file_string, lem_file_string, sc_file_string, sw_kw_file_string, - # cons_manual, lem_manual, sc_manual, and sw_kw_manual + # pattern_replacements_file_string, lem_file_string, sc_file_string, sw_kw_file_string, + # pattern_replacements_manual, lem_manual, sc_manual, and sw_kw_manual - all_options = [file_strings.get('consolidations_file[]'), + all_options = [file_strings.get('pattern_replacements_file[]'), file_strings.get('lemmas_file[]'), file_strings.get('special_characters_file[]'), file_strings.get('stop_words_file[]'), - request.form['consolidations'], + request.form['pattern_replacements'], request.form['lemmas'], request.form['special_characters'], request.form['stop_words']] @@ -878,33 +935,33 @@ def scrub(text: str, gutenberg: bool, lower: bool, punct: bool, apos: bool, storage_filenames = sorted( [constants.STOPWORD_FILENAME, constants.LEMMA_FILENAME, - constants.CONSOLIDATION_FILENAME, constants.SPECIAL_CHAR_FILENAME]) + constants.PATTERN_REPLACEMENTS_FILENAME, constants.SPECIAL_CHAR_FILENAME]) option_strings = prepare_additional_options( opt_uploads, storage_options, storage_folder, storage_filenames) - # handle uploaded FILES: consolidations, lemmas, special characters, + # handle uploaded FILES: pattern_replacements, lemmas, special characters, # stop-keep words - cons_file_string = option_strings[0] + pattern_replacements_file_string = option_strings[0] lem_file_string = option_strings[1] sc_file_string = option_strings[2] sw_kw_file_string = option_strings[3] - # handle manual entries: consolidations, lemmas, special characters, + # handle manual entries: pattern_replacements, lemmas, special characters, # stop-keep words - cons_manual = option_strings[4] + pattern_replacements_manual = option_strings[4] lem_manual = option_strings[5] sc_manual = option_strings[6] sw_kw_manual = option_strings[7] # Scrubbing order: # - # Note: lemmas and consolidations do NOT work on tags; in short, + # Note: lemmas and pattern_replacements do NOT work on tags; in short, # these manipulations do not change inside any tags # # 0. Gutenberg # 1. lower # (not applied in tags ever; - # lemmas/consolidations/specialChars/stopKeepWords changed; + # lemmas/pattern_replacements/specialChars/stopKeepWords changed; # text not changed at this point) # 2. special characters # 3. tags - scrub tags @@ -914,7 +971,7 @@ def scrub(text: str, gutenberg: bool, lower: bool, punct: bool, apos: bool, # 5. digits (text not changed at this point, not applied in tags ever) # 6. white space (text not changed at this point, not applied in tags ever, # otherwise tag attributes will be messed up) - # 7. consolidations + # 7. pattern_replacements # (text not changed at this point, not applied in tags ever) # 8. lemmatize (text not changed at this point, not applied in tags ever) # 9. stop words/keep words @@ -923,7 +980,7 @@ def scrub(text: str, gutenberg: bool, lower: bool, punct: bool, apos: bool, # apply: # 0. remove Gutenberg boiler plate (if any) # 1. lowercase - # 2. consolidation + # 2. pattern_replacements # 3. lemmatize # 4. stop words # 5. remove punctuation, digits, and whitespace without changing all the @@ -950,13 +1007,13 @@ def to_lower_function(orig_text: str) -> str: # since lower is ON, apply lowercase to other options # apply to contents of any uploaded files - cons_file_string = cons_file_string.lower() + pattern_replacements_file_string = pattern_replacements_file_string.lower() lem_file_string = lem_file_string.lower() sc_file_string = sc_file_string.lower() sw_kw_file_string = sw_kw_file_string.lower() # apply to contents manually entered - cons_manual = cons_manual.lower() + pattern_replacements_manual = pattern_replacements_manual.lower() lem_manual = lem_manual.lower() sc_manual = sc_manual.lower() sw_kw_manual = sw_kw_manual.lower() @@ -1024,21 +1081,21 @@ def total_removal_function(orig_text: str) -> str: """ return orig_text.translate(total_removal_map) - # -- 7. consolidations --------------------------------------------------- - def consolidation_function(orig_text: str) -> str: + # -- 7. pattern_replacements --------------------------------------------------- + def pattern_replacements_function(orig_text: str) -> str: """Replaces characters according to user input strings. :param orig_text: A text string. - :return: The text with characters swapped according to cons_file_string - and cons_manual. + :return: The text with characters swapped according to pattern_replacements_file_string + and pattern_replacements_manual. """ replacer_string = handle_file_and_manual_strings( - file_string=cons_file_string, manual_string=cons_manual, + file_string=pattern_replacements_file_string, manual_string=pattern_replacements_manual, storage_folder=storage_folder, storage_filenames=storage_filenames, storage_number=0) - text = replacement_handler( - text=orig_text, replacer_string=replacer_string, is_lemma=False) + text = pattern_replacement_handler( + text=orig_text, replacer_string=replacer_string) return text # -- 8. lemmatize -------------------------------------------------------- @@ -1091,7 +1148,7 @@ def stop_keep_words_function(orig_text: str) -> str: # apply all the functions and exclude tag functions = [to_lower_function, - consolidation_function, + pattern_replacements_function, lemmatize_function, total_removal_function, stop_keep_words_function] diff --git a/lexos/static/help/scrub-help.html b/lexos/static/help/scrub-help.html index 762522dfa..96ccf82b9 100644 --- a/lexos/static/help/scrub-help.html +++ b/lexos/static/help/scrub-help.html @@ -75,7 +75,7 @@

Converts all uppercase characters to lowercase characters so that the tokens "The" and "the" will be considered as the same term. In addition, all contents (whether in uploaded files or entered manually) for the - Stop Words/Keep Words, Lemmas, Consolidations, or Special Characters + Stop Words/Keep Words, Pattern Replacements, Lemmas, or Special Characters options will also have all uppercase characters changed to lowercase. Lowercase is not applied inside any HTML, XML, or SGML markup tags remaining in the text. @@ -212,6 +212,34 @@

click here

+
  • Pattern Replacements
    • +

    + Replaces a list of character patterns with substitutions. This is + typically to consolidate symbols considered equivalent or to provide + highly specific cleaning functions. +

    +

    + For example, in Old English, the character "eth" ð is interchangeable + with the character "thorn" þ. The Pattern Replacements option allows you to + choose to merge the two using a single character. +

    +

    + Pattern replacements should be entered in the format: 
    ð > þ
    Where you wish to + change all occurrences of ð to þ. Multiple + consolidations can be separated by commas or line breaks. All replacements will be + applied in the order in which they are listed. +

    +

    + If you find that you need to replace the > character itself, place a backslash + before it like \>. +

    +

    + Pattern replacements can be entered manually in the provided form field or + uploaded from a file. Note that the "Make Lowercase" option will be applied + to your list of characters if that option is also selected. To replace + entire words (terms) with other words, you should use the Lemma option. +

    +
  • Lemmas

    • Replaces all instances of terms in a list with a common replacement @@ -234,28 +262,6 @@

    You can manually enter a list of lemmas or upload your own.

    -
  • Consolidations
    • -

    - Replaces a list of characters with a different character. This is - typically to consolidate symbols considered equivalent. -

    -

    - For example, in Old English, the character "eth" ð is interchangeable - with the character "thorn" þ. The Consolidations option allows you to - choose to merge the two using a single character. -

    -

    - Consolidations should be entered in the format: 
    ð: þ
    Where you wish to - change all occurrences of ð to þ . Multiple - consolidations can be separated by commas or line breaks. -

    -

    - Consolidations can be entered manually in the provided form field or - uploaded from a file. Note that the "Make Lowercase" option will be applied - to your list of characters if that option is also selected. To replace - entire words (terms) with other words, you should use the Lemma option. -

    -
  • Special Characters

    • Replaces character entities with their glyph equivalents. diff --git a/lexos/static/js/scrub.js b/lexos/static/js/scrub.js index 7a3d5bf2d..476d86ac4 100644 --- a/lexos/static/js/scrub.js +++ b/lexos/static/js/scrub.js @@ -55,7 +55,7 @@ $(function(){ }); // Initialize the upload buttons - initialize_upload_buttons(["lemmas", "consolidations", + initialize_upload_buttons(["lemmas", "patterns", "stop-words", "special-characters"]); // Initialize the tooltips @@ -255,8 +255,8 @@ function update_document_previews(response){ function initialize_tooltips(){ // "Scrub Tags" - create_tooltip("#scrub-tags-tooltip-button", `Handle tags such as - those used in XML, HTML, or SGML. Click the "Options" button + create_tooltip("#scrub-tags-tooltip-button", `Handle tags such as + those used in XML, HTML, or SGML. Click the "Options" button to the left to control how each tag will be handled.`); // "Keep Hyphens" @@ -285,13 +285,11 @@ function initialize_tooltips(){ For example, "cyng, kyng:king" will replace every occurrence of "cyng" and "kyng" with "king".`); - // "Consolidations" - create_tooltip("#consolidations-tooltip-button", `Upload or input a list - of consolidations (character replacements). Enter the characters you - want to replace separated by comma. Then, add a colon and follow it - with the replacement character. Enter each replacement on a separate - line. For example, "a, b:c" will replace every occurrence of "a" and - "b" with "c".`); + // "Pattern Replacement" + create_tooltip("#patterns-tooltip-button", `Upload or input a list + of character patterns to replace. Enter the characters you + want to replace separated from their replacement values by ">". + To use regex, prefix the pattern with "REGEX:"`); // "Stop and Keep Words" create_tooltip("#stop-words-tooltip-button", `Upload or input a list of diff --git a/lexos/templates/scrub.html b/lexos/templates/scrub.html index 44dca3e78..6e2e9da52 100644 --- a/lexos/templates/scrub.html +++ b/lexos/templates/scrub.html @@ -60,41 +60,41 @@

    Scrubbing Options

    - -
    + +
    -

    Lemmas

    +

    Pattern Replacement

    - Upload - ? + Upload + ?
    - +
    - +
    - -
    + +
    -

    Consolidations

    +

    Lemmas

    - Upload - ? + Upload + ?
    - +
    - +
    From 286d08eb211e6d05701d6fe0e59ab91ec5f5ebae Mon Sep 17 00:00:00 2001 From: Scott Kleinman Date: Mon, 8 Jun 2020 21:55:36 -0700 Subject: [PATCH 2/4] Add replaement with spaces --- lexos/processors/prepare/scrubber.py | 5 ++-- lexos/static/help/scrub-help.html | 42 ++++++++-------------------- 2 files changed, 15 insertions(+), 32 deletions(-) diff --git a/lexos/processors/prepare/scrubber.py b/lexos/processors/prepare/scrubber.py index b010a697f..7e068b4a6 100644 --- a/lexos/processors/prepare/scrubber.py +++ b/lexos/processors/prepare/scrubber.py @@ -225,8 +225,10 @@ def pattern_replacement_handler(text: str, # Remove whitespace around the separator and then split replacement_line = re.sub(r'\s+>\s+', '>', replacement_line) pattern, substitution = re.split(pat_for_sep, replacement_line) - print(substitution) + # Handle string internal greater than sign substitution = substitution.replace('\\>', '>') + # Convert \s token to a space + substitution = substitution.replace('\\s', ' ') # If the pattern has the prefix REGEX:, remove it and set regex=True if pattern.lower().startswith('regex:'): regex = True @@ -234,7 +236,6 @@ def pattern_replacement_handler(text: str, else: regex = False replacement_jobs.append((regex, pattern, substitution)) - print(replacement_jobs) # Do the replacement if regex == True: text = re.sub(pattern, substitution, text) diff --git a/lexos/static/help/scrub-help.html b/lexos/static/help/scrub-help.html index 96ccf82b9..8e2d8b127 100644 --- a/lexos/static/help/scrub-help.html +++ b/lexos/static/help/scrub-help.html @@ -231,9 +231,19 @@

    If you find that you need to replace the > character itself, place a backslash - before it like \>. + before it like \>. To replace a character with a space, use \s. For + instance, the pattern - > \s will replace hyphens with spaces.

    -

    +

    + Lexos accepts regular expression (regex) patterns if the pattern begins with REGEX:. + For example, REGEX:^c > k will change c to k only at the beginning + of the text. If you are using regex capture groups, you can reference them with \1, + \2, etc. For example, the pattern REGEX:(hi)-(ho) > \2-\1 will change + "hi-ho" to "ho-hi". A useful regular expressions tutorial can be found at + RegexOne. Regex patterns can also be + tested at Regex101. +

    +

    Pattern replacements can be entered manually in the provided form field or uploaded from a file. Note that the "Make Lowercase" option will be applied to your list of characters if that option is also selected. To replace @@ -293,31 +303,3 @@

    Multiple transformation rules should be listed on separate lines.

    - - -

    Replacing Patterns

    - -

    - Sometimes it is necessary to replace a pattern rather than a precise string. - For instance, if a document contains multiple URLs like - http://lexos.wheatoncollege.edu and - http://scalar.usc.edu/works/lexos/, and you need to strip - these URLs, a method is required for matching all URLs without knowing - what they are in advance. One technique is to apply regular expression - (regex) pattern matching. Lexos uses regular expressions internally to - perform some of its scrubbing options, but, as of version 3.0, it does - not provide a way for users to supply their own regular expression - patterns when scrubbing. If users need to strip or replace patterns by - regular expressions, it will be necessary to perform that action using - a separate script or tool prior to using Lexos. A useful - regular expressions tutorial can be found at - RegexOne. - Most modern text editors like - Sublime Text and - TextWrangler - accept regular expressions in their search and replace functions, and - users may find them to be a convenient means of performing actions with - regular expressions. We hope to add a regular expression pattern matching - to Lexos in the future. -

    - From 4303b47ac36d190cebd51dd4e7f217d6d868ed86 Mon Sep 17 00:00:00 2001 From: Scott Kleinman Date: Wed, 10 Jun 2020 13:51:35 -0700 Subject: [PATCH 3/4] Fix linting errors --- lexos/helpers/error_messages.py | 5 ++-- lexos/processors/prepare/scrubber.py | 35 ++++++++++++++++------------ 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/lexos/helpers/error_messages.py b/lexos/helpers/error_messages.py index 78e9f9954..b529a6115 100644 --- a/lexos/helpers/error_messages.py +++ b/lexos/helpers/error_messages.py @@ -33,8 +33,9 @@ "Too many values on right side of replacement string." REPLACEMENT_NO_LEFT_HAND_MESSAGE = \ "Missing value on the left side of replacement string." -UNESCAPED_GREATER_THAN_SIGN_MESSAGE = \ - "Please place a backslash before all examples of `>` in the pattern you wish to replace." +UNESCAPED_GREATER_THAN_SIGN_MESSAGE = "Please place a backslash before all" \ + " examples of `>` in the pattern you wish to" \ + " replace." # ---------------------------------------------------------------------------- diff --git a/lexos/processors/prepare/scrubber.py b/lexos/processors/prepare/scrubber.py index 41115e3d5..c341a2521 100644 --- a/lexos/processors/prepare/scrubber.py +++ b/lexos/processors/prepare/scrubber.py @@ -194,8 +194,8 @@ def replacement_handler(text: str, def pattern_replacement_handler(text: str, - replacer_string: str) -> str: - """Handles pattern replacement lines found in the scrub-alteration-upload files. + replacer_string: str) -> str: + """Handle pattern replacement lines found in scrub-alteration-upload files. :param text: A unicode string with the whole text to be altered. :param replacer_string: A formatted string input with newline-separated @@ -204,8 +204,8 @@ def pattern_replacement_handler(text: str, :returns: The input string with replacements made. """ - # Convert HTML character entities to Unicode if HTML is selected *and* there - # are further entities entered in the form field + # Convert HTML character entities to Unicode if HTML is selected *and* + # there are further entities entered in the form field if request.form['special_characters_preset'] == 'HTML': text = html.unescape(text) @@ -235,7 +235,8 @@ def pattern_replacement_handler(text: str, substitution = substitution.replace('\\>', '>') # Convert \s token to a space substitution = substitution.replace('\\s', ' ') - # If the pattern has the prefix REGEX:, remove it and set regex=True + # If the pattern has the prefix REGEX:, remove it, + # and set regex=True if pattern.lower().startswith('regex:'): regex = True pattern = re.sub(r'^REGEX:', '', pattern, flags=re.IGNORECASE) @@ -243,7 +244,7 @@ def pattern_replacement_handler(text: str, regex = False replacement_jobs.append((regex, pattern, substitution)) # Do the replacement - if regex == True: + if regex is True: text = re.sub(pattern, substitution, text) else: text = text.replace(pattern, substitution) @@ -889,8 +890,9 @@ def prepare_additional_options(opt_uploads: Dict[str, FileStorage], file_strings[key] = "" # Create an array of option strings: - # pattern_replacements_file_string, lem_file_string, sc_file_string, sw_kw_file_string, - # pattern_replacements_manual, lem_manual, sc_manual, and sw_kw_manual + # pat_replacements_file_string, lem_file_string, sc_file_string, + # sw_kw_file_string, pattern_replacements_manual, lem_manual, sc_manual, + # and sw_kw_manual all_options = [file_strings.get('pattern_replacements_file[]'), file_strings.get('lemmas_file[]'), @@ -942,13 +944,14 @@ def scrub(text: str, gutenberg: bool, lower: bool, punct: bool, apos: bool, storage_filenames = sorted( [constants.STOPWORD_FILENAME, constants.LEMMA_FILENAME, - constants.PATTERN_REPLACEMENTS_FILENAME, constants.SPECIAL_CHAR_FILENAME]) + constants.PATTERN_REPLACEMENTS_FILENAME, + constants.SPECIAL_CHAR_FILENAME]) option_strings = prepare_additional_options( opt_uploads, storage_options, storage_folder, storage_filenames) # handle uploaded FILES: pattern_replacements, lemmas, special characters, # stop-keep words - pattern_replacements_file_string = option_strings[0] + pat_replacements_file_string = option_strings[0] lem_file_string = option_strings[1] sc_file_string = option_strings[2] sw_kw_file_string = option_strings[3] @@ -1014,7 +1017,7 @@ def to_lower_function(orig_text: str) -> str: # since lower is ON, apply lowercase to other options # apply to contents of any uploaded files - pattern_replacements_file_string = pattern_replacements_file_string.lower() + pat_replacements_file_string = pat_replacements_file_string.lower() lem_file_string = lem_file_string.lower() sc_file_string = sc_file_string.lower() sw_kw_file_string = sw_kw_file_string.lower() @@ -1102,13 +1105,15 @@ def pattern_replacements_function(orig_text: str) -> str: """Replaces characters according to user input strings. :param orig_text: A text string. - :return: The text with characters swapped according to pattern_replacements_file_string - and pattern_replacements_manual. + :return: The text with characters swapped according to + pat_replacements_file_string and pattern_replacements_manual. """ replacer_string = handle_file_and_manual_strings( - file_string=pattern_replacements_file_string, manual_string=pattern_replacements_manual, - storage_folder=storage_folder, storage_filenames=storage_filenames, + file_string=pat_replacements_file_string, + manual_string=pattern_replacements_manual, + storage_folder=storage_folder, + storage_filenames=storage_filenames, storage_number=0) text = pattern_replacement_handler( text=orig_text, replacer_string=replacer_string) From 113aab7952422580cf4509634a1df282ca315ce3 Mon Sep 17 00:00:00 2001 From: Scott Kleinman Date: Wed, 10 Jun 2020 13:55:20 -0700 Subject: [PATCH 4/4] Fix overlong line --- lexos/processors/prepare/scrubber.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lexos/processors/prepare/scrubber.py b/lexos/processors/prepare/scrubber.py index c341a2521..596c2614a 100644 --- a/lexos/processors/prepare/scrubber.py +++ b/lexos/processors/prepare/scrubber.py @@ -1100,7 +1100,7 @@ def total_removal_function(orig_text: str) -> str: """ return orig_text.translate(total_removal_map) - # -- 7. pattern_replacements --------------------------------------------------- + # -- 7. pattern_replacements -------------------------------------------- def pattern_replacements_function(orig_text: str) -> str: """Replaces characters according to user input strings.