From 1ace745ab4a7b236ad1c2f6f3dce2ca2c66a2c17 Mon Sep 17 00:00:00 2001 From: Michael Wood Date: Sat, 6 Dec 2025 12:02:29 -0500 Subject: [PATCH 1/4] Wrap name (n in ${n/../..}) in B2P markup so python code runs --- fix_string.c | 46 +++++++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/fix_string.c b/fix_string.c index 944c94a..d108488 100755 --- a/fix_string.c +++ b/fix_string.c @@ -848,25 +848,24 @@ static char * emitVariable(char *vblNameP, _BOOL is_variable_name_braced, int in } break; case '/': // c = *start2P + //case '%': + //case '#': { char *name, *value, *lpatsub, *pattern, *repl, *replacement, *p; int delim, quoted=0; char rawflag; burpT bufferBackup, newBackup; - _BOOL is_global_replace = FALSE; + _BOOL is_operator_doubled = FALSE; // Extract the substitution parameters: name, pattern, replacement. // Much of the following code borrows from bash's parameter_brace_patsub (). memset(&bufferBackup, 0, sizeof(bufferBackup)); memset(&newBackup, 0, sizeof(newBackup)); - *start2P = '\0'; // temp edit - name = strdup(vblNameP); - *start2P = c; // revert edit value = start2P+1; - if (*value == '/') + if (*value == c) { - is_global_replace = TRUE; + is_operator_doubled = TRUE; value++; } lpatsub = strdup(value); @@ -883,19 +882,36 @@ static char * emitVariable(char *vblNameP, _BOOL is_variable_name_braced, int in if (repl && *repl == '\0') repl = (char *)NULL; - // Expand the pattern and replacement and build the python around them. + // Expand the pattern, name and replacement and build the python around them. // We want fix_string() to do the dirty work, but this requires staging & unstaging // the global buffers carefully because fix_string can alter them too. // Back up the real buffers before running fix_string() swap_burps(&g_buffer, &bufferBackup); swap_burps(&g_new, &newBackup); - // Expand & pythonify the pattern, and just expand the replacement + // Expand & pythonify the pattern g_globConversionState = CONVERTING; got = FIX_STRING; pattern = strdup(fix_string(lpatsub, FIX_STRING, &got)); + // Just expand the name and replacement g_globConversionState = INACTIVE; + + got = FIX_STRING; + // Extract the name from vblName. To guard against global memory issues, + // set it up with strdup() so that we are working on virgin territory. + // (1) Mark end of vblName in the original buffer + *start2P = '\0'; + // (2) In the future we should be able to say "name = strdup(vblNameP)" + // as we expect name will not need expanding. But for now, to make the python + // self-consistent we need these three steps to insert the regrettable "str(XXX.val)" markup + p = malloc(strlen(vblNameP)+2); + sprintf(p, "$%s", vblNameP); // '$' tricks fix_string() into treating name like a var name + name = strdup(fix_string(p, FIX_STRING, &got)); + // (3) Unmark the end of vblName + *start2P = c; + replacement = strdup(fix_string(repl, FIX_STRING, &got)); + // Ensure the glob-regex conversion is only attempted once g_globConversionState = PROTECTING; // Restore the buffers swap_burps(&g_buffer, &bufferBackup); @@ -904,10 +920,11 @@ static char * emitVariable(char *vblNameP, _BOOL is_variable_name_braced, int in // Construct the python code g_translate.m_uses.m_re = TRUE; burp(&g_new, "re.sub(%s%s, %s, %s", (strchr("\"\'", pattern[0]))?"r":"", pattern, replacement, name); - if (!is_global_replace) + if (!is_operator_doubled) burps(&g_new, ", count=1"); burpc(&g_new, ')'); + free(p); free(name); free(lpatsub); free(pattern); @@ -1517,11 +1534,11 @@ static fix_typeE substitute(fix_typeE want) want1 = (is_file_expansion ? FIX_STRING : want); // The strange treatment of quotes in the following code is meant to trade bash quoting for python quoting. - // For example, echo ${abc}de"fg"hi would translate to print(str(abc)+"defghi"). + // For example, echo de"fg"hi would translate to print("defghi"). // The strategy is to strip the quotes from literal text in the input, determine where python // needs them, and insert new ones there. To keep track of this we use the markers {START/END}_QUOTE. // Meanwhile we cannot just toss the quotes from the special bash variable "$*". - // That's what 'quoted' is used to keep track of as you can see in emitSimpleVariable().. + // That's what 'quoted' keeps track of, as you can see in the "*" handler inside emitSimpleVariable(). for (P = g_buffer.m_P; ; ++P) { @@ -1624,8 +1641,11 @@ static fix_typeE substitute(fix_typeE want) } // switch (c) } - else if (g_globConversionState == PROTECTING) + else if (g_globConversionState == PROTECTING) { + // Having ensured the glob-regex conversion is only attempted once + // for a given input, we revert to normal execution in the future g_globConversionState = INACTIVE; + } quote_removal = FALSE; @@ -2076,7 +2096,7 @@ log_deactivate(); resultP = NULL; arrayPP = brace_expand((char *) startP); // calling bash API here if (arrayPP) { - // Transform array into a pythonic list: [x, y, z] in g_braced + // Transform array into a pythonic list, writing [x, y, z] to g_braced if (arrayPP[0]) { fix_typeE single_want_type; int i; From b4baac81be8065a032edb2685c1a56f7b19a3da9 Mon Sep 17 00:00:00 2001 From: Michael Wood Date: Wed, 10 Dec 2025 07:48:21 -0500 Subject: [PATCH 2/4] Add left and right string chopping operators --- fix_string.c | 225 ++++++++++++++++++++----------- fix_string.h | 1 + tests/v4_tests/chop_strings.sh | 14 ++ tests/v4_tests/search_replace.sh | 2 +- 4 files changed, 160 insertions(+), 82 deletions(-) create mode 100755 tests/v4_tests/chop_strings.sh diff --git a/fix_string.c b/fix_string.c index d108488..12098de 100755 --- a/fix_string.c +++ b/fix_string.c @@ -847,71 +847,102 @@ static char * emitVariable(char *vblNameP, _BOOL is_variable_name_braced, int in --start2P; } break; - case '/': // c = *start2P - //case '%': - //case '#': + case '/': + case '%': + case '#': { - char *name, *value, *lpatsub, *pattern, *repl, *replacement, *p; - int delim, quoted=0; - char rawflag; + char *name, *value, *parameters, *pattern, *anchored_pattern, *repl, *replacement, *p; + int delim, quoted=0, plen; + char *rawflag; burpT bufferBackup, newBackup; _BOOL is_operator_doubled = FALSE; - // Extract the substitution parameters: name, pattern, replacement. + // Extract the name and the parameters: (1) pattern and (2) replacement ('/' operator only). // Much of the following code borrows from bash's parameter_brace_patsub (). memset(&bufferBackup, 0, sizeof(bufferBackup)); memset(&newBackup, 0, sizeof(newBackup)); value = start2P+1; if (*value == c) - { + { //TODO this needs to be generalized to handle /% and /% operators is_operator_doubled = TRUE; value++; } - lpatsub = strdup(value); - *(strrchr(lpatsub, '}')) = '\0'; - delim = skip_to_delim (lpatsub, ((*value == '/') ? 1 : 0), "/", 0); - if (lpatsub[delim] == '/') + parameters = strdup(value); + *(strrchr(parameters, '}')) = '\0'; + + if (c == '/') { - lpatsub[delim] = 0; - repl = lpatsub + delim + 1; + delim = skip_to_delim (parameters, ((*value == '/') ? 1 : 0), "/", 0); + if (parameters[delim] == '/') + { + parameters[delim] = 0; + repl = parameters + delim + 1; + } + else + repl = (char *)NULL; + if (repl && *repl == '\0') + repl = (char *)NULL; } - else - repl = (char *)NULL; - if (repl && *repl == '\0') - repl = (char *)NULL; - // Expand the pattern, name and replacement and build the python around them. + // Expand the pattern, name and (if op='/') replacement and build the python around them. // We want fix_string() to do the dirty work, but this requires staging & unstaging // the global buffers carefully because fix_string can alter them too. + got = FIX_STRING; // Back up the real buffers before running fix_string() swap_burps(&g_buffer, &bufferBackup); swap_burps(&g_new, &newBackup); // Expand & pythonify the pattern - g_globConversionState = CONVERTING; - got = FIX_STRING; - pattern = strdup(fix_string(lpatsub, FIX_STRING, &got)); - // Just expand the name and replacement - g_globConversionState = INACTIVE; + g_globConversionState = !(is_operator_doubled || c=='/') ? CONVERTING_UNGREEDY : CONVERTING; + pattern = strdup(fix_string(parameters, FIX_STRING, &got)); - got = FIX_STRING; - // Extract the name from vblName. To guard against global memory issues, - // set it up with strdup() so that we are working on virgin territory. - // (1) Mark end of vblName in the original buffer + // Finish the pattern with raw marker and/or anchor if required + plen = strlen(pattern); + switch (c) + { + case '#': + anchored_pattern = malloc(plen+6); + if (*pattern == 'r') { // Quoted-string patterns + sprintf(anchored_pattern, "%.2s^%s", pattern, pattern+2); + } else { // Other patterns e.g. variable names + sprintf(anchored_pattern, "r'^'+%s", pattern); + } + break; + case '%': + anchored_pattern = malloc(plen+6); + p = pattern + plen-1; // last char of pattern + if (strchr("\"\'", p)) { //if (*pattern == 'r') { + sprintf(anchored_pattern, "%s", pattern); + sprintf(p, "$%c", *p); + } + else + sprintf(anchored_pattern, "%s+r'$'", pattern); + break; + case '/': + //TODO This is where to add logic for '/#' and '/%' operators + anchored_pattern = pattern; + break; + } + + // Extract the name from vblName + g_globConversionState = INACTIVE; + // To guard against global memory issues, copy name into a new buffer + // (1) Mark end of name in the original buffer *start2P = '\0'; - // (2) In the future we should be able to say "name = strdup(vblNameP)" - // as we expect name will not need expanding. But for now, to make the python - // self-consistent we need these three steps to insert the regrettable "str(XXX.val)" markup + // (2) (TODO) Basically do "name = strdup(vblNameP)" as we expect name will not need + // expanding. But for now, to make the python self-consistent we need to insert + // the regrettable "str(XXX.val)" markup, We do this with 3 LOC: p = malloc(strlen(vblNameP)+2); sprintf(p, "$%s", vblNameP); // '$' tricks fix_string() into treating name like a var name name = strdup(fix_string(p, FIX_STRING, &got)); // (3) Unmark the end of vblName *start2P = c; - replacement = strdup(fix_string(repl, FIX_STRING, &got)); - // Ensure the glob-regex conversion is only attempted once + // Expand the replacement + replacement = strdup( (c=='/') ? fix_string(repl, FIX_STRING, &got) : "''"); + // Ensure the glob-to-regex conversion is only attempted once g_globConversionState = PROTECTING; // Restore the buffers swap_burps(&g_buffer, &bufferBackup); @@ -919,18 +950,19 @@ static char * emitVariable(char *vblNameP, _BOOL is_variable_name_braced, int in // Construct the python code g_translate.m_uses.m_re = TRUE; - burp(&g_new, "re.sub(%s%s, %s, %s", (strchr("\"\'", pattern[0]))?"r":"", pattern, replacement, name); - if (!is_operator_doubled) + burp(&g_new, "re.sub(%s, %s, %s", anchored_pattern, replacement, name); + if (c=='/' && !is_operator_doubled) burps(&g_new, ", count=1"); burpc(&g_new, ')'); free(p); free(name); - free(lpatsub); + free(parameters); + if (anchored_pattern != pattern) free(anchored_pattern); free(pattern); free(replacement); - endP = strrchr(vblNameP, '}'); + endP = strrchr(vblNameP, '}'); goto done; } break; @@ -1381,7 +1413,7 @@ static char * emit_delimited(char *startP, int in_quotes, fix_typeE want, fix_ty return endP; } -// combine_types(): Applies type modifier to a value in g_new to make it compatible with its predecessor +// combine_types(): Applies type modifier to the latest piece of g_new to make it compatible with its predecessor // before joining them together static fix_typeE combine_types(int offset, fix_typeE want_type, fix_typeE was_type, fix_typeE new_type) { @@ -1432,7 +1464,7 @@ static fix_typeE combine_types(int offset, fix_typeE want_type, fix_typeE was_ty } if (was_type == FIX_NONE) { - // Do nothing more + // Do nothing more. e.g. we have only one component log_return_msg("Untyped value, will not cast"); return new_type; } @@ -1440,8 +1472,15 @@ static fix_typeE combine_types(int offset, fix_typeE want_type, fix_typeE was_ty switch (was_type) { case FIX_ARRAY: case FIX_STRING: - // Join two things together - P = burp_insert(&g_new, offset, "+"); + // Join two pieces together + if ((g_globConversionState == CONVERTING || g_globConversionState == CONVERTING_UNGREEDY) + && (g_new.m_P[offset-1] == 'r')) { + // Join at the raw marker right behind the START_QUOTE + P = burp_insert(&g_new, offset-1, "+"); + } else { + // Join at the START_QUOTE + P = burp_insert(&g_new, offset, "+"); + } log_return_msg("Concatenated"); return new_type; } } @@ -1484,11 +1523,11 @@ static fix_typeE substitute(fix_typeE want) fix_typeE got; // What I had fix_typeE got1; // What I'm now seeing fix_typeE want1; - int i, startquote_offset, quoted, c, offset; + int i, startquote_offset, offset, quoted, c, output_c; const int NOT_QUOTED = -1; char **arrayPP, *P, *P1, glob_type; _BOOL is_file_expansion, quote_removal; - _BOOL is_outside_quotes, burp_extra_glob_symbol = FALSE; + _BOOL is_outside_quotes, burp_extra_glob_symbols = FALSE; log_enter("substitute (want=%t)", want); @@ -1506,7 +1545,7 @@ static fix_typeE substitute(fix_typeE want) is_file_expansion = FALSE; offset = 0; - if (g_globConversionState == INACTIVE) { // INACTIVE by default. Changes when expr is like ${s/x/y} + if (g_globConversionState == INACTIVE) { // INACTIVE by default. Changes inside string modification operators. for (P = g_buffer.m_P; c = *P ; ++P) { switch (c) { case '*': @@ -1575,7 +1614,7 @@ static fix_typeE substitute(fix_typeE want) offset = g_new.m_lth; P1 = emit_delimited(P, quoted, want1, &got1); if (g_globConversionState != INACTIVE) // This can change in emit_delim() so check it again - is_file_expansion = FALSE; + is_file_expansion = FALSE; if (P1 && P1 != P) { got = combine_types(offset, want1, got, got1); P = --P1; @@ -1584,91 +1623,113 @@ static fix_typeE substitute(fix_typeE want) // Restore to where we were. //TODO is this always correct even if emit_delim() changes the buffer length? g_new.m_lth = offset; g_new.m_P[offset] = '\0'; - break; + break; } // end switch block - // Other characters are generally copied straight from input to output. - // But first we might need to convert shell globbing expressions to pythonic regexes - if (g_globConversionState == CONVERTING) - { - //TODO A perfect solution would track nested glob_types incl. NULLs in a stack and unwind it to print them out + // Other characters are generally copied straight from input to output... + output_c = c; + + // ... but we need to make exceptions when converting shell globbing expressions to pythonic regexes + switch (g_globConversionState) { + case CONVERTING: + case CONVERTING_UNGREEDY: + //TODO A perfect solution would use recursion to handle nested glob_types incl. NULLs switch (c) { // Look for the symbols relevant to extended globbing, POSIX globbing or both - case '!': case '+': if (*(P+1) == '(') { glob_type = c; // Save the glob symbol, print it later - c = *++P; // Skip to the '(', print that now + output_c = *++P; // Skip to the '(', print that now + } + break; + case '!': + if (*(P+1) == '(') { + // '!(..)' becomes '(?!..)' + burp_extra_glob_symbols = TRUE; + glob_type = c; + output_c = *++P; } break; case '@': if (*(P+1) == '(') { - c = *++P; // Don't flag as glob and skip the '@' sign + output_c = *++P; // Don't flag as glob and skip the '@' sign + // What bash sees as @(..) python sees as (..) } break; case ')': // Make sure the ')' is globbing-related if (glob_type != 0) { - // Set the the stage to burp both ')' and the glob type. - burp_extra_glob_symbol = TRUE; + // Set the stage to burp both ')' and the glob type. + burp_extra_glob_symbols = TRUE; + output_c = *P; } break; case '*': // '*' can indicate an extended glob OR a POSIX glob if ((*(P+1) == '(')) { glob_type = c; - c = *++P; + output_c = *++P; } else { // Need to convert "*" to ".*" glob_type = c; - c = '.'; - burp_extra_glob_symbol = TRUE; + output_c = '.'; + burp_extra_glob_symbols = TRUE; } break; case '?': // '?' can indicate an extended glob OR a POSIX glob if ((*(P+1) == '(')) { glob_type = c; - c = *++P; + output_c = *++P; } else { // We will just replace '?' with '.'. No need to remember the type. glob_type = 0; - c = '.'; + output_c = '.'; } break; - // case '[': // Bracketed globs are the same in bash and python. Nothing to do here. + // case '[': // Bracketed globs are the same in bash and python. Nothing to do here. + default: + output_c = c; } // switch (c) - - } - else if (g_globConversionState == PROTECTING) { - // Having ensured the glob-regex conversion is only attempted once - // for a given input, we revert to normal execution in the future - g_globConversionState = INACTIVE; - } + break; + case PROTECTING: + // Having ensured the glob-regex conversion is only attempted once + // for a given input, we revert to normal execution in the future + g_globConversionState = INACTIVE; + break; + } // switch (g_globConversionState) quote_removal = FALSE; - // Saw some normal character + // Saw some normal character: Make sure it gets QUOTED. Escape it if needed. Then post the character. if (startquote_offset == NOT_QUOTED) { - // Set the START marker for a new subsection of text that will need quoting + // Set the START_QUOTE marker for a new subsection of text that will need quoting + if (g_globConversionState == CONVERTING || g_globConversionState == CONVERTING_UNGREEDY) { + // Put a raw string marker before the quote + burpc(&g_new, 'r'); + } startquote_offset = g_new.m_lth; burpc(&g_new, START_QUOTE); } if (c == '\\') { if (P[1]) { burpc(&g_new, '\\'); - c = *++P; + output_c = *++P; } } + // Most important step: Append the possibly-modified character to output + burpc(&g_new, output_c); - // Most important step: copy input to output - burpc(&g_new, c); - - // If glob conversion requires us to print another character, do that now - if (burp_extra_glob_symbol) { + // If glob conversion requires us to print more symbols, do that now + if (burp_extra_glob_symbols) { + if (glob_type == '!') + burpc(&g_new, '?'); burpc(&g_new, glob_type); + // Also append the ungreedy sign '?' if needed + if ((glob_type == '+' || glob_type == '*') && g_globConversionState == CONVERTING_UNGREEDY) + burpc(&g_new, '?'); // Reset the state - burp_extra_glob_symbol = FALSE; + burp_extra_glob_symbols = FALSE; glob_type = 0; } } @@ -1875,8 +1936,8 @@ static void compactWhiteSpace(void) } } -// unmarkQuotes(): Remove {START/END}_QUOTE markers (not quote characters) -// from g_buffer. (These markers are sometimes added by substitute().) +// unmarkQuotes(): Replace {START/END}_QUOTE markers with '"' characters +// in g_buffer. (These markers are sometimes added by substitute().) static void unmarkQuotes(_BOOL delete_quotes) { char *P, *P1; @@ -1893,13 +1954,15 @@ static void unmarkQuotes(_BOOL delete_quotes) break; case END_QUOTE: if (P[1] == START_QUOTE) { + // Delete adjacent END-START pairs to merge quoted chunks ++P; continue; - } + } // N.B. We fall through here case START_QUOTE: if (delete_quotes && !expand_depth) { continue; } + // Normal step: Turn START/END_QUOTE into '"' c = '"'; } *P1++ = c; diff --git a/fix_string.h b/fix_string.h index a66af4f..0d09e3f 100755 --- a/fix_string.h +++ b/fix_string.h @@ -8,6 +8,7 @@ typedef int _BOOL; typedef enum { INACTIVE = 0, CONVERTING, + CONVERTING_UNGREEDY, PROTECTING } globConversionStateE; diff --git a/tests/v4_tests/chop_strings.sh b/tests/v4_tests/chop_strings.sh new file mode 100755 index 0000000..ea35bb9 --- /dev/null +++ b/tests/v4_tests/chop_strings.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# Try out some string chops with literal, variable, and globbing forms. +s=aasdf90babasfgbbacxv +key=f +repl=acxv +echo ${s#a*f} +echo ${s##a*f} +echo ${s%a*v} +echo ${s%%a+(?)*v} # Known bug; requires glob-to-regex to run recursively +echo ${s#+(a)*$key} +echo ${s##+(a)*${key}__} +echo ${s%%+(b)$repl} + diff --git a/tests/v4_tests/search_replace.sh b/tests/v4_tests/search_replace.sh index 9bc7460..7eb87f4 100755 --- a/tests/v4_tests/search_replace.sh +++ b/tests/v4_tests/search_replace.sh @@ -4,7 +4,7 @@ s=01234567890abcdefgh key=7 repl=haha -echo ${s/+(90|ba)/__} +echo ${s/+(90|qr)/__} echo ${s/456/___} echo ${s//456/___} echo ${s/$key/_${repl}_} From f7c5a3c932f60ed37ecf43ea797067bf3d3fff2f Mon Sep 17 00:00:00 2001 From: Michael Wood Date: Wed, 10 Dec 2025 07:56:16 -0500 Subject: [PATCH 3/4] Type compatibility edit --- fix_string.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fix_string.c b/fix_string.c index 12098de..cd76f92 100755 --- a/fix_string.c +++ b/fix_string.c @@ -913,7 +913,7 @@ static char * emitVariable(char *vblNameP, _BOOL is_variable_name_braced, int in case '%': anchored_pattern = malloc(plen+6); p = pattern + plen-1; // last char of pattern - if (strchr("\"\'", p)) { //if (*pattern == 'r') { + if (strchr("\"\'", p) != NULL) { //if (*pattern == 'r') { sprintf(anchored_pattern, "%s", pattern); sprintf(p, "$%c", *p); } From d5e9fb6f4325b61134b4f6dc09cd16a7220eef35 Mon Sep 17 00:00:00 2001 From: Michael Wood Date: Wed, 10 Dec 2025 07:58:50 -0500 Subject: [PATCH 4/4] Type compatibility edit --- fix_string.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fix_string.c b/fix_string.c index cd76f92..14e7e16 100755 --- a/fix_string.c +++ b/fix_string.c @@ -913,7 +913,7 @@ static char * emitVariable(char *vblNameP, _BOOL is_variable_name_braced, int in case '%': anchored_pattern = malloc(plen+6); p = pattern + plen-1; // last char of pattern - if (strchr("\"\'", p) != NULL) { //if (*pattern == 'r') { + if (strchr("\"\'", *p) != NULL) { //if (*pattern == 'r') { sprintf(anchored_pattern, "%s", pattern); sprintf(p, "$%c", *p); }