From 530ba0ec67ae433723150732b5e3c4f47f770651 Mon Sep 17 00:00:00 2001 From: Gregory Paradis Date: Sat, 15 Aug 2026 20:25:55 +0000 Subject: [PATCH 1/2] Harden generated-model runtime and unblock 2024 FABLE conversion - Add static INDIRECT(ADDRESS(ROW()+-k, COLUMN()+-k)) resolution and repair the corrupted structured-reference prefix defect so the 2024 Canada FABLE workbook converts end to end (410,299 formulas, 0 untranslated, 0 output-closure blockers). - Execute the generated model scenario (10,274 outputs) and fix all 349 runtime output errors: Excel-faithful error strings from _sf_arith, _sf_compare, _sf_sum, _sf_average, _sf_ln, VALUE, and NUMBERVALUE; coercion-safe ordering comparisons; _SfRangeView resolution in _sf_index. - Keep the 1,741 static circular-dependency warnings documented as phantom whole-column SUMIFS cycles that never form runtime cycles. - Add regression tests covering the corrupted-ref repair, static INDIRECT, arithmetic/comparison coercion, SUM error propagation, and INDEX range views. Full test run: 217 passed, 1 skipped. --- CHANGE_LOG.md | 49 ++++ src/modelwright/extraction.py | 32 ++- src/modelwright/formulas.py | 117 +++++++++- src/modelwright/generation.py | 373 ++++++++++++++++++++++++++++-- src/modelwright/graph.py | 4 +- src/modelwright/references.py | 59 ++++- tests/test_dependency_graph.py | 41 ++++ tests/test_formula_translation.py | 161 ++++++++++++- tests/test_openpyxl_extraction.py | 50 ++++ tests/test_python_generation.py | 300 ++++++++++++++++++++++++ 10 files changed, 1143 insertions(+), 43 deletions(-) diff --git a/CHANGE_LOG.md b/CHANGE_LOG.md index 5643947..9c6d6e5 100644 --- a/CHANGE_LOG.md +++ b/CHANGE_LOG.md @@ -2,6 +2,55 @@ This file records completed project work in chronological order. +## 2026-08-15 + +- Unblocked full-workbook conversion of the 2024 Canada FABLE Calculator workbook (Zenodo record + 14755928, `2024_Open_FABLECalculator.xlsx`): the extract, graph, translate, infer-contract, and + generate steps now all complete cleanly (410,299 formulas translated, 0 untranslated, 0 output + closure blockers, `generated=True`) for both the pristine and edited workbook copies. +- Implemented static `INDIRECT(ADDRESS(ROW() +- k, COLUMN() +- k))` resolution (used by 55 cells as + a "value of the cell directly above" pattern): `ROW`/`COLUMN` parse to the formula cell's + coordinates, `ADDRESS` folds to a literal address string, `INDIRECT` resolves a static address to + a cell reference; `static_indirect_cell_reference` in `references.py` adds the resolved target to + `FormulaRecord.raw_references` so the dependency graph creates the execution edge. +- Added `repair_corrupted_structured_references` in `references.py` for the source-level defect + where a structured reference appears with a duplicated table prefix + (`name[] name[[#This Row],[Column]]`, 330 cells); the dangling prefix is dropped during + tokenization and raw-reference extraction while `FormulaRecord.raw_formula` keeps provenance. +- Added `ADDRESS` and `INDIRECT` to `SUPPORTED_FUNCTIONS`; non-static INDIRECT/ADDRESS forms report + `unsupported_function` instead of translating. +- Added regression tests: repaired corrupted structured reference, static INDIRECT translation, + non-static INDIRECT rejection, and static INDIRECT raw-reference extraction. Full test run: + 212 passed, 1 skipped. +- Executed the generated 2024 Canada FABLE model end to end (`calculate({})`, 10,274 outputs, + ~100 s) and eliminated all 349 runtime output errors found during scenario execution. +- Hardened generated runtime semantics in `generation.py` so invalid workbook arithmetic and math + return Excel-faithful error strings instead of raising Python exceptions: + `_sf_arith` now coerces numeric strings, treats blank as zero, returns `#DIV/0!` for division by + zero, and `#NUM!` for negative-base non-integer `^`; ordering comparisons (`>`, `>=`, `<`, `<=`) + render through a new coercion-safe `_sf_compare` (Excel number None: progress(message) def _extract_formula(cell_ref: str, raw_formula: str, cached_value: JsonValue) -> FormulaRecord: + tokenized_formula = repair_corrupted_structured_references(raw_formula) try: - tokenizer = Tokenizer(raw_formula) + tokenizer = Tokenizer(tokenized_formula) except Exception as error: return FormulaRecord( raw_formula=raw_formula, @@ -544,6 +549,9 @@ def _extract_formula(cell_ref: str, raw_formula: str, cached_value: JsonValue) - raw_references = tuple( token.value for token in tokenizer.items if token.type == "OPERAND" and token.subtype == "RANGE" ) + static_indirect_reference = static_indirect_cell_reference(cell_ref, raw_formula) + if static_indirect_reference is not None and static_indirect_reference not in raw_references: + raw_references = raw_references + (static_indirect_reference,) functions = tuple( token.value[:-1].upper() for token in tokenizer.items if token.type == "FUNC" and token.subtype == "OPEN" ) @@ -636,6 +644,20 @@ def _json_value(value: Any) -> JsonValue: return str(value) +def _formula_cell_value(value: Any) -> str: + """Return the formula text for a formula cell value. + + OpenPyXL stores array formula cells (and similar formula objects) as wrapper + objects that expose the formula text through their ``text`` attribute instead + of as plain strings. + """ + if isinstance(value, str): + return value + if isinstance(value, ArrayFormula): + return value.text if value.text else str(value) + return str(value) + + def _is_external_reference(reference: str) -> bool: return "[" in reference and "]" in reference and ("." in reference.split("]", 1)[0] or "!" in reference) @@ -657,7 +679,7 @@ def _bracketed_parts(reference: str) -> tuple[str, ...]: if character == "]": depth -= 1 if depth == 0: - part = "".join(current) + part = "".join(current).strip() current = [] if part.startswith("[") and part.endswith("]"): parts.extend(_bracketed_parts(part)) @@ -672,4 +694,4 @@ def _bracketed_parts(reference: str) -> tuple[str, ...]: def _clean_structured_selector(selector: str) -> str: - return selector.removeprefix("@").replace("''", "'") + return selector.strip().removeprefix("@").replace("''", "'") diff --git a/src/modelwright/formulas.py b/src/modelwright/formulas.py index 7f2bd74..fd12f9a 100644 --- a/src/modelwright/formulas.py +++ b/src/modelwright/formulas.py @@ -15,7 +15,9 @@ from modelwright.extraction import CellRecord from modelwright.graph import DependencyEdge, DependencyGraph from modelwright.references import WorkbookReference +from modelwright.references import cell_reference_coordinates from modelwright.references import normalize_reference +from modelwright.references import repair_corrupted_structured_references JsonValue = str | int | float | bool | None | list[Any] | dict[str, Any] @@ -26,21 +28,31 @@ { "AND", "AVERAGE", + "AVERAGEIF", + "AVERAGEIFS", "CONCATENATE", "COUNTIF", "COUNTIFS", "IF", "IFERROR", "IFNA", + "INDEX", + "LN", + "MATCH", "MAX", "MIN", + "MINIFS", + "NUMBERVALUE", "OR", "OFFSET", "ROUND", "SUM", "SUMIF", "SUMIFS", + "VALUE", "VLOOKUP", + "ADDRESS", + "INDIRECT", } ) SUPPORTED_OPERATORS = frozenset({"+", "-", "*", "/", "^", "&", ">", ">=", "<", "<=", "=", "<>", "(", ")", ","}) @@ -343,6 +355,8 @@ def _parse_primary(self) -> FormulaExpressionNode: return FormulaExpressionNode.literal(token.value) if token.kind == "logical": return FormulaExpressionNode.literal(token.value == "TRUE") + if token.kind == "error": + return FormulaExpressionNode.literal(token.value) if token.kind == "reference": return FormulaExpressionNode.reference_to(self._resolved_reference(token.value)) if token.kind == "identifier": @@ -357,6 +371,24 @@ def _parse_function_call(self, function_name: str) -> FormulaExpressionNode: self._expect("(") raw_function_name = function_name.upper() function_name = _normalized_function_name(raw_function_name) + if function_name == "ROW": + if (token := self._peek()) is not None and token.value == ")": + self._advance() + return FormulaExpressionNode.literal(cell_reference_coordinates(self.cell.cell_ref)[0]) + raise FormulaTranslationError( + "unsupported_function", + "ROW with arguments is not supported", + raw_function_name, + ) + if function_name == "COLUMN": + if (token := self._peek()) is not None and token.value == ")": + self._advance() + return FormulaExpressionNode.literal(cell_reference_coordinates(self.cell.cell_ref)[1]) + raise FormulaTranslationError( + "unsupported_function", + "COLUMN with arguments is not supported", + raw_function_name, + ) if function_name not in SUPPORTED_FUNCTIONS: raise FormulaTranslationError( "unsupported_function", @@ -378,6 +410,10 @@ def _parse_function_call(self, function_name: str) -> FormulaExpressionNode: self._expect(")") if function_name == "OFFSET": return _static_offset_reference(arguments) + if function_name == "ADDRESS": + return _static_address_reference(arguments) + if function_name == "INDIRECT": + return _static_indirect_reference(self, arguments) return FormulaExpressionNode.function_call(function_name, tuple(arguments)) def _resolved_reference(self, raw_reference: str) -> WorkbookReference: @@ -437,7 +473,8 @@ def _expect(self, value: str) -> None: def _formula_tokens(raw_formula: str) -> tuple[_FormulaToken, ...]: tokens: list[_FormulaToken] = [] - for token in Tokenizer(raw_formula).items: + repaired_formula = repair_corrupted_structured_references(raw_formula) + for token in Tokenizer(repaired_formula).items: if token.type == "WHITE-SPACE": continue if token.type == "FUNC" and token.subtype == "OPEN": @@ -463,11 +500,8 @@ def _formula_tokens(raw_formula: str) -> tuple[_FormulaToken, ...]: tokens.append(_FormulaToken("logical", token.value.upper())) continue if token.type == "OPERAND" and token.subtype == "ERROR": - raise FormulaTranslationError( - "unsupported_error_reference", - "formula contains an unsupported error reference", - token.value, - ) + tokens.append(_FormulaToken("error", token.value)) + continue if token.type == "OPERAND" and token.subtype == "RANGE": tokens.append(_FormulaToken("reference", token.value)) continue @@ -530,6 +564,77 @@ def _literal_integer(node: FormulaExpressionNode) -> int | None: return None +def _static_address_reference(arguments: list[FormulaExpressionNode]) -> FormulaExpressionNode: + if len(arguments) not in {2, 3, 4}: + raise FormulaTranslationError( + "unsupported_function", + "ADDRESS requires two to four arguments", + "ADDRESS", + ) + row = _static_number(arguments[0]) + column = _static_number(arguments[1]) + if row < 1 or column < 1 or int(row) != row or int(column) != column: + raise FormulaTranslationError( + "unsupported_function", + "ADDRESS row and column must be static positive integers", + "ADDRESS", + ) + address = f"{get_column_letter(int(column))}{int(row)}" + return FormulaExpressionNode.literal(address) + + +def _static_indirect_reference(parser: "_FormulaParser", arguments: list[FormulaExpressionNode]) -> FormulaExpressionNode: + if len(arguments) != 1: + raise FormulaTranslationError( + "unsupported_function", + "INDIRECT requires exactly one argument", + "INDIRECT", + ) + address = _static_address_text(arguments[0]) + if address is None: + raise FormulaTranslationError( + "unsupported_function", + "INDIRECT argument must be a static cell address", + "INDIRECT", + ) + return FormulaExpressionNode.reference_to(parser._resolved_reference(address)) + + +def _static_address_text(node: FormulaExpressionNode) -> str | None: + if node.kind == "literal" and isinstance(node.value, str): + return node.value + return None + + +def _static_number(node: FormulaExpressionNode) -> int | float: + if node.kind == "literal": + value = node.value + if isinstance(value, (int, float)) and not isinstance(value, bool): + return value + if node.kind == "unary": + (operand,) = node.operands + if node.operator == "-": + return -_static_number(operand) + if node.operator == "+": + return _static_number(operand) + if node.kind == "binary": + left = _static_number(node.operands[0]) + right = _static_number(node.operands[1]) + if node.operator == "+": + return left + right + if node.operator == "-": + return left - right + if node.operator == "*": + return left * right + if node.operator == "/": + return left / right + raise FormulaTranslationError( + "unsupported_function", + "function argument is not statically computable", + "", + ) + + def _shift_cell_reference( reference: WorkbookReference, *, diff --git a/src/modelwright/generation.py b/src/modelwright/generation.py index 1e20d0e..d4e343b 100644 --- a/src/modelwright/generation.py +++ b/src/modelwright/generation.py @@ -575,6 +575,98 @@ def _render_module( " return 0 if value is None else value", "", "", + "def _sf_is_error(value):", + " return isinstance(value, str) and value.startswith('#')", + "", + "", + "def _sf_arith(operator, left, right):", + " left_value = _sf_value(left)", + " right_value = _sf_value(right)", + " for value in (left_value, right_value):", + " if _sf_is_error(value):", + " return value", + " left_number = _sf_coerce_number(left_value)", + " right_number = _sf_coerce_number(right_value)", + " if left_number is None or right_number is None:", + " return '#VALUE!'", + " if operator == '+':", + " return left_number + right_number", + " if operator == '-':", + " return left_number - right_number", + " if operator == '*':", + " return left_number * right_number", + " if operator == '/':", + " if right_number == 0:", + " return '#DIV/0!'", + " return left_number / right_number", + " if operator == '^':", + " if left_number == 0 and right_number < 0:", + " return '#DIV/0!'", + " if left_number < 0 and right_number != int(right_number):", + " return '#NUM!'", + " return left_number ** right_number", + " raise ValueError(f'unsupported arithmetic operator: {operator}')", + "", + "", + "def _sf_coerce_number(value):", + " value = _sf_value(value)", + " if isinstance(value, bool):", + " return int(value)", + " if value is None or value == '':", + " return 0", + " if isinstance(value, (int, float)):", + " return value", + " if isinstance(value, str):", + " try:", + " return float(value)", + " except ValueError:", + " return None", + " return None", + "", + "", + "def _sf_compare_operand(value):", + " value = _sf_value(value)", + " if isinstance(value, bool):", + " return int(value), 0", + " if value is None or value == '':", + " return 0, 0", + " if isinstance(value, (int, float)):", + " return value, 0", + " if isinstance(value, str):", + " try:", + " return float(value), 0", + " except ValueError:", + " return value, 1", + " return str(value), 1", + "", + "", + "def _sf_compare(operator, left, right):", + " left_value = _sf_value(left)", + " right_value = _sf_value(right)", + " for value in (left_value, right_value):", + " if _sf_is_error(value):", + " return value", + " left_operand, left_kind = _sf_compare_operand(left_value)", + " right_operand, right_kind = _sf_compare_operand(right_value)", + " if left_kind != right_kind:", + " result = -1 if left_kind < right_kind else 1", + " elif left_kind == 0:", + " result = (left_operand > right_operand) - (left_operand < right_operand)", + " else:", + " left_text = str(left_operand).upper()", + " right_text = str(right_operand).upper()", + " result = (left_text > right_text) - (left_text < right_text)", + " if operator == '<':", + " return result < 0", + " if operator == '<=':", + " return result <= 0", + " if operator == '>':", + " return result > 0", + " if operator == '>=':", + " return result >= 0", + " raise ValueError(f'unsupported comparison operator: {operator}')", + "", + "", "def _sf_direct_reference(value):", " value = _sf_value(value)", " return 0 if value is None else value", @@ -587,6 +679,53 @@ def _render_module( " return value", "", "", + "def _sf_excel_value(value):", + " value = _sf_value(value)", + " if value is None:", + " return 0", + " if isinstance(value, bool):", + " return int(value)", + " if isinstance(value, (int, float)):", + " return value", + " try:", + " return float(str(value).replace(',', '.'))", + " except (TypeError, ValueError):", + " return '#VALUE!'", + "", + "", + "def _sf_excel_numbervalue(value, *separators):", + " value = _sf_value(value)", + " if value is None or value == '':", + " return 0", + " if isinstance(value, (int, float)):", + " return value", + " text = str(value)", + " if len(separators) >= 2 and separators[1]:", + " text = text.replace(str(separators[1]), '')", + " decimal = str(separators[0]) if separators and separators[0] else '.'", + " if decimal != '.':", + " text = text.replace(decimal, '.')", + " try:", + " return float(text)", + " except (TypeError, ValueError):", + " return '#VALUE!'", + "", + "", + "def _sf_ln(value):", + " value = _sf_value(value)", + " if isinstance(value, str):", + " try:", + " value = float(value)", + " except ValueError:", + " return '#VALUE!'", + " if value is None or value == '':", + " value = 0", + " if not isinstance(value, (int, float)) or value <= 0:", + " return '#NUM!'", + " import math", + " return math.log(value)", + "", + "", "@lru_cache(maxsize=4096)", "def _sf_numeric_value(value):", " if isinstance(value, bool):", @@ -602,8 +741,32 @@ def _render_module( "", "", "def _sf_average(values):", - " values = list(values)", - " return sum(values) / len(values)", + " total = 0", + " count = 0", + " for value in values:", + " value = _sf_value(value)", + " if _sf_is_error(value):", + " return value", + " if isinstance(value, bool):", + " total += int(value)", + " count += 1", + " elif isinstance(value, (int, float)):", + " total += value", + " count += 1", + " return total / count if count else '#DIV/0!'", + "", + "", + "def _sf_sum(*operands):", + " total = 0", + " for value in _sf_flatten(operands):", + " value = _sf_value(value)", + " if _sf_is_error(value):", + " return value", + " if isinstance(value, bool):", + " total += int(value)", + " elif isinstance(value, (int, float)):", + " total += value", + " return total", "", "", "def _sf_iferror(value_fn, fallback):", @@ -660,14 +823,8 @@ def _render_module( " return _sf_criteria_equal(value, expected)", " if operator == '<>':", " return not _sf_criteria_equal(value, expected)", - " if operator == '>':", - " return value > expected", - " if operator == '>=':", - " return value >= expected", - " if operator == '<':", - " return value < expected", - " if operator == '<=':", - " return value <= expected", + " if operator in ('>', '>=', '<', '<='):", + " return _sf_compare(operator, value, expected)", " raise ValueError(f'unsupported criteria operator: {operator}')", "", "", @@ -697,6 +854,9 @@ def _render_module( " criteria_values = tuple(_sf_flatten((criteria_range,)))", " sum_values = criteria_values if sum_range is None else tuple(_sf_flatten_lazy((sum_range,)))", " matcher = _sf_criteria_matcher(criteria)", + " for sum_value in sum_values:", + " if not callable(sum_value) and _sf_is_error(sum_value):", + " return sum_value", " total = 0", " for criteria_value, sum_value in zip(criteria_values, sum_values):", " if matcher(criteria_value):", @@ -711,6 +871,9 @@ def _render_module( "", "def _sf_sumifs(sum_range, *criteria_pairs):", " sum_values = tuple(_sf_flatten_lazy((sum_range,)))", + " for sum_value in sum_values:", + " if not callable(sum_value) and _sf_is_error(sum_value):", + " return sum_value", " criteria_ranges = [tuple(_sf_flatten((criteria_range,))) for criteria_range, _criteria in criteria_pairs]", " criteria_matchers = tuple(_sf_criteria_matcher(criteria) for _range, criteria in criteria_pairs)", " total = 0", @@ -732,6 +895,65 @@ def _render_module( " )", "", "", + "def _sf_averageif(criteria_range, criteria, average_range=None):", + " criteria_values = tuple(_sf_flatten((criteria_range,)))", + " average_values = criteria_values if average_range is None else tuple(_sf_flatten_lazy((average_range,)))", + " for average_value in average_values:", + " if not callable(average_value) and _sf_is_error(average_value):", + " return average_value", + " matcher = _sf_criteria_matcher(criteria)", + " total = 0", + " count = 0", + " for criteria_value, average_value in zip(criteria_values, average_values):", + " if not matcher(criteria_value):", + " continue", + " value = _sf_value(average_value)", + " if value is None or isinstance(value, str):", + " continue", + " total += value", + " count += 1", + " return total / count", + "", + "", + "def _sf_averageifs(average_range, *criteria_pairs):", + " average_values = tuple(_sf_flatten_lazy((average_range,)))", + " for average_value in average_values:", + " if not callable(average_value) and _sf_is_error(average_value):", + " return average_value", + " criteria_ranges = [tuple(_sf_flatten((criteria_range,))) for criteria_range, _criteria in criteria_pairs]", + " criteria_matchers = tuple(_sf_criteria_matcher(criteria) for _range, criteria in criteria_pairs)", + " total = 0", + " count = 0", + " for index, average_value in enumerate(average_values):", + " if not all(matcher(criteria_range[index]) for criteria_range, matcher in zip(criteria_ranges, criteria_matchers)):", + " continue", + " value = _sf_value(average_value)", + " if value is None or isinstance(value, str):", + " continue", + " total += value", + " count += 1", + " return total / count", + "", + "", + "def _sf_minifs(min_range, *criteria_pairs):", + " min_values = tuple(_sf_flatten_lazy((min_range,)))", + " for min_value in min_values:", + " if not callable(min_value) and _sf_is_error(min_value):", + " return min_value", + " criteria_ranges = [tuple(_sf_flatten((criteria_range,))) for criteria_range, _criteria in criteria_pairs]", + " criteria_matchers = tuple(_sf_criteria_matcher(criteria) for _range, criteria in criteria_pairs)", + " minimum = None", + " for index, min_value in enumerate(min_values):", + " if not all(matcher(criteria_range[index]) for criteria_range, matcher in zip(criteria_ranges, criteria_matchers)):", + " continue", + " value = _sf_value(min_value)", + " if value is None or isinstance(value, str):", + " continue", + " if minimum is None or value < minimum:", + " minimum = value", + " return 0 if minimum is None else minimum", + "", + "", "def _sf_range_lookup_enabled(range_lookup):", " if isinstance(range_lookup, str):", " return range_lookup.upper() not in {'FALSE', '0'}", @@ -767,6 +989,84 @@ def _render_module( " return candidate[column_index]", "", "", + "def _sf_match(lookup_value, lookup_array, match_type=1):", + " values = tuple(_sf_flatten((lookup_array,)))", + " match_type = _sf_value(match_type)", + " if isinstance(match_type, str):", + " try:", + " match_type = int(float(match_type))", + " except (TypeError, ValueError):", + " raise ValueError('#VALUE!') from None", + " if match_type == 0:", + " for index, value in enumerate(values):", + " if _sf_lookup_equal(value, lookup_value):", + " return index + 1", + " return '#N/A'", + " if match_type == 1:", + " candidate = None", + " for index, value in enumerate(values):", + " try:", + " matched = value <= lookup_value", + " except TypeError:", + " continue", + " if matched:", + " candidate = index + 1", + " else:", + " break", + " return '#N/A' if candidate is None else candidate", + " if match_type == -1:", + " candidate = None", + " for index, value in enumerate(values):", + " try:", + " matched = value >= lookup_value", + " except TypeError:", + " continue", + " if matched:", + " candidate = index + 1", + " break", + " return '#N/A' if candidate is None else candidate", + " raise ValueError('#VALUE!')", + "", + "", + "def _sf_index(array, row_num, col_num=None):", + " array = _sf_value(array)", + " if isinstance(array, _SfRangeView):", + " if array.max_col == array.min_col:", + " rows = tuple(array.values())", + " else:", + " width = array.max_col - array.min_col + 1", + " values = array.values()", + " rows = tuple(tuple(values[index:index + width]) for index in range(0, len(values), width))", + " elif isinstance(array, (list, tuple)) and any(isinstance(item, (list, tuple)) for item in array):", + " rows = tuple(tuple(row) for row in array)", + " elif isinstance(array, (list, tuple)):", + " rows = tuple(array)", + " else:", + " rows = (array,)", + " try:", + " row_index = int(_sf_value(row_num)) - 1", + " except (TypeError, ValueError):", + " return '#VALUE!'", + " if row_index < 0 or row_index >= len(rows):", + " return '#REF!'", + " row = rows[row_index]", + " if col_num is None:", + " return row", + " try:", + " col_value = _sf_value(col_num)", + " except (TypeError, ValueError):", + " return '#VALUE!'", + " if col_value == 0:", + " return row", + " try:", + " col_index = int(col_value) - 1", + " except (TypeError, ValueError):", + " return '#VALUE!'", + " if not isinstance(row, (list, tuple)) or col_index < 0 or col_index >= len(row):", + " return '#REF!'", + " return row[col_index]", + "", + "", f"def {contract.entrypoint}(inputs=None):", " inputs = {} if inputs is None else dict(inputs)", " _cache = {}", @@ -934,16 +1234,17 @@ def _render_expression(node: FormulaExpressionNode | None) -> str: if node.kind == "binary": left, right = node.operands if node.operator == "^": - return f"(_sf_number({_render_expression(left)}) ** _sf_number({_render_expression(right)}))" + return f"_sf_arith('^', {_render_expression(left)}, {_render_expression(right)})" if node.operator == "&": return f"(str({_render_expression(left)}) + str({_render_expression(right)}))" - return f"(_sf_number({_render_expression(left)}) {node.operator} _sf_number({_render_expression(right)}))" + return f"_sf_arith({node.operator!r}, {_render_expression(left)}, {_render_expression(right)})" if node.kind == "comparison": left, right = node.operands + if node.operator in {">", ">=", "<", "<="}: + return f"_sf_compare({node.operator!r}, {_render_expression(left)}, {_render_expression(right)})" if node.operator in {"=", "<>"}: return f"_sf_compare_criteria({_render_expression(left)}, {node.operator!r}, {_render_expression(right)})" - operator = _python_comparison_operator(node.operator) - return f"({_render_expression(left)} {operator} {_render_expression(right)})" + raise ValueError(f"unsupported comparison operator: {node.operator}") if node.kind == "function_call": return _render_function_call(node) @@ -976,7 +1277,7 @@ def _render_function_call(node: FormulaExpressionNode) -> str: if node.function_name == "OR": return f"any(_sf_flatten({_render_argument_tuple(node.operands)}))" if node.function_name == "SUM": - return f"sum(_sf_flatten({_render_argument_tuple(node.operands)}))" + return f"_sf_sum({_render_argument_tuple(node.operands)})" if node.function_name == "MIN": return f"min(_sf_flatten({_render_argument_tuple(node.operands)}))" if node.function_name == "MAX": @@ -993,6 +1294,10 @@ def _render_function_call(node: FormulaExpressionNode) -> str: if len(node.operands) != 2: raise ValueError("COUNTIF requires two operands") return f"_sf_countif({_render_function_arguments(node.operands)})" + if node.function_name == "AVERAGEIF": + if len(node.operands) not in {2, 3}: + raise ValueError("AVERAGEIF requires two or three operands") + return f"_sf_averageif({_render_function_arguments(node.operands)})" if node.function_name == "SUMIFS": if len(node.operands) < 3 or len(node.operands) % 2 != 1: raise ValueError("SUMIFS requires a sum range followed by criteria range/criteria pairs") @@ -1001,6 +1306,34 @@ def _render_function_call(node: FormulaExpressionNode) -> str: if len(node.operands) < 2 or len(node.operands) % 2 != 0: raise ValueError("COUNTIFS requires criteria range/criteria pairs") return f"_sf_countifs({_render_criteria_function_arguments(node.operands)})" + if node.function_name == "AVERAGEIFS": + if len(node.operands) < 3 or len(node.operands) % 2 != 1: + raise ValueError("AVERAGEIFS requires an average range followed by criteria range/criteria pairs") + return f"_sf_averageifs({_render_criteria_function_arguments(node.operands)})" + if node.function_name == "MINIFS": + if len(node.operands) < 3 or len(node.operands) % 2 != 1: + raise ValueError("MINIFS requires a min range followed by criteria range/criteria pairs") + return f"_sf_minifs({_render_criteria_function_arguments(node.operands)})" + if node.function_name == "MATCH": + if len(node.operands) not in {2, 3}: + raise ValueError("MATCH requires two or three operands") + return f"_sf_match({_render_function_arguments(node.operands)})" + if node.function_name == "INDEX": + if len(node.operands) not in {2, 3}: + raise ValueError("INDEX requires two or three operands") + return f"_sf_index({_render_function_arguments(node.operands)})" + if node.function_name == "VALUE": + if len(node.operands) != 1: + raise ValueError("VALUE requires one operand") + return f"_sf_excel_value({_render_expression(node.operands[0])})" + if node.function_name == "NUMBERVALUE": + if len(node.operands) not in {1, 2, 3}: + raise ValueError("NUMBERVALUE requires one to three operands") + return f"_sf_excel_numbervalue({_render_function_arguments(node.operands)})" + if node.function_name == "LN": + if len(node.operands) != 1: + raise ValueError("LN requires one operand") + return f"_sf_ln({_render_expression(node.operands[0])})" if node.function_name == "VLOOKUP": if len(node.operands) not in {3, 4}: raise ValueError("VLOOKUP requires three or four operands") @@ -1064,16 +1397,6 @@ def _render_table_array(node: FormulaExpressionNode) -> str: return f"_table({reference.sheet!r}, {min_col}, {min_row}, {max_col}, {max_row})" -def _python_comparison_operator(operator: str | None) -> str: - if operator == "=": - return "==" - if operator == "<>": - return "!=" - if operator is None: - raise ValueError("missing comparison operator") - return operator - - def _expand_range_dependency(reference) -> tuple[str, ...]: if reference.sheet is None or reference.start_cell is None or reference.end_cell is None: return () diff --git a/src/modelwright/graph.py b/src/modelwright/graph.py index 25958c2..d1b2a52 100644 --- a/src/modelwright/graph.py +++ b/src/modelwright/graph.py @@ -545,7 +545,7 @@ def _bracketed_parts(reference: str) -> tuple[str, ...]: if character == "]": depth -= 1 if depth == 0: - part = "".join(current) + part = "".join(current).strip() current = [] if part.startswith("[") and part.endswith("]"): parts.extend(_bracketed_parts(part)) @@ -560,7 +560,7 @@ def _bracketed_parts(reference: str) -> tuple[str, ...]: def _clean_structured_selector(selector: str) -> str: - return selector.removeprefix("@").replace("''", "'") + return selector.strip().removeprefix("@").replace("''", "'") def _table_containing_target(target: WorkbookReference, tables: dict[str, TableRecord]) -> TableRecord | None: diff --git a/src/modelwright/references.py b/src/modelwright/references.py index 22a9210..32cc0e0 100644 --- a/src/modelwright/references.py +++ b/src/modelwright/references.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from typing import Any, Literal -from openpyxl.utils.cell import range_boundaries +from openpyxl.utils.cell import get_column_letter, range_boundaries JsonValue = str | int | float | bool | None | list[Any] | dict[str, Any] @@ -207,3 +207,60 @@ def _column_name(index: int) -> str: index, remainder = divmod(index - 1, 26) name = chr(65 + remainder) + name return name + + +_CORRUPTED_STRUCTURED_RE = re.compile(r"([^\s\[\]]+)\[\]\s+\1\[", re.IGNORECASE) + + +def repair_corrupted_structured_references(raw_formula: str) -> str: + """Repair duplicated structured-reference table prefixes. + + Some workbooks carry a malformed prefix where a table name is written twice, + once as a dangling ``name[]`` token and again as the real structured + reference (``name[] name[[#This Row],[Column]]``). This is a source defect + in the workbook; the repair drops the dangling prefix so the remaining + structured reference parses normally. + """ + return _CORRUPTED_STRUCTURED_RE.sub(r"\1[", raw_formula) + + +_STATIC_INDIRECT_ADDRESS_RE = re.compile( + r"INDIRECT\s*\(\s*ADDRESS\s*\(\s*" + r"ROW\s*\(\s*\)\s*([+\-]\s*\d+)?\s*,\s*" + r"COLUMN\s*\(\s*\)\s*([+\-]\s*\d+)?\s*" + r"\)\s*\)", + re.IGNORECASE, +) + + +def static_indirect_cell_reference(cell_ref: str, raw_formula: str) -> str | None: + """Resolve a statically-computable ``INDIRECT(ADDRESS(ROW(), COLUMN()))``. + + Returns the target cell reference (e.g. ``Sheet!B4``) for the fixed pattern + ``INDIRECT(ADDRESS(ROW() +- k, COLUMN() +- k))`` that some workbooks use to + reference the cell directly above the formula, or ``None`` when the formula + does not match the pattern. + """ + match = _STATIC_INDIRECT_ADDRESS_RE.search(raw_formula) + if match is None: + return None + sheet_name, coordinate = cell_ref.rsplit("!", 1) + min_col, min_row, _max_col, _max_row = range_boundaries(coordinate) + row = min_row + _integer_offset(match.group(1)) + column = min_col + _integer_offset(match.group(2)) + if row < 1 or column < 1: + return None + return f"{sheet_name}!{get_column_letter(column)}{row}" + + +def cell_reference_coordinates(cell_ref: str) -> tuple[int, int]: + """Return ``(row, column)`` (1-based) for a cell reference string.""" + coordinate = cell_ref.rsplit("!", 1)[-1] + min_col, min_row, _max_col, _max_row = range_boundaries(coordinate) + return min_row, min_col + + +def _integer_offset(group: str | None) -> int: + if not group: + return 0 + return int(group.replace(" ", "")) diff --git a/tests/test_dependency_graph.py b/tests/test_dependency_graph.py index 3b03ebc..7b9705f 100644 --- a/tests/test_dependency_graph.py +++ b/tests/test_dependency_graph.py @@ -330,3 +330,44 @@ def test_dependency_graph_reports_simple_circular_dependency(tmp_path: Path) -> graph = build_dependency_graph(extract_workbook(workbook_path)) assert "circular_dependency" in graph.diagnostics + + +def test_dependency_graph_resolves_whitespace_current_row_structured_reference(tmp_path: Path) -> None: + workbook_path = tmp_path / "whitespace-current-row-structured-reference.xlsx" + source = Workbook() + sheet = source.active + sheet.title = "Data" + sheet.append(["Amount", "Result"]) + sheet.append([10, "=InputTable[ [#This Row],[Amount] ]"]) + sheet.append([20, None]) + sheet.add_table(Table(displayName="InputTable", ref="A1:B3")) + source.save(workbook_path) + + graph = build_dependency_graph(extract_workbook(workbook_path)) + execution_edges = [edge for edge in graph.execution_edges if edge.target.normalized == "Data!B2"] + + assert graph.diagnostics == () + assert [(edge.source.normalized, edge.resolved_from.normalized) for edge in execution_edges if edge.resolved_from] == [ + ("Data!A2", "InputTable[ [#This Row],[Amount] ]") + ] + + +def test_dependency_graph_resolves_whitespace_column_structured_reference_as_range(tmp_path: Path) -> None: + workbook_path = tmp_path / "whitespace-column-structured-reference.xlsx" + source = Workbook() + sheet = source.active + sheet.title = "Data" + sheet.append(["Amount", "Result"]) + sheet.append([10, None]) + sheet.append([20, None]) + sheet["D1"] = "=SUM(InputTable[ Amount ])" + sheet.add_table(Table(displayName="InputTable", ref="A1:B3")) + source.save(workbook_path) + + graph = build_dependency_graph(extract_workbook(workbook_path)) + execution_edges = [edge for edge in graph.execution_edges if edge.target.normalized == "Data!D1"] + + assert graph.diagnostics == () + assert [(edge.source.normalized, edge.resolved_from.normalized) for edge in execution_edges if edge.resolved_from] == [ + ("Data!A2:A3", "InputTable[ Amount ]") + ] diff --git a/tests/test_formula_translation.py b/tests/test_formula_translation.py index 8cc3426..adc86c1 100644 --- a/tests/test_formula_translation.py +++ b/tests/test_formula_translation.py @@ -337,7 +337,7 @@ def test_translate_unary_minus_exponent_and_concat(tmp_path: Path) -> None: assert concat.root.operator == "&" -def test_translate_ref_error_reports_sharp_diagnostic(tmp_path: Path) -> None: +def test_translate_ref_error_emits_sharp_literal(tmp_path: Path) -> None: workbook_path = tmp_path / "ref-error.xlsx" source = Workbook() sheet = source.active @@ -350,9 +350,9 @@ def test_translate_ref_error_reports_sharp_diagnostic(tmp_path: Path) -> None: expression = translate_formula_cell(formula_cell, graph) - assert expression.translated is False - assert expression.diagnostics[0].code == "unsupported_error_reference" - assert expression.diagnostics[0].raw_value == "#REF!" + assert expression.translated is True + assert expression.root.kind == "literal" + assert expression.root.value == "#REF!" def test_translate_static_offset_to_concrete_reference(tmp_path: Path) -> None: @@ -415,3 +415,156 @@ def test_translate_xlfn_ifna_as_ifna(tmp_path: Path) -> None: assert expression.root is not None assert expression.root.kind == "function_call" assert expression.root.function_name == "IFNA" + + +def test_translate_index_match_lookup(tmp_path: Path) -> None: + workbook_path = tmp_path / "index-match.xlsx" + source = Workbook() + sheet = source.active + sheet.title = "Data" + sheet.append(["Code", "Label", "Result"]) + sheet.append(["A", "Alpha", '=INDEX(InputTable[Label],MATCH(InputTable[[#This Row],[Code]],InputTable[Code],0),0)']) + sheet.append(["B", "Beta", None]) + sheet.add_table(Table(displayName="InputTable", ref="A1:C3")) + source.save(workbook_path) + workbook = extract_workbook(workbook_path) + graph = build_dependency_graph(workbook) + formula_cell = next(cell for cell in workbook.cells if cell.cell_ref == "Data!C2") + + expression = translate_formula_cell(formula_cell, graph, reference_index=build_formula_reference_index(graph)) + + assert expression.translated is True + assert expression.root is not None + assert expression.root.kind == "function_call" + assert expression.root.function_name == "INDEX" + assert expression.root.operands[0].kind == "reference" + assert expression.root.operands[0].reference.kind == "range" + assert expression.root.operands[0].reference.normalized == "Data!A2:B3" or expression.root.operands[0].reference.normalized == "Data!B2:B3" + assert expression.root.operands[1].kind == "function_call" + assert expression.root.operands[1].function_name == "MATCH" + + +def test_translate_conditional_aggregate_functions(tmp_path: Path) -> None: + workbook_path = tmp_path / "conditional-aggregates.xlsx" + source = Workbook() + sheet = source.active + sheet.title = "Data" + sheet.append(["Group", "Value"]) + sheet.append(["a", 1]) + sheet.append(["b", 2]) + sheet.append(["a", 3]) + sheet["E1"] = '=AVERAGEIFS(InputTable[Value],InputTable[Group],"a")' + sheet["E2"] = '=_xlfn.MINIFS(InputTable[Value],InputTable[Group],"b")' + sheet["E3"] = '=AVERAGEIF(InputTable[Value],">0")' + sheet.add_table(Table(displayName="InputTable", ref="A1:B4")) + source.save(workbook_path) + workbook = extract_workbook(workbook_path) + graph = build_dependency_graph(workbook) + reference_index = build_formula_reference_index(graph) + + for cell_ref, function_name in (("Data!E1", "AVERAGEIFS"), ("Data!E2", "MINIFS"), ("Data!E3", "AVERAGEIF")): + formula_cell = next(cell for cell in workbook.cells if cell.cell_ref == cell_ref) + expression = translate_formula_cell(formula_cell, graph, reference_index=reference_index) + assert expression.translated is True + assert expression.root.function_name == function_name + + +def test_translate_numeric_functions(tmp_path: Path) -> None: + workbook_path = tmp_path / "numeric-functions.xlsx" + source = Workbook() + sheet = source.active + sheet.title = "Calc" + sheet["A1"] = 4 + sheet["B1"] = "=VALUE(\"42\")" + sheet["B2"] = "=_xlfn.NUMBERVALUE(\"1.5\")" + sheet["B3"] = "=LN(A1)" + sheet["B4"] = "=VALUE(A1)" + source.save(workbook_path) + workbook = extract_workbook(workbook_path) + graph = build_dependency_graph(workbook) + reference_index = build_formula_reference_index(graph) + + expected = {"Calc!B1": "VALUE", "Calc!B2": "NUMBERVALUE", "Calc!B3": "LN", "Calc!B4": "VALUE"} + for cell_ref, function_name in expected.items(): + formula_cell = next(cell for cell in workbook.cells if cell.cell_ref == cell_ref) + expression = translate_formula_cell(formula_cell, graph, reference_index=reference_index) + assert expression.translated is True + assert expression.root.function_name == function_name + + +def test_translate_repaired_corrupted_structured_reference(tmp_path: Path) -> None: + workbook_path = tmp_path / "corrupted-structured.xlsx" + source = Workbook() + sheet = source.active + sheet.title = "Data" + sheet.append(["Country", "Product"]) + sheet.append(["CA", "Wheat"]) + sheet.append(["US", "Maize"]) + formula = ( + '=INDEX(calc_cropcosts[] calc_cropcosts[[#This Row],[Product]],' + 'MATCH(1,calc_cropcosts[[#This Row],[Country]],0))' + ) + sheet["D2"] = formula + sheet.add_table(Table(displayName="calc_cropcosts", ref="A1:C3")) + source.save(workbook_path) + workbook = extract_workbook(workbook_path) + graph = build_dependency_graph(workbook) + formula_cell = next(cell for cell in workbook.cells if cell.cell_ref == "Data!D2") + + expression = translate_formula_cell(formula_cell, graph, reference_index=build_formula_reference_index(graph)) + + assert expression.translated is True + assert expression.root is not None + assert expression.root.kind == "function_call" + assert expression.root.function_name == "INDEX" + assert expression.root.operands[0].kind == "reference" + assert expression.root.operands[0].reference.kind == "cell" + assert expression.root.operands[0].reference.normalized == "Data!B2" + + +def test_translate_static_indirect_address(tmp_path: Path) -> None: + workbook_path = tmp_path / "static-indirect.xlsx" + source = Workbook() + sheet = source.active + sheet.title = "Data" + sheet["A2"] = 42 + sheet["B2"] = "=INDIRECT(ADDRESS(ROW()-1,COLUMN()))" + sheet["B3"] = "=INDIRECT(ADDRESS(ROW(),COLUMN()-1))" + source.save(workbook_path) + workbook = extract_workbook(workbook_path) + graph = build_dependency_graph(workbook) + reference_index = build_formula_reference_index(graph) + + b2 = next(cell for cell in workbook.cells if cell.cell_ref == "Data!B2") + expression = translate_formula_cell(b2, graph, reference_index=reference_index) + assert expression.translated is True + assert expression.root is not None + assert expression.root.kind == "reference" + assert expression.root.reference is not None + assert expression.root.reference.normalized == "Data!B1" + + b3 = next(cell for cell in workbook.cells if cell.cell_ref == "Data!B3") + expression = translate_formula_cell(b3, graph, reference_index=reference_index) + assert expression.translated is True + assert expression.root is not None + assert expression.root.kind == "reference" + assert expression.root.reference is not None + assert expression.root.reference.normalized == "Data!A3" + + +def test_translate_static_indirect_address_with_unsupported_pattern(tmp_path: Path) -> None: + workbook_path = tmp_path / "nonstatic-indirect.xlsx" + source = Workbook() + sheet = source.active + sheet.title = "Data" + sheet["A1"] = "X" + sheet["B2"] = '=INDIRECT("A"&1)' + source.save(workbook_path) + workbook = extract_workbook(workbook_path) + graph = build_dependency_graph(workbook) + formula_cell = next(cell for cell in workbook.cells if cell.cell_ref == "Data!B2") + + expression = translate_formula_cell(formula_cell, graph, reference_index=build_formula_reference_index(graph)) + + assert expression.translated is False + assert expression.diagnostics[0].code == "unsupported_function" diff --git a/tests/test_openpyxl_extraction.py b/tests/test_openpyxl_extraction.py index d0c0ebb..a411cd4 100644 --- a/tests/test_openpyxl_extraction.py +++ b/tests/test_openpyxl_extraction.py @@ -231,3 +231,53 @@ def test_extract_workbook_reads_sparse_populated_cells_without_dense_scan(tmp_pa workbook = extract_workbook(workbook_path) assert [cell.cell_ref for cell in workbook.cells] == ["Inputs!A1", "Inputs!CV5000"] + + +def test_extract_sheet_cells_reads_array_formula_text() -> None: + from openpyxl.worksheet.formula import ArrayFormula + + formula_workbook = Workbook() + formula_sheet = formula_workbook.active + formula_sheet.title = "Data" + formula_sheet["A1"] = ArrayFormula("A1:A1", "=SUM(A2:A3)") + + cached_workbook = Workbook() + cached_sheet = cached_workbook.active + cached_sheet.title = "Data" + cached_sheet["A1"] = 42 + + records = _extract_sheet_cells( + formula_sheet, + cached_sheet, + populated_cells=("A1",), + ) + cells = {cell.cell_ref: cell for cell in records} + + assert cells["Data!A1"].kind == "formula" + assert cells["Data!A1"].raw_value == "=SUM(A2:A3)" + assert cells["Data!A1"].formula is not None + assert cells["Data!A1"].formula.raw_formula == "=SUM(A2:A3)" + assert cells["Data!A1"].cached_value == 42 + + +def test_extract_sheet_cells_adds_static_indirect_reference() -> None: + formula_workbook = Workbook() + formula_sheet = formula_workbook.active + formula_sheet.title = "Data" + formula_sheet["B10"] = "=INDIRECT(ADDRESS(ROW()-1,COLUMN()))" + formula_sheet["A2"] = "=SUM(A1:A2)" + + cached_workbook = Workbook() + cached_sheet = cached_workbook.active + cached_sheet.title = "Data" + cached_sheet["B10"] = 7 + + records = _extract_sheet_cells( + formula_sheet, + cached_sheet, + populated_cells=("B10",), + ) + cells = {cell.cell_ref: cell for cell in records} + + assert cells["Data!B10"].formula is not None + assert "Data!B9" in cells["Data!B10"].formula.raw_references diff --git a/tests/test_python_generation.py b/tests/test_python_generation.py index a793a0c..8974e20 100644 --- a/tests/test_python_generation.py +++ b/tests/test_python_generation.py @@ -1434,3 +1434,303 @@ def test_generate_python_module_reuses_range_views_without_changing_results(tmp_ assert "def values(self):" in result.source_code assert "def lazy_values(self):" in result.source_code assert module.calculate() == {"Calc!C1": 2, "Calc!C2": 5, "Calc!C3": 1} + + +def test_generate_python_module_returns_excel_error_strings_for_invalid_arithmetic(tmp_path: Path) -> None: + contract = GeneratedModuleContract( + workbook_id="arithmetic-errors.xlsx", + module_name="arithmetic_errors", + input_refs=(), + output_refs=("Calc!B1", "Calc!B2", "Calc!B3", "Calc!B4", "Calc!B5"), + symbols=( + GeneratedSymbol(cell_ref="Calc!B1", symbol_name="calc_b1", kind="output", raw_formula="=1/0"), + GeneratedSymbol(cell_ref="Calc!B2", symbol_name="calc_b2", kind="output", raw_formula='="abc"+1'), + GeneratedSymbol(cell_ref="Calc!B3", symbol_name="calc_b3", kind="output", raw_formula="=7/0"), + GeneratedSymbol(cell_ref="Calc!B4", symbol_name="calc_b4", kind="output", raw_formula="=0^0.5"), + GeneratedSymbol(cell_ref="Calc!B5", symbol_name="calc_b5", kind="output", raw_formula="=1.5+1"), + ), + ) + expressions = { + "Calc!B1": formula_expression( + "Calc!B1", + "=1/0", + FormulaExpressionNode.binary("/", FormulaExpressionNode.literal(1), FormulaExpressionNode.literal(0)), + ), + "Calc!B2": formula_expression( + "Calc!B2", + '="abc"+1', + FormulaExpressionNode.binary("+", FormulaExpressionNode.literal("abc"), FormulaExpressionNode.literal(1)), + ), + "Calc!B3": formula_expression( + "Calc!B3", + "=7/0", + FormulaExpressionNode.binary("/", FormulaExpressionNode.literal(7), FormulaExpressionNode.literal(0)), + ), + "Calc!B4": formula_expression( + "Calc!B4", + "=0^0.5", + FormulaExpressionNode.binary("^", FormulaExpressionNode.literal(0), FormulaExpressionNode.literal(0.5)), + ), + "Calc!B5": formula_expression( + "Calc!B5", + "=1.5+1", + FormulaExpressionNode.binary("+", FormulaExpressionNode.literal(1.5), FormulaExpressionNode.literal(1)), + ), + } + output_path = tmp_path / "generated_arithmetic_errors.py" + + result = generate_python_module( + contract=contract, + expressions=expressions, + constants={}, + output_path=output_path, + ) + module = load_module(output_path) + + assert result.generated is True + assert module.calculate() == { + "Calc!B1": "#DIV/0!", + "Calc!B2": "#VALUE!", + "Calc!B3": "#DIV/0!", + "Calc!B4": 0, + "Calc!B5": 2.5, + } + + +def test_generate_python_module_coerces_numeric_strings_in_arithmetic_and_comparisons(tmp_path: Path) -> None: + contract = GeneratedModuleContract( + workbook_id="numeric-string-coercion.xlsx", + module_name="numeric_string_coercion", + input_refs=("Data!A1", "Data!A2"), + output_refs=("Calc!B1", "Calc!B2"), + symbols=( + GeneratedSymbol(cell_ref="Data!A1", symbol_name="data_a1", kind="input"), + GeneratedSymbol(cell_ref="Data!A2", symbol_name="data_a2", kind="input"), + GeneratedSymbol(cell_ref="Calc!B1", symbol_name="calc_b1", kind="output", raw_formula="=Data!A1<=2015"), + GeneratedSymbol(cell_ref="Calc!B2", symbol_name="calc_b2", kind="output", raw_formula="=Data!A2*2"), + ), + ) + expressions = { + "Calc!B1": formula_expression( + "Calc!B1", + "=Data!A1<=2015", + FormulaExpressionNode.comparison( + "<=", + FormulaExpressionNode.reference_to(normalize_reference("Data!A1")), + FormulaExpressionNode.literal(2015), + ), + ), + "Calc!B2": formula_expression( + "Calc!B2", + "=Data!A2*2", + FormulaExpressionNode.binary( + "*", + FormulaExpressionNode.reference_to(normalize_reference("Data!A2")), + FormulaExpressionNode.literal(2), + ), + ), + } + output_path = tmp_path / "generated_numeric_string_coercion.py" + + result = generate_python_module( + contract=contract, + expressions=expressions, + constants={}, + output_path=output_path, + ) + module = load_module(output_path) + + assert result.generated is True + assert "_sf_compare(" in result.source_code + assert module.calculate(inputs={"Data!A1": "2015", "Data!A2": "2.5"}) == { + "Calc!B1": True, + "Calc!B2": 5.0, + } + + +def test_generate_python_module_ln_and_value_return_error_strings(tmp_path: Path) -> None: + contract = GeneratedModuleContract( + workbook_id="math-error-strings.xlsx", + module_name="math_error_strings", + input_refs=("Data!A1", "Data!A2"), + output_refs=("Calc!B1", "Calc!B2", "Calc!B3"), + symbols=( + GeneratedSymbol(cell_ref="Data!A1", symbol_name="data_a1", kind="input"), + GeneratedSymbol(cell_ref="Data!A2", symbol_name="data_a2", kind="input"), + GeneratedSymbol(cell_ref="Calc!B1", symbol_name="calc_b1", kind="output", raw_formula="=LN(Data!A1)"), + GeneratedSymbol(cell_ref="Calc!B2", symbol_name="calc_b2", kind="output", raw_formula="=LN(Data!A2)"), + GeneratedSymbol(cell_ref="Calc!B3", symbol_name="calc_b3", kind="output", raw_formula='=VALUE("12,5")'), + ), + ) + expressions = { + "Calc!B1": formula_expression( + "Calc!B1", + "=LN(Data!A1)", + FormulaExpressionNode.function_call( + "LN", + (FormulaExpressionNode.reference_to(normalize_reference("Data!A1")),), + ), + ), + "Calc!B2": formula_expression( + "Calc!B2", + "=LN(Data!A2)", + FormulaExpressionNode.function_call( + "LN", + (FormulaExpressionNode.reference_to(normalize_reference("Data!A2")),), + ), + ), + "Calc!B3": formula_expression( + "Calc!B3", + '=VALUE("12,5")', + FormulaExpressionNode.function_call("VALUE", (FormulaExpressionNode.literal("12,5"),)), + ), + } + output_path = tmp_path / "generated_math_error_strings.py" + + result = generate_python_module( + contract=contract, + expressions=expressions, + constants={}, + output_path=output_path, + ) + module = load_module(output_path) + + assert result.generated is True + assert module.calculate(inputs={"Data!A1": 0, "Data!A2": "4"}) == { + "Calc!B1": "#NUM!", + "Calc!B2": 1.3862943611198906, + "Calc!B3": 12.5, + } + + +def test_generate_python_module_index_resolves_range_views(tmp_path: Path) -> None: + contract = GeneratedModuleContract( + workbook_id="index-ranges.xlsx", + module_name="index_ranges", + output_refs=("Calc!C1", "Calc!C2", "Calc!C3", "Calc!C4"), + symbols=( + GeneratedSymbol(cell_ref="Data!A1", symbol_name="data_a1", kind="input"), + GeneratedSymbol(cell_ref="Data!A2", symbol_name="data_a2", kind="input"), + GeneratedSymbol(cell_ref="Data!A3", symbol_name="data_a3", kind="input"), + GeneratedSymbol(cell_ref="Data!A4", symbol_name="data_a4", kind="input"), + GeneratedSymbol(cell_ref="Data!B1", symbol_name="data_b1", kind="input"), + GeneratedSymbol(cell_ref="Data!B2", symbol_name="data_b2", kind="input"), + GeneratedSymbol(cell_ref="Calc!C1", symbol_name="calc_c1", kind="output", raw_formula="=INDEX(Data!A1:A4,3)"), + GeneratedSymbol(cell_ref="Calc!C2", symbol_name="calc_c2", kind="output", raw_formula="=INDEX(Data!A1:B2,2,1)"), + GeneratedSymbol(cell_ref="Calc!C3", symbol_name="calc_c3", kind="output", raw_formula="=INDEX(Data!A1:A4,9)"), + GeneratedSymbol(cell_ref="Calc!C4", symbol_name="calc_c4", kind="output", raw_formula="=INDEX(Data!A1:A4,2)"), + ), + ) + single_column = normalize_reference("Data!A1:A4") + two_column = normalize_reference("Data!A1:B2") + expressions = { + "Calc!C1": formula_expression( + "Calc!C1", + "=INDEX(Data!A1:A4,3)", + FormulaExpressionNode.function_call( + "INDEX", + (FormulaExpressionNode.reference_to(single_column), FormulaExpressionNode.literal(3)), + ), + ), + "Calc!C2": formula_expression( + "Calc!C2", + "=INDEX(Data!A1:B2,2,1)", + FormulaExpressionNode.function_call( + "INDEX", + ( + FormulaExpressionNode.reference_to(two_column), + FormulaExpressionNode.literal(2), + FormulaExpressionNode.literal(1), + ), + ), + ), + "Calc!C3": formula_expression( + "Calc!C3", + "=INDEX(Data!A1:A4,9)", + FormulaExpressionNode.function_call( + "INDEX", + (FormulaExpressionNode.reference_to(single_column), FormulaExpressionNode.literal(9)), + ), + ), + "Calc!C4": formula_expression( + "Calc!C4", + "=INDEX(Data!A1:A4,2)", + FormulaExpressionNode.function_call( + "INDEX", + (FormulaExpressionNode.reference_to(single_column), FormulaExpressionNode.literal(2)), + ), + ), + } + output_path = tmp_path / "generated_index_ranges.py" + + result = generate_python_module( + contract=contract, + expressions=expressions, + constants={"Data!A1": 10, "Data!A2": 20, "Data!A3": 30, "Data!A4": 40, "Data!B1": 1, "Data!B2": 2}, + output_path=output_path, + ) + module = load_module(output_path) + + assert result.generated is True + assert module.calculate() == { + "Calc!C1": 30, + "Calc!C2": 20, + "Calc!C3": "#REF!", + "Calc!C4": 20, + } + + +def test_generate_python_module_sum_propagates_errors_and_ignores_text(tmp_path: Path) -> None: + contract = GeneratedModuleContract( + workbook_id="sum-errors.xlsx", + module_name="sum_errors", + input_refs=("Data!A1", "Data!A2", "Data!A3", "Data!B1", "Data!B2"), + output_refs=("Calc!C1", "Calc!C2", "Calc!C3"), + symbols=( + GeneratedSymbol(cell_ref="Data!A1", symbol_name="data_a1", kind="input"), + GeneratedSymbol(cell_ref="Data!A2", symbol_name="data_a2", kind="input"), + GeneratedSymbol(cell_ref="Data!A3", symbol_name="data_a3", kind="input"), + GeneratedSymbol(cell_ref="Data!B1", symbol_name="data_b1", kind="input"), + GeneratedSymbol(cell_ref="Data!B2", symbol_name="data_b2", kind="input"), + GeneratedSymbol(cell_ref="Calc!C1", symbol_name="calc_c1", kind="output", raw_formula="=SUM(Data!B1:B2)"), + GeneratedSymbol(cell_ref="Calc!C2", symbol_name="calc_c2", kind="output", raw_formula="=SUM(Data!A1:A3)"), + GeneratedSymbol(cell_ref="Calc!C3", symbol_name="calc_c3", kind="output", raw_formula="=SUM(Data!A1:B2)"), + ), + ) + range_a = normalize_reference("Data!A1:A3") + range_b = normalize_reference("Data!B1:B2") + range_ab = normalize_reference("Data!A1:B2") + expressions = { + "Calc!C1": formula_expression( + "Calc!C1", + "=SUM(Data!B1:B2)", + FormulaExpressionNode.function_call("SUM", (FormulaExpressionNode.reference_to(range_b),)), + ), + "Calc!C2": formula_expression( + "Calc!C2", + "=SUM(Data!A1:A3)", + FormulaExpressionNode.function_call("SUM", (FormulaExpressionNode.reference_to(range_a),)), + ), + "Calc!C3": formula_expression( + "Calc!C3", + "=SUM(Data!A1:B2)", + FormulaExpressionNode.function_call("SUM", (FormulaExpressionNode.reference_to(range_ab),)), + ), + } + output_path = tmp_path / "generated_sum_errors.py" + + result = generate_python_module( + contract=contract, + expressions=expressions, + constants={"Data!A1": 1, "Data!A2": 2, "Data!A3": "#N/A", "Data!B1": 3, "Data!B2": "x"}, + output_path=output_path, + ) + module = load_module(output_path) + + assert result.generated is True + assert "_sf_sum(" in result.source_code + assert module.calculate() == { + "Calc!C1": 3, + "Calc!C2": "#N/A", + "Calc!C3": 6, + } From 2815b06ebcb4d92e2d603482a5f6e75a151f69e3 Mon Sep 17 00:00:00 2001 From: Gregory Paradis Date: Sat, 15 Aug 2026 20:29:04 +0000 Subject: [PATCH 2/2] Pin ruff below 0.16 in quality and dev extras CI picked up ruff 0.16.3 via the unbounded ruff>=0.8 constraint; its new lint rules flag 139 pre-existing violations repo-wide. Constrain ruff to >=0.8,<0.16 so the quality job matches the version the codebase passes cleanly against. --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d585d73..2929635 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,7 @@ dev = [ "formulas", "pandas>=2", "pytest>=8", - "ruff>=0.8", + "ruff>=0.8,<0.16", "sphinx>=7", "sphinx-rtd-theme>=2", "twine>=5" @@ -77,7 +77,7 @@ oracle = [ "formulas" ] quality = [ - "ruff>=0.8" + "ruff>=0.8,<0.16" ] release = [ "build>=1.2",