diff --git a/common/log_parser/apply_waivers.py b/common/log_parser/apply_waivers.py index 94ad92f0..d79599da 100755 --- a/common/log_parser/apply_waivers.py +++ b/common/log_parser/apply_waivers.py @@ -14,23 +14,152 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Apply waiver files to parsed ACS JSON results.""" + import json -import sys import re import argparse def clean_description(desc): + """Normalize a description string for older description-based waiver matching.""" desc = desc.strip().lower() desc = re.sub(r'\s+', ' ', desc) # Replace multiple spaces with a single space desc = re.sub(r'[^\w\s-]', '', desc) # Remove special characters except hyphens return desc -def _as_list(v): - if v is None: +def _as_list(value): + """Return a value as a list so callers can handle one or many entries.""" + if value is None: return [] - return v if isinstance(v, list) else [v] + return value if isinstance(value, list) else [value] + +# New BSA/SBSA JSON no longer needs a separate sub_Rule_ID field. When only +# sub_Test_Number is present, the rule id is the text before the first ':'. +def _rule_id_from_test_number(test_number): + """Extract the rule id from a visible test number such as 'ITS_04 : 1535'.""" + if not isinstance(test_number, str): + return '' + return test_number.split(':', 1)[0].strip() + +def _subtest_rule_id(subtest): + """Return the best available rule id for old and new subtest JSON.""" + return subtest.get('sub_Rule_ID') or _rule_id_from_test_number(subtest.get('sub_Test_Number')) + +# Walk waiver input and parser output the same way. Both "SubTests" from waiver +# files and "subtests" from parsed JSON can now contain children at any depth. +def _iter_nested_subtests(subtests): + """Yield every subtest from a recursive subtest tree.""" + for subtest in subtests or []: + yield subtest + yield from _iter_nested_subtests( + subtest.get('subtests', []) or subtest.get('SubTests', []) + ) + +def _is_failed_result(result): + """Return True when a result string represents any failed status.""" + if not isinstance(result, str): + return False + result_upper = result.upper() + return 'FAILED' in result_upper or 'FAILURE' in result_upper or 'FAIL' in result_upper + +def _has_waiver_result(result): + """Return True when a result string is already marked with waiver status.""" + return isinstance(result, str) and 'WITH WAIVER' in result.upper() + +def _append_waiver_to_result(result): + """Add the waiver suffix once to an existing result string.""" + if not isinstance(result, str) or _has_waiver_result(result): + return result + return result + ' (WITH WAIVER)' + +# BSA/SBSA use Test_result, while some older parsers use test_result. Keep the +# marking in one place so suite/testsuite/testcase waivers behave consistently. +def _mark_failed_case_waived(testcase, reason): + """Mark a failed testcase as waived when it is not already waived.""" + result_key = 'Test_result' if 'Test_result' in testcase else 'test_result' + test_result = testcase.get(result_key, '') + if _is_failed_result(test_result) and not _has_waiver_result(test_result): + testcase[result_key] = _append_waiver_to_result(test_result) + testcase['waiver_reason'] = reason + return True + return False + +# Suite, testsuite, and testcase waivers apply to the whole failed branch. Mark +# every failed nested child so HTML and JSON show the waiver at the exact rule. +def _mark_failed_subtests_waived(subtests, reason): + """Mark every failed subtest in a nested branch as waived.""" + marked = False + for subtest in _iter_nested_subtests(subtests): + sub_result = subtest.get('sub_test_result') + if _is_failed_result(sub_result) and not _has_waiver_result(sub_result): + subtest['sub_test_result'] = _append_waiver_to_result(sub_result) + subtest['waiver_reason'] = reason + marked = True + return marked + +# If a failed parent only contains waived failed children, mark that parent as +# failed-with-waiver too. Passing/skipped/not-implemented children do not block +# this because they are not unwaived failures. +def _propagate_nested_bsa_waivers(testcase): + """Move waiver state upward through failed BSA/SBSA nested parents.""" + def propagate_subtest(subtest): + """Return failed, unwaived failed, and waived failed state for one branch.""" + child_has_failed = False + child_has_unwaived_failed = False + child_has_waived_failed = False + + for child in subtest.get('subtests', []) or []: + child_failed, child_unwaived, child_waived = propagate_subtest(child) + child_has_failed = child_has_failed or child_failed + child_has_unwaived_failed = child_has_unwaived_failed or child_unwaived + child_has_waived_failed = child_has_waived_failed or child_waived + + result = subtest.get('sub_test_result', '') + is_failed = _is_failed_result(result) + has_waiver = _has_waiver_result(result) + + if ( + is_failed + and not has_waiver + and subtest.get('subtests') + and child_has_waived_failed + and not child_has_unwaived_failed + ): + subtest['sub_test_result'] = _append_waiver_to_result(result) + subtest.setdefault('waiver_reason', 'All failed nested subtests were waived.') + has_waiver = True + + return ( + is_failed or child_has_failed, + (is_failed and not has_waiver) or child_has_unwaived_failed, + (is_failed and has_waiver) or child_has_waived_failed + ) + + has_failed = False + has_unwaived_failed = False + has_waived_failed = False + + for subtest in testcase.get('subtests', []) or []: + sub_failed, sub_unwaived, sub_waived = propagate_subtest(subtest) + has_failed = has_failed or sub_failed + has_unwaived_failed = has_unwaived_failed or sub_unwaived + has_waived_failed = has_waived_failed or sub_waived + + test_result = testcase.get('Test_result', testcase.get('test_result', '')) + if ( + _is_failed_result(test_result) + and not _has_waiver_result(test_result) + and has_failed + and has_waived_failed + and not has_unwaived_failed + ): + _mark_failed_case_waived( + testcase, + 'All failed nested subtests were waived.' + ) def load_waivers(waiver_data, suite_name): + """Collect waiver entries by scope for the requested suite.""" suite_level_waivers = [] testsuite_level_waivers = [] subsuite_level_waivers = [] # Added for SubSuite-level waivers @@ -80,20 +209,26 @@ def load_waivers(waiver_data, suite_name): # Also load SubTest-level waivers for this testcase if 'SubTests' in testcase_entry: - for subtest_entry in testcase_entry['SubTests']: + # Nested waiver entries are flattened here, but + # each entry still keeps its testcase scope. + for subtest_entry in _iter_nested_subtests(testcase_entry['SubTests']): subtest_id = subtest_entry.get('sub_Rule_ID') + subtest_number = subtest_entry.get('sub_Test_Number') + subtest_path = subtest_entry.get('sub_Test_Path') subtest_desc = subtest_entry.get('sub_Test_Description') subtest_reason = subtest_entry.get('Reason') - if (subtest_id or subtest_desc) and subtest_reason: + if (subtest_id or subtest_number or subtest_path or subtest_desc) and subtest_reason: subtest_level_waivers.append({ 'Test_case': testcase_name, 'sub_Rule_ID': subtest_id, + 'sub_Test_Number': subtest_number, + 'sub_Test_Path': subtest_path, 'sub_Test_Description': subtest_desc, 'Reason': subtest_reason }) - elif (subtest_id or subtest_desc) and not subtest_reason: + elif (subtest_id or subtest_number or subtest_path or subtest_desc) and not subtest_reason: if verbose: - subtest_key = subtest_id or subtest_desc + subtest_key = subtest_path or subtest_number or subtest_id or subtest_desc print(f"ERROR: Waiver for SubTest '{subtest_key}' under Test_case '{testcase_name}' is missing 'Reason'. Skipping SubTest-level waiver.") if suite_name.upper() == 'SCMI': if 'TestCases' in test_suite: @@ -143,7 +278,7 @@ def load_waivers(waiver_data, suite_name): # Collect SubTest-level waivers subtests = test_suite.get('TestCase', {}).get('SubTests', []) or test_suite.get('SubSuite', {}).get('TestCase', {}).get('SubTests', []) - for subtest in subtests: + for subtest in _iter_nested_subtests(subtests): if 'Reason' not in subtest or not subtest['Reason']: subtest_id_or_desc = subtest.get('SubTestID') or subtest.get('sub_Test_Description') or 'Unknown' if verbose: @@ -175,19 +310,17 @@ def load_waivers(waiver_data, suite_name): return suite_level_waivers, testsuite_level_waivers, subsuite_level_waivers, testcase_level_waivers, subtest_level_waivers def apply_suite_level_waivers(test_suite_entry, suite_waivers): + """Apply suite-level waivers to failed results below a suite.""" # Apply waivers to all applicable failed subtests in the suite for waiver in suite_waivers: reason = waiver['Reason'] # Apply to testcases (BSA/SBSA/SCMI) for testcase in test_suite_entry.get('testcases', []): - test_result = testcase.get('test_result', '') - if isinstance(test_result, str): - if 'FAILED' in test_result.upper() and '(WITH WAIVER)' not in test_result.upper(): - testcase['test_result'] = test_result + ' (WITH WAIVER)' - testcase['waiver_reason'] = reason - if verbose: - print(f"Suite-level waiver applied to testcase '{testcase.get('Test_case')}' with reason: {reason}") + if _mark_failed_case_waived(testcase, reason): + _mark_failed_subtests_waived(testcase.get('subtests', []), reason) + if verbose: + print(f"Suite-level waiver applied to testcase '{testcase.get('Test_case')}' with reason: {reason}") # For other structures: apply to subtests for subtest in test_suite_entry.get('subtests', []): @@ -229,13 +362,14 @@ def apply_suite_level_waivers(test_suite_entry, suite_waivers): updated_fail_reasons = [fr + ' (WITH WAIVER)' for fr in existing_fail_reasons] sub_test_result['fail_reasons'] = updated_fail_reasons elif isinstance(sub_test_result, str): - ru = sub_test_result.upper() - if 'FAILED' in ru or 'FAILURE' in ru or 'FAIL' in ru: - if '(WITH WAIVER)' not in ru: + result_upper = sub_test_result.upper() + if 'FAILED' in result_upper or 'FAILURE' in result_upper or 'FAIL' in result_upper: + if '(WITH WAIVER)' not in result_upper: subtest['sub_test_result'] += ' (WITH WAIVER)' subtest['waiver_reason'] = reason def apply_testsuite_level_waivers(test_suite_entry, testsuite_waivers): + """Apply testsuite-level waivers to matching failed testcases or subtests.""" # Get the test_suite_name considering different keys test_suite_name = test_suite_entry.get('Test_suite') or test_suite_entry.get('Test_suite_name') # Apply waivers to all applicable failed tests within specific TestSuites @@ -245,13 +379,10 @@ def apply_testsuite_level_waivers(test_suite_entry, testsuite_waivers): if test_suite_name == target_testsuite: # For BSA/SBSA: apply to testcases for testcase in test_suite_entry.get('testcases', []): - test_result = testcase.get('Test_result', '') - if isinstance(test_result, str): - if 'FAILED' in test_result.upper() and '(WITH WAIVER)' not in test_result.upper(): - testcase['Test_result'] = test_result + ' (WITH WAIVER)' - testcase['waiver_reason'] = reason - if verbose: - print(f"TestSuite-level waiver applied to testcase '{testcase.get('Test_case')}' in TestSuite '{target_testsuite}' with reason: {reason}") + if _mark_failed_case_waived(testcase, reason): + _mark_failed_subtests_waived(testcase.get('subtests', []), reason) + if verbose: + print(f"TestSuite-level waiver applied to testcase '{testcase.get('Test_case')}' in TestSuite '{target_testsuite}' with reason: {reason}") # For other structures: apply to subtests for subtest in test_suite_entry.get('subtests', []): @@ -295,13 +426,14 @@ def apply_testsuite_level_waivers(test_suite_entry, testsuite_waivers): updated_fail_reasons = [fr + ' (WITH WAIVER)' for fr in existing_fail_reasons] sub_test_result['fail_reasons'] = updated_fail_reasons elif isinstance(sub_test_result, str): - ru = sub_test_result.upper() - if 'FAILED' in ru or 'FAILURE' in ru: - if '(WITH WAIVER)' not in ru: + result_upper = sub_test_result.upper() + if 'FAILED' in result_upper or 'FAILURE' in result_upper: + if '(WITH WAIVER)' not in result_upper: subtest['sub_test_result'] += ' (WITH WAIVER)' subtest['waiver_reason'] = reason def apply_subsuite_level_waivers(test_suite_entry, subsuite_waivers): + """Apply subsuite-level waivers where the parsed JSON has subsuite results.""" # Apply waivers to all applicable failed subtests within specific SubSuites for waiver in subsuite_waivers: target_subsuite = waiver['SubSuite'] @@ -331,6 +463,7 @@ def apply_subsuite_level_waivers(test_suite_entry, subsuite_waivers): print(f"SubSuite-level waiver applied to subtest '{subtest.get('sub_Test_Description')}' with reason: {reason}") def apply_testcase_level_waivers(test_suite_entry, testcase_waivers): + """Apply testcase-level waivers to matching failed testcases.""" # Apply waivers to all applicable failed subtests within specific Test_cases for waiver in testcase_waivers: target_testcase = waiver['Test_case'] @@ -343,13 +476,10 @@ def apply_testcase_level_waivers(test_suite_entry, testcase_waivers): test_case_id = test_case_name.split(':')[0].strip() if ':' in test_case_name else test_case_name if test_case_id == target_testcase or test_case_name == target_testcase: - test_result = testcase.get('Test_result', '') - if isinstance(test_result, str): - if 'FAILED' in test_result.upper() and '(WITH WAIVER)' not in test_result.upper(): - testcase['Test_result'] = test_result + ' (WITH WAIVER)' - testcase['waiver_reason'] = reason - if verbose: - print(f"Test_case-level waiver applied to testcase '{test_case_name}' with reason: {reason}") + if _mark_failed_case_waived(testcase, reason): + _mark_failed_subtests_waived(testcase.get('subtests', []), reason) + if verbose: + print(f"Test_case-level waiver applied to testcase '{test_case_name}' with reason: {reason}") # For other structures: apply to subtests if test_suite_entry.get('Test_case') == target_testcase: @@ -394,21 +524,26 @@ def apply_testcase_level_waivers(test_suite_entry, testcase_waivers): updated_fail_reasons = [fr + ' (WITH WAIVER)' for fr in existing_fail_reasons] sub_test_result['fail_reasons'] = updated_fail_reasons elif isinstance(sub_test_result, str): - ru = sub_test_result.upper() - if 'FAILED' in ru or 'FAILURE' in ru: - if '(WITH WAIVER)' not in ru: + result_upper = sub_test_result.upper() + if 'FAILED' in result_upper or 'FAILURE' in result_upper: + if '(WITH WAIVER)' not in result_upper: subtest['sub_test_result'] += ' (WITH WAIVER)' subtest['waiver_reason'] = reason def apply_subtest_level_waivers(test_suite_entry, subtest_waivers, suite_name): + """Apply subtest-level waivers, including nested BSA/SBSA subtests.""" # For BSA/SBSA: apply waivers to subtests within testcases if suite_name.upper() in ['BSA', 'SBSA']: for testcase in test_suite_entry.get('testcases', []): testcase_name = testcase.get('Test_case', '') testcase_id = testcase_name.split(':')[0].strip() if ':' in testcase_name else testcase_name - for subtest in testcase.get('subtests', []): - sub_rule_id = subtest.get('sub_Rule_ID', '') + # Check every nested subtest, not only direct children of the + # testcase, so deeper SBSA/BSA rule failures can be waived. + for subtest in _iter_nested_subtests(testcase.get('subtests', [])): + sub_rule_id = _subtest_rule_id(subtest) + sub_test_number = subtest.get('sub_Test_Number', '') + sub_test_path = subtest.get('sub_Test_Path', '') sub_test_desc = subtest.get('sub_Test_Description', '') sub_test_result = subtest.get('sub_test_result', '') @@ -421,13 +556,22 @@ def apply_subtest_level_waivers(test_suite_entry, subtest_waivers, suite_name): if waiver_testcase_id != testcase_id and waiver_testcase != testcase_id: continue - # Match by sub_Rule_ID or sub_Test_Description + # sub_Test_Path is safest for nested logs because the same + # sub_Test_Number can appear in different parent branches. + # Number, rule id, and description are kept as fallbacks for + # old waiver files and hand-written waivers. waiver_id = waiver.get('sub_Rule_ID', '') + waiver_number = waiver.get('sub_Test_Number', '') + waiver_path = waiver.get('sub_Test_Path', '') waiver_desc = waiver.get('sub_Test_Description', '') reason = waiver.get('Reason', '') match = False - if waiver_id and waiver_id == sub_rule_id: + if waiver_path and waiver_path == sub_test_path: + match = True + elif waiver_number and waiver_number == sub_test_number: + match = True + elif waiver_id and waiver_id == sub_rule_id: match = True elif waiver_desc and waiver_desc == sub_test_desc: match = True @@ -627,32 +771,33 @@ def apply_subtest_level_waivers(test_suite_entry, subtest_waivers, suite_name): break def apply_waivers(suite_name, json_file, waiver_file='waiver.json', output_json_file=None): + """Apply all matching waivers to one parsed JSON file.""" # Load the JSON data try: - with open(json_file, 'r') as f: - json_data = json.load(f) - except Exception as e: + with open(json_file, 'r', encoding='utf-8') as json_handle: + json_data = json.load(json_handle) + except Exception as err: if verbose: - print(f"WARNING: Failed to read or parse {json_file}: {e}") + print(f"WARNING: Failed to read or parse {json_file}: {err}") return # Load waiver.json try: - with open(waiver_file, 'r') as f: - waiver_data = json.load(f) - except Exception as e: + with open(waiver_file, 'r', encoding='utf-8') as waiver_handle: + waiver_data = json.load(waiver_handle) + except Exception as err: if verbose: - print(f"INFO: Failed to read or parse {waiver_file}: {e}") + print(f"INFO: Failed to read or parse {waiver_file}: {err}") return # Load test_category.json if provided if output_json_file: try: - with open(output_json_file, 'r') as f: - output_json_data = json.load(f) - except Exception as e: + with open(output_json_file, 'r', encoding='utf-8') as output_handle: + output_json_data = json.load(output_handle) + except Exception as err: if verbose: - print(f"WARNING: Failed to read or parse {output_json_file}: {e}") + print(f"WARNING: Failed to read or parse {output_json_file}: {err}") output_json_data = None else: output_json_data = None @@ -692,9 +837,9 @@ def apply_waivers(suite_name, json_file, waiver_file='waiver.json', output_json_ else: # Check if the test suite is waivable according to test_category.json waivable = False - for catID, catData in output_json_data.items(): - # catData is a list - for row in catData: + for _category_id, category_rows in output_json_data.items(): + # category_rows is a list from test_category.json. + for row in category_rows: if row.get("Suite", "").lower() == suite_name.lower() and row.get("Test Suite", "").lower() == test_suite_name.lower(): if row.get("Waivable", "").lower() == "yes": waivable = True @@ -741,36 +886,12 @@ def apply_waivers(suite_name, json_file, waiver_file='waiver.json', output_json_ if subtest_level_waivers: apply_subtest_level_waivers(test_suite_entry, subtest_level_waivers, suite_name) - # === FIX: Recompute testcase-level Test_result based on subtests (BSA/SBSA) === - # After waivers are applied to subtests, testcase result must reflect the actual status + # Subtest waivers are applied at the leaf/branch where they match. This + # pass then updates failed parents only when no failed child remains + # unwaived, keeping mixed waived/unwaived failures visible. if suite_name.upper() in ('BSA', 'SBSA'): for testcase in test_suite_entry.get('testcases', []): - subtests = testcase.get('subtests', []) - if not subtests: - continue # No subtests to check, keep original - - # Check subtest results to determine testcase result - has_non_waivered_failed = False - has_waivered_failed = False - has_passed = False - has_skipped = False - - for subtest in subtests: - sub_result = subtest.get('sub_test_result', '') - if isinstance(sub_result, str): - sub_upper = sub_result.upper() - if 'FAILED' in sub_upper: - if '(WITH WAIVER)' in sub_upper: - has_waivered_failed = True - else: - has_non_waivered_failed = True - elif 'PASSED' in sub_upper: - has_passed = True - elif 'SKIPPED' in sub_upper: - has_skipped = True - - # Keep the original Test_result from the log file - # Parent testcase result is always set by the parser from the END line + _propagate_nested_bsa_waivers(testcase) # Update test suite summary # Determine the summary field based on suite name @@ -797,25 +918,25 @@ def apply_waivers(suite_name, json_file, waiver_file='waiver.json', output_json_ # Process results from testcases only for testcase in testcases: - test_result = testcase.get('test_result', '') + test_result = testcase.get('Test_result', testcase.get('test_result', '')) if isinstance(test_result, str): - r = test_result.upper() - if 'FAILED' in r: - if 'WITH WAIVER' in r: + result_upper = test_result.upper() + if 'FAILED' in result_upper: + if 'WITH WAIVER' in result_upper: waived += 1 else: failed += 1 - elif 'PAL NOT SUPPORTED' in r: + elif 'PAL NOT SUPPORTED' in result_upper: pal += 1 - elif 'TEST NOT IMPLEMENTED' in r: + elif 'TEST NOT IMPLEMENTED' in result_upper: notimpl += 1 - elif 'SKIPPED' in r: + elif 'SKIPPED' in result_upper: skipped += 1 - elif 'STATUS' in r: + elif 'STATUS' in result_upper: warnings += 1 - elif 'PARTIAL' in r: + elif 'PARTIAL' in result_upper: partial += 1 - elif 'PASS' in r: + elif 'PASS' in result_upper: passed += 1 test_suite_entry['test_suite_summary'] = { @@ -841,19 +962,19 @@ def apply_waivers(suite_name, json_file, waiver_file='waiver.json', output_json_ for testcase in test_suite_entry.get('testcases', []): test_result = testcase.get('Test_result', '') if isinstance(test_result, str): - r = test_result.upper() - if 'FAILED' in r: - if '(WITH WAIVER)' in r: + result_upper = test_result.upper() + if 'FAILED' in result_upper: + if '(WITH WAIVER)' in result_upper: total_failed_with_waiver += 1 else: total_failed += 1 - elif 'ABORTED' in r: + elif 'ABORTED' in result_upper: total_aborted += 1 - elif 'SKIPPED' in r: + elif 'SKIPPED' in result_upper: total_skipped += 1 - elif 'WARNING' in r: + elif 'WARNING' in result_upper: total_warnings += 1 - elif 'PASS' in r: + elif 'PASS' in result_upper: total_passed += 1 test_suite_entry['test_suite_summary'] = { @@ -884,19 +1005,19 @@ def apply_waivers(suite_name, json_file, waiver_file='waiver.json', output_json_ total_skipped += sub_test_result.get('SKIPPED', 0) total_warnings += sub_test_result.get('WARNINGS', 0) elif isinstance(sub_test_result, str): - r = sub_test_result.upper() - if 'PASS' in r: + result_upper = sub_test_result.upper() + if 'PASS' in result_upper: total_passed += 1 - elif 'FAIL' in r: - if '(WITH WAIVER)' in r: + elif 'FAIL' in result_upper: + if '(WITH WAIVER)' in result_upper: total_failed_with_waiver += 1 else: total_failed += 1 - elif 'ABORTED' in r: + elif 'ABORTED' in result_upper: total_aborted += 1 - elif 'SKIPPED' in r: + elif 'SKIPPED' in result_upper: total_skipped += 1 - elif 'WARNING' in r: + elif 'WARNING' in result_upper: total_warnings += 1 else: # anything else (IGNORED, NOT SUPPORTED, etc.) @@ -1123,15 +1244,16 @@ def apply_waivers(suite_name, json_file, waiver_file='waiver.json', output_json_ # Write the updated JSON data back to the file try: - with open(json_file, 'w') as f: - json.dump(json_data, f, indent=4) + with open(json_file, 'w', encoding='utf-8') as json_handle: + json.dump(json_data, json_handle, indent=4) print(f"Waivers successfully applied and '{json_file}' has been updated.") - except Exception as e: + except Exception as err: if verbose: - print(f"ERROR: Failed to write updated data to {json_file}: {e}") + print(f"ERROR: Failed to write updated data to {json_file}: {err}") return def main(): + """Parse command line arguments and apply waivers.""" parser = argparse.ArgumentParser(description='Apply waivers to test suite JSON results.') parser.add_argument('suite_name', help='Name of the test suite') parser.add_argument('json_file', help='Path to the JSON file') diff --git a/common/log_parser/bsa/json_to_html.py b/common/log_parser/bsa/json_to_html.py index 161b7198..9154c36f 100755 --- a/common/log_parser/bsa/json_to_html.py +++ b/common/log_parser/bsa/json_to_html.py @@ -14,22 +14,44 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json -import matplotlib.pyplot as plt +"""Generate BSA/SBSA HTML reports from parsed JSON results.""" + import base64 +import json +import os +import sys from io import BytesIO + from jinja2 import Template -import os +import matplotlib.pyplot as plt # pylint: disable=import-error # Helper function to retrieve dictionary values in a case-insensitive manner -def get_case_insensitive(d, key, default=0): - for k, v in d.items(): - if k.lower() == key.lower(): - return v +def get_case_insensitive(data, key, default=0): + """Return a dictionary value by key without requiring exact key casing.""" + for dict_key, value in data.items(): + if dict_key.lower() == key.lower(): + return value return default +def has_nested_subtests(subtests): + """Return True when any subtest contains child subtests.""" + for subtest in subtests or []: + child_subtests = subtest.get("subtests", []) + if child_subtests or has_nested_subtests(child_subtests): + return True + return False + +def annotate_nested_subtests(test_results): + """Mark each testcase that needs expand/collapse controls in HTML.""" + for test_suite in test_results: + for testcase in test_suite.get("testcases", []): + testcase["has_nested_subtests"] = has_nested_subtests( + testcase.get("subtests", []) + ) + # Function to generate bar chart for test results def generate_bar_chart(suite_summary): + """Build a base64 encoded bar chart image for suite summary counts.""" labels = [ 'Passed', 'Failed', @@ -69,11 +91,11 @@ def generate_bar_chart(suite_summary): # Add percentage labels on top of the bars total_tests = suite_summary.get('total_rules_run', 0) or sum(sizes) - for bar, size in zip(bars, sizes): - yval = bar.get_height() + for chart_bar, size in zip(bars, sizes): + yval = chart_bar.get_height() percentage = (size / total_tests) * 100 if total_tests > 0 else 0 plt.text( - bar.get_x() + bar.get_width()/2, + chart_bar.get_x() + chart_bar.get_width()/2, yval + max(sizes)*0.01, f'{percentage:.2f}%', ha='center', @@ -95,7 +117,16 @@ def generate_bar_chart(suite_summary): return base64.b64encode(buffer.getvalue()).decode('utf-8') # Function to generate HTML content for both summary and detailed pages -def generate_html(suite_summary, test_results, chart_data, output_html_path, test_suite_name, is_summary_page=True): +def generate_html( # pylint: disable=too-many-arguments,too-many-positional-arguments + suite_summary, + test_results, + chart_data, + output_html_path, + test_suite_name, + is_summary_page=True): + """Render either the detailed or summary BSA/SBSA HTML report.""" + annotate_nested_subtests(test_results) + # Template for both summary and detailed pages template = Template(""" @@ -225,9 +256,186 @@ def generate_html(suite_summary, test_results, chart_data, output_html_path, tes text-align: center; font-weight: normal; } + /* Keep rule numbers readable while indentation shows nesting. */ + .subtest-number { + white-space: nowrap; + } + .testcase-toggle, + .subtest-toggle, + .subtest-control { + border: 1px solid #6c7a89; + cursor: pointer; + font-weight: 600; + transition: background-color 0.15s ease, border-color 0.15s ease, + box-shadow 0.15s ease, color 0.15s ease; + } + .testcase-toggle, + .subtest-toggle { + display: inline-flex; + align-items: center; + justify-content: center; + width: 22px; + height: 22px; + margin-right: 8px; + border-radius: 50%; + background: #f7fbff; + border-color: #6fa8dc; + color: #1f5f99; + font-family: Arial, sans-serif; + font-size: 14px; + line-height: 1; + box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.08); + } + .testcase-toggle:hover, + .subtest-toggle:hover { + background: #e8f3ff; + border-color: #2f80c7; + } + .testcase-toggle[aria-expanded="true"], + .subtest-toggle[aria-expanded="true"] { + background: #edf7ed; + border-color: #58a55c; + color: #256b2a; + } + .testcase-toggle[aria-expanded="false"], + .subtest-toggle[aria-expanded="false"] { + background: #fff7e6; + border-color: #d89614; + color: #8a5a00; + } + .testcase-toggle-placeholder, + .subtest-toggle-placeholder { + display: inline-block; + width: 22px; + height: 22px; + margin-right: 8px; + } + .testcase-number { + white-space: nowrap; + } + .subtest-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + margin-bottom: 8px; + } + .subtest-title { + color: #2c3e50; + font-weight: bold; + } + .subtest-controls { + display: flex; + flex-wrap: wrap; + justify-content: flex-end; + gap: 8px; + } + .subtest-control { + display: inline-flex; + align-items: center; + gap: 6px; + padding: 6px 12px; + border-radius: 4px; + font-size: 12px; + line-height: 1.2; + box-shadow: 0 1px 2px rgba(0, 0, 0, 0.12); + } + .expand-subtests { + background: #2f80c7; + border-color: #2878b5; + color: #fff; + } + .expand-subtests:hover:not(:disabled) { + background: #236a9f; + border-color: #1f5f99; + } + .collapse-subtests { + background: #fff; + border-color: #9aa9b5; + color: #2c3e50; + } + .collapse-subtests:hover:not(:disabled) { + background: #eef3f7; + border-color: #6c7a89; + } + .subtest-control:disabled { + cursor: default; + opacity: 0.45; + box-shadow: none; + } + .control-symbol { + display: inline-flex; + align-items: center; + justify-content: center; + width: 16px; + height: 16px; + border-radius: 50%; + background: rgba(255, 255, 255, 0.25); + font-weight: bold; + line-height: 1; + } + .collapse-subtests .control-symbol { + background: #edf2f7; + color: #34495e; + } + .testcase-toggle:focus-visible, + .subtest-toggle:focus-visible, + .subtest-control:focus-visible { + outline: 2px solid #1f77b4; + outline-offset: 2px; + } + @media (max-width: 700px) { + .subtest-header { + align-items: flex-start; + flex-direction: column; + } + .subtest-controls { + justify-content: flex-start; + } + } + .subtest-row-hidden { + display: none; + } + {# Render BSA/SBSA subtests recursively so the HTML follows the same + parent-child order as the JSON and original nested log. #} + {% macro render_subtest_rows(subtests, parent_path='') %} + {% for subtest in subtests %} + {% set nesting_level = subtest.sub_Test_Level | default(1) | int %} + {% set subtest_path = subtest.sub_Test_Path | default(subtest.sub_Test_Number) %} + {% set has_children = subtest.subtests is defined and subtest.subtests %} + {# The row tooltip contains the full sub_Test_Path for comparison + with logs and for precise waiver targeting. #} + + + {% if has_children %} + + {% else %} + + {% endif %} + {{ subtest.sub_Test_Number }} + + {{ subtest.sub_Test_Description }} + + {{ subtest.sub_test_result }} + + + {% if 'FAILED (WITH WAIVER)' in subtest.sub_test_result %} + {{ subtest.waiver_reason | default("N/A") }} + {% else %} + N/A + {% endif %} + + + {% if has_children %} + {# Child rows are printed immediately after their parent with + additional indentation from sub_Test_Level. #} + {{ render_subtest_rows(subtest.subtests, subtest_path) }} + {% endif %} + {% endfor %} + {% endmacro %}

{{ test_suite_name }} Test Summary

{% if not is_summary_page %} @@ -298,8 +506,9 @@ def generate_html(suite_summary, test_results, chart_data, output_html_path, tes {% if not is_summary_page %} -
+
{% for test in test_results %} + {% set suite_index = loop.index0 %}
Test Suite: {{ test.Test_suite }}
@@ -312,8 +521,16 @@ def generate_html(suite_summary, test_results, chart_data, output_html_path, tes {% for testcase in test.testcases %} + {% set subtest_table_id = "subtests-" ~ suite_index ~ "-" ~ loop.index0 %} - + {% if testcase.subtests %} - + @@ -367,6 +583,201 @@ def generate_html(suite_summary, test_results, chart_data, output_html_path, tes {% endfor %} {% endif %} + """) @@ -415,13 +826,14 @@ def generate_html(suite_summary, test_results, chart_data, output_html_path, tes ) # Save to HTML file - with open(output_html_path, "w") as file: + with open(output_html_path, "w", encoding="utf-8") as file: file.write(html_content) # Main function to process the JSON file and generate the HTML report def main(input_json_file, detailed_html_file, summary_html_file): + """Load parsed JSON and generate detailed and summary HTML files.""" # Load JSON data - with open(input_json_file, 'r') as json_file: + with open(input_json_file, 'r', encoding="utf-8") as json_file: data = json.load(json_file) # Extract the test results @@ -445,25 +857,43 @@ def main(input_json_file, detailed_html_file, summary_html_file): } # Get the test suite name from the input JSON file name - test_suite_name = os.path.splitext(os.path.basename(input_json_file))[0].upper() + test_suite_name = os.path.splitext( + os.path.basename(input_json_file) + )[0].upper() # Generate bar chart chart_data = generate_bar_chart(suite_summary) # Generate the detailed summary page - generate_html(suite_summary, test_results, chart_data, detailed_html_file, test_suite_name, is_summary_page=False) + generate_html( + suite_summary, + test_results, + chart_data, + detailed_html_file, + test_suite_name, + is_summary_page=False + ) # Generate the summary page with the bar chart - generate_html(suite_summary, test_results, chart_data, summary_html_file, test_suite_name, is_summary_page=True) + generate_html( + suite_summary, + test_results, + chart_data, + summary_html_file, + test_suite_name, + is_summary_page=True + ) if __name__ == "__main__": - import sys if len(sys.argv) != 4: - print("Usage: python json_to_html.py ") + print( + "Usage: python json_to_html.py " + " " + ) sys.exit(1) - input_json_file = sys.argv[1] - detailed_html_file = sys.argv[2] # This will be the detailed HTML report - summary_html_file = sys.argv[3] # This will be the summary HTML report + input_path = sys.argv[1] + detailed_path = sys.argv[2] # This will be the detailed HTML report + summary_path = sys.argv[3] # This will be the summary HTML report - main(input_json_file, detailed_html_file, summary_html_file) + main(input_path, detailed_path, summary_path) diff --git a/common/log_parser/bsa/logs_to_json.py b/common/log_parser/bsa/logs_to_json.py index d0c3b456..74d7f116 100644 --- a/common/log_parser/bsa/logs_to_json.py +++ b/common/log_parser/bsa/logs_to_json.py @@ -73,9 +73,9 @@ def classify_status(status_text): summary_category = "Skipped" return formatted_result, summary_category - # STATUS → warnings - if up.startswith("STATUS:"): - formatted_result = "STATUS" + # Warnings + if up.startswith("STATUS:") or up in ("WARNING", "WARN") or up.startswith("WARNING"): + formatted_result = "WARNING" if "WARN" in up else "STATUS" summary_category = "Warnings" return formatted_result, summary_category @@ -121,6 +121,133 @@ def update_summary_counts(summary, summary_category, formatted_result): elif summary_category == "Warnings": summary["Warnings"] += 1 +def make_test_number(rule_id, test_index): + return f"{rule_id} : {test_index or '-'}" + +# A frame is one rule that has started but has not reached its Result/END line. +# Keeping these frames on a stack lets the parser attach each completed child +# rule to the nearest still-open parent rule. +def make_rule_frame(suite, rule_id, test_index, description, parent, current_source): + test_number = make_test_number(rule_id, test_index) + if parent: + child_counts = parent.setdefault("_child_counts", defaultdict(int)) + child_counts[test_number] += 1 + occurrence = child_counts[test_number] + # If the same rule appears twice under one parent, keep both paths unique + # without changing the visible sub_Test_Number used by old waivers. + display_number = test_number if occurrence == 1 else f"{test_number} [{occurrence}]" + root = parent.get("root") or parent + level = parent.get("level", 0) + 1 + path = parent.get("path", [parent.get("number", "")]) + [display_number] + else: + occurrence = 1 + root = None + level = 0 + path = [test_number] + + frame = { + "suite": suite, + "rule_id": rule_id, + "index": test_index or "-", + "description": description, + "number": test_number, + "path": path, + "level": level, + "parent": parent, + "root": root, + "source": current_source, + "subtests": [], + "occurrence": occurrence + } + if parent is None: + frame["root"] = frame + return frame + +def subtest_entry_from_frame(frame, formatted_result): + # Completed child rules become recursive subtests. sub_Test_Path mirrors the + # log branch so a partner can compare JSON/HTML directly with the log. + entry = { + "sub_Test_Number": frame.get("number", make_test_number(frame.get("rule_id"), frame.get("index"))), + "sub_Test_Description": frame.get("description", ""), + "sub_test_result": formatted_result, + "sub_Test_Level": frame.get("level", 1), + "sub_Test_Path": " / ".join(frame.get("path", [])) + } + subtests = frame.get("subtests", []) + if subtests: + entry["subtests"] = subtests + return entry + +def testcase_from_frame(frame, formatted_result): + testcase = { + "Test_case": frame.get("number", make_test_number(frame.get("rule_id"), frame.get("index"))), + "Test_case_description": frame.get("description", ""), + "Test_result": formatted_result, + "_source": frame.get("source", "unknown") + } + subtests = frame.get("subtests", []) + if subtests: + testcase["subtests"] = subtests + return testcase + +def find_frame_from_top(rule_stack, rule_id): + # END lines only carry the rule id. If the same id appears more than once + # in nested logs, the nearest open frame is the one that should close. + for idx in range(len(rule_stack) - 1, -1, -1): + if rule_stack[idx].get("rule_id") == rule_id: + return idx + return None + +def complete_rule_frame(frame, formatted_result, summary_category, + testcases_per_suite, suite_summaries, total_summary): + if frame.get("parent") is not None: + # Nested rules are stored under their parent. Suite totals count only + # completed top-level rules, matching the old BSA/SBSA behavior. + parent = frame.get("parent") + if parent is not None: + parent.setdefault("subtests", []).append( + subtest_entry_from_frame(frame, formatted_result) + ) + return + + testcase = testcase_from_frame(frame, formatted_result) + + tcs = init_summary() + update_summary_counts(tcs, summary_category, formatted_result) + testcase["Test_case_summary"] = tcs + + suite = frame.get("suite", "") + testcases_per_suite[suite].append(testcase) + update_summary_counts(suite_summaries[suite], summary_category, formatted_result) + update_summary_counts(total_summary, summary_category, formatted_result) + +def iter_subtests(subtests): + # Flatten a nested subtest tree for matching/merging, without changing the + # recursive JSON structure that partners see in the final output. + for subtest in subtests or []: + yield subtest + yield from iter_subtests(subtest.get("subtests", [])) + +def subtest_key(subtest): + # Use the full nested path when available. The visible rule number is kept + # as a fallback so old flat logs still merge as before. + return subtest.get("sub_Test_Path") or subtest.get("sub_Test_Number") + +def merge_matching_subtests(existing_subtests, override_subtests): + # When UEFI and Linux logs contain the same testcase, keep the UEFI tree as + # the base and overlay Linux results only on matching subtests. Prefer the + # full path so repeated nested rule numbers do not overwrite each other. + existing_by_key = { + subtest_key(st): st + for st in iter_subtests(existing_subtests) + if subtest_key(st) + } + for override in iter_subtests(override_subtests): + key = subtest_key(override) + if key in existing_by_key: + existing_by_key[key].clear() + existing_by_key[key].update(override) + def main(input_files, output_file): # Per-suite list of testcases testcases_per_suite = defaultdict(list) @@ -129,14 +256,9 @@ def main(input_files, output_file): # Global summary total_summary = init_summary() - # Active main testcases (B_* rules etc.) - # key: rule_id -> metadata - active_main = {} - # Active subtests - # key: rule_id -> metadata - active_sub = {} - # Stack of currently open main test IDs (for nesting) - parent_stack = [] + # Stack of currently open rule instances. Nested BSA/SBSA logs can reuse + # the same rule ID in different branches, so this must be instance based. + rule_stack = [] current_suite = "" @@ -188,16 +310,28 @@ def main(input_files, output_file): continue # ---------------- New log format support ---------------- - # Newer BSA logs can look like: - # *** Running tests *** - # : : + # Newer BSA/SBSA logs can nest rule groups: + # : : + # === Start tests for rules referenced by === + # : : + # Result: + # === End tests for rules referenced by === # Result: suite_hdr = re.match(r'^\*\*\*\s+Running\s+(.+?)\s+tests\s+\*\*\*$', line) if suite_hdr: current_suite = suite_hdr.group(1).strip().replace(" ", "_") continue - # RULE line (main or subtest, determined by indentation) + referenced_rules_marker = re.match( + r'^===\s+(?:Start|End)\s+tests\s+for\s+rules\s+referenced\s+by\s+([A-Za-z0-9_]+)\s+===$', + line + ) + if referenced_rules_marker: + continue + + # RULE line. A rule is top-level only when it starts without + # indentation. Indented rule lines become children of the last + # still-open frame on the stack. rule_line = re.match(r'^([A-Za-z0-9_]+)\s*:\s*(-|\d+)\s*:\s*(.*)$', line) if rule_line: rule_id = rule_line.group(1).strip() @@ -205,101 +339,41 @@ def main(input_files, output_file): desc = (rule_line.group(3) or "").strip() suite = current_suite or "" - if not is_indented: - # Main rule: start a new parent context - meta = { - "suite": suite, - "rule_id": rule_id, - "index": test_index, - "description": desc, - "subtests": [] - } - active_main[rule_id] = meta - # reset stack to this main rule (prevents accidental nesting across rules) - parent_stack[:] = [rule_id] - else: - # Subtest: attach to nearest main rule in the stack (if any) - parent_id = None - for rid in reversed(parent_stack): - if rid in active_main: - parent_id = rid - break - meta = { - "suite": suite, - "rule_id": rule_id, - "index": test_index, - "description": desc, - "parent": parent_id, - "is_indented": is_indented - } - active_sub[rule_id] = meta - parent_stack.append(rule_id) + parent = rule_stack[-1] if (is_indented and rule_stack) else None + if not is_indented and rule_stack: + # A new top-level rule should only appear after the + # previous top-level result. If a malformed log leaves + # frames open, avoid attaching the new testcase to them. + rule_stack.clear() + rule_stack.append( + make_rule_frame(suite, rule_id, test_index, desc, parent, current_source) + ) continue - # Result line (belongs to the last opened rule in new-format logs) + # In the new log format, Result closes the most recently opened rule. + # That rule is either emitted as a testcase or attached to its parent. if line.upper().startswith("RESULT:"): status_text = line.split(":", 1)[1].strip() - if not parent_stack: + if not rule_stack: continue - rule_id = parent_stack[-1] + frame = rule_stack.pop() formatted_result, summary_category = classify_status(status_text) - - # Subtest completion - if rule_id in active_sub: - sub_meta = active_sub.pop(rule_id) - parent_id = sub_meta.get("parent") - if parent_id and parent_id in active_main: - parent_meta = active_main[parent_id] - sub_entry = { - "sub_Test_Number": f"{rule_id} : {sub_meta.get('index', '-')}", - "sub_Rule_ID": rule_id, - "sub_Test_Description": sub_meta.get("description", ""), - "sub_test_result": formatted_result - } - parent_meta["subtests"].append(sub_entry) - - # Pop this subtest from the stack - parent_stack.pop() - continue - - # Main testcase completion - if rule_id in active_main: - meta = active_main.pop(rule_id) - - # Pop main from stack - if parent_stack and parent_stack[-1] == rule_id: - parent_stack.pop() - - suite = meta.get("suite", "") - desc = meta.get("description", "") - index = meta.get("index", "-") - subtests = meta.get("subtests", []) - - testcase = { - "Test_case": f"{rule_id} : {index}", - "Test_case_description": desc, - "Test_result": formatted_result - } - testcase["_source"] = current_source - if subtests: - testcase["subtests"] = subtests - - tcs = init_summary() - update_summary_counts(tcs, summary_category, formatted_result) - testcase["Test_case_summary"] = tcs - - testcases_per_suite[suite].append(testcase) - update_summary_counts(suite_summaries[suite], summary_category, formatted_result) - update_summary_counts(total_summary, summary_category, formatted_result) - - continue - + complete_rule_frame( + frame, + formatted_result, + summary_category, + testcases_per_suite, + suite_summaries, + total_summary + ) continue # -------------- End new log format support -------------- # START : + # Old-format START/END logs use the same stack. Flat old logs stay + # flat, while any indented old-format child rules can still nest. start_match = re.match( r'^START\s+([^\s:]+)\s+([A-Za-z0-9_]+)\s+([^\s:]+)\s*:\s*(.*)$', line @@ -321,35 +395,13 @@ def main(input_files, output_file): # Normalize index test_index = index_tok if index_tok != "" else "-" - # Decide if this is a main testcase or a subtest based on indentation: - # - Non-indented lines = main testcases (B_*, S_*, GPU_*, PCI_ER_*, etc.) - # - Indented lines = subtests (nested under current parent) - is_main = not is_indented - - if is_main: - # Main rule - meta = { - "suite": current_suite, - "rule_id": rule_id, - "index": test_index, - "description": desc, - "subtests": [] - } - active_main[rule_id] = meta - parent_stack.append(rule_id) - else: - # Subrule / subtest under current parent (if indented or non-B_ rule) - parent_id = parent_stack[-1] if parent_stack else None - meta = { - "suite": current_suite, - "rule_id": rule_id, - "index": test_index, - "description": desc, - "parent": parent_id, - "is_indented": is_indented - } - active_sub[rule_id] = meta + parent = rule_stack[-1] if (is_indented and rule_stack) else None + if not is_indented and rule_stack: + rule_stack.clear() + rule_stack.append( + make_rule_frame(current_suite, rule_id, test_index, desc, parent, current_source) + ) continue # END line: @@ -361,61 +413,21 @@ def main(input_files, output_file): formatted_result, summary_category = classify_status(status_text) - # Check if this is a subtest (in active_sub) - if rule_id in active_sub: - sub_meta = active_sub.pop(rule_id) - parent_id = sub_meta.get("parent") - if parent_id and parent_id in active_main: - parent_meta = active_main[parent_id] - sub_entry = { - "sub_Test_Number": f"{rule_id} : {sub_meta.get('index', '-')}", - "sub_Rule_ID": rule_id, - "sub_Test_Description": sub_meta.get("description", ""), - "sub_test_result": formatted_result - } - parent_meta["subtests"].append(sub_entry) - # No summary update for subtests - continue - - # Check if this is a main testcase (in active_main) - if rule_id in active_main: - meta = active_main.pop(rule_id) - - # Pop from parent stack if this was the last main opened - if parent_stack and parent_stack[-1] == rule_id: - parent_stack.pop() - - suite = meta.get("suite", "") - desc = meta.get("description", "") - index = meta.get("index", "-") - subtests = meta.get("subtests", []) - - # Build testcase object - testcase = { - "Test_case": f"{rule_id} : {index}", - "Test_case_description": desc, - "Test_result": formatted_result - } - testcase["_source"] = current_source - if subtests: - testcase["subtests"] = subtests - - # Per-testcase summary (one-hot) - tcs = init_summary() - update_summary_counts(tcs, summary_category, formatted_result) - testcase["Test_case_summary"] = tcs - - # Append to suite - testcases_per_suite[suite].append(testcase) - - # Update per-suite summary - update_summary_counts(suite_summaries[suite], summary_category, formatted_result) - # Update global summary - update_summary_counts(total_summary, summary_category, formatted_result) - + # END names the rule being closed. Search from the top of the + # stack so repeated rule IDs close the nearest matching instance. + frame_idx = find_frame_from_top(rule_stack, rule_id) + if frame_idx is None: continue - # Ignore END lines for unknown rule IDs (not in active_sub or active_main) + frame = rule_stack.pop(frame_idx) + complete_rule_frame( + frame, + formatted_result, + summary_category, + testcases_per_suite, + suite_summaries, + total_summary + ) continue # Ignore all other lines (debug, informational, etc.) @@ -448,18 +460,18 @@ def main(input_files, output_file): linux_tc = tc if src == "linux" else None if linux_tc: - # For B_PER_08, keep UEFI testcase result. For others, overwrite testcase result from Linux. + # For B_PER_08, keep UEFI testcase result. For other duplicate + # testcases, Linux has the final testcase-level result. if key != "B_PER_08 : -": existing_tc["Test_result"] = linux_tc.get("Test_result") existing_tc["Test_case_summary"] = linux_tc.get("Test_case_summary") - # Override only matching subtests (do not add Linux-only subtests). - uefi_subtests = {st.get("sub_Test_Number"): st for st in existing_tc.get("subtests", [])} - for st in linux_tc.get("subtests", []) or []: - sub_key = st.get("sub_Test_Number") - if sub_key in uefi_subtests: - uefi_subtests[sub_key] = st - existing_tc["subtests"] = list(uefi_subtests.values()) + # Override only matching subtests. Linux-only subtests are not + # appended because the UEFI tree is the report structure. + merge_matching_subtests( + existing_tc.get("subtests", []), + linux_tc.get("subtests", []) or [] + ) continue testcases_per_suite = processed_testcases diff --git a/docs/acs_results_structure.md b/docs/acs_results_structure.md index 1fe95f46..3cdee468 100644 --- a/docs/acs_results_structure.md +++ b/docs/acs_results_structure.md @@ -37,7 +37,8 @@ Comprehensive reference for JSON output structures across all test suite parsers | **sub_Test_Number** | String | String | String | String | String | String | String | String | String | | **sub_Test_Description** | String | String | String | String | String | String | String | String | String | | **sub_Test_GUID** | - | - | String | - | String | - | - | - | - | -| **sub_Rule_ID** | String | - | - | - | - | - | - | - | - | +| **sub_Test_Level** | Integer | - | - | - | - | - | - | - | - | +| **sub_Test_Path** | String | - | - | - | - | - | - | - | - | | **sub_test_result** | **String:**
• PASSED
• FAILED
• SKIPPED | **dict:**
• PASSED
• FAILED
• SKIPPED
• ABORTED
• WARNINGS
• FAILED_WITH_WAIVER
• pass_reasons (Array)
• fail_reasons (Array)
• skip_reasons (Array)
• abort_reasons (Array)
• warning_reasons (Array) | **String:**
• PASSED
• FAILED
• SKIPPED
• ABORTED
• WARNING | **dict:**
• PASSED
• FAILED
• SKIPPED
• ABORTED
• WARNINGS
• FAILED_WITH_WAIVER
• pass_reasons (Array)
• fail_reasons (Array)
• skip_reasons (Array)
• abort_reasons (Array)
• warning_reasons (Array) | **String:**
• PASSED
• FAILED
• SKIPPED
• ABORTED
• WARNING | **String:**
• PASSED
• FAILED
• SKIPPED
• ABORTED
• WARNING | **dict:**
• PASSED
• FAILED
• SKIPPED
• ABORTED
• WARNINGS
• FAILED_WITH_WAIVER
• pass_reasons (Array)
• fail_reasons (Array)
• skip_reasons (Array)
• abort_reasons (Array)
• warning_reasons (Array)
• waiver_reason (Array) | **dict:**
• PASSED
• FAILED
• SKIPPED
• ABORTED
• WARNINGS
• pass_reasons (Array)
• fail_reasons (Array)
• skip_reasons (Array)
• abort_reasons (Array)
• warning_reasons (Array) | **String:**
• PASSED
• FAILED
• SKIPPED
• ABORTED
• WARNING | | **reason** | - | - | String | - | String | String | - | - | String | @@ -54,6 +55,8 @@ Comprehensive reference for JSON output structures across all test suite parsers 1. **PFDI** - ONLY suite with top-level Array (not dict with test_results) 2. **BSA** - Uses unique summary field names (Passed, Failed, Skipped vs total_passed, total_failed, total_skipped) + - BSA/SBSA `subtests[]` can be nested recursively and use `sub_Test_Path` for exact log-path identity. + - New BSA/SBSA JSON does not emit `sub_Rule_ID`; waiver files may still use it as a legacy matcher. 3. **Naming variations:** - `suite_summary` (most) vs `Suite_summary` (PFDI - capital S) - `testcases[]` (BSA - lowercase) vs `Test_cases[]` (SBMR - capital T) diff --git a/docs/acs_schema_guide.md b/docs/acs_schema_guide.md index 3d75f13e..7edc2510 100644 --- a/docs/acs_schema_guide.md +++ b/docs/acs_schema_guide.md @@ -85,6 +85,8 @@ Each `bsa_test_case`: - `Test_case`, `Test_case_description`, `Test_result`, `Test_case_summary` - Optional: `subtests`, `waiver_reason` - If `Test_result` is `FAILED (WITH WAIVER)`, `waiver_reason` is required +- BSA/SBSA `subtests` may be nested recursively. Nested subtests use `sub_Test_Number`, `sub_Test_Description`, `sub_test_result`, `sub_Test_Level`, `sub_Test_Path`, and optional child `subtests`. +- New BSA/SBSA JSON does not emit `sub_Rule_ID`; waiver files may still use `sub_Rule_ID` as a legacy matcher. ### 4.2 FWTS / BBSR-FWTS diff --git a/docs/log_parser_guide.md b/docs/log_parser_guide.md index 5381a0c8..e309cb56 100644 --- a/docs/log_parser_guide.md +++ b/docs/log_parser_guide.md @@ -339,7 +339,8 @@ Suite Level "Reason": "Required: TestCase-level waiver", "SubTests": [ { - "sub_Rule_ID": "", + "sub_Test_Path": "", + "sub_Test_Number": "", "sub_Test_Description": "", "Reason": "Required: SubTest-level waiver" } @@ -353,6 +354,8 @@ Suite Level } ``` +For BSA/SBSA nested subtests, prefer `sub_Test_Path`. It uniquely identifies the branch when the same rule or subtest number appears under multiple parents. The waiver parser also accepts `sub_Test_Number`, legacy `sub_Rule_ID`, and exact `sub_Test_Description` for compatibility. + ### Waiver Examples #### 1. Suite-Level Waiver (applies to all failed tests in suite) @@ -402,18 +405,16 @@ Suite Level #### 4. SubTest-Level Waiver ```json { - "Suite": "BSA", + "Suite": "SBSA", "TestSuites": [ { "TestSuite": "PCIE", "TestCases": [ { - "Test_case": "B_PER_08", - "Reason": "Root complex behavior varies by board implementation", + "Test_case": "S_L6PCI_1", "SubTests": [ { - "sub_Rule_ID": "PCI_MM_01", - "sub_Test_Description": "PCIe Device Memory mapping support", + "sub_Test_Path": "S_L6PCI_1 : - / B_REP_1 : - / JKZMT : - / PCI_MM_01 : -", "Reason": "Device memory decode not supported on this platform" } ] @@ -424,6 +425,8 @@ Suite Level } ``` +BSA/SBSA waivers using `sub_Test_Number` or legacy `sub_Rule_ID` still work, but they can match every nested occurrence of that number or rule under the testcase. Use `sub_Test_Path` when only one branch should be waived. + #### 5. Standalone/OS Tests Waiver Format ```json { @@ -462,7 +465,7 @@ Suite Level "Test_case": "B_PER_08", "SubTests": [ { - "sub_Rule_ID": "PCI_MM_01", + "sub_Test_Path": "B_PER_08 : - / PCI_MM_01 : -", "Reason": "Device memory decode not supported" } ] @@ -512,8 +515,8 @@ Suite Level 3. **Apply waivers**: - Suite-level: Applies to ALL failed tests in suite - TestSuite-level: Applies to ALL failed tests in that TestSuite - - TestCase-level: Applies to specific TestCase and its SubTests - - SubTest-level: Applies only to specific SubTest + - TestCase-level: Applies to the specific TestCase and its nested failed SubTests + - SubTest-level: Applies only to specific SubTests; BSA/SBSA matching uses `sub_Test_Path`, then `sub_Test_Number`, then legacy `sub_Rule_ID`, then exact description 4. **Mark results**: - Original: `FAILED` @@ -529,6 +532,7 @@ Suite Level - **Reason is REQUIRED** for each waiver entry; missing reasons are skipped (quietly unless verbose) - Waivers cascade down (Suite → TestSuite → TestCase → SubTest) - Waivers only apply to **failed** tests (passing tests are not affected) +- For nested BSA/SBSA failures, if all failed nested children under a failed parent are waived, the parent subtest/testcase is also marked `FAILED (WITH WAIVER)` - Waiver reasons appear in both detailed HTML and summary reports - Multiple waivers can be applied to the same suite @@ -668,37 +672,66 @@ if not waivable: { "test_results": [ { - "Test_suite": "TIMER", + "Test_suite": "PCIE", "testcases": [ { - "Test_case": "B_TIMER_01", - "Test_case_description": "Check generic timer implementation", - "Test_result": "PASSED" - }, - { - "Test_case": "B_TIMER_02", - "Test_case_description": "Verify timer interrupt", - "Test_result": "FAILED (WITH WAIVER)", - "waiver_reason": "Timer interrupt not supported on this platform" + "Test_case": "S_L6PCI_1 : -", + "Test_case_description": "Check PCIe On-chip Peripherals", + "Test_result": "FAILED", + "subtests": [ + { + "sub_Test_Number": "B_REP_1 : -", + "sub_Test_Description": "Check RCiEP Devices", + "sub_test_result": "FAILED", + "sub_Test_Level": 1, + "sub_Test_Path": "S_L6PCI_1 : - / B_REP_1 : -", + "subtests": [ + { + "sub_Test_Number": "JKZMT : -", + "sub_Test_Description": "", + "sub_test_result": "FAILED", + "sub_Test_Level": 2, + "sub_Test_Path": "S_L6PCI_1 : - / B_REP_1 : - / JKZMT : -", + "subtests": [ + { + "sub_Test_Number": "PCI_MM_01 : -", + "sub_Test_Description": "PCIe Device Memory mapping support", + "sub_test_result": "FAILED", + "sub_Test_Level": 3, + "sub_Test_Path": "S_L6PCI_1 : - / B_REP_1 : - / JKZMT : - / PCI_MM_01 : -" + } + ] + } + ] + } + ], + "Test_case_summary": { + "Total Rules Run": 1, + "Passed": 0, + "Failed": 1, + "Total_failed_with_waiver": 0 + } } ], "test_suite_summary": { - "Total Rules Run": 2, - "Passed": 1, - "Failed": 0, - "Total_failed_with_waiver": 1 + "Total Rules Run": 1, + "Passed": 0, + "Failed": 1, + "Total_failed_with_waiver": 0 } } ], "suite_summary": { - "Total Rules Run": 2, - "Passed": 1, - "Failed": 0, - "Total_failed_with_waiver": 1 + "Total Rules Run": 1, + "Passed": 0, + "Failed": 1, + "Total_failed_with_waiver": 0 } } ``` +BSA/SBSA `subtests` can recurse to any depth. `sub_Test_Path` is built from the visible log nesting and is intended for partners to compare directly with the log and for precise waiver matching. New BSA/SBSA JSON does not emit `sub_Rule_ID`; waiver files may still use it as a legacy matcher. + #### FWTS/SCT JSON Structure ```json { @@ -892,7 +925,7 @@ python3 apply_waivers.py 1. Load waiver.json 2. Extract suite-specific waivers 3. Load test results JSON -4. Match waivers to failed tests (hierarchy-based) +4. Match waivers to failed tests (hierarchy-based; recursive for BSA/SBSA nested subtests) 5. Update test status to "FAILED (WITH WAIVER)" 6. Add waiver_reason field 7. Save updated JSON @@ -904,6 +937,8 @@ python3 apply_waivers.py - TestCase-level - SubTest-level +For BSA/SBSA subtest-level waivers, matching priority is `sub_Test_Path`, `sub_Test_Number`, legacy `sub_Rule_ID`, then exact `sub_Test_Description`. + ### 4. logs_to_json.py (per suite) **Purpose**: Parse raw log files into structured JSON @@ -930,12 +965,20 @@ python3 apply_waivers.py 6. Write to output JSON ``` +**BSA/SBSA nested rules**: +- The BSA/SBSA parser uses a rule stack so any number of nested rule groups can be represented. +- A top-level rule is emitted as a testcase. Rules that run inside it are emitted under recursive `subtests`. +- Each BSA/SBSA subtest contains `sub_Test_Number`, `sub_Test_Description`, `sub_test_result`, `sub_Test_Level`, and `sub_Test_Path`. +- `sub_Test_Path` mirrors the log nesting and is stable for comparison with the log and precise waiver matching. +- Only rules with a completed `Result:` line are emitted as completed JSON entries. + ### 5. json_to_html.py (per suite) **Purpose**: Generate HTML reports from JSON **Outputs**: - Detailed HTML: Complete test breakdown - Summary HTML: Pass/Fail counts, embedded in main summary +- BSA/SBSA HTML renders nested `subtests` recursively with indentation. **Template Variables**: - Test suite name diff --git a/docs/waiver_guide.md b/docs/waiver_guide.md index 17e2de07..8582f42e 100644 --- a/docs/waiver_guide.md +++ b/docs/waiver_guide.md @@ -29,11 +29,11 @@ The script processes log files, applies waivers using the embedded `apply_waiver Waivers can be applied at various levels within a test suite. The hierarchy is as follows: -1. **Suite-Level Waiver**: Applies to all failed subtests in the entire suite. -2. **TestSuite-Level Waiver**: Applies to all failed subtests within a specific test suite. +1. **Suite-Level Waiver**: Applies to all failed tests in the entire suite. +2. **TestSuite-Level Waiver**: Applies to all failed tests within a specific test suite. 3. **SubSuite-Level Waiver**: Applies to all failed subtests within a specific sub-suite (applicable for suites like `SCT`, `Standalone`, `BBSR-SCT`, `BBSR-FWTS`). -4. **TestCase-Level Waiver**: Applies to all failed subtests within a specific test case (applicable for suites like `SCT`, `Standalone`, `BBSR-SCT`, `BBSR-FWTS`). -5. **SubTest-Level Waiver**: Applies to individual subtests based on `SubTestID` or `sub_Test_Description`. +4. **TestCase-Level Waiver**: Applies to a specific failed test case and its failed nested subtests where that suite supports test case waivers. +5. **SubTest-Level Waiver**: Applies to individual subtests. For BSA/SBSA, prefer `sub_Test_Path` for nested subtests. Waivers are applied in the order listed above. If a waiver is applicable at multiple levels, the most specific waiver (lowest level) takes precedence. @@ -112,7 +112,7 @@ Applies to all failed subtests within a specific sub-suite. Applicable for suite ### TestCase-Level Waiver -Applies to all failed subtests within a specific test case. Applicable for suites like `SCT`, `Standalone`, `BBSR-SCT`, and `BBSR-FWTS`. +Applies to a specific failed test case and its failed nested subtests where that suite supports test case waivers. For BSA/SBSA, use the `TestCases` array under a `TestSuite` entry. For SCT, Standalone, BBSR-SCT, and BBSR-FWTS, use the `TestCase` object form shown below. ```json { @@ -137,27 +137,39 @@ Applies to all failed subtests within a specific test case. Applicable for suite ### SubTest-Level Waiver -Applies to individual subtests based on `SubTestID` or `sub_Test_Description`. +Applies to individual subtests. + +For BSA/SBSA nested logs, use `sub_Test_Path` when possible. It is the most precise key because the same rule can appear under different parent rules. The parser also supports `sub_Test_Number`, legacy `sub_Rule_ID`, and exact `sub_Test_Description` matching for compatibility. + +For other suites, use the suite-specific key such as `SubTestID`, `sub_Test_GUID`, or `sub_Test_Description`. ```json { "Suites": [ { - "Suite": "SUITE_NAME", + "Suite": "BSA_OR_SBSA", "TestSuites": [ { - "TestCase": { - "SubTests": [ - { - "SubTestID": "SUB_TEST_ID", - "Reason": "REASON_FOR_WAIVER" - }, - { - "sub_Test_Description": "SUB_TEST_DESCRIPTION", - "Reason": "REASON_FOR_WAIVER" - } - ] - } + "TestSuite": "TEST_SUITE_NAME", + "TestCases": [ + { + "Test_case": "TEST_CASE_ID_OR_FULL_TEST_CASE", + "SubTests": [ + { + "sub_Test_Path": "TEST_CASE : INDEX / CHILD_RULE : INDEX / NESTED_RULE : INDEX", + "Reason": "REASON_FOR_WAIVER" + }, + { + "sub_Test_Number": "CHILD_RULE : INDEX", + "Reason": "REASON_FOR_WAIVER" + }, + { + "sub_Rule_ID": "LEGACY_RULE_ID", + "Reason": "REASON_FOR_WAIVER" + } + ] + } + ] } ] } @@ -165,8 +177,11 @@ Applies to individual subtests based on `SubTestID` or `sub_Test_Description`. } ``` -- **`SubTestID`**: Identifier of the subtest (used in suites like `BSA`, `SBSA`). -- **`sub_Test_Description`**: Description of the subtest (used in suites like `FWTS`, `Standalone`, `SCT`, `BBSR-FWTS`, `BBSR-SCT`). +- **`sub_Test_Path`**: Preferred BSA/SBSA nested subtest identifier. Use the exact value from the JSON or HTML row tooltip. +- **`sub_Test_Number`**: BSA/SBSA subtest number, for example `PCI_MM_01 : -`. This can be ambiguous if the same subtest number appears in more than one nested branch. +- **`sub_Rule_ID`**: Legacy BSA/SBSA key. It is still accepted by the waiver parser but is not emitted in new BSA/SBSA JSON. +- **`SubTestID` / `sub_Test_GUID`**: Identifier used by other suites where applicable. +- **`sub_Test_Description`**: Description of the subtest. Prefer path or ID matching when available. - **`Reason`**: Explanation for the waiver. --- @@ -229,7 +244,7 @@ Some suites (e.g., SCT) support sub-suite waivers. Example shown for completenes ] } ``` -Note: SubSuite/TestCase-level waivers are supported only for certain suites (SCT, Standalone, BBSR-SCT, BBSR-FWTS). Keep this as an example if you’re documenting behavior. +Note: SubSuite-level waivers are supported only for certain suites (SCT, Standalone, BBSR-SCT, BBSR-FWTS). TestCase-level waivers are also supported for BSA/SBSA through the `TestCases` array form. ### 4. TestCase-Level Waiver @@ -255,7 +270,63 @@ Waives all failed subtests within the specified test case only. ### 5. SubTest-Level Waiver -Targets specific subtests by exact description text. +For BSA/SBSA, target nested subtests by exact `sub_Test_Path`. This avoids ambiguity when the same rule appears in multiple branches. + +```json +{ + "Suites": [ + { + "Suite": "SBSA", + "TestSuites": [ + { + "TestSuite": "PCIE", + "TestCases": [ + { + "Test_case": "S_L6PCI_1", + "SubTests": [ + { + "sub_Test_Path": "S_L6PCI_1 : - / B_REP_1 : - / JKZMT : - / PCI_MM_01 : -", + "Reason": "Device memory decode is not supported on this platform." + } + ] + } + ] + } + ] + } + ] +} +``` + +Waivers using `sub_Test_Number` or legacy `sub_Rule_ID` still work, but they can match more than one nested occurrence when the same number or rule appears in multiple branches: + +```json +{ + "Suites": [ + { + "Suite": "SBSA", + "TestSuites": [ + { + "TestSuite": "PCIE", + "TestCases": [ + { + "Test_case": "S_L6PCI_1", + "SubTests": [ + { + "sub_Rule_ID": "PCI_MM_01", + "Reason": "Device memory decode is not supported on this platform." + } + ] + } + ] + } + ] + } + ] +} +``` + +For Standalone and similar suites, target specific subtests by exact description text. ```json { @@ -299,11 +370,11 @@ The `apply_waivers.py` script processes waivers in a hierarchical manner: 1. **Load Waivers**: It reads the `waiver.json` file and extracts waivers relevant to the specified `SUITE_NAME`. 2. **Apply Waivers**: - - **Suite-Level**: Waivers are applied to all failed subtests in the suite. - - **TestSuite-Level**: Waivers are applied to all failed subtests within the specified test suites. + - **Suite-Level**: Waivers are applied to all failed tests in the suite. + - **TestSuite-Level**: Waivers are applied to all failed tests within the specified test suites. - **SubSuite-Level**: Waivers are applied to all failed subtests within the specified sub-suites. - - **TestCase-Level**: Waivers are applied to all failed subtests within the specified test cases. - - **SubTest-Level**: Waivers are applied to individual subtests based on `SubTestID` or `sub_Test_Description`. + - **TestCase-Level**: Waivers are applied to the matching failed test case and its failed nested subtests. + - **SubTest-Level**: Waivers are applied to individual subtests. BSA/SBSA matching uses `sub_Test_Path`, then `sub_Test_Number`, then legacy `sub_Rule_ID`, then exact description. 3. **Update Test Results**: The script adjusts pass/fail counts and annotates waived tests with the waiver reason. 4. **Output**: The updated test results JSON file replaces the original file, reflecting the applied waivers. @@ -311,14 +382,14 @@ The `apply_waivers.py` script processes waivers in a hierarchical manner: ## Important Notes -- **Suites Supporting SubSuite/TestCase-Level Waivers**: Only certain suites (`SCT`, `Standalone`, `BBSR-SCT`, `BBSR-FWTS`) support waivers at the SubSuite and TestCase levels. +- **Suites Supporting SubSuite/TestCase-Level Waivers**: SubSuite-level waivers are supported only for certain suites (`SCT`, `Standalone`, `BBSR-SCT`, `BBSR-FWTS`). TestCase-level waivers are supported for BSA/SBSA and selected other suites. - **Waiver Application Order**: Waivers are applied from the highest level (suite) to the lowest level (subtest). Lower-level waivers override higher-level waivers if both are applicable. +- **Nested BSA/SBSA Subtests**: If all failed nested subtests under a failed BSA/SBSA parent are waived, the waiver state propagates up to the failed parent subtest and testcase. - **Updating Test Results**: The script modifies the original test results JSON file. Ensure you have a backup if you need to retain the original data. -- **Waiver Effect on Statistics**: Failed tests that are waived are counted under `FAILED_WITH_WAIVER`, and the overall failed count is adjusted accordingly. +- **Waiver Effect on Statistics**: Failed tests that are waived are counted under `FAILED_WITH_WAIVER` or `Total_failed_with_waiver`, depending on the suite summary format, and the overall failed count is adjusted accordingly. - **Error Handling**: The script skips waiver entries that are missing required fields and provides warnings if verbosity is enabled. - **Case Sensitivity**: Suite and test names are case-sensitive. Ensure names in `waiver.json` match those in the test results. --- By following the templates and guidelines provided in this document, you can effectively apply waivers to your test suite results, ensuring accurate representation of your platform's compliance and capabilities. -
{{ testcase.Test_case }} + {% if testcase.subtests %} + + {% else %} + + {% endif %} + {{ testcase.Test_case }} + {{ testcase.Test_case_description }} {{ testcase.Test_result }} @@ -327,10 +544,24 @@ def generate_html(suite_summary, test_results, chart_data, output_html_path, tes
- Subtests: - +
+ Subtests: + {% if testcase.has_nested_subtests %} +
+ + +
+ {% endif %} +
+
@@ -340,22 +571,7 @@ def generate_html(suite_summary, test_results, chart_data, output_html_path, tes - {% for subtest in testcase.subtests %} - - - - - - - {% endfor %} + {{ render_subtest_rows(testcase.subtests) }}
Sub Test Number
{{ subtest.sub_Test_Number }}{{ subtest.sub_Test_Description }} - {{ subtest.sub_test_result }} - - {% if 'FAILED (WITH WAIVER)' in subtest.sub_test_result %} - {{ subtest.waiver_reason | default("N/A") }} - {% else %} - N/A - {% endif %} -