diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 00000000..ff138a2a --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -u + +echo "===== PRE-COMMIT CHECKS =====" + +REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null)" || { + echo "Pre-commit hook requires a real git clone/worktree." + exit 1 +} + +cd "$REPO_ROOT" || exit 1 + +if command -v python3 >/dev/null 2>&1; then + PYTHON_BIN="python3" +elif command -v python >/dev/null 2>&1; then + PYTHON_BIN="python" +else + echo "Python not found in PATH." + exit 1 +fi + +if [ -f "$REPO_ROOT/common/acs_test_framework_runner/report.py" ]; then + REPORT_SCRIPT="$REPO_ROOT/common/acs_test_framework_runner/report.py" + +else + echo "report.py not found." + echo "Checked:" + echo " - $REPO_ROOT/common/acs_test_framework_runner/report.py" + exit 1 +fi + +HOOK_LOG_DIR="$REPO_ROOT/common/reports" +HOOK_LOG="$HOOK_LOG_DIR/precommit_report.log" + +mkdir -p "$HOOK_LOG_DIR" + +"$PYTHON_BIN" "$REPORT_SCRIPT" 2>&1 | tee "$HOOK_LOG" +STATUS=${PIPESTATUS[0]} + + +if [ "$STATUS" -ne 0 ]; then + echo "Pre-commit checks failed." + echo "Log saved at: $HOOK_LOG" + exit 1 +fi + +echo "All checks passed. Proceeding with commit." +exit 0 diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..87309575 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ + +__pycache__/ +*.py[cod] + +common/reports/ +reports/ +__pycache__/ +*.py[cod] + +pylint_recent.log diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 00000000..0d39b4a2 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,641 @@ +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint +# in a server-like mode. +clear-cache-post-run=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold under which the program will exit with error. +fail-under=10.0 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= + +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, +# it can't be used as an escape character. +ignore-paths= + +# Files or directories matching the regular expression patterns are skipped. +# The regex matches against base names, not paths. The default value ignores +# Emacs file locks +ignore-patterns=^\.# + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs=0 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.11 + +# Discover python modules and packages in the file system subtree. +recursive=no + +# Add paths to the list of the source roots. Supports globbing patterns. The +# source root is an absolute path or a path relative to the current working +# directory used to determine a package namespace for modules located under the +# source root. +source-roots= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. If left empty, argument names will be checked with the set +# naming style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +#class-attribute-rgx= + +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. If left empty, class names will be checked with the set naming style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. If left empty, function names will be checked with the set +# naming style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Regular expression matching correct type alias names. If left empty, type +# alias names will be checked with the set naming style. +#typealias-rgx= + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. If left empty, variable names will be checked with the set +# naming style. +#variable-rgx= + + +[CLASSES] + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + asyncSetUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= + +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.BaseException,builtins.Exception + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow explicit reexports by alias from a package __init__. +allow-reexport-from-package=no + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence=HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable= + R0801, + broad-exception-caught, + broad-exception-raised, + invalid-name, + logging-fstring-interpolation, + missing-class-docstring, + missing-function-docstring, + missing-module-docstring, + redefined-outer-name, + too-few-public-methods, + too-many-branches, + too-many-lines, + too-many-locals, + too-many-statements, + unspecified-encoding +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + + +[METHOD_ARGS] + +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + +# Regular expression of note tags to take in consideration. +notes-rgx= + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +#output-format= + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[SIMILARITIES] + +# Comments are removed from the similarity computation +ignore-comments=yes + +# Docstrings are removed from the similarity computation +ignore-docstrings=yes + +# Imports are removed from the similarity computation +ignore-imports=yes + +# Signatures are removed from the similarity computation +ignore-signatures=yes + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. No available dictionaries : You need to install +# both the python package and the system dependency for enchant to work.. +spelling-dict= + +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins=no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of names allowed to shadow builtins +allowed-redefined-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io + + diff --git a/common/acs_test_framework_manifests/acs-info.yaml b/common/acs_test_framework_manifests/acs-info.yaml new file mode 100644 index 00000000..ba882455 --- /dev/null +++ b/common/acs_test_framework_manifests/acs-info.yaml @@ -0,0 +1,283 @@ +suites: + - name: acs_info + files: + - common/log_parser/acs_info.py + + cases: + - name: file_exists + type: file_exists + + - name: python_compiles + type: py_compile + + - name: cli_help_flag + type: cli + args: + - --help + expect_exit_code: 0 + expect_stdout_or_stderr_contains: + - "Collect ACS-like system info" + + - name: cli_generates_json_from_dmidecode_only + type: cli + scenario: + kind: acs_info + dmidecode: + vendor: VendorA + product: BoardA + family: FamilyA + firmware: FW-1.2.3 + args: + - --dmidecode_log + - "{dir}/dmidecode.log" + - --output_dir + - "{dir}/out" + expect_exit_code: 0 + post_checks: + - type: exists + path: "{dir}/out/acs_info.json" + - type: file_contains + path: "{dir}/out/acs_info.json" + text: "\"Vendor\": \"VendorA\"" + - type: file_contains + path: "{dir}/out/acs_info.json" + text: "\"System Name\": \"BoardA\"" + - type: file_contains + path: "{dir}/out/acs_info.json" + text: "\"SoC Family\": \"FamilyA\"" + - type: file_contains + path: "{dir}/out/acs_info.json" + text: "\"Firmware Version\": \"FW-1.2.3\"" + + - name: cli_merges_acs_and_system_config + type: cli + scenario: + kind: acs_info + dmidecode: + vendor: VendorB + product: BoardB + family: FamilyB + firmware: FW-2.0 + acs_config: + Band: SystemReady Band + ACS version: "6.1" + system_config: + Board Revision: RevC + args: + - --dmidecode_log + - "{dir}/dmidecode.log" + - --acs_config_path + - "{dir}/acs_config.txt" + - --system_config_path + - "{dir}/system_config.txt" + - --output_dir + - "{dir}/out" + expect_exit_code: 0 + post_checks: + - type: file_contains + path: "{dir}/out/acs_info.json" + text: "\"ACS version\": \"6.1\"" + - type: file_contains + path: "{dir}/out/acs_info.json" + text: "\"Board Revision\": \"RevC\"" + - type: file_contains + path: "{dir}/out/acs_info.json" + text: "\"Band\": \"SystemReady Band\"" + + - name: cli_stops_parsing_at_user_defined_configs_marker + type: cli + scenario: + kind: acs_info + dmidecode: + vendor: VendorC + product: BoardC + family: FamilyC + firmware: FW-3.0 + text_files: + acs_config.txt: | + Band: SystemReady Band + ACS version: 7.0 + # User-defined configs + Secret Field: should_not_appear + args: + - --dmidecode_log + - "{dir}/dmidecode.log" + - --acs_config_path + - "{dir}/acs_config.txt" + - --output_dir + - "{dir}/out" + expect_exit_code: 0 + post_checks: + - type: file_contains + path: "{dir}/out/acs_info.json" + text: "\"ACS version\": \"7.0\"" + - type: file_not_contains + path: "{dir}/out/acs_info.json" + text: "should_not_appear" + + - name: cli_reads_utf16_uefi_version + type: cli + scenario: + kind: acs_info + dmidecode: + vendor: VendorD + product: BoardD + family: FamilyD + firmware: FW-4.0 + args: + - --dmidecode_log + - "{dir}/dmidecode.log" + - --uefi_version_log + - "{dir}/uefi_version.log" + - --output_dir + - "{dir}/out" + bin_files: + uefi_version.log: + hex: "fffe55004500460049002000760032002e0039000a00" + expect_exit_code: 0 + post_checks: + - type: file_contains + path: "{dir}/out/acs_info.json" + text: "\"UEFI Version\": \"UEFI v2.9\"" + + - name: cli_adds_bmc_firmware_only_for_systemready_band + type: cli + scenario: + kind: acs_info + dmidecode: + vendor: VendorE + product: BoardE + family: FamilyE + firmware: FW-5.0 + acs_config: + Band: SystemReady Band + text_files: + ipmitool.log: | + Device ID : 32 + Firmware Revision : 1.45 + args: + - --dmidecode_log + - "{dir}/dmidecode.log" + - --acs_config_path + - "{dir}/acs_config.txt" + - --ipmitool_log + - "{dir}/ipmitool.log" + - --output_dir + - "{dir}/out" + expect_exit_code: 0 + post_checks: + - type: file_contains + path: "{dir}/out/acs_info.json" + text: "\"BMC Firmware Version\": \"1.45\"" + + - name: cli_does_not_add_bmc_firmware_for_non_systemready_band + type: cli + scenario: + kind: acs_info + dmidecode: + vendor: VendorF + product: BoardF + family: FamilyF + firmware: FW-6.0 + acs_config: + Band: DeviceTree Band + text_files: + ipmitool.log: | + Firmware Revision : 9.99 + args: + - --dmidecode_log + - "{dir}/dmidecode.log" + - --acs_config_path + - "{dir}/acs_config.txt" + - --ipmitool_log + - "{dir}/ipmitool.log" + - --output_dir + - "{dir}/out" + expect_exit_code: 0 + post_checks: + - type: file_not_contains + path: "{dir}/out/acs_info.json" + text: "\"BMC Firmware Version\"" + + - name: cli_uses_first_dmidecode_values_not_later_duplicates + type: cli + text_files: + dmidecode.log: | + Handle 0x0000, DMI type 0, 24 bytes + BIOS Information + Version: FW-FIRST + Handle 0x0001, DMI type 1, 27 bytes + System Information + Manufacturer: VendorFirst + Product Name: ProductFirst + Family: FamilyFirst + Handle 0x0002, DMI type 0, 24 bytes + BIOS Information + Version: FW-SECOND + Handle 0x0003, DMI type 1, 27 bytes + System Information + Manufacturer: VendorSecond + Product Name: ProductSecond + Family: FamilySecond + args: + - --dmidecode_log + - "{dir}/dmidecode.log" + - --output_dir + - "{dir}/out" + expect_exit_code: 0 + post_checks: + - type: file_contains + path: "{dir}/out/acs_info.json" + text: "\"Vendor\": \"VendorFirst\"" + - type: file_not_contains + path: "{dir}/out/acs_info.json" + text: "VendorSecond" + + - name: cli_systemready_band_is_case_and_space_insensitive + type: cli + scenario: + kind: acs_info + dmidecode: + vendor: V + product: P + family: F + firmware: FW-1 + text_files: + acs_config.txt: | + Band: systemready band + args: + - --dmidecode_log + - "{dir}/dmidecode.log" + - --acs_config_path + - "{dir}/acs_config.txt" + - --output_dir + - "{dir}/out" + expect_exit_code: 0 + post_checks: + - type: file_contains + path: "{dir}/out/acs_info.json" + text: "\"BMC Firmware Version\": \"Unknown\"" + + - name: cli_config_value_with_extra_colon_is_preserved + type: cli + scenario: + kind: acs_info + dmidecode: + vendor: V + product: P + family: F + firmware: FW-1 + system_config: + Firmware Notes: build:nightly:123 + args: + - --dmidecode_log + - "{dir}/dmidecode.log" + - --system_config_path + - "{dir}/system_config.txt" + - --output_dir + - "{dir}/out" + expect_exit_code: 0 + post_checks: + - type: file_contains + path: "{dir}/out/acs_info.json" + text: "\"Firmware Notes\": \"build:nightly:123\"" \ No newline at end of file diff --git a/common/acs_test_framework_manifests/apply_waivers.yaml b/common/acs_test_framework_manifests/apply_waivers.yaml new file mode 100644 index 00000000..5396335c --- /dev/null +++ b/common/acs_test_framework_manifests/apply_waivers.yaml @@ -0,0 +1,702 @@ +suites: + - name: apply_waivers + files: + - common/log_parser/apply_waivers.py + + cases: + - name: file_exists + type: file_exists + + - name: python_compiles + type: py_compile + + - name: cli_help_flag + type: cli + args: + - --help + expect_exit_code: 0 + expect_stdout_or_stderr_contains: + - "Apply waivers to test suite JSON results." + + - name: cli_missing_json_file_returns_zero_with_warning + type: cli + args: + - FWTS + - "{dir}/missing.json" + expect_exit_code: 0 + expect_stdout_or_stderr_contains: + - "WARNING: Failed to read or parse" + + - name: fwts_suite_level_waiver_marks_failed_subtest + type: cli + text_files: + fwts.json: | + { + "test_results": [ + { + "Test_suite": "DemoSuite", + "subtests": [ + { + "sub_Test_Description": "failure one", + "sub_Test_Number": "1", + "sub_test_result": "FAILED" + } + ] + } + ] + } + waiver.json: | + { + "Suites": [ + { + "Suite": "FWTS", + "Reason": "Known FWTS issue" + } + ] + } + args: + - FWTS + - "{dir}/fwts.json" + - "{dir}/waiver.json" + expect_exit_code: 0 + post_checks: + - type: file_contains + path: "{dir}/fwts.json" + text: "WITH WAIVER" + - type: file_not_contains + path: "{dir}/fwts.json" + text: "(WITH WAIVER) (WITH WAIVER)" + + - name: testsuite_level_waiver_applies_only_to_matching_testsuite + type: cli + text_files: + fwts.json: | + { + "test_results": [ + { + "Test_suite": "MatchSuite", + "subtests": [ + { + "sub_Test_Description": "alpha", + "sub_Test_Number": "1", + "sub_test_result": "FAILED" + } + ] + }, + { + "Test_suite": "OtherSuite", + "subtests": [ + { + "sub_Test_Description": "beta", + "sub_Test_Number": "1", + "sub_test_result": "FAILED" + } + ] + } + ] + } + waiver.json: | + { + "Suites": [ + { + "Suite": "FWTS", + "TestSuites": [ + { + "TestSuite": "MatchSuite", + "Reason": "Specific suite waiver" + } + ] + } + ] + } + args: + - FWTS + - "{dir}/fwts.json" + - "{dir}/waiver.json" + expect_exit_code: 0 + post_checks: + - type: file_contains + path: "{dir}/fwts.json" + text: "Specific suite waiver" + - type: file_contains + path: "{dir}/fwts.json" + text: "\"Test_suite\": \"OtherSuite\"" + + - name: category_non_waivable_blocks_waiver_application + type: cli + text_files: + fwts.json: | + { + "test_results": [ + { + "Test_suite": "DemoSuite", + "subtests": [ + { + "sub_Test_Description": "failure one", + "sub_Test_Number": "1", + "sub_test_result": "FAILED" + } + ] + } + ] + } + waiver.json: | + { + "Suites": [ + { + "Suite": "FWTS", + "Reason": "Known FWTS issue" + } + ] + } + category.json: | + { + "cat1": [ + { + "Suite": "FWTS", + "Test Suite": "DemoSuite", + "Waivable": "No" + } + ] + } + args: + - FWTS + - "{dir}/fwts.json" + - "{dir}/waiver.json" + - "{dir}/category.json" + expect_exit_code: 0 + post_checks: + - type: file_not_contains + path: "{dir}/fwts.json" + text: "WITH WAIVER" + + - name: category_waivable_yes_allows_waiver_application + type: cli + text_files: + fwts.json: | + { + "test_results": [ + { + "Test_suite": "DemoSuite", + "subtests": [ + { + "sub_Test_Description": "failure one", + "sub_Test_Number": "1", + "sub_test_result": "FAILED" + } + ] + } + ] + } + waiver.json: | + { + "Suites": [ + { + "Suite": "FWTS", + "Reason": "Known FWTS issue" + } + ] + } + category.json: | + { + "cat1": [ + { + "Suite": "FWTS", + "Test Suite": "DemoSuite", + "Waivable": "Yes" + } + ] + } + args: + - FWTS + - "{dir}/fwts.json" + - "{dir}/waiver.json" + - "{dir}/category.json" + expect_exit_code: 0 + post_checks: + - type: file_contains + path: "{dir}/fwts.json" + text: "WITH WAIVER" + - type: file_not_contains + path: "{dir}/fwts.json" + text: "(WITH WAIVER) (WITH WAIVER)" + + - name: fwts_subtest_description_waiver_applies + type: cli + text_files: + fwts.json: | + { + "test_results": [ + { + "Test_suite": "DemoSuite", + "subtests": [ + { + "sub_Test_Description": "something", + "sub_Test_Number": "42", + "sub_test_result": "FAILED" + } + ] + } + ] + } + waiver.json: | + { + "Suites": [ + { + "Suite": "FWTS", + "TestSuites": [ + { + "TestSuite": "DemoSuite", + "TestCase": { + "SubTests": [ + { + "sub_Test_Description": "something", + "Reason": "Description-based waiver" + } + ] + } + } + ] + } + ] + } + args: + - FWTS + - "{dir}/fwts.json" + - "{dir}/waiver.json" + expect_exit_code: 0 + post_checks: + - type: file_contains + path: "{dir}/fwts.json" + text: "WITH WAIVER" + - type: file_contains + path: "{dir}/fwts.json" + text: "Description-based waiver" + + - name: standalone_subtest_description_waiver_applies + type: cli + text_files: + standalone.json: | + { + "test_results": [ + { + "Test_suite": "Network", + "Test_case": "ethtool_test", + "subtests": [ + { + "sub_Test_Description": "Ping to www.arm.com on eth0", + "sub_test_result": "FAILED" + } + ], + "test_suite_summary": { + "total_passed": 0, + "total_failed": 1, + "total_failed_with_waiver": 0, + "total_aborted": 0, + "total_skipped": 0, + "total_warnings": 0 + } + } + ], + "suite_summary": { + "total_passed": 0, + "total_failed": 1, + "total_failed_with_waiver": 0, + "total_aborted": 0, + "total_skipped": 0, + "total_warnings": 0 + } + } + waiver.json: | + { + "Suites": [ + { + "Suite": "Standalone", + "TestSuites": [ + { + "TestCase": { + "Test_case": "ethtool_test", + "SubTests": [ + { + "sub_Test_Description": "Ping to www.arm.com on eth0", + "Reason": "Network stack excluded for this SKU" + } + ] + } + } + ] + } + ] + } + args: + - Standalone + - "{dir}/standalone.json" + - "{dir}/waiver.json" + expect_exit_code: 0 + post_checks: + - type: regex + path: "{dir}/standalone.json" + pattern: '"sub_test_result": "[^"]*WITH WAIVER[^"]*"' + - type: file_contains + path: "{dir}/standalone.json" + text: "Network stack excluded for this SKU" + - type: file_contains + path: "{dir}/standalone.json" + text: "\"total_failed\": 0" + - type: file_contains + path: "{dir}/standalone.json" + text: "\"total_failed_with_waiver\": 1" + + - name: fwts_does_not_duplicate_existing_with_waiver_marker + type: cli + text_files: + fwts.json: | + { + "test_results": [ + { + "Test_suite": "DemoSuite", + "subtests": [ + { + "sub_Test_Description": "failure one", + "sub_Test_Number": "1", + "sub_test_result": "FAILED (WITH WAIVER)" + } + ] + } + ] + } + waiver.json: | + { + "Suites": [ + { + "Suite": "FWTS", + "Reason": "Known issue" + } + ] + } + args: + - FWTS + - "{dir}/fwts.json" + - "{dir}/waiver.json" + expect_exit_code: 0 + post_checks: + - type: file_contains + path: "{dir}/fwts.json" + text: "WITH WAIVER" + - type: file_not_contains + path: "{dir}/fwts.json" + text: "(WITH WAIVER) (WITH WAIVER)" + + - name: malformed_waiver_file_does_not_modify_json + type: cli + text_files: + fwts.json: | + { + "test_results": [ + { + "Test_suite": "DemoSuite", + "subtests": [ + { + "sub_Test_Description": "failure one", + "sub_Test_Number": "1", + "sub_test_result": "FAILED" + } + ] + } + ] + } + waiver.json: | + { invalid json + args: + - FWTS + - "{dir}/fwts.json" + - "{dir}/waiver.json" + expect_exit_code: 0 + expect_stdout_or_stderr_contains: + - "INFO: Failed to read or parse" + post_checks: + - type: file_not_contains + path: "{dir}/fwts.json" + text: "WITH WAIVER" + + - name: bsa_testcase_waiver_marks_failed_case + type: cli + text_files: + bsa.json: | + { + "test_results": [ + { + "Test_suite": "RuleGroupA", + "testcases": [ + { + "Test_case": "RULE_001", + "Test_result": "FAILED" + } + ] + } + ] + } + waiver.json: | + { + "Suites": [ + { + "Suite": "BSA", + "TestSuites": [ + { + "TestSuite": "RuleGroupA", + "TestCases": [ + { + "Test_case": "RULE_001", + "Reason": "BSA waiver" + } + ] + } + ] + } + ] + } + args: + - BSA + - "{dir}/bsa.json" + - "{dir}/waiver.json" + expect_exit_code: 0 + post_checks: + - type: file_contains + path: "{dir}/bsa.json" + text: "WITH WAIVER" + - type: file_not_contains + path: "{dir}/bsa.json" + text: "(WITH WAIVER) (WITH WAIVER)" + + - name: quiet_mode_runs_and_updates_json + type: cli + text_files: + fwts.json: | + { + "test_results": [ + { + "Test_suite": "DemoSuite", + "subtests": [ + { + "sub_Test_Description": "failure one", + "sub_Test_Number": "1", + "sub_test_result": "FAILED" + } + ] + } + ] + } + waiver.json: | + { + "Suites": [ + { + "Suite": "FWTS", + "Reason": "Known FWTS issue" + } + ] + } + args: + - FWTS + - "{dir}/fwts.json" + - "{dir}/waiver.json" + - --quiet + expect_exit_code: 0 + post_checks: + - type: file_contains + path: "{dir}/fwts.json" + text: "WITH WAIVER" + + - name: no_matching_suite_in_waiver_file_leaves_json_unchanged + type: cli + text_files: + fwts.json: | + { + "test_results": [ + { + "Test_suite": "DemoSuite", + "subtests": [ + { + "sub_Test_Description": "failure one", + "sub_Test_Number": "1", + "sub_test_result": "FAILED" + } + ] + } + ] + } + waiver.json: | + { + "Suites": [ + { + "Suite": "BSA", + "Reason": "Wrong suite waiver" + } + ] + } + args: + - FWTS + - "{dir}/fwts.json" + - "{dir}/waiver.json" + expect_exit_code: 0 + expect_stdout_or_stderr_contains: + - "No valid waivers found for suite 'FWTS'" + post_checks: + - type: file_not_contains + path: "{dir}/fwts.json" + text: "WITH WAIVER" + + - name: missing_reason_in_waiver_is_ignored + type: cli + text_files: + fwts.json: | + { + "test_results": [ + { + "Test_suite": "DemoSuite", + "subtests": [ + { + "sub_Test_Description": "failure one", + "sub_Test_Number": "1", + "sub_test_result": "FAILED" + } + ] + } + ] + } + waiver.json: | + { + "Suites": [ + { + "Suite": "FWTS", + "TestSuites": [ + { + "TestSuite": "DemoSuite", + "Reason": "" + } + ] + } + ] + } + args: + - FWTS + - "{dir}/fwts.json" + - "{dir}/waiver.json" + expect_exit_code: 0 + expect_stdout_or_stderr_contains: + - "missing 'Reason'" + post_checks: + - type: file_not_contains + path: "{dir}/fwts.json" + text: "WITH WAIVER" + + + + - name: bsa_subtest_waiver_applies_by_sub_rule_id + type: cli + text_files: + bsa.json: | + { + "test_results": [ + { + "Test_suite": "RuleGroupA", + "testcases": [ + { + "Test_case": "RULE_001", + "Test_result": "FAILED", + "subtests": [ + { + "sub_Rule_ID": "SUB_01", + "sub_Test_Description": "first subtest", + "sub_test_result": "FAILED" + }, + { + "sub_Rule_ID": "SUB_02", + "sub_Test_Description": "second subtest", + "sub_test_result": "PASSED" + } + ] + } + ] + } + ] + } + waiver.json: | + { + "Suites": [ + { + "Suite": "BSA", + "TestSuites": [ + { + "TestSuite": "RuleGroupA", + "TestCases": [ + { + "Test_case": "RULE_001", + "Reason": "Case waiver placeholder", + "SubTests": [ + { + "sub_Rule_ID": "SUB_01", + "Reason": "BSA subtest waiver" + } + ] + } + ] + } + ] + } + ] + } + args: + - BSA + - "{dir}/bsa.json" + - "{dir}/waiver.json" + expect_exit_code: 0 + post_checks: + - type: file_contains + path: "{dir}/bsa.json" + text: "BSA subtest waiver" + - type: file_contains + path: "{dir}/bsa.json" + text: "WITH WAIVER" + - type: file_contains + path: "{dir}/bsa.json" + text: "\"sub_Rule_ID\": \"SUB_02\"" + + - name: passed_result_is_not_modified_by_waiver + type: cli + text_files: + fwts.json: | + { + "test_results": [ + { + "Test_suite": "DemoSuite", + "subtests": [ + { + "sub_Test_Description": "already passed", + "sub_Test_Number": "1", + "sub_test_result": "PASSED" + } + ] + } + ] + } + waiver.json: | + { + "Suites": [ + { + "Suite": "FWTS", + "Reason": "Should not affect passed results" + } + ] + } + args: + - FWTS + - "{dir}/fwts.json" + - "{dir}/waiver.json" + expect_exit_code: 0 + post_checks: + - type: file_contains + path: "{dir}/fwts.json" + text: "\"sub_test_result\": \"PASSED\"" + - type: file_not_contains + path: "{dir}/fwts.json" + text: "WITH WAIVER" diff --git a/common/acs_test_framework_manifests/capsule-ondisk-reporting-vars-check.yaml b/common/acs_test_framework_manifests/capsule-ondisk-reporting-vars-check.yaml new file mode 100644 index 00000000..d2561974 --- /dev/null +++ b/common/acs_test_framework_manifests/capsule-ondisk-reporting-vars-check.yaml @@ -0,0 +1,839 @@ +suites: + - name: capsule_ondisk_reporting_vars_check + files: + - common/linux_scripts/capsule_ondisk_reporting_vars_check.py + + cases: + - name: file_exists + type: file_exists + description: "Verify that the capsule on-disk reporting variables checker script exists." + + - name: python_compiles + type: py_compile + description: "Ensure that the script compiles successfully as valid Python." + + - name: has_main_guard + type: main_guard + description: "Check that the script contains a standard __main__ guard." + + - name: status_true_returns_passed + type: py_function + function: status + args: [true] + expect_return: "PASSED" + + - name: status_false_returns_warning + type: py_function + function: status + args: [false] + expect_return: "WARNING" + + - name: decode_valid_utf16le_returns_capsule_name + type: py_function + function: decode_char16_11_no_nul + args: + - !!binary | + QwBhAHAAcwB1AGwAZQAwADAAMAAxAA== + expect_return: "Capsule0001" + + - name: missing_efivarfs_returns_1_and_logs_warning + type: module_main_with_env + scenario: + kind: capsule_vars + efivarfs_present: false + efivar_dir: missing_efivars + log_file: capsule.log + expect_exit_code: 1 + post_checks: + - type: exists + path: "{dir}/capsule.log" + - type: file_contains + path: "{dir}/capsule.log" + text: "Please ensure efivarfs is enabled and mounted" + - type: file_contains + path: "{dir}/capsule.log" + text: "Overall Capsule On-Disk Update Reporting Variables Result: WARNING" + + - name: missing_os_indications_returns_2_and_skips + type: module_main_with_env + scenario: + kind: capsule_vars + log_file: capsule.log + expect_exit_code: 2 + post_checks: + - type: exists + path: "{dir}/capsule.log" + - type: file_contains + path: "{dir}/capsule.log" + text: "OsIndicationsSupported not found or unreadable" + - type: file_contains + path: "{dir}/capsule.log" + text: "Overall Capsule On-Disk Update Reporting Variables Result: SKIPPED" + + - name: os_indications_without_capsule_support_bit_returns_2 + type: module_main_with_env + scenario: + kind: capsule_vars + log_file: capsule.log + os_indications: + value: 0 + attrs: 0x07 + expect_exit_code: 2 + post_checks: + - type: file_contains + path: "{dir}/capsule.log" + text: "INFO: OsIndicationsSupported value: 0x0" + - type: file_contains + path: "{dir}/capsule.log" + text: "INFO: Capsule on-disk reporting variables test is not applicable - SKIPPED" + - type: file_contains + path: "{dir}/capsule.log" + text: "Overall Capsule On-Disk Update Reporting Variables Result: SKIPPED" + + - name: os_indications_short_data_treated_as_unreadable + type: module_main_with_env + scenario: + kind: capsule_vars + log_file: capsule.log + os_indications: + hex: "070000000400" + expect_exit_code: 2 + post_checks: + - type: file_contains + path: "{dir}/capsule.log" + text: "OsIndicationsSupported not found or unreadable" + - type: file_contains + path: "{dir}/capsule.log" + text: "Overall Capsule On-Disk Update Reporting Variables Result: SKIPPED" + + - name: support_claimed_but_missing_capsule_variables_returns_3 + type: module_main_with_env + scenario: + kind: capsule_vars + log_file: capsule.log + os_indications: + supported: true + attrs: 0x07 + expect_exit_code: 3 + post_checks: + - type: file_contains + path: "{dir}/capsule.log" + text: "CapsuleMax Variable - Not Found or Not Accessible" + - type: file_contains + path: "{dir}/capsule.log" + text: "CapsuleLast Variable - Not Found or Not Accessible" + - type: file_contains + path: "{dir}/capsule.log" + text: "No CapsuleNNNN reporting variables found - WARNING" + - type: file_contains + path: "{dir}/capsule.log" + text: "Overall Capsule On-Disk Update Reporting Variables Result: WARNING" + + - name: capsulemax_wrong_attributes_returns_3 + type: module_main_with_env + scenario: + kind: capsule_vars + log_file: capsule.log + os_indications: + supported: true + attrs: 0x07 + capsule_max: + name: Capsule0001 + attrs: 0x07 + capsule_last: + name: Capsule0001 + attrs: 0x07 + capsule_entries: + - name: Capsule0001 + attrs: 0x07 + payload_hex: "AA" + expect_exit_code: 3 + post_checks: + - type: file_contains + path: "{dir}/capsule.log" + text: "CapsuleMax Variable Attribute Test" + - type: file_contains + path: "{dir}/capsule.log" + text: "actual attributes Value:0x7" + - type: file_contains + path: "{dir}/capsule.log" + text: "Status - WARNING" + + - name: capsulelast_wrong_attributes_returns_3 + type: module_main_with_env + scenario: + kind: capsule_vars + log_file: capsule.log + os_indications: + supported: true + attrs: 0x07 + capsule_max: + name: Capsule0001 + attrs: 0x06 + capsule_last: + name: Capsule0001 + attrs: 0x06 + capsule_entries: + - name: Capsule0001 + attrs: 0x07 + payload_hex: "AA" + expect_exit_code: 3 + post_checks: + - type: file_contains + path: "{dir}/capsule.log" + text: "CapsuleLast Variable Attribute Test" + - type: file_contains + path: "{dir}/capsule.log" + text: "actual attributes Value:0x6" + - type: file_contains + path: "{dir}/capsule.log" + text: "Status - WARNING" + + - name: malformed_capsulelast_value_returns_3 + type: module_main_with_env + scenario: + kind: capsule_vars + log_file: capsule.log + os_indications: + supported: true + attrs: 0x07 + capsule_max: + name: Capsule0001 + attrs: 0x06 + capsule_last: + hex: "07000000420061006400560061006c0075006500000000000000" + capsule_entries: + - name: Capsule0001 + attrs: 0x07 + payload_hex: "AA" + expect_exit_code: 3 + post_checks: + - type: file_contains + path: "{dir}/capsule.log" + text: "CapsuleLast Variable Test: WARNING" + - type: file_contains + path: "{dir}/capsule.log" + text: "Overall Capsule On-Disk Update Reporting Variables Result: WARNING" + + - name: capsulemax_too_short_returns_3 + type: module_main_with_env + scenario: + kind: capsule_vars + log_file: capsule.log + os_indications: + supported: true + attrs: 0x07 + capsule_max: + hex: "0600000041004200" + capsule_last: + name: Capsule0001 + attrs: 0x07 + capsule_entries: + - name: Capsule0001 + attrs: 0x07 + payload_hex: "AA" + expect_exit_code: 3 + post_checks: + - type: file_contains + path: "{dir}/capsule.log" + text: "CapsuleMax Variable Test: WARNING" + + - name: missing_capsulelast_returns_3 + type: module_main_with_env + scenario: + kind: capsule_vars + log_file: capsule.log + os_indications: + supported: true + attrs: 0x07 + capsule_max: + name: Capsule0001 + attrs: 0x06 + capsule_entries: + - name: Capsule0001 + attrs: 0x07 + payload_hex: "AA" + expect_exit_code: 3 + post_checks: + - type: file_contains + path: "{dir}/capsule.log" + text: "CapsuleLast Variable - Not Found or Not Accessible" + + - name: inaccessible_capsule_nnnn_variable_returns_3 + type: module_main_with_env + scenario: + kind: capsule_vars + log_file: capsule.log + os_indications: + supported: true + attrs: 0x07 + capsule_max: + name: Capsule0001 + attrs: 0x06 + capsule_last: + name: Capsule0001 + attrs: 0x07 + capsule_entries: + - name: Capsule0001 + hex: "01" + expect_exit_code: 3 + post_checks: + - type: file_contains + path: "{dir}/capsule.log" + text: "CapsuleNNNN Variable Test: Capsule0001 Variable Test: WARNING" + + - name: capsule_nnnn_wrong_attributes_returns_3 + type: module_main_with_env + scenario: + kind: capsule_vars + log_file: capsule.log + os_indications: + supported: true + attrs: 0x07 + capsule_max: + name: Capsule0001 + attrs: 0x06 + capsule_last: + name: Capsule0001 + attrs: 0x07 + capsule_entries: + - name: Capsule0001 + attrs: 0x06 + payload_hex: "AA" + expect_exit_code: 3 + post_checks: + - type: file_contains + path: "{dir}/capsule.log" + text: "CapsuleNNNN Variable Test: Capsule0001 Variable Test: PASSED" + - type: file_contains + path: "{dir}/capsule.log" + text: "actual attributes Value:0x6" + - type: file_contains + path: "{dir}/capsule.log" + text: "Status - WARNING" + + - name: multiple_capsule_nnnn_one_bad_one_good_returns_3 + type: module_main_with_env + scenario: + kind: capsule_vars + log_file: capsule.log + os_indications: + supported: true + attrs: 0x07 + capsule_max: + name: Capsule0002 + attrs: 0x06 + capsule_last: + name: Capsule0002 + attrs: 0x07 + capsule_entries: + - name: Capsule0001 + attrs: 0x07 + payload_hex: "AA" + - name: Capsule0002 + attrs: 0x06 + payload_hex: "BB" + expect_exit_code: 3 + post_checks: + - type: file_contains + path: "{dir}/capsule.log" + text: "CapsuleNNNN Variable Test: Capsule0001 Variable Test: PASSED" + - type: file_contains + path: "{dir}/capsule.log" + text: "CapsuleNNNN Variable Test: Capsule0002 Variable Test: PASSED" + - type: file_contains + path: "{dir}/capsule.log" + text: "actual attributes Value:0x6" + - type: file_contains + path: "{dir}/capsule.log" + text: "Overall Capsule On-Disk Update Reporting Variables Result: WARNING" + + - name: all_valid_variables_return_0_and_pass + type: module_main_with_env + patch_constants: + EFIVAR_PATH: "{dir}/efivars" + LOG_FILE: "{dir}/capsule.log" + dir_structure: + - path: efivars + bin_files: + efivars/OsIndicationsSupported-8be4df61-93ca-11d2-aa0d-00e098032b8c: + hex: "070000000400000000000000" + efivars/CapsuleMax-39b68c46-f7fb-441b-b6ec-16b0f69821f3: + hex: "06000000430061007000730075006c0065003000300030003100" + efivars/CapsuleLast-39b68c46-f7fb-441b-b6ec-16b0f69821f3: + hex: "07000000430061007000730075006c0065003000300030003100" + efivars/Capsule0001-39b68c46-f7fb-441b-b6ec-16b0f69821f3: + hex: "07000000AA" + expect_exit_code: 0 + post_checks: + - type: ordered_contains + path: "{dir}/capsule.log" + texts: + - "Testing Capsule On-Disk Update Reporting Variables" + - "CapsuleMax Variable Test: PASSED" + - "CapsuleLast Variable Test: PASSED" + - "CapsuleNNNN Variable Test: Capsule0001 Variable Test: PASSED" + - "Overall Capsule On-Disk Update Reporting Variables Result: PASSED" + + - name: cli_mock_open_raises_oserror_for_existing_capsulelast + type: module_main_with_env + scenario: + kind: capsule_vars + log_file: capsule.log + os_indications: + supported: true + attrs: 0x07 + capsule_max: + name: Capsule0001 + attrs: 0x06 + capsule_last: + name: Capsule0001 + attrs: 0x07 + capsule_entries: + - name: Capsule0001 + attrs: 0x07 + payload_hex: "AA" + mocks: + builtins.open: + factory: mock_helpers.passthrough_router + inject_original_as: real + kwargs: + rules: + - label: capsulelast-read-failure + required: true + when: + args: + 0: + contains: "CapsuleLast-39b68c46-f7fb-441b-b6ec-16b0f69821f3" + 1: + contains: "rb" + raise: + type: py:builtins.OSError + args: ["mocked read failure"] + expect_exit_code: 3 + post_checks: + - type: file_contains + path: "{dir}/capsule.log" + text: "CapsuleLast Variable - Not Found or Not Accessible" + - type: file_contains + path: "{dir}/capsule.log" + text: "Overall Capsule On-Disk Update Reporting Variables Result: WARNING" + + - name: cli_mock_listdir_failure_for_existing_efivarfs + type: module_main_with_env + scenario: + kind: capsule_vars + log_file: capsule.log + os_indications: + supported: true + attrs: 0x07 + capsule_max: + name: Capsule0001 + attrs: 0x06 + capsule_last: + name: Capsule0001 + attrs: 0x07 + mocks: + "{module}.os.listdir": + factory: mock_helpers.passthrough_router + inject_original_as: real + kwargs: + rules: + - label: efivarfs-listdir-failure + required: true + when: + args: + 0: + contains: "efivars" + raise: + type: py:builtins.OSError + args: ["mocked listdir failure"] + expect_exit_code: 3 + post_checks: + - type: file_contains + path: "{dir}/capsule.log" + text: "efivarfs not accessible - WARNING" + - type: file_contains + path: "{dir}/capsule.log" + text: "Overall Capsule On-Disk Update Reporting Variables Result: WARNING" + + - name: cli_mock_read_efi_var_os_indications_unreadable_even_when_file_exists + type: module_main_with_env + scenario: + kind: capsule_vars + log_file: capsule.log + os_indications: + supported: true + attrs: 0x07 + mocks: + "{module}.read_efi_var": + factory: mock_helpers.passthrough_router + inject_original_as: real + kwargs: + rules: + - label: os-indications-read-returns-none + required: true + when: + args: + 0: + equals: "OsIndicationsSupported" + return: [null, null] + expect_exit_code: 2 + post_checks: + - type: file_contains + path: "{dir}/capsule.log" + text: "OsIndicationsSupported not found or unreadable" + - type: file_contains + path: "{dir}/capsule.log" + text: "Overall Capsule On-Disk Update Reporting Variables Result: SKIPPED" + + - name: cli_mock_read_efi_var_capsulemax_missing_even_when_file_exists + type: module_main_with_env + scenario: + kind: capsule_vars + log_file: capsule.log + os_indications: + supported: true + attrs: 0x07 + capsule_max: + name: Capsule0001 + attrs: 0x06 + capsule_last: + name: Capsule0001 + attrs: 0x07 + capsule_entries: + - name: Capsule0001 + attrs: 0x07 + payload_hex: "AA" + mocks: + "{module}.read_efi_var": + factory: mock_helpers.passthrough_router + inject_original_as: real + kwargs: + rules: + - label: capsulemax-read-returns-none + required: true + when: + args: + 0: + equals: "CapsuleMax" + return: [null, null] + expect_exit_code: 3 + post_checks: + - type: file_contains + path: "{dir}/capsule.log" + text: "CapsuleMax Variable - Not Found or Not Accessible" + - type: file_contains + path: "{dir}/capsule.log" + text: "Overall Capsule On-Disk Update Reporting Variables Result: WARNING" + + - name: ondisk_supported_without_capsule_entries_currently_passes_overall + type: module_main_with_env + warn_only: true + scenario: + kind: capsule_vars + log_file: capsule.log + os_indications: + supported: true + attrs: 0x06 + capsule_max: + name: Capsule0001 + attrs: 0x06 + capsule_last: + name: Capsule0001 + attrs: 0x07 + expect_exit_code: 0 + post_checks: + - type: file_contains + path: "{dir}/capsule.log" + text: "No CapsuleNNNN reporting variables found - WARNING" + - type: file_contains + path: "{dir}/capsule.log" + text: "Overall Capsule On-Disk Update Reporting Variables Result: PASSED" + + - name: capsulemax_exact_char16_11_length_should_be_enforced + type: module_main_with_env + warn_only: true + scenario: + kind: capsule_vars + log_file: capsule.log + os_indications: + supported: true + attrs: 0x06 + capsule_max: + name: Capsule0001 + attrs: 0x06 + extra_utf16: EXTRA + capsule_last: + name: Capsule0001 + attrs: 0x07 + capsule_entries: + - name: Capsule0001 + attrs: 0x07 + payload_hex: "AA" + expect_exit_code: 0 + post_checks: + - type: file_contains + path: "{dir}/capsule.log" + text: "CapsuleMax - Found, Value - Capsule0001" + - type: file_contains + path: "{dir}/capsule.log" + text: "Overall Capsule On-Disk Update Reporting Variables Result: PASSED" + + - name: capsule_nnnn_payload_structure_should_not_be_ignored + type: module_main_with_env + warn_only: true + scenario: + kind: capsule_vars + log_file: capsule.log + os_indications: + supported: true + attrs: 0x06 + capsule_max: + name: Capsule0001 + attrs: 0x06 + capsule_last: + name: Capsule0001 + attrs: 0x07 + capsule_entries: + - name: Capsule0001 + hex: "07000000" + expect_exit_code: 0 + post_checks: + - type: file_contains + path: "{dir}/capsule.log" + text: "CapsuleNNNN Variable Test: Capsule0001 - Found" + - type: file_contains + path: "{dir}/capsule.log" + text: "CapsuleNNNN Variable Test: Capsule0001 Variable Test: PASSED" + - type: file_contains + path: "{dir}/capsule.log" + text: "Overall Capsule On-Disk Update Reporting Variables Result: PASSED" + + - name: missing_os_indications_is_reported_as_not_found + type: module_main_with_env + warn_only: true + scenario: + kind: capsule_vars + log_file: capsule.log + expect_exit_code: 2 + post_checks: + - type: file_contains + path: "{dir}/capsule.log" + text: "OsIndicationsSupported not found or unreadable" + + - name: logging_failure_silently_skips_log_creation + type: module_main_with_env + warn_only: true + scenario: + kind: capsule_vars + efivarfs_present: false + log_file: capsule.log + mocks: + builtins.open: + factory: mock_helpers.passthrough_router + inject_original_as: real + kwargs: + rules: + - label: log-write-failure + required: true + when: + args: + 0: + contains: "capsule.log" + 1: + contains: "a" + raise: + type: py:builtins.OSError + args: ["mocked log write failure"] + expect_exit_code: 1 + post_checks: + - type: not_exists + path: "{dir}/capsule.log" + + - name: unreadable_os_indications_is_reported_as_unreadable + type: module_main_with_env + warn_only: true + scenario: + kind: capsule_vars + log_file: capsule.log + os_indications: + supported: true + attrs: 0x07 + mocks: + builtins.open: + factory: mock_helpers.passthrough_router + inject_original_as: real + kwargs: + rules: + - label: os-indications-open-failure + required: true + when: + args: + 0: + contains: "OsIndicationsSupported-8be4df61-93ca-11d2-aa0d-00e098032b8c" + 1: + contains: "rb" + raise: + type: py:builtins.OSError + args: ["mocked read failure"] + expect_exit_code: 2 + post_checks: + - type: file_contains + path: "{dir}/capsule.log" + text: "OsIndicationsSupported not found or unreadable" + + - name: capsulemax_with_additional_attribute_bits_is_rejected + type: module_main_with_env + warn_only: true + scenario: + kind: capsule_vars + log_file: log.txt + os_indications: + supported: true + attrs: 0x07 + capsule_max: + name: Capsule0001 + attrs: 0x0E + capsule_last: + name: Capsule0001 + attrs: 0x07 + capsule_entries: + - name: Capsule0001 + attrs: 0x07 + payload_hex: "AA" + expect_exit_code: 3 + description: "Document the current behavior that requires an exact CapsuleMax attribute match and rejects additional bits." + post_checks: + - type: file_contains + path: "{dir}/log.txt" + text: "Status - WARNING" + + - name: permission_denied_during_capsule_enumeration_is_identified + type: module_main_with_env + warn_only: true + scenario: + kind: capsule_vars + log_file: capsule.log + os_indications: + supported: true + attrs: 0x07 + capsule_max: + name: Capsule0001 + attrs: 0x06 + capsule_last: + name: Capsule0001 + attrs: 0x07 + mocks: + os.listdir: + factory: mock_helpers.passthrough_router + inject_original_as: real + kwargs: + rules: + - label: permission-denied-listdir + required: true + when: + args: + 0: + contains: "efivars" + raise: + type: py:builtins.PermissionError + args: ["denied"] + expect_exit_code: 3 + post_checks: + - type: file_contains + path: "{dir}/capsule.log" + text: "efivarfs not accessible - WARNING" + + - name: malformed_capsule_entry_content_is_rejected + type: module_main_with_env + warn_only: true + scenario: + kind: capsule_vars + log_file: log.txt + os_indications: + supported: true + attrs: 0x07 + capsule_entries: + - name: Capsule0001 + hex: "FFFFFFFFFFFFFFFF" + expect_exit_code: 3 + + - name: oversized_capsulemax_value_is_currently_accepted + type: module_main_with_env + warn_only: true + patch_constants: + EFIVAR_PATH: "{dir}/efivars" + LOG_FILE: "{dir}/log.txt" + dir_structure: + - path: efivars + bin_files: + efivars/OsIndicationsSupported-8be4df61-93ca-11d2-aa0d-00e098032b8c: + hex: "070000000400000000000000" + efivars/CapsuleMax-39b68c46-f7fb-441b-b6ec-16b0f69821f3: + hex: "06000000430061007000730075006c006500300030003000310058005800" + efivars/CapsuleLast-39b68c46-f7fb-441b-b6ec-16b0f69821f3: + hex: "07000000430061007000730075006c0065003000300030003100" + efivars/Capsule0001-39b68c46-f7fb-441b-b6ec-16b0f69821f3: + hex: "07000000AA" + expect_exit_code: 0 + post_checks: + - type: file_contains + path: "{dir}/log.txt" + text: "CapsuleMax - Found, Value - Capsule0001" + - type: file_contains + path: "{dir}/log.txt" + text: "Overall Capsule On-Disk Update Reporting Variables Result: PASSED" + + - name: invalid_utf16_in_capsulelast_is_reported + type: module_main_with_env + warn_only: true + scenario: + kind: capsule_vars + log_file: log.txt + os_indications: + supported: true + attrs: 0x07 + capsule_max: + name: Capsule0001 + attrs: 0x06 + capsule_last: + hex: "07000000FFFF" + capsule_entries: + - name: Capsule0001 + attrs: 0x07 + payload_hex: "AA" + expect_exit_code: 3 + post_checks: + - type: file_contains + path: "{dir}/log.txt" + text: "CapsuleLast Variable Test: WARNING" + - type: file_contains + path: "{dir}/log.txt" + text: "Overall Capsule On-Disk Update Reporting Variables Result: WARNING" + + - name: capsulelast_is_not_cross_checked_against_discovered_entry + type: module_main_with_env + warn_only: true + patch_constants: + EFIVAR_PATH: "{dir}/efivars" + LOG_FILE: "{dir}/log.txt" + dir_structure: + - path: efivars + bin_files: + efivars/OsIndicationsSupported-8be4df61-93ca-11d2-aa0d-00e098032b8c: + hex: "070000000400000000000000" + efivars/CapsuleMax-39b68c46-f7fb-441b-b6ec-16b0f69821f3: + hex: "06000000430061007000730075006c0065003000300030003100" + efivars/CapsuleLast-39b68c46-f7fb-441b-b6ec-16b0f69821f3: + hex: "07000000430061007000730075006c0065003000300030003900" + efivars/Capsule0001-39b68c46-f7fb-441b-b6ec-16b0f69821f3: + hex: "07000000AA" + expect_exit_code: 0 + post_checks: + - type: file_contains + path: "{dir}/log.txt" + text: "CapsuleLast Variable Test: PASSED" + - type: file_contains + path: "{dir}/log.txt" + text: "Overall Capsule On-Disk Update Reporting Variables Result: PASSED" diff --git a/common/acs_test_framework_manifests/ethtool-test.yaml b/common/acs_test_framework_manifests/ethtool-test.yaml new file mode 100644 index 00000000..16958e96 --- /dev/null +++ b/common/acs_test_framework_manifests/ethtool-test.yaml @@ -0,0 +1,345 @@ +suites: + - name: ethtool_test + files: + - common/linux_scripts/ethtool-test.py + + cases: + - name: file_exists + type: file_exists + description: Verify the ethtool test script is present + + - name: python_compiles + type: py_compile + description: Verify the script has no Python syntax errors + + - name: has_main_guard + type: main_guard + description: Verify the script can run as a standalone CLI entry point + + - name: expected_test_order_keywords_exist + type: source_contains_all + description: Verify the source includes all expected Ethernet validation stages + patterns: + - "Detect interface" + - "Bring up" + - "ethtool present" + - "Self-test supported" + - "ethtool self tests" + - "Link detected" + - "IPv4 address present" + - "Gateway Address present" + - "Ping gateway (IPv4)" + - "Ping www.arm.com (IPv4)" + - "wget and curl" + - "IPv6 address present" + - "Ping ipv6.google.com (IPv6)" + + - name: cli_no_ethernet_interfaces_exits_nonzero + type: module_cli + description: Verify the script exits with failure when no usable Ethernet interface is detected + scenario: + kind: ethtool + interfaces: [] + tools: + ethtool: absent + ping: absent + wget: absent + curl: absent + dhclient: absent + udhcpc: absent + expect_exit_nonzero: true + expect_stdout_or_stderr_contains: + - "No ethernet interfaces detected via ip linux command, Exiting" + + - name: cli_virtual_interface_only + type: module_cli + description: Verify virtual interfaces alone are not treated as compliant physical NICs + scenario: + kind: ethtool + required_compliant_interfaces: 0 + interfaces: + - name: docker0 + kind: virtual + state: down + mtu: 1500 + device: /sys/devices/virtual/net/docker0 + tools: + ethtool: /usr/sbin/ethtool + ping: absent + wget: absent + curl: absent + dhclient: absent + udhcpc: absent + text_files: + system_config.txt: | + total_number_of_network_controllers: 0 + args: + - "{dir}/system_config.txt" + expect_stdout_or_stderr_contains: + - "docker0" + + - name: cli_sysfs_link_only + type: module_cli + description: Verify sysfs carrier and operstate can identify a healthy physical NIC without ethtool + scenario: + kind: ethtool + required_compliant_interfaces: 1 + interfaces: + - name: eth0 + kind: physical + state: up + carrier: 1 + operstate: up + device: /sys/devices/pci0000:00/0000:00:01.0/net/eth0 + tools: + ethtool: absent + ping: absent + wget: absent + curl: absent + dhclient: absent + udhcpc: absent + text_files: + system_config.txt: | + total_number_of_network_controllers: 1 + args: + - "{dir}/system_config.txt" + expect_stdout_or_stderr_contains: + - "eth0" + + - name: cli_full_success + type: module_cli + description: Verify the end-to-end happy path for one healthy physical interface + scenario: + kind: ethtool + required_compliant_interfaces: 1 + interfaces: + - name: eth1 + kind: physical + state: up + carrier: 1 + operstate: up + device: /sys/devices/pci0000:00/0000:00:02.0/net/eth1 + ipv4: + - address: 192.0.2.10/24 + dynamic: true + ipv6: + - address: 2001:db8::10/64 + routes: + default: default via 192.0.2.1 dev eth1 proto dhcp + route_get: 8.8.8.8 via 192.0.2.1 dev eth1 src 192.0.2.10 + ethtool: + link_detected: true + self_test_supported: true + self_test: + returncode: 1 + stdout: "self-test failed\n" + stderr: "" + connectivity: + gateway_ping: pass + arm_ping: pass + ipv6_ping: pass + curl: pass + wget: pass + tools: + ethtool: /usr/bin/ethtool + ping: /usr/bin/ping + wget: /usr/bin/wget + curl: /usr/bin/curl + dhclient: absent + udhcpc: absent + text_files: + system_config.txt: | + total_number_of_network_controllers: 1 + args: + - "{dir}/system_config.txt" + expect_stdout_or_stderr_contains: + - "eth1" + + - name: cli_required_compliant_ifaces_fail_when_not_enough_pass + type: module_cli + description: Verify compliance fails when detected interfaces exist but too few actually pass validation + scenario: + kind: ethtool + required_compliant_interfaces: 2 + interfaces: + - name: eth0 + kind: physical + state: up + carrier: 1 + operstate: up + device: /sys/devices/pci0000:00/0000:00:03.0/net/eth0 + ipv4: + - address: 192.0.2.20/24 + dynamic: true + routes: + default: default via 192.0.2.1 dev eth0 proto dhcp + route_get: 8.8.8.8 via 192.0.2.1 dev eth0 src 192.0.2.20 + ethtool: + link_detected: true + self_test_supported: false + connectivity: + gateway_ping: pass + arm_ping: pass + curl: pass + wget: pass + - name: eth1 + kind: physical + state: down + carrier: 0 + operstate: down + device: /sys/devices/pci0000:00/0000:00:04.0/net/eth1 + ethtool: + link_detected: false + self_test_supported: false + tools: + ethtool: /usr/bin/ethtool + ping: /usr/bin/ping + wget: /usr/bin/wget + curl: /usr/bin/curl + dhclient: absent + udhcpc: absent + text_files: + system_config.txt: | + total_number_of_network_controllers: 2 + args: + - "{dir}/system_config.txt" + expect_stdout_or_stderr_contains: + - "eth0" + - "eth1" + + - name: external_connectivity_failures_do_not_override_local_interface_health + type: module_cli + description: Verify external internet failures do not automatically mark a locally healthy NIC as unhealthy + warn_only: true + scenario: + kind: ethtool + required_compliant_interfaces: 1 + interfaces: + - name: eth0 + kind: physical + state: up + carrier: 1 + operstate: up + device: /sys/devices/pci0000:00/0000:00:01.0/net/eth0 + ipv4: + - address: 192.0.2.10/24 + dynamic: true + ipv6: + - address: 2001:db8::10/64 + routes: + default: default via 192.0.2.1 dev eth0 proto dhcp + route_get: 8.8.8.8 via 192.0.2.1 dev eth0 src 192.0.2.10 + ethtool: + link_detected: true + self_test_supported: false + connectivity: + gateway_ping: pass + arm_ping: fail + ipv6_ping: fail + curl: fail + wget: fail + tools: + ethtool: /usr/bin/ethtool + ping: /usr/bin/ping + wget: /usr/bin/wget + curl: /usr/bin/curl + dhclient: absent + udhcpc: absent + text_files: + system_config.txt: | + total_number_of_network_controllers: 1 + args: + - "{dir}/system_config.txt" + expect_stdout_or_stderr_contains: + - "Ethtool Compliance :" + - "PASSED" + + - name: route_validation_does_not_depend_on_public_ip_probe + type: module_cli + description: Verify default-route validation still works even when public route probing fails + warn_only: true + scenario: + kind: ethtool + required_compliant_interfaces: 1 + interfaces: + - name: eth0 + kind: physical + state: up + carrier: 1 + operstate: up + device: /sys/devices/pci0000:00/0000:00:01.0/net/eth0 + ipv4: + - address: 10.0.0.10/24 + dynamic: true + routes: + default: default via 10.0.0.1 dev eth0 proto dhcp + route_get: + returncode: 1 + stdout: "" + stderr: "network unreachable\n" + ethtool: + link_detected: true + self_test_supported: false + tools: + ethtool: /usr/bin/ethtool + ping: /usr/bin/ping + wget: absent + curl: absent + dhclient: absent + udhcpc: absent + text_files: + system_config.txt: | + total_number_of_network_controllers: 1 + args: + - "{dir}/system_config.txt" + expect_stdout_or_stderr_contains: + - "Ping gateway (IPv4)" + - "PASSED" + + - name: compliance_requires_more_than_link_when_external_checks_degrade + type: module_cli + description: Verify overall compliance can fail even when link is up if higher-level connectivity checks degrade + warn_only: true + scenario: + kind: ethtool + required_compliant_interfaces: 1 + interfaces: + - name: eth0 + kind: physical + state: up + carrier: 1 + operstate: up + device: /sys/devices/pci0000:00/0000:00:01.0/net/eth0 + ipv4: + - address: 192.0.2.10/24 + dynamic: true + ipv6: + - address: 2001:db8::10/64 + routes: + default: default via 192.0.2.1 dev eth0 proto dhcp + route_get: 8.8.8.8 via 192.0.2.1 dev eth0 src 192.0.2.10 + ethtool: + link_detected: true + self_test_supported: false + connectivity: + gateway_ping: pass + arm_ping: fail + ipv6_ping: fail + curl: fail + wget: fail + tools: + ethtool: /usr/bin/ethtool + ping: /usr/bin/ping + wget: /usr/bin/wget + curl: /usr/bin/curl + dhclient: absent + udhcpc: absent + text_files: + system_config.txt: | + total_number_of_network_controllers: 1 + args: + - "{dir}/system_config.txt" + expect_stdout_or_stderr_contains: + - "Ethtool Compliance :" + - "FAILED" + diff --git a/common/acs_test_framework_manifests/extract-capsule-fw-version.yaml b/common/acs_test_framework_manifests/extract-capsule-fw-version.yaml new file mode 100644 index 00000000..ef16827c --- /dev/null +++ b/common/acs_test_framework_manifests/extract-capsule-fw-version.yaml @@ -0,0 +1,153 @@ +suites: + - name: extract_capsule_fw_version + files: + - common/linux_scripts/extract_capsule_fw_version.py + + cases: + - name: file_exists + type: file_exists + description: "Verify that the firmware version extraction script exists at the expected repository path." + + - name: python_compiles + type: py_compile + description: "Ensure the script is syntactically valid Python before executing runtime tests." + + - name: cli_no_args_fails + type: cli + description: "Validate that the script enforces required CLI arguments and fails when none are provided." + args: [] + expect_exit_code: 1 + expect_stdout_or_stderr_contains: + - "Usage: python3 extract_capsule_fw_version.py" + + - name: cli_one_arg_fails + type: cli + description: "Ensure the script rejects execution when only the regex pattern is provided without the input file." + args: + - "Capsule=([0-9A-F]+)" + expect_exit_code: 1 + expect_stdout_or_stderr_contains: + - "Usage: python3 extract_capsule_fw_version.py" + + - name: cli_missing_file_is_nonzero + type: cli + description: "Verify that the script exits with failure when the input file does not exist." + args: + - "Capsule=([0-9A-F]{4})" + - "{dir}/missing.log" + expect_exit_nonzero: true + + - name: cli_extracts_two_matching_values_in_order + type: cli + description: "Validate that the script correctly extracts multiple matching firmware version values in the order they appear in the input file." + args: + - "Capsule=([0-9A-F]{4})" + - "{dir}/input.log" + text_files: + input.log: | + Start + Capsule=1A2B + Noise + Capsule=00FF + End + expect_exit_code: 0 + expect_stdout_or_stderr_regex: + - "(?s)1A2B.*00FF" + + - name: cli_no_matches_returns_zero_cleanly + type: cli + description: "Ensure the script handles input with no matching patterns gracefully and still exits successfully." + args: + - "Capsule=([0-9A-F]{4})" + - "{dir}/input.log" + text_files: + input.log: | + Alpha + Beta + Gamma + expect_exit_code: 0 + + - name: cli_pattern_without_group_returns_zero_cleanly + type: cli + description: "Verify behavior when regex pattern has no capture group; script should still run without failure." + args: + - "Capsule=[0-9A-F]{4}" + - "{dir}/input.log" + text_files: + input.log: | + Capsule=ABCD + Capsule=EF01 + expect_exit_code: 0 + + - name: cli_only_prints_group_1_not_full_match + type: cli + description: "Ensure only the captured group is printed and not the full regex match string." + args: + - "Capsule=([0-9A-F]{4})" + - "{dir}/input.log" + text_files: + input.log: | + build Capsule=DEAD done + expect_exit_code: 0 + expect_stdout_or_stderr_regex: + - "(?m)^DEAD$" + + - name: cli_multiple_matches_same_line_prints_first_search_per_line + type: cli + description: "Validate that only the first match per line is captured when multiple matches exist in a single line." + args: + - "Capsule=([0-9A-F]{4})" + - "{dir}/input.log" + text_files: + input.log: | + Capsule=AAAA Capsule=BBBB + expect_exit_code: 0 + expect_stdout_or_stderr_regex: + - "(?m)^AAAA$" + + - name: cli_mixed_case_hex_matches + type: cli + description: "Ensure the regex can match both uppercase and lowercase hexadecimal values." + args: + - "Capsule=([0-9A-Fa-f]{4})" + - "{dir}/input.log" + text_files: + input.log: | + Capsule=AbCd + expect_exit_code: 0 + expect_stdout_or_stderr_contains: + - "AbCd" + + - name: cli_two_capture_groups_prints_only_first_group + type: cli + description: "Ensure that when multiple capture groups are present, only the first group is printed." + args: + - "Capsule=([0-9A-F]{4}) status=(OK)" + - "{dir}/input.log" + text_files: + input.log: | + Capsule=ABCD status=OK + expect_exit_code: 0 + expect_stdout_or_stderr_regex: + - "(?m)^ABCD$" + + - name: cli_invalid_regex_fails + type: cli + description: "Validate that invalid regex patterns are detected and cause script failure." + args: + - "Capsule=([0-9A-F]{4}" + - "{dir}/input.log" + text_files: + input.log: | + Capsule=ABCD + expect_exit_nonzero: true + + - name: cli_empty_file_returns_zero + type: cli + description: "Ensure the script handles empty input files gracefully without crashing." + args: + - "Capsule=([0-9A-F]{4})" + - "{dir}/empty.log" + text_files: + empty.log: "" + expect_exit_code: 0 \ No newline at end of file diff --git a/common/acs_test_framework_manifests/generate_acs_summary.yaml b/common/acs_test_framework_manifests/generate_acs_summary.yaml new file mode 100644 index 00000000..57ec7204 --- /dev/null +++ b/common/acs_test_framework_manifests/generate_acs_summary.yaml @@ -0,0 +1,399 @@ +suites: + - name: generate_acs_summary + files: + - common/log_parser/generate_acs_summary.py + + cases: + - name: file_exists + type: file_exists + + - name: python_compiles + type: py_compile + + - name: has_main_guard + type: main_guard + + - name: cli_requires_many_inputs + type: cli + args: [] + expect_exit_nonzero: true + + - name: cli_generates_html_with_summary_sections + type: cli + text_files: + bsa.html: | + + BSA Test Summary

BSA Test Summary

Result Summary

StatusTotal
Passed1
BSA content
+ sbsa.html: | + + SBSA Test Summary

SBSA Test Summary

Result Summary

StatusTotal
Passed1
SBSA content
+ fwts.html: | + + FWTS Test Summary

FWTS Test Summary

Result Summary

StatusTotal
Passed1
FWTS content
+ sct.html: | + + SCT Test Summary

SCT Test Summary

Result Summary

StatusTotal
Passed1
SCT content
+ bbsr_fwts.html: | + + FWTS Test Summary

FWTS Test Summary

Result Summary

StatusTotal
Passed1
BBSR FWTS content
+ bbsr_sct.html: | + + SCT Test Summary

SCT Test Summary

Result Summary

StatusTotal
Passed1
BBSR SCT content
+ bbsr_tpm.html: | + + TPM Test Summary

TPM Test Summary

Result Summary

StatusTotal
Passed1
BBSR TPM content
+ pfdi.html: | + + PFDI Test Summary

PFDI Test Summary

Result Summary

StatusTotal
Passed1
PFDI content
+ post_script.html: | + + Post-Script Test Summary

Post-Script Test Summary

Result Summary

StatusTotal
Passed1
POST content
+ standalone.html: | + + Standalone Test Summary

Standalone Test Summary

Result Summary

StatusTotal
Passed1
Standalone content
+ os_tests.html: | + + OS Tests Test Summary

OS Tests Test Summary

Result Summary

StatusTotal
Passed1
OS Tests content
+ capsule.html: | + + Capsule Update Test Summary

Capsule Update Test Summary

Result Summary

StatusTotal
Passed1
Capsule content
+ sbmr_ib.html: | + + SBMR-IB Test Summary

SBMR-IB Test Summary

Result Summary

StatusTotal
Passed1
SBMR IB content
+ sbmr_oob.html: | + + SBMR-OOB Test Summary

SBMR-OOB Test Summary

Result Summary

StatusTotal
Passed1
SBMR OOB content
+ scmi.html: | + + SCMI Test Summary

SCMI Test Summary

Result Summary

StatusTotal
Passed1
SCMI content
+ acs_info.json: | + { + "System Info": { + "Vendor": "VendorZ", + "System Name": "SystemZ", + "SoC Family": "FamilyZ", + "Firmware Version": "FW-Z", + "BMC Firmware Version": "BMC-1" + } + } + merged.json: | + { + "ACS Results Summary": { + "Overall Compliance Result": "Compliant", + "BBSR compliance results": "Compliant", + "SCMI compliance results": "Compliant" + } + } + args: + - --merged_json + - "{dir}/merged.json" + - "{dir}/bsa.html" + - "{dir}/sbsa.html" + - "{dir}/fwts.html" + - "{dir}/sct.html" + - "{dir}/bbsr_fwts.html" + - "{dir}/bbsr_sct.html" + - "{dir}/bbsr_tpm.html" + - "{dir}/pfdi.html" + - "{dir}/post_script.html" + - "{dir}/standalone.html" + - "{dir}/os_tests.html" + - "{dir}/capsule.html" + - "{dir}/sbmr_ib.html" + - "{dir}/sbmr_oob.html" + - "{dir}/scmi.html" + - "{dir}/summary.html" + - --acs_info_json + - "{dir}/acs_info.json" + expect_exit_code: 0 + post_checks: + - type: exists + path: "{dir}/summary.html" + - type: file_not_empty + path: "{dir}/summary.html" + - type: file_contains + path: "{dir}/summary.html" + text: "BSA content" + - type: file_contains + path: "{dir}/summary.html" + text: "FWTS content" + - type: file_contains + path: "{dir}/summary.html" + text: "SCMI content" + - type: file_not_contains + path: "{dir}/summary.html" + text: "Result Summary" + + - name: cli_without_merged_json_still_generates_html + type: cli + text_files: + bsa.html: | + + BSA Test Summary

BSA Test Summary

Result Summary

StatusTotal
Passed1
bsa
+ sbsa.html: | + + SBSA Test Summary

SBSA Test Summary

Result Summary

StatusTotal
Passed1
sbsa
+ fwts.html: | + + FWTS Test Summary

FWTS Test Summary

Result Summary

StatusTotal
Passed1
fwts
+ sct.html: | + + SCT Test Summary

SCT Test Summary

Result Summary

StatusTotal
Passed1
sct
+ bbsr_fwts.html: | + + FWTS Test Summary

FWTS Test Summary

Result Summary

StatusTotal
Passed1
bbsr_fwts
+ bbsr_sct.html: | + + SCT Test Summary

SCT Test Summary

Result Summary

StatusTotal
Passed1
bbsr_sct
+ bbsr_tpm.html: | + + TPM Test Summary

TPM Test Summary

Result Summary

StatusTotal
Passed1
bbsr_tpm
+ pfdi.html: | + + PFDI Test Summary

PFDI Test Summary

Result Summary

StatusTotal
Passed1
pfdi
+ post_script.html: | + + Post-Script Test Summary

Post-Script Test Summary

Result Summary

StatusTotal
Passed1
post_script
+ standalone.html: | + + Standalone Test Summary

Standalone Test Summary

Result Summary

StatusTotal
Passed1
standalone
+ os_tests.html: | + + OS Tests Test Summary

OS Tests Test Summary

Result Summary

StatusTotal
Passed1
os_tests
+ capsule.html: | + + Capsule Update Test Summary

Capsule Update Test Summary

Result Summary

StatusTotal
Passed1
capsule
+ sbmr_ib.html: | + + SBMR-IB Test Summary

SBMR-IB Test Summary

Result Summary

StatusTotal
Passed1
sbmr_ib
+ sbmr_oob.html: | + + SBMR-OOB Test Summary

SBMR-OOB Test Summary

Result Summary

StatusTotal
Passed1
sbmr_oob
+ scmi.html: | + + SCMI Test Summary

SCMI Test Summary

Result Summary

StatusTotal
Passed1
scmi
+ args: + - "{dir}/bsa.html" + - "{dir}/sbsa.html" + - "{dir}/fwts.html" + - "{dir}/sct.html" + - "{dir}/bbsr_fwts.html" + - "{dir}/bbsr_sct.html" + - "{dir}/bbsr_tpm.html" + - "{dir}/pfdi.html" + - "{dir}/post_script.html" + - "{dir}/standalone.html" + - "{dir}/os_tests.html" + - "{dir}/capsule.html" + - "{dir}/sbmr_ib.html" + - "{dir}/sbmr_oob.html" + - "{dir}/scmi.html" + - "{dir}/summary.html" + expect_exit_code: 0 + post_checks: + - type: exists + path: "{dir}/summary.html" + - type: file_contains + path: "{dir}/summary.html" + text: "standalone" + + + - name: cli_removes_result_summary_headings_case_insensitively + type: cli + text_files: + bsa.html: | + + BSA Test Summary

BSA Test Summary

RESULT SUMMARY

StatusTotal
Passed1
BSA
+ sbsa.html: | + + SBSA Test Summary

SBSA Test Summary

Result Summary

StatusTotal
Passed1
sbsa
+ fwts.html: | + + FWTS Test Summary

FWTS Test Summary

Result Summary

StatusTotal
Passed1
fwts
+ sct.html: | + + SCT Test Summary

SCT Test Summary

Result Summary

StatusTotal
Passed1
sct
+ bbsr_fwts.html: | + + FWTS Test Summary

FWTS Test Summary

Result Summary

StatusTotal
Passed1
bbsr_fwts
+ bbsr_sct.html: | + + SCT Test Summary

SCT Test Summary

Result Summary

StatusTotal
Passed1
bbsr_sct
+ bbsr_tpm.html: | + + TPM Test Summary

TPM Test Summary

Result Summary

StatusTotal
Passed1
bbsr_tpm
+ pfdi.html: | + + PFDI Test Summary

PFDI Test Summary

Result Summary

StatusTotal
Passed1
pfdi
+ post_script.html: | + + Post-Script Test Summary

Post-Script Test Summary

Result Summary

StatusTotal
Passed1
post_script
+ standalone.html: | + + Standalone Test Summary

Standalone Test Summary

Result Summary

StatusTotal
Passed1
standalone
+ os_tests.html: | + + OS Tests Test Summary

OS Tests Test Summary

Result Summary

StatusTotal
Passed1
os_tests
+ capsule.html: | + + Capsule Update Test Summary

Capsule Update Test Summary

Result Summary

StatusTotal
Passed1
capsule
+ sbmr_ib.html: | + + SBMR-IB Test Summary

SBMR-IB Test Summary

Result Summary

StatusTotal
Passed1
sbmr_ib
+ sbmr_oob.html: | + + SBMR-OOB Test Summary

SBMR-OOB Test Summary

Result Summary

StatusTotal
Passed1
sbmr_oob
+ scmi.html: | + + SCMI Test Summary

SCMI Test Summary

Result Summary

StatusTotal
Passed1
scmi
+ args: + - "{dir}/bsa.html" + - "{dir}/sbsa.html" + - "{dir}/fwts.html" + - "{dir}/sct.html" + - "{dir}/bbsr_fwts.html" + - "{dir}/bbsr_sct.html" + - "{dir}/bbsr_tpm.html" + - "{dir}/pfdi.html" + - "{dir}/post_script.html" + - "{dir}/standalone.html" + - "{dir}/os_tests.html" + - "{dir}/capsule.html" + - "{dir}/sbmr_ib.html" + - "{dir}/sbmr_oob.html" + - "{dir}/scmi.html" + - "{dir}/summary.html" + expect_exit_code: 0 + post_checks: + - type: file_contains + path: "{dir}/summary.html" + text: "BSA" + - type: file_not_contains + path: "{dir}/summary.html" + text: "RESULT SUMMARY" + + - name: cli_parses_merged_json_compliance_details + type: cli + text_files: + bsa.html: | + + BSA Test Summary

BSA Test Summary

Result Summary

StatusTotal
Passed1
BSA detail parsing content
+ merged.json: | + { + "Suite_Name: acs_info": { + "ACS Results Summary": { + "Overall Compliance Result": "Not Compliant : Mandatory - (not run: BSA, FWTS; failed: SCT) : Recommended - (not run: SBSA; failed: PFDI)", + "BBSR compliance results": "Not Compliant : Mandatory - (not run: BBSR-FWTS; failed: BBSR-SCT)", + "SCMI compliance results": "Not Compliant" + } + } + } + args: + - --merged_json + - "{dir}/merged.json" + - "{dir}/bsa.html" + - "{dir}/sbsa_missing.html" + - "{dir}/fwts_missing.html" + - "{dir}/sct_missing.html" + - "{dir}/bbsr_fwts_missing.html" + - "{dir}/bbsr_sct_missing.html" + - "{dir}/bbsr_tpm_missing.html" + - "{dir}/pfdi_missing.html" + - "{dir}/post_script_missing.html" + - "{dir}/standalone_missing.html" + - "{dir}/os_tests_missing.html" + - "{dir}/capsule_missing.html" + - "{dir}/sbmr_ib_missing.html" + - "{dir}/sbmr_oob_missing.html" + - "{dir}/scmi_missing.html" + - "{dir}/summary.html" + expect_exit_code: 0 + post_checks: + - type: exists + path: "{dir}/summary.html" + - type: file_contains + path: "{dir}/summary.html" + text: "not run: BSA, FWTS" + - type: file_contains + path: "{dir}/summary.html" + text: "failed: SCT" + - type: file_contains + path: "{dir}/summary.html" + text: "not run: SBSA" + - type: file_contains + path: "{dir}/summary.html" + text: "failed: PFDI" + - type: file_contains + path: "{dir}/summary.html" + text: "not run: BBSR-FWTS" + - type: file_contains + path: "{dir}/summary.html" + text: "failed: BBSR-SCT" + - type: file_contains + path: "{dir}/summary.html" + text: "failed: SCMI" + + - name: cli_injects_test_suite_info_into_detailed_html + type: cli + text_files: + bsa.html: | + + BSA Test Summary

BSA Test Summary

Result Summary

StatusTotal
Passed1
BSA suite info content
+ bsa_detailed.html: | + +
Test Suite: PE
details body
+ merged.json: | + { + "Suite_Name: acs_info": { + "ACS Results Summary": { + "Overall Compliance Result": "Compliant" + } + }, + "Suite_Name: BSA": { + "test_results": [ + { + "Test_suite": "PE", + "Test_suite_info": [ + "PCIe root complex present", + "BAR sizing checked" + ] + } + ] + } + } + args: + - --merged_json + - "{dir}/merged.json" + - "{dir}/bsa.html" + - "{dir}/sbsa_missing.html" + - "{dir}/fwts_missing.html" + - "{dir}/sct_missing.html" + - "{dir}/bbsr_fwts_missing.html" + - "{dir}/bbsr_sct_missing.html" + - "{dir}/bbsr_tpm_missing.html" + - "{dir}/pfdi_missing.html" + - "{dir}/post_script_missing.html" + - "{dir}/standalone_missing.html" + - "{dir}/os_tests_missing.html" + - "{dir}/capsule_missing.html" + - "{dir}/sbmr_ib_missing.html" + - "{dir}/sbmr_oob_missing.html" + - "{dir}/scmi_missing.html" + - "{dir}/summary.html" + expect_exit_code: 0 + post_checks: + - type: exists + path: "{dir}/summary.html" + - type: file_contains + path: "{dir}/bsa_detailed.html" + text: "Test_suite_info:" + - type: file_contains + path: "{dir}/bsa_detailed.html" + text: "PCIe root complex present" + - type: file_contains + path: "{dir}/bsa_detailed.html" + text: "BAR sizing checked" + - type: file_contains + path: "{dir}/bsa_detailed.html" + text: "5' + - type: file_contains + path: "{dir}/summary.html" + text: '0' + + - name: os_tests_json_to_html_specific + + # OS tests specific validation. + files: + - common/log_parser/os_tests/json_to_html.py + + defaults: + type: cli + env: + MPLBACKEND: Agg + + cases: + # Malformed JSON must fail loudly. + - name: os_tests_invalid_json_input + scripts: + os_test.json: | + { invalid json + args: + - "{dir}/os_test.json" + - "{dir}/detail.html" + - "{dir}/summary.html" + expect_exit_code_in: [1, 2] + expect_stdout_or_stderr_regex: + - "(?i)(error|exception|traceback|usage|json)" + + # Verifies the handling of an unsupported sub_test_result shape. + # Outside SR-single mode, the summary should count it as failed. + - name: os_tests_unknown_subtest_is_treated_as_failed_outside_sr_single_mode + warn_only: true + scripts: + os_custom.json: | + { + "os_name": "DemoOS", + "test_results": [ + { + "Test_suite_name": "OS Tests", + "Test_suite_description": "Unknown status handling", + "Test_case": "Case 1", + "Test_case_description": "Unknown subtest result", + "subtests": [ + { + "sub_Test_Number": "1", + "sub_Test_Description": "Unknown result object", + "sub_test_result": {} + } + ] + } + ] + } + args: + - "{dir}/os_custom.json" + - "{dir}/detail.html" + - "{dir}/summary.html" + expect_exit_code: 0 + post_checks: + - type: exists + path: "{dir}/detail.html" + - type: file_contains + path: "{dir}/detail.html" + text: "Case 1" + - type: file_contains + path: "{dir}/detail.html" + text: "INFO" + - type: exists + path: "{dir}/summary.html" + - type: file_contains + path: "{dir}/summary.html" + text: "Result Summary" + - type: file_contains + path: "{dir}/summary.html" + text: '1' + + - name: pfdi_json_to_html_specific + + # PFDI-specific validation. + files: + - common/log_parser/pfdi/json_to_html.py + + defaults: + type: cli + env: + MPLBACKEND: Agg + + cases: + # Malformed JSON must fail loudly. + - name: pfdi_invalid_json_input + scripts: + pfdi.json: | + { invalid json + args: + - "{dir}/pfdi.json" + - "{dir}/detail.html" + - "{dir}/summary.html" + expect_exit_code_in: [1, 2] + expect_stdout_or_stderr_regex: + - "(?i)(error|exception|traceback|json)" + + # Nested waiver_reason should be promoted and displayed at subtest level. + - name: pfdi_nested_waiver_reason_is_promoted_to_subtest + warn_only: true + scripts: + pfdi.json: | + [ + { + "Test_suite": "PFDI", + "subtests": [ + { + "sub_Test_Number": "1", + "sub_Test_Description": "Nested waiver reason", + "sub_test_result": { + "waiver_reason": "nested waiver detail" + } + } + ], + "test_suite_summary": { + "total_passed": 1, + "total_failed": 0, + "total_failed_with_waiver": 0, + "total_aborted": 0, + "total_skipped": 0, + "total_warnings": 0 + } + }, + { + "Suite_summary": "overall" + } + ] + args: + - "{dir}/pfdi.json" + - "{dir}/detail.html" + - "{dir}/summary.html" + expect_exit_code: 0 + post_checks: + - type: exists + path: "{dir}/detail.html" + - type: file_contains + path: "{dir}/detail.html" + text: "PFDI" + - type: file_contains + path: "{dir}/detail.html" + text: "Nested waiver reason" + - type: file_contains + path: "{dir}/detail.html" + text: "nested waiver detail" + - type: exists + path: "{dir}/summary.html" + - type: file_contains + path: "{dir}/summary.html" + text: "Result Summary" + - type: file_contains + path: "{dir}/summary.html" + text: '1' + + - name: post_script_json_to_html_specific + + # Post-script specific validation. + files: + - common/log_parser/post_script/json_to_html.py + + defaults: + type: cli + env: + MPLBACKEND: Agg + + cases: + # Malformed JSON must fail loudly. + - name: post_script_invalid_json_input + scripts: + post.json: | + { invalid json + args: + - "{dir}/post.json" + - "{dir}/detail.html" + - "{dir}/summary.html" + expect_exit_code_in: [1, 2] + expect_stdout_or_stderr_regex: + - "(?i)(error|exception|traceback|json)" + + # FAILED_WITH_WAIVER should be rendered distinctly in both detail and summary output. + - name: post_script_failed_with_waiver_is_rendered_separately + warn_only: true + scripts: + post.json: | + { + "suite_summary": { + "total_passed": 0, + "total_failed": 0, + "total_failed_with_waiver": 1, + "total_aborted": 0, + "total_skipped": 0, + "total_warnings": 0 + }, + "test_results": [ + { + "Test_suite": "Post Script", + "Test_suite_description": "Waiver handling", + "subtests": [ + { + "sub_Test_Number": "1", + "sub_Test_Description": "Waived failure", + "sub_test_result": { + "FAILED_WITH_WAIVER": 1, + "waiver_reason": "approved" + } + } + ] + } + ] + } + args: + - "{dir}/post.json" + - "{dir}/detail.html" + - "{dir}/summary.html" + expect_exit_code: 0 + post_checks: + - type: exists + path: "{dir}/detail.html" + - type: file_contains + path: "{dir}/detail.html" + text: "Post Script" + - type: file_contains + path: "{dir}/detail.html" + text: "Waived failure" + - type: file_contains + path: "{dir}/detail.html" + text: "FAILED WITH WAIVER" + - type: file_contains + path: "{dir}/detail.html" + text: "approved" + - type: exists + path: "{dir}/summary.html" + - type: file_contains + path: "{dir}/summary.html" + text: "Result Summary" + - type: file_contains + path: "{dir}/summary.html" + text: '1' + + - name: sbmr_json_to_html_specific + + # SBMR-specific validation. + files: + - common/log_parser/sbmr/json_to_html.py + + defaults: + type: cli + env: + MPLBACKEND: Agg + + cases: + # Malformed JSON must fail loudly. + - name: sbmr_invalid_json_input + scripts: + sbmr.json: | + { invalid json + args: + - "{dir}/sbmr.json" + - "{dir}/detail.html" + - "{dir}/summary.html" + expect_exit_code_in: [1, 2] + expect_stdout_or_stderr_regex: + - "(?i)(error|exception|traceback|json)" + + # Same malformed-input check, but with an extra report path argument. + - name: sbmr_invalid_json_input_with_report_path + scripts: + sbmr.json: | + { invalid json + report.html: | + report + args: + - "{dir}/sbmr.json" + - "{dir}/detail.html" + - "{dir}/summary.html" + - "{dir}/report.html" + expect_exit_code_in: [1, 2] + expect_stdout_or_stderr_regex: + - "(?i)(error|exception|traceback|json)" + + # When top-level summary is missing, the implementation should recompute it from testcase data. + - name: sbmr_recomputes_summary_when_top_level_summary_missing + warn_only: true + scripts: + sbmr.json: | + { + "test_results": [ + { + "Test_suite": "SBMR IB", + "test_suite_summary": { + "total_passed": 1, + "total_failed": 0, + "total_failed_with_waiver": 0, + "total_aborted": 0, + "total_skipped": 0, + "total_warnings": 0 + }, + "subtests": [ + { + "sub_Test_Number": "1", + "sub_Test_Description": "Passed case", + "sub_test_result": "PASSED" + } + ] + } + ] + } + args: + - "{dir}/sbmr.json" + - "{dir}/detail.html" + - "{dir}/summary.html" + expect_exit_code: 0 + post_checks: + - type: exists + path: "{dir}/detail.html" + - type: file_contains + path: "{dir}/detail.html" + text: "SBMR IB" + - type: file_contains + path: "{dir}/detail.html" + text: "Passed case" + - type: file_contains + path: "{dir}/detail.html" + text: "PASSED" + - type: exists + path: "{dir}/summary.html" + - type: file_contains + path: "{dir}/summary.html" + text: "Overall Summary" + - type: file_contains + path: "{dir}/summary.html" + text: '1' + + - name: scmi_json_to_html_specific + + # SCMI-specific validation. + files: + - common/log_parser/scmi/json_to_html.py + + defaults: + type: cli + env: + MPLBACKEND: Agg + + cases: + # Malformed JSON must fail loudly. + - name: scmi_invalid_json_input + scripts: + scmi.json: | + { invalid json + args: + - "{dir}/scmi.json" + - "{dir}/detail.html" + - "{dir}/summary.html" + expect_exit_code_in: [1, 2] + expect_stdout_or_stderr_regex: + - "(?i)(error|exception|traceback|json)" + + # Summary should be tallied from testcase-level results when that is the source of truth. + - name: scmi_summary_is_tallied_from_testcase_results + warn_only: true + scripts: + scmi.json: | + { + "test_results": [ + { + "Test_suite": "SCMI", + "reason": "Suite reason", + "testcases": [ + { + "Test_case": "Case 1", + "Test_case_description": "One passed testcase", + "Test_result": "PASSED", + "reason": "ok" + } + ] + } + ] + } + args: + - "{dir}/scmi.json" + - "{dir}/detail.html" + - "{dir}/summary.html" + expect_exit_code: 0 + post_checks: + - type: exists + path: "{dir}/detail.html" + - type: file_contains + path: "{dir}/detail.html" + text: "SCMI" + - type: file_contains + path: "{dir}/detail.html" + text: "Case 1" + - type: file_contains + path: "{dir}/detail.html" + text: "PASSED" + - type: exists + path: "{dir}/summary.html" + - type: file_contains + path: "{dir}/summary.html" + text: "Result Summary" + - type: file_contains + path: "{dir}/summary.html" + text: '1' + + - name: standalone_tests_json_to_html_specific + + # Standalone-tests specific validation. + files: + - common/log_parser/standalone_tests/json_to_html.py + + defaults: + type: cli + env: + MPLBACKEND: Agg + + cases: + # Malformed JSON must fail loudly. + - name: standalone_invalid_json_input + scripts: + standalone.json: | + { invalid json + args: + - "{dir}/standalone.json" + - "{dir}/detail.html" + - "{dir}/summary.html" + expect_exit_code_in: [1, 2] + expect_stdout_or_stderr_regex: + - "(?i)(error|exception|traceback|usage|json)" + + # Unknown subtest result shape should still render in detail, + # while summary currently counts the testcase as passed. + - name: unknown_subtest_status_appears_in_detail_but_summary_counts_test_as_passed + warn_only: true + scripts: + standalone.json: | + { + "test_results": [ + { + "Test_suite": "Standalone", + "Test_suite_description": "Unknown status handling", + "Test_case": "Case 1", + "Test_case_description": "Unsupported status shape", + "subtests": [ + { + "sub_Test_Number": "1", + "sub_Test_Description": "Status is empty dict", + "sub_test_result": {} + } + ] + } + ] + } + args: + - "{dir}/standalone.json" + - "{dir}/detail.html" + - "{dir}/summary.html" + expect_exit_code: 0 + post_checks: + - type: exists + path: "{dir}/detail.html" + - type: file_contains + path: "{dir}/detail.html" + text: "Standalone" + - type: file_contains + path: "{dir}/detail.html" + text: "Case 1" + - type: file_contains + path: "{dir}/detail.html" + text: "UNKNOWN" + - type: exists + path: "{dir}/summary.html" + - type: file_contains + path: "{dir}/summary.html" + text: "Result Summary" + - type: file_contains + path: "{dir}/summary.html" + text: '1' + + # WARNING should take priority over FAILED_WITH_WAIVER in summary classification. + - name: warning_status_takes_priority_over_failed_with_waiver + warn_only: true + scripts: + standalone.json: | + { + "test_results": [ + { + "Test_suite": "Standalone", + "Test_suite_description": "Priority handling", + "Test_case": "Case 1", + "Test_case_description": "Warnings and waiver together", + "subtests": [ + { + "sub_Test_Number": "1", + "sub_Test_Description": "Warning subtest", + "sub_test_result": { + "WARNINGS": 1, + "warning_reasons": ["warning path"] + } + }, + { + "sub_Test_Number": "2", + "sub_Test_Description": "Waiver subtest", + "sub_test_result": { + "FAILED_WITH_WAIVER": 1, + "waiver_reason": "accepted waiver" + } + } + ] + } + ] + } + args: + - "{dir}/standalone.json" + - "{dir}/detail.html" + - "{dir}/summary.html" + expect_exit_code: 0 + post_checks: + - type: exists + path: "{dir}/detail.html" + - type: file_contains + path: "{dir}/detail.html" + text: "Warning subtest" + - type: file_contains + path: "{dir}/detail.html" + text: "Waiver subtest" + - type: file_contains + path: "{dir}/detail.html" + text: "FAILED WITH WAIVER" + - type: file_contains + path: "{dir}/detail.html" + text: "accepted waiver" + - type: exists + path: "{dir}/summary.html" + - type: file_contains + path: "{dir}/summary.html" + text: "Result Summary" + - type: file_contains + path: "{dir}/summary.html" + text: '1' + - type: file_contains + path: "{dir}/summary.html" + text: '0' + + # Nested reason lists should be flattened into a single rendered reason column. + - name: nested_reason_lists_are_flattened_into_single_reason_column + warn_only: true + scripts: + standalone.json: | + { + "test_results": [ + { + "Test_suite": "Standalone", + "Test_suite_description": "Reason flattening", + "Test_case": "Case 1", + "Test_case_description": "Nested reasons", + "subtests": [ + { + "sub_Test_Number": "1", + "sub_Test_Description": "Nested fail reasons", + "sub_test_result": { + "FAILED": 1, + "fail_reasons": [["reason A", "reason B"], ["reason C"]] + } + } + ] + } + ] + } + args: + - "{dir}/standalone.json" + - "{dir}/detail.html" + - "{dir}/summary.html" + expect_exit_code: 0 + post_checks: + - type: exists + path: "{dir}/detail.html" + - type: file_contains + path: "{dir}/detail.html" + text: "Nested fail reasons" + - type: file_contains + path: "{dir}/detail.html" + text: "FAILED" + - type: file_contains + path: "{dir}/detail.html" + text: "reason A" + - type: file_contains + path: "{dir}/detail.html" + text: "reason B" + - type: file_contains + path: "{dir}/detail.html" + text: "reason C" + - type: exists + path: "{dir}/summary.html" + - type: file_contains + path: "{dir}/summary.html" + text: "Result Summary" + - type: file_contains + path: "{dir}/summary.html" + text: '1' \ No newline at end of file diff --git a/common/acs_test_framework_manifests/logs-to-json.yaml b/common/acs_test_framework_manifests/logs-to-json.yaml new file mode 100644 index 00000000..c31a7c37 --- /dev/null +++ b/common/acs_test_framework_manifests/logs-to-json.yaml @@ -0,0 +1,284 @@ +suites: + + - name: logs_to_json_common + files: + - common/log_parser/bbr/fwts/logs_to_json.py + - common/log_parser/bbr/sct/logs_to_json.py + - common/log_parser/bbr/tpm/logs_to_json.py + - common/log_parser/bsa/logs_to_json.py + - common/log_parser/os_tests/logs_to_json.py + - common/log_parser/pfdi/logs_to_json.py + - common/log_parser/post_script/logs_to_json.py + - common/log_parser/sbmr/logs_to_json.py + - common/log_parser/scmi/logs_to_json.py + - common/log_parser/standalone_tests/logs_to_json.py + - common/log_parser/os_tests/sr_logs_to_json.py + - common/log_parser/bbr/sct/logs_to_json_edk2.py + + cases: + - name: file_exists + type: file_exists + + - name: python_compiles + type: py_compile + + - name: contains_json_usage + type: source_contains + pattern: "json" + + - name: contains_parse_logic + type: source_contains_any + patterns: + - "parse" + - "parser" + - "parse_" + + - name: contains_result_collection_logic + type: source_contains_any + patterns: + - "subtest" + - "test_results" + - "results" + + # 🔧 FIXED: works for edk2 also + - name: contains_summary_logic + type: source_contains_any + patterns: + - "summary" + - "suite_summary" + - "total_" + - "\"result\"" + - "\"reason\"" + + - name: contains_output_logic + type: source_contains_any + patterns: + - "json.dump" + - "write" + + - name: has_main_guard + type: main_guard + + +# ========================= +# SR LOGS TO JSON +# ========================= + + - name: sr_logs_to_json_specific + files: + - common/log_parser/os_tests/sr_logs_to_json.py + + cases: + - name: has_sr_keyword + type: source_contains_all + patterns: + - "OS_RELEASE_FILE_NAME" + - "cat-etc-os-release.txt" + - "\"os_test\"" + + - name: has_sr_parse_logic + type: source_contains_all + patterns: + - "def build_results(" + - "def parse_os_release(" + - "def parse_post_script_errors(" + + - name: has_sr_status_handling + type: source_contains_all + patterns: + - "\"PASSED\"" + - "\"FAILED\"" + - "\"WARNINGS\"" + + - name: cli_requires_three_args + type: cli + args: [] + expect_exit_nonzero: true + expect_stdout_or_stderr_contains: + - "Usage:" + + - name: cli_builds_json_for_rhel_and_sle + type: cli + command: "./run_case.sh" + timeout_sec: 5 + scripts: + run_case.sh: | + #!/bin/sh + set -eu + + mkdir -p oslogs/rhel oslogs/sle + + cat > oslogs/rhel/cat-etc-os-release.txt < oslogs/sle/cat-etc-os-release.txt < post-script.log < oslogs/rhel/cat-etc-os-release.txt < oslogs/rhel/cat-etc-os-release.txt < oslogs/sle/cat-etc-os-release.txt < oslogs/ubuntu/cat-etc-os-release.txt < post-script.log < edk2-test-parser.log < + BSA Test Summary

BSA Test Summary

Result Summary

StatusTotal
Passed1
BSA merge marker
+ fwts_summary.html: | + + FWTS Test Summary

FWTS Test Summary

Result Summary

StatusTotal
Failed1
FWTS merge marker
+ args: + - "{dir}/bsa_summary.html" + - "{dir}/fwts_summary.html" + - "{dir}/acs_summary.html" + post_checks: + - type: file_not_empty + path: "{dir}/acs_summary.html" + - type: ordered_contains + path: "{dir}/acs_summary.html" + texts: + - "BSA Test Summary" + - "BSA merge marker" + - "FWTS Test Summary" + - "FWTS merge marker" + + - name: cli_preserves_blank_lines_during_merge + description: "Verify that blank lines present in repo-like log content are preserved in the merged output, ensuring formatting-sensitive diagnostic text is not unintentionally normalized or collapsed." + text_files: + # Simulate a plain-text ACS info fragment where an intentional blank line + # separates sections and must survive the merge unchanged. + acs_results.log: | + Suite_Name: acs_info + + Overall Compliance Result: Compliant + extensions.log: | + SCMI compliance results: Compliant + args: + - "{dir}/acs_results.log" + - "{dir}/extensions.log" + - "{dir}/acs_summary.html" + post_checks: + - type: regex + path: "{dir}/acs_summary.html" + pattern: "Suite_Name: acs_info\\n\\nOverall Compliance Result: Compliant" + + - name: cli_output_file_is_created_when_inputs_exist + description: "Verify that the script creates the requested output file when valid repo-like summary inputs are supplied, even in a minimal merge scenario." + text_files: + # Two minimal summary pages are enough here because the test only cares + # that a merged artifact is written, not about exact merged ordering. + sbsa_summary.html: | + + SBSA Test Summary

SBSA Test Summary

Result Summary

StatusTotal
Passed1
SBSA marker
+ scmi_summary.html: | + + SCMI Test Summary

SCMI Test Summary

Result Summary

StatusTotal
Passed1
SCMI marker
+ args: + - "{dir}/sbsa_summary.html" + - "{dir}/scmi_summary.html" + - "{dir}/acs_summary.html" + post_checks: + - type: exists + path: "{dir}/acs_summary.html" + + - name: cli_handles_empty_first_input + description: "Verify that the script handles an empty first summary input file gracefully and still merges content from the remaining non-empty summary into the output." + text_files: + # Empty string means "existing but empty file", which is different from a + # missing path and exercises a different branch in the merge flow. + empty_bsa_summary.html: "" + os_tests_summary.html: | + + OS Tests Test Summary

OS Tests Test Summary

Result Summary

StatusTotal
Passed1
OS tests payload marker
+ args: + - "{dir}/empty_bsa_summary.html" + - "{dir}/os_tests_summary.html" + - "{dir}/acs_summary.html" + post_checks: + - type: file_contains + path: "{dir}/acs_summary.html" + text: "OS tests payload marker" + + - name: cli_handles_both_inputs_empty + description: "Verify that the script handles the case where both summary input files are empty, completing cleanly and still producing an output artifact." + text_files: + empty_bsa_summary.html: "" + empty_fwts_summary.html: "" + args: + - "{dir}/empty_bsa_summary.html" + - "{dir}/empty_fwts_summary.html" + - "{dir}/acs_summary.html" + post_checks: + - type: exists + path: "{dir}/acs_summary.html" + + - name: cli_missing_second_input_fails_even_if_first_exists + description: "Verify that the script fails when the second summary input file is missing even if the first summary input file exists, ensuring partial input availability is not treated as sufficient." + text_files: + # Keep the first file valid so the failure is isolated to the missing + # second input rather than a generic all-inputs-missing condition. + bsa_summary.html: | + + BSA Test Summary

BSA Test Summary

Result Summary

StatusTotal
Passed1
BSA missing-second marker
+ args: + - "{dir}/bsa_summary.html" + - "{dir}/fwts_summary_missing.html" + - "{dir}/acs_summary.html" + expect_exit_nonzero: true + + - name: cli_overwrites_existing_output_file + description: "Verify that an already-existing ACS summary output file is overwritten with fresh merged summary content rather than appended to or left unchanged." + text_files: + # Seed the destination with stale content to prove merge_summary.py + # overwrites the old artifact instead of appending to it. + bsa_summary.html: | + + BSA Test Summary

BSA Test Summary

Result Summary

StatusTotal
Passed1
BSA fresh marker
+ fwts_summary.html: | + + FWTS Test Summary

FWTS Test Summary

Result Summary

StatusTotal
Failed1
FWTS fresh marker
+ acs_summary.html: | + + Old ACS Summarystale ACS summary content + args: + - "{dir}/bsa_summary.html" + - "{dir}/fwts_summary.html" + - "{dir}/acs_summary.html" + post_checks: + - type: file_contains + path: "{dir}/acs_summary.html" + text: "BSA fresh marker" + - type: file_contains + path: "{dir}/acs_summary.html" + text: "FWTS fresh marker" + - type: file_not_contains + path: "{dir}/acs_summary.html" + text: "stale ACS summary content" + + - name: cli_binary_or_non_utf8_input_fails_cleanly + description: "Verify that the script exits non-zero when a summary input file contains bytes that are invalid under the active text decoding mode." + env: + # Force deterministic UTF-8 decoding so 0xff is always undecodable, + # regardless of the host environment's default text codec. + PYTHONUTF8: "1" + bin_files: + corrupt_summary.html: + hex: "ff" + args: + - "{dir}/corrupt_summary.html" + - "{dir}/acs_summary.html" + expect_exit_nonzero: true + + - name: cli_output_parent_directory_missing_fails + description: "Verify that the script fails when the requested ACS summary output path points into a parent directory that does not exist." + text_files: + # The input summary is valid here; only the destination directory is + # deliberately invalid so the failure reason stays narrow and obvious. + bsa_summary.html: | + + BSA Test Summary

BSA Test Summary

Result Summary

StatusTotal
Passed1
BSA output-path marker
+ args: + - "{dir}/bsa_summary.html" + - "{dir}/missing_dir/acs_summary.html" + expect_exit_nonzero: true diff --git a/common/acs_test_framework_manifests/parser_common.yaml b/common/acs_test_framework_manifests/parser_common.yaml new file mode 100644 index 00000000..a3e71b6f --- /dev/null +++ b/common/acs_test_framework_manifests/parser_common.yaml @@ -0,0 +1,343 @@ +suites: + - name: parser_common + files: + - common/parser/Parser.py + + cases: + - name: file_exists + type: file_exists + + - name: python_compiles + type: py_compile + + - name: has_main_guard + type: main_guard + + - name: cli_help_flag_shows_usage_and_flags + type: cli + args: + - --help + expect_exit_code: 0 + expect_output: + - "usage:" + - "-bsa" + - "-sbsa" + - "-fwts" + - "-automation" + - "--config" + + - name: cli_no_args_prints_invalid_command_message + type: cli + args: [] + scripts: + config.ini: | + [AUTOMATION] + config_enabled_for_automation_run = true + expect_exit_code: 0 + expect_output: + - "Please specify a valid command or use --help for more information." + + - name: cli_bsa_disabled_or_missing_section_returns_message + type: cli + args: + - -bsa + - --config + - "{dir}/config.ini" + scripts: + config.ini: | + [AUTOMATION] + config_enabled_for_automation_run = true + expect_exit_code: 0 + expect_output: + - "BSA section is disabled or missing." + + - name: cli_bsa_enabled_builds_full_command + type: cli + args: + - -bsa + - --config + - "{dir}/config.ini" + scripts: + config.ini: | + [BSA] + automation_bsa_run = true + bsa_level = 3 + bsa_skip_rules = rule1,rule2 + bsa_verbose = 1 + expect_exit_code: 0 + expect_output: + - "/bin/bsa" + - "-l 3" + - "--skip rule1,rule2" + - "-v 1" + + - name: cli_bsa_enabled_with_only_required_flag_prints_base_command + type: cli + args: + - -bsa + - --config + - "{dir}/config.ini" + scripts: + config.ini: | + [BSA] + automation_bsa_run = true + expect_exit_code: 0 + expect_output: + - "/bin/bsa" + + - name: cli_sbsa_disabled_or_missing_section_returns_message + type: cli + args: + - -sbsa + - --config + - "{dir}/config.ini" + scripts: + config.ini: | + [AUTOMATION] + config_enabled_for_automation_run = false + expect_exit_code: 0 + expect_output: + - "SBSA section is disabled or missing." + + - name: cli_sbsa_enabled_builds_full_command + type: cli + args: + - -sbsa + - --config + - "{dir}/config.ini" + scripts: + config.ini: | + [SBSA] + automation_sbsa_run = true + sbsa_level = 4 + sbsa_skip_rules = skip_a,skip_b + sbsa_verbose = 2 + expect_exit_code: 0 + expect_output: + - "/bin/sbsa" + - "-l 4" + - "--skip skip_a,skip_b" + - "-v 2" + + - name: cli_sbsa_enabled_with_only_required_flag_prints_base_command + type: cli + args: + - -sbsa + - --config + - "{dir}/config.ini" + scripts: + config.ini: | + [SBSA] + automation_sbsa_run = true + expect_exit_code: 0 + expect_output: + - "/bin/sbsa" + + - name: cli_fwts_disabled_or_missing_section_returns_message + type: cli + args: + - -fwts + - --config + - "{dir}/config.ini" + scripts: + config.ini: | + [AUTOMATION] + config_enabled_for_automation_run = true + expect_exit_code: 0 + expect_output: + - "FWTS section is disabled or missing." + + - name: cli_fwts_enabled_builds_module_command + type: cli + args: + - -fwts + - --config + - "{dir}/config.ini" + scripts: + config.ini: | + [FWTS] + automation_fwts_run = true + fwts_modules = acpi + expect_exit_code: 0 + expect_output: + - "fwts acpi" + + - name: cli_fwts_enabled_without_module_prints_base_command + type: cli + args: + - -fwts + - --config + - "{dir}/config.ini" + scripts: + config.ini: | + [FWTS] + automation_fwts_run = true + expect_exit_code: 0 + expect_output: + - "fwts" + + - name: cli_automation_true_prints_true + type: cli + args: + - -automation + - --config + - "{dir}/config.ini" + scripts: + config.ini: | + [AUTOMATION] + config_enabled_for_automation_run = true + expect_exit_code: 0 + expect_output: + - "True" + + - name: cli_automation_false_prints_false + type: cli + args: + - -automation + - --config + - "{dir}/config.ini" + scripts: + config.ini: | + [AUTOMATION] + config_enabled_for_automation_run = false + expect_exit_code: 0 + expect_output: + - "False" + + - name: cli_automation_missing_section_reports_missing_and_false + type: cli + args: + - -automation + - --config + - "{dir}/config.ini" + scripts: + config.ini: | + [BSA] + automation_bsa_run = true + expect_exit_code: 0 + expect_output: + - "Section AUTOMATION is missing in the configuration file." + - "False" + + - name: cli_automation_bsa_run_true_prints_true + type: cli + args: + - -automation_bsa_run + - --config + - "{dir}/config.ini" + scripts: + config.ini: | + [BSA] + automation_bsa_run = true + expect_exit_code: 0 + expect_output: + - "True" + + - name: cli_automation_bsa_run_missing_section_reports_missing_and_false + type: cli + args: + - -automation_bsa_run + - --config + - "{dir}/config.ini" + scripts: + config.ini: | + [AUTOMATION] + config_enabled_for_automation_run = true + expect_exit_code: 0 + expect_output: + - "Section BSA is missing in the configuration file." + - "False" + + - name: cli_automation_sbsa_run_false_prints_false + type: cli + args: + - -automation_sbsa_run + - --config + - "{dir}/config.ini" + scripts: + config.ini: | + [SBSA] + automation_sbsa_run = false + expect_exit_code: 0 + expect_output: + - "False" + + - name: cli_automation_fwts_run_true_prints_true + type: cli + args: + - -automation_fwts_run + - --config + - "{dir}/config.ini" + scripts: + config.ini: | + [FWTS] + automation_fwts_run = true + expect_exit_code: 0 + expect_output: + - "True" + + - name: cli_automation_bbsr_fwts_run_true_prints_true + type: cli + args: + - -automation_bbsr_fwts_run + - --config + - "{dir}/config.ini" + scripts: + config.ini: | + [BBSR_FWTS] + automation_bbsr_fwts_run = true + expect_exit_code: 0 + expect_output: + - "True" + + - name: cli_automation_bbsr_tpm_run_false_prints_false + type: cli + args: + - -automation_bbsr_tpm_run + - --config + - "{dir}/config.ini" + scripts: + config.ini: | + [BBSR_TPM] + automation_bbsr_tpm_run = false + expect_exit_code: 0 + expect_output: + - "False" + + - name: cli_automation_sbmr_in_band_run_true_prints_true + type: cli + args: + - -automation_sbmr_in_band_run + - --config + - "{dir}/config.ini" + scripts: + config.ini: | + [SBMR] + automation_sbmr_in_band_run = true + expect_exit_code: 0 + expect_output: + - "True" + + - name: cli_invalid_boolean_in_bsa_section_fails + type: cli + args: + - -bsa + - --config + - "{dir}/config.ini" + scripts: + config.ini: | + [BSA] + automation_bsa_run = maybe + expect_exit_nonzero: true + + - name: cli_invalid_boolean_in_automation_check_fails + type: cli + args: + - -automation + - --config + - "{dir}/config.ini" + scripts: + config.ini: | + [AUTOMATION] + config_enabled_for_automation_run = maybe + expect_exit_nonzero: true diff --git a/common/acs_test_framework_manifests/read-write-check-blk-devices.yaml b/common/acs_test_framework_manifests/read-write-check-blk-devices.yaml new file mode 100644 index 00000000..9e973b3d --- /dev/null +++ b/common/acs_test_framework_manifests/read-write-check-blk-devices.yaml @@ -0,0 +1,430 @@ +x_module_case: &module_case + type: module_main_with_env + expect_exit_code: 0 + +suites: + - name: read_write_check_blk_devices + files: + - common/linux_scripts/read_write_check_blk_devices.py + + cases: + - name: file_exists + type: file_exists + description: "Verify that read_write_check_blk_devices.py exists at the expected repository path before any deeper structural or runtime checks are executed." + + - name: python_compiles + type: py_compile + description: "Ensure that the script compiles successfully as valid Python so syntax-level regressions are caught before runtime behavior is exercised." + + - name: cli_no_disks_runs_cleanly + <<: *module_case + description: "Verify that when no block devices are detected, the script still runs cleanly and only performs the top-level discovery command." + scenario: + kind: blk_devices + prompt: no + disks: [] + expect_stdout_or_stderr_contains: + - "INFO: Detected following block devices with lsblk command :" + post_checks: + - type: file_not_contains + path: "{dir}/blk_commands.log" + text: "gdisk -l /dev/" + - type: file_not_contains + path: "{dir}/blk_commands.log" + text: "dd if=/dev/" + + - name: cli_skips_mtd_and_ram_disks + <<: *module_case + description: "Verify that MTD block devices and RAM disks are surfaced by discovery but no later disk-processing commands are run for them." + scenario: + kind: blk_devices + prompt: no + disks: + - name: mtdblock0 + kind: mtdblock + - name: ram0 + kind: ram + expect_stdout_or_stderr_contains: + - "INFO: Skipping MTD block device /dev/mtdblock0" + - "INFO: Skipping RAM disk /dev/ram0" + post_checks: + - type: file_not_contains + path: "{dir}/blk_commands.log" + text: "timeout 10 gdisk -l /dev/mtdblock0" + - type: file_not_contains + path: "{dir}/blk_commands.log" + text: "timeout 10 gdisk -l /dev/ram0" + + - name: cli_raw_disk_block_read_success_and_no_write_prompt + <<: *module_case + description: "Verify that a disk without a valid partition table is treated as a raw device, receives only the initial read and mount check, and does not enter free-space or write-side operations when the user declines." + scenario: + kind: blk_devices + prompt: no + disks: + - name: sda + table: raw + block_read: pass + write_check: + mounted: false + expect_stdout_or_stderr_contains: + - "INFO: No valid partition table found for sda, treating as raw device." + - "INFO: No partitions detected for sda, treating as raw device." + - "INFO: Performing block read on /dev/sda" + - "INFO: Block read on /dev/sda successful" + post_checks: + - type: ordered_contains + path: "{dir}/blk_commands.log" + texts: + - "timeout 10 gdisk -l /dev/sda" + - "dd if=/dev/sda of=/dev/null bs=1M count=1" + - "findmnt -n /dev/sda" + - type: file_not_contains + path: "{dir}/blk_commands.log" + text: "df -B 512 /dev/sda --output=used,avail" + - type: file_not_contains + path: "{dir}/blk_commands.log" + text: "dd if=/dev/sda of=sda_backup.bin" + + - name: cli_mbr_precious_partition_is_skipped + <<: *module_case + description: "Verify that an MBR EFI System partition is classified as precious metadata and skipped for block read/write activity." + scenario: + kind: blk_devices + prompt: no + disks: + - name: sda + table: mbr + partitions: + - name: sda1 + boot: true + start: 2048 + sectors: 204800 + size: 100M + id: ef + type_name: EFI System + used_blocks: 10 + available_blocks: 20 + expect_stdout_or_stderr_contains: + - "INFO: Partition table type : MBR" + - "INFO: sda1 partition is PRECIOUS" + - "Skipping block read/write" + post_checks: + - type: ordered_contains + path: "{dir}/blk_commands.log" + texts: + - "fdisk -l /dev/sda" + - "df -B 512 /dev/sda1 --output=used,avail" + - type: file_not_contains + path: "{dir}/blk_commands.log" + text: "dd if=/dev/sda1 of=/dev/null bs=1M count=1" + + - name: write_check_restores_original_block_after_successful_verification + type: py_function + warn_only: true + function: perform_write_check + description: "Verify that a successful write verification restores the original block and removes the temporary artifacts." + scenario: + kind: blk_write_check + device: sda1 + partition_id: "0x83" + prompt: yes + used_blocks: 10 + available_blocks: 20 + write_flow: + readback: match + expect_stdout_or_stderr_contains: + - "INFO: Creating backup of the current block before write check..." + - "INFO: Writing test data to the device for write check..." + - "INFO: Reading back the test data for verification..." + - "INFO: write check passed on /dev/sda1." + - "INFO: Backup restored for /dev/sda1." + post_checks: + - type: file_contains + path: "{dir}/blk_commands.log" + text: "dd if=sda1_backup.bin of=/dev/sda1 bs=512 count=1 seek=10" + - type: not_exists + path: "{dir}/hello.txt" + - type: not_exists + path: "{dir}/read_hello.txt" + - type: not_exists + path: "{dir}/sda1_backup.bin" + + - name: cleanup_runs_even_when_restore_is_not_reached + type: py_function + warn_only: true + function: perform_write_check + description: "Verify that verification failure still restores the backup and removes the temporary artifacts." + scenario: + kind: blk_write_check + device: sda1 + partition_id: "0x83" + prompt: yes + used_blocks: 10 + available_blocks: 20 + write_flow: + readback: mismatch + restore: skip + expect_stdout_or_stderr_contains: + - "INFO: Creating backup of the current block before write check..." + - "INFO: Writing test data to the device for write check..." + - "INFO: Reading back the test data for verification..." + - "WARNING: Data integrity check failed for /dev/sda1. Possible data corruption." + post_checks: + - type: file_contains + path: "{dir}/blk_commands.log" + text: "dd if=sda1_backup.bin of=/dev/sda1 bs=512 count=1 seek=10" + - type: not_exists + path: "{dir}/hello.txt" + - type: not_exists + path: "{dir}/read_hello.txt" + - type: not_exists + path: "{dir}/sda1_backup.bin" + + - name: mbr_parser_limitation_missing_required_columns_is_warned_and_block_io_is_skipped + type: module_main_with_env + warn_only: true + expect_exit_code: 0 + description: "Document the current MBR parser limitation when required fdisk columns are missing and verify that block I/O is skipped safely." + scenario: + kind: blk_devices + prompt: no + disks: + - name: sda + table: mbr + partitions: + - name: sda1 + fdisk_output: | + Disk /dev/sda: 16 GiB + Device Start End Sectors Size Type + /dev/sda1 2048 206847 204800 100M EFI System + expect_stdout_or_stderr_contains: + - "INFO: Partition table type : MBR" + - "WARNING: Could not parse enough MBR partition IDs." + post_checks: + - type: file_contains + path: "{dir}/blk_commands.log" + text: "fdisk -l /dev/sda" + - type: file_not_contains + path: "{dir}/blk_commands.log" + text: "dd if=/dev/sda1 of=/dev/null bs=1M count=1" + + - name: gpt_partition_metadata_parsing_skips_partition_when_sgdisk_output_is_incomplete + type: module_main_with_env + warn_only: true + expect_exit_code: 0 + description: "Document the current GPT metadata parsing limitation when sgdisk output is incomplete and verify that block I/O is skipped safely." + scenario: + kind: blk_devices + prompt: yes + disks: + - name: sda + table: gpt + partitions: + - name: sda1 + sgdisk_output: | + Partition GUID code: C12A7328-F81F-11D2-BA4B-00A0C93EC93B (EFI System partition) + First sector: 2048 + Last sector: 206847 + Partition size: 204800 sectors + expect_stdout_or_stderr_contains: + - "INFO: Partition table type : GPT" + - "INFO: Unable to parse sgdisk info for sda1. Skipping." + post_checks: + - type: file_contains + path: "{dir}/blk_commands.log" + text: "sgdisk -i=1 /dev/sda" + - type: file_not_contains + path: "{dir}/blk_commands.log" + text: "dd if=/dev/sda1 of=/dev/null bs=1M count=1" + + - name: mounted_partition_skips_write_check_and_never_queries_space + <<: *module_case + warn_only: true + description: "Verify that mounted partitions are skipped before free-space checks and any write-side dd operations." + scenario: + kind: blk_devices + prompt: yes + disks: + - name: sda + table: mbr + partitions: + - name: sda1 + start: 2048 + sectors: 204800 + size: 100M + id: 83 + type_name: Linux + mounted: true + expect_stdout_or_stderr_contains: + - "INFO: Performing block read on /dev/sda1 mbr_part_id = 0x83" + - "INFO: Block read on /dev/sda1 mbr_part_id = 0x83 successful" + - "INFO: /dev/sda1 is mounted, skipping write test." + post_checks: + - type: ordered_contains + path: "{dir}/blk_commands.log" + texts: + - "dd if=/dev/sda1 of=/dev/null bs=1M count=1" + - "findmnt -n /dev/sda1" + - type: file_not_contains + path: "{dir}/blk_commands.log" + text: "df -B 512 /dev/sda1 --output=used,avail" + - type: file_not_contains + path: "{dir}/blk_commands.log" + text: "dd if=/dev/sda1 of=sda1_backup.bin" + + - name: raw_disk_path_never_reaches_write_side_dd_when_user_declines + <<: *module_case + warn_only: true + description: "Verify that the raw-disk path stops before any write-side operations when the user declines the write prompt." + scenario: + kind: blk_devices + prompt: no + disks: + - name: sda + table: raw + block_read: pass + write_check: + mounted: false + expect_stdout_or_stderr_contains: + - "INFO: No valid partition table found for sda, treating as raw device." + - "INFO: Performing block read on /dev/sda" + - "INFO: Block read on /dev/sda successful" + post_checks: + - type: ordered_contains + path: "{dir}/blk_commands.log" + texts: + - "dd if=/dev/sda of=/dev/null bs=1M count=1" + - "findmnt -n /dev/sda" + - type: file_not_contains + path: "{dir}/blk_commands.log" + text: "df -B 512 /dev/sda --output=used,avail" + - type: file_not_contains + path: "{dir}/blk_commands.log" + text: "dd if=/dev/sda of=sda_backup.bin" + + - name: readback_exception_leaves_backup_and_temp_files_behind + type: py_function + warn_only: true + function: perform_write_check + description: "Document the current behavior where a readback exception aborts cleanup and leaves the helper artifacts behind." + scenario: + kind: blk_write_check + device: sda1 + partition_id: "0x83" + prompt: yes + used_blocks: 10 + available_blocks: 20 + write_flow: + readback: error + readback_error: mocked readback failure + restore: skip + expect_exception: CalledProcessError + expect_stdout_or_stderr_contains: + - "INFO: Creating backup of the current block before write check..." + - "INFO: Writing test data to the device for write check..." + - "INFO: Reading back the test data for verification..." + post_checks: + - type: exists + path: "{dir}/hello.txt" + - type: not_exists + path: "{dir}/read_hello.txt" + - type: exists + path: "{dir}/sda1_backup.bin" + - type: file_not_contains + path: "{dir}/blk_commands.log" + text: "dd if=sda1_backup.bin of=/dev/sda1 bs=512 count=1 seek=10" + + - name: write_prompt_timeout_defaults_to_safe_decline + type: py_function + warn_only: true + function: perform_write_check + description: "Verify that a safe-decline response from the write prompt prevents any write-side operations from starting." + scenario: + kind: blk_write_check + device: sda1 + partition_id: "0x83" + prompt: timeout + post_checks: + - type: file_not_contains + path: "{dir}/blk_commands.log" + text: "dd if=/dev/sda1 of=sda1_backup.bin" + + - name: raw_disk_write_prompt_is_allowed_without_partition_type_guard + type: module_main_with_env + warn_only: true + expect_exit_code: 0 + description: "Document the current behavior that allows a raw disk to enter the write path once the user explicitly agrees." + scenario: + kind: blk_devices + prompt: yes + disks: + - name: sda + table: raw + block_read: pass + write_check: + mounted: false + used_blocks: 10 + available_blocks: 20 + write_flow: + readback: match + expect_stdout_or_stderr_contains: + - "INFO: No valid partition table found for sda, treating as raw device." + - "INFO: Performing block read on /dev/sda" + - "INFO: Block read on /dev/sda successful" + - "INFO: Creating backup of the current block before write check..." + - "INFO: write check passed on /dev/sda." + - "INFO: Backup restored for /dev/sda." + post_checks: + - type: ordered_contains + path: "{dir}/blk_commands.log" + texts: + - "findmnt -n /dev/sda" + - "df -B 512 /dev/sda --output=used,avail" + - "dd if=/dev/sda of=sda_backup.bin bs=512 count=1 skip=10" + - "dd if=hello.txt of=/dev/sda bs=512 count=1 seek=10" + - "dd if=/dev/sda of=read_hello.txt bs=512 count=1 skip=10" + - "dd if=sda_backup.bin of=/dev/sda bs=512 count=1 seek=10" + - type: not_exists + path: "{dir}/hello.txt" + - type: not_exists + path: "{dir}/read_hello.txt" + - type: not_exists + path: "{dir}/sda_backup.bin" + + - name: mounted_guard_applies_only_to_write_phase_not_initial_block_read + <<: *module_case + warn_only: true + description: "Verify that mounted partitions still undergo the initial safe block read while the write phase is skipped." + scenario: + kind: blk_devices + prompt: yes + disks: + - name: sda + table: mbr + partitions: + - name: sda1 + boot: true + start: 2048 + sectors: 204800 + size: 100M + id: 83 + type_name: Linux + mounted: true + expect_stdout_or_stderr_contains: + - "INFO: Performing block read on /dev/sda1 mbr_part_id = 0x83" + - "INFO: Block read on /dev/sda1 mbr_part_id = 0x83 successful" + - "INFO: /dev/sda1 is mounted, skipping write test." + post_checks: + - type: ordered_contains + path: "{dir}/blk_commands.log" + texts: + - "dd if=/dev/sda1 of=/dev/null bs=1M count=1" + - "findmnt -n /dev/sda1" + - type: file_not_contains + path: "{dir}/blk_commands.log" + text: "df -B 512 /dev/sda1 --output=used,avail" + - type: file_not_contains + path: "{dir}/blk_commands.log" + text: "dd if=/dev/sda1 of=sda1_backup.bin" + diff --git a/common/acs_test_framework_manifests/runtime-device-mapping-conflict-checker.yaml b/common/acs_test_framework_manifests/runtime-device-mapping-conflict-checker.yaml new file mode 100644 index 00000000..73511d3b --- /dev/null +++ b/common/acs_test_framework_manifests/runtime-device-mapping-conflict-checker.yaml @@ -0,0 +1,975 @@ +# Safe scenario-based runtime-device-mapping YAML. +# +# Requires mock_loader support for: +# +# scenario: +# kind: runtime_device_mapping +# dts: | +# ... +# memmap: | +# ... +# memmap_hex: "..." +# +# The mock_loader should materialize: +# dts -> text_files.device_tree.dts +# memmap -> text_files.memmap.log +# memmap_hex -> bin_files.memmap.log.hex +# +# IMPORTANT: +# The main scenario corpus still uses the proven CLI case_runner.py path. +# Native fault-injection cases may use in-process handlers when that is the +# cleaner seam. + +x_case_runner: &case_runner | + #!/usr/bin/env python3 + from __future__ import annotations + + import importlib.util + import sys + from pathlib import Path + + def main() -> int: + if len(sys.argv) != 2: + print("usage: case_runner.py ", file=sys.stderr) + return 2 + + work_dir = Path.cwd() + target = Path(sys.argv[1]).resolve() + + spec = importlib.util.spec_from_file_location( + "runtime_checker_under_test", + target, + ) + if spec is None or spec.loader is None: + print(f"failed to load target: {{target}}", file=sys.stderr) + return 2 + + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + + module.DTS_PATH = work_dir / "device_tree.dts" + module.MEMMAP_PATH = work_dir / "memmap.log" + module.OUT_LOG_PATH = work_dir / "runtime_device_mapping_conflict_test.log" + module._LOG_FH = None + + try: + module.main() + except SystemExit as exc: + code = exc.code + if isinstance(code, int): + return code + return 1 + + log_path = work_dir / "runtime_device_mapping_conflict_test.log" + if log_path.exists(): + print(log_path.read_text(encoding="utf-8", errors="replace"), end="") + + return 0 + + if __name__ == "__main__": + raise SystemExit(main()) + +x_cli_case_base: &cli_case_base + type: cli + command: "{dir}/case_runner.py" + args: + - "{file}" + expect_exit_code: 0 + scripts: + case_runner.py: *case_runner + +suites: + - name: runtime_device_mapping_conflict_checker + files: + - common/linux_scripts/runtime_device_mapping_conflict_checker.py + + cases: + - name: file_exists + type: file_exists + description: "Verify that the runtime device mapping conflict checker script exists." + + - name: python_compiles + type: py_compile + description: "Ensure that the script compiles successfully as valid Python." + + - name: has_main_guard + type: main_guard + description: "Check that the script contains a standard __main__ guard." + + - name: cli_missing_both_inputs_warns + <<: *cli_case_base + scenario: + kind: runtime_device_mapping + expect_output: + - "DEBUG: DTS file not found" + - "RESULTS: WARNINGS" + post_checks: + - type: exists + path: "{dir}/runtime_device_mapping_conflict_test.log" + + - name: cli_missing_memmap_warns + <<: *cli_case_base + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + uart@3000 { + reg = <0x0 0x3000 0x0 0x100>; + }; + }; + }; + expect_output: + - "DEBUG: Memmap file not found" + - "RESULTS: WARNINGS" + post_checks: + - type: exists + path: "{dir}/runtime_device_mapping_conflict_test.log" + + - name: cli_non_overlapping_regions_pass + <<: *cli_case_base + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + uart@3000 { + reg = <0x0 0x3000 0x0 0x100>; + }; + }; + }; + memmap: | + RT_Code 0x1000-0x1fff 1 0 + RT_Data 0x5000-0x5fff 1 0 + expect_output: + - "All UEFI Memmap segments CHECKED" + - "All DTS MMIO ranges CHECKED" + - "Total segments checked: 2" + - "Total DTS ranges checked: 1" + - "No overlaps found between UEFI runtime regions and DTS MMIO ranges" + - "RESULTS: PASSED" + post_checks: + - type: exists + path: "{dir}/runtime_device_mapping_conflict_test.log" + + - name: cli_overlapping_regions_fail + <<: *cli_case_base + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + uart@3000 { + reg = <0x0 0x3000 0x0 0x100>; + }; + }; + }; + memmap: | + RT_Code 0x3000-0x30ff 1 0 + expect_output: + - "RESULTS: FAILED" + - "overlaps DTS /soc/uart@3000" + post_checks: + - type: exists + path: "{dir}/runtime_device_mapping_conflict_test.log" + + - name: cli_disabled_node_is_ignored + <<: *cli_case_base + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + uart@3000 { + status = "disabled"; + reg = <0x0 0x3000 0x0 0x100>; + }; + }; + }; + memmap: | + RT_Code 0x3000-0x30ff 1 0 + expect_output: + - "No DTS MMIO ranges extracted." + - "RESULTS: PASSED" + post_checks: + - type: exists + path: "{dir}/runtime_device_mapping_conflict_test.log" + + - name: cli_disabled_ancestor_is_ignored + <<: *cli_case_base + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + status = "disabled"; + #address-cells = <2>; + #size-cells = <2>; + ranges; + + uart@3000 { + reg = <0x0 0x3000 0x0 0x100>; + }; + }; + }; + memmap: | + RT_Code 0x3000-0x30ff 1 0 + expect_output: + - "No DTS MMIO ranges extracted." + - "RESULTS: PASSED" + + - name: cli_mmap_reg_name_is_ignored + <<: *cli_case_base + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + flash@3000 { + reg = <0x0 0x3000 0x0 0x100>; + reg-names = "mmap"; + }; + }; + }; + memmap: | + RT_Code 0x3000-0x30ff 1 0 + expect_output: + - "No DTS MMIO ranges extracted." + - "RESULTS: PASSED" + post_checks: + - type: exists + path: "{dir}/runtime_device_mapping_conflict_test.log" + + - name: cli_reg_names_with_mmap_and_register_block_skips_only_mmap + <<: *cli_case_base + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + flash@3000 { + reg = <0x0 0x3000 0x0 0x100 0x0 0x4000 0x0 0x100>; + reg-names = "ctrl\0mmap"; + }; + }; + }; + memmap: | + RT_Code 0x3000-0x30ff 1 0 + expect_output: + - "Total DTS ranges checked: 1" + - "/soc/flash@3000" + - "RESULTS: FAILED" + + - name: cli_parent_offset_translation_detects_conflict + <<: *cli_case_base + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + syscon@4000 { + reg = <0x0 0x4000 0x0 0x1000>; + + efuse@100 { + reg = <0x0 0x100 0x0 0x20>; + }; + }; + }; + }; + memmap: | + RT_Data 0x4100-0x411f 1 0 + expect_output: + - "RESULTS: FAILED" + - "/soc/syscon@4000/efuse@100" + post_checks: + - type: exists + path: "{dir}/runtime_device_mapping_conflict_test.log" + + - name: cli_ranges_translation_detects_conflict + <<: *cli_case_base + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + bus@0 { + #address-cells = <2>; + #size-cells = <2>; + ranges = <0x0 0x0 0x0 0x8000 0x0 0x10000>; + + uart@2000 { + reg = <0x0 0x2000 0x0 0x100>; + }; + }; + }; + }; + memmap: | + RT_Code 0xA000-0xA0FF 1 0 + expect_output: + - "RESULTS: FAILED" + - "[T] /soc/bus@0/uart@2000" + + - name: cli_no_matching_range_skips_node_and_passes + <<: *cli_case_base + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + bus@0 { + #address-cells = <2>; + #size-cells = <2>; + ranges = <0x0 0x0 0x0 0x8000 0x0 0x1000>; + + uart@3000 { + reg = <0x0 0x3000 0x0 0x100>; + }; + }; + }; + }; + memmap: | + RT_Code 0xB000-0xB0FF 1 0 + expect_output: + - "No DTS MMIO ranges extracted." + - "RESULTS: PASSED" + + - name: cli_reserved_memory_is_ignored + <<: *cli_case_base + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + carveout@3000 { + reg = <0x0 0x3000 0x0 0x100>; + }; + }; + }; + memmap: | + RT_Code 0x3000-0x30ff 1 0 + expect_output: + - "No DTS MMIO ranges extracted." + - "RESULTS: PASSED" + + - name: cli_memory_node_is_ignored + <<: *cli_case_base + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + memory@80000000 { + device_type = "memory"; + reg = <0x0 0x80000000 0x0 0x100000>; + }; + }; + memmap: | + RT_Code 0x80000000-0x80000fff 1 0 + expect_output: + - "No DTS MMIO ranges extracted." + - "RESULTS: PASSED" + + - name: cli_storage_like_node_is_ignored + <<: *cli_case_base + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + spi@3000 { + compatible = "jedec,spi-nor"; + reg = <0x0 0x3000 0x0 0x100>; + }; + }; + memmap: | + RT_Code 0x3000-0x30ff 1 0 + expect_output: + - "No DTS MMIO ranges extracted." + - "RESULTS: PASSED" + + - name: cli_utf16_memmap_is_parsed + <<: *cli_case_base + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + uart@3000 { + reg = <0x0 0x3000 0x0 0x100>; + }; + }; + }; + memmap_hex: "fffe520054005f0043006f006400650020003000780031003000300030002d0030007800310066006600660020003100200030000a00" + expect_output: + - "Total segments checked: 1" + - "RESULTS: PASSED" + post_checks: + - type: exists + path: "{dir}/runtime_device_mapping_conflict_test.log" + + - name: cli_malformed_memmap_line_is_logged + <<: *cli_case_base + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + uart@9000 { + reg = <0x0 0x9000 0x0 0x100>; + }; + }; + }; + memmap: | + RT_Code badline + RT_Data 0x1000-0x1fff 1 0 + expect_output: + - "Skipped malformed memmap line" + - "Skipped 1 malformed lines from memmap" + - "RESULTS: PASSED" + post_checks: + - type: exists + path: "{dir}/runtime_device_mapping_conflict_test.log" + + - name: cli_deduplicates_identical_memmap_entries + <<: *cli_case_base + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + uart@9000 { + reg = <0x0 0x9000 0x0 0x100>; + }; + }; + }; + memmap: | + RT_Data 0x1000-0x1fff 1 0 + RT_Data 0x1000-0x1fff 1 0 + expect_output: + - "Total segments checked: 1" + - "RESULTS: PASSED" + + - name: cli_hex_without_0x_in_memmap_is_accepted + <<: *cli_case_base + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + uart@4096 { + reg = <0x0 0x4096 0x0 0x100>; + }; + }; + }; + memmap: | + RT_Code 1000-10ff 1 0 + expect_output: + - "Total segments checked: 1" + - "RESULTS: PASSED" + + - name: cli_symbolic_reg_cells_known_limitation + <<: *cli_case_base + warn_only: true + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + uart@3000 { + reg = ; + }; + }; + }; + memmap: | + RT_Code 0x3000-0x30ff 1 0 + expect_output: + - "No DTS MMIO ranges extracted." + - "RESULTS: PASSED" + + - name: cli_multiline_reg_property_warning + <<: *cli_case_base + warn_only: true + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + uart@3000 { + reg = < + 0x0 0x3000 + 0x0 0x100 + >; + }; + }; + }; + memmap: | + RT_Code 0x3000-0x30ff 1 0 + expect_output: + - "INFO: No DTS MMIO ranges extracted." + - "RESULTS: PASSED" + + - name: cli_multiline_ranges_property_warning + <<: *cli_case_base + warn_only: true + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + bus@0 { + #address-cells = <2>; + #size-cells = <2>; + ranges = < + 0x0 0x0 + 0x0 0x8000 + 0x0 0x10000 + >; + + uart@2000 { + reg = <0x0 0x2000 0x0 0x100>; + }; + }; + }; + }; + memmap: | + RT_Code 0xA000-0xA0FF 1 0 + expect_output: + - "Total DTS ranges checked: 1" + - "INFO: [T] /soc/bus@0/uart@2000" + - "RESULTS: PASSED" + + - name: cli_multiline_reg_names_warning + <<: *cli_case_base + warn_only: true + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + flash@3000 { + reg = <0x0 0x3000 0x0 0x100 0x0 0x4000 0x0 0x100>; + reg-names = + "ctrl\0mmap"; + }; + }; + }; + memmap: | + RT_Code 0x3000-0x30ff 1 0 + expect_output: + - "Total DTS ranges checked: 2" + - "Detected 1 conflict(s)" + - "RESULTS: FAILED" + + - name: cli_multiline_quoted_compatible_property_warning + <<: *cli_case_base + warn_only: true + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + spi@3000 { + compatible = + "jedec,spi-nor"; + reg = <0x0 0x3000 0x0 0x100>; + }; + }; + memmap: | + RT_Code 0x3000-0x30ff 1 0 + expect_output: + - "Total DTS ranges checked: 1" + - "Detected 1 conflict(s)" + - "RESULTS: FAILED" + + - name: cli_multiline_status_property_warning + <<: *cli_case_base + warn_only: true + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + uart@3000 { + status = + "disabled"; + reg = <0x0 0x3000 0x0 0x100>; + }; + }; + }; + memmap: | + RT_Code 0x3000-0x30ff 1 0 + expect_output: + - "Total DTS ranges checked: 1" + - "Detected 1 conflict(s)" + - "RESULTS: FAILED" + + - name: cli_multiline_device_type_memory_property_warning + <<: *cli_case_base + warn_only: true + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + memory@80000000 { + device_type = + "memory"; + reg = <0x0 0x80000000 0x0 0x100000>; + }; + }; + memmap: | + RT_Code 0x80000000-0x80000fff 1 0 + expect_output: + - "No DTS MMIO ranges extracted." + - "RESULTS: PASSED" + + - name: cli_ranges_with_incomplete_cell_tuple_warning + <<: *cli_case_base + warn_only: true + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + bus@0 { + #address-cells = <2>; + #size-cells = <2>; + ranges = <0x0 0x0 0x0 0x8000 0x0>; + + uart@2000 { + reg = <0x0 0x2000 0x0 0x100>; + }; + }; + }; + }; + memmap: | + RT_Code 0xA000-0xA0FF 1 0 + expect_output: + - "No DTS MMIO ranges extracted." + - "RESULTS: PASSED" + + - name: cli_ranges_with_extra_trailing_cells_warning + <<: *cli_case_base + warn_only: true + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + bus@0 { + #address-cells = <2>; + #size-cells = <2>; + ranges = <0x0 0x0 0x0 0x8000 0x0 0x10000 0xdeadbeef>; + + uart@2000 { + reg = <0x0 0x2000 0x0 0x100>; + }; + }; + }; + }; + memmap: | + RT_Code 0xA000-0xA0FF 1 0 + expect_output: + - "Total DTS ranges checked: 1" + - "Detected 1 conflict(s)" + - "RESULTS: FAILED" + + - name: cli_parent_reg_multiple_windows_offset_heuristic_warning + <<: *cli_case_base + warn_only: true + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + syscon@4000 { + reg = <0x0 0x4000 0x0 0x100 0x0 0x8000 0x0 0x100>; + + efuse@80 { + reg = <0x0 0x80 0x0 0x20>; + }; + }; + }; + }; + memmap: | + RT_Data 0x4080-0x409f 1 0 + expect_output: + - "/soc/syscon@4000/efuse@80" + - "Detected 2 conflict(s)" + - "RESULTS: FAILED" + + - name: cli_reg_names_count_mismatch_warning + <<: *cli_case_base + warn_only: true + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + flash@3000 { + reg = <0x0 0x3000 0x0 0x100 0x0 0x4000 0x0 0x100>; + reg-names = "mmap"; + }; + }; + }; + memmap: | + RT_Code 0x4000-0x40ff 1 0 + expect_output: + - "Total DTS ranges checked: 1" + - "Detected 1 conflict(s)" + - "RESULTS: FAILED" + + - name: cli_uppercase_disabled_status_value_warning + <<: *cli_case_base + warn_only: true + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + uart@3000 { + status = "DISABLED"; + reg = <0x0 0x3000 0x0 0x100>; + }; + }; + }; + memmap: | + RT_Code 0x3000-0x30ff 1 0 + expect_output: + - "No DTS MMIO ranges extracted." + - "RESULTS: PASSED" + + - name: cli_ranges_present_but_empty_cells_warning + <<: *cli_case_base + warn_only: true + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + bus@0 { + #address-cells = <2>; + #size-cells = <2>; + ranges = <>; + + uart@2000 { + reg = <0x0 0x2000 0x0 0x100>; + }; + }; + }; + }; + memmap: | + RT_Code 0xA000-0xA0FF 1 0 + expect_output: + - "Total DTS ranges checked: 1" + - "INFO: [T] /soc/bus@0/uart@2000" + - "RESULTS: PASSED" + + - name: cli_comments_are_stripped_before_parse + <<: *cli_case_base + scenario: + kind: runtime_device_mapping + dts: | + /dts-v1/; + /* block comment */ + / { + soc { + #address-cells = <2>; // inline comment + #size-cells = <2>; + ranges; + uart@3000 { + reg = <0x0 0x3000 0x0 0x100>; // end of line comment + }; + }; + }; + memmap: | + RT_Code 0x3000-0x30ff 1 0 + expect_output: + - "Total DTS ranges checked: 1" + - "RESULTS: FAILED" + + # Native fault-injection cases stay declarative through scenario flags. + + - name: cli_missing_dts_file_is_warned_natively + type: module_main_with_env + warn_only: true + scenario: + kind: runtime_device_mapping + missing_dts: true + memmap: | + RT_Code 0x1000-0x1fff 1 0 + post_checks: + - type: exists + path: "{dir}/runtime_device_mapping_conflict_test.log" + - type: file_contains + path: "{dir}/runtime_device_mapping_conflict_test.log" + text: "DEBUG: DTS file not found:" + - type: file_contains + path: "{dir}/runtime_device_mapping_conflict_test.log" + text: "RESULTS: WARNINGS" + + - name: cli_missing_memmap_file_is_warned_natively + type: module_main_with_env + warn_only: true + scenario: + kind: runtime_device_mapping + missing_memmap: true + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + uart@3000 { + reg = <0x0 0x3000 0x0 0x100>; + }; + }; + }; + post_checks: + - type: exists + path: "{dir}/runtime_device_mapping_conflict_test.log" + - type: file_contains + path: "{dir}/runtime_device_mapping_conflict_test.log" + text: "DEBUG: Memmap file not found:" + - type: file_contains + path: "{dir}/runtime_device_mapping_conflict_test.log" + text: "RESULTS: WARNINGS" + + - name: cli_mock_log_open_failure_warning + type: py_function + warn_only: true + function: main + description: "Document the current behavior where opening the runtime checker log aborts the run before any warning summary is written." + scenario: + kind: runtime_device_mapping + log_open_error: mocked log open failure + dts: | + /dts-v1/; + / { + soc { + #address-cells = <2>; + #size-cells = <2>; + ranges; + uart@3000 { + reg = <0x0 0x3000 0x0 0x100>; + }; + }; + }; + memmap: | + RT_Code 0x1000-0x1fff 1 0 + expect_exception: OSError + post_checks: + - type: not_exists + path: "{dir}/runtime_device_mapping_conflict_test.log" diff --git a/common/acs_test_framework_manifests/verify-tpm-measurements.yaml b/common/acs_test_framework_manifests/verify-tpm-measurements.yaml new file mode 100644 index 00000000..d30bd8d7 --- /dev/null +++ b/common/acs_test_framework_manifests/verify-tpm-measurements.yaml @@ -0,0 +1,526 @@ +x_verify_tpm_cli: &verify_tpm_cli + type: cli + command: "./run_case.sh" + args: + - "{file}" + expect_exit_code: 0 + timeout_sec: 20 + shell: false + scripts: + run_case.sh: | + #!/bin/sh + set -eu + python3 "$1" "$PWD/pcr.yaml" "$PWD/event.yaml" + +x_valid_tpm_events: &valid_tpm_events + - EventNum: 1 + PCRIndex: 0 + EventType: EV_NO_ACTION + SpecID: + - specVersionMajor: 2 + + - EventNum: 2 + PCRIndex: 0 + EventType: EV_POST_CODE + Event: "BL_31" + + - EventNum: 3 + PCRIndex: 0 + EventType: EV_POST_CODE + Event: "SECURE_RT_EL3" + + - EventNum: 4 + PCRIndex: 7 + EventType: EV_EFI_VARIABLE_DRIVER_CONFIG + Event: + UnicodeName: SecureBoot + + - EventNum: 5 + PCRIndex: 7 + EventType: EV_EFI_VARIABLE_DRIVER_CONFIG + Event: + UnicodeName: PK + + - EventNum: 6 + PCRIndex: 7 + EventType: EV_EFI_VARIABLE_DRIVER_CONFIG + Event: + UnicodeName: KEK + + - EventNum: 7 + PCRIndex: 7 + EventType: EV_EFI_VARIABLE_DRIVER_CONFIG + Event: + UnicodeName: db + + - EventNum: 8 + PCRIndex: 7 + EventType: EV_EFI_VARIABLE_DRIVER_CONFIG + Event: + UnicodeName: dbx + + - EventNum: 9 + PCRIndex: 1 + EventType: EV_EFI_VARIABLE_BOOT + Event: + UnicodeName: BootOrder + + - EventNum: 10 + PCRIndex: 1 + EventType: EV_EFI_VARIABLE_BOOT2 + Event: + UnicodeName: Boot0001 + + - EventNum: 11 + PCRIndex: 4 + EventType: EV_EFI_ACTION + Event: "Calling EFI Application from Boot Option" + + - EventNum: 12 + PCRIndex: 1 + EventType: EV_EFI_HANDOFF_TABLES + Event: "SMBIOS" + + - EventNum: 13 + PCRIndex: 0 + EventType: EV_SEPARATOR + Event: "sep0" + + - EventNum: 14 + PCRIndex: 1 + EventType: EV_SEPARATOR + Event: "sep1" + + - EventNum: 15 + PCRIndex: 2 + EventType: EV_SEPARATOR + Event: "sep2" + + - EventNum: 16 + PCRIndex: 3 + EventType: EV_SEPARATOR + Event: "sep3" + + - EventNum: 17 + PCRIndex: 4 + EventType: EV_SEPARATOR + Event: "sep4" + + - EventNum: 18 + PCRIndex: 5 + EventType: EV_SEPARATOR + Event: "sep5" + + - EventNum: 19 + PCRIndex: 6 + EventType: EV_SEPARATOR + Event: "sep6" + + - EventNum: 20 + PCRIndex: 7 + EventType: EV_SEPARATOR + Event: "sep7" + + - EventNum: 21 + PCRIndex: 1 + EventType: EV_TABLE_OF_DEVICES + Event: "SYS_CONFIG_UART0" + + - EventNum: 22 + PCRIndex: 5 + EventType: EV_EFI_ACTION + Event: "Exit Boot Services Invocation" + +x_without_boot_variable_events: &without_boot_variable_events + - EventNum: 1 + PCRIndex: 0 + EventType: EV_NO_ACTION + SpecID: + - specVersionMajor: 2 + + - EventNum: 2 + PCRIndex: 0 + EventType: EV_POST_CODE + Event: "BL_31" + + - EventNum: 3 + PCRIndex: 0 + EventType: EV_POST_CODE + Event: "SECURE_RT_EL3" + + - EventNum: 4 + PCRIndex: 7 + EventType: EV_EFI_VARIABLE_DRIVER_CONFIG + Event: + UnicodeName: SecureBoot + + - EventNum: 5 + PCRIndex: 7 + EventType: EV_EFI_VARIABLE_DRIVER_CONFIG + Event: + UnicodeName: PK + + - EventNum: 6 + PCRIndex: 7 + EventType: EV_EFI_VARIABLE_DRIVER_CONFIG + Event: + UnicodeName: KEK + + - EventNum: 7 + PCRIndex: 7 + EventType: EV_EFI_VARIABLE_DRIVER_CONFIG + Event: + UnicodeName: db + + - EventNum: 8 + PCRIndex: 7 + EventType: EV_EFI_VARIABLE_DRIVER_CONFIG + Event: + UnicodeName: dbx + + - EventNum: 9 + PCRIndex: 4 + EventType: EV_EFI_ACTION + Event: "Calling EFI Application from Boot Option" + + - EventNum: 10 + PCRIndex: 1 + EventType: EV_EFI_HANDOFF_TABLES + Event: "SMBIOS" + + - EventNum: 11 + PCRIndex: 0 + EventType: EV_SEPARATOR + Event: "sep0" + + - EventNum: 12 + PCRIndex: 1 + EventType: EV_SEPARATOR + Event: "sep1" + + - EventNum: 13 + PCRIndex: 2 + EventType: EV_SEPARATOR + Event: "sep2" + + - EventNum: 14 + PCRIndex: 3 + EventType: EV_SEPARATOR + Event: "sep3" + + - EventNum: 15 + PCRIndex: 4 + EventType: EV_SEPARATOR + Event: "sep4" + + - EventNum: 16 + PCRIndex: 5 + EventType: EV_SEPARATOR + Event: "sep5" + + - EventNum: 17 + PCRIndex: 6 + EventType: EV_SEPARATOR + Event: "sep6" + + - EventNum: 18 + PCRIndex: 7 + EventType: EV_SEPARATOR + Event: "sep7" + + - EventNum: 19 + PCRIndex: 1 + EventType: EV_TABLE_OF_DEVICES + Event: "SYS_CONFIG_UART0" + + - EventNum: 20 + PCRIndex: 5 + EventType: EV_EFI_ACTION + Event: "Exit Boot Services Invocation" + +suites: + - name: verify_tpm_measurements + files: + - common/linux_scripts/verify_tpm_measurements.py + + cases: + # Baseline repository-integrity check. + - name: file_exists + type: file_exists + description: "Verify that the TPM measurement verification script exists." + + # Fast syntax-level validation. + - name: python_compiles + type: py_compile + description: "Verify that the script compiles successfully as valid Python." + + # Required CLI contract validation: no arguments supplied. + - name: cli_no_args_fails + type: cli + command: python3 + args: + - "{file}" + expect_exit_code: 1 + expect_stdout_or_stderr_contains: + - "Usage: python3 verify_tpm_measurements.py" + + # Required CLI contract validation: only one argument supplied. + - name: cli_one_arg_fails + type: cli + command: python3 + args: + - "{file}" + - "{dir}/pcr.yaml" + expect_exit_code: 1 + expect_stdout_or_stderr_contains: + - "Usage: python3 verify_tpm_measurements.py" + + # Negative-path file validation. + - name: cli_missing_files_fail_nonzero + type: cli + command: python3 + args: + - "{file}" + - "{dir}/missing_pcr.yaml" + - "{dir}/missing_event.yaml" + expect_exit_nonzero: true + expect_stdout_or_stderr_contains: + - "FAIL" + + # Malformed-input parsing test. + - name: cli_invalid_event_yaml_fails + type: cli + command: python3 + args: + - "{file}" + - "{dir}/pcr.yaml" + - "{dir}/event.yaml" + text_files: + pcr.yaml: | + sha256: + - "a0" + - "a1" + - "a2" + - "a3" + - "a4" + - "a5" + - "a6" + - "a7" + event.yaml: | + not: [valid + expect_exit_nonzero: true + expect_stdout_or_stderr_contains: + - "FAIL" + + # Injected I/O failure test. + - name: cli_mock_open_raises_on_event_log + type: module_cli + scenario: + kind: verify_tpm + pcrs: + sha256: + - "a0" + - "a1" + - "a2" + - "a3" + - "a4" + - "a5" + - "a6" + - "a7" + event_pcrs: + sha256: + - "a0" + - "a1" + - "a2" + - "a3" + - "a4" + - "a5" + - "a6" + - "a7" + events: [] + event_log_open_error: mocked read failure + expect_exit_code: 1 + expect_stdout_or_stderr_contains: + - "FAIL" + - "mocked read failure" + + # Injected YAML-loader failure test. + - name: cli_mock_yaml_safe_load_raises_on_event_log + type: module_cli + scenario: + kind: verify_tpm + pcrs: + sha256: + - "a0" + - "a1" + - "a2" + - "a3" + - "a4" + - "a5" + - "a6" + - "a7" + event_pcrs: + sha256: + - "a0" + - "a1" + - "a2" + - "a3" + - "a4" + - "a5" + - "a6" + - "a7" + events: [] + event_log_yaml_error: mocked event parse failure + expect_exit_code: 1 + expect_stdout_or_stderr_contains: + - "FAIL" + - "mocked event parse failure" + + # Full positive-path event-log scenario. + - name: cli_all_required_measurements_pass + <<: *verify_tpm_cli + scenario: + kind: verify_tpm + pcrs: + sha256: + - "pcr0" + - "pcr1" + - "pcr2" + - "pcr3" + - "pcr4" + - "pcr5" + - "pcr6" + - "pcr7" + event_pcrs: + sha256: + - "pcr0" + - "pcr1" + - "pcr2" + - "pcr3" + - "pcr4" + - "pcr5" + - "pcr6" + - "pcr7" + events: *valid_tpm_events + expect_stdout_or_stderr_contains: + - "Verify that the cumulative SHA256 measurements from the event log match the TPM PCRs 0-7" + - "Verify the first event is the EV_NO_ACTION for the Specification ID version" + - "Verify EV_POST_CODE events for measurements of firmware to PCR[0] with recommended strings" + - "Verify secure boot policy measurements are made into PCR[7] with EV_EFI_VARIABLE_DRIVER_CONFIG" + - "Verify BootOrder and Boot#### variables are measured in PCR[1] withEV_EFI_VARIABLE_BOOT/EV_EFI_VARIABLE_BOOT2" + - "Verify boot attempts measured in PCR[4] with EV_EFI_ACTION event type" + - "Verify security relevant configuration data are measured in into PCR[1] with EV_EFI_HANDOFF_TABLES" + - "Verify presence of EV_SEPARATOR event for each PCR" + - "Verify EV_TABLE_OF_DEVICES events for measurements of config data to PCR[1] with recommended strings" + - "Verify presence of “Exit Boot Services Invocation” event with EV_EFI_ACTION type" + - "PASS" + + # PCR mismatch while event classes are otherwise present. + - name: cli_pcr_mismatch_reports_fail_but_runs + <<: *verify_tpm_cli + scenario: + kind: verify_tpm + pcrs: + sha256: + - "good0" + - "good1" + - "good2" + - "good3" + - "good4" + - "good5" + - "good6" + - "good7" + event_pcrs: + sha256: + - "good0" + - "bad1" + - "good2" + - "good3" + - "good4" + - "good5" + - "good6" + - "good7" + events: *valid_tpm_events + expect_stdout_or_stderr_contains: + - "PCR[1] measurements does not match for sha256" + - "FAIL" + + # Boot variable measurements intentionally omitted. + - name: cli_boot_variable_missing_reports_fail_but_runs + <<: *verify_tpm_cli + scenario: + kind: verify_tpm + pcrs: + sha256: + - "pcr0" + - "pcr1" + - "pcr2" + - "pcr3" + - "pcr4" + - "pcr5" + - "pcr6" + - "pcr7" + event_pcrs: + sha256: + - "pcr0" + - "pcr1" + - "pcr2" + - "pcr3" + - "pcr4" + - "pcr5" + - "pcr6" + - "pcr7" + events: *without_boot_variable_events + expect_stdout_or_stderr_contains: + - "BootOrder/Boot#### variable measurements not found" + - "FAIL" + + # Supported-algorithm negative path. + - name: pcr_comparison_requires_at_least_one_supported_algorithm_match + <<: *verify_tpm_cli + warn_only: true + scenario: + kind: verify_tpm + pcrs: + sha1: + - "p0" + - "p1" + - "p2" + - "p3" + - "p4" + - "p5" + - "p6" + - "p7" + event_pcrs: + sha1: + - "p0" + - "p1" + - "p2" + - "p3" + - "p4" + - "p5" + - "p6" + - "p7" + events: [] + expect_stdout_or_stderr_contains: + - "FAIL" + + # Truncated-PCR-array negative path. + - name: truncated_pcr_arrays_are_reported_cleanly + <<: *verify_tpm_cli + warn_only: true + scenario: + kind: verify_tpm + pcrs: + sha256: + - "p0" + - "p1" + - "p2" + event_pcrs: + sha256: + - "p0" + - "p1" + - "p2" + events: [] + expect_stdout_or_stderr_contains: + - "FAIL" diff --git a/common/acs_test_framework_runner/__init__.py b/common/acs_test_framework_runner/__init__.py new file mode 100644 index 00000000..3a588c3a --- /dev/null +++ b/common/acs_test_framework_runner/__init__.py @@ -0,0 +1,134 @@ +from .mock_loader import _resolve_runner_module_from_stack, stateful_run_router +from .mock_loader import ConfigError as MockLoaderConfigError +from .mock_helpers import ( + build_char16_payload, + build_df_output, + build_efi_var_bytes, + build_ethtool_ip_address_output, + build_ethtool_ip_link_line, + build_ethtool_ip_link_show_line, + build_fdisk_output, + build_os_indications_var, + build_run_result_from_outcome, + build_sgdisk_partition_output, + check_output_router, + default_device_path, + noop, + normalize_ethtool_tool_path, + passthrough_router, + route_gateway, + run_router, + scenario_truthy, + which_router, +) +from .case_data_builders import CaseBuildError, render_post_check_path +from .case_data_builders import expand_template as base_expand_template +from .runner_checks import ( + DEFAULT_CLI_TIMEOUT_SEC, + DESTRUCTIVE_TEST_ENV, + REAL_SUBPROCESS_RUN, + ConfigError, + CommandRunResult, + detect_project_root, + ensure_list, + ensure_list_of_strings, + ensure_string_or_list_of_strings, + RunCaseOptions, + SkipCase, + TestMeta, + TestOutcome, + create_outcome, + collect_function_names, + create_runner_temp_dir, + format_outcome_message, + load_yaml_config, + merge_mappings, + normalize_suites, + resolve_target_path, + run_single_check, + sanitize_name, + load_module_from_path, + normalize_completed_stream, + read_source, + sanitize_xml_text, +) +from .runner_reporting import ( + append_combined_case_log, + append_run_header, + build_config_error_outcome, + build_report_path, + cleanup_old_pytest_xml_reports, + create_placeholder_xml, + print_group_summary, + remove_placeholder_xml, + write_case_log, + write_junit_xml, +) +__all__ = [ + "MockLoaderConfigError", + "stateful_run_router", + + "build_char16_payload", + "build_df_output", + "build_efi_var_bytes", + "build_ethtool_ip_address_output", + "build_ethtool_ip_link_line", + "build_ethtool_ip_link_show_line", + "build_fdisk_output", + "build_os_indications_var", + "build_run_result_from_outcome", + "build_sgdisk_partition_output", + "check_output_router", + "default_device_path", + "noop", + "normalize_ethtool_tool_path", + "passthrough_router", + "route_gateway", + "run_router", + "scenario_truthy", + "which_router", + "base_expand_template", + + "CaseBuildError", + "render_post_check_path", + + "DEFAULT_CLI_TIMEOUT_SEC", + "DESTRUCTIVE_TEST_ENV", + "REAL_SUBPROCESS_RUN", + "ConfigError", + "CommandRunResult", + "detect_project_root", + "ensure_list", + "ensure_list_of_strings", + "ensure_string_or_list_of_strings", + "RunCaseOptions", + "SkipCase", + "TestMeta", + "TestOutcome", + "create_outcome", + "collect_function_names", + "create_runner_temp_dir", + "format_outcome_message", + "load_yaml_config", + "merge_mappings", + "normalize_suites", + "resolve_target_path", + "run_single_check", + "sanitize_name", + "load_module_from_path", + "normalize_completed_stream", + "read_source", + "sanitize_xml_text", + + "append_combined_case_log", + "append_run_header", + "build_config_error_outcome", + "build_report_path", + "cleanup_old_pytest_xml_reports", + "create_placeholder_xml", + "print_group_summary", + "remove_placeholder_xml", + "write_case_log", + "write_junit_xml", +] + diff --git a/common/acs_test_framework_runner/case_data_builders.py b/common/acs_test_framework_runner/case_data_builders.py new file mode 100644 index 00000000..070d81c1 --- /dev/null +++ b/common/acs_test_framework_runner/case_data_builders.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any, Mapping + + +class CaseBuildError(Exception): + """Raised when case workspace data is invalid or cannot be materialized.""" + + +def expand_template(value: str, work_dir: Path, file_path: Path) -> str: + return ( + value.replace("{dir}", str(work_dir)) + .replace("{file}", str(file_path)) + .replace("{filename}", file_path.name) + ) + + +def replace_dir_tokens(value: Any, temp_dir: Path) -> Any: + if isinstance(value, str): + return value.replace("{dir}", str(temp_dir)) + if isinstance(value, list): + return [replace_dir_tokens(item, temp_dir) for item in value] + if isinstance(value, tuple): + return tuple(replace_dir_tokens(item, temp_dir) for item in value) + if isinstance(value, dict): + return { + key: replace_dir_tokens(item, temp_dir) + for key, item in value.items() + } + return value + + +def build_runtime_case_for_tempdir( + case_def: Mapping[str, Any], + temp_dir: Path, +) -> dict[str, Any]: + runtime_case = dict(case_def) + runtime_case["scripts"] = replace_dir_tokens( + runtime_case.get("scripts", {}), + temp_dir, + ) + runtime_case["bin_files"] = replace_dir_tokens( + runtime_case.get("bin_files", {}), + temp_dir, + ) + runtime_case["text_files"] = replace_dir_tokens( + runtime_case.get("text_files", {}), + temp_dir, + ) + runtime_case["dir_structure"] = replace_dir_tokens( + runtime_case.get("dir_structure", []), + temp_dir, + ) + runtime_case["patch_constants"] = replace_dir_tokens( + runtime_case.get("patch_constants", {}), + temp_dir, + ) + return runtime_case + + +def render_post_check_path(raw_path: str, work_dir: Path) -> Path: + return Path(raw_path.format(dir=str(work_dir))) + + +def _write_text_if_changed(path: Path, content: str, *, executable: bool = False) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + + if path.exists(): + try: + existing = path.read_text(encoding="utf-8") + if existing == content: + if executable: + path.chmod(0o755) + return + except Exception: + pass + + path.write_text(content, encoding="utf-8") + if executable: + path.chmod(0o755) + + +def _write_bytes_if_changed(path: Path, content: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + + if path.exists(): + try: + existing = path.read_bytes() + if existing == content: + return + except Exception: + pass + + path.write_bytes(content) + + +def prepare_case_files(work_dir: Path, case_def: Mapping[str, Any]) -> None: + scripts = case_def.get("scripts", {}) + if scripts is not None: + if not isinstance(scripts, dict): + raise CaseBuildError("'scripts' must be a mapping") + for name, content in scripts.items(): + if not isinstance(name, str) or not isinstance(content, str): + raise CaseBuildError("'scripts' entries must be string -> string") + script_path = work_dir / name + _write_text_if_changed(script_path, content, executable=True) + + bin_files = case_def.get("bin_files", {}) + if bin_files is not None: + if not isinstance(bin_files, dict): + raise CaseBuildError("'bin_files' must be a mapping") + for name, spec in bin_files.items(): + if not isinstance(name, str) or not isinstance(spec, dict): + raise CaseBuildError("'bin_files' entries must be string -> mapping") + + target_file = work_dir / name + hex_data = spec.get("hex") + text_data = spec.get("text") + + if hex_data is not None: + if not isinstance(hex_data, str): + raise CaseBuildError( + f"Invalid hex data type for bin_files entry '{name}'" + ) + try: + payload = bytes.fromhex(hex_data) + except ValueError as exc: + raise CaseBuildError( + f"Invalid hex data for bin_files entry '{name}': {exc}" + ) from exc + _write_bytes_if_changed(target_file, payload) + elif text_data is not None: + if not isinstance(text_data, str): + raise CaseBuildError( + f"Invalid text data type for bin_files entry '{name}'" + ) + _write_bytes_if_changed(target_file, text_data.encode("utf-8")) + else: + raise CaseBuildError( + "Each 'bin_files' entry requires string key 'hex' or 'text'" + ) + + text_files = case_def.get("text_files", {}) + if text_files is not None: + if not isinstance(text_files, dict): + raise CaseBuildError("'text_files' must be a mapping") + for name, content in text_files.items(): + if not isinstance(name, str) or not isinstance(content, str): + raise CaseBuildError("'text_files' entries must be string -> string") + target_file = work_dir / name + _write_text_if_changed(target_file, content) + + +def materialize_case_workspace( + work_dir: Path, + case_def: Mapping[str, Any], +) -> list[str]: + details: list[str] = [] + + dir_structure = case_def.get("dir_structure", []) + if dir_structure is None: + dir_structure = [] + if not isinstance(dir_structure, list): + raise CaseBuildError("'dir_structure' must be a list") + + for index, entry in enumerate(dir_structure, start=1): + if not isinstance(entry, dict): + raise CaseBuildError(f"'dir_structure[{index}]' must be a mapping") + rel_path = entry.get("path") + if not isinstance(rel_path, str) or not rel_path.strip(): + raise CaseBuildError( + f"'dir_structure[{index}].path' must be a non-empty string" + ) + dir_path = work_dir / rel_path + dir_path.mkdir(parents=True, exist_ok=True) + details.append(f"Created directory: {dir_path}") + + prepare_case_files(work_dir, case_def) + return details diff --git a/common/acs_test_framework_runner/mock_helpers.py b/common/acs_test_framework_runner/mock_helpers.py new file mode 100644 index 00000000..ec302d2f --- /dev/null +++ b/common/acs_test_framework_runner/mock_helpers.py @@ -0,0 +1,759 @@ +from __future__ import annotations + +import re +from pathlib import Path +from subprocess import CalledProcessError, CompletedProcess +from typing import Any +from unittest.mock import Mock + + +def noop(*args: Any, **kwargs: Any) -> None: + """Simple no-op helper for replacing sleeps and signal handlers.""" + del args, kwargs + + +def basic_mock(**kwargs: Any) -> Mock: + """ + Generic Mock factory. + + Example: + factory: mock_helpers.basic_mock + attrs: + return_value: 123 + """ + return Mock(**kwargs) + + +def completed_process_mock( + stdout: str = "", + stderr: str = "", + returncode: int = 0, +) -> Mock: + """ + Mock for subprocess.run returning a CompletedProcess-like object. + """ + mock_obj = Mock() + mock_obj.return_value = CompletedProcess( + args=[], + returncode=returncode, + stdout=stdout, + stderr=stderr, + ) + return mock_obj + + +def simple_check_output(output: str = "", as_bytes: bool = True): + """ + Mock factory for subprocess.check_output with one fixed output. + """ + def _mock_check_output(*args, **kwargs): + del args, kwargs + return output.encode() if as_bytes else output + + return _mock_check_output + + +def _pick_routed_value(value: Any, state: dict[str, int], key: str) -> Any: + """Return a routed value, supporting sticky sequential lists.""" + if not isinstance(value, list): + return value + if not value: + raise ValueError("Router response lists must not be empty") + index = state.get(key, 0) + state[key] = index + 1 + if index >= len(value): + return value[-1] + return value[index] + + +def check_output_router( + responses: dict[str, str] | None = None, + errors: dict[str, str] | None = None, + default_output: str = "", + as_bytes: bool = True, + use_contains: bool = True, +): + """ + Route subprocess.check_output return values based on command text. + + responses: + "ip link": "eth0\\neth1\\n" + "ethtool eth0": "Link detected: yes\\n" + + errors: + "ethtool eth9": "No such device" + """ + responses = responses or {} + errors = errors or {} + call_state: dict[str, int] = {} + + def _match(pattern: str, cmd_text: str) -> bool: + if pattern.startswith("regex:"): + return re.search(pattern[6:], cmd_text) is not None + if use_contains: + return pattern in cmd_text + return pattern == cmd_text + + def _mock_check_output(cmd, *args, **kwargs): + del args, kwargs + cmd_text = " ".join(cmd) if isinstance(cmd, (list, tuple)) else str(cmd) + + for pattern, output in responses.items(): + if _match(pattern, cmd_text): + selected = _pick_routed_value(output, call_state, pattern) + return selected.encode() if as_bytes else selected + + for pattern, error_text in errors.items(): + if _match(pattern, cmd_text): + selected = _pick_routed_value(error_text, call_state, f"error:{pattern}") + raise CalledProcessError(1, cmd, output=selected) + + return default_output.encode() if as_bytes else default_output + + return _mock_check_output + + +def run_router( + responses: dict[str, Any] | None = None, + **options: Any, +): + """ + Route subprocess.run behavior based on command text. + + responses format: + "lsblk": {"stdout": "sda\\n", "stderr": "", "returncode": 0} + "dd": {"stdout": "", "stderr": "write fail", "returncode": 1} + """ + responses = responses or {} + default_stdout = str(options.pop("default_stdout", "")) + default_stderr = str(options.pop("default_stderr", "")) + default_returncode = int(options.pop("default_returncode", 0)) + unmatched_stderr_template = options.pop("unmatched_stderr_template", None) + if unmatched_stderr_template is not None and not isinstance( + unmatched_stderr_template, + str, + ): + raise TypeError("'unmatched_stderr_template' must be a string") + use_contains = bool(options.pop("use_contains", True)) + call_log_path = options.pop("call_log_path", None) + if call_log_path is not None and not isinstance(call_log_path, str): + raise TypeError("'call_log_path' must be a string") + if options: + unsupported = ", ".join(sorted(options)) + raise TypeError(f"Unsupported run_router option(s): {unsupported}") + + call_state: dict[str, int] = {} + + def _match(pattern: str, cmd_text: str) -> bool: + if pattern.startswith("regex:"): + return re.search(pattern[6:], cmd_text) is not None + if use_contains: + return pattern in cmd_text + return pattern == cmd_text + + def _mock_run(cmd, *args, **kwargs): + del args, kwargs + cmd_text = " ".join(cmd) if isinstance(cmd, (list, tuple)) else str(cmd) + + if call_log_path is not None: + log_path = Path(call_log_path) + log_path.parent.mkdir(parents=True, exist_ok=True) + with log_path.open("a", encoding="utf-8") as handle: + handle.write(f"{cmd_text}\n") + + for pattern, result in responses.items(): + if _match(pattern, cmd_text): + resolved = _pick_routed_value(result, call_state, pattern) + if not isinstance(resolved, dict): + raise TypeError("run_router responses must resolve to mappings") + return CompletedProcess( + args=cmd, + returncode=resolved.get("returncode", 0), + stdout=resolved.get("stdout", ""), + stderr=resolved.get("stderr", ""), + ) + + stderr_text = default_stderr + if unmatched_stderr_template is not None: + stderr_text = unmatched_stderr_template.format(cmd=cmd_text) + + return CompletedProcess( + args=cmd, + returncode=default_returncode, + stdout=default_stdout, + stderr=stderr_text, + ) + + return _mock_run + + +def path_read_text_router( + responses: dict[str, str] | None = None, + default_text: str = "", + use_contains: bool = True, +): + """ + Router for Path.read_text() based on path string. + + responses: + "/sys/class/net/eth0/speed": "1000" + """ + responses = responses or {} + + def _match(pattern: str, path_text: str) -> bool: + if pattern.startswith("regex:"): + return re.search(pattern[6:], path_text) is not None + if use_contains: + return pattern in path_text + return pattern == path_text + + def _mock_read_text(self, *args, **kwargs): + del args, kwargs + path_text = str(self) + + for pattern, text in responses.items(): + if _match(pattern, path_text): + return text + + return default_text + + return _mock_read_text + + +def which_return(value: str | None): + """ + Mock shutil.which return value. + """ + mock_obj = Mock() + mock_obj.return_value = value + return mock_obj + + +def which_router( + responses: dict[str, str | None] | None = None, + default: str | None = None, +): + """Route shutil.which(...) results by tool name.""" + responses = responses or {} + + def _mock_which(name: str, *args, **kwargs): + del args, kwargs + return responses.get(name, default) + + return _mock_which + + +def scenario_truthy(value: Any, *, default: bool = False) -> bool: + """Coerce common YAML-ish truthy values used in scenario definitions.""" + if value is None: + return default + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().lower() in { + "1", + "true", + "yes", + "present", + "up", + "pass", + "passed", + "on", + } + return bool(value) + + +def normalize_ethtool_tool_path(tool_name: str, raw_value: Any) -> str | None: + """Normalize a scenario tool entry into a concrete which() return value.""" + if raw_value is None: + return None + + default_path = "/usr/sbin/ethtool" if tool_name == "ethtool" else f"/usr/bin/{tool_name}" + if isinstance(raw_value, bool): + return default_path if raw_value else None + if not isinstance(raw_value, str): + raise TypeError( + f"scenario.tools['{tool_name}'] must be a string or boolean, " + f"got {type(raw_value).__name__}" + ) + + lowered = raw_value.strip().lower() + if lowered in {"present", "yes", "true"}: + return default_path + if lowered in {"absent", "no", "false", ""}: + return None + return raw_value + + +def normalize_ip_entries( + value: Any, + *, + default_dynamic: bool = False, +) -> list[dict[str, Any]]: + """Normalize IPv4/IPv6 scenario entries into a list of address mappings.""" + if value is None: + return [] + + raw_items = [value] if isinstance(value, (str, dict)) else value + if not isinstance(raw_items, list): + raise TypeError("scenario ipv4/ipv6 entries must be a list, string, or mapping") + + entries: list[dict[str, Any]] = [] + for item in raw_items: + if isinstance(item, str): + entries.append({"address": item, "dynamic": default_dynamic}) + continue + if not isinstance(item, dict): + raise TypeError("scenario ipv4/ipv6 entries must be strings or mappings") + address = item.get("address") + if not isinstance(address, str): + raise TypeError("scenario ip entries require a string 'address'") + entries.append( + { + "address": address, + "dynamic": scenario_truthy(item.get("dynamic"), default=default_dynamic), + } + ) + return entries + + +def build_ip_link_flags(kind: str, state: str) -> str: + """Render the flags field used by `ip link` output.""" + flags = ["LOOPBACK"] if kind == "loopback" else ["BROADCAST", "MULTICAST"] + if state.lower() == "up": + flags.append("UP") + return ",".join(flags) + + +def build_ethtool_ip_link_line(index: int, iface: dict[str, Any]) -> str: + """Render one line from `ip -o link` for an ethtool scenario interface.""" + name = iface["name"] + kind = iface.get("kind", "physical") + state = str(iface.get("state", "down")).upper() + mtu = int(iface.get("mtu", 65536 if kind == "loopback" else 1500)) + flags = build_ip_link_flags(kind, state) + + if kind == "loopback": + return ( + f"{index}: {name}: <{flags}> mtu {mtu} qdisc noop " + f"state {state} mode DEFAULT group default qlen 1000" + ) + + mac = iface.get("mac", f"02:00:00:00:00:{index:02x}") + return ( + f"{index}: {name}: <{flags}> mtu {mtu} qdisc noop state {state} " + f"mode DEFAULT group default qlen 1000 link/ether {mac}" + ) + + +def build_ethtool_ip_link_show_line(index: int, iface: dict[str, Any]) -> str: + """Render one line from `ip link show `.""" + name = iface["name"] + kind = iface.get("kind", "physical") + state = str(iface.get("state", "down")).upper() + mtu = int(iface.get("mtu", 65536 if kind == "loopback" else 1500)) + flags = build_ip_link_flags(kind, state) + return f"{index}: {name}: <{flags}> mtu {mtu} state {state}\n" + + +def build_ethtool_ip_address_output(index: int, iface: dict[str, Any]) -> str: + """Render `ip address show dev ` output for a scenario interface.""" + name = iface["name"] + kind = iface.get("kind", "physical") + state = str(iface.get("state", "down")).upper() + mtu = int(iface.get("mtu", 65536 if kind == "loopback" else 1500)) + flags = build_ip_link_flags(kind, state) + + lines = [f"{index}: {name}: <{flags}> mtu {mtu}"] + ipv4_entries = normalize_ip_entries( + iface.get("ipv4") or (iface.get("addresses") or {}).get("ipv4"), + default_dynamic=False, + ) + ipv6_entries = normalize_ip_entries( + iface.get("ipv6") or (iface.get("addresses") or {}).get("ipv6"), + default_dynamic=False, + ) + + for entry in ipv4_entries: + dynamic_text = " dynamic" if entry["dynamic"] else "" + lines.append(f" inet {entry['address']} scope global{dynamic_text} {name}") + for entry in ipv6_entries: + lines.append(f" inet6 {entry['address']} scope global") + return "\n".join(lines) + "\n" + + +def default_device_path(index: int, iface: dict[str, Any]) -> str: + """Build a default resolved sysfs device path for an interface.""" + kind = iface.get("kind", "physical") + name = iface["name"] + if kind == "virtual": + return f"/sys/devices/virtual/net/{name}" + return f"/sys/devices/pci0000:00/0000:00:{index:02x}.0/net/{name}" + + +def route_gateway(route_line: str | None) -> str | None: + """Extract the IPv4 gateway address from a route output line.""" + if not route_line: + return None + match = re.search(r"\bvia\s+(\d{1,3}(?:\.\d{1,3}){3})", route_line) + return match.group(1) if match else None + + +def build_run_result_from_outcome( + outcome: Any, + *, + success_stdout: str = "", + failure_stdout: str = "", + failure_stderr: str = "", +) -> dict[str, Any]: + """Convert a simple pass/fail scenario outcome into run_router response data.""" + if isinstance(outcome, dict): + return { + "returncode": outcome.get("returncode", 0), + "stdout": outcome.get("stdout", ""), + "stderr": outcome.get("stderr", ""), + } + + outcome_text = str(outcome).strip().lower() + if outcome_text in {"pass", "passed", "ok", "success"}: + return {"returncode": 0, "stdout": success_stdout, "stderr": ""} + if outcome_text in {"warn", "warning", "fail", "failed"}: + return { + "returncode": 1, + "stdout": failure_stdout, + "stderr": failure_stderr, + } + raise ValueError(f"Unsupported connectivity outcome: {outcome!r}") + + +def build_df_output(used_blocks: int, available_blocks: int) -> str: + """Render `df -B 512 --output=used,avail` output.""" + return f"Used Avail\n{used_blocks} {available_blocks}\n" + + +def build_fdisk_output(disk: str, partitions: list[dict[str, Any]]) -> str: + """Render minimal `fdisk -l` output that the parser can consume.""" + lines = [ + f"Disk /dev/{disk}: 16 GiB", + "Device Boot Start End Sectors Size Id Type", + ] + + for index, part in enumerate(partitions, start=1): + name = part.get("name", f"{disk}{index}") + boot_flag = "*" if scenario_truthy(part.get("boot"), default=False) else "" + start = int(part.get("start", 2048)) + sectors = int(part.get("sectors", 204800)) + end = int(part.get("end", start + sectors - 1)) + size = str(part.get("size", "100M")) + part_id = str(part.get("id", "83")).upper().removeprefix("0X") + type_name = str(part.get("type_name", "Linux")) + prefix = f"/dev/{name}" + if boot_flag: + lines.append( + f"{prefix} {boot_flag} {start} {end} {sectors} {size} {part_id} {type_name}" + ) + else: + lines.append(f"{prefix} {start} {end} {sectors} {size} {part_id} {type_name}") + + return "\n".join(lines) + "\n" + + +def build_sgdisk_partition_output(partition: dict[str, Any]) -> str: + """Render minimal `sgdisk -i=N` output for one GPT partition.""" + guid = str( + partition.get( + "guid", + "0FC63DAF-8483-4772-8E79-3D69D8477DE4", + ) + ).upper() + guid_name = str(partition.get("guid_name", "Linux filesystem")) + if "attribute_flags" in partition: + flags = str(partition["attribute_flags"]) + else: + flags = "0000000000000001" if scenario_truthy( + partition.get("platform_required"), + default=False, + ) else "0000000000000000" + + return ( + f"Partition GUID code: {guid} ({guid_name})\n" + f"Attribute flags: {flags}\n" + ) + + +def build_efi_var_bytes(attrs: int, payload: bytes = b"") -> bytes: + """Build raw efivarfs contents: 4-byte LE attrs followed by payload bytes.""" + if attrs < 0: + raise ValueError("EFI variable attributes must be non-negative") + return int(attrs).to_bytes(4, byteorder="little", signed=False) + payload + + +def build_char16_payload(value: str, extra_utf16: str = "") -> bytes: + """Encode a UEFI CHAR16 payload in UTF-16LE.""" + return (value + extra_utf16).encode("utf-16le") + + +def build_os_indications_var( + value: int, + *, + attrs: int = 0x07, +) -> bytes: + """Build the OsIndicationsSupported efivarfs payload.""" + if value < 0: + raise ValueError("OsIndicationsSupported value must be non-negative") + payload = int(value).to_bytes(8, byteorder="little", signed=False) + return build_efi_var_bytes(attrs, payload) + + +def _match_rule_value(expected: Any, actual: Any) -> bool: + """Match a rule value against the actual call argument.""" + if isinstance(expected, dict): + if "equals" in expected: + return actual == expected["equals"] + if "contains" in expected: + return str(expected["contains"]) in str(actual) + if "regex" in expected: + return re.search(str(expected["regex"]), str(actual)) is not None + if "in" in expected: + options = expected["in"] + if not isinstance(options, list): + raise TypeError("'in' matcher requires a list") + return actual in options + raise ValueError(f"Unsupported matcher spec: {expected!r}") + + return actual == expected + + +def _rule_matches( + when: dict[str, Any] | None, + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> bool: + """Return True when the rule selector matches the current call.""" + if when is None: + return True + if not isinstance(when, dict): + raise TypeError("'when' must be a mapping") + + arg_rules = when.get("args", {}) + if arg_rules is None: + arg_rules = {} + if not isinstance(arg_rules, dict): + raise TypeError("'when.args' must be a mapping") + + for index, expected in arg_rules.items(): + if not isinstance(index, int): + raise TypeError("'when.args' keys must be integers") + if index >= len(args) or not _match_rule_value(expected, args[index]): + return False + + kwarg_rules = when.get("kwargs", {}) + if kwarg_rules is None: + kwarg_rules = {} + if not isinstance(kwarg_rules, dict): + raise TypeError("'when.kwargs' must be a mapping") + + for key, expected in kwarg_rules.items(): + if key not in kwargs or not _match_rule_value(expected, kwargs[key]): + return False + + return True + + +def _build_exception(spec: Any) -> BaseException: + """Build an exception from a passthrough router rule.""" + if isinstance(spec, BaseException): + return spec + + if isinstance(spec, type) and issubclass(spec, BaseException): + return spec() + + if not isinstance(spec, dict): + raise TypeError( + "Exception rule must be an exception instance, exception class, " + f"or mapping. Got: {type(spec).__name__}" + ) + + exc_type = spec.get("type") + args = spec.get("args", []) + kwargs = spec.get("kwargs", {}) + + if not isinstance(exc_type, type) or not issubclass(exc_type, BaseException): + raise TypeError("'raise.type' must be an exception class") + if not isinstance(args, list): + raise TypeError("'raise.args' must be a list") + if not isinstance(kwargs, dict): + raise TypeError("'raise.kwargs' must be a dict") + + return exc_type(*args, **kwargs) + + +class MockExpectationError(AssertionError): + """Raised when a declarative mock rule was expected to match but did not.""" + + +def _rule_label(rule: dict[str, Any], index: int) -> str: + """Return a stable human-readable label for a passthrough rule.""" + label = rule.get("label") + if isinstance(label, str) and label.strip(): + return label.strip() + return f"rule[{index + 1}]" + + +def _validated_call_count( + value: Any, + *, + field_name: str, + rule_name: str, +) -> int | None: + """Validate an optional per-rule call count constraint.""" + if value is None: + return None + if not isinstance(value, int) or value < 0: + raise TypeError( + f"'{field_name}' for {rule_name} must be a non-negative integer" + ) + return value + + +def passthrough_router( + real: Any, + rules: list[dict[str, Any]] | None = None, + *, + label: str | None = None, + require_any_rule_hit: bool = False, +): + """ + Wrap a real callable and apply declarative YAML-friendly overrides. + + Example rule: + - when: + args: + 0: "/tmp/event.yaml" + 1: + contains: "r" + raise: + type: OSError + args: ["mocked open failure"] + """ + rules = rules or [] + if not isinstance(rules, list): + raise TypeError("'rules' must be a list") + if label is not None and (not isinstance(label, str) or not label.strip()): + raise TypeError("'label' must be a non-empty string when provided") + if not isinstance(require_any_rule_hit, bool): + raise TypeError("'require_any_rule_hit' must be a boolean") + + normalized_rules: list[dict[str, Any]] = [] + for index, rule in enumerate(rules): + if not isinstance(rule, dict): + raise TypeError("Each passthrough rule must be a mapping") + + rule_name = _rule_label(rule, index) + required = rule.get("required", False) + if not isinstance(required, bool): + raise TypeError(f"'required' for {rule_name} must be a boolean") + + exact_calls = _validated_call_count( + rule.get("exact_calls"), + field_name="exact_calls", + rule_name=rule_name, + ) + min_calls = _validated_call_count( + rule.get("min_calls"), + field_name="min_calls", + rule_name=rule_name, + ) + max_calls = _validated_call_count( + rule.get("max_calls"), + field_name="max_calls", + rule_name=rule_name, + ) + if exact_calls is not None and (min_calls is not None or max_calls is not None): + raise TypeError( + f"{rule_name} cannot define 'exact_calls' together with " + "'min_calls' or 'max_calls'" + ) + if ( + min_calls is not None + and max_calls is not None + and min_calls > max_calls + ): + raise TypeError( + f"{rule_name} has invalid call constraints: min_calls > max_calls" + ) + + normalized_rules.append( + { + "raw": rule, + "name": rule_name, + "required": required, + "exact_calls": exact_calls, + "min_calls": min_calls, + "max_calls": max_calls, + "hits": 0, + } + ) + + router_label = label.strip() if isinstance(label, str) else None + + def _wrapped(*args, **kwargs): + for rule_info in normalized_rules: + rule = rule_info["raw"] + if not _rule_matches(rule.get("when"), args, kwargs): + continue + rule_info["hits"] += 1 + + if "raise" in rule: + raise _build_exception(rule["raise"]) + if "return" in rule: + return rule["return"] + if rule.get("call_real", True): + return real(*args, **kwargs) + return None + + return real(*args, **kwargs) + + def _verify() -> None: + failures: list[str] = [] + total_hits = sum(int(rule["hits"]) for rule in normalized_rules) + + if require_any_rule_hit and total_hits == 0: + failures.append("expected at least one passthrough rule to match") + + for rule_info in normalized_rules: + hits = int(rule_info["hits"]) + rule_name = str(rule_info["name"]) + exact_calls = rule_info["exact_calls"] + min_calls = rule_info["min_calls"] + max_calls = rule_info["max_calls"] + + if rule_info["required"] and hits == 0: + failures.append(f"{rule_name} was never hit") + if exact_calls is not None and hits != exact_calls: + failures.append( + f"{rule_name} expected exactly {exact_calls} hit(s), got {hits}" + ) + if min_calls is not None and hits < min_calls: + failures.append( + f"{rule_name} expected at least {min_calls} hit(s), got {hits}" + ) + if max_calls is not None and hits > max_calls: + failures.append( + f"{rule_name} expected at most {max_calls} hit(s), got {hits}" + ) + + if failures: + hit_summary = ", ".join( + f"{rule_info['name']}={rule_info['hits']}" + for rule_info in normalized_rules + ) + prefix = f"{router_label}: " if router_label else "" + if not hit_summary: + hit_summary = "" + raise MockExpectationError( + f"{prefix}{'; '.join(failures)}. Hit counts: {hit_summary}" + ) + + setattr(_wrapped, "_mock_verify", _verify) + return _wrapped diff --git a/common/acs_test_framework_runner/mock_loader.py b/common/acs_test_framework_runner/mock_loader.py new file mode 100644 index 00000000..58f50a70 --- /dev/null +++ b/common/acs_test_framework_runner/mock_loader.py @@ -0,0 +1,749 @@ +from __future__ import annotations + +import importlib +import inspect +import types +from contextlib import ExitStack +from pathlib import Path +from subprocess import CompletedProcess +from typing import Any +from typing import Callable +from typing import Mapping +from unittest.mock import Mock +from unittest.mock import patch + +try: # Support package imports and direct harness module loading. + from .case_data_builders import expand_template as base_expand_template +except ImportError: # pragma: no cover - exercised by flat-module harness imports. + from case_data_builders import expand_template as base_expand_template + + + +def _stateful_read_content(spec: Any, state: dict[str, Any], work_dir: Path) -> bytes: + if isinstance(spec, bytes): + return spec + if isinstance(spec, str): + return spec.encode("utf-8") + if not isinstance(spec, dict): + raise TypeError("Stateful content spec must be bytes, string, or mapping") + if "from_state" in spec: + value = state.get(str(spec["from_state"]), b"") + if isinstance(value, bytes): + return value + if isinstance(value, str): + return value.encode("utf-8") + raise TypeError("State value for content must be bytes or string") + if "from_file" in spec: + file_path = work_dir / str(spec["from_file"]) + return file_path.read_bytes() + if "hex" in spec: + return bytes.fromhex(str(spec["hex"])) + raise ValueError(f"Unsupported stateful content spec: {spec!r}") + + +def _stateful_resolve_value(spec: Any, state: dict[str, Any], work_dir: Path) -> Any: + if isinstance(spec, dict): + if "from_state" in spec: + return state.get(str(spec["from_state"])) + if "from_file" in spec or "hex" in spec: + return _stateful_read_content(spec, state, work_dir) + return spec + + +def stateful_run_router( + responses: dict[str, Any] | None = None, + **options: Any, +): + """Stateful subprocess.run router for scenario-specific command flows.""" + state = dict(options.pop("state", {}) or {}) + rules = list(options.pop("rules", []) or []) + responses = dict(responses or {}) + default_stdout = str(options.pop("default_stdout", "")) + default_stderr = str(options.pop("default_stderr", "")) + default_returncode = int(options.pop("default_returncode", 0)) + unmatched_stderr_template = options.pop("unmatched_stderr_template", None) + if unmatched_stderr_template is not None and not isinstance( + unmatched_stderr_template, + str, + ): + raise TypeError("'unmatched_stderr_template' must be a string") + use_contains = bool(options.pop("use_contains", True)) + call_log_path = options.pop("call_log_path", None) + if call_log_path is not None and not isinstance(call_log_path, str): + raise TypeError("'call_log_path' must be a string") + if options: + unsupported = ", ".join(sorted(options)) + raise TypeError(f"Unsupported stateful_run_router option(s): {unsupported}") + + response_state: dict[str, int] = {} + + def _match(pattern: str, cmd_text: str) -> bool: + if use_contains: + return pattern in cmd_text + return pattern == cmd_text + + def _mock_run(cmd, *args, **kwargs): + del args, kwargs + cmd_text = " ".join(cmd) if isinstance(cmd, (list, tuple)) else str(cmd) + + if call_log_path is not None: + log_path = Path(call_log_path) + log_path.parent.mkdir(parents=True, exist_ok=True) + with log_path.open("a", encoding="utf-8") as handle: + handle.write(f"{cmd_text}\n") + + for rule in rules: + if not isinstance(rule, dict): + raise TypeError("Each stateful_run_router rule must be a mapping") + pattern = rule.get("command") + if not isinstance(pattern, str): + raise TypeError("stateful_run_router rule requires string 'command'") + if not _match(pattern, cmd_text): + continue + + for key, value_spec in (rule.get("set_state") or {}).items(): + state[str(key)] = _stateful_resolve_value(value_spec, state, Path.cwd()) + + write_file = rule.get("write_file") + if write_file is not None: + if ( + not isinstance(write_file, dict) + or "path" not in write_file + or "content" not in write_file + ): + raise TypeError("write_file must be a mapping with 'path' and 'content'") + target = Path.cwd() / str(write_file["path"]) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes( + _stateful_read_content( + write_file["content"], + state, + Path.cwd(), + ) + ) + + if "raise" in rule: + raise build_exception_from_spec(rule["raise"]) + + result = rule.get("result", {}) + if not isinstance(result, dict): + raise TypeError("stateful_run_router rule 'result' must be a mapping") + return CompletedProcess( + args=cmd, + returncode=int(result.get("returncode", 0)), + stdout=str(result.get("stdout", "")), + stderr=str(result.get("stderr", "")), + ) + + for pattern, result in responses.items(): + if not _match(pattern, cmd_text): + continue + resolved = result + if isinstance(result, list): + if not result: + raise TypeError("stateful_run_router response lists must not be empty") + index = response_state.get(pattern, 0) + response_state[pattern] = index + 1 + resolved = result[index] if index < len(result) else result[-1] + if not isinstance(resolved, dict): + raise TypeError("stateful_run_router responses must resolve to mappings") + return CompletedProcess( + args=cmd, + returncode=int(resolved.get("returncode", 0)), + stdout=str(resolved.get("stdout", "")), + stderr=str(resolved.get("stderr", "")), + ) + + stderr_text = default_stderr + if unmatched_stderr_template is not None: + stderr_text = unmatched_stderr_template.format(cmd=cmd_text) + + return CompletedProcess( + args=cmd, + returncode=default_returncode, + stdout=default_stdout, + stderr=stderr_text, + ) + + return _mock_run + + +class ConfigError(Exception): + """Raised when a YAML configuration is invalid.""" + + +def load_dotted_object(dotted_path: str) -> Any: + """Load object from dotted path like 'pkg.mod.func' or 'pkg.mod.obj.attr'.""" + if not isinstance(dotted_path, str) or "." not in dotted_path: + raise ValueError(f"Invalid dotted path: {dotted_path!r}") + + parts = dotted_path.split(".") + last_error: Exception | None = None + + for index in range(len(parts) - 1, 0, -1): + module_name = ".".join(parts[:index]) + attr_parts = parts[index:] + + try: + obj = importlib.import_module(module_name) + except ImportError as exc: + last_error = exc + continue + + try: + for attr_name in attr_parts: + obj = getattr(obj, attr_name) + return obj + except AttributeError as exc: + raise AttributeError(f"Invalid dotted path: {dotted_path!r}") from exc + + raise ValueError(f"Invalid dotted path: {dotted_path!r}") from last_error + + +def resolve_special_value(value: Any) -> Any: + """ + Resolve special YAML values. + + Supported: + - 'py:package.module.object' -> imported Python object + """ + if isinstance(value, str) and value.startswith("py:"): + return load_dotted_object(value[3:]) + if isinstance(value, list): + return [resolve_special_value(item) for item in value] + if isinstance(value, tuple): + return tuple(resolve_special_value(item) for item in value) + if isinstance(value, dict): + return { + key: resolve_special_value(item) + for key, item in value.items() + } + return value + + +def build_exception_from_spec(spec: Any) -> BaseException: + """ + Build an exception instance from YAML spec. + + Supported forms: + - py:builtins.OSError + - {type: py:builtins.OSError, args: ["message"]} + - an already-created exception instance + """ + resolved = resolve_special_value(spec) + + if isinstance(resolved, BaseException): + return resolved + + if isinstance(resolved, type) and issubclass(resolved, BaseException): + return resolved() + + if not isinstance(resolved, dict): + raise TypeError( + "Exception spec must resolve to an exception instance, exception " + f"class, or mapping. Got: {type(resolved).__name__}" + ) + + exc_type = resolved.get("type") + if not isinstance(exc_type, type) or not issubclass(exc_type, BaseException): + raise TypeError( + "Exception spec 'type' must resolve to an exception class, " + f"got: {exc_type!r}" + ) + + args = resolved.get("args", []) + kwargs = resolved.get("kwargs", {}) + + if not isinstance(args, list): + raise TypeError( + f"Exception spec 'args' must be a list, got: {type(args).__name__}" + ) + if not isinstance(kwargs, dict): + raise TypeError( + f"Exception spec 'kwargs' must be a dict, got: {type(kwargs).__name__}" + ) + + return exc_type(*args, **kwargs) + + +def resolve_side_effect_value(value: Any) -> Any: + """Resolve side-effect values, including exception descriptors.""" + if isinstance(value, list): + return [resolve_side_effect_value(item) for item in value] + + if isinstance(value, dict) and set(value.keys()) == {"exception"}: + return build_exception_from_spec(value["exception"]) + + return resolve_special_value(value) + + +def set_nested_attr(obj: Any, attr_path: str, value: Any) -> None: + """ + Set nested attribute path like: + return_value.stdout + return_value.returncode + """ + parts = attr_path.split(".") + current = obj + + for part in parts[:-1]: + current = getattr(current, part) + + setattr(current, parts[-1], value) + + +def build_mock_from_spec(target: str, spec: Any) -> Any: + """ + Build replacement object from YAML spec. + + Supported forms: + + 1) String shorthand: + mocks: + shutil.which: unittest.mock.Mock + + 2) Dict form: + mocks: + subprocess.run: + factory: unittest.mock.Mock + kwargs: + name: run_mock + attrs: + return_value.stdout: "hello" + return_value.returncode: 0 + + 3) Implicit Mock with direct behavior: + mocks: + yaml.safe_load: + side_effect: + - {"sha256": ["a0", "a1"]} + - exception: + type: py:yaml.YAMLError + args: ["mocked parse failure"] + + 4) Factory that needs the original callable: + mocks: + builtins.open: + factory: mock_helpers.passthrough_router + inject_original_as: real + kwargs: + rules: [...] + + 5) Direct callable factory (used by generated scenarios): + mocks: + time.sleep: + value: + """ + if isinstance(spec, str): + factory = load_dotted_object(spec) + return factory() + + if not isinstance(spec, dict): + raise TypeError(f"Unsupported mock spec type: {type(spec).__name__}") + + factory_ref = spec.get("factory") + if ( + factory_ref is not None + and not isinstance(factory_ref, str) + and not callable(factory_ref) + ): + raise TypeError( + "'factory' must be a string or callable when provided, " + f"got: {type(factory_ref).__name__}" + ) + + if "value" in spec: + return resolve_special_value(spec["value"]) + + original = load_dotted_object(target) + + raw_args = spec.get("args", []) + if raw_args is None: + raw_args = [] + if not isinstance(raw_args, list): + raise TypeError(f"'args' must be a list, got: {type(raw_args).__name__}") + + raw_kwargs = spec.get("kwargs", {}) + if raw_kwargs is None: + raw_kwargs = {} + if not isinstance(raw_kwargs, dict): + raise TypeError(f"'kwargs' must be a dict, got: {type(raw_kwargs).__name__}") + + args = [resolve_special_value(value) for value in raw_args] + kwargs = {key: resolve_special_value(value) for key, value in raw_kwargs.items()} + + inject_original_as = spec.get("inject_original_as") + if inject_original_as is not None: + if not isinstance(inject_original_as, str) or not inject_original_as.strip(): + raise TypeError("'inject_original_as' must be a non-empty string") + kwargs[inject_original_as] = original + + if spec.get("wraps_original", False): + kwargs["wraps"] = original + + if factory_ref: + if isinstance(factory_ref, str): + factory = load_dotted_object(factory_ref) + else: + factory = factory_ref + else: + factory = Mock + + obj = factory(*args, **kwargs) + + raw_attrs = spec.get("attrs", {}) + if raw_attrs is None: + raw_attrs = {} + + if not isinstance(raw_attrs, dict): + raise TypeError(f"'attrs' must be a dict, got: {type(raw_attrs).__name__}") + + if "return_value" in spec: + obj.return_value = resolve_special_value(spec["return_value"]) + + if "side_effect" in spec: + obj.side_effect = resolve_side_effect_value(spec["side_effect"]) + + for attr_name, raw_value in raw_attrs.items(): + value = resolve_special_value(raw_value) + set_nested_attr(obj, attr_name, value) + + return obj + + +def expand_patch_target( + target: str, + target_context: Mapping[str, str] | None = None, +) -> str: + """Expand supported placeholders such as '{module}' in patch targets.""" + expanded = target + if not target_context: + return expanded + + for key, value in target_context.items(): + expanded = expanded.replace(f"{{{key}}}", str(value)) + return expanded + + +def apply_case_mocks( + mock_map: dict[str, Any] | None, + *, + target_context: Mapping[str, str] | None = None, +) -> ExitStack: + """ + Apply case mocks and return an ExitStack. + + Caller should use: + with apply_case_mocks(case.get("mocks")): + ... + """ + stack = ExitStack() + + if not mock_map: + return stack + + if not isinstance(mock_map, dict): + raise TypeError(f"'mocks' must be a dict, got: {type(mock_map).__name__}") + + for target, spec in mock_map.items(): + if not isinstance(target, str): + raise TypeError( + f"Mock target must be a string, got: {type(target).__name__}" + ) + expanded_target = expand_patch_target(target, target_context) + replacement = build_mock_from_spec(expanded_target, spec) + verifier = getattr(replacement, "_mock_verify", None) + if callable(verifier): + stack.callback(verifier) + stack.enter_context(patch(expanded_target, replacement)) + + return stack + + +def expand_template( + value: str, + work_dir: Path, + file_path: Path, + extra_tokens: dict[str, str] | None = None, +) -> str: + """Expand common runtime tokens such as {dir}, {file}, and optional extras.""" + expanded = base_expand_template(value, work_dir, file_path) + if not extra_tokens: + return expanded + + for key, item in extra_tokens.items(): + expanded = expanded.replace(f"{{{key}}}", str(item)) + return expanded + + +def expand_case_value( + value: Any, + work_dir: Path, + file_path: Path, + extra_tokens: dict[str, str] | None = None, +) -> Any: + """Recursively expand runtime tokens inside case values.""" + if isinstance(value, str): + return expand_template(value, work_dir, file_path, extra_tokens) + if isinstance(value, list): + return [expand_case_value(item, work_dir, file_path, extra_tokens) for item in value] + if isinstance(value, tuple): + return tuple( + expand_case_value(item, work_dir, file_path, extra_tokens) + for item in value + ) + if isinstance(value, dict): + return { + key: expand_case_value(item, work_dir, file_path, extra_tokens) + for key, item in value.items() + } + return value + + +def expand_case_mapping( + mapping: dict[str, Any] | None, + work_dir: Path, + file_path: Path, + extra_tokens: dict[str, str] | None = None, +) -> dict[str, Any]: + """Expand tokens across a case mapping, including its keys.""" + if not mapping: + return {} + + expanded: dict[str, Any] = {} + for key, value in mapping.items(): + expanded_key = ( + expand_template(key, work_dir, file_path, extra_tokens) + if isinstance(key, str) + else key + ) + expanded[expanded_key] = expand_case_value( + value, + work_dir, + file_path, + extra_tokens, + ) + return expanded + + +def merge_case_definitions( + generated_case: dict[str, Any], + case_def: dict[str, Any], +) -> dict[str, Any]: + """Merge generated scenario data with explicit user-specified case fields.""" + merged = dict(generated_case) + merged.update(case_def) + + for field_name in ( + "scripts", + "bin_files", + "text_files", + "patch_constants", + "mocks", + "kwargs", + ): + generated = generated_case.get(field_name) or {} + explicit = case_def.get(field_name) or {} + if generated or explicit: + merged[field_name] = {**generated, **explicit} + + generated_dirs = list(generated_case.get("dir_structure") or []) + explicit_dirs = list(case_def.get("dir_structure") or []) + if generated_dirs or explicit_dirs: + merged["dir_structure"] = [*generated_dirs, *explicit_dirs] + + if "args" not in case_def and "args" in generated_case: + merged["args"] = list(generated_case["args"]) + + return merged + + +def _resolve_runner_module_from_stack() -> types.ModuleType: + """ + Resolve the target module from the runner call stack. + + module_main_with_env currently calls ``module.main()`` directly. For scripts + that only expose a main guard, scenario builders can inject a synthetic + ``main`` callable via patch_constants, and that callable recovers the target + module from the runner frame. + """ + frame = inspect.currentframe() + current = frame.f_back if frame is not None else None + try: + while current is not None: + candidate = current.f_locals.get("module") + if isinstance(candidate, types.ModuleType): + return candidate + current = current.f_back + finally: + del frame + del current + + raise RuntimeError("Unable to resolve target module from runner stack") + + +# Keep scenario imports below the shared helpers so the split builder modules can +# import shared functionality from this file without circular-import breakage. +try: # Support package imports and direct harness module loading. + from .mock_loader_hardware_scenarios import build_blk_devices_scenario_case + from .mock_loader_hardware_scenarios import build_blk_write_check_scenario_case + from .mock_loader_hardware_scenarios import build_capsule_vars_scenario_case + from .mock_loader_hardware_scenarios import build_ethtool_scenario_case + from .mock_loader_hardware_scenarios import build_runtime_device_mapping_scenario_case + from .mock_loader_hardware_scenarios import build_verify_tpm_events + from .mock_loader_hardware_scenarios import build_verify_tpm_scenario_case + from .mock_loader_parser_scenarios import build_acs_info_scenario_case + from .mock_loader_parser_scenarios import build_extract_capsule_fw_version_scenario_case + from .mock_loader_parser_scenarios import build_merge_jsons_scenario_case +except ImportError: # pragma: no cover - exercised by flat-module harness imports. + from mock_loader_hardware_scenarios import build_blk_devices_scenario_case + from mock_loader_hardware_scenarios import build_blk_write_check_scenario_case + from mock_loader_hardware_scenarios import build_capsule_vars_scenario_case + from mock_loader_hardware_scenarios import build_ethtool_scenario_case + from mock_loader_hardware_scenarios import build_runtime_device_mapping_scenario_case + from mock_loader_hardware_scenarios import build_verify_tpm_events + from mock_loader_hardware_scenarios import build_verify_tpm_scenario_case + from mock_loader_parser_scenarios import build_acs_info_scenario_case + from mock_loader_parser_scenarios import build_extract_capsule_fw_version_scenario_case + from mock_loader_parser_scenarios import build_merge_jsons_scenario_case + + +SCENARIO_CASE_BUILDERS: dict[ + str, + Callable[[dict[str, Any], Path], dict[str, Any]], +] = { + "ethtool": build_ethtool_scenario_case, + "capsule_vars": build_capsule_vars_scenario_case, + "verify_tpm": build_verify_tpm_scenario_case, + "acs_info": build_acs_info_scenario_case, + "merge_jsons": build_merge_jsons_scenario_case, + "blk_devices": build_blk_devices_scenario_case, + "blk_write_check": build_blk_write_check_scenario_case, + "runtime_device_mapping": build_runtime_device_mapping_scenario_case, +} + + +def build_generated_case_definition( + case_def: dict[str, Any], + work_dir: Path, + file_path: Path, + extra_tokens: dict[str, str] | None = None, +) -> dict[str, Any]: + """Build an implicit/generated case definition from scenario input.""" + scenario = expand_case_value( + case_def.get("scenario"), + work_dir, + file_path, + extra_tokens, + ) + if not scenario: + return {} + if not isinstance(scenario, dict): + raise ConfigError("'scenario' must resolve to a mapping") + + kind = scenario.get("kind") + if kind is None and file_path.name == "ethtool-test.py": + kind = "ethtool" + builder = SCENARIO_CASE_BUILDERS.get(str(kind)) + if builder is None: + raise ConfigError(f"Unsupported scenario kind: {kind!r}") + return builder(scenario, work_dir) + + +def build_case_runtime_definition( + case_def: dict[str, Any], + work_dir: Path, + file_path: Path, + extra_tokens: dict[str, str] | None = None, +) -> dict[str, Any]: + """Build the final runtime case after scenario generation and token expansion.""" + generated_case = build_generated_case_definition( + case_def, + work_dir, + file_path, + extra_tokens, + ) + merged_case = merge_case_definitions(generated_case, case_def) + + runtime_case = dict(merged_case) + runtime_case["scripts"] = expand_case_mapping( + merged_case.get("scripts", {}), + work_dir, + file_path, + extra_tokens, + ) + runtime_case["bin_files"] = expand_case_mapping( + merged_case.get("bin_files", {}), + work_dir, + file_path, + extra_tokens, + ) + runtime_case["text_files"] = expand_case_mapping( + merged_case.get("text_files", {}), + work_dir, + file_path, + extra_tokens, + ) + runtime_case["dir_structure"] = expand_case_value( + merged_case.get("dir_structure", []), + work_dir, + file_path, + extra_tokens, + ) + runtime_case["patch_constants"] = expand_case_value( + merged_case.get("patch_constants", {}), + work_dir, + file_path, + extra_tokens, + ) + runtime_case["mocks"] = expand_case_mapping( + merged_case.get("mocks", {}), + work_dir, + file_path, + extra_tokens, + ) + runtime_case["args"] = expand_case_value( + merged_case.get("args", []), + work_dir, + file_path, + extra_tokens, + ) + runtime_case["kwargs"] = expand_case_value( + merged_case.get("kwargs", {}), + work_dir, + file_path, + extra_tokens, + ) + return runtime_case + + +__all__ = [ + "ConfigError", + "SCENARIO_CASE_BUILDERS", + "_resolve_runner_module_from_stack", + "apply_case_mocks", + "build_acs_info_scenario_case", + "build_blk_devices_scenario_case", + "build_blk_write_check_scenario_case", + "build_capsule_vars_scenario_case", + "build_case_runtime_definition", + "build_ethtool_scenario_case", + "build_exception_from_spec", + "build_extract_capsule_fw_version_scenario_case", + "build_generated_case_definition", + "build_merge_jsons_scenario_case", + "build_mock_from_spec", + "build_runtime_device_mapping_scenario_case", + "build_verify_tpm_events", + "build_verify_tpm_scenario_case", + "expand_case_mapping", + "expand_case_value", + "expand_patch_target", + "expand_template", + "load_dotted_object", + "merge_case_definitions", + "resolve_side_effect_value", + "resolve_special_value", + "set_nested_attr", + "stateful_run_router", +] diff --git a/common/acs_test_framework_runner/mock_loader_hardware_scenarios.py b/common/acs_test_framework_runner/mock_loader_hardware_scenarios.py new file mode 100644 index 00000000..b217a937 --- /dev/null +++ b/common/acs_test_framework_runner/mock_loader_hardware_scenarios.py @@ -0,0 +1,1757 @@ +from __future__ import annotations + +import json +from pathlib import Path +from subprocess import CalledProcessError +from typing import Any +from typing import Mapping + +try: # Support package imports and direct harness module loading. + from .mock_loader import ConfigError + from .mock_loader import _resolve_runner_module_from_stack + from .mock_loader import stateful_run_router + from .mock_helpers import build_char16_payload + from .mock_helpers import build_df_output + from .mock_helpers import build_efi_var_bytes + from .mock_helpers import build_ethtool_ip_address_output + from .mock_helpers import build_ethtool_ip_link_line + from .mock_helpers import build_ethtool_ip_link_show_line + from .mock_helpers import build_fdisk_output + from .mock_helpers import build_os_indications_var + from .mock_helpers import build_run_result_from_outcome + from .mock_helpers import build_sgdisk_partition_output + from .mock_helpers import check_output_router + from .mock_helpers import default_device_path + from .mock_helpers import noop + from .mock_helpers import normalize_ethtool_tool_path + from .mock_helpers import passthrough_router + from .mock_helpers import route_gateway + from .mock_helpers import run_router + from .mock_helpers import scenario_truthy + from .mock_helpers import which_router +except ImportError: # pragma: no cover - exercised by flat-module harness imports. + from mock_loader import ConfigError + from mock_loader import _resolve_runner_module_from_stack + from mock_loader import stateful_run_router + from mock_helpers import build_char16_payload + from mock_helpers import build_df_output + from mock_helpers import build_efi_var_bytes + from mock_helpers import build_ethtool_ip_address_output + from mock_helpers import build_ethtool_ip_link_line + from mock_helpers import build_ethtool_ip_link_show_line + from mock_helpers import build_fdisk_output + from mock_helpers import build_os_indications_var + from mock_helpers import build_run_result_from_outcome + from mock_helpers import build_sgdisk_partition_output + from mock_helpers import check_output_router + from mock_helpers import default_device_path + from mock_helpers import noop + from mock_helpers import normalize_ethtool_tool_path + from mock_helpers import passthrough_router + from mock_helpers import route_gateway + from mock_helpers import run_router + from mock_helpers import scenario_truthy + from mock_helpers import which_router + + +def build_ethtool_scenario_case( + scenario: dict[str, Any], + work_dir: Path, +) -> dict[str, Any]: + """Build runtime files, args, and mocks for an ethtool scenario.""" + interfaces = scenario.get("interfaces", []) + if interfaces is None: + interfaces = [] + if not isinstance(interfaces, list): + raise ConfigError("scenario.interfaces must be a list") + + tool_paths = { + name: normalize_ethtool_tool_path(name, raw_value) + for name, raw_value in (scenario.get("tools") or {}).items() + } + + ip_link_lines = [ + build_ethtool_ip_link_line(index, iface) + for index, iface in enumerate(interfaces, start=1) + ] + run_responses: dict[str, Any] = { + "ip route show default": {"returncode": 0, "stdout": "", "stderr": ""}, + "ip -o route show table all default": {"returncode": 0, "stdout": "", "stderr": ""}, + } + read_text_rules: list[dict[str, Any]] = [] + resolve_rules: list[dict[str, Any]] = [] + default_route_lines: list[str] = [] + route_get_output = scenario.get("route_get") + + for index, iface in enumerate(interfaces, start=1): + name = iface["name"] + kind = str(iface.get("kind", "physical")) + + run_responses[f"ip link show {name}"] = { + "returncode": 0, + "stdout": build_ethtool_ip_link_show_line(index, iface), + "stderr": "", + } + run_responses[f"ip link set dev {name} down"] = { + "returncode": 0, + "stdout": "", + "stderr": "", + } + run_responses[f"ip link set dev {name} up"] = { + "returncode": 0, + "stdout": "", + "stderr": "", + } + run_responses[f"ip address show dev {name}"] = { + "returncode": 0, + "stdout": build_ethtool_ip_address_output(index, iface), + "stderr": "", + } + + if tool_paths.get("dhclient") is not None: + run_responses[f"dhclient -r {name}"] = { + "returncode": 0, + "stdout": "", + "stderr": "", + } + run_responses[f"dhclient -1 {name}"] = { + "returncode": 0, + "stdout": "", + "stderr": "", + } + if tool_paths.get("udhcpc") is not None: + run_responses[f"udhcpc -n -q -i {name}"] = { + "returncode": 0, + "stdout": "", + "stderr": "", + } + + if kind != "loopback": + device_path = str(iface.get("device") or default_device_path(index, iface)) + resolve_rules.append( + { + "when": {"args": {0: {"contains": f"/sys/class/net/{name}/device"}}}, + "return": device_path, + } + ) + read_text_rules.append( + { + "when": {"args": {0: {"contains": f"/sys/class/net/{name}/carrier"}}}, + "return": str(iface.get("carrier", "1" if kind == "physical" else "0")), + } + ) + read_text_rules.append( + { + "when": {"args": {0: {"contains": f"/sys/class/net/{name}/operstate"}}}, + "return": str(iface.get("operstate", "up" if kind == "physical" else "down")), + } + ) + + if tool_paths.get("ethtool") is not None and kind != "loopback": + ethtool_cfg = iface.get("ethtool") or {} + link_detected = scenario_truthy( + ethtool_cfg.get("link_detected"), + default=str(iface.get("carrier", "0")) == "1", + ) + self_test_supported = scenario_truthy( + ethtool_cfg.get("self_test_supported"), + default=False, + ) + run_responses[f"ethtool {name}"] = { + "returncode": 0, + "stdout": ( + f"Settings for {name}\n" + f"\tLink detected: {'yes' if link_detected else 'no'}\n" + ), + "stderr": "", + } + run_responses[f"ethtool -i {name}"] = { + "returncode": 0, + "stdout": ( + "driver: fake\n" + f"supports-test: {'yes' if self_test_supported else 'no'}\n" + ), + "stderr": "", + } + if self_test_supported or "self_test" in ethtool_cfg: + self_test = ethtool_cfg.get("self_test", "pass") + run_responses[f"ethtool -t {name}"] = build_run_result_from_outcome( + self_test, + success_stdout="The test result is PASS\n", + failure_stdout="self-test failed\n", + ) + + routes = iface.get("routes") or {} + default_route = routes.get("default") or iface.get("default_route") + if default_route: + default_route_lines.append(str(default_route)) + if route_get_output is None: + route_get_output = routes.get("route_get") or iface.get("route_get") + + connectivity = iface.get("connectivity") or {} + gateway = route_gateway(str(default_route)) if default_route else None + if gateway and "gateway_ping" in connectivity: + run_responses[ + f"ping -c 3 -W 10 -I {name} {gateway}" + ] = build_run_result_from_outcome( + connectivity["gateway_ping"], + success_stdout="3 packets transmitted, 3 received, 0% packet loss\n", + failure_stdout="3 packets transmitted, 0 received, 100% packet loss\n", + ) + if "arm_ping" in connectivity: + run_responses[ + f"ping -c 3 -W 10 -I {name} www.arm.com" + ] = build_run_result_from_outcome( + connectivity["arm_ping"], + success_stdout="3 packets transmitted, 3 received, 0% packet loss\n", + failure_stdout="3 packets transmitted, 0 received, 100% packet loss\n", + ) + if "ipv6_ping" in connectivity: + ipv6_result = build_run_result_from_outcome( + connectivity["ipv6_ping"], + success_stdout="3 packets transmitted, 3 received, 0% packet loss\n", + failure_stdout="3 packets transmitted, 0 received, 100% packet loss\n", + ) + run_responses[f"ping -6 -c 3 -I {name} ipv6.google.com"] = ipv6_result + run_responses[f"ping6 -c 3 -I {name} ipv6.google.com"] = ipv6_result + if "wget" in connectivity: + run_responses[ + "wget --spider --timeout=10 https://www.arm.com" + ] = build_run_result_from_outcome( + connectivity["wget"], + success_stdout="", + failure_stderr="network unreachable\n", + ) + if "curl" in connectivity: + run_responses[ + f"curl -Is --connect-timeout 20 --interface {name} https://www.arm.com" + ] = build_run_result_from_outcome( + connectivity["curl"], + success_stdout="HTTP/2 200\n", + failure_stderr="connect failed\n", + ) + + if default_route_lines: + routes_stdout = "\n".join(default_route_lines) + "\n" + run_responses["ip route show default"] = { + "returncode": 0, + "stdout": routes_stdout, + "stderr": "", + } + run_responses["ip -o route show table all default"] = { + "returncode": 0, + "stdout": routes_stdout, + "stderr": "", + } + + if route_get_output: + route_text = str(route_get_output) + if not route_text.endswith("\n"): + route_text += "\n" + run_responses["ip route get 8.8.8.8"] = { + "returncode": 0, + "stdout": route_text, + "stderr": "", + } + + required = scenario.get("required_compliant_interfaces") + if required is None: + required = sum( + 1 + for iface in interfaces + if str(iface.get("kind", "physical")) == "physical" + ) + + system_config_path = work_dir / "system_config.txt" + return { + "args": [str(system_config_path)], + "text_files": { + "system_config.txt": f"total_number_of_network_controllers: {required}\n", + }, + "mocks": { + "signal.signal": {"value": noop}, + "time.sleep": {"value": noop}, + "shutil.which": { + "factory": which_router, + "kwargs": {"responses": tool_paths, "default": None}, + }, + "subprocess.check_output": { + "factory": check_output_router, + "kwargs": { + "responses": { + "ip -o link": "\n".join(ip_link_lines) + ("\n" if ip_link_lines else "") + }, + "use_contains": False, + }, + }, + "subprocess.run": { + "factory": run_router, + "kwargs": { + "responses": run_responses, + "default_returncode": 1, + "use_contains": False, + "unmatched_stderr_template": "UNMOCKED COMMAND: {cmd}\n", + }, + }, + "pathlib.Path.read_text": { + "factory": passthrough_router, + "inject_original_as": "real", + "kwargs": {"rules": read_text_rules}, + }, + "pathlib.Path.resolve": { + "factory": passthrough_router, + "inject_original_as": "real", + "kwargs": {"rules": resolve_rules}, + }, + }, + } + + +PRECIOUS_MBR_IDS = {"0xF8", "0xEF"} +PRECIOUS_GPT_GUIDS = { + "C12A7328-F81F-11D2-BA4B-00A0C93EC93B", + "21686148-6449-6E6F-744E-656564454649", + "3DE21764-95BD-54BD-A5C3-4ABE786F38A8", +} + + +def _completed_process_spec( + *, + returncode: int = 0, + stdout: str = "", + stderr: str = "", +) -> dict[str, Any]: + return {"returncode": returncode, "stdout": stdout, "stderr": stderr} + + +def _join_stdout_lines(items: list[str]) -> str: + if not items: + return "" + return "\n".join(items) + "\n" + + +def _normalize_mbr_partition_id(value: Any) -> str: + text = str(value).strip().upper() + if not text: + return "" + if text.startswith("0X"): + text = text[2:] + return f"0x{text}" + + +def _normalize_prompt_response(value: Any, *, default: str = "no") -> str: + """Normalize YAML prompt values into explicit yes/no/timeout responses.""" + if value is None: + text = "" + elif isinstance(value, bool): + text = "yes" if value else "no" + else: + text = str(value).strip().lower() + + if not text: + return default + + normalized = { + "timeout": "timeout", + "1": "yes", + "true": "yes", + "yes": "yes", + "y": "yes", + "on": "yes", + "0": "no", + "false": "no", + "no": "no", + "n": "no", + "off": "no", + } + return normalized.get(text, text) + + +def _platform_required(partition: dict[str, Any]) -> bool: + if "attribute_flags" in partition: + flags = str(partition["attribute_flags"]).strip() + try: + return (int(flags, 16) & 1) == 1 + except ValueError as exc: + raise ConfigError( + f"Invalid GPT attribute_flags value: {partition['attribute_flags']!r}" + ) from exc + return scenario_truthy(partition.get("platform_required"), default=False) + + +def _add_blk_write_check_run_responses( + run_responses: dict[str, Any], + target: str, + write_cfg: dict[str, Any] | None, +) -> None: + cfg = write_cfg or {} + mounted = scenario_truthy(cfg.get("mounted"), default=False) + run_responses[f"findmnt -n /dev/{target}"] = _completed_process_spec( + returncode=0 if mounted else 1, + stdout=f"/dev/{target} /mnt/{target} ext4 rw,relatime 0 0\n" if mounted else "", + ) + + if mounted: + return + + if "used_blocks" not in cfg and "available_blocks" not in cfg: + return + + used_blocks = int(cfg.get("used_blocks", 0)) + available_blocks = int(cfg.get("available_blocks", 0)) + run_responses[f"df -B 512 /dev/{target} --output=used,avail"] = _completed_process_spec( + stdout=build_df_output(used_blocks, available_blocks) + ) + + +def _build_blk_write_check_stateful_rules( + device: str, + cfg: dict[str, Any], +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + write_flow = cfg.get("write_flow") + if write_flow is None: + return {}, [] + if not isinstance(write_flow, dict): + raise ConfigError("blk_write_check.write_flow must be a mapping when provided") + + initial_bytes = cfg.get("initial_device_bytes") + if initial_bytes is None: + initial_text = str(cfg.get("initial_device_text", "B" * 512)) + initial_bytes = initial_text.encode("utf-8") + elif isinstance(initial_bytes, str): + try: + initial_bytes = bytes.fromhex(initial_bytes) + except ValueError as exc: + raise ConfigError( + "blk_write_check.initial_device_bytes must be bytes or hex string" + ) from exc + elif not isinstance(initial_bytes, bytes): + raise ConfigError("blk_write_check.initial_device_bytes must be bytes or hex string") + + mismatch_bytes = write_flow.get("mismatch_bytes") + if mismatch_bytes is None: + mismatch_bytes = b"X" * max(len(initial_bytes), 512) + elif isinstance(mismatch_bytes, str): + try: + mismatch_bytes = bytes.fromhex(mismatch_bytes) + except ValueError as exc: + raise ConfigError( + "blk_write_check.write_flow.mismatch_bytes must be bytes or hex string" + ) from exc + elif not isinstance(mismatch_bytes, bytes): + raise ConfigError("blk_write_check.write_flow.mismatch_bytes must be bytes or hex string") + + used_blocks = int(cfg.get("used_blocks", 0)) + backup_name = f"{device}_backup.bin" + device_state_key = f"{device}:device_bytes" + mismatch_state_key = f"{device}:mismatch_bytes" + state = { + device_state_key: initial_bytes, + mismatch_state_key: mismatch_bytes, + } + rules: list[dict[str, Any]] = [] + + backup_command = ( + f"dd if=/dev/{device} of={backup_name} bs=512 count=1 skip={used_blocks}" + ) + backup_mode = str(write_flow.get("backup", "success")).lower() + if backup_mode in {"success", "pass", "ok"}: + rules.append({ + "command": backup_command, + "write_file": { + "path": backup_name, + "content": {"from_state": device_state_key}, + }, + "result": {"returncode": 0, "stdout": "", "stderr": ""}, + }) + elif backup_mode == "error": + error_text = str(write_flow.get("backup_error", "mocked backup failure")) + rules.append({ + "command": backup_command, + "raise": { + "type": CalledProcessError, + "args": [1, backup_command], + "kwargs": {"output": "", "stderr": error_text}, + }, + }) + elif backup_mode not in {"skip", "none"}: + raise ConfigError( + "blk_write_check.write_flow.backup must be success, error, skip, or none" + ) + + write_command = ( + f"dd if=hello.txt of=/dev/{device} bs=512 count=1 seek={used_blocks}" + ) + write_mode = str(write_flow.get("write", "success")).lower() + if write_mode in {"success", "pass", "ok"}: + written_content = ( + {"from_state": mismatch_state_key} + if str(write_flow.get("readback", "match")).lower() == "mismatch" + else {"from_file": "hello.txt"} + ) + rules.append({ + "command": write_command, + "set_state": {device_state_key: written_content}, + "result": {"returncode": 0, "stdout": "", "stderr": ""}, + }) + elif write_mode == "error": + error_text = str(write_flow.get("write_error", "mocked write failure")) + rules.append({ + "command": write_command, + "raise": { + "type": CalledProcessError, + "args": [1, write_command], + "kwargs": {"output": "", "stderr": error_text}, + }, + }) + elif write_mode not in {"skip", "none"}: + raise ConfigError( + "blk_write_check.write_flow.write must be success, error, skip, or none" + ) + + read_command = ( + f"dd if=/dev/{device} of=read_hello.txt bs=512 count=1 skip={used_blocks}" + ) + readback_mode = str(write_flow.get("readback", "match")).lower() + if readback_mode in {"match", "mismatch"}: + rules.append({ + "command": read_command, + "write_file": { + "path": "read_hello.txt", + "content": {"from_state": device_state_key}, + }, + "result": {"returncode": 0, "stdout": "", "stderr": ""}, + }) + elif readback_mode == "error": + error_text = str(write_flow.get("readback_error", "mocked readback failure")) + rules.append({ + "command": read_command, + "raise": { + "type": CalledProcessError, + "args": [1, read_command], + "kwargs": {"output": "", "stderr": error_text}, + }, + }) + else: + raise ConfigError("blk_write_check.write_flow.readback must be match, mismatch, or error") + + restore_command = ( + f"dd if={backup_name} of=/dev/{device} bs=512 count=1 seek={used_blocks}" + ) + restore_mode = str(write_flow.get("restore", "success")).lower() + if restore_mode in {"success", "pass", "ok"}: + rules.append({ + "command": restore_command, + "set_state": {device_state_key: {"from_file": backup_name}}, + "result": {"returncode": 0, "stdout": "", "stderr": ""}, + }) + elif restore_mode == "error": + error_text = str(write_flow.get("restore_error", "mocked restore failure")) + rules.append({ + "command": restore_command, + "raise": { + "type": CalledProcessError, + "args": [1, restore_command], + "kwargs": {"output": "", "stderr": error_text}, + }, + }) + elif restore_mode not in {"skip", "none"}: + raise ConfigError( + "blk_write_check.write_flow.restore must be success, error, skip, or none" + ) + + return state, rules + + +BLK_DEVICE_BANNER = "*" * 128 + + +def _print_blk_devices_separator() -> None: + print(f"\n{BLK_DEVICE_BANNER}\n") + + +def _run_blk_device_command(module: Any, command: str) -> Any: + return module.subprocess.run( + command, + shell=True, + text=True, + check=False, + capture_output=True, + ) + + +def _list_blk_devices(module: Any) -> list[str]: + command = "lsblk -e 7 -d | grep disk | awk '{print $1}'" + result = _run_blk_device_command(module, command) + return result.stdout.split() + + +def _print_blk_devices_header(disks: list[str]) -> None: + _print_blk_devices_separator() + print(" Read block devices tool\n") + print(BLK_DEVICE_BANNER) + print("INFO: Detected following block devices with lsblk command :") + for num, disk in enumerate(disks): + print(f"{num}: {disk}") + _print_blk_devices_separator() + + +def _detect_blk_partition_table(module: Any, disk: str) -> str: + command = f"timeout 10 gdisk -l /dev/{disk}" + result = _run_blk_device_command(module, command) + if "MBR: MBR only" in result.stdout: + return "MBR" + if "GPT: present" in result.stdout: + return "GPT" + print(f"INFO: No valid partition table found for {disk}, treating as raw device.") + return "RAW" + + +def _count_blk_partitions(module: Any, disk: str) -> int: + command = f"lsblk -rn -o NAME,TYPE /dev/{disk} | grep -c part" + result = _run_blk_device_command(module, command) + num_parts_str = result.stdout.strip() + return int(num_parts_str) if num_parts_str else 0 + + +def _run_blk_read(module: Any, partition_label: str, context: str = "") -> bool: + context_suffix = f" {context}" if context else "" + print(f"INFO: Performing block read on /dev/{partition_label}{context_suffix}") + command = f"dd if=/dev/{partition_label} bs=1M count=1 > /dev/null" + result = _run_blk_device_command(module, command) + outcome = "successful" if result.returncode == 0 else "failed" + print(f"INFO: Block read on /dev/{partition_label}{context_suffix} {outcome}") + return result.returncode == 0 + + +def _print_precious_partition( + module: Any, + partition_label: str, + identifier: str, + precious_parts: Mapping[str, str], + *, + suffix: str = "", +) -> None: + for key, value in precious_parts.items(): + if value != identifier: + continue + print(f"INFO: {partition_label} partition is PRECIOUS{suffix}") + used_blocks, _ = module.get_partition_space(f"/dev/{partition_label}") + print( + "INFO: Number of 512B blocks used on " + f"/dev/{partition_label}: {used_blocks}" + ) + print(f" {key} : {value}") + print(" Skipping block read/write...") + break + + +def _parse_mbr_partition_ids(fdisk_stdout: str) -> list[str]: + table_header_row = [ + "Device", + "Boot", + "Start", + "End", + "Sectors", + "Size", + "Id", + "Type", + ] + lines = fdisk_stdout.strip().split("\n") + collect_lines = False + mbr_part_ids: list[str] = [] + + for line in lines: + if collect_lines: + columns = line.split() + if len(columns) < 6: + continue + if columns[1] == "*" and len(columns) >= 7: + mbr_part_ids.append("0x" + columns[6].upper()) + else: + mbr_part_ids.append("0x" + columns[5].upper()) + continue + if all(substring in line for substring in table_header_row): + collect_lines = True + + return mbr_part_ids + + +def _parse_gpt_partition_info( + module: Any, + sgdisk_stdout: str, +) -> tuple[str, int] | None: + guid_regex = r"Partition GUID code: ([\w-]+) \(" + attr_flag_regex = r"Attribute flags: ([0-9A-Fa-f]+)" + guid_code_match = module.re.search(guid_regex, sgdisk_stdout) + attribute_flags_match = module.re.search(attr_flag_regex, sgdisk_stdout) + if not guid_code_match or not attribute_flags_match: + return None + + partition_guid_code = guid_code_match.group(1) + attribute_flags_hex = attribute_flags_match.group(1) + attribute_flags_int = int(attribute_flags_hex, 16) + return partition_guid_code, attribute_flags_int & 1 + + +def _handle_raw_blk_device(module: Any, disk: str) -> None: + print(f"INFO: No partitions detected for {disk}, treating as raw device.") + if _run_blk_read(module, disk): + module.perform_write_check(disk, "", {}) + _print_blk_devices_separator() + + +def _handle_mbr_blk_device( + module: Any, + disk: str, + part_labels: list[str], + num_parts: int, +) -> None: + command = f"fdisk -l /dev/{disk}" + result = _run_blk_device_command(module, command) + mbr_part_ids = _parse_mbr_partition_ids(result.stdout) + + if len(mbr_part_ids) < num_parts: + print( + "WARNING: Could not parse enough MBR partition IDs. " + f"Found {len(mbr_part_ids)}, expected {num_parts}." + ) + + process_count = min(len(part_labels), len(mbr_part_ids), num_parts) + for index in range(process_count): + part_label = part_labels[index] + partition_id = mbr_part_ids[index] + print(f"\nINFO: Partition : /dev/{part_label} Partition type : {partition_id}") + + if partition_id in module.precious_parts_mbr.values(): + _print_precious_partition( + module, + part_label, + partition_id, + module.precious_parts_mbr, + ) + continue + + if _run_blk_read(module, part_label, f"mbr_part_id = {partition_id}"): + module.perform_write_check( + part_label, + partition_id, + module.precious_parts_mbr, + ) + + _print_blk_devices_separator() + + +def _handle_gpt_blk_device( + module: Any, + disk: str, + part_labels: list[str], + num_parts: int, +) -> None: + process_count = min(len(part_labels), num_parts) + for index in range(process_count): + part_label = part_labels[index] + command = f"sgdisk -i={index + 1} /dev/{disk}" + result = _run_blk_device_command(module, command) + parsed = _parse_gpt_partition_info(module, result.stdout) + + if parsed is None: + print(f"INFO: Unable to parse sgdisk info for {part_label}. Skipping.") + continue + + partition_guid_code, lsb = parsed + print( + f"\nINFO: Partition : /dev/{part_label} " + f"Partition type GUID : {partition_guid_code} " + f"\"Platform required bit\" : {lsb}" + ) + + if lsb == 1: + print( + "INFO: Platform required attribute set for " + f"{part_label} partition, skipping block read/write..." + ) + continue + + if partition_guid_code in module.precious_parts_gpt.values(): + _print_precious_partition( + module, + part_label, + partition_guid_code, + module.precious_parts_gpt, + suffix=".", + ) + continue + + if _run_blk_read(module, part_label, f"part_guid = {partition_guid_code}"): + module.perform_write_check( + part_label, + partition_guid_code, + module.precious_parts_gpt, + ) + + _print_blk_devices_separator() + + +def _process_blk_device(module: Any, disk: str) -> None: + print(f"INFO: Block device : /dev/{disk}") + part_table = _detect_blk_partition_table(module, disk) + print(f"INFO: Partition table type : {part_table}\n") + + num_parts = _count_blk_partitions(module, disk) + if part_table == "RAW" or num_parts == 0: + _handle_raw_blk_device(module, disk) + return + + part_labels = module.get_partition_labels(disk) + if len(part_labels) < num_parts: + print( + "WARNING: Mismatch in partition count. Found " + f"{len(part_labels)} partition labels, but lsblk reported " + f"{num_parts} partitions for {disk}. Proceeding with the ones we have..." + ) + + if part_table == "MBR": + _handle_mbr_blk_device(module, disk, part_labels, num_parts) + return + if part_table == "GPT": + _handle_gpt_blk_device(module, disk, part_labels, num_parts) + return + + print( + "INFO: Invalid partition table, expected MBR or GPT " + f"reported type = {part_table}" + ) + + +def _run_blk_devices_module_main() -> int: + """ + Synthetic main() for block-device scripts that only define a main guard. + + This mirrors the repository script flow closely enough for YAML scenario + tests while still exercising the target module's own helper functions. + """ + module = _resolve_runner_module_from_stack() + + try: + disks = _list_blk_devices(module) + _print_blk_devices_header(disks) + + for disk in disks: + if module.is_mtd_block_device(disk): + print(f"INFO: Skipping MTD block device /dev/{disk}") + continue + + if module.is_ram_disk(disk): + print(f"INFO: Skipping RAM disk /dev/{disk}") + continue + + _process_blk_device(module, disk) + + return 0 + except Exception as exc: # pylint: disable=broad-exception-caught + print(f"Error occurred: {exc}") + return 1 + + +def build_blk_devices_scenario_case( + scenario: dict[str, Any], + work_dir: Path, +) -> dict[str, Any]: + """Build runtime mocks for the block-device script main flow.""" + disks = scenario.get("disks", []) + if disks is None: + disks = [] + if not isinstance(disks, list): + raise ConfigError("scenario.disks must be a list") + + prompt = _normalize_prompt_response(scenario.get("prompt", "no")) + prompt_response = "no" if prompt == "timeout" else prompt + call_log_path = str(work_dir / "blk_commands.log") + disk_listing_lines: list[str] = [] + run_responses: dict[str, Any] = {} + stateful_state: dict[str, Any] = {} + stateful_rules: list[dict[str, Any]] = [] + + for disk_index, disk in enumerate(disks, start=1): + if not isinstance(disk, dict): + raise ConfigError(f"scenario.disks[{disk_index}] must be a mapping") + if "name" not in disk: + raise ConfigError(f"scenario.disks[{disk_index}] requires key 'name'") + + disk_name = str(disk["name"]) + disk_listing_lines.append(f"{disk_name} disk") + + disk_kind = str(disk.get("kind", "disk")).lower() + if disk_name.startswith("mtdblock") or disk_kind in {"mtd", "mtdblock"}: + continue + if disk_name.startswith("ram") or disk_kind == "ram": + continue + + table = str(disk.get("table", "raw")).lower() + if table not in {"raw", "mbr", "gpt"}: + raise ConfigError( + f"scenario.disks[{disk_index}].table must be one of raw, mbr, gpt" + ) + + partitions = disk.get("partitions", []) + if partitions is None: + partitions = [] + if not isinstance(partitions, list): + raise ConfigError(f"scenario.disks[{disk_index}].partitions must be a list") + + reported_partition_count = disk.get("reported_partition_count", len(partitions)) + if not isinstance(reported_partition_count, int): + raise ConfigError( + f"scenario.disks[{disk_index}].reported_partition_count must be an integer" + ) + + gdisk_stdout = str( + disk.get( + "gdisk_output", + ( + "MBR: MBR only\n" + if table == "mbr" + else "GPT: present\n" if table == "gpt" else "no partition table here\n" + ), + ) + ) + if gdisk_stdout and not gdisk_stdout.endswith("\n"): + gdisk_stdout += "\n" + + run_responses[f"timeout 10 gdisk -l /dev/{disk_name}"] = _completed_process_spec( + stdout=gdisk_stdout + ) + + if table == "raw" or reported_partition_count == 0: + run_responses[f"lsblk -rn -o NAME,TYPE /dev/{disk_name}"] = _completed_process_spec( + stdout="" + ) + run_responses[f"dd if=/dev/{disk_name} of=/dev/null bs=1M count=1"] = ( + build_run_result_from_outcome(disk.get("block_read", "pass")) + ) + disk_write_cfg = disk.get("write_check") or disk + _add_blk_write_check_run_responses( + run_responses, + disk_name, + disk_write_cfg, + ) + state, rules = _build_blk_write_check_stateful_rules( + disk_name, + disk_write_cfg, + ) + stateful_state.update(state) + stateful_rules.extend(rules) + continue + + partition_labels: list[str] = [] + for part_index, partition in enumerate(partitions, start=1): + if not isinstance(partition, dict): + raise ConfigError( + f"scenario.disks[{disk_index}].partitions[{part_index}] must be a mapping" + ) + partition_labels.append(str(partition.get("name", f"{disk_name}{part_index}"))) + + lsblk_partition_lines = [f"{label} part" for label in partition_labels] + run_responses[f"lsblk -rn -o NAME,TYPE /dev/{disk_name}"] = _completed_process_spec( + stdout=_join_stdout_lines(lsblk_partition_lines) + ) + + if table == "mbr": + fdisk_stdout = disk.get("fdisk_output") + if fdisk_stdout is None: + fdisk_stdout = build_fdisk_output(disk_name, partitions) + else: + fdisk_stdout = str(fdisk_stdout) + if fdisk_stdout and not fdisk_stdout.endswith("\n"): + fdisk_stdout += "\n" + + run_responses[f"fdisk -l /dev/{disk_name}"] = _completed_process_spec( + stdout=fdisk_stdout + ) + + for part_index, partition in enumerate(partitions, start=1): + partition_name = str(partition.get("name", f"{disk_name}{part_index}")) + partition_id = _normalize_mbr_partition_id(partition.get("id", "83")) + + if ( + partition_id in PRECIOUS_MBR_IDS + or "used_blocks" in partition + or "available_blocks" in partition + ): + run_responses[ + f"df -B 512 /dev/{partition_name} --output=used,avail" + ] = _completed_process_spec( + stdout=build_df_output( + int(partition.get("used_blocks", 0)), + int(partition.get("available_blocks", 0)), + ) + ) + + if partition_id in PRECIOUS_MBR_IDS: + continue + + run_responses[ + f"dd if=/dev/{partition_name} of=/dev/null bs=1M count=1" + ] = build_run_result_from_outcome(partition.get("block_read", "pass")) + partition_write_cfg = partition.get("write_check") or partition + _add_blk_write_check_run_responses( + run_responses, + partition_name, + partition_write_cfg, + ) + state, rules = _build_blk_write_check_stateful_rules( + partition_name, + partition_write_cfg, + ) + stateful_state.update(state) + stateful_rules.extend(rules) + + continue + + for part_index, partition in enumerate(partitions, start=1): + partition_name = str(partition.get("name", f"{disk_name}{part_index}")) + partition_guid = str( + partition.get("guid", "0FC63DAF-8483-4772-8E79-3D69D8477DE4") + ).upper() + required = _platform_required(partition) + + sgdisk_stdout = partition.get("sgdisk_output") + if sgdisk_stdout is None: + sgdisk_stdout = build_sgdisk_partition_output(partition) + else: + sgdisk_stdout = str(sgdisk_stdout) + if sgdisk_stdout and not sgdisk_stdout.endswith("\n"): + sgdisk_stdout += "\n" + + run_responses[f"sgdisk -i={part_index} /dev/{disk_name}"] = _completed_process_spec( + stdout=sgdisk_stdout + ) + + if ( + partition_guid in PRECIOUS_GPT_GUIDS + or required + or "used_blocks" in partition + or "available_blocks" in partition + ): + run_responses[ + f"df -B 512 /dev/{partition_name} --output=used,avail" + ] = _completed_process_spec( + stdout=build_df_output( + int(partition.get("used_blocks", 0)), + int(partition.get("available_blocks", 0)), + ) + ) + + if partition_guid in PRECIOUS_GPT_GUIDS or required: + continue + + run_responses[ + f"dd if=/dev/{partition_name} of=/dev/null bs=1M count=1" + ] = build_run_result_from_outcome(partition.get("block_read", "pass")) + partition_write_cfg = partition.get("write_check") or partition + _add_blk_write_check_run_responses( + run_responses, + partition_name, + partition_write_cfg, + ) + state, rules = _build_blk_write_check_stateful_rules( + partition_name, + partition_write_cfg, + ) + stateful_state.update(state) + stateful_rules.extend(rules) + + run_responses["lsblk -e 7 -d -n -o NAME,TYPE"] = _completed_process_spec( + stdout=_join_stdout_lines(disk_listing_lines) + ) + + run_mock_spec: dict[str, Any] = { + "factory": run_router, + "kwargs": { + "responses": run_responses, + "default_returncode": 1, + "use_contains": False, + "unmatched_stderr_template": "UNMOCKED COMMAND: {cmd}\n", + "call_log_path": call_log_path, + }, + } + if stateful_rules: + run_mock_spec = { + "factory": stateful_run_router, + "kwargs": { + "state": stateful_state, + "rules": stateful_rules, + "responses": run_responses, + "default_returncode": 1, + "use_contains": False, + "unmatched_stderr_template": "UNMOCKED COMMAND: {cmd}\n", + "call_log_path": call_log_path, + }, + } + + return { + "mocks": { + "{module}.input_with_timeout": {"return_value": prompt_response}, + "subprocess.run": run_mock_spec, + }, + } + + +def build_blk_write_check_scenario_case( + scenario: dict[str, Any], + work_dir: Path, +) -> dict[str, Any]: + """Build runtime mocks for direct perform_write_check scenarios.""" + device = scenario.get("device") or scenario.get("partition_label") + if not isinstance(device, str) or not device.strip(): + raise ConfigError("blk_write_check requires string key 'device'") + + partition_id = str(scenario.get("partition_id", "")) + precious_parts = scenario.get("precious_parts", {}) + if not isinstance(precious_parts, dict): + raise ConfigError("blk_write_check.precious_parts must be a mapping") + + prompt = _normalize_prompt_response(scenario.get("prompt", "no")) + prompt_result = "no" if prompt == "timeout" else prompt + call_log_path = str(work_dir / "blk_commands.log") + run_responses: dict[str, Any] = {} + _add_blk_write_check_run_responses(run_responses, device, scenario) + state, rules = _build_blk_write_check_stateful_rules(device, scenario) + if not rules: + return { + "args": [device, partition_id, precious_parts], + "mocks": { + "{module}.input_with_timeout": {"return_value": prompt_result}, + "subprocess.run": { + "factory": run_router, + "kwargs": { + "responses": run_responses, + "default_returncode": 1, + "use_contains": False, + "unmatched_stderr_template": "UNMOCKED COMMAND: {cmd}\n", + "call_log_path": call_log_path, + }, + }, + }, + } + + return { + "args": [device, partition_id, precious_parts], + "mocks": { + "{module}.input_with_timeout": {"return_value": prompt_result}, + "subprocess.run": { + "factory": stateful_run_router, + "kwargs": { + "state": state, + "rules": rules, + "responses": run_responses, + "default_returncode": 1, + "use_contains": False, + "unmatched_stderr_template": "UNMOCKED COMMAND: {cmd}\n", + "call_log_path": call_log_path, + }, + }, + }, + } + + +def _ensure_string_list(value: Any, field_name: str) -> list[str]: + if value is None: + return [] + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise ConfigError(f"{field_name} must be a list of strings") + return list(value) + + +def _normalize_verify_tpm_banks(value: Any, field_name: str) -> dict[str, list[str]]: + if not isinstance(value, dict) or not value: + raise ConfigError(f"{field_name} must be a non-empty mapping") + + normalized: dict[str, list[str]] = {} + for algo, entries in value.items(): + if not isinstance(algo, str): + raise ConfigError(f"{field_name} keys must be strings") + if not isinstance(entries, list) or not all(isinstance(item, str) for item in entries): + raise ConfigError(f"{field_name}.{algo} must be a list of strings") + normalized[algo] = list(entries) + return normalized + + +def _verify_tpm_event( + event_num: int, + pcr_index: int, + event_type: str, + *, + event: Any = None, + spec_version_major: int | None = None, +) -> dict[str, Any]: + entry: dict[str, Any] = { + "EventNum": event_num, + "PCRIndex": pcr_index, + "EventType": event_type, + } + if spec_version_major is not None: + entry["SpecID"] = [{"specVersionMajor": spec_version_major}] + elif event is not None: + entry["Event"] = event + return entry + + +def build_verify_tpm_events(scenario: dict[str, Any]) -> list[dict[str, Any]]: + """Build the default event list for verify_tpm_measurements.py.""" + if "events" in scenario: + explicit_events = scenario["events"] + if not isinstance(explicit_events, list): + raise ConfigError("verify_tpm.events must be a list when provided") + return list(explicit_events) + + spec_version_major = int(scenario.get("spec_version_major", 2)) + post_code_events = _ensure_string_list( + scenario.get("post_code_events", ["BL_31", "SECURE_RT_EL3"]), + "verify_tpm.post_code_events", + ) + secure_boot_vars = _ensure_string_list( + scenario.get("secure_boot_vars", ["SecureBoot", "PK", "KEK", "db", "dbx"]), + "verify_tpm.secure_boot_vars", + ) + boot_variables = _ensure_string_list( + scenario.get("boot_variables", ["BootOrder", "Boot0001"]), + "verify_tpm.boot_variables", + ) + handoff_events = _ensure_string_list( + scenario.get("handoff_events", ["SMBIOS"]), + "verify_tpm.handoff_events", + ) + table_of_devices = _ensure_string_list( + scenario.get("table_of_devices", ["SYS_CONFIG_UART0"]), + "verify_tpm.table_of_devices", + ) + + separator_pcrs = scenario.get("separator_pcrs", list(range(8))) + if not isinstance(separator_pcrs, list) or not all( + isinstance(item, int) for item in separator_pcrs + ): + raise ConfigError("verify_tpm.separator_pcrs must be a list of integers") + + boot_attempt = scenario.get("boot_attempt", "Calling EFI Application from Boot Option") + if boot_attempt is not None and not isinstance(boot_attempt, str): + raise ConfigError("verify_tpm.boot_attempt must be a string or null") + + exit_boot_services = scenario_truthy( + scenario.get("exit_boot_services"), + default=True, + ) + + events: list[dict[str, Any]] = [] + next_event_num = 1 + + events.append( + _verify_tpm_event( + next_event_num, + 0, + "EV_NO_ACTION", + spec_version_major=spec_version_major, + ) + ) + next_event_num += 1 + + for item in post_code_events: + events.append(_verify_tpm_event(next_event_num, 0, "EV_POST_CODE", event=item)) + next_event_num += 1 + + for var_name in secure_boot_vars: + events.append( + _verify_tpm_event( + next_event_num, + 7, + "EV_EFI_VARIABLE_DRIVER_CONFIG", + event={"UnicodeName": var_name}, + ) + ) + next_event_num += 1 + + for var_name in boot_variables: + event_type = "EV_EFI_VARIABLE_BOOT" if var_name == "BootOrder" else "EV_EFI_VARIABLE_BOOT2" + events.append( + _verify_tpm_event( + next_event_num, + 1, + event_type, + event={"UnicodeName": var_name}, + ) + ) + next_event_num += 1 + + if boot_attempt is not None: + events.append( + _verify_tpm_event( + next_event_num, + 4, + "EV_EFI_ACTION", + event=boot_attempt, + ) + ) + next_event_num += 1 + + for item in handoff_events: + events.append( + _verify_tpm_event(next_event_num, 1, "EV_EFI_HANDOFF_TABLES", event=item) + ) + next_event_num += 1 + + for pcr_index in separator_pcrs: + events.append( + _verify_tpm_event( + next_event_num, + pcr_index, + "EV_SEPARATOR", + event=f"sep{pcr_index}", + ) + ) + next_event_num += 1 + + for item in table_of_devices: + events.append( + _verify_tpm_event(next_event_num, 1, "EV_TABLE_OF_DEVICES", event=item) + ) + next_event_num += 1 + + if exit_boot_services: + events.append( + _verify_tpm_event( + next_event_num, + 5, + "EV_EFI_ACTION", + event="Exit Boot Services Invocation", + ) + ) + + return events + + +def build_verify_tpm_scenario_case( + scenario: dict[str, Any], + work_dir: Path, +) -> dict[str, Any]: + """Build runtime files and args for verify_tpm_measurements.py scenarios.""" + pcr_banks = _normalize_verify_tpm_banks(scenario.get("pcrs"), "verify_tpm.pcrs") + event_pcrs = _normalize_verify_tpm_banks( + scenario.get("event_pcrs", pcr_banks), + "verify_tpm.event_pcrs", + ) + + event_log_raw = scenario.get("event_log_raw") + if event_log_raw is not None: + if not isinstance(event_log_raw, str): + raise ConfigError("verify_tpm.event_log_raw must be a string") + event_contents = event_log_raw + if event_contents and not event_contents.endswith("\n"): + event_contents += "\n" + else: + event_doc = { + "pcrs": event_pcrs, + "events": build_verify_tpm_events(scenario), + } + event_contents = json.dumps(event_doc, indent=2) + "\n" + + pcr_contents = json.dumps(pcr_banks, indent=2) + "\n" + pcr_path = work_dir / "pcr.yaml" + event_path = work_dir / "event.yaml" + + generated: dict[str, Any] = { + "args": [str(pcr_path), str(event_path)], + "text_files": { + "pcr.yaml": pcr_contents, + "event.yaml": event_contents, + }, + } + + mocks: dict[str, Any] = {} + + event_log_open_error = scenario.get("event_log_open_error") + if event_log_open_error is not None: + if not isinstance(event_log_open_error, str) or not event_log_open_error.strip(): + raise ConfigError( + "verify_tpm.event_log_open_error must be a non-empty string" + ) + mocks["builtins.open"] = { + "factory": "mock_helpers.passthrough_router", + "inject_original_as": "real", + "kwargs": { + "label": "verify-tpm-open-router", + "rules": [ + { + "label": "event-log-open-error", + "required": True, + "when": { + "args": { + 0: {"equals": str(event_path)}, + 1: {"contains": "r"}, + } + }, + "raise": { + "type": "py:builtins.OSError", + "args": [event_log_open_error], + }, + } + ], + }, + } + + event_log_yaml_error = scenario.get("event_log_yaml_error") + if event_log_yaml_error is not None: + if not isinstance(event_log_yaml_error, str) or not event_log_yaml_error.strip(): + raise ConfigError( + "verify_tpm.event_log_yaml_error must be a non-empty string" + ) + mocks["yaml.safe_load"] = { + "factory": "mock_helpers.passthrough_router", + "inject_original_as": "real", + "kwargs": { + "label": "verify-tpm-yaml-router", + "rules": [ + { + "label": "event-log-yaml-error", + "required": True, + "when": { + "args": { + 0: {"contains": str(event_path)}, + } + }, + "raise": { + "type": "py:yaml.YAMLError", + "args": [event_log_yaml_error], + }, + } + ], + }, + } + + if mocks: + generated["mocks"] = mocks + + return generated + + +CAPSULE_REPORT_GUID = "39b68c46-f7fb-441b-b6ec-16b0f69821f3" +GLOBAL_VARIABLE_GUID = "8be4df61-93ca-11d2-aa0d-00e098032b8c" +DEFAULT_ATTR_CAPSULE_MAX = 0x06 +DEFAULT_ATTR_CAPSULE_LAST = 0x07 +DEFAULT_ATTR_CAPSULE_NNNN = 0x07 +DEFAULT_ATTR_OS_INDICATIONS = 0x07 +FILE_CAPSULE_DELIVERY_SUPPORTED = 0x4 + + +def _parse_int_like(value: Any, field_name: str) -> int: + if isinstance(value, bool): + return int(value) + if isinstance(value, int): + return value + if isinstance(value, str): + text = value.strip() + if not text: + raise ConfigError(f"{field_name} must not be empty") + try: + return int(text, 0) + except ValueError as exc: + raise ConfigError( + f"{field_name} must be an integer or int-like string" + ) from exc + raise ConfigError(f"{field_name} must be an integer or int-like string") + + +def _decode_hex_bytes(hex_text: str, field_name: str) -> bytes: + try: + return bytes.fromhex(hex_text) + except ValueError as exc: + raise ConfigError(f"{field_name} contains invalid hex data") from exc + + +def _resolve_case_relative_path(work_dir: Path, raw_path: str) -> Path: + candidate = Path(raw_path) + return candidate if candidate.is_absolute() else work_dir / candidate + + +def _build_capsule_named_var_bytes( + spec: Any, + *, + default_attrs: int, + field_name: str, +) -> bytes: + if isinstance(spec, str): + return build_efi_var_bytes(default_attrs, build_char16_payload(spec)) + + if not isinstance(spec, dict): + raise ConfigError(f"{field_name} must be a string or mapping") + + if "hex" in spec: + return _decode_hex_bytes(str(spec["hex"]), f"{field_name}.hex") + + attrs = _parse_int_like(spec.get("attrs", default_attrs), f"{field_name}.attrs") + + if "payload_hex" in spec: + payload = _decode_hex_bytes( + str(spec["payload_hex"]), + f"{field_name}.payload_hex", + ) + else: + name = spec.get("name", spec.get("value", "")) + if not isinstance(name, str): + raise ConfigError(f"{field_name}.name must be a string") + extra_utf16 = spec.get("extra_utf16", "") + if not isinstance(extra_utf16, str): + raise ConfigError(f"{field_name}.extra_utf16 must be a string") + payload = build_char16_payload(name, extra_utf16) + + return build_efi_var_bytes(attrs, payload) + + +def _build_capsule_entry_bytes( + spec: Any, + *, + field_name: str, +) -> tuple[str, bytes]: + if isinstance(spec, str): + return spec, build_efi_var_bytes( + DEFAULT_ATTR_CAPSULE_NNNN, + _decode_hex_bytes("AA", f"{field_name}.payload_hex"), + ) + + if not isinstance(spec, dict): + raise ConfigError(f"{field_name} entries must be strings or mappings") + + name = spec.get("name") + if not isinstance(name, str) or not name.strip(): + raise ConfigError(f"{field_name}.name must be a non-empty string") + + if "hex" in spec: + payload_bytes = _decode_hex_bytes(str(spec["hex"]), f"{field_name}.hex") + return name, payload_bytes + + attrs = _parse_int_like( + spec.get("attrs", DEFAULT_ATTR_CAPSULE_NNNN), + f"{field_name}.attrs", + ) + payload_hex = str(spec.get("payload_hex", "AA")) + payload = _decode_hex_bytes(payload_hex, f"{field_name}.payload_hex") + return name, build_efi_var_bytes(attrs, payload) + + +def _build_os_indications_bytes(spec: Any) -> bytes: + if isinstance(spec, bool): + value = FILE_CAPSULE_DELIVERY_SUPPORTED if spec else 0 + return build_os_indications_var(value, attrs=DEFAULT_ATTR_OS_INDICATIONS) + + if isinstance(spec, (int, str)): + value = _parse_int_like(spec, "capsule_vars.os_indications") + return build_os_indications_var(value, attrs=DEFAULT_ATTR_OS_INDICATIONS) + + if not isinstance(spec, dict): + raise ConfigError( + "capsule_vars.os_indications must be a bool, int-like, or mapping" + ) + + if "hex" in spec: + return _decode_hex_bytes( + str(spec["hex"]), + "capsule_vars.os_indications.hex", + ) + + attrs = _parse_int_like( + spec.get("attrs", DEFAULT_ATTR_OS_INDICATIONS), + "capsule_vars.os_indications.attrs", + ) + if "value" in spec: + value = _parse_int_like( + spec["value"], + "capsule_vars.os_indications.value", + ) + else: + supported = scenario_truthy(spec.get("supported"), default=False) + value = FILE_CAPSULE_DELIVERY_SUPPORTED if supported else 0 + return build_os_indications_var(value, attrs=attrs) + + +def build_capsule_vars_scenario_case( + scenario: dict[str, Any], + work_dir: Path, +) -> dict[str, Any]: + """Build efivarfs fixtures and default module patches for capsule variable checks.""" + efivar_dir_raw = scenario.get("efivar_dir", "efivars") + if not isinstance(efivar_dir_raw, str) or not efivar_dir_raw.strip(): + raise ConfigError("capsule_vars.efivar_dir must be a non-empty string") + log_file_raw = scenario.get("log_file", "capsule.log") + if not isinstance(log_file_raw, str) or not log_file_raw.strip(): + raise ConfigError("capsule_vars.log_file must be a non-empty string") + + efivar_path = _resolve_case_relative_path(work_dir, efivar_dir_raw) + log_file_path = _resolve_case_relative_path(work_dir, log_file_raw) + efivarfs_present = scenario_truthy( + scenario.get("efivarfs_present"), + default=True, + ) + + generated: dict[str, Any] = { + "patch_constants": { + "EFIVAR_PATH": str(efivar_path), + "LOG_FILE": str(log_file_path), + } + } + + if not efivarfs_present: + return generated + + generated["dir_structure"] = [{"path": str(efivar_path.relative_to(work_dir))}] + bin_files: dict[str, dict[str, str]] = {} + + def add_var(filename: str, payload: bytes) -> None: + rel_path = str((efivar_path.relative_to(work_dir) / filename).as_posix()) + bin_files[rel_path] = {"hex": payload.hex()} + + os_indications = scenario.get("os_indications") + if os_indications is not None: + add_var( + f"OsIndicationsSupported-{GLOBAL_VARIABLE_GUID}", + _build_os_indications_bytes(os_indications), + ) + + capsule_max = scenario.get("capsule_max") + if capsule_max is not None: + add_var( + f"CapsuleMax-{CAPSULE_REPORT_GUID}", + _build_capsule_named_var_bytes( + capsule_max, + default_attrs=DEFAULT_ATTR_CAPSULE_MAX, + field_name="capsule_vars.capsule_max", + ), + ) + + capsule_last = scenario.get("capsule_last") + if capsule_last is not None: + add_var( + f"CapsuleLast-{CAPSULE_REPORT_GUID}", + _build_capsule_named_var_bytes( + capsule_last, + default_attrs=DEFAULT_ATTR_CAPSULE_LAST, + field_name="capsule_vars.capsule_last", + ), + ) + + capsule_entries = scenario.get("capsule_entries", []) + if capsule_entries is None: + capsule_entries = [] + if not isinstance(capsule_entries, list): + raise ConfigError("capsule_vars.capsule_entries must be a list") + + for index, entry in enumerate(capsule_entries, start=1): + name, payload = _build_capsule_entry_bytes( + entry, + field_name=f"capsule_vars.capsule_entries[{index}]", + ) + add_var(f"{name}-{CAPSULE_REPORT_GUID}", payload) + + if bin_files: + generated["bin_files"] = bin_files + + return generated + + +def build_runtime_device_mapping_scenario_case( + scenario: dict[str, Any], + work_dir: Path, +) -> dict[str, Any]: + """Build DTS and memmap fixtures for runtime_device_mapping_conflict_checker.py.""" + text_files: dict[str, str] = {} + bin_files: dict[str, dict[str, str]] = {} + mocks: dict[str, Any] = {} + + dts_path = work_dir / "device_tree.dts" + memmap_path = work_dir / "memmap.log" + log_path = work_dir / "runtime_device_mapping_conflict_test.log" + + dts = scenario.get("dts") + memmap = scenario.get("memmap") + memmap_hex = scenario.get("memmap_hex") + missing_dts = scenario_truthy(scenario.get("missing_dts"), default=False) + missing_memmap = scenario_truthy(scenario.get("missing_memmap"), default=False) + log_open_error = scenario.get("log_open_error") + + if missing_dts and dts is not None: + raise ConfigError( + "runtime_device_mapping cannot define both dts and missing_dts" + ) + if missing_memmap and (memmap is not None or memmap_hex is not None): + raise ConfigError( + "runtime_device_mapping cannot define memmap/memmap_hex together with " + "missing_memmap" + ) + if log_open_error is not None and ( + not isinstance(log_open_error, str) or not log_open_error.strip() + ): + raise ConfigError( + "runtime_device_mapping.log_open_error must be a non-empty string" + ) + + if dts is not None: + if not isinstance(dts, str): + raise ConfigError("runtime_device_mapping.dts must be a string") + text_files["device_tree.dts"] = dts + + if memmap is not None and memmap_hex is not None: + raise ConfigError( + "runtime_device_mapping cannot define both memmap and memmap_hex" + ) + + if memmap is not None: + if not isinstance(memmap, str): + raise ConfigError("runtime_device_mapping.memmap must be a string") + text_files["memmap.log"] = memmap + + if memmap_hex is not None: + if not isinstance(memmap_hex, str): + raise ConfigError("runtime_device_mapping.memmap_hex must be a string") + bin_files["memmap.log"] = {"hex": memmap_hex} + + if log_open_error is not None: + mocks["pathlib.Path.open"] = { + "factory": "mock_helpers.passthrough_router", + "inject_original_as": "real", + "kwargs": { + "label": "runtime-device-mapping-open-router", + "rules": [ + { + "label": "runtime-log-open-error", + "required": True, + "when": { + "args": { + 0: {"equals": log_path}, + } + }, + "raise": { + "type": "py:builtins.OSError", + "args": [log_open_error], + }, + } + ], + }, + } + + generated: dict[str, Any] = { + "patch_constants": { + "DTS_PATH": dts_path, + "MEMMAP_PATH": memmap_path, + "OUT_LOG_PATH": log_path, + } + } + + if text_files: + generated["text_files"] = text_files + + if bin_files: + generated["bin_files"] = bin_files + + if mocks: + generated["mocks"] = mocks + + return generated diff --git a/common/acs_test_framework_runner/mock_loader_parser_scenarios.py b/common/acs_test_framework_runner/mock_loader_parser_scenarios.py new file mode 100644 index 00000000..5dd3fd22 --- /dev/null +++ b/common/acs_test_framework_runner/mock_loader_parser_scenarios.py @@ -0,0 +1,344 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +try: # Support package imports and direct harness module loading. + from .mock_loader import ConfigError + from .mock_loader import _resolve_runner_module_from_stack +except ImportError: # pragma: no cover - exercised by flat-module harness imports. + from mock_loader import ConfigError + from mock_loader import _resolve_runner_module_from_stack + + +def _readable_text_block(lines: list[str]) -> str: + if not lines: + return "" + return "\n".join(lines) + "\n" + + +def _normalize_simple_key_value_text(value: Any, field_name: str) -> str: + if value is None: + return "" + if isinstance(value, str): + return value if value.endswith("\n") or not value else value + "\n" + if not isinstance(value, dict): + raise ConfigError(f"{field_name} must be a string or mapping") + return _readable_text_block([f"{key}: {item}" for key, item in value.items()]) + + +def _normalize_json_text(value: Any, field_name: str) -> str: + if isinstance(value, str): + return value if value.endswith("\n") or not value else value + "\n" + try: + return json.dumps(value, indent=4) + "\n" + except TypeError as exc: + raise ConfigError( + f"{field_name} must be a string or JSON-serializable value" + ) from exc + + +def _normalize_text_block(value: Any, field_name: str) -> str: + if not isinstance(value, str): + raise ConfigError(f"{field_name} must be a string") + return value if value.endswith("\n") or not value else value + "\n" + + +def _build_merge_jsons_acs_info_payload(value: Any) -> dict[str, Any]: + if isinstance(value, str): + return {"ACS Results Summary": {"Overall Compliance Result": value}} + if not isinstance(value, dict): + raise ConfigError("merge_jsons.acs_info must be a string or mapping") + + if "ACS Results Summary" in value: + summary = value["ACS Results Summary"] + if not isinstance(summary, dict): + raise ConfigError( + "merge_jsons.acs_info['ACS Results Summary'] must be a mapping" + ) + return dict(value) + + return {"ACS Results Summary": dict(value)} + + +def _build_dmidecode_text(spec: Any) -> str: + if isinstance(spec, str): + return spec if spec.endswith("\n") or not spec else spec + "\n" + if not isinstance(spec, dict): + raise ConfigError("acs_info.dmidecode must be a string or mapping") + firmware = str(spec.get("firmware", spec.get("firmware_version", "Unknown"))) + vendor = str(spec.get("vendor", spec.get("manufacturer", "Unknown"))) + product = str(spec.get("product", spec.get("system_name", "Unknown"))) + family = str(spec.get("family", spec.get("soc_family", "Unknown"))) + return _readable_text_block([ + "Handle 0x0000, DMI type 0, 24 bytes", + "BIOS Information", + f" Version: {firmware}", + "", + "Handle 0x0001, DMI type 1, 27 bytes", + "System Information", + f" Manufacturer: {vendor}", + f" Product Name: {product}", + f" Family: {family}", + ]) + + +def build_acs_info_scenario_case( + scenario: dict[str, Any], + work_dir: Path, +) -> dict[str, Any]: + """Build fixture files and args for acs_info.py.""" + output_dir = str(scenario.get("output_dir", "out")) + args = ["--dmidecode_log", str(work_dir / "dmidecode.log")] + text_files: dict[str, str] = { + "dmidecode.log": _build_dmidecode_text(scenario.get("dmidecode", {})), + } + bin_files: dict[str, dict[str, str]] = {} + + acs_config = scenario.get("acs_config") + if acs_config is not None: + text_files["acs_config.txt"] = _normalize_simple_key_value_text( + acs_config, + "acs_info.acs_config", + ) + args.extend(["--acs_config_path", str(work_dir / "acs_config.txt")]) + + system_config = scenario.get("system_config") + if system_config is not None: + text_files["system_config.txt"] = _normalize_simple_key_value_text( + system_config, + "acs_info.system_config", + ) + args.extend(["--system_config_path", str(work_dir / "system_config.txt")]) + + ipmitool = scenario.get("ipmitool") + if ipmitool is not None: + if isinstance(ipmitool, str): + text_files["ipmitool.log"] = ( + ipmitool + if ipmitool.endswith("\n") or not ipmitool + else ipmitool + "\n" + ) + elif isinstance(ipmitool, dict): + fw_rev = ipmitool.get( + "firmware_revision", + ipmitool.get("Firmware Revision", "Unknown"), + ) + text_files["ipmitool.log"] = _readable_text_block( + [ + "Device ID : 32", + f"Firmware Revision : {fw_rev}", + ] + ) + else: + raise ConfigError("acs_info.ipmitool must be a string or mapping") + args.extend(["--ipmitool_log", str(work_dir / "ipmitool.log")]) + + uefi_version = scenario.get("uefi_version") + if uefi_version is not None: + encoding = "utf-8" + value = uefi_version + if isinstance(uefi_version, dict): + value = str(uefi_version.get("text", "")) + encoding = str(uefi_version.get("encoding", "utf-8")) + elif not isinstance(uefi_version, str): + raise ConfigError("acs_info.uefi_version must be a string or mapping") + payload = str(value) + if payload and not payload.endswith("\n"): + payload += "\n" + bin_files["uefi_version.log"] = {"hex": payload.encode(encoding).hex()} + args.extend(["--uefi_version_log", str(work_dir / "uefi_version.log")]) + + args.extend(["--output_dir", str(work_dir / output_dir)]) + generated: dict[str, Any] = {"args": args, "text_files": text_files} + if bin_files: + generated["bin_files"] = bin_files + return generated + + +def _build_merge_jsons_module_main( + *, + work_dir: Path, + input_files: list[str], + output_files: list[str], + dt_or_sr_mode: str | None = None, + yocto_flag_path: str | None = None, +) -> Any: + def _run_merge_jsons_module_main() -> int: + module = _resolve_runner_module_from_stack() + + if yocto_flag_path is not None: + setattr( + module, + "YOCTO_FLAG_PATH", + str((work_dir / yocto_flag_path).resolve()), + ) + if dt_or_sr_mode is not None: + setattr(module, "DT_OR_SR_MODE", dt_or_sr_mode) + + resolved_inputs = [ + str((work_dir / input_name).resolve()) for input_name in input_files + ] + for output_name in output_files: + module.merge_json_files( + resolved_inputs, + str((work_dir / output_name).resolve()), + ) + return 0 + + return _run_merge_jsons_module_main + + +def build_merge_jsons_scenario_case( + scenario: dict[str, Any], + work_dir: Path, +) -> dict[str, Any]: + text_files: dict[str, str] = {} + input_names: list[str] = [] + + def _append_input( + name: Any, + *, + field_name: str, + content: str | None = None, + ) -> None: + if not isinstance(name, str) or not name.strip(): + raise ConfigError(f"{field_name} must be a non-empty string") + if content is not None: + if name in text_files: + raise ConfigError(f"Duplicate merge_jsons input file: {name}") + text_files[name] = content + input_names.append(name) + + acs_info = scenario.get("acs_info") + if acs_info is not None: + acs_info_name = scenario.get("acs_info_name", "acs_info.json") + _append_input( + acs_info_name, + field_name="merge_jsons.acs_info_name", + content=_normalize_json_text( + _build_merge_jsons_acs_info_payload(acs_info), + "merge_jsons.acs_info", + ), + ) + + input_files = scenario.get("input_files") + if input_files is not None: + if not isinstance(input_files, dict): + raise ConfigError("merge_jsons.input_files must be a mapping") + for name, content in input_files.items(): + _append_input( + name, + field_name=f"merge_jsons.input_files[{name!r}]", + content=_normalize_json_text( + content, + f"merge_jsons.input_files[{name!r}]", + ), + ) + + invalid_files = scenario.get("invalid_files") + if invalid_files is not None: + if not isinstance(invalid_files, dict): + raise ConfigError("merge_jsons.invalid_files must be a mapping") + for name, content in invalid_files.items(): + _append_input( + name, + field_name=f"merge_jsons.invalid_files[{name!r}]", + content=_normalize_text_block( + content, + f"merge_jsons.invalid_files[{name!r}]", + ), + ) + + missing_input_files = scenario.get("missing_input_files", []) + if missing_input_files is not None: + if not isinstance(missing_input_files, list) or not all( + isinstance(item, str) and item.strip() for item in missing_input_files + ): + raise ConfigError( + "merge_jsons.missing_input_files must be a list of strings" + ) + for name in missing_input_files: + _append_input( + name, + field_name="merge_jsons.missing_input_files[]", + ) + + if not input_names: + raise ConfigError( + "merge_jsons scenario requires at least one input via " + "acs_info, input_files, invalid_files, or missing_input_files" + ) + + raw_output_files = scenario.get("output_files", scenario.get("output_file", "merged.json")) + if isinstance(raw_output_files, str): + output_files = [raw_output_files] + elif isinstance(raw_output_files, list) and all( + isinstance(item, str) and item.strip() for item in raw_output_files + ): + output_files = list(raw_output_files) + else: + raise ConfigError( + "merge_jsons.output_files must be a string or a list of strings" + ) + + dt_or_sr_mode = scenario.get("dt_or_sr_mode") + if dt_or_sr_mode is not None: + if not isinstance(dt_or_sr_mode, str) or dt_or_sr_mode not in {"DT", "SR"}: + raise ConfigError("merge_jsons.dt_or_sr_mode must be 'DT' or 'SR'") + + yocto_flag_path = scenario.get("yocto_flag_path") + if yocto_flag_path is not None: + if not isinstance(yocto_flag_path, str) or not yocto_flag_path.strip(): + raise ConfigError( + "merge_jsons.yocto_flag_path must be a non-empty string" + ) + text_files.setdefault(yocto_flag_path, "") + + generated_case = { + "args": [ + str((work_dir / output_files[0]).resolve()), + *[str((work_dir / input_name).resolve()) for input_name in input_names], + ] + if len(output_files) == 1 + else [], + "patch_constants": { + "main": _build_merge_jsons_module_main( + work_dir=work_dir, + input_files=input_names, + output_files=output_files, + dt_or_sr_mode=dt_or_sr_mode, + yocto_flag_path=yocto_flag_path, + ), + }, + "text_files": text_files, + } + return generated_case + + +def build_extract_capsule_fw_version_scenario_case( + scenario: dict[str, Any], + work_dir: Path, +) -> dict[str, Any]: + """Build fixture files and args for extract_capsule_fw_version.py.""" + pattern = scenario.get("pattern") + if not isinstance(pattern, str) or not pattern.strip(): + raise ConfigError("extract_capsule_fw_version.pattern must be a non-empty string") + + input_name = str(scenario.get("input_name", "input.log")) + raw_input = scenario.get("input_text") + if raw_input is None: + lines = scenario.get("input_lines", []) + if not isinstance(lines, list) or not all(isinstance(item, str) for item in lines): + raise ConfigError("extract_capsule_fw_version.input_lines must be a list of strings") + raw_input = _readable_text_block(lines) + elif not isinstance(raw_input, str): + raise ConfigError("extract_capsule_fw_version.input_text must be a string") + if raw_input and not raw_input.endswith("\n"): + raw_input += "\n" + + return { + "args": [pattern, str(work_dir / input_name)], + "text_files": {input_name: raw_input}, + } diff --git a/common/acs_test_framework_runner/pytest_runner.py b/common/acs_test_framework_runner/pytest_runner.py new file mode 100644 index 00000000..7bad92bc --- /dev/null +++ b/common/acs_test_framework_runner/pytest_runner.py @@ -0,0 +1,631 @@ +from __future__ import annotations + +import argparse +import shutil +import subprocess +import sys +import traceback +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from typing import Any + +try: # Support package imports and direct harness module loading. + from .case_data_builders import CaseBuildError + from .runner_checks import ( + ConfigError, + RunCaseOptions, + SkipCase, + TestMeta, + TestOutcome, + create_outcome, + detect_project_root, + format_outcome_message, + load_yaml_config, + normalize_suites, + resolve_target_path, + run_single_check, + sanitize_name, + ) + from .runner_reporting import ( + append_combined_case_log, + append_run_header, + build_config_error_outcome, + build_report_path, + cleanup_old_pytest_xml_reports, + create_placeholder_xml, + print_group_summary, + remove_placeholder_xml, + sanitize_xml_text, + write_case_log, + write_junit_xml, + ) +except ImportError: # pragma: no cover - exercised by flat-module harness imports. + from case_data_builders import CaseBuildError + from runner_checks import ( + ConfigError, + RunCaseOptions, + SkipCase, + TestMeta, + TestOutcome, + create_outcome, + detect_project_root, + format_outcome_message, + load_yaml_config, + normalize_suites, + resolve_target_path, + run_single_check, + sanitize_name, + ) + from runner_reporting import ( + append_combined_case_log, + append_run_header, + build_config_error_outcome, + build_report_path, + cleanup_old_pytest_xml_reports, + create_placeholder_xml, + print_group_summary, + remove_placeholder_xml, + sanitize_xml_text, + write_case_log, + write_junit_xml, + ) + +SCRIPT_DIR = Path(__file__).resolve().parent +PROJECT_ROOT = detect_project_root(SCRIPT_DIR) +HARNESS_DIR = PROJECT_ROOT / "common" / "acs_test_framework_runner" +TEST_YAML_DIR = PROJECT_ROOT / "common" / "acs_test_framework_manifests" +REPORTS_DIR = PROJECT_ROOT / "common" / "reports" +SUPPORTED_SUFFIXES = {".yaml", ".yml"} +IN_PROCESS_CASE_TYPES = {"py_function", "module_main_with_env", "module_cli"} + + +def discover_yaml_files() -> list[Path]: + if not TEST_YAML_DIR.exists() or not TEST_YAML_DIR.is_dir(): + return [] + return sorted( + path + for path in TEST_YAML_DIR.rglob("*") + if path.is_file() and path.suffix.lower() in SUPPORTED_SUFFIXES + ) + + +def get_group_name(yaml_file: Path) -> str: + rel_path = yaml_file.relative_to(TEST_YAML_DIR) + return rel_path.parts[0] if len(rel_path.parts) > 1 else yaml_file.stem + + + +def get_file_work_dir(suite_name: str, file_entry: str) -> Path: + return ( + REPORTS_DIR + / "_work" + / sanitize_name(suite_name) + / sanitize_name(Path(file_entry).stem) + ) + + +def requires_serial_case_execution(suite_cases: list[dict[str, Any]]) -> bool: + return any( + str(case_def.get("type", "cli")) in IN_PROCESS_CASE_TYPES + for case_def in suite_cases + ) + + +def run_case( + suite_name: str, + file_entry: str, + case_index: int, + case_def: dict[str, Any], + options: RunCaseOptions | None = None, +) -> TestOutcome: + if options is None: + options = RunCaseOptions() + + file_path = resolve_target_path(file_entry) + case_name = case_def["name"].strip() + testcase_name = f"{suite_name}::{Path(file_entry).name}::{case_name}" + case_type = str(case_def.get("type", "cli")) + + file_work_dir = get_file_work_dir(suite_name, file_entry) + work_dir = file_work_dir / sanitize_name(f"{case_index}_{case_name}") + work_dir.mkdir(parents=True, exist_ok=True) + + effective_case = dict(case_def) + if options.suite_command is not None and "command" not in effective_case: + effective_case["command"] = options.suite_command + + meta = TestMeta( + suite_name=suite_name, + phase="case", + test_type=case_type, + ) + + def persist_case_logs(status: str, outcome: TestOutcome) -> None: + case_description = effective_case.get("description") + append_combined_case_log( + file_work_dir=file_work_dir, + testcase_name=testcase_name, + status=status, + message=outcome.message, + details=outcome.details, + description=case_description, + ) + write_case_log( + case_work_dir=work_dir, + testcase_name=testcase_name, + status=status, + message=outcome.message, + details=outcome.details, + description=case_description, + ) + + try: + passed, message, details, is_error = run_single_check( + file_path, effective_case, work_dir + ) + + warn_only = bool(effective_case.get("warn_only", False)) + final_passed = passed + final_warning = False + + if warn_only and not passed and not is_error: + final_passed = True + final_warning = True + + outcome = create_outcome( + testcase_name=testcase_name, + file_path=file_entry, + passed=final_passed, + message=sanitize_xml_text(message), + meta=meta, + details=sanitize_xml_text(details), + error=is_error, + warning=final_warning, + ) + + status = ( + "ERROR" + if is_error + else ( + "WARNING" + if outcome.warning + else ("PASS" if outcome.passed else "FAIL") + ) + ) + persist_case_logs(status, outcome) + return outcome + except SkipCase as exc: + outcome = create_outcome( + testcase_name=testcase_name, + file_path=file_entry, + passed=True, + message=sanitize_xml_text(str(exc)), + meta=meta, + details=sanitize_xml_text( + f"Case skipped in work directory: {work_dir}\nReason: {exc}" + ), + skipped=True, + ) + persist_case_logs("SKIPPED", outcome) + return outcome + except (ConfigError, CaseBuildError) as exc: + outcome = create_outcome( + testcase_name=testcase_name, + file_path=file_entry, + passed=False, + message=sanitize_xml_text( + format_outcome_message("Configuration error", str(exc)) + ), + meta=meta, + details=sanitize_xml_text(traceback.format_exc()), + error=True, + ) + persist_case_logs("ERROR", outcome) + return outcome + except Exception as exc: # pylint: disable=broad-exception-caught + outcome = create_outcome( + testcase_name=testcase_name, + file_path=file_entry, + passed=False, + message=sanitize_xml_text( + format_outcome_message("Unhandled exception", str(exc)) + ), + meta=meta, + details=sanitize_xml_text(traceback.format_exc()), + error=True, + ) + persist_case_logs("ERROR", outcome) + return outcome + + +def git_changed_paths_for_query(args: list[str]) -> set[Path]: + result = subprocess.run( + ["git", *args], + cwd=PROJECT_ROOT, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + return set() + + paths: set[Path] = set() + for line in result.stdout.splitlines(): + line = line.strip() + if not line: + continue + paths.add((PROJECT_ROOT / line).resolve()) + return paths + + +def get_recently_changed_paths() -> set[Path]: + changed: set[Path] = set() + changed.update(git_changed_paths_for_query(["diff", "--name-only"])) + changed.update(git_changed_paths_for_query(["diff", "--cached", "--name-only"])) + changed.update( + git_changed_paths_for_query(["ls-files", "--others", "--exclude-standard"]) + ) + return changed + + +def path_is_within(path: Path, root: Path) -> bool: + try: + path.relative_to(root) + except ValueError: + return False + return True + + +def harness_sources_changed(changed_paths: set[Path]) -> bool: + for changed_path in changed_paths: + if not path_is_within(changed_path, HARNESS_DIR): + continue + if "__pycache__" in changed_path.parts: + continue + return True + return False + + +def normalize_target_for_matching(file_entry: str) -> Path: + return resolve_target_path(file_entry) + + +def format_yaml_config_warning(yaml_file: Path, message: str) -> str: + rel_path = yaml_file.relative_to(PROJECT_ROOT).as_posix() + return f"Skipping {rel_path} due to configuration error: {message}" + + +def load_yaml_suites_for_selection( + yaml_file: Path, +) -> tuple[list[dict[str, Any]] | None, str | None]: + try: + config = load_yaml_config(yaml_file) + suites = normalize_suites(config) + except ConfigError as exc: + return None, format_yaml_config_warning(yaml_file, str(exc)) + return suites, None + + +def get_yaml_target_entries(yaml_file: Path) -> tuple[set[str], str | None]: + suites, warning = load_yaml_suites_for_selection(yaml_file) + if suites is None: + return set(), warning + + targets: set[str] = set() + for suite in suites: + for file_entry in suite["files"]: + targets.add(Path(file_entry).as_posix()) + return targets, None + + +def yaml_targets_changed_files( + yaml_file: Path, + changed_paths: set[Path], +) -> tuple[set[str], str | None]: + suites, warning = load_yaml_suites_for_selection(yaml_file) + if suites is None: + return set(), warning + + matched: set[str] = set() + for suite in suites: + for file_entry in suite["files"]: + target_path = normalize_target_for_matching(file_entry) + if target_path in changed_paths: + matched.add(Path(file_entry).as_posix()) + return matched, None + + +def select_yaml_runs( + yaml_files: list[Path], +) -> tuple[list[tuple[Path, set[str]]], list[str]]: + changed_paths = get_recently_changed_paths() + + if not changed_paths: + return [], [] + + run_all_yaml_groups = harness_sources_changed(changed_paths) + selected_map: dict[Path, set[str]] = {} + config_warnings: list[str] = [] + + for yaml_file in yaml_files: + yaml_abs = yaml_file.resolve() + matched_targets, warning = yaml_targets_changed_files(yaml_file, changed_paths) + if warning is not None: + config_warnings.append(warning) + + if run_all_yaml_groups or yaml_abs in changed_paths: + yaml_targets, target_warning = get_yaml_target_entries(yaml_file) + if target_warning is not None and target_warning not in config_warnings: + config_warnings.append(target_warning) + matched_targets.update(yaml_targets) + + if matched_targets: + selected_map[yaml_file] = matched_targets + + return list(selected_map.items()), config_warnings + + +def select_yaml_runs_for_target( + yaml_files: list[Path], + target: str, +) -> tuple[list[tuple[Path, set[str]]], list[str]]: + target_path = resolve_target_path(target) + selected_runs: list[tuple[Path, set[str]]] = [] + config_warnings: list[str] = [] + + for yaml_file in yaml_files: + suites, warning = load_yaml_suites_for_selection(yaml_file) + if suites is None: + if warning is not None: + config_warnings.append(warning) + continue + + matched_targets: set[str] = set() + for suite in suites: + for file_entry in suite["files"]: + if resolve_target_path(file_entry) == target_path: + matched_targets.add(Path(file_entry).as_posix()) + + if matched_targets: + selected_runs.append((yaml_file, matched_targets)) + + return selected_runs, config_warnings + + + +def run_yaml( + yaml_file: Path, + selected_targets: set[str], + jobs: int = 4, +) -> int: + REPORTS_DIR.mkdir(parents=True, exist_ok=True) + + group_name = get_group_name(yaml_file) + xml_report = build_report_path(group_name, yaml_file) + + print(f"\n[INFO] Running group : {group_name}") + print( + f"[INFO] YAML file : " + f"{yaml_file.relative_to(PROJECT_ROOT).as_posix()}" + ) + print( + f"[INFO] XML report : " + f"{xml_report.relative_to(PROJECT_ROOT).as_posix()}" + ) + + try: + config = load_yaml_config(yaml_file) + suites = normalize_suites(config) + except ConfigError as exc: + outcome = build_config_error_outcome( + yaml_file=yaml_file, + message=format_outcome_message("Configuration error", str(exc)), + details=traceback.format_exc(), + ) + write_junit_xml(xml_report, group_name, yaml_file, [outcome]) + print_group_summary(group_name, [outcome], xml_report) + return 1 + + outcomes: list[TestOutcome] = [] + + for suite_index, suite in enumerate(suites, start=1): + suite_name = suite["name"] + run_case_options = RunCaseOptions( + suite_command=suite.get("command"), + ) + suite_files = suite["files"] + suite_cases = suite["cases"] + + filtered_files = [] + for file_entry in suite_files: + normalized_file_entry = Path(file_entry).as_posix() + if normalized_file_entry in selected_targets: + filtered_files.append(file_entry) + + if not filtered_files: + continue + + print(f"[INFO] Suite : {suite_name}") + + if not suite_cases: + outcomes.append( + create_outcome( + testcase_name=f"{suite_name}::no_cases", + file_path="", + passed=False, + message="Suite has no cases defined", + meta=TestMeta( + suite_name=suite_name, + phase="suite", + test_type="config", + ), + details=f"suites[{suite_index}] has an empty 'cases' list.", + error=True, + ) + ) + continue + + for file_entry in filtered_files: + print(f"[INFO] Target file : {file_entry}") + + file_work_dir = get_file_work_dir(suite_name, file_entry) + + if file_work_dir.exists(): + shutil.rmtree(file_work_dir) + file_work_dir.mkdir(parents=True, exist_ok=True) + + append_run_header( + file_work_dir=file_work_dir, + suite_name=suite_name, + yaml_file=yaml_file, + targets=[file_entry], + ) + + case_jobs = list(enumerate(suite_cases, start=1)) + file_jobs = 1 if requires_serial_case_execution(suite_cases) else jobs + + if file_jobs <= 1: + for case_index, case_def in case_jobs: + outcomes.append( + run_case( + suite_name=suite_name, + file_entry=file_entry, + case_index=case_index, + case_def=case_def, + options=run_case_options, + ) + ) + else: + case_results: list[tuple[int, TestOutcome]] = [] + with ThreadPoolExecutor(max_workers=file_jobs) as executor: + future_map = { + executor.submit( + run_case, + suite_name, + file_entry, + case_index, + case_def, + run_case_options, + ): case_index + for case_index, case_def in case_jobs + } + for future in as_completed(future_map): + case_index = future_map[future] + case_results.append((case_index, future.result())) + + for _case_index, outcome in sorted( + case_results, + key=lambda item: item[0], + ): + outcomes.append(outcome) + + if not outcomes: + return 0 + + write_junit_xml(xml_report, group_name, yaml_file, outcomes) + print_group_summary(group_name, outcomes, xml_report) + + return 0 if all(item.passed or item.skipped or item.warning for item in outcomes) else 1 + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Run YAML-driven tests for impacted files or a manual target." + ) + parser.add_argument( + "--target", + help="Run tests only for the specified Python file target from YAML suites", + ) + parser.add_argument( + "--jobs", + type=int, + default=4, + help="Number of YAML test cases to run in parallel (default: 4)", + ) + args = parser.parse_args() + + jobs = max(1, args.jobs) + + REPORTS_DIR.mkdir(parents=True, exist_ok=True) + cleanup_old_pytest_xml_reports() + + yaml_files = discover_yaml_files() + + if not yaml_files: + reason = ( + f"No YAML files found in " + f"{TEST_YAML_DIR.relative_to(PROJECT_ROOT).as_posix()}" + ) + print(f"WARNING: {reason}") + create_placeholder_xml(reason) + return 0 + + remove_placeholder_xml() + + if args.target: + print(f"[INFO] Manual target override enabled: {args.target}") + selected_runs, config_warnings = select_yaml_runs_for_target( + yaml_files, + args.target, + ) + else: + selected_runs, config_warnings = select_yaml_runs(yaml_files) + + for warning in config_warnings: + print(f"[WARNING] {warning}") + + if not selected_runs: + if args.target: + if config_warnings: + message = ( + f"No valid YAML test groups found for manual target: {args.target}" + ) + print(f"[ERROR] {message}") + print( + f"[INFO] Reports written to: " + f"{REPORTS_DIR.relative_to(PROJECT_ROOT).as_posix()}" + ) + create_placeholder_xml(message) + return 1 + message = f"No YAML test groups found for manual target: {args.target}" + else: + message = "No impacted YAML test groups found for changed files." + + print(f"[INFO] {message}") + print( + f"[INFO] Reports written to: " + f"{REPORTS_DIR.relative_to(PROJECT_ROOT).as_posix()}" + ) + create_placeholder_xml(message) + return 0 + + if args.target: + print("[INFO] Running YAML test groups for manual target:") + else: + print("[INFO] Running only impacted YAML test groups:") + + for yaml_file, selected_targets in selected_runs: + print(f" - {yaml_file.relative_to(PROJECT_ROOT).as_posix()}") + for target in sorted(selected_targets): + print(f" * target: {target}") + + overall_exit_code = 0 + for yaml_file, selected_targets in selected_runs: + exit_code = run_yaml( + yaml_file, + selected_targets=selected_targets, + jobs=jobs, + ) + if exit_code != 0: + overall_exit_code = exit_code + + print("\n[INFO] Custom YAML test execution completed.") + print( + f"[INFO] Reports written to: " + f"{REPORTS_DIR.relative_to(PROJECT_ROOT).as_posix()}" + ) + return overall_exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/common/acs_test_framework_runner/report.py b/common/acs_test_framework_runner/report.py new file mode 100644 index 00000000..198a7453 --- /dev/null +++ b/common/acs_test_framework_runner/report.py @@ -0,0 +1,992 @@ +from __future__ import annotations + +import argparse +import re +import shutil +import subprocess +import sys +import xml.etree.ElementTree as ET +from datetime import datetime +from pathlib import Path + +try: # Support package imports and direct harness module loading. + from .runner_checks import ( + ConfigError, + detect_project_root, + load_yaml_config, + normalize_suites, + resolve_target_path, + sanitize_name, + ) +except ImportError: # pragma: no cover - exercised by flat-module harness imports. + from runner_checks import ( + ConfigError, + detect_project_root, + load_yaml_config, + normalize_suites, + resolve_target_path, + sanitize_name, + ) + + +SCRIPT_DIR = Path(__file__).resolve().parent +PROJECT_ROOT = detect_project_root(SCRIPT_DIR) + +RUNNER_FILE = SCRIPT_DIR / "pytest_runner.py" +REPORTS_DIR = PROJECT_ROOT / "common" / "reports" +TEST_YAML_DIR = PROJECT_ROOT / "common" / "acs_test_framework_manifests" +SUPPORTED_YAML_SUFFIXES = {".yaml", ".yml"} + +PYLINT_XML = REPORTS_DIR / "pylint-report.xml" +PYLINT_LOG = REPORTS_DIR / "pylint.log" + +MYPY_XML = REPORTS_DIR / "mypy-report.xml" +MYPY_LOG = REPORTS_DIR / "mypy.log" + +PYTEST_LOG = REPORTS_DIR / "pytest.log" + +MAX_LOG_ENTRIES = 50 +LOG_SEPARATOR = "=" * 100 + + +class Color: + GREEN = "\033[92m" + RED = "\033[91m" + YELLOW = "\033[93m" + BLUE = "\033[94m" + RESET = "\033[0m" + + +def color_text(text: str, color: str) -> str: + return f"{color}{text}{Color.RESET}" + + +def ensure_reports_dir() -> None: + REPORTS_DIR.mkdir(parents=True, exist_ok=True) + + +def append_log_history(log_path: Path, content: str, max_entries: int = MAX_LOG_ENTRIES) -> None: + ensure_reports_dir() + + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + normalized_content = content.rstrip("\n") + new_entry = ( + f"{LOG_SEPARATOR}\n" + f"RUN AT: {timestamp}\n" + f"{LOG_SEPARATOR}\n" + f"{normalized_content}\n" + ) + + if log_path.exists(): + existing = log_path.read_text(encoding="utf-8") + else: + existing = "" + + existing = existing.strip() + entries: list[str] = [] + + if existing: + parts = re.split( + rf"(?m)(?=^{re.escape(LOG_SEPARATOR)}\nRUN AT: )", + existing, + ) + entries = [part.strip() for part in parts if part.strip()] + + entries.append(new_entry.strip()) + entries = entries[-max_entries:] + + final_content = "\n\n".join(entries) + "\n" + log_path.write_text(final_content, encoding="utf-8") + + +def run_git_command(args: list[str]) -> tuple[int, str, str]: + result = subprocess.run( + ["git", *args], + cwd=PROJECT_ROOT, + capture_output=True, + text=True, + check=False, + ) + return result.returncode, result.stdout.strip(), result.stderr.strip() + + +def get_commit_info() -> dict[str, str]: + info = { + "commit": "NO_COMMIT_AVAILABLE", + "branch": "UNKNOWN_BRANCH", + "status": "UNKNOWN_STATUS", + } + + code, stdout, _ = run_git_command(["rev-parse", "--is-inside-work-tree"]) + if code != 0 or stdout.lower() != "true": + info["status"] = "NOT_A_GIT_REPOSITORY" + return info + + code, stdout, _ = run_git_command(["rev-parse", "--abbrev-ref", "HEAD"]) + if code == 0 and stdout: + info["branch"] = stdout + + code, stdout, _ = run_git_command(["rev-parse", "HEAD"]) + if code == 0 and stdout: + info["commit"] = stdout + info["status"] = "COMMIT_FOUND" + else: + info["status"] = "NO_COMMIT_YET" + + return info + + +def resolve_manual_target(target: str | None) -> Path | None: + if not target: + return None + + raw = Path(target) + if raw.is_absolute(): + return raw.resolve() + return (PROJECT_ROOT / raw).resolve() + + +def get_manual_python_target(target: str | None, tool_name: str) -> list[Path] | None: + resolved_target = resolve_manual_target(target) + if resolved_target is None: + return None + + if not resolved_target.exists() or not resolved_target.is_file(): + print( + f"{color_text('WARNING:', Color.YELLOW)} " + f"Manual target for {tool_name} not found: {resolved_target}" + ) + return [] + + if resolved_target.suffix != ".py": + print( + f"{color_text('WARNING:', Color.YELLOW)} " + f"Manual target is not a Python file for {tool_name}: {resolved_target}" + ) + return [] + + return [resolved_target] + + +def is_generated_report_artifact(path: Path) -> bool: + """Return True when a path points inside a generated report work directory.""" + try: + rel_path = path.relative_to(PROJECT_ROOT) + except ValueError: + return False + + parts = rel_path.parts + if len(parts) >= 3 and parts[:2] == ("common", "reports"): + return parts[2].startswith(("_", "tpm_check_")) + + if len(parts) >= 2 and parts[0] == "reports": + return parts[1].startswith(("_", "tpm_check_")) + + return False + + +def is_yaml_suite_file(path: Path) -> bool: + try: + path.relative_to(TEST_YAML_DIR) + except ValueError: + return False + return path.is_file() and path.suffix.lower() in SUPPORTED_YAML_SUFFIXES + + +def get_python_targets_from_yaml_file(yaml_file: Path) -> set[Path]: + try: + config = load_yaml_config(yaml_file) + suites = normalize_suites(config) + except ConfigError: + return set() + + targets: set[Path] = set() + for suite in suites: + for file_entry in suite["files"]: + target_path = resolve_target_path(file_entry) + if target_path.suffix != ".py": + continue + if target_path.exists() and target_path.is_file(): + targets.add(target_path) + return targets + + +def get_recently_changed_paths() -> list[Path]: + changed_files: set[str] = set() + + git_queries = [ + ["diff", "--name-only"], + ["diff", "--cached", "--name-only"], + ["ls-files", "--others", "--exclude-standard"], + ] + + for query in git_queries: + code, stdout, _ = run_git_command(query) + if code != 0: + continue + + for item in stdout.splitlines(): + item = item.strip() + if item: + changed_files.add(item) + + paths: list[Path] = [] + for item in sorted(changed_files): + path = (PROJECT_ROOT / item).resolve() + if path.exists() and path.is_file() and not is_generated_report_artifact(path): + paths.append(path) + + return paths + + +def run_pytest(target: str | None = None, jobs: int = 4) -> tuple[int, str, str]: + if not RUNNER_FILE.exists(): + return 1, "", f"ERROR: Runner file not found: {RUNNER_FILE}" + + cmd = [sys.executable, str(RUNNER_FILE)] + if target: + cmd.extend(["--target", target]) + cmd.extend(["--jobs", str(max(1, jobs))]) + + result = subprocess.run( + cmd, + cwd=PROJECT_ROOT, + capture_output=True, + text=True, + check=False, + ) + + stdout = result.stdout or "" + stderr = result.stderr or "" + + log_content = ( + "STDOUT\n" + + "=" * 80 + + "\n" + + stdout + + "\n\nSTDERR\n" + + "=" * 80 + + "\n" + + stderr + + "\n" + ) + append_log_history(PYTEST_LOG, log_content) + + return result.returncode, stdout, stderr + + +def collect_pytest_case_logs(target: str | None = None) -> list[dict[str, str]]: + work_root = REPORTS_DIR / "_work" + if not work_root.exists(): + return [] + + testcases: list[dict[str, str]] = [] + seen_names: set[str] = set() + + pattern = re.compile( + r"(?ms)^=+\n" + r"TEST CASE:\s*(?P.+?)\n" + r"(?:DESCRIPTION:\s*(?P.+?)\n)?" + r"STATUS:\s*(?P.+?)\n" + r"MESSAGE:\s*(?P.*?)(?=\n=+\nTEST CASE:|\Z)" + ) + + log_paths: list[Path] + resolved_target = resolve_manual_target(target) + + if resolved_target is not None: + target_stem = sanitize_name(resolved_target.stem) + log_paths = sorted(work_root.rglob(f"{target_stem}/combined.log")) + else: + log_paths = sorted(work_root.rglob("combined.log")) + + for log_path in log_paths: + content = log_path.read_text(encoding="utf-8", errors="replace") + + for match in pattern.finditer(content): + name = match.group("name").strip() + if name in seen_names: + continue + seen_names.add(name) + + raw_status = match.group("status").strip().upper() + + if raw_status == "PASS": + normalized_status = "passed" + elif raw_status == "FAIL": + normalized_status = "failed" + elif raw_status == "ERROR": + normalized_status = "error" + elif raw_status in {"WARNING", "SKIPPED"}: + normalized_status = "warning" + else: + normalized_status = raw_status.lower() + + testcases.append( + { + "name": name, + "description": (match.group("description") or "").strip(), + "status": normalized_status, + "details": match.group("message").strip(), + } + ) + + return testcases + + +def print_pytest_summary(commit_info: dict[str, str], target: str | None = None) -> None: + print(f"\n{color_text('========== PYTEST REPORT ==========', Color.BLUE)}\n") + print(f"Branch : {commit_info['branch']}") + print(f"Commit : {commit_info['commit']}") + print(f"Git : {commit_info['status']}") + + testcases = collect_pytest_case_logs(target) + + if not testcases: + print( + f"\n{color_text('WARNING:', Color.YELLOW)} " + "No pytest case logs were found." + ) + print(f"\nFull pytest log: {PYTEST_LOG.relative_to(PROJECT_ROOT)}") + print(f"\n{color_text('===================================', Color.BLUE)}\n") + return + + total = len(testcases) + passed = sum(1 for tc in testcases if tc["status"] == "passed") + failed = sum(1 for tc in testcases if tc["status"] == "failed") + errors = sum(1 for tc in testcases if tc["status"] == "error") + warnings = sum(1 for tc in testcases if tc["status"] == "warning") + + print(f"\n{color_text('Total :', Color.BLUE)} {total}") + print(f"{color_text('Passed :', Color.GREEN)} {passed}") + print(f"{color_text('Failed :', Color.RED)} {failed}") + print(f"{color_text('Errors :', Color.RED)} {errors}") + print(f"{color_text('Warnings:', Color.YELLOW)} {warnings}") + + print(f"\n{color_text('All Test Cases:', Color.BLUE)}") + + warning_groups: dict[str, list[dict[str, str]]] = {} + + for tc in testcases: + status = tc["status"] + + if status == "passed": + label = color_text("[passed]", Color.GREEN) + print(f" - {tc['name']} {label}") + elif status in {"failed", "error"}: + label = color_text(f"[{status}]", Color.RED) + print(f" - {tc['name']} {label}") + elif status == "warning": + short_msg = tc["details"].splitlines()[0].strip() if tc["details"] else "Warning" + warning_groups.setdefault(short_msg, []).append(tc) + else: + label = color_text(f"[{status}]", Color.YELLOW) + print(f" - {tc['name']} {label}") + + single_warning_groups = { + msg: items for msg, items in warning_groups.items() if len(items) == 1 + } + repeated_warning_groups = { + msg: items for msg, items in warning_groups.items() if len(items) > 1 + } + + for _msg, items in single_warning_groups.items(): + tc = items[0] + print(f" - {tc['name']} {color_text('[warning]', Color.YELLOW)}") + + if repeated_warning_groups: + print(f"\n{color_text('Collapsed Repeated Warnings:', Color.YELLOW)}") + for msg, items in repeated_warning_groups.items(): + print(f" - {color_text('[warning]', Color.YELLOW)} {msg} ({len(items)} case(s))") + for tc in items: + print(f" * {tc['name']}") + + print(f"\nFull pytest log: {PYTEST_LOG.relative_to(PROJECT_ROOT)}") + print(f"\n{color_text('===================================', Color.BLUE)}\n") + + +def get_recently_changed_python_files() -> list[Path]: + python_files: set[Path] = set() + + for path in get_recently_changed_paths(): + if path.suffix == ".py": + python_files.add(path) + continue + if is_yaml_suite_file(path): + python_files.update(get_python_targets_from_yaml_file(path)) + + return sorted(python_files) + + +def extract_pylint_score(output: str) -> str: + score_patterns = [ + r"rated at\s+(-?\d+(?:\.\d+)?)\/10", + r"Your code has been rated at\s+(-?\d+(?:\.\d+)?)\/10", + ] + + for pattern in score_patterns: + match = re.search(pattern, output, flags=re.IGNORECASE) + if match: + return f"{match.group(1)}/10" + + return "N/A" + + +def run_pylint_command(cmd: list[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + cmd, + cwd=PROJECT_ROOT, + capture_output=True, + text=True, + check=False, + ) + + +def run_pylint(target: str | None = None) -> int: + ensure_reports_dir() + + pylint_exe = shutil.which("pylint") + if pylint_exe is None: + print(f"{color_text('WARNING:', Color.YELLOW)} pylint not found in PATH.") + create_empty_pylint_xml("pylint not installed") + append_log_history(PYLINT_LOG, "pylint not installed") + return 0 + + manual_targets = get_manual_python_target(target, "pylint") + targets = ( + manual_targets + if manual_targets is not None + else get_recently_changed_python_files() + ) + + if not targets: + print( + f"{color_text('WARNING:', Color.YELLOW)} " + "No recently changed Python files found by git for pylint." + ) + create_empty_pylint_xml("no recently changed python files found") + append_log_history(PYLINT_LOG, "no recently changed python files found") + return 0 + + print( + color_text( + "[INFO] Running pylint on recently changed Python files:", + Color.BLUE, + ) + ) + for target_path in targets: + try: + rel = target_path.relative_to(PROJECT_ROOT) + except ValueError: + rel = target_path + print(f" - {rel}") + + parseable_cmd = [ + pylint_exe, + "--output-format=parseable", + "--score=y", + *[str(path) for path in targets], + ] + score_cmd = [ + pylint_exe, + "--score=y", + *[str(path) for path in targets], + ] + + parseable_result = run_pylint_command(parseable_cmd) + score_result = run_pylint_command(score_cmd) + pylint_score = extract_pylint_score( + "\n".join( + [ + score_result.stdout or "", + score_result.stderr or "", + parseable_result.stdout or "", + parseable_result.stderr or "", + ] + ) + ) + + log_content = ( + "PARSEABLE STDOUT\n" + + "=" * 80 + + "\n" + + (parseable_result.stdout or "") + + "\n\nPARSEABLE STDERR\n" + + "=" * 80 + + "\n" + + (parseable_result.stderr or "") + + "\n\nSCORE STDOUT\n" + + "=" * 80 + + "\n" + + (score_result.stdout or "") + + "\n\nSCORE STDERR\n" + + "=" * 80 + + "\n" + + (score_result.stderr or "") + + "\n\nEXTRACTED SCORE\n" + + "=" * 80 + + "\n" + + pylint_score + + "\n" + ) + append_log_history(PYLINT_LOG, log_content) + + print( + f"{color_text('[INFO]', Color.BLUE)} " + f"Full pylint log saved to: {PYLINT_LOG.relative_to(PROJECT_ROOT)}" + ) + + write_pylint_xml( + parseable_stdout=parseable_result.stdout, + parseable_stderr=parseable_result.stderr, + targets=targets, + pylint_score=pylint_score, + ) + return parseable_result.returncode + + +def write_pylint_xml( + parseable_stdout: str, + parseable_stderr: str, + targets: list[Path], + pylint_score: str, +) -> None: + commit_info = get_commit_info() + + testsuite = ET.Element("testsuite") + testsuite.set("name", "pylint") + testsuite.set("tests", str(len(targets))) + + failures = 0 + issues_by_file: dict[str, list[str]] = {} + + for line in parseable_stdout.splitlines(): + line = line.strip() + if not line: + continue + + file_key = line.split(":", 1)[0].strip() + issues_by_file.setdefault(file_key, []).append(line) + + properties = ET.SubElement(testsuite, "properties") + + score_prop = ET.SubElement(properties, "property") + score_prop.set("name", "pylint_score") + score_prop.set("value", pylint_score) + + for target in targets: + try: + rel = str(target.relative_to(PROJECT_ROOT)) + except ValueError: + rel = str(target) + + testcase = ET.SubElement(testsuite, "testcase") + testcase.set("classname", "pylint") + testcase.set("name", rel) + + issues = ( + issues_by_file.get(str(target), []) + or issues_by_file.get(rel, []) + ) + if issues: + failures += 1 + failure = ET.SubElement(testcase, "failure") + failure.set("message", f"{len(issues)} pylint issue(s)") + failure.text = "\n".join(issues) + + testsuite.set("failures", str(failures)) + testsuite.set("errors", "0") + testsuite.set("skipped", "0") + + system_out = ET.SubElement(testsuite, "system-out") + system_out.text = ( + f"branch={commit_info['branch']}\n" + f"commit={commit_info['commit']}\n" + f"git_status={commit_info['status']}\n" + f"pylint_score={pylint_score}\n" + f"{parseable_stderr.strip()}".strip() + ) + + tree = ET.ElementTree(testsuite) + tree.write(PYLINT_XML, encoding="utf-8", xml_declaration=True) + + +def create_empty_pylint_xml(reason: str) -> None: + commit_info = get_commit_info() + + testsuite = ET.Element("testsuite") + testsuite.set("name", "pylint") + testsuite.set("tests", "0") + testsuite.set("failures", "0") + testsuite.set("errors", "0") + testsuite.set("skipped", "0") + + properties = ET.SubElement(testsuite, "properties") + score_prop = ET.SubElement(properties, "property") + score_prop.set("name", "pylint_score") + score_prop.set("value", "N/A") + + system_out = ET.SubElement(testsuite, "system-out") + system_out.text = ( + f"branch={commit_info['branch']}\n" + f"commit={commit_info['commit']}\n" + f"git_status={commit_info['status']}\n" + f"pylint_score=N/A\n" + f"reason={reason}" + ) + + tree = ET.ElementTree(testsuite) + tree.write(PYLINT_XML, encoding="utf-8", xml_declaration=True) + + +def get_pylint_score_from_xml(root: ET.Element) -> str: + properties = root.find("properties") + if properties is not None: + for prop in properties.findall("property"): + if prop.attrib.get("name") == "pylint_score": + return prop.attrib.get("value", "N/A") + + system_out = root.findtext("system-out", default="") + match = re.search(r"pylint_score=(.+)", system_out) + if match: + return match.group(1).strip() + + return "N/A" + + +def print_pylint_summary(commit_info: dict[str, str]) -> None: + print(f"\n{color_text('========== PYLINT REPORT ==========', Color.BLUE)}\n") + print(f"Branch : {commit_info['branch']}") + print(f"Commit : {commit_info['commit']}") + print(f"Git : {commit_info['status']}") + + if not PYLINT_XML.exists(): + print(f"\n{color_text('WARNING:', Color.YELLOW)} No pylint XML report found.") + print(f"\n{color_text('===================================', Color.BLUE)}\n") + return + + try: + tree = ET.parse(PYLINT_XML) + root = tree.getroot() + except Exception as exc: + print( + f"\n{color_text('WARNING:', Color.YELLOW)} " + f"Failed to parse pylint XML report: {exc}" + ) + print(f"\n{color_text('===================================', Color.BLUE)}\n") + return + + total = int(root.attrib.get("tests", 0)) + failures = int(root.attrib.get("failures", 0)) + errors = int(root.attrib.get("errors", 0)) + skipped = int(root.attrib.get("skipped", 0)) + passed = total - failures - errors - skipped + pylint_score = get_pylint_score_from_xml(root) + + print(f"\n{color_text('Total :', Color.BLUE)} {total}") + print(f"{color_text('Passed :', Color.GREEN)} {passed}") + print(f"{color_text('Failed :', Color.RED)} {failures}") + print(f"{color_text('Errors :', Color.RED)} {errors}") + print(f"{color_text('Warnings:', Color.YELLOW)} {skipped}") + print(f"{color_text('Score :', Color.BLUE)} {pylint_score}") + + print(f"\n{color_text('Detailed Issues:', Color.BLUE)}\n") + + for testcase in root.findall("testcase"): + failure = testcase.find("failure") + if failure is None: + continue + + file_name = testcase.attrib.get("name", "unknown") + details = (failure.text or "").strip().splitlines() + + print(color_text(file_name, Color.RED)) + for line in details: + parts = line.split(":", 2) + if len(parts) >= 3: + line_no = parts[1] + rest = parts[2].strip() + print(color_text(f" - line {line_no}: {rest}", Color.RED)) + else: + print(color_text(f" - {line}", Color.RED)) + print() + + system_out = root.findtext("system-out", default="").strip() + if system_out and total == 0: + print(f"Note: {system_out}") + + print(f"\nFull pylint log: {PYLINT_LOG.relative_to(PROJECT_ROOT)}") + print(f"\n{color_text('===================================', Color.BLUE)}\n") + + +def run_mypy_command(cmd: list[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + cmd, + cwd=PROJECT_ROOT, + capture_output=True, + text=True, + check=False, + ) + + +def run_mypy(target: str | None = None) -> int: + ensure_reports_dir() + + mypy_exe = shutil.which("mypy") + if mypy_exe is None: + print(f"{color_text('WARNING:', Color.YELLOW)} mypy not found in PATH.") + create_empty_mypy_xml("mypy not installed") + append_log_history(MYPY_LOG, "mypy not installed") + return 0 + + manual_targets = get_manual_python_target(target, "mypy") + targets = ( + manual_targets + if manual_targets is not None + else get_recently_changed_python_files() + ) + + if not targets: + print( + f"{color_text('WARNING:', Color.YELLOW)} " + "No recently changed Python files found by git for mypy." + ) + create_empty_mypy_xml("no recently changed python files found") + append_log_history(MYPY_LOG, "no recently changed python files found") + return 0 + + print( + color_text( + "[INFO] Running mypy on recently changed Python files:", + Color.BLUE, + ) + ) + for target_path in targets: + try: + rel = target_path.relative_to(PROJECT_ROOT) + except ValueError: + rel = target_path + print(f" - {rel}") + + cmd = [ + mypy_exe, + "--show-error-codes", + "--no-color-output", + "--no-error-summary", + *[str(path) for path in targets], + ] + result = run_mypy_command(cmd) + + log_content = ( + "STDOUT\n" + + "=" * 80 + + "\n" + + (result.stdout or "") + + "\n\nSTDERR\n" + + "=" * 80 + + "\n" + + (result.stderr or "") + + "\n" + ) + append_log_history(MYPY_LOG, log_content) + + print( + f"{color_text('[INFO]', Color.BLUE)} " + f"Full mypy log saved to: {MYPY_LOG.relative_to(PROJECT_ROOT)}" + ) + + write_mypy_xml( + stdout=result.stdout, + stderr=result.stderr, + targets=targets, + ) + return result.returncode + + +def write_mypy_xml( + stdout: str, + stderr: str, + targets: list[Path], +) -> None: + commit_info = get_commit_info() + + testsuite = ET.Element("testsuite") + testsuite.set("name", "mypy") + testsuite.set("tests", str(len(targets))) + + failures = 0 + issues_by_file: dict[str, list[str]] = {} + + for raw_line in stdout.splitlines(): + line = raw_line.strip() + if not line: + continue + + match = re.match( + r"^(.*?\.py)(?::\d+)?(?::\d+)?:\s*(error|note):\s*(.*)$", + line, + ) + if match: + file_key = match.group(1).strip() + issues_by_file.setdefault(file_key, []).append(line) + + for target in targets: + try: + rel = str(target.relative_to(PROJECT_ROOT)) + except ValueError: + rel = str(target) + + testcase = ET.SubElement(testsuite, "testcase") + testcase.set("classname", "mypy") + testcase.set("name", rel) + + issues = ( + issues_by_file.get(str(target), []) + or issues_by_file.get(rel, []) + ) + if issues: + failures += 1 + failure = ET.SubElement(testcase, "failure") + failure.set("message", f"{len(issues)} mypy issue(s)") + failure.text = "\n".join(issues) + + testsuite.set("failures", str(failures)) + testsuite.set("errors", "0") + testsuite.set("skipped", "0") + + system_out = ET.SubElement(testsuite, "system-out") + system_out.text = ( + f"branch={commit_info['branch']}\n" + f"commit={commit_info['commit']}\n" + f"git_status={commit_info['status']}\n" + f"{stderr.strip()}".strip() + ) + + tree = ET.ElementTree(testsuite) + tree.write(MYPY_XML, encoding="utf-8", xml_declaration=True) + + +def create_empty_mypy_xml(reason: str) -> None: + commit_info = get_commit_info() + + testsuite = ET.Element("testsuite") + testsuite.set("name", "mypy") + testsuite.set("tests", "0") + testsuite.set("failures", "0") + testsuite.set("errors", "0") + testsuite.set("skipped", "0") + + ET.SubElement(testsuite, "properties") + + system_out = ET.SubElement(testsuite, "system-out") + system_out.text = ( + f"branch={commit_info['branch']}\n" + f"commit={commit_info['commit']}\n" + f"git_status={commit_info['status']}\n" + f"reason={reason}" + ) + + tree = ET.ElementTree(testsuite) + tree.write(MYPY_XML, encoding="utf-8", xml_declaration=True) + + +def print_mypy_summary(commit_info: dict[str, str]) -> None: + print(f"\n{color_text('=========== MYPY REPORT ===========', Color.BLUE)}\n") + print(f"Branch : {commit_info['branch']}") + print(f"Commit : {commit_info['commit']}") + print(f"Git : {commit_info['status']}") + + if not MYPY_XML.exists(): + print(f"\n{color_text('WARNING:', Color.YELLOW)} No mypy XML report found.") + print(f"\n{color_text('===================================', Color.BLUE)}\n") + return + + try: + tree = ET.parse(MYPY_XML) + root = tree.getroot() + except Exception as exc: + print( + f"\n{color_text('WARNING:', Color.YELLOW)} " + f"Failed to parse mypy XML report: {exc}" + ) + print(f"\n{color_text('===================================', Color.BLUE)}\n") + return + + total = int(root.attrib.get("tests", 0)) + failures = int(root.attrib.get("failures", 0)) + errors = int(root.attrib.get("errors", 0)) + skipped = int(root.attrib.get("skipped", 0)) + passed = total - failures - errors - skipped + + print(f"\n{color_text('Total :', Color.BLUE)} {total}") + print(f"{color_text('Passed :', Color.GREEN)} {passed}") + print(f"{color_text('Failed :', Color.RED)} {failures}") + print(f"{color_text('Errors :', Color.RED)} {errors}") + print(f"{color_text('Warnings:', Color.YELLOW)} {skipped}") + + print(f"\n{color_text('Detailed Issues:', Color.BLUE)}\n") + + for testcase in root.findall("testcase"): + failure = testcase.find("failure") + if failure is None: + continue + + file_name = testcase.attrib.get("name", "unknown") + details = (failure.text or "").strip().splitlines() + + print(color_text(file_name, Color.RED)) + for line in details: + print(color_text(f" - {line}", Color.RED)) + print() + + system_out = root.findtext("system-out", default="").strip() + if system_out and total == 0: + print(f"Note: {system_out}") + + print(f"\nFull mypy log: {MYPY_LOG.relative_to(PROJECT_ROOT)}") + print(f"\n{color_text('===================================', Color.BLUE)}\n") + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Generate test reports and run analysis tools." + ) + parser.add_argument( + "target", + nargs="?", + help="Manual target file for focused analysis", + ) + parser.add_argument( + "--jobs", + type=int, + default=4, + help="Number of YAML test cases to run in parallel (default: 4)", + ) + args = parser.parse_args() + + ensure_reports_dir() + commit_info = get_commit_info() + + if args.target: + print( + f"{color_text('[INFO]', Color.BLUE)} " + f"Manual target override enabled: {args.target}" + ) + + pytest_exit_code, _pytest_stdout, _pytest_stderr = run_pytest(args.target, jobs=args.jobs) + pylint_exit_code = run_pylint(args.target) + mypy_exit_code = run_mypy(args.target) + + print_pytest_summary(commit_info, args.target) + print_pylint_summary(commit_info) + print_mypy_summary(commit_info) + + if pytest_exit_code != 0: + return pytest_exit_code + if pylint_exit_code != 0: + return pylint_exit_code + return mypy_exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/common/acs_test_framework_runner/runner_check_execution.py b/common/acs_test_framework_runner/runner_check_execution.py new file mode 100644 index 00000000..bf448f90 --- /dev/null +++ b/common/acs_test_framework_runner/runner_check_execution.py @@ -0,0 +1,1210 @@ +from __future__ import annotations + +import importlib.util +import io +import os +import py_compile +import re +import shlex +import shutil +import subprocess +import sys +import traceback +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path +from typing import Any, Callable + +try: # Support package imports and direct harness module loading. + from .case_data_builders import ( + CaseBuildError, + expand_template, + materialize_case_workspace, + prepare_case_files, + ) + from .mock_helpers import MockExpectationError + from .mock_loader import ConfigError as MockLoaderConfigError + from .mock_loader import apply_case_mocks, build_case_runtime_definition + from .runner_checks import ( + DEFAULT_CLI_TIMEOUT_SEC, + DESTRUCTIVE_TEST_ENV, + REAL_SUBPROCESS_RUN, + CommandRunResult, + ConfigError, + SkipCase, + collect_function_names, + create_runner_temp_dir, + ensure_list, + ensure_list_of_strings, + ensure_string_or_list_of_strings, + load_module_from_path, + normalize_completed_stream, + read_source, + sanitize_xml_text, + ) +except ImportError: # pragma: no cover - exercised by flat-module harness imports. + from case_data_builders import ( + CaseBuildError, + expand_template, + materialize_case_workspace, + prepare_case_files, + ) + from mock_helpers import MockExpectationError + from mock_loader import ConfigError as MockLoaderConfigError + from mock_loader import apply_case_mocks, build_case_runtime_definition + from runner_checks import ( + DEFAULT_CLI_TIMEOUT_SEC, + DESTRUCTIVE_TEST_ENV, + REAL_SUBPROCESS_RUN, + CommandRunResult, + ConfigError, + SkipCase, + collect_function_names, + create_runner_temp_dir, + ensure_list, + ensure_list_of_strings, + ensure_string_or_list_of_strings, + load_module_from_path, + normalize_completed_stream, + read_source, + sanitize_xml_text, + ) + + +def append_log_file_details( + details_lines: list[str], + runtime_case: dict[str, Any], +) -> None: + try: # Support package imports and direct harness module loading. + from .runner_checks import append_log_file_details as shared_append_log_file_details + except ImportError: # pragma: no cover - exercised by flat-module harness imports. + from runner_checks import append_log_file_details as shared_append_log_file_details + + shared_append_log_file_details(details_lines, runtime_case) + + +def run_post_checks(work_dir: Path, post_checks: Any) -> tuple[bool, list[str]]: + try: # Support package imports and direct harness module loading. + from .runner_checks import run_post_checks as shared_run_post_checks + except ImportError: # pragma: no cover - exercised by flat-module harness imports. + from runner_checks import run_post_checks as shared_run_post_checks + + return shared_run_post_checks(work_dir, post_checks) + + +def validate_output_expectations( + case_def: dict[str, Any], + stdout: str, + stderr: str, +) -> tuple[bool, list[str]]: + try: # Support package imports and direct harness module loading. + from .runner_checks import ( + validate_output_expectations as shared_validate_output_expectations, + ) + except ImportError: # pragma: no cover - exercised by flat-module harness imports. + from runner_checks import ( + validate_output_expectations as shared_validate_output_expectations, + ) + + return shared_validate_output_expectations(case_def, stdout, stderr) + + +def build_command_from_spec( + file_path: Path, + spec: dict[str, Any], + work_dir: Path, +) -> tuple[list[str] | str, bool]: + raw_args = spec.get("args", []) + if not isinstance(raw_args, list) or not all(isinstance(arg, str) for arg in raw_args): + raise ConfigError("'args' must be a list of strings") + + command_value = spec.get("command") + if not isinstance(command_value, str): + raise ConfigError("'command' must be a string") + + shell_mode = bool(spec.get("shell", False)) + expanded_command = expand_template(command_value, work_dir, file_path) + expanded_args = [expand_template(arg, work_dir, file_path) for arg in raw_args] + + if shell_mode: + parts = [expanded_command, *[shlex.quote(arg) for arg in expanded_args]] + return " ".join(parts), True + + return [expanded_command, *expanded_args], False + + +def build_runtime_env( + file_path: Path, + env_spec: Any, + work_dir: Path, +) -> dict[str, str]: + env = os.environ.copy() + if env_spec is None: + return env + + if not isinstance(env_spec, dict): + raise ConfigError("'env' must be a mapping") + + for key, value in env_spec.items(): + if not isinstance(key, str) or not isinstance(value, str): + raise ConfigError("'env' entries must be string -> string") + env[key] = expand_template(value, work_dir, file_path) + + return env + + +def execute_command_spec( + file_path: Path, + spec: dict[str, Any], + work_dir: Path, + stdin_text: str | None = None, +) -> CommandRunResult: + timeout_sec = spec.get("timeout_sec", DEFAULT_CLI_TIMEOUT_SEC) + if not isinstance(timeout_sec, int) or timeout_sec <= 0: + raise ConfigError("'timeout_sec' must be a positive integer") + + cmd, shell_mode = build_command_from_spec(file_path, spec, work_dir) + run_env = build_runtime_env(file_path, spec.get("env"), work_dir) + + cwd_value = spec.get("cwd") + if cwd_value is None: + cwd = work_dir + else: + if not isinstance(cwd_value, str): + raise ConfigError("'cwd' must be a string") + cwd = Path(expand_template(cwd_value, work_dir, file_path)) + + timed_out = False + stdout = "" + stderr = "" + exit_code: int | None = None + + try: + completed = REAL_SUBPROCESS_RUN( + cmd, + cwd=str(cwd), + capture_output=True, + text=True, + input=stdin_text, + check=False, + env=run_env, + timeout=timeout_sec, + shell=shell_mode, + ) + stdout = normalize_completed_stream(completed.stdout) + stderr = normalize_completed_stream(completed.stderr) + exit_code = completed.returncode + except subprocess.TimeoutExpired as exc: + timed_out = True + stdout = normalize_completed_stream(exc.stdout) + stderr = normalize_completed_stream(exc.stderr) + + command_text = cmd if isinstance(cmd, str) else " ".join(cmd) + return CommandRunResult( + command_text=command_text, + stdout=stdout, + stderr=stderr, + exit_code=exit_code, + timed_out=timed_out, + timeout_sec=timeout_sec, + shell_mode=shell_mode, + ) + + +def normalize_probe_spec(probe: Any) -> dict[str, Any]: + if isinstance(probe, str): + return { + "command": probe, + "args": [], + "shell": True, + "timeout_sec": DEFAULT_CLI_TIMEOUT_SEC, + } + if isinstance(probe, dict): + return dict(probe) + raise ConfigError("Command probe must be a string or mapping") + + +def apply_skip_controls( + file_path: Path, + case_def: dict[str, Any], + work_dir: Path, +) -> None: + skip_unless_env = case_def.get("skip_unless_env", {}) + if skip_unless_env: + if not isinstance(skip_unless_env, dict): + raise ConfigError("'skip_unless_env' must be a mapping") + for key, expected in skip_unless_env.items(): + current = os.environ.get(key) + if isinstance(expected, bool): + present = bool(current) + if present != expected: + raise SkipCase( + f"[HW WARNING] Env presence mismatch for {key!r}: " + f"got {present}, expected {expected}" + ) + continue + if current != expected: + raise SkipCase( + f"[HW WARNING] Env mismatch for {key!r}: " + f"got {current!r}, expected {expected!r}" + ) + + skip_unless_paths_exist = case_def.get("skip_unless_paths_exist", []) + for raw_path in ensure_string_or_list_of_strings( + skip_unless_paths_exist, + "skip_unless_paths_exist", + ): + resolved = Path(expand_template(raw_path, work_dir, file_path)) + if not resolved.exists(): + raise SkipCase(f"[HW WARNING] Missing required path: {resolved}") + + skip_unless_commands_succeed = case_def.get("skip_unless_commands_succeed", []) + probes = ensure_list(skip_unless_commands_succeed, "skip_unless_commands_succeed") + for probe in probes: + probe_spec = normalize_probe_spec(probe) + result = execute_command_spec(file_path, probe_spec, work_dir) + if result.timed_out: + raise SkipCase( + f"[HW WARNING] Probe command timed out after {result.timeout_sec}s: " + f"{result.command_text}" + ) + if result.exit_code != 0: + raise SkipCase( + f"[HW WARNING] Probe command failed with exit code " + f"{result.exit_code}: {result.command_text}" + ) + + if case_def.get("requires_destructive", False): + if os.environ.get(DESTRUCTIVE_TEST_ENV) != "1": + raise SkipCase( + f"[HW WARNING] Destructive test skipped. Set " + f"{DESTRUCTIVE_TEST_ENV}=1 to enable." + ) + + required_env = case_def.get("required_env", {}) + if required_env: + if not isinstance(required_env, dict): + raise ConfigError("'required_env' must be a mapping") + for key, expected in required_env.items(): + current = os.environ.get(key) + if current != expected: + raise ConfigError( + f"required_env mismatch for {key!r}: " + f"got {current!r}, expected {expected!r}" + ) + + +def check_file_exists( + file_path: Path, + _case_def: dict[str, Any], + _work_dir: Path, +) -> tuple[bool, str, str, bool]: + exists = file_path.exists() + message = "File exists" if exists else "File does not exist" + return exists, message, "", False + + +def check_py_compile( + file_path: Path, + _case_def: dict[str, Any], + _work_dir: Path, +) -> tuple[bool, str, str, bool]: + try: + py_compile.compile(str(file_path), doraise=True) + return True, "Python compilation succeeded", "", False + except py_compile.PyCompileError as exc: + return False, "Python compilation failed", str(exc), False + + +def check_source_contains( + file_path: Path, + case_def: dict[str, Any], + _work_dir: Path, +) -> tuple[bool, str, str, bool]: + pattern = case_def.get("pattern") + if not isinstance(pattern, str): + raise ConfigError("'source_contains' requires a string 'pattern'") + + source = read_source(file_path) + passed = pattern in source + message = f"Found pattern: {pattern}" if passed else f"Pattern not found: {pattern}" + return passed, message, "", False + + +def check_source_contains_any( + file_path: Path, + case_def: dict[str, Any], + _work_dir: Path, +) -> tuple[bool, str, str, bool]: + patterns = ensure_list_of_strings(case_def.get("patterns"), "patterns") + source = read_source(file_path) + matched = [pattern for pattern in patterns if pattern in source] + passed = bool(matched) + details = "Matched patterns: " + ", ".join(matched) if matched else "" + message = "At least one pattern matched" if passed else "No patterns matched" + return passed, message, details, False + + +def check_source_contains_all( + file_path: Path, + case_def: dict[str, Any], + _work_dir: Path, +) -> tuple[bool, str, str, bool]: + patterns = ensure_list_of_strings(case_def.get("patterns"), "patterns") + source = read_source(file_path) + missing = [pattern for pattern in patterns if pattern not in source] + passed = not missing + details = "Missing patterns: " + ", ".join(missing) if missing else "" + message = "All patterns matched" if passed else "Some patterns were not found" + return passed, message, details, False + + +def check_function_exists( + file_path: Path, + case_def: dict[str, Any], + _work_dir: Path, +) -> tuple[bool, str, str, bool]: + function_name = case_def.get("function") + if not isinstance(function_name, str): + raise ConfigError("'function_exists' requires a string 'function'") + + names = collect_function_names(file_path) + passed = function_name in names + message = ( + f"Function found: {function_name}" + if passed + else f"Function not found: {function_name}" + ) + return passed, message, "", False + + +def check_function_exists_any( + file_path: Path, + case_def: dict[str, Any], + _work_dir: Path, +) -> tuple[bool, str, str, bool]: + functions = ensure_list_of_strings(case_def.get("functions"), "functions") + names = collect_function_names(file_path) + matched = [name for name in functions if name in names] + passed = bool(matched) + details = "Matched functions: " + ", ".join(matched) if matched else "" + message = "At least one function matched" if passed else "No functions matched" + return passed, message, details, False + + +def check_main_guard( + file_path: Path, + _case_def: dict[str, Any], + _work_dir: Path, +) -> tuple[bool, str, str, bool]: + source = read_source(file_path) + patterns = [ + 'if __name__ == "__main__":', + "if __name__ == '__main__':", + ] + passed = any(pattern in source for pattern in patterns) + message = "Main guard found" if passed else "Main guard not found" + return passed, message, "", False + + +def check_py_function( + file_path: Path, + case_def: dict[str, Any], + work_dir: Path, +) -> tuple[bool, str, str, bool]: + module = load_module_from_path(file_path) + module_name = module.__name__ + stdout_buffer = io.StringIO() + stderr_buffer = io.StringIO() + original_cwd = Path.cwd() + original_env = os.environ.copy() + + try: + details_lines: list[str] = [f"Work dir: {work_dir}"] + function_name = case_def.get("function") + if not isinstance(function_name, str): + raise ConfigError("'py_function' requires a string 'function'") + + if not hasattr(module, function_name): + return False, f"Function not found: {function_name}", "", False + + try: + runtime_case = build_case_runtime_definition( + case_def, + work_dir, + file_path, + extra_tokens={"module": module.__name__}, + ) + details_lines.extend(materialize_case_workspace(work_dir, runtime_case)) + except (MockLoaderConfigError, ValueError, TypeError) as exc: + raise ConfigError(str(exc)) from exc + except CaseBuildError as exc: + raise ConfigError(str(exc)) from exc + + func = getattr(module, function_name) + args = runtime_case.get("args", []) + kwargs = runtime_case.get("kwargs", {}) + run_env = build_cli_env(file_path, runtime_case, work_dir) + + if not isinstance(args, list): + raise ConfigError("'py_function.args' must be a list") + if not isinstance(kwargs, dict): + raise ConfigError("'py_function.kwargs' must be a mapping") + + patch_constants = runtime_case.get("patch_constants", {}) + if patch_constants is not None: + if not isinstance(patch_constants, dict): + raise ConfigError("'py_function.patch_constants' must be a mapping") + for attr_name, attr_value in patch_constants.items(): + setattr(module, attr_name, attr_value) + + expected_exception = runtime_case.get("expect_exception") + + passed = True + message = "" + is_error = False + result: Any = None + raised_exc: Exception | None = None + + try: + os.chdir(work_dir) + os.environ.clear() + os.environ.update(run_env) + with apply_case_mocks( + runtime_case.get("mocks"), + target_context={"module": module.__name__}, + ): + with redirect_stdout(stdout_buffer), redirect_stderr(stderr_buffer): + result = func(*args, **kwargs) + except MockExpectationError as exc: + raised_exc = exc + details_lines.append(f"Mock verification failed: {exc}") + except Exception as exc: # pylint: disable=broad-exception-caught + raised_exc = exc + details_lines.append(traceback.format_exc()) + finally: + os.chdir(original_cwd) + os.environ.clear() + os.environ.update(original_env) + + stdout = normalize_completed_stream(stdout_buffer.getvalue()) + stderr = normalize_completed_stream(stderr_buffer.getvalue()) + details_lines.extend( + [ + "--- STDOUT ---", + stdout.rstrip(), + "--- STDERR ---", + stderr.rstrip(), + ] + ) + + if raised_exc is not None: + if isinstance(raised_exc, MockExpectationError): + passed = False + message = f"Mock verification failed: {raised_exc}" + is_error = True + elif expected_exception: + passed = raised_exc.__class__.__name__ == expected_exception + message = ( + f"Raised expected exception: {expected_exception}" + if passed + else ( + f"Expected {expected_exception}, got " + f"{raised_exc.__class__.__name__}: {raised_exc}" + ) + ) + is_error = not passed + else: + passed = False + message = f"Unexpected exception: {raised_exc}" + is_error = True + elif expected_exception: + passed = False + message = f"Expected exception {expected_exception}, but function returned" + details_lines.append(repr(result)) + else: + expected_return = runtime_case.get("expect_return") + if "expect_return" in runtime_case and result != expected_return: + passed = False + message = f"Expected return {expected_return!r}, got {result!r}" + + expected_fragment = runtime_case.get("expect_return_contains") + if expected_fragment is not None and expected_fragment not in str(result): + passed = False + message = ( + f"Expected return to contain {expected_fragment!r}, got {result!r}" + ) + + if passed: + message = f"Function returned {result!r}" + + output_passed, output_conditions = validate_output_expectations( + runtime_case, + stdout, + stderr, + ) + post_passed, post_messages = run_post_checks( + work_dir, + runtime_case.get("post_checks"), + ) + if post_messages: + details_lines.extend(["--- POST CHECKS ---", *post_messages]) + + conditions: list[str] = [] + if not passed and message: + conditions.append(message) + conditions.extend(output_conditions) + conditions.extend(message for message in post_messages if "FAIL" in message) + + final_passed = passed and output_passed and post_passed + final_message = ( + message + if final_passed + else "\n\n" + ("\n" + ("-" * 80) + "\n").join(conditions) + if conditions + else "Function check failed" + ) + + details = sanitize_xml_text("\n".join(details_lines).strip()) + return final_passed, sanitize_xml_text(final_message), details, is_error + finally: + sys.modules.pop(module_name, None) + + +def check_module_main_with_env( + file_path: Path, + case_def: dict[str, Any], + _work_dir: Path, +) -> tuple[bool, str, str, bool]: + temp_dir = create_runner_temp_dir() + details_lines: list[str] = [f"Temp dir: {temp_dir}"] + module: Any | None = None + exit_code: int | None = None + stdout_buffer = io.StringIO() + stderr_buffer = io.StringIO() + original_cwd = Path.cwd() + original_env = os.environ.copy() + + try: + try: + runtime_case = build_case_runtime_definition( + case_def, + temp_dir, + file_path, + ) + details_lines.extend(materialize_case_workspace(temp_dir, runtime_case)) + except (CaseBuildError, MockLoaderConfigError, ValueError, TypeError) as exc: + return False, f"Configuration error: {exc}", "\n".join(details_lines), True + + run_env = build_cli_env(file_path, runtime_case, temp_dir) + module = load_module_from_path(file_path) + + patch_constants = runtime_case.get("patch_constants", {}) + for attr_name, attr_value in patch_constants.items(): + setattr(module, attr_name, attr_value) + details_lines.append(f"Patched constant: {attr_name}={attr_value!r}") + + try: + os.chdir(temp_dir) + os.environ.clear() + os.environ.update(run_env) + with apply_case_mocks( + runtime_case.get("mocks"), + target_context={"module": module.__name__}, + ): + with redirect_stdout(stdout_buffer), redirect_stderr(stderr_buffer): + main_result = module.main() + exit_code = main_result if isinstance(main_result, int) else 0 + except MockExpectationError as exc: + stdout = normalize_completed_stream(stdout_buffer.getvalue()) + stderr = normalize_completed_stream(stderr_buffer.getvalue()) + details_lines.extend( + [ + f"Exit code: {exit_code}", + "--- STDOUT ---", + stdout.rstrip(), + "--- STDERR ---", + stderr.rstrip(), + f"Mock verification failed: {exc}", + ] + ) + append_log_file_details(details_lines, runtime_case) + return ( + False, + f"Mock verification failed: {exc}", + sanitize_xml_text("\n".join(details_lines).strip()), + False, + ) + except SystemExit as exc: + exit_code = exc.code if isinstance(exc.code, int) else 0 + finally: + os.chdir(original_cwd) + os.environ.clear() + os.environ.update(original_env) + + stdout = normalize_completed_stream(stdout_buffer.getvalue()) + stderr = normalize_completed_stream(stderr_buffer.getvalue()) + runtime_case["_actual_exit_code"] = exit_code + details_lines.append(f"Exit code: {exit_code}") + details_lines.extend( + [ + "--- STDOUT ---", + stdout.rstrip(), + "--- STDERR ---", + stderr.rstrip(), + ] + ) + + output_passed, output_conditions = validate_output_expectations( + runtime_case, + stdout, + stderr, + ) + post_passed, post_messages = run_post_checks( + temp_dir, + runtime_case.get("post_checks"), + ) + details_lines.extend(["--- POST CHECKS ---", *post_messages]) + + conditions = [ + *output_conditions, + *[message for message in post_messages if "FAIL" in message], + ] + if not output_passed or not post_passed: + append_log_file_details(details_lines, runtime_case) + return ( + False, + "\n\n" + ("\n" + ("-" * 80) + "\n").join(conditions), + sanitize_xml_text("\n".join(details_lines).strip()), + False, + ) + + return ( + True, + "Module main() check passed", + sanitize_xml_text("\n".join(details_lines).strip()), + False, + ) + + except Exception as exc: # pylint: disable=broad-exception-caught + details_lines.append(traceback.format_exc()) + return ( + False, + f"Unhandled exception: {exc}", + sanitize_xml_text("\n".join(details_lines).strip()), + not bool(case_def.get("warn_only", False)), + ) + + finally: + if module is not None: + sys.modules.pop(module.__name__, None) + shutil.rmtree(temp_dir, ignore_errors=True) + + +def check_path_exists( + file_path: Path, + case_def: dict[str, Any], + work_dir: Path, +) -> tuple[bool, str, str, bool]: + raw_path = case_def.get("path") + if not isinstance(raw_path, str): + raise ConfigError("'path_exists' requires a string 'path'") + resolved = Path(expand_template(raw_path, work_dir, file_path)) + passed = resolved.exists() + message = f"Path exists: {resolved}" if passed else f"Path not found: {resolved}" + return passed, message, "", False + + +def check_path_not_exists( + file_path: Path, + case_def: dict[str, Any], + work_dir: Path, +) -> tuple[bool, str, str, bool]: + raw_path = case_def.get("path") + if not isinstance(raw_path, str): + raise ConfigError("'path_not_exists' requires a string 'path'") + resolved = Path(expand_template(raw_path, work_dir, file_path)) + passed = not resolved.exists() + message = ( + f"Path correctly absent: {resolved}" + if passed + else f"Path unexpectedly exists: {resolved}" + ) + return passed, message, "", False + + +def run_command_assertion( + file_path: Path, + case_def: dict[str, Any], + work_dir: Path, +) -> tuple[CommandRunResult, list[str], bool]: + spec = { + "command": case_def.get("command"), + "args": case_def.get("args", []), + "shell": case_def.get("shell", False), + "timeout_sec": case_def.get("timeout_sec", DEFAULT_CLI_TIMEOUT_SEC), + "cwd": case_def.get("cwd"), + "env": case_def.get("env"), + } + result = execute_command_spec(file_path, spec, work_dir) + details = [ + f"Command: {result.command_text}", + f"Shell mode: {'yes' if result.shell_mode else 'no'}", + f"Timeout seconds: {result.timeout_sec}", + f"Timed out: {'yes' if result.timed_out else 'no'}", + f"Exit code: {result.exit_code}", + "--- STDOUT ---", + result.stdout.rstrip(), + "--- STDERR ---", + result.stderr.rstrip(), + ] + return result, details, result.timed_out + + +def check_command_success( + file_path: Path, + case_def: dict[str, Any], + work_dir: Path, +) -> tuple[bool, str, str, bool]: + result, details, timed_out = run_command_assertion(file_path, case_def, work_dir) + if timed_out: + return ( + False, + f"Command timed out after {result.timeout_sec}s", + "\n".join(details), + True, + ) + passed = result.exit_code == 0 + message = "Command succeeded" if passed else f"Expected exit code 0, got {result.exit_code}" + return passed, message, "\n".join(details), False + + +def check_command_exit_code( + file_path: Path, + case_def: dict[str, Any], + work_dir: Path, +) -> tuple[bool, str, str, bool]: + expected_exit_code = case_def.get("expect_exit_code") + if not isinstance(expected_exit_code, int): + raise ConfigError("'command_exit_code' requires integer 'expect_exit_code'") + result, details, timed_out = run_command_assertion(file_path, case_def, work_dir) + if timed_out: + return ( + False, + f"Command timed out after {result.timeout_sec}s", + "\n".join(details), + True, + ) + passed = result.exit_code == expected_exit_code + message = ( + f"Command exited with expected code {expected_exit_code}" + if passed + else f"Expected exit code {expected_exit_code}, got {result.exit_code}" + ) + return passed, message, "\n".join(details), False + + +def check_command_output_contains( + file_path: Path, + case_def: dict[str, Any], + work_dir: Path, +) -> tuple[bool, str, str, bool]: + expect_text = case_def.get("expect_text") + if not isinstance(expect_text, str): + raise ConfigError("'command_output_contains' requires string 'expect_text'") + result, details, timed_out = run_command_assertion(file_path, case_def, work_dir) + if timed_out: + return ( + False, + f"Command timed out after {result.timeout_sec}s", + "\n".join(details), + True, + ) + merged = result.stdout + "\n" + result.stderr + passed = expect_text in merged + message = ( + f"Command output contains {expect_text!r}" + if passed + else f"Command output missing {expect_text!r}" + ) + return passed, message, "\n".join(details), False + + +def check_command_output_regex( + file_path: Path, + case_def: dict[str, Any], + work_dir: Path, +) -> tuple[bool, str, str, bool]: + expect_pattern = case_def.get("expect_pattern") + if not isinstance(expect_pattern, str): + raise ConfigError("'command_output_regex' requires string 'expect_pattern'") + result, details, timed_out = run_command_assertion(file_path, case_def, work_dir) + if timed_out: + return ( + False, + f"Command timed out after {result.timeout_sec}s", + "\n".join(details), + True, + ) + merged = result.stdout + "\n" + result.stderr + passed = re.search(expect_pattern, merged, flags=re.MULTILINE) is not None + message = ( + f"Command output matches regex {expect_pattern!r}" + if passed + else f"Command output does not match regex {expect_pattern!r}" + ) + return passed, message, "\n".join(details), False + + +def build_cli_command( + file_path: Path, + case_def: dict[str, Any], + work_dir: Path, +) -> tuple[list[str] | str, bool]: + raw_args = case_def.get("args", []) + command_override = case_def.get("command") + shell_mode = bool(case_def.get("shell", False)) + + expanded_args = [expand_template(arg, work_dir, file_path) for arg in raw_args] + + if command_override: + command_value = expand_template(command_override, work_dir, file_path) + if shell_mode: + parts = [command_value, *[shlex.quote(arg) for arg in expanded_args]] + return " ".join(parts), True + return [command_value, *expanded_args], False + + base_cmd = [sys.executable, str(file_path), *expanded_args] + if shell_mode: + return " ".join(shlex.quote(part) for part in base_cmd), True + return base_cmd, False + + +def build_cli_env( + file_path: Path, + case_def: dict[str, Any], + work_dir: Path, +) -> dict[str, str]: + env = os.environ.copy() + raw_env = case_def.get("env") + if raw_env is None: + return env + + if not isinstance(raw_env, dict): + raise ConfigError("'env' must be a mapping") + + for key, value in raw_env.items(): + if not isinstance(key, str) or not isinstance(value, str): + raise ConfigError("'env' entries must be string -> string") + env[key] = expand_template(value, work_dir, file_path) + + return env + + +def check_cli( + file_path: Path, + case_def: dict[str, Any], + work_dir: Path, +) -> tuple[bool, str, str, bool]: + try: + runtime_case = build_case_runtime_definition( + case_def, + work_dir, + file_path, + ) + prepare_case_files(work_dir, runtime_case) + except (CaseBuildError, MockLoaderConfigError, ValueError, TypeError) as exc: + raise ConfigError(str(exc)) from exc + + stdin_text = runtime_case.get("stdin") + if stdin_text is not None and not isinstance(stdin_text, str): + raise ConfigError("'stdin' must be a string") + + timeout_sec = runtime_case.get("timeout_sec", DEFAULT_CLI_TIMEOUT_SEC) + if not isinstance(timeout_sec, int) or timeout_sec <= 0: + raise ConfigError("'timeout_sec' must be a positive integer") + + expect_timeout = bool(runtime_case.get("expect_timeout", False)) + cmd, shell_mode = build_cli_command(file_path, runtime_case, work_dir) + run_env = build_cli_env(file_path, runtime_case, work_dir) + + timed_out = False + stdout = "" + stderr = "" + exit_code: int | None = None + + try: + completed = REAL_SUBPROCESS_RUN( + cmd, + cwd=str(work_dir), + capture_output=True, + text=True, + input=stdin_text, + check=False, + env=run_env, + timeout=timeout_sec, + shell=shell_mode, + ) + stdout = normalize_completed_stream(completed.stdout) + stderr = normalize_completed_stream(completed.stderr) + exit_code = completed.returncode + except subprocess.TimeoutExpired as exc: + timed_out = True + stdout = normalize_completed_stream(exc.stdout) + stderr = normalize_completed_stream(exc.stderr) + + runtime_case["_actual_exit_code"] = exit_code + + conditions: list[str] = [] + output_passed = True + + if timed_out: + if expect_timeout: + output_passed = True + else: + output_passed = False + conditions.append(f"CLI command timed out after {timeout_sec}s") + else: + if expect_timeout: + output_passed = False + conditions.append("Expected command to time out, but it completed") + else: + output_passed, output_conditions = validate_output_expectations( + runtime_case, + stdout, + stderr, + ) + conditions.extend(output_conditions) + + post_passed, post_messages = run_post_checks( + work_dir, + runtime_case.get("post_checks"), + ) + conditions.extend(message for message in post_messages if "FAIL" in message) + + passed = output_passed and post_passed and not conditions + + command_text = cmd if isinstance(cmd, str) else " ".join(cmd) + details_lines = [ + f"Command: {command_text}", + f"Shell mode: {'yes' if shell_mode else 'no'}", + f"Timeout seconds: {timeout_sec}", + f"Timed out: {'yes' if timed_out else 'no'}", + f"Exit code: {exit_code}", + "--- STDOUT ---", + stdout.rstrip(), + "--- STDERR ---", + stderr.rstrip(), + ] + + if post_messages: + details_lines.extend(["--- POST CHECKS ---", *post_messages]) + + message = ( + "CLI check passed" + if passed + else "\n\n" + ("\n" + ("-" * 80) + "\n").join(conditions) + ) + message = sanitize_xml_text(message) + details = sanitize_xml_text("\n".join(details_lines).strip()) + + is_error = timed_out and not expect_timeout + return passed, message, details, is_error + + +def check_module_cli( + file_path: Path, + case_def: dict[str, Any], + work_dir: Path, +) -> tuple[bool, str, str, bool]: + details_lines: list[str] = [f"Work dir: {work_dir}"] + + try: + runtime_case = build_case_runtime_definition( + case_def, + work_dir, + file_path, + ) + details_lines.extend(materialize_case_workspace(work_dir, runtime_case)) + except (CaseBuildError, MockLoaderConfigError, ValueError, TypeError) as exc: + raise ConfigError(str(exc)) from exc + + stdin_text = runtime_case.get("stdin") + if stdin_text is not None and not isinstance(stdin_text, str): + raise ConfigError("'stdin' must be a string") + + raw_args = runtime_case.get("args", []) + if not isinstance(raw_args, list) or not all(isinstance(arg, str) for arg in raw_args): + raise ConfigError("'args' must be a list of strings") + + run_env = build_cli_env(file_path, runtime_case, work_dir) + patch_constants = runtime_case.get("patch_constants", {}) + if not isinstance(patch_constants, dict): + raise ConfigError("'patch_constants' must be a mapping") + + command_text = " ".join( + [shlex.quote(str(file_path)), *(shlex.quote(arg) for arg in raw_args)] + ) + details_lines.append(f"Command: python {command_text}") + + stdout_buffer = io.StringIO() + stderr_buffer = io.StringIO() + original_argv = sys.argv[:] + original_stdin = sys.stdin + original_cwd = Path.cwd() + original_main = sys.modules.get("__main__") + original_env = os.environ.copy() + exit_code = 0 + spec = importlib.util.spec_from_file_location("__main__", str(file_path)) + if spec is None or spec.loader is None: + raise ConfigError(f"Could not load module from {file_path}") + script_module = importlib.util.module_from_spec(spec) + + for attr_name, attr_value in patch_constants.items(): + setattr(script_module, attr_name, attr_value) + details_lines.append(f"Patched constant: {attr_name}={attr_value!r}") + + try: + sys.argv = [str(file_path), *raw_args] + if stdin_text is not None: + sys.stdin = io.StringIO(stdin_text) + os.chdir(work_dir) + os.environ.clear() + os.environ.update(run_env) + sys.modules["__main__"] = script_module + + try: + with apply_case_mocks( + runtime_case.get("mocks"), + target_context={"module": "__main__"}, + ): + with redirect_stdout(stdout_buffer), redirect_stderr(stderr_buffer): + spec.loader.exec_module(script_module) + except SystemExit as exc: + exit_code = exc.code if isinstance(exc.code, int) else 0 + except MockExpectationError as exc: + stdout = normalize_completed_stream(stdout_buffer.getvalue()) + stderr = normalize_completed_stream(stderr_buffer.getvalue()) + details_lines.extend( + [ + f"Exit code: {exit_code}", + "--- STDOUT ---", + stdout.rstrip(), + "--- STDERR ---", + stderr.rstrip(), + ] + ) + return ( + False, + f"Mock verification failed: {exc}", + "\n".join(details_lines), + False, + ) + except Exception as exc: # pylint: disable=broad-exception-caught + stdout = normalize_completed_stream(stdout_buffer.getvalue()) + stderr = normalize_completed_stream(stderr_buffer.getvalue()) + details_lines.extend( + [ + f"Exit code: {exit_code}", + "--- STDOUT ---", + stdout.rstrip(), + "--- STDERR ---", + stderr.rstrip(), + traceback.format_exc(), + ] + ) + return ( + False, + f"Unhandled exception: {exc}", + "\n".join(details_lines), + not bool(case_def.get("warn_only", False)), + ) + finally: + sys.argv = original_argv + sys.stdin = original_stdin + os.chdir(original_cwd) + os.environ.clear() + os.environ.update(original_env) + if original_main is None: + sys.modules.pop("__main__", None) + else: + sys.modules["__main__"] = original_main + + stdout = normalize_completed_stream(stdout_buffer.getvalue()) + stderr = normalize_completed_stream(stderr_buffer.getvalue()) + runtime_case["_actual_exit_code"] = exit_code + + output_passed, output_conditions = validate_output_expectations( + runtime_case, + stdout, + stderr, + ) + post_passed, post_messages = run_post_checks( + work_dir, + runtime_case.get("post_checks"), + ) + conditions = [*output_conditions, *[msg for msg in post_messages if "FAIL" in msg]] + + passed = output_passed and post_passed and not conditions + details_lines.extend( + [ + f"Exit code: {exit_code}", + "--- STDOUT ---", + stdout.rstrip(), + "--- STDERR ---", + stderr.rstrip(), + ] + ) + if post_messages: + details_lines.extend(["--- POST CHECKS ---", *post_messages]) + + message = ( + "Module CLI check passed" + if passed + else "\n\n" + ("\n" + ("-" * 80) + "\n").join(conditions) + ) + message = sanitize_xml_text(message) + details = sanitize_xml_text("\n".join(details_lines).strip()) + return passed, message, details, False + + +CHECK_HANDLERS: dict[ + str, + Callable[[Path, dict[str, Any], Path], tuple[bool, str, str, bool]], +] = { + "file_exists": check_file_exists, + "py_compile": check_py_compile, + "source_contains": check_source_contains, + "source_contains_any": check_source_contains_any, + "source_contains_all": check_source_contains_all, + "function_exists": check_function_exists, + "function_exists_any": check_function_exists_any, + "main_guard": check_main_guard, + "cli": check_cli, + "module_cli": check_module_cli, + "py_function": check_py_function, + "module_main_with_env": check_module_main_with_env, + "path_exists": check_path_exists, + "path_not_exists": check_path_not_exists, + "command_success": check_command_success, + "command_exit_code": check_command_exit_code, + "command_output_contains": check_command_output_contains, + "command_output_regex": check_command_output_regex, +} + + +def run_single_check( + file_path: Path, + case_def: dict[str, Any], + work_dir: Path, +) -> tuple[bool, str, str, bool]: + case_type = case_def.get("type", "cli") + if not isinstance(case_type, str): + raise ConfigError("Case 'type' must be a string") + + apply_skip_controls(file_path, case_def, work_dir) + + handler = CHECK_HANDLERS.get(case_type) + if handler is None: + raise ConfigError(f"Unsupported test type: {case_type}") + + return handler(file_path, case_def, work_dir) diff --git a/common/acs_test_framework_runner/runner_check_validation.py b/common/acs_test_framework_runner/runner_check_validation.py new file mode 100644 index 00000000..152fe326 --- /dev/null +++ b/common/acs_test_framework_runner/runner_check_validation.py @@ -0,0 +1,552 @@ +from __future__ import annotations + +from typing import Any + +try: # Support package imports and direct harness module loading. + from .runner_checks import ( + DEFAULT_CLI_TIMEOUT_SEC, + ConfigError, + ensure_list, + ensure_list_of_strings, + ensure_string_or_list_of_strings, + merge_mappings, + ) +except ImportError: # pragma: no cover - exercised by flat-module harness imports. + from runner_checks import ( + DEFAULT_CLI_TIMEOUT_SEC, + ConfigError, + ensure_list, + ensure_list_of_strings, + ensure_string_or_list_of_strings, + merge_mappings, + ) + + +def normalize_suite_files(files_value: Any, field_name: str) -> list[str]: + items = ensure_list(files_value, field_name) + if not items: + raise ConfigError(f"'{field_name}' must not be empty") + + normalized: list[str] = [] + for item in items: + if isinstance(item, str): + normalized.append(item) + continue + if isinstance(item, dict) and isinstance(item.get("path"), str): + normalized.append(item["path"]) + continue + raise ConfigError( + f"Each entry in '{field_name}' must be either a string or {{path: ...}}" + ) + return normalized + + +def validate_post_checks(post_checks: Any, field_name: str) -> None: + if post_checks is None: + return + + checks = ensure_list(post_checks, field_name) + supported = { + "exists", + "not_exists", + "file_contains", + "file_not_contains", + "file_not_empty", + "regex", + "ordered_contains", + } + + for index, check in enumerate(checks, start=1): + entry_name = f"{field_name}[{index}]" + if not isinstance(check, dict): + raise ConfigError(f"{entry_name} must be a mapping") + + check_type = check.get("type") + if not isinstance(check_type, str): + raise ConfigError(f"{entry_name}.type must be a string") + + if check_type not in supported: + raise ConfigError( + f"{entry_name}.type unsupported: {check_type}. " + f"Supported types: {', '.join(sorted(supported))}" + ) + + raw_path = check.get("path") + if not isinstance(raw_path, str): + raise ConfigError(f"{entry_name}.path must be a string") + + if check_type in {"file_contains", "file_not_contains"}: + text = check.get("text") + if not isinstance(text, str): + raise ConfigError(f"{entry_name}.text must be a string") + + if check_type == "regex": + pattern = check.get("pattern") + if not isinstance(pattern, str): + raise ConfigError(f"{entry_name}.pattern must be a string") + + if check_type == "ordered_contains": + texts = check.get("texts") + if not isinstance(texts, list) or not all( + isinstance(item, str) for item in texts + ): + raise ConfigError(f"{entry_name}.texts must be a list of strings") + + +def validate_env_mapping( + value: Any, + field_name: str, + *, + allow_bool_values: bool = False, +) -> None: + if not isinstance(value, dict): + raise ConfigError(f"{field_name} must be a mapping") + + valid_value_types = (str, bool) if allow_bool_values else (str,) + for key, item in value.items(): + if not isinstance(key, str): + raise ConfigError(f"{field_name} keys must be strings") + if not isinstance(item, valid_value_types): + raise ConfigError( + f"{field_name}['{key}'] must be " + f"{'string/bool' if allow_bool_values else 'a string'}" + ) + + +def validate_command_probe_list(value: Any, field_name: str) -> None: + probes = ensure_list(value, field_name) + for index, probe in enumerate(probes, start=1): + entry_name = f"{field_name}[{index}]" + if isinstance(probe, str): + continue + if not isinstance(probe, dict): + raise ConfigError(f"{entry_name} must be a string or mapping") + + command = probe.get("command") + if not isinstance(command, str): + raise ConfigError(f"{entry_name}.command must be a string") + + args = probe.get("args") + if args is not None and ( + not isinstance(args, list) or not all(isinstance(item, str) for item in args) + ): + raise ConfigError(f"{entry_name}.args must be a list of strings") + + shell_value = probe.get("shell") + if shell_value is not None and not isinstance(shell_value, bool): + raise ConfigError(f"{entry_name}.shell must be a boolean") + + timeout_sec = probe.get("timeout_sec") + if timeout_sec is not None and ( + not isinstance(timeout_sec, int) or timeout_sec <= 0 + ): + raise ConfigError(f"{entry_name}.timeout_sec must be a positive integer") + + cwd = probe.get("cwd") + if cwd is not None and not isinstance(cwd, str): + raise ConfigError(f"{entry_name}.cwd must be a string") + + env = probe.get("env") + if env is not None: + validate_env_mapping(env, f"{entry_name}.env") + + +def validate_common_case_controls(case_def: dict[str, Any], field_name: str) -> None: + skip_unless_env = case_def.get("skip_unless_env") + if skip_unless_env is not None: + validate_env_mapping( + skip_unless_env, + f"{field_name}.skip_unless_env", + allow_bool_values=True, + ) + + skip_unless_paths_exist = case_def.get("skip_unless_paths_exist") + if skip_unless_paths_exist is not None: + ensure_string_or_list_of_strings( + skip_unless_paths_exist, + f"{field_name}.skip_unless_paths_exist", + ) + + skip_unless_commands_succeed = case_def.get("skip_unless_commands_succeed") + if skip_unless_commands_succeed is not None: + validate_command_probe_list( + skip_unless_commands_succeed, + f"{field_name}.skip_unless_commands_succeed", + ) + + requires_destructive = case_def.get("requires_destructive") + if requires_destructive is not None and not isinstance(requires_destructive, bool): + raise ConfigError(f"{field_name}.requires_destructive must be a boolean") + + required_env = case_def.get("required_env") + if required_env is not None: + validate_env_mapping(required_env, f"{field_name}.required_env") + + warn_only = case_def.get("warn_only") + if warn_only is not None and not isinstance(warn_only, bool): + raise ConfigError(f"{field_name}.warn_only must be a boolean") + + +def validate_mock_spec(spec: Any, field_name: str) -> None: + """Validate one mocks. spec conservatively.""" + if isinstance(spec, str): + return + + if not isinstance(spec, dict): + raise ConfigError( + f"{field_name} must be a string or mapping, got {type(spec).__name__}" + ) + + if "factory" in spec and spec["factory"] is not None and not isinstance( + spec["factory"], str + ): + raise ConfigError(f"{field_name}.factory must be a string when provided") + + if "inject_original_as" in spec and not isinstance( + spec["inject_original_as"], str + ): + raise ConfigError(f"{field_name}.inject_original_as must be a string") + + if "args" in spec and spec["args"] is not None and not isinstance(spec["args"], list): + raise ConfigError(f"{field_name}.args must be a list") + + if "kwargs" in spec and spec["kwargs"] is not None and not isinstance( + spec["kwargs"], dict + ): + raise ConfigError(f"{field_name}.kwargs must be a mapping") + + if "attrs" in spec and spec["attrs"] is not None and not isinstance( + spec["attrs"], dict + ): + raise ConfigError(f"{field_name}.attrs must be a mapping") + + +def validate_case_mocks(case_def: dict[str, Any], field_name: str) -> None: + """Validate optional mocks/scenario fields.""" + mocks = case_def.get("mocks") + if mocks is not None: + if not isinstance(mocks, dict): + raise ConfigError(f"{field_name}.mocks must be a mapping") + for target, spec in mocks.items(): + if not isinstance(target, str) or not target.strip(): + raise ConfigError(f"{field_name}.mocks keys must be non-empty strings") + validate_mock_spec(spec, f"{field_name}.mocks[{target!r}]") + + scenario = case_def.get("scenario") + if scenario is not None and not isinstance(scenario, dict): + raise ConfigError(f"{field_name}.scenario must be a mapping") + + +def validate_case_schema(case_def: dict[str, Any], field_name: str) -> None: + case_name = case_def.get("name") + if not isinstance(case_name, str) or not case_name.strip(): + raise ConfigError(f"{field_name}.name must be a non-empty string") + + case_type = case_def.get("type", "cli") + if not isinstance(case_type, str): + raise ConfigError(f"{field_name}.type must be a string") + + if case_type not in { + "file_exists", + "py_compile", + "source_contains", + "source_contains_any", + "source_contains_all", + "function_exists", + "function_exists_any", + "main_guard", + "cli", + "module_cli", + "py_function", + "module_main_with_env", + "path_exists", + "path_not_exists", + "command_success", + "command_exit_code", + "command_output_contains", + "command_output_regex", + }: + raise ConfigError(f"{field_name}.type unsupported: {case_type}") + + validate_common_case_controls(case_def, field_name) + validate_case_mocks(case_def, field_name) + + if "scripts" in case_def and case_def["scripts"] is not None: + scripts = case_def["scripts"] + if not isinstance(scripts, dict): + raise ConfigError(f"{field_name}.scripts must be a mapping") + for key, value in scripts.items(): + if not isinstance(key, str) or not isinstance(value, str): + raise ConfigError( + f"{field_name}.scripts entries must be string -> string" + ) + + if "bin_files" in case_def and case_def["bin_files"] is not None: + bin_files = case_def["bin_files"] + if not isinstance(bin_files, dict): + raise ConfigError(f"{field_name}.bin_files must be a mapping") + for key, spec in bin_files.items(): + if not isinstance(key, str) or not isinstance(spec, dict): + raise ConfigError( + f"{field_name}.bin_files entries must be string -> mapping" + ) + hex_data = spec.get("hex") + text_data = spec.get("text") + if hex_data is None and text_data is None: + raise ConfigError( + f"{field_name}.bin_files['{key}'] requires 'hex' or 'text'" + ) + if hex_data is not None and not isinstance(hex_data, str): + raise ConfigError( + f"{field_name}.bin_files['{key}'].hex must be a string" + ) + if text_data is not None and not isinstance(text_data, str): + raise ConfigError( + f"{field_name}.bin_files['{key}'].text must be a string" + ) + + if "text_files" in case_def and case_def["text_files"] is not None: + text_files = case_def["text_files"] + if not isinstance(text_files, dict): + raise ConfigError(f"{field_name}.text_files must be a mapping") + for key, value in text_files.items(): + if not isinstance(key, str) or not isinstance(value, str): + raise ConfigError( + f"{field_name}.text_files entries must be string -> string" + ) + + if "patch_constants" in case_def and case_def["patch_constants"] is not None: + patch_constants = case_def["patch_constants"] + if not isinstance(patch_constants, dict): + raise ConfigError(f"{field_name}.patch_constants must be a mapping") + for key, value in patch_constants.items(): + if not isinstance(key, str): + raise ConfigError(f"{field_name}.patch_constants keys must be strings") + if not isinstance(value, (str, int, float, bool)): + raise ConfigError( + f"{field_name}.patch_constants['{key}'] " + "must be a scalar string/int/float/bool" + ) + + if "dir_structure" in case_def and case_def["dir_structure"] is not None: + dir_structure = case_def["dir_structure"] + if not isinstance(dir_structure, list): + raise ConfigError(f"{field_name}.dir_structure must be a list") + for index, entry in enumerate(dir_structure, start=1): + if not isinstance(entry, dict): + raise ConfigError(f"{field_name}.dir_structure[{index}] must be a mapping") + if not isinstance(entry.get("path"), str): + raise ConfigError( + f"{field_name}.dir_structure[{index}].path must be a string" + ) + + validate_post_checks(case_def.get("post_checks"), f"{field_name}.post_checks") + + if case_type == "py_function": + function_name = case_def.get("function") + if not isinstance(function_name, str) or not function_name.strip(): + raise ConfigError(f"{field_name}.function must be a non-empty string") + + args = case_def.get("args", []) + if not isinstance(args, list): + raise ConfigError(f"{field_name}.args must be a list") + + kwargs = case_def.get("kwargs") + if kwargs is not None and not isinstance(kwargs, dict): + raise ConfigError(f"{field_name}.kwargs must be a mapping") + + if "expect_exception" in case_def and not isinstance( + case_def["expect_exception"], str + ): + raise ConfigError(f"{field_name}.expect_exception must be a string") + + if "expect_return_contains" in case_def and not isinstance( + case_def["expect_return_contains"], str + ): + raise ConfigError(f"{field_name}.expect_return_contains must be a string") + return + + if case_type == "module_main_with_env": + expected_exit_code = case_def.get("expect_exit_code") + if expected_exit_code is not None and not isinstance(expected_exit_code, int): + raise ConfigError(f"{field_name}.expect_exit_code must be an integer") + return + + if case_type in {"path_exists", "path_not_exists"}: + path_value = case_def.get("path") + if not isinstance(path_value, str): + raise ConfigError(f"{field_name}.path must be a string") + return + + if case_type in { + "command_success", + "command_exit_code", + "command_output_contains", + "command_output_regex", + }: + command = case_def.get("command") + if not isinstance(command, str): + raise ConfigError(f"{field_name}.command must be a string") + + args = case_def.get("args", []) + if not isinstance(args, list) or not all(isinstance(arg, str) for arg in args): + raise ConfigError(f"{field_name}.args must be a list of strings") + + shell_value = case_def.get("shell", False) + if not isinstance(shell_value, bool): + raise ConfigError(f"{field_name}.shell must be a boolean") + + timeout_sec = case_def.get("timeout_sec", DEFAULT_CLI_TIMEOUT_SEC) + if not isinstance(timeout_sec, int) or timeout_sec <= 0: + raise ConfigError(f"{field_name}.timeout_sec must be a positive integer") + + cwd = case_def.get("cwd") + if cwd is not None and not isinstance(cwd, str): + raise ConfigError(f"{field_name}.cwd must be a string") + + env = case_def.get("env") + if env is not None: + validate_env_mapping(env, f"{field_name}.env") + + if case_type == "command_exit_code": + expected_exit_code = case_def.get("expect_exit_code") + if not isinstance(expected_exit_code, int): + raise ConfigError(f"{field_name}.expect_exit_code must be an integer") + + if case_type == "command_output_contains": + expect_text = case_def.get("expect_text") + if not isinstance(expect_text, str): + raise ConfigError(f"{field_name}.expect_text must be a string") + + if case_type == "command_output_regex": + expect_pattern = case_def.get("expect_pattern") + if not isinstance(expect_pattern, str): + raise ConfigError(f"{field_name}.expect_pattern must be a string") + return + + if case_type not in {"cli", "module_cli"}: + return + + raw_args = case_def.get("args", []) + if not isinstance(raw_args, list) or not all(isinstance(arg, str) for arg in raw_args): + raise ConfigError(f"{field_name}.args must be a list of strings") + + command = case_def.get("command") + if command is not None and not isinstance(command, str): + raise ConfigError(f"{field_name}.command must be a string") + + stdin_text = case_def.get("stdin") + if stdin_text is not None and not isinstance(stdin_text, str): + raise ConfigError(f"{field_name}.stdin must be a string") + + timeout_sec = case_def.get("timeout_sec", DEFAULT_CLI_TIMEOUT_SEC) + if not isinstance(timeout_sec, int) or timeout_sec <= 0: + raise ConfigError(f"{field_name}.timeout_sec must be a positive integer") + + shell_value = case_def.get("shell", False) + if not isinstance(shell_value, bool): + raise ConfigError(f"{field_name}.shell must be a boolean") + + expect_timeout = case_def.get("expect_timeout", False) + if not isinstance(expect_timeout, bool): + raise ConfigError(f"{field_name}.expect_timeout must be a boolean") + + env = case_def.get("env") + if env is not None: + validate_env_mapping(env, f"{field_name}.env") + + expected_exit_code = case_def.get("expect_exit_code") + if expected_exit_code is not None and not isinstance(expected_exit_code, int): + raise ConfigError(f"{field_name}.expect_exit_code must be an integer") + + expect_exit_nonzero = case_def.get("expect_exit_nonzero", False) + if not isinstance(expect_exit_nonzero, bool): + raise ConfigError(f"{field_name}.expect_exit_nonzero must be a boolean") + + expect_exit_code_in = case_def.get("expect_exit_code_in") + if expect_exit_code_in is not None: + if not isinstance(expect_exit_code_in, list) or not all( + isinstance(item, int) for item in expect_exit_code_in + ): + raise ConfigError( + f"{field_name}.expect_exit_code_in must be a list of integers" + ) + + expect_stdout_or_stderr_regex = case_def.get("expect_stdout_or_stderr_regex") + if expect_stdout_or_stderr_regex is not None: + if isinstance(expect_stdout_or_stderr_regex, str): + return + if not isinstance(expect_stdout_or_stderr_regex, list) or not all( + isinstance(item, str) for item in expect_stdout_or_stderr_regex + ): + raise ConfigError( + f"{field_name}.expect_stdout_or_stderr_regex must be a string or list of strings" + ) + + +def normalize_cases( + value: Any, + field_name: str, + defaults: dict[str, Any] | None = None, +) -> list[dict[str, Any]]: + items = ensure_list(value, field_name) + normalized: list[dict[str, Any]] = [] + suite_defaults = defaults or {} + + for index, case_def in enumerate(items, start=1): + if not isinstance(case_def, dict): + raise ConfigError(f"{field_name}[{index}] must be a mapping") + merged_case = merge_mappings(suite_defaults, case_def) + if case_def.get("expect_exit_nonzero") and "expect_exit_code" not in case_def: + merged_case.pop("expect_exit_code", None) + if "expect_exit_code_in" in case_def and "expect_exit_code" not in case_def: + merged_case.pop("expect_exit_code", None) + validate_case_schema(merged_case, f"{field_name}[{index}]") + normalized.append(merged_case) + + return normalized + + +def normalize_suites(config: dict[str, Any]) -> list[dict[str, Any]]: + raw_suites = config.get("suites") + if raw_suites is None: + raise ConfigError("Top-level key 'suites' is required") + + suites = ensure_list(raw_suites, "suites") + if not suites: + raise ConfigError("'suites' must not be empty") + + normalized: list[dict[str, Any]] = [] + for index, suite in enumerate(suites, start=1): + if not isinstance(suite, dict): + raise ConfigError(f"suites[{index}] must be a mapping") + + name = suite.get("name") + if not isinstance(name, str) or not name.strip(): + raise ConfigError(f"suites[{index}] requires a non-empty string 'name'") + + suite_command = suite.get("command") + if suite_command is not None and not isinstance(suite_command, str): + raise ConfigError(f"suites[{index}].command must be a string") + + defaults = suite.get("defaults") or {} + if not isinstance(defaults, dict): + raise ConfigError(f"suites[{index}].defaults must be a mapping") + + files = normalize_suite_files(suite.get("files"), f"suites[{index}].files") + cases = normalize_cases( + suite.get("cases", []), + f"suites[{index}].cases", + defaults=defaults, + ) + + normalized.append( + { + "name": name.strip(), + "command": suite_command, + "files": files, + "cases": cases, + } + ) + + return normalized diff --git a/common/acs_test_framework_runner/runner_checks.py b/common/acs_test_framework_runner/runner_checks.py new file mode 100644 index 00000000..00559353 --- /dev/null +++ b/common/acs_test_framework_runner/runner_checks.py @@ -0,0 +1,731 @@ +from __future__ import annotations + +import ast +import importlib.util +import re +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import yaml + +try: # Support package imports and direct harness module loading. + from .case_data_builders import render_post_check_path +except ImportError: # pragma: no cover - exercised by flat-module harness imports. + from case_data_builders import render_post_check_path + + +SCRIPT_DIR = Path(__file__).resolve().parent + + +def detect_project_root(script_dir: Path) -> Path: + if script_dir.parent.name == "common": + return script_dir.parent.parent + return script_dir.parent + + +PROJECT_ROOT = detect_project_root(SCRIPT_DIR) +TEST_YAML_DIR = PROJECT_ROOT / "common" / "acs_test_framework_manifests" +REPORTS_DIR = PROJECT_ROOT / "common" / "reports" +RUNNER_WORK_DIR = REPORTS_DIR / "_runner_work" +SUPPORTED_SUFFIXES = {".yaml", ".yml"} +DEFAULT_CLI_TIMEOUT_SEC = 20 +DESTRUCTIVE_TEST_ENV = "RUN_DESTRUCTIVE_HW_TESTS" +# Runner-managed process launches must not be affected by case-level +# subprocess.run mocks that target the global subprocess module. +REAL_SUBPROCESS_RUN = subprocess.run + + +@dataclass +class TestMeta: + suite_name: str + phase: str + test_type: str + + +@dataclass +class TestOutcome: + testcase_name: str + file_path: str + passed: bool + message: str + meta: TestMeta + details: str = "" + flags: dict[str, bool] = field( + default_factory=lambda: { + "error": False, + "skipped": False, + "warning": False, + } + ) + + @property + def error(self) -> bool: + return self.flags["error"] + + @property + def skipped(self) -> bool: + return self.flags["skipped"] + + @property + def warning(self) -> bool: + return self.flags["warning"] + + +class ConfigError(Exception): + """Raised when a YAML configuration is invalid.""" + + +class SkipCase(Exception): + """Raised when a case should be skipped due to environment gating.""" + + +@dataclass +class CommandRunResult: + command_text: str + stdout: str + stderr: str + exit_code: int | None + timed_out: bool + timeout_sec: int + shell_mode: bool + + +@dataclass(frozen=True) +class RunCaseOptions: + suite_command: str | None = None + + +def create_runner_temp_dir(prefix: str = "runner_env_") -> Path: + RUNNER_WORK_DIR.mkdir(parents=True, exist_ok=True) + for _ in range(100): + candidate = RUNNER_WORK_DIR / f"{prefix}{uuid4().hex[:8]}" + try: + candidate.mkdir(parents=False, exist_ok=False) + return candidate + except FileExistsError: + continue + raise OSError(f"Could not create runner temp dir under {RUNNER_WORK_DIR}") + + +def load_yaml_config(yaml_file: Path) -> dict[str, Any]: + try: + with yaml_file.open("r", encoding="utf-8") as handle: + data = yaml.safe_load(handle) or {} + except yaml.YAMLError as exc: + raise ConfigError(f"Failed to parse YAML: {exc}") from exc + + if not isinstance(data, dict): + raise ConfigError("Top-level YAML content must be a mapping") + + return data + + +def ensure_list(value: Any, field_name: str) -> list[Any]: + if not isinstance(value, list): + raise ConfigError(f"'{field_name}' must be a list") + return value + + +def ensure_list_of_strings(value: Any, field_name: str) -> list[str]: + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise ConfigError(f"'{field_name}' must be a list of strings") + return value + + +def ensure_string_or_list_of_strings(value: Any, field_name: str) -> list[str]: + if value is None: + return [] + if isinstance(value, str): + return [value] + return ensure_list_of_strings(value, field_name) + + +def merge_mappings( + base: dict[str, Any], + override: dict[str, Any], +) -> dict[str, Any]: + """Recursively merge mappings with override precedence.""" + merged: dict[str, Any] = dict(base) + for key, value in override.items(): + existing = merged.get(key) + if isinstance(existing, dict) and isinstance(value, dict): + merged[key] = merge_mappings(existing, value) + else: + merged[key] = value + return merged + + +def sanitize_name(value: str) -> str: + return "".join( + char if char.isalnum() or char in {"-", "_", "."} else "_" + for char in value + ) + + +def is_valid_xml_char(code: int) -> bool: + return ( + code in {0x9, 0xA, 0xD} + or 0x20 <= code <= 0xD7FF + or 0xE000 <= code <= 0xFFFD + or 0x10000 <= code <= 0x10FFFF + ) + + +def sanitize_xml_text(value: Any) -> str: + if value is None: + return "" + if not isinstance(value, str): + value = str(value) + + cleaned: list[str] = [] + for char in value: + if is_valid_xml_char(ord(char)): + cleaned.append(char) + return "".join(cleaned) + + +def resolve_target_path(file_entry: str) -> Path: + raw = Path(file_entry) + if raw.is_absolute(): + return raw.resolve() + return (PROJECT_ROOT / raw).resolve() + + +def read_source(file_path: Path) -> str: + return file_path.read_text(encoding="utf-8") + + +def parse_ast(file_path: Path) -> ast.AST: + return ast.parse(read_source(file_path), filename=str(file_path)) + + +def collect_function_names(file_path: Path) -> set[str]: + tree = parse_ast(file_path) + names: set[str] = set() + + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + names.add(node.name) + + return names + + +def format_outcome_message(prefix: str, text: str) -> str: + text = text.strip() + return prefix if not text else f"{prefix}: {text}" + + +def create_outcome( + testcase_name: str, + file_path: str, + passed: bool, + message: str, + meta: TestMeta, + **kwargs: Any, +) -> TestOutcome: + return TestOutcome( + testcase_name=testcase_name, + file_path=file_path, + passed=passed, + message=message, + meta=meta, + details=kwargs.get("details", ""), + flags={ + "error": kwargs.get("error", False), + "skipped": kwargs.get("skipped", False), + "warning": kwargs.get("warning", False), + }, + ) + + +def build_runner_module_name(file_path: Path) -> str: + return f"runner_module_{sanitize_name(file_path.stem)}_{uuid4().hex}" + + +def load_module_from_path(file_path: Path) -> Any: + module_name = build_runner_module_name(file_path) + spec = importlib.util.spec_from_file_location( + module_name, + str(file_path), + ) + if spec is None or spec.loader is None: + raise ConfigError(f"Could not load module from {file_path}") + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + try: + spec.loader.exec_module(module) + except Exception: + sys.modules.pop(module_name, None) + raise + return module + + +def normalize_completed_stream(stream: Any) -> str: + if stream is None: + return "" + if isinstance(stream, str): + return stream + if isinstance(stream, bytes): + return stream.decode("utf-8", errors="replace") + return str(stream) + + +try: # Support package imports and direct harness module loading. + from .runner_check_validation import ( + normalize_cases, + normalize_suite_files, + normalize_suites, + validate_case_mocks, + validate_case_schema, + validate_command_probe_list, + validate_common_case_controls, + validate_env_mapping, + validate_mock_spec, + validate_post_checks, + ) +except ImportError: # pragma: no cover - exercised by flat-module harness imports. + from runner_check_validation import ( + normalize_cases, + normalize_suite_files, + normalize_suites, + validate_case_mocks, + validate_case_schema, + validate_command_probe_list, + validate_common_case_controls, + validate_env_mapping, + validate_mock_spec, + validate_post_checks, + ) + + +def run_post_checks(work_dir: Path, post_checks: Any) -> tuple[bool, list[str]]: + if post_checks is None: + return True, [] + + checks = ensure_list(post_checks, "post_checks") + messages: list[str] = [] + all_passed = True + + for index, check in enumerate(checks, start=1): + if not isinstance(check, dict): + raise ConfigError(f"post_checks[{index}] must be a mapping") + + check_type = check.get("type") + raw_path = check.get("path") + + if not isinstance(check_type, str): + raise ConfigError(f"post_checks[{index}] requires string key 'type'") + if not isinstance(raw_path, str): + raise ConfigError(f"post_checks[{index}] requires string key 'path'") + + resolved = render_post_check_path(raw_path, work_dir) + + if check_type == "exists": + exists = resolved.exists() + messages.append( + f"post_check exists: {resolved} -> {'PASS' if exists else 'FAIL'}" + ) + if not exists: + all_passed = False + continue + + if check_type == "not_exists": + missing = not resolved.exists() + messages.append( + f"post_check not_exists: {resolved} -> {'PASS' if missing else 'FAIL'}" + ) + if not missing: + all_passed = False + continue + + if check_type == "file_not_empty": + passed = ( + resolved.exists() + and resolved.is_file() + and resolved.stat().st_size > 0 + ) + messages.append( + f"post_check file_not_empty: {resolved} -> {'PASS' if passed else 'FAIL'}" + ) + if not passed: + all_passed = False + continue + + if check_type in {"file_contains", "file_not_contains"}: + text = check.get("text") + if not isinstance(text, str): + raise ConfigError(f"post_checks[{index}] requires string key 'text'") + + if not resolved.exists() or not resolved.is_file(): + messages.append( + f"post_check {check_type}: {resolved} contains {text!r} -> FAIL" + ) + all_passed = False + continue + + contents = resolved.read_text(encoding="utf-8", errors="replace") + contains = text in contents + passed = contains if check_type == "file_contains" else not contains + + messages.append( + f"post_check {check_type}: {resolved} contains {text!r} -> " + f"{'PASS' if passed else 'FAIL'}" + ) + if not passed: + all_passed = False + continue + + if check_type == "regex": + pattern = check.get("pattern") + if not isinstance(pattern, str): + raise ConfigError( + f"post_checks[{index}] requires string key 'pattern'" + ) + + if not resolved.exists() or not resolved.is_file(): + messages.append( + f"post_check regex: {resolved} matches {pattern!r} -> FAIL" + ) + all_passed = False + continue + + contents = resolved.read_text(encoding="utf-8", errors="replace") + passed = re.search(pattern, contents, flags=re.MULTILINE) is not None + messages.append( + f"post_check regex: {resolved} matches {pattern!r} -> " + f"{'PASS' if passed else 'FAIL'}" + ) + if not passed: + all_passed = False + continue + + if check_type == "ordered_contains": + texts = check.get("texts") + if not isinstance(texts, list) or not all( + isinstance(item, str) for item in texts + ): + raise ConfigError( + f"post_checks[{index}] requires key 'texts' as list[str]" + ) + + if not resolved.exists() or not resolved.is_file(): + messages.append(f"post_check ordered_contains: {resolved} -> FAIL") + all_passed = False + continue + + contents = resolved.read_text(encoding="utf-8", errors="replace") + start = 0 + passed = True + for text in texts: + idx = contents.find(text, start) + if idx == -1: + passed = False + messages.append( + f"post_check ordered_contains: {resolved} " + f"missing {text!r} in order -> FAIL" + ) + all_passed = False + break + start = idx + len(text) + + if passed: + messages.append(f"post_check ordered_contains: {resolved} -> PASS") + continue + + raise ConfigError(f"post_checks[{index}] unsupported type: {check_type}") + + return all_passed, messages + + +def format_output_block(title: str, text: str) -> str: + cleaned = text.rstrip() + if not cleaned: + cleaned = "" + return f"{title}\n{cleaned}" + + +def append_log_file_details( + details_lines: list[str], + runtime_case: dict[str, Any], +) -> None: + patch_constants = runtime_case.get("patch_constants", {}) + if not isinstance(patch_constants, dict): + return + + log_path_value = patch_constants.get("LOG_FILE") + if not isinstance(log_path_value, str): + return + + log_path = Path(log_path_value) + details_lines.append("--- LOG FILE ---") + + if not log_path.exists() or not log_path.is_file(): + details_lines.append(f"") + return + + log_text = log_path.read_text(encoding="utf-8", errors="replace").rstrip() + details_lines.append(log_text if log_text else "") + + +def format_expectation_failure( + check_type: str, + expected: str, + stdout: str, + stderr: str, + *, + actual_label: str = "", +) -> str: + lines = [ + "PHASE: expectation_check", + f"CHECK TYPE: {check_type}", + "", + "EXPECTED:", + f" {expected}", + "", + format_output_block("ACTUAL STDOUT:", stdout), + "", + format_output_block("ACTUAL STDERR:", stderr), + ] + if actual_label: + lines.extend(["", "ACTUAL:", f" {actual_label}"]) + return "\n".join(lines) + + +def validate_output_expectations( + case_def: dict[str, Any], + stdout: str, + stderr: str, +) -> tuple[bool, list[str]]: + conditions: list[str] = [] + passed = True + + expected_exit_code = case_def.get("expect_exit_code") + expect_exit_nonzero = bool(case_def.get("expect_exit_nonzero", False)) + + if expected_exit_code is not None and not isinstance(expected_exit_code, int): + raise ConfigError("'expect_exit_code' must be an integer") + + actual_exit_code = case_def.get("_actual_exit_code") + + if actual_exit_code is not None: + if expected_exit_code is not None: + if actual_exit_code != expected_exit_code: + passed = False + conditions.append( + "\n".join( + [ + "PHASE: expectation_check", + "CHECK TYPE: expect_exit_code", + "", + "EXPECTED:", + f" exit code = {expected_exit_code}", + "", + "ACTUAL:", + f" exit code = {actual_exit_code}", + ] + ) + ) + elif expect_exit_nonzero and actual_exit_code == 0: + passed = False + conditions.append( + "\n".join( + [ + "PHASE: expectation_check", + "CHECK TYPE: expect_exit_nonzero", + "", + "EXPECTED:", + " non-zero exit code", + "", + "ACTUAL:", + f" exit code = {actual_exit_code}", + ] + ) + ) + + expect_output = case_def.get("expect_output") + if expect_output is not None: + candidates = ( + [expect_output] + if isinstance(expect_output, str) + else ensure_list_of_strings(expect_output, "expect_output") + ) + merged = stdout + "\n" + stderr + for candidate in candidates: + if candidate not in merged: + passed = False + conditions.append( + format_expectation_failure( + check_type="expect_output", + expected=candidate, + stdout=stdout, + stderr=stderr, + ) + ) + + stdout_contains = case_def.get("expect_stdout_contains") + if stdout_contains is not None: + if not isinstance(stdout_contains, str): + raise ConfigError("'expect_stdout_contains' must be a string") + if stdout_contains not in stdout: + passed = False + conditions.append( + format_expectation_failure( + check_type="expect_stdout_contains", + expected=stdout_contains, + stdout=stdout, + stderr=stderr, + ) + ) + + stderr_contains = case_def.get("expect_stderr_contains") + if stderr_contains is not None: + if not isinstance(stderr_contains, str): + raise ConfigError("'expect_stderr_contains' must be a string") + if stderr_contains not in stderr: + passed = False + conditions.append( + format_expectation_failure( + check_type="expect_stderr_contains", + expected=stderr_contains, + stdout=stdout, + stderr=stderr, + ) + ) + + either_contains = case_def.get("expect_stdout_or_stderr_contains") + if either_contains is not None: + candidates = ( + [either_contains] + if isinstance(either_contains, str) + else ensure_list_of_strings( + either_contains, + "expect_stdout_or_stderr_contains", + ) + ) + for candidate in candidates: + if candidate not in stdout and candidate not in stderr: + passed = False + conditions.append( + format_expectation_failure( + check_type="expect_stdout_or_stderr_contains", + expected=candidate, + stdout=stdout, + stderr=stderr, + ) + ) + + expect_stdout_or_stderr_regex = case_def.get("expect_stdout_or_stderr_regex") + if expect_stdout_or_stderr_regex is not None: + candidates = ( + [expect_stdout_or_stderr_regex] + if isinstance(expect_stdout_or_stderr_regex, str) + else ensure_list_of_strings( + expect_stdout_or_stderr_regex, + "expect_stdout_or_stderr_regex", + ) + ) + merged = stdout + "\n" + stderr + for pattern in candidates: + if re.search(pattern, merged, flags=re.MULTILINE) is None: + passed = False + conditions.append( + format_expectation_failure( + check_type="expect_stdout_or_stderr_regex", + expected=pattern, + stdout=stdout, + stderr=stderr, + ) + ) + + expect_exit_code_in = case_def.get("expect_exit_code_in") + if expect_exit_code_in is not None: + if not isinstance(expect_exit_code_in, list) or not all( + isinstance(item, int) for item in expect_exit_code_in + ): + raise ConfigError("'expect_exit_code_in' must be a list of integers") + if actual_exit_code not in expect_exit_code_in: + passed = False + conditions.append( + "\n".join( + [ + "PHASE: expectation_check", + "CHECK TYPE: expect_exit_code_in", + "", + "EXPECTED:", + f" exit code in {expect_exit_code_in}", + "", + "ACTUAL:", + f" exit code = {actual_exit_code}", + ] + ) + ) + + return passed, conditions + + +try: # Support package imports and direct harness module loading. + from .runner_check_execution import ( + CHECK_HANDLERS, + apply_skip_controls, + build_cli_command, + build_cli_env, + build_command_from_spec, + build_runtime_env, + check_cli, + check_command_exit_code, + check_command_output_contains, + check_command_output_regex, + check_command_success, + check_file_exists, + check_function_exists, + check_function_exists_any, + check_main_guard, + check_module_cli, + check_module_main_with_env, + check_path_exists, + check_path_not_exists, + check_py_compile, + check_py_function, + check_source_contains, + check_source_contains_all, + check_source_contains_any, + execute_command_spec, + normalize_probe_spec, + run_command_assertion, + run_single_check, + ) +except ImportError: # pragma: no cover - exercised by flat-module harness imports. + from runner_check_execution import ( + CHECK_HANDLERS, + apply_skip_controls, + build_cli_command, + build_cli_env, + build_command_from_spec, + build_runtime_env, + check_cli, + check_command_exit_code, + check_command_output_contains, + check_command_output_regex, + check_command_success, + check_file_exists, + check_function_exists, + check_function_exists_any, + check_main_guard, + check_module_cli, + check_module_main_with_env, + check_path_exists, + check_path_not_exists, + check_py_compile, + check_py_function, + check_source_contains, + check_source_contains_all, + check_source_contains_any, + execute_command_spec, + normalize_probe_spec, + run_command_assertion, + run_single_check, + ) diff --git a/common/acs_test_framework_runner/runner_reporting.py b/common/acs_test_framework_runner/runner_reporting.py new file mode 100644 index 00000000..14fa1f29 --- /dev/null +++ b/common/acs_test_framework_runner/runner_reporting.py @@ -0,0 +1,360 @@ +from __future__ import annotations + +import shutil +import xml.etree.ElementTree as xml_et +from datetime import datetime +from pathlib import Path +from threading import Lock + +try: # Support package imports and direct harness module loading. + from .runner_checks import ( + TestMeta, + TestOutcome, + create_outcome, + detect_project_root, + sanitize_name, + sanitize_xml_text, + ) +except ImportError: # pragma: no cover - exercised by flat-module harness imports. + from runner_checks import ( + TestMeta, + TestOutcome, + create_outcome, + detect_project_root, + sanitize_name, + sanitize_xml_text, + ) + +SCRIPT_DIR = Path(__file__).resolve().parent +PROJECT_ROOT = detect_project_root(SCRIPT_DIR) +REPORTS_DIR = PROJECT_ROOT / "common" / "reports" +PLACEHOLDER_XML = REPORTS_DIR / "pytest-placeholder.xml" +LOG_SEPARATOR = "=" * 100 +LOG_WRITE_LOCK = Lock() + + +def append_run_header( + file_work_dir: Path, + suite_name: str, + yaml_file: Path, + targets: list[str], +) -> None: + file_work_dir.mkdir(parents=True, exist_ok=True) + log_path = file_work_dir / "combined.log" + + try: + yaml_display = yaml_file.relative_to(PROJECT_ROOT).as_posix() + except ValueError: + yaml_display = str(yaml_file) + + lines = [ + "=" * 100, + f"RUN START: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", + f"YAML: {yaml_display}", + f"SUITE: {suite_name}", + f"TARGETS: {', '.join(targets)}", + "=" * 100, + "", + ] + + with LOG_WRITE_LOCK: + with log_path.open("a", encoding="utf-8") as handle: + handle.write("\n".join(lines)) + handle.write("\n") + + +def build_report_path(group_name: str, yaml_file: Path) -> Path: + safe_group = sanitize_name(group_name) + safe_yaml = sanitize_name(yaml_file.stem) + if safe_group == safe_yaml: + return REPORTS_DIR / f"{safe_yaml}.xml" + return REPORTS_DIR / f"{safe_group}__{safe_yaml}.xml" + + +def create_placeholder_xml(reason: str) -> None: + REPORTS_DIR.mkdir(parents=True, exist_ok=True) + + testsuite = xml_et.Element("testsuite") + testsuite.set("name", sanitize_xml_text("pytest")) + testsuite.set("tests", "0") + testsuite.set("failures", "0") + testsuite.set("errors", "0") + testsuite.set("skipped", "0") + + properties = xml_et.SubElement(testsuite, "properties") + + reason_prop = xml_et.SubElement(properties, "property") + reason_prop.set("name", "reason") + reason_prop.set("value", sanitize_xml_text(reason)) + + placeholder_prop = xml_et.SubElement(properties, "property") + placeholder_prop.set("name", "placeholder") + placeholder_prop.set("value", "true") + + system_out = xml_et.SubElement(testsuite, "system-out") + system_out.text = sanitize_xml_text(reason) + + xml_et.ElementTree(testsuite).write( + PLACEHOLDER_XML, + encoding="utf-8", + xml_declaration=True, + ) + + +def remove_placeholder_xml() -> None: + if PLACEHOLDER_XML.exists(): + PLACEHOLDER_XML.unlink() + + +def cleanup_old_pytest_xml_reports() -> None: + REPORTS_DIR.mkdir(parents=True, exist_ok=True) + for xml_file in REPORTS_DIR.glob("*.xml"): + if xml_file.name == "pylint-report.xml": + continue + xml_file.unlink(missing_ok=True) + + work_root = REPORTS_DIR / "_work" + if work_root.exists(): + shutil.rmtree(work_root, ignore_errors=True) + + +def build_case_log_lines( + testcase_name: str, + status: str, + message: str, + details: str, + description: str | None = None, +) -> list[str]: + normalized_description = "" + if description is not None: + normalized_description = " ".join( + line.strip() + for line in sanitize_xml_text(description).splitlines() + if line.strip() + ) + + lines = [ + "=" * 80, + f"TEST CASE: {testcase_name}", + ] + if normalized_description: + lines.append(f"DESCRIPTION: {normalized_description}") + lines.extend([ + f"STATUS: {status}", + f"MESSAGE: {message}", + ]) + + if details: + lines.extend(["", details.rstrip()]) + + return lines + + +def append_combined_case_log( + file_work_dir: Path, + testcase_name: str, + status: str, + message: str, + details: str, + description: str | None = None, +) -> None: + file_work_dir.mkdir(parents=True, exist_ok=True) + log_path = file_work_dir / "combined.log" + lines = build_case_log_lines( + testcase_name, + status, + message, + details, + description=description, + ) + + with LOG_WRITE_LOCK: + with log_path.open("a", encoding="utf-8") as handle: + handle.write("\n".join(lines).rstrip()) + handle.write("\n\n") + + +def write_case_log( + case_work_dir: Path, + testcase_name: str, + status: str, + message: str, + details: str, + description: str | None = None, +) -> None: + case_work_dir.mkdir(parents=True, exist_ok=True) + log_path = case_work_dir / "case.log" + lines = build_case_log_lines( + testcase_name, + status, + message, + details, + description=description, + ) + + with LOG_WRITE_LOCK: + log_path.write_text( + "\n".join(lines).rstrip() + "\n", + encoding="utf-8", + ) + + +def write_junit_xml( + xml_report: Path, + suite_name: str, + yaml_file: Path, + outcomes: list[TestOutcome], +) -> None: + tests = len(outcomes) + failures = sum( + 1 + for item in outcomes + if not item.passed and not item.error and not item.skipped and not item.warning + ) + errors = sum(1 for item in outcomes if item.error) + skipped = sum(1 for item in outcomes if item.skipped) + # Keep the XML summary property aligned with the console summary, where + # hardware-gated skips are surfaced as warnings rather than hard failures. + warnings = sum(1 for item in outcomes if item.skipped or item.warning) + passed = sum( + 1 for item in outcomes if item.passed and not item.skipped and not item.warning + ) + + testsuite = xml_et.Element("testsuite") + testsuite.set("name", sanitize_xml_text(suite_name)) + testsuite.set("tests", str(tests)) + testsuite.set("failures", str(failures)) + testsuite.set("errors", str(errors)) + testsuite.set("skipped", str(skipped)) + + properties = xml_et.SubElement(testsuite, "properties") + + yaml_prop = xml_et.SubElement(properties, "property") + yaml_prop.set("name", "yaml_file") + yaml_prop.set( + "value", + sanitize_xml_text(yaml_file.relative_to(PROJECT_ROOT).as_posix()), + ) + + passed_prop = xml_et.SubElement(properties, "property") + passed_prop.set("name", "passed") + passed_prop.set("value", sanitize_xml_text(str(passed))) + + warnings_prop = xml_et.SubElement(properties, "property") + warnings_prop.set("name", "warnings") + warnings_prop.set("value", sanitize_xml_text(str(warnings))) + + for outcome in outcomes: + testcase = xml_et.SubElement(testsuite, "testcase") + testcase.set( + "classname", + sanitize_xml_text(sanitize_name(outcome.file_path or suite_name)), + ) + testcase.set("name", sanitize_xml_text(outcome.testcase_name)) + testcase.set("file", sanitize_xml_text(outcome.file_path)) + + if outcome.skipped: + skipped_node = xml_et.SubElement(testcase, "skipped") + skipped_node.set("message", sanitize_xml_text(outcome.message)) + skipped_node.text = sanitize_xml_text(outcome.details) + elif not outcome.passed and outcome.error: + error_node = xml_et.SubElement(testcase, "error") + error_node.set("message", sanitize_xml_text(outcome.message)) + error_node.text = sanitize_xml_text(outcome.details) + elif not outcome.passed: + failure_node = xml_et.SubElement(testcase, "failure") + failure_node.set("message", sanitize_xml_text(outcome.message)) + failure_node.text = sanitize_xml_text(outcome.details) + + system_out = xml_et.SubElement(testcase, "system-out") + body = [ + f"file={outcome.file_path}", + f"suite={outcome.meta.suite_name}", + f"phase={outcome.meta.phase}", + f"type={outcome.meta.test_type}", + f"warning={outcome.warning}", + f"message={outcome.message}", + ] + if outcome.details: + body.extend(["details:", outcome.details]) + + system_out.text = sanitize_xml_text("\n".join(body)) + + REPORTS_DIR.mkdir(parents=True, exist_ok=True) + xml_et.ElementTree(testsuite).write( + xml_report, + encoding="utf-8", + xml_declaration=True, + ) + + +def print_group_summary( + suite_name: str, + outcomes: list[TestOutcome], + xml_report: Path, +) -> None: + total = len(outcomes) + passed = sum( + 1 for item in outcomes if item.passed and not item.skipped and not item.warning + ) + failures = sum( + 1 + for item in outcomes + if not item.passed and not item.error and not item.skipped and not item.warning + ) + errors = sum(1 for item in outcomes if item.error) + warnings = sum(1 for item in outcomes if item.skipped or item.warning) + + print(f"\n[INFO] Finished group : {suite_name}") + print( + f"[INFO] XML report : " + f"{xml_report.relative_to(PROJECT_ROOT).as_posix()}" + ) + print(f"Total : {total}") + print(f"Passed : {passed}") + print(f"Failed : {failures}") + print(f"Errors : {errors}") + print(f"Warnings: {warnings}") + + failed_items = [ + item for item in outcomes + if not item.passed and not item.skipped and not item.warning + ] + if failed_items: + print("\n[FAILURES]") + for item in failed_items: + kind = "ERROR" if item.error else "FAIL" + print( + f" - [{kind}] " + f"{item.file_path} :: {item.testcase_name} :: {item.message}" + ) + + warning_items = [item for item in outcomes if item.skipped or item.warning] + if warning_items: + print("\n[WARNINGS]") + for item in warning_items: + label = "[WARNING]" if item.warning else "[HW WARNING]" + print( + f" - {label} " + f"{item.file_path} :: {item.testcase_name} :: {item.message}" + ) + + +def build_config_error_outcome( + yaml_file: Path, + message: str, + details: str, +) -> TestOutcome: + return create_outcome( + testcase_name="config::load_yaml", + file_path=yaml_file.relative_to(PROJECT_ROOT).as_posix(), + passed=False, + message=sanitize_xml_text(message), + meta=TestMeta( + suite_name="config", + phase="config", + test_type="config", + ), + details=sanitize_xml_text(details), + error=True, + ) diff --git a/common/acs_test_framework_runner/test_pytest_runner_selection.py b/common/acs_test_framework_runner/test_pytest_runner_selection.py new file mode 100644 index 00000000..d7697109 --- /dev/null +++ b/common/acs_test_framework_runner/test_pytest_runner_selection.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +import importlib.util +import shutil +import sys +from pathlib import Path + + +HARNESS_DIR = Path(__file__).resolve().parent +if str(HARNESS_DIR) not in sys.path: + sys.path.insert(0, str(HARNESS_DIR)) + + +def load_harness_module(module_name: str, filename: str): + spec = importlib.util.spec_from_file_location(module_name, HARNESS_DIR / filename) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +pytest_runner = load_harness_module("yaml_harness_pytest_runner", "pytest_runner.py") +report = load_harness_module("yaml_harness_report", "report.py") +runner_reporting = load_harness_module( + "yaml_harness_runner_reporting", + "runner_reporting.py", +) + + +def test_harness_sources_changed_ignores_pycache() -> None: + changed_paths = { + pytest_runner.HARNESS_DIR / "__pycache__" / "pytest_runner.cpython-313.pyc", + } + + assert pytest_runner.harness_sources_changed(changed_paths) is False + + +def test_select_yaml_runs_runs_all_groups_when_harness_changes(monkeypatch) -> None: + yaml_files = [ + pytest_runner.PROJECT_ROOT / "common" / "acs_test_framework_manifests" / "group_one.yaml", + pytest_runner.PROJECT_ROOT / "common" / "acs_test_framework_manifests" / "group_two.yaml", + ] + changed_paths = {pytest_runner.HARNESS_DIR / "mock_loader.py"} + + monkeypatch.setattr( + pytest_runner, + "get_recently_changed_paths", + lambda: changed_paths, + ) + monkeypatch.setattr( + pytest_runner, + "yaml_targets_changed_files", + lambda yaml_file, _changed: ({f"{yaml_file.stem}.py"}, None), + ) + monkeypatch.setattr( + pytest_runner, + "get_yaml_target_entries", + lambda yaml_file: ({f"{yaml_file.stem}.py", f"{yaml_file.stem}_extra.py"}, None), + ) + + selected_runs, warnings = pytest_runner.select_yaml_runs(yaml_files) + + assert warnings == [] + assert selected_runs == [ + (yaml_files[0], {"group_one.py", "group_one_extra.py"}), + (yaml_files[1], {"group_two.py", "group_two_extra.py"}), + ] + + +def test_select_yaml_runs_preserves_target_only_selection(monkeypatch) -> None: + yaml_files = [pytest_runner.PROJECT_ROOT / "common" / "acs_test_framework_manifests" / "group_one.yaml"] + + monkeypatch.setattr( + pytest_runner, + "get_recently_changed_paths", + lambda: {pytest_runner.PROJECT_ROOT / "common" / "linux_scripts" / "target.py"}, + ) + monkeypatch.setattr( + pytest_runner, + "yaml_targets_changed_files", + lambda _yaml_file, _changed: ({"target.py"}, None), + ) + get_yaml_target_entries_called = False + + def fake_get_yaml_target_entries(_yaml_file: Path) -> tuple[set[str], None]: + nonlocal get_yaml_target_entries_called + get_yaml_target_entries_called = True + return {"target.py", "unrelated.py"}, None + + monkeypatch.setattr( + pytest_runner, + "get_yaml_target_entries", + fake_get_yaml_target_entries, + ) + + selected_runs, warnings = pytest_runner.select_yaml_runs(yaml_files) + + assert warnings == [] + assert get_yaml_target_entries_called is False + assert selected_runs == [(yaml_files[0], {"target.py"})] + + +def test_report_changed_yaml_adds_python_targets_for_static_checks(monkeypatch) -> None: + yaml_path = report.PROJECT_ROOT / "common" / "acs_test_framework_manifests" / "group_one.yaml" + yaml_target = report.PROJECT_ROOT / "common" / "linux_scripts" / "target.py" + direct_python_change = report.PROJECT_ROOT / "common" / "linux_scripts" / "direct.py" + + monkeypatch.setattr( + report, + "get_recently_changed_paths", + lambda: [yaml_path, direct_python_change], + ) + monkeypatch.setattr( + report, + "is_yaml_suite_file", + lambda changed_path: changed_path == yaml_path, + ) + monkeypatch.setattr( + report, + "get_python_targets_from_yaml_file", + lambda changed_yaml: {yaml_target} if changed_yaml == yaml_path else set(), + ) + + assert report.get_recently_changed_python_files() == [ + direct_python_change, + yaml_target, + ] + + +def test_report_yaml_target_filter_keeps_only_existing_python_files(monkeypatch) -> None: + yaml_path = report.PROJECT_ROOT / "common" / "acs_test_framework_manifests" / "group_one.yaml" + py_target = report.PROJECT_ROOT / "common" / "linux_scripts" / "target.py" + sh_target = report.PROJECT_ROOT / "common" / "linux_scripts" / "target.sh" + + monkeypatch.setattr(report, "load_yaml_config", lambda _yaml_file: {"suites": []}) + monkeypatch.setattr( + report, + "normalize_suites", + lambda _config: [{"files": ["common/linux_scripts/target.py", "common/linux_scripts/target.sh"]}], + ) + + def fake_exists(self) -> bool: + return self in {yaml_path, py_target, sh_target} + + def fake_is_file(self) -> bool: + return self in {yaml_path, py_target, sh_target} + + monkeypatch.setattr(Path, "exists", fake_exists) + monkeypatch.setattr(Path, "is_file", fake_is_file) + + assert report.get_python_targets_from_yaml_file(yaml_path) == {py_target} + + +def test_collect_pytest_case_logs_reads_persisted_case_description( + monkeypatch, +) -> None: + temp_path = HARNESS_DIR / "_case_log_test_reports" + shutil.rmtree(temp_path, ignore_errors=True) + try: + runner_reporting.append_combined_case_log( + file_work_dir=temp_path / "_work" / "suite" / "target", + testcase_name="suite::target.py::case_with_description", + status="PASS", + message="case passed", + details="", + description="First line of description\nSecond line of description", + ) + monkeypatch.setattr(report, "REPORTS_DIR", temp_path) + + assert report.collect_pytest_case_logs() == [ + { + "name": "suite::target.py::case_with_description", + "description": "First line of description Second line of description", + "status": "passed", + "details": "case passed", + } + ] + finally: + shutil.rmtree(temp_path, ignore_errors=True) + + +def test_collect_pytest_case_logs_skips_missing_case_description( + monkeypatch, +) -> None: + temp_path = HARNESS_DIR / "_case_log_test_reports" + shutil.rmtree(temp_path, ignore_errors=True) + try: + runner_reporting.append_combined_case_log( + file_work_dir=temp_path / "_work" / "suite" / "target", + testcase_name="suite::target.py::case_without_description", + status="PASS", + message="case passed", + details="", + ) + monkeypatch.setattr(report, "REPORTS_DIR", temp_path) + + assert report.collect_pytest_case_logs() == [ + { + "name": "suite::target.py::case_without_description", + "description": "", + "status": "passed", + "details": "case passed", + } + ] + finally: + shutil.rmtree(temp_path, ignore_errors=True) diff --git a/docs/yaml_test_framework_readme.md b/docs/yaml_test_framework_readme.md new file mode 100644 index 00000000..04aee825 --- /dev/null +++ b/docs/yaml_test_framework_readme.md @@ -0,0 +1,522 @@ +# YAML Test Framework README + +## Purpose + +This framework lets us test repository scripts by declaring suites and cases in YAML instead of writing a separate Python test module for every script. + +It is built around: + +- `common/yaml_test_harness/report.py` as the main user-facing entry point +- `common/test_yaml/` for YAML suite definitions +- `common/yaml_test_harness/pytest_runner.py` as the underlying YAML execution engine +- `common/yaml_test_harness/runner_checks.py` for schema validation, execution modes, and expectations +- `common/yaml_test_harness/mock_loader.py` for scenario expansion and mock construction +- `common/yaml_test_harness/case_data_builders.py` for reusable case workspace builders and template expansion +- `common/yaml_test_harness/mock_helpers.py` for reusable mock routers and verification helpers +- `common/reports/` for JUnit XML, combined logs, and per-case work directories + +For normal use, run `python3 common/yaml_test_harness/report.py` from the repository root. Internally it calls `common/yaml_test_harness/pytest_runner.py`, which is a standalone Python runner rather than a standard pytest parametrization file. + +## Prerequisites + +Required: + +- Python 3 +- `PyYAML` + +Install the required Python package with: + +```bash +python3 -m pip install PyYAML +``` + +Optional but recommended when using `report.py`: + +- `pylint` +- `mypy` + +Install them with: + +```bash +python3 -m pip install pylint mypy +``` + +Notes: + +- The YAML framework itself only needs `PyYAML` in addition to the Python standard library. +- If `pylint` or `mypy` are not installed, `report.py` still runs the YAML test flow and prints warnings instead of failing only because those tools are missing. +- Pylint uses the repo-root `.pylintrc` file automatically because `report.py` runs pylint from the repository root. The checked-in config file is `.pylintrc`, not `.pylint.rc`. +- Git is recommended for the default impacted-file mode. If Git metadata is unavailable, use focused runs such as `python3 common/yaml_test_harness/report.py common/log_parser/merge_jsons.py`. +- Hardware-gated suites may also rely on platform commands such as `ip`, `ethtool`, `/bin/sh`, `awk`, `grep`, and `readlink`. Those are not needed for normal scenario-backed runs on a single target unless that target's cases explicitly probe them. + +## How It Works + +At a high level, each run goes through the following stages: + +1. Discover every `.yaml` and `.yml` file under `common/test_yaml/`. +2. Select which YAML suites to run. + - Default mode runs only suites whose `files:` entries match changed files from Git. + - Manual mode runs suites for one explicit target passed to `common/yaml_test_harness/report.py` as its positional `target` argument. +3. Normalize and validate the YAML schema. + - Top-level `suites:` is required. + - Each suite must define `name`, `files`, and `cases`. + - Optional suite-level `defaults` are merged into every case. +4. Build a runtime case. + - If a case contains `scenario:`, the scenario builder generates `args`, `text_files`, `bin_files`, `mocks`, and `patch_constants`. + - Case-level values override generated values where appropriate. + - Runtime tokens such as `{dir}`, `{file}`, and `{filename}` are expanded. +5. Materialize a per-case workspace under `common/reports/_work/...`. + - Generated files, helper scripts, and directory structures are created here. +6. Execute the case using the requested case type. +7. Validate exit code, stdout/stderr content, timeout behavior, and optional post-checks. +8. Write XML and text logs into `common/reports/`. + +The work directory is isolated per case, which keeps generated fixtures and side effects from leaking across cases. + +## Running The Framework + +Use the repo-root wrapper path: + +```bash +python3 common/yaml_test_harness/report.py +``` + +Useful variants: + +```bash +python3 common/yaml_test_harness/report.py common/log_parser/merge_jsons.py +python3 common/yaml_test_harness/report.py common/linux_scripts/verify_tpm_measurements.py +``` + +Notes: + +- `common/yaml_test_harness/report.py` is the main command to share with other users of this framework. +- The positional `target` argument is the simplest way to run focused coverage for one script. +- Example: `python3 common/yaml_test_harness/report.py common/log_parser/merge_jsons.py` +- `report.py` runs the YAML-based test flow and also prints the paired `pylint` and `mypy` summaries for the same target set. +- Internally, `report.py` forwards the YAML test portion to `common/yaml_test_harness/pytest_runner.py`. +- Default mode depends on Git change detection. If the repo is not a Git worktree, or there are no detected changes, nothing will be selected. +- When `report.py` auto-collects changed Python files for `pylint` and `mypy`, it skips generated artifacts under `common/reports/` and `reports/`, including directories such as `_work`, `_runner_work`, and `tpm_check_*`. +- XML reports are written to `common/reports/*.xml`. +- Case logs and generated fixtures go under `common/reports/_work/`. + +## Local Git Hooks + +The repo includes a local pre-commit hook at `.githooks/pre-commit`. + +What it does: + +- runs `common/yaml_test_harness/report.py` from the repository root +- uses the default impacted-file selection logic +- writes a hook-specific console capture to `common/reports/precommit_report.log` +- blocks the commit when the report flow exits non-zero + +Configure the hook path for this clone: + +```bash +git config core.hooksPath .githooks +git config --get core.hooksPath +``` + +If needed, make the hook executable: + +```bash +chmod +x .githooks/pre-commit +``` + +Run it manually without creating a commit: + +```bash +bash .githooks/pre-commit +``` + +Notes: + +- Local hooks only work in a real Git clone/worktree. They do not apply to an extracted ZIP snapshot. +- The hook script is Bash-based and intended to be run from a normal Linux shell. +- To disable the custom hook path for a clone, run `git config --unset core.hooksPath`. + +## Report Structure + +Main output directory: + +```text +common/reports/ + .xml + precommit_report.log + pylint-report.xml + pylint.log + mypy-report.xml + mypy.log + pytest.log + pytest-placeholder.xml # only when no YAML groups are selected + _work/ + / + / + combined.log + _/ + case.log + ...generated files for that case... + _runner_work/ + runner_env_/ # internal temp dirs for some in-process modes +``` + +What the main files mean: + +- `common/reports/.xml`: JUnit-style XML for one YAML suite group. Example: `merge_jsons.xml`. +- `common/reports/precommit_report.log`: stdout/stderr captured by the local pre-commit hook. +- `common/reports/pylint-report.xml`: XML summary produced by the `pylint` part of `report.py`. +- `common/reports/mypy-report.xml`: XML summary produced by the `mypy` part of `report.py`. +- `common/reports/pytest.log`: captured stdout/stderr from the YAML runner. +- `common/reports/pylint.log`: captured stdout/stderr from pylint, including the scored and parseable outputs used by the wrapper. +- `common/reports/mypy.log`: captured stdout/stderr from mypy. +- `common/reports/_work///combined.log`: one running log for all cases that executed against that target file in that suite. +- `common/reports/_work////case.log`: one detailed log for a single case, including command, timeout, exit code, stdout/stderr, and post-check details when applicable. +- `common/reports/_runner_work/runner_env_/`: internal scratch directories used by runner-managed execution modes such as `module_main_with_env`. + +## Logs Info + +`pytest.log`, `pylint.log`, and `mypy.log` are history logs maintained by `report.py`: + +- Each run is appended with a timestamped separator. +- The wrapper keeps the most recent 50 log entries. +- These files are the first place to look if the console output was truncated or if you need the raw stdout/stderr from a previous run. +- Generated report artifacts under `common/reports/` and `reports/` are not treated as source targets for the wrapper's default `pylint` and `mypy` changed-file mode. + +`precommit_report.log` is the hook-specific run log: + +- It is overwritten by each hook run rather than kept as a rolling history log. +- Use it when a commit is blocked and you want the exact hook console output in one file. + +`combined.log` is useful for target-level triage: + +- It starts with a run header showing the YAML file, suite name, and target path. +- It then lists every executed test case with `TEST CASE`, `STATUS`, and `MESSAGE`. +- Use it when you want one compact chronological view of what happened for a target file. + +`case.log` is useful for root-cause analysis: + +- Runtime cases usually include command line, shell mode, timeout, exit code, stdout, stderr, and post-check output. +- Mock verification failures and configuration errors are also written here. +- Generated per-case files live beside the case log in the same case work directory. + +XML report details: + +- Each testcase in the JUnit XML includes `file`, `suite`, `phase`, `type`, `warning`, `message`, and optional `details` in `system-out`. +- The XML `` section records at least the source YAML file and summary counts such as passed and warnings. + +## YAML Structure + +Minimal shape: + +```yaml +suites: + - name: example_suite + files: + - common/linux_scripts/example.py + defaults: + timeout_sec: 20 + cases: + - name: file_exists + type: file_exists + + - name: cli_happy_path + type: cli + args: + - "{dir}/input.yaml" + text_files: + input.yaml: | + enabled: true + expect_exit_code: 0 + expect_stdout_or_stderr_contains: + - "PASS" +``` + +Suite fields: + +| Field | Meaning | +| --- | --- | +| `name` | Logical suite name used in reports and work directories | +| `files` | One or more target files covered by the suite | +| `command` | Optional suite-wide command override | +| `defaults` | Optional case defaults merged into each case | +| `cases` | List of case definitions | + +Common case fields: + +| Field | Meaning | +| --- | --- | +| `name` | Required case name | +| `type` | Execution/check type | +| `description` | Optional explanation for humans | +| `args` / `kwargs` | Arguments for CLI or function-style cases | +| `command` | Override the default interpreter command | +| `env` | Environment variables for the case | +| `stdin` | Text passed to stdin | +| `timeout_sec` | Command timeout | +| `scripts` | Helper scripts written into the case workspace | +| `text_files` / `bin_files` | Generated fixture files written into the workspace | +| `dir_structure` | Directories to create before execution | +| `mocks` | Explicit patch definitions | +| `patch_constants` | Attribute overrides patched into imported modules | +| `post_checks` | File existence/content checks after execution | +| `warn_only` | Treat a functional failure as a warning instead of a failing case | +| `skip_unless_env` | Skip unless required env vars are present or match | +| `skip_unless_paths_exist` | Skip when required hardware-visible paths do not exist | +| `skip_unless_commands_succeed` | Skip when probe commands fail | +| `requires_destructive` | Skip unless `RUN_DESTRUCTIVE_HW_TESTS=1` is set | + +Output expectation fields commonly used by runtime cases: + +- `expect_exit_code` +- `expect_exit_code_in` +- `expect_exit_nonzero` +- `expect_timeout` +- `expect_output` +- `expect_stdout_contains` +- `expect_stderr_contains` +- `expect_stdout_or_stderr_contains` +- `expect_stdout_or_stderr_regex` + +Supported `post_checks`: + +- `exists` +- `not_exists` +- `file_contains` +- `file_not_contains` +- `file_not_empty` +- `regex` +- `ordered_contains` + +Runtime tokens: + +- `{dir}`: per-case work directory +- `{file}`: target file path being tested +- `{filename}`: basename of the target file + +Case-type-specific fields: + +- `py_function`: requires `function`; can also use `expect_exception` and `expect_return_contains` +- `path_exists` / `path_not_exists`: require `path` +- `command_success`, `command_exit_code`, `command_output_contains`, and `command_output_regex`: use `command`, `args`, optional `shell`, optional `cwd`, optional `env`, and optional `timeout_sec` +- `command_exit_code`: requires `expect_exit_code` +- `command_output_contains`: requires `expect_text` +- `command_output_regex`: requires `expect_pattern` + +## Supported Case Types + +Static and source-level checks: + +- `file_exists` +- `py_compile` +- `source_contains` +- `source_contains_any` +- `source_contains_all` +- `function_exists` +- `function_exists_any` +- `main_guard` +- `path_exists` +- `path_not_exists` + +Runtime and execution checks: + +- `cli` +- `module_cli` +- `py_function` +- `module_main_with_env` + +Command/probe checks: + +- `command_success` +- `command_exit_code` +- `command_output_contains` +- `command_output_regex` + +When to use which runtime type: + +- `cli`: runs the target in a real subprocess. Use this when you want real process behavior and do not need in-process patching. +- `module_cli`: executes the script as `__main__` in-process. Use this when you need deterministic mocks, generated patch constants, or scenario-generated runtime behavior. +- `module_main_with_env`: imports the module, patches attributes, changes cwd/env, and calls `main()` directly. +- `py_function`: imports the module and calls one named function. + +## Choosing The Right Test Style + +Start with the least powerful style that can still prove the requirement. + +### 1. Static checks + +Use static checks when you only need to prove source or presence facts: + +- the file exists and parses +- a function or main guard is present +- required source text or patterns exist +- a required path exists or does not exist on the current machine + +Choose this style when: + +- you want the fastest and least brittle coverage +- the behavior under test is structural rather than runtime behavior +- running the script would add noise without adding confidence + +Do not use static checks when you actually need to prove exit codes, stdout/stderr, generated files, side effects, or control flow. + +### 2. Subprocess CLI execution + +Use `type: cli` when the script should be exercised the way a user runs it: + +- real interpreter startup +- real `sys.argv`, stdin, stdout, and stderr flow +- real process exit codes and timeout behavior +- cases where shell mode, `command`, `cwd`, or environment handling matter + +Choose this style when: + +- process boundaries matter +- you want the closest match to production CLI behavior +- you do not need in-process mocks or patched constants + +Important limitation: + +- `cli` runs in a separate process, so in-process `mocks:` and `patch_constants:` do not affect the target script. +- `cli` can still use generated files, args, env, and `post_checks`, but it is the wrong choice when the test depends on patched Python internals. + +### 3. In-Process Execution + +Use in-process execution when you need the runner to patch or control Python state inside the target: + +- `module_cli`: best when the script is still being tested as a CLI entry point, but you need deterministic `mocks:`, generated `patch_constants:`, or scenario-generated runtime setup +- `module_main_with_env`: best when the module exposes a real `main()` and you want to call it directly under a controlled cwd and environment +- `py_function`: best for narrow unit-like checks of one function's return value or raised exception + +Choose this style when: + +- you need mocked `subprocess`, `open`, helper functions, or module attributes +- you want scenario builders to inject runtime setup into the same interpreter +- you need tighter control than a real subprocess allows + +Tradeoff: + +- in-process modes are less faithful to a true separate process than `cli` +- `module_main_with_env` skips the normal `__main__` execution path and calls `main()` directly, so use it only when that is the behavior you actually want to verify + +### 4. Scenario-Backed Cases + +A scenario-backed case is not a separate `type:`. It is a runtime case that also includes `scenario:` so a builder in `mock_loader.py` can generate `args`, files, mocks, and patch constants for you. + +Use a scenario-backed case when: + +- the same domain setup is repeated across many cases +- the script is hardware-facing or environment-heavy +- hand-writing `text_files:`, `bin_files:`, `mocks:`, and `patch_constants:` for every case would be noisy and error-prone + +Choose this style when: + +- an existing scenario kind already models the domain well +- the important part of the test is the behavior, not the low-level mock wiring +- you want consistent fixture generation across a whole suite + +Practical rule: + +- most scenario-backed cases should use `module_cli` or `module_main_with_env`, because scenario builders often generate `mocks:` and `patch_constants:` that only in-process execution can see +- use `cli` with `scenario:` only when the scenario is effectively just generating files, args, or environment input and does not rely on in-process patching + +Short decision guide: + +1. If a structural check is enough, use a static case type. +2. If real process behavior matters, use `cli`. +3. If mocks or patched Python state matter, use an in-process type. +4. If setup is repetitive or domain-specific, add `scenario:` on top of the appropriate runtime type. + +## Scenarios And Mocking + +Scenario builders reduce repetitive low-level setup. The currently registered scenario kinds are: + +| Scenario kind | What it auto-generates | +| --- | --- | +| `ethtool` | Network interface fixtures, tool availability, command routers, sysfs-style reads, connectivity outcomes | +| `verify_tpm` | `pcr.yaml`, `event.yaml`, event-log fault injection mocks, expected argument wiring | +| `capsule_vars` | efivarfs-style binary fixtures and capsule variable defaults | +| `acs_info` | ACS info inputs and mocked platform data | +| `merge_jsons` | input JSON trees, ACS info payloads, mode-specific runtime setup | +| `blk_devices` | block-device discovery, partition layout, and subprocess behavior for the script main flow | +| `blk_write_check` | direct write/readback/restore flows for block-device write checks | +| `runtime_device_mapping` | generated DTS/memmap/log inputs and selected file-open patches | + +`blk_devices` is aligned with the current `common/linux_scripts/read_write_check_blk_devices.py` implementation. For full-flow cases, it is intended to drive the real `main()` path under an in-process runtime type, patch `{module}.input_with_timeout`, and mock the current command forms such as `lsblk -e 7 -d -n -o NAME,TYPE`, `lsblk -rn -o NAME,TYPE /dev/`, and `dd if=/dev/ of=/dev/null bs=1M count=1`. + +Example of a scenario-backed case: + +```yaml +- name: cli_mock_open_raises_on_event_log + type: module_cli + scenario: + kind: verify_tpm + pcrs: + sha256: ["pcr0", "pcr1", "pcr2", "pcr3", "pcr4", "pcr5", "pcr6", "pcr7"] + event_pcrs: + sha256: ["pcr0", "pcr1", "pcr2", "pcr3", "pcr4", "pcr5", "pcr6", "pcr7"] + event_log_open_error: mocked read failure + expect_exit_nonzero: true + expect_stdout_or_stderr_contains: + - "mocked read failure" +``` + +Explicit `mocks:` are still supported for one-off cases. Mock targets can use `{module}` as a placeholder when the target module name is only known at runtime. + +Scenario-generated passthrough rules can be marked as required, so the test fails if the expected mocked path is never actually hit. That prevents silent mock misses. + +## Current Coverage + +The current YAML catalog covers both utility scripts and hardware-facing flows. + +Scenario-backed script coverage: + +| Suite group | Main focus | +| --- | --- | +| `ethtool_test` | interface discovery, virtual vs physical NIC filtering, tool presence, link state, IPv4/IPv6, gateway and internet probes | +| `verify_tpm_measurements` | CLI validation, missing/invalid inputs, PCR/event matching, event-log read and parse failures | +| `capsule_ondisk_reporting_vars_check` | efivarfs variable parsing, attributes, capsule reporting entries, warn-only documentation of current behavior | +| `runtime_device_mapping_conflict_checker` | DTS and memmap inputs, conflict detection, parser failures, log-open failures, warn-only known behavior | +| `read_write_check_blk_devices` | raw disks, MBR/GPT layouts, precious partitions, write/readback/restore flows, destructive gating | +| `acs_info` | platform-info extraction and formatted output paths | +| `merge_jsons` | merge modes, missing/invalid inputs, ACS info interactions, output generation | + +Generic and utility coverage: + +| Suite group | Main focus | +| --- | --- | +| `apply_waivers` | waiver application logic and CLI behavior | +| `generate_acs_summary` | summary generation behavior | +| `logs_to_json_common`, `sr_logs_to_json_specific`, `edk2_logs_to_json_specific` | log parsing and JSON conversion flows | +| `json_to_html_common` plus suite-specific HTML suites | HTML report generation across FWTS, SCT, TPM, BSA, SBMR, SCMI, standalone, and related outputs | +| `merge_summary` | summary merge behavior | +| `parser_common` | shared parser helpers | +| `extract_capsule_fw_version` | firmware version extraction cases | +| `validate_sh` | shell script validation and environment checks | + +Hardware-gated coverage: + +- `ethtool_test_hardware` +- `verify_tpm_measurements_hardware` +- `capsule_ondisk_reporting_vars_check_hardware` +- `runtime_device_mapping_conflict_checker_hardware` +- `read_write_check_blk_devices_hardware` + +These hardware suites use `skip_unless_paths_exist` and `skip_unless_commands_succeed` so they are skipped cleanly on machines that do not expose the required platform state. + +## Adding A New Case + +Recommended workflow: + +1. Pick the existing YAML suite that already targets the script you want to cover. +2. Use the simplest case type that expresses the requirement. + - Static check if only source structure matters. + - `cli` if subprocess behavior matters and mocks are not needed. + - `module_cli` if you need deterministic mocking or scenario-generated setup. +3. Reuse suite `defaults` for shared flags like `timeout_sec` or expected exit code. +4. Prefer a scenario builder if the domain already has one. +5. Use explicit `mocks:` only for narrow fault injection that is not worth promoting into a scenario builder yet. +6. Add `post_checks` when the script's output is file-based. +7. Run `python3 common/yaml_test_harness/report.py ` and inspect `common/reports/` if the case fails. + +