diff --git a/.githooks/pre-commit b/.githooks/pre-commit
new file mode 100755
index 00000000..ff138a2a
--- /dev/null
+++ b/.githooks/pre-commit
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+set -u
+
+echo "===== PRE-COMMIT CHECKS ====="
+
+REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null)" || {
+ echo "Pre-commit hook requires a real git clone/worktree."
+ exit 1
+}
+
+cd "$REPO_ROOT" || exit 1
+
+if command -v python3 >/dev/null 2>&1; then
+ PYTHON_BIN="python3"
+elif command -v python >/dev/null 2>&1; then
+ PYTHON_BIN="python"
+else
+ echo "Python not found in PATH."
+ exit 1
+fi
+
+if [ -f "$REPO_ROOT/common/acs_test_framework_runner/report.py" ]; then
+ REPORT_SCRIPT="$REPO_ROOT/common/acs_test_framework_runner/report.py"
+
+else
+ echo "report.py not found."
+ echo "Checked:"
+ echo " - $REPO_ROOT/common/acs_test_framework_runner/report.py"
+ exit 1
+fi
+
+HOOK_LOG_DIR="$REPO_ROOT/common/reports"
+HOOK_LOG="$HOOK_LOG_DIR/precommit_report.log"
+
+mkdir -p "$HOOK_LOG_DIR"
+
+"$PYTHON_BIN" "$REPORT_SCRIPT" 2>&1 | tee "$HOOK_LOG"
+STATUS=${PIPESTATUS[0]}
+
+
+if [ "$STATUS" -ne 0 ]; then
+ echo "Pre-commit checks failed."
+ echo "Log saved at: $HOOK_LOG"
+ exit 1
+fi
+
+echo "All checks passed. Proceeding with commit."
+exit 0
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 00000000..87309575
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,10 @@
+
+__pycache__/
+*.py[cod]
+
+common/reports/
+reports/
+__pycache__/
+*.py[cod]
+
+pylint_recent.log
diff --git a/.pylintrc b/.pylintrc
new file mode 100644
index 00000000..0d39b4a2
--- /dev/null
+++ b/.pylintrc
@@ -0,0 +1,641 @@
+[MAIN]
+
+# Analyse import fallback blocks. This can be used to support both Python 2 and
+# 3 compatible code, which means that the block might have code that exists
+# only in one or another interpreter, leading to false positives when analysed.
+analyse-fallback-blocks=no
+
+# Clear in-memory caches upon conclusion of linting. Useful if running pylint
+# in a server-like mode.
+clear-cache-post-run=no
+
+# Load and enable all available extensions. Use --list-extensions to see a list
+# all available extensions.
+#enable-all-extensions=
+
+# In error mode, messages with a category besides ERROR or FATAL are
+# suppressed, and no reports are done by default. Error mode is compatible with
+# disabling specific errors.
+#errors-only=
+
+# Always return a 0 (non-error) status code, even if lint errors are found.
+# This is primarily useful in continuous integration scripts.
+#exit-zero=
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code.
+extension-pkg-allow-list=
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code. (This is an alternative name to extension-pkg-allow-list
+# for backward compatibility.)
+extension-pkg-whitelist=
+
+# Return non-zero exit code if any of these messages/categories are detected,
+# even if score is above --fail-under value. Syntax same as enable. Messages
+# specified are enabled, while categories only check already-enabled messages.
+fail-on=
+
+# Specify a score threshold under which the program will exit with error.
+fail-under=10.0
+
+# Interpret the stdin as a python script, whose filename needs to be passed as
+# the module_or_package argument.
+#from-stdin=
+
+# Files or directories to be skipped. They should be base names, not paths.
+ignore=CVS
+
+# Add files or directories matching the regular expressions patterns to the
+# ignore-list. The regex matches against paths and can be in Posix or Windows
+# format. Because '\\' represents the directory delimiter on Windows systems,
+# it can't be used as an escape character.
+ignore-paths=
+
+# Files or directories matching the regular expression patterns are skipped.
+# The regex matches against base names, not paths. The default value ignores
+# Emacs file locks
+ignore-patterns=^\.#
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis). It
+# supports qualified module names, as well as Unix pattern matching.
+ignored-modules=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
+# number of processors available to use, and will cap the count on Windows to
+# avoid hangs.
+jobs=0
+
+# Control the amount of potential inferred values when inferring a single
+# object. This can help the performance when dealing with large functions or
+# complex, nested conditions.
+limit-inference-results=100
+
+# List of plugins (as comma separated values of python module names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# Minimum Python version to use for version dependent checks. Will default to
+# the version used to run pylint.
+py-version=3.11
+
+# Discover python modules and packages in the file system subtree.
+recursive=no
+
+# Add paths to the list of the source roots. Supports globbing patterns. The
+# source root is an absolute path or a path relative to the current working
+# directory used to determine a package namespace for modules located under the
+# source root.
+source-roots=
+
+# When enabled, pylint would attempt to guess common misconfiguration and emit
+# user-friendly hints instead of false-positive error messages.
+suggestion-mode=yes
+
+# Allow loading of arbitrary C extensions. Extensions are imported into the
+# active Python interpreter and may run arbitrary code.
+unsafe-load-any-extension=no
+
+# In verbose mode, extra non-checker-related info will be displayed.
+#verbose=
+
+
+[BASIC]
+
+# Naming style matching correct argument names.
+argument-naming-style=snake_case
+
+# Regular expression matching correct argument names. Overrides argument-
+# naming-style. If left empty, argument names will be checked with the set
+# naming style.
+#argument-rgx=
+
+# Naming style matching correct attribute names.
+attr-naming-style=snake_case
+
+# Regular expression matching correct attribute names. Overrides attr-naming-
+# style. If left empty, attribute names will be checked with the set naming
+# style.
+#attr-rgx=
+
+# Bad variable names which should always be refused, separated by a comma.
+bad-names=foo,
+ bar,
+ baz,
+ toto,
+ tutu,
+ tata
+
+# Bad variable names regexes, separated by a comma. If names match any regex,
+# they will always be refused
+bad-names-rgxs=
+
+# Naming style matching correct class attribute names.
+class-attribute-naming-style=any
+
+# Regular expression matching correct class attribute names. Overrides class-
+# attribute-naming-style. If left empty, class attribute names will be checked
+# with the set naming style.
+#class-attribute-rgx=
+
+# Naming style matching correct class constant names.
+class-const-naming-style=UPPER_CASE
+
+# Regular expression matching correct class constant names. Overrides class-
+# const-naming-style. If left empty, class constant names will be checked with
+# the set naming style.
+#class-const-rgx=
+
+# Naming style matching correct class names.
+class-naming-style=PascalCase
+
+# Regular expression matching correct class names. Overrides class-naming-
+# style. If left empty, class names will be checked with the set naming style.
+#class-rgx=
+
+# Naming style matching correct constant names.
+const-naming-style=UPPER_CASE
+
+# Regular expression matching correct constant names. Overrides const-naming-
+# style. If left empty, constant names will be checked with the set naming
+# style.
+#const-rgx=
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+# Naming style matching correct function names.
+function-naming-style=snake_case
+
+# Regular expression matching correct function names. Overrides function-
+# naming-style. If left empty, function names will be checked with the set
+# naming style.
+#function-rgx=
+
+# Good variable names which should always be accepted, separated by a comma.
+good-names=i,
+ j,
+ k,
+ ex,
+ Run,
+ _
+
+# Good variable names regexes, separated by a comma. If names match any regex,
+# they will always be accepted
+good-names-rgxs=
+
+# Include a hint for the correct naming format with invalid-name.
+include-naming-hint=no
+
+# Naming style matching correct inline iteration names.
+inlinevar-naming-style=any
+
+# Regular expression matching correct inline iteration names. Overrides
+# inlinevar-naming-style. If left empty, inline iteration names will be checked
+# with the set naming style.
+#inlinevar-rgx=
+
+# Naming style matching correct method names.
+method-naming-style=snake_case
+
+# Regular expression matching correct method names. Overrides method-naming-
+# style. If left empty, method names will be checked with the set naming style.
+#method-rgx=
+
+# Naming style matching correct module names.
+module-naming-style=snake_case
+
+# Regular expression matching correct module names. Overrides module-naming-
+# style. If left empty, module names will be checked with the set naming style.
+#module-rgx=
+
+# Colon-delimited sets of names that determine each other's naming style when
+# the name regexes allow several styles.
+name-group=
+
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=^_
+
+# List of decorators that produce properties, such as abc.abstractproperty. Add
+# to this list to register other decorators that produce valid properties.
+# These decorators are taken in consideration only for invalid-name.
+property-classes=abc.abstractproperty
+
+# Regular expression matching correct type alias names. If left empty, type
+# alias names will be checked with the set naming style.
+#typealias-rgx=
+
+# Regular expression matching correct type variable names. If left empty, type
+# variable names will be checked with the set naming style.
+#typevar-rgx=
+
+# Naming style matching correct variable names.
+variable-naming-style=snake_case
+
+# Regular expression matching correct variable names. Overrides variable-
+# naming-style. If left empty, variable names will be checked with the set
+# naming style.
+#variable-rgx=
+
+
+[CLASSES]
+
+# Warn about protected attribute access inside special methods
+check-protected-access-in-special-methods=no
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,
+ __new__,
+ setUp,
+ asyncSetUp,
+ __post_init__
+
+# List of member names, which should be excluded from the protected access
+# warning.
+exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=mcs
+
+
+[DESIGN]
+
+# List of regular expressions of class ancestor names to ignore when counting
+# public methods (see R0903)
+exclude-too-few-public-methods=
+
+# List of qualified class names to ignore when counting class parents (see
+# R0901)
+ignored-parents=
+
+# Maximum number of arguments for function / method.
+max-args=5
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
+
+# Maximum number of boolean expressions in an if statement (see R0916).
+max-bool-expr=5
+
+# Maximum number of branch for function / method body.
+max-branches=12
+
+# Maximum number of locals for function / method body.
+max-locals=15
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+# Maximum number of return / yield for function / method body.
+max-returns=6
+
+# Maximum number of statements in function / method body.
+max-statements=50
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when caught.
+overgeneral-exceptions=builtins.BaseException,builtins.Exception
+
+
+[FORMAT]
+
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
+
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines=^\s*(# )??$
+
+# Number of spaces of indent required inside a hanging or continued line.
+indent-after-paren=4
+
+# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
+# tab).
+indent-string=' '
+
+# Maximum number of characters on a single line.
+max-line-length=100
+
+# Maximum number of lines in a module.
+max-module-lines=1000
+
+# Allow the body of a class to be on the same line as the declaration if body
+# contains single statement.
+single-line-class-stmt=no
+
+# Allow the body of an if to be on the same line as the test if there is no
+# else.
+single-line-if-stmt=no
+
+
+[IMPORTS]
+
+# List of modules that can be imported at any level, not just the top level
+# one.
+allow-any-import-level=
+
+# Allow explicit reexports by alias from a package __init__.
+allow-reexport-from-package=no
+
+# Allow wildcard imports from modules that define __all__.
+allow-wildcard-with-all=no
+
+# Deprecated modules which should not be used, separated by a comma.
+deprecated-modules=
+
+# Output a graph (.gv or any supported image format) of external dependencies
+# to the given file (report RP0402 must not be disabled).
+ext-import-graph=
+
+# Output a graph (.gv or any supported image format) of all (i.e. internal and
+# external) dependencies to the given file (report RP0402 must not be
+# disabled).
+import-graph=
+
+# Output a graph (.gv or any supported image format) of internal dependencies
+# to the given file (report RP0402 must not be disabled).
+int-import-graph=
+
+# Force import order to recognize a module as part of the standard
+# compatibility libraries.
+known-standard-library=
+
+# Force import order to recognize a module as part of a third party library.
+known-third-party=enchant
+
+# Couples of modules and preferred modules, separated by a comma.
+preferred-modules=
+
+
+[LOGGING]
+
+# The type of string formatting that logging methods do. `old` means using %
+# formatting, `new` is for `{}` formatting.
+logging-format-style=old
+
+# Logging modules to check that the string format arguments are in logging
+# function parameter format.
+logging-modules=logging
+
+
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE,
+# UNDEFINED.
+confidence=HIGH,
+ CONTROL_FLOW,
+ INFERENCE,
+ INFERENCE_FAILURE,
+ UNDEFINED
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once). You can also use "--disable=all" to
+# disable everything first and then re-enable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use "--disable=all --enable=classes
+# --disable=W".
+disable=
+ R0801,
+ broad-exception-caught,
+ broad-exception-raised,
+ invalid-name,
+ logging-fstring-interpolation,
+ missing-class-docstring,
+ missing-function-docstring,
+ missing-module-docstring,
+ redefined-outer-name,
+ too-few-public-methods,
+ too-many-branches,
+ too-many-lines,
+ too-many-locals,
+ too-many-statements,
+ unspecified-encoding
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+enable=c-extension-no-member
+
+
+
+[METHOD_ARGS]
+
+# List of qualified names (i.e., library.method) which require a timeout
+# parameter e.g. 'requests.api.get,requests.api.post'
+timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,
+ XXX,
+ TODO
+
+# Regular expression of note tags to take in consideration.
+notes-rgx=
+
+
+[REFACTORING]
+
+# Maximum number of nested blocks for function / method body
+max-nested-blocks=5
+
+# Complete name of functions that never returns. When checking for
+# inconsistent-return-statements if a never returning function is called then
+# it will be considered as an explicit return statement and no message will be
+# printed.
+never-returning-functions=sys.exit,argparse.parse_error
+
+
+[REPORTS]
+
+# Python expression which should return a score less than or equal to 10. You
+# have access to the variables 'fatal', 'error', 'warning', 'refactor',
+# 'convention', and 'info' which contain the number of messages in each
+# category, as well as 'statement' which is the total number of statements
+# analyzed. This score is used by the global evaluation report (RP0004).
+evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10))
+
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details.
+msg-template=
+
+# Set the output format. Available formats are text, parseable, colorized, json
+# and msvs (visual studio). You can also give a reporter class, e.g.
+# mypackage.mymodule.MyReporterClass.
+#output-format=
+
+# Tells whether to display a full report or only the messages.
+reports=no
+
+# Activate the evaluation score.
+score=yes
+
+
+[SIMILARITIES]
+
+# Comments are removed from the similarity computation
+ignore-comments=yes
+
+# Docstrings are removed from the similarity computation
+ignore-docstrings=yes
+
+# Imports are removed from the similarity computation
+ignore-imports=yes
+
+# Signatures are removed from the similarity computation
+ignore-signatures=yes
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+
+[SPELLING]
+
+# Limits count of emitted suggestions for spelling mistakes.
+max-spelling-suggestions=4
+
+# Spelling dictionary name. No available dictionaries : You need to install
+# both the python package and the system dependency for enchant to work..
+spelling-dict=
+
+# List of comma separated words that should be considered directives if they
+# appear at the beginning of a comment and should not be checked.
+spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy:
+
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
+
+# A path to a file that contains the private dictionary; one word per line.
+spelling-private-dict-file=
+
+# Tells whether to store unknown words to the private dictionary (see the
+# --spelling-private-dict-file option) instead of raising a message.
+spelling-store-unknown-words=no
+
+
+[STRING]
+
+# This flag controls whether inconsistent-quotes generates a warning when the
+# character used as a quote delimiter is used inconsistently within a module.
+check-quote-consistency=no
+
+# This flag controls whether the implicit-str-concat should generate a warning
+# on implicit string concatenation in sequences defined over several lines.
+check-str-concat-over-line-jumps=no
+
+
+[TYPECHECK]
+
+# List of decorators that produce context managers, such as
+# contextlib.contextmanager. Add to this list to register other decorators that
+# produce valid context managers.
+contextmanager-decorators=contextlib.contextmanager
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=
+
+# Tells whether to warn about missing members when the owner of the attribute
+# is inferred to be None.
+ignore-none=yes
+
+# This flag controls whether pylint should warn about no-member and similar
+# checks whenever an opaque object is returned when inferring. The inference
+# can return multiple potential results while evaluating a Python object, but
+# some branches might not be evaluated, which results in partial inference. In
+# that case, it might be useful to still emit no-member and other checks for
+# the rest of the inferred objects.
+ignore-on-opaque-inference=yes
+
+# List of symbolic message names to ignore for Mixin members.
+ignored-checks-for-mixins=no-member,
+ not-async-context-manager,
+ not-context-manager,
+ attribute-defined-outside-init
+
+# List of class names for which member attributes should not be checked (useful
+# for classes with dynamically set attributes). This supports the use of
+# qualified names.
+ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace
+
+# Show a hint with possible names when a member name was not found. The aspect
+# of finding the hint is based on edit distance.
+missing-member-hint=yes
+
+# The minimum edit distance a name should have in order to be considered a
+# similar match for a missing member name.
+missing-member-hint-distance=1
+
+# The total number of similar names that should be taken in consideration when
+# showing a hint for a missing member.
+missing-member-max-choices=1
+
+# Regex pattern to define which classes are considered mixins.
+mixin-class-rgx=.*[Mm]ixin
+
+# List of decorators that change the signature of a decorated function.
+signature-mutators=
+
+
+[VARIABLES]
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid defining new builtins when possible.
+additional-builtins=
+
+# Tells whether unused global variables should be treated as a violation.
+allow-global-unused-variables=yes
+
+# List of names allowed to shadow builtins
+allowed-redefined-builtins=
+
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,
+ _cb
+
+# A regular expression matching the name of dummy variables (i.e. expected to
+# not be used).
+dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
+
+# Argument names that match this expression will be ignored.
+ignored-argument-names=_.*|^ignored_|^unused_
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# List of qualified module names which can have objects that can redefine
+# builtins.
+redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io
+
+
diff --git a/common/acs_test_framework_manifests/acs-info.yaml b/common/acs_test_framework_manifests/acs-info.yaml
new file mode 100644
index 00000000..ba882455
--- /dev/null
+++ b/common/acs_test_framework_manifests/acs-info.yaml
@@ -0,0 +1,283 @@
+suites:
+ - name: acs_info
+ files:
+ - common/log_parser/acs_info.py
+
+ cases:
+ - name: file_exists
+ type: file_exists
+
+ - name: python_compiles
+ type: py_compile
+
+ - name: cli_help_flag
+ type: cli
+ args:
+ - --help
+ expect_exit_code: 0
+ expect_stdout_or_stderr_contains:
+ - "Collect ACS-like system info"
+
+ - name: cli_generates_json_from_dmidecode_only
+ type: cli
+ scenario:
+ kind: acs_info
+ dmidecode:
+ vendor: VendorA
+ product: BoardA
+ family: FamilyA
+ firmware: FW-1.2.3
+ args:
+ - --dmidecode_log
+ - "{dir}/dmidecode.log"
+ - --output_dir
+ - "{dir}/out"
+ expect_exit_code: 0
+ post_checks:
+ - type: exists
+ path: "{dir}/out/acs_info.json"
+ - type: file_contains
+ path: "{dir}/out/acs_info.json"
+ text: "\"Vendor\": \"VendorA\""
+ - type: file_contains
+ path: "{dir}/out/acs_info.json"
+ text: "\"System Name\": \"BoardA\""
+ - type: file_contains
+ path: "{dir}/out/acs_info.json"
+ text: "\"SoC Family\": \"FamilyA\""
+ - type: file_contains
+ path: "{dir}/out/acs_info.json"
+ text: "\"Firmware Version\": \"FW-1.2.3\""
+
+ - name: cli_merges_acs_and_system_config
+ type: cli
+ scenario:
+ kind: acs_info
+ dmidecode:
+ vendor: VendorB
+ product: BoardB
+ family: FamilyB
+ firmware: FW-2.0
+ acs_config:
+ Band: SystemReady Band
+ ACS version: "6.1"
+ system_config:
+ Board Revision: RevC
+ args:
+ - --dmidecode_log
+ - "{dir}/dmidecode.log"
+ - --acs_config_path
+ - "{dir}/acs_config.txt"
+ - --system_config_path
+ - "{dir}/system_config.txt"
+ - --output_dir
+ - "{dir}/out"
+ expect_exit_code: 0
+ post_checks:
+ - type: file_contains
+ path: "{dir}/out/acs_info.json"
+ text: "\"ACS version\": \"6.1\""
+ - type: file_contains
+ path: "{dir}/out/acs_info.json"
+ text: "\"Board Revision\": \"RevC\""
+ - type: file_contains
+ path: "{dir}/out/acs_info.json"
+ text: "\"Band\": \"SystemReady Band\""
+
+ - name: cli_stops_parsing_at_user_defined_configs_marker
+ type: cli
+ scenario:
+ kind: acs_info
+ dmidecode:
+ vendor: VendorC
+ product: BoardC
+ family: FamilyC
+ firmware: FW-3.0
+ text_files:
+ acs_config.txt: |
+ Band: SystemReady Band
+ ACS version: 7.0
+ # User-defined configs
+ Secret Field: should_not_appear
+ args:
+ - --dmidecode_log
+ - "{dir}/dmidecode.log"
+ - --acs_config_path
+ - "{dir}/acs_config.txt"
+ - --output_dir
+ - "{dir}/out"
+ expect_exit_code: 0
+ post_checks:
+ - type: file_contains
+ path: "{dir}/out/acs_info.json"
+ text: "\"ACS version\": \"7.0\""
+ - type: file_not_contains
+ path: "{dir}/out/acs_info.json"
+ text: "should_not_appear"
+
+ - name: cli_reads_utf16_uefi_version
+ type: cli
+ scenario:
+ kind: acs_info
+ dmidecode:
+ vendor: VendorD
+ product: BoardD
+ family: FamilyD
+ firmware: FW-4.0
+ args:
+ - --dmidecode_log
+ - "{dir}/dmidecode.log"
+ - --uefi_version_log
+ - "{dir}/uefi_version.log"
+ - --output_dir
+ - "{dir}/out"
+ bin_files:
+ uefi_version.log:
+ hex: "fffe55004500460049002000760032002e0039000a00"
+ expect_exit_code: 0
+ post_checks:
+ - type: file_contains
+ path: "{dir}/out/acs_info.json"
+ text: "\"UEFI Version\": \"UEFI v2.9\""
+
+ - name: cli_adds_bmc_firmware_only_for_systemready_band
+ type: cli
+ scenario:
+ kind: acs_info
+ dmidecode:
+ vendor: VendorE
+ product: BoardE
+ family: FamilyE
+ firmware: FW-5.0
+ acs_config:
+ Band: SystemReady Band
+ text_files:
+ ipmitool.log: |
+ Device ID : 32
+ Firmware Revision : 1.45
+ args:
+ - --dmidecode_log
+ - "{dir}/dmidecode.log"
+ - --acs_config_path
+ - "{dir}/acs_config.txt"
+ - --ipmitool_log
+ - "{dir}/ipmitool.log"
+ - --output_dir
+ - "{dir}/out"
+ expect_exit_code: 0
+ post_checks:
+ - type: file_contains
+ path: "{dir}/out/acs_info.json"
+ text: "\"BMC Firmware Version\": \"1.45\""
+
+ - name: cli_does_not_add_bmc_firmware_for_non_systemready_band
+ type: cli
+ scenario:
+ kind: acs_info
+ dmidecode:
+ vendor: VendorF
+ product: BoardF
+ family: FamilyF
+ firmware: FW-6.0
+ acs_config:
+ Band: DeviceTree Band
+ text_files:
+ ipmitool.log: |
+ Firmware Revision : 9.99
+ args:
+ - --dmidecode_log
+ - "{dir}/dmidecode.log"
+ - --acs_config_path
+ - "{dir}/acs_config.txt"
+ - --ipmitool_log
+ - "{dir}/ipmitool.log"
+ - --output_dir
+ - "{dir}/out"
+ expect_exit_code: 0
+ post_checks:
+ - type: file_not_contains
+ path: "{dir}/out/acs_info.json"
+ text: "\"BMC Firmware Version\""
+
+ - name: cli_uses_first_dmidecode_values_not_later_duplicates
+ type: cli
+ text_files:
+ dmidecode.log: |
+ Handle 0x0000, DMI type 0, 24 bytes
+ BIOS Information
+ Version: FW-FIRST
+ Handle 0x0001, DMI type 1, 27 bytes
+ System Information
+ Manufacturer: VendorFirst
+ Product Name: ProductFirst
+ Family: FamilyFirst
+ Handle 0x0002, DMI type 0, 24 bytes
+ BIOS Information
+ Version: FW-SECOND
+ Handle 0x0003, DMI type 1, 27 bytes
+ System Information
+ Manufacturer: VendorSecond
+ Product Name: ProductSecond
+ Family: FamilySecond
+ args:
+ - --dmidecode_log
+ - "{dir}/dmidecode.log"
+ - --output_dir
+ - "{dir}/out"
+ expect_exit_code: 0
+ post_checks:
+ - type: file_contains
+ path: "{dir}/out/acs_info.json"
+ text: "\"Vendor\": \"VendorFirst\""
+ - type: file_not_contains
+ path: "{dir}/out/acs_info.json"
+ text: "VendorSecond"
+
+ - name: cli_systemready_band_is_case_and_space_insensitive
+ type: cli
+ scenario:
+ kind: acs_info
+ dmidecode:
+ vendor: V
+ product: P
+ family: F
+ firmware: FW-1
+ text_files:
+ acs_config.txt: |
+ Band: systemready band
+ args:
+ - --dmidecode_log
+ - "{dir}/dmidecode.log"
+ - --acs_config_path
+ - "{dir}/acs_config.txt"
+ - --output_dir
+ - "{dir}/out"
+ expect_exit_code: 0
+ post_checks:
+ - type: file_contains
+ path: "{dir}/out/acs_info.json"
+ text: "\"BMC Firmware Version\": \"Unknown\""
+
+ - name: cli_config_value_with_extra_colon_is_preserved
+ type: cli
+ scenario:
+ kind: acs_info
+ dmidecode:
+ vendor: V
+ product: P
+ family: F
+ firmware: FW-1
+ system_config:
+ Firmware Notes: build:nightly:123
+ args:
+ - --dmidecode_log
+ - "{dir}/dmidecode.log"
+ - --system_config_path
+ - "{dir}/system_config.txt"
+ - --output_dir
+ - "{dir}/out"
+ expect_exit_code: 0
+ post_checks:
+ - type: file_contains
+ path: "{dir}/out/acs_info.json"
+ text: "\"Firmware Notes\": \"build:nightly:123\""
\ No newline at end of file
diff --git a/common/acs_test_framework_manifests/apply_waivers.yaml b/common/acs_test_framework_manifests/apply_waivers.yaml
new file mode 100644
index 00000000..5396335c
--- /dev/null
+++ b/common/acs_test_framework_manifests/apply_waivers.yaml
@@ -0,0 +1,702 @@
+suites:
+ - name: apply_waivers
+ files:
+ - common/log_parser/apply_waivers.py
+
+ cases:
+ - name: file_exists
+ type: file_exists
+
+ - name: python_compiles
+ type: py_compile
+
+ - name: cli_help_flag
+ type: cli
+ args:
+ - --help
+ expect_exit_code: 0
+ expect_stdout_or_stderr_contains:
+ - "Apply waivers to test suite JSON results."
+
+ - name: cli_missing_json_file_returns_zero_with_warning
+ type: cli
+ args:
+ - FWTS
+ - "{dir}/missing.json"
+ expect_exit_code: 0
+ expect_stdout_or_stderr_contains:
+ - "WARNING: Failed to read or parse"
+
+ - name: fwts_suite_level_waiver_marks_failed_subtest
+ type: cli
+ text_files:
+ fwts.json: |
+ {
+ "test_results": [
+ {
+ "Test_suite": "DemoSuite",
+ "subtests": [
+ {
+ "sub_Test_Description": "failure one",
+ "sub_Test_Number": "1",
+ "sub_test_result": "FAILED"
+ }
+ ]
+ }
+ ]
+ }
+ waiver.json: |
+ {
+ "Suites": [
+ {
+ "Suite": "FWTS",
+ "Reason": "Known FWTS issue"
+ }
+ ]
+ }
+ args:
+ - FWTS
+ - "{dir}/fwts.json"
+ - "{dir}/waiver.json"
+ expect_exit_code: 0
+ post_checks:
+ - type: file_contains
+ path: "{dir}/fwts.json"
+ text: "WITH WAIVER"
+ - type: file_not_contains
+ path: "{dir}/fwts.json"
+ text: "(WITH WAIVER) (WITH WAIVER)"
+
+ - name: testsuite_level_waiver_applies_only_to_matching_testsuite
+ type: cli
+ text_files:
+ fwts.json: |
+ {
+ "test_results": [
+ {
+ "Test_suite": "MatchSuite",
+ "subtests": [
+ {
+ "sub_Test_Description": "alpha",
+ "sub_Test_Number": "1",
+ "sub_test_result": "FAILED"
+ }
+ ]
+ },
+ {
+ "Test_suite": "OtherSuite",
+ "subtests": [
+ {
+ "sub_Test_Description": "beta",
+ "sub_Test_Number": "1",
+ "sub_test_result": "FAILED"
+ }
+ ]
+ }
+ ]
+ }
+ waiver.json: |
+ {
+ "Suites": [
+ {
+ "Suite": "FWTS",
+ "TestSuites": [
+ {
+ "TestSuite": "MatchSuite",
+ "Reason": "Specific suite waiver"
+ }
+ ]
+ }
+ ]
+ }
+ args:
+ - FWTS
+ - "{dir}/fwts.json"
+ - "{dir}/waiver.json"
+ expect_exit_code: 0
+ post_checks:
+ - type: file_contains
+ path: "{dir}/fwts.json"
+ text: "Specific suite waiver"
+ - type: file_contains
+ path: "{dir}/fwts.json"
+ text: "\"Test_suite\": \"OtherSuite\""
+
+ - name: category_non_waivable_blocks_waiver_application
+ type: cli
+ text_files:
+ fwts.json: |
+ {
+ "test_results": [
+ {
+ "Test_suite": "DemoSuite",
+ "subtests": [
+ {
+ "sub_Test_Description": "failure one",
+ "sub_Test_Number": "1",
+ "sub_test_result": "FAILED"
+ }
+ ]
+ }
+ ]
+ }
+ waiver.json: |
+ {
+ "Suites": [
+ {
+ "Suite": "FWTS",
+ "Reason": "Known FWTS issue"
+ }
+ ]
+ }
+ category.json: |
+ {
+ "cat1": [
+ {
+ "Suite": "FWTS",
+ "Test Suite": "DemoSuite",
+ "Waivable": "No"
+ }
+ ]
+ }
+ args:
+ - FWTS
+ - "{dir}/fwts.json"
+ - "{dir}/waiver.json"
+ - "{dir}/category.json"
+ expect_exit_code: 0
+ post_checks:
+ - type: file_not_contains
+ path: "{dir}/fwts.json"
+ text: "WITH WAIVER"
+
+ - name: category_waivable_yes_allows_waiver_application
+ type: cli
+ text_files:
+ fwts.json: |
+ {
+ "test_results": [
+ {
+ "Test_suite": "DemoSuite",
+ "subtests": [
+ {
+ "sub_Test_Description": "failure one",
+ "sub_Test_Number": "1",
+ "sub_test_result": "FAILED"
+ }
+ ]
+ }
+ ]
+ }
+ waiver.json: |
+ {
+ "Suites": [
+ {
+ "Suite": "FWTS",
+ "Reason": "Known FWTS issue"
+ }
+ ]
+ }
+ category.json: |
+ {
+ "cat1": [
+ {
+ "Suite": "FWTS",
+ "Test Suite": "DemoSuite",
+ "Waivable": "Yes"
+ }
+ ]
+ }
+ args:
+ - FWTS
+ - "{dir}/fwts.json"
+ - "{dir}/waiver.json"
+ - "{dir}/category.json"
+ expect_exit_code: 0
+ post_checks:
+ - type: file_contains
+ path: "{dir}/fwts.json"
+ text: "WITH WAIVER"
+ - type: file_not_contains
+ path: "{dir}/fwts.json"
+ text: "(WITH WAIVER) (WITH WAIVER)"
+
+ - name: fwts_subtest_description_waiver_applies
+ type: cli
+ text_files:
+ fwts.json: |
+ {
+ "test_results": [
+ {
+ "Test_suite": "DemoSuite",
+ "subtests": [
+ {
+ "sub_Test_Description": "something",
+ "sub_Test_Number": "42",
+ "sub_test_result": "FAILED"
+ }
+ ]
+ }
+ ]
+ }
+ waiver.json: |
+ {
+ "Suites": [
+ {
+ "Suite": "FWTS",
+ "TestSuites": [
+ {
+ "TestSuite": "DemoSuite",
+ "TestCase": {
+ "SubTests": [
+ {
+ "sub_Test_Description": "something",
+ "Reason": "Description-based waiver"
+ }
+ ]
+ }
+ }
+ ]
+ }
+ ]
+ }
+ args:
+ - FWTS
+ - "{dir}/fwts.json"
+ - "{dir}/waiver.json"
+ expect_exit_code: 0
+ post_checks:
+ - type: file_contains
+ path: "{dir}/fwts.json"
+ text: "WITH WAIVER"
+ - type: file_contains
+ path: "{dir}/fwts.json"
+ text: "Description-based waiver"
+
+ - name: standalone_subtest_description_waiver_applies
+ type: cli
+ text_files:
+ standalone.json: |
+ {
+ "test_results": [
+ {
+ "Test_suite": "Network",
+ "Test_case": "ethtool_test",
+ "subtests": [
+ {
+ "sub_Test_Description": "Ping to www.arm.com on eth0",
+ "sub_test_result": "FAILED"
+ }
+ ],
+ "test_suite_summary": {
+ "total_passed": 0,
+ "total_failed": 1,
+ "total_failed_with_waiver": 0,
+ "total_aborted": 0,
+ "total_skipped": 0,
+ "total_warnings": 0
+ }
+ }
+ ],
+ "suite_summary": {
+ "total_passed": 0,
+ "total_failed": 1,
+ "total_failed_with_waiver": 0,
+ "total_aborted": 0,
+ "total_skipped": 0,
+ "total_warnings": 0
+ }
+ }
+ waiver.json: |
+ {
+ "Suites": [
+ {
+ "Suite": "Standalone",
+ "TestSuites": [
+ {
+ "TestCase": {
+ "Test_case": "ethtool_test",
+ "SubTests": [
+ {
+ "sub_Test_Description": "Ping to www.arm.com on eth0",
+ "Reason": "Network stack excluded for this SKU"
+ }
+ ]
+ }
+ }
+ ]
+ }
+ ]
+ }
+ args:
+ - Standalone
+ - "{dir}/standalone.json"
+ - "{dir}/waiver.json"
+ expect_exit_code: 0
+ post_checks:
+ - type: regex
+ path: "{dir}/standalone.json"
+ pattern: '"sub_test_result": "[^"]*WITH WAIVER[^"]*"'
+ - type: file_contains
+ path: "{dir}/standalone.json"
+ text: "Network stack excluded for this SKU"
+ - type: file_contains
+ path: "{dir}/standalone.json"
+ text: "\"total_failed\": 0"
+ - type: file_contains
+ path: "{dir}/standalone.json"
+ text: "\"total_failed_with_waiver\": 1"
+
+ - name: fwts_does_not_duplicate_existing_with_waiver_marker
+ type: cli
+ text_files:
+ fwts.json: |
+ {
+ "test_results": [
+ {
+ "Test_suite": "DemoSuite",
+ "subtests": [
+ {
+ "sub_Test_Description": "failure one",
+ "sub_Test_Number": "1",
+ "sub_test_result": "FAILED (WITH WAIVER)"
+ }
+ ]
+ }
+ ]
+ }
+ waiver.json: |
+ {
+ "Suites": [
+ {
+ "Suite": "FWTS",
+ "Reason": "Known issue"
+ }
+ ]
+ }
+ args:
+ - FWTS
+ - "{dir}/fwts.json"
+ - "{dir}/waiver.json"
+ expect_exit_code: 0
+ post_checks:
+ - type: file_contains
+ path: "{dir}/fwts.json"
+ text: "WITH WAIVER"
+ - type: file_not_contains
+ path: "{dir}/fwts.json"
+ text: "(WITH WAIVER) (WITH WAIVER)"
+
+ - name: malformed_waiver_file_does_not_modify_json
+ type: cli
+ text_files:
+ fwts.json: |
+ {
+ "test_results": [
+ {
+ "Test_suite": "DemoSuite",
+ "subtests": [
+ {
+ "sub_Test_Description": "failure one",
+ "sub_Test_Number": "1",
+ "sub_test_result": "FAILED"
+ }
+ ]
+ }
+ ]
+ }
+ waiver.json: |
+ { invalid json
+ args:
+ - FWTS
+ - "{dir}/fwts.json"
+ - "{dir}/waiver.json"
+ expect_exit_code: 0
+ expect_stdout_or_stderr_contains:
+ - "INFO: Failed to read or parse"
+ post_checks:
+ - type: file_not_contains
+ path: "{dir}/fwts.json"
+ text: "WITH WAIVER"
+
+ - name: bsa_testcase_waiver_marks_failed_case
+ type: cli
+ text_files:
+ bsa.json: |
+ {
+ "test_results": [
+ {
+ "Test_suite": "RuleGroupA",
+ "testcases": [
+ {
+ "Test_case": "RULE_001",
+ "Test_result": "FAILED"
+ }
+ ]
+ }
+ ]
+ }
+ waiver.json: |
+ {
+ "Suites": [
+ {
+ "Suite": "BSA",
+ "TestSuites": [
+ {
+ "TestSuite": "RuleGroupA",
+ "TestCases": [
+ {
+ "Test_case": "RULE_001",
+ "Reason": "BSA waiver"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+ args:
+ - BSA
+ - "{dir}/bsa.json"
+ - "{dir}/waiver.json"
+ expect_exit_code: 0
+ post_checks:
+ - type: file_contains
+ path: "{dir}/bsa.json"
+ text: "WITH WAIVER"
+ - type: file_not_contains
+ path: "{dir}/bsa.json"
+ text: "(WITH WAIVER) (WITH WAIVER)"
+
+ - name: quiet_mode_runs_and_updates_json
+ type: cli
+ text_files:
+ fwts.json: |
+ {
+ "test_results": [
+ {
+ "Test_suite": "DemoSuite",
+ "subtests": [
+ {
+ "sub_Test_Description": "failure one",
+ "sub_Test_Number": "1",
+ "sub_test_result": "FAILED"
+ }
+ ]
+ }
+ ]
+ }
+ waiver.json: |
+ {
+ "Suites": [
+ {
+ "Suite": "FWTS",
+ "Reason": "Known FWTS issue"
+ }
+ ]
+ }
+ args:
+ - FWTS
+ - "{dir}/fwts.json"
+ - "{dir}/waiver.json"
+ - --quiet
+ expect_exit_code: 0
+ post_checks:
+ - type: file_contains
+ path: "{dir}/fwts.json"
+ text: "WITH WAIVER"
+
+ - name: no_matching_suite_in_waiver_file_leaves_json_unchanged
+ type: cli
+ text_files:
+ fwts.json: |
+ {
+ "test_results": [
+ {
+ "Test_suite": "DemoSuite",
+ "subtests": [
+ {
+ "sub_Test_Description": "failure one",
+ "sub_Test_Number": "1",
+ "sub_test_result": "FAILED"
+ }
+ ]
+ }
+ ]
+ }
+ waiver.json: |
+ {
+ "Suites": [
+ {
+ "Suite": "BSA",
+ "Reason": "Wrong suite waiver"
+ }
+ ]
+ }
+ args:
+ - FWTS
+ - "{dir}/fwts.json"
+ - "{dir}/waiver.json"
+ expect_exit_code: 0
+ expect_stdout_or_stderr_contains:
+ - "No valid waivers found for suite 'FWTS'"
+ post_checks:
+ - type: file_not_contains
+ path: "{dir}/fwts.json"
+ text: "WITH WAIVER"
+
+ - name: missing_reason_in_waiver_is_ignored
+ type: cli
+ text_files:
+ fwts.json: |
+ {
+ "test_results": [
+ {
+ "Test_suite": "DemoSuite",
+ "subtests": [
+ {
+ "sub_Test_Description": "failure one",
+ "sub_Test_Number": "1",
+ "sub_test_result": "FAILED"
+ }
+ ]
+ }
+ ]
+ }
+ waiver.json: |
+ {
+ "Suites": [
+ {
+ "Suite": "FWTS",
+ "TestSuites": [
+ {
+ "TestSuite": "DemoSuite",
+ "Reason": ""
+ }
+ ]
+ }
+ ]
+ }
+ args:
+ - FWTS
+ - "{dir}/fwts.json"
+ - "{dir}/waiver.json"
+ expect_exit_code: 0
+ expect_stdout_or_stderr_contains:
+ - "missing 'Reason'"
+ post_checks:
+ - type: file_not_contains
+ path: "{dir}/fwts.json"
+ text: "WITH WAIVER"
+
+
+
+ - name: bsa_subtest_waiver_applies_by_sub_rule_id
+ type: cli
+ text_files:
+ bsa.json: |
+ {
+ "test_results": [
+ {
+ "Test_suite": "RuleGroupA",
+ "testcases": [
+ {
+ "Test_case": "RULE_001",
+ "Test_result": "FAILED",
+ "subtests": [
+ {
+ "sub_Rule_ID": "SUB_01",
+ "sub_Test_Description": "first subtest",
+ "sub_test_result": "FAILED"
+ },
+ {
+ "sub_Rule_ID": "SUB_02",
+ "sub_Test_Description": "second subtest",
+ "sub_test_result": "PASSED"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+ waiver.json: |
+ {
+ "Suites": [
+ {
+ "Suite": "BSA",
+ "TestSuites": [
+ {
+ "TestSuite": "RuleGroupA",
+ "TestCases": [
+ {
+ "Test_case": "RULE_001",
+ "Reason": "Case waiver placeholder",
+ "SubTests": [
+ {
+ "sub_Rule_ID": "SUB_01",
+ "Reason": "BSA subtest waiver"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+ args:
+ - BSA
+ - "{dir}/bsa.json"
+ - "{dir}/waiver.json"
+ expect_exit_code: 0
+ post_checks:
+ - type: file_contains
+ path: "{dir}/bsa.json"
+ text: "BSA subtest waiver"
+ - type: file_contains
+ path: "{dir}/bsa.json"
+ text: "WITH WAIVER"
+ - type: file_contains
+ path: "{dir}/bsa.json"
+ text: "\"sub_Rule_ID\": \"SUB_02\""
+
+ - name: passed_result_is_not_modified_by_waiver
+ type: cli
+ text_files:
+ fwts.json: |
+ {
+ "test_results": [
+ {
+ "Test_suite": "DemoSuite",
+ "subtests": [
+ {
+ "sub_Test_Description": "already passed",
+ "sub_Test_Number": "1",
+ "sub_test_result": "PASSED"
+ }
+ ]
+ }
+ ]
+ }
+ waiver.json: |
+ {
+ "Suites": [
+ {
+ "Suite": "FWTS",
+ "Reason": "Should not affect passed results"
+ }
+ ]
+ }
+ args:
+ - FWTS
+ - "{dir}/fwts.json"
+ - "{dir}/waiver.json"
+ expect_exit_code: 0
+ post_checks:
+ - type: file_contains
+ path: "{dir}/fwts.json"
+ text: "\"sub_test_result\": \"PASSED\""
+ - type: file_not_contains
+ path: "{dir}/fwts.json"
+ text: "WITH WAIVER"
diff --git a/common/acs_test_framework_manifests/capsule-ondisk-reporting-vars-check.yaml b/common/acs_test_framework_manifests/capsule-ondisk-reporting-vars-check.yaml
new file mode 100644
index 00000000..d2561974
--- /dev/null
+++ b/common/acs_test_framework_manifests/capsule-ondisk-reporting-vars-check.yaml
@@ -0,0 +1,839 @@
+suites:
+ - name: capsule_ondisk_reporting_vars_check
+ files:
+ - common/linux_scripts/capsule_ondisk_reporting_vars_check.py
+
+ cases:
+ - name: file_exists
+ type: file_exists
+ description: "Verify that the capsule on-disk reporting variables checker script exists."
+
+ - name: python_compiles
+ type: py_compile
+ description: "Ensure that the script compiles successfully as valid Python."
+
+ - name: has_main_guard
+ type: main_guard
+ description: "Check that the script contains a standard __main__ guard."
+
+ - name: status_true_returns_passed
+ type: py_function
+ function: status
+ args: [true]
+ expect_return: "PASSED"
+
+ - name: status_false_returns_warning
+ type: py_function
+ function: status
+ args: [false]
+ expect_return: "WARNING"
+
+ - name: decode_valid_utf16le_returns_capsule_name
+ type: py_function
+ function: decode_char16_11_no_nul
+ args:
+ - !!binary |
+ QwBhAHAAcwB1AGwAZQAwADAAMAAxAA==
+ expect_return: "Capsule0001"
+
+ - name: missing_efivarfs_returns_1_and_logs_warning
+ type: module_main_with_env
+ scenario:
+ kind: capsule_vars
+ efivarfs_present: false
+ efivar_dir: missing_efivars
+ log_file: capsule.log
+ expect_exit_code: 1
+ post_checks:
+ - type: exists
+ path: "{dir}/capsule.log"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "Please ensure efivarfs is enabled and mounted"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "Overall Capsule On-Disk Update Reporting Variables Result: WARNING"
+
+ - name: missing_os_indications_returns_2_and_skips
+ type: module_main_with_env
+ scenario:
+ kind: capsule_vars
+ log_file: capsule.log
+ expect_exit_code: 2
+ post_checks:
+ - type: exists
+ path: "{dir}/capsule.log"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "OsIndicationsSupported not found or unreadable"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "Overall Capsule On-Disk Update Reporting Variables Result: SKIPPED"
+
+ - name: os_indications_without_capsule_support_bit_returns_2
+ type: module_main_with_env
+ scenario:
+ kind: capsule_vars
+ log_file: capsule.log
+ os_indications:
+ value: 0
+ attrs: 0x07
+ expect_exit_code: 2
+ post_checks:
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "INFO: OsIndicationsSupported value: 0x0"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "INFO: Capsule on-disk reporting variables test is not applicable - SKIPPED"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "Overall Capsule On-Disk Update Reporting Variables Result: SKIPPED"
+
+ - name: os_indications_short_data_treated_as_unreadable
+ type: module_main_with_env
+ scenario:
+ kind: capsule_vars
+ log_file: capsule.log
+ os_indications:
+ hex: "070000000400"
+ expect_exit_code: 2
+ post_checks:
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "OsIndicationsSupported not found or unreadable"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "Overall Capsule On-Disk Update Reporting Variables Result: SKIPPED"
+
+ - name: support_claimed_but_missing_capsule_variables_returns_3
+ type: module_main_with_env
+ scenario:
+ kind: capsule_vars
+ log_file: capsule.log
+ os_indications:
+ supported: true
+ attrs: 0x07
+ expect_exit_code: 3
+ post_checks:
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "CapsuleMax Variable - Not Found or Not Accessible"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "CapsuleLast Variable - Not Found or Not Accessible"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "No CapsuleNNNN reporting variables found - WARNING"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "Overall Capsule On-Disk Update Reporting Variables Result: WARNING"
+
+ - name: capsulemax_wrong_attributes_returns_3
+ type: module_main_with_env
+ scenario:
+ kind: capsule_vars
+ log_file: capsule.log
+ os_indications:
+ supported: true
+ attrs: 0x07
+ capsule_max:
+ name: Capsule0001
+ attrs: 0x07
+ capsule_last:
+ name: Capsule0001
+ attrs: 0x07
+ capsule_entries:
+ - name: Capsule0001
+ attrs: 0x07
+ payload_hex: "AA"
+ expect_exit_code: 3
+ post_checks:
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "CapsuleMax Variable Attribute Test"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "actual attributes Value:0x7"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "Status - WARNING"
+
+ - name: capsulelast_wrong_attributes_returns_3
+ type: module_main_with_env
+ scenario:
+ kind: capsule_vars
+ log_file: capsule.log
+ os_indications:
+ supported: true
+ attrs: 0x07
+ capsule_max:
+ name: Capsule0001
+ attrs: 0x06
+ capsule_last:
+ name: Capsule0001
+ attrs: 0x06
+ capsule_entries:
+ - name: Capsule0001
+ attrs: 0x07
+ payload_hex: "AA"
+ expect_exit_code: 3
+ post_checks:
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "CapsuleLast Variable Attribute Test"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "actual attributes Value:0x6"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "Status - WARNING"
+
+ - name: malformed_capsulelast_value_returns_3
+ type: module_main_with_env
+ scenario:
+ kind: capsule_vars
+ log_file: capsule.log
+ os_indications:
+ supported: true
+ attrs: 0x07
+ capsule_max:
+ name: Capsule0001
+ attrs: 0x06
+ capsule_last:
+ hex: "07000000420061006400560061006c0075006500000000000000"
+ capsule_entries:
+ - name: Capsule0001
+ attrs: 0x07
+ payload_hex: "AA"
+ expect_exit_code: 3
+ post_checks:
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "CapsuleLast Variable Test: WARNING"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "Overall Capsule On-Disk Update Reporting Variables Result: WARNING"
+
+ - name: capsulemax_too_short_returns_3
+ type: module_main_with_env
+ scenario:
+ kind: capsule_vars
+ log_file: capsule.log
+ os_indications:
+ supported: true
+ attrs: 0x07
+ capsule_max:
+ hex: "0600000041004200"
+ capsule_last:
+ name: Capsule0001
+ attrs: 0x07
+ capsule_entries:
+ - name: Capsule0001
+ attrs: 0x07
+ payload_hex: "AA"
+ expect_exit_code: 3
+ post_checks:
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "CapsuleMax Variable Test: WARNING"
+
+ - name: missing_capsulelast_returns_3
+ type: module_main_with_env
+ scenario:
+ kind: capsule_vars
+ log_file: capsule.log
+ os_indications:
+ supported: true
+ attrs: 0x07
+ capsule_max:
+ name: Capsule0001
+ attrs: 0x06
+ capsule_entries:
+ - name: Capsule0001
+ attrs: 0x07
+ payload_hex: "AA"
+ expect_exit_code: 3
+ post_checks:
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "CapsuleLast Variable - Not Found or Not Accessible"
+
+ - name: inaccessible_capsule_nnnn_variable_returns_3
+ type: module_main_with_env
+ scenario:
+ kind: capsule_vars
+ log_file: capsule.log
+ os_indications:
+ supported: true
+ attrs: 0x07
+ capsule_max:
+ name: Capsule0001
+ attrs: 0x06
+ capsule_last:
+ name: Capsule0001
+ attrs: 0x07
+ capsule_entries:
+ - name: Capsule0001
+ hex: "01"
+ expect_exit_code: 3
+ post_checks:
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "CapsuleNNNN Variable Test: Capsule0001 Variable Test: WARNING"
+
+ - name: capsule_nnnn_wrong_attributes_returns_3
+ type: module_main_with_env
+ scenario:
+ kind: capsule_vars
+ log_file: capsule.log
+ os_indications:
+ supported: true
+ attrs: 0x07
+ capsule_max:
+ name: Capsule0001
+ attrs: 0x06
+ capsule_last:
+ name: Capsule0001
+ attrs: 0x07
+ capsule_entries:
+ - name: Capsule0001
+ attrs: 0x06
+ payload_hex: "AA"
+ expect_exit_code: 3
+ post_checks:
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "CapsuleNNNN Variable Test: Capsule0001 Variable Test: PASSED"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "actual attributes Value:0x6"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "Status - WARNING"
+
+ - name: multiple_capsule_nnnn_one_bad_one_good_returns_3
+ type: module_main_with_env
+ scenario:
+ kind: capsule_vars
+ log_file: capsule.log
+ os_indications:
+ supported: true
+ attrs: 0x07
+ capsule_max:
+ name: Capsule0002
+ attrs: 0x06
+ capsule_last:
+ name: Capsule0002
+ attrs: 0x07
+ capsule_entries:
+ - name: Capsule0001
+ attrs: 0x07
+ payload_hex: "AA"
+ - name: Capsule0002
+ attrs: 0x06
+ payload_hex: "BB"
+ expect_exit_code: 3
+ post_checks:
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "CapsuleNNNN Variable Test: Capsule0001 Variable Test: PASSED"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "CapsuleNNNN Variable Test: Capsule0002 Variable Test: PASSED"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "actual attributes Value:0x6"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "Overall Capsule On-Disk Update Reporting Variables Result: WARNING"
+
+ - name: all_valid_variables_return_0_and_pass
+ type: module_main_with_env
+ patch_constants:
+ EFIVAR_PATH: "{dir}/efivars"
+ LOG_FILE: "{dir}/capsule.log"
+ dir_structure:
+ - path: efivars
+ bin_files:
+ efivars/OsIndicationsSupported-8be4df61-93ca-11d2-aa0d-00e098032b8c:
+ hex: "070000000400000000000000"
+ efivars/CapsuleMax-39b68c46-f7fb-441b-b6ec-16b0f69821f3:
+ hex: "06000000430061007000730075006c0065003000300030003100"
+ efivars/CapsuleLast-39b68c46-f7fb-441b-b6ec-16b0f69821f3:
+ hex: "07000000430061007000730075006c0065003000300030003100"
+ efivars/Capsule0001-39b68c46-f7fb-441b-b6ec-16b0f69821f3:
+ hex: "07000000AA"
+ expect_exit_code: 0
+ post_checks:
+ - type: ordered_contains
+ path: "{dir}/capsule.log"
+ texts:
+ - "Testing Capsule On-Disk Update Reporting Variables"
+ - "CapsuleMax Variable Test: PASSED"
+ - "CapsuleLast Variable Test: PASSED"
+ - "CapsuleNNNN Variable Test: Capsule0001 Variable Test: PASSED"
+ - "Overall Capsule On-Disk Update Reporting Variables Result: PASSED"
+
+ - name: cli_mock_open_raises_oserror_for_existing_capsulelast
+ type: module_main_with_env
+ scenario:
+ kind: capsule_vars
+ log_file: capsule.log
+ os_indications:
+ supported: true
+ attrs: 0x07
+ capsule_max:
+ name: Capsule0001
+ attrs: 0x06
+ capsule_last:
+ name: Capsule0001
+ attrs: 0x07
+ capsule_entries:
+ - name: Capsule0001
+ attrs: 0x07
+ payload_hex: "AA"
+ mocks:
+ builtins.open:
+ factory: mock_helpers.passthrough_router
+ inject_original_as: real
+ kwargs:
+ rules:
+ - label: capsulelast-read-failure
+ required: true
+ when:
+ args:
+ 0:
+ contains: "CapsuleLast-39b68c46-f7fb-441b-b6ec-16b0f69821f3"
+ 1:
+ contains: "rb"
+ raise:
+ type: py:builtins.OSError
+ args: ["mocked read failure"]
+ expect_exit_code: 3
+ post_checks:
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "CapsuleLast Variable - Not Found or Not Accessible"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "Overall Capsule On-Disk Update Reporting Variables Result: WARNING"
+
+ - name: cli_mock_listdir_failure_for_existing_efivarfs
+ type: module_main_with_env
+ scenario:
+ kind: capsule_vars
+ log_file: capsule.log
+ os_indications:
+ supported: true
+ attrs: 0x07
+ capsule_max:
+ name: Capsule0001
+ attrs: 0x06
+ capsule_last:
+ name: Capsule0001
+ attrs: 0x07
+ mocks:
+ "{module}.os.listdir":
+ factory: mock_helpers.passthrough_router
+ inject_original_as: real
+ kwargs:
+ rules:
+ - label: efivarfs-listdir-failure
+ required: true
+ when:
+ args:
+ 0:
+ contains: "efivars"
+ raise:
+ type: py:builtins.OSError
+ args: ["mocked listdir failure"]
+ expect_exit_code: 3
+ post_checks:
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "efivarfs not accessible - WARNING"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "Overall Capsule On-Disk Update Reporting Variables Result: WARNING"
+
+ - name: cli_mock_read_efi_var_os_indications_unreadable_even_when_file_exists
+ type: module_main_with_env
+ scenario:
+ kind: capsule_vars
+ log_file: capsule.log
+ os_indications:
+ supported: true
+ attrs: 0x07
+ mocks:
+ "{module}.read_efi_var":
+ factory: mock_helpers.passthrough_router
+ inject_original_as: real
+ kwargs:
+ rules:
+ - label: os-indications-read-returns-none
+ required: true
+ when:
+ args:
+ 0:
+ equals: "OsIndicationsSupported"
+ return: [null, null]
+ expect_exit_code: 2
+ post_checks:
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "OsIndicationsSupported not found or unreadable"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "Overall Capsule On-Disk Update Reporting Variables Result: SKIPPED"
+
+ - name: cli_mock_read_efi_var_capsulemax_missing_even_when_file_exists
+ type: module_main_with_env
+ scenario:
+ kind: capsule_vars
+ log_file: capsule.log
+ os_indications:
+ supported: true
+ attrs: 0x07
+ capsule_max:
+ name: Capsule0001
+ attrs: 0x06
+ capsule_last:
+ name: Capsule0001
+ attrs: 0x07
+ capsule_entries:
+ - name: Capsule0001
+ attrs: 0x07
+ payload_hex: "AA"
+ mocks:
+ "{module}.read_efi_var":
+ factory: mock_helpers.passthrough_router
+ inject_original_as: real
+ kwargs:
+ rules:
+ - label: capsulemax-read-returns-none
+ required: true
+ when:
+ args:
+ 0:
+ equals: "CapsuleMax"
+ return: [null, null]
+ expect_exit_code: 3
+ post_checks:
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "CapsuleMax Variable - Not Found or Not Accessible"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "Overall Capsule On-Disk Update Reporting Variables Result: WARNING"
+
+ - name: ondisk_supported_without_capsule_entries_currently_passes_overall
+ type: module_main_with_env
+ warn_only: true
+ scenario:
+ kind: capsule_vars
+ log_file: capsule.log
+ os_indications:
+ supported: true
+ attrs: 0x06
+ capsule_max:
+ name: Capsule0001
+ attrs: 0x06
+ capsule_last:
+ name: Capsule0001
+ attrs: 0x07
+ expect_exit_code: 0
+ post_checks:
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "No CapsuleNNNN reporting variables found - WARNING"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "Overall Capsule On-Disk Update Reporting Variables Result: PASSED"
+
+ - name: capsulemax_exact_char16_11_length_should_be_enforced
+ type: module_main_with_env
+ warn_only: true
+ scenario:
+ kind: capsule_vars
+ log_file: capsule.log
+ os_indications:
+ supported: true
+ attrs: 0x06
+ capsule_max:
+ name: Capsule0001
+ attrs: 0x06
+ extra_utf16: EXTRA
+ capsule_last:
+ name: Capsule0001
+ attrs: 0x07
+ capsule_entries:
+ - name: Capsule0001
+ attrs: 0x07
+ payload_hex: "AA"
+ expect_exit_code: 0
+ post_checks:
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "CapsuleMax - Found, Value - Capsule0001"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "Overall Capsule On-Disk Update Reporting Variables Result: PASSED"
+
+ - name: capsule_nnnn_payload_structure_should_not_be_ignored
+ type: module_main_with_env
+ warn_only: true
+ scenario:
+ kind: capsule_vars
+ log_file: capsule.log
+ os_indications:
+ supported: true
+ attrs: 0x06
+ capsule_max:
+ name: Capsule0001
+ attrs: 0x06
+ capsule_last:
+ name: Capsule0001
+ attrs: 0x07
+ capsule_entries:
+ - name: Capsule0001
+ hex: "07000000"
+ expect_exit_code: 0
+ post_checks:
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "CapsuleNNNN Variable Test: Capsule0001 - Found"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "CapsuleNNNN Variable Test: Capsule0001 Variable Test: PASSED"
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "Overall Capsule On-Disk Update Reporting Variables Result: PASSED"
+
+ - name: missing_os_indications_is_reported_as_not_found
+ type: module_main_with_env
+ warn_only: true
+ scenario:
+ kind: capsule_vars
+ log_file: capsule.log
+ expect_exit_code: 2
+ post_checks:
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "OsIndicationsSupported not found or unreadable"
+
+ - name: logging_failure_silently_skips_log_creation
+ type: module_main_with_env
+ warn_only: true
+ scenario:
+ kind: capsule_vars
+ efivarfs_present: false
+ log_file: capsule.log
+ mocks:
+ builtins.open:
+ factory: mock_helpers.passthrough_router
+ inject_original_as: real
+ kwargs:
+ rules:
+ - label: log-write-failure
+ required: true
+ when:
+ args:
+ 0:
+ contains: "capsule.log"
+ 1:
+ contains: "a"
+ raise:
+ type: py:builtins.OSError
+ args: ["mocked log write failure"]
+ expect_exit_code: 1
+ post_checks:
+ - type: not_exists
+ path: "{dir}/capsule.log"
+
+ - name: unreadable_os_indications_is_reported_as_unreadable
+ type: module_main_with_env
+ warn_only: true
+ scenario:
+ kind: capsule_vars
+ log_file: capsule.log
+ os_indications:
+ supported: true
+ attrs: 0x07
+ mocks:
+ builtins.open:
+ factory: mock_helpers.passthrough_router
+ inject_original_as: real
+ kwargs:
+ rules:
+ - label: os-indications-open-failure
+ required: true
+ when:
+ args:
+ 0:
+ contains: "OsIndicationsSupported-8be4df61-93ca-11d2-aa0d-00e098032b8c"
+ 1:
+ contains: "rb"
+ raise:
+ type: py:builtins.OSError
+ args: ["mocked read failure"]
+ expect_exit_code: 2
+ post_checks:
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "OsIndicationsSupported not found or unreadable"
+
+ - name: capsulemax_with_additional_attribute_bits_is_rejected
+ type: module_main_with_env
+ warn_only: true
+ scenario:
+ kind: capsule_vars
+ log_file: log.txt
+ os_indications:
+ supported: true
+ attrs: 0x07
+ capsule_max:
+ name: Capsule0001
+ attrs: 0x0E
+ capsule_last:
+ name: Capsule0001
+ attrs: 0x07
+ capsule_entries:
+ - name: Capsule0001
+ attrs: 0x07
+ payload_hex: "AA"
+ expect_exit_code: 3
+ description: "Document the current behavior that requires an exact CapsuleMax attribute match and rejects additional bits."
+ post_checks:
+ - type: file_contains
+ path: "{dir}/log.txt"
+ text: "Status - WARNING"
+
+ - name: permission_denied_during_capsule_enumeration_is_identified
+ type: module_main_with_env
+ warn_only: true
+ scenario:
+ kind: capsule_vars
+ log_file: capsule.log
+ os_indications:
+ supported: true
+ attrs: 0x07
+ capsule_max:
+ name: Capsule0001
+ attrs: 0x06
+ capsule_last:
+ name: Capsule0001
+ attrs: 0x07
+ mocks:
+ os.listdir:
+ factory: mock_helpers.passthrough_router
+ inject_original_as: real
+ kwargs:
+ rules:
+ - label: permission-denied-listdir
+ required: true
+ when:
+ args:
+ 0:
+ contains: "efivars"
+ raise:
+ type: py:builtins.PermissionError
+ args: ["denied"]
+ expect_exit_code: 3
+ post_checks:
+ - type: file_contains
+ path: "{dir}/capsule.log"
+ text: "efivarfs not accessible - WARNING"
+
+ - name: malformed_capsule_entry_content_is_rejected
+ type: module_main_with_env
+ warn_only: true
+ scenario:
+ kind: capsule_vars
+ log_file: log.txt
+ os_indications:
+ supported: true
+ attrs: 0x07
+ capsule_entries:
+ - name: Capsule0001
+ hex: "FFFFFFFFFFFFFFFF"
+ expect_exit_code: 3
+
+ - name: oversized_capsulemax_value_is_currently_accepted
+ type: module_main_with_env
+ warn_only: true
+ patch_constants:
+ EFIVAR_PATH: "{dir}/efivars"
+ LOG_FILE: "{dir}/log.txt"
+ dir_structure:
+ - path: efivars
+ bin_files:
+ efivars/OsIndicationsSupported-8be4df61-93ca-11d2-aa0d-00e098032b8c:
+ hex: "070000000400000000000000"
+ efivars/CapsuleMax-39b68c46-f7fb-441b-b6ec-16b0f69821f3:
+ hex: "06000000430061007000730075006c006500300030003000310058005800"
+ efivars/CapsuleLast-39b68c46-f7fb-441b-b6ec-16b0f69821f3:
+ hex: "07000000430061007000730075006c0065003000300030003100"
+ efivars/Capsule0001-39b68c46-f7fb-441b-b6ec-16b0f69821f3:
+ hex: "07000000AA"
+ expect_exit_code: 0
+ post_checks:
+ - type: file_contains
+ path: "{dir}/log.txt"
+ text: "CapsuleMax - Found, Value - Capsule0001"
+ - type: file_contains
+ path: "{dir}/log.txt"
+ text: "Overall Capsule On-Disk Update Reporting Variables Result: PASSED"
+
+ - name: invalid_utf16_in_capsulelast_is_reported
+ type: module_main_with_env
+ warn_only: true
+ scenario:
+ kind: capsule_vars
+ log_file: log.txt
+ os_indications:
+ supported: true
+ attrs: 0x07
+ capsule_max:
+ name: Capsule0001
+ attrs: 0x06
+ capsule_last:
+ hex: "07000000FFFF"
+ capsule_entries:
+ - name: Capsule0001
+ attrs: 0x07
+ payload_hex: "AA"
+ expect_exit_code: 3
+ post_checks:
+ - type: file_contains
+ path: "{dir}/log.txt"
+ text: "CapsuleLast Variable Test: WARNING"
+ - type: file_contains
+ path: "{dir}/log.txt"
+ text: "Overall Capsule On-Disk Update Reporting Variables Result: WARNING"
+
+ - name: capsulelast_is_not_cross_checked_against_discovered_entry
+ type: module_main_with_env
+ warn_only: true
+ patch_constants:
+ EFIVAR_PATH: "{dir}/efivars"
+ LOG_FILE: "{dir}/log.txt"
+ dir_structure:
+ - path: efivars
+ bin_files:
+ efivars/OsIndicationsSupported-8be4df61-93ca-11d2-aa0d-00e098032b8c:
+ hex: "070000000400000000000000"
+ efivars/CapsuleMax-39b68c46-f7fb-441b-b6ec-16b0f69821f3:
+ hex: "06000000430061007000730075006c0065003000300030003100"
+ efivars/CapsuleLast-39b68c46-f7fb-441b-b6ec-16b0f69821f3:
+ hex: "07000000430061007000730075006c0065003000300030003900"
+ efivars/Capsule0001-39b68c46-f7fb-441b-b6ec-16b0f69821f3:
+ hex: "07000000AA"
+ expect_exit_code: 0
+ post_checks:
+ - type: file_contains
+ path: "{dir}/log.txt"
+ text: "CapsuleLast Variable Test: PASSED"
+ - type: file_contains
+ path: "{dir}/log.txt"
+ text: "Overall Capsule On-Disk Update Reporting Variables Result: PASSED"
diff --git a/common/acs_test_framework_manifests/ethtool-test.yaml b/common/acs_test_framework_manifests/ethtool-test.yaml
new file mode 100644
index 00000000..16958e96
--- /dev/null
+++ b/common/acs_test_framework_manifests/ethtool-test.yaml
@@ -0,0 +1,345 @@
+suites:
+ - name: ethtool_test
+ files:
+ - common/linux_scripts/ethtool-test.py
+
+ cases:
+ - name: file_exists
+ type: file_exists
+ description: Verify the ethtool test script is present
+
+ - name: python_compiles
+ type: py_compile
+ description: Verify the script has no Python syntax errors
+
+ - name: has_main_guard
+ type: main_guard
+ description: Verify the script can run as a standalone CLI entry point
+
+ - name: expected_test_order_keywords_exist
+ type: source_contains_all
+ description: Verify the source includes all expected Ethernet validation stages
+ patterns:
+ - "Detect interface"
+ - "Bring up"
+ - "ethtool present"
+ - "Self-test supported"
+ - "ethtool self tests"
+ - "Link detected"
+ - "IPv4 address present"
+ - "Gateway Address present"
+ - "Ping gateway (IPv4)"
+ - "Ping www.arm.com (IPv4)"
+ - "wget and curl"
+ - "IPv6 address present"
+ - "Ping ipv6.google.com (IPv6)"
+
+ - name: cli_no_ethernet_interfaces_exits_nonzero
+ type: module_cli
+ description: Verify the script exits with failure when no usable Ethernet interface is detected
+ scenario:
+ kind: ethtool
+ interfaces: []
+ tools:
+ ethtool: absent
+ ping: absent
+ wget: absent
+ curl: absent
+ dhclient: absent
+ udhcpc: absent
+ expect_exit_nonzero: true
+ expect_stdout_or_stderr_contains:
+ - "No ethernet interfaces detected via ip linux command, Exiting"
+
+ - name: cli_virtual_interface_only
+ type: module_cli
+ description: Verify virtual interfaces alone are not treated as compliant physical NICs
+ scenario:
+ kind: ethtool
+ required_compliant_interfaces: 0
+ interfaces:
+ - name: docker0
+ kind: virtual
+ state: down
+ mtu: 1500
+ device: /sys/devices/virtual/net/docker0
+ tools:
+ ethtool: /usr/sbin/ethtool
+ ping: absent
+ wget: absent
+ curl: absent
+ dhclient: absent
+ udhcpc: absent
+ text_files:
+ system_config.txt: |
+ total_number_of_network_controllers: 0
+ args:
+ - "{dir}/system_config.txt"
+ expect_stdout_or_stderr_contains:
+ - "docker0"
+
+ - name: cli_sysfs_link_only
+ type: module_cli
+ description: Verify sysfs carrier and operstate can identify a healthy physical NIC without ethtool
+ scenario:
+ kind: ethtool
+ required_compliant_interfaces: 1
+ interfaces:
+ - name: eth0
+ kind: physical
+ state: up
+ carrier: 1
+ operstate: up
+ device: /sys/devices/pci0000:00/0000:00:01.0/net/eth0
+ tools:
+ ethtool: absent
+ ping: absent
+ wget: absent
+ curl: absent
+ dhclient: absent
+ udhcpc: absent
+ text_files:
+ system_config.txt: |
+ total_number_of_network_controllers: 1
+ args:
+ - "{dir}/system_config.txt"
+ expect_stdout_or_stderr_contains:
+ - "eth0"
+
+ - name: cli_full_success
+ type: module_cli
+ description: Verify the end-to-end happy path for one healthy physical interface
+ scenario:
+ kind: ethtool
+ required_compliant_interfaces: 1
+ interfaces:
+ - name: eth1
+ kind: physical
+ state: up
+ carrier: 1
+ operstate: up
+ device: /sys/devices/pci0000:00/0000:00:02.0/net/eth1
+ ipv4:
+ - address: 192.0.2.10/24
+ dynamic: true
+ ipv6:
+ - address: 2001:db8::10/64
+ routes:
+ default: default via 192.0.2.1 dev eth1 proto dhcp
+ route_get: 8.8.8.8 via 192.0.2.1 dev eth1 src 192.0.2.10
+ ethtool:
+ link_detected: true
+ self_test_supported: true
+ self_test:
+ returncode: 1
+ stdout: "self-test failed\n"
+ stderr: ""
+ connectivity:
+ gateway_ping: pass
+ arm_ping: pass
+ ipv6_ping: pass
+ curl: pass
+ wget: pass
+ tools:
+ ethtool: /usr/bin/ethtool
+ ping: /usr/bin/ping
+ wget: /usr/bin/wget
+ curl: /usr/bin/curl
+ dhclient: absent
+ udhcpc: absent
+ text_files:
+ system_config.txt: |
+ total_number_of_network_controllers: 1
+ args:
+ - "{dir}/system_config.txt"
+ expect_stdout_or_stderr_contains:
+ - "eth1"
+
+ - name: cli_required_compliant_ifaces_fail_when_not_enough_pass
+ type: module_cli
+ description: Verify compliance fails when detected interfaces exist but too few actually pass validation
+ scenario:
+ kind: ethtool
+ required_compliant_interfaces: 2
+ interfaces:
+ - name: eth0
+ kind: physical
+ state: up
+ carrier: 1
+ operstate: up
+ device: /sys/devices/pci0000:00/0000:00:03.0/net/eth0
+ ipv4:
+ - address: 192.0.2.20/24
+ dynamic: true
+ routes:
+ default: default via 192.0.2.1 dev eth0 proto dhcp
+ route_get: 8.8.8.8 via 192.0.2.1 dev eth0 src 192.0.2.20
+ ethtool:
+ link_detected: true
+ self_test_supported: false
+ connectivity:
+ gateway_ping: pass
+ arm_ping: pass
+ curl: pass
+ wget: pass
+ - name: eth1
+ kind: physical
+ state: down
+ carrier: 0
+ operstate: down
+ device: /sys/devices/pci0000:00/0000:00:04.0/net/eth1
+ ethtool:
+ link_detected: false
+ self_test_supported: false
+ tools:
+ ethtool: /usr/bin/ethtool
+ ping: /usr/bin/ping
+ wget: /usr/bin/wget
+ curl: /usr/bin/curl
+ dhclient: absent
+ udhcpc: absent
+ text_files:
+ system_config.txt: |
+ total_number_of_network_controllers: 2
+ args:
+ - "{dir}/system_config.txt"
+ expect_stdout_or_stderr_contains:
+ - "eth0"
+ - "eth1"
+
+ - name: external_connectivity_failures_do_not_override_local_interface_health
+ type: module_cli
+ description: Verify external internet failures do not automatically mark a locally healthy NIC as unhealthy
+ warn_only: true
+ scenario:
+ kind: ethtool
+ required_compliant_interfaces: 1
+ interfaces:
+ - name: eth0
+ kind: physical
+ state: up
+ carrier: 1
+ operstate: up
+ device: /sys/devices/pci0000:00/0000:00:01.0/net/eth0
+ ipv4:
+ - address: 192.0.2.10/24
+ dynamic: true
+ ipv6:
+ - address: 2001:db8::10/64
+ routes:
+ default: default via 192.0.2.1 dev eth0 proto dhcp
+ route_get: 8.8.8.8 via 192.0.2.1 dev eth0 src 192.0.2.10
+ ethtool:
+ link_detected: true
+ self_test_supported: false
+ connectivity:
+ gateway_ping: pass
+ arm_ping: fail
+ ipv6_ping: fail
+ curl: fail
+ wget: fail
+ tools:
+ ethtool: /usr/bin/ethtool
+ ping: /usr/bin/ping
+ wget: /usr/bin/wget
+ curl: /usr/bin/curl
+ dhclient: absent
+ udhcpc: absent
+ text_files:
+ system_config.txt: |
+ total_number_of_network_controllers: 1
+ args:
+ - "{dir}/system_config.txt"
+ expect_stdout_or_stderr_contains:
+ - "Ethtool Compliance :"
+ - "PASSED"
+
+ - name: route_validation_does_not_depend_on_public_ip_probe
+ type: module_cli
+ description: Verify default-route validation still works even when public route probing fails
+ warn_only: true
+ scenario:
+ kind: ethtool
+ required_compliant_interfaces: 1
+ interfaces:
+ - name: eth0
+ kind: physical
+ state: up
+ carrier: 1
+ operstate: up
+ device: /sys/devices/pci0000:00/0000:00:01.0/net/eth0
+ ipv4:
+ - address: 10.0.0.10/24
+ dynamic: true
+ routes:
+ default: default via 10.0.0.1 dev eth0 proto dhcp
+ route_get:
+ returncode: 1
+ stdout: ""
+ stderr: "network unreachable\n"
+ ethtool:
+ link_detected: true
+ self_test_supported: false
+ tools:
+ ethtool: /usr/bin/ethtool
+ ping: /usr/bin/ping
+ wget: absent
+ curl: absent
+ dhclient: absent
+ udhcpc: absent
+ text_files:
+ system_config.txt: |
+ total_number_of_network_controllers: 1
+ args:
+ - "{dir}/system_config.txt"
+ expect_stdout_or_stderr_contains:
+ - "Ping gateway (IPv4)"
+ - "PASSED"
+
+ - name: compliance_requires_more_than_link_when_external_checks_degrade
+ type: module_cli
+ description: Verify overall compliance can fail even when link is up if higher-level connectivity checks degrade
+ warn_only: true
+ scenario:
+ kind: ethtool
+ required_compliant_interfaces: 1
+ interfaces:
+ - name: eth0
+ kind: physical
+ state: up
+ carrier: 1
+ operstate: up
+ device: /sys/devices/pci0000:00/0000:00:01.0/net/eth0
+ ipv4:
+ - address: 192.0.2.10/24
+ dynamic: true
+ ipv6:
+ - address: 2001:db8::10/64
+ routes:
+ default: default via 192.0.2.1 dev eth0 proto dhcp
+ route_get: 8.8.8.8 via 192.0.2.1 dev eth0 src 192.0.2.10
+ ethtool:
+ link_detected: true
+ self_test_supported: false
+ connectivity:
+ gateway_ping: pass
+ arm_ping: fail
+ ipv6_ping: fail
+ curl: fail
+ wget: fail
+ tools:
+ ethtool: /usr/bin/ethtool
+ ping: /usr/bin/ping
+ wget: /usr/bin/wget
+ curl: /usr/bin/curl
+ dhclient: absent
+ udhcpc: absent
+ text_files:
+ system_config.txt: |
+ total_number_of_network_controllers: 1
+ args:
+ - "{dir}/system_config.txt"
+ expect_stdout_or_stderr_contains:
+ - "Ethtool Compliance :"
+ - "FAILED"
+
diff --git a/common/acs_test_framework_manifests/extract-capsule-fw-version.yaml b/common/acs_test_framework_manifests/extract-capsule-fw-version.yaml
new file mode 100644
index 00000000..ef16827c
--- /dev/null
+++ b/common/acs_test_framework_manifests/extract-capsule-fw-version.yaml
@@ -0,0 +1,153 @@
+suites:
+ - name: extract_capsule_fw_version
+ files:
+ - common/linux_scripts/extract_capsule_fw_version.py
+
+ cases:
+ - name: file_exists
+ type: file_exists
+ description: "Verify that the firmware version extraction script exists at the expected repository path."
+
+ - name: python_compiles
+ type: py_compile
+ description: "Ensure the script is syntactically valid Python before executing runtime tests."
+
+ - name: cli_no_args_fails
+ type: cli
+ description: "Validate that the script enforces required CLI arguments and fails when none are provided."
+ args: []
+ expect_exit_code: 1
+ expect_stdout_or_stderr_contains:
+ - "Usage: python3 extract_capsule_fw_version.py"
+
+ - name: cli_one_arg_fails
+ type: cli
+ description: "Ensure the script rejects execution when only the regex pattern is provided without the input file."
+ args:
+ - "Capsule=([0-9A-F]+)"
+ expect_exit_code: 1
+ expect_stdout_or_stderr_contains:
+ - "Usage: python3 extract_capsule_fw_version.py"
+
+ - name: cli_missing_file_is_nonzero
+ type: cli
+ description: "Verify that the script exits with failure when the input file does not exist."
+ args:
+ - "Capsule=([0-9A-F]{4})"
+ - "{dir}/missing.log"
+ expect_exit_nonzero: true
+
+ - name: cli_extracts_two_matching_values_in_order
+ type: cli
+ description: "Validate that the script correctly extracts multiple matching firmware version values in the order they appear in the input file."
+ args:
+ - "Capsule=([0-9A-F]{4})"
+ - "{dir}/input.log"
+ text_files:
+ input.log: |
+ Start
+ Capsule=1A2B
+ Noise
+ Capsule=00FF
+ End
+ expect_exit_code: 0
+ expect_stdout_or_stderr_regex:
+ - "(?s)1A2B.*00FF"
+
+ - name: cli_no_matches_returns_zero_cleanly
+ type: cli
+ description: "Ensure the script handles input with no matching patterns gracefully and still exits successfully."
+ args:
+ - "Capsule=([0-9A-F]{4})"
+ - "{dir}/input.log"
+ text_files:
+ input.log: |
+ Alpha
+ Beta
+ Gamma
+ expect_exit_code: 0
+
+ - name: cli_pattern_without_group_returns_zero_cleanly
+ type: cli
+ description: "Verify behavior when regex pattern has no capture group; script should still run without failure."
+ args:
+ - "Capsule=[0-9A-F]{4}"
+ - "{dir}/input.log"
+ text_files:
+ input.log: |
+ Capsule=ABCD
+ Capsule=EF01
+ expect_exit_code: 0
+
+ - name: cli_only_prints_group_1_not_full_match
+ type: cli
+ description: "Ensure only the captured group is printed and not the full regex match string."
+ args:
+ - "Capsule=([0-9A-F]{4})"
+ - "{dir}/input.log"
+ text_files:
+ input.log: |
+ build Capsule=DEAD done
+ expect_exit_code: 0
+ expect_stdout_or_stderr_regex:
+ - "(?m)^DEAD$"
+
+ - name: cli_multiple_matches_same_line_prints_first_search_per_line
+ type: cli
+ description: "Validate that only the first match per line is captured when multiple matches exist in a single line."
+ args:
+ - "Capsule=([0-9A-F]{4})"
+ - "{dir}/input.log"
+ text_files:
+ input.log: |
+ Capsule=AAAA Capsule=BBBB
+ expect_exit_code: 0
+ expect_stdout_or_stderr_regex:
+ - "(?m)^AAAA$"
+
+ - name: cli_mixed_case_hex_matches
+ type: cli
+ description: "Ensure the regex can match both uppercase and lowercase hexadecimal values."
+ args:
+ - "Capsule=([0-9A-Fa-f]{4})"
+ - "{dir}/input.log"
+ text_files:
+ input.log: |
+ Capsule=AbCd
+ expect_exit_code: 0
+ expect_stdout_or_stderr_contains:
+ - "AbCd"
+
+ - name: cli_two_capture_groups_prints_only_first_group
+ type: cli
+ description: "Ensure that when multiple capture groups are present, only the first group is printed."
+ args:
+ - "Capsule=([0-9A-F]{4}) status=(OK)"
+ - "{dir}/input.log"
+ text_files:
+ input.log: |
+ Capsule=ABCD status=OK
+ expect_exit_code: 0
+ expect_stdout_or_stderr_regex:
+ - "(?m)^ABCD$"
+
+ - name: cli_invalid_regex_fails
+ type: cli
+ description: "Validate that invalid regex patterns are detected and cause script failure."
+ args:
+ - "Capsule=([0-9A-F]{4}"
+ - "{dir}/input.log"
+ text_files:
+ input.log: |
+ Capsule=ABCD
+ expect_exit_nonzero: true
+
+ - name: cli_empty_file_returns_zero
+ type: cli
+ description: "Ensure the script handles empty input files gracefully without crashing."
+ args:
+ - "Capsule=([0-9A-F]{4})"
+ - "{dir}/empty.log"
+ text_files:
+ empty.log: ""
+ expect_exit_code: 0
\ No newline at end of file
diff --git a/common/acs_test_framework_manifests/generate_acs_summary.yaml b/common/acs_test_framework_manifests/generate_acs_summary.yaml
new file mode 100644
index 00000000..57ec7204
--- /dev/null
+++ b/common/acs_test_framework_manifests/generate_acs_summary.yaml
@@ -0,0 +1,399 @@
+suites:
+ - name: generate_acs_summary
+ files:
+ - common/log_parser/generate_acs_summary.py
+
+ cases:
+ - name: file_exists
+ type: file_exists
+
+ - name: python_compiles
+ type: py_compile
+
+ - name: has_main_guard
+ type: main_guard
+
+ - name: cli_requires_many_inputs
+ type: cli
+ args: []
+ expect_exit_nonzero: true
+
+ - name: cli_generates_html_with_summary_sections
+ type: cli
+ text_files:
+ bsa.html: |
+
+
BSA Test SummaryBSA Test Summary
Result Summary
BSA content
+ sbsa.html: |
+
+ SBSA Test SummarySBSA Test Summary
Result Summary
SBSA content
+ fwts.html: |
+
+ FWTS Test SummaryFWTS Test Summary
Result Summary
FWTS content
+ sct.html: |
+
+ SCT Test SummarySCT Test Summary
Result Summary
SCT content
+ bbsr_fwts.html: |
+
+ FWTS Test SummaryFWTS Test Summary
Result Summary
BBSR FWTS content
+ bbsr_sct.html: |
+
+ SCT Test SummarySCT Test Summary
Result Summary
BBSR SCT content
+ bbsr_tpm.html: |
+
+ TPM Test SummaryTPM Test Summary
Result Summary
BBSR TPM content
+ pfdi.html: |
+
+ PFDI Test SummaryPFDI Test Summary
Result Summary
PFDI content
+ post_script.html: |
+
+ Post-Script Test SummaryPost-Script Test Summary
Result Summary
POST content
+ standalone.html: |
+
+ Standalone Test SummaryStandalone Test Summary
Result Summary
Standalone content
+ os_tests.html: |
+
+ OS Tests Test SummaryOS Tests Test Summary
Result Summary
OS Tests content
+ capsule.html: |
+
+ Capsule Update Test SummaryCapsule Update Test Summary
Result Summary
Capsule content
+ sbmr_ib.html: |
+
+ SBMR-IB Test SummarySBMR-IB Test Summary
Result Summary
SBMR IB content
+ sbmr_oob.html: |
+
+ SBMR-OOB Test SummarySBMR-OOB Test Summary
Result Summary
SBMR OOB content
+ scmi.html: |
+
+ SCMI Test SummarySCMI Test Summary
Result Summary
SCMI content
+ acs_info.json: |
+ {
+ "System Info": {
+ "Vendor": "VendorZ",
+ "System Name": "SystemZ",
+ "SoC Family": "FamilyZ",
+ "Firmware Version": "FW-Z",
+ "BMC Firmware Version": "BMC-1"
+ }
+ }
+ merged.json: |
+ {
+ "ACS Results Summary": {
+ "Overall Compliance Result": "Compliant",
+ "BBSR compliance results": "Compliant",
+ "SCMI compliance results": "Compliant"
+ }
+ }
+ args:
+ - --merged_json
+ - "{dir}/merged.json"
+ - "{dir}/bsa.html"
+ - "{dir}/sbsa.html"
+ - "{dir}/fwts.html"
+ - "{dir}/sct.html"
+ - "{dir}/bbsr_fwts.html"
+ - "{dir}/bbsr_sct.html"
+ - "{dir}/bbsr_tpm.html"
+ - "{dir}/pfdi.html"
+ - "{dir}/post_script.html"
+ - "{dir}/standalone.html"
+ - "{dir}/os_tests.html"
+ - "{dir}/capsule.html"
+ - "{dir}/sbmr_ib.html"
+ - "{dir}/sbmr_oob.html"
+ - "{dir}/scmi.html"
+ - "{dir}/summary.html"
+ - --acs_info_json
+ - "{dir}/acs_info.json"
+ expect_exit_code: 0
+ post_checks:
+ - type: exists
+ path: "{dir}/summary.html"
+ - type: file_not_empty
+ path: "{dir}/summary.html"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: "BSA content"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: "FWTS content"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: "SCMI content"
+ - type: file_not_contains
+ path: "{dir}/summary.html"
+ text: "Result Summary"
+
+ - name: cli_without_merged_json_still_generates_html
+ type: cli
+ text_files:
+ bsa.html: |
+
+ BSA Test SummaryBSA Test Summary
Result Summary
bsa
+ sbsa.html: |
+
+ SBSA Test SummarySBSA Test Summary
Result Summary
sbsa
+ fwts.html: |
+
+ FWTS Test SummaryFWTS Test Summary
Result Summary
fwts
+ sct.html: |
+
+ SCT Test SummarySCT Test Summary
Result Summary
sct
+ bbsr_fwts.html: |
+
+ FWTS Test SummaryFWTS Test Summary
Result Summary
bbsr_fwts
+ bbsr_sct.html: |
+
+ SCT Test SummarySCT Test Summary
Result Summary
bbsr_sct
+ bbsr_tpm.html: |
+
+ TPM Test SummaryTPM Test Summary
Result Summary
bbsr_tpm
+ pfdi.html: |
+
+ PFDI Test SummaryPFDI Test Summary
Result Summary
pfdi
+ post_script.html: |
+
+ Post-Script Test SummaryPost-Script Test Summary
Result Summary
post_script
+ standalone.html: |
+
+ Standalone Test SummaryStandalone Test Summary
Result Summary
standalone
+ os_tests.html: |
+
+ OS Tests Test SummaryOS Tests Test Summary
Result Summary
os_tests
+ capsule.html: |
+
+ Capsule Update Test SummaryCapsule Update Test Summary
Result Summary
capsule
+ sbmr_ib.html: |
+
+ SBMR-IB Test SummarySBMR-IB Test Summary
Result Summary
sbmr_ib
+ sbmr_oob.html: |
+
+ SBMR-OOB Test SummarySBMR-OOB Test Summary
Result Summary
sbmr_oob
+ scmi.html: |
+
+ SCMI Test SummarySCMI Test Summary
Result Summary
scmi
+ args:
+ - "{dir}/bsa.html"
+ - "{dir}/sbsa.html"
+ - "{dir}/fwts.html"
+ - "{dir}/sct.html"
+ - "{dir}/bbsr_fwts.html"
+ - "{dir}/bbsr_sct.html"
+ - "{dir}/bbsr_tpm.html"
+ - "{dir}/pfdi.html"
+ - "{dir}/post_script.html"
+ - "{dir}/standalone.html"
+ - "{dir}/os_tests.html"
+ - "{dir}/capsule.html"
+ - "{dir}/sbmr_ib.html"
+ - "{dir}/sbmr_oob.html"
+ - "{dir}/scmi.html"
+ - "{dir}/summary.html"
+ expect_exit_code: 0
+ post_checks:
+ - type: exists
+ path: "{dir}/summary.html"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: "standalone"
+
+
+ - name: cli_removes_result_summary_headings_case_insensitively
+ type: cli
+ text_files:
+ bsa.html: |
+
+ BSA Test SummaryBSA Test Summary
RESULT SUMMARY
BSA
+ sbsa.html: |
+
+ SBSA Test SummarySBSA Test Summary
Result Summary
sbsa
+ fwts.html: |
+
+ FWTS Test SummaryFWTS Test Summary
Result Summary
fwts
+ sct.html: |
+
+ SCT Test SummarySCT Test Summary
Result Summary
sct
+ bbsr_fwts.html: |
+
+ FWTS Test SummaryFWTS Test Summary
Result Summary
bbsr_fwts
+ bbsr_sct.html: |
+
+ SCT Test SummarySCT Test Summary
Result Summary
bbsr_sct
+ bbsr_tpm.html: |
+
+ TPM Test SummaryTPM Test Summary
Result Summary
bbsr_tpm
+ pfdi.html: |
+
+ PFDI Test SummaryPFDI Test Summary
Result Summary
pfdi
+ post_script.html: |
+
+ Post-Script Test SummaryPost-Script Test Summary
Result Summary
post_script
+ standalone.html: |
+
+ Standalone Test SummaryStandalone Test Summary
Result Summary
standalone
+ os_tests.html: |
+
+ OS Tests Test SummaryOS Tests Test Summary
Result Summary
os_tests
+ capsule.html: |
+
+ Capsule Update Test SummaryCapsule Update Test Summary
Result Summary
capsule
+ sbmr_ib.html: |
+
+ SBMR-IB Test SummarySBMR-IB Test Summary
Result Summary
sbmr_ib
+ sbmr_oob.html: |
+
+ SBMR-OOB Test SummarySBMR-OOB Test Summary
Result Summary
sbmr_oob
+ scmi.html: |
+
+ SCMI Test SummarySCMI Test Summary
Result Summary
scmi
+ args:
+ - "{dir}/bsa.html"
+ - "{dir}/sbsa.html"
+ - "{dir}/fwts.html"
+ - "{dir}/sct.html"
+ - "{dir}/bbsr_fwts.html"
+ - "{dir}/bbsr_sct.html"
+ - "{dir}/bbsr_tpm.html"
+ - "{dir}/pfdi.html"
+ - "{dir}/post_script.html"
+ - "{dir}/standalone.html"
+ - "{dir}/os_tests.html"
+ - "{dir}/capsule.html"
+ - "{dir}/sbmr_ib.html"
+ - "{dir}/sbmr_oob.html"
+ - "{dir}/scmi.html"
+ - "{dir}/summary.html"
+ expect_exit_code: 0
+ post_checks:
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: "BSA"
+ - type: file_not_contains
+ path: "{dir}/summary.html"
+ text: "RESULT SUMMARY"
+
+ - name: cli_parses_merged_json_compliance_details
+ type: cli
+ text_files:
+ bsa.html: |
+
+ BSA Test SummaryBSA Test Summary
Result Summary
BSA detail parsing content
+ merged.json: |
+ {
+ "Suite_Name: acs_info": {
+ "ACS Results Summary": {
+ "Overall Compliance Result": "Not Compliant : Mandatory - (not run: BSA, FWTS; failed: SCT) : Recommended - (not run: SBSA; failed: PFDI)",
+ "BBSR compliance results": "Not Compliant : Mandatory - (not run: BBSR-FWTS; failed: BBSR-SCT)",
+ "SCMI compliance results": "Not Compliant"
+ }
+ }
+ }
+ args:
+ - --merged_json
+ - "{dir}/merged.json"
+ - "{dir}/bsa.html"
+ - "{dir}/sbsa_missing.html"
+ - "{dir}/fwts_missing.html"
+ - "{dir}/sct_missing.html"
+ - "{dir}/bbsr_fwts_missing.html"
+ - "{dir}/bbsr_sct_missing.html"
+ - "{dir}/bbsr_tpm_missing.html"
+ - "{dir}/pfdi_missing.html"
+ - "{dir}/post_script_missing.html"
+ - "{dir}/standalone_missing.html"
+ - "{dir}/os_tests_missing.html"
+ - "{dir}/capsule_missing.html"
+ - "{dir}/sbmr_ib_missing.html"
+ - "{dir}/sbmr_oob_missing.html"
+ - "{dir}/scmi_missing.html"
+ - "{dir}/summary.html"
+ expect_exit_code: 0
+ post_checks:
+ - type: exists
+ path: "{dir}/summary.html"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: "not run: BSA, FWTS"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: "failed: SCT"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: "not run: SBSA"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: "failed: PFDI"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: "not run: BBSR-FWTS"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: "failed: BBSR-SCT"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: "failed: SCMI"
+
+ - name: cli_injects_test_suite_info_into_detailed_html
+ type: cli
+ text_files:
+ bsa.html: |
+
+ BSA Test SummaryBSA Test Summary
Result Summary
BSA suite info content
+ bsa_detailed.html: |
+
+ details body
+ merged.json: |
+ {
+ "Suite_Name: acs_info": {
+ "ACS Results Summary": {
+ "Overall Compliance Result": "Compliant"
+ }
+ },
+ "Suite_Name: BSA": {
+ "test_results": [
+ {
+ "Test_suite": "PE",
+ "Test_suite_info": [
+ "PCIe root complex present",
+ "BAR sizing checked"
+ ]
+ }
+ ]
+ }
+ }
+ args:
+ - --merged_json
+ - "{dir}/merged.json"
+ - "{dir}/bsa.html"
+ - "{dir}/sbsa_missing.html"
+ - "{dir}/fwts_missing.html"
+ - "{dir}/sct_missing.html"
+ - "{dir}/bbsr_fwts_missing.html"
+ - "{dir}/bbsr_sct_missing.html"
+ - "{dir}/bbsr_tpm_missing.html"
+ - "{dir}/pfdi_missing.html"
+ - "{dir}/post_script_missing.html"
+ - "{dir}/standalone_missing.html"
+ - "{dir}/os_tests_missing.html"
+ - "{dir}/capsule_missing.html"
+ - "{dir}/sbmr_ib_missing.html"
+ - "{dir}/sbmr_oob_missing.html"
+ - "{dir}/scmi_missing.html"
+ - "{dir}/summary.html"
+ expect_exit_code: 0
+ post_checks:
+ - type: exists
+ path: "{dir}/summary.html"
+ - type: file_contains
+ path: "{dir}/bsa_detailed.html"
+ text: "Test_suite_info:"
+ - type: file_contains
+ path: "{dir}/bsa_detailed.html"
+ text: "PCIe root complex present"
+ - type: file_contains
+ path: "{dir}/bsa_detailed.html"
+ text: "BAR sizing checked"
+ - type: file_contains
+ path: "{dir}/bsa_detailed.html"
+ text: "5'
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: '| 0 | '
+
+ - name: os_tests_json_to_html_specific
+
+ # OS tests specific validation.
+ files:
+ - common/log_parser/os_tests/json_to_html.py
+
+ defaults:
+ type: cli
+ env:
+ MPLBACKEND: Agg
+
+ cases:
+ # Malformed JSON must fail loudly.
+ - name: os_tests_invalid_json_input
+ scripts:
+ os_test.json: |
+ { invalid json
+ args:
+ - "{dir}/os_test.json"
+ - "{dir}/detail.html"
+ - "{dir}/summary.html"
+ expect_exit_code_in: [1, 2]
+ expect_stdout_or_stderr_regex:
+ - "(?i)(error|exception|traceback|usage|json)"
+
+ # Verifies the handling of an unsupported sub_test_result shape.
+ # Outside SR-single mode, the summary should count it as failed.
+ - name: os_tests_unknown_subtest_is_treated_as_failed_outside_sr_single_mode
+ warn_only: true
+ scripts:
+ os_custom.json: |
+ {
+ "os_name": "DemoOS",
+ "test_results": [
+ {
+ "Test_suite_name": "OS Tests",
+ "Test_suite_description": "Unknown status handling",
+ "Test_case": "Case 1",
+ "Test_case_description": "Unknown subtest result",
+ "subtests": [
+ {
+ "sub_Test_Number": "1",
+ "sub_Test_Description": "Unknown result object",
+ "sub_test_result": {}
+ }
+ ]
+ }
+ ]
+ }
+ args:
+ - "{dir}/os_custom.json"
+ - "{dir}/detail.html"
+ - "{dir}/summary.html"
+ expect_exit_code: 0
+ post_checks:
+ - type: exists
+ path: "{dir}/detail.html"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "Case 1"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "INFO"
+ - type: exists
+ path: "{dir}/summary.html"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: "Result Summary"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: '1 | '
+
+ - name: pfdi_json_to_html_specific
+
+ # PFDI-specific validation.
+ files:
+ - common/log_parser/pfdi/json_to_html.py
+
+ defaults:
+ type: cli
+ env:
+ MPLBACKEND: Agg
+
+ cases:
+ # Malformed JSON must fail loudly.
+ - name: pfdi_invalid_json_input
+ scripts:
+ pfdi.json: |
+ { invalid json
+ args:
+ - "{dir}/pfdi.json"
+ - "{dir}/detail.html"
+ - "{dir}/summary.html"
+ expect_exit_code_in: [1, 2]
+ expect_stdout_or_stderr_regex:
+ - "(?i)(error|exception|traceback|json)"
+
+ # Nested waiver_reason should be promoted and displayed at subtest level.
+ - name: pfdi_nested_waiver_reason_is_promoted_to_subtest
+ warn_only: true
+ scripts:
+ pfdi.json: |
+ [
+ {
+ "Test_suite": "PFDI",
+ "subtests": [
+ {
+ "sub_Test_Number": "1",
+ "sub_Test_Description": "Nested waiver reason",
+ "sub_test_result": {
+ "waiver_reason": "nested waiver detail"
+ }
+ }
+ ],
+ "test_suite_summary": {
+ "total_passed": 1,
+ "total_failed": 0,
+ "total_failed_with_waiver": 0,
+ "total_aborted": 0,
+ "total_skipped": 0,
+ "total_warnings": 0
+ }
+ },
+ {
+ "Suite_summary": "overall"
+ }
+ ]
+ args:
+ - "{dir}/pfdi.json"
+ - "{dir}/detail.html"
+ - "{dir}/summary.html"
+ expect_exit_code: 0
+ post_checks:
+ - type: exists
+ path: "{dir}/detail.html"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "PFDI"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "Nested waiver reason"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "nested waiver detail"
+ - type: exists
+ path: "{dir}/summary.html"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: "Result Summary"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: '1 | '
+
+ - name: post_script_json_to_html_specific
+
+ # Post-script specific validation.
+ files:
+ - common/log_parser/post_script/json_to_html.py
+
+ defaults:
+ type: cli
+ env:
+ MPLBACKEND: Agg
+
+ cases:
+ # Malformed JSON must fail loudly.
+ - name: post_script_invalid_json_input
+ scripts:
+ post.json: |
+ { invalid json
+ args:
+ - "{dir}/post.json"
+ - "{dir}/detail.html"
+ - "{dir}/summary.html"
+ expect_exit_code_in: [1, 2]
+ expect_stdout_or_stderr_regex:
+ - "(?i)(error|exception|traceback|json)"
+
+ # FAILED_WITH_WAIVER should be rendered distinctly in both detail and summary output.
+ - name: post_script_failed_with_waiver_is_rendered_separately
+ warn_only: true
+ scripts:
+ post.json: |
+ {
+ "suite_summary": {
+ "total_passed": 0,
+ "total_failed": 0,
+ "total_failed_with_waiver": 1,
+ "total_aborted": 0,
+ "total_skipped": 0,
+ "total_warnings": 0
+ },
+ "test_results": [
+ {
+ "Test_suite": "Post Script",
+ "Test_suite_description": "Waiver handling",
+ "subtests": [
+ {
+ "sub_Test_Number": "1",
+ "sub_Test_Description": "Waived failure",
+ "sub_test_result": {
+ "FAILED_WITH_WAIVER": 1,
+ "waiver_reason": "approved"
+ }
+ }
+ ]
+ }
+ ]
+ }
+ args:
+ - "{dir}/post.json"
+ - "{dir}/detail.html"
+ - "{dir}/summary.html"
+ expect_exit_code: 0
+ post_checks:
+ - type: exists
+ path: "{dir}/detail.html"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "Post Script"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "Waived failure"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "FAILED WITH WAIVER"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "approved"
+ - type: exists
+ path: "{dir}/summary.html"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: "Result Summary"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: '1 | '
+
+ - name: sbmr_json_to_html_specific
+
+ # SBMR-specific validation.
+ files:
+ - common/log_parser/sbmr/json_to_html.py
+
+ defaults:
+ type: cli
+ env:
+ MPLBACKEND: Agg
+
+ cases:
+ # Malformed JSON must fail loudly.
+ - name: sbmr_invalid_json_input
+ scripts:
+ sbmr.json: |
+ { invalid json
+ args:
+ - "{dir}/sbmr.json"
+ - "{dir}/detail.html"
+ - "{dir}/summary.html"
+ expect_exit_code_in: [1, 2]
+ expect_stdout_or_stderr_regex:
+ - "(?i)(error|exception|traceback|json)"
+
+ # Same malformed-input check, but with an extra report path argument.
+ - name: sbmr_invalid_json_input_with_report_path
+ scripts:
+ sbmr.json: |
+ { invalid json
+ report.html: |
+ report
+ args:
+ - "{dir}/sbmr.json"
+ - "{dir}/detail.html"
+ - "{dir}/summary.html"
+ - "{dir}/report.html"
+ expect_exit_code_in: [1, 2]
+ expect_stdout_or_stderr_regex:
+ - "(?i)(error|exception|traceback|json)"
+
+ # When top-level summary is missing, the implementation should recompute it from testcase data.
+ - name: sbmr_recomputes_summary_when_top_level_summary_missing
+ warn_only: true
+ scripts:
+ sbmr.json: |
+ {
+ "test_results": [
+ {
+ "Test_suite": "SBMR IB",
+ "test_suite_summary": {
+ "total_passed": 1,
+ "total_failed": 0,
+ "total_failed_with_waiver": 0,
+ "total_aborted": 0,
+ "total_skipped": 0,
+ "total_warnings": 0
+ },
+ "subtests": [
+ {
+ "sub_Test_Number": "1",
+ "sub_Test_Description": "Passed case",
+ "sub_test_result": "PASSED"
+ }
+ ]
+ }
+ ]
+ }
+ args:
+ - "{dir}/sbmr.json"
+ - "{dir}/detail.html"
+ - "{dir}/summary.html"
+ expect_exit_code: 0
+ post_checks:
+ - type: exists
+ path: "{dir}/detail.html"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "SBMR IB"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "Passed case"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "PASSED"
+ - type: exists
+ path: "{dir}/summary.html"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: "Overall Summary"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: '1 | '
+
+ - name: scmi_json_to_html_specific
+
+ # SCMI-specific validation.
+ files:
+ - common/log_parser/scmi/json_to_html.py
+
+ defaults:
+ type: cli
+ env:
+ MPLBACKEND: Agg
+
+ cases:
+ # Malformed JSON must fail loudly.
+ - name: scmi_invalid_json_input
+ scripts:
+ scmi.json: |
+ { invalid json
+ args:
+ - "{dir}/scmi.json"
+ - "{dir}/detail.html"
+ - "{dir}/summary.html"
+ expect_exit_code_in: [1, 2]
+ expect_stdout_or_stderr_regex:
+ - "(?i)(error|exception|traceback|json)"
+
+ # Summary should be tallied from testcase-level results when that is the source of truth.
+ - name: scmi_summary_is_tallied_from_testcase_results
+ warn_only: true
+ scripts:
+ scmi.json: |
+ {
+ "test_results": [
+ {
+ "Test_suite": "SCMI",
+ "reason": "Suite reason",
+ "testcases": [
+ {
+ "Test_case": "Case 1",
+ "Test_case_description": "One passed testcase",
+ "Test_result": "PASSED",
+ "reason": "ok"
+ }
+ ]
+ }
+ ]
+ }
+ args:
+ - "{dir}/scmi.json"
+ - "{dir}/detail.html"
+ - "{dir}/summary.html"
+ expect_exit_code: 0
+ post_checks:
+ - type: exists
+ path: "{dir}/detail.html"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "SCMI"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "Case 1"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "PASSED"
+ - type: exists
+ path: "{dir}/summary.html"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: "Result Summary"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: '1 | '
+
+ - name: standalone_tests_json_to_html_specific
+
+ # Standalone-tests specific validation.
+ files:
+ - common/log_parser/standalone_tests/json_to_html.py
+
+ defaults:
+ type: cli
+ env:
+ MPLBACKEND: Agg
+
+ cases:
+ # Malformed JSON must fail loudly.
+ - name: standalone_invalid_json_input
+ scripts:
+ standalone.json: |
+ { invalid json
+ args:
+ - "{dir}/standalone.json"
+ - "{dir}/detail.html"
+ - "{dir}/summary.html"
+ expect_exit_code_in: [1, 2]
+ expect_stdout_or_stderr_regex:
+ - "(?i)(error|exception|traceback|usage|json)"
+
+ # Unknown subtest result shape should still render in detail,
+ # while summary currently counts the testcase as passed.
+ - name: unknown_subtest_status_appears_in_detail_but_summary_counts_test_as_passed
+ warn_only: true
+ scripts:
+ standalone.json: |
+ {
+ "test_results": [
+ {
+ "Test_suite": "Standalone",
+ "Test_suite_description": "Unknown status handling",
+ "Test_case": "Case 1",
+ "Test_case_description": "Unsupported status shape",
+ "subtests": [
+ {
+ "sub_Test_Number": "1",
+ "sub_Test_Description": "Status is empty dict",
+ "sub_test_result": {}
+ }
+ ]
+ }
+ ]
+ }
+ args:
+ - "{dir}/standalone.json"
+ - "{dir}/detail.html"
+ - "{dir}/summary.html"
+ expect_exit_code: 0
+ post_checks:
+ - type: exists
+ path: "{dir}/detail.html"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "Standalone"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "Case 1"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "UNKNOWN"
+ - type: exists
+ path: "{dir}/summary.html"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: "Result Summary"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: '1 | '
+
+ # WARNING should take priority over FAILED_WITH_WAIVER in summary classification.
+ - name: warning_status_takes_priority_over_failed_with_waiver
+ warn_only: true
+ scripts:
+ standalone.json: |
+ {
+ "test_results": [
+ {
+ "Test_suite": "Standalone",
+ "Test_suite_description": "Priority handling",
+ "Test_case": "Case 1",
+ "Test_case_description": "Warnings and waiver together",
+ "subtests": [
+ {
+ "sub_Test_Number": "1",
+ "sub_Test_Description": "Warning subtest",
+ "sub_test_result": {
+ "WARNINGS": 1,
+ "warning_reasons": ["warning path"]
+ }
+ },
+ {
+ "sub_Test_Number": "2",
+ "sub_Test_Description": "Waiver subtest",
+ "sub_test_result": {
+ "FAILED_WITH_WAIVER": 1,
+ "waiver_reason": "accepted waiver"
+ }
+ }
+ ]
+ }
+ ]
+ }
+ args:
+ - "{dir}/standalone.json"
+ - "{dir}/detail.html"
+ - "{dir}/summary.html"
+ expect_exit_code: 0
+ post_checks:
+ - type: exists
+ path: "{dir}/detail.html"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "Warning subtest"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "Waiver subtest"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "FAILED WITH WAIVER"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "accepted waiver"
+ - type: exists
+ path: "{dir}/summary.html"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: "Result Summary"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: '1 | '
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: '0 | '
+
+ # Nested reason lists should be flattened into a single rendered reason column.
+ - name: nested_reason_lists_are_flattened_into_single_reason_column
+ warn_only: true
+ scripts:
+ standalone.json: |
+ {
+ "test_results": [
+ {
+ "Test_suite": "Standalone",
+ "Test_suite_description": "Reason flattening",
+ "Test_case": "Case 1",
+ "Test_case_description": "Nested reasons",
+ "subtests": [
+ {
+ "sub_Test_Number": "1",
+ "sub_Test_Description": "Nested fail reasons",
+ "sub_test_result": {
+ "FAILED": 1,
+ "fail_reasons": [["reason A", "reason B"], ["reason C"]]
+ }
+ }
+ ]
+ }
+ ]
+ }
+ args:
+ - "{dir}/standalone.json"
+ - "{dir}/detail.html"
+ - "{dir}/summary.html"
+ expect_exit_code: 0
+ post_checks:
+ - type: exists
+ path: "{dir}/detail.html"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "Nested fail reasons"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "FAILED"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "reason A"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "reason B"
+ - type: file_contains
+ path: "{dir}/detail.html"
+ text: "reason C"
+ - type: exists
+ path: "{dir}/summary.html"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: "Result Summary"
+ - type: file_contains
+ path: "{dir}/summary.html"
+ text: '1 | '
\ No newline at end of file
diff --git a/common/acs_test_framework_manifests/logs-to-json.yaml b/common/acs_test_framework_manifests/logs-to-json.yaml
new file mode 100644
index 00000000..c31a7c37
--- /dev/null
+++ b/common/acs_test_framework_manifests/logs-to-json.yaml
@@ -0,0 +1,284 @@
+suites:
+
+ - name: logs_to_json_common
+ files:
+ - common/log_parser/bbr/fwts/logs_to_json.py
+ - common/log_parser/bbr/sct/logs_to_json.py
+ - common/log_parser/bbr/tpm/logs_to_json.py
+ - common/log_parser/bsa/logs_to_json.py
+ - common/log_parser/os_tests/logs_to_json.py
+ - common/log_parser/pfdi/logs_to_json.py
+ - common/log_parser/post_script/logs_to_json.py
+ - common/log_parser/sbmr/logs_to_json.py
+ - common/log_parser/scmi/logs_to_json.py
+ - common/log_parser/standalone_tests/logs_to_json.py
+ - common/log_parser/os_tests/sr_logs_to_json.py
+ - common/log_parser/bbr/sct/logs_to_json_edk2.py
+
+ cases:
+ - name: file_exists
+ type: file_exists
+
+ - name: python_compiles
+ type: py_compile
+
+ - name: contains_json_usage
+ type: source_contains
+ pattern: "json"
+
+ - name: contains_parse_logic
+ type: source_contains_any
+ patterns:
+ - "parse"
+ - "parser"
+ - "parse_"
+
+ - name: contains_result_collection_logic
+ type: source_contains_any
+ patterns:
+ - "subtest"
+ - "test_results"
+ - "results"
+
+ # 🔧 FIXED: works for edk2 also
+ - name: contains_summary_logic
+ type: source_contains_any
+ patterns:
+ - "summary"
+ - "suite_summary"
+ - "total_"
+ - "\"result\""
+ - "\"reason\""
+
+ - name: contains_output_logic
+ type: source_contains_any
+ patterns:
+ - "json.dump"
+ - "write"
+
+ - name: has_main_guard
+ type: main_guard
+
+
+# =========================
+# SR LOGS TO JSON
+# =========================
+
+ - name: sr_logs_to_json_specific
+ files:
+ - common/log_parser/os_tests/sr_logs_to_json.py
+
+ cases:
+ - name: has_sr_keyword
+ type: source_contains_all
+ patterns:
+ - "OS_RELEASE_FILE_NAME"
+ - "cat-etc-os-release.txt"
+ - "\"os_test\""
+
+ - name: has_sr_parse_logic
+ type: source_contains_all
+ patterns:
+ - "def build_results("
+ - "def parse_os_release("
+ - "def parse_post_script_errors("
+
+ - name: has_sr_status_handling
+ type: source_contains_all
+ patterns:
+ - "\"PASSED\""
+ - "\"FAILED\""
+ - "\"WARNINGS\""
+
+ - name: cli_requires_three_args
+ type: cli
+ args: []
+ expect_exit_nonzero: true
+ expect_stdout_or_stderr_contains:
+ - "Usage:"
+
+ - name: cli_builds_json_for_rhel_and_sle
+ type: cli
+ command: "./run_case.sh"
+ timeout_sec: 5
+ scripts:
+ run_case.sh: |
+ #!/bin/sh
+ set -eu
+
+ mkdir -p oslogs/rhel oslogs/sle
+
+ cat > oslogs/rhel/cat-etc-os-release.txt < oslogs/sle/cat-etc-os-release.txt < post-script.log < oslogs/rhel/cat-etc-os-release.txt < oslogs/rhel/cat-etc-os-release.txt < oslogs/sle/cat-etc-os-release.txt < oslogs/ubuntu/cat-etc-os-release.txt < post-script.log < edk2-test-parser.log <
+ BSA Test SummaryBSA Test Summary
Result Summary
BSA merge marker
+ fwts_summary.html: |
+
+ FWTS Test SummaryFWTS Test Summary
Result Summary
FWTS merge marker
+ args:
+ - "{dir}/bsa_summary.html"
+ - "{dir}/fwts_summary.html"
+ - "{dir}/acs_summary.html"
+ post_checks:
+ - type: file_not_empty
+ path: "{dir}/acs_summary.html"
+ - type: ordered_contains
+ path: "{dir}/acs_summary.html"
+ texts:
+ - "BSA Test Summary"
+ - "BSA merge marker"
+ - "FWTS Test Summary"
+ - "FWTS merge marker"
+
+ - name: cli_preserves_blank_lines_during_merge
+ description: "Verify that blank lines present in repo-like log content are preserved in the merged output, ensuring formatting-sensitive diagnostic text is not unintentionally normalized or collapsed."
+ text_files:
+ # Simulate a plain-text ACS info fragment where an intentional blank line
+ # separates sections and must survive the merge unchanged.
+ acs_results.log: |
+ Suite_Name: acs_info
+
+ Overall Compliance Result: Compliant
+ extensions.log: |
+ SCMI compliance results: Compliant
+ args:
+ - "{dir}/acs_results.log"
+ - "{dir}/extensions.log"
+ - "{dir}/acs_summary.html"
+ post_checks:
+ - type: regex
+ path: "{dir}/acs_summary.html"
+ pattern: "Suite_Name: acs_info\\n\\nOverall Compliance Result: Compliant"
+
+ - name: cli_output_file_is_created_when_inputs_exist
+ description: "Verify that the script creates the requested output file when valid repo-like summary inputs are supplied, even in a minimal merge scenario."
+ text_files:
+ # Two minimal summary pages are enough here because the test only cares
+ # that a merged artifact is written, not about exact merged ordering.
+ sbsa_summary.html: |
+
+ SBSA Test SummarySBSA Test Summary
Result Summary
SBSA marker
+ scmi_summary.html: |
+
+ SCMI Test SummarySCMI Test Summary
Result Summary
SCMI marker
+ args:
+ - "{dir}/sbsa_summary.html"
+ - "{dir}/scmi_summary.html"
+ - "{dir}/acs_summary.html"
+ post_checks:
+ - type: exists
+ path: "{dir}/acs_summary.html"
+
+ - name: cli_handles_empty_first_input
+ description: "Verify that the script handles an empty first summary input file gracefully and still merges content from the remaining non-empty summary into the output."
+ text_files:
+ # Empty string means "existing but empty file", which is different from a
+ # missing path and exercises a different branch in the merge flow.
+ empty_bsa_summary.html: ""
+ os_tests_summary.html: |
+
+ OS Tests Test SummaryOS Tests Test Summary
Result Summary
OS tests payload marker
+ args:
+ - "{dir}/empty_bsa_summary.html"
+ - "{dir}/os_tests_summary.html"
+ - "{dir}/acs_summary.html"
+ post_checks:
+ - type: file_contains
+ path: "{dir}/acs_summary.html"
+ text: "OS tests payload marker"
+
+ - name: cli_handles_both_inputs_empty
+ description: "Verify that the script handles the case where both summary input files are empty, completing cleanly and still producing an output artifact."
+ text_files:
+ empty_bsa_summary.html: ""
+ empty_fwts_summary.html: ""
+ args:
+ - "{dir}/empty_bsa_summary.html"
+ - "{dir}/empty_fwts_summary.html"
+ - "{dir}/acs_summary.html"
+ post_checks:
+ - type: exists
+ path: "{dir}/acs_summary.html"
+
+ - name: cli_missing_second_input_fails_even_if_first_exists
+ description: "Verify that the script fails when the second summary input file is missing even if the first summary input file exists, ensuring partial input availability is not treated as sufficient."
+ text_files:
+ # Keep the first file valid so the failure is isolated to the missing
+ # second input rather than a generic all-inputs-missing condition.
+ bsa_summary.html: |
+
+ BSA Test SummaryBSA Test Summary
Result Summary
BSA missing-second marker
+ args:
+ - "{dir}/bsa_summary.html"
+ - "{dir}/fwts_summary_missing.html"
+ - "{dir}/acs_summary.html"
+ expect_exit_nonzero: true
+
+ - name: cli_overwrites_existing_output_file
+ description: "Verify that an already-existing ACS summary output file is overwritten with fresh merged summary content rather than appended to or left unchanged."
+ text_files:
+ # Seed the destination with stale content to prove merge_summary.py
+ # overwrites the old artifact instead of appending to it.
+ bsa_summary.html: |
+
+ BSA Test SummaryBSA Test Summary
Result Summary
BSA fresh marker
+ fwts_summary.html: |
+
+ FWTS Test SummaryFWTS Test Summary
Result Summary
FWTS fresh marker
+ acs_summary.html: |
+
+ Old ACS Summarystale ACS summary content
+ args:
+ - "{dir}/bsa_summary.html"
+ - "{dir}/fwts_summary.html"
+ - "{dir}/acs_summary.html"
+ post_checks:
+ - type: file_contains
+ path: "{dir}/acs_summary.html"
+ text: "BSA fresh marker"
+ - type: file_contains
+ path: "{dir}/acs_summary.html"
+ text: "FWTS fresh marker"
+ - type: file_not_contains
+ path: "{dir}/acs_summary.html"
+ text: "stale ACS summary content"
+
+ - name: cli_binary_or_non_utf8_input_fails_cleanly
+ description: "Verify that the script exits non-zero when a summary input file contains bytes that are invalid under the active text decoding mode."
+ env:
+ # Force deterministic UTF-8 decoding so 0xff is always undecodable,
+ # regardless of the host environment's default text codec.
+ PYTHONUTF8: "1"
+ bin_files:
+ corrupt_summary.html:
+ hex: "ff"
+ args:
+ - "{dir}/corrupt_summary.html"
+ - "{dir}/acs_summary.html"
+ expect_exit_nonzero: true
+
+ - name: cli_output_parent_directory_missing_fails
+ description: "Verify that the script fails when the requested ACS summary output path points into a parent directory that does not exist."
+ text_files:
+ # The input summary is valid here; only the destination directory is
+ # deliberately invalid so the failure reason stays narrow and obvious.
+ bsa_summary.html: |
+
+ BSA Test SummaryBSA Test Summary
Result Summary
BSA output-path marker
+ args:
+ - "{dir}/bsa_summary.html"
+ - "{dir}/missing_dir/acs_summary.html"
+ expect_exit_nonzero: true
diff --git a/common/acs_test_framework_manifests/parser_common.yaml b/common/acs_test_framework_manifests/parser_common.yaml
new file mode 100644
index 00000000..a3e71b6f
--- /dev/null
+++ b/common/acs_test_framework_manifests/parser_common.yaml
@@ -0,0 +1,343 @@
+suites:
+ - name: parser_common
+ files:
+ - common/parser/Parser.py
+
+ cases:
+ - name: file_exists
+ type: file_exists
+
+ - name: python_compiles
+ type: py_compile
+
+ - name: has_main_guard
+ type: main_guard
+
+ - name: cli_help_flag_shows_usage_and_flags
+ type: cli
+ args:
+ - --help
+ expect_exit_code: 0
+ expect_output:
+ - "usage:"
+ - "-bsa"
+ - "-sbsa"
+ - "-fwts"
+ - "-automation"
+ - "--config"
+
+ - name: cli_no_args_prints_invalid_command_message
+ type: cli
+ args: []
+ scripts:
+ config.ini: |
+ [AUTOMATION]
+ config_enabled_for_automation_run = true
+ expect_exit_code: 0
+ expect_output:
+ - "Please specify a valid command or use --help for more information."
+
+ - name: cli_bsa_disabled_or_missing_section_returns_message
+ type: cli
+ args:
+ - -bsa
+ - --config
+ - "{dir}/config.ini"
+ scripts:
+ config.ini: |
+ [AUTOMATION]
+ config_enabled_for_automation_run = true
+ expect_exit_code: 0
+ expect_output:
+ - "BSA section is disabled or missing."
+
+ - name: cli_bsa_enabled_builds_full_command
+ type: cli
+ args:
+ - -bsa
+ - --config
+ - "{dir}/config.ini"
+ scripts:
+ config.ini: |
+ [BSA]
+ automation_bsa_run = true
+ bsa_level = 3
+ bsa_skip_rules = rule1,rule2
+ bsa_verbose = 1
+ expect_exit_code: 0
+ expect_output:
+ - "/bin/bsa"
+ - "-l 3"
+ - "--skip rule1,rule2"
+ - "-v 1"
+
+ - name: cli_bsa_enabled_with_only_required_flag_prints_base_command
+ type: cli
+ args:
+ - -bsa
+ - --config
+ - "{dir}/config.ini"
+ scripts:
+ config.ini: |
+ [BSA]
+ automation_bsa_run = true
+ expect_exit_code: 0
+ expect_output:
+ - "/bin/bsa"
+
+ - name: cli_sbsa_disabled_or_missing_section_returns_message
+ type: cli
+ args:
+ - -sbsa
+ - --config
+ - "{dir}/config.ini"
+ scripts:
+ config.ini: |
+ [AUTOMATION]
+ config_enabled_for_automation_run = false
+ expect_exit_code: 0
+ expect_output:
+ - "SBSA section is disabled or missing."
+
+ - name: cli_sbsa_enabled_builds_full_command
+ type: cli
+ args:
+ - -sbsa
+ - --config
+ - "{dir}/config.ini"
+ scripts:
+ config.ini: |
+ [SBSA]
+ automation_sbsa_run = true
+ sbsa_level = 4
+ sbsa_skip_rules = skip_a,skip_b
+ sbsa_verbose = 2
+ expect_exit_code: 0
+ expect_output:
+ - "/bin/sbsa"
+ - "-l 4"
+ - "--skip skip_a,skip_b"
+ - "-v 2"
+
+ - name: cli_sbsa_enabled_with_only_required_flag_prints_base_command
+ type: cli
+ args:
+ - -sbsa
+ - --config
+ - "{dir}/config.ini"
+ scripts:
+ config.ini: |
+ [SBSA]
+ automation_sbsa_run = true
+ expect_exit_code: 0
+ expect_output:
+ - "/bin/sbsa"
+
+ - name: cli_fwts_disabled_or_missing_section_returns_message
+ type: cli
+ args:
+ - -fwts
+ - --config
+ - "{dir}/config.ini"
+ scripts:
+ config.ini: |
+ [AUTOMATION]
+ config_enabled_for_automation_run = true
+ expect_exit_code: 0
+ expect_output:
+ - "FWTS section is disabled or missing."
+
+ - name: cli_fwts_enabled_builds_module_command
+ type: cli
+ args:
+ - -fwts
+ - --config
+ - "{dir}/config.ini"
+ scripts:
+ config.ini: |
+ [FWTS]
+ automation_fwts_run = true
+ fwts_modules = acpi
+ expect_exit_code: 0
+ expect_output:
+ - "fwts acpi"
+
+ - name: cli_fwts_enabled_without_module_prints_base_command
+ type: cli
+ args:
+ - -fwts
+ - --config
+ - "{dir}/config.ini"
+ scripts:
+ config.ini: |
+ [FWTS]
+ automation_fwts_run = true
+ expect_exit_code: 0
+ expect_output:
+ - "fwts"
+
+ - name: cli_automation_true_prints_true
+ type: cli
+ args:
+ - -automation
+ - --config
+ - "{dir}/config.ini"
+ scripts:
+ config.ini: |
+ [AUTOMATION]
+ config_enabled_for_automation_run = true
+ expect_exit_code: 0
+ expect_output:
+ - "True"
+
+ - name: cli_automation_false_prints_false
+ type: cli
+ args:
+ - -automation
+ - --config
+ - "{dir}/config.ini"
+ scripts:
+ config.ini: |
+ [AUTOMATION]
+ config_enabled_for_automation_run = false
+ expect_exit_code: 0
+ expect_output:
+ - "False"
+
+ - name: cli_automation_missing_section_reports_missing_and_false
+ type: cli
+ args:
+ - -automation
+ - --config
+ - "{dir}/config.ini"
+ scripts:
+ config.ini: |
+ [BSA]
+ automation_bsa_run = true
+ expect_exit_code: 0
+ expect_output:
+ - "Section AUTOMATION is missing in the configuration file."
+ - "False"
+
+ - name: cli_automation_bsa_run_true_prints_true
+ type: cli
+ args:
+ - -automation_bsa_run
+ - --config
+ - "{dir}/config.ini"
+ scripts:
+ config.ini: |
+ [BSA]
+ automation_bsa_run = true
+ expect_exit_code: 0
+ expect_output:
+ - "True"
+
+ - name: cli_automation_bsa_run_missing_section_reports_missing_and_false
+ type: cli
+ args:
+ - -automation_bsa_run
+ - --config
+ - "{dir}/config.ini"
+ scripts:
+ config.ini: |
+ [AUTOMATION]
+ config_enabled_for_automation_run = true
+ expect_exit_code: 0
+ expect_output:
+ - "Section BSA is missing in the configuration file."
+ - "False"
+
+ - name: cli_automation_sbsa_run_false_prints_false
+ type: cli
+ args:
+ - -automation_sbsa_run
+ - --config
+ - "{dir}/config.ini"
+ scripts:
+ config.ini: |
+ [SBSA]
+ automation_sbsa_run = false
+ expect_exit_code: 0
+ expect_output:
+ - "False"
+
+ - name: cli_automation_fwts_run_true_prints_true
+ type: cli
+ args:
+ - -automation_fwts_run
+ - --config
+ - "{dir}/config.ini"
+ scripts:
+ config.ini: |
+ [FWTS]
+ automation_fwts_run = true
+ expect_exit_code: 0
+ expect_output:
+ - "True"
+
+ - name: cli_automation_bbsr_fwts_run_true_prints_true
+ type: cli
+ args:
+ - -automation_bbsr_fwts_run
+ - --config
+ - "{dir}/config.ini"
+ scripts:
+ config.ini: |
+ [BBSR_FWTS]
+ automation_bbsr_fwts_run = true
+ expect_exit_code: 0
+ expect_output:
+ - "True"
+
+ - name: cli_automation_bbsr_tpm_run_false_prints_false
+ type: cli
+ args:
+ - -automation_bbsr_tpm_run
+ - --config
+ - "{dir}/config.ini"
+ scripts:
+ config.ini: |
+ [BBSR_TPM]
+ automation_bbsr_tpm_run = false
+ expect_exit_code: 0
+ expect_output:
+ - "False"
+
+ - name: cli_automation_sbmr_in_band_run_true_prints_true
+ type: cli
+ args:
+ - -automation_sbmr_in_band_run
+ - --config
+ - "{dir}/config.ini"
+ scripts:
+ config.ini: |
+ [SBMR]
+ automation_sbmr_in_band_run = true
+ expect_exit_code: 0
+ expect_output:
+ - "True"
+
+ - name: cli_invalid_boolean_in_bsa_section_fails
+ type: cli
+ args:
+ - -bsa
+ - --config
+ - "{dir}/config.ini"
+ scripts:
+ config.ini: |
+ [BSA]
+ automation_bsa_run = maybe
+ expect_exit_nonzero: true
+
+ - name: cli_invalid_boolean_in_automation_check_fails
+ type: cli
+ args:
+ - -automation
+ - --config
+ - "{dir}/config.ini"
+ scripts:
+ config.ini: |
+ [AUTOMATION]
+ config_enabled_for_automation_run = maybe
+ expect_exit_nonzero: true
diff --git a/common/acs_test_framework_manifests/read-write-check-blk-devices.yaml b/common/acs_test_framework_manifests/read-write-check-blk-devices.yaml
new file mode 100644
index 00000000..9e973b3d
--- /dev/null
+++ b/common/acs_test_framework_manifests/read-write-check-blk-devices.yaml
@@ -0,0 +1,430 @@
+x_module_case: &module_case
+ type: module_main_with_env
+ expect_exit_code: 0
+
+suites:
+ - name: read_write_check_blk_devices
+ files:
+ - common/linux_scripts/read_write_check_blk_devices.py
+
+ cases:
+ - name: file_exists
+ type: file_exists
+ description: "Verify that read_write_check_blk_devices.py exists at the expected repository path before any deeper structural or runtime checks are executed."
+
+ - name: python_compiles
+ type: py_compile
+ description: "Ensure that the script compiles successfully as valid Python so syntax-level regressions are caught before runtime behavior is exercised."
+
+ - name: cli_no_disks_runs_cleanly
+ <<: *module_case
+ description: "Verify that when no block devices are detected, the script still runs cleanly and only performs the top-level discovery command."
+ scenario:
+ kind: blk_devices
+ prompt: no
+ disks: []
+ expect_stdout_or_stderr_contains:
+ - "INFO: Detected following block devices with lsblk command :"
+ post_checks:
+ - type: file_not_contains
+ path: "{dir}/blk_commands.log"
+ text: "gdisk -l /dev/"
+ - type: file_not_contains
+ path: "{dir}/blk_commands.log"
+ text: "dd if=/dev/"
+
+ - name: cli_skips_mtd_and_ram_disks
+ <<: *module_case
+ description: "Verify that MTD block devices and RAM disks are surfaced by discovery but no later disk-processing commands are run for them."
+ scenario:
+ kind: blk_devices
+ prompt: no
+ disks:
+ - name: mtdblock0
+ kind: mtdblock
+ - name: ram0
+ kind: ram
+ expect_stdout_or_stderr_contains:
+ - "INFO: Skipping MTD block device /dev/mtdblock0"
+ - "INFO: Skipping RAM disk /dev/ram0"
+ post_checks:
+ - type: file_not_contains
+ path: "{dir}/blk_commands.log"
+ text: "timeout 10 gdisk -l /dev/mtdblock0"
+ - type: file_not_contains
+ path: "{dir}/blk_commands.log"
+ text: "timeout 10 gdisk -l /dev/ram0"
+
+ - name: cli_raw_disk_block_read_success_and_no_write_prompt
+ <<: *module_case
+ description: "Verify that a disk without a valid partition table is treated as a raw device, receives only the initial read and mount check, and does not enter free-space or write-side operations when the user declines."
+ scenario:
+ kind: blk_devices
+ prompt: no
+ disks:
+ - name: sda
+ table: raw
+ block_read: pass
+ write_check:
+ mounted: false
+ expect_stdout_or_stderr_contains:
+ - "INFO: No valid partition table found for sda, treating as raw device."
+ - "INFO: No partitions detected for sda, treating as raw device."
+ - "INFO: Performing block read on /dev/sda"
+ - "INFO: Block read on /dev/sda successful"
+ post_checks:
+ - type: ordered_contains
+ path: "{dir}/blk_commands.log"
+ texts:
+ - "timeout 10 gdisk -l /dev/sda"
+ - "dd if=/dev/sda of=/dev/null bs=1M count=1"
+ - "findmnt -n /dev/sda"
+ - type: file_not_contains
+ path: "{dir}/blk_commands.log"
+ text: "df -B 512 /dev/sda --output=used,avail"
+ - type: file_not_contains
+ path: "{dir}/blk_commands.log"
+ text: "dd if=/dev/sda of=sda_backup.bin"
+
+ - name: cli_mbr_precious_partition_is_skipped
+ <<: *module_case
+ description: "Verify that an MBR EFI System partition is classified as precious metadata and skipped for block read/write activity."
+ scenario:
+ kind: blk_devices
+ prompt: no
+ disks:
+ - name: sda
+ table: mbr
+ partitions:
+ - name: sda1
+ boot: true
+ start: 2048
+ sectors: 204800
+ size: 100M
+ id: ef
+ type_name: EFI System
+ used_blocks: 10
+ available_blocks: 20
+ expect_stdout_or_stderr_contains:
+ - "INFO: Partition table type : MBR"
+ - "INFO: sda1 partition is PRECIOUS"
+ - "Skipping block read/write"
+ post_checks:
+ - type: ordered_contains
+ path: "{dir}/blk_commands.log"
+ texts:
+ - "fdisk -l /dev/sda"
+ - "df -B 512 /dev/sda1 --output=used,avail"
+ - type: file_not_contains
+ path: "{dir}/blk_commands.log"
+ text: "dd if=/dev/sda1 of=/dev/null bs=1M count=1"
+
+ - name: write_check_restores_original_block_after_successful_verification
+ type: py_function
+ warn_only: true
+ function: perform_write_check
+ description: "Verify that a successful write verification restores the original block and removes the temporary artifacts."
+ scenario:
+ kind: blk_write_check
+ device: sda1
+ partition_id: "0x83"
+ prompt: yes
+ used_blocks: 10
+ available_blocks: 20
+ write_flow:
+ readback: match
+ expect_stdout_or_stderr_contains:
+ - "INFO: Creating backup of the current block before write check..."
+ - "INFO: Writing test data to the device for write check..."
+ - "INFO: Reading back the test data for verification..."
+ - "INFO: write check passed on /dev/sda1."
+ - "INFO: Backup restored for /dev/sda1."
+ post_checks:
+ - type: file_contains
+ path: "{dir}/blk_commands.log"
+ text: "dd if=sda1_backup.bin of=/dev/sda1 bs=512 count=1 seek=10"
+ - type: not_exists
+ path: "{dir}/hello.txt"
+ - type: not_exists
+ path: "{dir}/read_hello.txt"
+ - type: not_exists
+ path: "{dir}/sda1_backup.bin"
+
+ - name: cleanup_runs_even_when_restore_is_not_reached
+ type: py_function
+ warn_only: true
+ function: perform_write_check
+ description: "Verify that verification failure still restores the backup and removes the temporary artifacts."
+ scenario:
+ kind: blk_write_check
+ device: sda1
+ partition_id: "0x83"
+ prompt: yes
+ used_blocks: 10
+ available_blocks: 20
+ write_flow:
+ readback: mismatch
+ restore: skip
+ expect_stdout_or_stderr_contains:
+ - "INFO: Creating backup of the current block before write check..."
+ - "INFO: Writing test data to the device for write check..."
+ - "INFO: Reading back the test data for verification..."
+ - "WARNING: Data integrity check failed for /dev/sda1. Possible data corruption."
+ post_checks:
+ - type: file_contains
+ path: "{dir}/blk_commands.log"
+ text: "dd if=sda1_backup.bin of=/dev/sda1 bs=512 count=1 seek=10"
+ - type: not_exists
+ path: "{dir}/hello.txt"
+ - type: not_exists
+ path: "{dir}/read_hello.txt"
+ - type: not_exists
+ path: "{dir}/sda1_backup.bin"
+
+ - name: mbr_parser_limitation_missing_required_columns_is_warned_and_block_io_is_skipped
+ type: module_main_with_env
+ warn_only: true
+ expect_exit_code: 0
+ description: "Document the current MBR parser limitation when required fdisk columns are missing and verify that block I/O is skipped safely."
+ scenario:
+ kind: blk_devices
+ prompt: no
+ disks:
+ - name: sda
+ table: mbr
+ partitions:
+ - name: sda1
+ fdisk_output: |
+ Disk /dev/sda: 16 GiB
+ Device Start End Sectors Size Type
+ /dev/sda1 2048 206847 204800 100M EFI System
+ expect_stdout_or_stderr_contains:
+ - "INFO: Partition table type : MBR"
+ - "WARNING: Could not parse enough MBR partition IDs."
+ post_checks:
+ - type: file_contains
+ path: "{dir}/blk_commands.log"
+ text: "fdisk -l /dev/sda"
+ - type: file_not_contains
+ path: "{dir}/blk_commands.log"
+ text: "dd if=/dev/sda1 of=/dev/null bs=1M count=1"
+
+ - name: gpt_partition_metadata_parsing_skips_partition_when_sgdisk_output_is_incomplete
+ type: module_main_with_env
+ warn_only: true
+ expect_exit_code: 0
+ description: "Document the current GPT metadata parsing limitation when sgdisk output is incomplete and verify that block I/O is skipped safely."
+ scenario:
+ kind: blk_devices
+ prompt: yes
+ disks:
+ - name: sda
+ table: gpt
+ partitions:
+ - name: sda1
+ sgdisk_output: |
+ Partition GUID code: C12A7328-F81F-11D2-BA4B-00A0C93EC93B (EFI System partition)
+ First sector: 2048
+ Last sector: 206847
+ Partition size: 204800 sectors
+ expect_stdout_or_stderr_contains:
+ - "INFO: Partition table type : GPT"
+ - "INFO: Unable to parse sgdisk info for sda1. Skipping."
+ post_checks:
+ - type: file_contains
+ path: "{dir}/blk_commands.log"
+ text: "sgdisk -i=1 /dev/sda"
+ - type: file_not_contains
+ path: "{dir}/blk_commands.log"
+ text: "dd if=/dev/sda1 of=/dev/null bs=1M count=1"
+
+ - name: mounted_partition_skips_write_check_and_never_queries_space
+ <<: *module_case
+ warn_only: true
+ description: "Verify that mounted partitions are skipped before free-space checks and any write-side dd operations."
+ scenario:
+ kind: blk_devices
+ prompt: yes
+ disks:
+ - name: sda
+ table: mbr
+ partitions:
+ - name: sda1
+ start: 2048
+ sectors: 204800
+ size: 100M
+ id: 83
+ type_name: Linux
+ mounted: true
+ expect_stdout_or_stderr_contains:
+ - "INFO: Performing block read on /dev/sda1 mbr_part_id = 0x83"
+ - "INFO: Block read on /dev/sda1 mbr_part_id = 0x83 successful"
+ - "INFO: /dev/sda1 is mounted, skipping write test."
+ post_checks:
+ - type: ordered_contains
+ path: "{dir}/blk_commands.log"
+ texts:
+ - "dd if=/dev/sda1 of=/dev/null bs=1M count=1"
+ - "findmnt -n /dev/sda1"
+ - type: file_not_contains
+ path: "{dir}/blk_commands.log"
+ text: "df -B 512 /dev/sda1 --output=used,avail"
+ - type: file_not_contains
+ path: "{dir}/blk_commands.log"
+ text: "dd if=/dev/sda1 of=sda1_backup.bin"
+
+ - name: raw_disk_path_never_reaches_write_side_dd_when_user_declines
+ <<: *module_case
+ warn_only: true
+ description: "Verify that the raw-disk path stops before any write-side operations when the user declines the write prompt."
+ scenario:
+ kind: blk_devices
+ prompt: no
+ disks:
+ - name: sda
+ table: raw
+ block_read: pass
+ write_check:
+ mounted: false
+ expect_stdout_or_stderr_contains:
+ - "INFO: No valid partition table found for sda, treating as raw device."
+ - "INFO: Performing block read on /dev/sda"
+ - "INFO: Block read on /dev/sda successful"
+ post_checks:
+ - type: ordered_contains
+ path: "{dir}/blk_commands.log"
+ texts:
+ - "dd if=/dev/sda of=/dev/null bs=1M count=1"
+ - "findmnt -n /dev/sda"
+ - type: file_not_contains
+ path: "{dir}/blk_commands.log"
+ text: "df -B 512 /dev/sda --output=used,avail"
+ - type: file_not_contains
+ path: "{dir}/blk_commands.log"
+ text: "dd if=/dev/sda of=sda_backup.bin"
+
+ - name: readback_exception_leaves_backup_and_temp_files_behind
+ type: py_function
+ warn_only: true
+ function: perform_write_check
+ description: "Document the current behavior where a readback exception aborts cleanup and leaves the helper artifacts behind."
+ scenario:
+ kind: blk_write_check
+ device: sda1
+ partition_id: "0x83"
+ prompt: yes
+ used_blocks: 10
+ available_blocks: 20
+ write_flow:
+ readback: error
+ readback_error: mocked readback failure
+ restore: skip
+ expect_exception: CalledProcessError
+ expect_stdout_or_stderr_contains:
+ - "INFO: Creating backup of the current block before write check..."
+ - "INFO: Writing test data to the device for write check..."
+ - "INFO: Reading back the test data for verification..."
+ post_checks:
+ - type: exists
+ path: "{dir}/hello.txt"
+ - type: not_exists
+ path: "{dir}/read_hello.txt"
+ - type: exists
+ path: "{dir}/sda1_backup.bin"
+ - type: file_not_contains
+ path: "{dir}/blk_commands.log"
+ text: "dd if=sda1_backup.bin of=/dev/sda1 bs=512 count=1 seek=10"
+
+ - name: write_prompt_timeout_defaults_to_safe_decline
+ type: py_function
+ warn_only: true
+ function: perform_write_check
+ description: "Verify that a safe-decline response from the write prompt prevents any write-side operations from starting."
+ scenario:
+ kind: blk_write_check
+ device: sda1
+ partition_id: "0x83"
+ prompt: timeout
+ post_checks:
+ - type: file_not_contains
+ path: "{dir}/blk_commands.log"
+ text: "dd if=/dev/sda1 of=sda1_backup.bin"
+
+ - name: raw_disk_write_prompt_is_allowed_without_partition_type_guard
+ type: module_main_with_env
+ warn_only: true
+ expect_exit_code: 0
+ description: "Document the current behavior that allows a raw disk to enter the write path once the user explicitly agrees."
+ scenario:
+ kind: blk_devices
+ prompt: yes
+ disks:
+ - name: sda
+ table: raw
+ block_read: pass
+ write_check:
+ mounted: false
+ used_blocks: 10
+ available_blocks: 20
+ write_flow:
+ readback: match
+ expect_stdout_or_stderr_contains:
+ - "INFO: No valid partition table found for sda, treating as raw device."
+ - "INFO: Performing block read on /dev/sda"
+ - "INFO: Block read on /dev/sda successful"
+ - "INFO: Creating backup of the current block before write check..."
+ - "INFO: write check passed on /dev/sda."
+ - "INFO: Backup restored for /dev/sda."
+ post_checks:
+ - type: ordered_contains
+ path: "{dir}/blk_commands.log"
+ texts:
+ - "findmnt -n /dev/sda"
+ - "df -B 512 /dev/sda --output=used,avail"
+ - "dd if=/dev/sda of=sda_backup.bin bs=512 count=1 skip=10"
+ - "dd if=hello.txt of=/dev/sda bs=512 count=1 seek=10"
+ - "dd if=/dev/sda of=read_hello.txt bs=512 count=1 skip=10"
+ - "dd if=sda_backup.bin of=/dev/sda bs=512 count=1 seek=10"
+ - type: not_exists
+ path: "{dir}/hello.txt"
+ - type: not_exists
+ path: "{dir}/read_hello.txt"
+ - type: not_exists
+ path: "{dir}/sda_backup.bin"
+
+ - name: mounted_guard_applies_only_to_write_phase_not_initial_block_read
+ <<: *module_case
+ warn_only: true
+ description: "Verify that mounted partitions still undergo the initial safe block read while the write phase is skipped."
+ scenario:
+ kind: blk_devices
+ prompt: yes
+ disks:
+ - name: sda
+ table: mbr
+ partitions:
+ - name: sda1
+ boot: true
+ start: 2048
+ sectors: 204800
+ size: 100M
+ id: 83
+ type_name: Linux
+ mounted: true
+ expect_stdout_or_stderr_contains:
+ - "INFO: Performing block read on /dev/sda1 mbr_part_id = 0x83"
+ - "INFO: Block read on /dev/sda1 mbr_part_id = 0x83 successful"
+ - "INFO: /dev/sda1 is mounted, skipping write test."
+ post_checks:
+ - type: ordered_contains
+ path: "{dir}/blk_commands.log"
+ texts:
+ - "dd if=/dev/sda1 of=/dev/null bs=1M count=1"
+ - "findmnt -n /dev/sda1"
+ - type: file_not_contains
+ path: "{dir}/blk_commands.log"
+ text: "df -B 512 /dev/sda1 --output=used,avail"
+ - type: file_not_contains
+ path: "{dir}/blk_commands.log"
+ text: "dd if=/dev/sda1 of=sda1_backup.bin"
+
diff --git a/common/acs_test_framework_manifests/runtime-device-mapping-conflict-checker.yaml b/common/acs_test_framework_manifests/runtime-device-mapping-conflict-checker.yaml
new file mode 100644
index 00000000..73511d3b
--- /dev/null
+++ b/common/acs_test_framework_manifests/runtime-device-mapping-conflict-checker.yaml
@@ -0,0 +1,975 @@
+# Safe scenario-based runtime-device-mapping YAML.
+#
+# Requires mock_loader support for:
+#
+# scenario:
+# kind: runtime_device_mapping
+# dts: |
+# ...
+# memmap: |
+# ...
+# memmap_hex: "..."
+#
+# The mock_loader should materialize:
+# dts -> text_files.device_tree.dts
+# memmap -> text_files.memmap.log
+# memmap_hex -> bin_files.memmap.log.hex
+#
+# IMPORTANT:
+# The main scenario corpus still uses the proven CLI case_runner.py path.
+# Native fault-injection cases may use in-process handlers when that is the
+# cleaner seam.
+
+x_case_runner: &case_runner |
+ #!/usr/bin/env python3
+ from __future__ import annotations
+
+ import importlib.util
+ import sys
+ from pathlib import Path
+
+ def main() -> int:
+ if len(sys.argv) != 2:
+ print("usage: case_runner.py ", file=sys.stderr)
+ return 2
+
+ work_dir = Path.cwd()
+ target = Path(sys.argv[1]).resolve()
+
+ spec = importlib.util.spec_from_file_location(
+ "runtime_checker_under_test",
+ target,
+ )
+ if spec is None or spec.loader is None:
+ print(f"failed to load target: {{target}}", file=sys.stderr)
+ return 2
+
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = module
+ spec.loader.exec_module(module)
+
+ module.DTS_PATH = work_dir / "device_tree.dts"
+ module.MEMMAP_PATH = work_dir / "memmap.log"
+ module.OUT_LOG_PATH = work_dir / "runtime_device_mapping_conflict_test.log"
+ module._LOG_FH = None
+
+ try:
+ module.main()
+ except SystemExit as exc:
+ code = exc.code
+ if isinstance(code, int):
+ return code
+ return 1
+
+ log_path = work_dir / "runtime_device_mapping_conflict_test.log"
+ if log_path.exists():
+ print(log_path.read_text(encoding="utf-8", errors="replace"), end="")
+
+ return 0
+
+ if __name__ == "__main__":
+ raise SystemExit(main())
+
+x_cli_case_base: &cli_case_base
+ type: cli
+ command: "{dir}/case_runner.py"
+ args:
+ - "{file}"
+ expect_exit_code: 0
+ scripts:
+ case_runner.py: *case_runner
+
+suites:
+ - name: runtime_device_mapping_conflict_checker
+ files:
+ - common/linux_scripts/runtime_device_mapping_conflict_checker.py
+
+ cases:
+ - name: file_exists
+ type: file_exists
+ description: "Verify that the runtime device mapping conflict checker script exists."
+
+ - name: python_compiles
+ type: py_compile
+ description: "Ensure that the script compiles successfully as valid Python."
+
+ - name: has_main_guard
+ type: main_guard
+ description: "Check that the script contains a standard __main__ guard."
+
+ - name: cli_missing_both_inputs_warns
+ <<: *cli_case_base
+ scenario:
+ kind: runtime_device_mapping
+ expect_output:
+ - "DEBUG: DTS file not found"
+ - "RESULTS: WARNINGS"
+ post_checks:
+ - type: exists
+ path: "{dir}/runtime_device_mapping_conflict_test.log"
+
+ - name: cli_missing_memmap_warns
+ <<: *cli_case_base
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+ uart@3000 {
+ reg = <0x0 0x3000 0x0 0x100>;
+ };
+ };
+ };
+ expect_output:
+ - "DEBUG: Memmap file not found"
+ - "RESULTS: WARNINGS"
+ post_checks:
+ - type: exists
+ path: "{dir}/runtime_device_mapping_conflict_test.log"
+
+ - name: cli_non_overlapping_regions_pass
+ <<: *cli_case_base
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+ uart@3000 {
+ reg = <0x0 0x3000 0x0 0x100>;
+ };
+ };
+ };
+ memmap: |
+ RT_Code 0x1000-0x1fff 1 0
+ RT_Data 0x5000-0x5fff 1 0
+ expect_output:
+ - "All UEFI Memmap segments CHECKED"
+ - "All DTS MMIO ranges CHECKED"
+ - "Total segments checked: 2"
+ - "Total DTS ranges checked: 1"
+ - "No overlaps found between UEFI runtime regions and DTS MMIO ranges"
+ - "RESULTS: PASSED"
+ post_checks:
+ - type: exists
+ path: "{dir}/runtime_device_mapping_conflict_test.log"
+
+ - name: cli_overlapping_regions_fail
+ <<: *cli_case_base
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+ uart@3000 {
+ reg = <0x0 0x3000 0x0 0x100>;
+ };
+ };
+ };
+ memmap: |
+ RT_Code 0x3000-0x30ff 1 0
+ expect_output:
+ - "RESULTS: FAILED"
+ - "overlaps DTS /soc/uart@3000"
+ post_checks:
+ - type: exists
+ path: "{dir}/runtime_device_mapping_conflict_test.log"
+
+ - name: cli_disabled_node_is_ignored
+ <<: *cli_case_base
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+ uart@3000 {
+ status = "disabled";
+ reg = <0x0 0x3000 0x0 0x100>;
+ };
+ };
+ };
+ memmap: |
+ RT_Code 0x3000-0x30ff 1 0
+ expect_output:
+ - "No DTS MMIO ranges extracted."
+ - "RESULTS: PASSED"
+ post_checks:
+ - type: exists
+ path: "{dir}/runtime_device_mapping_conflict_test.log"
+
+ - name: cli_disabled_ancestor_is_ignored
+ <<: *cli_case_base
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ status = "disabled";
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+
+ uart@3000 {
+ reg = <0x0 0x3000 0x0 0x100>;
+ };
+ };
+ };
+ memmap: |
+ RT_Code 0x3000-0x30ff 1 0
+ expect_output:
+ - "No DTS MMIO ranges extracted."
+ - "RESULTS: PASSED"
+
+ - name: cli_mmap_reg_name_is_ignored
+ <<: *cli_case_base
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+ flash@3000 {
+ reg = <0x0 0x3000 0x0 0x100>;
+ reg-names = "mmap";
+ };
+ };
+ };
+ memmap: |
+ RT_Code 0x3000-0x30ff 1 0
+ expect_output:
+ - "No DTS MMIO ranges extracted."
+ - "RESULTS: PASSED"
+ post_checks:
+ - type: exists
+ path: "{dir}/runtime_device_mapping_conflict_test.log"
+
+ - name: cli_reg_names_with_mmap_and_register_block_skips_only_mmap
+ <<: *cli_case_base
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+ flash@3000 {
+ reg = <0x0 0x3000 0x0 0x100 0x0 0x4000 0x0 0x100>;
+ reg-names = "ctrl\0mmap";
+ };
+ };
+ };
+ memmap: |
+ RT_Code 0x3000-0x30ff 1 0
+ expect_output:
+ - "Total DTS ranges checked: 1"
+ - "/soc/flash@3000"
+ - "RESULTS: FAILED"
+
+ - name: cli_parent_offset_translation_detects_conflict
+ <<: *cli_case_base
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+
+ syscon@4000 {
+ reg = <0x0 0x4000 0x0 0x1000>;
+
+ efuse@100 {
+ reg = <0x0 0x100 0x0 0x20>;
+ };
+ };
+ };
+ };
+ memmap: |
+ RT_Data 0x4100-0x411f 1 0
+ expect_output:
+ - "RESULTS: FAILED"
+ - "/soc/syscon@4000/efuse@100"
+ post_checks:
+ - type: exists
+ path: "{dir}/runtime_device_mapping_conflict_test.log"
+
+ - name: cli_ranges_translation_detects_conflict
+ <<: *cli_case_base
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+
+ bus@0 {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges = <0x0 0x0 0x0 0x8000 0x0 0x10000>;
+
+ uart@2000 {
+ reg = <0x0 0x2000 0x0 0x100>;
+ };
+ };
+ };
+ };
+ memmap: |
+ RT_Code 0xA000-0xA0FF 1 0
+ expect_output:
+ - "RESULTS: FAILED"
+ - "[T] /soc/bus@0/uart@2000"
+
+ - name: cli_no_matching_range_skips_node_and_passes
+ <<: *cli_case_base
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+
+ bus@0 {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges = <0x0 0x0 0x0 0x8000 0x0 0x1000>;
+
+ uart@3000 {
+ reg = <0x0 0x3000 0x0 0x100>;
+ };
+ };
+ };
+ };
+ memmap: |
+ RT_Code 0xB000-0xB0FF 1 0
+ expect_output:
+ - "No DTS MMIO ranges extracted."
+ - "RESULTS: PASSED"
+
+ - name: cli_reserved_memory_is_ignored
+ <<: *cli_case_base
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ reserved-memory {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+
+ carveout@3000 {
+ reg = <0x0 0x3000 0x0 0x100>;
+ };
+ };
+ };
+ memmap: |
+ RT_Code 0x3000-0x30ff 1 0
+ expect_output:
+ - "No DTS MMIO ranges extracted."
+ - "RESULTS: PASSED"
+
+ - name: cli_memory_node_is_ignored
+ <<: *cli_case_base
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ memory@80000000 {
+ device_type = "memory";
+ reg = <0x0 0x80000000 0x0 0x100000>;
+ };
+ };
+ memmap: |
+ RT_Code 0x80000000-0x80000fff 1 0
+ expect_output:
+ - "No DTS MMIO ranges extracted."
+ - "RESULTS: PASSED"
+
+ - name: cli_storage_like_node_is_ignored
+ <<: *cli_case_base
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ spi@3000 {
+ compatible = "jedec,spi-nor";
+ reg = <0x0 0x3000 0x0 0x100>;
+ };
+ };
+ memmap: |
+ RT_Code 0x3000-0x30ff 1 0
+ expect_output:
+ - "No DTS MMIO ranges extracted."
+ - "RESULTS: PASSED"
+
+ - name: cli_utf16_memmap_is_parsed
+ <<: *cli_case_base
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+ uart@3000 {
+ reg = <0x0 0x3000 0x0 0x100>;
+ };
+ };
+ };
+ memmap_hex: "fffe520054005f0043006f006400650020003000780031003000300030002d0030007800310066006600660020003100200030000a00"
+ expect_output:
+ - "Total segments checked: 1"
+ - "RESULTS: PASSED"
+ post_checks:
+ - type: exists
+ path: "{dir}/runtime_device_mapping_conflict_test.log"
+
+ - name: cli_malformed_memmap_line_is_logged
+ <<: *cli_case_base
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+ uart@9000 {
+ reg = <0x0 0x9000 0x0 0x100>;
+ };
+ };
+ };
+ memmap: |
+ RT_Code badline
+ RT_Data 0x1000-0x1fff 1 0
+ expect_output:
+ - "Skipped malformed memmap line"
+ - "Skipped 1 malformed lines from memmap"
+ - "RESULTS: PASSED"
+ post_checks:
+ - type: exists
+ path: "{dir}/runtime_device_mapping_conflict_test.log"
+
+ - name: cli_deduplicates_identical_memmap_entries
+ <<: *cli_case_base
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+ uart@9000 {
+ reg = <0x0 0x9000 0x0 0x100>;
+ };
+ };
+ };
+ memmap: |
+ RT_Data 0x1000-0x1fff 1 0
+ RT_Data 0x1000-0x1fff 1 0
+ expect_output:
+ - "Total segments checked: 1"
+ - "RESULTS: PASSED"
+
+ - name: cli_hex_without_0x_in_memmap_is_accepted
+ <<: *cli_case_base
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+ uart@4096 {
+ reg = <0x0 0x4096 0x0 0x100>;
+ };
+ };
+ };
+ memmap: |
+ RT_Code 1000-10ff 1 0
+ expect_output:
+ - "Total segments checked: 1"
+ - "RESULTS: PASSED"
+
+ - name: cli_symbolic_reg_cells_known_limitation
+ <<: *cli_case_base
+ warn_only: true
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+ uart@3000 {
+ reg = ;
+ };
+ };
+ };
+ memmap: |
+ RT_Code 0x3000-0x30ff 1 0
+ expect_output:
+ - "No DTS MMIO ranges extracted."
+ - "RESULTS: PASSED"
+
+ - name: cli_multiline_reg_property_warning
+ <<: *cli_case_base
+ warn_only: true
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+ uart@3000 {
+ reg = <
+ 0x0 0x3000
+ 0x0 0x100
+ >;
+ };
+ };
+ };
+ memmap: |
+ RT_Code 0x3000-0x30ff 1 0
+ expect_output:
+ - "INFO: No DTS MMIO ranges extracted."
+ - "RESULTS: PASSED"
+
+ - name: cli_multiline_ranges_property_warning
+ <<: *cli_case_base
+ warn_only: true
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+
+ bus@0 {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges = <
+ 0x0 0x0
+ 0x0 0x8000
+ 0x0 0x10000
+ >;
+
+ uart@2000 {
+ reg = <0x0 0x2000 0x0 0x100>;
+ };
+ };
+ };
+ };
+ memmap: |
+ RT_Code 0xA000-0xA0FF 1 0
+ expect_output:
+ - "Total DTS ranges checked: 1"
+ - "INFO: [T] /soc/bus@0/uart@2000"
+ - "RESULTS: PASSED"
+
+ - name: cli_multiline_reg_names_warning
+ <<: *cli_case_base
+ warn_only: true
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+ flash@3000 {
+ reg = <0x0 0x3000 0x0 0x100 0x0 0x4000 0x0 0x100>;
+ reg-names =
+ "ctrl\0mmap";
+ };
+ };
+ };
+ memmap: |
+ RT_Code 0x3000-0x30ff 1 0
+ expect_output:
+ - "Total DTS ranges checked: 2"
+ - "Detected 1 conflict(s)"
+ - "RESULTS: FAILED"
+
+ - name: cli_multiline_quoted_compatible_property_warning
+ <<: *cli_case_base
+ warn_only: true
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ spi@3000 {
+ compatible =
+ "jedec,spi-nor";
+ reg = <0x0 0x3000 0x0 0x100>;
+ };
+ };
+ memmap: |
+ RT_Code 0x3000-0x30ff 1 0
+ expect_output:
+ - "Total DTS ranges checked: 1"
+ - "Detected 1 conflict(s)"
+ - "RESULTS: FAILED"
+
+ - name: cli_multiline_status_property_warning
+ <<: *cli_case_base
+ warn_only: true
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+ uart@3000 {
+ status =
+ "disabled";
+ reg = <0x0 0x3000 0x0 0x100>;
+ };
+ };
+ };
+ memmap: |
+ RT_Code 0x3000-0x30ff 1 0
+ expect_output:
+ - "Total DTS ranges checked: 1"
+ - "Detected 1 conflict(s)"
+ - "RESULTS: FAILED"
+
+ - name: cli_multiline_device_type_memory_property_warning
+ <<: *cli_case_base
+ warn_only: true
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ memory@80000000 {
+ device_type =
+ "memory";
+ reg = <0x0 0x80000000 0x0 0x100000>;
+ };
+ };
+ memmap: |
+ RT_Code 0x80000000-0x80000fff 1 0
+ expect_output:
+ - "No DTS MMIO ranges extracted."
+ - "RESULTS: PASSED"
+
+ - name: cli_ranges_with_incomplete_cell_tuple_warning
+ <<: *cli_case_base
+ warn_only: true
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+
+ bus@0 {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges = <0x0 0x0 0x0 0x8000 0x0>;
+
+ uart@2000 {
+ reg = <0x0 0x2000 0x0 0x100>;
+ };
+ };
+ };
+ };
+ memmap: |
+ RT_Code 0xA000-0xA0FF 1 0
+ expect_output:
+ - "No DTS MMIO ranges extracted."
+ - "RESULTS: PASSED"
+
+ - name: cli_ranges_with_extra_trailing_cells_warning
+ <<: *cli_case_base
+ warn_only: true
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+
+ bus@0 {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges = <0x0 0x0 0x0 0x8000 0x0 0x10000 0xdeadbeef>;
+
+ uart@2000 {
+ reg = <0x0 0x2000 0x0 0x100>;
+ };
+ };
+ };
+ };
+ memmap: |
+ RT_Code 0xA000-0xA0FF 1 0
+ expect_output:
+ - "Total DTS ranges checked: 1"
+ - "Detected 1 conflict(s)"
+ - "RESULTS: FAILED"
+
+ - name: cli_parent_reg_multiple_windows_offset_heuristic_warning
+ <<: *cli_case_base
+ warn_only: true
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+
+ syscon@4000 {
+ reg = <0x0 0x4000 0x0 0x100 0x0 0x8000 0x0 0x100>;
+
+ efuse@80 {
+ reg = <0x0 0x80 0x0 0x20>;
+ };
+ };
+ };
+ };
+ memmap: |
+ RT_Data 0x4080-0x409f 1 0
+ expect_output:
+ - "/soc/syscon@4000/efuse@80"
+ - "Detected 2 conflict(s)"
+ - "RESULTS: FAILED"
+
+ - name: cli_reg_names_count_mismatch_warning
+ <<: *cli_case_base
+ warn_only: true
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+ flash@3000 {
+ reg = <0x0 0x3000 0x0 0x100 0x0 0x4000 0x0 0x100>;
+ reg-names = "mmap";
+ };
+ };
+ };
+ memmap: |
+ RT_Code 0x4000-0x40ff 1 0
+ expect_output:
+ - "Total DTS ranges checked: 1"
+ - "Detected 1 conflict(s)"
+ - "RESULTS: FAILED"
+
+ - name: cli_uppercase_disabled_status_value_warning
+ <<: *cli_case_base
+ warn_only: true
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+ uart@3000 {
+ status = "DISABLED";
+ reg = <0x0 0x3000 0x0 0x100>;
+ };
+ };
+ };
+ memmap: |
+ RT_Code 0x3000-0x30ff 1 0
+ expect_output:
+ - "No DTS MMIO ranges extracted."
+ - "RESULTS: PASSED"
+
+ - name: cli_ranges_present_but_empty_cells_warning
+ <<: *cli_case_base
+ warn_only: true
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+
+ bus@0 {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges = <>;
+
+ uart@2000 {
+ reg = <0x0 0x2000 0x0 0x100>;
+ };
+ };
+ };
+ };
+ memmap: |
+ RT_Code 0xA000-0xA0FF 1 0
+ expect_output:
+ - "Total DTS ranges checked: 1"
+ - "INFO: [T] /soc/bus@0/uart@2000"
+ - "RESULTS: PASSED"
+
+ - name: cli_comments_are_stripped_before_parse
+ <<: *cli_case_base
+ scenario:
+ kind: runtime_device_mapping
+ dts: |
+ /dts-v1/;
+ /* block comment */
+ / {
+ soc {
+ #address-cells = <2>; // inline comment
+ #size-cells = <2>;
+ ranges;
+ uart@3000 {
+ reg = <0x0 0x3000 0x0 0x100>; // end of line comment
+ };
+ };
+ };
+ memmap: |
+ RT_Code 0x3000-0x30ff 1 0
+ expect_output:
+ - "Total DTS ranges checked: 1"
+ - "RESULTS: FAILED"
+
+ # Native fault-injection cases stay declarative through scenario flags.
+
+ - name: cli_missing_dts_file_is_warned_natively
+ type: module_main_with_env
+ warn_only: true
+ scenario:
+ kind: runtime_device_mapping
+ missing_dts: true
+ memmap: |
+ RT_Code 0x1000-0x1fff 1 0
+ post_checks:
+ - type: exists
+ path: "{dir}/runtime_device_mapping_conflict_test.log"
+ - type: file_contains
+ path: "{dir}/runtime_device_mapping_conflict_test.log"
+ text: "DEBUG: DTS file not found:"
+ - type: file_contains
+ path: "{dir}/runtime_device_mapping_conflict_test.log"
+ text: "RESULTS: WARNINGS"
+
+ - name: cli_missing_memmap_file_is_warned_natively
+ type: module_main_with_env
+ warn_only: true
+ scenario:
+ kind: runtime_device_mapping
+ missing_memmap: true
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+ uart@3000 {
+ reg = <0x0 0x3000 0x0 0x100>;
+ };
+ };
+ };
+ post_checks:
+ - type: exists
+ path: "{dir}/runtime_device_mapping_conflict_test.log"
+ - type: file_contains
+ path: "{dir}/runtime_device_mapping_conflict_test.log"
+ text: "DEBUG: Memmap file not found:"
+ - type: file_contains
+ path: "{dir}/runtime_device_mapping_conflict_test.log"
+ text: "RESULTS: WARNINGS"
+
+ - name: cli_mock_log_open_failure_warning
+ type: py_function
+ warn_only: true
+ function: main
+ description: "Document the current behavior where opening the runtime checker log aborts the run before any warning summary is written."
+ scenario:
+ kind: runtime_device_mapping
+ log_open_error: mocked log open failure
+ dts: |
+ /dts-v1/;
+ / {
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+ uart@3000 {
+ reg = <0x0 0x3000 0x0 0x100>;
+ };
+ };
+ };
+ memmap: |
+ RT_Code 0x1000-0x1fff 1 0
+ expect_exception: OSError
+ post_checks:
+ - type: not_exists
+ path: "{dir}/runtime_device_mapping_conflict_test.log"
diff --git a/common/acs_test_framework_manifests/verify-tpm-measurements.yaml b/common/acs_test_framework_manifests/verify-tpm-measurements.yaml
new file mode 100644
index 00000000..d30bd8d7
--- /dev/null
+++ b/common/acs_test_framework_manifests/verify-tpm-measurements.yaml
@@ -0,0 +1,526 @@
+x_verify_tpm_cli: &verify_tpm_cli
+ type: cli
+ command: "./run_case.sh"
+ args:
+ - "{file}"
+ expect_exit_code: 0
+ timeout_sec: 20
+ shell: false
+ scripts:
+ run_case.sh: |
+ #!/bin/sh
+ set -eu
+ python3 "$1" "$PWD/pcr.yaml" "$PWD/event.yaml"
+
+x_valid_tpm_events: &valid_tpm_events
+ - EventNum: 1
+ PCRIndex: 0
+ EventType: EV_NO_ACTION
+ SpecID:
+ - specVersionMajor: 2
+
+ - EventNum: 2
+ PCRIndex: 0
+ EventType: EV_POST_CODE
+ Event: "BL_31"
+
+ - EventNum: 3
+ PCRIndex: 0
+ EventType: EV_POST_CODE
+ Event: "SECURE_RT_EL3"
+
+ - EventNum: 4
+ PCRIndex: 7
+ EventType: EV_EFI_VARIABLE_DRIVER_CONFIG
+ Event:
+ UnicodeName: SecureBoot
+
+ - EventNum: 5
+ PCRIndex: 7
+ EventType: EV_EFI_VARIABLE_DRIVER_CONFIG
+ Event:
+ UnicodeName: PK
+
+ - EventNum: 6
+ PCRIndex: 7
+ EventType: EV_EFI_VARIABLE_DRIVER_CONFIG
+ Event:
+ UnicodeName: KEK
+
+ - EventNum: 7
+ PCRIndex: 7
+ EventType: EV_EFI_VARIABLE_DRIVER_CONFIG
+ Event:
+ UnicodeName: db
+
+ - EventNum: 8
+ PCRIndex: 7
+ EventType: EV_EFI_VARIABLE_DRIVER_CONFIG
+ Event:
+ UnicodeName: dbx
+
+ - EventNum: 9
+ PCRIndex: 1
+ EventType: EV_EFI_VARIABLE_BOOT
+ Event:
+ UnicodeName: BootOrder
+
+ - EventNum: 10
+ PCRIndex: 1
+ EventType: EV_EFI_VARIABLE_BOOT2
+ Event:
+ UnicodeName: Boot0001
+
+ - EventNum: 11
+ PCRIndex: 4
+ EventType: EV_EFI_ACTION
+ Event: "Calling EFI Application from Boot Option"
+
+ - EventNum: 12
+ PCRIndex: 1
+ EventType: EV_EFI_HANDOFF_TABLES
+ Event: "SMBIOS"
+
+ - EventNum: 13
+ PCRIndex: 0
+ EventType: EV_SEPARATOR
+ Event: "sep0"
+
+ - EventNum: 14
+ PCRIndex: 1
+ EventType: EV_SEPARATOR
+ Event: "sep1"
+
+ - EventNum: 15
+ PCRIndex: 2
+ EventType: EV_SEPARATOR
+ Event: "sep2"
+
+ - EventNum: 16
+ PCRIndex: 3
+ EventType: EV_SEPARATOR
+ Event: "sep3"
+
+ - EventNum: 17
+ PCRIndex: 4
+ EventType: EV_SEPARATOR
+ Event: "sep4"
+
+ - EventNum: 18
+ PCRIndex: 5
+ EventType: EV_SEPARATOR
+ Event: "sep5"
+
+ - EventNum: 19
+ PCRIndex: 6
+ EventType: EV_SEPARATOR
+ Event: "sep6"
+
+ - EventNum: 20
+ PCRIndex: 7
+ EventType: EV_SEPARATOR
+ Event: "sep7"
+
+ - EventNum: 21
+ PCRIndex: 1
+ EventType: EV_TABLE_OF_DEVICES
+ Event: "SYS_CONFIG_UART0"
+
+ - EventNum: 22
+ PCRIndex: 5
+ EventType: EV_EFI_ACTION
+ Event: "Exit Boot Services Invocation"
+
+x_without_boot_variable_events: &without_boot_variable_events
+ - EventNum: 1
+ PCRIndex: 0
+ EventType: EV_NO_ACTION
+ SpecID:
+ - specVersionMajor: 2
+
+ - EventNum: 2
+ PCRIndex: 0
+ EventType: EV_POST_CODE
+ Event: "BL_31"
+
+ - EventNum: 3
+ PCRIndex: 0
+ EventType: EV_POST_CODE
+ Event: "SECURE_RT_EL3"
+
+ - EventNum: 4
+ PCRIndex: 7
+ EventType: EV_EFI_VARIABLE_DRIVER_CONFIG
+ Event:
+ UnicodeName: SecureBoot
+
+ - EventNum: 5
+ PCRIndex: 7
+ EventType: EV_EFI_VARIABLE_DRIVER_CONFIG
+ Event:
+ UnicodeName: PK
+
+ - EventNum: 6
+ PCRIndex: 7
+ EventType: EV_EFI_VARIABLE_DRIVER_CONFIG
+ Event:
+ UnicodeName: KEK
+
+ - EventNum: 7
+ PCRIndex: 7
+ EventType: EV_EFI_VARIABLE_DRIVER_CONFIG
+ Event:
+ UnicodeName: db
+
+ - EventNum: 8
+ PCRIndex: 7
+ EventType: EV_EFI_VARIABLE_DRIVER_CONFIG
+ Event:
+ UnicodeName: dbx
+
+ - EventNum: 9
+ PCRIndex: 4
+ EventType: EV_EFI_ACTION
+ Event: "Calling EFI Application from Boot Option"
+
+ - EventNum: 10
+ PCRIndex: 1
+ EventType: EV_EFI_HANDOFF_TABLES
+ Event: "SMBIOS"
+
+ - EventNum: 11
+ PCRIndex: 0
+ EventType: EV_SEPARATOR
+ Event: "sep0"
+
+ - EventNum: 12
+ PCRIndex: 1
+ EventType: EV_SEPARATOR
+ Event: "sep1"
+
+ - EventNum: 13
+ PCRIndex: 2
+ EventType: EV_SEPARATOR
+ Event: "sep2"
+
+ - EventNum: 14
+ PCRIndex: 3
+ EventType: EV_SEPARATOR
+ Event: "sep3"
+
+ - EventNum: 15
+ PCRIndex: 4
+ EventType: EV_SEPARATOR
+ Event: "sep4"
+
+ - EventNum: 16
+ PCRIndex: 5
+ EventType: EV_SEPARATOR
+ Event: "sep5"
+
+ - EventNum: 17
+ PCRIndex: 6
+ EventType: EV_SEPARATOR
+ Event: "sep6"
+
+ - EventNum: 18
+ PCRIndex: 7
+ EventType: EV_SEPARATOR
+ Event: "sep7"
+
+ - EventNum: 19
+ PCRIndex: 1
+ EventType: EV_TABLE_OF_DEVICES
+ Event: "SYS_CONFIG_UART0"
+
+ - EventNum: 20
+ PCRIndex: 5
+ EventType: EV_EFI_ACTION
+ Event: "Exit Boot Services Invocation"
+
+suites:
+ - name: verify_tpm_measurements
+ files:
+ - common/linux_scripts/verify_tpm_measurements.py
+
+ cases:
+ # Baseline repository-integrity check.
+ - name: file_exists
+ type: file_exists
+ description: "Verify that the TPM measurement verification script exists."
+
+ # Fast syntax-level validation.
+ - name: python_compiles
+ type: py_compile
+ description: "Verify that the script compiles successfully as valid Python."
+
+ # Required CLI contract validation: no arguments supplied.
+ - name: cli_no_args_fails
+ type: cli
+ command: python3
+ args:
+ - "{file}"
+ expect_exit_code: 1
+ expect_stdout_or_stderr_contains:
+ - "Usage: python3 verify_tpm_measurements.py"
+
+ # Required CLI contract validation: only one argument supplied.
+ - name: cli_one_arg_fails
+ type: cli
+ command: python3
+ args:
+ - "{file}"
+ - "{dir}/pcr.yaml"
+ expect_exit_code: 1
+ expect_stdout_or_stderr_contains:
+ - "Usage: python3 verify_tpm_measurements.py"
+
+ # Negative-path file validation.
+ - name: cli_missing_files_fail_nonzero
+ type: cli
+ command: python3
+ args:
+ - "{file}"
+ - "{dir}/missing_pcr.yaml"
+ - "{dir}/missing_event.yaml"
+ expect_exit_nonzero: true
+ expect_stdout_or_stderr_contains:
+ - "FAIL"
+
+ # Malformed-input parsing test.
+ - name: cli_invalid_event_yaml_fails
+ type: cli
+ command: python3
+ args:
+ - "{file}"
+ - "{dir}/pcr.yaml"
+ - "{dir}/event.yaml"
+ text_files:
+ pcr.yaml: |
+ sha256:
+ - "a0"
+ - "a1"
+ - "a2"
+ - "a3"
+ - "a4"
+ - "a5"
+ - "a6"
+ - "a7"
+ event.yaml: |
+ not: [valid
+ expect_exit_nonzero: true
+ expect_stdout_or_stderr_contains:
+ - "FAIL"
+
+ # Injected I/O failure test.
+ - name: cli_mock_open_raises_on_event_log
+ type: module_cli
+ scenario:
+ kind: verify_tpm
+ pcrs:
+ sha256:
+ - "a0"
+ - "a1"
+ - "a2"
+ - "a3"
+ - "a4"
+ - "a5"
+ - "a6"
+ - "a7"
+ event_pcrs:
+ sha256:
+ - "a0"
+ - "a1"
+ - "a2"
+ - "a3"
+ - "a4"
+ - "a5"
+ - "a6"
+ - "a7"
+ events: []
+ event_log_open_error: mocked read failure
+ expect_exit_code: 1
+ expect_stdout_or_stderr_contains:
+ - "FAIL"
+ - "mocked read failure"
+
+ # Injected YAML-loader failure test.
+ - name: cli_mock_yaml_safe_load_raises_on_event_log
+ type: module_cli
+ scenario:
+ kind: verify_tpm
+ pcrs:
+ sha256:
+ - "a0"
+ - "a1"
+ - "a2"
+ - "a3"
+ - "a4"
+ - "a5"
+ - "a6"
+ - "a7"
+ event_pcrs:
+ sha256:
+ - "a0"
+ - "a1"
+ - "a2"
+ - "a3"
+ - "a4"
+ - "a5"
+ - "a6"
+ - "a7"
+ events: []
+ event_log_yaml_error: mocked event parse failure
+ expect_exit_code: 1
+ expect_stdout_or_stderr_contains:
+ - "FAIL"
+ - "mocked event parse failure"
+
+ # Full positive-path event-log scenario.
+ - name: cli_all_required_measurements_pass
+ <<: *verify_tpm_cli
+ scenario:
+ kind: verify_tpm
+ pcrs:
+ sha256:
+ - "pcr0"
+ - "pcr1"
+ - "pcr2"
+ - "pcr3"
+ - "pcr4"
+ - "pcr5"
+ - "pcr6"
+ - "pcr7"
+ event_pcrs:
+ sha256:
+ - "pcr0"
+ - "pcr1"
+ - "pcr2"
+ - "pcr3"
+ - "pcr4"
+ - "pcr5"
+ - "pcr6"
+ - "pcr7"
+ events: *valid_tpm_events
+ expect_stdout_or_stderr_contains:
+ - "Verify that the cumulative SHA256 measurements from the event log match the TPM PCRs 0-7"
+ - "Verify the first event is the EV_NO_ACTION for the Specification ID version"
+ - "Verify EV_POST_CODE events for measurements of firmware to PCR[0] with recommended strings"
+ - "Verify secure boot policy measurements are made into PCR[7] with EV_EFI_VARIABLE_DRIVER_CONFIG"
+ - "Verify BootOrder and Boot#### variables are measured in PCR[1] withEV_EFI_VARIABLE_BOOT/EV_EFI_VARIABLE_BOOT2"
+ - "Verify boot attempts measured in PCR[4] with EV_EFI_ACTION event type"
+ - "Verify security relevant configuration data are measured in into PCR[1] with EV_EFI_HANDOFF_TABLES"
+ - "Verify presence of EV_SEPARATOR event for each PCR"
+ - "Verify EV_TABLE_OF_DEVICES events for measurements of config data to PCR[1] with recommended strings"
+ - "Verify presence of “Exit Boot Services Invocation” event with EV_EFI_ACTION type"
+ - "PASS"
+
+ # PCR mismatch while event classes are otherwise present.
+ - name: cli_pcr_mismatch_reports_fail_but_runs
+ <<: *verify_tpm_cli
+ scenario:
+ kind: verify_tpm
+ pcrs:
+ sha256:
+ - "good0"
+ - "good1"
+ - "good2"
+ - "good3"
+ - "good4"
+ - "good5"
+ - "good6"
+ - "good7"
+ event_pcrs:
+ sha256:
+ - "good0"
+ - "bad1"
+ - "good2"
+ - "good3"
+ - "good4"
+ - "good5"
+ - "good6"
+ - "good7"
+ events: *valid_tpm_events
+ expect_stdout_or_stderr_contains:
+ - "PCR[1] measurements does not match for sha256"
+ - "FAIL"
+
+ # Boot variable measurements intentionally omitted.
+ - name: cli_boot_variable_missing_reports_fail_but_runs
+ <<: *verify_tpm_cli
+ scenario:
+ kind: verify_tpm
+ pcrs:
+ sha256:
+ - "pcr0"
+ - "pcr1"
+ - "pcr2"
+ - "pcr3"
+ - "pcr4"
+ - "pcr5"
+ - "pcr6"
+ - "pcr7"
+ event_pcrs:
+ sha256:
+ - "pcr0"
+ - "pcr1"
+ - "pcr2"
+ - "pcr3"
+ - "pcr4"
+ - "pcr5"
+ - "pcr6"
+ - "pcr7"
+ events: *without_boot_variable_events
+ expect_stdout_or_stderr_contains:
+ - "BootOrder/Boot#### variable measurements not found"
+ - "FAIL"
+
+ # Supported-algorithm negative path.
+ - name: pcr_comparison_requires_at_least_one_supported_algorithm_match
+ <<: *verify_tpm_cli
+ warn_only: true
+ scenario:
+ kind: verify_tpm
+ pcrs:
+ sha1:
+ - "p0"
+ - "p1"
+ - "p2"
+ - "p3"
+ - "p4"
+ - "p5"
+ - "p6"
+ - "p7"
+ event_pcrs:
+ sha1:
+ - "p0"
+ - "p1"
+ - "p2"
+ - "p3"
+ - "p4"
+ - "p5"
+ - "p6"
+ - "p7"
+ events: []
+ expect_stdout_or_stderr_contains:
+ - "FAIL"
+
+ # Truncated-PCR-array negative path.
+ - name: truncated_pcr_arrays_are_reported_cleanly
+ <<: *verify_tpm_cli
+ warn_only: true
+ scenario:
+ kind: verify_tpm
+ pcrs:
+ sha256:
+ - "p0"
+ - "p1"
+ - "p2"
+ event_pcrs:
+ sha256:
+ - "p0"
+ - "p1"
+ - "p2"
+ events: []
+ expect_stdout_or_stderr_contains:
+ - "FAIL"
diff --git a/common/acs_test_framework_runner/__init__.py b/common/acs_test_framework_runner/__init__.py
new file mode 100644
index 00000000..3a588c3a
--- /dev/null
+++ b/common/acs_test_framework_runner/__init__.py
@@ -0,0 +1,134 @@
+from .mock_loader import _resolve_runner_module_from_stack, stateful_run_router
+from .mock_loader import ConfigError as MockLoaderConfigError
+from .mock_helpers import (
+ build_char16_payload,
+ build_df_output,
+ build_efi_var_bytes,
+ build_ethtool_ip_address_output,
+ build_ethtool_ip_link_line,
+ build_ethtool_ip_link_show_line,
+ build_fdisk_output,
+ build_os_indications_var,
+ build_run_result_from_outcome,
+ build_sgdisk_partition_output,
+ check_output_router,
+ default_device_path,
+ noop,
+ normalize_ethtool_tool_path,
+ passthrough_router,
+ route_gateway,
+ run_router,
+ scenario_truthy,
+ which_router,
+)
+from .case_data_builders import CaseBuildError, render_post_check_path
+from .case_data_builders import expand_template as base_expand_template
+from .runner_checks import (
+ DEFAULT_CLI_TIMEOUT_SEC,
+ DESTRUCTIVE_TEST_ENV,
+ REAL_SUBPROCESS_RUN,
+ ConfigError,
+ CommandRunResult,
+ detect_project_root,
+ ensure_list,
+ ensure_list_of_strings,
+ ensure_string_or_list_of_strings,
+ RunCaseOptions,
+ SkipCase,
+ TestMeta,
+ TestOutcome,
+ create_outcome,
+ collect_function_names,
+ create_runner_temp_dir,
+ format_outcome_message,
+ load_yaml_config,
+ merge_mappings,
+ normalize_suites,
+ resolve_target_path,
+ run_single_check,
+ sanitize_name,
+ load_module_from_path,
+ normalize_completed_stream,
+ read_source,
+ sanitize_xml_text,
+)
+from .runner_reporting import (
+ append_combined_case_log,
+ append_run_header,
+ build_config_error_outcome,
+ build_report_path,
+ cleanup_old_pytest_xml_reports,
+ create_placeholder_xml,
+ print_group_summary,
+ remove_placeholder_xml,
+ write_case_log,
+ write_junit_xml,
+)
+__all__ = [
+ "MockLoaderConfigError",
+ "stateful_run_router",
+
+ "build_char16_payload",
+ "build_df_output",
+ "build_efi_var_bytes",
+ "build_ethtool_ip_address_output",
+ "build_ethtool_ip_link_line",
+ "build_ethtool_ip_link_show_line",
+ "build_fdisk_output",
+ "build_os_indications_var",
+ "build_run_result_from_outcome",
+ "build_sgdisk_partition_output",
+ "check_output_router",
+ "default_device_path",
+ "noop",
+ "normalize_ethtool_tool_path",
+ "passthrough_router",
+ "route_gateway",
+ "run_router",
+ "scenario_truthy",
+ "which_router",
+ "base_expand_template",
+
+ "CaseBuildError",
+ "render_post_check_path",
+
+ "DEFAULT_CLI_TIMEOUT_SEC",
+ "DESTRUCTIVE_TEST_ENV",
+ "REAL_SUBPROCESS_RUN",
+ "ConfigError",
+ "CommandRunResult",
+ "detect_project_root",
+ "ensure_list",
+ "ensure_list_of_strings",
+ "ensure_string_or_list_of_strings",
+ "RunCaseOptions",
+ "SkipCase",
+ "TestMeta",
+ "TestOutcome",
+ "create_outcome",
+ "collect_function_names",
+ "create_runner_temp_dir",
+ "format_outcome_message",
+ "load_yaml_config",
+ "merge_mappings",
+ "normalize_suites",
+ "resolve_target_path",
+ "run_single_check",
+ "sanitize_name",
+ "load_module_from_path",
+ "normalize_completed_stream",
+ "read_source",
+ "sanitize_xml_text",
+
+ "append_combined_case_log",
+ "append_run_header",
+ "build_config_error_outcome",
+ "build_report_path",
+ "cleanup_old_pytest_xml_reports",
+ "create_placeholder_xml",
+ "print_group_summary",
+ "remove_placeholder_xml",
+ "write_case_log",
+ "write_junit_xml",
+]
+
diff --git a/common/acs_test_framework_runner/case_data_builders.py b/common/acs_test_framework_runner/case_data_builders.py
new file mode 100644
index 00000000..070d81c1
--- /dev/null
+++ b/common/acs_test_framework_runner/case_data_builders.py
@@ -0,0 +1,180 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Mapping
+
+
+class CaseBuildError(Exception):
+ """Raised when case workspace data is invalid or cannot be materialized."""
+
+
+def expand_template(value: str, work_dir: Path, file_path: Path) -> str:
+ return (
+ value.replace("{dir}", str(work_dir))
+ .replace("{file}", str(file_path))
+ .replace("{filename}", file_path.name)
+ )
+
+
+def replace_dir_tokens(value: Any, temp_dir: Path) -> Any:
+ if isinstance(value, str):
+ return value.replace("{dir}", str(temp_dir))
+ if isinstance(value, list):
+ return [replace_dir_tokens(item, temp_dir) for item in value]
+ if isinstance(value, tuple):
+ return tuple(replace_dir_tokens(item, temp_dir) for item in value)
+ if isinstance(value, dict):
+ return {
+ key: replace_dir_tokens(item, temp_dir)
+ for key, item in value.items()
+ }
+ return value
+
+
+def build_runtime_case_for_tempdir(
+ case_def: Mapping[str, Any],
+ temp_dir: Path,
+) -> dict[str, Any]:
+ runtime_case = dict(case_def)
+ runtime_case["scripts"] = replace_dir_tokens(
+ runtime_case.get("scripts", {}),
+ temp_dir,
+ )
+ runtime_case["bin_files"] = replace_dir_tokens(
+ runtime_case.get("bin_files", {}),
+ temp_dir,
+ )
+ runtime_case["text_files"] = replace_dir_tokens(
+ runtime_case.get("text_files", {}),
+ temp_dir,
+ )
+ runtime_case["dir_structure"] = replace_dir_tokens(
+ runtime_case.get("dir_structure", []),
+ temp_dir,
+ )
+ runtime_case["patch_constants"] = replace_dir_tokens(
+ runtime_case.get("patch_constants", {}),
+ temp_dir,
+ )
+ return runtime_case
+
+
+def render_post_check_path(raw_path: str, work_dir: Path) -> Path:
+ return Path(raw_path.format(dir=str(work_dir)))
+
+
+def _write_text_if_changed(path: Path, content: str, *, executable: bool = False) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+
+ if path.exists():
+ try:
+ existing = path.read_text(encoding="utf-8")
+ if existing == content:
+ if executable:
+ path.chmod(0o755)
+ return
+ except Exception:
+ pass
+
+ path.write_text(content, encoding="utf-8")
+ if executable:
+ path.chmod(0o755)
+
+
+def _write_bytes_if_changed(path: Path, content: bytes) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+
+ if path.exists():
+ try:
+ existing = path.read_bytes()
+ if existing == content:
+ return
+ except Exception:
+ pass
+
+ path.write_bytes(content)
+
+
+def prepare_case_files(work_dir: Path, case_def: Mapping[str, Any]) -> None:
+ scripts = case_def.get("scripts", {})
+ if scripts is not None:
+ if not isinstance(scripts, dict):
+ raise CaseBuildError("'scripts' must be a mapping")
+ for name, content in scripts.items():
+ if not isinstance(name, str) or not isinstance(content, str):
+ raise CaseBuildError("'scripts' entries must be string -> string")
+ script_path = work_dir / name
+ _write_text_if_changed(script_path, content, executable=True)
+
+ bin_files = case_def.get("bin_files", {})
+ if bin_files is not None:
+ if not isinstance(bin_files, dict):
+ raise CaseBuildError("'bin_files' must be a mapping")
+ for name, spec in bin_files.items():
+ if not isinstance(name, str) or not isinstance(spec, dict):
+ raise CaseBuildError("'bin_files' entries must be string -> mapping")
+
+ target_file = work_dir / name
+ hex_data = spec.get("hex")
+ text_data = spec.get("text")
+
+ if hex_data is not None:
+ if not isinstance(hex_data, str):
+ raise CaseBuildError(
+ f"Invalid hex data type for bin_files entry '{name}'"
+ )
+ try:
+ payload = bytes.fromhex(hex_data)
+ except ValueError as exc:
+ raise CaseBuildError(
+ f"Invalid hex data for bin_files entry '{name}': {exc}"
+ ) from exc
+ _write_bytes_if_changed(target_file, payload)
+ elif text_data is not None:
+ if not isinstance(text_data, str):
+ raise CaseBuildError(
+ f"Invalid text data type for bin_files entry '{name}'"
+ )
+ _write_bytes_if_changed(target_file, text_data.encode("utf-8"))
+ else:
+ raise CaseBuildError(
+ "Each 'bin_files' entry requires string key 'hex' or 'text'"
+ )
+
+ text_files = case_def.get("text_files", {})
+ if text_files is not None:
+ if not isinstance(text_files, dict):
+ raise CaseBuildError("'text_files' must be a mapping")
+ for name, content in text_files.items():
+ if not isinstance(name, str) or not isinstance(content, str):
+ raise CaseBuildError("'text_files' entries must be string -> string")
+ target_file = work_dir / name
+ _write_text_if_changed(target_file, content)
+
+
+def materialize_case_workspace(
+ work_dir: Path,
+ case_def: Mapping[str, Any],
+) -> list[str]:
+ details: list[str] = []
+
+ dir_structure = case_def.get("dir_structure", [])
+ if dir_structure is None:
+ dir_structure = []
+ if not isinstance(dir_structure, list):
+ raise CaseBuildError("'dir_structure' must be a list")
+
+ for index, entry in enumerate(dir_structure, start=1):
+ if not isinstance(entry, dict):
+ raise CaseBuildError(f"'dir_structure[{index}]' must be a mapping")
+ rel_path = entry.get("path")
+ if not isinstance(rel_path, str) or not rel_path.strip():
+ raise CaseBuildError(
+ f"'dir_structure[{index}].path' must be a non-empty string"
+ )
+ dir_path = work_dir / rel_path
+ dir_path.mkdir(parents=True, exist_ok=True)
+ details.append(f"Created directory: {dir_path}")
+
+ prepare_case_files(work_dir, case_def)
+ return details
diff --git a/common/acs_test_framework_runner/mock_helpers.py b/common/acs_test_framework_runner/mock_helpers.py
new file mode 100644
index 00000000..ec302d2f
--- /dev/null
+++ b/common/acs_test_framework_runner/mock_helpers.py
@@ -0,0 +1,759 @@
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from subprocess import CalledProcessError, CompletedProcess
+from typing import Any
+from unittest.mock import Mock
+
+
+def noop(*args: Any, **kwargs: Any) -> None:
+ """Simple no-op helper for replacing sleeps and signal handlers."""
+ del args, kwargs
+
+
+def basic_mock(**kwargs: Any) -> Mock:
+ """
+ Generic Mock factory.
+
+ Example:
+ factory: mock_helpers.basic_mock
+ attrs:
+ return_value: 123
+ """
+ return Mock(**kwargs)
+
+
+def completed_process_mock(
+ stdout: str = "",
+ stderr: str = "",
+ returncode: int = 0,
+) -> Mock:
+ """
+ Mock for subprocess.run returning a CompletedProcess-like object.
+ """
+ mock_obj = Mock()
+ mock_obj.return_value = CompletedProcess(
+ args=[],
+ returncode=returncode,
+ stdout=stdout,
+ stderr=stderr,
+ )
+ return mock_obj
+
+
+def simple_check_output(output: str = "", as_bytes: bool = True):
+ """
+ Mock factory for subprocess.check_output with one fixed output.
+ """
+ def _mock_check_output(*args, **kwargs):
+ del args, kwargs
+ return output.encode() if as_bytes else output
+
+ return _mock_check_output
+
+
+def _pick_routed_value(value: Any, state: dict[str, int], key: str) -> Any:
+ """Return a routed value, supporting sticky sequential lists."""
+ if not isinstance(value, list):
+ return value
+ if not value:
+ raise ValueError("Router response lists must not be empty")
+ index = state.get(key, 0)
+ state[key] = index + 1
+ if index >= len(value):
+ return value[-1]
+ return value[index]
+
+
+def check_output_router(
+ responses: dict[str, str] | None = None,
+ errors: dict[str, str] | None = None,
+ default_output: str = "",
+ as_bytes: bool = True,
+ use_contains: bool = True,
+):
+ """
+ Route subprocess.check_output return values based on command text.
+
+ responses:
+ "ip link": "eth0\\neth1\\n"
+ "ethtool eth0": "Link detected: yes\\n"
+
+ errors:
+ "ethtool eth9": "No such device"
+ """
+ responses = responses or {}
+ errors = errors or {}
+ call_state: dict[str, int] = {}
+
+ def _match(pattern: str, cmd_text: str) -> bool:
+ if pattern.startswith("regex:"):
+ return re.search(pattern[6:], cmd_text) is not None
+ if use_contains:
+ return pattern in cmd_text
+ return pattern == cmd_text
+
+ def _mock_check_output(cmd, *args, **kwargs):
+ del args, kwargs
+ cmd_text = " ".join(cmd) if isinstance(cmd, (list, tuple)) else str(cmd)
+
+ for pattern, output in responses.items():
+ if _match(pattern, cmd_text):
+ selected = _pick_routed_value(output, call_state, pattern)
+ return selected.encode() if as_bytes else selected
+
+ for pattern, error_text in errors.items():
+ if _match(pattern, cmd_text):
+ selected = _pick_routed_value(error_text, call_state, f"error:{pattern}")
+ raise CalledProcessError(1, cmd, output=selected)
+
+ return default_output.encode() if as_bytes else default_output
+
+ return _mock_check_output
+
+
+def run_router(
+ responses: dict[str, Any] | None = None,
+ **options: Any,
+):
+ """
+ Route subprocess.run behavior based on command text.
+
+ responses format:
+ "lsblk": {"stdout": "sda\\n", "stderr": "", "returncode": 0}
+ "dd": {"stdout": "", "stderr": "write fail", "returncode": 1}
+ """
+ responses = responses or {}
+ default_stdout = str(options.pop("default_stdout", ""))
+ default_stderr = str(options.pop("default_stderr", ""))
+ default_returncode = int(options.pop("default_returncode", 0))
+ unmatched_stderr_template = options.pop("unmatched_stderr_template", None)
+ if unmatched_stderr_template is not None and not isinstance(
+ unmatched_stderr_template,
+ str,
+ ):
+ raise TypeError("'unmatched_stderr_template' must be a string")
+ use_contains = bool(options.pop("use_contains", True))
+ call_log_path = options.pop("call_log_path", None)
+ if call_log_path is not None and not isinstance(call_log_path, str):
+ raise TypeError("'call_log_path' must be a string")
+ if options:
+ unsupported = ", ".join(sorted(options))
+ raise TypeError(f"Unsupported run_router option(s): {unsupported}")
+
+ call_state: dict[str, int] = {}
+
+ def _match(pattern: str, cmd_text: str) -> bool:
+ if pattern.startswith("regex:"):
+ return re.search(pattern[6:], cmd_text) is not None
+ if use_contains:
+ return pattern in cmd_text
+ return pattern == cmd_text
+
+ def _mock_run(cmd, *args, **kwargs):
+ del args, kwargs
+ cmd_text = " ".join(cmd) if isinstance(cmd, (list, tuple)) else str(cmd)
+
+ if call_log_path is not None:
+ log_path = Path(call_log_path)
+ log_path.parent.mkdir(parents=True, exist_ok=True)
+ with log_path.open("a", encoding="utf-8") as handle:
+ handle.write(f"{cmd_text}\n")
+
+ for pattern, result in responses.items():
+ if _match(pattern, cmd_text):
+ resolved = _pick_routed_value(result, call_state, pattern)
+ if not isinstance(resolved, dict):
+ raise TypeError("run_router responses must resolve to mappings")
+ return CompletedProcess(
+ args=cmd,
+ returncode=resolved.get("returncode", 0),
+ stdout=resolved.get("stdout", ""),
+ stderr=resolved.get("stderr", ""),
+ )
+
+ stderr_text = default_stderr
+ if unmatched_stderr_template is not None:
+ stderr_text = unmatched_stderr_template.format(cmd=cmd_text)
+
+ return CompletedProcess(
+ args=cmd,
+ returncode=default_returncode,
+ stdout=default_stdout,
+ stderr=stderr_text,
+ )
+
+ return _mock_run
+
+
+def path_read_text_router(
+ responses: dict[str, str] | None = None,
+ default_text: str = "",
+ use_contains: bool = True,
+):
+ """
+ Router for Path.read_text() based on path string.
+
+ responses:
+ "/sys/class/net/eth0/speed": "1000"
+ """
+ responses = responses or {}
+
+ def _match(pattern: str, path_text: str) -> bool:
+ if pattern.startswith("regex:"):
+ return re.search(pattern[6:], path_text) is not None
+ if use_contains:
+ return pattern in path_text
+ return pattern == path_text
+
+ def _mock_read_text(self, *args, **kwargs):
+ del args, kwargs
+ path_text = str(self)
+
+ for pattern, text in responses.items():
+ if _match(pattern, path_text):
+ return text
+
+ return default_text
+
+ return _mock_read_text
+
+
+def which_return(value: str | None):
+ """
+ Mock shutil.which return value.
+ """
+ mock_obj = Mock()
+ mock_obj.return_value = value
+ return mock_obj
+
+
+def which_router(
+ responses: dict[str, str | None] | None = None,
+ default: str | None = None,
+):
+ """Route shutil.which(...) results by tool name."""
+ responses = responses or {}
+
+ def _mock_which(name: str, *args, **kwargs):
+ del args, kwargs
+ return responses.get(name, default)
+
+ return _mock_which
+
+
+def scenario_truthy(value: Any, *, default: bool = False) -> bool:
+ """Coerce common YAML-ish truthy values used in scenario definitions."""
+ if value is None:
+ return default
+ if isinstance(value, bool):
+ return value
+ if isinstance(value, str):
+ return value.strip().lower() in {
+ "1",
+ "true",
+ "yes",
+ "present",
+ "up",
+ "pass",
+ "passed",
+ "on",
+ }
+ return bool(value)
+
+
+def normalize_ethtool_tool_path(tool_name: str, raw_value: Any) -> str | None:
+ """Normalize a scenario tool entry into a concrete which() return value."""
+ if raw_value is None:
+ return None
+
+ default_path = "/usr/sbin/ethtool" if tool_name == "ethtool" else f"/usr/bin/{tool_name}"
+ if isinstance(raw_value, bool):
+ return default_path if raw_value else None
+ if not isinstance(raw_value, str):
+ raise TypeError(
+ f"scenario.tools['{tool_name}'] must be a string or boolean, "
+ f"got {type(raw_value).__name__}"
+ )
+
+ lowered = raw_value.strip().lower()
+ if lowered in {"present", "yes", "true"}:
+ return default_path
+ if lowered in {"absent", "no", "false", ""}:
+ return None
+ return raw_value
+
+
+def normalize_ip_entries(
+ value: Any,
+ *,
+ default_dynamic: bool = False,
+) -> list[dict[str, Any]]:
+ """Normalize IPv4/IPv6 scenario entries into a list of address mappings."""
+ if value is None:
+ return []
+
+ raw_items = [value] if isinstance(value, (str, dict)) else value
+ if not isinstance(raw_items, list):
+ raise TypeError("scenario ipv4/ipv6 entries must be a list, string, or mapping")
+
+ entries: list[dict[str, Any]] = []
+ for item in raw_items:
+ if isinstance(item, str):
+ entries.append({"address": item, "dynamic": default_dynamic})
+ continue
+ if not isinstance(item, dict):
+ raise TypeError("scenario ipv4/ipv6 entries must be strings or mappings")
+ address = item.get("address")
+ if not isinstance(address, str):
+ raise TypeError("scenario ip entries require a string 'address'")
+ entries.append(
+ {
+ "address": address,
+ "dynamic": scenario_truthy(item.get("dynamic"), default=default_dynamic),
+ }
+ )
+ return entries
+
+
+def build_ip_link_flags(kind: str, state: str) -> str:
+ """Render the flags field used by `ip link` output."""
+ flags = ["LOOPBACK"] if kind == "loopback" else ["BROADCAST", "MULTICAST"]
+ if state.lower() == "up":
+ flags.append("UP")
+ return ",".join(flags)
+
+
+def build_ethtool_ip_link_line(index: int, iface: dict[str, Any]) -> str:
+ """Render one line from `ip -o link` for an ethtool scenario interface."""
+ name = iface["name"]
+ kind = iface.get("kind", "physical")
+ state = str(iface.get("state", "down")).upper()
+ mtu = int(iface.get("mtu", 65536 if kind == "loopback" else 1500))
+ flags = build_ip_link_flags(kind, state)
+
+ if kind == "loopback":
+ return (
+ f"{index}: {name}: <{flags}> mtu {mtu} qdisc noop "
+ f"state {state} mode DEFAULT group default qlen 1000"
+ )
+
+ mac = iface.get("mac", f"02:00:00:00:00:{index:02x}")
+ return (
+ f"{index}: {name}: <{flags}> mtu {mtu} qdisc noop state {state} "
+ f"mode DEFAULT group default qlen 1000 link/ether {mac}"
+ )
+
+
+def build_ethtool_ip_link_show_line(index: int, iface: dict[str, Any]) -> str:
+ """Render one line from `ip link show `."""
+ name = iface["name"]
+ kind = iface.get("kind", "physical")
+ state = str(iface.get("state", "down")).upper()
+ mtu = int(iface.get("mtu", 65536 if kind == "loopback" else 1500))
+ flags = build_ip_link_flags(kind, state)
+ return f"{index}: {name}: <{flags}> mtu {mtu} state {state}\n"
+
+
+def build_ethtool_ip_address_output(index: int, iface: dict[str, Any]) -> str:
+ """Render `ip address show dev ` output for a scenario interface."""
+ name = iface["name"]
+ kind = iface.get("kind", "physical")
+ state = str(iface.get("state", "down")).upper()
+ mtu = int(iface.get("mtu", 65536 if kind == "loopback" else 1500))
+ flags = build_ip_link_flags(kind, state)
+
+ lines = [f"{index}: {name}: <{flags}> mtu {mtu}"]
+ ipv4_entries = normalize_ip_entries(
+ iface.get("ipv4") or (iface.get("addresses") or {}).get("ipv4"),
+ default_dynamic=False,
+ )
+ ipv6_entries = normalize_ip_entries(
+ iface.get("ipv6") or (iface.get("addresses") or {}).get("ipv6"),
+ default_dynamic=False,
+ )
+
+ for entry in ipv4_entries:
+ dynamic_text = " dynamic" if entry["dynamic"] else ""
+ lines.append(f" inet {entry['address']} scope global{dynamic_text} {name}")
+ for entry in ipv6_entries:
+ lines.append(f" inet6 {entry['address']} scope global")
+ return "\n".join(lines) + "\n"
+
+
+def default_device_path(index: int, iface: dict[str, Any]) -> str:
+ """Build a default resolved sysfs device path for an interface."""
+ kind = iface.get("kind", "physical")
+ name = iface["name"]
+ if kind == "virtual":
+ return f"/sys/devices/virtual/net/{name}"
+ return f"/sys/devices/pci0000:00/0000:00:{index:02x}.0/net/{name}"
+
+
+def route_gateway(route_line: str | None) -> str | None:
+ """Extract the IPv4 gateway address from a route output line."""
+ if not route_line:
+ return None
+ match = re.search(r"\bvia\s+(\d{1,3}(?:\.\d{1,3}){3})", route_line)
+ return match.group(1) if match else None
+
+
+def build_run_result_from_outcome(
+ outcome: Any,
+ *,
+ success_stdout: str = "",
+ failure_stdout: str = "",
+ failure_stderr: str = "",
+) -> dict[str, Any]:
+ """Convert a simple pass/fail scenario outcome into run_router response data."""
+ if isinstance(outcome, dict):
+ return {
+ "returncode": outcome.get("returncode", 0),
+ "stdout": outcome.get("stdout", ""),
+ "stderr": outcome.get("stderr", ""),
+ }
+
+ outcome_text = str(outcome).strip().lower()
+ if outcome_text in {"pass", "passed", "ok", "success"}:
+ return {"returncode": 0, "stdout": success_stdout, "stderr": ""}
+ if outcome_text in {"warn", "warning", "fail", "failed"}:
+ return {
+ "returncode": 1,
+ "stdout": failure_stdout,
+ "stderr": failure_stderr,
+ }
+ raise ValueError(f"Unsupported connectivity outcome: {outcome!r}")
+
+
+def build_df_output(used_blocks: int, available_blocks: int) -> str:
+ """Render `df -B 512 --output=used,avail` output."""
+ return f"Used Avail\n{used_blocks} {available_blocks}\n"
+
+
+def build_fdisk_output(disk: str, partitions: list[dict[str, Any]]) -> str:
+ """Render minimal `fdisk -l` output that the parser can consume."""
+ lines = [
+ f"Disk /dev/{disk}: 16 GiB",
+ "Device Boot Start End Sectors Size Id Type",
+ ]
+
+ for index, part in enumerate(partitions, start=1):
+ name = part.get("name", f"{disk}{index}")
+ boot_flag = "*" if scenario_truthy(part.get("boot"), default=False) else ""
+ start = int(part.get("start", 2048))
+ sectors = int(part.get("sectors", 204800))
+ end = int(part.get("end", start + sectors - 1))
+ size = str(part.get("size", "100M"))
+ part_id = str(part.get("id", "83")).upper().removeprefix("0X")
+ type_name = str(part.get("type_name", "Linux"))
+ prefix = f"/dev/{name}"
+ if boot_flag:
+ lines.append(
+ f"{prefix} {boot_flag} {start} {end} {sectors} {size} {part_id} {type_name}"
+ )
+ else:
+ lines.append(f"{prefix} {start} {end} {sectors} {size} {part_id} {type_name}")
+
+ return "\n".join(lines) + "\n"
+
+
+def build_sgdisk_partition_output(partition: dict[str, Any]) -> str:
+ """Render minimal `sgdisk -i=N` output for one GPT partition."""
+ guid = str(
+ partition.get(
+ "guid",
+ "0FC63DAF-8483-4772-8E79-3D69D8477DE4",
+ )
+ ).upper()
+ guid_name = str(partition.get("guid_name", "Linux filesystem"))
+ if "attribute_flags" in partition:
+ flags = str(partition["attribute_flags"])
+ else:
+ flags = "0000000000000001" if scenario_truthy(
+ partition.get("platform_required"),
+ default=False,
+ ) else "0000000000000000"
+
+ return (
+ f"Partition GUID code: {guid} ({guid_name})\n"
+ f"Attribute flags: {flags}\n"
+ )
+
+
+def build_efi_var_bytes(attrs: int, payload: bytes = b"") -> bytes:
+ """Build raw efivarfs contents: 4-byte LE attrs followed by payload bytes."""
+ if attrs < 0:
+ raise ValueError("EFI variable attributes must be non-negative")
+ return int(attrs).to_bytes(4, byteorder="little", signed=False) + payload
+
+
+def build_char16_payload(value: str, extra_utf16: str = "") -> bytes:
+ """Encode a UEFI CHAR16 payload in UTF-16LE."""
+ return (value + extra_utf16).encode("utf-16le")
+
+
+def build_os_indications_var(
+ value: int,
+ *,
+ attrs: int = 0x07,
+) -> bytes:
+ """Build the OsIndicationsSupported efivarfs payload."""
+ if value < 0:
+ raise ValueError("OsIndicationsSupported value must be non-negative")
+ payload = int(value).to_bytes(8, byteorder="little", signed=False)
+ return build_efi_var_bytes(attrs, payload)
+
+
+def _match_rule_value(expected: Any, actual: Any) -> bool:
+ """Match a rule value against the actual call argument."""
+ if isinstance(expected, dict):
+ if "equals" in expected:
+ return actual == expected["equals"]
+ if "contains" in expected:
+ return str(expected["contains"]) in str(actual)
+ if "regex" in expected:
+ return re.search(str(expected["regex"]), str(actual)) is not None
+ if "in" in expected:
+ options = expected["in"]
+ if not isinstance(options, list):
+ raise TypeError("'in' matcher requires a list")
+ return actual in options
+ raise ValueError(f"Unsupported matcher spec: {expected!r}")
+
+ return actual == expected
+
+
+def _rule_matches(
+ when: dict[str, Any] | None,
+ args: tuple[Any, ...],
+ kwargs: dict[str, Any],
+) -> bool:
+ """Return True when the rule selector matches the current call."""
+ if when is None:
+ return True
+ if not isinstance(when, dict):
+ raise TypeError("'when' must be a mapping")
+
+ arg_rules = when.get("args", {})
+ if arg_rules is None:
+ arg_rules = {}
+ if not isinstance(arg_rules, dict):
+ raise TypeError("'when.args' must be a mapping")
+
+ for index, expected in arg_rules.items():
+ if not isinstance(index, int):
+ raise TypeError("'when.args' keys must be integers")
+ if index >= len(args) or not _match_rule_value(expected, args[index]):
+ return False
+
+ kwarg_rules = when.get("kwargs", {})
+ if kwarg_rules is None:
+ kwarg_rules = {}
+ if not isinstance(kwarg_rules, dict):
+ raise TypeError("'when.kwargs' must be a mapping")
+
+ for key, expected in kwarg_rules.items():
+ if key not in kwargs or not _match_rule_value(expected, kwargs[key]):
+ return False
+
+ return True
+
+
+def _build_exception(spec: Any) -> BaseException:
+ """Build an exception from a passthrough router rule."""
+ if isinstance(spec, BaseException):
+ return spec
+
+ if isinstance(spec, type) and issubclass(spec, BaseException):
+ return spec()
+
+ if not isinstance(spec, dict):
+ raise TypeError(
+ "Exception rule must be an exception instance, exception class, "
+ f"or mapping. Got: {type(spec).__name__}"
+ )
+
+ exc_type = spec.get("type")
+ args = spec.get("args", [])
+ kwargs = spec.get("kwargs", {})
+
+ if not isinstance(exc_type, type) or not issubclass(exc_type, BaseException):
+ raise TypeError("'raise.type' must be an exception class")
+ if not isinstance(args, list):
+ raise TypeError("'raise.args' must be a list")
+ if not isinstance(kwargs, dict):
+ raise TypeError("'raise.kwargs' must be a dict")
+
+ return exc_type(*args, **kwargs)
+
+
+class MockExpectationError(AssertionError):
+ """Raised when a declarative mock rule was expected to match but did not."""
+
+
+def _rule_label(rule: dict[str, Any], index: int) -> str:
+ """Return a stable human-readable label for a passthrough rule."""
+ label = rule.get("label")
+ if isinstance(label, str) and label.strip():
+ return label.strip()
+ return f"rule[{index + 1}]"
+
+
+def _validated_call_count(
+ value: Any,
+ *,
+ field_name: str,
+ rule_name: str,
+) -> int | None:
+ """Validate an optional per-rule call count constraint."""
+ if value is None:
+ return None
+ if not isinstance(value, int) or value < 0:
+ raise TypeError(
+ f"'{field_name}' for {rule_name} must be a non-negative integer"
+ )
+ return value
+
+
+def passthrough_router(
+ real: Any,
+ rules: list[dict[str, Any]] | None = None,
+ *,
+ label: str | None = None,
+ require_any_rule_hit: bool = False,
+):
+ """
+ Wrap a real callable and apply declarative YAML-friendly overrides.
+
+ Example rule:
+ - when:
+ args:
+ 0: "/tmp/event.yaml"
+ 1:
+ contains: "r"
+ raise:
+ type: OSError
+ args: ["mocked open failure"]
+ """
+ rules = rules or []
+ if not isinstance(rules, list):
+ raise TypeError("'rules' must be a list")
+ if label is not None and (not isinstance(label, str) or not label.strip()):
+ raise TypeError("'label' must be a non-empty string when provided")
+ if not isinstance(require_any_rule_hit, bool):
+ raise TypeError("'require_any_rule_hit' must be a boolean")
+
+ normalized_rules: list[dict[str, Any]] = []
+ for index, rule in enumerate(rules):
+ if not isinstance(rule, dict):
+ raise TypeError("Each passthrough rule must be a mapping")
+
+ rule_name = _rule_label(rule, index)
+ required = rule.get("required", False)
+ if not isinstance(required, bool):
+ raise TypeError(f"'required' for {rule_name} must be a boolean")
+
+ exact_calls = _validated_call_count(
+ rule.get("exact_calls"),
+ field_name="exact_calls",
+ rule_name=rule_name,
+ )
+ min_calls = _validated_call_count(
+ rule.get("min_calls"),
+ field_name="min_calls",
+ rule_name=rule_name,
+ )
+ max_calls = _validated_call_count(
+ rule.get("max_calls"),
+ field_name="max_calls",
+ rule_name=rule_name,
+ )
+ if exact_calls is not None and (min_calls is not None or max_calls is not None):
+ raise TypeError(
+ f"{rule_name} cannot define 'exact_calls' together with "
+ "'min_calls' or 'max_calls'"
+ )
+ if (
+ min_calls is not None
+ and max_calls is not None
+ and min_calls > max_calls
+ ):
+ raise TypeError(
+ f"{rule_name} has invalid call constraints: min_calls > max_calls"
+ )
+
+ normalized_rules.append(
+ {
+ "raw": rule,
+ "name": rule_name,
+ "required": required,
+ "exact_calls": exact_calls,
+ "min_calls": min_calls,
+ "max_calls": max_calls,
+ "hits": 0,
+ }
+ )
+
+ router_label = label.strip() if isinstance(label, str) else None
+
+ def _wrapped(*args, **kwargs):
+ for rule_info in normalized_rules:
+ rule = rule_info["raw"]
+ if not _rule_matches(rule.get("when"), args, kwargs):
+ continue
+ rule_info["hits"] += 1
+
+ if "raise" in rule:
+ raise _build_exception(rule["raise"])
+ if "return" in rule:
+ return rule["return"]
+ if rule.get("call_real", True):
+ return real(*args, **kwargs)
+ return None
+
+ return real(*args, **kwargs)
+
+ def _verify() -> None:
+ failures: list[str] = []
+ total_hits = sum(int(rule["hits"]) for rule in normalized_rules)
+
+ if require_any_rule_hit and total_hits == 0:
+ failures.append("expected at least one passthrough rule to match")
+
+ for rule_info in normalized_rules:
+ hits = int(rule_info["hits"])
+ rule_name = str(rule_info["name"])
+ exact_calls = rule_info["exact_calls"]
+ min_calls = rule_info["min_calls"]
+ max_calls = rule_info["max_calls"]
+
+ if rule_info["required"] and hits == 0:
+ failures.append(f"{rule_name} was never hit")
+ if exact_calls is not None and hits != exact_calls:
+ failures.append(
+ f"{rule_name} expected exactly {exact_calls} hit(s), got {hits}"
+ )
+ if min_calls is not None and hits < min_calls:
+ failures.append(
+ f"{rule_name} expected at least {min_calls} hit(s), got {hits}"
+ )
+ if max_calls is not None and hits > max_calls:
+ failures.append(
+ f"{rule_name} expected at most {max_calls} hit(s), got {hits}"
+ )
+
+ if failures:
+ hit_summary = ", ".join(
+ f"{rule_info['name']}={rule_info['hits']}"
+ for rule_info in normalized_rules
+ )
+ prefix = f"{router_label}: " if router_label else ""
+ if not hit_summary:
+ hit_summary = ""
+ raise MockExpectationError(
+ f"{prefix}{'; '.join(failures)}. Hit counts: {hit_summary}"
+ )
+
+ setattr(_wrapped, "_mock_verify", _verify)
+ return _wrapped
diff --git a/common/acs_test_framework_runner/mock_loader.py b/common/acs_test_framework_runner/mock_loader.py
new file mode 100644
index 00000000..58f50a70
--- /dev/null
+++ b/common/acs_test_framework_runner/mock_loader.py
@@ -0,0 +1,749 @@
+from __future__ import annotations
+
+import importlib
+import inspect
+import types
+from contextlib import ExitStack
+from pathlib import Path
+from subprocess import CompletedProcess
+from typing import Any
+from typing import Callable
+from typing import Mapping
+from unittest.mock import Mock
+from unittest.mock import patch
+
+try: # Support package imports and direct harness module loading.
+ from .case_data_builders import expand_template as base_expand_template
+except ImportError: # pragma: no cover - exercised by flat-module harness imports.
+ from case_data_builders import expand_template as base_expand_template
+
+
+
+def _stateful_read_content(spec: Any, state: dict[str, Any], work_dir: Path) -> bytes:
+ if isinstance(spec, bytes):
+ return spec
+ if isinstance(spec, str):
+ return spec.encode("utf-8")
+ if not isinstance(spec, dict):
+ raise TypeError("Stateful content spec must be bytes, string, or mapping")
+ if "from_state" in spec:
+ value = state.get(str(spec["from_state"]), b"")
+ if isinstance(value, bytes):
+ return value
+ if isinstance(value, str):
+ return value.encode("utf-8")
+ raise TypeError("State value for content must be bytes or string")
+ if "from_file" in spec:
+ file_path = work_dir / str(spec["from_file"])
+ return file_path.read_bytes()
+ if "hex" in spec:
+ return bytes.fromhex(str(spec["hex"]))
+ raise ValueError(f"Unsupported stateful content spec: {spec!r}")
+
+
+def _stateful_resolve_value(spec: Any, state: dict[str, Any], work_dir: Path) -> Any:
+ if isinstance(spec, dict):
+ if "from_state" in spec:
+ return state.get(str(spec["from_state"]))
+ if "from_file" in spec or "hex" in spec:
+ return _stateful_read_content(spec, state, work_dir)
+ return spec
+
+
+def stateful_run_router(
+ responses: dict[str, Any] | None = None,
+ **options: Any,
+):
+ """Stateful subprocess.run router for scenario-specific command flows."""
+ state = dict(options.pop("state", {}) or {})
+ rules = list(options.pop("rules", []) or [])
+ responses = dict(responses or {})
+ default_stdout = str(options.pop("default_stdout", ""))
+ default_stderr = str(options.pop("default_stderr", ""))
+ default_returncode = int(options.pop("default_returncode", 0))
+ unmatched_stderr_template = options.pop("unmatched_stderr_template", None)
+ if unmatched_stderr_template is not None and not isinstance(
+ unmatched_stderr_template,
+ str,
+ ):
+ raise TypeError("'unmatched_stderr_template' must be a string")
+ use_contains = bool(options.pop("use_contains", True))
+ call_log_path = options.pop("call_log_path", None)
+ if call_log_path is not None and not isinstance(call_log_path, str):
+ raise TypeError("'call_log_path' must be a string")
+ if options:
+ unsupported = ", ".join(sorted(options))
+ raise TypeError(f"Unsupported stateful_run_router option(s): {unsupported}")
+
+ response_state: dict[str, int] = {}
+
+ def _match(pattern: str, cmd_text: str) -> bool:
+ if use_contains:
+ return pattern in cmd_text
+ return pattern == cmd_text
+
+ def _mock_run(cmd, *args, **kwargs):
+ del args, kwargs
+ cmd_text = " ".join(cmd) if isinstance(cmd, (list, tuple)) else str(cmd)
+
+ if call_log_path is not None:
+ log_path = Path(call_log_path)
+ log_path.parent.mkdir(parents=True, exist_ok=True)
+ with log_path.open("a", encoding="utf-8") as handle:
+ handle.write(f"{cmd_text}\n")
+
+ for rule in rules:
+ if not isinstance(rule, dict):
+ raise TypeError("Each stateful_run_router rule must be a mapping")
+ pattern = rule.get("command")
+ if not isinstance(pattern, str):
+ raise TypeError("stateful_run_router rule requires string 'command'")
+ if not _match(pattern, cmd_text):
+ continue
+
+ for key, value_spec in (rule.get("set_state") or {}).items():
+ state[str(key)] = _stateful_resolve_value(value_spec, state, Path.cwd())
+
+ write_file = rule.get("write_file")
+ if write_file is not None:
+ if (
+ not isinstance(write_file, dict)
+ or "path" not in write_file
+ or "content" not in write_file
+ ):
+ raise TypeError("write_file must be a mapping with 'path' and 'content'")
+ target = Path.cwd() / str(write_file["path"])
+ target.parent.mkdir(parents=True, exist_ok=True)
+ target.write_bytes(
+ _stateful_read_content(
+ write_file["content"],
+ state,
+ Path.cwd(),
+ )
+ )
+
+ if "raise" in rule:
+ raise build_exception_from_spec(rule["raise"])
+
+ result = rule.get("result", {})
+ if not isinstance(result, dict):
+ raise TypeError("stateful_run_router rule 'result' must be a mapping")
+ return CompletedProcess(
+ args=cmd,
+ returncode=int(result.get("returncode", 0)),
+ stdout=str(result.get("stdout", "")),
+ stderr=str(result.get("stderr", "")),
+ )
+
+ for pattern, result in responses.items():
+ if not _match(pattern, cmd_text):
+ continue
+ resolved = result
+ if isinstance(result, list):
+ if not result:
+ raise TypeError("stateful_run_router response lists must not be empty")
+ index = response_state.get(pattern, 0)
+ response_state[pattern] = index + 1
+ resolved = result[index] if index < len(result) else result[-1]
+ if not isinstance(resolved, dict):
+ raise TypeError("stateful_run_router responses must resolve to mappings")
+ return CompletedProcess(
+ args=cmd,
+ returncode=int(resolved.get("returncode", 0)),
+ stdout=str(resolved.get("stdout", "")),
+ stderr=str(resolved.get("stderr", "")),
+ )
+
+ stderr_text = default_stderr
+ if unmatched_stderr_template is not None:
+ stderr_text = unmatched_stderr_template.format(cmd=cmd_text)
+
+ return CompletedProcess(
+ args=cmd,
+ returncode=default_returncode,
+ stdout=default_stdout,
+ stderr=stderr_text,
+ )
+
+ return _mock_run
+
+
+class ConfigError(Exception):
+ """Raised when a YAML configuration is invalid."""
+
+
+def load_dotted_object(dotted_path: str) -> Any:
+ """Load object from dotted path like 'pkg.mod.func' or 'pkg.mod.obj.attr'."""
+ if not isinstance(dotted_path, str) or "." not in dotted_path:
+ raise ValueError(f"Invalid dotted path: {dotted_path!r}")
+
+ parts = dotted_path.split(".")
+ last_error: Exception | None = None
+
+ for index in range(len(parts) - 1, 0, -1):
+ module_name = ".".join(parts[:index])
+ attr_parts = parts[index:]
+
+ try:
+ obj = importlib.import_module(module_name)
+ except ImportError as exc:
+ last_error = exc
+ continue
+
+ try:
+ for attr_name in attr_parts:
+ obj = getattr(obj, attr_name)
+ return obj
+ except AttributeError as exc:
+ raise AttributeError(f"Invalid dotted path: {dotted_path!r}") from exc
+
+ raise ValueError(f"Invalid dotted path: {dotted_path!r}") from last_error
+
+
+def resolve_special_value(value: Any) -> Any:
+ """
+ Resolve special YAML values.
+
+ Supported:
+ - 'py:package.module.object' -> imported Python object
+ """
+ if isinstance(value, str) and value.startswith("py:"):
+ return load_dotted_object(value[3:])
+ if isinstance(value, list):
+ return [resolve_special_value(item) for item in value]
+ if isinstance(value, tuple):
+ return tuple(resolve_special_value(item) for item in value)
+ if isinstance(value, dict):
+ return {
+ key: resolve_special_value(item)
+ for key, item in value.items()
+ }
+ return value
+
+
+def build_exception_from_spec(spec: Any) -> BaseException:
+ """
+ Build an exception instance from YAML spec.
+
+ Supported forms:
+ - py:builtins.OSError
+ - {type: py:builtins.OSError, args: ["message"]}
+ - an already-created exception instance
+ """
+ resolved = resolve_special_value(spec)
+
+ if isinstance(resolved, BaseException):
+ return resolved
+
+ if isinstance(resolved, type) and issubclass(resolved, BaseException):
+ return resolved()
+
+ if not isinstance(resolved, dict):
+ raise TypeError(
+ "Exception spec must resolve to an exception instance, exception "
+ f"class, or mapping. Got: {type(resolved).__name__}"
+ )
+
+ exc_type = resolved.get("type")
+ if not isinstance(exc_type, type) or not issubclass(exc_type, BaseException):
+ raise TypeError(
+ "Exception spec 'type' must resolve to an exception class, "
+ f"got: {exc_type!r}"
+ )
+
+ args = resolved.get("args", [])
+ kwargs = resolved.get("kwargs", {})
+
+ if not isinstance(args, list):
+ raise TypeError(
+ f"Exception spec 'args' must be a list, got: {type(args).__name__}"
+ )
+ if not isinstance(kwargs, dict):
+ raise TypeError(
+ f"Exception spec 'kwargs' must be a dict, got: {type(kwargs).__name__}"
+ )
+
+ return exc_type(*args, **kwargs)
+
+
+def resolve_side_effect_value(value: Any) -> Any:
+ """Resolve side-effect values, including exception descriptors."""
+ if isinstance(value, list):
+ return [resolve_side_effect_value(item) for item in value]
+
+ if isinstance(value, dict) and set(value.keys()) == {"exception"}:
+ return build_exception_from_spec(value["exception"])
+
+ return resolve_special_value(value)
+
+
+def set_nested_attr(obj: Any, attr_path: str, value: Any) -> None:
+ """
+ Set nested attribute path like:
+ return_value.stdout
+ return_value.returncode
+ """
+ parts = attr_path.split(".")
+ current = obj
+
+ for part in parts[:-1]:
+ current = getattr(current, part)
+
+ setattr(current, parts[-1], value)
+
+
+def build_mock_from_spec(target: str, spec: Any) -> Any:
+ """
+ Build replacement object from YAML spec.
+
+ Supported forms:
+
+ 1) String shorthand:
+ mocks:
+ shutil.which: unittest.mock.Mock
+
+ 2) Dict form:
+ mocks:
+ subprocess.run:
+ factory: unittest.mock.Mock
+ kwargs:
+ name: run_mock
+ attrs:
+ return_value.stdout: "hello"
+ return_value.returncode: 0
+
+ 3) Implicit Mock with direct behavior:
+ mocks:
+ yaml.safe_load:
+ side_effect:
+ - {"sha256": ["a0", "a1"]}
+ - exception:
+ type: py:yaml.YAMLError
+ args: ["mocked parse failure"]
+
+ 4) Factory that needs the original callable:
+ mocks:
+ builtins.open:
+ factory: mock_helpers.passthrough_router
+ inject_original_as: real
+ kwargs:
+ rules: [...]
+
+ 5) Direct callable factory (used by generated scenarios):
+ mocks:
+ time.sleep:
+ value:
+ """
+ if isinstance(spec, str):
+ factory = load_dotted_object(spec)
+ return factory()
+
+ if not isinstance(spec, dict):
+ raise TypeError(f"Unsupported mock spec type: {type(spec).__name__}")
+
+ factory_ref = spec.get("factory")
+ if (
+ factory_ref is not None
+ and not isinstance(factory_ref, str)
+ and not callable(factory_ref)
+ ):
+ raise TypeError(
+ "'factory' must be a string or callable when provided, "
+ f"got: {type(factory_ref).__name__}"
+ )
+
+ if "value" in spec:
+ return resolve_special_value(spec["value"])
+
+ original = load_dotted_object(target)
+
+ raw_args = spec.get("args", [])
+ if raw_args is None:
+ raw_args = []
+ if not isinstance(raw_args, list):
+ raise TypeError(f"'args' must be a list, got: {type(raw_args).__name__}")
+
+ raw_kwargs = spec.get("kwargs", {})
+ if raw_kwargs is None:
+ raw_kwargs = {}
+ if not isinstance(raw_kwargs, dict):
+ raise TypeError(f"'kwargs' must be a dict, got: {type(raw_kwargs).__name__}")
+
+ args = [resolve_special_value(value) for value in raw_args]
+ kwargs = {key: resolve_special_value(value) for key, value in raw_kwargs.items()}
+
+ inject_original_as = spec.get("inject_original_as")
+ if inject_original_as is not None:
+ if not isinstance(inject_original_as, str) or not inject_original_as.strip():
+ raise TypeError("'inject_original_as' must be a non-empty string")
+ kwargs[inject_original_as] = original
+
+ if spec.get("wraps_original", False):
+ kwargs["wraps"] = original
+
+ if factory_ref:
+ if isinstance(factory_ref, str):
+ factory = load_dotted_object(factory_ref)
+ else:
+ factory = factory_ref
+ else:
+ factory = Mock
+
+ obj = factory(*args, **kwargs)
+
+ raw_attrs = spec.get("attrs", {})
+ if raw_attrs is None:
+ raw_attrs = {}
+
+ if not isinstance(raw_attrs, dict):
+ raise TypeError(f"'attrs' must be a dict, got: {type(raw_attrs).__name__}")
+
+ if "return_value" in spec:
+ obj.return_value = resolve_special_value(spec["return_value"])
+
+ if "side_effect" in spec:
+ obj.side_effect = resolve_side_effect_value(spec["side_effect"])
+
+ for attr_name, raw_value in raw_attrs.items():
+ value = resolve_special_value(raw_value)
+ set_nested_attr(obj, attr_name, value)
+
+ return obj
+
+
+def expand_patch_target(
+ target: str,
+ target_context: Mapping[str, str] | None = None,
+) -> str:
+ """Expand supported placeholders such as '{module}' in patch targets."""
+ expanded = target
+ if not target_context:
+ return expanded
+
+ for key, value in target_context.items():
+ expanded = expanded.replace(f"{{{key}}}", str(value))
+ return expanded
+
+
+def apply_case_mocks(
+ mock_map: dict[str, Any] | None,
+ *,
+ target_context: Mapping[str, str] | None = None,
+) -> ExitStack:
+ """
+ Apply case mocks and return an ExitStack.
+
+ Caller should use:
+ with apply_case_mocks(case.get("mocks")):
+ ...
+ """
+ stack = ExitStack()
+
+ if not mock_map:
+ return stack
+
+ if not isinstance(mock_map, dict):
+ raise TypeError(f"'mocks' must be a dict, got: {type(mock_map).__name__}")
+
+ for target, spec in mock_map.items():
+ if not isinstance(target, str):
+ raise TypeError(
+ f"Mock target must be a string, got: {type(target).__name__}"
+ )
+ expanded_target = expand_patch_target(target, target_context)
+ replacement = build_mock_from_spec(expanded_target, spec)
+ verifier = getattr(replacement, "_mock_verify", None)
+ if callable(verifier):
+ stack.callback(verifier)
+ stack.enter_context(patch(expanded_target, replacement))
+
+ return stack
+
+
+def expand_template(
+ value: str,
+ work_dir: Path,
+ file_path: Path,
+ extra_tokens: dict[str, str] | None = None,
+) -> str:
+ """Expand common runtime tokens such as {dir}, {file}, and optional extras."""
+ expanded = base_expand_template(value, work_dir, file_path)
+ if not extra_tokens:
+ return expanded
+
+ for key, item in extra_tokens.items():
+ expanded = expanded.replace(f"{{{key}}}", str(item))
+ return expanded
+
+
+def expand_case_value(
+ value: Any,
+ work_dir: Path,
+ file_path: Path,
+ extra_tokens: dict[str, str] | None = None,
+) -> Any:
+ """Recursively expand runtime tokens inside case values."""
+ if isinstance(value, str):
+ return expand_template(value, work_dir, file_path, extra_tokens)
+ if isinstance(value, list):
+ return [expand_case_value(item, work_dir, file_path, extra_tokens) for item in value]
+ if isinstance(value, tuple):
+ return tuple(
+ expand_case_value(item, work_dir, file_path, extra_tokens)
+ for item in value
+ )
+ if isinstance(value, dict):
+ return {
+ key: expand_case_value(item, work_dir, file_path, extra_tokens)
+ for key, item in value.items()
+ }
+ return value
+
+
+def expand_case_mapping(
+ mapping: dict[str, Any] | None,
+ work_dir: Path,
+ file_path: Path,
+ extra_tokens: dict[str, str] | None = None,
+) -> dict[str, Any]:
+ """Expand tokens across a case mapping, including its keys."""
+ if not mapping:
+ return {}
+
+ expanded: dict[str, Any] = {}
+ for key, value in mapping.items():
+ expanded_key = (
+ expand_template(key, work_dir, file_path, extra_tokens)
+ if isinstance(key, str)
+ else key
+ )
+ expanded[expanded_key] = expand_case_value(
+ value,
+ work_dir,
+ file_path,
+ extra_tokens,
+ )
+ return expanded
+
+
+def merge_case_definitions(
+ generated_case: dict[str, Any],
+ case_def: dict[str, Any],
+) -> dict[str, Any]:
+ """Merge generated scenario data with explicit user-specified case fields."""
+ merged = dict(generated_case)
+ merged.update(case_def)
+
+ for field_name in (
+ "scripts",
+ "bin_files",
+ "text_files",
+ "patch_constants",
+ "mocks",
+ "kwargs",
+ ):
+ generated = generated_case.get(field_name) or {}
+ explicit = case_def.get(field_name) or {}
+ if generated or explicit:
+ merged[field_name] = {**generated, **explicit}
+
+ generated_dirs = list(generated_case.get("dir_structure") or [])
+ explicit_dirs = list(case_def.get("dir_structure") or [])
+ if generated_dirs or explicit_dirs:
+ merged["dir_structure"] = [*generated_dirs, *explicit_dirs]
+
+ if "args" not in case_def and "args" in generated_case:
+ merged["args"] = list(generated_case["args"])
+
+ return merged
+
+
+def _resolve_runner_module_from_stack() -> types.ModuleType:
+ """
+ Resolve the target module from the runner call stack.
+
+ module_main_with_env currently calls ``module.main()`` directly. For scripts
+ that only expose a main guard, scenario builders can inject a synthetic
+ ``main`` callable via patch_constants, and that callable recovers the target
+ module from the runner frame.
+ """
+ frame = inspect.currentframe()
+ current = frame.f_back if frame is not None else None
+ try:
+ while current is not None:
+ candidate = current.f_locals.get("module")
+ if isinstance(candidate, types.ModuleType):
+ return candidate
+ current = current.f_back
+ finally:
+ del frame
+ del current
+
+ raise RuntimeError("Unable to resolve target module from runner stack")
+
+
+# Keep scenario imports below the shared helpers so the split builder modules can
+# import shared functionality from this file without circular-import breakage.
+try: # Support package imports and direct harness module loading.
+ from .mock_loader_hardware_scenarios import build_blk_devices_scenario_case
+ from .mock_loader_hardware_scenarios import build_blk_write_check_scenario_case
+ from .mock_loader_hardware_scenarios import build_capsule_vars_scenario_case
+ from .mock_loader_hardware_scenarios import build_ethtool_scenario_case
+ from .mock_loader_hardware_scenarios import build_runtime_device_mapping_scenario_case
+ from .mock_loader_hardware_scenarios import build_verify_tpm_events
+ from .mock_loader_hardware_scenarios import build_verify_tpm_scenario_case
+ from .mock_loader_parser_scenarios import build_acs_info_scenario_case
+ from .mock_loader_parser_scenarios import build_extract_capsule_fw_version_scenario_case
+ from .mock_loader_parser_scenarios import build_merge_jsons_scenario_case
+except ImportError: # pragma: no cover - exercised by flat-module harness imports.
+ from mock_loader_hardware_scenarios import build_blk_devices_scenario_case
+ from mock_loader_hardware_scenarios import build_blk_write_check_scenario_case
+ from mock_loader_hardware_scenarios import build_capsule_vars_scenario_case
+ from mock_loader_hardware_scenarios import build_ethtool_scenario_case
+ from mock_loader_hardware_scenarios import build_runtime_device_mapping_scenario_case
+ from mock_loader_hardware_scenarios import build_verify_tpm_events
+ from mock_loader_hardware_scenarios import build_verify_tpm_scenario_case
+ from mock_loader_parser_scenarios import build_acs_info_scenario_case
+ from mock_loader_parser_scenarios import build_extract_capsule_fw_version_scenario_case
+ from mock_loader_parser_scenarios import build_merge_jsons_scenario_case
+
+
+SCENARIO_CASE_BUILDERS: dict[
+ str,
+ Callable[[dict[str, Any], Path], dict[str, Any]],
+] = {
+ "ethtool": build_ethtool_scenario_case,
+ "capsule_vars": build_capsule_vars_scenario_case,
+ "verify_tpm": build_verify_tpm_scenario_case,
+ "acs_info": build_acs_info_scenario_case,
+ "merge_jsons": build_merge_jsons_scenario_case,
+ "blk_devices": build_blk_devices_scenario_case,
+ "blk_write_check": build_blk_write_check_scenario_case,
+ "runtime_device_mapping": build_runtime_device_mapping_scenario_case,
+}
+
+
+def build_generated_case_definition(
+ case_def: dict[str, Any],
+ work_dir: Path,
+ file_path: Path,
+ extra_tokens: dict[str, str] | None = None,
+) -> dict[str, Any]:
+ """Build an implicit/generated case definition from scenario input."""
+ scenario = expand_case_value(
+ case_def.get("scenario"),
+ work_dir,
+ file_path,
+ extra_tokens,
+ )
+ if not scenario:
+ return {}
+ if not isinstance(scenario, dict):
+ raise ConfigError("'scenario' must resolve to a mapping")
+
+ kind = scenario.get("kind")
+ if kind is None and file_path.name == "ethtool-test.py":
+ kind = "ethtool"
+ builder = SCENARIO_CASE_BUILDERS.get(str(kind))
+ if builder is None:
+ raise ConfigError(f"Unsupported scenario kind: {kind!r}")
+ return builder(scenario, work_dir)
+
+
+def build_case_runtime_definition(
+ case_def: dict[str, Any],
+ work_dir: Path,
+ file_path: Path,
+ extra_tokens: dict[str, str] | None = None,
+) -> dict[str, Any]:
+ """Build the final runtime case after scenario generation and token expansion."""
+ generated_case = build_generated_case_definition(
+ case_def,
+ work_dir,
+ file_path,
+ extra_tokens,
+ )
+ merged_case = merge_case_definitions(generated_case, case_def)
+
+ runtime_case = dict(merged_case)
+ runtime_case["scripts"] = expand_case_mapping(
+ merged_case.get("scripts", {}),
+ work_dir,
+ file_path,
+ extra_tokens,
+ )
+ runtime_case["bin_files"] = expand_case_mapping(
+ merged_case.get("bin_files", {}),
+ work_dir,
+ file_path,
+ extra_tokens,
+ )
+ runtime_case["text_files"] = expand_case_mapping(
+ merged_case.get("text_files", {}),
+ work_dir,
+ file_path,
+ extra_tokens,
+ )
+ runtime_case["dir_structure"] = expand_case_value(
+ merged_case.get("dir_structure", []),
+ work_dir,
+ file_path,
+ extra_tokens,
+ )
+ runtime_case["patch_constants"] = expand_case_value(
+ merged_case.get("patch_constants", {}),
+ work_dir,
+ file_path,
+ extra_tokens,
+ )
+ runtime_case["mocks"] = expand_case_mapping(
+ merged_case.get("mocks", {}),
+ work_dir,
+ file_path,
+ extra_tokens,
+ )
+ runtime_case["args"] = expand_case_value(
+ merged_case.get("args", []),
+ work_dir,
+ file_path,
+ extra_tokens,
+ )
+ runtime_case["kwargs"] = expand_case_value(
+ merged_case.get("kwargs", {}),
+ work_dir,
+ file_path,
+ extra_tokens,
+ )
+ return runtime_case
+
+
+__all__ = [
+ "ConfigError",
+ "SCENARIO_CASE_BUILDERS",
+ "_resolve_runner_module_from_stack",
+ "apply_case_mocks",
+ "build_acs_info_scenario_case",
+ "build_blk_devices_scenario_case",
+ "build_blk_write_check_scenario_case",
+ "build_capsule_vars_scenario_case",
+ "build_case_runtime_definition",
+ "build_ethtool_scenario_case",
+ "build_exception_from_spec",
+ "build_extract_capsule_fw_version_scenario_case",
+ "build_generated_case_definition",
+ "build_merge_jsons_scenario_case",
+ "build_mock_from_spec",
+ "build_runtime_device_mapping_scenario_case",
+ "build_verify_tpm_events",
+ "build_verify_tpm_scenario_case",
+ "expand_case_mapping",
+ "expand_case_value",
+ "expand_patch_target",
+ "expand_template",
+ "load_dotted_object",
+ "merge_case_definitions",
+ "resolve_side_effect_value",
+ "resolve_special_value",
+ "set_nested_attr",
+ "stateful_run_router",
+]
diff --git a/common/acs_test_framework_runner/mock_loader_hardware_scenarios.py b/common/acs_test_framework_runner/mock_loader_hardware_scenarios.py
new file mode 100644
index 00000000..b217a937
--- /dev/null
+++ b/common/acs_test_framework_runner/mock_loader_hardware_scenarios.py
@@ -0,0 +1,1757 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from subprocess import CalledProcessError
+from typing import Any
+from typing import Mapping
+
+try: # Support package imports and direct harness module loading.
+ from .mock_loader import ConfigError
+ from .mock_loader import _resolve_runner_module_from_stack
+ from .mock_loader import stateful_run_router
+ from .mock_helpers import build_char16_payload
+ from .mock_helpers import build_df_output
+ from .mock_helpers import build_efi_var_bytes
+ from .mock_helpers import build_ethtool_ip_address_output
+ from .mock_helpers import build_ethtool_ip_link_line
+ from .mock_helpers import build_ethtool_ip_link_show_line
+ from .mock_helpers import build_fdisk_output
+ from .mock_helpers import build_os_indications_var
+ from .mock_helpers import build_run_result_from_outcome
+ from .mock_helpers import build_sgdisk_partition_output
+ from .mock_helpers import check_output_router
+ from .mock_helpers import default_device_path
+ from .mock_helpers import noop
+ from .mock_helpers import normalize_ethtool_tool_path
+ from .mock_helpers import passthrough_router
+ from .mock_helpers import route_gateway
+ from .mock_helpers import run_router
+ from .mock_helpers import scenario_truthy
+ from .mock_helpers import which_router
+except ImportError: # pragma: no cover - exercised by flat-module harness imports.
+ from mock_loader import ConfigError
+ from mock_loader import _resolve_runner_module_from_stack
+ from mock_loader import stateful_run_router
+ from mock_helpers import build_char16_payload
+ from mock_helpers import build_df_output
+ from mock_helpers import build_efi_var_bytes
+ from mock_helpers import build_ethtool_ip_address_output
+ from mock_helpers import build_ethtool_ip_link_line
+ from mock_helpers import build_ethtool_ip_link_show_line
+ from mock_helpers import build_fdisk_output
+ from mock_helpers import build_os_indications_var
+ from mock_helpers import build_run_result_from_outcome
+ from mock_helpers import build_sgdisk_partition_output
+ from mock_helpers import check_output_router
+ from mock_helpers import default_device_path
+ from mock_helpers import noop
+ from mock_helpers import normalize_ethtool_tool_path
+ from mock_helpers import passthrough_router
+ from mock_helpers import route_gateway
+ from mock_helpers import run_router
+ from mock_helpers import scenario_truthy
+ from mock_helpers import which_router
+
+
+def build_ethtool_scenario_case(
+ scenario: dict[str, Any],
+ work_dir: Path,
+) -> dict[str, Any]:
+ """Build runtime files, args, and mocks for an ethtool scenario."""
+ interfaces = scenario.get("interfaces", [])
+ if interfaces is None:
+ interfaces = []
+ if not isinstance(interfaces, list):
+ raise ConfigError("scenario.interfaces must be a list")
+
+ tool_paths = {
+ name: normalize_ethtool_tool_path(name, raw_value)
+ for name, raw_value in (scenario.get("tools") or {}).items()
+ }
+
+ ip_link_lines = [
+ build_ethtool_ip_link_line(index, iface)
+ for index, iface in enumerate(interfaces, start=1)
+ ]
+ run_responses: dict[str, Any] = {
+ "ip route show default": {"returncode": 0, "stdout": "", "stderr": ""},
+ "ip -o route show table all default": {"returncode": 0, "stdout": "", "stderr": ""},
+ }
+ read_text_rules: list[dict[str, Any]] = []
+ resolve_rules: list[dict[str, Any]] = []
+ default_route_lines: list[str] = []
+ route_get_output = scenario.get("route_get")
+
+ for index, iface in enumerate(interfaces, start=1):
+ name = iface["name"]
+ kind = str(iface.get("kind", "physical"))
+
+ run_responses[f"ip link show {name}"] = {
+ "returncode": 0,
+ "stdout": build_ethtool_ip_link_show_line(index, iface),
+ "stderr": "",
+ }
+ run_responses[f"ip link set dev {name} down"] = {
+ "returncode": 0,
+ "stdout": "",
+ "stderr": "",
+ }
+ run_responses[f"ip link set dev {name} up"] = {
+ "returncode": 0,
+ "stdout": "",
+ "stderr": "",
+ }
+ run_responses[f"ip address show dev {name}"] = {
+ "returncode": 0,
+ "stdout": build_ethtool_ip_address_output(index, iface),
+ "stderr": "",
+ }
+
+ if tool_paths.get("dhclient") is not None:
+ run_responses[f"dhclient -r {name}"] = {
+ "returncode": 0,
+ "stdout": "",
+ "stderr": "",
+ }
+ run_responses[f"dhclient -1 {name}"] = {
+ "returncode": 0,
+ "stdout": "",
+ "stderr": "",
+ }
+ if tool_paths.get("udhcpc") is not None:
+ run_responses[f"udhcpc -n -q -i {name}"] = {
+ "returncode": 0,
+ "stdout": "",
+ "stderr": "",
+ }
+
+ if kind != "loopback":
+ device_path = str(iface.get("device") or default_device_path(index, iface))
+ resolve_rules.append(
+ {
+ "when": {"args": {0: {"contains": f"/sys/class/net/{name}/device"}}},
+ "return": device_path,
+ }
+ )
+ read_text_rules.append(
+ {
+ "when": {"args": {0: {"contains": f"/sys/class/net/{name}/carrier"}}},
+ "return": str(iface.get("carrier", "1" if kind == "physical" else "0")),
+ }
+ )
+ read_text_rules.append(
+ {
+ "when": {"args": {0: {"contains": f"/sys/class/net/{name}/operstate"}}},
+ "return": str(iface.get("operstate", "up" if kind == "physical" else "down")),
+ }
+ )
+
+ if tool_paths.get("ethtool") is not None and kind != "loopback":
+ ethtool_cfg = iface.get("ethtool") or {}
+ link_detected = scenario_truthy(
+ ethtool_cfg.get("link_detected"),
+ default=str(iface.get("carrier", "0")) == "1",
+ )
+ self_test_supported = scenario_truthy(
+ ethtool_cfg.get("self_test_supported"),
+ default=False,
+ )
+ run_responses[f"ethtool {name}"] = {
+ "returncode": 0,
+ "stdout": (
+ f"Settings for {name}\n"
+ f"\tLink detected: {'yes' if link_detected else 'no'}\n"
+ ),
+ "stderr": "",
+ }
+ run_responses[f"ethtool -i {name}"] = {
+ "returncode": 0,
+ "stdout": (
+ "driver: fake\n"
+ f"supports-test: {'yes' if self_test_supported else 'no'}\n"
+ ),
+ "stderr": "",
+ }
+ if self_test_supported or "self_test" in ethtool_cfg:
+ self_test = ethtool_cfg.get("self_test", "pass")
+ run_responses[f"ethtool -t {name}"] = build_run_result_from_outcome(
+ self_test,
+ success_stdout="The test result is PASS\n",
+ failure_stdout="self-test failed\n",
+ )
+
+ routes = iface.get("routes") or {}
+ default_route = routes.get("default") or iface.get("default_route")
+ if default_route:
+ default_route_lines.append(str(default_route))
+ if route_get_output is None:
+ route_get_output = routes.get("route_get") or iface.get("route_get")
+
+ connectivity = iface.get("connectivity") or {}
+ gateway = route_gateway(str(default_route)) if default_route else None
+ if gateway and "gateway_ping" in connectivity:
+ run_responses[
+ f"ping -c 3 -W 10 -I {name} {gateway}"
+ ] = build_run_result_from_outcome(
+ connectivity["gateway_ping"],
+ success_stdout="3 packets transmitted, 3 received, 0% packet loss\n",
+ failure_stdout="3 packets transmitted, 0 received, 100% packet loss\n",
+ )
+ if "arm_ping" in connectivity:
+ run_responses[
+ f"ping -c 3 -W 10 -I {name} www.arm.com"
+ ] = build_run_result_from_outcome(
+ connectivity["arm_ping"],
+ success_stdout="3 packets transmitted, 3 received, 0% packet loss\n",
+ failure_stdout="3 packets transmitted, 0 received, 100% packet loss\n",
+ )
+ if "ipv6_ping" in connectivity:
+ ipv6_result = build_run_result_from_outcome(
+ connectivity["ipv6_ping"],
+ success_stdout="3 packets transmitted, 3 received, 0% packet loss\n",
+ failure_stdout="3 packets transmitted, 0 received, 100% packet loss\n",
+ )
+ run_responses[f"ping -6 -c 3 -I {name} ipv6.google.com"] = ipv6_result
+ run_responses[f"ping6 -c 3 -I {name} ipv6.google.com"] = ipv6_result
+ if "wget" in connectivity:
+ run_responses[
+ "wget --spider --timeout=10 https://www.arm.com"
+ ] = build_run_result_from_outcome(
+ connectivity["wget"],
+ success_stdout="",
+ failure_stderr="network unreachable\n",
+ )
+ if "curl" in connectivity:
+ run_responses[
+ f"curl -Is --connect-timeout 20 --interface {name} https://www.arm.com"
+ ] = build_run_result_from_outcome(
+ connectivity["curl"],
+ success_stdout="HTTP/2 200\n",
+ failure_stderr="connect failed\n",
+ )
+
+ if default_route_lines:
+ routes_stdout = "\n".join(default_route_lines) + "\n"
+ run_responses["ip route show default"] = {
+ "returncode": 0,
+ "stdout": routes_stdout,
+ "stderr": "",
+ }
+ run_responses["ip -o route show table all default"] = {
+ "returncode": 0,
+ "stdout": routes_stdout,
+ "stderr": "",
+ }
+
+ if route_get_output:
+ route_text = str(route_get_output)
+ if not route_text.endswith("\n"):
+ route_text += "\n"
+ run_responses["ip route get 8.8.8.8"] = {
+ "returncode": 0,
+ "stdout": route_text,
+ "stderr": "",
+ }
+
+ required = scenario.get("required_compliant_interfaces")
+ if required is None:
+ required = sum(
+ 1
+ for iface in interfaces
+ if str(iface.get("kind", "physical")) == "physical"
+ )
+
+ system_config_path = work_dir / "system_config.txt"
+ return {
+ "args": [str(system_config_path)],
+ "text_files": {
+ "system_config.txt": f"total_number_of_network_controllers: {required}\n",
+ },
+ "mocks": {
+ "signal.signal": {"value": noop},
+ "time.sleep": {"value": noop},
+ "shutil.which": {
+ "factory": which_router,
+ "kwargs": {"responses": tool_paths, "default": None},
+ },
+ "subprocess.check_output": {
+ "factory": check_output_router,
+ "kwargs": {
+ "responses": {
+ "ip -o link": "\n".join(ip_link_lines) + ("\n" if ip_link_lines else "")
+ },
+ "use_contains": False,
+ },
+ },
+ "subprocess.run": {
+ "factory": run_router,
+ "kwargs": {
+ "responses": run_responses,
+ "default_returncode": 1,
+ "use_contains": False,
+ "unmatched_stderr_template": "UNMOCKED COMMAND: {cmd}\n",
+ },
+ },
+ "pathlib.Path.read_text": {
+ "factory": passthrough_router,
+ "inject_original_as": "real",
+ "kwargs": {"rules": read_text_rules},
+ },
+ "pathlib.Path.resolve": {
+ "factory": passthrough_router,
+ "inject_original_as": "real",
+ "kwargs": {"rules": resolve_rules},
+ },
+ },
+ }
+
+
+PRECIOUS_MBR_IDS = {"0xF8", "0xEF"}
+PRECIOUS_GPT_GUIDS = {
+ "C12A7328-F81F-11D2-BA4B-00A0C93EC93B",
+ "21686148-6449-6E6F-744E-656564454649",
+ "3DE21764-95BD-54BD-A5C3-4ABE786F38A8",
+}
+
+
+def _completed_process_spec(
+ *,
+ returncode: int = 0,
+ stdout: str = "",
+ stderr: str = "",
+) -> dict[str, Any]:
+ return {"returncode": returncode, "stdout": stdout, "stderr": stderr}
+
+
+def _join_stdout_lines(items: list[str]) -> str:
+ if not items:
+ return ""
+ return "\n".join(items) + "\n"
+
+
+def _normalize_mbr_partition_id(value: Any) -> str:
+ text = str(value).strip().upper()
+ if not text:
+ return ""
+ if text.startswith("0X"):
+ text = text[2:]
+ return f"0x{text}"
+
+
+def _normalize_prompt_response(value: Any, *, default: str = "no") -> str:
+ """Normalize YAML prompt values into explicit yes/no/timeout responses."""
+ if value is None:
+ text = ""
+ elif isinstance(value, bool):
+ text = "yes" if value else "no"
+ else:
+ text = str(value).strip().lower()
+
+ if not text:
+ return default
+
+ normalized = {
+ "timeout": "timeout",
+ "1": "yes",
+ "true": "yes",
+ "yes": "yes",
+ "y": "yes",
+ "on": "yes",
+ "0": "no",
+ "false": "no",
+ "no": "no",
+ "n": "no",
+ "off": "no",
+ }
+ return normalized.get(text, text)
+
+
+def _platform_required(partition: dict[str, Any]) -> bool:
+ if "attribute_flags" in partition:
+ flags = str(partition["attribute_flags"]).strip()
+ try:
+ return (int(flags, 16) & 1) == 1
+ except ValueError as exc:
+ raise ConfigError(
+ f"Invalid GPT attribute_flags value: {partition['attribute_flags']!r}"
+ ) from exc
+ return scenario_truthy(partition.get("platform_required"), default=False)
+
+
+def _add_blk_write_check_run_responses(
+ run_responses: dict[str, Any],
+ target: str,
+ write_cfg: dict[str, Any] | None,
+) -> None:
+ cfg = write_cfg or {}
+ mounted = scenario_truthy(cfg.get("mounted"), default=False)
+ run_responses[f"findmnt -n /dev/{target}"] = _completed_process_spec(
+ returncode=0 if mounted else 1,
+ stdout=f"/dev/{target} /mnt/{target} ext4 rw,relatime 0 0\n" if mounted else "",
+ )
+
+ if mounted:
+ return
+
+ if "used_blocks" not in cfg and "available_blocks" not in cfg:
+ return
+
+ used_blocks = int(cfg.get("used_blocks", 0))
+ available_blocks = int(cfg.get("available_blocks", 0))
+ run_responses[f"df -B 512 /dev/{target} --output=used,avail"] = _completed_process_spec(
+ stdout=build_df_output(used_blocks, available_blocks)
+ )
+
+
+def _build_blk_write_check_stateful_rules(
+ device: str,
+ cfg: dict[str, Any],
+) -> tuple[dict[str, Any], list[dict[str, Any]]]:
+ write_flow = cfg.get("write_flow")
+ if write_flow is None:
+ return {}, []
+ if not isinstance(write_flow, dict):
+ raise ConfigError("blk_write_check.write_flow must be a mapping when provided")
+
+ initial_bytes = cfg.get("initial_device_bytes")
+ if initial_bytes is None:
+ initial_text = str(cfg.get("initial_device_text", "B" * 512))
+ initial_bytes = initial_text.encode("utf-8")
+ elif isinstance(initial_bytes, str):
+ try:
+ initial_bytes = bytes.fromhex(initial_bytes)
+ except ValueError as exc:
+ raise ConfigError(
+ "blk_write_check.initial_device_bytes must be bytes or hex string"
+ ) from exc
+ elif not isinstance(initial_bytes, bytes):
+ raise ConfigError("blk_write_check.initial_device_bytes must be bytes or hex string")
+
+ mismatch_bytes = write_flow.get("mismatch_bytes")
+ if mismatch_bytes is None:
+ mismatch_bytes = b"X" * max(len(initial_bytes), 512)
+ elif isinstance(mismatch_bytes, str):
+ try:
+ mismatch_bytes = bytes.fromhex(mismatch_bytes)
+ except ValueError as exc:
+ raise ConfigError(
+ "blk_write_check.write_flow.mismatch_bytes must be bytes or hex string"
+ ) from exc
+ elif not isinstance(mismatch_bytes, bytes):
+ raise ConfigError("blk_write_check.write_flow.mismatch_bytes must be bytes or hex string")
+
+ used_blocks = int(cfg.get("used_blocks", 0))
+ backup_name = f"{device}_backup.bin"
+ device_state_key = f"{device}:device_bytes"
+ mismatch_state_key = f"{device}:mismatch_bytes"
+ state = {
+ device_state_key: initial_bytes,
+ mismatch_state_key: mismatch_bytes,
+ }
+ rules: list[dict[str, Any]] = []
+
+ backup_command = (
+ f"dd if=/dev/{device} of={backup_name} bs=512 count=1 skip={used_blocks}"
+ )
+ backup_mode = str(write_flow.get("backup", "success")).lower()
+ if backup_mode in {"success", "pass", "ok"}:
+ rules.append({
+ "command": backup_command,
+ "write_file": {
+ "path": backup_name,
+ "content": {"from_state": device_state_key},
+ },
+ "result": {"returncode": 0, "stdout": "", "stderr": ""},
+ })
+ elif backup_mode == "error":
+ error_text = str(write_flow.get("backup_error", "mocked backup failure"))
+ rules.append({
+ "command": backup_command,
+ "raise": {
+ "type": CalledProcessError,
+ "args": [1, backup_command],
+ "kwargs": {"output": "", "stderr": error_text},
+ },
+ })
+ elif backup_mode not in {"skip", "none"}:
+ raise ConfigError(
+ "blk_write_check.write_flow.backup must be success, error, skip, or none"
+ )
+
+ write_command = (
+ f"dd if=hello.txt of=/dev/{device} bs=512 count=1 seek={used_blocks}"
+ )
+ write_mode = str(write_flow.get("write", "success")).lower()
+ if write_mode in {"success", "pass", "ok"}:
+ written_content = (
+ {"from_state": mismatch_state_key}
+ if str(write_flow.get("readback", "match")).lower() == "mismatch"
+ else {"from_file": "hello.txt"}
+ )
+ rules.append({
+ "command": write_command,
+ "set_state": {device_state_key: written_content},
+ "result": {"returncode": 0, "stdout": "", "stderr": ""},
+ })
+ elif write_mode == "error":
+ error_text = str(write_flow.get("write_error", "mocked write failure"))
+ rules.append({
+ "command": write_command,
+ "raise": {
+ "type": CalledProcessError,
+ "args": [1, write_command],
+ "kwargs": {"output": "", "stderr": error_text},
+ },
+ })
+ elif write_mode not in {"skip", "none"}:
+ raise ConfigError(
+ "blk_write_check.write_flow.write must be success, error, skip, or none"
+ )
+
+ read_command = (
+ f"dd if=/dev/{device} of=read_hello.txt bs=512 count=1 skip={used_blocks}"
+ )
+ readback_mode = str(write_flow.get("readback", "match")).lower()
+ if readback_mode in {"match", "mismatch"}:
+ rules.append({
+ "command": read_command,
+ "write_file": {
+ "path": "read_hello.txt",
+ "content": {"from_state": device_state_key},
+ },
+ "result": {"returncode": 0, "stdout": "", "stderr": ""},
+ })
+ elif readback_mode == "error":
+ error_text = str(write_flow.get("readback_error", "mocked readback failure"))
+ rules.append({
+ "command": read_command,
+ "raise": {
+ "type": CalledProcessError,
+ "args": [1, read_command],
+ "kwargs": {"output": "", "stderr": error_text},
+ },
+ })
+ else:
+ raise ConfigError("blk_write_check.write_flow.readback must be match, mismatch, or error")
+
+ restore_command = (
+ f"dd if={backup_name} of=/dev/{device} bs=512 count=1 seek={used_blocks}"
+ )
+ restore_mode = str(write_flow.get("restore", "success")).lower()
+ if restore_mode in {"success", "pass", "ok"}:
+ rules.append({
+ "command": restore_command,
+ "set_state": {device_state_key: {"from_file": backup_name}},
+ "result": {"returncode": 0, "stdout": "", "stderr": ""},
+ })
+ elif restore_mode == "error":
+ error_text = str(write_flow.get("restore_error", "mocked restore failure"))
+ rules.append({
+ "command": restore_command,
+ "raise": {
+ "type": CalledProcessError,
+ "args": [1, restore_command],
+ "kwargs": {"output": "", "stderr": error_text},
+ },
+ })
+ elif restore_mode not in {"skip", "none"}:
+ raise ConfigError(
+ "blk_write_check.write_flow.restore must be success, error, skip, or none"
+ )
+
+ return state, rules
+
+
+BLK_DEVICE_BANNER = "*" * 128
+
+
+def _print_blk_devices_separator() -> None:
+ print(f"\n{BLK_DEVICE_BANNER}\n")
+
+
+def _run_blk_device_command(module: Any, command: str) -> Any:
+ return module.subprocess.run(
+ command,
+ shell=True,
+ text=True,
+ check=False,
+ capture_output=True,
+ )
+
+
+def _list_blk_devices(module: Any) -> list[str]:
+ command = "lsblk -e 7 -d | grep disk | awk '{print $1}'"
+ result = _run_blk_device_command(module, command)
+ return result.stdout.split()
+
+
+def _print_blk_devices_header(disks: list[str]) -> None:
+ _print_blk_devices_separator()
+ print(" Read block devices tool\n")
+ print(BLK_DEVICE_BANNER)
+ print("INFO: Detected following block devices with lsblk command :")
+ for num, disk in enumerate(disks):
+ print(f"{num}: {disk}")
+ _print_blk_devices_separator()
+
+
+def _detect_blk_partition_table(module: Any, disk: str) -> str:
+ command = f"timeout 10 gdisk -l /dev/{disk}"
+ result = _run_blk_device_command(module, command)
+ if "MBR: MBR only" in result.stdout:
+ return "MBR"
+ if "GPT: present" in result.stdout:
+ return "GPT"
+ print(f"INFO: No valid partition table found for {disk}, treating as raw device.")
+ return "RAW"
+
+
+def _count_blk_partitions(module: Any, disk: str) -> int:
+ command = f"lsblk -rn -o NAME,TYPE /dev/{disk} | grep -c part"
+ result = _run_blk_device_command(module, command)
+ num_parts_str = result.stdout.strip()
+ return int(num_parts_str) if num_parts_str else 0
+
+
+def _run_blk_read(module: Any, partition_label: str, context: str = "") -> bool:
+ context_suffix = f" {context}" if context else ""
+ print(f"INFO: Performing block read on /dev/{partition_label}{context_suffix}")
+ command = f"dd if=/dev/{partition_label} bs=1M count=1 > /dev/null"
+ result = _run_blk_device_command(module, command)
+ outcome = "successful" if result.returncode == 0 else "failed"
+ print(f"INFO: Block read on /dev/{partition_label}{context_suffix} {outcome}")
+ return result.returncode == 0
+
+
+def _print_precious_partition(
+ module: Any,
+ partition_label: str,
+ identifier: str,
+ precious_parts: Mapping[str, str],
+ *,
+ suffix: str = "",
+) -> None:
+ for key, value in precious_parts.items():
+ if value != identifier:
+ continue
+ print(f"INFO: {partition_label} partition is PRECIOUS{suffix}")
+ used_blocks, _ = module.get_partition_space(f"/dev/{partition_label}")
+ print(
+ "INFO: Number of 512B blocks used on "
+ f"/dev/{partition_label}: {used_blocks}"
+ )
+ print(f" {key} : {value}")
+ print(" Skipping block read/write...")
+ break
+
+
+def _parse_mbr_partition_ids(fdisk_stdout: str) -> list[str]:
+ table_header_row = [
+ "Device",
+ "Boot",
+ "Start",
+ "End",
+ "Sectors",
+ "Size",
+ "Id",
+ "Type",
+ ]
+ lines = fdisk_stdout.strip().split("\n")
+ collect_lines = False
+ mbr_part_ids: list[str] = []
+
+ for line in lines:
+ if collect_lines:
+ columns = line.split()
+ if len(columns) < 6:
+ continue
+ if columns[1] == "*" and len(columns) >= 7:
+ mbr_part_ids.append("0x" + columns[6].upper())
+ else:
+ mbr_part_ids.append("0x" + columns[5].upper())
+ continue
+ if all(substring in line for substring in table_header_row):
+ collect_lines = True
+
+ return mbr_part_ids
+
+
+def _parse_gpt_partition_info(
+ module: Any,
+ sgdisk_stdout: str,
+) -> tuple[str, int] | None:
+ guid_regex = r"Partition GUID code: ([\w-]+) \("
+ attr_flag_regex = r"Attribute flags: ([0-9A-Fa-f]+)"
+ guid_code_match = module.re.search(guid_regex, sgdisk_stdout)
+ attribute_flags_match = module.re.search(attr_flag_regex, sgdisk_stdout)
+ if not guid_code_match or not attribute_flags_match:
+ return None
+
+ partition_guid_code = guid_code_match.group(1)
+ attribute_flags_hex = attribute_flags_match.group(1)
+ attribute_flags_int = int(attribute_flags_hex, 16)
+ return partition_guid_code, attribute_flags_int & 1
+
+
+def _handle_raw_blk_device(module: Any, disk: str) -> None:
+ print(f"INFO: No partitions detected for {disk}, treating as raw device.")
+ if _run_blk_read(module, disk):
+ module.perform_write_check(disk, "", {})
+ _print_blk_devices_separator()
+
+
+def _handle_mbr_blk_device(
+ module: Any,
+ disk: str,
+ part_labels: list[str],
+ num_parts: int,
+) -> None:
+ command = f"fdisk -l /dev/{disk}"
+ result = _run_blk_device_command(module, command)
+ mbr_part_ids = _parse_mbr_partition_ids(result.stdout)
+
+ if len(mbr_part_ids) < num_parts:
+ print(
+ "WARNING: Could not parse enough MBR partition IDs. "
+ f"Found {len(mbr_part_ids)}, expected {num_parts}."
+ )
+
+ process_count = min(len(part_labels), len(mbr_part_ids), num_parts)
+ for index in range(process_count):
+ part_label = part_labels[index]
+ partition_id = mbr_part_ids[index]
+ print(f"\nINFO: Partition : /dev/{part_label} Partition type : {partition_id}")
+
+ if partition_id in module.precious_parts_mbr.values():
+ _print_precious_partition(
+ module,
+ part_label,
+ partition_id,
+ module.precious_parts_mbr,
+ )
+ continue
+
+ if _run_blk_read(module, part_label, f"mbr_part_id = {partition_id}"):
+ module.perform_write_check(
+ part_label,
+ partition_id,
+ module.precious_parts_mbr,
+ )
+
+ _print_blk_devices_separator()
+
+
+def _handle_gpt_blk_device(
+ module: Any,
+ disk: str,
+ part_labels: list[str],
+ num_parts: int,
+) -> None:
+ process_count = min(len(part_labels), num_parts)
+ for index in range(process_count):
+ part_label = part_labels[index]
+ command = f"sgdisk -i={index + 1} /dev/{disk}"
+ result = _run_blk_device_command(module, command)
+ parsed = _parse_gpt_partition_info(module, result.stdout)
+
+ if parsed is None:
+ print(f"INFO: Unable to parse sgdisk info for {part_label}. Skipping.")
+ continue
+
+ partition_guid_code, lsb = parsed
+ print(
+ f"\nINFO: Partition : /dev/{part_label} "
+ f"Partition type GUID : {partition_guid_code} "
+ f"\"Platform required bit\" : {lsb}"
+ )
+
+ if lsb == 1:
+ print(
+ "INFO: Platform required attribute set for "
+ f"{part_label} partition, skipping block read/write..."
+ )
+ continue
+
+ if partition_guid_code in module.precious_parts_gpt.values():
+ _print_precious_partition(
+ module,
+ part_label,
+ partition_guid_code,
+ module.precious_parts_gpt,
+ suffix=".",
+ )
+ continue
+
+ if _run_blk_read(module, part_label, f"part_guid = {partition_guid_code}"):
+ module.perform_write_check(
+ part_label,
+ partition_guid_code,
+ module.precious_parts_gpt,
+ )
+
+ _print_blk_devices_separator()
+
+
+def _process_blk_device(module: Any, disk: str) -> None:
+ print(f"INFO: Block device : /dev/{disk}")
+ part_table = _detect_blk_partition_table(module, disk)
+ print(f"INFO: Partition table type : {part_table}\n")
+
+ num_parts = _count_blk_partitions(module, disk)
+ if part_table == "RAW" or num_parts == 0:
+ _handle_raw_blk_device(module, disk)
+ return
+
+ part_labels = module.get_partition_labels(disk)
+ if len(part_labels) < num_parts:
+ print(
+ "WARNING: Mismatch in partition count. Found "
+ f"{len(part_labels)} partition labels, but lsblk reported "
+ f"{num_parts} partitions for {disk}. Proceeding with the ones we have..."
+ )
+
+ if part_table == "MBR":
+ _handle_mbr_blk_device(module, disk, part_labels, num_parts)
+ return
+ if part_table == "GPT":
+ _handle_gpt_blk_device(module, disk, part_labels, num_parts)
+ return
+
+ print(
+ "INFO: Invalid partition table, expected MBR or GPT "
+ f"reported type = {part_table}"
+ )
+
+
+def _run_blk_devices_module_main() -> int:
+ """
+ Synthetic main() for block-device scripts that only define a main guard.
+
+ This mirrors the repository script flow closely enough for YAML scenario
+ tests while still exercising the target module's own helper functions.
+ """
+ module = _resolve_runner_module_from_stack()
+
+ try:
+ disks = _list_blk_devices(module)
+ _print_blk_devices_header(disks)
+
+ for disk in disks:
+ if module.is_mtd_block_device(disk):
+ print(f"INFO: Skipping MTD block device /dev/{disk}")
+ continue
+
+ if module.is_ram_disk(disk):
+ print(f"INFO: Skipping RAM disk /dev/{disk}")
+ continue
+
+ _process_blk_device(module, disk)
+
+ return 0
+ except Exception as exc: # pylint: disable=broad-exception-caught
+ print(f"Error occurred: {exc}")
+ return 1
+
+
+def build_blk_devices_scenario_case(
+ scenario: dict[str, Any],
+ work_dir: Path,
+) -> dict[str, Any]:
+ """Build runtime mocks for the block-device script main flow."""
+ disks = scenario.get("disks", [])
+ if disks is None:
+ disks = []
+ if not isinstance(disks, list):
+ raise ConfigError("scenario.disks must be a list")
+
+ prompt = _normalize_prompt_response(scenario.get("prompt", "no"))
+ prompt_response = "no" if prompt == "timeout" else prompt
+ call_log_path = str(work_dir / "blk_commands.log")
+ disk_listing_lines: list[str] = []
+ run_responses: dict[str, Any] = {}
+ stateful_state: dict[str, Any] = {}
+ stateful_rules: list[dict[str, Any]] = []
+
+ for disk_index, disk in enumerate(disks, start=1):
+ if not isinstance(disk, dict):
+ raise ConfigError(f"scenario.disks[{disk_index}] must be a mapping")
+ if "name" not in disk:
+ raise ConfigError(f"scenario.disks[{disk_index}] requires key 'name'")
+
+ disk_name = str(disk["name"])
+ disk_listing_lines.append(f"{disk_name} disk")
+
+ disk_kind = str(disk.get("kind", "disk")).lower()
+ if disk_name.startswith("mtdblock") or disk_kind in {"mtd", "mtdblock"}:
+ continue
+ if disk_name.startswith("ram") or disk_kind == "ram":
+ continue
+
+ table = str(disk.get("table", "raw")).lower()
+ if table not in {"raw", "mbr", "gpt"}:
+ raise ConfigError(
+ f"scenario.disks[{disk_index}].table must be one of raw, mbr, gpt"
+ )
+
+ partitions = disk.get("partitions", [])
+ if partitions is None:
+ partitions = []
+ if not isinstance(partitions, list):
+ raise ConfigError(f"scenario.disks[{disk_index}].partitions must be a list")
+
+ reported_partition_count = disk.get("reported_partition_count", len(partitions))
+ if not isinstance(reported_partition_count, int):
+ raise ConfigError(
+ f"scenario.disks[{disk_index}].reported_partition_count must be an integer"
+ )
+
+ gdisk_stdout = str(
+ disk.get(
+ "gdisk_output",
+ (
+ "MBR: MBR only\n"
+ if table == "mbr"
+ else "GPT: present\n" if table == "gpt" else "no partition table here\n"
+ ),
+ )
+ )
+ if gdisk_stdout and not gdisk_stdout.endswith("\n"):
+ gdisk_stdout += "\n"
+
+ run_responses[f"timeout 10 gdisk -l /dev/{disk_name}"] = _completed_process_spec(
+ stdout=gdisk_stdout
+ )
+
+ if table == "raw" or reported_partition_count == 0:
+ run_responses[f"lsblk -rn -o NAME,TYPE /dev/{disk_name}"] = _completed_process_spec(
+ stdout=""
+ )
+ run_responses[f"dd if=/dev/{disk_name} of=/dev/null bs=1M count=1"] = (
+ build_run_result_from_outcome(disk.get("block_read", "pass"))
+ )
+ disk_write_cfg = disk.get("write_check") or disk
+ _add_blk_write_check_run_responses(
+ run_responses,
+ disk_name,
+ disk_write_cfg,
+ )
+ state, rules = _build_blk_write_check_stateful_rules(
+ disk_name,
+ disk_write_cfg,
+ )
+ stateful_state.update(state)
+ stateful_rules.extend(rules)
+ continue
+
+ partition_labels: list[str] = []
+ for part_index, partition in enumerate(partitions, start=1):
+ if not isinstance(partition, dict):
+ raise ConfigError(
+ f"scenario.disks[{disk_index}].partitions[{part_index}] must be a mapping"
+ )
+ partition_labels.append(str(partition.get("name", f"{disk_name}{part_index}")))
+
+ lsblk_partition_lines = [f"{label} part" for label in partition_labels]
+ run_responses[f"lsblk -rn -o NAME,TYPE /dev/{disk_name}"] = _completed_process_spec(
+ stdout=_join_stdout_lines(lsblk_partition_lines)
+ )
+
+ if table == "mbr":
+ fdisk_stdout = disk.get("fdisk_output")
+ if fdisk_stdout is None:
+ fdisk_stdout = build_fdisk_output(disk_name, partitions)
+ else:
+ fdisk_stdout = str(fdisk_stdout)
+ if fdisk_stdout and not fdisk_stdout.endswith("\n"):
+ fdisk_stdout += "\n"
+
+ run_responses[f"fdisk -l /dev/{disk_name}"] = _completed_process_spec(
+ stdout=fdisk_stdout
+ )
+
+ for part_index, partition in enumerate(partitions, start=1):
+ partition_name = str(partition.get("name", f"{disk_name}{part_index}"))
+ partition_id = _normalize_mbr_partition_id(partition.get("id", "83"))
+
+ if (
+ partition_id in PRECIOUS_MBR_IDS
+ or "used_blocks" in partition
+ or "available_blocks" in partition
+ ):
+ run_responses[
+ f"df -B 512 /dev/{partition_name} --output=used,avail"
+ ] = _completed_process_spec(
+ stdout=build_df_output(
+ int(partition.get("used_blocks", 0)),
+ int(partition.get("available_blocks", 0)),
+ )
+ )
+
+ if partition_id in PRECIOUS_MBR_IDS:
+ continue
+
+ run_responses[
+ f"dd if=/dev/{partition_name} of=/dev/null bs=1M count=1"
+ ] = build_run_result_from_outcome(partition.get("block_read", "pass"))
+ partition_write_cfg = partition.get("write_check") or partition
+ _add_blk_write_check_run_responses(
+ run_responses,
+ partition_name,
+ partition_write_cfg,
+ )
+ state, rules = _build_blk_write_check_stateful_rules(
+ partition_name,
+ partition_write_cfg,
+ )
+ stateful_state.update(state)
+ stateful_rules.extend(rules)
+
+ continue
+
+ for part_index, partition in enumerate(partitions, start=1):
+ partition_name = str(partition.get("name", f"{disk_name}{part_index}"))
+ partition_guid = str(
+ partition.get("guid", "0FC63DAF-8483-4772-8E79-3D69D8477DE4")
+ ).upper()
+ required = _platform_required(partition)
+
+ sgdisk_stdout = partition.get("sgdisk_output")
+ if sgdisk_stdout is None:
+ sgdisk_stdout = build_sgdisk_partition_output(partition)
+ else:
+ sgdisk_stdout = str(sgdisk_stdout)
+ if sgdisk_stdout and not sgdisk_stdout.endswith("\n"):
+ sgdisk_stdout += "\n"
+
+ run_responses[f"sgdisk -i={part_index} /dev/{disk_name}"] = _completed_process_spec(
+ stdout=sgdisk_stdout
+ )
+
+ if (
+ partition_guid in PRECIOUS_GPT_GUIDS
+ or required
+ or "used_blocks" in partition
+ or "available_blocks" in partition
+ ):
+ run_responses[
+ f"df -B 512 /dev/{partition_name} --output=used,avail"
+ ] = _completed_process_spec(
+ stdout=build_df_output(
+ int(partition.get("used_blocks", 0)),
+ int(partition.get("available_blocks", 0)),
+ )
+ )
+
+ if partition_guid in PRECIOUS_GPT_GUIDS or required:
+ continue
+
+ run_responses[
+ f"dd if=/dev/{partition_name} of=/dev/null bs=1M count=1"
+ ] = build_run_result_from_outcome(partition.get("block_read", "pass"))
+ partition_write_cfg = partition.get("write_check") or partition
+ _add_blk_write_check_run_responses(
+ run_responses,
+ partition_name,
+ partition_write_cfg,
+ )
+ state, rules = _build_blk_write_check_stateful_rules(
+ partition_name,
+ partition_write_cfg,
+ )
+ stateful_state.update(state)
+ stateful_rules.extend(rules)
+
+ run_responses["lsblk -e 7 -d -n -o NAME,TYPE"] = _completed_process_spec(
+ stdout=_join_stdout_lines(disk_listing_lines)
+ )
+
+ run_mock_spec: dict[str, Any] = {
+ "factory": run_router,
+ "kwargs": {
+ "responses": run_responses,
+ "default_returncode": 1,
+ "use_contains": False,
+ "unmatched_stderr_template": "UNMOCKED COMMAND: {cmd}\n",
+ "call_log_path": call_log_path,
+ },
+ }
+ if stateful_rules:
+ run_mock_spec = {
+ "factory": stateful_run_router,
+ "kwargs": {
+ "state": stateful_state,
+ "rules": stateful_rules,
+ "responses": run_responses,
+ "default_returncode": 1,
+ "use_contains": False,
+ "unmatched_stderr_template": "UNMOCKED COMMAND: {cmd}\n",
+ "call_log_path": call_log_path,
+ },
+ }
+
+ return {
+ "mocks": {
+ "{module}.input_with_timeout": {"return_value": prompt_response},
+ "subprocess.run": run_mock_spec,
+ },
+ }
+
+
+def build_blk_write_check_scenario_case(
+ scenario: dict[str, Any],
+ work_dir: Path,
+) -> dict[str, Any]:
+ """Build runtime mocks for direct perform_write_check scenarios."""
+ device = scenario.get("device") or scenario.get("partition_label")
+ if not isinstance(device, str) or not device.strip():
+ raise ConfigError("blk_write_check requires string key 'device'")
+
+ partition_id = str(scenario.get("partition_id", ""))
+ precious_parts = scenario.get("precious_parts", {})
+ if not isinstance(precious_parts, dict):
+ raise ConfigError("blk_write_check.precious_parts must be a mapping")
+
+ prompt = _normalize_prompt_response(scenario.get("prompt", "no"))
+ prompt_result = "no" if prompt == "timeout" else prompt
+ call_log_path = str(work_dir / "blk_commands.log")
+ run_responses: dict[str, Any] = {}
+ _add_blk_write_check_run_responses(run_responses, device, scenario)
+ state, rules = _build_blk_write_check_stateful_rules(device, scenario)
+ if not rules:
+ return {
+ "args": [device, partition_id, precious_parts],
+ "mocks": {
+ "{module}.input_with_timeout": {"return_value": prompt_result},
+ "subprocess.run": {
+ "factory": run_router,
+ "kwargs": {
+ "responses": run_responses,
+ "default_returncode": 1,
+ "use_contains": False,
+ "unmatched_stderr_template": "UNMOCKED COMMAND: {cmd}\n",
+ "call_log_path": call_log_path,
+ },
+ },
+ },
+ }
+
+ return {
+ "args": [device, partition_id, precious_parts],
+ "mocks": {
+ "{module}.input_with_timeout": {"return_value": prompt_result},
+ "subprocess.run": {
+ "factory": stateful_run_router,
+ "kwargs": {
+ "state": state,
+ "rules": rules,
+ "responses": run_responses,
+ "default_returncode": 1,
+ "use_contains": False,
+ "unmatched_stderr_template": "UNMOCKED COMMAND: {cmd}\n",
+ "call_log_path": call_log_path,
+ },
+ },
+ },
+ }
+
+
+def _ensure_string_list(value: Any, field_name: str) -> list[str]:
+ if value is None:
+ return []
+ if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
+ raise ConfigError(f"{field_name} must be a list of strings")
+ return list(value)
+
+
+def _normalize_verify_tpm_banks(value: Any, field_name: str) -> dict[str, list[str]]:
+ if not isinstance(value, dict) or not value:
+ raise ConfigError(f"{field_name} must be a non-empty mapping")
+
+ normalized: dict[str, list[str]] = {}
+ for algo, entries in value.items():
+ if not isinstance(algo, str):
+ raise ConfigError(f"{field_name} keys must be strings")
+ if not isinstance(entries, list) or not all(isinstance(item, str) for item in entries):
+ raise ConfigError(f"{field_name}.{algo} must be a list of strings")
+ normalized[algo] = list(entries)
+ return normalized
+
+
+def _verify_tpm_event(
+ event_num: int,
+ pcr_index: int,
+ event_type: str,
+ *,
+ event: Any = None,
+ spec_version_major: int | None = None,
+) -> dict[str, Any]:
+ entry: dict[str, Any] = {
+ "EventNum": event_num,
+ "PCRIndex": pcr_index,
+ "EventType": event_type,
+ }
+ if spec_version_major is not None:
+ entry["SpecID"] = [{"specVersionMajor": spec_version_major}]
+ elif event is not None:
+ entry["Event"] = event
+ return entry
+
+
+def build_verify_tpm_events(scenario: dict[str, Any]) -> list[dict[str, Any]]:
+ """Build the default event list for verify_tpm_measurements.py."""
+ if "events" in scenario:
+ explicit_events = scenario["events"]
+ if not isinstance(explicit_events, list):
+ raise ConfigError("verify_tpm.events must be a list when provided")
+ return list(explicit_events)
+
+ spec_version_major = int(scenario.get("spec_version_major", 2))
+ post_code_events = _ensure_string_list(
+ scenario.get("post_code_events", ["BL_31", "SECURE_RT_EL3"]),
+ "verify_tpm.post_code_events",
+ )
+ secure_boot_vars = _ensure_string_list(
+ scenario.get("secure_boot_vars", ["SecureBoot", "PK", "KEK", "db", "dbx"]),
+ "verify_tpm.secure_boot_vars",
+ )
+ boot_variables = _ensure_string_list(
+ scenario.get("boot_variables", ["BootOrder", "Boot0001"]),
+ "verify_tpm.boot_variables",
+ )
+ handoff_events = _ensure_string_list(
+ scenario.get("handoff_events", ["SMBIOS"]),
+ "verify_tpm.handoff_events",
+ )
+ table_of_devices = _ensure_string_list(
+ scenario.get("table_of_devices", ["SYS_CONFIG_UART0"]),
+ "verify_tpm.table_of_devices",
+ )
+
+ separator_pcrs = scenario.get("separator_pcrs", list(range(8)))
+ if not isinstance(separator_pcrs, list) or not all(
+ isinstance(item, int) for item in separator_pcrs
+ ):
+ raise ConfigError("verify_tpm.separator_pcrs must be a list of integers")
+
+ boot_attempt = scenario.get("boot_attempt", "Calling EFI Application from Boot Option")
+ if boot_attempt is not None and not isinstance(boot_attempt, str):
+ raise ConfigError("verify_tpm.boot_attempt must be a string or null")
+
+ exit_boot_services = scenario_truthy(
+ scenario.get("exit_boot_services"),
+ default=True,
+ )
+
+ events: list[dict[str, Any]] = []
+ next_event_num = 1
+
+ events.append(
+ _verify_tpm_event(
+ next_event_num,
+ 0,
+ "EV_NO_ACTION",
+ spec_version_major=spec_version_major,
+ )
+ )
+ next_event_num += 1
+
+ for item in post_code_events:
+ events.append(_verify_tpm_event(next_event_num, 0, "EV_POST_CODE", event=item))
+ next_event_num += 1
+
+ for var_name in secure_boot_vars:
+ events.append(
+ _verify_tpm_event(
+ next_event_num,
+ 7,
+ "EV_EFI_VARIABLE_DRIVER_CONFIG",
+ event={"UnicodeName": var_name},
+ )
+ )
+ next_event_num += 1
+
+ for var_name in boot_variables:
+ event_type = "EV_EFI_VARIABLE_BOOT" if var_name == "BootOrder" else "EV_EFI_VARIABLE_BOOT2"
+ events.append(
+ _verify_tpm_event(
+ next_event_num,
+ 1,
+ event_type,
+ event={"UnicodeName": var_name},
+ )
+ )
+ next_event_num += 1
+
+ if boot_attempt is not None:
+ events.append(
+ _verify_tpm_event(
+ next_event_num,
+ 4,
+ "EV_EFI_ACTION",
+ event=boot_attempt,
+ )
+ )
+ next_event_num += 1
+
+ for item in handoff_events:
+ events.append(
+ _verify_tpm_event(next_event_num, 1, "EV_EFI_HANDOFF_TABLES", event=item)
+ )
+ next_event_num += 1
+
+ for pcr_index in separator_pcrs:
+ events.append(
+ _verify_tpm_event(
+ next_event_num,
+ pcr_index,
+ "EV_SEPARATOR",
+ event=f"sep{pcr_index}",
+ )
+ )
+ next_event_num += 1
+
+ for item in table_of_devices:
+ events.append(
+ _verify_tpm_event(next_event_num, 1, "EV_TABLE_OF_DEVICES", event=item)
+ )
+ next_event_num += 1
+
+ if exit_boot_services:
+ events.append(
+ _verify_tpm_event(
+ next_event_num,
+ 5,
+ "EV_EFI_ACTION",
+ event="Exit Boot Services Invocation",
+ )
+ )
+
+ return events
+
+
+def build_verify_tpm_scenario_case(
+ scenario: dict[str, Any],
+ work_dir: Path,
+) -> dict[str, Any]:
+ """Build runtime files and args for verify_tpm_measurements.py scenarios."""
+ pcr_banks = _normalize_verify_tpm_banks(scenario.get("pcrs"), "verify_tpm.pcrs")
+ event_pcrs = _normalize_verify_tpm_banks(
+ scenario.get("event_pcrs", pcr_banks),
+ "verify_tpm.event_pcrs",
+ )
+
+ event_log_raw = scenario.get("event_log_raw")
+ if event_log_raw is not None:
+ if not isinstance(event_log_raw, str):
+ raise ConfigError("verify_tpm.event_log_raw must be a string")
+ event_contents = event_log_raw
+ if event_contents and not event_contents.endswith("\n"):
+ event_contents += "\n"
+ else:
+ event_doc = {
+ "pcrs": event_pcrs,
+ "events": build_verify_tpm_events(scenario),
+ }
+ event_contents = json.dumps(event_doc, indent=2) + "\n"
+
+ pcr_contents = json.dumps(pcr_banks, indent=2) + "\n"
+ pcr_path = work_dir / "pcr.yaml"
+ event_path = work_dir / "event.yaml"
+
+ generated: dict[str, Any] = {
+ "args": [str(pcr_path), str(event_path)],
+ "text_files": {
+ "pcr.yaml": pcr_contents,
+ "event.yaml": event_contents,
+ },
+ }
+
+ mocks: dict[str, Any] = {}
+
+ event_log_open_error = scenario.get("event_log_open_error")
+ if event_log_open_error is not None:
+ if not isinstance(event_log_open_error, str) or not event_log_open_error.strip():
+ raise ConfigError(
+ "verify_tpm.event_log_open_error must be a non-empty string"
+ )
+ mocks["builtins.open"] = {
+ "factory": "mock_helpers.passthrough_router",
+ "inject_original_as": "real",
+ "kwargs": {
+ "label": "verify-tpm-open-router",
+ "rules": [
+ {
+ "label": "event-log-open-error",
+ "required": True,
+ "when": {
+ "args": {
+ 0: {"equals": str(event_path)},
+ 1: {"contains": "r"},
+ }
+ },
+ "raise": {
+ "type": "py:builtins.OSError",
+ "args": [event_log_open_error],
+ },
+ }
+ ],
+ },
+ }
+
+ event_log_yaml_error = scenario.get("event_log_yaml_error")
+ if event_log_yaml_error is not None:
+ if not isinstance(event_log_yaml_error, str) or not event_log_yaml_error.strip():
+ raise ConfigError(
+ "verify_tpm.event_log_yaml_error must be a non-empty string"
+ )
+ mocks["yaml.safe_load"] = {
+ "factory": "mock_helpers.passthrough_router",
+ "inject_original_as": "real",
+ "kwargs": {
+ "label": "verify-tpm-yaml-router",
+ "rules": [
+ {
+ "label": "event-log-yaml-error",
+ "required": True,
+ "when": {
+ "args": {
+ 0: {"contains": str(event_path)},
+ }
+ },
+ "raise": {
+ "type": "py:yaml.YAMLError",
+ "args": [event_log_yaml_error],
+ },
+ }
+ ],
+ },
+ }
+
+ if mocks:
+ generated["mocks"] = mocks
+
+ return generated
+
+
+CAPSULE_REPORT_GUID = "39b68c46-f7fb-441b-b6ec-16b0f69821f3"
+GLOBAL_VARIABLE_GUID = "8be4df61-93ca-11d2-aa0d-00e098032b8c"
+DEFAULT_ATTR_CAPSULE_MAX = 0x06
+DEFAULT_ATTR_CAPSULE_LAST = 0x07
+DEFAULT_ATTR_CAPSULE_NNNN = 0x07
+DEFAULT_ATTR_OS_INDICATIONS = 0x07
+FILE_CAPSULE_DELIVERY_SUPPORTED = 0x4
+
+
+def _parse_int_like(value: Any, field_name: str) -> int:
+ if isinstance(value, bool):
+ return int(value)
+ if isinstance(value, int):
+ return value
+ if isinstance(value, str):
+ text = value.strip()
+ if not text:
+ raise ConfigError(f"{field_name} must not be empty")
+ try:
+ return int(text, 0)
+ except ValueError as exc:
+ raise ConfigError(
+ f"{field_name} must be an integer or int-like string"
+ ) from exc
+ raise ConfigError(f"{field_name} must be an integer or int-like string")
+
+
+def _decode_hex_bytes(hex_text: str, field_name: str) -> bytes:
+ try:
+ return bytes.fromhex(hex_text)
+ except ValueError as exc:
+ raise ConfigError(f"{field_name} contains invalid hex data") from exc
+
+
+def _resolve_case_relative_path(work_dir: Path, raw_path: str) -> Path:
+ candidate = Path(raw_path)
+ return candidate if candidate.is_absolute() else work_dir / candidate
+
+
+def _build_capsule_named_var_bytes(
+ spec: Any,
+ *,
+ default_attrs: int,
+ field_name: str,
+) -> bytes:
+ if isinstance(spec, str):
+ return build_efi_var_bytes(default_attrs, build_char16_payload(spec))
+
+ if not isinstance(spec, dict):
+ raise ConfigError(f"{field_name} must be a string or mapping")
+
+ if "hex" in spec:
+ return _decode_hex_bytes(str(spec["hex"]), f"{field_name}.hex")
+
+ attrs = _parse_int_like(spec.get("attrs", default_attrs), f"{field_name}.attrs")
+
+ if "payload_hex" in spec:
+ payload = _decode_hex_bytes(
+ str(spec["payload_hex"]),
+ f"{field_name}.payload_hex",
+ )
+ else:
+ name = spec.get("name", spec.get("value", ""))
+ if not isinstance(name, str):
+ raise ConfigError(f"{field_name}.name must be a string")
+ extra_utf16 = spec.get("extra_utf16", "")
+ if not isinstance(extra_utf16, str):
+ raise ConfigError(f"{field_name}.extra_utf16 must be a string")
+ payload = build_char16_payload(name, extra_utf16)
+
+ return build_efi_var_bytes(attrs, payload)
+
+
+def _build_capsule_entry_bytes(
+ spec: Any,
+ *,
+ field_name: str,
+) -> tuple[str, bytes]:
+ if isinstance(spec, str):
+ return spec, build_efi_var_bytes(
+ DEFAULT_ATTR_CAPSULE_NNNN,
+ _decode_hex_bytes("AA", f"{field_name}.payload_hex"),
+ )
+
+ if not isinstance(spec, dict):
+ raise ConfigError(f"{field_name} entries must be strings or mappings")
+
+ name = spec.get("name")
+ if not isinstance(name, str) or not name.strip():
+ raise ConfigError(f"{field_name}.name must be a non-empty string")
+
+ if "hex" in spec:
+ payload_bytes = _decode_hex_bytes(str(spec["hex"]), f"{field_name}.hex")
+ return name, payload_bytes
+
+ attrs = _parse_int_like(
+ spec.get("attrs", DEFAULT_ATTR_CAPSULE_NNNN),
+ f"{field_name}.attrs",
+ )
+ payload_hex = str(spec.get("payload_hex", "AA"))
+ payload = _decode_hex_bytes(payload_hex, f"{field_name}.payload_hex")
+ return name, build_efi_var_bytes(attrs, payload)
+
+
+def _build_os_indications_bytes(spec: Any) -> bytes:
+ if isinstance(spec, bool):
+ value = FILE_CAPSULE_DELIVERY_SUPPORTED if spec else 0
+ return build_os_indications_var(value, attrs=DEFAULT_ATTR_OS_INDICATIONS)
+
+ if isinstance(spec, (int, str)):
+ value = _parse_int_like(spec, "capsule_vars.os_indications")
+ return build_os_indications_var(value, attrs=DEFAULT_ATTR_OS_INDICATIONS)
+
+ if not isinstance(spec, dict):
+ raise ConfigError(
+ "capsule_vars.os_indications must be a bool, int-like, or mapping"
+ )
+
+ if "hex" in spec:
+ return _decode_hex_bytes(
+ str(spec["hex"]),
+ "capsule_vars.os_indications.hex",
+ )
+
+ attrs = _parse_int_like(
+ spec.get("attrs", DEFAULT_ATTR_OS_INDICATIONS),
+ "capsule_vars.os_indications.attrs",
+ )
+ if "value" in spec:
+ value = _parse_int_like(
+ spec["value"],
+ "capsule_vars.os_indications.value",
+ )
+ else:
+ supported = scenario_truthy(spec.get("supported"), default=False)
+ value = FILE_CAPSULE_DELIVERY_SUPPORTED if supported else 0
+ return build_os_indications_var(value, attrs=attrs)
+
+
+def build_capsule_vars_scenario_case(
+ scenario: dict[str, Any],
+ work_dir: Path,
+) -> dict[str, Any]:
+ """Build efivarfs fixtures and default module patches for capsule variable checks."""
+ efivar_dir_raw = scenario.get("efivar_dir", "efivars")
+ if not isinstance(efivar_dir_raw, str) or not efivar_dir_raw.strip():
+ raise ConfigError("capsule_vars.efivar_dir must be a non-empty string")
+ log_file_raw = scenario.get("log_file", "capsule.log")
+ if not isinstance(log_file_raw, str) or not log_file_raw.strip():
+ raise ConfigError("capsule_vars.log_file must be a non-empty string")
+
+ efivar_path = _resolve_case_relative_path(work_dir, efivar_dir_raw)
+ log_file_path = _resolve_case_relative_path(work_dir, log_file_raw)
+ efivarfs_present = scenario_truthy(
+ scenario.get("efivarfs_present"),
+ default=True,
+ )
+
+ generated: dict[str, Any] = {
+ "patch_constants": {
+ "EFIVAR_PATH": str(efivar_path),
+ "LOG_FILE": str(log_file_path),
+ }
+ }
+
+ if not efivarfs_present:
+ return generated
+
+ generated["dir_structure"] = [{"path": str(efivar_path.relative_to(work_dir))}]
+ bin_files: dict[str, dict[str, str]] = {}
+
+ def add_var(filename: str, payload: bytes) -> None:
+ rel_path = str((efivar_path.relative_to(work_dir) / filename).as_posix())
+ bin_files[rel_path] = {"hex": payload.hex()}
+
+ os_indications = scenario.get("os_indications")
+ if os_indications is not None:
+ add_var(
+ f"OsIndicationsSupported-{GLOBAL_VARIABLE_GUID}",
+ _build_os_indications_bytes(os_indications),
+ )
+
+ capsule_max = scenario.get("capsule_max")
+ if capsule_max is not None:
+ add_var(
+ f"CapsuleMax-{CAPSULE_REPORT_GUID}",
+ _build_capsule_named_var_bytes(
+ capsule_max,
+ default_attrs=DEFAULT_ATTR_CAPSULE_MAX,
+ field_name="capsule_vars.capsule_max",
+ ),
+ )
+
+ capsule_last = scenario.get("capsule_last")
+ if capsule_last is not None:
+ add_var(
+ f"CapsuleLast-{CAPSULE_REPORT_GUID}",
+ _build_capsule_named_var_bytes(
+ capsule_last,
+ default_attrs=DEFAULT_ATTR_CAPSULE_LAST,
+ field_name="capsule_vars.capsule_last",
+ ),
+ )
+
+ capsule_entries = scenario.get("capsule_entries", [])
+ if capsule_entries is None:
+ capsule_entries = []
+ if not isinstance(capsule_entries, list):
+ raise ConfigError("capsule_vars.capsule_entries must be a list")
+
+ for index, entry in enumerate(capsule_entries, start=1):
+ name, payload = _build_capsule_entry_bytes(
+ entry,
+ field_name=f"capsule_vars.capsule_entries[{index}]",
+ )
+ add_var(f"{name}-{CAPSULE_REPORT_GUID}", payload)
+
+ if bin_files:
+ generated["bin_files"] = bin_files
+
+ return generated
+
+
+def build_runtime_device_mapping_scenario_case(
+ scenario: dict[str, Any],
+ work_dir: Path,
+) -> dict[str, Any]:
+ """Build DTS and memmap fixtures for runtime_device_mapping_conflict_checker.py."""
+ text_files: dict[str, str] = {}
+ bin_files: dict[str, dict[str, str]] = {}
+ mocks: dict[str, Any] = {}
+
+ dts_path = work_dir / "device_tree.dts"
+ memmap_path = work_dir / "memmap.log"
+ log_path = work_dir / "runtime_device_mapping_conflict_test.log"
+
+ dts = scenario.get("dts")
+ memmap = scenario.get("memmap")
+ memmap_hex = scenario.get("memmap_hex")
+ missing_dts = scenario_truthy(scenario.get("missing_dts"), default=False)
+ missing_memmap = scenario_truthy(scenario.get("missing_memmap"), default=False)
+ log_open_error = scenario.get("log_open_error")
+
+ if missing_dts and dts is not None:
+ raise ConfigError(
+ "runtime_device_mapping cannot define both dts and missing_dts"
+ )
+ if missing_memmap and (memmap is not None or memmap_hex is not None):
+ raise ConfigError(
+ "runtime_device_mapping cannot define memmap/memmap_hex together with "
+ "missing_memmap"
+ )
+ if log_open_error is not None and (
+ not isinstance(log_open_error, str) or not log_open_error.strip()
+ ):
+ raise ConfigError(
+ "runtime_device_mapping.log_open_error must be a non-empty string"
+ )
+
+ if dts is not None:
+ if not isinstance(dts, str):
+ raise ConfigError("runtime_device_mapping.dts must be a string")
+ text_files["device_tree.dts"] = dts
+
+ if memmap is not None and memmap_hex is not None:
+ raise ConfigError(
+ "runtime_device_mapping cannot define both memmap and memmap_hex"
+ )
+
+ if memmap is not None:
+ if not isinstance(memmap, str):
+ raise ConfigError("runtime_device_mapping.memmap must be a string")
+ text_files["memmap.log"] = memmap
+
+ if memmap_hex is not None:
+ if not isinstance(memmap_hex, str):
+ raise ConfigError("runtime_device_mapping.memmap_hex must be a string")
+ bin_files["memmap.log"] = {"hex": memmap_hex}
+
+ if log_open_error is not None:
+ mocks["pathlib.Path.open"] = {
+ "factory": "mock_helpers.passthrough_router",
+ "inject_original_as": "real",
+ "kwargs": {
+ "label": "runtime-device-mapping-open-router",
+ "rules": [
+ {
+ "label": "runtime-log-open-error",
+ "required": True,
+ "when": {
+ "args": {
+ 0: {"equals": log_path},
+ }
+ },
+ "raise": {
+ "type": "py:builtins.OSError",
+ "args": [log_open_error],
+ },
+ }
+ ],
+ },
+ }
+
+ generated: dict[str, Any] = {
+ "patch_constants": {
+ "DTS_PATH": dts_path,
+ "MEMMAP_PATH": memmap_path,
+ "OUT_LOG_PATH": log_path,
+ }
+ }
+
+ if text_files:
+ generated["text_files"] = text_files
+
+ if bin_files:
+ generated["bin_files"] = bin_files
+
+ if mocks:
+ generated["mocks"] = mocks
+
+ return generated
diff --git a/common/acs_test_framework_runner/mock_loader_parser_scenarios.py b/common/acs_test_framework_runner/mock_loader_parser_scenarios.py
new file mode 100644
index 00000000..5dd3fd22
--- /dev/null
+++ b/common/acs_test_framework_runner/mock_loader_parser_scenarios.py
@@ -0,0 +1,344 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+try: # Support package imports and direct harness module loading.
+ from .mock_loader import ConfigError
+ from .mock_loader import _resolve_runner_module_from_stack
+except ImportError: # pragma: no cover - exercised by flat-module harness imports.
+ from mock_loader import ConfigError
+ from mock_loader import _resolve_runner_module_from_stack
+
+
+def _readable_text_block(lines: list[str]) -> str:
+ if not lines:
+ return ""
+ return "\n".join(lines) + "\n"
+
+
+def _normalize_simple_key_value_text(value: Any, field_name: str) -> str:
+ if value is None:
+ return ""
+ if isinstance(value, str):
+ return value if value.endswith("\n") or not value else value + "\n"
+ if not isinstance(value, dict):
+ raise ConfigError(f"{field_name} must be a string or mapping")
+ return _readable_text_block([f"{key}: {item}" for key, item in value.items()])
+
+
+def _normalize_json_text(value: Any, field_name: str) -> str:
+ if isinstance(value, str):
+ return value if value.endswith("\n") or not value else value + "\n"
+ try:
+ return json.dumps(value, indent=4) + "\n"
+ except TypeError as exc:
+ raise ConfigError(
+ f"{field_name} must be a string or JSON-serializable value"
+ ) from exc
+
+
+def _normalize_text_block(value: Any, field_name: str) -> str:
+ if not isinstance(value, str):
+ raise ConfigError(f"{field_name} must be a string")
+ return value if value.endswith("\n") or not value else value + "\n"
+
+
+def _build_merge_jsons_acs_info_payload(value: Any) -> dict[str, Any]:
+ if isinstance(value, str):
+ return {"ACS Results Summary": {"Overall Compliance Result": value}}
+ if not isinstance(value, dict):
+ raise ConfigError("merge_jsons.acs_info must be a string or mapping")
+
+ if "ACS Results Summary" in value:
+ summary = value["ACS Results Summary"]
+ if not isinstance(summary, dict):
+ raise ConfigError(
+ "merge_jsons.acs_info['ACS Results Summary'] must be a mapping"
+ )
+ return dict(value)
+
+ return {"ACS Results Summary": dict(value)}
+
+
+def _build_dmidecode_text(spec: Any) -> str:
+ if isinstance(spec, str):
+ return spec if spec.endswith("\n") or not spec else spec + "\n"
+ if not isinstance(spec, dict):
+ raise ConfigError("acs_info.dmidecode must be a string or mapping")
+ firmware = str(spec.get("firmware", spec.get("firmware_version", "Unknown")))
+ vendor = str(spec.get("vendor", spec.get("manufacturer", "Unknown")))
+ product = str(spec.get("product", spec.get("system_name", "Unknown")))
+ family = str(spec.get("family", spec.get("soc_family", "Unknown")))
+ return _readable_text_block([
+ "Handle 0x0000, DMI type 0, 24 bytes",
+ "BIOS Information",
+ f" Version: {firmware}",
+ "",
+ "Handle 0x0001, DMI type 1, 27 bytes",
+ "System Information",
+ f" Manufacturer: {vendor}",
+ f" Product Name: {product}",
+ f" Family: {family}",
+ ])
+
+
+def build_acs_info_scenario_case(
+ scenario: dict[str, Any],
+ work_dir: Path,
+) -> dict[str, Any]:
+ """Build fixture files and args for acs_info.py."""
+ output_dir = str(scenario.get("output_dir", "out"))
+ args = ["--dmidecode_log", str(work_dir / "dmidecode.log")]
+ text_files: dict[str, str] = {
+ "dmidecode.log": _build_dmidecode_text(scenario.get("dmidecode", {})),
+ }
+ bin_files: dict[str, dict[str, str]] = {}
+
+ acs_config = scenario.get("acs_config")
+ if acs_config is not None:
+ text_files["acs_config.txt"] = _normalize_simple_key_value_text(
+ acs_config,
+ "acs_info.acs_config",
+ )
+ args.extend(["--acs_config_path", str(work_dir / "acs_config.txt")])
+
+ system_config = scenario.get("system_config")
+ if system_config is not None:
+ text_files["system_config.txt"] = _normalize_simple_key_value_text(
+ system_config,
+ "acs_info.system_config",
+ )
+ args.extend(["--system_config_path", str(work_dir / "system_config.txt")])
+
+ ipmitool = scenario.get("ipmitool")
+ if ipmitool is not None:
+ if isinstance(ipmitool, str):
+ text_files["ipmitool.log"] = (
+ ipmitool
+ if ipmitool.endswith("\n") or not ipmitool
+ else ipmitool + "\n"
+ )
+ elif isinstance(ipmitool, dict):
+ fw_rev = ipmitool.get(
+ "firmware_revision",
+ ipmitool.get("Firmware Revision", "Unknown"),
+ )
+ text_files["ipmitool.log"] = _readable_text_block(
+ [
+ "Device ID : 32",
+ f"Firmware Revision : {fw_rev}",
+ ]
+ )
+ else:
+ raise ConfigError("acs_info.ipmitool must be a string or mapping")
+ args.extend(["--ipmitool_log", str(work_dir / "ipmitool.log")])
+
+ uefi_version = scenario.get("uefi_version")
+ if uefi_version is not None:
+ encoding = "utf-8"
+ value = uefi_version
+ if isinstance(uefi_version, dict):
+ value = str(uefi_version.get("text", ""))
+ encoding = str(uefi_version.get("encoding", "utf-8"))
+ elif not isinstance(uefi_version, str):
+ raise ConfigError("acs_info.uefi_version must be a string or mapping")
+ payload = str(value)
+ if payload and not payload.endswith("\n"):
+ payload += "\n"
+ bin_files["uefi_version.log"] = {"hex": payload.encode(encoding).hex()}
+ args.extend(["--uefi_version_log", str(work_dir / "uefi_version.log")])
+
+ args.extend(["--output_dir", str(work_dir / output_dir)])
+ generated: dict[str, Any] = {"args": args, "text_files": text_files}
+ if bin_files:
+ generated["bin_files"] = bin_files
+ return generated
+
+
+def _build_merge_jsons_module_main(
+ *,
+ work_dir: Path,
+ input_files: list[str],
+ output_files: list[str],
+ dt_or_sr_mode: str | None = None,
+ yocto_flag_path: str | None = None,
+) -> Any:
+ def _run_merge_jsons_module_main() -> int:
+ module = _resolve_runner_module_from_stack()
+
+ if yocto_flag_path is not None:
+ setattr(
+ module,
+ "YOCTO_FLAG_PATH",
+ str((work_dir / yocto_flag_path).resolve()),
+ )
+ if dt_or_sr_mode is not None:
+ setattr(module, "DT_OR_SR_MODE", dt_or_sr_mode)
+
+ resolved_inputs = [
+ str((work_dir / input_name).resolve()) for input_name in input_files
+ ]
+ for output_name in output_files:
+ module.merge_json_files(
+ resolved_inputs,
+ str((work_dir / output_name).resolve()),
+ )
+ return 0
+
+ return _run_merge_jsons_module_main
+
+
+def build_merge_jsons_scenario_case(
+ scenario: dict[str, Any],
+ work_dir: Path,
+) -> dict[str, Any]:
+ text_files: dict[str, str] = {}
+ input_names: list[str] = []
+
+ def _append_input(
+ name: Any,
+ *,
+ field_name: str,
+ content: str | None = None,
+ ) -> None:
+ if not isinstance(name, str) or not name.strip():
+ raise ConfigError(f"{field_name} must be a non-empty string")
+ if content is not None:
+ if name in text_files:
+ raise ConfigError(f"Duplicate merge_jsons input file: {name}")
+ text_files[name] = content
+ input_names.append(name)
+
+ acs_info = scenario.get("acs_info")
+ if acs_info is not None:
+ acs_info_name = scenario.get("acs_info_name", "acs_info.json")
+ _append_input(
+ acs_info_name,
+ field_name="merge_jsons.acs_info_name",
+ content=_normalize_json_text(
+ _build_merge_jsons_acs_info_payload(acs_info),
+ "merge_jsons.acs_info",
+ ),
+ )
+
+ input_files = scenario.get("input_files")
+ if input_files is not None:
+ if not isinstance(input_files, dict):
+ raise ConfigError("merge_jsons.input_files must be a mapping")
+ for name, content in input_files.items():
+ _append_input(
+ name,
+ field_name=f"merge_jsons.input_files[{name!r}]",
+ content=_normalize_json_text(
+ content,
+ f"merge_jsons.input_files[{name!r}]",
+ ),
+ )
+
+ invalid_files = scenario.get("invalid_files")
+ if invalid_files is not None:
+ if not isinstance(invalid_files, dict):
+ raise ConfigError("merge_jsons.invalid_files must be a mapping")
+ for name, content in invalid_files.items():
+ _append_input(
+ name,
+ field_name=f"merge_jsons.invalid_files[{name!r}]",
+ content=_normalize_text_block(
+ content,
+ f"merge_jsons.invalid_files[{name!r}]",
+ ),
+ )
+
+ missing_input_files = scenario.get("missing_input_files", [])
+ if missing_input_files is not None:
+ if not isinstance(missing_input_files, list) or not all(
+ isinstance(item, str) and item.strip() for item in missing_input_files
+ ):
+ raise ConfigError(
+ "merge_jsons.missing_input_files must be a list of strings"
+ )
+ for name in missing_input_files:
+ _append_input(
+ name,
+ field_name="merge_jsons.missing_input_files[]",
+ )
+
+ if not input_names:
+ raise ConfigError(
+ "merge_jsons scenario requires at least one input via "
+ "acs_info, input_files, invalid_files, or missing_input_files"
+ )
+
+ raw_output_files = scenario.get("output_files", scenario.get("output_file", "merged.json"))
+ if isinstance(raw_output_files, str):
+ output_files = [raw_output_files]
+ elif isinstance(raw_output_files, list) and all(
+ isinstance(item, str) and item.strip() for item in raw_output_files
+ ):
+ output_files = list(raw_output_files)
+ else:
+ raise ConfigError(
+ "merge_jsons.output_files must be a string or a list of strings"
+ )
+
+ dt_or_sr_mode = scenario.get("dt_or_sr_mode")
+ if dt_or_sr_mode is not None:
+ if not isinstance(dt_or_sr_mode, str) or dt_or_sr_mode not in {"DT", "SR"}:
+ raise ConfigError("merge_jsons.dt_or_sr_mode must be 'DT' or 'SR'")
+
+ yocto_flag_path = scenario.get("yocto_flag_path")
+ if yocto_flag_path is not None:
+ if not isinstance(yocto_flag_path, str) or not yocto_flag_path.strip():
+ raise ConfigError(
+ "merge_jsons.yocto_flag_path must be a non-empty string"
+ )
+ text_files.setdefault(yocto_flag_path, "")
+
+ generated_case = {
+ "args": [
+ str((work_dir / output_files[0]).resolve()),
+ *[str((work_dir / input_name).resolve()) for input_name in input_names],
+ ]
+ if len(output_files) == 1
+ else [],
+ "patch_constants": {
+ "main": _build_merge_jsons_module_main(
+ work_dir=work_dir,
+ input_files=input_names,
+ output_files=output_files,
+ dt_or_sr_mode=dt_or_sr_mode,
+ yocto_flag_path=yocto_flag_path,
+ ),
+ },
+ "text_files": text_files,
+ }
+ return generated_case
+
+
+def build_extract_capsule_fw_version_scenario_case(
+ scenario: dict[str, Any],
+ work_dir: Path,
+) -> dict[str, Any]:
+ """Build fixture files and args for extract_capsule_fw_version.py."""
+ pattern = scenario.get("pattern")
+ if not isinstance(pattern, str) or not pattern.strip():
+ raise ConfigError("extract_capsule_fw_version.pattern must be a non-empty string")
+
+ input_name = str(scenario.get("input_name", "input.log"))
+ raw_input = scenario.get("input_text")
+ if raw_input is None:
+ lines = scenario.get("input_lines", [])
+ if not isinstance(lines, list) or not all(isinstance(item, str) for item in lines):
+ raise ConfigError("extract_capsule_fw_version.input_lines must be a list of strings")
+ raw_input = _readable_text_block(lines)
+ elif not isinstance(raw_input, str):
+ raise ConfigError("extract_capsule_fw_version.input_text must be a string")
+ if raw_input and not raw_input.endswith("\n"):
+ raw_input += "\n"
+
+ return {
+ "args": [pattern, str(work_dir / input_name)],
+ "text_files": {input_name: raw_input},
+ }
diff --git a/common/acs_test_framework_runner/pytest_runner.py b/common/acs_test_framework_runner/pytest_runner.py
new file mode 100644
index 00000000..7bad92bc
--- /dev/null
+++ b/common/acs_test_framework_runner/pytest_runner.py
@@ -0,0 +1,631 @@
+from __future__ import annotations
+
+import argparse
+import shutil
+import subprocess
+import sys
+import traceback
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import Any
+
+try: # Support package imports and direct harness module loading.
+ from .case_data_builders import CaseBuildError
+ from .runner_checks import (
+ ConfigError,
+ RunCaseOptions,
+ SkipCase,
+ TestMeta,
+ TestOutcome,
+ create_outcome,
+ detect_project_root,
+ format_outcome_message,
+ load_yaml_config,
+ normalize_suites,
+ resolve_target_path,
+ run_single_check,
+ sanitize_name,
+ )
+ from .runner_reporting import (
+ append_combined_case_log,
+ append_run_header,
+ build_config_error_outcome,
+ build_report_path,
+ cleanup_old_pytest_xml_reports,
+ create_placeholder_xml,
+ print_group_summary,
+ remove_placeholder_xml,
+ sanitize_xml_text,
+ write_case_log,
+ write_junit_xml,
+ )
+except ImportError: # pragma: no cover - exercised by flat-module harness imports.
+ from case_data_builders import CaseBuildError
+ from runner_checks import (
+ ConfigError,
+ RunCaseOptions,
+ SkipCase,
+ TestMeta,
+ TestOutcome,
+ create_outcome,
+ detect_project_root,
+ format_outcome_message,
+ load_yaml_config,
+ normalize_suites,
+ resolve_target_path,
+ run_single_check,
+ sanitize_name,
+ )
+ from runner_reporting import (
+ append_combined_case_log,
+ append_run_header,
+ build_config_error_outcome,
+ build_report_path,
+ cleanup_old_pytest_xml_reports,
+ create_placeholder_xml,
+ print_group_summary,
+ remove_placeholder_xml,
+ sanitize_xml_text,
+ write_case_log,
+ write_junit_xml,
+ )
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+PROJECT_ROOT = detect_project_root(SCRIPT_DIR)
+HARNESS_DIR = PROJECT_ROOT / "common" / "acs_test_framework_runner"
+TEST_YAML_DIR = PROJECT_ROOT / "common" / "acs_test_framework_manifests"
+REPORTS_DIR = PROJECT_ROOT / "common" / "reports"
+SUPPORTED_SUFFIXES = {".yaml", ".yml"}
+IN_PROCESS_CASE_TYPES = {"py_function", "module_main_with_env", "module_cli"}
+
+
+def discover_yaml_files() -> list[Path]:
+ if not TEST_YAML_DIR.exists() or not TEST_YAML_DIR.is_dir():
+ return []
+ return sorted(
+ path
+ for path in TEST_YAML_DIR.rglob("*")
+ if path.is_file() and path.suffix.lower() in SUPPORTED_SUFFIXES
+ )
+
+
+def get_group_name(yaml_file: Path) -> str:
+ rel_path = yaml_file.relative_to(TEST_YAML_DIR)
+ return rel_path.parts[0] if len(rel_path.parts) > 1 else yaml_file.stem
+
+
+
+def get_file_work_dir(suite_name: str, file_entry: str) -> Path:
+ return (
+ REPORTS_DIR
+ / "_work"
+ / sanitize_name(suite_name)
+ / sanitize_name(Path(file_entry).stem)
+ )
+
+
+def requires_serial_case_execution(suite_cases: list[dict[str, Any]]) -> bool:
+ return any(
+ str(case_def.get("type", "cli")) in IN_PROCESS_CASE_TYPES
+ for case_def in suite_cases
+ )
+
+
+def run_case(
+ suite_name: str,
+ file_entry: str,
+ case_index: int,
+ case_def: dict[str, Any],
+ options: RunCaseOptions | None = None,
+) -> TestOutcome:
+ if options is None:
+ options = RunCaseOptions()
+
+ file_path = resolve_target_path(file_entry)
+ case_name = case_def["name"].strip()
+ testcase_name = f"{suite_name}::{Path(file_entry).name}::{case_name}"
+ case_type = str(case_def.get("type", "cli"))
+
+ file_work_dir = get_file_work_dir(suite_name, file_entry)
+ work_dir = file_work_dir / sanitize_name(f"{case_index}_{case_name}")
+ work_dir.mkdir(parents=True, exist_ok=True)
+
+ effective_case = dict(case_def)
+ if options.suite_command is not None and "command" not in effective_case:
+ effective_case["command"] = options.suite_command
+
+ meta = TestMeta(
+ suite_name=suite_name,
+ phase="case",
+ test_type=case_type,
+ )
+
+ def persist_case_logs(status: str, outcome: TestOutcome) -> None:
+ case_description = effective_case.get("description")
+ append_combined_case_log(
+ file_work_dir=file_work_dir,
+ testcase_name=testcase_name,
+ status=status,
+ message=outcome.message,
+ details=outcome.details,
+ description=case_description,
+ )
+ write_case_log(
+ case_work_dir=work_dir,
+ testcase_name=testcase_name,
+ status=status,
+ message=outcome.message,
+ details=outcome.details,
+ description=case_description,
+ )
+
+ try:
+ passed, message, details, is_error = run_single_check(
+ file_path, effective_case, work_dir
+ )
+
+ warn_only = bool(effective_case.get("warn_only", False))
+ final_passed = passed
+ final_warning = False
+
+ if warn_only and not passed and not is_error:
+ final_passed = True
+ final_warning = True
+
+ outcome = create_outcome(
+ testcase_name=testcase_name,
+ file_path=file_entry,
+ passed=final_passed,
+ message=sanitize_xml_text(message),
+ meta=meta,
+ details=sanitize_xml_text(details),
+ error=is_error,
+ warning=final_warning,
+ )
+
+ status = (
+ "ERROR"
+ if is_error
+ else (
+ "WARNING"
+ if outcome.warning
+ else ("PASS" if outcome.passed else "FAIL")
+ )
+ )
+ persist_case_logs(status, outcome)
+ return outcome
+ except SkipCase as exc:
+ outcome = create_outcome(
+ testcase_name=testcase_name,
+ file_path=file_entry,
+ passed=True,
+ message=sanitize_xml_text(str(exc)),
+ meta=meta,
+ details=sanitize_xml_text(
+ f"Case skipped in work directory: {work_dir}\nReason: {exc}"
+ ),
+ skipped=True,
+ )
+ persist_case_logs("SKIPPED", outcome)
+ return outcome
+ except (ConfigError, CaseBuildError) as exc:
+ outcome = create_outcome(
+ testcase_name=testcase_name,
+ file_path=file_entry,
+ passed=False,
+ message=sanitize_xml_text(
+ format_outcome_message("Configuration error", str(exc))
+ ),
+ meta=meta,
+ details=sanitize_xml_text(traceback.format_exc()),
+ error=True,
+ )
+ persist_case_logs("ERROR", outcome)
+ return outcome
+ except Exception as exc: # pylint: disable=broad-exception-caught
+ outcome = create_outcome(
+ testcase_name=testcase_name,
+ file_path=file_entry,
+ passed=False,
+ message=sanitize_xml_text(
+ format_outcome_message("Unhandled exception", str(exc))
+ ),
+ meta=meta,
+ details=sanitize_xml_text(traceback.format_exc()),
+ error=True,
+ )
+ persist_case_logs("ERROR", outcome)
+ return outcome
+
+
+def git_changed_paths_for_query(args: list[str]) -> set[Path]:
+ result = subprocess.run(
+ ["git", *args],
+ cwd=PROJECT_ROOT,
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+ if result.returncode != 0:
+ return set()
+
+ paths: set[Path] = set()
+ for line in result.stdout.splitlines():
+ line = line.strip()
+ if not line:
+ continue
+ paths.add((PROJECT_ROOT / line).resolve())
+ return paths
+
+
+def get_recently_changed_paths() -> set[Path]:
+ changed: set[Path] = set()
+ changed.update(git_changed_paths_for_query(["diff", "--name-only"]))
+ changed.update(git_changed_paths_for_query(["diff", "--cached", "--name-only"]))
+ changed.update(
+ git_changed_paths_for_query(["ls-files", "--others", "--exclude-standard"])
+ )
+ return changed
+
+
+def path_is_within(path: Path, root: Path) -> bool:
+ try:
+ path.relative_to(root)
+ except ValueError:
+ return False
+ return True
+
+
+def harness_sources_changed(changed_paths: set[Path]) -> bool:
+ for changed_path in changed_paths:
+ if not path_is_within(changed_path, HARNESS_DIR):
+ continue
+ if "__pycache__" in changed_path.parts:
+ continue
+ return True
+ return False
+
+
+def normalize_target_for_matching(file_entry: str) -> Path:
+ return resolve_target_path(file_entry)
+
+
+def format_yaml_config_warning(yaml_file: Path, message: str) -> str:
+ rel_path = yaml_file.relative_to(PROJECT_ROOT).as_posix()
+ return f"Skipping {rel_path} due to configuration error: {message}"
+
+
+def load_yaml_suites_for_selection(
+ yaml_file: Path,
+) -> tuple[list[dict[str, Any]] | None, str | None]:
+ try:
+ config = load_yaml_config(yaml_file)
+ suites = normalize_suites(config)
+ except ConfigError as exc:
+ return None, format_yaml_config_warning(yaml_file, str(exc))
+ return suites, None
+
+
+def get_yaml_target_entries(yaml_file: Path) -> tuple[set[str], str | None]:
+ suites, warning = load_yaml_suites_for_selection(yaml_file)
+ if suites is None:
+ return set(), warning
+
+ targets: set[str] = set()
+ for suite in suites:
+ for file_entry in suite["files"]:
+ targets.add(Path(file_entry).as_posix())
+ return targets, None
+
+
+def yaml_targets_changed_files(
+ yaml_file: Path,
+ changed_paths: set[Path],
+) -> tuple[set[str], str | None]:
+ suites, warning = load_yaml_suites_for_selection(yaml_file)
+ if suites is None:
+ return set(), warning
+
+ matched: set[str] = set()
+ for suite in suites:
+ for file_entry in suite["files"]:
+ target_path = normalize_target_for_matching(file_entry)
+ if target_path in changed_paths:
+ matched.add(Path(file_entry).as_posix())
+ return matched, None
+
+
+def select_yaml_runs(
+ yaml_files: list[Path],
+) -> tuple[list[tuple[Path, set[str]]], list[str]]:
+ changed_paths = get_recently_changed_paths()
+
+ if not changed_paths:
+ return [], []
+
+ run_all_yaml_groups = harness_sources_changed(changed_paths)
+ selected_map: dict[Path, set[str]] = {}
+ config_warnings: list[str] = []
+
+ for yaml_file in yaml_files:
+ yaml_abs = yaml_file.resolve()
+ matched_targets, warning = yaml_targets_changed_files(yaml_file, changed_paths)
+ if warning is not None:
+ config_warnings.append(warning)
+
+ if run_all_yaml_groups or yaml_abs in changed_paths:
+ yaml_targets, target_warning = get_yaml_target_entries(yaml_file)
+ if target_warning is not None and target_warning not in config_warnings:
+ config_warnings.append(target_warning)
+ matched_targets.update(yaml_targets)
+
+ if matched_targets:
+ selected_map[yaml_file] = matched_targets
+
+ return list(selected_map.items()), config_warnings
+
+
+def select_yaml_runs_for_target(
+ yaml_files: list[Path],
+ target: str,
+) -> tuple[list[tuple[Path, set[str]]], list[str]]:
+ target_path = resolve_target_path(target)
+ selected_runs: list[tuple[Path, set[str]]] = []
+ config_warnings: list[str] = []
+
+ for yaml_file in yaml_files:
+ suites, warning = load_yaml_suites_for_selection(yaml_file)
+ if suites is None:
+ if warning is not None:
+ config_warnings.append(warning)
+ continue
+
+ matched_targets: set[str] = set()
+ for suite in suites:
+ for file_entry in suite["files"]:
+ if resolve_target_path(file_entry) == target_path:
+ matched_targets.add(Path(file_entry).as_posix())
+
+ if matched_targets:
+ selected_runs.append((yaml_file, matched_targets))
+
+ return selected_runs, config_warnings
+
+
+
+def run_yaml(
+ yaml_file: Path,
+ selected_targets: set[str],
+ jobs: int = 4,
+) -> int:
+ REPORTS_DIR.mkdir(parents=True, exist_ok=True)
+
+ group_name = get_group_name(yaml_file)
+ xml_report = build_report_path(group_name, yaml_file)
+
+ print(f"\n[INFO] Running group : {group_name}")
+ print(
+ f"[INFO] YAML file : "
+ f"{yaml_file.relative_to(PROJECT_ROOT).as_posix()}"
+ )
+ print(
+ f"[INFO] XML report : "
+ f"{xml_report.relative_to(PROJECT_ROOT).as_posix()}"
+ )
+
+ try:
+ config = load_yaml_config(yaml_file)
+ suites = normalize_suites(config)
+ except ConfigError as exc:
+ outcome = build_config_error_outcome(
+ yaml_file=yaml_file,
+ message=format_outcome_message("Configuration error", str(exc)),
+ details=traceback.format_exc(),
+ )
+ write_junit_xml(xml_report, group_name, yaml_file, [outcome])
+ print_group_summary(group_name, [outcome], xml_report)
+ return 1
+
+ outcomes: list[TestOutcome] = []
+
+ for suite_index, suite in enumerate(suites, start=1):
+ suite_name = suite["name"]
+ run_case_options = RunCaseOptions(
+ suite_command=suite.get("command"),
+ )
+ suite_files = suite["files"]
+ suite_cases = suite["cases"]
+
+ filtered_files = []
+ for file_entry in suite_files:
+ normalized_file_entry = Path(file_entry).as_posix()
+ if normalized_file_entry in selected_targets:
+ filtered_files.append(file_entry)
+
+ if not filtered_files:
+ continue
+
+ print(f"[INFO] Suite : {suite_name}")
+
+ if not suite_cases:
+ outcomes.append(
+ create_outcome(
+ testcase_name=f"{suite_name}::no_cases",
+ file_path="",
+ passed=False,
+ message="Suite has no cases defined",
+ meta=TestMeta(
+ suite_name=suite_name,
+ phase="suite",
+ test_type="config",
+ ),
+ details=f"suites[{suite_index}] has an empty 'cases' list.",
+ error=True,
+ )
+ )
+ continue
+
+ for file_entry in filtered_files:
+ print(f"[INFO] Target file : {file_entry}")
+
+ file_work_dir = get_file_work_dir(suite_name, file_entry)
+
+ if file_work_dir.exists():
+ shutil.rmtree(file_work_dir)
+ file_work_dir.mkdir(parents=True, exist_ok=True)
+
+ append_run_header(
+ file_work_dir=file_work_dir,
+ suite_name=suite_name,
+ yaml_file=yaml_file,
+ targets=[file_entry],
+ )
+
+ case_jobs = list(enumerate(suite_cases, start=1))
+ file_jobs = 1 if requires_serial_case_execution(suite_cases) else jobs
+
+ if file_jobs <= 1:
+ for case_index, case_def in case_jobs:
+ outcomes.append(
+ run_case(
+ suite_name=suite_name,
+ file_entry=file_entry,
+ case_index=case_index,
+ case_def=case_def,
+ options=run_case_options,
+ )
+ )
+ else:
+ case_results: list[tuple[int, TestOutcome]] = []
+ with ThreadPoolExecutor(max_workers=file_jobs) as executor:
+ future_map = {
+ executor.submit(
+ run_case,
+ suite_name,
+ file_entry,
+ case_index,
+ case_def,
+ run_case_options,
+ ): case_index
+ for case_index, case_def in case_jobs
+ }
+ for future in as_completed(future_map):
+ case_index = future_map[future]
+ case_results.append((case_index, future.result()))
+
+ for _case_index, outcome in sorted(
+ case_results,
+ key=lambda item: item[0],
+ ):
+ outcomes.append(outcome)
+
+ if not outcomes:
+ return 0
+
+ write_junit_xml(xml_report, group_name, yaml_file, outcomes)
+ print_group_summary(group_name, outcomes, xml_report)
+
+ return 0 if all(item.passed or item.skipped or item.warning for item in outcomes) else 1
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(
+ description="Run YAML-driven tests for impacted files or a manual target."
+ )
+ parser.add_argument(
+ "--target",
+ help="Run tests only for the specified Python file target from YAML suites",
+ )
+ parser.add_argument(
+ "--jobs",
+ type=int,
+ default=4,
+ help="Number of YAML test cases to run in parallel (default: 4)",
+ )
+ args = parser.parse_args()
+
+ jobs = max(1, args.jobs)
+
+ REPORTS_DIR.mkdir(parents=True, exist_ok=True)
+ cleanup_old_pytest_xml_reports()
+
+ yaml_files = discover_yaml_files()
+
+ if not yaml_files:
+ reason = (
+ f"No YAML files found in "
+ f"{TEST_YAML_DIR.relative_to(PROJECT_ROOT).as_posix()}"
+ )
+ print(f"WARNING: {reason}")
+ create_placeholder_xml(reason)
+ return 0
+
+ remove_placeholder_xml()
+
+ if args.target:
+ print(f"[INFO] Manual target override enabled: {args.target}")
+ selected_runs, config_warnings = select_yaml_runs_for_target(
+ yaml_files,
+ args.target,
+ )
+ else:
+ selected_runs, config_warnings = select_yaml_runs(yaml_files)
+
+ for warning in config_warnings:
+ print(f"[WARNING] {warning}")
+
+ if not selected_runs:
+ if args.target:
+ if config_warnings:
+ message = (
+ f"No valid YAML test groups found for manual target: {args.target}"
+ )
+ print(f"[ERROR] {message}")
+ print(
+ f"[INFO] Reports written to: "
+ f"{REPORTS_DIR.relative_to(PROJECT_ROOT).as_posix()}"
+ )
+ create_placeholder_xml(message)
+ return 1
+ message = f"No YAML test groups found for manual target: {args.target}"
+ else:
+ message = "No impacted YAML test groups found for changed files."
+
+ print(f"[INFO] {message}")
+ print(
+ f"[INFO] Reports written to: "
+ f"{REPORTS_DIR.relative_to(PROJECT_ROOT).as_posix()}"
+ )
+ create_placeholder_xml(message)
+ return 0
+
+ if args.target:
+ print("[INFO] Running YAML test groups for manual target:")
+ else:
+ print("[INFO] Running only impacted YAML test groups:")
+
+ for yaml_file, selected_targets in selected_runs:
+ print(f" - {yaml_file.relative_to(PROJECT_ROOT).as_posix()}")
+ for target in sorted(selected_targets):
+ print(f" * target: {target}")
+
+ overall_exit_code = 0
+ for yaml_file, selected_targets in selected_runs:
+ exit_code = run_yaml(
+ yaml_file,
+ selected_targets=selected_targets,
+ jobs=jobs,
+ )
+ if exit_code != 0:
+ overall_exit_code = exit_code
+
+ print("\n[INFO] Custom YAML test execution completed.")
+ print(
+ f"[INFO] Reports written to: "
+ f"{REPORTS_DIR.relative_to(PROJECT_ROOT).as_posix()}"
+ )
+ return overall_exit_code
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/common/acs_test_framework_runner/report.py b/common/acs_test_framework_runner/report.py
new file mode 100644
index 00000000..198a7453
--- /dev/null
+++ b/common/acs_test_framework_runner/report.py
@@ -0,0 +1,992 @@
+from __future__ import annotations
+
+import argparse
+import re
+import shutil
+import subprocess
+import sys
+import xml.etree.ElementTree as ET
+from datetime import datetime
+from pathlib import Path
+
+try: # Support package imports and direct harness module loading.
+ from .runner_checks import (
+ ConfigError,
+ detect_project_root,
+ load_yaml_config,
+ normalize_suites,
+ resolve_target_path,
+ sanitize_name,
+ )
+except ImportError: # pragma: no cover - exercised by flat-module harness imports.
+ from runner_checks import (
+ ConfigError,
+ detect_project_root,
+ load_yaml_config,
+ normalize_suites,
+ resolve_target_path,
+ sanitize_name,
+ )
+
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+PROJECT_ROOT = detect_project_root(SCRIPT_DIR)
+
+RUNNER_FILE = SCRIPT_DIR / "pytest_runner.py"
+REPORTS_DIR = PROJECT_ROOT / "common" / "reports"
+TEST_YAML_DIR = PROJECT_ROOT / "common" / "acs_test_framework_manifests"
+SUPPORTED_YAML_SUFFIXES = {".yaml", ".yml"}
+
+PYLINT_XML = REPORTS_DIR / "pylint-report.xml"
+PYLINT_LOG = REPORTS_DIR / "pylint.log"
+
+MYPY_XML = REPORTS_DIR / "mypy-report.xml"
+MYPY_LOG = REPORTS_DIR / "mypy.log"
+
+PYTEST_LOG = REPORTS_DIR / "pytest.log"
+
+MAX_LOG_ENTRIES = 50
+LOG_SEPARATOR = "=" * 100
+
+
+class Color:
+ GREEN = "\033[92m"
+ RED = "\033[91m"
+ YELLOW = "\033[93m"
+ BLUE = "\033[94m"
+ RESET = "\033[0m"
+
+
+def color_text(text: str, color: str) -> str:
+ return f"{color}{text}{Color.RESET}"
+
+
+def ensure_reports_dir() -> None:
+ REPORTS_DIR.mkdir(parents=True, exist_ok=True)
+
+
+def append_log_history(log_path: Path, content: str, max_entries: int = MAX_LOG_ENTRIES) -> None:
+ ensure_reports_dir()
+
+ timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+ normalized_content = content.rstrip("\n")
+ new_entry = (
+ f"{LOG_SEPARATOR}\n"
+ f"RUN AT: {timestamp}\n"
+ f"{LOG_SEPARATOR}\n"
+ f"{normalized_content}\n"
+ )
+
+ if log_path.exists():
+ existing = log_path.read_text(encoding="utf-8")
+ else:
+ existing = ""
+
+ existing = existing.strip()
+ entries: list[str] = []
+
+ if existing:
+ parts = re.split(
+ rf"(?m)(?=^{re.escape(LOG_SEPARATOR)}\nRUN AT: )",
+ existing,
+ )
+ entries = [part.strip() for part in parts if part.strip()]
+
+ entries.append(new_entry.strip())
+ entries = entries[-max_entries:]
+
+ final_content = "\n\n".join(entries) + "\n"
+ log_path.write_text(final_content, encoding="utf-8")
+
+
+def run_git_command(args: list[str]) -> tuple[int, str, str]:
+ result = subprocess.run(
+ ["git", *args],
+ cwd=PROJECT_ROOT,
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+ return result.returncode, result.stdout.strip(), result.stderr.strip()
+
+
+def get_commit_info() -> dict[str, str]:
+ info = {
+ "commit": "NO_COMMIT_AVAILABLE",
+ "branch": "UNKNOWN_BRANCH",
+ "status": "UNKNOWN_STATUS",
+ }
+
+ code, stdout, _ = run_git_command(["rev-parse", "--is-inside-work-tree"])
+ if code != 0 or stdout.lower() != "true":
+ info["status"] = "NOT_A_GIT_REPOSITORY"
+ return info
+
+ code, stdout, _ = run_git_command(["rev-parse", "--abbrev-ref", "HEAD"])
+ if code == 0 and stdout:
+ info["branch"] = stdout
+
+ code, stdout, _ = run_git_command(["rev-parse", "HEAD"])
+ if code == 0 and stdout:
+ info["commit"] = stdout
+ info["status"] = "COMMIT_FOUND"
+ else:
+ info["status"] = "NO_COMMIT_YET"
+
+ return info
+
+
+def resolve_manual_target(target: str | None) -> Path | None:
+ if not target:
+ return None
+
+ raw = Path(target)
+ if raw.is_absolute():
+ return raw.resolve()
+ return (PROJECT_ROOT / raw).resolve()
+
+
+def get_manual_python_target(target: str | None, tool_name: str) -> list[Path] | None:
+ resolved_target = resolve_manual_target(target)
+ if resolved_target is None:
+ return None
+
+ if not resolved_target.exists() or not resolved_target.is_file():
+ print(
+ f"{color_text('WARNING:', Color.YELLOW)} "
+ f"Manual target for {tool_name} not found: {resolved_target}"
+ )
+ return []
+
+ if resolved_target.suffix != ".py":
+ print(
+ f"{color_text('WARNING:', Color.YELLOW)} "
+ f"Manual target is not a Python file for {tool_name}: {resolved_target}"
+ )
+ return []
+
+ return [resolved_target]
+
+
+def is_generated_report_artifact(path: Path) -> bool:
+ """Return True when a path points inside a generated report work directory."""
+ try:
+ rel_path = path.relative_to(PROJECT_ROOT)
+ except ValueError:
+ return False
+
+ parts = rel_path.parts
+ if len(parts) >= 3 and parts[:2] == ("common", "reports"):
+ return parts[2].startswith(("_", "tpm_check_"))
+
+ if len(parts) >= 2 and parts[0] == "reports":
+ return parts[1].startswith(("_", "tpm_check_"))
+
+ return False
+
+
+def is_yaml_suite_file(path: Path) -> bool:
+ try:
+ path.relative_to(TEST_YAML_DIR)
+ except ValueError:
+ return False
+ return path.is_file() and path.suffix.lower() in SUPPORTED_YAML_SUFFIXES
+
+
+def get_python_targets_from_yaml_file(yaml_file: Path) -> set[Path]:
+ try:
+ config = load_yaml_config(yaml_file)
+ suites = normalize_suites(config)
+ except ConfigError:
+ return set()
+
+ targets: set[Path] = set()
+ for suite in suites:
+ for file_entry in suite["files"]:
+ target_path = resolve_target_path(file_entry)
+ if target_path.suffix != ".py":
+ continue
+ if target_path.exists() and target_path.is_file():
+ targets.add(target_path)
+ return targets
+
+
+def get_recently_changed_paths() -> list[Path]:
+ changed_files: set[str] = set()
+
+ git_queries = [
+ ["diff", "--name-only"],
+ ["diff", "--cached", "--name-only"],
+ ["ls-files", "--others", "--exclude-standard"],
+ ]
+
+ for query in git_queries:
+ code, stdout, _ = run_git_command(query)
+ if code != 0:
+ continue
+
+ for item in stdout.splitlines():
+ item = item.strip()
+ if item:
+ changed_files.add(item)
+
+ paths: list[Path] = []
+ for item in sorted(changed_files):
+ path = (PROJECT_ROOT / item).resolve()
+ if path.exists() and path.is_file() and not is_generated_report_artifact(path):
+ paths.append(path)
+
+ return paths
+
+
+def run_pytest(target: str | None = None, jobs: int = 4) -> tuple[int, str, str]:
+ if not RUNNER_FILE.exists():
+ return 1, "", f"ERROR: Runner file not found: {RUNNER_FILE}"
+
+ cmd = [sys.executable, str(RUNNER_FILE)]
+ if target:
+ cmd.extend(["--target", target])
+ cmd.extend(["--jobs", str(max(1, jobs))])
+
+ result = subprocess.run(
+ cmd,
+ cwd=PROJECT_ROOT,
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+
+ stdout = result.stdout or ""
+ stderr = result.stderr or ""
+
+ log_content = (
+ "STDOUT\n"
+ + "=" * 80
+ + "\n"
+ + stdout
+ + "\n\nSTDERR\n"
+ + "=" * 80
+ + "\n"
+ + stderr
+ + "\n"
+ )
+ append_log_history(PYTEST_LOG, log_content)
+
+ return result.returncode, stdout, stderr
+
+
+def collect_pytest_case_logs(target: str | None = None) -> list[dict[str, str]]:
+ work_root = REPORTS_DIR / "_work"
+ if not work_root.exists():
+ return []
+
+ testcases: list[dict[str, str]] = []
+ seen_names: set[str] = set()
+
+ pattern = re.compile(
+ r"(?ms)^=+\n"
+ r"TEST CASE:\s*(?P.+?)\n"
+ r"(?:DESCRIPTION:\s*(?P.+?)\n)?"
+ r"STATUS:\s*(?P.+?)\n"
+ r"MESSAGE:\s*(?P.*?)(?=\n=+\nTEST CASE:|\Z)"
+ )
+
+ log_paths: list[Path]
+ resolved_target = resolve_manual_target(target)
+
+ if resolved_target is not None:
+ target_stem = sanitize_name(resolved_target.stem)
+ log_paths = sorted(work_root.rglob(f"{target_stem}/combined.log"))
+ else:
+ log_paths = sorted(work_root.rglob("combined.log"))
+
+ for log_path in log_paths:
+ content = log_path.read_text(encoding="utf-8", errors="replace")
+
+ for match in pattern.finditer(content):
+ name = match.group("name").strip()
+ if name in seen_names:
+ continue
+ seen_names.add(name)
+
+ raw_status = match.group("status").strip().upper()
+
+ if raw_status == "PASS":
+ normalized_status = "passed"
+ elif raw_status == "FAIL":
+ normalized_status = "failed"
+ elif raw_status == "ERROR":
+ normalized_status = "error"
+ elif raw_status in {"WARNING", "SKIPPED"}:
+ normalized_status = "warning"
+ else:
+ normalized_status = raw_status.lower()
+
+ testcases.append(
+ {
+ "name": name,
+ "description": (match.group("description") or "").strip(),
+ "status": normalized_status,
+ "details": match.group("message").strip(),
+ }
+ )
+
+ return testcases
+
+
+def print_pytest_summary(commit_info: dict[str, str], target: str | None = None) -> None:
+ print(f"\n{color_text('========== PYTEST REPORT ==========', Color.BLUE)}\n")
+ print(f"Branch : {commit_info['branch']}")
+ print(f"Commit : {commit_info['commit']}")
+ print(f"Git : {commit_info['status']}")
+
+ testcases = collect_pytest_case_logs(target)
+
+ if not testcases:
+ print(
+ f"\n{color_text('WARNING:', Color.YELLOW)} "
+ "No pytest case logs were found."
+ )
+ print(f"\nFull pytest log: {PYTEST_LOG.relative_to(PROJECT_ROOT)}")
+ print(f"\n{color_text('===================================', Color.BLUE)}\n")
+ return
+
+ total = len(testcases)
+ passed = sum(1 for tc in testcases if tc["status"] == "passed")
+ failed = sum(1 for tc in testcases if tc["status"] == "failed")
+ errors = sum(1 for tc in testcases if tc["status"] == "error")
+ warnings = sum(1 for tc in testcases if tc["status"] == "warning")
+
+ print(f"\n{color_text('Total :', Color.BLUE)} {total}")
+ print(f"{color_text('Passed :', Color.GREEN)} {passed}")
+ print(f"{color_text('Failed :', Color.RED)} {failed}")
+ print(f"{color_text('Errors :', Color.RED)} {errors}")
+ print(f"{color_text('Warnings:', Color.YELLOW)} {warnings}")
+
+ print(f"\n{color_text('All Test Cases:', Color.BLUE)}")
+
+ warning_groups: dict[str, list[dict[str, str]]] = {}
+
+ for tc in testcases:
+ status = tc["status"]
+
+ if status == "passed":
+ label = color_text("[passed]", Color.GREEN)
+ print(f" - {tc['name']} {label}")
+ elif status in {"failed", "error"}:
+ label = color_text(f"[{status}]", Color.RED)
+ print(f" - {tc['name']} {label}")
+ elif status == "warning":
+ short_msg = tc["details"].splitlines()[0].strip() if tc["details"] else "Warning"
+ warning_groups.setdefault(short_msg, []).append(tc)
+ else:
+ label = color_text(f"[{status}]", Color.YELLOW)
+ print(f" - {tc['name']} {label}")
+
+ single_warning_groups = {
+ msg: items for msg, items in warning_groups.items() if len(items) == 1
+ }
+ repeated_warning_groups = {
+ msg: items for msg, items in warning_groups.items() if len(items) > 1
+ }
+
+ for _msg, items in single_warning_groups.items():
+ tc = items[0]
+ print(f" - {tc['name']} {color_text('[warning]', Color.YELLOW)}")
+
+ if repeated_warning_groups:
+ print(f"\n{color_text('Collapsed Repeated Warnings:', Color.YELLOW)}")
+ for msg, items in repeated_warning_groups.items():
+ print(f" - {color_text('[warning]', Color.YELLOW)} {msg} ({len(items)} case(s))")
+ for tc in items:
+ print(f" * {tc['name']}")
+
+ print(f"\nFull pytest log: {PYTEST_LOG.relative_to(PROJECT_ROOT)}")
+ print(f"\n{color_text('===================================', Color.BLUE)}\n")
+
+
+def get_recently_changed_python_files() -> list[Path]:
+ python_files: set[Path] = set()
+
+ for path in get_recently_changed_paths():
+ if path.suffix == ".py":
+ python_files.add(path)
+ continue
+ if is_yaml_suite_file(path):
+ python_files.update(get_python_targets_from_yaml_file(path))
+
+ return sorted(python_files)
+
+
+def extract_pylint_score(output: str) -> str:
+ score_patterns = [
+ r"rated at\s+(-?\d+(?:\.\d+)?)\/10",
+ r"Your code has been rated at\s+(-?\d+(?:\.\d+)?)\/10",
+ ]
+
+ for pattern in score_patterns:
+ match = re.search(pattern, output, flags=re.IGNORECASE)
+ if match:
+ return f"{match.group(1)}/10"
+
+ return "N/A"
+
+
+def run_pylint_command(cmd: list[str]) -> subprocess.CompletedProcess[str]:
+ return subprocess.run(
+ cmd,
+ cwd=PROJECT_ROOT,
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+
+
+def run_pylint(target: str | None = None) -> int:
+ ensure_reports_dir()
+
+ pylint_exe = shutil.which("pylint")
+ if pylint_exe is None:
+ print(f"{color_text('WARNING:', Color.YELLOW)} pylint not found in PATH.")
+ create_empty_pylint_xml("pylint not installed")
+ append_log_history(PYLINT_LOG, "pylint not installed")
+ return 0
+
+ manual_targets = get_manual_python_target(target, "pylint")
+ targets = (
+ manual_targets
+ if manual_targets is not None
+ else get_recently_changed_python_files()
+ )
+
+ if not targets:
+ print(
+ f"{color_text('WARNING:', Color.YELLOW)} "
+ "No recently changed Python files found by git for pylint."
+ )
+ create_empty_pylint_xml("no recently changed python files found")
+ append_log_history(PYLINT_LOG, "no recently changed python files found")
+ return 0
+
+ print(
+ color_text(
+ "[INFO] Running pylint on recently changed Python files:",
+ Color.BLUE,
+ )
+ )
+ for target_path in targets:
+ try:
+ rel = target_path.relative_to(PROJECT_ROOT)
+ except ValueError:
+ rel = target_path
+ print(f" - {rel}")
+
+ parseable_cmd = [
+ pylint_exe,
+ "--output-format=parseable",
+ "--score=y",
+ *[str(path) for path in targets],
+ ]
+ score_cmd = [
+ pylint_exe,
+ "--score=y",
+ *[str(path) for path in targets],
+ ]
+
+ parseable_result = run_pylint_command(parseable_cmd)
+ score_result = run_pylint_command(score_cmd)
+ pylint_score = extract_pylint_score(
+ "\n".join(
+ [
+ score_result.stdout or "",
+ score_result.stderr or "",
+ parseable_result.stdout or "",
+ parseable_result.stderr or "",
+ ]
+ )
+ )
+
+ log_content = (
+ "PARSEABLE STDOUT\n"
+ + "=" * 80
+ + "\n"
+ + (parseable_result.stdout or "")
+ + "\n\nPARSEABLE STDERR\n"
+ + "=" * 80
+ + "\n"
+ + (parseable_result.stderr or "")
+ + "\n\nSCORE STDOUT\n"
+ + "=" * 80
+ + "\n"
+ + (score_result.stdout or "")
+ + "\n\nSCORE STDERR\n"
+ + "=" * 80
+ + "\n"
+ + (score_result.stderr or "")
+ + "\n\nEXTRACTED SCORE\n"
+ + "=" * 80
+ + "\n"
+ + pylint_score
+ + "\n"
+ )
+ append_log_history(PYLINT_LOG, log_content)
+
+ print(
+ f"{color_text('[INFO]', Color.BLUE)} "
+ f"Full pylint log saved to: {PYLINT_LOG.relative_to(PROJECT_ROOT)}"
+ )
+
+ write_pylint_xml(
+ parseable_stdout=parseable_result.stdout,
+ parseable_stderr=parseable_result.stderr,
+ targets=targets,
+ pylint_score=pylint_score,
+ )
+ return parseable_result.returncode
+
+
+def write_pylint_xml(
+ parseable_stdout: str,
+ parseable_stderr: str,
+ targets: list[Path],
+ pylint_score: str,
+) -> None:
+ commit_info = get_commit_info()
+
+ testsuite = ET.Element("testsuite")
+ testsuite.set("name", "pylint")
+ testsuite.set("tests", str(len(targets)))
+
+ failures = 0
+ issues_by_file: dict[str, list[str]] = {}
+
+ for line in parseable_stdout.splitlines():
+ line = line.strip()
+ if not line:
+ continue
+
+ file_key = line.split(":", 1)[0].strip()
+ issues_by_file.setdefault(file_key, []).append(line)
+
+ properties = ET.SubElement(testsuite, "properties")
+
+ score_prop = ET.SubElement(properties, "property")
+ score_prop.set("name", "pylint_score")
+ score_prop.set("value", pylint_score)
+
+ for target in targets:
+ try:
+ rel = str(target.relative_to(PROJECT_ROOT))
+ except ValueError:
+ rel = str(target)
+
+ testcase = ET.SubElement(testsuite, "testcase")
+ testcase.set("classname", "pylint")
+ testcase.set("name", rel)
+
+ issues = (
+ issues_by_file.get(str(target), [])
+ or issues_by_file.get(rel, [])
+ )
+ if issues:
+ failures += 1
+ failure = ET.SubElement(testcase, "failure")
+ failure.set("message", f"{len(issues)} pylint issue(s)")
+ failure.text = "\n".join(issues)
+
+ testsuite.set("failures", str(failures))
+ testsuite.set("errors", "0")
+ testsuite.set("skipped", "0")
+
+ system_out = ET.SubElement(testsuite, "system-out")
+ system_out.text = (
+ f"branch={commit_info['branch']}\n"
+ f"commit={commit_info['commit']}\n"
+ f"git_status={commit_info['status']}\n"
+ f"pylint_score={pylint_score}\n"
+ f"{parseable_stderr.strip()}".strip()
+ )
+
+ tree = ET.ElementTree(testsuite)
+ tree.write(PYLINT_XML, encoding="utf-8", xml_declaration=True)
+
+
+def create_empty_pylint_xml(reason: str) -> None:
+ commit_info = get_commit_info()
+
+ testsuite = ET.Element("testsuite")
+ testsuite.set("name", "pylint")
+ testsuite.set("tests", "0")
+ testsuite.set("failures", "0")
+ testsuite.set("errors", "0")
+ testsuite.set("skipped", "0")
+
+ properties = ET.SubElement(testsuite, "properties")
+ score_prop = ET.SubElement(properties, "property")
+ score_prop.set("name", "pylint_score")
+ score_prop.set("value", "N/A")
+
+ system_out = ET.SubElement(testsuite, "system-out")
+ system_out.text = (
+ f"branch={commit_info['branch']}\n"
+ f"commit={commit_info['commit']}\n"
+ f"git_status={commit_info['status']}\n"
+ f"pylint_score=N/A\n"
+ f"reason={reason}"
+ )
+
+ tree = ET.ElementTree(testsuite)
+ tree.write(PYLINT_XML, encoding="utf-8", xml_declaration=True)
+
+
+def get_pylint_score_from_xml(root: ET.Element) -> str:
+ properties = root.find("properties")
+ if properties is not None:
+ for prop in properties.findall("property"):
+ if prop.attrib.get("name") == "pylint_score":
+ return prop.attrib.get("value", "N/A")
+
+ system_out = root.findtext("system-out", default="")
+ match = re.search(r"pylint_score=(.+)", system_out)
+ if match:
+ return match.group(1).strip()
+
+ return "N/A"
+
+
+def print_pylint_summary(commit_info: dict[str, str]) -> None:
+ print(f"\n{color_text('========== PYLINT REPORT ==========', Color.BLUE)}\n")
+ print(f"Branch : {commit_info['branch']}")
+ print(f"Commit : {commit_info['commit']}")
+ print(f"Git : {commit_info['status']}")
+
+ if not PYLINT_XML.exists():
+ print(f"\n{color_text('WARNING:', Color.YELLOW)} No pylint XML report found.")
+ print(f"\n{color_text('===================================', Color.BLUE)}\n")
+ return
+
+ try:
+ tree = ET.parse(PYLINT_XML)
+ root = tree.getroot()
+ except Exception as exc:
+ print(
+ f"\n{color_text('WARNING:', Color.YELLOW)} "
+ f"Failed to parse pylint XML report: {exc}"
+ )
+ print(f"\n{color_text('===================================', Color.BLUE)}\n")
+ return
+
+ total = int(root.attrib.get("tests", 0))
+ failures = int(root.attrib.get("failures", 0))
+ errors = int(root.attrib.get("errors", 0))
+ skipped = int(root.attrib.get("skipped", 0))
+ passed = total - failures - errors - skipped
+ pylint_score = get_pylint_score_from_xml(root)
+
+ print(f"\n{color_text('Total :', Color.BLUE)} {total}")
+ print(f"{color_text('Passed :', Color.GREEN)} {passed}")
+ print(f"{color_text('Failed :', Color.RED)} {failures}")
+ print(f"{color_text('Errors :', Color.RED)} {errors}")
+ print(f"{color_text('Warnings:', Color.YELLOW)} {skipped}")
+ print(f"{color_text('Score :', Color.BLUE)} {pylint_score}")
+
+ print(f"\n{color_text('Detailed Issues:', Color.BLUE)}\n")
+
+ for testcase in root.findall("testcase"):
+ failure = testcase.find("failure")
+ if failure is None:
+ continue
+
+ file_name = testcase.attrib.get("name", "unknown")
+ details = (failure.text or "").strip().splitlines()
+
+ print(color_text(file_name, Color.RED))
+ for line in details:
+ parts = line.split(":", 2)
+ if len(parts) >= 3:
+ line_no = parts[1]
+ rest = parts[2].strip()
+ print(color_text(f" - line {line_no}: {rest}", Color.RED))
+ else:
+ print(color_text(f" - {line}", Color.RED))
+ print()
+
+ system_out = root.findtext("system-out", default="").strip()
+ if system_out and total == 0:
+ print(f"Note: {system_out}")
+
+ print(f"\nFull pylint log: {PYLINT_LOG.relative_to(PROJECT_ROOT)}")
+ print(f"\n{color_text('===================================', Color.BLUE)}\n")
+
+
+def run_mypy_command(cmd: list[str]) -> subprocess.CompletedProcess[str]:
+ return subprocess.run(
+ cmd,
+ cwd=PROJECT_ROOT,
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+
+
+def run_mypy(target: str | None = None) -> int:
+ ensure_reports_dir()
+
+ mypy_exe = shutil.which("mypy")
+ if mypy_exe is None:
+ print(f"{color_text('WARNING:', Color.YELLOW)} mypy not found in PATH.")
+ create_empty_mypy_xml("mypy not installed")
+ append_log_history(MYPY_LOG, "mypy not installed")
+ return 0
+
+ manual_targets = get_manual_python_target(target, "mypy")
+ targets = (
+ manual_targets
+ if manual_targets is not None
+ else get_recently_changed_python_files()
+ )
+
+ if not targets:
+ print(
+ f"{color_text('WARNING:', Color.YELLOW)} "
+ "No recently changed Python files found by git for mypy."
+ )
+ create_empty_mypy_xml("no recently changed python files found")
+ append_log_history(MYPY_LOG, "no recently changed python files found")
+ return 0
+
+ print(
+ color_text(
+ "[INFO] Running mypy on recently changed Python files:",
+ Color.BLUE,
+ )
+ )
+ for target_path in targets:
+ try:
+ rel = target_path.relative_to(PROJECT_ROOT)
+ except ValueError:
+ rel = target_path
+ print(f" - {rel}")
+
+ cmd = [
+ mypy_exe,
+ "--show-error-codes",
+ "--no-color-output",
+ "--no-error-summary",
+ *[str(path) for path in targets],
+ ]
+ result = run_mypy_command(cmd)
+
+ log_content = (
+ "STDOUT\n"
+ + "=" * 80
+ + "\n"
+ + (result.stdout or "")
+ + "\n\nSTDERR\n"
+ + "=" * 80
+ + "\n"
+ + (result.stderr or "")
+ + "\n"
+ )
+ append_log_history(MYPY_LOG, log_content)
+
+ print(
+ f"{color_text('[INFO]', Color.BLUE)} "
+ f"Full mypy log saved to: {MYPY_LOG.relative_to(PROJECT_ROOT)}"
+ )
+
+ write_mypy_xml(
+ stdout=result.stdout,
+ stderr=result.stderr,
+ targets=targets,
+ )
+ return result.returncode
+
+
+def write_mypy_xml(
+ stdout: str,
+ stderr: str,
+ targets: list[Path],
+) -> None:
+ commit_info = get_commit_info()
+
+ testsuite = ET.Element("testsuite")
+ testsuite.set("name", "mypy")
+ testsuite.set("tests", str(len(targets)))
+
+ failures = 0
+ issues_by_file: dict[str, list[str]] = {}
+
+ for raw_line in stdout.splitlines():
+ line = raw_line.strip()
+ if not line:
+ continue
+
+ match = re.match(
+ r"^(.*?\.py)(?::\d+)?(?::\d+)?:\s*(error|note):\s*(.*)$",
+ line,
+ )
+ if match:
+ file_key = match.group(1).strip()
+ issues_by_file.setdefault(file_key, []).append(line)
+
+ for target in targets:
+ try:
+ rel = str(target.relative_to(PROJECT_ROOT))
+ except ValueError:
+ rel = str(target)
+
+ testcase = ET.SubElement(testsuite, "testcase")
+ testcase.set("classname", "mypy")
+ testcase.set("name", rel)
+
+ issues = (
+ issues_by_file.get(str(target), [])
+ or issues_by_file.get(rel, [])
+ )
+ if issues:
+ failures += 1
+ failure = ET.SubElement(testcase, "failure")
+ failure.set("message", f"{len(issues)} mypy issue(s)")
+ failure.text = "\n".join(issues)
+
+ testsuite.set("failures", str(failures))
+ testsuite.set("errors", "0")
+ testsuite.set("skipped", "0")
+
+ system_out = ET.SubElement(testsuite, "system-out")
+ system_out.text = (
+ f"branch={commit_info['branch']}\n"
+ f"commit={commit_info['commit']}\n"
+ f"git_status={commit_info['status']}\n"
+ f"{stderr.strip()}".strip()
+ )
+
+ tree = ET.ElementTree(testsuite)
+ tree.write(MYPY_XML, encoding="utf-8", xml_declaration=True)
+
+
+def create_empty_mypy_xml(reason: str) -> None:
+ commit_info = get_commit_info()
+
+ testsuite = ET.Element("testsuite")
+ testsuite.set("name", "mypy")
+ testsuite.set("tests", "0")
+ testsuite.set("failures", "0")
+ testsuite.set("errors", "0")
+ testsuite.set("skipped", "0")
+
+ ET.SubElement(testsuite, "properties")
+
+ system_out = ET.SubElement(testsuite, "system-out")
+ system_out.text = (
+ f"branch={commit_info['branch']}\n"
+ f"commit={commit_info['commit']}\n"
+ f"git_status={commit_info['status']}\n"
+ f"reason={reason}"
+ )
+
+ tree = ET.ElementTree(testsuite)
+ tree.write(MYPY_XML, encoding="utf-8", xml_declaration=True)
+
+
+def print_mypy_summary(commit_info: dict[str, str]) -> None:
+ print(f"\n{color_text('=========== MYPY REPORT ===========', Color.BLUE)}\n")
+ print(f"Branch : {commit_info['branch']}")
+ print(f"Commit : {commit_info['commit']}")
+ print(f"Git : {commit_info['status']}")
+
+ if not MYPY_XML.exists():
+ print(f"\n{color_text('WARNING:', Color.YELLOW)} No mypy XML report found.")
+ print(f"\n{color_text('===================================', Color.BLUE)}\n")
+ return
+
+ try:
+ tree = ET.parse(MYPY_XML)
+ root = tree.getroot()
+ except Exception as exc:
+ print(
+ f"\n{color_text('WARNING:', Color.YELLOW)} "
+ f"Failed to parse mypy XML report: {exc}"
+ )
+ print(f"\n{color_text('===================================', Color.BLUE)}\n")
+ return
+
+ total = int(root.attrib.get("tests", 0))
+ failures = int(root.attrib.get("failures", 0))
+ errors = int(root.attrib.get("errors", 0))
+ skipped = int(root.attrib.get("skipped", 0))
+ passed = total - failures - errors - skipped
+
+ print(f"\n{color_text('Total :', Color.BLUE)} {total}")
+ print(f"{color_text('Passed :', Color.GREEN)} {passed}")
+ print(f"{color_text('Failed :', Color.RED)} {failures}")
+ print(f"{color_text('Errors :', Color.RED)} {errors}")
+ print(f"{color_text('Warnings:', Color.YELLOW)} {skipped}")
+
+ print(f"\n{color_text('Detailed Issues:', Color.BLUE)}\n")
+
+ for testcase in root.findall("testcase"):
+ failure = testcase.find("failure")
+ if failure is None:
+ continue
+
+ file_name = testcase.attrib.get("name", "unknown")
+ details = (failure.text or "").strip().splitlines()
+
+ print(color_text(file_name, Color.RED))
+ for line in details:
+ print(color_text(f" - {line}", Color.RED))
+ print()
+
+ system_out = root.findtext("system-out", default="").strip()
+ if system_out and total == 0:
+ print(f"Note: {system_out}")
+
+ print(f"\nFull mypy log: {MYPY_LOG.relative_to(PROJECT_ROOT)}")
+ print(f"\n{color_text('===================================', Color.BLUE)}\n")
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(
+ description="Generate test reports and run analysis tools."
+ )
+ parser.add_argument(
+ "target",
+ nargs="?",
+ help="Manual target file for focused analysis",
+ )
+ parser.add_argument(
+ "--jobs",
+ type=int,
+ default=4,
+ help="Number of YAML test cases to run in parallel (default: 4)",
+ )
+ args = parser.parse_args()
+
+ ensure_reports_dir()
+ commit_info = get_commit_info()
+
+ if args.target:
+ print(
+ f"{color_text('[INFO]', Color.BLUE)} "
+ f"Manual target override enabled: {args.target}"
+ )
+
+ pytest_exit_code, _pytest_stdout, _pytest_stderr = run_pytest(args.target, jobs=args.jobs)
+ pylint_exit_code = run_pylint(args.target)
+ mypy_exit_code = run_mypy(args.target)
+
+ print_pytest_summary(commit_info, args.target)
+ print_pylint_summary(commit_info)
+ print_mypy_summary(commit_info)
+
+ if pytest_exit_code != 0:
+ return pytest_exit_code
+ if pylint_exit_code != 0:
+ return pylint_exit_code
+ return mypy_exit_code
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/common/acs_test_framework_runner/runner_check_execution.py b/common/acs_test_framework_runner/runner_check_execution.py
new file mode 100644
index 00000000..bf448f90
--- /dev/null
+++ b/common/acs_test_framework_runner/runner_check_execution.py
@@ -0,0 +1,1210 @@
+from __future__ import annotations
+
+import importlib.util
+import io
+import os
+import py_compile
+import re
+import shlex
+import shutil
+import subprocess
+import sys
+import traceback
+from contextlib import redirect_stderr, redirect_stdout
+from pathlib import Path
+from typing import Any, Callable
+
+try: # Support package imports and direct harness module loading.
+ from .case_data_builders import (
+ CaseBuildError,
+ expand_template,
+ materialize_case_workspace,
+ prepare_case_files,
+ )
+ from .mock_helpers import MockExpectationError
+ from .mock_loader import ConfigError as MockLoaderConfigError
+ from .mock_loader import apply_case_mocks, build_case_runtime_definition
+ from .runner_checks import (
+ DEFAULT_CLI_TIMEOUT_SEC,
+ DESTRUCTIVE_TEST_ENV,
+ REAL_SUBPROCESS_RUN,
+ CommandRunResult,
+ ConfigError,
+ SkipCase,
+ collect_function_names,
+ create_runner_temp_dir,
+ ensure_list,
+ ensure_list_of_strings,
+ ensure_string_or_list_of_strings,
+ load_module_from_path,
+ normalize_completed_stream,
+ read_source,
+ sanitize_xml_text,
+ )
+except ImportError: # pragma: no cover - exercised by flat-module harness imports.
+ from case_data_builders import (
+ CaseBuildError,
+ expand_template,
+ materialize_case_workspace,
+ prepare_case_files,
+ )
+ from mock_helpers import MockExpectationError
+ from mock_loader import ConfigError as MockLoaderConfigError
+ from mock_loader import apply_case_mocks, build_case_runtime_definition
+ from runner_checks import (
+ DEFAULT_CLI_TIMEOUT_SEC,
+ DESTRUCTIVE_TEST_ENV,
+ REAL_SUBPROCESS_RUN,
+ CommandRunResult,
+ ConfigError,
+ SkipCase,
+ collect_function_names,
+ create_runner_temp_dir,
+ ensure_list,
+ ensure_list_of_strings,
+ ensure_string_or_list_of_strings,
+ load_module_from_path,
+ normalize_completed_stream,
+ read_source,
+ sanitize_xml_text,
+ )
+
+
+def append_log_file_details(
+ details_lines: list[str],
+ runtime_case: dict[str, Any],
+) -> None:
+ try: # Support package imports and direct harness module loading.
+ from .runner_checks import append_log_file_details as shared_append_log_file_details
+ except ImportError: # pragma: no cover - exercised by flat-module harness imports.
+ from runner_checks import append_log_file_details as shared_append_log_file_details
+
+ shared_append_log_file_details(details_lines, runtime_case)
+
+
+def run_post_checks(work_dir: Path, post_checks: Any) -> tuple[bool, list[str]]:
+ try: # Support package imports and direct harness module loading.
+ from .runner_checks import run_post_checks as shared_run_post_checks
+ except ImportError: # pragma: no cover - exercised by flat-module harness imports.
+ from runner_checks import run_post_checks as shared_run_post_checks
+
+ return shared_run_post_checks(work_dir, post_checks)
+
+
+def validate_output_expectations(
+ case_def: dict[str, Any],
+ stdout: str,
+ stderr: str,
+) -> tuple[bool, list[str]]:
+ try: # Support package imports and direct harness module loading.
+ from .runner_checks import (
+ validate_output_expectations as shared_validate_output_expectations,
+ )
+ except ImportError: # pragma: no cover - exercised by flat-module harness imports.
+ from runner_checks import (
+ validate_output_expectations as shared_validate_output_expectations,
+ )
+
+ return shared_validate_output_expectations(case_def, stdout, stderr)
+
+
+def build_command_from_spec(
+ file_path: Path,
+ spec: dict[str, Any],
+ work_dir: Path,
+) -> tuple[list[str] | str, bool]:
+ raw_args = spec.get("args", [])
+ if not isinstance(raw_args, list) or not all(isinstance(arg, str) for arg in raw_args):
+ raise ConfigError("'args' must be a list of strings")
+
+ command_value = spec.get("command")
+ if not isinstance(command_value, str):
+ raise ConfigError("'command' must be a string")
+
+ shell_mode = bool(spec.get("shell", False))
+ expanded_command = expand_template(command_value, work_dir, file_path)
+ expanded_args = [expand_template(arg, work_dir, file_path) for arg in raw_args]
+
+ if shell_mode:
+ parts = [expanded_command, *[shlex.quote(arg) for arg in expanded_args]]
+ return " ".join(parts), True
+
+ return [expanded_command, *expanded_args], False
+
+
+def build_runtime_env(
+ file_path: Path,
+ env_spec: Any,
+ work_dir: Path,
+) -> dict[str, str]:
+ env = os.environ.copy()
+ if env_spec is None:
+ return env
+
+ if not isinstance(env_spec, dict):
+ raise ConfigError("'env' must be a mapping")
+
+ for key, value in env_spec.items():
+ if not isinstance(key, str) or not isinstance(value, str):
+ raise ConfigError("'env' entries must be string -> string")
+ env[key] = expand_template(value, work_dir, file_path)
+
+ return env
+
+
+def execute_command_spec(
+ file_path: Path,
+ spec: dict[str, Any],
+ work_dir: Path,
+ stdin_text: str | None = None,
+) -> CommandRunResult:
+ timeout_sec = spec.get("timeout_sec", DEFAULT_CLI_TIMEOUT_SEC)
+ if not isinstance(timeout_sec, int) or timeout_sec <= 0:
+ raise ConfigError("'timeout_sec' must be a positive integer")
+
+ cmd, shell_mode = build_command_from_spec(file_path, spec, work_dir)
+ run_env = build_runtime_env(file_path, spec.get("env"), work_dir)
+
+ cwd_value = spec.get("cwd")
+ if cwd_value is None:
+ cwd = work_dir
+ else:
+ if not isinstance(cwd_value, str):
+ raise ConfigError("'cwd' must be a string")
+ cwd = Path(expand_template(cwd_value, work_dir, file_path))
+
+ timed_out = False
+ stdout = ""
+ stderr = ""
+ exit_code: int | None = None
+
+ try:
+ completed = REAL_SUBPROCESS_RUN(
+ cmd,
+ cwd=str(cwd),
+ capture_output=True,
+ text=True,
+ input=stdin_text,
+ check=False,
+ env=run_env,
+ timeout=timeout_sec,
+ shell=shell_mode,
+ )
+ stdout = normalize_completed_stream(completed.stdout)
+ stderr = normalize_completed_stream(completed.stderr)
+ exit_code = completed.returncode
+ except subprocess.TimeoutExpired as exc:
+ timed_out = True
+ stdout = normalize_completed_stream(exc.stdout)
+ stderr = normalize_completed_stream(exc.stderr)
+
+ command_text = cmd if isinstance(cmd, str) else " ".join(cmd)
+ return CommandRunResult(
+ command_text=command_text,
+ stdout=stdout,
+ stderr=stderr,
+ exit_code=exit_code,
+ timed_out=timed_out,
+ timeout_sec=timeout_sec,
+ shell_mode=shell_mode,
+ )
+
+
+def normalize_probe_spec(probe: Any) -> dict[str, Any]:
+ if isinstance(probe, str):
+ return {
+ "command": probe,
+ "args": [],
+ "shell": True,
+ "timeout_sec": DEFAULT_CLI_TIMEOUT_SEC,
+ }
+ if isinstance(probe, dict):
+ return dict(probe)
+ raise ConfigError("Command probe must be a string or mapping")
+
+
+def apply_skip_controls(
+ file_path: Path,
+ case_def: dict[str, Any],
+ work_dir: Path,
+) -> None:
+ skip_unless_env = case_def.get("skip_unless_env", {})
+ if skip_unless_env:
+ if not isinstance(skip_unless_env, dict):
+ raise ConfigError("'skip_unless_env' must be a mapping")
+ for key, expected in skip_unless_env.items():
+ current = os.environ.get(key)
+ if isinstance(expected, bool):
+ present = bool(current)
+ if present != expected:
+ raise SkipCase(
+ f"[HW WARNING] Env presence mismatch for {key!r}: "
+ f"got {present}, expected {expected}"
+ )
+ continue
+ if current != expected:
+ raise SkipCase(
+ f"[HW WARNING] Env mismatch for {key!r}: "
+ f"got {current!r}, expected {expected!r}"
+ )
+
+ skip_unless_paths_exist = case_def.get("skip_unless_paths_exist", [])
+ for raw_path in ensure_string_or_list_of_strings(
+ skip_unless_paths_exist,
+ "skip_unless_paths_exist",
+ ):
+ resolved = Path(expand_template(raw_path, work_dir, file_path))
+ if not resolved.exists():
+ raise SkipCase(f"[HW WARNING] Missing required path: {resolved}")
+
+ skip_unless_commands_succeed = case_def.get("skip_unless_commands_succeed", [])
+ probes = ensure_list(skip_unless_commands_succeed, "skip_unless_commands_succeed")
+ for probe in probes:
+ probe_spec = normalize_probe_spec(probe)
+ result = execute_command_spec(file_path, probe_spec, work_dir)
+ if result.timed_out:
+ raise SkipCase(
+ f"[HW WARNING] Probe command timed out after {result.timeout_sec}s: "
+ f"{result.command_text}"
+ )
+ if result.exit_code != 0:
+ raise SkipCase(
+ f"[HW WARNING] Probe command failed with exit code "
+ f"{result.exit_code}: {result.command_text}"
+ )
+
+ if case_def.get("requires_destructive", False):
+ if os.environ.get(DESTRUCTIVE_TEST_ENV) != "1":
+ raise SkipCase(
+ f"[HW WARNING] Destructive test skipped. Set "
+ f"{DESTRUCTIVE_TEST_ENV}=1 to enable."
+ )
+
+ required_env = case_def.get("required_env", {})
+ if required_env:
+ if not isinstance(required_env, dict):
+ raise ConfigError("'required_env' must be a mapping")
+ for key, expected in required_env.items():
+ current = os.environ.get(key)
+ if current != expected:
+ raise ConfigError(
+ f"required_env mismatch for {key!r}: "
+ f"got {current!r}, expected {expected!r}"
+ )
+
+
+def check_file_exists(
+ file_path: Path,
+ _case_def: dict[str, Any],
+ _work_dir: Path,
+) -> tuple[bool, str, str, bool]:
+ exists = file_path.exists()
+ message = "File exists" if exists else "File does not exist"
+ return exists, message, "", False
+
+
+def check_py_compile(
+ file_path: Path,
+ _case_def: dict[str, Any],
+ _work_dir: Path,
+) -> tuple[bool, str, str, bool]:
+ try:
+ py_compile.compile(str(file_path), doraise=True)
+ return True, "Python compilation succeeded", "", False
+ except py_compile.PyCompileError as exc:
+ return False, "Python compilation failed", str(exc), False
+
+
+def check_source_contains(
+ file_path: Path,
+ case_def: dict[str, Any],
+ _work_dir: Path,
+) -> tuple[bool, str, str, bool]:
+ pattern = case_def.get("pattern")
+ if not isinstance(pattern, str):
+ raise ConfigError("'source_contains' requires a string 'pattern'")
+
+ source = read_source(file_path)
+ passed = pattern in source
+ message = f"Found pattern: {pattern}" if passed else f"Pattern not found: {pattern}"
+ return passed, message, "", False
+
+
+def check_source_contains_any(
+ file_path: Path,
+ case_def: dict[str, Any],
+ _work_dir: Path,
+) -> tuple[bool, str, str, bool]:
+ patterns = ensure_list_of_strings(case_def.get("patterns"), "patterns")
+ source = read_source(file_path)
+ matched = [pattern for pattern in patterns if pattern in source]
+ passed = bool(matched)
+ details = "Matched patterns: " + ", ".join(matched) if matched else ""
+ message = "At least one pattern matched" if passed else "No patterns matched"
+ return passed, message, details, False
+
+
+def check_source_contains_all(
+ file_path: Path,
+ case_def: dict[str, Any],
+ _work_dir: Path,
+) -> tuple[bool, str, str, bool]:
+ patterns = ensure_list_of_strings(case_def.get("patterns"), "patterns")
+ source = read_source(file_path)
+ missing = [pattern for pattern in patterns if pattern not in source]
+ passed = not missing
+ details = "Missing patterns: " + ", ".join(missing) if missing else ""
+ message = "All patterns matched" if passed else "Some patterns were not found"
+ return passed, message, details, False
+
+
+def check_function_exists(
+ file_path: Path,
+ case_def: dict[str, Any],
+ _work_dir: Path,
+) -> tuple[bool, str, str, bool]:
+ function_name = case_def.get("function")
+ if not isinstance(function_name, str):
+ raise ConfigError("'function_exists' requires a string 'function'")
+
+ names = collect_function_names(file_path)
+ passed = function_name in names
+ message = (
+ f"Function found: {function_name}"
+ if passed
+ else f"Function not found: {function_name}"
+ )
+ return passed, message, "", False
+
+
+def check_function_exists_any(
+ file_path: Path,
+ case_def: dict[str, Any],
+ _work_dir: Path,
+) -> tuple[bool, str, str, bool]:
+ functions = ensure_list_of_strings(case_def.get("functions"), "functions")
+ names = collect_function_names(file_path)
+ matched = [name for name in functions if name in names]
+ passed = bool(matched)
+ details = "Matched functions: " + ", ".join(matched) if matched else ""
+ message = "At least one function matched" if passed else "No functions matched"
+ return passed, message, details, False
+
+
+def check_main_guard(
+ file_path: Path,
+ _case_def: dict[str, Any],
+ _work_dir: Path,
+) -> tuple[bool, str, str, bool]:
+ source = read_source(file_path)
+ patterns = [
+ 'if __name__ == "__main__":',
+ "if __name__ == '__main__':",
+ ]
+ passed = any(pattern in source for pattern in patterns)
+ message = "Main guard found" if passed else "Main guard not found"
+ return passed, message, "", False
+
+
+def check_py_function(
+ file_path: Path,
+ case_def: dict[str, Any],
+ work_dir: Path,
+) -> tuple[bool, str, str, bool]:
+ module = load_module_from_path(file_path)
+ module_name = module.__name__
+ stdout_buffer = io.StringIO()
+ stderr_buffer = io.StringIO()
+ original_cwd = Path.cwd()
+ original_env = os.environ.copy()
+
+ try:
+ details_lines: list[str] = [f"Work dir: {work_dir}"]
+ function_name = case_def.get("function")
+ if not isinstance(function_name, str):
+ raise ConfigError("'py_function' requires a string 'function'")
+
+ if not hasattr(module, function_name):
+ return False, f"Function not found: {function_name}", "", False
+
+ try:
+ runtime_case = build_case_runtime_definition(
+ case_def,
+ work_dir,
+ file_path,
+ extra_tokens={"module": module.__name__},
+ )
+ details_lines.extend(materialize_case_workspace(work_dir, runtime_case))
+ except (MockLoaderConfigError, ValueError, TypeError) as exc:
+ raise ConfigError(str(exc)) from exc
+ except CaseBuildError as exc:
+ raise ConfigError(str(exc)) from exc
+
+ func = getattr(module, function_name)
+ args = runtime_case.get("args", [])
+ kwargs = runtime_case.get("kwargs", {})
+ run_env = build_cli_env(file_path, runtime_case, work_dir)
+
+ if not isinstance(args, list):
+ raise ConfigError("'py_function.args' must be a list")
+ if not isinstance(kwargs, dict):
+ raise ConfigError("'py_function.kwargs' must be a mapping")
+
+ patch_constants = runtime_case.get("patch_constants", {})
+ if patch_constants is not None:
+ if not isinstance(patch_constants, dict):
+ raise ConfigError("'py_function.patch_constants' must be a mapping")
+ for attr_name, attr_value in patch_constants.items():
+ setattr(module, attr_name, attr_value)
+
+ expected_exception = runtime_case.get("expect_exception")
+
+ passed = True
+ message = ""
+ is_error = False
+ result: Any = None
+ raised_exc: Exception | None = None
+
+ try:
+ os.chdir(work_dir)
+ os.environ.clear()
+ os.environ.update(run_env)
+ with apply_case_mocks(
+ runtime_case.get("mocks"),
+ target_context={"module": module.__name__},
+ ):
+ with redirect_stdout(stdout_buffer), redirect_stderr(stderr_buffer):
+ result = func(*args, **kwargs)
+ except MockExpectationError as exc:
+ raised_exc = exc
+ details_lines.append(f"Mock verification failed: {exc}")
+ except Exception as exc: # pylint: disable=broad-exception-caught
+ raised_exc = exc
+ details_lines.append(traceback.format_exc())
+ finally:
+ os.chdir(original_cwd)
+ os.environ.clear()
+ os.environ.update(original_env)
+
+ stdout = normalize_completed_stream(stdout_buffer.getvalue())
+ stderr = normalize_completed_stream(stderr_buffer.getvalue())
+ details_lines.extend(
+ [
+ "--- STDOUT ---",
+ stdout.rstrip(),
+ "--- STDERR ---",
+ stderr.rstrip(),
+ ]
+ )
+
+ if raised_exc is not None:
+ if isinstance(raised_exc, MockExpectationError):
+ passed = False
+ message = f"Mock verification failed: {raised_exc}"
+ is_error = True
+ elif expected_exception:
+ passed = raised_exc.__class__.__name__ == expected_exception
+ message = (
+ f"Raised expected exception: {expected_exception}"
+ if passed
+ else (
+ f"Expected {expected_exception}, got "
+ f"{raised_exc.__class__.__name__}: {raised_exc}"
+ )
+ )
+ is_error = not passed
+ else:
+ passed = False
+ message = f"Unexpected exception: {raised_exc}"
+ is_error = True
+ elif expected_exception:
+ passed = False
+ message = f"Expected exception {expected_exception}, but function returned"
+ details_lines.append(repr(result))
+ else:
+ expected_return = runtime_case.get("expect_return")
+ if "expect_return" in runtime_case and result != expected_return:
+ passed = False
+ message = f"Expected return {expected_return!r}, got {result!r}"
+
+ expected_fragment = runtime_case.get("expect_return_contains")
+ if expected_fragment is not None and expected_fragment not in str(result):
+ passed = False
+ message = (
+ f"Expected return to contain {expected_fragment!r}, got {result!r}"
+ )
+
+ if passed:
+ message = f"Function returned {result!r}"
+
+ output_passed, output_conditions = validate_output_expectations(
+ runtime_case,
+ stdout,
+ stderr,
+ )
+ post_passed, post_messages = run_post_checks(
+ work_dir,
+ runtime_case.get("post_checks"),
+ )
+ if post_messages:
+ details_lines.extend(["--- POST CHECKS ---", *post_messages])
+
+ conditions: list[str] = []
+ if not passed and message:
+ conditions.append(message)
+ conditions.extend(output_conditions)
+ conditions.extend(message for message in post_messages if "FAIL" in message)
+
+ final_passed = passed and output_passed and post_passed
+ final_message = (
+ message
+ if final_passed
+ else "\n\n" + ("\n" + ("-" * 80) + "\n").join(conditions)
+ if conditions
+ else "Function check failed"
+ )
+
+ details = sanitize_xml_text("\n".join(details_lines).strip())
+ return final_passed, sanitize_xml_text(final_message), details, is_error
+ finally:
+ sys.modules.pop(module_name, None)
+
+
+def check_module_main_with_env(
+ file_path: Path,
+ case_def: dict[str, Any],
+ _work_dir: Path,
+) -> tuple[bool, str, str, bool]:
+ temp_dir = create_runner_temp_dir()
+ details_lines: list[str] = [f"Temp dir: {temp_dir}"]
+ module: Any | None = None
+ exit_code: int | None = None
+ stdout_buffer = io.StringIO()
+ stderr_buffer = io.StringIO()
+ original_cwd = Path.cwd()
+ original_env = os.environ.copy()
+
+ try:
+ try:
+ runtime_case = build_case_runtime_definition(
+ case_def,
+ temp_dir,
+ file_path,
+ )
+ details_lines.extend(materialize_case_workspace(temp_dir, runtime_case))
+ except (CaseBuildError, MockLoaderConfigError, ValueError, TypeError) as exc:
+ return False, f"Configuration error: {exc}", "\n".join(details_lines), True
+
+ run_env = build_cli_env(file_path, runtime_case, temp_dir)
+ module = load_module_from_path(file_path)
+
+ patch_constants = runtime_case.get("patch_constants", {})
+ for attr_name, attr_value in patch_constants.items():
+ setattr(module, attr_name, attr_value)
+ details_lines.append(f"Patched constant: {attr_name}={attr_value!r}")
+
+ try:
+ os.chdir(temp_dir)
+ os.environ.clear()
+ os.environ.update(run_env)
+ with apply_case_mocks(
+ runtime_case.get("mocks"),
+ target_context={"module": module.__name__},
+ ):
+ with redirect_stdout(stdout_buffer), redirect_stderr(stderr_buffer):
+ main_result = module.main()
+ exit_code = main_result if isinstance(main_result, int) else 0
+ except MockExpectationError as exc:
+ stdout = normalize_completed_stream(stdout_buffer.getvalue())
+ stderr = normalize_completed_stream(stderr_buffer.getvalue())
+ details_lines.extend(
+ [
+ f"Exit code: {exit_code}",
+ "--- STDOUT ---",
+ stdout.rstrip(),
+ "--- STDERR ---",
+ stderr.rstrip(),
+ f"Mock verification failed: {exc}",
+ ]
+ )
+ append_log_file_details(details_lines, runtime_case)
+ return (
+ False,
+ f"Mock verification failed: {exc}",
+ sanitize_xml_text("\n".join(details_lines).strip()),
+ False,
+ )
+ except SystemExit as exc:
+ exit_code = exc.code if isinstance(exc.code, int) else 0
+ finally:
+ os.chdir(original_cwd)
+ os.environ.clear()
+ os.environ.update(original_env)
+
+ stdout = normalize_completed_stream(stdout_buffer.getvalue())
+ stderr = normalize_completed_stream(stderr_buffer.getvalue())
+ runtime_case["_actual_exit_code"] = exit_code
+ details_lines.append(f"Exit code: {exit_code}")
+ details_lines.extend(
+ [
+ "--- STDOUT ---",
+ stdout.rstrip(),
+ "--- STDERR ---",
+ stderr.rstrip(),
+ ]
+ )
+
+ output_passed, output_conditions = validate_output_expectations(
+ runtime_case,
+ stdout,
+ stderr,
+ )
+ post_passed, post_messages = run_post_checks(
+ temp_dir,
+ runtime_case.get("post_checks"),
+ )
+ details_lines.extend(["--- POST CHECKS ---", *post_messages])
+
+ conditions = [
+ *output_conditions,
+ *[message for message in post_messages if "FAIL" in message],
+ ]
+ if not output_passed or not post_passed:
+ append_log_file_details(details_lines, runtime_case)
+ return (
+ False,
+ "\n\n" + ("\n" + ("-" * 80) + "\n").join(conditions),
+ sanitize_xml_text("\n".join(details_lines).strip()),
+ False,
+ )
+
+ return (
+ True,
+ "Module main() check passed",
+ sanitize_xml_text("\n".join(details_lines).strip()),
+ False,
+ )
+
+ except Exception as exc: # pylint: disable=broad-exception-caught
+ details_lines.append(traceback.format_exc())
+ return (
+ False,
+ f"Unhandled exception: {exc}",
+ sanitize_xml_text("\n".join(details_lines).strip()),
+ not bool(case_def.get("warn_only", False)),
+ )
+
+ finally:
+ if module is not None:
+ sys.modules.pop(module.__name__, None)
+ shutil.rmtree(temp_dir, ignore_errors=True)
+
+
+def check_path_exists(
+ file_path: Path,
+ case_def: dict[str, Any],
+ work_dir: Path,
+) -> tuple[bool, str, str, bool]:
+ raw_path = case_def.get("path")
+ if not isinstance(raw_path, str):
+ raise ConfigError("'path_exists' requires a string 'path'")
+ resolved = Path(expand_template(raw_path, work_dir, file_path))
+ passed = resolved.exists()
+ message = f"Path exists: {resolved}" if passed else f"Path not found: {resolved}"
+ return passed, message, "", False
+
+
+def check_path_not_exists(
+ file_path: Path,
+ case_def: dict[str, Any],
+ work_dir: Path,
+) -> tuple[bool, str, str, bool]:
+ raw_path = case_def.get("path")
+ if not isinstance(raw_path, str):
+ raise ConfigError("'path_not_exists' requires a string 'path'")
+ resolved = Path(expand_template(raw_path, work_dir, file_path))
+ passed = not resolved.exists()
+ message = (
+ f"Path correctly absent: {resolved}"
+ if passed
+ else f"Path unexpectedly exists: {resolved}"
+ )
+ return passed, message, "", False
+
+
+def run_command_assertion(
+ file_path: Path,
+ case_def: dict[str, Any],
+ work_dir: Path,
+) -> tuple[CommandRunResult, list[str], bool]:
+ spec = {
+ "command": case_def.get("command"),
+ "args": case_def.get("args", []),
+ "shell": case_def.get("shell", False),
+ "timeout_sec": case_def.get("timeout_sec", DEFAULT_CLI_TIMEOUT_SEC),
+ "cwd": case_def.get("cwd"),
+ "env": case_def.get("env"),
+ }
+ result = execute_command_spec(file_path, spec, work_dir)
+ details = [
+ f"Command: {result.command_text}",
+ f"Shell mode: {'yes' if result.shell_mode else 'no'}",
+ f"Timeout seconds: {result.timeout_sec}",
+ f"Timed out: {'yes' if result.timed_out else 'no'}",
+ f"Exit code: {result.exit_code}",
+ "--- STDOUT ---",
+ result.stdout.rstrip(),
+ "--- STDERR ---",
+ result.stderr.rstrip(),
+ ]
+ return result, details, result.timed_out
+
+
+def check_command_success(
+ file_path: Path,
+ case_def: dict[str, Any],
+ work_dir: Path,
+) -> tuple[bool, str, str, bool]:
+ result, details, timed_out = run_command_assertion(file_path, case_def, work_dir)
+ if timed_out:
+ return (
+ False,
+ f"Command timed out after {result.timeout_sec}s",
+ "\n".join(details),
+ True,
+ )
+ passed = result.exit_code == 0
+ message = "Command succeeded" if passed else f"Expected exit code 0, got {result.exit_code}"
+ return passed, message, "\n".join(details), False
+
+
+def check_command_exit_code(
+ file_path: Path,
+ case_def: dict[str, Any],
+ work_dir: Path,
+) -> tuple[bool, str, str, bool]:
+ expected_exit_code = case_def.get("expect_exit_code")
+ if not isinstance(expected_exit_code, int):
+ raise ConfigError("'command_exit_code' requires integer 'expect_exit_code'")
+ result, details, timed_out = run_command_assertion(file_path, case_def, work_dir)
+ if timed_out:
+ return (
+ False,
+ f"Command timed out after {result.timeout_sec}s",
+ "\n".join(details),
+ True,
+ )
+ passed = result.exit_code == expected_exit_code
+ message = (
+ f"Command exited with expected code {expected_exit_code}"
+ if passed
+ else f"Expected exit code {expected_exit_code}, got {result.exit_code}"
+ )
+ return passed, message, "\n".join(details), False
+
+
+def check_command_output_contains(
+ file_path: Path,
+ case_def: dict[str, Any],
+ work_dir: Path,
+) -> tuple[bool, str, str, bool]:
+ expect_text = case_def.get("expect_text")
+ if not isinstance(expect_text, str):
+ raise ConfigError("'command_output_contains' requires string 'expect_text'")
+ result, details, timed_out = run_command_assertion(file_path, case_def, work_dir)
+ if timed_out:
+ return (
+ False,
+ f"Command timed out after {result.timeout_sec}s",
+ "\n".join(details),
+ True,
+ )
+ merged = result.stdout + "\n" + result.stderr
+ passed = expect_text in merged
+ message = (
+ f"Command output contains {expect_text!r}"
+ if passed
+ else f"Command output missing {expect_text!r}"
+ )
+ return passed, message, "\n".join(details), False
+
+
+def check_command_output_regex(
+ file_path: Path,
+ case_def: dict[str, Any],
+ work_dir: Path,
+) -> tuple[bool, str, str, bool]:
+ expect_pattern = case_def.get("expect_pattern")
+ if not isinstance(expect_pattern, str):
+ raise ConfigError("'command_output_regex' requires string 'expect_pattern'")
+ result, details, timed_out = run_command_assertion(file_path, case_def, work_dir)
+ if timed_out:
+ return (
+ False,
+ f"Command timed out after {result.timeout_sec}s",
+ "\n".join(details),
+ True,
+ )
+ merged = result.stdout + "\n" + result.stderr
+ passed = re.search(expect_pattern, merged, flags=re.MULTILINE) is not None
+ message = (
+ f"Command output matches regex {expect_pattern!r}"
+ if passed
+ else f"Command output does not match regex {expect_pattern!r}"
+ )
+ return passed, message, "\n".join(details), False
+
+
+def build_cli_command(
+ file_path: Path,
+ case_def: dict[str, Any],
+ work_dir: Path,
+) -> tuple[list[str] | str, bool]:
+ raw_args = case_def.get("args", [])
+ command_override = case_def.get("command")
+ shell_mode = bool(case_def.get("shell", False))
+
+ expanded_args = [expand_template(arg, work_dir, file_path) for arg in raw_args]
+
+ if command_override:
+ command_value = expand_template(command_override, work_dir, file_path)
+ if shell_mode:
+ parts = [command_value, *[shlex.quote(arg) for arg in expanded_args]]
+ return " ".join(parts), True
+ return [command_value, *expanded_args], False
+
+ base_cmd = [sys.executable, str(file_path), *expanded_args]
+ if shell_mode:
+ return " ".join(shlex.quote(part) for part in base_cmd), True
+ return base_cmd, False
+
+
+def build_cli_env(
+ file_path: Path,
+ case_def: dict[str, Any],
+ work_dir: Path,
+) -> dict[str, str]:
+ env = os.environ.copy()
+ raw_env = case_def.get("env")
+ if raw_env is None:
+ return env
+
+ if not isinstance(raw_env, dict):
+ raise ConfigError("'env' must be a mapping")
+
+ for key, value in raw_env.items():
+ if not isinstance(key, str) or not isinstance(value, str):
+ raise ConfigError("'env' entries must be string -> string")
+ env[key] = expand_template(value, work_dir, file_path)
+
+ return env
+
+
+def check_cli(
+ file_path: Path,
+ case_def: dict[str, Any],
+ work_dir: Path,
+) -> tuple[bool, str, str, bool]:
+ try:
+ runtime_case = build_case_runtime_definition(
+ case_def,
+ work_dir,
+ file_path,
+ )
+ prepare_case_files(work_dir, runtime_case)
+ except (CaseBuildError, MockLoaderConfigError, ValueError, TypeError) as exc:
+ raise ConfigError(str(exc)) from exc
+
+ stdin_text = runtime_case.get("stdin")
+ if stdin_text is not None and not isinstance(stdin_text, str):
+ raise ConfigError("'stdin' must be a string")
+
+ timeout_sec = runtime_case.get("timeout_sec", DEFAULT_CLI_TIMEOUT_SEC)
+ if not isinstance(timeout_sec, int) or timeout_sec <= 0:
+ raise ConfigError("'timeout_sec' must be a positive integer")
+
+ expect_timeout = bool(runtime_case.get("expect_timeout", False))
+ cmd, shell_mode = build_cli_command(file_path, runtime_case, work_dir)
+ run_env = build_cli_env(file_path, runtime_case, work_dir)
+
+ timed_out = False
+ stdout = ""
+ stderr = ""
+ exit_code: int | None = None
+
+ try:
+ completed = REAL_SUBPROCESS_RUN(
+ cmd,
+ cwd=str(work_dir),
+ capture_output=True,
+ text=True,
+ input=stdin_text,
+ check=False,
+ env=run_env,
+ timeout=timeout_sec,
+ shell=shell_mode,
+ )
+ stdout = normalize_completed_stream(completed.stdout)
+ stderr = normalize_completed_stream(completed.stderr)
+ exit_code = completed.returncode
+ except subprocess.TimeoutExpired as exc:
+ timed_out = True
+ stdout = normalize_completed_stream(exc.stdout)
+ stderr = normalize_completed_stream(exc.stderr)
+
+ runtime_case["_actual_exit_code"] = exit_code
+
+ conditions: list[str] = []
+ output_passed = True
+
+ if timed_out:
+ if expect_timeout:
+ output_passed = True
+ else:
+ output_passed = False
+ conditions.append(f"CLI command timed out after {timeout_sec}s")
+ else:
+ if expect_timeout:
+ output_passed = False
+ conditions.append("Expected command to time out, but it completed")
+ else:
+ output_passed, output_conditions = validate_output_expectations(
+ runtime_case,
+ stdout,
+ stderr,
+ )
+ conditions.extend(output_conditions)
+
+ post_passed, post_messages = run_post_checks(
+ work_dir,
+ runtime_case.get("post_checks"),
+ )
+ conditions.extend(message for message in post_messages if "FAIL" in message)
+
+ passed = output_passed and post_passed and not conditions
+
+ command_text = cmd if isinstance(cmd, str) else " ".join(cmd)
+ details_lines = [
+ f"Command: {command_text}",
+ f"Shell mode: {'yes' if shell_mode else 'no'}",
+ f"Timeout seconds: {timeout_sec}",
+ f"Timed out: {'yes' if timed_out else 'no'}",
+ f"Exit code: {exit_code}",
+ "--- STDOUT ---",
+ stdout.rstrip(),
+ "--- STDERR ---",
+ stderr.rstrip(),
+ ]
+
+ if post_messages:
+ details_lines.extend(["--- POST CHECKS ---", *post_messages])
+
+ message = (
+ "CLI check passed"
+ if passed
+ else "\n\n" + ("\n" + ("-" * 80) + "\n").join(conditions)
+ )
+ message = sanitize_xml_text(message)
+ details = sanitize_xml_text("\n".join(details_lines).strip())
+
+ is_error = timed_out and not expect_timeout
+ return passed, message, details, is_error
+
+
+def check_module_cli(
+ file_path: Path,
+ case_def: dict[str, Any],
+ work_dir: Path,
+) -> tuple[bool, str, str, bool]:
+ details_lines: list[str] = [f"Work dir: {work_dir}"]
+
+ try:
+ runtime_case = build_case_runtime_definition(
+ case_def,
+ work_dir,
+ file_path,
+ )
+ details_lines.extend(materialize_case_workspace(work_dir, runtime_case))
+ except (CaseBuildError, MockLoaderConfigError, ValueError, TypeError) as exc:
+ raise ConfigError(str(exc)) from exc
+
+ stdin_text = runtime_case.get("stdin")
+ if stdin_text is not None and not isinstance(stdin_text, str):
+ raise ConfigError("'stdin' must be a string")
+
+ raw_args = runtime_case.get("args", [])
+ if not isinstance(raw_args, list) or not all(isinstance(arg, str) for arg in raw_args):
+ raise ConfigError("'args' must be a list of strings")
+
+ run_env = build_cli_env(file_path, runtime_case, work_dir)
+ patch_constants = runtime_case.get("patch_constants", {})
+ if not isinstance(patch_constants, dict):
+ raise ConfigError("'patch_constants' must be a mapping")
+
+ command_text = " ".join(
+ [shlex.quote(str(file_path)), *(shlex.quote(arg) for arg in raw_args)]
+ )
+ details_lines.append(f"Command: python {command_text}")
+
+ stdout_buffer = io.StringIO()
+ stderr_buffer = io.StringIO()
+ original_argv = sys.argv[:]
+ original_stdin = sys.stdin
+ original_cwd = Path.cwd()
+ original_main = sys.modules.get("__main__")
+ original_env = os.environ.copy()
+ exit_code = 0
+ spec = importlib.util.spec_from_file_location("__main__", str(file_path))
+ if spec is None or spec.loader is None:
+ raise ConfigError(f"Could not load module from {file_path}")
+ script_module = importlib.util.module_from_spec(spec)
+
+ for attr_name, attr_value in patch_constants.items():
+ setattr(script_module, attr_name, attr_value)
+ details_lines.append(f"Patched constant: {attr_name}={attr_value!r}")
+
+ try:
+ sys.argv = [str(file_path), *raw_args]
+ if stdin_text is not None:
+ sys.stdin = io.StringIO(stdin_text)
+ os.chdir(work_dir)
+ os.environ.clear()
+ os.environ.update(run_env)
+ sys.modules["__main__"] = script_module
+
+ try:
+ with apply_case_mocks(
+ runtime_case.get("mocks"),
+ target_context={"module": "__main__"},
+ ):
+ with redirect_stdout(stdout_buffer), redirect_stderr(stderr_buffer):
+ spec.loader.exec_module(script_module)
+ except SystemExit as exc:
+ exit_code = exc.code if isinstance(exc.code, int) else 0
+ except MockExpectationError as exc:
+ stdout = normalize_completed_stream(stdout_buffer.getvalue())
+ stderr = normalize_completed_stream(stderr_buffer.getvalue())
+ details_lines.extend(
+ [
+ f"Exit code: {exit_code}",
+ "--- STDOUT ---",
+ stdout.rstrip(),
+ "--- STDERR ---",
+ stderr.rstrip(),
+ ]
+ )
+ return (
+ False,
+ f"Mock verification failed: {exc}",
+ "\n".join(details_lines),
+ False,
+ )
+ except Exception as exc: # pylint: disable=broad-exception-caught
+ stdout = normalize_completed_stream(stdout_buffer.getvalue())
+ stderr = normalize_completed_stream(stderr_buffer.getvalue())
+ details_lines.extend(
+ [
+ f"Exit code: {exit_code}",
+ "--- STDOUT ---",
+ stdout.rstrip(),
+ "--- STDERR ---",
+ stderr.rstrip(),
+ traceback.format_exc(),
+ ]
+ )
+ return (
+ False,
+ f"Unhandled exception: {exc}",
+ "\n".join(details_lines),
+ not bool(case_def.get("warn_only", False)),
+ )
+ finally:
+ sys.argv = original_argv
+ sys.stdin = original_stdin
+ os.chdir(original_cwd)
+ os.environ.clear()
+ os.environ.update(original_env)
+ if original_main is None:
+ sys.modules.pop("__main__", None)
+ else:
+ sys.modules["__main__"] = original_main
+
+ stdout = normalize_completed_stream(stdout_buffer.getvalue())
+ stderr = normalize_completed_stream(stderr_buffer.getvalue())
+ runtime_case["_actual_exit_code"] = exit_code
+
+ output_passed, output_conditions = validate_output_expectations(
+ runtime_case,
+ stdout,
+ stderr,
+ )
+ post_passed, post_messages = run_post_checks(
+ work_dir,
+ runtime_case.get("post_checks"),
+ )
+ conditions = [*output_conditions, *[msg for msg in post_messages if "FAIL" in msg]]
+
+ passed = output_passed and post_passed and not conditions
+ details_lines.extend(
+ [
+ f"Exit code: {exit_code}",
+ "--- STDOUT ---",
+ stdout.rstrip(),
+ "--- STDERR ---",
+ stderr.rstrip(),
+ ]
+ )
+ if post_messages:
+ details_lines.extend(["--- POST CHECKS ---", *post_messages])
+
+ message = (
+ "Module CLI check passed"
+ if passed
+ else "\n\n" + ("\n" + ("-" * 80) + "\n").join(conditions)
+ )
+ message = sanitize_xml_text(message)
+ details = sanitize_xml_text("\n".join(details_lines).strip())
+ return passed, message, details, False
+
+
+CHECK_HANDLERS: dict[
+ str,
+ Callable[[Path, dict[str, Any], Path], tuple[bool, str, str, bool]],
+] = {
+ "file_exists": check_file_exists,
+ "py_compile": check_py_compile,
+ "source_contains": check_source_contains,
+ "source_contains_any": check_source_contains_any,
+ "source_contains_all": check_source_contains_all,
+ "function_exists": check_function_exists,
+ "function_exists_any": check_function_exists_any,
+ "main_guard": check_main_guard,
+ "cli": check_cli,
+ "module_cli": check_module_cli,
+ "py_function": check_py_function,
+ "module_main_with_env": check_module_main_with_env,
+ "path_exists": check_path_exists,
+ "path_not_exists": check_path_not_exists,
+ "command_success": check_command_success,
+ "command_exit_code": check_command_exit_code,
+ "command_output_contains": check_command_output_contains,
+ "command_output_regex": check_command_output_regex,
+}
+
+
+def run_single_check(
+ file_path: Path,
+ case_def: dict[str, Any],
+ work_dir: Path,
+) -> tuple[bool, str, str, bool]:
+ case_type = case_def.get("type", "cli")
+ if not isinstance(case_type, str):
+ raise ConfigError("Case 'type' must be a string")
+
+ apply_skip_controls(file_path, case_def, work_dir)
+
+ handler = CHECK_HANDLERS.get(case_type)
+ if handler is None:
+ raise ConfigError(f"Unsupported test type: {case_type}")
+
+ return handler(file_path, case_def, work_dir)
diff --git a/common/acs_test_framework_runner/runner_check_validation.py b/common/acs_test_framework_runner/runner_check_validation.py
new file mode 100644
index 00000000..152fe326
--- /dev/null
+++ b/common/acs_test_framework_runner/runner_check_validation.py
@@ -0,0 +1,552 @@
+from __future__ import annotations
+
+from typing import Any
+
+try: # Support package imports and direct harness module loading.
+ from .runner_checks import (
+ DEFAULT_CLI_TIMEOUT_SEC,
+ ConfigError,
+ ensure_list,
+ ensure_list_of_strings,
+ ensure_string_or_list_of_strings,
+ merge_mappings,
+ )
+except ImportError: # pragma: no cover - exercised by flat-module harness imports.
+ from runner_checks import (
+ DEFAULT_CLI_TIMEOUT_SEC,
+ ConfigError,
+ ensure_list,
+ ensure_list_of_strings,
+ ensure_string_or_list_of_strings,
+ merge_mappings,
+ )
+
+
+def normalize_suite_files(files_value: Any, field_name: str) -> list[str]:
+ items = ensure_list(files_value, field_name)
+ if not items:
+ raise ConfigError(f"'{field_name}' must not be empty")
+
+ normalized: list[str] = []
+ for item in items:
+ if isinstance(item, str):
+ normalized.append(item)
+ continue
+ if isinstance(item, dict) and isinstance(item.get("path"), str):
+ normalized.append(item["path"])
+ continue
+ raise ConfigError(
+ f"Each entry in '{field_name}' must be either a string or {{path: ...}}"
+ )
+ return normalized
+
+
+def validate_post_checks(post_checks: Any, field_name: str) -> None:
+ if post_checks is None:
+ return
+
+ checks = ensure_list(post_checks, field_name)
+ supported = {
+ "exists",
+ "not_exists",
+ "file_contains",
+ "file_not_contains",
+ "file_not_empty",
+ "regex",
+ "ordered_contains",
+ }
+
+ for index, check in enumerate(checks, start=1):
+ entry_name = f"{field_name}[{index}]"
+ if not isinstance(check, dict):
+ raise ConfigError(f"{entry_name} must be a mapping")
+
+ check_type = check.get("type")
+ if not isinstance(check_type, str):
+ raise ConfigError(f"{entry_name}.type must be a string")
+
+ if check_type not in supported:
+ raise ConfigError(
+ f"{entry_name}.type unsupported: {check_type}. "
+ f"Supported types: {', '.join(sorted(supported))}"
+ )
+
+ raw_path = check.get("path")
+ if not isinstance(raw_path, str):
+ raise ConfigError(f"{entry_name}.path must be a string")
+
+ if check_type in {"file_contains", "file_not_contains"}:
+ text = check.get("text")
+ if not isinstance(text, str):
+ raise ConfigError(f"{entry_name}.text must be a string")
+
+ if check_type == "regex":
+ pattern = check.get("pattern")
+ if not isinstance(pattern, str):
+ raise ConfigError(f"{entry_name}.pattern must be a string")
+
+ if check_type == "ordered_contains":
+ texts = check.get("texts")
+ if not isinstance(texts, list) or not all(
+ isinstance(item, str) for item in texts
+ ):
+ raise ConfigError(f"{entry_name}.texts must be a list of strings")
+
+
+def validate_env_mapping(
+ value: Any,
+ field_name: str,
+ *,
+ allow_bool_values: bool = False,
+) -> None:
+ if not isinstance(value, dict):
+ raise ConfigError(f"{field_name} must be a mapping")
+
+ valid_value_types = (str, bool) if allow_bool_values else (str,)
+ for key, item in value.items():
+ if not isinstance(key, str):
+ raise ConfigError(f"{field_name} keys must be strings")
+ if not isinstance(item, valid_value_types):
+ raise ConfigError(
+ f"{field_name}['{key}'] must be "
+ f"{'string/bool' if allow_bool_values else 'a string'}"
+ )
+
+
+def validate_command_probe_list(value: Any, field_name: str) -> None:
+ probes = ensure_list(value, field_name)
+ for index, probe in enumerate(probes, start=1):
+ entry_name = f"{field_name}[{index}]"
+ if isinstance(probe, str):
+ continue
+ if not isinstance(probe, dict):
+ raise ConfigError(f"{entry_name} must be a string or mapping")
+
+ command = probe.get("command")
+ if not isinstance(command, str):
+ raise ConfigError(f"{entry_name}.command must be a string")
+
+ args = probe.get("args")
+ if args is not None and (
+ not isinstance(args, list) or not all(isinstance(item, str) for item in args)
+ ):
+ raise ConfigError(f"{entry_name}.args must be a list of strings")
+
+ shell_value = probe.get("shell")
+ if shell_value is not None and not isinstance(shell_value, bool):
+ raise ConfigError(f"{entry_name}.shell must be a boolean")
+
+ timeout_sec = probe.get("timeout_sec")
+ if timeout_sec is not None and (
+ not isinstance(timeout_sec, int) or timeout_sec <= 0
+ ):
+ raise ConfigError(f"{entry_name}.timeout_sec must be a positive integer")
+
+ cwd = probe.get("cwd")
+ if cwd is not None and not isinstance(cwd, str):
+ raise ConfigError(f"{entry_name}.cwd must be a string")
+
+ env = probe.get("env")
+ if env is not None:
+ validate_env_mapping(env, f"{entry_name}.env")
+
+
+def validate_common_case_controls(case_def: dict[str, Any], field_name: str) -> None:
+ skip_unless_env = case_def.get("skip_unless_env")
+ if skip_unless_env is not None:
+ validate_env_mapping(
+ skip_unless_env,
+ f"{field_name}.skip_unless_env",
+ allow_bool_values=True,
+ )
+
+ skip_unless_paths_exist = case_def.get("skip_unless_paths_exist")
+ if skip_unless_paths_exist is not None:
+ ensure_string_or_list_of_strings(
+ skip_unless_paths_exist,
+ f"{field_name}.skip_unless_paths_exist",
+ )
+
+ skip_unless_commands_succeed = case_def.get("skip_unless_commands_succeed")
+ if skip_unless_commands_succeed is not None:
+ validate_command_probe_list(
+ skip_unless_commands_succeed,
+ f"{field_name}.skip_unless_commands_succeed",
+ )
+
+ requires_destructive = case_def.get("requires_destructive")
+ if requires_destructive is not None and not isinstance(requires_destructive, bool):
+ raise ConfigError(f"{field_name}.requires_destructive must be a boolean")
+
+ required_env = case_def.get("required_env")
+ if required_env is not None:
+ validate_env_mapping(required_env, f"{field_name}.required_env")
+
+ warn_only = case_def.get("warn_only")
+ if warn_only is not None and not isinstance(warn_only, bool):
+ raise ConfigError(f"{field_name}.warn_only must be a boolean")
+
+
+def validate_mock_spec(spec: Any, field_name: str) -> None:
+ """Validate one mocks. spec conservatively."""
+ if isinstance(spec, str):
+ return
+
+ if not isinstance(spec, dict):
+ raise ConfigError(
+ f"{field_name} must be a string or mapping, got {type(spec).__name__}"
+ )
+
+ if "factory" in spec and spec["factory"] is not None and not isinstance(
+ spec["factory"], str
+ ):
+ raise ConfigError(f"{field_name}.factory must be a string when provided")
+
+ if "inject_original_as" in spec and not isinstance(
+ spec["inject_original_as"], str
+ ):
+ raise ConfigError(f"{field_name}.inject_original_as must be a string")
+
+ if "args" in spec and spec["args"] is not None and not isinstance(spec["args"], list):
+ raise ConfigError(f"{field_name}.args must be a list")
+
+ if "kwargs" in spec and spec["kwargs"] is not None and not isinstance(
+ spec["kwargs"], dict
+ ):
+ raise ConfigError(f"{field_name}.kwargs must be a mapping")
+
+ if "attrs" in spec and spec["attrs"] is not None and not isinstance(
+ spec["attrs"], dict
+ ):
+ raise ConfigError(f"{field_name}.attrs must be a mapping")
+
+
+def validate_case_mocks(case_def: dict[str, Any], field_name: str) -> None:
+ """Validate optional mocks/scenario fields."""
+ mocks = case_def.get("mocks")
+ if mocks is not None:
+ if not isinstance(mocks, dict):
+ raise ConfigError(f"{field_name}.mocks must be a mapping")
+ for target, spec in mocks.items():
+ if not isinstance(target, str) or not target.strip():
+ raise ConfigError(f"{field_name}.mocks keys must be non-empty strings")
+ validate_mock_spec(spec, f"{field_name}.mocks[{target!r}]")
+
+ scenario = case_def.get("scenario")
+ if scenario is not None and not isinstance(scenario, dict):
+ raise ConfigError(f"{field_name}.scenario must be a mapping")
+
+
+def validate_case_schema(case_def: dict[str, Any], field_name: str) -> None:
+ case_name = case_def.get("name")
+ if not isinstance(case_name, str) or not case_name.strip():
+ raise ConfigError(f"{field_name}.name must be a non-empty string")
+
+ case_type = case_def.get("type", "cli")
+ if not isinstance(case_type, str):
+ raise ConfigError(f"{field_name}.type must be a string")
+
+ if case_type not in {
+ "file_exists",
+ "py_compile",
+ "source_contains",
+ "source_contains_any",
+ "source_contains_all",
+ "function_exists",
+ "function_exists_any",
+ "main_guard",
+ "cli",
+ "module_cli",
+ "py_function",
+ "module_main_with_env",
+ "path_exists",
+ "path_not_exists",
+ "command_success",
+ "command_exit_code",
+ "command_output_contains",
+ "command_output_regex",
+ }:
+ raise ConfigError(f"{field_name}.type unsupported: {case_type}")
+
+ validate_common_case_controls(case_def, field_name)
+ validate_case_mocks(case_def, field_name)
+
+ if "scripts" in case_def and case_def["scripts"] is not None:
+ scripts = case_def["scripts"]
+ if not isinstance(scripts, dict):
+ raise ConfigError(f"{field_name}.scripts must be a mapping")
+ for key, value in scripts.items():
+ if not isinstance(key, str) or not isinstance(value, str):
+ raise ConfigError(
+ f"{field_name}.scripts entries must be string -> string"
+ )
+
+ if "bin_files" in case_def and case_def["bin_files"] is not None:
+ bin_files = case_def["bin_files"]
+ if not isinstance(bin_files, dict):
+ raise ConfigError(f"{field_name}.bin_files must be a mapping")
+ for key, spec in bin_files.items():
+ if not isinstance(key, str) or not isinstance(spec, dict):
+ raise ConfigError(
+ f"{field_name}.bin_files entries must be string -> mapping"
+ )
+ hex_data = spec.get("hex")
+ text_data = spec.get("text")
+ if hex_data is None and text_data is None:
+ raise ConfigError(
+ f"{field_name}.bin_files['{key}'] requires 'hex' or 'text'"
+ )
+ if hex_data is not None and not isinstance(hex_data, str):
+ raise ConfigError(
+ f"{field_name}.bin_files['{key}'].hex must be a string"
+ )
+ if text_data is not None and not isinstance(text_data, str):
+ raise ConfigError(
+ f"{field_name}.bin_files['{key}'].text must be a string"
+ )
+
+ if "text_files" in case_def and case_def["text_files"] is not None:
+ text_files = case_def["text_files"]
+ if not isinstance(text_files, dict):
+ raise ConfigError(f"{field_name}.text_files must be a mapping")
+ for key, value in text_files.items():
+ if not isinstance(key, str) or not isinstance(value, str):
+ raise ConfigError(
+ f"{field_name}.text_files entries must be string -> string"
+ )
+
+ if "patch_constants" in case_def and case_def["patch_constants"] is not None:
+ patch_constants = case_def["patch_constants"]
+ if not isinstance(patch_constants, dict):
+ raise ConfigError(f"{field_name}.patch_constants must be a mapping")
+ for key, value in patch_constants.items():
+ if not isinstance(key, str):
+ raise ConfigError(f"{field_name}.patch_constants keys must be strings")
+ if not isinstance(value, (str, int, float, bool)):
+ raise ConfigError(
+ f"{field_name}.patch_constants['{key}'] "
+ "must be a scalar string/int/float/bool"
+ )
+
+ if "dir_structure" in case_def and case_def["dir_structure"] is not None:
+ dir_structure = case_def["dir_structure"]
+ if not isinstance(dir_structure, list):
+ raise ConfigError(f"{field_name}.dir_structure must be a list")
+ for index, entry in enumerate(dir_structure, start=1):
+ if not isinstance(entry, dict):
+ raise ConfigError(f"{field_name}.dir_structure[{index}] must be a mapping")
+ if not isinstance(entry.get("path"), str):
+ raise ConfigError(
+ f"{field_name}.dir_structure[{index}].path must be a string"
+ )
+
+ validate_post_checks(case_def.get("post_checks"), f"{field_name}.post_checks")
+
+ if case_type == "py_function":
+ function_name = case_def.get("function")
+ if not isinstance(function_name, str) or not function_name.strip():
+ raise ConfigError(f"{field_name}.function must be a non-empty string")
+
+ args = case_def.get("args", [])
+ if not isinstance(args, list):
+ raise ConfigError(f"{field_name}.args must be a list")
+
+ kwargs = case_def.get("kwargs")
+ if kwargs is not None and not isinstance(kwargs, dict):
+ raise ConfigError(f"{field_name}.kwargs must be a mapping")
+
+ if "expect_exception" in case_def and not isinstance(
+ case_def["expect_exception"], str
+ ):
+ raise ConfigError(f"{field_name}.expect_exception must be a string")
+
+ if "expect_return_contains" in case_def and not isinstance(
+ case_def["expect_return_contains"], str
+ ):
+ raise ConfigError(f"{field_name}.expect_return_contains must be a string")
+ return
+
+ if case_type == "module_main_with_env":
+ expected_exit_code = case_def.get("expect_exit_code")
+ if expected_exit_code is not None and not isinstance(expected_exit_code, int):
+ raise ConfigError(f"{field_name}.expect_exit_code must be an integer")
+ return
+
+ if case_type in {"path_exists", "path_not_exists"}:
+ path_value = case_def.get("path")
+ if not isinstance(path_value, str):
+ raise ConfigError(f"{field_name}.path must be a string")
+ return
+
+ if case_type in {
+ "command_success",
+ "command_exit_code",
+ "command_output_contains",
+ "command_output_regex",
+ }:
+ command = case_def.get("command")
+ if not isinstance(command, str):
+ raise ConfigError(f"{field_name}.command must be a string")
+
+ args = case_def.get("args", [])
+ if not isinstance(args, list) or not all(isinstance(arg, str) for arg in args):
+ raise ConfigError(f"{field_name}.args must be a list of strings")
+
+ shell_value = case_def.get("shell", False)
+ if not isinstance(shell_value, bool):
+ raise ConfigError(f"{field_name}.shell must be a boolean")
+
+ timeout_sec = case_def.get("timeout_sec", DEFAULT_CLI_TIMEOUT_SEC)
+ if not isinstance(timeout_sec, int) or timeout_sec <= 0:
+ raise ConfigError(f"{field_name}.timeout_sec must be a positive integer")
+
+ cwd = case_def.get("cwd")
+ if cwd is not None and not isinstance(cwd, str):
+ raise ConfigError(f"{field_name}.cwd must be a string")
+
+ env = case_def.get("env")
+ if env is not None:
+ validate_env_mapping(env, f"{field_name}.env")
+
+ if case_type == "command_exit_code":
+ expected_exit_code = case_def.get("expect_exit_code")
+ if not isinstance(expected_exit_code, int):
+ raise ConfigError(f"{field_name}.expect_exit_code must be an integer")
+
+ if case_type == "command_output_contains":
+ expect_text = case_def.get("expect_text")
+ if not isinstance(expect_text, str):
+ raise ConfigError(f"{field_name}.expect_text must be a string")
+
+ if case_type == "command_output_regex":
+ expect_pattern = case_def.get("expect_pattern")
+ if not isinstance(expect_pattern, str):
+ raise ConfigError(f"{field_name}.expect_pattern must be a string")
+ return
+
+ if case_type not in {"cli", "module_cli"}:
+ return
+
+ raw_args = case_def.get("args", [])
+ if not isinstance(raw_args, list) or not all(isinstance(arg, str) for arg in raw_args):
+ raise ConfigError(f"{field_name}.args must be a list of strings")
+
+ command = case_def.get("command")
+ if command is not None and not isinstance(command, str):
+ raise ConfigError(f"{field_name}.command must be a string")
+
+ stdin_text = case_def.get("stdin")
+ if stdin_text is not None and not isinstance(stdin_text, str):
+ raise ConfigError(f"{field_name}.stdin must be a string")
+
+ timeout_sec = case_def.get("timeout_sec", DEFAULT_CLI_TIMEOUT_SEC)
+ if not isinstance(timeout_sec, int) or timeout_sec <= 0:
+ raise ConfigError(f"{field_name}.timeout_sec must be a positive integer")
+
+ shell_value = case_def.get("shell", False)
+ if not isinstance(shell_value, bool):
+ raise ConfigError(f"{field_name}.shell must be a boolean")
+
+ expect_timeout = case_def.get("expect_timeout", False)
+ if not isinstance(expect_timeout, bool):
+ raise ConfigError(f"{field_name}.expect_timeout must be a boolean")
+
+ env = case_def.get("env")
+ if env is not None:
+ validate_env_mapping(env, f"{field_name}.env")
+
+ expected_exit_code = case_def.get("expect_exit_code")
+ if expected_exit_code is not None and not isinstance(expected_exit_code, int):
+ raise ConfigError(f"{field_name}.expect_exit_code must be an integer")
+
+ expect_exit_nonzero = case_def.get("expect_exit_nonzero", False)
+ if not isinstance(expect_exit_nonzero, bool):
+ raise ConfigError(f"{field_name}.expect_exit_nonzero must be a boolean")
+
+ expect_exit_code_in = case_def.get("expect_exit_code_in")
+ if expect_exit_code_in is not None:
+ if not isinstance(expect_exit_code_in, list) or not all(
+ isinstance(item, int) for item in expect_exit_code_in
+ ):
+ raise ConfigError(
+ f"{field_name}.expect_exit_code_in must be a list of integers"
+ )
+
+ expect_stdout_or_stderr_regex = case_def.get("expect_stdout_or_stderr_regex")
+ if expect_stdout_or_stderr_regex is not None:
+ if isinstance(expect_stdout_or_stderr_regex, str):
+ return
+ if not isinstance(expect_stdout_or_stderr_regex, list) or not all(
+ isinstance(item, str) for item in expect_stdout_or_stderr_regex
+ ):
+ raise ConfigError(
+ f"{field_name}.expect_stdout_or_stderr_regex must be a string or list of strings"
+ )
+
+
+def normalize_cases(
+ value: Any,
+ field_name: str,
+ defaults: dict[str, Any] | None = None,
+) -> list[dict[str, Any]]:
+ items = ensure_list(value, field_name)
+ normalized: list[dict[str, Any]] = []
+ suite_defaults = defaults or {}
+
+ for index, case_def in enumerate(items, start=1):
+ if not isinstance(case_def, dict):
+ raise ConfigError(f"{field_name}[{index}] must be a mapping")
+ merged_case = merge_mappings(suite_defaults, case_def)
+ if case_def.get("expect_exit_nonzero") and "expect_exit_code" not in case_def:
+ merged_case.pop("expect_exit_code", None)
+ if "expect_exit_code_in" in case_def and "expect_exit_code" not in case_def:
+ merged_case.pop("expect_exit_code", None)
+ validate_case_schema(merged_case, f"{field_name}[{index}]")
+ normalized.append(merged_case)
+
+ return normalized
+
+
+def normalize_suites(config: dict[str, Any]) -> list[dict[str, Any]]:
+ raw_suites = config.get("suites")
+ if raw_suites is None:
+ raise ConfigError("Top-level key 'suites' is required")
+
+ suites = ensure_list(raw_suites, "suites")
+ if not suites:
+ raise ConfigError("'suites' must not be empty")
+
+ normalized: list[dict[str, Any]] = []
+ for index, suite in enumerate(suites, start=1):
+ if not isinstance(suite, dict):
+ raise ConfigError(f"suites[{index}] must be a mapping")
+
+ name = suite.get("name")
+ if not isinstance(name, str) or not name.strip():
+ raise ConfigError(f"suites[{index}] requires a non-empty string 'name'")
+
+ suite_command = suite.get("command")
+ if suite_command is not None and not isinstance(suite_command, str):
+ raise ConfigError(f"suites[{index}].command must be a string")
+
+ defaults = suite.get("defaults") or {}
+ if not isinstance(defaults, dict):
+ raise ConfigError(f"suites[{index}].defaults must be a mapping")
+
+ files = normalize_suite_files(suite.get("files"), f"suites[{index}].files")
+ cases = normalize_cases(
+ suite.get("cases", []),
+ f"suites[{index}].cases",
+ defaults=defaults,
+ )
+
+ normalized.append(
+ {
+ "name": name.strip(),
+ "command": suite_command,
+ "files": files,
+ "cases": cases,
+ }
+ )
+
+ return normalized
diff --git a/common/acs_test_framework_runner/runner_checks.py b/common/acs_test_framework_runner/runner_checks.py
new file mode 100644
index 00000000..00559353
--- /dev/null
+++ b/common/acs_test_framework_runner/runner_checks.py
@@ -0,0 +1,731 @@
+from __future__ import annotations
+
+import ast
+import importlib.util
+import re
+import subprocess
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+from uuid import uuid4
+
+import yaml
+
+try: # Support package imports and direct harness module loading.
+ from .case_data_builders import render_post_check_path
+except ImportError: # pragma: no cover - exercised by flat-module harness imports.
+ from case_data_builders import render_post_check_path
+
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+
+
+def detect_project_root(script_dir: Path) -> Path:
+ if script_dir.parent.name == "common":
+ return script_dir.parent.parent
+ return script_dir.parent
+
+
+PROJECT_ROOT = detect_project_root(SCRIPT_DIR)
+TEST_YAML_DIR = PROJECT_ROOT / "common" / "acs_test_framework_manifests"
+REPORTS_DIR = PROJECT_ROOT / "common" / "reports"
+RUNNER_WORK_DIR = REPORTS_DIR / "_runner_work"
+SUPPORTED_SUFFIXES = {".yaml", ".yml"}
+DEFAULT_CLI_TIMEOUT_SEC = 20
+DESTRUCTIVE_TEST_ENV = "RUN_DESTRUCTIVE_HW_TESTS"
+# Runner-managed process launches must not be affected by case-level
+# subprocess.run mocks that target the global subprocess module.
+REAL_SUBPROCESS_RUN = subprocess.run
+
+
+@dataclass
+class TestMeta:
+ suite_name: str
+ phase: str
+ test_type: str
+
+
+@dataclass
+class TestOutcome:
+ testcase_name: str
+ file_path: str
+ passed: bool
+ message: str
+ meta: TestMeta
+ details: str = ""
+ flags: dict[str, bool] = field(
+ default_factory=lambda: {
+ "error": False,
+ "skipped": False,
+ "warning": False,
+ }
+ )
+
+ @property
+ def error(self) -> bool:
+ return self.flags["error"]
+
+ @property
+ def skipped(self) -> bool:
+ return self.flags["skipped"]
+
+ @property
+ def warning(self) -> bool:
+ return self.flags["warning"]
+
+
+class ConfigError(Exception):
+ """Raised when a YAML configuration is invalid."""
+
+
+class SkipCase(Exception):
+ """Raised when a case should be skipped due to environment gating."""
+
+
+@dataclass
+class CommandRunResult:
+ command_text: str
+ stdout: str
+ stderr: str
+ exit_code: int | None
+ timed_out: bool
+ timeout_sec: int
+ shell_mode: bool
+
+
+@dataclass(frozen=True)
+class RunCaseOptions:
+ suite_command: str | None = None
+
+
+def create_runner_temp_dir(prefix: str = "runner_env_") -> Path:
+ RUNNER_WORK_DIR.mkdir(parents=True, exist_ok=True)
+ for _ in range(100):
+ candidate = RUNNER_WORK_DIR / f"{prefix}{uuid4().hex[:8]}"
+ try:
+ candidate.mkdir(parents=False, exist_ok=False)
+ return candidate
+ except FileExistsError:
+ continue
+ raise OSError(f"Could not create runner temp dir under {RUNNER_WORK_DIR}")
+
+
+def load_yaml_config(yaml_file: Path) -> dict[str, Any]:
+ try:
+ with yaml_file.open("r", encoding="utf-8") as handle:
+ data = yaml.safe_load(handle) or {}
+ except yaml.YAMLError as exc:
+ raise ConfigError(f"Failed to parse YAML: {exc}") from exc
+
+ if not isinstance(data, dict):
+ raise ConfigError("Top-level YAML content must be a mapping")
+
+ return data
+
+
+def ensure_list(value: Any, field_name: str) -> list[Any]:
+ if not isinstance(value, list):
+ raise ConfigError(f"'{field_name}' must be a list")
+ return value
+
+
+def ensure_list_of_strings(value: Any, field_name: str) -> list[str]:
+ if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
+ raise ConfigError(f"'{field_name}' must be a list of strings")
+ return value
+
+
+def ensure_string_or_list_of_strings(value: Any, field_name: str) -> list[str]:
+ if value is None:
+ return []
+ if isinstance(value, str):
+ return [value]
+ return ensure_list_of_strings(value, field_name)
+
+
+def merge_mappings(
+ base: dict[str, Any],
+ override: dict[str, Any],
+) -> dict[str, Any]:
+ """Recursively merge mappings with override precedence."""
+ merged: dict[str, Any] = dict(base)
+ for key, value in override.items():
+ existing = merged.get(key)
+ if isinstance(existing, dict) and isinstance(value, dict):
+ merged[key] = merge_mappings(existing, value)
+ else:
+ merged[key] = value
+ return merged
+
+
+def sanitize_name(value: str) -> str:
+ return "".join(
+ char if char.isalnum() or char in {"-", "_", "."} else "_"
+ for char in value
+ )
+
+
+def is_valid_xml_char(code: int) -> bool:
+ return (
+ code in {0x9, 0xA, 0xD}
+ or 0x20 <= code <= 0xD7FF
+ or 0xE000 <= code <= 0xFFFD
+ or 0x10000 <= code <= 0x10FFFF
+ )
+
+
+def sanitize_xml_text(value: Any) -> str:
+ if value is None:
+ return ""
+ if not isinstance(value, str):
+ value = str(value)
+
+ cleaned: list[str] = []
+ for char in value:
+ if is_valid_xml_char(ord(char)):
+ cleaned.append(char)
+ return "".join(cleaned)
+
+
+def resolve_target_path(file_entry: str) -> Path:
+ raw = Path(file_entry)
+ if raw.is_absolute():
+ return raw.resolve()
+ return (PROJECT_ROOT / raw).resolve()
+
+
+def read_source(file_path: Path) -> str:
+ return file_path.read_text(encoding="utf-8")
+
+
+def parse_ast(file_path: Path) -> ast.AST:
+ return ast.parse(read_source(file_path), filename=str(file_path))
+
+
+def collect_function_names(file_path: Path) -> set[str]:
+ tree = parse_ast(file_path)
+ names: set[str] = set()
+
+ for node in ast.walk(tree):
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+ names.add(node.name)
+
+ return names
+
+
+def format_outcome_message(prefix: str, text: str) -> str:
+ text = text.strip()
+ return prefix if not text else f"{prefix}: {text}"
+
+
+def create_outcome(
+ testcase_name: str,
+ file_path: str,
+ passed: bool,
+ message: str,
+ meta: TestMeta,
+ **kwargs: Any,
+) -> TestOutcome:
+ return TestOutcome(
+ testcase_name=testcase_name,
+ file_path=file_path,
+ passed=passed,
+ message=message,
+ meta=meta,
+ details=kwargs.get("details", ""),
+ flags={
+ "error": kwargs.get("error", False),
+ "skipped": kwargs.get("skipped", False),
+ "warning": kwargs.get("warning", False),
+ },
+ )
+
+
+def build_runner_module_name(file_path: Path) -> str:
+ return f"runner_module_{sanitize_name(file_path.stem)}_{uuid4().hex}"
+
+
+def load_module_from_path(file_path: Path) -> Any:
+ module_name = build_runner_module_name(file_path)
+ spec = importlib.util.spec_from_file_location(
+ module_name,
+ str(file_path),
+ )
+ if spec is None or spec.loader is None:
+ raise ConfigError(f"Could not load module from {file_path}")
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[module_name] = module
+ try:
+ spec.loader.exec_module(module)
+ except Exception:
+ sys.modules.pop(module_name, None)
+ raise
+ return module
+
+
+def normalize_completed_stream(stream: Any) -> str:
+ if stream is None:
+ return ""
+ if isinstance(stream, str):
+ return stream
+ if isinstance(stream, bytes):
+ return stream.decode("utf-8", errors="replace")
+ return str(stream)
+
+
+try: # Support package imports and direct harness module loading.
+ from .runner_check_validation import (
+ normalize_cases,
+ normalize_suite_files,
+ normalize_suites,
+ validate_case_mocks,
+ validate_case_schema,
+ validate_command_probe_list,
+ validate_common_case_controls,
+ validate_env_mapping,
+ validate_mock_spec,
+ validate_post_checks,
+ )
+except ImportError: # pragma: no cover - exercised by flat-module harness imports.
+ from runner_check_validation import (
+ normalize_cases,
+ normalize_suite_files,
+ normalize_suites,
+ validate_case_mocks,
+ validate_case_schema,
+ validate_command_probe_list,
+ validate_common_case_controls,
+ validate_env_mapping,
+ validate_mock_spec,
+ validate_post_checks,
+ )
+
+
+def run_post_checks(work_dir: Path, post_checks: Any) -> tuple[bool, list[str]]:
+ if post_checks is None:
+ return True, []
+
+ checks = ensure_list(post_checks, "post_checks")
+ messages: list[str] = []
+ all_passed = True
+
+ for index, check in enumerate(checks, start=1):
+ if not isinstance(check, dict):
+ raise ConfigError(f"post_checks[{index}] must be a mapping")
+
+ check_type = check.get("type")
+ raw_path = check.get("path")
+
+ if not isinstance(check_type, str):
+ raise ConfigError(f"post_checks[{index}] requires string key 'type'")
+ if not isinstance(raw_path, str):
+ raise ConfigError(f"post_checks[{index}] requires string key 'path'")
+
+ resolved = render_post_check_path(raw_path, work_dir)
+
+ if check_type == "exists":
+ exists = resolved.exists()
+ messages.append(
+ f"post_check exists: {resolved} -> {'PASS' if exists else 'FAIL'}"
+ )
+ if not exists:
+ all_passed = False
+ continue
+
+ if check_type == "not_exists":
+ missing = not resolved.exists()
+ messages.append(
+ f"post_check not_exists: {resolved} -> {'PASS' if missing else 'FAIL'}"
+ )
+ if not missing:
+ all_passed = False
+ continue
+
+ if check_type == "file_not_empty":
+ passed = (
+ resolved.exists()
+ and resolved.is_file()
+ and resolved.stat().st_size > 0
+ )
+ messages.append(
+ f"post_check file_not_empty: {resolved} -> {'PASS' if passed else 'FAIL'}"
+ )
+ if not passed:
+ all_passed = False
+ continue
+
+ if check_type in {"file_contains", "file_not_contains"}:
+ text = check.get("text")
+ if not isinstance(text, str):
+ raise ConfigError(f"post_checks[{index}] requires string key 'text'")
+
+ if not resolved.exists() or not resolved.is_file():
+ messages.append(
+ f"post_check {check_type}: {resolved} contains {text!r} -> FAIL"
+ )
+ all_passed = False
+ continue
+
+ contents = resolved.read_text(encoding="utf-8", errors="replace")
+ contains = text in contents
+ passed = contains if check_type == "file_contains" else not contains
+
+ messages.append(
+ f"post_check {check_type}: {resolved} contains {text!r} -> "
+ f"{'PASS' if passed else 'FAIL'}"
+ )
+ if not passed:
+ all_passed = False
+ continue
+
+ if check_type == "regex":
+ pattern = check.get("pattern")
+ if not isinstance(pattern, str):
+ raise ConfigError(
+ f"post_checks[{index}] requires string key 'pattern'"
+ )
+
+ if not resolved.exists() or not resolved.is_file():
+ messages.append(
+ f"post_check regex: {resolved} matches {pattern!r} -> FAIL"
+ )
+ all_passed = False
+ continue
+
+ contents = resolved.read_text(encoding="utf-8", errors="replace")
+ passed = re.search(pattern, contents, flags=re.MULTILINE) is not None
+ messages.append(
+ f"post_check regex: {resolved} matches {pattern!r} -> "
+ f"{'PASS' if passed else 'FAIL'}"
+ )
+ if not passed:
+ all_passed = False
+ continue
+
+ if check_type == "ordered_contains":
+ texts = check.get("texts")
+ if not isinstance(texts, list) or not all(
+ isinstance(item, str) for item in texts
+ ):
+ raise ConfigError(
+ f"post_checks[{index}] requires key 'texts' as list[str]"
+ )
+
+ if not resolved.exists() or not resolved.is_file():
+ messages.append(f"post_check ordered_contains: {resolved} -> FAIL")
+ all_passed = False
+ continue
+
+ contents = resolved.read_text(encoding="utf-8", errors="replace")
+ start = 0
+ passed = True
+ for text in texts:
+ idx = contents.find(text, start)
+ if idx == -1:
+ passed = False
+ messages.append(
+ f"post_check ordered_contains: {resolved} "
+ f"missing {text!r} in order -> FAIL"
+ )
+ all_passed = False
+ break
+ start = idx + len(text)
+
+ if passed:
+ messages.append(f"post_check ordered_contains: {resolved} -> PASS")
+ continue
+
+ raise ConfigError(f"post_checks[{index}] unsupported type: {check_type}")
+
+ return all_passed, messages
+
+
+def format_output_block(title: str, text: str) -> str:
+ cleaned = text.rstrip()
+ if not cleaned:
+ cleaned = ""
+ return f"{title}\n{cleaned}"
+
+
+def append_log_file_details(
+ details_lines: list[str],
+ runtime_case: dict[str, Any],
+) -> None:
+ patch_constants = runtime_case.get("patch_constants", {})
+ if not isinstance(patch_constants, dict):
+ return
+
+ log_path_value = patch_constants.get("LOG_FILE")
+ if not isinstance(log_path_value, str):
+ return
+
+ log_path = Path(log_path_value)
+ details_lines.append("--- LOG FILE ---")
+
+ if not log_path.exists() or not log_path.is_file():
+ details_lines.append(f"")
+ return
+
+ log_text = log_path.read_text(encoding="utf-8", errors="replace").rstrip()
+ details_lines.append(log_text if log_text else "")
+
+
+def format_expectation_failure(
+ check_type: str,
+ expected: str,
+ stdout: str,
+ stderr: str,
+ *,
+ actual_label: str = "",
+) -> str:
+ lines = [
+ "PHASE: expectation_check",
+ f"CHECK TYPE: {check_type}",
+ "",
+ "EXPECTED:",
+ f" {expected}",
+ "",
+ format_output_block("ACTUAL STDOUT:", stdout),
+ "",
+ format_output_block("ACTUAL STDERR:", stderr),
+ ]
+ if actual_label:
+ lines.extend(["", "ACTUAL:", f" {actual_label}"])
+ return "\n".join(lines)
+
+
+def validate_output_expectations(
+ case_def: dict[str, Any],
+ stdout: str,
+ stderr: str,
+) -> tuple[bool, list[str]]:
+ conditions: list[str] = []
+ passed = True
+
+ expected_exit_code = case_def.get("expect_exit_code")
+ expect_exit_nonzero = bool(case_def.get("expect_exit_nonzero", False))
+
+ if expected_exit_code is not None and not isinstance(expected_exit_code, int):
+ raise ConfigError("'expect_exit_code' must be an integer")
+
+ actual_exit_code = case_def.get("_actual_exit_code")
+
+ if actual_exit_code is not None:
+ if expected_exit_code is not None:
+ if actual_exit_code != expected_exit_code:
+ passed = False
+ conditions.append(
+ "\n".join(
+ [
+ "PHASE: expectation_check",
+ "CHECK TYPE: expect_exit_code",
+ "",
+ "EXPECTED:",
+ f" exit code = {expected_exit_code}",
+ "",
+ "ACTUAL:",
+ f" exit code = {actual_exit_code}",
+ ]
+ )
+ )
+ elif expect_exit_nonzero and actual_exit_code == 0:
+ passed = False
+ conditions.append(
+ "\n".join(
+ [
+ "PHASE: expectation_check",
+ "CHECK TYPE: expect_exit_nonzero",
+ "",
+ "EXPECTED:",
+ " non-zero exit code",
+ "",
+ "ACTUAL:",
+ f" exit code = {actual_exit_code}",
+ ]
+ )
+ )
+
+ expect_output = case_def.get("expect_output")
+ if expect_output is not None:
+ candidates = (
+ [expect_output]
+ if isinstance(expect_output, str)
+ else ensure_list_of_strings(expect_output, "expect_output")
+ )
+ merged = stdout + "\n" + stderr
+ for candidate in candidates:
+ if candidate not in merged:
+ passed = False
+ conditions.append(
+ format_expectation_failure(
+ check_type="expect_output",
+ expected=candidate,
+ stdout=stdout,
+ stderr=stderr,
+ )
+ )
+
+ stdout_contains = case_def.get("expect_stdout_contains")
+ if stdout_contains is not None:
+ if not isinstance(stdout_contains, str):
+ raise ConfigError("'expect_stdout_contains' must be a string")
+ if stdout_contains not in stdout:
+ passed = False
+ conditions.append(
+ format_expectation_failure(
+ check_type="expect_stdout_contains",
+ expected=stdout_contains,
+ stdout=stdout,
+ stderr=stderr,
+ )
+ )
+
+ stderr_contains = case_def.get("expect_stderr_contains")
+ if stderr_contains is not None:
+ if not isinstance(stderr_contains, str):
+ raise ConfigError("'expect_stderr_contains' must be a string")
+ if stderr_contains not in stderr:
+ passed = False
+ conditions.append(
+ format_expectation_failure(
+ check_type="expect_stderr_contains",
+ expected=stderr_contains,
+ stdout=stdout,
+ stderr=stderr,
+ )
+ )
+
+ either_contains = case_def.get("expect_stdout_or_stderr_contains")
+ if either_contains is not None:
+ candidates = (
+ [either_contains]
+ if isinstance(either_contains, str)
+ else ensure_list_of_strings(
+ either_contains,
+ "expect_stdout_or_stderr_contains",
+ )
+ )
+ for candidate in candidates:
+ if candidate not in stdout and candidate not in stderr:
+ passed = False
+ conditions.append(
+ format_expectation_failure(
+ check_type="expect_stdout_or_stderr_contains",
+ expected=candidate,
+ stdout=stdout,
+ stderr=stderr,
+ )
+ )
+
+ expect_stdout_or_stderr_regex = case_def.get("expect_stdout_or_stderr_regex")
+ if expect_stdout_or_stderr_regex is not None:
+ candidates = (
+ [expect_stdout_or_stderr_regex]
+ if isinstance(expect_stdout_or_stderr_regex, str)
+ else ensure_list_of_strings(
+ expect_stdout_or_stderr_regex,
+ "expect_stdout_or_stderr_regex",
+ )
+ )
+ merged = stdout + "\n" + stderr
+ for pattern in candidates:
+ if re.search(pattern, merged, flags=re.MULTILINE) is None:
+ passed = False
+ conditions.append(
+ format_expectation_failure(
+ check_type="expect_stdout_or_stderr_regex",
+ expected=pattern,
+ stdout=stdout,
+ stderr=stderr,
+ )
+ )
+
+ expect_exit_code_in = case_def.get("expect_exit_code_in")
+ if expect_exit_code_in is not None:
+ if not isinstance(expect_exit_code_in, list) or not all(
+ isinstance(item, int) for item in expect_exit_code_in
+ ):
+ raise ConfigError("'expect_exit_code_in' must be a list of integers")
+ if actual_exit_code not in expect_exit_code_in:
+ passed = False
+ conditions.append(
+ "\n".join(
+ [
+ "PHASE: expectation_check",
+ "CHECK TYPE: expect_exit_code_in",
+ "",
+ "EXPECTED:",
+ f" exit code in {expect_exit_code_in}",
+ "",
+ "ACTUAL:",
+ f" exit code = {actual_exit_code}",
+ ]
+ )
+ )
+
+ return passed, conditions
+
+
+try: # Support package imports and direct harness module loading.
+ from .runner_check_execution import (
+ CHECK_HANDLERS,
+ apply_skip_controls,
+ build_cli_command,
+ build_cli_env,
+ build_command_from_spec,
+ build_runtime_env,
+ check_cli,
+ check_command_exit_code,
+ check_command_output_contains,
+ check_command_output_regex,
+ check_command_success,
+ check_file_exists,
+ check_function_exists,
+ check_function_exists_any,
+ check_main_guard,
+ check_module_cli,
+ check_module_main_with_env,
+ check_path_exists,
+ check_path_not_exists,
+ check_py_compile,
+ check_py_function,
+ check_source_contains,
+ check_source_contains_all,
+ check_source_contains_any,
+ execute_command_spec,
+ normalize_probe_spec,
+ run_command_assertion,
+ run_single_check,
+ )
+except ImportError: # pragma: no cover - exercised by flat-module harness imports.
+ from runner_check_execution import (
+ CHECK_HANDLERS,
+ apply_skip_controls,
+ build_cli_command,
+ build_cli_env,
+ build_command_from_spec,
+ build_runtime_env,
+ check_cli,
+ check_command_exit_code,
+ check_command_output_contains,
+ check_command_output_regex,
+ check_command_success,
+ check_file_exists,
+ check_function_exists,
+ check_function_exists_any,
+ check_main_guard,
+ check_module_cli,
+ check_module_main_with_env,
+ check_path_exists,
+ check_path_not_exists,
+ check_py_compile,
+ check_py_function,
+ check_source_contains,
+ check_source_contains_all,
+ check_source_contains_any,
+ execute_command_spec,
+ normalize_probe_spec,
+ run_command_assertion,
+ run_single_check,
+ )
diff --git a/common/acs_test_framework_runner/runner_reporting.py b/common/acs_test_framework_runner/runner_reporting.py
new file mode 100644
index 00000000..14fa1f29
--- /dev/null
+++ b/common/acs_test_framework_runner/runner_reporting.py
@@ -0,0 +1,360 @@
+from __future__ import annotations
+
+import shutil
+import xml.etree.ElementTree as xml_et
+from datetime import datetime
+from pathlib import Path
+from threading import Lock
+
+try: # Support package imports and direct harness module loading.
+ from .runner_checks import (
+ TestMeta,
+ TestOutcome,
+ create_outcome,
+ detect_project_root,
+ sanitize_name,
+ sanitize_xml_text,
+ )
+except ImportError: # pragma: no cover - exercised by flat-module harness imports.
+ from runner_checks import (
+ TestMeta,
+ TestOutcome,
+ create_outcome,
+ detect_project_root,
+ sanitize_name,
+ sanitize_xml_text,
+ )
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+PROJECT_ROOT = detect_project_root(SCRIPT_DIR)
+REPORTS_DIR = PROJECT_ROOT / "common" / "reports"
+PLACEHOLDER_XML = REPORTS_DIR / "pytest-placeholder.xml"
+LOG_SEPARATOR = "=" * 100
+LOG_WRITE_LOCK = Lock()
+
+
+def append_run_header(
+ file_work_dir: Path,
+ suite_name: str,
+ yaml_file: Path,
+ targets: list[str],
+) -> None:
+ file_work_dir.mkdir(parents=True, exist_ok=True)
+ log_path = file_work_dir / "combined.log"
+
+ try:
+ yaml_display = yaml_file.relative_to(PROJECT_ROOT).as_posix()
+ except ValueError:
+ yaml_display = str(yaml_file)
+
+ lines = [
+ "=" * 100,
+ f"RUN START: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
+ f"YAML: {yaml_display}",
+ f"SUITE: {suite_name}",
+ f"TARGETS: {', '.join(targets)}",
+ "=" * 100,
+ "",
+ ]
+
+ with LOG_WRITE_LOCK:
+ with log_path.open("a", encoding="utf-8") as handle:
+ handle.write("\n".join(lines))
+ handle.write("\n")
+
+
+def build_report_path(group_name: str, yaml_file: Path) -> Path:
+ safe_group = sanitize_name(group_name)
+ safe_yaml = sanitize_name(yaml_file.stem)
+ if safe_group == safe_yaml:
+ return REPORTS_DIR / f"{safe_yaml}.xml"
+ return REPORTS_DIR / f"{safe_group}__{safe_yaml}.xml"
+
+
+def create_placeholder_xml(reason: str) -> None:
+ REPORTS_DIR.mkdir(parents=True, exist_ok=True)
+
+ testsuite = xml_et.Element("testsuite")
+ testsuite.set("name", sanitize_xml_text("pytest"))
+ testsuite.set("tests", "0")
+ testsuite.set("failures", "0")
+ testsuite.set("errors", "0")
+ testsuite.set("skipped", "0")
+
+ properties = xml_et.SubElement(testsuite, "properties")
+
+ reason_prop = xml_et.SubElement(properties, "property")
+ reason_prop.set("name", "reason")
+ reason_prop.set("value", sanitize_xml_text(reason))
+
+ placeholder_prop = xml_et.SubElement(properties, "property")
+ placeholder_prop.set("name", "placeholder")
+ placeholder_prop.set("value", "true")
+
+ system_out = xml_et.SubElement(testsuite, "system-out")
+ system_out.text = sanitize_xml_text(reason)
+
+ xml_et.ElementTree(testsuite).write(
+ PLACEHOLDER_XML,
+ encoding="utf-8",
+ xml_declaration=True,
+ )
+
+
+def remove_placeholder_xml() -> None:
+ if PLACEHOLDER_XML.exists():
+ PLACEHOLDER_XML.unlink()
+
+
+def cleanup_old_pytest_xml_reports() -> None:
+ REPORTS_DIR.mkdir(parents=True, exist_ok=True)
+ for xml_file in REPORTS_DIR.glob("*.xml"):
+ if xml_file.name == "pylint-report.xml":
+ continue
+ xml_file.unlink(missing_ok=True)
+
+ work_root = REPORTS_DIR / "_work"
+ if work_root.exists():
+ shutil.rmtree(work_root, ignore_errors=True)
+
+
+def build_case_log_lines(
+ testcase_name: str,
+ status: str,
+ message: str,
+ details: str,
+ description: str | None = None,
+) -> list[str]:
+ normalized_description = ""
+ if description is not None:
+ normalized_description = " ".join(
+ line.strip()
+ for line in sanitize_xml_text(description).splitlines()
+ if line.strip()
+ )
+
+ lines = [
+ "=" * 80,
+ f"TEST CASE: {testcase_name}",
+ ]
+ if normalized_description:
+ lines.append(f"DESCRIPTION: {normalized_description}")
+ lines.extend([
+ f"STATUS: {status}",
+ f"MESSAGE: {message}",
+ ])
+
+ if details:
+ lines.extend(["", details.rstrip()])
+
+ return lines
+
+
+def append_combined_case_log(
+ file_work_dir: Path,
+ testcase_name: str,
+ status: str,
+ message: str,
+ details: str,
+ description: str | None = None,
+) -> None:
+ file_work_dir.mkdir(parents=True, exist_ok=True)
+ log_path = file_work_dir / "combined.log"
+ lines = build_case_log_lines(
+ testcase_name,
+ status,
+ message,
+ details,
+ description=description,
+ )
+
+ with LOG_WRITE_LOCK:
+ with log_path.open("a", encoding="utf-8") as handle:
+ handle.write("\n".join(lines).rstrip())
+ handle.write("\n\n")
+
+
+def write_case_log(
+ case_work_dir: Path,
+ testcase_name: str,
+ status: str,
+ message: str,
+ details: str,
+ description: str | None = None,
+) -> None:
+ case_work_dir.mkdir(parents=True, exist_ok=True)
+ log_path = case_work_dir / "case.log"
+ lines = build_case_log_lines(
+ testcase_name,
+ status,
+ message,
+ details,
+ description=description,
+ )
+
+ with LOG_WRITE_LOCK:
+ log_path.write_text(
+ "\n".join(lines).rstrip() + "\n",
+ encoding="utf-8",
+ )
+
+
+def write_junit_xml(
+ xml_report: Path,
+ suite_name: str,
+ yaml_file: Path,
+ outcomes: list[TestOutcome],
+) -> None:
+ tests = len(outcomes)
+ failures = sum(
+ 1
+ for item in outcomes
+ if not item.passed and not item.error and not item.skipped and not item.warning
+ )
+ errors = sum(1 for item in outcomes if item.error)
+ skipped = sum(1 for item in outcomes if item.skipped)
+ # Keep the XML summary property aligned with the console summary, where
+ # hardware-gated skips are surfaced as warnings rather than hard failures.
+ warnings = sum(1 for item in outcomes if item.skipped or item.warning)
+ passed = sum(
+ 1 for item in outcomes if item.passed and not item.skipped and not item.warning
+ )
+
+ testsuite = xml_et.Element("testsuite")
+ testsuite.set("name", sanitize_xml_text(suite_name))
+ testsuite.set("tests", str(tests))
+ testsuite.set("failures", str(failures))
+ testsuite.set("errors", str(errors))
+ testsuite.set("skipped", str(skipped))
+
+ properties = xml_et.SubElement(testsuite, "properties")
+
+ yaml_prop = xml_et.SubElement(properties, "property")
+ yaml_prop.set("name", "yaml_file")
+ yaml_prop.set(
+ "value",
+ sanitize_xml_text(yaml_file.relative_to(PROJECT_ROOT).as_posix()),
+ )
+
+ passed_prop = xml_et.SubElement(properties, "property")
+ passed_prop.set("name", "passed")
+ passed_prop.set("value", sanitize_xml_text(str(passed)))
+
+ warnings_prop = xml_et.SubElement(properties, "property")
+ warnings_prop.set("name", "warnings")
+ warnings_prop.set("value", sanitize_xml_text(str(warnings)))
+
+ for outcome in outcomes:
+ testcase = xml_et.SubElement(testsuite, "testcase")
+ testcase.set(
+ "classname",
+ sanitize_xml_text(sanitize_name(outcome.file_path or suite_name)),
+ )
+ testcase.set("name", sanitize_xml_text(outcome.testcase_name))
+ testcase.set("file", sanitize_xml_text(outcome.file_path))
+
+ if outcome.skipped:
+ skipped_node = xml_et.SubElement(testcase, "skipped")
+ skipped_node.set("message", sanitize_xml_text(outcome.message))
+ skipped_node.text = sanitize_xml_text(outcome.details)
+ elif not outcome.passed and outcome.error:
+ error_node = xml_et.SubElement(testcase, "error")
+ error_node.set("message", sanitize_xml_text(outcome.message))
+ error_node.text = sanitize_xml_text(outcome.details)
+ elif not outcome.passed:
+ failure_node = xml_et.SubElement(testcase, "failure")
+ failure_node.set("message", sanitize_xml_text(outcome.message))
+ failure_node.text = sanitize_xml_text(outcome.details)
+
+ system_out = xml_et.SubElement(testcase, "system-out")
+ body = [
+ f"file={outcome.file_path}",
+ f"suite={outcome.meta.suite_name}",
+ f"phase={outcome.meta.phase}",
+ f"type={outcome.meta.test_type}",
+ f"warning={outcome.warning}",
+ f"message={outcome.message}",
+ ]
+ if outcome.details:
+ body.extend(["details:", outcome.details])
+
+ system_out.text = sanitize_xml_text("\n".join(body))
+
+ REPORTS_DIR.mkdir(parents=True, exist_ok=True)
+ xml_et.ElementTree(testsuite).write(
+ xml_report,
+ encoding="utf-8",
+ xml_declaration=True,
+ )
+
+
+def print_group_summary(
+ suite_name: str,
+ outcomes: list[TestOutcome],
+ xml_report: Path,
+) -> None:
+ total = len(outcomes)
+ passed = sum(
+ 1 for item in outcomes if item.passed and not item.skipped and not item.warning
+ )
+ failures = sum(
+ 1
+ for item in outcomes
+ if not item.passed and not item.error and not item.skipped and not item.warning
+ )
+ errors = sum(1 for item in outcomes if item.error)
+ warnings = sum(1 for item in outcomes if item.skipped or item.warning)
+
+ print(f"\n[INFO] Finished group : {suite_name}")
+ print(
+ f"[INFO] XML report : "
+ f"{xml_report.relative_to(PROJECT_ROOT).as_posix()}"
+ )
+ print(f"Total : {total}")
+ print(f"Passed : {passed}")
+ print(f"Failed : {failures}")
+ print(f"Errors : {errors}")
+ print(f"Warnings: {warnings}")
+
+ failed_items = [
+ item for item in outcomes
+ if not item.passed and not item.skipped and not item.warning
+ ]
+ if failed_items:
+ print("\n[FAILURES]")
+ for item in failed_items:
+ kind = "ERROR" if item.error else "FAIL"
+ print(
+ f" - [{kind}] "
+ f"{item.file_path} :: {item.testcase_name} :: {item.message}"
+ )
+
+ warning_items = [item for item in outcomes if item.skipped or item.warning]
+ if warning_items:
+ print("\n[WARNINGS]")
+ for item in warning_items:
+ label = "[WARNING]" if item.warning else "[HW WARNING]"
+ print(
+ f" - {label} "
+ f"{item.file_path} :: {item.testcase_name} :: {item.message}"
+ )
+
+
+def build_config_error_outcome(
+ yaml_file: Path,
+ message: str,
+ details: str,
+) -> TestOutcome:
+ return create_outcome(
+ testcase_name="config::load_yaml",
+ file_path=yaml_file.relative_to(PROJECT_ROOT).as_posix(),
+ passed=False,
+ message=sanitize_xml_text(message),
+ meta=TestMeta(
+ suite_name="config",
+ phase="config",
+ test_type="config",
+ ),
+ details=sanitize_xml_text(details),
+ error=True,
+ )
diff --git a/common/acs_test_framework_runner/test_pytest_runner_selection.py b/common/acs_test_framework_runner/test_pytest_runner_selection.py
new file mode 100644
index 00000000..d7697109
--- /dev/null
+++ b/common/acs_test_framework_runner/test_pytest_runner_selection.py
@@ -0,0 +1,208 @@
+from __future__ import annotations
+
+import importlib.util
+import shutil
+import sys
+from pathlib import Path
+
+
+HARNESS_DIR = Path(__file__).resolve().parent
+if str(HARNESS_DIR) not in sys.path:
+ sys.path.insert(0, str(HARNESS_DIR))
+
+
+def load_harness_module(module_name: str, filename: str):
+ spec = importlib.util.spec_from_file_location(module_name, HARNESS_DIR / filename)
+ assert spec is not None
+ assert spec.loader is not None
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+pytest_runner = load_harness_module("yaml_harness_pytest_runner", "pytest_runner.py")
+report = load_harness_module("yaml_harness_report", "report.py")
+runner_reporting = load_harness_module(
+ "yaml_harness_runner_reporting",
+ "runner_reporting.py",
+)
+
+
+def test_harness_sources_changed_ignores_pycache() -> None:
+ changed_paths = {
+ pytest_runner.HARNESS_DIR / "__pycache__" / "pytest_runner.cpython-313.pyc",
+ }
+
+ assert pytest_runner.harness_sources_changed(changed_paths) is False
+
+
+def test_select_yaml_runs_runs_all_groups_when_harness_changes(monkeypatch) -> None:
+ yaml_files = [
+ pytest_runner.PROJECT_ROOT / "common" / "acs_test_framework_manifests" / "group_one.yaml",
+ pytest_runner.PROJECT_ROOT / "common" / "acs_test_framework_manifests" / "group_two.yaml",
+ ]
+ changed_paths = {pytest_runner.HARNESS_DIR / "mock_loader.py"}
+
+ monkeypatch.setattr(
+ pytest_runner,
+ "get_recently_changed_paths",
+ lambda: changed_paths,
+ )
+ monkeypatch.setattr(
+ pytest_runner,
+ "yaml_targets_changed_files",
+ lambda yaml_file, _changed: ({f"{yaml_file.stem}.py"}, None),
+ )
+ monkeypatch.setattr(
+ pytest_runner,
+ "get_yaml_target_entries",
+ lambda yaml_file: ({f"{yaml_file.stem}.py", f"{yaml_file.stem}_extra.py"}, None),
+ )
+
+ selected_runs, warnings = pytest_runner.select_yaml_runs(yaml_files)
+
+ assert warnings == []
+ assert selected_runs == [
+ (yaml_files[0], {"group_one.py", "group_one_extra.py"}),
+ (yaml_files[1], {"group_two.py", "group_two_extra.py"}),
+ ]
+
+
+def test_select_yaml_runs_preserves_target_only_selection(monkeypatch) -> None:
+ yaml_files = [pytest_runner.PROJECT_ROOT / "common" / "acs_test_framework_manifests" / "group_one.yaml"]
+
+ monkeypatch.setattr(
+ pytest_runner,
+ "get_recently_changed_paths",
+ lambda: {pytest_runner.PROJECT_ROOT / "common" / "linux_scripts" / "target.py"},
+ )
+ monkeypatch.setattr(
+ pytest_runner,
+ "yaml_targets_changed_files",
+ lambda _yaml_file, _changed: ({"target.py"}, None),
+ )
+ get_yaml_target_entries_called = False
+
+ def fake_get_yaml_target_entries(_yaml_file: Path) -> tuple[set[str], None]:
+ nonlocal get_yaml_target_entries_called
+ get_yaml_target_entries_called = True
+ return {"target.py", "unrelated.py"}, None
+
+ monkeypatch.setattr(
+ pytest_runner,
+ "get_yaml_target_entries",
+ fake_get_yaml_target_entries,
+ )
+
+ selected_runs, warnings = pytest_runner.select_yaml_runs(yaml_files)
+
+ assert warnings == []
+ assert get_yaml_target_entries_called is False
+ assert selected_runs == [(yaml_files[0], {"target.py"})]
+
+
+def test_report_changed_yaml_adds_python_targets_for_static_checks(monkeypatch) -> None:
+ yaml_path = report.PROJECT_ROOT / "common" / "acs_test_framework_manifests" / "group_one.yaml"
+ yaml_target = report.PROJECT_ROOT / "common" / "linux_scripts" / "target.py"
+ direct_python_change = report.PROJECT_ROOT / "common" / "linux_scripts" / "direct.py"
+
+ monkeypatch.setattr(
+ report,
+ "get_recently_changed_paths",
+ lambda: [yaml_path, direct_python_change],
+ )
+ monkeypatch.setattr(
+ report,
+ "is_yaml_suite_file",
+ lambda changed_path: changed_path == yaml_path,
+ )
+ monkeypatch.setattr(
+ report,
+ "get_python_targets_from_yaml_file",
+ lambda changed_yaml: {yaml_target} if changed_yaml == yaml_path else set(),
+ )
+
+ assert report.get_recently_changed_python_files() == [
+ direct_python_change,
+ yaml_target,
+ ]
+
+
+def test_report_yaml_target_filter_keeps_only_existing_python_files(monkeypatch) -> None:
+ yaml_path = report.PROJECT_ROOT / "common" / "acs_test_framework_manifests" / "group_one.yaml"
+ py_target = report.PROJECT_ROOT / "common" / "linux_scripts" / "target.py"
+ sh_target = report.PROJECT_ROOT / "common" / "linux_scripts" / "target.sh"
+
+ monkeypatch.setattr(report, "load_yaml_config", lambda _yaml_file: {"suites": []})
+ monkeypatch.setattr(
+ report,
+ "normalize_suites",
+ lambda _config: [{"files": ["common/linux_scripts/target.py", "common/linux_scripts/target.sh"]}],
+ )
+
+ def fake_exists(self) -> bool:
+ return self in {yaml_path, py_target, sh_target}
+
+ def fake_is_file(self) -> bool:
+ return self in {yaml_path, py_target, sh_target}
+
+ monkeypatch.setattr(Path, "exists", fake_exists)
+ monkeypatch.setattr(Path, "is_file", fake_is_file)
+
+ assert report.get_python_targets_from_yaml_file(yaml_path) == {py_target}
+
+
+def test_collect_pytest_case_logs_reads_persisted_case_description(
+ monkeypatch,
+) -> None:
+ temp_path = HARNESS_DIR / "_case_log_test_reports"
+ shutil.rmtree(temp_path, ignore_errors=True)
+ try:
+ runner_reporting.append_combined_case_log(
+ file_work_dir=temp_path / "_work" / "suite" / "target",
+ testcase_name="suite::target.py::case_with_description",
+ status="PASS",
+ message="case passed",
+ details="",
+ description="First line of description\nSecond line of description",
+ )
+ monkeypatch.setattr(report, "REPORTS_DIR", temp_path)
+
+ assert report.collect_pytest_case_logs() == [
+ {
+ "name": "suite::target.py::case_with_description",
+ "description": "First line of description Second line of description",
+ "status": "passed",
+ "details": "case passed",
+ }
+ ]
+ finally:
+ shutil.rmtree(temp_path, ignore_errors=True)
+
+
+def test_collect_pytest_case_logs_skips_missing_case_description(
+ monkeypatch,
+) -> None:
+ temp_path = HARNESS_DIR / "_case_log_test_reports"
+ shutil.rmtree(temp_path, ignore_errors=True)
+ try:
+ runner_reporting.append_combined_case_log(
+ file_work_dir=temp_path / "_work" / "suite" / "target",
+ testcase_name="suite::target.py::case_without_description",
+ status="PASS",
+ message="case passed",
+ details="",
+ )
+ monkeypatch.setattr(report, "REPORTS_DIR", temp_path)
+
+ assert report.collect_pytest_case_logs() == [
+ {
+ "name": "suite::target.py::case_without_description",
+ "description": "",
+ "status": "passed",
+ "details": "case passed",
+ }
+ ]
+ finally:
+ shutil.rmtree(temp_path, ignore_errors=True)
diff --git a/docs/yaml_test_framework_readme.md b/docs/yaml_test_framework_readme.md
new file mode 100644
index 00000000..04aee825
--- /dev/null
+++ b/docs/yaml_test_framework_readme.md
@@ -0,0 +1,522 @@
+# YAML Test Framework README
+
+## Purpose
+
+This framework lets us test repository scripts by declaring suites and cases in YAML instead of writing a separate Python test module for every script.
+
+It is built around:
+
+- `common/yaml_test_harness/report.py` as the main user-facing entry point
+- `common/test_yaml/` for YAML suite definitions
+- `common/yaml_test_harness/pytest_runner.py` as the underlying YAML execution engine
+- `common/yaml_test_harness/runner_checks.py` for schema validation, execution modes, and expectations
+- `common/yaml_test_harness/mock_loader.py` for scenario expansion and mock construction
+- `common/yaml_test_harness/case_data_builders.py` for reusable case workspace builders and template expansion
+- `common/yaml_test_harness/mock_helpers.py` for reusable mock routers and verification helpers
+- `common/reports/` for JUnit XML, combined logs, and per-case work directories
+
+For normal use, run `python3 common/yaml_test_harness/report.py` from the repository root. Internally it calls `common/yaml_test_harness/pytest_runner.py`, which is a standalone Python runner rather than a standard pytest parametrization file.
+
+## Prerequisites
+
+Required:
+
+- Python 3
+- `PyYAML`
+
+Install the required Python package with:
+
+```bash
+python3 -m pip install PyYAML
+```
+
+Optional but recommended when using `report.py`:
+
+- `pylint`
+- `mypy`
+
+Install them with:
+
+```bash
+python3 -m pip install pylint mypy
+```
+
+Notes:
+
+- The YAML framework itself only needs `PyYAML` in addition to the Python standard library.
+- If `pylint` or `mypy` are not installed, `report.py` still runs the YAML test flow and prints warnings instead of failing only because those tools are missing.
+- Pylint uses the repo-root `.pylintrc` file automatically because `report.py` runs pylint from the repository root. The checked-in config file is `.pylintrc`, not `.pylint.rc`.
+- Git is recommended for the default impacted-file mode. If Git metadata is unavailable, use focused runs such as `python3 common/yaml_test_harness/report.py common/log_parser/merge_jsons.py`.
+- Hardware-gated suites may also rely on platform commands such as `ip`, `ethtool`, `/bin/sh`, `awk`, `grep`, and `readlink`. Those are not needed for normal scenario-backed runs on a single target unless that target's cases explicitly probe them.
+
+## How It Works
+
+At a high level, each run goes through the following stages:
+
+1. Discover every `.yaml` and `.yml` file under `common/test_yaml/`.
+2. Select which YAML suites to run.
+ - Default mode runs only suites whose `files:` entries match changed files from Git.
+ - Manual mode runs suites for one explicit target passed to `common/yaml_test_harness/report.py` as its positional `target` argument.
+3. Normalize and validate the YAML schema.
+ - Top-level `suites:` is required.
+ - Each suite must define `name`, `files`, and `cases`.
+ - Optional suite-level `defaults` are merged into every case.
+4. Build a runtime case.
+ - If a case contains `scenario:`, the scenario builder generates `args`, `text_files`, `bin_files`, `mocks`, and `patch_constants`.
+ - Case-level values override generated values where appropriate.
+ - Runtime tokens such as `{dir}`, `{file}`, and `{filename}` are expanded.
+5. Materialize a per-case workspace under `common/reports/_work/...`.
+ - Generated files, helper scripts, and directory structures are created here.
+6. Execute the case using the requested case type.
+7. Validate exit code, stdout/stderr content, timeout behavior, and optional post-checks.
+8. Write XML and text logs into `common/reports/`.
+
+The work directory is isolated per case, which keeps generated fixtures and side effects from leaking across cases.
+
+## Running The Framework
+
+Use the repo-root wrapper path:
+
+```bash
+python3 common/yaml_test_harness/report.py
+```
+
+Useful variants:
+
+```bash
+python3 common/yaml_test_harness/report.py common/log_parser/merge_jsons.py
+python3 common/yaml_test_harness/report.py common/linux_scripts/verify_tpm_measurements.py
+```
+
+Notes:
+
+- `common/yaml_test_harness/report.py` is the main command to share with other users of this framework.
+- The positional `target` argument is the simplest way to run focused coverage for one script.
+- Example: `python3 common/yaml_test_harness/report.py common/log_parser/merge_jsons.py`
+- `report.py` runs the YAML-based test flow and also prints the paired `pylint` and `mypy` summaries for the same target set.
+- Internally, `report.py` forwards the YAML test portion to `common/yaml_test_harness/pytest_runner.py`.
+- Default mode depends on Git change detection. If the repo is not a Git worktree, or there are no detected changes, nothing will be selected.
+- When `report.py` auto-collects changed Python files for `pylint` and `mypy`, it skips generated artifacts under `common/reports/` and `reports/`, including directories such as `_work`, `_runner_work`, and `tpm_check_*`.
+- XML reports are written to `common/reports/*.xml`.
+- Case logs and generated fixtures go under `common/reports/_work/`.
+
+## Local Git Hooks
+
+The repo includes a local pre-commit hook at `.githooks/pre-commit`.
+
+What it does:
+
+- runs `common/yaml_test_harness/report.py` from the repository root
+- uses the default impacted-file selection logic
+- writes a hook-specific console capture to `common/reports/precommit_report.log`
+- blocks the commit when the report flow exits non-zero
+
+Configure the hook path for this clone:
+
+```bash
+git config core.hooksPath .githooks
+git config --get core.hooksPath
+```
+
+If needed, make the hook executable:
+
+```bash
+chmod +x .githooks/pre-commit
+```
+
+Run it manually without creating a commit:
+
+```bash
+bash .githooks/pre-commit
+```
+
+Notes:
+
+- Local hooks only work in a real Git clone/worktree. They do not apply to an extracted ZIP snapshot.
+- The hook script is Bash-based and intended to be run from a normal Linux shell.
+- To disable the custom hook path for a clone, run `git config --unset core.hooksPath`.
+
+## Report Structure
+
+Main output directory:
+
+```text
+common/reports/
+ .xml
+ precommit_report.log
+ pylint-report.xml
+ pylint.log
+ mypy-report.xml
+ mypy.log
+ pytest.log
+ pytest-placeholder.xml # only when no YAML groups are selected
+ _work/
+ /
+ /
+ combined.log
+ _/
+ case.log
+ ...generated files for that case...
+ _runner_work/
+ runner_env_/ # internal temp dirs for some in-process modes
+```
+
+What the main files mean:
+
+- `common/reports/.xml`: JUnit-style XML for one YAML suite group. Example: `merge_jsons.xml`.
+- `common/reports/precommit_report.log`: stdout/stderr captured by the local pre-commit hook.
+- `common/reports/pylint-report.xml`: XML summary produced by the `pylint` part of `report.py`.
+- `common/reports/mypy-report.xml`: XML summary produced by the `mypy` part of `report.py`.
+- `common/reports/pytest.log`: captured stdout/stderr from the YAML runner.
+- `common/reports/pylint.log`: captured stdout/stderr from pylint, including the scored and parseable outputs used by the wrapper.
+- `common/reports/mypy.log`: captured stdout/stderr from mypy.
+- `common/reports/_work///combined.log`: one running log for all cases that executed against that target file in that suite.
+- `common/reports/_work////case.log`: one detailed log for a single case, including command, timeout, exit code, stdout/stderr, and post-check details when applicable.
+- `common/reports/_runner_work/runner_env_/`: internal scratch directories used by runner-managed execution modes such as `module_main_with_env`.
+
+## Logs Info
+
+`pytest.log`, `pylint.log`, and `mypy.log` are history logs maintained by `report.py`:
+
+- Each run is appended with a timestamped separator.
+- The wrapper keeps the most recent 50 log entries.
+- These files are the first place to look if the console output was truncated or if you need the raw stdout/stderr from a previous run.
+- Generated report artifacts under `common/reports/` and `reports/` are not treated as source targets for the wrapper's default `pylint` and `mypy` changed-file mode.
+
+`precommit_report.log` is the hook-specific run log:
+
+- It is overwritten by each hook run rather than kept as a rolling history log.
+- Use it when a commit is blocked and you want the exact hook console output in one file.
+
+`combined.log` is useful for target-level triage:
+
+- It starts with a run header showing the YAML file, suite name, and target path.
+- It then lists every executed test case with `TEST CASE`, `STATUS`, and `MESSAGE`.
+- Use it when you want one compact chronological view of what happened for a target file.
+
+`case.log` is useful for root-cause analysis:
+
+- Runtime cases usually include command line, shell mode, timeout, exit code, stdout, stderr, and post-check output.
+- Mock verification failures and configuration errors are also written here.
+- Generated per-case files live beside the case log in the same case work directory.
+
+XML report details:
+
+- Each testcase in the JUnit XML includes `file`, `suite`, `phase`, `type`, `warning`, `message`, and optional `details` in `system-out`.
+- The XML `` section records at least the source YAML file and summary counts such as passed and warnings.
+
+## YAML Structure
+
+Minimal shape:
+
+```yaml
+suites:
+ - name: example_suite
+ files:
+ - common/linux_scripts/example.py
+ defaults:
+ timeout_sec: 20
+ cases:
+ - name: file_exists
+ type: file_exists
+
+ - name: cli_happy_path
+ type: cli
+ args:
+ - "{dir}/input.yaml"
+ text_files:
+ input.yaml: |
+ enabled: true
+ expect_exit_code: 0
+ expect_stdout_or_stderr_contains:
+ - "PASS"
+```
+
+Suite fields:
+
+| Field | Meaning |
+| --- | --- |
+| `name` | Logical suite name used in reports and work directories |
+| `files` | One or more target files covered by the suite |
+| `command` | Optional suite-wide command override |
+| `defaults` | Optional case defaults merged into each case |
+| `cases` | List of case definitions |
+
+Common case fields:
+
+| Field | Meaning |
+| --- | --- |
+| `name` | Required case name |
+| `type` | Execution/check type |
+| `description` | Optional explanation for humans |
+| `args` / `kwargs` | Arguments for CLI or function-style cases |
+| `command` | Override the default interpreter command |
+| `env` | Environment variables for the case |
+| `stdin` | Text passed to stdin |
+| `timeout_sec` | Command timeout |
+| `scripts` | Helper scripts written into the case workspace |
+| `text_files` / `bin_files` | Generated fixture files written into the workspace |
+| `dir_structure` | Directories to create before execution |
+| `mocks` | Explicit patch definitions |
+| `patch_constants` | Attribute overrides patched into imported modules |
+| `post_checks` | File existence/content checks after execution |
+| `warn_only` | Treat a functional failure as a warning instead of a failing case |
+| `skip_unless_env` | Skip unless required env vars are present or match |
+| `skip_unless_paths_exist` | Skip when required hardware-visible paths do not exist |
+| `skip_unless_commands_succeed` | Skip when probe commands fail |
+| `requires_destructive` | Skip unless `RUN_DESTRUCTIVE_HW_TESTS=1` is set |
+
+Output expectation fields commonly used by runtime cases:
+
+- `expect_exit_code`
+- `expect_exit_code_in`
+- `expect_exit_nonzero`
+- `expect_timeout`
+- `expect_output`
+- `expect_stdout_contains`
+- `expect_stderr_contains`
+- `expect_stdout_or_stderr_contains`
+- `expect_stdout_or_stderr_regex`
+
+Supported `post_checks`:
+
+- `exists`
+- `not_exists`
+- `file_contains`
+- `file_not_contains`
+- `file_not_empty`
+- `regex`
+- `ordered_contains`
+
+Runtime tokens:
+
+- `{dir}`: per-case work directory
+- `{file}`: target file path being tested
+- `{filename}`: basename of the target file
+
+Case-type-specific fields:
+
+- `py_function`: requires `function`; can also use `expect_exception` and `expect_return_contains`
+- `path_exists` / `path_not_exists`: require `path`
+- `command_success`, `command_exit_code`, `command_output_contains`, and `command_output_regex`: use `command`, `args`, optional `shell`, optional `cwd`, optional `env`, and optional `timeout_sec`
+- `command_exit_code`: requires `expect_exit_code`
+- `command_output_contains`: requires `expect_text`
+- `command_output_regex`: requires `expect_pattern`
+
+## Supported Case Types
+
+Static and source-level checks:
+
+- `file_exists`
+- `py_compile`
+- `source_contains`
+- `source_contains_any`
+- `source_contains_all`
+- `function_exists`
+- `function_exists_any`
+- `main_guard`
+- `path_exists`
+- `path_not_exists`
+
+Runtime and execution checks:
+
+- `cli`
+- `module_cli`
+- `py_function`
+- `module_main_with_env`
+
+Command/probe checks:
+
+- `command_success`
+- `command_exit_code`
+- `command_output_contains`
+- `command_output_regex`
+
+When to use which runtime type:
+
+- `cli`: runs the target in a real subprocess. Use this when you want real process behavior and do not need in-process patching.
+- `module_cli`: executes the script as `__main__` in-process. Use this when you need deterministic mocks, generated patch constants, or scenario-generated runtime behavior.
+- `module_main_with_env`: imports the module, patches attributes, changes cwd/env, and calls `main()` directly.
+- `py_function`: imports the module and calls one named function.
+
+## Choosing The Right Test Style
+
+Start with the least powerful style that can still prove the requirement.
+
+### 1. Static checks
+
+Use static checks when you only need to prove source or presence facts:
+
+- the file exists and parses
+- a function or main guard is present
+- required source text or patterns exist
+- a required path exists or does not exist on the current machine
+
+Choose this style when:
+
+- you want the fastest and least brittle coverage
+- the behavior under test is structural rather than runtime behavior
+- running the script would add noise without adding confidence
+
+Do not use static checks when you actually need to prove exit codes, stdout/stderr, generated files, side effects, or control flow.
+
+### 2. Subprocess CLI execution
+
+Use `type: cli` when the script should be exercised the way a user runs it:
+
+- real interpreter startup
+- real `sys.argv`, stdin, stdout, and stderr flow
+- real process exit codes and timeout behavior
+- cases where shell mode, `command`, `cwd`, or environment handling matter
+
+Choose this style when:
+
+- process boundaries matter
+- you want the closest match to production CLI behavior
+- you do not need in-process mocks or patched constants
+
+Important limitation:
+
+- `cli` runs in a separate process, so in-process `mocks:` and `patch_constants:` do not affect the target script.
+- `cli` can still use generated files, args, env, and `post_checks`, but it is the wrong choice when the test depends on patched Python internals.
+
+### 3. In-Process Execution
+
+Use in-process execution when you need the runner to patch or control Python state inside the target:
+
+- `module_cli`: best when the script is still being tested as a CLI entry point, but you need deterministic `mocks:`, generated `patch_constants:`, or scenario-generated runtime setup
+- `module_main_with_env`: best when the module exposes a real `main()` and you want to call it directly under a controlled cwd and environment
+- `py_function`: best for narrow unit-like checks of one function's return value or raised exception
+
+Choose this style when:
+
+- you need mocked `subprocess`, `open`, helper functions, or module attributes
+- you want scenario builders to inject runtime setup into the same interpreter
+- you need tighter control than a real subprocess allows
+
+Tradeoff:
+
+- in-process modes are less faithful to a true separate process than `cli`
+- `module_main_with_env` skips the normal `__main__` execution path and calls `main()` directly, so use it only when that is the behavior you actually want to verify
+
+### 4. Scenario-Backed Cases
+
+A scenario-backed case is not a separate `type:`. It is a runtime case that also includes `scenario:` so a builder in `mock_loader.py` can generate `args`, files, mocks, and patch constants for you.
+
+Use a scenario-backed case when:
+
+- the same domain setup is repeated across many cases
+- the script is hardware-facing or environment-heavy
+- hand-writing `text_files:`, `bin_files:`, `mocks:`, and `patch_constants:` for every case would be noisy and error-prone
+
+Choose this style when:
+
+- an existing scenario kind already models the domain well
+- the important part of the test is the behavior, not the low-level mock wiring
+- you want consistent fixture generation across a whole suite
+
+Practical rule:
+
+- most scenario-backed cases should use `module_cli` or `module_main_with_env`, because scenario builders often generate `mocks:` and `patch_constants:` that only in-process execution can see
+- use `cli` with `scenario:` only when the scenario is effectively just generating files, args, or environment input and does not rely on in-process patching
+
+Short decision guide:
+
+1. If a structural check is enough, use a static case type.
+2. If real process behavior matters, use `cli`.
+3. If mocks or patched Python state matter, use an in-process type.
+4. If setup is repetitive or domain-specific, add `scenario:` on top of the appropriate runtime type.
+
+## Scenarios And Mocking
+
+Scenario builders reduce repetitive low-level setup. The currently registered scenario kinds are:
+
+| Scenario kind | What it auto-generates |
+| --- | --- |
+| `ethtool` | Network interface fixtures, tool availability, command routers, sysfs-style reads, connectivity outcomes |
+| `verify_tpm` | `pcr.yaml`, `event.yaml`, event-log fault injection mocks, expected argument wiring |
+| `capsule_vars` | efivarfs-style binary fixtures and capsule variable defaults |
+| `acs_info` | ACS info inputs and mocked platform data |
+| `merge_jsons` | input JSON trees, ACS info payloads, mode-specific runtime setup |
+| `blk_devices` | block-device discovery, partition layout, and subprocess behavior for the script main flow |
+| `blk_write_check` | direct write/readback/restore flows for block-device write checks |
+| `runtime_device_mapping` | generated DTS/memmap/log inputs and selected file-open patches |
+
+`blk_devices` is aligned with the current `common/linux_scripts/read_write_check_blk_devices.py` implementation. For full-flow cases, it is intended to drive the real `main()` path under an in-process runtime type, patch `{module}.input_with_timeout`, and mock the current command forms such as `lsblk -e 7 -d -n -o NAME,TYPE`, `lsblk -rn -o NAME,TYPE /dev/`, and `dd if=/dev/ of=/dev/null bs=1M count=1`.
+
+Example of a scenario-backed case:
+
+```yaml
+- name: cli_mock_open_raises_on_event_log
+ type: module_cli
+ scenario:
+ kind: verify_tpm
+ pcrs:
+ sha256: ["pcr0", "pcr1", "pcr2", "pcr3", "pcr4", "pcr5", "pcr6", "pcr7"]
+ event_pcrs:
+ sha256: ["pcr0", "pcr1", "pcr2", "pcr3", "pcr4", "pcr5", "pcr6", "pcr7"]
+ event_log_open_error: mocked read failure
+ expect_exit_nonzero: true
+ expect_stdout_or_stderr_contains:
+ - "mocked read failure"
+```
+
+Explicit `mocks:` are still supported for one-off cases. Mock targets can use `{module}` as a placeholder when the target module name is only known at runtime.
+
+Scenario-generated passthrough rules can be marked as required, so the test fails if the expected mocked path is never actually hit. That prevents silent mock misses.
+
+## Current Coverage
+
+The current YAML catalog covers both utility scripts and hardware-facing flows.
+
+Scenario-backed script coverage:
+
+| Suite group | Main focus |
+| --- | --- |
+| `ethtool_test` | interface discovery, virtual vs physical NIC filtering, tool presence, link state, IPv4/IPv6, gateway and internet probes |
+| `verify_tpm_measurements` | CLI validation, missing/invalid inputs, PCR/event matching, event-log read and parse failures |
+| `capsule_ondisk_reporting_vars_check` | efivarfs variable parsing, attributes, capsule reporting entries, warn-only documentation of current behavior |
+| `runtime_device_mapping_conflict_checker` | DTS and memmap inputs, conflict detection, parser failures, log-open failures, warn-only known behavior |
+| `read_write_check_blk_devices` | raw disks, MBR/GPT layouts, precious partitions, write/readback/restore flows, destructive gating |
+| `acs_info` | platform-info extraction and formatted output paths |
+| `merge_jsons` | merge modes, missing/invalid inputs, ACS info interactions, output generation |
+
+Generic and utility coverage:
+
+| Suite group | Main focus |
+| --- | --- |
+| `apply_waivers` | waiver application logic and CLI behavior |
+| `generate_acs_summary` | summary generation behavior |
+| `logs_to_json_common`, `sr_logs_to_json_specific`, `edk2_logs_to_json_specific` | log parsing and JSON conversion flows |
+| `json_to_html_common` plus suite-specific HTML suites | HTML report generation across FWTS, SCT, TPM, BSA, SBMR, SCMI, standalone, and related outputs |
+| `merge_summary` | summary merge behavior |
+| `parser_common` | shared parser helpers |
+| `extract_capsule_fw_version` | firmware version extraction cases |
+| `validate_sh` | shell script validation and environment checks |
+
+Hardware-gated coverage:
+
+- `ethtool_test_hardware`
+- `verify_tpm_measurements_hardware`
+- `capsule_ondisk_reporting_vars_check_hardware`
+- `runtime_device_mapping_conflict_checker_hardware`
+- `read_write_check_blk_devices_hardware`
+
+These hardware suites use `skip_unless_paths_exist` and `skip_unless_commands_succeed` so they are skipped cleanly on machines that do not expose the required platform state.
+
+## Adding A New Case
+
+Recommended workflow:
+
+1. Pick the existing YAML suite that already targets the script you want to cover.
+2. Use the simplest case type that expresses the requirement.
+ - Static check if only source structure matters.
+ - `cli` if subprocess behavior matters and mocks are not needed.
+ - `module_cli` if you need deterministic mocking or scenario-generated setup.
+3. Reuse suite `defaults` for shared flags like `timeout_sec` or expected exit code.
+4. Prefer a scenario builder if the domain already has one.
+5. Use explicit `mocks:` only for narrow fault injection that is not worth promoting into a scenario builder yet.
+6. Add `post_checks` when the script's output is file-based.
+7. Run `python3 common/yaml_test_harness/report.py ` and inspect `common/reports/` if the case fails.
+
+