diff --git a/README.md b/README.md index a14c185c..0c54e435 100644 --- a/README.md +++ b/README.md @@ -631,7 +631,7 @@ optional arguments: usage: fluster.py run [-h] [-j JOBS] [-t TIMEOUT] [-ff] [-q] [-ts TESTSUITES [TESTSUITES ...]] [-tv TESTVECTORS [TESTVECTORS ...]] [-sv SKIPVECTORS [SKIPVECTORS ...]] [-d DECODERS [DECODERS ...]] [-s] -[-so SUMMARY_OUTPUT] [-f {md,csv,junitxml}] [-k] [-th THRESHOLD] +[-so SUMMARY_OUTPUT] [-f {md,csv,junitxml}] [-k] [-p PROFILES [PROFILES ...]] [-th THRESHOLD] [-tth TIME_THRESHOLD] [-v] optional arguments: @@ -657,6 +657,8 @@ optional arguments: -f {md,csv,json,junitxml}, --format {md,csv,json,junitxml} specify the format for the summary file -k, --keep keep output files generated during the test + -p PROFILES [PROFILES ...], --profiles PROFILES [PROFILES ...] + run only test vectors matching the given profiles (e.g. 'VP9 Profile 0') -th THRESHOLD, --threshold THRESHOLD set exit code to 2 if threshold tests are not success. exit code is 0 otherwise @@ -666,6 +668,20 @@ optional arguments: -v, --verbose show stdout and stderr of commands executed ``` +**Profile Filtering:** +- The `-p/--profiles` option filters test vectors by their codec profile. Only vectors + with a matching profile are run; vectors without profile metadata are excluded. +- Multiple profiles can be specified (OR logic). +- Profile names are case-insensitive (e.g. `-p Main`, `-p "VP9 Profile 0"`). +- VP9 profiles are automatically inferred from test vector names + (`vp90-` prefix → Profile 0, `vp91-` → Profile 1, etc.). +- Vectors without a profile field in the JSON file are always excluded + when profile filtering is active. +- Examples: + - `./fluster.py run -ts JVT-AVC_V1 -p Baseline` runs only Baseline profile vectors + - `./fluster.py run -ts VP9-TEST-VECTORS -p "VP9 Profile 0"` runs only Profile 0 vectors + - `./fluster.py run -ts JVT-AVC_V1 -p Baseline Main High` runs vectors matching any of the three profiles + ### Download ```bash diff --git a/fluster/codec.py b/fluster/codec.py index 6ce4dd8c..36ba8091 100644 --- a/fluster/codec.py +++ b/fluster/codec.py @@ -16,6 +16,7 @@ # License along with this library. If not, see . from enum import Enum +from typing import Optional class Codec(Enum): @@ -67,43 +68,68 @@ class OutputFormat(Enum): class Profile(Enum): - """Profile""" + """Profile - NONE = "None" + Each member carries a ``codec`` attribute identifying its codec. + """ + + codec: Codec + + NONE = ("None", Codec.NONE) # H.264 - CONSTRAINED_BASELINE = "Constrained Baseline" - BASELINE = "Baseline" - EXTENDED = "Extended" - MAIN = "Main" - HIGH = "High" - HIGH_10 = "High 10" - HIGH_10_INTRA = "High 10 Intra" - HIGH_4_2_2 = "High 4:2:2" - HIGH_4_2_2_INTRA = "High 4:2:2 Intra" - HIGH_4_4_4_INTRA = "High 4:4:4 Intra" - HIGH_4_4_4_PREDICTIVE = "High 4:4:4 Predictive" - CAVLC_4_4_4 = "CAVLC 4:4:4" - CAVLC_4_4_4_INTRA = "CAVLC 4:4:4 Intra" - - MAIN_10 = "Main 10" - MAIN_STILL_PICTURE = "Main Still Picture" - MAIN_4_2_2_10 = "Main 4:2:2 10" - MAIN_4_4_4_12 = "Main 4:4:4 12" + CONSTRAINED_BASELINE = ("Constrained Baseline", Codec.H264) + BASELINE = ("Baseline", Codec.H264) + EXTENDED = ("Extended", Codec.H264) + MAIN = ("Main", Codec.H264) + HIGH = ("High", Codec.H264) + HIGH_10 = ("High 10", Codec.H264) + HIGH_10_INTRA = ("High 10 Intra", Codec.H264) + HIGH_4_2_2 = ("High 4:2:2", Codec.H264) + HIGH_4_2_2_INTRA = ("High 4:2:2 Intra", Codec.H264) + HIGH_4_4_4_INTRA = ("High 4:4:4 Intra", Codec.H264) + HIGH_4_4_4_PREDICTIVE = ("High 4:4:4 Predictive", Codec.H264) + CAVLC_4_4_4 = ("CAVLC 4:4:4", Codec.H264) + CAVLC_4_4_4_INTRA = ("CAVLC 4:4:4 Intra", Codec.H264) + + MAIN_10 = ("Main 10", Codec.H264) + MAIN_STILL_PICTURE = ("Main Still Picture", Codec.H264) + MAIN_4_2_2_10 = ("Main 4:2:2 10", Codec.H264) + MAIN_4_4_4_12 = ("Main 4:4:4 12", Codec.H264) # H.266 - MAIN_10_4_4_4 = "Main 10 4:4:4" - MAIN_10_STILL_PICTURE = "Main 10 Still Picture" - MAIN_10_4_4_4_STILL_PICTURE = "Main 10 4:4:4 Still Picture" - MULTILAYER_MAIN_10 = "Multilayer Main 10" - MULTILAYER_MAIN_10_4_4_4 = "Multilayer Main 10 4:4:4" + MAIN_10_4_4_4 = ("Main 10 4:4:4", Codec.H266) + MAIN_10_STILL_PICTURE = ("Main 10 Still Picture", Codec.H266) + MAIN_10_4_4_4_STILL_PICTURE = ("Main 10 4:4:4 Still Picture", Codec.H266) + MULTILAYER_MAIN_10 = ("Multilayer Main 10", Codec.H266) + MULTILAYER_MAIN_10_4_4_4 = ("Multilayer Main 10 4:4:4", Codec.H266) # MPEG2 video - PROFILE_4_2_2 = "4:2:2" - SIMPLE = "Simple" + PROFILE_4_2_2 = ("4:2:2", Codec.MPEG2_VIDEO) + SIMPLE = ("Simple", Codec.MPEG2_VIDEO) # MPEG4 video - SIMPLE_PROFILE = "Simple Profile" - ADVANCED_SIMPLE_PROFILE = "Advanced Simple Profile" - SIMPLE_STUDIO_PROFILE = "Simple Studio Profile" - ERROR_RESILIENT_SIMPLE_SCALABLE_PROFILE = "Error Resilient Simple Scalable Profile" + SIMPLE_PROFILE = ("Simple Profile", Codec.MPEG4_VIDEO) + ADVANCED_SIMPLE_PROFILE = ("Advanced Simple Profile", Codec.MPEG4_VIDEO) + SIMPLE_STUDIO_PROFILE = ("Simple Studio Profile", Codec.MPEG4_VIDEO) + ERROR_RESILIENT_SIMPLE_SCALABLE_PROFILE = ( + "Error Resilient Simple Scalable Profile", + Codec.MPEG4_VIDEO, + ) + + # VP9 + VP9_PROFILE_0 = ("VP9 Profile 0", Codec.VP9) + VP9_PROFILE_1 = ("VP9 Profile 1", Codec.VP9) + VP9_PROFILE_2 = ("VP9 Profile 2", Codec.VP9) + VP9_PROFILE_3 = ("VP9 Profile 3", Codec.VP9) + + def __new__(cls, display_name: str, codec: Optional[Codec] = None) -> "Profile": + if codec is None: + for member in cls: + if member.value == display_name: + return member + raise ValueError(f"{display_name!r} is not a valid {cls.__qualname__}") + obj = object.__new__(cls) + obj._value_ = display_name + obj.codec = codec + return obj diff --git a/fluster/fluster.py b/fluster/fluster.py index feb1b254..92b7c164 100644 --- a/fluster/fluster.py +++ b/fluster/fluster.py @@ -58,6 +58,7 @@ def __init__( verbose: bool = False, summary_output: str = "", summary_format: str = "", + profiles: Optional[List[str]] = None, ): self.jobs = jobs self.timeout = timeout @@ -77,6 +78,8 @@ def __init__( self.verbose = verbose self.summary_output = summary_output self.summary_format = summary_format + self.profiles_names = profiles + self.profiles: List[Profile] = [] def to_test_suite_context( self, @@ -98,6 +101,7 @@ def to_test_suite_context( skip_vectors=skip_vectors, keep_files=self.keep_files, verbose=self.verbose, + profiles=self.profiles, ) return ts_context @@ -233,6 +237,25 @@ def list_test_suites( if len(self.test_suites) == 0: print(f' No test suites found in "{self.test_suites_dir}"') + def list_profiles(self, codec: Optional[Codec] = None) -> None: + """List all available profiles""" + print("\nList of available profiles:") + profiles_dict: Dict[Codec, List[Profile]] = {} + for profile in Profile: + if profile == Profile.NONE: + continue + current_codec = profile.codec + if current_codec not in profiles_dict: + profiles_dict[current_codec] = [] + profiles_dict[current_codec].append(profile) + + for current_codec, profile_list in profiles_dict.items(): + if codec and codec != current_codec: + continue + print(f"\n{current_codec}") + for profile in profile_list: + print(f" {profile.value}") + @staticmethod def _get_matches(in_list: List[str], check_list: List[Any], name: str) -> List[Any]: if in_list: @@ -258,6 +281,14 @@ def _normalize_context(self, ctx: Context) -> None: ctx.test_vectors_names = [x.lower() for x in ctx.test_vectors_names] if ctx.skip_vectors_names: ctx.skip_vectors_names = [x.lower() for x in ctx.skip_vectors_names] + if ctx.profiles_names: + profile_values = {p.value.lower(): p for p in Profile} + for name in ctx.profiles_names: + if name.lower() in profile_values: + ctx.profiles.append(profile_values[name.lower()]) + else: + available = ", ".join(sorted(p.value for p in Profile if p != Profile.NONE)) + sys.exit(f"Unknown profile '{name}'. Available profiles: {available}") ctx.test_suites = self._get_matches(ctx.test_suites_names, self.test_suites, "test suite") ctx.decoders = self._get_matches(ctx.decoders_names, self.decoders, "decoders") diff --git a/fluster/main.py b/fluster/main.py index 4bb1e180..a1d79ce6 100644 --- a/fluster/main.py +++ b/fluster/main.py @@ -280,6 +280,12 @@ def _add_run_cmd(self, subparsers: Any) -> None: help="set exit code to 3 if test suite takes longer than threshold seconds. exit code is 0 otherwise", type=float, ) + subparser.add_argument( + "-p", + "--profiles", + help="run only test vectors for the given profiles (e.g. 'VP9 Profile 0')", + nargs="+", + ) subparser.add_argument( "-v", "--verbose", @@ -365,6 +371,7 @@ def _add_download_cmd(self, subparsers: Any) -> None: def _list_cmd(args: Any, fluster: Fluster) -> None: fluster.list_test_suites(show_test_vectors=args.testvectors, test_suites=args.testsuites, codec=args.codec) fluster.list_decoders(check=args.check, verbose=args.verbose, codec=args.codec) + fluster.list_profiles(codec=args.codec) @staticmethod def _run_cmd(args: Any, fluster: Fluster) -> None: @@ -385,6 +392,7 @@ def _run_cmd(args: Any, fluster: Fluster) -> None: verbose=args.verbose, summary_output=args.summary_output, summary_format=args.format, + profiles=args.profiles, ) try: fluster.run_test_suites(context) diff --git a/fluster/test_suite.py b/fluster/test_suite.py index 0fc3eab6..0e879047 100644 --- a/fluster/test_suite.py +++ b/fluster/test_suite.py @@ -30,7 +30,7 @@ from unittest.result import TestResult from fluster import utils -from fluster.codec import Codec +from fluster.codec import Codec, Profile from fluster.decoder import Decoder, get_reference_decoder_for_codec from fluster.test import MD5ComparisonTest, PixelComparisonTest, ReferenceComparisonTest, SampleComparisonTest, Test from fluster.test_vector import TestVector, TestVectorResult @@ -98,6 +98,7 @@ def __init__( verbose: bool = False, reference_decoder: Optional[Decoder] = None, test_vector_names: Optional[Set[str]] = None, + profiles: Optional[List[Profile]] = None, ): self.jobs = jobs self.decoder = decoder @@ -113,6 +114,7 @@ def __init__( self.verbose = verbose self.reference_decoder = reference_decoder self.test_vector_names = test_vector_names + self.profiles = profiles class TestMethod(Enum): @@ -165,6 +167,18 @@ def clone(self) -> "TestSuite": """Create a deep copy of the object""" return copy.deepcopy(self) + @staticmethod + def _infer_vp9_profile(name: str) -> Optional[Profile]: + if name.startswith("vp90-"): + return Profile.VP9_PROFILE_0 + if name.startswith("vp91-"): + return Profile.VP9_PROFILE_1 + if name.startswith("vp92-"): + return Profile.VP9_PROFILE_2 + if name.startswith("vp93-"): + return Profile.VP9_PROFILE_3 + return None + @classmethod def from_json_file(cls: Type["TestSuite"], filename: str, resources_dir: str) -> "TestSuite": """Create a TestSuite instance from a file""" @@ -181,7 +195,15 @@ def from_json_file(cls: Type["TestSuite"], filename: str, resources_dir: str) -> data.pop("test_vectors_not_run", None) data.pop("test_vectors_not_supported", None) data.pop("time_taken", None) - return cls(filename, resources_dir, **data) + test_suite = cls(filename, resources_dir, **data) + # Infer VP9 profile from filename when not explicitly set + if test_suite.codec == Codec.VP9: + for test_vector in test_suite.test_vectors.values(): + if test_vector.profile is None: + inferred = cls._infer_vp9_profile(test_vector.name) + if inferred: + test_vector.profile = inferred + return test_suite def to_json_file(self, filename: str) -> None: """Serialize the test suite to a file""" @@ -577,6 +599,8 @@ def generate_tests(self, ctx: Context) -> List[Test]: name_lower = test_vector.name.lower() if ctx.test_vector_names is not None and name_lower not in ctx.test_vector_names: continue + if ctx.profiles and test_vector.profile not in ctx.profiles: + continue if ctx.skip_vectors and name_lower in ctx.skip_vectors: skip = True diff --git a/fluster/test_vector.py b/fluster/test_vector.py index 8889752b..7896b8d9 100644 --- a/fluster/test_vector.py +++ b/fluster/test_vector.py @@ -18,7 +18,7 @@ from enum import Enum from typing import Any, Dict, List, Optional, Type -from fluster.codec import OutputFormat, Profile +from fluster.codec import Codec, OutputFormat, Profile class TestVectorResult(Enum): @@ -45,6 +45,7 @@ def __init__( output_format: OutputFormat, result: str, profile: Optional[Profile] = None, + codec: Optional[Codec] = None, optional_params: Optional[Dict[str, Any]] = None, ): # JSON members @@ -53,6 +54,7 @@ def __init__( self.source_checksum = source_checksum self.input_file = input_file self.profile = profile + self.codec = codec or (profile.codec if profile else None) self.optional_params = optional_params self.output_format = output_format self.result = result @@ -73,6 +75,7 @@ def from_json(cls: Type["TestVector"], data: Any) -> Any: # We only define profile if the paramter is found in .json of test suite if "profile" in data: data["profile"] = Profile(data["profile"]) + data["codec"] = data["profile"].codec return (data["name"], cls(**data)) @@ -85,8 +88,10 @@ def data_to_serialize(self) -> Dict[str, object]: data["output_format"] = str(self.output_format.value) if self.profile is not None: data["profile"] = str(self.profile.value) + data["codec"] = str(self.codec) else: data.pop("profile") + data.pop("codec", None) if self.optional_params is not None: data["optional_params"] = self.optional_params else: