Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -280,7 +280,7 @@ V7 artifact authorization uses `ARTIFACT_UPLOAD_SECRET` only on the server to si

## Version identities

- Published project release: `1.3.0-rc.2` / `2026-09-21` in `release.json`; client/0.3.1, protocol `7.1`.
- Published project release: `1.3.0-rc.3` / `2026-09-22` in `release.json`; client/0.3.2, protocol `7.1`.
- Historical project release: `1.2.0` / `client/0.2.0`, protocol `7.0` (historical timing).
- Candidate client implementation/minimum version: `client/0.3.0`.
- Candidate benchmark protocol version: `7.1`, timer boundary `ffmpeg-process-v1`.
Expand Down
76 changes: 53 additions & 23 deletions client/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@
print_info, print_success, print_warning, print_error, print_batch_summary,
)

CLIENT_VERSION = "client/0.3.1"
CLIENT_VERSION = "client/0.3.2"
# UI/package patches do not change the server's frozen protocol 7.1 contract.
PROTOCOL_MINIMUM_CLIENT_VERSION = "client/0.3.0"
PUBLICATION_CONSENT_VERSION = 1
Expand Down Expand Up @@ -305,27 +305,31 @@ def progress(stage, **details):
return wrapped


def _preparation_preflight(args, *, base_url=None):
def _preparation_preflight(args, *, base_url=None, event_sink=None):
check_preparation_cancelled()
if not getattr(args, "no_submit", False):
preparation_progress("compatibility")
try:
check_compatibility(base_url or args.base_url, CLIENT_VERSION)
except Exception as exc:
print(f"Compatibility check failed before preparation: {exc}. Use --no-submit for local collection.", file=sys.stderr)
message = f"Compatibility check failed before preparation: {exc}. Use --no-submit for local collection."
print(message, file=sys.stderr)
_emit_event(event_sink, "run_error", scope="preparation", code=5, message=message)
return 5
return _preparation_runtime_integrity()
return _preparation_runtime_integrity(event_sink=event_sink)


def _preparation_runtime_integrity():
def _preparation_runtime_integrity(event_sink=None):
check_preparation_cancelled()
if bool(getattr(sys, "frozen", False)) or os.environ.get("ENCODINGDB_RUNTIME_LOCK_PATH"):
from .runtime_lock import verify_runtime_lock
preparation_progress("runtime")
try:
verify_runtime_lock(ffmpeg_path=config.ffmpeg_exe(), ffprobe_path=config.ffprobe_exe())
except Exception as exc:
print(f"Runtime integrity check failed before preparation: {exc}", file=sys.stderr)
message = f"Runtime integrity check failed before preparation: {exc}"
print(message, file=sys.stderr)
_emit_event(event_sink, "run_error", scope="preparation", code=2, message=message)
return 2
return 0

Expand Down Expand Up @@ -1260,13 +1264,15 @@ def run_sweep_mode(
print(f"Unsupported sweep mode: {mode}", file=sys.stderr)
return 4
base_args = _apply_submission_policy(base_args, interactive=interactive)
preflight_rc = _preparation_preflight(base_args)
preflight_rc = _preparation_preflight(base_args, event_sink=event_sink)
if preflight_rc:
return preflight_rc
presets_cfg = presets_cfg if presets_cfg is not None else load_presets_config(PRESETS_CONFIG_PATH)
candidates = list_all_available_encoders()
if not candidates:
print("No available encoders found in this ffmpeg build.", file=sys.stderr)
message = "No available encoders found in this ffmpeg build."
print(message, file=sys.stderr)
_emit_event(event_sink, "run_error", scope="preparation", code=4, message=message)
return 4
plan = sweep_plan.plan_sweep(
mode,
Expand All @@ -1275,14 +1281,18 @@ def run_sweep_mode(
is_usable=_probe_encoder_usable_with_cancel,
)
if plan.is_empty():
print("No usable encoder on this machine supports a sweep.", file=sys.stderr)
message = "No usable encoder on this machine supports a sweep."
print(message, file=sys.stderr)
_emit_event(event_sink, "run_error", scope="preparation", code=4, message=message)
return 4
for name, reason in plan.skipped:
print_info(f"Skipped {sweep_plan_label(name)}: {reason}")
try:
suite_clips = _prepare_sweep_clips(plan.clip_policy)
except Exception as exc:
print(f"EncodingDB Test Suite v1 is unavailable: {exc}", file=sys.stderr)
message = f"EncodingDB Test Suite v1 is unavailable: {exc}"
print(message, file=sys.stderr)
_emit_event(event_sink, "run_error", scope="preparation", code=3, message=message)
return 3
tasks: List[Dict[str, Any]] = []
for step in plan.steps:
Expand Down Expand Up @@ -1414,29 +1424,37 @@ def run_benchmark_batch(
) -> int:
duration_minutes = float(getattr(args, "max_duration_minutes", 60))
if not math.isfinite(duration_minutes) or not math.isfinite(duration_minutes * 60) or duration_minutes <= 0:
print("--max-duration-minutes must be positive and finite", file=sys.stderr)
message = "--max-duration-minutes must be positive and finite"
print(message, file=sys.stderr)
_emit_event(event_sink, "run_error", scope="batch", code=4, message=message)
return 4
preflight_rc = _preparation_preflight(args, base_url=base_url)
preflight_rc = _preparation_preflight(args, base_url=base_url, event_sink=event_sink)
if preflight_rc:
return preflight_rc
ok, ffmpeg_version = ensure_ffmpeg_and_ffprobe()
if not ok:
print("ffmpeg/ffprobe not found in PATH. Please install ffmpeg.", file=sys.stderr)
message = "ffmpeg/ffprobe not found in PATH. Please install ffmpeg."
print(message, file=sys.stderr)
_emit_event(event_sink, "run_error", scope="batch", code=2, message=message)
return 2
if getattr(args, "local_metrics", False):
quality_ok, quality_rc = _ensure_local_quality_stack(event_sink=event_sink, scope="batch")
if not quality_ok:
return quality_rc
suite_clip = tasks[0].get("suiteClip") if tasks else None
if not isinstance(suite_clip, PreparedSuiteClip):
print("Batch benchmark requires EncodingDB Test Suite v1 clip identities.", file=sys.stderr)
message = "Batch benchmark requires EncodingDB Test Suite v1 clip identities."
print(message, file=sys.stderr)
_emit_event(event_sink, "run_error", scope="batch", code=3, message=message)
return 3
input_path = suite_clip.path
default_input_hash = suite_clip.input_hash
protocol_config = _build_protocol_config()
planned_attempts = len(tasks) * (protocol_config.warmup_runs + protocol_config.minimum_measured_runs + protocol_config.max_adaptive_repeats)
if planned_attempts > int(getattr(args, "max_attempts", 100)):
print(f"Campaign can require {planned_attempts} encodes, exceeding --max-attempts. Select fewer recipes or set an explicit budget.", file=sys.stderr)
message = f"Campaign can require {planned_attempts} encodes, exceeding --max-attempts. Select fewer recipes or set an explicit budget."
print(message, file=sys.stderr)
_emit_event(event_sink, "run_error", scope="batch", code=4, message=message)
return 4
campaign_seed = getattr(args, "campaign_seed", None)
if campaign_seed is None:
Expand Down Expand Up @@ -2261,20 +2279,26 @@ def run_v7_suite_clip_mode(
interactive: bool = False,
) -> int:
base_args = _apply_submission_policy(base_args, interactive=interactive)
preflight_rc = _preparation_preflight(base_args)
preflight_rc = _preparation_preflight(base_args, event_sink=event_sink)
if preflight_rc:
return preflight_rc
clip_id = str(getattr(base_args, "v7_suite_clip", "") or "").strip()
campaign_scope = str(getattr(base_args, "campaign", "quick") or "quick")
try:
suite_clips = (_prepare_full_suite() if getattr(base_args, "campaign", "quick") == "full"
suite_clips = (_prepare_full_suite() if campaign_scope == "full"
else [_prepare_named_suite_clip(clip_id) if clip_id else _prepare_quick_suite_clip()])
except Exception as exc:
print(f"Unable to prepare suite clip {clip_id}: {exc}", file=sys.stderr)
label = clip_id or ("all seven frozen clips" if campaign_scope == "full" else "the default quick clip")
message = f"Unable to prepare suite clip {label}: {exc}"
print(message, file=sys.stderr)
_emit_event(event_sink, "run_error", scope="preparation", code=3, message=message)
return 3

requested_codec = str(getattr(base_args, "codec", "") or "").strip()
if not requested_codec:
print("--codec is required for noninteractive v7 suite clip mode.", file=sys.stderr)
message = "--codec is required for noninteractive v7 suite clip mode."
print(message, file=sys.stderr)
_emit_event(event_sink, "run_error", scope="preparation", code=4, message=message)
return 4
if has_encoder(requested_codec):
resolved_encoder = requested_codec
Expand All @@ -2284,10 +2308,14 @@ def run_v7_suite_clip_mode(
else:
resolved_encoder = None
if not resolved_encoder or not has_encoder(resolved_encoder):
print(f"Requested encoder '{requested_codec}' is not available.", file=sys.stderr)
message = f"Requested encoder '{requested_codec}' is not available."
print(message, file=sys.stderr)
_emit_event(event_sink, "run_error", scope="preparation", code=4, message=message)
return 4
if is_hardware_encoder_name(resolved_encoder) and not is_hardware_encoder_usable(resolved_encoder):
print(f"Selected hardware encoder '{resolved_encoder}' is not usable on this machine.", file=sys.stderr)
message = f"Selected hardware encoder '{resolved_encoder}' is not usable on this machine."
print(message, file=sys.stderr)
_emit_event(event_sink, "run_error", scope="preparation", code=4, message=message)
return 4

preset_list = [value.strip() for value in str(getattr(base_args, "presets", "") or "").split(",") if value.strip()]
Expand All @@ -2297,7 +2325,9 @@ def run_v7_suite_clip_mode(
target_bitrate_kbps = getattr(base_args, "target_bitrate_kbps", None)
if target_bitrate_kbps is not None or resolved_encoder.lower().endswith("_videotoolbox"):
if target_bitrate_kbps is None or target_bitrate_kbps <= 0:
print("VideoToolbox v7 runs require --target-bitrate-kbps.", file=sys.stderr)
message = "VideoToolbox v7 runs require --target-bitrate-kbps."
print(message, file=sys.stderr)
_emit_event(event_sink, "run_error", scope="preparation", code=4, message=message)
return 4
task_rate_control = {"mode": "vbr", "targetBitrateKbps": int(target_bitrate_kbps)}
else:
Expand Down Expand Up @@ -2340,7 +2370,7 @@ def run_v7_suite_clip_mode(
@_preparation_operation
def _resume_campaign(args, *, event_sink=None, cancel_event=None, interactive=False):
args = _apply_submission_policy(args, interactive=interactive)
preflight_rc = _preparation_preflight(args)
preflight_rc = _preparation_preflight(args, event_sink=event_sink)
if preflight_rc:
return preflight_rc
try:
Expand Down
54 changes: 51 additions & 3 deletions client/suite.py
Original file line number Diff line number Diff line change
Expand Up @@ -966,14 +966,53 @@ def _copy_preparation_file(source, destination):
_copy_preparation_stream(reader, writer, path=source, total=os.path.getsize(source))


def _suite_pack_target_access_error(target_root: str) -> Optional[str]:
"""Explain why an existing extracted root can be neither reused nor replaced.

A cache created by a different account or an administrator-privileged run is
invisible to os.path.exists and undeletable by the normal user; returning a
cause here prevents a multi-gigabyte re-extraction that cannot land anyway.
"""
if not os.path.isdir(target_root):
return None
try:
with os.scandir(target_root) as entries:
for _ in entries:
break
except PermissionError as exc:
return (
f"suite cache at {target_root} exists but is not accessible to the current user ({exc}); "
"it was created by a different account or an administrator-privileged run - delete that "
"cache folder, then start the run again"
)
except OSError as exc:
return (
f"suite cache at {target_root} cannot be inspected ({exc}); delete that cache folder, "
"then start the run again"
)
try:
with open(os.path.join(target_root, "manifest.json"), "rb"):
return None
except FileNotFoundError:
return None
except OSError as exc:
return (
f"suite cache at {target_root} cannot be read ({exc}); it was likely created by a "
"different account or an administrator-privileged run - delete that cache folder, then "
"start the run again"
)


def _extract_suite_pack(pack_path: str, metadata: Mapping[str, Any], cache_root: Optional[str] = None) -> str:
target_root = _suite_pack_extract_root(metadata, cache_root)
canonical_root = os.path.join(target_root, "canonical")
try:
_verify_extracted_suite_pack(target_root, metadata, verify_media=False)
return canonical_root
except Exception:
pass
access_error = _suite_pack_target_access_error(target_root)
if access_error:
raise RuntimeError(access_error) from None
parent_dir = os.path.dirname(target_root)
os.makedirs(parent_dir, exist_ok=True)
staging_root = tempfile.mkdtemp(prefix="suite-pack-", dir=parent_dir)
Expand All @@ -989,8 +1028,17 @@ def _extract_suite_pack(pack_path: str, metadata: Mapping[str, Any], cache_root:
with archive.extractfile(member) as source, open(destination, "xb") as target:
_copy_preparation_stream(source, target, path=member.name, total=member.size)
_verify_extracted_suite_pack(staging_root, metadata)
shutil.rmtree(target_root, ignore_errors=True)
os.replace(staging_root, target_root)
try:
if os.path.isdir(target_root):
shutil.rmtree(target_root)
os.replace(staging_root, target_root)
except OSError as swap_exc:
raise RuntimeError(
f"verified suite content could not be installed into {target_root} because the "
f"existing cache folder could not be replaced ({swap_exc}); close any program "
"holding that folder (for example an Explorer window), delete it if asked, and "
"start the run again"
) from swap_exc
except BaseException:
shutil.rmtree(staging_root, ignore_errors=True)
raise
Expand Down
2 changes: 1 addition & 1 deletion client/tests/test_encoding_regressions.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ def setUp(self) -> None:
def test_corrected_metrics_use_distinguishable_client_version(self) -> None:
from client import main as client_main

self.assertEqual(client_main.CLIENT_VERSION, "client/0.3.1")
self.assertEqual(client_main.CLIENT_VERSION, "client/0.3.2")
self.assertEqual(client_main.PROTOCOL_MINIMUM_CLIENT_VERSION, "client/0.3.0")

def test_vmaf_passes_distorted_input_before_reference(self) -> None:
Expand Down
40 changes: 40 additions & 0 deletions client/tests/test_preparation.py
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,46 @@ def test_progress_is_throttled_and_does_not_start_budget(self):
self.assertIsNone(campaign._MEASUREMENT_BUDGET.get())
progress.assert_called_once()

def test_sweep_suite_unavailable_surfaces_causal_run_error(self):
events = []
presets = main.load_presets_config(main.PRESETS_CONFIG_PATH)
with mock.patch.object(main, 'list_all_available_encoders', return_value=['libx264']), \
mock.patch.object(main, '_probe_encoder_usable_with_cancel', return_value=True), \
mock.patch.object(main, '_prepare_sweep_clips',
side_effect=RuntimeError('suite pack could not be acquired: offline')):
rc = main.run_sweep_mode(mode='small', base_args=self.args(no_submit=True),
event_sink=events.append, presets_cfg=presets)
self.assertEqual(rc, 3)
errors = [event for event in events if event.get('type') == 'run_error']
self.assertEqual([event['code'] for event in errors], [3])
self.assertEqual(errors[0]['scope'], 'preparation')
self.assertIn('offline', errors[0]['message'])

def test_v7_quick_prep_failure_surfaces_causal_run_error(self):
events = []
with mock.patch.object(main, '_prepare_quick_suite_clip',
side_effect=RuntimeError('pack download failed: HTTP 403')):
rc = main.run_v7_suite_clip_mode(base_args=self.args(no_submit=True), event_sink=events.append)
self.assertEqual(rc, 3)
errors = [event for event in events if event.get('type') == 'run_error']
self.assertEqual([event['code'] for event in errors], [3])
self.assertIn('the default quick clip', errors[0]['message'])
self.assertIn('HTTP 403', errors[0]['message'])

def test_batch_missing_clip_identity_surfaces_causal_run_error(self):
events = []
hardware = main.HardwareInfo('Test CPU', 'none', 16, 'TestOS')
with mock.patch.object(main, 'ensure_ffmpeg_and_ffprobe', return_value=(True, 'ffmpeg test')):
rc = main.run_benchmark_batch(
hardware=hardware, base_url='https://invalid.example',
args=self.args(no_submit=True, max_attempts=100, max_duration_minutes=60),
tasks=[{'encoder': 'libx264', 'preset': 'fast', 'crf': 24, 'rateControl': None}],
event_sink=events.append)
self.assertEqual(rc, 3)
errors = [event for event in events if event.get('type') == 'run_error']
self.assertEqual([event['code'] for event in errors], [3])
self.assertIn('clip identities', errors[0]['message'])


if __name__ == '__main__':
unittest.main()
4 changes: 2 additions & 2 deletions client/tests/test_release_preflight.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,8 @@ def test_release_json_declares_coherent_frozen_release(self) -> None:
payload = json.loads((release_manifest_lib.ROOT_DIR / "release.json").read_text(encoding="utf-8"))

self.assertEqual(payload["suiteVersion"], "encodingdb-test-suite-v1")
self.assertEqual(payload["projectVersion"], "1.3.0-rc.2")
self.assertEqual(payload["releaseDate"], "2026-09-21")
self.assertEqual(payload["projectVersion"], "1.3.0-rc.3")
self.assertEqual(payload["releaseDate"], "2026-09-22")
for tree in ("client", "server"):
root = release_manifest_lib.ROOT_DIR / tree / "resources/test_suite_v1"
status = json.loads((root / "finalization-status.json").read_text())
Expand Down
Loading
Loading