diff --git a/.github/workflows/docker-hub-publish.yml b/.github/workflows/docker-hub-publish.yml index 1450134..6330e7b 100644 --- a/.github/workflows/docker-hub-publish.yml +++ b/.github/workflows/docker-hub-publish.yml @@ -34,7 +34,7 @@ jobs: - name: Build the Docker image - run: docker build . --file Dockerfile -t mobilesecurity/mdast_cli:2026.8.5 -t mobilesecurity/mdast_cli:latest + run: docker build . --file Dockerfile -t mobilesecurity/mdast_cli:2026.8.6 -t mobilesecurity/mdast_cli:latest - name: Docker Hub push latest image @@ -42,5 +42,4 @@ jobs: - name: Docker Hub push tagged image - run: docker push mobilesecurity/mdast_cli:2026.8.5 - + run: docker push mobilesecurity/mdast_cli:2026.8.6 diff --git a/README.md b/README.md index d58c1e7..2c346f2 100644 --- a/README.md +++ b/README.md @@ -752,6 +752,15 @@ and/or `--summary_report_json_file_name` overrides the name and forces that form > report service returns 502) is a **warning, not a failure**: the scan already > succeeded, so the CLI still exits 0 and any other requested report is still saved. +**Asynchronous reports on microservices.** A large PDF can be prepared +asynchronously: the report endpoint may return `202 Accepted` while the report is +still being generated. The CLI waits and retries until the endpoint returns `200`, +using `Retry-After` when the server provides it as integer seconds, otherwise +falling back to 10 seconds. Use `--report-timeout ` (alias: +`--report_timeout`) to control the maximum wait for report readiness. The default +is 1800 seconds (30 minutes). If the timeout is reached, the scan still stays +successful and the report step is reported as a warning. + > **In Docker:** the default `scan_report_.*` name is written to the > container's working directory, which is discarded when the container exits. Mount > a host directory and point the report there with an absolute path (e.g. diff --git a/mdast_cli/__init__.py b/mdast_cli/__init__.py index 69cfa87..1db54d3 100644 --- a/mdast_cli/__init__.py +++ b/mdast_cli/__init__.py @@ -1 +1 @@ -__version__ = '2026.8.5' +__version__ = '2026.8.6' diff --git a/mdast_cli/helpers/const.py b/mdast_cli/helpers/const.py index 7fcb2ff..356a234 100644 --- a/mdast_cli/helpers/const.py +++ b/mdast_cli/helpers/const.py @@ -39,6 +39,7 @@ class DastState: LONG_TRY = 60480 # ~1 week at SLEEP_TIMEOUT=10s (matches --long_wait docs) END_SCAN_TIMEOUT = 30 SLEEP_TIMEOUT = 10 +REPORT_TIMEOUT = 1800 # 30 minutes for async report preparation on microservices # HTTP timeout constants HTTP_REQUEST_TIMEOUT = 30 diff --git a/mdast_cli/mdast_scan.py b/mdast_cli/mdast_scan.py index 6a9a559..f38a407 100644 --- a/mdast_cli/mdast_scan.py +++ b/mdast_cli/mdast_scan.py @@ -24,7 +24,8 @@ from mdast_cli.distribution_systems.rumarket import rumarket_download_app from mdast_cli.distribution_systems.rustore import rustore_download_app from mdast_cli.helpers.const import (ANDROID_EXTENSIONS, DEFAULT_ANDROID_ARCHITECTURE, DEFAULT_IOS_ARCHITECTURE, - END_SCAN_TIMEOUT, LONG_TRY, SLEEP_TIMEOUT, TRY, DastState, DastStateDict) + END_SCAN_TIMEOUT, LONG_TRY, REPORT_TIMEOUT, SLEEP_TIMEOUT, TRY, DastState, + DastStateDict) from mdast_cli.helpers.exit_codes import ExitCode from mdast_cli.helpers.helpers import check_app_md5, resolve_report_targets from mdast_cli_core.token import mDastToken as mDast @@ -348,6 +349,10 @@ def parse_args(): help='File name for saving PDF report with scan results. ' 'Optional parameter. If specified, the PDF report will be saved to this file ' '(implies --report_format pdf).') + scan_group.add_argument('--report-timeout', '--report_timeout', dest='report_timeout', type=int, + default=REPORT_TIMEOUT, + help='Maximum time in seconds to wait for asynchronous report preparation ' + f'on microservices installations. Default: {REPORT_TIMEOUT}.') scan_group.add_argument('--nowait', '-nw', action='store_true', help='Do not wait for scan completion. ' 'If set, utility will start scan and exit immediately. ' @@ -400,6 +405,9 @@ def parse_args(): args = parser.parse_args() + if args.report_timeout <= 0: + parser.error('--report-timeout must be a positive integer number of seconds') + if args.distribution_system == 'file' and args.file_path is None: parser.error('"--distribution_system file" requires "--file_path" argument to be set') elif args.distribution_system == 'nexus' and ( diff --git a/mdast_cli/ms_flow.py b/mdast_cli/ms_flow.py index 9c12cfe..6e11117 100644 --- a/mdast_cli/ms_flow.py +++ b/mdast_cli/ms_flow.py @@ -21,9 +21,9 @@ from mdast_cli.helpers.const import (ACTIVE_STAGES, ANDROID_EXTENSIONS, DEFAULT_ANDROID_ARCHITECTURE, DEFAULT_IOS_ARCHITECTURE, END_SCAN_TIMEOUT, LONG_TRY, OS_ANDROID, - OS_IOS, PRE_START_STAGES, SLEEP_TIMEOUT, TERMINAL_SCAN_PAIRS, TRY, - UPLOAD_TIMEOUT_ENV_VAR, UPLOAD_TIMEOUT_MAX, UPLOAD_TIMEOUT_MIN, - ENGINE_ACTIVE_STATUS, ScanStage, ScanStageStatus) + OS_IOS, PRE_START_STAGES, REPORT_TIMEOUT, SLEEP_TIMEOUT, + TERMINAL_SCAN_PAIRS, TRY, UPLOAD_TIMEOUT_ENV_VAR, UPLOAD_TIMEOUT_MAX, + UPLOAD_TIMEOUT_MIN, ENGINE_ACTIVE_STATUS, ScanStage, ScanStageStatus) from mdast_cli.helpers.exit_codes import ExitCode from mdast_cli.helpers.helpers import check_app_md5, resolve_report_targets from mdast_cli_core.factory import architecture_items @@ -562,12 +562,13 @@ def download_reports(mdast, scan_id, arguments): ignored - the local file name is a CLI argument. """ targets = resolve_report_targets(arguments, scan_id) + report_timeout = getattr(arguments, 'report_timeout', REPORT_TIMEOUT) failures = [] if 'pdf' in targets: target = targets['pdf'] logger.info(f'Create and download pdf report for scan {scan_id} to file {target}.') - resp = _download_report(mdast.download_report, scan_id, 'PDF report') + resp = _download_report(mdast.download_report, scan_id, 'PDF report', report_timeout=report_timeout) if resp is None: failures.append('PDF') else: @@ -578,7 +579,7 @@ def download_reports(mdast, scan_id, arguments): if 'json' in targets: target = targets['json'] logger.info(f'Download JSON summary report for scan {scan_id} to file {target}.') - resp = _download_report(mdast.download_scan_json_result, scan_id, 'JSON report') + resp = _download_report(mdast.download_scan_json_result, scan_id, 'JSON report', report_timeout=report_timeout) if resp is None: failures.append('JSON') else: @@ -600,28 +601,71 @@ def download_reports(mdast, scan_id, arguments): 'Retry the report download later.') -def _download_report(fetch, scan_id, label): - """Download one report with transient retry. Returns Response or None (soft-fail).""" +def _retry_after_delay(resp, label): + """Retry-After support is intentionally limited to integer seconds.""" + raw = resp.headers.get('Retry-After') + if raw in (None, ''): + return SLEEP_TIMEOUT + try: + delay = int(raw) + except ValueError: + logger.warning(f'{label} returned invalid Retry-After={sanitize(raw)!r}; ' + f'using {SLEEP_TIMEOUT} seconds.') + return SLEEP_TIMEOUT + if delay < 0: + logger.warning(f'{label} returned negative Retry-After={delay}; ' + f'using {SLEEP_TIMEOUT} seconds.') + return SLEEP_TIMEOUT + return delay + + +def _download_report(fetch, scan_id, label, report_timeout=REPORT_TIMEOUT): + """Download one report, waiting for async 202 and retrying transient errors.""" last = None - for attempt in range(POLL_TRANSIENT_RETRIES): + transient_attempts = 0 + deadline = time.monotonic() + report_timeout + + while True: try: resp = fetch(scan_id) except requests.RequestException as ex: + transient_attempts += 1 logger.warning(f'{label} request failed ({type(ex).__name__}), ' - f'retry {attempt + 1}/{POLL_TRANSIENT_RETRIES}') - if attempt + 1 < POLL_TRANSIENT_RETRIES: + f'retry {transient_attempts}/{POLL_TRANSIENT_RETRIES}') + if transient_attempts < POLL_TRANSIENT_RETRIES: time.sleep(SLEEP_TIMEOUT) - continue + continue + break + if resp.status_code == 200: return resp - last = resp - if resp.status_code in POLL_TRANSIENT_CODES and attempt + 1 < POLL_TRANSIENT_RETRIES: - logger.warning(f'{label} returned {resp.status_code} (transient), ' - f'retry {attempt + 1}/{POLL_TRANSIENT_RETRIES}') - time.sleep(SLEEP_TIMEOUT) + + if resp.status_code == 202: + remaining = deadline - time.monotonic() + if remaining <= 0: + logger.error(f'{label} was not ready within {report_timeout} seconds. ' + 'Retry the report download later.') + return None + + delay = min(_retry_after_delay(resp, label), remaining) + logger.info(f'{label} is still being prepared (HTTP 202), ' + f'waiting {delay:.0f} seconds before retry.') + time.sleep(delay) continue + + last = resp + if resp.status_code in POLL_TRANSIENT_CODES: + transient_attempts += 1 + if transient_attempts < POLL_TRANSIENT_RETRIES: + logger.warning(f'{label} returned {resp.status_code} (transient), ' + f'retry {transient_attempts}/{POLL_TRANSIENT_RETRIES}') + time.sleep(SLEEP_TIMEOUT) + continue break + if last is not None: logger.error(f'{label} download failed (HTTP {last.status_code}): ' f'{sanitize(extract_error_message(last))}') + else: + logger.error(f'{label} download failed after {POLL_TRANSIENT_RETRIES} attempts (network).') return None diff --git a/setup.py b/setup.py index 546a936..56054c9 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name="mdast_cli", - version='2026.8.5', + version='2026.8.6', python_requires='>=3.12', diff --git a/tests/test_negative_e2e.py b/tests/test_negative_e2e.py index 44f46b2..284a1a0 100644 --- a/tests/test_negative_e2e.py +++ b/tests/test_negative_e2e.py @@ -67,6 +67,10 @@ def test_mode_detection_unreachable_exit_6(mocked_responses, monkeypatch, tmp_ap assert run_main(monkeypatch, base_argv(tmp_apk)) == 6 +def test_report_timeout_must_be_positive(monkeypatch, tmp_apk): + assert run_main(monkeypatch, base_argv(tmp_apk, '--report-timeout', '0')) == 2 + + # --- upload ----------------------------------------------------------------- def test_ms_upload_gateway_502_exhausted_exit_6(mocked_responses, monkeypatch, tmp_apk, diff --git a/tests/test_poll_resilience.py b/tests/test_poll_resilience.py index cbd394b..cc72b3f 100644 --- a/tests/test_poll_resilience.py +++ b/tests/test_poll_resilience.py @@ -88,3 +88,81 @@ def test_report_network_exception_then_success(client, no_sleep): fetch = mock.Mock(side_effect=seq) resp = ms_flow._download_report(fetch, 55, 'PDF report') assert resp is resp_ok + + +def test_report_202_then_202_then_success(client, mocked_responses, monkeypatch): + """Async report preparation is polled until the final 200 response.""" + sleeps = [] + monkeypatch.setattr(ms_flow.time, 'sleep', sleeps.append) + mocked_responses.add(responses.GET, REPORT_URL, status=202, + headers={'Retry-After': '30'}) + mocked_responses.add(responses.GET, REPORT_URL, status=202) + mocked_responses.add(responses.GET, REPORT_URL, body=b'%PDF-1.4 x', + content_type='application/pdf') + + resp = ms_flow._download_report(client.download_report, 55, 'PDF report', + report_timeout=120) + + assert resp is not None + assert resp.content == b'%PDF-1.4 x' + assert sleeps == [30, ms_flow.SLEEP_TIMEOUT] + + +def test_report_202_does_not_consume_transient_retry_budget(client, mocked_responses, + no_sleep, monkeypatch): + """HTTP 202 is a normal pending state, not one of transient retry attempts.""" + monkeypatch.setattr(ms_flow, 'POLL_TRANSIENT_RETRIES', 2) + for _ in range(5): + mocked_responses.add(responses.GET, REPORT_URL, status=202) + mocked_responses.add(responses.GET, REPORT_URL, body=b'%PDF-1.4 x', + content_type='application/pdf') + + resp = ms_flow._download_report(client.download_report, 55, 'PDF report', + report_timeout=120) + + assert resp is not None + assert len([c for c in mocked_responses.calls if c.request.url.startswith(REPORT_URL)]) == 6 + + +def test_report_invalid_retry_after_uses_default_sleep(client, mocked_responses, monkeypatch): + """Retry-After is deliberately supported only as integer seconds.""" + sleeps = [] + monkeypatch.setattr(ms_flow.time, 'sleep', sleeps.append) + mocked_responses.add(responses.GET, REPORT_URL, status=202, + headers={'Retry-After': 'later'}) + mocked_responses.add(responses.GET, REPORT_URL, body=b'%PDF-1.4 x', + content_type='application/pdf') + + resp = ms_flow._download_report(client.download_report, 55, 'PDF report', + report_timeout=120) + + assert resp is not None + assert sleeps == [ms_flow.SLEEP_TIMEOUT] + + +def test_report_202_timeout_soft_fails_to_none(monkeypatch): + """A report that stays pending past the report timeout remains a soft-fail.""" + resp_pending = mock.Mock(status_code=202, headers={}) + fetch = mock.Mock(return_value=resp_pending) + monotonic_values = iter([0, 2]) + monkeypatch.setattr(ms_flow.time, 'monotonic', lambda: next(monotonic_values)) + + resp = ms_flow._download_report(fetch, 55, 'PDF report', report_timeout=1) + + assert resp is None + assert fetch.call_count == 1 + + +def test_report_202_then_transient_then_success(client, mocked_responses, no_sleep): + """Transient failures during async report waiting use their own retry budget.""" + mocked_responses.add(responses.GET, REPORT_URL, status=202) + mocked_responses.add(responses.GET, REPORT_URL, status=502, json={'error_code': 'busy'}) + mocked_responses.add(responses.GET, REPORT_URL, status=202) + mocked_responses.add(responses.GET, REPORT_URL, body=b'%PDF-1.4 x', + content_type='application/pdf') + + resp = ms_flow._download_report(client.download_report, 55, 'PDF report', + report_timeout=120) + + assert resp is not None + assert resp.content == b'%PDF-1.4 x'