diff --git a/mdast_cli/distribution_systems/appstore.py b/mdast_cli/distribution_systems/appstore.py index 6f42966..2b1ed9e 100644 --- a/mdast_cli/distribution_systems/appstore.py +++ b/mdast_cli/distribution_systems/appstore.py @@ -3,6 +3,8 @@ import pickle import plistlib import shutil +import time +import unicodedata import zipfile from functools import lru_cache @@ -12,12 +14,35 @@ from urllib3 import Retry from tqdm import tqdm -from mdast_cli.distribution_systems.appstore_client.store import StoreClient, StoreException +from mdast_cli.distribution_systems.appstore_client.store import ( + FAILURE_LICENSE_NOT_FOUND, + DOWNLOAD_MAX_ATTEMPTS, + DOWNLOAD_RETRY_FAILURES, + DOWNLOAD_RETRY_PAUSE, + FAILURES_NEEDING_REAUTH, + StoreClient, + StoreException, +) from mdast_cli.helpers.file_utils import ensure_download_dir, cleanup_file logger = logging.getLogger(__name__) +def sanitize_file_name(name): + """Make an App Store display name safe to use as a file name. + + Apple ships bidi and other invisible control characters inside bundleDisplayName - + WhatsApp, for one, starts with U+200E - and a path that begins with an invisible + character quietly breaks scripts downstream. Path separators are stripped for the + same reason. + """ + cleaned = ''.join( + ch for ch in (name or '') + if unicodedata.category(ch) not in ('Cc', 'Cf') and ch not in '/\\' + ).strip() + return cleaned or 'application' + + def download_file(url, download_path, file_path): with requests.get(url, stream=True, verify=False) as r: if r.status_code != 200: @@ -118,14 +143,17 @@ def login(self, force=False): pickle.dump(self.store, file) logger.info(f'Dumped session for {self.apple_id}') except StoreException as e: - raise RuntimeError(f'Failed to download application. Seems like your credentials are incorrect ' - f'or your 2FA code expired. Message: {e.req} {e.err_msg} {e.err_type}') + raise RuntimeError(f'Failed to log into iTunes. This is either wrong credentials / an expired 2FA ' + f'code, or Apple refusing the request from this host. ' + f'Message: {e.req} {e.err_msg} {e.err_type}') def get_app_info(self, app_id=None, bundle_id=None, country='US'): if not app_id and not bundle_id: raise 'One of properties ApplicationID or BundleID should be set' - self.login(True) + # Deliberately not a forced login: Apple's auth endpoint is unreliable, so an + # already established session is worth far more than a fresh one. + self.login() resp_info = self.store.find_app(app_id=app_id, bundle_id=bundle_id, country=country).json() try: app_info = resp_info['results'][0] @@ -146,6 +174,36 @@ def get_app_info(self, app_id=None, bundle_id=None, country='US'): 'icon_url': app_info['artworkUrl100'] } + def _download_info_with_retries(self, app_id): + """Fetch download info, riding out Apple's random failures. + + 9610 means the account holds no license yet, so the app is bought and retried. + 5002 is Apple failing at random on an app it serves fine moments later, so the + same request is simply repeated. + """ + last_error = None + for attempt in range(1, DOWNLOAD_MAX_ATTEMPTS + 1): + try: + return self.store.download(app_id) + except StoreException as e: + last_error = e + if e.err_type == FAILURE_LICENSE_NOT_FOUND: + logger.info('No license found for this app yet, purchasing and retrying') + self.store.purchase(app_id) + continue + if e.err_type not in DOWNLOAD_RETRY_FAILURES or attempt == DOWNLOAD_MAX_ATTEMPTS: + logger.warning( + 'store.download failed: failureType=%s, app_id=%s', e.err_type, app_id, + ) + raise + logger.info( + 'App Store returned failureType %s for app %s (attempt %s/%s), ' + 'retrying in %.0fs', + e.err_type, app_id, attempt, DOWNLOAD_MAX_ATTEMPTS, DOWNLOAD_RETRY_PAUSE, + ) + time.sleep(DOWNLOAD_RETRY_PAUSE) + raise last_error + def _download_app_int(self, download_path, app_id=None, bundle_id=None, country='US', file_name=None): if not app_id: logger.info(f'Trying to find app in App Store with bundle id {bundle_id}') @@ -161,41 +219,12 @@ def _download_app_int(self, download_path, app_id=None, bundle_id=None, country= app_id = app_info["trackId"] logger.info(f'Trying to purchase app with id {app_id}') - purchase_resp = self.store.purchase(app_id) - logger.debug( - 'Purchase response: status_code=%s, content_length=%s', - purchase_resp.status_code, - len(purchase_resp.content), - ) - if purchase_resp.status_code == 200: + if self.store.purchase(app_id): logger.info(f'App was successfully purchased for {self.apple_id} account') - elif purchase_resp.status_code == 500: - logger.info(f'This app was purchased before for {self.apple_id} account') else: - logger.warning( - 'Unexpected purchase response: status_code=%s, body_preview=%r', - purchase_resp.status_code, - (purchase_resp.text[:500] if purchase_resp.text else ''), - ) + logger.info(f'This app was purchased before for {self.apple_id} account') logger.info(f'Retrieving download info for app with id: {app_id}') - try: - download_resp = self.store.download(app_id) - except Exception as e: - logger.warning( - 'store.download failed: exception=%s, app_id=%s', - type(e).__name__, - app_id, - exc_info=True, - ) - raise - if not download_resp.songList: - logger.error( - 'Download response has no songList: cancel_purchase_batch=%s, ' - 'customerMessage=%r', - getattr(download_resp, 'cancel_purchase_batch', None), - getattr(download_resp, 'customerMessage', None), - ) - raise RuntimeError('Failed to get app download info! Check your parameters') + download_resp = self._download_info_with_retries(app_id) downloaded_app_info = download_resp.songList[0] logger.debug( @@ -224,7 +253,7 @@ def _download_app_int(self, download_path, app_id=None, bundle_id=None, country= f'Downloading app is {app_name} ({app_bundle_id}) with app_id {app_id} and version {app_version}') if not file_name: - file_name = '%s-%s.ipa' % (app_name, app_version) + file_name = '%s-%s.ipa' % (sanitize_file_name(app_name), app_version) else: file_name = '%s-%s.ipa' % (file_name, app_version) @@ -278,16 +307,20 @@ def _download_app_int(self, download_path, app_id=None, bundle_id=None, country= return file_path, md5 def download_app(self, download_path, app_id=None, bundle_id=None, country='US', file_name=None): - file_path, md5 = None, None - for force in (False, True): # try first time with possible stored session, second time with forced login + # A stored session is retried once with a forced re-login, but only when Apple + # says the session itself went stale. Forcing a re-login drops the session cache, + # and Apple's auth endpoint is currently unreliable enough that throwing away a + # working session over an unrelated error (a missing license, say) can leave the + # download unrecoverable. + for force in (False, True): try: self.login(force=force) - file_path, md5 = self._download_app_int(download_path, app_id, bundle_id, country, file_name) - break + return self._download_app_int(download_path, app_id, bundle_id, country, file_name) except StoreException as e: - if not self.login_by_session: # login by credentials, still with error - raise RuntimeError(f'Failed to download application. Seems like your app_id does not exist ' - f'or you did not purchase this paid app from apple account before. ' + session_is_stale = self.login_by_session and e.err_type in FAILURES_NEEDING_REAUTH + if not session_is_stale: + raise RuntimeError(f'Failed to download application from App Store. ' f'Message: {e.req} {e.err_msg} {e.err_type}') + logger.info('App Store session went stale (failureType %s), logging in again', e.err_type) - return file_path, md5 + return None, None diff --git a/mdast_cli/distribution_systems/appstore_client/store.py b/mdast_cli/distribution_systems/appstore_client/store.py index 519a2ad..79b2890 100755 --- a/mdast_cli/distribution_systems/appstore_client/store.py +++ b/mdast_cli/distribution_systems/appstore_client/store.py @@ -1,7 +1,10 @@ import hashlib import logging +import os import plistlib +import random import re +import time from typing import Optional import requests @@ -9,31 +12,85 @@ # Apple "bag" service: returns endpoint definitions (auth URL). Required since ~2025. BAG_URL_TEMPLATE = "https://init.itunes.apple.com/bag.xml?guid=%s" -# Apple's 26HOTFIX24 (June 2026) moved login to the native auth endpoint. The bag now -# advertises ".../auth/v1/native" (no "/fast"); the endpoint only works WITH the "/fast/" -# sub-path AND a trailing slash, otherwise Apple replies 301 + an HTML redirect page that -# breaks the plist parser. See majd/ipatool#486. +# Apple's bag advertises the native auth endpoint, which only answers correctly when the +# path ends with "/fast/" (trailing slash). Since July 2026 that endpoint also answers +# 204/403/404/503 with an empty, non-plist body for many clients; the legacy MZFinance +# endpoint still works, but replies 302 to an assigned pod host, and the original plist +# body (with attempt=1) has to be reposted there. See majd/ipatool#513 / PR #514. AUTH_HOST = "auth.itunes.apple.com" -NATIVE_FAST_PATH = "/auth/v1/native/fast/" -DEFAULT_AUTH_URL = "https://" + AUTH_HOST + NATIVE_FAST_PATH +DEFAULT_AUTH_URL = "https://" + AUTH_HOST + "/auth/v1/native/fast/" +LEGACY_AUTH_URL = "https://buy.itunes.apple.com/WebObjects/MZFinance.woa/wa/authenticate" +# Statuses that mean "this endpoint is not usable right now" when the body is not a plist. +# Apple's edge also emits bare 301/302 responses that carry no Location header at all, so +# the redirect statuses belong here too: without them a broken redirect aborts the login. +AUTH_FALLBACK_STATUSES = (204, 301, 302, 303, 307, 308, 403, 404, 429, 500, 502, 503) +# Apple answers this endpoint erratically and getting through is partly luck. Measured +# in August 2026: 80 closely spaced requests got 0 usable answers, another burst got in +# on the 34th, and a single request after two minutes of silence logged in immediately. +# Spacing attempts out is not a guarantee, but it reached a working login in ~10 requests +# where bursts needed dozens, so the backoff grows exponentially rather than hammering. +# Jitter keeps parallel CLI runs from lining up into a burst of their own. +AUTH_MAX_ROUNDS = int(os.environ.get("MDAST_APPSTORE_AUTH_ROUNDS", "8")) +AUTH_ROUND_BACKOFF = float(os.environ.get("MDAST_APPSTORE_AUTH_BACKOFF", "20")) +AUTH_MAX_BACKOFF = float(os.environ.get("MDAST_APPSTORE_AUTH_MAX_BACKOFF", "150")) +AUTH_BACKOFF_JITTER = 5.0 +AUTH_MAX_REDIRECTS = 4 BUY_DOMAIN = "buy.itunes.apple.com" -def _normalize_auth_endpoint(endpoint: Optional[str]) -> Optional[str]: - """Ensure the native auth endpoint carries the '/fast/' sub-path with a trailing slash. +def _normalize_auth_endpoint(endpoint): + """Add the trailing slash the native auth endpoint requires. - Apple's bag advertises ".../auth/v1/native"; without "/fast/" the request gets a - 301 + HTML redirect that the plist parser chokes on (majd/ipatool#486). + Apple's bag returns ".../auth/v1/native/fast"; posting without the trailing slash + gets a 301/204 with an HTML or empty body that the plist parser chokes on + (majd/ipatool#507). The legacy MZFinance endpoint is left untouched. """ - if endpoint and AUTH_HOST in endpoint: - if not (endpoint.endswith("/fast") or endpoint.endswith("/fast/")): - endpoint = endpoint.rstrip("/") + "/fast" - if not endpoint.endswith("/"): - endpoint = endpoint + "/" + if endpoint and "/native/" in endpoint and not endpoint.endswith("/"): + return endpoint + "/" return endpoint -PURCHASE_PATH = "/WebObjects/MZBuy.woa/wa/buyProduct" + + +class _AuthEndpointUnusable(Exception): + """Raised internally when an auth endpoint should be retried elsewhere.""" + + def __init__(self, status_code, detail=""): + self.status_code = status_code + self.detail = detail + super().__init__("auth endpoint unusable (HTTP %s) %s" % (status_code, detail)) + + +# buyProduct lives on MZFinance, not MZBuy: the MZBuy variant answers HTTP 200 with +# m-allowed=False / cancel-purchase-batch=True ("Unable to process your request.") for +# every app, so no license is ever created and the download that follows fails with +# failureType 9610. Verified against Apple in August 2026; ipatool uses the same path. +PURCHASE_PATH = "/WebObjects/MZFinance.woa/wa/buyProduct" DOWNLOAD_PATH = "/WebObjects/MZFinance.woa/wa/volumeStoreDownloadProduct" +# Apple failure types (mirrors ipatool pkg/appstore/constants.go). +FAILURE_INVALID_CREDENTIALS = '-5000' +FAILURE_DEVICE_VERIFICATION_FAILED = '1008' +FAILURE_PASSWORD_TOKEN_EXPIRED = '2034' +FAILURE_SIGN_IN_REQUIRED = '2042' +FAILURE_TEMPORARILY_UNAVAILABLE = '2059' +FAILURE_LICENSE_ALREADY_EXISTS = '5002' +FAILURE_LICENSE_NOT_FOUND = '9610' +# Failure types that mean "the session went stale, log in again and retry". +FAILURES_NEEDING_REAUTH = ( + FAILURE_DEVICE_VERIFICATION_FAILED, + FAILURE_PASSWORD_TOKEN_EXPIRED, + FAILURE_SIGN_IN_REQUIRED, +) +# 5002 is context dependent: on buyProduct it means the account already owns the app +# (success), while on volumeStoreDownloadProduct Apple throws it at random. Measured in +# August 2026 over one session: attempt 1 failed for two apps, attempts 2-4 succeeded for +# both, attempt 5 failed again for one of them - and a fresh login (13 minutes of auth +# backoff) did not clear it. So it is retried in place rather than treated as a stale +# session the way ipatool does (majd/ipatool#468); re-authenticating costs minutes and +# does not help. +DOWNLOAD_RETRY_FAILURES = (FAILURE_LICENSE_ALREADY_EXISTS,) +DOWNLOAD_MAX_ATTEMPTS = int(os.environ.get("MDAST_APPSTORE_DOWNLOAD_ATTEMPTS", "5")) +DOWNLOAD_RETRY_PAUSE = float(os.environ.get("MDAST_APPSTORE_DOWNLOAD_PAUSE", "6")) + from mdast_cli.distribution_systems.appstore_client.schemas.store_authenticate_req import StoreAuthenticateReq from mdast_cli.distribution_systems.appstore_client.schemas.store_authenticate_resp import StoreAuthenticateResp from mdast_cli.distribution_systems.appstore_client.schemas.store_buyproduct_req import StoreBuyproductReq @@ -154,70 +211,132 @@ def get_bag(self) -> str: logger.warning("Could not parse bag response, falling back to default auth URL") return DEFAULT_AUTH_URL - def authenticate(self, appleId, password): - if not self.guid: - self.guid = self._generateGuid(appleId) - auth_url = self.get_bag() - max_attempts = 4 + def _post_authenticate(self, url, appleId, password, attempt): + req = StoreAuthenticateReq( + appleId=appleId, + password=password, + attempt=str(attempt), + createSession=None, + guid=self.guid, + rmp='0', + why='signIn', + ) + return self.sess.post( + url, + headers={ + "Accept": "*/*", + "Content-Type": "application/x-www-form-urlencoded", + "User-Agent": APPSTORE_USER_AGENT, + }, + data=plistlib.dumps(req.as_dict()), + allow_redirects=False, + verify=False, + timeout=60, + ) + + def _authenticate_at(self, auth_url, appleId, password): + """Authenticate against a single endpoint, following Apple's pod redirect. + + The legacy endpoint answers 302 with a Location pointing at the account's pod + (e.g. https://p7-buy.itunes.apple.com/...?Pod=7&PRH=7). The original plist body + must be reposted there unchanged - in particular attempt stays 1, otherwise Apple + rejects the request (majd/ipatool#514). + """ + url = auth_url attempt = 1 + redirects = 0 r = None - while attempt <= max_attempts: - req = StoreAuthenticateReq( - appleId=appleId, - password=password, - attempt=str(attempt), - createSession=None, - guid=self.guid, - rmp='0', - why='signIn', - ) - r = self.sess.post( - auth_url, - headers={ - "Accept": "*/*", - "Content-Type": "application/x-www-form-urlencoded", - "User-Agent": APPSTORE_USER_AGENT, - }, - data=plistlib.dumps(req.as_dict()), - allow_redirects=False, - verify=False, - ) - if r.status_code == 302: - auth_url = r.headers.get('Location') - if not auth_url: - raise StoreException("authenticate", "Missing Location header on redirect", None) - attempt += 1 + while True: + r = self._post_authenticate(url, appleId, password, attempt) + if r.status_code in (301, 302, 303, 307, 308) and r.headers.get('Location'): + redirects += 1 + if redirects > AUTH_MAX_REDIRECTS: + raise _AuthEndpointUnusable(r.status_code, "too many redirects") + url = r.headers['Location'] + logger.debug("Auth redirected to pod endpoint: %s", url) + continue # attempt is intentionally NOT incremented here + try: + resp = StoreAuthenticateResp.from_dict(plistlib.loads(r.content)) + except plistlib.InvalidFileException: + _log_response_on_plist_error(r, "authenticate") + raise _AuthEndpointUnusable(r.status_code, "non-plist body") from None + if resp.m_allowed: + return r, resp + # Apple sometimes rejects the very first attempt with an invalid-credentials + # failure; a single retry with attempt=2 clears it (ipatool does the same). + if attempt == 1 and str(resp.failureType) == '-5000': + attempt = 2 continue - break - if r is None or r.status_code == 302: - raise StoreException("authenticate", "Too many redirects", None) - try: - resp = StoreAuthenticateResp.from_dict(plistlib.loads(r.content)) - except plistlib.InvalidFileException as e: - _log_response_on_plist_error(r, "authenticate") - raise StoreException( - "authenticate", - "Server response is not valid plist (possibly HTML error page or empty). See log for response details.", - None, - ) from e - if not resp.m_allowed: raise StoreException("authenticate", resp.customerMessage, resp.failureType) + def authenticate(self, appleId, password): + if not self.guid: + self.guid = self._generateGuid(appleId) + + endpoints = [] + for candidate in (self.get_bag(), DEFAULT_AUTH_URL, LEGACY_AUTH_URL): + if candidate and candidate not in endpoints: + endpoints.append(candidate) + + last_failure = None + for round_no in range(1, AUTH_MAX_ROUNDS + 1): + for auth_url in endpoints: + try: + r, resp = self._authenticate_at(auth_url, appleId, password) + self._store_auth_result(r, resp, auth_url) + return resp + except _AuthEndpointUnusable as e: + if e.status_code not in AUTH_FALLBACK_STATUSES: + raise StoreException( + "authenticate", + "Server response is not valid plist (HTTP %s). See log for details." + % e.status_code, + None, + ) from e + last_failure = e + logger.warning( + "Auth endpoint %s unusable (HTTP %s, %s), trying next endpoint", + auth_url, e.status_code, e.detail, + ) + if round_no < AUTH_MAX_ROUNDS: + delay = min(AUTH_ROUND_BACKOFF * (2 ** (round_no - 1)), AUTH_MAX_BACKOFF) + delay += random.uniform(0, AUTH_BACKOFF_JITTER) + logger.info( + "All App Store auth endpoints failed (round %s/%s), waiting %.0fs before " + "retrying - Apple's auth endpoint is erratic, spacing attempts out helps", + round_no, AUTH_MAX_ROUNDS, delay, + ) + time.sleep(delay) + + raise StoreException( + "authenticate", + "Apple rejected every authentication endpoint (last status: HTTP %s). " + "This is an Apple-side/network block rather than a credentials problem: " + "retry later or from a different egress IP (see majd/ipatool#513)." + % (last_failure.status_code if last_failure else "unknown"), + None, + ) + + def _store_auth_result(self, r, resp, auth_url): self.sess.headers['X-Dsid'] = self.sess.headers['iCloud-Dsid'] = str(resp.download_queue_info.dsid) - self.sess.headers['X-Apple-Store-Front'] = r.headers.get('x-set-apple-store-front') + store_front = r.headers.get('x-set-apple-store-front') + if store_front: + self.sess.headers['X-Apple-Store-Front'] = store_front + self.store_front = store_front self.sess.headers['X-Token'] = resp.passwordToken + self.dsid = resp.download_queue_info.dsid + pod_header = r.headers.get("pod") or r.headers.get("Pod") if pod_header: self.pod = pod_header.strip() else: - # Auth URL from bag may be e.g. https://p25-buy.itunes.apple.com/... — extract pod - match = re.search(r"^https?://p(\d+)-" + re.escape(BUY_DOMAIN), auth_url) + # The pod redirect lands on e.g. https://p7-buy.itunes.apple.com/...?Pod=7 + match = re.search(r"https?://p(\d+)-" + re.escape(BUY_DOMAIN), r.url or auth_url) self.pod = match.group(1) if match else None if self.pod: logger.debug("Using pod for buy host: %s", self.pod) self.account_name = resp.accountInfo.address.firstName + " " + resp.accountInfo.address.lastName - return resp def _buy_host(self) -> str: """Host for purchase/download (pod-specific if set).""" @@ -240,7 +359,13 @@ def find_app(self, app_id=None, bundle_id=None, country="US"): }, verify=False) - def purchase(self, app_id, productType='C'): + def purchase(self, app_id, productType='C', pricingParameters='STDQ'): + """Acquire a license for the app. + + Returns True when a new license was created, False when the account already + owned it. Raises StoreException when Apple refuses - notably the response is a + HTTP 200 either way, so the plist has to be inspected rather than the status. + """ url = "https://%s%s" % (self._buy_host(), PURCHASE_PATH) req = StoreBuyproductReq( guid=self.guid, @@ -249,23 +374,50 @@ def purchase(self, app_id, productType='C'): price='0', productType=productType, - pricingParameters='STDQ', + pricingParameters=pricingParameters, hasAskedToFulfillPreorder='true', buyWithoutAuthorization='true', hasDoneAgeCheck='true', ) - payload = req.as_dict() - return self.sess.post( + r = self.sess.post( url, headers={ "Content-Type": "application/x-apple-plist", "User-Agent": APPSTORE_USER_AGENT, }, - data=plistlib.dumps(payload), + data=plistlib.dumps(req.as_dict()), verify=False, + timeout=60, ) + logger.debug("buyProduct response: status=%s, content_length=%s", r.status_code, len(r.content)) + + try: + data = plistlib.loads(r.content) + except plistlib.InvalidFileException as e: + _log_response_on_plist_error(r, "buyProduct") + raise StoreException( + "buyProduct", "Server response is not valid plist. See log for response details.", None, + ) from e + + failure_type = str(data.get('failureType') or '') + message = data.get('customerMessage') or '' + + # Apple reports "already owned" either as failureType 5002 or as a bare HTTP 500. + if failure_type == FAILURE_LICENSE_ALREADY_EXISTS or r.status_code == 500: + logger.info('App is already licensed for this Apple ID') + return False + if failure_type or data.get('cancel-purchase-batch') or data.get('m-allowed') is False: + logger.warning( + "buyProduct rejected: failureType=%r, customerMessage=%r, app_id=%s", + failure_type, message, app_id, + ) + raise StoreException('buyProduct', message or 'failed to purchase app', failure_type or None) + if data.get('jingleDocType') != 'purchaseSuccess' or data.get('status') != 0: + raise StoreException('buyProduct', message or 'failed to purchase app', failure_type or None) + + return True def download(self, app_id, app_ver_id=""): req = StoreDownloadReq(creditDisplay="", guid=self.guid, salableAdamId=app_id, appExtVrsId=app_ver_id) @@ -294,7 +446,11 @@ def download(self, app_id, app_ver_id=""): "Server response is not valid plist. See log for response details.", None, ) from e - if resp.cancel_purchase_batch: + failure_type = str(resp.failureType or '') + # No songList means no download info, whatever the HTTP status says. Surface + # Apple's own failure type so callers can react: 9610 means the account holds no + # license for the app (buy it first), 1008/2034/2042 mean the session went stale. + if resp.cancel_purchase_batch or failure_type or not resp.songList: logger.warning( "App Store download rejected: customerMessage=%r, failureType=%r, app_id=%s", resp.customerMessage, @@ -302,7 +458,9 @@ def download(self, app_id, app_ver_id=""): app_id, ) raise StoreException( - "volumeStoreDownloadProduct", resp.customerMessage, resp.failureType + "volumeStoreDownloadProduct", + resp.customerMessage or 'Apple returned no download info for this app', + failure_type or None, ) return resp diff --git a/mdast_cli/mdast_scan.py b/mdast_cli/mdast_scan.py index f38a407..6183403 100644 --- a/mdast_cli/mdast_scan.py +++ b/mdast_cli/mdast_scan.py @@ -165,7 +165,7 @@ def parse_args(): appstore_group.add_argument('--appstore_bundle_id', type=str, help='Application Bundle ID (package identifier). ' 'Either --appstore_app_id or --appstore_bundle_id must be specified. ' - 'Example: com.instagram.ios, com.whatsapp.WhatsApp') + 'Example: com.burbn.instagram, net.whatsapp.WhatsApp') appstore_group.add_argument('--appstore_apple_id', type=str, help='Apple ID email address for iTunes/App Store login. ' 'Required parameter when --distribution_system is set to "appstore". ' diff --git a/tests/test_appstore_auth.py b/tests/test_appstore_auth.py new file mode 100644 index 0000000..e69e22c --- /dev/null +++ b/tests/test_appstore_auth.py @@ -0,0 +1,287 @@ +"""Regression tests for the App Store authentication flow. + +Apple broke the native auth endpoint in July 2026: it answers 204/403/404/503 with an +empty, non-plist body. The working flow (mirrored from majd/ipatool#514) is: + + native /auth/v1/native/fast/ -> 204/403/404/503 + legacy MZFinance authenticate -> 302 Location: https://pN-buy.itunes.apple.com/... + repost the SAME plist body (attempt still 1) to the pod URL -> 200 + plist +""" +import plistlib + +import pytest + +from mdast_cli.distribution_systems.appstore_client import store as store_mod +from mdast_cli.distribution_systems.appstore_client.store import ( + LEGACY_AUTH_URL, + StoreClient, + StoreException, + _normalize_auth_endpoint, +) + +POD_URL = "https://p7-buy.itunes.apple.com/WebObjects/MZFinance.woa/wa/authenticate?Pod=7&PRH=7" + +SUCCESS_PLIST = { + "m-allowed": True, + "passwordToken": "token-123", + "download-queue-info": {"dsid": 4242}, + "accountInfo": {"address": {"firstName": "Test", "lastName": "User"}}, +} + + +class FakeResponse: + def __init__(self, status_code, content=b"", headers=None, url=""): + self.status_code = status_code + self.content = content + self.headers = headers or {} + self.url = url + self.text = content.decode("utf-8", "replace") + + +class FakeSession: + """Records POSTs and replays a scripted list of responses.""" + + def __init__(self, responses): + self._responses = list(responses) + self.headers = {} + self.calls = [] + + def post(self, url, headers=None, data=None, **kwargs): + self.calls.append({"url": url, "body": plistlib.loads(data)}) + return self._responses.pop(0) + + +def _client(responses): + client = StoreClient(FakeSession(responses), guid="12367150C7F5") + return client + + +@pytest.fixture(autouse=True) +def _no_sleep(monkeypatch): + monkeypatch.setattr(store_mod.time, "sleep", lambda *_: None) + + +@pytest.fixture +def _bag_returns_legacy(monkeypatch): + monkeypatch.setattr(StoreClient, "get_bag", lambda self: LEGACY_AUTH_URL) + + +@pytest.mark.parametrize( + "endpoint,expected", + [ + ("https://auth.itunes.apple.com/auth/v1/native/fast", "https://auth.itunes.apple.com/auth/v1/native/fast/"), + ("https://auth.itunes.apple.com/auth/v1/native/fast/", "https://auth.itunes.apple.com/auth/v1/native/fast/"), + (LEGACY_AUTH_URL, LEGACY_AUTH_URL), + (None, None), + ], +) +def test_normalize_auth_endpoint(endpoint, expected): + assert _normalize_auth_endpoint(endpoint) == expected + + +def test_pod_redirect_reposts_body_with_attempt_one(_bag_returns_legacy): + """Apple rejects the pod repost if `attempt` is bumped, so it must stay 1.""" + client = _client([ + FakeResponse(302, headers={"Location": POD_URL}), + FakeResponse(200, plistlib.dumps(SUCCESS_PLIST), headers={"pod": "7"}, url=POD_URL), + ]) + + resp = client.authenticate("user@example.com", "secret123456") + + assert resp.passwordToken == "token-123" + assert client.account_name == "Test User" + assert client.pod == "7" + urls = [c["url"] for c in client.sess.calls] + assert urls == [LEGACY_AUTH_URL, POD_URL] + assert [c["body"]["attempt"] for c in client.sess.calls] == ["1", "1"] + assert client.sess.calls[0]["body"] == client.sess.calls[1]["body"] + + +@pytest.mark.parametrize("status", [204, 403, 404, 503]) +def test_falls_back_to_next_endpoint_on_empty_body(monkeypatch, status): + """A non-plist body on the bag endpoint must not abort the login.""" + monkeypatch.setattr(StoreClient, "get_bag", lambda self: "https://auth.itunes.apple.com/auth/v1/native/fast/") + client = _client([ + FakeResponse(status, b""), + FakeResponse(302, headers={"Location": POD_URL}), + FakeResponse(200, plistlib.dumps(SUCCESS_PLIST), url=POD_URL), + ]) + + client.authenticate("user@example.com", "secret123456") + + urls = [c["url"] for c in client.sess.calls] + assert urls == ["https://auth.itunes.apple.com/auth/v1/native/fast/", LEGACY_AUTH_URL, POD_URL] + assert client.pod == "7" # derived from the pod URL when the header is absent + + +def test_invalid_credentials_are_reported_not_retried_forever(_bag_returns_legacy): + failure = {"m-allowed": False, "customerMessage": "Your Apple ID or password was incorrect.", + "failureType": "1234"} + client = _client([FakeResponse(200, plistlib.dumps(failure))]) + + with pytest.raises(StoreException) as exc: + client.authenticate("user@example.com", "wrong") + + assert "incorrect" in str(exc.value) + assert len(client.sess.calls) == 1 + + +def test_first_attempt_invalid_credentials_is_retried_once(_bag_returns_legacy): + """Apple spuriously fails attempt 1 with -5000; attempt 2 clears it.""" + failure = {"m-allowed": False, "customerMessage": "retry", "failureType": "-5000"} + client = _client([ + FakeResponse(200, plistlib.dumps(failure)), + FakeResponse(200, plistlib.dumps(SUCCESS_PLIST)), + ]) + + client.authenticate("user@example.com", "secret123456") + + assert [c["body"]["attempt"] for c in client.sess.calls] == ["1", "2"] + + +def test_all_endpoints_blocked_raises_actionable_error(_bag_returns_legacy): + client = _client([FakeResponse(403, b"") for _ in range(2 * store_mod.AUTH_MAX_ROUNDS)]) + + with pytest.raises(StoreException) as exc: + client.authenticate("user@example.com", "secret123456") + + assert "Apple-side/network block" in str(exc.value) + + +@pytest.mark.parametrize("status", [301, 302]) +def test_redirect_without_location_is_retried(_bag_returns_legacy, status): + """Apple's edge emits bare 30x responses with no Location; that must not abort login.""" + client = _client([ + FakeResponse(status, b""), # bag/legacy endpoint, broken redirect + FakeResponse(status, b""), # native endpoint, same + FakeResponse(302, headers={"Location": POD_URL}), # next round: real redirect + FakeResponse(200, plistlib.dumps(SUCCESS_PLIST), url=POD_URL), + ]) + + client.authenticate("user@example.com", "secret123456") + + assert [c["url"] for c in client.sess.calls][-1] == POD_URL + + +# --- purchase / download ----------------------------------------------------------- +# buyProduct on MZBuy answers HTTP 200 with m-allowed=False for every app, so no license +# is ever created and the download that follows fails with failureType 9610. The license +# only gets created on the MZFinance path. + +def _authed_client(responses): + client = _client(responses) + client.pod = "12" + return client + + +def test_purchase_uses_mzfinance_path(): + client = _authed_client([FakeResponse(200, plistlib.dumps({"jingleDocType": "purchaseSuccess", "status": 0}))]) + + assert client.purchase("310633997") is True + assert client.sess.calls[0]["url"] == "https://p12-buy.itunes.apple.com/WebObjects/MZFinance.woa/wa/buyProduct" + + +def test_purchase_reports_already_owned(): + client = _authed_client([FakeResponse(200, plistlib.dumps({"failureType": "5002", + "customerMessage": "An unknown error has occurred"}))]) + + assert client.purchase("310633997") is False + + +def test_purchase_rejection_is_not_reported_as_success(): + """HTTP 200 + m-allowed=False is a refusal; the old code logged it as a success.""" + refusal = {"failureType": "", "m-allowed": False, "cancel-purchase-batch": True, + "customerMessage": "Unable to process your request."} + client = _authed_client([FakeResponse(200, plistlib.dumps(refusal))]) + + with pytest.raises(StoreException) as exc: + client.purchase("310633997") + + assert "Unable to process your request." in str(exc.value) + + +def test_download_without_songlist_surfaces_failure_type(): + client = _authed_client([FakeResponse(200, plistlib.dumps({"failureType": "9610", + "customerMessage": "License not found."}))]) + + with pytest.raises(StoreException) as exc: + client.download("284882215") + + assert exc.value.err_type == "9610" + assert "License not found." in str(exc.value) + + +def test_download_returns_song_list(): + payload = {"songList": [{"songId": 1, "URL": "https://example.invalid/app.ipa", "md5": "abc", + "metadata": {"bundleDisplayName": "App", "bundleShortVersionString": "1.0", + "softwareVersionBundleId": "com.example.app"}}]} + client = _authed_client([FakeResponse(200, plistlib.dumps(payload))]) + + resp = client.download("389801252") + + assert len(resp.songList) == 1 + assert client.sess.calls[0]["url"].startswith( + "https://p12-buy.itunes.apple.com/WebObjects/MZFinance.woa/wa/volumeStoreDownloadProduct?guid=") + + +def test_5002_means_already_owned_on_purchase_but_a_random_failure_on_download(): + """Apple reuses failureType 5002 for two different things; the fix must not conflate them.""" + from mdast_cli.distribution_systems.appstore_client.store import ( + DOWNLOAD_RETRY_FAILURES, FAILURES_NEEDING_REAUTH, FAILURE_LICENSE_ALREADY_EXISTS) + + # Not a stale session: a fresh login does not clear it, so re-authenticating is wasted. + assert FAILURE_LICENSE_ALREADY_EXISTS not in FAILURES_NEEDING_REAUTH + assert FAILURE_LICENSE_ALREADY_EXISTS in DOWNLOAD_RETRY_FAILURES + + client = _authed_client([FakeResponse(200, plistlib.dumps({"failureType": "5002"}))]) + assert client.purchase("389801252") is False # purchase: already owned, not an error + + +def test_download_info_retries_through_random_5002(monkeypatch): + """Apple throws 5002 at random; repeating the same request clears it.""" + from mdast_cli.distribution_systems import appstore as appstore_mod + + monkeypatch.setattr(appstore_mod.time, "sleep", lambda *_: None) + store = _authed_client([ + FakeResponse(200, plistlib.dumps({"failureType": "5002"})), + FakeResponse(200, plistlib.dumps({"failureType": "5002"})), + FakeResponse(200, plistlib.dumps({"songList": [{"songId": 1, "URL": "https://x.invalid", + "md5": "abc", "metadata": {}}]})), + ]) + app = appstore_mod.AppStore.__new__(appstore_mod.AppStore) + app.store = store + + resp = app._download_info_with_retries("389801252") + + assert len(resp.songList) == 1 + assert len(store.sess.calls) == 3 # two failures ridden out, no re-login + + +def test_download_info_buys_a_missing_license_then_retries(monkeypatch): + from mdast_cli.distribution_systems import appstore as appstore_mod + + monkeypatch.setattr(appstore_mod.time, "sleep", lambda *_: None) + store = _authed_client([ + FakeResponse(200, plistlib.dumps({"failureType": "9610", "customerMessage": "License not found."})), + FakeResponse(200, plistlib.dumps({"jingleDocType": "purchaseSuccess", "status": 0})), + FakeResponse(200, plistlib.dumps({"songList": [{"songId": 1, "URL": "https://x.invalid", + "md5": "abc", "metadata": {}}]})), + ]) + app = appstore_mod.AppStore.__new__(appstore_mod.AppStore) + app.store = store + + resp = app._download_info_with_retries("284882215") + + assert len(resp.songList) == 1 + assert "buyProduct" in store.sess.calls[1]["url"] + + +@pytest.mark.parametrize("raw,expected", [ + ("‎WhatsApp", "WhatsApp"), # Apple ships a bidi mark in bundleDisplayName + ("Instagram", "Instagram"), + ("Foo/Bar", "FooBar"), # a separator would redirect the download path + ("‎", "application"), # nothing printable left +]) +def test_sanitize_file_name(raw, expected): + from mdast_cli.distribution_systems.appstore import sanitize_file_name + assert sanitize_file_name(raw) == expected