From a21dfdcbd4618c90c9eb146d004b88dba304b43a Mon Sep 17 00:00:00 2001 From: assafw <13516529+assafw@users.noreply.github.com> Date: Sun, 7 Jun 2026 18:53:44 +1000 Subject: [PATCH] fix: run blocking requests download in thread pool to unfreeze parallel downloads fast_async_download was using the synchronous requests library directly on the event loop, blocking all concurrent downloads whenever a connection was active. Offload to run_in_executor so the event loop stays free, and dispatch progress callbacks via call_soon_threadsafe to keep rich's Live display thread-safe. Co-Authored-By: Claude Sonnet 4.6 --- streamrip/client/downloadable.py | 33 +++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/streamrip/client/downloadable.py b/streamrip/client/downloadable.py index 3b29b504..584e172e 100644 --- a/streamrip/client/downloadable.py +++ b/streamrip/client/downloadable.py @@ -37,18 +37,18 @@ def generate_temp_path(url: str): ) -async def fast_async_download(path, url, headers, callback): - """Synchronous download with yield for every 1MB read. +def _do_download(path, url, headers, callback, loop): + """Blocking download using requests. Runs in a thread pool via fast_async_download. + + Large chunk size avoids the CPU-bound problem caused by yielding to the event loop + on every small read (the original aiohttp/aiofiles approach capped total speed at ~10MB/s). - Using aiofiles/aiohttp resulted in a yield to the event loop for every 1KB, - which made file downloads CPU-bound. This resulted in a ~10MB max total download - speed. This fixes the issue by only yielding to the event loop for every 1MB read. + callback is dispatched back onto the event loop (thread-safe) since rich's Live + display must only be updated from the main thread. """ chunk_size: int = 2**17 # 131 KB - counter = 0 - yield_every = 8 # 1 MB - with open(path, "wb") as file: # noqa: ASYNC101 - with requests.get( # noqa: ASYNC100 + with open(path, "wb") as file: + with requests.get( url, headers=headers, allow_redirects=True, @@ -56,10 +56,17 @@ async def fast_async_download(path, url, headers, callback): ) as resp: for chunk in resp.iter_content(chunk_size=chunk_size): file.write(chunk) - callback(len(chunk)) - if counter % yield_every == 0: - await asyncio.sleep(0) - counter += 1 + loop.call_soon_threadsafe(callback, len(chunk)) + + +async def fast_async_download(path, url, headers, callback): + """Run the blocking download in a thread pool so the event loop stays free. + + Keeping requests + large chunks for throughput, but offloading to a thread + so concurrent downloads are not frozen while one connection is active. + """ + loop = asyncio.get_event_loop() + await loop.run_in_executor(None, _do_download, path, url, headers, callback, loop) @dataclass(slots=True)