Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Remove the default download timeout of 5 minutes for a single file #81

Merged
merged 1 commit into from
Jan 5, 2022
Merged

Remove the default download timeout of 5 minutes for a single file #81

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 15 additions & 14 deletions parfive/downloader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,9 +265,9 @@ async def run_download(self, timeouts=None):
----------
timeouts : `dict`, optional
Overrides for the default timeouts for http downloads. Supported
keys are any accepted by the `aiohttp.ClientTimeout` class. Defaults
to 5 minutes for total session timeout and 90 seconds for socket
read timeout.
keys are any accepted by the `aiohttp.ClientTimeout` class.
Defaults to no timeout for total session timeout (overriding the
aiohttp 5 minute default) and 90 seconds for socket read timeout.

Returns
-------
Expand All @@ -284,7 +284,7 @@ async def run_download(self, timeouts=None):
if "PARFIVE_DEBUG" in os.environ:
self._configure_debug()

timeouts = timeouts or {"total": float(os.environ.get("PARFIVE_TOTAL_TIMEOUT", 5 * 60)),
timeouts = timeouts or {"total": float(os.environ.get("PARFIVE_TOTAL_TIMEOUT", 0)),
"sock_read": float(os.environ.get("PARFIVE_SOCK_READ_TIMEOUT", 90))}

total_files = self.queued_downloads
Expand Down Expand Up @@ -322,9 +322,9 @@ def download(self, timeouts=None):
----------
timeouts : `dict`, optional
Overrides for the default timeouts for http downloads. Supported
keys are any accepted by the `aiohttp.ClientTimeout` class. Defaults
to 5 minutes for total session timeout and 90 seconds for socket
read timeout.
keys are any accepted by the `aiohttp.ClientTimeout` class.
Defaults to no timeout for total session timeout (overriding the
aiohttp 5 minute default) and 90 seconds for socket read timeout.

Returns
-------
Expand Down Expand Up @@ -553,13 +553,14 @@ async def _get_http(self, session, *, url, filepath_partial, chunksize=None,
session, url, chunksize, None, timeout, downloaded_chunk_queue, **kwargs
))
)
# Close the initial request here before we start transferring data.

# run all the download workers
await asyncio.gather(*download_workers)
# join() waits till all the items in the queue have been processed
await downloaded_chunk_queue.join()
writer.cancel()
return str(filepath)
# run all the download workers
await asyncio.gather(*download_workers)
# join() waits till all the items in the queue have been processed
await downloaded_chunk_queue.join()
writer.cancel()
return str(filepath)

except Exception as e:
raise FailedDownload(filepath_partial, url, e)
Expand Down Expand Up @@ -650,7 +651,7 @@ async def _http_download_worker(self, session, url, chunksize, http_range, timeo
offset = 0

async with session.get(url, timeout=timeout, headers=headers, **kwargs) as resp:
parfive.log.debug("%s request made to %s with headers=%s",
parfive.log.debug("%s request made for download to %s with headers=%s",
resp.request_info.method,
resp.request_info.url,
resp.request_info.headers)
Expand Down
2 changes: 1 addition & 1 deletion parfive/tests/test_downloader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -460,7 +460,7 @@ def test_proxy_passed_as_kwargs_to_get(tmpdir, url, proxy):
assert patched.called, "`ClientSession._request` not called"
assert list(patched.call_args) == [('GET', url),
{'allow_redirects': True,
'timeout': ClientTimeout(total=300, connect=None, sock_read=90, sock_connect=None),
'timeout': ClientTimeout(total=0, connect=None, sock_read=90, sock_connect=None),
'proxy': proxy
}]

Expand Down