Skip to content

Commit

Permalink
fix wget download (#59957)
Browse files Browse the repository at this point in the history
* fix wget download

* fix wget
  • Loading branch information
lyuwenyu authored Dec 15, 2023
1 parent e64a054 commit c5f6862
Showing 1 changed file with 24 additions and 14 deletions.
38 changes: 24 additions & 14 deletions python/paddle/utils/download.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import tarfile
import time
import zipfile
from urllib.parse import urlparse

import httpx

Expand Down Expand Up @@ -196,22 +197,31 @@ def _get_download(url, fullname):
return False


def _wget_download(url, fullname):
# using wget to download url
tmp_fullname = fullname + "_tmp"
# –user-agent
command = f'wget -O {tmp_fullname} -t {DOWNLOAD_RETRY_LIMIT} {url}'
subprc = subprocess.Popen(
command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
_ = subprc.communicate()

if subprc.returncode != 0:
raise RuntimeError(
f'{command} failed. Please make sure `wget` is installed or {url} exists'
def _wget_download(url: str, fullname: str):
try:
assert urlparse(url).scheme in (
'http',
'https',
), 'Only support https and http url'
# using wget to download url
tmp_fullname = fullname + "_tmp"
# –user-agent
command = f'wget -O {tmp_fullname} -t {DOWNLOAD_RETRY_LIMIT} {url}'
subprc = subprocess.Popen(
command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
_ = subprc.communicate()

if subprc.returncode != 0:
raise RuntimeError(
f'{command} failed. Please make sure `wget` is installed or {url} exists'
)

shutil.move(tmp_fullname, fullname)

shutil.move(tmp_fullname, fullname)
except Exception as e: # requests.exceptions.ConnectionError
logger.info(f"Downloading {url} failed with exception {str(e)}")
return False

return fullname

Expand Down

0 comments on commit c5f6862

Please # to comment.