diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a478f2..49f452f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: python -m pip install .[test] pip list - name: Test with pytest - run: pytest -svv --cov=blender_downloader --cov-config=setup.cfg + run: pytest -svv #- name: Coveralls #run: coveralls #env: diff --git a/.gitignore b/.gitignore index f3807fc..7eab415 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ blender-* *.img Blender .vscode +.coverage +htmlcov diff --git a/blender_downloader/__init__.py b/blender_downloader/__init__.py index 7e2341f..0878b9d 100644 --- a/blender_downloader/__init__.py +++ b/blender_downloader/__init__.py @@ -8,6 +8,7 @@ import os import re import shutil +import subprocess import sys import tarfile import tempfile @@ -31,13 +32,18 @@ BLENDER_MANUAL_VERSIONS_URL = "https://docs.blender.org/PROD/versions.json" BLENDER_DAILY_BUILDS_URL = "https://builder.blender.org/download/daily/" MINIMUM_VERSION_SUPPPORTED = "2.57" -SUPPORTED_FILETYPES_EXTRACTION = [".bz2", ".gz", ".xz", ".zip", ".dmg"] NIGHLY_RELEASES_CACHE_EXPIRATION = 60 * 60 * 24 # 1 day CACHE = Cache( user_data_dir(appname=__title__, appauthor=__author__, version=__version__) ) +def removesuffix(string, suffix): # polyfill for Python < 3.9 + if string.endswith(suffix): + return string[: -len(suffix)] + return string + + class BlenderVersion: """Blender versions object for easy comparations support.""" @@ -104,6 +110,23 @@ def GET(url, expire=259200, use_cache=True): # 3 days for expiration return response.decode("utf-8") +def get_toplevel_dirnames_from_paths(paths): + """Extracts the names of the directories in the top level + from a set of paths constituting multiple directory trees. + """ + toplevel_dirnames = [] + for path in paths: + # not optimal implementation, but crossplatform for sure + parent, _ = os.path.split(path) + is_file = parent == "" + if not is_file: + while parent: + previous_parent = parent + parent, _ = os.path.split(previous_parent) + toplevel_dirnames.append(previous_parent) + return toplevel_dirnames + + def build_parser(): parser = argparse.ArgumentParser(description=__description__) parser.add_argument( @@ -742,13 +765,6 @@ def extract_release(zipped_filepath, quiet=False): output_directory = os.path.abspath(os.path.dirname(zipped_filepath)) extension = os.path.splitext(zipped_filepath)[1] - if extension not in SUPPORTED_FILETYPES_EXTRACTION: - sys.stderr.write( - f"Blender compressed release file '{zipped_filename}' extraction" - " is not supported by blender-downloader.\n" - ) - sys.exit(1) - # filepath of the extracted directory, don't confuse it with # `output_directory`, that is the directory where the file to extract # is located @@ -760,10 +776,32 @@ def extract_release(zipped_filepath, quiet=False): with zipfile.ZipFile(zipped_filepath, "r") as f: namelist = f.namelist() - extracted_directory_filepath = os.path.join( - output_directory, - namelist[0].split(os.sep)[0], - ) + + # ensure that Blender is extracted in a top level directory + # + # MacOS versions previous to 2.79 are released in a ZIP file + # with multiple directories and files in the root, which results + # in a messy extraction with a lot of files in the current + # working directory + root_dirnames = get_toplevel_dirnames_from_paths(namelist) + + # `not root_dirnames` when only files are found in the ZIP + if len(root_dirnames) > 1 or not root_dirnames: + output_directory = os.path.join(output_directory, "Blender") + else: + output_directory = os.path.join(output_directory, root_dirnames[0]) + + # don't overwrite existing non empty directory extracting + if os.path.isdir(output_directory) and os.listdir(output_directory): + sys.stderr.write( + f"The directory '{output_directory}' where the files will" + " be extracted already exists and is not empty. Extraction" + " skipped.\n" + ) + sys.exit(1) + + extracted_directory_filepath = output_directory + progress_bar_kwargs = dict( total=len(namelist), desc=f"Extracting '{zipped_filename}'", @@ -772,58 +810,117 @@ def extract_release(zipped_filepath, quiet=False): ) for file in tqdm(**progress_bar_kwargs): f.extract(member=file, path=output_directory) - elif extension in [".bz2", ".gz", ".xz"]: + + elif extension in [".bz2", ".gz", ".xz", ".tar.gz", ".tar.bz2", ".tar.xz"]: if not quiet: sys.stderr.write(f"Decompressing '{zipped_filename}'...\n") with tarfile.open(zipped_filepath, "r") as f: - members = f.getmembers() + files = f.getmembers() + paths = [file.name for file in files] + + root_dirnames = get_toplevel_dirnames_from_paths(paths) + + if len(root_dirnames) > 1 or not root_dirnames: + output_directory = os.path.join(output_directory, "Blender") + else: + output_directory = os.path.join(output_directory, root_dirnames[0]) + + extracted_directory_filepath = output_directory - extracted_directory_filepath = os.path.join( - output_directory, - members[0].name.split(os.sep)[0], - ) progress_bar_kwargs = dict( - total=len(members), + total=len(files), desc=f"Extracting '{zipped_filename}'", - iterable=members, + iterable=files, disable=quiet, ) for file in tqdm(**progress_bar_kwargs): f.extract(member=file, path=output_directory) - else: # extension == ".dmg": + + elif extension == ".dmg": running_os = get_running_os() if running_os != "macos": - sys.stderr.write( - "blender-downloader can't mount MacOS '.dmg' image files like" - f" '{extracted_directory_filepath}' in" - f" {running_os.capitalize()}, so you should install Blender" - " manually.\n" - ) - sys.exit(1) + # we are not in MacOS, so we need the binaries dmg2img and 7z + # to decompress the `.dmg` file downloaded + dmg2img, sevenz = shutil.which("dmg2img"), shutil.which("7z") + if dmg2img is None or sevenz is None: + required_programs = [] + if dmg2img is None: + required_programs.append("'dmg2img'") + if sevenz is None: + required_programs.append("'7z'") + plural_suffix = "s" if len(required_programs) > 1 else "" + sys.stderr.write( + f"You need to install the program{plural_suffix}" + f" {' and '.join(required_programs)} to extract the" + f" DMG Blender release located at {zipped_filepath}" + f" inside a {running_os.capitalize()} platform.\n" + ) + sys.exit(1) - extracted_directory_filepath = os.path.join( - output_directory, os.path.basename(zipped_filepath).rstrip(".dmg") - ) + img_filepath = removesuffix(zipped_filepath, "dmg") + "img" + if os.path.isfile(img_filepath): + os.remove(img_filepath) - import dmglib - - with dmglib.attachedDiskImage(zipped_filepath) as mounted_dmg: - contents_parent_dirpath = None - for dirpath, _, _ in os.walk(mounted_dmg[0]): - if ( - os.path.basename(dirpath) == "Contents" - and "blender.app" in dirpath.lower() - ): - contents_parent_dirpath = os.path.abspath( - os.path.dirname(dirpath), - ) - break - shutil.copytree( - contents_parent_dirpath, - extracted_directory_filepath, + dmg2img_proc = subprocess.Popen( + ["dmg2img", zipped_filepath], + stderr=sys.stderr, + stdout=sys.stdout, + env=os.environ, + ) + dmg2img_proc.communicate() + if dmg2img_proc.returncode != 0: + sys.exit(dmg2img_proc.returncode) + + seven7_proc = subprocess.Popen( + ["7z", "x", img_filepath], + stderr=sys.stderr, + stdout=sys.stdout, + env=os.environ, + ) + seven7_proc.communicate() + if seven7_proc.returncode != 0: + sys.exit(seven7_proc.returncode) + + extracted_directory_filepath = os.path.join(output_directory, "Blender") + else: + # we are inside MacOS, use the DMG CLI utility included in + # the system through the dmglib Python wrapper + extracted_directory_filepath = os.path.join( + output_directory, + removesuffix(os.path.basename(zipped_filepath), ".dmg"), ) + import dmglib + + with dmglib.attachedDiskImage(zipped_filepath) as mounted_dmg: + contents_parent_dirpath = None + for dirpath, _, _ in os.walk(mounted_dmg[0]): + if ( + os.path.basename(dirpath) == "Contents" + and "blender.app" in dirpath.lower() + ): + contents_parent_dirpath = os.path.abspath( + os.path.dirname(dirpath), + ) + break + shutil.copytree( + contents_parent_dirpath, + extracted_directory_filepath, + ) + else: + if "-e" in sys.argv and "--extract" not in sys.argv: + extract_option = "-e" + elif "-e" not in sys.argv and "--extract" in sys.argv: + extract_option = "--extract" + else: + extract_option = "-e/--extract" + sys.stderr.write( + f"File extension '{extension}' extraction not supported by" + f" '{extract_option}' command line option.\n" + ) + sys.exit(1) + return extracted_directory_filepath @@ -1156,7 +1253,7 @@ def run(args=[]): return 0 -def main(): +def main(): # pragma: no cover sys.exit(run(args=sys.argv[1:])) diff --git a/setup.cfg b/setup.cfg index d6ccde4..34ccf4a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -57,6 +57,9 @@ test = pytest==6.2.5 pytest-cov==3.0.0 +[tool:pytest] +addopts = --cov=blender_downloader --cov-report=html --cov-config=setup.cfg + [coverage:report] exclude_lines = pragma: no cover diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..a436452 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,7 @@ +import os +import sys + + +TESTS_DIR = os.path.abspath(os.path.dirname(__file__)) +if TESTS_DIR not in sys.path: + sys.path.append(TESTS_DIR) diff --git a/tests/test_extract_release.py b/tests/test_extract_release.py index e30c7bd..3478503 100644 --- a/tests/test_extract_release.py +++ b/tests/test_extract_release.py @@ -3,99 +3,100 @@ import contextlib import io import os -import shutil import tarfile -import tempfile import uuid import zipfile import pytest +from testing_utils import SUPPORTED_EXTENSIONS_FOR_EXTRACTION + +from blender_downloader import extract_release -from blender_downloader import ( - SUPPORTED_FILETYPES_EXTRACTION, - extract_release, - get_running_os, -) +# TODO: test when 'root_dirnames == 1' +# TODO: test when 'root_dirnames > 1' -# NOTE: MacOS '.dmg' not tested -EXTENSIONS = [".zip", ".tar.bz2", ".tar.gz", ".tar.xz", ".fakeextension"] +# TODO: MacOS '.dmg' not tested -def create_zipped_file_by_extension(extension): +def create_zipped_file_by_extension(tmp_path, extension, files): fake_release_zipped_filepath = os.path.join( - tempfile.gettempdir(), + tmp_path, f"{uuid.uuid4().hex}{extension}", ) - _f = tempfile.NamedTemporaryFile("w", delete=False) - _f.write("foo\n") - f = open(_f.name) - _f.close() + def files_generator(): + for fname in files: + with open(os.path.join(tmp_path, f"{fname}.txt"), "w") as f: + f.write(f"{fname}\n") + filepath = f.name + yield filepath if extension == ".zip": with zipfile.ZipFile(fake_release_zipped_filepath, "w") as zipf: - zipf.write(f.name) + for filepath in files_generator(): + zipf.write(filepath, os.path.relpath(filepath, tmp_path)) elif extension in [".tar.bz2", ".tar.gz", ".tar.xz"]: format = extension.split(".")[2] with tarfile.open(fake_release_zipped_filepath, f"w:{format}") as zipf: - zipf.add(f.name) - return (fake_release_zipped_filepath, f) + for filepath in files_generator(): + zipf.add(filepath, os.path.relpath(filepath, tmp_path)) + else: + raise NotImplementedError( + f"Tests for extraction of files with '{extension}' are not implemented" + ) + return fake_release_zipped_filepath -@pytest.mark.skipif( - get_running_os() == "macos", - reason="MacOS extraction not covered by 'test_extract_release' test.", +@pytest.mark.parametrize( + "expected_files", + (["foo"], ["foo", "bar"]), + ids=('["foo.txt"]', '["foo.txt","bar.txt"]'), +) +@pytest.mark.parametrize( + "extension", SUPPORTED_EXTENSIONS_FOR_EXTRACTION[4:] + [".fakeextension"] ) -@pytest.mark.parametrize("extension", EXTENSIONS) @pytest.mark.parametrize("quiet", (True, False)) -def test_extract_release(extension, quiet): - if quiet is True: - return - mocked_stderr = io.StringIO() - if ("." + extension.split(".")[-1]) not in SUPPORTED_FILETYPES_EXTRACTION: +def test_extract_release(expected_files, extension, quiet, tmp_path): + stderr = io.StringIO() + if extension not in SUPPORTED_EXTENSIONS_FOR_EXTRACTION: with pytest.raises(SystemExit): - with contextlib.redirect_stderr(mocked_stderr): + with contextlib.redirect_stderr(stderr): extract_release(f"foo{extension}", quiet=quiet) - assert mocked_stderr.getvalue() == ( - f"Blender compressed release file 'foo{extension}' extraction is" - " not supported by blender-downloader.\n" + assert stderr.getvalue() == ( + f"File extension '{extension}' extraction not supported by" + " '-e/--extract' command line option.\n" ) return - fake_release_zipped_filepath, content_f = create_zipped_file_by_extension( + fake_release_zipped_filepath = create_zipped_file_by_extension( + tmp_path, extension, + expected_files, ) - attempt = 0 - while attempt < 2: - with contextlib.redirect_stderr(mocked_stderr): - directory_filepath = extract_release( - fake_release_zipped_filepath, - quiet=quiet, - ) - - assert os.path.isdir(directory_filepath) - files = os.listdir(directory_filepath) - if len(files) > 1: - shutil.rmtree(directory_filepath) - attempt += 1 - else: - break - assert len(files) == 1 - with open(os.path.join(directory_filepath, files[0])) as f: - assert f.read() == content_f.read() - - if quiet is False: - stderr_lines = mocked_stderr.getvalue().splitlines() - fake_release_zipped_filename = os.path.basename(fake_release_zipped_filepath) - assert stderr_lines[0] == f"Decompressing '{fake_release_zipped_filename}'..." - assert stderr_lines[2].startswith( - f"Extracting '{fake_release_zipped_filename}': ", + with contextlib.redirect_stderr(stderr): + directory_filepath = extract_release( + fake_release_zipped_filepath, + quiet=quiet, ) + assert directory_filepath == os.path.join(tmp_path, "Blender") + + output_filenames = os.listdir(directory_filepath) + assert len(output_filenames) == len(expected_files) + + expected_filenames = [f"{f}.txt" for f in expected_files] - # cleanup - os.remove(fake_release_zipped_filepath) - content_f.close() - os.remove(content_f.name) - shutil.rmtree(directory_filepath) + for filename in output_filenames: + assert filename in expected_filenames + + filepath = os.path.join(directory_filepath, filename) + with open(filepath) as f: + content = f.read() + + expected_content = filename.replace(".txt", "") + "\n" + assert content == expected_content + if quiet is False: + stderr_lines = stderr.getvalue().splitlines() + assert stderr_lines[0].startswith("Decompressing") + assert stderr_lines[2].startswith("Extracting") diff --git a/tests/test_get_legacy_release_download_url.py b/tests/test_get_legacy_release_download_url.py index 3c6fa94..49dc206 100644 --- a/tests/test_get_legacy_release_download_url.py +++ b/tests/test_get_legacy_release_download_url.py @@ -8,10 +8,10 @@ from urllib.request import urlsplit import pytest +from testing_utils import SUPPORTED_EXTENSIONS_FOR_EXTRACTION from blender_downloader import ( MINIMUM_VERSION_SUPPPORTED, - SUPPORTED_FILETYPES_EXTRACTION, BlenderVersion, get_legacy_release_download_url, ) @@ -203,4 +203,4 @@ def assert_url(url_end_schema): # check that filetype is supported for extraction extension = os.path.splitext(os.path.basename(urlsplit(url).path))[1] - assert extension in SUPPORTED_FILETYPES_EXTRACTION + assert extension in SUPPORTED_EXTENSIONS_FOR_EXTRACTION diff --git a/tests/testing_utils.py b/tests/testing_utils.py new file mode 100644 index 0000000..c97dd6e --- /dev/null +++ b/tests/testing_utils.py @@ -0,0 +1,10 @@ +SUPPORTED_EXTENSIONS_FOR_EXTRACTION = [ + ".dmg", + ".bz2", + ".gz", + ".xz", + ".zip", + ".tar.bz2", + ".tar.gz", + ".tar.xz", +]