Crossplatform MacOS releases extraction (#52)

* Allow to extract release MacOS 2.79

* Handle unknown file extension in extraction

* Ensure always a top level directory when extracting

* Update tests

* Improve code coverage

* Exclude coverage for main function

* Support extraction of DMG outside MacOS

* Add useful comment

.github/workflows/ci.yml

@@ -36,7 +36,7 @@ jobs:
           python -m pip install .[test]
           pip list
       - name: Test with pytest
-        run: pytest -svv --cov=blender_downloader --cov-config=setup.cfg
+        run: pytest -svv
       #- name: Coveralls
         #run: coveralls
         #env:

.gitignore

@@ -10,3 +10,5 @@ blender-*
 *.img
 Blender
 .vscode
+.coverage
+htmlcov

blender_downloader/__init__.py

@@ -8,6 +8,7 @@
 import os
 import re
 import shutil
+import subprocess
 import sys
 import tarfile
 import tempfile
@@ -31,13 +32,18 @@
 BLENDER_MANUAL_VERSIONS_URL = "https://docs.blender.org/PROD/versions.json"
 BLENDER_DAILY_BUILDS_URL = "https://builder.blender.org/download/daily/"
 MINIMUM_VERSION_SUPPPORTED = "2.57"
-SUPPORTED_FILETYPES_EXTRACTION = [".bz2", ".gz", ".xz", ".zip", ".dmg"]
 NIGHLY_RELEASES_CACHE_EXPIRATION = 60 * 60 * 24  # 1 day
 CACHE = Cache(
     user_data_dir(appname=__title__, appauthor=__author__, version=__version__)
 )
 
 
+def removesuffix(string, suffix):  # polyfill for Python < 3.9
+    if string.endswith(suffix):
+        return string[: -len(suffix)]
+    return string
+
+
 class BlenderVersion:
     """Blender versions object for easy comparations support."""
 
@@ -104,6 +110,23 @@ def GET(url, expire=259200, use_cache=True):  # 3 days for expiration
     return response.decode("utf-8")
 
 
+def get_toplevel_dirnames_from_paths(paths):
+    """Extracts the names of the directories in the top level
+    from a set of paths constituting multiple directory trees.
+    """
+    toplevel_dirnames = []
+    for path in paths:
+        # not optimal implementation, but crossplatform for sure
+        parent, _ = os.path.split(path)
+        is_file = parent == ""
+        if not is_file:
+            while parent:
+                previous_parent = parent
+                parent, _ = os.path.split(previous_parent)
+            toplevel_dirnames.append(previous_parent)
+    return toplevel_dirnames
+
+
 def build_parser():
     parser = argparse.ArgumentParser(description=__description__)
     parser.add_argument(
@@ -742,13 +765,6 @@ def extract_release(zipped_filepath, quiet=False):
     output_directory = os.path.abspath(os.path.dirname(zipped_filepath))
     extension = os.path.splitext(zipped_filepath)[1]
 
-    if extension not in SUPPORTED_FILETYPES_EXTRACTION:
-        sys.stderr.write(
-            f"Blender compressed release file '{zipped_filename}' extraction"
-            " is not supported by blender-downloader.\n"
-        )
-        sys.exit(1)
-
     # filepath of the extracted directory, don't confuse it with
     # `output_directory`, that is the directory where the file to extract
     # is located
@@ -760,10 +776,32 @@ def extract_release(zipped_filepath, quiet=False):
 
         with zipfile.ZipFile(zipped_filepath, "r") as f:
             namelist = f.namelist()
-            extracted_directory_filepath = os.path.join(
-                output_directory,
-                namelist[0].split(os.sep)[0],
-            )
+
+            # ensure that Blender is extracted in a top level directory
+            #
+            # MacOS versions previous to 2.79 are released in a ZIP file
+            # with multiple directories and files in the root, which results
+            # in a messy extraction with a lot of files in the current
+            # working directory
+            root_dirnames = get_toplevel_dirnames_from_paths(namelist)
+
+            # `not root_dirnames` when only files are found in the ZIP
+            if len(root_dirnames) > 1 or not root_dirnames:
+                output_directory = os.path.join(output_directory, "Blender")
+            else:
+                output_directory = os.path.join(output_directory, root_dirnames[0])
+
+            # don't overwrite existing non empty directory extracting
+            if os.path.isdir(output_directory) and os.listdir(output_directory):
+                sys.stderr.write(
+                    f"The directory '{output_directory}' where the files will"
+                    " be extracted already exists and is not empty. Extraction"
+                    " skipped.\n"
+                )
+                sys.exit(1)
+
+            extracted_directory_filepath = output_directory
+
             progress_bar_kwargs = dict(
                 total=len(namelist),
                 desc=f"Extracting '{zipped_filename}'",
@@ -772,58 +810,117 @@ def extract_release(zipped_filepath, quiet=False):
             )
             for file in tqdm(**progress_bar_kwargs):
                 f.extract(member=file, path=output_directory)
-    elif extension in [".bz2", ".gz", ".xz"]:
+
+    elif extension in [".bz2", ".gz", ".xz", ".tar.gz", ".tar.bz2", ".tar.xz"]:
         if not quiet:
             sys.stderr.write(f"Decompressing '{zipped_filename}'...\n")
 
         with tarfile.open(zipped_filepath, "r") as f:
-            members = f.getmembers()
+            files = f.getmembers()
+            paths = [file.name for file in files]
+
+            root_dirnames = get_toplevel_dirnames_from_paths(paths)
+
+            if len(root_dirnames) > 1 or not root_dirnames:
+                output_directory = os.path.join(output_directory, "Blender")
+            else:
+                output_directory = os.path.join(output_directory, root_dirnames[0])
+
+            extracted_directory_filepath = output_directory
 
-            extracted_directory_filepath = os.path.join(
-                output_directory,
-                members[0].name.split(os.sep)[0],
-            )
             progress_bar_kwargs = dict(
-                total=len(members),
+                total=len(files),
                 desc=f"Extracting '{zipped_filename}'",
-                iterable=members,
+                iterable=files,
                 disable=quiet,
             )
             for file in tqdm(**progress_bar_kwargs):
                 f.extract(member=file, path=output_directory)
-    else:  # extension == ".dmg":
+
+    elif extension == ".dmg":
         running_os = get_running_os()
         if running_os != "macos":
-            sys.stderr.write(
-                "blender-downloader can't mount MacOS '.dmg' image files like"
-                f" '{extracted_directory_filepath}' in"
-                f" {running_os.capitalize()}, so you should install Blender"
-                " manually.\n"
-            )
-            sys.exit(1)
+            # we are not in MacOS, so we need the binaries dmg2img and 7z
+            # to decompress the `.dmg` file downloaded
+            dmg2img, sevenz = shutil.which("dmg2img"), shutil.which("7z")
+            if dmg2img is None or sevenz is None:
+                required_programs = []
+                if dmg2img is None:
+                    required_programs.append("'dmg2img'")
+                if sevenz is None:
+                    required_programs.append("'7z'")
+                plural_suffix = "s" if len(required_programs) > 1 else ""
+                sys.stderr.write(
+                    f"You need to install the program{plural_suffix}"
+                    f" {' and '.join(required_programs)} to extract the"
+                    f" DMG Blender release located at {zipped_filepath}"
+                    f" inside a {running_os.capitalize()} platform.\n"
+                )
+                sys.exit(1)
 
-        extracted_directory_filepath = os.path.join(
-            output_directory, os.path.basename(zipped_filepath).rstrip(".dmg")
-        )
+            img_filepath = removesuffix(zipped_filepath, "dmg") + "img"
+            if os.path.isfile(img_filepath):
+                os.remove(img_filepath)
 
-        import dmglib
-
-        with dmglib.attachedDiskImage(zipped_filepath) as mounted_dmg:
-            contents_parent_dirpath = None
-            for dirpath, _, _ in os.walk(mounted_dmg[0]):
-                if (
-                    os.path.basename(dirpath) == "Contents"
-                    and "blender.app" in dirpath.lower()
-                ):
-                    contents_parent_dirpath = os.path.abspath(
-                        os.path.dirname(dirpath),
-                    )
-                    break
-            shutil.copytree(
-                contents_parent_dirpath,
-                extracted_directory_filepath,
+            dmg2img_proc = subprocess.Popen(
+                ["dmg2img", zipped_filepath],
+                stderr=sys.stderr,
+                stdout=sys.stdout,
+                env=os.environ,
+            )
+            dmg2img_proc.communicate()
+            if dmg2img_proc.returncode != 0:
+                sys.exit(dmg2img_proc.returncode)
+
+            seven7_proc = subprocess.Popen(
+                ["7z", "x", img_filepath],
+                stderr=sys.stderr,
+                stdout=sys.stdout,
+                env=os.environ,
+            )
+            seven7_proc.communicate()
+            if seven7_proc.returncode != 0:
+                sys.exit(seven7_proc.returncode)
+
+            extracted_directory_filepath = os.path.join(output_directory, "Blender")
+        else:
+            # we are inside MacOS, use the DMG CLI utility included in
+            # the system through the dmglib Python wrapper
+            extracted_directory_filepath = os.path.join(
+                output_directory,
+                removesuffix(os.path.basename(zipped_filepath), ".dmg"),
             )
 
+            import dmglib
+
+            with dmglib.attachedDiskImage(zipped_filepath) as mounted_dmg:
+                contents_parent_dirpath = None
+                for dirpath, _, _ in os.walk(mounted_dmg[0]):
+                    if (
+                        os.path.basename(dirpath) == "Contents"
+                        and "blender.app" in dirpath.lower()
+                    ):
+                        contents_parent_dirpath = os.path.abspath(
+                            os.path.dirname(dirpath),
+                        )
+                        break
+                shutil.copytree(
+                    contents_parent_dirpath,
+                    extracted_directory_filepath,
+                )
+    else:
+        if "-e" in sys.argv and "--extract" not in sys.argv:
+            extract_option = "-e"
+        elif "-e" not in sys.argv and "--extract" in sys.argv:
+            extract_option = "--extract"
+        else:
+            extract_option = "-e/--extract"
+        sys.stderr.write(
+            f"File extension '{extension}' extraction not supported by"
+            f" '{extract_option}' command line option.\n"
+        )
+        sys.exit(1)
+
     return extracted_directory_filepath
 
 
@@ -1156,7 +1253,7 @@ def run(args=[]):
     return 0
 
 
-def main():
+def main():  # pragma: no cover
     sys.exit(run(args=sys.argv[1:]))
 
 

setup.cfg

@@ -57,6 +57,9 @@ test =
     pytest==6.2.5
     pytest-cov==3.0.0
 
+[tool:pytest]
+addopts = --cov=blender_downloader --cov-report=html --cov-config=setup.cfg
+
 [coverage:report]
 exclude_lines =
     pragma: no cover

tests/conftest.py

@@ -0,0 +1,7 @@
+import os
+import sys
+
+
+TESTS_DIR = os.path.abspath(os.path.dirname(__file__))
+if TESTS_DIR not in sys.path:
+    sys.path.append(TESTS_DIR)