From 67feca293580eb9325c4f756cf1257a0e7444e4f Mon Sep 17 00:00:00 2001 From: Giuseppe Natale Date: Wed, 19 Feb 2020 23:02:23 +0000 Subject: [PATCH] Refatcor 'deliver_notebooks' and 'download_overlays' in pynq.utils (#1011) * Refatcor 'deliver_notebooks' and 'download_overlays' in pynq.utils * Add comment in pynq_utils documentation regarding structure of .link JSON for global resource --- .../overlay_design_methodology/pynq_utils.rst | 15 +- pynq/_cli/get_notebooks.py | 13 +- pynq/bitstream.py | 11 +- pynq/utils.py | 578 +++++++++++------- 4 files changed, 374 insertions(+), 243 deletions(-) diff --git a/docs/source/overlay_design_methodology/pynq_utils.rst b/docs/source/overlay_design_methodology/pynq_utils.rst index e55097957b..a2e289f0a4 100644 --- a/docs/source/overlay_design_methodology/pynq_utils.rst +++ b/docs/source/overlay_design_methodology/pynq_utils.rst @@ -25,9 +25,20 @@ the URL where the overlay can be downloaded, and the MD5 checksum of the file } } +In case the resource to be downloaded is *global* the *url* and *md5sum* +entries should be at the top-level of the JSON dictionary. In this case, no +device-based resolution will be performed. + +.. code-block :: javascript + + { + "url": "https://link.to/resource.extension", + "md5sum": "1df38cf582c4c5d0c8e3ca38be8f1eb3" + } + PYNQ provides a ``download_overlays`` setuptools command which will process any -link files in your repository and download the overlays for your board and -place them alongside the link files. To run the the download command +link files in your repository and download overlays and resources for your +board and place them alongside the link files. To run the the download command automatically at build time the utils module provides a ``build_py`` command class that can be used in place of the normal ``build_py`` phase. For more details on how to include this command in your setup.py see the diff --git a/pynq/_cli/get_notebooks.py b/pynq/_cli/get_notebooks.py index 71047ec8b1..b245fbae0b 100755 --- a/pynq/_cli/get_notebooks.py +++ b/pynq/_cli/get_notebooks.py @@ -187,14 +187,15 @@ def main(): raise FileExistsError("Target notebooks directory already " "exists. Specify another path or use " "the 'force' option to proceed") - overlays_lookup = not args.ignore_overlays + overlays_res_lookup = not args.ignore_overlays try: ## Ignoring notebooks from main `pynq.notebooks` namespace as of now # src_path = notebooks_ext_man.extension_path(NOTEBOOKS_GROUP) # logger.info("Delivering notebooks from '{}'...".format( # NOTEBOOKS_GROUP)) # deliver_notebooks(device, src_path, delivery_path, - # NOTEBOOKS_GROUP, overlays_lookup=overlays_lookup) + # NOTEBOOKS_GROUP, + # overlays_res_lookup=overlays_res_lookup) ## for ext in notebooks_ext_man.list: if args.notebooks and "all" not in args.notebooks and \ @@ -228,13 +229,13 @@ def main(): # pre- or post-processing calling # `pynq.utils:deliver_notebooks` inside, or an entirely custom # delivery procedure - ext_mod.deliver_notebooks(device, src_path, delivery_path, - ext.name, folder=folder, - overlays_lookup=overlays_lookup) + ext_mod.deliver_notebooks( + device, src_path, delivery_path, ext.name, folder=folder, + overlays_res_lookup=overlays_res_lookup) else: deliver_notebooks(device, src_path, delivery_path, ext.name, folder=folder, - overlays_lookup=overlays_lookup) + overlays_res_lookup=overlays_res_lookup) except (Exception, KeyboardInterrupt) as e: raise e finally: diff --git a/pynq/bitstream.py b/pynq/bitstream.py index 0e98decd3d..ef416f377b 100644 --- a/pynq/bitstream.py +++ b/pynq/bitstream.py @@ -32,11 +32,8 @@ __email__ = "pynq_support@xilinx.com" import os -from datetime import datetime -import struct -import numpy as np from .devicetree import get_dtbo_path -from .utils import _find_local_overlay +from .utils import _find_local_overlay_res PYNQ_PATH = os.path.dirname(os.path.realpath(__file__)) @@ -45,14 +42,14 @@ def _resolve_bitstream(bitfile_path, device): if os.path.isfile(bitfile_path): return bitfile_path if os.path.isdir(bitfile_path + ".d") and hasattr(device, 'name'): - device_name = device.name split_bitfile = os.path.split(bitfile_path) - local_bitfile = _find_local_overlay(device.name, split_bitfile[1], - split_bitfile[0]) + local_bitfile = _find_local_overlay_res(device.name, split_bitfile[1], + split_bitfile[0]) if local_bitfile is not None: return local_bitfile return None + class Bitstream: """This class instantiates the meta class for PL bitstream (full/partial). diff --git a/pynq/utils.py b/pynq/utils.py index 5c1b7942ec..0adc836eb1 100755 --- a/pynq/utils.py +++ b/pynq/utils.py @@ -206,41 +206,57 @@ def _download_file(download_link, path, md5sum=None): path)) -def _find_local_overlay(device_name, overlay_filename, src_path): - """Get path of overlay to use. +def _find_local_overlay_res(device_name, overlay_res_filename, src_path): + """Inspects ``overlay_res.ext.d` directory for an available + ``overlay_res.ext`` file for ``device_name``. + Returns ``None`` if ``device_name`` is not found. - Inspects `overlay_name.d` directory for an available overlay for - `device_name`. If a `overlay_name` file is also found, always return - that one. - Returns `None` if device_name is not found. + If a ``overlay_res.ext`` file is also found, always return that one + without doing any resolution based on ``device_name``. Parameters ---------- device_name: str - The target device name (XRT shell) - overlay_name: str - the overlay filename + The target device name + overlay_res_filename: str + The target filename to resolve src_path: str - the path where to perform this search + The path where to perform this search """ - overlay_path = os.path.join(src_path, overlay_filename) - if os.path.isfile(overlay_path): - return overlay_path - overlay_filename_split = os.path.splitext(overlay_filename) - overlay_filename_ext = "{}.{}{}".format(overlay_filename_split[0], - device_name, - overlay_filename_split[1]) - overlay_path = os.path.join(src_path, overlay_filename + ".d", - overlay_filename_ext) - if os.path.isfile(overlay_path): - return overlay_path + overlay_res_path = os.path.join(src_path, overlay_res_filename) + if os.path.isfile(overlay_res_path): + return overlay_res_path + overlay_res_filename_split = os.path.splitext(overlay_res_filename) + overlay_res_filename_ext = "{}.{}{}".format(overlay_res_filename_split[0], + device_name, + overlay_res_filename_split[1]) + overlay_res_path = os.path.join(src_path, overlay_res_filename + ".d", + overlay_res_filename_ext) + if os.path.isfile(overlay_res_path): + return overlay_res_path return None -def _find_remote_overlay(device_name, links_json_path): - """Get overlay download link and checksum from links json file. +def _find_remote_overlay_res(device_name, links_json_path): + """Get download link for ``overlay_res.ext`` file and related checksum from + ``overlay_res.ext.link`` json file, based on ``device_name``. - Expected return content from the links json file is a dict with two + The ``.link`` file is generally a dict of device names and associated url + and md5sum. + + .. code-block:: python3 + { + "device_1": { + "url": "https://link.to/overlay.xclbin", + "md5sum": "da1e100gh8e7becb810976e37875de38" + }. + "device_2": { + "url": "https://link.to/overlay.xclbin", + "md5sum": "da1e100gh8e7becb810976e37875de38" + } + } + + Expected return content from the ``.link`` json file is a dict with two entries: .. code-block:: python3 @@ -250,20 +266,25 @@ def _find_remote_overlay(device_name, links_json_path): "md5sum": "da1e100gh8e7becb810976e37875de38" } - Returns `None` if device_name is not found + Returns `None` if ``device_name`` is not found. + + If the ``.link`` file contains a *url* and *md5sum* entries at the top + level, these are returned and no device-based resolution is performed. Parameters ---------- device_name: str - The target device name (XRT shell) + The target device name links_json_path: str - The full path to the links json file + The full path to the ``.link`` json file """ with open(links_json_path) as f: links = json.load(f) - if device_name in links: - return links[device_name] - return None + if "url" in links and "md5sum" in links: + return {"url": links["url"], "md5sum": links["md5sum"]} + if device_name in links: + return links[device_name] + return None class OverlayNotFoundError(Exception): @@ -272,35 +293,145 @@ class OverlayNotFoundError(Exception): pass +def _resolve_overlay_res_from_folder(device_name, overlay_res_folder, src_path, + dst_path, rel_path, files_to_copy): + """Resolve ``overlay_res.ext`` file from ``overlay_res.ext.d`` folder, + based on ``device_name``. Updates ``files_to_copy`` with the resolved file + to use. If a ``overlay_res.ext.link`` file is found, resolution is skipped + here. This is to avoid inspecting the ``overlay_res.ext.d`` folder twice. + See ``_resolve_overlay_res_from_link()``. + """ + overlay_res_filename = os.path.splitext(overlay_res_folder)[0] + # Avoid checking a .d folder twice when also a + # related .link file is found + if not os.path.isfile(os.path.join(src_path, + overlay_res_filename + ".link")): + overlay_res_src_path = _find_local_overlay_res(device_name, + overlay_res_filename, + src_path) + if overlay_res_src_path: + overlay_res_dst_path = os.path.join(dst_path, rel_path, + overlay_res_filename) + files_to_copy[overlay_res_src_path] = overlay_res_dst_path + else: + raise OverlayNotFoundError(overlay_res_filename) + + +def _resolve_overlay_res_from_link(device_name, overlay_res_link, src_path, + dst_path, rel_path, files_to_copy, + files_to_move, logger): + """Resolve ``overlay_res.ext`` file from ``overlay_res.ext.link`` file, + based on ``device_name``. Updates ``files_to_copy`` with the resolved file + to use if found locally (by inspecting ``overlay_res.ext.d`` folder), or + updates ``files_to_move`` in case the file is downloaded. + """ + overlay_res_filename = os.path.splitext(overlay_res_link)[0] + overlay_res_dst_path = os.path.join(dst_path, rel_path, + overlay_res_filename) + overlay_res_src_path = _find_local_overlay_res(device_name, + overlay_res_filename, + src_path) + if overlay_res_src_path: + files_to_copy[overlay_res_src_path] = overlay_res_dst_path + else: + overlay_res_download_dict = _find_remote_overlay_res( + device_name, os.path.join(src_path, overlay_res_link)) + if overlay_res_download_dict: + # attempt overlay_res.ext file download + try: + tmp_file = tempfile.mkstemp()[1] + logger.info("Downloading file '{}'. This may take a while" + "...".format(overlay_res_filename)) + _download_file( + overlay_res_download_dict["url"], + tmp_file, + overlay_res_download_dict["md5sum"] + ) + files_to_move[tmp_file] = overlay_res_dst_path + except DownloadedFileChecksumError: + raise OverlayNotFoundError(overlay_res_filename) + else: + raise OverlayNotFoundError(overlay_res_filename) + + +def _copy_and_move_files(files_to_copy, files_to_move): + """Copy and move files and folders. ``files_to_copy`` and ``files_to_move`` + are expected to be dict where the key is the source path, and the value is + destination path. + """ + # copy files and folders + for src, dst in files_to_copy.items(): + if os.path.isfile(src): + mkpath(os.path.dirname(dst)) + copy_file(src, dst) + else: + copy_tree(src, dst) + # and move files previously downloaded + for src, dst in files_to_move.items(): + shutil.move(src, dst) + + +def _roll_back_copy(files_to_copy, files_to_move): + """Roll-back previously performed copy of files and folders. + ``files_to_copy`` and ``files_to_move`` are expected to be dict where the + key is the source path, and the value is destination path. + """ + for _, dst in files_to_copy.items(): + if os.path.isfile(dst): + os.remove(dst) + while(len(os.listdir(os.path.dirname(dst))) == 0): + os.rmdir(os.path.dirname(dst)) + dst = os.path.dirname(dst) + elif os.path.isdir(dst): + remove_tree(dst) + for _, dst in files_to_move.items(): + if os.path.isfile(dst): + os.remove(dst) + while(len(os.listdir(os.path.dirname(dst))) == 0): + os.rmdir(os.path.dirname(dst)) + dst = os.path.dirname(dst) + + def deliver_notebooks(device_name, src_path, dst_path, name, folder=False, - overlays_lookup=True): - """Deliver notebooks (and possibly overlays) to target destination path. - - If overlays are delivered alongside notebooks, the following resolution - strategy is applied when inspecting the `src_path`: - - 1. If an overlay file is found, prioritize that file and do not perform - any overlay resolution - 2. In case step 1 fails, if a overlay_name.d folder is found, try to - retrieve the overlay from there. The overlays in this folder are - expected to contain the device name as a string, before the file - extension - 3. In case step 2 fails, if there is an overlay_name.link file, attempt - to download the correct overlay from the provided link, assumed that - a valid entry is available in the links json - 4. If all steps fail, remove the associated notebooks if the overlay - could not be found or downloaded, and warns the user. - - In the other case, notebooks are simply copied as is without any checks. + overlays_res_lookup=True): + """Deliver notebooks to target destination path. + + If a ``overlay_res.ext.link`` file or a ``overlay_res.ext.d`` folders is + found, then ``overlay_res.ext`` (where ``.ext`` represents a generic file + extension) is considered to be a file that need to be resolved dynamically, + based on ``device_name``. + The following resolution strategy is applied when inspecting ``src_path``: + + 1. If an ``overlay_res.ext`` file is found, prioritize that file and do + not perform any resolution. + 2. In case step 1 fails, if a ``overlay_res.ext.d`` folder is found, + try to retrieve the right ``overlau_res.ext`` file from there. The + files in this folder are expected to contain the device name as a + string, before the file extension ``.ext``. + Format should be ``overlay_res.device_name.ext``. + 3. In case step 2 fails, if there is an ``overlay_res.ext.link`` file, + attempt to download the correct file from the provided url, assumed + that a valid entry for ``device_name`` is available in the ``.link`` + json file. + 4. If all steps fail, notebooks that are in the same folder as + ``overlay_res.ext`` are not delivered, and the user is warned. + + For simplicity, it is assumed that ``.link`` files and ``.d`` folders are + located next to the notebooks that use the associated resource. Folders + that does not contain notebooks will not be inspected. + + In case no ``.link`` or ``overlay_res.d`` files are found, notebooks are + simply copied as is, no resolution is performed. It is assumed that for this scenario, overlays are delivered somewhere else. Parameters ---------- device_name: str - The target device name for the notebooks. If an overlay is already - found, no overlay resolution will be done and `device_name` will be - ignored for that overlay, as it is assumed that the overlay file is + The target device name to use when doing resolution of ``.link`` + files and ``.d`` folders. If an ``overlay_res.ext`` file is also + found, no resolution will be done and ``device_name`` will be + ignored, as it is assumed that the ``overlay_res.ext`` file is prioritized and no automatic resolution is expected src_path: str The source path to copy from @@ -309,11 +440,12 @@ def deliver_notebooks(device_name, src_path, dst_path, name, folder=False, name: str The name of the notebooks module folder: bool - Indicates whether to use `name` as target folder to copy notebooks, - inside `dst_path`. Notebooks will be copied directly in `dst_path` - if `None`. - overlays_lookup: bool - Dictates whether automatic overlays lookup must be performed. + Indicates whether to use ``name`` as target folder to copy + notebooks, inside ``dst_path``. Notebooks will be copied directly + in ``dst_path`` if ``False``. + overlays_res_lookup: bool + Dynamic resolution of ``.link`` files and ``.d`` folders is + disabled if ```False``. """ logger = get_logger() dst_fullpath = os.path.join(dst_path, name) if folder else dst_path @@ -322,6 +454,8 @@ def deliver_notebooks(device_name, src_path, dst_path, name, folder=False, for root, dirs, files in os.walk(src_path): # If there is at least one notebook, inspect the folder if [f for f in files if f.endswith(".ipynb")]: + # If folder is in the list of files to copy, remove it as it is + # going to be inspected if root in files_to_copy: files_to_copy.pop(root) relpath = os.path.relpath(root, src_path) @@ -331,55 +465,20 @@ def deliver_notebooks(device_name, src_path, dst_path, name, folder=False, files_to_move_tmp = {} for d in dirs: if d.endswith(".d"): - if overlays_lookup: - overlay_name = os.path.splitext(d)[0] - if not os.path.isfile(os.path.join( - root, overlay_name + ".link")): - overlay_src_path = _find_local_overlay( - device_name, overlay_name, root) - if overlay_src_path: - overlay_dst_path = os.path.join( - dst_fullpath, relpath, overlay_name) - files_to_copy_tmp[overlay_src_path] = \ - overlay_dst_path - else: - raise OverlayNotFoundError + if overlays_res_lookup: + _resolve_overlay_res_from_folder( + device_name, d, root, dst_fullpath, relpath, + files_to_copy_tmp) elif d != "__pycache__": # exclude __pycache__ folder dir_dst_path = os.path.join(dst_fullpath, relpath, d) files_to_copy_tmp[os.path.join(root, d)] = \ dir_dst_path for f in files: if f.endswith(".link"): - if overlays_lookup: - overlay_name = os.path.splitext(f)[0] - overlay_dst_path = os.path.join(dst_fullpath, - relpath, - overlay_name) - overlay_src_path = _find_local_overlay( - device_name, overlay_name, root) - if overlay_src_path: - files_to_copy_tmp[overlay_src_path] = \ - overlay_dst_path - else: - overlay_download_dict = _find_remote_overlay( - device_name, os.path.join(root, f)) - if overlay_download_dict: - # attempt overlay download - try: - tmp_file = tempfile.mkstemp()[1] - logger.info("Downloading file '{}'. " - "This may take a while" - "...".format(overlay_name)) - _download_file( - overlay_download_dict["url"], - tmp_file, - overlay_download_dict["md5sum"]) - files_to_move_tmp[tmp_file] = \ - overlay_dst_path - except DownloadedFileChecksumError: - raise OverlayNotFoundError - else: - raise OverlayNotFoundError + if overlays_res_lookup: + _resolve_overlay_res_from_link( + device_name, f, root, dst_fullpath, relpath, + files_to_copy_tmp, files_to_move_tmp, logger) else: file_dst_path = os.path.join(dst_fullpath, relpath, f) files_to_copy_tmp[os.path.join(root, f)] = \ @@ -389,13 +488,13 @@ def deliver_notebooks(device_name, src_path, dst_path, name, folder=False, files_to_copy.update(files_to_copy_tmp) # and files_to_move_tmp to files_to_move files_to_move.update(files_to_move_tmp) - except OverlayNotFoundError: - # files_to_copy not updated, notebooks here skipped + except OverlayNotFoundError as e: + # files_to_copy not updated, folder skipped if relpath: nb_str = os.path.join(name, relpath) - logger.info("Could not find valid overlay for notebooks " - "'{}', these notebooks will not be " - "delivered".format(nb_str)) + logger.info("Could not resolve file '{}' in folder " + "'{}', notebooks will not be " + "delivered".format(str(e), nb_str)) try: # exclude root __init__.py from copy, if it exists files_to_copy.pop(os.path.join(src_path, "__init__.py")) @@ -404,46 +503,159 @@ def deliver_notebooks(device_name, src_path, dst_path, name, folder=False, try: if not files_to_copy: logger.info("The notebooks module '{}' could not be delivered. " - "No valid overlays were found".format(name)) + "The module has no notebooks, or no valid overlays " + "were found".format(name)) else: - # now copy files and folders - for src, dst in files_to_copy.items(): - if os.path.isfile(src): - mkpath(os.path.dirname(dst)) - copy_file(src, dst) - else: - copy_tree(src, dst) - # and move files previously downloaded - for src, dst in files_to_move.items(): - shutil.move(src, dst) + _copy_and_move_files(files_to_copy, files_to_move) except (Exception, KeyboardInterrupt) as e: # roll-back copy logger.info("Exception detected. Cleaning up as the delivery process " "did not complete...") - for _, dst in files_to_copy.items(): - if os.path.isfile(dst): - os.remove(dst) - while(len(os.listdir(os.path.dirname(dst))) == 0): - os.rmdir(os.path.dirname(dst)) - dst = os.path.dirname(dst) - elif os.path.isdir(dst): - remove_tree(dst) - for _, dst in files_to_move.items(): - if os.path.isfile(dst): - os.remove(dst) - while(len(os.listdir(os.path.dirname(dst))) == 0): - os.rmdir(os.path.dirname(dst)) - dst = os.path.dirname(dst) + _roll_back_copy(files_to_copy, files_to_move) raise e +def _resolve_global_overlay_res(overlay_res_link, src_path, logger, + fail=False): + """Resolve resource that is global to every device (using a ``device=None`` + when calling ``_find_remote_overlay_res``). File is downloaded in + ``src_path``. + """ + overlay_res_filename = os.path.splitext(overlay_res_link)[0] + overlay_res_download_dict = \ + _find_remote_overlay_res(None, + os.path.join(src_path, overlay_res_link)) + if overlay_res_download_dict: + overlay_res_fullpath = os.path.join( + src_path, overlay_res_filename) + try: + logger.info("Downloading file '{}'. " + "This may take a while" + "...".format( + overlay_res_filename)) + _download_file( + overlay_res_download_dict["url"], + overlay_res_fullpath, + overlay_res_download_dict["md5sum"]) + except Exception as e: + if fail: + raise e + finally: + if not os.path.isfile( + overlay_res_fullpath): + err_msg = "Could not resolve file '{}'".format( + overlay_res_filename) + logger.info(err_msg) + return True # overlay_res_download_dict was not empty + return False + + +def _resolve_devices_overlay_res(overlay_res_link, src_path, devices, logger, + fail=False): + """Resolve ``overlay_res.ext`` file for every device in ``devices``. + Files are downloaded in a ``overlay_res.ext.d`` folder in ``src_path``. + """ + overlay_res_filename = os.path.splitext(overlay_res_link)[0] + for device in devices: + overlay_res_src_path = _find_local_overlay_res(device, + overlay_res_filename, + src_path) + err_msg = "Could not resolve file '{}' for " \ + "device '{}'".format(overlay_res_filename, + device) + if not overlay_res_src_path: + overlay_res_download_dict = _find_remote_overlay_res( + device, os.path.join(src_path, overlay_res_link)) + if overlay_res_download_dict: + overlay_res_download_path = os.path.join( + src_path, overlay_res_filename + ".d") + overlay_res_filename_split = \ + os.path.splitext(overlay_res_filename) + overlay_res_filename_ext = "{}.{}{}".format( + overlay_res_filename_split[0], device, + overlay_res_filename_split[1]) + mkpath(overlay_res_download_path) + overlay_res_fullpath = os.path.join(overlay_res_download_path, + overlay_res_filename_ext) + try: + logger.info("Downloading file '{}'. " + "This may take a while" + "...".format( + overlay_res_filename)) + _download_file( + overlay_res_download_dict["url"], + overlay_res_fullpath, + overlay_res_download_dict["md5sum"]) + except Exception as e: + if fail: + raise e + finally: + if not os.path.isfile( + overlay_res_fullpath): + logger.info(err_msg) + if len(os.listdir(overlay_res_download_path)) == 0: + os.rmdir(overlay_res_download_path) + else: + if fail: + raise OverlayNotFoundError(err_msg) + logger.info(err_msg) + + +def _resolve_all_overlay_res_from_link(overlay_res_link, src_path, logger, + fail=False): + """Resolve every entry of ``.link`` files regardless of detected devices. + """ + overlay_res_filename = os.path.splitext(overlay_res_link)[0] + with open(os.path.join(src_path, overlay_res_link)) as f: + links = json.load(f) + if not _resolve_global_overlay_res(overlay_res_link, src_path, logger, + fail): + for device, download_link_dict in links.items(): + if not _find_local_overlay_res( + device, overlay_res_filename, src_path): + err_msg = "Could not resolve file '{}' for " \ + "device '{}'".format(overlay_res_filename, + device) + overlay_res_download_path = os.path.join( + src_path, overlay_res_filename + ".d") + overlay_res_filename_split = \ + os.path.splitext(overlay_res_filename) + overlay_res_filename_ext = "{}.{}{}".format( + overlay_res_filename_split[0], device, + overlay_res_filename_split[1]) + mkpath(overlay_res_download_path) + overlay_res_fullpath = os.path.join( + overlay_res_download_path, + overlay_res_filename_ext) + try: + logger.info("Downloading file '{}'. " + "This may take a while" + "...".format( + overlay_res_filename)) + _download_file( + download_link_dict["url"], + overlay_res_fullpath, + download_link_dict["md5sum"]) + except Exception as e: + if fail: + raise e + finally: + if not os.path.isfile( + overlay_res_fullpath): + logger.info(err_msg) + if len(os.listdir( + overlay_res_download_path)) == 0: + os.rmdir(overlay_res_download_path) + + def download_overlays(path, download_all=False, fail=False): """Download overlays for detected devices in destination path. - Downloads overlays for all detected devices using 'overlay_filename.link' - json files. Downloaded overlays are put in a 'overlay_filename.d' - directory, with the device name added to their filename. - If target overlay already exists, automatic resolution is skipped. + Resolve ``overlay_res.ext`` files from ``overlay_res.ext.link`` + json files. Downloaded ``overlay_res.ext`` files are put in a + ``overlay_res.ext.d`` directory, with the device name added to their + filename, as ``overlay_res.device_name.ext``. + If target ``overlay_res.ext`` already exists, resolution is skipped. Parameters ---------- @@ -465,104 +677,14 @@ def download_overlays(path, download_all=False, fail=False): raise e devices = [] for root, dirs, files in os.walk(path): - if not download_all: - for d in dirs: - if d.endswith(".d"): - overlay_name = os.path.splitext(d)[0] - if not os.path.isfile(os.path.join( - root, overlay_name + ".link")): - for device in devices: - overlay_src_path = _find_local_overlay( - device, overlay_name, root) - if not overlay_src_path: - err_msg = "Could not find overlay '{}' for " \ - "device '{}'".format(overlay_name, - device) - if fail: - raise OverlayNotFoundError(err_msg) - logger.info(err_msg) for f in files: if f.endswith(".link"): - overlay_name = os.path.splitext(f)[0] if not download_all: - for device in devices: - overlay_src_path = _find_local_overlay(device, - overlay_name, - root) - err_msg = "Could not find overlay '{}' for " \ - "device '{}'".format(overlay_name, device) - if not overlay_src_path: - overlay_download_dict = _find_remote_overlay( - device, os.path.join(root, f)) - if overlay_download_dict: - overlay_download_path = os.path.join( - root, overlay_name + ".d") - overlay_filename_split = \ - os.path.splitext(overlay_name) - overlay_filename_ext = "{}.{}{}".format( - overlay_filename_split[0], device, - overlay_filename_split[1]) - mkpath(overlay_download_path) - overlay_fullpath = os.path.join( - overlay_download_path, - overlay_filename_ext) - try: - logger.info("Downloading file '{}'. " - "This may take a while" - "...".format(overlay_name)) - _download_file( - overlay_download_dict["url"], - overlay_fullpath, - overlay_download_dict["md5sum"]) - except Exception as e: - if fail: - raise e - finally: - if not os.path.isfile(overlay_fullpath): - logger.log(err_msg) - if len(os.listdir( - overlay_download_path)) == 0: - os.rmdir(overlay_download_path) - else: - if fail: - raise OverlayNotFoundError(err_msg) - logger.log(err_msg) + if not _resolve_global_overlay_res(f, root, logger, fail): + _resolve_devices_overlay_res(f, root, devices, logger, + fail) else: # download all overlays regardless of detected devices - with open(os.path.join(root, f)) as f: - links = json.load(f) - for device, download_link_dict in links.items(): - if not _find_local_overlay(device, overlay_name, - root): - err_msg = "Could not find overlay '{}' for " \ - "device '{}'".format(overlay_name, device) - overlay_download_path = os.path.join( - root, overlay_name + ".d") - overlay_filename_split = \ - os.path.splitext(overlay_name) - overlay_filename_ext = "{}.{}{}".format( - overlay_filename_split[0], device, - overlay_filename_split[1]) - mkpath(overlay_download_path) - overlay_fullpath = os.path.join( - overlay_download_path, - overlay_filename_ext) - try: - logger.info("Downloading file '{}'. " - "This may take a while" - "...".format(overlay_name)) - _download_file( - download_link_dict["url"], - overlay_fullpath, - download_link_dict["md5sum"]) - except Exception as e: - if fail: - raise e - finally: - if not os.path.isfile(overlay_fullpath): - logger.log(err_msg) - if len(os.listdir( - overlay_download_path)) == 0: - os.rmdir(overlay_download_path) + _resolve_all_overlay_res_from_link(f, root, logger, fail) class _download_overlays(dist_build):