diff --git a/quantecon/util/notebooks.py b/quantecon/util/notebooks.py index 51ef1c3a6..804081c8b 100644 --- a/quantecon/util/notebooks.py +++ b/quantecon/util/notebooks.py @@ -9,7 +9,7 @@ Files on the REMOTE Github Server can be organised into folders but they will end up at the root level of when downloaded as a support File -"https://github.com/QuantEcon/QuantEcon.notebooks/raw/master/dependencies/mpi/something.py" --> ./somthing.py +"https://github.com/QuantEcon/QuantEcon.notebooks/raw/master/dependencies/mpi/something.py" --> ./something.py TODO ---- @@ -21,57 +21,61 @@ import os import requests +import warnings #-Remote Structure-# REPO = "https://github.com/QuantEcon/QuantEcon.notebooks" RAW = "raw" BRANCH = "master" -DEPS = "dependencies" #Hard Coded Dependencies Folder on QuantEcon.notebooks +FOLDER = "dependencies" #Hard Coded Dependencies Folder on QuantEcon.notebooks -def fetch_nb_dependencies(files, repo=REPO, raw=RAW, branch=BRANCH, deps=DEPS, overwrite=False, verbose=True): +def fetch_nb_dependencies(files, repo=REPO, raw=RAW, branch=BRANCH, folder=FOLDER, overwrite=False, verbose=True): """ Retrieve raw files from QuantEcon.notebooks or any other Github repo Parameters ---------- file_list list or dict - A list of files to specify a collection of filenames + A list of files to specify a collection of filenames A dict of dir : list(files) to specify a directory repo str, optional(default=REPO) + raw str, optional(defualt=RAW) + This is here in case github changes access to their raw files through web links branch str, optional(default=BRANCH) - deps str, optional(default=DEPS) + folder str, optional(default=FOLDER) overwrite bool, optional(default=False) verbose bool, optional(default=True) - TODO - ---- - 1. Should we update this to allow people to specify their own folders on a different GitHub repo? - """ #-Generate Common Data Structure-# if type(files) == list: files = {"" : files} + status = [] + #-Obtain each requested file-# for directory in files.keys(): if directory != "": - if verbose: print("Parsing directory: %s") + if verbose: print("Parsing directory: %s"%directory) for fl in files[directory]: if directory != "": fl = directory+"/"+fl #-Check for Local Copy of File (Default Behaviour is to Skip)-# if not overwrite: if os.path.isfile(fl): - if verbose: print("A file named %s already exists in the specified directory ... skipping download."%fl) + if verbose: print("A file named %s already exists in the specified directory ... skipping download." % fl) + status.append(False) continue else: if verbose: print("Overwriting file %s ..."%fl) if verbose: print("Fetching file: %s"%fl) #-Get file in OS agnostic way using requests-# - url = "/".join([repo,raw,branch,deps,fl]) + url = "/".join([repo,raw,branch,folder,fl]) r = requests.get(url) with open(fl, "wb") as fl: fl.write(r.content) + status.append(True) + return status diff --git a/quantecon/util/tests/test_file.md b/quantecon/util/tests/test_file.md new file mode 100644 index 000000000..3ec45702f --- /dev/null +++ b/quantecon/util/tests/test_file.md @@ -0,0 +1 @@ +This is a Simple Test File for test_notebooks.py \ No newline at end of file diff --git a/quantecon/util/tests/test_notebooks.py b/quantecon/util/tests/test_notebooks.py new file mode 100644 index 000000000..8e39e2dab --- /dev/null +++ b/quantecon/util/tests/test_notebooks.py @@ -0,0 +1,38 @@ +""" +Tests for Notebook Utilities + +Functions +--------- +fetch_nb_dependencies + +""" + +from quantecon.util import fetch_nb_dependencies +import unittest +import os + +FILES = ['test_file.md'] +REPO = "https://github.com/QuantEcon/QuantEcon.py" +RAW = "raw" +BRANCH = "master" +FOLDER = "quantecon/util/tests/" + +class TestNotebookUtils(unittest.TestCase): + + def test_fetch_nb_dependencies(self): + """ + Run First and Test Download + """ + status = fetch_nb_dependencies(files=FILES, repo=REPO, raw=RAW, branch=BRANCH, folder=FOLDER) + self.assertFalse(False in status) + + def test_fetch_nb_dependencies_overwrite(self): + """ + Run Second and Ensure file is skipped by checking a False is found in status + """ + status = fetch_nb_dependencies(files=FILES, repo=REPO, raw=RAW, branch=BRANCH, folder=FOLDER) #First will succeed + status = fetch_nb_dependencies(files=FILES, repo=REPO, raw=RAW, branch=BRANCH, folder=FOLDER) #Second should skip + self.assertTrue(False in status) + + def tearDown(self): + os.remove("test_file.md") \ No newline at end of file