From 4a358bc398dbe571ab66454b8524f56fa8128e79 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Thu, 12 Jan 2017 10:08:15 +1100 Subject: [PATCH 1/6] improve dependency fetcher with clearer parameters --- quantecon/util/notebooks.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/quantecon/util/notebooks.py b/quantecon/util/notebooks.py index 51ef1c3a6..7b7768f96 100644 --- a/quantecon/util/notebooks.py +++ b/quantecon/util/notebooks.py @@ -26,28 +26,26 @@ REPO = "https://github.com/QuantEcon/QuantEcon.notebooks" RAW = "raw" BRANCH = "master" -DEPS = "dependencies" #Hard Coded Dependencies Folder on QuantEcon.notebooks +FOLDER = "dependencies" #Hard Coded Dependencies Folder on QuantEcon.notebooks -def fetch_nb_dependencies(files, repo=REPO, raw=RAW, branch=BRANCH, deps=DEPS, overwrite=False, verbose=True): +def fetch_nb_dependencies(files, repo=REPO, raw=RAW, branch=BRANCH, folder=FOLDER, overwrite=False, verbose=True): """ Retrieve raw files from QuantEcon.notebooks or any other Github repo Parameters ---------- file_list list or dict - A list of files to specify a collection of filenames + A list of files to specify a collection of filenames A dict of dir : list(files) to specify a directory repo str, optional(default=REPO) + raw str, optional(defualt=RAW) + This is here in case github changes access to their raw files through web links branch str, optional(default=BRANCH) - deps str, optional(default=DEPS) + folder str, optional(default=FOLDER) overwrite bool, optional(default=False) verbose bool, optional(default=True) - TODO - ---- - 1. Should we update this to allow people to specify their own folders on a different GitHub repo? - """ #-Generate Common Data Structure-# @@ -70,7 +68,7 @@ def fetch_nb_dependencies(files, repo=REPO, raw=RAW, branch=BRANCH, deps=DEPS, o if verbose: print("Overwriting file %s ..."%fl) if verbose: print("Fetching file: %s"%fl) #-Get file in OS agnostic way using requests-# - url = "/".join([repo,raw,branch,deps,fl]) + url = "/".join([repo,raw,branch,folder,fl]) r = requests.get(url) with open(fl, "wb") as fl: fl.write(r.content) From b4dc2774b55a364fb13033dfaadfb080f7fc716d Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Thu, 12 Jan 2017 10:50:40 +1100 Subject: [PATCH 2/6] fix print statment issue --- quantecon/util/notebooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quantecon/util/notebooks.py b/quantecon/util/notebooks.py index 7b7768f96..27e29eded 100644 --- a/quantecon/util/notebooks.py +++ b/quantecon/util/notebooks.py @@ -55,7 +55,7 @@ def fetch_nb_dependencies(files, repo=REPO, raw=RAW, branch=BRANCH, folder=FOLDE #-Obtain each requested file-# for directory in files.keys(): if directory != "": - if verbose: print("Parsing directory: %s") + if verbose: print("Parsing directory: %s"%directory) for fl in files[directory]: if directory != "": fl = directory+"/"+fl From 7f4e04ac4990f793872ea251de88cac5d5bfd524 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Thu, 23 Feb 2017 10:05:48 +1100 Subject: [PATCH 3/6] adding a test for nb_fetch utility --- quantecon/util/notebooks.py | 10 ++++++-- quantecon/util/tests/test_notebooks.py | 32 ++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 quantecon/util/tests/test_notebooks.py diff --git a/quantecon/util/notebooks.py b/quantecon/util/notebooks.py index 27e29eded..804081c8b 100644 --- a/quantecon/util/notebooks.py +++ b/quantecon/util/notebooks.py @@ -9,7 +9,7 @@ Files on the REMOTE Github Server can be organised into folders but they will end up at the root level of when downloaded as a support File -"https://github.com/QuantEcon/QuantEcon.notebooks/raw/master/dependencies/mpi/something.py" --> ./somthing.py +"https://github.com/QuantEcon/QuantEcon.notebooks/raw/master/dependencies/mpi/something.py" --> ./something.py TODO ---- @@ -21,6 +21,7 @@ import os import requests +import warnings #-Remote Structure-# REPO = "https://github.com/QuantEcon/QuantEcon.notebooks" @@ -52,6 +53,8 @@ def fetch_nb_dependencies(files, repo=REPO, raw=RAW, branch=BRANCH, folder=FOLDE if type(files) == list: files = {"" : files} + status = [] + #-Obtain each requested file-# for directory in files.keys(): if directory != "": @@ -62,7 +65,8 @@ def fetch_nb_dependencies(files, repo=REPO, raw=RAW, branch=BRANCH, folder=FOLDE #-Check for Local Copy of File (Default Behaviour is to Skip)-# if not overwrite: if os.path.isfile(fl): - if verbose: print("A file named %s already exists in the specified directory ... skipping download."%fl) + if verbose: print("A file named %s already exists in the specified directory ... skipping download." % fl) + status.append(False) continue else: if verbose: print("Overwriting file %s ..."%fl) @@ -72,4 +76,6 @@ def fetch_nb_dependencies(files, repo=REPO, raw=RAW, branch=BRANCH, folder=FOLDE r = requests.get(url) with open(fl, "wb") as fl: fl.write(r.content) + status.append(True) + return status diff --git a/quantecon/util/tests/test_notebooks.py b/quantecon/util/tests/test_notebooks.py new file mode 100644 index 000000000..e23c14e21 --- /dev/null +++ b/quantecon/util/tests/test_notebooks.py @@ -0,0 +1,32 @@ +""" +Tests for Notebook Utilities + +Functions +--------- +fetch_nb_dependencies + +""" + +from quantecon.util import fetch_nb_dependencies +import unittest + +FILES = ['README.md'] +REPO = "https://github.com/QuantEcon/QuantEcon.py" +RAW = "raw" +BRANCH = "master" + +class TestNotebookUtils(unittest.TestCase): + + def test_fetch_nb_dependencies(self): + """ + Run First and Test Download + """ + status = fetch_nb_dependencies(files=FILES, repo=REPO, raw=RAW, branch=BRANCH) + self.assertFalse(False in status) + + def test_fetch_nb_dependencies_overwrite(self): + """ + Run Second and Ensure file is skipped by checking a False is found in status + """ + status = fetch_nb_dependencies(files=FILES, repo=REPO, raw=RAW, branch=BRANCH) + self.assertTrue(False in status) \ No newline at end of file From 7cb53bae45cdf38575aa79fc743f4889950ec866 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Thu, 23 Feb 2017 10:17:01 +1100 Subject: [PATCH 4/6] fix issue with tabs introduced by editor --- quantecon/util/tests/test_notebooks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/quantecon/util/tests/test_notebooks.py b/quantecon/util/tests/test_notebooks.py index e23c14e21..a4f5c49cc 100644 --- a/quantecon/util/tests/test_notebooks.py +++ b/quantecon/util/tests/test_notebooks.py @@ -18,15 +18,15 @@ class TestNotebookUtils(unittest.TestCase): def test_fetch_nb_dependencies(self): - """ - Run First and Test Download - """ + """ + Run First and Test Download + """ status = fetch_nb_dependencies(files=FILES, repo=REPO, raw=RAW, branch=BRANCH) self.assertFalse(False in status) def test_fetch_nb_dependencies_overwrite(self): - """ - Run Second and Ensure file is skipped by checking a False is found in status - """ + """ + Run Second and Ensure file is skipped by checking a False is found in status + """ status = fetch_nb_dependencies(files=FILES, repo=REPO, raw=RAW, branch=BRANCH) self.assertTrue(False in status) \ No newline at end of file From 302934bfd8b30ee1b33cdfb60ca36021df153746 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Thu, 23 Feb 2017 10:30:55 +1100 Subject: [PATCH 5/6] improve cleanup process of test by removing the downloaded file --- quantecon/util/tests/test_notebooks.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/quantecon/util/tests/test_notebooks.py b/quantecon/util/tests/test_notebooks.py index a4f5c49cc..298c59289 100644 --- a/quantecon/util/tests/test_notebooks.py +++ b/quantecon/util/tests/test_notebooks.py @@ -9,6 +9,7 @@ from quantecon.util import fetch_nb_dependencies import unittest +import os FILES = ['README.md'] REPO = "https://github.com/QuantEcon/QuantEcon.py" @@ -28,5 +29,9 @@ def test_fetch_nb_dependencies_overwrite(self): """ Run Second and Ensure file is skipped by checking a False is found in status """ - status = fetch_nb_dependencies(files=FILES, repo=REPO, raw=RAW, branch=BRANCH) - self.assertTrue(False in status) \ No newline at end of file + status = fetch_nb_dependencies(files=FILES, repo=REPO, raw=RAW, branch=BRANCH) #First will succeed + status = fetch_nb_dependencies(files=FILES, repo=REPO, raw=RAW, branch=BRANCH) #Second should skip + self.assertTrue(False in status) + + def tearDown(self): + os.remove("README.md") \ No newline at end of file From c5c90c1948eb380a93c7a3c7927032f5279a7ac1 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Thu, 23 Feb 2017 10:42:13 +1100 Subject: [PATCH 6/6] update to remove conflict with using README.md file as a test. File already exists in travis environment --- quantecon/util/tests/test_file.md | 1 + quantecon/util/tests/test_notebooks.py | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 quantecon/util/tests/test_file.md diff --git a/quantecon/util/tests/test_file.md b/quantecon/util/tests/test_file.md new file mode 100644 index 000000000..3ec45702f --- /dev/null +++ b/quantecon/util/tests/test_file.md @@ -0,0 +1 @@ +This is a Simple Test File for test_notebooks.py \ No newline at end of file diff --git a/quantecon/util/tests/test_notebooks.py b/quantecon/util/tests/test_notebooks.py index 298c59289..8e39e2dab 100644 --- a/quantecon/util/tests/test_notebooks.py +++ b/quantecon/util/tests/test_notebooks.py @@ -11,10 +11,11 @@ import unittest import os -FILES = ['README.md'] +FILES = ['test_file.md'] REPO = "https://github.com/QuantEcon/QuantEcon.py" RAW = "raw" BRANCH = "master" +FOLDER = "quantecon/util/tests/" class TestNotebookUtils(unittest.TestCase): @@ -22,16 +23,16 @@ def test_fetch_nb_dependencies(self): """ Run First and Test Download """ - status = fetch_nb_dependencies(files=FILES, repo=REPO, raw=RAW, branch=BRANCH) + status = fetch_nb_dependencies(files=FILES, repo=REPO, raw=RAW, branch=BRANCH, folder=FOLDER) self.assertFalse(False in status) def test_fetch_nb_dependencies_overwrite(self): """ Run Second and Ensure file is skipped by checking a False is found in status """ - status = fetch_nb_dependencies(files=FILES, repo=REPO, raw=RAW, branch=BRANCH) #First will succeed - status = fetch_nb_dependencies(files=FILES, repo=REPO, raw=RAW, branch=BRANCH) #Second should skip + status = fetch_nb_dependencies(files=FILES, repo=REPO, raw=RAW, branch=BRANCH, folder=FOLDER) #First will succeed + status = fetch_nb_dependencies(files=FILES, repo=REPO, raw=RAW, branch=BRANCH, folder=FOLDER) #Second should skip self.assertTrue(False in status) def tearDown(self): - os.remove("README.md") \ No newline at end of file + os.remove("test_file.md") \ No newline at end of file