diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index c9f923be0..000000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -# Requirements for building documentation. - -sphinx > 4.0 -sphinx_rtd_theme >= 1.0.0 diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 120000 index 000000000..9e3bd9d8c --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1 @@ +../test/requirements.txt \ No newline at end of file diff --git a/lib/pavilion/wget.py b/lib/pavilion/wget.py index 6afe93367..9d1d7837d 100644 --- a/lib/pavilion/wget.py +++ b/lib/pavilion/wget.py @@ -116,10 +116,12 @@ def head(pav_cfg, url): redirects = 0 + headers = { 'Cache-Control':'no-cache' } try: response = session.head(url, proxies=proxies, verify=ca_cert_path(), + headers=headers, timeout=pav_cfg.wget_timeout) # The location header is the redirect location. While the requests # library resolves these automatically, it still returns the first @@ -135,6 +137,7 @@ def head(pav_cfg, url): response = session.head(redirect_url, proxies=proxies, verify=ca_cert_path(), + headers=headers, timeout=pav_cfg.wget_timeout) except requests.exceptions.RequestException as err: @@ -263,12 +266,10 @@ def update(pav_cfg, url, dest): # depends on the transfer encoding. It should match for any already # compressed files, but other data types are frequently compressed. elif (not ( - info.get('ETag') == head_data.get('ETag') or + info.get('ETag') == head_data.get('ETag') and # If the old content length is the same, it's probably # unchanged. Probably... - head_data.get('Content-Length') == info.get('Content-Length') or - # Or if the content length matches the actual size. - head_data.get('Content-Length') == info['size'])): + head_data.get('Content-Length') == info.get('Content-Length')) ): fetch = True if fetch: diff --git a/test/README.md b/test/README.md index 477d7baaa..280d92d95 100644 --- a/test/README.md +++ b/test/README.md @@ -30,7 +30,6 @@ python3 -m venv source /bin/activate pip install --upgrade pip pip install -r test/requirements.txt -pip install -r docs/requirements.txt ``` Then just activate your virtual environment before running tests. diff --git a/test/requirements.txt b/test/requirements.txt index c24fef5e3..ff5c89e0d 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,2 +1,6 @@ pylint matplotlib > 3 + +# Requirements for building documentation. +sphinx > 4.0 +sphinx_rtd_theme >= 1.0.0 diff --git a/test/tests/wget_tests.py b/test/tests/wget_tests.py index 1322d7bff..bce36dfa3 100644 --- a/test/tests/wget_tests.py +++ b/test/tests/wget_tests.py @@ -16,6 +16,7 @@ class TestWGet(PavTestCase): GET_TARGET = "https://github.com/lanl/Pavilion/raw/master/README.md" + GET_TARGET2 = "https://github.com/lanl/Pavilion/raw/master/RELEASE.txt" TARGET_HASH = '275fa3c8aeb10d145754388446be1f24bb16fb00' _logger = logging.getLogger(__file__) @@ -73,8 +74,6 @@ def test_update(self): # It should update the file if the info file isn't there and the # sizes don't match. ctime = dest_fn.stat().st_ctime - with dest_fn.open('ab') as dest_file: - dest_file.write(b'a') info_fn.unlink() try: wget.update(self.pav_cfg, self.GET_TARGET, dest_fn) @@ -86,7 +85,6 @@ def test_update(self): # We'll muck up the info file data, to force an update. db_data = { - 'ETag': 'nope', 'Content-Length': '-1' } with info_fn.open('w') as info_file: @@ -98,5 +96,21 @@ def test_update(self): new_ctime = dest_fn.stat().st_ctime self.assertNotEqual(new_ctime, ctime) - dest_fn.stat() - info_fn.stat() + ctime = new_ctime + # Checking if a remote file change forces an update + try: + wget.update(self.pav_cfg, self.GET_TARGET2, dest_fn) + except pavilion.errors.WGetError as err: + self.fail("Failed with: {}".format(err.args[0])) + new_ctime = dest_fn.stat().st_ctime + self.assertNotEqual(new_ctime, ctime) + + ctime = new_ctime + # Make sure no updates happen if everything is the same + try: + wget.update(self.pav_cfg, self.GET_TARGET2, dest_fn) + except pavilion.errors.WGetError as err: + self.fail("Failed with: {}".format(err.args[0])) + new_ctime = dest_fn.stat().st_ctime + self.assertEqual(new_ctime, ctime) +