From 4c8c3a07b674708430d2bfb6365c8de459e39eb4 Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Fri, 1 Mar 2024 13:34:44 -0700 Subject: [PATCH 01/11] fix for issue with source_download set to latest --- lib/pavilion/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pavilion/builder.py b/lib/pavilion/builder.py index 99b3fe859..f34e1a430 100644 --- a/lib/pavilion/builder.py +++ b/lib/pavilion/builder.py @@ -319,7 +319,7 @@ def _update_src(self): if (src_url is not None and ((src_download == 'missing' and found_src_path is None) - or src_download == 'latest')): + or src_download == 'latest' and found_src_path is None)): if self._download_dest is None: raise TestBuilderError( From 85f58b4afaa3103a101906d85ee1cbeb354f89c1 Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Mon, 4 Mar 2024 12:46:15 -0700 Subject: [PATCH 02/11] need to unlink the original file --- test/tests/builder_tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/tests/builder_tests.py b/test/tests/builder_tests.py index eb975d582..132dabf52 100644 --- a/test/tests/builder_tests.py +++ b/test/tests/builder_tests.py @@ -291,6 +291,9 @@ def test_src_urls(self): self._quick_test(config, build=False, finalize=False) self.assertEqual(orig_time, expected_path.stat().st_mtime) + if expected_path.exists(): + expected_path.unlink() + # Here it should be updated. We're playing a weird trick here though, # by pointing to a completely different url. config = copy.deepcopy(config) From fbd1320d4e2d5d0a329a19ae792f59b9dd52ee59 Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Tue, 5 Mar 2024 12:39:41 -0700 Subject: [PATCH 03/11] Revert "need to unlink the original file" This reverts commit 85f58b4afaa3103a101906d85ee1cbeb354f89c1. --- test/tests/builder_tests.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/tests/builder_tests.py b/test/tests/builder_tests.py index 132dabf52..eb975d582 100644 --- a/test/tests/builder_tests.py +++ b/test/tests/builder_tests.py @@ -291,9 +291,6 @@ def test_src_urls(self): self._quick_test(config, build=False, finalize=False) self.assertEqual(orig_time, expected_path.stat().st_mtime) - if expected_path.exists(): - expected_path.unlink() - # Here it should be updated. We're playing a weird trick here though, # by pointing to a completely different url. config = copy.deepcopy(config) From 02fba19efae2c87fc34a76d3aa7b89b254df9f85 Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Tue, 5 Mar 2024 12:40:40 -0700 Subject: [PATCH 04/11] Revert "fix for issue with source_download set to latest" This reverts commit 4c8c3a07b674708430d2bfb6365c8de459e39eb4. --- lib/pavilion/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pavilion/builder.py b/lib/pavilion/builder.py index f34e1a430..99b3fe859 100644 --- a/lib/pavilion/builder.py +++ b/lib/pavilion/builder.py @@ -319,7 +319,7 @@ def _update_src(self): if (src_url is not None and ((src_download == 'missing' and found_src_path is None) - or src_download == 'latest' and found_src_path is None)): + or src_download == 'latest')): if self._download_dest is None: raise TestBuilderError( From 72f9a64f687fe4e67ba0bf0c106878f32c98ac89 Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Thu, 21 Mar 2024 10:18:42 -0600 Subject: [PATCH 05/11] inverted logic (or vs and in not) --- lib/pavilion/wget.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pavilion/wget.py b/lib/pavilion/wget.py index 6afe93367..f3c737488 100644 --- a/lib/pavilion/wget.py +++ b/lib/pavilion/wget.py @@ -263,10 +263,10 @@ def update(pav_cfg, url, dest): # depends on the transfer encoding. It should match for any already # compressed files, but other data types are frequently compressed. elif (not ( - info.get('ETag') == head_data.get('ETag') or + info.get('ETag') == head_data.get('ETag') and # If the old content length is the same, it's probably # unchanged. Probably... - head_data.get('Content-Length') == info.get('Content-Length') or + head_data.get('Content-Length') == info.get('Content-Length') and # Or if the content length matches the actual size. head_data.get('Content-Length') == info['size'])): fetch = True From 395eae0b4fdfc439e9d438bd41fcd198b5b9fb49 Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Thu, 21 Mar 2024 10:51:24 -0600 Subject: [PATCH 06/11] fix for source download logic and tests for wget --- docs/requirements.txt | 5 +---- test/README.md | 1 - test/requirements.txt | 4 ++++ test/tests/wget_tests.py | 24 +++++++++++++++++++----- 4 files changed, 24 insertions(+), 10 deletions(-) mode change 100644 => 120000 docs/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index c9f923be0..000000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -# Requirements for building documentation. - -sphinx > 4.0 -sphinx_rtd_theme >= 1.0.0 diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 120000 index 000000000..9e3bd9d8c --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1 @@ +../test/requirements.txt \ No newline at end of file diff --git a/test/README.md b/test/README.md index 477d7baaa..280d92d95 100644 --- a/test/README.md +++ b/test/README.md @@ -30,7 +30,6 @@ python3 -m venv source /bin/activate pip install --upgrade pip pip install -r test/requirements.txt -pip install -r docs/requirements.txt ``` Then just activate your virtual environment before running tests. diff --git a/test/requirements.txt b/test/requirements.txt index c24fef5e3..ff5c89e0d 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,2 +1,6 @@ pylint matplotlib > 3 + +# Requirements for building documentation. +sphinx > 4.0 +sphinx_rtd_theme >= 1.0.0 diff --git a/test/tests/wget_tests.py b/test/tests/wget_tests.py index 1322d7bff..bce36dfa3 100644 --- a/test/tests/wget_tests.py +++ b/test/tests/wget_tests.py @@ -16,6 +16,7 @@ class TestWGet(PavTestCase): GET_TARGET = "https://github.com/lanl/Pavilion/raw/master/README.md" + GET_TARGET2 = "https://github.com/lanl/Pavilion/raw/master/RELEASE.txt" TARGET_HASH = '275fa3c8aeb10d145754388446be1f24bb16fb00' _logger = logging.getLogger(__file__) @@ -73,8 +74,6 @@ def test_update(self): # It should update the file if the info file isn't there and the # sizes don't match. ctime = dest_fn.stat().st_ctime - with dest_fn.open('ab') as dest_file: - dest_file.write(b'a') info_fn.unlink() try: wget.update(self.pav_cfg, self.GET_TARGET, dest_fn) @@ -86,7 +85,6 @@ def test_update(self): # We'll muck up the info file data, to force an update. db_data = { - 'ETag': 'nope', 'Content-Length': '-1' } with info_fn.open('w') as info_file: @@ -98,5 +96,21 @@ def test_update(self): new_ctime = dest_fn.stat().st_ctime self.assertNotEqual(new_ctime, ctime) - dest_fn.stat() - info_fn.stat() + ctime = new_ctime + # Checking if a remote file change forces an update + try: + wget.update(self.pav_cfg, self.GET_TARGET2, dest_fn) + except pavilion.errors.WGetError as err: + self.fail("Failed with: {}".format(err.args[0])) + new_ctime = dest_fn.stat().st_ctime + self.assertNotEqual(new_ctime, ctime) + + ctime = new_ctime + # Make sure no updates happen if everything is the same + try: + wget.update(self.pav_cfg, self.GET_TARGET2, dest_fn) + except pavilion.errors.WGetError as err: + self.fail("Failed with: {}".format(err.args[0])) + new_ctime = dest_fn.stat().st_ctime + self.assertEqual(new_ctime, ctime) + From f5143922c164f308ed548660f8793ebd215e31c1 Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Thu, 21 Mar 2024 11:25:08 -0600 Subject: [PATCH 07/11] debugging CI --- .github/workflows/unittests.yml | 4 ++-- lib/pavilion/wget.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index db72aaa4d..cbbc48e3b 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -10,7 +10,7 @@ jobs: # The debug job is meant for debugging CI related issues. debug: # Comment out the following line to enable. - if: ${{ false }} + # if: ${{ false }} runs-on: ubuntu-latest @@ -42,7 +42,7 @@ jobs: # Set run_tests to run only the specific tests you need to fix. - name: run_tests run: | - ./test/run_tests + ./test/run_tests -q -o update - name: zip_working_dir if: always() diff --git a/lib/pavilion/wget.py b/lib/pavilion/wget.py index f3c737488..06bf6283e 100644 --- a/lib/pavilion/wget.py +++ b/lib/pavilion/wget.py @@ -246,6 +246,7 @@ def update(pav_cfg, url, dest): # If the file doesn't exist, just get it. if not dest.exists(): + print( "does not exist" ) fetch = True else: head_data = head(pav_cfg, url) @@ -256,6 +257,7 @@ def update(pav_cfg, url, dest): # matching Content-Length and fetch it if we can't. if (not info_path.exists() and (head_data.get('Content-Length') != info['size'])): + print( "no content length match: ", info, head_data ) fetch = True # If we do have an info file and neither the ETag or content length @@ -269,6 +271,7 @@ def update(pav_cfg, url, dest): head_data.get('Content-Length') == info.get('Content-Length') and # Or if the content length matches the actual size. head_data.get('Content-Length') == info['size'])): + print( "content differs: ", info, head_data ) fetch = True if fetch: From 7a39cd9af23e5fd0c2bfd9e0bed07a0586d0db1f Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Thu, 21 Mar 2024 11:34:31 -0600 Subject: [PATCH 08/11] continue testing --- lib/pavilion/wget.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pavilion/wget.py b/lib/pavilion/wget.py index 06bf6283e..dcba3e02f 100644 --- a/lib/pavilion/wget.py +++ b/lib/pavilion/wget.py @@ -271,7 +271,10 @@ def update(pav_cfg, url, dest): head_data.get('Content-Length') == info.get('Content-Length') and # Or if the content length matches the actual size. head_data.get('Content-Length') == info['size'])): - print( "content differs: ", info, head_data ) + print( "ETag: ", info.get('ETag'), head_data.get('ETag') ) + print( "content length: ", info.get('Content-Length'), + head_data.get('Content-Length') ) + print( "content differs: ", info['size'] ) fetch = True if fetch: From b4e5e21bcc28bcc57908dd8330ac3c745b8ba4d7 Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Thu, 21 Mar 2024 11:44:18 -0600 Subject: [PATCH 09/11] debug --- lib/pavilion/wget.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/pavilion/wget.py b/lib/pavilion/wget.py index dcba3e02f..da4f994ff 100644 --- a/lib/pavilion/wget.py +++ b/lib/pavilion/wget.py @@ -268,9 +268,7 @@ def update(pav_cfg, url, dest): info.get('ETag') == head_data.get('ETag') and # If the old content length is the same, it's probably # unchanged. Probably... - head_data.get('Content-Length') == info.get('Content-Length') and - # Or if the content length matches the actual size. - head_data.get('Content-Length') == info['size'])): + head_data.get('Content-Length') == info.get('Content-Length') ): print( "ETag: ", info.get('ETag'), head_data.get('ETag') ) print( "content length: ", info.get('Content-Length'), head_data.get('Content-Length') ) From ae10e8aead8df2ee4905640a27395f69a4e75a42 Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Thu, 21 Mar 2024 11:45:45 -0600 Subject: [PATCH 10/11] debug --- lib/pavilion/wget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pavilion/wget.py b/lib/pavilion/wget.py index da4f994ff..30a16e9ca 100644 --- a/lib/pavilion/wget.py +++ b/lib/pavilion/wget.py @@ -268,7 +268,7 @@ def update(pav_cfg, url, dest): info.get('ETag') == head_data.get('ETag') and # If the old content length is the same, it's probably # unchanged. Probably... - head_data.get('Content-Length') == info.get('Content-Length') ): + head_data.get('Content-Length') == info.get('Content-Length')) ): print( "ETag: ", info.get('ETag'), head_data.get('ETag') ) print( "content length: ", info.get('Content-Length'), head_data.get('Content-Length') ) From 82605ceb3c4b52c8a11dd86c02e64b77529a15f2 Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Thu, 21 Mar 2024 12:51:04 -0600 Subject: [PATCH 11/11] fix for source download was issue with header caching --- .github/workflows/unittests.yml | 4 ++-- lib/pavilion/wget.py | 9 +++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index cbbc48e3b..db72aaa4d 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -10,7 +10,7 @@ jobs: # The debug job is meant for debugging CI related issues. debug: # Comment out the following line to enable. - # if: ${{ false }} + if: ${{ false }} runs-on: ubuntu-latest @@ -42,7 +42,7 @@ jobs: # Set run_tests to run only the specific tests you need to fix. - name: run_tests run: | - ./test/run_tests -q -o update + ./test/run_tests - name: zip_working_dir if: always() diff --git a/lib/pavilion/wget.py b/lib/pavilion/wget.py index 30a16e9ca..9d1d7837d 100644 --- a/lib/pavilion/wget.py +++ b/lib/pavilion/wget.py @@ -116,10 +116,12 @@ def head(pav_cfg, url): redirects = 0 + headers = { 'Cache-Control':'no-cache' } try: response = session.head(url, proxies=proxies, verify=ca_cert_path(), + headers=headers, timeout=pav_cfg.wget_timeout) # The location header is the redirect location. While the requests # library resolves these automatically, it still returns the first @@ -135,6 +137,7 @@ def head(pav_cfg, url): response = session.head(redirect_url, proxies=proxies, verify=ca_cert_path(), + headers=headers, timeout=pav_cfg.wget_timeout) except requests.exceptions.RequestException as err: @@ -246,7 +249,6 @@ def update(pav_cfg, url, dest): # If the file doesn't exist, just get it. if not dest.exists(): - print( "does not exist" ) fetch = True else: head_data = head(pav_cfg, url) @@ -257,7 +259,6 @@ def update(pav_cfg, url, dest): # matching Content-Length and fetch it if we can't. if (not info_path.exists() and (head_data.get('Content-Length') != info['size'])): - print( "no content length match: ", info, head_data ) fetch = True # If we do have an info file and neither the ETag or content length @@ -269,10 +270,6 @@ def update(pav_cfg, url, dest): # If the old content length is the same, it's probably # unchanged. Probably... head_data.get('Content-Length') == info.get('Content-Length')) ): - print( "ETag: ", info.get('ETag'), head_data.get('ETag') ) - print( "content length: ", info.get('Content-Length'), - head_data.get('Content-Length') ) - print( "content differs: ", info['size'] ) fetch = True if fetch: