From c77707f12543f0791d2a6d56fdf1ad1ed28a8066 Mon Sep 17 00:00:00 2001
From: Maria Francine Therese Ruiz Lapid <lapid@lanl.gov>
Date: Tue, 19 Mar 2024 13:00:52 -0600
Subject: [PATCH 1/2] WIP: added source_copy: False option
---
lib/pavilion/builder.py | 20 +++++++++++++-------
lib/pavilion/test_config/file_format.py | 4 ++++
lib/pavilion/utils.py | 14 ++++++++++++++
3 files changed, 31 insertions(+), 7 deletions(-)
diff --git a/lib/pavilion/builder.py b/lib/pavilion/builder.py
index 99b3fe859..43d4f9feb 100644
--- a/lib/pavilion/builder.py
+++ b/lib/pavilion/builder.py
@@ -26,6 +26,8 @@
from pavilion.test_config import parse_timeout
from pavilion.test_config.spack import SpackEnvConfig
+from pavilion.output import dbg_print
+
class TestBuilder:
"""Manages a test build and their organization.
@@ -638,7 +640,6 @@ def _setup_build_dir(self, dest, tracker: BuildTracker):
:param tracker: Build tracker for this build.
:return: None
"""
-
umask = os.umask(0)
os.umask(umask)
@@ -666,18 +667,23 @@ def _setup_build_dir(self, dest, tracker: BuildTracker):
elif src_path.is_dir():
# Recursively copy the src directory to the build directory.
+ # FRANCINE: add option to symlink everything recursively rather than copy
tracker.update(
state=STATES.BUILDING,
note=("Copying source directory {} for build {} "
"as the build directory."
.format(src_path, dest)))
- utils.copytree(
- src_path.as_posix(),
- dest.as_posix(),
- copy_function=shutil.copyfile,
- copystat=utils.make_umask_filtered_copystat(umask),
- symlinks=True)
+ source_copy = self._config.get('source_copy')
+ if source_copy.lower() == 'true':
+ utils.copytree(
+ src_path.as_posix(),
+ dest.as_posix(),
+ copy_function=shutil.copyfile,
+ copystat=utils.make_umask_filtered_copystat(umask),
+ symlinks=True)
+ else:
+ utils.symlinktree(src_path, dest)
elif src_path.is_file():
category, subtype = utils.get_mime_type(src_path)
diff --git a/lib/pavilion/test_config/file_format.py b/lib/pavilion/test_config/file_format.py
index 9b95bbe98..a7fcc2938 100644
--- a/lib/pavilion/test_config/file_format.py
+++ b/lib/pavilion/test_config/file_format.py
@@ -663,6 +663,10 @@ class TestConfigLoader(yc.YamlConfigLoader):
"source, tracking changes by "
"file size/timestamp/hash."
),
+ yc.StrElem(
+ 'source_copy', default='True', choices=['true', 'false', 'True', 'False'],
+ help_text="Whether to copy everything in source_path into the working dir."
+ ),
yc.KeyedElem(
'spack', elements=[
yc.ListElem(
diff --git a/lib/pavilion/utils.py b/lib/pavilion/utils.py
index d791d1680..c9fdcc449 100644
--- a/lib/pavilion/utils.py
+++ b/lib/pavilion/utils.py
@@ -14,6 +14,8 @@
from typing import Iterator, Union, TextIO
from typing import List, Dict
+from pavilion.output import dbg_print
+
def glob_to_re(glob):
"""Translate the given glob to one that is compatible with (extended) grep.
@@ -193,6 +195,18 @@ def path_is_external(path: Path):
return not_up_refs - up_refs <= 0
+def symlinktree(source_directory, destination_directory):
+ for root, dirs, files in os.walk(source_directory):
+ for file in files:
+ src_path = os.path.join(root, file)
+ rel_path = os.path.relpath(src_path, source_directory)
+ dst_path = os.path.join(destination_directory, rel_path)
+
+ # Create
+ os.makedirs(os.path.dirname(dst_path), exist_ok=True)
+ os.symlink(src_path, dst_path)
+
+
def flat_walk(path, *args, **kwargs) -> Iterator[Path]:
"""Perform an os.walk on path, but simply generate each item walked over.
From f1e1690054a6ecf7504f566be051c2e608cdb299 Mon Sep 17 00:00:00 2001
From: Maria Francine Therese Ruiz Lapid <lapid@lanl.gov>
Date: Tue, 19 Mar 2024 13:41:06 -0600
Subject: [PATCH 2/2] added some documentation
---
lib/pavilion/utils.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/lib/pavilion/utils.py b/lib/pavilion/utils.py
index c9fdcc449..8a5d3f092 100644
--- a/lib/pavilion/utils.py
+++ b/lib/pavilion/utils.py
@@ -196,14 +196,17 @@ def path_is_external(path: Path):
def symlinktree(source_directory, destination_directory):
+ """Recursively create symlinks from test source directory to builds directory in working_dir"""
+
for root, dirs, files in os.walk(source_directory):
for file in files:
src_path = os.path.join(root, file)
rel_path = os.path.relpath(src_path, source_directory)
dst_path = os.path.join(destination_directory, rel_path)
- # Create
+ # Create parent dir if it doesn't already exist
os.makedirs(os.path.dirname(dst_path), exist_ok=True)
+ # Create the symlink
os.symlink(src_path, dst_path)