From e9eb8016907f8687e68bd9790df6f4bd1b8211fa Mon Sep 17 00:00:00 2001 From: Mike Purvis Date: Fri, 22 Jul 2016 22:59:22 -0400 Subject: [PATCH 1/4] Add new initial command stage to mutex all reads of the installspace. --- catkin_tools/execution/executor.py | 5 +-- catkin_tools/execution/jobs.py | 7 ++-- catkin_tools/jobs/catkin.py | 37 ++++++++++++++++++--- catkin_tools/jobs/cmake.py | 34 +++++++++++++++++-- catkin_tools/jobs/utils.py | 45 +++++++++++--------------- catkin_tools/verbs/catkin_build/cli.py | 5 +-- 6 files changed, 88 insertions(+), 45 deletions(-) diff --git a/catkin_tools/execution/executor.py b/catkin_tools/execution/executor.py index 9d07ba4e..7ab8a179 100644 --- a/catkin_tools/execution/executor.py +++ b/catkin_tools/execution/executor.py @@ -59,9 +59,6 @@ def async_job(verb, job, threadpool, locks, event_queue, log_path): # Jobs start occuping a jobserver job occupying_job = True - # Load environment for this job - job_env = job.getenv(os.environ) - # Execute each stage of this job for stage in job.stages: # Logger reference in this scope for error reporting @@ -102,7 +99,7 @@ def async_job(verb, job, threadpool, locks, event_queue, log_path): while True: try: # Update the environment for this stage (respects overrides) - stage.update_env(job_env) + stage.update_env(job.env) # Get the logger protocol_type = stage.logger_factory(verb, job.jid, stage.label, event_queue, log_path) diff --git a/catkin_tools/execution/jobs.py b/catkin_tools/execution/jobs.py index bdd6d20b..1826926e 100644 --- a/catkin_tools/execution/jobs.py +++ b/catkin_tools/execution/jobs.py @@ -24,7 +24,7 @@ class Job(object): """A Job is a series of operations, each of which is considered a "stage" of the job.""" - def __init__(self, jid, deps, env_loader, stages, continue_on_failure=True): + def __init__(self, jid, deps, env, stages, continue_on_failure=True): """ jid: Unique job identifier deps: Dependencies (in terms of other jid's) @@ -33,7 +33,7 @@ def __init__(self, jid, deps, env_loader, stages, continue_on_failure=True): """ self.jid = jid self.deps = deps - self.env_loader = env_loader + self.env = env self.stages = stages self.continue_on_failure = continue_on_failure @@ -48,6 +48,3 @@ def all_deps_succeeded(self, completed_jobs): def any_deps_failed(self, completed_jobs): """Return True if any dependencies which have been completed have failed.""" return any([not completed_jobs.get(dep_id, True) for dep_id in self.deps]) - - def getenv(self, env): - return self.env_loader(env) diff --git a/catkin_tools/jobs/catkin.py b/catkin_tools/jobs/catkin.py index 91d05133..a82fac63 100644 --- a/catkin_tools/jobs/catkin.py +++ b/catkin_tools/jobs/catkin.py @@ -35,7 +35,7 @@ from .commands.make import MAKE_EXEC from .utils import copyfiles -from .utils import get_env_loader +from .utils import load_env from .utils import makedirs from .utils import rmfiles @@ -350,10 +350,23 @@ def create_catkin_build_job(context, package, package_path, dependencies, force_ install_space = context.package_install_space(package) # Package metadata path metadata_path = context.package_metadata_path(package) + # Environment dictionary for the job, which will be built + # up by the executions in the getenv stage. + job_env = dict(os.environ) # Create job stages stages = [] + # Get environment for job. + stages.append(FunctionStage( + 'getenv', + load_env, + locked_resource='installspace', + job_env=job_env, + package=package, + context=context + )) + # Create package build space stages.append(FunctionStage( 'mkdir', @@ -478,7 +491,7 @@ def create_catkin_build_job(context, package, package_path, dependencies, force_ return Job( jid=package.name, deps=dependencies, - env_loader=get_env_loader(package, context), + env=job_env, stages=stages) @@ -493,12 +506,26 @@ def create_catkin_clean_job( clean_install): """Generate a Job that cleans a catkin package""" - stages = [] - # Package build space path build_space = context.package_build_space(package) # Package metadata path metadata_path = context.package_metadata_path(package) + # Environment dictionary for the job, which will be built + # up by the executions in the getenv stage. + job_env = dict(os.environ) + + # Create job stages + stages = [] + + # Get environment for job. + stages.append(FunctionStage( + 'getenv', + load_env, + locked_resource='installspace', + job_env=job_env, + package=package, + context=context + )) # Remove installed files if clean_install: @@ -566,7 +593,7 @@ def create_catkin_clean_job( return Job( jid=package.name, deps=dependencies, - env_loader=get_env_loader(package, context), + env=job_env, stages=stages) diff --git a/catkin_tools/jobs/cmake.py b/catkin_tools/jobs/cmake.py index e751eea2..c8b774b7 100644 --- a/catkin_tools/jobs/cmake.py +++ b/catkin_tools/jobs/cmake.py @@ -31,7 +31,7 @@ from .commands.make import MAKE_EXEC from .utils import copyfiles -from .utils import get_env_loader +from .utils import load_env from .utils import makedirs from .utils import rmfiles @@ -217,6 +217,9 @@ def create_cmake_build_job(context, package, package_path, dependencies, force_c build_space = context.package_build_space(package) # Package metadata path metadata_path = context.package_metadata_path(package) + # Environment dictionary for the job, which will be built + # up by the executions in the getenv stage. + job_env = dict(os.environ) # Get actual staging path dest_path = context.package_dest_path(package) @@ -225,6 +228,16 @@ def create_cmake_build_job(context, package, package_path, dependencies, force_c # Create job stages stages = [] + # Get environment for job. + stages.append(FunctionStage( + 'getenv', + load_env, + locked_resource='installspace', + job_env=job_env, + package=package, + context=context + )) + # Create package build space stages.append(FunctionStage( 'mkdir', @@ -321,7 +334,7 @@ def create_cmake_build_job(context, package, package_path, dependencies, force_c return Job( jid=package.name, deps=dependencies, - env_loader=get_env_loader(package, context), + env=job_env, stages=stages) @@ -340,9 +353,24 @@ def create_cmake_clean_job( build_space = context.package_build_space(package) # Package metadata path metadata_path = context.package_metadata_path(package) + # Environment dictionary for the job, which will be built + # up by the executions in the getenv stage. + job_env = dict(os.environ) + # Create job stages stages = [] + # Get environment for job. + stages.append(FunctionStage( + 'getenv', + load_env, + locked_resource='installspace', + job_env=job_env, + package=package, + context=context + )) + + # Remove installed files if clean_install and context.install: installed_files = get_installed_files(context.package_metadata_path(package)) stages.append(FunctionStage( @@ -381,7 +409,7 @@ def create_cmake_clean_job( return Job( jid=package.name, deps=dependencies, - env_loader=get_env_loader(package, context), + env=job_env, stages=stages) diff --git a/catkin_tools/jobs/utils.py b/catkin_tools/jobs/utils.py index 9473ab98..cd42cbf7 100644 --- a/catkin_tools/jobs/utils.py +++ b/catkin_tools/jobs/utils.py @@ -50,32 +50,25 @@ def get_env_loaders(package, context): return sources -def get_env_loader(package, context): - """This function returns a function object which extends a base environment - based on a set of environments to load.""" - - def load_env(base_env): - # Copy the base environment to extend - job_env = dict(base_env) - # Get the paths to the env loaders - env_loader_paths = get_env_loaders(package, context) - # If DESTDIR is set, set _CATKIN_SETUP_DIR as well - if context.destdir is not None: - job_env['_CATKIN_SETUP_DIR'] = context.package_dest_path(package) - - for env_loader_path in env_loader_paths: - # print(' - Loading resultspace env from: {}'.format(env_loader_path)) - resultspace_env = get_resultspace_environment( - os.path.split(env_loader_path)[0], - base_env=job_env, - quiet=True, - cached=context.use_env_cache, - strict=False) - job_env.update(resultspace_env) - - return job_env - - return load_env +def load_env(logger, event_queue, job_env, package, context): + # Get the paths to the env loaders + env_loader_paths = get_env_loaders(package, context) + # If DESTDIR is set, set _CATKIN_SETUP_DIR as well + if context.destdir is not None: + job_env['_CATKIN_SETUP_DIR'] = context.package_dest_path(package) + + for env_loader_path in env_loader_paths: + if logger: + logger.out('Loading environment from: {}'.format(env_loader_path)) + resultspace_env = get_resultspace_environment( + os.path.split(env_loader_path)[0], + base_env=job_env, + quiet=True, + cached=context.use_env_cache, + strict=False) + job_env.update(resultspace_env) + + return 0 def makedirs(logger, event_queue, path): diff --git a/catkin_tools/verbs/catkin_build/cli.py b/catkin_tools/verbs/catkin_build/cli.py index f6b3b7dc..f5bb4c7c 100644 --- a/catkin_tools/verbs/catkin_build/cli.py +++ b/catkin_tools/verbs/catkin_build/cli.py @@ -48,7 +48,7 @@ import catkin_tools.execution.job_server as job_server -from catkin_tools.jobs.utils import get_env_loader +from catkin_tools.jobs.utils import load_env from catkin_tools.metadata import find_enclosing_workspace from catkin_tools.metadata import get_metadata @@ -228,7 +228,8 @@ def print_build_env(context, package_name): # Load the environment used by this package for building for pth, pkg in workspace_packages.items(): if pkg.name == package_name: - environ = get_env_loader(pkg, context)(os.environ) + environ = dict(os.environ) + load_env(None, None, environ, pkg, context) print(format_env_dict(environ)) return 0 print('[build] Error: Package `{}` not in workspace.'.format(package_name), From da435947c2666968bfe8c3b52963381d79374641 Mon Sep 17 00:00:00 2001 From: Mike Purvis Date: Wed, 10 Aug 2016 16:15:24 -0400 Subject: [PATCH 2/4] Changed load_env -> loadenv. This better aligns the function name with other similar helpers in catkin_tools.jobs.utils. --- catkin_tools/jobs/catkin.py | 8 ++++---- catkin_tools/jobs/cmake.py | 8 ++++---- catkin_tools/jobs/utils.py | 2 +- catkin_tools/verbs/catkin_build/cli.py | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/catkin_tools/jobs/catkin.py b/catkin_tools/jobs/catkin.py index a82fac63..f400b682 100644 --- a/catkin_tools/jobs/catkin.py +++ b/catkin_tools/jobs/catkin.py @@ -35,7 +35,7 @@ from .commands.make import MAKE_EXEC from .utils import copyfiles -from .utils import load_env +from .utils import loadenv from .utils import makedirs from .utils import rmfiles @@ -357,10 +357,10 @@ def create_catkin_build_job(context, package, package_path, dependencies, force_ # Create job stages stages = [] - # Get environment for job. + # Load environment for job. stages.append(FunctionStage( - 'getenv', - load_env, + 'loadenv', + loadenv, locked_resource='installspace', job_env=job_env, package=package, diff --git a/catkin_tools/jobs/cmake.py b/catkin_tools/jobs/cmake.py index c8b774b7..b60114e3 100644 --- a/catkin_tools/jobs/cmake.py +++ b/catkin_tools/jobs/cmake.py @@ -31,7 +31,7 @@ from .commands.make import MAKE_EXEC from .utils import copyfiles -from .utils import load_env +from .utils import loadenv from .utils import makedirs from .utils import rmfiles @@ -228,10 +228,10 @@ def create_cmake_build_job(context, package, package_path, dependencies, force_c # Create job stages stages = [] - # Get environment for job. + # Load environment for job. stages.append(FunctionStage( - 'getenv', - load_env, + 'loadenv', + loadenv, locked_resource='installspace', job_env=job_env, package=package, diff --git a/catkin_tools/jobs/utils.py b/catkin_tools/jobs/utils.py index cd42cbf7..eefd9b0a 100644 --- a/catkin_tools/jobs/utils.py +++ b/catkin_tools/jobs/utils.py @@ -50,7 +50,7 @@ def get_env_loaders(package, context): return sources -def load_env(logger, event_queue, job_env, package, context): +def loadenv(logger, event_queue, job_env, package, context): # Get the paths to the env loaders env_loader_paths = get_env_loaders(package, context) # If DESTDIR is set, set _CATKIN_SETUP_DIR as well diff --git a/catkin_tools/verbs/catkin_build/cli.py b/catkin_tools/verbs/catkin_build/cli.py index f5bb4c7c..460a3466 100644 --- a/catkin_tools/verbs/catkin_build/cli.py +++ b/catkin_tools/verbs/catkin_build/cli.py @@ -48,7 +48,7 @@ import catkin_tools.execution.job_server as job_server -from catkin_tools.jobs.utils import load_env +from catkin_tools.jobs.utils import loadenv from catkin_tools.metadata import find_enclosing_workspace from catkin_tools.metadata import get_metadata @@ -229,7 +229,7 @@ def print_build_env(context, package_name): for pth, pkg in workspace_packages.items(): if pkg.name == package_name: environ = dict(os.environ) - load_env(None, None, environ, pkg, context) + loadenv(None, None, environ, pkg, context) print(format_env_dict(environ)) return 0 print('[build] Error: Package `{}` not in workspace.'.format(package_name), From 22d2c8230175ecbaa75fa7107d2ae4801fde1235 Mon Sep 17 00:00:00 2001 From: Mike Purvis Date: Wed, 10 Aug 2016 15:49:43 -0400 Subject: [PATCH 3/4] Drop job env from clean jobs. --- catkin_tools/jobs/catkin.py | 22 +++++----------------- catkin_tools/jobs/cmake.py | 19 +++---------------- 2 files changed, 8 insertions(+), 33 deletions(-) diff --git a/catkin_tools/jobs/catkin.py b/catkin_tools/jobs/catkin.py index f400b682..f7a2bec8 100644 --- a/catkin_tools/jobs/catkin.py +++ b/catkin_tools/jobs/catkin.py @@ -351,7 +351,7 @@ def create_catkin_build_job(context, package, package_path, dependencies, force_ # Package metadata path metadata_path = context.package_metadata_path(package) # Environment dictionary for the job, which will be built - # up by the executions in the getenv stage. + # up by the executions in the loadenv stage. job_env = dict(os.environ) # Create job stages @@ -506,26 +506,14 @@ def create_catkin_clean_job( clean_install): """Generate a Job that cleans a catkin package""" + stages = [] + # Package build space path build_space = context.package_build_space(package) # Package metadata path metadata_path = context.package_metadata_path(package) - # Environment dictionary for the job, which will be built - # up by the executions in the getenv stage. - job_env = dict(os.environ) - - # Create job stages - stages = [] - - # Get environment for job. - stages.append(FunctionStage( - 'getenv', - load_env, - locked_resource='installspace', - job_env=job_env, - package=package, - context=context - )) + # Environment dictionary for the job, empty for a clean job + job_env = {} # Remove installed files if clean_install: diff --git a/catkin_tools/jobs/cmake.py b/catkin_tools/jobs/cmake.py index b60114e3..5d503304 100644 --- a/catkin_tools/jobs/cmake.py +++ b/catkin_tools/jobs/cmake.py @@ -218,7 +218,7 @@ def create_cmake_build_job(context, package, package_path, dependencies, force_c # Package metadata path metadata_path = context.package_metadata_path(package) # Environment dictionary for the job, which will be built - # up by the executions in the getenv stage. + # up by the executions in the loadenv stage. job_env = dict(os.environ) # Get actual staging path @@ -353,24 +353,11 @@ def create_cmake_clean_job( build_space = context.package_build_space(package) # Package metadata path metadata_path = context.package_metadata_path(package) - # Environment dictionary for the job, which will be built - # up by the executions in the getenv stage. - job_env = dict(os.environ) + # Environment dictionary for the job, empty for a clean job + job_env = {} - # Create job stages stages = [] - # Get environment for job. - stages.append(FunctionStage( - 'getenv', - load_env, - locked_resource='installspace', - job_env=job_env, - package=package, - context=context - )) - - # Remove installed files if clean_install and context.install: installed_files = get_installed_files(context.package_metadata_path(package)) stages.append(FunctionStage( From 2aaa8cb25314c6cbbf7a45df06337017b3c1341c Mon Sep 17 00:00:00 2001 From: Mike Purvis Date: Tue, 16 Aug 2016 19:40:02 -0400 Subject: [PATCH 4/4] Drop unused 'import os' --- catkin_tools/execution/executor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/catkin_tools/execution/executor.py b/catkin_tools/execution/executor.py index 7ab8a179..c9126e59 100644 --- a/catkin_tools/execution/executor.py +++ b/catkin_tools/execution/executor.py @@ -14,7 +14,6 @@ from __future__ import print_function -import os import traceback from itertools import tee