From 55b31dbfa9ff371bfcb927f7c6b21f3f8e82f8b9 Mon Sep 17 00:00:00 2001 From: David Katz <41651296+DavidKatz-il@users.noreply.github.com> Date: Tue, 7 Jul 2020 00:16:52 +0300 Subject: [PATCH] update 'dask_executor' docstring. (#2684) --- .../dagster-dask/dagster_dask/executor.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/python_modules/libraries/dagster-dask/dagster_dask/executor.py b/python_modules/libraries/dagster-dask/dagster_dask/executor.py index 99316e3632b6c..c7e15090bcd7c 100644 --- a/python_modules/libraries/dagster-dask/dagster_dask/executor.py +++ b/python_modules/libraries/dagster-dask/dagster_dask/executor.py @@ -60,9 +60,12 @@ def dask_executor(init_context): '''Dask-based executor. + The 'cluster' can be one of the following: + ('local', 'yarn', 'ssh', 'pbs', 'moab', 'sge', 'lsf', 'slurm', 'oar', 'kube'). + If the Dask executor is used without providing executor-specific config, a local Dask cluster will be created (as when calling :py:class:`dask.distributed.Client() ` - without specifying the scheduler address). + with :py:class:`dask.distributed.LocalCluster() `). The Dask executor optionally takes the following config: @@ -70,15 +73,11 @@ def dask_executor(init_context): cluster: { - local?: # The cluster type, one of the following ('local', 'yarn', 'ssh', 'pbs', 'moab', 'sge', 'lsf', 'slurm', 'oar', 'kube'). + local?: # takes distributed.LocalCluster parameters { - address?: '127.0.0.1:8786', # The address of a Dask scheduler timeout?: 5, # Timeout duration for initial connection to the scheduler - scheduler_file?: '/path/to/file' # Path to a file with scheduler information - # Whether to connect directly to the workers, or ask the scheduler to serve as - # intermediary - direct_to_workers?: False, - heartbeat_interval?: 1000, # Time in milliseconds between heartbeats to scheduler + n_workers?: 4 # Number of workers to start + threads_per_worker?: 1 # Number of threads per each worker } }