From 2591294dff301d4b5085e282607b9d693b71693d Mon Sep 17 00:00:00 2001 From: Ash Berlin-Taylor Date: Mon, 8 Jun 2020 09:33:17 +0100 Subject: [PATCH] Add note about using dag_run.conf in BashOperator (#9143) (cherry picked from commit 4d8599e8b0520ff4226fbad72f724afae50fdd08) (cherry picked from commit 74d6d1ddc79a74799ec301fbf5f84954252e0afe) --- .../example_trigger_target_dag.py | 4 +-- airflow/operators/bash_operator.py | 33 ++++++++++++++++++- docs/howto/operator/bash.rst | 31 +++++++++++++++++ 3 files changed, 65 insertions(+), 3 deletions(-) diff --git a/airflow/example_dags/example_trigger_target_dag.py b/airflow/example_dags/example_trigger_target_dag.py index c583439efa7c1..2129ea5032257 100644 --- a/airflow/example_dags/example_trigger_target_dag.py +++ b/airflow/example_dags/example_trigger_target_dag.py @@ -66,7 +66,7 @@ def run_this_func(ds, **kwargs): # You can also access the DagRun object in templates bash_task = BashOperator( task_id="bash_task", - bash_command='echo "Here is the message: ' - '{{ dag_run.conf["message"] if dag_run else "" }}" ', + bash_command='echo "Here is the message: $message"', + env={'message': '{{ dag_run.conf["message"] if dag_run else "" }}'}, dag=dag, ) diff --git a/airflow/operators/bash_operator.py b/airflow/operators/bash_operator.py index ec1058c7fb55b..e27e74656de37 100644 --- a/airflow/operators/bash_operator.py +++ b/airflow/operators/bash_operator.py @@ -33,7 +33,7 @@ class BashOperator(BaseOperator): - """ + r""" Execute a Bash script, command or set of commands. .. seealso:: @@ -53,6 +53,37 @@ class BashOperator(BaseOperator): :type env: dict :param output_encoding: Output encoding of bash command :type output_encoding: str + + .. warning:: + + Care should be taken with "user" input or when using Jinja templates in the + ``bash_command``, as this bash operator does not perform any escaping or + sanitization of the command. + + This applies mostly to using "dag_run" conf, as that can be submitted via + users in the Web UI. Most of the default template variables are not at + risk. + + For example, do **not** do this: + + .. code-block:: python + + bash_task = BashOperator( + task_id="bash_task", + bash_command='echo "Here is the message: \'{{ dag_run.conf["message"] if dag_run else "" }}\'"', + ) + + Instead, you should pass this via the ``env`` kwarg and use double-quotes + inside the bash_command, as below: + + .. code-block:: python + + bash_task = BashOperator( + task_id="bash_task", + bash_command='echo "here is the message: \'$message\'"', + env={'message': '{{ dag_run.conf["message"] if dag_run else "" }}'}, + ) + """ template_fields = ('bash_command', 'env') template_ext = ('.sh', '.bash',) diff --git a/docs/howto/operator/bash.rst b/docs/howto/operator/bash.rst index 9828122f8d13d..7956b2efde7ee 100644 --- a/docs/howto/operator/bash.rst +++ b/docs/howto/operator/bash.rst @@ -41,6 +41,37 @@ You can use :ref:`Jinja templates ` to parameterize the :start-after: [START howto_operator_bash_template] :end-before: [END howto_operator_bash_template] + +.. warning:: + + Care should be taken with "user" input or when using Jinja templates in the + ``bash_command``, as this bash operator does not perform any escaping or + sanitization of the command. + + This applies mostly to using "dag_run" conf, as that can be submitted via + users in the Web UI. Most of the default template variables are not at + risk. + +For example, do **not** do this: + +.. code-block:: python + + bash_task = BashOperator( + task_id="bash_task", + bash_command='echo "Here is the message: \'{{ dag_run.conf["message"] if dag_run else "" }}\'"', + ) + +Instead, you should pass this via the ``env`` kwarg and use double-quotes +inside the bash_command, as below: + +.. code-block:: python + + bash_task = BashOperator( + task_id="bash_task", + bash_command='echo "here is the message: \'$message\'"', + env={'message': '{{ dag_run.conf["message"] if dag_run else "" }}'}, + ) + Troubleshooting ---------------