Merge pull request #764 from MetOffice/761_move_output_under_workflow…

…_run_dir

Move all website files under the workflow shared directory

cset-workflow/app/housekeeping/bin/housekeep.sh

@@ -22,5 +22,5 @@ then
     # Housekeeping: Standard
     echo 'Removing intermediate data.'
     rm -rfv -- "$CYLC_WORKFLOW_SHARE_DIR"/cycle/*/data \
-              "$CYLC_WORKFLOW_SHARE_DIR"/plots/*/intermediate
+              "$CYLC_WORKFLOW_SHARE_DIR"/web/plots/*/intermediate
 fi

cset-workflow/app/install_website_skeleton/bin/install-website.sh

@@ -1,28 +1,28 @@
 #!/bin/bash
 
-# Copies the static files for the web interface into the correct location,
-# optionally removing previous files there.
+# Copies the static files for the web interface into the correct location, and
+# creates a symbolic link under the web server's document root.
 
 set -euo pipefail
 IFS="$(printf '\n\t')"
 
-if [[ "$CLEAN_WEB_DIR" == True ]]; then
-  echo "Removing existing files at $WEB_DIR"
-  rm -rf -- "$WEB_DIR"
-fi
+# Strip trailing slashes in case they have been added in the config. Otherwise
+# they break the symlinks.
+WEB_DIR="${WEB_DIR%/}"
+
+# Remove existing output ahead of creating new symlink.
+echo "Removing any existing output link at $WEB_DIR"
+rm -vfr -- "$WEB_DIR"
 
 echo "Installing website files to $WEB_DIR"
 # If we end up needing a build step for the website, here is where to run it.
 
+# Create directory for web content.
+mkdir -v "${CYLC_WORKFLOW_SHARE_DIR}/web"
 # Copy static HTML/CSS/JS.
-if mkdir -v "$WEB_DIR"; then
-  cp -rv html/* "$WEB_DIR"
-  # Create symbolic link to plots directory.
-  # NOTE: While its good for space, it means `cylc clean` removes plots.
-  ln -s "${CYLC_WORKFLOW_SHARE_DIR}/plots" "${WEB_DIR}/plots"
-else
-  # Fail task if directory already exists.
-  >&2 echo "Web directory already exists, refusing to overwrite."
-  >&2 echo "Web directory: $WEB_DIR"
-  false
-fi
+cp -rv html/* "${CYLC_WORKFLOW_SHARE_DIR}/web"
+# Create directory for plots.
+mkdir -p "${CYLC_WORKFLOW_SHARE_DIR}/web/plots"
+# Create symbolic link to web directory.
+# NOTE: While good for space, it means `cylc clean` removes output.
+ln -s "${CYLC_WORKFLOW_SHARE_DIR}/web" "$WEB_DIR"

cset-workflow/app/run_cset_recipe/bin/run-cset-recipe.py

@@ -39,8 +39,8 @@ def append_to_index(record: dict):
 
     Record should have the form {"Category Name": {"recipe_id": "Plot Name"}}
     """
-    # Plot index is at {run}/share/plots/index.json
-    index_path = Path(os.getenv("CYLC_WORKFLOW_SHARE_DIR"), "plots/index.json")
+    # Plot index is at {run}/share/web/plots/index.json
+    index_path = Path(os.getenv("CYLC_WORKFLOW_SHARE_DIR"), "web/plots/index.json")
     with open(index_path, "a+t", encoding="UTF-8") as fp:
         # Lock file until closed.
         fcntl.flock(fp, fcntl.LOCK_EX)
@@ -111,7 +111,7 @@ def recipe_id():
 def output_directory():
     """Get the plot output directory for the recipe."""
     share_directory = os.environ["CYLC_WORKFLOW_SHARE_DIR"]
-    return f"{share_directory}/plots/{recipe_id()}"
+    return f"{share_directory}/web/plots/{recipe_id()}"
 
 
 def data_directory():

cset-workflow/flow.cylc

@@ -128,8 +128,6 @@ URL = https://metoffice.github.io/CSET
 
     [[install_website_skeleton]]
     # Copies the static files that make up the web interface.
-        [[[environment]]]
-        CLEAN_WEB_DIR = {{CLEAN_WEB_DIR}}
 
     [[fetch_fcst]]
     # Fetch data from disk or a file based archival system.

cset-workflow/meta/rose-meta.conf

@@ -189,16 +189,6 @@ type=quoted
 compulsory=true
 sort-key=web1
 
-[template variables=CLEAN_WEB_DIR]
-ns=General
-description=Delete existing output in WEB_DIR.
-help=Whether to remove any existing files in WEB_DIR before running CSET. CSET
-    will not overwrite files, so if this is not set the workflow will stop on
-    encountering pre-existing files.
-type=python_boolean
-compulsory=true
-sort-key=web3
-
 [template variables=CSET_RUNAHEAD_LIMIT]
 ns=General
 description=Number of simultaneous cycles.

cset-workflow/rose-suite.conf.example

@@ -1,5 +1,4 @@
 [template variables]
-CLEAN_WEB_DIR=True
 !!CONDA_METPLUS_VENV_LOCATION=""
 CONDA_PATH=""
 CONDA_VENV_CREATE=True