Move all website files under workflow shared dir

Now the full output of CSET is under the workflow run directory, giving
more consistent behaviour around cylc clean, moving output, and keeping
old output when rerunning.

All web content is now saved under ${CYLC_WORKFLOW_SHARE_DIR}/web/ and
a symbolic link is created from the webserver's directory.

Fixes #761

cset-workflow/app/housekeeping/bin/housekeep.sh

@@ -22,5 +22,5 @@ then
     # Housekeeping: Standard
     echo 'Removing intermediate data.'
     rm -rfv -- "$CYLC_WORKFLOW_SHARE_DIR"/cycle/*/data \
-              "$CYLC_WORKFLOW_SHARE_DIR"/plots/*/intermediate
+              "$CYLC_WORKFLOW_SHARE_DIR"/web/plots/*/intermediate
 fi

cset-workflow/app/install_website_skeleton/bin/install-website.sh

@@ -1,28 +1,28 @@
 #!/bin/bash
 
-# Copies the static files for the web interface into the correct location,
-# optionally removing previous files there.
+# Copies the static files for the web interface into the correct location, and
+# creates a symbolic link under the web server's document root.
 
 set -euo pipefail
 IFS="$(printf '\n\t')"
 
-if [[ "$CLEAN_WEB_DIR" == True ]]; then
-  echo "Removing existing files at $WEB_DIR"
-  rm -rf -- "$WEB_DIR"
-fi
+# Strip trailing slashes in case they have been added in the config. Otherwise
+# they break the symlinks.
+WEB_DIR="${WEB_DIR%/}"
+
+# Remove existing output ahead of creating new symlink.
+echo "Removing any existing output link at $WEB_DIR"
+rm -vfr -- "$WEB_DIR"
 
 echo "Installing website files to $WEB_DIR"
 # If we end up needing a build step for the website, here is where to run it.
 
+# Create directory for web content.
+mkdir -v "${CYLC_WORKFLOW_SHARE_DIR}/web"
 # Copy static HTML/CSS/JS.
-if mkdir -v "$WEB_DIR"; then
-  cp -rv html/* "$WEB_DIR"
-  # Create symbolic link to plots directory.
-  # NOTE: While its good for space, it means `cylc clean` removes plots.
-  ln -s "${CYLC_WORKFLOW_SHARE_DIR}/plots" "${WEB_DIR}/plots"
-else
-  # Fail task if directory already exists.
-  >&2 echo "Web directory already exists, refusing to overwrite."
-  >&2 echo "Web directory: $WEB_DIR"
-  false
-fi
+cp -rv html/* "${CYLC_WORKFLOW_SHARE_DIR}/web"
+# Create directory for plots.
+mkdir -p "${CYLC_WORKFLOW_SHARE_DIR}/web/plots"
+# Create symbolic link to web directory.
+# NOTE: While good for space, it means `cylc clean` removes output.
+ln -s "${CYLC_WORKFLOW_SHARE_DIR}/web" "$WEB_DIR"

cset-workflow/app/run_cset_recipe/bin/run-cset-recipe.py

@@ -39,8 +39,8 @@ def append_to_index(record: dict):
 
     Record should have the form {"Category Name": {"recipe_id": "Plot Name"}}
     """
-    # Plot index is at {run}/share/plots/index.json
-    index_path = Path(os.getenv("CYLC_WORKFLOW_SHARE_DIR"), "plots/index.json")
+    # Plot index is at {run}/share/web/plots/index.json
+    index_path = Path(os.getenv("CYLC_WORKFLOW_SHARE_DIR"), "web/plots/index.json")
     with open(index_path, "a+t", encoding="UTF-8") as fp:
         # Lock file until closed.
         fcntl.flock(fp, fcntl.LOCK_EX)
@@ -111,7 +111,7 @@ def recipe_id():
 def output_directory():
     """Get the plot output directory for the recipe."""
     share_directory = os.environ["CYLC_WORKFLOW_SHARE_DIR"]
-    return f"{share_directory}/plots/{recipe_id()}"
+    return f"{share_directory}/web/plots/{recipe_id()}"
 
 
 def data_directory():

cset-workflow/flow.cylc

@@ -128,8 +128,6 @@ URL = https://metoffice.github.io/CSET
 
     [[install_website_skeleton]]
     # Copies the static files that make up the web interface.
-        [[[environment]]]
-        CLEAN_WEB_DIR = {{CLEAN_WEB_DIR}}
 
     [[fetch_fcst]]
     # Fetch data from disk or a file based archival system.

cset-workflow/meta/rose-meta.conf

@@ -189,16 +189,6 @@ type=quoted
 compulsory=true
 sort-key=web1
 
-[template variables=CLEAN_WEB_DIR]
-ns=General
-description=Delete existing output in WEB_DIR.
-help=Whether to remove any existing files in WEB_DIR before running CSET. CSET
-    will not overwrite files, so if this is not set the workflow will stop on
-    encountering pre-existing files.
-type=python_boolean
-compulsory=true
-sort-key=web3
-
 [template variables=CSET_RUNAHEAD_LIMIT]
 ns=General
 description=Number of simultaneous cycles.

cset-workflow/rose-suite.conf.example

@@ -1,5 +1,4 @@
 [template variables]
-CLEAN_WEB_DIR=True
 !!CONDA_METPLUS_VENV_LOCATION=""
 CONDA_PATH=""
 CONDA_VENV_CREATE=True