From 686bac42dd4b0af833d5d96375a2c965d7427e70 Mon Sep 17 00:00:00 2001 From: James Frost Date: Tue, 30 Jul 2024 16:46:48 +0100 Subject: [PATCH] Move all website files under workflow shared dir Now the full output of CSET is under the workflow run directory, giving more consistent behaviour around cylc clean, moving output, and keeping old output when rerunning. All web content is now saved under ${CYLC_WORKFLOW_SHARE_DIR}/web/ and a symbolic link is created from the webserver's directory. Fixes #761 --- .../app/housekeeping/bin/housekeep.sh | 2 +- .../bin/install-website.sh | 34 +++++++++---------- .../run_cset_recipe/bin/run-cset-recipe.py | 6 ++-- cset-workflow/flow.cylc | 2 -- cset-workflow/meta/rose-meta.conf | 10 ------ cset-workflow/rose-suite.conf.example | 1 - 6 files changed, 21 insertions(+), 34 deletions(-) diff --git a/cset-workflow/app/housekeeping/bin/housekeep.sh b/cset-workflow/app/housekeeping/bin/housekeep.sh index 27a9f4160..a084f4208 100755 --- a/cset-workflow/app/housekeeping/bin/housekeep.sh +++ b/cset-workflow/app/housekeeping/bin/housekeep.sh @@ -22,5 +22,5 @@ then # Housekeeping: Standard echo 'Removing intermediate data.' rm -rfv -- "$CYLC_WORKFLOW_SHARE_DIR"/cycle/*/data \ - "$CYLC_WORKFLOW_SHARE_DIR"/plots/*/intermediate + "$CYLC_WORKFLOW_SHARE_DIR"/web/plots/*/intermediate fi diff --git a/cset-workflow/app/install_website_skeleton/bin/install-website.sh b/cset-workflow/app/install_website_skeleton/bin/install-website.sh index 4e40f1d34..d3ae85cac 100755 --- a/cset-workflow/app/install_website_skeleton/bin/install-website.sh +++ b/cset-workflow/app/install_website_skeleton/bin/install-website.sh @@ -1,28 +1,28 @@ #!/bin/bash -# Copies the static files for the web interface into the correct location, -# optionally removing previous files there. +# Copies the static files for the web interface into the correct location, and +# creates a symbolic link under the web server's document root. set -euo pipefail IFS="$(printf '\n\t')" -if [[ "$CLEAN_WEB_DIR" == True ]]; then - echo "Removing existing files at $WEB_DIR" - rm -rf -- "$WEB_DIR" -fi +# Strip trailing slashes in case they have been added in the config. Otherwise +# they break the symlinks. +WEB_DIR="${WEB_DIR%/}" + +# Remove existing output ahead of creating new symlink. +echo "Removing any existing output link at $WEB_DIR" +rm -vfr -- "$WEB_DIR" echo "Installing website files to $WEB_DIR" # If we end up needing a build step for the website, here is where to run it. +# Create directory for web content. +mkdir -v "${CYLC_WORKFLOW_SHARE_DIR}/web" # Copy static HTML/CSS/JS. -if mkdir -v "$WEB_DIR"; then - cp -rv html/* "$WEB_DIR" - # Create symbolic link to plots directory. - # NOTE: While its good for space, it means `cylc clean` removes plots. - ln -s "${CYLC_WORKFLOW_SHARE_DIR}/plots" "${WEB_DIR}/plots" -else - # Fail task if directory already exists. - >&2 echo "Web directory already exists, refusing to overwrite." - >&2 echo "Web directory: $WEB_DIR" - false -fi +cp -rv html/* "${CYLC_WORKFLOW_SHARE_DIR}/web" +# Create directory for plots. +mkdir -p "${CYLC_WORKFLOW_SHARE_DIR}/web/plots" +# Create symbolic link to web directory. +# NOTE: While good for space, it means `cylc clean` removes output. +ln -s "${CYLC_WORKFLOW_SHARE_DIR}/web" "$WEB_DIR" diff --git a/cset-workflow/app/run_cset_recipe/bin/run-cset-recipe.py b/cset-workflow/app/run_cset_recipe/bin/run-cset-recipe.py index 10bf6e708..525727788 100755 --- a/cset-workflow/app/run_cset_recipe/bin/run-cset-recipe.py +++ b/cset-workflow/app/run_cset_recipe/bin/run-cset-recipe.py @@ -39,8 +39,8 @@ def append_to_index(record: dict): Record should have the form {"Category Name": {"recipe_id": "Plot Name"}} """ - # Plot index is at {run}/share/plots/index.json - index_path = Path(os.getenv("CYLC_WORKFLOW_SHARE_DIR"), "plots/index.json") + # Plot index is at {run}/share/web/plots/index.json + index_path = Path(os.getenv("CYLC_WORKFLOW_SHARE_DIR"), "web/plots/index.json") with open(index_path, "a+t", encoding="UTF-8") as fp: # Lock file until closed. fcntl.flock(fp, fcntl.LOCK_EX) @@ -111,7 +111,7 @@ def recipe_id(): def output_directory(): """Get the plot output directory for the recipe.""" share_directory = os.environ["CYLC_WORKFLOW_SHARE_DIR"] - return f"{share_directory}/plots/{recipe_id()}" + return f"{share_directory}/web/plots/{recipe_id()}" def data_directory(): diff --git a/cset-workflow/flow.cylc b/cset-workflow/flow.cylc index 11bb6646c..0247455d6 100644 --- a/cset-workflow/flow.cylc +++ b/cset-workflow/flow.cylc @@ -128,8 +128,6 @@ URL = https://metoffice.github.io/CSET [[install_website_skeleton]] # Copies the static files that make up the web interface. - [[[environment]]] - CLEAN_WEB_DIR = {{CLEAN_WEB_DIR}} [[fetch_fcst]] # Fetch data from disk or a file based archival system. diff --git a/cset-workflow/meta/rose-meta.conf b/cset-workflow/meta/rose-meta.conf index 4cf6342c2..8aff436bc 100644 --- a/cset-workflow/meta/rose-meta.conf +++ b/cset-workflow/meta/rose-meta.conf @@ -189,16 +189,6 @@ type=quoted compulsory=true sort-key=web1 -[template variables=CLEAN_WEB_DIR] -ns=General -description=Delete existing output in WEB_DIR. -help=Whether to remove any existing files in WEB_DIR before running CSET. CSET - will not overwrite files, so if this is not set the workflow will stop on - encountering pre-existing files. -type=python_boolean -compulsory=true -sort-key=web3 - [template variables=CSET_RUNAHEAD_LIMIT] ns=General description=Number of simultaneous cycles. diff --git a/cset-workflow/rose-suite.conf.example b/cset-workflow/rose-suite.conf.example index 6027607a5..81aea41a4 100644 --- a/cset-workflow/rose-suite.conf.example +++ b/cset-workflow/rose-suite.conf.example @@ -1,5 +1,4 @@ [template variables] -CLEAN_WEB_DIR=True !!CONDA_METPLUS_VENV_LOCATION="" CONDA_PATH="" CONDA_VENV_CREATE=True