diff --git a/src/CSET/cset_workflow/app/finish_website/bin/finish_website.py b/src/CSET/cset_workflow/app/finish_website/bin/finish_website.py
index c79d5dde7..f315774bb 100755
--- a/src/CSET/cset_workflow/app/finish_website/bin/finish_website.py
+++ b/src/CSET/cset_workflow/app/finish_website/bin/finish_website.py
@@ -24,8 +24,11 @@
import json
import logging
import os
+import shlex
import shutil
+import subprocess
import sys
+import tarfile
import time
from importlib.metadata import version
from pathlib import Path
@@ -40,12 +43,8 @@
logger = logging.getLogger(__name__)
-def install_website_skeleton(www_root_link: Path, www_content: Path):
+def install_website_skeleton(www_content: Path):
"""Copy static website files and create symlink from web document root."""
- # Remove existing link to output ahead of creating new symlink.
- logger.info("Removing any existing output link at %s.", www_root_link)
- www_root_link.unlink(missing_ok=True)
-
logger.info("Installing website files to %s.", www_content)
# Create directory for web content.
www_content.mkdir(parents=True, exist_ok=True)
@@ -56,6 +55,13 @@ def install_website_skeleton(www_root_link: Path, www_content: Path):
plot_dir = www_content / "plots"
plot_dir.mkdir(exist_ok=True)
+
+def symlink_website(www_root_link: Path, www_content: Path):
+ """Create a symlink to the web root."""
+ # Remove existing link to output ahead of creating new symlink.
+ logger.info("Removing any existing output link at %s.", www_root_link)
+ www_root_link.unlink(missing_ok=True)
+
logger.info("Linking %s to web content.", www_root_link)
# Ensure parent directories of WEB_DIR exist.
www_root_link.parent.mkdir(parents=True, exist_ok=True)
@@ -140,6 +146,86 @@ def copy_rose_config(www_content: Path):
shutil.copyfile(rose_suite_conf, web_conf_file)
+def tar_plots(plots_dir: Path, metadata: dict, tar: tarfile.TarFile):
+ """Add a single CSET plot directory to the tarball."""
+ # Path relative to the plots directory
+ plot_path = Path(metadata["path"])
+ # Absolute path
+ plot_fullpath = plots_dir.joinpath(plot_path)
+
+ if not (plot_fullpath / "CSET.log").exists():
+ # Not CSET output, we don't know how do deal with this yet
+ logger.warning("Unable to tar directory %s", plot_path)
+ return
+
+ with open(plot_fullpath / "meta.json") as f:
+ plot_meta = json.load(f)
+
+ # Add all the files to the tarball
+ def add_file(name):
+ tar.add(plot_fullpath / name, arcname=plot_path / name)
+
+ add_file("meta.json")
+ add_file("CSET.log")
+ for plot in plot_meta["plots"]:
+ add_file(plot)
+
+ # Change the path in the metadata
+ metadata["path"] = f"tarplot.html?path={metadata['path']}"
+
+ # Remove the plot directory
+ shutil.rmtree(plot_fullpath)
+
+
+def tar_website(www_content: Path):
+ """Tar up the website content to reduce file count."""
+ plots_dir = www_content / "plots"
+ tarpath = plots_dir / "plots.tar"
+ new_index = []
+
+ with (
+ open(plots_dir / "index.jsonl", encoding="UTF-8") as index,
+ tarfile.open(tarpath, mode="w") as tar,
+ ):
+ # Read each directory metadata, tar up the contents, then record the updated index information
+ lines = index.readlines()
+ for line in lines:
+ metadata = json.loads(line)
+ tar_plots(plots_dir, metadata, tar)
+ new_index.append(metadata)
+
+ with open(plots_dir / "index.jsonl", "w", encoding="UTF-8") as index:
+ # Write out the updated index info
+ for metadata in new_index:
+ json.dump(metadata, index, separators=(",", ":"))
+ index.write("\n")
+
+ # Create an index with the start and end location of each member
+ tarindex = {}
+ with tarfile.open(tarpath, mode="r") as tar:
+ for member in tar.getmembers():
+ tarindex[member.name] = [
+ member.offset_data,
+ member.offset_data + member.size,
+ ]
+ with open(plots_dir / "tarindex.json", "w") as f:
+ json.dump(tarindex, f)
+
+
+def rsync_website(www_root_link: Path, www_content: Path):
+ """Rsync the website content to the root directory."""
+ cmd = [
+ "rsync",
+ "--archive",
+ "--verbose",
+ "--delete",
+ str(www_content) + "/",
+ str(www_root_link),
+ ]
+ logger.info(shlex.join(cmd))
+ subprocess.run(cmd, check=True)
+
+
def run():
"""Do the final steps to finish the website."""
# Strip trailing slashes in case they have been added in the config.
@@ -147,12 +233,18 @@ def run():
www_root_link = Path(os.environ["WEB_DIR"].rstrip("/"))
www_content = Path(os.environ["CYLC_WORKFLOW_SHARE_DIR"] + "/web")
- install_website_skeleton(www_root_link, www_content)
+ install_website_skeleton(www_content)
copy_rose_config(www_content)
construct_index(www_content)
+ tar_website(www_content)
bust_cache(www_content)
update_workflow_status(www_content)
+ if os.environ.get("RSYNC_WEB_DIR", "False").lower() == "true":
+ rsync_website(www_root_link, www_content)
+ else:
+ symlink_website(www_root_link, www_content)
+
if __name__ == "__main__": # pragma: no cover
run()
diff --git a/src/CSET/cset_workflow/app/finish_website/file/html/plots/cset-plot.css b/src/CSET/cset_workflow/app/finish_website/file/html/plots/cset-plot.css
new file mode 100644
index 000000000..495873dbc
--- /dev/null
+++ b/src/CSET/cset_workflow/app/finish_website/file/html/plots/cset-plot.css
@@ -0,0 +1,61 @@
+body {
+ display: flex;
+ flex-wrap: wrap;
+ justify-content: space-around;
+ margin: 0;
+ font-family: sans-serif;
+}
+
+input:invalid {
+ box-shadow: 0 0 2px 2px red;
+}
+
+.hidden {
+ display: none !important;
+}
+
+.visually-hidden {
+ position: absolute;
+ left: -9999px;
+}
+
+main {
+ flex: 30em;
+ height: 100vh;
+}
+
+main>canvas {
+ width: 100%;
+ height: calc(100% - 3em);
+ object-fit: contain;
+}
+
+#sequence_controls {
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ margin: 0.5em 0;
+ border: none;
+ padding: 0;
+}
+
+#sequence_controls input[type="number"] {
+ width: 6em;
+}
+
+#sequence_controls input[type="range"] {
+ width: min(60em, calc(100% - 16em));
+ margin: 0 1em;
+}
+
+#description-container {
+ flex: 20em;
+ max-height: calc(100vh - min(1em, 5vh));
+ max-width: 50em;
+ margin: min(1em, 5vh);
+ padding: 0 1em;
+ background-color: whitesmoke;
+ box-shadow: 4px 4px 4px grey;
+ outline: darkgrey solid 1px;
+ overflow: auto;
+}
diff --git a/src/CSET/cset_workflow/app/finish_website/file/html/plots/tarplot.html b/src/CSET/cset_workflow/app/finish_website/file/html/plots/tarplot.html
new file mode 100644
index 000000000..801a966c2
--- /dev/null
+++ b/src/CSET/cset_workflow/app/finish_website/file/html/plots/tarplot.html
@@ -0,0 +1,26 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/CSET/cset_workflow/app/finish_website/file/html/plots/tarplot.js b/src/CSET/cset_workflow/app/finish_website/file/html/plots/tarplot.js
new file mode 100644
index 000000000..642d0d67f
--- /dev/null
+++ b/src/CSET/cset_workflow/app/finish_website/file/html/plots/tarplot.js
@@ -0,0 +1,117 @@
+"use strict";
+
+function update_displayed_plot(event) {
+ const plot_url = plot_urls[event.srcElement.value - 1];
+ show_plot(plot_url);
+}
+
+function show_plot(url) {
+ // Retrieve a plot from the tarball and put it on the canvas
+ const canvas = document.querySelector("#plot");
+ const ctx = canvas.getContext("2d")
+ ctx.reset()
+
+ getBlobFromTar(url)
+ .then(createImageBitmap)
+ .then((img) => {
+ canvas.width = img.width;
+ canvas.height = img.height;
+ ctx.drawImage(img, 0, 0);
+ });
+}
+
+function display_sequence_controls(plot_urls) {
+ // Only show sequence controls if there are multiple plots.
+ if (plot_urls == undefined || plot_urls.length < 2) {
+ return;
+ }
+
+ // Enable sequence controls.
+ const sequence_controls = document.querySelector("#sequence_controls");
+ sequence_controls.classList.remove("hidden")
+
+ // Wire up the controls.
+ const sequence_range = document.querySelector("#sequence_range");
+ sequence_range.max = plot_urls.length;
+ sequence_range.addEventListener("input", () => {
+ sequence_number.value = sequence_range.value;
+ });
+ sequence_range.addEventListener("input", update_displayed_plot);
+
+ const sequence_number = document.querySelector("#sequence_number");
+ sequence_number.max = plot_urls.length;
+ sequence_number.addEventListener("input", () => {
+ sequence_range.value = sequence_number.value;
+ });
+ sequence_number.addEventListener("change", update_displayed_plot);
+}
+
+let tar_index = null;
+async function getFromTar(path) {
+ // Tarfile to load
+ const tarfile = 'plots.tar';
+ const key = path;
+
+ // Tarfile index
+ if (tar_index === null) {
+ const r = await(fetch('tarindex.json'));
+ tar_index = await r.json();
+ }
+
+ if (! (key in tar_index)) {
+ console.log(key);
+ throw 404;
+ }
+
+ const start = tar_index[key][0];
+ const end = tar_index[key][1] - 1;
+
+ const slice = await fetch(tarfile, {headers: {Range: `bytes=${start}-${end}`}, cache: 'default'});
+
+ return slice
+}
+
+let tar_cache = {};
+async function getBlobFromTar(path) {
+ // Return a blob from the tarball, caching results
+ if (path in tar_cache) {
+ return tar_cache[path];
+ }
+ const slice = await getFromTar(path);
+ tar_cache[path] = slice.blob();
+ return tar_cache[path];
+}
+
+async function get_metadata() {
+ // Retrieve the Json metadata for this plot directory from the tarball
+ const meta = await getFromTar(`${path}/meta.json`);
+ return meta.json();
+}
+
+let plot_urls = [];
+function setup_plots(meta) {
+ // Set up the plot using the metadata info
+ document.getElementById("title").innerHTML = meta["title"];
+ document.getElementById("desc-title").innerHTML = meta["title"];
+ document.getElementById("desc-description").innerHTML = meta["description"];
+
+ // Set up the plot URLs
+ const plots = meta["plots"];
+ plot_urls = plots.map((p) => `${path}/${p}`);
+
+ display_sequence_controls(plot_urls);
+
+ // Show the first plot
+ show_plot(plot_urls[0]);
+}
+
+let path = "";
+function setup() {
+ // Get the path to the plot directory from the url then setup the page
+ const searchParams = new URLSearchParams(window.location.search);
+ path = searchParams.get("path");
+
+ get_metadata().then(setup_plots);
+}
+
+document.addEventListener('DOMContentLoaded', setup);
diff --git a/src/CSET/cset_workflow/flow.cylc b/src/CSET/cset_workflow/flow.cylc
index 738518d2a..f4b79f7b0 100644
--- a/src/CSET/cset_workflow/flow.cylc
+++ b/src/CSET/cset_workflow/flow.cylc
@@ -204,6 +204,7 @@ final cycle point = {{CSET_TRIAL_END_DATE}}
# Create the diagnostic viewing website.
[[[environment]]]
WEB_DIR = {{WEB_DIR}}
+ RSYNC_WEB_DIR = {{RSYNC_WEB_DIR}}
[[send_email]]
# Send email to notify that the workflow is complete.
diff --git a/src/CSET/cset_workflow/meta/rose-meta.conf b/src/CSET/cset_workflow/meta/rose-meta.conf
index e94c2a8e5..c2f64da64 100644
--- a/src/CSET/cset_workflow/meta/rose-meta.conf
+++ b/src/CSET/cset_workflow/meta/rose-meta.conf
@@ -135,6 +135,17 @@ type=quoted
compulsory=true
sort-key=setup-g-web2
+[template variables=RSYNC_WEB_DIR]
+ns=Setup
+title=Rsync web files
+description=Copy files to WEB_DIR with rsync
+help=If True files will be rsynced to WEB_DIR instead of symlinked. Use this if
+ the Cylc workflow directory is not visible to the web server, e.g. if the
+ web server runs on a different server or can only see certain filesystems.
+type=python_boolean
+compulsory=true
+sort-key=setup-g-web3
+
[template variables=LOGLEVEL]
ns=Setup
title=Logging level
diff --git a/src/CSET/cset_workflow/meta/rose-meta.conf.jinja2 b/src/CSET/cset_workflow/meta/rose-meta.conf.jinja2
index 9b6383064..74bfcb5ab 100644
--- a/src/CSET/cset_workflow/meta/rose-meta.conf.jinja2
+++ b/src/CSET/cset_workflow/meta/rose-meta.conf.jinja2
@@ -135,6 +135,17 @@ type=quoted
compulsory=true
sort-key=setup-g-web2
+[template variables=RSYNC_WEB_DIR]
+ns=Setup
+title=Rsync web files
+description=Copy files to WEB_DIR with rsync
+help=If True files will be rsynced to WEB_DIR instead of symlinked. Use this if
+ the Cylc workflow directory is not visible to the web server, e.g. if the
+ web server runs on a different server or can only see certain filesystems.
+type=python_boolean
+compulsory=true
+sort-key=setup-g-web3
+
[template variables=LOGLEVEL]
ns=Setup
title=Logging level
diff --git a/src/CSET/cset_workflow/rose-suite.conf.example b/src/CSET/cset_workflow/rose-suite.conf.example
index 646464bfb..ed0bded1a 100644
--- a/src/CSET/cset_workflow/rose-suite.conf.example
+++ b/src/CSET/cset_workflow/rose-suite.conf.example
@@ -209,6 +209,7 @@ VIOLENT_RAIN_PRESENCE_SPATIAL_DIFFERENCE=False
VIOLENT_RAIN_PRESENCE_SPATIAL_PLOT=False
WEB_ADDR=""
WEB_DIR=""
+RSYNC_WEB_DIR=False
!!WMO_BLOCK_STTN_NMBRS=[]
!!m10_analysis_offset="PT0H"
!!m10_data_path=""
diff --git a/src/CSET/cset_workflow/site/nci-gadi.cylc b/src/CSET/cset_workflow/site/nci-gadi.cylc
index 543ef4ea5..61714542a 100644
--- a/src/CSET/cset_workflow/site/nci-gadi.cylc
+++ b/src/CSET/cset_workflow/site/nci-gadi.cylc
@@ -11,6 +11,9 @@
# # 'access_c3_dn' or 'custom'
# CUSTOM_ODB2_PATTERN # Path to custom ODB2 files if using
# # 'custom' system
+# RSYNC_WEB_DIR # Set to True if you want to use the /g/data/vx53
+# # webserver with e.g.
+# # WEB_DIR="/g/data/vx53/$USER/$CYLC_WORKFLOW_NAME"
#
# Leave METPLUS_FCST_DIR and METPLUS_OBS_DIR unset
diff --git a/tests/workflow_utils/test_finish_website.py b/tests/workflow_utils/test_finish_website.py
index 6187b24ed..302edba70 100644
--- a/tests/workflow_utils/test_finish_website.py
+++ b/tests/workflow_utils/test_finish_website.py
@@ -27,11 +27,13 @@ def test_install_website_skeleton(monkeypatch, tmp_path):
www_content = tmp_path / "web"
www_root_link = tmp_path / "www/CSET"
monkeypatch.chdir("src/CSET/cset_workflow/app/finish_website/file")
- finish_website.install_website_skeleton(www_root_link, www_content)
+ finish_website.install_website_skeleton(www_content)
assert www_content.is_dir()
assert (www_content / "index.html").is_file()
assert (www_content / "script.js").is_file()
assert (www_content / "plots").is_dir()
+
+ finish_website.symlink_website(www_root_link, www_content)
assert www_root_link.is_symlink()
assert www_root_link.resolve() == www_content.resolve()
@@ -200,14 +202,16 @@ def check_args(www_root_link: Path, www_content: Path):
assert www_content == Path("/share/web")
increment_counter()
- monkeypatch.setattr(finish_website, "install_website_skeleton", check_args)
+ monkeypatch.setattr(finish_website, "install_website_skeleton", check_single_arg)
monkeypatch.setattr(finish_website, "copy_rose_config", check_single_arg)
monkeypatch.setattr(finish_website, "construct_index", check_single_arg)
+ monkeypatch.setattr(finish_website, "tar_website", check_single_arg)
monkeypatch.setattr(finish_website, "bust_cache", check_single_arg)
monkeypatch.setattr(finish_website, "update_workflow_status", check_single_arg)
+ monkeypatch.setattr(finish_website, "symlink_website", check_args)
monkeypatch.setenv("WEB_DIR", "/var/www/cset")
monkeypatch.setenv("CYLC_WORKFLOW_SHARE_DIR", "/share")
# Check that it runs all the needed subfunctions.
finish_website.run()
- assert counter == 5
+ assert counter == 7