Put workflow plots in a tarball

src/CSET/cset_workflow/app/finish_website/bin/finish_website.py

@@ -24,8 +24,11 @@
 import json
 import logging
 import os
+import shlex
 import shutil
+import subprocess
 import sys
+import tarfile
 import time
 from importlib.metadata import version
 from pathlib import Path
@@ -40,12 +43,8 @@
 logger = logging.getLogger(__name__)
 
 
-def install_website_skeleton(www_root_link: Path, www_content: Path):
+def install_website_skeleton(www_content: Path):
     """Copy static website files and create symlink from web document root."""
-    # Remove existing link to output ahead of creating new symlink.
-    logger.info("Removing any existing output link at %s.", www_root_link)
-    www_root_link.unlink(missing_ok=True)
-
     logger.info("Installing website files to %s.", www_content)
     # Create directory for web content.
     www_content.mkdir(parents=True, exist_ok=True)
@@ -56,6 +55,13 @@ def install_website_skeleton(www_root_link: Path, www_content: Path):
     plot_dir = www_content / "plots"
     plot_dir.mkdir(exist_ok=True)
 
+
+def symlink_website(www_root_link: Path, www_content: Path):
+    """Create a symlink to the web root."""
+    # Remove existing link to output ahead of creating new symlink.
+    logger.info("Removing any existing output link at %s.", www_root_link)
+    www_root_link.unlink(missing_ok=True)
+
     logger.info("Linking %s to web content.", www_root_link)
     # Ensure parent directories of WEB_DIR exist.
     www_root_link.parent.mkdir(parents=True, exist_ok=True)
@@ -140,19 +146,105 @@ def copy_rose_config(www_content: Path):
     shutil.copyfile(rose_suite_conf, web_conf_file)
 
 
+def tar_plots(plots_dir: Path, metadata: dict, tar: tarfile.TarFile):
+    """Add a single CSET plot directory to the tarball."""
+    # Path relative to the plots directory
+    plot_path = Path(metadata["path"])
+    # Absolute path
+    plot_fullpath = plots_dir.joinpath(plot_path)
+
+    if not (plot_fullpath / "CSET.log").exists():
+        # Not CSET output, we don't know how do deal with this yet
+        logger.warning("Unable to tar directory %s", plot_path)
+        return
+
+    with open(plot_fullpath / "meta.json") as f:
+        plot_meta = json.load(f)
+
+    # Add all the files to the tarball
+    def add_file(name):
+        tar.add(plot_fullpath / name, arcname=plot_path / name)
+
+    add_file("meta.json")
+    add_file("CSET.log")
+    for plot in plot_meta["plots"]:
+        add_file(plot)
+
+    # Change the path in the metadata
+    metadata["path"] = f"tarplot.html?path={metadata['path']}"
+
+    # Remove the plot directory
+    shutil.rmtree(plot_fullpath)
+
+
+def tar_website(www_content: Path):
+    """Tar up the website content to reduce file count."""
+    plots_dir = www_content / "plots"
+    tarpath = plots_dir / "plots.tar"
+    new_index = []
+
+    with (
+        open(plots_dir / "index.jsonl", encoding="UTF-8") as index,
+        tarfile.open(tarpath, mode="w") as tar,
+    ):
+        # Read each directory metadata, tar up the contents, then record the updated index information
+        lines = index.readlines()
+        for line in lines:
+            metadata = json.loads(line)
+            tar_plots(plots_dir, metadata, tar)
+            new_index.append(metadata)
+
+    with open(plots_dir / "index.jsonl", "w", encoding="UTF-8") as index:
+        # Write out the updated index info
+        for metadata in new_index:
+            json.dump(metadata, index, separators=(",", ":"))
+            index.write("\n")
+
+    # Create an index with the start and end location of each member
+    tarindex = {}
+    with tarfile.open(tarpath, mode="r") as tar:
+        for member in tar.getmembers():
+            tarindex[member.name] = [
+                member.offset_data,
+                member.offset_data + member.size,
+            ]
+    with open(plots_dir / "tarindex.json", "w") as f:
+        json.dump(tarindex, f)
+
+
+def rsync_website(www_root_link: Path, www_content: Path):
+    """Rsync the website content to the root directory."""
+    cmd = [
+        "rsync",
+        "--archive",
+        "--verbose",
+        "--delete",
+        str(www_content) + "/",
+        str(www_root_link),
+    ]
+    logger.info(shlex.join(cmd))
+    subprocess.run(cmd, check=True)
+
+
 def run():
     """Do the final steps to finish the website."""
     # Strip trailing slashes in case they have been added in the config.
     # Otherwise they break the symlinks.
     www_root_link = Path(os.environ["WEB_DIR"].rstrip("/"))
     www_content = Path(os.environ["CYLC_WORKFLOW_SHARE_DIR"] + "/web")
 
-    install_website_skeleton(www_root_link, www_content)
+    install_website_skeleton(www_content)
     copy_rose_config(www_content)
     construct_index(www_content)
+    tar_website(www_content)
     bust_cache(www_content)
     update_workflow_status(www_content)
 
+    if os.environ.get("RSYNC_WEB_DIR", "False").lower() == "true":
+        rsync_website(www_root_link, www_content)
+    else:
+        symlink_website(www_root_link, www_content)
+
 
 if __name__ == "__main__":  # pragma: no cover
     run()

src/CSET/cset_workflow/app/finish_website/file/html/plots/cset-plot.css

@@ -0,0 +1,61 @@
+body {
+    display: flex;
+    flex-wrap: wrap;
+    justify-content: space-around;
+    margin: 0;
+    font-family: sans-serif;
+}
+
+input:invalid {
+    box-shadow: 0 0 2px 2px red;
+}
+
+.hidden {
+    display: none !important;
+}
+
+.visually-hidden {
+    position: absolute;
+    left: -9999px;
+}
+
+main {
+    flex: 30em;
+    height: 100vh;
+}
+
+main>canvas {
+    width: 100%;
+    height: calc(100% - 3em);
+    object-fit: contain;
+}
+
+#sequence_controls {
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    margin: 0.5em 0;
+    border: none;
+    padding: 0;
+}
+
+#sequence_controls input[type="number"] {
+    width: 6em;
+}
+
+#sequence_controls input[type="range"] {
+    width: min(60em, calc(100% - 16em));
+    margin: 0 1em;
+}
+
+#description-container {
+    flex: 20em;
+    max-height: calc(100vh - min(1em, 5vh));
+    max-width: 50em;
+    margin: min(1em, 5vh);
+    padding: 0 1em;
+    background-color: whitesmoke;
+    box-shadow: 4px 4px 4px grey;
+    outline: darkgrey solid 1px;
+    overflow: auto;
+}

src/CSET/cset_workflow/app/finish_website/file/html/plots/tarplot.html

@@ -0,0 +1,26 @@
+<!doctype html>
+<html>
+    <head>
+        <meta charset="UTF-8">
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        <title id="title"></title>
+        <link href="cset-plot.css" rel="stylesheet"/>
+        <script src="tarplot.js"></script>
+    </head>
+    <body>
+        <main>
+            <fieldset id="sequence_controls" class="hidden">
+                <label for="sequence_range">Select plot:</label>
+                <label class="visually-hidden" for="sequence_number">Select plot</label>
+                <input type="range" id="sequence_range" value="1" min="1" max="1">
+                <input type="number" id="sequence_number" value="1" min="1" max="1">
+            </fieldset>
+            <canvas id="plot" alt="plot">
+        </main>
+        <aside id="description-container">
+            <h1 id="desc-title"></h1>
+            <p id="desc-description"></p>
+            <hr>
+        </aside>
+    </body>
+</html>

src/CSET/cset_workflow/app/finish_website/file/html/plots/tarplot.js

@@ -0,0 +1,117 @@
+"use strict";
+
+function update_displayed_plot(event) {
+    const plot_url = plot_urls[event.srcElement.value - 1];
+    show_plot(plot_url);
+}
+
+function show_plot(url) {
+    // Retrieve a plot from the tarball and put it on the canvas
+    const canvas = document.querySelector("#plot");
+    const ctx = canvas.getContext("2d")
+    ctx.reset()
+
+    getBlobFromTar(url)
+        .then(createImageBitmap)
+        .then((img) => {
+            canvas.width = img.width;
+            canvas.height = img.height;
+            ctx.drawImage(img, 0, 0);
+        });
+}
+
+function display_sequence_controls(plot_urls) {
+    // Only show sequence controls if there are multiple plots.
+    if (plot_urls == undefined || plot_urls.length < 2) {
+        return;
+    }
+
+    // Enable sequence controls.
+    const sequence_controls = document.querySelector("#sequence_controls");
+    sequence_controls.classList.remove("hidden")
+
+    // Wire up the controls.
+    const sequence_range = document.querySelector("#sequence_range");
+    sequence_range.max = plot_urls.length;
+    sequence_range.addEventListener("input", () => {
+        sequence_number.value = sequence_range.value;
+    });
+    sequence_range.addEventListener("input", update_displayed_plot);
+
+    const sequence_number = document.querySelector("#sequence_number");
+    sequence_number.max = plot_urls.length;
+    sequence_number.addEventListener("input", () => {
+        sequence_range.value = sequence_number.value;
+    });
+    sequence_number.addEventListener("change", update_displayed_plot);
+}
+
+let tar_index = null;
+async function getFromTar(path) {
+    // Tarfile to load
+    const tarfile = 'plots.tar';
+    const key = path;
+
+    // Tarfile index
+    if (tar_index === null) {
+        const r = await(fetch('tarindex.json'));
+        tar_index = await r.json();
+    }
+
+    if (! (key in tar_index)) {
+        console.log(key);
+        throw 404;
+    }
+
+    const start = tar_index[key][0];
+    const end = tar_index[key][1] - 1;
+
+    const slice = await fetch(tarfile, {headers: {Range: `bytes=${start}-${end}`}, cache: 'default'});
+
+    return slice
+}
+
+let tar_cache = {};
+async function getBlobFromTar(path) {
+    // Return a blob from the tarball, caching results
+    if (path in tar_cache) {
+        return tar_cache[path];
+    }
+    const slice = await getFromTar(path);
+    tar_cache[path] = slice.blob();
+    return tar_cache[path];
+}
+
+async function get_metadata() {
+    // Retrieve the Json metadata for this plot directory from the tarball
+    const meta = await getFromTar(`${path}/meta.json`);
+    return meta.json();
+}
+
+let plot_urls = [];
+function setup_plots(meta) {
+    // Set up the plot using the metadata info
+    document.getElementById("title").innerHTML = meta["title"];
+    document.getElementById("desc-title").innerHTML = meta["title"];
+    document.getElementById("desc-description").innerHTML = meta["description"];
+
+    // Set up the plot URLs
+    const plots = meta["plots"];
+    plot_urls = plots.map((p) => `${path}/${p}`);
+
+    display_sequence_controls(plot_urls);
+
+    // Show the first plot
+    show_plot(plot_urls[0]);
+}
+
+let path = "";
+function setup() {
+    // Get the path to the plot directory from the url then setup the page
+    const searchParams = new URLSearchParams(window.location.search);
+    path = searchParams.get("path");
+
+    get_metadata().then(setup_plots);
+}
+
+document.addEventListener('DOMContentLoaded', setup);

src/CSET/cset_workflow/flow.cylc

@@ -204,6 +204,7 @@ final cycle point = {{CSET_TRIAL_END_DATE}}
     # Create the diagnostic viewing website.
         [[[environment]]]
         WEB_DIR = {{WEB_DIR}}
+        RSYNC_WEB_DIR = {{RSYNC_WEB_DIR}}
 
     [[send_email]]
     # Send email to notify that the workflow is complete.

src/CSET/cset_workflow/meta/rose-meta.conf.jinja2

@@ -135,6 +135,17 @@ type=quoted
 compulsory=true
 sort-key=setup-g-web2
 
+[template variables=RSYNC_WEB_DIR]
+ns=Setup
+title=Rsync web files
+description=Copy files to WEB_DIR with rsync
+help=If True files will be rsynced to WEB_DIR instead of symlinked. Use this if
+     the Cylc workflow directory is not visible to the web server, e.g. if the
+     web server runs on a different server or can only see certain filesystems.
+type=python_boolean
+compulsory=true
+sort-key=setup-g-web3
+
 [template variables=LOGLEVEL]
 ns=Setup
 title=Logging level