Commpath

config.h.in

@@ -6,6 +6,12 @@
 /* Whether we are using a broken srun */
 #undef BROKEN_SRUN
 
+/* Colon-separated list of potential back-end cache directories */
+#undef CACHEPATHS
+
+/* Back-end directory for communication and housekeeping */
+#undef COMMPATH
+
 /* Define if were using biter for client/server communication */
 #undef COMM_BITER
 
@@ -137,9 +143,6 @@
 /* Default mode for slurm launch */
 #undef SLURMLAUNCH_ENABLED
 
-/* The default local directory for Spindle */
-#undef SPINDLE_LOC
-
 /* The default colon-separated list of directories that Spindle will not cache
    files out of */
 #undef SPINDLE_LOCAL_PREFIX

configure

@@ -848,6 +848,9 @@ enable_maintainer_mode
 with_default_port
 with_default_num_ports
 with_localstorage
+with_cachepaths
+with_cachepath
+with_commpath
 with_default_local_prefix
 with_testrm
 with_rm
@@ -1590,7 +1593,13 @@ Optional Packages:
   --with-default-numports=NUM
                           Number of TCP/IP ports to scan for Spindle server
                           communication
-  --with-localstorage=DIR Directory on back-ends for storing relocated files
+  --with-localstorage=DIR (obsolete)
+                          Use --with-cachepaths and --with-commpath instead.
+  --with-cachepaths=DIR   Colon-separated list of potential back-end cache
+                          directories
+,
+  --with-compath=DIR      Back-end directory for communication and
+                          housekeeping
   --with-default-local-prefix=DIRS
                           Colon-seperated list of directories that Spindle
                           will not cache files out of
@@ -16664,17 +16673,37 @@ fi
 
 # Check whether --with-localstorage was given.
 if test "${with_localstorage+set}" = set; then :
-  withval=$with_localstorage; SPINDLE_LOC=${withval}
+  withval=$with_localstorage; as_fn_error $? "requested obsolete option --with-localstorage.  Use --with-cachepaths and --with-commpath instead." "$LINENO" 5
+fi
+
+
+# Check whether --with-cachepaths was given.
+if test "${with_cachepaths+set}" = set; then :
+  withval=$with_cachepaths; CACHEPATHS=${withval}
+else
+  CACHEPATHS=$DEFAULT_LOC
+fi
+
+
+# Check whether --with-cachepath was given.
+if test "${with_cachepath+set}" = set; then :
+  withval=$with_cachepath; as_fn_error $? "use --with-cachepaths=DIRS (plural) instead of --with-cachepath=DIR to specify one or more cache paths" "$LINENO" 5
+fi
+
+
+# Check whether --with-commpath was given.
+if test "${with_commpath+set}" = set; then :
+  withval=$with_commpath; COMMPATH=${withval}
 else
-  SPINDLE_LOC=$DEFAULT_LOC
+  COMMPATH=$DEFAULT_LOC
 fi
 
 
 # Check whether --with-default-local-prefix was given.
 if test "${with_default_local_prefix+set}" = set; then :
   withval=$with_default_local_prefix; SPINDLE_LOCAL_PREFIX=${withval}
 else
-  SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$SPINDLE_LOC"
+  SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH"
 fi
 
 
@@ -16694,7 +16723,12 @@ _ACEOF
 
 
 cat >>confdefs.h <<_ACEOF
-#define SPINDLE_LOC "$SPINDLE_LOC"
+#define COMMPATH "$COMMPATH"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define CACHEPATHS "$CACHEPATHS"
 _ACEOF
 
 

configure.common.ac

@@ -18,17 +18,30 @@ AC_ARG_WITH(default-num-ports,
             [NUM_COBO_PORTS=${withval}],
             [NUM_COBO_PORTS=$DEFAULT_NUM_COBO_PORTS])
 AC_ARG_WITH(localstorage,
-            [AS_HELP_STRING([--with-localstorage=DIR],[Directory on back-ends for storing relocated files])],
-            [SPINDLE_LOC=${withval}],
-            [SPINDLE_LOC=$DEFAULT_LOC])
+            [AS_HELP_STRING([--with-localstorage=DIR (obsolete)],[Use --with-cachepaths and --with-commpath instead.])],
+            [AC_MSG_ERROR(requested obsolete option --with-localstorage.  Use --with-cachepaths and --with-commpath instead.)],
+            [])
+AC_ARG_WITH(cachepaths,
+            [AS_HELP_STRING([--with-cachepaths=DIR],[Colon-separated list of potential back-end cache directories])],
+            [CACHEPATHS=${withval}],
+            [CACHEPATHS=$DEFAULT_LOC])
+AC_ARG_WITH(cachepath,
+            [[],[]],
+            [AC_MSG_ERROR(use --with-cachepaths=DIRS (plural) instead of --with-cachepath=DIR to specify one or more cache paths)],
+            [])
+AC_ARG_WITH(commpath,
+            [AS_HELP_STRING([--with-compath=DIR],[Back-end directory for communication and housekeeping])],
+            [COMMPATH=${withval}],
+            [COMMPATH=$DEFAULT_LOC])
 AC_ARG_WITH(default-local-prefix,
             [AS_HELP_STRING([--with-default-local-prefix=DIRS],[Colon-seperated list of directories that Spindle will not cache files out of])],
             [SPINDLE_LOCAL_PREFIX=${withval}],
-            [SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$SPINDLE_LOC"])
+            [SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH"])
 AC_DEFINE_UNQUOTED([SPINDLE_PORT],[$SPINDLE_PORT],[The default port for Spindle])
 AC_DEFINE_UNQUOTED([NUM_COBO_PORTS],[$NUM_COBO_PORTS],[Number of ports for COBO to search for an open port])
 AC_DEFINE_UNQUOTED([SPINDLE_MAX_PORT],[$(($SPINDLE_PORT + $NUM_COBO_PORTS - 1))],[The maximum port value])
-AC_DEFINE_UNQUOTED([SPINDLE_LOC],"[$SPINDLE_LOC]",[The default local directory for Spindle])
+AC_DEFINE_UNQUOTED([COMMPATH],"[$COMMPATH]",[Back-end directory for communication and housekeeping])
+AC_DEFINE_UNQUOTED([CACHEPATHS],"[$CACHEPATHS]",[Colon-separated list of potential back-end cache directories])
 AC_DEFINE_UNQUOTED([SPINDLE_LOCAL_PREFIX],"[$SPINDLE_LOCAL_PREFIX]",[The default colon-separated list of directories that Spindle will not cache files out of])
 
 TESTRM=unknown

containers/spindle-flux-ubuntu/Dockerfile

@@ -5,6 +5,8 @@ FROM fluxrm/flux-sched:${flux_sched_version} AS builder
 ARG replicas=4
 ENV workers=${replicas}
 USER root
+ENV TMPDIR=/tmp
+RUN echo 'TMPDIR="/tmp"' >> /etc/environment
 
 RUN DEBIAN_FRONTEND="noninteractive" apt-get update \
     && apt-get -qq install -y --no-install-recommends \

containers/spindle-flux-ubuntu/scripts/build_spindle.sh

@@ -4,7 +4,7 @@ set -euxo pipefail
 
 mkdir -p /home/${USER}/Spindle-build
 cd /home/${USER}/Spindle-build
-/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=flux --enable-flux-plugin --with-localstorage=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
+/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=flux --enable-flux-plugin --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
 make -j$(nproc)
 make install
 

containers/spindle-serial-ubuntu/Dockerfile

@@ -1,6 +1,8 @@
 ARG ubuntu_version=noble
 FROM ubuntu:${ubuntu_version}
 USER root
+ENV TMPDIR=/tmp
+RUN echo 'TMPDIR="/tmp"' >> /etc/environment
 
 RUN DEBIAN_FRONTEND="noninteractive" apt-get update \
 # install latest pkg utils:

containers/spindle-serial-ubuntu/scripts/build_spindle.sh

@@ -4,7 +4,7 @@ set -euxo pipefail
 
 mkdir -p /home/${USER}/Spindle-build
 cd /home/${USER}/Spindle-build
-/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=serial --with-localstorage=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
+/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=serial --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
 make -j$(nproc)
 make install
 

containers/spindle-slurm-ubuntu/base/Dockerfile

@@ -1,6 +1,8 @@
 ARG UBUNTU_VERSION=noble
 FROM ubuntu:${UBUNTU_VERSION}
 USER root
+ENV TMPDIR=/tmp
+RUN echo 'TMPDIR="/tmp"' >> /etc/environment
 
 RUN apt-get update \
  && DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \

containers/spindle-slurm-ubuntu/testing-plugin/Dockerfile

@@ -2,6 +2,8 @@ ARG BASE_VERSION=latest
 FROM ghcr.io/llnl/spindle-slurm-base:${BASE_VERSION}
 ARG replicas=4
 ENV workers=${replicas}
+ENV TMPDIR=/tmp
+RUN echo 'TMPDIR="/tmp"' >> /etc/environment
 
 ARG BUILD_ROOT=containers/spindle-slurm-ubuntu/testing-plugin
 

containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh

@@ -3,7 +3,7 @@ set -euxo pipefail
 
 mkdir -p /home/${USER}/Spindle-build
 cd /home/${USER}/Spindle-build
-/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm-plugin --enable-slurm-plugin --with-localstorage=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
+/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm-plugin --enable-slurm-plugin --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
 make -j$(nproc)
 make install
 

containers/spindle-slurm-ubuntu/testing-srun/scripts/build_spindle.sh

@@ -3,7 +3,7 @@ set -euxo pipefail
 
 mkdir -p /home/${USER}/Spindle-build
 cd /home/${USER}/Spindle-build
-/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm --with-localstorage=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
+/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
 make -j$(nproc)
 make install
 

containers/spindle-slurm-ubuntu/testing/Dockerfile

@@ -2,6 +2,8 @@ ARG BASE_VERSION=latest
 FROM ghcr.io/llnl/spindle-slurm-base:${BASE_VERSION}
 ARG replicas=4
 ENV workers=${replicas}
+ENV TMPDIR=/tmp
+RUN echo 'TMPDIR="/tmp"' >> /etc/environment
 
 ARG BUILD_ROOT=containers/spindle-slurm-ubuntu/testing
 

containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh

@@ -3,7 +3,7 @@ set -euxo pipefail
 
 mkdir -p /home/${USER}/Spindle-build
 cd /home/${USER}/Spindle-build
-/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm --with-rsh-launch --with-rsh-cmd=/usr/bin/ssh --with-localstorage=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
+/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm --with-rsh-launch --with-rsh-cmd=/usr/bin/ssh --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
 make -j$(nproc)
 make install
 

src/client/beboot/spindle_bootstrap.c

@@ -53,7 +53,7 @@ static int rankinfo[4]={-1,-1,-1,-1};
 number_t number;
 static int use_cache;
 static unsigned int cachesize;
-static char *location, *number_s, *orig_location, *symbolic_location;
+static char *commpath, *number_s, *symbolic_commpath;
 static char **cmdline;
 static char *executable;
 static char *client_lib;
@@ -92,7 +92,7 @@ extern char *realize(char *path);
 static int establish_connection()
 {
    debug_printf2("Opening connection to server\n");
-   ldcsid = client_open_connection(location, number);
+   ldcsid = client_open_connection(commpath, number);
    if (ldcsid == -1) 
       return -1;
 
@@ -108,14 +108,18 @@ static void setup_environment()
 {
    char rankinfo_str[256];
    snprintf(rankinfo_str, 256, "%d %d %d %d %d", ldcsid, rankinfo[0], rankinfo[1], rankinfo[2], rankinfo[3]);
-   
+
    char *connection_str = NULL;
    if (opts & OPT_RELOCAOUT) 
       connection_str = client_get_connection_string(ldcsid);
 
+   char *chosen_parsed_cachepath = NULL;
+   send_cachepath_query( ldcsid , NULL, &chosen_parsed_cachepath);
+   assert( chosen_parsed_cachepath );
+
    setenv("LD_AUDIT", client_lib, 1);
-   setenv("LDCS_LOCATION", location, 1);
-   setenv("LDCS_ORIG_LOCATION", orig_location, 1);
+   setenv("LDCS_COMMPATH", commpath, 1);
+   setenv("LDCS_CHOSEN_PARSED_CACHEPATH", chosen_parsed_cachepath, 1);
    setenv("LDCS_NUMBER", number_s, 1);
    setenv("LDCS_RANKINFO", rankinfo_str, 1);
    if (connection_str)
@@ -161,7 +165,8 @@ static int parse_cmdline(int argc, char *argv[])
       daemon_args[i - 3] = NULL;
    }
 
-   symbolic_location = argv[i++];
+   symbolic_commpath = argv[i++];
+   i++; // Skip over candidate_cachepaths.
    number_s = argv[i++];
    number = (number_t) strtoul(number_s, NULL, 0);
    opts_s = argv[i++];
@@ -174,7 +179,7 @@ static int parse_cmdline(int argc, char *argv[])
    return 0;
 }
 
-static void launch_daemon(char *location)
+static void launch_daemon(char *commpath)
 {
    /*grand-child fork, then execv daemon.  By grand-child forking we ensure that
      the app won't get confused by seeing an unknown process as a child. */
@@ -184,12 +189,12 @@ static void launch_daemon(char *location)
    char unique_file[MAX_PATH_LEN+1];
    char buffer[32];
 
-   result = spindle_mkdir(location);
+   result = spindle_mkdir(commpath);
    if (result == -1) {
       debug_printf("Exiting due to spindle_mkdir error\n");
       exit(-1);
    }
-   snprintf(unique_file, MAX_PATH_LEN, "%s/spindle_daemon_pid", location);
+   snprintf(unique_file, MAX_PATH_LEN, "%s/spindle_daemon_pid", commpath);
    unique_file[MAX_PATH_LEN] = '\0';
    fd = open(unique_file, O_CREAT | O_EXCL | O_WRONLY, 0600);
    if (fd == -1) {
@@ -344,14 +349,14 @@ int main(int argc, char *argv[])
       }
    }
 
-   orig_location = parse_location(symbolic_location, number);
-   if (!orig_location) {
+   char *orig_commpath = parse_location(symbolic_commpath, number);
+   if (!orig_commpath) {
       return -1;
    }
-   location = realize(orig_location);
+   commpath = realize(orig_commpath);
 
    if (daemon_args) {
-      launch_daemon(location);
+      launch_daemon(commpath);
    }
 
    result = establish_connection();
@@ -375,7 +380,7 @@ int main(int argc, char *argv[])
 #else
       shm_cache_limit = cachesize;
 #endif
-      shmcache_init(location, number, cachesize, shm_cache_limit);
+      shmcache_init(commpath, number, cachesize, shm_cache_limit);
       use_cache = 1;
    }