Updated to add systematic errors switch and job scripts.

AraCorrelator/RayTraceCorrelator_support/make_timing_tables/config.dagman

@@ -0,0 +1,2 @@
+DAGMAN_MAX_JOBS_SUBMITTED = 1000
+DAGMAN_MAX_JOBS_IDLE = 1000

AraCorrelator/RayTraceCorrelator_support/make_timing_tables/makeRTArrivalTimeTables.cpp

@@ -22,7 +22,7 @@
 #include "Settings.h"
 
 std::map<int, Position> GetAntLocationsInEarthCoords(int station, IceModel *iceModel);
-void CalculateTables(RayTraceCorrelator *theCorrelator, int solNum, int iceModelidx, const std::string &tableDir);
+void CalculateTables(RayTraceCorrelator *theCorrelator, int solNum, int iceModelidx, const std::string &tableDir, int systematics);
 Position CalculateStationCOG(std::map<int, Position> antennaLocations);
 void CalculateArrivalInformation(
     RaySolver *raySolver,
@@ -38,32 +38,34 @@ int main(int argc, char **argv)
 
     gStyle->SetOptStat(0);
 
-    if(argc<4) {
-        std::cout << "Usage\n" << argv[0] << " <station> <radius> <output location>\n";
-        std::cout << "e.g.\n" << argv[0] << " 2 300 /path/to/my/home/dir \n";
+    if(argc<5) {
+        std::cout << "Usage\n" << argv[0] << " <station> <radius> <output location> <systematic> \n";
+        std::cout << "e.g.\n" << argv[0] << " 2 300 /path/to/my/home/dir 0 \n";
         return 0;
     }
 
     int station = atoi(argv[1]);
     double radius = atof(argv[2]);
+    int systematics = atof(argv[4]);
+    std::cout << "Systematics set to: " << systematics << endl;
 
     double angular_size = 1.;
-    int iceModelidx = 50;
+    int iceModelidx = 40;
+    // Use 40 for the UNL Modified (PA model). Related slide: https://aradocs.wipac.wisc.edu/docs/0022/002222/001/inIceMC_Hughes_A5locations_10222020.pdf
     int unixTime = 0;
     int numAntennas = 16;
 
-
     std::string tempFileName = "temp.txt"; // the tables don't exist yet, so we feed it a dummy path
     RayTraceCorrelator *theCorrelator = new RayTraceCorrelator(station, numAntennas, 
         radius, angular_size, tempFileName, tempFileName
     );
 
-    CalculateTables(theCorrelator, 0, iceModelidx, argv[3]);
-    CalculateTables(theCorrelator, 1, iceModelidx, argv[3]);
+    CalculateTables(theCorrelator, 0, iceModelidx, argv[3], systematics);
+    CalculateTables(theCorrelator, 1, iceModelidx, argv[3], systematics);
 
 }
 
-void CalculateTables(RayTraceCorrelator *theCorrelator, int solNum, int iceModelidx, const std::string &tableDir){
+void CalculateTables(RayTraceCorrelator *theCorrelator, int solNum, int iceModelidx, const std::string &tableDir, int systematics){
 
     int numThetaBins = theCorrelator->GetNumThetaBins();
     int numPhiBins = theCorrelator->GetNumPhiBins();
@@ -99,6 +101,26 @@ void CalculateTables(RayTraceCorrelator *theCorrelator, int solNum, int iceModel
 
     settings->NOFZ=1; // make sure n(z) is turned on
     settings->RAY_TRACE_ICE_MODEL_PARAMS = iceModelidx; // set the ice model as user requested
+    // Use 40 for the UNL Modified (PA model). Related slide: https://aradocs.wipac.wisc.edu/docs/0022/002222/001/inIceMC_Hughes_A5locations_10222020.pdf
+    // See here for updated systematic errors: https://aradocs.wipac.wisc.edu/cgi-bin/DocDB/ShowDocument?docid=3525
+
+    // Make it possible to run with upper/lower bounds of systematic error sources
+    if (systematics == 1){
+
+      settings->SYSTEMATICS_nofz_delta_ns = 0.08368;
+      settings->SYSTEMATICS_nofz_delta_nd = 0.00800;
+      settings->SYSTEMATICS_nofz_delta_nc = 0.00040;
+
+    }
+    else if (systematics == -1){
+      settings->SYSTEMATICS_nofz_delta_ns = -0.16000;
+      settings->SYSTEMATICS_nofz_delta_nd = -0.00600;
+      settings->SYSTEMATICS_nofz_delta_nc = -0.00470;
+
+    }
+    std::cout << "SYSTEMATICS_nofz_delta_ns: " << settings->SYSTEMATICS_nofz_delta_ns << endl;
+    std::cout << "SYSTEMATICS_nofz_delta_nd: " << settings->SYSTEMATICS_nofz_delta_nd << endl;
+    std::cout << "SYSTEMATICS_nofz_delta_nc: " << settings->SYSTEMATICS_nofz_delta_nc << endl;
 
     // open output file BEFORE making root containers
     char fileName[500];
@@ -221,6 +243,7 @@ void CalculateArrivalInformation(
     // outputs holder
     std::vector < std::vector < double > > outputs;
     vector < vector < vector <double> > > RayStep;
+
     raySolver->Solve_Ray(source, target, iceModel, outputs, settings, RayStep);
 
     if ((outputs.size() > 0)) {

AraCorrelator/RayTraceCorrelator_support/make_timing_tables/make_tables.dag

@@ -0,0 +1,138 @@
+CONFIG config.dagman
+
+## Station 1
+# Up
+JOB st1_r300_sysup   make_tables.sub
+VARS st1_r300_sysup  nodename="$(JOB)" station="100" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="300" systematic="1"
+
+JOB st1_r40_sysup1   make_tables.sub
+VARS st1_r40_sysup1  nodename="$(JOB)" station="100" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="48.02" systematic="1"
+
+JOB st1_r40_sysup2   make_tables.sub
+VARS st1_r40_sysup2  nodename="$(JOB)" station="100" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="48.91" systematic="1"
+
+# Down
+JOB st1_r300_sysdown make_tables.sub
+VARS st1_r300_sysdown nodename="$(JOB)" station="100" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="300" systematic="-1"
+
+JOB st1_r40_sysdown1 make_tables.sub
+VARS st1_r40_sysdown1 nodename="$(JOB)" station="100" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="48.02" systematic="-1"
+
+JOB st1_r40_sysdown2 make_tables.sub
+VARS st1_r40_sysdown2 nodename="$(JOB)" station="100" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="48.91" systematic="-1"
+
+## Station 2
+# Up
+JOB st2_r300_sysup   make_tables.sub
+VARS st2_r300_sysup  nodename="$(JOB)" station="2" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="300" systematic="1"
+
+JOB st2_r40_sysup1   make_tables.sub
+VARS st2_r40_sysup1  nodename="$(JOB)" station="2" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="42.86" systematic="1"
+
+JOB st2_r40_sysup2   make_tables.sub
+VARS st2_r40_sysup2  nodename="$(JOB)" station="2" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="44.07" systematic="1"
+
+JOB st2_r40_sysup3   make_tables.sub
+VARS st2_r40_sysup3  nodename="$(JOB)" station="2" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="47.51" systematic="1"
+
+JOB st2_r40_sysup4   make_tables.sub
+VARS st2_r40_sysup4  nodename="$(JOB)" station="2" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="47.96" systematic="1"
+
+
+# Down
+JOB st2_r300_sysdown make_tables.sub
+VARS st2_r300_sysdown nodename="$(JOB)" station="2" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="300" systematic="-1"
+
+JOB st2_r40_sysdown1 make_tables.sub
+VARS st2_r40_sysdown1 nodename="$(JOB)" station="2" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="42.86" systematic="-1"
+
+JOB st2_r40_sysdown2 make_tables.sub
+VARS st2_r40_sysdown2 nodename="$(JOB)" station="2" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="44.07" systematic="-1"
+
+JOB st2_r40_sysdown3 make_tables.sub
+VARS st2_r40_sysdown3 nodename="$(JOB)" station="2" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="47.51" systematic="-1"
+
+JOB st2_r40_sysdown4 make_tables.sub
+VARS st2_r40_sysdown4 nodename="$(JOB)" station="2" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="47.96" systematic="-1"
+
+
+## Station 3
+# Up
+JOB st3_r300_sysup   make_tables.sub
+VARS st3_r300_sysup  nodename="$(JOB)" station="3" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="300" systematic="1"
+
+JOB st3_r40_sysup1   make_tables.sub
+VARS st3_r40_sysup1  nodename="$(JOB)" station="3" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="41.08" systematic="1"
+
+JOB st3_r40_sysup2   make_tables.sub
+VARS st3_r40_sysup2  nodename="$(JOB)" station="3" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="41.11" systematic="1"
+
+JOB st3_r40_sysup3   make_tables.sub
+VARS st3_r40_sysup3  nodename="$(JOB)" station="3" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="41.77" systematic="1"
+
+JOB st3_r40_sysup4   make_tables.sub
+VARS st3_r40_sysup4  nodename="$(JOB)" station="3" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="41.90" systematic="1"
+
+
+# Down
+JOB st3_r300_sysdown make_tables.sub
+VARS st3_r300_sysdown nodename="$(JOB)" station="3" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="300" systematic="-1"
+
+JOB st3_r40_sysdown1 make_tables.sub
+VARS st3_r40_sysdown1 nodename="$(JOB)" station="3" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="41.08" systematic="-1"
+
+JOB st3_r40_sysdown2 make_tables.sub
+VARS st3_r40_sysdown2 nodename="$(JOB)" station="3" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="41.11" systematic="-1"
+
+JOB st3_r40_sysdown3 make_tables.sub
+VARS st3_r40_sysdown3 nodename="$(JOB)" station="3" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="41.77" systematic="-1"
+
+JOB st3_r40_sysdown4 make_tables.sub
+VARS st3_r40_sysdown4 nodename="$(JOB)" station="3" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="41.90" systematic="-1"
+
+
+## Station 4
+# Up
+JOB st4_r300_sysup   make_tables.sub
+VARS st4_r300_sysup  nodename="$(JOB)" station="4" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="300" systematic="1"
+
+JOB st4_r50_sysup1   make_tables.sub
+VARS st4_r50_sysup1  nodename="$(JOB)" station="4" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="52.60" systematic="1"
+
+JOB st4_r50_sysup2   make_tables.sub
+VARS st4_r50_sysup2  nodename="$(JOB)" station="4" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="53.18" systematic="1"
+
+
+# Down
+JOB st4_r300_sysdown make_tables.sub
+VARS st4_r300_sysdown nodename="$(JOB)" station="4" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="300" systematic="-1"
+
+JOB st4_r50_sysdown1 make_tables.sub
+VARS st4_r50_sysdown1 nodename="$(JOB)" station="4" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="52.60" systematic="-1"
+
+JOB st4_r50_sysdown2 make_tables.sub
+VARS st4_r50_sysdown2 nodename="$(JOB)" station="4" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="53.18" systematic="-1"
+
+
+## Station 5
+# Up
+JOB st5_r300_sysup   make_tables.sub
+VARS st5_r300_sysup  nodename="$(JOB)" station="5" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="300" systematic="1"
+
+JOB st5_r50_sysup1   make_tables.sub
+VARS st5_r50_sysup1  nodename="$(JOB)" station="5" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="49.23" systematic="1"
+
+JOB st5_r50_sysup2   make_tables.sub
+VARS st5_r50_sysup2  nodename="$(JOB)" station="5" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="49.29" systematic="1"
+
+
+# Down
+JOB st5_r300_sysdown make_tables.sub
+VARS st5_r300_sysdown nodename="$(JOB)" station="5" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="300" systematic="-1"
+
+JOB st5_r50_sysdown1 make_tables.sub
+VARS st5_r50_sysdown1 nodename="$(JOB)" station="5" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="49.23" systematic="-1"
+
+JOB st5_r50_sysdown2 make_tables.sub
+VARS st5_r50_sysdown2 nodename="$(JOB)" station="5" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="49.29" systematic="-1"
+

AraCorrelator/RayTraceCorrelator_support/make_timing_tables/make_tables.sh

@@ -1,32 +1,27 @@
 #!/bin/bash
-#SBATCH --job-name=makeTables
-#SBATCH --nodes=1
-#SBATCH --ntasks-per-node=1
-#SBATCH --time=04:00:00
-#SBATCH --export=ALL
-#SBATCH --array=0-33
-#SBATCH --output=logs/make_tables_%A_%a.out
-#SBATCH --error=logs/make_tables_%A_%a.err
 
-# this is just an example slurm script of how you might make many
-# of the ray trace timing tables on a cluster
+## We need to pass the following variables:
+# 1. Station number
+# 2. The output directory to store the table in
+# 3. The calpulser distance for the station
+# 4. The systematic error we're including on the ice model
+#	(1 for up, -1 for down, 0 (or anything) for nothing)
+station=$1
+outputdir=$2
+radius=$3
+systematic=$4
 
-# declare -a radii=( 
-#   100. 111.02320645 123.26152371 136.84889595 151.93403228
-#   168.68203434 187.27620324 207.92004576 230.83950166 256.28541651
-#   284.53628709 315.90130945 350.72376298 389.38476746 432.30745428
-#   479.96159749 532.86875528 591.60797831 656.82214716 729.22500848
-#   809.60898669 898.85385677 997.93637312 1107.94095982 1230.07157922
-#  1365.66490893 1516.20497132 1683.33937559 1868.89735029 2074.90976364
-#  2303.63135064 2557.56539037 2839.49110356 3152.49407017 3500.
-# )
+source ~/.bashrc
 
-declare -a radii=(
-  0  150  300  450  600  750  900 1050 1200 1350 1500 1650 1800 1950
- 2100 2250 2400 2550 2700 2850 3000 3150 3300 3450 3600 3750 3900 4050
- 4200 4350 4500 4650 4800 4950
-)
-cd /mnt/home/baclark/ara/araroot_pedloader/src/AraCorrelator/RayTraceCorrelator_support/make_timing_tables
-source /mnt/home/baclark/ara/energy_reco_work/energy_reco/code/env_arasim.py
-# echo ${radii[SLURM_ARRAY_TASK_ID]}
-./makeRTArrivalTimeTables 2 ${radii[SLURM_ARRAY_TASK_ID]} . 
+## Get the environment variables necessary for running makeRTArrivalTables
+#source /cvmfs/ara.opensciencegrid.org/trunk/alma9/setup.sh
+### We need to point to our AraSim version with the systematics
+new_ray_tables ## Need to load AraProc, Fivestation, etc. + AraSim on the correct branch
+
+
+cd ~/ray_trace_tables
+
+## Cobalt wants you to put outputs on TMPDIR I guess
+## So send it there and copy over 
+./makeRTArrivalTimeTables "${station}" "${radius}" "${TMPDIR}" "${systematic}" 
+cp "${TMPDIR}"/* "${outputdir}"

AraCorrelator/RayTraceCorrelator_support/make_timing_tables/make_tables.sub

@@ -0,0 +1,17 @@
+log    = /scratch/amachtay/ray_trace_tables/log/$(nodename).log
+output = /scratch/amachtay/ray_trace_tables/out/$(nodename).out
+error  = /scratch/amachtay/ray_trace_tables/err/$(nodename).err
+stream_output = True
+stream_error  = True
+requirements  = has_avx2
+request_cpus  = 1
+request_disk  = 1GB
+request_memory = 1000
+Universe = vanilla
+should_transfer_files   = YES
+when_to_transfer_output = ON_EXIT
+
+executable = /home/amachtay/ray_trace_tables/make_tables.sh
+arguments  = $(station) $(outputdir) $(radius) $(systematic)
+
+queue