diff --git a/AraCorrelator/RayTraceCorrelator_support/make_timing_tables/config.dagman b/AraCorrelator/RayTraceCorrelator_support/make_timing_tables/config.dagman new file mode 100644 index 0000000..cf07cf3 --- /dev/null +++ b/AraCorrelator/RayTraceCorrelator_support/make_timing_tables/config.dagman @@ -0,0 +1,2 @@ +DAGMAN_MAX_JOBS_SUBMITTED = 1000 +DAGMAN_MAX_JOBS_IDLE = 1000 diff --git a/AraCorrelator/RayTraceCorrelator_support/make_timing_tables/makeRTArrivalTimeTables.cpp b/AraCorrelator/RayTraceCorrelator_support/make_timing_tables/makeRTArrivalTimeTables.cpp index 667072f..c3d01a5 100644 --- a/AraCorrelator/RayTraceCorrelator_support/make_timing_tables/makeRTArrivalTimeTables.cpp +++ b/AraCorrelator/RayTraceCorrelator_support/make_timing_tables/makeRTArrivalTimeTables.cpp @@ -22,7 +22,7 @@ #include "Settings.h" std::map GetAntLocationsInEarthCoords(int station, IceModel *iceModel); -void CalculateTables(RayTraceCorrelator *theCorrelator, int solNum, int iceModelidx, const std::string &tableDir); +void CalculateTables(RayTraceCorrelator *theCorrelator, int solNum, int iceModelidx, const std::string &tableDir, int systematics); Position CalculateStationCOG(std::map antennaLocations); void CalculateArrivalInformation( RaySolver *raySolver, @@ -38,32 +38,34 @@ int main(int argc, char **argv) gStyle->SetOptStat(0); - if(argc<4) { - std::cout << "Usage\n" << argv[0] << " \n"; - std::cout << "e.g.\n" << argv[0] << " 2 300 /path/to/my/home/dir \n"; + if(argc<5) { + std::cout << "Usage\n" << argv[0] << " \n"; + std::cout << "e.g.\n" << argv[0] << " 2 300 /path/to/my/home/dir 0 \n"; return 0; } int station = atoi(argv[1]); double radius = atof(argv[2]); + int systematics = atof(argv[4]); + std::cout << "Systematics set to: " << systematics << endl; double angular_size = 1.; - int iceModelidx = 50; + int iceModelidx = 40; + // Use 40 for the UNL Modified (PA model). Related slide: https://aradocs.wipac.wisc.edu/docs/0022/002222/001/inIceMC_Hughes_A5locations_10222020.pdf int unixTime = 0; int numAntennas = 16; - std::string tempFileName = "temp.txt"; // the tables don't exist yet, so we feed it a dummy path RayTraceCorrelator *theCorrelator = new RayTraceCorrelator(station, numAntennas, radius, angular_size, tempFileName, tempFileName ); - CalculateTables(theCorrelator, 0, iceModelidx, argv[3]); - CalculateTables(theCorrelator, 1, iceModelidx, argv[3]); + CalculateTables(theCorrelator, 0, iceModelidx, argv[3], systematics); + CalculateTables(theCorrelator, 1, iceModelidx, argv[3], systematics); } -void CalculateTables(RayTraceCorrelator *theCorrelator, int solNum, int iceModelidx, const std::string &tableDir){ +void CalculateTables(RayTraceCorrelator *theCorrelator, int solNum, int iceModelidx, const std::string &tableDir, int systematics){ int numThetaBins = theCorrelator->GetNumThetaBins(); int numPhiBins = theCorrelator->GetNumPhiBins(); @@ -99,6 +101,26 @@ void CalculateTables(RayTraceCorrelator *theCorrelator, int solNum, int iceModel settings->NOFZ=1; // make sure n(z) is turned on settings->RAY_TRACE_ICE_MODEL_PARAMS = iceModelidx; // set the ice model as user requested + // Use 40 for the UNL Modified (PA model). Related slide: https://aradocs.wipac.wisc.edu/docs/0022/002222/001/inIceMC_Hughes_A5locations_10222020.pdf + // See here for updated systematic errors: https://aradocs.wipac.wisc.edu/cgi-bin/DocDB/ShowDocument?docid=3525 + + // Make it possible to run with upper/lower bounds of systematic error sources + if (systematics == 1){ + + settings->SYSTEMATICS_nofz_delta_ns = 0.08368; + settings->SYSTEMATICS_nofz_delta_nd = 0.00800; + settings->SYSTEMATICS_nofz_delta_nc = 0.00040; + + } + else if (systematics == -1){ + settings->SYSTEMATICS_nofz_delta_ns = -0.16000; + settings->SYSTEMATICS_nofz_delta_nd = -0.00600; + settings->SYSTEMATICS_nofz_delta_nc = -0.00470; + + } + std::cout << "SYSTEMATICS_nofz_delta_ns: " << settings->SYSTEMATICS_nofz_delta_ns << endl; + std::cout << "SYSTEMATICS_nofz_delta_nd: " << settings->SYSTEMATICS_nofz_delta_nd << endl; + std::cout << "SYSTEMATICS_nofz_delta_nc: " << settings->SYSTEMATICS_nofz_delta_nc << endl; // open output file BEFORE making root containers char fileName[500]; @@ -221,6 +243,7 @@ void CalculateArrivalInformation( // outputs holder std::vector < std::vector < double > > outputs; vector < vector < vector > > RayStep; + raySolver->Solve_Ray(source, target, iceModel, outputs, settings, RayStep); if ((outputs.size() > 0)) { diff --git a/AraCorrelator/RayTraceCorrelator_support/make_timing_tables/make_tables.dag b/AraCorrelator/RayTraceCorrelator_support/make_timing_tables/make_tables.dag new file mode 100644 index 0000000..c34f8f8 --- /dev/null +++ b/AraCorrelator/RayTraceCorrelator_support/make_timing_tables/make_tables.dag @@ -0,0 +1,138 @@ +CONFIG config.dagman + +## Station 1 +# Up +JOB st1_r300_sysup make_tables.sub +VARS st1_r300_sysup nodename="$(JOB)" station="100" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="300" systematic="1" + +JOB st1_r40_sysup1 make_tables.sub +VARS st1_r40_sysup1 nodename="$(JOB)" station="100" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="48.02" systematic="1" + +JOB st1_r40_sysup2 make_tables.sub +VARS st1_r40_sysup2 nodename="$(JOB)" station="100" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="48.91" systematic="1" + +# Down +JOB st1_r300_sysdown make_tables.sub +VARS st1_r300_sysdown nodename="$(JOB)" station="100" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="300" systematic="-1" + +JOB st1_r40_sysdown1 make_tables.sub +VARS st1_r40_sysdown1 nodename="$(JOB)" station="100" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="48.02" systematic="-1" + +JOB st1_r40_sysdown2 make_tables.sub +VARS st1_r40_sysdown2 nodename="$(JOB)" station="100" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="48.91" systematic="-1" + +## Station 2 +# Up +JOB st2_r300_sysup make_tables.sub +VARS st2_r300_sysup nodename="$(JOB)" station="2" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="300" systematic="1" + +JOB st2_r40_sysup1 make_tables.sub +VARS st2_r40_sysup1 nodename="$(JOB)" station="2" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="42.86" systematic="1" + +JOB st2_r40_sysup2 make_tables.sub +VARS st2_r40_sysup2 nodename="$(JOB)" station="2" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="44.07" systematic="1" + +JOB st2_r40_sysup3 make_tables.sub +VARS st2_r40_sysup3 nodename="$(JOB)" station="2" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="47.51" systematic="1" + +JOB st2_r40_sysup4 make_tables.sub +VARS st2_r40_sysup4 nodename="$(JOB)" station="2" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="47.96" systematic="1" + + +# Down +JOB st2_r300_sysdown make_tables.sub +VARS st2_r300_sysdown nodename="$(JOB)" station="2" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="300" systematic="-1" + +JOB st2_r40_sysdown1 make_tables.sub +VARS st2_r40_sysdown1 nodename="$(JOB)" station="2" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="42.86" systematic="-1" + +JOB st2_r40_sysdown2 make_tables.sub +VARS st2_r40_sysdown2 nodename="$(JOB)" station="2" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="44.07" systematic="-1" + +JOB st2_r40_sysdown3 make_tables.sub +VARS st2_r40_sysdown3 nodename="$(JOB)" station="2" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="47.51" systematic="-1" + +JOB st2_r40_sysdown4 make_tables.sub +VARS st2_r40_sysdown4 nodename="$(JOB)" station="2" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="47.96" systematic="-1" + + +## Station 3 +# Up +JOB st3_r300_sysup make_tables.sub +VARS st3_r300_sysup nodename="$(JOB)" station="3" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="300" systematic="1" + +JOB st3_r40_sysup1 make_tables.sub +VARS st3_r40_sysup1 nodename="$(JOB)" station="3" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="41.08" systematic="1" + +JOB st3_r40_sysup2 make_tables.sub +VARS st3_r40_sysup2 nodename="$(JOB)" station="3" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="41.11" systematic="1" + +JOB st3_r40_sysup3 make_tables.sub +VARS st3_r40_sysup3 nodename="$(JOB)" station="3" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="41.77" systematic="1" + +JOB st3_r40_sysup4 make_tables.sub +VARS st3_r40_sysup4 nodename="$(JOB)" station="3" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="41.90" systematic="1" + + +# Down +JOB st3_r300_sysdown make_tables.sub +VARS st3_r300_sysdown nodename="$(JOB)" station="3" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="300" systematic="-1" + +JOB st3_r40_sysdown1 make_tables.sub +VARS st3_r40_sysdown1 nodename="$(JOB)" station="3" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="41.08" systematic="-1" + +JOB st3_r40_sysdown2 make_tables.sub +VARS st3_r40_sysdown2 nodename="$(JOB)" station="3" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="41.11" systematic="-1" + +JOB st3_r40_sysdown3 make_tables.sub +VARS st3_r40_sysdown3 nodename="$(JOB)" station="3" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="41.77" systematic="-1" + +JOB st3_r40_sysdown4 make_tables.sub +VARS st3_r40_sysdown4 nodename="$(JOB)" station="3" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="41.90" systematic="-1" + + +## Station 4 +# Up +JOB st4_r300_sysup make_tables.sub +VARS st4_r300_sysup nodename="$(JOB)" station="4" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="300" systematic="1" + +JOB st4_r50_sysup1 make_tables.sub +VARS st4_r50_sysup1 nodename="$(JOB)" station="4" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="52.60" systematic="1" + +JOB st4_r50_sysup2 make_tables.sub +VARS st4_r50_sysup2 nodename="$(JOB)" station="4" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="53.18" systematic="1" + + +# Down +JOB st4_r300_sysdown make_tables.sub +VARS st4_r300_sysdown nodename="$(JOB)" station="4" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="300" systematic="-1" + +JOB st4_r50_sysdown1 make_tables.sub +VARS st4_r50_sysdown1 nodename="$(JOB)" station="4" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="52.60" systematic="-1" + +JOB st4_r50_sysdown2 make_tables.sub +VARS st4_r50_sysdown2 nodename="$(JOB)" station="4" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="53.18" systematic="-1" + + +## Station 5 +# Up +JOB st5_r300_sysup make_tables.sub +VARS st5_r300_sysup nodename="$(JOB)" station="5" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="300" systematic="1" + +JOB st5_r50_sysup1 make_tables.sub +VARS st5_r50_sysup1 nodename="$(JOB)" station="5" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="49.23" systematic="1" + +JOB st5_r50_sysup2 make_tables.sub +VARS st5_r50_sysup2 nodename="$(JOB)" station="5" outputdir="/data/user/amachtay/ray_trace_tables/up" radius="49.29" systematic="1" + + +# Down +JOB st5_r300_sysdown make_tables.sub +VARS st5_r300_sysdown nodename="$(JOB)" station="5" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="300" systematic="-1" + +JOB st5_r50_sysdown1 make_tables.sub +VARS st5_r50_sysdown1 nodename="$(JOB)" station="5" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="49.23" systematic="-1" + +JOB st5_r50_sysdown2 make_tables.sub +VARS st5_r50_sysdown2 nodename="$(JOB)" station="5" outputdir="/data/user/amachtay/ray_trace_tables/down" radius="49.29" systematic="-1" + diff --git a/AraCorrelator/RayTraceCorrelator_support/make_timing_tables/make_tables.sh b/AraCorrelator/RayTraceCorrelator_support/make_timing_tables/make_tables.sh index ddf39e5..a223300 100644 --- a/AraCorrelator/RayTraceCorrelator_support/make_timing_tables/make_tables.sh +++ b/AraCorrelator/RayTraceCorrelator_support/make_timing_tables/make_tables.sh @@ -1,32 +1,27 @@ #!/bin/bash -#SBATCH --job-name=makeTables -#SBATCH --nodes=1 -#SBATCH --ntasks-per-node=1 -#SBATCH --time=04:00:00 -#SBATCH --export=ALL -#SBATCH --array=0-33 -#SBATCH --output=logs/make_tables_%A_%a.out -#SBATCH --error=logs/make_tables_%A_%a.err -# this is just an example slurm script of how you might make many -# of the ray trace timing tables on a cluster +## We need to pass the following variables: +# 1. Station number +# 2. The output directory to store the table in +# 3. The calpulser distance for the station +# 4. The systematic error we're including on the ice model +# (1 for up, -1 for down, 0 (or anything) for nothing) +station=$1 +outputdir=$2 +radius=$3 +systematic=$4 -# declare -a radii=( -# 100. 111.02320645 123.26152371 136.84889595 151.93403228 -# 168.68203434 187.27620324 207.92004576 230.83950166 256.28541651 -# 284.53628709 315.90130945 350.72376298 389.38476746 432.30745428 -# 479.96159749 532.86875528 591.60797831 656.82214716 729.22500848 -# 809.60898669 898.85385677 997.93637312 1107.94095982 1230.07157922 -# 1365.66490893 1516.20497132 1683.33937559 1868.89735029 2074.90976364 -# 2303.63135064 2557.56539037 2839.49110356 3152.49407017 3500. -# ) +source ~/.bashrc -declare -a radii=( - 0 150 300 450 600 750 900 1050 1200 1350 1500 1650 1800 1950 - 2100 2250 2400 2550 2700 2850 3000 3150 3300 3450 3600 3750 3900 4050 - 4200 4350 4500 4650 4800 4950 -) -cd /mnt/home/baclark/ara/araroot_pedloader/src/AraCorrelator/RayTraceCorrelator_support/make_timing_tables -source /mnt/home/baclark/ara/energy_reco_work/energy_reco/code/env_arasim.py -# echo ${radii[SLURM_ARRAY_TASK_ID]} -./makeRTArrivalTimeTables 2 ${radii[SLURM_ARRAY_TASK_ID]} . +## Get the environment variables necessary for running makeRTArrivalTables +#source /cvmfs/ara.opensciencegrid.org/trunk/alma9/setup.sh +### We need to point to our AraSim version with the systematics +new_ray_tables ## Need to load AraProc, Fivestation, etc. + AraSim on the correct branch + + +cd ~/ray_trace_tables + +## Cobalt wants you to put outputs on TMPDIR I guess +## So send it there and copy over +./makeRTArrivalTimeTables "${station}" "${radius}" "${TMPDIR}" "${systematic}" +cp "${TMPDIR}"/* "${outputdir}" diff --git a/AraCorrelator/RayTraceCorrelator_support/make_timing_tables/make_tables.sub b/AraCorrelator/RayTraceCorrelator_support/make_timing_tables/make_tables.sub new file mode 100644 index 0000000..71a5be0 --- /dev/null +++ b/AraCorrelator/RayTraceCorrelator_support/make_timing_tables/make_tables.sub @@ -0,0 +1,17 @@ +log = /scratch/amachtay/ray_trace_tables/log/$(nodename).log +output = /scratch/amachtay/ray_trace_tables/out/$(nodename).out +error = /scratch/amachtay/ray_trace_tables/err/$(nodename).err +stream_output = True +stream_error = True +requirements = has_avx2 +request_cpus = 1 +request_disk = 1GB +request_memory = 1000 +Universe = vanilla +should_transfer_files = YES +when_to_transfer_output = ON_EXIT + +executable = /home/amachtay/ray_trace_tables/make_tables.sh +arguments = $(station) $(outputdir) $(radius) $(systematic) + +queue