diff --git a/scripts/ANALYSIS.anasum.sh b/scripts/ANALYSIS.anasum.sh index 7b58e593..07801675 100755 --- a/scripts/ANALYSIS.anasum.sh +++ b/scripts/ANALYSIS.anasum.sh @@ -87,8 +87,8 @@ sed -e "s|FILELIST|$FLIST|" \ chmod u+x "$FSCRIPT.sh" echo "$FSCRIPT.sh" -SUBC="$(dirname "$0")/helper_scripts/UTILITY.readSubmissionCommand.sh" -# run locally or on cluster +SUBC=`$( dirname "$0" )/helper_scripts/UTILITY.readSubmissionCommand.sh` +# run locally or on cluster (expand shell variables in submission string) SUBC=`eval "echo \"$SUBC\""` if [[ $SUBC == *"ERROR"* ]]; then echo "$SUBC" @@ -113,5 +113,3 @@ elif [[ $SUBC == *parallel* ]]; then elif [[ $SUBC == *simple* ]]; then "$FSCRIPT.sh" |& tee $FSCRIPT.log fi - -exit diff --git a/scripts/ANALYSIS.anasum_allcuts.sh b/scripts/ANALYSIS.anasum_allcuts.sh index b7dbe2e4..cc6cc2a6 100755 --- a/scripts/ANALYSIS.anasum_allcuts.sh +++ b/scripts/ANALYSIS.anasum_allcuts.sh @@ -63,7 +63,7 @@ do fi echo $CDIR mkdir -p "$TMPDIR/${CDIR}" - ./ANALYSIS.anasum_parallel_from_runlist.sh ${RUNL} \ + $(dirname "$0")/ANALYSIS.anasum_parallel_from_runlist.sh ${RUNL} \ "$TMPDIR/${CDIR}" \ ${C} \ ${IGNORETYPE} \ @@ -71,7 +71,7 @@ do $PREDIR $SKIPIFPROCESSED | tee -a ${TMPLOG} elif [[ $RUNTYPE == "V2DL3" ]]; then mkdir -p "$TMPDIR/v2dl3_${C}" - ./ANALYSIS.v2dl3.sh ${RUNL} \ + $(dirname "$0")/ANALYSIS.v2dl3.sh ${RUNL} \ "$TMPDIR/v2dl3_${C}" \ ${C} | tee -a ${TMPLOG} else diff --git a/scripts/ANALYSIS.anasum_combine.sh b/scripts/ANALYSIS.anasum_combine.sh index 17a54e20..093928d8 100755 --- a/scripts/ANALYSIS.anasum_combine.sh +++ b/scripts/ANALYSIS.anasum_combine.sh @@ -76,6 +76,15 @@ SUBSCRIPT=$( dirname "$0" )"/helper_scripts/ANALYSIS.anasum_combine_sub" FSCRIPT="$LOGDIR/anasum_combine-$DATE-RUN$RUN-$(date +%s)" echo "Run script written to $FSCRIPT" +# Check that anasum output files exist before combining +NANASUMFILES=$(find "$DDIR" -maxdepth 1 -name "*.anasum.root" | wc -l) +if [[ $NANASUMFILES -eq 0 ]]; then + echo "Error: no .anasum.root files found in $DDIR" + echo "Run ANALYSIS.anasum_parallel_from_runlist.sh and wait for completion before combining." + exit 1 +fi +echo "Found $NANASUMFILES anasum output files in $DDIR" + sed -e "s|RRUNLIST|$RUNLIST|" \ -e "s|DDDIR|$DDIR|" \ -e "s|RRUNP|$RUNP|" \ diff --git a/scripts/ANALYSIS.mscw_energy.sh b/scripts/ANALYSIS.mscw_energy.sh index 82459345..3780ee3c 100755 --- a/scripts/ANALYSIS.mscw_energy.sh +++ b/scripts/ANALYSIS.mscw_energy.sh @@ -12,7 +12,7 @@ if [ "$#" -lt 2 ]; then echo " MSCW_ENERGY data analysis: submit jobs from a simple run list -ANALYSIS.mscw_energy.sh [output directory] [evndisp directory] [preprocessing skip] [Rec ID] [ATM] +ANALYSIS.mscw_energy.sh [output directory] [evndisp directory] [preprocessing skip] [Rec ID] required parameters: @@ -32,9 +32,6 @@ optional parameters: [Rec ID] reconstruction ID. Default 0 (see EVNDISP.reconstruction.runparameter) - [ATM] set atmosphere ID (overwrite the value from the evndisp stage) - - The analysis type (cleaning method; direction reconstruction) is read from the \$VERITAS_ANALYSIS_TYPE environmental variable (e.g., AP_DISP, NN_DISP; here set to: \"$VERITAS_ANALYSIS_TYPE\"). @@ -60,7 +57,6 @@ RUNLIST=$1 [[ "$3" ]] && INPUTDIR=$3 || INPUTDIR="$VERITAS_PREPROCESSED_DATA_DIR/${VERITAS_ANALYSIS_TYPE:0:2}/evndisp" [[ "$4" ]] && SKIP=$4 || SKIP=1 [[ "$5" ]] && ID=$5 || ID=0 -[[ "$6" ]] && FORCEDATMO=$6 DISPBDT="1" # Read runlist diff --git a/scripts/ANALYSIS.v2dl3.sh b/scripts/ANALYSIS.v2dl3.sh index c0f2ab79..77f3cad5 100755 --- a/scripts/ANALYSIS.v2dl3.sh +++ b/scripts/ANALYSIS.v2dl3.sh @@ -54,7 +54,7 @@ DATE=`date +"%y%m%d"` LOGDIR="$VERITAS_USER_LOG_DIR/V2DL3-${DATE}-$(uuidgen)/" mkdir -p "$LOGDIR" echo -e "Log files will be written to:\n $LOGDIR" -rm -f ${LOGIDR}/x* 2>/dev/null +rm -f ${LOGDIR}/x* 2>/dev/null # split run list into smaller run lists sort -u "${RUNLIST}" -o "${LOGDIR}/$(basename "${RUNLIST}")" @@ -98,11 +98,11 @@ do JOBID=$( echo "$JOBID" | grep -oP "Your job [0-9.-:]+" | awk '{ print $3 }' ) fi - echo "RUN $AFILE JOBID $JOBID" - echo "RUN $AFILE SCRIPT $FSCRIPT.sh" + echo "RUN $J JOBID $JOBID" + echo "RUN $J SCRIPT $FSCRIPT.sh" if [[ $SUBC != */dev/null* ]] ; then - echo "RUN $AFILE OLOG $FSCRIPT.sh.o$JOBID" - echo "RUN $AFILE ELOG $FSCRIPT.sh.e$JOBID" + echo "RUN $J OLOG $FSCRIPT.sh.o$JOBID" + echo "RUN $J ELOG $FSCRIPT.sh.e$JOBID" fi elif [[ $SUBC == *condor* ]]; then $(dirname "$0")/helper_scripts/UTILITY.condorSubmission.sh $FSCRIPT.sh $h_vmem $tmpdir_size @@ -116,7 +116,7 @@ do $SUBC $FSCRIPT.sh elif [[ $SUBC == *parallel* ]]; then echo "$FSCRIPT.sh &> $FSCRIPT.log" >> "$LOGDIR/runscripts.$TIMETAG.dat" - echo "RUN $AFILE OLOG $FSCRIPT.log" + echo "RUN $J OLOG $FSCRIPT.log" elif [[ "$SUBC" == *simple* ]] ; then "$FSCRIPT.sh" |& tee "$FSCRIPT.log" fi diff --git a/scripts/IRF.combine_lookup_table_parts.sh b/scripts/IRF.combine_lookup_table_parts.sh index f7cc1fe3..f4a43197 100755 --- a/scripts/IRF.combine_lookup_table_parts.sh +++ b/scripts/IRF.combine_lookup_table_parts.sh @@ -74,7 +74,7 @@ echo -e "Output files will be written to:\n $ODIR" mkdir -p "$ODIR" chmod g+w "$ODIR" -if [[ -f $OFILE ]]; then +if [[ -f "$ODIR/$OFILE" ]]; then echo "ERROR: table file $ODIR/$OFILE exists; move it or delete it" exit 1 fi diff --git a/scripts/IRF.generate_lookup_table_parts.sh b/scripts/IRF.generate_lookup_table_parts.sh index 13b8b053..907e00b0 100755 --- a/scripts/IRF.generate_lookup_table_parts.sh +++ b/scripts/IRF.generate_lookup_table_parts.sh @@ -112,18 +112,18 @@ if [[ $SUBC == *"ERROR"* ]]; then exit 1 fi if [[ $SUBC == *qsub* ]]; then - JOBID=`$SUBC $FSCRIPT.sh` + JOBID=`$SUBC $FSCRIPT` echo "JOBID: $JOBID" elif [[ $SUBC == *condor* ]]; then - $(dirname "$0")/helper_scripts/UTILITY.condorSubmission.sh $FSCRIPT.sh $h_vmem $tmpdir_size + $(dirname "$0")/helper_scripts/UTILITY.condorSubmission.sh $FSCRIPT $h_vmem $tmpdir_size echo "-------------------------------------------------------------------------------" echo "Job submission using HTCondor - run the following script to submit jobs:" echo "$EVNDISPSCRIPTS/helper_scripts/submit_scripts_to_htcondor.sh ${LOGDIR} submit" echo "-------------------------------------------------------------------------------" elif [[ $SUBC == *sbatch* ]]; then - $SUBC $FSCRIPT.sh + $SUBC $FSCRIPT elif [[ $SUBC == *parallel* ]]; then - echo "$FSCRIPT.sh &> $FSCRIPT.log" >> "$LOGDIR/runscripts.dat" + echo "$FSCRIPT &> $FSCRIPT.log" >> "$LOGDIR/runscripts.dat" elif [[ "$SUBC" == *simple* ]]; then - "$FSCRIPT.sh" | tee "$FSCRIPT.log" + "$FSCRIPT" | tee "$FSCRIPT.log" fi diff --git a/scripts/IRF.mscw_energy_MC.sh b/scripts/IRF.mscw_energy_MC.sh index 8aa705f2..48d9c203 100755 --- a/scripts/IRF.mscw_energy_MC.sh +++ b/scripts/IRF.mscw_energy_MC.sh @@ -137,18 +137,18 @@ if [[ $SUBC == *"ERROR"* ]]; then exit 1 fi if [[ $SUBC == *qsub* ]]; then - JOBID=`$SUBC $FSCRIPT.sh` + JOBID=`$SUBC $FSCRIPT` echo "JOBID: $JOBID" elif [[ $SUBC == *condor* ]]; then - $(dirname "$0")/helper_scripts/UTILITY.condorSubmission.sh $FSCRIPT.sh $h_vmem $tmpdir_size + $(dirname "$0")/helper_scripts/UTILITY.condorSubmission.sh $FSCRIPT $h_vmem $tmpdir_size echo "-------------------------------------------------------------------------------" echo "Job submission using HTCondor - run the following script to submit jobs:" echo "$EVNDISPSCRIPTS/helper_scripts/submit_scripts_to_htcondor.sh ${LOGDIR} submit" echo "-------------------------------------------------------------------------------" elif [[ $SUBC == *sbatch* ]]; then - $SUBC $FSCRIPT.sh + $SUBC $FSCRIPT elif [[ $SUBC == *parallel* ]]; then - echo "$FSCRIPT.sh &> $FSCRIPT.log" >> "$LOGDIR/runscripts.dat" + echo "$FSCRIPT &> $FSCRIPT.log" >> "$LOGDIR/runscripts.dat" elif [[ "$SUBC" == *simple* ]]; then - "$FSCRIPT.sh" | tee "$FSCRIPT.log" + "$FSCRIPT" | tee "$FSCRIPT.log" fi diff --git a/scripts/IRF.optimizeTMVAforGammaHadronSeparation.sh b/scripts/IRF.optimizeTMVAforGammaHadronSeparation.sh index 00341f44..7506ef9e 100755 --- a/scripts/IRF.optimizeTMVAforGammaHadronSeparation.sh +++ b/scripts/IRF.optimizeTMVAforGammaHadronSeparation.sh @@ -80,6 +80,7 @@ fi if [[ ! -e $VERITAS_EVNDISP_AUX_DIR/EffectiveAreas/${EFFFILE} ]]; then echo "Error - effective area file not found ${EFFFILE}" + exit 1 fi if [[ ${EPOCH:0:2} == "V4" ]] || [[ ${EPOCH:0:2} == "V5" ]]; then RUNPAR="$VERITAS_EVNDISP_AUX_DIR/ParameterFiles/TMVA.BDT.V4.runparameter" @@ -88,8 +89,8 @@ else fi ##################################### # energy bins -if grep -q "^* ENERGYBINS" "$RUNPAR"; then - ENBINS=$( cat "$RUNPAR" | grep "^* ENERGYBINS 1" | sed -e 's/* ENERGYBINS 1//' | sed -e 's/ /\n/g') +if grep -q "^\* ENERGYBINS" "$RUNPAR"; then + ENBINS=$( cat "$RUNPAR" | grep "^\* ENERGYBINS" | sed -e 's/\* ENERGYBINS//' | sed -e 's/ /\n/g') declare -a EBINARRAY=( $ENBINS ) #convert to array count1=1 NENE=$((${#EBINARRAY[@]}-$count1)) #get number of bins diff --git a/scripts/IRF.production.sh b/scripts/IRF.production.sh index 61735277..02f766ec 100755 --- a/scripts/IRF.production.sh +++ b/scripts/IRF.production.sh @@ -308,7 +308,7 @@ for VX in $EPOCH; do if [[ $IRFTYPE == "ANAXGBANGRES" ]]; then MSCWDIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANATYPE}/${SIMTYPE}/${VX}_ATM${ATM}_gamma/MSCW_RECID${RECID}_DISP" echo "XGB reconstruction reading from $MSCWDIR" - ./IRF.dispXGB.sh "stereo_analysis" "${MSCWDIR}" "${MSCWDIR}" + $(dirname "$0")/IRF.dispXGB.sh "stereo_analysis" "${MSCWDIR}" "${MSCWDIR}" continue fi ############################################# @@ -316,7 +316,7 @@ for VX in $EPOCH; do if [[ $IRFTYPE == "ANAXGBGH" ]]; then MSCWDIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANATYPE}/${SIMTYPE}/${VX}_ATM${ATM}_gamma/MSCW_RECID${RECID}_DISP" echo "XGB classification reading from $MSCWDIR" - ./IRF.dispXGB.sh "classification" "${MSCWDIR}" "${MSCWDIR}" + $(dirname "$0")/IRF.dispXGB.sh "classification" "${MSCWDIR}" "${MSCWDIR}" continue fi ############################################# @@ -327,7 +327,7 @@ for VX in $EPOCH; do ODIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANATYPE}/${SIMTYPE}/${VX}_ATM${ATM}_gamma/TrainXGBGammaHadron" echo "XGB Classification Training" echo "${BCKDIR}" "${RUNPAR}" "${ODIR}" "${SIMTYPE}" "${VX}" "${ATM}" - ./IRF.trainXGBforGammaHadronSeparationTraining.sh "${BCKDIR}" "${RUNPAR}" "${ODIR}" "${SIMTYPE}" "${VX}" "${ATM}" + $(dirname "$0")/IRF.trainXGBforGammaHadronSeparationTraining.sh "${BCKDIR}" "${RUNPAR}" "${ODIR}" "${SIMTYPE}" "${VX}" "${ATM}" continue fi ############################################# @@ -335,54 +335,50 @@ for VX in $EPOCH; do # train per epoch and atmosphere and for each cut # (cut as sizesecondmax cut is applied) if [[ $IRFTYPE == "TRAINTMVA" ]] || [[ $IRFTYPE == "OPTIMIZETMVA" ]]; then - for VX in $EPOCH; do - for ATM in $ATMOS; do - for C in ${CUTTYPES[@]}; do - echo "Training/optimising TMVA for $C cuts, ${VX} ATM${ATM}" - BDTDIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANATYPE}/BDTtraining" - MVADIR="${BDTDIR}/GammaHadronBDTs_${VX:0:2}/${VX}_ATM${ATM}/${C/PointSource-/}/" - # list of background files - TRAINDIR="${BDTDIR}/mscw_${VX:0:2}/" - if [[ $DISPBDT == "1" ]]; then - TRAINDIR="${BDTDIR}/mscw_${VX:0:2}_DISP/" - MVADIR="${BDTDIR}/GammaHadronBDTs_${VX:0:2}_DISP/${VX}_ATM${ATM}/${C/PointSource-/}/" - fi - mkdir -p -v "${MVADIR}" - if [[ $IRFTYPE == "TRAINTMVA" ]]; then - # retrieve size cut - CUTFIL="$VERITAS_EVNDISP_AUX_DIR"/GammaHadronCutFiles/ANASUM.GammaHadron-Cut-${C}-TMVA-Preselection.dat - echo "CUTFILE: $CUTFIL" - SIZECUT=`grep "* sizesecondmax" $CUTFIL | grep ${EPOCH:0:2} | awk '{print $3}' | sort -u` - if [ -z "$SIZECUT" ] - then - echo "No size cut found; skipping cut $C" - continue - fi - echo "Size cut applied: $SIZECUT" - RUNPAR="TMVA.BDT.runparameter" - if [[ ${EPOCH:0:2} == "V4" ]] || [[ ${EPOCH:0:2} == "V5" ]]; then - cp -f "$VERITAS_EVNDISP_AUX_DIR"/ParameterFiles/TMVA.BDT.V4.runparameter "$MVADIR"/"$RUNPAR" - else - cp -f "$VERITAS_EVNDISP_AUX_DIR"/ParameterFiles/"$RUNPAR" "$MVADIR"/"$RUNPAR" - fi - sed -i "s/TMVASIZECUT/${SIZECUT}/" "$MVADIR"/"$RUNPAR" - if [[ $CUTFIL = *"NTel3"* ]]; then - sed -i "s/NImages>1/NImages>2/" "$MVADIR"/"$RUNPAR" - fi - ./IRF.trainTMVAforGammaHadronSeparation.sh \ - "${TRAINDIR}" \ - "$MVADIR"/"$RUNPAR" \ - "${MVADIR}" BDT ${SIMTYPE} ${VX} "${ATM}" - # Cut optimization - elif [[ $IRFTYPE == "OPTIMIZETMVA" ]]; then - echo "OPTIMIZE TMVA $C ${BDTDIR}/BackgroundRates/${VX:0:2}" - ./IRF.optimizeTMVAforGammaHadronSeparation.sh \ - "${BDTDIR}/BackgroundRates/${VX:0:2}" \ - "${C/PointSource-/}" \ - ${SIMTYPE} ${VX} "${ATM}" - fi - done - done + for C in ${CUTTYPES[@]}; do + echo "Training/optimising TMVA for $C cuts, ${VX} ATM${ATM}" + BDTDIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANATYPE}/BDTtraining" + MVADIR="${BDTDIR}/GammaHadronBDTs_${VX:0:2}/${VX}_ATM${ATM}/${C/PointSource-/}/" + # list of background files + TRAINDIR="${BDTDIR}/mscw_${VX:0:2}/" + if [[ $DISPBDT == "1" ]]; then + TRAINDIR="${BDTDIR}/mscw_${VX:0:2}_DISP/" + MVADIR="${BDTDIR}/GammaHadronBDTs_${VX:0:2}_DISP/${VX}_ATM${ATM}/${C/PointSource-/}/" + fi + mkdir -p -v "${MVADIR}" + if [[ $IRFTYPE == "TRAINTMVA" ]]; then + # retrieve size cut + CUTFIL="$VERITAS_EVNDISP_AUX_DIR"/GammaHadronCutFiles/ANASUM.GammaHadron-Cut-${C}-TMVA-Preselection.dat + echo "CUTFILE: $CUTFIL" + SIZECUT=`grep "* sizesecondmax" $CUTFIL | grep ${VX:0:2} | awk '{print $3}' | sort -u` + if [ -z "$SIZECUT" ] + then + echo "No size cut found; skipping cut $C" + continue + fi + echo "Size cut applied: $SIZECUT" + RUNPAR="TMVA.BDT.runparameter" + if [[ ${VX:0:2} == "V4" ]] || [[ ${VX:0:2} == "V5" ]]; then + cp -f "$VERITAS_EVNDISP_AUX_DIR"/ParameterFiles/TMVA.BDT.V4.runparameter "$MVADIR"/"$RUNPAR" + else + cp -f "$VERITAS_EVNDISP_AUX_DIR"/ParameterFiles/"$RUNPAR" "$MVADIR"/"$RUNPAR" + fi + sed -i "s/TMVASIZECUT/${SIZECUT}/" "$MVADIR"/"$RUNPAR" + if [[ $CUTFIL = *"NTel3"* ]]; then + sed -i "s/NImages>1/NImages>2/" "$MVADIR"/"$RUNPAR" + fi + $(dirname "$0")/IRF.trainTMVAforGammaHadronSeparation.sh \ + "${TRAINDIR}" \ + "$MVADIR"/"$RUNPAR" \ + "${MVADIR}" BDT ${SIMTYPE} ${VX} "${ATM}" + # Cut optimization + elif [[ $IRFTYPE == "OPTIMIZETMVA" ]]; then + echo "OPTIMIZE TMVA $C ${BDTDIR}/BackgroundRates/${VX:0:2}" + $(dirname "$0")/IRF.optimizeTMVAforGammaHadronSeparation.sh \ + "${BDTDIR}/BackgroundRates/${VX:0:2}" \ + "${C/PointSource-/}" \ + ${SIMTYPE} ${VX} "${ATM}" + fi done continue fi diff --git a/scripts/IRF.selectRunsForGammaHadronSeparationTraining.sh b/scripts/IRF.selectRunsForGammaHadronSeparationTraining.sh index 0cc02265..6ed94597 100755 --- a/scripts/IRF.selectRunsForGammaHadronSeparationTraining.sh +++ b/scripts/IRF.selectRunsForGammaHadronSeparationTraining.sh @@ -57,7 +57,7 @@ if [[ "${RUNPAR##*.}" == "json" ]]; then } }') else - ZEBINS=$( cat "$RUNPAR" | grep "^* ZENBINS " | sed -e 's/* ZENBINS//' | sed -e 's/ /\n/g') + ZEBINS=$(grep "^\* ZENBINS " "$RUNPAR" | sed -e 's/\* ZENBINS//' | sed -e 's/ /\n/g') fi echo "Zenith angle definition: $ZEBINS" declare -a ZEBINARRAY=( $ZEBINS ) #convert to array diff --git a/scripts/IRF.trainXGBforAngularReconstructionBinned.sh b/scripts/IRF.trainXGBforAngularReconstructionBinned.sh index 08025876..b28fbfae 100755 --- a/scripts/IRF.trainXGBforAngularReconstructionBinned.sh +++ b/scripts/IRF.trainXGBforAngularReconstructionBinned.sh @@ -73,7 +73,7 @@ MSCWLIST="$ODIR/xgbFiles.list" rm -f ${MSCWLIST} touch ${MSCWLIST} -INDIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma/MSCW_RECID0_DISP" +INDIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma/MSCW_RECID${RECID}_DISP" STEREO_PAR="$VERITAS_EVNDISP_AUX_DIR/ParameterFiles/XGB-stereo-parameter.json" TRAIN_ANGLES=$(jq -r ".zenith[] | select(.id==\"$ZA\") | .train | join(\" \")" $STEREO_PAR) diff --git a/scripts/db_scripts/db_combine_dqm_fits.py b/scripts/db_scripts/db_combine_dqm_fits.py index aa899259..51f7b966 100755 --- a/scripts/db_scripts/db_combine_dqm_fits.py +++ b/scripts/db_scripts/db_combine_dqm_fits.py @@ -49,7 +49,7 @@ def combine_tables(file_list, output_file): combined_tables.extend(batch_tables) - final_combined_table = vstack(combined_tables, join_type="exact") + final_combined_table = vstack(combined_tables, join_type="outer") logging.info(f"Writing combined table to {output_file}") final_combined_table.write(output_file, overwrite=True) diff --git a/scripts/db_scripts/db_run.sh b/scripts/db_scripts/db_run.sh index 03164b87..e394419a 100755 --- a/scripts/db_scripts/db_run.sh +++ b/scripts/db_scripts/db_run.sh @@ -74,6 +74,7 @@ get_start_time() else field_name="data_start_time" fi + start_time_index=0 while IFS="|" read -ra a; do if [[ ${a[0]} == "run_id" ]]; then for (( j=0; j<${#a[@]}; j++ )); @@ -97,6 +98,7 @@ get_end_time() else field_name="data_end_time" fi + end_time_index=0 while IFS="|" read -ra a; do if [[ ${a[0]} == "run_id" ]]; then for (( j=0; j<${#a[@]}; j++ )); @@ -170,6 +172,7 @@ get_source_id() else OFIL=$(cat $OFIL) fi + source_index=0 while IFS="|" read -ra a; do if [[ ${a[0]} == "run_id" ]]; then for (( j=0; j<${#a[@]}; j++ )); @@ -289,9 +292,11 @@ read_pointing() read_run_from_DB runinfo read_run_from_DB rundqm -# don't test and read if tar file exists -# (implementation of testing missing) -if [[ ! -e $(getDBTextFileDirectory ${RUN}).tar.gz ]] || [[ ${OVERWRITE} == "1" ]]; then +# Re-read laser data if tarball is missing OR if sidecar .laserrun file is absent +RUND="$(getDBTextFileDirectory ${RUN})" +if [[ ! -e "${RUND}.tar.gz" ]] || \ + [[ $(get_file_status ${RUN} "${RUND}/${RUN}.laserrun") == "0" ]] || \ + [[ ${OVERWRITE} == "1" ]]; then read_laser_run_and_dqm read_laser_calibration fi diff --git a/scripts/db_scripts/db_write_fits.py b/scripts/db_scripts/db_write_fits.py index 51139e68..d57c23c8 100755 --- a/scripts/db_scripts/db_write_fits.py +++ b/scripts/db_scripts/db_write_fits.py @@ -106,7 +106,8 @@ def extract_l3_rate(run, temp_run_dir): table = table[condition] time_diffs = np.diff(np.array(table["timestamp"])) / 1000.0 l3_values = np.diff(np.array(table["L3"], dtype=float)) - l3_values /= time_diffs + with np.errstate(divide="ignore", invalid="ignore"): + l3_values = np.where(time_diffs > 0, l3_values / time_diffs, np.nan) if len(l3_values) > 0: l3_mean = np.mean(l3_values) @@ -129,8 +130,10 @@ def extract_l3_rate(run, temp_run_dir): l3_busy = np.array(table["L3orVDAQBusyScaler"], dtype=float) busy = np.diff(l3_busy) ten_mhz = np.diff(np.array(table["TenMHzScaler"])) - dead_time = np.mean(busy / ten_mhz) - dead_time_std = np.std(busy / ten_mhz) + with np.errstate(divide="ignore", invalid="ignore"): + ratio = np.where(ten_mhz > 0, busy / ten_mhz, np.nan) + dead_time = np.mean(ratio) + dead_time_std = np.std(ratio) except ValueError: dead_time = np.nan dead_time_std = np.nan @@ -426,9 +429,9 @@ def main(): file_path = os.path.join(temp_run_files, file_name) logging.info("Converting %s", file_path) table = read_file(file_path) - if "dqm" in file_name.lower(): - convert_table_comment_to_ascii(table) if table is not None: + if "dqm" in file_name.lower(): + convert_table_comment_to_ascii(table) hdu.append(fits.BinTableHDU(table)) hdu[-1].name = file_name.split(".")[1] diff --git a/scripts/helper_scripts/ANALYSIS.anasum_sub.sh b/scripts/helper_scripts/ANALYSIS.anasum_sub.sh index eb60af6d..86ae2393 100755 --- a/scripts/helper_scripts/ANALYSIS.anasum_sub.sh +++ b/scripts/helper_scripts/ANALYSIS.anasum_sub.sh @@ -74,22 +74,6 @@ rm -f $OUTPUTLOGFILE touch $OUTPUTLOGFILE -prepare_atmo_string() -{ - ATMO=$1 - EPOCH=$2 - OBSL=$3 - # V4 and V5: grisu sims with ATM21/22 - if [[ $EPOCH == *"V4"* ]] || [[ $EPOCH == *"V5"* ]]; then - ATMO=${ATMO/6/2} - fi - # V6 UV only for ATM 61 - if [[ $EPOCH == *"V6"* ]] && [[ $OBSL == "obsFilter" ]]; then - ATMO=${ATMO/62/61} - fi - echo "$ATMO" -} - inspect_executables() { if [ -n "$EVNDISP_APPTAINER" ]; then @@ -144,8 +128,6 @@ if [[ $FLIST == "NOTDEFINED" ]]; then OBSL=$(echo $RUNINFO | awk '{print $4}') TELTOANA=`echo $RUNINFO | awk '{print "T"$(5)}'` - ATMO=$(prepare_atmo_string $ATMO $EPOCH $OBSL) - REPLACESIMTYPEEff=$(prepare_irf_string $EPOCH $OBSL $SIMTYPE 0) REPLACESIMTYPERad=$(prepare_irf_string $EPOCH $OBSL $SIMTYPE 1) diff --git a/scripts/helper_scripts/ANALYSIS.evndisp_sub.sh b/scripts/helper_scripts/ANALYSIS.evndisp_sub.sh index 0a0f6c57..da1d382e 100755 --- a/scripts/helper_scripts/ANALYSIS.evndisp_sub.sh +++ b/scripts/helper_scripts/ANALYSIS.evndisp_sub.sh @@ -74,11 +74,18 @@ unpack_db_textdirectory() echo "DBTEXT FILE for $RRUN $DBRUNFIL" >&2 if [[ -e ${DBRUNFIL} ]]; then mkdir -p ${TMP_DBTEXTDIRECTORY}/${SRUN} - tar -xzf ${DBRUNFIL} -C ${TMP_DBTEXTDIRECTORY}/${SRUN}/ + tar -xzf ${DBRUNFIL} -C ${TMP_DBTEXTDIRECTORY}/${SRUN}/ || return 1 else echo "DBTEXT FILE not found ($DBRUNFIL)" >&2 + return 1 fi - echo "${TMP_DBTEXTDIRECTORY}/${SRUN}/${RRUN}/${RRUN}.laserrun" + + DBTEXTFILE="${TMP_DBTEXTDIRECTORY}/${SRUN}/${RRUN}/${RRUN}.laserrun" + if [[ ! -r ${DBTEXTFILE} ]]; then + echo "DBTEXT FILE not readable (${DBTEXTFILE})" >&2 + return 1 + fi + echo "${DBTEXTFILE}" } sub_dir() @@ -98,11 +105,15 @@ if [[ "${DBTEXTDIRECTORY}" != "0" ]]; then echo "UNPACKING DBTEXT from $RUN ${DBTEXTDIRECTORY}" TMP_DBTEXTDIRECTORY="${TEMPDIR}/DBTEXT" TMP_LASERRUN=$(unpack_db_textdirectory $RUN $TMP_DBTEXTDIRECTORY) + if [[ $? -ne 0 || ! -r ${TMP_LASERRUN} ]]; then + echo "failed to unpack DBTEXT laser-run metadata for run $RUN" >&2 + exit 1 + fi LRUNID=$(cat ${TMP_LASERRUN} | grep -v run_id | awk -F "|" '{print $1}') for LL in ${LRUNID} do echo " unpacking flasher/laser run: $LL" - unpack_db_textdirectory $LL $TMP_DBTEXTDIRECTORY + unpack_db_textdirectory $LL $TMP_DBTEXTDIRECTORY >/dev/null || exit 1 done echo "DBTEXT directory $(ls -l $TMP_DBTEXTDIRECTORY)" @@ -136,6 +147,10 @@ get_run_date() # check if run is on disk if [[ "${DBTEXTDIRECTORY}" != "0" ]]; then RUNINFO=$(sub_dir ${TMP_DBTEXTDIRECTORY} ${RUN})/${RUN}/${RUN}.runinfo + if [[ ! -r ${RUNINFO} ]]; then + echo "DBTEXT run info not readable (${RUNINFO})" >&2 + exit 1 + fi RUNDATE=$(get_run_date ${RUNINFO}) echo "RUN $RUN $RUNINFO $RUNDATE" ls -l ${TMP_DBTEXTDIRECTORY} diff --git a/scripts/helper_scripts/ANALYSIS.mscw_energy_sub.sh b/scripts/helper_scripts/ANALYSIS.mscw_energy_sub.sh index ab143ec3..4e3356ad 100755 --- a/scripts/helper_scripts/ANALYSIS.mscw_energy_sub.sh +++ b/scripts/helper_scripts/ANALYSIS.mscw_energy_sub.sh @@ -137,13 +137,13 @@ get_disp_dir() elif (( $(echo "$ZA < 58" | bc -l) )); then DISPDIR="${DISPDIR}/55deg/" elif (( $(echo "$ZA < 62" | bc -l) )); then - if [[ ${SIMTYPE_RUN} == "CARE_RedHV" ]]; then # fix for incomplete MC set + if [[ ${HVSETTINGS} == "obsLowHV" ]]; then # fix for incomplete RedHV MC set DISPDIR="${DISPDIR}/55deg/" else DISPDIR="${DISPDIR}/60deg/" fi else - if [[ ${SIMTYPE_RUN} == "CARE_RedHV" ]]; then # fix for incomplete MC set + if [[ ${HVSETTINGS} == "obsLowHV" ]]; then # fix for incomplete RedHV MC set DISPDIR="${DISPDIR}/55deg/" else DISPDIR="${DISPDIR}/60deg/" @@ -156,6 +156,10 @@ if [[ $DISPBDT == "1" ]]; then ZA=$($EVNDISPSYS/bin/printRunParameter $INFILEPATH -zenith | awk '{print $4}') DISPDIR=$(get_disp_dir ${ZA}) echo "DISPDIR (Zenith is $ZA deg): " $DISPDIR + if [[ ! -d "$DISPDIR" ]]; then + echo "Error: DispBDT directory $DISPDIR not found, exiting..." + exit 1 + fi fi ################################# @@ -206,7 +210,7 @@ $EVNDISPSYS/bin/mscw_energy \ echo "$(inspect_executables)" >> ${MSCWLOGFILE} # write DISP directory into log file (as tmp directories are used) -if [[ DISPBDT != "NOTSET" ]]; then +if [[ $DISPBDT != "NOTSET" ]]; then echo "" >> ${MSCWLOGFILE} echo "dispBDT XML files read from ${DISPDIR}" >> ${MSCWLOGFILE} fi diff --git a/scripts/helper_scripts/ANALYSIS.v2dl3_sub.sh b/scripts/helper_scripts/ANALYSIS.v2dl3_sub.sh index c99db244..3333ac48 100755 --- a/scripts/helper_scripts/ANALYSIS.v2dl3_sub.sh +++ b/scripts/helper_scripts/ANALYSIS.v2dl3_sub.sh @@ -55,7 +55,8 @@ check_conda_installation source activate base conda activate v2dl3Eventdisplay-${V2DL3VERSION} -pip install -e "${V2DL3SYS%/}-v${V2DL3VERSION}" +# Install only if not already present (avoid slow per-job reinstall) +pip show v2dl3-eventdisplay &>/dev/null 2>&1 || pip install -e "${V2DL3SYS%/}-v${V2DL3VERSION}" V2DL3OPT="--fuzzy_boundary zenith 0.05 --fuzzy_boundary pedvar 0.5 --save_multiplicity" # selection for full-gamma files diff --git a/scripts/helper_scripts/IRF.compress_evndisp_MC_sub.sh b/scripts/helper_scripts/IRF.compress_evndisp_MC_sub.sh index 0f7f833b..c2003a96 100755 --- a/scripts/helper_scripts/IRF.compress_evndisp_MC_sub.sh +++ b/scripts/helper_scripts/IRF.compress_evndisp_MC_sub.sh @@ -45,7 +45,7 @@ add_log_file() compress_file() { - if command -v zstd /dev/null; then + if command -v zstd &>/dev/null; then zstd ${1} zstd --test ${1}.zst else diff --git a/scripts/helper_scripts/IRF.evndisp_MC_sub.sh b/scripts/helper_scripts/IRF.evndisp_MC_sub.sh index 07a2fc3e..400ef5b5 100755 --- a/scripts/helper_scripts/IRF.evndisp_MC_sub.sh +++ b/scripts/helper_scripts/IRF.evndisp_MC_sub.sh @@ -91,6 +91,7 @@ if [ -e "$V4N" ]; then mkdir -p "$TMPDIR" VBF_BASENAME="${VBFNAME%.zst}" echo "Checking processed files in $V4N" + shopt -s nullglob for file in "$V4N"/*.root.zst; do tmpfile="$TMPDIR/$(basename "${file%.zst}")" zstd -d -c "$file" > "$tmpfile" @@ -99,6 +100,7 @@ if [ -e "$V4N" ]; then exit fi done + shopt -u nullglob rm -rf "$TMPDIR" fi @@ -322,7 +324,7 @@ compare_log_file evndisptzeroLog $ODIR/$ONAME.tzero.log ### compress evndisp root file compress_file() { - if command -v zstd /dev/null; then + if command -v zstd &>/dev/null; then zstd ${1} zstd --test ${1}.zst else diff --git a/scripts/helper_scripts/IRF.lookup_table_parallel_sub.sh b/scripts/helper_scripts/IRF.lookup_table_parallel_sub.sh index b51ec7bb..868bb810 100755 --- a/scripts/helper_scripts/IRF.lookup_table_parallel_sub.sh +++ b/scripts/helper_scripts/IRF.lookup_table_parallel_sub.sh @@ -64,7 +64,7 @@ if [ -n "$(find ${INDIR} -name "*[0-9].root" 2>/dev/null)" ]; then echo "Copying evndisp root files to ${DDIR}" find ${INDIR} -name "*[0-9].root" -exec cp -v {} ${DDIR} \; elif [ -n "$(find ${INDIR} -name "*[0-9].root.zst" 2>/dev/null)" ]; then - if command -v zstd /dev/null; then + if command -v zstd &>/dev/null; then echo "Copying evndisp root.zst files to ${DDIR}" FLIST=$(find "${INDIR}/" -name "*[0-9].root.zst") for F in $FLIST diff --git a/scripts/helper_scripts/IRF.mscw_energy_MC_sub.sh b/scripts/helper_scripts/IRF.mscw_energy_MC_sub.sh index 3fd4d167..63873312 100755 --- a/scripts/helper_scripts/IRF.mscw_energy_MC_sub.sh +++ b/scripts/helper_scripts/IRF.mscw_energy_MC_sub.sh @@ -73,7 +73,6 @@ inspect_executables() fi } - # mscw_energy command line options MOPT="-noNoTrigger -nomctree -writeReconstructedEventsOnly=1 -arrayrecid=${RECID} -tablefile $TABFILE" # dispBDT reconstruction @@ -104,14 +103,16 @@ if [ $DISPBDT -eq 1 ]; then elif [[ "${ZA}" -lt "58" ]]; then DISPDIR="${DISPDIR}/55deg/" elif [[ "${ZA}" -lt "62" ]]; then - DISPDIR="${DISPDIR}/60deg/" - if [[ ${SIMTYPE} == "CARE_RedHV" ]]; then # fix for incomplete MC set + if [[ ${SIMTYPE} == CARE_RedHV* ]]; then # fix for incomplete RedHV MC set DISPDIR="${DISPDIR}/55deg/" + else + DISPDIR="${DISPDIR}/60deg/" fi else - DISPDIR="${DISPDIR}/65deg/" - if [[ ${SIMTYPE} == "CARE_RedHV" ]]; then # fix for incomplete MC set + if [[ ${SIMTYPE} == CARE_RedHV* ]]; then # fix for incomplete RedHV MC set DISPDIR="${DISPDIR}/55deg/" + else + DISPDIR="${DISPDIR}/65deg/" fi fi # unzip XML files into DDIR @@ -130,11 +131,11 @@ fi rm -f $OSUBDIR/$OFILE.log rm -f $OSUBDIR/$OFILE.list echo "INDIR ${INDIR}" -if [ -n "$(find "${INDIR}/" -name "*[0-9].root" 2>/dev/null)" ]; then +if [ -n "$(find "${INDIR}/" -maxdepth 1 -name "*[0-9].root" 2>/dev/null)" ]; then echo "Using evndisp root files from ${INDIR}" - cp -v "${INDIR}/*[0-9].root" "$DDIR" + find "${INDIR}/" -maxdepth 1 -name "*[0-9].root" -exec cp -v {} "$DDIR" \; elif [ -n "$(find "${INDIR}/" -name "*[0-9].root.zst" 2>/dev/null)" ]; then - if command -v zstd /dev/null; then + if command -v zstd &>/dev/null; then echo "Copying evndisp root.zst files to ${DDIR}" FLIST=$(find "${INDIR}/" -name "*[0-9].root.zst") for F in $FLIST diff --git a/scripts/helper_scripts/IRF.optimizeTMVAforGammaHadronSeparation_sub.sh b/scripts/helper_scripts/IRF.optimizeTMVAforGammaHadronSeparation_sub.sh index 2892d54c..6c0e1ec7 100755 --- a/scripts/helper_scripts/IRF.optimizeTMVAforGammaHadronSeparation_sub.sh +++ b/scripts/helper_scripts/IRF.optimizeTMVAforGammaHadronSeparation_sub.sh @@ -54,7 +54,6 @@ then rm -f ${RATEFILE}.log # calculate rates from Crab Nebula and from background rates - rm -f ${MVADIR}/rates.log "$EVNDISPSYS"/bin/calculateCrabRateFromMC \ ${EFFAREA} \ ${RATEFILE}.root \ diff --git a/scripts/helper_scripts/IRF.trainXGBforAngularReconstruction_sub.sh b/scripts/helper_scripts/IRF.trainXGBforAngularReconstruction_sub.sh index e6afb030..74c218da 100755 --- a/scripts/helper_scripts/IRF.trainXGBforAngularReconstruction_sub.sh +++ b/scripts/helper_scripts/IRF.trainXGBforAngularReconstruction_sub.sh @@ -13,8 +13,8 @@ N="5000000" MAXCORES=48 # temporary (scratch) directory -if [[ -n $TMPDIR ]]; then - TEMPDIR=$TMPDIR/$(basename $MSCW_FILE .root) +if [[ -n "$TMPDIR" ]]; then + TEMPDIR="${TMPDIR}/XGB-$(basename "$LLIST" .list)-$(uuidgen)" else TEMPDIR="$VERITAS_USER_DATA_DIR/TMPDIR" fi diff --git a/scripts/set_environment.sh b/scripts/set_environment.sh index 006b3e41..5b1b333b 100755 --- a/scripts/set_environment.sh +++ b/scripts/set_environment.sh @@ -61,7 +61,7 @@ export V2DL3SYS=${USERAFSDIR}/EVNDISP/EVNDISP-400/GITHUB_Eventdisplay/PreProcess export EVNDISP_MLSYS=${USERAFSDIR}/EVNDISP/EVNDISP-400/GITHUB_Eventdisplay/Eventdisplay-ML # EVENTDISPLAY using apptainers if [[ $PROCESS == "apptainer"* ]]; then - export EVNDISP_APPTAINER="$VERITAS_DATA_DIR/shared/APPTAINERS/eventdisplay_v4_${EVNDISPVERSION}-rc3.sif" + export EVNDISP_APPTAINER="$VERITAS_DATA_DIR/shared/APPTAINERS/eventdisplay_v4_${EVNDISPVERSION}-rc4.sif" export EVNDISP_ENV="--env VERITAS_DATA_DIR=${VERITAS_DATA_DIR},VERITAS_EVNDISP_AUX_DIR=${VERITAS_EVNDISP_AUX_DIR},VERITAS_USER_DATA_DIR=${VERITAS_USER_DATA_DIR},VERITAS_USER_LOG_DIR=${VERITAS_USER_LOG_DIR}" export EVNDISPSYS="apptainer exec --no-mount bind-paths --cleanenv ${EVNDISP_APPTAINER} /opt/EventDisplay_v4/" # Alma Linux 9 (al9) processing