Feature/add option to alter job directories

Prod/cmsCondorData.py

@@ -20,6 +20,7 @@
 parser.add_option("-n",dest="nPerJob",type="int",default=1,help="NUMBER of files processed per job",metavar="NUMBER")
 parser.add_option("-q","--flavour",dest="jobFlavour",type="str",default="workday",help="job FLAVOUR",metavar="FLAVOUR")
 parser.add_option("-p","--proxy",dest="proxyPath",type="str",default="noproxy",help="Proxy path")
+parser.add_option("-s","--suffix",dest="jobsDirSuffix",type="str",default="",help="Suffix of the Jobs directory")
 
 opts, args = parser.parse_args()
 
@@ -30,7 +31,8 @@
 help_text += '\n<remoteDir> (mandatory) = directory where the files will be transfered (e.g. on EOS)'
 help_text += '\n<proxyPath> (optional) = location of your voms cms proxy. Note: keep your proxy in a private directory.'
 help_text += '\n<nPerJob> (optional) = number of files processed per batch job (default=1)'
-help_text += '\n<flavour> (optional) = job flavour (default=workday)\n'
+help_text += '\n<flavour> (optional) = job flavour (default=workday)'
+help_text += '\n<suffix> (optional) = job directory suffix (default=\"\")\n'
 
 
 cfgFileName = str(args[0])
@@ -43,6 +45,11 @@
 print ('remote directory = %s'%remoteDir)
 # print 'job flavour = %s'%opts.jobFlavour
 
+jobsDirSuffix = ""
+if opts.jobsDirSuffix:
+		jobsDirSuffix = f"_{opts.jobsDirSuffix}"
+		print (f'Jobs directory suffix = {jobsDirSuffix}')
+
 
 # #make directories for the jobs
 # try:
@@ -90,7 +97,9 @@
 
 # create the run_cfg.py 
 
-mainJobDir = MYDIR+'/Jobs'
+jobsDir = 'Jobs'+jobsDirSuffix
+
+mainJobDir = MYDIR+'/'+jobsDir
 os.system('mkdir -p %s'%mainJobDir)
 
 process.source.fileNames = cms.untracked.vstring("INPUTFILES")
@@ -121,7 +130,7 @@
 for i in range(0, nJobs):
     #print 'total: %d/%d  ;  %.1f %% processed '%(j,my_sum,(100*float(j)/float(my_sum)))
 
-    jobDir = MYDIR+'/Jobs/Job_%s/'%str(i)
+    jobDir = MYDIR+'/'+jobsDir+'/Job_%s/'%str(i)
     os.system('mkdir -p %s'%jobDir)
 
     iFileMin = i*opts.nPerJob
@@ -171,9 +180,9 @@
 #requirements = "(OpSysAndVer =?= \"CentOS7\")"
 #condor_str += f"requirements = {requirements}\n"
 if "/eos/user/" in MYDIR or "/eos/home" in MYDIR:
-    condor_str += "queue filename matching (./Jobs/Job_*/*.sh)"
+    condor_str += f"queue filename matching (.{jobsDir}/Job_*/*.sh)"
 else:
-    condor_str += "queue filename matching ("+MYDIR+"/Jobs/Job_*/*.sh)"
+    condor_str += "queue filename matching ("+MYDIR+f"/{jobsDir}/Job_*/*.sh)"
 condor_name = MYDIR+"/condor_cluster.sub"
 condor_file = open(condor_name, "w")
 condor_file.write(condor_str)

Prod/cmsCondorMC.py

@@ -32,6 +32,7 @@ def runCommand(commandLine):
 parser.add_option("-n",dest="nPerJob",type="int",default=1,help="NUMBER of files processed per job",metavar="NUMBER")
 parser.add_option("-q","--flavour",dest="jobFlavour",type="str",default="workday",help="job FLAVOUR",metavar="FLAVOUR")
 parser.add_option("-p","--proxy",dest="proxyPath",type="str",default="noproxy",help="Proxy path")
+parser.add_option("-s","--suffix",dest="jobsDirSuffix",type="str",default="",help="Suffix of the Jobs directory")
 
 opts, args = parser.parse_args()
 
@@ -42,7 +43,8 @@ def runCommand(commandLine):
 help_text += '\n<remoteDir> (mandatory) = directory where the files will be transfered (e.g. on EOS)'
 help_text += '\n<proxyPath> (optional) = location of your voms cms proxy. Note: keep your proxy in a private directory.'
 help_text += '\n<nPerJob> (optional) = number of files processed per batch job (default=1)'
-help_text += '\n<flavour> (optional) = job flavour (default=workday)\n'
+help_text += '\n<flavour> (optional) = job flavour (default=workday)'
+help_text += '\n<suffix> (optional) = job directory suffix (default=\"\")\n'
 
 
 cfgFileName = str(args[0])
@@ -55,10 +57,18 @@ def runCommand(commandLine):
 print('remote directory = %s'%remoteDir)
 print('job flavour = %s'%opts.jobFlavour)
 
+jobsDirSuffix = ""
+if opts.jobsDirSuffix:
+		jobsDirSuffix = f"_{opts.jobsDirSuffix}"
+		print (f'Jobs directory suffix = {jobsDirSuffix}')
+
 #make directories for the jobs
+
+jobsDir = 'Jobs'+jobsDirSuffix
+
 try:
-    os.system('rm -rf Jobs')
-    os.system('mkdir Jobs')
+    os.system(f'rm -rf {jobsDir}')
+    os.system(f'mkdir {jobsDir}')
 except:
     print("err!")
     pass
@@ -127,7 +137,7 @@ def runCommand(commandLine):
 
 # create the run_cfg.py 
 
-mainJobDir = MYDIR+'/Jobs'
+mainJobDir = MYDIR+'/'+jobsDir
 os.system('mkdir -p %s'%mainJobDir)
 
 process.source.fileNames = cms.untracked.vstring("INPUTFILES")
@@ -165,7 +175,7 @@ def runCommand(commandLine):
     newdataset = str(dataset).replace("b'","")
     datasetName=newdataset.lstrip("/")
     datasetName=datasetName.replace("/","_")
-    datasetJobDir='Jobs/'+datasetName
+    datasetJobDir=jobsDir'/'+datasetName
     datasetRemoteDir=remoteDir+'/'+datasetName
     os.system('mkdir '+datasetJobDir)
     os.system('mkdir '+datasetRemoteDir)
@@ -236,9 +246,9 @@ def runCommand(commandLine):
 condor_str += "log = $Fp(filename)hlt.log\n"
 condor_str += '+JobFlavour = "%s"\n'%opts.jobFlavour
 if "/eos/user/" in MYDIR or "/eos/home" in MYDIR:
-    condor_str += "queue filename matching (./Jobs/*/Job_*/*.sh)"
+    condor_str += f"queue filename matching (./{jobsDir}/*/Job_*/*.sh)"
 else:
-    condor_str += "queue filename matching ("+MYDIR+"/Jobs/*/Job_*/*.sh)"
+    condor_str += "queue filename matching ("+MYDIR+f"/{jobsDir}/*/Job_*/*.sh)"
 condor_name = MYDIR+"/condor_cluster.sub"
 condor_file = open(condor_name, "w")
 condor_file.write(condor_str)

Rates/condorScriptForRatesData.py

@@ -17,19 +17,21 @@
 parser.add_option("-n",dest="nPerJob",type="int",default=5,help="NUMBER of files processed per job",metavar="NUMBER")
 parser.add_option("-q","--flavour",dest="jobFlavour",type="str",default="workday",help="job FLAVOUR",metavar="FLAVOUR")
 parser.add_option("-m","--maps",dest="maps",type="str",default="nomaps",help="ARG='nomaps' (default option, don't use maps to get dataste/groups/etc. rates), 'somemaps' (get dataset/groups/etc. rates but with no study of dataset merging), 'allmaps' (get dataset/groups/etc. rates and also study dataset merging)",metavar="ARG")
+parser.add_option("-s","--suffix",dest="dirSuffix",type="str",default="",help="Suffix of the Jobs and Results directories")
 
 opts, args = parser.parse_args()
 
 
 error_text = '\n\nError: wrong <json>=%s or <CMSSWrel>=%s inputs\n' %(opts.jsonFile, opts.cmsEnv)
-help_text = '\npython3 batchScriptForRates.py -j <json> -e <CMSSWrel> -i <infilesDir> -f <filetype> -n <nPerJob> -q <jobFlavour> -m <merging>'
+help_text  = '\npython3 condorScriptForRatesData.py -j <json> -e <CMSSWrel> -i <infilesDir> -f <filetype> -n <nPerJob> -q <jobFlavour> -m <merging>'
 help_text += '\n<json> (mandatory argument) = text file with the LS range in json format'
 help_text += '\n<CMSSWrel> (mandatory) = directory where the top of a CMSSW release is located'
 help_text += '\n<infilesDir> (optional) = directory where the input root files are located (default = will take whatever is in the filesInput.py file)'
 help_text += '\n<filetype> (optional) = "custom" (default option) or "RAW"'
 help_text += '\n<nPerJob> (optional) = number of files processed per batch job (default=5)'
-help_text += '\n<flavour> (optional) = job flavour (default=workday)\n'
-help_text += '\n<maps> (optional) = "nomaps" (default option) or "somemaps" or "allmaps""\n'
+help_text += '\n<flavour> (optional) = job flavour (default=workday)'
+help_text += '\n<maps> (optional) = "nomaps" (default option) or "somemaps" or "allmaps"'
+help_text += '\n<suffix> (optional) = suffix for the Jobs and Results directories (default=\"\")\n'
 
 if opts.jsonFile == "nojson" or opts.cmsEnv == "noenv":
     print(error_text)
@@ -41,20 +43,27 @@
 print('file type = %s'%opts.fileType)
 print('job flavour = %s'%opts.jobFlavour)
 
+dirSuffix = ""
+if opts.dirSuffix
+		dirSuffix = f"_{opts.dirSuffix}"
+		print (f'Jobs and Results directories suffix = _{dirSuffix}')
+
+jobsDir = 'Jobs' + dirSuffix
 
 #make directories for the jobs
 try:
-    os.system('rm -rf Jobs')
-    os.system('mkdir Jobs')
-    os.system('mkdir Jobs/sub_raw')
+    os.system(f'rm -rf {jobsDir}')
+    os.system(f'mkdir {jobsDir}')
+    os.system(f'mkdir {jobsDir}/sub_raw')
 except:
     print("err!")
     pass
 
+resultsDir = "Results" + dirSuffix
 
 sub_total = open("sub_total.jobb","w")
-sub_total.write("rm Results/Data/Raw/*/*.csv\n")
-sub_total.write("rm Results/Data/Raw/*/*.root\n")
+sub_total.write(f"rm {resultsDir}/Data/Raw/*/*.csv\n")
+sub_total.write(f"rm {resultsDir}/Data/Raw/*/*.root\n")
 
 
 if opts.inputFilesDir != "no":
@@ -80,15 +89,15 @@
     #print 'total: %d/%d  ;  %.1f %% processed '%(j,my_sum,(100*float(j)/float(my_sum)))
 
     tmp_jobname="sub_%s.sh"%(str(i))
-    tmp_job=open(MYDIR+'/Jobs/sub_raw/'+tmp_jobname,'w')
+    tmp_job=open(MYDIR+f'/{jobsDir}/sub_raw/'+tmp_jobname,'w')
     tmp_job.write("cd %s\n"%(MYDIR))
     tmp_job.write("cd %s\n"%(opts.cmsEnv))
     tmp_job.write("eval `scramv1 runtime -sh`\n")
     tmp_job.write("cd -\n")
     tmp_job.write("python3 triggerCountsFromTriggerResults.py -i %s -j %s -s %s -f %s -m %s\n"%(infile, opts.jsonFile, str(i), opts.fileType, opts.maps))
-    tmp_job.write("\npython3 handleFileTransfer.py -d %s -s %s"%(MYDIR, str(i)))
+    tmp_job.write(f"\npython3 handleFileTransfer.py -d {MYDIR} -s {i} -S {dirSuffix}")
     tmp_job.close()
-    tmp_job_dir = MYDIR+'/Jobs/sub_raw/'+tmp_jobname
+    tmp_job_dir = MYDIR+f'/{jobsDir}/sub_raw/'+tmp_jobname
     os.system("chmod +x %s"%(tmp_job_dir))
 
 
@@ -97,7 +106,7 @@
     if k==loop_mark or i==len(fileInputNames)-1:
         k=0
         Tjobsname = "sub_%s.sh"%i
-        Tjob_dir = '%s/Jobs/Job_%s/'%(MYDIR, str(i))
+        Tjob_dir = f'%s/{jobsDir}/Job_%s/'%(MYDIR, str(i))
         os.system("mkdir %s"%Tjob_dir)
         Tjob = open(Tjob_dir+Tjobsname,"w")
         Tjob.write("%s"%(tmp_text))
@@ -116,9 +125,9 @@
 #requirements = "(OpSysAndVer =?= \"CentOS7\")"
 #condor_str += f"requirements = {requirements}\n"
 if "/eos/user/" in MYDIR or "/eos/home" in MYDIR:
-    condor_str += "queue filename matching (./Jobs/Job_*/*.sh)"
+    condor_str += f"queue filename matching (./{jobsDir}/Job_*/*.sh)"
 else:
-    condor_str += "queue filename matching ("+MYDIR+"/Jobs/Job_*/*.sh)"
+    condor_str += f"queue filename matching ("+MYDIR+"/{jobsDir}/Job_*/*.sh)"
 condor_name = MYDIR+"/condor_cluster.sub"
 condor_file = open(condor_name, "w")
 condor_file.write(condor_str)

Rates/config_makeCondorJobsData.py

@@ -54,6 +54,11 @@
 #Job flavour
 #flavour = "microcentury"
 flavour = "espresso"
+
+# Jobs and Rates directories suffix
+# Change this if you want to run multiple rates tests in parallel with different settings
+dirSuffix = ""
+
 '''
 --------------------------OPTIONS TO BE FILLED OUT-----------------------------------------
 '''
@@ -69,6 +74,9 @@
 if isUnusual:
     command += " -n %s -q %s" %(n, flavour)
 
+if dirSuffix:
+		command += f" -s _{dirSuffix}"
+
 command += " -m %s" %maps
 os.system(command)
 

Rates/handleFileTransfer.py

@@ -7,27 +7,36 @@
 parser.add_option("-d","--directory",dest="dir",type="str",default="nodir",help="working DIRECTORY",metavar="DIRECTORY")
 parser.add_option("-m","--dataset",dest="MCdataset",type="str",default="Data",help="MCdataset string")
 parser.add_option("-s","--finalstring",dest="fstr",type="str",default="nostr",help="final STRING",metavar="STRING")
+parser.add_option("-S","--dirSuffix",dest="dirSuffix",type="str",default="",help="Suffix of the Results directory")
 
 (opts, args) = parser.parse_args()
 
-os.system("mkdir %s/Results"%opts.dir)
+head_dir = opts.dir
+
+dirSuffix = ""
+if opts.dirSuffix
+		dirSuffix = f"_{opts.dirSuffix}"
+
+results_dir = "Results" + dirSuffix
+
+os.system(f"mkdir {head_dir}/{results_dir}")
 MCorData="MC"
 MCorData2="MC/"+opts.MCdataset
 inDir="Jobs/"+opts.MCdataset
 if opts.MCdataset == "Data":
     MCorData = "Data"
     MCorData2 = "Data"
     inDir = "Jobs"
-os.system("mkdir %s/Results/%s"%(opts.dir, MCorData))
+os.system(f"mkdir {head_dir}/{results_dir}/{MCorData}")
 if opts.MCdataset != "Data":
-    os.system("mkdir %s/Results/%s"%(opts.dir, MCorData2))
-rawDir="%s/Results/%s/Raw"%(opts.dir, MCorData2)
-os.system("mkdir %s"%rawDir)
-    
+    os.system(f"mkdir {head_dir}/{results_dir}/{MCorData2}")
+rawDir=f"{head_dir}/{results_dir}/{MCorData2}/Raw"
+os.system(f"mkdir {rawDir}")
+
 for name in mergeNames:
-    os.system("mkdir %s/%s"%(rawDir, mergeNames[name]))
-os.system("mkdir %s/Root"%rawDir)
-os.system("mkdir %s/Global"%rawDir)
+    os.system(f"mkdir {rawDir}/{mergeNames[name]}")
+os.system(f"mkdir {rawDir}/Root")
+os.system(f"mkdir {rawDir}/Global")
 
 
 for filename in list(mergeNames.keys()):