diff --git a/Prod/cmsCondorData.py b/Prod/cmsCondorData.py index 896a852..eb8687a 100755 --- a/Prod/cmsCondorData.py +++ b/Prod/cmsCondorData.py @@ -20,6 +20,7 @@ parser.add_option("-n",dest="nPerJob",type="int",default=1,help="NUMBER of files processed per job",metavar="NUMBER") parser.add_option("-q","--flavour",dest="jobFlavour",type="str",default="workday",help="job FLAVOUR",metavar="FLAVOUR") parser.add_option("-p","--proxy",dest="proxyPath",type="str",default="noproxy",help="Proxy path") +parser.add_option("-s","--suffix",dest="jobsDirSuffix",type="str",default="",help="Suffix of the Jobs directory") opts, args = parser.parse_args() @@ -30,7 +31,8 @@ help_text += '\n (mandatory) = directory where the files will be transfered (e.g. on EOS)' help_text += '\n (optional) = location of your voms cms proxy. Note: keep your proxy in a private directory.' help_text += '\n (optional) = number of files processed per batch job (default=1)' -help_text += '\n (optional) = job flavour (default=workday)\n' +help_text += '\n (optional) = job flavour (default=workday)' +help_text += '\n (optional) = job directory suffix (default=\"\")\n' cfgFileName = str(args[0]) @@ -43,6 +45,11 @@ print ('remote directory = %s'%remoteDir) # print 'job flavour = %s'%opts.jobFlavour +jobsDirSuffix = "" +if opts.jobsDirSuffix: + jobsDirSuffix = f"_{opts.jobsDirSuffix}" + print (f'Jobs directory suffix = {jobsDirSuffix}') + # #make directories for the jobs # try: @@ -90,7 +97,9 @@ # create the run_cfg.py -mainJobDir = MYDIR+'/Jobs' +jobsDir = 'Jobs'+jobsDirSuffix + +mainJobDir = MYDIR+'/'+jobsDir os.system('mkdir -p %s'%mainJobDir) process.source.fileNames = cms.untracked.vstring("INPUTFILES") @@ -121,7 +130,7 @@ for i in range(0, nJobs): #print 'total: %d/%d ; %.1f %% processed '%(j,my_sum,(100*float(j)/float(my_sum))) - jobDir = MYDIR+'/Jobs/Job_%s/'%str(i) + jobDir = MYDIR+'/'+jobsDir+'/Job_%s/'%str(i) os.system('mkdir -p %s'%jobDir) iFileMin = i*opts.nPerJob @@ -171,9 +180,9 @@ #requirements = "(OpSysAndVer =?= \"CentOS7\")" #condor_str += f"requirements = {requirements}\n" if "/eos/user/" in MYDIR or "/eos/home" in MYDIR: - condor_str += "queue filename matching (./Jobs/Job_*/*.sh)" + condor_str += f"queue filename matching (.{jobsDir}/Job_*/*.sh)" else: - condor_str += "queue filename matching ("+MYDIR+"/Jobs/Job_*/*.sh)" + condor_str += "queue filename matching ("+MYDIR+f"/{jobsDir}/Job_*/*.sh)" condor_name = MYDIR+"/condor_cluster.sub" condor_file = open(condor_name, "w") condor_file.write(condor_str) diff --git a/Prod/cmsCondorMC.py b/Prod/cmsCondorMC.py index 92914ea..bda9b0c 100755 --- a/Prod/cmsCondorMC.py +++ b/Prod/cmsCondorMC.py @@ -32,6 +32,7 @@ def runCommand(commandLine): parser.add_option("-n",dest="nPerJob",type="int",default=1,help="NUMBER of files processed per job",metavar="NUMBER") parser.add_option("-q","--flavour",dest="jobFlavour",type="str",default="workday",help="job FLAVOUR",metavar="FLAVOUR") parser.add_option("-p","--proxy",dest="proxyPath",type="str",default="noproxy",help="Proxy path") +parser.add_option("-s","--suffix",dest="jobsDirSuffix",type="str",default="",help="Suffix of the Jobs directory") opts, args = parser.parse_args() @@ -42,7 +43,8 @@ def runCommand(commandLine): help_text += '\n (mandatory) = directory where the files will be transfered (e.g. on EOS)' help_text += '\n (optional) = location of your voms cms proxy. Note: keep your proxy in a private directory.' help_text += '\n (optional) = number of files processed per batch job (default=1)' -help_text += '\n (optional) = job flavour (default=workday)\n' +help_text += '\n (optional) = job flavour (default=workday)' +help_text += '\n (optional) = job directory suffix (default=\"\")\n' cfgFileName = str(args[0]) @@ -55,10 +57,18 @@ def runCommand(commandLine): print('remote directory = %s'%remoteDir) print('job flavour = %s'%opts.jobFlavour) +jobsDirSuffix = "" +if opts.jobsDirSuffix: + jobsDirSuffix = f"_{opts.jobsDirSuffix}" + print (f'Jobs directory suffix = {jobsDirSuffix}') + #make directories for the jobs + +jobsDir = 'Jobs'+jobsDirSuffix + try: - os.system('rm -rf Jobs') - os.system('mkdir Jobs') + os.system(f'rm -rf {jobsDir}') + os.system(f'mkdir {jobsDir}') except: print("err!") pass @@ -127,7 +137,7 @@ def runCommand(commandLine): # create the run_cfg.py -mainJobDir = MYDIR+'/Jobs' +mainJobDir = MYDIR+'/'+jobsDir os.system('mkdir -p %s'%mainJobDir) process.source.fileNames = cms.untracked.vstring("INPUTFILES") @@ -165,7 +175,7 @@ def runCommand(commandLine): newdataset = str(dataset).replace("b'","") datasetName=newdataset.lstrip("/") datasetName=datasetName.replace("/","_") - datasetJobDir='Jobs/'+datasetName + datasetJobDir=jobsDir'/'+datasetName datasetRemoteDir=remoteDir+'/'+datasetName os.system('mkdir '+datasetJobDir) os.system('mkdir '+datasetRemoteDir) @@ -236,9 +246,9 @@ def runCommand(commandLine): condor_str += "log = $Fp(filename)hlt.log\n" condor_str += '+JobFlavour = "%s"\n'%opts.jobFlavour if "/eos/user/" in MYDIR or "/eos/home" in MYDIR: - condor_str += "queue filename matching (./Jobs/*/Job_*/*.sh)" + condor_str += f"queue filename matching (./{jobsDir}/*/Job_*/*.sh)" else: - condor_str += "queue filename matching ("+MYDIR+"/Jobs/*/Job_*/*.sh)" + condor_str += "queue filename matching ("+MYDIR+f"/{jobsDir}/*/Job_*/*.sh)" condor_name = MYDIR+"/condor_cluster.sub" condor_file = open(condor_name, "w") condor_file.write(condor_str) diff --git a/Rates/condorScriptForRatesData.py b/Rates/condorScriptForRatesData.py index 9dbe68a..ccc8eb0 100755 --- a/Rates/condorScriptForRatesData.py +++ b/Rates/condorScriptForRatesData.py @@ -17,19 +17,21 @@ parser.add_option("-n",dest="nPerJob",type="int",default=5,help="NUMBER of files processed per job",metavar="NUMBER") parser.add_option("-q","--flavour",dest="jobFlavour",type="str",default="workday",help="job FLAVOUR",metavar="FLAVOUR") parser.add_option("-m","--maps",dest="maps",type="str",default="nomaps",help="ARG='nomaps' (default option, don't use maps to get dataste/groups/etc. rates), 'somemaps' (get dataset/groups/etc. rates but with no study of dataset merging), 'allmaps' (get dataset/groups/etc. rates and also study dataset merging)",metavar="ARG") +parser.add_option("-s","--suffix",dest="dirSuffix",type="str",default="",help="Suffix of the Jobs and Results directories") opts, args = parser.parse_args() error_text = '\n\nError: wrong =%s or =%s inputs\n' %(opts.jsonFile, opts.cmsEnv) -help_text = '\npython3 batchScriptForRates.py -j -e -i -f -n -q -m ' +help_text = '\npython3 condorScriptForRatesData.py -j -e -i -f -n -q -m ' help_text += '\n (mandatory argument) = text file with the LS range in json format' help_text += '\n (mandatory) = directory where the top of a CMSSW release is located' help_text += '\n (optional) = directory where the input root files are located (default = will take whatever is in the filesInput.py file)' help_text += '\n (optional) = "custom" (default option) or "RAW"' help_text += '\n (optional) = number of files processed per batch job (default=5)' -help_text += '\n (optional) = job flavour (default=workday)\n' -help_text += '\n (optional) = "nomaps" (default option) or "somemaps" or "allmaps""\n' +help_text += '\n (optional) = job flavour (default=workday)' +help_text += '\n (optional) = "nomaps" (default option) or "somemaps" or "allmaps"' +help_text += '\n (optional) = suffix for the Jobs and Results directories (default=\"\")\n' if opts.jsonFile == "nojson" or opts.cmsEnv == "noenv": print(error_text) @@ -41,20 +43,27 @@ print('file type = %s'%opts.fileType) print('job flavour = %s'%opts.jobFlavour) +dirSuffix = "" +if opts.dirSuffix + dirSuffix = f"_{opts.dirSuffix}" + print (f'Jobs and Results directories suffix = _{dirSuffix}') + +jobsDir = 'Jobs' + dirSuffix #make directories for the jobs try: - os.system('rm -rf Jobs') - os.system('mkdir Jobs') - os.system('mkdir Jobs/sub_raw') + os.system(f'rm -rf {jobsDir}') + os.system(f'mkdir {jobsDir}') + os.system(f'mkdir {jobsDir}/sub_raw') except: print("err!") pass +resultsDir = "Results" + dirSuffix sub_total = open("sub_total.jobb","w") -sub_total.write("rm Results/Data/Raw/*/*.csv\n") -sub_total.write("rm Results/Data/Raw/*/*.root\n") +sub_total.write(f"rm {resultsDir}/Data/Raw/*/*.csv\n") +sub_total.write(f"rm {resultsDir}/Data/Raw/*/*.root\n") if opts.inputFilesDir != "no": @@ -80,15 +89,15 @@ #print 'total: %d/%d ; %.1f %% processed '%(j,my_sum,(100*float(j)/float(my_sum))) tmp_jobname="sub_%s.sh"%(str(i)) - tmp_job=open(MYDIR+'/Jobs/sub_raw/'+tmp_jobname,'w') + tmp_job=open(MYDIR+f'/{jobsDir}/sub_raw/'+tmp_jobname,'w') tmp_job.write("cd %s\n"%(MYDIR)) tmp_job.write("cd %s\n"%(opts.cmsEnv)) tmp_job.write("eval `scramv1 runtime -sh`\n") tmp_job.write("cd -\n") tmp_job.write("python3 triggerCountsFromTriggerResults.py -i %s -j %s -s %s -f %s -m %s\n"%(infile, opts.jsonFile, str(i), opts.fileType, opts.maps)) - tmp_job.write("\npython3 handleFileTransfer.py -d %s -s %s"%(MYDIR, str(i))) + tmp_job.write(f"\npython3 handleFileTransfer.py -d {MYDIR} -s {i} -S {dirSuffix}") tmp_job.close() - tmp_job_dir = MYDIR+'/Jobs/sub_raw/'+tmp_jobname + tmp_job_dir = MYDIR+f'/{jobsDir}/sub_raw/'+tmp_jobname os.system("chmod +x %s"%(tmp_job_dir)) @@ -97,7 +106,7 @@ if k==loop_mark or i==len(fileInputNames)-1: k=0 Tjobsname = "sub_%s.sh"%i - Tjob_dir = '%s/Jobs/Job_%s/'%(MYDIR, str(i)) + Tjob_dir = f'%s/{jobsDir}/Job_%s/'%(MYDIR, str(i)) os.system("mkdir %s"%Tjob_dir) Tjob = open(Tjob_dir+Tjobsname,"w") Tjob.write("%s"%(tmp_text)) @@ -116,9 +125,9 @@ #requirements = "(OpSysAndVer =?= \"CentOS7\")" #condor_str += f"requirements = {requirements}\n" if "/eos/user/" in MYDIR or "/eos/home" in MYDIR: - condor_str += "queue filename matching (./Jobs/Job_*/*.sh)" + condor_str += f"queue filename matching (./{jobsDir}/Job_*/*.sh)" else: - condor_str += "queue filename matching ("+MYDIR+"/Jobs/Job_*/*.sh)" + condor_str += f"queue filename matching ("+MYDIR+"/{jobsDir}/Job_*/*.sh)" condor_name = MYDIR+"/condor_cluster.sub" condor_file = open(condor_name, "w") condor_file.write(condor_str) diff --git a/Rates/config_makeCondorJobsData.py b/Rates/config_makeCondorJobsData.py index a5892ae..1c513ea 100755 --- a/Rates/config_makeCondorJobsData.py +++ b/Rates/config_makeCondorJobsData.py @@ -54,6 +54,11 @@ #Job flavour #flavour = "microcentury" flavour = "espresso" + +# Jobs and Rates directories suffix +# Change this if you want to run multiple rates tests in parallel with different settings +dirSuffix = "" + ''' --------------------------OPTIONS TO BE FILLED OUT----------------------------------------- ''' @@ -69,6 +74,9 @@ if isUnusual: command += " -n %s -q %s" %(n, flavour) +if dirSuffix: + command += f" -s _{dirSuffix}" + command += " -m %s" %maps os.system(command) diff --git a/Rates/handleFileTransfer.py b/Rates/handleFileTransfer.py index cd535e2..95c51df 100644 --- a/Rates/handleFileTransfer.py +++ b/Rates/handleFileTransfer.py @@ -7,10 +7,19 @@ parser.add_option("-d","--directory",dest="dir",type="str",default="nodir",help="working DIRECTORY",metavar="DIRECTORY") parser.add_option("-m","--dataset",dest="MCdataset",type="str",default="Data",help="MCdataset string") parser.add_option("-s","--finalstring",dest="fstr",type="str",default="nostr",help="final STRING",metavar="STRING") +parser.add_option("-S","--dirSuffix",dest="dirSuffix",type="str",default="",help="Suffix of the Results directory") (opts, args) = parser.parse_args() -os.system("mkdir %s/Results"%opts.dir) +head_dir = opts.dir + +dirSuffix = "" +if opts.dirSuffix + dirSuffix = f"_{opts.dirSuffix}" + +results_dir = "Results" + dirSuffix + +os.system(f"mkdir {head_dir}/{results_dir}") MCorData="MC" MCorData2="MC/"+opts.MCdataset inDir="Jobs/"+opts.MCdataset @@ -18,16 +27,16 @@ MCorData = "Data" MCorData2 = "Data" inDir = "Jobs" -os.system("mkdir %s/Results/%s"%(opts.dir, MCorData)) +os.system(f"mkdir {head_dir}/{results_dir}/{MCorData}") if opts.MCdataset != "Data": - os.system("mkdir %s/Results/%s"%(opts.dir, MCorData2)) -rawDir="%s/Results/%s/Raw"%(opts.dir, MCorData2) -os.system("mkdir %s"%rawDir) - + os.system(f"mkdir {head_dir}/{results_dir}/{MCorData2}") +rawDir=f"{head_dir}/{results_dir}/{MCorData2}/Raw" +os.system(f"mkdir {rawDir}") + for name in mergeNames: - os.system("mkdir %s/%s"%(rawDir, mergeNames[name])) -os.system("mkdir %s/Root"%rawDir) -os.system("mkdir %s/Global"%rawDir) + os.system(f"mkdir {rawDir}/{mergeNames[name]}") +os.system(f"mkdir {rawDir}/Root") +os.system(f"mkdir {rawDir}/Global") for filename in list(mergeNames.keys()):