FourTops/LaunchParallelMacro.py at CMSSW_72X · qpythonn/FourTops · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
import xml.etree.cElementTree as ET
import subprocess
import time
import os

tree = ET.ElementTree(file='config/Run2DiLepton_TOPTREES.xml')

root = tree.getroot()
datasets = root.find('datasets')
procsDone = 0
procsStarted = 0
numCores = 8
args = []
execCommands = []
jobSize = 1000000
for d in datasets:
    if d.attrib['add'] == '1':
        args.append(["./MACRO", d.attrib['name'], d.attrib['title'], d.attrib['add'], d.attrib['color'], d.attrib['ls'], d.attrib['lw'], d.attrib['normf'], d.attrib['EqLumi'], d.attrib['xsection'], d.attrib['PreselEff'], d.attrib['filenames']])
outfiles = []
fileNames = []
processes = []
tempList = []
if not os.path.exists("Terminal_Output"):
    os.makedirs("Terminal_Output")

for row in args:
    if row[3] == '1':
        title = row[1]

        totalEvents = float(row[8])*float(row[9])
#        tempList = list(row)
#        tempList.extend(["", ""])
        if (totalEvents > jobSize):
            endEvent = 0
            while (totalEvents-(endEvent*jobSize) > 0):
                startStr = str(endEvent*jobSize)
                endStr = str((endEvent+1)*jobSize)
                tempList = list(row)
                tempList.extend(["", ""])
                tempList[12] = startStr
                tempList[13] = endStr
                tempList[1] = title+"_"+str(endEvent+1)
                fileNames.append("Terminal_Output/"+tempList[1]+".out")
                execCommands.append(tempList)
                print 'Job {} Created'.format(tempList[1])
                #popen = subprocess.Popen(execCommands, stdout=outfile)
                #processes.append(popen)
                endEvent += 1
        else:
            tempList = list(row)
            tempList.extend(["", ""])
            tempList[12] = "0"
            tempList[13] = str(jobSize)
            fileNames.append("Terminal_Output/"+tempList[1]+".out")
            execCommands.append(tempList)
            #popen = subprocess.Popen(execCommands, stdout=outfile)
            #processes.append(popen)
#        popen.wait()
#        for i in row:
#            print i
for i, row in enumerate(execCommands):
    outfile = open(fileNames[i], 'w')
    print 'File {} opened'.format(fileNames[i])
    outfiles.append(outfile)
    row.insert(0, "nohup")
    popen = subprocess.Popen(row, stdout=outfiles[i])
    print 'Job {} begun'.format(row[2])
    processes.append(popen)
    procsStarted += 1
    print 'Jobs {} of {} started.  Timestamp: {}'.format(procsStarted, len(execCommands), time.ctime())
    while (procsStarted-procsDone) >= (numCores/2):
        time.sleep(60)
        procsDone = 0
        for proc in processes:
            if proc.poll() != None:
                procsDone+= 1
        print '{} jobs of {} Finished.  Timestamp: {}'.format(procsDone, len(execCommands), time.ctime())
while (procsDone != len(execCommands)):  #This loop controls the status output for the last 4 jobs that are still running when the above for loop terminates
        time.sleep(60)
        procsDone = 0
        for proc in processes:
            if proc.poll() != None:
                procsDone+= 1
        print '{} jobs of {} Finished.  Timestamp: {}'.format(procsDone, len(execCommands), time.ctime())


#while procsDone < len(processes):
#    time.sleep(60)
#    procsDone = 0
#    counter = 0
#    for proc in processes:
#        counter += 1
#        if proc.poll() != None:
#            procsDone += 1
#            print 'Job {} complete.'.format(counter)
#        else:
#            print 'Job {} still running.'.format(counter)
#    print '{} jobs of {} completed.  Timestamp: {}'.format(procsDone, len(processes), time.ctime())