-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathTEST_output_processor.py
executable file
·100 lines (83 loc) · 7.44 KB
/
TEST_output_processor.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
#!/usr/bin/env python
__import__('sys').path.append(__import__('os').path.join(__import__('os').path.dirname(__file__), ''))
__import__('testfwk').setup(__file__)
# - prolog marker
from testfwk import TestsuiteStream, run_test, str_dict_testsuite, try_catch
from grid_control.output_processor import JobResult, OutputProcessor, TaskOutputProcessor
def display(value):
raw = None
for key in list(value):
if key == JobResult.RAW:
raw = str_dict_testsuite(value.pop(key))
elif key in JobResult.enum_value_list:
value[JobResult.enum2str(key)] = value.pop(key)
print('result: ' + str_dict_testsuite(value) + '\nraw: ' + str(raw))
def modify_log(lines):
sstr = 'Not a gzipped'
for line in lines:
if sstr in line:
line = line[:line.find(sstr) + len(sstr)] + '...'
yield line
TestsuiteStream.set_modify(modify_log)
class Test_OutputProcessor:
"""
>>> try_catch(lambda: OutputProcessor().process('.'), 'AbstractError', 'process is an abstract function')
caught
>>> jip = OutputProcessor.create_instance('JobInfoProcessor')
>>> try_catch(lambda: jip.process('work.jobdb/output.job_F1'), 'JobResultError', 'does not exist')
caught
>>> try_catch(lambda: jip.process('work.jobdb/output.job_F2'), 'JobResultError', 'is empty')
caught
>>> try_catch(lambda: jip.process('work.jobdb/output.job_F3'), 'JobResultError', 'Unable to read job result file')
caught
>>> try_catch(lambda: jip.process('work.jobdb/output.job_F4'), 'JobResultError', 'is incomplete')
caught
>>> display(jip.process('work.jobdb/output.job_0'))
result: {'EXITCODE': 0, 'JOBNUM': 0, 'MESSAGE': None}
raw: {'FILE': '"f9ed86240f138b0814fbb6bbce98c516 skim.root kappa_2011-11-28_FS_442_Jet_2011A_RR_Nov08_0.root srm://dgridsrm-fzk.gridka.de:8443/srm/managerv2?SFN=/pnfs/gridka.de/dcms/disk-only/users/stober"', 'TIME': 27417}
>>> display(jip.process('work.jobdb/output.job_1'))
result: {'EXITCODE': 123, 'JOBNUM': 155, 'MESSAGE': None}
raw: {'FILE': '"6a24adfdc1a710b23ba7e5744bb69bc7 skim.root kappa_2011-11-28_FS_442_Jet_2011A_RR_Nov08_155.root srm://dgridsrm-fzk.gridka.de:8443/srm/managerv2?SFN=/pnfs/gridka.de/dcms/disk-only/users/stober"', 'TIME': 32571}
>>> display(jip.process('work.jobdb/output.job_2'))
result: {'EXITCODE': 0, 'JOBNUM': 0, 'MESSAGE': None}
raw: {'FILE': '"dc5310668534534957283c8d07be51e5 output.exp Dataset1_job_0_output.exp dir:///home/fred/grid-control/docs/XXX"', 'FILE1': '"6e63ef4f3c68d0fe35c3ead00a990c13 output.hallo Dataset1_job_0_output.hallo dir:///home/fred/grid-control/docs/XXX"', 'TIME': 17}
>>> display(jip.process('work.jobdb/output.job_3'))
result: {'EXITCODE': 0, 'JOBNUM': 0, 'MESSAGE': None}
raw: {'FILE': '"cd4ce73f52940d6daffbbb82d29bcac8 histo_reconstruction.root pfembTauTau_data_embedding_SingleMu_2011A_PR_v1_0_0_pt0_0_histo_reconstruction.root srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/pnfs/desy.de/cms/tier2/store/user/aburgmei/embedding/20110420-emb2/"', 'FILE1': '"e41fa44c6595595428c79db135930bc6 histo_simulation.root pfembTauTau_data_embedding_SingleMu_2011A_PR_v1_0_0_pt0_0_histo_simulation.root srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/pnfs/desy.de/cms/tier2/store/user/aburgmei/embedding/20110420-emb2/"', 'FILE2': '"acac0de54ef6c683ce479b227ba1fc85 embedded_RECO.root pfembTauTau_data_embedding_SingleMu_2011A_PR_v1_0_0_pt0_0_embedded_RECO.root srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/pnfs/desy.de/cms/tier2/store/user/aburgmei/embedding/20110420-emb2/"', 'TIME': 496}
>>> display(jip.process('work.jobdb/output.job_4'))
result: {'EXITCODE': 0, 'JOBNUM': 0, 'MESSAGE': None}
raw: {'FILE': '"efa0f7787043f4eda6ef289e2d44f0bb log.txt DYJetsToLLM50_RunIIFall15MiniAODv2_PU25nsData2015v1_13TeV_MINIAOD_madgraph-pythia8/DYJetsToLLM50_RunIIFall15MiniAODv2_PU25nsData2015v1_13TeV_MINIAOD_madgraph-pythia8_job_0_log.txt srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/pnfs/desy.de/cms/tier2/store/user/tmuller/artus/2016-04-22_13-29_analysis/output"', 'FILE1': '"43a198d3a7796d2c89f17238bec2d5dc output.root DYJetsToLLM50_RunIIFall15MiniAODv2_PU25nsData2015v1_13TeV_MINIAOD_madgraph-pythia8/DYJetsToLLM50_RunIIFall15MiniAODv2_PU25nsData2015v1_13TeV_MINIAOD_madgraph-pythia8_job_0_output.root srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/pnfs/desy.de/cms/tier2/store/user/tmuller/artus/2016-04-22_13-29_analysis/output"', 'OUTPUT_FILE_0_DEST': '"DYJetsToLLM50_RunIIFall15MiniAODv2_PU25nsData2015v1_13TeV_MINIAOD_madgraph-pythia8/DYJetsToLLM50_RunIIFall15MiniAODv2_PU25nsData2015v1_13TeV_MINIAOD_madgraph-pythia8_job_0_log.txt"', 'OUTPUT_FILE_0_HASH': 'efa0f7787043f4eda6ef289e2d44f0bb', 'OUTPUT_FILE_0_LOCAL': '"log.txt"', 'OUTPUT_FILE_0_PATH': '"srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/pnfs/desy.de/cms/tier2/store/user/tmuller/artus/2016-04-22_13-29_analysis/output"', 'OUTPUT_FILE_0_SIZE': 61407, 'OUTPUT_FILE_1_DEST': '"DYJetsToLLM50_RunIIFall15MiniAODv2_PU25nsData2015v1_13TeV_MINIAOD_madgraph-pythia8/DYJetsToLLM50_RunIIFall15MiniAODv2_PU25nsData2015v1_13TeV_MINIAOD_madgraph-pythia8_job_0_output.root"', 'OUTPUT_FILE_1_HASH': '43a198d3a7796d2c89f17238bec2d5dc', 'OUTPUT_FILE_1_LOCAL': '"output.root"', 'OUTPUT_FILE_1_PATH': '"srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/pnfs/desy.de/cms/tier2/store/user/tmuller/artus/2016-04-22_13-29_analysis/output"', 'OUTPUT_FILE_1_SIZE': 1619503, 'TIME': 387, 'TIMESTAMP_CMSSW_EPILOG1_DONE': 1461325248, 'TIMESTAMP_CMSSW_EPILOG1_START': 1461324879, 'TIMESTAMP_CMSSW_STARTUP_DONE': 1461324879, 'TIMESTAMP_CMSSW_STARTUP_START': 1461324873, 'TIMESTAMP_DEPLOYMENT_DONE': 1461324868, 'TIMESTAMP_DEPLOYMENT_START': 1461324867, 'TIMESTAMP_EXECUTION_DONE': 1461325248, 'TIMESTAMP_EXECUTION_START': 1461324872, 'TIMESTAMP_SE_IN_DONE': 1461324872, 'TIMESTAMP_SE_IN_START': 1461324868, 'TIMESTAMP_SE_OUT_DONE': 1461325254, 'TIMESTAMP_SE_OUT_START': 1461325248, 'TIMESTAMP_WRAPPER_DONE': 1461325255, 'TIMESTAMP_WRAPPER_START': 1461324867}
>>> djip = OutputProcessor.create_instance('DebugJobInfoProcessor')
>>> djip.process('work.jobdb/output.job_0') == jip.process('work.jobdb/output.job_0')
True
>>> djip._display_files.append('gc.test.gz')
>>> djip._display_files.append('gc.fail.gz')
>>> display(djip.process('work.jobdb/output.job_1'))
0000-00-00 00:00:00 - jobs.output:ERROR - gc.stdout
STDOUT message1
STDOUT message2
STDOUT message3
--------------------------------------------------
0000-00-00 00:00:00 - jobs.output:ERROR - Log file does not exist: gc.stderr
0000-00-00 00:00:00 - jobs.output:ERROR - gc.test.gz
TEST message1
TEST message2
TEST message3
--------------------------------------------------
0000-00-00 00:00:00 - jobs.output:ERROR - Unable to display gc.fail.gz: XXError: Not a gzipped...
result: {'EXITCODE': 123, 'JOBNUM': 155, 'MESSAGE': None}
raw: {'FILE': '"6a24adfdc1a710b23ba7e5744bb69bc7 skim.root kappa_2011-11-28_FS_442_Jet_2011A_RR_Nov08_155.root srm://dgridsrm-fzk.gridka.de:8443/srm/managerv2?SFN=/pnfs/gridka.de/dcms/disk-only/users/stober"', 'TIME': 32571}
>>> fip = OutputProcessor.create_instance('FileInfoProcessor')
>>> fip.process('work.jobdb/output.job_F1')
Unable to process job information: JobResultError: Unable to process output directory 'work.jobdb/output.job_F1'
----------
JobResultError: Job result file 'work.jobdb/output.job_F1/job.info' does not exist
"""
class Test_TaskOutputProcessor:
"""
>>> try_catch(lambda: TaskOutputProcessor().process(dn=None, task=None), 'AbstractError', 'process is an abstract function')
caught
>>> TaskOutputProcessor.create_instance('SandboxProcessor').process(dn=None, task=None)
True
"""
run_test()