1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
|
from dbaccess import execQuery, database
from misc import (
idToText, textToId, metricIdToLowerIsBetter, benchmarkToComponents,
getSnapshots, getTimeSeries, extractChanges, extractTimeSeriesStats,
computeLastChangeStabilityScores, printJSONHeader)
class GetTimeSeriesStats:
def __init__(
self, host, platform, branch, sha11, sha12, difftol,
test_case_filter):
self.host = host
self.platform = platform
self.branch = branch
self.sha11 = sha11
self.sha12 = sha12
self.difftol = difftol
self.test_case_filter = test_case_filter
self.host_id = textToId("host", host)
self.platform_id = textToId("platform", platform)
self.branch_id = textToId("branch", branch)
self.sha11_id = textToId("sha1", sha11)
self.sha12_id = textToId("sha1", sha12)
def execute(self):
self.snapshots = getSnapshots(
self.host_id, self.platform_id, self.branch_id, self.sha11_id,
self.sha12_id)
self.bmstats_list = self.computeBMStatsList()
self.writeOutput()
# Computes per-benchmark change statistics.
# Returns an n-tuple of 2-tuples:
#
# (testCase, testFunction, dataTag, metric, MS, LSD, NC, LCSS, LC,
# BC, WC)
#
# Note: the stats (MS, ..., WC) are documented elsewhere.
#
def computeBMStatsList(self):
# Get all distinct benchmark/metric combinations that match the
# host/platform/branch context and are within the selected snapshot
# interval:
query = (
"SELECT DISTINCT benchmarkId, metricId FROM result WHERE " +
" hostId = " + str(self.host_id) + " AND platformId = " +
str(self.platform_id) + " AND branchId = " + str(self.branch_id) +
" AND sha1id IN (")
first = True
for s in self.snapshots:
if not first:
query += ", "
else:
first = False
query += str(s[0])
query += ");"
bmark_metrics = execQuery(query);
bmstats_list = []
max_lcdb = -1
max_lcda = -1
# Compute basic stats for each time series:
#for benchmark_id, metric_id in bmark_metrics[800:810]:
for benchmark_id, metric_id in bmark_metrics:
benchmark = idToText("benchmark", benchmark_id)
# if benchmark != "tst_qhostinfo:lookupSpeed(WithoutCache)":
# continue
# if benchmark != "tst_QSslSocket:systemCaCertificates()":
# continue
test_case, test_function, data_tag = (
benchmarkToComponents(benchmark))
if ((self.test_case_filter != None)
and (not test_case in self.test_case_filter)):
continue
(time_series, tot_ninvalid, tot_nzeros, median_of_rses,
rse_of_medians) = getTimeSeries(
self.host_id, self.platform_id, self.branch_id,
self.snapshots, benchmark_id, metric_id)
changes = extractChanges(
time_series, metricIdToLowerIsBetter(metric_id), self.difftol)
stats = extractTimeSeriesStats(time_series, changes, self.snapshots)
stats["ni"] = tot_ninvalid
stats["nz"] = tot_nzeros
if median_of_rses >= 0:
stats["med_of_rses"] = median_of_rses
if rse_of_medians >= 0:
stats["rse_of_meds"] = rse_of_medians
if "lcdb" in stats:
max_lcdb = max(max_lcdb, stats["lcdb"])
max_lcda = max(max_lcda, stats["lcda"])
stats["test_case"] = test_case
stats["test_function"] = test_function
stats["data_tag"] = data_tag
stats["metric"] = idToText("metric", metric_id)
stats["lib"] = metricIdToLowerIsBetter(metric_id)
stats["time_series"] = time_series # ### Store here for now
bmstats_list.append(stats)
# Add stability scores for the last change (if any) of each benchmark:
for bmstats in bmstats_list:
if "pos_lc" in bmstats:
lcss, lcss_ls = computeLastChangeStabilityScores(
zip(*(bmstats["time_series"]))[1], bmstats["lcdb"],
bmstats["lcda"], max_lcdb, max_lcda, bmstats["pos_lc"],
bmstats["pos_pc"], bmstats["lc_at_ls"])
bmstats["lcss"] = lcss
if lcss_ls >= 0:
bmstats["lcss_ls"] = lcss_ls
return tuple(bmstats_list)
def writeOutputAsJSON(self):
printJSONHeader()
print "{"
# Context:
print "\"database\": \"" + str(database()) + "\", "
print "\"host\": \"" + str(self.host) + "\", "
print "\"platform\": \"" + str(self.platform) + "\", "
print "\"branch\": \"" + str(self.branch) + "\", "
# (note that self.sha11 and self.sha12 may not be in
# chronological order, but self.snapshots is:)
print (
"\"sha11\": \"" + str(idToText("sha1", self.snapshots[0][0])) +
"\", ")
print (
"\"sha12\": \"" + str(idToText("sha1", self.snapshots[-1][0])) +
"\", ")
print "\"difftol\": " + str(self.difftol) + ""
# Snapshots:
print ", \"snapshots\": ["
first_row = True
for sha1_id, timestamp in self.snapshots:
if not first_row:
print ",",
first_row = False
print (
"[\"" + str(idToText("sha1", sha1_id)) + "\", " +
str(timestamp) + "]")
print "]"
# Per-benchmark stats:
print ", \"per_bm_stats\": ["
first_row = True
for stats in self.bmstats_list:
if not first_row:
print ","
first_row = False
print "{"
print "\"ms\": " + str(stats["ms"]) + ", "
if "lsd" in stats:
print "\"lsd\": " + str(stats["lsd"]) + ", "
print "\"ni\": " + str(stats["ni"]) + ", "
print "\"nz\": " + str(stats["nz"]) + ", "
print "\"nc\": " + str(stats["nc"]) + ", "
if "med_of_rses" in stats:
print "\"med_of_rses\": " + str(stats["med_of_rses"]) + ", "
if "rse_of_meds" in stats:
print "\"rse_of_meds\": " + str(stats["rse_of_meds"]) + ", "
if "lc" in stats:
print "\"lcss\": " + str(stats["lcss"]) + ", "
print "\"lc\": " + str(stats["lc"]) + ", "
print "\"bc\": " + str(stats["bc"]) + ", "
print "\"wc\": " + str(stats["wc"]) + ", "
if "lcss_ls" in stats:
print "\"lcss_ls\": " + str(stats["lcss_ls"]) + ", "
print "\"mt\": \"" + stats["metric"] + "\", "
print "\"lib\": \"" + ("1" if stats["lib"] else "0") + "\", "
print (
"\"bm\": \"" + stats["test_case"] + ":" +
stats["test_function"] + "(" + stats["data_tag"] + ")\"")
print "}"
print "]"
print "}"
class GetTimeSeriesStatsAsJSON(GetTimeSeriesStats):
def writeOutput(self):
self.writeOutputAsJSON()
|
