summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorjasplin <qt-info@nokia.com>2010-05-10 16:53:29 +0200
committerjasplin <qt-info@nokia.com>2010-05-10 16:53:29 +0200
commit65589d5d1e1678227f12d9e50d60b3c02f7617b4 (patch)
tree6687f8a11e2d2b673f5d6363e834f17eab9814c5
parentc0c856290e3232dc0d6dd6e9a5a1c22e45d128e2 (diff)
Added ASF Scorecard feature.
-rw-r--r--src/bm/asfstats.cpp211
-rw-r--r--src/bm/asfstats.h74
-rw-r--r--src/bm/bm.pro4
-rw-r--r--src/bm/bmmisc.cpp31
-rw-r--r--src/bm/bmmisc.h4
-rw-r--r--src/bm/bmrequest.cpp700
-rw-r--r--src/bm/bmrequest.h44
-rw-r--r--src/bm/resulthistoryinfo.cpp97
-rw-r--r--src/bm/resulthistoryinfo.h10
-rw-r--r--src/bmclient/main.cpp162
-rw-r--r--src/bmweb/indexsection.js96
11 files changed, 1264 insertions, 169 deletions
diff --git a/src/bm/asfstats.cpp b/src/bm/asfstats.cpp
new file mode 100644
index 0000000..960e1c2
--- /dev/null
+++ b/src/bm/asfstats.cpp
@@ -0,0 +1,211 @@
+/****************************************************************************
+**
+** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
+** Contact: Qt Software Information (qt-info@nokia.com)
+**
+** This file is part of the BM project on Qt Labs.
+**
+** This file may be used under the terms of the GNU General Public
+** License version 2.0 or 3.0 as published by the Free Software Foundation
+** and appearing in the file LICENSE.GPL included in the packaging of
+** this file. Please review the following information to ensure GNU
+** General Public Licensing requirements will be met:
+** http://www.fsf.org/licensing/licenses/info/GPLv2.html and
+** http://www.gnu.org/copyleft/gpl.html.
+**
+** If you are unsure which license is appropriate for your use, please
+** contact the sales department at qt-sales@nokia.com.
+**
+** This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
+** WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
+**
+****************************************************************************/
+
+#include "asfstats.h"
+#include "bmmisc.h"
+#include <QList>
+#include <QMap>
+#include <QDebug>
+
+/* NOTES
+
+Glossary:
+
+- RH: Result history.
+
+
+Params:
+
+- diffTolerance (in percent: 0 <= x <= 100)
+- stabTolerance (a positive integer: x >= 2)
+
+- sfTolerance (in percent) - the lowest tolerable SF (i.e. at or above is considered stable)
+- lfTolerance (in percent) - the lowest tolerable LF (i.e. at or above is considered stable)
+- maxLDTolerance(in percent) - the highest tolerable maxLD (i.e. at or below is considered stable)
+
+-------------------------------------------------
+
+Def: An equality subsequence (ESS) is a subsequence v1, v2, ..., vn of a RH
+ for which the following condition holds:
+
+ ∀ i >= 1 : 100 * (max(vi, v1) / min(vi, v1) - 1) <= diffTolerance
+
+
+Def: A maximal equality subsequence (MaxESS) is one of the subsequences formed by
+ partitioning a RH into the smallest possible number of ESS'es.
+
+
+Def: The stability fraction (SF) of a RH is the fraction (given as a
+ percentage: 0 <= SF <= 100) of its MaxESS'es that are stable.
+ More precisely,
+
+ SF = 100 * (stableMaxESS / totalMaxESS),
+
+ where stableMaxESS is the the number of MaxESS'es that have a length of at least
+ stabTolerance and totalMaxESS is the total number of MaxESS'es.
+
+Def: The level fraction (LF) of a RH is the fraction (given as a percentage: 0 <= LF <= 100)
+ of its levels that are unique (distinct).
+ More precisely,
+
+ LF = 100 * (uniqueLevels / totalMaxESS)
+
+ where uniqueLevels is the number of unique levels and totalMaxESS is the total number of
+ MaxESS'es. Note that each MaxESS has a level (which is currently defined as its first value).
+ Example: A RH with levels 2, 10, 2, 10, 2 has a LF of 40% (2 of 5 levels are unique),
+ whereas one with levels 2, 11, 3, 12, 4 has a LF of 100% (5 of 5 levels are unique).
+
+Def: The maximum level distance (maxLD) of a RH is the relative difference (given as a percentage;
+ 0 <= maxLD <= 100) between the highest and lowest level.
+ More precisely:
+
+ maxLD = 100 * ((maxLevel / minLevel) - 1)
+
+-------
+
+Step 1: Identifying unstable RHs
+
+ A RH is considered unstable if at least one of the following conditions are true.
+ Except for the first condition, the conditions are evaluated for the raw
+ (i.e. unsmoothed) data.
+
+ Cond 1: At least one zero value occurs
+ Cond 2: The RH doesn't have a _smoothed_ value before fromTimestamp.
+ Cond 3: SF < sfTolerance
+ Cond 4: LF < lfTolerance
+ Cond 5: maxLD < maxLDTolerance
+
+ Stats produced:
+ Stat 1: Count for Cond 1
+ Stat 2: Count for Cond 2
+ Stat 3: Count for Cond 3
+ Stat 4: Count for Cond 4
+ Stat 5: Count for Cond 5
+ Stat 6: Total unstable count
+
+ Other output:
+ The list of BM context IDs of the unstable RHs.
+
+
+Step 2: Sample the stable and smoothed RHs at fromTimestamp and toTimestamp and compare the
+ difference.
+ For each RH, the two values to be compared - v1 and v2 - are the latest smoothed
+ values that are not later than fromTimestamp and toTimestamp respectively.
+ If 100 * (max(v1, v2) / min(v1, v2) <= diffTolerance, the values are considered equal.
+ Otherwise, the diff is computed as v1 - v2 for a "lower is better" metric or v2 - v1
+ for a "higher is better" metric. If diff < 0, the RH regressed, otherwise it improved.
+
+ Stats produced:
+ Stat 1: Regressed count.
+ Stat 2: Unchanged count.
+ Stat 4: Improved count.
+
+*/
+
+
+// ### 2 B DOCUMENTED!
+void ASFStats::compute(
+ const QList<ResultHistoryInfo *> &rhInfos, UnstableStats *usStats, int *regressed,
+ int *unchanged, int *improved)
+{
+ Q_ASSERT((fromTimestamp >= 0) && (fromTimestamp <= toTimestamp));
+ Q_ASSERT((diffTolerance >= 0.0) && (diffTolerance <= 100.0));
+ Q_ASSERT(stabTolerance >= 2);
+ Q_ASSERT((sfTolerance >= 0.0) && (sfTolerance <= 100.0));
+ Q_ASSERT((lfTolerance >= 0.0) && (lfTolerance <= 100.0));
+
+ usStats->unstable.fill(false, rhInfos.size());
+ *regressed = *unchanged = *improved = 0;
+
+ for (int i = 0; i < rhInfos.size(); ++i) {
+ bool zerosFound = false;
+ int total = 0;
+ int stable = 0;
+ int uniqueLevels = 0;
+ qreal minLevel = 0;
+ qreal maxLevel = 0;
+
+ // Compute stability stats for raw (unsmoothed) data ...
+ rhInfos.at(i)->computeStabilityStats(
+ diffTolerance, stabTolerance, &zerosFound, &total, &stable, &uniqueLevels, &minLevel,
+ &maxLevel);
+
+ Q_ASSERT(total > 0);
+ const qreal sf = 100 * (stable / qreal(total));
+ const qreal lf = 100 * (uniqueLevels / qreal(total));
+
+ const qreal maxLD = zerosFound ? -1 : (100 * ((maxLevel / minLevel) - 1));
+
+ int fromPos = -1;
+ rhInfos.at(i)->findSmoothPos(fromTimestamp, &fromPos);
+
+ bool unstable = false;
+ if (zerosFound) {
+ unstable = true;
+ ++(usStats->zeroCount);
+ }
+ if (fromPos == -1) {
+ unstable = true;
+ ++(usStats->lowFromPosCount);
+ }
+ if (sf < sfTolerance) {
+ unstable = true;
+ ++(usStats->lowSFCount);
+ }
+ if (lf < lfTolerance) {
+ unstable = true;
+ ++(usStats->lowLFCount);
+ }
+ if (maxLD > maxLDTolerance) {
+ unstable = true;
+ ++(usStats->highMaxLDCount);
+ }
+
+ if (unstable) {
+
+ // RH is unstable, so mark it as such ...
+ usStats->unstable.setBit(i);
+
+ } else {
+ // RH is stable, so sample the smoothed values at fromTimestamp and toTimestamp and
+ // classify the difference as 'regressed', 'unchanged', or 'improved' ...
+
+ int toPos = -1;
+ const bool ok = rhInfos.at(i)->findSmoothPos(toTimestamp, &toPos);
+ Q_ASSERT(ok);
+
+ const qreal v1 = rhInfos.at(i)->value(fromPos);
+ const qreal v2 = rhInfos.at(i)->value(toPos);
+ if ((100 * (qMax(v1, v2) / qMin(v1, v2) - 1)) <= diffTolerance) {
+ ++(*unchanged);
+ } else {
+ const qreal diff =
+ BMMisc::lowerIsBetter(rhInfos.at(i)->metric()) ? (v1 - v2) : (v2 - v1);
+ if (diff < 0)
+ ++(*regressed);
+ else
+ ++(*improved);
+ }
+ }
+ }
+}
diff --git a/src/bm/asfstats.h b/src/bm/asfstats.h
new file mode 100644
index 0000000..c153376
--- /dev/null
+++ b/src/bm/asfstats.h
@@ -0,0 +1,74 @@
+/****************************************************************************
+**
+** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
+** Contact: Qt Software Information (qt-info@nokia.com)
+**
+** This file is part of the BM project on Qt Labs.
+**
+** This file may be used under the terms of the GNU General Public
+** License version 2.0 or 3.0 as published by the Free Software Foundation
+** and appearing in the file LICENSE.GPL included in the packaging of
+** this file. Please review the following information to ensure GNU
+** General Public Licensing requirements will be met:
+** http://www.fsf.org/licensing/licenses/info/GPLv2.html and
+** http://www.gnu.org/copyleft/gpl.html.
+**
+** If you are unsure which license is appropriate for your use, please
+** contact the sales department at qt-sales@nokia.com.
+**
+** This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
+** WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
+**
+****************************************************************************/
+
+#ifndef ASFSTATS_H
+#define ASFSTATS_H
+
+#include "resulthistoryinfo.h"
+
+// ### 2 B DOCUMENTED!
+
+class ASFStats {
+public:
+ ASFStats(
+ const qreal diffTolerance, const int stabTolerance, const qreal sfTolerance,
+ const qreal lfTolerance, const qreal maxLDTolerance, const int fromTimestamp,
+ const int toTimestamp)
+ : diffTolerance(diffTolerance)
+ , stabTolerance(stabTolerance)
+ , sfTolerance(sfTolerance)
+ , lfTolerance(lfTolerance)
+ , maxLDTolerance(maxLDTolerance)
+ , fromTimestamp(fromTimestamp)
+ , toTimestamp(toTimestamp)
+ {}
+
+ struct UnstableStats {
+ QBitArray unstable;
+ int zeroCount;
+ int lowFromPosCount;
+ int lowSFCount;
+ int lowLFCount;
+ int highMaxLDCount;
+ UnstableStats()
+ : zeroCount(0), lowFromPosCount(), lowSFCount(0), lowLFCount(0), highMaxLDCount(0) {}
+ };
+
+ void compute(
+ const QList<ResultHistoryInfo *> &rhInfos, UnstableStats *usStats, int *regressed,
+ int *unchanged, int *improved);
+
+private:
+ // Tolerances:
+ qreal diffTolerance; // difference
+ int stabTolerance; // stability
+ qreal sfTolerance; // stability fraction
+ qreal lfTolerance; // level fraction
+ qreal maxLDTolerance; // maximum level difference
+
+ // Timestamps:
+ int fromTimestamp;
+ int toTimestamp;
+};
+
+#endif // ASFSTATS_H
diff --git a/src/bm/bm.pro b/src/bm/bm.pro
index 70ae5dd..1bf4f1e 100644
--- a/src/bm/bm.pro
+++ b/src/bm/bm.pro
@@ -2,9 +2,9 @@ TEMPLATE = lib
CONFIG += shared
TARGET = bm
SOURCES += bm.cpp bmrequest.cpp bmmisc.cpp plotter.cpp resulthistoryinfo.cpp cache.cpp index.cpp \
- dataqualitystats.cpp
+ dataqualitystats.cpp asfstats.cpp
HEADERS += bm.h bmrequest.h bmmisc.h plotter.h resulthistoryinfo.h cache.h index.h \
- dataqualitystats.h
+ dataqualitystats.h asfstats.h
QT += network
QT += xml
QT += sql
diff --git a/src/bm/bmmisc.cpp b/src/bm/bmmisc.cpp
index 90d4062..23bebda 100644
--- a/src/bm/bmmisc.cpp
+++ b/src/bm/bmmisc.cpp
@@ -187,6 +187,37 @@ bool BMMisc::hasOption(const QStringList &args, const QString &option)
return getOption(args, option, &dummyValues, 0, 0);
}
+// ### 2 B DOCUMENTED!
+bool BMMisc::getDoubleOption(
+ const QStringList &args, const QString &option, qreal *value, QString *error)
+{
+ QStringList values;
+ if (getOption(args, option, &values, 1, 0, error)) {
+ bool ok;
+ *value = values.at(0).toDouble(&ok);
+ if (!ok) {
+ *error = QString("failed to extract %1 as a double").arg(option);
+ return false;
+ }
+ } else {
+ if (error->isEmpty())
+ *error = QString("%1 option not found").arg(option);
+ return false;
+ }
+
+ return true;
+}
+
+// ### 2 B DOCUMENTED!
+bool BMMisc::getClampedPercentageOption(
+ const QStringList &args, const QString &option, qreal *value, QString *error)
+{
+ if (!getDoubleOption(args, option, value, error))
+ return false;
+ *value = qMin(qMax(*value, 0.0), 100.0);
+ return true;
+}
+
// Computes the median value of the numbers in \a values.
qreal BMMisc::median(const QList<qreal> &values)
{
diff --git a/src/bm/bmmisc.h b/src/bm/bmmisc.h
index 057adac..62469bd 100644
--- a/src/bm/bmmisc.h
+++ b/src/bm/bmmisc.h
@@ -57,6 +57,10 @@ public:
const QStringList &args, const QString &option, QList<QStringList> *values, int n,
QString *error);
static bool hasOption(const QStringList &args, const QString &option);
+ static bool getDoubleOption(
+ const QStringList &args, const QString &option, qreal *value, QString *error);
+ static bool getClampedPercentageOption(
+ const QStringList &args, const QString &option, qreal *value, QString *error);
static qreal median(const QList<qreal> &values);
static int medianPos(const QList<qreal> &values, bool lastDuplicate = true);
static qreal v2y(
diff --git a/src/bm/bmrequest.cpp b/src/bm/bmrequest.cpp
index 0025c0a..fa09701 100644
--- a/src/bm/bmrequest.cpp
+++ b/src/bm/bmrequest.cpp
@@ -24,6 +24,7 @@
#include "bmrequest.h"
#include "resulthistoryinfo.h"
#include "dataqualitystats.h"
+#include "asfstats.h"
#include "plotter.h"
#include "cache.h"
#include <QSqlDatabase>
@@ -107,6 +108,8 @@ BMRequest * BMRequest::create(const QByteArray &data, const QString &msgType)
request = new BMRequest_GetBMTree(doc);
else if (type == "GetIXHistories")
request = new BMRequest_GetIXHistories(doc);
+ else if (type == "ASFStatsGetValues")
+ request = new BMRequest_ASFStatsGetValues(doc);
#ifdef BMDEBUG
else
qDebug() << "invalid request type:" << type;
@@ -4842,6 +4845,143 @@ void BMRequest_GetResult::handleReply_Raw(const QStringList &args) const
}
}
+// ### 2 B DOCUMENTED!
+static bool createFilteredRHInfos(
+ QList<ResultHistoryInfo *> *rhInfos, int *totCandidates, const QString &reqName,
+ QSqlDatabase *database, const QStringList &testCaseFilter, const QStringList &metricFilter,
+ const QStringList &platformFilter, const QStringList &hostFilter,
+ const QStringList &branchFilter, const int medianWinSize, QString *error,
+ const bool debugPrint = false)
+{
+ QSqlQuery query(*database);
+
+ // Get the initial list of candidate result histories by getting the BM context ID and metric
+ // for all result histories that match the logical OR-values.
+ // NOTE: Results measuring the "events" metric are skipped for now since
+ // they seem to be of little value when computing the performance index (they are
+ // typically integers close to (and often at) zero) ...
+
+ if (debugPrint)
+ fprintf(stderr, "fetching list of relevant result histories ... ");
+
+ QString query_s =
+ "SELECT bmcontext.id, metric.name"
+ " FROM bmcontext, benchmark, context, metric, platform, host, branch"
+ " WHERE benchmarkId = benchmark.id"
+ " AND contextId = context.id"
+ " AND metricId = metric.id"
+ " AND platformId = platform.id"
+ " AND hostId = host.id"
+ " AND branchId = branch.id"
+ " AND metric.name != 'events'";
+
+ if (!testCaseFilter.isEmpty()) {
+ query_s += " AND (";
+ for (int i = 0; i < testCaseFilter.size(); ++i)
+ query_s += QString("%1benchmark.testCase = '%2'")
+ .arg((i > 0) ? " OR " : "")
+ .arg(testCaseFilter.at(i));
+ query_s += ")";
+ }
+
+ if (!metricFilter.isEmpty()) {
+ query_s += " AND (";
+ for (int i = 0; i < metricFilter.size(); ++i)
+ query_s +=
+ QString("%1metric.name = '%2'").arg((i > 0) ? " OR " : "").arg(metricFilter.at(i));
+ query_s += ")";
+ }
+
+ if (!platformFilter.isEmpty()) {
+ query_s += " AND (";
+ for (int i = 0; i < platformFilter.size(); ++i)
+ query_s += QString("%1platform.name = '%2'")
+ .arg((i > 0) ? " OR " : "")
+ .arg(platformFilter.at(i));
+ query_s += ")";
+ }
+
+ if (!hostFilter.isEmpty()) {
+ query_s += " AND (";
+ for (int i = 0; i < hostFilter.size(); ++i)
+ query_s += QString("%1host.name = '%2'")
+ .arg((i > 0) ? " OR " : "")
+ .arg(hostFilter.at(i));
+ query_s += ")";
+ }
+
+ if (!branchFilter.isEmpty()) {
+ query_s += " AND (";
+ for (int i = 0; i < branchFilter.size(); ++i)
+ query_s += QString("%1(branch.gitRepo || branch.gitBranch) = '%2'")
+ .arg((i > 0) ? " OR " : "")
+ .arg(branchFilter.at(i));
+ query_s += ")";
+ }
+
+ query_s += ";";
+
+ if (!query.exec(query_s)) {
+ *error = errorReply(query, reqName, "failed to get bmcontext IDs (exec() failed)");
+ return false;
+ }
+ QList<int> bmcontextIds;
+ QStringList metrics;
+ while (query.next()) {
+ bmcontextIds += query.value(0).toInt();
+ metrics += query.value(1).toString();
+ }
+ if (debugPrint)
+ fprintf(stderr, "done\n");
+
+ if (bmcontextIds.isEmpty()) {
+ *error = errorReply(reqName, "no matching result histories found");
+ return false;
+ }
+
+
+ // Loop over initial result history candidates ...
+ for (int i = 0; i < bmcontextIds.size(); ++i) {
+
+ if (!(i % qMax(1, (bmcontextIds.size() / 20)))) {
+ if (debugPrint)
+ fprintf(
+ stderr, "fetching result histories (%0.0f%% done)\r",
+ 100 * (qreal(i) / bmcontextIds.size()));
+ }
+
+ // Get the time series ...
+
+ if (!query.exec(
+ QString(
+ "SELECT timestamp, value FROM result "
+ "WHERE bmcontextId=%1 "
+ "ORDER BY timestamp ASC;")
+ .arg(bmcontextIds.at(i)))) {
+ *error = errorReply(query, reqName, "failed to get time series (exec() failed)");
+ return false;
+ }
+ QList<int> timestamps;
+ QList<qreal> values;
+ while (query.next()) {
+ timestamps += query.value(0).toInt();
+ values += query.value(1).toDouble();
+ }
+
+ ResultHistoryInfo *rhInfo =
+ new ResultHistoryInfo(
+ bmcontextIds.at(i), timestamps, values, medianWinSize, metrics.at(i));
+ rhInfos->append(rhInfo);
+ }
+
+ *totCandidates = bmcontextIds.size();
+
+ if (debugPrint)
+ fprintf(stderr, "fetching result histories (100%% done)\n");
+
+ return true;
+}
+
// --- IndexGetValues ---
QByteArray BMRequest_IndexGetValues::toRequestBuffer(QString *)
@@ -4895,7 +5035,7 @@ QByteArray BMRequest_IndexGetValues::toReplyBuffer()
static int calls_ = 0;
fprintf(
stderr,
- "BMRequest_IndexGetValues2::toReplyBuffer() called (%d), cacheKey: >%s<\n",
+ "BMRequest_IndexGetValues::toReplyBuffer() called (%d), cacheKey: >%s<\n",
calls_++, argsElem.attributeNode("cacheKey").value().toLatin1().data());
}
if (ok) {
@@ -4949,140 +5089,22 @@ QByteArray BMRequest_IndexGetValues::toReplyBuffer()
branchFilter.append(branchNodes.at(i).toElement().attributeNode("name").value());
- QSqlQuery *query;
-
- // Get the initial list of candidate result histories by getting the BM context ID and metric
- // for all result histories that match the logical OR-values.
- // NOTE: Results measuring the "events" metric are skipped for now since
- // they seem to be of little value when computing the performance index (they are
- // typically integers close to (and often at) zero) ...
- query = createQuery();
-
- if (debugPrint)
- fprintf(stderr, "fetching list of relevant result histories ... ");
-
- QString query_s =
- "SELECT bmcontext.id, metric.name"
- " FROM bmcontext, benchmark, context, metric, platform, host, branch"
- " WHERE benchmarkId = benchmark.id"
- " AND contextId = context.id"
- " AND metricId = metric.id"
- " AND platformId = platform.id"
- " AND hostId = host.id"
- " AND branchId = branch.id"
- " AND metric.name != 'events'";
-
- if (!testCaseFilter.isEmpty()) {
- query_s += " AND (";
- for (int i = 0; i < testCaseFilter.size(); ++i)
- query_s += QString("%1benchmark.testCase = '%2'")
- .arg((i > 0) ? " OR " : "")
- .arg(testCaseFilter.at(i));
- query_s += ")";
- }
-
- if (!metricFilter.isEmpty()) {
- query_s += " AND (";
- for (int i = 0; i < metricFilter.size(); ++i)
- query_s +=
- QString("%1metric.name = '%2'").arg((i > 0) ? " OR " : "").arg(metricFilter.at(i));
- query_s += ")";
- }
-
- if (!platformFilter.isEmpty()) {
- query_s += " AND (";
- for (int i = 0; i < platformFilter.size(); ++i)
- query_s += QString("%1platform.name = '%2'")
- .arg((i > 0) ? " OR " : "")
- .arg(platformFilter.at(i));
- query_s += ")";
- }
-
- if (!hostFilter.isEmpty()) {
- query_s += " AND (";
- for (int i = 0; i < hostFilter.size(); ++i)
- query_s += QString("%1host.name = '%2'")
- .arg((i > 0) ? " OR " : "")
- .arg(hostFilter.at(i));
- query_s += ")";
- }
-
- if (!branchFilter.isEmpty()) {
- query_s += " AND (";
- for (int i = 0; i < branchFilter.size(); ++i)
- query_s += QString("%1(branch.gitRepo || branch.gitBranch) = '%2'")
- .arg((i > 0) ? " OR " : "")
- .arg(branchFilter.at(i));
- query_s += ")";
- }
-
- query_s += ";";
-
- if (!query->exec(query_s)) {
- deleteQuery(query);
- return xmlConvert(
- errorReply(*query, name(), "failed to get bmcontext IDs (exec() failed)"));
- }
- QList<int> bmcontextIds;
- QStringList metrics;
- while (query->next()) {
- bmcontextIds += query->value(0).toInt();
- metrics += query->value(1).toString();
- }
- deleteQuery(query);
- if (debugPrint)
- fprintf(stderr, "done\n");
-
- if (bmcontextIds.isEmpty())
- return xmlConvert(errorReply(name(), "no matching result histories found"));
-
-
QList<ResultHistoryInfo *> rhInfos;
+ int totRHCandidates;
+ if (!createFilteredRHInfos(
+ &rhInfos, &totRHCandidates, name(), database, testCaseFilter, metricFilter,
+ platformFilter, hostFilter, branchFilter, medianWinSize, &error, debugPrint)) {
- // Loop over initial result history candidates ...
- for (int i = 0; i < bmcontextIds.size(); ++i) {
-
- if (!(i % qMax(1, (bmcontextIds.size() / 20)))) {
- if (debugPrint)
- fprintf(
- stderr, "fetching result histories (%0.0f%% done)\r",
- 100 * (qreal(i) / bmcontextIds.size()));
- }
-
- // Get the time series ...
- query = createQuery();
-
- if (!query->exec(
- QString(
- "SELECT timestamp, value FROM result "
- "WHERE bmcontextId=%1 "
- "ORDER BY timestamp ASC;")
- .arg(bmcontextIds.at(i)))) {
- deleteQuery(query);
- return xmlConvert(
- errorReply(*query, name(), "failed to get time series (exec() failed)"));
- }
- QList<int> timestamps;
- QList<qreal> values;
- while (query->next()) {
- timestamps += query->value(0).toInt();
- values += query->value(1).toDouble();
- }
- deleteQuery(query);
+ // Free dynamic memory ...
+ for (int i = 0; i < rhInfos.size(); ++i)
+ delete rhInfos.at(i);
- ResultHistoryInfo *rhInfo =
- new ResultHistoryInfo(
- bmcontextIds.at(i), timestamps, values, medianWinSize, metrics.at(i));
- rhInfos.append(rhInfo);
+ return xmlConvert(error);
}
- if (debugPrint)
- fprintf(stderr, "fetching result histories (100%% done)\n");
-
reply = QString("<reply type=\"%1\" >").arg(name());
-
int nonPositiveRH = 0;
int baseValuePos;
@@ -5092,12 +5114,10 @@ QByteArray BMRequest_IndexGetValues::toReplyBuffer()
QList<int> evalTimestamps;
IndexAlgorithm1 index(rhInfos, baseTimestamp, &evalTimestamps, &nonPositiveRH);
- if (!index.isValid()) {
+ if (!index.isValid())
return xmlConvert(
errorReply(
name(), QString("failed to compute index (1): %1").arg(index.invalidReason())));
- }
-
QList<qreal> indexValues;
QList<int> contrCounts;
@@ -5108,10 +5128,9 @@ QByteArray BMRequest_IndexGetValues::toReplyBuffer()
if (!index.computeValues(
&indexValues, &baseValuePos, &contrCounts, &error_, &topContr, topContrLimit,
- dataQualityStats ? &dqStats : static_cast<DataQualityStats *>(0))) {
+ dataQualityStats ? &dqStats : static_cast<DataQualityStats *>(0)))
return xmlConvert(
errorReply(name(), QString("failed to compute index (2): %1").arg(error_)));
- }
// Add index values to reply ...
Q_ASSERT(indexValues.size() == evalTimestamps.size());
@@ -5191,14 +5210,15 @@ QByteArray BMRequest_IndexGetValues::toReplyBuffer()
reply += QString(
"<args baseTimestamp=\"%1\" medianWinSize=\"%2\" totalRH=\"%3\" nonPositiveRH=\"%4\" "
"baseValuePos=\"%5\" cacheKey=\"%6\" /></reply>")
- .arg(baseTimestamp).arg(medianWinSize).arg(bmcontextIds.size()).arg(nonPositiveRH)
+ .arg(baseTimestamp).arg(medianWinSize).arg(totRHCandidates).arg(nonPositiveRH)
.arg(baseValuePos).arg(cacheKey_);
Cache::instance()->put(cacheKey_, reply);
return xmlConvert(reply);
}
-void BMRequest_IndexGetValues::appendToFilterTable(
+// ### 2 B DOCUMENTED!
+static void appendToFilterTable(
const QStringList &filter, const QString &filterName, QString *reply)
{
reply->append(
@@ -7200,3 +7220,415 @@ void BMRequest_GetIXHistories::handleReply_Image(const QStringList &args) const
"failed to create plot for main index contributors: %1").arg(error_));
}
}
+
+
+// --- ASFStatsGetValues ---
+QByteArray BMRequest_ASFStatsGetValues::toRequestBuffer(QString *)
+{
+ QString request =
+ QString(
+ "<request type=\"%1\"><args medianWinSize=\"%2\" cacheKey=\"%3\" "
+ "fromTimestamp=\"%4\" toTimestamp=\"%5\" diffTol=\"%6\" stabTol=\"%7\" "
+ "sfTol=\"%8\" lfTol=\"%9\" maxLDTol=\"%10\" />")
+ .arg(name())
+ .arg(medianWinSize)
+ .arg(cacheKey)
+ .arg(fromTimestamp)
+ .arg(toTimestamp)
+ .arg(diffTol)
+ .arg(stabTol)
+ .arg(sfTol)
+ .arg(lfTol)
+ .arg(maxLDTol);
+
+ for (int i = 0; i < testCaseFilter.size(); ++i)
+ request += QString("<testCase name=\"%1\" />").arg(testCaseFilter.at(i));
+
+ for (int i = 0; i < metricFilter.size(); ++i)
+ request += QString("<metric name=\"%1\" />").arg(metricFilter.at(i));
+
+ for (int i = 0; i < platformFilter.size(); ++i)
+ request += QString("<platform name=\"%1\" />").arg(platformFilter.at(i));
+
+ for (int i = 0; i < hostFilter.size(); ++i)
+ request += QString("<host name=\"%1\" />").arg(hostFilter.at(i));
+
+ for (int i = 0; i < branchFilter.size(); ++i)
+ request += QString("<branch name=\"%1\" />").arg(branchFilter.at(i));
+
+ request += "</request>";
+
+ return xmlConvert(request);
+}
+
+QByteArray BMRequest_ASFStatsGetValues::toReplyBuffer()
+{
+ QDomElement argsElem = doc.elementsByTagName("args").at(0).toElement();
+
+ QString error;
+ QString reply;
+ bool ok;
+
+ const bool debugPrint = false;
+
+ // Check if a cached reply can be returned ...
+ int cacheKey_ = argsElem.attributeNode("cacheKey").value().toInt(&ok);
+ if (debugPrint) {
+ static int calls_ = 0;
+ fprintf(
+ stderr,
+ "BMRequest_ASFStatsGetValues::toReplyBuffer() called (%d), cacheKey: >%s<\n",
+ calls_++, argsElem.attributeNode("cacheKey").value().toLatin1().data());
+ }
+ if (ok) {
+ reply = Cache::instance()->get(cacheKey_);
+ if (!reply.isEmpty()) {
+ if (debugPrint)
+ fprintf(stderr, "returning cached reply (key = %d)\n", cacheKey_);
+ return xmlConvert(reply);
+ }
+ }
+
+ // Compute from scratch ...
+ if (debugPrint)
+ fprintf(stderr, "computing from scratch ...\n");
+
+ // Get from timestamp ...
+ fromTimestamp = argsElem.attributeNode("fromTimestamp").value().toInt(&ok);
+ Q_ASSERT(ok);
+
+ // Get to timestamp ...
+ toTimestamp = argsElem.attributeNode("toTimestamp").value().toInt(&ok);
+ Q_ASSERT(ok);
+
+ // Get median window size ...
+ medianWinSize = argsElem.attributeNode("medianWinSize").value().toInt(&ok);
+ Q_ASSERT(ok);
+
+ // Get tolerance values ...
+ diffTol = argsElem.attributeNode("diffTol").value().toDouble(&ok);
+ Q_ASSERT(ok);
+ stabTol = argsElem.attributeNode("stabTol").value().toInt(&ok);
+ Q_ASSERT(ok);
+ sfTol = argsElem.attributeNode("sfTol").value().toDouble(&ok);
+ Q_ASSERT(ok);
+ lfTol = argsElem.attributeNode("lfTol").value().toDouble(&ok);
+ Q_ASSERT(ok);
+ maxLDTol = argsElem.attributeNode("maxLDTol").value().toDouble(&ok);
+ Q_ASSERT(ok);
+
+ // Get filters ...
+ QDomNodeList testCaseNodes = doc.elementsByTagName("testCase");
+ for (int i = 0; i < testCaseNodes.size(); ++i)
+ testCaseFilter.append(testCaseNodes.at(i).toElement().attributeNode("name").value());
+
+ QDomNodeList metricNodes = doc.elementsByTagName("metric");
+ for (int i = 0; i < metricNodes.size(); ++i)
+ metricFilter.append(metricNodes.at(i).toElement().attributeNode("name").value());
+
+ QDomNodeList platformNodes = doc.elementsByTagName("platform");
+ for (int i = 0; i < platformNodes.size(); ++i)
+ platformFilter.append(platformNodes.at(i).toElement().attributeNode("name").value());
+
+ QDomNodeList hostNodes = doc.elementsByTagName("host");
+ for (int i = 0; i < hostNodes.size(); ++i)
+ hostFilter.append(hostNodes.at(i).toElement().attributeNode("name").value());
+
+ QDomNodeList branchNodes = doc.elementsByTagName("branch");
+ for (int i = 0; i < branchNodes.size(); ++i)
+ branchFilter.append(branchNodes.at(i).toElement().attributeNode("name").value());
+
+
+ QList<ResultHistoryInfo *> rhInfos;
+ int totRHCandidates;
+ if (!createFilteredRHInfos(
+ &rhInfos, &totRHCandidates, name(), database, testCaseFilter, metricFilter,
+ platformFilter, hostFilter, branchFilter, medianWinSize, &error, debugPrint)) {
+
+ // Free dynamic memory ...
+ for (int i = 0; i < rhInfos.size(); ++i)
+ delete rhInfos.at(i);
+
+ return xmlConvert(error);
+ }
+
+
+ reply = QString("<reply type=\"%1\" >").arg(name());
+
+ ASFStats asfStats(diffTol, stabTol, sfTol, lfTol, maxLDTol, fromTimestamp, toTimestamp);
+ ASFStats::UnstableStats usStats;
+ int regressed = 0;
+ int unchanged = 0;
+ int improved = 0;
+ asfStats.compute(rhInfos, &usStats, &regressed, &unchanged, &improved);
+
+ // Add stats to reply ...
+ reply += QString(
+ "<stats regressed=\"%1\" unchanged=\"%2\" improved=\"%3\" unstable=\"%4\" "
+ "usZeroCount=\"%5\" usLowFromPosCount=\"%6\" usLowSFCount=\"%7\" usLowLFCount=\"%8\" "
+ "usHighMaxLDCount=\"%9\" />")
+ .arg(regressed).arg(unchanged).arg(improved)
+ .arg(usStats.unstable.count(true))
+ .arg(usStats.zeroCount)
+ .arg(usStats.lowFromPosCount)
+ .arg(usStats.lowSFCount)
+ .arg(usStats.lowLFCount)
+ .arg(usStats.highMaxLDCount);
+
+ // Add IDs of unstable result histories to reply ...
+ reply += "<unstableIDs>";
+ for (int i = 0; i < usStats.unstable.size(); ++i)
+ if (usStats.unstable.testBit(i))
+ reply += QString("%1 ").arg(rhInfos.at(i)->bmcontextId());
+ reply += "</unstableIDs>";
+
+ //----------------------------------------------------------------------------------------
+
+ for (int i = 0; i < testCaseFilter.size(); ++i)
+ reply += QString("<testCase name=\"%1\" />").arg(testCaseFilter.at(i));
+
+ for (int i = 0; i < metricFilter.size(); ++i)
+ reply += QString("<metric name=\"%1\" />").arg(metricFilter.at(i));
+
+ for (int i = 0; i < platformFilter.size(); ++i)
+ reply += QString("<platform name=\"%1\" />").arg(platformFilter.at(i));
+
+ for (int i = 0; i < hostFilter.size(); ++i)
+ reply += QString("<host name=\"%1\" />").arg(hostFilter.at(i));
+
+ for (int i = 0; i < branchFilter.size(); ++i)
+ reply += QString("<branch name=\"%1\" />").arg(branchFilter.at(i));
+
+ // Finalize the reply and cache it ...
+ cacheKey_ = Cache::instance()->nextId();
+ reply += QString(
+ "<args medianWinSize=\"%1\" cacheKey=\"%2\" "
+ "fromTimestamp=\"%3\" toTimestamp=\"%4\" diffTol=\"%5\" stabTol=\"%6\" "
+ "sfTol=\"%7\" lfTol=\"%8\" maxLDTol=\"%9\" /></reply>")
+ .arg(medianWinSize).arg(cacheKey_).arg(fromTimestamp).arg(toTimestamp).arg(diffTol)
+ .arg(stabTol).arg(sfTol).arg(lfTol).arg(maxLDTol);
+ Cache::instance()->put(cacheKey_, reply);
+
+ // Free dynamic memory ...
+ for (int i = 0; i < rhInfos.size(); ++i)
+ delete rhInfos.at(i);
+
+ return xmlConvert(reply);
+}
+
+void BMRequest_ASFStatsGetValues::handleReply_HTML(const QStringList &args) const
+{
+ QString error;
+
+ error = doc.elementsByTagName("reply").at(0).toElement().attributeNode("error").value();
+ if (!error.isEmpty()) {
+ BMMisc::printHTMLErrorPage(
+ QString("failed to create details page: error evaluating ASF stats: %1").arg(error));
+ return;
+ }
+
+ QStringList optValues;
+
+ // Get server ...
+ QString server;
+ if (BMMisc::getOption(args, "-server", &optValues, 1, 0, &error)) {
+ server = optValues.first().trimmed();
+ } else {
+ BMMisc::printHTMLErrorPage("-server option not found");
+ return;
+ }
+
+ // Get style sheet ...
+ QString styleSheet;
+ if (BMMisc::getOption(args, "-stylesheet", &optValues, 1, 0, &error)) {
+ styleSheet = optValues.first().trimmed();
+ } else {
+ BMMisc::printHTMLErrorPage("-stylesheet option not found");
+ return;
+ }
+
+ // *** Header ***
+ QString reply = QString("<html>\n<head>\n");
+ reply += QString("<link rel=\"stylesheet\" type=\"text/css\" href=\"%1\" />\n").arg(styleSheet);
+ reply += QString("</head>\n<body>\n");
+
+ reply += "<span style=\"font-size:18\">ASF Stats (for scorecard)</span>";
+
+ reply += "<br /><br /><table><tr><td style=\"background-color:#eeeeee\">";
+
+ reply += "This page shows statistics suitable for presentation on the ASF Scorecard ";
+ reply += "<span style=\"color:red\"> (More documentation to follow soon!)</span>.";
+
+ reply += "</td></tr></table>\n";
+
+ QDomElement argsElem = doc.elementsByTagName("args").at(0).toElement();
+
+ bool ok;
+
+
+ reply += "\n<br /><br /><table style=\"border:0px\">\n";
+ reply += "<tr>\n";
+
+ // Left main table ...
+
+ reply += "<td style=\"border:0px\">\n";
+
+ reply += "<fieldset><legend style=\"font-size:12\">Parameters</legend>";
+
+ // 'Misc' table ...
+ fromTimestamp = argsElem.attributeNode("fromTimestamp").value().toInt(&ok);
+ Q_ASSERT(ok);
+ toTimestamp = argsElem.attributeNode("toTimestamp").value().toInt(&ok);
+ Q_ASSERT(ok);
+ medianWinSize = argsElem.attributeNode("medianWinSize").value().toInt(&ok);
+ Q_ASSERT(ok);
+ diffTol = argsElem.attributeNode("diffTol").value().toDouble(&ok);
+ Q_ASSERT(ok);
+ stabTol = argsElem.attributeNode("stabTol").value().toInt(&ok);
+ Q_ASSERT(ok);
+ sfTol = argsElem.attributeNode("sfTol").value().toInt(&ok);
+ Q_ASSERT(ok);
+ lfTol = argsElem.attributeNode("lfTol").value().toInt(&ok);
+ Q_ASSERT(ok);
+ maxLDTol = argsElem.attributeNode("maxLDTol").value().toDouble(&ok);
+ Q_ASSERT(ok);
+
+ QDateTime dateTime;
+ reply += "\n<br /><table style=\"text-align:right\">\n";
+ dateTime.setTime_t(fromTimestamp);
+ reply += QString("<tr><td>From time:</td><td>%1</td></tr>\n").arg(dateTime.toString());
+ dateTime.setTime_t(toTimestamp);
+ reply += QString("<tr><td>To time:</td><td>%1</td></tr>\n").arg(dateTime.toString());
+ reply += QString("<tr><td>Median window size:</td><td>%1</td></tr>\n").arg(medianWinSize);
+ reply += QString("<tr><td>Difference tolerance (%):</td><td>%1</td></tr>\n").arg(diffTol);
+ reply += QString("<tr><td>Stability tolerance:</td><td>%1</td></tr>\n").arg(stabTol);
+ reply += QString("<tr><td>Stability fraction tolerance (%):</td><td>%1</td></tr>\n").arg(sfTol);
+ reply += QString("<tr><td>Unique level fraction tolerance (%):</td><td>%1</td></tr>\n")
+ .arg(lfTol);
+ reply += QString("<tr><td>Maximim level distance tolerance (%):</td><td>%1</td></tr>\n")
+ .arg(maxLDTol);
+ reply += "</table>\n";
+
+
+ // 'Filters' table ...
+ QDomNodeList testCaseNodes = doc.elementsByTagName("testCase");
+ for (int i = 0; i < testCaseNodes.size(); ++i)
+ testCaseFilter.append(testCaseNodes.at(i).toElement().attributeNode("name").value());
+
+ QDomNodeList metricNodes = doc.elementsByTagName("metric");
+ for (int i = 0; i < metricNodes.size(); ++i)
+ metricFilter.append(metricNodes.at(i).toElement().attributeNode("name").value());
+
+ QDomNodeList platformNodes = doc.elementsByTagName("platform");
+ for (int i = 0; i < platformNodes.size(); ++i)
+ platformFilter.append(platformNodes.at(i).toElement().attributeNode("name").value());
+
+ QDomNodeList hostNodes = doc.elementsByTagName("host");
+ for (int i = 0; i < hostNodes.size(); ++i)
+ hostFilter.append(hostNodes.at(i).toElement().attributeNode("name").value());
+
+ QDomNodeList branchNodes = doc.elementsByTagName("branch");
+ for (int i = 0; i < branchNodes.size(); ++i)
+ branchFilter.append(branchNodes.at(i).toElement().attributeNode("name").value());
+
+ reply += "\n<br /><b>Filters:</b><table>\n";
+ appendToFilterTable(testCaseFilter, "Test case", &reply);
+ appendToFilterTable(metricFilter, "Metric", &reply);
+ appendToFilterTable(platformFilter, "Platform", &reply);
+ appendToFilterTable(hostFilter, "Host", &reply);
+ appendToFilterTable(branchFilter, "Branch", &reply);
+ reply += "</table>";
+
+ reply += "</fieldset>";
+
+ reply += "</td>\n";
+
+ // Right main table ...
+
+ reply += "<td style=\"border:0px\">\n";
+
+ // 'Stats' table ...
+
+ QDomNodeList statsNodes = doc.elementsByTagName("stats");
+ QDomElement statsElem = statsNodes.at(0).toElement();
+
+ reply += "<fieldset><legend style=\"font-size:12\">Statistics</legend>";
+
+ const int regressed = statsElem.attributeNode("regressed").value().toInt(&ok);
+ Q_ASSERT(ok);
+ const int unchanged = statsElem.attributeNode("unchanged").value().toInt(&ok);
+ Q_ASSERT(ok);
+ const int improved = statsElem.attributeNode("improved").value().toInt(&ok);
+ Q_ASSERT(ok);
+ const int unstable = statsElem.attributeNode("unstable").value().toInt(&ok);
+ Q_ASSERT(ok);
+ const int total = regressed + unchanged + improved + unstable;
+ const int usZeroCount = statsElem.attributeNode("usZeroCount").value().toInt(&ok);
+ Q_ASSERT(ok);
+ const int usLowFromPosCount =
+ statsElem.attributeNode("usLowFromPosCount").value().toInt(&ok);
+ Q_ASSERT(ok);
+ const int usLowSFCount = statsElem.attributeNode("usLowSFCount").value().toInt(&ok);
+ Q_ASSERT(ok);
+ const int usLowLFCount = statsElem.attributeNode("usLowLFCount").value().toInt(&ok);
+ Q_ASSERT(ok);
+ const int usHighMaxLDCount = statsElem.attributeNode("usHighMaxLDCount").value().toInt(&ok);
+ Q_ASSERT(ok);
+
+ const QString usColor("#ffdddd");
+
+ reply += "<br /><table style=\"text-align:right\">\n";
+
+ reply += QString("<tr><td><b>Total:</b></td><td><b>%1</b></td><td><b>100.00 %</b></td></tr>\n")
+ .arg(total);
+ if (total > 0) {
+ reply += QString("<tr><td>Regressed:</td><td>%1</td><td>%2 %</td></tr>\n")
+ .arg(regressed).arg(QString().setNum(100 * (regressed / qreal(total)), 'f', 2));
+ reply += QString("<tr><td>Unchanged:</td><td>%1</td><td>%2 %</td></tr>\n")
+ .arg(unchanged).arg(QString().setNum(100 * (unchanged / qreal(total)), 'f', 2));
+ reply += QString("<tr><td>Improved:</td><td>%1</td><td>%2 %</td></tr>\n")
+ .arg(improved).arg(QString().setNum(100 * (improved / qreal(total)), 'f', 2));
+ reply += QString(
+ "<tr style=\"background-color:%1\"><td>Unstable:</td><td>%2</td>"
+ "<td>%3 %</td></tr>\n")
+ .arg(usColor).arg(unstable)
+ .arg(QString().setNum(100 * (unstable / qreal(total)), 'f', 2));
+ }
+
+ reply += "</table>\n";
+
+ if (unstable > 0) {
+ reply += "<br /><b>Categories of instabilities:</b>";
+ reply += QString("<table style=\"text-align:right; background-color:%1\">\n").arg(usColor);
+ reply += QString(
+ "<tr><td>Zero existence:</td><td>%1</td><td>%2 %</td></tr>\n")
+ .arg(usZeroCount).arg(QString().setNum(100 * (usZeroCount / qreal(unstable)), 'f', 2));
+ reply += QString("<tr><td>No data before 'from' time:</td><td>%1</td><td>%2 %</td></tr>\n")
+ .arg(usLowFromPosCount)
+ .arg(QString().setNum(100 * (usLowFromPosCount / qreal(unstable)), 'f', 2));
+ reply += QString(
+ "<tr><td>Stable fraction too low:</td><td>%1</td><td>%2 %</td></tr>\n")
+ .arg(usLowSFCount)
+ .arg(QString().setNum(100 * (usLowSFCount / qreal(unstable)), 'f', 2));
+ reply += QString(
+ "<tr><td>Unique level fraction too low:</td><td>%1</td><td>%2 %</td></tr>\n")
+ .arg(usLowLFCount)
+ .arg(QString().setNum(100 * (usLowLFCount / qreal(unstable)), 'f', 2));
+ reply += QString(
+ "<tr><td>Max level difference too high:</td><td>%1</td><td>%2 %</td></tr>\n")
+ .arg(usHighMaxLDCount)
+ .arg(QString().setNum(100 * (usHighMaxLDCount / qreal(unstable)), 'f', 2));
+ reply += "</table>\n";
+ }
+
+ reply += "</fieldset>";
+
+ reply += "</td>\n";
+
+ reply += "</tr>\n";
+ reply += "<table>\n";
+
+ reply += "</body></html>";
+
+ BMMisc::printHTMLOutput(reply);
+}
diff --git a/src/bm/bmrequest.h b/src/bm/bmrequest.h
index 2b97b65..8d90d88 100644
--- a/src/bm/bmrequest.h
+++ b/src/bm/bmrequest.h
@@ -598,9 +598,6 @@ private:
void handleReply_JSON(const QStringList &args) const { Q_UNUSED(args); /* 2 B DONE! */ }
void handleReply_HTML(const QStringList &args) const;
void handleReply_Image(const QStringList &args) const;
-
- static void appendToFilterTable(
- const QStringList &filter, const QString &filterName, QString *reply);
};
// ### 2 B DOCUMENTED (in bmclient.html and bmproto.html)
@@ -747,4 +744,45 @@ private:
void handleReply_Image(const QStringList &args) const;
};
+// ### 2 B DOCUMENTED (in bmclient.html and bmproto.html)
+class BMRequest_ASFStatsGetValues : public BMRequest
+{
+public:
+ BMRequest_ASFStatsGetValues(
+ const int medianWinSize, const QString &cacheKey, const QStringList &testCaseFilter,
+ const QStringList &metricFilter, const QStringList &platformFilter,
+ const QStringList &hostFilter, const QStringList &branchFilter,
+ const int fromTimestamp, const int toTimestamp, const qreal diffTol, const int stabTol,
+ const qreal sfTol, const qreal lfTol, const qreal maxLDTol)
+ : medianWinSize(medianWinSize), cacheKey(cacheKey), testCaseFilter(testCaseFilter)
+ , metricFilter(metricFilter), platformFilter(platformFilter), hostFilter(hostFilter)
+ , branchFilter(branchFilter), fromTimestamp(fromTimestamp), toTimestamp(toTimestamp)
+ , diffTol(diffTol), stabTol(stabTol), sfTol(sfTol), lfTol(lfTol), maxLDTol(maxLDTol)
+ {}
+ BMRequest_ASFStatsGetValues(const QDomDocument &doc) : BMRequest(doc) {}
+
+private:
+ mutable int medianWinSize;
+ QString cacheKey;
+ mutable QStringList testCaseFilter;
+ mutable QStringList metricFilter;
+ mutable QStringList platformFilter;
+ mutable QStringList hostFilter;
+ mutable QStringList branchFilter;
+ mutable int fromTimestamp;
+ mutable int toTimestamp;
+ mutable qreal diffTol;
+ mutable qreal stabTol;
+ mutable qreal sfTol;
+ mutable qreal lfTol;
+ mutable qreal maxLDTol;
+
+ QString name() const { return "ASFStatsGetValues"; }
+ QByteArray toRequestBuffer(QString *error);
+ QByteArray toReplyBuffer();
+ void handleReply_Raw(const QStringList &args) const { Q_UNUSED(args); /* 2 B DONE! */ };
+ void handleReply_JSON(const QStringList &args) const { Q_UNUSED(args); /* 2 B DONE! */ }
+ void handleReply_HTML(const QStringList &args) const;
+};
+
#endif // BMREQUEST_H
diff --git a/src/bm/resulthistoryinfo.cpp b/src/bm/resulthistoryinfo.cpp
index cb18924..0e2d23b 100644
--- a/src/bm/resulthistoryinfo.cpp
+++ b/src/bm/resulthistoryinfo.cpp
@@ -111,16 +111,6 @@ void ResultHistoryInfo::markOutliers() const
}
// ### 2 B DOCUMENTED!
-static void appendMaxESSStats(int seqSize, int *total, int *stable, int stabTolerance)
-{
- if (seqSize <= 0)
- return;
- (*total)++;
- if (seqSize >= stabTolerance)
- (*stable)++;
-}
-
-// ### 2 B DOCUMENTED!
bool ResultHistoryInfo::equal(int i, int j, int diffTolerance) const
{
if ((i < 0) || (j < 0) || (i >= values.size()) || (j >= values.size()))
@@ -142,8 +132,49 @@ bool ResultHistoryInfo::equal(int i, int j, int diffTolerance) const
}
// ### 2 B DOCUMENTED!
+void ResultHistoryInfo::startSubsequence(
+ int pos, qreal diffTolerance, QMap<int, int> *uniqueLevels, qreal *minLevel,
+ qreal *maxLevel, bool first) const
+{
+ Q_ASSERT(pos >= 0);
+ Q_ASSERT(pos < values.size());
+
+ const qreal v = values.at(pos);
+
+ if (first) {
+ *minLevel = *maxLevel = v;
+ } else {
+ *minLevel = qMin(*minLevel, v);
+ *maxLevel = qMax(*maxLevel, v);
+ }
+
+ QMap<int, int>::iterator it;
+ for (it = uniqueLevels->begin(); it != uniqueLevels->end(); ++it)
+ if (equal(pos, it.key(), diffTolerance))
+ break;
+
+ if (it == uniqueLevels->end()) {
+ // Insert new unique level ...
+ uniqueLevels->insert(pos, 1);
+ } else {
+ // Add to existing unique level ...
+ ++(it.value());
+ }
+}
+
+// ### 2 B DOCUMENTED!
+void ResultHistoryInfo::endSubsequence(int seqSize, int *total, int *stable, int stabTolerance)
+{
+ if (seqSize <= 0)
+ return;
+ (*total)++;
+ if (seqSize >= stabTolerance)
+ (*stable)++;
+}
+
+// ### 2 B DOCUMENTED!
void ResultHistoryInfo::computeMaxESSStats(
- qreal diffTolerance, int stabTolerance, int *total, int *stable)
+ qreal diffTolerance, int stabTolerance, int *total, int *stable) const
{
*total = *stable = 0;
@@ -154,7 +185,7 @@ void ResultHistoryInfo::computeMaxESSStats(
if (isOutlier(i)) {
if (i == values.size() - 1)
- appendMaxESSStats(seqSize, total, stable, stabTolerance);
+ endSubsequence(seqSize, total, stable, stabTolerance);
} else {
@@ -163,14 +194,52 @@ void ResultHistoryInfo::computeMaxESSStats(
if (basePos == -1) {
basePos = i;
} else if (!equal(basePos, i, diffTolerance)) {
- appendMaxESSStats(seqSize, total, stable, stabTolerance);
+ endSubsequence(seqSize, total, stable, stabTolerance);
seqSize = 1;
basePos = i;
}
if (i == values.size() - 1)
- appendMaxESSStats(seqSize, total, stable, stabTolerance);
+ endSubsequence(seqSize, total, stable, stabTolerance);
+ }
+
+ }
+}
+
+// ### 2 B DOCUMENTED!
+void ResultHistoryInfo::computeStabilityStats(
+ qreal diffTolerance, int stabTolerance, bool *zerosFound, int *total, int *stable,
+ int *uniqueLevels, qreal *minLevel, qreal *maxLevel) const
+{
+ *zerosFound = false;
+ *total = *stable = *uniqueLevels = 0;
+ *minLevel = *maxLevel = 0;
+
+ QMap<int, int> uniqueLevels_; // <pos, count>
+
+ int basePos = -1;
+ int seqSize = 0;
+
+ for (int i = 0; i < values.size(); ++i) {
+
+ ++seqSize;
+
+ if ((!(*zerosFound)) && (values.at(i) == 0.0))
+ *zerosFound = true;
+
+ if (basePos == -1) {
+ basePos = i;
+ startSubsequence(basePos, diffTolerance, &uniqueLevels_, minLevel, maxLevel, true);
+ } else if (!equal(basePos, i, diffTolerance)) {
+ endSubsequence(seqSize, total, stable, stabTolerance);
+ seqSize = 1;
+ basePos = i;
+ startSubsequence(basePos, diffTolerance, &uniqueLevels_, minLevel, maxLevel);
}
+ if (i == values.size() - 1)
+ endSubsequence(seqSize, total, stable, stabTolerance);
}
+
+ *uniqueLevels = uniqueLevels_.size();
}
diff --git a/src/bm/resulthistoryinfo.h b/src/bm/resulthistoryinfo.h
index a65757f..b5bdff0 100644
--- a/src/bm/resulthistoryinfo.h
+++ b/src/bm/resulthistoryinfo.h
@@ -26,6 +26,7 @@
#include <QString>
#include <QList>
+#include <QMap>
#include <QBitArray>
class ResultHistoryInfo {
@@ -69,7 +70,10 @@ public:
QString testFunction() const { return testFunction_; }
QString dataTag() const { return dataTag_; }
- void computeMaxESSStats(qreal diffTolerance, int stabTolerance, int *total, int *stable);
+ void computeMaxESSStats(qreal diffTolerance, int stabTolerance, int *total, int *stable) const;
+ void computeStabilityStats(
+ qreal diffTolerance, int stabTolerance, bool *zerosFound, int *total, int *stable,
+ int *uniqueLevels, qreal *minLevel, qreal *maxLevel) const;
private:
int bmcontextId_;
@@ -96,6 +100,10 @@ private:
void markOutliers() const;
bool equal(int i, int j, int diffTolerance) const;
+ void startSubsequence(
+ int pos, qreal diffTolerance, QMap<int, int> *uniqueLevels, qreal *minLevel,
+ qreal *maxLevel, bool first = false) const;
+ static void endSubsequence(int seqSize, int *total, int *stable, int stabTolerance);
};
#endif // RESULTHISTORYINFO_H
diff --git a/src/bmclient/main.cpp b/src/bmclient/main.cpp
index 238d0f4..104f481 100644
--- a/src/bmclient/main.cpp
+++ b/src/bmclient/main.cpp
@@ -116,6 +116,9 @@ private:
BMRequest * createIndexPutConfigRequest(const QStringList &args, QString *error) const;
BMRequest * createGetHistoriesRequest(const QStringList &args, QString *error) const;
BMRequest * createGetIXHistoriesRequest(const QStringList &args, QString *error) const;
+ BMRequest * createASFStatsGetValuesRequest(
+ const QStringList &args, QString *error,
+ const QString &command = "asfstats get values") const;
mutable BMRequest::OutputFormat explicitOutputFormat;
mutable bool useExplicitOutputFormat;
BMRequest::OutputFormat outputFormat() const;
@@ -788,6 +791,32 @@ BMRequest * Executor::createRequest(const QStringList &args, QString *error) con
setOutputFormat(BMRequest::HTML);
return request;
+ } else if (
+ (args.size() >= 3) && (args.at(0) == "asfstats") && (args.at(1) == "get")
+ && (args.at(2) == "values")) {
+ // --- 'asfstats get values' command ---
+
+ return createASFStatsGetValuesRequest(args, error);
+
+ // } else if (
+ // (args.size() >= 3) && (args.at(0) == "asfstats") && (args.at(1) == "get")
+ // && (args.at(2) == "plot")) {
+ // // --- 'asfstats get plot' command ---
+
+ // BMRequest *request = createASFStatsGetValuesRequest(args, error, "asfstats get plot");
+ // setOutputFormat(BMRequest::Image);
+ // return request;
+
+ } else if (
+ (args.size() >= 3) && (args.at(0) == "asfstats") && (args.at(1) == "get")
+ && (args.at(2) == "detailspage")) {
+ // --- 'asfstats get detailspage' command ---
+
+ BMRequest *request =
+ createASFStatsGetValuesRequest(args, error, "asfstats get detailspage");
+ setOutputFormat(BMRequest::HTML);
+ return request;
+
} else if ((args.size() >= 2) && (args.at(0) == "get") && (args.at(1) == "detailspage")) {
// --- 'get detailspage' command ---
@@ -1269,6 +1298,126 @@ BMRequest * Executor::createGetIXHistoriesRequest(const QStringList &args, QStri
return new BMRequest_GetIXHistories(evalTimestamp, medianWinSize, rankedInfos, cacheKey);
}
+BMRequest * Executor::createASFStatsGetValuesRequest(
+ const QStringList &args, QString *error, const QString &command) const
+{
+ Q_UNUSED(command);
+
+ QStringList values;
+
+ // Get median window size ...
+ int medianWinSize = -1;
+ if (BMMisc::getOption(args, "-medianwinsize", &values, 1, 0, error)) {
+ bool ok;
+ medianWinSize = values.at(0).toInt(&ok);
+ if ((!ok) || (medianWinSize < 1)) {
+ *error = "failed to extract median window size as a positive integer";
+ return 0;
+ }
+ } else {
+ if (error->isEmpty())
+ *error = "-medianwinsize option not found";
+ return 0;
+ }
+
+ // Get cache key ...
+ QString cacheKey;
+ if (BMMisc::getOption(args, "-cachekey", &values, 1, 0, error)) {
+ cacheKey = values.at(0).trimmed();
+ bool ok;
+ cacheKey.toInt(&ok);
+ if (!ok) {
+ *error = "failed to extract cache key as an integer";
+ return 0;
+ }
+ } else if (!error->isEmpty()) {
+ return 0;
+ }
+
+ // Get filters ...
+ QStringList testCaseFilter;
+ if (!BMMisc::getMultiOption(args, "-testcase", &testCaseFilter, error))
+ return 0;
+ QStringList metricFilter;
+ if (!BMMisc::getMultiOption(args, "-metric", &metricFilter, error))
+ return 0;
+ QStringList platformFilter;
+ if (!BMMisc::getMultiOption(args, "-platform", &platformFilter, error))
+ return 0;
+ QStringList hostFilter;
+ if (!BMMisc::getMultiOption(args, "-host", &hostFilter, error))
+ return 0;
+ QStringList branchFilter;
+ if (!BMMisc::getMultiOption(args, "-branch", &branchFilter, error))
+ return 0;
+
+ // Get ASF-specific params ...
+
+ // ... timestamps ...
+
+ int fromTimestamp = -1;
+ if (BMMisc::getOption(args, "-fromtime", &values, 1, 0, error)) {
+ bool ok;
+ fromTimestamp = values.at(0).toInt(&ok);
+ if ((!ok) || (fromTimestamp < 0)) {
+ *error = "failed to extract 'from' time as a non-negative integer";
+ return 0;
+ }
+ } else {
+ if (error->isEmpty())
+ *error = QString("-fromtime option not found");
+ return 0;
+ }
+
+ int toTimestamp = -1;
+ if (BMMisc::getOption(args, "-totime", &values, 1, 0, error)) {
+ bool ok;
+ toTimestamp = values.at(0).toInt(&ok);
+ if ((!ok) || (toTimestamp < fromTimestamp)) {
+ *error = QString("failed to extract 'to' time as an integer < %1").arg(fromTimestamp);
+ return 0;
+ }
+ } else {
+ if (error->isEmpty())
+ *error = QString("-totime option not found");
+ return 0;
+ }
+
+ // ... tolerance values ...
+ qreal diffTol = -1;
+ if (!BMMisc::getClampedPercentageOption(args, "-difftol", &diffTol, error))
+ return 0;
+
+ int stabTol = -1;
+ if (BMMisc::getOption(args, "-stabtol", &values, 1, 0, error)) {
+ bool ok;
+ stabTol = values.at(0).toInt(&ok);
+ if ((!ok) || (stabTol < 2)) {
+ *error = "failed to extract stability tolerance as an integer >= 2";
+ return 0;
+ }
+ } else {
+ if (error->isEmpty())
+ *error = QString("-stabtol option not found");
+ return 0;
+ }
+
+ qreal sfTol = -1;
+ if (!BMMisc::getClampedPercentageOption(args, "-sftol", &sfTol, error))
+ return 0;
+ qreal lfTol = -1;
+ if (!BMMisc::getClampedPercentageOption(args, "-lftol", &lfTol, error))
+ return 0;
+ qreal maxLDTol = -1;
+ if (!BMMisc::getDoubleOption(args, "-maxldtol", &maxLDTol, error))
+ return 0;
+
+
+ return new BMRequest_ASFStatsGetValues(
+ medianWinSize, cacheKey, testCaseFilter, metricFilter, platformFilter, hostFilter,
+ branchFilter, fromTimestamp, toTimestamp, diffTol, stabTol, sfTol, lfTol, maxLDTol);
+}
+
// ### 2 B DOCUMENTED!
static void splitQuotedArgs(const QString &arg_s, QStringList *args)
{
@@ -1688,6 +1837,19 @@ class DirectExecutor : public Executor
"get ixhistories detailspage <SAME AS 'get ixhistories'> except that the mandatory \\\n"
" -stylesheet option is recognized\n"
+ <<
+ "asfstats get values \\\n"
+ " -medianwinsize <...> -fromtime <...> -totime <...> -difftol <...> \\\n"
+ " -stabtol <...> -sftol <...> -lftol <...> -maxldtol <...> \\\n"
+ " [-testcase <...> -testcase <...> ...] [-metric <...> -metric <...> ...] \\\n"
+ " [-platform <...> -platform <...> ...] [-host <...> -host <...> ...] \\\n"
+ " [-branch <...> -branch <...> ...]\n"
+
+ <<
+ "asfstats get detailspage <SAME AS 'asfstats get values'> except that the \\\n"
+ " optional -httpHeader and -cachekey options, and the mandatory \\\n"
+ " -stylesheet option are recognized\n"
+
;
qDebug() << "\nNote: the -server option may be replaced by a BMSERVER=<host>:<port> "
diff --git a/src/bmweb/indexsection.js b/src/bmweb/indexsection.js
index f07c635..18cc52c 100644
--- a/src/bmweb/indexsection.js
+++ b/src/bmweb/indexsection.js
@@ -96,14 +96,11 @@ function updateComputeASFStatsLink(defaultMedianWinSize)
url += " asfstats get detailspage";
url += " -stylesheet " + document.styleSheets[0].href;
-
// Add median window size ...
medianWinSize = getIntFromTextInput("ixMedianWinSize", defaultMedianWinSize);
document.getElementById("ixMedianWinSize").value = medianWinSize;
-
url += " -medianwinsize " + medianWinSize;
-
// Add filters ...
url += createFilterOptionsString("-testcase", getCheckedFilterValues("ixTestCaseFilter"));
url += createFilterOptionsString("-metric", getCheckedFilterValues("ixMetricFilter"));
@@ -111,11 +108,18 @@ function updateComputeASFStatsLink(defaultMedianWinSize)
url += createFilterOptionsString("-host", getCheckedFilterValues("ixHostFilter"));
url += createFilterOptionsString("-branch", getCheckedFilterValues("ixBranchFilter"));
+ // Add ASF-specific parameters ...
+ url += " -fromtime " + document.getElementById("ixASFStatsFromTime").value
+ + " -totime " + document.getElementById("ixASFStatsToTime").value
+ + " -difftol " + document.getElementById("ixASFStatsDiffTol").value
+ + " -stabtol " + selectedOptions("ixASFStatsStabTol")[0]
+ + " -sftol " + document.getElementById("ixASFStatsSFTol").value
+ + " -lftol " + document.getElementById("ixASFStatsLFTol").value
+ + " -maxldtol " + document.getElementById("ixASFStatsMaxLDTol").value;
// Update link ...
link = document.getElementById("ixComputeASFStatsLink");
-// link.setAttribute("href", url);
- link.setAttribute("href", "http://www.klokka.no"); // 4 NOW!
+ link.setAttribute("href", url);
link.setAttribute("target", "_blank"); // prevent page from opening in same window
}
@@ -653,14 +657,9 @@ function IndexSection()
legend.appendChild(document.createTextNode("Data Quality Statistics"));
legend.innerHTML = "ASF Scorecard Statistics";
legend.setAttribute("style", "font-size:12");
- table = fieldset.appendChild(document.createElement("table"));
- table.setAttribute("style", "border:0px; padding:0px");
// ... 'compute ASF stats' link ...
- tr = table.insertRow(table.rows.length);
- td = tr.insertCell(0);
- td.setAttribute("style", "border:0px; padding:2px");
- link = td.appendChild(document.createElement("a"));
+ link = fieldset.appendChild(document.createElement("a"));
link.id = "ixComputeASFStatsLink";
link.appendChild(document.createTextNode("Compute"));
link.setAttribute("class", "ixLink_index");
@@ -671,8 +670,12 @@ function IndexSection()
+ defaultMedianWinSize
+ ")"
);
+ fieldset.appendChild(document.createElement("br"));
+
+ table = fieldset.appendChild(document.createElement("table"));
+ table.setAttribute("style", "border:0px; padding:0px");
- // ... 'from time' ...
+ // ... 'from timestamp' ...
tr = table.insertRow(table.rows.length);
td = tr.insertCell(0);
td.innerHTML = "From timestamp (secs since 1970): ";
@@ -680,10 +683,10 @@ function IndexSection()
td = tr.insertCell(1);
input = td.appendChild(document.createElement("input"));
input.setAttribute("type", "text");
- input.setAttribute("id", "ixASFFromTime");
+ input.setAttribute("id", "ixASFStatsFromTime");
input.setAttribute("value", "-1");
- // ... 'to time' ...
+ // ... 'to timestamp' ...
tr = table.insertRow(table.rows.length);
td = tr.insertCell(0);
td.innerHTML = "To timestamp (secs since 1970): ";
@@ -691,9 +694,72 @@ function IndexSection()
td = tr.insertCell(1);
input = td.appendChild(document.createElement("input"));
input.setAttribute("type", "text");
- input.setAttribute("id", "ixASFToTime");
+ input.setAttribute("id", "ixASFStatsToTime");
input.setAttribute("value", "-1");
+ // ... 'difference tolerance' ...
+ tr = table.insertRow(table.rows.length);
+ td = tr.insertCell(0);
+ td.setAttribute("style", "text-align:right");
+ td.innerHTML = "Difference tolerance (%): ";
+ td = tr.insertCell(1);
+ td.setAttribute("style", "text-align:left");
+ input = td.appendChild(document.createElement("input"));
+ input.setAttribute("type", "text");
+ input.setAttribute("id", "ixASFStatsDiffTol");
+ input.setAttribute("Value", "10");
+
+ // ... 'stability tolerance' ...
+ tr = table.insertRow(table.rows.length);
+ td = tr.insertCell(0);
+ td.setAttribute("style", "text-align:right");
+ td.innerHTML = "Stability tolerance: ";
+ td = tr.insertCell(1);
+ td.setAttribute("style", "text-align:left");
+ select = td.appendChild(document.createElement("select"));
+ select.setAttribute("id", "ixASFStatsStabTol");
+ select.options[0] = new Option("2", "2");
+ select.options[1] = new Option("3", "3");
+ select.options[2] = new Option("4", "4");
+ select.options[3] = new Option("5", "5");
+ select.options[4] = new Option("6", "6");
+ select.options[2].selected = true;
+
+ // ... 'stability fraction tolerance' ...
+ tr = table.insertRow(table.rows.length);
+ td = tr.insertCell(0);
+ td.setAttribute("style", "text-align:right");
+ td.innerHTML = "Stability fraction tolerance (%): ";
+ td = tr.insertCell(1);
+ td.setAttribute("style", "text-align:left");
+ input = td.appendChild(document.createElement("input"));
+ input.setAttribute("type", "text");
+ input.setAttribute("id", "ixASFStatsSFTol");
+ input.setAttribute("Value", "10");
+
+ // ... 'level fraction tolerance' ...
+ tr = table.insertRow(table.rows.length);
+ td = tr.insertCell(0);
+ td.setAttribute("style", "text-align:right");
+ td.innerHTML = "Unique level fraction tolerance (%): ";
+ td = tr.insertCell(1);
+ td.setAttribute("style", "text-align:left");
+ input = td.appendChild(document.createElement("input"));
+ input.setAttribute("type", "text");
+ input.setAttribute("id", "ixASFStatsLFTol");
+ input.setAttribute("Value", "10");
+
+ // ... 'max level difference tolerance' ...
+ tr = table.insertRow(table.rows.length);
+ td = tr.insertCell(0);
+ td.setAttribute("style", "text-align:right");
+ td.innerHTML = "Max level difference tolerance (%): ";
+ td = tr.insertCell(1);
+ td.setAttribute("style", "text-align:left");
+ input = td.appendChild(document.createElement("input"));
+ input.setAttribute("type", "text");
+ input.setAttribute("id", "ixASFStatsMaxLDTol");
+ input.setAttribute("Value", "10");
// Create 'Configuration' section ...
section.appendChild(document.createElement("br"));