diff options
| author | jasplin <qt-info@nokia.com> | 2010-05-10 16:53:29 +0200 |
|---|---|---|
| committer | jasplin <qt-info@nokia.com> | 2010-05-10 16:53:29 +0200 |
| commit | 65589d5d1e1678227f12d9e50d60b3c02f7617b4 (patch) | |
| tree | 6687f8a11e2d2b673f5d6363e834f17eab9814c5 | |
| parent | c0c856290e3232dc0d6dd6e9a5a1c22e45d128e2 (diff) | |
Added ASF Scorecard feature.
| -rw-r--r-- | src/bm/asfstats.cpp | 211 | ||||
| -rw-r--r-- | src/bm/asfstats.h | 74 | ||||
| -rw-r--r-- | src/bm/bm.pro | 4 | ||||
| -rw-r--r-- | src/bm/bmmisc.cpp | 31 | ||||
| -rw-r--r-- | src/bm/bmmisc.h | 4 | ||||
| -rw-r--r-- | src/bm/bmrequest.cpp | 700 | ||||
| -rw-r--r-- | src/bm/bmrequest.h | 44 | ||||
| -rw-r--r-- | src/bm/resulthistoryinfo.cpp | 97 | ||||
| -rw-r--r-- | src/bm/resulthistoryinfo.h | 10 | ||||
| -rw-r--r-- | src/bmclient/main.cpp | 162 | ||||
| -rw-r--r-- | src/bmweb/indexsection.js | 96 |
11 files changed, 1264 insertions, 169 deletions
diff --git a/src/bm/asfstats.cpp b/src/bm/asfstats.cpp new file mode 100644 index 0000000..960e1c2 --- /dev/null +++ b/src/bm/asfstats.cpp @@ -0,0 +1,211 @@ +/**************************************************************************** +** +** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +** Contact: Qt Software Information (qt-info@nokia.com) +** +** This file is part of the BM project on Qt Labs. +** +** This file may be used under the terms of the GNU General Public +** License version 2.0 or 3.0 as published by the Free Software Foundation +** and appearing in the file LICENSE.GPL included in the packaging of +** this file. Please review the following information to ensure GNU +** General Public Licensing requirements will be met: +** http://www.fsf.org/licensing/licenses/info/GPLv2.html and +** http://www.gnu.org/copyleft/gpl.html. +** +** If you are unsure which license is appropriate for your use, please +** contact the sales department at qt-sales@nokia.com. +** +** This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE +** WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. +** +****************************************************************************/ + +#include "asfstats.h" +#include "bmmisc.h" +#include <QList> +#include <QMap> +#include <QDebug> + +/* NOTES + +Glossary: + +- RH: Result history. + + +Params: + +- diffTolerance (in percent: 0 <= x <= 100) +- stabTolerance (a positive integer: x >= 2) + +- sfTolerance (in percent) - the lowest tolerable SF (i.e. at or above is considered stable) +- lfTolerance (in percent) - the lowest tolerable LF (i.e. at or above is considered stable) +- maxLDTolerance(in percent) - the highest tolerable maxLD (i.e. at or below is considered stable) + +------------------------------------------------- + +Def: An equality subsequence (ESS) is a subsequence v1, v2, ..., vn of a RH + for which the following condition holds: + + ∀ i >= 1 : 100 * (max(vi, v1) / min(vi, v1) - 1) <= diffTolerance + + +Def: A maximal equality subsequence (MaxESS) is one of the subsequences formed by + partitioning a RH into the smallest possible number of ESS'es. + + +Def: The stability fraction (SF) of a RH is the fraction (given as a + percentage: 0 <= SF <= 100) of its MaxESS'es that are stable. + More precisely, + + SF = 100 * (stableMaxESS / totalMaxESS), + + where stableMaxESS is the the number of MaxESS'es that have a length of at least + stabTolerance and totalMaxESS is the total number of MaxESS'es. + +Def: The level fraction (LF) of a RH is the fraction (given as a percentage: 0 <= LF <= 100) + of its levels that are unique (distinct). + More precisely, + + LF = 100 * (uniqueLevels / totalMaxESS) + + where uniqueLevels is the number of unique levels and totalMaxESS is the total number of + MaxESS'es. Note that each MaxESS has a level (which is currently defined as its first value). + Example: A RH with levels 2, 10, 2, 10, 2 has a LF of 40% (2 of 5 levels are unique), + whereas one with levels 2, 11, 3, 12, 4 has a LF of 100% (5 of 5 levels are unique). + +Def: The maximum level distance (maxLD) of a RH is the relative difference (given as a percentage; + 0 <= maxLD <= 100) between the highest and lowest level. + More precisely: + + maxLD = 100 * ((maxLevel / minLevel) - 1) + +------- + +Step 1: Identifying unstable RHs + + A RH is considered unstable if at least one of the following conditions are true. + Except for the first condition, the conditions are evaluated for the raw + (i.e. unsmoothed) data. + + Cond 1: At least one zero value occurs + Cond 2: The RH doesn't have a _smoothed_ value before fromTimestamp. + Cond 3: SF < sfTolerance + Cond 4: LF < lfTolerance + Cond 5: maxLD < maxLDTolerance + + Stats produced: + Stat 1: Count for Cond 1 + Stat 2: Count for Cond 2 + Stat 3: Count for Cond 3 + Stat 4: Count for Cond 4 + Stat 5: Count for Cond 5 + Stat 6: Total unstable count + + Other output: + The list of BM context IDs of the unstable RHs. + + +Step 2: Sample the stable and smoothed RHs at fromTimestamp and toTimestamp and compare the + difference. + For each RH, the two values to be compared - v1 and v2 - are the latest smoothed + values that are not later than fromTimestamp and toTimestamp respectively. + If 100 * (max(v1, v2) / min(v1, v2) <= diffTolerance, the values are considered equal. + Otherwise, the diff is computed as v1 - v2 for a "lower is better" metric or v2 - v1 + for a "higher is better" metric. If diff < 0, the RH regressed, otherwise it improved. + + Stats produced: + Stat 1: Regressed count. + Stat 2: Unchanged count. + Stat 4: Improved count. + +*/ + + +// ### 2 B DOCUMENTED! +void ASFStats::compute( + const QList<ResultHistoryInfo *> &rhInfos, UnstableStats *usStats, int *regressed, + int *unchanged, int *improved) +{ + Q_ASSERT((fromTimestamp >= 0) && (fromTimestamp <= toTimestamp)); + Q_ASSERT((diffTolerance >= 0.0) && (diffTolerance <= 100.0)); + Q_ASSERT(stabTolerance >= 2); + Q_ASSERT((sfTolerance >= 0.0) && (sfTolerance <= 100.0)); + Q_ASSERT((lfTolerance >= 0.0) && (lfTolerance <= 100.0)); + + usStats->unstable.fill(false, rhInfos.size()); + *regressed = *unchanged = *improved = 0; + + for (int i = 0; i < rhInfos.size(); ++i) { + bool zerosFound = false; + int total = 0; + int stable = 0; + int uniqueLevels = 0; + qreal minLevel = 0; + qreal maxLevel = 0; + + // Compute stability stats for raw (unsmoothed) data ... + rhInfos.at(i)->computeStabilityStats( + diffTolerance, stabTolerance, &zerosFound, &total, &stable, &uniqueLevels, &minLevel, + &maxLevel); + + Q_ASSERT(total > 0); + const qreal sf = 100 * (stable / qreal(total)); + const qreal lf = 100 * (uniqueLevels / qreal(total)); + + const qreal maxLD = zerosFound ? -1 : (100 * ((maxLevel / minLevel) - 1)); + + int fromPos = -1; + rhInfos.at(i)->findSmoothPos(fromTimestamp, &fromPos); + + bool unstable = false; + if (zerosFound) { + unstable = true; + ++(usStats->zeroCount); + } + if (fromPos == -1) { + unstable = true; + ++(usStats->lowFromPosCount); + } + if (sf < sfTolerance) { + unstable = true; + ++(usStats->lowSFCount); + } + if (lf < lfTolerance) { + unstable = true; + ++(usStats->lowLFCount); + } + if (maxLD > maxLDTolerance) { + unstable = true; + ++(usStats->highMaxLDCount); + } + + if (unstable) { + + // RH is unstable, so mark it as such ... + usStats->unstable.setBit(i); + + } else { + // RH is stable, so sample the smoothed values at fromTimestamp and toTimestamp and + // classify the difference as 'regressed', 'unchanged', or 'improved' ... + + int toPos = -1; + const bool ok = rhInfos.at(i)->findSmoothPos(toTimestamp, &toPos); + Q_ASSERT(ok); + + const qreal v1 = rhInfos.at(i)->value(fromPos); + const qreal v2 = rhInfos.at(i)->value(toPos); + if ((100 * (qMax(v1, v2) / qMin(v1, v2) - 1)) <= diffTolerance) { + ++(*unchanged); + } else { + const qreal diff = + BMMisc::lowerIsBetter(rhInfos.at(i)->metric()) ? (v1 - v2) : (v2 - v1); + if (diff < 0) + ++(*regressed); + else + ++(*improved); + } + } + } +} diff --git a/src/bm/asfstats.h b/src/bm/asfstats.h new file mode 100644 index 0000000..c153376 --- /dev/null +++ b/src/bm/asfstats.h @@ -0,0 +1,74 @@ +/**************************************************************************** +** +** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +** Contact: Qt Software Information (qt-info@nokia.com) +** +** This file is part of the BM project on Qt Labs. +** +** This file may be used under the terms of the GNU General Public +** License version 2.0 or 3.0 as published by the Free Software Foundation +** and appearing in the file LICENSE.GPL included in the packaging of +** this file. Please review the following information to ensure GNU +** General Public Licensing requirements will be met: +** http://www.fsf.org/licensing/licenses/info/GPLv2.html and +** http://www.gnu.org/copyleft/gpl.html. +** +** If you are unsure which license is appropriate for your use, please +** contact the sales department at qt-sales@nokia.com. +** +** This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE +** WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. +** +****************************************************************************/ + +#ifndef ASFSTATS_H +#define ASFSTATS_H + +#include "resulthistoryinfo.h" + +// ### 2 B DOCUMENTED! + +class ASFStats { +public: + ASFStats( + const qreal diffTolerance, const int stabTolerance, const qreal sfTolerance, + const qreal lfTolerance, const qreal maxLDTolerance, const int fromTimestamp, + const int toTimestamp) + : diffTolerance(diffTolerance) + , stabTolerance(stabTolerance) + , sfTolerance(sfTolerance) + , lfTolerance(lfTolerance) + , maxLDTolerance(maxLDTolerance) + , fromTimestamp(fromTimestamp) + , toTimestamp(toTimestamp) + {} + + struct UnstableStats { + QBitArray unstable; + int zeroCount; + int lowFromPosCount; + int lowSFCount; + int lowLFCount; + int highMaxLDCount; + UnstableStats() + : zeroCount(0), lowFromPosCount(), lowSFCount(0), lowLFCount(0), highMaxLDCount(0) {} + }; + + void compute( + const QList<ResultHistoryInfo *> &rhInfos, UnstableStats *usStats, int *regressed, + int *unchanged, int *improved); + +private: + // Tolerances: + qreal diffTolerance; // difference + int stabTolerance; // stability + qreal sfTolerance; // stability fraction + qreal lfTolerance; // level fraction + qreal maxLDTolerance; // maximum level difference + + // Timestamps: + int fromTimestamp; + int toTimestamp; +}; + +#endif // ASFSTATS_H diff --git a/src/bm/bm.pro b/src/bm/bm.pro index 70ae5dd..1bf4f1e 100644 --- a/src/bm/bm.pro +++ b/src/bm/bm.pro @@ -2,9 +2,9 @@ TEMPLATE = lib CONFIG += shared TARGET = bm SOURCES += bm.cpp bmrequest.cpp bmmisc.cpp plotter.cpp resulthistoryinfo.cpp cache.cpp index.cpp \ - dataqualitystats.cpp + dataqualitystats.cpp asfstats.cpp HEADERS += bm.h bmrequest.h bmmisc.h plotter.h resulthistoryinfo.h cache.h index.h \ - dataqualitystats.h + dataqualitystats.h asfstats.h QT += network QT += xml QT += sql diff --git a/src/bm/bmmisc.cpp b/src/bm/bmmisc.cpp index 90d4062..23bebda 100644 --- a/src/bm/bmmisc.cpp +++ b/src/bm/bmmisc.cpp @@ -187,6 +187,37 @@ bool BMMisc::hasOption(const QStringList &args, const QString &option) return getOption(args, option, &dummyValues, 0, 0); } +// ### 2 B DOCUMENTED! +bool BMMisc::getDoubleOption( + const QStringList &args, const QString &option, qreal *value, QString *error) +{ + QStringList values; + if (getOption(args, option, &values, 1, 0, error)) { + bool ok; + *value = values.at(0).toDouble(&ok); + if (!ok) { + *error = QString("failed to extract %1 as a double").arg(option); + return false; + } + } else { + if (error->isEmpty()) + *error = QString("%1 option not found").arg(option); + return false; + } + + return true; +} + +// ### 2 B DOCUMENTED! +bool BMMisc::getClampedPercentageOption( + const QStringList &args, const QString &option, qreal *value, QString *error) +{ + if (!getDoubleOption(args, option, value, error)) + return false; + *value = qMin(qMax(*value, 0.0), 100.0); + return true; +} + // Computes the median value of the numbers in \a values. qreal BMMisc::median(const QList<qreal> &values) { diff --git a/src/bm/bmmisc.h b/src/bm/bmmisc.h index 057adac..62469bd 100644 --- a/src/bm/bmmisc.h +++ b/src/bm/bmmisc.h @@ -57,6 +57,10 @@ public: const QStringList &args, const QString &option, QList<QStringList> *values, int n, QString *error); static bool hasOption(const QStringList &args, const QString &option); + static bool getDoubleOption( + const QStringList &args, const QString &option, qreal *value, QString *error); + static bool getClampedPercentageOption( + const QStringList &args, const QString &option, qreal *value, QString *error); static qreal median(const QList<qreal> &values); static int medianPos(const QList<qreal> &values, bool lastDuplicate = true); static qreal v2y( diff --git a/src/bm/bmrequest.cpp b/src/bm/bmrequest.cpp index 0025c0a..fa09701 100644 --- a/src/bm/bmrequest.cpp +++ b/src/bm/bmrequest.cpp @@ -24,6 +24,7 @@ #include "bmrequest.h" #include "resulthistoryinfo.h" #include "dataqualitystats.h" +#include "asfstats.h" #include "plotter.h" #include "cache.h" #include <QSqlDatabase> @@ -107,6 +108,8 @@ BMRequest * BMRequest::create(const QByteArray &data, const QString &msgType) request = new BMRequest_GetBMTree(doc); else if (type == "GetIXHistories") request = new BMRequest_GetIXHistories(doc); + else if (type == "ASFStatsGetValues") + request = new BMRequest_ASFStatsGetValues(doc); #ifdef BMDEBUG else qDebug() << "invalid request type:" << type; @@ -4842,6 +4845,143 @@ void BMRequest_GetResult::handleReply_Raw(const QStringList &args) const } } +// ### 2 B DOCUMENTED! +static bool createFilteredRHInfos( + QList<ResultHistoryInfo *> *rhInfos, int *totCandidates, const QString &reqName, + QSqlDatabase *database, const QStringList &testCaseFilter, const QStringList &metricFilter, + const QStringList &platformFilter, const QStringList &hostFilter, + const QStringList &branchFilter, const int medianWinSize, QString *error, + const bool debugPrint = false) +{ + QSqlQuery query(*database); + + // Get the initial list of candidate result histories by getting the BM context ID and metric + // for all result histories that match the logical OR-values. + // NOTE: Results measuring the "events" metric are skipped for now since + // they seem to be of little value when computing the performance index (they are + // typically integers close to (and often at) zero) ... + + if (debugPrint) + fprintf(stderr, "fetching list of relevant result histories ... "); + + QString query_s = + "SELECT bmcontext.id, metric.name" + " FROM bmcontext, benchmark, context, metric, platform, host, branch" + " WHERE benchmarkId = benchmark.id" + " AND contextId = context.id" + " AND metricId = metric.id" + " AND platformId = platform.id" + " AND hostId = host.id" + " AND branchId = branch.id" + " AND metric.name != 'events'"; + + if (!testCaseFilter.isEmpty()) { + query_s += " AND ("; + for (int i = 0; i < testCaseFilter.size(); ++i) + query_s += QString("%1benchmark.testCase = '%2'") + .arg((i > 0) ? " OR " : "") + .arg(testCaseFilter.at(i)); + query_s += ")"; + } + + if (!metricFilter.isEmpty()) { + query_s += " AND ("; + for (int i = 0; i < metricFilter.size(); ++i) + query_s += + QString("%1metric.name = '%2'").arg((i > 0) ? " OR " : "").arg(metricFilter.at(i)); + query_s += ")"; + } + + if (!platformFilter.isEmpty()) { + query_s += " AND ("; + for (int i = 0; i < platformFilter.size(); ++i) + query_s += QString("%1platform.name = '%2'") + .arg((i > 0) ? " OR " : "") + .arg(platformFilter.at(i)); + query_s += ")"; + } + + if (!hostFilter.isEmpty()) { + query_s += " AND ("; + for (int i = 0; i < hostFilter.size(); ++i) + query_s += QString("%1host.name = '%2'") + .arg((i > 0) ? " OR " : "") + .arg(hostFilter.at(i)); + query_s += ")"; + } + + if (!branchFilter.isEmpty()) { + query_s += " AND ("; + for (int i = 0; i < branchFilter.size(); ++i) + query_s += QString("%1(branch.gitRepo || branch.gitBranch) = '%2'") + .arg((i > 0) ? " OR " : "") + .arg(branchFilter.at(i)); + query_s += ")"; + } + + query_s += ";"; + + if (!query.exec(query_s)) { + *error = errorReply(query, reqName, "failed to get bmcontext IDs (exec() failed)"); + return false; + } + QList<int> bmcontextIds; + QStringList metrics; + while (query.next()) { + bmcontextIds += query.value(0).toInt(); + metrics += query.value(1).toString(); + } + if (debugPrint) + fprintf(stderr, "done\n"); + + if (bmcontextIds.isEmpty()) { + *error = errorReply(reqName, "no matching result histories found"); + return false; + } + + + // Loop over initial result history candidates ... + for (int i = 0; i < bmcontextIds.size(); ++i) { + + if (!(i % qMax(1, (bmcontextIds.size() / 20)))) { + if (debugPrint) + fprintf( + stderr, "fetching result histories (%0.0f%% done)\r", + 100 * (qreal(i) / bmcontextIds.size())); + } + + // Get the time series ... + + if (!query.exec( + QString( + "SELECT timestamp, value FROM result " + "WHERE bmcontextId=%1 " + "ORDER BY timestamp ASC;") + .arg(bmcontextIds.at(i)))) { + *error = errorReply(query, reqName, "failed to get time series (exec() failed)"); + return false; + } + QList<int> timestamps; + QList<qreal> values; + while (query.next()) { + timestamps += query.value(0).toInt(); + values += query.value(1).toDouble(); + } + + ResultHistoryInfo *rhInfo = + new ResultHistoryInfo( + bmcontextIds.at(i), timestamps, values, medianWinSize, metrics.at(i)); + rhInfos->append(rhInfo); + } + + *totCandidates = bmcontextIds.size(); + + if (debugPrint) + fprintf(stderr, "fetching result histories (100%% done)\n"); + + return true; +} + // --- IndexGetValues --- QByteArray BMRequest_IndexGetValues::toRequestBuffer(QString *) @@ -4895,7 +5035,7 @@ QByteArray BMRequest_IndexGetValues::toReplyBuffer() static int calls_ = 0; fprintf( stderr, - "BMRequest_IndexGetValues2::toReplyBuffer() called (%d), cacheKey: >%s<\n", + "BMRequest_IndexGetValues::toReplyBuffer() called (%d), cacheKey: >%s<\n", calls_++, argsElem.attributeNode("cacheKey").value().toLatin1().data()); } if (ok) { @@ -4949,140 +5089,22 @@ QByteArray BMRequest_IndexGetValues::toReplyBuffer() branchFilter.append(branchNodes.at(i).toElement().attributeNode("name").value()); - QSqlQuery *query; - - // Get the initial list of candidate result histories by getting the BM context ID and metric - // for all result histories that match the logical OR-values. - // NOTE: Results measuring the "events" metric are skipped for now since - // they seem to be of little value when computing the performance index (they are - // typically integers close to (and often at) zero) ... - query = createQuery(); - - if (debugPrint) - fprintf(stderr, "fetching list of relevant result histories ... "); - - QString query_s = - "SELECT bmcontext.id, metric.name" - " FROM bmcontext, benchmark, context, metric, platform, host, branch" - " WHERE benchmarkId = benchmark.id" - " AND contextId = context.id" - " AND metricId = metric.id" - " AND platformId = platform.id" - " AND hostId = host.id" - " AND branchId = branch.id" - " AND metric.name != 'events'"; - - if (!testCaseFilter.isEmpty()) { - query_s += " AND ("; - for (int i = 0; i < testCaseFilter.size(); ++i) - query_s += QString("%1benchmark.testCase = '%2'") - .arg((i > 0) ? " OR " : "") - .arg(testCaseFilter.at(i)); - query_s += ")"; - } - - if (!metricFilter.isEmpty()) { - query_s += " AND ("; - for (int i = 0; i < metricFilter.size(); ++i) - query_s += - QString("%1metric.name = '%2'").arg((i > 0) ? " OR " : "").arg(metricFilter.at(i)); - query_s += ")"; - } - - if (!platformFilter.isEmpty()) { - query_s += " AND ("; - for (int i = 0; i < platformFilter.size(); ++i) - query_s += QString("%1platform.name = '%2'") - .arg((i > 0) ? " OR " : "") - .arg(platformFilter.at(i)); - query_s += ")"; - } - - if (!hostFilter.isEmpty()) { - query_s += " AND ("; - for (int i = 0; i < hostFilter.size(); ++i) - query_s += QString("%1host.name = '%2'") - .arg((i > 0) ? " OR " : "") - .arg(hostFilter.at(i)); - query_s += ")"; - } - - if (!branchFilter.isEmpty()) { - query_s += " AND ("; - for (int i = 0; i < branchFilter.size(); ++i) - query_s += QString("%1(branch.gitRepo || branch.gitBranch) = '%2'") - .arg((i > 0) ? " OR " : "") - .arg(branchFilter.at(i)); - query_s += ")"; - } - - query_s += ";"; - - if (!query->exec(query_s)) { - deleteQuery(query); - return xmlConvert( - errorReply(*query, name(), "failed to get bmcontext IDs (exec() failed)")); - } - QList<int> bmcontextIds; - QStringList metrics; - while (query->next()) { - bmcontextIds += query->value(0).toInt(); - metrics += query->value(1).toString(); - } - deleteQuery(query); - if (debugPrint) - fprintf(stderr, "done\n"); - - if (bmcontextIds.isEmpty()) - return xmlConvert(errorReply(name(), "no matching result histories found")); - - QList<ResultHistoryInfo *> rhInfos; + int totRHCandidates; + if (!createFilteredRHInfos( + &rhInfos, &totRHCandidates, name(), database, testCaseFilter, metricFilter, + platformFilter, hostFilter, branchFilter, medianWinSize, &error, debugPrint)) { - // Loop over initial result history candidates ... - for (int i = 0; i < bmcontextIds.size(); ++i) { - - if (!(i % qMax(1, (bmcontextIds.size() / 20)))) { - if (debugPrint) - fprintf( - stderr, "fetching result histories (%0.0f%% done)\r", - 100 * (qreal(i) / bmcontextIds.size())); - } - - // Get the time series ... - query = createQuery(); - - if (!query->exec( - QString( - "SELECT timestamp, value FROM result " - "WHERE bmcontextId=%1 " - "ORDER BY timestamp ASC;") - .arg(bmcontextIds.at(i)))) { - deleteQuery(query); - return xmlConvert( - errorReply(*query, name(), "failed to get time series (exec() failed)")); - } - QList<int> timestamps; - QList<qreal> values; - while (query->next()) { - timestamps += query->value(0).toInt(); - values += query->value(1).toDouble(); - } - deleteQuery(query); + // Free dynamic memory ... + for (int i = 0; i < rhInfos.size(); ++i) + delete rhInfos.at(i); - ResultHistoryInfo *rhInfo = - new ResultHistoryInfo( - bmcontextIds.at(i), timestamps, values, medianWinSize, metrics.at(i)); - rhInfos.append(rhInfo); + return xmlConvert(error); } - if (debugPrint) - fprintf(stderr, "fetching result histories (100%% done)\n"); - reply = QString("<reply type=\"%1\" >").arg(name()); - int nonPositiveRH = 0; int baseValuePos; @@ -5092,12 +5114,10 @@ QByteArray BMRequest_IndexGetValues::toReplyBuffer() QList<int> evalTimestamps; IndexAlgorithm1 index(rhInfos, baseTimestamp, &evalTimestamps, &nonPositiveRH); - if (!index.isValid()) { + if (!index.isValid()) return xmlConvert( errorReply( name(), QString("failed to compute index (1): %1").arg(index.invalidReason()))); - } - QList<qreal> indexValues; QList<int> contrCounts; @@ -5108,10 +5128,9 @@ QByteArray BMRequest_IndexGetValues::toReplyBuffer() if (!index.computeValues( &indexValues, &baseValuePos, &contrCounts, &error_, &topContr, topContrLimit, - dataQualityStats ? &dqStats : static_cast<DataQualityStats *>(0))) { + dataQualityStats ? &dqStats : static_cast<DataQualityStats *>(0))) return xmlConvert( errorReply(name(), QString("failed to compute index (2): %1").arg(error_))); - } // Add index values to reply ... Q_ASSERT(indexValues.size() == evalTimestamps.size()); @@ -5191,14 +5210,15 @@ QByteArray BMRequest_IndexGetValues::toReplyBuffer() reply += QString( "<args baseTimestamp=\"%1\" medianWinSize=\"%2\" totalRH=\"%3\" nonPositiveRH=\"%4\" " "baseValuePos=\"%5\" cacheKey=\"%6\" /></reply>") - .arg(baseTimestamp).arg(medianWinSize).arg(bmcontextIds.size()).arg(nonPositiveRH) + .arg(baseTimestamp).arg(medianWinSize).arg(totRHCandidates).arg(nonPositiveRH) .arg(baseValuePos).arg(cacheKey_); Cache::instance()->put(cacheKey_, reply); return xmlConvert(reply); } -void BMRequest_IndexGetValues::appendToFilterTable( +// ### 2 B DOCUMENTED! +static void appendToFilterTable( const QStringList &filter, const QString &filterName, QString *reply) { reply->append( @@ -7200,3 +7220,415 @@ void BMRequest_GetIXHistories::handleReply_Image(const QStringList &args) const "failed to create plot for main index contributors: %1").arg(error_)); } } + + +// --- ASFStatsGetValues --- +QByteArray BMRequest_ASFStatsGetValues::toRequestBuffer(QString *) +{ + QString request = + QString( + "<request type=\"%1\"><args medianWinSize=\"%2\" cacheKey=\"%3\" " + "fromTimestamp=\"%4\" toTimestamp=\"%5\" diffTol=\"%6\" stabTol=\"%7\" " + "sfTol=\"%8\" lfTol=\"%9\" maxLDTol=\"%10\" />") + .arg(name()) + .arg(medianWinSize) + .arg(cacheKey) + .arg(fromTimestamp) + .arg(toTimestamp) + .arg(diffTol) + .arg(stabTol) + .arg(sfTol) + .arg(lfTol) + .arg(maxLDTol); + + for (int i = 0; i < testCaseFilter.size(); ++i) + request += QString("<testCase name=\"%1\" />").arg(testCaseFilter.at(i)); + + for (int i = 0; i < metricFilter.size(); ++i) + request += QString("<metric name=\"%1\" />").arg(metricFilter.at(i)); + + for (int i = 0; i < platformFilter.size(); ++i) + request += QString("<platform name=\"%1\" />").arg(platformFilter.at(i)); + + for (int i = 0; i < hostFilter.size(); ++i) + request += QString("<host name=\"%1\" />").arg(hostFilter.at(i)); + + for (int i = 0; i < branchFilter.size(); ++i) + request += QString("<branch name=\"%1\" />").arg(branchFilter.at(i)); + + request += "</request>"; + + return xmlConvert(request); +} + +QByteArray BMRequest_ASFStatsGetValues::toReplyBuffer() +{ + QDomElement argsElem = doc.elementsByTagName("args").at(0).toElement(); + + QString error; + QString reply; + bool ok; + + const bool debugPrint = false; + + // Check if a cached reply can be returned ... + int cacheKey_ = argsElem.attributeNode("cacheKey").value().toInt(&ok); + if (debugPrint) { + static int calls_ = 0; + fprintf( + stderr, + "BMRequest_ASFStatsGetValues::toReplyBuffer() called (%d), cacheKey: >%s<\n", + calls_++, argsElem.attributeNode("cacheKey").value().toLatin1().data()); + } + if (ok) { + reply = Cache::instance()->get(cacheKey_); + if (!reply.isEmpty()) { + if (debugPrint) + fprintf(stderr, "returning cached reply (key = %d)\n", cacheKey_); + return xmlConvert(reply); + } + } + + // Compute from scratch ... + if (debugPrint) + fprintf(stderr, "computing from scratch ...\n"); + + // Get from timestamp ... + fromTimestamp = argsElem.attributeNode("fromTimestamp").value().toInt(&ok); + Q_ASSERT(ok); + + // Get to timestamp ... + toTimestamp = argsElem.attributeNode("toTimestamp").value().toInt(&ok); + Q_ASSERT(ok); + + // Get median window size ... + medianWinSize = argsElem.attributeNode("medianWinSize").value().toInt(&ok); + Q_ASSERT(ok); + + // Get tolerance values ... + diffTol = argsElem.attributeNode("diffTol").value().toDouble(&ok); + Q_ASSERT(ok); + stabTol = argsElem.attributeNode("stabTol").value().toInt(&ok); + Q_ASSERT(ok); + sfTol = argsElem.attributeNode("sfTol").value().toDouble(&ok); + Q_ASSERT(ok); + lfTol = argsElem.attributeNode("lfTol").value().toDouble(&ok); + Q_ASSERT(ok); + maxLDTol = argsElem.attributeNode("maxLDTol").value().toDouble(&ok); + Q_ASSERT(ok); + + // Get filters ... + QDomNodeList testCaseNodes = doc.elementsByTagName("testCase"); + for (int i = 0; i < testCaseNodes.size(); ++i) + testCaseFilter.append(testCaseNodes.at(i).toElement().attributeNode("name").value()); + + QDomNodeList metricNodes = doc.elementsByTagName("metric"); + for (int i = 0; i < metricNodes.size(); ++i) + metricFilter.append(metricNodes.at(i).toElement().attributeNode("name").value()); + + QDomNodeList platformNodes = doc.elementsByTagName("platform"); + for (int i = 0; i < platformNodes.size(); ++i) + platformFilter.append(platformNodes.at(i).toElement().attributeNode("name").value()); + + QDomNodeList hostNodes = doc.elementsByTagName("host"); + for (int i = 0; i < hostNodes.size(); ++i) + hostFilter.append(hostNodes.at(i).toElement().attributeNode("name").value()); + + QDomNodeList branchNodes = doc.elementsByTagName("branch"); + for (int i = 0; i < branchNodes.size(); ++i) + branchFilter.append(branchNodes.at(i).toElement().attributeNode("name").value()); + + + QList<ResultHistoryInfo *> rhInfos; + int totRHCandidates; + if (!createFilteredRHInfos( + &rhInfos, &totRHCandidates, name(), database, testCaseFilter, metricFilter, + platformFilter, hostFilter, branchFilter, medianWinSize, &error, debugPrint)) { + + // Free dynamic memory ... + for (int i = 0; i < rhInfos.size(); ++i) + delete rhInfos.at(i); + + return xmlConvert(error); + } + + + reply = QString("<reply type=\"%1\" >").arg(name()); + + ASFStats asfStats(diffTol, stabTol, sfTol, lfTol, maxLDTol, fromTimestamp, toTimestamp); + ASFStats::UnstableStats usStats; + int regressed = 0; + int unchanged = 0; + int improved = 0; + asfStats.compute(rhInfos, &usStats, ®ressed, &unchanged, &improved); + + // Add stats to reply ... + reply += QString( + "<stats regressed=\"%1\" unchanged=\"%2\" improved=\"%3\" unstable=\"%4\" " + "usZeroCount=\"%5\" usLowFromPosCount=\"%6\" usLowSFCount=\"%7\" usLowLFCount=\"%8\" " + "usHighMaxLDCount=\"%9\" />") + .arg(regressed).arg(unchanged).arg(improved) + .arg(usStats.unstable.count(true)) + .arg(usStats.zeroCount) + .arg(usStats.lowFromPosCount) + .arg(usStats.lowSFCount) + .arg(usStats.lowLFCount) + .arg(usStats.highMaxLDCount); + + // Add IDs of unstable result histories to reply ... + reply += "<unstableIDs>"; + for (int i = 0; i < usStats.unstable.size(); ++i) + if (usStats.unstable.testBit(i)) + reply += QString("%1 ").arg(rhInfos.at(i)->bmcontextId()); + reply += "</unstableIDs>"; + + //---------------------------------------------------------------------------------------- + + for (int i = 0; i < testCaseFilter.size(); ++i) + reply += QString("<testCase name=\"%1\" />").arg(testCaseFilter.at(i)); + + for (int i = 0; i < metricFilter.size(); ++i) + reply += QString("<metric name=\"%1\" />").arg(metricFilter.at(i)); + + for (int i = 0; i < platformFilter.size(); ++i) + reply += QString("<platform name=\"%1\" />").arg(platformFilter.at(i)); + + for (int i = 0; i < hostFilter.size(); ++i) + reply += QString("<host name=\"%1\" />").arg(hostFilter.at(i)); + + for (int i = 0; i < branchFilter.size(); ++i) + reply += QString("<branch name=\"%1\" />").arg(branchFilter.at(i)); + + // Finalize the reply and cache it ... + cacheKey_ = Cache::instance()->nextId(); + reply += QString( + "<args medianWinSize=\"%1\" cacheKey=\"%2\" " + "fromTimestamp=\"%3\" toTimestamp=\"%4\" diffTol=\"%5\" stabTol=\"%6\" " + "sfTol=\"%7\" lfTol=\"%8\" maxLDTol=\"%9\" /></reply>") + .arg(medianWinSize).arg(cacheKey_).arg(fromTimestamp).arg(toTimestamp).arg(diffTol) + .arg(stabTol).arg(sfTol).arg(lfTol).arg(maxLDTol); + Cache::instance()->put(cacheKey_, reply); + + // Free dynamic memory ... + for (int i = 0; i < rhInfos.size(); ++i) + delete rhInfos.at(i); + + return xmlConvert(reply); +} + +void BMRequest_ASFStatsGetValues::handleReply_HTML(const QStringList &args) const +{ + QString error; + + error = doc.elementsByTagName("reply").at(0).toElement().attributeNode("error").value(); + if (!error.isEmpty()) { + BMMisc::printHTMLErrorPage( + QString("failed to create details page: error evaluating ASF stats: %1").arg(error)); + return; + } + + QStringList optValues; + + // Get server ... + QString server; + if (BMMisc::getOption(args, "-server", &optValues, 1, 0, &error)) { + server = optValues.first().trimmed(); + } else { + BMMisc::printHTMLErrorPage("-server option not found"); + return; + } + + // Get style sheet ... + QString styleSheet; + if (BMMisc::getOption(args, "-stylesheet", &optValues, 1, 0, &error)) { + styleSheet = optValues.first().trimmed(); + } else { + BMMisc::printHTMLErrorPage("-stylesheet option not found"); + return; + } + + // *** Header *** + QString reply = QString("<html>\n<head>\n"); + reply += QString("<link rel=\"stylesheet\" type=\"text/css\" href=\"%1\" />\n").arg(styleSheet); + reply += QString("</head>\n<body>\n"); + + reply += "<span style=\"font-size:18\">ASF Stats (for scorecard)</span>"; + + reply += "<br /><br /><table><tr><td style=\"background-color:#eeeeee\">"; + + reply += "This page shows statistics suitable for presentation on the ASF Scorecard "; + reply += "<span style=\"color:red\"> (More documentation to follow soon!)</span>."; + + reply += "</td></tr></table>\n"; + + QDomElement argsElem = doc.elementsByTagName("args").at(0).toElement(); + + bool ok; + + + reply += "\n<br /><br /><table style=\"border:0px\">\n"; + reply += "<tr>\n"; + + // Left main table ... + + reply += "<td style=\"border:0px\">\n"; + + reply += "<fieldset><legend style=\"font-size:12\">Parameters</legend>"; + + // 'Misc' table ... + fromTimestamp = argsElem.attributeNode("fromTimestamp").value().toInt(&ok); + Q_ASSERT(ok); + toTimestamp = argsElem.attributeNode("toTimestamp").value().toInt(&ok); + Q_ASSERT(ok); + medianWinSize = argsElem.attributeNode("medianWinSize").value().toInt(&ok); + Q_ASSERT(ok); + diffTol = argsElem.attributeNode("diffTol").value().toDouble(&ok); + Q_ASSERT(ok); + stabTol = argsElem.attributeNode("stabTol").value().toInt(&ok); + Q_ASSERT(ok); + sfTol = argsElem.attributeNode("sfTol").value().toInt(&ok); + Q_ASSERT(ok); + lfTol = argsElem.attributeNode("lfTol").value().toInt(&ok); + Q_ASSERT(ok); + maxLDTol = argsElem.attributeNode("maxLDTol").value().toDouble(&ok); + Q_ASSERT(ok); + + QDateTime dateTime; + reply += "\n<br /><table style=\"text-align:right\">\n"; + dateTime.setTime_t(fromTimestamp); + reply += QString("<tr><td>From time:</td><td>%1</td></tr>\n").arg(dateTime.toString()); + dateTime.setTime_t(toTimestamp); + reply += QString("<tr><td>To time:</td><td>%1</td></tr>\n").arg(dateTime.toString()); + reply += QString("<tr><td>Median window size:</td><td>%1</td></tr>\n").arg(medianWinSize); + reply += QString("<tr><td>Difference tolerance (%):</td><td>%1</td></tr>\n").arg(diffTol); + reply += QString("<tr><td>Stability tolerance:</td><td>%1</td></tr>\n").arg(stabTol); + reply += QString("<tr><td>Stability fraction tolerance (%):</td><td>%1</td></tr>\n").arg(sfTol); + reply += QString("<tr><td>Unique level fraction tolerance (%):</td><td>%1</td></tr>\n") + .arg(lfTol); + reply += QString("<tr><td>Maximim level distance tolerance (%):</td><td>%1</td></tr>\n") + .arg(maxLDTol); + reply += "</table>\n"; + + + // 'Filters' table ... + QDomNodeList testCaseNodes = doc.elementsByTagName("testCase"); + for (int i = 0; i < testCaseNodes.size(); ++i) + testCaseFilter.append(testCaseNodes.at(i).toElement().attributeNode("name").value()); + + QDomNodeList metricNodes = doc.elementsByTagName("metric"); + for (int i = 0; i < metricNodes.size(); ++i) + metricFilter.append(metricNodes.at(i).toElement().attributeNode("name").value()); + + QDomNodeList platformNodes = doc.elementsByTagName("platform"); + for (int i = 0; i < platformNodes.size(); ++i) + platformFilter.append(platformNodes.at(i).toElement().attributeNode("name").value()); + + QDomNodeList hostNodes = doc.elementsByTagName("host"); + for (int i = 0; i < hostNodes.size(); ++i) + hostFilter.append(hostNodes.at(i).toElement().attributeNode("name").value()); + + QDomNodeList branchNodes = doc.elementsByTagName("branch"); + for (int i = 0; i < branchNodes.size(); ++i) + branchFilter.append(branchNodes.at(i).toElement().attributeNode("name").value()); + + reply += "\n<br /><b>Filters:</b><table>\n"; + appendToFilterTable(testCaseFilter, "Test case", &reply); + appendToFilterTable(metricFilter, "Metric", &reply); + appendToFilterTable(platformFilter, "Platform", &reply); + appendToFilterTable(hostFilter, "Host", &reply); + appendToFilterTable(branchFilter, "Branch", &reply); + reply += "</table>"; + + reply += "</fieldset>"; + + reply += "</td>\n"; + + // Right main table ... + + reply += "<td style=\"border:0px\">\n"; + + // 'Stats' table ... + + QDomNodeList statsNodes = doc.elementsByTagName("stats"); + QDomElement statsElem = statsNodes.at(0).toElement(); + + reply += "<fieldset><legend style=\"font-size:12\">Statistics</legend>"; + + const int regressed = statsElem.attributeNode("regressed").value().toInt(&ok); + Q_ASSERT(ok); + const int unchanged = statsElem.attributeNode("unchanged").value().toInt(&ok); + Q_ASSERT(ok); + const int improved = statsElem.attributeNode("improved").value().toInt(&ok); + Q_ASSERT(ok); + const int unstable = statsElem.attributeNode("unstable").value().toInt(&ok); + Q_ASSERT(ok); + const int total = regressed + unchanged + improved + unstable; + const int usZeroCount = statsElem.attributeNode("usZeroCount").value().toInt(&ok); + Q_ASSERT(ok); + const int usLowFromPosCount = + statsElem.attributeNode("usLowFromPosCount").value().toInt(&ok); + Q_ASSERT(ok); + const int usLowSFCount = statsElem.attributeNode("usLowSFCount").value().toInt(&ok); + Q_ASSERT(ok); + const int usLowLFCount = statsElem.attributeNode("usLowLFCount").value().toInt(&ok); + Q_ASSERT(ok); + const int usHighMaxLDCount = statsElem.attributeNode("usHighMaxLDCount").value().toInt(&ok); + Q_ASSERT(ok); + + const QString usColor("#ffdddd"); + + reply += "<br /><table style=\"text-align:right\">\n"; + + reply += QString("<tr><td><b>Total:</b></td><td><b>%1</b></td><td><b>100.00 %</b></td></tr>\n") + .arg(total); + if (total > 0) { + reply += QString("<tr><td>Regressed:</td><td>%1</td><td>%2 %</td></tr>\n") + .arg(regressed).arg(QString().setNum(100 * (regressed / qreal(total)), 'f', 2)); + reply += QString("<tr><td>Unchanged:</td><td>%1</td><td>%2 %</td></tr>\n") + .arg(unchanged).arg(QString().setNum(100 * (unchanged / qreal(total)), 'f', 2)); + reply += QString("<tr><td>Improved:</td><td>%1</td><td>%2 %</td></tr>\n") + .arg(improved).arg(QString().setNum(100 * (improved / qreal(total)), 'f', 2)); + reply += QString( + "<tr style=\"background-color:%1\"><td>Unstable:</td><td>%2</td>" + "<td>%3 %</td></tr>\n") + .arg(usColor).arg(unstable) + .arg(QString().setNum(100 * (unstable / qreal(total)), 'f', 2)); + } + + reply += "</table>\n"; + + if (unstable > 0) { + reply += "<br /><b>Categories of instabilities:</b>"; + reply += QString("<table style=\"text-align:right; background-color:%1\">\n").arg(usColor); + reply += QString( + "<tr><td>Zero existence:</td><td>%1</td><td>%2 %</td></tr>\n") + .arg(usZeroCount).arg(QString().setNum(100 * (usZeroCount / qreal(unstable)), 'f', 2)); + reply += QString("<tr><td>No data before 'from' time:</td><td>%1</td><td>%2 %</td></tr>\n") + .arg(usLowFromPosCount) + .arg(QString().setNum(100 * (usLowFromPosCount / qreal(unstable)), 'f', 2)); + reply += QString( + "<tr><td>Stable fraction too low:</td><td>%1</td><td>%2 %</td></tr>\n") + .arg(usLowSFCount) + .arg(QString().setNum(100 * (usLowSFCount / qreal(unstable)), 'f', 2)); + reply += QString( + "<tr><td>Unique level fraction too low:</td><td>%1</td><td>%2 %</td></tr>\n") + .arg(usLowLFCount) + .arg(QString().setNum(100 * (usLowLFCount / qreal(unstable)), 'f', 2)); + reply += QString( + "<tr><td>Max level difference too high:</td><td>%1</td><td>%2 %</td></tr>\n") + .arg(usHighMaxLDCount) + .arg(QString().setNum(100 * (usHighMaxLDCount / qreal(unstable)), 'f', 2)); + reply += "</table>\n"; + } + + reply += "</fieldset>"; + + reply += "</td>\n"; + + reply += "</tr>\n"; + reply += "<table>\n"; + + reply += "</body></html>"; + + BMMisc::printHTMLOutput(reply); +} diff --git a/src/bm/bmrequest.h b/src/bm/bmrequest.h index 2b97b65..8d90d88 100644 --- a/src/bm/bmrequest.h +++ b/src/bm/bmrequest.h @@ -598,9 +598,6 @@ private: void handleReply_JSON(const QStringList &args) const { Q_UNUSED(args); /* 2 B DONE! */ } void handleReply_HTML(const QStringList &args) const; void handleReply_Image(const QStringList &args) const; - - static void appendToFilterTable( - const QStringList &filter, const QString &filterName, QString *reply); }; // ### 2 B DOCUMENTED (in bmclient.html and bmproto.html) @@ -747,4 +744,45 @@ private: void handleReply_Image(const QStringList &args) const; }; +// ### 2 B DOCUMENTED (in bmclient.html and bmproto.html) +class BMRequest_ASFStatsGetValues : public BMRequest +{ +public: + BMRequest_ASFStatsGetValues( + const int medianWinSize, const QString &cacheKey, const QStringList &testCaseFilter, + const QStringList &metricFilter, const QStringList &platformFilter, + const QStringList &hostFilter, const QStringList &branchFilter, + const int fromTimestamp, const int toTimestamp, const qreal diffTol, const int stabTol, + const qreal sfTol, const qreal lfTol, const qreal maxLDTol) + : medianWinSize(medianWinSize), cacheKey(cacheKey), testCaseFilter(testCaseFilter) + , metricFilter(metricFilter), platformFilter(platformFilter), hostFilter(hostFilter) + , branchFilter(branchFilter), fromTimestamp(fromTimestamp), toTimestamp(toTimestamp) + , diffTol(diffTol), stabTol(stabTol), sfTol(sfTol), lfTol(lfTol), maxLDTol(maxLDTol) + {} + BMRequest_ASFStatsGetValues(const QDomDocument &doc) : BMRequest(doc) {} + +private: + mutable int medianWinSize; + QString cacheKey; + mutable QStringList testCaseFilter; + mutable QStringList metricFilter; + mutable QStringList platformFilter; + mutable QStringList hostFilter; + mutable QStringList branchFilter; + mutable int fromTimestamp; + mutable int toTimestamp; + mutable qreal diffTol; + mutable qreal stabTol; + mutable qreal sfTol; + mutable qreal lfTol; + mutable qreal maxLDTol; + + QString name() const { return "ASFStatsGetValues"; } + QByteArray toRequestBuffer(QString *error); + QByteArray toReplyBuffer(); + void handleReply_Raw(const QStringList &args) const { Q_UNUSED(args); /* 2 B DONE! */ }; + void handleReply_JSON(const QStringList &args) const { Q_UNUSED(args); /* 2 B DONE! */ } + void handleReply_HTML(const QStringList &args) const; +}; + #endif // BMREQUEST_H diff --git a/src/bm/resulthistoryinfo.cpp b/src/bm/resulthistoryinfo.cpp index cb18924..0e2d23b 100644 --- a/src/bm/resulthistoryinfo.cpp +++ b/src/bm/resulthistoryinfo.cpp @@ -111,16 +111,6 @@ void ResultHistoryInfo::markOutliers() const } // ### 2 B DOCUMENTED! -static void appendMaxESSStats(int seqSize, int *total, int *stable, int stabTolerance) -{ - if (seqSize <= 0) - return; - (*total)++; - if (seqSize >= stabTolerance) - (*stable)++; -} - -// ### 2 B DOCUMENTED! bool ResultHistoryInfo::equal(int i, int j, int diffTolerance) const { if ((i < 0) || (j < 0) || (i >= values.size()) || (j >= values.size())) @@ -142,8 +132,49 @@ bool ResultHistoryInfo::equal(int i, int j, int diffTolerance) const } // ### 2 B DOCUMENTED! +void ResultHistoryInfo::startSubsequence( + int pos, qreal diffTolerance, QMap<int, int> *uniqueLevels, qreal *minLevel, + qreal *maxLevel, bool first) const +{ + Q_ASSERT(pos >= 0); + Q_ASSERT(pos < values.size()); + + const qreal v = values.at(pos); + + if (first) { + *minLevel = *maxLevel = v; + } else { + *minLevel = qMin(*minLevel, v); + *maxLevel = qMax(*maxLevel, v); + } + + QMap<int, int>::iterator it; + for (it = uniqueLevels->begin(); it != uniqueLevels->end(); ++it) + if (equal(pos, it.key(), diffTolerance)) + break; + + if (it == uniqueLevels->end()) { + // Insert new unique level ... + uniqueLevels->insert(pos, 1); + } else { + // Add to existing unique level ... + ++(it.value()); + } +} + +// ### 2 B DOCUMENTED! +void ResultHistoryInfo::endSubsequence(int seqSize, int *total, int *stable, int stabTolerance) +{ + if (seqSize <= 0) + return; + (*total)++; + if (seqSize >= stabTolerance) + (*stable)++; +} + +// ### 2 B DOCUMENTED! void ResultHistoryInfo::computeMaxESSStats( - qreal diffTolerance, int stabTolerance, int *total, int *stable) + qreal diffTolerance, int stabTolerance, int *total, int *stable) const { *total = *stable = 0; @@ -154,7 +185,7 @@ void ResultHistoryInfo::computeMaxESSStats( if (isOutlier(i)) { if (i == values.size() - 1) - appendMaxESSStats(seqSize, total, stable, stabTolerance); + endSubsequence(seqSize, total, stable, stabTolerance); } else { @@ -163,14 +194,52 @@ void ResultHistoryInfo::computeMaxESSStats( if (basePos == -1) { basePos = i; } else if (!equal(basePos, i, diffTolerance)) { - appendMaxESSStats(seqSize, total, stable, stabTolerance); + endSubsequence(seqSize, total, stable, stabTolerance); seqSize = 1; basePos = i; } if (i == values.size() - 1) - appendMaxESSStats(seqSize, total, stable, stabTolerance); + endSubsequence(seqSize, total, stable, stabTolerance); + } + + } +} + +// ### 2 B DOCUMENTED! +void ResultHistoryInfo::computeStabilityStats( + qreal diffTolerance, int stabTolerance, bool *zerosFound, int *total, int *stable, + int *uniqueLevels, qreal *minLevel, qreal *maxLevel) const +{ + *zerosFound = false; + *total = *stable = *uniqueLevels = 0; + *minLevel = *maxLevel = 0; + + QMap<int, int> uniqueLevels_; // <pos, count> + + int basePos = -1; + int seqSize = 0; + + for (int i = 0; i < values.size(); ++i) { + + ++seqSize; + + if ((!(*zerosFound)) && (values.at(i) == 0.0)) + *zerosFound = true; + + if (basePos == -1) { + basePos = i; + startSubsequence(basePos, diffTolerance, &uniqueLevels_, minLevel, maxLevel, true); + } else if (!equal(basePos, i, diffTolerance)) { + endSubsequence(seqSize, total, stable, stabTolerance); + seqSize = 1; + basePos = i; + startSubsequence(basePos, diffTolerance, &uniqueLevels_, minLevel, maxLevel); } + if (i == values.size() - 1) + endSubsequence(seqSize, total, stable, stabTolerance); } + + *uniqueLevels = uniqueLevels_.size(); } diff --git a/src/bm/resulthistoryinfo.h b/src/bm/resulthistoryinfo.h index a65757f..b5bdff0 100644 --- a/src/bm/resulthistoryinfo.h +++ b/src/bm/resulthistoryinfo.h @@ -26,6 +26,7 @@ #include <QString> #include <QList> +#include <QMap> #include <QBitArray> class ResultHistoryInfo { @@ -69,7 +70,10 @@ public: QString testFunction() const { return testFunction_; } QString dataTag() const { return dataTag_; } - void computeMaxESSStats(qreal diffTolerance, int stabTolerance, int *total, int *stable); + void computeMaxESSStats(qreal diffTolerance, int stabTolerance, int *total, int *stable) const; + void computeStabilityStats( + qreal diffTolerance, int stabTolerance, bool *zerosFound, int *total, int *stable, + int *uniqueLevels, qreal *minLevel, qreal *maxLevel) const; private: int bmcontextId_; @@ -96,6 +100,10 @@ private: void markOutliers() const; bool equal(int i, int j, int diffTolerance) const; + void startSubsequence( + int pos, qreal diffTolerance, QMap<int, int> *uniqueLevels, qreal *minLevel, + qreal *maxLevel, bool first = false) const; + static void endSubsequence(int seqSize, int *total, int *stable, int stabTolerance); }; #endif // RESULTHISTORYINFO_H diff --git a/src/bmclient/main.cpp b/src/bmclient/main.cpp index 238d0f4..104f481 100644 --- a/src/bmclient/main.cpp +++ b/src/bmclient/main.cpp @@ -116,6 +116,9 @@ private: BMRequest * createIndexPutConfigRequest(const QStringList &args, QString *error) const; BMRequest * createGetHistoriesRequest(const QStringList &args, QString *error) const; BMRequest * createGetIXHistoriesRequest(const QStringList &args, QString *error) const; + BMRequest * createASFStatsGetValuesRequest( + const QStringList &args, QString *error, + const QString &command = "asfstats get values") const; mutable BMRequest::OutputFormat explicitOutputFormat; mutable bool useExplicitOutputFormat; BMRequest::OutputFormat outputFormat() const; @@ -788,6 +791,32 @@ BMRequest * Executor::createRequest(const QStringList &args, QString *error) con setOutputFormat(BMRequest::HTML); return request; + } else if ( + (args.size() >= 3) && (args.at(0) == "asfstats") && (args.at(1) == "get") + && (args.at(2) == "values")) { + // --- 'asfstats get values' command --- + + return createASFStatsGetValuesRequest(args, error); + + // } else if ( + // (args.size() >= 3) && (args.at(0) == "asfstats") && (args.at(1) == "get") + // && (args.at(2) == "plot")) { + // // --- 'asfstats get plot' command --- + + // BMRequest *request = createASFStatsGetValuesRequest(args, error, "asfstats get plot"); + // setOutputFormat(BMRequest::Image); + // return request; + + } else if ( + (args.size() >= 3) && (args.at(0) == "asfstats") && (args.at(1) == "get") + && (args.at(2) == "detailspage")) { + // --- 'asfstats get detailspage' command --- + + BMRequest *request = + createASFStatsGetValuesRequest(args, error, "asfstats get detailspage"); + setOutputFormat(BMRequest::HTML); + return request; + } else if ((args.size() >= 2) && (args.at(0) == "get") && (args.at(1) == "detailspage")) { // --- 'get detailspage' command --- @@ -1269,6 +1298,126 @@ BMRequest * Executor::createGetIXHistoriesRequest(const QStringList &args, QStri return new BMRequest_GetIXHistories(evalTimestamp, medianWinSize, rankedInfos, cacheKey); } +BMRequest * Executor::createASFStatsGetValuesRequest( + const QStringList &args, QString *error, const QString &command) const +{ + Q_UNUSED(command); + + QStringList values; + + // Get median window size ... + int medianWinSize = -1; + if (BMMisc::getOption(args, "-medianwinsize", &values, 1, 0, error)) { + bool ok; + medianWinSize = values.at(0).toInt(&ok); + if ((!ok) || (medianWinSize < 1)) { + *error = "failed to extract median window size as a positive integer"; + return 0; + } + } else { + if (error->isEmpty()) + *error = "-medianwinsize option not found"; + return 0; + } + + // Get cache key ... + QString cacheKey; + if (BMMisc::getOption(args, "-cachekey", &values, 1, 0, error)) { + cacheKey = values.at(0).trimmed(); + bool ok; + cacheKey.toInt(&ok); + if (!ok) { + *error = "failed to extract cache key as an integer"; + return 0; + } + } else if (!error->isEmpty()) { + return 0; + } + + // Get filters ... + QStringList testCaseFilter; + if (!BMMisc::getMultiOption(args, "-testcase", &testCaseFilter, error)) + return 0; + QStringList metricFilter; + if (!BMMisc::getMultiOption(args, "-metric", &metricFilter, error)) + return 0; + QStringList platformFilter; + if (!BMMisc::getMultiOption(args, "-platform", &platformFilter, error)) + return 0; + QStringList hostFilter; + if (!BMMisc::getMultiOption(args, "-host", &hostFilter, error)) + return 0; + QStringList branchFilter; + if (!BMMisc::getMultiOption(args, "-branch", &branchFilter, error)) + return 0; + + // Get ASF-specific params ... + + // ... timestamps ... + + int fromTimestamp = -1; + if (BMMisc::getOption(args, "-fromtime", &values, 1, 0, error)) { + bool ok; + fromTimestamp = values.at(0).toInt(&ok); + if ((!ok) || (fromTimestamp < 0)) { + *error = "failed to extract 'from' time as a non-negative integer"; + return 0; + } + } else { + if (error->isEmpty()) + *error = QString("-fromtime option not found"); + return 0; + } + + int toTimestamp = -1; + if (BMMisc::getOption(args, "-totime", &values, 1, 0, error)) { + bool ok; + toTimestamp = values.at(0).toInt(&ok); + if ((!ok) || (toTimestamp < fromTimestamp)) { + *error = QString("failed to extract 'to' time as an integer < %1").arg(fromTimestamp); + return 0; + } + } else { + if (error->isEmpty()) + *error = QString("-totime option not found"); + return 0; + } + + // ... tolerance values ... + qreal diffTol = -1; + if (!BMMisc::getClampedPercentageOption(args, "-difftol", &diffTol, error)) + return 0; + + int stabTol = -1; + if (BMMisc::getOption(args, "-stabtol", &values, 1, 0, error)) { + bool ok; + stabTol = values.at(0).toInt(&ok); + if ((!ok) || (stabTol < 2)) { + *error = "failed to extract stability tolerance as an integer >= 2"; + return 0; + } + } else { + if (error->isEmpty()) + *error = QString("-stabtol option not found"); + return 0; + } + + qreal sfTol = -1; + if (!BMMisc::getClampedPercentageOption(args, "-sftol", &sfTol, error)) + return 0; + qreal lfTol = -1; + if (!BMMisc::getClampedPercentageOption(args, "-lftol", &lfTol, error)) + return 0; + qreal maxLDTol = -1; + if (!BMMisc::getDoubleOption(args, "-maxldtol", &maxLDTol, error)) + return 0; + + + return new BMRequest_ASFStatsGetValues( + medianWinSize, cacheKey, testCaseFilter, metricFilter, platformFilter, hostFilter, + branchFilter, fromTimestamp, toTimestamp, diffTol, stabTol, sfTol, lfTol, maxLDTol); +} + // ### 2 B DOCUMENTED! static void splitQuotedArgs(const QString &arg_s, QStringList *args) { @@ -1688,6 +1837,19 @@ class DirectExecutor : public Executor "get ixhistories detailspage <SAME AS 'get ixhistories'> except that the mandatory \\\n" " -stylesheet option is recognized\n" + << + "asfstats get values \\\n" + " -medianwinsize <...> -fromtime <...> -totime <...> -difftol <...> \\\n" + " -stabtol <...> -sftol <...> -lftol <...> -maxldtol <...> \\\n" + " [-testcase <...> -testcase <...> ...] [-metric <...> -metric <...> ...] \\\n" + " [-platform <...> -platform <...> ...] [-host <...> -host <...> ...] \\\n" + " [-branch <...> -branch <...> ...]\n" + + << + "asfstats get detailspage <SAME AS 'asfstats get values'> except that the \\\n" + " optional -httpHeader and -cachekey options, and the mandatory \\\n" + " -stylesheet option are recognized\n" + ; qDebug() << "\nNote: the -server option may be replaced by a BMSERVER=<host>:<port> " diff --git a/src/bmweb/indexsection.js b/src/bmweb/indexsection.js index f07c635..18cc52c 100644 --- a/src/bmweb/indexsection.js +++ b/src/bmweb/indexsection.js @@ -96,14 +96,11 @@ function updateComputeASFStatsLink(defaultMedianWinSize) url += " asfstats get detailspage"; url += " -stylesheet " + document.styleSheets[0].href; - // Add median window size ... medianWinSize = getIntFromTextInput("ixMedianWinSize", defaultMedianWinSize); document.getElementById("ixMedianWinSize").value = medianWinSize; - url += " -medianwinsize " + medianWinSize; - // Add filters ... url += createFilterOptionsString("-testcase", getCheckedFilterValues("ixTestCaseFilter")); url += createFilterOptionsString("-metric", getCheckedFilterValues("ixMetricFilter")); @@ -111,11 +108,18 @@ function updateComputeASFStatsLink(defaultMedianWinSize) url += createFilterOptionsString("-host", getCheckedFilterValues("ixHostFilter")); url += createFilterOptionsString("-branch", getCheckedFilterValues("ixBranchFilter")); + // Add ASF-specific parameters ... + url += " -fromtime " + document.getElementById("ixASFStatsFromTime").value + + " -totime " + document.getElementById("ixASFStatsToTime").value + + " -difftol " + document.getElementById("ixASFStatsDiffTol").value + + " -stabtol " + selectedOptions("ixASFStatsStabTol")[0] + + " -sftol " + document.getElementById("ixASFStatsSFTol").value + + " -lftol " + document.getElementById("ixASFStatsLFTol").value + + " -maxldtol " + document.getElementById("ixASFStatsMaxLDTol").value; // Update link ... link = document.getElementById("ixComputeASFStatsLink"); -// link.setAttribute("href", url); - link.setAttribute("href", "http://www.klokka.no"); // 4 NOW! + link.setAttribute("href", url); link.setAttribute("target", "_blank"); // prevent page from opening in same window } @@ -653,14 +657,9 @@ function IndexSection() legend.appendChild(document.createTextNode("Data Quality Statistics")); legend.innerHTML = "ASF Scorecard Statistics"; legend.setAttribute("style", "font-size:12"); - table = fieldset.appendChild(document.createElement("table")); - table.setAttribute("style", "border:0px; padding:0px"); // ... 'compute ASF stats' link ... - tr = table.insertRow(table.rows.length); - td = tr.insertCell(0); - td.setAttribute("style", "border:0px; padding:2px"); - link = td.appendChild(document.createElement("a")); + link = fieldset.appendChild(document.createElement("a")); link.id = "ixComputeASFStatsLink"; link.appendChild(document.createTextNode("Compute")); link.setAttribute("class", "ixLink_index"); @@ -671,8 +670,12 @@ function IndexSection() + defaultMedianWinSize + ")" ); + fieldset.appendChild(document.createElement("br")); + + table = fieldset.appendChild(document.createElement("table")); + table.setAttribute("style", "border:0px; padding:0px"); - // ... 'from time' ... + // ... 'from timestamp' ... tr = table.insertRow(table.rows.length); td = tr.insertCell(0); td.innerHTML = "From timestamp (secs since 1970): "; @@ -680,10 +683,10 @@ function IndexSection() td = tr.insertCell(1); input = td.appendChild(document.createElement("input")); input.setAttribute("type", "text"); - input.setAttribute("id", "ixASFFromTime"); + input.setAttribute("id", "ixASFStatsFromTime"); input.setAttribute("value", "-1"); - // ... 'to time' ... + // ... 'to timestamp' ... tr = table.insertRow(table.rows.length); td = tr.insertCell(0); td.innerHTML = "To timestamp (secs since 1970): "; @@ -691,9 +694,72 @@ function IndexSection() td = tr.insertCell(1); input = td.appendChild(document.createElement("input")); input.setAttribute("type", "text"); - input.setAttribute("id", "ixASFToTime"); + input.setAttribute("id", "ixASFStatsToTime"); input.setAttribute("value", "-1"); + // ... 'difference tolerance' ... + tr = table.insertRow(table.rows.length); + td = tr.insertCell(0); + td.setAttribute("style", "text-align:right"); + td.innerHTML = "Difference tolerance (%): "; + td = tr.insertCell(1); + td.setAttribute("style", "text-align:left"); + input = td.appendChild(document.createElement("input")); + input.setAttribute("type", "text"); + input.setAttribute("id", "ixASFStatsDiffTol"); + input.setAttribute("Value", "10"); + + // ... 'stability tolerance' ... + tr = table.insertRow(table.rows.length); + td = tr.insertCell(0); + td.setAttribute("style", "text-align:right"); + td.innerHTML = "Stability tolerance: "; + td = tr.insertCell(1); + td.setAttribute("style", "text-align:left"); + select = td.appendChild(document.createElement("select")); + select.setAttribute("id", "ixASFStatsStabTol"); + select.options[0] = new Option("2", "2"); + select.options[1] = new Option("3", "3"); + select.options[2] = new Option("4", "4"); + select.options[3] = new Option("5", "5"); + select.options[4] = new Option("6", "6"); + select.options[2].selected = true; + + // ... 'stability fraction tolerance' ... + tr = table.insertRow(table.rows.length); + td = tr.insertCell(0); + td.setAttribute("style", "text-align:right"); + td.innerHTML = "Stability fraction tolerance (%): "; + td = tr.insertCell(1); + td.setAttribute("style", "text-align:left"); + input = td.appendChild(document.createElement("input")); + input.setAttribute("type", "text"); + input.setAttribute("id", "ixASFStatsSFTol"); + input.setAttribute("Value", "10"); + + // ... 'level fraction tolerance' ... + tr = table.insertRow(table.rows.length); + td = tr.insertCell(0); + td.setAttribute("style", "text-align:right"); + td.innerHTML = "Unique level fraction tolerance (%): "; + td = tr.insertCell(1); + td.setAttribute("style", "text-align:left"); + input = td.appendChild(document.createElement("input")); + input.setAttribute("type", "text"); + input.setAttribute("id", "ixASFStatsLFTol"); + input.setAttribute("Value", "10"); + + // ... 'max level difference tolerance' ... + tr = table.insertRow(table.rows.length); + td = tr.insertCell(0); + td.setAttribute("style", "text-align:right"); + td.innerHTML = "Max level difference tolerance (%): "; + td = tr.insertCell(1); + td.setAttribute("style", "text-align:left"); + input = td.appendChild(document.createElement("input")); + input.setAttribute("type", "text"); + input.setAttribute("id", "ixASFStatsMaxLDTol"); + input.setAttribute("Value", "10"); // Create 'Configuration' section ... section.appendChild(document.createElement("br")); |
