Compiler: Aggregate and print aotstats

This patch enables the aggregation and printing of aotstats recorded by
qmlcachegen for compiled qml files. The aotstats files for individual
qml files are aggregated into module-level aotstats files and then into
one global aotstats file. This file is then presented into a more human
friendly format.

The all_aotstats target can be used to print the collected stats of all
the compiled files and modules.

Due to CMake configuration errors, the feature has temporarily been
disabled on Xcode. This should be fixed before soon.
Created QTBUG-125995.

Task-number: QTBUG-124667
Change-Id: I0c82142626743e9c1af98516c553f4dd7bc6da13
Reviewed-by: Fabian Kosmale <[email protected]>

9 files changed, 420 insertions, 0 deletions

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index ca83200cf5..56992dda9b 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -95,6 +95,7 @@ add_subdirectory(qmldom)
 
 # Build qmlcachegen now, so that we can use it in src/imports.
 if(QT_FEATURE_xmlstreamwriter)
+    add_subdirectory(../tools/qmlaotstats qmlaotstats)
     add_subdirectory(../tools/qmlcachegen qmlcachegen)
 endif()
 
diff --git a/src/qml/Qt6QmlMacros.cmake b/src/qml/Qt6QmlMacros.cmake
index 4eab6a2fad..07848342c8 100644
--- a/src/qml/Qt6QmlMacros.cmake
+++ b/src/qml/Qt6QmlMacros.cmake
@@ -945,6 +945,67 @@ Check https://doc.qt.io/qt-6/qt-cmake-policy-qtp0001.html for policy details."
             ")
         endif()
     endif()
+
+    if("${CMAKE_VERSION}" VERSION_GREATER_EQUAL "3.19.0" AND NOT CMAKE_GENERATOR STREQUAL "Xcode")
+        set(id qmlaotstats_aggregation)
+        cmake_language(DEFER DIRECTORY ${PROJECT_BINARY_DIR} GET_CALL ${id} call)
+
+        if("${call}" STREQUAL "")
+            cmake_language(EVAL CODE "cmake_language(DEFER DIRECTORY ${PROJECT_BINARY_DIR} "
+                "ID ${id} CALL _qt_internal_deferred_aggregate_aotstats_files ${target})")
+        endif()
+    else()
+        if(NOT TARGET all_aotstats)
+            if(CMAKE_GENERATOR STREQUAL "Xcode") #TODO: QTBUG-125995
+                add_custom_target(
+                    all_aotstats
+                    ${CMAKE_COMMAND} -E echo "aotstats is not supported on Xcode"
+                )
+            else()
+                add_custom_target(
+                    all_aotstats
+                    ${CMAKE_COMMAND} -E echo "aotstats is not supported on CMake versions < 3.19"
+                )
+            endif()
+        endif()
+    endif()
+endfunction()
+
+function(_qt_internal_deferred_aggregate_aotstats_files target)
+    get_property(module_aotstats_files GLOBAL PROPERTY "module_aotstats_files")
+    list(JOIN module_aotstats_files "\n" lines)
+    set(aotstats_list_file "${PROJECT_BINARY_DIR}/.rcc/qmlcache/all_aotstats.aotstatslist")
+    file(WRITE ${aotstats_list_file} ${lines})
+
+    set(all_aotstats_file ${PROJECT_BINARY_DIR}/.rcc/qmlcache/all_aotstats.aotstats)
+    set(formatted_stats_file ${PROJECT_BINARY_DIR}/.rcc/qmlcache/all_aotstats.txt)
+
+    _qt_internal_get_tool_wrapper_script_path(tool_wrapper)
+    add_custom_command(
+        OUTPUT
+            ${all_aotstats_file}
+            ${formatted_stats_file}
+        DEPENDS ${module_aotstats_files}
+        COMMAND
+            ${tool_wrapper}
+            $<TARGET_FILE:Qt6::qmlaotstats>
+            aggregate
+            ${aotstats_list_file}
+            ${all_aotstats_file}
+        COMMAND
+            ${tool_wrapper}
+            $<TARGET_FILE:Qt6::qmlaotstats>
+            format
+            ${all_aotstats_file}
+            ${formatted_stats_file}
+    )
+
+    if(NOT TARGET all_aotstats)
+        add_custom_target(all_aotstats
+            DEPENDS ${formatted_stats_file}
+            COMMAND ${CMAKE_COMMAND} -E cat ${formatted_stats_file}
+        )
+    endif()
 endfunction()
 
 function(_qt_internal_write_deferred_qmlls_ini_file)
@@ -2843,6 +2904,7 @@ function(qt6_target_qml_sources target)
             set(aotstats_file "")
             if("${qml_file_src}" MATCHES ".+\\.qml")
                 set(aotstats_file "${compiled_file}.aotstats")
+                list(APPEND aotstats_files ${aotstats_file})
             endif()
 
             _qt_internal_get_tool_wrapper_script_path(tool_wrapper)
@@ -2893,6 +2955,29 @@ function(qt6_target_qml_sources target)
         endif()
     endforeach()
 
+    if(NOT "${arg_URI}" STREQUAL "")
+        list(JOIN aotstats_files "\n" aotstats_files_lines)
+        set(module_aotstats_list_file "${CMAKE_CURRENT_BINARY_DIR}/.rcc/qmlcache/module_${arg_URI}.aotstatslist")
+        file(WRITE ${module_aotstats_list_file} ${aotstats_files_lines})
+
+        # Aggregate qml file aotstats into module-level aotstats
+        _qt_internal_get_tool_wrapper_script_path(tool_wrapper)
+        set(output "${CMAKE_CURRENT_BINARY_DIR}/.rcc/qmlcache/module_${arg_URI}.aotstats")
+        add_custom_command(
+            OUTPUT ${output}
+            DEPENDS ${aotstats_files}
+            COMMAND
+                ${tool_wrapper}
+                $<TARGET_FILE:Qt6::qmlaotstats>
+                aggregate
+                ${module_aotstats_list_file}
+                ${output}
+        )
+
+        # Collect module-level aotstats files for later aggregation at the project level
+        set_property(GLOBAL APPEND PROPERTY "module_aotstats_files" ${output})
+    endif()
+
     if(ANDROID)
         _qt_internal_collect_qml_root_paths("${target}" ${arg_QML_FILES})
     endif()
diff --git a/src/qmlcompiler/CMakeLists.txt b/src/qmlcompiler/CMakeLists.txt
index 43d044ac88..445e5c7c12 100644
--- a/src/qmlcompiler/CMakeLists.txt
+++ b/src/qmlcompiler/CMakeLists.txt
@@ -17,6 +17,7 @@ qt_internal_add_module(QmlCompiler
         qqmljscompilepass_p.h
         qqmljscompiler.cpp qqmljscompiler_p.h
         qqmljscompilerstats.cpp qqmljscompilerstats_p.h
+        qqmljscompilerstatsreporter.cpp qqmljscompilerstatsreporter_p.h
         qqmljsfunctioninitializer.cpp qqmljsfunctioninitializer_p.h
         qqmljsimporter.cpp qqmljsimporter_p.h
         qqmljsimportvisitor.cpp qqmljsimportvisitor_p.h
diff --git a/src/qmlcompiler/qqmljscompilerstats.cpp b/src/qmlcompiler/qqmljscompilerstats.cpp
index ac926b942f..b65afd412f 100644
--- a/src/qmlcompiler/qqmljscompilerstats.cpp
+++ b/src/qmlcompiler/qqmljscompilerstats.cpp
@@ -26,6 +26,60 @@ bool QQmlJS::AotStatsEntry::operator<(const AotStatsEntry &other) const
     return line < other.line;
 }
 
+void AotStats::insert(AotStats other)
+{
+    for (const auto &[moduleUri, moduleStats] : other.m_entries.asKeyValueRange()) {
+        m_entries[moduleUri].insert(moduleStats);
+    }
+}
+
+std::optional<QList<QString>> extractAotstatsFilesList(const QString &aotstatsListPath)
+{
+    QFile aotstatsListFile(aotstatsListPath);
+    if (!aotstatsListFile.open(QIODevice::ReadOnly | QIODevice::ReadOnly | QIODevice::Text)) {
+        qDebug().noquote() << u"Could not open \"%1\" for reading"_s.arg(aotstatsListFile.fileName());
+        return std::nullopt;
+    }
+
+    QStringList aotstatsFiles;
+    QTextStream stream(&aotstatsListFile);
+    while (!stream.atEnd())
+        aotstatsFiles.append(stream.readLine());
+
+    return aotstatsFiles;
+}
+
+std::optional<AotStats> AotStats::parseAotstatsFile(const QString &aotstatsPath)
+{
+    QFile file(aotstatsPath);
+    if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
+        qDebug().noquote() << u"Could not open \"%1\""_s.arg(aotstatsPath);
+        return std::nullopt;
+    }
+
+    return AotStats::fromJsonDocument(QJsonDocument::fromJson(file.readAll()));
+}
+
+std::optional<AotStats> AotStats::aggregateAotstatsList(const QString &aotstatsListPath)
+{
+    const auto aotstatsFiles = extractAotstatsFilesList(aotstatsListPath);
+    if (!aotstatsFiles.has_value())
+        return std::nullopt;
+
+    AotStats aggregated;
+    if (aotstatsFiles->empty())
+        return aggregated;
+
+    for (const auto &aotstatsFile : aotstatsFiles.value()) {
+        auto parsed = parseAotstatsFile(aotstatsFile);
+        if (!parsed.has_value())
+            return std::nullopt;
+        aggregated.insert(parsed.value());
+    }
+
+    return aggregated;
+}
+
 AotStats AotStats::fromJsonDocument(const QJsonDocument &document)
 {
     QJsonArray modulesArray = document.array();
diff --git a/src/qmlcompiler/qqmljscompilerstats_p.h b/src/qmlcompiler/qqmljscompilerstats_p.h
index fd4f7273bc..abd89c4a20 100644
--- a/src/qmlcompiler/qqmljscompilerstats_p.h
+++ b/src/qmlcompiler/qqmljscompilerstats_p.h
@@ -51,9 +51,13 @@ public:
     }
 
     void addEntry(const QString &moduleId, const QString &filepath, AotStatsEntry entry);
+    void insert(AotStats other);
 
     bool saveToDisk(const QString &filepath) const;
 
+    static std::optional<AotStats> parseAotstatsFile(const QString &aotstatsPath);
+    static std::optional<AotStats> aggregateAotstatsList(const QString &aotstatsListPath);
+
     static AotStats fromJsonDocument(const QJsonDocument &);
     QJsonDocument toJsonDocument() const;
 
diff --git a/src/qmlcompiler/qqmljscompilerstatsreporter.cpp b/src/qmlcompiler/qqmljscompilerstatsreporter.cpp
new file mode 100644
index 0000000000..1cb17f71fe
--- /dev/null
+++ b/src/qmlcompiler/qqmljscompilerstatsreporter.cpp
@@ -0,0 +1,118 @@
+// Copyright (C) 2024 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+
+#include "qqmljscompilerstatsreporter_p.h"
+
+#include <QFileInfo>
+
+QT_BEGIN_NAMESPACE
+
+namespace QQmlJS {
+
+using namespace Qt::StringLiterals;
+
+AotStatsReporter::AotStatsReporter(const AotStats &aotstats) : m_aotstats(aotstats)
+{
+    for (const auto &[moduleUri, fileEntries] : aotstats.entries().asKeyValueRange()) {
+        for (const auto &[filepath, statsEntries] : fileEntries.asKeyValueRange()) {
+            for (const auto &entry : statsEntries) {
+                m_fileCounters[moduleUri][filepath].codegens += 1;
+                if (entry.codegenSuccessful) {
+                    m_fileCounters[moduleUri][filepath].successes += 1;
+                    m_successDurations.append(entry.codegenDuration);
+                }
+            }
+            m_moduleCounters[moduleUri].codegens += m_fileCounters[moduleUri][filepath].codegens;
+            m_moduleCounters[moduleUri].successes += m_fileCounters[moduleUri][filepath].successes;
+        }
+        m_totalCounters.codegens += m_moduleCounters[moduleUri].codegens;
+        m_totalCounters.successes += m_moduleCounters[moduleUri].successes;
+    }
+}
+
+void AotStatsReporter::formatDetailedStats(QTextStream &s) const
+{
+    s << "############ AOT COMPILATION STATS ############\n";
+    for (const auto &[moduleUri, fileStats] : m_aotstats.entries().asKeyValueRange()) {
+        s << u"Module %1:\n"_s.arg(moduleUri);
+        if (fileStats.empty()) {
+            s << "No attempts at compiling a binding or function\n";
+            continue;
+        }
+
+        for (const auto &[filename, entries] : fileStats.asKeyValueRange()) {
+            s << u"--File %1\n"_s.arg(filename);
+            if (entries.empty()) {
+                s << "  No attempts at compiling a binding or function\n";
+                continue;
+            }
+
+            int successes = m_fileCounters[moduleUri][filename].successes;
+            s << "  " << formatSuccessRate(entries.size(), successes) << "\n";
+
+            for (const auto &stat : entries) {
+                s << u"    %1: [%2:%3:%4]\n"_s.arg(stat.functionName)
+                                .arg(QFileInfo(filename).fileName())
+                                .arg(stat.line)
+                                .arg(stat.column);
+                s << u"      result: "_s << (stat.codegenSuccessful
+                                                     ? u"Success\n"_s
+                                                     : u"Error: "_s + stat.errorMessage + u'\n');
+                s << u"      duration: %1us\n"_s.arg(stat.codegenDuration.count());
+            }
+            s << "\n";
+        }
+    }
+}
+
+void AotStatsReporter::formatSummary(QTextStream &s) const
+{
+    s << "############ AOT COMPILATION STATS SUMMARY ############\n";
+    if (m_totalCounters.codegens == 0) {
+        s << "No attempted compilations to Cpp for bindings or functions.\n";
+        return;
+    }
+
+    for (const auto &moduleUri : m_aotstats.entries().keys()) {
+        const auto &counters = m_moduleCounters[moduleUri];
+        s << u"Module %1: "_s.arg(moduleUri)
+          << formatSuccessRate(counters.codegens, counters.successes) << "\n";
+    }
+
+    s << "Total results: " << formatSuccessRate(m_totalCounters.codegens, m_totalCounters.successes);
+    s << "\n";
+
+    if (m_totalCounters.successes != 0) {
+        auto totalDuration = std::accumulate(m_successDurations.cbegin(), m_successDurations.cend(),
+                                             std::chrono::microseconds(0));
+        const auto averageDuration = totalDuration.count() / m_totalCounters.successes;
+        s << u"Successful codegens took an average of %1us\n"_s.arg(averageDuration);
+    }
+}
+
+QString AotStatsReporter::format() const
+{
+    QString output;
+    QTextStream s(&output);
+
+    formatDetailedStats(s);
+    formatSummary(s);
+
+    return output;
+}
+
+QString AotStatsReporter::formatSuccessRate(int codegens, int successes) const
+{
+    if (codegens == 0)
+        return u"No attempted compilations"_s;
+
+    return u"%1 of %2 (%3%4) bindings or functions compiled to Cpp successfully"_s
+            .arg(successes)
+            .arg(codegens)
+            .arg(double(successes) / codegens * 100, 0, 'g', 4)
+            .arg(u"%"_s);
+}
+
+} // namespace QQmlJS
+
+QT_END_NAMESPACE
diff --git a/src/qmlcompiler/qqmljscompilerstatsreporter_p.h b/src/qmlcompiler/qqmljscompilerstatsreporter_p.h
new file mode 100644
index 0000000000..9acf4adac8
--- /dev/null
+++ b/src/qmlcompiler/qqmljscompilerstatsreporter_p.h
@@ -0,0 +1,57 @@
+// Copyright (C) 2024 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+
+#ifndef QQMLJSCOMPILERSTATSREPORTER_P_H
+#define QQMLJSCOMPILERSTATSREPORTER_P_H
+
+//
+//  W A R N I N G
+//  -------------
+//
+// This file is not part of the Qt API.  It exists purely as an
+// implementation detail.  This header file may change from version to
+// version without notice, or even be removed.
+//
+// We mean it.
+
+#include <QTextStream>
+
+#include <qtqmlcompilerexports.h>
+
+#include <private/qqmljscompilerstats_p.h>
+
+QT_BEGIN_NAMESPACE
+
+namespace QQmlJS {
+
+class Q_QMLCOMPILER_EXPORT AotStatsReporter
+{
+public:
+    AotStatsReporter(const QQmlJS::AotStats &);
+
+    QString format() const;
+
+private:
+    void formatDetailedStats(QTextStream &) const;
+    void formatSummary(QTextStream &) const;
+    QString formatSuccessRate(int codegens, int successes) const;
+
+    const AotStats &m_aotstats;
+
+    struct Counters
+    {
+        int successes = 0;
+        int codegens = 0;
+    };
+
+    Counters m_totalCounters;
+    QHash<QString, Counters> m_moduleCounters;
+    QHash<QString, QHash<QString, Counters>> m_fileCounters;
+    QList<std::chrono::microseconds> m_successDurations;
+};
+
+} // namespace QQmlJS
+
+QT_END_NAMESPACE
+
+#endif // QQMLJSCOMPILERSTATSREPORTER_P_H
diff --git a/tools/qmlaotstats/CMakeLists.txt b/tools/qmlaotstats/CMakeLists.txt
new file mode 100644
index 0000000000..1511f19e4b
--- /dev/null
+++ b/tools/qmlaotstats/CMakeLists.txt
@@ -0,0 +1,17 @@
+# Copyright (C) 2024 The Qt Company Ltd.
+# SPDX-License-Identifier: BSD-3-Clause
+
+qt_get_tool_target_name(target_name qmlaotstats)
+qt_internal_add_tool(${target_name}
+    TARGET_DESCRIPTION "QML ahead-of-time compiler statistics aggregator"
+    TOOLS_TARGET Qml # special case
+    INSTALL_DIR "${INSTALL_LIBEXECDIR}"
+    SOURCES
+        main.cpp
+    LIBRARIES
+        Qt::CorePrivate
+        Qt::QmlPrivate
+        Qt::QmlCompilerPrivate
+        Qt::QmlToolingSettingsPrivate
+)
+qt_internal_return_unless_building_tools()
diff --git a/tools/qmlaotstats/main.cpp b/tools/qmlaotstats/main.cpp
new file mode 100644
index 0000000000..24b34efec3
--- /dev/null
+++ b/tools/qmlaotstats/main.cpp
@@ -0,0 +1,83 @@
+// Copyright (C) 2024 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+
+#include <QCommandLineParser>
+#include <QCoreApplication>
+#include <QDir>
+#include <QFile>
+#include <QFileInfo>
+#include <QJsonArray>
+#include <QJsonDocument>
+#include <QJsonObject>
+
+#include <private/qqmljscompilerstats_p.h>
+#include <private/qqmljscompilerstatsreporter_p.h>
+
+using namespace Qt::Literals::StringLiterals;
+
+bool saveFormattedStats(const QString &stats, const QString &outputPath)
+{
+    QString directory = QFileInfo(outputPath).dir().path();
+    if (!QDir().mkpath(directory)) {
+        qDebug() << "Could not ensure the existence of" << directory;
+        return false;
+    }
+
+    QFile outputFile(outputPath);
+    if (!outputFile.open(QIODevice::Text | QIODevice::WriteOnly)) {
+        qDebug() << "Could not open file" << outputPath;
+        return false;
+    }
+
+    if (outputFile.write(stats.toLatin1()) == -1) {
+        qDebug() << "Could not write formatted AOT stats to" << outputPath;
+        return false;
+    } else {
+        qDebug() << "Formatted AOT stats saved to" << outputPath;
+    }
+
+    return true;
+}
+
+int main(int argc, char **argv)
+{
+    QCoreApplication app(argc, argv);
+    QCoreApplication::setApplicationVersion(QLatin1String(QT_VERSION_STR));
+
+    QCommandLineParser parser;
+    parser.addHelpOption();
+    parser.setApplicationDescription("Internal development tool.");
+    parser.addPositionalArgument("mode", "Choose whether to aggregate or display aotstats files",
+                                 "[aggregate|format]");
+    parser.addPositionalArgument("input", "Aggregate mode: the aotstatslist file to aggregate. "
+                                          "Format mode: the aotstats file to display.");
+    parser.addPositionalArgument("output", "Aggregate mode: the path where to store the "
+                                           "aggregated aotstats. Format mode: the the path where "
+                                           "the formatted output will be saved.");
+    parser.process(app);
+
+    const auto &positionalArgs = parser.positionalArguments();
+    if (positionalArgs.size() != 3) {
+        qDebug().noquote() << parser.helpText();
+        return EXIT_FAILURE;
+    }
+
+    const auto &mode = positionalArgs.first();
+    if (mode == u"aggregate"_s) {
+        const auto aggregated = QQmlJS::AotStats::aggregateAotstatsList(positionalArgs[1]);
+        if (!aggregated.has_value())
+            return EXIT_FAILURE;
+        if (!aggregated->saveToDisk(positionalArgs[2]))
+            return EXIT_FAILURE;
+
+    } else if (mode == u"format"_s) {
+        const auto aotstats = QQmlJS::AotStats::parseAotstatsFile(positionalArgs[1]);
+        if (!aotstats.has_value())
+            return EXIT_FAILURE;
+        const QQmlJS::AotStatsReporter reporter(aotstats.value());
+        if (!saveFormattedStats(reporter.format(), positionalArgs[2]))
+            return EXIT_FAILURE;
+    }
+
+    return EXIT_SUCCESS;
+}