Add snippet extraction to shiboken

Add a snippet attribute to inject-code and conversion-rule
instructing shiboken to extract code from a source file using
annotations.

Task-number: PYSIDE-834
Change-Id: I576c4a48fe68e9d26fe46e324af5baa88a5c1d34
Reviewed-by: Cristian Maureira-Fredes <[email protected]>
Reviewed-by: Christian Tismer <[email protected]>

6 files changed, 103 insertions, 9 deletions

diff --git a/sources/shiboken2/ApiExtractor/doc/typesystem_manipulating_objects.rst b/sources/shiboken2/ApiExtractor/doc/typesystem_manipulating_objects.rst
index 531c4ece8..12b866ad7 100644
--- a/sources/shiboken2/ApiExtractor/doc/typesystem_manipulating_objects.rst
+++ b/sources/shiboken2/ApiExtractor/doc/typesystem_manipulating_objects.rst
@@ -11,6 +11,9 @@ inject-code
     given type or function, and it is a child of the :ref:`object-type`, :ref:`value-type`,
     :ref:`modify-function` and :ref:`add-function` nodes.
 
+    The code can be embedded into XML (be careful to use the correct XML entities
+    for characters like '<', '>', '&'):
+
     .. code-block:: xml
 
          <value-type>
@@ -20,6 +23,18 @@ inject-code
              </inject-code>
          </value-type>
 
+    or obtained from an external file:
+
+    .. code-block:: xml
+
+         <value-type>
+             <inject-code class="native | target | target-declaration"
+                 position="beginning | end" since="..."
+                 file="external_source.cpp"
+                 snippet="label"/>
+         </value-type>
+
+
     The ``class`` attribute specifies which module of the generated code that
     will be affected by the code injection. The ``class`` attribute accepts the
     following values:
@@ -28,6 +43,8 @@ inject-code
         * target: The binding code
         * target-declaration: The code will be injected into the generated header
           file containing the c++ wrapper class definition.
+        * file: The file name
+        * snippet: The snippet label (optional)
 
     If the ``position`` attribute is set to *beginning* (the default), the code
     is inserted at the beginning of the function. If it is set to *end*, the code
@@ -35,6 +52,16 @@ inject-code
 
     The ``since`` attribute specify the API version where this code was injected.
 
+    If a ``snippet`` label is given, the code between annotations of the form
+
+    .. code-block:: c++
+
+        // @snippet label
+        ...
+        // @snippet label
+
+    will be extracted.
+
 modify-field
 ^^^^^^^^^^^^
 
@@ -152,3 +179,4 @@ conversion-rule
 
     .. note:: You can also use the conversion-rule node to specify :ref:`how the conversion of a single function argument should be done in a function <conversion-rule>`.
 
+    The ``file`` and ``snippet`` attributes are also supported (see :ref:`inject-code` nodes).
diff --git a/sources/shiboken2/ApiExtractor/tests/injectedcode.txt b/sources/shiboken2/ApiExtractor/tests/injectedcode.txt
new file mode 100644
index 000000000..872898810
--- /dev/null
+++ b/sources/shiboken2/ApiExtractor/tests/injectedcode.txt
@@ -0,0 +1,5 @@
+// Bla
+// @snippet label
+code line
+// @snippet label
+// Bla
diff --git a/sources/shiboken2/ApiExtractor/tests/testcodeinjection.cpp b/sources/shiboken2/ApiExtractor/tests/testcodeinjection.cpp
index 68da25373..9f71b495a 100644
--- a/sources/shiboken2/ApiExtractor/tests/testcodeinjection.cpp
+++ b/sources/shiboken2/ApiExtractor/tests/testcodeinjection.cpp
@@ -34,17 +34,43 @@
 #include <abstractmetalang.h>
 #include <typesystem.h>
 
-void TestCodeInjections::testReadFileUtf8()
+void TestCodeInjections::testReadFile_data()
 {
+    QTest::addColumn<QString>("filePath");
+    QTest::addColumn<QString>("snippet");
+    QTest::addColumn<QString>("expected");
+
+    QTest::newRow("utf8")
+        << QString::fromLatin1(":/utf8code.txt")
+        << QString()
+        << QString::fromUtf8("\xC3\xA1\xC3\xA9\xC3\xAD\xC3\xB3\xC3\xBA");
+
+    QTest::newRow("snippet")
+        << QString::fromLatin1(":/injectedcode.txt")
+        << QString::fromLatin1("label")
+        << QString::fromLatin1("code line");
+}
+
+void TestCodeInjections::testReadFile()
+{
+    QFETCH(QString, filePath);
+    QFETCH(QString, snippet);
+    QFETCH(QString, expected);
+
     const char* cppCode ="struct A {};\n";
     int argc = 0;
     char *argv[] = {NULL};
     QCoreApplication app(argc, argv);
+
+    QString attribute = QLatin1String("file='") + filePath + QLatin1Char('\'');
+    if (!snippet.isEmpty())
+        attribute += QLatin1String(" snippet='") + snippet + QLatin1Char('\'');
+
     QString xmlCode = QLatin1String("\
     <typesystem package=\"Foo\">\n\
         <value-type name='A'>\n\
-            <conversion-rule file=':/utf8code.txt'/>\n\
-            <inject-code class='target' file=':/utf8code.txt'/>\n\
+            <conversion-rule ") + attribute + QLatin1String("/>\n\
+            <inject-code class='target' ") + attribute + QLatin1String("/>\n\
         </value-type>\n\
         <value-type name='A::B'/>\n\
     </typesystem>\n");
@@ -54,10 +80,9 @@ void TestCodeInjections::testReadFileUtf8()
     const AbstractMetaClass *classA = AbstractMetaClass::findClass(classes, QLatin1String("A"));
     QCOMPARE(classA->typeEntry()->codeSnips().count(), 1);
     QString code = classA->typeEntry()->codeSnips().first().code();
-    QString utf8Data = QString::fromUtf8("\xC3\xA1\xC3\xA9\xC3\xAD\xC3\xB3\xC3\xBA");
-    QVERIFY(code.indexOf(utf8Data) != -1);
+    QVERIFY(code.indexOf(expected) != -1);
     code = classA->typeEntry()->conversionRule();
-    QVERIFY(code.indexOf(utf8Data) != -1);
+    QVERIFY(code.indexOf(expected) != -1);
 }
 
 void TestCodeInjections::testInjectWithValidApiVersion()
diff --git a/sources/shiboken2/ApiExtractor/tests/testcodeinjection.h b/sources/shiboken2/ApiExtractor/tests/testcodeinjection.h
index bd5e7ece1..1ac873970 100644
--- a/sources/shiboken2/ApiExtractor/tests/testcodeinjection.h
+++ b/sources/shiboken2/ApiExtractor/tests/testcodeinjection.h
@@ -37,7 +37,8 @@ class TestCodeInjections : public QObject
 {
     Q_OBJECT
 private slots:
-    void testReadFileUtf8();
+    void testReadFile_data();
+    void testReadFile();
     void testInjectWithValidApiVersion();
     void testInjectWithInvalidApiVersion();
 };
diff --git a/sources/shiboken2/ApiExtractor/tests/testcodeinjection.qrc b/sources/shiboken2/ApiExtractor/tests/testcodeinjection.qrc
index 61d59567b..fd7616bd2 100644
--- a/sources/shiboken2/ApiExtractor/tests/testcodeinjection.qrc
+++ b/sources/shiboken2/ApiExtractor/tests/testcodeinjection.qrc
@@ -1,5 +1,6 @@
 <RCC>
     <qresource>
         <file>utf8code.txt</file>
+        <file>injectedcode.txt</file>
     </qresource>
 </RCC>
diff --git a/sources/shiboken2/ApiExtractor/typesystem.cpp b/sources/shiboken2/ApiExtractor/typesystem.cpp
index 21c35bda6..2c7f5eeaa 100644
--- a/sources/shiboken2/ApiExtractor/typesystem.cpp
+++ b/sources/shiboken2/ApiExtractor/typesystem.cpp
@@ -90,6 +90,7 @@ static inline QString writeAttribute() { return QStringLiteral("write"); }
 static inline QString replaceAttribute() { return QStringLiteral("replace"); }
 static inline QString toAttribute() { return QStringLiteral("to"); }
 static inline QString signatureAttribute() { return QStringLiteral("signature"); }
+static inline QString snippetAttribute() { return QStringLiteral("snippet"); }
 static inline QString staticAttribute() { return QStringLiteral("static"); }
 static inline QString threadAttribute() { return QStringLiteral("thread"); }
 static inline QString sourceAttribute() { return QStringLiteral("source"); }
@@ -128,6 +129,31 @@ static bool setRejectionRegularExpression(const QString &patternIn,
     return true;
 }
 
+// Extract a snippet from a file within annotation "// @snippet label".
+static QString extractSnippet(const QString &code, const QString &snippetLabel)
+{
+    if (snippetLabel.isEmpty())
+        return code;
+    const QString pattern = QStringLiteral(R"(^\s*//\s*@snippet\s+)")
+        + QRegularExpression::escape(snippetLabel)
+        + QStringLiteral(R"(\s*$)");
+    const QRegularExpression snippetRe(pattern);
+    Q_ASSERT(snippetRe.isValid());
+
+    bool useLine = false;
+    QString result;
+    const auto lines = code.splitRef(QLatin1Char('\n'));
+    for (const QStringRef &line : lines) {
+        if (snippetRe.match(line).hasMatch()) {
+            useLine = !useLine;
+            if (!useLine)
+                break; // End of snippet reached
+        } else if (useLine)
+            result += line.toString() + QLatin1Char('\n');
+    }
+    return result;
+}
+
 template <class EnumType, Qt::CaseSensitivity cs = Qt::CaseInsensitive>
 struct EnumLookup
 {
@@ -1546,6 +1572,7 @@ bool Handler::parseCustomConversion(const QXmlStreamReader &,
     }
 
     QString sourceFile;
+    QString snippetLabel;
     TypeSystem::Language lang = TypeSystem::NativeCode;
     for (int i = attributes->size() - 1; i >= 0; --i) {
         const QStringRef name = attributes->at(i).qualifiedName();
@@ -1558,6 +1585,8 @@ bool Handler::parseCustomConversion(const QXmlStreamReader &,
             }
         } else if (name == QLatin1String("file")) {
             sourceFile = attributes->takeAt(i).value().toString();
+        } else if (name == snippetAttribute()) {
+            snippetLabel = attributes->takeAt(i).value().toString();
         }
     }
 
@@ -1585,7 +1614,9 @@ bool Handler::parseCustomConversion(const QXmlStreamReader &,
 
             QFile conversionSource(sourceFile);
             if (conversionSource.open(QIODevice::ReadOnly | QIODevice::Text)) {
-                topElement.entry->setConversionRule(QLatin1String(conversionFlag) + QString::fromUtf8(conversionSource.readAll()));
+                const QString conversionRule =
+                    extractSnippet(QString::fromUtf8(conversionSource.readAll()), snippetLabel);
+                topElement.entry->setConversionRule(QLatin1String(conversionFlag) + conversionRule);
             } else {
                 qCWarning(lcShiboken).noquote().nospace()
                     << "File containing conversion code for "
@@ -2197,6 +2228,7 @@ bool Handler::parseInjectCode(const QXmlStreamReader &,
     TypeSystem::CodeSnipPosition position = TypeSystem::CodeSnipPositionBeginning;
     TypeSystem::Language lang = TypeSystem::TargetLangCode;
     QString fileName;
+    QString snippetLabel;
     for (int i = attributes->size() - 1; i >= 0; --i) {
         const QStringRef name = attributes->at(i).qualifiedName();
         if (name == classAttribute()) {
@@ -2215,6 +2247,8 @@ bool Handler::parseInjectCode(const QXmlStreamReader &,
             }
         } else if (name == QLatin1String("file")) {
             fileName = attributes->takeAt(i).value().toString();
+        } else if (name == snippetAttribute()) {
+            snippetLabel = attributes->takeAt(i).value().toString();
         }
     }
 
@@ -2235,7 +2269,7 @@ bool Handler::parseInjectCode(const QXmlStreamReader &,
                                                 "// START of custom code block [file: ");
                 content += fileName;
                 content += QLatin1String("]\n");
-                content += QString::fromUtf8(codeFile.readAll());
+                content += extractSnippet(QString::fromUtf8(codeFile.readAll()), snippetLabel);
                 content += QLatin1String("\n// END of custom code block [file: ");
                 content += fileName;
                 content += QLatin1String("]\n// ========================================================================\n");