clangbackend: Fix token length calculation

This was broken for tokens with characters that take up more than one
byte.

Fixes: QTCREATORBUG-25715
Change-Id: Ic9842b960af8d6f12487f582b100cb2edcf48cc1
Reviewed-by: Christian Stenger <[email protected]>

3 files changed, 11 insertions, 1 deletions

diff --git a/src/tools/clangbackend/source/tokeninfo.cpp b/src/tools/clangbackend/source/tokeninfo.cpp
index 09413145a8e..c33dd008c94 100644
--- a/src/tools/clangbackend/source/tokeninfo.cpp
+++ b/src/tools/clangbackend/source/tokeninfo.cpp
@@ -49,7 +49,9 @@ TokenInfo::TokenInfo(const Cursor &cursor,
     m_line = start.line();
     m_column = start.column();
     m_offset = start.offset();
-    m_length = end.offset() - start.offset();
+    m_length = token->spelling().operator Utf8String().toString().size();
+    if (m_length == 0)
+        m_length = end.offset() - start.offset(); // Just for safety.
 }
 
 bool TokenInfo::hasInvalidMainType() const
diff --git a/tests/unit/unittest/data/highlightingmarks.cpp b/tests/unit/unittest/data/highlightingmarks.cpp
index f4c9aa74bbb..fc3f792b85e 100644
--- a/tests/unit/unittest/data/highlightingmarks.cpp
+++ b/tests/unit/unittest/data/highlightingmarks.cpp
@@ -788,3 +788,5 @@ static inline constexpr vecn<T, S> operator<(vecn<T, S> a, vecn<T, S> b)
     }
     return x;
 }
+
+const char *cyrillic = "б";
diff --git a/tests/unit/unittest/tokenprocessor-test.cpp b/tests/unit/unittest/tokenprocessor-test.cpp
index d076f815016..38755dce663 100644
--- a/tests/unit/unittest/tokenprocessor-test.cpp
+++ b/tests/unit/unittest/tokenprocessor-test.cpp
@@ -1807,6 +1807,12 @@ TEST_F(TokenProcessor, OperatorInTemplate)
     ASSERT_THAT(infos[9], HasOnlyType(HighlightingType::Punctuation));
 }
 
+TEST_F(TokenProcessor, CyrillicString)
+{
+    const auto infos = translationUnit.tokenInfosInRange(sourceRange(792, 28));
+    ASSERT_THAT(infos[5], IsHighlightingMark(792u, 24u, 3u, HighlightingType::StringLiteral));
+}
+
 Data *TokenProcessor::d;
 
 void TokenProcessor::SetUpTestCase()