Removing SimpleToken

7 files changed, 55 insertions, 163 deletions

diff --git a/src/libs/cplusplus/BackwardsScanner.cpp b/src/libs/cplusplus/BackwardsScanner.cpp
index e6c31691769..93db1209b94 100644
--- a/src/libs/cplusplus/BackwardsScanner.cpp
+++ b/src/libs/cplusplus/BackwardsScanner.cpp
@@ -52,19 +52,19 @@ BackwardsScanner::BackwardsScanner(const QTextCursor &cursor, int maxBlockCount,
     _startToken = _tokens.size();
 }
 
-SimpleToken BackwardsScanner::LA(int index) const
+Token BackwardsScanner::LA(int index) const
 { return const_cast<BackwardsScanner *>(this)->fetchToken(_startToken - index); }
 
-SimpleToken BackwardsScanner::operator[](int index) const
+Token BackwardsScanner::operator[](int index) const
 { return const_cast<BackwardsScanner *>(this)->fetchToken(index); }
 
-const SimpleToken &BackwardsScanner::fetchToken(int tokenIndex)
+const Token &BackwardsScanner::fetchToken(int tokenIndex)
 {
     while (_offset + tokenIndex < 0) {
         _block = _block.previous();
         if (_blocksTokenized == _maxBlockCount || !_block.isValid()) {
             ++_offset;
-            _tokens.prepend(SimpleToken()); // sentinel
+            _tokens.prepend(Token()); // sentinel
             break;
         } else {
             ++_blocksTokenized;
@@ -73,10 +73,10 @@ const SimpleToken &BackwardsScanner::fetchToken(int tokenIndex)
             _text.prepend(QLatin1Char('\n'));
             _text.prepend(blockText);
 
-            QList<SimpleToken> adaptedTokens;
+            QList<Token> adaptedTokens;
             for (int i = 0; i < _tokens.size(); ++i) {
-                SimpleToken t = _tokens.at(i);
-                t.setPosition(t.position() + blockText.length() + 1);
+                Token t = _tokens.at(i);
+                t.offset += + blockText.length() + 1;
                 adaptedTokens.append(t);
             }
 
@@ -100,19 +100,19 @@ QString BackwardsScanner::text() const
 
 QString BackwardsScanner::mid(int index) const
 {
-    const SimpleToken &firstToken = _tokens.at(index + _offset);
+    const Token &firstToken = _tokens.at(index + _offset);
     return _text.mid(firstToken.begin());
 }
 
 QString BackwardsScanner::text(int index) const
 {
-    const SimpleToken &firstToken = _tokens.at(index + _offset);
+    const Token &firstToken = _tokens.at(index + _offset);
     return _text.mid(firstToken.begin(), firstToken.length());
 }
 
 QStringRef BackwardsScanner::textRef(int index) const
 {
-    const SimpleToken &firstToken = _tokens.at(index + _offset);
+    const Token &firstToken = _tokens.at(index + _offset);
     return _text.midRef(firstToken.begin(), firstToken.length());
 }
 
@@ -181,11 +181,11 @@ int BackwardsScanner::startOfLine(int index) const
     const BackwardsScanner tk(*this);
 
     forever {
-        const SimpleToken &tok = tk[index - 1];
+        const Token &tok = tk[index - 1];
 
         if (tok.is(T_EOF_SYMBOL))
             break;
-        else if (tok.followsNewline())
+        else if (tok.newline())
             return index - 1;
 
         --index;
@@ -201,7 +201,7 @@ int BackwardsScanner::startOfBlock(int index) const
     const int start = index;
 
     forever {
-        SimpleToken token = tk[index - 1];
+        Token token = tk[index - 1];
 
         if (token.is(T_EOF_SYMBOL)) {
             break;
@@ -234,9 +234,10 @@ int BackwardsScanner::startOfBlock(int index) const
 
 QString BackwardsScanner::indentationString(int index) const
 {
-    const SimpleToken tokenAfterNewline = operator[](startOfLine(index + 1));
-    const int newlinePos = qMax(0, _text.lastIndexOf(QLatin1Char('\n'), tokenAfterNewline.position()));
-    return _text.mid(newlinePos, tokenAfterNewline.position() - newlinePos);
+    const Token tokenAfterNewline = operator[](startOfLine(index + 1));
+    const int newlinePos = qMax(0, _text.lastIndexOf(QLatin1Char('\n'),
+                                                     tokenAfterNewline.begin()));
+    return _text.mid(newlinePos, tokenAfterNewline.begin() - newlinePos);
 }
 
 
diff --git a/src/libs/cplusplus/BackwardsScanner.h b/src/libs/cplusplus/BackwardsScanner.h
index e348c72a27d..26b45f5667c 100644
--- a/src/libs/cplusplus/BackwardsScanner.h
+++ b/src/libs/cplusplus/BackwardsScanner.h
@@ -55,10 +55,10 @@ public:
     QString text(int index) const;
     QStringRef textRef(int index) const;
     // 1-based
-    SimpleToken LA(int index) const;
+    Token LA(int index) const;
 
     // n-la token is [startToken - n]
-    SimpleToken operator[](int index) const; // ### deprecate
+    Token operator[](int index) const; // ### deprecate
 
     QString indentationString(int index) const;
 
@@ -71,10 +71,10 @@ public:
     static int previousBlockState(const QTextBlock &block);
 
 private:
-    const SimpleToken &fetchToken(int tokenIndex);
+    const Token &fetchToken(int tokenIndex);
 
 private:
-    QList<SimpleToken> _tokens;
+    QList<Token> _tokens;
     int _offset;
     int _blocksTokenized;
     QTextBlock _block;
diff --git a/src/libs/cplusplus/ExpressionUnderCursor.cpp b/src/libs/cplusplus/ExpressionUnderCursor.cpp
index c840ad398dd..d70bf3428eb 100644
--- a/src/libs/cplusplus/ExpressionUnderCursor.cpp
+++ b/src/libs/cplusplus/ExpressionUnderCursor.cpp
@@ -56,7 +56,7 @@ int ExpressionUnderCursor::startOfExpression(BackwardsScanner &tk, int index)
     index = startOfExpression_helper(tk, index);
 
     if (_jumpedComma) {
-        const SimpleToken &tok = tk[index - 1];
+        const Token &tok = tk[index - 1];
 
         switch (tok.kind()) {
         case T_COMMA:
@@ -204,7 +204,7 @@ int ExpressionUnderCursor::startOfExpression_helper(BackwardsScanner &tk, int in
     return index;
 }
 
-bool ExpressionUnderCursor::isAccessToken(const SimpleToken &tk)
+bool ExpressionUnderCursor::isAccessToken(const Token &tk)
 {
     switch (tk.kind()) {
     case T_COLON_COLON:
@@ -237,12 +237,12 @@ int ExpressionUnderCursor::startOfFunctionCall(const QTextCursor &cursor) const
     int index = scanner.startToken();
 
     forever {
-        const SimpleToken &tk = scanner[index - 1];
+        const Token &tk = scanner[index - 1];
 
         if (tk.is(T_EOF_SYMBOL))
             break;
         else if (tk.is(T_LPAREN))
-            return scanner.startPosition() + tk.position();
+            return scanner.startPosition() + tk.begin();
         else if (tk.is(T_RPAREN)) {
             int matchingBrace = scanner.startOfMatchingBrace(index);
 
diff --git a/src/libs/cplusplus/ExpressionUnderCursor.h b/src/libs/cplusplus/ExpressionUnderCursor.h
index 3972aca6e4f..a18b40b6e9c 100644
--- a/src/libs/cplusplus/ExpressionUnderCursor.h
+++ b/src/libs/cplusplus/ExpressionUnderCursor.h
@@ -42,7 +42,6 @@ QT_END_NAMESPACE
 namespace CPlusPlus {
 
 class BackwardsScanner;
-class SimpleToken;
 
 class CPLUSPLUS_EXPORT ExpressionUnderCursor
 {
@@ -56,7 +55,7 @@ public:
 private:
     int startOfExpression(BackwardsScanner &tk, int index);
     int startOfExpression_helper(BackwardsScanner &tk, int index);
-    bool isAccessToken(const SimpleToken &tk);
+    bool isAccessToken(const Token &tk);
 
 private:
     bool _jumpedComma;
diff --git a/src/libs/cplusplus/MatchingText.cpp b/src/libs/cplusplus/MatchingText.cpp
index b14348e117b..6dc7515d8d0 100644
--- a/src/libs/cplusplus/MatchingText.cpp
+++ b/src/libs/cplusplus/MatchingText.cpp
@@ -152,7 +152,7 @@ QString MatchingText::insertMatchingBrace(const QTextCursor &cursor, const QStri
     const int startToken = tk.startToken();
     int index = startToken;
 
-    const SimpleToken &token = tk[index - 1];
+    const Token &token = tk[index - 1];
 
     if (text.at(0) == QLatin1Char('"') && (token.is(T_STRING_LITERAL) || token.is(T_WIDE_STRING_LITERAL))) {
         if (text.length() != 1)
@@ -220,7 +220,7 @@ QString MatchingText::insertParagraphSeparator(const QTextCursor &tc) const
 
     --index; // consume the `{'
 
-    const SimpleToken &token = tk[index - 1];
+    const Token &token = tk[index - 1];
 
     if (token.is(T_STRING_LITERAL) && tk[index - 2].is(T_EXTERN)) {
         // recognized extern "C"
@@ -230,7 +230,7 @@ QString MatchingText::insertParagraphSeparator(const QTextCursor &tc) const
         int i = index - 1;
 
         forever {
-            const SimpleToken &current = tk[i - 1];
+            const Token &current = tk[i - 1];
 
             if (current.is(T_EOF_SYMBOL))
                 break;
@@ -290,7 +290,7 @@ QString MatchingText::insertParagraphSeparator(const QTextCursor &tc) const
         }
 
         // look at the token before the matched brace
-        const SimpleToken &tokenBeforeBrace = tk[lparenIndex - 1];
+        const Token &tokenBeforeBrace = tk[lparenIndex - 1];
 
         if (tokenBeforeBrace.is(T_IF)) {
             // recognized an if statement
diff --git a/src/libs/cplusplus/SimpleLexer.cpp b/src/libs/cplusplus/SimpleLexer.cpp
index 83f3a5189c2..1efa043e7bd 100644
--- a/src/libs/cplusplus/SimpleLexer.cpp
+++ b/src/libs/cplusplus/SimpleLexer.cpp
@@ -37,47 +37,6 @@
 
 using namespace CPlusPlus;
 
-SimpleToken::SimpleToken(const Token &token)
-    : _kind(token.f.kind)
-    , _flags(0)
-    , _position(token.begin())
-    , _length(token.f.length)
-{
-    f._whitespace = token.f.whitespace;
-    f._newline = token.f.newline;
-}
-
-bool SimpleToken::isLiteral() const
-{
-    return _kind >= T_FIRST_LITERAL && _kind <= T_LAST_LITERAL;
-}
-
-bool SimpleToken::isOperator() const
-{
-    return _kind >= T_FIRST_OPERATOR && _kind <= T_LAST_OPERATOR;
-}
-
-bool SimpleToken::isKeyword() const
-{
-    return _kind >= T_FIRST_KEYWORD && _kind < T_FIRST_QT_KEYWORD;
-}
-
-bool SimpleToken::isComment() const
-{
-    return _kind == T_COMMENT || _kind == T_DOXY_COMMENT ||
-            _kind == T_CPP_COMMENT || _kind == T_CPP_DOXY_COMMENT;
-}
-
-bool SimpleToken::isObjCAtKeyword() const
-{
-    return _kind >= T_FIRST_OBJC_AT_KEYWORD && _kind <= T_LAST_OBJC_AT_KEYWORD;
-}
-
-const char *SimpleToken::name() const
-{
-    return Token::name(_kind);
-}
-
 SimpleLexer::SimpleLexer()
     : _lastState(0),
       _skipComments(false),
@@ -119,9 +78,9 @@ void SimpleLexer::setSkipComments(bool skipComments)
     _skipComments = skipComments;
 }
 
-QList<SimpleToken> SimpleLexer::operator()(const QString &text, int state)
+QList<Token> SimpleLexer::operator()(const QString &text, int state)
 {
-    QList<SimpleToken> tokens;
+    QList<Token> tokens;
 
     const QByteArray bytes = text.toLatin1();
     const char *firstChar = bytes.constData();
@@ -131,6 +90,7 @@ QList<SimpleToken> SimpleLexer::operator()(const QString &text, int state)
     lex.setQtMocRunEnabled(_qtMocRunEnabled);
     lex.setObjCEnabled(_objCEnabled);
     lex.setStartWithNewline(true);
+    lex.setObjCEnabled(_objCEnabled);
 
     if (! _skipComments)
         lex.setScanCommentTokens(true);
@@ -147,57 +107,53 @@ QList<SimpleToken> SimpleLexer::operator()(const QString &text, int state)
             break;
 
         QStringRef spell = text.midRef(lex.tokenOffset(), lex.tokenLength());
-        SimpleToken simpleTk(tk);
         lex.setScanAngleStringLiteralTokens(false);
 
         if (tk.f.newline && tk.is(T_POUND))
             inPreproc = true;
-        else if (inPreproc && tokens.size() == 1 && simpleTk.is(T_IDENTIFIER) &&
+        else if (inPreproc && tokens.size() == 1 && tk.is(T_IDENTIFIER) &&
                  spell == QLatin1String("include"))
             lex.setScanAngleStringLiteralTokens(true);
         else if (_objCEnabled
-                 && inPreproc && tokens.size() == 1 && simpleTk.is(T_IDENTIFIER) &&
+                 && inPreproc && tokens.size() == 1 && tk.is(T_IDENTIFIER) &&
                  spell == QLatin1String("import"))
             lex.setScanAngleStringLiteralTokens(true);
 
-        if (_objCEnabled && tk.is(T_IDENTIFIER))
-            simpleTk.f._objcTypeQualifier = (classifyObjectiveCContextKeyword(firstChar + tk.offset, tk.f.length) != Token_identifier);
-
-        tokens.append(simpleTk);
+        tokens.append(tk);
     }
 
     _lastState = lex.state();
     return tokens;
 }
 
-int SimpleLexer::tokenAt(const QList<SimpleToken> &tokens, int offset)
+int SimpleLexer::tokenAt(const QList<Token> &tokens, unsigned offset)
 {
     for (int index = tokens.size() - 1; index >= 0; --index) {
-        const SimpleToken &tk = tokens.at(index);
-        if (tk.position() <= offset && tk.end() >= offset)
+        const Token &tk = tokens.at(index);
+        if (tk.begin() <= offset && tk.end() >= offset)
             return index;
     }
 
     return -1;
 }
 
-SimpleToken SimpleLexer::tokenAt(const QString &text,
-                                 int offset,
-                                 int state,
-                                 bool qtMocRunEnabled)
+Token SimpleLexer::tokenAt(const QString &text,
+                           unsigned offset,
+                           int state,
+                           bool qtMocRunEnabled)
 {
     SimpleLexer tokenize;
     tokenize.setQtMocRunEnabled(qtMocRunEnabled);
-    const QList<SimpleToken> tokens = tokenize(text, state);
+    const QList<Token> tokens = tokenize(text, state);
     const int tokenIdx = tokenAt(tokens, offset);
-    return (tokenIdx == -1) ? SimpleToken() : tokens.at(tokenIdx);
+    return (tokenIdx == -1) ? Token() : tokens.at(tokenIdx);
 }
 
-int SimpleLexer::tokenBefore(const QList<SimpleToken> &tokens, int offset)
+int SimpleLexer::tokenBefore(const QList<Token> &tokens, unsigned offset)
 {
     for (int index = tokens.size() - 1; index >= 0; --index) {
-        const SimpleToken &tk = tokens.at(index);
-        if (tk.position() <= offset)
+        const Token &tk = tokens.at(index);
+        if (tk.begin() <= offset)
             return index;
     }
 
diff --git a/src/libs/cplusplus/SimpleLexer.h b/src/libs/cplusplus/SimpleLexer.h
index cf451cacc6b..b745b1b8099 100644
--- a/src/libs/cplusplus/SimpleLexer.h
+++ b/src/libs/cplusplus/SimpleLexer.h
@@ -39,73 +39,6 @@ namespace CPlusPlus {
 class SimpleLexer;
 class Token;
 
-class CPLUSPLUS_EXPORT SimpleToken
-{
-public:
-    SimpleToken(const Token &token);
-
-    SimpleToken()
-        : _kind(0)
-        , _flags(0)
-        , _position(0)
-        , _length(0)
-    { }
-
-    inline int kind() const
-    { return _kind; }
-
-    inline int position() const
-    { return _position; }
-
-    inline int length() const
-    { return _length; }
-
-    inline int begin() const
-    { return _position; }
-
-    inline int end() const
-    { return _position + _length; }
-
-    inline bool followsNewline() const
-    { return f._newline; }
-
-    inline bool followsWhitespace() const
-    { return f._whitespace; }
-
-    inline bool is(int k) const    { return _kind == k; }
-    inline bool isNot(int k) const { return _kind != k; }
-
-    bool isLiteral() const;
-    bool isOperator() const;
-    bool isKeyword() const;
-    bool isComment() const;
-    bool isObjCAtKeyword() const;
-    bool isObjCTypeQualifier() const { return f._objcTypeQualifier; }
-
-    const char *name() const;
-
-    // internal
-    inline void setPosition(int position)
-    { _position = position; }
-
-public:
-    short _kind;
-    union {
-        short _flags;
-
-        struct {
-            unsigned _newline: 1;
-            unsigned _whitespace: 1;
-            unsigned _objcTypeQualifier: 1;
-        } f;
-    };
-
-    int _position;
-    int _length;
-
-    friend class SimpleLexer;
-};
-
 class CPLUSPLUS_EXPORT SimpleLexer
 {
 public:
@@ -121,15 +54,18 @@ public:
     bool objCEnabled() const;
     void setObjCEnabled(bool onoff);
 
-    QList<SimpleToken> operator()(const QString &text, int state = 0);
+    QList<Token> operator()(const QString &text, int state = 0);
 
     int state() const
     { return _lastState; }
 
-    static int tokenAt(const QList<SimpleToken> &tokens, int offset);
-    static SimpleToken tokenAt(const QString &text, int offset, int state, bool qtMocRunEnabled = false);
+    static int tokenAt(const QList<Token> &tokens, unsigned offset);
+    static Token tokenAt(const QString &text,
+                         unsigned offset,
+                         int state,
+                         bool qtMocRunEnabled = false);
 
-    static int tokenBefore(const QList<SimpleToken> &tokens, int offset);
+    static int tokenBefore(const QList<Token> &tokens, unsigned offset);
 
 private:
     int _lastState;