qqmljslexer: split T_FUNCTION_STAR into T_FUNCTION and T_STAR

Partially reverts commit 4ba86a97011dda2d8792ee23f0e86015715a592e
that tried to reduce the number of parser conflicts by introducing
new parser rules and a new token T_FUNCTION_STAR.

The new token T_FUNCTION_STAR seems unnecessary: removing it does not
make new conflicts appear in the parser. Furthermore, it allows
removing some special handling from the lexer.

Therefore, remove all occurrences of T_FUNCTION_STAR and replace it with
T_FUNCTION and T_STAR. Add the sourcelocation of the star to the
FunctionExpression so it becomes available (previously the star was
located at the end of the function-token sourcelocation).

Change the parser to create Function Declaration early in its
FunctionStar rule, so it can save the sourcelocation of the "function"
keyword and the "*" token in FunctionDeclaration.

Also fix indexes in the parser where T_FUNCTION T_STAR does shift all
the following arguments by one position to the left.

Pick-to: 6.8
Fixes: QTBUG-126676
Change-Id: I4c405c136137628e58469b91f2b93dc2edb110e3
Reviewed-by: Fabian Kosmale <[email protected]>
Reviewed-by: Olivier De Cannière <[email protected]>

3 files changed, 37 insertions, 78 deletions

diff --git a/src/qml/parser/qqmljs.g b/src/qml/parser/qqmljs.g
index 32b609f5ff..866f66ef1d 100644
--- a/src/qml/parser/qqmljs.g
+++ b/src/qml/parser/qqmljs.g
@@ -13,7 +13,6 @@
 %token T_DIVIDE_EQ "/="         T_DO "do"                   T_DOT "."
 %token T_ELSE "else"            T_EQ "="                    T_EQ_EQ "=="
 %token T_EQ_EQ_EQ "==="         T_FINALLY "finally"         T_FOR "for"
-%token T_FUNCTION_STAR "function *"
 %token T_FUNCTION "function"    T_GE ">="                   T_GT ">"
 %token T_GT_GT ">>"             T_GT_GT_EQ ">>="            T_GT_GT_GT ">>>"
 %token T_GT_GT_GT_EQ ">>>="     T_IDENTIFIER "identifier"   T_IF "if"
@@ -3483,7 +3482,7 @@ ExpressionStatementLookahead: ;
         int token = lookaheadToken(lexer);
         if (token == T_LBRACE)
             pushToken(T_FORCE_BLOCK);
-        else if (token == T_FUNCTION || token == T_FUNCTION_STAR || token == T_CLASS || token == T_LET || token == T_CONST)
+        else if (token == T_FUNCTION || token == T_CLASS || token == T_LET || token == T_CONST)
             pushToken(T_FORCE_DECLARATION);
     } break;
 ./
@@ -4167,6 +4166,7 @@ MethodDefinition: T_STAR PropertyName GeneratorLParen StrictFormalParameters T_R
         if (!ensureNoFunctionTypeAnnotations(sym(6).TypeAnnotation, sym(4).FormalParameterList))
             return false;
         AST::FunctionExpression *f = new (pool) AST::FunctionExpression(stringRef(2), sym(4).FormalParameterList, sym(8).StatementList);
+        f->starToken = loc(1);
         f->functionToken = sym(2).PropertyName->firstSourceLocation();
         f->lparenToken = loc(3);
         f->rparenToken = loc(5);
@@ -4235,17 +4235,27 @@ GeneratorRBrace: T_RBRACE;
     } break;
 ./
 
-FunctionStar: T_FUNCTION_STAR %prec REDUCE_HERE;
+FunctionStar: T_FUNCTION T_STAR %prec REDUCE_HERE;
+/.
+    case $rule_number: {
+        AST::FunctionDeclaration *node = new (pool) AST::FunctionDeclaration(QStringView(), nullptr, nullptr);
+        node->functionToken = loc(1);
+        node->starToken = loc(2);
+        sym(1).FunctionDeclaration = node;
+    } break;
+./
 
 GeneratorDeclaration: FunctionStar BindingIdentifier GeneratorLParen FormalParameters T_RPAREN FunctionLBrace GeneratorBody GeneratorRBrace;
 /.
     case $rule_number: {
-        AST::FunctionDeclaration *node = new (pool) AST::FunctionDeclaration(stringRef(2), sym(4).FormalParameterList, sym(7).StatementList);
-        node->functionToken = loc(1);
+        AST::FunctionDeclaration *node = sym(1).FunctionDeclaration;
         node->identifierToken = loc(2);
+        node->name = stringRef(2);
         node->lparenToken = loc(3);
+        node->formals = sym(4).FormalParameterList;
         node->rparenToken = loc(5);
         node->lbraceToken = loc(6);
+        node->body = sym(7).StatementList;
         node->rbraceToken = loc(8);
         node->isGenerator = true;
         sym(1).Node = node;
@@ -4256,42 +4266,45 @@ GeneratorDeclaration_Default: GeneratorDeclaration;
 GeneratorDeclaration_Default: FunctionStar GeneratorLParen FormalParameters T_RPAREN FunctionLBrace GeneratorBody GeneratorRBrace;
 /.
     case $rule_number: {
-        AST::FunctionDeclaration *node = new (pool) AST::FunctionDeclaration(QStringView(), sym(3).FormalParameterList, sym(6).StatementList);
-        node->functionToken = loc(1);
+        AST::FunctionDeclaration *node = sym(1).FunctionDeclaration;
         node->lparenToken = loc(2);
+        node->formals = sym(3).FormalParameterList;
         node->rparenToken = loc(4);
         node->lbraceToken = loc(5);
+        node->body = sym(6).StatementList;
         node->rbraceToken = loc(7);
         node->isGenerator = true;
         sym(1).Node = node;
     } break;
 ./
 
-GeneratorExpression: T_FUNCTION_STAR BindingIdentifier GeneratorLParen FormalParameters T_RPAREN FunctionLBrace GeneratorBody GeneratorRBrace;
+GeneratorExpression: T_FUNCTION T_STAR BindingIdentifier GeneratorLParen FormalParameters T_RPAREN FunctionLBrace GeneratorBody GeneratorRBrace;
 /.
     case $rule_number: {
-        AST::FunctionExpression *node = new (pool) AST::FunctionExpression(stringRef(2), sym(4).FormalParameterList, sym(7).StatementList);
+        AST::FunctionExpression *node = new (pool) AST::FunctionExpression(stringRef(3), sym(5).FormalParameterList, sym(8).StatementList);
         node->functionToken = loc(1);
-        if (!stringRef(2).isNull())
-          node->identifierToken = loc(2);
-        node->lparenToken = loc(3);
-        node->rparenToken = loc(5);
-        node->lbraceToken = loc(6);
-        node->rbraceToken = loc(8);
+        node->starToken = loc(2);
+        if (!stringRef(3).isNull())
+          node->identifierToken = loc(3);
+        node->lparenToken = loc(4);
+        node->rparenToken = loc(6);
+        node->lbraceToken = loc(7);
+        node->rbraceToken = loc(9);
         node->isGenerator = true;
         sym(1).Node = node;
     } break;
 ./
 
-GeneratorExpression: T_FUNCTION_STAR GeneratorLParen FormalParameters T_RPAREN FunctionLBrace GeneratorBody GeneratorRBrace;
+GeneratorExpression: T_FUNCTION T_STAR GeneratorLParen FormalParameters T_RPAREN FunctionLBrace GeneratorBody GeneratorRBrace;
 /.
     case $rule_number: {
-        AST::FunctionExpression *node = new (pool) AST::FunctionExpression(QStringView(), sym(3).FormalParameterList, sym(6).StatementList);
+        AST::FunctionExpression *node = new (pool) AST::FunctionExpression(QStringView(), sym(4).FormalParameterList, sym(7).StatementList);
         node->functionToken = loc(1);
-        node->lparenToken = loc(2);
-        node->rparenToken = loc(4);
-        node->lbraceToken = loc(5);
-        node->rbraceToken = loc(7);
+        node->starToken = loc(2);
+        node->lparenToken = loc(3);
+        node->rparenToken = loc(5);
+        node->lbraceToken = loc(6);
+        node->rbraceToken = loc(8);
         node->isGenerator = true;
         sym(1).Node = node;
     } break;
@@ -4678,7 +4691,7 @@ ExportDeclarationLookahead: ;
 /.
     case $rule_number: {
         int token = lookaheadToken(lexer);
-        if (token == T_FUNCTION || token == T_FUNCTION_STAR || token == T_CLASS)
+        if (token == T_FUNCTION || token == T_CLASS)
             pushToken(T_FORCE_DECLARATION);
     } break;
 ./
diff --git a/src/qml/parser/qqmljsast_p.h b/src/qml/parser/qqmljsast_p.h
index bfeab7518c..504ef413f9 100644
--- a/src/qml/parser/qqmljsast_p.h
+++ b/src/qml/parser/qqmljsast_p.h
@@ -2416,6 +2416,8 @@ public:
     StatementList *body;
     TypeAnnotation *typeAnnotation;
     SourceLocation functionToken;
+    // for generators:
+    SourceLocation starToken;
     SourceLocation identifierToken;
     SourceLocation lparenToken;
     SourceLocation rparenToken;
diff --git a/src/qml/parser/qqmljslexer.cpp b/src/qml/parser/qqmljslexer.cpp
index aa95f6302f..56ae5192b4 100644
--- a/src/qml/parser/qqmljslexer.cpp
+++ b/src/qml/parser/qqmljslexer.cpp
@@ -933,61 +933,6 @@ again:
             if (!identifierWithEscapeChars)
                 kind = classify(_tokenStartPtr, _tokenLength, parseModeFlags());
 
-            if (kind == T_FUNCTION) {
-                continue_skipping:
-                    while (_codePtr < _endPtr && _state.currentChar.isSpace())
-                        scanChar();
-                    if (_state.currentChar == u'*') {
-                        _tokenLength = _codePtr - _tokenStartPtr - 1;
-                        kind = T_FUNCTION_STAR;
-                        scanChar();
-                    } else if (_state.currentChar == u'/') {
-                        scanChar();
-                        switch (_state.currentChar.unicode()) {
-                        case u'*':
-                            scanChar();
-                            while (_codePtr <= _endPtr) {
-                                if (_state.currentChar == u'*') {
-                                    scanChar();
-                                    if (_state.currentChar == u'/') {
-                                        scanChar();
-                                        if (_engine) {
-                                            _engine->addComment(tokenOffset() + 2,
-                                                                _codePtr - _tokenStartPtr - 1 - 4,
-                                                                tokenStartLine(),
-                                                                tokenStartColumn() + 2);
-                                        }
-                                        if (_lexMode == LexMode::LineByLine)
-                                            return T_COMMENT;
-                                        goto continue_skipping;
-                                    }
-                                } else {
-                                    scanChar();
-                                }
-                            }
-                            if (_lexMode == LexMode::LineByLine)
-                                return T_PARTIAL_COMMENT;
-                            else
-                                goto continue_skipping;
-                        case u'/':
-                            while (_codePtr <= _endPtr && !isLineTerminator()) {
-                                scanChar();
-                            }
-                            if (_engine) {
-                                _engine->addComment(tokenOffset() + 2,
-                                                    _codePtr - _tokenStartPtr - 1 - 2,
-                                                    tokenStartLine(), tokenStartColumn() + 2);
-                            }
-                            if (_lexMode == LexMode::LineByLine)
-                                return T_COMMENT;
-                            else
-                                goto continue_skipping;
-                        default:
-                            break;
-                        }
-                    }
-            }
-
             if (_engine) {
                 if (kind == T_IDENTIFIER && identifierWithEscapeChars)
                     _tokenSpell = _engine->newStringRef(_tokenText);
@@ -1657,7 +1602,6 @@ static const int uriTokens[] = {
     QQmlJSGrammar::T_FINALLY,
     QQmlJSGrammar::T_FOR,
     QQmlJSGrammar::T_FUNCTION,
-    QQmlJSGrammar::T_FUNCTION_STAR,
     QQmlJSGrammar::T_IF,
     QQmlJSGrammar::T_IN,
     QQmlJSGrammar::T_OF,