function Lexer::postprocessTokens

Parameters

list<Token> $tokens:

1 call to Lexer::postprocessTokens()

Lexer::tokenize in vendor/nikic/php-parser/lib/PhpParser/Lexer.php

Tokenize the provided source code.

File

vendor/nikic/php-parser/lib/PhpParser/Lexer.php, line 69

Class

Lexer

Namespace

PhpParser

Code

protected function postprocessTokens(array &$tokens, ErrorHandler $errorHandler) : void {
    // This function reports errors (bad characters and unterminated comments) in the token
    // array, and performs certain canonicalizations:
    //  * Use PHP 8.1 T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG and
    //    T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG tokens used to disambiguate intersection types.
    //  * Add a sentinel token with ID 0.
    $numTokens = \count($tokens);
    if ($numTokens === 0) {
        // Empty input edge case: Just add the sentinel token.
        $tokens[] = new Token(0, "\x00", 1, 0);
        return;
    }
    for ($i = 0; $i < $numTokens; $i++) {
        $token = $tokens[$i];
        if ($token->id === \T_BAD_CHARACTER) {
            $this->handleInvalidCharacter($token, $errorHandler);
        }
        if ($token->id === \ord('&')) {
            $next = $i + 1;
            while (isset($tokens[$next]) && $tokens[$next]->id === \T_WHITESPACE) {
                $next++;
            }
            $followedByVarOrVarArg = isset($tokens[$next]) && $tokens[$next]->is([
                \T_VARIABLE,
                \T_ELLIPSIS,
            ]);
            $token->id = $followedByVarOrVarArg ? \T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG : \T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG;
        }
    }
    // Check for unterminated comment
    $lastToken = $tokens[$numTokens - 1];
    if ($this->isUnterminatedComment($lastToken)) {
        $errorHandler->handleError(new Error('Unterminated comment', [
            'startLine' => $lastToken->line,
            'endLine' => $lastToken->getEndLine(),
            'startFilePos' => $lastToken->pos,
            'endFilePos' => $lastToken->getEndPos(),
        ]));
    }
    // Add sentinel token.
    $tokens[] = new Token(0, "\x00", $lastToken->getEndLine(), $lastToken->getEndPos());
}