function Lexer::tokenize

Same name in this branch

1. 11.1.x vendor/phpstan/phpdoc-parser/src/Lexer/Lexer.php \PHPStan\PhpDocParser\Lexer\Lexer::tokenize()
2. 11.1.x vendor/nikic/php-parser/lib/PhpParser/Lexer.php \PhpParser\Lexer::tokenize()

File

vendor/twig/twig/src/Lexer.php, line 174

Class

Lexer

@author Fabien Potencier <fabien@symfony.com>

Namespace

Twig

Code

public function tokenize(Source $source) : TokenStream {
    $this->initialize();
    $this->source = $source;
    $this->code = str_replace([
        "\r\n",
        "\r",
    ], "\n", $source->getCode());
    $this->cursor = 0;
    $this->lineno = 1;
    $this->end = \strlen($this->code);
    $this->tokens = [];
    $this->state = self::STATE_DATA;
    $this->states = [];
    $this->brackets = [];
    $this->position = -1;
    // find all token starts in one go
    preg_match_all($this->regexes['lex_tokens_start'], $this->code, $matches, \PREG_OFFSET_CAPTURE);
    $this->positions = $matches;
    while ($this->cursor < $this->end) {
        // dispatch to the lexing functions depending
        // on the current state
        switch ($this->state) {
            case self::STATE_DATA:
                $this->lexData();
                break;
            case self::STATE_BLOCK:
                $this->lexBlock();
                break;
            case self::STATE_VAR:
                $this->lexVar();
                break;
            case self::STATE_STRING:
                $this->lexString();
                break;
            case self::STATE_INTERPOLATION:
                $this->lexInterpolation();
                break;
        }
    }
    $this->pushToken(Token::EOF_TYPE);
    if ($this->brackets) {
        [
            $expect,
            $lineno,
        ] = array_pop($this->brackets);
        throw new SyntaxError(\sprintf('Unclosed "%s".', $expect), $lineno, $this->source);
    }
    return new TokenStream($this->tokens, $this->source);
}