function Lexer::tokenize

Same name in this branch

11.1.x vendor/nikic/php-parser/lib/PhpParser/Lexer.php \PhpParser\Lexer::tokenize()
11.1.x vendor/twig/twig/src/Lexer.php \Twig\Lexer::tokenize()

Return value

list<array{string, int, int}>

File

vendor/phpstan/phpdoc-parser/src/Lexer/Lexer.php, line 111

Class

Lexer: Implementation based on Nette Tokenizer (New BSD License; https://github.com/nette/tokenizer)

Namespace

PHPStan\PhpDocParser\Lexer

Code

public function tokenize(string $s) : array {
    if ($this->regexp === null) {
        $this->regexp = $this->generateRegexp();
    }
    preg_match_all($this->regexp, $s, $matches, PREG_SET_ORDER);
    $tokens = [];
    $line = 1;
    foreach ($matches as $match) {
        $type = (int) $match['MARK'];
        $tokens[] = [
            $match[0],
            $type,
            $line,
        ];
        if ($type !== self::TOKEN_PHPDOC_EOL) {
            continue;
        }
        $line++;
    }
    $tokens[] = [
        '',
        self::TOKEN_END,
        $line,
    ];
    return $tokens;
}