function Tokenizer::parse

Same name in this branch

1. 11.1.x vendor/masterminds/html5/src/HTML5/Parser/Tokenizer.php \Masterminds\HTML5\Parser\Tokenizer::parse()

File

vendor/theseer/tokenizer/src/Tokenizer.php, line 42

Class

Tokenizer

Namespace

TheSeer\Tokenizer

Code

public function parse(string $source) : TokenCollection {
    $result = new TokenCollection();
    if ($source === '') {
        return $result;
    }
    $tokens = \token_get_all($source);
    $lastToken = new Token($tokens[0][2], 'Placeholder', '');
    foreach ($tokens as $pos => $tok) {
        if (\is_string($tok)) {
            $token = new Token($lastToken->getLine(), $this->map[$tok], $tok);
            $result->addToken($token);
            $lastToken = $token;
            continue;
        }
        $line = $tok[2];
        $values = \preg_split('/\\R+/Uu', $tok[1]);
        if (!$values) {
            $result->addToken(new Token($line, \token_name($tok[0]), '{binary data}'));
            continue;
        }
        foreach ($values as $v) {
            $token = new Token($line, \token_name($tok[0]), $v);
            $lastToken = $token;
            $line++;
            if ($v === '') {
                continue;
            }
            $result->addToken($token);
        }
    }
    return $this->fillBlanks($result, $lastToken->getLine());
}