function Comment::tokenizeString

Creates an array of tokens when given some PHP code.

Starts by using token_get_all() but does a lot of extra processing to insert information about the context of the token.

Parameters

string $string The string to tokenize.:

string $eolChar The EOL character to use for splitting strings.:

int $stackPtr The position of the first token in the file.:

Return value

array<int, array<string, string|int|array<int>>>

File

vendor/squizlabs/php_codesniffer/src/Tokenizers/Comment.php, line 30

Class

Comment

Namespace

PHP_CodeSniffer\Tokenizers

Code

public function tokenizeString($string, $eolChar, $stackPtr) {
    if (PHP_CODESNIFFER_VERBOSITY > 1) {
        echo "\t\t*** START COMMENT TOKENIZING ***" . PHP_EOL;
    }
    $tokens = [];
    $numChars = strlen($string);
    
    /*
        Doc block comments start with /*, but typically contain an
        extra star when they are used for function and class comments.
    */
    $char = $numChars - strlen(ltrim($string, '/*'));
    $lastChars = substr($string, -2);
    if ($char === $numChars && $lastChars === '*/') {
        // Edge case: docblock without whitespace or contents.
        $openTag = substr($string, 0, -2);
        $string = $lastChars;
    }
    else {
        $openTag = substr($string, 0, $char);
        $string = ltrim($string, '/*');
    }
    $tokens[$stackPtr] = [
        'content' => $openTag,
        'code' => T_DOC_COMMENT_OPEN_TAG,
        'type' => 'T_DOC_COMMENT_OPEN_TAG',
        'comment_tags' => [],
    ];
    $openPtr = $stackPtr;
    $stackPtr++;
    if (PHP_CODESNIFFER_VERBOSITY > 1) {
        $content = Common::prepareForOutput($openTag);
        echo "\t\tCreate comment token: T_DOC_COMMENT_OPEN_TAG => {$content}" . PHP_EOL;
    }
    
    /*
        Strip off the close tag so it doesn't interfere with any
        of our comment line processing. The token will be added to the
        stack just before we return it.
    */
    $closeTag = [
        'content' => substr($string, strlen(rtrim($string, '/*'))),
        'code' => T_DOC_COMMENT_CLOSE_TAG,
        'type' => 'T_DOC_COMMENT_CLOSE_TAG',
        'comment_opener' => $openPtr,
    ];
    if ($closeTag['content'] === false) {
        // In PHP < 8.0 substr() can return `false` instead of always returning a string.
        $closeTag['content'] = '';
    }
    $string = rtrim($string, '/*');
    
    /*
        Process each line of the comment.
    */
    $lines = explode($eolChar, $string);
    $numLines = count($lines);
    foreach ($lines as $lineNum => $string) {
        if ($lineNum !== $numLines - 1) {
            $string .= $eolChar;
        }
        $char = 0;
        $numChars = strlen($string);
        // We've started a new line, so process the indent.
        $space = $this->collectWhitespace($string, $char, $numChars);
        if ($space !== null) {
            $tokens[$stackPtr] = $space;
            $stackPtr++;
            if (PHP_CODESNIFFER_VERBOSITY > 1) {
                $content = Common::prepareForOutput($space['content']);
                echo "\t\tCreate comment token: T_DOC_COMMENT_WHITESPACE => {$content}" . PHP_EOL;
            }
            $char += strlen($space['content']);
            if ($char === $numChars) {
                break;
            }
        }
        if ($string === '') {
            continue;
        }
        if ($lineNum > 0 && $string[$char] === '*') {
            // This is a function or class doc block line.
            $char++;
            $tokens[$stackPtr] = [
                'content' => '*',
                'code' => T_DOC_COMMENT_STAR,
                'type' => 'T_DOC_COMMENT_STAR',
            ];
            $stackPtr++;
            if (PHP_CODESNIFFER_VERBOSITY > 1) {
                echo "\t\tCreate comment token: T_DOC_COMMENT_STAR => *" . PHP_EOL;
            }
        }
        // Now we are ready to process the actual content of the line.
        $lineTokens = $this->processLine($string, $eolChar, $char, $numChars);
        foreach ($lineTokens as $lineToken) {
            $tokens[$stackPtr] = $lineToken;
            if (PHP_CODESNIFFER_VERBOSITY > 1) {
                $content = Common::prepareForOutput($lineToken['content']);
                $type = $lineToken['type'];
                echo "\t\tCreate comment token: {$type} => {$content}" . PHP_EOL;
            }
            if ($lineToken['code'] === T_DOC_COMMENT_TAG) {
                $tokens[$openPtr]['comment_tags'][] = $stackPtr;
            }
            $stackPtr++;
        }
    }
    
    //end foreach
    $tokens[$stackPtr] = $closeTag;
    $tokens[$openPtr]['comment_closer'] = $stackPtr;
    if (PHP_CODESNIFFER_VERBOSITY > 1) {
        $content = Common::prepareForOutput($closeTag['content']);
        echo "\t\tCreate comment token: T_DOC_COMMENT_CLOSE_TAG => {$content}" . PHP_EOL;
    }
    if (PHP_CODESNIFFER_VERBOSITY > 1) {
        echo "\t\t*** END COMMENT TOKENIZING ***" . PHP_EOL;
    }
    return $tokens;
}