class JS

class JS extends Tokenizer {
    
    /**
     * A list of tokens that are allowed to open a scope.
     *
     * This array also contains information about what kind of token the scope
     * opener uses to open and close the scope, if the token strictly requires
     * an opener, if the token can share a scope closer, and who it can be shared
     * with. An example of a token that shares a scope closer is a CASE scope.
     *
     * @var array
     */
    public $scopeOpeners = [
        T_IF => [
            'start' => [
                T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
            ],
            'end' => [
                T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
            ],
            'strict' => false,
            'shared' => false,
            'with' => [],
        ],
        T_TRY => [
            'start' => [
                T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
            ],
            'end' => [
                T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
            ],
            'strict' => true,
            'shared' => false,
            'with' => [],
        ],
        T_CATCH => [
            'start' => [
                T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
            ],
            'end' => [
                T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
            ],
            'strict' => true,
            'shared' => false,
            'with' => [],
        ],
        T_ELSE => [
            'start' => [
                T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
            ],
            'end' => [
                T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
            ],
            'strict' => false,
            'shared' => false,
            'with' => [],
        ],
        T_FOR => [
            'start' => [
                T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
            ],
            'end' => [
                T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
            ],
            'strict' => false,
            'shared' => false,
            'with' => [],
        ],
        T_CLASS => [
            'start' => [
                T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
            ],
            'end' => [
                T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
            ],
            'strict' => true,
            'shared' => false,
            'with' => [],
        ],
        T_FUNCTION => [
            'start' => [
                T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
            ],
            'end' => [
                T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
            ],
            'strict' => false,
            'shared' => false,
            'with' => [],
        ],
        T_WHILE => [
            'start' => [
                T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
            ],
            'end' => [
                T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
            ],
            'strict' => false,
            'shared' => false,
            'with' => [],
        ],
        T_DO => [
            'start' => [
                T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
            ],
            'end' => [
                T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
            ],
            'strict' => true,
            'shared' => false,
            'with' => [],
        ],
        T_SWITCH => [
            'start' => [
                T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
            ],
            'end' => [
                T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
            ],
            'strict' => true,
            'shared' => false,
            'with' => [],
        ],
        T_CASE => [
            'start' => [
                T_COLON => T_COLON,
            ],
            'end' => [
                T_BREAK => T_BREAK,
                T_RETURN => T_RETURN,
                T_CONTINUE => T_CONTINUE,
                T_THROW => T_THROW,
            ],
            'strict' => true,
            'shared' => true,
            'with' => [
                T_DEFAULT => T_DEFAULT,
                T_CASE => T_CASE,
                T_SWITCH => T_SWITCH,
            ],
        ],
        T_DEFAULT => [
            'start' => [
                T_COLON => T_COLON,
            ],
            'end' => [
                T_BREAK => T_BREAK,
                T_RETURN => T_RETURN,
                T_CONTINUE => T_CONTINUE,
                T_THROW => T_THROW,
            ],
            'strict' => true,
            'shared' => true,
            'with' => [
                T_CASE => T_CASE,
                T_SWITCH => T_SWITCH,
            ],
        ],
    ];
    
    /**
     * A list of tokens that end the scope.
     *
     * This array is just a unique collection of the end tokens
     * from the _scopeOpeners array. The data is duplicated here to
     * save time during parsing of the file.
     *
     * @var array
     */
    public $endScopeTokens = [
        T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
        T_BREAK => T_BREAK,
    ];
    
    /**
     * A list of special JS tokens and their types.
     *
     * @var array
     */
    protected $tokenValues = [
        'class' => 'T_CLASS',
        'function' => 'T_FUNCTION',
        'prototype' => 'T_PROTOTYPE',
        'try' => 'T_TRY',
        'catch' => 'T_CATCH',
        'return' => 'T_RETURN',
        'throw' => 'T_THROW',
        'break' => 'T_BREAK',
        'switch' => 'T_SWITCH',
        'continue' => 'T_CONTINUE',
        'if' => 'T_IF',
        'else' => 'T_ELSE',
        'do' => 'T_DO',
        'while' => 'T_WHILE',
        'for' => 'T_FOR',
        'var' => 'T_VAR',
        'case' => 'T_CASE',
        'default' => 'T_DEFAULT',
        'true' => 'T_TRUE',
        'false' => 'T_FALSE',
        'null' => 'T_NULL',
        'this' => 'T_THIS',
        'typeof' => 'T_TYPEOF',
        '(' => 'T_OPEN_PARENTHESIS',
        ')' => 'T_CLOSE_PARENTHESIS',
        '{' => 'T_OPEN_CURLY_BRACKET',
        '}' => 'T_CLOSE_CURLY_BRACKET',
        '[' => 'T_OPEN_SQUARE_BRACKET',
        ']' => 'T_CLOSE_SQUARE_BRACKET',
        '?' => 'T_INLINE_THEN',
        '.' => 'T_OBJECT_OPERATOR',
        '+' => 'T_PLUS',
        '-' => 'T_MINUS',
        '*' => 'T_MULTIPLY',
        '%' => 'T_MODULUS',
        '/' => 'T_DIVIDE',
        '^' => 'T_LOGICAL_XOR',
        ',' => 'T_COMMA',
        ';' => 'T_SEMICOLON',
        ':' => 'T_COLON',
        '<' => 'T_LESS_THAN',
        '>' => 'T_GREATER_THAN',
        '<<' => 'T_SL',
        '>>' => 'T_SR',
        '>>>' => 'T_ZSR',
        '<<=' => 'T_SL_EQUAL',
        '>>=' => 'T_SR_EQUAL',
        '>>>=' => 'T_ZSR_EQUAL',
        '<=' => 'T_IS_SMALLER_OR_EQUAL',
        '>=' => 'T_IS_GREATER_OR_EQUAL',
        '=>' => 'T_DOUBLE_ARROW',
        '!' => 'T_BOOLEAN_NOT',
        '||' => 'T_BOOLEAN_OR',
        '&&' => 'T_BOOLEAN_AND',
        '|' => 'T_BITWISE_OR',
        '&' => 'T_BITWISE_AND',
        '!=' => 'T_IS_NOT_EQUAL',
        '!==' => 'T_IS_NOT_IDENTICAL',
        '=' => 'T_EQUAL',
        '==' => 'T_IS_EQUAL',
        '===' => 'T_IS_IDENTICAL',
        '-=' => 'T_MINUS_EQUAL',
        '+=' => 'T_PLUS_EQUAL',
        '*=' => 'T_MUL_EQUAL',
        '/=' => 'T_DIV_EQUAL',
        '%=' => 'T_MOD_EQUAL',
        '++' => 'T_INC',
        '--' => 'T_DEC',
        '//' => 'T_COMMENT',
        '/*' => 'T_COMMENT',
        '/**' => 'T_DOC_COMMENT',
        '*/' => 'T_COMMENT',
    ];
    
    /**
     * A list string delimiters.
     *
     * @var array
     */
    protected $stringTokens = [
        '\'' => '\'',
        '"' => '"',
    ];
    
    /**
     * A list tokens that start and end comments.
     *
     * @var array
     */
    protected $commentTokens = [
        '//' => null,
        '/*' => '*/',
        '/**' => '*/',
    ];
    
    /**
     * Initialise the tokenizer.
     *
     * Pre-checks the content to see if it looks minified.
     *
     * @param string                  $content The content to tokenize.
     * @param \PHP_CodeSniffer\Config $config  The config data for the run.
     * @param string                  $eolChar The EOL char used in the content.
     *
     * @return void
     * @throws \PHP_CodeSniffer\Exceptions\TokenizerException If the file appears to be minified.
     */
    public function __construct($content, Config $config, $eolChar = '\\n') {
        if ($this->isMinifiedContent($content, $eolChar) === true) {
            throw new TokenizerException('File appears to be minified and cannot be processed');
        }
        parent::__construct($content, $config, $eolChar);
    }
    
    //end __construct()
    
    /**
     * Creates an array of tokens when given some JS code.
     *
     * @param string $string The string to tokenize.
     *
     * @return array
     */
    public function tokenize($string) {
        if (PHP_CODESNIFFER_VERBOSITY > 1) {
            echo "\t*** START JS TOKENIZING ***" . PHP_EOL;
        }
        $maxTokenLength = 0;
        foreach ($this->tokenValues as $token => $values) {
            if (strlen($token) > $maxTokenLength) {
                $maxTokenLength = strlen($token);
            }
        }
        $tokens = [];
        $inString = '';
        $stringChar = null;
        $inComment = '';
        $buffer = '';
        $preStringBuffer = '';
        $cleanBuffer = false;
        $commentTokenizer = new Comment();
        $tokens[] = [
            'code' => T_OPEN_TAG,
            'type' => 'T_OPEN_TAG',
            'content' => '',
        ];
        // Convert newlines to single characters for ease of
        // processing. We will change them back later.
        $string = str_replace($this->eolChar, "\n", $string);
        $chars = str_split($string);
        $numChars = count($chars);
        for ($i = 0; $i < $numChars; $i++) {
            $char = $chars[$i];
            if (PHP_CODESNIFFER_VERBOSITY > 1) {
                $content = Util\Common::prepareForOutput($char);
                $bufferContent = Util\Common::prepareForOutput($buffer);
                if ($inString !== '') {
                    echo "\t";
                }
                if ($inComment !== '') {
                    echo "\t";
                }
                echo "\tProcess char {$i} => {$content} (buffer: {$bufferContent})" . PHP_EOL;
            }
            
            //end if
            if ($inString === '' && $inComment === '' && $buffer !== '') {
                // If the buffer only has whitespace and we are about to
                // add a character, store the whitespace first.
                if (trim($char) !== '' && trim($buffer) === '') {
                    $tokens[] = [
                        'code' => T_WHITESPACE,
                        'type' => 'T_WHITESPACE',
                        'content' => str_replace("\n", $this->eolChar, $buffer),
                    ];
                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        $content = Util\Common::prepareForOutput($buffer);
                        echo "\t=> Added token T_WHITESPACE ({$content})" . PHP_EOL;
                    }
                    $buffer = '';
                }
                // If the buffer is not whitespace and we are about to
                // add a whitespace character, store the content first.
                if ($inString === '' && $inComment === '' && trim($char) === '' && trim($buffer) !== '') {
                    $tokens[] = [
                        'code' => T_STRING,
                        'type' => 'T_STRING',
                        'content' => str_replace("\n", $this->eolChar, $buffer),
                    ];
                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        $content = Util\Common::prepareForOutput($buffer);
                        echo "\t=> Added token T_STRING ({$content})" . PHP_EOL;
                    }
                    $buffer = '';
                }
            }
            
            //end if
            // Process strings.
            if ($inComment === '' && isset($this->stringTokens[$char]) === true) {
                if ($inString === $char) {
                    // This could be the end of the string, but make sure it
                    // is not escaped first.
                    $escapes = 0;
                    for ($x = $i - 1; $x >= 0; $x--) {
                        if ($chars[$x] !== '\\') {
                            break;
                        }
                        $escapes++;
                    }
                    if ($escapes === 0 || $escapes % 2 === 0) {
                        // There is an even number escape chars,
                        // so this is not escaped, it is the end of the string.
                        $tokens[] = [
                            'code' => T_CONSTANT_ENCAPSED_STRING,
                            'type' => 'T_CONSTANT_ENCAPSED_STRING',
                            'content' => str_replace("\n", $this->eolChar, $buffer) . $char,
                        ];
                        if (PHP_CODESNIFFER_VERBOSITY > 1) {
                            echo "\t\t* found end of string *" . PHP_EOL;
                            $content = Util\Common::prepareForOutput($buffer . $char);
                            echo "\t=> Added token T_CONSTANT_ENCAPSED_STRING ({$content})" . PHP_EOL;
                        }
                        $buffer = '';
                        $preStringBuffer = '';
                        $inString = '';
                        $stringChar = null;
                        continue;
                    }
                    
                    //end if
                }
                else {
                    if ($inString === '') {
                        $inString = $char;
                        $stringChar = $i;
                        $preStringBuffer = $buffer;
                        if (PHP_CODESNIFFER_VERBOSITY > 1) {
                            echo "\t\t* looking for string closer *" . PHP_EOL;
                        }
                    }
                }
                
                //end if
            }
            
            //end if
            if ($inString !== '' && $char === "\n") {
                // Unless this newline character is escaped, the string did not
                // end before the end of the line, which means it probably
                // wasn't a string at all (maybe a regex).
                if ($chars[$i - 1] !== '\\') {
                    $i = $stringChar;
                    $buffer = $preStringBuffer;
                    $preStringBuffer = '';
                    $inString = '';
                    $stringChar = null;
                    $char = $chars[$i];
                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        echo "\t\t* found newline before end of string, bailing *" . PHP_EOL;
                    }
                }
            }
            $buffer .= $char;
            // We don't look for special tokens inside strings,
            // so if we are in a string, we can continue here now
            // that the current char is in the buffer.
            if ($inString !== '') {
                continue;
            }
            // Special case for T_DIVIDE which can actually be
            // the start of a regular expression.
            if ($buffer === $char && $char === '/' && $chars[$i + 1] !== '*') {
                $regex = $this->getRegexToken($i, $string, $chars, $tokens);
                if ($regex !== null) {
                    $tokens[] = [
                        'code' => T_REGULAR_EXPRESSION,
                        'type' => 'T_REGULAR_EXPRESSION',
                        'content' => $regex['content'],
                    ];
                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        $content = Util\Common::prepareForOutput($regex['content']);
                        echo "\t=> Added token T_REGULAR_EXPRESSION ({$content})" . PHP_EOL;
                    }
                    $i = $regex['end'];
                    $buffer = '';
                    $cleanBuffer = false;
                    continue;
                }
                
                //end if
            }
            
            //end if
            // Check for known tokens, but ignore tokens found that are not at
            // the end of a string, like FOR and this.FORmat.
            if (isset($this->tokenValues[strtolower($buffer)]) === true && (preg_match('|[a-zA-z0-9_]|', $char) === 0 || isset($chars[$i + 1]) === false || preg_match('|[a-zA-z0-9_]|', $chars[$i + 1]) === 0)) {
                $matchedToken = false;
                $lookAheadLength = $maxTokenLength - strlen($buffer);
                if ($lookAheadLength > 0) {
                    // The buffer contains a token type, but we need
                    // to look ahead at the next chars to see if this is
                    // actually part of a larger token. For example,
                    // FOR and FOREACH.
                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        echo "\t\t* buffer possibly contains token, looking ahead {$lookAheadLength} chars *" . PHP_EOL;
                    }
                    $charBuffer = $buffer;
                    for ($x = 1; $x <= $lookAheadLength; $x++) {
                        if (isset($chars[$i + $x]) === false) {
                            break;
                        }
                        $charBuffer .= $chars[$i + $x];
                        if (PHP_CODESNIFFER_VERBOSITY > 1) {
                            $content = Util\Common::prepareForOutput($charBuffer);
                            echo "\t\t=> Looking ahead {$x} chars => {$content}" . PHP_EOL;
                        }
                        if (isset($this->tokenValues[strtolower($charBuffer)]) === true) {
                            // We've found something larger that matches
                            // so we can ignore this char. Except for 1 very specific
                            // case where a comment like /**/ needs to tokenize as
                            // T_COMMENT and not T_DOC_COMMENT.
                            $oldType = $this->tokenValues[strtolower($buffer)];
                            $newType = $this->tokenValues[strtolower($charBuffer)];
                            if ($oldType === 'T_COMMENT' && $newType === 'T_DOC_COMMENT' && $chars[$i + $x + 1] === '/') {
                                if (PHP_CODESNIFFER_VERBOSITY > 1) {
                                    echo "\t\t* look ahead ignored T_DOC_COMMENT, continuing *" . PHP_EOL;
                                }
                            }
                            else {
                                if (PHP_CODESNIFFER_VERBOSITY > 1) {
                                    echo "\t\t* look ahead found more specific token ({$newType}), ignoring {$i} *" . PHP_EOL;
                                }
                                $matchedToken = true;
                                break;
                            }
                        }
                        
                        //end if
                    }
                    
                    //end for
                }
                
                //end if
                if ($matchedToken === false) {
                    if (PHP_CODESNIFFER_VERBOSITY > 1 && $lookAheadLength > 0) {
                        echo "\t\t* look ahead found nothing *" . PHP_EOL;
                    }
                    $value = $this->tokenValues[strtolower($buffer)];
                    if ($value === 'T_FUNCTION' && $buffer !== 'function') {
                        // The function keyword needs to be all lowercase or else
                        // it is just a function called "Function".
                        $value = 'T_STRING';
                    }
                    $tokens[] = [
                        'code' => constant($value),
                        'type' => $value,
                        'content' => $buffer,
                    ];
                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        $content = Util\Common::prepareForOutput($buffer);
                        echo "\t=> Added token {$value} ({$content})" . PHP_EOL;
                    }
                    $cleanBuffer = true;
                }
                
                //end if
            }
            else {
                if (isset($this->tokenValues[strtolower($char)]) === true) {
                    // No matter what token we end up using, we don't
                    // need the content in the buffer any more because we have
                    // found a valid token.
                    $newContent = substr(str_replace("\n", $this->eolChar, $buffer), 0, -1);
                    if ($newContent !== '') {
                        $tokens[] = [
                            'code' => T_STRING,
                            'type' => 'T_STRING',
                            'content' => $newContent,
                        ];
                        if (PHP_CODESNIFFER_VERBOSITY > 1) {
                            $content = Util\Common::prepareForOutput(substr($buffer, 0, -1));
                            echo "\t=> Added token T_STRING ({$content})" . PHP_EOL;
                        }
                    }
                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        echo "\t\t* char is token, looking ahead " . ($maxTokenLength - 1) . ' chars *' . PHP_EOL;
                    }
                    // The char is a token type, but we need to look ahead at the
                    // next chars to see if this is actually part of a larger token.
                    // For example, = and ===.
                    $charBuffer = $char;
                    $matchedToken = false;
                    for ($x = 1; $x <= $maxTokenLength; $x++) {
                        if (isset($chars[$i + $x]) === false) {
                            break;
                        }
                        $charBuffer .= $chars[$i + $x];
                        if (PHP_CODESNIFFER_VERBOSITY > 1) {
                            $content = Util\Common::prepareForOutput($charBuffer);
                            echo "\t\t=> Looking ahead {$x} chars => {$content}" . PHP_EOL;
                        }
                        if (isset($this->tokenValues[strtolower($charBuffer)]) === true) {
                            // We've found something larger that matches
                            // so we can ignore this char.
                            if (PHP_CODESNIFFER_VERBOSITY > 1) {
                                $type = $this->tokenValues[strtolower($charBuffer)];
                                echo "\t\t* look ahead found more specific token ({$type}), ignoring {$i} *" . PHP_EOL;
                            }
                            $matchedToken = true;
                            break;
                        }
                    }
                    
                    //end for
                    if ($matchedToken === false) {
                        $value = $this->tokenValues[strtolower($char)];
                        $tokens[] = [
                            'code' => constant($value),
                            'type' => $value,
                            'content' => $char,
                        ];
                        if (PHP_CODESNIFFER_VERBOSITY > 1) {
                            echo "\t\t* look ahead found nothing *" . PHP_EOL;
                            $content = Util\Common::prepareForOutput($char);
                            echo "\t=> Added token {$value} ({$content})" . PHP_EOL;
                        }
                        $cleanBuffer = true;
                    }
                    else {
                        $buffer = $char;
                    }
                    
                    //end if
                }
            }
            
            //end if
            // Keep track of content inside comments.
            if ($inComment === '' && array_key_exists($buffer, $this->commentTokens) === true) {
                // This is not really a comment if the content
                // looks like \// (i.e., it is escaped).
                if (isset($chars[$i - 2]) === true && $chars[$i - 2] === '\\') {
                    $lastToken = array_pop($tokens);
                    $lastContent = $lastToken['content'];
                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        $value = $this->tokenValues[strtolower($lastContent)];
                        $content = Util\Common::prepareForOutput($lastContent);
                        echo "\t=> Removed token {$value} ({$content})" . PHP_EOL;
                    }
                    $lastChars = str_split($lastContent);
                    $lastNumChars = count($lastChars);
                    for ($x = 0; $x < $lastNumChars; $x++) {
                        $lastChar = $lastChars[$x];
                        $value = $this->tokenValues[strtolower($lastChar)];
                        $tokens[] = [
                            'code' => constant($value),
                            'type' => $value,
                            'content' => $lastChar,
                        ];
                        if (PHP_CODESNIFFER_VERBOSITY > 1) {
                            $content = Util\Common::prepareForOutput($lastChar);
                            echo "\t=> Added token {$value} ({$content})" . PHP_EOL;
                        }
                    }
                }
                else {
                    // We have started a comment.
                    $inComment = $buffer;
                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        echo "\t\t* looking for end of comment *" . PHP_EOL;
                    }
                }
                
                //end if
            }
            else {
                if ($inComment !== '') {
                    if ($this->commentTokens[$inComment] === null) {
                        // Comment ends at the next newline.
                        if (strpos($buffer, "\n") !== false) {
                            $inComment = '';
                        }
                    }
                    else {
                        if ($this->commentTokens[$inComment] === $buffer) {
                            $inComment = '';
                        }
                    }
                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        if ($inComment === '') {
                            echo "\t\t* found end of comment *" . PHP_EOL;
                        }
                    }
                    if ($inComment === '' && $cleanBuffer === false) {
                        $tokens[] = [
                            'code' => T_STRING,
                            'type' => 'T_STRING',
                            'content' => str_replace("\n", $this->eolChar, $buffer),
                        ];
                        if (PHP_CODESNIFFER_VERBOSITY > 1) {
                            $content = Util\Common::prepareForOutput($buffer);
                            echo "\t=> Added token T_STRING ({$content})" . PHP_EOL;
                        }
                        $buffer = '';
                    }
                }
            }
            
            //end if
            if ($cleanBuffer === true) {
                $buffer = '';
                $cleanBuffer = false;
            }
        }
        
        //end for
        if (empty($buffer) === false) {
            if ($inString !== '') {
                // The string did not end before the end of the file,
                // which means there was probably a syntax error somewhere.
                $tokens[] = [
                    'code' => T_STRING,
                    'type' => 'T_STRING',
                    'content' => str_replace("\n", $this->eolChar, $buffer),
                ];
                if (PHP_CODESNIFFER_VERBOSITY > 1) {
                    $content = Util\Common::prepareForOutput($buffer);
                    echo "\t=> Added token T_STRING ({$content})" . PHP_EOL;
                }
            }
            else {
                // Buffer contains whitespace from the end of the file.
                $tokens[] = [
                    'code' => T_WHITESPACE,
                    'type' => 'T_WHITESPACE',
                    'content' => str_replace("\n", $this->eolChar, $buffer),
                ];
                if (PHP_CODESNIFFER_VERBOSITY > 1) {
                    $content = Util\Common::prepareForOutput($buffer);
                    echo "\t=> Added token T_WHITESPACE ({$content})" . PHP_EOL;
                }
            }
            
            //end if
        }
        
        //end if
        $tokens[] = [
            'code' => T_CLOSE_TAG,
            'type' => 'T_CLOSE_TAG',
            'content' => '',
        ];
        
        /*
            Now that we have done some basic tokenizing, we need to
            modify the tokens to join some together and split some apart
            so they match what the PHP tokenizer does.
        */
        $finalTokens = [];
        $newStackPtr = 0;
        $numTokens = count($tokens);
        for ($stackPtr = 0; $stackPtr < $numTokens; $stackPtr++) {
            $token = $tokens[$stackPtr];
            
            /*
                Look for comments and join the tokens together.
            */
            if ($token['code'] === T_COMMENT || $token['code'] === T_DOC_COMMENT) {
                $newContent = '';
                $tokenContent = $token['content'];
                $endContent = null;
                if (isset($this->commentTokens[$tokenContent]) === true) {
                    $endContent = $this->commentTokens[$tokenContent];
                }
                while ($tokenContent !== $endContent) {
                    if ($endContent === null && strpos($tokenContent, $this->eolChar) !== false) {
                        // A null end token means the comment ends at the end of
                        // the line so we look for newlines and split the token.
                        $tokens[$stackPtr]['content'] = substr($tokenContent, strpos($tokenContent, $this->eolChar) + strlen($this->eolChar));
                        $tokenContent = substr($tokenContent, 0, strpos($tokenContent, $this->eolChar) + strlen($this->eolChar));
                        // If the substr failed, skip the token as the content
                        // will now be blank.
                        if ($tokens[$stackPtr]['content'] !== false && $tokens[$stackPtr]['content'] !== '') {
                            $stackPtr--;
                        }
                        break;
                    }
                    
                    //end if
                    $stackPtr++;
                    $newContent .= $tokenContent;
                    if (isset($tokens[$stackPtr]) === false) {
                        break;
                    }
                    $tokenContent = $tokens[$stackPtr]['content'];
                }
                
                //end while
                if ($token['code'] === T_DOC_COMMENT) {
                    $commentTokens = $commentTokenizer->tokenizeString($newContent . $tokenContent, $this->eolChar, $newStackPtr);
                    foreach ($commentTokens as $commentToken) {
                        $finalTokens[$newStackPtr] = $commentToken;
                        $newStackPtr++;
                    }
                    continue;
                }
                else {
                    // Save the new content in the current token so
                    // the code below can chop it up on newlines.
                    $token['content'] = $newContent . $tokenContent;
                }
            }
            
            //end if
            
            /*
                If this token has newlines in its content, split each line up
                and create a new token for each line. We do this so it's easier
                to ascertain where errors occur on a line.
                Note that $token[1] is the token's content.
            */
            if (strpos($token['content'], $this->eolChar) !== false) {
                $tokenLines = explode($this->eolChar, $token['content']);
                $numLines = count($tokenLines);
                for ($i = 0; $i < $numLines; $i++) {
                    $newToken = [
                        'content' => $tokenLines[$i],
                    ];
                    if ($i === $numLines - 1) {
                        if ($tokenLines[$i] === '') {
                            break;
                        }
                    }
                    else {
                        $newToken['content'] .= $this->eolChar;
                    }
                    $newToken['type'] = $token['type'];
                    $newToken['code'] = $token['code'];
                    $finalTokens[$newStackPtr] = $newToken;
                    $newStackPtr++;
                }
            }
            else {
                $finalTokens[$newStackPtr] = $token;
                $newStackPtr++;
            }
            
            //end if
            // Convert numbers, including decimals.
            if ($token['code'] === T_STRING || $token['code'] === T_OBJECT_OPERATOR) {
                $newContent = '';
                $oldStackPtr = $stackPtr;
                while (preg_match('|^[0-9\\.]+$|', $tokens[$stackPtr]['content']) !== 0) {
                    $newContent .= $tokens[$stackPtr]['content'];
                    $stackPtr++;
                }
                if ($newContent !== '' && $newContent !== '.') {
                    $finalTokens[$newStackPtr - 1]['content'] = $newContent;
                    if (ctype_digit($newContent) === true) {
                        $finalTokens[$newStackPtr - 1]['code'] = constant('T_LNUMBER');
                        $finalTokens[$newStackPtr - 1]['type'] = 'T_LNUMBER';
                    }
                    else {
                        $finalTokens[$newStackPtr - 1]['code'] = constant('T_DNUMBER');
                        $finalTokens[$newStackPtr - 1]['type'] = 'T_DNUMBER';
                    }
                    $stackPtr--;
                    continue;
                }
                else {
                    $stackPtr = $oldStackPtr;
                }
            }
            
            //end if
            // Convert the token after an object operator into a string, in most cases.
            if ($token['code'] === T_OBJECT_OPERATOR) {
                for ($i = $stackPtr + 1; $i < $numTokens; $i++) {
                    if (isset(Util\Tokens::$emptyTokens[$tokens[$i]['code']]) === true) {
                        continue;
                    }
                    if ($tokens[$i]['code'] !== T_PROTOTYPE && $tokens[$i]['code'] !== T_LNUMBER && $tokens[$i]['code'] !== T_DNUMBER) {
                        $tokens[$i]['code'] = T_STRING;
                        $tokens[$i]['type'] = 'T_STRING';
                    }
                    break;
                }
            }
        }
        
        //end for
        if (PHP_CODESNIFFER_VERBOSITY > 1) {
            echo "\t*** END TOKENIZING ***" . PHP_EOL;
        }
        return $finalTokens;
    }
    
    //end tokenize()
    
    /**
     * Tokenizes a regular expression if one is found.
     *
     * If a regular expression is not found, NULL is returned.
     *
     * @param int    $char   The index of the possible regex start character.
     * @param string $string The complete content of the string being tokenized.
     * @param array  $chars  An array of characters being tokenized.
     * @param array  $tokens The current array of tokens found in the string.
     *
     * @return array<string, string>|null
     */
    public function getRegexToken($char, $string, $chars, $tokens) {
        $beforeTokens = [
            T_EQUAL => true,
            T_IS_NOT_EQUAL => true,
            T_IS_IDENTICAL => true,
            T_IS_NOT_IDENTICAL => true,
            T_OPEN_PARENTHESIS => true,
            T_OPEN_SQUARE_BRACKET => true,
            T_RETURN => true,
            T_BOOLEAN_OR => true,
            T_BOOLEAN_AND => true,
            T_BOOLEAN_NOT => true,
            T_BITWISE_OR => true,
            T_BITWISE_AND => true,
            T_COMMA => true,
            T_COLON => true,
            T_TYPEOF => true,
            T_INLINE_THEN => true,
            T_INLINE_ELSE => true,
        ];
        $afterTokens = [
            ',' => true,
            ')' => true,
            ']' => true,
            ';' => true,
            ' ' => true,
            '.' => true,
            ':' => true,
            $this->eolChar => true,
        ];
        // Find the last non-whitespace token that was added
        // to the tokens array.
        $numTokens = count($tokens);
        for ($prev = $numTokens - 1; $prev >= 0; $prev--) {
            if (isset(Util\Tokens::$emptyTokens[$tokens[$prev]['code']]) === false) {
                break;
            }
        }
        if (isset($beforeTokens[$tokens[$prev]['code']]) === false) {
            return null;
        }
        // This is probably a regular expression, so look for the end of it.
        if (PHP_CODESNIFFER_VERBOSITY > 1) {
            echo "\t* token possibly starts a regular expression *" . PHP_EOL;
        }
        $numChars = count($chars);
        for ($next = $char + 1; $next < $numChars; $next++) {
            if ($chars[$next] === '/') {
                // Just make sure this is not escaped first.
                if ($chars[$next - 1] !== '\\') {
                    // In the simple form: /.../ so we found the end.
                    break;
                }
                else {
                    if ($chars[$next - 2] === '\\') {
                        // In the form: /...\\/ so we found the end.
                        break;
                    }
                }
            }
            else {
                $possibleEolChar = substr($string, $next, strlen($this->eolChar));
                if ($possibleEolChar === $this->eolChar) {
                    // This is the last token on the line and regular
                    // expressions need to be defined on a single line,
                    // so this is not a regular expression.
                    break;
                }
            }
        }
        if ($chars[$next] !== '/') {
            if (PHP_CODESNIFFER_VERBOSITY > 1) {
                echo "\t* could not find end of regular expression *" . PHP_EOL;
            }
            return null;
        }
        while (preg_match('|[a-zA-Z]|', $chars[$next + 1]) !== 0) {
            // The token directly after the end of the regex can
            // be modifiers like global and case insensitive
            // (.e.g, /pattern/gi).
            $next++;
        }
        $regexEnd = $next;
        if (PHP_CODESNIFFER_VERBOSITY > 1) {
            echo "\t* found end of regular expression at token {$regexEnd} *" . PHP_EOL;
        }
        for ($next += 1; $next < $numChars; $next++) {
            if ($chars[$next] !== ' ') {
                break;
            }
            else {
                $possibleEolChar = substr($string, $next, strlen($this->eolChar));
                if ($possibleEolChar === $this->eolChar) {
                    // This is the last token on the line.
                    break;
                }
            }
        }
        if (isset($afterTokens[$chars[$next]]) === false) {
            if (PHP_CODESNIFFER_VERBOSITY > 1) {
                echo "\t* tokens after regular expression do not look correct *" . PHP_EOL;
            }
            return null;
        }
        // This is a regular expression, so join all the tokens together.
        $content = '';
        for ($x = $char; $x <= $regexEnd; $x++) {
            $content .= $chars[$x];
        }
        $token = [
            'start' => $char,
            'end' => $regexEnd,
            'content' => $content,
        ];
        return $token;
    }
    
    //end getRegexToken()
    
    /**
     * Performs additional processing after main tokenizing.
     *
     * This additional processing looks for properties, closures, labels and objects.
     *
     * @return void
     */
    public function processAdditional() {
        if (PHP_CODESNIFFER_VERBOSITY > 1) {
            echo "\t*** START ADDITIONAL JS PROCESSING ***" . PHP_EOL;
        }
        $numTokens = count($this->tokens);
        $classStack = [];
        for ($i = 0; $i < $numTokens; $i++) {
            if (PHP_CODESNIFFER_VERBOSITY > 1) {
                $type = $this->tokens[$i]['type'];
                $content = Util\Common::prepareForOutput($this->tokens[$i]['content']);
                echo str_repeat("\t", count($classStack));
                echo "\tProcess token {$i}: {$type} => {$content}" . PHP_EOL;
            }
            // Looking for functions that are actually closures.
            if ($this->tokens[$i]['code'] === T_FUNCTION && isset($this->tokens[$i]['scope_opener']) === true) {
                for ($x = $i + 1; $x < $numTokens; $x++) {
                    if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === false) {
                        break;
                    }
                }
                if ($this->tokens[$x]['code'] === T_OPEN_PARENTHESIS) {
                    $this->tokens[$i]['code'] = T_CLOSURE;
                    $this->tokens[$i]['type'] = 'T_CLOSURE';
                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        $line = $this->tokens[$i]['line'];
                        echo str_repeat("\t", count($classStack));
                        echo "\t* token {$i} on line {$line} changed from T_FUNCTION to T_CLOSURE *" . PHP_EOL;
                    }
                    for ($x = $this->tokens[$i]['scope_opener'] + 1; $x < $this->tokens[$i]['scope_closer']; $x++) {
                        if (isset($this->tokens[$x]['conditions'][$i]) === false) {
                            continue;
                        }
                        $this->tokens[$x]['conditions'][$i] = T_CLOSURE;
                        if (PHP_CODESNIFFER_VERBOSITY > 1) {
                            $type = $this->tokens[$x]['type'];
                            echo str_repeat("\t", count($classStack));
                            echo "\t\t* cleaned {$x} ({$type}) *" . PHP_EOL;
                        }
                    }
                }
                
                //end if
                continue;
            }
            else {
                if ($this->tokens[$i]['code'] === T_OPEN_CURLY_BRACKET && isset($this->tokens[$i]['scope_condition']) === false && isset($this->tokens[$i]['bracket_closer']) === true) {
                    $condition = $this->tokens[$i]['conditions'];
                    $condition = end($condition);
                    if ($condition === T_CLASS) {
                        // Possibly an ES6 method. To be classified as one, the previous
                        // non-empty tokens need to be a set of parenthesis, and then a string
                        // (the method name).
                        for ($parenCloser = $i - 1; $parenCloser > 0; $parenCloser--) {
                            if (isset(Util\Tokens::$emptyTokens[$this->tokens[$parenCloser]['code']]) === false) {
                                break;
                            }
                        }
                        if ($this->tokens[$parenCloser]['code'] === T_CLOSE_PARENTHESIS) {
                            $parenOpener = $this->tokens[$parenCloser]['parenthesis_opener'];
                            for ($name = $parenOpener - 1; $name > 0; $name--) {
                                if (isset(Util\Tokens::$emptyTokens[$this->tokens[$name]['code']]) === false) {
                                    break;
                                }
                            }
                            if ($this->tokens[$name]['code'] === T_STRING) {
                                // We found a method name.
                                if (PHP_CODESNIFFER_VERBOSITY > 1) {
                                    $line = $this->tokens[$name]['line'];
                                    echo str_repeat("\t", count($classStack));
                                    echo "\t* token {$name} on line {$line} changed from T_STRING to T_FUNCTION *" . PHP_EOL;
                                }
                                $closer = $this->tokens[$i]['bracket_closer'];
                                $this->tokens[$name]['code'] = T_FUNCTION;
                                $this->tokens[$name]['type'] = 'T_FUNCTION';
                                foreach ([
                                    $name,
                                    $i,
                                    $closer,
                                ] as $token) {
                                    $this->tokens[$token]['scope_condition'] = $name;
                                    $this->tokens[$token]['scope_opener'] = $i;
                                    $this->tokens[$token]['scope_closer'] = $closer;
                                    $this->tokens[$token]['parenthesis_opener'] = $parenOpener;
                                    $this->tokens[$token]['parenthesis_closer'] = $parenCloser;
                                    $this->tokens[$token]['parenthesis_owner'] = $name;
                                }
                                $this->tokens[$parenOpener]['parenthesis_owner'] = $name;
                                $this->tokens[$parenCloser]['parenthesis_owner'] = $name;
                                for ($x = $i + 1; $x < $closer; $x++) {
                                    $this->tokens[$x]['conditions'][$name] = T_FUNCTION;
                                    ksort($this->tokens[$x]['conditions'], SORT_NUMERIC);
                                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                                        $type = $this->tokens[$x]['type'];
                                        echo str_repeat("\t", count($classStack));
                                        echo "\t\t* added T_FUNCTION condition to {$x} ({$type}) *" . PHP_EOL;
                                    }
                                }
                                continue;
                            }
                            
                            //end if
                        }
                        
                        //end if
                    }
                    
                    //end if
                    $classStack[] = $i;
                    $closer = $this->tokens[$i]['bracket_closer'];
                    $this->tokens[$i]['code'] = T_OBJECT;
                    $this->tokens[$i]['type'] = 'T_OBJECT';
                    $this->tokens[$closer]['code'] = T_CLOSE_OBJECT;
                    $this->tokens[$closer]['type'] = 'T_CLOSE_OBJECT';
                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        echo str_repeat("\t", count($classStack));
                        echo "\t* token {$i} converted from T_OPEN_CURLY_BRACKET to T_OBJECT *" . PHP_EOL;
                        echo str_repeat("\t", count($classStack));
                        echo "\t* token {$closer} converted from T_CLOSE_CURLY_BRACKET to T_CLOSE_OBJECT *" . PHP_EOL;
                    }
                    for ($x = $i + 1; $x < $closer; $x++) {
                        $this->tokens[$x]['conditions'][$i] = T_OBJECT;
                        ksort($this->tokens[$x]['conditions'], SORT_NUMERIC);
                        if (PHP_CODESNIFFER_VERBOSITY > 1) {
                            $type = $this->tokens[$x]['type'];
                            echo str_repeat("\t", count($classStack));
                            echo "\t\t* added T_OBJECT condition to {$x} ({$type}) *" . PHP_EOL;
                        }
                    }
                }
                else {
                    if ($this->tokens[$i]['code'] === T_CLOSE_OBJECT) {
                        array_pop($classStack);
                    }
                    else {
                        if ($this->tokens[$i]['code'] === T_COLON) {
                            // If it is a scope opener, it belongs to a
                            // DEFAULT or CASE statement.
                            if (isset($this->tokens[$i]['scope_condition']) === true) {
                                continue;
                            }
                            // Make sure this is not part of an inline IF statement.
                            for ($x = $i - 1; $x >= 0; $x--) {
                                if ($this->tokens[$x]['code'] === T_INLINE_THEN) {
                                    $this->tokens[$i]['code'] = T_INLINE_ELSE;
                                    $this->tokens[$i]['type'] = 'T_INLINE_ELSE';
                                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                                        echo str_repeat("\t", count($classStack));
                                        echo "\t* token {$i} converted from T_COLON to T_INLINE_THEN *" . PHP_EOL;
                                    }
                                    continue 2;
                                }
                                else {
                                    if ($this->tokens[$x]['line'] < $this->tokens[$i]['line']) {
                                        break;
                                    }
                                }
                            }
                            // The string to the left of the colon is either a property or label.
                            for ($label = $i - 1; $label >= 0; $label--) {
                                if (isset(Util\Tokens::$emptyTokens[$this->tokens[$label]['code']]) === false) {
                                    break;
                                }
                            }
                            if ($this->tokens[$label]['code'] !== T_STRING && $this->tokens[$label]['code'] !== T_CONSTANT_ENCAPSED_STRING) {
                                continue;
                            }
                            if (empty($classStack) === false) {
                                $this->tokens[$label]['code'] = T_PROPERTY;
                                $this->tokens[$label]['type'] = 'T_PROPERTY';
                                if (PHP_CODESNIFFER_VERBOSITY > 1) {
                                    echo str_repeat("\t", count($classStack));
                                    echo "\t* token {$label} converted from T_STRING to T_PROPERTY *" . PHP_EOL;
                                }
                            }
                            else {
                                $this->tokens[$label]['code'] = T_LABEL;
                                $this->tokens[$label]['type'] = 'T_LABEL';
                                if (PHP_CODESNIFFER_VERBOSITY > 1) {
                                    echo str_repeat("\t", count($classStack));
                                    echo "\t* token {$label} converted from T_STRING to T_LABEL *" . PHP_EOL;
                                }
                            }
                            
                            //end if
                        }
                    }
                }
            }
            
            //end if
        }
        
        //end for
        if (PHP_CODESNIFFER_VERBOSITY > 1) {
            echo "\t*** END ADDITIONAL JS PROCESSING ***" . PHP_EOL;
        }
    }
    
    //end processAdditional()

}