class RuboCop::AST::ProcessedSource

def tokenize(parser)
  begin
    ast, comments, tokens = parser.tokenize(@buffer)
    ast ||= nil # force `false` to `nil`, see https://github.com/whitequark/parser/pull/722
  rescue Parser::SyntaxError
    # All errors are in diagnostics. No need to handle exception.
    comments = []
    tokens = []
  end
  ast&.complete!
  tokens.map! { |t| Token.from_parser_token(t) }
  [ast, comments, tokens]
end