class Parser::Base

def tokenize(source_buffer, recover=false)
  @lexer.tokens = []
  @lexer.comments = []
  begin
    ast = parse(source_buffer)
  rescue Parser::SyntaxError
    raise if !recover
  end
  [ ast, @lexer.comments, @lexer.tokens ]
ensure
  @lexer.tokens = nil
  @lexer.comments = nil
end