class Prism::Translation::Parser

def tokenize(source_buffer, recover = false)
  @source_buffer = source_buffer
  source = source_buffer.source
  offset_cache = build_offset_cache(source)
  result =
    begin
      unwrap(@parser.parse_lex(source, **prism_options), offset_cache)
    rescue ::Parser::SyntaxError
      raise if !recover
    end
  program, tokens = result.value
  ast = build_ast(program, offset_cache) if result.success?
  [
    ast,
    build_comments(result.comments, offset_cache),
    build_tokens(tokens, offset_cache)
  ]
ensure
  @source_buffer = nil
end