class RuboCop::AST::ProcessedSource

rubocop:disable Metrics/ClassLength
It also provides a convenient way to access source lines.
and other information such as disabled lines for cops.
ProcessedSource contains objects which are generated by Parser

def self.from_file(path, ruby_version, parser_engine: :default)

def self.from_file(path, ruby_version, parser_engine: :default)
  file = File.read(path, mode: 'rb')
  new(file, ruby_version, path, parser_engine: parser_engine)
end

def self.lazy_tokens_parser_class(base)

Other tags:
    Api: - private
def self.lazy_tokens_parser_class(base)
  @lazy_tokens_parser_classes ||= {}
  @lazy_tokens_parser_classes[base] ||= Class.new(base) { include PrismLazyTokens }
end

def [](*args)

def [](*args)
  lines[*args]
end

def ast_with_comments

def ast_with_comments
  return if !ast || !comments
  @ast_with_comments ||= Parser::Source::Comment.associate_by_identity(ast, comments)
end

def blank?

def blank?
  ast.nil?
end

def checksum

Raw source checksum for tracking infinite loops.
def checksum
  @checksum ||= Digest::SHA1.hexdigest(@raw_source)
end

def comment_at_line(line)

Returns:
  • (Comment, nil) - the comment at that line, if any.
def comment_at_line(line)
  comment_index[line]
end

def comment_index

def comment_index
  @comment_index ||= {}.tap do |hash|
    comments.each { |c| hash[c.location.line] = c }
  end
end

def comments_before_line(line)

Deprecated:
  • Use `each_comment_in_lines`
def comments_before_line(line)
  each_comment_in_lines(0..line).to_a
end

def contains_comment?(source_range)

Returns:
  • (Boolean) - if any of the lines in the given `source_range` has a comment.
def contains_comment?(source_range)
  each_comment_in_lines(source_range.line..source_range.last_line).any?
end

def create_parser(ruby_version, parser_engine, prism_result)

rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength
def create_parser(ruby_version, parser_engine, prism_result)
  # NOTE: Since it is intended for use with Ruby LSP, it targets only Prism.
  # If there is no reuse of a pre-parsed result, such as in Ruby LSP,
  # regular parsing with Prism occurs.
  prism = PrismPreparsed.new(prism_result) if prism_result && parser_engine != :parser_whitequark
  parser_instance = case parser_engine
                    when :prism
                      PrismParser.new(ruby_version, prism: prism || Prism)
                    when :parser_prism
                      translation_parser(ruby_version, prism)
                    else
                      parser_class(ruby_version, parser_engine).new(Builder.new)
                    end
  parser_instance.tap do |parser|
    # On JRuby there's a risk that we hang in tokenize() if we
    # don't set the all errors as fatal flag. The problem is caused by a bug
    # in Racc that is discussed in issue #93 of the whitequark/parser
    # project on GitHub.
    parser.diagnostics.all_errors_are_fatal = (RUBY_ENGINE != 'ruby')
    parser.diagnostics.ignore_warnings = false
    parser.diagnostics.consumer = lambda do |diagnostic|
      @diagnostics << diagnostic
    end
  end
end

def current_line(token)

def current_line(token)
  lines[token.line - 1]
end

def default_parser_engine(ruby_version)

Parser gem as well, so without it the `prism` engine is the default.
compatible with Ruby 3.4. The translation layer of Prism needs the
the Parser gem does not support Ruby 3.5 or later and is not fully
Prism is used for all Ruby versions it can parse (3.3 and later);
def default_parser_engine(ruby_version)
  if ruby_version < 3.3
    :parser_whitequark
  elsif ParserSource.vendored?
    :prism
  else
    :parser_prism
  end
end

def each_comment(&block)

Deprecated:
  • Use `comments.each`
def each_comment(&block)
  comments.each(&block)
end

def each_comment_in_lines(line_range)

Enumerates on the comments contained with the given `line_range`
def each_comment_in_lines(line_range)
  return to_enum(:each_comment_in_lines, line_range) unless block_given?
  line_range.each do |line|
    if (comment = comment_index[line])
      yield comment
    end
  end
end

def each_token(&block)

Deprecated:
  • Use `tokens.each`
def each_token(&block)
  tokens.each(&block)
end

def file_path

def file_path
  buffer.name
end

def find_comment(&block)

Deprecated:
  • Use `comment_at_line`, `each_comment_in_lines`, or `comments.find`
def find_comment(&block)
  comments.find(&block)
end

def find_token(&block)

Deprecated:
  • Use `tokens.find`
def find_token(&block)
  tokens.find(&block)
end

def first_token_index(range_or_node)

def first_token_index(range_or_node)
  begin_pos = source_range(range_or_node).begin_pos
  sorted_tokens.bsearch_index { |token| token.begin_pos >= begin_pos }
end

def first_token_of(range_or_node)

def first_token_of(range_or_node)
  sorted_tokens[first_token_index(range_or_node)]
end

def following_line(token)

def following_line(token)
  lines[token.line]
end

def initialize(

def initialize(
  source, ruby_version, path = nil, parser_engine: :default, prism_result: nil
)
  parser_engine = normalize_parser_engine(parser_engine, ruby_version)
  # Defaults source encoding to UTF-8, regardless of the encoding it has
  # been read with, which could be non-utf8 depending on the default
  # external encoding.
  source = (+source).force_encoding(Encoding::UTF_8) unless source.encoding == Encoding::UTF_8
  @raw_source = source
  @path = path
  @diagnostics = []
  @ruby_version = ruby_version
  @parser_engine = parser_engine
  @parser_error = nil
  parse(source, ruby_version, parser_engine, prism_result)
end

def last_token_index(range_or_node)

def last_token_index(range_or_node)
  end_pos = source_range(range_or_node).end_pos
  sorted_tokens.bsearch_index { |token| token.end_pos >= end_pos }
end

def last_token_of(range_or_node)

def last_token_of(range_or_node)
  sorted_tokens[last_token_index(range_or_node)]
end

def line_indentation(line_number)

def line_indentation(line_number)
  lines[line_number - 1]
    .match(/^(\s*)/)[1]
    .to_s
    .length
end

def line_with_comment?(line)

Returns:
  • (Boolean) - if the given line number has a comment.
def line_with_comment?(line)
  comment_index.include?(line)
end

def lines

possible __END__ and everything that comes after.
Returns the source lines, line break characters removed, excluding a
def lines
  @lines ||= begin
    all_lines = @buffer.source_lines
    if all_lines.include?('__END__')
      trim_lines_after_data_marker(all_lines)
    else
      # Don't consult the tokens (which may have to be built first) when
      # the source can't contain a data section.
      all_lines.dup
    end
  end
end

def normalize_parser_engine(parser_engine, ruby_version)

def normalize_parser_engine(parser_engine, ruby_version)
  parser_engine = parser_engine.to_sym
  unless PARSER_ENGINES.include?(parser_engine)
    raise ArgumentError, 'The keyword argument `parser_engine` accepts `default`, ' \
                         "`parser_whitequark`, `parser_prism`, or `prism`, but `#{parser_engine}` " \
                         'was passed.'
  end
  if parser_engine == :default
    default_parser_engine(ruby_version)
  else
    parser_engine
  end
end

def parse(source, ruby_version, parser_engine, prism_result)

def parse(source, ruby_version, parser_engine, prism_result)
  @buffer = Parser::Source::Buffer.new(@path || STRING_SOURCE_NAME, 1)
  begin
    @buffer.source = source
  rescue EncodingError, Parser::UnknownEncodingInMagicComment => e
    @parser_error = e
    @ast = nil
    @comments = []
    @tokens = []
    return
  end
  parser = create_parser(ruby_version, parser_engine, prism_result)
  @ast, @comments, tokens = tokenize(parser)
  store_tokens(tokens)
end

def parse_and_lex(parser)

def parse_and_lex(parser)
  if parser.respond_to?(:tokenize_deferred)
    parser.tokenize_deferred(@buffer)
  else
    parser.tokenize(@buffer)
  end
end

def parser_class(ruby_version, parser_engine)

rubocop:disable-next Lint/FloatComparison, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
def parser_class(ruby_version, parser_engine)
  ParserSource.ensure_parser_gem(parser_engine)
  case parser_engine
  when :parser_whitequark
    case ruby_version
    when 1.9
      require 'parser/ruby19'
      Parser::Ruby19
    when 2.0
      require 'parser/ruby20'
      Parser::Ruby20
    when 2.1
      require 'parser/ruby21'
      Parser::Ruby21
    when 2.2
      require 'parser/ruby22'
      Parser::Ruby22
    when 2.3
      require 'parser/ruby23'
      Parser::Ruby23
    when 2.4
      require 'parser/ruby24'
      Parser::Ruby24
    when 2.5
      require 'parser/ruby25'
      Parser::Ruby25
    when 2.6
      require 'parser/ruby26'
      Parser::Ruby26
    when 2.7
      require 'parser/ruby27'
      Parser::Ruby27
    when 2.8, 3.0
      require 'parser/ruby30'
      Parser::Ruby30
    when 3.1
      require 'parser/ruby31'
      Parser::Ruby31
    when 3.2
      require 'parser/ruby32'
      Parser::Ruby32
    when 3.3
      require 'parser/ruby33'
      Parser::Ruby33
    when 3.4
      require 'parser/ruby34'
      Parser::Ruby34
    else
      raise ArgumentError, 'RuboCop supports target Ruby versions 3.4 and below with ' \
                           "`parser`. Specified target Ruby version: #{ruby_version.inspect}"
    end
  when :parser_prism
    case ruby_version
    when 3.3
      Prism::Translation::Parser33
    when 3.4
      Prism::Translation::Parser34
    when 3.5, 4.0
      Prism::Translation::Parser40
    when 4.1
      Prism::Translation::Parser41
    else
      raise ArgumentError, 'RuboCop supports target Ruby versions 3.3 and above with Prism. ' \
                           "Specified target Ruby version: #{ruby_version.inspect}"
    end
  end
end

def parser_tokens

def parser_tokens
  if @deferred_parser_tokens
    @parser_tokens = @deferred_parser_tokens.call
    @deferred_parser_tokens = nil
  end
  @parser_tokens
end

def preceding_line(token)

def preceding_line(token)
  return nil if token.line < 2
  lines[token.line - 2]
end

def sort_tokens

def sort_tokens
  # Combine the position and the index into a single integer to get a
  # stable sort without allocating an array per token for the key.
  size = tokens.size
  tokens.sort_by.with_index { |token, i| (token.begin_pos * size) + i }
end

def sorted_tokens

lexer emits out of band.
commonly, whenever the source contains comments, whose tokens the
argument (the heredoc contents' tokens are interleaved) and, more
The tokens list is out of order when a heredoc is passed as a method
def sorted_tokens
  # Callers only ever read from the returned array, so it is safe to
  # reuse `tokens` as is when it's already in order.
  @sorted_tokens ||= if tokens_sorted?
                       tokens
                     else
                       sort_tokens
                     end
end

def source_range(range_or_node)

def source_range(range_or_node)
  if range_or_node.respond_to?(:source_range)
    range_or_node.source_range
  else
    range_or_node
  end
end

def start_with?(string)

def start_with?(string)
  return false if self[0].nil?
  self[0].start_with?(string)
end

def store_tokens(tokens)

to be performed when the tokens are first accessed.
The tokens may be an already converted array, or a deferred conversion
def store_tokens(tokens)
  if tokens.is_a?(Proc)
    @deferred_parser_tokens = tokens
  else
    @parser_tokens = tokens
  end
end

def tokenize(parser)

def tokenize(parser)
  begin
    ast, comments, tokens = parse_and_lex(parser)
    ast ||= nil # force `false` to `nil`, see https://github.com/whitequark/parser/pull/722
  rescue Parser::SyntaxError
    # All errors are in diagnostics. No need to handle exception.
    comments = []
    tokens = []
  rescue StandardError => e
    # The parser gem may fail on invalid sources with an exception other than `Parser::SyntaxError`
    # and without a diagnostic, e.g. with `RegexpError` for `/\c\xFF/ =~ ""`. Keep the exception
    # so that it can be reported instead of crashing the caller.
    @parser_error = e
    comments = []
    tokens = []
  end
  ast&.complete!
  [ast, comments, tokens]
end

def tokens

every caller needs them.
lazily on first access, since their conversion is costly and not
The tokens of the source. With the prism engine the tokens are built
def tokens
  @tokens ||= begin
    tokens = parser_tokens.map { |t| Token.from_parser_token(t) }
    # The parser tokens are no longer needed, and the deferred conversion
    # holds on to the parser instance, so release both.
    @parser_tokens = nil
    tokens
  end
end

def tokens_sorted?

def tokens_sorted?
  # `each_cons` allocates an array per pair, which would defeat the
  # point of checking before sorting.
  (1...tokens.size).all? { |i| tokens[i - 1].begin_pos <= tokens[i].begin_pos }
end

def tokens_within(range_or_node)

def tokens_within(range_or_node)
  begin_index = first_token_index(range_or_node)
  end_index = last_token_index(range_or_node)
  sorted_tokens[begin_index..end_index]
end

def translation_parser(ruby_version, prism)

def translation_parser(ruby_version, prism)
  parser_class = self.class.lazy_tokens_parser_class(parser_class(ruby_version, :parser_prism))
  prism ? parser_class.new(BuilderPrism.new, parser: prism) : parser_class.new(BuilderPrism.new)
end

def trim_lines_after_data_marker(all_lines)

string or heredoc, which is what the last token's line disambiguates.
`__END__` only starts a data section when it isn't nested inside a
def trim_lines_after_data_marker(all_lines)
  last_token_line = tokens.any? ? tokens.last.line : all_lines.size
  result = []
  all_lines.each_with_index do |line, ix|
    break if ix >= last_token_line && line == '__END__'
    result << line
  end
  result
end

def valid_syntax?

def valid_syntax?
  return false if @parser_error
  @diagnostics.none? { |d| INVALID_LEVELS.include?(d.level) }
end