class RuboCop::AST::ProcessedSource
rubocop:disable Metrics/ClassLength
It also provides a convenient way to access source lines.
and other information such as disabled lines for cops.
ProcessedSource contains objects which are generated by Parser
def self.from_file(path, ruby_version, parser_engine: :default)
def self.from_file(path, ruby_version, parser_engine: :default) file = File.read(path, mode: 'rb') new(file, ruby_version, path, parser_engine: parser_engine) end
def self.lazy_tokens_parser_class(base)
- Api: - private
def self.lazy_tokens_parser_class(base) @lazy_tokens_parser_classes ||= {} @lazy_tokens_parser_classes[base] ||= Class.new(base) { include PrismLazyTokens } end
def [](*args)
def [](*args) lines[*args] end
def ast_with_comments
def ast_with_comments return if !ast || !comments @ast_with_comments ||= Parser::Source::Comment.associate_by_identity(ast, comments) end
def blank?
def blank? ast.nil? end
def checksum
def checksum @checksum ||= Digest::SHA1.hexdigest(@raw_source) end
def comment_at_line(line)
-
(Comment, nil)- the comment at that line, if any.
def comment_at_line(line) comment_index[line] end
def comment_index
def comment_index @comment_index ||= {}.tap do |hash| comments.each { |c| hash[c.location.line] = c } end end
def comments_before_line(line)
- Use `each_comment_in_lines`
def comments_before_line(line) each_comment_in_lines(0..line).to_a end
def contains_comment?(source_range)
-
(Boolean)- if any of the lines in the given `source_range` has a comment.
def contains_comment?(source_range) each_comment_in_lines(source_range.line..source_range.last_line).any? end
def create_parser(ruby_version, parser_engine, prism_result)
def create_parser(ruby_version, parser_engine, prism_result) # NOTE: Since it is intended for use with Ruby LSP, it targets only Prism. # If there is no reuse of a pre-parsed result, such as in Ruby LSP, # regular parsing with Prism occurs. prism = PrismPreparsed.new(prism_result) if prism_result && parser_engine != :parser_whitequark parser_instance = case parser_engine when :prism PrismParser.new(ruby_version, prism: prism || Prism) when :parser_prism translation_parser(ruby_version, prism) else parser_class(ruby_version, parser_engine).new(Builder.new) end parser_instance.tap do |parser| # On JRuby there's a risk that we hang in tokenize() if we # don't set the all errors as fatal flag. The problem is caused by a bug # in Racc that is discussed in issue #93 of the whitequark/parser # project on GitHub. parser.diagnostics.all_errors_are_fatal = (RUBY_ENGINE != 'ruby') parser.diagnostics.ignore_warnings = false parser.diagnostics.consumer = lambda do |diagnostic| @diagnostics << diagnostic end end end
def current_line(token)
def current_line(token) lines[token.line - 1] end
def default_parser_engine(ruby_version)
compatible with Ruby 3.4. The translation layer of Prism needs the
the Parser gem does not support Ruby 3.5 or later and is not fully
Prism is used for all Ruby versions it can parse (3.3 and later);
def default_parser_engine(ruby_version) if ruby_version < 3.3 :parser_whitequark elsif ParserSource.vendored? :prism else :parser_prism end end
def each_comment(&block)
- Use `comments.each`
def each_comment(&block) comments.each(&block) end
def each_comment_in_lines(line_range)
def each_comment_in_lines(line_range) return to_enum(:each_comment_in_lines, line_range) unless block_given? line_range.each do |line| if (comment = comment_index[line]) yield comment end end end
def each_token(&block)
- Use `tokens.each`
def each_token(&block) tokens.each(&block) end
def file_path
def file_path buffer.name end
def find_comment(&block)
- Use `comment_at_line`, `each_comment_in_lines`, or `comments.find`
def find_comment(&block) comments.find(&block) end
def find_token(&block)
- Use `tokens.find`
def find_token(&block) tokens.find(&block) end
def first_token_index(range_or_node)
def first_token_index(range_or_node) begin_pos = source_range(range_or_node).begin_pos sorted_tokens.bsearch_index { |token| token.begin_pos >= begin_pos } end
def first_token_of(range_or_node)
def first_token_of(range_or_node) sorted_tokens[first_token_index(range_or_node)] end
def following_line(token)
def following_line(token) lines[token.line] end
def initialize(
def initialize( source, ruby_version, path = nil, parser_engine: :default, prism_result: nil ) parser_engine = normalize_parser_engine(parser_engine, ruby_version) # Defaults source encoding to UTF-8, regardless of the encoding it has # been read with, which could be non-utf8 depending on the default # external encoding. source = (+source).force_encoding(Encoding::UTF_8) unless source.encoding == Encoding::UTF_8 @raw_source = source @path = path @diagnostics = [] @ruby_version = ruby_version @parser_engine = parser_engine @parser_error = nil parse(source, ruby_version, parser_engine, prism_result) end
def last_token_index(range_or_node)
def last_token_index(range_or_node) end_pos = source_range(range_or_node).end_pos sorted_tokens.bsearch_index { |token| token.end_pos >= end_pos } end
def last_token_of(range_or_node)
def last_token_of(range_or_node) sorted_tokens[last_token_index(range_or_node)] end
def line_indentation(line_number)
def line_indentation(line_number) lines[line_number - 1] .match(/^(\s*)/)[1] .to_s .length end
def line_with_comment?(line)
-
(Boolean)- if the given line number has a comment.
def line_with_comment?(line) comment_index.include?(line) end
def lines
Returns the source lines, line break characters removed, excluding a
def lines @lines ||= begin all_lines = @buffer.source_lines if all_lines.include?('__END__') trim_lines_after_data_marker(all_lines) else # Don't consult the tokens (which may have to be built first) when # the source can't contain a data section. all_lines.dup end end end
def normalize_parser_engine(parser_engine, ruby_version)
def normalize_parser_engine(parser_engine, ruby_version) parser_engine = parser_engine.to_sym unless PARSER_ENGINES.include?(parser_engine) raise ArgumentError, 'The keyword argument `parser_engine` accepts `default`, ' \ "`parser_whitequark`, `parser_prism`, or `prism`, but `#{parser_engine}` " \ 'was passed.' end if parser_engine == :default default_parser_engine(ruby_version) else parser_engine end end
def parse(source, ruby_version, parser_engine, prism_result)
def parse(source, ruby_version, parser_engine, prism_result) @buffer = Parser::Source::Buffer.new(@path || STRING_SOURCE_NAME, 1) begin @buffer.source = source rescue EncodingError, Parser::UnknownEncodingInMagicComment => e @parser_error = e @ast = nil @comments = [] @tokens = [] return end parser = create_parser(ruby_version, parser_engine, prism_result) @ast, @comments, tokens = tokenize(parser) store_tokens(tokens) end
def parse_and_lex(parser)
def parse_and_lex(parser) if parser.respond_to?(:tokenize_deferred) parser.tokenize_deferred(@buffer) else parser.tokenize(@buffer) end end
def parser_class(ruby_version, parser_engine)
def parser_class(ruby_version, parser_engine) ParserSource.ensure_parser_gem(parser_engine) case parser_engine when :parser_whitequark case ruby_version when 1.9 require 'parser/ruby19' Parser::Ruby19 when 2.0 require 'parser/ruby20' Parser::Ruby20 when 2.1 require 'parser/ruby21' Parser::Ruby21 when 2.2 require 'parser/ruby22' Parser::Ruby22 when 2.3 require 'parser/ruby23' Parser::Ruby23 when 2.4 require 'parser/ruby24' Parser::Ruby24 when 2.5 require 'parser/ruby25' Parser::Ruby25 when 2.6 require 'parser/ruby26' Parser::Ruby26 when 2.7 require 'parser/ruby27' Parser::Ruby27 when 2.8, 3.0 require 'parser/ruby30' Parser::Ruby30 when 3.1 require 'parser/ruby31' Parser::Ruby31 when 3.2 require 'parser/ruby32' Parser::Ruby32 when 3.3 require 'parser/ruby33' Parser::Ruby33 when 3.4 require 'parser/ruby34' Parser::Ruby34 else raise ArgumentError, 'RuboCop supports target Ruby versions 3.4 and below with ' \ "`parser`. Specified target Ruby version: #{ruby_version.inspect}" end when :parser_prism case ruby_version when 3.3 Prism::Translation::Parser33 when 3.4 Prism::Translation::Parser34 when 3.5, 4.0 Prism::Translation::Parser40 when 4.1 Prism::Translation::Parser41 else raise ArgumentError, 'RuboCop supports target Ruby versions 3.3 and above with Prism. ' \ "Specified target Ruby version: #{ruby_version.inspect}" end end end
def parser_tokens
def parser_tokens if @deferred_parser_tokens @parser_tokens = @deferred_parser_tokens.call @deferred_parser_tokens = nil end @parser_tokens end
def preceding_line(token)
def preceding_line(token) return nil if token.line < 2 lines[token.line - 2] end
def sort_tokens
def sort_tokens # Combine the position and the index into a single integer to get a # stable sort without allocating an array per token for the key. size = tokens.size tokens.sort_by.with_index { |token, i| (token.begin_pos * size) + i } end
def sorted_tokens
commonly, whenever the source contains comments, whose tokens the
argument (the heredoc contents' tokens are interleaved) and, more
The tokens list is out of order when a heredoc is passed as a method
def sorted_tokens # Callers only ever read from the returned array, so it is safe to # reuse `tokens` as is when it's already in order. @sorted_tokens ||= if tokens_sorted? tokens else sort_tokens end end
def source_range(range_or_node)
def source_range(range_or_node) if range_or_node.respond_to?(:source_range) range_or_node.source_range else range_or_node end end
def start_with?(string)
def start_with?(string) return false if self[0].nil? self[0].start_with?(string) end
def store_tokens(tokens)
The tokens may be an already converted array, or a deferred conversion
def store_tokens(tokens) if tokens.is_a?(Proc) @deferred_parser_tokens = tokens else @parser_tokens = tokens end end
def tokenize(parser)
def tokenize(parser) begin ast, comments, tokens = parse_and_lex(parser) ast ||= nil # force `false` to `nil`, see https://github.com/whitequark/parser/pull/722 rescue Parser::SyntaxError # All errors are in diagnostics. No need to handle exception. comments = [] tokens = [] rescue StandardError => e # The parser gem may fail on invalid sources with an exception other than `Parser::SyntaxError` # and without a diagnostic, e.g. with `RegexpError` for `/\c\xFF/ =~ ""`. Keep the exception # so that it can be reported instead of crashing the caller. @parser_error = e comments = [] tokens = [] end ast&.complete! [ast, comments, tokens] end
def tokens
lazily on first access, since their conversion is costly and not
The tokens of the source. With the prism engine the tokens are built
def tokens @tokens ||= begin tokens = parser_tokens.map { |t| Token.from_parser_token(t) } # The parser tokens are no longer needed, and the deferred conversion # holds on to the parser instance, so release both. @parser_tokens = nil tokens end end
def tokens_sorted?
def tokens_sorted? # `each_cons` allocates an array per pair, which would defeat the # point of checking before sorting. (1...tokens.size).all? { |i| tokens[i - 1].begin_pos <= tokens[i].begin_pos } end
def tokens_within(range_or_node)
def tokens_within(range_or_node) begin_index = first_token_index(range_or_node) end_index = last_token_index(range_or_node) sorted_tokens[begin_index..end_index] end
def translation_parser(ruby_version, prism)
def translation_parser(ruby_version, prism) parser_class = self.class.lazy_tokens_parser_class(parser_class(ruby_version, :parser_prism)) prism ? parser_class.new(BuilderPrism.new, parser: prism) : parser_class.new(BuilderPrism.new) end
def trim_lines_after_data_marker(all_lines)
`__END__` only starts a data section when it isn't nested inside a
def trim_lines_after_data_marker(all_lines) last_token_line = tokens.any? ? tokens.last.line : all_lines.size result = [] all_lines.each_with_index do |line, ix| break if ix >= last_token_line && line == '__END__' result << line end result end
def valid_syntax?
def valid_syntax? return false if @parser_error @diagnostics.none? { |d| INVALID_LEVELS.include?(d.level) } end