class RDoc::Parser::Ruby

def collect_first_comment
  skip_tkspace
  comment = ''.dup
  comment = RDoc::Encoding.change_encoding comment, @encoding if @encoding
  first_line = true
  first_comment_tk_kind = nil
  line_no = nil
  tk = get_tk
  while tk && (:on_comment == tk[:kind] or :on_embdoc == tk[:kind])
    comment_body = retrieve_comment_body(tk)
    if first_line and comment_body =~ /\A#!/ then
      skip_tkspace
      tk = get_tk
    elsif first_line and comment_body =~ /\A#\s*-\*-/ then
      first_line = false
      skip_tkspace
      tk = get_tk
    else
      break if first_comment_tk_kind and not first_comment_tk_kind === tk[:kind]
      first_comment_tk_kind = tk[:kind]
      line_no = tk[:line_no] if first_line
      first_line = false
      comment << comment_body
      tk = get_tk
      if :on_nl === tk then
        skip_tkspace_without_nl
        tk = get_tk
      end
    end
  end
  unget_tk tk
  new_comment comment, line_no
end

def consume_trailing_spaces # :nodoc:
  skip_tkspace_without_nl
end

def create_attr(container, single, name, rw, comment) # :nodoc:
  att = RDoc::Attr.new get_tkread, name, rw, comment, singleton: single == SINGLE
  record_location att
  container.add_attribute att
  @stats.add_attribute att
  att
end

def create_module_alias(container, constant, rhs_name) # :nodoc:
  mod = if rhs_name =~ /^::/ then
          @store.find_class_or_module rhs_name
        else
          container.find_module_named rhs_name
        end
  container.add_module_alias mod, rhs_name, constant, @top_level
end

def error(msg)
  msg = make_message msg
  abort msg
end

def get_bool
  skip_tkspace
  tk = get_tk
  if :on_kw == tk[:kind] && 'true' == tk[:text]
    true
  elsif :on_kw == tk[:kind] && ('false' == tk[:text] || 'nil' == tk[:text])
    false
  else
    unget_tk tk
    true
  end
end

def get_class_or_module(container, ignore_constants = false)
  skip_tkspace
  name_t = get_tk
  given_name = ''.dup
  # class ::A -> A is in the top level
  if :on_op == name_t[:kind] and '::' == name_t[:text] then # bug
    name_t = get_tk
    container = @top_level
    given_name << '::'
  end
  skip_tkspace_without_nl
  given_name << name_t[:text]
  is_self = name_t[:kind] == :on_op && name_t[:text] == '<<'
  new_modules = []
  while !is_self && (tk = peek_tk) and :on_op == tk[:kind] and '::' == tk[:text] do
    prev_container = container
    container = container.find_module_named name_t[:text]
    container ||=
      if ignore_constants then
        c = RDoc::NormalModule.new name_t[:text]
        c.store = @store
        new_modules << [prev_container, c]
        c
      else
        c = prev_container.add_module RDoc::NormalModule, name_t[:text]
        c.ignore unless prev_container.document_children
        @top_level.add_to_classes_or_modules c
        c
      end
    record_location container
    get_tk
    skip_tkspace
    if :on_lparen == peek_tk[:kind] # ProcObjectInConstant::()
      parse_method_or_yield_parameters
      break
    end
    name_t = get_tk
    unless :on_const == name_t[:kind] || :on_ident == name_t[:kind]
      raise RDoc::Error, "Invalid class or module definition: #{given_name}"
    end
    if prev_container == container and !ignore_constants
      given_name = name_t[:text]
    else
      given_name << '::' + name_t[:text]
    end
  end
  skip_tkspace_without_nl
  return [container, name_t, given_name, new_modules]
end

def get_class_specification
  tk = peek_tk
  if tk.nil?
    return ''
  elsif :on_kw == tk[:kind] && 'self' == tk[:text]
    return 'self'
  elsif :on_gvar == tk[:kind]
    return ''
  end
  res = get_constant
  skip_tkspace_without_nl
  get_tkread # empty out read buffer
  tk = get_tk
  return res unless tk
  case tk[:kind]
  when :on_nl, :on_comment, :on_embdoc, :on_semicolon then
    unget_tk(tk)
    return res
  end
  res += parse_call_parameters(tk)
  res
end

def get_constant
  res = ""
  skip_tkspace_without_nl
  tk = get_tk
  while tk && ((:on_op == tk[:kind] && '::' == tk[:text]) || :on_const == tk[:kind]) do
    res += tk[:text]
    tk = get_tk
  end
  unget_tk(tk)
  res
end

def get_end_token(tk) # :nodoc:
  case tk[:kind]
  when :on_lparen
    token = RDoc::Parser::RipperStateLex::Token.new
    token[:kind] = :on_rparen
    token[:text] = ')'
    token
  when :on_rparen
    nil
  else
    token = RDoc::Parser::RipperStateLex::Token.new
    token[:kind] = :on_nl
    token[:text] = "\n"
    token
  end
end

def get_included_module_with_optional_parens
  skip_tkspace_without_nl
  get_tkread
  tk = get_tk
  end_token = get_end_token tk
  return '' unless end_token
  nest = 0
  continue = false
  only_constant = true
  while tk != nil do
    is_element_of_constant = false
    case tk[:kind]
    when :on_semicolon then
      break if nest == 0
    when :on_lbracket then
      nest += 1
    when :on_rbracket then
      nest -= 1
    when :on_lbrace then
      nest += 1
    when :on_rbrace then
      nest -= 1
      if nest <= 0
        # we might have a.each { |i| yield i }
        unget_tk(tk) if nest < 0
        break
      end
    when :on_lparen then
      nest += 1
    when end_token[:kind] then
      if end_token[:kind] == :on_rparen
        nest -= 1
        break if nest <= 0
      else
        break if nest <= 0
      end
    when :on_rparen then
      nest -= 1
    when :on_comment, :on_embdoc then
      @read.pop
      if :on_nl == end_token[:kind] and "\n" == tk[:text][-1] and
        (!continue or (tk[:state] & Ripper::EXPR_LABEL) != 0) then
        break if !continue and nest <= 0
      end
    when :on_comma then
      continue = true
    when :on_ident then
      continue = false if continue
    when :on_kw then
      case tk[:text]
      when 'def', 'do', 'case', 'for', 'begin', 'class', 'module'
        nest += 1
      when 'if', 'unless', 'while', 'until', 'rescue'
        # postfix if/unless/while/until/rescue must be EXPR_LABEL
        nest += 1 unless (tk[:state] & Ripper::EXPR_LABEL) != 0
      when 'end'
        nest -= 1
        break if nest == 0
      end
    when :on_const then
      is_element_of_constant = true
    when :on_op then
      is_element_of_constant = true if '::' == tk[:text]
    end
    only_constant = false unless is_element_of_constant
    tk = get_tk
  end
  if only_constant
    get_tkread_clean(/\s+/, ' ')
  else
    ''
  end
end

def get_method_container(container, name_t) # :nodoc:
  prev_container = container
  container = container.find_module_named(name_t[:text])
  unless container then
    constant = prev_container.constants.find do |const|
      const.name == name_t[:text]
    end
    if constant then
      parse_method_dummy prev_container
      return
    end
  end
  unless container then
    # TODO seems broken, should starting at Object in @store
    obj = name_t[:text].split("::").inject(Object) do |state, item|
      state.const_get(item)
    end rescue nil
    type = obj.class == Class ? RDoc::NormalClass : RDoc::NormalModule
    unless [Class, Module].include?(obj.class) then
      warn("Couldn't find #{name_t[:text]}. Assuming it's a module")
    end
    if type == RDoc::NormalClass then
      sclass = obj.superclass ? obj.superclass.name : nil
      container = prev_container.add_class type, name_t[:text], sclass
    else
      container = prev_container.add_module type, name_t[:text]
    end
    record_location container
  end
  container
end

def get_symbol_or_name
  tk = get_tk
  case tk[:kind]
  when :on_symbol then
    text = tk[:text].sub(/^:/, '')
    next_tk = peek_tk
    if next_tk && :on_op == next_tk[:kind] && '=' == next_tk[:text] then
      get_tk
      text << '='
    end
    text
  when :on_ident, :on_const, :on_gvar, :on_cvar, :on_ivar, :on_op, :on_kw then
    tk[:text]
  when :on_tstring, :on_dstring then
    tk[:text][1..-2]
  else
    raise RDoc::Error, "Name or symbol expected (got #{tk})"
  end
end

def get_tkread_clean(pattern, replacement) # :nodoc:
  read = get_tkread.gsub(pattern, replacement).strip
  return '' if read == ';'
  read
end

def get_visibility_information(tk, single) # :nodoc:
  vis_type  = tk[:text]
  singleton = single == SINGLE
  vis =
    case vis_type
    when 'private'   then :private
    when 'protected' then :protected
    when 'public'    then :public
    when 'private_class_method' then
      singleton = true
      :private
    when 'public_class_method' then
      singleton = true
      :public
    when 'module_function' then
      singleton = true
      :public
    else
      raise RDoc::Error, "Invalid visibility: #{tk.name}"
    end
  return vis_type, vis, singleton
end

def initialize(top_level, content, options, stats)
  super
  content = handle_tab_width(content)
  @size = 0
  @token_listeners = nil
  content = RDoc::Encoding.remove_magic_comment content
  @scanner = RDoc::Parser::RipperStateLex.parse(content)
  @content = content
  @scanner_point = 0
  @prev_seek = nil
  @markup = @options.markup
  @track_visibility = :nodoc != @options.visibility
  @encoding = @options.encoding
  reset
end

def look_for_directives_in(container, comment)
  @preprocess.handle comment, container do |directive, param|
    case directive
    when 'method', 'singleton-method',
         'attr', 'attr_accessor', 'attr_reader', 'attr_writer' then
      false # handled elsewhere
    when 'section' then
      break unless container.kind_of?(RDoc::Context)
      container.set_current_section param, comment.dup
      comment.text = ''
      break
    end
  end
  comment.remove_private
end

def make_message(message)
  prefix = "#{@file_name}:".dup
  tk = peek_tk
  prefix << "#{tk[:line_no]}:#{tk[:char_no]}:" if tk
  "#{prefix} #{message}"
end

def new_comment(comment, line_no = nil)
  c = RDoc::Comment.new comment, @top_level, :ruby
  c.line = line_no
  c.format = @markup
  c
end

def parse_alias(context, single, tk, comment)
  line_no = tk[:line_no]
  skip_tkspace
  if :on_lparen === peek_tk[:kind] then
    get_tk
    skip_tkspace
  end
  new_name = get_symbol_or_name
  skip_tkspace
  if :on_comma === peek_tk[:kind] then
    get_tk
    skip_tkspace
  end
  begin
    old_name = get_symbol_or_name
  rescue RDoc::Error
    return
  end
  al = RDoc::Alias.new(get_tkread, old_name, new_name, comment, singleton: single == SINGLE)
  record_location al
  al.line   = line_no
  read_documentation_modifiers al, RDoc::ATTR_MODIFIERS
  if al.document_self or not @track_visibility
    context.add_alias al
    @stats.add_alias al
  end
  al
end

def parse_attr(context, single, tk, comment)
  line_no = tk[:line_no]
  args = parse_symbol_arg 1
  if args.size > 0 then
    name = args[0]
    rw = "R"
    skip_tkspace_without_nl
    tk = get_tk
    if :on_comma == tk[:kind] then
      rw = "RW" if get_bool
    else
      unget_tk tk
    end
    att = create_attr context, single, name, rw, comment
    att.line   = line_no
    read_documentation_modifiers att, RDoc::ATTR_MODIFIERS
  else
    warn "'attr' ignored - looks like a variable"
  end
end

def parse_attr_accessor(context, single, tk, comment)
  line_no = tk[:line_no]
  args = parse_symbol_arg
  rw = "?"
  tmp = RDoc::CodeObject.new
  read_documentation_modifiers tmp, RDoc::ATTR_MODIFIERS
  # TODO In most other places we let the context keep track of document_self
  # and add found items appropriately but here we do not.  I'm not sure why.
  return if @track_visibility and not tmp.document_self
  case tk[:text]
  when "attr_reader"   then rw = "R"
  when "attr_writer"   then rw = "W"
  when "attr_accessor" then rw = "RW"
  else
    rw = '?'
  end
  for name in args
    att = create_attr context, single, name, rw, comment
    att.line   = line_no
  end
end

def parse_call_parameters(tk)
  end_token = case tk[:kind]
              when :on_lparen
                :on_rparen
              when :on_rparen
                return ""
              else
                :on_nl
              end
  nest = 0
  loop do
    break if tk.nil?
    case tk[:kind]
    when :on_semicolon
      break
    when :on_lparen
      nest += 1
    when end_token
      if end_token == :on_rparen
        nest -= 1
        break if RDoc::Parser::RipperStateLex.end?(tk) and nest <= 0
      else
        break if RDoc::Parser::RipperStateLex.end?(tk)
      end
    when :on_comment, :on_embdoc
      unget_tk(tk)
      break
    when :on_op
      if tk[:text] =~ /^(.{1,2})?=$/
        unget_tk(tk)
        break
      end
    end
    tk = get_tk
  end
  get_tkread_clean "\n", " "
end

def parse_class(container, single, tk, comment)
  line_no = tk[:line_no]
  declaration_context = container
  container, name_t, given_name, = get_class_or_module container
  if name_t[:kind] == :on_const
    cls = parse_class_regular container, declaration_context, single,
      name_t, given_name, comment
  elsif name_t[:kind] == :on_op && name_t[:text] == '<<'
    case name = skip_parentheses { get_class_specification }
    when 'self', container.name
      read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS
      parse_statements container, SINGLE
      return # don't update line
    else
      cls = parse_class_singleton container, name, comment
    end
  else
    warn "Expected class name or '<<'. Got #{name_t[:kind]}: #{name_t[:text].inspect}"
    return
  end
  cls.line   = line_no
  # after end modifiers
  read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS
  cls
end

def parse_class_regular(container, declaration_context, single, # :nodoc:
                        name_t, given_name, comment)
  superclass = '::Object'
  if given_name =~ /^::/ then
    declaration_context = @top_level
    given_name = $'
  end
  tk = peek_tk
  if tk[:kind] == :on_op && tk[:text] == '<' then
    get_tk
    skip_tkspace
    superclass = get_class_specification
    superclass = '(unknown)' if superclass.empty?
  end
  cls_type = single == SINGLE ? RDoc::SingleClass : RDoc::NormalClass
  cls = declaration_context.add_class cls_type, given_name, superclass
  cls.ignore unless container.document_children
  read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS
  record_location cls
  cls.add_comment comment, @top_level
  @top_level.add_to_classes_or_modules cls
  @stats.add_class cls
  suppress_parents container, declaration_context unless cls.document_self
  parse_statements cls
  cls
end

def parse_class_singleton(container, name, comment) # :nodoc:
  other = @store.find_class_named name
  unless other then
    if name =~ /^::/ then
      name = $'
      container = @top_level
    end
    other = container.add_module RDoc::NormalModule, name
    record_location other
    # class << $gvar
    other.ignore if name.empty?
    other.add_comment comment, @top_level
  end
  # notify :nodoc: all if not a constant-named class/module
  # (and remove any comment)
  unless name =~ /\A(::)?[A-Z]/ then
    other.document_self = nil
    other.document_children = false
    other.clear_comment
  end
  @top_level.add_to_classes_or_modules other
  @stats.add_class other
  read_documentation_modifiers other, RDoc::CLASS_MODIFIERS
  parse_statements(other, SINGLE)
  other
end

def parse_comment(container, tk, comment)
  return parse_comment_tomdoc container, tk, comment if @markup == 'tomdoc'
  column  = tk[:char_no]
  line_no = comment.line.nil? ? tk[:line_no] : comment.line
  comment.text = comment.text.sub(/(^# +:?)(singleton-)(method:)/, '\1\3')
  singleton = !!$~
  co =
    if (comment.text = comment.text.sub(/^# +:?method: *(\S*).*?\n/i, '')) && !!$~ then
      line_no += $`.count("\n")
      parse_comment_ghost container, comment.text, $1, column, line_no, comment
    elsif (comment.text = comment.text.sub(/# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i, '')) && !!$~ then
      parse_comment_attr container, $1, $3, comment
    end
  if co then
    co.singleton = singleton
    co.line      = line_no
  end
  true
end

def parse_comment_attr(container, type, name, comment) # :nodoc:
  return if name.empty?
  rw = case type
       when 'attr_reader' then 'R'
       when 'attr_writer' then 'W'
       else 'RW'
       end
  create_attr container, NORMAL, name, rw, comment
end

def parse_comment_ghost(container, text, name, column, line_no, # :nodoc:
                        comment)
  name = nil if name.empty?
  meth = RDoc::GhostMethod.new get_tkread, name
  record_location meth
  meth.start_collecting_tokens
  indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column)
  position_comment = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment)
  position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
  newline = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n")
  meth.add_tokens [position_comment, newline, indent]
  meth.params =
    if text.sub!(/^#\s+:?args?:\s*(.*?)\s*$/i, '') then
      $1
    else
      ''
    end
  comment.normalize
  meth.call_seq = comment.extract_call_seq
  return unless meth.name
  container.add_method meth
  meth.comment = comment
  @stats.add_method meth
  meth
end

def parse_comment_tomdoc(container, tk, comment)
  return unless signature = RDoc::TomDoc.signature(comment)
  column  = tk[:char_no]
  line_no = tk[:line_no]
  name, = signature.split %r%[ \(]%, 2
  meth = RDoc::GhostMethod.new get_tkread, name
  record_location meth
  meth.line      = line_no
  meth.start_collecting_tokens
  indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column)
  position_comment = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment)
  position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
  newline = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n")
  meth.add_tokens [position_comment, newline, indent]
  meth.call_seq = signature
  comment.normalize
  return unless meth.name
  container.add_method meth
  meth.comment = comment
  @stats.add_method meth
end

def parse_constant(container, tk, comment, ignore_constants = false)
  line_no = tk[:line_no]
  name = tk[:text]
  skip_tkspace_without_nl
  return unless name =~ /^\w+$/
  new_modules = []
  if :on_op == peek_tk[:kind] && '::' == peek_tk[:text] then
    unget_tk tk
    container, name_t, _, new_modules = get_class_or_module container, true
    name = name_t[:text]
  end
  is_array_or_hash = false
  if peek_tk && :on_lbracket == peek_tk[:kind]
    get_tk
    nest = 1
    while bracket_tk = get_tk
      case bracket_tk[:kind]
      when :on_lbracket
        nest += 1
      when :on_rbracket
        nest -= 1
        break if nest == 0
      end
    end
    skip_tkspace_without_nl
    is_array_or_hash = true
  end
  unless peek_tk && :on_op == peek_tk[:kind] && '=' == peek_tk[:text] then
    return false
  end
  get_tk
  unless ignore_constants
    new_modules.each do |prev_c, new_module|
      prev_c.add_module_by_normal_module new_module
      new_module.ignore unless prev_c.document_children
      @top_level.add_to_classes_or_modules new_module
    end
  end
  value = ''
  con = RDoc::Constant.new name, value, comment
  body = parse_constant_body container, con, is_array_or_hash
  return unless body
  con.value = body
  record_location con
  con.line   = line_no
  read_documentation_modifiers con, RDoc::CONSTANT_MODIFIERS
  return if is_array_or_hash
  @stats.add_constant con
  container.add_constant con
  true
end

def parse_constant_body(container, constant, is_array_or_hash) # :nodoc:
  nest     = 0
  rhs_name = ''.dup
  get_tkread
  tk = get_tk
  body = nil
  loop do
    break if tk.nil?
    if :on_semicolon == tk[:kind] then
      break if nest <= 0
    elsif [:on_tlambeg, :on_lparen, :on_lbrace, :on_lbracket].include?(tk[:kind]) then
      nest += 1
    elsif (:on_kw == tk[:kind] && 'def' == tk[:text]) then
      nest += 1
    elsif (:on_kw == tk[:kind] && %w{do if unless case begin}.include?(tk[:text])) then
      if (tk[:state] & Ripper::EXPR_LABEL) == 0
        nest += 1
      end
    elsif [:on_rparen, :on_rbrace, :on_rbracket].include?(tk[:kind]) ||
          (:on_kw == tk[:kind] && 'end' == tk[:text]) then
      nest -= 1
    elsif (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) then
      unget_tk tk
      if nest <= 0 and RDoc::Parser::RipperStateLex.end?(tk) then
        body = get_tkread_clean(/^[ \t]+/, '')
        read_documentation_modifiers constant, RDoc::CONSTANT_MODIFIERS
        break
      else
        read_documentation_modifiers constant, RDoc::CONSTANT_MODIFIERS
      end
    elsif :on_const == tk[:kind] then
      rhs_name << tk[:text]
      next_tk = peek_tk
      if nest <= 0 and (next_tk.nil? || :on_nl == next_tk[:kind]) then
        create_module_alias container, constant, rhs_name unless is_array_or_hash
        break
      end
    elsif :on_nl == tk[:kind] then
      if nest <= 0 and RDoc::Parser::RipperStateLex.end?(tk) then
        unget_tk tk
        break
      end
    elsif :on_op == tk[:kind] && '::' == tk[:text]
      rhs_name << '::'
    end
    tk = get_tk
  end
  body ? body : get_tkread_clean(/^[ \t]+/, '')
end

def parse_constant_visibility(container, single, tk)
  args = parse_symbol_arg
  case tk[:text]
  when 'private_constant'
    vis = :private
  when 'public_constant'
    vis = :public
  else
    raise RDoc::Error, 'Unreachable'
  end
  container.set_constant_visibility_for args, vis
end

def parse_extend_or_include(klass, container, comment) # :nodoc:
  loop do
    skip_tkspace_comment
    name = get_included_module_with_optional_parens
    unless name.empty? then
      obj = container.add klass, name, comment
      record_location obj
    end
    return if peek_tk.nil? || :on_comma != peek_tk[:kind]
    get_tk
  end
end

def parse_identifier(container, single, tk, comment) # :nodoc:
  case tk[:text]
  when 'private', 'protected', 'public', 'private_class_method',
       'public_class_method', 'module_function' then
    parse_visibility container, single, tk
    return true
  when 'private_constant', 'public_constant'
    parse_constant_visibility container, single, tk
    return true
  when 'attr' then
    parse_attr container, single, tk, comment
  when /^attr_(reader|writer|accessor)$/ then
    parse_attr_accessor container, single, tk, comment
  when 'alias_method' then
    parse_alias container, single, tk, comment
  when 'require', 'include' then
    # ignore
  else
    if comment.text =~ /\A#\#$/ then
      case comment.text
      when /^# +:?attr(_reader|_writer|_accessor)?:/ then
        parse_meta_attr container, single, tk, comment
      else
        method = parse_meta_method container, single, tk, comment
        method.params = container.params if
          container.params
        method.block_params = container.block_params if
          container.block_params
      end
    end
  end
  false
end

def parse_included_with_activesupport_concern(container, comment) # :nodoc:
  skip_tkspace_without_nl
  tk = get_tk
  unless tk[:kind] == :on_lbracket || (tk[:kind] == :on_kw && tk[:text] == 'do')
    unget_tk tk
    return nil # should be a block
  end
  parse_statements container
  container
end

def parse_meta_attr(context, single, tk, comment)
  args = parse_symbol_arg
  rw = "?"
  # If nodoc is given, don't document any of them
  tmp = RDoc::CodeObject.new
  read_documentation_modifiers tmp, RDoc::ATTR_MODIFIERS
  regexp = /^# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i
  if regexp =~ comment.text then
    comment.text = comment.text.sub(regexp, '')
    rw = case $1
         when 'attr_reader' then 'R'
         when 'attr_writer' then 'W'
         else 'RW'
         end
    name = $3 unless $3.empty?
  end
  if name then
    att = create_attr context, single, name, rw, comment
  else
    args.each do |attr_name|
      att = create_attr context, single, attr_name, rw, comment
    end
  end
  att
end

def parse_meta_method(container, single, tk, comment)
  column  = tk[:char_no]
  line_no = tk[:line_no]
  start_collecting_tokens
  add_token tk
  add_token_listener self
  skip_tkspace_without_nl
  comment.text = comment.text.sub(/(^# +:?)(singleton-)(method:)/, '\1\3')
  singleton = !!$~
  name = parse_meta_method_name comment, tk
  return unless name
  meth = RDoc::MetaMethod.new get_tkread, name, singleton: singleton
  record_location meth
  meth.line   = line_no
  remove_token_listener self
  meth.start_collecting_tokens
  indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column)
  position_comment = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment)
  position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
  newline = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n")
  meth.add_tokens [position_comment, newline, indent]
  meth.add_tokens @token_stream
  parse_meta_method_params container, single, meth, tk, comment
  meth.comment = comment
  @stats.add_method meth
  meth
end

def parse_meta_method_name(comment, tk) # :nodoc:
  if comment.text.sub!(/^# +:?method: *(\S*).*?\n/i, '') then
    return $1 unless $1.empty?
  end
  name_t = get_tk
  if :on_symbol == name_t[:kind] then
    name_t[:text][1..-1]
  elsif :on_tstring == name_t[:kind] then
    name_t[:text][1..-2]
  elsif :on_op == name_t[:kind] && '=' == name_t[:text] then # ignore
    remove_token_listener self
    nil
  else
    warn "unknown name token #{name_t.inspect} for meta-method '#{tk[:text]}'"
    'unknown'
  end
end

def parse_meta_method_params(container, single, meth, tk, comment) # :nodoc:
  token_listener meth do
    meth.params = ''
    look_for_directives_in meth, comment
    comment.normalize
    meth.call_seq = comment.extract_call_seq
    container.add_method meth
    last_tk = tk
    while tk = get_tk do
      if :on_semicolon == tk[:kind] then
        break
      elsif :on_nl == tk[:kind] then
        break unless last_tk and :on_comma == last_tk[:kind]
      elsif :on_sp == tk[:kind] then
        # expression continues
      elsif :on_kw == tk[:kind] && 'do' == tk[:text] then
        parse_statements container, single, meth
        break
      else
        last_tk = tk
      end
    end
  end
end

def parse_method(container, single, tk, comment)
  singleton = nil
  added_container = false
  name = nil
  column  = tk[:char_no]
  line_no = tk[:line_no]
  start_collecting_tokens
  add_token tk
  token_listener self do
    prev_container = container
    name, container, singleton = parse_method_name container
    added_container = container != prev_container
  end
  return unless name
  meth = RDoc::AnyMethod.new get_tkread, name, singleton: single == SINGLE ? true : singleton
  look_for_directives_in meth, comment
  if singleton
    # `current_line_visibility' is useless because it works against
    # the normal method named as same as the singleton method, after
    # the latter was defined.  Of course these are different things.
    container.current_line_visibility = :public
  end
  record_location meth
  meth.line   = line_no
  meth.start_collecting_tokens
  indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column)
  token = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment)
  token[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
  newline = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n")
  meth.add_tokens [token, newline, indent]
  meth.add_tokens @token_stream
  parse_method_params_and_body container, single, meth, added_container
  comment.normalize
  meth.call_seq = comment.extract_call_seq
  meth.comment = comment
  # after end modifiers
  read_documentation_modifiers meth, RDoc::METHOD_MODIFIERS
  @stats.add_method meth
end

def parse_method_dummy(container)
  dummy = RDoc::Context.new
  dummy.parent = container
  dummy.store  = container.store
  skip_method dummy
end

def parse_method_name(container) # :nodoc:
  skip_tkspace
  name_t = get_tk
  back_tk = skip_tkspace_without_nl
  singleton = false
  dot = get_tk
  if dot[:kind] == :on_period || (dot[:kind] == :on_op && dot[:text] == '::') then
    singleton = true
    name, container = parse_method_name_singleton container, name_t
  else
    unget_tk dot
    back_tk.reverse_each do |token|
      unget_tk token
    end
    name = parse_method_name_regular container, name_t
  end
  return name, container, singleton
end

def parse_method_name_regular(container, name_t) # :nodoc:
  if :on_op == name_t[:kind] && (%w{* & [] []= <<}.include?(name_t[:text])) then
    name_t[:text]
  else
    unless [:on_kw, :on_const, :on_ident].include?(name_t[:kind]) then
      warn "expected method name token, . or ::, got #{name_t.inspect}"
      skip_method container
      return
    end
    name_t[:text]
  end
end

def parse_method_name_singleton(container, name_t) # :nodoc:
  skip_tkspace
  name_t2 = get_tk
  if (:on_kw == name_t[:kind] && 'self' == name_t[:text]) || (:on_op == name_t[:kind] && '%' == name_t[:text]) then
    # NOTE: work around '[' being consumed early
    if :on_lbracket == name_t2[:kind]
      get_tk
      name = '[]'
    else
      name = name_t2[:text]
    end
  elsif :on_const == name_t[:kind] then
    name = name_t2[:text]
    container = get_method_container container, name_t
    return unless container
    name
  elsif :on_ident == name_t[:kind] || :on_ivar == name_t[:kind] || :on_gvar == name_t[:kind] then
    parse_method_dummy container
    name = nil
  elsif (:on_kw == name_t[:kind]) && ('true' == name_t[:text] || 'false' == name_t[:text] || 'nil' == name_t[:text]) then
    klass_name = "#{name_t[:text].capitalize}Class"
    container = @store.find_class_named klass_name
    container ||= @top_level.add_class RDoc::NormalClass, klass_name
    name = name_t2[:text]
  else
    warn "unexpected method name token #{name_t.inspect}"
    # break
    skip_method container
    name = nil
  end
  return name, container
end

def parse_method_or_yield_parameters(method = nil,
                                     modifiers = RDoc::METHOD_MODIFIERS)
  skip_tkspace_without_nl
  tk = get_tk
  end_token = get_end_token tk
  return '' unless end_token
  nest = 0
  continue = false
  while tk != nil do
    case tk[:kind]
    when :on_semicolon then
      break if nest == 0
    when :on_lbracket then
      nest += 1
    when :on_rbracket then
      nest -= 1
    when :on_lbrace then
      nest += 1
    when :on_rbrace then
      nest -= 1
      if nest <= 0
        # we might have a.each { |i| yield i }
        unget_tk(tk) if nest < 0
        break
      end
    when :on_lparen then
      nest += 1
    when end_token[:kind] then
      if end_token[:kind] == :on_rparen
        nest -= 1
        break if nest <= 0
      else
        break
      end
    when :on_rparen then
      nest -= 1
    when :on_comment, :on_embdoc then
      @read.pop
      if :on_nl == end_token[:kind] and "\n" == tk[:text][-1] and
        (!continue or (tk[:state] & Ripper::EXPR_LABEL) != 0) then
        if method && method.block_params.nil? then
          unget_tk tk
          read_documentation_modifiers method, modifiers
        end
        break if !continue and nest <= 0
      end
    when :on_comma then
      continue = true
    when :on_ident then
      continue = false if continue
    end
    tk = get_tk
  end
  get_tkread_clean(/\s+/, ' ')
end

def parse_method_parameters(method)
  res = parse_method_or_yield_parameters method
  res = "(#{res})" unless res =~ /\A\(/
  method.params = res unless method.params
  return if  method.block_params
  skip_tkspace_without_nl
  read_documentation_modifiers method, RDoc::METHOD_MODIFIERS
end

def parse_method_params_and_body(container, single, meth, added_container)
  token_listener meth do
    parse_method_parameters meth
    if meth.document_self or not @track_visibility then
      container.add_method meth
    elsif added_container then
      container.document_self = false
    end
    # Having now read the method parameters and documentation modifiers, we
    # now know whether we have to rename #initialize to ::new
    if meth.name == "initialize" && !meth.singleton then
      if meth.dont_rename_initialize then
        meth.visibility = :protected
      else
        meth.singleton = true
        meth.name = "new"
        meth.visibility = :public
      end
    end
    parse_statements container, single, meth
  end
end

def parse_module(container, single, tk, comment)
  container, name_t, = get_class_or_module container
  name = name_t[:text]
  mod = container.add_module RDoc::NormalModule, name
  mod.ignore unless container.document_children
  record_location mod
  read_documentation_modifiers mod, RDoc::CLASS_MODIFIERS
  mod.add_comment comment, @top_level
  parse_statements mod
  # after end modifiers
  read_documentation_modifiers mod, RDoc::CLASS_MODIFIERS
  @stats.add_module mod
end

def parse_require(context, comment)
  skip_tkspace_comment
  tk = get_tk
  if :on_lparen == tk[:kind] then
    skip_tkspace_comment
    tk = get_tk
  end
  name = tk[:text][1..-2] if :on_tstring == tk[:kind]
  if name then
    @top_level.add_require RDoc::Require.new(name, comment)
  else
    unget_tk tk
  end
end

def parse_rescue
  skip_tkspace_without_nl
  while tk = get_tk
    case tk[:kind]
    when :on_nl, :on_semicolon, :on_comment then
      break
    when :on_comma then
      skip_tkspace_without_nl
      get_tk if :on_nl == peek_tk[:kind]
    end
    skip_tkspace_without_nl
  end
end

def parse_statements(container, single = NORMAL, current_method = nil,
                     comment = new_comment(''))
  raise 'no' unless RDoc::Comment === comment
  comment = RDoc::Encoding.change_encoding comment, @encoding if @encoding
  nest = 1
  save_visibility = container.visibility
  container.visibility = :public unless current_method
  non_comment_seen = true
  while tk = get_tk do
    keep_comment = false
    try_parse_comment = false
    non_comment_seen = true unless (:on_comment == tk[:kind] or :on_embdoc == tk[:kind])
    case tk[:kind]
    when :on_nl, :on_ignored_nl, :on_comment, :on_embdoc then
      if :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind]
        skip_tkspace
        tk = get_tk
      else
        past_tokens = @read.size > 1 ? @read[0..-2] : []
        nl_position = 0
        past_tokens.reverse.each_with_index do |read_tk, i|
          if read_tk =~ /^\n$/ then
            nl_position = (past_tokens.size - 1) - i
            break
          elsif read_tk =~ /^#.*\n$/ then
            nl_position = ((past_tokens.size - 1) - i) + 1
            break
          end
        end
        comment_only_line = past_tokens[nl_position..-1].all?{ |c| c =~ /^\s+$/ }
        unless comment_only_line then
          tk = get_tk
        end
      end
      if tk and (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) then
        if non_comment_seen then
          # Look for RDoc in a comment about to be thrown away
          non_comment_seen = parse_comment container, tk, comment unless
            comment.empty?
          comment = ''
          comment = RDoc::Encoding.change_encoding comment, @encoding if @encoding
        end
        line_no = nil
        while tk and (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) do
          comment_body = retrieve_comment_body(tk)
          line_no = tk[:line_no] if comment.empty?
          comment += comment_body
          comment << "\n" unless comment_body =~ /\n\z/
          if comment_body.size > 1 && comment_body =~ /\n\z/ then
            skip_tkspace_without_nl # leading spaces
          end
          tk = get_tk
        end
        comment = new_comment comment, line_no
        unless comment.empty? then
          look_for_directives_in container, comment
          if container.done_documenting then
            throw :eof if RDoc::TopLevel === container
            container.ongoing_visibility = save_visibility
          end
        end
        keep_comment = true
      else
        non_comment_seen = true
      end
      unget_tk tk
      keep_comment = true
      container.current_line_visibility = nil
    when :on_kw then
      case tk[:text]
      when 'class' then
        parse_class container, single, tk, comment
      when 'module' then
        parse_module container, single, tk, comment
      when 'def' then
        parse_method container, single, tk, comment
      when 'alias' then
        parse_alias container, single, tk, comment unless current_method
      when 'yield' then
        if current_method.nil? then
          warn "Warning: yield outside of method" if container.document_self
        else
          parse_yield container, single, tk, current_method
        end
      when 'until', 'while' then
        if (tk[:state] & Ripper::EXPR_LABEL) == 0
          nest += 1
          skip_optional_do_after_expression
        end
      # Until and While can have a 'do', which shouldn't increase the nesting.
      # We can't solve the general case, but we can handle most occurrences by
      # ignoring a do at the end of a line.
      # 'for' is trickier
      when 'for' then
        nest += 1
        skip_for_variable
        skip_optional_do_after_expression
      when 'case', 'do', 'if', 'unless', 'begin' then
        if (tk[:state] & Ripper::EXPR_LABEL) == 0
          nest += 1
        end
      when 'super' then
        current_method.calls_super = true if current_method
      when 'rescue' then
        parse_rescue
      when 'end' then
        nest -= 1
        if nest == 0 then
          container.ongoing_visibility = save_visibility
          parse_comment container, tk, comment unless comment.empty?
          return
        end
      end
    when :on_const then
      unless parse_constant container, tk, comment, current_method then
        try_parse_comment = true
      end
    when :on_ident then
      if nest == 1 and current_method.nil? then
        keep_comment = parse_identifier container, single, tk, comment
      end
      case tk[:text]
      when "require" then
        parse_require container, comment
      when "include" then
        parse_extend_or_include RDoc::Include, container, comment
      when "extend" then
        parse_extend_or_include RDoc::Extend, container, comment
      when "included" then
        parse_included_with_activesupport_concern container, comment
      end
    else
      try_parse_comment = nest == 1
    end
    if try_parse_comment then
      non_comment_seen = parse_comment container, tk, comment unless
        comment.empty?
      keep_comment = false
    end
    unless keep_comment then
      comment = new_comment ''
      comment = RDoc::Encoding.change_encoding comment, @encoding if @encoding
      container.params = nil
      container.block_params = nil
    end
    consume_trailing_spaces
  end
  container.params = nil
  container.block_params = nil
end

def parse_symbol_arg(no = nil)
  skip_tkspace_comment
  tk = get_tk
  if tk[:kind] == :on_lparen
    parse_symbol_arg_paren no
  else
    parse_symbol_arg_space no, tk
  end
end

def parse_symbol_arg_paren(no) # :nodoc:
  args = []
  loop do
    skip_tkspace_comment
    if tk1 = parse_symbol_in_arg
      args.push tk1
      break if no and args.size >= no
    end
    skip_tkspace_comment
    case (tk2 = get_tk)[:kind]
    when :on_rparen
      break
    when :on_comma
    else
      warn("unexpected token: '#{tk2.inspect}'") if $DEBUG_RDOC
      break
    end
  end
  args
end

def parse_symbol_arg_space(no, tk) # :nodoc:
  args = []
  unget_tk tk
  if tk = parse_symbol_in_arg
    args.push tk
    return args if no and args.size >= no
  end
  loop do
    skip_tkspace_without_nl
    tk1 = get_tk
    if tk1.nil? || :on_comma != tk1[:kind] then
      unget_tk tk1
      break
    end
    skip_tkspace_comment
    if tk = parse_symbol_in_arg
      args.push tk
      break if no and args.size >= no
    end
  end
  args
end

def parse_symbol_in_arg
  tk = get_tk
  if :on_symbol == tk[:kind] then
    tk[:text].sub(/^:/, '')
  elsif :on_tstring == tk[:kind] then
    tk[:text][1..-2]
  elsif :on_dstring == tk[:kind] or :on_ident == tk[:kind] then
    nil # ignore
  else
    warn("Expected symbol or string, got #{tk.inspect}") if $DEBUG_RDOC
    nil
  end
end

def parse_top_level_statements(container)
  comment = collect_first_comment
  look_for_directives_in container, comment
  throw :eof if container.done_documenting
  @markup = comment.format
  # HACK move if to RDoc::Context#comment=
  container.comment = comment if container.document_self unless comment.empty?
  parse_statements container, NORMAL, nil, comment
end

def parse_visibility(container, single, tk)
  vis_type, vis, singleton = get_visibility_information tk, single
  skip_tkspace_comment false
  ptk = peek_tk
  # Ryan Davis suggested the extension to ignore modifiers, because he
  # often writes
  #
  #   protected unless $TESTING
  #
  if [:on_nl, :on_semicolon].include?(ptk[:kind]) || (:on_kw == ptk[:kind] && (['if', 'unless'].include?(ptk[:text]))) then
    container.ongoing_visibility = vis
  elsif :on_kw == ptk[:kind] && 'def' == ptk[:text]
    container.current_line_visibility = vis
  else
    update_visibility container, vis_type, vis, singleton
  end
end

def parse_yield(context, single, tk, method)
  return if method.block_params
  get_tkread
  method.block_params = parse_method_or_yield_parameters
end

def read_directive(allowed)
  tokens = []
  while tk = get_tk do
    tokens << tk
    if :on_nl == tk[:kind] or (:on_kw == tk[:kind] && 'def' == tk[:text]) then
      return
    elsif :on_comment == tk[:kind] or :on_embdoc == tk[:kind] then
      return unless tk[:text] =~ /:?\b([\w-]+):\s*(.*)/
      directive = $1.downcase
      return [directive, $2] if allowed.include? directive
      return
    end
  end
ensure
  unless tokens.length == 1 and (:on_comment == tokens.first[:kind] or :on_embdoc == tokens.first[:kind]) then
    tokens.reverse_each do |token|
      unget_tk token
    end
  end
end

def read_documentation_modifiers(context, allowed)
  skip_tkspace_without_nl
  directive, value = read_directive allowed
  return unless directive
  @preprocess.handle_directive '', directive, value, context do |dir, param|
    if %w[notnew not_new not-new].include? dir then
      context.dont_rename_initialize = true
      true
    end
  end
end

def record_location(container) # :nodoc:
  case container
  when RDoc::ClassModule then
    @top_level.add_to_classes_or_modules container
  end
  container.record_location @top_level
end

def retrieve_comment_body(tk)
  if :on_embdoc == tk[:kind]
    tk[:text].gsub(/\A=begin.*\n/, '').gsub(/=end\n?\z/, '')
  else
    tk[:text]
  end
end

def scan
  reset
  catch :eof do
    begin
      parse_top_level_statements @top_level
    rescue StandardError => e
      if @content.include?('<%') and @content.include?('%>') then
        # Maybe, this is ERB.
        $stderr.puts "\033[2KRDoc detects ERB file. Skips it for compatibility:"
        $stderr.puts @file_name
        return
      end
      if @scanner_point >= @scanner.size
        now_line_no = @scanner[@scanner.size - 1][:line_no]
      else
        now_line_no = peek_tk[:line_no]
      end
      first_tk_index = @scanner.find_index { |tk| tk[:line_no] == now_line_no }
      last_tk_index = @scanner.find_index { |tk| tk[:line_no] == now_line_no + 1 }
      last_tk_index = last_tk_index ? last_tk_index - 1 : @scanner.size - 1
      code = @scanner[first_tk_index..last_tk_index].map{ |t| t[:text] }.join
      $stderr.puts <<-EOF
self.class} failure around line #{now_line_no} of
@file_name}
      EOF
      unless code.empty? then
        $stderr.puts code
        $stderr.puts
      end
      raise e
    end
  end
  @top_level
end

def skip_for_variable
  skip_tkspace_without_nl
  get_tk
  skip_tkspace_without_nl
  tk = get_tk
  unget_tk(tk) unless :on_kw == tk[:kind] and 'in' == tk[:text]
end

def skip_method(container)
  meth = RDoc::AnyMethod.new "", "anon"
  parse_method_parameters meth
  parse_statements container, false, meth
end

def skip_optional_do_after_expression
  skip_tkspace_without_nl
  tk = get_tk
  b_nest = 0
  nest = 0
  loop do
    break unless tk
    case tk[:kind]
    when :on_semicolon, :on_nl, :on_ignored_nl then
      break if b_nest.zero?
    when :on_lparen then
      nest += 1
    when :on_rparen then
      nest -= 1
    when :on_kw then
      case tk[:text]
      when 'begin'
        b_nest += 1
      when 'end'
        b_nest -= 1
      when 'do'
        break if nest.zero?
      end
    when :on_comment, :on_embdoc then
      if b_nest.zero? and "\n" == tk[:text][-1] then
        break
      end
    end
    tk = get_tk
  end
  skip_tkspace_without_nl
  get_tk if peek_tk && :on_kw == peek_tk[:kind] && 'do' == peek_tk[:text]
end

def skip_parentheses(&block)
  left_tk = peek_tk
  if :on_lparen == left_tk[:kind]
    get_tk
    ret = skip_parentheses(&block)
    right_tk = peek_tk
    if :on_rparen == right_tk[:kind]
      get_tk
    end
    ret
  else
    yield
  end
end

def skip_tkspace_comment(skip_nl = true)
  loop do
    skip_nl ? skip_tkspace : skip_tkspace_without_nl
    next_tk = peek_tk
    return if next_tk.nil? || (:on_comment != next_tk[:kind] and :on_embdoc != next_tk[:kind])
    get_tk
  end
end

def suppress_parents(container, ancestor) # :nodoc:
  while container and container != ancestor do
    container.suppress unless container.documented?
    container = container.parent
  end
end

def tk_nl?(tk)
  :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind]
end

def update_visibility(container, vis_type, vis, singleton) # :nodoc:
  new_methods = []
  case vis_type
  when 'module_function' then
    args = parse_symbol_arg
    container.set_visibility_for args, :private, false
    container.methods_matching args do |m|
      s_m = m.dup
      record_location s_m
      s_m.singleton = true
      new_methods << s_m
    end
  when 'public_class_method', 'private_class_method' then
    args = parse_symbol_arg
    container.methods_matching args, true do |m|
      if m.parent != container then
        m = m.dup
        record_location m
        new_methods << m
      end
      m.visibility = vis
    end
  else
    args = parse_symbol_arg
    container.set_visibility_for args, vis, singleton
  end
  new_methods.each do |method|
    case method
    when RDoc::AnyMethod then
      container.add_method method
    when RDoc::Attr then
      container.add_attribute method
    end
    method.visibility = vis
  end
end

def warn(message)
  @options.warn make_message message
end