class RDoc::RD::BlockParser

def next_token # :nodoc:
  # preprocessing
  # if it is not in RD part
  # => method
  while @in_part != "rd"
    line = @src[@i]
    @i += 1 # next line

    case line
    # src end
    when false
      return [false, false]
    # RD part begin
    when /^=begin\s*(?:\bRD\b.*)?\s*$/
      if @in_part # if in non-RD part
        @part_content.push(line)
      else
        @in_part = "rd"
        return [:WHITELINE, "=begin\n"] # <= for textblockand
      end
    # non-RD part begin
    when /^=begin\s+(\w+)/
      part = $1
=begin # not imported to RDoc
      if @in_part # if in non-RD part
        @part_content.push(line)
      else
        @in_part = part if @tree.filter[part] # if filter exists
#  p "BEGIN_PART: #{@in_part}" # DEBUG
      end
=end
      @in_part = part
    # non-RD part end
    when /^=end(?:$|[\s\0\C-d\C-z])/
      if @in_part # if in non-RD part
=begin # not imported to RDoc
#  p "END_PART: #{@in_part}" # DEBUG
        # make Part-in object
        part = RDoc::RD::Part.new(@part_content.join(""), @tree, "r")
        @part_content.clear
        # call filter, part_out is output(Part object)
        part_out = @tree.filter[@in_part].call(part)

        if @tree.filter[@in_part].mode == :rd # if output is RD formatted
          subtree = parse_subtree(part_out.to_a)
        else # if output is target formatted
          basename = Tempfile.create(["rdtmp", ".#{@in_part}"], @tree.tmp_dir) do |tmpfile|
            tmpfile.print(part_out)
            File.basename(tmpfile.path)
          end
          subtree = parse_subtree(["=begin\n", "<<< #{basename}\n", "=end\n"])
        end
        @in_part = nil
        return [:SUBTREE, subtree]
=end
      end
    else
=begin # not imported to RDoc
      if @in_part # if in non-RD part
        @part_content.push(line)
      end
=end
    end
  end

  @current_indent = @indent_stack.join("")
  line = @src[@i]
  case line
  when false
    if_current_indent_equal("") do
      [false, false]
    end
  when /^=end/
    if_current_indent_equal("") do
      @in_part = nil
      [:WHITELINE, "=end"] # MUST CHANGE??
    end
  when /^\s*$/
    @i += 1 # next line
    return [:WHITELINE, ':WHITELINE']
  when /^\#/  # comment line
    @i += 1 # next line
    self.next_token()
  when /^(={1,4})(?!=)\s*(?=\S)/, /^(\+{1,2})(?!\+)\s*(?=\S)/
    rest = $'                    # '
    rest.strip!
    mark = $1
    if_current_indent_equal("") do
      return [:HEADLINE, [MARK_TO_LEVEL[mark], rest]]
    end
  when /^<<<\s*(\S+)/
    file = $1
    if_current_indent_equal("") do
      suffix = file[-3 .. -1]
      if suffix == ".rd" or suffix == ".rb"
        subtree = parse_subtree(get_included(file))
        [:SUBTREE, subtree]
      else
        [:INCLUDE, file]
      end
    end
  when /^(\s*)\*(\s*)/
    rest = $'                   # '
    newIndent = $2
    if_current_indent_equal($1) do
      if @in_verbatim
        [:STRINGLINE, line]
      else
        @indent_stack.push("\s" + newIndent)
        [:ITEMLISTLINE, rest]
      end
    end
  when /^(\s*)(\(\d+\))(\s*)/
    rest = $'                     # '
    mark = $2
    newIndent = $3
    if_current_indent_equal($1) do
      if @in_verbatim
        [:STRINGLINE, line]
      else
        @indent_stack.push("\s" * mark.size + newIndent)
        [:ENUMLISTLINE, rest]
      end
    end
  when /^(\s*):(\s*)/
    rest = $'                    # '
    newIndent = $2
    if_current_indent_equal($1) do
      if @in_verbatim
        [:STRINGLINE, line]
      else
        @indent_stack.push("\s#{$2}")
        [:DESCLISTLINE, rest]
      end
    end
  when /^(\s*)---(?!-|\s*$)/
    indent = $1
    rest = $'
    /\s*/ === rest
    term = $'
    new_indent = $&
    if_current_indent_equal(indent) do
      if @in_verbatim
        [:STRINGLINE, line]
      else
        @indent_stack.push("\s\s\s" + new_indent)
        [:METHODLISTLINE, term]
      end
    end
  when /^(\s*)/
    if_current_indent_equal($1) do
      [:STRINGLINE, line]
    end
  else
    raise "[BUG] parsing error may occurred."
  end
end