class RDoc::RubyLex

def identify_identifier
  token = ""
  if peek(0) =~ /[$@]/
    token.concat(c = getc)
    if c == "@" and peek(0) == "@"
      token.concat getc
    end
  end
  # HACK to avoid a warning the regexp is hidden behind an eval
  # HACK need a better way to detect oniguruma
  @identifier_re ||= if defined? Encoding then
                       eval '/[\p{Alnum}_]/u'
                     else
                       eval '/[\w\x80-\xff]/'
                     end
  while (ch = getc) =~ @identifier_re
    print " :#{ch}: " if RDoc::RubyLex.debug?
    token.concat ch
  end
  ungetc
  if (ch == "!" || ch == "?") && token[0,1] =~ /\w/ && peek(0) != "="
    token.concat getc
  end
  # almost fix token
  case token
  when /^\$/
    return Token(TkGVAR, token)
  when /^\@\@/
    @lex_state = EXPR_END
    # p Token(TkCVAR, token)
    return Token(TkCVAR, token)
  when /^\@/
    @lex_state = EXPR_END
    return Token(TkIVAR, token)
  end
  if @lex_state != EXPR_DOT
    print token, "\n" if RDoc::RubyLex.debug?
    token_c, *trans = TkReading2Token[token]
    if token_c
      # reserved word?
      if (@lex_state != EXPR_BEG &&
          @lex_state != EXPR_FNAME &&
          trans[1])
        # modifiers
        token_c = TkSymbol2Token[trans[1]]
        @lex_state = trans[0]
      else
        if @lex_state != EXPR_FNAME
          if ENINDENT_CLAUSE.include?(token)
            # check for ``class = val'' etc.
            valid = true
            case token
            when "class"
              valid = false unless peek_match?(/^\s*(<<|\w|::)/)
            when "def"
              valid = false if peek_match?(/^\s*(([+-\/*&\|^]|<<|>>|\|\||\&\&)=|\&\&|\|\|)/)
            when "do"
              valid = false if peek_match?(/^\s*([+-\/*]?=|\*|<|>|\&)/)
            when *ENINDENT_CLAUSE
              valid = false if peek_match?(/^\s*([+-\/*]?=|\*|<|>|\&|\|)/)
            else
              # no nothing
            end
            if valid
              if token == "do"
                if ![TkFOR, TkWHILE, TkUNTIL].include?(@indent_stack.last)
                  @indent += 1
                  @indent_stack.push token_c
                end
              else
                @indent += 1
                @indent_stack.push token_c
              end
            end
          elsif DEINDENT_CLAUSE.include?(token)
            @indent -= 1
            @indent_stack.pop
          end
          @lex_state = trans[0]
        else
          @lex_state = EXPR_END
        end
      end
      return Token(token_c, token)
    end
  end
  if @lex_state == EXPR_FNAME
    @lex_state = EXPR_END
    if peek(0) == '='
      token.concat getc
    end
  elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
    @lex_state = EXPR_ARG
  else
    @lex_state = EXPR_END
  end
  if token[0, 1] =~ /[A-Z]/
    return Token(TkCONSTANT, token)
  elsif token[token.size - 1, 1] =~ /[!?]/
    return Token(TkFID, token)
  else
    return Token(TkIDENTIFIER, token)
  end
end