module RDoc::RubyToken

def self.def_token(token_n, super_token = Token, reading = nil, *opts)
  token_n = token_n.id2name if token_n.kind_of?(Symbol)
  if const_defined?(token_n)
    IRB.fail AlreadyDefinedToken, token_n
  end
  token_c = eval("class #{token_n} < #{super_token}; end; #{token_n}")
  if reading
    TkToken2Reading[token_c] = reading
    return if TkReading2Token[reading]
    if opts.empty?
      TkReading2Token[reading] = [token_c]
    else
      TkReading2Token[reading] = [token_c].concat(opts)
    end
  end
  TkSymbol2Token[token_n.intern] = token_c
end

def Token(token, value = nil)
  value ||= TkToken2Reading[token]
  case token
  when String
    if (tk = TkReading2Token[token]).nil?
      IRB.fail TkReading2TokenNoKey, token
    end
    tk = Token(tk[0], value)
    if tk.kind_of?(TkOp) then
      tk.name = token
    end
  when Symbol
    if (tk = TkSymbol2Token[token]).nil?
      IRB.fail TkSymbol2TokenNoKey, token
    end
    tk = Token(tk[0], value)
  else
    if token.instance_method(:initialize).arity == 3 then
      tk = token.new(@prev_seek, @prev_line_no, @prev_char_no)
      tk.set_text value
    else
      tk = token.new(@prev_seek, @prev_line_no, @prev_char_no, value)
    end
  end
  tk
end

def set_token_position(line, char)
  @prev_line_no = line
  @prev_char_no = char
end