class RDoc::TomDoc

def tokenize(text)
  text = text.sub(/\A(Public|Internal|Deprecated):\s+/, '')
  setup_scanner text
  until @s.eos? do
    pos = @s.pos
    # leading spaces will be reflected by the column of the next token
    # the only thing we loose are trailing spaces at the end of the file
    next if @s.scan(/ +/)
    @tokens << case
               when @s.scan(/\r?\n/) then
                 token = [:NEWLINE, @s.matched, *pos]
                 @s.newline!
                 token
               when @s.scan(/(Examples|Signature)$/) then
                 @tokens << [:HEADER, 3, *pos]
                 [:TEXT, @s[1], *pos]
               when @s.scan(/([:\w][\w\[\]]*)[ ]+- /) then
                 [:NOTE, @s[1], *pos]
               else
                 @s.scan(/.*/)
                 [:TEXT, @s.matched.sub(/\r$/, ''), *pos]
               end
  end
  self
end