class Rouge::Formatter

def token_lines(tokens, &b)
  return enum_for(:token_lines, tokens) unless block_given?
  out = []
  tokens.each do |tok, val|
    val.scan %r/\n|[^\n]+/ do |s|
      if s == "\n"
        yield out
        out = []
      else
        out << [tok, s]
      end
    end
  end
  # for inputs not ending in a newline
  yield out if out.any?
end