module ERB::Util

def self.tokenize(source) # :nodoc:

:nodoc:
nobody should use it.
Tokenizes a line of ERB. This is really just for error reporting and 
def self.tokenize(source) # :nodoc:
  require "strscan"
  source = StringScanner.new(source.chomp)
  tokens = []
  start_re = /<%(?:={1,2}|-|\#|%)?/m
  finish_re = /(?:[-=])?%>/m
  while !source.eos?
    pos = source.pos
    source.scan_until(/(?:#{start_re}|#{finish_re})/)
    raise NotImplementedError if source.matched.nil?
    len = source.pos - source.matched.bytesize - pos
    case source.matched
    when start_re
      tokens << [:TEXT, source.string[pos, len]] if len > 0
      tokens << [:OPEN, source.matched]
      if source.scan(/(.*?)(?=#{finish_re}|\z)/m)
        tokens << [:CODE, source.matched] unless source.matched.empty?
        tokens << [:CLOSE, source.scan(finish_re)] unless source.eos?
      else
        raise NotImplementedError
      end
    when finish_re
      tokens << [:CODE, source.string[pos, len]] if len > 0
      tokens << [:CLOSE, source.matched]
    else
      raise NotImplementedError, source.matched
    end
  end
  tokens
end