class CodeRay::Scanners::Scanner

def tokenize source = nil, options = {}
  options = @options.merge(options)
  @tokens = options[:tokens] || @tokens || Tokens.new
  @tokens.scanner = self if @tokens.respond_to? :scanner=
  case source
  when Array
    self.string = self.class.normalize(source.join)
  when nil
    reset
  else
    self.string = self.class.normalize(source)
  end
  
  begin
    scan_tokens @tokens, options
  rescue => e
    message = "Error in %s#scan_tokens, initial state was: %p" % [self.class, defined?(state) && state]
    raise_inspect e.message, @tokens, message, 30, e.backtrace
  end
  
  @cached_tokens = @tokens
  if source.is_a? Array
    @tokens.split_into_parts(*source.map { |part| part.size })
  else
    @tokens
  end
end