module Sass::Constant

def tokenize(value)
  escaped = false
  is_string = false
  beginning_of_token = true
  str = ''
  to_return = []
  reset_str = Proc.new do
    to_return << str unless str.empty?
    ''
  end
  value.each_byte do |byte|
    unless escaped
      if byte == ESCAPE_CHAR
        escaped = true
        next
      end
      last = to_return[-1]
      # Do we need to open or close a string literal?
      if byte == STRING_CHAR
        is_string = !is_string
        # Adjacent strings should be concatenated
        if is_string && last && (!last.is_a?(Symbol) || last == :close)
          to_return << :concat
        end
        str = reset_str.call
        next
      end
      unless is_string
        # Are we looking at whitespace?
        if WHITESPACE.include?(byte)
          str = reset_str.call
          next
        end
        symbol = SYMBOLS[byte]
        # Adjacent values without an operator should be concatenated
        if (symbol.nil? || symbol == :open || symbol == :const) &&
            last && (!last.is_a?(Symbol) || last == :close)
          to_return << :concat
        end
        # String then open with no whitespace means funcall
        if symbol == :open && !str.empty?
          str = reset_str.call
          to_return << :funcall
        end
        # Time for a unary minus!
        if beginning_of_token && symbol == :minus
          beginning_of_token = true
          to_return << :neg
          next
        end
        # Are we looking at an operator?
        if symbol && (symbol != :mod || str.empty?)
          str = reset_str.call
          beginning_of_token = symbol != :close
          to_return << symbol
          next
        end
      end
    end
    escaped = false
    beginning_of_token = false
    str << byte.chr
  end
  if is_string
    raise Sass::SyntaxError.new("Unterminated string: #{value.dump}.")
  end
  str = reset_str.call
  to_return
end