class Crass::Tokenizer

def consume: () -> nil

def consume
  return nil if @s.eos?
  @s.mark
  # Consume comments.
  if comment_token = consume_comments
    if @options[:preserve_comments]
      return comment_token
    else
      return consume
    end
  end
  # Consume whitespace.
  return create_token(:whitespace) if @s.scan(RE_WHITESPACE)
  char = @s.consume
  case char.to_sym
  when :'"'
    consume_string
  when :'#'
    if @s.peek =~ RE_NAME || valid_escape?(@s.peek(2))
      create_token(:hash,
        :type  => start_identifier?(@s.peek(3)) ? :id : :unrestricted,
        :value => consume_name)
    else
      create_token(:delim, :value => char)
    end
  when :'$'
    if @s.peek == '='
      @s.consume
      create_token(:suffix_match)
    else
      create_token(:delim, :value => char)
    end
  when :"'"
    consume_string
  when :'('
    create_token(:'(')
  when :')'
    create_token(:')')
  when :*
    if @s.peek == '='
      @s.consume
      create_token(:substring_match)
    # Non-standard: Preserve the IE * hack.
    elsif @options[:preserve_hacks] && @s.peek =~ RE_NAME_START
      @s.reconsume
      consume_ident
    else
      create_token(:delim, :value => char)
    end
  when :+
    if start_number?
      @s.reconsume
      consume_numeric
    else
      create_token(:delim, :value => char)
    end
  when :','
    create_token(:comma)
  when :-
    nextTwoChars   = @s.peek(2)
    nextThreeChars = char + nextTwoChars
    if start_number?(nextThreeChars)
      @s.reconsume
      consume_numeric
    elsif nextTwoChars == '->'
      @s.consume
      @s.consume
      create_token(:cdc)
    elsif start_identifier?(nextThreeChars)
      @s.reconsume
      consume_ident
    else
      create_token(:delim, :value => char)
    end
  when :'.'
    if start_number?
      @s.reconsume
      consume_numeric
    else
      create_token(:delim, :value => char)
    end
  when :':'
    create_token(:colon)
  when :';'
    create_token(:semicolon)
  when :<
    if @s.peek(3) == '!--'
      @s.consume
      @s.consume
      @s.consume
      create_token(:cdo)
    else
      create_token(:delim, :value => char)
    end
  when :'@'
    if start_identifier?(@s.peek(3))
      create_token(:at_keyword, :value => consume_name)
    else
      create_token(:delim, :value => char)
    end
  when :'['
    create_token(:'[')
  when :'\\'
    if valid_escape?
      @s.reconsume
      consume_ident
    else
      # Parse error.
      create_token(:delim,
        :error => true,
        :value => char)
    end
  when :']'
    create_token(:']')
  when :'^'
    if @s.peek == '='
      @s.consume
      create_token(:prefix_match)
    else
      create_token(:delim, :value => char)
    end
  when :'{'
    create_token(:'{')
  when :'}'
    create_token(:'}')
  when :U, :u
    if @s.peek(2) =~ RE_UNICODE_RANGE_START
      @s.consume
      consume_unicode_range
    else
      @s.reconsume
      consume_ident
    end
  when :|
    case @s.peek
    when '='
      @s.consume
      create_token(:dash_match)
    when '|'
      @s.consume
      create_token(:column)
    else
      create_token(:delim, :value => char)
    end
  when :~
    if @s.peek == '='
      @s.consume
      create_token(:include_match)
    else
      create_token(:delim, :value => char)
    end
  else
    case char
    when RE_DIGIT
      @s.reconsume
      consume_numeric
    when RE_NAME_START
      @s.reconsume
      consume_ident
    else
      create_token(:delim, :value => char)
    end
  end
end