class Crass::Tokenizer

def consume_url
  value = String.new
  @s.scan(RE_WHITESPACE)
  until @s.eos?
    case char = @s.consume
    when ')'
      break
    when RE_WHITESPACE
      @s.scan(RE_WHITESPACE)
      if @s.eos? || @s.peek == ')'
        @s.consume
        break
      else
        return create_token(:bad_url, :value => value + consume_bad_url)
      end
    when '"', "'", '(', RE_NON_PRINTABLE
      # Parse error.
      return create_token(:bad_url,
        :error => true,
        :value => value + consume_bad_url)
    when '\\'
      if valid_escape?
        value << consume_escaped
      else
        # Parse error.
        return create_token(:bad_url,
          :error => true,
          :value => value + consume_bad_url
        )
      end
    else
      value << char
    end
  end
  create_token(:url, :value => value)
end