class Cucumber::CucumberExpressions::CucumberExpressionTokenizer

def tokenize(expression)
  @expression = expression
  tokens = []
  @buffer = []
  previous_token_type = TokenType::START_OF_LINE
  treat_as_text = false
  @escaped = 0
  @buffer_start_index = 0
  codepoints = expression.codepoints
  if codepoints.empty?
    tokens.push(Token.new(TokenType::START_OF_LINE, '', 0, 0))
  end
  codepoints.each do |codepoint|
    if !treat_as_text && Token.is_escape_character(codepoint)
      @escaped += 1
      treat_as_text = true
      next
    end
    current_token_type = token_type_of(codepoint, treat_as_text)
    treat_as_text = false
    if should_create_new_token?(previous_token_type, current_token_type)
      token = convert_buffer_to_token(previous_token_type)
      previous_token_type = current_token_type
      @buffer.push(codepoint)
      tokens.push(token)
    else
      previous_token_type = current_token_type
      @buffer.push(codepoint)
    end
  end
  if @buffer.length > 0
    token = convert_buffer_to_token(previous_token_type)
    tokens.push(token)
  end
  raise TheEndOfLineCannotBeEscaped.new(expression) if treat_as_text
  tokens.push(Token.new(TokenType::END_OF_LINE, '', codepoints.length, codepoints.length))
  tokens
end