class CodeRay::Scanners::SQL

def scan_tokens encoder, options
  
  state = :initial
  string_type = nil
  string_content = ''
  name_expected = false
  
  until eos?
    
    if state == :initial
      
      if match = scan(/ \s+ | \\\n /x)
        encoder.text_token match, :space
      
      elsif match = scan(/(?:--\s?|#).*/)
        encoder.text_token match, :comment
        
      elsif match = scan(%r( /\* (!)? (?: .*? \*/ | .* ) )mx)
        encoder.text_token match, self[1] ? :directive : :comment
        
      elsif match = scan(/ [*\/=<>:;,!&^|()\[\]{}~%] | [-+\.](?!\d) /x)
        name_expected = true if match == '.' && check(/[A-Za-z_]/)
        encoder.text_token match, :operator
        
      elsif match = scan(/(#{STRING_PREFIXES})?([`"'])/o)
        prefix = self[1]
        string_type = self[2]
        encoder.begin_group :string
        encoder.text_token prefix, :modifier if prefix
        match = string_type
        state = :string
        encoder.text_token match, :delimiter
        
      elsif match = scan(/ @? [A-Za-z_][A-Za-z_0-9]* /x)
        encoder.text_token match, name_expected ? :ident : (match[0] == ?@ ? :variable : IDENT_KIND[match])
        name_expected = false
        
      elsif match = scan(/0[xX][0-9A-Fa-f]+/)
        encoder.text_token match, :hex
        
      elsif match = scan(/0[0-7]+(?![89.eEfF])/)
        encoder.text_token match, :octal
        
      elsif match = scan(/[-+]?(?>\d+)(?![.eEfF])/)
        encoder.text_token match, :integer
        
      elsif match = scan(/[-+]?(?:\d[fF]|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+)/)
        encoder.text_token match, :float
      
      elsif match = scan(/\\N/)
        encoder.text_token match, :predefined_constant
        
      else
        encoder.text_token getch, :error
        
      end
      
    elsif state == :string
      if match = scan(/[^\\"'`]+/)
        string_content << match
        next
      elsif match = scan(/["'`]/)
        if string_type == match
          if peek(1) == string_type  # doubling means escape
            string_content << string_type << getch
            next
          end
          unless string_content.empty?
            encoder.text_token string_content, :content
            string_content = ''
          end
          encoder.text_token match, :delimiter
          encoder.end_group :string
          state = :initial
          string_type = nil
        else
          string_content << match
        end
      elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
        unless string_content.empty?
          encoder.text_token string_content, :content
          string_content = ''
        end
        encoder.text_token match, :char
      elsif match = scan(/ \\ . /mox)
        string_content << match
        next
      elsif match = scan(/ \\ | $ /x)
        unless string_content.empty?
          encoder.text_token string_content, :content
          string_content = ''
        end
        encoder.text_token match, :error
        state = :initial
      else
        raise "else case \" reached; %p not handled." % peek(1), encoder
      end
      
    else
      raise 'else-case reached', encoder
      
    end
    
  end
  
  if state == :string
    encoder.end_group state
  end
  
  encoder
  
end