class CodeRay::Scanners::Clojure

def scan_tokens encoder, options
  
  state = :initial
  kind = nil
  
  until eos?
    
    case state
    when :initial
      if match = scan(/ \s+ | \\\n | , /x)
        encoder.text_token match, :space
      elsif match = scan(/['`\(\[\)\]\{\}]|\#[({]|~@?|[@\^]/)
        encoder.text_token match, :operator
      elsif match = scan(/;.*/)
        encoder.text_token match, :comment  # TODO: recognize (comment ...) too
      elsif match = scan(/\#?\\(?:newline|space|.?)/)
        encoder.text_token match, :char
      elsif match = scan(/\#[ft]/)
        encoder.text_token match, :predefined_constant
      elsif match = scan(/#{IDENTIFIER}/o)
        kind = IDENT_KIND[match]
        encoder.text_token match, kind
        if rest? && kind == :keyword
          if kind = KEYWORD_NEXT_TOKEN_KIND[match]
            encoder.text_token match, :space if match = scan(/\s+/o)
            encoder.text_token match, kind if match = scan(/#{IDENTIFIER}/o)
          end
        end
      elsif match = scan(/#{SYMBOL}/o)
        encoder.text_token match, :symbol
      elsif match = scan(/\./)
        encoder.text_token match, :operator
      elsif match = scan(/ \# \^ #{IDENTIFIER} /ox)
        encoder.text_token match, :type
      elsif match = scan(/ (\#)? " /x)
        state = self[1] ? :regexp : :string
        encoder.begin_group state
        encoder.text_token match, :delimiter
      elsif match = scan(/#{NUM}/o) and not matched.empty?
        encoder.text_token match, match[/[.e\/]/i] ? :float : :integer
      else
        encoder.text_token getch, :error
      end
      
    when :string, :regexp
      if match = scan(/[^"\\]+|\\.?/)
        encoder.text_token match, :content
      elsif match = scan(/"/)
        encoder.text_token match, :delimiter
        encoder.end_group state
        state = :initial
      else
        raise_inspect "else case \" reached; %p not handled." % peek(1),
          encoder, state
      end
      
    else
      raise 'else case reached'
      
    end
    
  end
  
  if [:string, :regexp].include? state
    encoder.end_group state
  end
  
  encoder
  
end