class CodeRay::Scanners::Groovy

def scan_tokens encoder, options
  
  state = :initial
  inline_block_stack = []
  inline_block_paren_depth = nil
  string_delimiter = nil
  import_clause = class_name_follows = last_token = after_def = false
  value_expected = true
  
  until eos?
    
    case state
    
    when :initial
      
      if match = scan(/ \s+ | \\\n /x)
        encoder.text_token match, :space
        if match.index ?\n
          import_clause = after_def = false
          value_expected = true unless value_expected
        end
        next
      
      elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
        value_expected = true
        after_def = false
        encoder.text_token match, :comment
      
      elsif bol? && match = scan(/ \#!.* /x)
        encoder.text_token match, :doctype
      
      elsif import_clause && match = scan(/ (?!as) #{IDENT} (?: \. #{IDENT} )* (?: \.\* )? /ox)
        after_def = value_expected = false
        encoder.text_token match, :include
      
      elsif match = scan(/ #{IDENT} | \[\] /ox)
        kind = IDENT_KIND[match]
        value_expected = (kind == :keyword) && KEYWORDS_EXPECTING_VALUE[match]
        if last_token == '.'
          kind = :ident
        elsif class_name_follows
          kind = :class
          class_name_follows = false
        elsif after_def && check(/\s*[({]/)
          kind = :method
          after_def = false
        elsif kind == :ident && last_token != '?' && check(/:/)
          kind = :key
        else
          class_name_follows = true if match == 'class' || (import_clause && match == 'as')
          import_clause = match == 'import'
          after_def = true if match == 'def'
        end
        encoder.text_token match, kind
      
      elsif match = scan(/;/)
        import_clause = after_def = false
        value_expected = true
        encoder.text_token match, :operator
      
      elsif match = scan(/\{/)
        class_name_follows = after_def = false
        value_expected = true
        encoder.text_token match, :operator
        if !inline_block_stack.empty?
          inline_block_paren_depth += 1
        end
      
      # TODO: ~'...', ~"..." and ~/.../ style regexps
      elsif match = scan(/ \.\.<? | \*?\.(?!\d)@? | \.& | \?:? | [,?:(\[] | -[->] | \+\+ |
          && | \|\| | \*\*=? | ==?~ | <=?>? | [-+*%^~&|>=!]=? | <<<?=? | >>>?=? /x)
        value_expected = true
        value_expected = :regexp if match == '~'
        after_def = false
        encoder.text_token match, :operator
      
      elsif match = scan(/ [)\]}] /x)
        value_expected = after_def = false
        if !inline_block_stack.empty? && match == '}'
          inline_block_paren_depth -= 1
          if inline_block_paren_depth == 0  # closing brace of inline block reached
            encoder.text_token match, :inline_delimiter
            encoder.end_group :inline
            state, string_delimiter, inline_block_paren_depth = inline_block_stack.pop
            next
          end
        end
        encoder.text_token match, :operator
      
      elsif check(/[\d.]/)
        after_def = value_expected = false
        if match = scan(/0[xX][0-9A-Fa-f]+/)
          encoder.text_token match, :hex
        elsif match = scan(/(?>0[0-7]+)(?![89.eEfF])/)
          encoder.text_token match, :oct
        elsif match = scan(/\d+[fFdD]|\d*\.\d+(?:[eE][+-]?\d+)?[fFdD]?|\d+[eE][+-]?\d+[fFdD]?/)
          encoder.text_token match, :float
        elsif match = scan(/\d+[lLgG]?/)
          encoder.text_token match, :integer
        end
        
      elsif match = scan(/'''|"""/)
        after_def = value_expected = false
        state = :multiline_string
        encoder.begin_group :string
        string_delimiter = match
        encoder.text_token match, :delimiter
        
      # TODO: record.'name' syntax
      elsif match = scan(/["']/)
        after_def = value_expected = false
        state = match == '/' ? :regexp : :string
        encoder.begin_group state
        string_delimiter = match
        encoder.text_token match, :delimiter
        
      elsif value_expected && match = scan(/\//)
        after_def = value_expected = false
        encoder.begin_group :regexp
        state = :regexp
        string_delimiter = '/'
        encoder.text_token match, :delimiter
        
      elsif match = scan(/ @ #{IDENT} /ox)
        after_def = value_expected = false
        encoder.text_token match, :annotation
        
      elsif match = scan(/\//)
        after_def = false
        value_expected = true
        encoder.text_token match, :operator
        
      else
        encoder.text_token getch, :error
        
      end
      
    when :string, :regexp, :multiline_string
      if match = scan(STRING_CONTENT_PATTERN[string_delimiter])
        encoder.text_token match, :content
        
      elsif match = scan(state == :multiline_string ? /'''|"""/ : /["'\/]/)
        encoder.text_token match, :delimiter
        if state == :regexp
          # TODO: regexp modifiers? s, m, x, i?
          modifiers = scan(/[ix]+/)
          encoder.text_token modifiers, :modifier if modifiers && !modifiers.empty?
        end
        state = :string if state == :multiline_string
        encoder.end_group state
        string_delimiter = nil
        after_def = value_expected = false
        state = :initial
        next
        
      elsif (state == :string || state == :multiline_string) &&
          (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox))
        if string_delimiter[0] == ?' && !(match == "\\\\" || match == "\\'")
          encoder.text_token match, :content
        else
          encoder.text_token match, :char
        end
      elsif state == :regexp && match = scan(/ \\ (?: #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
        encoder.text_token match, :char
        
      elsif match = scan(/ \$ #{IDENT} /mox)
        encoder.begin_group :inline
        encoder.text_token '$', :inline_delimiter
        match = match[1..-1]
        encoder.text_token match, IDENT_KIND[match]
        encoder.end_group :inline
        next
      elsif match = scan(/ \$ \{ /x)
        encoder.begin_group :inline
        encoder.text_token match, :inline_delimiter
        inline_block_stack << [state, string_delimiter, inline_block_paren_depth]
        inline_block_paren_depth = 1
        state = :initial
        next
        
      elsif match = scan(/ \$ /mx)
        encoder.text_token match, :content
        
      elsif match = scan(/ \\. /mx)
        encoder.text_token match, :content  # TODO: Shouldn't this be :error?
        
      elsif match = scan(/ \\ | \n /x)
        encoder.end_group state
        encoder.text_token match, :error
        after_def = value_expected = false
        state = :initial
        
      else
        raise_inspect "else case \" reached; %p not handled." % peek(1), encoder
        
      end
      
    else
      raise_inspect 'Unknown state', encoder
      
    end
    
    last_token = match unless [:space, :comment, :doctype].include? kind
    
  end
  
  if [:multiline_string, :string, :regexp].include? state
    encoder.end_group state
  end
  
  encoder
end