class CodeRay::Scanners::Java

def scan_tokens encoder, options
  state = :initial
  string_delimiter = nil
  package_name_expected = false
  class_name_follows = false
  last_token_dot = false
  until eos?
    case state
    when :initial
      if match = scan(/ \s+ | \\\n /x)
        encoder.text_token match, :space
        next
      
      elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
        encoder.text_token match, :comment
        next
      
      elsif package_name_expected && match = scan(/ #{IDENT} (?: \. #{IDENT} )* /ox)
        encoder.text_token match, package_name_expected
      
      elsif match = scan(/ #{IDENT} | \[\] /ox)
        kind = IDENT_KIND[match]
        if last_token_dot
          kind = :ident
        elsif class_name_follows
          kind = :class
          class_name_follows = false
        else
          case match
          when 'import'
            package_name_expected = :include
          when 'package'
            package_name_expected = :namespace
          when 'class', 'interface'
            class_name_follows = true
          end
        end
        encoder.text_token match, kind
      
      elsif match = scan(/ \.(?!\d) | [,?:()\[\]}] | -- | \+\+ | && | \|\| | \*\*=? | [-+*\/%^~&|<>=!]=? | <<<?=? | >>>?=? /x)
        encoder.text_token match, :operator
      
      elsif match = scan(/;/)
        package_name_expected = false
        encoder.text_token match, :operator
      
      elsif match = scan(/\{/)
        class_name_follows = false
        encoder.text_token match, :operator
      
      elsif check(/[\d.]/)
        if match = scan(/0[xX][0-9A-Fa-f]+/)
          encoder.text_token match, :hex
        elsif match = scan(/(?>0[0-7]+)(?![89.eEfF])/)
          encoder.text_token match, :octal
        elsif match = scan(/\d+[fFdD]|\d*\.\d+(?:[eE][+-]?\d+)?[fFdD]?|\d+[eE][+-]?\d+[fFdD]?/)
          encoder.text_token match, :float
        elsif match = scan(/\d+[lL]?/)
          encoder.text_token match, :integer
        end
      elsif match = scan(/["']/)
        state = :string
        encoder.begin_group state
        string_delimiter = match
        encoder.text_token match, :delimiter
      elsif match = scan(/ @ #{IDENT} /ox)
        encoder.text_token match, :annotation
      else
        encoder.text_token getch, :error
      end
    when :string
      if match = scan(STRING_CONTENT_PATTERN[string_delimiter])
        encoder.text_token match, :content
      elsif match = scan(/["'\/]/)
        encoder.text_token match, :delimiter
        encoder.end_group state
        state = :initial
        string_delimiter = nil
      elsif state == :string && (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox))
        if string_delimiter == "'" && !(match == "\\\\" || match == "\\'")
          encoder.text_token match, :content
        else
          encoder.text_token match, :char
        end
      elsif match = scan(/\\./m)
        encoder.text_token match, :content
      elsif match = scan(/ \\ | $ /x)
        encoder.end_group state
        state = :initial
        encoder.text_token match, :error unless match.empty?
      else
        raise_inspect "else case \" reached; %p not handled." % peek(1), encoder
      end
    else
      raise_inspect 'Unknown state', encoder
    end
    
    last_token_dot = match == '.'
    
  end
  if state == :string
    encoder.end_group state
  end
  encoder
end