class IDL::Scanner

def next_token
  sign = nil
  str = '' #initialize empty string
  while true
    ch = @in.getc
    if ch.nil?
      if @ifdef.size>0 and !in_expansion?
        parse_error 'mismatched #if/#endif'
      end
      if more_source?
        leave_source
        next
      else
        return [false, nil]
      end
    end
    if WHITESPACE.include? ch
      @in.skipwhile {|c| WHITESPACE.include?(c) }
      next
    end
    if str.empty? && ch == ?\#
      parse_directive
      next
    end
    unless do_parse?
      skipline
      next
    end
    str << ch
    case
    when BREAKCHARS.include?(ch)
      if SHIFTCHARS.include?(ch) && @in.lookc == ch
        # '<<' or '>>'
        str << @in.getc
      end
      return [str, str]
    when ch == ANNOTATION
      if @in_annotation
        return [str, str]
      else
        # return token returned by parse_annotation or parse next token recursively
        return parse_annotation || next_token
      end
    when ch == ?: #
      if @in.lookc == ?: #
        @in.skipc
        return %w(:: ::)
      else
        return %w(: :)
      end
    when ch == ?L
      _nxtc = @in.lookc
      if _nxtc == ?\'  #' #single quote, for a character literal.
        @in.skipc # skip 'L'
        _nxtc = @in.lookc
        ret = if _nxtc == ?\\
          @in.skipc
          next_escape_str
        elsif _nxtc == ?\' #'
          [ nil, nil ]
        else
          [ :char, '' << @in.getc ]
        end
        if @in.lookc != ?\' #'
          parse_error "wide character literal must be single wide character enclosed in \"'\""
        end
        @in.skipc
        return [ :wide_character_literal, ret ]
      elsif _nxtc == ?\" #" #double quote, for a string literal.
        ret = []
        chs = ''
        @in.skipc # skip 'L'
        while true
          _nxtc = @in.lookc
          if _nxtc == ?\\
            @in.skipc
            ret << [:char, chs] unless chs.empty?
            chs = ''
            ret << next_escape_str
          elsif _nxtc == ?\" #"
            @in.skipc
            ret << [:char, chs] unless chs.empty?
            return [ :wide_string_literal, ret ]
          else
            chs << @in.getc
          end
        end
      else
        return next_identifier(ch)
      end
    when IDCHARS.include?(ch)
      return next_identifier(ch)
    when ch == ?/ #
      _nxtc = @in.lookc
      if _nxtc == ?*
        # skip comment like a `/* ... */'
        @in.skipc # forward stream beyond `/*'
        ch1 = nil
        @in.skipuntil { |ch_|
          ch0 = ch1; ch1 = ch_
          ch0 == ?* and ch1 == ?/ #
        }
        if @in.lookc.nil?
          parse_error "cannot find comment closing brace (\'*/\'). "
        end
        @in.skipc
        str = '' # reset
        next
      elsif _nxtc == ?/
        # skip comment like a `// ...\n'
        @in.skipc
        unless @scan_comment  # scan_comment will be true when parsing commented annotations
          _nxtc = @in.lookc
          if _nxtc == ANNOTATION
            @in.skipc
            # return token returned by parse_annotation or parse next token recursively
            return parse_annotation(true) || next_token
          else
            @in.skipuntil {|c| LFCR.include?(c) }
          end
        end
        str = '' # reset
        next
      else
        return %w(/ /)
      end
    when SIGNS.include?(ch)
      _nxtc = @in.lookc
      if DIGITS.include? _nxtc
        sign = ch
        str = '' # reset
        next
      else
        return [str, str]
      end
    when (?1..?9).include?(ch)
      @in.mark(sign, ch)
      @in.skipwhile {|c| DIGITS.include?(c) }
      num_type = ([?., ?e, ?E, ?d, ?D].include?(@in.lookc)) ? skipfloat_or_fixed : :integer_literal
      r = @in.getregion
      if num_type == :floating_pt_literal
        return [:floating_pt_literal, r.to_f]
      elsif num_type == :fixed_pt_literal
        return [:fixed_pt_literal, r]
      else
        return [:integer_literal, r.to_i]
      end
    when ch == DOT #
      @in.mark(ch)
      @in.skipwhile {|c| DIGITS.include?(c) }
      num_type = (DOT != @in.lookc) ? skipfloat_or_fixed : nil
      s = @in.getregion
      if s == '.'
        parse_error 'token consisting of single dot (.) is invalid.'
      end
      if num_type == :floating_pt_literal
        return [:floating_pt_literal, s.to_f]
      elsif num_type == :fixed_pt_literal
        return [:fixed_pt_literal, s]
      else
        parse_error 'invalid floating point constant.'
      end
    when ch == ?0
      @in.mark(sign, ch)
      _nxtc = @in.lookc
      if _nxtc == ?x || _nxtc == ?X
        @in.skipc
        @in.skipwhile { |ch_| HEXCHARS.include? ch_ }
        s = @in.getregion
        return [:integer_literal, s.hex]
      else
        dec = false
        @in.skipwhile {|c| OCTALS.include?(c) }
        if (?8..?9).include? @in.lookc
          dec = TRUE
          @in.skipwhile {|c| DIGITS.include?(c) }
        end
        num_type = ([?., ?e, ?E, ?d, ?D].include?(@in.lookc)) ? skipfloat_or_fixed : :integer_literal
        s = @in.getregion
        ret = if num_type == :floating_pt_literal
          [:floating_pt_literal, s.to_f]
        elsif num_type == :fixed_pt_literal
          [:fixed_pt_literal, s]
        elsif dec
          parse_error "decimal literal starting with '0' should be octal ('0'..'7' only): #{s}"
        else
          [:integer_literal, s.oct]
        end
        return ret
      end
    when ch == ?\'  #' #single quote, for a character literal.
      _nxtc = @in.lookc
      ret = if _nxtc == ?\\
        @in.skipc
        next_escape
      elsif _nxtc == ?\' #'
        0
      elsif _nxtc
        ('' << @in.getc).unpack('C').first
      end
      if @in.lookc != ?\' #'
        parse_error "character literal must be single character enclosed in \"'\""
      end
      @in.skipc
      return [ :character_literal, ret ]
    when ch == ?\" #" #double quote, for a string literal.
      ret = ''
      while true
        _nxtc = @in.lookc
        if _nxtc == ?\\
          @in.skipc
          ret << next_escape
        elsif _nxtc == ?\" #"
          @in.skipc
          return [ :string_literal, ret ]
        elsif _nxtc
          ret << @in.getc
        else
          parse_error 'unterminated string literal'
        end
      end
    else
      parse_error 'illegal character [' << ch << ']'
    end #of case
  end #of while
  parse_error 'unexcepted error'
end #of method next_token