class IDL::Scanner

def do_parse?
  @ifdef.empty? || @ifdef.last
end

def enter_expansion(src, define)
  @stack << [:define, nil, nil, @in, nil]
  @expansions << define
  @in = In.new(StrIStream.new(src), @in.position.name, @in.position.line, @in.position.column)
end

def enter_include(src)
  if @directiver.is_included?(src)
    @directiver.declare_include(src)
  else
    fpath = find_include(src)
    if fpath.nil?
      parse_error "Cannot open include file '#{src}'"
    end
    @stack << [:include, @prefix, @ifdef, @in, @ifskip]
    @prefix = nil
    @ifdef = Array.new
    @in = In.new(File.open(fpath, 'r'), fpath)
    @directiver.enter_include(src)
    @directiver.pragma_prefix(nil)
  end
end

def eval_directive(s)
  IDL.log(2,"** RIDL - eval_directive(#{s})")
  rc = eval(s)
  case rc
  when FalseClass, TrueClass
    rc
  when Numeric
    rc != 0
  else
    parse_error "invalid preprocessor expression."
  end
end

def extract_annotation()
  annotation_body = nil
  # next token should be '(' in case of normal/single value annotation
  # or anything else in case of marker annotation
  token = next_token
  if token.first == '('
    begin
      # identifier or value (in case of single value annotation) expected
      token = next_token
      if token.first == ')' # marker annotation; leave body empty
        annotation_body = { }
      else
        parse_error 'annotation member expected!' unless token.first == :identifier || is_literal?(token.first)
        s1 = token.last
        token = next_token # ')'  (in case of single value annotation) or '='
        if token.first == ')'
          parse_error 'invalid annotation member' if annotation_body
          annotation_body = { :value => s1 }
        else
          parse_error 'invalid annotation member' unless token.first == '='
          token, annotation_value = extract_annotation_value()
          parse_error 'invalid annotation body' unless token.first == ',' || token.first == ')'
          (annotation_body ||= {})[s1] = annotation_value
        end
      end
    end until token.first == ')'
    token = next_token # need to get next token
  else
    # marker annotation or symbolic value; leave body nil
  end
  return [token, annotation_body]
end

def extract_annotation_value()
  annotation_value = nil
  token = next_token # needs '{' (array) or literal or identifier (which means nested annotation object or enum value)
  if token.first == '{'
    # extract array of values (literals or identifiers) separated by ','
    annotation_value = []
    begin
      token, ann_value = extract_annotation_value()
      parse_error 'invalid annotation value array' unless token.first == ',' || token.first == '}'
      annotation_value << ann_value
    end until token.first == '}'
    token = next_token
  elsif token.first == :identifier
    member_annotation_id = token.last
    # get nested body
    token, member_annotation_body = extract_annotation()
    # determin vaue type; if it has a body it is an annotation instance
    if member_annotation_body
      annotation_value = { member_annotation_id => member_annotation_body }
    else  # otherwise it is a symbolic value
      annotation_value = member_annotation_id.to_sym
    end
  else
    parse_error 'invalid annotation member' unless is_literal?(token.first)
    annotation_value = token.last
    token = next_token
  end
  return [token, annotation_value]
end

def find_include(fname)
  path = if File.file?(fname) && File.readable?(fname)
    fname
  else
    fp = @includepaths.find do |p|
      f = p + "/" + fname
      File.file?(f) && File.readable?(f)
    end
    fp += '/' + fname if !fp.nil?
    fp
  end
end

def getline
  s = ""
  while TRUE
    ch = @in.lookc
    break if ch.nil?
    case
    when (ch == ?\") #"
      s << @in.getc # opening quote
      while TRUE
        if @in.lookc == ?\\
          # escape sequence
          s << @in.getc
          esctyp, escstr = next_escape_str(true)
          s << escstr
        elsif @in.lookc == ?\" #"
          break
        elsif @in.lookc
          # normal character
          s << @in.getc
        else
          parse_error "unterminated string literal"
        end
      end
      s << @in.getc # closing quote
    when (ch == ?\') #' # quoted character
      s << @in.getc # opening quote
      if @in.lookc == ?\\
        # escape sequence
        s << @in.getc
        esctyp, escstr = next_escape_str(true)
        s << escstr
      elsif @in.lookc && @in.lookc != ?\' #'
        # normal character
        s << @in.getc
      end
      if @in.lookc != ?\' #'
        parse_error "character literal must be single character enclosed in \"'\""
      end
      s << @in.getc # closing quote
    when LFCR.include?(ch)
      @in.skipwhile() { |ch_| LFCR.include? ch_ }
      break
    when ch == ?/
      @in.skipc
      if @in.lookc == ?/
        # //-style comment; skip till eol
        @in.gets
        break
      elsif @in.lookc == ?*
        # /*...*/ style comment; skip comment
        ch1 = nil
        @in.skipuntil { |ch_|
          ch0 = ch1; ch1 = ch_
          ch0 == ?* and ch1 == ?/ #
        }
        if @in.lookc.nil?
          parse_error "cannot find comment closing brace (\'*/\'). "
        end
        @in.skipc
      else
        s << ch
      end
    when ch == ?\\
      @in.skipc
      if LFCR.include?(@in.lookc)
        # line continuation
        @in.skipwhile() { |ch_| LFCR.include? ch_ }
        if @in.lookc.nil?
          parse_error "line continuation character ('\\') not allowed as last character in file."
        end
      else
        s << ch
      end
    else
      @in.skipc
      s << ch
    end
  end
  s
end

def in_expansion?
  more_source? and @stack.last[0] == :define
end

def initialize(src, directiver, params = {})
  @includepaths = params[:includepaths] || []
  @stack = []
  @expansions = []
  @prefix = nil
  @directiver = directiver
  @defined = TokenRegistry.new
  # initialize with predefined macros
  if params[:macros]
    params[:macros].each do |(name, value)|
      @defined[name] = value
    end
  end
  @ifdef = Array.new
  @ifskip = false
  @ifnest = 0
  i = nil
  nm = ''
  case src
  when String
    i = StrIStream.new(src)
    nm = '<string>'
  when File
    i = src
    nm = src.path
  when IO
    i = src
    nm = '<io>'
  else
    parse_error "illegal type for input source: #{src.class} "
  end
  @in = In.new(i, nm)
  @scan_comment = false # true if parsing commented annotation
  @in_annotation = false # true if parsing annotation
end

def is_expanded?(define)
  @expansions.include?(define)
end

def is_literal?(o)
  return LITERALS.include?(o)
end

def leave_source()
  if @stack.size>0
    if @stack.last[0] == :include
      @directiver.leave_include
      type, @prefix, @ifdef, @in, @ifskip = @stack.pop
      @directiver.pragma_prefix(@prefix)
    else
      type, prefix_, ifdef_, @in, elsif_ = @stack.pop
      @expansions.pop
    end
  end
end

def more_source?
  @stack.size>0
end

def next_escape
  ret = 0
  case (ch = @in.getc)
  when nil
    parse_error 'illegal escape sequence'
  when ?0..?7
    ret = ""
    ret << ch
    1.upto(2) {
      ch = @in.lookc
      if (?0..?7).include? ch
        ret << ch
      else
        break
      end
      @in.skipc
    }
    ret = ret.oct
  when ?x # i'm not sure '\x' should be 0 or 'x'. currently returns 0.
    ret = ""
    1.upto(2) {
      ch = @in.lookc
      if HEXCHARS.include? ch
        ret << ch
      else
        break
      end
      @in.skipc
    }
    ret = ret.hex
  when ?u
    ret = ""
    1.upto(4) {
      ch = @in.lookc
      if HEXCHARS.include? ch
        ret << ch
      else
        break
      end
      @in.skipc
    }
    ret = ret.hex
  when ?n, ?t, ?v, ?b, ?r, ?f, ?a
    ret = ESCTBL[ch]
  else
    ret = ('' << ch).unpack('C').first
  end
  return ret
end

def next_escape_str(keep_type_ch = false)
  ret = 0
  case (ch = @in.getc)
  when nil
    parse_error 'illegal escape sequence'
  when ?0..?7
    ret = ""
    ret << ch
    1.upto(2) {
      ch = @in.lookc
      if (?0..?7).include? ch
        ret << ch
      else
        break
      end
      @in.skipc
    }
    ret = [ :oct, ret ]
  when ?x # i'm not sure '\x' should be 0 or 'x'. currently returns 0.
    ret = ""
    ret << ch if keep_type_ch
    1.upto(2) {
      ch = @in.lookc
      if HEXCHARS.include? ch
        ret << ch
      else
        break
      end
      @in.skipc
    }
    ret = [ :hex2, ret ]
  when ?u
    ret = ""
    ret << ch if keep_type_ch
    1.upto(4) {
      ch = @in.lookc
      if HEXCHARS.include? ch
        ret << ch
      else
        break
      end
      @in.skipc
    }
    ret = [ :hex4, ret ]
  when ?n, ?t, ?v, ?b, ?r, ?f, ?a
    ret = ''
    ret << ch
    ret = [ :esc, ret ]
  else
    ret = ''
    ret << ch
    ret = [ :esc_ch, ch ]
  end
  return ret
end

def next_identifier(first = nil)
  @in.mark(first)
  while TRUE
    case @in.lookc
    when nil
      break
    when ?0..?9, ?a..?z, ?A..?Z, ?_
      @in.skipc
    else
      break
    end
  end
  s0 = @in.getregion
  s1 = s0.downcase
  # simple check
  if (s0.length == 0)
    parse_error "identifier expected!"
  else
    case s0[0]
    when ?a..?z, ?A..?Z
    when ?_   ## if starts with CORBA IDL escape => remove
      s0.slice!(0)
    else
      parse_error "identifier must begin with alphabet character: #{s0}"
    end
  end
  # preprocessor check
  if @defined.has_key?(s0) and !is_expanded?(s0)
    # enter expansion as new source
    enter_expansion(@defined[s0], s0)
    # call next_token to parse expanded source
    next_token
  # keyword check
  elsif @in_annotation
    if BOOL_LITERALS.has_key?(s1)
      [ :boolean_literal, BOOL_LITERALS[s1] ]
    else
      [ :identifier, s0 ]
    end
  elsif (a = KEYWORDS.assoc(s1)).nil?
    # check for language mapping keyword except when
    # - this is an IDL escaped ('_' prefix) identifier
    [ :identifier, Identifier.new(s0, s1[0] == ?_ ? s0 : chk_identifier(s0)) ]
  elsif s0 == a[1]
    [ a[1], nil ]
  else
    parse_error "`#{s0}' collides with a keyword `#{a[1]}'"
  end
end

def next_token
  sign = nil
  str = "" #initialize empty string
  while TRUE
    ch = @in.getc
    if ch.nil?
      if @ifdef.size>0 and !in_expansion?
        parse_error "mismatched #if/#endif"
      end
      if more_source?
        leave_source
        next
      else
        return [FALSE, nil]
      end
    end
    if WHITESPACE.include? ch
      @in.skipwhile( WHITESPACE )
      next
    end
    if str.empty? && ch == ?\#
      parse_directive
      next
    end
    unless do_parse?
      skipline
      next
    end
    str << ch
    case
    when BREAKCHARS.include?(ch)
      if SHIFTCHARS.include?(ch) && @in.lookc == ch
        # '<<' or '>>'
        str << @in.getc
      end
      return [str, str]
    when ch == ANNOTATION
      if @in_annotation
        return [str, str]
      else
        return parse_annotation()
      end
    when ch == ?: #
      if @in.lookc == ?: #
        @in.skipc
        return ["::", "::"]
      else
        return [":", ":"]
      end
    when ch == ?L
      _nxtc = @in.lookc
      if _nxtc == ?\'  #' #single quote, for a character literal.
        ret = 0
        @in.skipc # skip 'L'
        _nxtc = @in.lookc
        if _nxtc == ?\\
          @in.skipc
          ret = next_escape_str
        elsif _nxtc == ?\' #'
          ret = [ nil, nil ]
        else
          ret = ''
          ret << @in.getc
          ret = [ :char, ret ]
        end
        if @in.lookc != ?\' #'
          parse_error "wide character literal must be single wide character enclosed in \"'\""
        end
        @in.skipc
        return [ :wide_character_literal, ret ]
      elsif _nxtc == ?\" #" #double quote, for a string literal.
        ret = []
        chs = ''
        @in.skipc # skip 'L'
        while TRUE
          _nxtc = @in.lookc
          if _nxtc == ?\\
            @in.skipc
            ret << [:char, chs] unless chs.empty?
            chs = ''
            ret << next_escape_str
          elsif _nxtc == ?\" #"
            @in.skipc
            ret << [:char, chs] unless chs.empty?
            return [ :wide_string_literal, ret ]
          else
            chs << @in.getc
          end
        end
      else
        return next_identifier(ch)
      end
    when IDCHARS.include?(ch)
      return next_identifier(ch)
    when ch == ?/ #
      _nxtc = @in.lookc
      if _nxtc == ?*
        # skip comment like a `/* ... */'
        @in.skipc # forward stream beyond `/*'
        ch1 = nil
        @in.skipuntil { |ch_|
          ch0 = ch1; ch1 = ch_
          ch0 == ?* and ch1 == ?/ #
        }
        if @in.lookc.nil?
          parse_error "cannot find comment closing brace (\'*/\'). "
        end
        @in.skipc
        str = "" # reset
        next
      elsif _nxtc == ?/
        # skip comment like a `// ...\n'
        @in.skipc
        unless @scan_comment  # scan_comment will be true when parsing commented annotations
          _nxtc = @in.lookc
          if _nxtc == ANNOTATION
            @in.skipc
            return parse_annotation(true)
          else
            @in.skipuntil(?\n, ?\r)
          end
        end
        str = "" # reset
        next
      else
        return [ "/", "/" ]
      end
    when ch == ?+ || ch == ?-
      _nxtc = @in.lookc
      if (?0..?9).include? _nxtc
        sign = ch
        str = "" # reset
        next
      else
        return [str, str]
      end
    when (?1..?9).include?(ch)
      @in.mark(sign, ch)
      sign = nil
      @in.skipwhile(?0..?9)
      num_type = ([?., ?e, ?E, ?d, ?D].include?(@in.lookc)) ? skipfloat_or_fixed : :integer_literal
      r = @in.getregion
      if num_type == :floating_pt_literal
        return [:floating_pt_literal, r.to_f]
      elsif num_type == :fixed_pt_literal
        return [:fixed_pt_literal, r]
      else
        return [:integer_literal, r.to_i]
      end
    when ch == ?. #
      @in.mark(ch)
      @in.skipwhile(?0..?9)
      num_type = (?. != @in.lookc) ? skipfloat_or_fixed : nil
      s = @in.getregion
      if s == "."
        parse_error "token consisting of single dot (.) is invalid."
      end
      if num_type == :floating_pt_literal
        return [:floating_pt_literal, s.to_f]
      elsif num_type == :fixed_pt_literal
        return [:fixed_pt_literal, s]
      else
        parse_error "invalid floating point constant."
      end
    when ch == ?0
      @in.mark(sign, ch)
      sign = nil
      _nxtc = @in.lookc
      if _nxtc == ?x || _nxtc == ?X
        @in.skipc
        @in.skipwhile() { |ch_| HEXCHARS.include? ch_ }
        s = @in.getregion
        return [:integer_literal, s.hex]
      else
        dec = FALSE
        @in.skipwhile(?0..?7)
        if (?8..?9).include? @in.lookc
          dec = TRUE
          @in.skipwhile(?0..?9)
        end
        num_type = ([?., ?e, ?E, ?d, ?D].include?(@in.lookc)) ? skipfloat_or_fixed : :integer_literal
        ret = nil
        s = @in.getregion
        if num_type == :floating_pt_literal
          ret = [:floating_pt_literal, s.to_f]
        elsif num_type == :fixed_pt_literal
          ret = [:fixed_pt_literal, s]
        elsif dec
          parse_error "decimal literal starting with '0' should be octal ('0'..'7' only): #{s}"
        else
          ret = [:integer_literal, s.oct]
        end
        return ret
      end
    when ch == ?\'  #' #single quote, for a character literal.
      ret = 0
      _nxtc = @in.lookc
      if _nxtc == ?\\
        @in.skipc
        ret = next_escape
      elsif _nxtc == ?\' #'
        ret = 0
      elsif _nxtc
        ret = ('' << @in.getc).unpack('C').first
      end
      if @in.lookc != ?\' #'
        parse_error "character literal must be single character enclosed in \"'\""
      end
      @in.skipc
      return [ :character_literal, ret ]
    when ch == ?\" #" #double quote, for a string literal.
      ret = ""
      while TRUE
        _nxtc = @in.lookc
        if _nxtc == ?\\
          @in.skipc
          ret << next_escape
        elsif _nxtc == ?\" #"
          @in.skipc
          return [ :string_literal, ret ]
        elsif _nxtc
          ret << @in.getc
        else
          parse_error "unterminated string literal"
        end
      end
    else
      parse_error 'illegal character [' << ch << ']'
    end #of case
  end #of while
  parse_error "unexcepted error"
end #of method next_token

def parse_annotation(in_comment = false)
  @in_annotation = true
  @scan_comment = in_comment
  begin
    # parse (possibly multiple) annotation(s)
    begin
      # next token should be identifier (must be on same line following '@')
      token = next_token
      parse_error 'annotation identifier expected!' unless token.first == :identifier
      annotation_id = token.last
      token, annotation_body = extract_annotation()
      # pass annotation to directiver for processing
      @directiver.define_annotation(annotation_id, annotation_body || {})
    end until token.first != ANNOTATION_STR
  ensure
    @in_annotation = false
    @scan_comment = false
  end
  # check identifier for keywords
  if token.first == :identifier
    # keyword check
    if (a = KEYWORDS.assoc(token.last)).nil?
      token = [ :identifier, Identifier.new(token.last, chk_identifier(token.last)) ]
    elsif token.last == a[1]
      token = [ a[1], nil ]
    else
      parse_error "`#{token.last}' collides with a keyword `#{a[1]}'"
    end
  end
  return token
end

def parse_directive
  @in.skipwhile(?\ , ?\t)
  s = getline
  /^(\w*)\s*/ === s
  s1,s2 = $1, $' #'
  if /(else|endif|elif)/ === s1
    if @ifdef.empty?
      parse_error "#else/#elif/#endif must not appear without preceding #if"
    end
    case s1
    when 'else'
      if @ifnest == 0
        if @ifskip # true branch has already been parsed
          @ifdef[@ifdef.size - 1] = false
        else
          @ifdef[@ifdef.size - 1] ^= true;
          @ifskip = @ifdef.last
        end
      end
    when 'endif'
      if @ifnest == 0
        @ifdef.pop
        @ifskip = @ifdef.last
      else
        @ifnest -= 1
      end
    else
      if @ifnest == 0
        if @ifskip || @ifdef[@ifdef.size - 1]
          # true branch has already been parsed so skip from now on
          @ifdef[@ifdef.size - 1] = false
          @ifskip = true
        else
          while s2 =~ /(^|[\W])defined\s*\(\s*(\w+)\s*\)/
             def_id = $2
             s2.gsub!(/(^|[\W])(defined\s*\(\s*\w+\s*\))/, '\1'+"#{@defined.has_key?(def_id).to_s}")
          end
          s2.gsub!(/(^|[\W])([A-Za-z_][\w]*)/) do |m_| "#{$1}#{resolve_define($2)}" end
          begin
            @ifdef[@ifdef.size - 1] = eval_directive(s2)
            @ifskip = @ifdef[@ifdef.size - 1]
          rescue IDL::ParseError
            raise
          rescue => ex
            p ex
            puts ex.backtrace.join("\n")
            parse_error "error evaluating #elif"
          end
        end
      end
    end
  elsif /(if|ifn?def)/ === s1
    case s1
    when /ifn?def/
      if do_parse?
        if not (/^(\w+)/ === s2)
          parse_error "no #if(n)def target."
        end
        @ifdef.push(@defined[$1].nil? ^ (s1 == "ifdef"))
        @ifskip = @ifdef.last
      else
        @ifnest += 1
      end
    when 'if'
      if do_parse?
        while s2 =~ /(^|[\W])defined\s*\(\s*(\w+)\s*\)/
           def_id = $2
           s2.gsub!(/(^|[\W])(defined\s*\(\s*\w+\s*\))/, '\1'+"#{@defined.has_key?(def_id).to_s}")
        end
        s2.gsub!(/(^|[\W])([A-Za-z_][\w]*)/) do |m_| "#{$1}#{resolve_define($2)}" end
        begin
          @ifdef.push(eval_directive(s2))
          @ifskip = @ifdef.last
        rescue IDL::ParseError
          raise
        rescue => ex
          p ex
          puts ex.backtrace.join("\n")
          parse_error "error evaluating #if"
        end
      else
        @ifnest += 1
      end
    end
  elsif do_parse?
    case s1
    when 'pragma'
      parse_pragma(s2)
    when 'error'
      parse_error(s2)
    when 'define'
      a = s2.split
      a[1] = true if a[1].nil?
      if a[0].nil?
        parse_error "no #define target."
      elsif not @defined[a[0]].nil?
        parse_error "#{a[0]} is already #define-d."
      end
      @defined[a[0]] = a[1]
    when 'undef'
      @defined.delete(s2)
    when 'include'
      if s2[0,1] == '"' || s2[0,1] == '<'
        if s2.size>2
          s2.strip!
          s2 = s2.slice(1..(s2.size-2))
        else
          s2 = ""
        end
      end
      enter_include(s2)
    when /[0-9]+/
      # ignore line directive
    else
      parse_error "unknown directive: #{s}."
    end
  end
end

def parse_error(msg, ex = nil)
  e = IDL::ParseError.new(msg, positions)
  e.set_backtrace(ex.backtrace) unless ex.nil?
  raise e
end

def parse_pragma(s)
  case s
  when /^ID\s+(.*)\s+"(.*)"\s*$/
    @directiver.pragma_id($1.strip, $2)
  when /^version\s+(.*)\s+([0-9]+)\.([0-9]+)\s*$/
    @directiver.pragma_version($1.strip, $2, $3)
  when /^prefix\s+"(.*)"\s*$/
    @prefix = $1
    @directiver.pragma_prefix(@prefix)
  else
    @directiver.handle_pragma(s)
  end
end

def positions
  @stack.reverse.inject(@in.nil? ? [] : [@in.position]) {|pos_arr,(type_,pfx_,ifdef_,in_,elsif_)| pos_arr << in_.position }
end

def resolve_define(id, stack = [])
  return id if ['true', 'false'].include?(id)
  IDL.log(3,"*** RIDL - resolve_define(#{id})")
  if @defined.has_key?(id)
    define_ = @defined[id]
    stack << id
    parse_error("circular macro reference detected for [#{define_}]") if stack.include?(define_)
    # resolve any nested macro definitions
    define_.gsub(/(^|[\W])([A-Za-z_][\w]*)/) do |m_| "#{$1}#{resolve_define($2, stack)}" end
  else
    '0' # unknown id
  end
end

def skipfloat_or_fixed
  if (@in.lookc == ?.)
    @in.skipc
    @in.skipwhile(?0..?9)
  end
  if [?e, ?E].include? @in.lookc
    @in.skipc
    @in.skipc if [?+, ?-].include? @in.lookc
    @in.skipwhile(?0..?9)
    return :floating_pt_literal
  elsif [?d, ?D].include? @in.lookc
    @in.skipc
    @in.skipc if [?+, ?-].include? @in.lookc
    @in.skipwhile(?0..?9)
    return :fixed_pt_literal
  end
  return :floating_pt_literal
end

def skipline
  while TRUE
    s = @in.gets
    until s.chomp!.nil?; end
    break unless s[s.length - 1] == ?\\
  end
end