module IRB::Color

def scan: (String code, allow_last_error: true) -> nil

def scan(code, allow_last_error:)
  verbose, $VERBOSE = $VERBOSE, nil
  RubyLex.compile_with_errors_suppressed(code) do |inner_code, line_no|
    lexer = Ripper::Lexer.new(inner_code, '(ripper)', line_no)
    byte_pos = 0
    line_positions = [0]
    inner_code.lines.each do |line|
      line_positions << line_positions.last + line.bytesize
    end
    on_scan = proc do |elem|
      start_pos = line_positions[elem.pos[0] - 1] + elem.pos[1]
      # yield uncolorable code
      if byte_pos < start_pos
        yield(nil, inner_code.byteslice(byte_pos...start_pos), nil)
      end
      if byte_pos <= start_pos
        str = elem.tok
        yield(elem.event, str, elem.state)
        byte_pos = start_pos + str.bytesize
      end
    end
    lexer.scan.each do |elem|
      next if allow_last_error and /meets end of file|unexpected end-of-input/ =~ elem.message
      on_scan.call(elem)
    end
    # yield uncolorable DATA section
    yield(nil, inner_code.byteslice(byte_pos...inner_code.bytesize), nil) if byte_pos < inner_code.bytesize
  end
ensure
  $VERBOSE = verbose
end