class Bundler::ParallelInstaller
def self.call(*args, **kwargs)
def self.call(*args, **kwargs) new(*args, **kwargs).call end
def bsd_make?
def bsd_make? return false unless Gem.freebsd_platform? make = ENV["MAKE"] || ENV["make"] || "make" !/\bgmake/i.match?(make) end
def call
def call if @rake do_download(@rake, 0) do_install(@rake, 0) Gem::Specification.reset end if @size > 1 install_with_worker else install_serially end handle_error if failed_specs.any? @specs ensure worker_pool&.stop end
def do_download(spec_install, worker_num)
def do_download(spec_install, worker_num) Plugin.hook(Plugin::Events::GEM_BEFORE_INSTALL, spec_install) gem_installer = Bundler::GemInstaller.new( spec_install.spec, @installer, @standalone, worker_num, @force, @local ) success, message = gem_installer.download if success spec_install.state = :downloaded else spec_install.error = "#{message}\n\n#{require_tree_for_spec(spec_install.spec)}" spec_install.state = :failed end spec_install end
def do_install(spec_install, worker_num)
def do_install(spec_install, worker_num) gem_installer = Bundler::GemInstaller.new( spec_install.spec, @installer, @standalone, worker_num, @force, @local ) success, message = gem_installer.install_from_spec if success spec_install.state = :installed spec_install.post_install_message = message unless message.nil? else spec_install.error = "#{message}\n\n#{require_tree_for_spec(spec_install.spec)}" spec_install.state = :failed end Plugin.hook(Plugin::Events::GEM_AFTER_INSTALL, spec_install) spec_install end
def enqueue_specs(installed_specs)
previously installed specifications. We continue until all specs
Later we call this lambda again to install specs that depended on
We enqueue all gem specs that do not have any dependencies.
Keys in the remains hash represent uninstalled gems specs.
def enqueue_specs(installed_specs) @specs.each do |spec| if spec.installed? installed_specs[spec.name] = true next end spec.state = :enqueued worker_pool.enq spec end end
def failed_specs
def failed_specs @specs.select(&:failed?) end
def finished_installing?
def finished_installing? @specs.all? do |spec| return true if spec.failed? spec.installed? end end
def handle_error
def handle_error errors = failed_specs.map(&:error) if exception = errors.find {|e| e.is_a?(Bundler::BundlerError) } raise exception end raise Bundler::InstallError, errors.join("\n\n") end
def initialize(installer, all_specs, size, standalone, force, local: false, skip: nil)
def initialize(installer, all_specs, size, standalone, force, local: false, skip: nil) @installer = installer @size = size @standalone = standalone @force = force @local = local @specs = all_specs.map {|s| SpecInstallation.new(s) } specs_by_name = @specs.to_h {|s| [s.name, s] } @specs.each do |spec_install| spec_install.dependencies = spec_install.spec.dependencies.filter_map do |dep| specs_by_name[dep.name] unless dep.type == :development || dep.name == spec_install.name end end @specs.each do |spec_install| spec_install.state = :installed if skip.include?(spec_install.name) end if skip @spec_set = all_specs @rake = @specs.find {|s| s.name == "rake" unless s.installed? } end
def install_serially
def install_serially until finished_installing? raise "failed to find a spec to enqueue while installing serially" unless spec_install = @specs.find(&:ready_to_enqueue?) spec_install.state = :enqueued do_download(spec_install, 0) do_install(spec_install, 0) end end
def install_with_worker
def install_with_worker with_jobserver do installed_specs = {} enqueue_specs(installed_specs) process_specs(installed_specs) until finished_installing? end end
def nmake?
Mirror how RubyGems' extension builder picks the make program so the
def nmake? make = ENV["MAKE"] || ENV["make"] make ||= "nmake" if RUBY_PLATFORM.include?("mswin") /\bnmake/i.match?(make.to_s) end
def process_specs(installed_specs)
processed so the call to `enqueue_specs` is important after every
Some specs might've had to wait til this spec was installed to be
remaining specs.
Dequeue a spec and save its post-install message and then enqueue the
def process_specs(installed_specs) spec = worker_pool.deq if spec.installed? installed_specs[spec.name] = true return elsif spec.failed? return elsif spec.ready_to_install?(installed_specs) spec.state = :installable end worker_pool.enq(spec, priority: spec.enqueue_with_priority?) end
def require_tree_for_spec(spec)
def require_tree_for_spec(spec) tree = @spec_set.what_required(spec) t = String.new("In #{File.basename(SharedHelpers.default_gemfile)}:\n") tree.each_with_index do |s, depth| t << " " * depth.succ << s.name unless tree.last == s t << %( was resolved to #{s.version}, which depends on) end t << %(\n) end t end
def with_jobserver
def with_jobserver # The jobserver hands tokens to child `make` processes through MAKEFLAGS # using the GNU make `--jobserver-auth` protocol. nmake, the default make # on mswin, instead reads MAKEFLAGS as bare option letters and aborts # every native extension build with `fatal error U1065: invalid option # '-'`. Skip the jobserver when nmake is in use. Other Windows toolchains # such as mingw use GNU make and keep working through the inherited pipe. return yield if nmake? || bsd_make? begin r, w = IO.pipe r.close_on_exec = false w.close_on_exec = false w.write("*" * @size) old_makeflags = ENV["MAKEFLAGS"] ENV["MAKEFLAGS"] = [old_makeflags, "--jobserver-auth=#{r.fileno},#{w.fileno}"].compact.join(" ") yield ensure # Restore MAKEFLAGS before closing the pipe so a close failure can't # leave the process with descriptors that point at a closed pipe. old_makeflags ? ENV["MAKEFLAGS"] = old_makeflags : ENV.delete("MAKEFLAGS") r&.close w&.close end end
def worker_pool
def worker_pool @worker_pool ||= Bundler::Worker.new @size, "Parallel Installer", lambda {|spec_install, worker_num| case spec_install.state when :enqueued do_download(spec_install, worker_num) when :installable do_install(spec_install, worker_num) else spec_install end } end