module Dependabot::SharedHelpers
def self.check_out_of_disk_memory_error(stderr, error_context)
def self.check_out_of_disk_memory_error(stderr, error_context) if stderr&.include?("No space left on device") || stderr&.include?("Out of diskspace") raise HelperSubprocessFailed.new( message: "No space left on device", error_class: "Dependabot::OutOfDisk", error_context: error_context ) elsif stderr&.include?("MemoryError") raise HelperSubprocessFailed.new( message: "MemoryError", error_class: "Dependabot::OutOfMemory", error_context: error_context ) end end
def self.check_out_of_memory_error(stderr, error_context, error_class)
def self.check_out_of_memory_error(stderr, error_context, error_class) return unless stderr&.include?("JavaScript heap out of memory") raise error_class.new( message: "JavaScript heap out of memory", error_class: "Dependabot::OutOfMemoryError", error_context: error_context ) end
def self.configure_git_to_use_https(host)
def self.configure_git_to_use_https(host) # NOTE: we use --global here (rather than --system) so that Dependabot # can be run without privileged access run_shell_command( "git config --global --replace-all url.https://#{host}/." \ "insteadOf ssh://git@#{host}/" ) run_shell_command( "git config --global --add url.https://#{host}/." \ "insteadOf ssh://git@#{host}:" ) run_shell_command( "git config --global --add url.https://#{host}/." \ "insteadOf git@#{host}:" ) run_shell_command( "git config --global --add url.https://#{host}/." \ "insteadOf git@#{host}/" ) run_shell_command( "git config --global --add url.https://#{host}/." \ "insteadOf git://#{host}/" ) end
def self.configure_git_to_use_https_with_credentials(credentials, safe_directories, git_config_global_path)
def self.configure_git_to_use_https_with_credentials(credentials, safe_directories, git_config_global_path) File.open(git_config_global_path, "w") do |file| file << "# Generated by dependabot/dependabot-core" end # Then add a file-based credential store that loads a file in this repo. # Under the hood this uses git credential-store, but it's invoked through # a wrapper binary that only allows non-mutating commands. Without this, # whenever the credentials are deemed to be invalid, they're erased. run_shell_command( "git config --global credential.helper " \ "'!#{credential_helper_path} --file #{Dir.pwd}/git.store'", allow_unsafe_shell_command: true, fingerprint: "git config --global credential.helper '<helper_command>'" ) # see https://github.blog/2022-04-12-git-security-vulnerability-announced/ safe_directories.each do |path| run_shell_command("git config --global --add safe.directory #{path}") end github_credentials = credentials .select { |c| c["type"] == "git_source" } .select { |c| c["host"] == "github.com" } .select { |c| c["password"] && c["username"] } # If multiple credentials are specified for github.com, pick the one that # *isn't* just an app token (since it must have been added deliberately) github_credential = github_credentials.find { |c| !c["password"]&.start_with?("v1.") } || github_credentials.first # Make sure we always have https alternatives for github.com. configure_git_to_use_https("github.com") if github_credential.nil? deduped_credentials = credentials - github_credentials + [github_credential].compact # Build the content for our credentials file git_store_content = "" deduped_credentials.each do |cred| next unless cred["type"] == "git_source" next unless cred["username"] && cred["password"] authenticated_url = "https://#{cred.fetch('username')}:#{cred.fetch('password')}" \ "@#{cred.fetch('host')}" git_store_content += authenticated_url + "\n" configure_git_to_use_https(cred.fetch("host")) end # Save the file File.write("git.store", git_store_content) end
def self.credential_helper_path
def self.credential_helper_path File.join(__dir__, "../../bin/git-credential-store-immutable") end
def self.escape_command(command)
def self.escape_command(command) CommandHelpers.escape_command(command) end
def self.excon_defaults(options = nil)
def self.excon_defaults(options = nil) options ||= {} headers = T.cast(options.delete(:headers), T.nilable(T::Hash[String, String])) { instrumentor: Dependabot::SimpleInstrumentor, connect_timeout: 5, write_timeout: 5, read_timeout: 20, retry_limit: 4, # Excon defaults to four retries, but let's set it explicitly for clarity omit_default_port: true, middlewares: excon_middleware, headers: excon_headers(headers) }.merge(options) end
def self.excon_headers(headers = nil)
def self.excon_headers(headers = nil) headers ||= {} { "User-Agent" => USER_AGENT }.merge(headers) end
def self.excon_middleware
def self.excon_middleware T.must(T.cast(Excon.defaults, T::Hash[Symbol, T::Array[T.class_of(Excon::Middleware::Base)]])[:middlewares]) + [Excon::Middleware::Decompress] + [Excon::Middleware::RedirectFollower] end
def self.find_safe_directories
def self.find_safe_directories # to preserve safe directories from global .gitconfig output, process = Open3.capture2("git config --global --get-all safe.directory") safe_directories = [] safe_directories = output.split("\n").compact if process.success? safe_directories end
def self.handle_json_parse_error(stdout, stderr, error_context, error_class)
def self.handle_json_parse_error(stdout, stderr, error_context, error_class) # If the JSON is invalid, the helper has likely failed # We should raise a more helpful error message message = if !stdout.strip.empty? stdout elsif !stderr.strip.empty? stderr else "No output from command" end error_class.new( message: message, error_class: "JSON::ParserError", error_context: error_context ) end
def self.helper_subprocess_bash_command(command:, stdin_data:, env:)
def self.helper_subprocess_bash_command(command:, stdin_data:, env:) escaped_stdin_data = stdin_data.gsub("\"", "\\\"") env_keys = env ? env.compact.map { |k, v| "#{k}=#{v}" }.join(" ") + " " : "" "$ cd #{Dir.pwd} && echo \"#{escaped_stdin_data}\" | #{env_keys}#{command}" end
def self.in_a_temporary_directory(directory = "/", &_block)
def self.in_a_temporary_directory(directory = "/", &_block) FileUtils.mkdir_p(Utils::BUMP_TMP_DIR_PATH) tmp_dir = Dir.mktmpdir(Utils::BUMP_TMP_FILE_PREFIX, Utils::BUMP_TMP_DIR_PATH) path = Pathname.new(File.join(tmp_dir, directory)).expand_path begin path = Pathname.new(File.join(tmp_dir, directory)).expand_path FileUtils.mkpath(path) Dir.chdir(path) { yield(path) } ensure FileUtils.rm_rf(tmp_dir) end end
def self.in_a_temporary_repo_directory(directory = "/", repo_contents_path = nil, &block)
def self.in_a_temporary_repo_directory(directory = "/", repo_contents_path = nil, &block) if repo_contents_path # If a workspace has been defined to allow orcestration of the git repo # by the runtime we should defer to it, otherwise we prepare the folder # for direct use and yield. if Dependabot::Workspace.active_workspace T.must(Dependabot::Workspace.active_workspace).change(&block) else path = Pathname.new(File.join(repo_contents_path, directory)).expand_path reset_git_repo(repo_contents_path) # Handle missing directories by creating an empty one and relying on the # file fetcher to raise a DependencyFileNotFound error FileUtils.mkdir_p(path) Dir.chdir(path) { yield(path) } end else in_a_temporary_directory(directory, &block) end end
def self.reset_git_repo(path)
def self.reset_git_repo(path) Dir.chdir(path) do run_shell_command("git reset HEAD --hard") run_shell_command("git clean -fx") end end
def self.run_helper_subprocess(command:, function:, args:, env: nil,
def self.run_helper_subprocess(command:, function:, args:, env: nil, stderr_to_stdout: false, allow_unsafe_shell_command: false, error_class: HelperSubprocessFailed, timeout: CommandHelpers::TIMEOUTS::DEFAULT) start = Time.now stdin_data = JSON.dump(function: function, args: args) cmd = allow_unsafe_shell_command ? command : escape_command(command) # NOTE: For debugging native helpers in specs and dry-run: outputs the # bash command to run in the tmp directory created by # in_a_temporary_directory if ENV["DEBUG_FUNCTION"] == function puts helper_subprocess_bash_command(stdin_data: stdin_data, command: cmd, env: env) # Pause execution so we can run helpers inside the temporary directory T.unsafe(self).debugger end env_cmd = [env, cmd].compact if Experiments.enabled?(:enable_shared_helpers_command_timeout) stdout, stderr, process = CommandHelpers.capture3_with_timeout( env_cmd, stdin_data: stdin_data, timeout: timeout ) else stdout, stderr, process = T.unsafe(Open3).capture3(*env_cmd, stdin_data: stdin_data) end time_taken = Time.now - start if ENV["DEBUG_HELPERS"] == "true" puts env_cmd puts function puts stdout puts stderr end # Some package managers output useful stuff to stderr instead of stdout so # we want to parse this, most package manager will output garbage here so # would mess up json response from stdout stdout = "#{stderr}\n#{stdout}" if stderr_to_stdout error_context = { command: command, function: function, args: args, time_taken: time_taken, stderr_output: stderr[0..50_000], # Truncate to ~100kb process_exit_value: process.to_s, process_termsig: process&.termsig } check_out_of_memory_error(stderr, error_context, error_class) begin response = JSON.parse(stdout) return response["result"] if process&.success? raise error_class.new( message: response["error"], error_class: response["error_class"], error_context: error_context, trace: response["trace"] ) rescue JSON::ParserError raise handle_json_parse_error(stdout, stderr, error_context, error_class) end end
def self.run_shell_command(command,
def self.run_shell_command(command, allow_unsafe_shell_command: false, cwd: nil, env: {}, fingerprint: nil, stderr_to_stdout: true, timeout: CommandHelpers::TIMEOUTS::DEFAULT) start = Time.now cmd = allow_unsafe_shell_command ? command : escape_command(command) puts cmd if ENV["DEBUG_HELPERS"] == "true" opts = {} opts[:chdir] = cwd if cwd env_cmd = [env || {}, cmd, opts].compact if Experiments.enabled?(:enable_shared_helpers_command_timeout) stdout, stderr, process = CommandHelpers.capture3_with_timeout( env_cmd, stderr_to_stdout: stderr_to_stdout, timeout: timeout ) elsif stderr_to_stdout stdout, process = Open3.capture2e(env || {}, cmd, opts) else stdout, stderr, process = Open3.capture3(env || {}, cmd, opts) end time_taken = Time.now - start # Raise an error with the output from the shell session if the # command returns a non-zero status return stdout || "" if process&.success? error_context = { command: cmd, fingerprint: fingerprint, time_taken: time_taken, process_exit_value: process.to_s } check_out_of_disk_memory_error(stderr, error_context) raise SharedHelpers::HelperSubprocessFailed.new( message: stderr_to_stdout ? (stdout || "") : "#{stderr}\n#{stdout}", error_context: error_context ) end
def self.scp_to_standard(uri)
def self.scp_to_standard(uri) return uri unless uri.start_with?("git@") "https://#{T.must(uri.split('git@').last).sub(%r{:/?}, '/')}" end
def self.with_git_configured(credentials:, &_block)
def self.with_git_configured(credentials:, &_block) safe_directories = find_safe_directories FileUtils.mkdir_p(Utils::BUMP_TMP_DIR_PATH) previous_config = ENV.fetch("GIT_CONFIG_GLOBAL", nil) # adding a random suffix to avoid conflicts when running in parallel # some package managers like bundler will modify the global git config git_config_global_path = File.expand_path("#{SecureRandom.hex(16)}.gitconfig", Utils::BUMP_TMP_DIR_PATH) previous_terminal_prompt = ENV.fetch("GIT_TERMINAL_PROMPT", nil) begin ENV["GIT_CONFIG_GLOBAL"] = git_config_global_path ENV["GIT_TERMINAL_PROMPT"] = "false" configure_git_to_use_https_with_credentials(credentials, safe_directories, git_config_global_path) yield ensure ENV["GIT_CONFIG_GLOBAL"] = previous_config ENV["GIT_TERMINAL_PROMPT"] = previous_terminal_prompt end rescue Errno::ENOSPC => e raise Dependabot::OutOfDisk, e.message ensure FileUtils.rm_f(T.must(git_config_global_path)) end