class Dependabot::Uv::Package::PackageDetailsFetcher
def auth_headers_for(index_url)
def auth_headers_for(index_url) credential = @credentials.find { |cred| cred["index-url"] == index_url } return {} unless credential { "Authorization" => "Basic #{Base64.strict_encode64( "#{credential[CREDENTIALS_USERNAME]}:#{credential[CREDENTIALS_PASSWORD]}" )}" } end
def build_python_requirement(req_string)
def build_python_requirement(req_string) return nil unless req_string requirement_class.new(CGI.unescapeHTML(req_string)) rescue Gem::Requirement::BadRequirementError nil end
def convert_language_version(version)
def convert_language_version(version) return ["python", nil] if version.nil? || version == "source" # Extract numeric parts dynamically (e.g., "cp37" -> "3.7", "py38" -> "3.8") extracted_version = version.scan(/\d+/).join(".") # Detect the language implementation language_name = if version.start_with?("cp") "cpython" # CPython implementation elsif version.start_with?("py") "python" # General Python compatibility else "unknown" # Fallback for unknown cases end # Ensure extracted version is valid before converting language_version = extracted_version.match?(/^\d+(\.\d+)*$/) ? Dependabot::Version.new(extracted_version) : nil Dependabot.logger.warn("Skipping invalid language_version: #{version.inspect}") if language_version.nil? [language_name, language_version] end
def extract_release_details_json_from_html(html_body)
def extract_release_details_json_from_html(html_body) doc = Nokogiri::HTML(html_body) releases = {} doc.css("a").each do |a_tag| details = version_details_from_link(a_tag.to_s) if details && details["version"] releases[details["version"]] ||= [] releases[details["version"]] << details end end releases end
def fetch
def fetch package_releases = registry_urls .select { |index_url| validate_index(index_url) } # Ensure only valid URLs .flat_map do |index_url| fetch_from_registry(index_url) || [] # Ensure it always returns an array rescue Excon::Error::Timeout, Excon::Error::Socket raise if MAIN_PYPI_INDEXES.include?(index_url) raise PrivateSourceTimedOut, sanitized_url(index_url) rescue URI::InvalidURIError raise DependencyFileNotResolvable, "Invalid URL: #{sanitized_url(index_url)}" end Dependabot::Package::PackageDetails.new( dependency: dependency, releases: package_releases.reverse.uniq(&:version) ) end
def fetch_from_html_registry(index_url)
def fetch_from_html_registry(index_url) Dependabot.logger.info( "Fetching release information from html registry at #{sanitized_url(index_url)} for #{dependency.name}" ) index_response = registry_response_for_dependency(index_url) if index_response.status == 401 || index_response.status == 403 registry_index_response = registry_index_response(index_url) if registry_index_response.status == 401 || registry_index_response.status == 403 raise PrivateSourceAuthenticationFailure, sanitized_url(index_url) end end version_releases = extract_release_details_json_from_html(index_response.body) releases = format_version_releases(version_releases) releases.sort_by(&:version).reverse end
def fetch_from_json_registry(index_url)
def fetch_from_json_registry(index_url) json_url = index_url.sub(%r{/simple/?$}i, "/pypi/") Dependabot.logger.info( "Fetching release information from json registry at #{sanitized_url(json_url)} for #{dependency.name}" ) response = registry_json_response_for_dependency(json_url) return nil unless response.status == 200 begin data = JSON.parse(response.body) version_releases = data["releases"] releases = format_version_releases(version_releases) releases.sort_by(&:version).reverse rescue JSON::ParserError Dependabot.logger.warn("JSON parsing error for #{json_url}. Falling back to HTML.") nil rescue StandardError => e Dependabot.logger.warn("Unexpected error while fetching JSON data: #{e.message}.") nil end end
def fetch_from_registry(index_url)
def fetch_from_registry(index_url) if Dependabot::Experiments.enabled?(:enable_cooldown_for_uv) metadata = fetch_from_json_registry(index_url) return metadata if metadata&.any? Dependabot.logger.warn("No valid versions found via JSON API. Falling back to HTML.") end fetch_from_html_registry(index_url) rescue StandardError => e Dependabot.logger.warn("Unexpected error in JSON fetch: #{e.message}. Falling back to HTML.") fetch_from_html_registry(index_url) end
def format_version_release(version, release_data)
def format_version_release(version, release_data) upload_time = release_data["upload_time"] released_at = Time.parse(upload_time) if upload_time yanked = release_data["yanked"] || false yanked_reason = release_data["yanked_reason"] downloads = release_data["downloads"] || -1 url = release_data["url"] package_type = release_data["packagetype"] language = package_language( python_version: release_data["python_version"], requires_python: release_data["requires_python"] ) release = Dependabot::Package::PackageRelease.new( version: Dependabot::Uv::Version.new(version), released_at: released_at, yanked: yanked, yanked_reason: yanked_reason, downloads: downloads, url: url, package_type: package_type, language: language ) release end
def format_version_releases(releases_json)
def format_version_releases(releases_json) return [] unless releases_json releases_json.each_with_object([]) do |(version, release_data_array), versions| release_data = release_data_array.last next unless release_data release = format_version_release(version, release_data) next unless release versions << release end end
def get_version_from_filename(filename)
def get_version_from_filename(filename) filename .gsub(/#{name_regex}-/i, "") .split(/-|\.tar\.|\.zip|\.whl/) .first end
def initialize(
def initialize( dependency:, dependency_files:, credentials: ) @dependency = dependency @dependency_files = dependency_files @credentials = credentials @registry_urls = T.let(nil, T.nilable(T::Array[String])) end
def name_regex
def name_regex parts = normalised_name.split(/[\s_.-]/).map { |n| Regexp.quote(n) } /#{parts.join("[\s_.-]")}/i end
def normalised_name
def normalised_name NameNormaliser.normalise(dependency.name) end
def package_language(python_version:, requires_python:)
def package_language(python_version:, requires_python:) # Extract language name and version language_name, language_version = convert_language_version(python_version) # Extract language requirement language_requirement = build_python_requirement(requires_python) return nil unless language_version || language_requirement # Return a Language object with all details Dependabot::Package::PackageLanguage.new( name: language_name, version: language_version, requirement: language_requirement ) end
def registry_index_response(index_url)
def registry_index_response(index_url) Dependabot::RegistryClient.get( url: index_url, headers: { "Accept" => APPLICATION_TEXT } ) end
def registry_json_response_for_dependency(json_url)
def registry_json_response_for_dependency(json_url) url = "#{json_url.chomp('/')}/#{@dependency.name}/json" Dependabot::RegistryClient.get( url: url, headers: { "Accept" => APPLICATION_JSON } ) end
def registry_response_for_dependency(index_url)
def registry_response_for_dependency(index_url) Dependabot::RegistryClient.get( url: index_url + normalised_name + "/", headers: { "Accept" => APPLICATION_TEXT } ) end
def registry_urls
def registry_urls @registry_urls ||= Package::PackageRegistryFinder.new( dependency_files: dependency_files, credentials: credentials, dependency: dependency ).registry_urls end
def requirement_class
def requirement_class dependency.requirement_class end
def requires_python_from_link(link)
def requires_python_from_link(link) raw_value = Nokogiri::XML(link) .at_css("a") &.attribute("data-requires-python") &.content return nil unless raw_value CGI.unescapeHTML(raw_value) # Decodes HTML entities like >=3 → >=3 end
def sanitized_url(index_url)
def sanitized_url(index_url) index_url.sub(%r{//([^/@]+)@}, "//redacted@") end
def validate_index(index_url)
def validate_index(index_url) return false unless index_url return true if index_url.match?(URI::DEFAULT_PARSER.regexp[:ABS_URI]) raise Dependabot::DependencyFileNotResolvable, "Invalid URL: #{sanitized_url(index_url)}" end
def version_class
def version_class dependency.version_class end
def version_details_from_link(link)
def version_details_from_link(link) return unless link doc = Nokogiri::XML(link) filename = doc.at_css("a")&.content url = doc.at_css("a")&.attributes&.fetch("href", nil)&.value return unless filename&.match?(name_regex) || url&.match?(name_regex) version = get_version_from_filename(filename) return unless version_class.correct?(version) { "version" => version, "requires_python" => requires_python_from_link(link), "yanked" => link.include?("data-yanked"), "url" => link } end