Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion cargo/lib/dependabot/cargo/file_parser.rb
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
require "dependabot/cargo/version"
require "dependabot/errors"
require "dependabot/cargo/registry_fetcher"
require "dependabot/cargo/registry_response"
require "dependabot/cargo/language"
require "dependabot/cargo/package_manager"

Expand Down Expand Up @@ -328,7 +329,11 @@ def sparse_registry_source_details(registry_name, index_url)
url = index_url.delete_prefix("sparse+")
url << "/" unless url.end_with?("/")
url << "config.json"
config_json = JSON.parse(RegistryClient.get(url: url, headers: headers).body)
response = RegistryClient.get(url: url, headers: headers)
RegistryResponse.raise_for_error_status!(response, index_url.delete_prefix("sparse+"))
config_json = RegistryResponse.with_parse_error_context(response, url) do
T.cast(JSON.parse(response.body), T::Hash[String, String])
end

{
type: "registry",
Expand Down
17 changes: 6 additions & 11 deletions cargo/lib/dependabot/cargo/metadata_finder.rb
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
require "dependabot/metadata_finders"
require "dependabot/metadata_finders/base"
require "dependabot/registry_client"
require "dependabot/cargo/registry_response"

module Dependabot
module Cargo
Expand Down Expand Up @@ -65,9 +66,12 @@ def crates_listing
hdrs = build_headers(index, dependency.source_string("name"))

url = metadata_fetch_url(dependency, index)
response = fetch_metadata(url, hdrs)
response = Dependabot::RegistryClient.get(url: url, headers: hdrs)
return @crates_listing = {} if response.status == 404

@crates_listing = parse_response(response, index)
RegistryResponse.raise_for_error_status!(response, index.delete_prefix("sparse+")) if index != CRATES_IO_API

@crates_listing = RegistryResponse.with_parse_error_context(response, url) { parse_response(response, index) }
end

sig do
Expand All @@ -86,15 +90,6 @@ def build_headers(index, registry_name)
hdrs
end

sig { params(url: String, headers: T::Hash[String, String]).returns(Excon::Response) }
def fetch_metadata(url, headers)
Excon.get(
url,
idempotent: true,
**SharedHelpers.excon_defaults(headers: headers)
)
end

sig { params(response: Excon::Response, index: String).returns(T::Hash[String, T.anything]) }
def parse_response(response, index)
if index.start_with?("sparse+")
Expand Down
19 changes: 8 additions & 11 deletions cargo/lib/dependabot/cargo/package/package_details_fetcher.rb
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
require "sorbet-runtime"
require "dependabot/registry_client"
require "dependabot/cargo"
require "dependabot/cargo/registry_response"
require "dependabot/package/package_release"
require "dependabot/package/package_details"

Expand Down Expand Up @@ -84,10 +85,15 @@ def crates_listing

Dependabot.logger.info("Calling #{url} to fetch metadata for #{dependency.name} from #{index}")

response = fetch_response(url, hdrs)
response = RegistryClient.get(url: url, headers: hdrs)
return {} if response.status == 404

@crates_listing = T.let(parse_response(response, index), T.nilable(T::Hash[String, T.anything]))
RegistryResponse.raise_for_error_status!(response, index.delete_prefix("sparse+")) if index != CRATES_IO_API

@crates_listing = T.let(
RegistryResponse.with_parse_error_context(response, url) { parse_response(response, index) },
T.nilable(T::Hash[String, T.anything])
)

Dependabot.logger.info("Fetched metadata for #{dependency.name} from #{index} successfully")

Expand Down Expand Up @@ -117,15 +123,6 @@ def auth_headers
{ "Authorization" => token }
end

sig { params(url: String, headers: T::Hash[String, String]).returns(Excon::Response) }
def fetch_response(url, headers)
Excon.get(
url,
idempotent: true,
**SharedHelpers.excon_defaults(headers: headers)
)
end

sig { params(response: Excon::Response, index: String).returns(T::Hash[String, T.anything]) }
def parse_response(response, index)
if index.start_with?("sparse+")
Expand Down
45 changes: 45 additions & 0 deletions cargo/lib/dependabot/cargo/registry_response.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
# typed: strong
# frozen_string_literal: true

require "excon"
require "json"
require "sorbet-runtime"
require "dependabot/error_report_context"
require "dependabot/errors"

module Dependabot
module Cargo
# Registries can answer with non-JSON error bodies (e.g. a 403 with a plain
# `Forbidden` body), which would otherwise surface as unknown JSON parse errors.
module RegistryResponse
extend T::Sig

AUTH_FAILURE_STATUSES = [401, 403].freeze

sig { params(response: Excon::Response, source: String).void }
def self.raise_for_error_status!(response, source)
return if (200..299).cover?(response.status)

raise PrivateSourceAuthenticationFailure, source if AUTH_FAILURE_STATUSES.include?(response.status)

raise PrivateSourceBadResponse, source
end

# Only the host is attached: registry paths can contain private crate names.
sig do
type_parameters(:R)
.params(response: Excon::Response, url: String, blk: T.proc.returns(T.type_parameter(:R)))
.returns(T.type_parameter(:R))
end
def self.with_parse_error_context(response, url, &blk)
yield
rescue JSON::ParserError => e
ErrorReportContext.attach(
e,
{ http_method: "GET", host: ErrorReportContext.host_for(url), http_status: response.status }
)
raise
end
end
end
end
39 changes: 39 additions & 0 deletions cargo/spec/dependabot/cargo/file_parser_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -562,6 +562,45 @@
index: "sparse+https://private.example.com/index/"
)
end

context "when the registry config request is forbidden" do
before do
stub_request(:get, "https://private.example.com/index/config.json")
.to_return(status: 403, body: "Forbidden")
end

it "raises a PrivateSourceAuthenticationFailure for the registry" do
expect { dependencies }.to raise_error(Dependabot::PrivateSourceAuthenticationFailure) do |error|
expect(error.source).to eq("https://private.example.com/index/")
end
end
end

context "when the registry config request fails" do
before do
stub_request(:get, "https://private.example.com/index/config.json")
.to_return(status: 502, body: "")
end

it "raises a PrivateSourceBadResponse for the registry" do
expect { dependencies }.to raise_error(Dependabot::PrivateSourceBadResponse)
end
end

context "when the registry config is not valid JSON" do
before do
stub_request(:get, "https://private.example.com/index/config.json")
.to_return(status: 200, body: "<html>login</html>")
end

it "attaches only the registry host to the parse error" do
expect { dependencies }.to raise_error(JSON::ParserError) do |error|
context = T.must(Dependabot::ErrorReportContext.build(error: error))
expect(context).to include("host" => "private.example.com", "http-status" => 200)
expect(context.values.join).not_to include("config.json")
end
end
end
end

context "when the input is unparseable" do
Expand Down
14 changes: 14 additions & 0 deletions cargo/spec/dependabot/cargo/metadata_finder_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,20 @@
end
end

context "when crates.io returns an unparseable response" do
before do
stub_request(:get, crates_url).to_return(status: 502, body: "Bad Gateway")
end

it "attaches only the registry host to the parse error" do
expect { source_url }.to raise_error(JSON::ParserError) do |error|
context = T.must(Dependabot::ErrorReportContext.build(error: error))
expect(context).to include("host" => "crates.io", "http-status" => 502)
expect(context.values.join).not_to include("bitflags")
end
end
end

context "when there is no recognised source link in the response" do
let(:crates_response) do
fixture("crates_io_responses", crates_fixture_name).gsub!("github.com", "example.com")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,5 +96,62 @@
expect(T.must(first_result.language).requirement).to eq(T.must(latest_release.language).requirement)
end
end

context "when crates.io returns an empty error response" do
before do
stub_request(:get, json_url).to_return(status: 503, body: "")
end

it "attaches only the registry host to the parse error" do
expect { fetch }.to raise_error(JSON::ParserError) do |error|
context = T.must(Dependabot::ErrorReportContext.build(error: error))
expect(context).to include("host" => "crates.io", "http-status" => 503)
expect(context.values.join).not_to include("es-entity")
end
end
end

context "with a dependency from a private registry" do
sig { returns(Dependabot::Dependency) }
let(:dependency) do
Dependabot::Dependency.new(
name: dependency_name,
version: "0.3.23",
requirements: [{
requirement: "==0.3.23",
file: "Cargo.toml",
groups: ["dependencies"],
source: { type: "registry", name: "my-registry", index: "https://private.example.com/index" }
}],
package_manager: "cargo"
)
end

context "when the registry rejects the credentials" do
before do
stub_request(:get, "https://private.example.com/index/es/-e/es-entity").to_return(
status: 403,
body: "Forbidden"
)
end

it "raises a PrivateSourceAuthenticationFailure for the registry" do
expect { fetch }.to raise_error(Dependabot::PrivateSourceAuthenticationFailure) do |error|
expect(T.cast(error, Dependabot::PrivateSourceAuthenticationFailure).source)
.to eq("https://private.example.com/index")
end
end
end

context "when the registry returns a server error" do
before do
stub_request(:get, "https://private.example.com/index/es/-e/es-entity").to_return(status: 500, body: "")
end

it "raises a PrivateSourceBadResponse" do
expect { fetch }.to raise_error(Dependabot::PrivateSourceBadResponse)
end
end
end
end
end
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
require "excon"
require "sorbet-runtime"
require "dependabot/swift"
require "dependabot/error_report_context"
require "dependabot/registry_client"

module Dependabot
module Terraform
Expand Down Expand Up @@ -51,13 +53,15 @@ def fetch_tag_and_release_date_from_provider # rubocop:disable Metrics/AbcSize,M
Dependabot.logger.info("Fetching provider release details from URL: #{url}")
result_lines = T.let([], T::Array[GitTagWithDetail])
# Fetch the releases from the provider API
response = Excon.get(url, headers: { "Accept" => "application/vnd.github.v3+json" })
response = Dependabot::RegistryClient.get(url: url, headers: { "Accept" => "application/vnd.github.v3+json" })
Dependabot.logger.error("Failed call details: #{response.body}") unless response.status == 200
return result_lines unless response.status == 200

# Parse the JSON response
releases = JSON.parse(response.body).fetch("included", [])
.select { |item| item["type"] == "provider-versions" }
releases = with_parse_error_context(response, url) do
JSON.parse(response.body).fetch("included", [])
.select { |item| item["type"] == "provider-versions" }
end
releases = releases.map { |release| release.fetch("attributes", {}) }
# Check if releases is an array and not empty
return result_lines unless releases.is_a?(Array) && !releases.empty?
Expand All @@ -81,13 +85,15 @@ def fetch_tag_and_release_date_from_module
Dependabot.logger.info("Fetching provider release details from URL: #{url}")
result_lines = T.let([], T::Array[GitTagWithDetail])
# Fetch the releases from the provider API
response = Excon.get(url, headers: { "Accept" => "application/vnd.github.v3+json" })
response = Dependabot::RegistryClient.get(url: url, headers: { "Accept" => "application/vnd.github.v3+json" })
Dependabot.logger.error("Failed call details: #{response.body}") unless response.status == 200
return result_lines unless response.status == 200

# Parse the JSON response
releases = JSON.parse(response.body).fetch("included", [])
.select { |item| item["type"] == "module-versions" }
releases = with_parse_error_context(response, url) do
JSON.parse(response.body).fetch("included", [])
.select { |item| item["type"] == "module-versions" }
end
releases = releases.map { |release| release.fetch("attributes", {}) }

# Extract version names and release dates into result_lines
Expand All @@ -101,6 +107,21 @@ def fetch_tag_and_release_date_from_module
result_lines.sort_by(&:tag).reverse
end

sig do
type_parameters(:R)
.params(response: Excon::Response, url: String, blk: T.proc.returns(T.type_parameter(:R)))
.returns(T.type_parameter(:R))
end
def with_parse_error_context(response, url, &blk)
yield
rescue JSON::ParserError => e
ErrorReportContext.attach(
e,
{ http_method: "GET", host: ErrorReportContext.host_for(url), http_status: response.status }
)
raise
end

sig { returns(T.nilable(T::Hash[T.any(String, Symbol), T.untyped])) }
def dependency_source_details
return nil unless @dependency.source_details
Expand Down
Loading
Loading