Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions python/lib/dependabot/python/dependency_grapher.rb
Original file line number Diff line number Diff line change
Expand Up @@ -142,9 +142,8 @@ def txt_files_to_keep
files_by_name = dependency_files.to_h { |file| [file.name, file] }

seeds = dependency_files.select do |file|
file.name.end_with?(".txt") &&
(RequirementsLayers.manifest_txt_filename?(file.name) ||
pip_compile_file_matcher.lockfile_for_pip_compile_file?(file))
RequirementsLayers.manifest_txt_file?(file) ||
(file.name.end_with?(".txt") && pip_compile_file_matcher.lockfile_for_pip_compile_file?(file))
end

# `.in` files are always retained, so a `.txt` they reference must be kept too.
Expand Down Expand Up @@ -493,7 +492,7 @@ def pip_requirements_file

@pip_requirements_file = T.let(
dependency_files.find { |f| f.name == "requirements.txt" } ||
dependency_files.find { |f| f.name.end_with?(".txt") && RequirementsLayers.manifest_txt_filename?(f.name) },
dependency_files.find { |file| RequirementsLayers.manifest_txt_file?(file) },
T.nilable(Dependabot::DependencyFile)
)
end
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,12 @@ class RequirementsLayers
# Examples: depend.txt, depends.txt, depend-test.txt, py3-depends.txt
DEPEND_TXT_REGEX = %r{(?:[-._]|^|/)depend(?:s)?(?:[-_/][^\s.]*)?\.txt$}i

# Whether a .txt filename looks like a real pip manifest rather than an unrelated .txt file.
sig { params(path: String).returns(T::Boolean) }
def self.manifest_txt_filename?(path)
# Use the repository path so `requirements/base.txt` is classified the same from either scan directory.
sig { params(file: Dependabot::DependencyFile).returns(T::Boolean) }
def self.manifest_txt_file?(file)
return false unless file.name.end_with?(".txt")

path = file.path
path.match?(REQUIREMENTS_TXT_REGEX) ||
path.match?(REQUIRE_TXT_REGEX) ||
path.match?(DEPENDENCIES_TXT_REGEX) ||
Expand Down Expand Up @@ -84,9 +87,7 @@ def groups
# like a real manifest, plus any `.in` file that has no compiled `.txt` counterpart.
sig { returns(T::Array[Dependabot::DependencyFile]) }
def layer_primaries
txt_primaries = requirement_family_files.select do |f|
f.name.end_with?(".txt") && self.class.manifest_txt_filename?(f.name)
end
txt_primaries = requirement_family_files.select { |file| self.class.manifest_txt_file?(file) }
compiled_stems = txt_primaries.map { |f| requirements_stem(f.name) }

in_primaries = requirement_family_files.select do |f|
Expand Down
53 changes: 53 additions & 0 deletions python/spec/dependabot/python/dependency_grapher_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -1330,6 +1330,59 @@ def resolved_package_names(snapshot)
end
end

describe "graphing a requirements directory with generic layer filenames" do
let(:layers) do
{
"base.txt" => "requests==2.32.5\n",
"local.txt" => "-r ./base.txt\ncolorama==0.4.6\n",
"production.txt" => "-r ./base.txt\ngunicorn==23.0.0\n"
}
end
let(:scan_directory) { "/requirements" }
let(:dependency_files) do
layers.map do |name, content|
name = "requirements/#{name}" if scan_directory == "/"
Dependabot::DependencyFile.new(name: name, content: content, directory: scan_directory)
end
end

def layer_dependencies
grapher.manifest_group_snapshots.to_h do |snapshot|
[snapshot.manifest_file.path, snapshot.resolved_dependencies.keys.sort]
end
end

shared_examples "graphs the same requirements files" do
it "attributes each layer to its repository path and includes its referenced dependencies" do
expect(layer_dependencies).to include(
"/requirements/base.txt" => ["pkg:pypi/requests@2.32.5"],
"/requirements/local.txt" => ["pkg:pypi/colorama@0.4.6", "pkg:pypi/requests@2.32.5"],
"/requirements/production.txt" => ["pkg:pypi/gunicorn@23.0.0", "pkg:pypi/requests@2.32.5"]
)
end
end

include_examples "graphs the same requirements files"

context "when scanned from the repository root with a tool-only pyproject.toml" do
let(:scan_directory) { "/" }
let(:dependency_files) do
super() + [Dependabot::DependencyFile.new(name: "pyproject.toml", content: "[tool.black]\nline-length = 88\n",
directory: "/")]
end

include_examples "graphs the same requirements files"
end

context "when there is just one layer" do
let(:layers) { { "base.txt" => "requests==2.32.5\n" } }

it "selects the requirements file for the single-manifest fallback" do
expect(layer_dependencies).to eq("/requirements/base.txt" => ["pkg:pypi/requests@2.32.5"])
end
end
end

# A pip/pip-compile directory can hold requirements layers AND a non-requirements manifest
# (setup.py/setup.cfg/pyproject.toml).
#
Expand Down
Loading