From d486f10bc67de5a7863d2b000edd2d808b4648e0 Mon Sep 17 00:00:00 2001 From: Jon Ruskin Date: Tue, 15 Mar 2022 14:10:55 -0700 Subject: [PATCH 1/3] use pip list to get packages for pip source this also removes the need for a requirements.txt file in the current directory --- docs/sources/pip.md | 5 -- lib/licensed/sources/pip.rb | 51 ++++++++++++------- test/sources/pip_test.rb | 97 ++----------------------------------- 3 files changed, 38 insertions(+), 115 deletions(-) diff --git a/docs/sources/pip.md b/docs/sources/pip.md index 2f75f257..6645eaa7 100644 --- a/docs/sources/pip.md +++ b/docs/sources/pip.md @@ -2,11 +2,6 @@ The pip source uses `pip` CLI commands to enumerate dependencies and properties. It is expected that `pip` is available in the `virtual_env_dir` specific directory before running `licensed`. -Your repository root should also contain a `requirements.txt` file which contains all the packages and dependences that are needed. You can generate one with `pip` using the command: -``` -pip freeze > requirements.txt -``` - A `virtualenv` directory is required before running `licensed`. You can setup a `virtualenv` by running the command: ``` virtualenv diff --git a/lib/licensed/sources/pip.rb b/lib/licensed/sources/pip.rb index db2796ef..0396fd8e 100644 --- a/lib/licensed/sources/pip.rb +++ b/lib/licensed/sources/pip.rb @@ -7,17 +7,14 @@ module Licensed module Sources class Pip < Source - VERSION_OPERATORS = %w(< > <= >= == !=).freeze - PACKAGE_REGEX = /^([\w\.-]+)(#{VERSION_OPERATORS.join("|")})?/ + PACKAGE_INFO_SEPARATOR = "\n---\n" def enabled? - return unless virtual_env_pip && Licensed::Shell.tool_available?(virtual_env_pip) - File.exist?(config.pwd.join("requirements.txt")) + virtual_env_pip && Licensed::Shell.tool_available?(virtual_env_pip) end def enumerate_dependencies - Parallel.map(packages_from_requirements_txt, in_threads: Parallel.processor_count) do |package_name| - package = package_info(package_name) + packages.map do |package| location = File.join(package["Location"], package["Name"].gsub("-", "_") + "-" + package["Version"] + ".dist-info") Dependency.new( name: package["Name"], @@ -34,25 +31,45 @@ def enumerate_dependencies private - def packages_from_requirements_txt - File.read(config.pwd.join("requirements.txt")) - .lines - .reject { |line| line.include?("://") } - .map { |line| line.strip.match(PACKAGE_REGEX) { |match| match.captures.first } } - .compact + # Returns parsed information for all packages used by the project, + # using `pip list` to determine what packages are used and `pip show` + # to gather package information + def packages + all_packages = pip_show_command(package_names) + all_packages.split(PACKAGE_INFO_SEPARATOR).reduce([]) do |accum, val| + accum << parse_package_info(val) + end + end + + # Returns the names of all of the packages used by the current project, + # as returned from `pip list` + def package_names + @package_names ||= begin + JSON.parse(pip_list_command).map { |package| package["name"] } + rescue JSON::ParserError => e + message = "Licensed was unable to parse the output from 'npm list'. JSON Error: #{e.message}" + raise Licensed::Sources::Source::Error, message + end end - def package_info(package_name) - p_info = pip_command(package_name).lines - p_info.each_with_object(Hash.new(0)) { |pkg, a| + # Returns a hash filled with package info parsed from the email-header formatted output + # returned by `pip show` + def parse_package_info(package_info) + package_info.lines.each_with_object(Hash.new(0)) { |pkg, a| k, v = pkg.split(":", 2) next if k.nil? || k.empty? a[k.strip] = v&.strip } end - def pip_command(*args) - Licensed::Shell.execute(virtual_env_pip, "--disable-pip-version-check", "show", *args) + # Returns the output from `pip list --format=json` + def pip_list_command + Licensed::Shell.execute(virtual_env_pip, "--disable-pip-version-check", "list", "--format=json") + end + + # Returns the output from `pip show ...` + def pip_show_command(packages) + Licensed::Shell.execute(virtual_env_pip, "--disable-pip-version-check", "show", *packages) end def virtual_env_pip diff --git a/test/sources/pip_test.rb b/test/sources/pip_test.rb index 8da4f182..a7bc3113 100644 --- a/test/sources/pip_test.rb +++ b/test/sources/pip_test.rb @@ -25,109 +25,20 @@ end describe "dependencies" do - it "detects dependencies without a version constraint" do - Dir.chdir fixtures do - dep = source.dependencies.detect { |d| d.name == "scapy" } - assert dep - assert_equal "pip", dep.record["type"] - assert dep.record["homepage"] - assert dep.record["summary"] - end - end - - it "detects dependencies with == version constraint" do + it "detects explicit dependencies" do Dir.chdir fixtures do dep = source.dependencies.detect { |d| d.name == "Jinja2" } assert dep + assert_equal "2.9.6", dep.version assert_equal "pip", dep.record["type"] assert dep.record["homepage"] assert dep.record["summary"] end end - it "detects dependencies with >= version constraint" do - Dir.chdir fixtures do - dep = source.dependencies.detect { |d| d.name == "requests" } - assert dep - assert_equal "pip", dep.record["type"] - assert dep.record["homepage"] - assert dep.record["summary"] - end - end - - it "detects dependencies with <= version constraint" do - Dir.chdir fixtures do - dep = source.dependencies.detect { |d| d.name == "tqdm" } - assert dep - assert_equal "pip", dep.record["type"] - assert dep.record["homepage"] - assert dep.record["summary"] - end - end - - it "detects dependencies with < version constraint" do - Dir.chdir fixtures do - dep = source.dependencies.detect { |d| d.name == "Pillow" } - assert dep - assert_equal "pip", dep.record["type"] - assert dep.record["homepage"] - assert dep.record["summary"] - end - end - - it "detects dependencies with > version constraint" do - Dir.chdir fixtures do - dep = source.dependencies.detect { |d| d.name == "Scrapy" } - assert dep - assert_equal "pip", dep.record["type"] - assert dep.record["homepage"] - assert dep.record["summary"] - end - end - - it "detects dependencies with != version constraint" do - Dir.chdir fixtures do - dep = source.dependencies.detect { |d| d.name == "numpy" } - assert dep - assert_equal "pip", dep.record["type"] - assert dep.record["homepage"] - assert dep.record["summary"] - end - end - - it "detects dependencies with whitespace between the package name and version operator" do - Dir.chdir fixtures do - dep = source.dependencies.detect { |d| d.name == "botocore" } - assert dep - assert_equal "pip", dep.record["type"] - assert dep.record["homepage"] - assert dep.record["summary"] - end - end - - it "detects dependencies with multiple version constraints" do - Dir.chdir fixtures do - dep = source.dependencies.detect { |d| d.name == "boto3" } - assert dep - assert_equal "pip", dep.record["type"] - assert dep.record["homepage"] - assert dep.record["summary"] - end - end - - it "detects dependencies with hyphens in package name" do - Dir.chdir fixtures do - dep = source.dependencies.detect { |d| d.name == "lazy-object-proxy" } - assert dep - assert_equal "pip", dep.record["type"] - assert dep.record["homepage"] - assert dep.record["summary"] - end - end - - it "detects dependencies with dots in package name" do + it "detects transitive dependencies" do Dir.chdir fixtures do - dep = source.dependencies.detect { |d| d.name == "backports.shutil-get-terminal-size" } + dep = source.dependencies.detect { |d| d.name == "google-auth-oauthlib" } assert dep assert_equal "pip", dep.record["type"] assert dep.record["homepage"] From 00120edf044e10f4e9c3c1180e8ce9e9499f5566 Mon Sep 17 00:00:00 2001 From: licensed-ci Date: Tue, 15 Mar 2022 21:20:29 +0000 Subject: [PATCH 2/3] Auto-update license files --- .licenses/bundler/bundler.dep.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.licenses/bundler/bundler.dep.yml b/.licenses/bundler/bundler.dep.yml index 1301be0e..f289f4bf 100644 --- a/.licenses/bundler/bundler.dep.yml +++ b/.licenses/bundler/bundler.dep.yml @@ -1,6 +1,6 @@ --- name: bundler -version: 2.3.8 +version: 2.3.9 type: bundler summary: The best way to manage your application's dependencies homepage: https://bundler.io From ae0db440cdb4d52e1d9cac677933c7866c71811e Mon Sep 17 00:00:00 2001 From: Jon Ruskin Date: Tue, 15 Mar 2022 14:59:39 -0700 Subject: [PATCH 3/3] use different transitive dependency it looks like the one I used before might have been a dependency of an older pip version that's no longer present in the version used in CI --- test/sources/pip_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sources/pip_test.rb b/test/sources/pip_test.rb index a7bc3113..699b4daa 100644 --- a/test/sources/pip_test.rb +++ b/test/sources/pip_test.rb @@ -38,7 +38,7 @@ it "detects transitive dependencies" do Dir.chdir fixtures do - dep = source.dependencies.detect { |d| d.name == "google-auth-oauthlib" } + dep = source.dependencies.detect { |d| d.name == "MarkupSafe" } assert dep assert_equal "pip", dep.record["type"] assert dep.record["homepage"]