Use python to parse python requires syntax and evaluate markers.

Not supported yet:
* Custom environment for marker evaluation.
* "extras" markers

Restore function to `--python-obey-requirements-txt`. This flag will
only work on python inputs that are provided as a directory, not a
wheel/tar.gz/etc.

Tests pass including the obey requirements.txt ones.

* Fix bug 'else File.directory?` -> `elsif File.directory?...` lol oops.
* Use Dir.entries instead of Dir.glob to allow python package extra
  syntax, with names like `django[bcrypt]` (Note, extras in requirements
  aren't evaluated yet)

Author: jordansissel

lib/fpm/package/pyfpm/parse_requires.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+
+#import pkg_resources
+import packaging.requirements
+import json
+import sys
+
+# Expect requirements lines via stdin.
+#requirements = pkg_resources.parse_requirements(sys.stdin)
+
+# Process environment markers, if any, and produce a list of requirements for the current environment.
+def evaluate_requirements(fd):
+    all_requirements = [packaging.requirements.Requirement(line) for line in sys.stdin]
+    
+    for req in all_requirements:
+        # XXX: Note: marker.evaluate() can be given a dict() containing environment values to overwrite
+        if req.marker is None or req.marker.evaluate():
+            if len(req.specifier) > 0:
+                for spec in req.specifier:
+                    yield "%s%s" % (req.name, spec)
+            else:
+                yield str(req.name)
+
+print(json.dumps(list(evaluate_requirements(sys.stdin))))

lib/fpm/package/python.rb

@@ -5,6 +5,7 @@
 require "rubygems"
 require "fileutils"
 require "tmpdir"
+require "json"
 
 # Support for python packages.
 #
@@ -324,9 +325,9 @@ def process_maintainer(headers)
   # * The path to a python sdist file ending in .tar.gz
   # * The path to a python wheel file ending in .whl
   def input(package)
-    if attributes[:python_obey_requirements_txt?]
-      raise "--python-obey-requirements-txt is temporarily unsupported at this time."
-    end
+    #if attributes[:python_obey_requirements_txt?]
+      #raise "--python-obey-requirements-txt is temporarily unsupported at this time."
+    #end
     explore_environment
 
     path_to_package = download_if_necessary(package, version)
@@ -337,6 +338,10 @@ def input(package)
         logger.error("The path doesn't appear to be a python package directory. I expected either a pypackage.toml or setup.py but found neither.", :package => package)
         raise "Unable to find python package; tried #{setup_py}"
       end
+
+      if attributes[:python_obey_requirements_txt?] && File.exist?(File.join(path_to_package, "requirements.txt"))
+        @requirements_txt = File.read(File.join(path_to_package, "requirements.txt"))
+      end
     end
 
     if File.file?(path_to_package)
@@ -355,7 +360,7 @@ def input(package)
         log.error("Failed building python package wheel format. This might be a bug in fpm.")
         raise "Failed building python package format."
       end
-    else File.directory?(path_to_package)
+    elsif File.directory?(path_to_package)
       logger.debug("Found directory and assuming it's a python source package.")
       safesystem(*attributes[:python_pip], "wheel", "--no-deps", "-w", build_path, path_to_package)
 
@@ -454,11 +459,12 @@ def download_if_necessary(package, version=nil)
 
       safesystem(*setup_cmd)
 
-      files = ::Dir.glob(File.join(target, "*.{whl,tar.gz,zip}"))
+      #files = ::Dir.glob(File.join(target, "*.{whl,tar.gz,zip}"))
+      files = ::Dir.entries(target).filter { |entry| entry =~ /\.(whl|tar\.gz|zip)$/ }
       if files.length != 1
         raise "Unexpected directory layout after `pip download ...`. This might be an fpm bug? The directory contains these files: #{files.inspect}"
       end
-      return files.first
+      return File.join(target, files.first)
     else
       # no pip, use easy_install
       logger.debug("no pip, defaulting to easy_install", :easy_install => attributes[:python_easyinstall])
@@ -467,7 +473,8 @@ def download_if_necessary(package, version=nil)
                  "--build-directory", target, want_pkg)
       # easy_install will put stuff in @tmpdir/packagename/, so find that:
       #  @tmpdir/somepackage/setup.py
-      dirs = ::Dir.glob(File.join(target, "*"))
+      #dirs = ::Dir.glob(File.join(target, "*"))
+      files = ::Dir.entries(target).filter { |entry| entry != "." && entry != ".." }
       if dirs.length != 1
         raise "Unexpected directory layout after easy_install. Maybe file a bug? The directory is #{build_path}"
       end
@@ -511,52 +518,46 @@ def load_package_info(path)
     self.maintainer = metadata.maintainer 
 
     if !attributes[:no_auto_depends?] and attributes[:python_dependencies?]
-      sys_platform = nil
-      execmd([attributes[:python_bin], "-c", "import sys; print(sys.platform)"], :stdin => false, :stderr => false) do |stdout|
-        sys_platform = stdout.read.chomp
-      end
-
-      dep_re = /^([^<>!= ]+)\s*(?:([~<>!=]{1,2})\s*(.*))?$/
-
       # Python Dependency specifiers are a somewhat complex format described here:
       # https://packaging.python.org/en/latest/specifications/dependency-specifiers/#environment-markers
       #
-      # It would be ideal to support the entire specifier format, but it's unclear if that's necessary 
-      # for most packaging situations.
-      #
-      # If a specifier is found to not work under fpm, please file an issue on the fpm issue tracker
-      # and hopefully support for it can be added.
+      # We can ask python's packaging module to parse and evaluate these.
+      # XXX: Allow users to override environnment values.
       #
       # Example:
       # Requires-Dist: tzdata; sys_platform = win32
       # Requires-Dist: asgiref>=3.8.1
 
-      metadata.requires.each do |text|
-        dep, environment = text.split(/ *; */)
+      dep_re = /^([^<>!= ]+)\s*(?:([~<>!=]{1,2})\s*(.*))?$/
+
+      reqs = []
+
+      # --python-obey-requirements-txt should replace the requirments listed from the metadata
+      if attributes[:python_obey_requirements_txt?] && !@requirements_txt.nil?
+        requires = @requirements_txt.split("\n")
+      else
+        requires = metadata.requires
+      end
+
+      # Evaluate python package requirements and only show ones matching the current environment
+      # (Environment markers, etc)
+      # Additionally, 'extra' features such as a requirement named `django[bcrypt]` isn't quite supported yet,
+      # since the marker.evaluate() needs to be passed some environment like { "extra": "bcrypt" }
+      execmd([attributes[:python_bin], File.expand_path(File.join("pyfpm", "parse_requires.py"), File.dirname(__FILE__))]) do |stdin, stdout, stderr|
+        requires.each { |r| stdin.puts(r) }
+        stdin.close
+        data = stdout.read
+        logger.pipe(stderr => :warn)
+        reqs += JSON.parse(data)
+      end
+
+      reqs.each do |dep|
         match = dep_re.match(dep)
         if match.nil?
           logger.error("Unable to parse dependency", :dependency => dep)
           raise FPM::InvalidPackageConfiguration, "Invalid dependency '#{dep}'"
         end
 
-        if environment
-          if environment.include?("sys_platform ==") && !environment.include?("sys_platform == #{sys_platform}")
-            logger.debug("Ignoring dependency because it doesn't match the current platform", :current => sys_platform, :target => environment, :dependency => text)
-            next
-          end
-
-          if environment.include?("extra ==")
-            logger.debug("Ignoring extra/optional dependency", :dependency => text, :extra => environment)
-            next
-          end
-
-          unsupported_markers = UNSUPPORTED_DEPENDENCY_MARKERS.filter { |m| environment.include?(m) }
-          if unsupported_markers.any?
-            logger.debug("Package contains an unsupported Requires-Dist 'environment marker' which fpm doesn't yet support. If you want support for these, please file an issue.", :markers => unsupported_markers, :dependency => text)
-            next
-          end
-        end # if environment
-
         name, cmp, version = match.captures
 
         next if attributes[:python_disable_dependency].include?(name)
@@ -593,7 +594,7 @@ def fix_name(name)
     if name.start_with?("python")
       # If the python package is called "python-foo" strip the "python-" part while
       # prepending the package name prefix.
-      return [attributes[:python_package_name_prefix], name.gsub(/^python-/, "")].join("-")
+      return [attributes[:ptython_package_name_prefix], name.gsub(/^python-/, "")].join("-")
     else
       return [attributes[:python_package_name_prefix], name].join("-")
     end

spec/fixtures/python/setup.py

@@ -10,9 +10,6 @@
       package_dir={},
       install_requires=[
           "Dependency1", "dependency2",
-          # XXX: I don't know what these python_version-dependent deps mean
-          # needs investigation
-          # Reference: PEP-0508
            'rtxt-dep3; python_version == "2.0"',
            'rtxt-dep4; python_version > "2.0"',
            ],

spec/fpm/package/python_spec.rb

@@ -149,7 +149,7 @@ def easy_install_default(python_bin, option)
       #       not supported by fpm.
       #       In this test, there are (at time of writing) some python_version markers and fpm doesn't
       #       support those.
-      insist { subject.dependencies.sort } == ["#{prefix}-dependency1","#{prefix}-dependency2"]
+      insist { subject.dependencies.sort } == ["#{prefix}-dependency1","#{prefix}-dependency2", "#{prefix}-rtxt-dep4"]
     end
 
     context "and :python_disable_dependency is set" do
@@ -160,7 +160,7 @@ def easy_install_default(python_bin, option)
       it "it should exclude the dependency" do
         subject.input(example_dir)
         prefix = subject.attributes[:python_package_name_prefix]
-        insist { subject.dependencies.sort } == ["#{prefix}-dependency2"]
+        insist { subject.dependencies.sort } == ["#{prefix}-dependency2", "#{prefix}-rtxt-dep4"]
       end
     end
   end
@@ -179,13 +179,14 @@ def easy_install_default(python_bin, option)
       it "it should prefix requirements.txt" do
         subject.input(example_dir)
         prefix = subject.attributes[:python_package_name_prefix]
-        insist { subject.dependencies.sort } == ["#{prefix}-rtxt-dep3", "#{prefix}-rtxt-dep4"]
+        insist { subject.dependencies.sort } == ["#{prefix}-rtxt-dep1 > 0.1", "#{prefix}-rtxt-dep2 = 0.1", "#{prefix}-rtxt-dep4"]
       end
 
       it "it should exclude the dependency" do
         subject.attributes[:python_disable_dependency] = "rtxt-dep1"
         subject.input(example_dir)
-        insist { subject.dependencies.sort } == ["python-rtxt-dep2 = 0.1", "python-rtxt-dep4  "]
+        prefix = subject.attributes[:python_package_name_prefix]
+        insist { subject.dependencies.sort } == ["#{prefix}-rtxt-dep2 = 0.1", "#{prefix}-rtxt-dep4"]
       end
     end
 
@@ -196,21 +197,20 @@ def easy_install_default(python_bin, option)
 
       it "it should load requirements.txt" do
         subject.input(example_dir)
-        insist { subject.dependencies.sort } == ["rtxt-dep1 > 0.1", "rtxt-dep2 = 0.1", "rtxt-dep4  "]
+        insist { subject.dependencies.sort } == ["rtxt-dep1 > 0.1", "rtxt-dep2 = 0.1", "rtxt-dep4"]
       end
 
       it "it should exclude the dependency" do
         subject.attributes[:python_disable_dependency] = "rtxt-dep1"
         subject.input(example_dir)
-        insist { subject.dependencies.sort } == ["rtxt-dep2 = 0.1", "rtxt-dep4  "]
+        insist { subject.dependencies.sort } == ["rtxt-dep2 = 0.1", "rtxt-dep4"]
       end
     end
   end
 
   context "python_scripts_executable is set" do
     it "should have scripts with a custom hashbang line" do
-      pending("Disabled on travis-ci becaulamese it always fails, and there is no way to debug it?") if is_travis
-      skip("Requires python3 executable") unless program_exists?("python3")
+      skip("setup.py-specific feature is no longer supported")
 
       subject.attributes[:python_scripts_executable] = "fancypants"
       # Newer versions of Django require Python 3.
@@ -243,7 +243,7 @@ def easy_install_default(python_bin, option)
       insist { subject.version } == "8.3.0"
       insist { subject.maintainer } == "Pallets <contact@palletsprojects.com>"
       insist { subject.architecture } == "all"
-      insist { subject.depends }.include? "python3-colorama"
+      insist { subject.dependencies } == [ ]
 
     end
   end