(PUP-6120) Pip package provider compatibility with CLI (#4832)

* (PUP-6120) Pip package provider compatibility with CLI

Reduce dependencies to only require pip installed.

This results in:

Removed complexity:

    Poll from PyPI API directly
    Special case proxies

Added functionality:

    pip can now properly manage pip package from pip repository

lib/puppet/provider/package/pip.rb

@@ -30,21 +30,43 @@ def self.parse(line)
   # that's managed by `pip` or an empty array if `pip` is not available.
   def self.instances
     packages = []
-    pip_cmd = cmd.map { |c| which(c) }.find { |c| c != nil }
+    pip_cmd = self.pip_cmd
     return [] unless pip_cmd
     execpipe "#{pip_cmd} freeze" do |process|
       process.collect do |line|
         next unless options = parse(line)
         packages << new(options)
       end
     end
+
+    # Pip can also upgrade pip, but it's not listed in freeze so need to special case it
+    # Pip list would also show pip installed version, but "pip list" doesn't exist for older versions of pip (E.G v1.0)
+    if version = self.pip_version
+      packages << new({:ensure => version, :name => File.basename(pip_cmd), :provider => name})
+    end
+
     packages
   end
 
   def self.cmd
     ["pip", "pip-python"]
   end
 
+  def self.pip_cmd
+    self.cmd.map { |c| which(c) }.find { |c| c != nil }
+  end
+
+  def self.pip_version
+    pip_cmd = self.pip_cmd
+    return nil unless pip_cmd
+
+    execpipe [pip_cmd, '--version'] do |process|
+      process.collect do |line|
+        return line.strip.match(/^pip (\d+\.\d+\.?\d*).*$/)[1]
+      end
+    end
+  end
+
   # Return structured information about a particular package or `nil` if
   # it is not installed or `pip` itself is not available.
   def query
@@ -54,26 +76,13 @@ def query
     return nil
   end
 
-  # Ask the PyPI API for the latest version number.  There is no local
-  # cache of PyPI's package list so this operation will always have to
-  # ask the web service.
+  # Use pip CLI to look up versions from PyPI repositories, honoring local pip config such as custom repositories
   def latest
-    http_proxy_host = Puppet::Util::HttpProxy.http_proxy_host
-    http_proxy_port = Puppet::Util::HttpProxy.http_proxy_port
-    if http_proxy_host && http_proxy_port
-      proxy = "#{http_proxy_host}:#{http_proxy_port}"
-    else
-      # nil is acceptable
-      proxy = http_proxy_host
+    return nil unless self.class.pip_cmd
+    if Puppet::Util::Package.versioncmp(self.class.pip_version, '1.5.4') == -1 # a < b
+      return latest_with_old_pip
     end
-
-    client = XMLRPC::Client.new2("http://pypi.python.org/pypi", proxy)
-    client.http_header_extra = {"Content-Type" => "text/xml"}
-    client.timeout = 10
-    result = client.call("package_releases", @resource[:name])
-    result.first
-  rescue Timeout::Error => detail
-    raise Puppet::Error, "Timeout while contacting pypi.python.org: #{detail}", detail.backtrace
+    latest_with_new_pip
   end
 
   # Install a package.  The ensure parameter may specify installed,
@@ -120,8 +129,14 @@ def update
   def lazy_pip(*args)
     pip *args
   rescue NoMethodError => e
+    # Ensure pip can upgrade pip, which usually puts pip into a new path /usr/local/bin/pip (compared to /usr/bin/pip)
+    # The path to pip needs to be looked up again in the subsequent request. Using the preferred approach as noted
+    # in provider.rb ensures this (copied below for reference)
+    #
+    # @note From provider.rb; It is preferred if the commands are not entered with absolute paths as this allows puppet
+    # to search for them using the PATH variable.
     if pathname = self.class.cmd.map { |c| which(c) }.find { |c| c != nil }
-      self.class.commands :pip => pathname
+      self.class.commands :pip => File.basename(pathname)
       pip *args
     else
       raise e, "Could not locate command #{self.class.cmd.join(' and ')}.", e.backtrace
@@ -131,4 +146,33 @@ def lazy_pip(*args)
   def install_options
     join_options(@resource[:install_options])
   end
+
+  def latest_with_new_pip
+    # Less resource intensive approach for pip version 1.5.4 and above
+    execpipe ["#{self.class.pip_cmd}", "install", "#{@resource[:name]}==versionplease"] do |process|
+      process.collect do |line|
+        # PIP OUTPUT: Could not find a version that satisfies the requirement Django==versionplease (from versions: 1.1.3, 1.8rc1)
+        if line =~ /from versions: /
+          textAfterLastMatch = $'
+          versionList = textAfterLastMatch.chomp(")\n").split(', ')
+          return versionList.last
+        end
+      end
+      return nil
+    end
+  end
+
+  def latest_with_old_pip
+    Dir.mktmpdir("puppet_pip") do |dir|
+      execpipe ["#{self.class.pip_cmd}", "install", "#{@resource[:name]}", "-d", "#{dir}", "-v"] do |process|
+        process.collect do |line|
+          # PIP OUTPUT: Using version 0.10.1 (newest of versions: 0.10.1, 0.10, 0.9, 0.8.1, 0.8, 0.7.2, 0.7.1, 0.7, 0.6.1, 0.6, 0.5.2, 0.5.1, 0.5, 0.4, 0.3.1, 0.3, 0.2, 0.1)
+          if line =~ /Using version (.+?) \(newest of versions/
+            return $1
+          end
+        end
+        return nil
+      end
+    end
+  end
 end