Do not download Python packages again

Maintain a file indicating that all downloads are finished
and ensure that all requirements are satisfied.

Implements: blueprint pip-download-once
Change-Id: I0ddd39885d195da9a59bef2453be63990a784d32

anvil/packaging/base.py

@@ -18,10 +18,7 @@
 # R0921: Abstract class not referenced
 #pylint: disable=R0902,R0921
 
-import pkg_resources
-
 from anvil import colorizer
-from anvil import env
 from anvil import exceptions as exc
 from anvil import log as logging
 from anvil.packaging.helpers import pip_helper
@@ -87,6 +84,7 @@ class DependencyHandler(object):
         self.forced_requires_filename = sh.joinpths(
             self.deps_dir, "forced-requires")
         self.pip_executable = str(self.distro.get_command_config('pip'))
+        # list of requirement strings
         self.pips_to_install = []
         self.forced_packages = []
         self.package_dirs = self._get_package_dirs(instances)
@@ -244,6 +242,10 @@ class DependencyHandler(object):
                       "\n".join(str(req) for req in self.forced_packages))
 
     def filter_download_requires(self):
+        """
+        :returns: a list of all requirements that must be downloaded
+        :rtype: list of str
+        """
         return self.pips_to_install
 
     def _try_download_dependencies(self, attempt, pips_to_download,
@@ -259,7 +261,7 @@ class DependencyHandler(object):
             "--build", pip_build_dir,
         ]
         cmdline.extend(sorted(pips_to_download))
-        download_filename = "pip-download-attempt-%s.out"
+        download_filename = "pip-download-attempt-%s.log"
         download_filename = download_filename % (attempt)
         out_filename = sh.joinpths(self.log_dir, download_filename)
         sh.execute_save_output(cmdline, out_filename=out_filename)
@@ -276,13 +278,29 @@ class DependencyHandler(object):
                 LOG.info("Dependency %s was automatically included.",
                          colorizer.quote(req))
 
-    def download_dependencies(self, clear_cache=False):
+    @staticmethod
-        """Download dependencies from `$deps_dir/download-requires`.
+    def _requirements_satisfied(pips_list, download_dir):
+        downloaded_req = [
+            pip_helper.get_archive_details(filename)["req"]
+            for filename in sh.listdir(download_dir, files_only=True)]
+        downloaded_req = dict(
+            (req.key, req.specs[0][1])
+            for req in downloaded_req)
+        for req_str in pips_list:
+            req = pip_helper.extract_requirement(req_str)
+            try:
+                downloaded_version = downloaded_req[req.key]
+            except KeyError:
+                return False
+            else:
+                if downloaded_version not in req:
+                    return False
+        return True
 
-        :param clear_cache: clear `$deps_dir/cache` dir (pip can work incorrectly
+    def download_dependencies(self):
-            when it has a cache)
+        """Download dependencies from `$deps_dir/download-requires`.
         """
-        sh.deldir(self.download_dir)
+        # NOTE(aababilov): do not drop download_dir - it can be reused
         sh.mkdirslist(self.download_dir, tracewriter=self.tracewriter)
         download_requires_filename = sh.joinpths(self.deps_dir,
                                                  "download-requires")
@@ -291,41 +309,49 @@ class DependencyHandler(object):
                       "\n".join(str(req) for req in raw_pips_to_download))
         if not raw_pips_to_download:
             return ([], [])
-        pip_dir = sh.joinpths(self.deps_dir, "pip")
+        downloaded_flag_file = sh.joinpths(self.deps_dir, "pip-downloaded")
-        pip_download_dir = sh.joinpths(pip_dir, "download")
+        # NOTE(aababilov): user could have changed persona, so,
-        pip_build_dir = sh.joinpths(pip_dir, "build")
+        # check that all requirements are downloaded
-        pip_cache_dir = sh.joinpths(pip_dir, "cache")
+        if sh.isfile(downloaded_flag_file) and self._requirements_satisfied(
-        if clear_cache:
+                raw_pips_to_download, self.download_dir):
-            sh.deldir(pip_cache_dir)
+            LOG.info("All python dependencies have been already downloaded")
-        pip_failures = []
+        else:
-        for attempt in xrange(self.MAX_PIP_DOWNLOAD_ATTEMPTS):
+            pip_dir = sh.joinpths(self.deps_dir, "pip")
-            # NOTE(aababilov): pip has issues with already downloaded files
+            pip_download_dir = sh.joinpths(pip_dir, "download")
-            sh.deldir(pip_download_dir)
+            pip_build_dir = sh.joinpths(pip_dir, "build")
-            sh.mkdir(pip_download_dir, recurse=True)
+            # NOTE(aababilov): do not clean the cache, it is always useful
-            sh.deldir(pip_build_dir)
+            pip_cache_dir = sh.joinpths(self.deps_dir, "pip-cache")
-            header = "Downloading %s python dependencies (attempt %s)"
+            pip_failures = []
-            header = header % (len(raw_pips_to_download), attempt)
+            for attempt in xrange(self.MAX_PIP_DOWNLOAD_ATTEMPTS):
-            utils.log_iterable(sorted(raw_pips_to_download),
+                # NOTE(aababilov): pip has issues with already downloaded files
-                               logger=LOG,
+                sh.deldir(pip_dir)
-                               header=header)
+                sh.mkdir(pip_download_dir, recurse=True)
-            failed = False
+                header = "Downloading %s python dependencies (attempt %s)"
-            try:
+                header = header % (len(raw_pips_to_download), attempt)
-                self._try_download_dependencies(attempt, raw_pips_to_download,
+                utils.log_iterable(sorted(raw_pips_to_download),
-                                                pip_download_dir,
+                                   logger=LOG,
-                                                pip_cache_dir, pip_build_dir)
+                                   header=header)
-                pip_failures = []
+                failed = False
-            except exc.ProcessExecutionError as e:
+                try:
-                LOG.exception("Failed downloading python dependencies")
+                    self._try_download_dependencies(attempt, raw_pips_to_download,
-                pip_failures.append(e)
+                                                    pip_download_dir,
-                failed = True
+                                                    pip_cache_dir, pip_build_dir)
-            if not failed:
+                    pip_failures = []
-                break
+                except exc.ProcessExecutionError as e:
-        if pip_failures:
+                    LOG.exception("Failed downloading python dependencies")
-            raise pip_failures[-1]
+                    pip_failures.append(e)
+                    failed = True
+                if not failed:
+                    break
+            for filename in sh.listdir(pip_download_dir, files_only=True):
+                sh.move(filename, self.download_dir, force=True)
+            sh.deldir(pip_dir)
+            if pip_failures:
+                raise pip_failures[-1]
+            with open(downloaded_flag_file, "w"):
+                pass
         pips_downloaded = [pip_helper.extract_requirement(p)
                            for p in raw_pips_to_download]
-        self._examine_download_dir(pips_downloaded, pip_download_dir)
+        self._examine_download_dir(pips_downloaded, self.download_dir)
-        for filename in sh.listdir(pip_download_dir, files_only=True):
-            sh.move(filename, self.download_dir)
         what_downloaded = sh.listdir(self.download_dir, files_only=True)
         return (pips_downloaded, what_downloaded)