Bunch of cleanups

- Use the egg-info providing function on
  archives and directories instead of having
  to find the name and version manually.
- Style cleanups.
- Remove dir/file before after context manager.
- Use the tracewriter so files/directories created
  get automatically deleted on uninstall/remove.

Change-Id: Ib6208cf1e83a34ac49fb1384f8f80b6e71879aff

anvil/actions/prepare.py

@@ -65,7 +65,7 @@ class PrepareAction(action.Action):
         dependency_handler.package_start()
         self._run_phase(
             action.PhaseFunctors(
-                start=lambda i: LOG.info("Packing %s", colorizer.quote(i.name)),
+                start=lambda i: LOG.info("Packaging %s.", colorizer.quote(i.name)),
                 run=dependency_handler.package_instance,
                 end=None,
             ),

anvil/packaging/helpers/pip_helper.py

@@ -28,7 +28,7 @@ from anvil import utils
 LOG = logging.getLogger(__name__)
 
 FREEZE_CMD = ['freeze', '--local']
-FILES_DETAILED = {}
+EGGS_DETAILED = {}
 
 
 def create_requirement(name, version=None):
@@ -48,36 +48,58 @@ def create_requirement(name, version=None):
     return pkg_resources.Requirement.parse(name)
 
 
+def get_directory_details(path):
+    if not sh.isdir(path):
+        raise IOError("Can not detail non-existent directory %s" % (path))
+
+    # Check if we already got the details of this dir previously
+    path = sh.abspth(path)
+    cache_key = "d:%s" % (sh.abspth(path))
+    if cache_key in EGGS_DETAILED:
+        return EGGS_DETAILED[cache_key]
+
+    req = pip_req.InstallRequirement.from_line(path)
+    req.source_dir = path
+    req.run_egg_info()
+
+    dependencies = []
+    for d in req.requirements():
+        if not d.startswith("-e") and d.find("#"):
+            d = d.split("#")[0]
+        d = d.strip()
+        if d:
+            dependencies.append(d)
+
+    details = {
+        'req': req.req,
+        'dependencies': dependencies,
+        'name': req.name,
+        'pkg_info': req.pkg_info(),
+        'dependency_links': req.dependency_links,
+        'version': req.installed_version,
+    }
+
+    EGGS_DETAILED[cache_key] = details
+    return details
+
+
 def get_archive_details(filename):
-    if not os.path.isfile(filename):
+    if not sh.isfile(filename):
         raise IOError("Can not detail non-existent file %s" % (filename))
 
     # Check if we already got the details of this file previously
-    #
+    cache_key = "f:%s:%s" % (sh.basename(filename), sh.getsize(filename))
-    # A filename and size cache key should be good enough to match against
+    if cache_key in EGGS_DETAILED:
-    # most non-malicous files...
+        return EGGS_DETAILED[cache_key]
-    cache_key = "%s:%s" % (sh.basename(filename), sh.getsize(filename))
-    if cache_key in FILES_DETAILED:
-        return FILES_DETAILED[cache_key]
 
     # Get pip to get us the egg-info.
     with utils.tempdir() as td:
         filename = sh.copy(filename, sh.joinpths(td, sh.basename(filename)))
         extract_to = sh.mkdir(sh.joinpths(td, 'build'))
         pip_util.unpack_file(filename, extract_to, content_type='', link='')
-        req = pip_req.InstallRequirement.from_line(extract_to)
+        details = get_directory_details(extract_to)
-        req.source_dir = extract_to
-        req.run_egg_info()
-        # Selectively extract pieces of the install requirement
-        details = {
-            'req': req.req,
-            'dependencies': req.requirements,
-            'name': req.name,
-            'pkg_info': req.pkg_info,
-            'dependency_links': req.dependency_links,
-        }
 
-    FILES_DETAILED[cache_key] = details
+    EGGS_DETAILED[cache_key] = details
     return details
 
 

anvil/packaging/yum.py

@@ -104,22 +104,19 @@ class YumDependencyHandler(base.DependencyHandler):
         return cmdline
 
     def package_instance(self, instance):
-        # clear before...
+        with sh.remove_before_after(self.rpmbuild_dir):
-        sh.deldir(self.rpmbuild_dir)
+            for dirname in (sh.joinpths(self.rpmbuild_dir, "SPECS"),
-        for dirname in (sh.joinpths(self.rpmbuild_dir, "SPECS"),
+                            sh.joinpths(self.rpmbuild_dir, "SOURCES")):
-                        sh.joinpths(self.rpmbuild_dir, "SOURCES")):
+                sh.mkdirslist(dirname, tracewriter=self.tracewriter)
-            sh.mkdir(dirname, recurse=True)
+            if instance.name == "general":
-        if instance.name == "general":
+                self._build_dependencies()
-            self._build_dependencies()
+                self._move_rpms("anvil-deps")
-            self._move_rpms("anvil-deps")
+                self._create_repo("anvil-deps")
-            self._create_repo("anvil-deps")
+            else:
-        else:
+                app_dir = instance.get_option("app_dir")
-            app_dir = instance.get_option("app_dir")
+                if sh.isdir(app_dir):
-            if sh.isdir(app_dir):
+                    self._build_openstack_package(app_dir)
-                self._build_openstack_package(app_dir)
+                    self._move_rpms("anvil")
-                self._move_rpms("anvil")
-        # ...and after
-        sh.deldir(self.rpmbuild_dir)
 
     def package_finish(self):
         self._create_repo("anvil")
@@ -127,33 +124,38 @@ class YumDependencyHandler(base.DependencyHandler):
     def _move_rpms(self, repo_name):
         repo_dir = sh.joinpths(self.anvil_repo_dir, repo_name)
         src_repo_dir = "%s-sources" % repo_dir
-        sh.mkdir(repo_dir, recurse=True)
+        for dirname in (repo_dir, src_repo_dir):
-        sh.mkdir(src_repo_dir, recurse=True)
+            sh.mkdirslist(dirname, tracewriter=self.tracewriter)
-        for filename in sh.listdir(sh.joinpths(self.rpmbuild_dir, "RPMS"),
+        for (src_dir, tgt_dir) in ([sh.joinpths(self.rpmbuild_dir, "RPMS"), repo_dir],
-                                   recursive=True, files_only=True):
+                                   [sh.joinpths(self.rpmbuild_dir, "SRPMS"), src_repo_dir]):
-            sh.move(filename, repo_dir, force=True)
+            rpms = []
-        for filename in sh.listdir(sh.joinpths(self.rpmbuild_dir, "SRPMS"),
+            for filename in sh.listdir(src_dir, recursive=True, files_only=True):
-                                   recursive=True, files_only=True):
+                sh.move(filename, tgt_dir, force=True)
-            sh.move(filename, src_repo_dir, force=True)
+                rpms.append(sh.basename(filename))
-        return repo_dir
+            if rpms:
+                utils.log_iterable(rpms,
+                                   header="Moved %s rpms to %s" % (len(rpms),
+                                                                   tgt_dir),
+                                   logger=LOG)
 
     def _create_repo(self, repo_name):
         repo_dir = sh.joinpths(self.anvil_repo_dir, repo_name)
         src_repo_dir = "%s-sources" % repo_dir
-        for a_dir in repo_dir, src_repo_dir:
+        for a_dir in (repo_dir, src_repo_dir):
             cmdline = ["createrepo", a_dir]
-            LOG.info("Creating repo at %s" % a_dir)
+            LOG.info("Creating repo at %s", a_dir)
             sh.execute(cmdline)
         repo_filename = sh.joinpths(self.anvil_repo_dir, "%s.repo" % repo_name)
-        LOG.info("Writing %s" % repo_filename)
+        LOG.info("Writing %s", repo_filename)
         (_fn, content) = utils.load_template("packaging", "common.repo")
         params = {
             "repo_name": repo_name,
             "baseurl_bin": "file://%s" % repo_dir,
             "baseurl_src": "file://%s" % src_repo_dir
         }
-        sh.write_file(
+        sh.write_file(repo_filename,
-            repo_filename, utils.expand_template(content, params))
+                      utils.expand_template(content, params),
+                      tracewriter=self.tracewriter)
 
     def _get_yum_available(self):
         yum_map = {}
@@ -271,45 +273,16 @@ class YumDependencyHandler(base.DependencyHandler):
                                        quiet=True)
                 p_bar.update(i + 1)
 
-    @staticmethod
-    def _python_setup_py_get(pkg_dir, field):
-        """
-        :param field: e.g., "name" or "version"
-        """
-        cmdline = [sys.executable, "setup.py", "--%s" % field]
-        value = sh.execute(cmdline, cwd=pkg_dir)[0].splitlines()[-1].strip()
-        if not value:
-            LOG.error("Cannot determine %s for %s", field, pkg_dir)
-        return value
-
     def _write_spec_file(self, pkg_dir, rpm_name, template_name, params):
-
+        egg_details = pip_helper.get_directory_details(pkg_dir)
-        def load_requirements(filename):
+        requires_what = egg_details['dependencies']
-            if not sh.isfile(filename):
+        params['requires'] = self._convert_names_python2rpm(requires_what)
-                return []
-            requires = []
-            with open(filename, "r") as requires_file:
-                for line in requires_file.readlines():
-                    line = line.split("#", 1)[0].strip()
-                    if line:
-                        requires.append(line)
-            return requires
-
-        if not params.setdefault("requires", []):
-            # TODO(harlowja): get from egg-info???
-            requires_what = []
-            for filename in ["%s/tools/pip-requires" % pkg_dir,
-                             "%s/requirements.txt" % pkg_dir]:
-                requires_what.extend(load_requirements(filename))
-            requires_what = self._convert_names_python2rpm(requires_what)
-            if requires_what:
-                params["requires"] = requires_what
-
         params["epoch"] = self.OPENSTACK_EPOCH
         content = utils.load_template(self.SPEC_TEMPLATE_DIR, template_name)[1]
         spec_filename = sh.joinpths(self.rpmbuild_dir, "SPECS",
                                     "%s.spec" % rpm_name)
-        sh.write_file(spec_filename, utils.expand_template(content, params),
+        sh.write_file(spec_filename,
+                      utils.expand_template(content, params),
                       tracewriter=self.tracewriter)
         return spec_filename
 
@@ -396,8 +369,9 @@ class YumDependencyHandler(base.DependencyHandler):
         rpm_name = None
         template_name = None
         if sh.isfile(sh.joinpths(pkg_dir, "setup.py")):
-            name = self._python_setup_py_get(pkg_dir, "name")
+            egg_info = pip_helper.get_directory_details(pkg_dir)
-            params["version"] = self._python_setup_py_get(pkg_dir, "version")
+            name = egg_info['name']
+            params["version"] = egg_info["version"]
             if component_name.endswith("client"):
                 clientname = utils.strip_prefix_suffix(name,
                                                        "python-", "client")

anvil/shell.py

@@ -17,6 +17,7 @@
 # R0915: Too many statements
 # pylint: disable=R0915
 
+import contextlib
 import distutils.spawn
 import getpass
 import grp
@@ -197,6 +198,22 @@ def execute_save_output(cmd, out_filename, **kwargs):
         execute(cmd, **kwargs)
 
 
+@contextlib.contextmanager
+def remove_before_after(path):
+
+    def delete_it(path):
+        if isdir(path):
+            deldir(path)
+        if isfile(path):
+            unlink(path)
+
+    delete_it(path)
+    try:
+        yield path
+    finally:
+        delete_it(path)
+
+
 def abspth(path):
     if not path:
         path = "/"