From 74ca7202fdf22b15558463d41f6e694718e09175 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 18 Jun 2013 12:20:00 -0700 Subject: [PATCH] Bunch of cleanups - Use the egg-info providing function on archives and directories instead of having to find the name and version manually. - Style cleanups. - Remove dir/file before after context manager. - Use the tracewriter so files/directories created get automatically deleted on uninstall/remove. Change-Id: Ib6208cf1e83a34ac49fb1384f8f80b6e71879aff --- anvil/actions/prepare.py | 2 +- anvil/packaging/helpers/pip_helper.py | 62 ++++++++++----- anvil/packaging/yum.py | 106 ++++++++++---------------- anvil/shell.py | 17 +++++ 4 files changed, 100 insertions(+), 87 deletions(-) diff --git a/anvil/actions/prepare.py b/anvil/actions/prepare.py index d558ef1d..fa0bad1b 100644 --- a/anvil/actions/prepare.py +++ b/anvil/actions/prepare.py @@ -65,7 +65,7 @@ class PrepareAction(action.Action): dependency_handler.package_start() self._run_phase( action.PhaseFunctors( - start=lambda i: LOG.info("Packing %s", colorizer.quote(i.name)), + start=lambda i: LOG.info("Packaging %s.", colorizer.quote(i.name)), run=dependency_handler.package_instance, end=None, ), diff --git a/anvil/packaging/helpers/pip_helper.py b/anvil/packaging/helpers/pip_helper.py index 3c44521d..b93a1cf9 100644 --- a/anvil/packaging/helpers/pip_helper.py +++ b/anvil/packaging/helpers/pip_helper.py @@ -28,7 +28,7 @@ from anvil import utils LOG = logging.getLogger(__name__) FREEZE_CMD = ['freeze', '--local'] -FILES_DETAILED = {} +EGGS_DETAILED = {} def create_requirement(name, version=None): @@ -48,36 +48,58 @@ def create_requirement(name, version=None): return pkg_resources.Requirement.parse(name) +def get_directory_details(path): + if not sh.isdir(path): + raise IOError("Can not detail non-existent directory %s" % (path)) + + # Check if we already got the details of this dir previously + path = sh.abspth(path) + cache_key = "d:%s" % (sh.abspth(path)) + if cache_key in EGGS_DETAILED: + return EGGS_DETAILED[cache_key] + + req = pip_req.InstallRequirement.from_line(path) + req.source_dir = path + req.run_egg_info() + + dependencies = [] + for d in req.requirements(): + if not d.startswith("-e") and d.find("#"): + d = d.split("#")[0] + d = d.strip() + if d: + dependencies.append(d) + + details = { + 'req': req.req, + 'dependencies': dependencies, + 'name': req.name, + 'pkg_info': req.pkg_info(), + 'dependency_links': req.dependency_links, + 'version': req.installed_version, + } + + EGGS_DETAILED[cache_key] = details + return details + + def get_archive_details(filename): - if not os.path.isfile(filename): + if not sh.isfile(filename): raise IOError("Can not detail non-existent file %s" % (filename)) # Check if we already got the details of this file previously - # - # A filename and size cache key should be good enough to match against - # most non-malicous files... - cache_key = "%s:%s" % (sh.basename(filename), sh.getsize(filename)) - if cache_key in FILES_DETAILED: - return FILES_DETAILED[cache_key] + cache_key = "f:%s:%s" % (sh.basename(filename), sh.getsize(filename)) + if cache_key in EGGS_DETAILED: + return EGGS_DETAILED[cache_key] # Get pip to get us the egg-info. with utils.tempdir() as td: filename = sh.copy(filename, sh.joinpths(td, sh.basename(filename))) extract_to = sh.mkdir(sh.joinpths(td, 'build')) pip_util.unpack_file(filename, extract_to, content_type='', link='') - req = pip_req.InstallRequirement.from_line(extract_to) - req.source_dir = extract_to - req.run_egg_info() - # Selectively extract pieces of the install requirement - details = { - 'req': req.req, - 'dependencies': req.requirements, - 'name': req.name, - 'pkg_info': req.pkg_info, - 'dependency_links': req.dependency_links, - } + details = get_directory_details(extract_to) - FILES_DETAILED[cache_key] = details + EGGS_DETAILED[cache_key] = details return details diff --git a/anvil/packaging/yum.py b/anvil/packaging/yum.py index 15e7d5b2..d133bd62 100644 --- a/anvil/packaging/yum.py +++ b/anvil/packaging/yum.py @@ -104,22 +104,19 @@ class YumDependencyHandler(base.DependencyHandler): return cmdline def package_instance(self, instance): - # clear before... - sh.deldir(self.rpmbuild_dir) - for dirname in (sh.joinpths(self.rpmbuild_dir, "SPECS"), - sh.joinpths(self.rpmbuild_dir, "SOURCES")): - sh.mkdir(dirname, recurse=True) - if instance.name == "general": - self._build_dependencies() - self._move_rpms("anvil-deps") - self._create_repo("anvil-deps") - else: - app_dir = instance.get_option("app_dir") - if sh.isdir(app_dir): - self._build_openstack_package(app_dir) - self._move_rpms("anvil") - # ...and after - sh.deldir(self.rpmbuild_dir) + with sh.remove_before_after(self.rpmbuild_dir): + for dirname in (sh.joinpths(self.rpmbuild_dir, "SPECS"), + sh.joinpths(self.rpmbuild_dir, "SOURCES")): + sh.mkdirslist(dirname, tracewriter=self.tracewriter) + if instance.name == "general": + self._build_dependencies() + self._move_rpms("anvil-deps") + self._create_repo("anvil-deps") + else: + app_dir = instance.get_option("app_dir") + if sh.isdir(app_dir): + self._build_openstack_package(app_dir) + self._move_rpms("anvil") def package_finish(self): self._create_repo("anvil") @@ -127,33 +124,38 @@ class YumDependencyHandler(base.DependencyHandler): def _move_rpms(self, repo_name): repo_dir = sh.joinpths(self.anvil_repo_dir, repo_name) src_repo_dir = "%s-sources" % repo_dir - sh.mkdir(repo_dir, recurse=True) - sh.mkdir(src_repo_dir, recurse=True) - for filename in sh.listdir(sh.joinpths(self.rpmbuild_dir, "RPMS"), - recursive=True, files_only=True): - sh.move(filename, repo_dir, force=True) - for filename in sh.listdir(sh.joinpths(self.rpmbuild_dir, "SRPMS"), - recursive=True, files_only=True): - sh.move(filename, src_repo_dir, force=True) - return repo_dir + for dirname in (repo_dir, src_repo_dir): + sh.mkdirslist(dirname, tracewriter=self.tracewriter) + for (src_dir, tgt_dir) in ([sh.joinpths(self.rpmbuild_dir, "RPMS"), repo_dir], + [sh.joinpths(self.rpmbuild_dir, "SRPMS"), src_repo_dir]): + rpms = [] + for filename in sh.listdir(src_dir, recursive=True, files_only=True): + sh.move(filename, tgt_dir, force=True) + rpms.append(sh.basename(filename)) + if rpms: + utils.log_iterable(rpms, + header="Moved %s rpms to %s" % (len(rpms), + tgt_dir), + logger=LOG) def _create_repo(self, repo_name): repo_dir = sh.joinpths(self.anvil_repo_dir, repo_name) src_repo_dir = "%s-sources" % repo_dir - for a_dir in repo_dir, src_repo_dir: + for a_dir in (repo_dir, src_repo_dir): cmdline = ["createrepo", a_dir] - LOG.info("Creating repo at %s" % a_dir) + LOG.info("Creating repo at %s", a_dir) sh.execute(cmdline) repo_filename = sh.joinpths(self.anvil_repo_dir, "%s.repo" % repo_name) - LOG.info("Writing %s" % repo_filename) + LOG.info("Writing %s", repo_filename) (_fn, content) = utils.load_template("packaging", "common.repo") params = { "repo_name": repo_name, "baseurl_bin": "file://%s" % repo_dir, "baseurl_src": "file://%s" % src_repo_dir } - sh.write_file( - repo_filename, utils.expand_template(content, params)) + sh.write_file(repo_filename, + utils.expand_template(content, params), + tracewriter=self.tracewriter) def _get_yum_available(self): yum_map = {} @@ -271,45 +273,16 @@ class YumDependencyHandler(base.DependencyHandler): quiet=True) p_bar.update(i + 1) - @staticmethod - def _python_setup_py_get(pkg_dir, field): - """ - :param field: e.g., "name" or "version" - """ - cmdline = [sys.executable, "setup.py", "--%s" % field] - value = sh.execute(cmdline, cwd=pkg_dir)[0].splitlines()[-1].strip() - if not value: - LOG.error("Cannot determine %s for %s", field, pkg_dir) - return value - def _write_spec_file(self, pkg_dir, rpm_name, template_name, params): - - def load_requirements(filename): - if not sh.isfile(filename): - return [] - requires = [] - with open(filename, "r") as requires_file: - for line in requires_file.readlines(): - line = line.split("#", 1)[0].strip() - if line: - requires.append(line) - return requires - - if not params.setdefault("requires", []): - # TODO(harlowja): get from egg-info??? - requires_what = [] - for filename in ["%s/tools/pip-requires" % pkg_dir, - "%s/requirements.txt" % pkg_dir]: - requires_what.extend(load_requirements(filename)) - requires_what = self._convert_names_python2rpm(requires_what) - if requires_what: - params["requires"] = requires_what - + egg_details = pip_helper.get_directory_details(pkg_dir) + requires_what = egg_details['dependencies'] + params['requires'] = self._convert_names_python2rpm(requires_what) params["epoch"] = self.OPENSTACK_EPOCH content = utils.load_template(self.SPEC_TEMPLATE_DIR, template_name)[1] spec_filename = sh.joinpths(self.rpmbuild_dir, "SPECS", "%s.spec" % rpm_name) - sh.write_file(spec_filename, utils.expand_template(content, params), + sh.write_file(spec_filename, + utils.expand_template(content, params), tracewriter=self.tracewriter) return spec_filename @@ -396,8 +369,9 @@ class YumDependencyHandler(base.DependencyHandler): rpm_name = None template_name = None if sh.isfile(sh.joinpths(pkg_dir, "setup.py")): - name = self._python_setup_py_get(pkg_dir, "name") - params["version"] = self._python_setup_py_get(pkg_dir, "version") + egg_info = pip_helper.get_directory_details(pkg_dir) + name = egg_info['name'] + params["version"] = egg_info["version"] if component_name.endswith("client"): clientname = utils.strip_prefix_suffix(name, "python-", "client") diff --git a/anvil/shell.py b/anvil/shell.py index fc9ab082..d860aace 100644 --- a/anvil/shell.py +++ b/anvil/shell.py @@ -17,6 +17,7 @@ # R0915: Too many statements # pylint: disable=R0915 +import contextlib import distutils.spawn import getpass import grp @@ -197,6 +198,22 @@ def execute_save_output(cmd, out_filename, **kwargs): execute(cmd, **kwargs) +@contextlib.contextmanager +def remove_before_after(path): + + def delete_it(path): + if isdir(path): + deldir(path) + if isfile(path): + unlink(path) + + delete_it(path) + try: + yield path + finally: + delete_it(path) + + def abspth(path): if not path: path = "/"