x/anvil
Code Issues Proposed changes

Bunch of cleanups

Browse Source 
- Use the egg-info providing function on
  archives and directories instead of having
  to find the name and version manually.
- Style cleanups.
- Remove dir/file before after context manager.
- Use the tracewriter so files/directories created
  get automatically deleted on uninstall/remove.

Change-Id: Ib6208cf1e83a34ac49fb1384f8f80b6e71879aff
This commit is contained in:
Joshua Harlow
2013-06-18 12:20:00 -07:00
parent ab958d95b0
commit 74ca7202fd
4 changed files with 100 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
anvil/actions/prepare.py
View File

@@ -65,7 +65,7 @@ class PrepareAction(action.Action):
dependency_handler.package_start()
self._run_phase(
action.PhaseFunctors(
start=lambda i: LOG.info("Packing %s", colorizer.quote(i.name)),
start=lambda i: LOG.info("Packaging %s.", colorizer.quote(i.name)),
run=dependency_handler.package_instance,
end=None,
),

62
anvil/packaging/helpers/pip_helper.py
View File

@@ -28,7 +28,7 @@ from anvil import utils
LOG = logging.getLogger(__name__)
FREEZE_CMD = ['freeze', '--local']
FILES_DETAILED = {}
EGGS_DETAILED = {}
def create_requirement(name, version=None):
@@ -48,36 +48,58 @@ def create_requirement(name, version=None):
return pkg_resources.Requirement.parse(name)
def get_directory_details(path):
if not sh.isdir(path):
raise IOError("Can not detail non-existent directory %s" % (path))
# Check if we already got the details of this dir previously
path = sh.abspth(path)
cache_key = "d:%s" % (sh.abspth(path))
if cache_key in EGGS_DETAILED:
return EGGS_DETAILED[cache_key]
req = pip_req.InstallRequirement.from_line(path)
req.source_dir = path
req.run_egg_info()
dependencies = []
for d in req.requirements():
if not d.startswith("-e") and d.find("#"):
d = d.split("#")[0]
d = d.strip()
if d:
dependencies.append(d)
details = {
'req': req.req,
'dependencies': dependencies,
'name': req.name,
'pkg_info': req.pkg_info(),
'dependency_links': req.dependency_links,
'version': req.installed_version,
}
EGGS_DETAILED[cache_key] = details
return details
def get_archive_details(filename):
if not os.path.isfile(filename):
if not sh.isfile(filename):
raise IOError("Can not detail non-existent file %s" % (filename))
# Check if we already got the details of this file previously
#
# A filename and size cache key should be good enough to match against
# most non-malicous files...
cache_key = "%s:%s" % (sh.basename(filename), sh.getsize(filename))
if cache_key in FILES_DETAILED:
return FILES_DETAILED[cache_key]
cache_key = "f:%s:%s" % (sh.basename(filename), sh.getsize(filename))
if cache_key in EGGS_DETAILED:
return EGGS_DETAILED[cache_key]
# Get pip to get us the egg-info.
with utils.tempdir() as td:
filename = sh.copy(filename, sh.joinpths(td, sh.basename(filename)))
extract_to = sh.mkdir(sh.joinpths(td, 'build'))
pip_util.unpack_file(filename, extract_to, content_type='', link='')
req = pip_req.InstallRequirement.from_line(extract_to)
req.source_dir = extract_to
req.run_egg_info()
# Selectively extract pieces of the install requirement
details = {
'req': req.req,
'dependencies': req.requirements,
'name': req.name,
'pkg_info': req.pkg_info,
'dependency_links': req.dependency_links,
}
details = get_directory_details(extract_to)
FILES_DETAILED[cache_key] = details
EGGS_DETAILED[cache_key] = details
return details

106
anvil/packaging/yum.py
View File

@@ -104,22 +104,19 @@ class YumDependencyHandler(base.DependencyHandler):
return cmdline
def package_instance(self, instance):
# clear before...
sh.deldir(self.rpmbuild_dir)
for dirname in (sh.joinpths(self.rpmbuild_dir, "SPECS"),
sh.joinpths(self.rpmbuild_dir, "SOURCES")):
sh.mkdir(dirname, recurse=True)
if instance.name == "general":
self._build_dependencies()
self._move_rpms("anvil-deps")
self._create_repo("anvil-deps")
else:
app_dir = instance.get_option("app_dir")
if sh.isdir(app_dir):
self._build_openstack_package(app_dir)
self._move_rpms("anvil")
# ...and after
sh.deldir(self.rpmbuild_dir)
with sh.remove_before_after(self.rpmbuild_dir):
for dirname in (sh.joinpths(self.rpmbuild_dir, "SPECS"),
sh.joinpths(self.rpmbuild_dir, "SOURCES")):
sh.mkdirslist(dirname, tracewriter=self.tracewriter)
if instance.name == "general":
self._build_dependencies()
self._move_rpms("anvil-deps")
self._create_repo("anvil-deps")
else:
app_dir = instance.get_option("app_dir")
if sh.isdir(app_dir):
self._build_openstack_package(app_dir)
self._move_rpms("anvil")
def package_finish(self):
self._create_repo("anvil")
@@ -127,33 +124,38 @@ class YumDependencyHandler(base.DependencyHandler):
def _move_rpms(self, repo_name):
repo_dir = sh.joinpths(self.anvil_repo_dir, repo_name)
src_repo_dir = "%s-sources" % repo_dir
sh.mkdir(repo_dir, recurse=True)
sh.mkdir(src_repo_dir, recurse=True)
for filename in sh.listdir(sh.joinpths(self.rpmbuild_dir, "RPMS"),
recursive=True, files_only=True):
sh.move(filename, repo_dir, force=True)
for filename in sh.listdir(sh.joinpths(self.rpmbuild_dir, "SRPMS"),
recursive=True, files_only=True):
sh.move(filename, src_repo_dir, force=True)
return repo_dir
for dirname in (repo_dir, src_repo_dir):
sh.mkdirslist(dirname, tracewriter=self.tracewriter)
for (src_dir, tgt_dir) in ([sh.joinpths(self.rpmbuild_dir, "RPMS"), repo_dir],
[sh.joinpths(self.rpmbuild_dir, "SRPMS"), src_repo_dir]):
rpms = []
for filename in sh.listdir(src_dir, recursive=True, files_only=True):
sh.move(filename, tgt_dir, force=True)
rpms.append(sh.basename(filename))
if rpms:
utils.log_iterable(rpms,
header="Moved %s rpms to %s" % (len(rpms),
tgt_dir),
logger=LOG)
def _create_repo(self, repo_name):
repo_dir = sh.joinpths(self.anvil_repo_dir, repo_name)
src_repo_dir = "%s-sources" % repo_dir
for a_dir in repo_dir, src_repo_dir:
for a_dir in (repo_dir, src_repo_dir):
cmdline = ["createrepo", a_dir]
LOG.info("Creating repo at %s" % a_dir)
LOG.info("Creating repo at %s", a_dir)
sh.execute(cmdline)
repo_filename = sh.joinpths(self.anvil_repo_dir, "%s.repo" % repo_name)
LOG.info("Writing %s" % repo_filename)
LOG.info("Writing %s", repo_filename)
(_fn, content) = utils.load_template("packaging", "common.repo")
params = {
"repo_name": repo_name,
"baseurl_bin": "file://%s" % repo_dir,
"baseurl_src": "file://%s" % src_repo_dir
}
sh.write_file(
repo_filename, utils.expand_template(content, params))
sh.write_file(repo_filename,
utils.expand_template(content, params),
tracewriter=self.tracewriter)
def _get_yum_available(self):
yum_map = {}
@@ -271,45 +273,16 @@ class YumDependencyHandler(base.DependencyHandler):
quiet=True)
p_bar.update(i + 1)
@staticmethod
def _python_setup_py_get(pkg_dir, field):
"""
:param field: e.g., "name" or "version"
"""
cmdline = [sys.executable, "setup.py", "--%s" % field]
value = sh.execute(cmdline, cwd=pkg_dir)[0].splitlines()[-1].strip()
if not value:
LOG.error("Cannot determine %s for %s", field, pkg_dir)
return value
def _write_spec_file(self, pkg_dir, rpm_name, template_name, params):
def load_requirements(filename):
if not sh.isfile(filename):
return []
requires = []
with open(filename, "r") as requires_file:
for line in requires_file.readlines():
line = line.split("#", 1)[0].strip()
if line:
requires.append(line)
return requires
if not params.setdefault("requires", []):
# TODO(harlowja): get from egg-info???
requires_what = []
for filename in ["%s/tools/pip-requires" % pkg_dir,
"%s/requirements.txt" % pkg_dir]:
requires_what.extend(load_requirements(filename))
requires_what = self._convert_names_python2rpm(requires_what)
if requires_what:
params["requires"] = requires_what
egg_details = pip_helper.get_directory_details(pkg_dir)
requires_what = egg_details['dependencies']
params['requires'] = self._convert_names_python2rpm(requires_what)
params["epoch"] = self.OPENSTACK_EPOCH
content = utils.load_template(self.SPEC_TEMPLATE_DIR, template_name)[1]
spec_filename = sh.joinpths(self.rpmbuild_dir, "SPECS",
"%s.spec" % rpm_name)
sh.write_file(spec_filename, utils.expand_template(content, params),
sh.write_file(spec_filename,
utils.expand_template(content, params),
tracewriter=self.tracewriter)
return spec_filename
@@ -396,8 +369,9 @@ class YumDependencyHandler(base.DependencyHandler):
rpm_name = None
template_name = None
if sh.isfile(sh.joinpths(pkg_dir, "setup.py")):
name = self._python_setup_py_get(pkg_dir, "name")
params["version"] = self._python_setup_py_get(pkg_dir, "version")
egg_info = pip_helper.get_directory_details(pkg_dir)
name = egg_info['name']
params["version"] = egg_info["version"]
if component_name.endswith("client"):
clientname = utils.strip_prefix_suffix(name,
"python-", "client")

17
anvil/shell.py
View File

@@ -17,6 +17,7 @@
# R0915: Too many statements
# pylint: disable=R0915
import contextlib
import distutils.spawn
import getpass
import grp
@@ -197,6 +198,22 @@ def execute_save_output(cmd, out_filename, **kwargs):
execute(cmd, **kwargs)
@contextlib.contextmanager
def remove_before_after(path):
def delete_it(path):
if isdir(path):
deldir(path)
if isfile(path):
unlink(path)
delete_it(path)
try:
yield path
finally:
delete_it(path)
def abspth(path):
if not path:
path = "/"