x/anvil
Code Issues Proposed changes

Add a prebuild stage that will build the dependencies first

Browse Source 
Change-Id: Icbf653b1bfb463f955df93ba4baf339e1a070e78
This commit is contained in:
Joshua Harlow
2014-08-18 18:41:20 -07:00
parent b243913be1
commit 460adbfa93
2 changed files with 76 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
anvil/packaging/yum.py
View File

@@ -77,12 +77,14 @@ class YumDependencyHandler(base.DependencyHandler):
super(YumDependencyHandler, self).__init__(distro, root_dir, instances, opts)
# Various paths we will use while operating
self.rpmbuild_dir = sh.joinpths(self.deps_dir, "rpmbuild")
self.prebuild_dir = sh.joinpths(self.deps_dir, "prebuild")
self.deps_repo_dir = sh.joinpths(self.deps_dir, "openstack-deps")
self.deps_src_repo_dir = sh.joinpths(self.deps_dir, "openstack-deps-sources")
self.rpm_sources_dir = sh.joinpths(self.rpmbuild_dir, "SOURCES")
self.anvil_repo_dir = sh.joinpths(self.root_dir, "repo")
self.build_requires_filename = sh.joinpths(self.deps_dir, "build-requires")
self.yum_satisfies_filename = sh.joinpths(self.deps_dir, "yum-satisfiable")
self.rpm_build_requires_filename = sh.joinpths(self.deps_dir, "rpm-build-requires")
# Executables we require to operate
self.rpmbuild_executable = sh.which("rpmbuild")
self.specprint_executable = sh.which('specprint', ["tools/"])
@@ -249,27 +251,18 @@ class YumDependencyHandler(base.DependencyHandler):
for sub_dir in sh.listdir(self.rpmbuild_dir, dirs_only=True):
search_dirs.append(sh.joinpths(sub_dir, "RPMS"))
moved = 0
for d in search_dirs:
moved += self._move_rpm_files(d, repo_dir)
for source_dir in search_dirs:
moved += self._move_rpm_files(source_dir, repo_dir)
return moved
build_requirements = self.requirements.get("build-requires")
if build_requirements:
utils.log_iterable(build_requirements,
header="Installing build requirements",
logger=LOG)
self.helper.transaction(install_pkgs=build_requirements,
tracewriter=self.tracewriter)
for repo_name in self.REPOS:
src_repo_dir = sh.joinpths(self.anvil_repo_dir, self.SRC_REPOS[repo_name])
src_repo_files = list_src_rpms(src_repo_dir)
if not src_repo_files:
continue
utils.log_iterable(src_repo_files,
header=('Building %s RPM packages from their'
' SRPMs for repo %s using %s jobs') %
(len(src_repo_files), self.SRC_REPOS[repo_name], self.jobs),
def build(repo_dir, repo_name, header_tpl):
repo_files = list_src_rpms(repo_dir)
if not repo_files:
return
utils.log_iterable(repo_files,
header=header_tpl % (len(repo_files),
self.SRC_REPOS[repo_name],
self.jobs),
logger=LOG)
rpmbuild_flags = "--rebuild"
if self.opts.get("usr_only", False):
@@ -278,7 +271,7 @@ class YumDependencyHandler(base.DependencyHandler):
self._create_rpmbuild_subdirs()
try:
self.py2rpm_helper.build_all_binaries(repo_name,
src_repo_dir,
repo_dir,
rpmbuild_flags,
self.tracewriter,
self.jobs)
@@ -288,6 +281,51 @@ class YumDependencyHandler(base.DependencyHandler):
if move_rpms(repo_name) > 0:
self._create_repo(repo_name)
def pre_build():
build_requirements = self.requirements.get("build-requires")
if build_requirements:
utils.log_iterable(build_requirements,
header="Installing build requirements",
logger=LOG)
self.helper.transaction(install_pkgs=build_requirements,
tracewriter=self.tracewriter)
build_requirements = ''
try:
build_requirements = sh.load_file(self.rpm_build_requires_filename)
except IOError as e:
if e.errno != errno.ENOENT:
raise
build_requirements = set(pkg_resources.yield_lines(build_requirements))
for repo_name in self.REPOS:
repo_dir = sh.joinpths(self.anvil_repo_dir, self.SRC_REPOS[repo_name])
matched_paths = []
for path in list_src_rpms(repo_dir):
package_name = sh.basename(path)[0:-len('.src.rpm')]
if package_name in build_requirements:
matched_paths.append(path)
build_requirements.discard(package_name)
if matched_paths:
with sh.remove_before(self.prebuild_dir) as prebuild_dir:
for path in matched_paths:
sh.move(path, sh.joinpths(prebuild_dir, sh.basename(path)))
build(prebuild_dir, repo_name,
'Prebuilding %s RPM packages from their SRPMs'
' for repo %s using %s jobs')
return build_requirements
unsatisfied_build_requirements = list(pre_build())
if unsatisfied_build_requirements:
utils.log_iterable(sorted(unsatisfied_build_requirements),
header="%s unsatisfied build requirements (these"
" will need to be satisfied by existing"
" repositories)" % len(unsatisfied_build_requirements),
logger=LOG)
for repo_name in self.REPOS:
repo_dir = sh.joinpths(self.anvil_repo_dir, self.SRC_REPOS[repo_name])
build(repo_dir, repo_name,
'Building %s RPM packages from their SRPMs for repo %s'
' using %s jobs')
def _move_srpms(self, repo_name, rpmbuild_dir=None):
if rpmbuild_dir is None:
rpmbuild_dir = self.rpmbuild_dir
@@ -531,13 +569,10 @@ class YumDependencyHandler(base.DependencyHandler):
tracewriter=self.tracewriter)
return spec_filename
def _copy_startup_scripts(self, instance, spec_filename):
def _copy_startup_scripts(self, instance, spec_details):
common_init_content = utils.load_template("packaging",
"common.init")[1]
cmd = [self.specprint_executable]
cmd.extend(['-f', spec_filename])
daemon_args = instance.get_option('daemon_args', default_value={})
spec_details = json.loads(sh.execute(cmd)[0])
for src in spec_details.get('sources', []):
script = sh.basename(src)
if not (script.endswith(".init")):
@@ -574,7 +609,22 @@ class YumDependencyHandler(base.DependencyHandler):
self._copy_sources(instance)
if patches:
self._copy_patches(patches)
self._copy_startup_scripts(instance, spec_filename)
cmdline = [self.specprint_executable]
cmdline.extend(['-f', spec_filename])
spec_details = json.loads(sh.execute(cmdline)[0])
try:
rpm_requires = spec_details['headers']['requires']
except (KeyError, TypeError):
pass
else:
buff = six.StringIO()
buff.write("# %s\n" % instance.name)
if rpm_requires:
for req in rpm_requires:
buff.write("%s\n" % req)
buff.write("\n")
sh.append_file(self.rpm_build_requires_filename, buff.getvalue())
self._copy_startup_scripts(instance, spec_details)
cmdline = [
self.rpmbuild_executable,
"-bs",

2
pylintrc
View File

@@ -263,7 +263,7 @@ max-statements=60
max-parents=7
# Maximum number of attributes for a class (see R0902).
max-attributes=12
max-attributes=20
# Minimum number of public methods for a class (see R0903).
min-public-methods=0