Use parallel make to build packages
Parallelize build of dependencies SRPMs and all binary RPMs. Implements: blueprint parallel-build Change-Id: Ieed5aa199e325524390327b823ce3fef0b801f4a
This commit is contained in:
Alessio Ababilov
parent
963eb3f2e7
commit
7cba1cd48d
@ -32,6 +32,7 @@ from anvil import env
|
||||
from anvil import exceptions as excp
|
||||
from anvil import log as logging
|
||||
from anvil import opts
|
||||
from anvil.packaging import yum
|
||||
from anvil import persona
|
||||
from anvil import settings
|
||||
from anvil import shell as sh
|
||||
@ -102,6 +103,7 @@ def run(args):
|
||||
except Exception as e:
|
||||
raise excp.OptionException("Error loading persona file: %s due to %s" % (persona_fn, e))
|
||||
|
||||
yum.YumDependencyHandler.jobs = args["jobs"]
|
||||
# Get the object we will be running with...
|
||||
runner = runner_cls(distro=dist,
|
||||
root_dir=root_dir,
|
||||
|
||||
@ -14,6 +14,8 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import multiprocessing
|
||||
|
||||
from optparse import IndentedHelpFormatter
|
||||
from optparse import (OptionParser, OptionGroup, OptionValueError)
|
||||
|
||||
@ -91,6 +93,13 @@ def parse(previous_settings=None):
|
||||
dest="action",
|
||||
metavar="ACTION",
|
||||
help="required action to perform: %s" % (_format_list(actions.names())))
|
||||
base_group.add_option("-j", "--jobs",
|
||||
action="store",
|
||||
type="int",
|
||||
dest="jobs",
|
||||
default=multiprocessing.cpu_count() + 1,
|
||||
metavar="JOBS",
|
||||
help="number of building jobs to run simultaneously (default: %default)")
|
||||
base_group.add_option("-d", "--directory",
|
||||
action="store",
|
||||
type="string",
|
||||
@ -132,6 +141,7 @@ def parse(previous_settings=None):
|
||||
values['dir'] = (options.dir or "")
|
||||
values['dryrun'] = (options.dryrun or False)
|
||||
values['action'] = (options.action or "")
|
||||
values['jobs'] = options.jobs
|
||||
values['persona_fn'] = options.persona_fn
|
||||
values['verbose'] = options.verbose
|
||||
values['usr_only'] = options.usr_only
|
||||
|
||||
@ -14,9 +14,10 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import re
|
||||
import copy
|
||||
import pkg_resources
|
||||
|
||||
import pkg_resources
|
||||
from pip import util as pip_util
|
||||
from pip import req as pip_req
|
||||
|
||||
@ -28,7 +29,7 @@ LOG = logging.getLogger(__name__)
|
||||
|
||||
FREEZE_CMD = ['freeze', '--local']
|
||||
EGGS_DETAILED = {}
|
||||
|
||||
PYTHON_KEY_VERSION_RE = re.compile("^(.+)-([0-9][0-9.a-zA-Z]*)$")
|
||||
|
||||
def create_requirement(name, version=None):
|
||||
name = pkg_resources.safe_name(name.strip())
|
||||
@ -48,7 +49,15 @@ def create_requirement(name, version=None):
|
||||
|
||||
|
||||
def extract(line):
|
||||
return pip_req.InstallRequirement.from_line(line)
|
||||
req = pip_req.InstallRequirement.from_line(line)
|
||||
# NOTE(aababilov): req.req.key can look like oslo.config-1.2.0a2,
|
||||
# so, split it
|
||||
if req.req:
|
||||
match = PYTHON_KEY_VERSION_RE.match(req.req.key)
|
||||
if match:
|
||||
req.req = pkg_resources.Requirement.parse(
|
||||
"%s>=%s" % (match.group(1), match.group(2)))
|
||||
return req
|
||||
|
||||
|
||||
def extract_requirement(line):
|
||||
|
||||
@ -16,9 +16,9 @@
|
||||
|
||||
import collections
|
||||
import os
|
||||
import pkg_resources
|
||||
import sys
|
||||
|
||||
import pkg_resources
|
||||
import rpm
|
||||
|
||||
from anvil import colorizer
|
||||
@ -74,6 +74,7 @@ class YumDependencyHandler(base.DependencyHandler):
|
||||
REPOS = ["anvil-deps", "anvil"]
|
||||
py2rpm_executable = sh.which("py2rpm", ["tools/"])
|
||||
rpmbuild_executable = sh.which("rpmbuild")
|
||||
jobs = 2
|
||||
|
||||
def __init__(self, distro, root_dir, instances, opts=None):
|
||||
super(YumDependencyHandler, self).__init__(distro, root_dir, instances, opts)
|
||||
@ -135,11 +136,14 @@ class YumDependencyHandler(base.DependencyHandler):
|
||||
params['version_suffix'] = version_suffix
|
||||
return params
|
||||
|
||||
def _create_rpmbuild_subdirs(self):
|
||||
for dirname in (sh.joinpths(self.rpmbuild_dir, "SPECS"),
|
||||
sh.joinpths(self.rpmbuild_dir, "SOURCES")):
|
||||
sh.mkdirslist(dirname)
|
||||
|
||||
def package_instance(self, instance):
|
||||
with sh.remove_before_after(self.rpmbuild_dir):
|
||||
for dirname in (sh.joinpths(self.rpmbuild_dir, "SPECS"),
|
||||
sh.joinpths(self.rpmbuild_dir, "SOURCES")):
|
||||
sh.mkdirslist(dirname)
|
||||
self._create_rpmbuild_subdirs()
|
||||
if instance.name in ["general"]:
|
||||
self._build_dependencies()
|
||||
self._move_srpms("anvil-deps")
|
||||
@ -171,30 +175,6 @@ class YumDependencyHandler(base.DependencyHandler):
|
||||
def _is_src_rpm(filename):
|
||||
return filename.endswith('.src.rpm')
|
||||
|
||||
def _build_single_binary(repo_dir, srpm_filename, rpmbuild_dir):
|
||||
# Extract needed info from the source rpm directly.
|
||||
ts = rpm.TransactionSet()
|
||||
with open(srpm_filename) as fd:
|
||||
hdr = ts.hdrFromFdno(fd)
|
||||
bin_rpm_filename = "%s-%s-%s.%s.rpm" % (hdr[rpm.RPMTAG_NAME],
|
||||
hdr[rpm.RPMTAG_VERSION],
|
||||
hdr[rpm.RPMTAG_RELEASE],
|
||||
hdr[rpm.RPMTAG_ARCH])
|
||||
if sh.isfile(sh.joinpths(repo_dir, bin_rpm_filename)):
|
||||
return
|
||||
# Rebuild the rpm from the source rpm.
|
||||
base_filename = sh.basename(srpm_filename)
|
||||
LOG.debug("Building RPM package from %s", base_filename)
|
||||
cmdline = ["yum-builddep", "-q", "-y", srpm_filename]
|
||||
sh.execute(cmdline)
|
||||
cmdline = ["rpmbuild",
|
||||
"--define", "_topdir %s" % self.rpmbuild_dir,
|
||||
"--rebuild", srpm_filename]
|
||||
if self.opts.get("usr_only", False):
|
||||
cmdline.extend(["--define", "usr_only 1"])
|
||||
out_filename = sh.joinpths(self.log_dir, "rpmbuild-build-%s.out" % base_filename)
|
||||
sh.execute_save_output(cmdline, out_filename=out_filename, quiet=True)
|
||||
|
||||
_install_build_requirements()
|
||||
|
||||
for repo_name in self.REPOS:
|
||||
@ -209,21 +189,38 @@ class YumDependencyHandler(base.DependencyHandler):
|
||||
if not src_repo_files:
|
||||
continue
|
||||
src_repo_base_files = [sh.basename(f) for f in src_repo_files]
|
||||
header = 'Building %s RPM packages from there SRPM backing files from repo %s'
|
||||
header = header % (len(src_repo_files), self.SRC_REPOS[repo_name])
|
||||
header = 'Building %s RPM packages from their SRPMs for repo %s using %s jobs'
|
||||
header = header % (len(src_repo_files), self.SRC_REPOS[repo_name], self.jobs)
|
||||
utils.log_iterable(src_repo_base_files, header=header, logger=LOG)
|
||||
with utils.progress_bar(name='Building', max_am=len(src_repo_files)) as p_bar:
|
||||
for (i, srpm_filename) in enumerate(src_repo_files):
|
||||
# Rebuild the rpm from the source rpm.
|
||||
with sh.remove_before_after(self.rpmbuild_dir):
|
||||
sh.mkdirslist(self.rpmbuild_dir)
|
||||
_build_single_binary(repo_dir, srpm_filename, self.rpmbuild_dir)
|
||||
self._move_files(sh.joinpths(self.rpmbuild_dir, "RPMS"), repo_dir)
|
||||
p_bar.update(i + 1)
|
||||
|
||||
for repo_name in self.REPOS:
|
||||
makefile_name = sh.joinpths(self.deps_dir, "binary-%s.mk" % repo_name)
|
||||
marks_dir = sh.joinpths(self.deps_dir, "marks-binary")
|
||||
sh.mkdirslist(marks_dir)
|
||||
(_fn, content) = utils.load_template("packaging/makefiles", "binary.mk")
|
||||
rpmbuild_flags = ("--rebuild --define '_topdir %s'" % self.rpmbuild_dir)
|
||||
if self.opts.get("usr_only", False):
|
||||
rpmbuild_flags += "--define 'usr_only 1'"
|
||||
params = {
|
||||
"SRC_REPO_DIR": src_repo_dir,
|
||||
"RPMBUILD_FLAGS": rpmbuild_flags,
|
||||
"LOGS_DIR": self.log_dir,
|
||||
}
|
||||
sh.write_file(makefile_name,
|
||||
utils.expand_template(content, params),
|
||||
tracewriter=self.tracewriter)
|
||||
with sh.remove_before_after(self.rpmbuild_dir):
|
||||
self._create_rpmbuild_subdirs()
|
||||
self._execute_make(makefile_name, marks_dir)
|
||||
self._move_files(sh.joinpths(self.rpmbuild_dir, "RPMS"),
|
||||
repo_dir)
|
||||
self._create_repo(repo_name)
|
||||
|
||||
def _execute_make(self, filename, marks_dir):
|
||||
sh.execute(
|
||||
["make", "-f", filename, "-j", str(self.jobs)],
|
||||
cwd=marks_dir,
|
||||
stdout_fh=sys.stdout, stderr_fh=sys.stderr)
|
||||
|
||||
def _move_srpms(self, repo_name):
|
||||
src_repo_name = self.SRC_REPOS[repo_name]
|
||||
src_repo_dir = sh.joinpths(self.anvil_repo_dir, src_repo_name)
|
||||
@ -354,22 +351,31 @@ class YumDependencyHandler(base.DependencyHandler):
|
||||
if not package_files:
|
||||
LOG.info("No SRPM package dependencies to build.")
|
||||
return
|
||||
package_files = sorted(package_files)
|
||||
makefile_name = sh.joinpths(self.deps_dir, "deps.mk")
|
||||
marks_dir = sh.joinpths(self.deps_dir, "marks-deps")
|
||||
sh.mkdirslist(marks_dir)
|
||||
(_fn, content) = utils.load_template("packaging/makefiles", "source.mk")
|
||||
scripts_dir = sh.abspth(sh.joinpths(settings.TEMPLATE_DIR, "packaging", "scripts"))
|
||||
py2rpm_options = self.py2rpm_start_cmdline()[1:] + [
|
||||
"--scripts-dir", scripts_dir,
|
||||
"--source-only",
|
||||
"--rpm-base", self.rpmbuild_dir,
|
||||
]
|
||||
params = {
|
||||
"DOWNLOADS_DIR": self.download_dir,
|
||||
"LOGS_DIR": self.log_dir,
|
||||
"PY2RPM": self.py2rpm_executable,
|
||||
"PY2RPM_FLAGS": " ".join(py2rpm_options),
|
||||
}
|
||||
sh.write_file(makefile_name,
|
||||
utils.expand_template(content, params),
|
||||
tracewriter=self.tracewriter)
|
||||
base_package_files = [sh.basename(f) for f in package_files]
|
||||
utils.log_iterable(base_package_files,
|
||||
header="Building %s SRPM packages" % (len(base_package_files)),
|
||||
header="Building %s SRPM packages using %s jobs" %
|
||||
(len(package_files), self.jobs),
|
||||
logger=LOG)
|
||||
with utils.progress_bar(name="Building", max_am=len(base_package_files)) as p_bar:
|
||||
for (i, filename) in enumerate(package_files):
|
||||
LOG.debug("Building SRPM package from %s", filename)
|
||||
scripts_dir = sh.abspth(sh.joinpths(settings.TEMPLATE_DIR, "packaging", "scripts"))
|
||||
cmdline = self.py2rpm_start_cmdline()
|
||||
cmdline.extend(["--scripts-dir", scripts_dir,
|
||||
"--source-only",
|
||||
"--", filename])
|
||||
out_filename = sh.joinpths(self.log_dir, "py2rpm-build-%s.out" % (sh.basename(filename)))
|
||||
sh.execute_save_output(cmdline, out_filename=out_filename, quiet=True)
|
||||
p_bar.update(i + 1)
|
||||
self._execute_make(makefile_name, marks_dir)
|
||||
|
||||
def _write_spec_file(self, instance, rpm_name, template_name, params):
|
||||
requires_what = params.get('requires')
|
||||
@ -435,7 +441,7 @@ class YumDependencyHandler(base.DependencyHandler):
|
||||
"--define", "_topdir %s" % self.rpmbuild_dir,
|
||||
spec_filename,
|
||||
]
|
||||
out_filename = sh.joinpths(self.log_dir, "rpmbuild-%s.out" % instance.name)
|
||||
out_filename = sh.joinpths(self.log_dir, "rpmbuild-%s.log" % instance.name)
|
||||
sh.execute_save_output(cmdline, out_filename=out_filename, quiet=True)
|
||||
|
||||
def _write_git_tarball(self, pkg_dir, spec_filename):
|
||||
@ -499,7 +505,7 @@ class YumDependencyHandler(base.DependencyHandler):
|
||||
app_dir = instance.get_option('app_dir')
|
||||
cmdline = self.py2rpm_start_cmdline()
|
||||
cmdline.extend(["--source-only", "--", app_dir])
|
||||
out_filename = sh.joinpths(self.log_dir, "py2rpm-build-%s.out" % (instance.name))
|
||||
out_filename = sh.joinpths(self.log_dir, "py2rpm-build-%s.log" % (instance.name))
|
||||
sh.execute_save_output(cmdline, cwd=app_dir, out_filename=out_filename,
|
||||
quiet=True)
|
||||
|
||||
|
||||
33
conf/templates/packaging/makefiles/binary.mk
Normal file
33
conf/templates/packaging/makefiles/binary.mk
Normal file
@ -0,0 +1,33 @@
|
||||
SRC_REPO_DIR := $SRC_REPO_DIR
|
||||
LOGS_DIR := $LOGS_DIR
|
||||
RPMBUILD := rpmbuild
|
||||
RPMBUILD_FLAGS := $RPMBUILD_FLAGS
|
||||
|
||||
#raw
|
||||
YUM_BUILDDEP := yum-builddep
|
||||
YUM_BUILDDEP_FLAGS := -q -y
|
||||
|
||||
REPO_NAME := $(shell basename $(SRC_REPO_DIR))
|
||||
BUILDDEP_MARK := builddep-$(REPO_NAME).mark
|
||||
MARKS := $(foreach archive,$(shell (cd $(SRC_REPO_DIR) && echo *src.rpm)),$(archive).mark)
|
||||
|
||||
|
||||
all: $(MARKS)
|
||||
|
||||
|
||||
# NOTE(aababilov): yum-builddep is buggy and can fail when several
|
||||
# package names are given, so, pass them one by one
|
||||
$(BUILDDEP_MARK):
|
||||
@echo "Installing build requirements for $(REPO_NAME)"
|
||||
@for pkg in $(SRC_REPO_DIR)/*; do \
|
||||
$(YUM_BUILDDEP) $(YUM_BUILDDEP_FLAGS) $$pkg; \
|
||||
done &> $(LOGS_DIR)/yum-builddep-$(REPO_NAME).log
|
||||
@touch "$@"
|
||||
@echo "Build requirements are installed"
|
||||
|
||||
|
||||
%.mark: $(SRC_REPO_DIR)/% $(BUILDDEP_MARK)
|
||||
@$(RPMBUILD) $(RPMBUILD_FLAGS) -- $< &> $(LOGS_DIR)/rpmbuild-$*.log
|
||||
@touch "$@"
|
||||
@echo "$* is processed"
|
||||
#end raw
|
||||
17
conf/templates/packaging/makefiles/source.mk
Normal file
17
conf/templates/packaging/makefiles/source.mk
Normal file
@ -0,0 +1,17 @@
|
||||
DOWNLOADS_DIR := $DOWNLOADS_DIR
|
||||
LOGS_DIR := $LOGS_DIR
|
||||
PY2RPM := $PY2RPM
|
||||
PY2RPM_FLAGS := $PY2RPM_FLAGS
|
||||
|
||||
#raw
|
||||
MARKS := $(foreach archive,$(shell ls $(DOWNLOADS_DIR)),$(archive).mark)
|
||||
|
||||
|
||||
all: $(MARKS)
|
||||
|
||||
|
||||
%.mark: $(DOWNLOADS_DIR)/%
|
||||
@$(PY2RPM) $(PY2RPM_FLAGS) -- $^ &> $(LOGS_DIR)/py2rpm-$*.log
|
||||
@touch "$@"
|
||||
@echo "$* is processed"
|
||||
#end raw
|
||||
@ -6,6 +6,8 @@ DISTRO_CONFIG=conf/distros/rhel.yaml
|
||||
## Bootstrap for Red Hat based distros
|
||||
REQUIRES='
|
||||
gcc
|
||||
|
||||
make
|
||||
git
|
||||
patch
|
||||
python
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user