Cleanups after new changes/features.

- Some style adjustments.
- Use SRC_REPOS for mapping repo name to source repository name.
- Split the build_binary into individual tiny functions.
- Fix up the progress bars and iterable logging.
- Ensure that we log that we wrote to /etc/yum.repos.d and leave a trace for later cleanup.
- Quiet some of the new executes (and put there output to files).
- Add a smithy clean_pip() function that will help avoid hitting https://github.com/pypa/pip/issues/982
- Further cleanup of smithy after it being partially rewritten.
- Instead of hard coding 'conf/distros/rhel.yaml' in smithy, take this from the sourced file.
- Ensure that we can't remove packages smithy requires to operate.
- Ensure the package version that py2rpm is building is also trimmed of zeros.

Change-Id: I2df8a47f0115de2684777b64db42e08d50ef4115
This commit is contained in:
Joshua Harlow 2013-06-21 20:00:41 -07:00
parent a362c5399a
commit d37e725773
6 changed files with 145 additions and 128 deletions

View File

@ -208,8 +208,9 @@ class DependencyHandler(object):
self.pip_executable
]
cmdline = cmdline + extra_pips + ["-r"] + requires_files
cmdline = (cmdline + ["--ignore-package"] +
OPENSTACK_PACKAGES + self.python_names)
cmdline.extend(["--ignore-package"])
cmdline.extend(OPENSTACK_PACKAGES)
cmdline.extend(self.python_names)
output = sh.execute(cmdline, check_exit_code=False)
self.pips_to_install = list(utils.splitlines_not_empty(output[0]))

View File

@ -67,6 +67,11 @@ class YumDependencyHandler(base.DependencyHandler):
'distribute',
'setuptools',
]
SRC_REPOS = {
'anvil': 'anvil-source',
"anvil-deps": "anvil-deps-source",
}
REPOS = ["anvil-deps", "anvil"]
py2rpm_executable = sh.which("py2rpm", ["tools/"])
rpmbuild_executable = sh.which("rpmbuild")
@ -134,7 +139,7 @@ class YumDependencyHandler(base.DependencyHandler):
with sh.remove_before_after(self.rpmbuild_dir):
for dirname in (sh.joinpths(self.rpmbuild_dir, "SPECS"),
sh.joinpths(self.rpmbuild_dir, "SOURCES")):
sh.mkdirslist(dirname, tracewriter=self.tracewriter)
sh.mkdirslist(dirname)
if instance.name in ["general"]:
self._build_dependencies()
self._move_srpms("anvil-deps")
@ -149,62 +154,88 @@ class YumDependencyHandler(base.DependencyHandler):
def _move_files(source_dir, target_dir):
if not sh.isdir(source_dir):
return
for filename in sh.listdir(source_dir,
recursive=True, files_only=True):
for filename in sh.listdir(source_dir, recursive=True, files_only=True):
sh.move(filename, target_dir, force=True)
def build_binary(self):
build_requires = self.requirements["build-requires"]
if build_requires:
cmdline = ["yum", "install", "-y"] + list(build_requires)
def _install_build_requirements():
build_requires = self.requirements["build-requires"]
if build_requires:
utils.log_iterable(sorted(build_requires),
header=("Installing %s build requirements" % len(build_requires)),
logger=LOG)
cmdline = ["yum", "install", "-y"] + list(build_requires)
sh.execute(cmdline)
def _is_src_rpm(filename):
return filename.endswith('.src.rpm')
def _build_single_binary(repo_dir, srpm_filename, rpmbuild_dir):
# Extract needed info from the source rpm directly.
ts = rpm.TransactionSet()
with open(srpm_filename) as fd:
hdr = ts.hdrFromFdno(fd)
bin_rpm_filename = "%s-%s-%s.%s.rpm" % (hdr[rpm.RPMTAG_NAME],
hdr[rpm.RPMTAG_VERSION],
hdr[rpm.RPMTAG_RELEASE],
hdr[rpm.RPMTAG_ARCH])
if sh.isfile(sh.joinpths(repo_dir, bin_rpm_filename)):
return
# Rebuild the rpm from the source rpm.
base_filename = sh.basename(srpm_filename)
LOG.debug("Building RPM package from %s", base_filename)
cmdline = ["yum-builddep", "-q", "-y", srpm_filename]
sh.execute(cmdline)
ts = rpm.TransactionSet()
for repo_name in ("anvil-deps", "anvil"):
cmdline = ["rpmbuild",
"--define", "_topdir %s" % self.rpmbuild_dir,
"--rebuild", srpm_filename]
if self.opts.get("usr_only", False):
cmdline.extend(["--define", "usr_only 1"])
out_filename = sh.joinpths(self.log_dir, "rpmbuild-build-%s.out" % base_filename)
sh.execute_save_output(cmdline, out_filename=out_filename, quiet=True)
_install_build_requirements()
for repo_name in self.REPOS:
repo_dir = sh.joinpths(self.anvil_repo_dir, repo_name)
sh.mkdirslist(repo_dir, tracewriter=self.tracewriter)
src_repo_dir = sh.joinpths(self.anvil_repo_dir,
"%s-sources" % repo_name),
for srpm_filename in sh.listdir(src_repo_dir, files_only=True):
if not srpm_filename.endswith(".src.rpm"):
continue
# Extract needed info from the source rpm directly.
with open(srpm_filename) as fd:
hdr = ts.hdrFromFdno(fd)
bin_rpm_filename = "%s-%s-%s.%s.rpm" % (hdr[rpm.RPMTAG_NAME],
hdr[rpm.RPMTAG_VERSION],
hdr[rpm.RPMTAG_RELEASE],
hdr[rpm.RPMTAG_ARCH])
# Woot it was already built!
if sh.isfile(sh.joinpths(repo_dir, bin_rpm_filename)):
LOG.info("Found RPM package %s", bin_rpm_filename)
continue
# Rebuild the rpm from the source rpm.
with remove_before_after(self.rpmbuild_dir):
base_filename = sh.basename(srpm_filename)
LOG.info("Building RPM package from %s", base_filename)
cmdline = ["yum-builddep", "-q", "-y", srpm_filename]
sh.execute(cmdline)
cmdline = ["rpmbuild",
"--define", "_topdir %s" % self.rpmbuild_dir,
"--rebuild", srpm_filename]
if self.opts.get("usr_only", False):
cmdline.extend(["--define", "usr_only 1"])
build_filename = sh.joinpths(self.log_dir,
"rpmbuild-%s.out" % base_filename)
sh.execute_save_output(cmdline, out_filename=build_filename)
self._move_files(sh.joinpths(self.rpmbuild_dir, "RPMS"),
repo_dir)
src_repo_dir = sh.joinpths(self.anvil_repo_dir, self.SRC_REPOS[repo_name])
if sh.isdir(src_repo_dir):
src_repo_files = sh.listdir(src_repo_dir, files_only=True)
src_repo_files = sorted([f for f in src_repo_files if _is_src_rpm(f)])
else:
src_repo_files = []
if not src_repo_files:
continue
src_repo_base_files = [sh.basename(f) for f in src_repo_files]
header = 'Building %s RPM packages from there SRPM backing files from repo %s'
header = header % (len(src_repo_files), self.SRC_REPOS[repo_name])
utils.log_iterable(src_repo_base_files, header=header, logger=LOG)
with utils.progress_bar(name='Building', max_am=len(src_repo_files)) as p_bar:
for (i, srpm_filename) in enumerate(src_repo_files):
# Rebuild the rpm from the source rpm.
with sh.remove_before_after(self.rpmbuild_dir):
sh.mkdirslist(self.rpmbuild_dir)
_build_single_binary(repo_dir, srpm_filename, self.rpmbuild_dir)
self._move_files(sh.joinpths(self.rpmbuild_dir, "RPMS"), repo_dir)
p_bar.update(i + 1)
for repo_name in self.REPOS:
self._create_repo(repo_name)
def _move_srpms(self, repo_name):
src_repo_dir = sh.joinpths(self.anvil_repo_dir, "%s-sources" % repo_name)
src_repo_name = self.SRC_REPOS[repo_name]
src_repo_dir = sh.joinpths(self.anvil_repo_dir, src_repo_name)
sh.mkdirslist(src_repo_dir, tracewriter=self.tracewriter)
self._move_files(sh.joinpths(self.rpmbuild_dir, "SRPMS"), src_repo_dir)
def _create_repo(self, repo_name):
repo_dir = sh.joinpths(self.anvil_repo_dir, repo_name)
src_repo_dir = "%s-sources" % repo_dir
src_repo_dir = sh.joinpths(self.anvil_repo_dir, self.SRC_REPOS[repo_name])
for a_dir in (repo_dir, src_repo_dir):
if not sh.isdir(a_dir):
sh.mkdirslist(a_dir, tracewriter=self.tracewriter)
cmdline = ["createrepo", a_dir]
LOG.info("Creating repo at %s", a_dir)
sh.execute(cmdline)
@ -216,13 +247,13 @@ class YumDependencyHandler(base.DependencyHandler):
"baseurl_bin": "file://%s" % repo_dir,
"baseurl_src": "file://%s" % src_repo_dir,
}
sh.write_file(repo_filename,
utils.expand_template(content, params),
sh.write_file(repo_filename, utils.expand_template(content, params),
tracewriter=self.tracewriter)
# Install *.repo file so that anvil deps will be available
# when building OpenStack
system_repo_filename = sh.joinpths(self.YUM_REPO_DIR, "%s.repo" % repo_name)
sh.copy(repo_filename, system_repo_filename)
LOG.info("Copying to %s", system_repo_filename)
self.tracewriter.file_touched(system_repo_filename)
def _get_yum_available(self):
@ -287,7 +318,7 @@ class YumDependencyHandler(base.DependencyHandler):
yum_map = self._get_yum_available()
pips_keys = set([p.key for p in pips_downloaded])
def filter_package_files(package_files):
def _filter_package_files(package_files):
package_reqs = []
package_keys = []
for filename in package_files:
@ -319,22 +350,30 @@ class YumDependencyHandler(base.DependencyHandler):
return filtered_files
LOG.info("Filtering %s downloaded files.", len(package_files))
package_files = filter_package_files(package_files)
package_files = _filter_package_files(package_files)
if not package_files:
LOG.info("No RPM packages of OpenStack dependencies to build")
LOG.info("No SRPM package dependencies to build.")
return
for filename in package_files:
LOG.info("Building SRPM package from %s", filename)
scripts_dir = sh.abspth(sh.joinpths(settings.TEMPLATE_DIR,
"packaging", "scripts"))
cmdline = self.py2rpm_start_cmdline()
cmdline.extend(["--scripts-dir", scripts_dir, "--source-only",
"--", filename])
sh.execute(cmdline)
package_files = sorted(package_files)
base_package_files = [sh.basename(f) for f in package_files]
utils.log_iterable(base_package_files,
header="Building %s SRPM packages" % (len(base_package_files)),
logger=LOG)
with utils.progress_bar(name="Building", max_am=len(base_package_files)) as p_bar:
for (i, filename) in enumerate(package_files):
LOG.debug("Building SRPM package from %s", filename)
scripts_dir = sh.abspth(sh.joinpths(settings.TEMPLATE_DIR, "packaging", "scripts"))
cmdline = self.py2rpm_start_cmdline()
cmdline.extend(["--scripts-dir", scripts_dir,
"--source-only",
"--", filename])
out_filename = sh.joinpths(self.log_dir, "py2rpm-build-%s.out" % (sh.basename(filename)))
sh.execute_save_output(cmdline, out_filename=out_filename, quiet=True)
p_bar.update(i + 1)
def _write_spec_file(self, instance, rpm_name, template_name, params):
requires_what = params.get('requires')
it not requires_what:
if not requires_what:
requires_what = []
requires_python = []
try:
@ -347,8 +386,7 @@ class YumDependencyHandler(base.DependencyHandler):
params["epoch"] = self.OPENSTACK_EPOCH
content = utils.load_template(self.SPEC_TEMPLATE_DIR, template_name)[1]
spec_filename = sh.joinpths(self.rpmbuild_dir, "SPECS", "%s.spec" % rpm_name)
sh.write_file(spec_filename,
utils.expand_template(content, params),
sh.write_file(spec_filename, utils.expand_template(content, params),
tracewriter=self.tracewriter)
return spec_filename
@ -397,11 +435,8 @@ class YumDependencyHandler(base.DependencyHandler):
"--define", "_topdir %s" % self.rpmbuild_dir,
spec_filename,
]
build_filename = sh.joinpths(self.log_dir,
"rpmbuild-%s" % sh.basename(spec_filename))
sh.execute_save_output(cmdline,
out_filename=build_filename,
quiet=True)
out_filename = sh.joinpths(self.log_dir, "rpmbuild-%s.out" % instance.name)
sh.execute_save_output(cmdline, out_filename=out_filename, quiet=True)
def _write_git_tarball(self, pkg_dir, spec_filename):
cmdline = [
@ -464,10 +499,8 @@ class YumDependencyHandler(base.DependencyHandler):
app_dir = instance.get_option('app_dir')
cmdline = self.py2rpm_start_cmdline()
cmdline.extend(["--source-only", "--", app_dir])
build_filename = sh.joinpths(self.log_dir, "py2rpm-%s.out" % (instance.name))
sh.execute_save_output(cmdline,
cwd=app_dir,
out_filename=build_filename,
out_filename = sh.joinpths(self.log_dir, "py2rpm-build-%s.out" % (instance.name))
sh.execute_save_output(cmdline, cwd=app_dir, out_filename=out_filename,
quiet=True)
def _build_openstack_package(self, instance):
@ -479,8 +512,7 @@ class YumDependencyHandler(base.DependencyHandler):
egg_name = instance.egg_info['name']
params["version"] = instance.egg_info["version"]
if self._is_client(instance.name, egg_name):
client_name = utils.strip_prefix_suffix(egg_name,
"python-", "client")
client_name = utils.strip_prefix_suffix(egg_name, "python-", "client")
if not client_name:
msg = "Bad client package name %s" % (egg_name)
raise excp.PackageException(msg)

View File

@ -144,8 +144,7 @@ def execute(cmd,
stderr_fh = subprocess.PIPE
try:
obj = subprocess.Popen(execute_cmd, stdin=stdin_fh, stdout=stdout_fh, stderr=stderr_fh,
close_fds=True, cwd=cwd, shell=shell,
env=process_env)
close_fds=True, cwd=cwd, shell=shell, env=process_env)
if process_input is not None:
result = obj.communicate(str(process_input))
else:

85
smithy
View File

@ -57,10 +57,19 @@ find_pip()
fi
}
clean_pip()
{
# https://github.com/pypa/pip/issues/982
if [ -n "$SUDO_USER" ]; then
rm -rf /tmp/pip-build-$SUDO_USER
fi
}
rpm_is_installed()
{
local name="$(basename "$1")"
rpm $RPM_OPTS "${name%.rpm}" &>/dev/null
return $?
}
cache_and_install_rpm_url()
@ -98,45 +107,6 @@ yum_install()
return $rc
}
install_rpm()
{
local rpm_path=$1
local py_name=$2
local always_build=$3
if [ -n "$rpm_path" -a -z "$always_build" ]; then
yum_install "$rpm_path"
if rpm_is_installed "$rpm_path"; then
return 0
fi
fi
if [ -z "$py_name" ]; then
return 1
fi
# RPM is not available. Try to build it on fly
# First download it.
pip_tmp_dir=$(mktemp -d)
find_pip
pip_opts="$PIP_OPTS -U -I"
$PIP_CMD install $pip_opts "$py_name" --download "$pip_tmp_dir"
# Now build it
echo "Building RPM for $py_name"
rpm_names=$("$PY2RPM_CMD" "$pip_tmp_dir/"* 2>/dev/null |
awk '/^Wrote: /{ print $2 }' | grep -v '.src.rpm' | sort -u | grep -v "python-distribute")
rm -rf "$pip_tmp_dir"
if [ -z "$rpm_names" ]; then
echo "No binary RPM was built for $py_name"
return 1
fi
for pkg in $rpm_names; do
echo "Installing RPM $pkg"
if [ -f "$pkg" ]; then
yum_install "$pkg"
fi
done
}
bootstrap_epel()
{
[ -z "$EPEL_RPM_URL" ] && return 0
@ -154,19 +124,23 @@ bootstrap_rpm_packages()
echo "Installing packages: $(echo $REQUIRES)"
for rpm in $REQUIRES; do
yum_install "$rpm"
if [ "$?" != "0" ]; then
echo "Failed installing $rpm"
return 1
fi
done
fi
}
bootstrap_python_rpms()
{
local package_map=$(python -c 'import yaml
local package_map=$(python -c "import yaml
try:
for k, v in yaml.safe_load(open("conf/distros/rhel.yaml"))["dependency_handler"]["package_map"].iteritems():
print "%s==%s" % (k, v)
except:
for k, v in yaml.safe_load(open('$DISTRO_CONFIG'))['dependency_handler']['package_map'].iteritems():
print '%s==%s' % (k, v)
except KeyError:
pass
')
")
# NOTE(aababilov): drop `<` and `<=` requirements because yum cannot
# handle them correctly
local python_names=$(cat requirements.txt test-requirements.txt |
@ -174,7 +148,7 @@ except:
local rpm_names=$("$PY2RPM_CMD" --package-map $package_map --convert $python_names |
while read req pack; do echo $pack; done | sort -u)
# Install all available RPMs
echo "Installing packages: $(echo $rpm_names)"
echo "Installing python requirement packages: $(echo $rpm_names)"
for rpm in $rpm_names; do
yum_install $rpm
done
@ -182,32 +156,35 @@ except:
local missing_python=""
for python_name in $python_names; do
rpm_name=$("$PY2RPM_CMD" --package-map $package_map --convert $python_name | awk '{print $2}' | sort -u)
if ! rpm -q $rpm_name &>/dev/null; then
if ! rpm_is_installed $rpm_name; then
missing_python="$missing_python $python_name"
fi
done
if [ -z "$missing_python" ]; then
return
fi
# Some RPMs are not available.
# Try to build them on fly.
# Some RPMs are not available, try to build them on fly.
# First download them...
local pip_tmp_dir=$(mktemp -d)
find_pip
local pip_opts="$PIP_OPTS -U -I"
echo "Downloading Python requirements:$missing_python"
echo "Downloading missing python requirements:$missing_python"
$PIP_CMD install $pip_opts $missing_python --download "$pip_tmp_dir"
# Now build them
echo "Building RPMs for $missing_python"
local rpm_names=$("$PY2RPM_CMD" --package-map $package_map -- "$pip_tmp_dir/"* 2>/dev/null |
awk '/^Wrote: /{ print $2 }' | grep -v '.src.rpm' | sort -u)
rm -rf "$pip_tmp_dir" /tmp/pip-build-$SUDO_USER
if [ -z "$rpm_names" ]; then
echo "No binary RPMs were built for$missing_python"
return 1
fi
echo "Installing missing python requirement packages: $(echo $rpm_names)"
for rpm in $rpm_names; do
yum_install "$rpm"
if [ "$?" != "0" ]; then
echo "Failed installing $rpm"
return 1
fi
done
}
@ -303,14 +280,14 @@ done
# Source immediately so that we can export the needed variables.
if [ -f "$BSCONF_FILE" ]; then
echo "Sourcing $BSCONF_FILE"
source $BSCONF_FILE
export REQUIRED_PACKAGES="$REQUIRES"
fi
export REQUIRED_PACKAGES="$REQUIRES"
if ! needs_bootstrap; then
clean_pip
run_smithy
clean_pip
elif ! $BOOTSTRAP; then
echo "This system needs to be updated in order to run anvil!" >&2
echo "Running 'sudo $SMITHY_NAME --bootstrap' will attempt to do so." >&2
@ -338,6 +315,7 @@ fi
echo "Bootstrapping $SHORTNAME $RELEASE"
echo "Please wait..."
clean_pip
for step in ${STEPS:?"Error: STEPS is undefined!"}; do
bootstrap_${step}
if [ $? != 0 ]; then
@ -345,6 +323,7 @@ for step in ${STEPS:?"Error: STEPS is undefined!"}; do
exit 1
fi
done
clean_pip
# Write the checksums of the bootstrap file
# which if new requirements are added will cause new checksums

View File

@ -1,6 +1,7 @@
# -*- sh -*-
STEPS="epel rpm_packages python_rpms"
EPEL_RPM_URL="http://mirrors.kernel.org/fedora-epel/6/i386/epel-release-6-8.noarch.rpm"
DISTRO_CONFIG=conf/distros/rhel.yaml
## Bootstrap for Red Hat based distros
REQUIRES='

View File

@ -412,10 +412,15 @@ def build_rpm(options, filename):
archive_name = "%s/dist/%s-%s.tar.gz" % (source_dir, pkg_name, version)
shutil.copy(archive_name, os.path.join(build_dir, "SOURCES"))
# We need to do this so that when a package such as hacking depends on
# flake8 v2 that we don't go ahead and build a v2.0 version.
#
# Note(harlowja): Not sure why rpm seems to not understand these are the same...
cleaned_version = trim_zeroes(version.replace('-','_'))
with open(spec_name, "w") as spec_file:
print >> spec_file, "%define pkg_name", pkg_name
print >> spec_file, "%define rpm_name", rpm_name
print >> spec_file, "%define version", version.replace('-','_')
print >> spec_file, "%define version", cleaned_version
print >> spec_file, "%define release", options.release
print >> spec_file, "%define unmangled_version", version
print >> spec_file, ""
@ -434,7 +439,7 @@ def build_rpm(options, filename):
tags.append(("Source0", archive_name))
tags.append(("License", info["license"]))
tags.append(("Group", "Development/Libraries"))
tags.append(("BuildRoot", "%{_tmppath}/%{pkg_name}-%{version}-%{release}-buildroot"))
tags.append(("BuildRoot", "%{_tmppath}/%{pkg_name}-%{unmangled_version}-%{release}-buildroot"))
tags.append(("Prefix", "%{_prefix}"))
if pkg_key not in arch_dependent:
if not os.path.exists(egg_info_path(source_dir, "ext_modules.txt")):