Build OpenStack packages from custom specs

Maintain spec files for OpenStack packages. Start OpenStack daemons
as native system services under different users. Write configuration
to /etc.

Implements: blueprint different-openstack-users
Implements: blueprint purge-config
Implements: blueprint package-novnc

Change-Id: I454c1e88011c75997d879bf8b90cd87c8db3f123
This commit is contained in:
Alessio Ababilov
2013-06-04 16:58:59 +04:00
committed by Joshua Harlow
parent b9699a75ad
commit 052daddfd7
68 changed files with 3070 additions and 1468 deletions

View File

@@ -123,7 +123,7 @@ class DependencyHandler(object):
splitlines()[-1].strip())
return python_names
def package(self):
def package_start(self):
requires_files = []
extra_pips = []
for inst in self.instances:
@@ -138,6 +138,12 @@ class DependencyHandler(object):
self.gather_pips_to_install(requires_files, extra_pips)
self.clean_pip_requires(requires_files)
def package_instance(self, instance):
pass
def package_finish(self):
pass
def install(self):
for inst in self.instances:
for pkg in inst.get_option("nopackages") or []:

View File

@@ -15,10 +15,10 @@
# under the License.
import collections
import pkg_resources
import sys
from datetime import datetime
import pkg_resources
import rpm
from anvil import colorizer
from anvil import env
@@ -26,6 +26,7 @@ from anvil import log as logging
from anvil.packaging import base
from anvil.packaging.helpers import pip_helper
from anvil.packaging.helpers import yum_helper
from anvil import settings
from anvil import shell as sh
from anvil import utils
@@ -45,8 +46,16 @@ class YumInstallHelper(base.InstallHelper):
class YumDependencyHandler(base.DependencyHandler):
OPENSTACK_DEPS_PACKAGE_NAME = "openstack-deps"
OPENSTACK_EPOCH = 2
SPEC_TEMPLATE_DIR = "packaging/specs"
API_NAMES = {
"nova": "Compute",
"glance": "Image",
"keystone": "Identity",
"cinder": "Volume",
"quantum": "Networking",
}
SERVER_NAMES = ["nova", "glance", "keystone", "quantum", "cinder"]
py2rpm_executable = sh.which("py2rpm", ["tools/"])
REPO_FN = "anvil.repo"
YUM_REPO_DIR = "/etc/yum.repos.d/"
@@ -54,6 +63,7 @@ class YumDependencyHandler(base.DependencyHandler):
'distribute',
'setuptools',
]
rpmbuild_executable = sh.which("rpmbuild")
def __init__(self, distro, root_dir, instances):
super(YumDependencyHandler, self).__init__(distro, root_dir, instances)
@@ -62,6 +72,8 @@ class YumDependencyHandler(base.DependencyHandler):
self.deps_src_repo_dir = sh.joinpths(self.deps_dir, "openstack-deps-sources")
self.anvil_repo_filename = sh.joinpths(self.deps_dir, self.REPO_FN)
self.helper = yum_helper.Helper()
self.rpm_sources_dir = sh.joinpths(self.rpmbuild_dir, "SOURCES")
self.anvil_repo_dir = sh.joinpths(self.root_dir, "repo")
def py2rpm_start_cmdline(self):
cmdline = [
@@ -88,12 +100,57 @@ class YumDependencyHandler(base.DependencyHandler):
] + arch_dependent
return cmdline
def package(self):
super(YumDependencyHandler, self).package()
self._write_all_deps_package()
self._build_dependencies()
self._build_openstack()
self._create_deps_repo()
def package_instance(self, instance):
# clear before...
sh.deldir(self.rpmbuild_dir)
for dirname in (sh.joinpths(self.rpmbuild_dir, "SPECS"),
sh.joinpths(self.rpmbuild_dir, "SOURCES")):
sh.mkdir(dirname, recurse=True)
if instance.name == "general":
self._build_dependencies()
self._move_rpms("anvil-deps")
self._create_repo("anvil-deps")
else:
app_dir = instance.get_option("app_dir")
if sh.isdir(app_dir):
self._build_openstack_package(app_dir)
self._move_rpms("anvil")
# ...and after
sh.deldir(self.rpmbuild_dir)
def package_finish(self):
self._create_repo("anvil")
def _move_rpms(self, repo_name):
repo_dir = sh.joinpths(self.anvil_repo_dir, repo_name)
src_repo_dir = "%s-sources" % repo_dir
sh.mkdir(repo_dir, recurse=True)
sh.mkdir(src_repo_dir, recurse=True)
for filename in sh.listdir(sh.joinpths(self.rpmbuild_dir, "RPMS"),
recursive=True, files_only=True):
sh.move(filename, repo_dir, force=True)
for filename in sh.listdir(sh.joinpths(self.rpmbuild_dir, "SRPMS"),
recursive=True, files_only=True):
sh.move(filename, src_repo_dir, force=True)
return repo_dir
def _create_repo(self, repo_name):
repo_dir = sh.joinpths(self.anvil_repo_dir, repo_name)
src_repo_dir = "%s-sources" % repo_dir
for a_dir in repo_dir, src_repo_dir:
cmdline = ["createrepo", a_dir]
LOG.info("Creating repo at %s" % a_dir)
sh.execute(cmdline)
repo_filename = sh.joinpths(self.anvil_repo_dir, "%s.repo" % repo_name)
LOG.info("Writing %s" % repo_filename)
(_fn, content) = utils.load_template("packaging", "common.repo")
params = {
"repo_name": repo_name,
"baseurl_bin": "file://%s" % repo_dir,
"baseurl_src": "file://%s" % src_repo_dir
}
sh.write_file(
repo_filename, utils.expand_template(content, params))
def _get_yum_available(self):
yum_map = {}
@@ -149,109 +206,6 @@ class YumDependencyHandler(base.DependencyHandler):
def _get_component_name(pkg_dir):
return sh.basename(sh.dirname(pkg_dir))
def _write_all_deps_package(self):
spec_filename = sh.joinpths(
self.rpmbuild_dir,
"SPECS",
"%s.spec" % self.OPENSTACK_DEPS_PACKAGE_NAME)
# Clean out previous dirs.
for dirname in (self.rpmbuild_dir, self.deps_repo_dir,
self.deps_src_repo_dir):
sh.deldir(dirname)
sh.mkdirslist(dirname, tracewriter=self.tracewriter)
def get_version_release():
right_now = datetime.now()
components = [
str(right_now.year),
str(right_now.month),
str(right_now.day),
]
return (".".join(components), right_now.strftime("%s"))
(version, release) = get_version_release()
spec_content = """Name: %s
Version: %s
Release: %s
License: Apache 2.0
Summary: OpenStack dependencies
BuildArch: noarch
""" % (self.OPENSTACK_DEPS_PACKAGE_NAME, version, release)
packages = {}
for inst in self.instances:
try:
for pack in inst.packages:
packages[pack["name"]] = pack
except AttributeError:
pass
scripts = {}
script_map = {
"pre-install": "%pre",
"post-install": "%post",
"pre-uninstall": "%preun",
"post-uninstall": "%postun",
}
for pack_name in sorted(packages.iterkeys()):
pack = packages[pack_name]
cont = [spec_content, "Requires: ", pack["name"]]
version = pack.get("version")
if version:
cont.append(" ")
cont.append(version)
cont.append("\n")
spec_content = "".join(cont)
for script_name in script_map.iterkeys():
try:
script_list = pack[script_name]
except (KeyError, ValueError):
continue
script_body = scripts.get(script_name, "")
script_body = "%s\n# %s\n" % (script_body, pack_name)
for script in script_list:
try:
line = " ".join(
sh.shellquote(word)
for word in script["cmd"])
except (KeyError, ValueError):
continue
if script.get("ignore_failure"):
ignore = " 2>/dev/null || true"
else:
ignore = ""
script_body = "".join((
script_body,
line,
ignore,
"\n"))
scripts[script_name] = script_body
spec_content += "\n%description\n\n"
for script_name in sorted(script_map.iterkeys()):
try:
script_body = scripts[script_name]
except KeyError:
pass
else:
spec_content = "%s\n%s\n%s\n" % (
spec_content,
script_map[script_name],
script_body)
spec_content += "\n%files\n"
sh.write_file(spec_filename, spec_content,
tracewriter=self.tracewriter)
cmdline = [
"rpmbuild", "-ba",
"--define", "_topdir %s" % self.rpmbuild_dir,
spec_filename,
]
LOG.info("Building %s RPM" % self.OPENSTACK_DEPS_PACKAGE_NAME)
sh.execute(cmdline)
def _build_dependencies(self):
(pips_downloaded, package_files) = self.download_dependencies()
@@ -313,44 +267,157 @@ BuildArch: noarch
quiet=True)
p_bar.update(i + 1)
def _build_openstack(self):
if not self.package_dirs:
LOG.warn("No RPM packages of OpenStack installs to build")
return
component_names = [self._get_component_name(d)
for d in self.package_dirs]
utils.log_iterable(sorted(component_names), logger=LOG,
header=("Building %s OpenStack RPM"
" packages") % (len(self.package_dirs)))
with utils.progress_bar(name='Building',
max_am=len(self.package_dirs)) as p_bar:
for (i, pkg_dir) in enumerate(sorted(self.package_dirs)):
component_name = self._get_component_name(pkg_dir)
cmdline = self.py2rpm_start_cmdline() + ["--", pkg_dir]
out_filename = sh.joinpths(self.log_dir,
"py2rpm.%s.out" % (component_name))
sh.execute_save_output(cmdline, out_filename=out_filename,
quiet=True)
p_bar.update(i + 1)
@staticmethod
def _python_setup_py_get(pkg_dir, field):
"""
:param field: e.g., "name" or "version"
"""
cmdline = [sys.executable, "setup.py", "--%s" % field]
value = sh.execute(cmdline, cwd=pkg_dir)[0].splitlines()[-1].strip()
if not value:
LOG.error("Cannot determine %s for %s", field, pkg_dir)
return value
def _create_deps_repo(self):
for filename in sh.listdir(sh.joinpths(self.rpmbuild_dir, "RPMS"),
recursive=True, files_only=True):
sh.move(filename, self.deps_repo_dir, force=True)
for filename in sh.listdir(sh.joinpths(self.rpmbuild_dir, "SRPMS"),
recursive=True, files_only=True):
sh.move(filename, self.deps_src_repo_dir, force=True)
for repo_dir in self.deps_repo_dir, self.deps_src_repo_dir:
cmdline = ["createrepo", repo_dir]
LOG.info("Creating repo at %s" % repo_dir)
sh.execute(cmdline)
LOG.info("Writing %s to %s", self.REPO_FN, self.anvil_repo_filename)
(_fn, content) = utils.load_template('packaging', self.REPO_FN)
params = {"baseurl_bin": "file://%s" % self.deps_repo_dir,
"baseurl_src": "file://%s" % self.deps_src_repo_dir}
sh.write_file(self.anvil_repo_filename,
utils.expand_template(content, params),
tracewriter=self.tracewriter)
def _write_spec_file(self, pkg_dir, rpm_name, template_name, params):
if not params.setdefault("requires", []):
requires_filename = "%s/tools/pip-requires" % pkg_dir
if sh.isfile(requires_filename):
requires_python = []
with open(requires_filename, "r") as requires_file:
for line in requires_file.readlines():
line = line.split("#", 1)[0].strip()
if line:
requires_python.append(line)
if requires_python:
params["requires"] = self._convert_names_python2rpm(
requires_python)
params["epoch"] = self.OPENSTACK_EPOCH
content = utils.load_template(self.SPEC_TEMPLATE_DIR, template_name)[1]
spec_filename = sh.joinpths(
self.rpmbuild_dir, "SPECS", "%s.spec" % rpm_name)
sh.write_file(spec_filename, utils.expand_template(content, params))
return spec_filename
def _copy_startup_scripts(self, spec_filename):
common_init_content = utils.load_template(
"packaging", "common.init")[1]
for src in rpm.spec(spec_filename).sources:
script = sh.basename(src[0])
if not (script.endswith(".init")):
continue
target_filename = sh.joinpths(self.rpm_sources_dir, script)
if sh.isfile(target_filename):
continue
bin_name = utils.strip_prefix_suffix(
script, "openstack-", ".init")
params = {
"bin": bin_name,
"package": bin_name.split("-", 1)[0],
}
sh.write_file(
target_filename,
utils.expand_template(common_init_content, params))
def _copy_sources(self, pkg_dir):
component_name = self._get_component_name(pkg_dir)
other_sources_dir = sh.joinpths(
settings.TEMPLATE_DIR, "packaging/sources", component_name)
if sh.isdir(other_sources_dir):
for filename in sh.listdir(other_sources_dir, files_only=True):
sh.copy(filename, self.rpm_sources_dir)
def _build_from_spec(self, pkg_dir, spec_filename):
if sh.isfile(sh.joinpths(pkg_dir, "setup.py")):
self._write_python_tarball(pkg_dir)
else:
self._write_git_tarball(pkg_dir, spec_filename)
self._copy_sources(pkg_dir)
self._copy_startup_scripts(spec_filename)
cmdline = [
self.rpmbuild_executable,
"-ba",
"--define", "_topdir %s" % self.rpmbuild_dir,
spec_filename,
]
sh.execute_save_output(
cmdline, sh.joinpths(self.log_dir, sh.basename(spec_filename)))
def _write_git_tarball(self, pkg_dir, spec_filename):
cmdline = [
"rpm",
"-q",
"--specfile", spec_filename,
"--qf", "%{NAME}-%{VERSION}\n"
]
tar_base = sh.execute(cmdline, cwd=pkg_dir)[0].splitlines()[0].strip()
# git 1.7.1 from RHEL doesn't understand --format=tar.gz
output_filename = sh.joinpths(
self.rpm_sources_dir, "%s.tar" % tar_base)
cmdline = [
"git",
"archive",
"--format=tar",
"--prefix=%s/" % tar_base,
"--output=%s" % output_filename,
"HEAD",
]
sh.execute(cmdline, cwd=pkg_dir)
cmdline = ["gzip", output_filename]
sh.execute(cmdline)
def _write_python_tarball(self, pkg_dir):
cmdline = [
sys.executable,
"setup.py",
"sdist",
"--formats", "gztar",
"--dist-dir", self.rpm_sources_dir,
]
sh.execute(cmdline, cwd=pkg_dir)
def _build_openstack_package(self, pkg_dir):
component_name = self._get_component_name(pkg_dir)
params = {}
rpm_name = None
template_name = None
if sh.isfile(sh.joinpths(pkg_dir, "setup.py")):
name = self._python_setup_py_get(pkg_dir, "name")
params["version"] = self._python_setup_py_get(pkg_dir, "version")
if component_name.endswith("client"):
clientname = utils.strip_prefix_suffix(
name, "python-", "client")
if not clientname:
LOG.error("Bad client package name %s", name)
return
params["clientname"] = clientname
params["apiname"] = self.API_NAMES.get(
clientname, clientname.title())
rpm_name = name
template_name = "python-commonclient.spec"
elif component_name in self.SERVER_NAMES:
rpm_name = "openstack-%s" % name
elif component_name == "horizon":
rpm_name = "python-django-horizon"
else:
rpm_name = component_name
template_name = "%s.spec" % rpm_name
spec_filename = sh.joinpths(
settings.TEMPLATE_DIR,
self.SPEC_TEMPLATE_DIR,
template_name)
if not sh.isfile(spec_filename):
rpm_name = None
if rpm_name:
template_name = template_name or "%s.spec" % rpm_name
spec_filename = self._write_spec_file(
pkg_dir, rpm_name, template_name, params)
self._build_from_spec(pkg_dir, spec_filename)
else:
cmdline = self.py2rpm_start_cmdline() + ["--", pkg_dir]
sh.execute_save_output(
cmdline,
cwd=pkg_dir,
out_filename=sh.joinpths(self.log_dir, component_name))
def _convert_names_python2rpm(self, python_names):
if not python_names:
@@ -360,24 +427,27 @@ BuildArch: noarch
for name in sh.execute(cmdline)[0].splitlines():
# name is "Requires: rpm-name"
try:
rpm_names.append(name.split(":")[1].strip())
rpm_names.append(name.split(":", 1)[1].strip())
except IndexError:
pass
return rpm_names
def install(self):
super(YumDependencyHandler, self).install()
repo_filename = sh.joinpths(self.YUM_REPO_DIR, self.REPO_FN)
# Ensure we copy the local repo file name to the main repo so that
# yum will find it when installing packages.
sh.write_file(repo_filename, sh.load_file(self.anvil_repo_filename),
tracewriter=self.tracewriter)
for repo_name in "anvil", "anvil-deps":
repo_filename = sh.joinpths(
self.anvil_repo_dir, "%s.repo" % repo_name)
if sh.isfile(repo_filename):
sh.write_file(
"%s/%s.repo" % (self.YUM_REPO_DIR, repo_name),
sh.load_file(repo_filename),
tracewriter=self.tracewriter)
# Erase it if its been previously installed.
cmdline = []
if self.helper.is_installed(self.OPENSTACK_DEPS_PACKAGE_NAME):
cmdline.append(self.OPENSTACK_DEPS_PACKAGE_NAME)
for p in self.nopackages:
if self.helper.is_installed(p):
cmdline.append(p)
@@ -389,13 +459,17 @@ BuildArch: noarch
cmdline = ["yum", "clean", "all"]
sh.execute(cmdline)
cmdline = ["yum", "install", "-y", self.OPENSTACK_DEPS_PACKAGE_NAME]
sh.execute(cmdline, stdout_fh=sys.stdout, stderr_fh=sys.stderr)
rpm_names = []
for inst in self.instances:
for p in inst.package_names():
if p not in self.nopackages:
rpm_names.append(p)
rpm_names = self._convert_names_python2rpm(self.python_names)
if rpm_names:
cmdline = ["yum", "install", "-y"] + rpm_names
cmdline = ["yum", "install", "-y"] + sorted(set(rpm_names))
sh.execute(cmdline, stdout_fh=sys.stdout, stderr_fh=sys.stderr)
for name in rpm_names:
self.tracewriter.package_installed(name)
def uninstall(self):
super(YumDependencyHandler, self).uninstall()
@@ -404,13 +478,14 @@ BuildArch: noarch
no_remove = env.get_key('REQUIRED_PACKAGES', '').split()
no_remove = sorted(set(no_remove))
rpm_names = []
for name in self._convert_names_python2rpm(self.python_names):
if self.helper.is_installed(name) and name not in no_remove:
rpm_names.append(name)
for inst in self.instances:
for p in inst.package_names():
if self.helper.is_installed(p) and p not in no_remove:
rpm_names.append(p)
if rpm_names:
cmdline = ["yum", "remove", "--remove-leaves", "-y"]
for p in no_remove:
cmdline.append("--exclude=%s" % (p))
cmdline.extend(rpm_names)
cmdline.extend(sorted(set(rpm_names)))
sh.execute(cmdline, stdout_fh=sys.stdout, stderr_fh=sys.stderr)