Ensure that we add in the missing paths.
Certain openstack components when built with sdist seem to be missing, which will break either pbr (and setup.py) or break doc building or i8ln building so make sure that we pickup these paths if they are not automatically included. Also include a few tracewriter tracking additions and few wording adjustments. Fixes bug: #1195842 Change-Id: I0ce68db9aa28fcc3b839c7583468927d0e2647c6
This commit is contained in:
Joshua Harlow
parent
b504addad5
commit
1075c1dc8e
@ -77,6 +77,7 @@ class DependencyHandler(object):
|
||||
self.instances = instances
|
||||
self.opts = opts or {}
|
||||
self.deps_dir = sh.joinpths(self.root_dir, "deps")
|
||||
self.downloaded_flag_file = sh.joinpths(self.deps_dir, "pip-downloaded")
|
||||
self.download_dir = sh.joinpths(self.deps_dir, "download")
|
||||
self.log_dir = sh.joinpths(self.deps_dir, "output")
|
||||
self.gathered_requires_filename = sh.joinpths(
|
||||
@ -302,18 +303,16 @@ class DependencyHandler(object):
|
||||
"""
|
||||
# NOTE(aababilov): do not drop download_dir - it can be reused
|
||||
sh.mkdirslist(self.download_dir, tracewriter=self.tracewriter)
|
||||
download_requires_filename = sh.joinpths(self.deps_dir,
|
||||
"download-requires")
|
||||
download_requires_filename = sh.joinpths(self.deps_dir, "download-requires")
|
||||
raw_pips_to_download = self.filter_download_requires()
|
||||
sh.write_file(download_requires_filename,
|
||||
"\n".join(str(req) for req in raw_pips_to_download))
|
||||
if not raw_pips_to_download:
|
||||
return ([], [])
|
||||
downloaded_flag_file = sh.joinpths(self.deps_dir, "pip-downloaded")
|
||||
# NOTE(aababilov): user could have changed persona, so,
|
||||
# check that all requirements are downloaded
|
||||
if sh.isfile(downloaded_flag_file) and self._requirements_satisfied(
|
||||
raw_pips_to_download, self.download_dir):
|
||||
if (sh.isfile(self.downloaded_flag_file) and
|
||||
self._requirements_satisfied(raw_pips_to_download, self.download_dir)):
|
||||
LOG.info("All python dependencies have been already downloaded")
|
||||
else:
|
||||
pip_dir = sh.joinpths(self.deps_dir, "pip")
|
||||
@ -325,7 +324,7 @@ class DependencyHandler(object):
|
||||
for attempt in xrange(self.MAX_PIP_DOWNLOAD_ATTEMPTS):
|
||||
# NOTE(aababilov): pip has issues with already downloaded files
|
||||
sh.deldir(pip_dir)
|
||||
sh.mkdir(pip_download_dir, recurse=True)
|
||||
sh.mkdirslist(pip_download_dir, tracewriter=self.tracewriter)
|
||||
header = "Downloading %s python dependencies (attempt %s)"
|
||||
header = header % (len(raw_pips_to_download), attempt)
|
||||
utils.log_iterable(sorted(raw_pips_to_download),
|
||||
@ -348,8 +347,8 @@ class DependencyHandler(object):
|
||||
sh.deldir(pip_dir)
|
||||
if pip_failures:
|
||||
raise pip_failures[-1]
|
||||
with open(downloaded_flag_file, "w"):
|
||||
pass
|
||||
sh.touch_file(self.downloaded_flag_file, die_if_there=False,
|
||||
quiet=True, tracewriter=self.tracewriter)
|
||||
pips_downloaded = [pip_helper.extract_requirement(p)
|
||||
for p in raw_pips_to_download]
|
||||
self._examine_download_dir(pips_downloaded, self.download_dir)
|
||||
|
||||
@ -15,11 +15,14 @@
|
||||
# under the License.
|
||||
|
||||
import collections
|
||||
import contextlib
|
||||
import os
|
||||
import pkg_resources
|
||||
import sys
|
||||
|
||||
import pkg_resources
|
||||
import gzip
|
||||
import rpm
|
||||
import tarfile
|
||||
|
||||
from anvil import colorizer
|
||||
from anvil import env
|
||||
@ -34,6 +37,19 @@ from anvil import utils
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
# Certain versions of pbr seem to miss these files, which causes the rpmbuild
|
||||
# phases to not complete correctly. Ensure that we don't miss them.
|
||||
ENSURE_NOT_MISSING = [
|
||||
'doc', # Without this one our rpm doc build won't work
|
||||
'README.rst', # Without this one pbr won't work (thus killing setup.py)
|
||||
'babel.cfg',
|
||||
'HACKING',
|
||||
'AUTHORS',
|
||||
'ChangeLog',
|
||||
'CONTRIBUTING.rst',
|
||||
'LICENSE',
|
||||
]
|
||||
|
||||
|
||||
class YumInstallHelper(base.InstallHelper):
|
||||
def pre_install(self, pkg, params=None):
|
||||
@ -139,7 +155,7 @@ class YumDependencyHandler(base.DependencyHandler):
|
||||
def _create_rpmbuild_subdirs(self):
|
||||
for dirname in (sh.joinpths(self.rpmbuild_dir, "SPECS"),
|
||||
sh.joinpths(self.rpmbuild_dir, "SOURCES")):
|
||||
sh.mkdirslist(dirname)
|
||||
sh.mkdirslist(dirname, tracewriter=self.tracewriter)
|
||||
|
||||
def package_instance(self, instance):
|
||||
with sh.remove_before_after(self.rpmbuild_dir):
|
||||
@ -189,13 +205,11 @@ class YumDependencyHandler(base.DependencyHandler):
|
||||
if not src_repo_files:
|
||||
continue
|
||||
src_repo_base_files = [sh.basename(f) for f in src_repo_files]
|
||||
header = 'Building %s RPM packages from their SRPMs for repo %s using %s jobs'
|
||||
header = header % (len(src_repo_files), self.SRC_REPOS[repo_name], self.jobs)
|
||||
utils.log_iterable(src_repo_base_files, header=header, logger=LOG)
|
||||
|
||||
LOG.info('Building %s RPM packages from their SRPMs for repo %s using %s jobs',
|
||||
len(src_repo_files), self.SRC_REPOS[repo_name], self.jobs)
|
||||
makefile_name = sh.joinpths(self.deps_dir, "binary-%s.mk" % repo_name)
|
||||
marks_dir = sh.joinpths(self.deps_dir, "marks-binary")
|
||||
sh.mkdirslist(marks_dir)
|
||||
sh.mkdirslist(marks_dir, tracewriter=self.tracewriter)
|
||||
(_fn, content) = utils.load_template("packaging/makefiles", "binary.mk")
|
||||
rpmbuild_flags = ("--rebuild --define '_topdir %s'" % self.rpmbuild_dir)
|
||||
if self.opts.get("usr_only", False):
|
||||
@ -347,13 +361,17 @@ class YumDependencyHandler(base.DependencyHandler):
|
||||
return filtered_files
|
||||
|
||||
LOG.info("Filtering %s downloaded files.", len(package_files))
|
||||
package_files = _filter_package_files(package_files)
|
||||
if not package_files:
|
||||
filtered_package_files = _filter_package_files(package_files)
|
||||
if not filtered_package_files:
|
||||
LOG.info("No SRPM package dependencies to build.")
|
||||
return
|
||||
for filename in package_files:
|
||||
if filename not in filtered_package_files:
|
||||
sh.unlink(filename)
|
||||
package_files = filtered_package_files
|
||||
makefile_name = sh.joinpths(self.deps_dir, "deps.mk")
|
||||
marks_dir = sh.joinpths(self.deps_dir, "marks-deps")
|
||||
sh.mkdirslist(marks_dir)
|
||||
sh.mkdirslist(marks_dir, tracewriter=self.tracewriter)
|
||||
(_fn, content) = utils.load_template("packaging/makefiles", "source.mk")
|
||||
scripts_dir = sh.abspth(sh.joinpths(settings.TEMPLATE_DIR, "packaging", "scripts"))
|
||||
py2rpm_options = self.py2rpm_start_cmdline()[1:] + [
|
||||
@ -370,11 +388,7 @@ class YumDependencyHandler(base.DependencyHandler):
|
||||
sh.write_file(makefile_name,
|
||||
utils.expand_template(content, params),
|
||||
tracewriter=self.tracewriter)
|
||||
base_package_files = [sh.basename(f) for f in package_files]
|
||||
utils.log_iterable(base_package_files,
|
||||
header="Building %s SRPM packages using %s jobs" %
|
||||
(len(package_files), self.jobs),
|
||||
logger=LOG)
|
||||
LOG.info("Building %s SRPM packages using %s jobs", len(package_files), self.jobs)
|
||||
self._execute_make(makefile_name, marks_dir)
|
||||
|
||||
def _write_spec_file(self, instance, rpm_name, template_name, params):
|
||||
@ -428,7 +442,7 @@ class YumDependencyHandler(base.DependencyHandler):
|
||||
def _build_from_spec(self, instance, spec_filename, patches=None):
|
||||
pkg_dir = instance.get_option('app_dir')
|
||||
if sh.isfile(sh.joinpths(pkg_dir, "setup.py")):
|
||||
self._write_python_tarball(pkg_dir)
|
||||
self._write_python_tarball(instance, pkg_dir, ENSURE_NOT_MISSING)
|
||||
else:
|
||||
self._write_git_tarball(pkg_dir, spec_filename)
|
||||
self._copy_sources(instance)
|
||||
@ -467,15 +481,48 @@ class YumDependencyHandler(base.DependencyHandler):
|
||||
cmdline = ["gzip", output_filename]
|
||||
sh.execute(cmdline)
|
||||
|
||||
def _write_python_tarball(self, pkg_dir):
|
||||
def _write_python_tarball(self, instance, pkg_dir, ensure_exists=None):
|
||||
|
||||
def prefix_exists(text, in_what):
|
||||
for t in in_what:
|
||||
if t.startswith(text):
|
||||
return True
|
||||
return False
|
||||
|
||||
pkg_name = instance.egg_info['name']
|
||||
version = instance.egg_info['version']
|
||||
base_name = "%s-%s" % (pkg_name, version)
|
||||
cmdline = [
|
||||
sys.executable,
|
||||
"setup.py",
|
||||
"sdist",
|
||||
"--formats", "gztar",
|
||||
"--formats=tar",
|
||||
"--dist-dir", self.rpm_sources_dir,
|
||||
]
|
||||
sh.execute(cmdline, cwd=pkg_dir)
|
||||
out_filename = sh.joinpths(self.log_dir, "sdist-%s.log" % (instance.name))
|
||||
sh.execute_save_output(cmdline, cwd=pkg_dir, out_filename=out_filename, quiet=True)
|
||||
archive_name = sh.joinpths(self.rpm_sources_dir, "%s.tar" % (base_name))
|
||||
if ensure_exists:
|
||||
with contextlib.closing(tarfile.open(archive_name, 'r')) as tfh:
|
||||
tar_entries = [t.path for t in tfh.getmembers()]
|
||||
missing_paths = {}
|
||||
for path in ensure_exists:
|
||||
tar_path = sh.joinpths(base_name, path)
|
||||
source_path = sh.joinpths(pkg_dir, path)
|
||||
if not prefix_exists(tar_path, tar_entries) and sh.exists(source_path):
|
||||
missing_paths[tar_path] = source_path
|
||||
if missing_paths:
|
||||
utils.log_iterable(sorted(missing_paths.keys()),
|
||||
logger=LOG,
|
||||
header='%s paths were not archived and will now be' % (len(missing_paths)))
|
||||
with contextlib.closing(tarfile.open(archive_name, 'a')) as tfh:
|
||||
for (tar_path, source_path) in missing_paths.items():
|
||||
tfh.add(source_path, tar_path)
|
||||
gz_archive_name = "%s.gz" % (archive_name)
|
||||
with contextlib.closing(gzip.open(gz_archive_name, 'wb')) as tz:
|
||||
with open(archive_name, 'rb') as fh:
|
||||
tz.write(fh.read())
|
||||
sh.unlink(archive_name)
|
||||
|
||||
@staticmethod
|
||||
def _is_client(instance_name, egg_name):
|
||||
|
||||
@ -29,5 +29,5 @@ $(BUILDDEP_MARK):
|
||||
%.mark: $(SRC_REPO_DIR)/% $(BUILDDEP_MARK)
|
||||
@$(RPMBUILD) $(RPMBUILD_FLAGS) -- $< &> $(LOGS_DIR)/rpmbuild-$*.log
|
||||
@touch "$@"
|
||||
@echo "$* is processed"
|
||||
@echo "$* has been processed."
|
||||
#end raw
|
||||
|
||||
@ -13,5 +13,5 @@ all: $(MARKS)
|
||||
%.mark: $(DOWNLOADS_DIR)/%
|
||||
@$(PY2RPM) $(PY2RPM_FLAGS) -- $^ &> $(LOGS_DIR)/py2rpm-$*.log
|
||||
@touch "$@"
|
||||
@echo "$* is processed"
|
||||
@echo "$* has been processed."
|
||||
#end raw
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user