Merge "Ensure that we add in the missing paths."
This commit is contained in:
Jenkins
Gerrit Code Review
commit
84068a9e57
@ -77,6 +77,7 @@ class DependencyHandler(object):
|
||||
self.instances = instances
|
||||
self.opts = opts or {}
|
||||
self.deps_dir = sh.joinpths(self.root_dir, "deps")
|
||||
self.downloaded_flag_file = sh.joinpths(self.deps_dir, "pip-downloaded")
|
||||
self.download_dir = sh.joinpths(self.deps_dir, "download")
|
||||
self.log_dir = sh.joinpths(self.deps_dir, "output")
|
||||
self.gathered_requires_filename = sh.joinpths(
|
||||
@ -302,18 +303,16 @@ class DependencyHandler(object):
|
||||
"""
|
||||
# NOTE(aababilov): do not drop download_dir - it can be reused
|
||||
sh.mkdirslist(self.download_dir, tracewriter=self.tracewriter)
|
||||
download_requires_filename = sh.joinpths(self.deps_dir,
|
||||
"download-requires")
|
||||
download_requires_filename = sh.joinpths(self.deps_dir, "download-requires")
|
||||
raw_pips_to_download = self.filter_download_requires()
|
||||
sh.write_file(download_requires_filename,
|
||||
"\n".join(str(req) for req in raw_pips_to_download))
|
||||
if not raw_pips_to_download:
|
||||
return ([], [])
|
||||
downloaded_flag_file = sh.joinpths(self.deps_dir, "pip-downloaded")
|
||||
# NOTE(aababilov): user could have changed persona, so,
|
||||
# check that all requirements are downloaded
|
||||
if sh.isfile(downloaded_flag_file) and self._requirements_satisfied(
|
||||
raw_pips_to_download, self.download_dir):
|
||||
if (sh.isfile(self.downloaded_flag_file) and
|
||||
self._requirements_satisfied(raw_pips_to_download, self.download_dir)):
|
||||
LOG.info("All python dependencies have been already downloaded")
|
||||
else:
|
||||
pip_dir = sh.joinpths(self.deps_dir, "pip")
|
||||
@ -325,7 +324,7 @@ class DependencyHandler(object):
|
||||
for attempt in xrange(self.MAX_PIP_DOWNLOAD_ATTEMPTS):
|
||||
# NOTE(aababilov): pip has issues with already downloaded files
|
||||
sh.deldir(pip_dir)
|
||||
sh.mkdir(pip_download_dir, recurse=True)
|
||||
sh.mkdirslist(pip_download_dir, tracewriter=self.tracewriter)
|
||||
header = "Downloading %s python dependencies (attempt %s)"
|
||||
header = header % (len(raw_pips_to_download), attempt)
|
||||
utils.log_iterable(sorted(raw_pips_to_download),
|
||||
@ -348,8 +347,8 @@ class DependencyHandler(object):
|
||||
sh.deldir(pip_dir)
|
||||
if pip_failures:
|
||||
raise pip_failures[-1]
|
||||
with open(downloaded_flag_file, "w"):
|
||||
pass
|
||||
sh.touch_file(self.downloaded_flag_file, die_if_there=False,
|
||||
quiet=True, tracewriter=self.tracewriter)
|
||||
pips_downloaded = [pip_helper.extract_requirement(p)
|
||||
for p in raw_pips_to_download]
|
||||
self._examine_download_dir(pips_downloaded, self.download_dir)
|
||||
|
||||
@ -15,11 +15,14 @@
|
||||
# under the License.
|
||||
|
||||
import collections
|
||||
import contextlib
|
||||
import os
|
||||
import pkg_resources
|
||||
import sys
|
||||
|
||||
import pkg_resources
|
||||
import gzip
|
||||
import rpm
|
||||
import tarfile
|
||||
|
||||
from anvil import colorizer
|
||||
from anvil import env
|
||||
@ -34,6 +37,19 @@ from anvil import utils
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
# Certain versions of pbr seem to miss these files, which causes the rpmbuild
|
||||
# phases to not complete correctly. Ensure that we don't miss them.
|
||||
ENSURE_NOT_MISSING = [
|
||||
'doc', # Without this one our rpm doc build won't work
|
||||
'README.rst', # Without this one pbr won't work (thus killing setup.py)
|
||||
'babel.cfg',
|
||||
'HACKING',
|
||||
'AUTHORS',
|
||||
'ChangeLog',
|
||||
'CONTRIBUTING.rst',
|
||||
'LICENSE',
|
||||
]
|
||||
|
||||
|
||||
class YumInstallHelper(base.InstallHelper):
|
||||
def pre_install(self, pkg, params=None):
|
||||
@ -139,7 +155,7 @@ class YumDependencyHandler(base.DependencyHandler):
|
||||
def _create_rpmbuild_subdirs(self):
|
||||
for dirname in (sh.joinpths(self.rpmbuild_dir, "SPECS"),
|
||||
sh.joinpths(self.rpmbuild_dir, "SOURCES")):
|
||||
sh.mkdirslist(dirname)
|
||||
sh.mkdirslist(dirname, tracewriter=self.tracewriter)
|
||||
|
||||
def package_instance(self, instance):
|
||||
with sh.remove_before_after(self.rpmbuild_dir):
|
||||
@ -189,13 +205,11 @@ class YumDependencyHandler(base.DependencyHandler):
|
||||
if not src_repo_files:
|
||||
continue
|
||||
src_repo_base_files = [sh.basename(f) for f in src_repo_files]
|
||||
header = 'Building %s RPM packages from their SRPMs for repo %s using %s jobs'
|
||||
header = header % (len(src_repo_files), self.SRC_REPOS[repo_name], self.jobs)
|
||||
utils.log_iterable(src_repo_base_files, header=header, logger=LOG)
|
||||
|
||||
LOG.info('Building %s RPM packages from their SRPMs for repo %s using %s jobs',
|
||||
len(src_repo_files), self.SRC_REPOS[repo_name], self.jobs)
|
||||
makefile_name = sh.joinpths(self.deps_dir, "binary-%s.mk" % repo_name)
|
||||
marks_dir = sh.joinpths(self.deps_dir, "marks-binary")
|
||||
sh.mkdirslist(marks_dir)
|
||||
sh.mkdirslist(marks_dir, tracewriter=self.tracewriter)
|
||||
(_fn, content) = utils.load_template("packaging/makefiles", "binary.mk")
|
||||
rpmbuild_flags = ("--rebuild --define '_topdir %s'" % self.rpmbuild_dir)
|
||||
if self.opts.get("usr_only", False):
|
||||
@ -347,13 +361,17 @@ class YumDependencyHandler(base.DependencyHandler):
|
||||
return filtered_files
|
||||
|
||||
LOG.info("Filtering %s downloaded files.", len(package_files))
|
||||
package_files = _filter_package_files(package_files)
|
||||
if not package_files:
|
||||
filtered_package_files = _filter_package_files(package_files)
|
||||
if not filtered_package_files:
|
||||
LOG.info("No SRPM package dependencies to build.")
|
||||
return
|
||||
for filename in package_files:
|
||||
if filename not in filtered_package_files:
|
||||
sh.unlink(filename)
|
||||
package_files = filtered_package_files
|
||||
makefile_name = sh.joinpths(self.deps_dir, "deps.mk")
|
||||
marks_dir = sh.joinpths(self.deps_dir, "marks-deps")
|
||||
sh.mkdirslist(marks_dir)
|
||||
sh.mkdirslist(marks_dir, tracewriter=self.tracewriter)
|
||||
(_fn, content) = utils.load_template("packaging/makefiles", "source.mk")
|
||||
scripts_dir = sh.abspth(sh.joinpths(settings.TEMPLATE_DIR, "packaging", "scripts"))
|
||||
py2rpm_options = self.py2rpm_start_cmdline()[1:] + [
|
||||
@ -370,11 +388,7 @@ class YumDependencyHandler(base.DependencyHandler):
|
||||
sh.write_file(makefile_name,
|
||||
utils.expand_template(content, params),
|
||||
tracewriter=self.tracewriter)
|
||||
base_package_files = [sh.basename(f) for f in package_files]
|
||||
utils.log_iterable(base_package_files,
|
||||
header="Building %s SRPM packages using %s jobs" %
|
||||
(len(package_files), self.jobs),
|
||||
logger=LOG)
|
||||
LOG.info("Building %s SRPM packages using %s jobs", len(package_files), self.jobs)
|
||||
self._execute_make(makefile_name, marks_dir)
|
||||
|
||||
def _write_spec_file(self, instance, rpm_name, template_name, params):
|
||||
@ -440,7 +454,7 @@ class YumDependencyHandler(base.DependencyHandler):
|
||||
def _build_from_spec(self, instance, spec_filename, patches=None):
|
||||
pkg_dir = instance.get_option('app_dir')
|
||||
if sh.isfile(sh.joinpths(pkg_dir, "setup.py")):
|
||||
self._write_python_tarball(pkg_dir)
|
||||
self._write_python_tarball(instance, pkg_dir, ENSURE_NOT_MISSING)
|
||||
else:
|
||||
self._write_git_tarball(pkg_dir, spec_filename)
|
||||
self._copy_sources(instance)
|
||||
@ -479,15 +493,48 @@ class YumDependencyHandler(base.DependencyHandler):
|
||||
cmdline = ["gzip", output_filename]
|
||||
sh.execute(cmdline)
|
||||
|
||||
def _write_python_tarball(self, pkg_dir):
|
||||
def _write_python_tarball(self, instance, pkg_dir, ensure_exists=None):
|
||||
|
||||
def prefix_exists(text, in_what):
|
||||
for t in in_what:
|
||||
if t.startswith(text):
|
||||
return True
|
||||
return False
|
||||
|
||||
pkg_name = instance.egg_info['name']
|
||||
version = instance.egg_info['version']
|
||||
base_name = "%s-%s" % (pkg_name, version)
|
||||
cmdline = [
|
||||
sys.executable,
|
||||
"setup.py",
|
||||
"sdist",
|
||||
"--formats", "gztar",
|
||||
"--formats=tar",
|
||||
"--dist-dir", self.rpm_sources_dir,
|
||||
]
|
||||
sh.execute(cmdline, cwd=pkg_dir)
|
||||
out_filename = sh.joinpths(self.log_dir, "sdist-%s.log" % (instance.name))
|
||||
sh.execute_save_output(cmdline, cwd=pkg_dir, out_filename=out_filename, quiet=True)
|
||||
archive_name = sh.joinpths(self.rpm_sources_dir, "%s.tar" % (base_name))
|
||||
if ensure_exists:
|
||||
with contextlib.closing(tarfile.open(archive_name, 'r')) as tfh:
|
||||
tar_entries = [t.path for t in tfh.getmembers()]
|
||||
missing_paths = {}
|
||||
for path in ensure_exists:
|
||||
tar_path = sh.joinpths(base_name, path)
|
||||
source_path = sh.joinpths(pkg_dir, path)
|
||||
if not prefix_exists(tar_path, tar_entries) and sh.exists(source_path):
|
||||
missing_paths[tar_path] = source_path
|
||||
if missing_paths:
|
||||
utils.log_iterable(sorted(missing_paths.keys()),
|
||||
logger=LOG,
|
||||
header='%s paths were not archived and will now be' % (len(missing_paths)))
|
||||
with contextlib.closing(tarfile.open(archive_name, 'a')) as tfh:
|
||||
for (tar_path, source_path) in missing_paths.items():
|
||||
tfh.add(source_path, tar_path)
|
||||
gz_archive_name = "%s.gz" % (archive_name)
|
||||
with contextlib.closing(gzip.open(gz_archive_name, 'wb')) as tz:
|
||||
with open(archive_name, 'rb') as fh:
|
||||
tz.write(fh.read())
|
||||
sh.unlink(archive_name)
|
||||
|
||||
@staticmethod
|
||||
def _is_client(instance_name, egg_name):
|
||||
|
||||
@ -29,5 +29,5 @@ $(BUILDDEP_MARK):
|
||||
%.mark: $(SRC_REPO_DIR)/% $(BUILDDEP_MARK)
|
||||
@$(RPMBUILD) $(RPMBUILD_FLAGS) -- $< &> $(LOGS_DIR)/rpmbuild-$*.log
|
||||
@touch "$@"
|
||||
@echo "$* is processed"
|
||||
@echo "$* has been processed."
|
||||
#end raw
|
||||
|
||||
@ -13,5 +13,5 @@ all: $(MARKS)
|
||||
%.mark: $(DOWNLOADS_DIR)/%
|
||||
@$(PY2RPM) $(PY2RPM_FLAGS) -- $^ &> $(LOGS_DIR)/py2rpm-$*.log
|
||||
@touch "$@"
|
||||
@echo "$* is processed"
|
||||
@echo "$* has been processed."
|
||||
#end raw
|
||||
|
||||
Loading…
Reference in New Issue
Block a user