starlingx/root
Code Issues Proposed changes
5c1e7e7b75
root/build-tools/mockchain-parallel-2.6
Scott Little 77576b7207 Add support CentOS-8/dnf/mock-2.6 based builds 
This update will retain support for CentOS-7/yum/mock-1.4 based builds.
The build environment will be queried to discover which environment
it is building in, and modify the commands we issue accordingly.

In CentOS 8, DNF replaces both YUM and REPOQUERY.
While DNF tries to be a transparent replacement of the old tools,
there are also subtle changes to the supported arguments.

I will provide independent mock.cfg.prototypes for centos7 vs centos8.
Changes in generate-centos-repo.sh under stx-tools will be required to
select the correct prototype.

Add support for mock 2.6. Mock 2.6 is python 3, and it processes the
'root' and 'rootdir' arguments slightly differently.

Also change the order of arguments to tar within default_build_srpm.
The latest tar only honors '--exclude' if it precedes other arguments.

Story: 2006729
Depends-On: https://review.opendev.org/762700
Signed-off-by: Scott Little <scott.little@windriver.com>
Change-Id: I826be2051e535e6a4c08ad17124f453b04210668
2020-12-08 14:13:28 -05:00

1222 lines
44 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/python3 -tt
# -*- coding: utf-8 -*-
# vim: noai:ts=4:sw=4:expandtab
# by skvidal@fedoraproject.org
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
# copyright 2012 Red Hat, Inc.
# SUMMARY
# mockchain
# take a mock config and a series of srpms
# rebuild them one at a time
# adding each to a local repo
# so they are available as build deps to next pkg being built
from __future__ import print_function
import cgi
# pylint: disable=deprecated-module
import optparse
import os
import re
import shutil
import subprocess
import sys
import tempfile
import time
import multiprocessing
import signal
import psutil
import requests
# pylint: disable=import-error
from six.moves.urllib_parse import urlsplit
import mockbuild.util
from stxRpmUtils import splitRpmFilename
# all of the variables below are substituted by the build system
__VERSION__="2.6"
SYSCONFDIR="/etc"
PYTHONDIR="/usr/lib/python3.6/site-packages"
PKGPYTHONDIR="/usr/lib/python3.6/site-packages/mockbuild"
MOCKCONFDIR = os.path.join(SYSCONFDIR, "mock")
# end build system subs
mockconfig_path = '/etc/mock'
def rpmName(path):
filename = os.path.basename(path)
(n, v, r, e, a) = splitRpmFilename(filename)
return n
def createrepo(path):
global max_workers
if os.path.exists(path + '/repodata/repomd.xml'):
comm = ['/usr/bin/createrepo_c', '--update', '--retain-old-md', "%d" % max_workers, "--workers", "%d" % max_workers, path]
else:
comm = ['/usr/bin/createrepo_c', "--workers", "%d" % max_workers, path]
cmd = subprocess.Popen(
comm, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = cmd.communicate()
return out, err
g_opts = optparse.Values()
def parse_args(args):
parser = optparse.OptionParser('\nmockchain -r mockcfg pkg1 [pkg2] [pkg3]')
parser.add_option(
'-r', '--root', default=None, dest='chroot',
metavar="CONFIG",
help="chroot config name/base to use in the mock build")
parser.add_option(
'-l', '--localrepo', default=None,
help="local path for the local repo, defaults to making its own")
parser.add_option(
'-c', '--continue', default=False, action='store_true',
dest='cont',
help="if a pkg fails to build, continue to the next one")
parser.add_option(
'-a', '--addrepo', default=[], action='append',
dest='repos',
help="add these repo baseurls to the chroot's yum config")
parser.add_option(
'--recurse', default=False, action='store_true',
help="if more than one pkg and it fails to build, try to build the rest and come back to it")
parser.add_option(
'--log', default=None, dest='logfile',
help="log to the file named by this option, defaults to not logging")
parser.add_option(
'--workers', default=1, dest='max_workers',
help="number of parallel build jobs")
parser.add_option(
'--worker-resources', default="", dest='worker_resources',
help="colon seperated list, how much mem in gb for each workers temfs")
parser.add_option(
'--basedir', default='/var/lib/mock', dest='basedir',
help="path to workspace")
parser.add_option(
'--tmp_prefix', default=None, dest='tmp_prefix',
help="tmp dir prefix - will default to username-pid if not specified")
parser.add_option(
'-m', '--mock-option', default=[], action='append',
dest='mock_option',
help="option to pass directly to mock")
parser.add_option(
'--mark-slow-name', default=[], action='append',
dest='slow_pkg_names_raw',
help="package name that is known to build slowly")
parser.add_option(
'--mark-slow-path', default=[], action='append',
dest='slow_pkgs_raw',
help="package path that is known to build slowly")
parser.add_option(
'--mark-big-name', default=[], action='append',
dest='big_pkg_names_raw',
help="package name that is known to require a lot of disk space to build")
parser.add_option(
'--mark-big-path', default=[], action='append',
dest='big_pkgs_raw',
help="package path that is known to require a lot of disk space to build")
parser.add_option(
'--srpm-dependency-file', default=None,
dest='srpm_dependency_file',
help="path to srpm dependency file")
parser.add_option(
'--rpm-dependency-file', default=None,
dest='rpm_dependency_file',
help="path to rpm dependency file")
parser.add_option(
'--rpm-to-srpm-map-file', default=None,
dest='rpm_to_srpm_map_file',
help="path to rpm to srpm map file")
opts, args = parser.parse_args(args)
if opts.recurse:
opts.cont = True
if not opts.chroot:
print("You must provide an argument to -r for the mock chroot")
sys.exit(1)
if len(sys.argv) < 3:
print("You must specify at least 1 package to build")
sys.exit(1)
return opts, args
REPOS_ID = []
slow_pkg_names={}
slow_pkgs={}
big_pkg_names={}
big_pkgs={}
def generate_repo_id(baseurl):
""" generate repository id for yum.conf out of baseurl """
repoid = "/".join(baseurl.split('//')[1:]).replace('/', '_')
repoid = re.sub(r'[^a-zA-Z0-9_]', '', repoid)
suffix = ''
i = 1
while repoid + suffix in REPOS_ID:
suffix = str(i)
i += 1
repoid = repoid + suffix
REPOS_ID.append(repoid)
return repoid
def set_build_idx(infile, destfile, build_idx, tmpfs_size_gb, opts):
# log(opts.logfile, "set_build_idx: infile=%s, destfile=%s, build_idx=%d, tmpfs_size_gb=%d" % (infile, destfile, build_idx, tmpfs_size_gb))
try:
with open(infile) as f:
code = compile(f.read(), infile, 'exec')
# pylint: disable=exec-used
exec(code)
config_opts['root'] = config_opts['root'].replace('b0', 'b{0}'.format(build_idx))
config_opts['rootdir'] = config_opts['rootdir'].replace('b0', 'b{0}'.format(build_idx))
config_opts['cache_topdir'] = config_opts['cache_topdir'].replace('b0', 'b{0}'.format(build_idx))
# log(opts.logfile, "set_build_idx: root=%s" % config_opts['root'])
# log(opts.logfile, "set_build_idx: cache_topdir=%s" % config_opts['cache_topdir'])
if tmpfs_size_gb > 0:
config_opts['plugin_conf']['tmpfs_enable'] = True
config_opts['plugin_conf']['tmpfs_opts'] = {}
config_opts['plugin_conf']['tmpfs_opts']['required_ram_mb'] = 1024
config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = "%dg" % tmpfs_size_gb
config_opts['plugin_conf']['tmpfs_opts']['mode'] = '0755'
config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = True
# log(opts.logfile, "set_build_idx: plugin_conf->tmpfs_enable=%s" % config_opts['plugin_conf']['tmpfs_enable'])
# log(opts.logfile, "set_build_idx: plugin_conf->tmpfs_opts->max_fs_size=%s" % config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'])
with open(destfile, 'w') as br_dest:
for k, v in list(config_opts.items()):
br_dest.write("config_opts[%r] = %r\n" % (k, v))
try:
log(opts.logfile, "set_build_idx: os.makedirs %s" % config_opts['cache_topdir'])
if not os.path.isdir(config_opts['cache_topdir']):
os.makedirs(config_opts['cache_topdir'], exist_ok=True)
except (IOError, OSError):
return False, "Could not create dir: %s" % config_opts['cache_topdir']
cache_dir = "%s/%s/mock" % (config_opts['basedir'], config_opts['root'])
try:
log(opts.logfile, "set_build_idx: os.makedirs %s" % cache_dir)
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
except (IOError, OSError):
return False, "Could not create dir: %s" % cache_dir
return True, ''
except (IOError, OSError):
return False, "Could not write mock config to %s" % destfile
return True, ''
def set_basedir(infile, destfile, basedir, opts):
log(opts.logfile, "set_basedir: infile=%s, destfile=%s, basedir=%s" % (infile, destfile, basedir))
try:
with open(infile) as f:
code = compile(f.read(), infile, 'exec')
# pylint: disable=exec-used
exec(code)
config_opts['basedir'] = basedir
config_opts['resultdir'] = '{0}/result'.format(basedir)
config_opts['backup_base_dir'] = '{0}/backup'.format(basedir)
config_opts['root'] = 'mock/b0'
config_opts['cache_topdir'] = '{0}/cache/b0'.format(basedir)
config_opts['rootdir'] = '{0}/mock/b0/root'.format(basedir)
with open(destfile, 'w') as br_dest:
for k, v in list(config_opts.items()):
br_dest.write("config_opts[%r] = %r\n" % (k, v))
return True, ''
except (IOError, OSError):
return False, "Could not write mock config to %s" % destfile
return True, ''
def add_local_repo(infile, destfile, baseurl, repoid=None):
"""take a mock chroot config and add a repo to it's yum.conf
infile = mock chroot config file
destfile = where to save out the result
baseurl = baseurl of repo you wish to add"""
global config_opts
try:
with open(infile) as f:
code = compile(f.read(), infile, 'exec')
# pylint: disable=exec-used
exec(code)
if not repoid:
repoid = generate_repo_id(baseurl)
else:
REPOS_ID.append(repoid)
localyumrepo = """
[%s]
name=%s
baseurl=%s
enabled=1
skip_if_unavailable=1
metadata_expire=0
cost=1
best=1
""" % (repoid, baseurl, baseurl)
config_opts['yum.conf'] += localyumrepo
with open(destfile, 'w') as br_dest:
for k, v in list(config_opts.items()):
br_dest.write("config_opts[%r] = %r\n" % (k, v))
return True, ''
except (IOError, OSError):
return False, "Could not write mock config to %s" % destfile
return True, ''
def do_build(opts, cfg, pkg):
# returns 0, cmd, out, err = failure
# returns 1, cmd, out, err = success
# returns 2, None, None, None = already built
signal.signal(signal.SIGTERM, child_signal_handler)
signal.signal(signal.SIGINT, child_signal_handler)
signal.signal(signal.SIGHUP, child_signal_handler)
signal.signal(signal.SIGABRT, child_signal_handler)
s_pkg = os.path.basename(pkg)
pdn = s_pkg.replace('.src.rpm', '')
resdir = '%s/%s' % (opts.local_repo_dir, pdn)
resdir = os.path.normpath(resdir)
if not os.path.exists(resdir):
os.makedirs(resdir)
success_file = resdir + '/success'
fail_file = resdir + '/fail'
if os.path.exists(success_file):
# return 2, None, None, None
sys.exit(2)
# clean it up if we're starting over :)
if os.path.exists(fail_file):
os.unlink(fail_file)
if opts.uniqueext == '':
mockcmd = ['/usr/bin/mock',
'--configdir', opts.config_path,
'--resultdir', resdir,
'--root', cfg, ]
else:
mockcmd = ['/usr/bin/mock',
'--configdir', opts.config_path,
'--resultdir', resdir,
'--uniqueext', opts.uniqueext,
'--root', cfg, ]
# Ensure repo is up-to-date.
# Note: Merely adding --update to mockcmd failed to update
mockcmd_update=mockcmd
mockcmd_update.append('--update')
cmd = subprocess.Popen(
mockcmd_update, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = cmd.communicate()
if cmd.returncode != 0:
if (isinstance(err, bytes)):
err = err.decode("utf-8")
sys.stderr.write(err)
# heuristic here, if user pass for mock "-d foo", but we must be care to leave
# "-d'foo bar'" or "--define='foo bar'" as is
compiled_re_1 = re.compile(r'^(-\S)\s+(.+)')
compiled_re_2 = re.compile(r'^(--[^ =])[ =](\.+)')
for option in opts.mock_option:
r_match = compiled_re_1.match(option)
if r_match:
mockcmd.extend([r_match.group(1), r_match.group(2)])
else:
r_match = compiled_re_2.match(option)
if r_match:
mockcmd.extend([r_match.group(1), r_match.group(2)])
else:
mockcmd.append(option)
print('building %s' % s_pkg)
mockcmd.append(pkg)
cmd = subprocess.Popen(
mockcmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = cmd.communicate()
if cmd.returncode == 0:
with open(success_file, 'w') as f:
f.write('done\n')
ret = 1
else:
if (isinstance(err, bytes)):
err = err.decode("utf-8")
sys.stderr.write(err)
with open(fail_file, 'w') as f:
f.write('undone\n')
ret = 0
# return ret, cmd, out, err
sys.exit(ret)
def log(lf, msg):
if lf:
now = time.time()
try:
with open(lf, 'a') as f:
f.write(str(now) + ':' + msg + '\n')
except (IOError, OSError) as e:
print('Could not write to logfile %s - %s' % (lf, str(e)))
print(msg)
config_opts = {}
worker_data = []
workers = 0
max_workers = 1
build_env = []
failed = []
built_pkgs = []
local_repo_dir = ""
pkg_to_name={}
name_to_pkg={}
srpm_dependencies_direct={}
rpm_dependencies_direct={}
rpm_to_srpm_map={}
no_dep_list = [ "bash", "kernel" , "kernel-rt" ]
def init_build_env(slots, opts, config_opts_in):
global build_env
orig_chroot_name=config_opts_in['chroot_name']
orig_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(orig_chroot_name))
# build_env.append({'state': 'Idle', 'cfg': orig_mock_config})
for i in range(0,slots):
new_chroot_name = "{0}.b{1}".format(orig_chroot_name, i)
new_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(new_chroot_name))
tmpfs_size_gb = 0
if opts.worker_resources == "":
if i > 0:
tmpfs_size_gb = 2 * (1 + slots - i)
else:
resource_array=opts.worker_resources.split(':')
if i < len(resource_array):
tmpfs_size_gb=int(resource_array[i])
else:
log(opts.logfile, "Error: worker-resources argument '%s' does not supply info for all %d workers" % (opts.worker_resources, slots))
sys.exit(1)
if i == 0 and tmpfs_size_gb != 0:
log(opts.logfile, "Error: worker-resources argument '%s' must pass '0' as first value" % (opts.worker_resources, slots))
sys.exit(1)
build_env.append({'state': 'Idle', 'cfg': new_mock_config, 'fs_size_gb': tmpfs_size_gb})
res, msg = set_build_idx(orig_mock_config, new_mock_config, i, tmpfs_size_gb, opts)
if not res:
log(opts.logfile, "Error: Could not write out local config: %s" % msg)
sys.exit(1)
idle_build_env_last_awarded = 0
def get_idle_build_env(slots):
global build_env
global idle_build_env_last_awarded
visited = 0
if slots < 1:
return -1
i = idle_build_env_last_awarded - 1
if i < 0 or i >= slots:
i = slots - 1
while visited < slots:
if build_env[i]['state'] == 'Idle':
build_env[i]['state'] = 'Busy'
idle_build_env_last_awarded = i
return i
visited = visited + 1
i = i - 1
if i < 0:
i = slots - 1
return -1
def release_build_env(idx):
global build_env
build_env[idx]['state'] = 'Idle'
def get_best_rc(a, b):
print("get_best_rc: a=%s" % str(a))
print("get_best_rc: b=%s" % str(b))
if (b == {}) and (a != {}):
return a
if (a == {}) and (b != {}):
return b
if (b['build_name'] is None) and (not a['build_name'] is None):
return a
if (a['build_name'] is None) and (not b['build_name'] is None):
return b
if a['unbuilt_deps'] < b['unbuilt_deps']:
return a
if b['unbuilt_deps'] < a['unbuilt_deps']:
return b
if a['depth'] < b['depth']:
return a
if b['depth'] < a['depth']:
return b
print("get_best_rc: uncertain %s vs %s" % (a,b))
return a
unbuilt_dep_list_print=False
def unbuilt_dep_list(name, unbuilt_pkg_names, depth, checked=None):
global srpm_dependencies_direct
global rpm_dependencies_direct
global rpm_to_srpm_map
global no_dep_list
global unbuilt_dep_list_print
first_iteration=False
unbuilt = []
if name in no_dep_list:
return unbuilt
if checked is None:
first_iteration=True
checked=[]
# Count unbuild dependencies
if first_iteration:
dependencies_direct=srpm_dependencies_direct
else:
dependencies_direct=rpm_dependencies_direct
if name in dependencies_direct:
for rdep in dependencies_direct[name]:
sdep='???'
if rdep in rpm_to_srpm_map:
sdep = rpm_to_srpm_map[rdep]
if rdep != name and sdep != name and not rdep in checked:
if (not first_iteration) and (sdep in no_dep_list):
continue
checked.append(rdep)
if sdep in unbuilt_pkg_names:
if not sdep in unbuilt:
unbuilt.append(sdep)
if depth > 0:
child_unbuilt = unbuilt_dep_list(rdep, unbuilt_pkg_names, depth-1, checked)
for sub_sdep in child_unbuilt:
if sub_sdep != name:
if not sub_sdep in unbuilt:
unbuilt.append(sub_sdep)
return unbuilt
def can_build_at_idx(build_idx, name, opts):
global pkg_to_name
global name_to_pkg
global big_pkgs
global big_pkg_names
global slow_pkgs
global slow_pkg_names
global build_env
fs_size_gb = 0
size_gb = 0
speed = 0
pkg = name_to_pkg[name]
if name in big_pkg_names:
size_gb=big_pkg_names[name]
if pkg in big_pkgs:
size_gb=big_pkgs[pkg]
if name in slow_pkg_names:
speed=slow_pkg_names[name]
if pkg in slow_pkgs:
speed=slow_pkgs[pkg]
fs_size_gb = build_env[build_idx]['fs_size_gb']
return fs_size_gb == 0 or fs_size_gb >= size_gb
def schedule(build_idx, pkgs, opts):
global worker_data
global pkg_to_name
global name_to_pkg
global big_pkgs
global big_pkg_names
global slow_pkgs
global slow_pkg_names
unbuilt_pkg_names=[]
building_pkg_names=[]
unprioritized_pkg_names=[]
for pkg in pkgs:
name = pkg_to_name[pkg]
unbuilt_pkg_names.append(name)
unprioritized_pkg_names.append(name)
prioritized_pkg_names=[]
for wd in worker_data:
pkg = wd['pkg']
if not pkg is None:
name = pkg_to_name[pkg]
building_pkg_names.append(name)
# log(opts.logfile, "schedule: build_idx=%d start" % build_idx)
if len(big_pkg_names) or len(big_pkgs):
next_unprioritized_pkg_names = unprioritized_pkg_names[:]
for name in unprioritized_pkg_names:
pkg = name_to_pkg[name]
if name in big_pkg_names or pkg in big_pkgs:
prioritized_pkg_names.append(name)
next_unprioritized_pkg_names.remove(name)
unprioritized_pkg_names = next_unprioritized_pkg_names[:]
if len(slow_pkg_names) or len(slow_pkgs):
next_unprioritized_pkg_names = unprioritized_pkg_names[:]
for name in unprioritized_pkg_names:
pkg = name_to_pkg[name]
if name in slow_pkg_names or pkg in slow_pkgs:
if can_build_at_idx(build_idx, name, opts):
prioritized_pkg_names.append(name)
next_unprioritized_pkg_names.remove(name)
unprioritized_pkg_names = next_unprioritized_pkg_names[:]
for name in unprioritized_pkg_names:
if can_build_at_idx(build_idx, name, opts):
prioritized_pkg_names.append(name)
name_out = schedule2(build_idx, prioritized_pkg_names, unbuilt_pkg_names, building_pkg_names, opts)
if not name_out is None:
pkg_out = name_to_pkg[name_out]
else:
pkg_out = None
# log(opts.logfile, "schedule: failed to translate '%s' to a pkg" % name_out)
# log(opts.logfile, "schedule: build_idx=%d end: out = %s -> %s" % (build_idx, str(name_out), str(pkg_out)))
return pkg_out
def schedule2(build_idx, pkg_names, unbuilt_pkg_names, building_pkg_names, opts):
global pkg_to_name
global name_to_pkg
global no_dep_list
max_depth = 3
if len(pkg_names) == 0:
return None
unbuilt_deps={}
building_deps={}
for depth in range(max_depth,-1,-1):
unbuilt_deps[depth]={}
building_deps[depth]={}
for depth in range(max_depth,-1,-1):
checked=[]
reordered_pkg_names = pkg_names[:]
# for name in reordered_pkg_names:
while len(reordered_pkg_names):
name = reordered_pkg_names.pop(0)
if name in checked:
continue
# log(opts.logfile, "checked.append(%s)" % name)
checked.append(name)
pkg = name_to_pkg[name]
# log(opts.logfile, "schedule2: check '%s', depth %d" % (name, depth))
if not name in unbuilt_deps[depth]:
unbuilt_deps[depth][name] = unbuilt_dep_list(name, unbuilt_pkg_names, depth)
if not name in building_deps[depth]:
building_deps[depth][name] = unbuilt_dep_list(name, building_pkg_names, depth)
# log(opts.logfile, "schedule2: unbuilt deps for pkg=%s, depth=%d: %s" % (name, depth, unbuilt_deps[depth][name]))
# log(opts.logfile, "schedule2: building deps for pkg=%s, depth=%d: %s" % (name, depth, building_deps[depth][name]))
if len(unbuilt_deps[depth][name]) == 0 and len(building_deps[depth][name]) == 0:
if can_build_at_idx(build_idx, name, opts):
log(opts.logfile, "schedule2: no unbuilt deps for '%s', searching at depth %d" % (name, depth))
return name
else:
# log(opts.logfile, "schedule2: Can't build '%s' on 'b%d'" % (name, build_idx))
continue
if not name in unbuilt_deps[0]:
unbuilt_deps[0][name] = unbuilt_dep_list(name, unbuilt_pkg_names, 0)
if not name in building_deps[0]:
building_deps[0][name] = unbuilt_dep_list(name, building_pkg_names, 0)
# log(opts.logfile, "schedule2: unbuilt deps for pkg=%s, depth=%d: %s" % (name, 0, unbuilt_deps[0][name]))
# log(opts.logfile, "schedule2: building deps for pkg=%s, depth=%d: %s" % (name, 0, building_deps[0][name]))
if (len(building_deps[depth][name]) == 0 and len(unbuilt_deps[depth][name]) == 1 and unbuilt_deps[depth][name][0] in no_dep_list) or (len(unbuilt_deps[depth][name]) == 0 and len(building_deps[depth][name]) == 1 and building_deps[depth][name][0] in no_dep_list):
if len(unbuilt_deps[0][name]) == 0 and len(building_deps[0][name]) == 0:
if can_build_at_idx(build_idx, name, opts):
log(opts.logfile, "schedule2: no unbuilt deps for '%s' except for indirect kernel dep, searching at depth %d" % (name, depth))
return name
else:
# log(opts.logfile, "schedule2: Can't build '%s' on 'b%d'" % (name, build_idx))
continue
loop = False
for dep_name in unbuilt_deps[depth][name]:
if name == dep_name:
continue
# log(opts.logfile, "name=%s depends on dep_name=%s, depth=%d" % (name, dep_name, depth))
if dep_name in checked:
continue
# log(opts.logfile, "schedule2: check '%s' indirect" % dep_name)
if not dep_name in unbuilt_deps[depth]:
unbuilt_deps[depth][dep_name] = unbuilt_dep_list(dep_name, unbuilt_pkg_names, depth)
if not dep_name in building_deps[depth]:
building_deps[depth][dep_name] = unbuilt_dep_list(dep_name, building_pkg_names, depth)
# log(opts.logfile, "schedule2: deps: unbuilt deps for %s -> %s, depth=%d: %s" % (name, dep_name, depth, unbuilt_deps[depth][dep_name]))
# log(opts.logfile, "schedule2: deps: building deps for %s -> %s, depth=%d: %s" % (name, dep_name, depth, building_deps[depth][dep_name]))
if len(unbuilt_deps[depth][dep_name]) == 0 and len(building_deps[depth][dep_name]) == 0:
if can_build_at_idx(build_idx, dep_name, opts):
log(opts.logfile, "schedule2: deps: no unbuilt deps for '%s', working towards '%s', searching at depth %d" % (dep_name, name, depth))
return dep_name
if not dep_name in unbuilt_deps[0]:
unbuilt_deps[0][dep_name] = unbuilt_dep_list(dep_name, unbuilt_pkg_names, 0)
if not dep_name in building_deps[0]:
building_deps[0][dep_name] = unbuilt_dep_list(dep_name, building_pkg_names, 0)
# log(opts.logfile, "schedule2: deps: unbuilt deps for %s -> %s, depth=%d: %s" % (name, dep_name, 0, unbuilt_deps[0][dep_name]))
# log(opts.logfile, "schedule2: deps: building deps for %s -> %s, depth=%d: %s" % (name, dep_name, 0, building_deps[0][dep_name]))
if (len(building_deps[depth][dep_name]) == 0 and len(unbuilt_deps[depth][dep_name]) == 1 and unbuilt_deps[depth][dep_name][0] in no_dep_list) or (len(unbuilt_deps[depth][dep_name]) == 0 and len(building_deps[depth][dep_name]) == 1 and building_deps[depth][dep_name][0] in no_dep_list):
if len(unbuilt_deps[0][dep_name]) == 0 and len(building_deps[0][dep_name]) == 0:
if can_build_at_idx(build_idx, dep_name, opts):
log(opts.logfile, "schedule2: no unbuilt deps for '%s' except for indirect kernel dep, working towards '%s', searching at depth %d" % (dep_name, name, depth))
return dep_name
if name in unbuilt_deps[0][dep_name]:
loop = True
# log(opts.logfile, "schedule2: loop detected: %s <-> %s" % (name, dep_name))
if loop and len(building_deps[depth][name]) == 0:
log(opts.logfile, "schedule2: loop detected, try to build '%s'" % name)
return name
for dep_name in unbuilt_deps[depth][name]:
if dep_name in reordered_pkg_names:
# log(opts.logfile, "schedule2: promote %s to work toward %s" % (dep_name, name))
reordered_pkg_names.remove(dep_name)
reordered_pkg_names.insert(0,dep_name)
# log(opts.logfile, "schedule2: Nothing buildable at this time")
return None
def read_deps(opts):
read_srpm_deps(opts)
read_rpm_deps(opts)
read_map_deps(opts)
def read_srpm_deps(opts):
global srpm_dependencies_direct
if opts.srpm_dependency_file == None:
return
if not os.path.exists(opts.srpm_dependency_file):
log(opts.logfile, "File not found: %s" % opts.srpm_dependency_file)
sys.exit(1)
with open(opts.srpm_dependency_file) as f:
lines = f.readlines()
for line in lines:
(name,deps) = line.rstrip().split(';')
srpm_dependencies_direct[name]=deps.split(',')
def read_rpm_deps(opts):
global rpm_dependencies_direct
if opts.rpm_dependency_file == None:
return
if not os.path.exists(opts.rpm_dependency_file):
log(opts.logfile, "File not found: %s" % opts.rpm_dependency_file)
sys.exit(1)
with open(opts.rpm_dependency_file) as f:
lines = f.readlines()
for line in lines:
(name,deps) = line.rstrip().split(';')
rpm_dependencies_direct[name]=deps.split(',')
def read_map_deps(opts):
global rpm_to_srpm_map
if opts.rpm_to_srpm_map_file == None:
return
if not os.path.exists(opts.rpm_to_srpm_map_file):
log(opts.logfile, "File not found: %s" % opts.rpm_to_srpm_map_file)
sys.exit(1)
with open(opts.rpm_to_srpm_map_file) as f:
lines = f.readlines()
for line in lines:
(rpm,srpm) = line.rstrip().split(';')
rpm_to_srpm_map[rpm]=srpm
def reaper(opts):
global built_pkgs
global failed
global worker_data
global workers
reaped = 0
need_createrepo = False
last_reaped = -1
while reaped > last_reaped:
last_reaped = reaped
for wd in worker_data:
p = wd['proc']
ret = p.exitcode
if ret is not None:
pkg = wd['pkg']
b = int(wd['build_index'])
p.join()
worker_data.remove(wd)
workers = workers - 1
reaped = reaped + 1
release_build_env(b)
log(opts.logfile, "End build on 'b%d': %s" % (b, pkg))
if ret == 0:
failed.append(pkg)
log(opts.logfile, "Error building %s on 'b%d'." % (os.path.basename(pkg), b))
if opts.recurse and not stop_signal:
log(opts.logfile, "Will try to build again (if some other package will succeed).")
else:
log(opts.logfile, "See logs/results in %s" % opts.local_repo_dir)
elif ret == 1:
log(opts.logfile, "Success building %s on 'b%d'" % (os.path.basename(pkg), b))
built_pkgs.append(pkg)
need_createrepo = True
elif ret == 2:
log(opts.logfile, "Skipping already built pkg %s" % os.path.basename(pkg))
if need_createrepo:
# createrepo with the new pkgs
err = createrepo(opts.local_repo_dir)[1]
if err.strip():
log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir)
log(opts.logfile, "Err: %s" % err)
return reaped
stop_signal = False
def on_terminate(proc):
print("process {} terminated with exit code {}".format(proc, proc.returncode))
def kill_proc_and_descentents(parent, need_stop=False, verbose=False):
global g_opts
if need_stop:
if verbose:
log(g_opts.logfile, "Stop %d" % parent.pid)
try:
parent.send_signal(signal.SIGSTOP)
except:
# perhaps mock still running as root, give it a sec to drop pivledges and try again
time.sleep(1)
parent.send_signal(signal.SIGSTOP)
try:
children = parent.children(recursive=False)
except:
children = []
for p in children:
kill_proc_and_descentents(p, need_stop=True, verbose=verbose)
if verbose:
log(g_opts.logfile, "Terminate %d" % parent.pid)
# parent.send_signal(signal.SIGTERM)
try:
parent.terminate()
except:
# perhaps mock still running as root, give it a sec to drop pivledges and try again
time.sleep(1)
parent.terminate()
if need_stop:
if verbose:
log(g_opts.logfile, "Continue %d" % parent.pid)
parent.send_signal(signal.SIGCONT)
def child_signal_handler(signum, frame):
global g_opts
my_pid = os.getpid()
# log(g_opts.logfile, "--------- child %d recieved signal %d" % (my_pid, signum))
p = psutil.Process(my_pid)
kill_proc_and_descentents(p)
try:
sys.exit(0)
except SystemExit as e:
os._exit(0)
def signal_handler(signum, frame):
global g_opts
global stop_signal
global workers
global worker_data
stop_signal = True
# Signal processes to complete
log(g_opts.logfile, "recieved signal %d, Terminating children" % signum)
for wd in worker_data:
p = wd['proc']
ret = p.exitcode
if ret is None:
# log(g_opts.logfile, "terminate child %d" % p.pid)
p.terminate()
else:
log(g_opts.logfile, "child return code was %d" % ret)
# Wait for remaining processes to complete
log(g_opts.logfile, "===== wait for signaled jobs to complete =====")
while len(worker_data) > 0:
log(g_opts.logfile, " remaining workers: %d" % workers)
reaped = reaper(g_opts)
if reaped == 0:
time.sleep(0.1)
try:
sys.exit(1)
except SystemExit as e:
os._exit(1)
def main(args):
opts, args = parse_args(args)
# take mock config + list of pkgs
global g_opts
global stop_signal
global build_env
global worker_data
global workers
global max_workers
global slow_pkg_names
global slow_pkgs
global big_pkg_names
global big_pkgs
max_workers = int(opts.max_workers)
global failed
global built_pkgs
cfg = opts.chroot
pkgs = args[1:]
# transform slow/big package options into dictionaries
for line in opts.slow_pkg_names_raw:
speed,name = line.split(":")
if speed != "":
slow_pkg_names[name]=int(speed)
for line in opts.slow_pkgs_raw:
speed,pkg = line.split(":")
if speed != "":
slow_pkgs[pkg]=int(speed)
for line in opts.big_pkg_names_raw:
size_gb,name = line.split(":")
if size_gb != "":
big_pkg_names[name]=int(size_gb)
for line in opts.big_pkgs_raw:
size_gb,pkg = line.split(":")
if size_gb != "":
big_pkgs[pkg]=int(size_gb)
# Set up a mapping between pkg path and pkg name
global pkg_to_name
global name_to_pkg
for pkg in pkgs:
if not pkg.endswith('.rpm'):
log(opts.logfile, "%s doesn't appear to be an rpm - skipping" % pkg)
continue
try:
name = rpmName(pkg)
except OSError as e:
print("Could not parse rpm %s" % pkg)
sys.exit(1)
pkg_to_name[pkg] = name
name_to_pkg[name] = pkg
read_deps(opts)
global config_opts
config_opts = mockbuild.util.load_config(mockconfig_path, cfg, None, __VERSION__, PKGPYTHONDIR)
if not opts.tmp_prefix:
try:
opts.tmp_prefix = os.getlogin()
except OSError as e:
print("Could not find login name for tmp dir prefix add --tmp_prefix")
sys.exit(1)
pid = os.getpid()
opts.uniqueext = '%s-%s' % (opts.tmp_prefix, pid)
if opts.basedir != "/var/lib/mock":
opts.uniqueext = ''
# create a tempdir for our local info
if opts.localrepo:
local_tmp_dir = os.path.abspath(opts.localrepo)
if not os.path.exists(local_tmp_dir):
os.makedirs(local_tmp_dir)
os.chmod(local_tmp_dir, 0o755)
else:
pre = 'mock-chain-%s-' % opts.uniqueext
local_tmp_dir = tempfile.mkdtemp(prefix=pre, dir='/var/tmp')
os.chmod(local_tmp_dir, 0o755)
if opts.logfile:
opts.logfile = os.path.join(local_tmp_dir, opts.logfile)
if os.path.exists(opts.logfile):
os.unlink(opts.logfile)
log(opts.logfile, "starting logfile: %s" % opts.logfile)
opts.local_repo_dir = os.path.normpath(local_tmp_dir + '/results/' + config_opts['chroot_name'] + '/')
if not os.path.exists(opts.local_repo_dir):
os.makedirs(opts.local_repo_dir, mode=0o755)
local_baseurl = "file://%s" % opts.local_repo_dir
log(opts.logfile, "results dir: %s" % opts.local_repo_dir)
opts.config_path = os.path.normpath(local_tmp_dir + '/configs/' + config_opts['chroot_name'] + '/')
if not os.path.exists(opts.config_path):
os.makedirs(opts.config_path, mode=0o755)
log(opts.logfile, "config dir: %s" % opts.config_path)
my_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(config_opts['chroot_name']))
# modify with localrepo
res, msg = add_local_repo(config_opts['config_file'], my_mock_config, local_baseurl, 'local_build_repo')
if not res:
log(opts.logfile, "Error: Could not write out local config: %s" % msg)
sys.exit(1)
for baseurl in opts.repos:
res, msg = add_local_repo(my_mock_config, my_mock_config, baseurl)
if not res:
log(opts.logfile, "Error: Could not add: %s to yum config in mock chroot: %s" % (baseurl, msg))
sys.exit(1)
res, msg = set_basedir(my_mock_config, my_mock_config, opts.basedir, opts)
if not res:
log(opts.logfile, "Error: Could not write out local config: %s" % msg)
sys.exit(1)
# these files needed from the mock.config dir to make mock run
for fn in ['site-defaults.cfg', 'logging.ini']:
pth = mockconfig_path + '/' + fn
shutil.copyfile(pth, opts.config_path + '/' + fn)
# createrepo on it
err = createrepo(opts.local_repo_dir)[1]
if err.strip():
log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir)
log(opts.logfile, "Err: %s" % err)
sys.exit(1)
init_build_env(max_workers, opts, config_opts)
download_dir = tempfile.mkdtemp()
downloaded_pkgs = {}
built_pkgs = []
try_again = True
to_be_built = pkgs
return_code = 0
num_of_tries = 0
g_opts = opts
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGHUP, signal_handler)
signal.signal(signal.SIGABRT, signal_handler)
while try_again and not stop_signal:
num_of_tries += 1
failed = []
log(opts.logfile, "===== iteration %d start =====" % num_of_tries)
to_be_built_scheduled = to_be_built[:]
need_reap = False
while len(to_be_built_scheduled) > 0:
# Free up a worker
while need_reap or workers >= max_workers:
need_reap = False
reaped = reaper(opts)
if reaped == 0:
time.sleep(0.1)
if workers < max_workers:
workers = workers + 1
b = get_idle_build_env(max_workers)
if b < 0:
log(opts.logfile, "Failed to find idle build env for: %s" % pkg)
workers = workers - 1
need_reap = True
continue
pkg = schedule(b, to_be_built_scheduled, opts)
if pkg is None:
if workers <= 1:
# Remember we have one build environmnet reserved, so can't test for zero workers
log(opts.logfile, "failed to schedule from: %s" % to_be_built_scheduled)
pkg = to_be_built_scheduled[0]
log(opts.logfile, "All workers idle, forcing build of pkg=%s" % pkg)
else:
release_build_env(b)
workers = workers - 1
need_reap = True
continue
to_be_built_scheduled.remove(pkg)
if not pkg.endswith('.rpm'):
log(opts.logfile, "%s doesn't appear to be an rpm - skipping" % pkg)
failed.append(pkg)
release_build_env(b)
need_reap = True
continue
elif pkg.startswith('http://') or pkg.startswith('https://') or pkg.startswith('ftp://'):
url = pkg
try:
log(opts.logfile, 'Fetching %s' % url)
r = requests.get(url)
# pylint: disable=no-member
if r.status_code == requests.codes.ok:
fn = urlsplit(r.url).path.rsplit('/', 1)[1]
if 'content-disposition' in r.headers:
_, params = cgi.parse_header(r.headers['content-disposition'])
if 'filename' in params and params['filename']:
fn = params['filename']
pkg = download_dir + '/' + fn
with open(pkg, 'wb') as fd:
for chunk in r.iter_content(4096):
fd.write(chunk)
except Exception as e:
log(opts.logfile, 'Error Downloading %s: %s' % (url, str(e)))
failed.append(url)
release_build_env(b)
need_reap = True
continue
else:
downloaded_pkgs[pkg] = url
log(opts.logfile, "Start build on 'b%d': %s" % (b, pkg))
# ret = do_build(opts, config_opts['chroot_name'], pkg)[0]
p = multiprocessing.Process(target=do_build, args=(opts, build_env[b]['cfg'], pkg))
worker_data.append({'proc': p, 'pkg': pkg, 'build_index': int(b)})
p.start()
# Wait for remaining processes to complete
log(opts.logfile, "===== wait for last jobs in iteration %d to complete =====" % num_of_tries)
while workers > 0:
reaped = reaper(opts)
if reaped == 0:
time.sleep(0.1)
log(opts.logfile, "===== iteration %d complete =====" % num_of_tries)
if failed and opts.recurse:
log(opts.logfile, "failed=%s" % failed)
log(opts.logfile, "to_be_built=%s" % to_be_built)
if len(failed) != len(to_be_built):
to_be_built = failed
try_again = True
log(opts.logfile, 'Some package succeeded, some failed.')
log(opts.logfile, 'Trying to rebuild %s failed pkgs, because --recurse is set.' % len(failed))
else:
if max_workers > 1:
max_workers = 1
to_be_built = failed
try_again = True
log(opts.logfile, 'Some package failed under parallel build.')
log(opts.logfile, 'Trying to rebuild %s failed pkgs with single thread, because --recurse is set.' % len(failed))
else:
log(opts.logfile, "")
log(opts.logfile, "*** Build Failed ***")
log(opts.logfile, "Tried %s times - following pkgs could not be successfully built:" % num_of_tries)
log(opts.logfile, "*** Build Failed ***")
for pkg in failed:
msg = pkg
if pkg in downloaded_pkgs:
msg = downloaded_pkgs[pkg]
log(opts.logfile, msg)
log(opts.logfile, "")
try_again = False
else:
try_again = False
if failed:
return_code = 2
# cleaning up our download dir
shutil.rmtree(download_dir, ignore_errors=True)
log(opts.logfile, "")
log(opts.logfile, "Results out to: %s" % opts.local_repo_dir)
log(opts.logfile, "")
log(opts.logfile, "Pkgs built: %s" % len(built_pkgs))
if built_pkgs:
if failed:
if len(built_pkgs):
log(opts.logfile, "Some packages successfully built in this order:")
else:
log(opts.logfile, "Packages successfully built in this order:")
for pkg in built_pkgs:
log(opts.logfile, pkg)
return return_code
if __name__ == "__main__":
sys.exit(main(sys.argv))
View Git Blame Copy Permalink