starlingx/root
Code Issues Proposed changes
205781e766
root/build-tools/mockchain-parallel-1.3.4
Scott Little b2286535c9 Update to build tools to support Build layering. 
1: Building rpms now produces an rpm.lst file.  This file serves to
list all rpms produced by the build.  A seperate rpm.lst file
is producede for each build type (std, rt, installer). The file is
co-resident with the matching repodata directory.

The rpm.lst files will need to be published by cengn for each layer build.

The download tools in stx-tools will need to find the rpm.lst files
of lower layer builds, and use it to direct the download of rpms
from the lower layer.

2: Building rpms now produces an image.inc file.  This file serves
to list all rpms that the layer recommends be built into an iso.
The file is stored under the $MY_WORKSPACE/<build-type> subdirectory,
although it has identical content for any build-type.

The image.inc file will need to be published by cengn for each layer
build.

The download tools in stx-tools will need to download the per-layer
image.inc files to the $MY_REPO/cgcs-centos-repo/layer_image_inc/
sub-directory, renaming the file in some layer specific way.

The build-iso tool will process any *.inc files found under
$MY_REPO/cgcs-centos-repo/layer_image_inc/ ,
adding those rpms to the iso.

3) Add a mechanism for layer specific mock.cfg prototypes.
This will allow special handling of the real-time (aka 'rt'),
repositories for layer builds that need to do a 'rt' build.

4) Add support for a $MY_REPO/cgcs-centos-repo/rt subdirectory,
a place to build a repo of real-time rpms originating from
lower layer builds.

The download tools in stx-tools will need to populate the new
rt repos.

As of this writing, non-rt rpms remain in $MY_REPO/cgcs-centos-repo/.
i.e. there is not a new $MY_REPO/cgcs-centos-repo/std/ directory.

5) Some changes to make us more flexible about where we find
realease and bsp files.

6) Found that kernel mudules were not reliably building against
the hearnel-headers of our modified kernel.  Found that adding '--update'
to our mock build command was not working.  Does mock expect '--update'
to only be used independently of a build command?  It does work when
used in that manner, so that's what we will do.

7) The build-pkgs, build-srpms, build-rpms family of commands
can take a layer argument and/or will read the LAYER environment
variable.  Current the only use of this variable is to modify
the build-info.  It does NOT limit the compile to packages
for a specific layer.

Story: 2006166
Task: 37094

Depends-On: https://review.opendev.org/698756
Depends-On: https://review.opendev.org/700819
Change-Id: I817e08a19cdabe08b3fcc47dee63a36b461c13c0
Co-Authored-by: Martin Chen <haochuan.z.chen@intel.com>
Signed-off-by: Scott Little <scott.little@windriver.com>
2020-02-07 16:36:40 -05:00

1220 lines
44 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/python2.7 -tt
# -*- coding: utf-8 -*-
# vim: noai:ts=4:sw=4:expandtab
# by skvidal@fedoraproject.org
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
# copyright 2012 Red Hat, Inc.
# SUMMARY
# mockchain
# take a mock config and a series of srpms
# rebuild them one at a time
# adding each to a local repo
# so they are available as build deps to next pkg being built
from __future__ import print_function
import cgi
# pylint: disable=deprecated-module
import optparse
import os
import re
import shutil
import subprocess
import sys
import tempfile
import time
import multiprocessing
import signal
import psutil
import requests
# pylint: disable=import-error
from six.moves.urllib_parse import urlsplit
import mockbuild.util
from stxRpmUtils import splitRpmFilename
# all of the variables below are substituted by the build system
__VERSION__="1.3.4"
SYSCONFDIR="/etc"
PYTHONDIR="/usr/lib/python2.7/site-packages"
PKGPYTHONDIR="/usr/lib/python2.7/site-packages/mockbuild"
MOCKCONFDIR = os.path.join(SYSCONFDIR, "mock")
# end build system subs
mockconfig_path = '/etc/mock'
def rpmName(path):
filename = os.path.basename(path)
(n, v, r, e, a) = splitRpmFilename(filename)
return n
def createrepo(path):
global max_workers
if os.path.exists(path + '/repodata/repomd.xml'):
comm = ['/usr/bin/createrepo_c', '--update', '--retain-old-md', "%d" % max_workers, "--workers", "%d" % max_workers, path]
else:
comm = ['/usr/bin/createrepo_c', "--workers", "%d" % max_workers, path]
cmd = subprocess.Popen(
comm, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = cmd.communicate()
return out, err
g_opts = optparse.Values()
def parse_args(args):
parser = optparse.OptionParser('\nmockchain -r mockcfg pkg1 [pkg2] [pkg3]')
parser.add_option(
'-r', '--root', default=None, dest='chroot',
metavar="CONFIG",
help="chroot config name/base to use in the mock build")
parser.add_option(
'-l', '--localrepo', default=None,
help="local path for the local repo, defaults to making its own")
parser.add_option(
'-c', '--continue', default=False, action='store_true',
dest='cont',
help="if a pkg fails to build, continue to the next one")
parser.add_option(
'-a', '--addrepo', default=[], action='append',
dest='repos',
help="add these repo baseurls to the chroot's yum config")
parser.add_option(
'--recurse', default=False, action='store_true',
help="if more than one pkg and it fails to build, try to build the rest and come back to it")
parser.add_option(
'--log', default=None, dest='logfile',
help="log to the file named by this option, defaults to not logging")
parser.add_option(
'--workers', default=1, dest='max_workers',
help="number of parallel build jobs")
parser.add_option(
'--worker-resources', default="", dest='worker_resources',
help="colon seperated list, how much mem in gb for each workers temfs")
parser.add_option(
'--basedir', default='/var/lib/mock', dest='basedir',
help="path to workspace")
parser.add_option(
'--tmp_prefix', default=None, dest='tmp_prefix',
help="tmp dir prefix - will default to username-pid if not specified")
parser.add_option(
'-m', '--mock-option', default=[], action='append',
dest='mock_option',
help="option to pass directly to mock")
parser.add_option(
'--mark-slow-name', default=[], action='append',
dest='slow_pkg_names_raw',
help="package name that is known to build slowly")
parser.add_option(
'--mark-slow-path', default=[], action='append',
dest='slow_pkgs_raw',
help="package path that is known to build slowly")
parser.add_option(
'--mark-big-name', default=[], action='append',
dest='big_pkg_names_raw',
help="package name that is known to require a lot of disk space to build")
parser.add_option(
'--mark-big-path', default=[], action='append',
dest='big_pkgs_raw',
help="package path that is known to require a lot of disk space to build")
parser.add_option(
'--srpm-dependency-file', default=None,
dest='srpm_dependency_file',
help="path to srpm dependency file")
parser.add_option(
'--rpm-dependency-file', default=None,
dest='rpm_dependency_file',
help="path to rpm dependency file")
parser.add_option(
'--rpm-to-srpm-map-file', default=None,
dest='rpm_to_srpm_map_file',
help="path to rpm to srpm map file")
opts, args = parser.parse_args(args)
if opts.recurse:
opts.cont = True
if not opts.chroot:
print("You must provide an argument to -r for the mock chroot")
sys.exit(1)
if len(sys.argv) < 3:
print("You must specify at least 1 package to build")
sys.exit(1)
return opts, args
REPOS_ID = []
slow_pkg_names={}
slow_pkgs={}
big_pkg_names={}
big_pkgs={}
def generate_repo_id(baseurl):
""" generate repository id for yum.conf out of baseurl """
repoid = "/".join(baseurl.split('//')[1:]).replace('/', '_')
repoid = re.sub(r'[^a-zA-Z0-9_]', '', repoid)
suffix = ''
i = 1
while repoid + suffix in REPOS_ID:
suffix = str(i)
i += 1
repoid = repoid + suffix
REPOS_ID.append(repoid)
return repoid
def set_build_idx(infile, destfile, build_idx, tmpfs_size_gb, opts):
# log(opts.logfile, "set_build_idx: infile=%s, destfile=%s, build_idx=%d, tmpfs_size_gb=%d" % (infile, destfile, build_idx, tmpfs_size_gb))
try:
with open(infile) as f:
code = compile(f.read(), infile, 'exec')
# pylint: disable=exec-used
exec(code)
config_opts['root'] = config_opts['root'].replace('b0', 'b{0}'.format(build_idx))
config_opts['cache_topdir'] = config_opts['cache_topdir'].replace('b0', 'b{0}'.format(build_idx))
# log(opts.logfile, "set_build_idx: root=%s" % config_opts['root'])
# log(opts.logfile, "set_build_idx: cache_topdir=%s" % config_opts['cache_topdir'])
if tmpfs_size_gb > 0:
config_opts['plugin_conf']['tmpfs_enable'] = True
config_opts['plugin_conf']['tmpfs_opts'] = {}
config_opts['plugin_conf']['tmpfs_opts']['required_ram_mb'] = 1024
config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = "%dg" % tmpfs_size_gb
config_opts['plugin_conf']['tmpfs_opts']['mode'] = '0755'
config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = True
# log(opts.logfile, "set_build_idx: plugin_conf->tmpfs_enable=%s" % config_opts['plugin_conf']['tmpfs_enable'])
# log(opts.logfile, "set_build_idx: plugin_conf->tmpfs_opts->max_fs_size=%s" % config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'])
with open(destfile, 'w') as br_dest:
for k, v in list(config_opts.items()):
br_dest.write("config_opts[%r] = %r\n" % (k, v))
try:
log(opts.logfile, "set_build_idx: os.makedirs %s" % config_opts['cache_topdir'])
if not os.path.isdir(config_opts['cache_topdir']):
os.makedirs(config_opts['cache_topdir'], exist_ok=True)
except (IOError, OSError):
return False, "Could not create dir: %s" % config_opts['cache_topdir']
cache_dir = "%s/%s/mock" % (config_opts['basedir'], config_opts['root'])
try:
log(opts.logfile, "set_build_idx: os.makedirs %s" % cache_dir)
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
except (IOError, OSError):
return False, "Could not create dir: %s" % cache_dir
return True, ''
except (IOError, OSError):
return False, "Could not write mock config to %s" % destfile
return True, ''
def set_basedir(infile, destfile, basedir, opts):
log(opts.logfile, "set_basedir: infile=%s, destfile=%s, basedir=%s" % (infile, destfile, basedir))
try:
with open(infile) as f:
code = compile(f.read(), infile, 'exec')
# pylint: disable=exec-used
exec(code)
config_opts['basedir'] = basedir
config_opts['resultdir'] = '{0}/result'.format(basedir)
config_opts['backup_base_dir'] = '{0}/backup'.format(basedir)
config_opts['root'] = 'mock/b0'
config_opts['cache_topdir'] = '{0}/cache/b0'.format(basedir)
with open(destfile, 'w') as br_dest:
for k, v in list(config_opts.items()):
br_dest.write("config_opts[%r] = %r\n" % (k, v))
return True, ''
except (IOError, OSError):
return False, "Could not write mock config to %s" % destfile
return True, ''
def add_local_repo(infile, destfile, baseurl, repoid=None):
"""take a mock chroot config and add a repo to it's yum.conf
infile = mock chroot config file
destfile = where to save out the result
baseurl = baseurl of repo you wish to add"""
global config_opts
try:
with open(infile) as f:
code = compile(f.read(), infile, 'exec')
# pylint: disable=exec-used
exec(code)
if not repoid:
repoid = generate_repo_id(baseurl)
else:
REPOS_ID.append(repoid)
localyumrepo = """
[%s]
name=%s
baseurl=%s
enabled=1
skip_if_unavailable=1
metadata_expire=0
cost=1
best=1
""" % (repoid, baseurl, baseurl)
config_opts['yum.conf'] += localyumrepo
with open(destfile, 'w') as br_dest:
for k, v in list(config_opts.items()):
br_dest.write("config_opts[%r] = %r\n" % (k, v))
return True, ''
except (IOError, OSError):
return False, "Could not write mock config to %s" % destfile
return True, ''
def do_build(opts, cfg, pkg):
# returns 0, cmd, out, err = failure
# returns 1, cmd, out, err = success
# returns 2, None, None, None = already built
signal.signal(signal.SIGTERM, child_signal_handler)
signal.signal(signal.SIGINT, child_signal_handler)
signal.signal(signal.SIGHUP, child_signal_handler)
signal.signal(signal.SIGABRT, child_signal_handler)
s_pkg = os.path.basename(pkg)
pdn = s_pkg.replace('.src.rpm', '')
resdir = '%s/%s' % (opts.local_repo_dir, pdn)
resdir = os.path.normpath(resdir)
if not os.path.exists(resdir):
os.makedirs(resdir)
success_file = resdir + '/success'
fail_file = resdir + '/fail'
if os.path.exists(success_file):
# return 2, None, None, None
sys.exit(2)
# clean it up if we're starting over :)
if os.path.exists(fail_file):
os.unlink(fail_file)
if opts.uniqueext == '':
mockcmd = ['/usr/bin/mock',
'--configdir', opts.config_path,
'--resultdir', resdir,
'-r', cfg, ]
else:
mockcmd = ['/usr/bin/mock',
'--configdir', opts.config_path,
'--resultdir', resdir,
'--uniqueext', opts.uniqueext,
'-r', cfg, ]
# Ensure repo is up-to-date.
# Note: Merely adding --update to mockcmd failed to update
mockcmd_update=mockcmd
mockcmd_update.append('--update')
cmd = subprocess.Popen(
mockcmd_update, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = cmd.communicate()
if cmd.returncode != 0:
if (isinstance(err, bytes)):
err = err.decode("utf-8")
sys.stderr.write(err)
# heuristic here, if user pass for mock "-d foo", but we must be care to leave
# "-d'foo bar'" or "--define='foo bar'" as is
compiled_re_1 = re.compile(r'^(-\S)\s+(.+)')
compiled_re_2 = re.compile(r'^(--[^ =])[ =](\.+)')
for option in opts.mock_option:
r_match = compiled_re_1.match(option)
if r_match:
mockcmd.extend([r_match.group(1), r_match.group(2)])
else:
r_match = compiled_re_2.match(option)
if r_match:
mockcmd.extend([r_match.group(1), r_match.group(2)])
else:
mockcmd.append(option)
print('building %s' % s_pkg)
mockcmd.append(pkg)
# print("mockcmd: %s" % str(mockcmd))
cmd = subprocess.Popen(
mockcmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = cmd.communicate()
if cmd.returncode == 0:
with open(success_file, 'w') as f:
f.write('done\n')
ret = 1
else:
if (isinstance(err, bytes)):
err = err.decode("utf-8")
sys.stderr.write(err)
with open(fail_file, 'w') as f:
f.write('undone\n')
ret = 0
# return ret, cmd, out, err
sys.exit(ret)
def log(lf, msg):
if lf:
now = time.time()
try:
with open(lf, 'a') as f:
f.write(str(now) + ':' + msg + '\n')
except (IOError, OSError) as e:
print('Could not write to logfile %s - %s' % (lf, str(e)))
print(msg)
config_opts = {}
worker_data = []
workers = 0
max_workers = 1
build_env = []
failed = []
built_pkgs = []
local_repo_dir = ""
pkg_to_name={}
name_to_pkg={}
srpm_dependencies_direct={}
rpm_dependencies_direct={}
rpm_to_srpm_map={}
no_dep_list = [ "bash", "kernel" , "kernel-rt" ]
def init_build_env(slots, opts, config_opts_in):
global build_env
orig_chroot_name=config_opts_in['chroot_name']
orig_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(orig_chroot_name))
# build_env.append({'state': 'Idle', 'cfg': orig_mock_config})
for i in range(0,slots):
new_chroot_name = "{0}.b{1}".format(orig_chroot_name, i)
new_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(new_chroot_name))
tmpfs_size_gb = 0
if opts.worker_resources == "":
if i > 0:
tmpfs_size_gb = 2 * (1 + slots - i)
else:
resource_array=opts.worker_resources.split(':')
if i < len(resource_array):
tmpfs_size_gb=int(resource_array[i])
else:
log(opts.logfile, "Error: worker-resources argument '%s' does not supply info for all %d workers" % (opts.worker_resources, slots))
sys.exit(1)
if i == 0 and tmpfs_size_gb != 0:
log(opts.logfile, "Error: worker-resources argument '%s' must pass '0' as first value" % (opts.worker_resources, slots))
sys.exit(1)
build_env.append({'state': 'Idle', 'cfg': new_mock_config, 'fs_size_gb': tmpfs_size_gb})
res, msg = set_build_idx(orig_mock_config, new_mock_config, i, tmpfs_size_gb, opts)
if not res:
log(opts.logfile, "Error: Could not write out local config: %s" % msg)
sys.exit(1)
idle_build_env_last_awarded = 0
def get_idle_build_env(slots):
global build_env
global idle_build_env_last_awarded
visited = 0
if slots < 1:
return -1
i = idle_build_env_last_awarded - 1
if i < 0 or i >= slots:
i = slots - 1
while visited < slots:
if build_env[i]['state'] == 'Idle':
build_env[i]['state'] = 'Busy'
idle_build_env_last_awarded = i
return i
visited = visited + 1
i = i - 1
if i < 0:
i = slots - 1
return -1
def release_build_env(idx):
global build_env
build_env[idx]['state'] = 'Idle'
def get_best_rc(a, b):
print("get_best_rc: a=%s" % str(a))
print("get_best_rc: b=%s" % str(b))
if (b == {}) and (a != {}):
return a
if (a == {}) and (b != {}):
return b
if (b['build_name'] is None) and (not a['build_name'] is None):
return a
if (a['build_name'] is None) and (not b['build_name'] is None):
return b
if a['unbuilt_deps'] < b['unbuilt_deps']:
return a
if b['unbuilt_deps'] < a['unbuilt_deps']:
return b
if a['depth'] < b['depth']:
return a
if b['depth'] < a['depth']:
return b
print("get_best_rc: uncertain %s vs %s" % (a,b))
return a
unbuilt_dep_list_print=False
def unbuilt_dep_list(name, unbuilt_pkg_names, depth, checked=None):
global srpm_dependencies_direct
global rpm_dependencies_direct
global rpm_to_srpm_map
global no_dep_list
global unbuilt_dep_list_print
first_iteration=False
unbuilt = []
if name in no_dep_list:
return unbuilt
if checked is None:
first_iteration=True
checked=[]
# Count unbuild dependencies
if first_iteration:
dependencies_direct=srpm_dependencies_direct
else:
dependencies_direct=rpm_dependencies_direct
if name in dependencies_direct:
for rdep in dependencies_direct[name]:
sdep='???'
if rdep in rpm_to_srpm_map:
sdep = rpm_to_srpm_map[rdep]
if rdep != name and sdep != name and not rdep in checked:
if (not first_iteration) and (sdep in no_dep_list):
continue
checked.append(rdep)
if sdep in unbuilt_pkg_names:
if not sdep in unbuilt:
unbuilt.append(sdep)
if depth > 0:
child_unbuilt = unbuilt_dep_list(rdep, unbuilt_pkg_names, depth-1, checked)
for sub_sdep in child_unbuilt:
if sub_sdep != name:
if not sub_sdep in unbuilt:
unbuilt.append(sub_sdep)
return unbuilt
def can_build_at_idx(build_idx, name, opts):
global pkg_to_name
global name_to_pkg
global big_pkgs
global big_pkg_names
global slow_pkgs
global slow_pkg_names
global build_env
fs_size_gb = 0
size_gb = 0
speed = 0
pkg = name_to_pkg[name]
if name in big_pkg_names:
size_gb=big_pkg_names[name]
if pkg in big_pkgs:
size_gb=big_pkgs[pkg]
if name in slow_pkg_names:
speed=slow_pkg_names[name]
if pkg in slow_pkgs:
speed=slow_pkgs[pkg]
fs_size_gb = build_env[build_idx]['fs_size_gb']
return fs_size_gb == 0 or fs_size_gb >= size_gb
def schedule(build_idx, pkgs, opts):
global worker_data
global pkg_to_name
global name_to_pkg
global big_pkgs
global big_pkg_names
global slow_pkgs
global slow_pkg_names
unbuilt_pkg_names=[]
building_pkg_names=[]
unprioritized_pkg_names=[]
for pkg in pkgs:
name = pkg_to_name[pkg]
unbuilt_pkg_names.append(name)
unprioritized_pkg_names.append(name)
prioritized_pkg_names=[]
for wd in worker_data:
pkg = wd['pkg']
if not pkg is None:
name = pkg_to_name[pkg]
building_pkg_names.append(name)
# log(opts.logfile, "schedule: build_idx=%d start" % build_idx)
if len(big_pkg_names) or len(big_pkgs):
next_unprioritized_pkg_names = unprioritized_pkg_names[:]
for name in unprioritized_pkg_names:
pkg = name_to_pkg[name]
if name in big_pkg_names or pkg in big_pkgs:
prioritized_pkg_names.append(name)
next_unprioritized_pkg_names.remove(name)
unprioritized_pkg_names = next_unprioritized_pkg_names[:]
if len(slow_pkg_names) or len(slow_pkgs):
next_unprioritized_pkg_names = unprioritized_pkg_names[:]
for name in unprioritized_pkg_names:
pkg = name_to_pkg[name]
if name in slow_pkg_names or pkg in slow_pkgs:
if can_build_at_idx(build_idx, name, opts):
prioritized_pkg_names.append(name)
next_unprioritized_pkg_names.remove(name)
unprioritized_pkg_names = next_unprioritized_pkg_names[:]
for name in unprioritized_pkg_names:
if can_build_at_idx(build_idx, name, opts):
prioritized_pkg_names.append(name)
name_out = schedule2(build_idx, prioritized_pkg_names, unbuilt_pkg_names, building_pkg_names, opts)
if not name_out is None:
pkg_out = name_to_pkg[name_out]
else:
pkg_out = None
# log(opts.logfile, "schedule: failed to translate '%s' to a pkg" % name_out)
# log(opts.logfile, "schedule: build_idx=%d end: out = %s -> %s" % (build_idx, str(name_out), str(pkg_out)))
return pkg_out
def schedule2(build_idx, pkg_names, unbuilt_pkg_names, building_pkg_names, opts):
global pkg_to_name
global name_to_pkg
global no_dep_list
max_depth = 3
if len(pkg_names) == 0:
return None
unbuilt_deps={}
building_deps={}
for depth in range(max_depth,-1,-1):
unbuilt_deps[depth]={}
building_deps[depth]={}
for depth in range(max_depth,-1,-1):
checked=[]
reordered_pkg_names = pkg_names[:]
# for name in reordered_pkg_names:
while len(reordered_pkg_names):
name = reordered_pkg_names.pop(0)
if name in checked:
continue
# log(opts.logfile, "checked.append(%s)" % name)
checked.append(name)
pkg = name_to_pkg[name]
# log(opts.logfile, "schedule2: check '%s', depth %d" % (name, depth))
if not name in unbuilt_deps[depth]:
unbuilt_deps[depth][name] = unbuilt_dep_list(name, unbuilt_pkg_names, depth)
if not name in building_deps[depth]:
building_deps[depth][name] = unbuilt_dep_list(name, building_pkg_names, depth)
# log(opts.logfile, "schedule2: unbuilt deps for pkg=%s, depth=%d: %s" % (name, depth, unbuilt_deps[depth][name]))
# log(opts.logfile, "schedule2: building deps for pkg=%s, depth=%d: %s" % (name, depth, building_deps[depth][name]))
if len(unbuilt_deps[depth][name]) == 0 and len(building_deps[depth][name]) == 0:
if can_build_at_idx(build_idx, name, opts):
log(opts.logfile, "schedule2: no unbuilt deps for '%s', searching at depth %d" % (name, depth))
return name
else:
# log(opts.logfile, "schedule2: Can't build '%s' on 'b%d'" % (name, build_idx))
continue
if not name in unbuilt_deps[0]:
unbuilt_deps[0][name] = unbuilt_dep_list(name, unbuilt_pkg_names, 0)
if not name in building_deps[0]:
building_deps[0][name] = unbuilt_dep_list(name, building_pkg_names, 0)
# log(opts.logfile, "schedule2: unbuilt deps for pkg=%s, depth=%d: %s" % (name, 0, unbuilt_deps[0][name]))
# log(opts.logfile, "schedule2: building deps for pkg=%s, depth=%d: %s" % (name, 0, building_deps[0][name]))
if (len(building_deps[depth][name]) == 0 and len(unbuilt_deps[depth][name]) == 1 and unbuilt_deps[depth][name][0] in no_dep_list) or (len(unbuilt_deps[depth][name]) == 0 and len(building_deps[depth][name]) == 1 and building_deps[depth][name][0] in no_dep_list):
if len(unbuilt_deps[0][name]) == 0 and len(building_deps[0][name]) == 0:
if can_build_at_idx(build_idx, name, opts):
log(opts.logfile, "schedule2: no unbuilt deps for '%s' except for indirect kernel dep, searching at depth %d" % (name, depth))
return name
else:
# log(opts.logfile, "schedule2: Can't build '%s' on 'b%d'" % (name, build_idx))
continue
loop = False
for dep_name in unbuilt_deps[depth][name]:
if name == dep_name:
continue
# log(opts.logfile, "name=%s depends on dep_name=%s, depth=%d" % (name, dep_name, depth))
if dep_name in checked:
continue
# log(opts.logfile, "schedule2: check '%s' indirect" % dep_name)
if not dep_name in unbuilt_deps[depth]:
unbuilt_deps[depth][dep_name] = unbuilt_dep_list(dep_name, unbuilt_pkg_names, depth)
if not dep_name in building_deps[depth]:
building_deps[depth][dep_name] = unbuilt_dep_list(dep_name, building_pkg_names, depth)
# log(opts.logfile, "schedule2: deps: unbuilt deps for %s -> %s, depth=%d: %s" % (name, dep_name, depth, unbuilt_deps[depth][dep_name]))
# log(opts.logfile, "schedule2: deps: building deps for %s -> %s, depth=%d: %s" % (name, dep_name, depth, building_deps[depth][dep_name]))
if len(unbuilt_deps[depth][dep_name]) == 0 and len(building_deps[depth][dep_name]) == 0:
if can_build_at_idx(build_idx, dep_name, opts):
log(opts.logfile, "schedule2: deps: no unbuilt deps for '%s', working towards '%s', searching at depth %d" % (dep_name, name, depth))
return dep_name
if not dep_name in unbuilt_deps[0]:
unbuilt_deps[0][dep_name] = unbuilt_dep_list(dep_name, unbuilt_pkg_names, 0)
if not dep_name in building_deps[0]:
building_deps[0][dep_name] = unbuilt_dep_list(dep_name, building_pkg_names, 0)
# log(opts.logfile, "schedule2: deps: unbuilt deps for %s -> %s, depth=%d: %s" % (name, dep_name, 0, unbuilt_deps[0][dep_name]))
# log(opts.logfile, "schedule2: deps: building deps for %s -> %s, depth=%d: %s" % (name, dep_name, 0, building_deps[0][dep_name]))
if (len(building_deps[depth][dep_name]) == 0 and len(unbuilt_deps[depth][dep_name]) == 1 and unbuilt_deps[depth][dep_name][0] in no_dep_list) or (len(unbuilt_deps[depth][dep_name]) == 0 and len(building_deps[depth][dep_name]) == 1 and building_deps[depth][dep_name][0] in no_dep_list):
if len(unbuilt_deps[0][dep_name]) == 0 and len(building_deps[0][dep_name]) == 0:
if can_build_at_idx(build_idx, dep_name, opts):
log(opts.logfile, "schedule2: no unbuilt deps for '%s' except for indirect kernel dep, working towards '%s', searching at depth %d" % (dep_name, name, depth))
return dep_name
if name in unbuilt_deps[0][dep_name]:
loop = True
# log(opts.logfile, "schedule2: loop detected: %s <-> %s" % (name, dep_name))
if loop and len(building_deps[depth][name]) == 0:
log(opts.logfile, "schedule2: loop detected, try to build '%s'" % name)
return name
for dep_name in unbuilt_deps[depth][name]:
if dep_name in reordered_pkg_names:
# log(opts.logfile, "schedule2: promote %s to work toward %s" % (dep_name, name))
reordered_pkg_names.remove(dep_name)
reordered_pkg_names.insert(0,dep_name)
# log(opts.logfile, "schedule2: Nothing buildable at this time")
return None
def read_deps(opts):
read_srpm_deps(opts)
read_rpm_deps(opts)
read_map_deps(opts)
def read_srpm_deps(opts):
global srpm_dependencies_direct
if opts.srpm_dependency_file == None:
return
if not os.path.exists(opts.srpm_dependency_file):
log(opts.logfile, "File not found: %s" % opts.srpm_dependency_file)
sys.exit(1)
with open(opts.srpm_dependency_file) as f:
lines = f.readlines()
for line in lines:
(name,deps) = line.rstrip().split(';')
srpm_dependencies_direct[name]=deps.split(',')
def read_rpm_deps(opts):
global rpm_dependencies_direct
if opts.rpm_dependency_file == None:
return
if not os.path.exists(opts.rpm_dependency_file):
log(opts.logfile, "File not found: %s" % opts.rpm_dependency_file)
sys.exit(1)
with open(opts.rpm_dependency_file) as f:
lines = f.readlines()
for line in lines:
(name,deps) = line.rstrip().split(';')
rpm_dependencies_direct[name]=deps.split(',')
def read_map_deps(opts):
global rpm_to_srpm_map
if opts.rpm_to_srpm_map_file == None:
return
if not os.path.exists(opts.rpm_to_srpm_map_file):
log(opts.logfile, "File not found: %s" % opts.rpm_to_srpm_map_file)
sys.exit(1)
with open(opts.rpm_to_srpm_map_file) as f:
lines = f.readlines()
for line in lines:
(rpm,srpm) = line.rstrip().split(';')
rpm_to_srpm_map[rpm]=srpm
def reaper(opts):
global built_pkgs
global failed
global worker_data
global workers
reaped = 0
need_createrepo = False
last_reaped = -1
while reaped > last_reaped:
last_reaped = reaped
for wd in worker_data:
p = wd['proc']
ret = p.exitcode
if ret is not None:
pkg = wd['pkg']
b = int(wd['build_index'])
p.join()
worker_data.remove(wd)
workers = workers - 1
reaped = reaped + 1
release_build_env(b)
log(opts.logfile, "End build on 'b%d': %s" % (b, pkg))
if ret == 0:
failed.append(pkg)
log(opts.logfile, "Error building %s on 'b%d'." % (os.path.basename(pkg), b))
if opts.recurse and not stop_signal:
log(opts.logfile, "Will try to build again (if some other package will succeed).")
else:
log(opts.logfile, "See logs/results in %s" % opts.local_repo_dir)
elif ret == 1:
log(opts.logfile, "Success building %s on 'b%d'" % (os.path.basename(pkg), b))
built_pkgs.append(pkg)
need_createrepo = True
elif ret == 2:
log(opts.logfile, "Skipping already built pkg %s" % os.path.basename(pkg))
if need_createrepo:
# createrepo with the new pkgs
err = createrepo(opts.local_repo_dir)[1]
if err.strip():
log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir)
log(opts.logfile, "Err: %s" % err)
return reaped
stop_signal = False
def on_terminate(proc):
print("process {} terminated with exit code {}".format(proc, proc.returncode))
def kill_proc_and_descentents(parent, need_stop=False, verbose=False):
global g_opts
if need_stop:
if verbose:
log(g_opts.logfile, "Stop %d" % parent.pid)
try:
parent.send_signal(signal.SIGSTOP)
except:
# perhaps mock still running as root, give it a sec to drop pivledges and try again
time.sleep(1)
parent.send_signal(signal.SIGSTOP)
children = parent.children(recursive=False)
for p in children:
kill_proc_and_descentents(p, need_stop=True, verbose=verbose)
if verbose:
log(g_opts.logfile, "Terminate %d" % parent.pid)
# parent.send_signal(signal.SIGTERM)
try:
parent.terminate()
except:
# perhaps mock still running as root, give it a sec to drop pivledges and try again
time.sleep(1)
parent.terminate()
if need_stop:
if verbose:
log(g_opts.logfile, "Continue %d" % parent.pid)
parent.send_signal(signal.SIGCONT)
def child_signal_handler(signum, frame):
global g_opts
my_pid = os.getpid()
# log(g_opts.logfile, "--------- child %d recieved signal %d" % (my_pid, signum))
p = psutil.Process(my_pid)
kill_proc_and_descentents(p)
try:
sys.exit(0)
except SystemExit as e:
os._exit(0)
def signal_handler(signum, frame):
global g_opts
global stop_signal
global workers
global worker_data
stop_signal = True
# Signal processes to complete
log(g_opts.logfile, "recieved signal %d, Terminating children" % signum)
for wd in worker_data:
p = wd['proc']
ret = p.exitcode
if ret is None:
# log(g_opts.logfile, "terminate child %d" % p.pid)
p.terminate()
else:
log(g_opts.logfile, "child return code was %d" % ret)
# Wait for remaining processes to complete
log(g_opts.logfile, "===== wait for signaled jobs to complete =====")
while len(worker_data) > 0:
log(g_opts.logfile, " remaining workers: %d" % workers)
reaped = reaper(g_opts)
if reaped == 0:
time.sleep(0.1)
try:
sys.exit(1)
except SystemExit as e:
os._exit(1)
def main(args):
opts, args = parse_args(args)
# take mock config + list of pkgs
global g_opts
global stop_signal
global build_env
global worker_data
global workers
global max_workers
global slow_pkg_names
global slow_pkgs
global big_pkg_names
global big_pkgs
max_workers = int(opts.max_workers)
global failed
global built_pkgs
cfg = opts.chroot
pkgs = args[1:]
# transform slow/big package options into dictionaries
for line in opts.slow_pkg_names_raw:
speed,name = line.split(":")
if speed != "":
slow_pkg_names[name]=int(speed)
for line in opts.slow_pkgs_raw:
speed,pkg = line.split(":")
if speed != "":
slow_pkgs[pkg]=int(speed)
for line in opts.big_pkg_names_raw:
size_gb,name = line.split(":")
if size_gb != "":
big_pkg_names[name]=int(size_gb)
for line in opts.big_pkgs_raw:
size_gb,pkg = line.split(":")
if size_gb != "":
big_pkgs[pkg]=int(size_gb)
# Set up a mapping between pkg path and pkg name
global pkg_to_name
global name_to_pkg
for pkg in pkgs:
if not pkg.endswith('.rpm'):
log(opts.logfile, "%s doesn't appear to be an rpm - skipping" % pkg)
continue
try:
name = rpmName(pkg)
except OSError as e:
print("Could not parse rpm %s" % pkg)
sys.exit(1)
pkg_to_name[pkg] = name
name_to_pkg[name] = pkg
read_deps(opts)
global config_opts
config_opts = mockbuild.util.load_config(mockconfig_path, cfg, None, __VERSION__, PKGPYTHONDIR)
if not opts.tmp_prefix:
try:
opts.tmp_prefix = os.getlogin()
except OSError as e:
print("Could not find login name for tmp dir prefix add --tmp_prefix")
sys.exit(1)
pid = os.getpid()
opts.uniqueext = '%s-%s' % (opts.tmp_prefix, pid)
if opts.basedir != "/var/lib/mock":
opts.uniqueext = ''
# create a tempdir for our local info
if opts.localrepo:
local_tmp_dir = os.path.abspath(opts.localrepo)
if not os.path.exists(local_tmp_dir):
os.makedirs(local_tmp_dir)
os.chmod(local_tmp_dir, 0o755)
else:
pre = 'mock-chain-%s-' % opts.uniqueext
local_tmp_dir = tempfile.mkdtemp(prefix=pre, dir='/var/tmp')
os.chmod(local_tmp_dir, 0o755)
if opts.logfile:
opts.logfile = os.path.join(local_tmp_dir, opts.logfile)
if os.path.exists(opts.logfile):
os.unlink(opts.logfile)
log(opts.logfile, "starting logfile: %s" % opts.logfile)
opts.local_repo_dir = os.path.normpath(local_tmp_dir + '/results/' + config_opts['chroot_name'] + '/')
if not os.path.exists(opts.local_repo_dir):
os.makedirs(opts.local_repo_dir, mode=0o755)
local_baseurl = "file://%s" % opts.local_repo_dir
log(opts.logfile, "results dir: %s" % opts.local_repo_dir)
opts.config_path = os.path.normpath(local_tmp_dir + '/configs/' + config_opts['chroot_name'] + '/')
if not os.path.exists(opts.config_path):
os.makedirs(opts.config_path, mode=0o755)
log(opts.logfile, "config dir: %s" % opts.config_path)
my_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(config_opts['chroot_name']))
# modify with localrepo
res, msg = add_local_repo(config_opts['config_file'], my_mock_config, local_baseurl, 'local_build_repo')
if not res:
log(opts.logfile, "Error: Could not write out local config: %s" % msg)
sys.exit(1)
for baseurl in opts.repos:
res, msg = add_local_repo(my_mock_config, my_mock_config, baseurl)
if not res:
log(opts.logfile, "Error: Could not add: %s to yum config in mock chroot: %s" % (baseurl, msg))
sys.exit(1)
res, msg = set_basedir(my_mock_config, my_mock_config, opts.basedir, opts)
if not res:
log(opts.logfile, "Error: Could not write out local config: %s" % msg)
sys.exit(1)
# these files needed from the mock.config dir to make mock run
for fn in ['site-defaults.cfg', 'logging.ini']:
pth = mockconfig_path + '/' + fn
shutil.copyfile(pth, opts.config_path + '/' + fn)
# createrepo on it
err = createrepo(opts.local_repo_dir)[1]
if err.strip():
log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir)
log(opts.logfile, "Err: %s" % err)
sys.exit(1)
init_build_env(max_workers, opts, config_opts)
download_dir = tempfile.mkdtemp()
downloaded_pkgs = {}
built_pkgs = []
try_again = True
to_be_built = pkgs
return_code = 0
num_of_tries = 0
g_opts = opts
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGHUP, signal_handler)
signal.signal(signal.SIGABRT, signal_handler)
while try_again and not stop_signal:
num_of_tries += 1
failed = []
log(opts.logfile, "===== iteration %d start =====" % num_of_tries)
to_be_built_scheduled = to_be_built[:]
need_reap = False
while len(to_be_built_scheduled) > 0:
# Free up a worker
while need_reap or workers >= max_workers:
need_reap = False
reaped = reaper(opts)
if reaped == 0:
time.sleep(0.1)
if workers < max_workers:
workers = workers + 1
b = get_idle_build_env(max_workers)
if b < 0:
log(opts.logfile, "Failed to find idle build env for: %s" % pkg)
workers = workers - 1
need_reap = True
continue
pkg = schedule(b, to_be_built_scheduled, opts)
if pkg is None:
if workers <= 1:
# Remember we have one build environmnet reserved, so can't test for zero workers
log(opts.logfile, "failed to schedule from: %s" % to_be_built_scheduled)
pkg = to_be_built_scheduled[0]
log(opts.logfile, "All workers idle, forcing build of pkg=%s" % pkg)
else:
release_build_env(b)
workers = workers - 1
need_reap = True
continue
to_be_built_scheduled.remove(pkg)
if not pkg.endswith('.rpm'):
log(opts.logfile, "%s doesn't appear to be an rpm - skipping" % pkg)
failed.append(pkg)
release_build_env(b)
need_reap = True
continue
elif pkg.startswith('http://') or pkg.startswith('https://') or pkg.startswith('ftp://'):
url = pkg
try:
log(opts.logfile, 'Fetching %s' % url)
r = requests.get(url)
# pylint: disable=no-member
if r.status_code == requests.codes.ok:
fn = urlsplit(r.url).path.rsplit('/', 1)[1]
if 'content-disposition' in r.headers:
_, params = cgi.parse_header(r.headers['content-disposition'])
if 'filename' in params and params['filename']:
fn = params['filename']
pkg = download_dir + '/' + fn
with open(pkg, 'wb') as fd:
for chunk in r.iter_content(4096):
fd.write(chunk)
except Exception as e:
log(opts.logfile, 'Error Downloading %s: %s' % (url, str(e)))
failed.append(url)
release_build_env(b)
need_reap = True
continue
else:
downloaded_pkgs[pkg] = url
log(opts.logfile, "Start build on 'b%d': %s" % (b, pkg))
# ret = do_build(opts, config_opts['chroot_name'], pkg)[0]
p = multiprocessing.Process(target=do_build, args=(opts, build_env[b]['cfg'], pkg))
worker_data.append({'proc': p, 'pkg': pkg, 'build_index': int(b)})
p.start()
# Wait for remaining processes to complete
log(opts.logfile, "===== wait for last jobs in iteration %d to complete =====" % num_of_tries)
while workers > 0:
reaped = reaper(opts)
if reaped == 0:
time.sleep(0.1)
log(opts.logfile, "===== iteration %d complete =====" % num_of_tries)
if failed and opts.recurse:
log(opts.logfile, "failed=%s" % failed)
log(opts.logfile, "to_be_built=%s" % to_be_built)
if len(failed) != len(to_be_built):
to_be_built = failed
try_again = True
log(opts.logfile, 'Some package succeeded, some failed.')
log(opts.logfile, 'Trying to rebuild %s failed pkgs, because --recurse is set.' % len(failed))
else:
if max_workers > 1:
max_workers = 1
to_be_built = failed
try_again = True
log(opts.logfile, 'Some package failed under parallel build.')
log(opts.logfile, 'Trying to rebuild %s failed pkgs with single thread, because --recurse is set.' % len(failed))
else:
log(opts.logfile, "")
log(opts.logfile, "*** Build Failed ***")
log(opts.logfile, "Tried %s times - following pkgs could not be successfully built:" % num_of_tries)
log(opts.logfile, "*** Build Failed ***")
for pkg in failed:
msg = pkg
if pkg in downloaded_pkgs:
msg = downloaded_pkgs[pkg]
log(opts.logfile, msg)
log(opts.logfile, "")
try_again = False
else:
try_again = False
if failed:
return_code = 2
# cleaning up our download dir
shutil.rmtree(download_dir, ignore_errors=True)
log(opts.logfile, "")
log(opts.logfile, "Results out to: %s" % opts.local_repo_dir)
log(opts.logfile, "")
log(opts.logfile, "Pkgs built: %s" % len(built_pkgs))
if built_pkgs:
if failed:
if len(built_pkgs):
log(opts.logfile, "Some packages successfully built in this order:")
else:
log(opts.logfile, "Packages successfully built in this order:")
for pkg in built_pkgs:
log(opts.logfile, pkg)
return return_code
if __name__ == "__main__":
sys.exit(main(sys.argv))
View Git Blame Copy Permalink