#!/usr/bin/python2.7 -tt # -*- coding: utf-8 -*- # vim: noai:ts=4:sw=4:expandtab # by skvidal@fedoraproject.org # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Library General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. # copyright 2012 Red Hat, Inc. # SUMMARY # mockchain # take a mock config and a series of srpms # rebuild them one at a time # adding each to a local repo # so they are available as build deps to next pkg being built from __future__ import print_function import cgi # pylint: disable=deprecated-module import optparse import os import re import shutil import subprocess import sys import tempfile import time import multiprocessing import signal import psutil import requests # pylint: disable=import-error from six.moves.urllib_parse import urlsplit import mockbuild.util from stxRpmUtils import splitRpmFilename # all of the variables below are substituted by the build system __VERSION__="1.3.4" SYSCONFDIR="/etc" PYTHONDIR="/usr/lib/python2.7/site-packages" PKGPYTHONDIR="/usr/lib/python2.7/site-packages/mockbuild" MOCKCONFDIR = os.path.join(SYSCONFDIR, "mock") # end build system subs mockconfig_path = '/etc/mock' def rpmName(path): filename = os.path.basename(path) (n, v, r, e, a) = splitRpmFilename(filename) return n def createrepo(path): global max_workers if os.path.exists(path + '/repodata/repomd.xml'): comm = ['/usr/bin/createrepo_c', '--update', '--retain-old-md', "%d" % max_workers, "--workers", "%d" % max_workers, path] else: comm = ['/usr/bin/createrepo_c', "--workers", "%d" % max_workers, path] cmd = subprocess.Popen( comm, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = cmd.communicate() return out, err g_opts = optparse.Values() def parse_args(args): parser = optparse.OptionParser('\nmockchain -r mockcfg pkg1 [pkg2] [pkg3]') parser.add_option( '-r', '--root', default=None, dest='chroot', metavar="CONFIG", help="chroot config name/base to use in the mock build") parser.add_option( '-l', '--localrepo', default=None, help="local path for the local repo, defaults to making its own") parser.add_option( '-c', '--continue', default=False, action='store_true', dest='cont', help="if a pkg fails to build, continue to the next one") parser.add_option( '-a', '--addrepo', default=[], action='append', dest='repos', help="add these repo baseurls to the chroot's yum config") parser.add_option( '--recurse', default=False, action='store_true', help="if more than one pkg and it fails to build, try to build the rest and come back to it") parser.add_option( '--log', default=None, dest='logfile', help="log to the file named by this option, defaults to not logging") parser.add_option( '--workers', default=1, dest='max_workers', help="number of parallel build jobs") parser.add_option( '--worker-resources', default="", dest='worker_resources', help="colon seperated list, how much mem in gb for each workers temfs") parser.add_option( '--basedir', default='/var/lib/mock', dest='basedir', help="path to workspace") parser.add_option( '--tmp_prefix', default=None, dest='tmp_prefix', help="tmp dir prefix - will default to username-pid if not specified") parser.add_option( '-m', '--mock-option', default=[], action='append', dest='mock_option', help="option to pass directly to mock") parser.add_option( '--mark-slow-name', default=[], action='append', dest='slow_pkg_names_raw', help="package name that is known to build slowly") parser.add_option( '--mark-slow-path', default=[], action='append', dest='slow_pkgs_raw', help="package path that is known to build slowly") parser.add_option( '--mark-big-name', default=[], action='append', dest='big_pkg_names_raw', help="package name that is known to require a lot of disk space to build") parser.add_option( '--mark-big-path', default=[], action='append', dest='big_pkgs_raw', help="package path that is known to require a lot of disk space to build") parser.add_option( '--srpm-dependency-file', default=None, dest='srpm_dependency_file', help="path to srpm dependency file") parser.add_option( '--rpm-dependency-file', default=None, dest='rpm_dependency_file', help="path to rpm dependency file") parser.add_option( '--rpm-to-srpm-map-file', default=None, dest='rpm_to_srpm_map_file', help="path to rpm to srpm map file") opts, args = parser.parse_args(args) if opts.recurse: opts.cont = True if not opts.chroot: print("You must provide an argument to -r for the mock chroot") sys.exit(1) if len(sys.argv) < 3: print("You must specify at least 1 package to build") sys.exit(1) return opts, args REPOS_ID = [] slow_pkg_names={} slow_pkgs={} big_pkg_names={} big_pkgs={} def generate_repo_id(baseurl): """ generate repository id for yum.conf out of baseurl """ repoid = "/".join(baseurl.split('//')[1:]).replace('/', '_') repoid = re.sub(r'[^a-zA-Z0-9_]', '', repoid) suffix = '' i = 1 while repoid + suffix in REPOS_ID: suffix = str(i) i += 1 repoid = repoid + suffix REPOS_ID.append(repoid) return repoid def set_build_idx(infile, destfile, build_idx, tmpfs_size_gb, opts): # log(opts.logfile, "set_build_idx: infile=%s, destfile=%s, build_idx=%d, tmpfs_size_gb=%d" % (infile, destfile, build_idx, tmpfs_size_gb)) try: with open(infile) as f: code = compile(f.read(), infile, 'exec') # pylint: disable=exec-used exec(code) config_opts['root'] = config_opts['root'].replace('b0', 'b{0}'.format(build_idx)) config_opts['cache_topdir'] = config_opts['cache_topdir'].replace('b0', 'b{0}'.format(build_idx)) # log(opts.logfile, "set_build_idx: root=%s" % config_opts['root']) # log(opts.logfile, "set_build_idx: cache_topdir=%s" % config_opts['cache_topdir']) if tmpfs_size_gb > 0: config_opts['plugin_conf']['tmpfs_enable'] = True config_opts['plugin_conf']['tmpfs_opts'] = {} config_opts['plugin_conf']['tmpfs_opts']['required_ram_mb'] = 1024 config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = "%dg" % tmpfs_size_gb config_opts['plugin_conf']['tmpfs_opts']['mode'] = '0755' config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = True # log(opts.logfile, "set_build_idx: plugin_conf->tmpfs_enable=%s" % config_opts['plugin_conf']['tmpfs_enable']) # log(opts.logfile, "set_build_idx: plugin_conf->tmpfs_opts->max_fs_size=%s" % config_opts['plugin_conf']['tmpfs_opts']['max_fs_size']) with open(destfile, 'w') as br_dest: for k, v in list(config_opts.items()): br_dest.write("config_opts[%r] = %r\n" % (k, v)) try: log(opts.logfile, "set_build_idx: os.makedirs %s" % config_opts['cache_topdir']) if not os.path.isdir(config_opts['cache_topdir']): os.makedirs(config_opts['cache_topdir'], exist_ok=True) except (IOError, OSError): return False, "Could not create dir: %s" % config_opts['cache_topdir'] cache_dir = "%s/%s/mock" % (config_opts['basedir'], config_opts['root']) try: log(opts.logfile, "set_build_idx: os.makedirs %s" % cache_dir) if not os.path.isdir(cache_dir): os.makedirs(cache_dir) except (IOError, OSError): return False, "Could not create dir: %s" % cache_dir return True, '' except (IOError, OSError): return False, "Could not write mock config to %s" % destfile return True, '' def set_basedir(infile, destfile, basedir, opts): log(opts.logfile, "set_basedir: infile=%s, destfile=%s, basedir=%s" % (infile, destfile, basedir)) try: with open(infile) as f: code = compile(f.read(), infile, 'exec') # pylint: disable=exec-used exec(code) config_opts['basedir'] = basedir config_opts['resultdir'] = '{0}/result'.format(basedir) config_opts['backup_base_dir'] = '{0}/backup'.format(basedir) config_opts['root'] = 'mock/b0' config_opts['cache_topdir'] = '{0}/cache/b0'.format(basedir) with open(destfile, 'w') as br_dest: for k, v in list(config_opts.items()): br_dest.write("config_opts[%r] = %r\n" % (k, v)) return True, '' except (IOError, OSError): return False, "Could not write mock config to %s" % destfile return True, '' def add_local_repo(infile, destfile, baseurl, repoid=None): """take a mock chroot config and add a repo to it's yum.conf infile = mock chroot config file destfile = where to save out the result baseurl = baseurl of repo you wish to add""" global config_opts try: with open(infile) as f: code = compile(f.read(), infile, 'exec') # pylint: disable=exec-used exec(code) if not repoid: repoid = generate_repo_id(baseurl) else: REPOS_ID.append(repoid) localyumrepo = """ [%s] name=%s baseurl=%s enabled=1 skip_if_unavailable=1 metadata_expire=0 cost=1 best=1 """ % (repoid, baseurl, baseurl) config_opts['yum.conf'] += localyumrepo with open(destfile, 'w') as br_dest: for k, v in list(config_opts.items()): br_dest.write("config_opts[%r] = %r\n" % (k, v)) return True, '' except (IOError, OSError): return False, "Could not write mock config to %s" % destfile return True, '' def do_build(opts, cfg, pkg): # returns 0, cmd, out, err = failure # returns 1, cmd, out, err = success # returns 2, None, None, None = already built signal.signal(signal.SIGTERM, child_signal_handler) signal.signal(signal.SIGINT, child_signal_handler) signal.signal(signal.SIGHUP, child_signal_handler) signal.signal(signal.SIGABRT, child_signal_handler) s_pkg = os.path.basename(pkg) pdn = s_pkg.replace('.src.rpm', '') resdir = '%s/%s' % (opts.local_repo_dir, pdn) resdir = os.path.normpath(resdir) if not os.path.exists(resdir): os.makedirs(resdir) success_file = resdir + '/success' fail_file = resdir + '/fail' if os.path.exists(success_file): # return 2, None, None, None sys.exit(2) # clean it up if we're starting over :) if os.path.exists(fail_file): os.unlink(fail_file) if opts.uniqueext == '': mockcmd = ['/usr/bin/mock', '--configdir', opts.config_path, '--resultdir', resdir, '-r', cfg, ] else: mockcmd = ['/usr/bin/mock', '--configdir', opts.config_path, '--resultdir', resdir, '--uniqueext', opts.uniqueext, '-r', cfg, ] # Ensure repo is up-to-date. # Note: Merely adding --update to mockcmd failed to update mockcmd_update=mockcmd mockcmd_update.append('--update') cmd = subprocess.Popen( mockcmd_update, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = cmd.communicate() if cmd.returncode != 0: if (isinstance(err, bytes)): err = err.decode("utf-8") sys.stderr.write(err) # heuristic here, if user pass for mock "-d foo", but we must be care to leave # "-d'foo bar'" or "--define='foo bar'" as is compiled_re_1 = re.compile(r'^(-\S)\s+(.+)') compiled_re_2 = re.compile(r'^(--[^ =])[ =](\.+)') for option in opts.mock_option: r_match = compiled_re_1.match(option) if r_match: mockcmd.extend([r_match.group(1), r_match.group(2)]) else: r_match = compiled_re_2.match(option) if r_match: mockcmd.extend([r_match.group(1), r_match.group(2)]) else: mockcmd.append(option) print('building %s' % s_pkg) mockcmd.append(pkg) # print("mockcmd: %s" % str(mockcmd)) cmd = subprocess.Popen( mockcmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = cmd.communicate() if cmd.returncode == 0: with open(success_file, 'w') as f: f.write('done\n') ret = 1 else: if (isinstance(err, bytes)): err = err.decode("utf-8") sys.stderr.write(err) with open(fail_file, 'w') as f: f.write('undone\n') ret = 0 # return ret, cmd, out, err sys.exit(ret) def log(lf, msg): if lf: now = time.time() try: with open(lf, 'a') as f: f.write(str(now) + ':' + msg + '\n') except (IOError, OSError) as e: print('Could not write to logfile %s - %s' % (lf, str(e))) print(msg) config_opts = {} worker_data = [] workers = 0 max_workers = 1 build_env = [] failed = [] built_pkgs = [] local_repo_dir = "" pkg_to_name={} name_to_pkg={} srpm_dependencies_direct={} rpm_dependencies_direct={} rpm_to_srpm_map={} no_dep_list = [ "bash", "kernel" , "kernel-rt" ] def init_build_env(slots, opts, config_opts_in): global build_env orig_chroot_name=config_opts_in['chroot_name'] orig_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(orig_chroot_name)) # build_env.append({'state': 'Idle', 'cfg': orig_mock_config}) for i in range(0,slots): new_chroot_name = "{0}.b{1}".format(orig_chroot_name, i) new_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(new_chroot_name)) tmpfs_size_gb = 0 if opts.worker_resources == "": if i > 0: tmpfs_size_gb = 2 * (1 + slots - i) else: resource_array=opts.worker_resources.split(':') if i < len(resource_array): tmpfs_size_gb=int(resource_array[i]) else: log(opts.logfile, "Error: worker-resources argument '%s' does not supply info for all %d workers" % (opts.worker_resources, slots)) sys.exit(1) if i == 0 and tmpfs_size_gb != 0: log(opts.logfile, "Error: worker-resources argument '%s' must pass '0' as first value" % (opts.worker_resources, slots)) sys.exit(1) build_env.append({'state': 'Idle', 'cfg': new_mock_config, 'fs_size_gb': tmpfs_size_gb}) res, msg = set_build_idx(orig_mock_config, new_mock_config, i, tmpfs_size_gb, opts) if not res: log(opts.logfile, "Error: Could not write out local config: %s" % msg) sys.exit(1) idle_build_env_last_awarded = 0 def get_idle_build_env(slots): global build_env global idle_build_env_last_awarded visited = 0 if slots < 1: return -1 i = idle_build_env_last_awarded - 1 if i < 0 or i >= slots: i = slots - 1 while visited < slots: if build_env[i]['state'] == 'Idle': build_env[i]['state'] = 'Busy' idle_build_env_last_awarded = i return i visited = visited + 1 i = i - 1 if i < 0: i = slots - 1 return -1 def release_build_env(idx): global build_env build_env[idx]['state'] = 'Idle' def get_best_rc(a, b): print("get_best_rc: a=%s" % str(a)) print("get_best_rc: b=%s" % str(b)) if (b == {}) and (a != {}): return a if (a == {}) and (b != {}): return b if (b['build_name'] is None) and (not a['build_name'] is None): return a if (a['build_name'] is None) and (not b['build_name'] is None): return b if a['unbuilt_deps'] < b['unbuilt_deps']: return a if b['unbuilt_deps'] < a['unbuilt_deps']: return b if a['depth'] < b['depth']: return a if b['depth'] < a['depth']: return b print("get_best_rc: uncertain %s vs %s" % (a,b)) return a unbuilt_dep_list_print=False def unbuilt_dep_list(name, unbuilt_pkg_names, depth, checked=None): global srpm_dependencies_direct global rpm_dependencies_direct global rpm_to_srpm_map global no_dep_list global unbuilt_dep_list_print first_iteration=False unbuilt = [] if name in no_dep_list: return unbuilt if checked is None: first_iteration=True checked=[] # Count unbuild dependencies if first_iteration: dependencies_direct=srpm_dependencies_direct else: dependencies_direct=rpm_dependencies_direct if name in dependencies_direct: for rdep in dependencies_direct[name]: sdep='???' if rdep in rpm_to_srpm_map: sdep = rpm_to_srpm_map[rdep] if rdep != name and sdep != name and not rdep in checked: if (not first_iteration) and (sdep in no_dep_list): continue checked.append(rdep) if sdep in unbuilt_pkg_names: if not sdep in unbuilt: unbuilt.append(sdep) if depth > 0: child_unbuilt = unbuilt_dep_list(rdep, unbuilt_pkg_names, depth-1, checked) for sub_sdep in child_unbuilt: if sub_sdep != name: if not sub_sdep in unbuilt: unbuilt.append(sub_sdep) return unbuilt def can_build_at_idx(build_idx, name, opts): global pkg_to_name global name_to_pkg global big_pkgs global big_pkg_names global slow_pkgs global slow_pkg_names global build_env fs_size_gb = 0 size_gb = 0 speed = 0 pkg = name_to_pkg[name] if name in big_pkg_names: size_gb=big_pkg_names[name] if pkg in big_pkgs: size_gb=big_pkgs[pkg] if name in slow_pkg_names: speed=slow_pkg_names[name] if pkg in slow_pkgs: speed=slow_pkgs[pkg] fs_size_gb = build_env[build_idx]['fs_size_gb'] return fs_size_gb == 0 or fs_size_gb >= size_gb def schedule(build_idx, pkgs, opts): global worker_data global pkg_to_name global name_to_pkg global big_pkgs global big_pkg_names global slow_pkgs global slow_pkg_names unbuilt_pkg_names=[] building_pkg_names=[] unprioritized_pkg_names=[] for pkg in pkgs: name = pkg_to_name[pkg] unbuilt_pkg_names.append(name) unprioritized_pkg_names.append(name) prioritized_pkg_names=[] for wd in worker_data: pkg = wd['pkg'] if not pkg is None: name = pkg_to_name[pkg] building_pkg_names.append(name) # log(opts.logfile, "schedule: build_idx=%d start" % build_idx) if len(big_pkg_names) or len(big_pkgs): next_unprioritized_pkg_names = unprioritized_pkg_names[:] for name in unprioritized_pkg_names: pkg = name_to_pkg[name] if name in big_pkg_names or pkg in big_pkgs: prioritized_pkg_names.append(name) next_unprioritized_pkg_names.remove(name) unprioritized_pkg_names = next_unprioritized_pkg_names[:] if len(slow_pkg_names) or len(slow_pkgs): next_unprioritized_pkg_names = unprioritized_pkg_names[:] for name in unprioritized_pkg_names: pkg = name_to_pkg[name] if name in slow_pkg_names or pkg in slow_pkgs: if can_build_at_idx(build_idx, name, opts): prioritized_pkg_names.append(name) next_unprioritized_pkg_names.remove(name) unprioritized_pkg_names = next_unprioritized_pkg_names[:] for name in unprioritized_pkg_names: if can_build_at_idx(build_idx, name, opts): prioritized_pkg_names.append(name) name_out = schedule2(build_idx, prioritized_pkg_names, unbuilt_pkg_names, building_pkg_names, opts) if not name_out is None: pkg_out = name_to_pkg[name_out] else: pkg_out = None # log(opts.logfile, "schedule: failed to translate '%s' to a pkg" % name_out) # log(opts.logfile, "schedule: build_idx=%d end: out = %s -> %s" % (build_idx, str(name_out), str(pkg_out))) return pkg_out def schedule2(build_idx, pkg_names, unbuilt_pkg_names, building_pkg_names, opts): global pkg_to_name global name_to_pkg global no_dep_list max_depth = 3 if len(pkg_names) == 0: return None unbuilt_deps={} building_deps={} for depth in range(max_depth,-1,-1): unbuilt_deps[depth]={} building_deps[depth]={} for depth in range(max_depth,-1,-1): checked=[] reordered_pkg_names = pkg_names[:] # for name in reordered_pkg_names: while len(reordered_pkg_names): name = reordered_pkg_names.pop(0) if name in checked: continue # log(opts.logfile, "checked.append(%s)" % name) checked.append(name) pkg = name_to_pkg[name] # log(opts.logfile, "schedule2: check '%s', depth %d" % (name, depth)) if not name in unbuilt_deps[depth]: unbuilt_deps[depth][name] = unbuilt_dep_list(name, unbuilt_pkg_names, depth) if not name in building_deps[depth]: building_deps[depth][name] = unbuilt_dep_list(name, building_pkg_names, depth) # log(opts.logfile, "schedule2: unbuilt deps for pkg=%s, depth=%d: %s" % (name, depth, unbuilt_deps[depth][name])) # log(opts.logfile, "schedule2: building deps for pkg=%s, depth=%d: %s" % (name, depth, building_deps[depth][name])) if len(unbuilt_deps[depth][name]) == 0 and len(building_deps[depth][name]) == 0: if can_build_at_idx(build_idx, name, opts): log(opts.logfile, "schedule2: no unbuilt deps for '%s', searching at depth %d" % (name, depth)) return name else: # log(opts.logfile, "schedule2: Can't build '%s' on 'b%d'" % (name, build_idx)) continue if not name in unbuilt_deps[0]: unbuilt_deps[0][name] = unbuilt_dep_list(name, unbuilt_pkg_names, 0) if not name in building_deps[0]: building_deps[0][name] = unbuilt_dep_list(name, building_pkg_names, 0) # log(opts.logfile, "schedule2: unbuilt deps for pkg=%s, depth=%d: %s" % (name, 0, unbuilt_deps[0][name])) # log(opts.logfile, "schedule2: building deps for pkg=%s, depth=%d: %s" % (name, 0, building_deps[0][name])) if (len(building_deps[depth][name]) == 0 and len(unbuilt_deps[depth][name]) == 1 and unbuilt_deps[depth][name][0] in no_dep_list) or (len(unbuilt_deps[depth][name]) == 0 and len(building_deps[depth][name]) == 1 and building_deps[depth][name][0] in no_dep_list): if len(unbuilt_deps[0][name]) == 0 and len(building_deps[0][name]) == 0: if can_build_at_idx(build_idx, name, opts): log(opts.logfile, "schedule2: no unbuilt deps for '%s' except for indirect kernel dep, searching at depth %d" % (name, depth)) return name else: # log(opts.logfile, "schedule2: Can't build '%s' on 'b%d'" % (name, build_idx)) continue loop = False for dep_name in unbuilt_deps[depth][name]: if name == dep_name: continue # log(opts.logfile, "name=%s depends on dep_name=%s, depth=%d" % (name, dep_name, depth)) if dep_name in checked: continue # log(opts.logfile, "schedule2: check '%s' indirect" % dep_name) if not dep_name in unbuilt_deps[depth]: unbuilt_deps[depth][dep_name] = unbuilt_dep_list(dep_name, unbuilt_pkg_names, depth) if not dep_name in building_deps[depth]: building_deps[depth][dep_name] = unbuilt_dep_list(dep_name, building_pkg_names, depth) # log(opts.logfile, "schedule2: deps: unbuilt deps for %s -> %s, depth=%d: %s" % (name, dep_name, depth, unbuilt_deps[depth][dep_name])) # log(opts.logfile, "schedule2: deps: building deps for %s -> %s, depth=%d: %s" % (name, dep_name, depth, building_deps[depth][dep_name])) if len(unbuilt_deps[depth][dep_name]) == 0 and len(building_deps[depth][dep_name]) == 0: if can_build_at_idx(build_idx, dep_name, opts): log(opts.logfile, "schedule2: deps: no unbuilt deps for '%s', working towards '%s', searching at depth %d" % (dep_name, name, depth)) return dep_name if not dep_name in unbuilt_deps[0]: unbuilt_deps[0][dep_name] = unbuilt_dep_list(dep_name, unbuilt_pkg_names, 0) if not dep_name in building_deps[0]: building_deps[0][dep_name] = unbuilt_dep_list(dep_name, building_pkg_names, 0) # log(opts.logfile, "schedule2: deps: unbuilt deps for %s -> %s, depth=%d: %s" % (name, dep_name, 0, unbuilt_deps[0][dep_name])) # log(opts.logfile, "schedule2: deps: building deps for %s -> %s, depth=%d: %s" % (name, dep_name, 0, building_deps[0][dep_name])) if (len(building_deps[depth][dep_name]) == 0 and len(unbuilt_deps[depth][dep_name]) == 1 and unbuilt_deps[depth][dep_name][0] in no_dep_list) or (len(unbuilt_deps[depth][dep_name]) == 0 and len(building_deps[depth][dep_name]) == 1 and building_deps[depth][dep_name][0] in no_dep_list): if len(unbuilt_deps[0][dep_name]) == 0 and len(building_deps[0][dep_name]) == 0: if can_build_at_idx(build_idx, dep_name, opts): log(opts.logfile, "schedule2: no unbuilt deps for '%s' except for indirect kernel dep, working towards '%s', searching at depth %d" % (dep_name, name, depth)) return dep_name if name in unbuilt_deps[0][dep_name]: loop = True # log(opts.logfile, "schedule2: loop detected: %s <-> %s" % (name, dep_name)) if loop and len(building_deps[depth][name]) == 0: log(opts.logfile, "schedule2: loop detected, try to build '%s'" % name) return name for dep_name in unbuilt_deps[depth][name]: if dep_name in reordered_pkg_names: # log(opts.logfile, "schedule2: promote %s to work toward %s" % (dep_name, name)) reordered_pkg_names.remove(dep_name) reordered_pkg_names.insert(0,dep_name) # log(opts.logfile, "schedule2: Nothing buildable at this time") return None def read_deps(opts): read_srpm_deps(opts) read_rpm_deps(opts) read_map_deps(opts) def read_srpm_deps(opts): global srpm_dependencies_direct if opts.srpm_dependency_file == None: return if not os.path.exists(opts.srpm_dependency_file): log(opts.logfile, "File not found: %s" % opts.srpm_dependency_file) sys.exit(1) with open(opts.srpm_dependency_file) as f: lines = f.readlines() for line in lines: (name,deps) = line.rstrip().split(';') srpm_dependencies_direct[name]=deps.split(',') def read_rpm_deps(opts): global rpm_dependencies_direct if opts.rpm_dependency_file == None: return if not os.path.exists(opts.rpm_dependency_file): log(opts.logfile, "File not found: %s" % opts.rpm_dependency_file) sys.exit(1) with open(opts.rpm_dependency_file) as f: lines = f.readlines() for line in lines: (name,deps) = line.rstrip().split(';') rpm_dependencies_direct[name]=deps.split(',') def read_map_deps(opts): global rpm_to_srpm_map if opts.rpm_to_srpm_map_file == None: return if not os.path.exists(opts.rpm_to_srpm_map_file): log(opts.logfile, "File not found: %s" % opts.rpm_to_srpm_map_file) sys.exit(1) with open(opts.rpm_to_srpm_map_file) as f: lines = f.readlines() for line in lines: (rpm,srpm) = line.rstrip().split(';') rpm_to_srpm_map[rpm]=srpm def reaper(opts): global built_pkgs global failed global worker_data global workers reaped = 0 need_createrepo = False last_reaped = -1 while reaped > last_reaped: last_reaped = reaped for wd in worker_data: p = wd['proc'] ret = p.exitcode if ret is not None: pkg = wd['pkg'] b = int(wd['build_index']) p.join() worker_data.remove(wd) workers = workers - 1 reaped = reaped + 1 release_build_env(b) log(opts.logfile, "End build on 'b%d': %s" % (b, pkg)) if ret == 0: failed.append(pkg) log(opts.logfile, "Error building %s on 'b%d'." % (os.path.basename(pkg), b)) if opts.recurse and not stop_signal: log(opts.logfile, "Will try to build again (if some other package will succeed).") else: log(opts.logfile, "See logs/results in %s" % opts.local_repo_dir) elif ret == 1: log(opts.logfile, "Success building %s on 'b%d'" % (os.path.basename(pkg), b)) built_pkgs.append(pkg) need_createrepo = True elif ret == 2: log(opts.logfile, "Skipping already built pkg %s" % os.path.basename(pkg)) if need_createrepo: # createrepo with the new pkgs err = createrepo(opts.local_repo_dir)[1] if err.strip(): log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir) log(opts.logfile, "Err: %s" % err) return reaped stop_signal = False def on_terminate(proc): print("process {} terminated with exit code {}".format(proc, proc.returncode)) def kill_proc_and_descentents(parent, need_stop=False, verbose=False): global g_opts if need_stop: if verbose: log(g_opts.logfile, "Stop %d" % parent.pid) try: parent.send_signal(signal.SIGSTOP) except: # perhaps mock still running as root, give it a sec to drop pivledges and try again time.sleep(1) parent.send_signal(signal.SIGSTOP) children = parent.children(recursive=False) for p in children: kill_proc_and_descentents(p, need_stop=True, verbose=verbose) if verbose: log(g_opts.logfile, "Terminate %d" % parent.pid) # parent.send_signal(signal.SIGTERM) try: parent.terminate() except: # perhaps mock still running as root, give it a sec to drop pivledges and try again time.sleep(1) parent.terminate() if need_stop: if verbose: log(g_opts.logfile, "Continue %d" % parent.pid) parent.send_signal(signal.SIGCONT) def child_signal_handler(signum, frame): global g_opts my_pid = os.getpid() # log(g_opts.logfile, "--------- child %d recieved signal %d" % (my_pid, signum)) p = psutil.Process(my_pid) kill_proc_and_descentents(p) try: sys.exit(0) except SystemExit as e: os._exit(0) def signal_handler(signum, frame): global g_opts global stop_signal global workers global worker_data stop_signal = True # Signal processes to complete log(g_opts.logfile, "recieved signal %d, Terminating children" % signum) for wd in worker_data: p = wd['proc'] ret = p.exitcode if ret is None: # log(g_opts.logfile, "terminate child %d" % p.pid) p.terminate() else: log(g_opts.logfile, "child return code was %d" % ret) # Wait for remaining processes to complete log(g_opts.logfile, "===== wait for signaled jobs to complete =====") while len(worker_data) > 0: log(g_opts.logfile, " remaining workers: %d" % workers) reaped = reaper(g_opts) if reaped == 0: time.sleep(0.1) try: sys.exit(1) except SystemExit as e: os._exit(1) def main(args): opts, args = parse_args(args) # take mock config + list of pkgs global g_opts global stop_signal global build_env global worker_data global workers global max_workers global slow_pkg_names global slow_pkgs global big_pkg_names global big_pkgs max_workers = int(opts.max_workers) global failed global built_pkgs cfg = opts.chroot pkgs = args[1:] # transform slow/big package options into dictionaries for line in opts.slow_pkg_names_raw: speed,name = line.split(":") if speed != "": slow_pkg_names[name]=int(speed) for line in opts.slow_pkgs_raw: speed,pkg = line.split(":") if speed != "": slow_pkgs[pkg]=int(speed) for line in opts.big_pkg_names_raw: size_gb,name = line.split(":") if size_gb != "": big_pkg_names[name]=int(size_gb) for line in opts.big_pkgs_raw: size_gb,pkg = line.split(":") if size_gb != "": big_pkgs[pkg]=int(size_gb) # Set up a mapping between pkg path and pkg name global pkg_to_name global name_to_pkg for pkg in pkgs: if not pkg.endswith('.rpm'): log(opts.logfile, "%s doesn't appear to be an rpm - skipping" % pkg) continue try: name = rpmName(pkg) except OSError as e: print("Could not parse rpm %s" % pkg) sys.exit(1) pkg_to_name[pkg] = name name_to_pkg[name] = pkg read_deps(opts) global config_opts config_opts = mockbuild.util.load_config(mockconfig_path, cfg, None, __VERSION__, PKGPYTHONDIR) if not opts.tmp_prefix: try: opts.tmp_prefix = os.getlogin() except OSError as e: print("Could not find login name for tmp dir prefix add --tmp_prefix") sys.exit(1) pid = os.getpid() opts.uniqueext = '%s-%s' % (opts.tmp_prefix, pid) if opts.basedir != "/var/lib/mock": opts.uniqueext = '' # create a tempdir for our local info if opts.localrepo: local_tmp_dir = os.path.abspath(opts.localrepo) if not os.path.exists(local_tmp_dir): os.makedirs(local_tmp_dir) os.chmod(local_tmp_dir, 0o755) else: pre = 'mock-chain-%s-' % opts.uniqueext local_tmp_dir = tempfile.mkdtemp(prefix=pre, dir='/var/tmp') os.chmod(local_tmp_dir, 0o755) if opts.logfile: opts.logfile = os.path.join(local_tmp_dir, opts.logfile) if os.path.exists(opts.logfile): os.unlink(opts.logfile) log(opts.logfile, "starting logfile: %s" % opts.logfile) opts.local_repo_dir = os.path.normpath(local_tmp_dir + '/results/' + config_opts['chroot_name'] + '/') if not os.path.exists(opts.local_repo_dir): os.makedirs(opts.local_repo_dir, mode=0o755) local_baseurl = "file://%s" % opts.local_repo_dir log(opts.logfile, "results dir: %s" % opts.local_repo_dir) opts.config_path = os.path.normpath(local_tmp_dir + '/configs/' + config_opts['chroot_name'] + '/') if not os.path.exists(opts.config_path): os.makedirs(opts.config_path, mode=0o755) log(opts.logfile, "config dir: %s" % opts.config_path) my_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(config_opts['chroot_name'])) # modify with localrepo res, msg = add_local_repo(config_opts['config_file'], my_mock_config, local_baseurl, 'local_build_repo') if not res: log(opts.logfile, "Error: Could not write out local config: %s" % msg) sys.exit(1) for baseurl in opts.repos: res, msg = add_local_repo(my_mock_config, my_mock_config, baseurl) if not res: log(opts.logfile, "Error: Could not add: %s to yum config in mock chroot: %s" % (baseurl, msg)) sys.exit(1) res, msg = set_basedir(my_mock_config, my_mock_config, opts.basedir, opts) if not res: log(opts.logfile, "Error: Could not write out local config: %s" % msg) sys.exit(1) # these files needed from the mock.config dir to make mock run for fn in ['site-defaults.cfg', 'logging.ini']: pth = mockconfig_path + '/' + fn shutil.copyfile(pth, opts.config_path + '/' + fn) # createrepo on it err = createrepo(opts.local_repo_dir)[1] if err.strip(): log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir) log(opts.logfile, "Err: %s" % err) sys.exit(1) init_build_env(max_workers, opts, config_opts) download_dir = tempfile.mkdtemp() downloaded_pkgs = {} built_pkgs = [] try_again = True to_be_built = pkgs return_code = 0 num_of_tries = 0 g_opts = opts signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGHUP, signal_handler) signal.signal(signal.SIGABRT, signal_handler) while try_again and not stop_signal: num_of_tries += 1 failed = [] log(opts.logfile, "===== iteration %d start =====" % num_of_tries) to_be_built_scheduled = to_be_built[:] need_reap = False while len(to_be_built_scheduled) > 0: # Free up a worker while need_reap or workers >= max_workers: need_reap = False reaped = reaper(opts) if reaped == 0: time.sleep(0.1) if workers < max_workers: workers = workers + 1 b = get_idle_build_env(max_workers) if b < 0: log(opts.logfile, "Failed to find idle build env for: %s" % pkg) workers = workers - 1 need_reap = True continue pkg = schedule(b, to_be_built_scheduled, opts) if pkg is None: if workers <= 1: # Remember we have one build environmnet reserved, so can't test for zero workers log(opts.logfile, "failed to schedule from: %s" % to_be_built_scheduled) pkg = to_be_built_scheduled[0] log(opts.logfile, "All workers idle, forcing build of pkg=%s" % pkg) else: release_build_env(b) workers = workers - 1 need_reap = True continue to_be_built_scheduled.remove(pkg) if not pkg.endswith('.rpm'): log(opts.logfile, "%s doesn't appear to be an rpm - skipping" % pkg) failed.append(pkg) release_build_env(b) need_reap = True continue elif pkg.startswith('http://') or pkg.startswith('https://') or pkg.startswith('ftp://'): url = pkg try: log(opts.logfile, 'Fetching %s' % url) r = requests.get(url) # pylint: disable=no-member if r.status_code == requests.codes.ok: fn = urlsplit(r.url).path.rsplit('/', 1)[1] if 'content-disposition' in r.headers: _, params = cgi.parse_header(r.headers['content-disposition']) if 'filename' in params and params['filename']: fn = params['filename'] pkg = download_dir + '/' + fn with open(pkg, 'wb') as fd: for chunk in r.iter_content(4096): fd.write(chunk) except Exception as e: log(opts.logfile, 'Error Downloading %s: %s' % (url, str(e))) failed.append(url) release_build_env(b) need_reap = True continue else: downloaded_pkgs[pkg] = url log(opts.logfile, "Start build on 'b%d': %s" % (b, pkg)) # ret = do_build(opts, config_opts['chroot_name'], pkg)[0] p = multiprocessing.Process(target=do_build, args=(opts, build_env[b]['cfg'], pkg)) worker_data.append({'proc': p, 'pkg': pkg, 'build_index': int(b)}) p.start() # Wait for remaining processes to complete log(opts.logfile, "===== wait for last jobs in iteration %d to complete =====" % num_of_tries) while workers > 0: reaped = reaper(opts) if reaped == 0: time.sleep(0.1) log(opts.logfile, "===== iteration %d complete =====" % num_of_tries) if failed and opts.recurse: log(opts.logfile, "failed=%s" % failed) log(opts.logfile, "to_be_built=%s" % to_be_built) if len(failed) != len(to_be_built): to_be_built = failed try_again = True log(opts.logfile, 'Some package succeeded, some failed.') log(opts.logfile, 'Trying to rebuild %s failed pkgs, because --recurse is set.' % len(failed)) else: if max_workers > 1: max_workers = 1 to_be_built = failed try_again = True log(opts.logfile, 'Some package failed under parallel build.') log(opts.logfile, 'Trying to rebuild %s failed pkgs with single thread, because --recurse is set.' % len(failed)) else: log(opts.logfile, "") log(opts.logfile, "*** Build Failed ***") log(opts.logfile, "Tried %s times - following pkgs could not be successfully built:" % num_of_tries) log(opts.logfile, "*** Build Failed ***") for pkg in failed: msg = pkg if pkg in downloaded_pkgs: msg = downloaded_pkgs[pkg] log(opts.logfile, msg) log(opts.logfile, "") try_again = False else: try_again = False if failed: return_code = 2 # cleaning up our download dir shutil.rmtree(download_dir, ignore_errors=True) log(opts.logfile, "") log(opts.logfile, "Results out to: %s" % opts.local_repo_dir) log(opts.logfile, "") log(opts.logfile, "Pkgs built: %s" % len(built_pkgs)) if built_pkgs: if failed: if len(built_pkgs): log(opts.logfile, "Some packages successfully built in this order:") else: log(opts.logfile, "Packages successfully built in this order:") for pkg in built_pkgs: log(opts.logfile, pkg) return return_code if __name__ == "__main__": sys.exit(main(sys.argv))