starlingx/root
Code Issues Proposed changes

Slow builds due to sub-optimal build order

Browse Source 
Problem:
   In the absence of a dependency cache, build-pkgs has
not information on which packages should be built first.
All it can do is try to build each package in a arbitrary order.
If the build fails, it is set aside and will be retired on
the next iteration.  Builds fail when a BuildRequired package
can't be found, because it hasn't been built yet.
   This was not an issue when cgcs-tis-repo, the home of the cache,
was stored in a git.

Solution:
   Have build-pkgs invoke create_dependancy_cache.py if the
cache is not present.

Note: We only run this automatically once.  If the user is
adding packages, or modifying the BuildRequires or Requires of
an existing package, they should re-run create_dependancy_cache.py
manually.

Change-Id: Id63e42903d9c0884470a990fb62fe1bafacd1849
Story: 2002835
Task: 24519
Signed-off-by: Scott Little <scott.little@windriver.com>
This commit is contained in:
Scott Little 2018-08-22 10:31:13 -04:00
parent cbc9c29c5c
commit 354ee9e9c8
2 changed files with 86 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
build-tools/build-pkgs
View File

@ -1,8 +1,16 @@
#!/bin/bash
# This program is a wrapper around build-pkgs-parallel and build-pkgs-serial
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
#
# This program is a wrapper around build-pkgs-parallel and build-pkgs-serial
#
BUILD_PKGS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
usage () {
echo ""
@ -24,6 +32,15 @@ if [ $? -ne 0 ]; then
SERIAL_FLAG=1
fi
# Make sure we have a dependency cache
DEP_CACHE="$MY_REPO/cgcs-tis-repo/dependancy-cache"
if [ ! -d $DEP_CACHE ]; then
echo "Dependency cache is missing. Creating it now."
$BUILD_PKGS_DIR/create_dependancy_cache.py > $MY_WORKSPACE/create_dependancy_cache.log
echo "Dependency cache created."
echo ""
fi
if [ $SERIAL_FLAG -eq 1 ]; then
echo "build-pkgs-serial $@"
build-pkgs-serial "$@"

103
build-tools/create_dependancy_cache.py
View File

@ -1,5 +1,43 @@
#!/usr/bin/python
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
#
# Create a RPM dependency cache frpm the RPMS found in
# 1) $MY_REPO/cgcs-centos-repo
# 2) $MY_WORKSPACE/$BUILD_TYPE/rpmbuild/
#
# Cache files are written to $MY_REPO/cgcs-tis-repo/dependancy-cache
# unless an alternate path is supplied.
#
# The cache is a set of files that are easily digested by
# common shell script tools. Each file has format
# <rpm-name>;<comma-seperated-list-of-rpm-names>
#
# The files created are:
# RPM-direct-descendants RPMS that have a direct Requires on X
# RPM-transitive-descendants RPMS that have a possibly indirect need for X
#
# RPM-direct-requires RPMS directly Required by X
# RPM-transitive-requires RPMS possibly indirectly Required by X
#
# SRPM-direct-descendants SRPMS whos RPMS have a direct Requires on RPMS built by X
# SRPM-transitive-descendants SRPMS whos RPMS have a possibly indirect need for RPMS built by X
#
# SRPM-direct-requires SRPMS whos RPMS satisfy a direct BuildRequires of X
# SRPM-transitive-requires SRPMS whos RPMS satisfy an indirect BuildRequires of X
#
# SRPM-direct-requires-rpm RPMS that satisfy a direct BuildRequires of X
# SRPM-transitive-requires-rpm RPMS that satisfy an indirect BuildRequires of X
#
# rpm-to-srpm Map RPM back to the SRPM that created it
# srpm-to-rpm Map a SRPM to the set of RPMS it builds
#
import xml.etree.ElementTree as ET
import fnmatch
import os
@ -29,39 +67,27 @@ if not os.path.isdir(repodata_dir):
publish_cache_dir="%s/cgcs-tis-repo/dependancy-cache" % os.environ['MY_REPO']
centos_repo_dir="%s/cgcs-centos-repo" % os.environ['MY_REPO']
tis_repo_dir="%s/cgcs-tis-repo" % os.environ['MY_REPO']
workspace_repo_dirs={}
for rt in rpm_types:
workspace_repo_dirs[rt]={}
for bt in build_types:
workspace_repo_dirs[rt][bt]="%s/%s/rpmbuild/%sS" % (os.environ['MY_WORKSPACE'], bt, rt)
workspace_repo_dirs[rt]={}
for bt in build_types:
workspace_repo_dirs[rt][bt]="%s/%s/rpmbuild/%sS" % (os.environ['MY_WORKSPACE'], bt, rt)
if not os.path.isdir(os.environ['MY_REPO']):
print("ERROR: directory not found MY_REPO=%s" % os.environ['MY_REPO'])
sys.exit(1)
print("ERROR: directory not found MY_REPO=%s" % os.environ['MY_REPO'])
sys.exit(1)
if not os.path.isdir(centos_repo_dir):
print("ERROR: directory not found %s" % centos_repo_dir)
sys.exit(1)
if not os.path.isdir(tis_repo_dir):
print("ERROR: directory not found %s" % tis_repo_dir)
sys.exit(1)
# bin_rpm_mirror_roots = ["%s/fedora/epel/7" % repodata_dir,
# "%s/CentOS/7.2.1511" % repodata_dir,
# "%s/CentOS/tis-r3/" % repodata_dir ]
# src_rpm_mirror_roots = ["%s/fedora/dl.fedoraproject.org/pub/epel/7/SRPMS" % repodata_dir,
# "%s/CentOS/vault.centos.org/7.2.1511" % repodata_dir,
# "%s/CentOS/tis-r3/Source" % repodata_dir ]
print("ERROR: directory not found %s" % centos_repo_dir)
sys.exit(1)
bin_rpm_mirror_roots = ["%s/Binary" % centos_repo_dir]
src_rpm_mirror_roots = ["%s/Source" % centos_repo_dir]
for bt in build_types:
bin_rpm_mirror_roots.append(workspace_repo_dirs['RPM'][bt])
src_rpm_mirror_roots.append(workspace_repo_dirs['SRPM'][bt])
bin_rpm_mirror_roots.append(workspace_repo_dirs['RPM'][bt])
src_rpm_mirror_roots.append(workspace_repo_dirs['SRPM'][bt])
parser = OptionParser('create_dependancy_cache')
parser.add_option('-c', '--cache_dir', action='store', type='string',
@ -82,9 +108,10 @@ if options.third_party_repo_dir:
print("ERROR: directory not found %s" % third_party_repo_dir)
sys.exit(1)
# Create directory if required
if not os.path.isdir(publish_cache_dir):
print("ERROR: directory not found %s" % publish_cache_dir)
sys.exit(1)
print("Creating directory: %s" % publish_cache_dir)
os.makedirs(publish_cache_dir, 0755)
# The Main data structure
pkg_data={}
@ -156,19 +183,21 @@ def get_repo_primary_data_list(rpm_type='RPM', arch_list=default_arch_list):
if rpm_type == 'RPM':
for d in bin_rpm_mirror_roots:
sub_list = file_search(d, 'repodata', 25)
rpm_repodata_roots.extend(sub_list)
if os.path.isdir(d):
sub_list = file_search(d, 'repodata', 25)
rpm_repodata_roots.extend(sub_list)
elif rpm_type == 'SRPM':
for d in src_rpm_mirror_roots:
sub_list = file_search(d, 'repodata', 5)
rpm_repodata_roots.extend(sub_list)
if os.path.isdir(d):
sub_list = file_search(d, 'repodata', 5)
rpm_repodata_roots.extend(sub_list)
else:
print "invalid rpm_type '%s', valid types are %s" % (rpm_type, str(rpm_types))
return repodata_list
for d in rpm_repodata_roots:
sub_list = file_search(d, '*primary.xml.gz', 2)
repodata_list.extend(sub_list)
sub_list = file_search(d, '*primary.xml.gz', 2)
repodata_list.extend(sub_list)
return repodata_list
@ -183,12 +212,14 @@ def get_repo_filelists_data_list(rpm_type='RPM', arch_list=default_arch_list):
if rpm_type == 'RPM':
for d in bin_rpm_mirror_roots:
sub_list = file_search(d, 'repodata', 25)
rpm_repodata_roots.extend(sub_list)
if os.path.isdir(d):
sub_list = file_search(d, 'repodata', 25)
rpm_repodata_roots.extend(sub_list)
elif rpm_type == 'SRPM':
for d in src_rpm_mirror_roots:
sub_list = file_search(d, 'repodata', 5)
rpm_repodata_roots.extend(sub_list)
if os.path.isdir(d):
sub_list = file_search(d, 'repodata', 5)
rpm_repodata_roots.extend(sub_list)
else:
print "invalid rpm_type '%s', valid types are %s" % (rpm_type, str(rpm_types))
return repodata_list
@ -348,7 +379,7 @@ def calulate_all_direct_requires_and_descendants(rpm_type='RPM'):
calulate_pkg_direct_requires_and_descendants(name, rpm_type=rpm_type)
def calulate_pkg_direct_requires_and_descendants(name, rpm_type='RPM'):
print "SAL: %s needs:" % name
print "%s needs:" % name
if not rpm_type in pkg_data:
print "Error: unknown rpm_type '%s'" % rpm_type
return
@ -405,7 +436,7 @@ def calulate_pkg_direct_requires_and_descendants(name, rpm_type='RPM'):
if not name in pkg_data[rpm_type]['pkg_direct_descendants'][pro]:
pkg_data[rpm_type]['pkg_direct_descendants'][pro].append(name)
print "SAL: %s -> %s" % (req, pro)
print " %s -> %s" % (req, pro)