Merge "Slow builds due to sub-optimal build order"

build-tools/build-pkgs

@@ -1,8 +1,16 @@
 #!/bin/bash
 
-# This program is a wrapper around build-pkgs-parallel and build-pkgs-serial
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+#
+# This program is a wrapper around build-pkgs-parallel and build-pkgs-serial
+#
+
+BUILD_PKGS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
 
 usage () {
     echo ""
@@ -24,6 +32,15 @@ if [ $? -ne 0 ]; then
     SERIAL_FLAG=1
 fi
 
+# Make sure we have a dependency cache
+DEP_CACHE="$MY_REPO/cgcs-tis-repo/dependancy-cache"
+if [ ! -d $DEP_CACHE ]; then
+    echo "Dependency cache is missing.  Creating it now."
+    $BUILD_PKGS_DIR/create_dependancy_cache.py > $MY_WORKSPACE/create_dependancy_cache.log
+    echo "Dependency cache created."
+    echo ""
+fi
+
 if [ $SERIAL_FLAG -eq 1 ]; then
     echo "build-pkgs-serial $@"
     build-pkgs-serial "$@"

build-tools/create_dependancy_cache.py

@@ -1,5 +1,43 @@
 #!/usr/bin/python
 
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+#
+# Create a RPM dependency cache frpm the RPMS found in
+# 1) $MY_REPO/cgcs-centos-repo
+# 2) $MY_WORKSPACE/$BUILD_TYPE/rpmbuild/
+#
+# Cache files are written to $MY_REPO/cgcs-tis-repo/dependancy-cache
+# unless an alternate path is supplied.
+#
+# The cache is a set of files that are easily digested by 
+# common shell script tools.  Each file has format
+#   <rpm-name>;<comma-seperated-list-of-rpm-names>
+#
+# The files created are:
+#   RPM-direct-descendants       RPMS that have a direct Requires on X
+#   RPM-transitive-descendants   RPMS that have a possibly indirect need for X
+#
+#   RPM-direct-requires          RPMS directly Required by X
+#   RPM-transitive-requires      RPMS possibly indirectly Required by X
+#
+#   SRPM-direct-descendants      SRPMS whos RPMS have a direct Requires on RPMS built by X
+#   SRPM-transitive-descendants  SRPMS whos RPMS have a possibly indirect need for RPMS built by X
+#
+#   SRPM-direct-requires         SRPMS whos RPMS satisfy a direct BuildRequires of X
+#   SRPM-transitive-requires     SRPMS whos RPMS satisfy an indirect BuildRequires of X
+#
+#   SRPM-direct-requires-rpm      RPMS that satisfy a direct BuildRequires of X
+#   SRPM-transitive-requires-rpm  RPMS that satisfy an indirect BuildRequires of X
+#
+#   rpm-to-srpm                   Map RPM back to the SRPM that created it
+#   srpm-to-rpm                   Map a SRPM to the set of RPMS it builds
+#
+
 import xml.etree.ElementTree as ET
 import fnmatch
 import os
@@ -29,7 +67,7 @@ if not os.path.isdir(repodata_dir):
 
 publish_cache_dir="%s/cgcs-tis-repo/dependancy-cache" % os.environ['MY_REPO']
 centos_repo_dir="%s/cgcs-centos-repo" % os.environ['MY_REPO']
-tis_repo_dir="%s/cgcs-tis-repo" % os.environ['MY_REPO']
+
 workspace_repo_dirs={}
 for rt in rpm_types:
     workspace_repo_dirs[rt]={}
@@ -44,18 +82,6 @@ if not os.path.isdir(centos_repo_dir):
     print("ERROR: directory not found %s" % centos_repo_dir)
     sys.exit(1)
 
-if not os.path.isdir(tis_repo_dir):
-   print("ERROR: directory not found %s" % tis_repo_dir)
-   sys.exit(1)
-
-# bin_rpm_mirror_roots = ["%s/fedora/epel/7" % repodata_dir, 
-#                         "%s/CentOS/7.2.1511" % repodata_dir, 
-#                         "%s/CentOS/tis-r3/" % repodata_dir ]
-
-# src_rpm_mirror_roots = ["%s/fedora/dl.fedoraproject.org/pub/epel/7/SRPMS" % repodata_dir, 
-#                         "%s/CentOS/vault.centos.org/7.2.1511" % repodata_dir, 
-#                         "%s/CentOS/tis-r3/Source" % repodata_dir ]
-
 bin_rpm_mirror_roots = ["%s/Binary" % centos_repo_dir]
 src_rpm_mirror_roots = ["%s/Source" % centos_repo_dir]
 
@@ -82,9 +108,10 @@ if options.third_party_repo_dir:
         print("ERROR: directory not found %s" % third_party_repo_dir)
         sys.exit(1)
 
+# Create directory if required
 if not os.path.isdir(publish_cache_dir):
-    print("ERROR: directory not found %s" % publish_cache_dir)
+    print("Creating directory: %s" % publish_cache_dir)
-    sys.exit(1)
+    os.makedirs(publish_cache_dir, 0755)
 
 # The Main data structure
 pkg_data={}
@@ -156,10 +183,12 @@ def get_repo_primary_data_list(rpm_type='RPM', arch_list=default_arch_list):
 
     if rpm_type == 'RPM':
         for d in bin_rpm_mirror_roots:
+            if os.path.isdir(d):
                 sub_list = file_search(d, 'repodata', 25)
                 rpm_repodata_roots.extend(sub_list)
     elif rpm_type == 'SRPM':
         for d in src_rpm_mirror_roots:
+            if os.path.isdir(d):
                 sub_list = file_search(d, 'repodata', 5)
                 rpm_repodata_roots.extend(sub_list)
     else:
@@ -183,10 +212,12 @@ def get_repo_filelists_data_list(rpm_type='RPM', arch_list=default_arch_list):
 
     if rpm_type == 'RPM':
         for d in bin_rpm_mirror_roots:
+            if os.path.isdir(d):
                 sub_list = file_search(d, 'repodata', 25)
                 rpm_repodata_roots.extend(sub_list)
     elif rpm_type == 'SRPM':
         for d in src_rpm_mirror_roots:
+            if os.path.isdir(d):
                 sub_list = file_search(d, 'repodata', 5)
                 rpm_repodata_roots.extend(sub_list)
     else:
@@ -348,7 +379,7 @@ def calulate_all_direct_requires_and_descendants(rpm_type='RPM'):
         calulate_pkg_direct_requires_and_descendants(name, rpm_type=rpm_type)
 
 def calulate_pkg_direct_requires_and_descendants(name, rpm_type='RPM'):
-    print "SAL: %s needs:" % name
+    print "%s needs:" % name
     if not rpm_type in pkg_data:
         print "Error: unknown rpm_type '%s'" % rpm_type
         return
@@ -405,7 +436,7 @@ def calulate_pkg_direct_requires_and_descendants(name, rpm_type='RPM'):
             if not name in pkg_data[rpm_type]['pkg_direct_descendants'][pro]:
                 pkg_data[rpm_type]['pkg_direct_descendants'][pro].append(name)
 
-        print "SAL:    %s -> %s" % (req, pro)
+        print "    %s -> %s" % (req, pro)