Change how build process treats unbuildable images

We have a list of images which are unbuildable for distro/buildtype/arch
combos. This patch renames it from SKIPPED_IMAGES to UNBUILDABLE_IMAGES
because 'skipped' images is something else in our code.

Now all unbuildable images are marked as such and so are their children.

At the end of build there is information provided about skipped images
(due to --skip-parents or --skip-existing options) and then information
about not buildable images. Both list should only contain entries
related to requested build.

Change-Id: If9b521339f564e483cba03d52e7c4eba271821a5

kolla/cmd/build.py

@@ -33,7 +33,7 @@ def main():
         statuses = build.run_build()
         if statuses:
             (bad_results, good_results, unmatched_results,
-             skipped_results) = statuses
+             skipped_results, unbuildable_results) = statuses
             if bad_results:
                 return 1
         return 0

kolla/image/build.py

@@ -71,16 +71,17 @@ STATUS_UNMATCHED = 'unmatched'
 STATUS_MATCHED = 'matched'
 STATUS_UNPROCESSED = 'unprocessed'
 STATUS_SKIPPED = 'skipped'
+STATUS_UNBUILDABLE = 'unbuildable'
 
 # All error status constants.
 STATUS_ERRORS = (STATUS_CONNECTION_ERROR, STATUS_PUSH_ERROR,
                  STATUS_ERROR, STATUS_PARENT_ERROR)
 
-# The dictionary of skipped images supports keys in the format:
+# The dictionary of unbuildable images supports keys in the format:
 # '<distro>+<installation_type>+<arch>' where each component is optional
 # and can be omitted along with the + separator which means that component
 # is irrelevant. Otherwise all must match for skip to happen.
-SKIPPED_IMAGES = {
+UNBUILDABLE_IMAGES = {
     'aarch64': {
         "cyborg-base",       # no binary package
         "kibana",            # no binary package
@@ -505,7 +506,7 @@ class BuildTask(DockerTask):
 
         self.logger.debug('Processing')
 
-        if image.status == STATUS_SKIPPED:
+        if image.status in [STATUS_SKIPPED, STATUS_UNBUILDABLE]:
             self.logger.info('Skipping %s' % image.name)
             return
 
@@ -741,6 +742,7 @@ class KollaWorker(object):
         self.image_statuses_good = dict()
         self.image_statuses_unmatched = dict()
         self.image_statuses_skipped = dict()
+        self.image_statuses_unbuildable = dict()
         self.maintainer = conf.maintainer
         self.distro_python_version = conf.distro_python_version
 
@@ -1000,44 +1002,80 @@ class KollaWorker(object):
                 else:
                     filter_ += self.conf.profiles[profile]
 
-        if filter_:
+        # mark unbuildable images and their children
-            patterns = re.compile(r"|".join(filter_).join('()'))
-            for image in self.images:
-                if image.status in (STATUS_MATCHED, STATUS_SKIPPED):
-                    continue
-                if re.search(patterns, image.name):
-                    image.status = STATUS_MATCHED
-                    while (image.parent is not None and
-                           image.parent.status not in (STATUS_MATCHED,
-                                                       STATUS_SKIPPED)):
-                        image = image.parent
-                        if self.conf.skip_parents:
-                            image.status = STATUS_SKIPPED
-                        elif (self.conf.skip_existing and
-                              image.in_docker_cache()):
-                            image.status = STATUS_SKIPPED
-                        else:
-                            image.status = STATUS_MATCHED
-                        LOG.debug('Image %s matched regex', image.name)
-                else:
-                    image.status = STATUS_UNMATCHED
-        else:
-            for image in self.images:
-                image.status = STATUS_MATCHED
-
-        # Unmatch (skip) unsupported images
         tag_element = r'(%s|%s|%s)' % (self.base,
                                        self.install_type,
                                        self.base_arch)
         tag_re = re.compile(r'^%s(\+%s)*$' % (tag_element, tag_element))
-        skipped_images = set()
+        unbuildable_images = set()
-        for set_tag in SKIPPED_IMAGES:
+        for set_tag in UNBUILDABLE_IMAGES:
             if tag_re.match(set_tag):
-                skipped_images.update(SKIPPED_IMAGES[set_tag])
+                unbuildable_images.update(UNBUILDABLE_IMAGES[set_tag])
-        if skipped_images:
+
+        if unbuildable_images:
             for image in self.images:
-                if image.name in skipped_images:
+                if image.name in unbuildable_images:
+                    image.status = STATUS_UNBUILDABLE
+                else:
+                    # let's check ancestors
+                    # if any of them is unbuildable then we mark it
+                    # and then mark image
+                    build_image = True
+                    ancestor_image = image
+                    while (ancestor_image.parent is not None):
+                        ancestor_image = ancestor_image.parent
+                        if ancestor_image.name in unbuildable_images or \
+                           ancestor_image.status == STATUS_UNBUILDABLE:
+                            build_image = False
+                            ancestor_image.status = STATUS_UNBUILDABLE
+                            break
+                    if not build_image:
+                        image.status = STATUS_UNBUILDABLE
+
+        # When we want to build a subset of images then filter_ part kicks in.
+        # Otherwise we just mark everything buildable as matched for build.
+
+        if filter_:
+            patterns = re.compile(r"|".join(filter_).join('()'))
+            for image in self.images:
+                # as we now list not buildable/skipped images we need to
+                # process them otherwise list will contain also not requested
+                # entries
+                if image.status == STATUS_MATCHED:
+                    continue
+                if re.search(patterns, image.name):
+                    if image.status not in [STATUS_SKIPPED,
+                                            STATUS_UNBUILDABLE]:
+                        image.status = STATUS_MATCHED
+
+                    # skip image if --skip-existing was given and image
+                    # was already built
+                    if (self.conf.skip_existing and image.in_docker_cache()):
+                        image.status = STATUS_SKIPPED
+
+                    # handle image ancestors
+                    ancestor_image = image
+                    while (ancestor_image.parent is not None and
+                           ancestor_image.parent.status not in
+                           (STATUS_MATCHED, STATUS_SKIPPED)):
+                        ancestor_image = ancestor_image.parent
+                        if self.conf.skip_parents:
+                            ancestor_image.status = STATUS_SKIPPED
+                        elif (self.conf.skip_existing and
+                              ancestor_image.in_docker_cache()):
+                            ancestor_image.status = STATUS_SKIPPED
+                        else:
+                            if ancestor_image.status != STATUS_UNBUILDABLE:
+                                ancestor_image.status = STATUS_MATCHED
+                        LOG.debug('Image %s matched regex', image.name)
+                else:
+                    # we do not care if it is skipped or not as we did not
+                    # request it
                     image.status = STATUS_UNMATCHED
+        else:
+            for image in self.images:
+                if image.status != STATUS_UNBUILDABLE:
+                    image.status = STATUS_MATCHED
 
     def summary(self):
         """Walk the dictionary of images statuses and print results."""
@@ -1053,6 +1091,7 @@ class KollaWorker(object):
             'failed': [],
             'not_matched': [],
             'skipped': [],
+            'unbuildable': [],
         }
 
         if self.image_statuses_good:
@@ -1097,26 +1136,38 @@ class KollaWorker(object):
                 })
 
         if self.image_statuses_skipped:
-            LOG.debug("================================")
+            LOG.info("===================================")
-            LOG.debug("Images skipped due build options")
+            LOG.info("Images skipped due to build options")
-            LOG.debug("================================")
+            LOG.info("===================================")
             for name in sorted(self.image_statuses_skipped.keys()):
-                LOG.debug(name)
+                LOG.info(name)
                 results['skipped'].append({
                     'name': name,
                 })
 
+        if self.image_statuses_unbuildable:
+            LOG.info("=========================================")
+            LOG.info("Images not buildable due to build options")
+            LOG.info("=========================================")
+            for name in sorted(self.image_statuses_unbuildable.keys()):
+                LOG.info(name)
+                results['unbuildable'].append({
+                    'name': name,
+                })
+
         return results
 
     def get_image_statuses(self):
         if any([self.image_statuses_bad,
                 self.image_statuses_good,
                 self.image_statuses_unmatched,
-                self.image_statuses_skipped]):
+                self.image_statuses_skipped,
+                self.image_statuses_unbuildable]):
             return (self.image_statuses_bad,
                     self.image_statuses_good,
                     self.image_statuses_unmatched,
-                    self.image_statuses_skipped)
+                    self.image_statuses_skipped,
+                    self.image_statuses_unbuildable)
         for image in self.images:
             if image.status == STATUS_BUILT:
                 self.image_statuses_good[image.name] = image.status
@@ -1124,12 +1175,15 @@ class KollaWorker(object):
                 self.image_statuses_unmatched[image.name] = image.status
             elif image.status == STATUS_SKIPPED:
                 self.image_statuses_skipped[image.name] = image.status
+            elif image.status == STATUS_UNBUILDABLE:
+                self.image_statuses_unbuildable[image.name] = image.status
             else:
                 self.image_statuses_bad[image.name] = image.status
         return (self.image_statuses_bad,
                 self.image_statuses_good,
                 self.image_statuses_unmatched,
-                self.image_statuses_skipped)
+                self.image_statuses_skipped,
+                self.image_statuses_unbuildable)
 
     def build_image_list(self):
         def process_source_installation(image, section):
@@ -1285,7 +1339,8 @@ class KollaWorker(object):
         queue = six.moves.queue.Queue()
 
         for image in self.images:
-            if image.status in (STATUS_UNMATCHED, STATUS_SKIPPED):
+            if image.status in (STATUS_UNMATCHED, STATUS_SKIPPED,
+                                STATUS_UNBUILDABLE):
                 # Don't bother queuing up build tasks for things that
                 # were not matched in the first place... (not worth the
                 # effort to run them, if they won't be used anyway).

kolla/tests/test_build.py

@@ -541,14 +541,14 @@ class MainTest(base.TestCase):
 
     @mock.patch.object(build, 'run_build')
     def test_images_built(self, mock_run_build):
-        image_statuses = ({}, {'img': 'built'}, {}, {})
+        image_statuses = ({}, {'img': 'built'}, {}, {}, {})
         mock_run_build.return_value = image_statuses
         result = build_cmd.main()
         self.assertEqual(0, result)
 
     @mock.patch.object(build, 'run_build')
     def test_images_unmatched(self, mock_run_build):
-        image_statuses = ({}, {}, {'img': 'unmatched'}, {})
+        image_statuses = ({}, {}, {'img': 'unmatched'}, {}, {})
         mock_run_build.return_value = image_statuses
         result = build_cmd.main()
         self.assertEqual(0, result)
@@ -561,7 +561,7 @@ class MainTest(base.TestCase):
 
     @mock.patch.object(build, 'run_build')
     def test_bad_images(self, mock_run_build):
-        image_statuses = ({'img': 'error'}, {}, {}, {})
+        image_statuses = ({'img': 'error'}, {}, {}, {}, {})
         mock_run_build.return_value = image_statuses
         result = build_cmd.main()
         self.assertEqual(1, result)
@@ -574,7 +574,14 @@ class MainTest(base.TestCase):
 
     @mock.patch.object(build, 'run_build')
     def test_skipped_images(self, mock_run_build):
-        image_statuses = ({}, {}, {}, {'img': 'skipped'})
+        image_statuses = ({}, {}, {}, {'img': 'skipped'}, {})
+        mock_run_build.return_value = image_statuses
+        result = build_cmd.main()
+        self.assertEqual(0, result)
+
+    @mock.patch.object(build, 'run_build')
+    def test_unbuildable_images(self, mock_run_build):
+        image_statuses = ({}, {}, {}, {}, {'img': 'unbuildable'})
         mock_run_build.return_value = image_statuses
         result = build_cmd.main()
         self.assertEqual(0, result)

tests/test_build.py

@@ -48,11 +48,11 @@ class BuildTest(object):
         with patch.object(sys, 'argv', self.build_args):
             LOG.info("Running with args %s", self.build_args)
             (bad_results, good_results, unmatched_results,
-             skipped_results) = build.run_build()
+             skipped_results, unbuildable_results) = build.run_build()
 
         failures = 0
         for image, result in bad_results.items():
-            if result is not 'error':
+            if result != 'error':
                 continue
             failures = failures + 1
             LOG.critical(">>> Expected image '%s' to succeed!", image)