Allow the use of previously built parent images

Consider the case where you have a set of tagged / released images to
production, and you want to apply a small hotfix, security patch, etc.
to one of them, say keystone-ssh.

Rebuilding keystone-ssh will cause a rebuild of base, openstack-base,
and keystone-base, potentially pulling in updated packages / new
dependencies, unless you have access to the exact machine they were
previously built on. Ideally you want to use the exact parent images
that are in production.

This patch adds a new build argument '--skip-parents', which will result
in only the image(s) specified by the regex and their children to be
built. Parents are expected to exist either on machine or in a registry.

Change-Id: I79d5f0b422f48d2dc36ae85dfa21668cf8177837
Implements: blueprint image-build-depth

kolla/cmd/build.py

@@ -29,7 +29,8 @@ from kolla.image import build
 def main():
     statuses = build.run_build()
     if statuses:
-        bad_results, good_results, unmatched_results = statuses
+        (bad_results, good_results, unmatched_results,
+         skipped_results) = statuses
         if bad_results:
             return 1
     return 0

kolla/common/config.py

@@ -156,6 +156,8 @@ _CLI_OPTS = [
                help='The base image name. Default is the same with base'),
     cfg.BoolOpt('debug', short='d', default=False,
                 help='Turn on debugging log level'),
+    cfg.BoolOpt('skip-parents', default=False,
+                help='Do not rebuild parents of matched images'),
     cfg.DictOpt('build-args',
                 help='Set docker build time variables'),
     cfg.BoolOpt('keep', default=False,

kolla/image/build.py

@@ -88,6 +88,7 @@ STATUS_BUILDING = 'building'
 STATUS_UNMATCHED = 'unmatched'
 STATUS_MATCHED = 'matched'
 STATUS_UNPROCESSED = 'unprocessed'
+STATUS_SKIPPED = 'skipped'
 
 # All error status constants.
 STATUS_ERRORS = (STATUS_CONNECTION_ERROR, STATUS_PUSH_ERROR,
@@ -257,7 +258,7 @@ class BuildTask(DockerTask):
 
     def run(self):
         self.builder(self.image)
-        if self.image.status == STATUS_BUILT:
+        if self.image.status in (STATUS_BUILT, STATUS_SKIPPED):
             self.success = True
 
     @property
@@ -395,6 +396,11 @@ class BuildTask(DockerTask):
             return len(os.listdir(items_path))
 
         self.logger.debug('Processing')
+
+        if image.status == STATUS_SKIPPED:
+            self.logger.info('Skipping %s (--skip-parents)' % image.name)
+            return
+
         if image.status == STATUS_UNMATCHED:
             return
 
@@ -567,6 +573,7 @@ class KollaWorker(object):
         self.image_statuses_bad = dict()
         self.image_statuses_good = dict()
         self.image_statuses_unmatched = dict()
+        self.image_statuses_skipped = dict()
         self.maintainer = conf.maintainer
 
     def _get_images_dir(self):
@@ -777,14 +784,18 @@ class KollaWorker(object):
         if filter_:
             patterns = re.compile(r"|".join(filter_).join('()'))
             for image in self.images:
-                if image.status == STATUS_MATCHED:
+                if image.status in (STATUS_MATCHED, STATUS_SKIPPED):
                     continue
                 if re.search(patterns, image.name):
                     image.status = STATUS_MATCHED
                     while (image.parent is not None and
-                           image.parent.status != STATUS_MATCHED):
+                           image.parent.status not in (STATUS_MATCHED,
+                                                       STATUS_SKIPPED)):
                         image = image.parent
-                        image.status = STATUS_MATCHED
+                        if self.conf.skip_parents:
+                            image.status = STATUS_SKIPPED
+                        else:
+                            image.status = STATUS_MATCHED
                         LOG.debug('Image %s matched regex', image.name)
                 else:
                     image.status = STATUS_UNMATCHED
@@ -805,6 +816,7 @@ class KollaWorker(object):
             'built': [],
             'failed': [],
             'not_matched': [],
+            'skipped': [],
         }
 
         if self.image_statuses_good:
@@ -838,25 +850,40 @@ class KollaWorker(object):
                     'name': name,
                 })
 
+        if self.image_statuses_skipped:
+            LOG.debug("=====================================")
+            LOG.debug("Images skipped due to --skip-parents")
+            LOG.debug("=====================================")
+            for name in self.image_statuses_skipped.keys():
+                LOG.debug(name)
+                results['skipped'].append({
+                    'name': name,
+                })
+
         return results
 
     def get_image_statuses(self):
         if any([self.image_statuses_bad,
                 self.image_statuses_good,
-                self.image_statuses_unmatched]):
+                self.image_statuses_unmatched,
+                self.image_statuses_skipped]):
             return (self.image_statuses_bad,
                     self.image_statuses_good,
-                    self.image_statuses_unmatched)
+                    self.image_statuses_unmatched,
+                    self.image_statuses_skipped)
         for image in self.images:
             if image.status == STATUS_BUILT:
                 self.image_statuses_good[image.name] = image.status
             elif image.status == STATUS_UNMATCHED:
                 self.image_statuses_unmatched[image.name] = image.status
+            elif image.status == STATUS_SKIPPED:
+                self.image_statuses_skipped[image.name] = image.status
             else:
                 self.image_statuses_bad[image.name] = image.status
         return (self.image_statuses_bad,
                 self.image_statuses_good,
-                self.image_statuses_unmatched)
+                self.image_statuses_unmatched,
+                self.image_statuses_skipped)
 
     def build_image_list(self):
         def process_source_installation(image, section):

kolla/tests/test_build.py

@@ -209,6 +209,7 @@ class KollaWorkerTest(base.TestCase):
         image.status = None
         image_child = FAKE_IMAGE_CHILD.copy()
         image_child.status = None
+        image_child.parent.status = None
         image_unmatched = FAKE_IMAGE_CHILD_UNMATCHED.copy()
         image_error = FAKE_IMAGE_CHILD_ERROR.copy()
         image_built = FAKE_IMAGE_CHILD_BUILT.copy()
@@ -292,6 +293,15 @@ class KollaWorkerTest(base.TestCase):
         return [image for image in images
                 if image.status == build.STATUS_MATCHED]
 
+    def test_skip_parents(self):
+        self.conf.set_override('regex', 'image-child')
+        self.conf.set_override('skip_parents', True)
+        kolla = build.KollaWorker(self.conf)
+        kolla.images = self.images
+        kolla.filter_images()
+
+        self.assertEqual(build.STATUS_SKIPPED, kolla.images[1].parent.status)
+
     def test_without_profile(self):
         kolla = build.KollaWorker(self.conf)
         kolla.images = self.images
@@ -354,14 +364,14 @@ class MainTest(base.TestCase):
 
     @mock.patch.object(build, 'run_build')
     def test_images_built(self, mock_run_build):
-        image_statuses = ({}, {'img': 'built'}, {})
+        image_statuses = ({}, {'img': 'built'}, {}, {})
         mock_run_build.return_value = image_statuses
         result = build_cmd.main()
         self.assertEqual(0, result)
 
     @mock.patch.object(build, 'run_build')
     def test_images_unmatched(self, mock_run_build):
-        image_statuses = ({}, {}, {'img': 'unmatched'})
+        image_statuses = ({}, {}, {'img': 'unmatched'}, {})
         mock_run_build.return_value = image_statuses
         result = build_cmd.main()
         self.assertEqual(0, result)
@@ -374,7 +384,7 @@ class MainTest(base.TestCase):
 
     @mock.patch.object(build, 'run_build')
     def test_bad_images(self, mock_run_build):
-        image_statuses = ({'img': 'error'}, {}, {})
+        image_statuses = ({'img': 'error'}, {}, {}, {})
         mock_run_build.return_value = image_statuses
         result = build_cmd.main()
         self.assertEqual(1, result)
@@ -383,3 +393,10 @@ class MainTest(base.TestCase):
     def test_run_build(self, mock_sys):
         result = build.run_build()
         self.assertTrue(result)
+
+    @mock.patch.object(build, 'run_build')
+    def test_skipped_images(self, mock_run_build):
+        image_statuses = ({}, {}, {}, {'img': 'skipped'})
+        mock_run_build.return_value = image_statuses
+        result = build_cmd.main()
+        self.assertEqual(0, result)

releasenotes/notes/build-skip-parents-102a82736935f027.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Added a new build argument '--skip-parents', which will result in only
+    the image(s) specified by the regex and their children to be built.
+    Parents are expected to exist either on machine or in a registry.

tests/test_build.py

@@ -41,7 +41,8 @@ class BuildTest(object):
     def runTest(self):
         with patch.object(sys, 'argv', self.build_args):
             LOG.info("Running with args %s", self.build_args)
-            bad_results, good_results, unmatched_results = build.run_build()
+            (bad_results, good_results, unmatched_results,
+             skipped_results) = build.run_build()
 
         failures = 0
         for image, result in bad_results.items():