Allow the use of previously built parent images

Consider the case where you have a set of tagged / released images to
production, and you want to apply a small hotfix, security patch, etc.
to one of them, say keystone-ssh.

Rebuilding keystone-ssh will cause a rebuild of base, openstack-base,
and keystone-base, potentially pulling in updated packages / new
dependencies, unless you have access to the exact machine they were
previously built on. Ideally you want to use the exact parent images
that are in production.

This patch adds a new build argument '--skip-parents', which will result
in only the image(s) specified by the regex and their children to be
built. Parents are expected to exist either on machine or in a registry.

Change-Id: I79d5f0b422f48d2dc36ae85dfa21668cf8177837
Implements: blueprint image-build-depth
This commit is contained in:
Paul Bourke 2017-01-27 17:25:56 +00:00 committed by Dave Walker
parent 2c801637e6
commit e5410950c4
6 changed files with 66 additions and 12 deletions

View File

@ -29,7 +29,8 @@ from kolla.image import build
def main(): def main():
statuses = build.run_build() statuses = build.run_build()
if statuses: if statuses:
bad_results, good_results, unmatched_results = statuses (bad_results, good_results, unmatched_results,
skipped_results) = statuses
if bad_results: if bad_results:
return 1 return 1
return 0 return 0

View File

@ -156,6 +156,8 @@ _CLI_OPTS = [
help='The base image name. Default is the same with base'), help='The base image name. Default is the same with base'),
cfg.BoolOpt('debug', short='d', default=False, cfg.BoolOpt('debug', short='d', default=False,
help='Turn on debugging log level'), help='Turn on debugging log level'),
cfg.BoolOpt('skip-parents', default=False,
help='Do not rebuild parents of matched images'),
cfg.DictOpt('build-args', cfg.DictOpt('build-args',
help='Set docker build time variables'), help='Set docker build time variables'),
cfg.BoolOpt('keep', default=False, cfg.BoolOpt('keep', default=False,

View File

@ -88,6 +88,7 @@ STATUS_BUILDING = 'building'
STATUS_UNMATCHED = 'unmatched' STATUS_UNMATCHED = 'unmatched'
STATUS_MATCHED = 'matched' STATUS_MATCHED = 'matched'
STATUS_UNPROCESSED = 'unprocessed' STATUS_UNPROCESSED = 'unprocessed'
STATUS_SKIPPED = 'skipped'
# All error status constants. # All error status constants.
STATUS_ERRORS = (STATUS_CONNECTION_ERROR, STATUS_PUSH_ERROR, STATUS_ERRORS = (STATUS_CONNECTION_ERROR, STATUS_PUSH_ERROR,
@ -257,7 +258,7 @@ class BuildTask(DockerTask):
def run(self): def run(self):
self.builder(self.image) self.builder(self.image)
if self.image.status == STATUS_BUILT: if self.image.status in (STATUS_BUILT, STATUS_SKIPPED):
self.success = True self.success = True
@property @property
@ -395,6 +396,11 @@ class BuildTask(DockerTask):
return len(os.listdir(items_path)) return len(os.listdir(items_path))
self.logger.debug('Processing') self.logger.debug('Processing')
if image.status == STATUS_SKIPPED:
self.logger.info('Skipping %s (--skip-parents)' % image.name)
return
if image.status == STATUS_UNMATCHED: if image.status == STATUS_UNMATCHED:
return return
@ -567,6 +573,7 @@ class KollaWorker(object):
self.image_statuses_bad = dict() self.image_statuses_bad = dict()
self.image_statuses_good = dict() self.image_statuses_good = dict()
self.image_statuses_unmatched = dict() self.image_statuses_unmatched = dict()
self.image_statuses_skipped = dict()
self.maintainer = conf.maintainer self.maintainer = conf.maintainer
def _get_images_dir(self): def _get_images_dir(self):
@ -777,14 +784,18 @@ class KollaWorker(object):
if filter_: if filter_:
patterns = re.compile(r"|".join(filter_).join('()')) patterns = re.compile(r"|".join(filter_).join('()'))
for image in self.images: for image in self.images:
if image.status == STATUS_MATCHED: if image.status in (STATUS_MATCHED, STATUS_SKIPPED):
continue continue
if re.search(patterns, image.name): if re.search(patterns, image.name):
image.status = STATUS_MATCHED image.status = STATUS_MATCHED
while (image.parent is not None and while (image.parent is not None and
image.parent.status != STATUS_MATCHED): image.parent.status not in (STATUS_MATCHED,
STATUS_SKIPPED)):
image = image.parent image = image.parent
image.status = STATUS_MATCHED if self.conf.skip_parents:
image.status = STATUS_SKIPPED
else:
image.status = STATUS_MATCHED
LOG.debug('Image %s matched regex', image.name) LOG.debug('Image %s matched regex', image.name)
else: else:
image.status = STATUS_UNMATCHED image.status = STATUS_UNMATCHED
@ -805,6 +816,7 @@ class KollaWorker(object):
'built': [], 'built': [],
'failed': [], 'failed': [],
'not_matched': [], 'not_matched': [],
'skipped': [],
} }
if self.image_statuses_good: if self.image_statuses_good:
@ -838,25 +850,40 @@ class KollaWorker(object):
'name': name, 'name': name,
}) })
if self.image_statuses_skipped:
LOG.debug("=====================================")
LOG.debug("Images skipped due to --skip-parents")
LOG.debug("=====================================")
for name in self.image_statuses_skipped.keys():
LOG.debug(name)
results['skipped'].append({
'name': name,
})
return results return results
def get_image_statuses(self): def get_image_statuses(self):
if any([self.image_statuses_bad, if any([self.image_statuses_bad,
self.image_statuses_good, self.image_statuses_good,
self.image_statuses_unmatched]): self.image_statuses_unmatched,
self.image_statuses_skipped]):
return (self.image_statuses_bad, return (self.image_statuses_bad,
self.image_statuses_good, self.image_statuses_good,
self.image_statuses_unmatched) self.image_statuses_unmatched,
self.image_statuses_skipped)
for image in self.images: for image in self.images:
if image.status == STATUS_BUILT: if image.status == STATUS_BUILT:
self.image_statuses_good[image.name] = image.status self.image_statuses_good[image.name] = image.status
elif image.status == STATUS_UNMATCHED: elif image.status == STATUS_UNMATCHED:
self.image_statuses_unmatched[image.name] = image.status self.image_statuses_unmatched[image.name] = image.status
elif image.status == STATUS_SKIPPED:
self.image_statuses_skipped[image.name] = image.status
else: else:
self.image_statuses_bad[image.name] = image.status self.image_statuses_bad[image.name] = image.status
return (self.image_statuses_bad, return (self.image_statuses_bad,
self.image_statuses_good, self.image_statuses_good,
self.image_statuses_unmatched) self.image_statuses_unmatched,
self.image_statuses_skipped)
def build_image_list(self): def build_image_list(self):
def process_source_installation(image, section): def process_source_installation(image, section):

View File

@ -209,6 +209,7 @@ class KollaWorkerTest(base.TestCase):
image.status = None image.status = None
image_child = FAKE_IMAGE_CHILD.copy() image_child = FAKE_IMAGE_CHILD.copy()
image_child.status = None image_child.status = None
image_child.parent.status = None
image_unmatched = FAKE_IMAGE_CHILD_UNMATCHED.copy() image_unmatched = FAKE_IMAGE_CHILD_UNMATCHED.copy()
image_error = FAKE_IMAGE_CHILD_ERROR.copy() image_error = FAKE_IMAGE_CHILD_ERROR.copy()
image_built = FAKE_IMAGE_CHILD_BUILT.copy() image_built = FAKE_IMAGE_CHILD_BUILT.copy()
@ -292,6 +293,15 @@ class KollaWorkerTest(base.TestCase):
return [image for image in images return [image for image in images
if image.status == build.STATUS_MATCHED] if image.status == build.STATUS_MATCHED]
def test_skip_parents(self):
self.conf.set_override('regex', 'image-child')
self.conf.set_override('skip_parents', True)
kolla = build.KollaWorker(self.conf)
kolla.images = self.images
kolla.filter_images()
self.assertEqual(build.STATUS_SKIPPED, kolla.images[1].parent.status)
def test_without_profile(self): def test_without_profile(self):
kolla = build.KollaWorker(self.conf) kolla = build.KollaWorker(self.conf)
kolla.images = self.images kolla.images = self.images
@ -354,14 +364,14 @@ class MainTest(base.TestCase):
@mock.patch.object(build, 'run_build') @mock.patch.object(build, 'run_build')
def test_images_built(self, mock_run_build): def test_images_built(self, mock_run_build):
image_statuses = ({}, {'img': 'built'}, {}) image_statuses = ({}, {'img': 'built'}, {}, {})
mock_run_build.return_value = image_statuses mock_run_build.return_value = image_statuses
result = build_cmd.main() result = build_cmd.main()
self.assertEqual(0, result) self.assertEqual(0, result)
@mock.patch.object(build, 'run_build') @mock.patch.object(build, 'run_build')
def test_images_unmatched(self, mock_run_build): def test_images_unmatched(self, mock_run_build):
image_statuses = ({}, {}, {'img': 'unmatched'}) image_statuses = ({}, {}, {'img': 'unmatched'}, {})
mock_run_build.return_value = image_statuses mock_run_build.return_value = image_statuses
result = build_cmd.main() result = build_cmd.main()
self.assertEqual(0, result) self.assertEqual(0, result)
@ -374,7 +384,7 @@ class MainTest(base.TestCase):
@mock.patch.object(build, 'run_build') @mock.patch.object(build, 'run_build')
def test_bad_images(self, mock_run_build): def test_bad_images(self, mock_run_build):
image_statuses = ({'img': 'error'}, {}, {}) image_statuses = ({'img': 'error'}, {}, {}, {})
mock_run_build.return_value = image_statuses mock_run_build.return_value = image_statuses
result = build_cmd.main() result = build_cmd.main()
self.assertEqual(1, result) self.assertEqual(1, result)
@ -383,3 +393,10 @@ class MainTest(base.TestCase):
def test_run_build(self, mock_sys): def test_run_build(self, mock_sys):
result = build.run_build() result = build.run_build()
self.assertTrue(result) self.assertTrue(result)
@mock.patch.object(build, 'run_build')
def test_skipped_images(self, mock_run_build):
image_statuses = ({}, {}, {}, {'img': 'skipped'})
mock_run_build.return_value = image_statuses
result = build_cmd.main()
self.assertEqual(0, result)

View File

@ -0,0 +1,6 @@
---
features:
- |
Added a new build argument '--skip-parents', which will result in only
the image(s) specified by the regex and their children to be built.
Parents are expected to exist either on machine or in a registry.

View File

@ -41,7 +41,8 @@ class BuildTest(object):
def runTest(self): def runTest(self):
with patch.object(sys, 'argv', self.build_args): with patch.object(sys, 'argv', self.build_args):
LOG.info("Running with args %s", self.build_args) LOG.info("Running with args %s", self.build_args)
bad_results, good_results, unmatched_results = build.run_build() (bad_results, good_results, unmatched_results,
skipped_results) = build.run_build()
failures = 0 failures = 0
for image, result in bad_results.items(): for image, result in bad_results.items():