Merge "Change how build process treats unbuildable images"
This commit is contained in:
commit
71c75a16a0
@ -33,7 +33,7 @@ def main():
|
|||||||
statuses = build.run_build()
|
statuses = build.run_build()
|
||||||
if statuses:
|
if statuses:
|
||||||
(bad_results, good_results, unmatched_results,
|
(bad_results, good_results, unmatched_results,
|
||||||
skipped_results) = statuses
|
skipped_results, unbuildable_results) = statuses
|
||||||
if bad_results:
|
if bad_results:
|
||||||
return 1
|
return 1
|
||||||
return 0
|
return 0
|
||||||
|
@ -71,16 +71,17 @@ STATUS_UNMATCHED = 'unmatched'
|
|||||||
STATUS_MATCHED = 'matched'
|
STATUS_MATCHED = 'matched'
|
||||||
STATUS_UNPROCESSED = 'unprocessed'
|
STATUS_UNPROCESSED = 'unprocessed'
|
||||||
STATUS_SKIPPED = 'skipped'
|
STATUS_SKIPPED = 'skipped'
|
||||||
|
STATUS_UNBUILDABLE = 'unbuildable'
|
||||||
|
|
||||||
# All error status constants.
|
# All error status constants.
|
||||||
STATUS_ERRORS = (STATUS_CONNECTION_ERROR, STATUS_PUSH_ERROR,
|
STATUS_ERRORS = (STATUS_CONNECTION_ERROR, STATUS_PUSH_ERROR,
|
||||||
STATUS_ERROR, STATUS_PARENT_ERROR)
|
STATUS_ERROR, STATUS_PARENT_ERROR)
|
||||||
|
|
||||||
# The dictionary of skipped images supports keys in the format:
|
# The dictionary of unbuildable images supports keys in the format:
|
||||||
# '<distro>+<installation_type>+<arch>' where each component is optional
|
# '<distro>+<installation_type>+<arch>' where each component is optional
|
||||||
# and can be omitted along with the + separator which means that component
|
# and can be omitted along with the + separator which means that component
|
||||||
# is irrelevant. Otherwise all must match for skip to happen.
|
# is irrelevant. Otherwise all must match for skip to happen.
|
||||||
SKIPPED_IMAGES = {
|
UNBUILDABLE_IMAGES = {
|
||||||
'aarch64': {
|
'aarch64': {
|
||||||
"cyborg-base", # no binary package
|
"cyborg-base", # no binary package
|
||||||
"kibana", # no binary package
|
"kibana", # no binary package
|
||||||
@ -505,7 +506,7 @@ class BuildTask(DockerTask):
|
|||||||
|
|
||||||
self.logger.debug('Processing')
|
self.logger.debug('Processing')
|
||||||
|
|
||||||
if image.status == STATUS_SKIPPED:
|
if image.status in [STATUS_SKIPPED, STATUS_UNBUILDABLE]:
|
||||||
self.logger.info('Skipping %s' % image.name)
|
self.logger.info('Skipping %s' % image.name)
|
||||||
return
|
return
|
||||||
|
|
||||||
@ -741,6 +742,7 @@ class KollaWorker(object):
|
|||||||
self.image_statuses_good = dict()
|
self.image_statuses_good = dict()
|
||||||
self.image_statuses_unmatched = dict()
|
self.image_statuses_unmatched = dict()
|
||||||
self.image_statuses_skipped = dict()
|
self.image_statuses_skipped = dict()
|
||||||
|
self.image_statuses_unbuildable = dict()
|
||||||
self.maintainer = conf.maintainer
|
self.maintainer = conf.maintainer
|
||||||
self.distro_python_version = conf.distro_python_version
|
self.distro_python_version = conf.distro_python_version
|
||||||
|
|
||||||
@ -1000,44 +1002,80 @@ class KollaWorker(object):
|
|||||||
else:
|
else:
|
||||||
filter_ += self.conf.profiles[profile]
|
filter_ += self.conf.profiles[profile]
|
||||||
|
|
||||||
if filter_:
|
# mark unbuildable images and their children
|
||||||
patterns = re.compile(r"|".join(filter_).join('()'))
|
|
||||||
for image in self.images:
|
|
||||||
if image.status in (STATUS_MATCHED, STATUS_SKIPPED):
|
|
||||||
continue
|
|
||||||
if re.search(patterns, image.name):
|
|
||||||
image.status = STATUS_MATCHED
|
|
||||||
while (image.parent is not None and
|
|
||||||
image.parent.status not in (STATUS_MATCHED,
|
|
||||||
STATUS_SKIPPED)):
|
|
||||||
image = image.parent
|
|
||||||
if self.conf.skip_parents:
|
|
||||||
image.status = STATUS_SKIPPED
|
|
||||||
elif (self.conf.skip_existing and
|
|
||||||
image.in_docker_cache()):
|
|
||||||
image.status = STATUS_SKIPPED
|
|
||||||
else:
|
|
||||||
image.status = STATUS_MATCHED
|
|
||||||
LOG.debug('Image %s matched regex', image.name)
|
|
||||||
else:
|
|
||||||
image.status = STATUS_UNMATCHED
|
|
||||||
else:
|
|
||||||
for image in self.images:
|
|
||||||
image.status = STATUS_MATCHED
|
|
||||||
|
|
||||||
# Unmatch (skip) unsupported images
|
|
||||||
tag_element = r'(%s|%s|%s)' % (self.base,
|
tag_element = r'(%s|%s|%s)' % (self.base,
|
||||||
self.install_type,
|
self.install_type,
|
||||||
self.base_arch)
|
self.base_arch)
|
||||||
tag_re = re.compile(r'^%s(\+%s)*$' % (tag_element, tag_element))
|
tag_re = re.compile(r'^%s(\+%s)*$' % (tag_element, tag_element))
|
||||||
skipped_images = set()
|
unbuildable_images = set()
|
||||||
for set_tag in SKIPPED_IMAGES:
|
for set_tag in UNBUILDABLE_IMAGES:
|
||||||
if tag_re.match(set_tag):
|
if tag_re.match(set_tag):
|
||||||
skipped_images.update(SKIPPED_IMAGES[set_tag])
|
unbuildable_images.update(UNBUILDABLE_IMAGES[set_tag])
|
||||||
if skipped_images:
|
|
||||||
|
if unbuildable_images:
|
||||||
for image in self.images:
|
for image in self.images:
|
||||||
if image.name in skipped_images:
|
if image.name in unbuildable_images:
|
||||||
|
image.status = STATUS_UNBUILDABLE
|
||||||
|
else:
|
||||||
|
# let's check ancestors
|
||||||
|
# if any of them is unbuildable then we mark it
|
||||||
|
# and then mark image
|
||||||
|
build_image = True
|
||||||
|
ancestor_image = image
|
||||||
|
while (ancestor_image.parent is not None):
|
||||||
|
ancestor_image = ancestor_image.parent
|
||||||
|
if ancestor_image.name in unbuildable_images or \
|
||||||
|
ancestor_image.status == STATUS_UNBUILDABLE:
|
||||||
|
build_image = False
|
||||||
|
ancestor_image.status = STATUS_UNBUILDABLE
|
||||||
|
break
|
||||||
|
if not build_image:
|
||||||
|
image.status = STATUS_UNBUILDABLE
|
||||||
|
|
||||||
|
# When we want to build a subset of images then filter_ part kicks in.
|
||||||
|
# Otherwise we just mark everything buildable as matched for build.
|
||||||
|
|
||||||
|
if filter_:
|
||||||
|
patterns = re.compile(r"|".join(filter_).join('()'))
|
||||||
|
for image in self.images:
|
||||||
|
# as we now list not buildable/skipped images we need to
|
||||||
|
# process them otherwise list will contain also not requested
|
||||||
|
# entries
|
||||||
|
if image.status == STATUS_MATCHED:
|
||||||
|
continue
|
||||||
|
if re.search(patterns, image.name):
|
||||||
|
if image.status not in [STATUS_SKIPPED,
|
||||||
|
STATUS_UNBUILDABLE]:
|
||||||
|
image.status = STATUS_MATCHED
|
||||||
|
|
||||||
|
# skip image if --skip-existing was given and image
|
||||||
|
# was already built
|
||||||
|
if (self.conf.skip_existing and image.in_docker_cache()):
|
||||||
|
image.status = STATUS_SKIPPED
|
||||||
|
|
||||||
|
# handle image ancestors
|
||||||
|
ancestor_image = image
|
||||||
|
while (ancestor_image.parent is not None and
|
||||||
|
ancestor_image.parent.status not in
|
||||||
|
(STATUS_MATCHED, STATUS_SKIPPED)):
|
||||||
|
ancestor_image = ancestor_image.parent
|
||||||
|
if self.conf.skip_parents:
|
||||||
|
ancestor_image.status = STATUS_SKIPPED
|
||||||
|
elif (self.conf.skip_existing and
|
||||||
|
ancestor_image.in_docker_cache()):
|
||||||
|
ancestor_image.status = STATUS_SKIPPED
|
||||||
|
else:
|
||||||
|
if ancestor_image.status != STATUS_UNBUILDABLE:
|
||||||
|
ancestor_image.status = STATUS_MATCHED
|
||||||
|
LOG.debug('Image %s matched regex', image.name)
|
||||||
|
else:
|
||||||
|
# we do not care if it is skipped or not as we did not
|
||||||
|
# request it
|
||||||
image.status = STATUS_UNMATCHED
|
image.status = STATUS_UNMATCHED
|
||||||
|
else:
|
||||||
|
for image in self.images:
|
||||||
|
if image.status != STATUS_UNBUILDABLE:
|
||||||
|
image.status = STATUS_MATCHED
|
||||||
|
|
||||||
def summary(self):
|
def summary(self):
|
||||||
"""Walk the dictionary of images statuses and print results."""
|
"""Walk the dictionary of images statuses and print results."""
|
||||||
@ -1053,6 +1091,7 @@ class KollaWorker(object):
|
|||||||
'failed': [],
|
'failed': [],
|
||||||
'not_matched': [],
|
'not_matched': [],
|
||||||
'skipped': [],
|
'skipped': [],
|
||||||
|
'unbuildable': [],
|
||||||
}
|
}
|
||||||
|
|
||||||
if self.image_statuses_good:
|
if self.image_statuses_good:
|
||||||
@ -1097,26 +1136,38 @@ class KollaWorker(object):
|
|||||||
})
|
})
|
||||||
|
|
||||||
if self.image_statuses_skipped:
|
if self.image_statuses_skipped:
|
||||||
LOG.debug("================================")
|
LOG.info("===================================")
|
||||||
LOG.debug("Images skipped due build options")
|
LOG.info("Images skipped due to build options")
|
||||||
LOG.debug("================================")
|
LOG.info("===================================")
|
||||||
for name in sorted(self.image_statuses_skipped.keys()):
|
for name in sorted(self.image_statuses_skipped.keys()):
|
||||||
LOG.debug(name)
|
LOG.info(name)
|
||||||
results['skipped'].append({
|
results['skipped'].append({
|
||||||
'name': name,
|
'name': name,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
if self.image_statuses_unbuildable:
|
||||||
|
LOG.info("=========================================")
|
||||||
|
LOG.info("Images not buildable due to build options")
|
||||||
|
LOG.info("=========================================")
|
||||||
|
for name in sorted(self.image_statuses_unbuildable.keys()):
|
||||||
|
LOG.info(name)
|
||||||
|
results['unbuildable'].append({
|
||||||
|
'name': name,
|
||||||
|
})
|
||||||
|
|
||||||
return results
|
return results
|
||||||
|
|
||||||
def get_image_statuses(self):
|
def get_image_statuses(self):
|
||||||
if any([self.image_statuses_bad,
|
if any([self.image_statuses_bad,
|
||||||
self.image_statuses_good,
|
self.image_statuses_good,
|
||||||
self.image_statuses_unmatched,
|
self.image_statuses_unmatched,
|
||||||
self.image_statuses_skipped]):
|
self.image_statuses_skipped,
|
||||||
|
self.image_statuses_unbuildable]):
|
||||||
return (self.image_statuses_bad,
|
return (self.image_statuses_bad,
|
||||||
self.image_statuses_good,
|
self.image_statuses_good,
|
||||||
self.image_statuses_unmatched,
|
self.image_statuses_unmatched,
|
||||||
self.image_statuses_skipped)
|
self.image_statuses_skipped,
|
||||||
|
self.image_statuses_unbuildable)
|
||||||
for image in self.images:
|
for image in self.images:
|
||||||
if image.status == STATUS_BUILT:
|
if image.status == STATUS_BUILT:
|
||||||
self.image_statuses_good[image.name] = image.status
|
self.image_statuses_good[image.name] = image.status
|
||||||
@ -1124,12 +1175,15 @@ class KollaWorker(object):
|
|||||||
self.image_statuses_unmatched[image.name] = image.status
|
self.image_statuses_unmatched[image.name] = image.status
|
||||||
elif image.status == STATUS_SKIPPED:
|
elif image.status == STATUS_SKIPPED:
|
||||||
self.image_statuses_skipped[image.name] = image.status
|
self.image_statuses_skipped[image.name] = image.status
|
||||||
|
elif image.status == STATUS_UNBUILDABLE:
|
||||||
|
self.image_statuses_unbuildable[image.name] = image.status
|
||||||
else:
|
else:
|
||||||
self.image_statuses_bad[image.name] = image.status
|
self.image_statuses_bad[image.name] = image.status
|
||||||
return (self.image_statuses_bad,
|
return (self.image_statuses_bad,
|
||||||
self.image_statuses_good,
|
self.image_statuses_good,
|
||||||
self.image_statuses_unmatched,
|
self.image_statuses_unmatched,
|
||||||
self.image_statuses_skipped)
|
self.image_statuses_skipped,
|
||||||
|
self.image_statuses_unbuildable)
|
||||||
|
|
||||||
def build_image_list(self):
|
def build_image_list(self):
|
||||||
def process_source_installation(image, section):
|
def process_source_installation(image, section):
|
||||||
@ -1285,7 +1339,8 @@ class KollaWorker(object):
|
|||||||
queue = six.moves.queue.Queue()
|
queue = six.moves.queue.Queue()
|
||||||
|
|
||||||
for image in self.images:
|
for image in self.images:
|
||||||
if image.status in (STATUS_UNMATCHED, STATUS_SKIPPED):
|
if image.status in (STATUS_UNMATCHED, STATUS_SKIPPED,
|
||||||
|
STATUS_UNBUILDABLE):
|
||||||
# Don't bother queuing up build tasks for things that
|
# Don't bother queuing up build tasks for things that
|
||||||
# were not matched in the first place... (not worth the
|
# were not matched in the first place... (not worth the
|
||||||
# effort to run them, if they won't be used anyway).
|
# effort to run them, if they won't be used anyway).
|
||||||
|
@ -541,14 +541,14 @@ class MainTest(base.TestCase):
|
|||||||
|
|
||||||
@mock.patch.object(build, 'run_build')
|
@mock.patch.object(build, 'run_build')
|
||||||
def test_images_built(self, mock_run_build):
|
def test_images_built(self, mock_run_build):
|
||||||
image_statuses = ({}, {'img': 'built'}, {}, {})
|
image_statuses = ({}, {'img': 'built'}, {}, {}, {})
|
||||||
mock_run_build.return_value = image_statuses
|
mock_run_build.return_value = image_statuses
|
||||||
result = build_cmd.main()
|
result = build_cmd.main()
|
||||||
self.assertEqual(0, result)
|
self.assertEqual(0, result)
|
||||||
|
|
||||||
@mock.patch.object(build, 'run_build')
|
@mock.patch.object(build, 'run_build')
|
||||||
def test_images_unmatched(self, mock_run_build):
|
def test_images_unmatched(self, mock_run_build):
|
||||||
image_statuses = ({}, {}, {'img': 'unmatched'}, {})
|
image_statuses = ({}, {}, {'img': 'unmatched'}, {}, {})
|
||||||
mock_run_build.return_value = image_statuses
|
mock_run_build.return_value = image_statuses
|
||||||
result = build_cmd.main()
|
result = build_cmd.main()
|
||||||
self.assertEqual(0, result)
|
self.assertEqual(0, result)
|
||||||
@ -561,7 +561,7 @@ class MainTest(base.TestCase):
|
|||||||
|
|
||||||
@mock.patch.object(build, 'run_build')
|
@mock.patch.object(build, 'run_build')
|
||||||
def test_bad_images(self, mock_run_build):
|
def test_bad_images(self, mock_run_build):
|
||||||
image_statuses = ({'img': 'error'}, {}, {}, {})
|
image_statuses = ({'img': 'error'}, {}, {}, {}, {})
|
||||||
mock_run_build.return_value = image_statuses
|
mock_run_build.return_value = image_statuses
|
||||||
result = build_cmd.main()
|
result = build_cmd.main()
|
||||||
self.assertEqual(1, result)
|
self.assertEqual(1, result)
|
||||||
@ -574,7 +574,14 @@ class MainTest(base.TestCase):
|
|||||||
|
|
||||||
@mock.patch.object(build, 'run_build')
|
@mock.patch.object(build, 'run_build')
|
||||||
def test_skipped_images(self, mock_run_build):
|
def test_skipped_images(self, mock_run_build):
|
||||||
image_statuses = ({}, {}, {}, {'img': 'skipped'})
|
image_statuses = ({}, {}, {}, {'img': 'skipped'}, {})
|
||||||
|
mock_run_build.return_value = image_statuses
|
||||||
|
result = build_cmd.main()
|
||||||
|
self.assertEqual(0, result)
|
||||||
|
|
||||||
|
@mock.patch.object(build, 'run_build')
|
||||||
|
def test_unbuildable_images(self, mock_run_build):
|
||||||
|
image_statuses = ({}, {}, {}, {}, {'img': 'unbuildable'})
|
||||||
mock_run_build.return_value = image_statuses
|
mock_run_build.return_value = image_statuses
|
||||||
result = build_cmd.main()
|
result = build_cmd.main()
|
||||||
self.assertEqual(0, result)
|
self.assertEqual(0, result)
|
||||||
|
@ -48,11 +48,11 @@ class BuildTest(object):
|
|||||||
with patch.object(sys, 'argv', self.build_args):
|
with patch.object(sys, 'argv', self.build_args):
|
||||||
LOG.info("Running with args %s", self.build_args)
|
LOG.info("Running with args %s", self.build_args)
|
||||||
(bad_results, good_results, unmatched_results,
|
(bad_results, good_results, unmatched_results,
|
||||||
skipped_results) = build.run_build()
|
skipped_results, unbuildable_results) = build.run_build()
|
||||||
|
|
||||||
failures = 0
|
failures = 0
|
||||||
for image, result in bad_results.items():
|
for image, result in bad_results.items():
|
||||||
if result is not 'error':
|
if result != 'error':
|
||||||
continue
|
continue
|
||||||
failures = failures + 1
|
failures = failures + 1
|
||||||
LOG.critical(">>> Expected image '%s' to succeed!", image)
|
LOG.critical(">>> Expected image '%s' to succeed!", image)
|
||||||
|
Loading…
Reference in New Issue
Block a user