Merge "Don't assume default tag exists in container repo" into stable/train

container-images/container_image_prepare_defaults.yaml

@@ -17,6 +17,7 @@ parameter_defaults:
       name_prefix: centos-binary-
       name_suffix: ''
       tag: current-tripleo
+      default_tag: True
       rhel_containers: false
 
       # Substitute neutron images based on driver. Can be 'other', 'ovn' or

releasenotes/notes/check_for_default_tag-09fe34d2ac434890.yaml

@@ -0,0 +1,6 @@
+---
+fixes:
+  - When the default tag doesn't exist in the container repo during container
+    image prepare, and a tag wasn't set in the actual input for
+    ContainerImagePrepare, the latest tag from the repo will be used instead of
+    failing with a not found error.

tripleo_common/image/image_uploader.py

@@ -695,19 +695,24 @@ class BaseImageUploader(object):
         wait=tenacity.wait_random_exponential(multiplier=1, max=10),
         stop=tenacity.stop_after_attempt(5)
     )
-    def _inspect(cls, image_url, session=None):
+    def _inspect(cls, image_url, session=None, default_tag=False):
         image, tag = cls._image_tag_from_url(image_url)
         parts = {
             'image': image,
             'tag': tag
         }
 
-        manifest_url = cls._build_url(
-            image_url, CALL_MANIFEST % parts
-        )
         tags_url = cls._build_url(
             image_url, CALL_TAGS % parts
         )
+        tags_r = RegistrySessionHelper.get(session, tags_url, timeout=30)
+        tags = tags_r.json()['tags']
+        if default_tag and tag not in tags:
+            parts['tag'] = tags[-1]
+
+        manifest_url = cls._build_url(
+            image_url, CALL_MANIFEST % parts
+        )
         manifest_headers = {'Accept': MEDIA_MANIFEST_V2}
 
         try:
@@ -724,8 +729,6 @@ class BaseImageUploader(object):
             else:
                 raise
 
-        tags_r = RegistrySessionHelper.get(session, tags_url, timeout=30)
-
         manifest_str = cls._get_response_text(manifest_r)
 
         if 'Docker-Content-Digest' in manifest_r.headers:
@@ -759,8 +762,6 @@ class BaseImageUploader(object):
             )
             config = config_r.json()
 
-        tags = tags_r.json()['tags']
-
         image, tag = cls._image_tag_from_url(image_url)
         name = '%s%s' % (image_url.netloc, image)
         created = config['created']
@@ -910,7 +911,8 @@ class BaseImageUploader(object):
             )
         return tag_label
 
-    def discover_image_tags(self, images, tag_from_label=None):
+    def discover_image_tags(self, images, tag_from_label=None,
+                            default_tag=False):
         image_urls = [self._image_to_url(i) for i in images]
 
         # prime self.insecure_registries by testing every image
@@ -919,7 +921,8 @@ class BaseImageUploader(object):
 
         discover_args = []
         for image in images:
-            discover_args.append((self, image, tag_from_label))
+            discover_args.append((self, image, tag_from_label,
+                                  default_tag))
 
         versioned_images = {}
         with futures.ThreadPoolExecutor(max_workers=16) as p:
@@ -2326,10 +2329,10 @@ class PythonImageUploader(BaseImageUploader):
         return image, manifest, config_str
 
     @classmethod
-    def _inspect(cls, image_url, session=None):
+    def _inspect(cls, image_url, session=None, default_tag=False):
         if image_url.scheme == 'docker':
             return super(PythonImageUploader, cls)._inspect(
-                image_url, session=session)
+                image_url, session=session, default_tag=default_tag)
         if image_url.scheme != 'containers-storage':
             raise ImageUploaderException('Inspect not implemented for %s' %
                                          image_url.geturl())
@@ -2490,7 +2493,7 @@ def upload_task(args):
 
 
 def discover_tag_from_inspect(args):
-    self, image, tag_from_label = args
+    self, image, tag_from_label, default_tag = args
     image_url = self._image_to_url(image)
     username, password = self.credentials_for_registry(image_url.netloc)
     try:
@@ -2503,7 +2506,7 @@ def discover_tag_from_inspect(args):
                 'missing registry credentials or the provided '
                 'container or namespace does not exist. %s' % e)
         raise
-    i = self._inspect(image_url, session=session)
+    i = self._inspect(image_url, session=session, default_tag=default_tag)
     session.close()
     if ':' in image_url.path:
         # break out the tag from the url to be the fallback tag

tripleo_common/image/kolla_builder.py

@@ -330,10 +330,14 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
     )
     uploader = manager.uploader('python')
     images = [i.get('imagename', '') for i in result]
+    if result:
+        default_tag = result[0].get('default_tag', False)
+    else:
+        default_tag = False
 
     if tag_from_label:
         image_version_tags = uploader.discover_image_tags(
-            images, tag_from_label)
+            images, tag_from_label, default_tag)
         for entry in result:
             imagename = entry.get('imagename', '')
             image_no_tag = imagename.rpartition(':')[0]
@@ -438,6 +442,12 @@ class KollaImageBuilder(base.BaseImageManager):
         hyphenated appropriately.
         '''
         mapping = dict(kwargs)
+        # set a flag to record whether the default tag is used or not. the
+        # logic here is that if the tag key is not already in mapping then it
+        # wil be added during the template render, so default_tag is set to
+        # True.
+        mapping['default_tag'] = 'tag' not in mapping
+
         if CONTAINER_IMAGES_DEFAULTS is None:
             return
         for k, v in CONTAINER_IMAGES_DEFAULTS.items():

tripleo_common/tests/image/test_image_uploader.py

@@ -527,12 +527,12 @@ class TestBaseImageUploader(base.TestCase):
         self.assertRaises(
             ImageUploaderException,
             image_uploader.discover_tag_from_inspect,
-            (self.uploader, 'docker.io/t/foo', 'rdo_version')
+            (self.uploader, 'docker.io/t/foo', 'rdo_version', False)
         )
         self.assertRaises(
             requests.exceptions.HTTPError,
             image_uploader.discover_tag_from_inspect,
-            (self.uploader, 'docker.io/t/foo', 'rdo_version')
+            (self.uploader, 'docker.io/t/foo', 'rdo_version', False)
         )
 
     @mock.patch('tripleo_common.image.image_uploader.'
@@ -554,63 +554,65 @@ class TestBaseImageUploader(base.TestCase):
         self.assertEqual(
             ('docker.io/t/foo', 'a'),
             image_uploader.discover_tag_from_inspect(
-                (self.uploader, 'docker.io/t/foo', 'rdo_version'))
+                (self.uploader, 'docker.io/t/foo', 'rdo_version', False))
         )
 
         # templated labels -> tag
         self.assertEqual(
             ('docker.io/t/foo', '1.0.0-20180125'),
             image_uploader.discover_tag_from_inspect(
-                (self.uploader, 'docker.io/t/foo', '{release}-{version}'))
+                (self.uploader, 'docker.io/t/foo', '{release}-{version}',
+                 False))
         )
 
         # simple label -> tag with fallback
         self.assertEqual(
             ('docker.io/t/foo', 'a'),
             image_uploader.discover_tag_from_inspect(
-                (self.uploader, 'docker.io/t/foo:a', 'bar'))
+                (self.uploader, 'docker.io/t/foo:a', 'bar', False))
         )
 
         # templated labels -> tag with fallback
         self.assertEqual(
             ('docker.io/t/foo', 'a'),
             image_uploader.discover_tag_from_inspect(
-                (self.uploader, 'docker.io/t/foo:a', '{releases}-{versions}'))
+                (self.uploader, 'docker.io/t/foo:a', '{releases}-{versions}',
+                 False))
         )
 
         # Invalid template
         self.assertRaises(
             ImageUploaderException,
             image_uploader.discover_tag_from_inspect,
-            (self.uploader, 'docker.io/t/foo', '{release}-{version')
+            (self.uploader, 'docker.io/t/foo', '{release}-{version', False)
         )
 
         # Missing label in template
         self.assertRaises(
             ImageUploaderException,
             image_uploader.discover_tag_from_inspect,
-            (self.uploader, 'docker.io/t/foo', '{releases}-{version}')
+            (self.uploader, 'docker.io/t/foo', '{releases}-{version}', False)
         )
 
         # no tag_from_label specified
         self.assertRaises(
             ImageUploaderException,
             image_uploader.discover_tag_from_inspect,
-            (self.uploader, 'docker.io/t/foo', None)
+            (self.uploader, 'docker.io/t/foo', None, False)
         )
 
         # missing RepoTags entry
         self.assertRaises(
             ImageUploaderException,
             image_uploader.discover_tag_from_inspect,
-            (self.uploader, 'docker.io/t/foo', 'build_version')
+            (self.uploader, 'docker.io/t/foo', 'build_version', False)
         )
 
         # missing Labels entry
         self.assertRaises(
             ImageUploaderException,
             image_uploader.discover_tag_from_inspect,
-            (self.uploader, 'docker.io/t/foo', 'version')
+            (self.uploader, 'docker.io/t/foo', 'version', False)
         )
 
         # inspect call failed
@@ -618,7 +620,7 @@ class TestBaseImageUploader(base.TestCase):
         self.assertRaises(
             ImageUploaderException,
             image_uploader.discover_tag_from_inspect,
-            (self.uploader, 'docker.io/t/foo', 'rdo_version')
+            (self.uploader, 'docker.io/t/foo', 'rdo_version', False)
         )
 
         # handle auth issues
@@ -632,12 +634,12 @@ class TestBaseImageUploader(base.TestCase):
         self.assertRaises(
             ImageUploaderException,
             image_uploader.discover_tag_from_inspect,
-            (self.uploader, 'docker.io/t/foo', 'rdo_version')
+            (self.uploader, 'docker.io/t/foo', 'rdo_version', False)
         )
         self.assertRaises(
             requests.exceptions.HTTPError,
             image_uploader.discover_tag_from_inspect,
-            (self.uploader, 'docker.io/t/foo', 'rdo_version')
+            (self.uploader, 'docker.io/t/foo', 'rdo_version', False)
         )
 
     @mock.patch('concurrent.futures.ThreadPoolExecutor')
@@ -665,9 +667,9 @@ class TestBaseImageUploader(base.TestCase):
         mock_map.assert_called_once_with(
             image_uploader.discover_tag_from_inspect,
             [
-                (self.uploader, 'docker.io/t/foo', 'rdo_release'),
+                (self.uploader, 'docker.io/t/foo', 'rdo_release', False),
-                (self.uploader, 'docker.io/t/bar', 'rdo_release'),
+                (self.uploader, 'docker.io/t/bar', 'rdo_release', False),
-                (self.uploader, 'docker.io/t/baz', 'rdo_release')
+                (self.uploader, 'docker.io/t/baz', 'rdo_release', False)
             ])
 
     @mock.patch('tripleo_common.image.image_uploader.'

tripleo_common/tests/image/test_kolla_builder.py

@@ -254,6 +254,7 @@ class TestKollaImageBuilderTemplate(base.TestCase):
                 'tag': 'current-tripleo',
                 'rhel_containers': False,
                 'neutron_driver': 'ovn',
+                'default_tag': True,
             },
             builder.container_images_template_inputs()
         )
@@ -281,6 +282,7 @@ class TestKollaImageBuilderTemplate(base.TestCase):
                 'tag': 'master',
                 'rhel_containers': False,
                 'neutron_driver': 'ovn',
+                'default_tag': False,
             },
             builder.container_images_template_inputs(
                 namespace='192.0.2.0:5000/tripleotrain',