Support caching http:// images locally in the direct deploy

For low RAM nodes we need a way to convert even http:// images to raw
and serve them locally. Add a new image_download_source value "local"
that works the same as "http" but also affects http:// images.

Change-Id: I8da968ccfad6e7a508e9b763b9e3f96007438b6a
Story: #2008075
Task: #40765

doc/source/admin/interfaces/deploy.rst

@@ -63,6 +63,14 @@ To use this deploy interface with a custom HTTP server, set
    image_download_source = http
    ...
 
+This configuration affects *glance* and ``file://`` images. If you want
+``http(s)://`` images to also be cached and served locally, use instead:
+
+.. code-block:: ini
+
+   [agent]
+   image_download_source = local
+
 You need to set up a workable HTTP server at each conductor node which with
 ``direct`` deploy interface enabled, and check http related options in the
 ironic configuration file to match the HTTP server configurations.

ironic/conf/agent.py

@@ -101,15 +101,16 @@ opts = [
                                     'from the Object Storage service.')),
                         ('http', _('IPA ramdisk retrieves instance image '
                                    'from HTTP service served at conductor '
-                                   'nodes.'))],
+                                   'nodes.')),
+                        ('local', _('Same as "http", but HTTP images '
+                                    'are also cached locally, converted '
+                                    'and served from the conductor'))],
                default='swift',
                mutable=True,
                help=_('Specifies whether direct deploy interface should try '
                       'to use the image source directly or if ironic should '
                       'cache the image on the conductor and serve it from '
-                      'ironic\'s own http server. This option takes effect '
-                      'only when instance image is provided from the Image '
-                      'service.')),
+                      'ironic\'s own http server.')),
     cfg.IntOpt('command_timeout',
                default=60,
                mutable=True,

ironic/drivers/modules/agent.py

@@ -172,6 +172,7 @@ def validate_http_provisioning_configuration(node):
     # 1. Glance images with image_download_source == http
     # 2. File images (since we need to serve them to IPA)
     if (not image_source.startswith('file://')
+            and CONF.agent.image_download_source != 'local'
             and (not service_utils.is_glance_image(image_source)
                  or CONF.agent.image_download_source == 'swift')):
         return

ironic/drivers/modules/deploy_utils.py

@@ -1085,7 +1085,8 @@ def build_instance_info_for_deploy(task):
         if not iwdi:
             instance_info['kernel'] = image_info['properties']['kernel_id']
             instance_info['ramdisk'] = image_info['properties']['ramdisk_id']
-    elif image_source.startswith('file://'):
+    elif (image_source.startswith('file://')
+          or CONF.agent.image_download_source == 'local'):
         _cache_and_convert_image(task, instance_info)
     else:
         _validate_image_url(node, image_source)

ironic/tests/unit/drivers/modules/test_agent.py

@@ -202,6 +202,17 @@ class TestAgentMethods(db_base.DbTestCase):
                                agent.validate_http_provisioning_configuration,
                                self.node)
 
+    def test_validate_http_provisioning_missing_args_local_http(self):
+        CONF.set_override('image_download_source', 'local', group='agent')
+        CONF.set_override('http_url', None, group='deploy')
+        i_info = self.node.instance_info
+        i_info['image_source'] = 'http://image-ref'
+        self.node.instance_info = i_info
+        self.assertRaisesRegex(exception.MissingParameterValue,
+                               'failed to validate http provisoning',
+                               agent.validate_http_provisioning_configuration,
+                               self.node)
+
 
 class TestAgentDeploy(db_base.DbTestCase):
     def setUp(self):

ironic/tests/unit/drivers/modules/test_deploy_utils.py

@@ -2073,6 +2073,39 @@ class TestBuildInstanceInfoForHttpProvisioning(db_base.DbTestCase):
             validate_href_mock.assert_called_once_with(
                 mock.ANY, expected_url, False)
 
+    @mock.patch.object(image_service.HttpImageService, 'validate_href',
+                       autospec=True)
+    def test_build_instance_info_local_image(self, validate_href_mock):
+        cfg.CONF.set_override('image_download_source', 'local', group='agent')
+        i_info = self.node.instance_info
+        driver_internal_info = self.node.driver_internal_info
+        i_info['image_source'] = 'http://image-ref'
+        i_info['image_checksum'] = 'aa'
+        i_info['root_gb'] = 10
+        i_info['image_checksum'] = 'aa'
+        driver_internal_info['is_whole_disk_image'] = True
+        self.node.instance_info = i_info
+        self.node.driver_internal_info = driver_internal_info
+        self.node.save()
+
+        expected_url = (
+            'http://172.172.24.10:8080/agent_images/%s' % self.node.uuid)
+
+        with task_manager.acquire(
+                self.context, self.node.uuid, shared=False) as task:
+
+            info = utils.build_instance_info_for_deploy(task)
+
+            self.assertEqual(expected_url, info['image_url'])
+            self.assertEqual('sha256', info['image_os_hash_algo'])
+            self.assertEqual('fake-checksum', info['image_os_hash_value'])
+            self.cache_image_mock.assert_called_once_with(
+                task.context, task.node, force_raw=True)
+            self.checksum_mock.assert_called_once_with(
+                self.fake_path, algorithm='sha256')
+            validate_href_mock.assert_called_once_with(
+                mock.ANY, expected_url, False)
+
 
 class TestStorageInterfaceUtils(db_base.DbTestCase):
     def setUp(self):

releasenotes/notes/http-local-4e8f32c6d5309f12.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Adds a new possible value for ``image_download_source``: ``local``.
+    When used, even ``http://`` images are downloaded, converted to RAW if
+    needed and served from the conductor's HTTP server. This feature targets
+    primarily nodes with low RAM.