Merge "Add configuration mold storage"

ironic/conf/default.py

@@ -423,6 +423,26 @@ webserver_opts = [
                       'with images.')),
 ]
 
+configuration_mold_opts = [
+    cfg.StrOpt('mold_storage',
+               default='swift',
+               help=_('Configuration mold storage location. Supports "swift" '
+                      'and "http". By default "swift".')),
+    cfg.StrOpt('mold_user',
+               help=_('User for "http" Basic auth. By default set empty.')),
+    cfg.StrOpt('mold_password',
+               help=_('Password for "http" Basic auth. By default set '
+                      'empty.')),
+    cfg.StrOpt('mold_retry_attempts',
+               default=3,
+               help=_('Retry attempts for saving or getting configuration '
+                      'molds.')),
+    cfg.StrOpt('mold_retry_interval',
+               default=3,
+               help=_('Retry interval for saving or getting configuration '
+                      'molds.'))
+]
+
 
 def register_opts(conf):
     conf.register_opts(api_opts)
@@ -438,3 +458,4 @@ def register_opts(conf):
     conf.register_opts(service_opts)
     conf.register_opts(utils_opts)
     conf.register_opts(webserver_opts)
+    conf.register_opts(configuration_mold_opts)

ironic/drivers/utils.py

@@ -12,6 +12,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import json
 import os
 import tempfile
 
@@ -20,6 +21,8 @@ from oslo_log import log as logging
 from oslo_serialization import base64
 from oslo_utils import strutils
 from oslo_utils import timeutils
+import requests
+import tenacity
 
 from ironic.common import exception
 from ironic.common.i18n import _
@@ -384,3 +387,85 @@ OPTIONAL_PROPERTIES = {
                                       "deprecated in favor of the new ones."
                                       "Defaults to 'Default'. Optional."),
 }
+
+
+def save_configuration_mold(task, url, data):
+    """Store configuration mold to indicated location.
+
+    :param task: A TaskManager instance.
+    :param name: URL of the configuration item to save to.
+    :param data: Content of JSON data to save.
+
+    :raises IronicException: If using Swift storage and no authentication
+        token found in task's context.
+    :raises HTTPError: If failed to complete HTTP request.
+    """
+    @tenacity.retry(
+        retry=tenacity.retry_if_exception_type(
+            requests.exceptions.ConnectionError),
+        stop=tenacity.stop_after_attempt(CONF.mold_retry_attempts),
+        wait=tenacity.wait_fixed(CONF.mold_retry_interval),
+        reraise=True
+    )
+    def _request(url, data, auth_header):
+        return requests.put(
+            url, data=json.dumps(data, indent=2), headers=auth_header)
+
+    auth_header = _get_auth_header(task)
+    response = _request(url, data, auth_header)
+    response.raise_for_status()
+
+
+def get_configuration_mold(task, url):
+    """Gets configuration mold from indicated location.
+
+    :param task: A TaskManager instance.
+    :param url: URL of the configuration item to get.
+
+    :returns: JSON configuration mold
+
+    :raises IronicException: If using Swift storage and no authentication
+        token found in task's context.
+    :raises HTTPError: If failed to complete HTTP request.
+    """
+    @tenacity.retry(
+        retry=tenacity.retry_if_exception_type(
+            requests.exceptions.ConnectionError),
+        stop=tenacity.stop_after_attempt(CONF.mold_retry_attempts),
+        wait=tenacity.wait_fixed(CONF.mold_retry_interval),
+        reraise=True
+    )
+    def _request(url, auth_header):
+        return requests.get(url, headers=auth_header)
+
+    auth_header = _get_auth_header(task)
+    response = _request(url, auth_header)
+    if response.status_code == requests.codes.ok:
+        return response.json()
+
+    response.raise_for_status()
+
+
+def _get_auth_header(task):
+    """Based on setup of configuration mold storage gets authentication header
+
+    :param task: A TaskManager instance.
+    :raises IronicException: If using Swift storage and no authentication
+        token found in task's context.
+    """
+    auth_header = None
+    if CONF.mold_storage == 'swift':
+        # TODO(ajya) Need to update to use Swift client and context session
+        auth_token = swift.get_swift_session().get_token()
+        if auth_token:
+            auth_header = {'X-Auth-Token': auth_token}
+        else:
+            raise exception.IronicException(
+                _('Missing auth_token for configuration mold access for node '
+                  '%s') % task.node.uuid)
+    elif CONF.mold_storage == 'http':
+        if CONF.mold_user and CONF.mold_password:
+            auth_header = {'Authorization': 'Basic %s'
+                           % base64.encode_as_text(
+                               '%s:%s' % (CONF.mold_user, CONF.mold_password))}
+    return auth_header

ironic/tests/unit/drivers/test_utils.py

@@ -19,6 +19,7 @@ from unittest import mock
 
 from oslo_config import cfg
 from oslo_utils import timeutils
+import requests
 
 from ironic.common import exception
 from ironic.common import swift
@@ -408,3 +409,270 @@ class MixinVendorInterfaceTestCase(db_base.DbTestCase):
             self.assertRaises(exception.InvalidParameterValue,
                               self.vendor.validate,
                               task, method='fake_method')
+
+
+class ConfigurationMoldTestCase(db_base.DbTestCase):
+
+    def setUp(self):
+        super(ConfigurationMoldTestCase, self).setUp()
+        self.node = obj_utils.create_test_node(self.context)
+
+    @mock.patch.object(swift, 'get_swift_session', autospec=True)
+    @mock.patch.object(requests, 'put', autospec=True)
+    def test_save_configuration_mold_swift(self, mock_put, mock_swift):
+        mock_session = mock.Mock()
+        mock_session.get_token.return_value = 'token'
+        mock_swift.return_value = mock_session
+        driver_utils.CONF.mold_storage = 'swift'
+        url = 'https://example.com/file1'
+        data = {'key': 'value'}
+
+        with task_manager.acquire(self.context, self.node.uuid) as task:
+            driver_utils.save_configuration_mold(task, url, data)
+
+        mock_put.assert_called_once_with(url, '{\n  "key": "value"\n}',
+                                         headers={'X-Auth-Token': 'token'})
+
+    @mock.patch.object(swift, 'get_swift_session', autospec=True)
+    @mock.patch.object(requests, 'put', autospec=True)
+    def test_save_configuration_mold_swift_noauth(self, mock_put, mock_swift):
+        mock_session = mock.Mock()
+        mock_session.get_token.return_value = None
+        mock_swift.return_value = mock_session
+        driver_utils.CONF.mold_storage = 'swift'
+        url = 'https://example.com/file1'
+        data = {'key': 'value'}
+
+        with task_manager.acquire(self.context, self.node.uuid) as task:
+            self.assertRaises(
+                exception.IronicException,
+                driver_utils.save_configuration_mold,
+                task, url, data)
+
+    @mock.patch.object(requests, 'put', autospec=True)
+    def test_save_configuration_mold_http(self, mock_put):
+        driver_utils.CONF.mold_storage = 'http'
+        driver_utils.CONF.mold_user = 'user'
+        driver_utils.CONF.mold_password = 'password'
+        url = 'https://example.com/file1'
+        data = {'key': 'value'}
+
+        with task_manager.acquire(self.context, self.node.uuid) as task:
+            driver_utils.save_configuration_mold(task, url, data)
+
+            mock_put.assert_called_once_with(
+                url, '{\n  "key": "value"\n}',
+                headers={'Authorization': 'Basic dXNlcjpwYXNzd29yZA=='})
+
+    @mock.patch.object(requests, 'put', autospec=True)
+    def test_save_configuration_mold_http_noauth(self, mock_put):
+        driver_utils.CONF.mold_storage = 'http'
+        driver_utils.CONF.mold_user = None
+        driver_utils.CONF.mold_password = None
+        url = 'https://example.com/file1'
+        data = {'key': 'value'}
+
+        with task_manager.acquire(self.context, self.node.uuid) as task:
+            driver_utils.save_configuration_mold(task, url, data)
+
+            mock_put.assert_called_once_with(
+                url, '{\n  "key": "value"\n}',
+                headers=None)
+
+    @mock.patch.object(requests, 'put', autospec=True)
+    def test_save_configuration_mold_http_error(self, mock_put):
+        driver_utils.CONF.mold_storage = 'http'
+        driver_utils.CONF.mold_user = 'user'
+        driver_utils.CONF.mold_password = 'password'
+        response = mock.MagicMock()
+        response.status_code = 404
+        response.raise_for_status.side_effect = requests.exceptions.HTTPError
+        mock_put.return_value = response
+
+        with task_manager.acquire(self.context, self.node.uuid) as task:
+            self.assertRaises(
+                requests.exceptions.HTTPError,
+                driver_utils.save_configuration_mold,
+                task,
+                'https://example.com/file2',
+                {'key': 'value'})
+            mock_put.assert_called_once_with(
+                'https://example.com/file2', '{\n  "key": "value"\n}',
+                headers={'Authorization': 'Basic dXNlcjpwYXNzd29yZA=='})
+
+    @mock.patch.object(requests, 'put', autospec=True)
+    def test_save_configuration_mold_connection_error(self, mock_put):
+        driver_utils.CONF.mold_storage = 'http'
+        driver_utils.CONF.mold_user = 'user'
+        driver_utils.CONF.mold_password = 'password'
+        driver_utils.CONF.mold_retry_interval = 0
+        driver_utils.CONF.mold_retry_attempts = 3
+        response = mock.MagicMock()
+        mock_put.side_effect = [
+            requests.exceptions.ConnectTimeout,
+            requests.exceptions.ConnectionError,
+            response]
+
+        with task_manager.acquire(self.context, self.node.uuid) as task:
+            driver_utils.save_configuration_mold(
+                task, 'https://example.com/file2', {'key': 'value'})
+            mock_put.assert_called_with(
+                'https://example.com/file2', '{\n  "key": "value"\n}',
+                headers={'Authorization': 'Basic dXNlcjpwYXNzd29yZA=='})
+            self.assertEqual(mock_put.call_count, 3)
+
+    @mock.patch.object(requests, 'put', autospec=True)
+    def test_save_configuration_mold_connection_error_exceeded(self, mock_put):
+        driver_utils.CONF.mold_storage = 'http'
+        driver_utils.CONF.mold_user = 'user'
+        driver_utils.CONF.mold_password = 'password'
+        driver_utils.CONF.mold_retry_interval = 0
+        driver_utils.CONF.mold_retry_attempts = 2
+        mock_put.side_effect = [
+            requests.exceptions.ConnectTimeout,
+            requests.exceptions.ConnectionError]
+
+        with task_manager.acquire(self.context, self.node.uuid) as task:
+            self.assertRaises(
+                requests.exceptions.ConnectionError,
+                driver_utils.save_configuration_mold,
+                task,
+                'https://example.com/file2',
+                {'key': 'value'})
+            mock_put.assert_called_with(
+                'https://example.com/file2', '{\n  "key": "value"\n}',
+                headers={'Authorization': 'Basic dXNlcjpwYXNzd29yZA=='})
+            self.assertEqual(mock_put.call_count, 2)
+
+    @mock.patch.object(swift, 'get_swift_session', autospec=True)
+    @mock.patch.object(requests, 'get', autospec=True)
+    def test_get_configuration_mold_swift(self, mock_get, mock_swift):
+        mock_session = mock.Mock()
+        mock_session.get_token.return_value = 'token'
+        mock_swift.return_value = mock_session
+        driver_utils.CONF.mold_storage = 'swift'
+        response = mock.MagicMock()
+        response.status_code = 200
+        response.json.return_value = {'key': 'value'}
+        mock_get.return_value = response
+        url = 'https://example.com/file1'
+
+        with task_manager.acquire(self.context, self.node.uuid) as task:
+            result = driver_utils.get_configuration_mold(task, url)
+
+            mock_get.assert_called_once_with(
+                url, headers={'X-Auth-Token': 'token'})
+            self.assertJsonEqual({'key': 'value'}, result)
+
+    @mock.patch.object(swift, 'get_swift_session', autospec=True)
+    @mock.patch.object(requests, 'get', autospec=True)
+    def test_get_configuration_mold_swift_noauth(self, mock_get, mock_swift):
+        mock_session = mock.Mock()
+        mock_session.get_token.return_value = None
+        mock_swift.return_value = mock_session
+        driver_utils.CONF.mold_storage = 'swift'
+        url = 'https://example.com/file1'
+
+        with task_manager.acquire(self.context, self.node.uuid) as task:
+            self.assertRaises(
+                exception.IronicException,
+                driver_utils.get_configuration_mold,
+                task, url)
+
+    @mock.patch.object(requests, 'get', autospec=True)
+    def test_get_configuration_mold_http(self, mock_get):
+        driver_utils.CONF.mold_storage = 'http'
+        driver_utils.CONF.mold_user = 'user'
+        driver_utils.CONF.mold_password = 'password'
+        response = mock.MagicMock()
+        response.status_code = 200
+        response.json.return_value = {'key': 'value'}
+        mock_get.return_value = response
+        url = 'https://example.com/file2'
+
+        with task_manager.acquire(self.context, self.node.uuid) as task:
+            result = driver_utils.get_configuration_mold(task, url)
+
+            mock_get.assert_called_once_with(
+                url, headers={'Authorization': 'Basic dXNlcjpwYXNzd29yZA=='})
+            self.assertJsonEqual({"key": "value"}, result)
+
+    @mock.patch.object(requests, 'get', autospec=True)
+    def test_get_configuration_mold_http_noauth(self, mock_get):
+        driver_utils.CONF.mold_storage = 'http'
+        driver_utils.CONF.mold_user = None
+        driver_utils.CONF.mold_password = None
+        response = mock.MagicMock()
+        response.status_code = 200
+        response.json.return_value = {'key': 'value'}
+        mock_get.return_value = response
+        url = 'https://example.com/file2'
+
+        with task_manager.acquire(self.context, self.node.uuid) as task:
+            result = driver_utils.get_configuration_mold(task, url)
+
+            mock_get.assert_called_once_with(url, headers=None)
+            self.assertJsonEqual({"key": "value"}, result)
+
+    @mock.patch.object(requests, 'get', autospec=True)
+    def test_get_configuration_mold_http_error(self, mock_get):
+        driver_utils.CONF.mold_storage = 'http'
+        driver_utils.CONF.mold_user = 'user'
+        driver_utils.CONF.mold_password = 'password'
+        response = mock.MagicMock()
+        response.status_code = 404
+        response.raise_for_status.side_effect = requests.exceptions.HTTPError
+        mock_get.return_value = response
+
+        with task_manager.acquire(self.context, self.node.uuid) as task:
+            self.assertRaises(
+                requests.exceptions.HTTPError,
+                driver_utils.get_configuration_mold,
+                task,
+                'https://example.com/file2')
+            mock_get.assert_called_once_with(
+                'https://example.com/file2',
+                headers={'Authorization': 'Basic dXNlcjpwYXNzd29yZA=='})
+
+    @mock.patch.object(requests, 'get', autospec=True)
+    def test_get_configuration_mold_connection_error(self, mock_get):
+        driver_utils.CONF.mold_storage = 'http'
+        driver_utils.CONF.mold_user = 'user'
+        driver_utils.CONF.mold_password = 'password'
+        driver_utils.CONF.mold_retry_interval = 0
+        driver_utils.CONF.mold_retry_attempts = 3
+        response = mock.MagicMock()
+        mock_get.side_effect = [
+            requests.exceptions.ConnectTimeout,
+            requests.exceptions.ConnectionError,
+            response]
+
+        with task_manager.acquire(self.context, self.node.uuid) as task:
+            driver_utils.get_configuration_mold(
+                task, 'https://example.com/file2')
+            mock_get.assert_called_with(
+                'https://example.com/file2',
+                headers={'Authorization': 'Basic dXNlcjpwYXNzd29yZA=='})
+            self.assertEqual(mock_get.call_count, 3)
+
+    @mock.patch.object(requests, 'get', autospec=True)
+    def test_get_configuration_mold_connection_error_exceeded(self, mock_get):
+        driver_utils.CONF.mold_storage = 'http'
+        driver_utils.CONF.mold_user = 'user'
+        driver_utils.CONF.mold_password = 'password'
+        driver_utils.CONF.mold_retry_interval = 0
+        driver_utils.CONF.mold_retry_attempts = 2
+        mock_get.side_effect = [
+            requests.exceptions.ConnectTimeout,
+            requests.exceptions.ConnectionError]
+
+        with task_manager.acquire(self.context, self.node.uuid) as task:
+            self.assertRaises(
+                requests.exceptions.ConnectionError,
+                driver_utils.get_configuration_mold,
+                task,
+                'https://example.com/file2')
+            mock_get.assert_called_with(
+                'https://example.com/file2',
+                headers={'Authorization': 'Basic dXNlcjpwYXNzd29yZA=='})
+            self.assertEqual(mock_get.call_count, 2)