Add FuelClient

Client for access to Fuel API.

blueprint fuel-engine

Change-Id: If1572357b58e66cff84a5d2256f7b4d3bff2f0e4

rally/fuelclient.py

@@ -0,0 +1,263 @@
+# Copyright 2013: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import json
+import re
+import requests
+import time
+
+from rally.openstack.common import log as logging
+
+LOG = logging.getLogger(__name__)
+
+FILTER_REG = re.compile(r'^([a-z]+)\s*([<>=!]=|<|>)\s*(.+)$')
+INT_REG = re.compile(r'^(\d+)(K|M|G|T)?$')
+
+
+class FuelException(Exception):
+    pass
+
+
+class FuelClientException(FuelException):
+
+    def __init__(self, code, body):
+        self.code = code
+        self.body = body
+
+    def __str__(self):
+        return ('FuelClientException. '
+                'Code: %(code)d Body: %(body)s' % {'code': self.code,
+                                                   'body': self.body})
+
+
+class FuelNetworkVerificationFailed(FuelException):
+    pass
+
+
+class FuelNode(object):
+
+    def __init__(self, node):
+        self.node = node
+        self.ATTRIBUTE_MAP = {
+            '==': lambda x, y: x == y,
+            '!=': lambda x, y: x != y,
+            '<=': lambda x, y: x <= y,
+            '>=': lambda x, y: x >= y,
+            '<': lambda x, y: x < y,
+            '>': lambda x, y: x > y,
+        }
+        self.FACTOR_MAP = {
+            'K': 1024,
+            'M': 1048576,
+            'G': 1073741824,
+            'T': 1099511627776,
+            None: 1,
+        }
+
+    def __getitem__(self, key):
+        return self.node[key]
+
+    def check_filters(self, filters):
+        return all((self.check(f) for f in filters))
+
+    def check(self, filter_string):
+        if self.node['cluster'] is not None:
+            return False
+        m = FILTER_REG.match(filter_string)
+        if m is None:
+            raise ValueError('Invalid filter: %s' % filter_string)
+        attribute, operator, value = m.groups()
+        return self._check(attribute, value, operator)
+
+    def _check(self, attribute, value, operator):
+        attribute = getattr(self, '_get_' + attribute)()
+        checker = self.ATTRIBUTE_MAP[operator]
+        m = INT_REG.match(value)
+        if m:
+            value = int(m.group(1)) * self.FACTOR_MAP[m.group(2)]
+        return checker(attribute, value)
+
+    def _get_ram(self):
+        return self.node['meta']['memory']['total']
+
+    def _get_mac(self):
+        return self.node['mac']
+
+    def _get_storage(self):
+        return sum((d['size'] for d in self.node['meta']['disks']))
+
+    def _get_cpus(self):
+        return self.node['meta']['cpu']['total']
+
+
+class FuelCluster(object):
+
+    def __init__(self, client, **config):
+        """Create Fuel cluster.
+
+        :param client:              FuelClient instance.
+        :param name:                Name
+        :param release:             Release id. Integer.
+        :param mode:                One of multinode, ha_compact
+        :param net_provider:        One of nova_network, neutron
+        :param net_segment_type:    One of gre, vlan.
+        :param dns_nameservers:     List of strings.
+        """
+
+        self.client = client
+        self.cluster = client.post('clusters', config)
+
+    def get_nodes(self):
+        return self.client.get('nodes?cluster_id=%d' % self.cluster['id'])
+
+    def set_nodes(self, nodes, roles):
+        if not nodes:
+            return
+        node_list = []
+        for n in nodes:
+            node_list.append({'id': n['id'],
+                              'pending_roles': roles,
+                              'pending_addition': True,
+                              'cluster_id': self.cluster['id']})
+        self.client.put('nodes', node_list)
+
+    def configure_network(self, config):
+        netconfig = self.get_network()
+        for network in netconfig['networks']:
+            if network['name'] in config:
+                network.update(config[network['name']])
+        self.set_network(netconfig)
+
+    def deploy(self):
+        self.client.put('clusters/%d/changes' % self.cluster['id'], {})
+        for task in self.client.get_tasks(self.cluster['id']):
+            if task['name'] == 'deploy':
+                task_id = task['id']
+                break
+        while 1:
+            time.sleep(10)
+            task = self.client.get_task(task_id)
+            if task['progress'] == 100:
+                return
+            LOG.info('Deployment in progress. %d%% done.' % task['progress'])
+
+    def get_network(self):
+        args = {'cluster_id': self.cluster['id'],
+                'net_provider': self.cluster['net_provider']}
+        url = ('clusters/%(cluster_id)d/network_configuration/'
+               '%(net_provider)s' % args)
+        return self.client.get(url)
+
+    def set_network(self, config):
+        self.verify_network(config)
+        args = {'cluster_id': self.cluster['id'],
+                'net_provider': self.cluster['net_provider']}
+        url = ('clusters/%(cluster_id)d/network_configuration/'
+               '%(net_provider)s' % args)
+        self.client.put(url, config)
+
+    def verify_network(self, config):
+        args = {'cluster_id': self.cluster['id'],
+                'net_provider': self.cluster['net_provider']}
+        url = ('clusters/%(cluster_id)d/network_configuration/'
+               '%(net_provider)s/verify' % args)
+        task_id = self.client.put(url, config)['id']
+        while 1:
+            time.sleep(5)
+            task = self.client.get_task(task_id)
+            if task['progress'] == 100:
+                if task['message']:
+                    raise FuelNetworkVerificationFailed(task['message'])
+                else:
+                    return
+            LOG.info('Network verification in progress.'
+                     ' %d%% done.' % task['progress'])
+
+    def get_attributes(self):
+        return self.client.get('clusters/%d/attributes' % self.cluster['id'])
+
+    def get_endpoint_ip(self):
+        if self.cluster['mode'].startswith('ha_'):
+            netdata = self.get_network()
+            return netdata['public_vip']
+
+        for node in self.get_nodes():
+            if "controller" in node['roles']:
+                for net in node['network_data']:
+                    if net['name'] == 'public':
+                        return net['ip'].split('/')[0]
+
+        raise FuelException('Unable to get endpoint ip.')
+
+
+class FuelNodesCollection(object):
+    nodes = []
+
+    def __init__(self, nodes):
+        for node in nodes:
+            self.nodes.append(FuelNode(node))
+
+    def pop(self, filters):
+        for i, node in enumerate(self.nodes):
+            if node.check_filters(filters):
+                return self.nodes.pop(i)
+
+
+class FuelClient(object):
+
+    def __init__(self, base_url):
+        self.base_url = base_url
+
+    def _request(self, method, url, data=None):
+        if data:
+            data = json.dumps(data)
+        headers = {'content-type': 'application/json'}
+        reply = getattr(requests, method)(self.base_url + url, data=data,
+                                          headers=headers)
+        if reply.status_code >= 300 or reply.status_code < 200:
+            raise FuelClientException(code=reply.status_code, body=reply.text)
+        if reply.text and reply.headers['content-type'] == 'application/json':
+            return json.loads(reply.text)
+        return reply
+
+    def get(self, url):
+        return self._request('get', url)
+
+    def post(self, url, data):
+        return self._request('post', url, data)
+
+    def put(self, url, data):
+        return self._request('put', url, data)
+
+    def delete(self, url):
+        return self._request('delete', url)
+
+    def get_releases(self):
+        return self.get('releases')
+
+    def get_task(self, task_id):
+        return self.get('tasks/%d' % task_id)
+
+    def get_tasks(self, cluster_id):
+        return self.get('tasks?cluster_id=%d' % cluster_id)
+
+    def get_node(self, node_id):
+        return self.get('nodes/%d' % node_id)
+
+    def get_nodes(self):
+        return FuelNodesCollection(self.get('nodes'))
+
+    def delete_cluster(self, cluster_id):
+        self.delete('clusters/%s' % cluster_id)

requirements.txt

@@ -12,5 +12,6 @@ python-keystoneclient>=0.4.1
 python-novaclient>=2.15.0
 python-neutronclient>=2.3.0,<3
 python-cinderclient>=1.0.6
+requests>=1.1
 SQLAlchemy>=0.7.8,<=0.7.99
 six>=1.4.1

tests/test_fuelclient.py

@@ -0,0 +1,337 @@
+# Copyright 2013: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from rally import fuelclient
+from tests import test
+
+import copy
+import mock
+
+
+class FuelNodeTestCase(test.TestCase):
+
+    def test_check(self):
+
+        node = {
+            "cluster": None,
+            "mac": "00:01:02:0a:0b:0c",
+            "meta": {
+                "memory": {"total": 42},
+                "cpu": {"total": 2},
+                "disks": [{"size": 22}, {"size": 33}]  # total 55
+            },
+        }
+
+        n = fuelclient.FuelNode(node)
+
+        self.assertFalse(n.check('ram==41'))
+        self.assertFalse(n.check('ram!=42'))
+        self.assertFalse(n.check('ram<=41'))
+        self.assertFalse(n.check('ram>=43'))
+        self.assertFalse(n.check('ram>43'))
+        self.assertFalse(n.check('ram<41'))
+        self.assertFalse(n.check('cpus>3'))
+
+        self.assertTrue(n.check('ram==42'))
+        self.assertTrue(n.check('ram!=41'))
+        self.assertTrue(n.check('ram<=43'))
+        self.assertTrue(n.check('ram<=42'))
+        self.assertTrue(n.check('ram>=41'))
+        self.assertTrue(n.check('ram>=42'))
+        self.assertTrue(n.check('ram<43'))
+        self.assertTrue(n.check('ram>41'))
+        self.assertTrue(n.check('cpus==2'))
+
+        self.assertTrue(n.check('mac==00:01:02:0a:0b:0c'))
+        self.assertTrue(n.check('mac!=00:01:02:0a:0b:0e'))
+        self.assertTrue(n.check('storage==55'))
+        self.assertTrue(n.check('storage<=1G'))
+        self.assertTrue(n.check('storage<1M'))
+
+
+class FuelNodesCollectionTestCase(test.TestCase):
+
+    def test_pop(self):
+        node = {
+            "cluster": None,
+            "mac": "00:01:02:0a:0b:0c",
+            "meta": {
+                "memory": {"total": 42},
+                "cpu": {"total": 2},
+                "disks": [{"size": 22}, {"size": 33}]  # total 55
+            },
+        }
+
+        nodes = [copy.deepcopy(node) for i in range(4)]
+        nodes[0]['meta']['cpu']['total'] = 1
+        nodes[1]['meta']['cpu']['total'] = 2
+        nodes[2]['meta']['memory']['total'] = 16
+        nodes[3]['cluster'] = 42  # node with cluster is occupied
+
+        nodes = fuelclient.FuelNodesCollection(nodes)
+
+        node_1cpu = nodes.pop(['cpus==1'])
+        self.assertEqual(node_1cpu._get_cpus(), 1)
+        self.assertEqual(len(nodes.nodes), 3)
+
+        node_2cpu = nodes.pop(['cpus==2'])
+        self.assertEqual(node_2cpu._get_cpus(), 2)
+        self.assertEqual(len(nodes.nodes), 2)
+
+        node_16ram_2cpu = nodes.pop(['ram>=16', 'cpus==2'])
+        self.assertEqual(node_16ram_2cpu._get_ram(), 16)
+        self.assertEqual(node_16ram_2cpu._get_cpus(), 2)
+        self.assertEqual(len(nodes.nodes), 1)
+        node_none = nodes.pop(['storage>4T'])
+        self.assertIsNone(node_none)
+
+
+class FuelClusterTestCase(test.TestCase):
+
+    def setUp(self):
+        super(FuelClusterTestCase, self).setUp()
+        self.client = mock.Mock()
+        self.config = {'name': 'Cluster'}
+        self.cluster = fuelclient.FuelCluster(self.client, **self.config)
+
+    def test_init(self):
+        self.client.post.assert_called_once_with('clusters', self.config)
+
+    def test_get_nodes(self):
+        self.cluster.cluster = {'id': 42}
+        self.cluster.get_nodes()
+        self.client.get.assert_called_once_with('nodes?cluster_id=42')
+
+    def test_set_nodes_empty(self):
+        self.assertIsNone(self.cluster.set_nodes([], []))
+
+    def test_set_nodes(self):
+        nodes = [{'id': 42}, {'id': 43}]
+        self.cluster.cluster = {'id': 1}
+        self.cluster.set_nodes(nodes, ['role1', 'role2'])
+
+        node42_args = {'cluster_id': 1,
+                       'pending_roles': ['role1', 'role2'],
+                       'pending_addition': True,
+                       'id': 42}
+        node43_args = {'cluster_id': 1,
+                       'pending_roles': ['role1', 'role2'],
+                       'pending_addition': True,
+                       'id': 43}
+        expected = [
+            mock.call.post('clusters', {'name': 'Cluster'}),
+            mock.call.put('nodes', [node42_args, node43_args])
+        ]
+        self.assertEqual(expected, self.client.mock_calls)
+
+    def test_configure_network(self):
+        current_network = {'networks': [{'name': 'public',
+                                         'key': 'old_val',
+                                         'key2': 'val2'}]}
+
+        netconfig = {'public': {'key': 'new_val'}}
+        self.cluster.get_network = mock.Mock(return_value=current_network)
+        self.cluster.set_network = mock.Mock()
+
+        self.cluster.configure_network(netconfig)
+
+        self.cluster.set_network.assert_called_once_with(
+            {'networks': [{'name': 'public',
+                           'key': 'new_val',
+                           'key2': 'val2'}]})
+
+    @mock.patch('rally.fuelclient.time.sleep')
+    def test_deploy(self, m_sleep):
+        call1 = {'progress': 50}
+        call2 = {'progress': 100}
+        self.client.get_task.side_effect = [call1, call2]
+
+        tasks = [{'name': 'deploy', 'id': 41}]
+        self.client.get_tasks.return_value = tasks
+
+        self.cluster.cluster = {'id': 42}
+        self.cluster.deploy()
+
+        expected = [
+            mock.call.post('clusters', {'name': 'Cluster'}),
+            mock.call.put('clusters/42/changes', {}),
+            mock.call.get_tasks(42),
+            mock.call.get_task(41),
+            mock.call.get_task(41)
+        ]
+        self.assertEqual(expected, self.client.mock_calls)
+
+    def test_get_network(self):
+        self.cluster.cluster = {'id': 42, 'net_provider': 'nova_network'}
+        self.cluster.get_network()
+        self.client.get.assert_called_once_with(
+            'clusters/42/network_configuration/nova_network')
+
+    def test_set_network(self):
+        self.cluster.cluster = {'id': 42, 'net_provider': 'nova_network'}
+        self.cluster.verify_network = mock.Mock()
+        self.cluster.set_network({'key': 'val'})
+
+        self.client.put.assert_called_once_with(
+            'clusters/42/network_configuration/nova_network', {'key': 'val'})
+        self.cluster.verify_network.assert_called_once_with({'key': 'val'})
+
+    @mock.patch('rally.fuelclient.time.sleep')
+    def test_verify_network(self, m_sleep):
+        call1 = {'progress': 50}
+        call2 = {'progress': 100, 'message': ''}
+
+        self.client.put.return_value = {'id': 42}
+        self.client.get_task.side_effect = [call1, call2]
+        self.cluster.cluster = {'id': 43, 'net_provider': 'nova_network'}
+
+        self.cluster.verify_network({'key': 'val'})
+
+        self.client.put.assert_called_once_with(
+            'clusters/43/network_configuration/nova_network/verify',
+            {'key': 'val'})
+        self.assertEqual([mock.call(42), mock.call(42)],
+                         self.client.get_task.mock_calls)
+
+    @mock.patch('rally.fuelclient.time.sleep')
+    def test_verify_network_fail(self, m_sleep):
+        self.client.put.return_value = {'id': 42}
+        self.client.get_task.return_value = {'progress': 100,
+                                             'message': 'error'}
+        self.cluster.cluster = {'id': 43, 'net_provider': 'nova_network'}
+        self.assertRaises(fuelclient.FuelNetworkVerificationFailed,
+                          self.cluster.verify_network, {'key': 'val'})
+
+    def test_get_attributes(self):
+        self.cluster.cluster = {'id': 52}
+        self.cluster.get_attributes()
+        self.client.get.assert_called_once_with('clusters/52/attributes')
+
+    def test_get_endpoint_ip_multinode(self):
+        self.cluster.cluster = {'mode': 'multinode'}
+        node1 = {'roles': ['compute', 'cinder']}
+        node2 = {'roles': ['controller'],
+                 'network_data': [{'name': 'private'},
+                                  {'name': 'public', 'ip': '42.42.42.42/24'}]}
+        fake_nodes = [node1, node2]
+        self.cluster.get_nodes = mock.Mock(return_value=fake_nodes)
+        ip = self.cluster.get_endpoint_ip()
+        self.assertEqual('42.42.42.42', ip)
+
+    def test_get_endpoint_ip_ha(self):
+        ip = '1.2.3.4'
+        self.cluster.cluster = {'id': 42, 'mode': 'ha_compact'}
+        self.cluster.get_network = mock.Mock(return_value={'public_vip': ip})
+        self.assertEqual(ip, self.cluster.get_endpoint_ip())
+
+
+class FuelClientTestCase(test.TestCase):
+
+    def setUp(self):
+        super(FuelClientTestCase, self).setUp()
+        self.client = fuelclient.FuelClient('http://10.20.0.2:8000/api/v1/')
+
+    @mock.patch('rally.fuelclient.requests')
+    def test__request_non_json(self, fake_req):
+        reply = mock.Mock()
+        reply.status_code = 200
+        reply.headers = {'content-type': 'application/x-httpd-php'}
+        reply.text = '{"reply": "ok"}'
+        fake_req.method = mock.Mock(return_value=reply)
+
+        retval = self.client._request('method', 'url', data={'key': 'value'})
+
+        self.assertEqual(retval, reply)
+
+    @mock.patch('rally.fuelclient.requests')
+    def test__request_non_2xx(self, fake_req):
+        reply = mock.Mock()
+        reply.status_code = 300
+        reply.headers = {'content-type': 'application/json'}
+        reply.text = '{"reply": "ok"}'
+        fake_req.method = mock.Mock(return_value=reply)
+        self.assertRaises(fuelclient.FuelClientException,
+                          self.client._request, 'method', 'url',
+                          data={'key': 'value'})
+
+    @mock.patch('rally.fuelclient.requests')
+    def test__request(self, fake_req):
+        reply = mock.Mock()
+        reply.status_code = 202
+        reply.headers = {'content-type': 'application/json'}
+        reply.text = '{"reply": "ok"}'
+        fake_req.method = mock.Mock(return_value=reply)
+
+        retval = self.client._request('method', 'url', data={'key': 'value'})
+        fake_req.method.assert_called_once_with(
+            'http://10.20.0.2:8000/api/v1/url',
+            headers={'content-type': 'application/json'},
+            data='{"key": "value"}')
+        self.assertEqual(retval, {'reply': 'ok'})
+
+    @mock.patch.object(fuelclient.FuelClient, '_request')
+    def test_get(self, m_request):
+        self.client.get('url')
+        m_request.assert_called_once_with('get', 'url')
+
+    @mock.patch.object(fuelclient.FuelClient, '_request')
+    def test_delete(self, m_request):
+        self.client.delete('url')
+        m_request.assert_called_once_with('delete', 'url')
+
+    @mock.patch.object(fuelclient.FuelClient, '_request')
+    def test_post(self, m_request):
+        self.client.post('url', {'key': 'val'})
+        m_request.assert_called_once_with('post', 'url', {'key': 'val'})
+
+    @mock.patch.object(fuelclient.FuelClient, '_request')
+    def test_put(self, m_request):
+        self.client.put('url', {'key': 'val'})
+        m_request.assert_called_once_with('put', 'url', {'key': 'val'})
+
+    @mock.patch.object(fuelclient.FuelClient, 'get')
+    def test_get_releases(self, m_get):
+        self.client.get_releases()
+        m_get.assert_called_once_with('releases')
+
+    @mock.patch.object(fuelclient.FuelClient, 'get')
+    def test_get_task(self, m_get):
+        self.client.get_task(42)
+        m_get.assert_called_once_with('tasks/42')
+
+    @mock.patch.object(fuelclient.FuelClient, 'get')
+    def test_get_tasks(self, m_get):
+        self.client.get_tasks(42)
+        m_get.assert_called_once_with('tasks?cluster_id=42')
+
+    @mock.patch.object(fuelclient.FuelClient, 'get')
+    def test_get_node(self, m_get):
+        self.client.get_node(42)
+        m_get.assert_called_once_with('nodes/42')
+
+    @mock.patch.object(fuelclient, 'FuelNodesCollection')
+    @mock.patch.object(fuelclient.FuelClient, 'get')
+    def test_get_nodes(self, m_get, m_collection):
+        m_get.return_value = 'fake_nodes'
+        m_collection.return_value = 'fake_collection'
+        retval = self.client.get_nodes()
+        self.assertEqual('fake_collection', retval)
+        m_collection.assert_called_once_with('fake_nodes')
+        m_get.assert_called_once_with('nodes')
+
+    @mock.patch.object(fuelclient.FuelClient, 'delete')
+    def test_delete_cluster(self, m_delete):
+        self.client.delete_cluster(42)
+        m_delete.assert_called_once_with('clusters/42')