diff --git a/rally/fuelclient.py b/rally/fuelclient.py new file mode 100644 index 0000000000..b79a6706b4 --- /dev/null +++ b/rally/fuelclient.py @@ -0,0 +1,263 @@ +# Copyright 2013: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json +import re +import requests +import time + +from rally.openstack.common import log as logging + +LOG = logging.getLogger(__name__) + +FILTER_REG = re.compile(r'^([a-z]+)\s*([<>=!]=|<|>)\s*(.+)$') +INT_REG = re.compile(r'^(\d+)(K|M|G|T)?$') + + +class FuelException(Exception): + pass + + +class FuelClientException(FuelException): + + def __init__(self, code, body): + self.code = code + self.body = body + + def __str__(self): + return ('FuelClientException. ' + 'Code: %(code)d Body: %(body)s' % {'code': self.code, + 'body': self.body}) + + +class FuelNetworkVerificationFailed(FuelException): + pass + + +class FuelNode(object): + + def __init__(self, node): + self.node = node + self.ATTRIBUTE_MAP = { + '==': lambda x, y: x == y, + '!=': lambda x, y: x != y, + '<=': lambda x, y: x <= y, + '>=': lambda x, y: x >= y, + '<': lambda x, y: x < y, + '>': lambda x, y: x > y, + } + self.FACTOR_MAP = { + 'K': 1024, + 'M': 1048576, + 'G': 1073741824, + 'T': 1099511627776, + None: 1, + } + + def __getitem__(self, key): + return self.node[key] + + def check_filters(self, filters): + return all((self.check(f) for f in filters)) + + def check(self, filter_string): + if self.node['cluster'] is not None: + return False + m = FILTER_REG.match(filter_string) + if m is None: + raise ValueError('Invalid filter: %s' % filter_string) + attribute, operator, value = m.groups() + return self._check(attribute, value, operator) + + def _check(self, attribute, value, operator): + attribute = getattr(self, '_get_' + attribute)() + checker = self.ATTRIBUTE_MAP[operator] + m = INT_REG.match(value) + if m: + value = int(m.group(1)) * self.FACTOR_MAP[m.group(2)] + return checker(attribute, value) + + def _get_ram(self): + return self.node['meta']['memory']['total'] + + def _get_mac(self): + return self.node['mac'] + + def _get_storage(self): + return sum((d['size'] for d in self.node['meta']['disks'])) + + def _get_cpus(self): + return self.node['meta']['cpu']['total'] + + +class FuelCluster(object): + + def __init__(self, client, **config): + """Create Fuel cluster. + + :param client: FuelClient instance. + :param name: Name + :param release: Release id. Integer. + :param mode: One of multinode, ha_compact + :param net_provider: One of nova_network, neutron + :param net_segment_type: One of gre, vlan. + :param dns_nameservers: List of strings. + """ + + self.client = client + self.cluster = client.post('clusters', config) + + def get_nodes(self): + return self.client.get('nodes?cluster_id=%d' % self.cluster['id']) + + def set_nodes(self, nodes, roles): + if not nodes: + return + node_list = [] + for n in nodes: + node_list.append({'id': n['id'], + 'pending_roles': roles, + 'pending_addition': True, + 'cluster_id': self.cluster['id']}) + self.client.put('nodes', node_list) + + def configure_network(self, config): + netconfig = self.get_network() + for network in netconfig['networks']: + if network['name'] in config: + network.update(config[network['name']]) + self.set_network(netconfig) + + def deploy(self): + self.client.put('clusters/%d/changes' % self.cluster['id'], {}) + for task in self.client.get_tasks(self.cluster['id']): + if task['name'] == 'deploy': + task_id = task['id'] + break + while 1: + time.sleep(10) + task = self.client.get_task(task_id) + if task['progress'] == 100: + return + LOG.info('Deployment in progress. %d%% done.' % task['progress']) + + def get_network(self): + args = {'cluster_id': self.cluster['id'], + 'net_provider': self.cluster['net_provider']} + url = ('clusters/%(cluster_id)d/network_configuration/' + '%(net_provider)s' % args) + return self.client.get(url) + + def set_network(self, config): + self.verify_network(config) + args = {'cluster_id': self.cluster['id'], + 'net_provider': self.cluster['net_provider']} + url = ('clusters/%(cluster_id)d/network_configuration/' + '%(net_provider)s' % args) + self.client.put(url, config) + + def verify_network(self, config): + args = {'cluster_id': self.cluster['id'], + 'net_provider': self.cluster['net_provider']} + url = ('clusters/%(cluster_id)d/network_configuration/' + '%(net_provider)s/verify' % args) + task_id = self.client.put(url, config)['id'] + while 1: + time.sleep(5) + task = self.client.get_task(task_id) + if task['progress'] == 100: + if task['message']: + raise FuelNetworkVerificationFailed(task['message']) + else: + return + LOG.info('Network verification in progress.' + ' %d%% done.' % task['progress']) + + def get_attributes(self): + return self.client.get('clusters/%d/attributes' % self.cluster['id']) + + def get_endpoint_ip(self): + if self.cluster['mode'].startswith('ha_'): + netdata = self.get_network() + return netdata['public_vip'] + + for node in self.get_nodes(): + if "controller" in node['roles']: + for net in node['network_data']: + if net['name'] == 'public': + return net['ip'].split('/')[0] + + raise FuelException('Unable to get endpoint ip.') + + +class FuelNodesCollection(object): + nodes = [] + + def __init__(self, nodes): + for node in nodes: + self.nodes.append(FuelNode(node)) + + def pop(self, filters): + for i, node in enumerate(self.nodes): + if node.check_filters(filters): + return self.nodes.pop(i) + + +class FuelClient(object): + + def __init__(self, base_url): + self.base_url = base_url + + def _request(self, method, url, data=None): + if data: + data = json.dumps(data) + headers = {'content-type': 'application/json'} + reply = getattr(requests, method)(self.base_url + url, data=data, + headers=headers) + if reply.status_code >= 300 or reply.status_code < 200: + raise FuelClientException(code=reply.status_code, body=reply.text) + if reply.text and reply.headers['content-type'] == 'application/json': + return json.loads(reply.text) + return reply + + def get(self, url): + return self._request('get', url) + + def post(self, url, data): + return self._request('post', url, data) + + def put(self, url, data): + return self._request('put', url, data) + + def delete(self, url): + return self._request('delete', url) + + def get_releases(self): + return self.get('releases') + + def get_task(self, task_id): + return self.get('tasks/%d' % task_id) + + def get_tasks(self, cluster_id): + return self.get('tasks?cluster_id=%d' % cluster_id) + + def get_node(self, node_id): + return self.get('nodes/%d' % node_id) + + def get_nodes(self): + return FuelNodesCollection(self.get('nodes')) + + def delete_cluster(self, cluster_id): + self.delete('clusters/%s' % cluster_id) diff --git a/requirements.txt b/requirements.txt index bf5dd93251..c000156dfa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,5 +12,6 @@ python-keystoneclient>=0.4.1 python-novaclient>=2.15.0 python-neutronclient>=2.3.0,<3 python-cinderclient>=1.0.6 +requests>=1.1 SQLAlchemy>=0.7.8,<=0.7.99 six>=1.4.1 diff --git a/tests/test_fuelclient.py b/tests/test_fuelclient.py new file mode 100644 index 0000000000..255447bc44 --- /dev/null +++ b/tests/test_fuelclient.py @@ -0,0 +1,337 @@ +# Copyright 2013: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from rally import fuelclient +from tests import test + +import copy +import mock + + +class FuelNodeTestCase(test.TestCase): + + def test_check(self): + + node = { + "cluster": None, + "mac": "00:01:02:0a:0b:0c", + "meta": { + "memory": {"total": 42}, + "cpu": {"total": 2}, + "disks": [{"size": 22}, {"size": 33}] # total 55 + }, + } + + n = fuelclient.FuelNode(node) + + self.assertFalse(n.check('ram==41')) + self.assertFalse(n.check('ram!=42')) + self.assertFalse(n.check('ram<=41')) + self.assertFalse(n.check('ram>=43')) + self.assertFalse(n.check('ram>43')) + self.assertFalse(n.check('ram<41')) + self.assertFalse(n.check('cpus>3')) + + self.assertTrue(n.check('ram==42')) + self.assertTrue(n.check('ram!=41')) + self.assertTrue(n.check('ram<=43')) + self.assertTrue(n.check('ram<=42')) + self.assertTrue(n.check('ram>=41')) + self.assertTrue(n.check('ram>=42')) + self.assertTrue(n.check('ram<43')) + self.assertTrue(n.check('ram>41')) + self.assertTrue(n.check('cpus==2')) + + self.assertTrue(n.check('mac==00:01:02:0a:0b:0c')) + self.assertTrue(n.check('mac!=00:01:02:0a:0b:0e')) + self.assertTrue(n.check('storage==55')) + self.assertTrue(n.check('storage<=1G')) + self.assertTrue(n.check('storage<1M')) + + +class FuelNodesCollectionTestCase(test.TestCase): + + def test_pop(self): + node = { + "cluster": None, + "mac": "00:01:02:0a:0b:0c", + "meta": { + "memory": {"total": 42}, + "cpu": {"total": 2}, + "disks": [{"size": 22}, {"size": 33}] # total 55 + }, + } + + nodes = [copy.deepcopy(node) for i in range(4)] + nodes[0]['meta']['cpu']['total'] = 1 + nodes[1]['meta']['cpu']['total'] = 2 + nodes[2]['meta']['memory']['total'] = 16 + nodes[3]['cluster'] = 42 # node with cluster is occupied + + nodes = fuelclient.FuelNodesCollection(nodes) + + node_1cpu = nodes.pop(['cpus==1']) + self.assertEqual(node_1cpu._get_cpus(), 1) + self.assertEqual(len(nodes.nodes), 3) + + node_2cpu = nodes.pop(['cpus==2']) + self.assertEqual(node_2cpu._get_cpus(), 2) + self.assertEqual(len(nodes.nodes), 2) + + node_16ram_2cpu = nodes.pop(['ram>=16', 'cpus==2']) + self.assertEqual(node_16ram_2cpu._get_ram(), 16) + self.assertEqual(node_16ram_2cpu._get_cpus(), 2) + self.assertEqual(len(nodes.nodes), 1) + node_none = nodes.pop(['storage>4T']) + self.assertIsNone(node_none) + + +class FuelClusterTestCase(test.TestCase): + + def setUp(self): + super(FuelClusterTestCase, self).setUp() + self.client = mock.Mock() + self.config = {'name': 'Cluster'} + self.cluster = fuelclient.FuelCluster(self.client, **self.config) + + def test_init(self): + self.client.post.assert_called_once_with('clusters', self.config) + + def test_get_nodes(self): + self.cluster.cluster = {'id': 42} + self.cluster.get_nodes() + self.client.get.assert_called_once_with('nodes?cluster_id=42') + + def test_set_nodes_empty(self): + self.assertIsNone(self.cluster.set_nodes([], [])) + + def test_set_nodes(self): + nodes = [{'id': 42}, {'id': 43}] + self.cluster.cluster = {'id': 1} + self.cluster.set_nodes(nodes, ['role1', 'role2']) + + node42_args = {'cluster_id': 1, + 'pending_roles': ['role1', 'role2'], + 'pending_addition': True, + 'id': 42} + node43_args = {'cluster_id': 1, + 'pending_roles': ['role1', 'role2'], + 'pending_addition': True, + 'id': 43} + expected = [ + mock.call.post('clusters', {'name': 'Cluster'}), + mock.call.put('nodes', [node42_args, node43_args]) + ] + self.assertEqual(expected, self.client.mock_calls) + + def test_configure_network(self): + current_network = {'networks': [{'name': 'public', + 'key': 'old_val', + 'key2': 'val2'}]} + + netconfig = {'public': {'key': 'new_val'}} + self.cluster.get_network = mock.Mock(return_value=current_network) + self.cluster.set_network = mock.Mock() + + self.cluster.configure_network(netconfig) + + self.cluster.set_network.assert_called_once_with( + {'networks': [{'name': 'public', + 'key': 'new_val', + 'key2': 'val2'}]}) + + @mock.patch('rally.fuelclient.time.sleep') + def test_deploy(self, m_sleep): + call1 = {'progress': 50} + call2 = {'progress': 100} + self.client.get_task.side_effect = [call1, call2] + + tasks = [{'name': 'deploy', 'id': 41}] + self.client.get_tasks.return_value = tasks + + self.cluster.cluster = {'id': 42} + self.cluster.deploy() + + expected = [ + mock.call.post('clusters', {'name': 'Cluster'}), + mock.call.put('clusters/42/changes', {}), + mock.call.get_tasks(42), + mock.call.get_task(41), + mock.call.get_task(41) + ] + self.assertEqual(expected, self.client.mock_calls) + + def test_get_network(self): + self.cluster.cluster = {'id': 42, 'net_provider': 'nova_network'} + self.cluster.get_network() + self.client.get.assert_called_once_with( + 'clusters/42/network_configuration/nova_network') + + def test_set_network(self): + self.cluster.cluster = {'id': 42, 'net_provider': 'nova_network'} + self.cluster.verify_network = mock.Mock() + self.cluster.set_network({'key': 'val'}) + + self.client.put.assert_called_once_with( + 'clusters/42/network_configuration/nova_network', {'key': 'val'}) + self.cluster.verify_network.assert_called_once_with({'key': 'val'}) + + @mock.patch('rally.fuelclient.time.sleep') + def test_verify_network(self, m_sleep): + call1 = {'progress': 50} + call2 = {'progress': 100, 'message': ''} + + self.client.put.return_value = {'id': 42} + self.client.get_task.side_effect = [call1, call2] + self.cluster.cluster = {'id': 43, 'net_provider': 'nova_network'} + + self.cluster.verify_network({'key': 'val'}) + + self.client.put.assert_called_once_with( + 'clusters/43/network_configuration/nova_network/verify', + {'key': 'val'}) + self.assertEqual([mock.call(42), mock.call(42)], + self.client.get_task.mock_calls) + + @mock.patch('rally.fuelclient.time.sleep') + def test_verify_network_fail(self, m_sleep): + self.client.put.return_value = {'id': 42} + self.client.get_task.return_value = {'progress': 100, + 'message': 'error'} + self.cluster.cluster = {'id': 43, 'net_provider': 'nova_network'} + self.assertRaises(fuelclient.FuelNetworkVerificationFailed, + self.cluster.verify_network, {'key': 'val'}) + + def test_get_attributes(self): + self.cluster.cluster = {'id': 52} + self.cluster.get_attributes() + self.client.get.assert_called_once_with('clusters/52/attributes') + + def test_get_endpoint_ip_multinode(self): + self.cluster.cluster = {'mode': 'multinode'} + node1 = {'roles': ['compute', 'cinder']} + node2 = {'roles': ['controller'], + 'network_data': [{'name': 'private'}, + {'name': 'public', 'ip': '42.42.42.42/24'}]} + fake_nodes = [node1, node2] + self.cluster.get_nodes = mock.Mock(return_value=fake_nodes) + ip = self.cluster.get_endpoint_ip() + self.assertEqual('42.42.42.42', ip) + + def test_get_endpoint_ip_ha(self): + ip = '1.2.3.4' + self.cluster.cluster = {'id': 42, 'mode': 'ha_compact'} + self.cluster.get_network = mock.Mock(return_value={'public_vip': ip}) + self.assertEqual(ip, self.cluster.get_endpoint_ip()) + + +class FuelClientTestCase(test.TestCase): + + def setUp(self): + super(FuelClientTestCase, self).setUp() + self.client = fuelclient.FuelClient('http://10.20.0.2:8000/api/v1/') + + @mock.patch('rally.fuelclient.requests') + def test__request_non_json(self, fake_req): + reply = mock.Mock() + reply.status_code = 200 + reply.headers = {'content-type': 'application/x-httpd-php'} + reply.text = '{"reply": "ok"}' + fake_req.method = mock.Mock(return_value=reply) + + retval = self.client._request('method', 'url', data={'key': 'value'}) + + self.assertEqual(retval, reply) + + @mock.patch('rally.fuelclient.requests') + def test__request_non_2xx(self, fake_req): + reply = mock.Mock() + reply.status_code = 300 + reply.headers = {'content-type': 'application/json'} + reply.text = '{"reply": "ok"}' + fake_req.method = mock.Mock(return_value=reply) + self.assertRaises(fuelclient.FuelClientException, + self.client._request, 'method', 'url', + data={'key': 'value'}) + + @mock.patch('rally.fuelclient.requests') + def test__request(self, fake_req): + reply = mock.Mock() + reply.status_code = 202 + reply.headers = {'content-type': 'application/json'} + reply.text = '{"reply": "ok"}' + fake_req.method = mock.Mock(return_value=reply) + + retval = self.client._request('method', 'url', data={'key': 'value'}) + fake_req.method.assert_called_once_with( + 'http://10.20.0.2:8000/api/v1/url', + headers={'content-type': 'application/json'}, + data='{"key": "value"}') + self.assertEqual(retval, {'reply': 'ok'}) + + @mock.patch.object(fuelclient.FuelClient, '_request') + def test_get(self, m_request): + self.client.get('url') + m_request.assert_called_once_with('get', 'url') + + @mock.patch.object(fuelclient.FuelClient, '_request') + def test_delete(self, m_request): + self.client.delete('url') + m_request.assert_called_once_with('delete', 'url') + + @mock.patch.object(fuelclient.FuelClient, '_request') + def test_post(self, m_request): + self.client.post('url', {'key': 'val'}) + m_request.assert_called_once_with('post', 'url', {'key': 'val'}) + + @mock.patch.object(fuelclient.FuelClient, '_request') + def test_put(self, m_request): + self.client.put('url', {'key': 'val'}) + m_request.assert_called_once_with('put', 'url', {'key': 'val'}) + + @mock.patch.object(fuelclient.FuelClient, 'get') + def test_get_releases(self, m_get): + self.client.get_releases() + m_get.assert_called_once_with('releases') + + @mock.patch.object(fuelclient.FuelClient, 'get') + def test_get_task(self, m_get): + self.client.get_task(42) + m_get.assert_called_once_with('tasks/42') + + @mock.patch.object(fuelclient.FuelClient, 'get') + def test_get_tasks(self, m_get): + self.client.get_tasks(42) + m_get.assert_called_once_with('tasks?cluster_id=42') + + @mock.patch.object(fuelclient.FuelClient, 'get') + def test_get_node(self, m_get): + self.client.get_node(42) + m_get.assert_called_once_with('nodes/42') + + @mock.patch.object(fuelclient, 'FuelNodesCollection') + @mock.patch.object(fuelclient.FuelClient, 'get') + def test_get_nodes(self, m_get, m_collection): + m_get.return_value = 'fake_nodes' + m_collection.return_value = 'fake_collection' + retval = self.client.get_nodes() + self.assertEqual('fake_collection', retval) + m_collection.assert_called_once_with('fake_nodes') + m_get.assert_called_once_with('nodes') + + @mock.patch.object(fuelclient.FuelClient, 'delete') + def test_delete_cluster(self, m_delete): + self.client.delete_cluster(42) + m_delete.assert_called_once_with('clusters/42')