Merge "Fix Opencontrail pollster according the API changes"

This commit is contained in:
Jenkins 2015-01-23 21:01:07 +00:00 committed by Gerrit Code Review
commit 5042b2da73
4 changed files with 311 additions and 182 deletions

View File

@ -14,6 +14,8 @@
# License for the specific language governing permissions and limitations
# under the License.
import copy
from oslo_config import cfg
import requests
import six
@ -37,53 +39,22 @@ class OpencontrailAPIFailed(Exception):
class AnalyticsAPIBaseClient(object):
"""Opencontrail Base Statistics REST API Client."""
def __init__(self, endpoint, username, password, domain, verify_ssl=True):
def __init__(self, endpoint, data):
self.endpoint = endpoint
self.username = username
self.password = password
self.domain = domain
self.verify_ssl = verify_ssl
self.sid = None
self.data = data or {}
def authenticate(self):
path = '/authenticate'
data = {'username': self.username,
'password': self.password,
'domain': self.domain}
def request(self, path, fqdn_uuid, data=None):
req_data = copy.copy(self.data)
if data:
req_data.update(data)
req_params = self._get_req_params(data=data)
url = urlparse.urljoin(self.endpoint, path)
resp = requests.post(url, **req_params)
if resp.status_code != 302:
raise OpencontrailAPIFailed(
_('Opencontrail API returned %(status)s %(reason)s') %
{'status': resp.status_code, 'reason': resp.reason})
self.sid = resp.cookies['connect.sid']
req_params = self._get_req_params(data=req_data)
def request(self, path, fqdn_uuid, data, retry=True):
if not self.sid:
self.authenticate()
if not data:
data = {'fqnUUID': fqdn_uuid}
else:
data['fqnUUID'] = fqdn_uuid
req_params = self._get_req_params(data=data,
cookies={'connect.sid': self.sid})
url = urlparse.urljoin(self.endpoint, path)
url = urlparse.urljoin(self.endpoint, path + fqdn_uuid)
self._log_req(url, req_params)
resp = requests.get(url, **req_params)
self._log_res(resp)
# it seems that the sid token has to be renewed
if resp.status_code == 302:
self.sid = 0
if retry:
return self.request(path, fqdn_uuid, data,
retry=False)
if resp.status_code != 200:
raise OpencontrailAPIFailed(
_('Opencontrail API returned %(status)s %(reason)s') %
@ -91,15 +62,13 @@ class AnalyticsAPIBaseClient(object):
return resp
def _get_req_params(self, params=None, data=None, cookies=None):
def _get_req_params(self, data=None):
req_params = {
'headers': {
'Accept': 'application/json'
},
'data': data,
'verify': self.verify_ssl,
'allow_redirects': False,
'cookies': cookies,
'timeout': CONF.http_timeout,
}
@ -143,24 +112,20 @@ class AnalyticsAPIBaseClient(object):
class NetworksAPIClient(AnalyticsAPIBaseClient):
"""Opencontrail Statistics REST API Client."""
def get_port_statistics(self, fqdn_uuid):
"""Get port statistics of a network
def get_vm_statistics(self, fqdn_uuid, data=None):
"""Get statistics of a virtual-machines.
URL:
/tenant/networking/virtual-machines/details
PARAMS:
fqdnUUID=fqdn_uuid
type=vn
{endpoint}/analytics/uves/virtual-machine/{fqdn_uuid}
"""
path = '/api/tenant/networking/virtual-machines/details'
resp = self.request(path, fqdn_uuid, {'type': 'vn'})
path = '/analytics/uves/virtual-machine/'
resp = self.request(path, fqdn_uuid, data)
return resp.json()
class Client(object):
def __init__(self, endpoint, username, password, domain, verify_ssl=True):
self.networks = NetworksAPIClient(endpoint, username, password,
domain, verify_ssl)
def __init__(self, endpoint, data=None):
self.networks = NetworksAPIClient(endpoint, data)

View File

@ -13,6 +13,9 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import re
from oslo_utils import timeutils
from six.moves.urllib import parse as urlparse
@ -25,6 +28,7 @@ class OpencontrailDriver(driver.Driver):
"""Driver of network analytics of Opencontrail.
This driver uses resources in "pipeline.yaml".
Resource requires below conditions:
* resource is url
@ -35,21 +39,26 @@ class OpencontrailDriver(driver.Driver):
* scheme:
The scheme of request url to Opencontrail Analytics endpoint.
(default http)
* username:
This is username used by Opencontrail Analytics.(default None)
* password:
This is password used by Opencontrail Analytics.(default None)
* domain:
This is domain used by Opencontrail Analytics.(default None)
* verify_ssl:
Specify if the certificate will be checked for https request.
(default false)
(default "http")
* virtual_network
Specify the virtual network.
(default None)
* fqdn_uuid:
Specify the VM fqdn UUID.
(default "*")
* resource:
The resource on which the counters are retrieved.
(default "if_stats_list")
* fip_stats_list:
Traffic on floating ips
* if_stats_list:
Traffic on VM interfaces
e.g.::
opencontrail://localhost:8143/?username=admin&password=admin&
scheme=https&domain=&verify_ssl=true
opencontrail://localhost:8081/?resource=fip_stats_list&
virtual_network=default-domain:openstack:public
"""
@staticmethod
def _prepare_cache(endpoint, params, cache):
@ -58,11 +67,7 @@ class OpencontrailDriver(driver.Driver):
return cache['network.statistics.opencontrail']
data = {
'o_client': client.Client(endpoint,
params['username'],
params['password'],
params.get('domain'),
params.get('verify_ssl') == 'true'),
'o_client': client.Client(endpoint),
'n_client': neutron_client.Client()
}
@ -95,22 +100,21 @@ class OpencontrailDriver(driver.Driver):
data = self._prepare_cache(endpoint, params, cache)
ports = data['n_client'].port_get_all()
ports_map = dict((port['id'], port['tenant_id']) for port in ports)
ports_map = dict((port['id'], port) for port in ports)
networks = data['n_client'].network_get_all()
for network in networks:
net_id = network['id']
resource = params.get('resource', ['if_stats_list'])[0]
fqdn_uuid = params.get('fqdn_uuid', ['*'])[0]
virtual_network = params.get('virtual_network', [None])[0]
timestamp = timeutils.utcnow().isoformat()
statistics = data['o_client'].networks.get_port_statistics(net_id)
statistics = data['o_client'].networks.get_vm_statistics(fqdn_uuid)
if not statistics:
continue
return
for value in statistics['value']:
for sample in iter(extractor, value, ports_map):
for sample in iter(extractor, value, ports_map,
resource, virtual_network):
if sample is not None:
sample[2]['network_id'] = net_id
yield sample + (timestamp, )
def _get_iter(self, meter_name):
@ -122,17 +126,65 @@ class OpencontrailDriver(driver.Driver):
return getattr(self, method_name, None)
@staticmethod
def _iter_port(extractor, value, ports_map):
ifstats = value['value']['UveVirtualMachineAgent']['if_stats_list']
for ifstat in ifstats:
name = ifstat['name']
device_owner_id, port_id = name.split(':')
def _explode_name(fq_name):
m = re.match(
"(?P<domain>[^:]+):(?P<project>.+):(?P<port_id>[^:]+)",
fq_name)
if not m:
return
return m.group('domain'), m.group('project'), m.group('port_id')
tenant_id = ports_map.get(port_id)
@staticmethod
def _get_resource_meta(ports_map, stat, resource, network):
if resource == 'fip_stats_list':
if network and (network != stat['virtual_network']):
return
name = stat['iface_name']
else:
name = stat['name']
domain, project, port_id = OpencontrailDriver._explode_name(name)
port = ports_map.get(port_id)
tenant_id = None
network_id = None
device_owner_id = None
if port:
tenant_id = port['tenant_id']
network_id = port['network_id']
device_owner_id = port['device_id']
resource_meta = {'device_owner_id': device_owner_id,
'tenant_id': tenant_id}
yield extractor(ifstat, port_id, resource_meta)
'network_id': network_id,
'project_id': tenant_id,
'project': project,
'resource': resource,
'domain': domain}
return port_id, resource_meta
@staticmethod
def _iter_port(extractor, value, ports_map, resource,
virtual_network=None):
stats = value['value']['UveVirtualMachineAgent'].get(resource, [])
for stat in stats:
if type(stat) is list:
for sub_stats, node in zip(*[iter(stat)] * 2):
for sub_stat in sub_stats:
result = OpencontrailDriver._get_resource_meta(
ports_map, sub_stat, resource, virtual_network)
if not result:
continue
port_id, resource_meta = result
yield extractor(sub_stat, port_id, resource_meta)
else:
result = OpencontrailDriver._get_resource_meta(
ports_map, stat, resource, virtual_network)
if not result:
continue
port_id, resource_meta = result
yield extractor(stat, port_id, resource_meta)
@staticmethod
def _switch_port_receive_packets(statistic, resource_id, resource_meta):

View File

@ -24,53 +24,44 @@ class TestOpencontrailClient(base.BaseTestCase):
def setUp(self):
super(TestOpencontrailClient, self).setUp()
self.client = client.Client('http://127.0.0.1:8143',
'admin', 'admin', None, False)
self.post_resp = mock.MagicMock()
self.post = mock.patch('requests.post',
return_value=self.post_resp).start()
self.post_resp.raw.version = 1.1
self.post_resp.status_code = 302
self.post_resp.reason = 'Moved'
self.post_resp.headers = {}
self.post_resp.cookies = {'connect.sid': 'aaa'}
self.post_resp.content = 'dummy'
self.client = client.Client('http://127.0.0.1:8081', {'arg1': 'aaa'})
self.get_resp = mock.MagicMock()
self.get = mock.patch('requests.get',
return_value=self.get_resp).start()
self.get_resp.raw_version = 1.1
self.get_resp.status_code = 200
self.post_resp.content = 'dqs'
def test_port_statistics(self):
uuid = 'bbb'
self.client.networks.get_port_statistics(uuid)
call_args = self.post.call_args_list[0][0]
call_kwargs = self.post.call_args_list[0][1]
expected_url = 'http://127.0.0.1:8143/authenticate'
self.assertEqual(expected_url, call_args[0])
data = call_kwargs.get('data')
expected_data = {'domain': None, 'password': 'admin',
'username': 'admin'}
self.assertEqual(expected_data, data)
def test_vm_statistics(self):
self.client.networks.get_vm_statistics('bbb')
call_args = self.get.call_args_list[0][0]
call_kwargs = self.get.call_args_list[0][1]
expected_url = ('http://127.0.0.1:8143/api/tenant/'
'networking/virtual-machines/details')
expected_url = ('http://127.0.0.1:8081/analytics/'
'uves/virtual-machine/bbb')
self.assertEqual(expected_url, call_args[0])
data = call_kwargs.get('data')
cookies = call_kwargs.get('cookies')
expected_data = {'fqnUUID': 'bbb', 'type': 'vn'}
expected_cookies = {'connect.sid': 'aaa'}
expected_data = {'arg1': 'aaa'}
self.assertEqual(expected_data, data)
def test_vm_statistics_params(self):
self.client.networks.get_vm_statistics('bbb',
{'resource': 'fip_stats_list',
'virtual_network': 'ccc'})
call_args = self.get.call_args_list[0][0]
call_kwargs = self.get.call_args_list[0][1]
expected_url = ('http://127.0.0.1:8081/analytics/'
'uves/virtual-machine/bbb')
self.assertEqual(expected_url, call_args[0])
data = call_kwargs.get('data')
expected_data = {'arg1': 'aaa',
'resource': 'fip_stats_list',
'virtual_network': 'ccc'}
self.assertEqual(expected_data, data)
self.assertEqual(expected_cookies, cookies)

View File

@ -31,11 +31,6 @@ class TestOpencontrailDriver(base.BaseTestCase):
return_value=self.fake_ports())
self.nc_ports.start()
self.nc_networks = mock.patch('ceilometer.neutron_client'
'.Client.network_get_all',
return_value=self.fake_networks())
self.nc_networks.start()
self.driver = driver.OpencontrailDriver()
self.parse_url = urlparse.ParseResult('opencontrail',
'127.0.0.1:8143',
@ -43,7 +38,8 @@ class TestOpencontrailDriver(base.BaseTestCase):
self.params = {'password': ['admin'],
'scheme': ['http'],
'username': ['admin'],
'verify_ssl': ['false']}
'verify_ssl': ['false'],
'resource': ['if_stats_list']}
@staticmethod
def fake_ports():
@ -58,20 +54,6 @@ class TestOpencontrailDriver(base.BaseTestCase):
'status': 'ACTIVE',
'tenant_id': '89271fa581ab4380bf172f868c3615f9'}]
@staticmethod
def fake_networks():
return [{'admin_state_up': True,
'id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'name': 'public',
'provider:network_type': 'gre',
'provider:physical_network': None,
'provider:segmentation_id': 2,
'router:external': True,
'shared': False,
'status': 'ACTIVE',
'subnets': [u'c4b6f5b8-3508-4896-b238-a441f25fb492'],
'tenant_id': '62d6f08bbd3a44f6ad6f00ca15cce4e5'}]
@staticmethod
def fake_port_stats():
return {"value": [{
@ -85,61 +67,200 @@ class TestOpencontrailDriver(base.BaseTestCase):
"out_bandwidth_usage": 0,
"out_pkts": 5,
"in_pkts": 6,
"name": ("674e553b-8df9-4321-87d9-93ba05b93558:"
"name": ("default-domain:demo:"
"96d49cc3-4e01-40ce-9cac-c0e32642a442")
}]}}}]}
}],
"fip_stats_list": [{
"in_bytes": 33,
"iface_name": ("default-domain:demo:"
"96d49cc3-4e01-40ce-9cac-c0e32642a442"),
"out_bytes": 44,
"out_pkts": 10,
"virtual_network": "default-domain:openstack:public",
"in_pkts": 11,
"ip_address": "1.1.1.1"
}]
}}}]}
def _test_meter(self, meter_name, expected):
@staticmethod
def fake_port_stats_with_node():
return {"value": [{
"name": "c588ebb7-ae52-485a-9f0c-b2791c5da196",
"value": {
"UveVirtualMachineAgent": {
"if_stats_list": [
[[{
"out_bytes": 22,
"in_bandwidth_usage": 0,
"in_bytes": 23,
"out_bandwidth_usage": 0,
"out_pkts": 5,
"in_pkts": 6,
"name": ("default-domain:demo:"
"96d49cc3-4e01-40ce-9cac-c0e32642a442")
}], 'node1'],
[[{
"out_bytes": 22,
"in_bandwidth_usage": 0,
"in_bytes": 23,
"out_bandwidth_usage": 0,
"out_pkts": 4,
"in_pkts": 13,
"name": ("default-domain:demo:"
"96d49cc3-4e01-40ce-9cac-c0e32642a442")}],
'node2']
]
}}}]}
def _test_meter(self, meter_name, expected, fake_port_stats=None):
if not fake_port_stats:
fake_port_stats = self.fake_port_stats()
with mock.patch('ceilometer.network.'
'statistics.opencontrail.'
'client.NetworksAPIClient.'
'get_port_statistics',
return_value=self.fake_port_stats()) as port_stats:
'get_vm_statistics',
return_value=fake_port_stats) as port_stats:
samples = self.driver.get_sample_data(meter_name, self.parse_url,
self.params, {})
self.assertEqual(expected, [s for s in samples])
net_id = '298a3088-a446-4d5a-bad8-f92ecacd786b'
port_stats.assert_called_with(net_id)
port_stats.assert_called_with('*')
def test_switch_port_receive_packets_with_node(self):
expected = [(6,
'96d49cc3-4e01-40ce-9cac-c0e32642a442',
{'device_owner_id':
'674e553b-8df9-4321-87d9-93ba05b93558',
'domain': 'default-domain',
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'if_stats_list'},
mock.ANY),
(13,
'96d49cc3-4e01-40ce-9cac-c0e32642a442',
{'device_owner_id':
'674e553b-8df9-4321-87d9-93ba05b93558',
'domain': 'default-domain',
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'if_stats_list'},
mock.ANY)]
self._test_meter('switch.port.receive.packets', expected,
self.fake_port_stats_with_node())
def test_switch_port_receive_packets(self):
expected = [
(6,
expected = [(6,
'96d49cc3-4e01-40ce-9cac-c0e32642a442',
{'device_owner_id': '674e553b-8df9-4321-87d9-93ba05b93558',
{'device_owner_id':
'674e553b-8df9-4321-87d9-93ba05b93558',
'domain': 'default-domain',
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'tenant_id': '89271fa581ab4380bf172f868c3615f9'},
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'if_stats_list'},
mock.ANY)]
self._test_meter('switch.port.receive.packets', expected)
def test_switch_port_transmit_packets(self):
expected = [
(5,
expected = [(5,
'96d49cc3-4e01-40ce-9cac-c0e32642a442',
{'device_owner_id': '674e553b-8df9-4321-87d9-93ba05b93558',
{'device_owner_id':
'674e553b-8df9-4321-87d9-93ba05b93558',
'domain': 'default-domain',
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'tenant_id': '89271fa581ab4380bf172f868c3615f9'},
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'if_stats_list'},
mock.ANY)]
self._test_meter('switch.port.transmit.packets', expected)
def test_switch_port_receive_bytes(self):
expected = [
(23,
expected = [(23,
'96d49cc3-4e01-40ce-9cac-c0e32642a442',
{'device_owner_id': '674e553b-8df9-4321-87d9-93ba05b93558',
{'device_owner_id':
'674e553b-8df9-4321-87d9-93ba05b93558',
'domain': 'default-domain',
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'tenant_id': '89271fa581ab4380bf172f868c3615f9'},
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'if_stats_list'},
mock.ANY)]
self._test_meter('switch.port.receive.bytes', expected)
def test_switch_port_transmit_bytes(self):
expected = [
(22,
expected = [(22,
'96d49cc3-4e01-40ce-9cac-c0e32642a442',
{'device_owner_id': '674e553b-8df9-4321-87d9-93ba05b93558',
{'device_owner_id':
'674e553b-8df9-4321-87d9-93ba05b93558',
'domain': 'default-domain',
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'tenant_id': '89271fa581ab4380bf172f868c3615f9'},
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'if_stats_list'},
mock.ANY)]
self._test_meter('switch.port.transmit.bytes', expected)
def test_switch_port_receive_packets_fip(self):
self.params['resource'] = ['fip_stats_list']
expected = [(11,
'96d49cc3-4e01-40ce-9cac-c0e32642a442',
{'device_owner_id':
'674e553b-8df9-4321-87d9-93ba05b93558',
'domain': 'default-domain',
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'fip_stats_list'},
mock.ANY)]
self._test_meter('switch.port.receive.packets', expected)
def test_switch_port_transmit_packets_fip(self):
self.params['resource'] = ['fip_stats_list']
expected = [(10,
'96d49cc3-4e01-40ce-9cac-c0e32642a442',
{'device_owner_id':
'674e553b-8df9-4321-87d9-93ba05b93558',
'domain': 'default-domain',
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'fip_stats_list'},
mock.ANY)]
self._test_meter('switch.port.transmit.packets', expected)
def test_switch_port_receive_bytes_fip(self):
self.params['resource'] = ['fip_stats_list']
expected = [(33,
'96d49cc3-4e01-40ce-9cac-c0e32642a442',
{'device_owner_id':
'674e553b-8df9-4321-87d9-93ba05b93558',
'domain': 'default-domain',
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'fip_stats_list'},
mock.ANY)]
self._test_meter('switch.port.receive.bytes', expected)
def test_switch_port_transmit_bytes_fip(self):
self.params['resource'] = ['fip_stats_list']
expected = [(44,
'96d49cc3-4e01-40ce-9cac-c0e32642a442',
{'device_owner_id':
'674e553b-8df9-4321-87d9-93ba05b93558',
'domain': 'default-domain',
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'fip_stats_list'},
mock.ANY)]
self._test_meter('switch.port.transmit.bytes', expected)
def test_switch_port_transmit_bytes_non_existing_network(self):
self.params['virtual_network'] = ['aaa']
self.params['resource'] = ['fip_stats_list']
self._test_meter('switch.port.transmit.bytes', [])