Pull metrics by using TLS enabled client
Magnum introduced TLS support in swarm bay, so swarm monitor needs to use TLS enabled client to pull data. In addition, this commit eliminates data pulling from the docker daemon of individual bay nodes, since they are not secured by TLS. Instead, we always pull from swarm. Due to that, additional text parsing is needed, since the output of docker.info from swarm is not well formed. Change-Id: I121bfa98622c240d5c777901794a35c0d8990345 Closes-Bug: #1503460
This commit is contained in:
parent
49cf17c12d
commit
30eacb52c9
|
@ -73,6 +73,12 @@ def docker_for_container(context, container):
|
||||||
if magnum_utils.is_uuid_like(container):
|
if magnum_utils.is_uuid_like(container):
|
||||||
container = objects.Container.get_by_uuid(context, container)
|
container = objects.Container.get_by_uuid(context, container)
|
||||||
bay = conductor_utils.retrieve_bay(context, container)
|
bay = conductor_utils.retrieve_bay(context, container)
|
||||||
|
with docker_for_bay(context, bay) as docker:
|
||||||
|
yield docker
|
||||||
|
|
||||||
|
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def docker_for_bay(context, bay):
|
||||||
baymodel = conductor_utils.retrieve_baymodel(context, bay)
|
baymodel = conductor_utils.retrieve_baymodel(context, bay)
|
||||||
|
|
||||||
tcp_url = 'tcp://%s:2376' % bay.api_address
|
tcp_url = 'tcp://%s:2376' % bay.api_address
|
||||||
|
|
|
@ -19,7 +19,7 @@ from oslo_config import cfg
|
||||||
from oslo_log import log
|
from oslo_log import log
|
||||||
import six
|
import six
|
||||||
|
|
||||||
from magnum.conductor.handlers.common import docker_client
|
from magnum.common import docker_utils
|
||||||
from magnum.i18n import _LW
|
from magnum.i18n import _LW
|
||||||
from magnum import objects
|
from magnum import objects
|
||||||
from magnum.objects.fields import BayType as bay_type
|
from magnum.objects.fields import BayType as bay_type
|
||||||
|
@ -39,7 +39,8 @@ CONF.import_opt('default_timeout',
|
||||||
@six.add_metaclass(abc.ABCMeta)
|
@six.add_metaclass(abc.ABCMeta)
|
||||||
class MonitorBase(object):
|
class MonitorBase(object):
|
||||||
|
|
||||||
def __init__(self, bay):
|
def __init__(self, context, bay):
|
||||||
|
self.context = context
|
||||||
self.bay = bay
|
self.bay = bay
|
||||||
|
|
||||||
@abc.abstractproperty
|
@abc.abstractproperty
|
||||||
|
@ -64,8 +65,8 @@ class MonitorBase(object):
|
||||||
|
|
||||||
class SwarmMonitor(MonitorBase):
|
class SwarmMonitor(MonitorBase):
|
||||||
|
|
||||||
def __init__(self, bay):
|
def __init__(self, context, bay):
|
||||||
super(SwarmMonitor, self).__init__(bay)
|
super(SwarmMonitor, self).__init__(context, bay)
|
||||||
self.data = {}
|
self.data = {}
|
||||||
self.data['nodes'] = []
|
self.data['nodes'] = []
|
||||||
self.data['containers'] = []
|
self.data['containers'] = []
|
||||||
|
@ -80,27 +81,23 @@ class SwarmMonitor(MonitorBase):
|
||||||
}
|
}
|
||||||
|
|
||||||
def pull_data(self):
|
def pull_data(self):
|
||||||
# pull data from each bay node
|
with docker_utils.docker_for_bay(self.context,
|
||||||
nodes = []
|
self.bay) as docker:
|
||||||
for node_addr in (self.bay.node_addresses + [self.bay.api_address]):
|
system_info = docker.info()
|
||||||
docker = self._docker_client(node_addr)
|
self.data['nodes'] = self._parse_node_info(system_info)
|
||||||
node_info = docker.info()
|
|
||||||
nodes.append(node_info)
|
|
||||||
self.data['nodes'] = nodes
|
|
||||||
|
|
||||||
# pull data from each container
|
# pull data from each container
|
||||||
containers = []
|
containers = []
|
||||||
docker = self._docker_swarm_client(self.bay)
|
for container in docker.containers(all=True):
|
||||||
for container in docker.containers(all=True):
|
try:
|
||||||
try:
|
container = docker.inspect_container(container['Id'])
|
||||||
container = docker.inspect_container(container['Id'])
|
except Exception as e:
|
||||||
except Exception as e:
|
LOG.warn(_LW("Ignore error [%(e)s] when inspecting "
|
||||||
LOG.warn(_LW("Ignore error [%(e)s] when inspecting container "
|
"container %(container_id)s."),
|
||||||
"%(container_id)s."),
|
{'e': e, 'container_id': container['Id']},
|
||||||
{'e': e, 'container_id': container['Id']},
|
exc_info=True)
|
||||||
exc_info=True)
|
containers.append(container)
|
||||||
containers.append(container)
|
self.data['containers'] = containers
|
||||||
self.data['containers'] = containers
|
|
||||||
|
|
||||||
def compute_memory_util(self):
|
def compute_memory_util(self):
|
||||||
mem_total = 0
|
mem_total = 0
|
||||||
|
@ -115,22 +112,49 @@ class SwarmMonitor(MonitorBase):
|
||||||
else:
|
else:
|
||||||
return mem_reserved * 100 / mem_total
|
return mem_reserved * 100 / mem_total
|
||||||
|
|
||||||
def _docker_client(self, api_address, port=2375):
|
def _parse_node_info(self, system_info):
|
||||||
tcp_url = 'tcp://%s:%s' % (api_address, port)
|
"""Parse system_info to retrieve memory size of each node.
|
||||||
return docker_client.DockerHTTPClient(
|
|
||||||
tcp_url,
|
|
||||||
CONF.docker.docker_remote_api_version,
|
|
||||||
CONF.docker.default_timeout
|
|
||||||
)
|
|
||||||
|
|
||||||
def _docker_swarm_client(self, bay):
|
:param system_info: The output returned by docker.info(). Example:
|
||||||
return self._docker_client(bay.api_address, port=2376)
|
{
|
||||||
|
u'Debug': False,
|
||||||
|
u'NEventsListener': 0,
|
||||||
|
u'DriverStatus': [
|
||||||
|
[u'\x08Strategy', u'spread'],
|
||||||
|
[u'\x08Filters', u'...'],
|
||||||
|
[u'\x08Nodes', u'2'],
|
||||||
|
[u'node1', u'10.0.0.4:2375'],
|
||||||
|
[u' \u2514 Containers', u'1'],
|
||||||
|
[u' \u2514 Reserved CPUs', u'0 / 1'],
|
||||||
|
[u' \u2514 Reserved Memory', u'0 B / 2.052 GiB'],
|
||||||
|
[u'node2', u'10.0.0.3:2375'],
|
||||||
|
[u' \u2514 Containers', u'2'],
|
||||||
|
[u' \u2514 Reserved CPUs', u'0 / 1'],
|
||||||
|
[u' \u2514 Reserved Memory', u'0 B / 2.052 GiB']
|
||||||
|
],
|
||||||
|
u'Containers': 3
|
||||||
|
}
|
||||||
|
:return: Memory size of each node. Excample:
|
||||||
|
[{'MemTotal': 2203318222.848},
|
||||||
|
{'MemTotal': 2203318222.848}]
|
||||||
|
"""
|
||||||
|
nodes = []
|
||||||
|
for info in system_info['DriverStatus']:
|
||||||
|
key = info[0]
|
||||||
|
value = info[1]
|
||||||
|
if key == u' \u2514 Reserved Memory':
|
||||||
|
memory = value # Example: '0 B / 2.052 GiB'
|
||||||
|
memory = memory.split('/')[1].strip() # Example: '2.052 GiB'
|
||||||
|
memory = memory.split(' ')[0] # Example: '2.052'
|
||||||
|
memory = float(memory) * 1024 * 1024 * 1024
|
||||||
|
nodes.append({'MemTotal': memory})
|
||||||
|
return nodes
|
||||||
|
|
||||||
|
|
||||||
def create_monitor(context, bay):
|
def create_monitor(context, bay):
|
||||||
baymodel = objects.BayModel.get_by_uuid(context, bay.baymodel_id)
|
baymodel = objects.BayModel.get_by_uuid(context, bay.baymodel_id)
|
||||||
if baymodel.coe == bay_type.SWARM:
|
if baymodel.coe == bay_type.SWARM:
|
||||||
return SwarmMonitor(bay)
|
return SwarmMonitor(context, bay)
|
||||||
|
|
||||||
# TODO(hongbin): add support for other bay types
|
# TODO(hongbin): add support for other bay types
|
||||||
LOG.debug("Cannot create monitor with bay type '%s'" % baymodel.coe)
|
LOG.debug("Cannot create monitor with bay type '%s'" % baymodel.coe)
|
||||||
|
|
|
@ -40,7 +40,7 @@ class MonitorsTestCase(base.TestCase):
|
||||||
bay = utils.get_test_bay(node_addresses=['1.2.3.4'],
|
bay = utils.get_test_bay(node_addresses=['1.2.3.4'],
|
||||||
api_address='5.6.7.8')
|
api_address='5.6.7.8')
|
||||||
self.bay = objects.Bay(self.context, **bay)
|
self.bay = objects.Bay(self.context, **bay)
|
||||||
self.monitor = monitors.SwarmMonitor(self.bay)
|
self.monitor = monitors.SwarmMonitor(self.context, self.bay)
|
||||||
p = mock.patch('magnum.conductor.monitors.SwarmMonitor.metrics_spec',
|
p = mock.patch('magnum.conductor.monitors.SwarmMonitor.metrics_spec',
|
||||||
new_callable=mock.PropertyMock)
|
new_callable=mock.PropertyMock)
|
||||||
self.mock_metrics_spec = p.start()
|
self.mock_metrics_spec = p.start()
|
||||||
|
@ -63,35 +63,35 @@ class MonitorsTestCase(base.TestCase):
|
||||||
monitor = monitors.create_monitor(self.context, self.bay)
|
monitor = monitors.create_monitor(self.context, self.bay)
|
||||||
self.assertIsNone(monitor)
|
self.assertIsNone(monitor)
|
||||||
|
|
||||||
@mock.patch('magnum.conductor.handlers.common.docker_client.'
|
@mock.patch('magnum.common.docker_utils.docker_for_bay')
|
||||||
'DockerHTTPClient')
|
def test_swarm_monitor_pull_data_success(self, mock_docker_for_bay):
|
||||||
def test_swarm_monitor_pull_data_success(self, mock_docker_http_client):
|
|
||||||
mock_docker = mock.MagicMock()
|
mock_docker = mock.MagicMock()
|
||||||
mock_docker.info.return_value = 'test_node'
|
mock_docker.info.return_value = {'DriverStatus': [[
|
||||||
|
u' \u2514 Reserved Memory', u'0 B / 1 GiB']]}
|
||||||
mock_docker.containers.return_value = [mock.MagicMock()]
|
mock_docker.containers.return_value = [mock.MagicMock()]
|
||||||
mock_docker.inspect_container.return_value = 'test_container'
|
mock_docker.inspect_container.return_value = 'test_container'
|
||||||
mock_docker_http_client.return_value = mock_docker
|
mock_docker_for_bay.return_value.__enter__.return_value = mock_docker
|
||||||
|
|
||||||
self.monitor.pull_data()
|
self.monitor.pull_data()
|
||||||
|
|
||||||
self.assertEqual(self.monitor.data['nodes'],
|
self.assertEqual(self.monitor.data['nodes'],
|
||||||
['test_node', 'test_node'])
|
[{'MemTotal': 1073741824.0}])
|
||||||
self.assertEqual(self.monitor.data['containers'], ['test_container'])
|
self.assertEqual(self.monitor.data['containers'], ['test_container'])
|
||||||
|
|
||||||
@mock.patch('magnum.conductor.handlers.common.docker_client.'
|
@mock.patch('magnum.common.docker_utils.docker_for_bay')
|
||||||
'DockerHTTPClient')
|
def test_swarm_monitor_pull_data_raise(self, mock_docker_for_bay):
|
||||||
def test_swarm_monitor_pull_data_raise(self, mock_docker_http_client):
|
|
||||||
mock_container = mock.MagicMock()
|
mock_container = mock.MagicMock()
|
||||||
mock_docker = mock.MagicMock()
|
mock_docker = mock.MagicMock()
|
||||||
mock_docker.info.return_value = 'test_node'
|
mock_docker.info.return_value = {'DriverStatus': [[
|
||||||
|
u' \u2514 Reserved Memory', u'0 B / 1 GiB']]}
|
||||||
mock_docker.containers.return_value = [mock_container]
|
mock_docker.containers.return_value = [mock_container]
|
||||||
mock_docker.inspect_container.side_effect = Exception("inspect error")
|
mock_docker.inspect_container.side_effect = Exception("inspect error")
|
||||||
mock_docker_http_client.return_value = mock_docker
|
mock_docker_for_bay.return_value.__enter__.return_value = mock_docker
|
||||||
|
|
||||||
self.monitor.pull_data()
|
self.monitor.pull_data()
|
||||||
|
|
||||||
self.assertEqual(self.monitor.data['nodes'],
|
self.assertEqual(self.monitor.data['nodes'],
|
||||||
['test_node', 'test_node'])
|
[{'MemTotal': 1073741824.0}])
|
||||||
self.assertEqual(self.monitor.data['containers'], [mock_container])
|
self.assertEqual(self.monitor.data['containers'], [mock_container])
|
||||||
|
|
||||||
def test_swarm_monitor_get_metric_names(self):
|
def test_swarm_monitor_get_metric_names(self):
|
||||||
|
|
Loading…
Reference in New Issue