From d95f400c63d3706d95fe1ba6f388a5adc870a9ae Mon Sep 17 00:00:00 2001 From: Hongbin Lu Date: Mon, 30 Nov 2015 15:42:17 -0500 Subject: [PATCH] Fix mesos monitor for handling multiple masters Magnum is going to enable HA mode for mesos bay, in which there could be more than one master nodes. One of the master node (leader) is active and the rest of them is not active. Mesos monitor needs to identify the leader and pull metrics from the leader only. Change-Id: Ic7d6cb64964db2758163bd231dec4190302155b2 Partial-Implements: blueprint mesos-multi-master-node --- magnum/conductor/mesos_monitor.py | 19 ++++++---- magnum/tests/unit/conductor/test_monitors.py | 38 +++++++++++++++----- 2 files changed, 42 insertions(+), 15 deletions(-) diff --git a/magnum/conductor/mesos_monitor.py b/magnum/conductor/mesos_monitor.py index d118a1f5e2..21625d56dd 100644 --- a/magnum/conductor/mesos_monitor.py +++ b/magnum/conductor/mesos_monitor.py @@ -34,16 +34,21 @@ class MesosMonitor(MonitorBase): def _build_url(self, url, protocol='http', port='80', path='/'): return protocol + '://' + url + ':' + port + path + def _is_leader(self, state): + return state['leader'] == state['pid'] + def pull_data(self): - mesos_master_url = self._build_url(self.bay.api_address, - port='5050', - path='/state') - state_json = jsonutils.loads(urlfetch.get(mesos_master_url)) self.data['mem_total'] = 0 self.data['mem_used'] = 0 - for slave in state_json['slaves']: - self.data['mem_total'] += slave['resources']['mem'] - self.data['mem_used'] += slave['used_resources']['mem'] + for master_addr in self.bay.master_addresses: + mesos_master_url = self._build_url(master_addr, port='5050', + path='/state') + master = jsonutils.loads(urlfetch.get(mesos_master_url)) + if self._is_leader(master): + for slave in master['slaves']: + self.data['mem_total'] += slave['resources']['mem'] + self.data['mem_used'] += slave['used_resources']['mem'] + break def compute_memory_util(self): if self.data['mem_total'] == 0: diff --git a/magnum/tests/unit/conductor/test_monitors.py b/magnum/tests/unit/conductor/test_monitors.py index 1fa1025310..0b9289b47b 100644 --- a/magnum/tests/unit/conductor/test_monitors.py +++ b/magnum/tests/unit/conductor/test_monitors.py @@ -43,7 +43,8 @@ class MonitorsTestCase(base.TestCase): super(MonitorsTestCase, self).setUp() bay = utils.get_test_bay(node_addresses=['1.2.3.4'], - api_address='https://5.6.7.8:2376') + api_address='https://5.6.7.8:2376', + master_addresses=['10.0.0.6']) self.bay = objects.Bay(self.context, **bay) self.monitor = swarm_monitor.SwarmMonitor(self.context, self.bay) self.k8s_monitor = k8s_monitor.K8sMonitor(self.context, self.bay) @@ -230,9 +231,22 @@ class MonitorsTestCase(base.TestCase): mem_util = self.k8s_monitor.compute_memory_util() self.assertEqual(0, mem_util) + def _test_mesos_monitor_pull_data( + self, mock_url_get, state_json, expected_mem_total, + expected_mem_used): + state_json = jsonutils.dumps(state_json) + mock_url_get.return_value = state_json + self.mesos_monitor.pull_data() + self.assertEqual(self.mesos_monitor.data['mem_total'], + expected_mem_total) + self.assertEqual(self.mesos_monitor.data['mem_used'], + expected_mem_used) + @mock.patch('magnum.common.urlfetch.get') def test_mesos_monitor_pull_data_success(self, mock_url_get): state_json = { + 'leader': 'master@10.0.0.6:5050', + 'pid': 'master@10.0.0.6:5050', 'slaves': [{ 'resources': { 'mem': 100 @@ -242,13 +256,21 @@ class MonitorsTestCase(base.TestCase): } }] } - state_json = jsonutils.dumps(state_json) - mock_url_get.return_value = state_json - self.mesos_monitor.pull_data() - self.assertEqual(self.mesos_monitor.data['mem_total'], - 100) - self.assertEqual(self.mesos_monitor.data['mem_used'], - 50) + self._test_mesos_monitor_pull_data(mock_url_get, state_json, 100, 50) + + @mock.patch('magnum.common.urlfetch.get') + def test_mesos_monitor_pull_data_success_not_leader(self, mock_url_get): + state_json = { + 'leader': 'master@10.0.0.6:5050', + 'pid': 'master@1.1.1.1:5050', + 'slaves': [] + } + self._test_mesos_monitor_pull_data(mock_url_get, state_json, 0, 0) + + @mock.patch('magnum.common.urlfetch.get') + def test_mesos_monitor_pull_data_success_no_master(self, mock_url_get): + self.bay.master_addresses = [] + self._test_mesos_monitor_pull_data(mock_url_get, {}, 0, 0) def test_mesos_monitor_get_metric_names(self): mesos_metric_spec = 'magnum.conductor.mesos_monitor.MesosMonitor.'\