# Copyright 2016 Canonical Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import sys import time import unittest from mock import patch, call, MagicMock import ceph.utils # python-apt is not installed as part of test-requirements but is imported by # some charmhelpers modules so create a fake import. mock_apt = MagicMock() sys.modules['apt'] = mock_apt mock_apt.apt_pkg = MagicMock() def config_side_effect(*args): if args[0] == 'source': return 'cloud:trusty-kilo' elif args[0] == 'key': return 'key' elif args[0] == 'release-version': return 'cloud:trusty-kilo' previous_node_start_time = time.time() - (9 * 60) def monitor_key_side_effect(*args): if args[1] == \ 'mon_ip-192-168-1-2_0.94.1_done': return False elif args[1] == \ 'mon_ip-192-168-1-2_0.94.1_start': # Return that the previous node started 9 minutes ago # NOTE(jamespage): # Pass back as string as this is what we actually get # from the monitor cluster return str(previous_node_start_time) class UpgradeRollingTestCase(unittest.TestCase): @patch('time.time') @patch.object(ceph.utils, 'log') @patch.object(ceph.utils, 'upgrade_monitor') @patch.object(ceph.utils, 'monitor_key_set') def test_lock_and_roll(self, monitor_key_set, upgrade_monitor, log, time): time.return_value = 1473279502.69 monitor_key_set.monitor_key_set.return_value = None ceph.utils.lock_and_roll(my_name='ip-192-168-1-2', version='hammer', service='mon', upgrade_key='admin') upgrade_monitor.assert_called_once_with('hammer') log.assert_has_calls( [ call('monitor_key_set ' 'mon_ip-192-168-1-2_hammer_start 1473279502.69'), call('Rolling'), call('Done'), call('monitor_key_set ' 'mon_ip-192-168-1-2_hammer_done 1473279502.69'), ]) @patch.object(ceph.utils, 'ceph_user') @patch.object(ceph.utils, 'socket') @patch.object(ceph.utils, 'mkdir') @patch.object(ceph.utils, 'apt_install') @patch.object(ceph.utils, 'chownr') @patch.object(ceph.utils, 'service_stop') @patch.object(ceph.utils, 'service_start') @patch.object(ceph.utils, 'log') @patch.object(ceph.utils, 'status_set') @patch.object(ceph.utils, 'apt_update') @patch.object(ceph.utils, 'add_source') @patch.object(ceph.utils, 'get_local_mon_ids') @patch.object(ceph.utils, 'systemd') @patch.object(ceph.utils, 'get_version') @patch.object(ceph.utils, 'config') def test_upgrade_monitor_hammer(self, config, get_version, systemd, local_mons, add_source, apt_update, status_set, log, service_start, service_stop, chownr, apt_install, mkdir, socket, ceph_user): get_version.side_effect = [0.80, 0.94] config.side_effect = config_side_effect systemd.return_value = False socket.gethostname.return_value = 'testmon' ceph_user.return_value = 'root' local_mons.return_value = ['a'] ceph.utils.upgrade_monitor('hammer') service_stop.assert_called_with('ceph-mon-all') service_start.assert_called_with('ceph-mon-all') add_source.assert_called_with('cloud:trusty-kilo', 'key') log.assert_has_calls( [ call('Current ceph version is 0.8'), call('Upgrading to: hammer') ] ) status_set.assert_has_calls([ call('maintenance', 'Upgrading monitor'), ]) mkdir.assert_called_with('/var/lib/ceph/mon/ceph-testmon', owner='root', group='root', perms=0o755) chownr.assert_not_called() @patch.object(ceph.utils, 'ceph_user') @patch.object(ceph.utils, 'socket') @patch.object(ceph.utils, 'mkdir') @patch.object(ceph.utils, 'apt_install') @patch.object(ceph.utils, 'chownr') @patch.object(ceph.utils, 'service_stop') @patch.object(ceph.utils, 'service_start') @patch.object(ceph.utils, 'log') @patch.object(ceph.utils, 'status_set') @patch.object(ceph.utils, 'apt_update') @patch.object(ceph.utils, 'add_source') @patch.object(ceph.utils, 'get_local_mon_ids') @patch.object(ceph.utils, 'systemd') @patch.object(ceph.utils, 'get_version') @patch.object(ceph.utils, 'config') def test_upgrade_monitor_jewel(self, config, get_version, systemd, local_mons, add_source, apt_update, status_set, log, service_start, service_stop, chownr, apt_install, mkdir, socket, ceph_user): get_version.side_effect = [0.94, 10.1] config.side_effect = config_side_effect systemd.return_value = False socket.gethostname.return_value = 'testmon' ceph_user.return_value = 'ceph' local_mons.return_value = ['a'] ceph.utils.upgrade_monitor('jewel') service_stop.assert_called_with('ceph-mon-all') service_start.assert_called_with('ceph-mon-all') add_source.assert_called_with('cloud:trusty-kilo', 'key') log.assert_has_calls( [ call('Current ceph version is 0.94'), call('Upgrading to: jewel') ] ) status_set.assert_has_calls([ call('maintenance', 'Upgrading monitor'), ]) chownr.assert_has_calls( [ call(group='ceph', owner='ceph', path='/var/lib/ceph', follow_links=True) ] ) mkdir.assert_called_with('/var/lib/ceph/mon/ceph-testmon', owner='ceph', group='ceph', perms=0o755) @patch.object(ceph.utils, 'ceph_user') @patch.object(ceph.utils, 'socket') @patch.object(ceph.utils, 'mkdir') @patch.object(ceph.utils, 'apt_install') @patch.object(ceph.utils, 'chownr') @patch.object(ceph.utils, 'service_stop') @patch.object(ceph.utils, 'service_start') @patch.object(ceph.utils, 'log') @patch.object(ceph.utils, 'status_set') @patch.object(ceph.utils, 'apt_update') @patch.object(ceph.utils, 'add_source') @patch.object(ceph.utils, 'systemd') @patch.object(ceph.utils, 'get_version') @patch.object(ceph.utils, 'config') def test_upgrade_monitor_luminous(self, config, get_version, systemd, add_source, apt_update, status_set, log, service_start, service_stop, chownr, apt_install, mkdir, socket, ceph_user): get_version.side_effect = [10.2, 12.2] config.side_effect = config_side_effect socket.gethostname.return_value = 'testmon' ceph_user.return_value = 'ceph' systemd.return_value = True ceph.utils.upgrade_monitor('luminous') service_stop.assert_called_with('ceph-mon') service_start.assert_called_with('ceph-mon') add_source.assert_called_with('cloud:trusty-kilo', 'key') log.assert_has_calls( [ call('Current ceph version is 10.2'), call('Upgrading to: luminous') ] ) status_set.assert_has_calls([ call('maintenance', 'Upgrading monitor'), ]) chownr.assert_not_called() mkdir.assert_called_with('/var/lib/ceph/mon/ceph-testmon', owner='ceph', group='ceph', perms=0o755) @patch.object(ceph.utils, 'bootstrap_manager') @patch.object(ceph.utils, 'wait_for_all_monitors_to_upgrade') @patch.object(ceph.utils, 'status_set') @patch.object(ceph.utils, 'lock_and_roll') @patch.object(ceph.utils, 'wait_on_previous_node') @patch.object(ceph.utils, 'get_mon_map') @patch.object(ceph.utils, 'socket') def _test_roll_monitor_cluster(self, socket, get_mon_map, wait_on_previous_node, lock_and_roll, status_set, wait_for_all_monitors_to_upgrade, bootstrap_manager, new_version): socket.gethostname.return_value = "ip-192-168-1-3" get_mon_map.return_value = { 'monmap': { 'mons': [ { 'name': 'ip-192-168-1-2', }, { 'name': 'ip-192-168-1-3', }, ] } } ceph.utils.roll_monitor_cluster(new_version=new_version, upgrade_key='admin') get_mon_map.assert_called_once_with('admin') wait_on_previous_node.assert_called_with( upgrade_key='admin', service='mon', previous_node='ip-192-168-1-2', version=new_version, ) status_set.assert_called_with( 'waiting', 'Waiting on ip-192-168-1-2 to finish upgrading') lock_and_roll.assert_called_with(my_name='ip-192-168-1-3', service='mon', upgrade_key='admin', version=new_version) if new_version == 'luminous': wait_for_all_monitors_to_upgrade.assert_called_with( new_version=new_version, upgrade_key='admin', ) bootstrap_manager.assert_called_once_with() else: wait_for_all_monitors_to_upgrade.assert_not_called() bootstrap_manager.assert_not_called() def test_roll_monitor_cluster_luminous(self): self._test_roll_monitor_cluster(new_version='luminous') def test_roll_monitor_cluster_jewel(self): self._test_roll_monitor_cluster(new_version='jewel') def test_roll_monitor_cluster_hammer(self): self._test_roll_monitor_cluster(new_version='hammer') @patch.object(ceph.utils, 'log') @patch.object(ceph.utils, 'time') @patch.object(ceph.utils, 'monitor_key_get') @patch.object(ceph.utils, 'monitor_key_exists') def test_wait_on_previous_node(self, monitor_key_exists, monitor_key_get, mock_time, log): tval = [previous_node_start_time] def fake_time(): tval[0] += 100 return tval[0] mock_time.time.side_effect = fake_time monitor_key_get.side_effect = monitor_key_side_effect monitor_key_exists.return_value = False ceph.utils.wait_on_previous_node(previous_node="ip-192-168-1-2", version='0.94.1', service='mon', upgrade_key='admin') # Make sure we checked to see if the previous node started monitor_key_get.assert_has_calls( [call('admin', 'mon_ip-192-168-1-2_0.94.1_start')] ) # Make sure we checked to see if the previous node was finished monitor_key_exists.assert_has_calls( [call('admin', 'mon_ip-192-168-1-2_0.94.1_done')] ) # Make sure we waited at last once before proceeding log.assert_has_calls( [call('Previous node is: ip-192-168-1-2')], [call('ip-192-168-1-2 is not finished. Waiting')], ) self.assertGreaterEqual(tval[0], previous_node_start_time + 600)