Helpers for writing Ceph OpenStack Charms
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

test_mon_upgrade_roll.py 13KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324
  1. # Copyright 2016 Canonical Ltd
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import sys
  15. import time
  16. import unittest
  17. from mock import patch, call, MagicMock
  18. import ceph.utils
  19. # python-apt is not installed as part of test-requirements but is imported by
  20. # some charmhelpers modules so create a fake import.
  21. mock_apt = MagicMock()
  22. sys.modules['apt'] = mock_apt
  23. mock_apt.apt_pkg = MagicMock()
  24. def config_side_effect(*args):
  25. if args[0] == 'source':
  26. return 'cloud:trusty-kilo'
  27. elif args[0] == 'key':
  28. return 'key'
  29. elif args[0] == 'release-version':
  30. return 'cloud:trusty-kilo'
  31. previous_node_start_time = time.time() - (9 * 60)
  32. def monitor_key_side_effect(*args):
  33. if args[1] == \
  34. 'mon_ip-192-168-1-2_0.94.1_done':
  35. return False
  36. elif args[1] == \
  37. 'mon_ip-192-168-1-2_0.94.1_start':
  38. # Return that the previous node started 9 minutes ago
  39. # NOTE(jamespage):
  40. # Pass back as string as this is what we actually get
  41. # from the monitor cluster
  42. return str(previous_node_start_time)
  43. class UpgradeRollingTestCase(unittest.TestCase):
  44. @patch('time.time')
  45. @patch.object(ceph.utils, 'log')
  46. @patch.object(ceph.utils, 'upgrade_monitor')
  47. @patch.object(ceph.utils, 'monitor_key_set')
  48. def test_lock_and_roll(self, monitor_key_set, upgrade_monitor, log, time):
  49. time.return_value = 1473279502.69
  50. monitor_key_set.monitor_key_set.return_value = None
  51. ceph.utils.lock_and_roll(my_name='ip-192-168-1-2',
  52. version='hammer',
  53. service='mon',
  54. upgrade_key='admin')
  55. upgrade_monitor.assert_called_once_with('hammer')
  56. log.assert_has_calls(
  57. [
  58. call('monitor_key_set '
  59. 'mon_ip-192-168-1-2_hammer_start 1473279502.69'),
  60. call('Rolling'),
  61. call('Done'),
  62. call('monitor_key_set '
  63. 'mon_ip-192-168-1-2_hammer_done 1473279502.69'),
  64. ])
  65. @patch.object(ceph.utils, 'ceph_user')
  66. @patch.object(ceph.utils, 'socket')
  67. @patch.object(ceph.utils, 'mkdir')
  68. @patch.object(ceph.utils, 'apt_install')
  69. @patch.object(ceph.utils, 'chownr')
  70. @patch.object(ceph.utils, 'service_stop')
  71. @patch.object(ceph.utils, 'service_start')
  72. @patch.object(ceph.utils, 'log')
  73. @patch.object(ceph.utils, 'status_set')
  74. @patch.object(ceph.utils, 'apt_update')
  75. @patch.object(ceph.utils, 'add_source')
  76. @patch.object(ceph.utils, 'get_local_mon_ids')
  77. @patch.object(ceph.utils, 'systemd')
  78. @patch.object(ceph.utils, 'get_version')
  79. @patch.object(ceph.utils, 'config')
  80. def test_upgrade_monitor_hammer(self, config, get_version,
  81. systemd, local_mons, add_source,
  82. apt_update, status_set, log,
  83. service_start, service_stop, chownr,
  84. apt_install, mkdir, socket,
  85. ceph_user):
  86. get_version.side_effect = [0.80, 0.94]
  87. config.side_effect = config_side_effect
  88. systemd.return_value = False
  89. socket.gethostname.return_value = 'testmon'
  90. ceph_user.return_value = 'root'
  91. local_mons.return_value = ['a']
  92. ceph.utils.upgrade_monitor('hammer')
  93. service_stop.assert_called_with('ceph-mon-all')
  94. service_start.assert_called_with('ceph-mon-all')
  95. add_source.assert_called_with('cloud:trusty-kilo', 'key')
  96. log.assert_has_calls(
  97. [
  98. call('Current ceph version is 0.8'),
  99. call('Upgrading to: hammer')
  100. ]
  101. )
  102. status_set.assert_has_calls([
  103. call('maintenance', 'Upgrading monitor'),
  104. ])
  105. mkdir.assert_called_with('/var/lib/ceph/mon/ceph-testmon',
  106. owner='root',
  107. group='root',
  108. perms=0o755)
  109. chownr.assert_not_called()
  110. @patch.object(ceph.utils, 'ceph_user')
  111. @patch.object(ceph.utils, 'socket')
  112. @patch.object(ceph.utils, 'mkdir')
  113. @patch.object(ceph.utils, 'apt_install')
  114. @patch.object(ceph.utils, 'chownr')
  115. @patch.object(ceph.utils, 'service_stop')
  116. @patch.object(ceph.utils, 'service_start')
  117. @patch.object(ceph.utils, 'log')
  118. @patch.object(ceph.utils, 'status_set')
  119. @patch.object(ceph.utils, 'apt_update')
  120. @patch.object(ceph.utils, 'add_source')
  121. @patch.object(ceph.utils, 'get_local_mon_ids')
  122. @patch.object(ceph.utils, 'systemd')
  123. @patch.object(ceph.utils, 'get_version')
  124. @patch.object(ceph.utils, 'config')
  125. def test_upgrade_monitor_jewel(self, config, get_version,
  126. systemd, local_mons, add_source,
  127. apt_update, status_set, log,
  128. service_start, service_stop, chownr,
  129. apt_install, mkdir, socket,
  130. ceph_user):
  131. get_version.side_effect = [0.94, 10.1]
  132. config.side_effect = config_side_effect
  133. systemd.return_value = False
  134. socket.gethostname.return_value = 'testmon'
  135. ceph_user.return_value = 'ceph'
  136. local_mons.return_value = ['a']
  137. ceph.utils.upgrade_monitor('jewel')
  138. service_stop.assert_called_with('ceph-mon-all')
  139. service_start.assert_called_with('ceph-mon-all')
  140. add_source.assert_called_with('cloud:trusty-kilo', 'key')
  141. log.assert_has_calls(
  142. [
  143. call('Current ceph version is 0.94'),
  144. call('Upgrading to: jewel')
  145. ]
  146. )
  147. status_set.assert_has_calls([
  148. call('maintenance', 'Upgrading monitor'),
  149. ])
  150. chownr.assert_has_calls(
  151. [
  152. call(group='ceph', owner='ceph', path='/var/lib/ceph',
  153. follow_links=True)
  154. ]
  155. )
  156. mkdir.assert_called_with('/var/lib/ceph/mon/ceph-testmon',
  157. owner='ceph',
  158. group='ceph',
  159. perms=0o755)
  160. @patch.object(ceph.utils, 'ceph_user')
  161. @patch.object(ceph.utils, 'socket')
  162. @patch.object(ceph.utils, 'mkdir')
  163. @patch.object(ceph.utils, 'apt_install')
  164. @patch.object(ceph.utils, 'chownr')
  165. @patch.object(ceph.utils, 'service_stop')
  166. @patch.object(ceph.utils, 'service_start')
  167. @patch.object(ceph.utils, 'log')
  168. @patch.object(ceph.utils, 'status_set')
  169. @patch.object(ceph.utils, 'apt_update')
  170. @patch.object(ceph.utils, 'add_source')
  171. @patch.object(ceph.utils, 'systemd')
  172. @patch.object(ceph.utils, 'get_version')
  173. @patch.object(ceph.utils, 'config')
  174. def test_upgrade_monitor_luminous(self, config, get_version,
  175. systemd, add_source,
  176. apt_update, status_set, log,
  177. service_start, service_stop, chownr,
  178. apt_install, mkdir, socket,
  179. ceph_user):
  180. get_version.side_effect = [10.2, 12.2]
  181. config.side_effect = config_side_effect
  182. socket.gethostname.return_value = 'testmon'
  183. ceph_user.return_value = 'ceph'
  184. systemd.return_value = True
  185. ceph.utils.upgrade_monitor('luminous')
  186. service_stop.assert_called_with('ceph-mon')
  187. service_start.assert_called_with('ceph-mon')
  188. add_source.assert_called_with('cloud:trusty-kilo', 'key')
  189. log.assert_has_calls(
  190. [
  191. call('Current ceph version is 10.2'),
  192. call('Upgrading to: luminous')
  193. ]
  194. )
  195. status_set.assert_has_calls([
  196. call('maintenance', 'Upgrading monitor'),
  197. ])
  198. chownr.assert_not_called()
  199. mkdir.assert_called_with('/var/lib/ceph/mon/ceph-testmon',
  200. owner='ceph',
  201. group='ceph',
  202. perms=0o755)
  203. @patch.object(ceph.utils, 'bootstrap_manager')
  204. @patch.object(ceph.utils, 'wait_for_all_monitors_to_upgrade')
  205. @patch.object(ceph.utils, 'status_set')
  206. @patch.object(ceph.utils, 'lock_and_roll')
  207. @patch.object(ceph.utils, 'wait_on_previous_node')
  208. @patch.object(ceph.utils, 'get_mon_map')
  209. @patch.object(ceph.utils, 'socket')
  210. def _test_roll_monitor_cluster(self,
  211. socket,
  212. get_mon_map,
  213. wait_on_previous_node,
  214. lock_and_roll,
  215. status_set,
  216. wait_for_all_monitors_to_upgrade,
  217. bootstrap_manager,
  218. new_version):
  219. socket.gethostname.return_value = "ip-192-168-1-3"
  220. get_mon_map.return_value = {
  221. 'monmap': {
  222. 'mons': [
  223. {
  224. 'name': 'ip-192-168-1-2',
  225. },
  226. {
  227. 'name': 'ip-192-168-1-3',
  228. },
  229. ]
  230. }
  231. }
  232. ceph.utils.roll_monitor_cluster(new_version=new_version,
  233. upgrade_key='admin')
  234. get_mon_map.assert_called_once_with('admin')
  235. wait_on_previous_node.assert_called_with(
  236. upgrade_key='admin',
  237. service='mon',
  238. previous_node='ip-192-168-1-2',
  239. version=new_version,
  240. )
  241. status_set.assert_called_with(
  242. 'waiting',
  243. 'Waiting on ip-192-168-1-2 to finish upgrading')
  244. lock_and_roll.assert_called_with(my_name='ip-192-168-1-3',
  245. service='mon',
  246. upgrade_key='admin',
  247. version=new_version)
  248. if new_version == 'luminous':
  249. wait_for_all_monitors_to_upgrade.assert_called_with(
  250. new_version=new_version,
  251. upgrade_key='admin',
  252. )
  253. bootstrap_manager.assert_called_once_with()
  254. else:
  255. wait_for_all_monitors_to_upgrade.assert_not_called()
  256. bootstrap_manager.assert_not_called()
  257. def test_roll_monitor_cluster_luminous(self):
  258. self._test_roll_monitor_cluster(new_version='luminous')
  259. def test_roll_monitor_cluster_jewel(self):
  260. self._test_roll_monitor_cluster(new_version='jewel')
  261. def test_roll_monitor_cluster_hammer(self):
  262. self._test_roll_monitor_cluster(new_version='hammer')
  263. @patch.object(ceph.utils, 'log')
  264. @patch.object(ceph.utils, 'time')
  265. @patch.object(ceph.utils, 'monitor_key_get')
  266. @patch.object(ceph.utils, 'monitor_key_exists')
  267. def test_wait_on_previous_node(self, monitor_key_exists, monitor_key_get,
  268. mock_time, log):
  269. tval = [previous_node_start_time]
  270. def fake_time():
  271. tval[0] += 100
  272. return tval[0]
  273. mock_time.time.side_effect = fake_time
  274. monitor_key_get.side_effect = monitor_key_side_effect
  275. monitor_key_exists.return_value = False
  276. ceph.utils.wait_on_previous_node(previous_node="ip-192-168-1-2",
  277. version='0.94.1',
  278. service='mon',
  279. upgrade_key='admin')
  280. # Make sure we checked to see if the previous node started
  281. monitor_key_get.assert_has_calls(
  282. [call('admin', 'mon_ip-192-168-1-2_0.94.1_start')]
  283. )
  284. # Make sure we checked to see if the previous node was finished
  285. monitor_key_exists.assert_has_calls(
  286. [call('admin', 'mon_ip-192-168-1-2_0.94.1_done')]
  287. )
  288. # Make sure we waited at last once before proceeding
  289. log.assert_has_calls(
  290. [call('Previous node is: ip-192-168-1-2')],
  291. [call('ip-192-168-1-2 is not finished. Waiting')],
  292. )
  293. self.assertGreaterEqual(tval[0], previous_node_start_time + 600)