Juju Charm - Ceph OSD
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

ceph_hooks.py 21KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650
  1. #!/usr/bin/env python3
  2. #
  3. # Copyright 2016 Canonical Ltd
  4. #
  5. # Licensed under the Apache License, Version 2.0 (the "License");
  6. # you may not use this file except in compliance with the License.
  7. # You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. import base64
  17. import json
  18. import glob
  19. import os
  20. import shutil
  21. import sys
  22. import socket
  23. import subprocess
  24. import netifaces
  25. sys.path.append('lib')
  26. import ceph.utils as ceph
  27. from charmhelpers.core import hookenv
  28. from charmhelpers.core.hookenv import (
  29. log,
  30. DEBUG,
  31. ERROR,
  32. INFO,
  33. config,
  34. relation_ids,
  35. related_units,
  36. relation_get,
  37. relation_set,
  38. Hooks,
  39. UnregisteredHookError,
  40. service_name,
  41. status_get,
  42. status_set,
  43. storage_get,
  44. storage_list,
  45. application_version_set,
  46. )
  47. from charmhelpers.core.host import (
  48. umount,
  49. mkdir,
  50. cmp_pkgrevno,
  51. service_reload,
  52. service_restart,
  53. add_to_updatedb_prunepath,
  54. restart_on_change,
  55. write_file,
  56. is_container,
  57. )
  58. from charmhelpers.fetch import (
  59. add_source,
  60. apt_install,
  61. apt_update,
  62. filter_installed_packages,
  63. get_upstream_version,
  64. )
  65. from charmhelpers.core.sysctl import create as create_sysctl
  66. from charmhelpers.contrib.openstack.context import (
  67. AppArmorContext,
  68. )
  69. from utils import (
  70. get_host_ip,
  71. get_networks,
  72. assert_charm_supports_ipv6,
  73. render_template,
  74. is_unit_paused_set,
  75. get_public_addr,
  76. get_cluster_addr,
  77. get_blacklist,
  78. get_journal_devices,
  79. )
  80. from charmhelpers.contrib.openstack.alternatives import install_alternative
  81. from charmhelpers.contrib.network.ip import (
  82. get_ipv6_addr,
  83. format_ipv6_addr,
  84. get_relation_ip,
  85. )
  86. from charmhelpers.contrib.storage.linux.ceph import (
  87. CephConfContext)
  88. from charmhelpers.contrib.storage.linux.utils import (
  89. is_device_mounted,
  90. )
  91. from charmhelpers.contrib.charmsupport import nrpe
  92. from charmhelpers.contrib.hardening.harden import harden
  93. from charmhelpers.core.unitdata import kv
  94. import charmhelpers.contrib.openstack.vaultlocker as vaultlocker
  95. hooks = Hooks()
  96. STORAGE_MOUNT_PATH = '/var/lib/ceph'
  97. def check_for_upgrade():
  98. if not os.path.exists(ceph._upgrade_keyring):
  99. log("Ceph upgrade keyring not detected, skipping upgrade checks.")
  100. return
  101. c = hookenv.config()
  102. old_version = ceph.resolve_ceph_version(c.previous('source') or
  103. 'distro')
  104. log('old_version: {}'.format(old_version))
  105. new_version = ceph.resolve_ceph_version(hookenv.config('source') or
  106. 'distro')
  107. log('new_version: {}'.format(new_version))
  108. # May be in a previous upgrade that was failed if the directories
  109. # still need an ownership update. Check this condition.
  110. resuming_upgrade = ceph.dirs_need_ownership_update('osd')
  111. if old_version == new_version and not resuming_upgrade:
  112. log("No new ceph version detected, skipping upgrade.", DEBUG)
  113. return
  114. if (ceph.UPGRADE_PATHS.get(old_version) == new_version) or\
  115. resuming_upgrade:
  116. if old_version == new_version:
  117. log('Attempting to resume possibly failed upgrade.',
  118. INFO)
  119. else:
  120. log("{} to {} is a valid upgrade path. Proceeding.".format(
  121. old_version, new_version))
  122. emit_cephconf(upgrading=True)
  123. ceph.roll_osd_cluster(new_version=new_version,
  124. upgrade_key='osd-upgrade')
  125. emit_cephconf(upgrading=False)
  126. else:
  127. # Log a helpful error message
  128. log("Invalid upgrade path from {} to {}. "
  129. "Valid paths are: {}".format(old_version,
  130. new_version,
  131. ceph.pretty_print_upgrade_paths()))
  132. def tune_network_adapters():
  133. interfaces = netifaces.interfaces()
  134. for interface in interfaces:
  135. if interface == "lo":
  136. # Skip the loopback
  137. continue
  138. log("Looking up {} for possible sysctl tuning.".format(interface))
  139. ceph.tune_nic(interface)
  140. @restart_on_change({'/etc/apparmor.d/usr.bin.ceph-osd': ['apparmor']},
  141. restart_functions={'apparmor': service_reload})
  142. def copy_profile_into_place():
  143. """
  144. Copy the apparmor profiles included with the charm
  145. into the /etc/apparmor.d directory.
  146. """
  147. new_install = False
  148. apparmor_dir = os.path.join(os.sep,
  149. 'etc',
  150. 'apparmor.d')
  151. for x in glob.glob('files/apparmor/*'):
  152. if not os.path.exists(os.path.join(apparmor_dir,
  153. os.path.basename(x))):
  154. new_install = True
  155. shutil.copy(x, apparmor_dir)
  156. return new_install
  157. class CephOsdAppArmorContext(AppArmorContext):
  158. """"Apparmor context for ceph-osd binary"""
  159. def __init__(self):
  160. super(CephOsdAppArmorContext, self).__init__()
  161. self.aa_profile = 'usr.bin.ceph-osd'
  162. def __call__(self):
  163. super(CephOsdAppArmorContext, self).__call__()
  164. if not self.ctxt:
  165. return self.ctxt
  166. self._ctxt.update({'aa_profile': self.aa_profile})
  167. return self.ctxt
  168. def use_vaultlocker():
  169. """Determine whether vaultlocker should be used for OSD encryption
  170. :returns: whether vaultlocker should be used for key management
  171. :rtype: bool
  172. :raises: ValueError if vaultlocker is enable but ceph < 12.2.4"""
  173. if (config('osd-encrypt') and
  174. config('osd-encrypt-keymanager') == ceph.VAULT_KEY_MANAGER):
  175. if cmp_pkgrevno('ceph', '12.2.4') < 0:
  176. msg = ('vault usage only supported with ceph >= 12.2.4')
  177. status_set('blocked', msg)
  178. raise ValueError(msg)
  179. else:
  180. return True
  181. return False
  182. def install_apparmor_profile():
  183. """
  184. Install ceph apparmor profiles and configure
  185. based on current setting of 'aa-profile-mode'
  186. configuration option.
  187. """
  188. log('Installing apparmor profile for ceph-osd')
  189. new_install = copy_profile_into_place()
  190. if new_install or config().changed('aa-profile-mode'):
  191. aa_context = CephOsdAppArmorContext()
  192. aa_context.setup_aa_profile()
  193. service_reload('apparmor')
  194. if ceph.systemd():
  195. for osd_id in ceph.get_local_osd_ids():
  196. service_restart('ceph-osd@{}'.format(osd_id))
  197. else:
  198. service_restart('ceph-osd-all')
  199. def install_udev_rules():
  200. """
  201. Install and reload udev rules for ceph-volume LV
  202. permissions
  203. """
  204. if is_container():
  205. log('Skipping udev rule installation '
  206. 'as unit is in a container', level=DEBUG)
  207. return
  208. for x in glob.glob('files/udev/*'):
  209. shutil.copy(x, '/lib/udev/rules.d')
  210. subprocess.check_call(['udevadm', 'control',
  211. '--reload-rules'])
  212. @hooks.hook('install.real')
  213. @harden()
  214. def install():
  215. add_source(config('source'), config('key'))
  216. apt_update(fatal=True)
  217. apt_install(packages=ceph.determine_packages(), fatal=True)
  218. if config('autotune'):
  219. tune_network_adapters()
  220. install_udev_rules()
  221. def az_info():
  222. az_info = ""
  223. config_az = config("availability_zone")
  224. juju_az_info = os.environ.get('JUJU_AVAILABILITY_ZONE')
  225. if juju_az_info:
  226. # NOTE(jamespage): avoid conflicting key with root
  227. # of crush hierarchy
  228. if juju_az_info == 'default':
  229. juju_az_info = 'default-rack'
  230. az_info = "{} rack={}".format(az_info, juju_az_info)
  231. if config_az:
  232. # NOTE(jamespage): avoid conflicting key with root
  233. # of crush hierarchy
  234. if config_az == 'default':
  235. config_az = 'default-row'
  236. az_info = "{} row={}".format(az_info, config_az)
  237. if az_info != "":
  238. log("AZ Info: " + az_info)
  239. return az_info
  240. def use_short_objects():
  241. '''
  242. Determine whether OSD's should be configured with
  243. limited object name lengths.
  244. @return: boolean indicating whether OSD's should be limited
  245. '''
  246. if cmp_pkgrevno('ceph', "10.2.0") >= 0:
  247. if config('osd-format') in ('ext4'):
  248. return True
  249. devices = config('osd-devices')
  250. if not devices:
  251. return False
  252. for device in devices.split():
  253. if device and not device.startswith('/dev'):
  254. # TODO: determine format of directory based
  255. # OSD location
  256. return True
  257. return False
  258. def get_ceph_context(upgrading=False):
  259. """Returns the current context dictionary for generating ceph.conf
  260. :param upgrading: bool - determines if the context is invoked as
  261. part of an upgrade proedure Setting this to true
  262. causes settings useful during an upgrade to be
  263. defined in the ceph.conf file
  264. """
  265. mon_hosts = get_mon_hosts()
  266. log('Monitor hosts are ' + repr(mon_hosts))
  267. networks = get_networks('ceph-public-network')
  268. public_network = ', '.join(networks)
  269. networks = get_networks('ceph-cluster-network')
  270. cluster_network = ', '.join(networks)
  271. cephcontext = {
  272. 'auth_supported': get_auth(),
  273. 'mon_hosts': ' '.join(mon_hosts),
  274. 'fsid': get_fsid(),
  275. 'old_auth': cmp_pkgrevno('ceph', "0.51") < 0,
  276. 'crush_initial_weight': config('crush-initial-weight'),
  277. 'osd_journal_size': config('osd-journal-size'),
  278. 'osd_max_backfills': config('osd-max-backfills'),
  279. 'osd_recovery_max_active': config('osd-recovery-max-active'),
  280. 'use_syslog': str(config('use-syslog')).lower(),
  281. 'ceph_public_network': public_network,
  282. 'ceph_cluster_network': cluster_network,
  283. 'loglevel': config('loglevel'),
  284. 'dio': str(config('use-direct-io')).lower(),
  285. 'short_object_len': use_short_objects(),
  286. 'upgrade_in_progress': upgrading,
  287. 'bluestore': config('bluestore'),
  288. 'bluestore_experimental': cmp_pkgrevno('ceph', '12.1.0') < 0,
  289. 'bluestore_block_wal_size': config('bluestore-block-wal-size'),
  290. 'bluestore_block_db_size': config('bluestore-block-db-size'),
  291. }
  292. if config('prefer-ipv6'):
  293. dynamic_ipv6_address = get_ipv6_addr()[0]
  294. if not public_network:
  295. cephcontext['public_addr'] = dynamic_ipv6_address
  296. if not cluster_network:
  297. cephcontext['cluster_addr'] = dynamic_ipv6_address
  298. else:
  299. cephcontext['public_addr'] = get_public_addr()
  300. cephcontext['cluster_addr'] = get_cluster_addr()
  301. if config('customize-failure-domain'):
  302. az = az_info()
  303. if az:
  304. cephcontext['crush_location'] = "root=default {} host={}" \
  305. .format(az, socket.gethostname())
  306. else:
  307. log(
  308. "Your Juju environment doesn't"
  309. "have support for Availability Zones"
  310. )
  311. # NOTE(dosaboy): these sections must correspond to what is supported in the
  312. # config template.
  313. sections = ['global', 'osd']
  314. cephcontext.update(CephConfContext(permitted_sections=sections)())
  315. return cephcontext
  316. def emit_cephconf(upgrading=False):
  317. # Install ceph.conf as an alternative to support
  318. # co-existence with other charms that write this file
  319. charm_ceph_conf = "/var/lib/charm/{}/ceph.conf".format(service_name())
  320. mkdir(os.path.dirname(charm_ceph_conf), owner=ceph.ceph_user(),
  321. group=ceph.ceph_user())
  322. with open(charm_ceph_conf, 'w') as cephconf:
  323. context = get_ceph_context(upgrading)
  324. cephconf.write(render_template('ceph.conf', context))
  325. install_alternative('ceph.conf', '/etc/ceph/ceph.conf',
  326. charm_ceph_conf, 90)
  327. @hooks.hook('config-changed')
  328. @harden()
  329. def config_changed():
  330. # Determine whether vaultlocker is required and install
  331. if use_vaultlocker():
  332. installed = len(filter_installed_packages(['vaultlocker'])) == 0
  333. if not installed:
  334. add_source('ppa:openstack-charmers/vaultlocker')
  335. apt_update(fatal=True)
  336. apt_install('vaultlocker', fatal=True)
  337. # Check if an upgrade was requested
  338. check_for_upgrade()
  339. # Pre-flight checks
  340. if config('osd-format') not in ceph.DISK_FORMATS:
  341. log('Invalid OSD disk format configuration specified', level=ERROR)
  342. sys.exit(1)
  343. if config('prefer-ipv6'):
  344. assert_charm_supports_ipv6()
  345. sysctl_dict = config('sysctl')
  346. if sysctl_dict:
  347. create_sysctl(sysctl_dict, '/etc/sysctl.d/50-ceph-osd-charm.conf')
  348. e_mountpoint = config('ephemeral-unmount')
  349. if e_mountpoint and ceph.filesystem_mounted(e_mountpoint):
  350. umount(e_mountpoint)
  351. prepare_disks_and_activate()
  352. install_apparmor_profile()
  353. add_to_updatedb_prunepath(STORAGE_MOUNT_PATH)
  354. @hooks.hook('storage.real')
  355. def prepare_disks_and_activate():
  356. # NOTE: vault/vaultlocker preflight check
  357. vault_kv = vaultlocker.VaultKVContext(vaultlocker.VAULTLOCKER_BACKEND)
  358. context = vault_kv()
  359. if use_vaultlocker() and not vault_kv.complete:
  360. log('Deferring OSD preparation as vault not ready',
  361. level=DEBUG)
  362. return
  363. elif use_vaultlocker() and vault_kv.complete:
  364. log('Vault ready, writing vaultlocker configuration',
  365. level=DEBUG)
  366. vaultlocker.write_vaultlocker_conf(context)
  367. osd_journal = get_journal_devices()
  368. if not osd_journal.isdisjoint(set(get_devices())):
  369. raise ValueError('`osd-journal` and `osd-devices` options must not'
  370. 'overlap.')
  371. log("got journal devs: {}".format(osd_journal), level=DEBUG)
  372. # pre-flight check of eligible device pristinity
  373. devices = get_devices()
  374. # if a device has been previously touched we need to consider it as
  375. # non-pristine. If it needs to be re-processed it has to be zapped
  376. # via the respective action which also clears the unitdata entry.
  377. db = kv()
  378. touched_devices = db.get('osd-devices', [])
  379. devices = [dev for dev in devices if dev not in touched_devices]
  380. log('Skipping osd devices previously processed by this unit: {}'
  381. .format(touched_devices))
  382. # filter osd-devices that are file system paths
  383. devices = [dev for dev in devices if dev.startswith('/dev')]
  384. # filter osd-devices that does not exist on this unit
  385. devices = [dev for dev in devices if os.path.exists(dev)]
  386. # filter osd-devices that are already mounted
  387. devices = [dev for dev in devices if not is_device_mounted(dev)]
  388. # filter osd-devices that are active bluestore devices
  389. devices = [dev for dev in devices
  390. if not ceph.is_active_bluestore_device(dev)]
  391. log('Checking for pristine devices: "{}"'.format(devices), level=DEBUG)
  392. if not all(ceph.is_pristine_disk(dev) for dev in devices):
  393. status_set('blocked',
  394. 'Non-pristine devices detected, consult '
  395. '`list-disks`, `zap-disk` and `blacklist-*` actions.')
  396. return
  397. if ceph.is_bootstrapped():
  398. log('ceph bootstrapped, rescanning disks')
  399. emit_cephconf()
  400. for dev in get_devices():
  401. ceph.osdize(dev, config('osd-format'),
  402. osd_journal,
  403. config('ignore-device-errors'),
  404. config('osd-encrypt'),
  405. config('bluestore'),
  406. config('osd-encrypt-keymanager'))
  407. # Make it fast!
  408. if config('autotune'):
  409. ceph.tune_dev(dev)
  410. ceph.start_osds(get_devices())
  411. def get_mon_hosts():
  412. hosts = []
  413. for relid in relation_ids('mon'):
  414. for unit in related_units(relid):
  415. addr = \
  416. relation_get('ceph-public-address',
  417. unit,
  418. relid) or get_host_ip(
  419. relation_get(
  420. 'private-address',
  421. unit,
  422. relid))
  423. if addr:
  424. hosts.append('{}:6789'.format(format_ipv6_addr(addr) or addr))
  425. return sorted(hosts)
  426. def get_fsid():
  427. return get_conf('fsid')
  428. def get_auth():
  429. return get_conf('auth')
  430. def get_conf(name):
  431. for relid in relation_ids('mon'):
  432. for unit in related_units(relid):
  433. conf = relation_get(name,
  434. unit, relid)
  435. if conf:
  436. return conf
  437. return None
  438. def get_devices():
  439. devices = []
  440. if config('osd-devices'):
  441. for path in config('osd-devices').split(' '):
  442. path = path.strip()
  443. # Make sure its a device which is specified using an
  444. # absolute path so that the current working directory
  445. # or any relative path under this directory is not used
  446. if os.path.isabs(path):
  447. devices.append(os.path.realpath(path))
  448. # List storage instances for the 'osd-devices'
  449. # store declared for this charm too, and add
  450. # their block device paths to the list.
  451. storage_ids = storage_list('osd-devices')
  452. devices.extend((storage_get('location', s) for s in storage_ids))
  453. # Filter out any devices in the action managed unit-local device blacklist
  454. _blacklist = get_blacklist()
  455. return [device for device in devices if device not in _blacklist]
  456. @hooks.hook('mon-relation-changed',
  457. 'mon-relation-departed')
  458. def mon_relation():
  459. bootstrap_key = relation_get('osd_bootstrap_key')
  460. upgrade_key = relation_get('osd_upgrade_key')
  461. if get_fsid() and get_auth() and bootstrap_key:
  462. log('mon has provided conf- scanning disks')
  463. emit_cephconf()
  464. ceph.import_osd_bootstrap_key(bootstrap_key)
  465. ceph.import_osd_upgrade_key(upgrade_key)
  466. prepare_disks_and_activate()
  467. else:
  468. log('mon cluster has not yet provided conf')
  469. @hooks.hook('upgrade-charm.real')
  470. @harden()
  471. def upgrade_charm():
  472. if get_fsid() and get_auth():
  473. emit_cephconf()
  474. apt_install(packages=filter_installed_packages(ceph.determine_packages()),
  475. fatal=True)
  476. install_udev_rules()
  477. @hooks.hook('nrpe-external-master-relation-joined',
  478. 'nrpe-external-master-relation-changed')
  479. def update_nrpe_config():
  480. # python-dbus is used by check_upstart_job
  481. apt_install('python3-dbus')
  482. hostname = nrpe.get_nagios_hostname()
  483. current_unit = nrpe.get_nagios_unit_name()
  484. nrpe_setup = nrpe.NRPE(hostname=hostname)
  485. nrpe_setup.add_check(
  486. shortname='ceph-osd',
  487. description='process check {%s}' % current_unit,
  488. check_cmd=('/bin/cat /var/lib/ceph/osd/ceph-*/whoami |'
  489. 'xargs -I@ status ceph-osd id=@ && exit 0 || exit 2')
  490. )
  491. nrpe_setup.write()
  492. @hooks.hook('secrets-storage-relation-joined')
  493. def secrets_storage_joined(relation_id=None):
  494. relation_set(relation_id=relation_id,
  495. secret_backend='charm-vaultlocker',
  496. isolated=True,
  497. access_address=get_relation_ip('secrets-storage'),
  498. hostname=socket.gethostname())
  499. @hooks.hook('secrets-storage-relation-changed')
  500. def secrets_storage_changed():
  501. vault_ca = relation_get('vault_ca')
  502. if vault_ca:
  503. vault_ca = base64.decodestring(json.loads(vault_ca).encode())
  504. write_file('/usr/local/share/ca-certificates/vault-ca.crt',
  505. vault_ca, perms=0o644)
  506. subprocess.check_call(['update-ca-certificates', '--fresh'])
  507. prepare_disks_and_activate()
  508. VERSION_PACKAGE = 'ceph-common'
  509. def assess_status():
  510. """Assess status of current unit"""
  511. # check to see if the unit is paused.
  512. application_version_set(get_upstream_version(VERSION_PACKAGE))
  513. if is_unit_paused_set():
  514. status_set('maintenance',
  515. "Paused. Use 'resume' action to resume normal service.")
  516. return
  517. # Check for mon relation
  518. if len(relation_ids('mon')) < 1:
  519. status_set('blocked', 'Missing relation: monitor')
  520. return
  521. # Check for monitors with presented addresses
  522. # Check for bootstrap key presentation
  523. monitors = get_mon_hosts()
  524. if len(monitors) < 1 or not get_conf('osd_bootstrap_key'):
  525. status_set('waiting', 'Incomplete relation: monitor')
  526. return
  527. # Check for vault
  528. if use_vaultlocker():
  529. if not relation_ids('secrets-storage'):
  530. status_set('blocked', 'Missing relation: vault')
  531. return
  532. if not vaultlocker.vault_relation_complete():
  533. status_set('waiting', 'Incomplete relation: vault')
  534. return
  535. # Check for OSD device creation parity i.e. at least some devices
  536. # must have been presented and used for this charm to be operational
  537. (prev_status, prev_message) = status_get()
  538. running_osds = ceph.get_running_osds()
  539. if not prev_message.startswith('Non-pristine'):
  540. if not running_osds:
  541. status_set('blocked',
  542. 'No block devices detected using current configuration')
  543. else:
  544. status_set('active',
  545. 'Unit is ready ({} OSD)'.format(len(running_osds)))
  546. @hooks.hook('update-status')
  547. @harden()
  548. def update_status():
  549. log('Updating status.')
  550. if __name__ == '__main__':
  551. try:
  552. hooks.execute(sys.argv)
  553. except UnregisteredHookError as e:
  554. log('Unknown hook {} - skipping.'.format(e))
  555. assess_status()