openstack
/
charm-ceph-osd
Code Issues Proposed changes

Updates to support Ceph Bobtail LTS 

- Add support for different OSD block device formats
- Add support for separate journal disks

Fixes

- Improve hostname -> ip address resolution (fixes issues with MAAS managed DNS)
- Improve zapping process for devices that have already been OSD's.
- Don't catch key errors from hooks in do_hooks.
- Resync utils.py, ceph.py across ceph charms.
This commit is contained in:
James Page 2013-02-08 11:03:53 +00:00
parent 0bce381b4c 247519b4fd
commit df28cb6218
6 changed files with 225 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
README → README.md
View File

18
config.yaml
View File

@ -7,6 +7,24 @@ options:
.
These devices are the range of devices that will be checked for and
used across all service units.
osd-journal:
type: string
description: |
The device to use as a shared journal drive for all OSD's. By default
no journal device will be used.
.
Only supported with ceph >= 0.55.
osd-format:
type: string
default: xfs
description: |
Format of filesystem to use for OSD devices; supported formats include:
.
xfs (Default >= 0.55)
ext4 (Only option < 0.55)
btrfs (experimental and not recommended)
.
Only supported with ceph >= 0.55.
osd-reformat:
type: string
description: |

119
hooks/ceph.py
View File

@ -12,8 +12,11 @@ import subprocess
import time
import utils
import os
import apt_pkg as apt
QUORUM = ['leader', 'peon']
LEADER = 'leader'
PEON = 'peon'
QUORUM = [LEADER, PEON]
def is_quorum():
@ -40,6 +43,30 @@ def is_quorum():
return False
def is_leader():
asok = "/var/run/ceph/ceph-mon.{}.asok".format(utils.get_unit_hostname())
cmd = [
"ceph",
"--admin-daemon",
asok,
"mon_status"
]
if os.path.exists(asok):
try:
result = json.loads(subprocess.check_output(cmd))
except subprocess.CalledProcessError:
return False
except ValueError:
# Non JSON response from mon_status
return False
if result['state'] == LEADER:
return True
else:
return False
else:
return False
def wait_for_quorum():
while not is_quorum():
time.sleep(3)
@ -58,6 +85,12 @@ def add_bootstrap_hint(peer):
# Ignore any errors for this call
subprocess.call(cmd)
DISK_FORMATS = [
'xfs',
'ext4',
'btrfs'
]
def is_osd_disk(dev):
try:
@ -81,6 +114,12 @@ def rescan_osd_devices():
subprocess.call(cmd)
def zap_disk(dev):
cmd = ['sgdisk', '--zap-all', dev]
subprocess.check_call(cmd)
_bootstrap_keyring = "/var/lib/ceph/bootstrap-osd/ceph.keyring"
@ -88,6 +127,11 @@ def is_bootstrapped():
return os.path.exists(_bootstrap_keyring)
def wait_for_bootstrap():
while (not is_bootstrapped()):
time.sleep(3)
def import_osd_bootstrap_key(key):
if not os.path.exists(_bootstrap_keyring):
cmd = [
@ -100,15 +144,53 @@ def import_osd_bootstrap_key(key):
subprocess.check_call(cmd)
# OSD caps taken from ceph-create-keys
_osd_bootstrap_caps = [
'allow command osd create ...',
'allow command osd crush set ...',
r'allow command auth add * osd allow\ * mon allow\ rwx',
'allow command mon getmap'
]
_osd_bootstrap_caps = {
'mon': [
'allow command osd create ...',
'allow command osd crush set ...',
r'allow command auth add * osd allow\ * mon allow\ rwx',
'allow command mon getmap'
]
}
def get_osd_bootstrap_key():
return get_named_key('bootstrap-osd', _osd_bootstrap_caps)
_radosgw_keyring = "/etc/ceph/keyring.rados.gateway"
def import_radosgw_key(key):
if not os.path.exists(_radosgw_keyring):
cmd = [
'ceph-authtool',
_radosgw_keyring,
'--create-keyring',
'--name=client.radosgw.gateway',
'--add-key={}'.format(key)
]
subprocess.check_call(cmd)
# OSD caps taken from ceph-create-keys
_radosgw_caps = {
'mon': ['allow r'],
'osd': ['allow rwx']
}
def get_radosgw_key():
return get_named_key('radosgw.gateway', _radosgw_caps)
_default_caps = {
'mon': ['allow r'],
'osd': ['allow rwx']
}
def get_named_key(name, caps=None):
caps = caps or _default_caps
cmd = [
'ceph',
'--name', 'mon.',
@ -116,9 +198,14 @@ def get_osd_bootstrap_key():
'/var/lib/ceph/mon/ceph-{}/keyring'.format(
utils.get_unit_hostname()
),
'auth', 'get-or-create', 'client.bootstrap-osd',
'mon', '; '.join(_osd_bootstrap_caps)
'auth', 'get-or-create', 'client.{}'.format(name),
]
# Add capabilities
for subsystem, subcaps in caps.iteritems():
cmd.extend([
subsystem,
'; '.join(subcaps),
])
output = subprocess.check_output(cmd).strip() # IGNORE:E1103
# get-or-create appears to have different output depending
# on whether its 'get' or 'create'
@ -132,3 +219,17 @@ def get_osd_bootstrap_key():
if 'key' in element:
key = element.split(' = ')[1].strip() # IGNORE:E1103
return key
def get_ceph_version():
apt.init()
cache = apt.Cache()
pkg = cache['ceph']
if pkg.current_ver:
return apt.upstream_version(pkg.current_ver.ver_str)
else:
return None
def version_compare(a, b):
return apt.version_compare(a, b)

60
hooks/hooks.py
View File

@ -18,14 +18,16 @@ import utils
def install_upstart_scripts():
for x in glob.glob('files/upstart/*.conf'):
shutil.copy(x, '/etc/init/')
# Only install upstart configurations for older versions
if ceph.get_ceph_version() < "0.55.1":
for x in glob.glob('files/upstart/*.conf'):
shutil.copy(x, '/etc/init/')
def install():
utils.juju_log('INFO', 'Begin install hook.')
utils.configure_source()
utils.install('ceph', 'gdisk', 'ntp')
utils.install('ceph', 'gdisk', 'ntp', 'btrfs-tools')
install_upstart_scripts()
utils.juju_log('INFO', 'End install hook.')
@ -37,21 +39,38 @@ def emit_cephconf():
cephcontext = {
'auth_supported': get_auth(),
'mon_hosts': ' '.join(mon_hosts),
'fsid': get_fsid()
'fsid': get_fsid(),
'version': ceph.get_ceph_version()
}
with open('/etc/ceph/ceph.conf', 'w') as cephconf:
cephconf.write(utils.render_template('ceph.conf', cephcontext))
JOURNAL_ZAPPED = '/var/lib/ceph/journal_zapped'
def config_changed():
utils.juju_log('INFO', 'Begin config-changed hook.')
# Pre-flight checks
if utils.config_get('osd-format') not in ceph.DISK_FORMATS:
utils.juju_log('CRITICAL',
'Invalid OSD disk format configuration specified')
sys.exit(1)
e_mountpoint = utils.config_get('ephemeral-unmount')
if (e_mountpoint != "" and
if (e_mountpoint and
filesystem_mounted(e_mountpoint)):
subprocess.call(['umount', e_mountpoint])
osd_journal = utils.config_get('osd-journal')
if (osd_journal and
not os.path.exists(JOURNAL_ZAPPED) and
os.path.exists(osd_journal)):
ceph.zap_disk(osd_journal)
with open(JOURNAL_ZAPPED, 'w') as zapped:
zapped.write('DONE')
if ceph.is_bootstrapped():
utils.juju_log('INFO', 'ceph bootstrapped, rescanning disks')
emit_cephconf()
@ -89,13 +108,13 @@ def get_conf(name):
for unit in utils.relation_list(relid):
conf = utils.relation_get(name,
unit, relid)
if conf != "":
if conf:
return conf
return None
def reformat_osd():
if utils.config_get('osd-reformat') != "":
if utils.config_get('osd-reformat'):
return True
else:
return False
@ -119,7 +138,23 @@ def osdize(dev):
'Looks like {} is in use, skipping.'.format(dev))
return
subprocess.call(['ceph-disk-prepare', dev])
cmd = ['ceph-disk-prepare']
# Later versions of ceph support more options
if ceph.get_ceph_version() >= "0.55":
osd_format = utils.config_get('osd-format')
if osd_format:
cmd.append('--fs-type')
cmd.append(osd_format)
cmd.append(dev)
osd_journal = utils.config_get('osd-journal')
if (osd_journal and
os.path.exists(osd_journal)):
cmd.append(osd_journal)
else:
# Just provide the device - no other options
# for older versions of ceph
cmd.append(dev)
subprocess.call(cmd)
def device_mounted(dev):
@ -136,7 +171,7 @@ def mon_relation():
bootstrap_key = utils.relation_get('osd_bootstrap_key')
if (get_fsid() and
get_auth() and
bootstrap_key != ""):
bootstrap_key):
utils.juju_log('INFO', 'mon has provided conf- scanning disks')
emit_cephconf()
ceph.import_osd_bootstrap_key(bootstrap_key)
@ -159,18 +194,11 @@ def upgrade_charm():
utils.juju_log('INFO', 'End upgrade-charm hook.')
def start():
# In case we're being redeployed to the same machines, try
# to make sure everything is running as soon as possible.
ceph.rescan_osd_devices()
utils.do_hooks({
'config-changed': config_changed,
'install': install,
'mon-relation-departed': mon_relation,
'mon-relation-changed': mon_relation,
'start': start,
'upgrade-charm': upgrade_charm,
})

57
hooks/utils.py
View File

@ -11,16 +11,19 @@ import os
import subprocess
import socket
import sys
import re
def do_hooks(hooks):
hook = os.path.basename(sys.argv[0])
try:
hooks[hook]()
hook_func = hooks[hook]
except KeyError:
juju_log('INFO',
"This charm doesn't know how to handle '{}'.".format(hook))
else:
hook_func()
def install(*pkgs):
@ -41,6 +44,12 @@ except ImportError:
install('python-jinja2')
import jinja2
try:
import dns.resolver
except ImportError:
install('python-dnspython')
import dns.resolver
def render_template(template_name, context, template_dir=TEMPLATES_DIR):
templates = jinja2.Environment(
@ -88,6 +97,18 @@ def configure_source():
]
subprocess.check_call(cmd)
def enable_pocket(pocket):
apt_sources = "/etc/apt/sources.list"
with open(apt_sources, "r") as sources:
lines = sources.readlines()
with open(apt_sources, "w") as sources:
for line in lines:
if pocket in line:
sources.write(re.sub('^# deb', 'deb', line))
else:
sources.write(line)
# Protocols
TCP = 'TCP'
UDP = 'UDP'
@ -136,7 +157,11 @@ def relation_get(attribute, unit=None, rid=None):
cmd.append(attribute)
if unit:
cmd.append(unit)
return subprocess.check_output(cmd).strip() # IGNORE:E1103
value = str(subprocess.check_output(cmd)).strip()
if value == "":
return None
else:
return value
def relation_set(**kwargs):
@ -159,7 +184,11 @@ def unit_get(attribute):
'unit-get',
attribute
]
return subprocess.check_output(cmd).strip() # IGNORE:E1103
value = str(subprocess.check_output(cmd)).strip()
if value == "":
return None
else:
return value
def config_get(attribute):
@ -167,7 +196,11 @@ def config_get(attribute):
'config-get',
attribute
]
return subprocess.check_output(cmd).strip() # IGNORE:E1103
value = str(subprocess.check_output(cmd)).strip()
if value == "":
return None
else:
return value
def get_unit_hostname():
@ -175,9 +208,13 @@ def get_unit_hostname():
def get_host_ip(hostname=unit_get('private-address')):
cmd = [
'dig',
'+short',
hostname
]
return subprocess.check_output(cmd).strip() # IGNORE:E1103
try:
# Test to see if already an IPv4 address
socket.inet_aton(hostname)
return hostname
except socket.error:
# This may throw an NXDOMAIN exception; in which case
# things are badly broken so just let it kill the hook
answers = dns.resolver.query(hostname, 'A')
if answers:
return answers[0].address

6
templates/ceph.conf
View File

@ -1,5 +1,11 @@
[global]
{% if version < "0.51" %}
auth supported = {{ auth_supported }}
{% else %}
auth cluster required = {{ auth_supported }}
auth service required = {{ auth_supported }}
auth client required = {{ auth_supported }}
{% endif %}
keyring = /etc/ceph/$cluster.$name.keyring
mon host = {{ mon_hosts }}
fsid = {{ fsid }}