openstack/charm-percona-cluster
Code Issues Proposed changes

more

Browse Source
This commit is contained in:
Edward Hope-Morley
2015-02-05 00:51:01 +00:00
parent d737b71a1d
commit 5e4fdcb35a
3 changed files with 281 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files

214
hooks/charmhelpers/contrib/database/mysql.py
View File

@@ -42,10 +42,13 @@ except ImportError:
import MySQLdb import MySQLdb
class MySQLHelper(): class MySQLHelper(object):
def __init__(self, host='localhost'): def __init__(self, rpasswdf_template, upasswdf_template, host='localhost'):
self.host = host self.host = host
# Password file path templates
self.root_passwd_file_template = rpasswdf_template
self.user_passwd_file_template = upasswdf_template
def connect(self, user='root', password=None): def connect(self, user='root', password=None):
self.connection = MySQLdb.connect(user=user, host=self.host, self.connection = MySQLdb.connect(user=user, host=self.host,
@@ -124,42 +127,50 @@ class MySQLHelper():
finally: finally:
cursor.close() cursor.close()
# These are percona-only since mysql charm uses /var/lib/mysql/... def migrate_passwords_to_peer_relation(self):
_root_passwd = '/var/lib/charm/{}/mysql.passwd' """Migrate any passwords storage on disk to cluster peer relation."""
_named_passwd = '/var/lib/charm/{}/mysql-{}.passwd' template = self.user_passwd_file_template
for f in glob.glob(template.format(service_name(), '*')):
_key = os.path.basename(f)
with open(f, 'r') as passwd:
_value = passwd.read().strip()
try:
peer_store(_key, _value)
os.unlink(f)
except ValueError:
# NOTE cluster relation not yet ready - skip for now
pass
def get_mysql_password_on_disk(username=None, password=None, passwd_file=None): def get_mysql_password_on_disk(self, username=None, password=None):
"""Retrieve, generate or store a mysql password for the provided username """Retrieve, generate or store a mysql password for the provided
on disk.""" username on disk."""
if not passwd_file:
if username: if username:
passwd_file = _named_passwd.format(service_name(), username) template = self.user_passwd_file_template
passwd_file = template.format(service_name(), username)
else: else:
passwd_file = _root_passwd.format(service_name()) template = self.root_passwd_file_template
passwd_file = template.format(service_name())
_password = None _password = None
if os.path.exists(passwd_file): if os.path.exists(passwd_file):
with open(passwd_file, 'r') as passwd: with open(passwd_file, 'r') as passwd:
_password = passwd.read().strip() _password = passwd.read().strip()
else: else:
mkdir(os.path.dirname(passwd_file), mkdir(os.path.dirname(passwd_file), owner='root', group='root',
owner='root', group='root',
perms=0o770) perms=0o770)
# Force permissions - for some reason the chmod in makedirs fails # Force permissions - for some reason the chmod in makedirs fails
os.chmod(os.path.dirname(passwd_file), 0o770) os.chmod(os.path.dirname(passwd_file), 0o770)
_password = password or pwgen(length=32) _password = password or pwgen(length=32)
write_file(passwd_file, _password, write_file(passwd_file, _password, owner='root', group='root',
owner='root', group='root',
perms=0o660) perms=0o660)
return _password return _password
def get_mysql_password(self, username=None, password=None):
def get_mysql_password(username=None, password=None, passwd_file=None): """Retrieve, generate or store a mysql password for the provided
"""Retrieve, generate or store a mysql password for the provided username username using peer relation cluster."""
using peer relation cluster.""" self.migrate_passwords_to_peer_relation()
migrate_passwords_to_peer_relation()
if username: if username:
_key = '{}.passwd'.format(username) _key = '{}.passwd'.format(username)
else: else:
@@ -172,33 +183,55 @@ def get_mysql_password(username=None, password=None, passwd_file=None):
peer_store(_key, _password) peer_store(_key, _password)
except ValueError: except ValueError:
# cluster relation is not yet started; use on-disk # cluster relation is not yet started; use on-disk
_password = get_mysql_password_on_disk(username, password, _password = self.get_mysql_password_on_disk(username, password)
passwd_file=passwd_file)
return _password return _password
def get_mysql_root_password(self, password=None):
def migrate_passwords_to_peer_relation():
"""Migrate any passwords storage on disk to cluster peer relation."""
for f in glob.glob('/var/lib/charm/{}/*.passwd'.format(service_name())):
_key = os.path.basename(f)
with open(f, 'r') as passwd:
_value = passwd.read().strip()
try:
peer_store(_key, _value)
os.unlink(f)
except ValueError:
# NOTE cluster relation not yet ready - skip for now
pass
def get_mysql_root_password(password=None, passwd_file=None):
"""Retrieve or generate mysql root password for service units.""" """Retrieve or generate mysql root password for service units."""
return get_mysql_password(username=None, password=password, return self.get_mysql_password(username=None, password=password)
passwd_file=passwd_file)
def get_allowed_units(self, database, username, relation_id=None):
"""Get list of units with access grants for database with username.
def configure_db(hostname, database, username, admin=False, passwd_file=None): This is typically used to provide shared-db relations with a list of
which units have been granted access to the given database.
"""
self.connect(password=self.get_mysql_root_password())
allowed_units = set()
for unit in related_units(relation_id):
settings = relation_get(rid=relation_id, unit=unit)
# First check for setting with prefix, then without
for attr in ["%s_hostname" % (database), 'hostname']:
hosts = settings.get(attr, None)
if hosts:
break
if hosts:
# hostname can be json-encoded list of hostnames
try:
hosts = json.loads(hosts)
except ValueError:
hosts = [hosts]
else:
hosts = [settings['private-address']]
if hosts:
for host in hosts:
if self.grant_exists(database, username, host):
log("Grant exists for host '%s' on db '%s'" %
(host, database), level=DEBUG)
if unit not in allowed_units:
allowed_units.add(unit)
else:
log("Grant does NOT exist for host '%s' on db '%s'" %
(host, database), level=DEBUG)
else:
log("No hosts found for grant check", level=INFO)
return allowed_units
def configure_db(self, hostname, database, username, admin=False):
"""Configure access to database for username from hostname.""" """Configure access to database for username from hostname."""
if config_get('prefer-ipv6'): if config_get('prefer-ipv6'):
remote_ip = hostname remote_ip = hostname
@@ -211,28 +244,27 @@ def configure_db(hostname, database, username, admin=False, passwd_file=None):
else: else:
remote_ip = '127.0.0.1' remote_ip = '127.0.0.1'
password = get_mysql_password(username, passwd_file=passwd_file) self.connect(password=self.get_mysql_root_password())
m_helper = MySQLHelper() if not self.database_exists(database):
m_helper.connect(password=get_mysql_root_password()) self.create_database(database)
if not m_helper.database_exists(database):
m_helper.create_database(database)
if not m_helper.grant_exists(database, password = self.get_mysql_password(username)
username, if not self.grant_exists(database, username, remote_ip):
remote_ip):
if not admin: if not admin:
m_helper.create_grant(database, username, remote_ip, password) self.create_grant(database, username, remote_ip, password)
else: else:
m_helper.create_admin_grant(username, remote_ip, password) self.create_admin_grant(username, remote_ip, password)
return password return password
# Going for the biggest page size to avoid wasted bytes. InnoDB page size is
# 16MB
DEFAULT_PAGE_SIZE = 16 * 1024 * 1024
class PerconaClusterHelper(object):
def human_to_bytes(human): # Going for the biggest page size to avoid wasted bytes. InnoDB page size is
# 16MB
DEFAULT_PAGE_SIZE = 16 * 1024 * 1024
def human_to_bytes(self, human):
"""Convert human readable configuration options to bytes.""" """Convert human readable configuration options to bytes."""
num_re = re.compile('^[0-9]+$') num_re = re.compile('^[0-9]+$')
if num_re.match(human): if num_re.match(human):
@@ -249,36 +281,33 @@ def human_to_bytes(human):
return int(human[:-1]) * factors[modifier] return int(human[:-1]) * factors[modifier]
if modifier == '%': if modifier == '%':
total_ram = human_to_bytes(get_mem_total()) total_ram = self.human_to_bytes(self.get_mem_total())
if is_32bit_system() and total_ram > sys_mem_limit(): if self.is_32bit_system() and total_ram > self.sys_mem_limit():
total_ram = sys_mem_limit() total_ram = self.sys_mem_limit()
factor = int(human[:-1]) * 0.01 factor = int(human[:-1]) * 0.01
pctram = total_ram * factor pctram = total_ram * factor
return int(pctram - (pctram % DEFAULT_PAGE_SIZE)) return int(pctram - (pctram % self.DEFAULT_PAGE_SIZE))
raise ValueError("Can only convert K,M,G, or T") raise ValueError("Can only convert K,M,G, or T")
def is_32bit_system(self):
def is_32bit_system():
"""Determine whether system is 32 or 64 bit.""" """Determine whether system is 32 or 64 bit."""
try: try:
return sys.maxsize < 2 ** 32 return sys.maxsize < 2 ** 32
except OverflowError: except OverflowError:
return False return False
def sys_mem_limit(self):
def sys_mem_limit():
"""Determine the default memory limit for the current service unit.""" """Determine the default memory limit for the current service unit."""
if platform.machine() in ['armv7l']: if platform.machine() in ['armv7l']:
_mem_limit = human_to_bytes('2700M') # experimentally determined _mem_limit = self.human_to_bytes('2700M') # experimentally determined
else: else:
# Limit for x86 based 32bit systems # Limit for x86 based 32bit systems
_mem_limit = human_to_bytes('4G') _mem_limit = self.human_to_bytes('4G')
return _mem_limit return _mem_limit
def get_mem_total(self):
def get_mem_total():
"""Calculate the total memory in the current service unit.""" """Calculate the total memory in the current service unit."""
with open('/proc/meminfo') as meminfo_file: with open('/proc/meminfo') as meminfo_file:
for line in meminfo_file: for line in meminfo_file:
@@ -287,8 +316,7 @@ def get_mem_total():
mtot, modifier = mem.strip().split(' ') mtot, modifier = mem.strip().split(' ')
return '%s%s' % (mtot, upper(modifier[0])) return '%s%s' % (mtot, upper(modifier[0]))
def parse_config(self):
def parse_config():
"""Parse charm configuration and calculate values for config files.""" """Parse charm configuration and calculate values for config files."""
config = config_get() config = config_get()
mysql_config = {} mysql_config = {}
@@ -296,7 +324,7 @@ def parse_config():
mysql_config['max_connections'] = config['max-connections'] mysql_config['max_connections'] = config['max-connections']
# Total memory available for dataset # Total memory available for dataset
dataset_bytes = human_to_bytes(config['dataset-size']) dataset_bytes = self.human_to_bytes(config['dataset-size'])
mysql_config['dataset_bytes'] = dataset_bytes mysql_config['dataset_bytes'] = dataset_bytes
if 'query-cache-type' in config: if 'query-cache-type' in config:
@@ -307,7 +335,7 @@ def parse_config():
# Calculate the query cache size automatically # Calculate the query cache size automatically
qcache_bytes = (dataset_bytes * 0.20) qcache_bytes = (dataset_bytes * 0.20)
qcache_bytes = int(qcache_bytes - qcache_bytes = int(qcache_bytes -
(qcache_bytes % DEFAULT_PAGE_SIZE)) (qcache_bytes % self.DEFAULT_PAGE_SIZE))
mysql_config['query_cache_size'] = qcache_bytes mysql_config['query_cache_size'] = qcache_bytes
dataset_bytes -= qcache_bytes dataset_bytes -= qcache_bytes
@@ -320,7 +348,7 @@ def parse_config():
mysql_config['query_cache_type'] = 0 mysql_config['query_cache_type'] = 0
# Set a sane default key_buffer size # Set a sane default key_buffer size
mysql_config['key_buffer'] = human_to_bytes('32M') mysql_config['key_buffer'] = self.human_to_bytes('32M')
if 'preferred-storage-engine' in config: if 'preferred-storage-engine' in config:
# Storage engine configuration # Storage engine configuration
@@ -343,45 +371,3 @@ def parse_config():
mysql_config['sync_binlog'] = 0 mysql_config['sync_binlog'] = 0
return mysql_config return mysql_config
def get_allowed_units(database, username, db_root_password, relation_id=None):
"""Get list of units with access grants for database with username.
This is typically used to provide shared-db relations with a list of which
units have been granted access to the given database.
"""
m_helper = MySQLHelper()
m_helper.connect(password=db_root_password)
allowed_units = set()
for unit in related_units(relation_id):
settings = relation_get(rid=relation_id, unit=unit)
# First check for setting with prefix, then without
for attr in ["%s_hostname" % (database), 'hostname']:
hosts = settings.get(attr, None)
if hosts:
break
if hosts:
# hostname can be json-encoded list of hostnames
try:
hosts = json.loads(hosts)
except ValueError:
hosts = [hosts]
else:
hosts = [settings['private-address']]
if hosts:
for host in hosts:
if m_helper.grant_exists(database, username, host):
log("Grant exists for host '%s' on db '%s'" %
(host, database), level=DEBUG)
if unit not in allowed_units:
allowed_units.add(unit)
else:
log("Grant does NOT exist for host '%s' on db '%s'" %
(host, database), level=DEBUG)
else:
log("No hosts found for grant check", level=INFO)
return allowed_units

61
hooks/percona_hooks.py
View File

@@ -49,13 +49,10 @@ from percona_utils import (
relation_clear, relation_clear,
assert_charm_supports_ipv6, assert_charm_supports_ipv6,
unit_sorted, unit_sorted,
get_db_helper,
) )
from charmhelpers.contrib.database.mysql import ( from charmhelpers.contrib.database.mysql import (
get_allowed_units, PerconaClusterHelper,
get_mysql_password,
get_mysql_root_password,
parse_config,
configure_db,
) )
from charmhelpers.contrib.hahelpers.cluster import ( from charmhelpers.contrib.hahelpers.cluster import (
peer_units, peer_units,
@@ -87,8 +84,10 @@ def install():
add_source(config('source')) add_source(config('source'))
configure_mysql_root_password(config('root-password')) configure_mysql_root_password(config('root-password'))
mysql_password = get_mysql_password(username='sstuser', db_helper = get_db_helper()
password=config('sst-password')) cfg_passwd = config('sst-password')
mysql_password = db_helper.get_mysql_password(username='sstuser',
password=cfg_passwd)
# Render base configuration (no cluster) # Render base configuration (no cluster)
render_config(mysql_password=mysql_password) render_config(mysql_password=mysql_password)
apt_update(fatal=True) apt_update(fatal=True)
@@ -101,8 +100,10 @@ def render_config(clustered=False, hosts=[], mysql_password=None):
os.makedirs(os.path.dirname(MY_CNF)) os.makedirs(os.path.dirname(MY_CNF))
if not mysql_password: if not mysql_password:
mysql_password = get_mysql_password(username='sstuser', db_helper = get_db_helper()
password=config('sst-password')) cfg_passwd = config('sst-password')
mysql_password = db_helper.get_mysql_password(username='sstuser',
password=cfg_passwd)
context = { context = {
'cluster_name': 'juju_cluster', 'cluster_name': 'juju_cluster',
@@ -123,7 +124,7 @@ def render_config(clustered=False, hosts=[], mysql_password=None):
else: else:
context['ipv6'] = False context['ipv6'] = False
context.update(parse_config()) context.update(PerconaClusterHelper().parse_config())
render(os.path.basename(MY_CNF), MY_CNF, context, perms=0o444) render(os.path.basename(MY_CNF), MY_CNF, context, perms=0o444)
@@ -202,23 +203,19 @@ def db_changed(relation_id=None, unit=None, admin=None):
if admin not in [True, False]: if admin not in [True, False]:
admin = relation_type() == 'db-admin' admin = relation_type() == 'db-admin'
database_name, _ = remote_unit().split("/") db_name, _ = remote_unit().split("/")
username = database_name username = db_name
password = configure_db(relation_get('private-address', db_helper = get_db_helper()
unit=unit, addr = relation_get('private-address', unit=unit, rid=relation_id)
rid=relation_id), password = db_helper.configure_db(addr, db_name, username, admin=admin)
database_name,
username,
admin=admin)
relation_set(relation_id=relation_id, relation_set(relation_id=relation_id,
relation_settings={ relation_settings={
'user': username, 'user': username,
'password': password, 'password': password,
'host': db_host, 'host': db_host,
'database': database_name, 'database': db_name,
} })
)
def get_db_host(client_hostname): def get_db_host(client_hostname):
@@ -232,7 +229,7 @@ def get_db_host(client_hostname):
return unit_get('private-address') return unit_get('private-address')
def configure_db_for_hosts(hosts, database, username): def configure_db_for_hosts(hosts, database, username, db_helper):
"""Hosts may be a json-encoded list of hosts or a single hostname.""" """Hosts may be a json-encoded list of hosts or a single hostname."""
try: try:
hosts = json.loads(hosts) hosts = json.loads(hosts)
@@ -244,7 +241,7 @@ def configure_db_for_hosts(hosts, database, username):
hosts = [hosts] hosts = [hosts]
for host in hosts: for host in hosts:
password = configure_db(host, database, username) password = db_helper.configure_db(host, database, username)
return password return password
@@ -280,7 +277,7 @@ def shared_db_changed(relation_id=None, unit=None):
db_host = unit_get('private-address') db_host = unit_get('private-address')
access_network = config('access-network') access_network = config('access-network')
rpasswd = get_mysql_root_password() db_helper = get_db_helper()
singleset = set(['database', 'username', 'hostname']) singleset = set(['database', 'username', 'hostname'])
if singleset.issubset(settings): if singleset.issubset(settings):
@@ -290,11 +287,11 @@ def shared_db_changed(relation_id=None, unit=None):
username = settings['username'] username = settings['username']
# NOTE: do this before querying access grants # NOTE: do this before querying access grants
password = configure_db_for_hosts(hostname, database, username) password = configure_db_for_hosts(hostname, database, username,
db_helper)
allowed_units = get_allowed_units(database, username, allowed_units = db_helper.get_allowed_units(database, username,
relation_id=relation_id, relation_id=relation_id)
db_root_password=rpasswd)
allowed_units = unit_sorted(allowed_units) allowed_units = unit_sorted(allowed_units)
allowed_units = ' '.join(allowed_units) allowed_units = ' '.join(allowed_units)
relation_set(relation_id=relation_id, allowed_units=allowed_units) relation_set(relation_id=relation_id, allowed_units=allowed_units)
@@ -339,11 +336,11 @@ def shared_db_changed(relation_id=None, unit=None):
username = databases[db]['username'] username = databases[db]['username']
# NOTE: do this before querying access grants # NOTE: do this before querying access grants
password = configure_db_for_hosts(hostname, database, username) password = configure_db_for_hosts(hostname, database, username,
db_helper)
a_units = get_allowed_units(database, username, a_units = db_helper.get_allowed_units(database, username,
relation_id=relation_id, relation_id=relation_id)
db_root_password=rpasswd)
a_units = ' '.join(unit_sorted(a_units)) a_units = ' '.join(unit_sorted(a_units))
allowed_units['%s_allowed_units' % (db)] = a_units allowed_units['%s_allowed_units' % (db)] = a_units

13
hooks/percona_utils.py
View File

@@ -26,7 +26,6 @@ from charmhelpers.contrib.network.ip import (
get_ipv6_addr get_ipv6_addr
) )
from charmhelpers.contrib.database.mysql import ( from charmhelpers.contrib.database.mysql import (
get_mysql_root_password,
MySQLHelper, MySQLHelper,
) )
@@ -133,9 +132,14 @@ SQL_SST_USER_SETUP_IPV6 = ("GRANT RELOAD, LOCK TABLES, REPLICATION CLIENT "
"BY '{}'") "BY '{}'")
def get_db_helper():
return MySQLHelper(rpasswdf_template='/var/lib/charm/{}/mysql.passwd',
upasswdf_template='/var/lib/charm/{}/mysql-{}.passwd')
def configure_sstuser(sst_password): def configure_sstuser(sst_password):
m_helper = MySQLHelper() m_helper = get_db_helper()
m_helper.connect(password=get_mysql_root_password()) m_helper.connect(password=m_helper.get_mysql_root_password())
m_helper.execute(SQL_SST_USER_SETUP.format(sst_password)) m_helper.execute(SQL_SST_USER_SETUP.format(sst_password))
m_helper.execute(SQL_SST_USER_SETUP_IPV6.format(sst_password)) m_helper.execute(SQL_SST_USER_SETUP_IPV6.format(sst_password))
@@ -147,7 +151,8 @@ def configure_mysql_root_password(password):
# Set both percona and mysql password options to cover # Set both percona and mysql password options to cover
# both upstream and distro packages. # both upstream and distro packages.
packages = ["percona-server-server", "mysql-server"] packages = ["percona-server-server", "mysql-server"]
root_pass = get_mysql_root_password(password) m_helper = get_db_helper()
root_pass = m_helper.get_mysql_root_password(password)
for package in packages: for package in packages:
dconf.stdin.write("%s %s/root_password password %s\n" % dconf.stdin.write("%s %s/root_password password %s\n" %
(package, package, root_pass)) (package, package, root_pass))