openstack/charm-percona-cluster
Code Issues Proposed changes

more

Browse Source
This commit is contained in:
Edward Hope-Morley
2015-02-05 00:51:01 +00:00
parent d737b71a1d
commit 5e4fdcb35a
3 changed files with 281 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files

500
hooks/charmhelpers/contrib/database/mysql.py
View File

@@ -42,10 +42,13 @@ except ImportError:
import MySQLdb
class MySQLHelper():
class MySQLHelper(object):
def __init__(self, host='localhost'):
def __init__(self, rpasswdf_template, upasswdf_template, host='localhost'):
self.host = host
# Password file path templates
self.root_passwd_file_template = rpasswdf_template
self.user_passwd_file_template = upasswdf_template
def connect(self, user='root', password=None):
self.connection = MySQLdb.connect(user=user, host=self.host,
@@ -124,264 +127,247 @@ class MySQLHelper():
finally:
cursor.close()
# These are percona-only since mysql charm uses /var/lib/mysql/...
_root_passwd = '/var/lib/charm/{}/mysql.passwd'
_named_passwd = '/var/lib/charm/{}/mysql-{}.passwd'
def migrate_passwords_to_peer_relation(self):
"""Migrate any passwords storage on disk to cluster peer relation."""
template = self.user_passwd_file_template
for f in glob.glob(template.format(service_name(), '*')):
_key = os.path.basename(f)
with open(f, 'r') as passwd:
_value = passwd.read().strip()
def get_mysql_password_on_disk(username=None, password=None, passwd_file=None):
"""Retrieve, generate or store a mysql password for the provided username
on disk."""
if not passwd_file:
if username:
passwd_file = _named_passwd.format(service_name(), username)
else:
passwd_file = _root_passwd.format(service_name())
_password = None
if os.path.exists(passwd_file):
with open(passwd_file, 'r') as passwd:
_password = passwd.read().strip()
else:
mkdir(os.path.dirname(passwd_file),
owner='root', group='root',
perms=0o770)
# Force permissions - for some reason the chmod in makedirs fails
os.chmod(os.path.dirname(passwd_file), 0o770)
_password = password or pwgen(length=32)
write_file(passwd_file, _password,
owner='root', group='root',
perms=0o660)
return _password
def get_mysql_password(username=None, password=None, passwd_file=None):
"""Retrieve, generate or store a mysql password for the provided username
using peer relation cluster."""
migrate_passwords_to_peer_relation()
if username:
_key = '{}.passwd'.format(username)
else:
_key = 'mysql.passwd'
try:
_password = peer_retrieve(_key)
if _password is None:
_password = password or pwgen(length=32)
peer_store(_key, _password)
except ValueError:
# cluster relation is not yet started; use on-disk
_password = get_mysql_password_on_disk(username, password,
passwd_file=passwd_file)
return _password
def migrate_passwords_to_peer_relation():
"""Migrate any passwords storage on disk to cluster peer relation."""
for f in glob.glob('/var/lib/charm/{}/*.passwd'.format(service_name())):
_key = os.path.basename(f)
with open(f, 'r') as passwd:
_value = passwd.read().strip()
try:
peer_store(_key, _value)
os.unlink(f)
except ValueError:
# NOTE cluster relation not yet ready - skip for now
pass
def get_mysql_root_password(password=None, passwd_file=None):
"""Retrieve or generate mysql root password for service units."""
return get_mysql_password(username=None, password=password,
passwd_file=passwd_file)
def configure_db(hostname, database, username, admin=False, passwd_file=None):
"""Configure access to database for username from hostname."""
if config_get('prefer-ipv6'):
remote_ip = hostname
elif hostname != unit_get('private-address'):
try:
remote_ip = socket.gethostbyname(hostname)
except Exception:
# socket.gethostbyname doesn't support ipv6
remote_ip = hostname
else:
remote_ip = '127.0.0.1'
password = get_mysql_password(username, passwd_file=passwd_file)
m_helper = MySQLHelper()
m_helper.connect(password=get_mysql_root_password())
if not m_helper.database_exists(database):
m_helper.create_database(database)
if not m_helper.grant_exists(database,
username,
remote_ip):
if not admin:
m_helper.create_grant(database, username, remote_ip, password)
else:
m_helper.create_admin_grant(username, remote_ip, password)
return password
# Going for the biggest page size to avoid wasted bytes. InnoDB page size is
# 16MB
DEFAULT_PAGE_SIZE = 16 * 1024 * 1024
def human_to_bytes(human):
"""Convert human readable configuration options to bytes."""
num_re = re.compile('^[0-9]+$')
if num_re.match(human):
return human
factors = {
'K': 1024,
'M': 1048576,
'G': 1073741824,
'T': 1099511627776
}
modifier = human[-1]
if modifier in factors:
return int(human[:-1]) * factors[modifier]
if modifier == '%':
total_ram = human_to_bytes(get_mem_total())
if is_32bit_system() and total_ram > sys_mem_limit():
total_ram = sys_mem_limit()
factor = int(human[:-1]) * 0.01
pctram = total_ram * factor
return int(pctram - (pctram % DEFAULT_PAGE_SIZE))
raise ValueError("Can only convert K,M,G, or T")
def is_32bit_system():
"""Determine whether system is 32 or 64 bit."""
try:
return sys.maxsize < 2 ** 32
except OverflowError:
return False
def sys_mem_limit():
"""Determine the default memory limit for the current service unit."""
if platform.machine() in ['armv7l']:
_mem_limit = human_to_bytes('2700M') # experimentally determined
else:
# Limit for x86 based 32bit systems
_mem_limit = human_to_bytes('4G')
return _mem_limit
def get_mem_total():
"""Calculate the total memory in the current service unit."""
with open('/proc/meminfo') as meminfo_file:
for line in meminfo_file:
key, mem = line.split(':', 2)
if key == 'MemTotal':
mtot, modifier = mem.strip().split(' ')
return '%s%s' % (mtot, upper(modifier[0]))
def parse_config():
"""Parse charm configuration and calculate values for config files."""
config = config_get()
mysql_config = {}
if 'max-connections' in config:
mysql_config['max_connections'] = config['max-connections']
# Total memory available for dataset
dataset_bytes = human_to_bytes(config['dataset-size'])
mysql_config['dataset_bytes'] = dataset_bytes
if 'query-cache-type' in config:
# Query Cache Configuration
mysql_config['query_cache_size'] = config['query-cache-size']
if (config['query-cache-size'] == -1 and
config['query-cache-type'] in ['ON', 'DEMAND']):
# Calculate the query cache size automatically
qcache_bytes = (dataset_bytes * 0.20)
qcache_bytes = int(qcache_bytes -
(qcache_bytes % DEFAULT_PAGE_SIZE))
mysql_config['query_cache_size'] = qcache_bytes
dataset_bytes -= qcache_bytes
# 5.5 allows the words, but not 5.1
if config['query-cache-type'] == 'ON':
mysql_config['query_cache_type'] = 1
elif config['query-cache-type'] == 'DEMAND':
mysql_config['query_cache_type'] = 2
else:
mysql_config['query_cache_type'] = 0
# Set a sane default key_buffer size
mysql_config['key_buffer'] = human_to_bytes('32M')
if 'preferred-storage-engine' in config:
# Storage engine configuration
preferred_engines = config['preferred-storage-engine'].split(',')
chunk_size = int(dataset_bytes / len(preferred_engines))
mysql_config['innodb_flush_log_at_trx_commit'] = 1
mysql_config['sync_binlog'] = 1
if 'InnoDB' in preferred_engines:
mysql_config['innodb_buffer_pool_size'] = chunk_size
if config['tuning-level'] == 'fast':
mysql_config['innodb_flush_log_at_trx_commit'] = 2
else:
mysql_config['innodb_buffer_pool_size'] = 0
mysql_config['default_storage_engine'] = preferred_engines[0]
if 'MyISAM' in preferred_engines:
mysql_config['key_buffer'] = chunk_size
if config['tuning-level'] == 'fast':
mysql_config['sync_binlog'] = 0
return mysql_config
def get_allowed_units(database, username, db_root_password, relation_id=None):
"""Get list of units with access grants for database with username.
This is typically used to provide shared-db relations with a list of which
units have been granted access to the given database.
"""
m_helper = MySQLHelper()
m_helper.connect(password=db_root_password)
allowed_units = set()
for unit in related_units(relation_id):
settings = relation_get(rid=relation_id, unit=unit)
# First check for setting with prefix, then without
for attr in ["%s_hostname" % (database), 'hostname']:
hosts = settings.get(attr, None)
if hosts:
break
if hosts:
# hostname can be json-encoded list of hostnames
try:
hosts = json.loads(hosts)
peer_store(_key, _value)
os.unlink(f)
except ValueError:
hosts = [hosts]
else:
hosts = [settings['private-address']]
# NOTE cluster relation not yet ready - skip for now
pass
if hosts:
for host in hosts:
if m_helper.grant_exists(database, username, host):
log("Grant exists for host '%s' on db '%s'" %
(host, database), level=DEBUG)
if unit not in allowed_units:
allowed_units.add(unit)
else:
log("Grant does NOT exist for host '%s' on db '%s'" %
(host, database), level=DEBUG)
def get_mysql_password_on_disk(self, username=None, password=None):
"""Retrieve, generate or store a mysql password for the provided
username on disk."""
if username:
template = self.user_passwd_file_template
passwd_file = template.format(service_name(), username)
else:
log("No hosts found for grant check", level=INFO)
template = self.root_passwd_file_template
passwd_file = template.format(service_name())
return allowed_units
_password = None
if os.path.exists(passwd_file):
with open(passwd_file, 'r') as passwd:
_password = passwd.read().strip()
else:
mkdir(os.path.dirname(passwd_file), owner='root', group='root',
perms=0o770)
# Force permissions - for some reason the chmod in makedirs fails
os.chmod(os.path.dirname(passwd_file), 0o770)
_password = password or pwgen(length=32)
write_file(passwd_file, _password, owner='root', group='root',
perms=0o660)
return _password
def get_mysql_password(self, username=None, password=None):
"""Retrieve, generate or store a mysql password for the provided
username using peer relation cluster."""
self.migrate_passwords_to_peer_relation()
if username:
_key = '{}.passwd'.format(username)
else:
_key = 'mysql.passwd'
try:
_password = peer_retrieve(_key)
if _password is None:
_password = password or pwgen(length=32)
peer_store(_key, _password)
except ValueError:
# cluster relation is not yet started; use on-disk
_password = self.get_mysql_password_on_disk(username, password)
return _password
def get_mysql_root_password(self, password=None):
"""Retrieve or generate mysql root password for service units."""
return self.get_mysql_password(username=None, password=password)
def get_allowed_units(self, database, username, relation_id=None):
"""Get list of units with access grants for database with username.
This is typically used to provide shared-db relations with a list of
which units have been granted access to the given database.
"""
self.connect(password=self.get_mysql_root_password())
allowed_units = set()
for unit in related_units(relation_id):
settings = relation_get(rid=relation_id, unit=unit)
# First check for setting with prefix, then without
for attr in ["%s_hostname" % (database), 'hostname']:
hosts = settings.get(attr, None)
if hosts:
break
if hosts:
# hostname can be json-encoded list of hostnames
try:
hosts = json.loads(hosts)
except ValueError:
hosts = [hosts]
else:
hosts = [settings['private-address']]
if hosts:
for host in hosts:
if self.grant_exists(database, username, host):
log("Grant exists for host '%s' on db '%s'" %
(host, database), level=DEBUG)
if unit not in allowed_units:
allowed_units.add(unit)
else:
log("Grant does NOT exist for host '%s' on db '%s'" %
(host, database), level=DEBUG)
else:
log("No hosts found for grant check", level=INFO)
return allowed_units
def configure_db(self, hostname, database, username, admin=False):
"""Configure access to database for username from hostname."""
if config_get('prefer-ipv6'):
remote_ip = hostname
elif hostname != unit_get('private-address'):
try:
remote_ip = socket.gethostbyname(hostname)
except Exception:
# socket.gethostbyname doesn't support ipv6
remote_ip = hostname
else:
remote_ip = '127.0.0.1'
self.connect(password=self.get_mysql_root_password())
if not self.database_exists(database):
self.create_database(database)
password = self.get_mysql_password(username)
if not self.grant_exists(database, username, remote_ip):
if not admin:
self.create_grant(database, username, remote_ip, password)
else:
self.create_admin_grant(username, remote_ip, password)
return password
class PerconaClusterHelper(object):
# Going for the biggest page size to avoid wasted bytes. InnoDB page size is
# 16MB
DEFAULT_PAGE_SIZE = 16 * 1024 * 1024
def human_to_bytes(self, human):
"""Convert human readable configuration options to bytes."""
num_re = re.compile('^[0-9]+$')
if num_re.match(human):
return human
factors = {
'K': 1024,
'M': 1048576,
'G': 1073741824,
'T': 1099511627776
}
modifier = human[-1]
if modifier in factors:
return int(human[:-1]) * factors[modifier]
if modifier == '%':
total_ram = self.human_to_bytes(self.get_mem_total())
if self.is_32bit_system() and total_ram > self.sys_mem_limit():
total_ram = self.sys_mem_limit()
factor = int(human[:-1]) * 0.01
pctram = total_ram * factor
return int(pctram - (pctram % self.DEFAULT_PAGE_SIZE))
raise ValueError("Can only convert K,M,G, or T")
def is_32bit_system(self):
"""Determine whether system is 32 or 64 bit."""
try:
return sys.maxsize < 2 ** 32
except OverflowError:
return False
def sys_mem_limit(self):
"""Determine the default memory limit for the current service unit."""
if platform.machine() in ['armv7l']:
_mem_limit = self.human_to_bytes('2700M') # experimentally determined
else:
# Limit for x86 based 32bit systems
_mem_limit = self.human_to_bytes('4G')
return _mem_limit
def get_mem_total(self):
"""Calculate the total memory in the current service unit."""
with open('/proc/meminfo') as meminfo_file:
for line in meminfo_file:
key, mem = line.split(':', 2)
if key == 'MemTotal':
mtot, modifier = mem.strip().split(' ')
return '%s%s' % (mtot, upper(modifier[0]))
def parse_config(self):
"""Parse charm configuration and calculate values for config files."""
config = config_get()
mysql_config = {}
if 'max-connections' in config:
mysql_config['max_connections'] = config['max-connections']
# Total memory available for dataset
dataset_bytes = self.human_to_bytes(config['dataset-size'])
mysql_config['dataset_bytes'] = dataset_bytes
if 'query-cache-type' in config:
# Query Cache Configuration
mysql_config['query_cache_size'] = config['query-cache-size']
if (config['query-cache-size'] == -1 and
config['query-cache-type'] in ['ON', 'DEMAND']):
# Calculate the query cache size automatically
qcache_bytes = (dataset_bytes * 0.20)
qcache_bytes = int(qcache_bytes -
(qcache_bytes % self.DEFAULT_PAGE_SIZE))
mysql_config['query_cache_size'] = qcache_bytes
dataset_bytes -= qcache_bytes
# 5.5 allows the words, but not 5.1
if config['query-cache-type'] == 'ON':
mysql_config['query_cache_type'] = 1
elif config['query-cache-type'] == 'DEMAND':
mysql_config['query_cache_type'] = 2
else:
mysql_config['query_cache_type'] = 0
# Set a sane default key_buffer size
mysql_config['key_buffer'] = self.human_to_bytes('32M')
if 'preferred-storage-engine' in config:
# Storage engine configuration
preferred_engines = config['preferred-storage-engine'].split(',')
chunk_size = int(dataset_bytes / len(preferred_engines))
mysql_config['innodb_flush_log_at_trx_commit'] = 1
mysql_config['sync_binlog'] = 1
if 'InnoDB' in preferred_engines:
mysql_config['innodb_buffer_pool_size'] = chunk_size
if config['tuning-level'] == 'fast':
mysql_config['innodb_flush_log_at_trx_commit'] = 2
else:
mysql_config['innodb_buffer_pool_size'] = 0
mysql_config['default_storage_engine'] = preferred_engines[0]
if 'MyISAM' in preferred_engines:
mysql_config['key_buffer'] = chunk_size
if config['tuning-level'] == 'fast':
mysql_config['sync_binlog'] = 0
return mysql_config

61
hooks/percona_hooks.py
View File

@@ -49,13 +49,10 @@ from percona_utils import (
relation_clear,
assert_charm_supports_ipv6,
unit_sorted,
get_db_helper,
)
from charmhelpers.contrib.database.mysql import (
get_allowed_units,
get_mysql_password,
get_mysql_root_password,
parse_config,
configure_db,
PerconaClusterHelper,
)
from charmhelpers.contrib.hahelpers.cluster import (
peer_units,
@@ -87,8 +84,10 @@ def install():
add_source(config('source'))
configure_mysql_root_password(config('root-password'))
mysql_password = get_mysql_password(username='sstuser',
password=config('sst-password'))
db_helper = get_db_helper()
cfg_passwd = config('sst-password')
mysql_password = db_helper.get_mysql_password(username='sstuser',
password=cfg_passwd)
# Render base configuration (no cluster)
render_config(mysql_password=mysql_password)
apt_update(fatal=True)
@@ -101,8 +100,10 @@ def render_config(clustered=False, hosts=[], mysql_password=None):
os.makedirs(os.path.dirname(MY_CNF))
if not mysql_password:
mysql_password = get_mysql_password(username='sstuser',
password=config('sst-password'))
db_helper = get_db_helper()
cfg_passwd = config('sst-password')
mysql_password = db_helper.get_mysql_password(username='sstuser',
password=cfg_passwd)
context = {
'cluster_name': 'juju_cluster',
@@ -123,7 +124,7 @@ def render_config(clustered=False, hosts=[], mysql_password=None):
else:
context['ipv6'] = False
context.update(parse_config())
context.update(PerconaClusterHelper().parse_config())
render(os.path.basename(MY_CNF), MY_CNF, context, perms=0o444)
@@ -202,23 +203,19 @@ def db_changed(relation_id=None, unit=None, admin=None):
if admin not in [True, False]:
admin = relation_type() == 'db-admin'
database_name, _ = remote_unit().split("/")
username = database_name
password = configure_db(relation_get('private-address',
unit=unit,
rid=relation_id),
database_name,
username,
admin=admin)
db_name, _ = remote_unit().split("/")
username = db_name
db_helper = get_db_helper()
addr = relation_get('private-address', unit=unit, rid=relation_id)
password = db_helper.configure_db(addr, db_name, username, admin=admin)
relation_set(relation_id=relation_id,
relation_settings={
'user': username,
'password': password,
'host': db_host,
'database': database_name,
}
)
'database': db_name,
})
def get_db_host(client_hostname):
@@ -232,7 +229,7 @@ def get_db_host(client_hostname):
return unit_get('private-address')
def configure_db_for_hosts(hosts, database, username):
def configure_db_for_hosts(hosts, database, username, db_helper):
"""Hosts may be a json-encoded list of hosts or a single hostname."""
try:
hosts = json.loads(hosts)
@@ -244,7 +241,7 @@ def configure_db_for_hosts(hosts, database, username):
hosts = [hosts]
for host in hosts:
password = configure_db(host, database, username)
password = db_helper.configure_db(host, database, username)
return password
@@ -280,7 +277,7 @@ def shared_db_changed(relation_id=None, unit=None):
db_host = unit_get('private-address')
access_network = config('access-network')
rpasswd = get_mysql_root_password()
db_helper = get_db_helper()
singleset = set(['database', 'username', 'hostname'])
if singleset.issubset(settings):
@@ -290,11 +287,11 @@ def shared_db_changed(relation_id=None, unit=None):
username = settings['username']
# NOTE: do this before querying access grants
password = configure_db_for_hosts(hostname, database, username)
password = configure_db_for_hosts(hostname, database, username,
db_helper)
allowed_units = get_allowed_units(database, username,
relation_id=relation_id,
db_root_password=rpasswd)
allowed_units = db_helper.get_allowed_units(database, username,
relation_id=relation_id)
allowed_units = unit_sorted(allowed_units)
allowed_units = ' '.join(allowed_units)
relation_set(relation_id=relation_id, allowed_units=allowed_units)
@@ -339,11 +336,11 @@ def shared_db_changed(relation_id=None, unit=None):
username = databases[db]['username']
# NOTE: do this before querying access grants
password = configure_db_for_hosts(hostname, database, username)
password = configure_db_for_hosts(hostname, database, username,
db_helper)
a_units = get_allowed_units(database, username,
relation_id=relation_id,
db_root_password=rpasswd)
a_units = db_helper.get_allowed_units(database, username,
relation_id=relation_id)
a_units = ' '.join(unit_sorted(a_units))
allowed_units['%s_allowed_units' % (db)] = a_units

13
hooks/percona_utils.py
View File

@@ -26,7 +26,6 @@ from charmhelpers.contrib.network.ip import (
get_ipv6_addr
)
from charmhelpers.contrib.database.mysql import (
get_mysql_root_password,
MySQLHelper,
)
@@ -133,9 +132,14 @@ SQL_SST_USER_SETUP_IPV6 = ("GRANT RELOAD, LOCK TABLES, REPLICATION CLIENT "
"BY '{}'")
def get_db_helper():
return MySQLHelper(rpasswdf_template='/var/lib/charm/{}/mysql.passwd',
upasswdf_template='/var/lib/charm/{}/mysql-{}.passwd')
def configure_sstuser(sst_password):
m_helper = MySQLHelper()
m_helper.connect(password=get_mysql_root_password())
m_helper = get_db_helper()
m_helper.connect(password=m_helper.get_mysql_root_password())
m_helper.execute(SQL_SST_USER_SETUP.format(sst_password))
m_helper.execute(SQL_SST_USER_SETUP_IPV6.format(sst_password))
@@ -147,7 +151,8 @@ def configure_mysql_root_password(password):
# Set both percona and mysql password options to cover
# both upstream and distro packages.
packages = ["percona-server-server", "mysql-server"]
root_pass = get_mysql_root_password(password)
m_helper = get_db_helper()
root_pass = m_helper.get_mysql_root_password(password)
for package in packages:
dconf.stdin.write("%s %s/root_password password %s\n" %
(package, package, root_pass))