diff --git a/config.yaml b/config.yaml index ffb00d9..ecf5c48 100644 --- a/config.yaml +++ b/config.yaml @@ -1,28 +1,18 @@ options: dataset-size: default: '80%' - description: How much data do you want to keep in memory in the DB. This will be used to tune settings in the database server appropriately. Any more specific settings will override these defaults though. This currently sets innodb_buffer_pool_size or key_cache_size depending on the setting in preferred-storage-engine. If query-cache-type is set to 'ON' or 'DEMAND' 20% of this is given to query-cache-size. Suffix this value with 'K','M','G', or 'T' to get the relevant kilo/mega/etc. bytes. If suffixed with %, one will get that percentage of RAM devoted to dataset and (if enabled) query cache. type: string - preferred-storage-engine: - default: InnoDB - type: string - description: Tune the server for usage of this storage engine. Other possible value is MyISAM. Comma separated will cause settings to split resources evenly among given engines. - tuning-level: - default: safest - type: string - description: Valid values are 'safest', 'fast', and 'unsafe'. If set to safest, all settings are tuned to have maximum safety at the cost of performance. Fast will turn off most controls, but may lose data on crashes. unsafe will turn off all protections. - query-cache-type: - default: "OFF" - type: string - description: Query cache is usually a good idea, but can hurt concurrency. Valid values are "OFF", "ON", or "DEMAND". http://dev.mysql.com/doc/refman/5.1/en/server-system-variables.html#sysvar_query_cache_type - query-cache-size: - default: -1 - type: int - description: Override the computed version from dataset-size. Still works if query-cache-type is "OFF" since sessions can override the cache type setting on their own. + description: How much data do you want to keep in memory in the DB. This will be used to tune settings in the database server appropriately. Suffix this value with 'K','M','G', or 'T' to get the relevant kilo/mega/etc. bytes. If suffixed with %, one will get that percentage of RAM devoted to dataset. max-connections: default: -1 type: int description: Maximum connections to allow. -1 means use the server's compiled in default. + root-password: + type: string + description: Root password for MySQL access; must be configured pre-deployment for Active-Active clusters. + sst-password: + type: string + description: Re-sync account password for new cluster nodes; must be configured pre-deployment for Active-Active clusters. vip: type: string description: Virtual IP to use to front Percona XtraDB Cluster in active/active HA configuration @@ -42,9 +32,3 @@ options: type: int default: 5490 description: Default multicast port number that will be used to communicate between HA Cluster nodes. - root-password: - type: string - description: Root password for MySQL access; must be configured pre-deployment for Active-Active clusters. - sst-password: - type: string - description: Re-sync account password for new cluster nodes; must be configured pre-deployment for Active-Active clusters. diff --git a/hooks/mysql.py b/hooks/mysql.py index aff4f4e..5579b5c 100644 --- a/hooks/mysql.py +++ b/hooks/mysql.py @@ -2,8 +2,13 @@ # TODO: Contribute to charm-helpers import socket import os +import re +import sys +import platform +from string import upper from charmhelpers.core.host import pwgen, write_file, mkdir from charmhelpers.core.hookenv import unit_get, service_name +from charmhelpers.core.hookenv import config as config_get from charmhelpers.fetch import apt_install, filter_installed_packages @@ -156,3 +161,113 @@ def configure_db(hostname, m_helper.create_admin_grant(username, remote_ip, password) return password + +# Going for the biggest page size to avoid wasted bytes. InnoDB page size is +# 16MB +DEFAULT_PAGE_SIZE = 16 * 1024 * 1024 + + +def human_to_bytes(human): + ''' Convert human readable configuration options to bytes ''' + num_re = re.compile('^[0-9]+$') + if num_re.match(human): + return human + factors = { + 'K': 1024, + 'M': 1048576, + 'G': 1073741824, + 'T': 1099511627776 + } + modifier = human[-1] + if modifier in factors: + return int(human[:-1]) * factors[modifier] + if modifier == '%': + total_ram = human_to_bytes(get_mem_total()) + if is_32bit_system() and total_ram > sys_mem_limit(): + total_ram = sys_mem_limit() + factor = int(human[:-1]) * 0.01 + pctram = total_ram * factor + return int(pctram - (pctram % DEFAULT_PAGE_SIZE)) + raise ValueError("Can only convert K,M,G, or T") + + +def is_32bit_system(): + ''' Determine whether system is 32 or 64 bit ''' + try: + _is_32bit_system = sys.maxsize < 2 ** 32 + except OverflowError: + _is_32bit_system = True + return _is_32bit_system + + +def sys_mem_limit(): + ''' Determine the default memory limit for the current service unit ''' + if platform.machine() in ['armv7l']: + _mem_limit = human_to_bytes('2700M') # experimentally determined + else: + # Limit for x86 based 32bit systems + _mem_limit = human_to_bytes('4G') + return _mem_limit + + +def get_mem_total(): + ''' Calculate the total memory in the current service unit ''' + with open('/proc/meminfo') as meminfo_file: + for line in meminfo_file: + (key, mem) = line.split(':', 2) + if key == 'MemTotal': + (mtot, modifier) = mem.strip().split(' ') + return '%s%s' % (mtot, upper(modifier[0])) + + +def parse_config(): + ''' Parse charm configuration and calculate values for config files ''' + config = config_get() + mysql_config = {} + if 'max-connections' in config: + mysql_config['max_connections'] = config['max-connections'] + + # Total memory available for dataset + dataset_bytes = human_to_bytes(config['dataset-size']) + mysql_config['dataset_bytes'] = dataset_bytes + + if 'query-cache-type' in config: + # Query Cache Configuration + mysql_config['query_cache_size'] = config['query-cache-size'] + if (config['query-cache-size'] == -1 and + config['query-cache-type'] in ['ON', 'DEMAND']): + # Calculate the query cache size automatically + qcache_bytes = (dataset_bytes * 0.20) + qcache_bytes = int(qcache_bytes - + (qcache_bytes % DEFAULT_PAGE_SIZE)) + mysql_config['query_cache_size'] = qcache_bytes + dataset_bytes -= qcache_bytes + # 5.5 allows the words, but not 5.1 + if config['query-cache-type'] == 'ON': + mysql_config['query_cache_type'] = 1 + elif config['query-cache-type'] == 'DEMAND': + mysql_config['query_cache_type'] = 2 + else: + mysql_config['query_cache_type'] = 0 + + # Set a sane default key_buffer size + mysql_config['key_buffer'] = human_to_bytes('32M') + + if 'preferred-storage-engine' in config: + # Storage engine configuration + preferred_engines = config['preferred-storage-engine'].split(',') + chunk_size = int(dataset_bytes / len(preferred_engines)) + mysql_config['innodb_flush_log_at_trx_commit'] = 1 + mysql_config['sync_binlog'] = 1 + if 'InnoDB' in preferred_engines: + mysql_config['innodb_buffer_pool_size'] = chunk_size + if config['tuning-level'] == 'fast': + mysql_config['innodb_flush_log_at_trx_commit'] = 2 + else: + mysql_config['innodb_buffer_pool_size'] = 0 + mysql_config['default_storage_engine'] = preferred_engines[0] + if 'MyISAM' in preferred_engines: + mysql_config['key_buffer'] = chunk_size + if config['tuning-level'] == 'fast': + mysql_config['sync_binlog'] = 0 + return mysql_config diff --git a/hooks/percona_hooks.py b/hooks/percona_hooks.py index c8766ee..46b18f3 100755 --- a/hooks/percona_hooks.py +++ b/hooks/percona_hooks.py @@ -1,5 +1,4 @@ #!/usr/bin/python -# TODO: Support relevant configuration options # TODO: Support changes to root and sstuser passwords import sys @@ -19,7 +18,8 @@ from charmhelpers.core.hookenv import ( ) from charmhelpers.core.host import ( service_restart, - file_hash + file_hash, + write_file ) from charmhelpers.fetch import ( apt_update, @@ -37,7 +37,7 @@ from percona_utils import ( configure_mysql_root_password, relation_clear, ) -from mysql import get_mysql_password +from mysql import get_mysql_password, parse_config from charmhelpers.contrib.hahelpers.cluster import ( peer_units, oldest_peer, @@ -67,18 +67,18 @@ def install(): def render_config(clustered=False, hosts=[]): if not os.path.exists(os.path.dirname(MY_CNF)): os.makedirs(os.path.dirname(MY_CNF)) - with open(MY_CNF, 'w') as conf: - context = { - 'cluster_name': 'juju_cluster', - 'private_address': get_host_ip(), - 'clustered': clustered, - 'cluster_hosts': ",".join(hosts), - 'sst_password': get_mysql_password(username='sstuser', - password=config('sst-password')) - } - conf.write(render_template(os.path.basename(MY_CNF), context)) - # TODO: set 0640 and change group to mysql if avaliable - os.chmod(MY_CNF, 0644) + context = { + 'cluster_name': 'juju_cluster', + 'private_address': get_host_ip(), + 'clustered': clustered, + 'cluster_hosts': ",".join(hosts), + 'sst_password': get_mysql_password(username='sstuser', + password=config('sst-password')) + } + context.update(parse_config()) + write_file(path=MY_CNF, + content=render_template(os.path.basename(MY_CNF), context), + perms=0444) @hooks.hook('cluster-relation-joined') diff --git a/hooks/percona_utils.py b/hooks/percona_utils.py index 83912be..0edd068 100644 --- a/hooks/percona_utils.py +++ b/hooks/percona_utils.py @@ -18,7 +18,7 @@ from charmhelpers.fetch import ( apt_install, filter_installed_packages, ) -from mysql import get_mysql_root_password +from mysql import get_mysql_root_password, MySQLHelper try: import jinja2 @@ -97,18 +97,14 @@ def get_cluster_hosts(): unit, relid))) return hosts -# TODO: refactor to use mysql helper when it support setting arbitary -# permissions -SQL_SST_USER_SETUP = """mysql --user=root --password={} << EOF -CREATE USER 'sstuser'@'localhost' IDENTIFIED BY '{}'; -GRANT RELOAD, LOCK TABLES, REPLICATION CLIENT ON *.* TO 'sstuser'@'localhost'; -EOF""" +SQL_SST_USER_SETUP = "GRANT RELOAD, LOCK TABLES, REPLICATION CLIENT ON *.*" \ +" TO 'sstuser'@'localhost' IDENTIFIED BY '{}'" def configure_sstuser(sst_password): - subprocess.check_call(SQL_SST_USER_SETUP.format(get_mysql_root_password(), - sst_password), - shell=True) + m_helper = MySQLHelper() + m_helper.connect(password=get_mysql_root_password()) + m_helper.execute(SQL_SST_USER_SETUP.format(sst_password)) # TODO: mysql charmhelper diff --git a/revision b/revision index f5c8955..ea90ee3 100644 --- a/revision +++ b/revision @@ -1 +1 @@ -32 +45 diff --git a/templates/my.cnf b/templates/my.cnf index be1e241..37e750b 100644 --- a/templates/my.cnf +++ b/templates/my.cnf @@ -1,3 +1,4 @@ +# Juju managed file - don't change as charm will overwrite your changed! [mysqld] datadir=/var/lib/mysql @@ -36,4 +37,17 @@ wsrep_cluster_name={{ cluster_name }} # Authentication for SST method wsrep_sst_auth="sstuser:{{ sst_password }}" +{% if max_connections != -1 %} +max_connections = {{ max_connections }} +{% endif %} + +# Fine tuning +key_buffer_size = {{ key_buffer }} +table_cache = 512 +max_allowed_packet = 16M + +# InnoDB buffer should consume 100% of the bytes of the dataset size +# query cache is not supported with Active/Active configuration +innodb_buffer_pool_size = {{ dataset_bytes }} + !includedir /etc/mysql/conf.d/