Rationalize configuration for percona/galera, add generic helpers for parsing mysql configuration options, use mysqlhelper for creation of SST user

config.yaml

@@ -1,28 +1,18 @@
 options:
     dataset-size:
         default: '80%'
-        description: How much data do you want to keep in memory in the DB. This will be used to tune settings in the database server appropriately. Any more specific settings will override these defaults though. This currently sets innodb_buffer_pool_size or key_cache_size depending on the setting in preferred-storage-engine. If query-cache-type is set to 'ON' or 'DEMAND' 20% of this is given to query-cache-size. Suffix this value with 'K','M','G', or 'T' to get the relevant kilo/mega/etc. bytes. If suffixed with %, one will get that percentage of RAM devoted to dataset and (if enabled) query cache.
         type: string
-    preferred-storage-engine:
-        default: InnoDB
-        type: string
-        description: Tune the server for usage of this storage engine. Other possible value is MyISAM. Comma separated will cause settings to split resources evenly among given engines.
-    tuning-level:
-        default: safest
-        type: string
-        description: Valid values are 'safest', 'fast', and 'unsafe'. If set to safest, all settings are tuned to have maximum safety at the cost of performance. Fast will turn off most controls, but may lose data on crashes. unsafe will turn off all protections.
-    query-cache-type:
-        default: "OFF"
-        type: string
-        description: Query cache is usually a good idea, but can hurt concurrency. Valid values are "OFF", "ON", or "DEMAND". http://dev.mysql.com/doc/refman/5.1/en/server-system-variables.html#sysvar_query_cache_type
-    query-cache-size:
-        default: -1
-        type: int
-        description: Override the computed version from dataset-size. Still works if query-cache-type is "OFF" since sessions can override the cache type setting on their own.
+        description: How much data do you want to keep in memory in the DB. This will be used to tune settings in the database server appropriately. Suffix this value with 'K','M','G', or 'T' to get the relevant kilo/mega/etc. bytes. If suffixed with %, one will get that percentage of RAM devoted to dataset.
     max-connections:
         default: -1
         type: int
         description: Maximum connections to allow. -1 means use the server's compiled in default.
+    root-password:
+        type: string
+        description: Root password for MySQL access; must be configured pre-deployment for Active-Active clusters.
+    sst-password:
+        type: string
+        description: Re-sync account password for new cluster nodes; must be configured pre-deployment for Active-Active clusters.
     vip:
         type: string
         description: Virtual IP to use to front Percona XtraDB Cluster in active/active HA configuration
@@ -42,9 +32,3 @@ options:
         type: int
         default: 5490
         description: Default multicast port number that will be used to communicate between HA Cluster nodes.
-    root-password:
-        type: string
-        description: Root password for MySQL access; must be configured pre-deployment for Active-Active clusters.
-    sst-password:
-        type: string
-        description: Re-sync account password for new cluster nodes; must be configured pre-deployment for Active-Active clusters.

hooks/mysql.py

@@ -2,8 +2,13 @@
 # TODO: Contribute to charm-helpers
 import socket
 import os
+import re
+import sys
+import platform
+from string import upper
 from charmhelpers.core.host import pwgen, write_file, mkdir
 from charmhelpers.core.hookenv import unit_get, service_name
+from charmhelpers.core.hookenv import config as config_get
 from charmhelpers.fetch import apt_install, filter_installed_packages
 
 
@@ -156,3 +161,113 @@ def configure_db(hostname,
             m_helper.create_admin_grant(username,
                                         remote_ip, password)
     return password
+
+# Going for the biggest page size to avoid wasted bytes. InnoDB page size is
+# 16MB
+DEFAULT_PAGE_SIZE = 16 * 1024 * 1024
+
+
+def human_to_bytes(human):
+    ''' Convert human readable configuration options to bytes '''
+    num_re = re.compile('^[0-9]+$')
+    if num_re.match(human):
+        return human
+    factors = {
+        'K': 1024,
+        'M': 1048576,
+        'G': 1073741824,
+        'T': 1099511627776
+    }
+    modifier = human[-1]
+    if modifier in factors:
+        return int(human[:-1]) * factors[modifier]
+    if modifier == '%':
+        total_ram = human_to_bytes(get_mem_total())
+        if is_32bit_system() and total_ram > sys_mem_limit():
+            total_ram = sys_mem_limit()
+        factor = int(human[:-1]) * 0.01
+        pctram = total_ram * factor
+        return int(pctram - (pctram % DEFAULT_PAGE_SIZE))
+    raise ValueError("Can only convert K,M,G, or T")
+
+
+def is_32bit_system():
+    ''' Determine whether system is 32 or 64 bit '''
+    try:
+        _is_32bit_system = sys.maxsize < 2 ** 32
+    except OverflowError:
+        _is_32bit_system = True
+    return _is_32bit_system
+
+
+def sys_mem_limit():
+    ''' Determine the default memory limit for the current service unit '''
+    if platform.machine() in ['armv7l']:
+        _mem_limit = human_to_bytes('2700M')  # experimentally determined
+    else:
+        # Limit for x86 based 32bit systems
+        _mem_limit = human_to_bytes('4G')
+    return _mem_limit
+
+
+def get_mem_total():
+    ''' Calculate the total memory in the current service unit '''
+    with open('/proc/meminfo') as meminfo_file:
+        for line in meminfo_file:
+            (key, mem) = line.split(':', 2)
+            if key == 'MemTotal':
+                (mtot, modifier) = mem.strip().split(' ')
+                return '%s%s' % (mtot, upper(modifier[0]))
+
+
+def parse_config():
+    ''' Parse charm configuration and calculate values for config files '''
+    config = config_get()
+    mysql_config = {}
+    if 'max-connections' in config:
+        mysql_config['max_connections'] = config['max-connections']
+
+    # Total memory available for dataset
+    dataset_bytes = human_to_bytes(config['dataset-size'])
+    mysql_config['dataset_bytes'] = dataset_bytes
+
+    if 'query-cache-type' in config:
+        # Query Cache Configuration
+        mysql_config['query_cache_size'] = config['query-cache-size']
+        if (config['query-cache-size'] == -1 and
+            config['query-cache-type'] in ['ON', 'DEMAND']):
+            # Calculate the query cache size automatically
+            qcache_bytes = (dataset_bytes * 0.20)
+            qcache_bytes = int(qcache_bytes -
+                               (qcache_bytes % DEFAULT_PAGE_SIZE))
+            mysql_config['query_cache_size'] = qcache_bytes
+            dataset_bytes -= qcache_bytes
+        # 5.5 allows the words, but not 5.1
+        if config['query-cache-type'] == 'ON':
+            mysql_config['query_cache_type'] = 1
+        elif config['query-cache-type'] == 'DEMAND':
+            mysql_config['query_cache_type'] = 2
+        else:
+            mysql_config['query_cache_type'] = 0
+
+    # Set a sane default key_buffer size
+    mysql_config['key_buffer'] = human_to_bytes('32M')
+
+    if 'preferred-storage-engine' in config:
+        # Storage engine configuration
+        preferred_engines = config['preferred-storage-engine'].split(',')
+        chunk_size = int(dataset_bytes / len(preferred_engines))
+        mysql_config['innodb_flush_log_at_trx_commit'] = 1
+        mysql_config['sync_binlog'] = 1
+        if 'InnoDB' in preferred_engines:
+            mysql_config['innodb_buffer_pool_size'] = chunk_size
+            if config['tuning-level'] == 'fast':
+                mysql_config['innodb_flush_log_at_trx_commit'] = 2
+        else:
+            mysql_config['innodb_buffer_pool_size'] = 0
+        mysql_config['default_storage_engine'] = preferred_engines[0]
+        if 'MyISAM' in preferred_engines:
+            mysql_config['key_buffer'] = chunk_size
+        if config['tuning-level'] == 'fast':
+            mysql_config['sync_binlog'] = 0
+    return mysql_config

hooks/percona_hooks.py

@@ -1,5 +1,4 @@
 #!/usr/bin/python
-# TODO: Support relevant configuration options
 # TODO: Support changes to root and sstuser passwords
 
 import sys
@@ -19,7 +18,8 @@ from charmhelpers.core.hookenv import (
 )
 from charmhelpers.core.host import (
     service_restart,
-    file_hash
+    file_hash,
+    write_file
 )
 from charmhelpers.fetch import (
     apt_update,
@@ -37,7 +37,7 @@ from percona_utils import (
     configure_mysql_root_password,
     relation_clear,
 )
-from mysql import get_mysql_password
+from mysql import get_mysql_password, parse_config
 from charmhelpers.contrib.hahelpers.cluster import (
     peer_units,
     oldest_peer,
@@ -67,18 +67,18 @@ def install():
 def render_config(clustered=False, hosts=[]):
     if not os.path.exists(os.path.dirname(MY_CNF)):
         os.makedirs(os.path.dirname(MY_CNF))
-    with open(MY_CNF, 'w') as conf:
-        context = {
-            'cluster_name': 'juju_cluster',
-            'private_address': get_host_ip(),
-            'clustered': clustered,
-            'cluster_hosts': ",".join(hosts),
-            'sst_password': get_mysql_password(username='sstuser',
-                                               password=config('sst-password'))
-        }
-        conf.write(render_template(os.path.basename(MY_CNF), context))
-    # TODO: set 0640 and change group to mysql if avaliable
-    os.chmod(MY_CNF, 0644)
+    context = {
+        'cluster_name': 'juju_cluster',
+        'private_address': get_host_ip(),
+        'clustered': clustered,
+        'cluster_hosts': ",".join(hosts),
+        'sst_password': get_mysql_password(username='sstuser',
+                                           password=config('sst-password'))
+    }
+    context.update(parse_config())
+    write_file(path=MY_CNF,
+               content=render_template(os.path.basename(MY_CNF), context),
+               perms=0444)
 
 
 @hooks.hook('cluster-relation-joined')

hooks/percona_utils.py

@@ -18,7 +18,7 @@ from charmhelpers.fetch import (
     apt_install,
     filter_installed_packages,
 )
-from mysql import get_mysql_root_password
+from mysql import get_mysql_root_password, MySQLHelper
 
 try:
     import jinja2
@@ -97,18 +97,14 @@ def get_cluster_hosts():
                                      unit, relid)))
     return hosts
 
-# TODO: refactor to use mysql helper when it support setting arbitary
-#       permissions
-SQL_SST_USER_SETUP = """mysql --user=root --password={} << EOF
-CREATE USER 'sstuser'@'localhost' IDENTIFIED BY '{}';
-GRANT RELOAD, LOCK TABLES, REPLICATION CLIENT ON *.* TO 'sstuser'@'localhost';
-EOF"""
+SQL_SST_USER_SETUP = "GRANT RELOAD, LOCK TABLES, REPLICATION CLIENT ON *.*" \
+" TO 'sstuser'@'localhost' IDENTIFIED BY '{}'"
 
 
 def configure_sstuser(sst_password):
-    subprocess.check_call(SQL_SST_USER_SETUP.format(get_mysql_root_password(),
-                                                    sst_password),
-                          shell=True)
+    m_helper = MySQLHelper()
+    m_helper.connect(password=get_mysql_root_password())
+    m_helper.execute(SQL_SST_USER_SETUP.format(sst_password))
 
 
 # TODO: mysql charmhelper

revision

@@ -1 +1 @@
-32
+45

templates/my.cnf

@@ -1,3 +1,4 @@
+# Juju managed file - don't change as charm will overwrite your changed!
 [mysqld]
 
 datadir=/var/lib/mysql
@@ -36,4 +37,17 @@ wsrep_cluster_name={{ cluster_name }}
 # Authentication for SST method
 wsrep_sst_auth="sstuser:{{ sst_password }}"
 
+{% if max_connections != -1 %}
+max_connections = {{ max_connections }}
+{% endif %}
+
+# Fine tuning
+key_buffer_size = {{ key_buffer }}
+table_cache = 512
+max_allowed_packet = 16M
+
+# InnoDB buffer should consume 100% of the bytes of the dataset size
+# query cache is not supported with Active/Active configuration
+innodb_buffer_pool_size = {{ dataset_bytes }}
+
 !includedir /etc/mysql/conf.d/