Browse Source

Rationalize configuration for percona/galera, add generic helpers for parsing mysql configuration options, use mysqlhelper for creation of SST user

changes/65/287065/1
James Page 7 years ago
parent
commit
5410686642
6 changed files with 158 additions and 49 deletions
  1. +7
    -23
      config.yaml
  2. +115
    -0
      hooks/mysql.py
  3. +15
    -15
      hooks/percona_hooks.py
  4. +6
    -10
      hooks/percona_utils.py
  5. +1
    -1
      revision
  6. +14
    -0
      templates/my.cnf

+ 7
- 23
config.yaml View File

@ -1,28 +1,18 @@
options:
dataset-size:
default: '80%'
description: How much data do you want to keep in memory in the DB. This will be used to tune settings in the database server appropriately. Any more specific settings will override these defaults though. This currently sets innodb_buffer_pool_size or key_cache_size depending on the setting in preferred-storage-engine. If query-cache-type is set to 'ON' or 'DEMAND' 20% of this is given to query-cache-size. Suffix this value with 'K','M','G', or 'T' to get the relevant kilo/mega/etc. bytes. If suffixed with %, one will get that percentage of RAM devoted to dataset and (if enabled) query cache.
type: string
preferred-storage-engine:
default: InnoDB
type: string
description: Tune the server for usage of this storage engine. Other possible value is MyISAM. Comma separated will cause settings to split resources evenly among given engines.
tuning-level:
default: safest
type: string
description: Valid values are 'safest', 'fast', and 'unsafe'. If set to safest, all settings are tuned to have maximum safety at the cost of performance. Fast will turn off most controls, but may lose data on crashes. unsafe will turn off all protections.
query-cache-type:
default: "OFF"
type: string
description: Query cache is usually a good idea, but can hurt concurrency. Valid values are "OFF", "ON", or "DEMAND". http://dev.mysql.com/doc/refman/5.1/en/server-system-variables.html#sysvar_query_cache_type
query-cache-size:
default: -1
type: int
description: Override the computed version from dataset-size. Still works if query-cache-type is "OFF" since sessions can override the cache type setting on their own.
description: How much data do you want to keep in memory in the DB. This will be used to tune settings in the database server appropriately. Suffix this value with 'K','M','G', or 'T' to get the relevant kilo/mega/etc. bytes. If suffixed with %, one will get that percentage of RAM devoted to dataset.
max-connections:
default: -1
type: int
description: Maximum connections to allow. -1 means use the server's compiled in default.
root-password:
type: string
description: Root password for MySQL access; must be configured pre-deployment for Active-Active clusters.
sst-password:
type: string
description: Re-sync account password for new cluster nodes; must be configured pre-deployment for Active-Active clusters.
vip:
type: string
description: Virtual IP to use to front Percona XtraDB Cluster in active/active HA configuration
@ -42,9 +32,3 @@ options:
type: int
default: 5490
description: Default multicast port number that will be used to communicate between HA Cluster nodes.
root-password:
type: string
description: Root password for MySQL access; must be configured pre-deployment for Active-Active clusters.
sst-password:
type: string
description: Re-sync account password for new cluster nodes; must be configured pre-deployment for Active-Active clusters.

+ 115
- 0
hooks/mysql.py View File

@ -2,8 +2,13 @@
# TODO: Contribute to charm-helpers
import socket
import os
import re
import sys
import platform
from string import upper
from charmhelpers.core.host import pwgen, write_file, mkdir
from charmhelpers.core.hookenv import unit_get, service_name
from charmhelpers.core.hookenv import config as config_get
from charmhelpers.fetch import apt_install, filter_installed_packages
@ -156,3 +161,113 @@ def configure_db(hostname,
m_helper.create_admin_grant(username,
remote_ip, password)
return password
# Going for the biggest page size to avoid wasted bytes. InnoDB page size is
# 16MB
DEFAULT_PAGE_SIZE = 16 * 1024 * 1024
def human_to_bytes(human):
''' Convert human readable configuration options to bytes '''
num_re = re.compile('^[0-9]+$')
if num_re.match(human):
return human
factors = {
'K': 1024,
'M': 1048576,
'G': 1073741824,
'T': 1099511627776
}
modifier = human[-1]
if modifier in factors:
return int(human[:-1]) * factors[modifier]
if modifier == '%':
total_ram = human_to_bytes(get_mem_total())
if is_32bit_system() and total_ram > sys_mem_limit():
total_ram = sys_mem_limit()
factor = int(human[:-1]) * 0.01
pctram = total_ram * factor
return int(pctram - (pctram % DEFAULT_PAGE_SIZE))
raise ValueError("Can only convert K,M,G, or T")
def is_32bit_system():
''' Determine whether system is 32 or 64 bit '''
try:
_is_32bit_system = sys.maxsize < 2 ** 32
except OverflowError:
_is_32bit_system = True
return _is_32bit_system
def sys_mem_limit():
''' Determine the default memory limit for the current service unit '''
if platform.machine() in ['armv7l']:
_mem_limit = human_to_bytes('2700M') # experimentally determined
else:
# Limit for x86 based 32bit systems
_mem_limit = human_to_bytes('4G')
return _mem_limit
def get_mem_total():
''' Calculate the total memory in the current service unit '''
with open('/proc/meminfo') as meminfo_file:
for line in meminfo_file:
(key, mem) = line.split(':', 2)
if key == 'MemTotal':
(mtot, modifier) = mem.strip().split(' ')
return '%s%s' % (mtot, upper(modifier[0]))
def parse_config():
''' Parse charm configuration and calculate values for config files '''
config = config_get()
mysql_config = {}
if 'max-connections' in config:
mysql_config['max_connections'] = config['max-connections']
# Total memory available for dataset
dataset_bytes = human_to_bytes(config['dataset-size'])
mysql_config['dataset_bytes'] = dataset_bytes
if 'query-cache-type' in config:
# Query Cache Configuration
mysql_config['query_cache_size'] = config['query-cache-size']
if (config['query-cache-size'] == -1 and
config['query-cache-type'] in ['ON', 'DEMAND']):
# Calculate the query cache size automatically
qcache_bytes = (dataset_bytes * 0.20)
qcache_bytes = int(qcache_bytes -
(qcache_bytes % DEFAULT_PAGE_SIZE))
mysql_config['query_cache_size'] = qcache_bytes
dataset_bytes -= qcache_bytes
# 5.5 allows the words, but not 5.1
if config['query-cache-type'] == 'ON':
mysql_config['query_cache_type'] = 1
elif config['query-cache-type'] == 'DEMAND':
mysql_config['query_cache_type'] = 2
else:
mysql_config['query_cache_type'] = 0
# Set a sane default key_buffer size
mysql_config['key_buffer'] = human_to_bytes('32M')
if 'preferred-storage-engine' in config:
# Storage engine configuration
preferred_engines = config['preferred-storage-engine'].split(',')
chunk_size = int(dataset_bytes / len(preferred_engines))
mysql_config['innodb_flush_log_at_trx_commit'] = 1
mysql_config['sync_binlog'] = 1
if 'InnoDB' in preferred_engines:
mysql_config['innodb_buffer_pool_size'] = chunk_size
if config['tuning-level'] == 'fast':
mysql_config['innodb_flush_log_at_trx_commit'] = 2
else:
mysql_config['innodb_buffer_pool_size'] = 0
mysql_config['default_storage_engine'] = preferred_engines[0]
if 'MyISAM' in preferred_engines:
mysql_config['key_buffer'] = chunk_size
if config['tuning-level'] == 'fast':
mysql_config['sync_binlog'] = 0
return mysql_config

+ 15
- 15
hooks/percona_hooks.py View File

@ -1,5 +1,4 @@
#!/usr/bin/python
# TODO: Support relevant configuration options
# TODO: Support changes to root and sstuser passwords
import sys
@ -19,7 +18,8 @@ from charmhelpers.core.hookenv import (
)
from charmhelpers.core.host import (
service_restart,
file_hash
file_hash,
write_file
)
from charmhelpers.fetch import (
apt_update,
@ -37,7 +37,7 @@ from percona_utils import (
configure_mysql_root_password,
relation_clear,
)
from mysql import get_mysql_password
from mysql import get_mysql_password, parse_config
from charmhelpers.contrib.hahelpers.cluster import (
peer_units,
oldest_peer,
@ -67,18 +67,18 @@ def install():
def render_config(clustered=False, hosts=[]):
if not os.path.exists(os.path.dirname(MY_CNF)):
os.makedirs(os.path.dirname(MY_CNF))
with open(MY_CNF, 'w') as conf:
context = {
'cluster_name': 'juju_cluster',
'private_address': get_host_ip(),
'clustered': clustered,
'cluster_hosts': ",".join(hosts),
'sst_password': get_mysql_password(username='sstuser',
password=config('sst-password'))
}
conf.write(render_template(os.path.basename(MY_CNF), context))
# TODO: set 0640 and change group to mysql if avaliable
os.chmod(MY_CNF, 0644)
context = {
'cluster_name': 'juju_cluster',
'private_address': get_host_ip(),
'clustered': clustered,
'cluster_hosts': ",".join(hosts),
'sst_password': get_mysql_password(username='sstuser',
password=config('sst-password'))
}
context.update(parse_config())
write_file(path=MY_CNF,
content=render_template(os.path.basename(MY_CNF), context),
perms=0444)
@hooks.hook('cluster-relation-joined')


+ 6
- 10
hooks/percona_utils.py View File

@ -18,7 +18,7 @@ from charmhelpers.fetch import (
apt_install,
filter_installed_packages,
)
from mysql import get_mysql_root_password
from mysql import get_mysql_root_password, MySQLHelper
try:
import jinja2
@ -97,18 +97,14 @@ def get_cluster_hosts():
unit, relid)))
return hosts
# TODO: refactor to use mysql helper when it support setting arbitary
# permissions
SQL_SST_USER_SETUP = """mysql --user=root --password={} << EOF
CREATE USER 'sstuser'@'localhost' IDENTIFIED BY '{}';
GRANT RELOAD, LOCK TABLES, REPLICATION CLIENT ON *.* TO 'sstuser'@'localhost';
EOF"""
SQL_SST_USER_SETUP = "GRANT RELOAD, LOCK TABLES, REPLICATION CLIENT ON *.*" \
" TO 'sstuser'@'localhost' IDENTIFIED BY '{}'"
def configure_sstuser(sst_password):
subprocess.check_call(SQL_SST_USER_SETUP.format(get_mysql_root_password(),
sst_password),
shell=True)
m_helper = MySQLHelper()
m_helper.connect(password=get_mysql_root_password())
m_helper.execute(SQL_SST_USER_SETUP.format(sst_password))
# TODO: mysql charmhelper


+ 1
- 1
revision View File

@ -1 +1 @@
32
45

+ 14
- 0
templates/my.cnf View File

@ -1,3 +1,4 @@
# Juju managed file - don't change as charm will overwrite your changed!
[mysqld]
datadir=/var/lib/mysql
@ -36,4 +37,17 @@ wsrep_cluster_name={{ cluster_name }}
# Authentication for SST method
wsrep_sst_auth="sstuser:{{ sst_password }}"
{% if max_connections != -1 %}
max_connections = {{ max_connections }}
{% endif %}
# Fine tuning
key_buffer_size = {{ key_buffer }}
table_cache = 512
max_allowed_packet = 16M
# InnoDB buffer should consume 100% of the bytes of the dataset size
# query cache is not supported with Active/Active configuration
innodb_buffer_pool_size = {{ dataset_bytes }}
!includedir /etc/mysql/conf.d/

Loading…
Cancel
Save