more

hooks/charmhelpers/contrib/database/mysql.py

@@ -42,10 +42,13 @@ except ImportError:
     import MySQLdb
 
 
-class MySQLHelper():
+class MySQLHelper(object):
 
-    def __init__(self, host='localhost'):
+    def __init__(self, rpasswdf_template, upasswdf_template, host='localhost'):
         self.host = host
+        # Password file path templates
+        self.root_passwd_file_template = rpasswdf_template
+        self.user_passwd_file_template = upasswdf_template
 
     def connect(self, user='root', password=None):
         self.connection = MySQLdb.connect(user=user, host=self.host,
@@ -124,42 +127,50 @@ class MySQLHelper():
         finally:
             cursor.close()
 
-# These are percona-only since mysql charm uses /var/lib/mysql/...
-_root_passwd = '/var/lib/charm/{}/mysql.passwd'
-_named_passwd = '/var/lib/charm/{}/mysql-{}.passwd'
+    def migrate_passwords_to_peer_relation(self):
+        """Migrate any passwords storage on disk to cluster peer relation."""
+        template = self.user_passwd_file_template
+        for f in glob.glob(template.format(service_name(), '*')):
+            _key = os.path.basename(f)
+            with open(f, 'r') as passwd:
+                _value = passwd.read().strip()
 
+            try:
+                peer_store(_key, _value)
+                os.unlink(f)
+            except ValueError:
+                # NOTE cluster relation not yet ready - skip for now
+                pass
 
-def get_mysql_password_on_disk(username=None, password=None, passwd_file=None):
-    """Retrieve, generate or store a mysql password for the provided username
-    on disk."""
-    if not passwd_file:
+    def get_mysql_password_on_disk(self, username=None, password=None):
+        """Retrieve, generate or store a mysql password for the provided
+        username on disk."""
         if username:
-            passwd_file = _named_passwd.format(service_name(), username)
+            template = self.user_passwd_file_template
+            passwd_file = template.format(service_name(), username)
         else:
-            passwd_file = _root_passwd.format(service_name())
+            template = self.root_passwd_file_template
+            passwd_file = template.format(service_name())
 
         _password = None
         if os.path.exists(passwd_file):
             with open(passwd_file, 'r') as passwd:
                 _password = passwd.read().strip()
         else:
-        mkdir(os.path.dirname(passwd_file),
-              owner='root', group='root',
+            mkdir(os.path.dirname(passwd_file), owner='root', group='root',
                   perms=0o770)
             # Force permissions - for some reason the chmod in makedirs fails
             os.chmod(os.path.dirname(passwd_file), 0o770)
             _password = password or pwgen(length=32)
-        write_file(passwd_file, _password,
-                   owner='root', group='root',
+            write_file(passwd_file, _password, owner='root', group='root',
                        perms=0o660)
 
         return _password
 
-
-def get_mysql_password(username=None, password=None, passwd_file=None):
-    """Retrieve, generate or store a mysql password for the provided username
-    using peer relation cluster."""
-    migrate_passwords_to_peer_relation()
+    def get_mysql_password(self, username=None, password=None):
+        """Retrieve, generate or store a mysql password for the provided
+        username using peer relation cluster."""
+        self.migrate_passwords_to_peer_relation()
         if username:
             _key = '{}.passwd'.format(username)
         else:
@@ -172,33 +183,55 @@ def get_mysql_password(username=None, password=None, passwd_file=None):
                 peer_store(_key, _password)
         except ValueError:
             # cluster relation is not yet started; use on-disk
-        _password = get_mysql_password_on_disk(username, password,
-                                               passwd_file=passwd_file)
+            _password = self.get_mysql_password_on_disk(username, password)
 
         return _password
 
-
-def migrate_passwords_to_peer_relation():
-    """Migrate any passwords storage on disk to cluster peer relation."""
-    for f in glob.glob('/var/lib/charm/{}/*.passwd'.format(service_name())):
-        _key = os.path.basename(f)
-        with open(f, 'r') as passwd:
-            _value = passwd.read().strip()
-        try:
-            peer_store(_key, _value)
-            os.unlink(f)
-        except ValueError:
-            # NOTE cluster relation not yet ready - skip for now
-            pass
-
-
-def get_mysql_root_password(password=None, passwd_file=None):
+    def get_mysql_root_password(self, password=None):
         """Retrieve or generate mysql root password for service units."""
-    return get_mysql_password(username=None, password=password,
-                              passwd_file=passwd_file)
+        return self.get_mysql_password(username=None, password=password)
 
+    def get_allowed_units(self, database, username, relation_id=None):
+        """Get list of units with access grants for database with username.
 
-def configure_db(hostname, database, username, admin=False, passwd_file=None):
+        This is typically used to provide shared-db relations with a list of
+        which units have been granted access to the given database.
+        """
+        self.connect(password=self.get_mysql_root_password())
+        allowed_units = set()
+        for unit in related_units(relation_id):
+            settings = relation_get(rid=relation_id, unit=unit)
+            # First check for setting with prefix, then without
+            for attr in ["%s_hostname" % (database), 'hostname']:
+                hosts = settings.get(attr, None)
+                if hosts:
+                    break
+
+            if hosts:
+                # hostname can be json-encoded list of hostnames
+                try:
+                    hosts = json.loads(hosts)
+                except ValueError:
+                    hosts = [hosts]
+            else:
+                hosts = [settings['private-address']]
+
+            if hosts:
+                for host in hosts:
+                    if self.grant_exists(database, username, host):
+                        log("Grant exists for host '%s' on db '%s'" %
+                            (host, database), level=DEBUG)
+                        if unit not in allowed_units:
+                            allowed_units.add(unit)
+                    else:
+                        log("Grant does NOT exist for host '%s' on db '%s'" %
+                            (host, database), level=DEBUG)
+            else:
+                log("No hosts found for grant check", level=INFO)
+
+        return allowed_units
+
+    def configure_db(self, hostname, database, username, admin=False):
         """Configure access to database for username from hostname."""
         if config_get('prefer-ipv6'):
             remote_ip = hostname
@@ -211,28 +244,27 @@ def configure_db(hostname, database, username, admin=False, passwd_file=None):
         else:
             remote_ip = '127.0.0.1'
 
-    password = get_mysql_password(username, passwd_file=passwd_file)
-    m_helper = MySQLHelper()
-    m_helper.connect(password=get_mysql_root_password())
-    if not m_helper.database_exists(database):
-        m_helper.create_database(database)
+        self.connect(password=self.get_mysql_root_password())
+        if not self.database_exists(database):
+            self.create_database(database)
 
-    if not m_helper.grant_exists(database,
-                                 username,
-                                 remote_ip):
+        password = self.get_mysql_password(username)
+        if not self.grant_exists(database, username, remote_ip):
             if not admin:
-            m_helper.create_grant(database, username, remote_ip, password)
+                self.create_grant(database, username, remote_ip, password)
             else:
-            m_helper.create_admin_grant(username, remote_ip, password)
+                self.create_admin_grant(username, remote_ip, password)
 
         return password
 
-# Going for the biggest page size to avoid wasted bytes. InnoDB page size is
-# 16MB
-DEFAULT_PAGE_SIZE = 16 * 1024 * 1024
 
+class PerconaClusterHelper(object):
 
-def human_to_bytes(human):
+    # Going for the biggest page size to avoid wasted bytes. InnoDB page size is
+    # 16MB
+    DEFAULT_PAGE_SIZE = 16 * 1024 * 1024
+
+    def human_to_bytes(self, human):
         """Convert human readable configuration options to bytes."""
         num_re = re.compile('^[0-9]+$')
         if num_re.match(human):
@@ -249,36 +281,33 @@ def human_to_bytes(human):
             return int(human[:-1]) * factors[modifier]
 
         if modifier == '%':
-        total_ram = human_to_bytes(get_mem_total())
-        if is_32bit_system() and total_ram > sys_mem_limit():
-            total_ram = sys_mem_limit()
+            total_ram = self.human_to_bytes(self.get_mem_total())
+            if self.is_32bit_system() and total_ram > self.sys_mem_limit():
+                total_ram = self.sys_mem_limit()
             factor = int(human[:-1]) * 0.01
             pctram = total_ram * factor
-        return int(pctram - (pctram % DEFAULT_PAGE_SIZE))
+            return int(pctram - (pctram % self.DEFAULT_PAGE_SIZE))
 
         raise ValueError("Can only convert K,M,G, or T")
 
-
-def is_32bit_system():
+    def is_32bit_system(self):
         """Determine whether system is 32 or 64 bit."""
         try:
             return sys.maxsize < 2 ** 32
         except OverflowError:
             return False
 
-
-def sys_mem_limit():
+    def sys_mem_limit(self):
         """Determine the default memory limit for the current service unit."""
         if platform.machine() in ['armv7l']:
-        _mem_limit = human_to_bytes('2700M')  # experimentally determined
+            _mem_limit = self.human_to_bytes('2700M')  # experimentally determined
         else:
             # Limit for x86 based 32bit systems
-        _mem_limit = human_to_bytes('4G')
+            _mem_limit = self.human_to_bytes('4G')
 
         return _mem_limit
 
-
-def get_mem_total():
+    def get_mem_total(self):
         """Calculate the total memory in the current service unit."""
         with open('/proc/meminfo') as meminfo_file:
             for line in meminfo_file:
@@ -287,8 +316,7 @@ def get_mem_total():
                     mtot, modifier = mem.strip().split(' ')
                     return '%s%s' % (mtot, upper(modifier[0]))
 
-
-def parse_config():
+    def parse_config(self):
         """Parse charm configuration and calculate values for config files."""
         config = config_get()
         mysql_config = {}
@@ -296,7 +324,7 @@ def parse_config():
             mysql_config['max_connections'] = config['max-connections']
 
         # Total memory available for dataset
-    dataset_bytes = human_to_bytes(config['dataset-size'])
+        dataset_bytes = self.human_to_bytes(config['dataset-size'])
         mysql_config['dataset_bytes'] = dataset_bytes
 
         if 'query-cache-type' in config:
@@ -307,7 +335,7 @@ def parse_config():
                 # Calculate the query cache size automatically
                 qcache_bytes = (dataset_bytes * 0.20)
                 qcache_bytes = int(qcache_bytes -
-                               (qcache_bytes % DEFAULT_PAGE_SIZE))
+                                   (qcache_bytes % self.DEFAULT_PAGE_SIZE))
                 mysql_config['query_cache_size'] = qcache_bytes
                 dataset_bytes -= qcache_bytes
 
@@ -320,7 +348,7 @@ def parse_config():
                 mysql_config['query_cache_type'] = 0
 
         # Set a sane default key_buffer size
-    mysql_config['key_buffer'] = human_to_bytes('32M')
+        mysql_config['key_buffer'] = self.human_to_bytes('32M')
 
         if 'preferred-storage-engine' in config:
             # Storage engine configuration
@@ -343,45 +371,3 @@ def parse_config():
                 mysql_config['sync_binlog'] = 0
 
         return mysql_config
-
-
-def get_allowed_units(database, username, db_root_password, relation_id=None):
-    """Get list of units with access grants for database with username.
-
-    This is typically used to provide shared-db relations with a list of which
-    units have been granted access to the given database.
-    """
-    m_helper = MySQLHelper()
-    m_helper.connect(password=db_root_password)
-    allowed_units = set()
-    for unit in related_units(relation_id):
-        settings = relation_get(rid=relation_id, unit=unit)
-        # First check for setting with prefix, then without
-        for attr in ["%s_hostname" % (database), 'hostname']:
-            hosts = settings.get(attr, None)
-            if hosts:
-                break
-
-        if hosts:
-            # hostname can be json-encoded list of hostnames
-            try:
-                hosts = json.loads(hosts)
-            except ValueError:
-                hosts = [hosts]
-        else:
-            hosts = [settings['private-address']]
-
-        if hosts:
-            for host in hosts:
-                if m_helper.grant_exists(database, username, host):
-                    log("Grant exists for host '%s' on db '%s'" %
-                        (host, database), level=DEBUG)
-                    if unit not in allowed_units:
-                        allowed_units.add(unit)
-                else:
-                    log("Grant does NOT exist for host '%s' on db '%s'" %
-                        (host, database), level=DEBUG)
-        else:
-            log("No hosts found for grant check", level=INFO)
-
-    return allowed_units

hooks/percona_hooks.py

@@ -49,13 +49,10 @@ from percona_utils import (
     relation_clear,
     assert_charm_supports_ipv6,
     unit_sorted,
+    get_db_helper,
 )
 from charmhelpers.contrib.database.mysql import (
-    get_allowed_units,
-    get_mysql_password,
-    get_mysql_root_password,
-    parse_config,
-    configure_db,
+    PerconaClusterHelper,
 )
 from charmhelpers.contrib.hahelpers.cluster import (
     peer_units,
@@ -87,8 +84,10 @@ def install():
         add_source(config('source'))
 
     configure_mysql_root_password(config('root-password'))
-    mysql_password = get_mysql_password(username='sstuser',
-                                        password=config('sst-password'))
+    db_helper = get_db_helper()
+    cfg_passwd = config('sst-password')
+    mysql_password = db_helper.get_mysql_password(username='sstuser',
+                                                  password=cfg_passwd)
     # Render base configuration (no cluster)
     render_config(mysql_password=mysql_password)
     apt_update(fatal=True)
@@ -101,8 +100,10 @@ def render_config(clustered=False, hosts=[], mysql_password=None):
         os.makedirs(os.path.dirname(MY_CNF))
 
     if not mysql_password:
-        mysql_password = get_mysql_password(username='sstuser',
-                                            password=config('sst-password'))
+        db_helper = get_db_helper()
+        cfg_passwd = config('sst-password')
+        mysql_password = db_helper.get_mysql_password(username='sstuser',
+                                                      password=cfg_passwd)
 
     context = {
         'cluster_name': 'juju_cluster',
@@ -123,7 +124,7 @@ def render_config(clustered=False, hosts=[], mysql_password=None):
     else:
         context['ipv6'] = False
 
-    context.update(parse_config())
+    context.update(PerconaClusterHelper().parse_config())
     render(os.path.basename(MY_CNF), MY_CNF, context, perms=0o444)
 
 
@@ -202,23 +203,19 @@ def db_changed(relation_id=None, unit=None, admin=None):
 
     if admin not in [True, False]:
         admin = relation_type() == 'db-admin'
-    database_name, _ = remote_unit().split("/")
-    username = database_name
-    password = configure_db(relation_get('private-address',
-                                         unit=unit,
-                                         rid=relation_id),
-                            database_name,
-                            username,
-                            admin=admin)
+    db_name, _ = remote_unit().split("/")
+    username = db_name
+    db_helper = get_db_helper()
+    addr = relation_get('private-address', unit=unit, rid=relation_id)
+    password = db_helper.configure_db(addr, db_name, username, admin=admin)
 
     relation_set(relation_id=relation_id,
                  relation_settings={
                      'user': username,
                      'password': password,
                      'host': db_host,
-                     'database': database_name,
-                 }
-                 )
+                     'database': db_name,
+                 })
 
 
 def get_db_host(client_hostname):
@@ -232,7 +229,7 @@ def get_db_host(client_hostname):
     return unit_get('private-address')
 
 
-def configure_db_for_hosts(hosts, database, username):
+def configure_db_for_hosts(hosts, database, username, db_helper):
     """Hosts may be a json-encoded list of hosts or a single hostname."""
     try:
         hosts = json.loads(hosts)
@@ -244,7 +241,7 @@ def configure_db_for_hosts(hosts, database, username):
         hosts = [hosts]
 
     for host in hosts:
-        password = configure_db(host, database, username)
+        password = db_helper.configure_db(host, database, username)
 
     return password
 
@@ -280,7 +277,7 @@ def shared_db_changed(relation_id=None, unit=None):
             db_host = unit_get('private-address')
 
     access_network = config('access-network')
-    rpasswd = get_mysql_root_password()
+    db_helper = get_db_helper()
 
     singleset = set(['database', 'username', 'hostname'])
     if singleset.issubset(settings):
@@ -290,11 +287,11 @@ def shared_db_changed(relation_id=None, unit=None):
         username = settings['username']
 
         # NOTE: do this before querying access grants
-        password = configure_db_for_hosts(hostname, database, username)
+        password = configure_db_for_hosts(hostname, database, username,
+                                          db_helper)
 
-        allowed_units = get_allowed_units(database, username,
-                                          relation_id=relation_id,
-                                          db_root_password=rpasswd)
+        allowed_units = db_helper.get_allowed_units(database, username,
+                                                    relation_id=relation_id)
         allowed_units = unit_sorted(allowed_units)
         allowed_units = ' '.join(allowed_units)
         relation_set(relation_id=relation_id, allowed_units=allowed_units)
@@ -339,11 +336,11 @@ def shared_db_changed(relation_id=None, unit=None):
                 username = databases[db]['username']
 
                 # NOTE: do this before querying access grants
-                password = configure_db_for_hosts(hostname, database, username)
+                password = configure_db_for_hosts(hostname, database, username,
+                                                  db_helper)
 
-                a_units = get_allowed_units(database, username,
-                                            relation_id=relation_id,
-                                            db_root_password=rpasswd)
+                a_units = db_helper.get_allowed_units(database, username,
+                                                      relation_id=relation_id)
                 a_units = ' '.join(unit_sorted(a_units))
                 allowed_units['%s_allowed_units' % (db)] = a_units
 

hooks/percona_utils.py

@@ -26,7 +26,6 @@ from charmhelpers.contrib.network.ip import (
     get_ipv6_addr
 )
 from charmhelpers.contrib.database.mysql import (
-    get_mysql_root_password,
     MySQLHelper,
 )
 
@@ -133,9 +132,14 @@ SQL_SST_USER_SETUP_IPV6 = ("GRANT RELOAD, LOCK TABLES, REPLICATION CLIENT "
                            "BY '{}'")
 
 
+def get_db_helper():
+    return MySQLHelper(rpasswdf_template='/var/lib/charm/{}/mysql.passwd',
+                       upasswdf_template='/var/lib/charm/{}/mysql-{}.passwd')
+
+
 def configure_sstuser(sst_password):
-    m_helper = MySQLHelper()
-    m_helper.connect(password=get_mysql_root_password())
+    m_helper = get_db_helper()
+    m_helper.connect(password=m_helper.get_mysql_root_password())
     m_helper.execute(SQL_SST_USER_SETUP.format(sst_password))
     m_helper.execute(SQL_SST_USER_SETUP_IPV6.format(sst_password))
 
@@ -147,7 +151,8 @@ def configure_mysql_root_password(password):
     # Set both percona and mysql password options to cover
     # both upstream and distro packages.
     packages = ["percona-server-server", "mysql-server"]
-    root_pass = get_mysql_root_password(password)
+    m_helper = get_db_helper()
+    root_pass = m_helper.get_mysql_root_password(password)
     for package in packages:
         dconf.stdin.write("%s %s/root_password password %s\n" %
                           (package, package, root_pass))