Implement validated restart on change
Add new metadata cache parameters. Use the charm-helpers' pausable_restart_on_change for config-changed Validate connectivity on restart to resolve LP Bug #1915842 See "Metadata Cache Options" https://dev.mysql.com/doc/mysql-router/8.0/en/mysql-router-conf-options.html Closes-Bug: #1915842 Change-Id: Icd30f9ade2c7aadbadaa57300c22786cde63957f
This commit is contained in:
parent
7f3719ffe4
commit
80339e4625
|
@ -20,3 +20,27 @@ options:
|
||||||
description: |
|
description: |
|
||||||
Base port number for RW interface. RO, xRW and xRO will
|
Base port number for RW interface. RO, xRW and xRO will
|
||||||
increment from base_port.
|
increment from base_port.
|
||||||
|
ttl:
|
||||||
|
type: float
|
||||||
|
default: .5
|
||||||
|
description: |
|
||||||
|
Time to live (in seconds) of information in the metadata cache.
|
||||||
|
Accepts either an integer or a floating point value. The granularity is
|
||||||
|
limited to milliseconds, where 0.001 equates to one millisecond.
|
||||||
|
Precision is truncated to the supported range; for example ttl=0.0119
|
||||||
|
is treated as 11 milliseconds. The value 0 means that the metadata
|
||||||
|
cache module queries the metadata continuously in a tight loop.
|
||||||
|
auth_cache_refresh_interval:
|
||||||
|
type: int
|
||||||
|
default: 2
|
||||||
|
description: |
|
||||||
|
Time (in seconds) between the auth-cache refresh attempts. Defaults to
|
||||||
|
2. The value must be smaller than auth_cache_ttl and ttl else Router
|
||||||
|
won't start.
|
||||||
|
auth_cache_ttl:
|
||||||
|
type: int
|
||||||
|
default: -1
|
||||||
|
description: |
|
||||||
|
Time (in seconds) until the cache becomes invalid if not refreshed.
|
||||||
|
Defaults to -1 (infinite). The value must be larger than
|
||||||
|
auth_cache_refresh_interval else Router won't start.
|
||||||
|
|
|
@ -16,6 +16,7 @@ import configparser
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import tenacity
|
||||||
|
|
||||||
import charms_openstack.charm
|
import charms_openstack.charm
|
||||||
import charms_openstack.adapters
|
import charms_openstack.adapters
|
||||||
|
@ -28,6 +29,7 @@ import charmhelpers.contrib.network.ip as ch_net_ip
|
||||||
import charmhelpers.contrib.database.mysql as mysql
|
import charmhelpers.contrib.database.mysql as mysql
|
||||||
|
|
||||||
import charmhelpers.contrib.openstack.templating as os_templating
|
import charmhelpers.contrib.openstack.templating as os_templating
|
||||||
|
import charmhelpers.contrib.openstack.utils as os_utils
|
||||||
|
|
||||||
|
|
||||||
# Flag Strings
|
# Flag Strings
|
||||||
|
@ -74,6 +76,10 @@ class MySQLRouterCharm(charms_openstack.charm.OpenStackCharm):
|
||||||
# For internal use with mysql.get_db_data
|
# For internal use with mysql.get_db_data
|
||||||
_unprefixed = "MRUP"
|
_unprefixed = "MRUP"
|
||||||
|
|
||||||
|
# mysql.MySQLdb._exceptions.OperationalError error 2013
|
||||||
|
# LP Bug #1915842
|
||||||
|
_waiting_for_initial_communication_packet_error = 2013
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def mysqlrouter_bin(self):
|
def mysqlrouter_bin(self):
|
||||||
"""Determine the path to the mysqlrouter binary.
|
"""Determine the path to the mysqlrouter binary.
|
||||||
|
@ -218,6 +224,24 @@ class MySQLRouterCharm(charms_openstack.charm.OpenStackCharm):
|
||||||
def mysqlrouter_group(self):
|
def mysqlrouter_group(self):
|
||||||
return "mysql"
|
return "mysql"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def ssl_ca(self):
|
||||||
|
"""Return the SSL Certificate Authority
|
||||||
|
|
||||||
|
:param self: Self
|
||||||
|
:type self: MySQLInnoDBClusterCharm instance
|
||||||
|
:returns: Cluster username
|
||||||
|
:rtype: str
|
||||||
|
:rtype: Union[str, None]
|
||||||
|
"""
|
||||||
|
if self.db_router_endpoint:
|
||||||
|
if self.db_router_endpoint.ssl_ca():
|
||||||
|
return json.loads(self.db_router_endpoint.ssl_ca())
|
||||||
|
|
||||||
|
@property
|
||||||
|
def restart_functions(self):
|
||||||
|
return {self.name: self.custom_restart_function}
|
||||||
|
|
||||||
def install(self):
|
def install(self):
|
||||||
"""Custom install function.
|
"""Custom install function.
|
||||||
|
|
||||||
|
@ -313,12 +337,16 @@ class MySQLRouterCharm(charms_openstack.charm.OpenStackCharm):
|
||||||
|
|
||||||
return states_to_check
|
return states_to_check
|
||||||
|
|
||||||
def check_mysql_connection(self):
|
def check_mysql_connection(self, reraise_on=None):
|
||||||
"""Check if an instance of MySQL is accessible.
|
"""Check if an instance of MySQL is accessible.
|
||||||
|
|
||||||
Attempt a connection to the given instance of mysql to determine if it
|
Attempt a connection to the given instance of mysql to determine if it
|
||||||
is running and accessible.
|
is running and accessible.
|
||||||
|
|
||||||
|
:param reraise_on: List of integer error codes to reraise exceptions.
|
||||||
|
For use with tenacity retries on specific
|
||||||
|
exceptions.
|
||||||
|
:type reraise_on: List[int]
|
||||||
:side effect: Uses get_db_helper to execute a connection to the DB.
|
:side effect: Uses get_db_helper to execute a connection to the DB.
|
||||||
:returns: True if connection succeeds or False if not
|
:returns: True if connection succeeds or False if not
|
||||||
:rtype: boolean
|
:rtype: boolean
|
||||||
|
@ -331,9 +359,13 @@ class MySQLRouterCharm(charms_openstack.charm.OpenStackCharm):
|
||||||
port=self.mysqlrouter_port,
|
port=self.mysqlrouter_port,
|
||||||
connect_timeout=self.mysql_connect_timeout)
|
connect_timeout=self.mysql_connect_timeout)
|
||||||
return True
|
return True
|
||||||
except mysql.MySQLdb._exceptions.OperationalError:
|
except mysql.MySQLdb._exceptions.OperationalError as e:
|
||||||
ch_core.hookenv.log("Could not connect to db", "DEBUG")
|
ch_core.hookenv.log("Could not connect to db", "DEBUG")
|
||||||
return False
|
if not reraise_on:
|
||||||
|
return False
|
||||||
|
else:
|
||||||
|
if e.args[0] in reraise_on:
|
||||||
|
raise e
|
||||||
|
|
||||||
def custom_assess_status_check(self):
|
def custom_assess_status_check(self):
|
||||||
"""Custom assess status check.
|
"""Custom assess status check.
|
||||||
|
@ -524,19 +556,11 @@ class MySQLRouterCharm(charms_openstack.charm.OpenStackCharm):
|
||||||
_ssl_ca = receiving_interface.ssl_ca()
|
_ssl_ca = receiving_interface.ssl_ca()
|
||||||
if _ssl_ca:
|
if _ssl_ca:
|
||||||
_ssl_ca = json.loads(_ssl_ca)
|
_ssl_ca = json.loads(_ssl_ca)
|
||||||
# We are using CA signed certificates with validation for TLS
|
|
||||||
# Set client_ssl_mode = PASSTHROUGH
|
|
||||||
self.update_config_parameter(
|
|
||||||
"DEFAULT", "client_ssl_mode", "PASSTHROUGH")
|
|
||||||
else:
|
else:
|
||||||
# Reset ssl_ca in case we previously had it set
|
# Reset ssl_ca in case we previously had it set
|
||||||
ch_core.hookenv.log("Proactively resetting ssl_ca", "DEBUG")
|
ch_core.hookenv.log("Proactively resetting ssl_ca", "DEBUG")
|
||||||
sending_interface.relations[
|
sending_interface.relations[
|
||||||
unit.relation.relation_id].to_publish_raw["ssl_ca"] = None
|
unit.relation.relation_id].to_publish_raw["ssl_ca"] = None
|
||||||
# We are using self-signed certificates for TLS
|
|
||||||
# Set client_ssl_mode = PREFERRED
|
|
||||||
self.update_config_parameter(
|
|
||||||
"DEFAULT", "client_ssl_mode", "PREFERRED")
|
|
||||||
|
|
||||||
if ch_core.hookenv.local_unit() in (json.loads(
|
if ch_core.hookenv.local_unit() in (json.loads(
|
||||||
receiving_interface.allowed_units(prefix=prefix))):
|
receiving_interface.allowed_units(prefix=prefix))):
|
||||||
|
@ -556,29 +580,100 @@ class MySQLRouterCharm(charms_openstack.charm.OpenStackCharm):
|
||||||
db_port=self.mysqlrouter_port,
|
db_port=self.mysqlrouter_port,
|
||||||
ssl_ca=_ssl_ca)
|
ssl_ca=_ssl_ca)
|
||||||
|
|
||||||
def update_config_parameter(self, heading, param, value):
|
def update_config_parameters(self, parameters):
|
||||||
"""Update configuration parameter.
|
"""Update configuration parameters using ConfigParser.
|
||||||
|
|
||||||
Update this instances mysqlrouter.conf with a key=value.
|
Update this instances mysqlrouter.conf with a dictionary set of
|
||||||
Such that under heading we get param = value.
|
configuration parameters and values of the form:
|
||||||
Restart on changed based on the mysqlrouter.conf.
|
|
||||||
|
|
||||||
:param heading: The config file heading i.e. 'DEFAULT'
|
parameters = {
|
||||||
:type heading: str
|
"HEADING": {
|
||||||
:param param: The parameter key
|
"param": "value"
|
||||||
:type param: str
|
}
|
||||||
:param value: The value getting set
|
}
|
||||||
:type value: str
|
|
||||||
:side effect: Writes the mysqlrouter.conf file and restarts services if
|
:param parameters: Dictionary of parameters
|
||||||
there is a change.
|
:type parameters: dict
|
||||||
|
:side effect: Writes the mysqlrouter.conf file
|
||||||
:returns: This function is called for its side effect
|
:returns: This function is called for its side effect
|
||||||
:rtype: None
|
:rtype: None
|
||||||
"""
|
"""
|
||||||
config = configparser.ConfigParser()
|
config = configparser.ConfigParser()
|
||||||
config.read(self.mysqlrouter_conf)
|
config.read(self.mysqlrouter_conf)
|
||||||
config[heading][param] = value
|
for heading in parameters.keys():
|
||||||
ch_core.hookenv.log("Updating {} with {} {} = {}".format(
|
for param in parameters[heading].keys():
|
||||||
self.mysqlrouter_conf, heading, param, value))
|
config[heading][param] = parameters[heading][param]
|
||||||
with self.restart_on_change():
|
ch_core.hookenv.log("Writing {}".format(
|
||||||
with open(self.mysqlrouter_conf, 'w') as configfile:
|
self.mysqlrouter_conf))
|
||||||
config.write(configfile)
|
with open(self.mysqlrouter_conf, 'w') as configfile:
|
||||||
|
config.write(configfile)
|
||||||
|
|
||||||
|
def config_changed(self):
|
||||||
|
"""Config changed.
|
||||||
|
|
||||||
|
Custom config changed as we are not using templates we need to update
|
||||||
|
config via ConfigParser. We only update after the mysql-router service
|
||||||
|
has bootstrapped.
|
||||||
|
|
||||||
|
:side effect: Calls update_config_parameter and restarts mysql-router
|
||||||
|
if the config file has changed.
|
||||||
|
:returns: This function is called for its side effect
|
||||||
|
:rtype: None
|
||||||
|
"""
|
||||||
|
if not os.path.exists(self.mysqlrouter_conf):
|
||||||
|
ch_core.hookenv.log(
|
||||||
|
"mysqlrouter.conf does not yet exist. "
|
||||||
|
"Skipping config changed.", "DEBUG")
|
||||||
|
return
|
||||||
|
|
||||||
|
_parameters = {
|
||||||
|
"metadata_cache:jujuCluster": {
|
||||||
|
"ttl": str(self.options.ttl),
|
||||||
|
"auth_cache_ttl": str(self.options.auth_cache_ttl),
|
||||||
|
"auth_cache_refresh_interval":
|
||||||
|
str(self.options.auth_cache_refresh_interval),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.ssl_ca:
|
||||||
|
ch_core.hookenv.log("TLS mode PASSTHROUGH", "DEBUG")
|
||||||
|
_parameters["DEFAULT"] = {"client_ssl_mode": "PASSTHROUGH"}
|
||||||
|
else:
|
||||||
|
ch_core.hookenv.log("TLS mode PREFERRED", "DEBUG")
|
||||||
|
_parameters["DEFAULT"] = {"client_ssl_mode": "PREFERRED"}
|
||||||
|
|
||||||
|
# NOTE: LP Bug #1917792
|
||||||
|
# Switch to context manager when work there is completed
|
||||||
|
@os_utils.pausable_restart_on_change(
|
||||||
|
self.restart_map, restart_functions=self.restart_functions)
|
||||||
|
def _config_changed(self, parameters):
|
||||||
|
ch_core.hookenv.log("Updating configuration parameters", "DEBUG")
|
||||||
|
self.update_config_parameters(parameters)
|
||||||
|
|
||||||
|
_config_changed(self, _parameters)
|
||||||
|
|
||||||
|
@tenacity.retry(wait=tenacity.wait_fixed(10),
|
||||||
|
retry=tenacity.retry_if_exception_type(
|
||||||
|
mysql.MySQLdb._exceptions.OperationalError),
|
||||||
|
reraise=True,
|
||||||
|
stop=tenacity.stop_after_attempt(5))
|
||||||
|
def custom_restart_function(self, service_name):
|
||||||
|
"""Tenacity retry custom restart function for restart_on_change
|
||||||
|
|
||||||
|
Custom restart function for use in restart_on_change contexts. Tenacity
|
||||||
|
retry enabled based on verification of connectivity.
|
||||||
|
|
||||||
|
:side effect: Calls service_stop and service_start on the mysql-router
|
||||||
|
service(s).
|
||||||
|
:returns: This function is called for its side effect
|
||||||
|
:rtype: None
|
||||||
|
"""
|
||||||
|
ch_core.hookenv.log(
|
||||||
|
"Custom restart of {}".format(service_name), "DEBUG")
|
||||||
|
self.service_stop(service_name)
|
||||||
|
self.service_start(service_name)
|
||||||
|
# Only raise an exception if it matches
|
||||||
|
# mysql.MySQLdb._exceptions.OperationalError error 2013
|
||||||
|
# LP Bug #1915842
|
||||||
|
self.check_mysql_connection(
|
||||||
|
reraise_on=[self._waiting_for_initial_communication_packet_error])
|
||||||
|
|
|
@ -99,5 +99,6 @@ def proxy_shared_db_responses(shared_db, db_router):
|
||||||
:type db_router_interface: MySQLRouterRequires object
|
:type db_router_interface: MySQLRouterRequires object
|
||||||
"""
|
"""
|
||||||
with charm.provide_charm_instance() as instance:
|
with charm.provide_charm_instance() as instance:
|
||||||
|
instance.config_changed()
|
||||||
instance.proxy_db_and_user_responses(db_router, shared_db)
|
instance.proxy_db_and_user_responses(db_router, shared_db)
|
||||||
instance.assess_status()
|
instance.assess_status()
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
psutil
|
psutil
|
||||||
mysqlclient
|
mysqlclient
|
||||||
|
tenacity
|
||||||
|
|
||||||
git+https://github.com/juju/charm-helpers.git#egg=charmhelpers
|
git+https://github.com/juju/charm-helpers.git#egg=charmhelpers
|
||||||
|
|
||||||
|
|
|
@ -600,15 +600,75 @@ class TestMySQLRouterCharm(test_utils.PatchHelper):
|
||||||
self.db_router, self.keystone_shared_db)
|
self.db_router, self.keystone_shared_db)
|
||||||
self.keystone_shared_db.set_db_connection_info.assert_not_called()
|
self.keystone_shared_db.set_db_connection_info.assert_not_called()
|
||||||
|
|
||||||
def test_update_config_parameter(self):
|
def test_update_config_parameters(self):
|
||||||
self.patch_object(mysql_router.configparser, "ConfigParser")
|
self.patch_object(mysql_router.configparser, "ConfigParser")
|
||||||
|
|
||||||
self.service_name = "mysql-router"
|
self.service_name = "mysql-router"
|
||||||
_mock_config_parser = mock.MagicMock()
|
_mock_config_parser = mock.MagicMock()
|
||||||
self.ConfigParser.return_value = _mock_config_parser
|
self.ConfigParser.return_value = _mock_config_parser
|
||||||
|
|
||||||
|
_params = {"DEFAULT": {"client_ssl_mode": "PREFERRED"}}
|
||||||
|
|
||||||
mrc = mysql_router.MySQLRouterCharm()
|
mrc = mysql_router.MySQLRouterCharm()
|
||||||
mrc.update_config_parameter("DEFAULT", "client_ssl_mode", "PREFERRED")
|
mrc.update_config_parameters(_params)
|
||||||
_mock_config_parser.read.assert_called_once()
|
_mock_config_parser.read.assert_called_once()
|
||||||
|
_mock_config_parser.__getitem__.assert_called_once_with('DEFAULT')
|
||||||
|
_mock_config_parser.__getitem__().__setitem__.assert_called_once_with(
|
||||||
|
'client_ssl_mode', 'PREFERRED')
|
||||||
_mock_config_parser.write.assert_called_once_with(
|
_mock_config_parser.write.assert_called_once_with(
|
||||||
self.mock_open()().__enter__())
|
self.mock_open()().__enter__())
|
||||||
|
|
||||||
|
def test_config_changed(self):
|
||||||
|
_config_data = {
|
||||||
|
"ttl": '5',
|
||||||
|
"auth_cache_ttl": '10',
|
||||||
|
"auth_cache_refresh_interval": '7',
|
||||||
|
}
|
||||||
|
_mock_decorator = mock.MagicMock()
|
||||||
|
_mock_decorated = mock.MagicMock()
|
||||||
|
_mock_decorator.return_value = _mock_decorated
|
||||||
|
|
||||||
|
def _fake_config(key=None):
|
||||||
|
return _config_data[key] if key else _config_data
|
||||||
|
|
||||||
|
self.patch_object(mysql_router.ch_core.hookenv, "config")
|
||||||
|
self.patch_object(mysql_router.os.path, "exists")
|
||||||
|
self.patch_object(mysql_router.os_utils, "pausable_restart_on_change")
|
||||||
|
self.config.side_effect = _fake_config
|
||||||
|
self.pausable_restart_on_change.return_value = _mock_decorator
|
||||||
|
self.service_name = "mysql-router"
|
||||||
|
self.endpoint_from_flag.return_value = self.db_router
|
||||||
|
self.db_router.ssl_ca.return_value = '"CACERT"'
|
||||||
|
|
||||||
|
_mock_update_config_parameters = mock.MagicMock()
|
||||||
|
mrc = mysql_router.MySQLRouterCharm()
|
||||||
|
mrc.update_config_parameters = _mock_update_config_parameters
|
||||||
|
|
||||||
|
_params = {
|
||||||
|
'metadata_cache:jujuCluster': _config_data,
|
||||||
|
'DEFAULT': {'client_ssl_mode': "PASSTHROUGH"},
|
||||||
|
}
|
||||||
|
|
||||||
|
# Not bootstrapped yet
|
||||||
|
self.exists.return_value = False
|
||||||
|
mrc.config_changed()
|
||||||
|
_mock_update_config_parameters.assert_not_called()
|
||||||
|
|
||||||
|
# Bootstrapped
|
||||||
|
self.exists.return_value = True
|
||||||
|
mrc.config_changed()
|
||||||
|
_mock_decorated.assert_has_calls([mock.call(mrc, _params)])
|
||||||
|
|
||||||
|
def test_custom_restart_function(self):
|
||||||
|
self.patch_object(mysql_router.ch_core.host, "service_stop")
|
||||||
|
self.patch_object(mysql_router.ch_core.host, "service_start")
|
||||||
|
self.service_name = "mysql-router"
|
||||||
|
_mock_check_mysql_connection = mock.MagicMock()
|
||||||
|
|
||||||
|
mrc = mysql_router.MySQLRouterCharm()
|
||||||
|
mrc.check_mysql_connection = _mock_check_mysql_connection
|
||||||
|
|
||||||
|
mrc.custom_restart_function(self.service_name)
|
||||||
|
self.service_stop.assert_called_once_with(self.service_name)
|
||||||
|
self.service_start.assert_called_once_with(self.service_name)
|
||||||
|
_mock_check_mysql_connection.assert_called_once()
|
||||||
|
|
Loading…
Reference in New Issue