openstack/fuel-library
Code Issues Proposed changes

Add read_timeout and infinite retries to MySQL conns

Browse Source 
read_timeout=60 is an explicit parameter added
for mysqldb to bail connections if no data is
received for 60s. Depends on MySQLdb 1.2.5
max_retries=-1 for all connections so that
APIs don't give up and die

Change-Id: Ib4a2cdcc287cbc53c18f7500d96f82d8099e0f35
Partial-Bug: #1285449
This commit is contained in:
Matthew Mosesohn 2014-03-03 20:45:55 +04:00 committed by Dmitry Borodaenko
parent 136d342a9d
commit b8f8476345
19 changed files with 81 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
deployment/puppet/ceilometer/manifests/db.pp
View File

@ -45,6 +45,7 @@ class ceilometer::db (
ceilometer_config {
'database/connection': value => $database_connection;
'database/max_retries': value => "-1";
}
Ceilometer_config['database/connection'] ~> Exec['ceilometer-dbsync']

4
deployment/puppet/cinder/manifests/base.pp
View File

@ -107,6 +107,10 @@ class cinder::base (
'DEFAULT/verbose': value => $verbose;
'DEFAULT/api_paste_config': value => '/etc/cinder/api-paste.ini';
}
cinder_config {
'DEFAULT/max_retries': value => '-1';
}
exec { 'cinder-manage db_sync':
command => $::cinder::params::db_sync_command,
path => '/usr/bin',

1
deployment/puppet/heat/manifests/db.pp
View File

@ -32,6 +32,7 @@ class heat::db (
heat_config {
'DEFAULT/sql_connection': value => $sql_connection;
'DEFAULT/max_retries': value => '-1';
}
file { 'db_sync_script' :

2
deployment/puppet/heat/manifests/init.pp
View File

@ -112,7 +112,7 @@ class heat(
}
class { 'heat::db' :
sql_connection => "mysql://${db_user}:${db_password}@${db_host}/${db_name}",
sql_connection => "mysql://${db_user}:${db_password}@${db_host}/${db_name}?read_timeout=60",
}
class { 'heat::api_cfn' :

5
deployment/puppet/keystone/manifests/init.pp
View File

@ -229,8 +229,11 @@ class keystone(
keystone_config { 'catalog/driver':
value => ' keystone.catalog.backends.sql.Catalog'
}
}
keystone_config {
'DATABASE/max_retries': value => '-1';
}
}
if $enabled {
$service_ensure = 'running'
} else {

3
deployment/puppet/murano/manifests/api.pp
View File

@ -37,7 +37,7 @@ class murano::api (
$murano_db_host = 'localhost',
) {
$api_database_connection = "mysql://${murano_db_name}:${murano_db_password}@${murano_db_host}:3306/${murano_db_name}"
$api_database_connection = "mysql://${murano_db_name}:${murano_db_password}@${murano_db_host}:3306/${murano_db_name}?read_timeout=60"
include murano::params
@ -92,6 +92,7 @@ class murano::api (
'DEFAULT/logging_default_format_string':
value => 'murano-api %(asctime)s %(levelname)s %(name)s [-] %(instance)s %(message)s';
'database/connection' : value => $api_database_connection;
'database/max_retries' : value => -1;
'database/auto_create' : value => $api_database_auto_create;
'reports/results_exchange' : value => $api_reports_results_exchange;
'reports/results_queue' : value => $api_reports_results_queue;

9
deployment/puppet/neutron/lib/puppet/parser/functions/sanitize_neutron_config.rb
View File

@ -108,8 +108,12 @@ class MrntNeutron
#rv = cfg.clone()
case cfg[:provider].to_s().downcase
when "mysql"
charset = cfg[:charset] ? "?charset=#{cfg[:charset]}" : ''
rv = "mysql://#{cfg[:username]}:#{cfg[:passwd]}@#{cfg[:host]}:#{cfg[:port]}/#{cfg[:database]}#{charset}"
url_opts = {
:read_timeout => cfg[:read_timeout],
:charset => cfg[:charset],
}.select{|k,v| !v.nil? and v != ''}.collect{|k,v| "#{k}=#{v}"}
optline = url_opts.size>0 ? "?#{url_opts.join('&')}" : ''
rv = "mysql://#{cfg[:username]}:#{cfg[:passwd]}@#{cfg[:host]}:#{cfg[:port]}/#{cfg[:database]}#{optline}"
when "pgsql"
raise(Puppet::ParseError, "unsupported database provider \"#{cfg[:provider]}\".")
when "sqlite"
@ -272,6 +276,7 @@ class MrntNeutron
:passwd => "neutron",
:reconnects => -1,
:reconnect_interval => 2,
:read_timeout => 60,
:charset => nil,
},
:keystone => {

1
deployment/puppet/neutron/manifests/init.pp
View File

@ -140,7 +140,6 @@ class neutron (
'keystone_authtoken/admin_password': value => $neutron_config['keystone']['admin_password'];
}
if defined(Anchor['neutron-server-config-done']) {
$endpoint_neutron_main_configuration = 'neutron-server-config-done'
} else {

60
deployment/puppet/neutron/spec/functions/sanitize_quantum_config__spec.rb
View File

@ -73,6 +73,7 @@ class NeutronConfig
'reconnect_interval' => 2,
'url' => nil,
'charset' => nil,
'read_timeout' => 60,
},
'keystone' => {
'auth_host' => "#{@def_v[:management_vip]}",
@ -277,7 +278,7 @@ describe 'sanitize_neutron_config with minimal incoming data' , :type => :puppet
it 'should return default config (DATABASE) if given only minimal parameter set' do
rv = scope.function_sanitize_neutron_config([cfg, 'neutron_settings'])
expect(rv['database']).to eq({
"url"=>"mysql://neutron:neutron@192.168.0.254:3306/neutron",
"url"=>"mysql://neutron:neutron@192.168.0.254:3306/neutron?read_timeout=60",
"provider"=>"mysql",
"host"=>"192.168.0.254",
"port"=>3306,
@ -457,7 +458,7 @@ describe 'sanitize_neutron_config' , :type => :puppet_function do
}
}
res_cfg['predefined_networks']['net04']['L2']['physnet'] = nil
res_cfg['database']['url'] = 'mysql://neutron:neutron@192.168.0.254:3306/neutron'
res_cfg['database']['url'] = 'mysql://neutron:neutron@192.168.0.254:3306/neutron?read_timeout=60'
rv = scope.function_sanitize_neutron_config([cfg, 'neutron_settings'])
expect(rv).to eq(res_cfg)
end
@ -466,7 +467,7 @@ describe 'sanitize_neutron_config' , :type => :puppet_function do
cfg = Marshal.load(Marshal.dump(@cfg))
cfg['neutron_settings']['L3'].delete('dhcp_agent')
res_cfg = Marshal.load(Marshal.dump(@res_cfg))
res_cfg['database']['url'] = 'mysql://neutron:neutron@192.168.0.254:3306/neutron'
res_cfg['database']['url'] = 'mysql://neutron:neutron@192.168.0.254:3306/neutron?read_timeout=60'
rv = scope.function_sanitize_neutron_config([@cfg, 'neutron_settings'])
expect(rv['amqp']).to eq(res_cfg['amqp'])
end
@ -475,7 +476,7 @@ describe 'sanitize_neutron_config' , :type => :puppet_function do
cfg = Marshal.load(Marshal.dump(@cfg))
cfg['neutron_settings']['L3'].delete('dhcp_agent')
res_cfg = Marshal.load(Marshal.dump(@res_cfg))
res_cfg['database']['url'] = 'mysql://neutron:neutron@192.168.0.254:3306/neutron'
res_cfg['database']['url'] = 'mysql://neutron:neutron@192.168.0.254:3306/neutron?read_timeout=60'
rv = scope.function_sanitize_neutron_config([cfg, 'neutron_settings'])
expect(rv['database']).to eq res_cfg['database']
end
@ -484,7 +485,7 @@ describe 'sanitize_neutron_config' , :type => :puppet_function do
cfg = Marshal.load(Marshal.dump(@cfg))
cfg['neutron_settings']['L3'].delete('dhcp_agent')
res_cfg = Marshal.load(Marshal.dump(@res_cfg))
res_cfg['database']['url'] = 'mysql://neutron:neutron@192.168.0.254:3306/neutron'
res_cfg['database']['url'] = 'mysql://neutron:neutron@192.168.0.254:3306/neutron?read_timeout=60'
rv = scope.function_sanitize_neutron_config([@cfg, 'neutron_settings'])
expect(rv['server']).to eq res_cfg['server']
end
@ -493,7 +494,7 @@ describe 'sanitize_neutron_config' , :type => :puppet_function do
cfg = Marshal.load(Marshal.dump(@cfg))
cfg['neutron_settings']['L3'].delete('dhcp_agent')
res_cfg = Marshal.load(Marshal.dump(@res_cfg))
res_cfg['database']['url'] = 'mysql://neutron:neutron@192.168.0.254:3306/neutron'
res_cfg['database']['url'] = 'mysql://neutron:neutron@192.168.0.254:3306/neutron?read_timeout=60'
rv = scope.function_sanitize_neutron_config([@cfg, 'neutron_settings'])
expect(rv['keystone']).to eq res_cfg['keystone']
end
@ -512,7 +513,7 @@ describe 'sanitize_neutron_config' , :type => :puppet_function do
"vlan_range" => nil
}
}
res_cfg['database']['url'] = 'mysql://neutron:neutron@192.168.0.254:3306/neutron'
res_cfg['database']['url'] = 'mysql://neutron:neutron@192.168.0.254:3306/neutron?read_timeout=60'
rv = scope.function_sanitize_neutron_config([@cfg, 'neutron_settings'])
expect(rv['L2']).to eq res_cfg['L2']
end
@ -521,7 +522,7 @@ describe 'sanitize_neutron_config' , :type => :puppet_function do
cfg = Marshal.load(Marshal.dump(@cfg))
cfg['neutron_settings']['L3'].delete('dhcp_agent')
res_cfg = Marshal.load(Marshal.dump(@res_cfg))
res_cfg['database']['url'] = 'mysql://neutron:neutron@192.168.0.254:3306/neutron'
res_cfg['database']['url'] = 'mysql://neutron:neutron@192.168.0.254:3306/neutron?read_timeout=60'
rv = scope.function_sanitize_neutron_config([@cfg, 'neutron_settings'])
expect(rv['L3']).to eq res_cfg['L3']
end
@ -531,7 +532,7 @@ describe 'sanitize_neutron_config' , :type => :puppet_function do
cfg['neutron_settings']['L3'].delete('dhcp_agent')
res_cfg = Marshal.load(Marshal.dump(@res_cfg))
res_cfg['predefined_networks']['net04']['L2']['physnet'] = nil
res_cfg['database']['url'] = 'mysql://neutron:neutron@192.168.0.254:3306/neutron'
res_cfg['database']['url'] = 'mysql://neutron:neutron@192.168.0.254:3306/neutron?read_timeout=60'
rv = scope.function_sanitize_neutron_config([@cfg, 'neutron_settings'])
expect(rv['predefined_networks']).to eq res_cfg['predefined_networks']
end
@ -540,7 +541,7 @@ describe 'sanitize_neutron_config' , :type => :puppet_function do
cfg = Marshal.load(Marshal.dump(@cfg))
cfg['neutron_settings']['L3'].delete('dhcp_agent')
res_cfg = Marshal.load(Marshal.dump(@res_cfg))
res_cfg['database']['url'] = 'mysql://neutron:neutron@192.168.0.254:3306/neutron'
res_cfg['database']['url'] = 'mysql://neutron:neutron@192.168.0.254:3306/neutron?read_timeout=60'
rv = scope.function_sanitize_neutron_config([@cfg, 'neutron_settings'])
expect(rv['predefined_routers']).to eq res_cfg['predefined_routers']
end
@ -548,7 +549,7 @@ describe 'sanitize_neutron_config' , :type => :puppet_function do
it 'should calculate database url if database properties not given' do
@cfg['neutron_settings']['database'] = {}
rv = scope.function_sanitize_neutron_config([@cfg, 'neutron_settings'])
expect(rv['database']['url']).to eq "mysql://neutron:neutron@192.168.0.254:3306/neutron"
expect(rv['database']['url']).to eq "mysql://neutron:neutron@192.168.0.254:3306/neutron?read_timeout=60"
end
it 'should calculate database url if some database properties given' do
@cfg['neutron_settings']['database'] = {
@ -560,7 +561,7 @@ describe 'sanitize_neutron_config' , :type => :puppet_function do
'port' => 666,
}
rv = scope.function_sanitize_neutron_config([@cfg, 'neutron_settings'])
expect(rv['database']['url']).to eq "mysql://qq_username:qq_password@5.4.3.2:666/qq_database"
expect(rv['database']['url']).to eq "mysql://qq_username:qq_password@5.4.3.2:666/qq_database?read_timeout=60"
end
it 'should can substitute values in deep level' do
@ -583,7 +584,7 @@ describe 'sanitize_neutron_config' , :type => :puppet_function do
}
}
res_cfg['predefined_networks']['net04']['L2']['physnet'] = nil
res_cfg['database']['url'] = 'mysql://neutron:neutron@192.168.0.254:3306/neutron'
res_cfg['database']['url'] = 'mysql://neutron:neutron@192.168.0.254:3306/neutron?read_timeout=60'
res_cfg['L2']['enable_tunneling'] = true
#should run.with_params(@cfg,'neutron_settings').and_return(res_cfg)
rv = scope.function_sanitize_neutron_config([cfg, 'neutron_settings'])
@ -743,7 +744,38 @@ describe MrntNeutron do
end
end
describe '.get_database_url' do
it 'should return database url without charset' do
it 'should return database url with read_timeout' do
MrntNeutron.get_database_url({
:provider => "mysql",
:host => "1.2.3.4",
:port => 3306,
:database => "q_database",
:username => "q_username",
:passwd => "q_passwd",
:read_timeout => 45,
}).should == "mysql://q_username:q_passwd@1.2.3.4:3306/q_database?read_timeout=45"
end
end
describe '.get_database_url' do
it 'should return database url with charset and read_timeout' do
gdu = MrntNeutron.get_database_url({
:provider => "mysql",
:host => "1.2.3.4",
:port => 3306,
:database => "q_database",
:username => "q_username",
:passwd => "q_passwd",
:charset => "xxx32",
:read_timeout => 45,
})
gdu.should =~ /charset=xxx32/
gdu.should =~ /read_timeout=45/
gdu.should =~ /\?/
gdu.should =~ /\&/
end
end
describe '.get_database_url' do
it 'should return database url without charset and read_timeout' do
MrntNeutron.get_database_url({
:provider => "mysql",
:host => "1.2.3.4",

4
deployment/puppet/nova/manifests/init.pp
View File

@ -251,6 +251,10 @@ if $use_syslog and !$debug { #syslog and nondebug case
'DEFAULT/osapi_volume_listen': value => $api_bind_address;
}
nova_config {
'DATABASE/max_retries': value => '-1';
}
if $monitoring_notifications {
nova_config {
'DEFAULT/notification_driver': value => 'nova.notifier.rabbit_notifier'

2
deployment/puppet/openstack/manifests/ceilometer.pp
View File

@ -50,7 +50,7 @@ class openstack::ceilometer (
# Configure the ceilometer database
# Only needed if ceilometer::agent::central or ceilometer::api are declared
class { '::ceilometer::db':
database_connection => "${db_type}://${db_user}:${db_password}@${db_host}/${db_dbname}",
database_connection => "${db_type}://${db_user}:${db_password}@${db_host}/${db_dbname}?read_timeout=60",
}
# Install the ceilometer-api service

2
deployment/puppet/openstack/manifests/controller.pp
View File

@ -399,7 +399,7 @@ class openstack::controller (
if $cinder {
if !defined(Class['openstack::cinder']) {
class {'openstack::cinder':
sql_connection => "mysql://${cinder_db_user}:${cinder_db_password}@${db_host}/${cinder_db_dbname}?charset=utf8",
sql_connection => "mysql://${cinder_db_user}:${cinder_db_password}@${db_host}/${cinder_db_dbname}?charset=utf8&read_timeout=60",
queue_provider => $queue_provider,
amqp_hosts => $amqp_hosts,
amqp_user => $amqp_user,

2
deployment/puppet/openstack/manifests/glance.pp
View File

@ -63,7 +63,7 @@ class openstack::glance (
# Configure the db string
case $db_type {
'mysql': {
$sql_connection = "mysql://${glance_db_user}:${glance_db_password}@${db_host}/${glance_db_dbname}"
$sql_connection = "mysql://${glance_db_user}:${glance_db_password}@${db_host}/${glance_db_dbname}?read_timeout=60"
}
}

2
deployment/puppet/openstack/manifests/keystone.pp
View File

@ -95,7 +95,7 @@ class openstack::keystone (
# Install and configure Keystone
if $db_type == 'mysql' {
$sql_conn = "mysql://${$db_user}:${db_password}@${db_host}/${db_name}"
$sql_conn = "mysql://${$db_user}:${db_password}@${db_host}/${db_name}?read_timeout=60"
} else {
fail("db_type ${db_type} is not supported")
}

2
deployment/puppet/openstack/manifests/nova/controller.pp
View File

@ -93,7 +93,7 @@ class openstack::nova::controller (
# Configure the db string
case $db_type {
'mysql': {
$nova_db = "mysql://${nova_db_user}:${nova_db_password}@${db_host}/${nova_db_dbname}"
$nova_db = "mysql://${nova_db_user}:${nova_db_password}@${db_host}/${nova_db_dbname}?read_timeout=60"
}
}

4
deployment/puppet/osnailyfacter/manifests/cluster_ha.pp
View File

@ -498,7 +498,7 @@ class osnailyfacter::cluster_ha {
network_manager => $network_manager,
network_config => $network_config,
multi_host => $multi_host,
sql_connection => "mysql://nova:${nova_hash[db_password]}@${::fuel_settings['management_vip']}/nova",
sql_connection => "mysql://nova:${nova_hash[db_password]}@${::fuel_settings['management_vip']}/nova?read_timeout=60",
queue_provider => $::queue_provider,
amqp_hosts => $amqp_hosts,
amqp_user => $rabbit_hash['user'],
@ -564,7 +564,7 @@ class osnailyfacter::cluster_ha {
$bind_host = false
}
class { 'openstack::cinder':
sql_connection => "mysql://cinder:${cinder_hash[db_password]}@${::fuel_settings['management_vip']}/cinder?charset=utf8",
sql_connection => "mysql://cinder:${cinder_hash[db_password]}@${::fuel_settings['management_vip']}/cinder?charset=utf8&read_timeout=60",
glance_api_servers => "${::fuel_settings['management_vip']}:9292",
bind_host => $bind_host,
queue_provider => $::queue_provider,

4
deployment/puppet/osnailyfacter/manifests/cluster_simple.pp
View File

@ -113,7 +113,7 @@ class osnailyfacter::cluster_simple {
# do not edit the below line
validate_re($::queue_provider, 'rabbitmq|qpid')
$sql_connection = "mysql://nova:${nova_hash[db_password]}@${controller_node_address}/nova"
$sql_connection = "mysql://nova:${nova_hash[db_password]}@${controller_node_address}/nova?read_timeout=60"
$mirror_type = 'external'
$multi_host = true
Exec { logoutput => true }
@ -423,7 +423,7 @@ class osnailyfacter::cluster_simple {
$bind_host = false
}
class { 'openstack::cinder':
sql_connection => "mysql://cinder:${cinder_hash[db_password]}@${controller_node_address}/cinder?charset=utf8",
sql_connection => "mysql://cinder:${cinder_hash[db_password]}@${controller_node_address}/cinder?charset=utf8&read_timeout=60",
glance_api_servers => "${controller_node_address}:9292",
queue_provider => $::queue_provider,
amqp_hosts => $amqp_hosts,

1
deployment/puppet/savanna/manifests/api.pp
View File

@ -72,6 +72,7 @@ class savanna::api (
'plugin:hdp/plugin_class' : value => $hdp_plugin_class;
'plugin:idh/plugin_class' : value => $idh_plugin_class;
'database/connection' : value => $sql_connection;
'database/max_retries' : value => -1;
}
$logging_file = '/etc/savanna/logging.conf'

2
deployment/puppet/savanna/manifests/init.pp
View File

@ -35,7 +35,7 @@ class savanna (
$syslog_log_facility_savanna = 'LOG_LOCAL0',
) {
$savanna_sql_connection = "mysql://${savanna_db_user}:${savanna_db_password}@${savanna_db_host}/${savanna_db_name}"
$savanna_sql_connection = "mysql://${savanna_db_user}:${savanna_db_password}@${savanna_db_host}/${savanna_db_name}?read_timeout=60"
$savanna_url_string = "SAVANNA_URL = '${savanna_api_protocol}://${savanna_api_host}:${savanna_api_port}/${savanna_api_version}'"
class { 'savanna::db::mysql':