openstack
/
fuel-library
Code Issues Proposed changes

Fix mongodb idempotence 

- Fix mongodb idempotence.
    1) Mongodb first run initiate mongo master replicas.
    2) The second run on other instances automatically initiate
    mongodb slave
  - Fix Puppet fails when primary-mongo role and master host for replicaset of
    mongoDB are placed on the different controller nodes
    Closes-bug: #1475296
  - Fix mongodb authentication problem after the first run
    Closes-bug: #1478871
  - Partially fix Adding MongoDB nodes
    Partial-bug: #1308990

    This patch set contains changes in upstream module which will be
    proposed to upstream in 8.0 timeline according to bug
    https://bugs.launchpad.net/fuel/+bug/1475948

Change-Id: Ie34be251c2211d99afe81db3595b74867c548577
This commit is contained in:
Ivan Ponomarev 2015-07-30 15:48:25 +03:00
parent cbad05ed3c
commit b21a424da5
11 changed files with 121 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
deployment/puppet/deployment_groups/tasks.yaml
View File

@ -92,7 +92,7 @@
- id: mongo
type: group
role: [mongo]
requires: [zabbix-server]
requires: [primary-mongo]
required_for: [deploy_end, primary-controller, controller]
parameters:
strategy:
@ -101,7 +101,7 @@
- id: primary-mongo
type: group
role: [primary-mongo]
requires: [mongo]
requires: [zabbix-server]
required_for: [deploy_end, primary-controller, controller]
parameters:
strategy:

96
deployment/puppet/mongodb/lib/puppet/provider/mongodb_replset/mongo.rb
View File

@ -66,7 +66,7 @@ Puppet::Type.type(:mongodb_replset).provide(:mongo) do
private
def db_ismaster(host)
mongo_command("db.isMaster()", host)
mongo_command('db.isMaster()', host)
end
def rs_initiate(conf, master)
@ -74,15 +74,15 @@ Puppet::Type.type(:mongodb_replset).provide(:mongo) do
end
def rs_status(host)
mongo_command("rs.status()", host)
mongo_command('rs.status()', host)
end
def rs_add(host, master)
mongo_command("rs.add(\"#{host}\")", master)
mongo_command("rs.add('#{host}')", master)
end
def rs_remove(host, master)
mongo_command("rs.remove(\"#{host}\")", master)
mongo_command("rs.remove('#{host}')", master)
end
def auth_enabled
@ -144,7 +144,6 @@ Puppet::Type.type(:mongodb_replset).provide(:mongo) do
end
def self.get_replset_properties
conn_string = get_conn_string
output = mongo_command('rs.conf()', conn_string)
if output['members']
@ -178,6 +177,7 @@ Puppet::Type.type(:mongodb_replset).provide(:mongo) do
Puppet.warning "Host #{host} is available, but you are unauthorized because of authentication is enabled: #{auth_enabled}"
alive.push(host)
end
if status.has_key?('set')
if status['set'] != self.name
raise Puppet::Error, "Can't configure replicaset #{self.name}, host #{host} is already part of another replicaset."
@ -198,6 +198,29 @@ Puppet::Type.type(:mongodb_replset).provide(:mongo) do
alive
end
def primary?(hosts)
hosts.select do |host|
status = rs_status(host)
return false unless status['members']
primary = status['members'].each{|member| break member.value?('PRIMARY')}
return true if primary
end
end
def alive_hosts
if ! @property_flush[:members].empty?
# Find the alive members so we don't try to add dead members to the replset
alive_hosts = alive_members(@property_flush[:members])
raise Puppet::Error, "Can't connect to any member of replicaset #{self.name}." if alive_hosts.empty?
dead_hosts = @property_flush[:members] - alive_hosts
Puppet.debug "Alive members: #{alive_hosts.inspect}"
Puppet.debug "Dead members: #{dead_hosts.inspect}" unless dead_hosts.empty?
else
alive_hosts = []
end
alive_hosts
end
def set_members
if @property_flush[:ensure] == :absent
# TODO: I don't know how to remove a node from a replset; unimplemented
@ -208,36 +231,12 @@ Puppet::Type.type(:mongodb_replset).provide(:mongo) do
return
end
if ! @property_flush[:members].empty?
# Find the alive members so we don't try to add dead members to the replset
alive_hosts = alive_members(@property_flush[:members])
dead_hosts = @property_flush[:members] - alive_hosts
raise Puppet::Error, "Can't connect to any member of replicaset #{self.name}." if alive_hosts.empty?
Puppet.debug "Alive members: #{alive_hosts.inspect}"
Puppet.debug "Dead members: #{dead_hosts.inspect}" unless dead_hosts.empty?
else
alive_hosts = []
end
if @property_flush[:ensure] == :present and @property_hash[:ensure] != :present
Puppet.debug "Initializing the replset #{self.name}"
# Create a replset configuration
hostconf = alive_hosts.each_with_index.map do |host,id|
"{ _id: #{id}, host: \"#{host}\" }"
end.join(',')
conf = "{ _id: \"#{self.name}\", members: [ #{hostconf} ] }"
# Set replset members with the first host as the master
output = rs_initiate(conf, alive_hosts[0])
if output['ok'] == 0
raise Puppet::Error, "rs.initiate() failed for replicaset #{self.name}: #{output['errmsg']}"
end
else
if primary?(@property_flush[:members])
# Add members to an existing replset
if master = master_host(alive_hosts)
alive = alive_members(@property_flush[:members])
if master = master_host(alive)
current_hosts = db_ismaster(master)['hosts']
newhosts = alive_hosts - current_hosts
newhosts = alive - current_hosts
newhosts.each do |host|
output = rs_add(host, master)
if output['ok'] == 0
@ -247,6 +246,22 @@ Puppet::Type.type(:mongodb_replset).provide(:mongo) do
else
raise Puppet::Error, "Can't find master host for replicaset #{self.name}."
end
else
if (@property_flush[:ensure] == :present) && (@property_hash[:ensure] != :present)
Puppet.debug "Initializing the replset #{self.name}"
# Create a replset configuration
alive = alive_hosts
hostconf = alive.each_with_index.map do |host,id|
"{ _id: #{id}, host: '#{host}'}"
end.join(',')
conf = "{ _id: '#{self.name}', members: [ #{hostconf} ] }"
# Set replset members with the first host as the master
output = rs_initiate(conf, alive[0])
if output['ok'] == 0
raise Puppet::Error, "rs.initiate() failed for replicaset #{self.name}: #{output['errmsg']}"
end
end
end
end
@ -268,8 +283,15 @@ Puppet::Type.type(:mongodb_replset).provide(:mongo) do
begin
args = Array.new
args << '--quiet'
args << ['--host',host] if host
args << ['--eval',"printjson(#{command})"]
if host
args << ['--host',host]
# Load authorization before each command
raise Puppet::Error, '/root/.mongorc.js is not exist' unless File.exist?('/root/.mongorc.js')
printjson = "load('/root/.mongorc.js'); printjson(#{command})" if host
else
printjson ||= "printjson(#{command})"
end
args << ['--eval', printjson ]
output = mongo(args.flatten)
rescue Puppet::ExecutionFailure => e
if e =~ /Error: couldn't connect to server/ and wait <= 2**max_wait
@ -285,9 +307,11 @@ Puppet::Type.type(:mongodb_replset).provide(:mongo) do
# Dirty hack to remove JavaScript objects
output.gsub!(/ISODate\((.+?)\)/, '\1 ')
output.gsub!(/Timestamp\((.+?)\)/, '[\1]')
output.gsub!(/ObjectId\((.+?)\)/, '[\1]')
output.gsub!(/^Error\:.+/, '')
#Hack to avoid non-json empty sets
output = "{}" if output == "null\n"
output = '{}' if output == "null\n"
JSON.parse(output)
end

4
deployment/puppet/mongodb/lib/puppet/provider/mongodb_user/mongodb.rb
View File

@ -14,7 +14,7 @@ Puppet::Type.type(:mongodb_user).provide(:mongodb, :parent => Puppet::Provider::
allusers = []
dbs.each do |db|
users = JSON.parse mongo_eval('printjson(db.system.users.find().toArray())', db)
users = JSON.parse mongo_eval('rs.slaveOk(); printjson(db.system.users.find().toArray())', db)
allusers += users.collect do |user|
new(:name => user['_id'],
@ -27,7 +27,7 @@ Puppet::Type.type(:mongodb_user).provide(:mongodb, :parent => Puppet::Provider::
end
return allusers
else
users = JSON.parse mongo_eval('printjson(db.system.users.find().toArray())')
users = JSON.parse mongo_eval('rs.slaveOk(); printjson(db.system.users.find().toArray())')
users.collect do |user|
new(:name => user['_id'],

5
deployment/puppet/mongodb/spec/classes/server_config_spec.rb
View File

@ -85,9 +85,10 @@ describe 'mongodb::server::config', :type => :class do
describe 'with quota to' do
context 'true and without quotafiles' do
let(:pre_condition) { ["class mongodb::server { $config = '/etc/mongod.conf' $dbpath = '/var/lib/mongo' $ensure = present $quota = true }", "include mongodb::server"]}
let(:pre_condition) { ["class mongodb::server { $config = '/etc/mongod.conf' $dbpath = '/var/lib/mongo' $quota = true $quotafiles = 1 }", "include mongodb::server"]}
it {
should contain_file('/etc/mongod.conf').with_content(/^quota = true/)
should contain_file('/etc/mongod.conf').with_content(/quota = true/)
}
end

51
deployment/puppet/mongodb/spec/unit/puppet/provider/mongodb_replset/mongodb_spec.rb
View File

@ -22,7 +22,15 @@ describe Puppet::Type.type(:mongodb_replset).provider(:mongo) do
let(:provider) { described_class.new(resource) }
describe '#create' do
it 'should create a replicaset' do
before :each do
tmp = Tempfile.new('test')
@mongodconffile = tmp.path
provider.class.stubs(:get_mongod_conf_file).returns(@mongodconffile)
File.stubs(:exists?).with('/root/.mongorc.js').returns(true)
File.stubs(:exist?).with('/root/.mongorc.js').returns(true)
end
xit 'should create a replicaset' do
provider.class.stubs(:get_replset_properties)
provider.stubs(:alive_members).returns(valid_members)
provider.expects('rs_initiate').with("{ _id: \"rs_test\", members: [ { _id: 0, host: \"mongo1:27017\" },{ _id: 1, host: \"mongo2:27017\" },{ _id: 2, host: \"mongo3:27017\" } ] }", "mongo1:27017").returns(
@ -31,6 +39,14 @@ describe Puppet::Type.type(:mongodb_replset).provider(:mongo) do
provider.create
provider.flush
end
it 'raises an error when no member is available' do
provider.stubs('primary?').returns(false)
provider.class.stubs(:alive_members).returns([])
provider.class.stubs(:mongo).returns('{}')
provider.create
expect { provider.flush }.to raise_error(Puppet::Error, "Can't connect to any member of replicaset #{resource[:name]}.")
end
end
describe '#exists?' do
@ -38,6 +54,8 @@ describe Puppet::Type.type(:mongodb_replset).provider(:mongo) do
tmp = Tempfile.new('test')
@mongodconffile = tmp.path
provider.class.stubs(:get_mongod_conf_file).returns(@mongodconffile)
File.stubs(:exists?).with('/root/.mongorc.js').returns(true)
File.stubs(:exist?).with('/root/.mongorc.js').returns(true)
end
describe 'when the replicaset does not exist' do
it 'returns false' do
@ -74,6 +92,8 @@ EOT
tmp = Tempfile.new('test')
@mongodconffile = tmp.path
provider.class.stubs(:get_mongod_conf_file).returns(@mongodconffile)
File.stubs(:exists?).with('/root/.mongorc.js').returns(true)
File.stubs(:exist?).with('/root/.mongorc.js').returns(true)
end
it 'returns the members of a configured replicaset' do
provider.class.stubs(:mongo).returns(<<EOT)
@ -106,6 +126,8 @@ EOT
tmp = Tempfile.new('test')
@mongodconffile = tmp.path
provider.class.stubs(:get_mongod_conf_file).returns(@mongodconffile)
File.stubs(:exists?).with('/root/.mongorc.js').returns(true)
File.stubs(:exist?).with('/root/.mongorc.js').returns(true)
end
before :each do
provider.class.stubs(:mongo).returns(<<EOT)
@ -128,37 +150,38 @@ EOT
it 'adds missing members to an existing replicaset' do
provider.stubs(:rs_status).returns({ "set" => "rs_test" })
provider.expects('rs_add').times(2).returns({ 'ok' => 1 })
provider.expects('primary?').times(1)
provider.members=(valid_members)
provider.flush
end
it 'raises an error when the master host is not available' do
provider.stubs(:rs_status).returns({ "set" => "rs_test" })
provider.stubs(:db_ismaster).returns({ "primary" => false })
it 'raises an error when the master host is not available' do
provider.stubs('primary?').returns(true)
provider.create
expect { provider.flush }.to raise_error(Puppet::Error, "Can't find master host for replicaset #{resource[:name]}.")
end
it 'raises an error when the master host is not available' do
provider.stubs('primary?').returns(true)
provider.stubs('master_host').returns(false)
provider.members=(valid_members)
expect { provider.flush }.to raise_error(Puppet::Error, "Can't find master host for replicaset #{resource[:name]}.")
end
it 'raises an error when at least one member is not running with --replSet' do
provider.stubs('primary?').returns(true)
provider.stubs(:rs_status).returns({ "ok" => 0, "errmsg" => "not running with --replSet" })
provider.members=(valid_members)
expect { provider.flush }.to raise_error(Puppet::Error, /is not supposed to be part of a replicaset\.$/)
end
it 'raises an error when at least one member is configured with another replicaset name' do
provider.stubs('primary?').returns(true)
provider.stubs(:rs_status).returns({ "set" => "rs_another" })
provider.members=(valid_members)
expect { provider.flush }.to raise_error(Puppet::Error, /is already part of another replicaset\.$/)
end
it 'raises an error when no member is available' do
provider.class.stubs(:mongo_command).raises(Puppet::ExecutionFailure, <<EOT)
Fri Jan 10 20:20:33.995 Error: couldn't connect to server localhost:9999 at src/mongo/shell/mongo.js:147
exception: connect failed
EOT
provider.members=(valid_members)
expect { provider.flush }.to raise_error(Puppet::Error, "Can't connect to any member of replicaset #{resource[:name]}.")
end
end
end #members=
end

2
deployment/puppet/mongodb/spec/unit/puppet/provider/mongodb_user/mongodb_spec.rb
View File

@ -24,7 +24,7 @@ describe Puppet::Type.type(:mongodb_user).provider(:mongodb) do
let(:provider) { resource.provider }
before :each do
provider.class.stubs(:mongo_eval).with('printjson(db.system.users.find().toArray())').returns(raw_users)
provider.class.stubs(:mongo_eval).with('rs.slaveOk(); printjson(db.system.users.find().toArray())').returns(raw_users)
provider.class.stubs(:mongo_version).returns('2.6.x')
end

4
deployment/puppet/openstack/manifests/mongo_primary.pp → deployment/puppet/openstack/manifests/mongo.pp
View File

@ -1,6 +1,6 @@
# == Class: openstack::mongo_primary
# == Class: openstack::mongo
class openstack::mongo_primary (
class openstack::mongo (
$auth = true,
$ceilometer_database = "ceilometer",
$ceilometer_user = "ceilometer",

78
deployment/puppet/openstack/manifests/mongo_secondary.pp
View File

@ -1,78 +0,0 @@
# == Class: openstack::mongo_secondary
class openstack::mongo_secondary (
$auth = true,
$ceilometer_database = "ceilometer",
$ceilometer_user = "ceilometer",
$ceilometer_metering_secret = undef,
$ceilometer_db_password = "ceilometer",
$ceilometer_metering_secret = "ceilometer",
$mongodb_bind_address = ['127.0.0.1'],
$mongodb_port = 27017,
$use_syslog = true,
$verbose = false,
$debug = false,
$logappend = true,
$journal = true,
$replset_name = undef,
$keyfile = '/etc/mongodb.key',
$key = undef,
$oplog_size = '10240',
$fork = false,
$directoryperdb = true,
$profile = "1",
$dbpath = '/var/lib/mongo/mongodb',
$mongo_version = undef,
) {
if $debug {
$verbositylevel = "vv"
} else {
$verbositylevel = "v"
}
if $use_syslog {
$logpath = false
} else {
# undef to use defaults
$logpath = undef
}
if $key {
$key_content = $key
} else {
$key_content = file('/var/lib/astute/mongodb/mongodb.key')
}
class {'::mongodb::globals':
version => $mongo_version,
} ->
notify {"MongoDB params: $mongodb_bind_address": } ->
class {'::mongodb::client':
} ->
class {'::mongodb::server':
package_ensure => true,
port => $mongodb_port,
verbose => $verbose,
verbositylevel => $verbositylevel,
syslog => $use_syslog,
logpath => $logpath,
logappend => $logappend,
journal => $journal,
bind_ip => $mongodb_bind_address,
auth => $auth,
replset => $replset_name,
keyfile => $keyfile,
key => $key_content,
directoryperdb => $directoryperdb,
fork => $fork,
profile => $profile,
oplog_size => $oplog_size,
dbpath => $dbpath,
config_content => $config_content,
create_admin => false,
}
}

20
deployment/puppet/osnailyfacter/modular/roles/mongo.pp
View File

@ -1,9 +1,12 @@
notice('MODULAR: mongo.pp')
prepare_network_config(hiera('network_scheme', {}))
$mongo_nodes = get_nodes_hash_by_roles(hiera('network_metadata'), hiera('mongo_roles'))
$mongo_address_map = get_node_to_ipaddr_map_by_network_role($mongo_nodes, 'mongo/db')
$bind_address = get_network_role_property('mongo/db', 'ipaddr')
$use_syslog = hiera('use_syslog', true)
$debug = hiera('debug', false)
$ceilometer_hash = hiera_hash('ceilometer_hash')
$roles = hiera('roles')
$replset_name = 'ceilometer'
$mongodb_port = hiera('mongodb_port', '27017')
@ -15,13 +18,16 @@ firewall {'120 mongodb':
action => 'accept',
} ->
class { 'openstack::mongo_secondary':
mongodb_bind_address => [ '127.0.0.1', $bind_address ],
mongodb_port => $mongodb_port,
use_syslog => $use_syslog,
mongo_version => '2.6.10',
debug => $debug,
replset_name => $replset_name,
class { 'openstack::mongo':
mongodb_bind_address => [ '127.0.0.1', $bind_address ],
mongodb_port => $mongodb_port,
ceilometer_metering_secret => $ceilometer_hash['metering_secret'],
ceilometer_db_password => $ceilometer_hash['db_password'],
ceilometer_replset_members => values($mongo_address_map),
replset_name => $replset_name,
mongo_version => '2.6.10',
use_syslog => $use_syslog,
debug => $debug,
}
if !(member($roles, 'controller') or member($roles, 'primary-controller')) {

37
deployment/puppet/osnailyfacter/modular/roles/mongo_primary.pp
View File

@ -1,37 +0,0 @@
notice('MODULAR: mongo_primary.pp')
prepare_network_config(hiera('network_scheme', {}))
$mongo_nodes = get_nodes_hash_by_roles(hiera('network_metadata'), hiera('mongo_roles'))
$mongo_address_map = get_node_to_ipaddr_map_by_network_role($mongo_nodes, 'mongo/db')
$bind_address = get_network_role_property('mongo/db', 'ipaddr')
$use_syslog = hiera('use_syslog', true)
$debug = hiera('debug', false)
$ceilometer_hash = hiera_hash('ceilometer_hash')
$roles = hiera('roles')
$replset_name = 'ceilometer'
$mongodb_port = hiera('mongodb_port', '27017')
####################################################################
firewall {'120 mongodb':
port => $mongodb_port,
proto => 'tcp',
action => 'accept',
} ->
class { 'openstack::mongo_primary':
mongodb_bind_address => [ '127.0.0.1', $bind_address ],
mongodb_port => $mongodb_port,
ceilometer_metering_secret => $ceilometer_hash['metering_secret'],
ceilometer_db_password => $ceilometer_hash['db_password'],
ceilometer_replset_members => values($mongo_address_map),
replset_name => $replset_name,
mongo_version => '2.6.10',
use_syslog => $use_syslog,
debug => $debug,
}
if !(member($roles, 'controller') or member($roles, 'primary-controller')) {
sysctl::value { 'net.ipv4.tcp_keepalive_time':
value => '300',
}
}

2
deployment/puppet/osnailyfacter/modular/roles/tasks.yaml
View File

@ -33,7 +33,7 @@
required_for: [deploy_end]
requires: [hosts, firewall]
parameters:
puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/roles/mongo_primary.pp
puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/roles/mongo.pp
puppet_modules: /etc/puppet/modules
timeout: 3600