From 831f144e81e03b94cf49660cf7378d27f122939e Mon Sep 17 00:00:00 2001 From: David Gurtner Date: Fri, 6 Jun 2014 13:41:16 +0200 Subject: [PATCH] Mon profile for roles/profiles pattern This provides a profile for monitor instances in the context of a roles/profiles pattern. The roles/profiles pattern is a way to combine ceph manifests into functional units: Profiles combine multiple manifest to provide a single service. For example a ceph monitor server needs the repository, packages, configuration and finally the monitor service. Roles define sets of profiles to configure a specific server. For example an allinone role would install the monitor as well as an osd profile. The learn more have a look at: http://www.slideshare.net/PuppetLabs/roles-talk All configuration happens via hiera. Example hiera files are provided. Change-Id: If07d18cdb202706c8478570027e80a85c8159d31 NB: this uses hiera autoloading and will only work with Puppet >=3.0 --- examples/common.yaml | 15 +- manifests/profile/mon.pp | 61 +++++ spec/classes/ceph_profile_mon_spec.rb | 108 ++++++++ spec/fixtures/hieradata/common.yaml | 28 ++ spec/spec_helper.rb | 16 ++ spec/system/ceph_profile_mon_spec.rb | 365 ++++++++++++++++++++++++++ 6 files changed, 586 insertions(+), 7 deletions(-) create mode 100644 manifests/profile/mon.pp create mode 100644 spec/classes/ceph_profile_mon_spec.rb create mode 100644 spec/fixtures/hieradata/common.yaml create mode 100644 spec/system/ceph_profile_mon_spec.rb diff --git a/examples/common.yaml b/examples/common.yaml index 343006c3..b2994448 100644 --- a/examples/common.yaml +++ b/examples/common.yaml @@ -15,13 +15,14 @@ ceph::profile::params::cluster_network: '10.12.13.0/24' ceph::profile::params::public_network: '10.11.12.0/24' ######## Keys -ceph::profiles::params::mon_key: 'AQATGHJTUCBqIBAA7M2yafV1xctn1pgr3GcKPg==' -ceph::profiles::params::mon_keyring: '/tmp/keyring' -ceph::profiles::params::admin_key: 'AQBMGHJTkC8HKhAAJ7NH255wYypgm1oVuV41MA==' -ceph::profiles::params::admin_key_mode: '0600' -ceph::profiles::params::bootstrap_osd_key: 'AQARG3JTsDDEHhAAVinHPiqvJkUi5Mww/URupw==' -ceph::profiles::params::bootstrap_mds_key: 'AQCztJdSyNb0NBAASA2yPZPuwXeIQnDJ9O8gVw==' -ceph::profiles::params::osds: +ceph::profile::params::mon_key: 'AQATGHJTUCBqIBAA7M2yafV1xctn1pgr3GcKPg==' +# as an alternative to specifying the mon key you can provide an exising keyring +#ceph::profile::params::mon_keyring: '/etc/ceph/ceph.mon.keyring' +ceph::profile::params::admin_key: 'AQBMGHJTkC8HKhAAJ7NH255wYypgm1oVuV41MA==' +ceph::profile::params::admin_key_mode: '0600' +ceph::profile::params::bootstrap_osd_key: 'AQARG3JTsDDEHhAAVinHPiqvJkUi5Mww/URupw==' +ceph::profile::params::bootstrap_mds_key: 'AQCztJdSyNb0NBAASA2yPZPuwXeIQnDJ9O8gVw==' +ceph::profile::params::osds: '/dev/sdc': journal: '/dev/sdb1' '/dev/sdd': diff --git a/manifests/profile/mon.pp b/manifests/profile/mon.pp new file mode 100644 index 00000000..d9c237b8 --- /dev/null +++ b/manifests/profile/mon.pp @@ -0,0 +1,61 @@ +# +# Copyright (C) 2014 Nine Internet Solutions AG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Author: David Gurtner +# +# Profile for a Ceph mon +# +class ceph::profile::mon inherits ceph::profile::base { + + Ceph_Config<| |> -> + ceph::mon { $::hostname: + authentication_type => $ceph::profile::params::authentication_type, + key => $ceph::profile::params::mon_key, + keyring => $ceph::profile::params::mon_keyring, + } + + Ceph::Key { + inject => true, + inject_as_id => 'mon.', + inject_keyring => "/var/lib/ceph/mon/ceph-${::hostname}/keyring", + } + + # this supports providing the key manually + if $ceph::profile::params::admin_key { + ceph::key { 'client.admin': + secret => $ceph::profile::params::admin_key, + cap_mon => 'allow *', + cap_osd => 'allow *', + cap_mds => 'allow', + mode => $ceph::profile::params::admin_key_mode, + } + } + + if $ceph::profile::params::bootstrap_osd_key { + ceph::key { 'client.bootstrap-osd': + secret => $ceph::profile::params::bootstrap_osd_key, + keyring_path => '/var/lib/ceph/bootstrap-osd/ceph.keyring', + cap_mon => 'allow profile bootstrap-osd', + } + } + + if $ceph::profile::params::bootstrap_mds_key { + ceph::key { 'client.bootstrap-mds': + secret => $ceph::profile::params::bootstrap_mds_key, + keyring_path => '/var/lib/ceph/bootstrap-mds/ceph.keyring', + cap_mon => 'allow profile bootstrap-mds', + } + } +} diff --git a/spec/classes/ceph_profile_mon_spec.rb b/spec/classes/ceph_profile_mon_spec.rb new file mode 100644 index 00000000..db1b93c1 --- /dev/null +++ b/spec/classes/ceph_profile_mon_spec.rb @@ -0,0 +1,108 @@ +# +# Copyright (C) 2014 Nine Internet Solutions AG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Author: David Gurtner +# +require 'spec_helper' + +describe 'ceph::profile::mon' do + + shared_examples_for 'ceph profile mon' do + it { should contain_ceph__mon('first').with( + :authentication_type => 'cephx', + :key => 'AQATGHJTUCBqIBAA7M2yafV1xctn1pgr3GcKPg==') + } + it { should contain_ceph__key('client.admin').that_requires('Ceph::Mon[first]').with( + :secret => 'AQBMGHJTkC8HKhAAJ7NH255wYypgm1oVuV41MA==', + :cap_mon => 'allow *', + :cap_osd => 'allow *', + :cap_mds => 'allow', + :mode => '0644', + :inject => true, + :inject_as_id => 'mon.', + :inject_keyring => '/var/lib/ceph/mon/ceph-first/keyring') + } + it { should contain_ceph__key('client.bootstrap-osd').that_requires('Ceph::Mon[first]').with( + :secret => 'AQARG3JTsDDEHhAAVinHPiqvJkUi5Mww/URupw==', + :keyring_path => '/var/lib/ceph/bootstrap-osd/ceph.keyring', + :cap_mon => 'allow profile bootstrap-osd', + :inject => true, + :inject_as_id => 'mon.', + :inject_keyring => '/var/lib/ceph/mon/ceph-first/keyring') + } + it { should contain_ceph__key('client.bootstrap-mds').that_requires('Ceph::Mon[first]').with( + :secret => 'AQCztJdSyNb0NBAASA2yPZPuwXeIQnDJ9O8gVw==', + :keyring_path => '/var/lib/ceph/bootstrap-mds/ceph.keyring', + :cap_mon => 'allow profile bootstrap-mds', + :inject => true, + :inject_as_id => 'mon.', + :inject_keyring => '/var/lib/ceph/mon/ceph-first/keyring') + } + end + + context 'on Debian' do + + let :facts do + { + :osfamily => 'Debian', + :lsbdistcodename => 'wheezy', + :operatingsystem => 'Debian', + :hostname => 'first', + } + end + + # dont actually run any tests. these cannot run under puppet 2.7 + # TODO: uncomment once 2.7 is deprecated + #it_configures 'ceph profile mon' + end + + context 'on Ubuntu' do + + let :facts do + { + :osfamily => 'Debian', + :lsbdistcodename => 'precise', + :operatingsystem => 'Ubuntu', + :hostname => 'first', + } + end + + # dont actually run any tests. these cannot run under puppet 2.7 + # TODO: uncomment once 2.7 is deprecated + #it_configures 'ceph profile mon' + end + + context 'on RHEL6' do + + let :facts do + { + :osfamily => 'RedHat', + :operatingsystem => 'RHEL6', + :hostname => 'first', + } + end + + # dont actually run any tests. these cannot run under puppet 2.7 + # TODO: uncomment once 2.7 is deprecated + #it_configures 'ceph profile mon' + end + +end +# Local Variables: +# compile-command: "cd ../.. ; +# bundle install --path=vendor; +# bundle exec rake spec +# " +# End: diff --git a/spec/fixtures/hieradata/common.yaml b/spec/fixtures/hieradata/common.yaml new file mode 100644 index 00000000..e113f5a1 --- /dev/null +++ b/spec/fixtures/hieradata/common.yaml @@ -0,0 +1,28 @@ +--- +######## Ceph +ceph::profile::params::release: 'firefly' + +######## Ceph.conf +ceph::profile::params::fsid: '4b5c8c0a-ff60-454b-a1b4-9747aa737d19' +ceph::profile::params::authentication_type: 'cephx' +ceph::profile::params::mon_initial_members: 'first, second' +ceph::profile::params::mon_host: '10.11.12.2:6789, 10.11.12.3:6789' +ceph::profile::params::osd_pool_default_pg_num: '200' +ceph::profile::params::osd_pool_default_pgp_num: '200' +ceph::profile::params::osd_pool_default_size: '2' +ceph::profile::params::osd_pool_default_min_size: '1' +ceph::profile::params::cluster_network: '10.12.13.0/24' +ceph::profile::params::public_network: '10.11.12.0/24' + +######## Keys +ceph::profile::params::mon_key: 'AQATGHJTUCBqIBAA7M2yafV1xctn1pgr3GcKPg==' +ceph::profile::params::admin_key: 'AQBMGHJTkC8HKhAAJ7NH255wYypgm1oVuV41MA==' +ceph::profile::params::admin_key_mode: '0644' +ceph::profile::params::bootstrap_osd_key: 'AQARG3JTsDDEHhAAVinHPiqvJkUi5Mww/URupw==' +ceph::profile::params::bootstrap_mds_key: 'AQCztJdSyNb0NBAASA2yPZPuwXeIQnDJ9O8gVw==' +ceph::profile::params::osds: + '/dev/sdc': + journal: '/dev/sdb1' + '/dev/sdd': + journal: '/dev/sdb2' + diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 9d0c281a..391f14ad 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -18,5 +18,21 @@ require 'puppetlabs_spec_helper/module_spec_helper' RSpec.configure do |c| + fixture_path = File.expand_path(File.join(File.dirname(__FILE__), 'fixtures')) + + c.hiera_config = File.join(fixture_path, 'hieradata/hiera.yaml') + c.alias_it_should_behave_like_to(:it_configures, 'configures') + + c.before(:all) do + data = YAML.load_file(c.hiera_config) + data[:yaml][:datadir] = File.join(fixture_path, 'hieradata') + File.open(c.hiera_config, 'w') do |f| + f.write data.to_yaml + end + end + + c.after(:all) do + `git checkout -- #{c.hiera_config}` + end end diff --git a/spec/system/ceph_profile_mon_spec.rb b/spec/system/ceph_profile_mon_spec.rb new file mode 100644 index 00000000..50018ffd --- /dev/null +++ b/spec/system/ceph_profile_mon_spec.rb @@ -0,0 +1,365 @@ +# +# Copyright 2014 (C) Nine Internet Solutions AG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Author: David Gurtner +# +require 'spec_helper_system' + +describe 'ceph::profile::mon' do + + releases = ENV['RELEASES'] ? ENV['RELEASES'].split : [ 'dumpling', 'emperor', 'firefly' ] + machines = ENV['MACHINES'] ? ENV['MACHINES'].split : [ 'first', 'second' ] + # passing it directly as unqoted array is not supported everywhere + packages = "[ 'python-ceph', 'ceph-common', 'librados2', 'librbd1', 'libcephfs1' ]" + fsid = 'a4807c9a-e76f-4666-a297-6d6cbc922e3a' + admin_key = 'AQBMGHJTkC8HKhAAJ7NH255wYypgm1oVuV41MA==' + mon_key = 'AQATGHJTUCBqIBAA7M2yafV1xctn1pgr3GcKPg==' + hieradata_common = '/var/lib/hiera/common.yaml' + hiera_shared = <<-EOS +--- +ceph::profile::params::fsid: '#{fsid}' + EOS + + releases.each do |release| + describe release do + after(:all) do + pp = <<-EOS + package { #{packages}: + ensure => purged + } + class { 'ceph::repo': + release => '#{release}', + ensure => absent, + } + EOS + + machines.each do |vm| + puppet_apply(:node => vm, :code => pp) do |r| + r.exit_code.should_not == 1 + end + end + end + + describe 'on one host' do + it 'should install one monitor' do + hiera = <<-EOS +ceph::profile::params::release: '#{release}' +ceph::profile::params::authentication_type: 'none' +ceph::profile::params::mon_initial_members: 'first' +ceph::profile::params::mon_host: '10.11.12.2:6789' + EOS + + file = Tempfile.new('hieradata') + begin + file.write(hiera_shared + hiera) + file.close + rcp(:sp => file.path, :dp => hieradata_common, :d => node) + ensure + file.unlink + end + + pp = <<-EOS + include ::ceph::profile::mon + EOS + + puppet_apply(pp) do |r| + r.exit_code.should_not == 1 + r.refresh + r.exit_code.should_not == 1 + end + + shell 'ceph -s' do |r| + r.stdout.should =~ /1 mons .* quorum 0 first/ + r.stderr.should be_empty + r.exit_code.should be_zero + end + end + + it 'should uninstall one monitor' do + pp = <<-EOS + ceph::mon { 'first': + ensure => absent, + } + EOS + + puppet_apply(pp) do |r| + r.exit_code.should_not == 1 + end + + osfamily = facter.facts['osfamily'] + operatingsystem = facter.facts['operatingsystem'] + + if osfamily == 'Debian' && operatingsystem == 'Ubuntu' + shell 'status ceph-mon id=first' do |r| + r.stdout.should be_empty + r.stderr.should =~ /Unknown instance: ceph.first/ + r.exit_code.should_not be_zero + end + end + if osfamily == 'RedHat' + shell 'service ceph status mon.first' do |r| + r.stdout.should =~ /mon.first not found/ + r.stderr.should be_empty + r.exit_code.should_not be_zero + end + end + end + end + + describe 'on one host', :cephx do + it 'should install one monitor with key' do + hiera = <<-EOS +ceph::profile::params::release: '#{release}' +ceph::profile::params::authentication_type: 'cephx' +ceph::profile::params::admin_key: '#{admin_key}' +ceph::profile::params::mon_key: '#{mon_key}' +ceph::profile::params::mon_initial_members: 'first' +ceph::profile::params::mon_host: '10.11.12.2:6789' + EOS + + file = Tempfile.new('hieradata') + begin + file.write(hiera_shared + hiera) + file.close + rcp(:sp => file.path, :dp => hieradata_common, :d => node) + ensure + file.unlink + end + + pp = <<-EOS + include ::ceph::profile::mon + EOS + + puppet_apply(pp) do |r| + r.exit_code.should_not == 1 + r.refresh + r.exit_code.should_not == 1 + end + + shell 'ceph -s' do |r| + r.stdout.should =~ /1 mons .* quorum 0 first/ + r.stderr.should be_empty + r.exit_code.should be_zero + end + + shell 'ceph auth list' do |r| + r.stdout.should =~ /#{admin_key}/ + r.exit_code.should be_zero + end + end + + it 'should uninstall one monitor' do + pp = <<-EOS + ceph::mon { 'first': + ensure => absent, + } + EOS + + puppet_apply(pp) do |r| + r.exit_code.should_not == 1 + end + + osfamily = facter.facts['osfamily'] + operatingsystem = facter.facts['operatingsystem'] + + if osfamily == 'Debian' && operatingsystem == 'Ubuntu' + shell 'status ceph-mon id=first' do |r| + r.stdout.should be_empty + r.stderr.should =~ /Unknown instance: ceph.first/ + r.exit_code.should_not be_zero + end + end + if osfamily == 'RedHat' + shell 'service ceph status mon.first' do |r| + r.stdout.should =~ /mon.first not found/ + r.stderr.should be_empty + r.exit_code.should_not be_zero + end + end + end + + it 'should install one monitor with keyring' do + key = 'AQCztJdSyNb0NBAASA2yPZPuwXeIQnDJ9O8gVw==' + keyring = "[mon.]\\n\\tkey = #{key}\\n\\tcaps mon = \"allow *\"" + keyring_path = "/tmp/keyring" + shell "echo -e '#{keyring}' > #{keyring_path}" + + hiera = <<-EOS +ceph::profile::params::release: '#{release}' +ceph::profile::params::authentication_type: 'cephx' +ceph::profile::params::admin_key: '#{admin_key}' +ceph::profile::params::mon_keyring: '#{keyring_path}' +ceph::profile::params::mon_initial_members: 'first' +ceph::profile::params::mon_host: '10.11.12.2:6789' + EOS + + file = Tempfile.new('hieradata') + begin + file.write(hiera_shared + hiera) + file.close + rcp(:sp => file.path, :dp => hieradata_common, :d => node) + ensure + file.unlink + end + + pp = <<-EOS + include ::ceph::profile::mon + EOS + + puppet_apply(pp) do |r| + r.exit_code.should_not == 1 + r.refresh + r.exit_code.should_not == 1 + end + shell 'ceph -s' do |r| + r.stdout.should =~ /1 mons .* quorum 0 first/ + + r.stderr.should be_empty + r.exit_code.should be_zero + end + + shell 'ceph auth list' do |r| + r.stdout.should =~ /#{admin_key}/ + r.exit_code.should be_zero + end + end + + it 'should uninstall one monitor' do + pp = <<-EOS + ceph::mon { 'first': + ensure => absent, + } + EOS + + puppet_apply(pp) do |r| + r.exit_code.should_not == 1 + end + + osfamily = facter.facts['osfamily'] + operatingsystem = facter.facts['operatingsystem'] + + if osfamily == 'Debian' && operatingsystem == 'Ubuntu' + shell 'status ceph-mon id=first' do |r| + r.stdout.should be_empty + r.stderr.should =~ /Unknown instance: ceph.first/ + r.exit_code.should_not be_zero + end + end + if osfamily == 'RedHat' + shell 'service ceph status mon.first' do |r| + r.stdout.should =~ /mon.first not found/ + r.stderr.should be_empty + r.exit_code.should_not be_zero + end + end + end + end + + describe 'on two hosts' do + it 'should be two hosts' do + machines.size.should == 2 + end + + it 'should install two monitors' do + [ 'first', 'second' ].each do |mon| + hiera = <<-EOS +ceph::profile::params::release: '#{release}' +ceph::profile::params::authentication_type: 'none' +ceph::profile::params::public_network: '10.11.12.0/24' +ceph::profile::params::mon_initial_members: 'first, second' +ceph::profile::params::mon_host: '10.11.12.2,10.11.12.3' + EOS + + file = Tempfile.new('hieradata') + begin + file.write(hiera_shared + hiera) + file.close + rcp(:sp => file.path, :dp => hieradata_common, :d => node(:name => mon)) + ensure + file.unlink + end + + pp = <<-EOS + include ::ceph::profile::mon + EOS + + puppet_apply(:node => mon, :code => pp) do |r| + r.exit_code.should_not == 1 + r.refresh + r.exit_code.should_not == 1 + end + end + + shell 'ceph -s' do |r| + r.stdout.should =~ /2 mons .* quorum 0,1 first,second/ + r.stderr.should be_empty + r.exit_code.should be_zero + end + end + + it 'should uninstall two monitors' do + machines.each do |mon| + pp = <<-EOS + ceph::mon { '#{mon}': + ensure => absent, + } + EOS + + puppet_apply(:node => mon, :code => pp) do |r| + r.exit_code.should_not == 1 + end + + osfamily = facter.facts['osfamily'] + operatingsystem = facter.facts['operatingsystem'] + + if osfamily == 'Debian' && operatingsystem == 'Ubuntu' + shell "status ceph-mon id=#{mon}" do |r| + r.stdout.should be_empty + r.stderr.should =~ /Unknown instance: ceph.#{mon}/ + r.exit_code.should_not be_zero + end + end + if osfamily == 'RedHat' + shell "service ceph status mon.#{mon}" do |r| + r.stdout.should =~ /mon.#{mon} not found/ + r.stderr.should be_empty + r.exit_code.should_not be_zero + end + end + end + end + end + end + end +end +# Local Variables: +# compile-command: "cd ../.. +# ( +# cd .rspec_system/vagrant_projects/two-ubuntu-server-12042-x64 +# vagrant destroy --force +# ) +# cp -a Gemfile-rspec-system Gemfile +# BUNDLE_PATH=/tmp/vendor bundle install --no-deployment +# MACHINES='first second' \ +# RELEASES=dumpling \ +# RS_DESTROY=no \ +# RS_SET=two-ubuntu-server-12042-x64 \ +# BUNDLE_PATH=/tmp/vendor \ +# bundle exec rake spec:system \ +# SPEC=spec/system/ceph_profile_mon_spec.rb \ +# SPEC_OPTS='--tag cephx' && +# git checkout Gemfile +# " +# End: