From 4e114c96d528bc7a7b9e2e5055fa65ea9338ce94 Mon Sep 17 00:00:00 2001 From: Dan Bode Date: Thu, 5 Apr 2012 15:45:49 -0700 Subject: [PATCH 1/9] Update README The keystone module has made a lot of progress. This commit updates the README to reflect that. --- deployment/puppet/keystone/README | 37 ------- deployment/puppet/keystone/README.markdown | 118 +++++++++++++++++++++ 2 files changed, 118 insertions(+), 37 deletions(-) delete mode 100644 deployment/puppet/keystone/README create mode 100644 deployment/puppet/keystone/README.markdown diff --git a/deployment/puppet/keystone/README b/deployment/puppet/keystone/README deleted file mode 100644 index ee3d85a77f..0000000000 --- a/deployment/puppet/keystone/README +++ /dev/null @@ -1,37 +0,0 @@ ---- - -module for managing keystone: - -Has only been tested mainly on Ubuntu Precise - -Has only been tested with the essex-4 version of keystone - -manifests: - -init.pp - - contains the keystone class that can be used - to install keystone from apt - -dev/install.pp - - installs keystone from source and sets up a dev - environment as recommended in the development guide - - http://keystone.openstack.org/setup.html - -native types: - - I would like to create types to manage the configuration of - keystone, including: - - - keystone_tenant - - keystone_user - - keystone_role - - keystone_user_role - - keystone_service - - keystone_endpoint - - The sole provider for all of these types is keystone - - These types can only be run on the same machine where keystone is configured diff --git a/deployment/puppet/keystone/README.markdown b/deployment/puppet/keystone/README.markdown new file mode 100644 index 0000000000..eb1bd82a43 --- /dev/null +++ b/deployment/puppet/keystone/README.markdown @@ -0,0 +1,118 @@ +# Overview # + +The module contains a collection of manifests and native types that are capable +of installing/managing/configuring Keystone. + +Keystone is the Identity service for OpenStack. + +# Tested use cases # + +This module has been tested against the dev version of Ubuntu Precise. + +It has only currently been tested as a single node installation of keystone. + +It is currently targetting essex support and is being actively developed against +packaging that are built off of trunk. + +# Dependencies: # + +This module has relatively few dependencies: + + https://github.com/puppetlabs/puppet-concat + # if used on Ubuntu + https://github.com/puppetlabs/puppet-apt + # if using mysql as a backend + https://github.com/puppetlabs/puppetlabs-mysql + +# Usage # +## Manifests ## + +### class keystone ### + +The keystone class sets up the basic configuration for the keystone service. + +It must be used together with a class that expresses the db backend to use: + +for example: + + class { 'keystone': + log_verbose => 'True', + service_token => 'my_secret_token' + } + + needs to be configured to use a backend database with either: + + class { 'keystone::config::sqlite': } + + or + + class { 'keystone::config::mysql': + password => 'keystone', + } + + +### setting up a keystone mysql db ### + + a keystone mysql database can be configured separately from + the service. + + If you need to actually install a mysql database server, you can use + the mysql::server class from the puppetlabs mysql module + + # check out the mysql module's README to learn more about + # how to more appropriately configure a server + class { 'mysql::server': } + + class { 'keystone::mysql': + dbname => 'keystone', + user => 'keystone', + password => 'keystone_password', + } + +## Native Types ## + + The Puppet support for keystone also includes native types that can be + used to manage the following keystone objects: + + - keystone_tenant + - keystone_user + - keystone_role + - keystone_user_role + - keystone_service + - keystone_endpoint + + These types will only work on an actualy keystone node (and they read keystone.conf + to figure out the admin port and admin token, which is kind of hacky, but the best + way I could think of.) + +### examples ### + + keystone_tenant { 'openstack': + ensure => present, + enabled => 'True', + } + keystone_user { 'openstack': + ensure => present, + enabled => 'True' + } + keystone_role { 'admin': + ensure => present, + } + keystone_user_role { 'admin@openstack': + roles => ['admin', 'superawesomedue'], + ensure => present + } + +### puppet resource ### + +These native types also allow for some interesting introspection using puppet resource + +To list all of the objects of a certain type in the keystone database, you can run: + + puppet resource + +For example: + + puppet resource keystone_tenant + + would list all know keystone tenants for a given keystone instance. From d0b4c750290b267946e4c2c368935f1eb85140f5 Mon Sep 17 00:00:00 2001 From: Dan Bode Date: Thu, 5 Apr 2012 16:56:07 -0700 Subject: [PATCH 2/9] Remove python-migrate dependency This has been fixed in the packages so I can remove this code. --- deployment/puppet/keystone/manifests/init.pp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/deployment/puppet/keystone/manifests/init.pp b/deployment/puppet/keystone/manifests/init.pp index 2bb8040ec7..c92023b520 100644 --- a/deployment/puppet/keystone/manifests/init.pp +++ b/deployment/puppet/keystone/manifests/init.pp @@ -23,13 +23,6 @@ class keystone( # this package dependency needs to be removed when it include 'keystone::params' include 'concat::setup' - # is added as a package dependency - # I filed the following ticket against the packages: 909941 - if(! defined(Package['python-migrate'])) { - package { 'python-migrate': - ensure => present, - } - } package { 'keystone': ensure => $package_ensure, From d574b371a02d69fd376756b9a49e5c8e9e387f0d Mon Sep 17 00:00:00 2001 From: Dan Bode Date: Thu, 5 Apr 2012 16:56:38 -0700 Subject: [PATCH 3/9] Remove keystone.conf.erb This file config has been moved to fragments --- .../keystone/templates/keystone.conf.erb | 81 ------------------- 1 file changed, 81 deletions(-) delete mode 100644 deployment/puppet/keystone/templates/keystone.conf.erb diff --git a/deployment/puppet/keystone/templates/keystone.conf.erb b/deployment/puppet/keystone/templates/keystone.conf.erb deleted file mode 100644 index a3d91332bf..0000000000 --- a/deployment/puppet/keystone/templates/keystone.conf.erb +++ /dev/null @@ -1,81 +0,0 @@ -[DEFAULT] -# Show more verbose log output (sets INFO log level output) -verbose = <%= log_verbose %> - -# Show debugging output in logs (sets DEBUG log level output) -debug = <%= log_debug %> - -# Which backend store should Keystone use by default. -# Default: 'sqlite' -# Available choices are 'sqlite' [future will include LDAP, PAM, etc] -default_store = <%= default_store %> - -# Log to this file. Make sure you do not set the same log -# file for both the API and registry servers! -log_file = /var/log/keystone/keystone.log - -# List of backends to be configured -backends = keystone.backends.sqlalchemy -#For LDAP support, add: ,keystone.backends.ldap - -# Dictionary Maps every service to a header.Missing services would get header -# X_(SERVICE_NAME) Key => Service Name, Value => Header Name -service-header-mappings = { - 'nova' : 'X-Server-Management-Url', - 'swift' : 'X-Storage-Url', - 'cdn' : 'X-CDN-Management-Url'} - -# Address to bind the API server -# TODO Properties defined within app not available via pipeline. -service_host = <%= bind_host %> - -# Port the bind the API server to -service_port = <%= bind_port %> - -# Address to bind the Admin API server -admin_host = <%= admin_bind_host %> - -# Port the bind the Admin API server to -admin_port = <%= admin_bind_port %> - -#Role that allows to perform admin operations. -keystone-admin-role = Admin - -#Role that allows to perform service admin operations. -keystone-service-admin-role = KeystoneServiceAdmin - -[keystone.backends.sqlalchemy] -# SQLAlchemy connection string for the reference implementation registry -# server. Any valid SQLAlchemy connection string is fine. -# See: http://bit.ly/ideIpI -sql_connection = sqlite:////var/lib/keystone/keystone.db -backend_entities = ['UserRoleAssociation', 'Endpoints', 'Role', 'Tenant', - 'User', 'Credentials', 'EndpointTemplates', 'Token', - 'Service'] - -# Period in seconds after which SQLAlchemy should reestablish its connection -# to the database. -sql_idle_timeout = 30 - -[pipeline:admin] -pipeline = - urlrewritefilter - admin_api - -[pipeline:keystone-legacy-auth] -pipeline = - urlrewritefilter - legacy_auth - service_api - -[app:service_api] -paste.app_factory = keystone.server:service_app_factory - -[app:admin_api] -paste.app_factory = keystone.server:admin_app_factory - -[filter:urlrewritefilter] -paste.filter_factory = keystone.middleware.url:filter_factory - -[filter:legacy_auth] -paste.filter_factory = keystone.frontends.legacy_token_auth:filter_factory From c09ada921b0e5153a66412ba55966cd5e49ba492 Mon Sep 17 00:00:00 2001 From: Dan Bode Date: Thu, 5 Apr 2012 16:57:33 -0700 Subject: [PATCH 4/9] fixen broken dependency It was set incorrectly to a nonexistent class. This commit sets it to ensure the sqlite db file is removed before the mysql db is created. --- deployment/puppet/keystone/manifests/mysql.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/puppet/keystone/manifests/mysql.pp b/deployment/puppet/keystone/manifests/mysql.pp index 95750139bf..edb583afb9 100644 --- a/deployment/puppet/keystone/manifests/mysql.pp +++ b/deployment/puppet/keystone/manifests/mysql.pp @@ -14,7 +14,7 @@ class keystone::mysql( file { '/var/lib/keystone/keystone.db': ensure => absent, subscribe => Package['keystone'], - before => Class['keystone::db'], + before => Mysql::Db[$dbname], } mysql::db { $dbname: From 9dc487a175c22d37a21898c357284bd555cdc9d3 Mon Sep 17 00:00:00 2001 From: Dan Bode Date: Thu, 5 Apr 2012 16:58:36 -0700 Subject: [PATCH 5/9] Improve spec test coverage --- .../spec/classes/keystone_mysql_spec.rb | 51 ++++++++++ .../keystone/spec/classes/keystone_spec.rb | 93 ++++++++++++++----- .../puppet/keystone/spec/spec_helper.rb | 5 + 3 files changed, 124 insertions(+), 25 deletions(-) create mode 100644 deployment/puppet/keystone/spec/classes/keystone_mysql_spec.rb diff --git a/deployment/puppet/keystone/spec/classes/keystone_mysql_spec.rb b/deployment/puppet/keystone/spec/classes/keystone_mysql_spec.rb new file mode 100644 index 0000000000..23734ac3ce --- /dev/null +++ b/deployment/puppet/keystone/spec/classes/keystone_mysql_spec.rb @@ -0,0 +1,51 @@ +require 'spec_helper' + +describe 'keystone::mysql' do + + let :facts do + { :osfamily => 'Debian' } + end + + let :param_defaults do + { + 'password' => 'keystone_default_password', + 'dbname' => 'keystone', + 'user' => 'keystone_admin', + 'host' => '127.0.0.1' + } + end + + [ + {}, + { + 'password' => 'password', + 'dbname' => 'not_keystone', + 'user' => 'dan', + 'host' => '127.0.0.2' + } + ].each do |p| + + let :params do + p + end + + let :param_values do + param_defaults.merge(p) + end + + it { should contain_file('/var/lib/keystone/keystone.db').with( + 'ensure' => 'absent', + 'subscribe' => 'Package[keystone]', + 'before' => "Mysql::Db[#{param_values['dbname']}]" + )} + + it { should contain_mysql__db(param_values['dbname']).with( + 'user' => param_values['user'], + 'password' => param_values['password'], + 'host' => param_values['host'], + 'charset' => 'latin1', + 'require' => 'Class[Mysql::Server]' + )} + end + +end diff --git a/deployment/puppet/keystone/spec/classes/keystone_spec.rb b/deployment/puppet/keystone/spec/classes/keystone_spec.rb index c7adc6d2cc..8004ddbe42 100644 --- a/deployment/puppet/keystone/spec/classes/keystone_spec.rb +++ b/deployment/puppet/keystone/spec/classes/keystone_spec.rb @@ -2,43 +2,63 @@ require 'spec_helper' describe 'keystone' do + let :concat_file do + { + :type => 'File', + :title => '/var/lib/puppet/concat/_etc_keystone_keystone.conf/fragments.concat.out' + } + end + let :default_params do { 'package_ensure' => 'present', + 'bind_host' => '0.0.0.0', + 'public_port' => '5000', + 'admin_port' => '35357', + 'admin_token' => 'service_token', + 'compute_port' => '3000', 'log_verbose' => 'False', 'log_debug' => 'False', - 'default_store' => 'sqlite', - 'bind_host' => '0.0.0.0', - 'bind_port' => '5000', - 'admin_bind_host' => '0.0.0.0', - 'admin_bind_port' => '5001' + 'use_syslog' => 'False', + 'catalog_type' => 'template' } end [{}, { 'package_ensure' => 'latest', + 'bind_host' => '127.0.0.1', + 'public_port' => '5001', + 'admin_port' => '35358', + 'admin_token' => 'service_token_override', + 'compute_port' => '3001', 'log_verbose' => 'True', 'log_debug' => 'True', - 'default_store' => 'ldap', - 'bind_host' => '127.0.0.1', - 'bind_port' => '50000', - 'admin_bind_host' => '127.0.0.1', - 'admin_bind_port' => '50001' + 'catalog_type' => 'sql' } ].each do |param_set| describe "when #{param_set == {} ? "using default" : "specifying"} class parameters" do let :param_hash do - param_set == {} ? default_params : param_set + default_params.merge(param_set) end let :params do param_set end + it { should contain_class('keystone::params') } + + it { should contain_class('concat::setup') } + it { should contain_package('keystone').with_ensure(param_hash['package_ensure']) } + it { should contain_group('keystone').with_ensure('present') } + it { should contain_user('keystone').with( + 'ensure' => 'present', + 'gid' => 'keystone' + ) } + it { should contain_file('/etc/keystone').with( 'ensure' => 'directory', 'owner' => 'keystone', @@ -47,8 +67,12 @@ describe 'keystone' do 'require' => 'Package[keystone]' ) } - # maybe keystone should always be with the API server? - it 'should refresh nova-api if they are on the same machine' + it { should contain_concat('/etc/keystone/keystone.conf').with( + 'owner' => 'keystone', + 'group' => 'keystone', + 'require' => 'Package[keystone]', + 'notify' => 'Service[keystone]' + )} it { should contain_service('keystone').with( 'ensure' => 'running', @@ -57,19 +81,38 @@ describe 'keystone' do 'hasrestart' => 'true' ) } - it 'should compile the template based on the class parameters' do - content = param_value(subject, 'file', 'keystone.conf', 'content') - expected_lines = [ - "verbose = #{param_hash['log_verbose']}", - "debug = #{param_hash['log_debug']}", - "default_store = #{param_hash['default_store']}", - "service_host = #{param_hash['bind_host']}", - "service_port = #{param_hash['bind_port']}", - "admin_host = #{param_hash['admin_bind_host']}", - "admin_port = #{param_hash['admin_bind_port']}" - ] - (content.split("\n") & expected_lines).should == expected_lines + it 'should correctly configure catalog based on catalog_type' + + it 'should create the expected DEFAULT configuration' do +#require 'ruby-debug';debugger + verify_contents( + subject, + '/var/lib/puppet/concat/_etc_keystone_keystone.conf/fragments/00_kestone-DEFAULT', + [ + "bind_host = #{param_hash['bind_host']}", + "public_port = #{param_hash['public_port']}", + "admin_port = #{param_hash['admin_port']}", + "admin_token = #{param_hash['admin_token']}", + "compute_port = #{param_hash['compute_port']}", + "verbose = #{param_hash['log_verbose']}", + "debug = #{param_hash['log_debug']}", + "log_file = /var/log/keystone/keystone.log", + "use_syslog = #{param_hash['use_syslog']}" + ] + ) end + it 'should create the expected identity section' do + verify_contents( + subject, + '/var/lib/puppet/concat/_etc_keystone_keystone.conf/fragments/03_kestone-identity', + [ + "[identity]", + "driver = keystone.identity.backends.sql.Identity" + ] + ) + end + it { should create_file( + '/var/lib/puppet/concat/_etc_keystone_keystone.conf/fragments/99_kestone-footer') } end end end diff --git a/deployment/puppet/keystone/spec/spec_helper.rb b/deployment/puppet/keystone/spec/spec_helper.rb index 44c24145fa..c337f33781 100644 --- a/deployment/puppet/keystone/spec/spec_helper.rb +++ b/deployment/puppet/keystone/spec/spec_helper.rb @@ -6,6 +6,11 @@ def param_value(subject, type, title, param) subject.resource(type, title).send(:parameters)[param.to_sym] end +def verify_contents(subject, title, expected_lines) + content = subject.resource('file', title).send(:parameters)[:content] + (content.split("\n") & expected_lines).should == expected_lines +end + RSpec.configure do |c| c.manifest_dir = File.expand_path(File.join(File.dirname(__FILE__), 'fixtures/manifests')) c.module_path = File.join(File.dirname(__FILE__), '../../') From 76ec99e3b5012f30d932b4e42a931cdabca3dc06 Mon Sep 17 00:00:00 2001 From: Dan Bode Date: Thu, 5 Apr 2012 21:39:40 -0700 Subject: [PATCH 6/9] Extend keystone example This commit fixes the keystone mysql example by ensuring that a mysql::server is installed. It also adds the keystone default roles for robustness. --- deployment/puppet/keystone/examples/site.pp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deployment/puppet/keystone/examples/site.pp b/deployment/puppet/keystone/examples/site.pp index 9e66555441..a6638abadb 100644 --- a/deployment/puppet/keystone/examples/site.pp +++ b/deployment/puppet/keystone/examples/site.pp @@ -29,6 +29,7 @@ node keystone { } node keystone_mysql { + class { 'mysql::server': } class { 'keystone::mysql': password => 'keystone', } @@ -39,7 +40,8 @@ node keystone_mysql { log_verbose => true, log_debug => true, catalog_type => 'sql', - } + }-> + class { 'keystone::roles::admin': } } node default { From 59ab56a29bdffb483d0ee6e97b111064fef01686 Mon Sep 17 00:00:00 2001 From: Dan Bode Date: Thu, 5 Apr 2012 23:40:41 -0700 Subject: [PATCH 7/9] Fix sql section name for keystone.conf template It was printing mysql instead of sql causing keystone to fail. This commit fixes the issue. --- deployment/puppet/keystone/templates/mysql.erb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deployment/puppet/keystone/templates/mysql.erb b/deployment/puppet/keystone/templates/mysql.erb index cc8a1ae7f5..0ff07d0e50 100644 --- a/deployment/puppet/keystone/templates/mysql.erb +++ b/deployment/puppet/keystone/templates/mysql.erb @@ -1,4 +1,5 @@ -[<%= name %>] + +[sql] connection = mysql://<%= "#{config['user']}:#{config['password']}@#{config['host']}/#{config['dbname']}" %> idle_timeout = <%= config['idle_timeout'] %> min_pool_size = <%= config['min_pool_size'] %> From 864640121bec93c15fbd8550932d30f723f1c2cc Mon Sep 17 00:00:00 2001 From: Dan Bode Date: Thu, 5 Apr 2012 23:53:18 -0700 Subject: [PATCH 8/9] Fix Mysql support Previously, the mysql support for keystone was not completely working. This commit resolves the following issues: - Adds a define that can authorize db users. - Ensures that keystone-manage db_sync is called when db is created. - Ensure that keystone::mysql is only configured after the keystone class. --- deployment/puppet/keystone/examples/site.pp | 6 +++--- deployment/puppet/keystone/manifests/mysql.pp | 8 ++++++++ .../puppet/keystone/manifests/mysql/access.pp | 13 +++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 deployment/puppet/keystone/manifests/mysql/access.pp diff --git a/deployment/puppet/keystone/examples/site.pp b/deployment/puppet/keystone/examples/site.pp index a6638abadb..bd0a95988c 100644 --- a/deployment/puppet/keystone/examples/site.pp +++ b/deployment/puppet/keystone/examples/site.pp @@ -30,9 +30,6 @@ node keystone { node keystone_mysql { class { 'mysql::server': } - class { 'keystone::mysql': - password => 'keystone', - } class { 'keystone::config::mysql': password => 'keystone' } @@ -41,6 +38,9 @@ node keystone_mysql { log_debug => true, catalog_type => 'sql', }-> + class { 'keystone::mysql': + password => 'keystone', + }-> class { 'keystone::roles::admin': } } diff --git a/deployment/puppet/keystone/manifests/mysql.pp b/deployment/puppet/keystone/manifests/mysql.pp index edb583afb9..1b4ca43a4a 100644 --- a/deployment/puppet/keystone/manifests/mysql.pp +++ b/deployment/puppet/keystone/manifests/mysql.pp @@ -25,4 +25,12 @@ class keystone::mysql( require => Class['mysql::server'], } + # this probably needs to happen more often than just when the db is + # created + exec { 'keystone-manage db_sync': + path => '/usr/bin', + refreshonly => true, + subscribe => Mysql::Db[$dbname], + } + } diff --git a/deployment/puppet/keystone/manifests/mysql/access.pp b/deployment/puppet/keystone/manifests/mysql/access.pp new file mode 100644 index 0000000000..9b85b89949 --- /dev/null +++ b/deployment/puppet/keystone/manifests/mysql/access.pp @@ -0,0 +1,13 @@ +define keystone::mysql::host_access ($user, $password, $database) { + database_user { "${user}@${name}": + password_hash => mysql_password($password), + provider => 'mysql', + require => Database[$database], + } + database_grant { "${user}@${name}/${database}": + # TODO figure out which privileges to grant. + privileges => "all", + provider => 'mysql', + require => Database_user["${user}@${name}"] + } +} From 6dbb9edaef89f8473847e263ac7f4c4a243dcb5d Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Fri, 6 Apr 2012 23:19:05 +0200 Subject: [PATCH 9/9] Use admin token from keystone config --- .../keystone/lib/puppet/provider/keystone_endpoint/keystone.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/puppet/keystone/lib/puppet/provider/keystone_endpoint/keystone.rb b/deployment/puppet/keystone/lib/puppet/provider/keystone_endpoint/keystone.rb index 63d565a99d..8120dd1945 100644 --- a/deployment/puppet/keystone/lib/puppet/provider/keystone_endpoint/keystone.rb +++ b/deployment/puppet/keystone/lib/puppet/provider/keystone_endpoint/keystone.rb @@ -107,7 +107,7 @@ Puppet::Type.type(:keystone_endpoint).provide( # TODO - this needs to be replaced with a call to endpoint-get # but endpoint-get is not currently supported from the admin url def self.get_service_id(endpoint_id) - `python -c "from keystoneclient.v2_0 import client ; import os ; print [e.service_id for e in client.Client(endpoint='http://127.0.0.1:35357/v2.0/', token='service_token').endpoints.list() if e.id == u'#{endpoint_id}'][0]"` + `python -c "from keystoneclient.v2_0 import client ; import os ; print [e.service_id for e in client.Client(endpoint='http://127.0.0.1:35357/v2.0/', token='#{admin_token}').endpoints.list() if e.id == u'#{endpoint_id}'][0]"` end end