Merge pull request #14 from bodepd/dev

Doc +  mysql update

deployment/puppet/keystone/README

@@ -1,37 +0,0 @@
----
-
-module for managing keystone:
-
-Has only been tested mainly on Ubuntu Precise
-
-Has only been tested with the essex-4 version of keystone
-
-manifests:
-
-init.pp
-
-  contains the keystone class that can be used
-  to install keystone from apt
-
-dev/install.pp
-
-  installs keystone from source and sets up a dev
-  environment as recommended in the development guide
-
-    http://keystone.openstack.org/setup.html
-
-native types:
-
-  I would like to create types to manage the configuration of
-  keystone, including:
-
-  - keystone_tenant
-  - keystone_user
-  - keystone_role
-  - keystone_user_role
-  - keystone_service
-  - keystone_endpoint
-
-  The sole provider for all of these types is keystone
-
-  These types can only be run on the same machine where keystone is configured

deployment/puppet/keystone/README.markdown

@@ -0,0 +1,118 @@
+# Overview #
+
+The module contains a collection of manifests and native types that are capable
+of installing/managing/configuring Keystone.
+
+Keystone is the Identity service for OpenStack.
+
+# Tested use cases #
+
+This module has been tested against the dev version of Ubuntu Precise.
+
+It has only currently been tested as a single node installation of keystone.
+
+It is currently targetting essex support and is being actively developed against
+packaging that are built off of trunk.
+
+# Dependencies: #
+
+This module has relatively few dependencies:
+
+  https://github.com/puppetlabs/puppet-concat
+  # if used on Ubuntu
+  https://github.com/puppetlabs/puppet-apt
+  # if using mysql as a backend
+  https://github.com/puppetlabs/puppetlabs-mysql
+
+# Usage #
+## Manifests ##
+
+### class keystone ###
+
+The keystone class sets up the basic configuration for the keystone service.
+
+It must be used together with a class that expresses the db backend to use:
+
+for example:
+
+    class { 'keystone':
+      log_verbose => 'True',
+      service_token => 'my_secret_token'
+    }
+
+  needs to be configured to use a backend database with either:
+
+    class { 'keystone::config::sqlite': }
+
+  or
+
+    class { 'keystone::config::mysql':
+      password => 'keystone',
+    }
+
+
+### setting up a keystone mysql db ###
+
+  a keystone mysql database can be configured separately from
+  the service.
+
+  If you need to actually install a mysql database server, you can use
+  the mysql::server class from the puppetlabs mysql module
+
+    # check out the mysql module's README to learn more about
+    # how to more appropriately configure a server
+    class { 'mysql::server': }
+
+    class { 'keystone::mysql':
+      dbname   => 'keystone',
+      user     => 'keystone',
+      password => 'keystone_password',
+    }
+
+## Native Types ##
+
+  The Puppet support for keystone also includes native types that can be
+  used to manage the following keystone objects:
+
+    - keystone_tenant
+    - keystone_user
+    - keystone_role
+    - keystone_user_role
+    - keystone_service
+    - keystone_endpoint
+
+  These types will only work on an actualy keystone node (and they read keystone.conf
+  to figure out the admin port and admin token, which is kind of hacky, but the best
+  way I could think of.)
+
+### examples ###
+
+  keystone_tenant { 'openstack':
+    ensure  => present,
+    enabled => 'True',
+  }
+  keystone_user { 'openstack':
+    ensure  => present,
+    enabled => 'True'
+  }
+  keystone_role { 'admin':
+    ensure => present,
+  }
+  keystone_user_role { 'admin@openstack':
+    roles => ['admin', 'superawesomedue'],
+    ensure => present
+  }
+
+### puppet resource ###
+
+These native types also allow for some interesting introspection using puppet resource
+
+To list all of the objects of a certain type in the keystone database, you can run:
+
+  puppet resource <type>
+
+For example:
+
+  puppet resource keystone_tenant
+
+  would list all know keystone tenants for a given keystone instance.

deployment/puppet/keystone/examples/site.pp

@@ -29,9 +29,7 @@ node keystone {
 }
 
 node keystone_mysql {
-  class { 'keystone::mysql':
-    password => 'keystone',
-  }
+  class { 'mysql::server': }
   class { 'keystone::config::mysql':
     password => 'keystone'
   }
@@ -39,7 +37,11 @@ node keystone_mysql {
     log_verbose  => true,
     log_debug    => true,
     catalog_type => 'sql',
-  }
+  }->
+  class { 'keystone::mysql':
+    password => 'keystone',
+  }->
+  class { 'keystone::roles::admin': }
 }
 
 node default {

deployment/puppet/keystone/lib/puppet/provider/keystone_endpoint/keystone.rb

@@ -107,7 +107,7 @@ Puppet::Type.type(:keystone_endpoint).provide(
     # TODO - this needs to be replaced with a call to endpoint-get
     # but endpoint-get is not currently supported from the admin url
     def self.get_service_id(endpoint_id)
-      `python -c "from keystoneclient.v2_0 import client ; import os ; print [e.service_id for e in client.Client(endpoint='http://127.0.0.1:35357/v2.0/', token='service_token').endpoints.list() if e.id == u'#{endpoint_id}'][0]"`
+      `python -c "from keystoneclient.v2_0 import client ; import os ; print [e.service_id for e in client.Client(endpoint='http://127.0.0.1:35357/v2.0/', token='#{admin_token}').endpoints.list() if e.id == u'#{endpoint_id}'][0]"`
     end
 
 end

deployment/puppet/keystone/manifests/init.pp

@@ -23,13 +23,6 @@ class keystone(
   # this package dependency needs to be removed when it
   include 'keystone::params'
   include 'concat::setup'
-  # is added as a package dependency
-  # I filed the following ticket against the packages: 909941
-  if(! defined(Package['python-migrate'])) {
-    package { 'python-migrate':
-      ensure => present,
-    }
-  }
 
   package { 'keystone':
     ensure => $package_ensure,

deployment/puppet/keystone/manifests/mysql.pp

@@ -14,7 +14,7 @@ class keystone::mysql(
   file { '/var/lib/keystone/keystone.db':
     ensure    => absent,
     subscribe => Package['keystone'],
-    before    => Class['keystone::db'],
+    before    => Mysql::Db[$dbname],
   }
 
   mysql::db { $dbname:
@@ -25,4 +25,12 @@ class keystone::mysql(
     require      => Class['mysql::server'],
   }
 
+  # this probably needs to happen more often than just when the db is
+  # created
+  exec { 'keystone-manage db_sync':
+    path        => '/usr/bin',
+    refreshonly => true,
+    subscribe   => Mysql::Db[$dbname],
+  }
+
 }

deployment/puppet/keystone/manifests/mysql/access.pp

@@ -0,0 +1,13 @@
+define keystone::mysql::host_access ($user, $password, $database)  {
+  database_user { "${user}@${name}":
+    password_hash => mysql_password($password),
+    provider => 'mysql',
+    require => Database[$database],
+  }
+  database_grant { "${user}@${name}/${database}":
+    # TODO figure out which privileges to grant.
+    privileges => "all",
+    provider => 'mysql',
+    require => Database_user["${user}@${name}"]
+  }
+}

deployment/puppet/keystone/spec/classes/keystone_mysql_spec.rb

@@ -0,0 +1,51 @@
+require 'spec_helper'
+
+describe 'keystone::mysql' do
+
+  let :facts do
+    { :osfamily => 'Debian' }
+  end
+
+  let :param_defaults do
+    {
+      'password' => 'keystone_default_password',
+      'dbname'   => 'keystone',
+      'user'     => 'keystone_admin',
+      'host'     => '127.0.0.1'
+    }
+  end
+
+  [
+    {},
+    {
+      'password' => 'password',
+      'dbname'   => 'not_keystone',
+      'user'     => 'dan',
+      'host'     => '127.0.0.2'
+    }
+  ].each do |p|
+
+    let :params do
+      p
+    end
+
+    let :param_values do
+      param_defaults.merge(p)
+    end
+
+    it { should contain_file('/var/lib/keystone/keystone.db').with(
+      'ensure'    => 'absent',
+      'subscribe' => 'Package[keystone]',
+      'before'    => "Mysql::Db[#{param_values['dbname']}]"
+    )}
+
+    it { should contain_mysql__db(param_values['dbname']).with(
+      'user'     => param_values['user'],
+      'password' => param_values['password'],
+      'host'     => param_values['host'],
+      'charset'  => 'latin1',
+      'require'  => 'Class[Mysql::Server]'
+    )}
+  end
+
+end

deployment/puppet/keystone/spec/classes/keystone_spec.rb

@@ -2,43 +2,63 @@ require 'spec_helper'
 
 describe 'keystone' do
 
+  let :concat_file do
+    {
+      :type  => 'File',
+      :title => '/var/lib/puppet/concat/_etc_keystone_keystone.conf/fragments.concat.out'
+    }
+  end
+
   let :default_params do
     {
       'package_ensure'  => 'present',
+      'bind_host'        => '0.0.0.0',
+      'public_port'     => '5000',
+      'admin_port'      => '35357',
+      'admin_token'     => 'service_token',
+      'compute_port'    => '3000',
       'log_verbose'     => 'False',
       'log_debug'       => 'False',
-      'default_store'   => 'sqlite',
-      'bind_host'       => '0.0.0.0',
-      'bind_port'       => '5000',
-      'admin_bind_host' => '0.0.0.0',
-      'admin_bind_port' => '5001'
+      'use_syslog'      => 'False',
+      'catalog_type'    => 'template'
     }
   end
 
   [{},
    {
       'package_ensure'  => 'latest',
+      'bind_host'        => '127.0.0.1',
+      'public_port'     => '5001',
+      'admin_port'      => '35358',
+      'admin_token'     => 'service_token_override',
+      'compute_port'    => '3001',
       'log_verbose'     => 'True',
       'log_debug'       => 'True',
-      'default_store'   => 'ldap',
-      'bind_host'       => '127.0.0.1',
-      'bind_port'       => '50000',
-      'admin_bind_host' => '127.0.0.1',
-      'admin_bind_port' => '50001'
+      'catalog_type'    => 'sql'
     }
   ].each do |param_set|
 
     describe "when #{param_set == {} ? "using default" : "specifying"} class parameters" do
       let :param_hash do
-        param_set == {} ? default_params : param_set
+        default_params.merge(param_set)
       end
 
       let :params do
         param_set
       end
 
+      it { should contain_class('keystone::params') }
+
+      it { should contain_class('concat::setup') }
+
       it { should contain_package('keystone').with_ensure(param_hash['package_ensure']) }
 
+      it { should contain_group('keystone').with_ensure('present') }
+      it { should contain_user('keystone').with(
+        'ensure' => 'present',
+        'gid'    => 'keystone'
+      ) }
+
       it { should contain_file('/etc/keystone').with(
         'ensure'     => 'directory',
         'owner'      => 'keystone',
@@ -47,8 +67,12 @@ describe 'keystone' do
         'require'    => 'Package[keystone]'
       ) }
 
-      # maybe keystone should always be with the API server?
-      it 'should refresh nova-api if they are on the same machine'
+      it { should contain_concat('/etc/keystone/keystone.conf').with(
+        'owner'   => 'keystone',
+        'group'   => 'keystone',
+        'require' => 'Package[keystone]',
+        'notify'  => 'Service[keystone]'
+      )}
 
       it { should contain_service('keystone').with(
         'ensure'     => 'running',
@@ -57,19 +81,38 @@ describe 'keystone' do
         'hasrestart' => 'true'
       ) }
 
-      it 'should compile the template based on the class parameters' do
-        content = param_value(subject, 'file', 'keystone.conf', 'content')
-        expected_lines = [
+      it 'should correctly configure catalog based on catalog_type'
+
+      it 'should create the expected DEFAULT configuration' do
+#require 'ruby-debug';debugger
+        verify_contents(
+          subject,
+          '/var/lib/puppet/concat/_etc_keystone_keystone.conf/fragments/00_kestone-DEFAULT',
+          [
+            "bind_host     = #{param_hash['bind_host']}",
+            "public_port   = #{param_hash['public_port']}",
+            "admin_port    = #{param_hash['admin_port']}",
+            "admin_token   = #{param_hash['admin_token']}",
+            "compute_port  = #{param_hash['compute_port']}",
             "verbose       = #{param_hash['log_verbose']}",
             "debug         = #{param_hash['log_debug']}",
-          "default_store = #{param_hash['default_store']}",
-          "service_host = #{param_hash['bind_host']}",
-          "service_port = #{param_hash['bind_port']}",
-          "admin_host = #{param_hash['admin_bind_host']}",
-          "admin_port = #{param_hash['admin_bind_port']}"
+            "log_file      = /var/log/keystone/keystone.log",
+            "use_syslog    = #{param_hash['use_syslog']}"
           ]
-        (content.split("\n") & expected_lines).should == expected_lines
+        )
       end
+      it 'should create the expected identity section' do
+        verify_contents(
+          subject,
+          '/var/lib/puppet/concat/_etc_keystone_keystone.conf/fragments/03_kestone-identity',
+          [
+            "[identity]",
+            "driver = keystone.identity.backends.sql.Identity"
+          ]
+        )
+      end
+      it { should create_file(
+        '/var/lib/puppet/concat/_etc_keystone_keystone.conf/fragments/99_kestone-footer') }
     end
   end
 end

deployment/puppet/keystone/spec/spec_helper.rb

@@ -6,6 +6,11 @@ def param_value(subject, type, title, param)
   subject.resource(type, title).send(:parameters)[param.to_sym]
 end
 
+def verify_contents(subject, title, expected_lines)
+  content = subject.resource('file', title).send(:parameters)[:content]
+  (content.split("\n") & expected_lines).should == expected_lines
+end
+
 RSpec.configure do |c|
   c.manifest_dir = File.expand_path(File.join(File.dirname(__FILE__), 'fixtures/manifests'))
   c.module_path = File.join(File.dirname(__FILE__), '../../')

deployment/puppet/keystone/templates/keystone.conf.erb

@@ -1,81 +0,0 @@
-[DEFAULT]
-# Show more verbose log output (sets INFO log level output)
-verbose = <%= log_verbose %>
-
-# Show debugging output in logs (sets DEBUG log level output)
-debug = <%= log_debug %>
-
-# Which backend store should Keystone use by default.
-# Default: 'sqlite'
-# Available choices are 'sqlite' [future will include LDAP, PAM, etc]
-default_store = <%= default_store %>
-
-# Log to this file. Make sure you do not set the same log
-# file for both the API and registry servers!
-log_file = /var/log/keystone/keystone.log
-
-# List of backends to be configured
-backends = keystone.backends.sqlalchemy
-#For LDAP support, add: ,keystone.backends.ldap
-
-# Dictionary Maps every service to a header.Missing services would get header
-# X_(SERVICE_NAME) Key => Service Name, Value => Header Name
-service-header-mappings = {
-	'nova' : 'X-Server-Management-Url',
-	'swift' : 'X-Storage-Url',
-	'cdn' : 'X-CDN-Management-Url'}
-
-# Address to bind the API server
-# TODO Properties defined within app not available via pipeline.
-service_host = <%= bind_host %>
-
-# Port the bind the API server to
-service_port = <%= bind_port %>
-
-# Address to bind the Admin API server
-admin_host = <%= admin_bind_host %>
-
-# Port the bind the Admin API server to
-admin_port = <%= admin_bind_port %>
-
-#Role that allows to perform admin operations.
-keystone-admin-role = Admin
-
-#Role that allows to perform service admin operations.
-keystone-service-admin-role = KeystoneServiceAdmin
-
-[keystone.backends.sqlalchemy]
-# SQLAlchemy connection string for the reference implementation registry
-# server. Any valid SQLAlchemy connection string is fine.
-# See: http://bit.ly/ideIpI
-sql_connection = sqlite:////var/lib/keystone/keystone.db
-backend_entities = ['UserRoleAssociation', 'Endpoints', 'Role', 'Tenant',
-                    'User', 'Credentials', 'EndpointTemplates', 'Token',
-                    'Service']
-
-# Period in seconds after which SQLAlchemy should reestablish its connection
-# to the database.
-sql_idle_timeout = 30
-
-[pipeline:admin]
-pipeline =
-	urlrewritefilter
-	admin_api
-
-[pipeline:keystone-legacy-auth]
-pipeline =
-	urlrewritefilter
-    legacy_auth
-    service_api
-
-[app:service_api]
-paste.app_factory = keystone.server:service_app_factory
-
-[app:admin_api]
-paste.app_factory = keystone.server:admin_app_factory
-
-[filter:urlrewritefilter]
-paste.filter_factory = keystone.middleware.url:filter_factory
-
-[filter:legacy_auth]
-paste.filter_factory = keystone.frontends.legacy_token_auth:filter_factory

deployment/puppet/keystone/templates/mysql.erb

@@ -1,4 +1,5 @@
-[<%= name %>]
+
+[sql]
 connection    = mysql://<%= "#{config['user']}:#{config['password']}@#{config['host']}/#{config['dbname']}" %>
 idle_timeout  = <%= config['idle_timeout'] %>
 min_pool_size = <%= config['min_pool_size'] %>