openstack/puppet-swift
Code Issues Proposed changes

Implement proxy pipline into fragments

Browse Source 
The swift pipeline is commonly used to plug
capabilities in.

This pipeline should be implemented in a way that
is easier to extend as people desire additional
pipelines.

This commit implements classes that can be used
to configure the following pipelines:
  - proxy::cache
  - proxy::healthcheck
  - proxy::keystone
  - proxy::swauth

It also reimplements swift::proxy to utilize those
pipelines for configuration.
This commit is contained in:
Dan Bode
2012-04-22 12:11:27 -07:00
parent 84d59985fc
commit 24a01d1bea
17 changed files with 403 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
manifests/proxy.pp
View File

@@ -12,29 +12,21 @@
# Required.
# [*port*] The port to which the proxy server will bind.
# Optional. Defaults to 8080.
# [*pipeline*] The list of elements of the swift proxy pipeline.
# Currently supports healthcheck, cache, proxy-server, and
# one of the following auth_types: tempauth, swauth, keystone.
# Each of the specified elements also need to be declared externally
# as a puppet class with the exception of proxy-server.
# Optional. Defaults to ['healthcheck', 'cache', 'tempauth', 'proxy-server']
# [*workers*] Number of threads to process requests.
# Optional. Defaults to the number of processors.
# [*auth_type*] - Type of authorization to use.
# valid values are tempauth, swauth, and keystone.
# Optional. Defaults to tempauth.
# [*allow_account_management*]
# Rather or not requests through this proxy can create and
# delete accounts. Optional. Defaults to true.
# [*account_autocreate*] Rather accounts should automatically be created.
# Has to be set to true for tempauth. Optional. Defaults to true.
# [*proxy_port*] Port that the swift proxy service will bind to.
# Optional. Defaults to 11211
# [*package_ensure*] Ensure state of the swift proxy package.
# Optional. Defaults to present.
# [*cache_servers*] A list of the memcache servers to be used. Entries
# should be in the form host:port.
# == sw auth specific configuration
# [*swauth_endpoint*]
# [*swauth_super_admin_user*]
#
# == Dependencies
#
# Class['memcached']
#
# == Examples
#
@@ -49,58 +41,75 @@
class swift::proxy(
$proxy_local_net_ip,
$port = '8080',
$pipeline = ['healthcheck', 'cache', 'tempauth', 'proxy-server'],
$workers = $::processorcount,
$cache_servers = ['127.0.0.1:11211'],
$allow_account_management = true,
$auth_type = 'tempauth',
$account_autocreate = true,
$swauth_endpoint = '127.0.0.1',
$swauth_super_admin_key = 'swauthkey',
$package_ensure = 'present'
) inherits swift {
) {
include 'swift::params'
include 'concat::setup'
validate_bool($account_autocreate)
validate_bool($allow_account_management)
validate_re($auth_type, 'tempauth|swauth|keystone')
validate_array($pipeline)
if(member($pipeline, 'tempauth')) {
$auth_type = 'tempauth'
} elsif(member($pipeline, 'swauth')) {
$auth_type = 'swauth'
} elsif(member($pipeline, 'keystone')) {
$auth_type = 'keystone'
} else {
warning('no auth type provided in the pipeline')
}
if(! member($pipeline, 'proxy-server')) {
warning("swift storage server ${type} must specify ${type}-server")
}
if($auth_type == 'tempauth' and ! $account_autocreate ){
fail("\$account_autocreate must be set to true when auth type is tempauth")
}
if $cache_server_ips =~ /^127\.0\.0\.1/ {
Class['memcached'] -> Class['swift::proxy']
}
if(auth_type == 'keystone') {
fail('Keystone is currently not supported, it should be supported soon :)')
}
package { 'swift-proxy':
name => $::swift::params::proxy_package_name,
ensure => $package_ensure,
}
if($auth_type == 'swauth') {
package { 'python-swauth':
ensure => $package_ensure,
before => Package['swift-proxy'],
}
}
file { "/etc/swift/proxy-server.conf":
ensure => present,
concat { '/etc/swift/proxy-server.conf':
owner => 'swift',
group => 'swift',
mode => 0660,
content => template('swift/proxy-server.conf.erb'),
mode => '0660',
require => Package['swift-proxy'],
}
$required_classes = split(
inline_template(
"<%=
(pipeline - ['proxy-server']).collect do |x|
'swift::proxy::' + x
end.join(',')
%>"), ',')
# you can now add your custom fragments at the user level
concat::fragment { 'swift_proxy':
target => "/etc/swift/proxy-server.conf",
content => template('swift/proxy-server.conf.erb'),
order => '00',
# require classes for each of the elements of the pipeline
# this is to ensure the user gets reasonable elements if he
# does not specify the backends for every specified element of
# the pipeline
before => Class[$required_classes],
}
service { 'swift-proxy':
name => $::swift::params::proxy_service_name,
ensure => running,
provider => $::swift::params::service_provider,
enable => true,
subscribe => File['/etc/swift/proxy-server.conf'],
provider => $::swift::params::service_provider,
subscribe => Concat['/etc/swift/proxy-server.conf'],
}
}

36
manifests/proxy/cache.pp Normal file
View File

@@ -0,0 +1,36 @@
#
# Configures the swift proxy memcache server
#
# [*memcache_servers*] A list of the memcache servers to be used. Entries
# should be in the form host:port.
#
# == Dependencies
#
# Class['memcached']
#
# == Examples
#
# == Authors
#
# Dan Bode dan@puppetlabs.com
#
# == Copyright
#
# Copyright 2011 Puppetlabs Inc, unless otherwise noted.
#
class swift::proxy::cache(
$memcache_servers = ['127.0.0.1:11211'],
) {
# require the memcached class if its on the same machine
if $memcache_servers =~ /^127\.0\.0\.1/ {
Class['memcached'] -> Class['swift::proxy::cache']
}
concat::fragment { 'swift_cache':
target => '/etc/swift/proxy-server.conf',
content => template('swift/proxy/cache.conf.erb'),
order => '23',
}
}

24
manifests/proxy/healthcheck.pp Normal file
View File

@@ -0,0 +1,24 @@
#
# Configure swift healthcheck.
#
# == Dependencies
#
# == Examples
#
# == Authors
#
# Dan Bode dan@puppetlabs.com
#
# == Copyright
#
# Copyright 2011 Puppetlabs Inc, unless otherwise noted.
#
class swift::proxy::healthcheck() {
concat::fragment { 'swift_healthcheck':
target => '/etc/swift/proxy-server.conf',
content => template('swift/proxy/healthcheck.conf.erb'),
order => '25',
}
}

32
manifests/proxy/keystone.pp Normal file
View File

@@ -0,0 +1,32 @@
class swift::proxy::keystone(
$admin_token = undef,
$admin_user = undef,
$admin_tenant_name = undef,
$admin_password = undef,
$delay_auth_decision = undef,
$auth_host = undef,
$auth_port = undef,
$auth_protocol = undef,
$operator_roles = ['admin', 'SwiftOperator'],
$is_admin = true,
$cache = 'swift.cache'
) {
concat::fragment { 'swift_keystone':
target => '/etc/swift/proxy-server.conf',
content => template('swift/proxy/keystone.conf.erb'),
order => '79',
}
keystone::client::authtoken { '/etc/swift/proxy-server.conf':
admin_token => $admin_token,
admin_user => $admin_user,
admin_tenant_name => $admin_tenant_name,
admin_password => $admin_password,
delay_auth_decision => $delay_auth_decision,
auth_host => $auth_host,
auth_port => $auth_port,
auth_protocol => $auth_protocol
}
}

19
manifests/proxy/swauth.pp Normal file
View File

@@ -0,0 +1,19 @@
# [*swauth_endpoint*]
# [*swauth_super_admin_user*]
class swift::proxy::swauth(
$swauth_endpoint = '127.0.0.1',
$swauth_super_admin_key = 'swauthkey',
) {
package { 'python-swauth':
ensure => $package_ensure,
before => Package['swift-proxy'],
}
concat::fragment { 'swift_proxy_swauth':
target => '/etc/swift/proxy-server.conf',
content => template('swift/proxy/swauth.conf.erb'),
order => '20',
}
}

9
manifests/proxy/tempauth.pp Normal file
View File

@@ -0,0 +1,9 @@
class swift::proxy::tempauth() {
concat::fragment { 'swift-proxy-swauth':
target => '/etc/swift/proxy-server.conf',
content => template('swift/proxy/tempauth.conf.erb'),
order => '01',
}
}

52
spec/classes/swift_proxy_cache_spec.rb Normal file
View File

@@ -0,0 +1,52 @@
require 'spec_helper'
describe 'swift::proxy::cache' do
let :facts do
{
:operatingsystem => 'Ubuntu',
:osfamily => 'Debian',
:processorcount => 1
}
end
let :pre_condition do
'class { "concat::setup": }
concat { "/etc/swift/proxy-server.conf": }
class { "memcached": max_memory => 1 }'
end
let :fragment_file do
"/var/lib/puppet/concat/_etc_swift_proxy-server.conf/fragments/23_swift_cache"
end
it { should contain_file(fragment_file).with_content(/[filter:cache]/) }
it { should contain_file(fragment_file).with_content(/use = egg:swift#memcache/) }
describe 'with defaults' do
it { should contain_file(fragment_file).with_content(/memcache_servers = 127\.0\.0\.1:11211/) }
end
describe 'with overridden memcache server' do
let :params do
{:memcache_servers => '10.0.0.1:1'}
end
it { should contain_file(fragment_file).with_content(/memcache_servers = 10\.0\.0\.1:1/) }
end
describe 'with overridden memcache server array' do
let :params do
{:memcache_servers => ['10.0.0.1:1', '10.0.0.2:2']}
end
it { should contain_file(fragment_file).with_content(/memcache_servers = 10\.0\.0\.1:1,10\.0\.0\.2:2/) }
end
end

17
spec/classes/swift_proxy_healthcheck_spec.rb Normal file
View File

@@ -0,0 +1,17 @@
require 'spec_helper'
describe 'swift::proxy::healthcheck' do
let :pre_condition do
'class { "concat::setup": }
concat { "/etc/swift/proxy-server.conf": }'
end
let :fragment_file do
"/var/lib/puppet/concat/_etc_swift_proxy-server.conf/fragments/25_swift_healthcheck"
end
it { should contain_file(fragment_file).with_content(/[filter:healthcheck]/) }
it { should contain_file(fragment_file).with_content(/use = egg:swift#healthcheck/) }
end

47
spec/classes/swift_proxy_keystone_spec.rb Normal file
View File

@@ -0,0 +1,47 @@
require 'spec_helper'
describe 'swift::proxy::keystone' do
let :fragment_file do
'/var/lib/puppet/concat/_etc_swift_proxy-server.conf/fragments/79_swift_keystone'
end
let :pre_condition do
'
include concat::setup
concat { "/etc/swift/proxy-server.conf": }
'
end
it { should contain_file(fragment_file).with_content(/[filter:keystone]/) }
it { should contain_file(fragment_file).with_content(/paste.filter_factory = keystone.middleware.swift_auth:filter_factory/) }
describe 'with defaults' do
it { should contain_file(fragment_file).with_content(/operator_roles = admin SwiftOperator/) }
it { should contain_file(fragment_file).with_content(/is_admin = true/) }
it { should contain_file(fragment_file).with_content(/cache = swift.cache/) }
it { should contain_keystone__client__authtoken('/etc/swift/proxy-server.conf') }
end
describe 'with parameter overrides' do
let :params do
{
:operator_roles => 'foo',
:is_admin => 'false',
:cache => 'somecache'
}
it { should contain_file(fragment_file).with_content(/operator_roles = foo/) }
it { should contain_file(fragment_file).with_content(/is_admin = false/) }
it { should contain_file(fragment_file).with_content(/cache = somecache/) }
end
end
end

167
spec/classes/swift_proxy_spec.rb
View File

@@ -18,8 +18,8 @@ describe 'swift::proxy' do
}
end
let :fixture_dir do
File.join(File.dirname(__FILE__), '..', 'fixtures')
let :fragment_path do
"/var/lib/puppet/concat/_etc_swift_proxy-server.conf/fragments/00_swift_proxy"
end
describe 'with proper dependencies' do
@@ -48,7 +48,7 @@ describe 'swift::proxy' do
{:ensure => 'running',
:provider => 'upstart',
:enable => true,
:subscribe => 'File[/etc/swift/proxy-server.conf]'
:subscribe => 'Concat[/etc/swift/proxy-server.conf]'
}
)}
it { should contain_file('/etc/swift/proxy-server.conf').with(
@@ -60,32 +60,29 @@ describe 'swift::proxy' do
}
)}
it 'should contain default config file' do
content = param_value(
subject,
'file', '/etc/swift/proxy-server.conf',
'content'
it 'should build the header file with all of the default contents' do
verify_contents(subject, fragment_path,
[
'[DEFAULT]',
'bind_port = 8080',
"workers = #{facts[:processorcount]}",
'user = swift',
'[pipeline:main]',
'pipeline = healthcheck cache tempauth proxy-server',
'[app:proxy-server]',
'use = egg:swift#proxy',
'allow_account_management = true',
'account_autocreate = true'
]
)
expected_lines =
[
'[DEFAULT]',
'bind_port = 8080',
"workers = #{facts[:processorcount]}",
'user = swift',
'[pipeline:main]',
'pipeline = healthcheck cache tempauth proxy-server',
'[app:proxy-server]',
'use = egg:swift#proxy',
'allow_account_management = true',
'account_autocreate = true',
'[filter:healthcheck]',
'use = egg:swift#healthcheck',
'[filter:cache]',
'use = egg:swift#memcache',
'memcache_servers = 127.0.0.1:11211'
]
(content.split("\n") & expected_lines).should =~ expected_lines
end
it { should contain_concat__fragment('swift_proxy').with_before(
[
'Class[Swift::Proxy::Healthcheck]',
'Class[Swift::Proxy::Cache]',
'Class[Swift::Proxy::Tempauth]'
]
)}
describe 'when more parameters are set' do
let :params do
@@ -93,55 +90,30 @@ describe 'swift::proxy' do
:proxy_local_net_ip => '10.0.0.2',
:port => '80',
:workers => 3,
:cache_servers => ['foo:1', 'bar:2'],
:allow_account_management => true
:pipeline => ['swauth', 'proxy-server'],
:allow_account_management => false,
:account_autocreate => false
}
end
it 'should contain default config file' do
content = param_value(
subject,
'file', '/etc/swift/proxy-server.conf',
'content'
)
expected_lines =
it 'should build the header file with provided values' do
verify_contents(subject, fragment_path,
[
'[DEFAULT]',
'bind_port = 80',
"workers = 3",
'allow_account_management = true',
'memcache_servers = foo:1,bar:2'
'user = swift',
'[pipeline:main]',
'pipeline = swauth proxy-server',
'[app:proxy-server]',
'use = egg:swift#proxy',
'allow_account_management = false',
'account_autocreate = false'
]
(content.split("\n") & expected_lines).should =~ expected_lines
end
end
describe 'when using tempauth' do
it { should_not contain_package('python-swauth') }
it 'should fail when setting account_autocreate to false' do
params[:auth_type] = 'tempauth'
params[:account_autocreate] = false
expect do
subject
end.should raise_error(Puppet::Error, /account_autocreate must be set to true when auth type is tempauth/)
end
it 'should contain tempauth configuration' do
content = param_value(
subject,
'file', '/etc/swift/proxy-server.conf',
'content'
)
expected_lines =
[
'pipeline = healthcheck cache tempauth proxy-server',
'[filter:tempauth]',
'use = egg:swift#tempauth',
'user_admin_admin = admin .admin .reseller_admin',
'user_test_tester = testing .admin',
'user_test2_tester2 = testing2 .admin',
'user_test_tester3 = testing3'
]
(content.split("\n") & expected_lines).should =~ expected_lines
end
it { should contain_concat__fragment('swift_proxy').with_before(
'Class[Swift::Proxy::Swauth]'
)}
end
describe 'when supplying bad values for parameters' do
@@ -156,64 +128,5 @@ describe 'swift::proxy' do
end
end
describe 'when using swauth' do
let :params do
{:proxy_local_net_ip => '127.0.0.1',
:auth_type => 'swauth' }
end
describe 'with defaults' do
it { should contain_package('python-swauth').with(
{:ensure => 'present',
:before => 'Package[swift-proxy]'
}
)}
it 'should create a config file with default swauth config' do
content = param_value(
subject,
'file', '/etc/swift/proxy-server.conf',
'content'
)
expected_lines =
[
'[filter:swauth]',
'use = egg:swauth#swauth',
'default_swift_cluster = local#127.0.0.1',
'super_admin_key = swauthkey'
]
(content.split("\n") & expected_lines).should =~ expected_lines
end
end
describe 'with parameter overrides' do
let :params do
{:proxy_local_net_ip => '127.0.0.1',
:auth_type => 'swauth',
:swauth_endpoint => '10.0.0.1',
:swauth_super_admin_key => 'key'
}
end
it 'should create a config file with default swauth config' do
content = param_value(
subject,
'file', '/etc/swift/proxy-server.conf',
'content'
)
expected_lines =
[
'[filter:swauth]',
'use = egg:swauth#swauth',
'default_swift_cluster = local#10.0.0.1',
'super_admin_key = key'
]
(content.split("\n") & expected_lines).should =~ expected_lines
end
end
end
end
end

47
spec/classes/swift_proxy_swauth_spec.rb Normal file
View File

@@ -0,0 +1,47 @@
require 'spec_helper'
describe 'swift::proxy::swauth' do
let :pre_condition do
'class { "concat::setup": }
concat { "/etc/swift/proxy-server.conf": }'
end
let :fragment_file do
"/var/lib/puppet/concat/_etc_swift_proxy-server.conf/fragments/20_swift_proxy_swauth"
end
it { should contain_package('python-swauth') }
it { should contain_file(fragment_file).with_content(/[filter:swauth]/) }
it { should contain_file(fragment_file).with_content(/use = egg:swauth#swauth/) }
describe 'with defaults' do
it { should contain_file(fragment_file).with_content(/default_swift_cluster = local#127\.0\.0\.1/) }
it { should contain_file(fragment_file).with_content(/super_admin_key = swauthkey/) }
end
describe 'with overridden endpoint' do
let :params do
{:swauth_endpoint => '10.0.0.1'}
end
it { should contain_file(fragment_file).with_content(/default_swift_cluster = local#10\.0\.0\.1/) }
end
describe 'with overridden admin key' do
let :params do
{:swauth_super_admin_key => 'foo'}
end
it { should contain_file(fragment_file).with_content(/super_admin_key = foo/) }
end
end

38
templates/proxy-server.conf.erb
View File

@@ -6,46 +6,10 @@ workers = <%= workers %>
user = swift
[pipeline:main]
# ratelimit?
pipeline = healthcheck cache <%= auth_type %> proxy-server
pipeline = <%= pipeline.to_a.join(' ') %>
[app:proxy-server]
use = egg:swift#proxy
allow_account_management = <%= allow_account_management %>
account_autocreate = <%= account_autocreate %>
<% if auth_type == 'swauth' -%>
[filter:swauth]
use = egg:swauth#swauth
# this line is not in the install docs?
default_swift_cluster = local#<%= swauth_endpoint %>
super_admin_key = <%= swauth_super_admin_key %>
<% elsif auth_type == 'tempauth' -%>
[filter:tempauth]
use = egg:swift#tempauth
user_admin_admin = admin .admin .reseller_admin
user_test_tester = testing .admin
user_test2_tester2 = testing2 .admin
user_test_tester3 = testing3
<% elsif auth_type == 'keystone' -%>
[filter:keystone]
use = egg:keystone#swiftauth
auth_protocol = https
auth_host = 127.0.0.0
auth_port = 35357
admin_token = 999888777666
delay_auth_decision = 0
service_protocol = https
service_host = 127.0.0.0
service_port = 5000
service_pass = dTpw
cache = swift.cache
<% end -%>
[filter:healthcheck]
use = egg:swift#healthcheck
[filter:cache]
use = egg:swift#memcache
# multi-proxy config not supported
memcache_servers = <%= cache_servers.to_a.join(',') %>

3
templates/proxy/cache.conf.erb Normal file
View File

@@ -0,0 +1,3 @@
[filter:cache]
use = egg:swift#memcache
memcache_servers = <%= memcache_servers.to_a.join(',') %>

4
templates/proxy/healthcheck.conf.erb Normal file
View File

@@ -0,0 +1,4 @@
[filter:healthcheck]
use = egg:swift#healthcheck

7
templates/proxy/keystone.conf.erb Normal file
View File

@@ -0,0 +1,7 @@
[filter:keystone]
paste.filter_factory = keystone.middleware.swift_auth:filter_factory
operator_roles = <%= operator_roles.to_a.join(' ') %>
is_admin = <%= is_admin %>
cache = <%= cache %>

7
templates/proxy/swauth.conf.erb Normal file
View File

@@ -0,0 +1,7 @@
[filter:swauth]
use = egg:swauth#swauth
# this line is not in the install docs?
default_swift_cluster = local#<%= swauth_endpoint %>
super_admin_key = <%= swauth_super_admin_key %>

8
templates/proxy/tempauth.conf.erb Normal file
View File

@@ -0,0 +1,8 @@
[filter:tempauth]
use = egg:swift#tempauth
user_admin_admin = admin .admin .reseller_admin
user_test_tester = testing .admin
user_test2_tester2 = testing2 .admin
user_test_tester3 = testing3