Implement multiple store configuration

This patch moves the default_store config option to the glance::api
class, and makes it possible to configure more than one store while
supplying a value for the default store to be used. If only one store
is given for glance_store/stores, the default store is automatically
set to be the same value. If multiple stores are given and no default
store is explicitly set, the config will fail and ask the user to
provide a default store.

Change-Id: I28a79ae36e673a3537ea16910d338666b65c80f7
Closes-bug: #1481460
Co-Authored-By: Alex Schultz <aschultz@mirantis.com>

manifests/api.pp

@@ -185,12 +185,21 @@
 #   (optional) CA certificate file to use to verify connecting clients
 #   Defaults to false, not set
 #
-# [*known_stores*]
+# [*stores*]
-#   (optional)List of which store classes and store class locations are
+#   (optional) List of which store classes and store class locations are
 #    currently known to glance at startup.
 #    Defaults to false.
 #    Example: ['glance.store.filesystem.Store','glance.store.http.Store']
 #
+# [*default_store*]
+#   (optional) The default backend store, should be given as a string. Value
+#   must be provided if more than one store is listed in 'stores'.
+#   Defaults to undef
+#
+# [*multi_store*]
+#   (optional) Boolean describing if multiple backends will be configured
+#   Defaults to false
+#
 # [*image_cache_dir*]
 #   (optional) Base directory that the Image Cache uses.
 #    Defaults to '/var/lib/glance/image-cache'.
@@ -234,6 +243,15 @@
 #       try_sleep: 10
 #   Defaults to {}
 #
+#  === deprecated parameters:
+#
+# [*known_stores*]
+#   (optional) DEPRECATED List of which store classes and store class
+#   locations are currently known to glance at startup. This parameter
+#   should be removed in the N release.
+#   Defaults to false.
+#   Example: ['glance.store.filesystem.Store','glance.store.http.Store']
+#
 class glance::api(
   $keystone_password,
   $package_ensure           = 'present',
@@ -270,7 +288,9 @@ class glance::api(
   $cert_file                = false,
   $key_file                 = false,
   $ca_file                  = false,
-  $known_stores             = false,
+  $stores                   = false,
+  $default_store            = undef,
+  $multi_store              = false,
   $database_connection      = undef,
   $database_idle_timeout    = undef,
   $database_min_pool_size   = undef,
@@ -286,6 +306,8 @@ class glance::api(
   $token_cache_time         = $::os_service_default,
   $validate                 = false,
   $validation_options       = {},
+  # DEPRECATED PARAMETERS
+  $known_stores             = false,
 ) inherits glance {
 
   include ::glance::policy
@@ -337,10 +359,50 @@ class glance::api(
     'glance_store/os_region_name':     value => $os_region_name;
   }
 
-  # known_stores config
+  # stores config
-  if $known_stores {
+  if $stores and $known_stores {
+    fail('known_stores and stores cannot both be assigned values')
+  } elsif $stores {
+    $stores_real = $stores
+  } elsif $known_stores {
+    warning('The known_stores parameter is deprecated, use stores instead')
+    $stores_real = $known_stores
+  }
+  if $default_store {
+    $default_store_real = $default_store
+  }
+  # determine value for glance_store/stores
+  if !empty($stores_real) {
+    if size(any2array($stores_real)) > 1 {
+      $final_stores_real = join($stores_real, ',')
+    } else {
+      $final_stores_real = $stores_real[0]
+    }
+    if !$default_store_real {
+      # set default store based on provided stores when it isn't explicitly set
+      warning("default_store not provided, it will be automatically set to ${stores_real[0]}")
+      $default_store_real = $stores_real[0]
+    }
+  } elsif $default_store_real {
+    # set stores based on default_store if only default_store is provided
+    $final_stores_real = $default_store
+  } else {
+    warning('Glance-api is being provisioned without any stores configured')
+  }
+
+  if $default_store_real and $multi_store {
     glance_api_config {
-      'glance_store/stores':  value => join($known_stores, ',');
+      'glance_store/default_store': value => $default_store_real;
+    }
+  } elsif $multi_store {
+    glance_api_config {
+      'glance_store/default_store': ensure => absent;
+    }
+  }
+
+  if $final_stores_real {
+    glance_api_config {
+      'glance_store/stores': value => $final_stores_real;
     }
   } else {
     glance_api_config {

manifests/backend/cinder.pp

@@ -52,6 +52,10 @@
 #   Should be a valid boolean value
 #   Defaults to false
 #
+# [*multi_store*]
+#   (optional) Boolean describing if multiple backends will be configured
+#   Defaults to false
+#
 
 class glance::backend::cinder(
   $os_region_name              = undef,
@@ -59,7 +63,8 @@ class glance::backend::cinder(
   $cinder_api_insecure         = false,
   $cinder_catalog_info         = 'volume:cinder:publicURL',
   $cinder_endpoint_template    = undef,
-  $cinder_http_retries         = '3'
+  $cinder_http_retries         = '3',
+  $multi_store                 = false,
 
 ) {
 
@@ -71,7 +76,10 @@ class glance::backend::cinder(
     'glance_store/cinder_api_insecure': value => $cinder_api_insecure;
     'glance_store/cinder_catalog_info': value => $cinder_catalog_info;
     'glance_store/cinder_http_retries': value => $cinder_http_retries;
-    'glance_store/default_store':       value => 'cinder';
+  }
+
+  if !$multi_store {
+    glance_api_config { 'glance_store/default_store': value => 'cinder'; }
   }
 
   glance_cache_config {

manifests/backend/file.pp

@@ -9,15 +9,24 @@
 #    default_store == file.
 #    Optional. Default: /var/lib/glance/images/
 #
+# [*multi_store*]
+#   (optional) Boolean describing if multiple backends will be configured
+#   Defaults to false
+#
+
 class glance::backend::file(
-  $filesystem_store_datadir = '/var/lib/glance/images/'
+  $filesystem_store_datadir = '/var/lib/glance/images/',
+  $multi_store              = false,
 ) {
 
   glance_api_config {
-    'glance_store/default_store':            value => 'file';
     'glance_store/filesystem_store_datadir': value => $filesystem_store_datadir;
   }
 
+  if !$multi_store {
+    glance_api_config { 'glance_store/default_store': value => 'file'; }
+  }
+
   glance_cache_config {
     'glance_store/filesystem_store_datadir': value => $filesystem_store_datadir;
   }

manifests/backend/rbd.pp

@@ -31,6 +31,10 @@
 #      to ceph cluster. If value <= 0, no timeout is set and
 #      default librados value is used.
 #
+# [*multi_store*]
+#   (optional) Boolean describing if multiple backends will be configured
+#   Defaults to false
+#
 
 class glance::backend::rbd(
   $rbd_store_user         = undef,
@@ -40,6 +44,7 @@ class glance::backend::rbd(
   $show_image_direct_url  = undef,
   $package_ensure         = 'present',
   $rados_connect_timeout  = '0',
+  $multi_store            = false,
 ) {
   include ::glance::params
 
@@ -48,7 +53,6 @@ class glance::backend::rbd(
   }
 
   glance_api_config {
-    'glance_store/default_store':          value => 'rbd';
     'glance_store/rbd_store_ceph_conf':    value => $rbd_store_ceph_conf;
     'glance_store/rbd_store_user':         value => $rbd_store_user;
     'glance_store/rbd_store_pool':         value => $rbd_store_pool;
@@ -56,6 +60,10 @@ class glance::backend::rbd(
     'glance_store/rados_connect_timeout':  value => $rados_connect_timeout;
   }
 
+  if !$multi_store {
+    glance_api_config { 'glance_store/default_store': value => 'rbd'; }
+  }
+
   package { 'python-ceph':
     ensure => $package_ensure,
     name   => $::glance::params::pyceph_package_name,

manifests/backend/s3.pp

@@ -46,9 +46,15 @@
 #    (Optional) The number of thread pools to perform a multipart upload in S3.
 #    Default: 10
 #
+# [*multi_store*]
+#   (optional) Boolean describing if multiple backends will be configured
+#   Defaults to false
+#
+#  === deprecated parameters:
+#
 #  [*default_store*]
-#    (Optional) Whether to set S3 as the default backend store.
+#   (Optional) DEPRECATED Whether to set S3 as the default backend store.
-#    Default: true
+#   Default: undef
 #
 class glance::backend::s3(
   $access_key,
@@ -61,7 +67,9 @@ class glance::backend::s3(
   $large_object_chunk_size  = 10,
   $object_buffer_dir        = undef,
   $thread_pools             = 10,
-  $default_store            = true
+  $multi_store              = false,
+  # deprecated parameters
+  $default_store            = undef,
 ) {
 
   if !is_integer($large_object_chunk_size) or $large_object_chunk_size < 5 {
@@ -72,6 +80,10 @@ class glance::backend::s3(
     fail('glance::backend::s3::bucket_url_format must be either "subdomain" or "path"')
   }
 
+  if $default_store {
+    warning('The default_store parameter is deprecated in glance::backend::s3, you should declare it in glance::api')
+  }
+
   glance_api_config {
     'glance_store/s3_store_access_key':              value => $access_key;
     'glance_store/s3_store_secret_key':              value => $secret_key;
@@ -84,14 +96,14 @@ class glance::backend::s3(
     'glance_store/s3_store_thread_pools':            value => $thread_pools;
   }
 
+  if !$multi_store {
+    glance_api_config { 'glance_store/default_store': value => 's3'; }
+  }
+
   if $object_buffer_dir {
     glance_api_config { 'glance_store/s3_store_object_buffer_dir':  value => $object_buffer_dir; }
   } else {
     glance_api_config { 'glance_store/s3_store_object_buffer_dir':  ensure => absent; }
   }
 
-  if $default_store {
-    glance_api_config { 'glance_store/default_store':   value => 's3'; }
-  }
-
 }

manifests/backend/swift.pp

@@ -38,21 +38,26 @@
 #    new images. String value.
 #    Default to 'ref1'.
 #
+# [*multi_store*]
+#   (optional) Boolean describing if multiple backends will be configured
+#   Defaults to false
+#
+
 class glance::backend::swift(
   $swift_store_user,
   $swift_store_key,
-  $swift_store_auth_address = 'http://127.0.0.1:5000/v2.0/',
+  $swift_store_auth_address            = 'http://127.0.0.1:5000/v2.0/',
-  $swift_store_container = 'glance',
+  $swift_store_container               = 'glance',
-  $swift_store_auth_version = '2',
+  $swift_store_auth_version            = '2',
-  $swift_store_large_object_size = '5120',
+  $swift_store_large_object_size       = '5120',
   $swift_store_create_container_on_put = false,
-  $swift_store_endpoint_type = 'internalURL',
+  $swift_store_endpoint_type           = 'internalURL',
-  $swift_store_region = undef,
+  $swift_store_region                  = undef,
-  $default_swift_reference = 'ref1',
+  $default_swift_reference             = 'ref1',
+  $multi_store                         = false,
 ) {
 
   glance_api_config {
-    'glance_store/default_store':              value => 'swift';
     'glance_store/swift_store_region':         value => $swift_store_region;
     'glance_store/swift_store_container':      value => $swift_store_container;
     'glance_store/swift_store_create_container_on_put':
@@ -66,6 +71,10 @@ class glance::backend::swift(
     'glance_store/default_swift_reference':    value => $default_swift_reference;
   }
 
+  if !$multi_store {
+    glance_api_config { 'glance_store/default_store': value => 'swift'; }
+  }
+
   glance_swift_config {
     "${default_swift_reference}/user":         value => $swift_store_user;
     "${default_swift_reference}/key":          value => $swift_store_key;

manifests/backend/vsphere.pp

@@ -57,6 +57,11 @@
 #   connection related issues.
 #    Defaults to '10'
 #
+# [*multi_store*]
+#   (optional) Boolean describing if multiple backends will be configured
+#   Defaults to false
+#
+
 class glance::backend::vsphere(
   $vcenter_host,
   $vcenter_user,
@@ -64,12 +69,13 @@ class glance::backend::vsphere(
   $vcenter_datacenter,
   $vcenter_datastore,
   $vcenter_image_dir,
-  $vcenter_api_insecure = 'False',
+  $vcenter_api_insecure       = 'False',
   $vcenter_task_poll_interval = '5',
-  $vcenter_api_retry_count = '10',
+  $vcenter_api_retry_count    = '10',
+  $multi_store                = false,
 ) {
+
   glance_api_config {
-    'glance_store/default_store': value             => 'vsphere';
     'glance_store/vmware_api_insecure': value       => $vcenter_api_insecure;
     'glance_store/vmware_server_host': value        => $vcenter_host;
     'glance_store/vmware_server_username': value    => $vcenter_user;
@@ -80,4 +86,8 @@ class glance::backend::vsphere(
     'glance_store/vmware_api_retry_count': value    => $vcenter_api_retry_count;
     'glance_store/vmware_datacenter_path': value    => $vcenter_datacenter;
   }
+
+  if !$multi_store {
+    glance_api_config {  'glance_store/default_store': value => 'vsphere'; }
+  }
 }

spec/classes/glance_api_spec.rb

@@ -32,6 +32,7 @@ describe 'glance::api' do
       :known_stores             => false,
       :delayed_delete           => '<SERVICE DEFAULT>',
       :scrub_time               => '<SERVICE DEFAULT>',
+      :default_store            => false,
       :image_cache_dir          => '/var/lib/glance/image-cache',
       :image_cache_stall_time   => '<SERVICE DEFAULT>',
       :image_cache_max_size     => '<SERVICE DEFAULT>',
@@ -244,7 +245,7 @@ describe 'glance::api' do
         it { is_expected.to contain_glance_api_config('DEFAULT/key_file').with_value('/tmp/key_file') }
       end
     end
-    describe 'with known_stores by default' do
+    describe 'with stores by default' do
       let :params do
         default_params
       end
@@ -252,14 +253,64 @@ describe 'glance::api' do
       it { is_expected.to_not contain_glance_api_config('glance_store/stores').with_value('false') }
     end
 
-    describe 'with known_stores override' do
+    describe 'with stores override' do
       let :params do
         default_params.merge({
-          :known_stores   => ['glance.store.filesystem.Store','glance.store.http.Store'],
+          :default_store => 'glance.store.filesystem.Store',
+          :stores        => ['glance.store.filesystem.Store','glance.store.http.Store'],
+          :multi_store   => true,
         })
       end
 
-      it { is_expected.to contain_glance_api_config('glance_store/stores').with_value("glance.store.filesystem.Store,glance.store.http.Store") }
+      it { is_expected.to contain_glance_api_config('glance_store/default_store').with_value('glance.store.filesystem.Store') }
+      it { is_expected.to contain_glance_api_config('glance_store/stores').with_value('glance.store.filesystem.Store,glance.store.http.Store') }
+    end
+
+    describe 'with single store override and no default store' do
+      let :params do
+        default_params.merge({
+          :stores      => ['glance.store.filesystem.Store'],
+          :multi_store => true,
+        })
+      end
+
+      it { is_expected.to contain_glance_api_config('glance_store/default_store').with_value('glance.store.filesystem.Store') }
+      it { is_expected.to contain_glance_api_config('glance_store/stores').with_value('glance.store.filesystem.Store') }
+    end
+
+    describe 'with multiple stores override and no default store' do
+      let :params do
+        default_params.merge({
+          :stores      => ['glance.store.filesystem.Store', 'glance.store.http.Store'],
+          :multi_store => true,
+        })
+      end
+
+      it { is_expected.to contain_glance_api_config('glance_store/default_store').with_value('glance.store.filesystem.Store') }
+      it { is_expected.to contain_glance_api_config('glance_store/stores').with_value('glance.store.filesystem.Store,glance.store.http.Store') }
+    end
+
+    describe 'with both stores and known_stores provided' do
+      let :params do
+        default_params.merge({
+          :stores       => ['glance.store.filesystem.Store'],
+          :known_stores => ['glance.store.http.store'],
+        })
+      end
+
+      it { is_expected.to raise_error(Puppet::Error, /known_stores and stores cannot both be assigned values/) }
+    end
+
+    describe 'with known_stores not set but with default_store' do
+      let :params do
+        default_params.merge({
+          :default_store => 'glance.store.filesystem.Store',
+          :multi_store   => true,
+        })
+      end
+
+      it { is_expected.to contain_glance_api_config('glance_store/default_store').with_value('glance.store.filesystem.Store') }
+      it { is_expected.to contain_glance_api_config('glance_store/stores').with_value('glance.store.filesystem.Store') }
     end
 
     describe 'while validating the service with default command' do

spec/classes/glance_backend_cinder_spec.rb

@@ -31,6 +31,7 @@ describe 'glance::backend::cinder' do
     context 'when default parameters' do
 
       it 'configures glance-api.conf' do
+        is_expected.to contain_glance_api_config('glance_store/default_store').with_value('cinder')
         is_expected.to contain_glance_api_config('glance_store/default_store').with_value('cinder')
         is_expected.to contain_glance_api_config('glance_store/cinder_api_insecure').with_value(false)
         is_expected.to contain_glance_api_config('glance_store/cinder_catalog_info').with_value('volume:cinder:publicURL')
@@ -58,7 +59,6 @@ describe 'glance::backend::cinder' do
         }
       end
       it 'configures glance-api.conf' do
-        is_expected.to contain_glance_api_config('glance_store/default_store').with_value('cinder')
         is_expected.to contain_glance_api_config('glance_store/cinder_api_insecure').with_value(true)
         is_expected.to contain_glance_api_config('glance_store/cinder_ca_certificates_file').with_value('/etc/ssh/ca.crt')
         is_expected.to contain_glance_api_config('glance_store/cinder_catalog_info').with_value('volume:cinder:internalURL')

spec/classes/glance_backend_s3_spec.rb

@@ -42,7 +42,6 @@ describe 'glance::backend::s3' do
           :large_object_chunk_size  => 20,
           :object_buffer_dir        => '/tmp',
           :thread_pools             => 20,
-          :default_store            => false
         }
       end
 
@@ -57,7 +56,6 @@ describe 'glance::backend::s3' do
         is_expected.to contain_glance_api_config('glance_store/s3_store_large_object_chunk_size').with_value('20')
         is_expected.to contain_glance_api_config('glance_store/s3_store_object_buffer_dir').with_value('/tmp')
         is_expected.to contain_glance_api_config('glance_store/s3_store_thread_pools').with_value('20')
-        is_expected.to_not contain_glance_api_config('glance_store/default_store')
       end
 
     end