openstack/puppet-glance
Code Issues Proposed changes

Re-add support for S3 backend

Browse Source 
Support for S3 backend was removed from glance_store during Newton
release[1] but it was restored during Ussuri release[2].

This change re-introduces support for S3 backend.

[1] 4432e60af2a2bdbfc456d8d99d93d1157fc86dcd
[2] 261bad3e433860742ecdaf93a9ff5609dcbfa432

Change-Id: I82cd7f006f9c85513ee8781e6e1a526816188844
This commit is contained in:
Takashi Kajinami 2021-12-21 21:45:58 +09:00
parent e832ed4069
commit 40315729bc
5 changed files with 277 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

124
manifests/backend/multistore/s3.pp Normal file
View File

@ -0,0 +1,124 @@
#
# Copyright 2021 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# == Define: glance::backend::multistore::s3
#
# Used to configure s3 backends for glance
#
# === Parameters:
#
# [*s3_store_host*]
# (required) The host where the S3 server is listening.
# Defaults to $::os_service_default.
#
# [*s3_store_access_key*]
# (required) The S3 query token access key.
# Defaults to $::os_service_default.
#
# [*s3_store_secret_key*]
# (required) The S3 query token secret key.
# Defaults to $::os_service_default.
#
# [*s3_store_bucket*]
# (required) The S3 bucket to be used to store the Glance data.
# Defaults to $::os_service_default.
#
# [*s3_store_create_bucket_on_put*]
# (optional) Determine whether S3 should create a new bucket.
# Defaults to $::os_service_default.
#
# [*s3_store_bucket_url_format*]
# (optional) The S3 calling format used to determine the object.
# Defaults to $::os_service_default.
#
# [*s3_store_large_object_size*]
# (optional) What size, in MB, should S3 start chunking image files and do
# a multipart upload in S3.
# Defaults to $::os_service_default.
#
# [*s3_store_large_object_chunk_size*]
# (optional) What multipart upload part size, in MB, should S3 use when
# uploading parts.
# Defaults to $::os_service_default.
#
# [*s3_store_thread_pools*]
# (optional) The number of thead pools to perform a multipart upload in S3.
# Defaults to $::os_service_default.
#
# [*store_description*]
# (optional) Provides constructive information about the store backend to
# end users.
# Defaults to $::os_service_default.
#
# [*manage_packages*]
# Optional. Whether we should manage the packages.
# Defaults to true,
#
# [*package_ensure*]
# Optional. Desired ensure state of packages.
# accepts latest or specific versions.
# Defaults to present.
#
define glance::backend::multistore::s3(
$s3_store_host,
$s3_store_access_key,
$s3_store_secret_key,
$s3_store_bucket,
$s3_store_create_bucket_on_put = $::os_service_default,
$s3_store_bucket_url_format = $::os_service_default,
$s3_store_large_object_size = $::os_service_default,
$s3_store_large_object_chunk_size = $::os_service_default,
$s3_store_thread_pools = $::os_service_default,
$store_description = $::os_service_default,
$manage_packages = true,
$package_ensure = 'present',
) {
include glance::deps
include glance::params
if $manage_packages {
ensure_packages('python-boto3', {
'ensure' => $package_ensure,
'name' => $::glance::params::boto3_package_name,
'tag' => ['openstack','glance-package'],
})
}
glance_api_config {
"${name}/s3_store_host": value => $s3_store_host;
"${name}/s3_store_access_key": value => $s3_store_access_key, secret => true;
"${name}/s3_store_secret_key": value => $s3_store_secret_key, secret => true;
"${name}/s3_store_bucket": value => $s3_store_bucket;
"${name}/s3_store_create_bucket_on_put": value => $s3_store_create_bucket_on_put;
"${name}/s3_store_bucket_url_format": value => $s3_store_bucket_url_format;
"${name}/s3_store_large_object_size": value => $s3_store_large_object_size;
"${name}/s3_store_large_object_chunk_size": value => $s3_store_large_object_chunk_size;
"${name}/s3_store_thread_pools": value => $s3_store_thread_pools;
"${name}/store_description": value => $store_description;
}
glance_cache_config {
"${name}/s3_store_host": value => $s3_store_host;
"${name}/s3_store_access_key": value => $s3_store_access_key, secret => true;
"${name}/s3_store_secret_key": value => $s3_store_secret_key, secret => true;
"${name}/s3_store_bucket": value => $s3_store_bucket;
"${name}/s3_store_create_bucket_on_put": value => $s3_store_create_bucket_on_put;
"${name}/s3_store_bucket_url_format": value => $s3_store_bucket_url_format;
"${name}/s3_store_large_object_size": value => $s3_store_large_object_size;
"${name}/s3_store_large_object_chunk_size": value => $s3_store_large_object_chunk_size;
"${name}/s3_store_thread_pools": value => $s3_store_thread_pools;
}
}

68
manifests/backend/s3.pp
View File

@ -1,68 +0,0 @@
# == class: glance::backend::s3
#
# configures the storage backend for glance
# as a s3 instance
#
# === parameters:
#
# [*access_key*]
# (Required) The S3 query token access key.
#
# [*secret_key*]
# (Required) The S3 query token secret key.
#
# [*host*]
# (Required) The host where the S3 server is listening.
#
# [*bucket*]
# (Required) The S3 bucket to be used to store the Glance data.
#
# [*bucket_url_format*]
# (Optional) The S3 calling format used to determine the bucket. Either
# 'subdomain' or 'path' can be used.
# Default: $::os_service_default.
#
# [*create_bucket_on_put*]
# (Optional) A boolean to determine if the S3 bucket should be created on
# upload if it does not exist or if an error should be returned to the user.
# Default: $::os_service_default.
#
# [*large_object_size*]
# (Optional) What size, in MB, should S3 start chunking image files and do a
# multipart upload in S3.
# Default: $::os_service_default.
#
# [*large_object_chunk_size*]
# (Optional) What multipart upload part size, in MB, should S3 use when
# uploading parts. The size must be greater than or equal to 5M.
# Default: $::os_service_default.
#
# [*object_buffer_dir*]
# (Optional) The local directory where uploads will be staged before they are
# transferred into S3.
# Default: $::os_service_default.
#
# [*thread_pools*]
# (Optional) The number of thread pools to perform a multipart upload in S3.
# Default: $::os_service_default.
#
# [*multi_store*]
# (optional) Boolean describing if multiple backends will be configured
# Defaults to false
#
class glance::backend::s3(
$access_key,
$secret_key,
$host,
$bucket,
$bucket_url_format = $::os_service_default,
$create_bucket_on_put = $::os_service_default,
$large_object_size = $::os_service_default,
$large_object_chunk_size = $::os_service_default,
$object_buffer_dir = $::os_service_default,
$thread_pools = $::os_service_default,
$multi_store = false,
) {
warning('glance::backend::s3 is deprecated and has no effect. Glance no longer supports the s3 backend.')
}

1
manifests/params.pp
View File

@ -8,6 +8,7 @@ class glance::params {
$cache_cleaner_command = 'glance-cache-cleaner'
$cache_pruner_command = 'glance-cache-pruner'
$group = 'glance'
$boto3_package_name = 'python3-boto3'
case $::osfamily {
'RedHat': {

10
releasenotes/notes/restore-s3-support-82e63d83b1184cdf.yaml Normal file
View File

@ -0,0 +1,10 @@
---
features:
- |
Support for S3 backend has been reintroduced, because s3 backend support
was restored in glance_store.
upgrade:
- |
The deprecated ``glance::backend::s3`` class has been removed. Use the new
``glance::backend::multistore::s3`` resource type to set up S3 backend.

142
spec/defines/glance_backend_multistore_s3_spec.rb Normal file
View File

@ -0,0 +1,142 @@
#
# Copyright 2021 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# Unit tests for glance::backend::multistore::s3
#
require 'spec_helper'
describe 'glance::backend::multistore::s3' do
let (:title) { 's3' }
let :params do
{
:s3_store_host => 'host',
:s3_store_access_key => 'access',
:s3_store_secret_key => 'secret',
:s3_store_bucket => 'bucket',
}
end
shared_examples_for 'glance::backend::multistore::s3' do
context 'with defaults' do
it 'installs boto3' do
is_expected.to contain_package('python-boto3').with(
:ensure => 'installed',
:name => 'python3-boto3',
:tag => ['openstack', 'glance-package']
)
end
it 'configures glance-api.conf' do
is_expected.to contain_glance_api_config('s3/store_description').with_value('<SERVICE DEFAULT>')
is_expected.to contain_glance_api_config('s3/s3_store_host').with_value('host')
is_expected.to contain_glance_api_config('s3/s3_store_access_key').with_value('access').with_secret(true)
is_expected.to contain_glance_api_config('s3/s3_store_secret_key').with_value('secret').with_secret(true)
is_expected.to contain_glance_api_config('s3/s3_store_bucket').with_value('bucket')
is_expected.to contain_glance_api_config('s3/s3_store_create_bucket_on_put').with_value('<SERVICE DEFAULT>')
is_expected.to contain_glance_api_config('s3/s3_store_bucket_url_format').with_value('<SERVICE DEFAULT>')
is_expected.to contain_glance_api_config('s3/s3_store_large_object_size').with_value('<SERVICE DEFAULT>')
is_expected.to contain_glance_api_config('s3/s3_store_large_object_chunk_size').with_value('<SERVICE DEFAULT>')
is_expected.to contain_glance_api_config('s3/s3_store_thread_pools').with_value('<SERVICE DEFAULT>')
end
it 'configures glance-cache.conf' do
is_expected.to_not contain_glance_cache_config('s3/store_description')
is_expected.to contain_glance_cache_config('s3/s3_store_host').with_value('host')
is_expected.to contain_glance_cache_config('s3/s3_store_access_key').with_value('access').with_secret(true)
is_expected.to contain_glance_cache_config('s3/s3_store_secret_key').with_value('secret').with_secret(true)
is_expected.to contain_glance_cache_config('s3/s3_store_bucket').with_value('bucket')
is_expected.to contain_glance_cache_config('s3/s3_store_create_bucket_on_put').with_value('<SERVICE DEFAULT>')
is_expected.to contain_glance_cache_config('s3/s3_store_bucket_url_format').with_value('<SERVICE DEFAULT>')
is_expected.to contain_glance_cache_config('s3/s3_store_large_object_size').with_value('<SERVICE DEFAULT>')
is_expected.to contain_glance_cache_config('s3/s3_store_large_object_chunk_size').with_value('<SERVICE DEFAULT>')
is_expected.to contain_glance_cache_config('s3/s3_store_thread_pools').with_value('<SERVICE DEFAULT>')
end
end
context 'when overriding datadir' do
before :each do
params.merge!({
:store_description => 's3store',
:s3_store_create_bucket_on_put => false,
:s3_store_bucket_url_format => 'auto',
:s3_store_large_object_size => 100,
:s3_store_large_object_chunk_size => 10,
:s3_store_thread_pools => 11,
:package_ensure => 'latest',
})
end
it 'installs boto3' do
is_expected.to contain_package('python-boto3').with(
:ensure => 'latest',
:name => 'python3-boto3',
:tag => ['openstack', 'glance-package']
)
end
it 'configures glance-api.conf' do
is_expected.to contain_glance_api_config('s3/store_description').with_value('s3store')
is_expected.to contain_glance_api_config('s3/s3_store_host').with_value('host')
is_expected.to contain_glance_api_config('s3/s3_store_access_key').with_value('access').with_secret(true)
is_expected.to contain_glance_api_config('s3/s3_store_secret_key').with_value('secret').with_secret(true)
is_expected.to contain_glance_api_config('s3/s3_store_bucket').with_value('bucket')
is_expected.to contain_glance_api_config('s3/s3_store_create_bucket_on_put').with_value(false)
is_expected.to contain_glance_api_config('s3/s3_store_bucket_url_format').with_value('auto')
is_expected.to contain_glance_api_config('s3/s3_store_large_object_size').with_value(100)
is_expected.to contain_glance_api_config('s3/s3_store_large_object_chunk_size').with_value(10)
is_expected.to contain_glance_api_config('s3/s3_store_thread_pools').with_value(11)
end
it 'configures glance-cache.conf' do
is_expected.to_not contain_glance_cache_config('s3/store_description')
is_expected.to contain_glance_cache_config('s3/s3_store_host').with_value('host')
is_expected.to contain_glance_cache_config('s3/s3_store_access_key').with_value('access').with_secret(true)
is_expected.to contain_glance_cache_config('s3/s3_store_secret_key').with_value('secret').with_secret(true)
is_expected.to contain_glance_cache_config('s3/s3_store_bucket').with_value('bucket')
is_expected.to contain_glance_cache_config('s3/s3_store_create_bucket_on_put').with_value(false)
is_expected.to contain_glance_cache_config('s3/s3_store_bucket_url_format').with_value('auto')
is_expected.to contain_glance_cache_config('s3/s3_store_large_object_size').with_value(100)
is_expected.to contain_glance_cache_config('s3/s3_store_large_object_chunk_size').with_value(10)
is_expected.to contain_glance_cache_config('s3/s3_store_thread_pools').with_value(11)
end
end
context 'when package management is disabled' do
before :each do
params.merge!({
:manage_packages => false
})
end
it 'does not manage packages' do
is_expected.to_not contain_package('python-boto3')
end
end
end
on_supported_os({
:supported_os => OSDefaults.get_supported_os
}).each do |os,facts|
context "on #{os}" do
let (:facts) do
facts.merge!(OSDefaults.get_facts())
end
it_behaves_like 'glance::backend::multistore::s3'
end
end
end