From 385a34fa77d0475f14c6f5b35f2972143519d780 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Sat, 26 Jan 2019 17:50:08 +0000 Subject: [PATCH] Add a Swift storage backend implementation This implements the storage backend interface added in the previous commit using Swift. It also adds some example configuration to the sample config file, and some setup to initialise a storage backend if one is configured. Change-Id: I8467486ed42f8674e2b1db635789e88bf4113850 --- etc/storyboard.conf.sample | 49 ++++++++++ requirements.txt | 2 + storyboard/api/app.py | 8 ++ storyboard/api/v1/storage/impls.py | 21 +++++ storyboard/api/v1/storage/swift_impl.py | 116 ++++++++++++++++++++++++ 5 files changed, 196 insertions(+) create mode 100644 storyboard/api/v1/storage/impls.py create mode 100644 storyboard/api/v1/storage/swift_impl.py diff --git a/etc/storyboard.conf.sample b/etc/storyboard.conf.sample index d99501af..a0622c83 100644 --- a/etc/storyboard.conf.sample +++ b/etc/storyboard.conf.sample @@ -197,3 +197,52 @@ lock_path = $state_path/lock # Password for the SMTP server. # smtp_password = + +[attachments] + +# Whether or not to enable attachment support. Requires a supported +# attachment storage backend to be available and configured. Disabled +# by default. +# enable_attachments = True + +# The type of storage backend to use for attachments. Currently only +# `swift` is a valid value. +# storage_backend = swift + +# Settings in this section are used when storage_backend is set to +# `swift`. Default values are set to work out of the box with a +# Swift all-in-one instance accessible at 127.0.0.1:8888. +[swift] + +# Name of the cloud in clouds.yaml which provides the object storage +# to use. If this is set then `auth_type`, `auth_url`, `user`, and +# `password` are ignored in favour of the auth configuration in your +# clouds.yaml file. This should be used in most cases, the other +# options are for supporting Swift legacy auth. +# cloud = + +# Authentication type to use for connecting to Swift. For legacy auth, +# this should be `v1password`. For all other auth, the `clouds.yaml` +# approach should be used instead. +# auth_type = v1password + +# Authentication endpoint for the Swift backend. +# auth_url = http://127.0.0.1:8888/auth/v1.0 + +# User to authenticate with Swift as. +# user = test:tester + +# Password for the configured Swift user. +# password = testing + +# Swift container to store attachments in. This will be created if it +# doesn't already exist. +# container = storyboard + +# The value to set X-Container-Meta-Temp-URL-Key to if the container +# needs to be created by StoryBoard. If your container already exists, +# you should ensure it has this metadata set separately. +# temp_url_key = secret_key + +# The time in seconds that generated Temp URL signatures are valid for. +# temp_url_timeout = 60 diff --git a/requirements.txt b/requirements.txt index 5e126034..10d30039 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,3 +30,5 @@ python_dateutil>=2.4.0 oslo.concurrency>=3.8.0 # Apache-2.0 oslo.i18n>=2.1.0 # Apache-2.0 #launchpadlib # Only for migration +python-swiftclient +openstacksdk \ No newline at end of file diff --git a/storyboard/api/app.py b/storyboard/api/app.py index cf90accd..7ea87eea 100644 --- a/storyboard/api/app.py +++ b/storyboard/api/app.py @@ -30,6 +30,8 @@ from storyboard.api.middleware import user_id_hook from storyboard.api.middleware import validation_hook from storyboard.api.v1.search import impls as search_engine_impls from storyboard.api.v1.search import search_engine +from storyboard.api.v1.storage import impls as storage_impls +from storyboard.api.v1.storage import storage from storyboard.notifications.notification_hook import NotificationHook from storyboard.plugin.scheduler import initialize_scheduler from storyboard.plugin.user_preferences import initialize_user_preferences @@ -99,6 +101,12 @@ def setup_app(pecan_config=None): search_engine_cls = search_engine_impls.ENGINE_IMPLS[search_engine_name] search_engine.set_engine(search_engine_cls()) + # Setup storage backend + if CONF.attachments.enable_attachments: + storage_type = CONF.attachments.storage_backend + storage_cls = storage_impls.STORAGE_IMPLS[storage_type] + storage.set_storage_backend(storage_cls()) + # Load user preference plugins initialize_user_preferences() diff --git a/storyboard/api/v1/storage/impls.py b/storyboard/api/v1/storage/impls.py new file mode 100644 index 00000000..a68793fe --- /dev/null +++ b/storyboard/api/v1/storage/impls.py @@ -0,0 +1,21 @@ +# Copyright (c) 2019 Adam Coldrick +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from storyboard.api.v1.storage.swift_impl import SwiftStorageImpl + + +STORAGE_IMPLS = { + "swift": SwiftStorageImpl +} diff --git a/storyboard/api/v1/storage/swift_impl.py b/storyboard/api/v1/storage/swift_impl.py new file mode 100644 index 00000000..e1cb1511 --- /dev/null +++ b/storyboard/api/v1/storage/swift_impl.py @@ -0,0 +1,116 @@ +# Copyright (c) 2019 Adam Coldrick +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from hashlib import sha1 +import hmac +from time import time +import uuid + +import openstack +from openstack import connection +from oslo_config import cfg +from six.moves.urllib import parse + +from storyboard.api.v1.storage.storage import StorageBackend + + +CONF = cfg.CONF + +SWIFT_OPTS = [ + cfg.StrOpt("cloud", + default="", + help="Name of the cloud which provides Swift as " + "used in `clouds.yaml`. Other auth-related " + "options are ignored if this is set."), + cfg.StrOpt("auth_url", + default="http://127.0.0.1:8888/auth/v1.0", + help="URL to use to obtain an auth token from swift."), + cfg.StrOpt("auth_type", + default="v1password", + help="Swift auth type, defaults to 'v1password' " + "(which is legacy auth)."), + cfg.StrOpt("user", + default="test:tester", + help="User to use when authenticating with Swift to obtain an " + "auth token."), + cfg.StrOpt("password", + default="testing", + help="Password to use when authenticating with Swift."), + cfg.StrOpt("container", + default="storyboard", + help="Swift container to store attachments in. Will be " + "created if it doesn't already exist."), + cfg.StrOpt("temp_url_key", + default="secret_key", + help="Temp URL secret key to set for the container if it " + "is created by StoryBoard."), + cfg.IntOpt("temp_url_timeout", + default=120, + help="Number of seconds that Swift tempurl signatures " + "are valid for after generation.") +] + +CONF.register_opts(SWIFT_OPTS, "swift") + + +class SwiftStorageImpl(StorageBackend): + """Implementation of an attachment storage backend using swift.""" + + def _get_connection(self): + if CONF.swift.cloud: + return connection.Connection( + cloud=CONF.swift.cloud, + service_types={'object-store'}) + + return openstack.connect( + auth_type=CONF.swift.auth_type, + auth_url=CONF.swift.auth_url, + username=CONF.swift.user, + password=CONF.swift.password, + ) + + def _ensure_container_exists(self, conn): + names = [container.name + for container in conn.object_store.containers()] + if CONF.swift.container not in names: + conn.object_store.create_container(CONF.swift.container) + conn.object_store.set_container_temp_url_key( + CONF.swift.container, CONF.swift.temp_url_key) + container = conn.object_store.set_container_metadata( + CONF.swift.container, read_ACL=".r:*") + + def get_upload_url(self): + conn = self._get_connection() + self._ensure_container_exists(conn) + + url = conn.object_store.get_endpoint() + return "%s/%s" % (url, CONF.swift.container) + + def get_auth(self): + conn = self._get_connection() + self._ensure_container_exists(conn) + + name = str(uuid.uuid4()) + endpoint = parse.urlparse(conn.object_store.get_endpoint()) + path = '/'.join((endpoint.path, CONF.swift.container, name)) + + method = 'PUT' + expires = int(time() + CONF.swift.temp_url_timeout) + hmac_body = '%s\n%s\n%s' % (method, expires, path) + hmac_body = hmac_body.encode('utf8') + + key = conn.object_store.get_temp_url_key(CONF.swift.container) + signature = hmac.new(key, hmac_body, sha1).hexdigest() + return expires, signature, name