diff --git a/etc/storyboard.conf.sample b/etc/storyboard.conf.sample index d99501af..a0622c83 100644 --- a/etc/storyboard.conf.sample +++ b/etc/storyboard.conf.sample @@ -197,3 +197,52 @@ lock_path = $state_path/lock # Password for the SMTP server. # smtp_password = + +[attachments] + +# Whether or not to enable attachment support. Requires a supported +# attachment storage backend to be available and configured. Disabled +# by default. +# enable_attachments = True + +# The type of storage backend to use for attachments. Currently only +# `swift` is a valid value. +# storage_backend = swift + +# Settings in this section are used when storage_backend is set to +# `swift`. Default values are set to work out of the box with a +# Swift all-in-one instance accessible at 127.0.0.1:8888. +[swift] + +# Name of the cloud in clouds.yaml which provides the object storage +# to use. If this is set then `auth_type`, `auth_url`, `user`, and +# `password` are ignored in favour of the auth configuration in your +# clouds.yaml file. This should be used in most cases, the other +# options are for supporting Swift legacy auth. +# cloud = + +# Authentication type to use for connecting to Swift. For legacy auth, +# this should be `v1password`. For all other auth, the `clouds.yaml` +# approach should be used instead. +# auth_type = v1password + +# Authentication endpoint for the Swift backend. +# auth_url = http://127.0.0.1:8888/auth/v1.0 + +# User to authenticate with Swift as. +# user = test:tester + +# Password for the configured Swift user. +# password = testing + +# Swift container to store attachments in. This will be created if it +# doesn't already exist. +# container = storyboard + +# The value to set X-Container-Meta-Temp-URL-Key to if the container +# needs to be created by StoryBoard. If your container already exists, +# you should ensure it has this metadata set separately. +# temp_url_key = secret_key + +# The time in seconds that generated Temp URL signatures are valid for. +# temp_url_timeout = 60 diff --git a/requirements.txt b/requirements.txt index 5e126034..10d30039 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,3 +30,5 @@ python_dateutil>=2.4.0 oslo.concurrency>=3.8.0 # Apache-2.0 oslo.i18n>=2.1.0 # Apache-2.0 #launchpadlib # Only for migration +python-swiftclient +openstacksdk \ No newline at end of file diff --git a/storyboard/api/app.py b/storyboard/api/app.py index cf90accd..7ea87eea 100644 --- a/storyboard/api/app.py +++ b/storyboard/api/app.py @@ -30,6 +30,8 @@ from storyboard.api.middleware import user_id_hook from storyboard.api.middleware import validation_hook from storyboard.api.v1.search import impls as search_engine_impls from storyboard.api.v1.search import search_engine +from storyboard.api.v1.storage import impls as storage_impls +from storyboard.api.v1.storage import storage from storyboard.notifications.notification_hook import NotificationHook from storyboard.plugin.scheduler import initialize_scheduler from storyboard.plugin.user_preferences import initialize_user_preferences @@ -99,6 +101,12 @@ def setup_app(pecan_config=None): search_engine_cls = search_engine_impls.ENGINE_IMPLS[search_engine_name] search_engine.set_engine(search_engine_cls()) + # Setup storage backend + if CONF.attachments.enable_attachments: + storage_type = CONF.attachments.storage_backend + storage_cls = storage_impls.STORAGE_IMPLS[storage_type] + storage.set_storage_backend(storage_cls()) + # Load user preference plugins initialize_user_preferences() diff --git a/storyboard/api/v1/storage/impls.py b/storyboard/api/v1/storage/impls.py new file mode 100644 index 00000000..a68793fe --- /dev/null +++ b/storyboard/api/v1/storage/impls.py @@ -0,0 +1,21 @@ +# Copyright (c) 2019 Adam Coldrick +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from storyboard.api.v1.storage.swift_impl import SwiftStorageImpl + + +STORAGE_IMPLS = { + "swift": SwiftStorageImpl +} diff --git a/storyboard/api/v1/storage/swift_impl.py b/storyboard/api/v1/storage/swift_impl.py new file mode 100644 index 00000000..e1cb1511 --- /dev/null +++ b/storyboard/api/v1/storage/swift_impl.py @@ -0,0 +1,116 @@ +# Copyright (c) 2019 Adam Coldrick +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from hashlib import sha1 +import hmac +from time import time +import uuid + +import openstack +from openstack import connection +from oslo_config import cfg +from six.moves.urllib import parse + +from storyboard.api.v1.storage.storage import StorageBackend + + +CONF = cfg.CONF + +SWIFT_OPTS = [ + cfg.StrOpt("cloud", + default="", + help="Name of the cloud which provides Swift as " + "used in `clouds.yaml`. Other auth-related " + "options are ignored if this is set."), + cfg.StrOpt("auth_url", + default="http://127.0.0.1:8888/auth/v1.0", + help="URL to use to obtain an auth token from swift."), + cfg.StrOpt("auth_type", + default="v1password", + help="Swift auth type, defaults to 'v1password' " + "(which is legacy auth)."), + cfg.StrOpt("user", + default="test:tester", + help="User to use when authenticating with Swift to obtain an " + "auth token."), + cfg.StrOpt("password", + default="testing", + help="Password to use when authenticating with Swift."), + cfg.StrOpt("container", + default="storyboard", + help="Swift container to store attachments in. Will be " + "created if it doesn't already exist."), + cfg.StrOpt("temp_url_key", + default="secret_key", + help="Temp URL secret key to set for the container if it " + "is created by StoryBoard."), + cfg.IntOpt("temp_url_timeout", + default=120, + help="Number of seconds that Swift tempurl signatures " + "are valid for after generation.") +] + +CONF.register_opts(SWIFT_OPTS, "swift") + + +class SwiftStorageImpl(StorageBackend): + """Implementation of an attachment storage backend using swift.""" + + def _get_connection(self): + if CONF.swift.cloud: + return connection.Connection( + cloud=CONF.swift.cloud, + service_types={'object-store'}) + + return openstack.connect( + auth_type=CONF.swift.auth_type, + auth_url=CONF.swift.auth_url, + username=CONF.swift.user, + password=CONF.swift.password, + ) + + def _ensure_container_exists(self, conn): + names = [container.name + for container in conn.object_store.containers()] + if CONF.swift.container not in names: + conn.object_store.create_container(CONF.swift.container) + conn.object_store.set_container_temp_url_key( + CONF.swift.container, CONF.swift.temp_url_key) + container = conn.object_store.set_container_metadata( + CONF.swift.container, read_ACL=".r:*") + + def get_upload_url(self): + conn = self._get_connection() + self._ensure_container_exists(conn) + + url = conn.object_store.get_endpoint() + return "%s/%s" % (url, CONF.swift.container) + + def get_auth(self): + conn = self._get_connection() + self._ensure_container_exists(conn) + + name = str(uuid.uuid4()) + endpoint = parse.urlparse(conn.object_store.get_endpoint()) + path = '/'.join((endpoint.path, CONF.swift.container, name)) + + method = 'PUT' + expires = int(time() + CONF.swift.temp_url_timeout) + hmac_body = '%s\n%s\n%s' % (method, expires, path) + hmac_body = hmac_body.encode('utf8') + + key = conn.object_store.get_temp_url_key(CONF.swift.container) + signature = hmac.new(key, hmac_body, sha1).hexdigest() + return expires, signature, name