Add a Swift storage backend implementation

This implements the storage backend interface added in the previous
commit using Swift. It also adds some example configuration to the
sample config file, and some setup to initialise a storage backend
if one is configured.

Change-Id: I8467486ed42f8674e2b1db635789e88bf4113850

etc/storyboard.conf.sample

@@ -197,3 +197,52 @@ lock_path = $state_path/lock
 
 # Password for the SMTP server.
 # smtp_password =
+
+[attachments]
+
+# Whether or not to enable attachment support. Requires a supported
+# attachment storage backend to be available and configured. Disabled
+# by default.
+# enable_attachments = True
+
+# The type of storage backend to use for attachments. Currently only
+# `swift` is a valid value.
+# storage_backend = swift
+
+# Settings in this section are used when storage_backend is set to
+# `swift`. Default values are set to work out of the box with a
+# Swift all-in-one instance accessible at 127.0.0.1:8888.
+[swift]
+
+# Name of the cloud in clouds.yaml which provides the object storage
+# to use. If this is set then `auth_type`, `auth_url`, `user`, and
+# `password` are ignored in favour of the auth configuration in your
+# clouds.yaml file. This should be used in most cases, the other
+# options are for supporting Swift legacy auth.
+# cloud =
+
+# Authentication type to use for connecting to Swift. For legacy auth,
+# this should be `v1password`. For all other auth, the `clouds.yaml`
+# approach should be used instead.
+# auth_type = v1password
+
+# Authentication endpoint for the Swift backend.
+# auth_url = http://127.0.0.1:8888/auth/v1.0
+
+# User to authenticate with Swift as.
+# user = test:tester
+
+# Password for the configured Swift user.
+# password = testing
+
+# Swift container to store attachments in. This will be created if it
+# doesn't already exist.
+# container = storyboard
+
+# The value to set X-Container-Meta-Temp-URL-Key to if the container
+# needs to be created by StoryBoard. If your container already exists,
+# you should ensure it has this metadata set separately.
+# temp_url_key = secret_key
+
+# The time in seconds that generated Temp URL signatures are valid for.
+# temp_url_timeout = 60

requirements.txt

@@ -30,3 +30,5 @@ python_dateutil>=2.4.0
 oslo.concurrency>=3.8.0         # Apache-2.0
 oslo.i18n>=2.1.0  # Apache-2.0
 #launchpadlib         # Only for migration
+python-swiftclient
+openstacksdk

storyboard/api/app.py

@@ -30,6 +30,8 @@ from storyboard.api.middleware import user_id_hook
 from storyboard.api.middleware import validation_hook
 from storyboard.api.v1.search import impls as search_engine_impls
 from storyboard.api.v1.search import search_engine
+from storyboard.api.v1.storage import impls as storage_impls
+from storyboard.api.v1.storage import storage
 from storyboard.notifications.notification_hook import NotificationHook
 from storyboard.plugin.scheduler import initialize_scheduler
 from storyboard.plugin.user_preferences import initialize_user_preferences
@@ -99,6 +101,12 @@ def setup_app(pecan_config=None):
     search_engine_cls = search_engine_impls.ENGINE_IMPLS[search_engine_name]
     search_engine.set_engine(search_engine_cls())
 
+    # Setup storage backend
+    if CONF.attachments.enable_attachments:
+        storage_type = CONF.attachments.storage_backend
+        storage_cls = storage_impls.STORAGE_IMPLS[storage_type]
+        storage.set_storage_backend(storage_cls())
+
     # Load user preference plugins
     initialize_user_preferences()
 

storyboard/api/v1/storage/impls.py

@@ -0,0 +1,21 @@
+# Copyright (c) 2019 Adam Coldrick
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from storyboard.api.v1.storage.swift_impl import SwiftStorageImpl
+
+
+STORAGE_IMPLS = {
+    "swift": SwiftStorageImpl
+}

storyboard/api/v1/storage/swift_impl.py

@@ -0,0 +1,116 @@
+# Copyright (c) 2019 Adam Coldrick
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from hashlib import sha1
+import hmac
+from time import time
+import uuid
+
+import openstack
+from openstack import connection
+from oslo_config import cfg
+from six.moves.urllib import parse
+
+from storyboard.api.v1.storage.storage import StorageBackend
+
+
+CONF = cfg.CONF
+
+SWIFT_OPTS = [
+    cfg.StrOpt("cloud",
+               default="",
+               help="Name of the cloud which provides Swift as "
+                    "used in `clouds.yaml`. Other auth-related "
+                    "options are ignored if this is set."),
+    cfg.StrOpt("auth_url",
+               default="http://127.0.0.1:8888/auth/v1.0",
+               help="URL to use to obtain an auth token from swift."),
+    cfg.StrOpt("auth_type",
+               default="v1password",
+               help="Swift auth type, defaults to 'v1password' "
+                    "(which is legacy auth)."),
+    cfg.StrOpt("user",
+               default="test:tester",
+               help="User to use when authenticating with Swift to obtain an "
+                    "auth token."),
+    cfg.StrOpt("password",
+               default="testing",
+               help="Password to use when authenticating with Swift."),
+    cfg.StrOpt("container",
+               default="storyboard",
+               help="Swift container to store attachments in. Will be "
+                    "created if it doesn't already exist."),
+    cfg.StrOpt("temp_url_key",
+               default="secret_key",
+               help="Temp URL secret key to set for the container if it "
+                    "is created by StoryBoard."),
+    cfg.IntOpt("temp_url_timeout",
+               default=120,
+               help="Number of seconds that Swift tempurl signatures "
+                    "are valid for after generation.")
+]
+
+CONF.register_opts(SWIFT_OPTS, "swift")
+
+
+class SwiftStorageImpl(StorageBackend):
+    """Implementation of an attachment storage backend using swift."""
+
+    def _get_connection(self):
+        if CONF.swift.cloud:
+            return connection.Connection(
+                cloud=CONF.swift.cloud,
+                service_types={'object-store'})
+
+        return openstack.connect(
+            auth_type=CONF.swift.auth_type,
+            auth_url=CONF.swift.auth_url,
+            username=CONF.swift.user,
+            password=CONF.swift.password,
+        )
+
+    def _ensure_container_exists(self, conn):
+        names = [container.name
+                 for container in conn.object_store.containers()]
+        if CONF.swift.container not in names:
+            conn.object_store.create_container(CONF.swift.container)
+            conn.object_store.set_container_temp_url_key(
+                CONF.swift.container, CONF.swift.temp_url_key)
+            container = conn.object_store.set_container_metadata(
+                CONF.swift.container, read_ACL=".r:*")
+
+    def get_upload_url(self):
+        conn = self._get_connection()
+        self._ensure_container_exists(conn)
+
+        url = conn.object_store.get_endpoint()
+        return "%s/%s" % (url, CONF.swift.container)
+
+    def get_auth(self):
+        conn = self._get_connection()
+        self._ensure_container_exists(conn)
+
+        name = str(uuid.uuid4())
+        endpoint = parse.urlparse(conn.object_store.get_endpoint())
+        path = '/'.join((endpoint.path, CONF.swift.container, name))
+
+        method = 'PUT'
+        expires = int(time() + CONF.swift.temp_url_timeout)
+        hmac_body = '%s\n%s\n%s' % (method, expires, path)
+        hmac_body = hmac_body.encode('utf8')
+
+        key = conn.object_store.get_temp_url_key(CONF.swift.container)
+        signature = hmac.new(key, hmac_body, sha1).hexdigest()
+        return expires, signature, name