Browse Source

Add a Swift storage backend implementation

This implements the storage backend interface added in the previous
commit using Swift. It also adds some example configuration to the
sample config file, and some setup to initialise a storage backend
if one is configured.

Change-Id: I8467486ed42f8674e2b1db635789e88bf4113850
changes/65/633365/5
Adam Coldrick 4 years ago
parent
commit
385a34fa77
  1. 49
      etc/storyboard.conf.sample
  2. 2
      requirements.txt
  3. 8
      storyboard/api/app.py
  4. 21
      storyboard/api/v1/storage/impls.py
  5. 116
      storyboard/api/v1/storage/swift_impl.py

49
etc/storyboard.conf.sample

@ -197,3 +197,52 @@ lock_path = $state_path/lock
# Password for the SMTP server.
# smtp_password =
[attachments]
# Whether or not to enable attachment support. Requires a supported
# attachment storage backend to be available and configured. Disabled
# by default.
# enable_attachments = True
# The type of storage backend to use for attachments. Currently only
# `swift` is a valid value.
# storage_backend = swift
# Settings in this section are used when storage_backend is set to
# `swift`. Default values are set to work out of the box with a
# Swift all-in-one instance accessible at 127.0.0.1:8888.
[swift]
# Name of the cloud in clouds.yaml which provides the object storage
# to use. If this is set then `auth_type`, `auth_url`, `user`, and
# `password` are ignored in favour of the auth configuration in your
# clouds.yaml file. This should be used in most cases, the other
# options are for supporting Swift legacy auth.
# cloud =
# Authentication type to use for connecting to Swift. For legacy auth,
# this should be `v1password`. For all other auth, the `clouds.yaml`
# approach should be used instead.
# auth_type = v1password
# Authentication endpoint for the Swift backend.
# auth_url = http://127.0.0.1:8888/auth/v1.0
# User to authenticate with Swift as.
# user = test:tester
# Password for the configured Swift user.
# password = testing
# Swift container to store attachments in. This will be created if it
# doesn't already exist.
# container = storyboard
# The value to set X-Container-Meta-Temp-URL-Key to if the container
# needs to be created by StoryBoard. If your container already exists,
# you should ensure it has this metadata set separately.
# temp_url_key = secret_key
# The time in seconds that generated Temp URL signatures are valid for.
# temp_url_timeout = 60

2
requirements.txt

@ -30,3 +30,5 @@ python_dateutil>=2.4.0
oslo.concurrency>=3.8.0 # Apache-2.0
oslo.i18n>=2.1.0 # Apache-2.0
#launchpadlib # Only for migration
python-swiftclient
openstacksdk

8
storyboard/api/app.py

@ -30,6 +30,8 @@ from storyboard.api.middleware import user_id_hook
from storyboard.api.middleware import validation_hook
from storyboard.api.v1.search import impls as search_engine_impls
from storyboard.api.v1.search import search_engine
from storyboard.api.v1.storage import impls as storage_impls
from storyboard.api.v1.storage import storage
from storyboard.notifications.notification_hook import NotificationHook
from storyboard.plugin.scheduler import initialize_scheduler
from storyboard.plugin.user_preferences import initialize_user_preferences
@ -99,6 +101,12 @@ def setup_app(pecan_config=None):
search_engine_cls = search_engine_impls.ENGINE_IMPLS[search_engine_name]
search_engine.set_engine(search_engine_cls())
# Setup storage backend
if CONF.attachments.enable_attachments:
storage_type = CONF.attachments.storage_backend
storage_cls = storage_impls.STORAGE_IMPLS[storage_type]
storage.set_storage_backend(storage_cls())
# Load user preference plugins
initialize_user_preferences()

21
storyboard/api/v1/storage/impls.py

@ -0,0 +1,21 @@
# Copyright (c) 2019 Adam Coldrick
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from storyboard.api.v1.storage.swift_impl import SwiftStorageImpl
STORAGE_IMPLS = {
"swift": SwiftStorageImpl
}

116
storyboard/api/v1/storage/swift_impl.py

@ -0,0 +1,116 @@
# Copyright (c) 2019 Adam Coldrick
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from hashlib import sha1
import hmac
from time import time
import uuid
import openstack
from openstack import connection
from oslo_config import cfg
from six.moves.urllib import parse
from storyboard.api.v1.storage.storage import StorageBackend
CONF = cfg.CONF
SWIFT_OPTS = [
cfg.StrOpt("cloud",
default="",
help="Name of the cloud which provides Swift as "
"used in `clouds.yaml`. Other auth-related "
"options are ignored if this is set."),
cfg.StrOpt("auth_url",
default="http://127.0.0.1:8888/auth/v1.0",
help="URL to use to obtain an auth token from swift."),
cfg.StrOpt("auth_type",
default="v1password",
help="Swift auth type, defaults to 'v1password' "
"(which is legacy auth)."),
cfg.StrOpt("user",
default="test:tester",
help="User to use when authenticating with Swift to obtain an "
"auth token."),
cfg.StrOpt("password",
default="testing",
help="Password to use when authenticating with Swift."),
cfg.StrOpt("container",
default="storyboard",
help="Swift container to store attachments in. Will be "
"created if it doesn't already exist."),
cfg.StrOpt("temp_url_key",
default="secret_key",
help="Temp URL secret key to set for the container if it "
"is created by StoryBoard."),
cfg.IntOpt("temp_url_timeout",
default=120,
help="Number of seconds that Swift tempurl signatures "
"are valid for after generation.")
]
CONF.register_opts(SWIFT_OPTS, "swift")
class SwiftStorageImpl(StorageBackend):
"""Implementation of an attachment storage backend using swift."""
def _get_connection(self):
if CONF.swift.cloud:
return connection.Connection(
cloud=CONF.swift.cloud,
service_types={'object-store'})
return openstack.connect(
auth_type=CONF.swift.auth_type,
auth_url=CONF.swift.auth_url,
username=CONF.swift.user,
password=CONF.swift.password,
)
def _ensure_container_exists(self, conn):
names = [container.name
for container in conn.object_store.containers()]
if CONF.swift.container not in names:
conn.object_store.create_container(CONF.swift.container)
conn.object_store.set_container_temp_url_key(
CONF.swift.container, CONF.swift.temp_url_key)
container = conn.object_store.set_container_metadata(
CONF.swift.container, read_ACL=".r:*")
def get_upload_url(self):
conn = self._get_connection()
self._ensure_container_exists(conn)
url = conn.object_store.get_endpoint()
return "%s/%s" % (url, CONF.swift.container)
def get_auth(self):
conn = self._get_connection()
self._ensure_container_exists(conn)
name = str(uuid.uuid4())
endpoint = parse.urlparse(conn.object_store.get_endpoint())
path = '/'.join((endpoint.path, CONF.swift.container, name))
method = 'PUT'
expires = int(time() + CONF.swift.temp_url_timeout)
hmac_body = '%s\n%s\n%s' % (method, expires, path)
hmac_body = hmac_body.encode('utf8')
key = conn.object_store.get_temp_url_key(CONF.swift.container)
signature = hmac.new(key, hmac_body, sha1).hexdigest()
return expires, signature, name
Loading…
Cancel
Save