Add Sheepdog store
Sheepdog is a distributed block storage. This patch enables Sheepdog cluster as a backend store for glance. Test: You can set up a simulated 3 node cluster on the local machine with following script: $ sudo apt-get install liburcu-dev $ git clone git://github.com/collie/sheepdog.git $ cd sheepdog $ ./autogen.sh; ./configure --disable-corosync $ make; sudo make install $ for i in 0 1 2; do sheep /tmp/store$i -n -c local -z $i -p 700$i;done $ collie cluster format Then change the default store in glance-api.conf as sheepdog blueprint: add-sheepdog-support Change-Id: I99907bbfc2e131146de9dd1a39f94a73cd2585e9
This commit is contained in:
Liu Yuan
parent
9daf38706f
commit
1757e7e0ae
doc/source
etc
glance
api/v1
store
tests
@ -314,7 +314,7 @@ Optional. Default: ``file``
|
||||
Can only be specified in configuration files.
|
||||
|
||||
Sets the storage backend to use by default when storing images in Glance.
|
||||
Available options for this option are (``file``, ``swift``, ``s3``, or ``rbd``).
|
||||
Available options for this option are (``file``, ``swift``, ``s3``, ``rbd``, or ``sheepdog``).
|
||||
|
||||
Configuring Glance Image Size Limit
|
||||
-----------------------------------
|
||||
@ -668,6 +668,40 @@ To set up a user named ``glance`` with minimal permissions, using a pool called
|
||||
ceph-authtool --gen-key --name client.glance --cap mon 'allow r' --cap osd 'allow rwx pool=images' /etc/glance/rbd.keyring
|
||||
ceph auth add client.glance -i /etc/glance/rbd.keyring
|
||||
|
||||
Configuring the Sheepdog Storage Backend
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
* ``sheepdog_store_address=ADDR``
|
||||
|
||||
Optional. Default: ``localhost``
|
||||
|
||||
Can only be specified in configuration files.
|
||||
|
||||
`This option is specific to the Sheepdog storage backend.`
|
||||
|
||||
Sets the IP address of the sheep daemon
|
||||
|
||||
* ``sheepdog_store_port=PORT``
|
||||
|
||||
Optional. Default: ``7000``
|
||||
|
||||
Can only be specified in configuration files.
|
||||
|
||||
`This option is specific to the Sheepdog storage backend.`
|
||||
|
||||
Sets the IP port of the sheep daemon
|
||||
|
||||
* ``sheepdog_store_chunk_size=SIZE_IN_MB``
|
||||
|
||||
Optional. Default: ``64``
|
||||
|
||||
Can only be specified in configuration files.
|
||||
|
||||
`This option is specific to the Sheepdog storage backend.`
|
||||
|
||||
Images will be chunked into objects of this size (in megabytes).
|
||||
For best performance, this should be a power of two.
|
||||
|
||||
Configuring the Image Cache
|
||||
---------------------------
|
||||
|
||||
|
||||
@ -18,6 +18,7 @@ default_store = file
|
||||
# glance.store.rbd.Store,
|
||||
# glance.store.s3.Store,
|
||||
# glance.store.swift.Store,
|
||||
# glance.store.sheepdog.Store,
|
||||
|
||||
|
||||
# Maximum image size (in bytes) that may be uploaded through the
|
||||
@ -320,6 +321,16 @@ rbd_store_pool = images
|
||||
# For best performance, this should be a power of two
|
||||
rbd_store_chunk_size = 8
|
||||
|
||||
# ============ Sheepdog Store Options =============================
|
||||
|
||||
sheepdog_store_address = localhost
|
||||
|
||||
sheepdog_store_port = 7000
|
||||
|
||||
# Images will be chunked into objects of this size (in megabytes).
|
||||
# For best performance, this should be a power of two
|
||||
sheepdog_store_chunk_size = 64
|
||||
|
||||
# ============ Delayed Delete Options =============================
|
||||
|
||||
# Turn on/off delayed delete
|
||||
|
||||
@ -49,6 +49,7 @@ registry_port = 9191
|
||||
# glance.store.rbd.Store,
|
||||
# glance.store.s3.Store,
|
||||
# glance.store.swift.Store,
|
||||
# glance.store.sheepdog.Store,
|
||||
|
||||
# ============ Filesystem Store Options ========================
|
||||
|
||||
|
||||
@ -277,7 +277,7 @@ class Controller(controller.BaseController):
|
||||
If the above constraint is violated, we reject with 400 "Bad Request".
|
||||
"""
|
||||
if source:
|
||||
for scheme in ['s3', 'swift', 'http', 'rbd']:
|
||||
for scheme in ['s3', 'swift', 'http', 'rbd', 'sheepdog']:
|
||||
if source.lower().startswith(scheme):
|
||||
return source
|
||||
msg = _("External sourcing not supported for store %s") % source
|
||||
|
||||
@ -40,6 +40,7 @@ store_opts = [
|
||||
'glance.store.rbd.Store',
|
||||
'glance.store.s3.Store',
|
||||
'glance.store.swift.Store',
|
||||
'glance.store.sheepdog.Store',
|
||||
],
|
||||
help=_('List of which store classes and store class locations '
|
||||
'are currently known to glance at startup.')),
|
||||
|
||||
301
glance/store/sheepdog.py
Normal file
301
glance/store/sheepdog.py
Normal file
@ -0,0 +1,301 @@
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2013 Taobao Inc.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Storage backend for Sheepdog storage system"""
|
||||
|
||||
import hashlib
|
||||
|
||||
from oslo.config import cfg
|
||||
|
||||
from glance.common import exception
|
||||
import glance.openstack.common.log as logging
|
||||
from glance.openstack.common import processutils
|
||||
import glance.store
|
||||
import glance.store.base
|
||||
import glance.store.location
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_ADDR = 'localhost'
|
||||
DEFAULT_PORT = '7000'
|
||||
DEFAULT_CHUNKSIZE = 64 # in MiB
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
sheepdog_opts = [
|
||||
cfg.IntOpt('sheepdog_store_chunk_size', default=DEFAULT_CHUNKSIZE,
|
||||
help=_('Images will be chunked into objects of this size '
|
||||
'(in megabytes). For best performance, this should be '
|
||||
'a power of two.')),
|
||||
cfg.StrOpt('sheepdog_store_port', default=DEFAULT_PORT,
|
||||
help=_('Port of sheep daemon.')),
|
||||
cfg.StrOpt('sheepdog_store_address', default=DEFAULT_ADDR,
|
||||
help=_('IP address of sheep daemon.'))
|
||||
]
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.register_opts(sheepdog_opts)
|
||||
|
||||
|
||||
class SheepdogImage:
|
||||
"""Class describing an image stored in Sheepdog storage."""
|
||||
|
||||
def __init__(self, addr, port, name, chunk_size):
|
||||
self.addr = addr
|
||||
self.port = port
|
||||
self.name = name
|
||||
self.chunk_size = chunk_size
|
||||
|
||||
def _run_command(self, command, data, *params):
|
||||
cmd = ("collie vdi %(command)s -a %(addr)s -p %(port)s %(name)s "
|
||||
"%(params)s" %
|
||||
{"command": command,
|
||||
"addr": self.addr,
|
||||
"port": self.port,
|
||||
"name": self.name,
|
||||
"params": " ".join(map(str, params))})
|
||||
|
||||
try:
|
||||
return processutils.execute(
|
||||
cmd, process_input=data, shell=True)[0]
|
||||
except processutils.ProcessExecutionError as exc:
|
||||
LOG.error(exc)
|
||||
raise glance.store.BackendException(exc)
|
||||
|
||||
def get_size(self):
|
||||
"""
|
||||
Return the size of the this iamge
|
||||
|
||||
Sheepdog Usage: collie vdi list -r -a address -p port image
|
||||
"""
|
||||
out = self._run_command("list -r", None)
|
||||
return long(out.split(' ')[3])
|
||||
|
||||
def read(self, offset, count):
|
||||
"""
|
||||
Read up to 'count' bytes from this image starting at 'offset' and
|
||||
return the data.
|
||||
|
||||
Sheepdog Usage: collie vdi read -a address -p port image offset len
|
||||
"""
|
||||
return self._run_command("read", None, str(offset), str(count))
|
||||
|
||||
def write(self, data, offset, count):
|
||||
"""
|
||||
Write up to 'count' bytes from the data to this image starting at
|
||||
'offset'
|
||||
|
||||
Sheepdog Usage: collie vdi write -a address -p port image offset len
|
||||
"""
|
||||
self._run_command("write", data, str(offset), str(count))
|
||||
|
||||
def create(self, size):
|
||||
"""
|
||||
Create this image in the Sheepdog cluster with size 'size'.
|
||||
|
||||
Sheepdog Usage: collie vdi create -a address -p port image size
|
||||
"""
|
||||
self._run_command("create", None, str(size))
|
||||
|
||||
def delete(self):
|
||||
"""
|
||||
Delete this image in the Sheepdog cluster
|
||||
|
||||
Sheepdog Usage: collie vdi delete -a address -p port image
|
||||
"""
|
||||
self._run_command("delete", None)
|
||||
|
||||
def exist(self):
|
||||
"""
|
||||
Check if this image exists in the Sheepdog cluster via 'list' command
|
||||
|
||||
Sheepdog Usage: collie vdi list -r -a address -p port image
|
||||
"""
|
||||
out = self._run_command("list -r", None)
|
||||
if not out:
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
|
||||
class StoreLocation(glance.store.location.StoreLocation):
|
||||
"""
|
||||
Class describing a Sheepdog URI. This is of the form:
|
||||
|
||||
sheepdog://image
|
||||
|
||||
"""
|
||||
|
||||
def process_specs(self):
|
||||
self.image = self.specs.get('image')
|
||||
|
||||
def get_uri(self):
|
||||
return "sheepdog://%s" % self.image
|
||||
|
||||
def parse_uri(self, uri):
|
||||
if not uri.startswith('sheepdog://'):
|
||||
raise exception.BadStoreUri(uri, "URI must start with %s://" %
|
||||
'sheepdog')
|
||||
self.image = uri[11:]
|
||||
|
||||
|
||||
class ImageIterator(object):
|
||||
"""
|
||||
Reads data from an Sheepdog image, one chunk at a time.
|
||||
"""
|
||||
|
||||
def __init__(self, image):
|
||||
self.image = image
|
||||
|
||||
def __iter__(self):
|
||||
image = self.image
|
||||
total = left = image.get_size()
|
||||
while left > 0:
|
||||
length = min(image.chunk_size, left)
|
||||
data = image.read(total - left, length)
|
||||
left -= len(data)
|
||||
yield data
|
||||
raise StopIteration()
|
||||
|
||||
|
||||
class Store(glance.store.base.Store):
|
||||
"""Sheepdog backend adapter."""
|
||||
|
||||
EXAMPLE_URL = "sheepdog://image"
|
||||
|
||||
def get_schemes(self):
|
||||
return ('sheepdog',)
|
||||
|
||||
def configure_add(self):
|
||||
"""
|
||||
Configure the Store to use the stored configuration options
|
||||
Any store that needs special configuration should implement
|
||||
this method. If the store was not able to successfully configure
|
||||
itself, it should raise `exception.BadStoreConfiguration`
|
||||
"""
|
||||
|
||||
try:
|
||||
self.chunk_size = CONF.sheepdog_store_chunk_size * 1024 * 1024
|
||||
self.addr = CONF.sheepdog_store_address
|
||||
self.port = CONF.sheepdog_store_port
|
||||
except cfg.ConfigFileValueError as e:
|
||||
reason = _("Error in store configuration: %s") % e
|
||||
LOG.error(reason)
|
||||
raise exception.BadStoreConfiguration(store_name='sheepdog',
|
||||
reason=reason)
|
||||
|
||||
try:
|
||||
processutils.execute("collie", shell=True)
|
||||
except processutils.ProcessExecutionError as exc:
|
||||
reason = _("Error in store configuration: %s") % exc
|
||||
LOG.error(reason)
|
||||
raise exception.BadStoreConfiguration(store_name='sheepdog',
|
||||
reason=reason)
|
||||
|
||||
def get(self, location):
|
||||
"""
|
||||
Takes a `glance.store.location.Location` object that indicates
|
||||
where to find the image file, and returns a generator for reading
|
||||
the image file
|
||||
|
||||
:param location `glance.store.location.Location` object, supplied
|
||||
from glance.store.location.get_location_from_uri()
|
||||
:raises `glance.exception.NotFound` if image does not exist
|
||||
"""
|
||||
|
||||
loc = location.store_location
|
||||
image = SheepdogImage(self.addr, self.port, loc.image,
|
||||
self.chunk_size)
|
||||
if not image.exist():
|
||||
raise exception.NotFound(_("Sheepdog image %s does not exist")
|
||||
% image.name)
|
||||
return (ImageIterator(image), image.get_size())
|
||||
|
||||
def get_size(self, location):
|
||||
"""
|
||||
Takes a `glance.store.location.Location` object that indicates
|
||||
where to find the image file and returns the image size
|
||||
|
||||
:param location `glance.store.location.Location` object, supplied
|
||||
from glance.store.location.get_location_from_uri()
|
||||
:raises `glance.exception.NotFound` if image does not exist
|
||||
:rtype int
|
||||
"""
|
||||
|
||||
loc = location.store_location
|
||||
image = SheepdogImage(self.addr, self.port, loc.image,
|
||||
self.chunk_size)
|
||||
if not image.exist():
|
||||
raise exception.NotFound(_("Sheepdog image %s does not exist")
|
||||
% image.name)
|
||||
return image.get_size()
|
||||
|
||||
def add(self, image_id, image_file, image_size):
|
||||
"""
|
||||
Stores an image file with supplied identifier to the backend
|
||||
storage system and returns a tuple containing information
|
||||
about the stored image.
|
||||
|
||||
:param image_id: The opaque image identifier
|
||||
:param image_file: The image data to write, as a file-like object
|
||||
:param image_size: The size of the image data to write, in bytes
|
||||
|
||||
:retval tuple of URL in backing store, bytes written, and checksum
|
||||
:raises `glance.common.exception.Duplicate` if the image already
|
||||
existed
|
||||
"""
|
||||
|
||||
image = SheepdogImage(self.addr, self.port, image_id,
|
||||
self.chunk_size)
|
||||
if image.exist():
|
||||
raise exception.Duplicate(_("Sheepdog image %s already exists")
|
||||
% image_id)
|
||||
|
||||
location = StoreLocation({'image': image_id})
|
||||
checksum = hashlib.md5()
|
||||
|
||||
image.create(image_size)
|
||||
|
||||
total = left = image_size
|
||||
while left > 0:
|
||||
length = min(self.chunk_size, left)
|
||||
data = image_file.read(length)
|
||||
image.write(data, total - left, length)
|
||||
left -= length
|
||||
checksum.update(data)
|
||||
|
||||
return (location.get_uri(), image_size, checksum.hexdigest())
|
||||
|
||||
def delete(self, location):
|
||||
"""
|
||||
Takes a `glance.store.location.Location` object that indicates
|
||||
where to find the image file to delete
|
||||
|
||||
:location `glance.store.location.Location` object, supplied
|
||||
from glance.store.location.get_location_from_uri()
|
||||
|
||||
:raises NotFound if image does not exist
|
||||
"""
|
||||
|
||||
loc = location.store_location
|
||||
image = SheepdogImage(self.addr, self.port, loc.image,
|
||||
self.chunk_size)
|
||||
if not image.exist():
|
||||
raise exception.NotFound(_("Sheepdog image %s does not exist") %
|
||||
loc.image)
|
||||
image.delete()
|
||||
81
glance/tests/functional/store/test_sheepdog.py
Normal file
81
glance/tests/functional/store/test_sheepdog.py
Normal file
@ -0,0 +1,81 @@
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2013 Taobao Inc.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Functional tests for the Sheepdog store interface
|
||||
"""
|
||||
|
||||
import os
|
||||
import os.path
|
||||
|
||||
import fixtures
|
||||
import oslo.config.cfg
|
||||
import testtools
|
||||
|
||||
from glance.store import BackendException
|
||||
import glance.store.sheepdog as sheepdog
|
||||
import glance.tests.functional.store as store_tests
|
||||
import glance.tests.utils
|
||||
|
||||
|
||||
class TestSheepdogStore(store_tests.BaseTestCase, testtools.TestCase):
|
||||
|
||||
store_cls_path = 'glance.store.sheepdog.Store'
|
||||
store_cls = glance.store.sheepdog.Store
|
||||
store_name = 'sheepdog'
|
||||
|
||||
def setUp(self):
|
||||
image = sheepdog.SheepdogImage(sheepdog.DEFAULT_ADDR,
|
||||
sheepdog.DEFAULT_PORT,
|
||||
"test",
|
||||
sheepdog.DEFAULT_CHUNKSIZE)
|
||||
try:
|
||||
image.create(512)
|
||||
except BackendException as e:
|
||||
msg = "Sheepdog cluster isn't set up"
|
||||
self.skipTest(msg)
|
||||
image.delete()
|
||||
|
||||
self.tmp_dir = self.useFixture(fixtures.TempDir()).path
|
||||
|
||||
config_file = os.path.join(self.tmp_dir, 'glance.conf')
|
||||
with open(config_file, 'w') as f:
|
||||
f.write("[DEFAULT]\n")
|
||||
f.write("default_store = sheepdog")
|
||||
|
||||
oslo.config.cfg.CONF(default_config_files=[config_file], args=[])
|
||||
super(TestSheepdogStore, self).setUp()
|
||||
|
||||
def get_store(self, **kwargs):
|
||||
store = sheepdog.Store(context=kwargs.get('context'))
|
||||
store.configure()
|
||||
store.configure_add()
|
||||
return store
|
||||
|
||||
def stash_image(self, image_id, image_data):
|
||||
image_size = len(image_data)
|
||||
image = sheepdog.SheepdogImage(sheepdog.DEFAULT_ADDR,
|
||||
sheepdog.DEFAULT_PORT,
|
||||
image_id,
|
||||
sheepdog.DEFAULT_CHUNKSIZE)
|
||||
image.create(image_size)
|
||||
total = left = image_size
|
||||
while left > 0:
|
||||
length = min(sheepdog.DEFAULT_CHUNKSIZE, left)
|
||||
image.write(image_data, total - left, length)
|
||||
left -= length
|
||||
|
||||
return 'sheepdog://%s' % image_id
|
||||
@ -52,6 +52,7 @@ class TestStoreLocation(base.StoreClearingUnitTest):
|
||||
'rbd://imagename',
|
||||
'rbd://fsid/pool/image/snap',
|
||||
'rbd://%2F/%2F/%2F/%2F',
|
||||
'sheepdog://imagename',
|
||||
]
|
||||
|
||||
for uri in good_store_uris:
|
||||
@ -361,6 +362,21 @@ class TestStoreLocation(base.StoreClearingUnitTest):
|
||||
bad_uri = 'rbd://' + unichr(300)
|
||||
self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri)
|
||||
|
||||
def test_sheepdog_store_location(self):
|
||||
"""
|
||||
Test the specific StoreLocation for the Sheepdog store
|
||||
"""
|
||||
uri = 'sheepdog://imagename'
|
||||
loc = glance.store.sheepdog.StoreLocation({})
|
||||
loc.parse_uri(uri)
|
||||
self.assertEqual('imagename', loc.image)
|
||||
|
||||
bad_uri = 'sheepdog:/image'
|
||||
self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri)
|
||||
|
||||
bad_uri = 'http://image'
|
||||
self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri)
|
||||
|
||||
def test_get_store_from_scheme(self):
|
||||
"""
|
||||
Test that the backend returned by glance.store.get_backend_class
|
||||
@ -377,7 +393,8 @@ class TestStoreLocation(base.StoreClearingUnitTest):
|
||||
'filesystem': glance.store.filesystem.Store,
|
||||
'http': glance.store.http.Store,
|
||||
'https': glance.store.http.Store,
|
||||
'rbd': glance.store.rbd.Store}
|
||||
'rbd': glance.store.rbd.Store,
|
||||
'sheepdog': glance.store.sheepdog.Store}
|
||||
|
||||
ctx = context.RequestContext()
|
||||
for scheme, store in good_results.items():
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user