diff --git a/doc/source/index.rst b/doc/source/index.rst index 06afb3424b..9b0ec0d41a 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -195,6 +195,122 @@ Notes: `application/octet-stream`. +.. toctree:: + :maxdepth: 1 + + +Adding a New Virtual Machine Image +---------------------------------- + +We have created a new virtual machine image in some way (created a +"golden image" or snapshotted/backed up an existing image) and we +wish to do two things: + + * Store the disk image data in Glance + * Store metadata about this image in Glance + +We can do the above two activities in a single call to the Glance API. +Assuming, like in the examples above, that a Glance API server is running +at `glance.openstack.org`, we issue a `POST` request to add an image to +Glance:: + + POST http://glance.openstack.org/images/ + +The metadata about the image is sent to Glance in HTTP headers. The body +of the HTTP request to the Glance API will be the MIME-encoded disk +image data. + + +Adding Image Metadata in HTTP Headers +************************************* + +Glance will view as image metadata any HTTP header that it receives in a +`POST` request where the header key is prefixed with the strings +`x-image-meta-` and `x-image-meta-property-`. + +The list of metadata headers that Glance accepts are listed below. + + * `x-image-meta-name` + + This header is required. Its value should be the name of the image. + + Note that the name of an image *is not unique to a Glance node*. It + would be an unrealistic expectation of users to know all the unique + names of all other user's images. + + * `x-image-meta-id` + + This header is optional. + + When present, Glance will use the supplied identifier for the image. + If the identifier already exists in that Glance node, then a + `409 Conflict` will be returned by Glance. + + When this header is *not* present, Glance will generate an identifier + for the image and return this identifier in the response (see below) + + * `x-image-meta-store` + + This header is optional. Valid values are one of `file` or `swift` + + When present, Glance will attempt to store the disk image data in the + backing store indicated by the value of the header. If the Glance node + does not support the backing store, Glance will return a `400 Bad Request`. + + When not present, Glance will store the disk image data in the backing + store that is marked default. See the configuration option `default_store` + for more information. + + * `x-image-meta-type` + + This header is required. Valid values are one of `kernel`, `machine`, `raw`, + or `ramdisk`. + + * `x-image-meta-size` + + This header is optional. + + When present, Glance assumes that the expected size of the request body + will be the value of this header. If the length in bytes of the request + body *does not match* the value of this header, Glance will return a + `400 Bad Request`. + + When not present, Glance will calculate the image's size based on the size + of the request body. + + * `x-image-meta-is_public` + + This header is optional. + + When present, Glance converts the value of the header to a boolean value, + so "on, 1, true" are all true values. When true, the image is marked as + a public image, meaning that any user may view its metadata and may read + the disk image from Glance. + + When not present, the image is assumed to be *not public* and specific to + a user. + + * `x-image-meta-property-*` + + When Glance receives any HTTP header whose key begins with the string prefix + `x-image-meta-property-`, Glance adds the key and value to a set of custom, + free-form image properties stored with the image. The key is the + lower-cased string following the prefix `x-image-meta-property-` with dashes + and punctuation replaced with underscores. + + For example, if the following HTTP header were sent:: + + x-image-meta-property-distro Ubuntu 10.10 + + Then a key/value pair of "distro"/"Ubuntu 10.10" will be stored with the + image in Glance. + + There is no limit on the number of free-form key/value attributes that can + be attached to the image. However, keep in mind that the 8K limit on the + size of all HTTP headers sent in a request will effectively limit the number + of image properties. + + .. toctree:: :maxdepth: 1 @@ -322,6 +438,129 @@ where `metadata` is a mapping of metadata about the image and `file` is a generator that yields chunks of image data. +.. toctree:: + :maxdepth: 1 + + +Adding a New Virtual Machine Image +---------------------------------- + +We have created a new virtual machine image in some way (created a +"golden image" or snapshotted/backed up an existing image) and we +wish to do two things: + + * Store the disk image data in Glance + * Store metadata about this image in Glance + +We can do the above two activities in a single call to the Glance client. +Assuming, like in the examples above, that a Glance API server is running +at `glance.openstack.org`, we issue a call to `glance.client.Client.add_image`. + +The method signature is as follows:: + + glance.client.Client.add_image(image_meta, image_data=None) + +The `image_meta` argument is a mapping containing various image metadata. The +`image_data` argument is the disk image data. + +The list of metadata that `image_meta` can contain are listed below. + + * `name` + + This key/value is required. Its value should be the name of the image. + + Note that the name of an image *is not unique to a Glance node*. It + would be an unrealistic expectation of users to know all the unique + names of all other user's images. + + * `id` + + This key/value is optional. + + When present, Glance will use the supplied identifier for the image. + If the identifier already exists in that Glance node, then a + `glance.common.exception.Duplicate` will be raised. + + When this key/value is *not* present, Glance will generate an identifier + for the image and return this identifier in the response (see below) + + * `store` + + This key/value is optional. Valid values are one of `file` or `swift` + + When present, Glance will attempt to store the disk image data in the + backing store indicated by the value. If the Glance node does not support + the backing store, Glance will raise a `glance.common.exception.BadRequest` + + When not present, Glance will store the disk image data in the backing + store that is marked default. See the configuration option `default_store` + for more information. + + * `type` + + This key/values is required. Valid values are one of `kernel`, `machine`, + `raw`, or `ramdisk`. + + * `size` + + This key/value is optional. + + When present, Glance assumes that the expected size of the request body + will be the value. If the length in bytes of the request body *does not + match* the value, Glance will raise a `glance.common.exception.BadRequest` + + When not present, Glance will calculate the image's size based on the size + of the request body. + + * `is_public` + + This key/value is optional. + + When present, Glance converts the value to a boolean value, so "on, 1, true" + are all true values. When true, the image is marked as a public image, + meaning that any user may view its metadata and may read the disk image from + Glance. + + When not present, the image is assumed to be *not public* and specific to + a user. + + * `properties` + + This key/value is optional. + + When present, the value is assumed to be a mapping of free-form key/value + attributes to store with the image. + + For example, if the following is the value of the `properties` key in the + `image_meta` argument:: + + {'distro': 'Ubuntu 10.10'} + + Then a key/value pair of "distro"/"Ubuntu 10.10" will be stored with the + image in Glance. + + There is no limit on the number of free-form key/value attributes that can + be attached to the image with `properties`. However, keep in mind that there + is a 8K limit on the size of all HTTP headers sent in a request and this + number will effectively limit the number of image properties. + +As a complete example, the following code would add a new machine image to +Glance:: + + from glance.client import Client + + c = Client("glance.openstack.org", 9292) + + meta = {'name': 'Ubuntu 10.10 5G', + 'type': 'machine', + 'is_public': True, + 'properties': {'distro': 'Ubuntu 10.10'}} + + new_meta = c.add_image(meta, open('/path/to/image.tar.gz')) + + print 'Stored image. Got identifier: %s' % new_meta['id'] + + .. toctree:: :maxdepth: 1 diff --git a/glance/client.py b/glance/client.py index 206e014dfe..679b02dc3a 100644 --- a/glance/client.py +++ b/glance/client.py @@ -46,11 +46,6 @@ class ClientConnectionError(Exception): pass -class BadInputError(Exception): - """Error resulting from a client sending bad input to a server""" - pass - - class ImageBodyIterator(object): """ @@ -150,7 +145,7 @@ class BaseClient(object): elif status_code == httplib.CONFLICT: raise exception.Duplicate elif status_code == httplib.BAD_REQUEST: - raise BadInputError + raise exception.BadInputError else: raise Exception("Unknown error occurred! %d" % status_code) @@ -235,17 +230,33 @@ class Client(BaseClient): """ Tells Glance about an image's metadata as well as optionally the image_data itself + + :param image_meta: Mapping of information about the + image + :param image_data: Optional string of raw image data + or file-like object that can be + used to read the image data + + :retval The newly-stored image's metadata. """ if not image_data and 'location' not in image_meta.keys(): raise exception.Invalid("You must either specify a location " "for the image or supply the actual " "image data when adding an image to " "Glance") - body = image_data headers = util.image_meta_to_http_headers(image_meta) + if image_data: + if hasattr(image_data, 'read'): + # TODO(jaypipes): This is far from efficient. Implement + # chunked transfer encoding if size is not in image_meta + body = image_data.read() + else: + body = image_data + headers['content-type'] = 'application/octet-stream' + else: + body = None res = self.do_request("POST", "/images", body, headers) - # Registry returns a JSONified dict(image=image_info) data = json.loads(res.read()) return data['image']['id'] diff --git a/glance/common/exception.py b/glance/common/exception.py index f00c27b34f..18e14f9d55 100644 --- a/glance/common/exception.py +++ b/glance/common/exception.py @@ -70,6 +70,11 @@ class Invalid(Error): pass +class BadInputError(Exception): + """Error resulting from a client sending bad input to a server""" + pass + + def wrap_exception(f): def _wrap(*args, **kw): try: diff --git a/glance/common/flags.py b/glance/common/flags.py index e9b2250bc2..2895af6978 100644 --- a/glance/common/flags.py +++ b/glance/common/flags.py @@ -173,3 +173,5 @@ DEFINE_string('sql_connection', 'sqlite:///%s/glance.sqlite' % os.path.abspath("./"), 'connection string for sql database') DEFINE_bool('verbose', False, 'show debug output') +DEFINE_string('default_store', 'file', + 'Default storage backend. Default: "file"') diff --git a/glance/registry/db/sqlalchemy/api.py b/glance/registry/db/sqlalchemy/api.py index 48ecc87d26..6d3fc874a6 100644 --- a/glance/registry/db/sqlalchemy/api.py +++ b/glance/registry/db/sqlalchemy/api.py @@ -52,10 +52,19 @@ def _deleted(context): def image_create(_context, values): + values['size'] = int(values['size']) + values['is_public'] = bool(values.get('is_public', False)) + properties = values.pop('properties', {}) + image_ref = models.Image() image_ref.update(values) image_ref.save() - return image_ref + + for key, value in properties.iteritems(): + prop_values = {'image_id': image_ref.id, 'key': key, 'value': value} + image_property_create(_context, prop_values) + + return image_get(_context, image_ref.id) def image_destroy(_context, image_id): @@ -102,18 +111,25 @@ def image_get_by_str(context, str_id): def image_update(_context, image_id, values): session = get_session() with session.begin(): + values['size'] = int(values['size']) + values['is_public'] = bool(values.get('is_public', False)) + properties = values.pop('properties', {}) + image_ref = models.Image.find(image_id, session=session) image_ref.update(values) image_ref.save(session=session) + for key, value in properties.iteritems(): + prop_values = {'image_id': image_ref.id, 'key': key, 'value': value} + image_property_create(_context, prop_values) + ################### def image_file_create(_context, values): image_file_ref = models.ImageFile() - for (key, value) in values.iteritems(): - image_file_ref[key] = value + image_file_ref.update(values) image_file_ref.save() return image_file_ref @@ -122,8 +138,7 @@ def image_file_create(_context, values): def image_property_create(_context, values): - image_properties_ref = models.ImageProperty() - for (key, value) in values.iteritems(): - image_properties_ref[key] = value - image_properties_ref.save() - return image_properties_ref + image_property_ref = models.ImageProperty() + image_property_ref.update(values) + image_property_ref.save() + return image_property_ref diff --git a/glance/server.py b/glance/server.py index 7310cdfa54..0296b490c7 100644 --- a/glance/server.py +++ b/glance/server.py @@ -23,6 +23,10 @@ Glance API Server Configuration Options --------------------- + `default_store`: When no x-image-meta-store header is sent for a + `POST /images` request, this store will be used + for storing the image data. Default: 'file' + """ import json @@ -39,7 +43,9 @@ from glance.common import flags from glance.common import wsgi from glance.store import (get_from_backend, delete_from_backend, - get_store_from_location) + get_store_from_location, + get_backend_class, + UnsupportedBackend) from glance import registry from glance import util @@ -174,9 +180,6 @@ class Controller(wsgi.Controller): :param request: The WSGI/Webob Request object - :see The `id_type` configuration option (default: uuid) determines - the type of identifier that Glance generates for an image - :raises HTTPBadRequest if no x-image-meta-location is missing and the request body is not application/octet-stream image data. @@ -195,7 +198,7 @@ class Controller(wsgi.Controller): "mime-encoded as application/" "octet-stream.", request=req) else: - if 'x-image-meta-store' in headers_keys: + if 'x-image-meta-store' in header_keys: image_store = req.headers['x-image-meta-store'] image_status = 'pending' # set to available when stored... image_in_body = True @@ -204,23 +207,34 @@ class Controller(wsgi.Controller): image_store = get_store_from_location(image_location) image_status = 'available' + # If image is the request body, validate that the requested + # or default store is capable of storing the image data... + if not image_store: + image_store = FLAGS.default_store + if image_in_body: + store = self.get_store_or_400(req, image_store) + image_meta = util.get_image_meta_from_headers(req) + image_meta['status'] = image_status image_meta['store'] = image_store - try: image_meta = registry.add_image_metadata(image_meta) if image_in_body: - #store = stores.get_store() - #store.add_image(req.body) + try: + location = store.add(image_meta['id'], req.body) + except exception.Duplicate, e: + return HTTPConflict(str(e), request=req) image_meta['status'] = 'available' - registries.update_image(image_meta) + image_meta['location'] = location + registry.update_image_metadata(image_meta['id'], image_meta) return dict(image=image_meta) except exception.Duplicate: - return HTTPConflict() + return HTTPConflict("An image with identifier %s already exists" + % image_meta['id'], request=req) except exception.Invalid: return HTTPBadRequest() @@ -275,6 +289,25 @@ class Controller(wsgi.Controller): request=request, content_type='text/plain') + def get_store_or_400(self, request, store_name): + """ + Grabs the storage backend for the supplied store name + or raises an HTTPBadRequest (400) response + + :param request: The WSGI/Webob Request object + :param id: The opaque image identifier + + :raises HTTPNotFound if image does not exist + """ + try: + return get_backend_class(store_name) + except UnsupportedBackend: + raise HTTPBadRequest(body='Requested store %s not available ' + 'for storage on this Glance node' + % store_name, + request=request, + content_type='text/plain') + class API(wsgi.Router): diff --git a/glance/store/__init__.py b/glance/store/__init__.py index ff4463afeb..f1922fe88f 100644 --- a/glance/store/__init__.py +++ b/glance/store/__init__.py @@ -66,7 +66,7 @@ def get_backend_class(backend): try: return BACKENDS[backend] except KeyError: - raise UnsupportedBackend("No backend found for '%s'" % scheme) + raise UnsupportedBackend("No backend found for '%s'" % backend) def get_from_backend(uri, **kwargs): diff --git a/glance/store/filesystem.py b/glance/store/filesystem.py index d5b1ae4122..2254b2c520 100644 --- a/glance/store/filesystem.py +++ b/glance/store/filesystem.py @@ -26,6 +26,11 @@ from glance.common import exception from glance.common import flags import glance.store + +flags.DEFINE_string('filesystem_store_datadir', '/var/lib/glance/images/', + 'Location to write image data. ' + 'Default: /var/lib/glance/images/') + FLAGS = flags.FLAGS @@ -94,4 +99,35 @@ class FilesystemBackend(glance.store.Backend): except OSError: raise exception.NotAuthorized("You cannot delete file %s" % fn) else: - raise exception.NotFound("Image file %s does not exist" % fn) + raise exception.NotFound("Image file %s does not exist" % fn) + + @classmethod + def add(cls, id, data): + """ + Stores image data to disk and returns a location that the image was + written to. By default, the backend writes the image data to a file + `//`, where is the value of + FLAGS.filesystem_store_datadir and is the supplied image ID. + + :param id: The opaque image identifier + :param data: The image data to write + + :retval The location that was written, with file:// scheme prepended + """ + + datadir = FLAGS.filesystem_store_datadir + + if not os.path.exists(datadir): + os.makedirs(datadir) + + filepath = os.path.join(datadir, str(id)) + + if os.path.exists(filepath): + raise exception.Duplicate("Image file %s already exists!" + % filepath) + + f = open(filepath, 'wb') + f.write(data) + f.close() + + return 'file://%s' % filepath diff --git a/tests/stubs.py b/tests/stubs.py index 2e847a8eb7..c80cb5044b 100644 --- a/tests/stubs.py +++ b/tests/stubs.py @@ -37,7 +37,7 @@ import glance.store.swift import glance.registry.db.sqlalchemy.api -FAKE_FILESYSTEM_ROOTDIR = os.path.join('//tmp', 'glance-tests') +FAKE_FILESYSTEM_ROOTDIR = os.path.join('/tmp', 'glance-tests') def stub_out_http_backend(stubs): @@ -356,6 +356,22 @@ def stub_out_registry_db_image_api(stubs): return values def image_update(self, _context, image_id, values): + + props = [] + + if 'properties' in values.keys(): + for k,v in values['properties'].iteritems(): + p = {} + p['key'] = k + p['value'] = v + p['deleted'] = False + p['created_at'] = datetime.datetime.utcnow() + p['updated_at'] = datetime.datetime.utcnow() + p['deleted_at'] = None + props.append(p) + + values['properties'] = props + image = self.image_get(_context, image_id) image.update(values) return image diff --git a/tests/unit/test_api.py b/tests/unit/test_api.py index fd11348125..34f83980d9 100644 --- a/tests/unit/test_api.py +++ b/tests/unit/test_api.py @@ -22,11 +22,14 @@ import stubout import webob from glance import server +from glance.common import flags from glance.registry import server as rserver from tests import stubs +FLAGS = flags.FLAGS -class TestImageController(unittest.TestCase): + +class TestRegistryAPI(unittest.TestCase): def setUp(self): """Establish a clean test environment""" self.stubs = stubout.StubOutForTesting() @@ -230,7 +233,71 @@ class TestImageController(unittest.TestCase): self.assertEquals(res.status_int, webob.exc.HTTPNotFound.code) + +class TestGlanceAPI(unittest.TestCase): + def setUp(self): + """Establish a clean test environment""" + self.stubs = stubout.StubOutForTesting() + stubs.stub_out_registry_and_store_server(self.stubs) + stubs.stub_out_registry_db_image_api(self.stubs) + stubs.stub_out_filesystem_backend() + self.orig_filesystem_store_datadir = FLAGS.filesystem_store_datadir + FLAGS.filesystem_store_datadir = stubs.FAKE_FILESYSTEM_ROOTDIR + + def tearDown(self): + """Clear the test environment""" + FLAGS.filesystem_store_datadir = self.orig_filesystem_store_datadir + stubs.clean_out_fake_filesystem_backend() + self.stubs.UnsetAll() + + def test_add_image_no_location_no_image_as_body(self): + """Tests raises BadRequest for no body and no loc header""" + fixture_headers = {'x-image-meta-store': 'file', + 'x-image-meta-name': 'fake image #3'} + + req = webob.Request.blank("/images") + req.method = 'POST' + for k,v in fixture_headers.iteritems(): + req.headers[k] = v + res = req.get_response(server.API()) + self.assertEquals(res.status_int, webob.exc.HTTPBadRequest.code) + + def test_add_image_bad_store(self): + """Tests raises BadRequest for invalid store header""" + fixture_headers = {'x-image-meta-store': 'bad', + 'x-image-meta-name': 'fake image #3'} + + req = webob.Request.blank("/images") + req.method = 'POST' + for k,v in fixture_headers.iteritems(): + req.headers[k] = v + + req.headers['Content-Type'] = 'application/octet-stream' + req.body = "chunk00000remainder" + res = req.get_response(server.API()) + self.assertEquals(res.status_int, webob.exc.HTTPBadRequest.code) + + def test_add_image_basic_file_store(self): + """Tests raises BadRequest for invalid store header""" + fixture_headers = {'x-image-meta-store': 'file', + 'x-image-meta-name': 'fake image #3'} + + req = webob.Request.blank("/images") + req.method = 'POST' + for k,v in fixture_headers.iteritems(): + req.headers[k] = v + + req.headers['Content-Type'] = 'application/octet-stream' + req.body = "chunk00000remainder" + res = req.get_response(server.API()) + self.assertEquals(res.status_int, 200) + + res_body = json.loads(res.body)['image'] + self.assertEquals(res_body['location'], + 'file:///tmp/glance-tests/3') + def test_image_meta(self): + """Test for HEAD /images/""" expected_headers = {'x-image-meta-id': 2, 'x-image-meta-name': 'fake image #2'} req = webob.Request.blank("/images/2") diff --git a/tests/unit/test_clients.py b/tests/unit/test_clients.py index 266191dcdc..fb12ffe42b 100644 --- a/tests/unit/test_clients.py +++ b/tests/unit/test_clients.py @@ -16,6 +16,7 @@ # under the License. import json +import os import stubout import StringIO import unittest @@ -24,9 +25,12 @@ import webob from glance import client from glance.registry import client as rclient +from glance.common import flags from glance.common import exception from tests import stubs +FLAGS = flags.FLAGS + class TestBadClients(unittest.TestCase): @@ -40,7 +44,7 @@ class TestBadClients(unittest.TestCase): 1) def test_bad_address(self): - """Test unsupported protocol raised""" + """Test ClientConnectionError raised""" c = client.Client(address="http://127.999.1.1/") self.assertRaises(client.ClientConnectionError, c.get_image, @@ -212,7 +216,7 @@ class TestRegistryClient(unittest.TestCase): 'location': "file:///tmp/glance-tests/2", } - self.assertRaises(client.BadInputError, + self.assertRaises(exception.BadInputError, self.client.add_image, fixture) @@ -279,10 +283,13 @@ class TestClient(unittest.TestCase): stubs.stub_out_registry_db_image_api(self.stubs) stubs.stub_out_registry_and_store_server(self.stubs) stubs.stub_out_filesystem_backend() + self.orig_filesystem_store_datadir = FLAGS.filesystem_store_datadir + FLAGS.filesystem_store_datadir = stubs.FAKE_FILESYSTEM_ROOTDIR self.client = client.Client() def tearDown(self): """Clear the test environment""" + FLAGS.filesystem_store_datadir = self.orig_filesystem_store_datadir stubs.clean_out_fake_filesystem_backend() self.stubs.UnsetAll() @@ -480,6 +487,85 @@ class TestClient(unittest.TestCase): self.assertEquals(data['status'], 'available') + def test_add_image_with_image_data_as_string(self): + """Tests can add image by passing image data as string""" + fixture = {'name': 'fake public image', + 'is_public': True, + 'type': 'kernel', + 'size': 19, + 'properties': {'distro': 'Ubuntu 10.04 LTS'} + } + + image_data_fixture = r"chunk0000remainder" + + new_id = self.client.add_image(fixture, image_data_fixture) + + self.assertEquals(3, new_id) + + new_meta, new_image_chunks = self.client.get_image(3) + + new_image_data = "" + for image_chunk in new_image_chunks: + new_image_data += image_chunk + + self.assertEquals(image_data_fixture, new_image_data) + for k,v in fixture.iteritems(): + self.assertEquals(v, new_meta[k]) + + def test_add_image_with_image_data_as_file(self): + """Tests can add image by passing image data as file""" + fixture = {'name': 'fake public image', + 'is_public': True, + 'type': 'kernel', + 'size': 19, + 'properties': {'distro': 'Ubuntu 10.04 LTS'} + } + + image_data_fixture = r"chunk0000remainder" + + tmp_image_filepath = '/tmp/rubbish-image' + + if os.path.exists(tmp_image_filepath): + os.unlink(tmp_image_filepath) + + tmp_file = open(tmp_image_filepath, 'wb') + tmp_file.write(image_data_fixture) + tmp_file.close() + + new_id = self.client.add_image(fixture, open(tmp_image_filepath)) + + self.assertEquals(3, new_id) + + if os.path.exists(tmp_image_filepath): + os.unlink(tmp_image_filepath) + + new_meta, new_image_chunks = self.client.get_image(3) + + new_image_data = "" + for image_chunk in new_image_chunks: + new_image_data += image_chunk + + self.assertEquals(image_data_fixture, new_image_data) + for k,v in fixture.iteritems(): + self.assertEquals(v, new_meta[k]) + + def test_add_image_with_bad_store(self): + """Tests BadRequest raised when supplying bad store name in meta""" + fixture = {'name': 'fake public image', + 'is_public': True, + 'type': 'kernel', + 'size': 19, + 'store': 'bad', + 'properties': {'distro': 'Ubuntu 10.04 LTS'} + } + + image_data_fixture = r"chunk0000remainder" + + self.assertRaises(exception.BadInputError, + self.client.add_image, + fixture, + image_data_fixture) + def test_update_image(self): """Tests that the /images PUT registry API updates the image""" fixture = {'name': 'fake public image #2',