remove twisted objectstore
This commit is contained in:
termie
@@ -1,180 +0,0 @@
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2010 United States Government as represented by the
|
||||
# Administrator of the National Aeronautics and Space Administration.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Simple object store using Blobs and JSON files on disk.
|
||||
"""
|
||||
|
||||
import bisect
|
||||
import datetime
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
|
||||
from nova import exception
|
||||
from nova import flags
|
||||
from nova import utils
|
||||
from nova.objectstore import stored
|
||||
|
||||
|
||||
FLAGS = flags.FLAGS
|
||||
flags.DECLARE('buckets_path', 'nova.objectstore.s3server')
|
||||
|
||||
|
||||
class Bucket(object):
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
self.path = os.path.abspath(os.path.join(FLAGS.buckets_path, name))
|
||||
if not self.path.startswith(os.path.abspath(FLAGS.buckets_path)) or \
|
||||
not os.path.isdir(self.path):
|
||||
raise exception.NotFound()
|
||||
|
||||
self.ctime = os.path.getctime(self.path)
|
||||
|
||||
def __repr__(self):
|
||||
return "<Bucket: %s>" % self.name
|
||||
|
||||
@staticmethod
|
||||
def all():
|
||||
""" list of all buckets """
|
||||
buckets = []
|
||||
for fn in glob.glob("%s/*.json" % FLAGS.buckets_path):
|
||||
try:
|
||||
json.load(open(fn))
|
||||
name = os.path.split(fn)[-1][:-5]
|
||||
buckets.append(Bucket(name))
|
||||
except:
|
||||
pass
|
||||
|
||||
return buckets
|
||||
|
||||
@staticmethod
|
||||
def create(bucket_name, context):
|
||||
"""Create a new bucket owned by a project.
|
||||
|
||||
@bucket_name: a string representing the name of the bucket to create
|
||||
@context: a nova.auth.api.ApiContext object representing who owns the
|
||||
bucket.
|
||||
|
||||
Raises:
|
||||
NotAuthorized: if the bucket is already exists or has invalid name
|
||||
"""
|
||||
path = os.path.abspath(os.path.join(
|
||||
FLAGS.buckets_path, bucket_name))
|
||||
if not path.startswith(os.path.abspath(FLAGS.buckets_path)) or \
|
||||
os.path.exists(path):
|
||||
raise exception.NotAuthorized()
|
||||
|
||||
os.makedirs(path)
|
||||
|
||||
with open(path + '.json', 'w') as f:
|
||||
json.dump({'ownerId': context.project_id}, f)
|
||||
|
||||
@property
|
||||
def metadata(self):
|
||||
""" dictionary of metadata around bucket,
|
||||
keys are 'Name' and 'CreationDate'
|
||||
"""
|
||||
|
||||
return {
|
||||
"Name": self.name,
|
||||
"CreationDate": datetime.datetime.utcfromtimestamp(self.ctime),
|
||||
}
|
||||
|
||||
@property
|
||||
def owner_id(self):
|
||||
try:
|
||||
with open(self.path + '.json') as f:
|
||||
return json.load(f)['ownerId']
|
||||
except:
|
||||
return None
|
||||
|
||||
def is_authorized(self, context):
|
||||
try:
|
||||
return context.is_admin or \
|
||||
self.owner_id == context.project_id
|
||||
except Exception, e:
|
||||
return False
|
||||
|
||||
def list_keys(self, prefix='', marker=None, max_keys=1000, terse=False):
|
||||
object_names = []
|
||||
path_length = len(self.path)
|
||||
for root, dirs, files in os.walk(self.path):
|
||||
for file_name in files:
|
||||
object_name = os.path.join(root, file_name)[path_length + 1:]
|
||||
object_names.append(object_name)
|
||||
object_names.sort()
|
||||
contents = []
|
||||
|
||||
start_pos = 0
|
||||
if marker:
|
||||
start_pos = bisect.bisect_right(object_names, marker, start_pos)
|
||||
if prefix:
|
||||
start_pos = bisect.bisect_left(object_names, prefix, start_pos)
|
||||
|
||||
truncated = False
|
||||
for object_name in object_names[start_pos:]:
|
||||
if not object_name.startswith(prefix):
|
||||
break
|
||||
if len(contents) >= max_keys:
|
||||
truncated = True
|
||||
break
|
||||
object_path = self._object_path(object_name)
|
||||
c = {"Key": object_name}
|
||||
if not terse:
|
||||
info = os.stat(object_path)
|
||||
c.update({
|
||||
"LastModified": datetime.datetime.utcfromtimestamp(
|
||||
info.st_mtime),
|
||||
"Size": info.st_size,
|
||||
})
|
||||
contents.append(c)
|
||||
marker = object_name
|
||||
|
||||
return {
|
||||
"Name": self.name,
|
||||
"Prefix": prefix,
|
||||
"Marker": marker,
|
||||
"MaxKeys": max_keys,
|
||||
"IsTruncated": truncated,
|
||||
"Contents": contents,
|
||||
}
|
||||
|
||||
def _object_path(self, object_name):
|
||||
fn = os.path.join(self.path, object_name)
|
||||
|
||||
if not fn.startswith(self.path):
|
||||
raise exception.NotAuthorized()
|
||||
|
||||
return fn
|
||||
|
||||
def delete(self):
|
||||
if len(os.listdir(self.path)) > 0:
|
||||
raise exception.NotEmpty()
|
||||
os.rmdir(self.path)
|
||||
os.remove(self.path + '.json')
|
||||
|
||||
def __getitem__(self, key):
|
||||
return stored.Object(self, key)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
with open(self._object_path(key), 'wb') as f:
|
||||
f.write(value)
|
||||
|
||||
def __delitem__(self, key):
|
||||
stored.Object(self, key).delete()
|
||||
@@ -1,478 +0,0 @@
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
#
|
||||
# Copyright 2010 OpenStack LLC.
|
||||
# Copyright 2010 United States Government as represented by the
|
||||
# Administrator of the National Aeronautics and Space Administration.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Copyright 2009 Facebook
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Implementation of an S3-like storage server based on local files.
|
||||
|
||||
Useful to test features that will eventually run on S3, or if you want to
|
||||
run something locally that was once running on S3.
|
||||
|
||||
We don't support all the features of S3, but it does work with the
|
||||
standard S3 client for the most basic semantics. To use the standard
|
||||
S3 client with this module::
|
||||
|
||||
c = S3.AWSAuthConnection("", "", server="localhost", port=8888,
|
||||
is_secure=False)
|
||||
c.create_bucket("mybucket")
|
||||
c.put("mybucket", "mykey", "a value")
|
||||
print c.get("mybucket", "mykey").body
|
||||
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import json
|
||||
import multiprocessing
|
||||
import os
|
||||
import urllib
|
||||
|
||||
from twisted.application import internet
|
||||
from twisted.application import service
|
||||
from twisted.web import error
|
||||
from twisted.web import resource
|
||||
from twisted.web import server
|
||||
from twisted.web import static
|
||||
|
||||
from nova import context
|
||||
from nova import exception
|
||||
from nova import flags
|
||||
from nova import log as logging
|
||||
from nova import utils
|
||||
from nova.auth import manager
|
||||
from nova.objectstore import bucket
|
||||
from nova.objectstore import image
|
||||
|
||||
|
||||
LOG = logging.getLogger('nova.objectstore.handler')
|
||||
FLAGS = flags.FLAGS
|
||||
flags.DEFINE_string('s3_listen_host', '', 'Host to listen on.')
|
||||
|
||||
|
||||
def render_xml(request, value):
|
||||
"""Writes value as XML string to request"""
|
||||
assert isinstance(value, dict) and len(value) == 1
|
||||
request.setHeader("Content-Type", "application/xml; charset=UTF-8")
|
||||
|
||||
name = value.keys()[0]
|
||||
request.write('<?xml version="1.0" encoding="UTF-8"?>\n')
|
||||
request.write('<' + utils.utf8(name) +
|
||||
' xmlns="http://doc.s3.amazonaws.com/2006-03-01">')
|
||||
_render_parts(value.values()[0], request.write)
|
||||
request.write('</' + utils.utf8(name) + '>')
|
||||
request.finish()
|
||||
|
||||
|
||||
def finish(request, content=None):
|
||||
"""Finalizer method for request"""
|
||||
if content:
|
||||
request.write(content)
|
||||
request.finish()
|
||||
|
||||
|
||||
def _render_parts(value, write_cb):
|
||||
"""Helper method to render different Python objects to XML"""
|
||||
if isinstance(value, basestring):
|
||||
write_cb(utils.xhtml_escape(value))
|
||||
elif isinstance(value, int) or isinstance(value, long):
|
||||
write_cb(str(value))
|
||||
elif isinstance(value, datetime.datetime):
|
||||
write_cb(value.strftime("%Y-%m-%dT%H:%M:%S.000Z"))
|
||||
elif isinstance(value, dict):
|
||||
for name, subvalue in value.iteritems():
|
||||
if not isinstance(subvalue, list):
|
||||
subvalue = [subvalue]
|
||||
for subsubvalue in subvalue:
|
||||
write_cb('<' + utils.utf8(name) + '>')
|
||||
_render_parts(subsubvalue, write_cb)
|
||||
write_cb('</' + utils.utf8(name) + '>')
|
||||
else:
|
||||
raise Exception(_("Unknown S3 value type %r"), value)
|
||||
|
||||
|
||||
def get_argument(request, key, default_value):
|
||||
"""Returns the request's value at key, or default_value
|
||||
if not found
|
||||
"""
|
||||
if key in request.args:
|
||||
return request.args[key][0]
|
||||
return default_value
|
||||
|
||||
|
||||
def get_context(request):
|
||||
"""Returns the supplied request's context object"""
|
||||
try:
|
||||
# Authorization Header format: 'AWS <access>:<secret>'
|
||||
authorization_header = request.getHeader('Authorization')
|
||||
if not authorization_header:
|
||||
raise exception.NotAuthorized()
|
||||
auth_header_value = authorization_header.split(' ')[1]
|
||||
access, _ignored, secret = auth_header_value.rpartition(':')
|
||||
am = manager.AuthManager()
|
||||
(user, project) = am.authenticate(access,
|
||||
secret,
|
||||
{},
|
||||
request.method,
|
||||
request.getRequestHostname(),
|
||||
request.uri,
|
||||
headers=request.getAllHeaders(),
|
||||
check_type='s3')
|
||||
rv = context.RequestContext(user, project)
|
||||
LOG.audit(_("Authenticated request"), context=rv)
|
||||
return rv
|
||||
except exception.Error as ex:
|
||||
LOG.debug(_("Authentication Failure: %s"), ex)
|
||||
raise exception.NotAuthorized()
|
||||
|
||||
|
||||
class ErrorHandlingResource(resource.Resource):
|
||||
"""Maps exceptions to 404 / 401 codes. Won't work for
|
||||
exceptions thrown after NOT_DONE_YET is returned.
|
||||
"""
|
||||
# TODO(unassigned) (calling-all-twisted-experts): This needs to be
|
||||
# plugged in to the right place in twisted...
|
||||
# This doesn't look like it's the right place
|
||||
# (consider exceptions in getChild; or after
|
||||
# NOT_DONE_YET is returned
|
||||
def render(self, request):
|
||||
"""Renders the response as XML"""
|
||||
try:
|
||||
return resource.Resource.render(self, request)
|
||||
except exception.NotFound:
|
||||
request.setResponseCode(404)
|
||||
return ''
|
||||
except exception.NotAuthorized:
|
||||
request.setResponseCode(403)
|
||||
return ''
|
||||
|
||||
|
||||
class S3(ErrorHandlingResource):
|
||||
"""Implementation of an S3-like storage server based on local files."""
|
||||
def __init__(self):
|
||||
ErrorHandlingResource.__init__(self)
|
||||
|
||||
def getChild(self, name, request): # pylint: disable=C0103
|
||||
"""Returns either the image or bucket resource"""
|
||||
request.context = get_context(request)
|
||||
if name == '':
|
||||
return self
|
||||
elif name == '_images':
|
||||
return ImagesResource()
|
||||
else:
|
||||
return BucketResource(name)
|
||||
|
||||
def render_GET(self, request): # pylint: disable=R0201
|
||||
"""Renders the GET request for a list of buckets as XML"""
|
||||
LOG.debug(_('List of buckets requested'), context=request.context)
|
||||
buckets = [b for b in bucket.Bucket.all()
|
||||
if b.is_authorized(request.context)]
|
||||
|
||||
render_xml(request, {"ListAllMyBucketsResult": {
|
||||
"Buckets": {"Bucket": [b.metadata for b in buckets]},
|
||||
}})
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
|
||||
class BucketResource(ErrorHandlingResource):
|
||||
"""A web resource containing an S3-like bucket"""
|
||||
def __init__(self, name):
|
||||
resource.Resource.__init__(self)
|
||||
self.name = name
|
||||
|
||||
def getChild(self, name, request):
|
||||
"""Returns the bucket resource itself, or the object resource
|
||||
the bucket contains if a name is supplied
|
||||
"""
|
||||
if name == '':
|
||||
return self
|
||||
else:
|
||||
return ObjectResource(bucket.Bucket(self.name), name)
|
||||
|
||||
def render_GET(self, request):
|
||||
"Returns the keys for the bucket resource"""
|
||||
LOG.debug(_("List keys for bucket %s"), self.name)
|
||||
|
||||
try:
|
||||
bucket_object = bucket.Bucket(self.name)
|
||||
except exception.NotFound:
|
||||
return error.NoResource(message="No such bucket").render(request)
|
||||
|
||||
if not bucket_object.is_authorized(request.context):
|
||||
LOG.audit(_("Unauthorized attempt to access bucket %s"),
|
||||
self.name, context=request.context)
|
||||
raise exception.NotAuthorized()
|
||||
|
||||
prefix = get_argument(request, "prefix", u"")
|
||||
marker = get_argument(request, "marker", u"")
|
||||
max_keys = int(get_argument(request, "max-keys", 1000))
|
||||
terse = int(get_argument(request, "terse", 0))
|
||||
|
||||
results = bucket_object.list_keys(prefix=prefix,
|
||||
marker=marker,
|
||||
max_keys=max_keys,
|
||||
terse=terse)
|
||||
render_xml(request, {"ListBucketResult": results})
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
def render_PUT(self, request):
|
||||
"Creates the bucket resource"""
|
||||
LOG.debug(_("Creating bucket %s"), self.name)
|
||||
LOG.debug("calling bucket.Bucket.create(%r, %r)",
|
||||
self.name,
|
||||
request.context)
|
||||
bucket.Bucket.create(self.name, request.context)
|
||||
request.finish()
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
def render_DELETE(self, request):
|
||||
"""Deletes the bucket resource"""
|
||||
LOG.debug(_("Deleting bucket %s"), self.name)
|
||||
bucket_object = bucket.Bucket(self.name)
|
||||
|
||||
if not bucket_object.is_authorized(request.context):
|
||||
LOG.audit(_("Unauthorized attempt to delete bucket %s"),
|
||||
self.name, context=request.context)
|
||||
raise exception.NotAuthorized()
|
||||
|
||||
bucket_object.delete()
|
||||
request.setResponseCode(204)
|
||||
return ''
|
||||
|
||||
|
||||
class ObjectResource(ErrorHandlingResource):
|
||||
"""The resource returned from a bucket"""
|
||||
def __init__(self, bucket, name):
|
||||
resource.Resource.__init__(self)
|
||||
self.bucket = bucket
|
||||
self.name = name
|
||||
|
||||
def render_GET(self, request):
|
||||
"""Returns the object
|
||||
|
||||
Raises NotAuthorized if user in request context is not
|
||||
authorized to delete the object.
|
||||
"""
|
||||
bname = self.bucket.name
|
||||
nm = self.name
|
||||
LOG.debug(_("Getting object: %(bname)s / %(nm)s") % locals())
|
||||
|
||||
if not self.bucket.is_authorized(request.context):
|
||||
LOG.audit(_("Unauthorized attempt to get object %(nm)s"
|
||||
" from bucket %(bname)s") % locals(),
|
||||
context=request.context)
|
||||
raise exception.NotAuthorized()
|
||||
|
||||
obj = self.bucket[urllib.unquote(self.name)]
|
||||
request.setHeader("Content-Type", "application/unknown")
|
||||
request.setHeader("Last-Modified",
|
||||
datetime.datetime.utcfromtimestamp(obj.mtime))
|
||||
request.setHeader("Etag", '"' + obj.md5 + '"')
|
||||
return static.File(obj.path).render_GET(request)
|
||||
|
||||
def render_PUT(self, request):
|
||||
"""Modifies/inserts the object and returns a result code
|
||||
|
||||
Raises NotAuthorized if user in request context is not
|
||||
authorized to delete the object.
|
||||
"""
|
||||
nm = self.name
|
||||
bname = self.bucket.name
|
||||
LOG.debug(_("Putting object: %(bname)s / %(nm)s") % locals())
|
||||
|
||||
if not self.bucket.is_authorized(request.context):
|
||||
LOG.audit(_("Unauthorized attempt to upload object %(nm)s to"
|
||||
" bucket %(bname)s") % locals(), context=request.context)
|
||||
raise exception.NotAuthorized()
|
||||
|
||||
key = urllib.unquote(self.name)
|
||||
request.content.seek(0, 0)
|
||||
self.bucket[key] = request.content.read()
|
||||
request.setHeader("Etag", '"' + self.bucket[key].md5 + '"')
|
||||
finish(request)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
def render_DELETE(self, request):
|
||||
"""Deletes the object and returns a result code
|
||||
|
||||
Raises NotAuthorized if user in request context is not
|
||||
authorized to delete the object.
|
||||
"""
|
||||
nm = self.name
|
||||
bname = self.bucket.name
|
||||
LOG.debug(_("Deleting object: %(bname)s / %(nm)s") % locals(),
|
||||
context=request.context)
|
||||
|
||||
if not self.bucket.is_authorized(request.context):
|
||||
LOG.audit(_("Unauthorized attempt to delete object %(nm)s from "
|
||||
"bucket %(bname)s") % locals(), context=request.context)
|
||||
raise exception.NotAuthorized()
|
||||
|
||||
del self.bucket[urllib.unquote(self.name)]
|
||||
request.setResponseCode(204)
|
||||
return ''
|
||||
|
||||
|
||||
class ImageResource(ErrorHandlingResource):
|
||||
"""A web resource representing a single image"""
|
||||
isLeaf = True
|
||||
|
||||
def __init__(self, name):
|
||||
resource.Resource.__init__(self)
|
||||
self.img = image.Image(name)
|
||||
|
||||
def render_GET(self, request):
|
||||
"""Returns the image file"""
|
||||
if not self.img.is_authorized(request.context, True):
|
||||
raise exception.NotAuthorized()
|
||||
return static.File(self.img.image_path,
|
||||
defaultType='application/octet-stream').\
|
||||
render_GET(request)
|
||||
|
||||
|
||||
class ImagesResource(resource.Resource):
|
||||
"""A web resource representing a list of images"""
|
||||
|
||||
def getChild(self, name, _request):
|
||||
"""Returns itself or an ImageResource if no name given"""
|
||||
if name == '':
|
||||
return self
|
||||
else:
|
||||
return ImageResource(name)
|
||||
|
||||
def render_GET(self, request): # pylint: disable=R0201
|
||||
""" returns a json listing of all images
|
||||
that a user has permissions to see """
|
||||
|
||||
images = [i for i in image.Image.all() \
|
||||
if i.is_authorized(request.context, readonly=True)]
|
||||
|
||||
# Bug #617776:
|
||||
# We used to have 'type' in the image metadata, but this field
|
||||
# should be called 'imageType', as per the EC2 specification.
|
||||
# For compat with old metadata files we copy type to imageType if
|
||||
# imageType is not present.
|
||||
# For compat with euca2ools (and any other clients using the
|
||||
# incorrect name) we copy imageType to type.
|
||||
# imageType is primary if we end up with both in the metadata file
|
||||
# (which should never happen).
|
||||
def decorate(m):
|
||||
if 'imageType' not in m and 'type' in m:
|
||||
m[u'imageType'] = m['type']
|
||||
elif 'imageType' in m:
|
||||
m[u'type'] = m['imageType']
|
||||
if 'displayName' not in m:
|
||||
m[u'displayName'] = u''
|
||||
return m
|
||||
|
||||
request.write(json.dumps([decorate(i.metadata) for i in images]))
|
||||
request.finish()
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
def render_PUT(self, request): # pylint: disable=R0201
|
||||
""" create a new registered image """
|
||||
|
||||
image_id = get_argument(request, 'image_id', u'')
|
||||
image_location = get_argument(request, 'image_location', u'')
|
||||
|
||||
image_path = os.path.join(FLAGS.images_path, image_id)
|
||||
if ((not image_path.startswith(FLAGS.images_path)) or
|
||||
os.path.exists(image_path)):
|
||||
LOG.audit(_("Not authorized to upload image: invalid directory "
|
||||
"%s"),
|
||||
image_path, context=request.context)
|
||||
raise exception.NotAuthorized()
|
||||
|
||||
bucket_object = bucket.Bucket(image_location.split("/")[0])
|
||||
|
||||
if not bucket_object.is_authorized(request.context):
|
||||
LOG.audit(_("Not authorized to upload image: unauthorized "
|
||||
"bucket %s"), bucket_object.name,
|
||||
context=request.context)
|
||||
raise exception.NotAuthorized()
|
||||
|
||||
LOG.audit(_("Starting image upload: %s"), image_id,
|
||||
context=request.context)
|
||||
p = multiprocessing.Process(target=image.Image.register_aws_image,
|
||||
args=(image_id, image_location, request.context))
|
||||
p.start()
|
||||
return ''
|
||||
|
||||
def render_POST(self, request): # pylint: disable=R0201
|
||||
"""Update image attributes: public/private"""
|
||||
|
||||
# image_id required for all requests
|
||||
image_id = get_argument(request, 'image_id', u'')
|
||||
image_object = image.Image(image_id)
|
||||
if not image_object.is_authorized(request.context):
|
||||
LOG.audit(_("Not authorized to update attributes of image %s"),
|
||||
image_id, context=request.context)
|
||||
raise exception.NotAuthorized()
|
||||
|
||||
operation = get_argument(request, 'operation', u'')
|
||||
if operation:
|
||||
# operation implies publicity toggle
|
||||
newstatus = (operation == 'add')
|
||||
LOG.audit(_("Toggling publicity flag of image %(image_id)s"
|
||||
" %(newstatus)r") % locals(), context=request.context)
|
||||
image_object.set_public(newstatus)
|
||||
else:
|
||||
# other attributes imply update
|
||||
LOG.audit(_("Updating user fields on image %s"), image_id,
|
||||
context=request.context)
|
||||
clean_args = {}
|
||||
for arg in request.args.keys():
|
||||
clean_args[arg] = request.args[arg][0]
|
||||
image_object.update_user_editable_fields(clean_args)
|
||||
return ''
|
||||
|
||||
def render_DELETE(self, request): # pylint: disable=R0201
|
||||
"""Delete a registered image"""
|
||||
image_id = get_argument(request, "image_id", u"")
|
||||
image_object = image.Image(image_id)
|
||||
|
||||
if not image_object.is_authorized(request.context):
|
||||
LOG.audit(_("Unauthorized attempt to delete image %s"),
|
||||
image_id, context=request.context)
|
||||
raise exception.NotAuthorized()
|
||||
|
||||
image_object.delete()
|
||||
LOG.audit(_("Deleted image: %s"), image_id, context=request.context)
|
||||
|
||||
request.setResponseCode(204)
|
||||
return ''
|
||||
|
||||
|
||||
def get_site():
|
||||
"""Support for WSGI-like interfaces"""
|
||||
root = S3()
|
||||
site = server.Site(root)
|
||||
return site
|
||||
|
||||
|
||||
def get_application():
|
||||
"""Support WSGI-like interfaces"""
|
||||
factory = get_site()
|
||||
application = service.Application("objectstore")
|
||||
# Disabled because of lack of proper introspection in Twisted
|
||||
# or possibly different versions of twisted?
|
||||
# pylint: disable=E1101
|
||||
objectStoreService = internet.TCPServer(FLAGS.s3_port, factory,
|
||||
interface=FLAGS.s3_listen_host)
|
||||
objectStoreService.setServiceParent(application)
|
||||
return application
|
||||
@@ -1,296 +0,0 @@
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2010 United States Government as represented by the
|
||||
# Administrator of the National Aeronautics and Space Administration.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Take uploaded bucket contents and register them as disk images (AMIs).
|
||||
Requires decryption using keys in the manifest.
|
||||
"""
|
||||
|
||||
|
||||
import binascii
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import tarfile
|
||||
from xml.etree import ElementTree
|
||||
|
||||
from nova import exception
|
||||
from nova import flags
|
||||
from nova import utils
|
||||
from nova.objectstore import bucket
|
||||
|
||||
|
||||
FLAGS = flags.FLAGS
|
||||
flags.DECLARE('images_path', 'nova.image.local')
|
||||
|
||||
|
||||
class Image(object):
|
||||
def __init__(self, image_id):
|
||||
self.image_id = image_id
|
||||
self.path = os.path.abspath(os.path.join(FLAGS.images_path, image_id))
|
||||
if not self.path.startswith(os.path.abspath(FLAGS.images_path)) or \
|
||||
not os.path.isdir(self.path):
|
||||
raise exception.NotFound
|
||||
|
||||
@property
|
||||
def image_path(self):
|
||||
return os.path.join(self.path, 'image')
|
||||
|
||||
def delete(self):
|
||||
for fn in ['info.json', 'image']:
|
||||
try:
|
||||
os.unlink(os.path.join(self.path, fn))
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
os.rmdir(self.path)
|
||||
except:
|
||||
pass
|
||||
|
||||
def is_authorized(self, context, readonly=False):
|
||||
# NOTE(devcamcar): Public images can be read by anyone,
|
||||
# but only modified by admin or owner.
|
||||
try:
|
||||
return (self.metadata['isPublic'] and readonly) or \
|
||||
context.is_admin or \
|
||||
self.metadata['imageOwnerId'] == context.project_id
|
||||
except:
|
||||
return False
|
||||
|
||||
def set_public(self, state):
|
||||
md = self.metadata
|
||||
md['isPublic'] = state
|
||||
with open(os.path.join(self.path, 'info.json'), 'w') as f:
|
||||
json.dump(md, f)
|
||||
|
||||
def update_user_editable_fields(self, args):
|
||||
"""args is from the request parameters, so requires extra cleaning"""
|
||||
fields = {'display_name': 'displayName', 'description': 'description'}
|
||||
info = self.metadata
|
||||
for field in fields.keys():
|
||||
if field in args:
|
||||
info[fields[field]] = args[field]
|
||||
with open(os.path.join(self.path, 'info.json'), 'w') as f:
|
||||
json.dump(info, f)
|
||||
|
||||
@staticmethod
|
||||
def all():
|
||||
images = []
|
||||
for fn in glob.glob("%s/*/info.json" % FLAGS.images_path):
|
||||
try:
|
||||
image_id = fn.split('/')[-2]
|
||||
images.append(Image(image_id))
|
||||
except:
|
||||
pass
|
||||
return images
|
||||
|
||||
@property
|
||||
def owner_id(self):
|
||||
return self.metadata['imageOwnerId']
|
||||
|
||||
@property
|
||||
def metadata(self):
|
||||
with open(os.path.join(self.path, 'info.json')) as f:
|
||||
return json.load(f)
|
||||
|
||||
@staticmethod
|
||||
def add(src, description, kernel=None, ramdisk=None, public=True):
|
||||
"""adds an image to imagestore
|
||||
|
||||
@type src: str
|
||||
@param src: location of the partition image on disk
|
||||
|
||||
@type description: str
|
||||
@param description: string describing the image contents
|
||||
|
||||
@type kernel: bool or str
|
||||
@param kernel: either TRUE meaning this partition is a kernel image or
|
||||
a string of the image id for the kernel
|
||||
|
||||
@type ramdisk: bool or str
|
||||
@param ramdisk: either TRUE meaning this partition is a ramdisk image
|
||||
or a string of the image id for the ramdisk
|
||||
|
||||
|
||||
@type public: bool
|
||||
@param public: determine if this is a public image or private
|
||||
|
||||
@rtype: str
|
||||
@return: a string with the image id
|
||||
"""
|
||||
|
||||
image_type = 'machine'
|
||||
image_id = utils.generate_uid('ami')
|
||||
|
||||
if kernel is True:
|
||||
image_type = 'kernel'
|
||||
image_id = utils.generate_uid('aki')
|
||||
if ramdisk is True:
|
||||
image_type = 'ramdisk'
|
||||
image_id = utils.generate_uid('ari')
|
||||
|
||||
image_path = os.path.join(FLAGS.images_path, image_id)
|
||||
os.makedirs(image_path)
|
||||
|
||||
shutil.copyfile(src, os.path.join(image_path, 'image'))
|
||||
|
||||
info = {
|
||||
'imageId': image_id,
|
||||
'imageLocation': description,
|
||||
'imageOwnerId': 'system',
|
||||
'isPublic': public,
|
||||
'architecture': 'x86_64',
|
||||
'imageType': image_type,
|
||||
'state': 'available'}
|
||||
|
||||
if type(kernel) is str and len(kernel) > 0:
|
||||
info['kernelId'] = kernel
|
||||
|
||||
if type(ramdisk) is str and len(ramdisk) > 0:
|
||||
info['ramdiskId'] = ramdisk
|
||||
|
||||
with open(os.path.join(image_path, 'info.json'), "w") as f:
|
||||
json.dump(info, f)
|
||||
|
||||
return image_id
|
||||
|
||||
@staticmethod
|
||||
def register_aws_image(image_id, image_location, context):
|
||||
image_path = os.path.join(FLAGS.images_path, image_id)
|
||||
os.makedirs(image_path)
|
||||
|
||||
bucket_name = image_location.split("/")[0]
|
||||
manifest_path = image_location[len(bucket_name) + 1:]
|
||||
bucket_object = bucket.Bucket(bucket_name)
|
||||
|
||||
manifest = ElementTree.fromstring(bucket_object[manifest_path].read())
|
||||
image_type = 'machine'
|
||||
|
||||
try:
|
||||
kernel_id = manifest.find("machine_configuration/kernel_id").text
|
||||
if kernel_id == 'true':
|
||||
image_type = 'kernel'
|
||||
except:
|
||||
kernel_id = None
|
||||
|
||||
try:
|
||||
ramdisk_id = manifest.find("machine_configuration/ramdisk_id").text
|
||||
if ramdisk_id == 'true':
|
||||
image_type = 'ramdisk'
|
||||
except:
|
||||
ramdisk_id = None
|
||||
|
||||
try:
|
||||
arch = manifest.find("machine_configuration/architecture").text
|
||||
except:
|
||||
arch = 'x86_64'
|
||||
|
||||
info = {
|
||||
'imageId': image_id,
|
||||
'imageLocation': image_location,
|
||||
'imageOwnerId': context.project_id,
|
||||
'isPublic': False, # FIXME: grab public from manifest
|
||||
'architecture': arch,
|
||||
'imageType': image_type}
|
||||
|
||||
if kernel_id:
|
||||
info['kernelId'] = kernel_id
|
||||
|
||||
if ramdisk_id:
|
||||
info['ramdiskId'] = ramdisk_id
|
||||
|
||||
def write_state(state):
|
||||
info['imageState'] = state
|
||||
with open(os.path.join(image_path, 'info.json'), "w") as f:
|
||||
json.dump(info, f)
|
||||
|
||||
write_state('pending')
|
||||
|
||||
encrypted_filename = os.path.join(image_path, 'image.encrypted')
|
||||
with open(encrypted_filename, 'w') as f:
|
||||
for filename in manifest.find("image").getiterator("filename"):
|
||||
shutil.copyfileobj(bucket_object[filename.text].file, f)
|
||||
|
||||
write_state('decrypting')
|
||||
|
||||
# FIXME: grab kernelId and ramdiskId from bundle manifest
|
||||
hex_key = manifest.find("image/ec2_encrypted_key").text
|
||||
encrypted_key = binascii.a2b_hex(hex_key)
|
||||
hex_iv = manifest.find("image/ec2_encrypted_iv").text
|
||||
encrypted_iv = binascii.a2b_hex(hex_iv)
|
||||
cloud_private_key = os.path.join(FLAGS.ca_path, "private/cakey.pem")
|
||||
|
||||
decrypted_filename = os.path.join(image_path, 'image.tar.gz')
|
||||
Image.decrypt_image(encrypted_filename, encrypted_key, encrypted_iv,
|
||||
cloud_private_key, decrypted_filename)
|
||||
|
||||
write_state('untarring')
|
||||
|
||||
image_file = Image.untarzip_image(image_path, decrypted_filename)
|
||||
shutil.move(os.path.join(image_path, image_file),
|
||||
os.path.join(image_path, 'image'))
|
||||
|
||||
write_state('available')
|
||||
os.unlink(decrypted_filename)
|
||||
os.unlink(encrypted_filename)
|
||||
|
||||
@staticmethod
|
||||
def decrypt_image(encrypted_filename, encrypted_key, encrypted_iv,
|
||||
cloud_private_key, decrypted_filename):
|
||||
key, err = utils.execute('openssl',
|
||||
'rsautl',
|
||||
'-decrypt',
|
||||
'-inkey', '%s' % cloud_private_key,
|
||||
process_input=encrypted_key,
|
||||
check_exit_code=False)
|
||||
if err:
|
||||
raise exception.Error(_("Failed to decrypt private key: %s")
|
||||
% err)
|
||||
iv, err = utils.execute('openssl',
|
||||
'rsautl',
|
||||
'-decrypt',
|
||||
'-inkey', '%s' % cloud_private_key,
|
||||
process_input=encrypted_iv,
|
||||
check_exit_code=False)
|
||||
if err:
|
||||
raise exception.Error(_("Failed to decrypt initialization "
|
||||
"vector: %s") % err)
|
||||
|
||||
_out, err = utils.execute('openssl',
|
||||
'enc',
|
||||
'-d',
|
||||
'-aes-128-cbc',
|
||||
'-in', '%s' % (encrypted_filename,),
|
||||
'-K', '%s' % (key,),
|
||||
'-iv', '%s' % (iv,),
|
||||
'-out', '%s' % (decrypted_filename,),
|
||||
check_exit_code=False)
|
||||
if err:
|
||||
raise exception.Error(_("Failed to decrypt image file "
|
||||
"%(image_file)s: %(err)s") %
|
||||
{'image_file': encrypted_filename,
|
||||
'err': err})
|
||||
|
||||
@staticmethod
|
||||
def untarzip_image(path, filename):
|
||||
tar_file = tarfile.open(filename, "r|gz")
|
||||
tar_file.extractall(path)
|
||||
image_file = tar_file.getnames()[0]
|
||||
tar_file.close()
|
||||
return image_file
|
||||
@@ -1,63 +0,0 @@
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2010 United States Government as represented by the
|
||||
# Administrator of the National Aeronautics and Space Administration.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Properties of an object stored within a bucket.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import nova.crypto
|
||||
from nova import exception
|
||||
|
||||
|
||||
class Object(object):
|
||||
def __init__(self, bucket, key):
|
||||
""" wrapper class of an existing key """
|
||||
self.bucket = bucket
|
||||
self.key = key
|
||||
self.path = bucket._object_path(key)
|
||||
if not os.path.isfile(self.path):
|
||||
raise exception.NotFound
|
||||
|
||||
def __repr__(self):
|
||||
return "<Object %s/%s>" % (self.bucket, self.key)
|
||||
|
||||
@property
|
||||
def md5(self):
|
||||
""" computes the MD5 of the contents of file """
|
||||
with open(self.path, "r") as f:
|
||||
return nova.crypto.compute_md5(f)
|
||||
|
||||
@property
|
||||
def mtime(self):
|
||||
""" mtime of file """
|
||||
return os.path.getmtime(self.path)
|
||||
|
||||
def read(self):
|
||||
""" read all contents of key into memory and return """
|
||||
return self.file.read()
|
||||
|
||||
@property
|
||||
def file(self):
|
||||
""" return a file object for the key """
|
||||
return open(self.path, 'rb')
|
||||
|
||||
def delete(self):
|
||||
""" deletes the file """
|
||||
os.unlink(self.path)
|
||||
@@ -35,31 +35,22 @@ from nova import log as logging
|
||||
from nova import rpc
|
||||
from nova import service
|
||||
from nova import test
|
||||
from nova import utils
|
||||
from nova.auth import manager
|
||||
from nova.compute import power_state
|
||||
from nova.api.ec2 import cloud
|
||||
from nova.api.ec2 import ec2utils
|
||||
from nova.image import local
|
||||
from nova.objectstore import image
|
||||
|
||||
|
||||
FLAGS = flags.FLAGS
|
||||
LOG = logging.getLogger('nova.tests.cloud')
|
||||
|
||||
# Temp dirs for working with image attributes through the cloud controller
|
||||
# (stole this from objectstore_unittest.py)
|
||||
OSS_TEMPDIR = tempfile.mkdtemp(prefix='test_oss-')
|
||||
IMAGES_PATH = os.path.join(OSS_TEMPDIR, 'images')
|
||||
os.makedirs(IMAGES_PATH)
|
||||
|
||||
|
||||
# TODO(termie): these tests are rather fragile, they should at the lest be
|
||||
# wiping database state after each run
|
||||
class CloudTestCase(test.TestCase):
|
||||
def setUp(self):
|
||||
super(CloudTestCase, self).setUp()
|
||||
self.flags(connection_type='fake',
|
||||
images_path=IMAGES_PATH)
|
||||
self.flags(connection_type='fake')
|
||||
|
||||
self.conn = rpc.Connection.instance()
|
||||
|
||||
@@ -70,6 +61,7 @@ class CloudTestCase(test.TestCase):
|
||||
self.compute = self.start_service('compute')
|
||||
self.scheduter = self.start_service('scheduler')
|
||||
self.network = self.start_service('network')
|
||||
self.image_service = utils.import_object(FLAGS.image_service)
|
||||
|
||||
self.manager = manager.AuthManager()
|
||||
self.user = self.manager.create_user('admin', 'admin', 'admin', True)
|
||||
@@ -318,41 +310,6 @@ class CloudTestCase(test.TestCase):
|
||||
LOG.debug(_("Terminating instance %s"), instance_id)
|
||||
rv = self.compute.terminate_instance(instance_id)
|
||||
|
||||
@staticmethod
|
||||
def _fake_set_image_description(ctxt, image_id, description):
|
||||
from nova.objectstore import handler
|
||||
|
||||
class req:
|
||||
pass
|
||||
|
||||
request = req()
|
||||
request.context = ctxt
|
||||
request.args = {'image_id': [image_id],
|
||||
'description': [description]}
|
||||
|
||||
resource = handler.ImagesResource()
|
||||
resource.render_POST(request)
|
||||
|
||||
def test_user_editable_image_endpoint(self):
|
||||
pathdir = os.path.join(FLAGS.images_path, 'ami-testing')
|
||||
os.mkdir(pathdir)
|
||||
info = {'isPublic': False}
|
||||
with open(os.path.join(pathdir, 'info.json'), 'w') as f:
|
||||
json.dump(info, f)
|
||||
img = image.Image('ami-testing')
|
||||
# self.cloud.set_image_description(self.context, 'ami-testing',
|
||||
# 'Foo Img')
|
||||
# NOTE(vish): Above won't work unless we start objectstore or create
|
||||
# a fake version of api/ec2/images.py conn that can
|
||||
# call methods directly instead of going through boto.
|
||||
# for now, just cheat and call the method directly
|
||||
self._fake_set_image_description(self.context, 'ami-testing',
|
||||
'Foo Img')
|
||||
self.assertEqual('Foo Img', img.metadata['description'])
|
||||
self._fake_set_image_description(self.context, 'ami-testing', '')
|
||||
self.assertEqual('', img.metadata['description'])
|
||||
shutil.rmtree(pathdir)
|
||||
|
||||
def test_update_of_instance_display_fields(self):
|
||||
inst = db.instance_create(self.context, {})
|
||||
ec2_id = ec2utils.id_to_ec2_id(inst['id'])
|
||||
|
||||
@@ -33,11 +33,9 @@ from boto.s3 import connection as s3
|
||||
from nova import context
|
||||
from nova import exception
|
||||
from nova import flags
|
||||
from nova import objectstore
|
||||
from nova import wsgi
|
||||
from nova import test
|
||||
from nova.auth import manager
|
||||
#from nova.exception import NotEmpty, NotFound
|
||||
from nova.objectstore import s3server
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user