metadata speed - revert logic changes, just caching
* reverts most of https://review.openstack.org/5042 * keep changes that cache the results so subsequent calls are fast fixes bug 954377 Change-Id: Ia047f1d04dc360dd657d4d80d3dff8c8101635cd
This commit is contained in:
Jesse Andrews
parent
f403575742
commit
4c8dbf81d9
@ -19,7 +19,6 @@
|
||||
"""Metadata request handler."""
|
||||
|
||||
import base64
|
||||
import collections
|
||||
|
||||
import webob.dec
|
||||
import webob.exc
|
||||
@ -33,12 +32,9 @@ from nova import exception
|
||||
from nova import flags
|
||||
from nova import log as logging
|
||||
from nova import network
|
||||
from nova.rpc import common as rpc_common
|
||||
from nova import utils
|
||||
from nova import volume
|
||||
from nova import wsgi
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
FLAGS = flags.FLAGS
|
||||
flags.DECLARE('use_forwarded_for', 'nova.api.auth')
|
||||
@ -83,36 +79,9 @@ class MetadataRequestHandler(wsgi.Application):
|
||||
self.compute_api = compute.API(
|
||||
network_api=self.network_api,
|
||||
volume_api=volume.API())
|
||||
|
||||
self.metadata_mapper = {
|
||||
'user-data': self.user_data,
|
||||
'meta-data': {
|
||||
'instance-id': self.instance_id,
|
||||
'instance-type': self.instance_type,
|
||||
'ami-id': self.ami_id,
|
||||
'kernel-id': self.kernel_id,
|
||||
'ramdisk-id': self.ramdisk_id,
|
||||
'block-device-mapping': self.block_device_mapping,
|
||||
'hostname': self.hostname,
|
||||
'public-hostname': self.hostname,
|
||||
'local-hostname': self.hostname,
|
||||
'local-ipv4': self.local_ipv4,
|
||||
'public-ipv4': self.public_ipv4,
|
||||
'security-groups': self.security_groups,
|
||||
'public-keys': self.public_keys,
|
||||
'ami-launch-index': self.ami_launch_index,
|
||||
'reservation-id': self.reservation_id,
|
||||
'placement': self.placement,
|
||||
'instance-action': self.instance_action,
|
||||
'ami-manifest-path': self.ami_manifest_path,
|
||||
}
|
||||
}
|
||||
|
||||
self._cache = memcache.Client(FLAGS.memcached_servers, debug=0)
|
||||
|
||||
def _format_instance_mapping(self, instance_ref):
|
||||
ctxt = context.get_admin_context()
|
||||
|
||||
def _format_instance_mapping(self, ctxt, instance_ref):
|
||||
root_device_name = instance_ref['root_device_name']
|
||||
if root_device_name is None:
|
||||
return _DEFAULT_MAPPINGS
|
||||
@ -158,127 +127,84 @@ class MetadataRequestHandler(wsgi.Application):
|
||||
|
||||
return mappings
|
||||
|
||||
def get_instance(self, address):
|
||||
"""get instance_ref for a given fixed_ip, raising
|
||||
exception.NotFound if unable to find instance
|
||||
|
||||
this will attempt to use memcache or fake memcache (an in-memory
|
||||
cache) to remove the DB query + RPC query for batched calls (eg
|
||||
cloud-init making dozens of queries on boot)
|
||||
"""
|
||||
|
||||
cache_key = 'metadata-%s' % address
|
||||
instance_dict = self._cache.get(cache_key)
|
||||
if instance_dict:
|
||||
return instance_dict
|
||||
|
||||
def get_metadata(self, address):
|
||||
if not address:
|
||||
raise exception.FixedIpNotFoundForAddress(address=address)
|
||||
|
||||
ctxt = context.get_admin_context()
|
||||
cache_key = 'metadata-%s' % address
|
||||
data = self._cache.get(cache_key)
|
||||
if data:
|
||||
return data
|
||||
|
||||
ctxt = context.get_admin_context()
|
||||
try:
|
||||
fixed_ip = self.network_api.get_fixed_ip_by_address(ctxt, address)
|
||||
except rpc_common.RemoteError:
|
||||
raise exception.FixedIpNotFoundForAddress(address=address)
|
||||
instance_ref = db.instance_get(ctxt, fixed_ip['instance_id'])
|
||||
instance_dict = utils.to_primitive(instance_ref)
|
||||
instance_ref = db.instance_get(ctxt, fixed_ip['instance_id'])
|
||||
except exception.NotFound:
|
||||
return None
|
||||
|
||||
self._cache.set(cache_key, instance_dict, 15)
|
||||
hostname = "%s.%s" % (instance_ref['hostname'], FLAGS.dhcp_domain)
|
||||
host = instance_ref['host']
|
||||
services = db.service_get_all_by_host(ctxt.elevated(), host)
|
||||
availability_zone = ec2utils.get_availability_zone_by_host(services,
|
||||
host)
|
||||
|
||||
return instance_dict
|
||||
|
||||
def user_data(self, address):
|
||||
instance_ref = self.get_instance(address)
|
||||
return base64.b64decode(instance_ref['user_data'])
|
||||
|
||||
def instance_id(self, address):
|
||||
instance_ref = self.get_instance(address)
|
||||
return ec2utils.id_to_ec2_id(instance_ref['id'])
|
||||
|
||||
def instance_type(self, address):
|
||||
instance_ref = self.get_instance(address)
|
||||
return instance_ref['instance_type']['name']
|
||||
|
||||
def ami_id(self, address):
|
||||
instance_ref = self.get_instance(address)
|
||||
image_id = instance_ref['image_ref']
|
||||
ctxt = context.get_admin_context()
|
||||
return ec2utils.glance_id_to_ec2_id(ctxt, image_id)
|
||||
|
||||
def kernel_id(self, address):
|
||||
instance_ref = self.get_instance(address)
|
||||
kernel_id = instance_ref.get('kernel_id')
|
||||
if kernel_id:
|
||||
image_type = ec2utils.image_type('kernel')
|
||||
ctxt = context.get_admin_context()
|
||||
return ec2utils.glance_id_to_ec2_id(ctxt, kernel_id, image_type)
|
||||
|
||||
def ramdisk_id(self, address):
|
||||
instance_ref = self.get_instance(address)
|
||||
ramdisk_id = instance_ref.get('ramdisk_id')
|
||||
if ramdisk_id:
|
||||
image_type = ec2utils.image_type('ramdisk')
|
||||
ctxt = context.get_admin_context()
|
||||
return ec2utils.image_ec2_id(ctxt, ramdisk_id, image_type)
|
||||
|
||||
def ami_launch_index(self, address):
|
||||
instance_ref = self.get_instance(address)
|
||||
return instance_ref['launch_index']
|
||||
|
||||
def block_device_mapping(self, address):
|
||||
instance_ref = self.get_instance(address)
|
||||
return self._format_instance_mapping(instance_ref)
|
||||
|
||||
def hostname(self, address):
|
||||
instance_ref = self.get_instance(address)
|
||||
return "%s.%s" % (instance_ref['hostname'], FLAGS.dhcp_domain)
|
||||
|
||||
def local_ipv4(self, address):
|
||||
return address
|
||||
|
||||
def public_ipv4(self, address):
|
||||
instance_ref = self.get_instance(address)
|
||||
ctxt = context.get_admin_context()
|
||||
ip_info = ec2utils.get_ip_info_for_instance(ctxt, instance_ref)
|
||||
floating_ips = ip_info['floating_ips']
|
||||
floating_ip = floating_ips and floating_ips[0] or ''
|
||||
return floating_ip
|
||||
|
||||
def reservation_id(self, address):
|
||||
instance_ref = self.get_instance(address)
|
||||
return instance_ref['reservation_id']
|
||||
|
||||
def placement(self, address):
|
||||
instance_ref = self.get_instance(address)
|
||||
host = instance_ref['host']
|
||||
ec2_id = ec2utils.id_to_ec2_id(instance_ref['id'])
|
||||
image_id = instance_ref['image_ref']
|
||||
ctxt = context.get_admin_context()
|
||||
# note(ja): original code had ctx.elevated?
|
||||
services = db.service_get_all_by_host(ctxt, host)
|
||||
zone = ec2utils.get_availability_zone_by_host(services, host)
|
||||
return {'availability-zone': zone}
|
||||
image_ec2_id = ec2utils.glance_id_to_ec2_id(ctxt, image_id)
|
||||
security_groups = db.security_group_get_by_instance(ctxt,
|
||||
instance_ref['id'])
|
||||
security_groups = [x['name'] for x in security_groups]
|
||||
mappings = self._format_instance_mapping(ctxt, instance_ref)
|
||||
data = {
|
||||
'user-data': base64.b64decode(instance_ref['user_data']),
|
||||
'meta-data': {
|
||||
'ami-id': image_ec2_id,
|
||||
'ami-launch-index': instance_ref['launch_index'],
|
||||
'ami-manifest-path': 'FIXME',
|
||||
'block-device-mapping': mappings,
|
||||
'hostname': hostname,
|
||||
'instance-action': 'none',
|
||||
'instance-id': ec2_id,
|
||||
'instance-type': instance_ref['instance_type']['name'],
|
||||
'local-hostname': hostname,
|
||||
'local-ipv4': address,
|
||||
'placement': {'availability-zone': availability_zone},
|
||||
'public-hostname': hostname,
|
||||
'public-ipv4': floating_ip,
|
||||
'reservation-id': instance_ref['reservation_id'],
|
||||
'security-groups': security_groups}}
|
||||
|
||||
def security_groups(self, address):
|
||||
instance_ref = self.get_instance(address)
|
||||
ctxt = context.get_admin_context()
|
||||
groups = db.security_group_get_by_instance(ctxt,
|
||||
instance_ref['id'])
|
||||
return [g['name'] for g in groups]
|
||||
|
||||
def public_keys(self, address):
|
||||
instance_ref = self.get_instance(address)
|
||||
# public-keys should be in meta-data only if user specified one
|
||||
if instance_ref['key_name']:
|
||||
return {'0': {'_name': instance_ref['key_name'],
|
||||
'openssh-key': instance_ref['key_data']}}
|
||||
data['meta-data']['public-keys'] = {
|
||||
'0': {'_name': instance_ref['key_name'],
|
||||
'openssh-key': instance_ref['key_data']}}
|
||||
|
||||
def ami_manifest_path(self, address):
|
||||
return 'Not Implemented'
|
||||
for image_type in ['kernel', 'ramdisk']:
|
||||
if instance_ref.get('%s_id' % image_type):
|
||||
image_id = instance_ref['%s_id' % image_type]
|
||||
image_type = ec2utils.image_type(image_type)
|
||||
ec2_id = ec2utils.glance_id_to_ec2_id(ctxt,
|
||||
image_id,
|
||||
image_type)
|
||||
data['meta-data']['%s-id' % image_type] = ec2_id
|
||||
|
||||
def instance_action(self, address):
|
||||
return 'none'
|
||||
if False: # TODO(vish): store ancestor ids
|
||||
data['ancestor-ami-ids'] = []
|
||||
if False: # TODO(vish): store product codes
|
||||
data['product-codes'] = []
|
||||
|
||||
def format_data(self, data):
|
||||
self._cache.set(cache_key, data, 15)
|
||||
|
||||
return data
|
||||
|
||||
def print_data(self, data):
|
||||
if isinstance(data, dict):
|
||||
output = ''
|
||||
for key in data:
|
||||
@ -298,21 +224,15 @@ class MetadataRequestHandler(wsgi.Application):
|
||||
else:
|
||||
return str(data)
|
||||
|
||||
def lookup(self, path, address):
|
||||
def lookup(self, path, data):
|
||||
items = path.split('/')
|
||||
data = self.metadata_mapper
|
||||
for item in items:
|
||||
if item:
|
||||
if not isinstance(data, dict):
|
||||
# FIXME(ja): should we check that we are at the end
|
||||
# of the path as well before we just return?
|
||||
return data
|
||||
if not item in data:
|
||||
return None
|
||||
data = data[item]
|
||||
if isinstance(data, collections.Callable):
|
||||
# lazy evaluation
|
||||
data = data(address)
|
||||
return data
|
||||
|
||||
@webob.dec.wsgify(RequestClass=wsgi.Request)
|
||||
@ -320,20 +240,19 @@ class MetadataRequestHandler(wsgi.Application):
|
||||
remote_address = req.remote_addr
|
||||
if FLAGS.use_forwarded_for:
|
||||
remote_address = req.headers.get('X-Forwarded-For', remote_address)
|
||||
|
||||
try:
|
||||
data = self.lookup(req.path_info,
|
||||
remote_address)
|
||||
except (exception.NotFound, exception.FixedIpNotFoundForAddress):
|
||||
LOG.error(_('Failed to get metadata for ip: %s'), remote_address)
|
||||
return webob.exc.HTTPNotFound()
|
||||
except:
|
||||
meta_data = self.get_metadata(remote_address)
|
||||
except Exception:
|
||||
LOG.exception(_('Failed to get metadata for ip: %s'),
|
||||
remote_address)
|
||||
msg = _('An unknown error has occurred. '
|
||||
'Please try your request again.')
|
||||
return webob.exc.HTTPInternalServerError(explanation=unicode(msg))
|
||||
|
||||
exc = webob.exc.HTTPInternalServerError(explanation=unicode(msg))
|
||||
return exc
|
||||
if meta_data is None:
|
||||
LOG.error(_('Failed to get metadata for ip: %s'), remote_address)
|
||||
raise webob.exc.HTTPNotFound()
|
||||
data = self.lookup(req.path_info, meta_data)
|
||||
if data is None:
|
||||
raise webob.exc.HTTPNotFound()
|
||||
return self.format_data(data)
|
||||
return self.print_data(data)
|
||||
|
||||
@ -119,7 +119,7 @@ class MetadataTestCase(test.TestCase):
|
||||
request = webob.Request.blank('/user-data')
|
||||
request.remote_addr = None
|
||||
response = request.get_response(self.app)
|
||||
self.assertEqual(response.status_int, 404)
|
||||
self.assertEqual(response.status_int, 500)
|
||||
|
||||
def test_user_data_invalid_url(self):
|
||||
request = webob.Request.blank('/user-data-invalid')
|
||||
@ -177,7 +177,9 @@ class MetadataTestCase(test.TestCase):
|
||||
'swap': '/dev/sdc',
|
||||
'ebs0': '/dev/sdh'}
|
||||
|
||||
self.assertEqual(self.app._format_instance_mapping(instance_ref0),
|
||||
self.assertEqual(self.app._format_instance_mapping(ctxt,
|
||||
instance_ref0),
|
||||
handler._DEFAULT_MAPPINGS)
|
||||
self.assertEqual(self.app._format_instance_mapping(instance_ref1),
|
||||
self.assertEqual(self.app._format_instance_mapping(ctxt,
|
||||
instance_ref1),
|
||||
expected)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user