Merge "Distributed image import"
This commit is contained in:
commit
878d7f49d8
@ -39,6 +39,7 @@ import glance.notifier
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.import_opt('public_endpoint', 'glance.api.versions')
|
||||
|
||||
|
||||
class ImageDataController(object):
|
||||
@ -349,6 +350,14 @@ class ImageDataController(object):
|
||||
msg = _("The image %s has data on staging") % image_id
|
||||
raise webob.exc.HTTPConflict(explanation=msg)
|
||||
|
||||
# NOTE(danms): Record this worker's
|
||||
# worker_self_reference_url in the image metadata so we
|
||||
# know who has the staging data.
|
||||
self_url = CONF.worker_self_reference_url or CONF.public_endpoint
|
||||
if self_url:
|
||||
image.extra_properties['os_glance_stage_host'] = self_url
|
||||
image_repo.save(image, from_state='uploading')
|
||||
|
||||
except exception.NotFound as e:
|
||||
raise webob.exc.HTTPNotFound(explanation=e.msg)
|
||||
|
||||
|
@ -26,6 +26,7 @@ from oslo_log import log as logging
|
||||
from oslo_serialization import jsonutils as json
|
||||
from oslo_utils import encodeutils
|
||||
from oslo_utils import timeutils as oslo_timeutils
|
||||
import requests
|
||||
import six
|
||||
from six.moves import http_client as http
|
||||
import six.moves.urllib.parse as urlparse
|
||||
@ -40,9 +41,10 @@ from glance.common import store_utils
|
||||
from glance.common import timeutils
|
||||
from glance.common import utils
|
||||
from glance.common import wsgi
|
||||
from glance import context as glance_context
|
||||
import glance.db
|
||||
import glance.gateway
|
||||
from glance.i18n import _, _LI, _LW
|
||||
from glance.i18n import _, _LE, _LI, _LW
|
||||
import glance.notifier
|
||||
import glance.schema
|
||||
|
||||
@ -56,6 +58,24 @@ CONF.import_opt('show_multiple_locations', 'glance.common.config')
|
||||
CONF.import_opt('hashing_algorithm', 'glance.common.config')
|
||||
|
||||
|
||||
def proxy_response_error(orig_code, orig_explanation):
|
||||
"""Construct a webob.exc.HTTPError exception on the fly.
|
||||
|
||||
The webob.exc.HTTPError classes are statically defined, intended
|
||||
to be straight subclasses of HTTPError, specifically with *class*
|
||||
level definitions of things we need to be dynamic. This method
|
||||
returns an exception class instance with those values set
|
||||
programmatically so we can raise it to mimic the response we
|
||||
got from a remote.
|
||||
"""
|
||||
|
||||
class ProxiedResponse(webob.exc.HTTPError):
|
||||
code = orig_code
|
||||
title = orig_explanation
|
||||
|
||||
return ProxiedResponse()
|
||||
|
||||
|
||||
class ImagesController(object):
|
||||
def __init__(self, db_api=None, policy_enforcer=None, notifier=None,
|
||||
store_api=None):
|
||||
@ -210,6 +230,75 @@ class ImagesController(object):
|
||||
{'image': image.image_id, 'task': task.task_id,
|
||||
'keys': ','.join(changed)})
|
||||
|
||||
def _proxy_request_to_stage_host(self, image, req, body=None):
|
||||
"""Proxy a request to a staging host.
|
||||
|
||||
When an image was staged on another worker, that worker may record its
|
||||
worker_self_reference_url on the image, indicating that other workers
|
||||
should proxy requests to it while the image is staged. This method
|
||||
replays our current request against the remote host, returns the
|
||||
result, and performs any response error translation required.
|
||||
|
||||
The remote request-id is used to replace the one on req.context so that
|
||||
a client sees the proper id used for the actual action.
|
||||
|
||||
:param image: The Image from the repo
|
||||
:param req: The webob.Request from the current request
|
||||
:param body: The request body or None
|
||||
:returns: The result from the remote host
|
||||
:raises: webob.exc.HTTPClientError matching the remote's error, or
|
||||
webob.exc.HTTPServerError if we were unable to contact the
|
||||
remote host.
|
||||
"""
|
||||
|
||||
stage_host = image.extra_properties['os_glance_stage_host']
|
||||
LOG.info(_LI('Proxying %s request to host %s '
|
||||
'which has image staged'),
|
||||
req.method, stage_host)
|
||||
client = glance_context.get_ksa_client(req.context)
|
||||
url = '%s%s' % (stage_host, req.path)
|
||||
req_id_hdr = 'x-openstack-request-id'
|
||||
request_method = getattr(client, req.method.lower())
|
||||
try:
|
||||
r = request_method(url, json=body, timeout=60)
|
||||
except (requests.exceptions.ConnectionError,
|
||||
requests.exceptions.ConnectTimeout) as e:
|
||||
LOG.error(_LE('Failed to proxy to %r: %s'), url, e)
|
||||
raise webob.exc.HTTPGatewayTimeout('Stage host is unavailable')
|
||||
except requests.exceptions.RequestException as e:
|
||||
LOG.error(_LE('Failed to proxy to %r: %s'), url, e)
|
||||
raise webob.exc.HTTPBadGateway('Stage host is unavailable')
|
||||
req_id_hdr = 'x-openstack-request-id'
|
||||
if req_id_hdr in r.headers:
|
||||
LOG.debug('Replying with remote request id %s' % (
|
||||
r.headers[req_id_hdr]))
|
||||
req.context.request_id = r.headers[req_id_hdr]
|
||||
if r.status_code // 100 != 2:
|
||||
raise proxy_response_error(r.status_code, r.reason)
|
||||
return image.image_id
|
||||
|
||||
@property
|
||||
def self_url(self):
|
||||
"""Return the URL we expect to point to us.
|
||||
|
||||
If this is set to a per-worker URL in worker_self_reference_url,
|
||||
that takes precedence. Otherwise we fall back to public_endpoint.
|
||||
"""
|
||||
return CONF.worker_self_reference_url or CONF.public_endpoint
|
||||
|
||||
def is_proxyable(self, image):
|
||||
"""Decide if an action is proxyable to a stage host.
|
||||
|
||||
If the image has a staging host recorded with a URL that does not match
|
||||
ours, then we can proxy our request to that host.
|
||||
|
||||
:param image: The Image from the repo
|
||||
:returns: bool indicating proxyable status
|
||||
"""
|
||||
return (
|
||||
'os_glance_stage_host' in image.extra_properties and
|
||||
image.extra_properties['os_glance_stage_host'] != self.self_url)
|
||||
|
||||
@utils.mutating
|
||||
def import_image(self, req, image_id, body):
|
||||
ctxt = req.context
|
||||
@ -308,6 +397,12 @@ class ImagesController(object):
|
||||
"enabled_backends %s") % uri)
|
||||
raise webob.exc.HTTPBadRequest(explanation=msg)
|
||||
|
||||
if self.is_proxyable(image) and import_method == 'glance-direct':
|
||||
# NOTE(danms): Image is staged on another worker; proxy the
|
||||
# import request to that worker with the user's token, as if
|
||||
# they had called it themselves.
|
||||
return self._proxy_request_to_stage_host(image, req, body)
|
||||
|
||||
task_input = {'image_id': image_id,
|
||||
'import_req': body,
|
||||
'backend': stores}
|
||||
@ -634,11 +729,59 @@ class ImagesController(object):
|
||||
|
||||
image_repo.save(image)
|
||||
|
||||
def _delete_image_on_remote(self, image, req):
|
||||
"""Proxy an image delete to a staging host.
|
||||
|
||||
When an image is staged and then deleted, the staging host still
|
||||
has local residue that needs to be cleaned up. If the request to
|
||||
delete arrived here, but we are not the stage host, we need to
|
||||
proxy it to the appropriate host.
|
||||
|
||||
If the delete succeeds, we return None (per DELETE semantics),
|
||||
indicating to the caller that it was handled.
|
||||
|
||||
If the delete fails on the remote end, we allow the
|
||||
HTTPClientError to bubble to our caller, which will return the
|
||||
error to the client.
|
||||
|
||||
If we fail to contact the remote server, we catch the
|
||||
HTTPServerError raised by our proxy method, verify that the
|
||||
image still exists, and return it. That indicates to the
|
||||
caller that it should proceed with the regular delete logic,
|
||||
which will satisfy the client's request, but leave the residue
|
||||
on the stage host (which is unavoidable).
|
||||
|
||||
:param image: The Image from the repo
|
||||
:param req: The webob.Request for this call
|
||||
:returns: None if successful, or a refreshed image if the proxy failed.
|
||||
:raises: webob.exc.HTTPClientError if so raised by the remote server.
|
||||
"""
|
||||
try:
|
||||
self._proxy_request_to_stage_host(image, req)
|
||||
except webob.exc.HTTPServerError:
|
||||
# This means we would have raised a 50x error, indicating
|
||||
# we did not succeed with the request to the remote host.
|
||||
# In this case, refresh the image from the repo, and if it
|
||||
# is not deleted, allow the regular delete process to
|
||||
# continue on the local worker to match the user's
|
||||
# expectations. If the image is already deleted, the caller
|
||||
# will catch this NotFound like normal.
|
||||
return self.gateway.get_repo(req.context).get(image.image_id)
|
||||
|
||||
@utils.mutating
|
||||
def delete(self, req, image_id):
|
||||
image_repo = self.gateway.get_repo(req.context)
|
||||
try:
|
||||
image = image_repo.get(image_id)
|
||||
if self.is_proxyable(image):
|
||||
# NOTE(danms): Image is staged on another worker; proxy the
|
||||
# delete request to that worker with the user's token, as if
|
||||
# they had called it themselves.
|
||||
image = self._delete_image_on_remote(image, req)
|
||||
if image is None:
|
||||
# Delete was proxied, so we are done here.
|
||||
return
|
||||
|
||||
# NOTE(abhishekk): Delete the data from staging area
|
||||
if CONF.enabled_backends:
|
||||
separator, staging_dir = store_utils.get_dir_separator()
|
||||
@ -1325,6 +1468,10 @@ class RequestDeserializer(wsgi.JSONRequestDeserializer):
|
||||
|
||||
|
||||
class ResponseSerializer(wsgi.JSONResponseSerializer):
|
||||
# These properties will be filtered out from the response and not
|
||||
# exposed to the client
|
||||
_hidden_properties = ['os_glance_stage_host']
|
||||
|
||||
def __init__(self, schema=None):
|
||||
super(ResponseSerializer, self).__init__()
|
||||
self.schema = schema or get_schema()
|
||||
@ -1344,7 +1491,8 @@ class ResponseSerializer(wsgi.JSONResponseSerializer):
|
||||
return []
|
||||
|
||||
try:
|
||||
image_view = dict(image.extra_properties)
|
||||
image_view = {k: v for k, v in dict(image.extra_properties).items()
|
||||
if k not in self._hidden_properties}
|
||||
attributes = ['name', 'disk_format', 'container_format',
|
||||
'visibility', 'size', 'virtual_size', 'status',
|
||||
'checksum', 'protected', 'min_ram', 'min_disk',
|
||||
|
@ -324,6 +324,15 @@ class _ImportActions(object):
|
||||
self._image.size = None
|
||||
break
|
||||
|
||||
def pop_extra_property(self, name):
|
||||
"""Delete the named extra_properties value, if present.
|
||||
|
||||
If the image.extra_properties dict contains the named key,
|
||||
delete it.
|
||||
:param name: The key to delete.
|
||||
"""
|
||||
self._image.extra_properties.pop(name, None)
|
||||
|
||||
|
||||
class _DeleteFromFS(task.Task):
|
||||
|
||||
@ -788,5 +797,6 @@ def get_flow(**kwargs):
|
||||
action.set_image_status('importing')
|
||||
action.add_importing_stores(stores)
|
||||
action.remove_failed_stores(stores)
|
||||
action.pop_extra_property('os_glance_stage_host')
|
||||
|
||||
return flow
|
||||
|
@ -589,6 +589,26 @@ roles in keystone (e.g., `admin`, `member`, and `reader`).
|
||||
|
||||
Related options:
|
||||
* [oslo_policy]/enforce_new_defaults
|
||||
""")),
|
||||
cfg.StrOpt('worker_self_reference_url',
|
||||
default=None,
|
||||
help=_("""
|
||||
The URL to this worker.
|
||||
|
||||
If this is set, other glance workers will know how to contact this one
|
||||
directly if needed. For image import, a single worker stages the image
|
||||
and other workers need to be able to proxy the import request to the
|
||||
right one.
|
||||
|
||||
If unset, this will be considered to be `public_endpoint`, which
|
||||
normally would be set to the same value on all workers, effectively
|
||||
disabling the proxying behavior.
|
||||
|
||||
Possible values:
|
||||
* A URL by which this worker is reachable from other workers
|
||||
|
||||
Related options:
|
||||
* public_endpoint
|
||||
|
||||
""")),
|
||||
]
|
||||
|
@ -20,6 +20,7 @@ import tempfile
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import fixtures
|
||||
from oslo_serialization import jsonutils
|
||||
from oslo_utils.secretutils import md5
|
||||
from oslo_utils import units
|
||||
@ -6880,3 +6881,123 @@ class TestCopyImagePermissions(functional.MultipleBackendFunctionalTest):
|
||||
response = requests.get(path, headers=self._headers())
|
||||
self.assertEqual(http.OK, response.status_code)
|
||||
self.assertIn('file2', jsonutils.loads(response.text)['stores'])
|
||||
|
||||
|
||||
class TestImportProxy(functional.SynchronousAPIBase):
|
||||
"""Test the image import proxy-to-stage-worker behavior.
|
||||
|
||||
This is done as a SynchronousAPIBase test with one mock for a couple of
|
||||
reasons:
|
||||
|
||||
1. The main functional tests can't handle a call with a token
|
||||
inside because of their paste config. Even if they did, they would
|
||||
not be able to validate it.
|
||||
2. The main functional tests don't support multiple API workers with
|
||||
separate config and making them work that way is non-trivial.
|
||||
|
||||
Functional tests are fairly synthetic and fixing or hacking over
|
||||
the above push us only further so. Using theh Synchronous API
|
||||
method is vastly easier, easier to verify, and tests the
|
||||
integration across the API calls, which is what is important.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
super(TestImportProxy, self).setUp()
|
||||
# Emulate a keystoneauth1 client for service-to-service communication
|
||||
self.ksa_client = self.useFixture(
|
||||
fixtures.MockPatch('glance.context.get_ksa_client')).mock
|
||||
|
||||
def test_import_proxy(self):
|
||||
resp = requests.Response()
|
||||
resp.status_code = 202
|
||||
resp.headers['x-openstack-request-id'] = 'req-remote'
|
||||
self.ksa_client.return_value.post.return_value = resp
|
||||
|
||||
# Stage it on worker1
|
||||
self.config(worker_self_reference_url='http://worker1')
|
||||
self.start_server()
|
||||
image_id = self._create_and_stage()
|
||||
|
||||
# Make sure we can't see the stage host key
|
||||
image = self.api_get('/v2/images/%s' % image_id).json
|
||||
self.assertIn('container_format', image)
|
||||
self.assertNotIn('os_glance_stage_host', image)
|
||||
|
||||
# Import call goes to worker2
|
||||
self.config(worker_self_reference_url='http://worker2')
|
||||
self.start_server()
|
||||
r = self._import_direct(image_id, ['store1'])
|
||||
|
||||
# Assert that it was proxied back to worker1
|
||||
self.assertEqual(202, r.status_code)
|
||||
self.assertEqual('req-remote', r.headers['x-openstack-request-id'])
|
||||
self.ksa_client.return_value.post.assert_called_once_with(
|
||||
'http://worker1/v2/images/%s/import' % image_id,
|
||||
timeout=60,
|
||||
json={'method': {'name': 'glance-direct'},
|
||||
'stores': ['store1'],
|
||||
'all_stores': False})
|
||||
|
||||
def test_import_proxy_fail_on_remote(self):
|
||||
resp = requests.Response()
|
||||
resp.url = '/v2'
|
||||
resp.status_code = 409
|
||||
resp.reason = 'Something Failed (tm)'
|
||||
self.ksa_client.return_value.post.return_value = resp
|
||||
self.ksa_client.return_value.delete.return_value = resp
|
||||
|
||||
# Stage it on worker1
|
||||
self.config(worker_self_reference_url='http://worker1')
|
||||
self.start_server()
|
||||
image_id = self._create_and_stage()
|
||||
|
||||
# Import call goes to worker2
|
||||
self.config(worker_self_reference_url='http://worker2')
|
||||
self.start_server()
|
||||
r = self._import_direct(image_id, ['store1'])
|
||||
|
||||
# Make sure we see the relevant details from worker1
|
||||
self.assertEqual(409, r.status_code)
|
||||
self.assertEqual('409 Something Failed (tm)', r.status)
|
||||
|
||||
# For a 40x, we should get the same on delete
|
||||
r = self.api_delete('/v2/images/%s' % image_id)
|
||||
self.assertEqual(409, r.status_code)
|
||||
self.assertEqual('409 Something Failed (tm)', r.status)
|
||||
|
||||
def _test_import_proxy_fail_requests(self, error, status):
|
||||
self.ksa_client.return_value.post.side_effect = error
|
||||
self.ksa_client.return_value.delete.side_effect = error
|
||||
|
||||
# Stage it on worker1
|
||||
self.config(worker_self_reference_url='http://worker1')
|
||||
self.start_server()
|
||||
image_id = self._create_and_stage()
|
||||
|
||||
# Import call goes to worker2
|
||||
self.config(worker_self_reference_url='http://worker2')
|
||||
self.start_server()
|
||||
r = self._import_direct(image_id, ['store1'])
|
||||
self.assertEqual(status, r.status)
|
||||
self.assertIn(b'Stage host is unavailable', r.body)
|
||||
|
||||
# Make sure we can still delete it
|
||||
r = self.api_delete('/v2/images/%s' % image_id)
|
||||
self.assertEqual(204, r.status_code)
|
||||
r = self.api_get('/v2/images/%s' % image_id)
|
||||
self.assertEqual(404, r.status_code)
|
||||
|
||||
def test_import_proxy_connection_refused(self):
|
||||
self._test_import_proxy_fail_requests(
|
||||
requests.exceptions.ConnectionError(),
|
||||
'504 Gateway Timeout')
|
||||
|
||||
def test_import_proxy_connection_timeout(self):
|
||||
self._test_import_proxy_fail_requests(
|
||||
requests.exceptions.ConnectTimeout(),
|
||||
'504 Gateway Timeout')
|
||||
|
||||
def test_import_proxy_connection_unknown_error(self):
|
||||
self._test_import_proxy_fail_requests(
|
||||
requests.exceptions.RequestException(),
|
||||
'502 Bad Gateway')
|
||||
|
@ -63,8 +63,11 @@ class TestApiImageImportTask(test_utils.BaseTestCase):
|
||||
|
||||
self.mock_task_repo = mock.MagicMock()
|
||||
self.mock_image_repo = mock.MagicMock()
|
||||
self.mock_image_repo.get.return_value.extra_properties = {
|
||||
'os_glance_import_task': TASK_ID1}
|
||||
self.mock_image = self.mock_image_repo.get.return_value
|
||||
self.mock_image.extra_properties = {
|
||||
'os_glance_import_task': TASK_ID1,
|
||||
'os_glance_stage_host': 'http://glance2',
|
||||
}
|
||||
|
||||
@mock.patch('glance.async_.flows.api_image_import._VerifyStaging.__init__')
|
||||
@mock.patch('taskflow.patterns.linear_flow.Flow.add')
|
||||
@ -103,6 +106,17 @@ class TestApiImageImportTask(test_utils.BaseTestCase):
|
||||
self._pass_uri(uri=test_uri, file_uri=expected_uri,
|
||||
import_req=self.gd_task_input['import_req'])
|
||||
|
||||
def test_get_flow_pops_stage_host(self):
|
||||
import_flow.get_flow(task_id=TASK_ID1, task_type=TASK_TYPE,
|
||||
task_repo=self.mock_task_repo,
|
||||
image_repo=self.mock_image_repo,
|
||||
image_id=IMAGE_ID1,
|
||||
import_req=self.gd_task_input['import_req'])
|
||||
self.assertNotIn('os_glance_stage_host',
|
||||
self.mock_image.extra_properties)
|
||||
self.assertIn('os_glance_import_task',
|
||||
self.mock_image.extra_properties)
|
||||
|
||||
|
||||
class TestImageLock(test_utils.BaseTestCase):
|
||||
def setUp(self):
|
||||
@ -899,6 +913,17 @@ class TestImportActions(test_utils.BaseTestCase):
|
||||
_('Unexpected exception when deleting from store foo.'))
|
||||
mock_log.warning.reset_mock()
|
||||
|
||||
def test_pop_extra_property(self):
|
||||
self.image.extra_properties = {'foo': '1', 'bar': 2}
|
||||
|
||||
# Should remove, if present
|
||||
self.actions.pop_extra_property('foo')
|
||||
self.assertEqual({'bar': 2}, self.image.extra_properties)
|
||||
|
||||
# Should not raise if missing
|
||||
self.actions.pop_extra_property('baz')
|
||||
self.assertEqual({'bar': 2}, self.image.extra_properties)
|
||||
|
||||
|
||||
@mock.patch('glance.common.scripts.utils.get_task')
|
||||
class TestCompleteTask(test_utils.BaseTestCase):
|
||||
|
@ -70,6 +70,7 @@ def get_fake_request(path='', method='POST', is_admin=False, user=USER1,
|
||||
|
||||
req = wsgi.Request.blank(path)
|
||||
req.method = method
|
||||
req.headers = {'x-openstack-request-id': 'my-req'}
|
||||
|
||||
kwargs = {
|
||||
'user': user,
|
||||
|
@ -18,6 +18,7 @@ import uuid
|
||||
from cursive import exception as cursive_exception
|
||||
import glance_store
|
||||
from glance_store._drivers import filesystem
|
||||
from oslo_config import cfg
|
||||
import six
|
||||
from six.moves import http_client as http
|
||||
import webob
|
||||
@ -30,6 +31,9 @@ from glance.tests.unit import base
|
||||
import glance.tests.unit.utils as unit_test_utils
|
||||
import glance.tests.utils as test_utils
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.import_opt('public_endpoint', 'glance.api.versions')
|
||||
|
||||
|
||||
class Raise(object):
|
||||
|
||||
@ -54,6 +58,7 @@ class FakeImage(object):
|
||||
self.container_format = container_format
|
||||
self.disk_format = disk_format
|
||||
self._status = status
|
||||
self.extra_properties = {}
|
||||
|
||||
@property
|
||||
def status(self):
|
||||
@ -553,6 +558,49 @@ class TestImagesController(base.StoreClearingUnitTest):
|
||||
self.assertRaises(webob.exc.HTTPConflict, self.controller.stage,
|
||||
request, image_id, 'YYYY', 4)
|
||||
|
||||
def _test_image_stage_records_host(self, expected_url):
|
||||
image_id = str(uuid.uuid4())
|
||||
request = unit_test_utils.get_fake_request()
|
||||
image = FakeImage(image_id=image_id)
|
||||
self.image_repo.result = image
|
||||
with mock.patch.object(filesystem.Store, 'add'):
|
||||
self.controller.stage(request, image_id, 'YYYY', 4)
|
||||
if expected_url is None:
|
||||
self.assertNotIn('os_glance_stage_host', image.extra_properties)
|
||||
else:
|
||||
self.assertEqual(expected_url,
|
||||
image.extra_properties['os_glance_stage_host'])
|
||||
|
||||
def test_image_stage_records_host_unset(self):
|
||||
# Make sure we do not set a null staging host, if we are not configured
|
||||
# to support worker-to-worker communication.
|
||||
self._test_image_stage_records_host(None)
|
||||
|
||||
def test_image_stage_records_host_public_endpoint(self):
|
||||
# Make sure we fall back to public_endpoint
|
||||
self.config(public_endpoint='http://lb.example.com')
|
||||
self._test_image_stage_records_host('http://lb.example.com')
|
||||
|
||||
def test_image_stage_records_host_self_url(self):
|
||||
# Make sure worker_self_reference_url takes precedence
|
||||
self.config(worker_self_reference_url='http://worker1.example.com')
|
||||
self._test_image_stage_records_host('http://worker1.example.com')
|
||||
|
||||
def test_image_stage_fail_does_not_set_host(self):
|
||||
# Make sure that if the store.add() fails, we do not claim to have the
|
||||
# image staged.
|
||||
self.config(public_endpoint='http://worker1.example.com')
|
||||
image_id = str(uuid.uuid4())
|
||||
request = unit_test_utils.get_fake_request()
|
||||
image = FakeImage(image_id=image_id)
|
||||
self.image_repo.result = image
|
||||
exc_cls = glance_store.exceptions.StorageFull
|
||||
with mock.patch.object(filesystem.Store, 'add', side_effect=exc_cls):
|
||||
self.assertRaises(webob.exc.HTTPRequestEntityTooLarge,
|
||||
self.controller.stage,
|
||||
request, image_id, 'YYYY', 4)
|
||||
self.assertNotIn('os_glance_stage_host', image.extra_properties)
|
||||
|
||||
|
||||
class TestImageDataDeserializer(test_utils.BaseTestCase):
|
||||
|
||||
|
@ -16,6 +16,7 @@
|
||||
import datetime
|
||||
import hashlib
|
||||
import os
|
||||
import requests
|
||||
from unittest import mock
|
||||
import uuid
|
||||
|
||||
@ -30,6 +31,7 @@ from six.moves import http_client as http
|
||||
from six.moves import range
|
||||
import testtools
|
||||
import webob
|
||||
import webob.exc
|
||||
|
||||
import glance.api.v2.image_actions
|
||||
import glance.api.v2.images
|
||||
@ -873,6 +875,184 @@ class TestImagesController(base.IsolatedUnitTest):
|
||||
{'method': {'name': 'web-download',
|
||||
'uri': 'fake_uri'}})
|
||||
|
||||
@mock.patch('glance.context.get_ksa_client')
|
||||
def test_image_import_proxies(self, mock_client):
|
||||
# Make sure that we proxy to the remote side when we need to
|
||||
self.config(
|
||||
worker_self_reference_url='http://glance-worker2.openstack.org')
|
||||
request = unit_test_utils.get_fake_request(
|
||||
'/v2/images/%s/import' % UUID4)
|
||||
with mock.patch.object(
|
||||
glance.api.authorization.ImageRepoProxy, 'get') as mock_get:
|
||||
mock_get.return_value = FakeImage(status='uploading')
|
||||
mock_get.return_value.extra_properties['os_glance_stage_host'] = (
|
||||
'https://glance-worker1.openstack.org')
|
||||
remote_hdrs = {'x-openstack-request-id': 'remote-req'}
|
||||
mock_resp = mock.MagicMock(location='/target',
|
||||
status_code=202,
|
||||
reason='Thanks',
|
||||
headers=remote_hdrs)
|
||||
mock_client.return_value.post.return_value = mock_resp
|
||||
r = self.controller.import_image(
|
||||
request, UUID4,
|
||||
{'method': {'name': 'glance-direct'}})
|
||||
|
||||
# Make sure we returned the ID like expected normally
|
||||
self.assertEqual(UUID4, r)
|
||||
|
||||
# Make sure we called the expected remote URL and passed
|
||||
# the body.
|
||||
mock_client.return_value.post.assert_called_once_with(
|
||||
('https://glance-worker1.openstack.org'
|
||||
'/v2/images/%s/import') % UUID4,
|
||||
json={'method': {'name': 'glance-direct'}},
|
||||
timeout=60)
|
||||
|
||||
# Make sure the remote request-id is returned to us
|
||||
self.assertEqual('remote-req', request.context.request_id)
|
||||
|
||||
@mock.patch('glance.context.get_ksa_client')
|
||||
def test_image_delete_proxies(self, mock_client):
|
||||
# Make sure that we proxy to the remote side when we need to
|
||||
self.config(
|
||||
worker_self_reference_url='http://glance-worker2.openstack.org')
|
||||
request = unit_test_utils.get_fake_request(
|
||||
'/v2/images/%s' % UUID4, method='DELETE')
|
||||
with mock.patch.object(
|
||||
glance.api.authorization.ImageRepoProxy, 'get') as mock_get:
|
||||
mock_get.return_value = FakeImage(status='uploading')
|
||||
mock_get.return_value.extra_properties['os_glance_stage_host'] = (
|
||||
'https://glance-worker1.openstack.org')
|
||||
remote_hdrs = {'x-openstack-request-id': 'remote-req'}
|
||||
mock_resp = mock.MagicMock(location='/target',
|
||||
status_code=202,
|
||||
reason='Thanks',
|
||||
headers=remote_hdrs)
|
||||
mock_client.return_value.delete.return_value = mock_resp
|
||||
self.controller.delete(request, UUID4)
|
||||
|
||||
# Make sure we called the expected remote URL and passed
|
||||
# the body.
|
||||
mock_client.return_value.delete.assert_called_once_with(
|
||||
('https://glance-worker1.openstack.org'
|
||||
'/v2/images/%s') % UUID4,
|
||||
json=None, timeout=60)
|
||||
|
||||
@mock.patch('glance.context.get_ksa_client')
|
||||
def test_image_import_proxies_error(self, mock_client):
|
||||
# Make sure that errors from the remote worker are proxied to our
|
||||
# client with the proper code and message
|
||||
self.config(
|
||||
worker_self_reference_url='http://glance-worker2.openstack.org')
|
||||
request = unit_test_utils.get_fake_request(
|
||||
'/v2/images/%s/import' % UUID4)
|
||||
with mock.patch.object(
|
||||
glance.api.authorization.ImageRepoProxy, 'get') as mock_get:
|
||||
mock_get.return_value = FakeImage(status='uploading')
|
||||
mock_get.return_value.extra_properties['os_glance_stage_host'] = (
|
||||
'https://glance-worker1.openstack.org')
|
||||
mock_resp = mock.MagicMock(location='/target',
|
||||
status_code=456,
|
||||
reason='No thanks')
|
||||
mock_client.return_value.post.return_value = mock_resp
|
||||
exc = self.assertRaises(webob.exc.HTTPError,
|
||||
self.controller.import_image,
|
||||
request, UUID4,
|
||||
{'method': {'name': 'glance-direct'}})
|
||||
self.assertEqual('456 No thanks', exc.status)
|
||||
mock_client.return_value.post.assert_called_once_with(
|
||||
('https://glance-worker1.openstack.org'
|
||||
'/v2/images/%s/import') % UUID4,
|
||||
json={'method': {'name': 'glance-direct'}},
|
||||
timeout=60)
|
||||
|
||||
@mock.patch('glance.context.get_ksa_client')
|
||||
def test_image_delete_proxies_error(self, mock_client):
|
||||
# Make sure that errors from the remote worker are proxied to our
|
||||
# client with the proper code and message
|
||||
self.config(
|
||||
worker_self_reference_url='http://glance-worker2.openstack.org')
|
||||
request = unit_test_utils.get_fake_request(
|
||||
'/v2/images/%s' % UUID4, method='DELETE')
|
||||
with mock.patch.object(
|
||||
glance.api.authorization.ImageRepoProxy, 'get') as mock_get:
|
||||
mock_get.return_value = FakeImage(status='uploading')
|
||||
mock_get.return_value.extra_properties['os_glance_stage_host'] = (
|
||||
'https://glance-worker1.openstack.org')
|
||||
remote_hdrs = {'x-openstack-request-id': 'remote-req'}
|
||||
mock_resp = mock.MagicMock(location='/target',
|
||||
status_code=456,
|
||||
reason='No thanks',
|
||||
headers=remote_hdrs)
|
||||
mock_client.return_value.delete.return_value = mock_resp
|
||||
exc = self.assertRaises(webob.exc.HTTPError,
|
||||
self.controller.delete, request, UUID4)
|
||||
self.assertEqual('456 No thanks', exc.status)
|
||||
|
||||
# Make sure we called the expected remote URL and passed
|
||||
# the body.
|
||||
mock_client.return_value.delete.assert_called_once_with(
|
||||
('https://glance-worker1.openstack.org'
|
||||
'/v2/images/%s') % UUID4,
|
||||
json=None, timeout=60)
|
||||
|
||||
@mock.patch('glance.context.get_ksa_client')
|
||||
@mock.patch.object(glance.api.authorization.ImageRepoProxy, 'get')
|
||||
@mock.patch.object(glance.api.authorization.ImageRepoProxy, 'remove')
|
||||
def test_image_delete_deletes_locally_on_error(self, mock_remove, mock_get,
|
||||
mock_client):
|
||||
# Make sure that if the proxy delete fails due to a connection error
|
||||
# that we continue with the delete ourselves.
|
||||
self.config(
|
||||
worker_self_reference_url='http://glance-worker2.openstack.org')
|
||||
request = unit_test_utils.get_fake_request(
|
||||
'/v2/images/%s' % UUID4, method='DELETE')
|
||||
|
||||
image = FakeImage(status='uploading')
|
||||
mock_get.return_value = image
|
||||
image.extra_properties['os_glance_stage_host'] = (
|
||||
'https://glance-worker1.openstack.org')
|
||||
image.delete = mock.MagicMock()
|
||||
|
||||
mock_client.return_value.delete.side_effect = (
|
||||
requests.exceptions.ConnectTimeout)
|
||||
|
||||
self.controller.delete(request, UUID4)
|
||||
|
||||
# Make sure we called delete on our image
|
||||
mock_get.return_value.delete.assert_called_once_with()
|
||||
mock_remove.assert_called_once_with(image)
|
||||
|
||||
# Make sure we called the expected remote URL and passed
|
||||
# the body.
|
||||
mock_client.return_value.delete.assert_called_once_with(
|
||||
('https://glance-worker1.openstack.org'
|
||||
'/v2/images/%s') % UUID4,
|
||||
json=None, timeout=60)
|
||||
|
||||
@mock.patch('glance.context.get_ksa_client')
|
||||
def test_image_import_no_proxy_non_direct(self, mock_client):
|
||||
# Make sure that we won't take the proxy path for import methods
|
||||
# other than glance-direct
|
||||
self.config(
|
||||
worker_self_reference_url='http://glance-worker2.openstack.org')
|
||||
request = unit_test_utils.get_fake_request(
|
||||
'/v2/images/%s/import' % UUID4)
|
||||
with mock.patch.object(
|
||||
glance.api.authorization.ImageRepoProxy, 'get') as mock_get:
|
||||
mock_get.return_value = FakeImage(status='queued')
|
||||
mock_get.return_value.extra_properties['os_glance_stage_host'] = (
|
||||
'https://glance-worker1.openstack.org')
|
||||
# This will fail validation after the point at which we would
|
||||
# have proxied to the remote side, just to avoid task setup.
|
||||
self.assertRaises(webob.exc.HTTPBadRequest,
|
||||
self.controller.import_image,
|
||||
request, UUID4,
|
||||
{'method': {'name': 'web-download',
|
||||
'url': 'not-a-url'}})
|
||||
# Make sure we did not try to proxy this web-download request
|
||||
mock_client.return_value.post.assert_not_called()
|
||||
|
||||
def test_create(self):
|
||||
request = unit_test_utils.get_fake_request()
|
||||
image = {'name': 'image-1'}
|
||||
@ -5044,6 +5224,17 @@ class TestImagesSerializer(test_utils.BaseTestCase):
|
||||
self.assertEqual(http.ACCEPTED, response.status_int)
|
||||
self.assertEqual('0', response.headers['Content-Length'])
|
||||
|
||||
def test_image_stage_host_hidden(self):
|
||||
# Make sure that os_glance_stage_host is not exposed to clients
|
||||
response = webob.Response()
|
||||
self.serializer.show(response,
|
||||
mock.MagicMock(extra_properties={
|
||||
'foo': 'bar',
|
||||
'os_glance_stage_host': 'http://foo'}))
|
||||
actual = jsonutils.loads(response.body)
|
||||
self.assertIn('foo', actual)
|
||||
self.assertNotIn('os_glance_stage_host', actual)
|
||||
|
||||
|
||||
class TestImagesSerializerWithUnicode(test_utils.BaseTestCase):
|
||||
|
||||
@ -5879,3 +6070,36 @@ class TestMultiImagesController(base.MultiIsolatedUnitTest):
|
||||
'stores': ["cheap"]})
|
||||
|
||||
self.assertEqual(UUID7, output)
|
||||
|
||||
|
||||
class TestProxyHelpers(base.IsolatedUnitTest):
|
||||
def test_proxy_response_error(self):
|
||||
e = glance.api.v2.images.proxy_response_error(123, 'Foo')
|
||||
self.assertIsInstance(e, webob.exc.HTTPError)
|
||||
self.assertEqual(123, e.code)
|
||||
self.assertEqual('123 Foo', e.status)
|
||||
|
||||
def test_is_proxyable(self):
|
||||
controller = glance.api.v2.images.ImagesController(None, None,
|
||||
None, None)
|
||||
self.config(worker_self_reference_url='http://worker1')
|
||||
mock_image = mock.MagicMock(extra_properties={})
|
||||
|
||||
self.assertFalse(controller.is_proxyable(mock_image))
|
||||
|
||||
mock_image.extra_properties['os_glance_stage_host'] = 'http://worker1'
|
||||
self.assertFalse(controller.is_proxyable(mock_image))
|
||||
|
||||
mock_image.extra_properties['os_glance_stage_host'] = 'http://worker2'
|
||||
self.assertTrue(controller.is_proxyable(mock_image))
|
||||
|
||||
def test_self_url(self):
|
||||
controller = glance.api.v2.images.ImagesController(None, None,
|
||||
None, None)
|
||||
self.assertIsNone(controller.self_url)
|
||||
|
||||
self.config(public_endpoint='http://lb.example.com')
|
||||
self.assertEqual('http://lb.example.com', controller.self_url)
|
||||
|
||||
self.config(worker_self_reference_url='http://worker1.example.com')
|
||||
self.assertEqual('http://worker1.example.com', controller.self_url)
|
||||
|
Loading…
Reference in New Issue
Block a user