Switch VMWare Datastore to use Requests
Previously the VMWare Datastore was using HTTPS Connections from httplib which do not verify the connection. Switching to requests allows the store to perform proper connection level verification for a secure connection. By switching to using requests, we will get several benefits: 1. Certificate verification when using HTTPS 2. Connection pooling when following redirects 3. Help handling redirects 4. Help with Chunked Encoding Partial-bug: 1436082 Co-authored-by: Sabari Kumar Murugesan <smurugesan@vmware.com> Change-Id: I8ff20b2f6bd0e05cd50e44a60ec89fd54f87e1b4
This commit is contained in:
Ian Cordasco
Sabari Kumar Murugesan
parent
d4eb2c9ed2
commit
91636e8b85
@ -31,15 +31,20 @@ try:
|
||||
from oslo_vmware import vim_util
|
||||
except ImportError:
|
||||
api = None
|
||||
from six.moves import http_client
|
||||
from six.moves import urllib
|
||||
|
||||
from six.moves import urllib
|
||||
import six.moves.urllib.parse as urlparse
|
||||
|
||||
import requests
|
||||
from requests import adapters
|
||||
from requests.packages.urllib3.util import retry
|
||||
import six
|
||||
# NOTE(jokke): simplified transition to py3, behaves like py2 xrange
|
||||
from six.moves import range
|
||||
|
||||
import glance_store
|
||||
from glance_store import capabilities
|
||||
from glance_store.common import utils
|
||||
from glance_store import exceptions
|
||||
from glance_store.i18n import _
|
||||
from glance_store.i18n import _LE
|
||||
@ -48,6 +53,7 @@ from glance_store import location
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
CHUNKSIZE = 1024 * 64 # 64kB
|
||||
MAX_REDIRECTS = 5
|
||||
DEFAULT_STORE_IMAGE_DIR = '/openstack_glance'
|
||||
DS_URL_PREFIX = '/folder'
|
||||
@ -138,49 +144,6 @@ class _Reader(object):
|
||||
return self._size
|
||||
|
||||
|
||||
class _ChunkReader(_Reader):
|
||||
|
||||
def __init__(self, data, verifier=None, blocksize=8192):
|
||||
self.blocksize = blocksize
|
||||
self.current_chunk = b""
|
||||
self.closed = False
|
||||
super(_ChunkReader, self).__init__(data, verifier)
|
||||
|
||||
def read(self, size=None):
|
||||
ret = b""
|
||||
while size is None or size >= len(self.current_chunk):
|
||||
ret += self.current_chunk
|
||||
if size is not None:
|
||||
size -= len(self.current_chunk)
|
||||
if self.closed:
|
||||
self.current_chunk = b""
|
||||
break
|
||||
self._get_chunk()
|
||||
else:
|
||||
ret += self.current_chunk[:size]
|
||||
self.current_chunk = self.current_chunk[size:]
|
||||
return ret
|
||||
|
||||
def _get_chunk(self):
|
||||
if not self.closed:
|
||||
chunk = self.data.read(self.blocksize)
|
||||
chunk_len = len(chunk)
|
||||
self._size += chunk_len
|
||||
self.checksum.update(chunk)
|
||||
if self.verifier:
|
||||
self.verifier.update(chunk)
|
||||
if chunk:
|
||||
if six.PY3:
|
||||
size_header = ('%x\r\n' % chunk_len).encode('ascii')
|
||||
self.current_chunk = b''.join((size_header, chunk,
|
||||
b'\r\n'))
|
||||
else:
|
||||
self.current_chunk = b'%x\r\n%s\r\n' % (chunk_len, chunk)
|
||||
else:
|
||||
self.current_chunk = b'0\r\n\r\n'
|
||||
self.closed = True
|
||||
|
||||
|
||||
class StoreLocation(location.StoreLocation):
|
||||
"""Class describing an VMware URI.
|
||||
|
||||
@ -248,6 +211,16 @@ class StoreLocation(location.StoreLocation):
|
||||
if ds_name:
|
||||
self.datastore_name = ds_name[0]
|
||||
|
||||
@property
|
||||
def https_url(self):
|
||||
"""
|
||||
Creates a https url that can be used to upload/download data from a
|
||||
vmware store.
|
||||
"""
|
||||
parsed_url = urlparse.urlparse(self.get_uri())
|
||||
new_url = parsed_url._replace(scheme='https')
|
||||
return urlparse.urlunparse(new_url)
|
||||
|
||||
|
||||
class Store(glance_store.Store):
|
||||
"""An implementation of the VMware datastore adapter."""
|
||||
@ -445,15 +418,13 @@ class Store(glance_store.Store):
|
||||
are 201 Created and 200 OK.
|
||||
"""
|
||||
ds = self.select_datastore(image_size)
|
||||
image_file = _Reader(image_file, verifier)
|
||||
headers = {}
|
||||
if image_size > 0:
|
||||
headers = {'Content-Length': image_size}
|
||||
image_file = _Reader(image_file, verifier)
|
||||
headers.update({'Content-Length': image_size})
|
||||
data = image_file
|
||||
else:
|
||||
# NOTE (arnaud): use chunk encoding when the image is still being
|
||||
# generated by the server (ex: stream optimized disks generated by
|
||||
# Nova).
|
||||
headers = {'Transfer-Encoding': 'chunked'}
|
||||
image_file = _ChunkReader(image_file, verifier)
|
||||
data = utils.chunkiter(image_file, CHUNKSIZE)
|
||||
loc = StoreLocation({'scheme': self.scheme,
|
||||
'server_host': self.server_host,
|
||||
'image_dir': self.store_image_dir,
|
||||
@ -463,13 +434,15 @@ class Store(glance_store.Store):
|
||||
# NOTE(arnaud): use a decorator when the config is not tied to self
|
||||
cookie = self._build_vim_cookie_header(True)
|
||||
headers = dict(headers)
|
||||
headers['Cookie'] = cookie
|
||||
conn_class = self._get_http_conn_class()
|
||||
conn = conn_class(loc.server_host)
|
||||
url = urllib.parse.quote('%s?%s' % (loc.path, loc.query))
|
||||
headers.update({'Cookie': cookie})
|
||||
session = new_session(self.api_insecure)
|
||||
|
||||
url = loc.https_url
|
||||
try:
|
||||
conn.request('PUT', url, image_file, headers)
|
||||
response = session.put(url, data=data, headers=headers)
|
||||
except IOError as e:
|
||||
# TODO(sigmavirus24): Figure out what the new exception type would
|
||||
# be in requests.
|
||||
# When a session is not authenticated, the socket is closed by
|
||||
# the server after sending the response. http_client has an open
|
||||
# issue with https that raises Broken Pipe
|
||||
@ -482,17 +455,19 @@ class Store(glance_store.Store):
|
||||
'url': url,
|
||||
'e': e}
|
||||
LOG.error(msg)
|
||||
raise exceptions.BackendException(msg)
|
||||
except Exception:
|
||||
with excutils.save_and_reraise_exception():
|
||||
LOG.exception(_LE('Failed to upload content of image '
|
||||
'%(image)s'), {'image': image_id})
|
||||
res = conn.getresponse()
|
||||
if res.status == http_client.CONFLICT:
|
||||
|
||||
res = response.raw
|
||||
if res.status == requests.codes.conflict:
|
||||
raise exceptions.Duplicate(_("Image file %(image_id)s already "
|
||||
"exists!") %
|
||||
{'image_id': image_id})
|
||||
|
||||
if res.status not in (http_client.CREATED, http_client.OK):
|
||||
if res.status not in (requests.codes.created, requests.codes.ok):
|
||||
msg = (_LE('Failed to upload content of image %(image)s. '
|
||||
'The request returned an unexpected status: %(status)s.'
|
||||
'\nThe response body:\n%(body)s') %
|
||||
@ -534,7 +509,15 @@ class Store(glance_store.Store):
|
||||
:param location: `glance_store.location.Location` object, supplied
|
||||
from glance_store.location.get_location_from_uri()
|
||||
"""
|
||||
return self._query(location, 'HEAD')[2]
|
||||
conn = None
|
||||
try:
|
||||
conn, resp, size = self._query(location, 'HEAD')
|
||||
return size
|
||||
finally:
|
||||
# NOTE(sabari): Close the connection as the request was made with
|
||||
# stream=True.
|
||||
if conn is not None:
|
||||
conn.close()
|
||||
|
||||
@capabilities.check
|
||||
def delete(self, location, context=None):
|
||||
@ -566,30 +549,59 @@ class Store(glance_store.Store):
|
||||
LOG.exception(_LE('Failed to delete image %(image)s '
|
||||
'content.') % {'image': location.image_id})
|
||||
|
||||
def _query(self, location, method, depth=0):
|
||||
if depth > MAX_REDIRECTS:
|
||||
def _query(self, location, method):
|
||||
session = new_session(self.api_insecure)
|
||||
loc = location.store_location
|
||||
redirects_followed = 0
|
||||
# TODO(sabari): The redirect logic was added to handle cases when the
|
||||
# backend redirects http url's to https. But the store never makes a
|
||||
# http request and hence this can be safely removed.
|
||||
while redirects_followed < MAX_REDIRECTS:
|
||||
conn, resp = self._retry_request(session, method, location)
|
||||
|
||||
# NOTE(sigmavirus24): _retry_request handles 4xx and 5xx errors so
|
||||
# if the response is not a redirect, we can return early.
|
||||
if not conn.is_redirect:
|
||||
break
|
||||
|
||||
redirects_followed += 1
|
||||
|
||||
location_header = conn.headers.get('location')
|
||||
if location_header:
|
||||
if resp.status not in (301, 302):
|
||||
reason = (_("The HTTP URL %(path)s attempted to redirect "
|
||||
"with an invalid %(status)s status code.")
|
||||
% {'path': loc.path, 'status': resp.status})
|
||||
LOG.info(reason)
|
||||
raise exceptions.BadStoreUri(message=reason)
|
||||
conn.close()
|
||||
location = self._new_location(location, location_header)
|
||||
else:
|
||||
# NOTE(sigmavirus24): We exceeded the maximum number of redirects
|
||||
msg = ("The HTTP URL exceeded %(max_redirects)s maximum "
|
||||
"redirects.", {'max_redirects': MAX_REDIRECTS})
|
||||
LOG.debug(msg)
|
||||
raise exceptions.MaxRedirectsExceeded(redirects=MAX_REDIRECTS)
|
||||
|
||||
content_length = int(resp.getheader('content-length', 0))
|
||||
|
||||
return (conn, resp, content_length)
|
||||
|
||||
def _retry_request(self, session, method, location):
|
||||
loc = location.store_location
|
||||
# NOTE(arnaud): use a decorator when the config is not tied to self
|
||||
for i in range(self.api_retry_count + 1):
|
||||
cookie = self._build_vim_cookie_header()
|
||||
headers = {'Cookie': cookie}
|
||||
try:
|
||||
conn = self._get_http_conn(method, loc, headers)
|
||||
resp = conn.getresponse()
|
||||
except Exception:
|
||||
with excutils.save_and_reraise_exception():
|
||||
LOG.exception(_LE('Failed to access image %(image)s '
|
||||
'content.') % {'image':
|
||||
location.image_id})
|
||||
conn = session.request(method, loc.https_url, headers=headers,
|
||||
stream=True)
|
||||
resp = conn.raw
|
||||
|
||||
if resp.status >= 400:
|
||||
if resp.status == http_client.UNAUTHORIZED:
|
||||
if resp.status == requests.codes.unauthorized:
|
||||
self.reset_session()
|
||||
continue
|
||||
if resp.status == http_client.NOT_FOUND:
|
||||
if resp.status == requests.codes.not_found:
|
||||
reason = _('VMware datastore could not find image at URI.')
|
||||
LOG.info(reason)
|
||||
raise exceptions.NotFound(message=reason)
|
||||
@ -598,34 +610,36 @@ class Store(glance_store.Store):
|
||||
LOG.debug(msg)
|
||||
raise exceptions.BadStoreUri(msg)
|
||||
break
|
||||
location_header = resp.getheader('location')
|
||||
if location_header:
|
||||
if resp.status not in (301, 302):
|
||||
reason = (_("The HTTP URL %(path)s attempted to redirect "
|
||||
"with an invalid %(status)s status code.")
|
||||
% {'path': loc.path, 'status': resp.status})
|
||||
LOG.info(reason)
|
||||
raise exceptions.BadStoreUri(message=reason)
|
||||
location_class = glance_store.location.Location
|
||||
new_loc = location_class(location.store_name,
|
||||
location.store_location.__class__,
|
||||
uri=location_header,
|
||||
image_id=location.image_id,
|
||||
store_specs=location.store_specs)
|
||||
return self._query(new_loc, method, depth + 1)
|
||||
content_length = int(resp.getheader('content-length', 0))
|
||||
return conn, resp
|
||||
|
||||
return (conn, resp, content_length)
|
||||
def _new_location(self, old_location, url):
|
||||
store_name = old_location.store_name
|
||||
store_class = old_location.store_location.__class__
|
||||
image_id = old_location.image_id
|
||||
store_specs = old_location.store_specs
|
||||
# Note(sabari): The redirect url will have a scheme 'http(s)', but the
|
||||
# store only accepts url with scheme 'vsphere'. Thus, replacing with
|
||||
# store's scheme.
|
||||
parsed_url = urlparse.urlparse(url)
|
||||
new_url = parsed_url._replace(scheme='vsphere')
|
||||
vsphere_url = urlparse.urlunparse(new_url)
|
||||
return glance_store.location.Location(store_name,
|
||||
store_class,
|
||||
self.conf,
|
||||
uri=vsphere_url,
|
||||
image_id=image_id,
|
||||
store_specs=store_specs)
|
||||
|
||||
def _get_http_conn(self, method, loc, headers, content=None):
|
||||
conn_class = self._get_http_conn_class()
|
||||
conn = conn_class(loc.server_host)
|
||||
url = urllib.parse.quote('%s?%s' % (loc.path, loc.query))
|
||||
conn.request(method, url, content, headers)
|
||||
|
||||
return conn
|
||||
|
||||
def _get_http_conn_class(self):
|
||||
if self.api_insecure:
|
||||
return http_client.HTTPConnection
|
||||
return http_client.HTTPSConnection
|
||||
def new_session(insecure=False, total_retries=None):
|
||||
session = requests.Session()
|
||||
if total_retries is not None:
|
||||
http_adapter = adapters.HTTPAdapter(
|
||||
max_retries=retry.Retry(total=total_retries))
|
||||
https_adapter = adapters.HTTPAdapter(
|
||||
max_retries=retry.Retry(total=total_retries))
|
||||
session.mount('http://', http_adapter)
|
||||
session.mount('https://', https_adapter)
|
||||
if insecure:
|
||||
session.verify = False
|
||||
return session
|
||||
|
||||
@ -65,24 +65,6 @@ def format_location(host_ip, folder_name, image_id, datastores):
|
||||
image_id, datacenter_path, datastore_name))
|
||||
|
||||
|
||||
class FakeHTTPConnection(object):
|
||||
|
||||
def __init__(self, status=200, *args, **kwargs):
|
||||
self.status = status
|
||||
self.no_response_body = kwargs.get('no_response_body', False)
|
||||
pass
|
||||
|
||||
def getresponse(self):
|
||||
return utils.FakeHTTPResponse(status=self.status,
|
||||
no_response_body=self.no_response_body)
|
||||
|
||||
def request(self, *_args, **_kwargs):
|
||||
pass
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
|
||||
def fake_datastore_obj(*args, **kwargs):
|
||||
dc_obj = oslo_datacenter.Datacenter(ref='fake-ref',
|
||||
name='fake-name')
|
||||
@ -130,8 +112,8 @@ class TestStore(base.StoreBaseTest,
|
||||
loc = location.get_location_from_uri(
|
||||
"vsphere://127.0.0.1/folder/openstack_glance/%s"
|
||||
"?dsName=ds1&dcPath=dc1" % FAKE_UUID, conf=self.conf)
|
||||
with self._mock_http_connection() as HttpConn:
|
||||
HttpConn.return_value = FakeHTTPConnection()
|
||||
with mock.patch('requests.Session.request') as HttpConn:
|
||||
HttpConn.return_value = utils.fake_response()
|
||||
(image_file, image_size) = self.store.get(loc)
|
||||
self.assertEqual(image_size, expected_image_size)
|
||||
chunks = [c for c in image_file]
|
||||
@ -146,8 +128,8 @@ class TestStore(base.StoreBaseTest,
|
||||
loc = location.get_location_from_uri(
|
||||
"vsphere://127.0.0.1/folder/openstack_glan"
|
||||
"ce/%s?dsName=ds1&dcPath=dc1" % FAKE_UUID, conf=self.conf)
|
||||
with self._mock_http_connection() as HttpConn:
|
||||
HttpConn.return_value = FakeHTTPConnection(status=404)
|
||||
with mock.patch('requests.Session.request') as HttpConn:
|
||||
HttpConn.return_value = utils.fake_response(status_code=404)
|
||||
self.assertRaises(exceptions.NotFound, self.store.get, loc)
|
||||
|
||||
@mock.patch.object(vm_store.Store, 'select_datastore')
|
||||
@ -170,8 +152,8 @@ class TestStore(base.StoreBaseTest,
|
||||
expected_image_id,
|
||||
VMWARE_DS['vmware_datastores'])
|
||||
image = six.BytesIO(expected_contents)
|
||||
with self._mock_http_connection() as HttpConn:
|
||||
HttpConn.return_value = FakeHTTPConnection()
|
||||
with mock.patch('requests.Session.request') as HttpConn:
|
||||
HttpConn.return_value = utils.fake_response()
|
||||
location, size, checksum, _ = self.store.add(expected_image_id,
|
||||
image,
|
||||
expected_size)
|
||||
@ -204,8 +186,8 @@ class TestStore(base.StoreBaseTest,
|
||||
expected_image_id,
|
||||
VMWARE_DS['vmware_datastores'])
|
||||
image = six.BytesIO(expected_contents)
|
||||
with self._mock_http_connection() as HttpConn:
|
||||
HttpConn.return_value = FakeHTTPConnection()
|
||||
with mock.patch('requests.Session.request') as HttpConn:
|
||||
HttpConn.return_value = utils.fake_response()
|
||||
location, size, checksum, _ = self.store.add(expected_image_id,
|
||||
image, 0)
|
||||
self.assertEqual(utils.sort_url_by_qs_keys(expected_location),
|
||||
@ -222,14 +204,14 @@ class TestStore(base.StoreBaseTest,
|
||||
size = FIVE_KB
|
||||
contents = b"*" * size
|
||||
image = six.BytesIO(contents)
|
||||
with self._mock_http_connection() as HttpConn:
|
||||
HttpConn.return_value = FakeHTTPConnection()
|
||||
with mock.patch('requests.Session.request') as HttpConn:
|
||||
HttpConn.return_value = utils.fake_response()
|
||||
self.store.add(image_id, image, size, verifier=verifier)
|
||||
|
||||
fake_reader.assert_called_with(image, verifier)
|
||||
|
||||
@mock.patch.object(vm_store.Store, 'select_datastore')
|
||||
@mock.patch('glance_store._drivers.vmware_datastore._ChunkReader')
|
||||
@mock.patch('glance_store._drivers.vmware_datastore._Reader')
|
||||
def test_add_with_verifier_size_zero(self, fake_reader, fake_select_ds):
|
||||
"""Test that the verifier is passed to the _ChunkReader during add."""
|
||||
verifier = mock.MagicMock(name='mock_verifier')
|
||||
@ -237,8 +219,8 @@ class TestStore(base.StoreBaseTest,
|
||||
size = FIVE_KB
|
||||
contents = b"*" * size
|
||||
image = six.BytesIO(contents)
|
||||
with self._mock_http_connection() as HttpConn:
|
||||
HttpConn.return_value = FakeHTTPConnection()
|
||||
with mock.patch('requests.Session.request') as HttpConn:
|
||||
HttpConn.return_value = utils.fake_response()
|
||||
self.store.add(image_id, image, 0, verifier=verifier)
|
||||
|
||||
fake_reader.assert_called_with(image, verifier)
|
||||
@ -249,12 +231,12 @@ class TestStore(base.StoreBaseTest,
|
||||
loc = location.get_location_from_uri(
|
||||
"vsphere://127.0.0.1/folder/openstack_glance/%s?"
|
||||
"dsName=ds1&dcPath=dc1" % FAKE_UUID, conf=self.conf)
|
||||
with self._mock_http_connection() as HttpConn:
|
||||
HttpConn.return_value = FakeHTTPConnection()
|
||||
with mock.patch('requests.Session.request') as HttpConn:
|
||||
HttpConn.return_value = utils.fake_response()
|
||||
vm_store.Store._service_content = mock.Mock()
|
||||
self.store.delete(loc)
|
||||
with self._mock_http_connection() as HttpConn:
|
||||
HttpConn.return_value = FakeHTTPConnection(status=404)
|
||||
with mock.patch('requests.Session.request') as HttpConn:
|
||||
HttpConn.return_value = utils.fake_response(status_code=404)
|
||||
self.assertRaises(exceptions.NotFound, self.store.get, loc)
|
||||
|
||||
@mock.patch('oslo_vmware.api.VMwareAPISession')
|
||||
@ -278,8 +260,8 @@ class TestStore(base.StoreBaseTest,
|
||||
loc = location.get_location_from_uri(
|
||||
"vsphere://127.0.0.1/folder/openstack_glance/%s"
|
||||
"?dsName=ds1&dcPath=dc1" % FAKE_UUID, conf=self.conf)
|
||||
with self._mock_http_connection() as HttpConn:
|
||||
HttpConn.return_value = FakeHTTPConnection()
|
||||
with mock.patch('requests.Session.request') as HttpConn:
|
||||
HttpConn.return_value = utils.fake_response()
|
||||
image_size = self.store.get_size(loc)
|
||||
self.assertEqual(image_size, 31)
|
||||
|
||||
@ -292,8 +274,8 @@ class TestStore(base.StoreBaseTest,
|
||||
loc = location.get_location_from_uri(
|
||||
"vsphere://127.0.0.1/folder/openstack_glan"
|
||||
"ce/%s?dsName=ds1&dcPath=dc1" % FAKE_UUID, conf=self.conf)
|
||||
with self._mock_http_connection() as HttpConn:
|
||||
HttpConn.return_value = FakeHTTPConnection(status=404)
|
||||
with mock.patch('requests.Session.request') as HttpConn:
|
||||
HttpConn.return_value = utils.fake_response(status_code=404)
|
||||
self.assertRaises(exceptions.NotFound, self.store.get_size, loc)
|
||||
|
||||
def test_reader_full(self):
|
||||
@ -324,76 +306,6 @@ class TestStore(base.StoreBaseTest,
|
||||
reader.read()
|
||||
verifier.update.assert_called_with(content)
|
||||
|
||||
def test_chunkreader_image_fits_in_blocksize(self):
|
||||
"""
|
||||
Test that the image file reader returns the expected chunk of data
|
||||
when the block size is larger than the image.
|
||||
"""
|
||||
content = b'XXX'
|
||||
image = six.BytesIO(content)
|
||||
expected_checksum = hashlib.md5(content).hexdigest()
|
||||
reader = vm_store._ChunkReader(image)
|
||||
ret = reader.read()
|
||||
if six.PY3:
|
||||
expected_chunk = ('%x\r\n%s\r\n'
|
||||
% (len(content), content.decode('ascii')))
|
||||
expected_chunk = expected_chunk.encode('ascii')
|
||||
else:
|
||||
expected_chunk = b'%x\r\n%s\r\n' % (len(content), content)
|
||||
last_chunk = b'0\r\n\r\n'
|
||||
self.assertEqual(expected_chunk + last_chunk, ret)
|
||||
self.assertEqual(len(content), reader.size)
|
||||
self.assertEqual(expected_checksum, reader.checksum.hexdigest())
|
||||
self.assertTrue(reader.closed)
|
||||
ret = reader.read()
|
||||
self.assertEqual(len(content), reader.size)
|
||||
self.assertEqual(expected_checksum, reader.checksum.hexdigest())
|
||||
self.assertTrue(reader.closed)
|
||||
self.assertEqual(b'', ret)
|
||||
|
||||
def test_chunkreader_image_larger_blocksize(self):
|
||||
"""
|
||||
Test that the image file reader returns the expected chunks when
|
||||
the block size specified is smaller than the image.
|
||||
"""
|
||||
content = b'XXX'
|
||||
image = six.BytesIO(content)
|
||||
expected_checksum = hashlib.md5(content).hexdigest()
|
||||
last_chunk = b'0\r\n\r\n'
|
||||
reader = vm_store._ChunkReader(image, blocksize=1)
|
||||
ret = reader.read()
|
||||
expected_chunk = b'1\r\nX\r\n'
|
||||
expected = (expected_chunk + expected_chunk + expected_chunk
|
||||
+ last_chunk)
|
||||
self.assertEqual(expected,
|
||||
ret)
|
||||
self.assertEqual(expected_checksum, reader.checksum.hexdigest())
|
||||
self.assertEqual(len(content), reader.size)
|
||||
self.assertTrue(reader.closed)
|
||||
|
||||
def test_chunkreader_size(self):
|
||||
"""Test that the image reader takes into account the specified size."""
|
||||
content = b'XXX'
|
||||
image = six.BytesIO(content)
|
||||
expected_checksum = hashlib.md5(content).hexdigest()
|
||||
reader = vm_store._ChunkReader(image, blocksize=1)
|
||||
ret = reader.read(size=3)
|
||||
self.assertEqual(b'1\r\n', ret)
|
||||
ret = reader.read(size=1)
|
||||
self.assertEqual(b'X', ret)
|
||||
ret = reader.read()
|
||||
self.assertEqual(expected_checksum, reader.checksum.hexdigest())
|
||||
self.assertEqual(len(content), reader.size)
|
||||
self.assertTrue(reader.closed)
|
||||
|
||||
def test_chunkreader_with_verifier(self):
|
||||
content = b'XXX'
|
||||
image = six.BytesIO(content)
|
||||
verifier = mock.MagicMock(name='mock_verifier')
|
||||
reader = vm_store._ChunkReader(image, verifier)
|
||||
reader.read(size=3)
|
||||
verifier.update.assert_called_with(content)
|
||||
|
||||
def test_sanity_check_api_retry_count(self):
|
||||
"""Test that sanity check raises if api_retry_count is <= 0."""
|
||||
self.store.conf.glance_store.vmware_api_retry_count = -1
|
||||
@ -475,8 +387,8 @@ class TestStore(base.StoreBaseTest,
|
||||
expected_contents = b"*" * expected_size
|
||||
image = six.BytesIO(expected_contents)
|
||||
self.session = mock.Mock()
|
||||
with self._mock_http_connection() as HttpConn:
|
||||
HttpConn.return_value = FakeHTTPConnection(status=401)
|
||||
with mock.patch('requests.Session.request') as HttpConn:
|
||||
HttpConn.return_value = utils.fake_response(status_code=401)
|
||||
self.assertRaises(exceptions.BackendException,
|
||||
self.store.add,
|
||||
expected_image_id, image, expected_size)
|
||||
@ -491,8 +403,8 @@ class TestStore(base.StoreBaseTest,
|
||||
image = six.BytesIO(expected_contents)
|
||||
self.session = mock.Mock()
|
||||
with self._mock_http_connection() as HttpConn:
|
||||
HttpConn.return_value = FakeHTTPConnection(status=500,
|
||||
no_response_body=True)
|
||||
HttpConn.return_value = utils.fake_response(status_code=500,
|
||||
no_response_body=True)
|
||||
self.assertRaises(exceptions.BackendException,
|
||||
self.store.add,
|
||||
expected_image_id, image, expected_size)
|
||||
@ -532,7 +444,7 @@ class TestStore(base.StoreBaseTest,
|
||||
expected_contents = b"*" * expected_size
|
||||
image = six.BytesIO(expected_contents)
|
||||
self.session = mock.Mock()
|
||||
with self._mock_http_connection() as HttpConn:
|
||||
with mock.patch('requests.Session.request') as HttpConn:
|
||||
HttpConn.request.side_effect = IOError
|
||||
self.assertRaises(exceptions.BackendException,
|
||||
self.store.add,
|
||||
@ -660,3 +572,58 @@ class TestStore(base.StoreBaseTest,
|
||||
'FindByInventoryPath',
|
||||
self.store.session.vim.service_content.searchIndex,
|
||||
inventoryPath=datacenter_path)
|
||||
|
||||
@mock.patch('oslo_vmware.api.VMwareAPISession')
|
||||
def test_http_get_redirect(self, mock_api_session):
|
||||
# Add two layers of redirects to the response stack, which will
|
||||
# return the default 200 OK with the expected data after resolving
|
||||
# both redirects.
|
||||
redirect1 = {"location": "https://example.com?dsName=ds1&dcPath=dc1"}
|
||||
redirect2 = {"location": "https://example.com?dsName=ds2&dcPath=dc2"}
|
||||
responses = [utils.fake_response(),
|
||||
utils.fake_response(status_code=302, headers=redirect1),
|
||||
utils.fake_response(status_code=301, headers=redirect2)]
|
||||
|
||||
def getresponse(*args, **kwargs):
|
||||
return responses.pop()
|
||||
|
||||
expected_image_size = 31
|
||||
expected_returns = ['I am a teapot, short and stout\n']
|
||||
loc = location.get_location_from_uri(
|
||||
"vsphere://127.0.0.1/folder/openstack_glance/%s"
|
||||
"?dsName=ds1&dcPath=dc1" % FAKE_UUID, conf=self.conf)
|
||||
with mock.patch('requests.Session.request') as HttpConn:
|
||||
HttpConn.side_effect = getresponse
|
||||
(image_file, image_size) = self.store.get(loc)
|
||||
self.assertEqual(image_size, expected_image_size)
|
||||
chunks = [c for c in image_file]
|
||||
self.assertEqual(expected_returns, chunks)
|
||||
|
||||
@mock.patch('oslo_vmware.api.VMwareAPISession')
|
||||
def test_http_get_max_redirects(self, mock_api_session):
|
||||
redirect = {"location": "https://example.com?dsName=ds1&dcPath=dc1"}
|
||||
responses = ([utils.fake_response(status_code=302, headers=redirect)]
|
||||
* (vm_store.MAX_REDIRECTS + 1))
|
||||
|
||||
def getresponse(*args, **kwargs):
|
||||
return responses.pop()
|
||||
|
||||
loc = location.get_location_from_uri(
|
||||
"vsphere://127.0.0.1/folder/openstack_glance/%s"
|
||||
"?dsName=ds1&dcPath=dc1" % FAKE_UUID, conf=self.conf)
|
||||
with mock.patch('requests.Session.request') as HttpConn:
|
||||
HttpConn.side_effect = getresponse
|
||||
self.assertRaises(exceptions.MaxRedirectsExceeded, self.store.get,
|
||||
loc)
|
||||
|
||||
@mock.patch('oslo_vmware.api.VMwareAPISession')
|
||||
def test_http_get_redirect_invalid(self, mock_api_session):
|
||||
redirect = {"location": "https://example.com?dsName=ds1&dcPath=dc1"}
|
||||
|
||||
loc = location.get_location_from_uri(
|
||||
"vsphere://127.0.0.1/folder/openstack_glance/%s"
|
||||
"?dsName=ds1&dcPath=dc1" % FAKE_UUID, conf=self.conf)
|
||||
with mock.patch('requests.Session.request') as HttpConn:
|
||||
HttpConn.return_value = utils.fake_response(status_code=307,
|
||||
headers=redirect)
|
||||
self.assertRaises(exceptions.BadStoreUri, self.store.get, loc)
|
||||
|
||||
@ -67,9 +67,9 @@ class FakeHTTPResponse(object):
|
||||
self.data.close()
|
||||
|
||||
|
||||
def fake_response(status_code=200, headers=None, content=None):
|
||||
def fake_response(status_code=200, headers=None, content=None, **kwargs):
|
||||
r = requests.models.Response()
|
||||
r.status_code = status_code
|
||||
r.headers = headers or {}
|
||||
r.raw = FakeHTTPResponse(status_code, headers, content)
|
||||
r.raw = FakeHTTPResponse(status_code, headers, content, kwargs)
|
||||
return r
|
||||
|
||||
@ -0,0 +1,6 @@
|
||||
---
|
||||
security:
|
||||
- Previously the VMWare Datastore was using HTTPS Connections from httplib
|
||||
which do not verify the connection. By switching to using requests library
|
||||
the VMware storage backend now verifies HTTPS connection to vCenter server
|
||||
and thus addresses the vulnerabilities described in OSSN-0033.
|
||||
Loading…
Reference in New Issue
Block a user