openstack
/
glance_store
Code Issues Proposed changes

Merge "Add proxy support to S3 Store"

This commit is contained in:
Jenkins 2015-08-19 12:33:38 +00:00 committed by Gerrit Code Review
parent caa3d20cd0 e3b6d640a9
commit d19a18c4e5
3 changed files with 95 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
glance_store/_drivers/s3.py
View File

@ -78,6 +78,18 @@ _S3_OPTS = [
cfg.IntOpt('s3_store_thread_pools', default=DEFAULT_THREAD_POOLS,
help=_('The number of thread pools to perform a multipart '
'upload in S3.')),
cfg.BoolOpt('s3_store_enable_proxy', default=False,
help=_('Enable the use of a proxy.')),
cfg.StrOpt('s3_store_proxy_host',
help=_('Address or hostname for the proxy server.')),
cfg.IntOpt('s3_store_proxy_port', default=8080,
help=_('The port to use when connecting over a proxy.')),
cfg.StrOpt('s3_store_proxy_user',
default=None,
help=_('The username to connect to the proxy.')),
cfg.StrOpt('s3_store_proxy_password', secret=True,
default=None,
help=_('The password to use when connecting over a proxy.'))
]
@ -371,6 +383,33 @@ class Store(glance_store.driver.Store):
reason=reason)
return result
def _create_connection(self, loc):
from boto.s3.connection import S3Connection
s3host, s3port = netutils.parse_host_port(loc.s3serviceurl, 80)
uformat = self.conf.glance_store.s3_store_bucket_url_format
calling_format = get_calling_format(s3_store_bucket_url_format=uformat)
use_proxy = self.conf.glance_store.s3_store_enable_proxy
if use_proxy:
proxy_host = self._option_get('s3_store_proxy_host')
proxy_user = self.conf.glance_store.s3_store_proxy_user
proxy_pass = self.conf.glance_store.s3_store_proxy_password
proxy_port = self.conf.glance_store.s3_store_proxy_port
return S3Connection(loc.accesskey, loc.secretkey,
proxy=proxy_host,
proxy_port=proxy_port,
proxy_user=proxy_user,
proxy_pass=proxy_pass,
is_secure=(loc.scheme == 's3+https'),
calling_format=calling_format)
return S3Connection(loc.accesskey, loc.secretkey,
host=s3host, port=s3port,
is_secure=(loc.scheme == 's3+https'),
calling_format=calling_format)
@capabilities.check
def get(self, location, offset=0, chunk_size=None, context=None):
"""
@ -410,16 +449,7 @@ class Store(glance_store.driver.Store):
def _retrieve_key(self, location):
loc = location.store_location
s3host, s3port = netutils.parse_host_port(loc.s3serviceurl, 80)
from boto.s3.connection import S3Connection
uformat = self.conf.glance_store.s3_store_bucket_url_format
calling_format = get_calling_format(s3_store_bucket_url_format=uformat)
s3_conn = S3Connection(loc.accesskey, loc.secretkey,
host=s3host, port=s3port,
is_secure=(loc.scheme == 's3+https'),
calling_format=calling_format)
s3_conn = self._create_connection(loc)
bucket_obj = get_bucket(s3_conn, loc.bucket)
key = get_key(bucket_obj, loc.key)
@ -459,8 +489,6 @@ class Store(glance_store.driver.Store):
<BUCKET> = ``s3_store_bucket``
<ID> = The id of the image being added
"""
from boto.s3.connection import S3Connection
loc = StoreLocation({'scheme': self.scheme,
'bucket': self.bucket,
'key': image_id,
@ -468,14 +496,7 @@ class Store(glance_store.driver.Store):
'accesskey': self.access_key,
'secretkey': self.secret_key}, self.conf)
s3host, s3port = netutils.parse_host_port(loc.s3serviceurl, 80)
uformat = self.conf.glance_store.s3_store_bucket_url_format
calling_format = get_calling_format(s3_store_bucket_url_format=uformat)
s3_conn = S3Connection(loc.accesskey, loc.secretkey,
host=s3host, port=s3port,
is_secure=(loc.scheme == 's3+https'),
calling_format=calling_format)
s3_conn = self._create_connection(loc)
create_bucket_if_missing(self.conf, self.bucket, s3_conn)
@ -671,16 +692,7 @@ class Store(glance_store.driver.Store):
:raises NotFound if image does not exist
"""
loc = location.store_location
s3host, s3port = netutils.parse_host_port(loc.s3serviceurl, 80)
from boto.s3.connection import S3Connection
uformat = self.conf.glance_store.s3_store_bucket_url_format
calling_format = get_calling_format(s3_store_bucket_url_format=uformat)
s3_conn = S3Connection(loc.accesskey, loc.secretkey,
host=s3host, port=s3port,
is_secure=(loc.scheme == 's3+https'),
calling_format=calling_format)
s3_conn = self._create_connection(loc)
bucket_obj = get_bucket(s3_conn, loc.bucket)
# Close the key when we're through.

5
glance_store/tests/unit/test_opts.py
View File

@ -86,6 +86,11 @@ class OptsTestCase(base.StoreBaseTest):
's3_store_large_object_size',
's3_store_large_object_chunk_size',
's3_store_thread_pools',
's3_store_enable_proxy',
's3_store_proxy_host',
's3_store_proxy_port',
's3_store_proxy_user',
's3_store_proxy_password',
'sheepdog_store_address',
'sheepdog_store_chunk_size',
'sheepdog_store_port',

49
glance_store/tests/unit/test_s3_store.py
View File

@ -40,7 +40,12 @@ S3_CONF = {'s3_store_access_key': 'user',
's3_store_host': 'localhost:8080',
's3_store_bucket': 'glance',
's3_store_large_object_size': 5, # over 5MB is large
's3_store_large_object_chunk_size': 6} # part size is 6MB
's3_store_large_object_chunk_size': 6, # part size is 6MB
's3_store_enable_proxy': False,
's3_store_proxy_host': None,
's3_store_proxy_port': 8080,
's3_store_proxy_user': 'user',
's3_store_proxy_password': 'foobar'}
# ensure that mpu api is used and parts are uploaded as expected
mpu_parts_uploaded = 0
@ -586,3 +591,45 @@ class TestStore(base.StoreBaseTest,
def test_calling_format_default(self):
self.assertIsInstance(s3.get_calling_format(),
boto.s3.connection.SubdomainCallingFormat)
def test_image_get_with_proxy_without_host(self):
"""Test s3 backend with unconfigured proxy connection."""
self.config(s3_store_enable_proxy=True)
loc = location.get_location_from_uri(
"s3://user:key@auth_address/glance/%s" % FAKE_UUID,
conf=self.conf)
self.assertRaises(exceptions.BadStoreConfiguration,
self.store.get, loc)
def test_image_get_with_proxy(self):
"""Test s3 get with proxy connection."""
self.config(s3_store_enable_proxy=True)
proxy_host = '127.0.0.1'
self.config(s3_store_proxy_host=proxy_host)
with mock.patch.object(boto.s3.connection, 'S3Connection') as m:
cf = s3.get_calling_format(bucket_format=None,
s3_store_bucket_url_format='subdomain')
with mock.patch.object(s3, 'get_calling_format') as gcf:
gcf.return_value = cf
loc = location.get_location_from_uri(
"s3://user:key@auth_address/glance/%s" % FAKE_UUID,
conf=self.conf)
self.store.get(loc)
accesskey = S3_CONF['s3_store_access_key']
secretkey = S3_CONF['s3_store_secret_key']
proxy_port = S3_CONF['s3_store_proxy_port']
proxy_pass = S3_CONF['s3_store_proxy_password']
proxy_user = S3_CONF['s3_store_proxy_user']
m.assert_called_with(accesskey, secretkey,
calling_format=cf,
is_secure=False,
proxy=proxy_host,
proxy_pass=proxy_pass,
proxy_port=proxy_port,
proxy_user=proxy_user)