From 009c8a7b929c9b961c9cf388c122cffe6f6ae41c Mon Sep 17 00:00:00 2001 From: Dmitrii Shcherbakov Date: Thu, 15 Jul 2021 20:49:16 +0300 Subject: [PATCH] Dynamically generate proxy settings for image syncs sstream-mirror-glance has several endpoints it needs to talk to: * Image mirrors - typically, public Internet endpoints; * Keystone - typically, a directly reachable endpoint; * Glance - typically, a directly reachable endpoint; * Object store (Swift) - typically, a directly reachable endpoint but sometimes it may be deployed externally and added to the region catalog in Keystone (in which case it might be accessible via a proxy only). While sstream-mirror-glance does not support specifying proxy settings for individual directions, since we know all of them based on the Keystone catalog, a list of endpoints to add to NO_PROXY environment variable can be generated dynamically. The complication is that image syncs are periodically done via a cron job so a juju-run invocation is needed to retrieve relevant proxy settings from model-config at each invocation of the synchronization script. Additionally, the charm is long-lived so there may be some environments that rely on legacy proxy settings. This change accounts for that and acts both on juju-prefixed (new) and unprefixed (legacy) proxy settings. Whether to use proxy settings for connections to the object store API is controlled by a charm option which the script is made to react to. Proxy settings are ignored for object store connections by default. Closes-Bug: #1843486 Change-Id: Ib1fc5d2eebf43d5f98bb8ee405a3799802c8b8dc --- config.yaml | 7 + files/glance_simplestreams_sync.py | 159 ++++++++++--- hooks/hooks.py | 2 + requirements.txt | 2 +- templates/mirrors.yaml | 1 + test-requirements.txt | 8 +- tox.ini | 4 +- unit_tests/test_glance_simplestreams_sync.py | 228 +++++++++++++++++++ 8 files changed, 374 insertions(+), 37 deletions(-) create mode 100644 unit_tests/test_glance_simplestreams_sync.py diff --git a/config.yaml b/config.yaml index fae3f06..8d4b09b 100644 --- a/config.yaml +++ b/config.yaml @@ -23,6 +23,13 @@ options: on a local Apache server running on the unit and endpoints will be registered referencing the local unit. This does not support HA or TLS and is for testing purposes only. + ignore_proxy_for_object_store: + type: boolean + default: true + description: | + Controls whether Juju model proxy settings are going to be used + by sstream-mirror-glance when connecting to object-store endpoints + from the Keystone catalog. frequency: type: string default: "daily" diff --git a/files/glance_simplestreams_sync.py b/files/glance_simplestreams_sync.py index 5d30313..84252f5 100755 --- a/files/glance_simplestreams_sync.py +++ b/files/glance_simplestreams_sync.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Copyright 2014 Canonical Ltd. # @@ -27,22 +27,28 @@ import atexit import base64 import copy import fcntl +import itertools import logging import os +import re import shutil import six -import sys import subprocess +import sys import tempfile import time import yaml +from keystoneclient import exceptions as keystone_exceptions from keystoneclient.v2_0 import client as keystone_client from keystoneclient.v3 import client as keystone_v3_client -from keystoneclient import exceptions as keystone_exceptions +if six.PY3: + from urllib import parse as urlparse +else: + import urlparse -def setup_logging(): +def setup_file_logging(): logfilename = '/var/log/glance-simplestreams-sync.log' if not os.path.exists(logfilename): @@ -60,10 +66,8 @@ def setup_logging(): logger.setLevel('DEBUG') logger.addHandler(h) - return logger - -log = setup_logging() +log = logging.getLogger() KEYRING = '/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg' CONF_FILE_DIR = '/etc/glance-simplestreams-sync' @@ -95,6 +99,12 @@ SSTREAM_LOG_FILE = os.path.join(SSTREAM_SNAP_COMMON, CACERT_FILE = os.path.join(SSTREAM_SNAP_COMMON, 'cacert.pem') SYSTEM_CACERT_FILE = '/etc/ssl/certs/ca-certificates.crt' +ENDPOINT_TYPES = [ + 'publicURL', + 'adminURL', + 'internalURL', +] + # TODOs: # - allow people to specify their own policy, since they can specify # their own mirrors. @@ -226,7 +236,7 @@ def set_openstack_env(id_conf, charm_conf): os.environ['OS_TENANT_NAME'] = id_conf['admin_tenant_name'] -def do_sync(charm_conf): +def do_sync(ksc, charm_conf): # NOTE(beisner): the user_agent variable was an unused assignment (lint). # It may be worth re-visiting its usage, intent and benefit with the @@ -234,6 +244,8 @@ def do_sync(charm_conf): # and not assigning it since it is not currently utilized. # user_agent = charm_conf.get("user_agent") + region_name = charm_conf['region'] + for mirror_info in charm_conf['mirror_list']: # NOTE: output directory must be under HOME # or snap cannot access it for stream files @@ -241,7 +253,7 @@ def do_sync(charm_conf): try: log.info("Configuring sync for url {}".format(mirror_info)) content_id = charm_conf['content_id_template'].format( - region=charm_conf['region']) + region=region_name) sync_command = [ "/snap/bin/simplestreams.sstream-mirror-glance", @@ -284,43 +296,135 @@ def do_sync(charm_conf): ] sync_command += mirror_info['item_filters'] + # Pass the current process' environment down along with proxy + # settings crafted for sstream-mirror-glance. + sstream_mirror_env = os.environ.copy() + sstream_mirror_env.update(get_sstream_mirror_proxy_env( + ksc, region_name, + charm_conf['ignore_proxy_for_object_store'], + )) + log.info("calling sstream-mirror-glance") - log.debug("command: {}".format(" ".join(sync_command))) - subprocess.check_call(sync_command) + log.debug("command: %s", " ".join(sync_command)) + log.debug("sstream-mirror environment: %s", sstream_mirror_env) + subprocess.check_call(sync_command, env=sstream_mirror_env) if not charm_conf['use_swift']: # Sync output directory to APACHE_DATA_DIR subprocess.check_call([ 'rsync', '-avz', - os.path.join(tmpdir, charm_conf['region'], 'streams'), + os.path.join(tmpdir, region_name, 'streams'), APACHE_DATA_DIR ]) finally: shutil.rmtree(tmpdir) -def update_product_streams_service(ksc, services, region): - """ - Updates URLs of product-streams endpoint to point to swift URLs. - """ +def get_sstream_mirror_proxy_env(ksc, region_name, + ignore_proxy_for_object_store=True): + '''Get proxy settings to be passed to sstreams-mirror-glance. + sstream-mirror-glance has multiple endpoints it needs to connect to: + + 1. Upstream image mirror (typically, an endpoint in public Internet); + 2. Keystone (typically, a directly reachable endpoint); + 3. Object storage (Swift) (typically a directly reachable endpoint). + 4. Glance (typically, a directly reachable endpoint). + + In a restricted environment where proxy settings have to be used for public + Internet connectivity we need to be explicit about hosts for which proxy + settings need to be used by sstream-mirror-glance. This function + dynamically builds a list of endpoints that need to be added to NO_PROXY + and optionally allows not including object storage endpoints into the + NO_PROXY list. + + :param ksc: An instance of a Keystone client. + :type ksc: :class: `keystoneclient.v3.client.Client` + :param str region_name: A name of the region to retrieve endpoints for. + :param bool ignore_proxy_for_object_store: Do not include object-store + endpoints into NO_PROXY. + ''' + proxy_settings = juju_proxy_settings() + if proxy_settings is None: + proxy_settings = {} + no_proxy_set = set() + else: + no_proxy_set = set(proxy_settings.get('NO_PROXY').split(',')) + additional_hosts = set([ + urlparse.urlparse(u).hostname for u in itertools.chain( + get_service_endpoints(ksc, 'identity', region_name).values(), + get_service_endpoints(ksc, 'image', region_name).values(), + get_service_endpoints(ksc, 'object-store', region_name).values() + if ignore_proxy_for_object_store else [], + )]) + no_proxy = ','.join(no_proxy_set | additional_hosts) + proxy_settings['NO_PROXY'] = no_proxy + proxy_settings['no_proxy'] = no_proxy + return proxy_settings + + +def update_product_streams_service(ksc, services, region): + """Updates URLs of product-streams endpoint to point to swift URLs.""" + object_store_endpoints = get_service_endpoints(ksc, 'object-store', region) + for endpoint_type in ENDPOINT_TYPES: + object_store_endpoints[endpoint_type] += "/{}".format(SWIFT_DATA_DIR) + + publicURL, internalURL, adminURL = (object_store_endpoints[t] + for t in ENDPOINT_TYPES) + # Update the relation to keystone to update the catalog URLs + update_endpoint_urls( + region, + publicURL, + internalURL, + adminURL, + ) + + +def get_service_endpoints(ksc, service_type, region_name): + """Get endpoints for a given service type from the Keystone catalog. + + :param ksc: An instance of a Keystone client. + :type ksc: :class: `keystoneclient.v3.client.Client` + :param str service_type: An endpoint service type to use. + :param str region_name: A name of the region to retrieve endpoints for. + :raises :class: `keystone_exceptions.EndpointNotFound` + """ try: catalog = { endpoint_type: ksc.service_catalog.url_for( - service_type='object-store', endpoint_type=endpoint_type) + service_type=service_type, endpoint_type=endpoint_type, + region_name=region_name) for endpoint_type in ['publicURL', 'internalURL', 'adminURL']} - except keystone_exceptions.EndpointNotFound as e: - log.warning("could not retrieve swift endpoint, not updating " - "product-streams endpoint: {}".format(e)) + except keystone_exceptions.EndpointNotFound: + log.error('could not retrieve any {} endpoints'.format(service_type)) raise + return catalog - for endpoint_type in ['publicURL', 'internalURL']: - catalog[endpoint_type] += "/{}".format(SWIFT_DATA_DIR) - # Update the relation to keystone to update the catalog URLs - update_endpoint_urls(region, catalog['publicURL'], - catalog['adminURL'], - catalog['internalURL']) +def juju_proxy_settings(): + """Get proxy settings from Juju environment. + + Get charm proxy settings from environment variables that correspond to + juju-http-proxy, juju-https-proxy juju-no-proxy (available as of 2.4.2, see + lp:1782236) or the legacy unprefixed settings. + + :rtype: None | dict[str, str] + """ + # Get proxy settings from the environment variables set by Juju. + juju_settings = { + m.groupdict()['var']: m.groupdict()['val'] + for m in re.finditer( + '^((JUJU_CHARM_)?(?P(HTTP|HTTPS|NO)_PROXY))=(?P.*)$', + juju_run_cmd(['env']), re.MULTILINE) + } + + proxy_settings = {} + for var in ['HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY']: + var_val = juju_settings.get(var) + if var_val: + proxy_settings[var] = var_val + proxy_settings[var.lower()] = var_val + return proxy_settings if proxy_settings else None def juju_run_cmd(cmd): @@ -430,7 +534,7 @@ def main(): log.info("Beginning image sync") status_set('maintenance', 'Synchronising images') - do_sync(charm_conf) + do_sync(ksc, charm_conf) ts = time.strftime("%x %X") # "Unit is ready" is one of approved message prefixes # Prefix the message with it will help zaza to understand the status. @@ -459,4 +563,5 @@ def main(): if __name__ == "__main__": + setup_file_logging() sys.exit(main()) diff --git a/hooks/hooks.py b/hooks/hooks.py index 2c768ff..b33cd97 100755 --- a/hooks/hooks.py +++ b/hooks/hooks.py @@ -174,6 +174,8 @@ class MirrorsConfigServiceContext(OSContextGenerator): name_prefix=config['name_prefix'], content_id_template=config['content_id_template'], use_swift=config['use_swift'], + ignore_proxy_for_object_store=config[ + 'ignore_proxy_for_object_store'], region=config['region'], cloud_name=config['cloud_name'], user_agent=config['user_agent'], diff --git a/requirements.txt b/requirements.txt index 360ecba..ead6e89 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ # requirements. They are intertwined. Also, Zaza itself should specify # all of its own requirements and if it doesn't, fix it there. # -pbr>=1.8.0,<1.9.0 +pbr==5.6.0 simplejson>=2.2.0 netifaces>=0.10.4 diff --git a/templates/mirrors.yaml b/templates/mirrors.yaml index 1298e0e..67c1e59 100644 --- a/templates/mirrors.yaml +++ b/templates/mirrors.yaml @@ -3,6 +3,7 @@ user_agent: {{ user_agent }} modify_hook_scripts: {{ modify_hook_scripts }} name_prefix: {{ name_prefix }} use_swift: {{ use_swift }} +ignore_proxy_for_object_store: {{ ignore_proxy_for_object_store }} region: {{ region }} cloud_name: {{ cloud_name }} content_id_template: {{ content_id_template }} diff --git a/test-requirements.txt b/test-requirements.txt index 9aea716..dba2c76 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,11 +8,6 @@ # all of its own requirements and if it doesn't, fix it there. # setuptools<50.0.0 # https://github.com/pypa/setuptools/commit/04e3df22df840c6bb244e9b27bc56750c44b7c85 -charm-tools>=2.4.4 - -# Workaround until https://github.com/juju/charm-tools/pull/589 gets -# published -keyring<21 requests>=2.18.4 @@ -21,7 +16,6 @@ requests>=2.18.4 mock>=1.2,<4.0.0; python_version < '3.6' mock>=1.2; python_version >= '3.6' -flake8>=2.2.4 stestr>=2.2.0 # Dependency of stestr. Workaround for @@ -42,7 +36,7 @@ oslo.utils<=3.41.0;python_version<'3.6' coverage>=4.5.2 pyudev # for ceph-* charm unit tests (need to fix the ceph-* charm unit tests/mocking) -git+https://github.com/openstack-charmers/zaza.git#egg=zaza;python_version>='3.0' +git+https://github.com/openstack-charmers/zaza.git#egg=zaza git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack # Needed for charm-glance: diff --git a/tox.ini b/tox.ini index ab9593f..9ba3f9f 100644 --- a/tox.ini +++ b/tox.ini @@ -65,8 +65,8 @@ deps = -r{toxinidir}/requirements.txt [testenv:pep8] basepython = python3 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt +deps = flake8==3.9.2 + charm-tools==2.8.3 commands = flake8 {posargs} hooks unit_tests tests actions lib files charm-proof diff --git a/unit_tests/test_glance_simplestreams_sync.py b/unit_tests/test_glance_simplestreams_sync.py new file mode 100644 index 0000000..ebf4852 --- /dev/null +++ b/unit_tests/test_glance_simplestreams_sync.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 + +''' +Copyright 2021 Canonical Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import files.glance_simplestreams_sync as gss +import mock +import unittest + +from keystoneclient import exceptions as keystone_exceptions + + +class TestGlanceSimpleStreamsSync(unittest.TestCase): + + def setUp(self): + self.maxDiff = 4096 + + @mock.patch('files.glance_simplestreams_sync.juju_run_cmd') + def test_proxy_settings(self, juju_run_cmd): + juju_run_cmd.return_value = ''' +LANG=C.UTF-8 +JUJU_CONTEXT_ID=glance-simplestreams-sync/0-run-commands-3325280900519425661 +JUJU_CHARM_HTTP_PROXY=http://squid.internal:3128 +JUJU_CHARM_HTTPS_PROXY=https://squid.internal:3128 +JUJU_CHARM_NO_PROXY=127.0.0.1,localhost,::1 +''' + self.assertEqual(gss.juju_proxy_settings(), { + "HTTP_PROXY": "http://squid.internal:3128", + "HTTPS_PROXY": "https://squid.internal:3128", + "NO_PROXY": "127.0.0.1,localhost,::1", + "http_proxy": "http://squid.internal:3128", + "https_proxy": "https://squid.internal:3128", + "no_proxy": "127.0.0.1,localhost,::1", + }) + + @mock.patch('files.glance_simplestreams_sync.juju_run_cmd') + def test_legacy_proxy_settings(self, juju_run_cmd): + juju_run_cmd.return_value = ''' +LANG=C.UTF-8 +JUJU_CONTEXT_ID=glance-simplestreams-sync/0-run-commands-3325280900519425661 +HTTP_PROXY=http://squid.internal:3128 +HTTPS_PROXY=https://squid.internal:3128 +NO_PROXY=127.0.0.1,localhost,::1 +''' + self.assertEqual(gss.juju_proxy_settings(), { + "HTTP_PROXY": "http://squid.internal:3128", + "HTTPS_PROXY": "https://squid.internal:3128", + "NO_PROXY": "127.0.0.1,localhost,::1", + "http_proxy": "http://squid.internal:3128", + "https_proxy": "https://squid.internal:3128", + "no_proxy": "127.0.0.1,localhost,::1", + }) + + @mock.patch('files.glance_simplestreams_sync.juju_run_cmd') + def test_proxy_settings_not_set(self, juju_run_cmd): + juju_run_cmd.return_value = ''' +LANG=C.UTF-8 +JUJU_CONTEXT_ID=glance-simplestreams-sync/0-run-commands-3325280900519425661 +''' + self.assertEqual(gss.juju_proxy_settings(), None) + + @mock.patch('files.glance_simplestreams_sync.get_service_endpoints') + @mock.patch('files.glance_simplestreams_sync.juju_proxy_settings') + def test_get_sstream_mirror_proxy_env(self, + juju_proxy_settings, + get_service_endpoints): + # Use a side effect instead of return value to avoid modification of + # the same dict in different invocations of the tested function. + def juju_proxy_settings_side_effect(): + return { + "HTTP_PROXY": "http://squid.internal:3128", + "HTTPS_PROXY": "https://squid.internal:3128", + "NO_PROXY": "127.0.0.1,localhost,::1", + "http_proxy": "http://squid.internal:3128", + "https_proxy": "https://squid.internal:3128", + "no_proxy": "127.0.0.1,localhost,::1", + } + + juju_proxy_settings.side_effect = juju_proxy_settings_side_effect + + def get_service_endpoints_side_effect(ksc, service_type, region_name): + return { + 'identity': { + 'publicURL': 'https://192.0.2.42:5000/v3', + 'internalURL': 'https://192.0.2.43:5000/v3', + 'adminURL': 'https://192.0.2.44:35357/v3', + }, + 'image': { + 'publicURL': 'https://192.0.2.45:9292', + 'internalURL': 'https://192.0.2.45:9292', + 'adminURL': 'https://192.0.2.47:9292', + }, + 'object-store': { + 'publicURL': 'https://192.0.2.90:443/swift/v1', + 'internalURL': 'https://192.0.2.90:443/swift/v1', + 'adminURL': 'https://192.0.2.90:443/swift', + }, + }[service_type] + + get_service_endpoints.side_effect = get_service_endpoints_side_effect + # Besides checking for proxy settings being set, make sure that + # object-store endpoints are added to NO_PROXY by default or when + # explicitly asked for. + for proxy_env in [ + gss.get_sstream_mirror_proxy_env( + mock.MagicMock(), 'TestRegion'), + gss.get_sstream_mirror_proxy_env( + mock.MagicMock(), 'TestRegion', + ignore_proxy_for_object_store=True)]: + self.assertEqual(proxy_env['HTTP_PROXY'], + 'http://squid.internal:3128') + self.assertEqual(proxy_env['http_proxy'], + 'http://squid.internal:3128') + self.assertEqual(proxy_env['HTTPS_PROXY'], + 'https://squid.internal:3128') + self.assertEqual(proxy_env['https_proxy'], + 'https://squid.internal:3128') + no_proxy_set = set(['127.0.0.1', 'localhost', '::1', '192.0.2.42', + '192.0.2.43', '192.0.2.44', '192.0.2.45', + '192.0.2.47', '192.0.2.90']) + self.assertEqual(set(proxy_env['NO_PROXY'].split(',')), + no_proxy_set) + self.assertEqual(set(proxy_env['no_proxy'].split(',')), + no_proxy_set) + + # Make sure that object-store endpoints are not included into + # NO_PROXY when this is explicitly being asked for. In this case + # the set of expected addresses in NO_PROXY should exclude 192.0.2.90. + proxy_env = gss.get_sstream_mirror_proxy_env( + mock.MagicMock(), + 'TestRegion', ignore_proxy_for_object_store=False) + self.assertEqual(proxy_env['HTTP_PROXY'], 'http://squid.internal:3128') + self.assertEqual(proxy_env['http_proxy'], 'http://squid.internal:3128') + self.assertEqual(proxy_env['HTTPS_PROXY'], + 'https://squid.internal:3128') + self.assertEqual(proxy_env['https_proxy'], + 'https://squid.internal:3128') + no_proxy_set_no_obj = set(['127.0.0.1', 'localhost', '::1', + '192.0.2.42', '192.0.2.43', '192.0.2.44', + '192.0.2.45', '192.0.2.47']) + self.assertEqual(set(proxy_env['NO_PROXY'].split(',')), + no_proxy_set_no_obj) + self.assertEqual(set(proxy_env['no_proxy'].split(',')), + no_proxy_set_no_obj) + + def no_juju_proxy_settings_side_effect(): + return None + + juju_proxy_settings.side_effect = no_juju_proxy_settings_side_effect + # Make sure that even if Juju does not have any proxy settings set, + # via the model, we are still adding endpoints to NO_PROXY for + # sstream-mirror-glance invocations because settings might be sourced + # from other files (see glance-simplestreams-sync.sh). + proxy_env = gss.get_sstream_mirror_proxy_env( + mock.MagicMock(), + 'TestRegion', ignore_proxy_for_object_store=False) + no_proxy_set_no_obj = set(['192.0.2.42', '192.0.2.43', '192.0.2.44', + '192.0.2.45', '192.0.2.47']) + self.assertEqual(set(proxy_env['NO_PROXY'].split(',')), + no_proxy_set_no_obj) + self.assertEqual(set(proxy_env['no_proxy'].split(',')), + no_proxy_set_no_obj) + + def test_get_service_endpoints(self): + + def url_for_side_effect(service_type, endpoint_type, region_name): + return { + 'TestRegion': { + 'identity': { + 'publicURL': 'https://10.5.2.42:443/swift/v1', + 'internalURL': 'https://10.5.2.42:443/swift/v1', + 'adminURL': 'https://10.5.2.42:443/swift/v1', + }, + 'image': { + 'publicURL': 'https://10.5.2.43:443/swift/v1', + 'internalURL': 'https://10.5.2.43:443/swift/v1', + 'adminURL': 'https://10.5.2.43:443/swift/v1', + }, + 'object-store': { + 'publicURL': 'https://10.5.2.44:443/swift/v1', + 'internalURL': 'https://10.5.2.44:443/swift/v1', + 'adminURL': 'https://10.5.2.44:443/swift/v1', + }, + } + }[region_name][service_type][endpoint_type] + + ksc = mock.MagicMock() + ksc.service_catalog.url_for.side_effect = url_for_side_effect + self.assertEqual( + gss.get_service_endpoints(ksc, 'identity', 'TestRegion'), { + 'publicURL': 'https://10.5.2.42:443/swift/v1', + 'internalURL': 'https://10.5.2.42:443/swift/v1', + 'adminURL': 'https://10.5.2.42:443/swift/v1', + } + ) + self.assertEqual( + gss.get_service_endpoints(ksc, 'image', 'TestRegion'), { + 'publicURL': 'https://10.5.2.43:443/swift/v1', + 'internalURL': 'https://10.5.2.43:443/swift/v1', + 'adminURL': 'https://10.5.2.43:443/swift/v1', + } + ) + self.assertEqual( + gss.get_service_endpoints(ksc, 'object-store', 'TestRegion'), { + 'publicURL': 'https://10.5.2.44:443/swift/v1', + 'internalURL': 'https://10.5.2.44:443/swift/v1', + 'adminURL': 'https://10.5.2.44:443/swift/v1', + } + ) + + ksc.service_catalog.url_for.side_effect = mock.MagicMock( + side_effect=keystone_exceptions.EndpointException('foo')) + + with self.assertRaises(keystone_exceptions.EndpointException): + gss.get_service_endpoints(ksc, 'test', 'TestRegion')