Dynamically generate proxy settings for image syncs

sstream-mirror-glance has several endpoints it needs to talk to:

* Image mirrors - typically, public Internet endpoints;
* Keystone - typically, a directly reachable endpoint;
* Glance - typically, a directly reachable endpoint;
* Object store (Swift) - typically, a directly reachable endpoint but
  sometimes it may be deployed externally and added to the region
  catalog in Keystone (in which case it might be accessible via a proxy
  only).

While sstream-mirror-glance does not support specifying proxy settings
for individual directions, since we know all of them based on the
Keystone catalog, a list of endpoints to add to NO_PROXY environment
variable can be generated dynamically.

The complication is that image syncs are periodically done via a cron
job so a juju-run invocation is needed to retrieve relevant proxy
settings from model-config at each invocation of the synchronization
script.

Additionally, the charm is long-lived so there may be some environments
that rely on legacy proxy settings. This change accounts for that and
acts both on juju-prefixed (new) and unprefixed (legacy) proxy settings.

Whether to use proxy settings for connections to the object store API
is controlled by a charm option which the script is made to react to.
Proxy settings are ignored for object store connections by default.

Closes-Bug: #1843486
Change-Id: Ib1fc5d2eebf43d5f98bb8ee405a3799802c8b8dc

config.yaml

@@ -23,6 +23,13 @@ options:
       on a local Apache server running on the unit and endpoints will be
       registered referencing the local unit.  This does not support HA
       or TLS and is for testing purposes only.
+  ignore_proxy_for_object_store:
+      type: boolean
+      default: true
+      description: |
+          Controls whether Juju model proxy settings are going to be used
+          by sstream-mirror-glance when connecting to object-store endpoints
+          from the Keystone catalog.
   frequency:
     type: string
     default: "daily"

files/glance_simplestreams_sync.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright 2014 Canonical Ltd.
 #
@@ -27,22 +27,28 @@ import atexit
 import base64
 import copy
 import fcntl
+import itertools
 import logging
 import os
+import re
 import shutil
 import six
-import sys
 import subprocess
+import sys
 import tempfile
 import time
 import yaml
 
+from keystoneclient import exceptions as keystone_exceptions
 from keystoneclient.v2_0 import client as keystone_client
 from keystoneclient.v3 import client as keystone_v3_client
-from keystoneclient import exceptions as keystone_exceptions
+if six.PY3:
+    from urllib import parse as urlparse
+else:
+    import urlparse
 
 
-def setup_logging():
+def setup_file_logging():
     logfilename = '/var/log/glance-simplestreams-sync.log'
 
     if not os.path.exists(logfilename):
@@ -60,10 +66,8 @@ def setup_logging():
     logger.setLevel('DEBUG')
     logger.addHandler(h)
 
-    return logger
 
-
+log = logging.getLogger()
-log = setup_logging()
 
 KEYRING = '/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg'
 CONF_FILE_DIR = '/etc/glance-simplestreams-sync'
@@ -95,6 +99,12 @@ SSTREAM_LOG_FILE = os.path.join(SSTREAM_SNAP_COMMON,
 CACERT_FILE = os.path.join(SSTREAM_SNAP_COMMON, 'cacert.pem')
 SYSTEM_CACERT_FILE = '/etc/ssl/certs/ca-certificates.crt'
 
+ENDPOINT_TYPES = [
+    'publicURL',
+    'adminURL',
+    'internalURL',
+]
+
 # TODOs:
 #   - allow people to specify their own policy, since they can specify
 #     their own mirrors.
@@ -226,7 +236,7 @@ def set_openstack_env(id_conf, charm_conf):
         os.environ['OS_TENANT_NAME'] = id_conf['admin_tenant_name']
 
 
-def do_sync(charm_conf):
+def do_sync(ksc, charm_conf):
 
     # NOTE(beisner): the user_agent variable was an unused assignment (lint).
     # It may be worth re-visiting its usage, intent and benefit with the
@@ -234,6 +244,8 @@ def do_sync(charm_conf):
     # and not assigning it since it is not currently utilized.
     # user_agent = charm_conf.get("user_agent")
 
+    region_name = charm_conf['region']
+
     for mirror_info in charm_conf['mirror_list']:
         # NOTE: output directory must be under HOME
         #       or snap cannot access it for stream files
@@ -241,7 +253,7 @@ def do_sync(charm_conf):
         try:
             log.info("Configuring sync for url {}".format(mirror_info))
             content_id = charm_conf['content_id_template'].format(
-                region=charm_conf['region'])
+                region=region_name)
 
             sync_command = [
                 "/snap/bin/simplestreams.sstream-mirror-glance",
@@ -284,43 +296,135 @@ def do_sync(charm_conf):
             ]
             sync_command += mirror_info['item_filters']
 
+            # Pass the current process' environment down along with proxy
+            # settings crafted for sstream-mirror-glance.
+            sstream_mirror_env = os.environ.copy()
+            sstream_mirror_env.update(get_sstream_mirror_proxy_env(
+                ksc, region_name,
+                charm_conf['ignore_proxy_for_object_store'],
+            ))
+
             log.info("calling sstream-mirror-glance")
-            log.debug("command: {}".format(" ".join(sync_command)))
+            log.debug("command: %s", " ".join(sync_command))
-            subprocess.check_call(sync_command)
+            log.debug("sstream-mirror environment: %s", sstream_mirror_env)
+            subprocess.check_call(sync_command, env=sstream_mirror_env)
 
             if not charm_conf['use_swift']:
                 # Sync output directory to APACHE_DATA_DIR
                 subprocess.check_call([
                     'rsync', '-avz',
-                    os.path.join(tmpdir, charm_conf['region'], 'streams'),
+                    os.path.join(tmpdir, region_name, 'streams'),
                     APACHE_DATA_DIR
                 ])
         finally:
             shutil.rmtree(tmpdir)
 
 
-def update_product_streams_service(ksc, services, region):
+def get_sstream_mirror_proxy_env(ksc, region_name,
-    """
+                                 ignore_proxy_for_object_store=True):
-    Updates URLs of product-streams endpoint to point to swift URLs.
+    '''Get proxy settings to be passed to sstreams-mirror-glance.
-    """
 
+    sstream-mirror-glance has multiple endpoints it needs to connect to:
+
+    1. Upstream image mirror (typically, an endpoint in public Internet);
+    2. Keystone (typically, a directly reachable endpoint);
+    3. Object storage (Swift) (typically a directly reachable endpoint).
+    4. Glance (typically, a directly reachable endpoint).
+
+    In a restricted environment where proxy settings have to be used for public
+    Internet connectivity we need to be explicit about hosts for which proxy
+    settings need to be used by sstream-mirror-glance. This function
+    dynamically builds a list of endpoints that need to be added to NO_PROXY
+    and optionally allows not including object storage endpoints into the
+    NO_PROXY list.
+
+    :param ksc: An instance of a Keystone client.
+    :type ksc: :class: `keystoneclient.v3.client.Client`
+    :param str region_name: A name of the region to retrieve endpoints for.
+    :param bool ignore_proxy_for_object_store: Do not include object-store
+                                               endpoints into NO_PROXY.
+    '''
+    proxy_settings = juju_proxy_settings()
+    if proxy_settings is None:
+        proxy_settings = {}
+        no_proxy_set = set()
+    else:
+        no_proxy_set = set(proxy_settings.get('NO_PROXY').split(','))
+    additional_hosts = set([
+        urlparse.urlparse(u).hostname for u in itertools.chain(
+            get_service_endpoints(ksc, 'identity', region_name).values(),
+            get_service_endpoints(ksc, 'image', region_name).values(),
+            get_service_endpoints(ksc, 'object-store', region_name).values()
+            if ignore_proxy_for_object_store else [],
+        )])
+    no_proxy = ','.join(no_proxy_set | additional_hosts)
+    proxy_settings['NO_PROXY'] = no_proxy
+    proxy_settings['no_proxy'] = no_proxy
+    return proxy_settings
+
+
+def update_product_streams_service(ksc, services, region):
+    """Updates URLs of product-streams endpoint to point to swift URLs."""
+    object_store_endpoints = get_service_endpoints(ksc, 'object-store', region)
+    for endpoint_type in ENDPOINT_TYPES:
+        object_store_endpoints[endpoint_type] += "/{}".format(SWIFT_DATA_DIR)
+
+    publicURL, internalURL, adminURL = (object_store_endpoints[t]
+                                        for t in ENDPOINT_TYPES)
+    # Update the relation to keystone to update the catalog URLs
+    update_endpoint_urls(
+        region,
+        publicURL,
+        internalURL,
+        adminURL,
+    )
+
+
+def get_service_endpoints(ksc, service_type, region_name):
+    """Get endpoints for a given service type from the Keystone catalog.
+
+    :param ksc: An instance of a Keystone client.
+    :type ksc: :class: `keystoneclient.v3.client.Client`
+    :param str service_type: An endpoint service type to use.
+    :param str region_name: A name of the region to retrieve endpoints for.
+    :raises :class: `keystone_exceptions.EndpointNotFound`
+    """
     try:
         catalog = {
             endpoint_type: ksc.service_catalog.url_for(
-                service_type='object-store', endpoint_type=endpoint_type)
+                service_type=service_type, endpoint_type=endpoint_type,
+                region_name=region_name)
             for endpoint_type in ['publicURL', 'internalURL', 'adminURL']}
-    except keystone_exceptions.EndpointNotFound as e:
+    except keystone_exceptions.EndpointNotFound:
-        log.warning("could not retrieve swift endpoint, not updating "
+        log.error('could not retrieve any {} endpoints'.format(service_type))
-                    "product-streams endpoint: {}".format(e))
         raise
+    return catalog
 
-    for endpoint_type in ['publicURL', 'internalURL']:
-        catalog[endpoint_type] += "/{}".format(SWIFT_DATA_DIR)
 
-    # Update the relation to keystone to update the catalog URLs
+def juju_proxy_settings():
-    update_endpoint_urls(region, catalog['publicURL'],
+    """Get proxy settings from Juju environment.
-                         catalog['adminURL'],
+
-                         catalog['internalURL'])
+    Get charm proxy settings from environment variables that correspond to
+    juju-http-proxy, juju-https-proxy juju-no-proxy (available as of 2.4.2, see
+    lp:1782236) or the legacy unprefixed settings.
+
+    :rtype: None | dict[str, str]
+    """
+    # Get proxy settings from the environment variables set by Juju.
+    juju_settings = {
+        m.groupdict()['var']: m.groupdict()['val']
+        for m in re.finditer(
+            '^((JUJU_CHARM_)?(?P<var>(HTTP|HTTPS|NO)_PROXY))=(?P<val>.*)$',
+            juju_run_cmd(['env']), re.MULTILINE)
+    }
+
+    proxy_settings = {}
+    for var in ['HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY']:
+        var_val = juju_settings.get(var)
+        if var_val:
+            proxy_settings[var] = var_val
+            proxy_settings[var.lower()] = var_val
+    return proxy_settings if proxy_settings else None
 
 
 def juju_run_cmd(cmd):
@@ -430,7 +534,7 @@ def main():
         log.info("Beginning image sync")
         status_set('maintenance', 'Synchronising images')
 
-        do_sync(charm_conf)
+        do_sync(ksc, charm_conf)
         ts = time.strftime("%x %X")
         # "Unit is ready" is one of approved message prefixes
         # Prefix the message with it will help zaza to understand the status.
@@ -459,4 +563,5 @@ def main():
 
 
 if __name__ == "__main__":
+    setup_file_logging()
     sys.exit(main())

hooks/hooks.py

@@ -174,6 +174,8 @@ class MirrorsConfigServiceContext(OSContextGenerator):
                     name_prefix=config['name_prefix'],
                     content_id_template=config['content_id_template'],
                     use_swift=config['use_swift'],
+                    ignore_proxy_for_object_store=config[
+                        'ignore_proxy_for_object_store'],
                     region=config['region'],
                     cloud_name=config['cloud_name'],
                     user_agent=config['user_agent'],

requirements.txt

@@ -7,7 +7,7 @@
 #       requirements.  They are intertwined.  Also, Zaza itself should specify
 #       all of its own requirements and if it doesn't, fix it there.
 #
-pbr>=1.8.0,<1.9.0
+pbr==5.6.0
 simplejson>=2.2.0
 netifaces>=0.10.4
 

templates/mirrors.yaml

@@ -3,6 +3,7 @@ user_agent: {{ user_agent }}
 modify_hook_scripts: {{ modify_hook_scripts }}
 name_prefix: {{ name_prefix }}
 use_swift: {{ use_swift }}
+ignore_proxy_for_object_store: {{ ignore_proxy_for_object_store }}
 region: {{ region }}
 cloud_name: {{ cloud_name }}
 content_id_template: {{ content_id_template }}

test-requirements.txt

@@ -8,11 +8,6 @@
 #       all of its own requirements and if it doesn't, fix it there.
 #
 setuptools<50.0.0  # https://github.com/pypa/setuptools/commit/04e3df22df840c6bb244e9b27bc56750c44b7c85
-charm-tools>=2.4.4
-
-# Workaround until https://github.com/juju/charm-tools/pull/589 gets
-# published
-keyring<21
 
 requests>=2.18.4
 
@@ -21,7 +16,6 @@ requests>=2.18.4
 mock>=1.2,<4.0.0; python_version < '3.6'
 mock>=1.2; python_version >= '3.6'
 
-flake8>=2.2.4
 stestr>=2.2.0
 
 # Dependency of stestr. Workaround for
@@ -42,7 +36,7 @@ oslo.utils<=3.41.0;python_version<'3.6'
 
 coverage>=4.5.2
 pyudev              # for ceph-* charm unit tests (need to fix the ceph-* charm unit tests/mocking)
-git+https://github.com/openstack-charmers/zaza.git#egg=zaza;python_version>='3.0'
+git+https://github.com/openstack-charmers/zaza.git#egg=zaza
 git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack
 
 # Needed for charm-glance:

tox.ini

@@ -65,8 +65,8 @@ deps = -r{toxinidir}/requirements.txt
 
 [testenv:pep8]
 basepython = python3
-deps = -r{toxinidir}/requirements.txt
+deps = flake8==3.9.2
-       -r{toxinidir}/test-requirements.txt
+       charm-tools==2.8.3
 commands = flake8 {posargs} hooks unit_tests tests actions lib files
            charm-proof
 

unit_tests/test_glance_simplestreams_sync.py

@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+
+'''
+Copyright 2021 Canonical Ltd.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import files.glance_simplestreams_sync as gss
+import mock
+import unittest
+
+from keystoneclient import exceptions as keystone_exceptions
+
+
+class TestGlanceSimpleStreamsSync(unittest.TestCase):
+
+    def setUp(self):
+        self.maxDiff = 4096
+
+    @mock.patch('files.glance_simplestreams_sync.juju_run_cmd')
+    def test_proxy_settings(self, juju_run_cmd):
+        juju_run_cmd.return_value = '''
+LANG=C.UTF-8
+JUJU_CONTEXT_ID=glance-simplestreams-sync/0-run-commands-3325280900519425661
+JUJU_CHARM_HTTP_PROXY=http://squid.internal:3128
+JUJU_CHARM_HTTPS_PROXY=https://squid.internal:3128
+JUJU_CHARM_NO_PROXY=127.0.0.1,localhost,::1
+'''
+        self.assertEqual(gss.juju_proxy_settings(), {
+            "HTTP_PROXY": "http://squid.internal:3128",
+            "HTTPS_PROXY": "https://squid.internal:3128",
+            "NO_PROXY": "127.0.0.1,localhost,::1",
+            "http_proxy": "http://squid.internal:3128",
+            "https_proxy": "https://squid.internal:3128",
+            "no_proxy": "127.0.0.1,localhost,::1",
+        })
+
+    @mock.patch('files.glance_simplestreams_sync.juju_run_cmd')
+    def test_legacy_proxy_settings(self, juju_run_cmd):
+        juju_run_cmd.return_value = '''
+LANG=C.UTF-8
+JUJU_CONTEXT_ID=glance-simplestreams-sync/0-run-commands-3325280900519425661
+HTTP_PROXY=http://squid.internal:3128
+HTTPS_PROXY=https://squid.internal:3128
+NO_PROXY=127.0.0.1,localhost,::1
+'''
+        self.assertEqual(gss.juju_proxy_settings(), {
+            "HTTP_PROXY": "http://squid.internal:3128",
+            "HTTPS_PROXY": "https://squid.internal:3128",
+            "NO_PROXY": "127.0.0.1,localhost,::1",
+            "http_proxy": "http://squid.internal:3128",
+            "https_proxy": "https://squid.internal:3128",
+            "no_proxy": "127.0.0.1,localhost,::1",
+        })
+
+    @mock.patch('files.glance_simplestreams_sync.juju_run_cmd')
+    def test_proxy_settings_not_set(self, juju_run_cmd):
+        juju_run_cmd.return_value = '''
+LANG=C.UTF-8
+JUJU_CONTEXT_ID=glance-simplestreams-sync/0-run-commands-3325280900519425661
+'''
+        self.assertEqual(gss.juju_proxy_settings(), None)
+
+    @mock.patch('files.glance_simplestreams_sync.get_service_endpoints')
+    @mock.patch('files.glance_simplestreams_sync.juju_proxy_settings')
+    def test_get_sstream_mirror_proxy_env(self,
+                                          juju_proxy_settings,
+                                          get_service_endpoints):
+        # Use a side effect instead of return value to avoid modification of
+        # the same dict in different invocations of the tested function.
+        def juju_proxy_settings_side_effect():
+            return {
+                "HTTP_PROXY": "http://squid.internal:3128",
+                "HTTPS_PROXY": "https://squid.internal:3128",
+                "NO_PROXY": "127.0.0.1,localhost,::1",
+                "http_proxy": "http://squid.internal:3128",
+                "https_proxy": "https://squid.internal:3128",
+                "no_proxy": "127.0.0.1,localhost,::1",
+            }
+
+        juju_proxy_settings.side_effect = juju_proxy_settings_side_effect
+
+        def get_service_endpoints_side_effect(ksc, service_type, region_name):
+            return {
+                'identity': {
+                    'publicURL': 'https://192.0.2.42:5000/v3',
+                    'internalURL': 'https://192.0.2.43:5000/v3',
+                    'adminURL': 'https://192.0.2.44:35357/v3',
+                },
+                'image': {
+                    'publicURL': 'https://192.0.2.45:9292',
+                    'internalURL': 'https://192.0.2.45:9292',
+                    'adminURL': 'https://192.0.2.47:9292',
+                },
+                'object-store': {
+                    'publicURL': 'https://192.0.2.90:443/swift/v1',
+                    'internalURL': 'https://192.0.2.90:443/swift/v1',
+                    'adminURL': 'https://192.0.2.90:443/swift',
+                },
+            }[service_type]
+
+        get_service_endpoints.side_effect = get_service_endpoints_side_effect
+        # Besides checking for proxy settings being set, make sure that
+        # object-store endpoints are added to NO_PROXY by default or when
+        # explicitly asked for.
+        for proxy_env in [
+                gss.get_sstream_mirror_proxy_env(
+                    mock.MagicMock(), 'TestRegion'),
+                gss.get_sstream_mirror_proxy_env(
+                    mock.MagicMock(), 'TestRegion',
+                    ignore_proxy_for_object_store=True)]:
+            self.assertEqual(proxy_env['HTTP_PROXY'],
+                             'http://squid.internal:3128')
+            self.assertEqual(proxy_env['http_proxy'],
+                             'http://squid.internal:3128')
+            self.assertEqual(proxy_env['HTTPS_PROXY'],
+                             'https://squid.internal:3128')
+            self.assertEqual(proxy_env['https_proxy'],
+                             'https://squid.internal:3128')
+            no_proxy_set = set(['127.0.0.1', 'localhost', '::1', '192.0.2.42',
+                                '192.0.2.43', '192.0.2.44', '192.0.2.45',
+                                '192.0.2.47', '192.0.2.90'])
+            self.assertEqual(set(proxy_env['NO_PROXY'].split(',')),
+                             no_proxy_set)
+            self.assertEqual(set(proxy_env['no_proxy'].split(',')),
+                             no_proxy_set)
+
+        # Make sure that object-store endpoints are not included into
+        # NO_PROXY when this is explicitly being asked for. In this case
+        # the set of expected addresses in NO_PROXY should exclude 192.0.2.90.
+        proxy_env = gss.get_sstream_mirror_proxy_env(
+            mock.MagicMock(),
+            'TestRegion', ignore_proxy_for_object_store=False)
+        self.assertEqual(proxy_env['HTTP_PROXY'], 'http://squid.internal:3128')
+        self.assertEqual(proxy_env['http_proxy'], 'http://squid.internal:3128')
+        self.assertEqual(proxy_env['HTTPS_PROXY'],
+                         'https://squid.internal:3128')
+        self.assertEqual(proxy_env['https_proxy'],
+                         'https://squid.internal:3128')
+        no_proxy_set_no_obj = set(['127.0.0.1', 'localhost', '::1',
+                                   '192.0.2.42', '192.0.2.43', '192.0.2.44',
+                                   '192.0.2.45', '192.0.2.47'])
+        self.assertEqual(set(proxy_env['NO_PROXY'].split(',')),
+                         no_proxy_set_no_obj)
+        self.assertEqual(set(proxy_env['no_proxy'].split(',')),
+                         no_proxy_set_no_obj)
+
+        def no_juju_proxy_settings_side_effect():
+            return None
+
+        juju_proxy_settings.side_effect = no_juju_proxy_settings_side_effect
+        # Make sure that even if Juju does not have any proxy settings set,
+        # via the model, we are still adding endpoints to NO_PROXY for
+        # sstream-mirror-glance invocations because settings might be sourced
+        # from other files (see glance-simplestreams-sync.sh).
+        proxy_env = gss.get_sstream_mirror_proxy_env(
+            mock.MagicMock(),
+            'TestRegion', ignore_proxy_for_object_store=False)
+        no_proxy_set_no_obj = set(['192.0.2.42', '192.0.2.43', '192.0.2.44',
+                                   '192.0.2.45', '192.0.2.47'])
+        self.assertEqual(set(proxy_env['NO_PROXY'].split(',')),
+                         no_proxy_set_no_obj)
+        self.assertEqual(set(proxy_env['no_proxy'].split(',')),
+                         no_proxy_set_no_obj)
+
+    def test_get_service_endpoints(self):
+
+        def url_for_side_effect(service_type, endpoint_type, region_name):
+            return {
+                'TestRegion': {
+                    'identity': {
+                        'publicURL': 'https://10.5.2.42:443/swift/v1',
+                        'internalURL': 'https://10.5.2.42:443/swift/v1',
+                        'adminURL': 'https://10.5.2.42:443/swift/v1',
+                    },
+                    'image': {
+                        'publicURL': 'https://10.5.2.43:443/swift/v1',
+                        'internalURL': 'https://10.5.2.43:443/swift/v1',
+                        'adminURL': 'https://10.5.2.43:443/swift/v1',
+                    },
+                    'object-store': {
+                        'publicURL': 'https://10.5.2.44:443/swift/v1',
+                        'internalURL': 'https://10.5.2.44:443/swift/v1',
+                        'adminURL': 'https://10.5.2.44:443/swift/v1',
+                    },
+                }
+            }[region_name][service_type][endpoint_type]
+
+        ksc = mock.MagicMock()
+        ksc.service_catalog.url_for.side_effect = url_for_side_effect
+        self.assertEqual(
+            gss.get_service_endpoints(ksc, 'identity', 'TestRegion'), {
+                'publicURL': 'https://10.5.2.42:443/swift/v1',
+                'internalURL': 'https://10.5.2.42:443/swift/v1',
+                'adminURL': 'https://10.5.2.42:443/swift/v1',
+            }
+        )
+        self.assertEqual(
+            gss.get_service_endpoints(ksc, 'image', 'TestRegion'), {
+                'publicURL': 'https://10.5.2.43:443/swift/v1',
+                'internalURL': 'https://10.5.2.43:443/swift/v1',
+                'adminURL': 'https://10.5.2.43:443/swift/v1',
+            }
+        )
+        self.assertEqual(
+            gss.get_service_endpoints(ksc, 'object-store', 'TestRegion'), {
+                'publicURL': 'https://10.5.2.44:443/swift/v1',
+                'internalURL': 'https://10.5.2.44:443/swift/v1',
+                'adminURL': 'https://10.5.2.44:443/swift/v1',
+            }
+        )
+
+        ksc.service_catalog.url_for.side_effect = mock.MagicMock(
+            side_effect=keystone_exceptions.EndpointException('foo'))
+
+        with self.assertRaises(keystone_exceptions.EndpointException):
+            gss.get_service_endpoints(ksc, 'test', 'TestRegion')