Update endpoint caches post network reconfig

After switching the management network to the admin network
we need to update the services endpoints cache of all
dcmanager, dcorch and cert-mon workers to the new admin value.

The fanout parameter is being added to the cast calls of the
RPC clients (dcorch and audit), this parameter makes the method
cast to all servers listening on a topic rather than just one
of them.

Test Plan:
PASS: dcmanager subcloud update (using admin network parameters)
1. Endpoints for the subcloud updated with admin ip value
2. subcloud availability = online
PASS: Verify that the subcloud is online shortly after succesful
completion of subcloud_update playbook
PASS: Verify that the service endpoints are updated in all workers'
endpoint caches for the subcloud
PASS: Manage the subcloud and verify that both dcmanager and dcorch
audits are working as expected
PASS: Perform a Identity sync:
1. openstack --os-region-name SystemController user create <new_user>
--domain <domain> --project <project> --password <password>
2. Log in into subcloud and verify the new user: openstack user list
PASS: Verify that the master token is refreshed successfully after an hour

Story: 2010319
Task: 47556

Depends-On: https://review.opendev.org/c/starlingx/config/+/877323

Signed-off-by: Hugo Brito <hugo.brito@windriver.com>
Change-Id: I149c864382b7c63d424f736bdb4eaac2a787b709

distributedcloud/dccommon/endpoint_cache.py

@@ -1,5 +1,5 @@
 # Copyright 2015 Huawei Technologies Co., Ltd.
-# Copyright (c) 2018-2021 Wind River Systems, Inc.
+# Copyright (c) 2018-2023 Wind River Systems, Inc.
 # All Rights Reserved
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -208,11 +208,16 @@ class EndpointCache(object):
         sess = session.Session(auth=auth)
         return sess
 
+    @lockutils.synchronized(LOCK_NAME)
+    def update_master_service_endpoint_region(self, region_name, endpoint_values):
+        EndpointCache.master_service_endpoint_map[region_name] = endpoint_values
+
     @lockutils.synchronized(LOCK_NAME)
     def get_cached_master_keystone_client_and_region_endpoint_map(self, region_name):
         if (EndpointCache.master_keystone_client is None):
             self._create_master_cached_data()
-            LOG.info("Generated Master keystone client and master token the very first time")
+            LOG.info("Generated Master keystone client and master token the "
+                     "very first time")
         else:
             token_expiring_soon = is_token_expiring_soon(token=EndpointCache.master_token)
 

distributedcloud/dccommon/tests/unit/test_endpoint_cache.py

@@ -1,5 +1,5 @@
 # Copyright 2015 Huawei Technologies Co., Ltd.
-# Copyright (c) 2017-2021 Wind River Systems, Inc.
+# Copyright (c) 2017-2023 Wind River Systems, Inc.
 # All Rights Reserved
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -16,6 +16,7 @@
 #
 
 import collections
+import copy
 import mock
 from mock import patch
 
@@ -34,11 +35,12 @@ FAKE_SUBCLOUD1_KEYSTONE_ENDPOINT = "https://[2620:10a:a001:ac05::7d02]:5001/v3"
 CENTRAL_REGION = "RegionOne"
 SUBCLOUD1_REGION = "subcloud1"
 
-FAKE_MASTER_SERVICE_ENDPOINT_MAP = \
-    {CENTRAL_REGION: {"sysinv": FAKE_REGIONONE_SYSINV_ENDPOINT,
-                      "keystone": FAKE_REGIONONE_KEYSTONE_ENDPOINT},
-     SUBCLOUD1_REGION: {"sysinv": FAKE_SUBCLOUD1_SYSINV_ENDPOINT,
-                        "keystone": FAKE_SUBCLOUD1_KEYSTONE_ENDPOINT}}
+FAKE_MASTER_SERVICE_ENDPOINT_MAP = {
+    CENTRAL_REGION: {"sysinv": FAKE_REGIONONE_SYSINV_ENDPOINT,
+                     "keystone": FAKE_REGIONONE_KEYSTONE_ENDPOINT},
+    SUBCLOUD1_REGION: {"sysinv": FAKE_SUBCLOUD1_SYSINV_ENDPOINT,
+                       "keystone": FAKE_SUBCLOUD1_KEYSTONE_ENDPOINT}
+}
 
 FAKE_SERVICE_ENDPOINT_MAP = {"sysinv": FAKE_REGIONONE_SYSINV_ENDPOINT,
                              "keystone": FAKE_REGIONONE_KEYSTONE_ENDPOINT}
@@ -139,3 +141,28 @@ class EndpointCacheTest(base.DCCommonTestCase):
         endpoint_cache.EndpointCache("RegionOne", None)
         services_list = endpoint_cache.EndpointCache.get_master_services_list()
         self.assertEqual(FAKE_SERVICES_LIST, services_list)
+
+    @patch.object(endpoint_cache.EndpointCache, 'get_admin_session')
+    @patch.object(tokens.TokenManager, 'validate')
+    @patch.object(endpoint_cache.EndpointCache,
+                  '_generate_master_service_endpoint_map')
+    def test_update_master_service_endpoint_region(
+            self, mock_generate_cached_data, mock_tokens_validate,
+            mock_admin_session):
+        mock_generate_cached_data.return_value = (
+            copy.deepcopy(FAKE_MASTER_SERVICE_ENDPOINT_MAP))
+        region_name = SUBCLOUD1_REGION
+        new_endpoints = {
+            'sysinv': 'https://[fake_ip]:6386/v1',
+            'keystone': 'https://[fake_ip]:5001/v3'
+        }
+        cache = endpoint_cache.EndpointCache("RegionOne", None)
+        self.assertEqual(
+            endpoint_cache.EndpointCache.master_service_endpoint_map,
+            FAKE_MASTER_SERVICE_ENDPOINT_MAP
+        )
+        cache.update_master_service_endpoint_region(region_name, new_endpoints)
+        self.assertNotEqual(
+            endpoint_cache.EndpointCache.master_service_endpoint_map,
+            FAKE_MASTER_SERVICE_ENDPOINT_MAP
+        )

distributedcloud/dcmanager/audit/rpcapi.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2021 Wind River Systems, Inc.
+# Copyright (c) 2020-2023 Wind River Systems, Inc.
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
 # a copy of the License at
@@ -80,6 +80,11 @@ class ManagerAuditClient(object):
         return self.cast(ctxt, self.make_msg('trigger_subcloud_patch_load_audits',
                                              subcloud_id=subcloud_id))
 
+    def trigger_subcloud_endpoints_update(self, ctxt, subcloud_name, endpoints):
+        return self.cast(ctxt, self.make_msg('trigger_subcloud_endpoints_update',
+                                             subcloud_name=subcloud_name,
+                                             endpoints=endpoints))
+
 
 class ManagerAuditWorkerClient(object):
     """Client side of the DC Manager Audit Worker rpc API.
@@ -109,10 +114,10 @@ class ManagerAuditWorkerClient(object):
             client = self._client
         return client.call(ctxt, method, **kwargs)
 
-    def cast(self, ctxt, msg, version=None):
+    def cast(self, ctxt, msg, fanout=None, version=None):
         method, kwargs = msg
-        if version is not None:
-            client = self._client.prepare(version=version)
+        if version or fanout:
+            client = self._client.prepare(fanout=fanout, version=version)
         else:
             client = self._client
         return client.cast(ctxt, method, **kwargs)
@@ -137,3 +142,9 @@ class ManagerAuditWorkerClient(object):
             kubernetes_audit_data=kubernetes_audit_data,
             do_openstack_audit=do_openstack_audit,
             kube_rootca_update_audit_data=kube_rootca_update_data))
+
+    def update_subcloud_endpoints(self, ctxt, subcloud_name, endpoints):
+        """Update endpoints of services for a subcloud region"""
+        return self.cast(ctxt, self.make_msg(
+            'update_subcloud_endpoints', subcloud_name=subcloud_name,
+            endpoints=endpoints), fanout=True, version=self.BASE_RPC_API_VERSION)

distributedcloud/dcmanager/audit/service.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2021 Wind River Systems, Inc.
+# Copyright (c) 2020-2023 Wind River Systems, Inc.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -158,6 +158,13 @@ class DCManagerAuditService(service.Service):
         return self.subcloud_audit_manager.trigger_subcloud_patch_load_audits(
             context, subcloud_id)
 
+    @request_context
+    def trigger_subcloud_endpoints_update(self, context, subcloud_name, endpoints):
+        """Trigger update endpoints of services for a subcloud region."""
+        LOG.info("Trigger update endpoints for subcloud %s", subcloud_name)
+        return self.subcloud_audit_manager.trigger_subcloud_endpoints_update(
+            context, subcloud_name, endpoints)
+
 
 class DCManagerAuditWorkerService(service.Service):
     """Lifecycle manager for a running audit service."""
@@ -255,3 +262,12 @@ class DCManagerAuditWorkerService(service.Service):
             kubernetes_audit_data,
             do_openstack_audit,
             kube_rootca_update_audit_data)
+
+    @request_context
+    def update_subcloud_endpoints(self, context, subcloud_name, endpoints):
+        """Update endpoints of services for a subcloud region"""
+        self.subcloud_audit_worker_manager.update_subcloud_endpoints(
+            context,
+            subcloud_name,
+            endpoints
+        )

distributedcloud/dcmanager/audit/subcloud_audit_manager.py

@@ -1,5 +1,5 @@
 # Copyright 2017 Ericsson AB.
-# Copyright (c) 2017-2022 Wind River Systems, Inc.
+# Copyright (c) 2017-2023 Wind River Systems, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -207,6 +207,11 @@ class SubcloudAuditManager(manager.Manager):
         }
         db_api.subcloud_audits_update(context, subcloud_id, values)
 
+    def trigger_subcloud_endpoints_update(self, context, subcloud_name, endpoints):
+        """Trigger update endpoints of services for a subcloud region."""
+        self.audit_worker_rpc_client.update_subcloud_endpoints(
+            context, subcloud_name, endpoints)
+
     def periodic_subcloud_audit(self):
         """Audit availability of subclouds."""
 

distributedcloud/dcmanager/audit/subcloud_audit_worker_manager.py

@@ -1,5 +1,5 @@
 # Copyright 2017 Ericsson AB.
-# Copyright (c) 2017-2022 Wind River Systems, Inc.
+# Copyright (c) 2017-2023 Wind River Systems, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -162,6 +162,20 @@ class SubcloudAuditWorkerManager(manager.Manager):
                                                 do_kubernetes_audit,
                                                 do_kube_rootca_update_audit)
 
+    def update_subcloud_endpoints(self, context, subcloud_name, endpoints):
+        try:
+            LOG.info("Updating service endpoints for subcloud %s "
+                     "in endpoint cache" % subcloud_name)
+            endpoint_cache = OpenStackDriver(
+                region_name=dccommon_consts.CLOUD_0).keystone_client.endpoint_cache
+            endpoint_cache.update_master_service_endpoint_region(
+                subcloud_name, endpoints)
+        except (keystone_exceptions.EndpointNotFound,
+                keystone_exceptions.ConnectFailure,
+                IndexError):
+            LOG.error("Failed to update the service endpoints "
+                      "for subcloud %s." % subcloud_name)
+
     def _update_subcloud_audit_fail_count(self, subcloud,
                                           audit_fail_count):
         """Update the subcloud's audit_fail_count directly to db.

distributedcloud/dcmanager/common/messaging.py

@@ -103,6 +103,11 @@ def get_rpc_client(timeout, **msg_target_kwargs):
                                     serializer=serializer)
 
 
+def get_transport():
+    """Return a configured oslo_messaging transport."""
+    return TRANSPORT
+
+
 def get_notifier(publisher_id):
     """Return a configured oslo_messaging notifier."""
     return NOTIFIER.prepare(publisher_id=publisher_id)

distributedcloud/dcmanager/manager/subcloud_manager.py

@@ -1804,7 +1804,7 @@ class SubcloudManager(manager.Manager):
             LOG.exception("Failed to create route to admin")
             return
         try:
-            self._update_services_endpoint(payload, m_ks_client)
+            self._update_services_endpoint(context, payload, m_ks_client)
         except Exception:
             LOG.exception("Failed to update endpoint %s" % subcloud_name)
             return
@@ -1826,35 +1826,47 @@ class SubcloudManager(manager.Manager):
                                        systemcontroller_gateway_ip,
                                        1)
 
-    def _update_services_endpoint(self, payload, m_ks_client):
+    def _update_services_endpoint(self, context, payload, m_ks_client):
         endpoint_ip = str(ipaddress.ip_network(payload.get('admin_subnet'))[2])
+        subcloud_name = payload.get('name')
         if netaddr.IPAddress(endpoint_ip).version == 6:
             endpoint_ip = '[' + endpoint_ip + ']'
 
+        services_endpoints = {
+            "keystone": "https://{}:5001/v3".format(endpoint_ip),
+            "sysinv": "https://{}:6386/v1".format(endpoint_ip),
+            "fm": "https://{}:18003".format(endpoint_ip),
+            "patching": "https://{}:5492".format(endpoint_ip),
+            "vim": "https://{}:4546".format(endpoint_ip)
+        }
+
         for endpoint in m_ks_client.keystone_client.endpoints.list(
-                region=payload['name']):
+                region=subcloud_name):
             service_type = m_ks_client.keystone_client.services.get(
                 endpoint.service_id).type
             if service_type == dccommon_consts.ENDPOINT_TYPE_PLATFORM:
-                admin_endpoint_url = "https://{}:6386/v1".format(endpoint_ip)
+                admin_endpoint_url = services_endpoints.get('sysinv')
             elif service_type == dccommon_consts.ENDPOINT_TYPE_IDENTITY:
-                admin_endpoint_url = "https://{}:5001/v3".format(endpoint_ip)
+                admin_endpoint_url = services_endpoints.get('keystone')
             elif service_type == dccommon_consts.ENDPOINT_TYPE_PATCHING:
-                admin_endpoint_url = "https://{}:5492".format(endpoint_ip)
+                admin_endpoint_url = services_endpoints.get('patching')
             elif service_type == dccommon_consts.ENDPOINT_TYPE_FM:
-                admin_endpoint_url = "https://{}:18003".format(endpoint_ip)
+                admin_endpoint_url = services_endpoints.get('fm')
             elif service_type == dccommon_consts.ENDPOINT_TYPE_NFV:
-                admin_endpoint_url = "https://{}:4546".format(endpoint_ip)
+                admin_endpoint_url = services_endpoints.get('vim')
             else:
                 LOG.exception("Endpoint Type Error: %s" % service_type)
             m_ks_client.keystone_client.endpoints.update(
                 endpoint, url=admin_endpoint_url)
-        # Clear the subcloud endpoint cache
-        OpenStackDriver(
-            region_name=dccommon_consts.DEFAULT_REGION_NAME,
-            region_clients=None
-        ).os_clients_dict[payload['name']] = collections.defaultdict(dict)
-        m_ks_client.endpoint_cache.re_initialize_master_keystone_client()
+        # Update service URLs in subcloud endpoint cache
+        self.audit_rpc_client.trigger_subcloud_endpoints_update(
+            context, subcloud_name, services_endpoints)
+        self.dcorch_rpc_client.update_subcloud_endpoints(
+            context, subcloud_name, services_endpoints)
+        # Update sysinv URL in cert-mon cache
+        dc_notification = dcmanager_rpc_client.DCManagerNotifications()
+        dc_notification.subcloud_sysinv_endpoint_update(
+            context, subcloud_name, services_endpoints.get("sysinv"))
 
     def _create_subcloud_update_overrides_file(
             self, payload, subcloud_name, filename_suffix):

distributedcloud/dcmanager/rpc/client.py

@@ -47,10 +47,10 @@ class RPCClient(object):
             client = self._client
         return client.call(ctxt, method, **kwargs)
 
-    def cast(self, ctxt, msg, version=None):
+    def cast(self, ctxt, msg, fanout=None, version=None):
         method, kwargs = msg
-        if version is not None:
-            client = self._client.prepare(version=version)
+        if fanout or version:
+            client = self._client.prepare(fanout=fanout, version=version)
         else:
             client = self._client
         return client.cast(ctxt, method, **kwargs)
@@ -210,3 +210,8 @@ class DCManagerNotifications(RPCClient):
     def subcloud_managed(self, ctxt, subcloud_name):
         return self.cast(ctxt, self.make_msg('subcloud_managed',
                                              subcloud_name=subcloud_name))
+
+    def subcloud_sysinv_endpoint_update(self, ctxt, subcloud_name, endpoint):
+        return self.cast(ctxt, self.make_msg(
+            'subcloud_sysinv_endpoint_update', subcloud_name=subcloud_name,
+            endpoint=endpoint), fanout=True, version=self.DCMANAGER_RPC_API_VERSION)

distributedcloud/dcmanager/tests/unit/audit/test_rpc_client.py

@@ -0,0 +1,80 @@
+# Copyright (c) 2023 Wind River Systems, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+import mock
+
+import oslo_messaging
+
+from dcmanager.audit import rpcapi as rpc_client
+from dcmanager.common import config
+from dcmanager.common import consts
+from dcmanager.common import messaging
+from dcmanager.tests import base
+from dcmanager.tests import utils
+
+config.register_options()
+
+
+class ManagerRpcAuditAPITestCase(base.DCManagerTestCase):
+
+    def setUp(self):
+        messaging.setup("fake://", optional=True)
+        self.addCleanup(messaging.cleanup)
+        self.context = utils.dummy_context()
+        super(ManagerRpcAuditAPITestCase, self).setUp()
+
+    def test_cast(self):
+        rpcapi = rpc_client.ManagerAuditWorkerClient()
+        transport = messaging.get_transport()
+        transport._send = mock.Mock()
+
+        fake_endpoints = {'service': 'fake_ip', 'service2': 'other_fake_ip'}
+
+        rpcapi.update_subcloud_endpoints(
+            self.context, 'subcloud', fake_endpoints)
+
+        exp_msg = {'method': 'update_subcloud_endpoints',
+                   'args': {'subcloud_name': 'subcloud',
+                            'endpoints': fake_endpoints},
+                   'version': '1.0'}
+
+        # With fanout a new target is created
+        new_target = oslo_messaging.Target(
+            fanout=True, version=rpcapi.BASE_RPC_API_VERSION,
+            topic=consts.TOPIC_DC_MANAGER_AUDIT_WORKER)
+        transport._send.assert_called_with(new_target,
+                                           mock.ANY,
+                                           exp_msg,
+                                           retry=None,
+                                           transport_options=None)
+
+        # Without fanout the target is the same
+        rpcapi.audit_subclouds(
+            self.context, ['subcloud1', 'subcloud2'],
+            True, False, True, True, False)
+
+        exp_msg2 = {'method': 'audit_subclouds',
+                    'args': {'subcloud_ids': ['subcloud1', 'subcloud2'],
+                             'patch_audit_data': True,
+                             'firmware_audit_data': False,
+                             'kubernetes_audit_data': True,
+                             'do_openstack_audit': True,
+                             'kube_rootca_update_audit_data': False},
+                    'version': '1.0'}
+
+        transport._send.assert_called_with(rpcapi._client.target,
+                                           mock.ANY,
+                                           exp_msg2,
+                                           retry=None,
+                                           transport_options=None)

distributedcloud/dcmanager/tests/unit/audit/test_subcloud_audit_manager.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2017-2022 Wind River Systems, Inc.
+# Copyright (c) 2017-2023 Wind River Systems, Inc.
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
 # a copy of the License at
@@ -30,6 +30,7 @@ class FakeAuditWorkerAPI(object):
 
     def __init__(self):
         self.audit_subclouds = mock.MagicMock()
+        self.update_subcloud_endpoints = mock.MagicMock()
 
 
 class FakePatchAudit(object):

distributedcloud/dcmanager/tests/unit/rpc/test_rpc_client.py

@@ -81,7 +81,7 @@ class ManagerRpcAPITestCase(base.DCManagerTestCase):
 
         # with version
         res = rpcapi.cast(self.context, msg, version='123')
-        client.prepare.assert_called_once_with(version='123')
+        client.prepare.assert_called_once_with(fanout=None, version='123')
         new_client = client.prepare.return_value
         new_client.cast.assert_called_once_with(self.context, 'fake_method',
                                                 key='value')

distributedcloud/dcorch/engine/generic_sync_manager.py

@@ -13,27 +13,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# Copyright (c) 2020-2022 Wind River Systems, Inc.
+# Copyright (c) 2020-2023 Wind River Systems, Inc.
 #
 
 import eventlet
 import collections  # noqa: H306
+import random
+
+from keystoneauth1 import exceptions as keystone_exceptions
 from oslo_log import log as logging
 from oslo_utils import timeutils
-import random
 
 from dccommon import consts as dccommon_consts
 from dcorch.common import consts as dco_consts
 from dcorch.common import context
 from dcorch.common import exceptions
 from dcorch.db import api as db_api
+from dcorch.drivers.openstack import sdk
 from dcorch.engine import scheduler
 from dcorch.engine import subcloud_lock
 from dcorch.engine.sync_services.identity import IdentitySyncThread
 from dcorch.engine.sync_services.sysinv import SysinvSyncThread
 from dcorch.objects import subcloud
 
-
 LOG = logging.getLogger(__name__)
 
 CHECK_AUDIT_INTERVAL = 300  # frequency to check for audit work
@@ -459,6 +461,20 @@ class GenericSyncManager(object):
         except KeyError:
             raise exceptions.SubcloudNotFound(region_name=subcloud_name)
 
+    def update_subcloud_endpoints(self, context, subcloud_name, endpoints):
+        try:
+            LOG.info("Updating service endpoints for subcloud %s in "
+                     "endpoint cache" % subcloud_name)
+            endpoint_cache = sdk.OpenStackDriver(
+                region_name=dccommon_consts.CLOUD_0).keystone_client.endpoint_cache
+            endpoint_cache.update_master_service_endpoint_region(
+                subcloud_name, endpoints)
+        except (keystone_exceptions.EndpointNotFound,
+                keystone_exceptions.ConnectFailure,
+                IndexError):
+            LOG.error("Failed to update services endpoints for "
+                      "subcloud: %s in dcorch." % subcloud_name)
+
     def initial_sync(self, context, subcloud_name):
         LOG.info('Initial sync subcloud %(sc)s %(id)s' %
                  {'sc': subcloud_name, 'id': self.engine_id})

distributedcloud/dcorch/engine/service.py

@@ -10,7 +10,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 #
-# Copyright (c) 2020-2022 Wind River Systems, Inc.
+# Copyright (c) 2020-2023 Wind River Systems, Inc.
 #
 
 import six
@@ -292,6 +292,10 @@ class EngineService(service.Service):
     def update_subcloud_version(self, ctxt, subcloud_name, sw_version):
         self.gsm.update_subcloud_version(ctxt, subcloud_name, sw_version)
 
+    @request_context
+    def update_subcloud_endpoints(self, ctxt, subcloud_name, endpoints):
+        self.gsm.update_subcloud_endpoints(ctxt, subcloud_name, endpoints)
+
     @request_context
     # The sync job info has been written to the DB, alert the sync engine
     # that there is work to do.

distributedcloud/dcorch/rpc/client.py

@@ -1,3 +1,4 @@
+# Copyright (c) 2017-2023 Wind River Systems, Inc.
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
 # a copy of the License at
@@ -48,10 +49,10 @@ class EngineClient(object):
             client = self._client
         return client.call(ctxt, method, **kwargs)
 
-    def cast(self, ctxt, msg, version=None):
+    def cast(self, ctxt, msg, fanout=None, version=None):
         method, kwargs = msg
-        if version is not None:
-            client = self._client.prepare(version=version)
+        if version or fanout:
+            client = self._client.prepare(fanout=fanout, version=version)
         else:
             client = self._client
         return client.cast(ctxt, method, **kwargs)
@@ -121,6 +122,11 @@ class EngineClient(object):
             self.make_msg('update_subcloud_version',
                           subcloud_name=subcloud_name, sw_version=sw_version))
 
+    def update_subcloud_endpoints(self, ctxt, subcloud_name, endpoints):
+        return self.cast(ctxt, self.make_msg(
+            'update_subcloud_endpoints', subcloud_name=subcloud_name,
+            endpoints=endpoints), fanout=True, version=self.BASE_RPC_API_VERSION)
+
     # The sync job info has been written to the DB, alert the sync engine
     # that there is work to do.
     def sync_request(self, ctxt, endpoint_type):