Browse Source

Handle compact services on multiple lines

This patch adds logic to handle compact service metadata that
has been split into multiple lines to avoid hitting the metadata
size limit.

Co-Authored-By: Grzegorz Grasza <xek@redhat.com>
Change-Id: Ida39f5768c67f982b2fe316f6fae4988a74c8534
Douglas Mendizábal 4 months ago
parent
commit
2c0091d23d
4 changed files with 106 additions and 32 deletions
  1. 14
    15
      novajoin/join.py
  2. 16
    16
      novajoin/notifications.py
  3. 43
    0
      novajoin/tests/unit/test_util.py
  4. 33
    1
      novajoin/util.py

+ 14
- 15
novajoin/join.py View File

@@ -12,7 +12,6 @@
12 12
 #    License for the specific language governing permissions and limitations
13 13
 #    under the License.
14 14
 
15
-import json
16 15
 import logging
17 16
 import traceback
18 17
 import uuid
@@ -26,7 +25,7 @@ from novajoin.glance import get_default_image_service
26 25
 from novajoin.ipa import IPAClient
27 26
 from novajoin import keystone_client
28 27
 from novajoin.nova import get_instance
29
-from novajoin.util import get_fqdn
28
+from novajoin import util
30 29
 
31 30
 
32 31
 CONF = cfg.CONF
@@ -200,7 +199,7 @@ class JoinController(Controller):
200 199
 
201 200
         ipaotp = uuid.uuid4().hex
202 201
 
203
-        data['hostname'] = get_fqdn(hostname_short, project_name)
202
+        data['hostname'] = util.get_fqdn(hostname_short, project_name)
204 203
         _, realm = self.ipaclient.get_host_and_realm()
205 204
         data['krb_realm'] = realm
206 205
 
@@ -220,10 +219,11 @@ class JoinController(Controller):
220 219
                             if key.startswith('managed_service_')]
221 220
         if managed_services:
222 221
             self.handle_services(data['hostname'], managed_services)
223
-        # compact json format
224
-        if 'compact_services' in metadata:
225
-            self.handle_compact_services(hostname_short,
226
-                                         metadata.get('compact_services'))
222
+
223
+        compact_services = util.get_compact_services(metadata)
224
+        if compact_services:
225
+            self.handle_compact_services(hostname_short, compact_services)
226
+
227 227
         self.ipaclient.flush_batch_operation()
228 228
 
229 229
         return data
@@ -250,13 +250,13 @@ class JoinController(Controller):
250 250
 
251 251
             self.ipaclient.service_add_host(principal, base_host)
252 252
 
253
-    def handle_compact_services(self, base_host_short, service_repr_json):
253
+    def handle_compact_services(self, base_host_short, service_repr):
254 254
         """Make any host/principal assignments passed from metadata
255 255
 
256
-        This takes a representation of the services and networks where the
257
-        services are listening on, and forms appropriate hostnames/service
258
-        principals based on this information. The representation looks as the
259
-        following:
256
+        This takes a dictionary representation of the services and networks
257
+        where the services are listening on, and forms appropriate
258
+        hostnames/service principals based on this information.
259
+        The dictionary representation looks as the following:
260 260
 
261 261
             {
262 262
                 "service1": [
@@ -286,15 +286,14 @@ class JoinController(Controller):
286 286
         """
287 287
         LOG.debug("In handle compact services")
288 288
 
289
-        service_repr = json.loads(service_repr_json)
290 289
         hosts_found = list()
291 290
         services_found = list()
292
-        base_host = get_fqdn(base_host_short)
291
+        base_host = util.get_fqdn(base_host_short)
293 292
 
294 293
         for service_name, net_list in service_repr.items():
295 294
             for network in net_list:
296 295
                 host_short = "%s.%s" % (base_host_short, network)
297
-                principal_host = get_fqdn(host_short)
296
+                principal_host = util.get_fqdn(host_short)
298 297
                 principal = "%s/%s" % (service_name, principal_host)
299 298
 
300 299
                 # add host if not present

+ 16
- 16
novajoin/notifications.py View File

@@ -17,13 +17,16 @@
17 17
 # notification_topic = notifications
18 18
 # notify_on_state_change = vm_state
19 19
 
20
-import json
21 20
 import sys
22 21
 import time
23 22
 
24 23
 import glanceclient as glance_client
25 24
 from neutronclient.v2_0 import client as neutron_client
26 25
 from novaclient import client as nova_client
26
+from oslo_log import log as logging
27
+import oslo_messaging
28
+from oslo_serialization import jsonutils
29
+
27 30
 from novajoin import config
28 31
 from novajoin import exception
29 32
 from novajoin.ipa import IPAClient
@@ -31,11 +34,7 @@ from novajoin import join
31 34
 from novajoin.keystone_client import get_session
32 35
 from novajoin.keystone_client import register_keystoneauth_opts
33 36
 from novajoin.nova import get_instance
34
-from novajoin.util import get_domain
35
-from novajoin.util import get_fqdn
36
-from oslo_log import log as logging
37
-import oslo_messaging
38
-from oslo_serialization import jsonutils
37
+from novajoin import util
39 38
 
40 39
 
41 40
 CONF = config.CONF
@@ -169,10 +168,12 @@ class NotificationEndpoint(object):
169 168
             if key.startswith('managed_service_')]
170 169
         if managed_services:
171 170
             join_controller.handle_services(hostname, managed_services)
172
-        # compact json format
173
-        if 'compact_services' in payload_metadata:
171
+
172
+        compact_services = util.get_compact_services(payload_metadata)
173
+        if compact_services:
174 174
             join_controller.handle_compact_services(
175
-                hostname_short, payload_metadata.get('compact_services'))
175
+                hostname_short, compact_services)
176
+
176 177
         ipa.flush_batch_operation()
177 178
 
178 179
     @event_handlers('compute.instance.delete.end')
@@ -262,16 +263,16 @@ class NotificationEndpoint(object):
262 263
         if metadata is None:
263 264
             return
264 265
 
265
-        if 'compact_services' in metadata:
266
+        compact_services = util.get_compact_services(metadata)
267
+        if compact_services:
266 268
             self.handle_compact_services(ipa, hostname_short,
267
-                                         metadata.get('compact_services'))
269
+                                         compact_services)
268 270
         managed_services = [metadata[key] for key in metadata.keys()
269 271
                             if key.startswith('managed_service_')]
270 272
         if managed_services:
271 273
             self.handle_managed_services(ipa, managed_services)
272 274
 
273
-    def handle_compact_services(self, ipa, host_short,
274
-                                service_repr_json):
275
+    def handle_compact_services(self, ipa, host_short, service_repr):
275 276
         """Reconstructs and removes subhosts for compact services.
276 277
 
277 278
            Data looks like this:
@@ -286,14 +287,13 @@ class NotificationEndpoint(object):
286 287
             integrity.
287 288
         """
288 289
         LOG.debug("In handle compact services")
289
-        service_repr = json.loads(service_repr_json)
290 290
         hosts_found = list()
291 291
 
292 292
         ipa.start_batch_operation()
293 293
         for service_name, net_list in service_repr.items():
294 294
             for network in net_list:
295 295
                 host = "%s.%s" % (host_short, network)
296
-                principal_host = get_fqdn(host)
296
+                principal_host = util.get_fqdn(host)
297 297
 
298 298
                 # remove host
299 299
                 if principal_host not in hosts_found:
@@ -330,7 +330,7 @@ class NotificationEndpoint(object):
330 330
     def _generate_hostname(self, hostname):
331 331
         # FIXME: Don't re-calculate the hostname, fetch it from somewhere
332 332
         project = 'foo'
333
-        domain = get_domain()
333
+        domain = util.get_domain()
334 334
         if CONF.project_subdomain:
335 335
             host = '%s.%s.%s' % (hostname, project, domain)
336 336
         else:

+ 43
- 0
novajoin/tests/unit/test_util.py View File

@@ -0,0 +1,43 @@
1
+# Copyright 2018 Red Hat, Inc.
2
+#
3
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
4
+#    not use this file except in compliance with the License. You may obtain
5
+#    a copy of the License at
6
+#
7
+#         http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+#    Unless required by applicable law or agreed to in writing, software
10
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12
+#    License for the specific language governing permissions and limitations
13
+#    under the License.
14
+
15
+"""
16
+Unit Tests for util functions
17
+"""
18
+
19
+import json
20
+import testtools
21
+
22
+from novajoin import util
23
+
24
+
25
+class TestUtil(testtools.TestCase):
26
+
27
+    def setUp(self):
28
+        super(TestUtil, self).setUp()
29
+
30
+    def test_get_compact_services(self):
31
+        result = {"http": ["internalapi", "ctlplane", "storage"],
32
+                  "rabbitmq": ["internalapi", "ctlplane"]}
33
+        old_metadata = {"compact_services": json.dumps(result)}
34
+        new_metadata = {
35
+            "compact_service_http": json.dumps(result['http']),
36
+            "compact_service_rabbitmq": json.dumps(result['rabbitmq'])}
37
+
38
+        self.assertDictEqual(util.get_compact_services(old_metadata), result)
39
+
40
+        self.assertDictEqual(util.get_compact_services(new_metadata), result)
41
+
42
+    def test_get_compact_services_empty(self):
43
+        self.assertIsNone(util.get_compact_services({}))

+ 33
- 1
novajoin/util.py View File

@@ -14,10 +14,13 @@
14 14
 
15 15
 """Utility functions shared between notify and server"""
16 16
 
17
-from novajoin.errors import ConfigurationError
17
+import json
18
+
18 19
 from oslo_config import cfg
19 20
 from oslo_log import log as logging
21
+import six
20 22
 
23
+from novajoin.errors import ConfigurationError
21 24
 from novajoin.ipa import ipalib_imported
22 25
 if ipalib_imported:
23 26
     from ipalib import api
@@ -54,3 +57,32 @@ def get_fqdn(hostname, project_name=None):
54 57
         return '%s.%s.%s' % (hostname, project_name, domain)
55 58
     else:
56 59
         return '%s.%s' % (hostname, domain)
60
+
61
+
62
+def get_compact_services(metadata):
63
+    """Retrieve and convert the compact_services from instance metadata.
64
+
65
+    This converts the new compact services format to the old/internal one.
66
+    The old format looks like:
67
+
68
+    "compact_services": {
69
+        "http": ["internalapi", "ctlplane", "storage"],
70
+        "rabbitmq": ["internalapi", "ctlplane"]
71
+    }
72
+
73
+    The new format contains service names inside the primary key:
74
+
75
+    "compact_services_http": ["internalapi", "ctlplane", "storage"],
76
+    "compact_services_rabbitmq": ["internalapi", "ctlplane"]
77
+    """
78
+    # compact key-per-service
79
+    compact_services = {key.split('_', 2)[-1]: json.loads(value)
80
+                        for key, value in six.iteritems(metadata)
81
+                        if key.startswith('compact_service_')}
82
+    if compact_services:
83
+        return compact_services
84
+    # legacy compact json format
85
+    if 'compact_services' in metadata:
86
+        return json.loads(metadata['compact_services'])
87
+
88
+    return None

Loading…
Cancel
Save