Handle compact services on multiple lines

This patch adds logic to handle compact service metadata that
has been split into multiple lines to avoid hitting the metadata
size limit.

Co-Authored-By: Grzegorz Grasza <xek@redhat.com>
Change-Id: Ida39f5768c67f982b2fe316f6fae4988a74c8534

novajoin/join.py

@@ -12,7 +12,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import json
 import logging
 import traceback
 import uuid
@@ -26,7 +25,7 @@ from novajoin.glance import get_default_image_service
 from novajoin.ipa import IPAClient
 from novajoin import keystone_client
 from novajoin.nova import get_instance
-from novajoin.util import get_fqdn
+from novajoin import util
 
 
 CONF = cfg.CONF
@@ -200,7 +199,7 @@ class JoinController(Controller):
 
         ipaotp = uuid.uuid4().hex
 
-        data['hostname'] = get_fqdn(hostname_short, project_name)
+        data['hostname'] = util.get_fqdn(hostname_short, project_name)
         _, realm = self.ipaclient.get_host_and_realm()
         data['krb_realm'] = realm
 
@@ -220,10 +219,11 @@ class JoinController(Controller):
                             if key.startswith('managed_service_')]
         if managed_services:
             self.handle_services(data['hostname'], managed_services)
-        # compact json format
+
-        if 'compact_services' in metadata:
+        compact_services = util.get_compact_services(metadata)
-            self.handle_compact_services(hostname_short,
+        if compact_services:
-                                         metadata.get('compact_services'))
+            self.handle_compact_services(hostname_short, compact_services)
+
         self.ipaclient.flush_batch_operation()
 
         return data
@@ -250,13 +250,13 @@ class JoinController(Controller):
 
             self.ipaclient.service_add_host(principal, base_host)
 
-    def handle_compact_services(self, base_host_short, service_repr_json):
+    def handle_compact_services(self, base_host_short, service_repr):
         """Make any host/principal assignments passed from metadata
 
-        This takes a representation of the services and networks where the
+        This takes a dictionary representation of the services and networks
-        services are listening on, and forms appropriate hostnames/service
+        where the services are listening on, and forms appropriate
-        principals based on this information. The representation looks as the
+        hostnames/service principals based on this information.
-        following:
+        The dictionary representation looks as the following:
 
             {
                 "service1": [
@@ -286,15 +286,14 @@ class JoinController(Controller):
         """
         LOG.debug("In handle compact services")
 
-        service_repr = json.loads(service_repr_json)
         hosts_found = list()
         services_found = list()
-        base_host = get_fqdn(base_host_short)
+        base_host = util.get_fqdn(base_host_short)
 
         for service_name, net_list in service_repr.items():
             for network in net_list:
                 host_short = "%s.%s" % (base_host_short, network)
-                principal_host = get_fqdn(host_short)
+                principal_host = util.get_fqdn(host_short)
                 principal = "%s/%s" % (service_name, principal_host)
 
                 # add host if not present

novajoin/notifications.py

@@ -17,13 +17,16 @@
 # notification_topic = notifications
 # notify_on_state_change = vm_state
 
-import json
 import sys
 import time
 
 import glanceclient as glance_client
 from neutronclient.v2_0 import client as neutron_client
 from novaclient import client as nova_client
+from oslo_log import log as logging
+import oslo_messaging
+from oslo_serialization import jsonutils
+
 from novajoin import config
 from novajoin import exception
 from novajoin.ipa import IPAClient
@@ -31,11 +34,7 @@ from novajoin import join
 from novajoin.keystone_client import get_session
 from novajoin.keystone_client import register_keystoneauth_opts
 from novajoin.nova import get_instance
-from novajoin.util import get_domain
+from novajoin import util
-from novajoin.util import get_fqdn
-from oslo_log import log as logging
-import oslo_messaging
-from oslo_serialization import jsonutils
 
 
 CONF = config.CONF
@@ -169,10 +168,12 @@ class NotificationEndpoint(object):
             if key.startswith('managed_service_')]
         if managed_services:
             join_controller.handle_services(hostname, managed_services)
-        # compact json format
+
-        if 'compact_services' in payload_metadata:
+        compact_services = util.get_compact_services(payload_metadata)
+        if compact_services:
             join_controller.handle_compact_services(
-                hostname_short, payload_metadata.get('compact_services'))
+                hostname_short, compact_services)
+
         ipa.flush_batch_operation()
 
     @event_handlers('compute.instance.delete.end')
@@ -262,16 +263,16 @@ class NotificationEndpoint(object):
         if metadata is None:
             return
 
-        if 'compact_services' in metadata:
+        compact_services = util.get_compact_services(metadata)
+        if compact_services:
             self.handle_compact_services(ipa, hostname_short,
-                                         metadata.get('compact_services'))
+                                         compact_services)
         managed_services = [metadata[key] for key in metadata.keys()
                             if key.startswith('managed_service_')]
         if managed_services:
             self.handle_managed_services(ipa, managed_services)
 
-    def handle_compact_services(self, ipa, host_short,
+    def handle_compact_services(self, ipa, host_short, service_repr):
-                                service_repr_json):
         """Reconstructs and removes subhosts for compact services.
 
            Data looks like this:
@@ -286,14 +287,13 @@ class NotificationEndpoint(object):
             integrity.
         """
         LOG.debug("In handle compact services")
-        service_repr = json.loads(service_repr_json)
         hosts_found = list()
 
         ipa.start_batch_operation()
         for service_name, net_list in service_repr.items():
             for network in net_list:
                 host = "%s.%s" % (host_short, network)
-                principal_host = get_fqdn(host)
+                principal_host = util.get_fqdn(host)
 
                 # remove host
                 if principal_host not in hosts_found:
@@ -330,7 +330,7 @@ class NotificationEndpoint(object):
     def _generate_hostname(self, hostname):
         # FIXME: Don't re-calculate the hostname, fetch it from somewhere
         project = 'foo'
-        domain = get_domain()
+        domain = util.get_domain()
         if CONF.project_subdomain:
             host = '%s.%s.%s' % (hostname, project, domain)
         else:

novajoin/tests/unit/test_util.py

@@ -0,0 +1,43 @@
+# Copyright 2018 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Unit Tests for util functions
+"""
+
+import json
+import testtools
+
+from novajoin import util
+
+
+class TestUtil(testtools.TestCase):
+
+    def setUp(self):
+        super(TestUtil, self).setUp()
+
+    def test_get_compact_services(self):
+        result = {"http": ["internalapi", "ctlplane", "storage"],
+                  "rabbitmq": ["internalapi", "ctlplane"]}
+        old_metadata = {"compact_services": json.dumps(result)}
+        new_metadata = {
+            "compact_service_http": json.dumps(result['http']),
+            "compact_service_rabbitmq": json.dumps(result['rabbitmq'])}
+
+        self.assertDictEqual(util.get_compact_services(old_metadata), result)
+
+        self.assertDictEqual(util.get_compact_services(new_metadata), result)
+
+    def test_get_compact_services_empty(self):
+        self.assertIsNone(util.get_compact_services({}))

novajoin/util.py

@@ -14,10 +14,13 @@
 
 """Utility functions shared between notify and server"""
 
-from novajoin.errors import ConfigurationError
+import json
+
 from oslo_config import cfg
 from oslo_log import log as logging
+import six
 
+from novajoin.errors import ConfigurationError
 from novajoin.ipa import ipalib_imported
 if ipalib_imported:
     from ipalib import api
@@ -54,3 +57,32 @@ def get_fqdn(hostname, project_name=None):
         return '%s.%s.%s' % (hostname, project_name, domain)
     else:
         return '%s.%s' % (hostname, domain)
+
+
+def get_compact_services(metadata):
+    """Retrieve and convert the compact_services from instance metadata.
+
+    This converts the new compact services format to the old/internal one.
+    The old format looks like:
+
+    "compact_services": {
+        "http": ["internalapi", "ctlplane", "storage"],
+        "rabbitmq": ["internalapi", "ctlplane"]
+    }
+
+    The new format contains service names inside the primary key:
+
+    "compact_services_http": ["internalapi", "ctlplane", "storage"],
+    "compact_services_rabbitmq": ["internalapi", "ctlplane"]
+    """
+    # compact key-per-service
+    compact_services = {key.split('_', 2)[-1]: json.loads(value)
+                        for key, value in six.iteritems(metadata)
+                        if key.startswith('compact_service_')}
+    if compact_services:
+        return compact_services
+    # legacy compact json format
+    if 'compact_services' in metadata:
+        return json.loads(metadata['compact_services'])
+
+    return None