Browse Source

Support message queue clusters in inter-cell communication

Since cells use oslo.messaging to specify and store the message queue URL,
multiple hosts can be specified by manually modifying that URL in the database.
However, there is no way to specify multiple hosts during cell creation phase.
This patch adds a --broker_hosts option to `nova-manage cell create` command,
which is analogous to the rabbit_hosts option in nova.conf and can be used to
specify multiple message queue servers as a comma separated list. Each server
is specified using hostname:port with both being mandatory. The existing
--hostname and --port options continue to remain but are only considered if no
--broker_hosts is specified.
Internally, each host is converted to a oslo.messaging.TransportHost
and added to the generated TransportURL.
This patch also adds unit tests for creation of the TransportHosts from
user given input.

Change-Id: I14de860b1d12f3e2c0169b58651d580792d6ce0e
Closes-Bug: 1178541
tags/2015.1.0b1
Dheeraj Gupta 4 years ago
parent
commit
cc17991d1a
2 changed files with 171 additions and 8 deletions
  1. 52
    8
      nova/cmd/manage.py
  2. 119
    0
      nova/tests/test_nova_manage.py

+ 52
- 8
nova/cmd/manage.py View File

@@ -1059,6 +1059,46 @@ class GetLogCommands(object):
1059 1059
 class CellCommands(object):
1060 1060
     """Commands for managing cells."""
1061 1061
 
1062
+    def _create_transport_hosts(self, username, password,
1063
+                                broker_hosts=None, hostname=None, port=None):
1064
+        """Returns a list of oslo.messaging.TransportHost objects."""
1065
+        transport_hosts = []
1066
+        # Either broker-hosts or hostname should be set
1067
+        if broker_hosts:
1068
+            hosts = broker_hosts.split(',')
1069
+            for host in hosts:
1070
+                host = host.strip()
1071
+                broker_hostname, broker_port = utils.parse_server_string(host)
1072
+                if not broker_port:
1073
+                    msg = _('Invalid broker_hosts value: %s. It should be'
1074
+                            ' in hostname:port format') % host
1075
+                    raise ValueError(msg)
1076
+                try:
1077
+                    broker_port = int(broker_port)
1078
+                except ValueError:
1079
+                    msg = _('Invalid port value: %s. It should be '
1080
+                             'an integer') % broker_port
1081
+                    raise ValueError(msg)
1082
+                transport_hosts.append(
1083
+                               messaging.TransportHost(
1084
+                                   hostname=broker_hostname,
1085
+                                   port=broker_port,
1086
+                                   username=username,
1087
+                                   password=password))
1088
+        else:
1089
+            try:
1090
+                port = int(port)
1091
+            except ValueError:
1092
+                msg = _("Invalid port value: %s. Should be an integer") % port
1093
+                raise ValueError(msg)
1094
+            transport_hosts.append(
1095
+                           messaging.TransportHost(
1096
+                               hostname=hostname,
1097
+                               port=port,
1098
+                               username=username,
1099
+                               password=password))
1100
+        return transport_hosts
1101
+
1062 1102
     @args('--name', metavar='<name>', help='Name for the new cell')
1063 1103
     @args('--cell_type', metavar='<parent|child>',
1064 1104
          help='Whether the cell is a parent or child')
@@ -1066,6 +1106,11 @@ class CellCommands(object):
1066 1106
          help='Username for the message broker in this cell')
1067 1107
     @args('--password', metavar='<password>',
1068 1108
          help='Password for the message broker in this cell')
1109
+    @args('--broker_hosts', metavar='<broker_hosts>',
1110
+         help='Comma separated list of message brokers in this cell. '
1111
+              'Each Broker is specified as hostname:port with both '
1112
+              'mandatory. This option overrides the --hostname '
1113
+              'and --port options (if provided). ')
1069 1114
     @args('--hostname', metavar='<hostname>',
1070 1115
          help='Address of the message broker in this cell')
1071 1116
     @args('--port', metavar='<number>',
@@ -1074,8 +1119,8 @@ class CellCommands(object):
1074 1119
          help='The virtual host of the message broker in this cell')
1075 1120
     @args('--woffset', metavar='<float>')
1076 1121
     @args('--wscale', metavar='<float>')
1077
-    def create(self, name, cell_type='child', username=None, password=None,
1078
-               hostname=None, port=None, virtual_host=None,
1122
+    def create(self, name, cell_type='child', username=None, broker_hosts=None,
1123
+               password=None, hostname=None, port=None, virtual_host=None,
1079 1124
                woffset=None, wscale=None):
1080 1125
 
1081 1126
         if cell_type not in ['parent', 'child']:
@@ -1083,13 +1128,12 @@ class CellCommands(object):
1083 1128
             return(2)
1084 1129
 
1085 1130
         # Set up the transport URL
1086
-        transport_host = messaging.TransportHost(hostname=hostname,
1087
-                                                 port=int(port),
1088
-                                                 username=username,
1089
-                                                 password=password)
1090
-
1131
+        transport_hosts = self._create_transport_hosts(
1132
+                                                 username, password,
1133
+                                                 broker_hosts, hostname,
1134
+                                                 port)
1091 1135
         transport_url = rpc.get_transport_url()
1092
-        transport_url.hosts.append(transport_host)
1136
+        transport_url.hosts.extend(transport_hosts)
1093 1137
         transport_url.virtual_host = virtual_host
1094 1138
 
1095 1139
         is_parent = cell_type == 'parent'

+ 119
- 0
nova/tests/test_nova_manage.py View File

@@ -17,6 +17,7 @@ import StringIO
17 17
 import sys
18 18
 
19 19
 import fixtures
20
+import mock
20 21
 
21 22
 from nova.cmd import manage
22 23
 from nova import context
@@ -346,3 +347,121 @@ class ServiceCommandsTestCase(test.TestCase):
346 347
 
347 348
     def test_service_disable_invalid_params(self):
348 349
         self.assertEqual(2, self.commands.disable('nohost', 'noservice'))
350
+
351
+
352
+class CellCommandsTestCase(test.TestCase):
353
+    def setUp(self):
354
+        super(CellCommandsTestCase, self).setUp()
355
+        self.commands = manage.CellCommands()
356
+
357
+    def test_create_transport_hosts_multiple(self):
358
+        """Test the _create_transport_hosts method
359
+        when broker_hosts is set.
360
+        """
361
+        brokers = "127.0.0.1:5672,127.0.0.2:5671"
362
+        thosts = self.commands._create_transport_hosts(
363
+                                           'guest', 'devstack',
364
+                                           broker_hosts=brokers)
365
+        self.assertEqual(2, len(thosts))
366
+        self.assertEqual('127.0.0.1', thosts[0].hostname)
367
+        self.assertEqual(5672, thosts[0].port)
368
+        self.assertEqual('127.0.0.2', thosts[1].hostname)
369
+        self.assertEqual(5671, thosts[1].port)
370
+
371
+    def test_create_transport_hosts_single(self):
372
+        """Test the _create_transport_hosts method when hostname is passed."""
373
+        thosts = self.commands._create_transport_hosts('guest', 'devstack',
374
+                                                       hostname='127.0.0.1',
375
+                                                       port=80)
376
+        self.assertEqual(1, len(thosts))
377
+        self.assertEqual('127.0.0.1', thosts[0].hostname)
378
+        self.assertEqual(80, thosts[0].port)
379
+
380
+    def test_create_transport_hosts_single_broker(self):
381
+        """Test the _create_transport_hosts method for single broker_hosts."""
382
+        thosts = self.commands._create_transport_hosts(
383
+                                              'guest', 'devstack',
384
+                                              broker_hosts='127.0.0.1:5672')
385
+        self.assertEqual(1, len(thosts))
386
+        self.assertEqual('127.0.0.1', thosts[0].hostname)
387
+        self.assertEqual(5672, thosts[0].port)
388
+
389
+    def test_create_transport_hosts_both(self):
390
+        """Test the _create_transport_hosts method when both broker_hosts
391
+        and hostname/port are passed.
392
+        """
393
+        thosts = self.commands._create_transport_hosts(
394
+                                              'guest', 'devstack',
395
+                                              broker_hosts='127.0.0.1:5672',
396
+                                              hostname='127.0.0.2', port=80)
397
+        self.assertEqual(1, len(thosts))
398
+        self.assertEqual('127.0.0.1', thosts[0].hostname)
399
+        self.assertEqual(5672, thosts[0].port)
400
+
401
+    def test_create_transport_hosts_wrong_val(self):
402
+        """Test the _create_transport_hosts method when broker_hosts
403
+        is wrongly sepcified
404
+        """
405
+        self.assertRaises(ValueError,
406
+                          self.commands._create_transport_hosts,
407
+                          'guest', 'devstack',
408
+                          broker_hosts='127.0.0.1:5672,127.0.0.1')
409
+
410
+    def test_create_transport_hosts_wrong_port_val(self):
411
+        """Test the _create_transport_hosts method when port in
412
+        broker_hosts is wrongly sepcified
413
+        """
414
+        self.assertRaises(ValueError,
415
+                          self.commands._create_transport_hosts,
416
+                          'guest', 'devstack',
417
+                          broker_hosts='127.0.0.1:')
418
+
419
+    def test_create_transport_hosts_wrong_port_arg(self):
420
+        """Test the _create_transport_hosts method when port
421
+        argument is wrongly sepcified
422
+        """
423
+        self.assertRaises(ValueError,
424
+                          self.commands._create_transport_hosts,
425
+                          'guest', 'devstack',
426
+                          hostname='127.0.0.1', port='ab')
427
+
428
+    @mock.patch.object(context, 'get_admin_context')
429
+    @mock.patch.object(db, 'cell_create')
430
+    def test_create_broker_hosts(self, mock_db_cell_create, mock_ctxt):
431
+        """Test the create function when broker_hosts is
432
+        passed
433
+        """
434
+        cell_tp_url = "fake://guest:devstack@127.0.0.1:5432"
435
+        cell_tp_url += ",guest:devstack@127.0.0.2:9999/"
436
+        ctxt = mock.sentinel
437
+        mock_ctxt.return_value = mock.sentinel
438
+        self.commands.create("test",
439
+                             broker_hosts='127.0.0.1:5432,127.0.0.2:9999',
440
+                             woffset=0, wscale=0,
441
+                             username="guest", password="devstack")
442
+        exp_values = {'name': "test",
443
+                      'is_parent': False,
444
+                      'transport_url': cell_tp_url,
445
+                      'weight_offset': 0.0,
446
+                      'weight_scale': 0.0}
447
+        mock_db_cell_create.assert_called_once_with(ctxt, exp_values)
448
+
449
+    @mock.patch.object(context, 'get_admin_context')
450
+    @mock.patch.object(db, 'cell_create')
451
+    def test_create_hostname(self, mock_db_cell_create, mock_ctxt):
452
+        """Test the create function when hostname and port is
453
+        passed
454
+        """
455
+        cell_tp_url = "fake://guest:devstack@127.0.0.1:9999/"
456
+        ctxt = mock.sentinel
457
+        mock_ctxt.return_value = mock.sentinel
458
+        self.commands.create("test",
459
+                             hostname='127.0.0.1', port="9999",
460
+                             woffset=0, wscale=0,
461
+                             username="guest", password="devstack")
462
+        exp_values = {'name': "test",
463
+                      'is_parent': False,
464
+                      'transport_url': cell_tp_url,
465
+                      'weight_offset': 0.0,
466
+                      'weight_scale': 0.0}
467
+        mock_db_cell_create.assert_called_once_with(ctxt, exp_values)

Loading…
Cancel
Save