Perform rolling upgrade of payload

Make use of the coordinator layer to ensure payload upgrades
happen at one unit at a time, starting with the charm leader.

Closes-Bug: #1902562
Change-Id: I6abc20651ba6f81172b0193d22fdde23e6d38ed0

src/layer.yaml

@@ -1,6 +1,7 @@
 includes:
   - layer:openstack
   - layer:leadership
+  - layer:coordinator
   - interface:nrpe-external-master
   - interface:ovsdb
   - interface:ovsdb-cluster

src/lib/charm/openstack/ovn_central.py

@@ -245,6 +245,7 @@ class BaseOVNCentralCharm(charms_openstack.charm.OpenStackCharm):
         :returns: None
         """
         if self.ovn_upgrade_available(self.release_pkg):
+            ch_core.hookenv.status_set('maintenance', 'Rolling upgrade')
             self.do_openstack_pkg_upgrade(upgrade_openstack=False)
             self.render_with_interfaces(interfaces_list)
 

src/reactive/ovn_central_handlers.py

@@ -14,6 +14,7 @@
 
 import charms.reactive as reactive
 import charms.leadership as leadership
+import charms.coordinator as coordinator
 
 import charms_openstack.bus
 import charms_openstack.charm as charm
@@ -62,6 +63,12 @@ def enable_install():
     """Enable the default install hook."""
     charm.use_defaults('charm.installed')
 
+    # These flags will be set on initial install.  We use these flags to ensure
+    # not performing certain actions during coordinated payload upgrades, but
+    # we don't want these provisions to interfere with initial clustering.
+    reactive.clear_flag('config.changed.source')
+    reactive.clear_flag('config.changed.ovn-source')
+
 
 @reactive.when_none('is-update-status-hook', 'charm.firewall_initialized')
 def initialize_firewall():
@@ -73,7 +80,11 @@ def initialize_firewall():
 
 @reactive.when_none('is-update-status-hook',
                     'leadership.set.nb_cid',
-                    'leadership.set.sb_cid')
+                    'leadership.set.sb_cid',
+                    'coordinator.granted.upgrade',
+                    'coordinator.requested.upgrade',
+                    'config.changed.source',
+                    'config.changed.ovn-source')
 @reactive.when('config.rendered',
                'certificates.connected',
                'certificates.available',
@@ -111,8 +122,11 @@ def announce_leader_ready():
         })
 
 
-@reactive.when_none('is-update-status-hook', 'leadership.set.nb_cid',
-                    'leadership.set.sb_cid')
+@reactive.when_none('is-update-status-hook',
+                    'leadership.set.nb_cid',
+                    'leadership.set.sb_cid',
+                    'coordinator.granted.upgrade',
+                    'coordinator.requested.upgrade')
 @reactive.when('charm.installed', 'leadership.is_leader',
                'ovsdb-peer.connected')
 def initialize_ovsdbs():
@@ -179,6 +193,22 @@ def publish_addr_to_clients():
 @reactive.when_none('is-update-status-hook')
 @reactive.when('ovsdb-peer.available')
 @reactive.when_any('config.changed.source', 'config.changed.ovn-source')
+def maybe_request_upgrade():
+    # The ovn-ctl script in the ovn-common package does schema upgrade based
+    # on non-presence of a value to `--db-nb-cluster-remote-addr` in
+    # /etc/default/ovn-central.  This is the case for the charm leader.
+    #
+    # The charm leader will perform DB schema upgrade as part of the package
+    # upgrade, and in order to succeed with that we must ensure the other
+    # units does not perform the package upgrade simultaneously.
+    #
+    # The coordinator library is based on leader storage and the leader will
+    # always be the first one to get the lock.
+    coordinator.acquire('upgrade')
+
+
+@reactive.when_none('is-update-status-hook')
+@reactive.when('ovsdb-peer.available', 'coordinator.granted.upgrade')
 def maybe_do_upgrade():
     ovsdb_peer = reactive.endpoint_from_flag('ovsdb-peer.available')
     with charm.provide_charm_instance() as ovn_charm:
@@ -186,7 +216,11 @@ def maybe_do_upgrade():
         ovn_charm.assess_status()
 
 
-@reactive.when_none('is-update-status-hook')
+@reactive.when_none('is-update-status-hook',
+                    'coordinator.granted.upgrade',
+                    'coordinator.requested.upgrade',
+                    'config.changed.source',
+                    'config.changed.ovn-source')
 @reactive.when('ovsdb-peer.available',
                'leadership.set.nb_cid',
                'leadership.set.sb_cid',

unit_tests/__init__.py

@@ -43,6 +43,8 @@ charms = mock.MagicMock()
 sys.modules['charms'] = charms
 charms.leadership = mock.MagicMock()
 sys.modules['charms.leadership'] = charms.leadership
+charms.coordinator = mock.MagicMock()
+sys.modules['charms.coordinator'] = charms.coordinator
 charms.reactive = mock.MagicMock()
 charms.reactive.when = _fake_decorator
 charms.reactive.when_all = _fake_decorator

unit_tests/test_reactive_ovn_central_handlers.py

@@ -32,7 +32,11 @@ class TestRegisteredHooks(test_utils.TestRegisteredHooks):
             'when_none': {
                 'announce_leader_ready': ('is-update-status-hook',
                                           'leadership.set.nb_cid',
-                                          'leadership.set.sb_cid'),
+                                          'leadership.set.sb_cid',
+                                          'coordinator.granted.upgrade',
+                                          'coordinator.requested.upgrade',
+                                          'config.changed.source',
+                                          'config.changed.ovn-source'),
                 'configure_firewall': ('is-update-status-hook',),
                 'enable_default_certificates': ('is-update-status-hook',
                                                 'leadership.is_leader',),
@@ -40,10 +44,17 @@ class TestRegisteredHooks(test_utils.TestRegisteredHooks):
                                         'charm.firewall_initialized',),
                 'initialize_ovsdbs': ('is-update-status-hook',
                                       'leadership.set.nb_cid',
-                                      'leadership.set.sb_cid',),
+                                      'leadership.set.sb_cid',
+                                      'coordinator.granted.upgrade',
+                                      'coordinator.requested.upgrade'),
                 'maybe_do_upgrade': ('is-update-status-hook',),
+                'maybe_request_upgrade': ('is-update-status-hook',),
                 'publish_addr_to_clients': ('is-update-status-hook',),
-                'render': ('is-update-status-hook',),
+                'render': ('is-update-status-hook',
+                           'coordinator.granted.upgrade',
+                           'coordinator.requested.upgrade',
+                           'config.changed.source',
+                           'config.changed.ovn-source'),
                 'configure_nrpe': ('charm.paused', 'is-update-status-hook',),
                 'stamp_fresh_deployment': ('charm.installed',
                                            'leadership.set.install_stamp'),
@@ -65,7 +76,9 @@ class TestRegisteredHooks(test_utils.TestRegisteredHooks):
                 'initialize_ovsdbs': ('charm.installed',
                                       'leadership.is_leader',
                                       'ovsdb-peer.connected',),
-                'maybe_do_upgrade': ('ovsdb-peer.available',),
+                'maybe_do_upgrade': ('ovsdb-peer.available',
+                                     'coordinator.granted.upgrade',),
+                'maybe_request_upgrade': ('ovsdb-peer.available',),
                 'publish_addr_to_clients': ('ovsdb-peer.available',
                                             'leadership.set.nb_cid',
                                             'leadership.set.sb_cid',
@@ -88,8 +101,8 @@ class TestRegisteredHooks(test_utils.TestRegisteredHooks):
                                    'nrpe-external-master.available',),
                 'enable_install': ('leadership.set.install_stamp',
                                    'leadership.set.upgrade_stamp'),
-                'maybe_do_upgrade': ('config.changed.source',
-                                     'config.changed.ovn-source'),
+                'maybe_request_upgrade': ('config.changed.source',
+                                          'config.changed.ovn-source'),
             },
             'when_not': {
                 'configure_deferred_restarts': ('is-update-status-hook',),