Fix version retrieval

During cluster deployment a situation can arise where there are
already osd relations but osds are not yet fully added to the cluster.
This can make version retrieval fail for osds. Retry version retrieval
to give the cluster a chance to settle.

Also update tests to install OpenStack from latest/edge

Change-Id: I12a1bcd32be2ed8a8e5ee0e304f716f5a190bd57

src/utils.py

@@ -18,6 +18,7 @@ import socket
 import subprocess
 import errno
 
+import tenacity
 from charmhelpers.core.hookenv import (
     DEBUG,
     cached,
@@ -327,10 +328,16 @@ def execute_post_osd_upgrade_steps(ceph_osd_release):
         log(message=msg, level='ERROR')
 
 
-def _all_ceph_versions_same():
-    """Checks that ceph-mon and ceph-osd have converged to the same version.
+@tenacity.retry(
+    wait=tenacity.wait_exponential(multiplier=1, max=10),
+    reraise=True,
+    stop=tenacity.stop_after_attempt(8))
+def _get_versions():
+    """Gets the ceph versions.
 
-    :return boolean: True if all same, false if not or command failed.
+    Retry if the commands fails to give the cluster time to settle.
+
+    :return tuple: (bool, dict) True if successful, False if not, and a dict
     """
     try:
         versions_command = 'ceph versions'
@@ -340,11 +347,25 @@ def _all_ceph_versions_same():
         if call_error.returncode == errno.EINVAL:
             log('Calling "ceph versions" failed. Command requires '
                 'luminous and above.', level='WARNING')
-            return False
+            return False, {}
         else:
             log('Calling "ceph versions" failed.', level='ERROR')
             raise OsdPostUpgradeError(call_error)
+    log('Versions: {}'.format(versions_str), level='DEBUG')
     versions_dict = json.loads(versions_str)
+    # provoke keyerror if we don't have osd versions yet to cause a retry
+    _ = versions_dict['osd']
+    return True, versions_dict
+
+
+def _all_ceph_versions_same():
+    """Checks that ceph-mon and ceph-osd have converged to the same version.
+
+    :return boolean: True if all same, false if not or command failed.
+    """
+    ok, versions_dict = _get_versions()
+    if not ok:
+        return False
     if len(versions_dict['overall']) > 1:
         log('All upgrades of mon and osd have not completed.')
         return False

tests/bundles/jammy-bobcat.yaml

@@ -77,11 +77,11 @@ applications:
       - '3'
       - '4'
       - '5'
-    channel: quincy/edge
+    channel: latest/edge
 
   ceph-mon:
     charm: ch:ceph-mon
-    channel: quincy/edge
+    channel: latest/edge
     num_units: 3
     options:
       source: *openstack-origin
@@ -96,7 +96,7 @@ applications:
     num_units: 1
     options:
       source: *openstack-origin
-    channel: quincy/edge
+    channel: latest/edge
     to: 
     - '17'
 
@@ -108,7 +108,7 @@ applications:
       openstack-origin: *openstack-origin
     to:
       - '10'
-    channel: 2023.1/edge
+    channel: latest/edge
 
   nova-compute:
     charm: ch:nova-compute
@@ -118,7 +118,7 @@ applications:
       libvirt-image-backend: rbd
     to:
       - '11'
-    channel: 2023.1/edge
+    channel: latest/edge
 
   glance:
     expose: True
@@ -128,7 +128,7 @@ applications:
       openstack-origin: *openstack-origin
     to:
       - '12'
-    channel: 2023.1/edge
+    channel: latest/edge
 
   cinder:
     expose: True
@@ -140,11 +140,11 @@ applications:
       openstack-origin: *openstack-origin
     to:
       - '13'
-    channel: 2023.1/edge
+    channel: latest/edge
 
   cinder-ceph:
     charm: ch:cinder-ceph
-    channel: 2023.1/edge
+    channel: latest/edge
 
   nova-cloud-controller:
     expose: True
@@ -154,7 +154,7 @@ applications:
       openstack-origin: *openstack-origin
     to:
       - '14'
-    channel: 2023.1/edge
+    channel: latest/edge
 
   placement:
     charm: ch:placement
@@ -163,7 +163,7 @@ applications:
       openstack-origin: *openstack-origin
     to:
       - '15'
-    channel: 2023.1/edge
+    channel: latest/edge
 
   prometheus2:
     charm: ch:prometheus2

unit_tests/test_ceph_utils.py

@@ -172,7 +172,7 @@ class CephUtilsTestCase(test_utils.CharmTestCase):
         self.assertTrue(
             return_bool,
             msg='all_ceph_versions_same returned False but should be True')
-        log.assert_not_called()
+        log.assert_called_once()
 
     @mock.patch.object(utils.subprocess, 'check_output')
     @mock.patch.object(utils.json, 'loads')
@@ -190,7 +190,7 @@ class CephUtilsTestCase(test_utils.CharmTestCase):
         self.assertFalse(
             return_bool,
             msg='all_ceph_versions_same returned True but should be False')
-        log.assert_called_once()
+        self.assertEquals(log.call_count, 2)
 
     @mock.patch.object(utils.subprocess, 'check_output')
     @mock.patch.object(utils.json, 'loads')
@@ -208,7 +208,7 @@ class CephUtilsTestCase(test_utils.CharmTestCase):
         self.assertFalse(
             return_bool,
             msg='all_ceph_versions_same returned True but should be False')
-        log.assert_called_once()
+        self.assertEquals(log.call_count, 2)
 
     @mock.patch.object(utils.subprocess, 'check_output')
     @mock.patch.object(utils, 'log')