Add tests: datastore upgrade

Change-Id: Idd42169182df809232b1f2ab3bdc8e6d4289c798

trove_tempest_plugin/config.py

@@ -73,4 +73,15 @@ DatabaseGroup = [
         default="lvmdriver-1",
         help="The Cinder volume type used for creating database instance."
     ),
+    cfg.DictOpt(
+        'default_datastore_versions',
+        default={'mysql': '5.7.29'},
+        help='The default datastore versions used to create instance',
+    ),
+    cfg.DictOpt(
+        'pre_upgrade_datastore_versions',
+        default={'mysql': '5.7.29'},
+        help='The datastore versions used to create instances that need to be '
+             'upgrade.',
+    ),
 ]

trove_tempest_plugin/tests/base.py

@@ -198,6 +198,38 @@ class BaseTroveTest(test.BaseTestCase):
             "instances", cls.instance_id)['instance']
         cls.instance_ip = cls.get_instance_ip(cls.instance)
 
+    def assert_single_item(self, items, **props):
+        return self.assert_multiple_items(items, 1, **props)[0]
+
+    def assert_multiple_items(self, items, count, **props):
+        """Check if a object is in a list of objects.
+
+        e.g. props is a sub-dict, items is a list of dicts.
+        """
+
+        def _matches(item, **props):
+            for prop_name, prop_val in props.items():
+                v = item[prop_name] if isinstance(
+                    item, dict) else getattr(item, prop_name)
+
+                if v != prop_val:
+                    return False
+
+            return True
+
+        filtered_items = list(
+            [item for item in items if _matches(item, **props)]
+        )
+
+        found = len(filtered_items)
+
+        if found != count:
+            LOG.info("[FAIL] items=%s, expected_props=%s", str(items), props)
+            self.fail("Wrong number of items found [props=%s, "
+                      "expected=%s, found=%s]" % (props, count, found))
+
+        return filtered_items
+
     @classmethod
     def create_instance(cls, name=None, datastore_version=None,
                         database=constants.DB_NAME, username=constants.DB_USER,
@@ -212,16 +244,25 @@ class BaseTroveTest(test.BaseTestCase):
         """
         name = name or cls.get_resource_name("instance")
 
-        # Get datastore version
+        # Get datastore version. Get from API if the default ds version is not
+        # configured.
         if not datastore_version:
-            res = cls.client.list_resources("datastores")
+            default_versions = CONF.database.default_datastore_versions
-            for d in res['datastores']:
+            datastore_version = default_versions.get(cls.datastore)
-                if d['name'] == cls.datastore:
+
-                    if d.get('default_version'):
+            if not datastore_version:
-                        datastore_version = d['default_version']
+                res = cls.client.list_resources("datastores")
-                    else:
+                for d in res['datastores']:
-                        datastore_version = d['versions'][0]['name']
+                    if d['name'] == cls.datastore:
-                    break
+                        if d.get('default_version'):
+                            datastore_version = d['default_version']
+                        else:
+                            datastore_version = d['versions'][0]['name']
+                        break
+
+        if not datastore_version:
+            message = ('Failed to get available datastore version.')
+            raise exceptions.TempestException(message)
 
         body = {
             "instance": {

trove_tempest_plugin/tests/scenario/base_actions.py

@@ -14,20 +14,22 @@
 import time
 
 from oslo_log import log as logging
+from tempest import config
 from tempest.lib import decorators
 
 from trove_tempest_plugin.tests import base as trove_base
+from trove_tempest_plugin.tests import constants
 from trove_tempest_plugin.tests import utils
 
 LOG = logging.getLogger(__name__)
+CONF = config.CONF
 
 
-def get_db_version(ip, username='test_user', password='password'):
+def get_db_version(ip, username=constants.DB_USER, password=constants.DB_PASS):
     LOG.info('Trying to access the database %s', ip)
 
     db_url = f'mysql+pymysql://{username}:{password}@{ip}:3306'
-    db_engine = utils.init_engine(db_url)
+    db_client = utils.SQLClient(db_url)
-    db_client = utils.SQLClient(db_engine)
 
     cmd = "SELECT @@GLOBAL.innodb_version;"
     ret = db_client.execute(cmd)
@@ -35,30 +37,68 @@ def get_db_version(ip, username='test_user', password='password'):
 
 
 class TestInstanceActionsBase(trove_base.BaseTroveTest):
+    @classmethod
+    def init_db(cls, *args, **kwargs):
+        pass
+
+    def insert_data_upgrade(self, *args, **kwargs):
+        pass
+
+    def verify_data_upgrade(self, *args, **kwargs):
+        pass
+
+    @classmethod
+    def resource_setup(cls):
+        super(TestInstanceActionsBase, cls).resource_setup()
+
+        # Initialize database
+        cls.init_db(cls.instance_ip, constants.DB_USER, constants.DB_PASS,
+                    constants.DB_NAME)
+
     @decorators.idempotent_id("be6dd514-27d6-11ea-a56a-98f2b3cc23a0")
     def test_instance_upgrade(self):
-        datastore = self.instance['datastore']['type']
+        cur_version = self.instance['datastore']['version']
-        version = self.instance['datastore']['version']
+        cfg_versions = CONF.database.pre_upgrade_datastore_versions
-        new_version = version
+        ds_version = cfg_versions.get(self.datastore)
-        datastore = self.client.get_resource("datastores", datastore)
+        if not ds_version:
-        for v in datastore['datastore']['versions']:
+            # Fall back to the instance datastore version. In this case, we are
-            if v['name'] != version:
+            # still testing the upgrade API but the datastore version doesn't
-                new_version = v['name']
+            # change actually.
-                break
+            ds_version = cur_version
 
-        LOG.info('Using datastore %s for instance upgrading', new_version)
+        name = self.get_resource_name("pre-upgrade")
+        LOG.info(f'Creating instance {name} with datastore version '
+                 f'{ds_version} for upgrade')
+        instance = self.create_instance(name=name,
+                                        datastore_version=ds_version)
+        self.wait_for_instance_status(instance['id'])
+        instance = self.client.get_resource(
+            "instances", instance['id'])['instance']
+        instance_ip = self.get_instance_ip(instance)
 
-        body = {
+        # Insert data before upgrading
-            "instance": {
+        self.init_db(instance_ip, constants.DB_USER, constants.DB_PASS,
-                "datastore_version": new_version
+                     constants.DB_NAME)
-            }
+        self.insert_data_upgrade(instance_ip, constants.DB_USER,
-        }
+                                 constants.DB_PASS, constants.DB_NAME)
-        self.client.patch_resource('instances', self.instance_id, body)
 
-        time.sleep(3)
+        new_version = cur_version
-        self.wait_for_instance_status(self.instance_id)
+        LOG.info(f"Upgrading instance {instance['id']} using datastore "
+                 f"{new_version}")
+        body = {"instance": {"datastore_version": new_version}}
+        self.client.patch_resource('instances', instance['id'], body)
 
-        time.sleep(3)
+        # Wait in case the instance status hasn't changed yet.
-        actual = get_db_version(self.instance_ip)
+        time.sleep(5)
+        self.wait_for_instance_status(instance['id'])
+        actual = get_db_version(instance_ip)
+        self.assertEqual(new_version, actual)
 
-        self.assertEqual(actual, new_version)
+        self.verify_data_upgrade(instance_ip, constants.DB_USER,
+                                 constants.DB_PASS, constants.DB_NAME)
+
+        # Delete the new instance explicitly to avoid too many instances
+        # during the test.
+        self.wait_for_instance_status(instance['id'],
+                                      expected_status="DELETED",
+                                      need_delete=True)

trove_tempest_plugin/tests/scenario/base_basic.py

@@ -27,8 +27,7 @@ class TestInstanceBasicMySQLBase(trove_base.BaseTroveTest):
         LOG.info('Trying to access the database %s', ip)
 
         db_url = f'mysql+pymysql://{username}:{password}@{ip}:3306'
-        db_engine = utils.init_engine(db_url)
+        db_client = utils.SQLClient(db_url)
-        db_client = utils.SQLClient(db_engine)
 
         cmd = "SELECT 1;"
         db_client.execute(cmd)

trove_tempest_plugin/tests/scenario/test_backup.py

@@ -27,16 +27,11 @@ class TestBackupMySQL(base_backup.TestBackupBase):
         LOG.info(f"Inserting data to database {database} on {ip}")
 
         db_url = f'mysql+pymysql://{username}:{password}@{ip}:3306/{database}'
-        db_engine = utils.init_engine(db_url)
+        db_client = utils.SQLClient(db_url)
-        db_client = utils.SQLClient(db_engine)
 
         cmds = [
-            "CREATE TABLE Persons (PersonID int, LastName varchar(255), "
+            "CREATE TABLE Persons (ID int, String varchar(255));",
-            "FirstName varchar(255), Address varchar(255), City "
+            "insert into Persons VALUES (1, 'Lingxian Kong');",
-            "varchar(255));",
-
-            "insert into Persons VALUES (1, 'Kong', 'Lingxian', '150 Willis "
-            "Street', 'Wellington');"
         ]
         db_client.execute(cmds)
 
@@ -46,19 +41,16 @@ class TestBackupMySQL(base_backup.TestBackupBase):
                  f"incremental backup")
 
         db_url = f'mysql+pymysql://{username}:{password}@{ip}:3306/{database}'
-        db_engine = utils.init_engine(db_url)
+        db_client = utils.SQLClient(db_url)
-        db_client = utils.SQLClient(db_engine)
 
         cmds = [
-            "insert into Persons VALUES (99, 'OpenStack', 'Trove', "
+            "insert into Persons VALUES (99, 'OpenStack');"
-            "'150 Willis Street', 'Wellington');"
         ]
         db_client.execute(cmds)
 
     def verify_data(self, ip, username, password, database):
         db_url = f'mysql+pymysql://{username}:{password}@{ip}:3306/{database}'
-        db_engine = utils.init_engine(db_url)
+        db_client = utils.SQLClient(db_url)
-        db_client = utils.SQLClient(db_engine)
 
         cmd = "select * from Persons;"
         ret = db_client.execute(cmd)
@@ -67,14 +59,12 @@ class TestBackupMySQL(base_backup.TestBackupBase):
         self.assertEqual(1, len(rows))
 
         result = dict(zip(keys, rows[0]))
-        expected = {'PersonID': 1, 'LastName': 'Kong', 'FirstName': 'Lingxian',
+        expected = {'ID': 1, 'String': 'Lingxian Kong'}
-                    'Address': '150 Willis Street', 'City': 'Wellington'}
         self.assertEqual(expected, result)
 
     def verify_data_inc(self, ip, username, password, database):
         db_url = f'mysql+pymysql://{username}:{password}@{ip}:3306/{database}'
-        db_engine = utils.init_engine(db_url)
+        db_client = utils.SQLClient(db_url)
-        db_client = utils.SQLClient(db_engine)
 
         cmd = "select * from Persons;"
         ret = db_client.execute(cmd)
@@ -87,13 +77,7 @@ class TestBackupMySQL(base_backup.TestBackupBase):
             actual.append(dict(zip(keys, rows[index])))
 
         expected = [
-            {
+            {'ID': 1, 'String': 'Lingxian Kong'},
-                'PersonID': 1, 'LastName': 'Kong', 'FirstName': 'Lingxian',
+            {'ID': 99, 'String': 'OpenStack'},
-                'Address': '150 Willis Street', 'City': 'Wellington'
-            },
-            {
-                'PersonID': 99, 'LastName': 'OpenStack', 'FirstName': 'Trove',
-                'Address': '150 Willis Street', 'City': 'Wellington'
-            },
         ]
         self.assertEqual(expected, actual)

trove_tempest_plugin/tests/scenario/test_instance_actions.py

@@ -11,13 +11,57 @@
 #    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
+from oslo_log import log as logging
 
 from trove_tempest_plugin.tests.scenario import base_actions
+from trove_tempest_plugin.tests import utils
+
+LOG = logging.getLogger(__name__)
 
 
 class TestInstanceActionsMySQL(base_actions.TestInstanceActionsBase):
     datastore = 'mysql'
 
+    @classmethod
+    def init_db(cls, ip, username, password, database):
+        LOG.info(f"Initializing database {database} on {ip}")
 
-class TestInstanceActionsMariaDB(base_actions.TestInstanceActionsBase):
+        db_url = f'mysql+pymysql://{username}:{password}@{ip}:3306/{database}'
+        db_client = utils.SQLClient(db_url)
+
+        cmds = [
+            "CREATE TABLE Persons (ID int, String varchar(255));",
+        ]
+        db_client.execute(cmds)
+
+    def insert_data_upgrade(self, ip, username, password, database):
+        LOG.info(f"Inserting data to database {database} on {ip} for "
+                 f"datastore upgrade")
+
+        db_url = f'mysql+pymysql://{username}:{password}@{ip}:3306/{database}'
+        db_client = utils.SQLClient(db_url)
+
+        cmds = [
+            "insert into Persons VALUES (99, 'Upgrade');"
+        ]
+        db_client.execute(cmds)
+
+    def verify_data_upgrade(self, ip, username, password, database):
+        db_url = f'mysql+pymysql://{username}:{password}@{ip}:3306/{database}'
+        db_client = utils.SQLClient(db_url)
+
+        cmd = "select * from Persons;"
+        ret = db_client.execute(cmd)
+        keys = ret.keys()
+        rows = ret.fetchall()
+        self.assertGreaterEqual(len(rows), 1)
+
+        result = []
+        for index in range(len(rows)):
+            result.append(dict(zip(keys, rows[index])))
+        expected = {'ID': 99, 'String': 'Upgrade'}
+        self.assert_single_item(result, **expected)
+
+
+class TestInstanceActionsMariaDB(TestInstanceActionsMySQL):
     datastore = 'mariadb'

trove_tempest_plugin/tests/scenario/test_instance_basic.py

@@ -18,5 +18,5 @@ class TestInstanceBasicMySQL(base_basic.TestInstanceBasicMySQLBase):
     datastore = 'mysql'
 
 
-class TestInstanceBasicMariaDB(base_basic.TestInstanceBasicMySQLBase):
+class TestInstanceBasicMariaDB(TestInstanceBasicMySQL):
     datastore = 'mariadb'

trove_tempest_plugin/tests/utils.py

@@ -56,8 +56,8 @@ def init_engine(db_url):
 
 
 class SQLClient(object):
-    def __init__(self, engine):
+    def __init__(self, url):
-        self.engine = engine
+        self.engine = init_engine(url)
 
     def execute(self, cmds, **kwargs):
         try: