Merge "Add support for module-reapply command"

trove/tests/scenario/groups/module_group.py

@@ -375,17 +375,33 @@ class ModuleInstCreateGroup(TestGroup):
         """Check that module-apply works."""
         self.test_runner.run_module_apply()
 
-    @test(runs_after=[module_query_empty])
+    @test(runs_after=[module_apply])
     def module_apply_wrong_module(self):
         """Ensure that module-apply for wrong module fails."""
         self.test_runner.run_module_apply_wrong_module()
 
-    @test(depends_on=[module_apply])
+    @test(depends_on=[module_apply_wrong_module])
+    def module_update_not_live(self):
+        """Ensure updating a non live_update module fails."""
+        self.test_runner.run_module_update_not_live()
+
+    @test(depends_on=[module_apply],
+          runs_after=[module_update_not_live])
     def module_list_instance_after_apply(self):
-        """Check that the instance has one module associated."""
+        """Check that the instance has the modules associated."""
         self.test_runner.run_module_list_instance_after_apply()
 
     @test(runs_after=[module_list_instance_after_apply])
+    def module_apply_live_update(self):
+        """Check that module-apply works for live_update."""
+        self.test_runner.run_module_apply_live_update()
+
+    @test(depends_on=[module_apply_live_update])
+    def module_list_instance_after_apply_live(self):
+        """Check that the instance has the right modules."""
+        self.test_runner.run_module_list_instance_after_apply_live()
+
+    @test(runs_after=[module_list_instance_after_apply_live])
     def module_instances_after_apply(self):
         """Check that the instance shows up in the list."""
         self.test_runner.run_module_instances_after_apply()
@@ -401,13 +417,18 @@ class ModuleInstCreateGroup(TestGroup):
         self.test_runner.run_module_query_after_apply()
 
     @test(runs_after=[module_query_after_apply])
+    def module_update_live_update(self):
+        """Check that update module works on 'live' applied module."""
+        self.test_runner.run_module_update_live_update()
+
+    @test(runs_after=[module_update_live_update])
     def module_apply_another(self):
         """Check that module-apply works for another module."""
         self.test_runner.run_module_apply_another()
 
     @test(depends_on=[module_apply_another])
     def module_list_instance_after_apply_another(self):
-        """Check that the instance has one module associated."""
+        """Check that the instance has the right modules again."""
         self.test_runner.run_module_list_instance_after_apply_another()
 
     @test(runs_after=[module_list_instance_after_apply_another])
@@ -420,7 +441,8 @@ class ModuleInstCreateGroup(TestGroup):
         """Check that the instance count is right after another apply."""
         self.test_runner.run_module_instance_count_after_apply_another()
 
-    @test(depends_on=[module_apply_another])
+    @test(depends_on=[module_apply_another],
+          runs_after=[module_instance_count_after_apply_another])
     def module_query_after_apply_another(self):
         """Check that module-query works after another apply."""
         self.test_runner.run_module_query_after_apply_another()
@@ -431,26 +453,26 @@ class ModuleInstCreateGroup(TestGroup):
         """Check that creating an instance with modules works."""
         self.test_runner.run_create_inst_with_mods()
 
-    @test(runs_after=[module_query_empty])
+    @test(runs_after=[create_inst_with_mods])
     def create_inst_with_wrong_module(self):
         """Ensure that creating an inst with wrong ds mod fails."""
         self.test_runner.run_create_inst_with_wrong_module()
 
-    @test(depends_on=[module_apply])
+    @test(depends_on=[module_apply],
+          runs_after=[create_inst_with_wrong_module])
     def module_delete_applied(self):
         """Ensure that deleting an applied module fails."""
         self.test_runner.run_module_delete_applied()
 
     @test(depends_on=[module_apply],
-          runs_after=[module_list_instance_after_apply,
-                      module_query_after_apply])
+          runs_after=[module_delete_applied])
     def module_remove(self):
         """Check that module-remove works."""
         self.test_runner.run_module_remove()
 
     @test(depends_on=[module_remove])
     def module_query_after_remove(self):
-        """Check that the instance has one module applied after remove."""
+        """Check that the instance has modules applied after remove."""
         self.test_runner.run_module_query_after_remove()
 
     @test(depends_on=[module_remove],
@@ -468,7 +490,7 @@ class ModuleInstCreateGroup(TestGroup):
     @test(depends_on=[module_apply],
           runs_after=[module_apply_another_again])
     def module_query_after_apply_another2(self):
-        """Check that module-query works after second apply."""
+        """Check that module-query works still."""
         self.test_runner.run_module_query_after_apply_another()
 
     @test(depends_on=[module_apply_another_again],
@@ -479,7 +501,7 @@ class ModuleInstCreateGroup(TestGroup):
 
     @test(depends_on=[module_remove_again])
     def module_query_empty_after_again(self):
-        """Check that the inst has one mod applied after 2nd remove."""
+        """Check that the inst has right mod applied after 2nd remove."""
         self.test_runner.run_module_query_after_remove()
 
     @test(depends_on=[module_remove_again],
@@ -533,6 +555,86 @@ class ModuleInstCreateWaitGroup(TestGroup):
         """Ensure that module-delete on auto-applied module fails."""
         self.test_runner.run_module_delete_auto_applied()
 
+    @test(runs_after=[module_delete_auto_applied])
+    def module_list_instance_after_mod_inst(self):
+        """Check that the new instance has the right modules."""
+        self.test_runner.run_module_list_instance_after_mod_inst()
+
+    @test(runs_after=[module_list_instance_after_mod_inst])
+    def module_instances_after_mod_inst(self):
+        """Check that the new instance shows up in the list."""
+        self.test_runner.run_module_instances_after_mod_inst()
+
+    @test(runs_after=[module_instances_after_mod_inst])
+    def module_instance_count_after_mod_inst(self):
+        """Check that the new instance count is right."""
+        self.test_runner.run_module_instance_count_after_mod_inst()
+
+    @test(runs_after=[module_instance_count_after_mod_inst])
+    def module_reapply_with_md5(self):
+        """Check that module reapply with md5 works."""
+        self.test_runner.run_module_reapply_with_md5()
+
+    @test(runs_after=[module_reapply_with_md5])
+    def module_reapply_with_md5_verify(self):
+        """Verify the dates after md5 reapply (no-op)."""
+        self.test_runner.run_module_reapply_with_md5_verify()
+
+    @test(runs_after=[module_reapply_with_md5_verify])
+    def module_list_instance_after_reapply_md5(self):
+        """Check that the instance's modules haven't changed."""
+        self.test_runner.run_module_list_instance_after_reapply_md5()
+
+    @test(runs_after=[module_list_instance_after_reapply_md5])
+    def module_instances_after_reapply_md5(self):
+        """Check that the new instance still shows up in the list."""
+        self.test_runner.run_module_instances_after_reapply_md5()
+
+    @test(runs_after=[module_instances_after_reapply_md5])
+    def module_instance_count_after_reapply_md5(self):
+        """Check that the instance count hasn't changed."""
+        self.test_runner.run_module_instance_count_after_reapply_md5()
+
+    @test(runs_after=[module_instance_count_after_reapply_md5])
+    def module_reapply_all(self):
+        """Check that module reapply works."""
+        self.test_runner.run_module_reapply_all()
+
+    @test(runs_after=[module_reapply_all])
+    def module_reapply_all_wait(self):
+        """Wait for module reapply to complete."""
+        self.test_runner.run_module_reapply_all_wait()
+
+    @test(runs_after=[module_reapply_all_wait])
+    def module_instance_count_after_reapply(self):
+        """Check that the reapply instance count is right."""
+        self.test_runner.run_module_instance_count_after_reapply()
+
+    @test(runs_after=[module_instance_count_after_reapply])
+    def module_reapply_with_force(self):
+        """Check that module reapply with force works."""
+        self.test_runner.run_module_reapply_with_force()
+
+    @test(runs_after=[module_reapply_with_force])
+    def module_reapply_with_force_wait(self):
+        """Wait for module reapply with force to complete."""
+        self.test_runner.run_module_reapply_with_force_wait()
+
+    @test(runs_after=[module_reapply_with_force_wait])
+    def module_list_instance_after_reapply_force(self):
+        """Check that the new instance still has the right modules."""
+        self.test_runner.run_module_list_instance_after_reapply()
+
+    @test(runs_after=[module_list_instance_after_reapply_force])
+    def module_instances_after_reapply_force(self):
+        """Check that the new instance still shows up in the list."""
+        self.test_runner.run_module_instances_after_reapply()
+
+    @test(runs_after=[module_instances_after_reapply_force])
+    def module_instance_count_after_reapply_force(self):
+        """Check that the instance count is right after reapply force."""
+        self.test_runner.run_module_instance_count_after_reapply()
+
 
 @test(depends_on_groups=[groups.MODULE_INST_CREATE_WAIT],
       groups=[GROUP, groups.MODULE_INST, groups.MODULE_INST_DELETE])
@@ -548,6 +650,11 @@ class ModuleInstDeleteGroup(TestGroup):
         """Check that instance with module can be deleted."""
         self.test_runner.run_delete_inst_with_mods()
 
+    @test(runs_after=[delete_inst_with_mods])
+    def remove_mods_from_main_inst(self):
+        """Check that modules can be removed from the main instance."""
+        self.test_runner.run_remove_mods_from_main_inst()
+
 
 @test(depends_on_groups=[groups.MODULE_INST_DELETE],
       groups=[GROUP, groups.MODULE_INST, groups.MODULE_INST_DELETE_WAIT],

trove/tests/scenario/runners/module_runners.py

@@ -18,9 +18,12 @@ import Crypto.Random
 from proboscis import SkipTest
 import re
 import tempfile
+import time
 
 from troveclient.compat import exceptions
 
+from trove.common import exception
+from trove.common.utils import poll_until
 from trove.guestagent.common import guestagent_utils
 from trove.guestagent.common import operating_system
 from trove.module import models
@@ -64,8 +67,12 @@ class ModuleRunner(TestRunner):
             {'suffix': '_updated', 'priority': False, 'order': 8},
         ]
 
+        self.apply_count = 0
         self.mod_inst_id = None
+        self.mod_inst_apply_count = 0
         self.temp_module = None
+        self.live_update_orig_md5 = None
+        self.reapply_max_upd_date = None
         self._module_type = None
 
         self.test_modules = []
@@ -104,6 +111,10 @@ class ModuleRunner(TestRunner):
     def update_test_module(self):
         return self._get_test_module(1)
 
+    @property
+    def live_update_test_module(self):
+        return self._find_live_update_module()
+
     def build_module_args(self, name_order=None):
         suffix = "_unknown"
         priority = False
@@ -153,6 +164,11 @@ class ModuleRunner(TestRunner):
             return mod.auto_apply and mod.tenant_id and mod.visible
         return self._find_module(_match, "Could not find auto-apply module")
 
+    def _find_live_update_module(self):
+        def _match(mod):
+            return mod.live_update and mod.tenant_id and mod.visible
+        return self._find_module(_match, "Could not find live-update module")
+
     def _find_all_tenant_module(self):
         def _match(mod):
             return mod.tenant_id is None and mod.visible
@@ -584,6 +600,7 @@ class ModuleRunner(TestRunner):
         self.assert_module_create(
             self.admin_client, 5,
             live_update=True)
+        self.live_update_orig_md5 = self.test_modules[-1].md5
 
     def run_module_create_admin_priority_apply(self):
         self.assert_module_create(
@@ -905,10 +922,13 @@ class ModuleRunner(TestRunner):
         rowcount = len(instance_count_list)
         self.assert_equal(expected_rows, rowcount,
                           "Wrong number of instance count records from module")
-        if expected_rows == 1:
-            self.assert_equal(expected_count,
-                              instance_count_list[0].instance_count,
-                              "Wrong count in record from module instances")
+        # expected_count is a dict of md5->count pairs.
+        if expected_rows and expected_count:
+            for row in instance_count_list:
+                self.assert_equal(
+                    expected_count[row.module_md5], row.instance_count,
+                    "Wrong count in record from module instances; md5: %s" %
+                    row.module_md5)
 
     def run_module_query_empty(self):
         self.assert_module_query(
@@ -918,7 +938,7 @@ class ModuleRunner(TestRunner):
     def run_module_query_after_remove(self):
         self.assert_module_query(
             self.auth_client, self.instance_info.id,
-            self.module_auto_apply_count_prior_to_create + 1)
+            self.module_auto_apply_count_prior_to_create + 2)
 
     def assert_module_query(self, client, instance_id, expected_count,
                             expected_http_code=200, expected_results=None):
@@ -955,6 +975,7 @@ class ModuleRunner(TestRunner):
     def run_module_apply(self):
         self.assert_module_apply(self.auth_client, self.instance_info.id,
                                  self.main_test_module)
+        self.apply_count += 1
 
     def assert_module_apply(self, client, instance_id, module,
                             expected_is_admin=False,
@@ -1041,15 +1062,16 @@ class ModuleRunner(TestRunner):
 
     def run_module_list_instance_after_apply(self):
         self.assert_module_list_instance(
-            self.auth_client, self.instance_info.id, 1)
+            self.auth_client, self.instance_info.id, self.apply_count)
 
     def run_module_apply_another(self):
         self.assert_module_apply(self.auth_client, self.instance_info.id,
                                  self.update_test_module)
+        self.apply_count += 1
 
     def run_module_list_instance_after_apply_another(self):
         self.assert_module_list_instance(
-            self.auth_client, self.instance_info.id, 2)
+            self.auth_client, self.instance_info.id, self.apply_count)
 
     def run_module_update_after_remove(self):
         name, description, contents, priority, order = (
@@ -1068,10 +1090,11 @@ class ModuleRunner(TestRunner):
 
     def run_module_instance_count_after_apply(self):
         self.assert_module_instance_count(
-            self.auth_client, self.main_test_module.id, 1, 1)
+            self.auth_client, self.main_test_module.id, 1,
+            {self.main_test_module.md5: 1})
 
     def run_module_query_after_apply(self):
-        expected_count = self.module_auto_apply_count_prior_to_create + 1
+        expected_count = self.module_auto_apply_count_prior_to_create + 2
         expected_results = self.create_default_query_expected_results(
             [self.main_test_module])
         self.assert_module_query(self.auth_client, self.instance_info.id,
@@ -1110,16 +1133,44 @@ class ModuleRunner(TestRunner):
 
     def run_module_instance_count_after_apply_another(self):
         self.assert_module_instance_count(
-            self.auth_client, self.main_test_module.id, 1, 1)
+            self.auth_client, self.main_test_module.id, 1,
+            {self.main_test_module.md5: 1})
 
     def run_module_query_after_apply_another(self):
-        expected_count = self.module_auto_apply_count_prior_to_create + 2
+        expected_count = self.module_auto_apply_count_prior_to_create + 3
         expected_results = self.create_default_query_expected_results(
             [self.main_test_module, self.update_test_module])
         self.assert_module_query(self.auth_client, self.instance_info.id,
                                  expected_count=expected_count,
                                  expected_results=expected_results)
 
+    def run_module_update_not_live(
+            self, expected_exception=exceptions.Forbidden,
+            expected_http_code=403):
+        client = self.auth_client
+        self.assert_raises(
+            expected_exception, expected_http_code,
+            client, client.modules.update,
+            self.main_test_module.id, description='Do not allow this change')
+
+    def run_module_apply_live_update(self):
+        module = self.live_update_test_module
+        self.assert_module_apply(self.auth_client, self.instance_info.id,
+                                 module, expected_is_admin=module.is_admin)
+        self.apply_count += 1
+
+    def run_module_list_instance_after_apply_live(self):
+        self.assert_module_list_instance(
+            self.auth_client, self.instance_info.id, self.apply_count)
+
+    def run_module_update_live_update(self):
+        module = self.live_update_test_module
+        new_contents = self.get_module_contents(name=module.name + '_upd')
+        self.assert_module_update(
+            self.admin_client,
+            module.id,
+            contents=new_contents)
+
     def run_module_update_after_remove_again(self):
         self.assert_module_update(
             self.auth_client,
@@ -1129,10 +1180,13 @@ class ModuleRunner(TestRunner):
             all_datastore_versions=True)
 
     def run_create_inst_with_mods(self, expected_http_code=200):
+        live_update = self.live_update_test_module
         self.mod_inst_id = self.assert_inst_mod_create(
-            self.main_test_module.id, '_module', expected_http_code)
+            [self.main_test_module.id, live_update.id],
+            '_module', expected_http_code)
+        self.mod_inst_apply_count += 2
 
-    def assert_inst_mod_create(self, module_id, name_suffix,
+    def assert_inst_mod_create(self, module_ids, name_suffix,
                                expected_http_code):
         client = self.auth_client
         inst = client.instances.create(
@@ -1142,7 +1196,7 @@ class ModuleRunner(TestRunner):
             datastore=self.instance_info.dbaas_datastore,
             datastore_version=self.instance_info.dbaas_datastore_version,
             nics=self.instance_info.nics,
-            modules=[module_id],
+            modules=module_ids,
         )
         self.assert_client_code(client, expected_http_code)
         self.register_debug_inst_ids(inst.id)
@@ -1188,7 +1242,7 @@ class ModuleRunner(TestRunner):
 
     def run_module_query_after_inst_create(self):
         auto_modules = self._find_all_auto_apply_modules(visible=True)
-        expected_count = 1 + len(auto_modules)
+        expected_count = self.mod_inst_apply_count + len(auto_modules)
         expected_results = self.create_default_query_expected_results(
             [self.main_test_module] + auto_modules)
         self.assert_module_query(self.auth_client, self.mod_inst_id,
@@ -1197,7 +1251,7 @@ class ModuleRunner(TestRunner):
 
     def run_module_retrieve_after_inst_create(self):
         auto_modules = self._find_all_auto_apply_modules(visible=True)
-        expected_count = 1 + len(auto_modules)
+        expected_count = self.mod_inst_apply_count + len(auto_modules)
         expected_results = self.create_default_query_expected_results(
             [self.main_test_module] + auto_modules)
         self.assert_module_retrieve(self.auth_client, self.mod_inst_id,
@@ -1242,7 +1296,7 @@ class ModuleRunner(TestRunner):
 
     def run_module_query_after_inst_create_admin(self):
         auto_modules = self._find_all_auto_apply_modules()
-        expected_count = 1 + len(auto_modules)
+        expected_count = self.mod_inst_apply_count + len(auto_modules)
         expected_results = self.create_default_query_expected_results(
             [self.main_test_module] + auto_modules, is_admin=True)
         self.assert_module_query(self.admin_client, self.mod_inst_id,
@@ -1250,9 +1304,8 @@ class ModuleRunner(TestRunner):
                                  expected_results=expected_results)
 
     def run_module_retrieve_after_inst_create_admin(self):
-        pass
         auto_modules = self._find_all_auto_apply_modules()
-        expected_count = 1 + len(auto_modules)
+        expected_count = self.mod_inst_apply_count + len(auto_modules)
         expected_results = self.create_default_query_expected_results(
             [self.main_test_module] + auto_modules, is_admin=True)
         self.assert_module_retrieve(self.admin_client, self.mod_inst_id,
@@ -1268,6 +1321,143 @@ class ModuleRunner(TestRunner):
             expected_exception, expected_http_code,
             client, client.modules.delete, module.id)
 
+    def run_module_list_instance_after_mod_inst(self):
+        self.assert_module_list_instance(
+            self.auth_client, self.mod_inst_id,
+            self.module_auto_apply_create_count + 2)
+
+    def run_module_instances_after_mod_inst(self):
+        self.assert_module_instances(
+            self.auth_client, self.live_update_test_module.id, 2)
+
+    def run_module_instance_count_after_mod_inst(self):
+        self.assert_module_instance_count(
+            self.auth_client, self.live_update_test_module.id, 2,
+            {self.live_update_test_module.md5: 1,
+             self.live_update_orig_md5: 1})
+
+    def run_module_reapply_with_md5(self, expected_http_code=202):
+        self.assert_module_reapply(
+            self.auth_client, self.live_update_test_module,
+            expected_http_code=expected_http_code,
+            md5=self.live_update_test_module.md5)
+
+    def assert_module_reapply(self, client, module, expected_http_code,
+                              md5=None, force=False):
+        self.reapply_max_upd_date = self.get_updated(client, module.id)
+        client.modules.reapply(module.id, md5=md5, force=force)
+        self.assert_client_code(client, expected_http_code)
+
+    def run_module_reapply_with_md5_verify(self):
+        # since this isn't supposed to do anything, we can't 'wait' for it to
+        # finish, since we'll never know. So just sleep for a couple seconds
+        # just to make sure.
+        time.sleep(2)
+        # Now we check that the max_updated_date field didn't change
+        module_id = self.live_update_test_module.id
+        instance_count_list = self.auth_client.modules.instances(
+            module_id, count_only=True)
+        mismatch = False
+        for instance_count in instance_count_list:
+            if self.reapply_max_upd_date != instance_count.max_updated_date:
+                mismatch = True
+        self.assert_true(
+            mismatch,
+            "Could not find record having max_updated_date different from %s" %
+            self.reapply_max_upd_date)
+
+    def run_module_list_instance_after_reapply_md5(self):
+        self.assert_module_list_instance(
+            self.auth_client, self.mod_inst_id,
+            self.module_auto_apply_create_count + 2)
+
+    def run_module_instances_after_reapply_md5(self):
+        self.assert_module_instances(
+            self.auth_client, self.live_update_test_module.id, 2)
+
+    def run_module_instance_count_after_reapply_md5(self):
+        self.assert_module_instance_count(
+            self.auth_client, self.live_update_test_module.id, 2,
+            {self.live_update_test_module.md5: 1,
+             self.live_update_orig_md5: 1})
+
+    def run_module_reapply_all(self, expected_http_code=202):
+        module_id = self.live_update_test_module.id
+        client = self.auth_client
+        self.reapply_max_upd_date = self.get_updated(client, module_id)
+        self.assert_module_reapply(
+            client, self.live_update_test_module,
+            expected_http_code=expected_http_code)
+
+    def run_module_reapply_all_wait(self):
+        self.wait_for_reapply(
+            self.auth_client, self.live_update_test_module.id,
+            md5=self.live_update_orig_md5)
+
+    def wait_for_reapply(self, client, module_id, updated=None, md5=None):
+        """Reapply is done when all the counts for 'md5' are gone.  If updated
+        is passed in, the min_updated_date must all be greater than it.
+        """
+        if not updated and not md5:
+            raise RuntimeError("Code error: Must pass in 'updated' or 'md5'.")
+        self.report.log("Waiting for all md5:%s modules to have an updated "
+                        "date greater than %s" % (md5, updated))
+
+        def _all_updated():
+            min_updated = self.get_updated(
+                client, module_id, max=False, md5=md5)
+            if md5:
+                return min_updated is None
+            return min_updated > updated
+
+        timeout = 60
+        try:
+            poll_until(_all_updated, time_out=timeout, sleep_time=5)
+            self.report.log("All instances now have the current module "
+                            "for md5: %s." % md5)
+        except exception.PollTimeOut:
+            self.fail("Some instances were not updated with the "
+                      "timeout: %ds" % timeout)
+
+    def get_updated(self, client, module_id, max=True, md5=None):
+        updated = None
+        instance_count_list = client.modules.instances(
+            module_id, count_only=True)
+        for instance_count in instance_count_list:
+            if not md5 or md5 == instance_count.module_md5:
+                if not updated or (
+                        (max and instance_count.max_updated_date > updated) or
+                        (not max and
+                         instance_count.min_updated_date < updated)):
+                    updated = (instance_count.max_updated_date
+                               if max else instance_count.min_updated_date)
+        return updated
+
+    def run_module_list_instance_after_reapply(self):
+        self.assert_module_list_instance(
+            self.auth_client, self.mod_inst_id,
+            self.module_auto_apply_create_count + 2)
+
+    def run_module_instances_after_reapply(self):
+        self.assert_module_instances(
+            self.auth_client, self.live_update_test_module.id, 2)
+
+    def run_module_instance_count_after_reapply(self):
+        self.assert_module_instance_count(
+            self.auth_client, self.live_update_test_module.id, 1,
+            {self.live_update_test_module.md5: 2})
+
+    def run_module_reapply_with_force(self, expected_http_code=202):
+        self.assert_module_reapply(
+            self.auth_client, self.live_update_test_module,
+            expected_http_code=expected_http_code,
+            force=True)
+
+    def run_module_reapply_with_force_wait(self):
+        self.wait_for_reapply(
+            self.auth_client, self.live_update_test_module.id,
+            updated=self.reapply_max_upd_date)
+
     def run_delete_inst_with_mods(self, expected_http_code=202):
         self.assert_delete_instance(self.mod_inst_id, expected_http_code)
 
@@ -1276,6 +1466,14 @@ class ModuleRunner(TestRunner):
         client.instances.delete(instance_id)
         self.assert_client_code(client, expected_http_code)
 
+    def run_remove_mods_from_main_inst(self, expected_http_code=200):
+        client = self.auth_client
+        modquery_list = client.instances.module_query(self.instance_info.id)
+        self.assert_client_code(client, expected_http_code)
+        for modquery in modquery_list:
+            client.instances.module_remove(self.instance_info.id, modquery.id)
+            self.assert_client_code(client, expected_http_code)
+
     def run_wait_for_delete_inst_with_mods(
             self, expected_last_state=['SHUTDOWN']):
         self.assert_all_gone(self.mod_inst_id, expected_last_state)