From f999b15bd4152ec3169046b05fa625a54f430150 Mon Sep 17 00:00:00 2001
From: Katarina Strenkova <kstrenko@redhat.com>
Date: Tue, 10 Oct 2023 15:16:15 +0000
Subject: [PATCH] Implement purge list for tempest cleanup

This patch adds an option to create purge list that logs every
resource created by Tempest. When used with tempest cleanup
command, it creates a new method of deleting resources created
during one or multiple Tempest runs.

This method solves the problem of accidentally deleting
resources which were not created by Tempest, but by users.

Change-Id: Ide81e6a41799bace211669951b4ceab8635b56ab
---
 .../notes/resource-list-cbf9779e8b434654.yaml |  11 +
 tempest/clients.py                            |  10 +
 tempest/cmd/cleanup.py                        |  81 ++-
 tempest/cmd/cleanup_service.py                | 102 +++-
 tempest/config.py                             |   9 +
 tempest/lib/common/rest_client.py             |  80 ++-
 .../services/compute/server_groups_client.py  |  11 +-
 tempest/tests/cmd/test_cleanup.py             |  23 +-
 tempest/tests/cmd/test_cleanup_services.py    | 473 +++++++++++++++++-
 tempest/tests/cmd/test_resource_list.json     |  11 +
 tempest/tests/lib/common/test_rest_client.py  | 105 ++++
 11 files changed, 890 insertions(+), 26 deletions(-)
 create mode 100644 releasenotes/notes/resource-list-cbf9779e8b434654.yaml
 create mode 100644 tempest/tests/cmd/test_resource_list.json

diff --git a/releasenotes/notes/resource-list-cbf9779e8b434654.yaml b/releasenotes/notes/resource-list-cbf9779e8b434654.yaml
new file mode 100644
index 0000000000..bbd2f16850
--- /dev/null
+++ b/releasenotes/notes/resource-list-cbf9779e8b434654.yaml
@@ -0,0 +1,11 @@
+---
+features:
+  - |
+    A new interface ``--resource-list`` has been introduced in the
+    ``tempest cleanup`` command to remove the resources created by
+    Tempest. A new config option in the default section, ``record_resources``,
+    is added to allow the recording of all resources created by Tempest.
+    A list of these resources will be saved in ``resource_list.json`` file,
+    which will be appended in case of multiple Tempest runs. This file
+    is intended to be used with the ``tempest cleanup`` command if it is
+    used with the newly added option ``--resource-list``.
diff --git a/tempest/clients.py b/tempest/clients.py
index 5338ed4c7c..e432120eba 100644
--- a/tempest/clients.py
+++ b/tempest/clients.py
@@ -13,8 +13,13 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import os
+
+from oslo_concurrency import lockutils
+
 from tempest import config
 from tempest.lib import auth
+from tempest.lib.common.rest_client import RestClient
 from tempest.lib import exceptions as lib_exc
 from tempest.lib.services import clients
 
@@ -35,6 +40,11 @@ class Manager(clients.ServiceClients):
         super(Manager, self).__init__(
             credentials=credentials, identity_uri=identity_uri, scope=scope,
             region=CONF.identity.region)
+        if CONF.record_resources:
+            RestClient.lock_dir = os.path.join(
+                lockutils.get_lock_path(CONF),
+                'tempest-rec-rw-lock')
+            RestClient.record_resources = True
         # TODO(andreaf) When clients are initialised without the right
         # parameters available, the calls below will trigger a KeyError.
         # We should catch that and raise a better error.
diff --git a/tempest/cmd/cleanup.py b/tempest/cmd/cleanup.py
index 2a406dedcf..8d06f930cf 100644
--- a/tempest/cmd/cleanup.py
+++ b/tempest/cmd/cleanup.py
@@ -87,6 +87,23 @@ Runtime Arguments
   ``saved_state.json`` file will be ignored and cleanup will be done based on
   the passed prefix only.
 
+* ``--resource-list``: Allows the use of file ``./resource_list.json``, which
+  contains all resources created by Tempest during all Tempest runs, to
+  create another method for removing only resources created by Tempest.
+  List of these resources is created when config option ``record_resources``
+  in default section is set to true. After using this option for cleanup,
+  the existing ``./resource_list.json`` is cleared from deleted resources.
+
+  When this option is used, ``saved_state.json`` file is not needed (no
+  need to run with ``--init-saved-state`` first). If there is any
+  ``saved_state.json`` file present and you run the tempest cleanup with
+  ``--resource-list``, the ``saved_state.json`` file will be ignored and
+  cleanup will be done based on the ``resource_list.json`` only.
+
+  If you run tempest cleanup with both ``--prefix`` and ``--resource-list``,
+  the ``--resource-list`` option will be ignored and cleanup will be done
+  based on the ``--prefix`` option only.
+
 * ``--help``: Print the help text for the command and parameters.
 
 .. [1] The ``_projects_to_clean`` dictionary in ``dry_run.json`` lists the
@@ -122,6 +139,7 @@ from tempest.lib import exceptions
 
 SAVED_STATE_JSON = "saved_state.json"
 DRY_RUN_JSON = "dry_run.json"
+RESOURCE_LIST_JSON = "resource_list.json"
 LOG = logging.getLogger(__name__)
 CONF = config.CONF
 
@@ -164,6 +182,7 @@ class TempestCleanup(command.Command):
         self.admin_mgr = clients.Manager(
             credentials.get_configured_admin_credentials())
         self.dry_run_data = {}
+        self.resource_data = {}
         self.json_data = {}
 
         # available services
@@ -177,12 +196,20 @@ class TempestCleanup(command.Command):
             self._init_state()
             return
 
-        self._load_json()
+        if parsed_args.prefix:
+            return
+
+        if parsed_args.resource_list:
+            self._load_resource_list()
+            return
+
+        self._load_saved_state()
 
     def _cleanup(self):
         LOG.info("Begin cleanup")
         is_dry_run = self.options.dry_run
         is_preserve = not self.options.delete_tempest_conf_objects
+        is_resource_list = self.options.resource_list
         is_save_state = False
         cleanup_prefix = self.options.prefix
 
@@ -194,8 +221,10 @@ class TempestCleanup(command.Command):
         # they are in saved state json. Therefore is_preserve is False
         kwargs = {'data': self.dry_run_data,
                   'is_dry_run': is_dry_run,
+                  'resource_list_json': self.resource_data,
                   'saved_state_json': self.json_data,
                   'is_preserve': False,
+                  'is_resource_list': is_resource_list,
                   'is_save_state': is_save_state,
                   'prefix': cleanup_prefix}
         project_service = cleanup_service.ProjectService(admin_mgr, **kwargs)
@@ -208,8 +237,10 @@ class TempestCleanup(command.Command):
 
         kwargs = {'data': self.dry_run_data,
                   'is_dry_run': is_dry_run,
+                  'resource_list_json': self.resource_data,
                   'saved_state_json': self.json_data,
                   'is_preserve': is_preserve,
+                  'is_resource_list': is_resource_list,
                   'is_save_state': is_save_state,
                   'prefix': cleanup_prefix,
                   'got_exceptions': self.GOT_EXCEPTIONS}
@@ -228,11 +259,17 @@ class TempestCleanup(command.Command):
                 f.write(json.dumps(self.dry_run_data, sort_keys=True,
                                    indent=2, separators=(',', ': ')))
 
+        if is_resource_list:
+            LOG.info("Clearing 'resource_list.json' file.")
+            with open(RESOURCE_LIST_JSON, 'w') as f:
+                f.write('{}')
+
     def _clean_project(self, project):
         LOG.debug("Cleaning project:  %s ", project['name'])
         is_dry_run = self.options.dry_run
         dry_run_data = self.dry_run_data
         is_preserve = not self.options.delete_tempest_conf_objects
+        is_resource_list = self.options.resource_list
         project_id = project['id']
         project_name = project['name']
         project_data = None
@@ -244,7 +281,9 @@ class TempestCleanup(command.Command):
         kwargs = {'data': project_data,
                   'is_dry_run': is_dry_run,
                   'saved_state_json': self.json_data,
+                  'resource_list_json': self.resource_data,
                   'is_preserve': is_preserve,
+                  'is_resource_list': is_resource_list,
                   'is_save_state': False,
                   'project_id': project_id,
                   'prefix': cleanup_prefix,
@@ -287,6 +326,19 @@ class TempestCleanup(command.Command):
                             "ignored when --init-saved-state is used so that "
                             "it can capture the true init state - all "
                             "resources present at that moment.")
+        parser.add_argument('--resource-list', action="store_true",
+                            dest='resource_list', default=False,
+                            help="Runs tempest cleanup with generated "
+                            "JSON file: " + RESOURCE_LIST_JSON + " to "
+                            "erase resources created during Tempest run. "
+                            "NOTE: To create " + RESOURCE_LIST_JSON + " "
+                            "set config option record_resources under default "
+                            "section in tempest.conf file to true. This "
+                            "option will be ignored when --init-saved-state "
+                            "is used so that it can capture the true init "
+                            "state - all resources present at that moment. "
+                            "This option will be ignored if passed with "
+                            "--prefix.")
         return parser
 
     def get_description(self):
@@ -304,6 +356,7 @@ class TempestCleanup(command.Command):
                   'is_dry_run': False,
                   'saved_state_json': data,
                   'is_preserve': False,
+                  'is_resource_list': False,
                   'is_save_state': True,
                   # must be None as we want to capture true init state
                   # (all resources present) thus no filtering based
@@ -326,15 +379,31 @@ class TempestCleanup(command.Command):
             f.write(json.dumps(data, sort_keys=True,
                                indent=2, separators=(',', ': ')))
 
-    def _load_json(self, saved_state_json=SAVED_STATE_JSON):
+    def _load_resource_list(self, resource_list_json=RESOURCE_LIST_JSON):
+        try:
+            with open(resource_list_json, 'rb') as json_file:
+                self.resource_data = json.load(json_file)
+        except IOError as ex:
+            LOG.exception(
+                "Failed loading 'resource_list.json', please "
+                "be sure you created this file by setting config "
+                "option record_resources in default section to true "
+                "prior to running tempest. Exception: %s", ex)
+            sys.exit(ex)
+        except Exception as ex:
+            LOG.exception(
+                "Exception parsing 'resource_list.json' : %s", ex)
+            sys.exit(ex)
+
+    def _load_saved_state(self, saved_state_json=SAVED_STATE_JSON):
         try:
             with open(saved_state_json, 'rb') as json_file:
                 self.json_data = json.load(json_file)
-
         except IOError as ex:
-            LOG.exception("Failed loading saved state, please be sure you"
-                          " have first run cleanup with --init-saved-state "
-                          "flag prior to running tempest. Exception: %s", ex)
+            LOG.exception(
+                "Failed loading saved state, please be sure you"
+                " have first run cleanup with --init-saved-state "
+                "flag prior to running tempest. Exception: %s", ex)
             sys.exit(ex)
         except Exception as ex:
             LOG.exception("Exception parsing saved state json : %s", ex)
diff --git a/tempest/cmd/cleanup_service.py b/tempest/cmd/cleanup_service.py
index 8651ab064d..b20294003b 100644
--- a/tempest/cmd/cleanup_service.py
+++ b/tempest/cmd/cleanup_service.py
@@ -120,6 +120,13 @@ class BaseService(object):
                  if item['name'].startswith(self.prefix)]
         return items
 
+    def _filter_by_resource_list(self, item_list, attr):
+        if attr not in self.resource_list_json:
+            return []
+        items = [item for item in item_list if item['id']
+                 in self.resource_list_json[attr].keys()]
+        return items
+
     def _filter_out_ids_from_saved(self, item_list, attr):
         items = [item for item in item_list if item['id']
                  not in self.saved_state_json[attr].keys()]
@@ -166,8 +173,11 @@ class SnapshotService(BaseService):
     def list(self):
         client = self.client
         snaps = client.list_snapshots()['snapshots']
+
         if self.prefix:
             snaps = self._filter_by_prefix(snaps)
+        elif self.is_resource_list:
+            snaps = self._filter_by_resource_list(snaps, 'snapshots')
         elif not self.is_save_state:
             # recreate list removing saved snapshots
             snaps = self._filter_out_ids_from_saved(snaps, 'snapshots')
@@ -205,8 +215,11 @@ class ServerService(BaseService):
         client = self.client
         servers_body = client.list_servers()
         servers = servers_body['servers']
+
         if self.prefix:
             servers = self._filter_by_prefix(servers)
+        elif self.is_resource_list:
+            servers = self._filter_by_resource_list(servers, 'servers')
         elif not self.is_save_state:
             # recreate list removing saved servers
             servers = self._filter_out_ids_from_saved(servers, 'servers')
@@ -238,9 +251,12 @@ class ServerGroupService(ServerService):
 
     def list(self):
         client = self.server_groups_client
-        sgs = client.list_server_groups()['server_groups']
+        sgs = client.list_server_groups(all_projects=True)['server_groups']
+
         if self.prefix:
             sgs = self._filter_by_prefix(sgs)
+        elif self.is_resource_list:
+            sgs = self._filter_by_resource_list(sgs, 'server_groups')
         elif not self.is_save_state:
             # recreate list removing saved server_groups
             sgs = self._filter_out_ids_from_saved(sgs, 'server_groups')
@@ -276,8 +292,13 @@ class KeyPairService(BaseService):
     def list(self):
         client = self.client
         keypairs = client.list_keypairs()['keypairs']
+
         if self.prefix:
             keypairs = self._filter_by_prefix(keypairs)
+        elif self.is_resource_list:
+            keypairs = [keypair for keypair in keypairs
+                        if keypair['keypair']['name']
+                        in self.resource_list_json['keypairs'].keys()]
         elif not self.is_save_state:
             # recreate list removing saved keypairs
             keypairs = [keypair for keypair in keypairs
@@ -317,8 +338,11 @@ class VolumeService(BaseService):
     def list(self):
         client = self.client
         vols = client.list_volumes()['volumes']
+
         if self.prefix:
             vols = self._filter_by_prefix(vols)
+        elif self.is_resource_list:
+            vols = self._filter_by_resource_list(vols, 'volumes')
         elif not self.is_save_state:
             # recreate list removing saved volumes
             vols = self._filter_out_ids_from_saved(vols, 'volumes')
@@ -462,8 +486,11 @@ class NetworkService(BaseNetworkService):
         client = self.networks_client
         networks = client.list_networks(**self.tenant_filter)
         networks = networks['networks']
+
         if self.prefix:
             networks = self._filter_by_prefix(networks)
+        elif self.is_resource_list:
+            networks = self._filter_by_resource_list(networks, 'networks')
         else:
             if not self.is_save_state:
                 # recreate list removing saved networks
@@ -500,15 +527,17 @@ class NetworkService(BaseNetworkService):
 class NetworkFloatingIpService(BaseNetworkService):
 
     def list(self):
-        if self.prefix:
-            # this means we're cleaning resources based on a certain prefix,
-            # this resource doesn't have a name, therefore return empty list
-            return []
         client = self.floating_ips_client
         flips = client.list_floatingips(**self.tenant_filter)
         flips = flips['floatingips']
 
-        if not self.is_save_state:
+        if self.prefix:
+            # this means we're cleaning resources based on a certain prefix,
+            # this resource doesn't have a name, therefore return empty list
+            return []
+        elif self.is_resource_list:
+            flips = self._filter_by_resource_list(flips, 'floatingips')
+        elif not self.is_save_state:
             # recreate list removing saved flips
             flips = self._filter_out_ids_from_saved(flips, 'floatingips')
         LOG.debug("List count, %s Network Floating IPs", len(flips))
@@ -543,8 +572,11 @@ class NetworkRouterService(BaseNetworkService):
         client = self.routers_client
         routers = client.list_routers(**self.tenant_filter)
         routers = routers['routers']
+
         if self.prefix:
             routers = self._filter_by_prefix(routers)
+        elif self.is_resource_list:
+            routers = self._filter_by_resource_list(routers, 'routers')
         else:
             if not self.is_save_state:
                 # recreate list removing saved routers
@@ -592,16 +624,19 @@ class NetworkRouterService(BaseNetworkService):
 class NetworkMeteringLabelRuleService(NetworkService):
 
     def list(self):
-        if self.prefix:
-            # this means we're cleaning resources based on a certain prefix,
-            # this resource doesn't have a name, therefore return empty list
-            return []
         client = self.metering_label_rules_client
         rules = client.list_metering_label_rules()
         rules = rules['metering_label_rules']
         rules = self._filter_by_tenant_id(rules)
 
-        if not self.is_save_state:
+        if self.prefix:
+            # this means we're cleaning resources based on a certain prefix,
+            # this resource doesn't have a name, therefore return empty list
+            return []
+        elif self.is_resource_list:
+            rules = self._filter_by_resource_list(
+                rules, 'metering_label_rules')
+        elif not self.is_save_state:
             rules = self._filter_out_ids_from_saved(
                 rules, 'metering_label_rules')
             # recreate list removing saved rules
@@ -638,8 +673,12 @@ class NetworkMeteringLabelService(BaseNetworkService):
         labels = client.list_metering_labels()
         labels = labels['metering_labels']
         labels = self._filter_by_tenant_id(labels)
+
         if self.prefix:
             labels = self._filter_by_prefix(labels)
+        elif self.is_resource_list:
+            labels = self._filter_by_resource_list(
+                labels, 'metering_labels')
         elif not self.is_save_state:
             # recreate list removing saved labels
             labels = self._filter_out_ids_from_saved(
@@ -677,8 +716,11 @@ class NetworkPortService(BaseNetworkService):
                  client.list_ports(**self.tenant_filter)['ports']
                  if port["device_owner"] == "" or
                  port["device_owner"].startswith("compute:")]
+
         if self.prefix:
             ports = self._filter_by_prefix(ports)
+        elif self.is_resource_list:
+            ports = self._filter_by_resource_list(ports, 'ports')
         else:
             if not self.is_save_state:
                 # recreate list removing saved ports
@@ -717,8 +759,12 @@ class NetworkSecGroupService(BaseNetworkService):
         secgroups = [secgroup for secgroup in
                      client.list_security_groups(**filter)['security_groups']
                      if secgroup['name'] != 'default']
+
         if self.prefix:
             secgroups = self._filter_by_prefix(secgroups)
+        elif self.is_resource_list:
+            secgroups = self._filter_by_resource_list(
+                secgroups, 'security_groups')
         else:
             if not self.is_save_state:
                 # recreate list removing saved security_groups
@@ -760,8 +806,11 @@ class NetworkSubnetService(BaseNetworkService):
         client = self.subnets_client
         subnets = client.list_subnets(**self.tenant_filter)
         subnets = subnets['subnets']
+
         if self.prefix:
             subnets = self._filter_by_prefix(subnets)
+        elif self.is_resource_list:
+            subnets = self._filter_by_resource_list(subnets, 'subnets')
         else:
             if not self.is_save_state:
                 # recreate list removing saved subnets
@@ -797,8 +846,11 @@ class NetworkSubnetPoolsService(BaseNetworkService):
     def list(self):
         client = self.subnetpools_client
         pools = client.list_subnetpools(**self.tenant_filter)['subnetpools']
+
         if self.prefix:
             pools = self._filter_by_prefix(pools)
+        elif self.is_resource_list:
+            pools = self._filter_by_resource_list(pools, 'subnetpools')
         else:
             if not self.is_save_state:
                 # recreate list removing saved subnet pools
@@ -838,13 +890,18 @@ class RegionService(BaseService):
         self.client = manager.regions_client
 
     def list(self):
+        client = self.client
+        regions = client.list_regions()
+
         if self.prefix:
             # this means we're cleaning resources based on a certain prefix,
             # this resource doesn't have a name, therefore return empty list
             return []
-        client = self.client
-        regions = client.list_regions()
-        if not self.is_save_state:
+        elif self.is_resource_list:
+            regions = self._filter_by_resource_list(
+                regions['regions'], 'regions')
+            return regions
+        elif not self.is_save_state:
             regions = self._filter_out_ids_from_saved(
                 regions['regions'], 'regions')
             LOG.debug("List count, %s Regions", len(regions))
@@ -884,8 +941,11 @@ class FlavorService(BaseService):
     def list(self):
         client = self.client
         flavors = client.list_flavors({"is_public": None})['flavors']
+
         if self.prefix:
             flavors = self._filter_by_prefix(flavors)
+        elif self.is_resource_list:
+            flavors = self._filter_by_resource_list(flavors, 'flavors')
         else:
             if not self.is_save_state:
                 # recreate list removing saved flavors
@@ -932,8 +992,11 @@ class ImageService(BaseService):
             marker = urllib.parse_qs(parsed.query)['marker'][0]
             response = client.list_images(params={"marker": marker})
             images.extend(response['images'])
+
         if self.prefix:
             images = self._filter_by_prefix(images)
+        elif self.is_resource_list:
+            images = self._filter_by_resource_list(images, 'images')
         else:
             if not self.is_save_state:
                 images = self._filter_out_ids_from_saved(images, 'images')
@@ -974,6 +1037,8 @@ class UserService(BaseService):
         users = self.client.list_users()['users']
         if self.prefix:
             users = self._filter_by_prefix(users)
+        elif self.is_resource_list:
+            users = self._filter_by_resource_list(users, 'users')
         else:
             if not self.is_save_state:
                 users = self._filter_out_ids_from_saved(users, 'users')
@@ -1015,8 +1080,11 @@ class RoleService(BaseService):
     def list(self):
         try:
             roles = self.client.list_roles()['roles']
+
             if self.prefix:
                 roles = self._filter_by_prefix(roles)
+            elif self.is_resource_list:
+                roles = self._filter_by_resource_list(roles, 'roles')
             elif not self.is_save_state:
                 # reconcile roles with saved state and never list admin role
                 roles = self._filter_out_ids_from_saved(roles, 'roles')
@@ -1056,8 +1124,11 @@ class ProjectService(BaseService):
 
     def list(self):
         projects = self.client.list_projects()['projects']
+
         if self.prefix:
             projects = self._filter_by_prefix(projects)
+        elif self.is_resource_list:
+            projects = self._filter_by_resource_list(projects, 'projects')
         else:
             if not self.is_save_state:
                 projects = self._filter_out_ids_from_saved(
@@ -1099,8 +1170,11 @@ class DomainService(BaseService):
     def list(self):
         client = self.client
         domains = client.list_domains()['domains']
+
         if self.prefix:
             domains = self._filter_by_prefix(domains)
+        elif self.is_resource_list:
+            domains = self._filter_by_resource_list(domains, 'domains')
         elif not self.is_save_state:
             domains = self._filter_out_ids_from_saved(domains, 'domains')
         LOG.debug("List count, %s Domains after reconcile", len(domains))
diff --git a/tempest/config.py b/tempest/config.py
index 0a084eaf1f..507d1161db 100644
--- a/tempest/config.py
+++ b/tempest/config.py
@@ -1317,6 +1317,15 @@ or
                     "to cleanup only the resources that match the prefix. "
                     "Make sure this prefix does not match with the resource "
                     "name you do not want Tempest cleanup CLI to delete."),
+    cfg.BoolOpt('record_resources',
+                default=False,
+                help="Allows to record all resources created by Tempest. "
+                     "These resources are stored in file resource_list.json, "
+                     "which can be later used for resource deletion by "
+                     "command tempest cleanup. The resource_list.json file "
+                     "will be appended in case of multiple Tempest runs, "
+                     "so the file will contain a list of resources created "
+                     "during all Tempest runs."),
 ]
 
 _opts = [
diff --git a/tempest/lib/common/rest_client.py b/tempest/lib/common/rest_client.py
index 6cf5b734cb..4f9e9ba663 100644
--- a/tempest/lib/common/rest_client.py
+++ b/tempest/lib/common/rest_client.py
@@ -21,6 +21,7 @@ import time
 import urllib
 import urllib3
 
+from fasteners import process_lock
 import jsonschema
 from oslo_log import log as logging
 from oslo_log import versionutils
@@ -78,6 +79,17 @@ class RestClient(object):
     # The version of the API this client implements
     api_version = None
 
+    # Directory for storing read-write lock
+    lock_dir = None
+
+    # An interprocess lock used when the recording of all resources created by
+    # Tempest is allowed.
+    rec_rw_lock = None
+
+    # Variable mirrors value in config option 'record_resources' that allows
+    # the recording of all resources created by Tempest.
+    record_resources = False
+
     LOG = logging.getLogger(__name__)
 
     def __init__(self, auth_provider, service, region,
@@ -297,7 +309,13 @@ class RestClient(object):
                  and the second the response body
         :rtype: tuple
         """
-        return self.request('POST', url, extra_headers, headers, body, chunked)
+        resp_header, resp_body = self.request(
+            'POST', url, extra_headers, headers, body, chunked)
+
+        if self.record_resources:
+            self.resource_record(resp_body)
+
+        return resp_header, resp_body
 
     def get(self, url, headers=None, extra_headers=False, chunked=False):
         """Send a HTTP GET request using keystone service catalog and auth
@@ -1006,6 +1024,66 @@ class RestClient(object):
         """Returns the primary type of resource this client works with."""
         return 'resource'
 
+    def resource_update(self, data, res_type, res_dict):
+        """Updates resource_list.json file with current resource."""
+        if not isinstance(res_dict, dict):
+            return
+
+        if not res_type.endswith('s'):
+            res_type += 's'
+
+        if res_type not in data:
+            data[res_type] = {}
+
+        if 'uuid' in res_dict:
+            data[res_type].update(
+                {res_dict.get('uuid'): res_dict.get('name')})
+        elif 'id' in res_dict:
+            data[res_type].update(
+                {res_dict.get('id'): res_dict.get('name')})
+        elif 'name' in res_dict:
+            data[res_type].update({res_dict.get('name'): ""})
+
+        self.rec_rw_lock.acquire_write_lock()
+        with open("resource_list.json", 'w+') as f:
+            f.write(json.dumps(data, indent=2, separators=(',', ': ')))
+        self.rec_rw_lock.release_write_lock()
+
+    def resource_record(self, resp_dict):
+        """Records resources into resource_list.json file."""
+        if self.rec_rw_lock is None:
+            path = self.lock_dir
+            self.rec_rw_lock = (
+                process_lock.InterProcessReaderWriterLock(path)
+            )
+
+        self.rec_rw_lock.acquire_read_lock()
+        try:
+            with open('resource_list.json', 'rb') as f:
+                data = json.load(f)
+        except IOError:
+            data = {}
+        self.rec_rw_lock.release_read_lock()
+
+        try:
+            resp_dict = json.loads(resp_dict.decode('utf-8'))
+        except (AttributeError, TypeError, ValueError):
+            return
+
+        # check if response has any keys
+        if not resp_dict.keys():
+            return
+
+        resource_type = list(resp_dict.keys())[0]
+
+        resource_dict = resp_dict[resource_type]
+
+        if isinstance(resource_dict, list):
+            for resource in resource_dict:
+                self.resource_update(data, resource_type, resource)
+        else:
+            self.resource_update(data, resource_type, resource_dict)
+
     @classmethod
     def validate_response(cls, schema, resp, body):
         # Only check the response if the status code is a success code
diff --git a/tempest/lib/services/compute/server_groups_client.py b/tempest/lib/services/compute/server_groups_client.py
index 98956538e3..5c1e623a17 100644
--- a/tempest/lib/services/compute/server_groups_client.py
+++ b/tempest/lib/services/compute/server_groups_client.py
@@ -14,6 +14,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from urllib import parse as urllib
+
 from oslo_serialization import jsonutils as json
 
 from tempest.lib.api_schema.response.compute.v2_1 import server_groups \
@@ -55,9 +57,14 @@ class ServerGroupsClient(base_compute_client.BaseComputeClient):
         self.validate_response(schema.delete_server_group, resp, body)
         return rest_client.ResponseBody(resp, body)
 
-    def list_server_groups(self):
+    def list_server_groups(self, **params):
         """List the server-groups."""
-        resp, body = self.get("os-server-groups")
+
+        url = 'os-server-groups'
+        if params:
+            url += '?%s' % urllib.urlencode(params)
+
+        resp, body = self.get(url)
         body = json.loads(body)
         schema = self.get_schema(self.schema_versions_info)
         self.validate_response(schema.list_server_groups, resp, body)
diff --git a/tempest/tests/cmd/test_cleanup.py b/tempest/tests/cmd/test_cleanup.py
index 69e735b55b..3efc9bd8fe 100644
--- a/tempest/tests/cmd/test_cleanup.py
+++ b/tempest/tests/cmd/test_cleanup.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 from unittest import mock
 
 from tempest.cmd import cleanup
@@ -20,12 +21,30 @@ from tempest.tests import base
 
 class TestTempestCleanup(base.TestCase):
 
-    def test_load_json(self):
+    def test_load_json_saved_state(self):
         # instantiate "empty" TempestCleanup
         c = cleanup.TempestCleanup(None, None, 'test')
         test_saved_json = 'tempest/tests/cmd/test_saved_state_json.json'
+        with open(test_saved_json, 'r') as f:
+            test_saved_json_content = json.load(f)
         # test if the file is loaded without any issues/exceptions
-        c._load_json(test_saved_json)
+        c.options = mock.Mock()
+        c.options.init_saved_state = True
+        c._load_saved_state(test_saved_json)
+        self.assertEqual(c.json_data, test_saved_json_content)
+
+    def test_load_json_resource_list(self):
+        # instantiate "empty" TempestCleanup
+        c = cleanup.TempestCleanup(None, None, 'test')
+        test_resource_list = 'tempest/tests/cmd/test_resource_list.json'
+        with open(test_resource_list, 'r') as f:
+            test_resource_list_content = json.load(f)
+        # test if the file is loaded without any issues/exceptions
+        c.options = mock.Mock()
+        c.options.init_saved_state = False
+        c.options.resource_list = True
+        c._load_resource_list(test_resource_list)
+        self.assertEqual(c.resource_data, test_resource_list_content)
 
     @mock.patch('tempest.cmd.cleanup.TempestCleanup.init')
     @mock.patch('tempest.cmd.cleanup.TempestCleanup._cleanup')
diff --git a/tempest/tests/cmd/test_cleanup_services.py b/tempest/tests/cmd/test_cleanup_services.py
index 6b3b4b7a59..2557145311 100644
--- a/tempest/tests/cmd/test_cleanup_services.py
+++ b/tempest/tests/cmd/test_cleanup_services.py
@@ -41,8 +41,10 @@ class TestBaseService(base.TestCase):
     def test_base_service_init(self):
         kwargs = {'data': {'data': 'test'},
                   'is_dry_run': False,
+                  'resource_list_json': {'resp': 'data'},
                   'saved_state_json': {'saved': 'data'},
                   'is_preserve': False,
+                  'is_resource_list': False,
                   'is_save_state': True,
                   'prefix': 'tempest',
                   'tenant_id': 'project_id',
@@ -50,8 +52,10 @@ class TestBaseService(base.TestCase):
         base = cleanup_service.BaseService(kwargs)
         self.assertEqual(base.data, kwargs['data'])
         self.assertFalse(base.is_dry_run)
+        self.assertEqual(base.resource_list_json, kwargs['resource_list_json'])
         self.assertEqual(base.saved_state_json, kwargs['saved_state_json'])
         self.assertFalse(base.is_preserve)
+        self.assertFalse(base.is_resource_list)
         self.assertTrue(base.is_save_state)
         self.assertEqual(base.tenant_filter['project_id'], kwargs['tenant_id'])
         self.assertEqual(base.got_exceptions, kwargs['got_exceptions'])
@@ -60,8 +64,10 @@ class TestBaseService(base.TestCase):
     def test_not_implemented_ex(self):
         kwargs = {'data': {'data': 'test'},
                   'is_dry_run': False,
+                  'resource_list_json': {'resp': 'data'},
                   'saved_state_json': {'saved': 'data'},
                   'is_preserve': False,
+                  'is_resource_list': False,
                   'is_save_state': False,
                   'prefix': 'tempest',
                   'tenant_id': 'project_id',
@@ -181,10 +187,20 @@ class BaseCmdServiceTests(MockFunctionsBase):
         "subnetpools": {'8acf64c1-43fc': 'saved-subnet-pool'},
         "regions": {'RegionOne': {}}
     }
+
+    resource_list = {
+        "keypairs": {'saved-key-pair': ""}
+    }
+
     # Mocked methods
     get_method = 'tempest.lib.common.rest_client.RestClient.get'
     delete_method = 'tempest.lib.common.rest_client.RestClient.delete'
     log_method = 'tempest.cmd.cleanup_service.LOG.exception'
+    filter_saved_state = 'tempest.cmd.cleanup_service.' \
+                         'BaseService._filter_out_ids_from_saved'
+    filter_resource_list = 'tempest.cmd.cleanup_service.' \
+                           'BaseService._filter_by_resource_list'
+    filter_prefix = 'tempest.cmd.cleanup_service.BaseService._filter_by_prefix'
     # Override parameters
     service_class = 'BaseService'
     response = None
@@ -192,17 +208,19 @@ class BaseCmdServiceTests(MockFunctionsBase):
 
     def _create_cmd_service(self, service_type, is_save_state=False,
                             is_preserve=False, is_dry_run=False,
-                            prefix=''):
+                            prefix='', is_resource_list=False):
         creds = fake_credentials.FakeKeystoneV3Credentials()
         os = clients.Manager(creds)
         return getattr(cleanup_service, service_type)(
             os,
+            is_resource_list=is_resource_list,
             is_save_state=is_save_state,
             is_preserve=is_preserve,
             is_dry_run=is_dry_run,
             prefix=prefix,
             project_id='b8e3ece07bb049138d224436756e3b57',
             data={},
+            resource_list_json=self.resource_list,
             saved_state_json=self.saved_state
             )
 
@@ -266,6 +284,38 @@ class BaseCmdServiceTests(MockFunctionsBase):
             self.assertNotIn(rsp['id'], self.conf_values.values())
             self.assertNotIn(rsp['name'], self.conf_values.values())
 
+    def _test_prefix_opt_precedence(self, delete_mock):
+        serv = self._create_cmd_service(
+            self.service_class, is_resource_list=True, prefix='tempest')
+        _, fixtures = self.run_function_with_mocks(
+            serv.run,
+            delete_mock
+        )
+
+        # Check that prefix was used for filtering
+        fixtures[2].mock.assert_called_once()
+
+        # Check that neither saved_state.json nor resource list was
+        # used for filtering
+        fixtures[0].mock.assert_not_called()
+        fixtures[1].mock.assert_not_called()
+
+    def _test_resource_list_opt_precedence(self, delete_mock):
+        serv = self._create_cmd_service(
+            self.service_class, is_resource_list=True)
+        _, fixtures = self.run_function_with_mocks(
+            serv.run,
+            delete_mock
+        )
+
+        # Check that resource list was used for filtering
+        fixtures[1].mock.assert_called_once()
+
+        # Check that neither saved_state.json nor prefix was
+        # used for filtering
+        fixtures[0].mock.assert_not_called()
+        fixtures[2].mock.assert_not_called()
+
 
 class TestSnapshotService(BaseCmdServiceTests):
 
@@ -320,6 +370,24 @@ class TestSnapshotService(BaseCmdServiceTests):
     def test_save_state(self):
         self._test_saved_state_true([(self.get_method, self.response, 200)])
 
+    def test_prefix_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_prefix_opt_precedence(delete_mock)
+
+    def test_resource_list_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_resource_list_opt_precedence(delete_mock)
+
 
 class TestServerService(BaseCmdServiceTests):
 
@@ -378,6 +446,24 @@ class TestServerService(BaseCmdServiceTests):
     def test_save_state(self):
         self._test_saved_state_true([(self.get_method, self.response, 200)])
 
+    def test_prefix_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_prefix_opt_precedence(delete_mock)
+
+    def test_resource_list_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_resource_list_opt_precedence(delete_mock)
+
 
 class TestServerGroupService(BaseCmdServiceTests):
 
@@ -429,6 +515,26 @@ class TestServerGroupService(BaseCmdServiceTests):
                                      (self.validate_response, 'validate', None)
                                      ])
 
+    def test_prefix_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.validate_response, 'validate', None),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_prefix_opt_precedence(delete_mock)
+
+    def test_resource_list_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.validate_response, 'validate', None),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_resource_list_opt_precedence(delete_mock)
+
 
 class TestKeyPairService(BaseCmdServiceTests):
 
@@ -493,6 +599,33 @@ class TestKeyPairService(BaseCmdServiceTests):
             (self.validate_response, 'validate', None)
         ])
 
+    def test_prefix_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.validate_response, 'validate', None),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_prefix_opt_precedence(delete_mock)
+
+    def test_resource_list_opt_precedence(self):
+        delete_mock = [(self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.validate_response, 'validate', None),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        serv = self._create_cmd_service(
+            self.service_class, is_resource_list=True)
+
+        _, fixtures = self.run_function_with_mocks(
+            serv.delete,
+            delete_mock
+        )
+
+        # Check that prefix was not used for filtering
+        fixtures[0].mock.assert_not_called()
+
 
 class TestVolumeService(BaseCmdServiceTests):
 
@@ -542,6 +675,24 @@ class TestVolumeService(BaseCmdServiceTests):
     def test_save_state(self):
         self._test_saved_state_true([(self.get_method, self.response, 200)])
 
+    def test_prefix_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_prefix_opt_precedence(delete_mock)
+
+    def test_resource_list_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_resource_list_opt_precedence(delete_mock)
+
 
 class TestVolumeQuotaService(BaseCmdServiceTests):
 
@@ -761,6 +912,24 @@ class TestNetworkService(BaseCmdServiceTests):
             })
         self._test_is_preserve_true([(self.get_method, self.response, 200)])
 
+    def test_prefix_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_prefix_opt_precedence(delete_mock)
+
+    def test_resource_list_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_resource_list_opt_precedence(delete_mock)
+
 
 class TestNetworkFloatingIpService(BaseCmdServiceTests):
 
@@ -823,6 +992,34 @@ class TestNetworkFloatingIpService(BaseCmdServiceTests):
     def test_save_state(self):
         self._test_saved_state_true([(self.get_method, self.response, 200)])
 
+    def test_prefix_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        serv = self._create_cmd_service(
+            self.service_class, is_resource_list=True, prefix='tempest')
+        _, fixtures = self.run_function_with_mocks(
+            serv.run,
+            delete_mock
+        )
+
+        # cleanup returns []
+        fixtures[0].mock.assert_not_called()
+        fixtures[1].mock.assert_not_called()
+        fixtures[2].mock.assert_not_called()
+
+    def test_resource_list_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_resource_list_opt_precedence(delete_mock)
+
 
 class TestNetworkRouterService(BaseCmdServiceTests):
 
@@ -937,6 +1134,24 @@ class TestNetworkRouterService(BaseCmdServiceTests):
             })
         self._test_is_preserve_true([(self.get_method, self.response, 200)])
 
+    def test_prefix_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_prefix_opt_precedence(delete_mock)
+
+    def test_resource_list_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_resource_list_opt_precedence(delete_mock)
+
 
 class TestNetworkMeteringLabelRuleService(BaseCmdServiceTests):
 
@@ -978,6 +1193,34 @@ class TestNetworkMeteringLabelRuleService(BaseCmdServiceTests):
     def test_save_state(self):
         self._test_saved_state_true([(self.get_method, self.response, 200)])
 
+    def test_prefix_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        serv = self._create_cmd_service(
+            self.service_class, is_resource_list=True, prefix='tempest')
+        _, fixtures = self.run_function_with_mocks(
+            serv.run,
+            delete_mock
+        )
+
+        # cleanup returns []
+        fixtures[0].mock.assert_not_called()
+        fixtures[1].mock.assert_not_called()
+        fixtures[2].mock.assert_not_called()
+
+    def test_resource_list_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_resource_list_opt_precedence(delete_mock)
+
 
 class TestNetworkMeteringLabelService(BaseCmdServiceTests):
 
@@ -1020,6 +1263,24 @@ class TestNetworkMeteringLabelService(BaseCmdServiceTests):
     def test_save_state(self):
         self._test_saved_state_true([(self.get_method, self.response, 200)])
 
+    def test_prefix_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_prefix_opt_precedence(delete_mock)
+
+    def test_resource_list_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_resource_list_opt_precedence(delete_mock)
+
 
 class TestNetworkPortService(BaseCmdServiceTests):
 
@@ -1118,6 +1379,24 @@ class TestNetworkPortService(BaseCmdServiceTests):
             })
         self._test_is_preserve_true([(self.get_method, self.response, 200)])
 
+    def test_prefix_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_prefix_opt_precedence(delete_mock)
+
+    def test_resource_list_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_resource_list_opt_precedence(delete_mock)
+
 
 class TestNetworkSecGroupService(BaseCmdServiceTests):
 
@@ -1196,6 +1475,24 @@ class TestNetworkSecGroupService(BaseCmdServiceTests):
             })
         self._test_is_preserve_true([(self.get_method, self.response, 200)])
 
+    def test_prefix_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_prefix_opt_precedence(delete_mock)
+
+    def test_resource_list_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_resource_list_opt_precedence(delete_mock)
+
 
 class TestNetworkSubnetService(BaseCmdServiceTests):
 
@@ -1272,6 +1569,24 @@ class TestNetworkSubnetService(BaseCmdServiceTests):
             })
         self._test_is_preserve_true([(self.get_method, self.response, 200)])
 
+    def test_prefix_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_prefix_opt_precedence(delete_mock)
+
+    def test_resource_list_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_resource_list_opt_precedence(delete_mock)
+
 
 class TestNetworkSubnetPoolsService(BaseCmdServiceTests):
 
@@ -1340,6 +1655,24 @@ class TestNetworkSubnetPoolsService(BaseCmdServiceTests):
             })
         self._test_is_preserve_true([(self.get_method, self.response, 200)])
 
+    def test_prefix_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_prefix_opt_precedence(delete_mock)
+
+    def test_resource_list_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_resource_list_opt_precedence(delete_mock)
+
 
 # begin global services
 class TestRegionService(BaseCmdServiceTests):
@@ -1392,6 +1725,34 @@ class TestRegionService(BaseCmdServiceTests):
     def test_save_state(self):
         self._test_saved_state_true([(self.get_method, self.response, 200)])
 
+    def test_prefix_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        serv = self._create_cmd_service(
+            self.service_class, is_resource_list=True, prefix='tempest')
+        _, fixtures = self.run_function_with_mocks(
+            serv.run,
+            delete_mock
+        )
+
+        # cleanup returns []
+        fixtures[0].mock.assert_not_called()
+        fixtures[1].mock.assert_not_called()
+        fixtures[2].mock.assert_not_called()
+
+    def test_resource_list_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_resource_list_opt_precedence(delete_mock)
+
 
 class TestDomainService(BaseCmdServiceTests):
 
@@ -1445,6 +1806,26 @@ class TestDomainService(BaseCmdServiceTests):
     def test_save_state(self):
         self._test_saved_state_true([(self.get_method, self.response, 200)])
 
+    def test_prefix_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None),
+                       (self.mock_update, 'update', None)]
+        self._test_prefix_opt_precedence(delete_mock)
+
+    def test_resource_list_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None),
+                       (self.mock_update, 'update', None)]
+        self._test_resource_list_opt_precedence(delete_mock)
+
 
 class TestProjectsService(BaseCmdServiceTests):
 
@@ -1518,6 +1899,24 @@ class TestProjectsService(BaseCmdServiceTests):
             })
         self._test_is_preserve_true([(self.get_method, self.response, 200)])
 
+    def test_prefix_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_prefix_opt_precedence(delete_mock)
+
+    def test_resource_list_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_resource_list_opt_precedence(delete_mock)
+
 
 class TestImagesService(BaseCmdServiceTests):
 
@@ -1597,6 +1996,24 @@ class TestImagesService(BaseCmdServiceTests):
             })
         self._test_is_preserve_true([(self.get_method, self.response, 200)])
 
+    def test_prefix_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_prefix_opt_precedence(delete_mock)
+
+    def test_resource_list_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_resource_list_opt_precedence(delete_mock)
+
 
 class TestFlavorService(BaseCmdServiceTests):
 
@@ -1670,6 +2087,24 @@ class TestFlavorService(BaseCmdServiceTests):
             })
         self._test_is_preserve_true([(self.get_method, self.response, 200)])
 
+    def test_prefix_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_prefix_opt_precedence(delete_mock)
+
+    def test_resource_list_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_resource_list_opt_precedence(delete_mock)
+
 
 class TestRoleService(BaseCmdServiceTests):
 
@@ -1716,6 +2151,24 @@ class TestRoleService(BaseCmdServiceTests):
     def test_save_state(self):
         self._test_saved_state_true([(self.get_method, self.response, 200)])
 
+    def test_prefix_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_prefix_opt_precedence(delete_mock)
+
+    def test_resource_list_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_resource_list_opt_precedence(delete_mock)
+
 
 class TestUserService(BaseCmdServiceTests):
 
@@ -1782,3 +2235,21 @@ class TestUserService(BaseCmdServiceTests):
                 "password_expires_at": "1893-11-06T15:32:17.000000",
             })
         self._test_is_preserve_true([(self.get_method, self.response, 200)])
+
+    def test_prefix_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_prefix_opt_precedence(delete_mock)
+
+    def test_resource_list_opt_precedence(self):
+        delete_mock = [(self.filter_saved_state, [], None),
+                       (self.filter_resource_list, [], None),
+                       (self.filter_prefix, [], None),
+                       (self.get_method, self.response, 200),
+                       (self.delete_method, 'error', None),
+                       (self.log_method, 'exception', None)]
+        self._test_resource_list_opt_precedence(delete_mock)
diff --git a/tempest/tests/cmd/test_resource_list.json b/tempest/tests/cmd/test_resource_list.json
new file mode 100644
index 0000000000..dfbc7900f6
--- /dev/null
+++ b/tempest/tests/cmd/test_resource_list.json
@@ -0,0 +1,11 @@
+{
+  "project": {
+    "ce4e7edf051c439d8b81c4bfe581c5ef": "test"
+  },
+  "keypairs": {
+    "tempest-keypair-1215039183": ""
+  },
+  "users": {
+    "74463c83f9d640fe84c4376527ceff26": "test"
+  }
+}
diff --git a/tempest/tests/lib/common/test_rest_client.py b/tempest/tests/lib/common/test_rest_client.py
index 81a76e02c2..9bc6f60717 100644
--- a/tempest/tests/lib/common/test_rest_client.py
+++ b/tempest/tests/lib/common/test_rest_client.py
@@ -13,6 +13,7 @@
 #    under the License.
 
 import copy
+from unittest import mock
 
 import fixtures
 import jsonschema
@@ -749,6 +750,110 @@ class TestExpectedSuccess(BaseRestClientTestClass):
                           expected_code, read_code)
 
 
+class TestRecordResources(BaseRestClientTestClass):
+
+    def setUp(self):
+        self.fake_http = fake_http.fake_httplib2()
+        super(TestRecordResources, self).setUp()
+
+    def _cleanup_test_resource_record(self):
+        # clear resource_list.json file
+        with open('resource_list.json', 'w') as f:
+            f.write('{}')
+
+    def test_post_record_resources(self):
+        self.rest_client.record_resources = True
+        __, return_dict = self.rest_client.post(self.url, {}, {})
+        self.assertEqual({}, return_dict['headers'])
+        self.assertEqual({}, return_dict['body'])
+
+    def test_resource_record_no_top_key(self):
+        test_body_no_key = b'{}'
+        self.rest_client.resource_record(test_body_no_key)
+
+    def test_resource_record_dict(self):
+        test_dict_body = b'{"project": {"id": "test-id", "name": ""}}\n'
+        self.rest_client.resource_record(test_dict_body)
+
+        with open('resource_list.json', 'r') as f:
+            content = f.read()
+            resource_list_content = json.loads(content)
+
+        test_resource_list = {
+            "projects": {"test-id": ""}
+        }
+        self.assertEqual(resource_list_content, test_resource_list)
+
+        # cleanup
+        self._cleanup_test_resource_record()
+
+    def test_resource_record_list(self):
+        test_list_body = '''{
+            "user": [
+                {
+                    "id": "test-uuid",
+                    "name": "test-name"
+                },
+                {
+                    "id": "test-uuid2",
+                    "name": "test-name2"
+                }
+            ]
+        }'''
+        test_list_body = test_list_body.encode('utf-8')
+        self.rest_client.resource_record(test_list_body)
+
+        with open('resource_list.json', 'r') as f:
+            content = f.read()
+            resource_list_content = json.loads(content)
+
+        test_resource_list = {
+            "users": {
+                "test-uuid": "test-name",
+                "test-uuid2": "test-name2"
+            }
+        }
+        self.assertEqual(resource_list_content, test_resource_list)
+
+        # cleanup
+        self._cleanup_test_resource_record()
+
+    def test_resource_update_id(self):
+        data = {}
+        res_dict = {'id': 'test-uuid', 'name': 'test-name'}
+
+        self.rest_client.rec_rw_lock = mock.MagicMock()
+        self.rest_client.resource_update(data, 'user', res_dict)
+        result = {'users': {'test-uuid': 'test-name'}}
+        self.assertEqual(data, result)
+
+    def test_resource_update_name(self):
+        data = {'keypairs': {}}
+        res_dict = {'name': 'test-keypair'}
+
+        self.rest_client.rec_rw_lock = mock.MagicMock()
+        self.rest_client.resource_update(data, 'keypair', res_dict)
+        result = {'keypairs': {'test-keypair': ""}}
+        self.assertEqual(data, result)
+
+    def test_resource_update_no_id(self):
+        data = {}
+        res_dict = {'type': 'test', 'description': 'example'}
+
+        self.rest_client.rec_rw_lock = mock.MagicMock()
+        self.rest_client.resource_update(data, 'projects', res_dict)
+        result = {'projects': {}}
+        self.assertEqual(data, result)
+
+    def test_resource_update_not_dict(self):
+        data = {}
+        res_dict = 'test-string'
+
+        self.rest_client.rec_rw_lock = mock.MagicMock()
+        self.rest_client.resource_update(data, 'user', res_dict)
+        self.assertEqual(data, {})
+
+
 class TestResponseBody(base.TestCase):
 
     def test_str(self):