[Ci] Get changes in resources after task run

This patch adds module tests.ci.osresources, which saves listing
of most important cloud resources before task run, and makes
comparison with another listing after task is complete.

Now we can check if there is a resources leak in cleanup

Change-Id: I5338c77ebd9390ec74a3f1a2028cc0b1ea1b51cf

rally/ui/templates/ci/index.mako

@@ -50,6 +50,7 @@
       <li><a href="console.html" class="rich check-gz">Benchmarking logs</a> <code>console.html</code>
       <li><a href="logs/">Logs of all services</a> <code>logs/</code>
       <li><a href="rally-plot/">Rally files</a> <code>rally-plot/</code>
+      <li><a href="rally-plot/resources_diff.txt.gz">Changes in resources</a> <code>rally-plot/resources_diff.txt</code>
     </ul>
 
     <h2>Job results, in different formats</h2>

tests/ci/osresources.py

@@ -0,0 +1,292 @@
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+"""List and compare most used OpenStack cloud resources."""
+
+
+import argparse
+import json
+import subprocess
+import sys
+
+from rally.common.plugin import discover
+from rally import consts
+from rally import osclients
+
+
+class ResourceManager(object):
+
+    REQUIRED_SERVICE = None
+    REPR_KEYS = ("id", "name", "tenant_id", "zone", "zoneName", "pool")
+
+    def __init__(self, clients):
+        self.clients = clients
+
+    def is_available(self):
+        if self.REQUIRED_SERVICE:
+            return self.REQUIRED_SERVICE in self.clients.services().values()
+        return True
+
+    @property
+    def client(self):
+        return getattr(self.clients, self.__class__.__name__.lower())()
+
+    def get_resources(self):
+        all_resources = []
+        cls = self.__class__.__name__.lower()
+        for prop in dir(self):
+            if not prop.startswith("list_"):
+                continue
+            f = getattr(self, prop)
+            resources = f() or []
+            resource_name = prop[5:][:-1]
+
+            for res in resources:
+                res_repr = []
+                for key in self.REPR_KEYS + (resource_name,):
+                    if isinstance(res, dict):
+                        value = res.get(key)
+                    else:
+                        value = getattr(res, key, None)
+                    if value:
+                        res_repr.append("%s:%s" % (key, value))
+                if not res_repr:
+                    raise ValueError("Failed to represent resource %r" % res)
+
+                all_resources.append(
+                    "%s %s %s" % (cls, resource_name, " ".join(res_repr)))
+        return all_resources
+
+
+class Keystone(ResourceManager):
+
+    def list_users(self):
+        return self.client.users.list()
+
+    def list_tenants(self):
+        return self.client.tenants.list()
+
+    def list_roles(self):
+        return self.client.roles.list()
+
+
+class Nova(ResourceManager):
+
+    def list_flavors(self):
+        return self.client.flavors.list()
+
+    def list_floating_ip_pools(self):
+        return self.client.floating_ip_pools.list()
+
+    def list_floating_ips(self):
+        return self.client.floating_ips.list()
+
+    def list_images(self):
+        return self.client.images.list()
+
+    def list_keypairs(self):
+        return self.client.keypairs.list()
+
+    def list_networks(self):
+        return self.client.networks.list()
+
+    def list_security_groups(self):
+        return self.client.security_groups.list(
+            search_opts={"all_tenants": True})
+
+    def list_servers(self):
+        return self.client.servers.list(
+            search_opts={"all_tenants": True})
+
+    def list_services(self):
+        return self.client.services.list()
+
+    def list_availability_zones(self):
+        return self.client.availability_zones.list()
+
+
+class Neutron(ResourceManager):
+
+    REQUIRED_SERVICE = consts.Service.NEUTRON
+
+    def has_extension(self, name):
+        extensions = self.client.list_extensions().get("extensions", [])
+        return any(ext.get("alias") == name for ext in extensions)
+
+    def list_networks(self):
+        return self.client.list_networks()["networks"]
+
+    def list_subnets(self):
+        return self.client.list_subnets()["subnets"]
+
+    def list_routers(self):
+        return self.client.list_routers()["routers"]
+
+    def list_ports(self):
+        return self.client.list_ports()["ports"]
+
+    def list_floatingips(self):
+        return self.client.list_floatingips()["floatingips"]
+
+    def list_security_groups(self):
+        return self.client.list_security_groups()["security_groups"]
+
+    def list_health_monitors(self):
+        if self.has_extension("lbaas"):
+            return self.client.list_health_monitors()["health_monitors"]
+
+    def list_pools(self):
+        if self.has_extension("lbaas"):
+            return self.client.list_pools()["pools"]
+
+    def list_vips(self):
+        if self.has_extension("lbaas"):
+            return self.client.list_vips()["vips"]
+
+
+class Glance(ResourceManager):
+
+    def list_images(self):
+        return self.client.images.list()
+
+
+class Heat(ResourceManager):
+
+    def list_resource_types(self):
+        return self.client.resource_types.list()
+
+    def list_stacks(self):
+        return self.client.stacks.list()
+
+
+class Cinder(ResourceManager):
+
+    def list_availability_zones(self):
+        return self.client.availability_zones.list()
+
+    def list_backups(self):
+        return self.client.backups.list()
+
+    def list_volume_snapshots(self):
+        return self.client.volume_snapshots.list()
+
+    def list_volume_types(self):
+        return self.client.volume_types.list()
+
+    def list_volumes(self):
+        return self.client.volumes.list(
+            search_opts={"all_tenants": True})
+
+
+class CloudResources(object):
+    """List and compare cloud resources.
+
+    resources = CloudResources(auth_url=..., ...)
+    saved_list = resources.list()
+
+    # Do something with the cloud ...
+
+    changes = resources.compare(saved_list)
+    has_changed = any(changes)
+    removed, added = changes
+    """
+
+    def __init__(self, **kwargs):
+        endpoint = osclients.objects.Endpoint(**kwargs)
+        self.clients = osclients.Clients(endpoint)
+
+    def _deduplicate(self, lst):
+        """Change list duplicates to make all items unique.
+
+        >>> resources._deduplicate(["a", "b", "c", "b", "b"])
+        >>> ['a', 'b', 'c', 'b (duplicate 1)', 'b (duplicate 2)'
+        """
+        deduplicated_list = []
+        for value in lst:
+            if value in deduplicated_list:
+                ctr = 0
+                try_value = value
+                while try_value in deduplicated_list:
+                    ctr += 1
+                    try_value = "%s (duplicate %i)" % (value, ctr)
+                value = try_value
+            deduplicated_list.append(value)
+        return deduplicated_list
+
+    def list(self):
+        managers_classes = discover.itersubclasses(ResourceManager)
+        resources = []
+        for cls in managers_classes:
+            manager = cls(self.clients)
+            if manager.is_available():
+                resources.extend(manager.get_resources())
+        return sorted(self._deduplicate(resources))
+
+    def compare(self, with_list):
+        saved_resources = set(with_list)
+        current_resources = set(self.list())
+        removed = saved_resources - current_resources
+        added = current_resources - saved_resources
+
+        return sorted(list(removed)), sorted(list(added))
+
+
+def main():
+
+    parser = argparse.ArgumentParser(
+        description=("Save list of OpenStack cloud resources or compare "
+                     "with previously saved list."))
+    parser.add_argument("--credentials",
+                        type=argparse.FileType("r"),
+                        metavar="<path/to/credentials.json>",
+                        help="cloud credentials in JSON format")
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument("--dump-list",
+                       type=argparse.FileType("w"),
+                       metavar="<path/to/output/list.json>",
+                       help="dump resources to given file in JSON format")
+    group.add_argument("--compare-with-list",
+                       type=argparse.FileType("r"),
+                       metavar="<path/to/existent/list.json>",
+                       help=("compare current resources with a list from "
+                             "given JSON file"))
+    args = parser.parse_args()
+
+    if args.credentials:
+        config = json.load(args.credentials)
+    else:
+        config = json.loads(subprocess.check_output(["rally", "deployment",
+                                                     "config"]))
+        config.update(config.pop("admin"))
+        del config["type"]
+
+    resources = CloudResources(**config)
+
+    if args.dump_list:
+        resources_list = resources.list()
+        json.dump(resources_list, args.dump_list, indent=2)
+    elif args.compare_with_list:
+        given_list = json.load(args.compare_with_list)
+        changes = resources.compare(with_list=given_list)
+        removed, added = changes
+        sys.stdout.write(
+            json.dumps({"removed": removed, "added": added}, indent=2))
+        if any(changes):
+            return 0  # `1' will fail gate job
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/ci/rally-gate.sh

@@ -60,6 +60,9 @@ rally show networks
 rally show secgroups
 rally show keypairs
 
+python $BASE/new/rally/tests/ci/osresources.py\
+    --dump-list resources_at_start.txt
+
 rally -v --rally-debug task start --task $TASK $TASK_ARGS
 
 mkdir -p rally-plot/extra
@@ -75,3 +78,8 @@ gzip -9 rally-plot/detailed_with_iterations.txt
 rally task report --out rally-plot/results.html
 gzip -9 rally-plot/results.html
 rally task sla_check | tee rally-plot/sla.txt
+
+python $BASE/new/rally/tests/ci/osresources.py\
+    --compare-with-list resources_at_start.txt\
+        | gzip > rally-plot/resources_diff.txt.gz
+cp resources_at_start.txt rally-plot/