Merge "Fuel version from cluster used in OSWL reports"

analytics/fuel_analytics/api/resources/csv_exporter.py

@@ -179,6 +179,7 @@ def get_oswls_query(resource_type, from_date=None, to_date=None):
         IS.creation_date.label('installation_created_date'),
         IS.modification_date.label('installation_updated_date'),
         IS.structure['fuel_release'].label('fuel_release_from_inst_info'),
+        IS.structure['clusters'].label('clusters'),
         IS.is_filtered).\
         join(IS, IS.master_node_uid == OSWS.master_node_uid).\
         filter(OSWS.resource_type == resource_type).\

analytics/fuel_analytics/api/resources/utils/oswl_stats_to_csv.py

@@ -95,25 +95,69 @@ class OswlStatsToCsv(object):
 
         return result
 
-    def handle_empty_version_info(self, oswl):
+    def _add_oswl_to_clusters_versions_cache(self, oswl, clusters_versions):
+        """Adds oswl clusters info into clusters_versions cache.
+
+        :param oswl: OSWL DB object
+        :type oswl: fuel_analytics.api.db.model.OpenStackWorkloadStats
+        :param clusters_versions: cache for saving cluster versions with
+        structure {mn_uid: {cluster_id: fuel_release}}
+        :type clusters_versions: dict
+        """
+
+        mn_uid = oswl.master_node_uid
+
+        # Result of csv_exporter.get_oswls_query contains info about all
+        # clusters in the installation. Thus we need to add clusters data
+        # into the cache only once for specified master_node_uid.
+        if mn_uid in clusters_versions:
+            return
+
+        if oswl.clusters is None:
+            return
+
+        clusters_versions[mn_uid] = {}
+
+        for cluster in oswl.clusters:
+            fuel_release = cluster.get('fuel_release')
+            if fuel_release:
+                clusters_versions[mn_uid][cluster['id']] = fuel_release
+
+    def handle_empty_version_info(self, oswl, clusters_versions):
         """Handles empty version info in oswl object
 
         For OSWLs with empty version_info data we compose version_info
         from InstallationStructure data and assign it to oswl object.
-        If we extract OpenStack release, os, name from
+        We bound InstallationStructure.structure.clusters to the oswl
-        InstallationStructure.structure.clusters we have performance
+        and extract fuel_release from clusters data. If fuel_release
-        degradation on fetching all clusters data in csv_exporter.get_oswls
+        info doesn't provided by clusters data then
-        thus only fuel_release will be used in case of empty version_info.
+        InstallationStructure.structure.fuel_release is used.
 
         :param oswl: OSWL DB object
         :type oswl: fuel_analytics.api.db.model.OpenStackWorkloadStats
+        :param clusters_versions: cache for saving cluster versions with
+        structure {mn_uid: {cluster_id: fuel_release}}
+        :type clusters_versions: dict
         """
         if oswl.version_info:
             return
 
-        fuel_release = oswl.fuel_release_from_inst_info or {}
+        self._add_oswl_to_clusters_versions_cache(oswl, clusters_versions)
+
+        mn_uid = oswl.master_node_uid
+        cluster_id = oswl.cluster_id
+
+        # Fetching fuel_release info from cache
+        fuel_release = clusters_versions.get(mn_uid, {}).get(cluster_id)
+
+        # If clusters data doesn't contain fuel_release info we are using
+        # info from installation info
+        if fuel_release is None:
+            fuel_release = oswl.fuel_release_from_inst_info or {}
+            fuel_release = fuel_release.get('release')
+
         oswl.version_info = {
-            'fuel_release': fuel_release.get('release')
+            'fuel_release': fuel_release
         }
 
     def get_flatten_resources(self, resource_type, oswl_keys_paths,
@@ -126,9 +170,15 @@ class OswlStatsToCsv(object):
         :return: generator on flatten resources info collection
         """
         app.logger.debug("Getting OSWL flatten %s info started", resource_type)
+
+        # Cache for saving cluster versions. Cache is used only if version_info
+        # is not provided in the oswl.
+        # Structure: {mn_uid: {cluster_id: fuel_release}}
+        clusters_versions = {}
+
         for oswl in oswls:
             try:
-                self.handle_empty_version_info(oswl)
+                self.handle_empty_version_info(oswl, clusters_versions)
                 flatten_oswl = export_utils.get_flatten_data(oswl_keys_paths,
                                                              oswl)
                 resource_data = oswl.resource_data

analytics/fuel_analytics/api/resources/utils/stats_to_csv.py

@@ -86,9 +86,8 @@ class StatsToCsv(object):
 
         plugin_key_paths = export_utils.get_keys_paths(plugin_skeleton)
         structure_key_paths = [['master_node_uid'],
-                               ['structure', 'fuel_packages'],
+                               ['structure', 'fuel_packages']]
-                               ['structure', 'fuel_release', 'release']]
+        cluster_key_paths = [['cluster_id'], ['cluster_fuel_version']]
-        cluster_key_paths = [['cluster_id']]
         result_key_paths = plugin_key_paths + cluster_key_paths + \
             structure_key_paths
         app.logger.debug("Plugin keys paths got")
@@ -202,6 +201,8 @@ class StatsToCsv(object):
 
                 for cluster in clusters:
                     cluster['cluster_id'] = cluster['id']
+                    cluster['cluster_fuel_version'] = \
+                        cluster.get('fuel_version')
                     flatten_cluster = export_utils.get_flatten_data(
                         cluster_keys_paths, cluster)
                     plugins = cluster.pop('installed_plugins', [])

analytics/fuel_analytics/test/api/resources/utils/test_oswl_stats_to_csv.py

@@ -28,6 +28,7 @@ from fuel_analytics.test.base import DbTest
 from fuel_analytics.api.app import app
 from fuel_analytics.api.app import db
 from fuel_analytics.api.common import consts
+from fuel_analytics.api.db.model import InstallationStructure
 from fuel_analytics.api.db.model import OpenStackWorkloadStats
 from fuel_analytics.api.resources.csv_exporter import get_oswls
 from fuel_analytics.api.resources.csv_exporter import get_oswls_query
@@ -853,3 +854,95 @@ class OswlStatsToCsvTest(OswlTest, DbTest):
         # The fourth oswl status True in is_modified, is_deleted
         check_resource_state(flatten_resources[6], 'fourth',
                              False, True, True)
+
+    def test_fuel_version_from_clusters_data_is_used(self):
+        master_node_uid = 'x'
+        exporter = OswlStatsToCsv()
+        resource_type = consts.OSWL_RESOURCE_TYPES.vm
+        version_from_cluster = '7.0'
+        version_from_version_info = '9.0'
+        version_from_installation_info = '8.0'
+        installation_date = datetime.utcnow().date() - timedelta(days=3)
+
+        # Upgraded Fuel and not upgraded cluster
+        structure = InstallationStructure(
+            master_node_uid=master_node_uid,
+            structure={
+                'fuel_release': {'release': version_from_installation_info},
+                'clusters_num': 2,
+                'clusters': [
+                    {'id': 1, 'fuel_release': version_from_cluster},
+                    {'id': 2}
+                ],
+                'unallocated_nodes_num_range': 0,
+                'allocated_nodes_num': 0
+            },
+            creation_date=installation_date,
+            is_filtered=False
+        )
+        db.session.add(structure)
+
+        oswls = [
+            OpenStackWorkloadStats(
+                master_node_uid=master_node_uid,
+                external_id=1,
+                cluster_id=1,
+                created_date=installation_date,
+                updated_time=datetime.utcnow().time(),
+                resource_type=resource_type,
+                resource_checksum='info_from_cluster',
+                resource_data={'current': [{'id': 1, 'status': 'enabled'}],
+                               'added': [], 'modified': [], 'removed': []},
+                version_info=None
+            ),
+            OpenStackWorkloadStats(
+                master_node_uid=master_node_uid,
+                external_id=3,
+                cluster_id=1,
+                created_date=installation_date + timedelta(days=1),
+                updated_time=datetime.utcnow().time(),
+                resource_type=resource_type,
+                resource_checksum='info_from_version_info',
+                resource_data={'current': [{'id': 1}],
+                               'added': [], 'modified': [], 'removed': []},
+                version_info={'fuel_release': version_from_version_info}
+            ),
+            OpenStackWorkloadStats(
+                master_node_uid=master_node_uid,
+                external_id=2,
+                cluster_id=2,
+                created_date=installation_date + timedelta(days=2),
+                updated_time=datetime.utcnow().time(),
+                resource_type=resource_type,
+                resource_checksum='info_from_installation_info',
+                resource_data={'current': [{'id': 1}],
+                               'added': [], 'modified': [], 'removed': []},
+                version_info=None
+            )
+        ]
+        for oswl in oswls:
+            db.session.add(oswl)
+
+        with app.test_request_context():
+            oswls_data = list(get_oswls(resource_type))
+
+        oswl_keys_paths, resource_keys_paths, csv_keys_paths = \
+            exporter.get_resource_keys_paths(resource_type)
+        fuel_release_pos = csv_keys_paths.index(
+            ['version_info', 'fuel_release'])
+        flatten_resources = list(exporter.get_flatten_resources(
+            resource_type, oswl_keys_paths, resource_keys_paths, oswls_data))
+
+        self.assertEqual(len(oswls), len(flatten_resources))
+
+        # Checking release info fetched from cluster
+        self.assertEqual(version_from_cluster,
+                         flatten_resources[0][fuel_release_pos])
+
+        # Checking release info fetched from oswl.version_info
+        self.assertEqual(version_from_version_info,
+                         flatten_resources[1][fuel_release_pos])
+
+        # Checking release info fetched from installation info
+        self.assertEqual(version_from_installation_info,
+                         flatten_resources[2][fuel_release_pos])

analytics/fuel_analytics/test/api/resources/utils/test_plugins_to_csv.py

@@ -36,14 +36,13 @@ class PluginsToCsvExportTest(InstStructureTest, DbTest):
         exporter = StatsToCsv()
         _, _, _, csv_keys_paths = exporter.get_plugin_keys_paths()
         self.assertTrue(['cluster_id'] in csv_keys_paths)
+        self.assertTrue(['cluster_fuel_version'] in csv_keys_paths)
         self.assertTrue(['master_node_uid'] in csv_keys_paths)
         self.assertTrue(['name'] in csv_keys_paths)
         self.assertTrue(['version'] in csv_keys_paths)
         self.assertTrue(['fuel_version'] in csv_keys_paths)
         self.assertTrue(['package_version'] in csv_keys_paths)
         self.assertTrue(['structure', 'fuel_packages'] in csv_keys_paths)
-        self.assertTrue(['structure', 'fuel_release', 'release'] in
-                        csv_keys_paths)
 
     def test_get_flatten_plugins(self):
         installations_num = 10
@@ -100,7 +99,8 @@ class PluginsToCsvExportTest(InstStructureTest, DbTest):
                 self.assertEqual(num - 1, len(list(flatten_plugins)))
 
     def test_fuel_release_info_in_flatten_plugins(self):
-        release = '8.0'
+        inst_fuel_version = '8.0'
+        cluster_fuel_version = '7.0'
         packages = ['z', 'a', 'c']
         inst_structures = [
             model.InstallationStructure(
@@ -108,10 +108,11 @@ class PluginsToCsvExportTest(InstStructureTest, DbTest):
                 creation_date=datetime.datetime.utcnow(),
                 is_filtered=False,
                 structure={
-                    'fuel_release': {'release': release},
+                    'fuel_release': {'release': inst_fuel_version},
                     'fuel_packages': packages,
                     'clusters': [{
                         'id': 1, 'nodes': [],
+                        'fuel_version': cluster_fuel_version,
                         'installed_plugins': [{
                             'name': 'plugin_a',
                             'version': 'plugin_version_0',
@@ -133,10 +134,10 @@ class PluginsToCsvExportTest(InstStructureTest, DbTest):
         flatten_plugins = exporter.get_flatten_plugins(
             structure_paths, cluster_paths, plugins_paths, inst_structures)
 
-        pos_release = csv_paths.index(['structure', 'fuel_release',
+        pos_fuel_version = csv_paths.index(['cluster_fuel_version'])
-                                       'release'])
         pos_packages = csv_paths.index(['structure', 'fuel_packages'])
         for flatten_plugin in flatten_plugins:
-            self.assertEqual(release, flatten_plugin[pos_release])
+            self.assertEqual(cluster_fuel_version,
+                             flatten_plugin[pos_fuel_version])
             self.assertEqual(' '.join(packages),
                              flatten_plugin[pos_packages])