995 lines
42 KiB
Python
995 lines
42 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# Copyright 2015 Mirantis, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import csv
|
|
from datetime import datetime
|
|
from datetime import timedelta
|
|
import mock
|
|
import six
|
|
import types
|
|
import uuid
|
|
|
|
from fuel_analytics.test.api.resources.utils.oswl_test import OswlTest
|
|
from fuel_analytics.test.base import DbTest
|
|
|
|
from fuel_analytics.api.app import app
|
|
from fuel_analytics.api.app import db
|
|
from fuel_analytics.api.common import consts
|
|
from fuel_analytics.api.db.model import InstallationStructure
|
|
from fuel_analytics.api.db.model import OpenStackWorkloadStats
|
|
from fuel_analytics.api.resources.csv_exporter import get_clusters_version_info
|
|
from fuel_analytics.api.resources.csv_exporter import get_oswls
|
|
from fuel_analytics.api.resources.csv_exporter import get_oswls_query
|
|
from fuel_analytics.api.resources.utils import export_utils
|
|
from fuel_analytics.api.resources.utils.oswl_stats_to_csv import OswlStatsToCsv
|
|
from fuel_analytics.api.resources.utils.skeleton import OSWL_SKELETONS
|
|
|
|
|
|
class OswlStatsToCsvTest(OswlTest, DbTest):
|
|
|
|
def test_get_keys_paths(self):
|
|
for resource_type in self.RESOURCE_TYPES:
|
|
exporter = OswlStatsToCsv()
|
|
oswl_keys_paths, resource_keys_paths, csv_keys_paths = \
|
|
exporter.get_resource_keys_paths(resource_type)
|
|
self.assertNotIn(['external_id'], oswl_keys_paths)
|
|
self.assertNotIn(['updated_time'], oswl_keys_paths)
|
|
self.assertNotIn(['release'], oswl_keys_paths)
|
|
self.assertIn(['version_info', 'fuel_version'], oswl_keys_paths)
|
|
self.assertIn(['version_info', 'release_version'],
|
|
oswl_keys_paths)
|
|
self.assertIn(['version_info', 'release_name'], oswl_keys_paths)
|
|
self.assertIn(['version_info', 'release_os'], oswl_keys_paths)
|
|
self.assertIn(['version_info', 'environment_version'],
|
|
oswl_keys_paths)
|
|
self.assertIn([resource_type, 'id'], resource_keys_paths)
|
|
self.assertIn([resource_type, 'is_added'], csv_keys_paths)
|
|
self.assertIn([resource_type, 'is_modified'], csv_keys_paths)
|
|
self.assertIn([resource_type, 'is_removed'], csv_keys_paths)
|
|
|
|
def test_get_flatten_resources(self):
|
|
for resource_type in self.RESOURCE_TYPES:
|
|
exporter = OswlStatsToCsv()
|
|
oswl_keys_paths, resource_keys_paths, csv_keys_paths = \
|
|
exporter.get_resource_keys_paths(resource_type)
|
|
oswls = self.generate_oswls(2, resource_type)
|
|
flatten_resources = exporter.get_flatten_resources(
|
|
resource_type, oswl_keys_paths, resource_keys_paths, oswls, {})
|
|
self.assertIsInstance(flatten_resources, types.GeneratorType)
|
|
for _ in flatten_resources:
|
|
pass
|
|
|
|
def test_flavor_ephemeral_in_flatten(self):
|
|
exporter = OswlStatsToCsv()
|
|
resource_type = consts.OSWL_RESOURCE_TYPES.flavor
|
|
oswl_keys_paths, resource_keys_paths, csv_keys_paths = \
|
|
exporter.get_resource_keys_paths(resource_type)
|
|
oswls = self.generate_oswls(1, resource_type)
|
|
flatten_resources = exporter.get_flatten_resources(
|
|
resource_type, oswl_keys_paths, resource_keys_paths, oswls, {})
|
|
|
|
ephemeral_idx = csv_keys_paths.index(['flavor', 'ephemeral'])
|
|
for fr in flatten_resources:
|
|
self.assertIsNotNone(fr[ephemeral_idx])
|
|
|
|
def test_get_additional_info(self):
|
|
exporter = OswlStatsToCsv()
|
|
added_num = 0
|
|
modified_num = 3
|
|
removed_num = 5
|
|
num = 1
|
|
for resource_type in self.RESOURCE_TYPES:
|
|
oswls = self.generate_oswls(
|
|
num,
|
|
resource_type,
|
|
added_num_range=(added_num, added_num),
|
|
modified_num_range=(modified_num, modified_num),
|
|
removed_num_range=(removed_num, removed_num)
|
|
)
|
|
oswl = oswls.next()
|
|
|
|
# Saving data for true JSON loading from DB object
|
|
db.session.add(oswl)
|
|
resource_data = oswl.resource_data
|
|
added_ids = set(d['id'] for d in resource_data['added'])
|
|
modified_ids = set(d['id'] for d in resource_data['modified'])
|
|
removed_ids = set(d['id'] for d in resource_data['removed'])
|
|
for resource in resource_data['current']:
|
|
resource_id = resource['id']
|
|
expected = [
|
|
resource_id in added_ids,
|
|
resource_id in modified_ids,
|
|
resource_id in removed_ids
|
|
]
|
|
added_ids = set(item['id'] for item in
|
|
resource_data.get('added', []))
|
|
modified_ids = set(item['id'] for item in
|
|
resource_data.get('removed', []))
|
|
removed_ids = set(item['id'] for item in
|
|
resource_data.get('modified', []))
|
|
|
|
actual = exporter.get_additional_resource_info(
|
|
resource, oswl.resource_type,
|
|
added_ids, modified_ids, removed_ids)
|
|
self.assertListEqual(expected, actual)
|
|
|
|
def test_export(self):
|
|
exporter = OswlStatsToCsv()
|
|
num = 200
|
|
with app.test_request_context():
|
|
for resource_type in self.RESOURCE_TYPES:
|
|
# Saving data for true JSON loading from DB object
|
|
oswls_saved = self.get_saved_oswls(num, resource_type)
|
|
# Saving installation structures for proper oswls filtering
|
|
self.get_saved_inst_structs(oswls_saved)
|
|
# Checking oswls filtered properly
|
|
oswls = list(get_oswls(resource_type))
|
|
self.assertEqual(num, len(oswls))
|
|
# Checking export
|
|
result = exporter.export(resource_type, oswls,
|
|
datetime.utcnow().date(), {})
|
|
self.assertIsInstance(result, types.GeneratorType)
|
|
output = six.StringIO(list(result))
|
|
reader = csv.reader(output)
|
|
for _ in reader:
|
|
pass
|
|
|
|
def test_export_on_empty_data(self):
|
|
exporter = OswlStatsToCsv()
|
|
for resource_type in self.RESOURCE_TYPES:
|
|
result = exporter.export(resource_type, [], {}, None)
|
|
self.assertIsInstance(result, types.GeneratorType)
|
|
output = six.StringIO(list(result))
|
|
reader = csv.reader(output)
|
|
for _ in reader:
|
|
pass
|
|
|
|
def test_get_last_sync_datetime(self):
|
|
exporter = OswlStatsToCsv()
|
|
for resource_type in self.RESOURCE_TYPES:
|
|
oswls_saved = self.get_saved_oswls(1, resource_type)
|
|
inst_sturcts = self.get_saved_inst_structs(oswls_saved)
|
|
inst_struct = inst_sturcts[0]
|
|
inst_struct.modification_date = None
|
|
|
|
oswls = get_oswls_query(resource_type).all()
|
|
oswl = oswls[0]
|
|
self.assertEqual(
|
|
inst_struct.creation_date,
|
|
exporter.get_last_sync_datetime(oswl)
|
|
)
|
|
|
|
inst_struct.modification_date = datetime.utcnow()
|
|
oswls = get_oswls_query(resource_type).all()
|
|
oswl = oswls[0]
|
|
self.assertEqual(
|
|
inst_struct.modification_date,
|
|
exporter.get_last_sync_datetime(oswl)
|
|
)
|
|
|
|
def test_stream_horizon_content(self):
|
|
exporter = OswlStatsToCsv()
|
|
created_days = 2
|
|
for resource_type in self.RESOURCE_TYPES:
|
|
# Generating oswl on specified date
|
|
oswls_saved = self.get_saved_oswls(
|
|
1, resource_type, created_date_range=(created_days,
|
|
created_days))
|
|
self.get_saved_inst_structs(
|
|
oswls_saved, creation_date_range=(created_days, created_days))
|
|
|
|
oswls = get_oswls_query(resource_type).all()
|
|
oswl = oswls[0]
|
|
oswl_idx = export_utils.get_index(
|
|
oswl, *exporter.OSWL_INDEX_FIELDS)
|
|
horizon = {
|
|
oswl_idx: oswl
|
|
}
|
|
|
|
# Checking horizon size is changed on date greater than specified
|
|
# date. Checks list format: [(on_date, horizon_size)]
|
|
checks = [
|
|
(datetime.utcnow().date() - timedelta(days=created_days + 1),
|
|
1),
|
|
(datetime.utcnow().date() - timedelta(days=created_days),
|
|
1),
|
|
(datetime.utcnow().date() - timedelta(days=created_days - 1),
|
|
0),
|
|
]
|
|
for on_date, horizon_size in checks:
|
|
for _ in exporter.stream_horizon_content(horizon, on_date):
|
|
pass
|
|
self.assertEqual(horizon_size, len(horizon))
|
|
|
|
def test_fill_date_gaps(self):
|
|
exporter = OswlStatsToCsv()
|
|
created_days = 5
|
|
for resource_type in self.RESOURCE_TYPES:
|
|
# Generating resource time series for one master node
|
|
oswls_saved = self.get_saved_oswls(
|
|
1, resource_type, created_date_range=(created_days,
|
|
created_days))
|
|
inst_sturcts = self.get_saved_inst_structs(
|
|
oswls_saved, creation_date_range=(created_days, created_days))
|
|
inst_struct = inst_sturcts[0]
|
|
|
|
# Checking only one record is present
|
|
inst_struct.modification_date = None
|
|
db.session.add(inst_struct)
|
|
oswls = get_oswls_query(resource_type).all()
|
|
oswl = oswls[0]
|
|
self.assertIsNotNone(oswl.installation_created_date)
|
|
self.assertIsNone(oswl.installation_updated_date)
|
|
|
|
oswls_seamless = exporter.fill_date_gaps(
|
|
oswls, datetime.utcnow().date())
|
|
self.assertEqual(1, len(list(oswls_seamless)))
|
|
|
|
# Checking record is duplicated
|
|
inst_struct.modification_date = datetime.utcnow()
|
|
db.session.add(inst_struct)
|
|
|
|
oswls = get_oswls_query(resource_type).all()
|
|
oswl = oswls[0]
|
|
self.assertIsNotNone(oswl.installation_created_date)
|
|
self.assertIsNotNone(oswl.installation_updated_date)
|
|
|
|
on_date_days = 1
|
|
on_date = (datetime.utcnow() - timedelta(days=on_date_days)).date()
|
|
oswls_seamless = list(exporter.fill_date_gaps(oswls, on_date))
|
|
self.assertEqual(created_days - on_date_days + 1,
|
|
len(oswls_seamless))
|
|
|
|
# Checking dates are seamless and grow
|
|
expected_date = oswls_seamless[0].stats_on_date
|
|
for oswl in oswls_seamless:
|
|
self.assertEqual(expected_date, oswl.stats_on_date)
|
|
expected_date += timedelta(days=1)
|
|
|
|
def test_fill_date_gaps_empty_data_is_not_failed(self):
|
|
exporter = OswlStatsToCsv()
|
|
oswls = exporter.fill_date_gaps([], datetime.utcnow().date())
|
|
self.assertIsInstance(oswls, types.GeneratorType)
|
|
|
|
def test_resource_data_on_oswl_duplication(self):
|
|
exporter = OswlStatsToCsv()
|
|
num = 20
|
|
for resource_type in self.RESOURCE_TYPES:
|
|
oswls_before = get_oswls_query(resource_type).count()
|
|
oswls_saved = self.get_saved_oswls(
|
|
num, resource_type,
|
|
added_num_range=(1, 5), removed_num_range=(1, 3),
|
|
modified_num_range=(1, 15)
|
|
)
|
|
self.get_saved_inst_structs(oswls_saved,
|
|
creation_date_range=(0, 0))
|
|
oswls = get_oswls_query(resource_type).all()
|
|
self.assertEqual(oswls_before + num, len(list(oswls)))
|
|
|
|
# Checking added, modified, removed not empty
|
|
for oswl in oswls:
|
|
resource_data = oswl.resource_data
|
|
self.assertTrue(len(resource_data['added']) > 0)
|
|
self.assertTrue(len(resource_data['modified']) > 0)
|
|
self.assertTrue(len(resource_data['removed']) > 0)
|
|
|
|
# Checking added, modified, removed empty on duplicated oswls
|
|
oswls_seamless = exporter.fill_date_gaps(
|
|
oswls, datetime.utcnow().date())
|
|
for oswl in oswls_seamless:
|
|
if oswl.created_date != oswl.stats_on_date:
|
|
resource_data = oswl.resource_data
|
|
self.assertEqual(0, len(resource_data['added']))
|
|
self.assertEqual(0, len(resource_data['modified']))
|
|
self.assertEqual(0, len(resource_data['removed']))
|
|
|
|
def test_fill_date_gaps_for_set_of_clusters(self):
|
|
exporter = OswlStatsToCsv()
|
|
created_days = 3
|
|
clusters_num = 2
|
|
insts_num = 3
|
|
for resource_type in self.RESOURCE_TYPES[:1]:
|
|
# Generating oswls
|
|
for _ in six.moves.range(insts_num):
|
|
mn_uid = six.text_type(uuid.uuid4())
|
|
oswls_saved = []
|
|
for cluster_id in six.moves.range(clusters_num):
|
|
created_date = datetime.utcnow().date() - \
|
|
timedelta(days=created_days)
|
|
oswl = OpenStackWorkloadStats(
|
|
master_node_uid=mn_uid,
|
|
external_id=cluster_id,
|
|
cluster_id=cluster_id,
|
|
created_date=created_date,
|
|
updated_time=datetime.utcnow().time(),
|
|
resource_type=resource_type,
|
|
resource_checksum='',
|
|
resource_data={}
|
|
)
|
|
db.session.add(oswl)
|
|
oswls_saved.append(oswl)
|
|
self.get_saved_inst_structs(oswls_saved,
|
|
creation_date_range=(0, 0))
|
|
# Checking all resources in seamless oswls
|
|
oswls = get_oswls_query(resource_type).all()
|
|
self.assertEqual(insts_num * clusters_num, len(oswls))
|
|
oswls_seamless = list(exporter.fill_date_gaps(
|
|
oswls, datetime.utcnow().date()))
|
|
self.assertEqual(insts_num * clusters_num * (created_days + 1),
|
|
len(list(oswls_seamless)))
|
|
|
|
# Checking dates do not decrease in seamless oswls
|
|
dates = [oswl.stats_on_date for oswl in oswls_seamless]
|
|
self.assertListEqual(sorted(dates), dates)
|
|
|
|
def test_filter_by_date(self):
|
|
exporter = OswlStatsToCsv()
|
|
num = 10
|
|
with app.test_request_context('/?from_date=2015-02-01'):
|
|
for resource_type in self.RESOURCE_TYPES:
|
|
# Creating oswls
|
|
oswls_saved = self.get_saved_oswls(num, resource_type)
|
|
self.get_saved_inst_structs(oswls_saved)
|
|
# Filtering oswls
|
|
oswls = get_oswls(resource_type)
|
|
result = exporter.export(resource_type, oswls,
|
|
datetime.utcnow().date(), {})
|
|
self.assertIsInstance(result, types.GeneratorType)
|
|
output = six.StringIO(list(result))
|
|
reader = csv.reader(output)
|
|
for _ in reader:
|
|
pass
|
|
|
|
def test_seamless_dates(self):
|
|
exporter = OswlStatsToCsv()
|
|
# Creating oswls with not continuous created dates
|
|
resource_type = consts.OSWL_RESOURCE_TYPES.vm
|
|
old_days = 7
|
|
new_days = 2
|
|
oswls_saved = [
|
|
OpenStackWorkloadStats(
|
|
master_node_uid='x',
|
|
external_id=1,
|
|
cluster_id=1,
|
|
created_date=(datetime.utcnow().date() -
|
|
timedelta(days=old_days)),
|
|
updated_time=datetime.utcnow().time(),
|
|
resource_type=resource_type,
|
|
resource_checksum='',
|
|
resource_data={}
|
|
),
|
|
OpenStackWorkloadStats(
|
|
master_node_uid='x',
|
|
external_id=2,
|
|
cluster_id=2,
|
|
created_date=(datetime.utcnow().date() -
|
|
timedelta(days=new_days)),
|
|
updated_time=datetime.utcnow().time(),
|
|
resource_type=resource_type,
|
|
resource_checksum='',
|
|
resource_data={}
|
|
),
|
|
]
|
|
for oswl in oswls_saved:
|
|
db.session.add(oswl)
|
|
self.get_saved_inst_structs(oswls_saved, creation_date_range=(0, 0))
|
|
|
|
with app.test_request_context():
|
|
oswls = get_oswls(resource_type)
|
|
oswls_seamless = list(exporter.fill_date_gaps(
|
|
oswls, datetime.utcnow().date()))
|
|
|
|
# Checking size of seamless report
|
|
single_record = old_days - new_days
|
|
number_of_records = new_days + 1 # current date should be in report
|
|
expected_num = single_record + number_of_records * len(oswls_saved)
|
|
actual_num = len(oswls_seamless)
|
|
self.assertEqual(expected_num, actual_num)
|
|
|
|
# Checking no gaps in dates
|
|
stats_on_date = oswls_seamless[0].stats_on_date
|
|
for o in oswls_seamless:
|
|
self.assertIn(
|
|
o.stats_on_date - stats_on_date,
|
|
(timedelta(days=0), timedelta(days=1))
|
|
)
|
|
stats_on_date = o.stats_on_date
|
|
|
|
def test_dates_filtering(self):
|
|
updated_at = datetime.utcnow()
|
|
base_date = updated_at.date() - timedelta(days=1)
|
|
exporter = OswlStatsToCsv()
|
|
resource_type = consts.OSWL_RESOURCE_TYPES.vm
|
|
oswl = OpenStackWorkloadStats(
|
|
master_node_uid='x',
|
|
external_id=1,
|
|
cluster_id=1,
|
|
created_date=base_date,
|
|
updated_time=updated_at.time(),
|
|
resource_type=resource_type,
|
|
resource_checksum='',
|
|
resource_data={'current': [{'id': 444, 'status': 'ACTIVE'}]}
|
|
)
|
|
db.session.add(oswl)
|
|
self.get_saved_inst_structs([oswl], creation_date_range=(0, 0))
|
|
|
|
req_params = '/?from_date={0}&to_date={1}'.format(
|
|
(base_date - timedelta(days=2)).isoformat(),
|
|
(base_date - timedelta(days=1)).isoformat()
|
|
)
|
|
with app.test_request_context(req_params):
|
|
oswls = list(get_oswls(resource_type))
|
|
self.assertEqual(0, len(oswls))
|
|
result = exporter.export(
|
|
resource_type,
|
|
oswls,
|
|
(base_date - timedelta(days=1)),
|
|
{}
|
|
)
|
|
# Only column names in result
|
|
self.assertEqual(1, len(list(result)))
|
|
|
|
req_params = '/?to_date={0}'.format(
|
|
(base_date - timedelta(days=1)).isoformat()
|
|
)
|
|
with app.test_request_context(req_params):
|
|
oswls = list(get_oswls(resource_type))
|
|
self.assertEqual(0, len(oswls))
|
|
result = exporter.export(
|
|
resource_type,
|
|
oswls,
|
|
base_date - timedelta(days=1),
|
|
{}
|
|
)
|
|
# Only column names in result
|
|
self.assertEqual(1, len(list(result)))
|
|
|
|
req_params = '/?_date={0}'.format(
|
|
base_date.isoformat()
|
|
)
|
|
with app.test_request_context(req_params):
|
|
oswls = list(get_oswls(resource_type))
|
|
self.assertEqual(1, len(oswls))
|
|
result = exporter.export(
|
|
resource_type,
|
|
oswls,
|
|
base_date + timedelta(days=1),
|
|
{}
|
|
)
|
|
# Not only column names in result
|
|
self.assertEqual(1 + 2, len(list(result)))
|
|
|
|
def test_resource_data_structure(self):
|
|
num = 20
|
|
for resource_type in self.RESOURCE_TYPES:
|
|
oswls = self.get_saved_oswls(num, resource_type)
|
|
for oswl in oswls:
|
|
for res_data in oswl.resource_data['current']:
|
|
# Checking all required for report data is in resource data
|
|
for key in six.iterkeys(OSWL_SKELETONS[resource_type]):
|
|
self.assertIn(key, res_data)
|
|
|
|
def test_oswl_invalid_data(self):
|
|
exporter = OswlStatsToCsv()
|
|
num = 10
|
|
for resource_type in self.RESOURCE_TYPES:
|
|
oswls_saved = self.get_saved_oswls(
|
|
num, resource_type, current_num_range=(1, 1),
|
|
removed_num_range=(0, 0), added_num_range=(0, 0),
|
|
modified_num_range=(0, 0))
|
|
# Saving installation structures for proper oswls filtering
|
|
self.get_saved_inst_structs(oswls_saved)
|
|
|
|
with app.test_request_context():
|
|
oswls = get_oswls(resource_type)
|
|
oswl_keys_paths, vm_keys_paths, csv_keys_paths = \
|
|
exporter.get_resource_keys_paths(resource_type)
|
|
|
|
side_effect = [[]] * num
|
|
side_effect[num / 2] = Exception
|
|
with mock.patch.object(exporter,
|
|
'get_additional_resource_info',
|
|
side_effect=side_effect):
|
|
flatten_resources = exporter.get_flatten_resources(
|
|
resource_type, oswl_keys_paths, vm_keys_paths,
|
|
oswls, {})
|
|
# Checking only invalid data is not exported
|
|
self.assertEqual(num - 1, len(list(flatten_resources)))
|
|
|
|
def test_volume_host_not_in_keys_paths(self):
|
|
exporter = OswlStatsToCsv()
|
|
resource_type = consts.OSWL_RESOURCE_TYPES.volume
|
|
oswl_keys_paths, resource_keys_paths, csv_keys_paths = \
|
|
exporter.get_resource_keys_paths(resource_type)
|
|
self.assertNotIn(['volume', 'host'], csv_keys_paths)
|
|
|
|
def test_is_filtered_oswls_export(self):
|
|
for resource_type in self.RESOURCE_TYPES:
|
|
# Creating filtered OSWLs
|
|
filtered_num = 15
|
|
filtered_oswls = self.get_saved_oswls(
|
|
filtered_num,
|
|
resource_type, current_num_range=(1, 1))
|
|
self.get_saved_inst_structs(filtered_oswls,
|
|
is_filtered_values=(True,))
|
|
# Creating not filtered OSWLs
|
|
not_filtered_num = 10
|
|
not_filtered_oswls = self.get_saved_oswls(
|
|
not_filtered_num,
|
|
resource_type, current_num_range=(1, 1))
|
|
self.get_saved_inst_structs(not_filtered_oswls)
|
|
|
|
# Checking only not filtered resources fetched
|
|
with app.test_request_context():
|
|
oswls = get_oswls_query(resource_type).all()
|
|
self.assertEqual(not_filtered_num, len(oswls))
|
|
for oswl in oswls:
|
|
self.assertIn(oswl.is_filtered, (False, None))
|
|
|
|
def test_release_info_in_oswl(self):
|
|
exporter = OswlStatsToCsv()
|
|
releases = ('6.0', '6.1', None)
|
|
num = 30
|
|
|
|
for resource_type in self.RESOURCE_TYPES:
|
|
# Creating OSWLs
|
|
oswls = self.get_saved_oswls(num, resource_type)
|
|
self.get_saved_inst_structs(oswls, releases=releases)
|
|
|
|
with app.test_request_context():
|
|
oswls = get_oswls_query(resource_type).all()
|
|
oswl_keys_paths, resource_keys_paths, csv_keys_paths = \
|
|
exporter.get_resource_keys_paths(resource_type)
|
|
|
|
# Checking release value in flatten resources
|
|
release_pos = csv_keys_paths.index(
|
|
['version_info', 'fuel_version'])
|
|
flatten_resources = exporter.get_flatten_resources(
|
|
resource_type, oswl_keys_paths, resource_keys_paths,
|
|
oswls, {})
|
|
for flatten_resource in flatten_resources:
|
|
release = flatten_resource[release_pos]
|
|
self.assertIn(release, releases)
|
|
|
|
def test_duplicated_oswls_skipped(self):
|
|
exporter = OswlStatsToCsv()
|
|
# Creating oswls duplicates
|
|
resource_type = consts.OSWL_RESOURCE_TYPES.vm
|
|
old_days = 7
|
|
new_days = 2
|
|
old_created_date = datetime.utcnow().date() - timedelta(days=old_days)
|
|
oswls_saved = [
|
|
OpenStackWorkloadStats(
|
|
master_node_uid='x',
|
|
external_id=1,
|
|
cluster_id=1,
|
|
created_date=old_created_date,
|
|
updated_time=datetime.utcnow().time(),
|
|
resource_type=resource_type,
|
|
resource_checksum='checksum',
|
|
resource_data={'current': [{'id': 1}], 'added': [{'id': 1}]}
|
|
),
|
|
OpenStackWorkloadStats(
|
|
master_node_uid='x',
|
|
external_id=2,
|
|
cluster_id=1,
|
|
created_date=(datetime.utcnow().date() -
|
|
timedelta(days=new_days)),
|
|
updated_time=datetime.utcnow().time(),
|
|
resource_type=resource_type,
|
|
resource_checksum='checksum',
|
|
resource_data={'current': [{'id': 1}], 'added': [{'id': 1}]}
|
|
),
|
|
]
|
|
for oswl in oswls_saved:
|
|
db.session.add(oswl)
|
|
self.get_saved_inst_structs(oswls_saved, creation_date_range=(0, 0))
|
|
|
|
with app.test_request_context():
|
|
oswls = get_oswls(resource_type)
|
|
oswls_seamless = list(exporter.fill_date_gaps(
|
|
oswls, datetime.utcnow().date()))
|
|
|
|
# Checking size of seamless report
|
|
expected_num = old_days + 1 # current date should be in report
|
|
actual_num = len(oswls_seamless)
|
|
self.assertEqual(expected_num, actual_num)
|
|
|
|
# Checking only old oswl in seamless_oswls
|
|
for o in oswls_seamless:
|
|
self.assertEqual(old_created_date, o.created_date)
|
|
|
|
def test_version_info_in_flatten_resource(self):
|
|
exporter = OswlStatsToCsv()
|
|
resource_type = consts.OSWL_RESOURCE_TYPES.vm
|
|
oswls_saved = [
|
|
OpenStackWorkloadStats(
|
|
master_node_uid='x',
|
|
external_id=1,
|
|
cluster_id=1,
|
|
created_date=datetime.utcnow().date(),
|
|
updated_time=datetime.utcnow().time(),
|
|
resource_type=resource_type,
|
|
resource_checksum='no_version_info',
|
|
resource_data={'current': [{'id': 1}]}
|
|
),
|
|
OpenStackWorkloadStats(
|
|
master_node_uid='y',
|
|
external_id=2,
|
|
cluster_id=2,
|
|
created_date=datetime.utcnow().date(),
|
|
updated_time=datetime.utcnow().time(),
|
|
resource_type=resource_type,
|
|
resource_checksum='empty_version_info',
|
|
resource_data={'current': [{'id': 1}]},
|
|
version_info={}
|
|
),
|
|
OpenStackWorkloadStats(
|
|
master_node_uid='z',
|
|
external_id=3,
|
|
cluster_id=3,
|
|
created_date=datetime.utcnow().date(),
|
|
updated_time=datetime.utcnow().time(),
|
|
resource_type=resource_type,
|
|
resource_checksum='with_version_info',
|
|
resource_data={'current': [{'id': 1}]},
|
|
version_info={
|
|
'release_version': 'liberty-9.0',
|
|
'release_os': 'Ubuntu',
|
|
'release_name': 'Liberty on Ubuntu 14.04',
|
|
'fuel_version': '9.0',
|
|
'environment_version': '9.0'
|
|
}
|
|
),
|
|
]
|
|
for oswl in oswls_saved:
|
|
db.session.add(oswl)
|
|
self.get_saved_inst_structs(oswls_saved, creation_date_range=(0, 0))
|
|
|
|
with app.test_request_context():
|
|
oswls = list(get_oswls(resource_type))
|
|
|
|
oswl_keys_paths, resource_keys_paths, csv_keys_paths = \
|
|
exporter.get_resource_keys_paths(resource_type)
|
|
fuel_release_pos = csv_keys_paths.index(
|
|
['version_info', 'fuel_version'])
|
|
flatten_resources = list(exporter.get_flatten_resources(
|
|
resource_type, oswl_keys_paths, resource_keys_paths, oswls, {}))
|
|
|
|
# Checking all oswls are in flatten resources
|
|
external_uid_pos = csv_keys_paths.index(['master_node_uid'])
|
|
expected_uids = set([oswl.master_node_uid for oswl in oswls])
|
|
actual_uids = set([d[external_uid_pos] for d in flatten_resources])
|
|
self.assertEqual(expected_uids, actual_uids)
|
|
|
|
# Checking every flatten_resources contain fuel_release_info
|
|
self.assertTrue(all(d[fuel_release_pos] for d in flatten_resources))
|
|
|
|
def test_all_resource_statuses_are_shown(self):
|
|
exporter = OswlStatsToCsv()
|
|
resource_type = consts.OSWL_RESOURCE_TYPES.vm
|
|
updated_time_str = datetime.utcnow().time().isoformat()
|
|
oswls_saved = [
|
|
OpenStackWorkloadStats(
|
|
master_node_uid='x',
|
|
external_id=1,
|
|
cluster_id=1,
|
|
created_date=(datetime.utcnow().date() -
|
|
timedelta(days=8)),
|
|
updated_time=datetime.utcnow().time(),
|
|
resource_type=resource_type,
|
|
resource_checksum='checksum',
|
|
resource_data={'current': [{'id': 1, 'status': 'enabled',
|
|
'tenant_id': 'first'}],
|
|
'added': [{'id': 1, 'time': updated_time_str}],
|
|
'modified': [], 'removed': []}
|
|
),
|
|
# Removing and adding back the same resource.
|
|
OpenStackWorkloadStats(
|
|
master_node_uid='x',
|
|
external_id=2,
|
|
cluster_id=1,
|
|
created_date=(datetime.utcnow().date() -
|
|
timedelta(days=6)),
|
|
updated_time=datetime.utcnow().time(),
|
|
resource_type=resource_type,
|
|
resource_checksum='checksum',
|
|
resource_data={
|
|
'current': [{'id': 1, 'status': 'enabled',
|
|
'tenant_id': 'second'}],
|
|
'added': [{'id': 1, 'time': updated_time_str}],
|
|
'modified': [],
|
|
'removed': [{'id': 1, 'status': 'enabled',
|
|
'time': updated_time_str,
|
|
'tenant_id': 'second'}]}
|
|
),
|
|
# Changing and restoring back resource
|
|
OpenStackWorkloadStats(
|
|
master_node_uid='x',
|
|
external_id=3,
|
|
cluster_id=1,
|
|
created_date=(datetime.utcnow().date() -
|
|
timedelta(days=4)),
|
|
updated_time=datetime.utcnow().time(),
|
|
resource_type=resource_type,
|
|
resource_checksum='checksum',
|
|
resource_data={
|
|
'current': [{'id': 1, 'enabled': True,
|
|
'tenant_id': 'third'}],
|
|
'added': [],
|
|
'modified': [
|
|
{'id': 1, 'enabled': False, 'time': updated_time_str},
|
|
{'id': 1, 'enabled': True, 'time': updated_time_str},
|
|
],
|
|
'removed': []
|
|
}
|
|
),
|
|
# Resource modified and finally deleted
|
|
OpenStackWorkloadStats(
|
|
master_node_uid='x',
|
|
external_id=4,
|
|
cluster_id=1,
|
|
created_date=(datetime.utcnow().date() -
|
|
timedelta(days=2)),
|
|
updated_time=datetime.utcnow().time(),
|
|
resource_type=resource_type,
|
|
resource_checksum='another_checksum',
|
|
resource_data={
|
|
'current': [],
|
|
'added': [],
|
|
'modified': [
|
|
{'id': 1, 'enabled': False, 'time': updated_time_str},
|
|
{'id': 1, 'enabled': True, 'time': updated_time_str},
|
|
],
|
|
'removed': [{'id': 1, 'enabled': True,
|
|
'tenant_id': 'fourth'}]
|
|
}
|
|
),
|
|
]
|
|
for oswl in oswls_saved:
|
|
db.session.add(oswl)
|
|
self.get_saved_inst_structs(oswls_saved, creation_date_range=(0, 0))
|
|
|
|
with app.test_request_context():
|
|
oswls = get_oswls(resource_type)
|
|
|
|
oswls_seamless = list(exporter.fill_date_gaps(
|
|
oswls, datetime.utcnow().date()))
|
|
|
|
oswl_keys_paths, resource_keys_paths, csv_keys_paths = \
|
|
exporter.get_resource_keys_paths(resource_type)
|
|
|
|
flatten_resources = list(exporter.get_flatten_resources(
|
|
resource_type, oswl_keys_paths, resource_keys_paths,
|
|
oswls_seamless, {}))
|
|
|
|
# Expected oswls num: 2 for 'first', 2 for 'second', 2 for 'third'
|
|
# and only one for finally removed 'fourth'
|
|
expected_oswls_num = 7
|
|
self.assertEqual(expected_oswls_num, len(flatten_resources))
|
|
|
|
is_added_pos = csv_keys_paths.index([resource_type, 'is_added'])
|
|
is_modified_pos = csv_keys_paths.index([resource_type, 'is_modified'])
|
|
is_removed_pos = csv_keys_paths.index([resource_type, 'is_removed'])
|
|
tenant_id_pos = csv_keys_paths.index([resource_type, 'tenant_id'])
|
|
|
|
def check_resource_state(resource, tenant_id, is_added,
|
|
is_modified, is_removed):
|
|
self.assertEqual(is_added, resource[is_added_pos])
|
|
self.assertEqual(is_modified, resource[is_modified_pos])
|
|
self.assertEqual(is_removed, resource[is_removed_pos])
|
|
self.assertEqual(tenant_id, resource[tenant_id_pos])
|
|
|
|
# The fist oswl status True only in is_added
|
|
check_resource_state(flatten_resources[0], 'first',
|
|
True, False, False)
|
|
|
|
# The first oswl status on the next day is False for
|
|
# is_added, is_modified, is_removed
|
|
check_resource_state(flatten_resources[1], 'first',
|
|
False, False, False)
|
|
|
|
# The second oswl status True in is_added, is_modified
|
|
check_resource_state(flatten_resources[2], 'second',
|
|
True, False, True)
|
|
|
|
# The second oswl status on the next day is False for
|
|
# is_added, is_modified, is_removed
|
|
check_resource_state(flatten_resources[3], 'second',
|
|
False, False, False)
|
|
|
|
# The third oswl status True only in is_modified
|
|
check_resource_state(flatten_resources[4], 'third',
|
|
False, True, False)
|
|
|
|
# The third oswl status on the next day is False for
|
|
# is_added, is_modified, is_removed
|
|
check_resource_state(flatten_resources[5], 'third',
|
|
False, False, False)
|
|
|
|
# The fourth oswl status True in is_modified, is_deleted
|
|
check_resource_state(flatten_resources[6], 'fourth',
|
|
False, True, True)
|
|
|
|
def test_fuel_version_from_clusters_data_is_used(self):
|
|
master_node_uid = 'x'
|
|
exporter = OswlStatsToCsv()
|
|
resource_type = consts.OSWL_RESOURCE_TYPES.vm
|
|
version_from_cluster = '7.0'
|
|
release_version_from_cluster = 'from_cluster_7.0'
|
|
version_from_version_info = '9.0'
|
|
release_version_from_version_info = 'from_version_info_9.0'
|
|
|
|
version_from_installation_info = '8.0'
|
|
release_version_from_inst_info = 'from_inst_info_8.0'
|
|
installation_date = datetime.utcnow().date() - timedelta(days=3)
|
|
|
|
# Upgraded Fuel and not upgraded cluster
|
|
structure = InstallationStructure(
|
|
master_node_uid=master_node_uid,
|
|
structure={
|
|
'fuel_release': {
|
|
'release': version_from_installation_info,
|
|
'openstack_version': release_version_from_inst_info
|
|
},
|
|
'clusters_num': 2,
|
|
'clusters': [
|
|
{'id': 1, 'fuel_version': version_from_cluster,
|
|
'release': {'version': release_version_from_cluster}},
|
|
{'id': 2}
|
|
],
|
|
'unallocated_nodes_num_range': 0,
|
|
'allocated_nodes_num': 0
|
|
},
|
|
creation_date=installation_date,
|
|
is_filtered=False
|
|
)
|
|
db.session.add(structure)
|
|
|
|
oswls = [
|
|
OpenStackWorkloadStats(
|
|
master_node_uid=master_node_uid,
|
|
external_id=1,
|
|
cluster_id=1,
|
|
created_date=installation_date,
|
|
updated_time=datetime.utcnow().time(),
|
|
resource_type=resource_type,
|
|
resource_checksum='info_from_cluster',
|
|
resource_data={'current': [{'id': 1, 'status': 'enabled'}],
|
|
'added': [], 'modified': [], 'removed': []},
|
|
version_info=None
|
|
),
|
|
OpenStackWorkloadStats(
|
|
master_node_uid=master_node_uid,
|
|
external_id=3,
|
|
cluster_id=1,
|
|
created_date=installation_date + timedelta(days=1),
|
|
updated_time=datetime.utcnow().time(),
|
|
resource_type=resource_type,
|
|
resource_checksum='info_from_version_info',
|
|
resource_data={'current': [{'id': 1}],
|
|
'added': [], 'modified': [], 'removed': []},
|
|
version_info={
|
|
'fuel_version': version_from_version_info,
|
|
'release_version': release_version_from_version_info
|
|
}
|
|
),
|
|
OpenStackWorkloadStats(
|
|
master_node_uid=master_node_uid,
|
|
external_id=2,
|
|
cluster_id=2,
|
|
created_date=installation_date + timedelta(days=2),
|
|
updated_time=datetime.utcnow().time(),
|
|
resource_type=resource_type,
|
|
resource_checksum='info_from_installation_info',
|
|
resource_data={'current': [{'id': 1}],
|
|
'added': [], 'modified': [], 'removed': []},
|
|
version_info=None
|
|
)
|
|
]
|
|
for oswl in oswls:
|
|
db.session.add(oswl)
|
|
|
|
with app.test_request_context():
|
|
oswls_data = list(get_oswls(resource_type))
|
|
clusters_version_info = get_clusters_version_info()
|
|
|
|
oswl_keys_paths, resource_keys_paths, csv_keys_paths = \
|
|
exporter.get_resource_keys_paths(resource_type)
|
|
fuel_release_pos = csv_keys_paths.index(
|
|
['version_info', 'fuel_version'])
|
|
release_version_pos = csv_keys_paths.index(
|
|
['version_info', 'release_version'])
|
|
flatten_resources = list(exporter.get_flatten_resources(
|
|
resource_type, oswl_keys_paths,
|
|
resource_keys_paths, oswls_data, clusters_version_info
|
|
))
|
|
|
|
self.assertEqual(len(oswls), len(flatten_resources))
|
|
|
|
# Checking version info fetched from cluster
|
|
self.assertEqual(version_from_cluster,
|
|
flatten_resources[0][fuel_release_pos])
|
|
self.assertEqual(release_version_from_cluster,
|
|
flatten_resources[0][release_version_pos])
|
|
|
|
# Checking version info fetched from oswl.version_info
|
|
self.assertEqual(version_from_version_info,
|
|
flatten_resources[1][fuel_release_pos])
|
|
self.assertEqual(release_version_from_version_info,
|
|
flatten_resources[1][release_version_pos])
|
|
|
|
# Checking version info fetched from installation info
|
|
self.assertEqual(version_from_installation_info,
|
|
flatten_resources[2][fuel_release_pos])
|
|
self.assertEqual(release_version_from_inst_info,
|
|
flatten_resources[2][release_version_pos])
|
|
|
|
def test_get_clusters_version_info(self):
|
|
mn_uid = 'x'
|
|
cluster_id = 1
|
|
empty_cluster_id = 2
|
|
mn_uid_no_clusters = 'xx'
|
|
release_name = 'release name'
|
|
version_from_cluster = '7.0'
|
|
release_version_from_cluster = 'from_cluster_7.0'
|
|
installation_date = datetime.utcnow().date() - timedelta(days=3)
|
|
|
|
expected_version_info = {
|
|
'release_version': release_version_from_cluster,
|
|
'release_os': None,
|
|
'release_name': release_name,
|
|
'fuel_version': version_from_cluster
|
|
}
|
|
|
|
structures = [
|
|
InstallationStructure(
|
|
master_node_uid=mn_uid,
|
|
structure={
|
|
'clusters': [
|
|
{'id': cluster_id,
|
|
'fuel_version': version_from_cluster,
|
|
'release': {'version': release_version_from_cluster,
|
|
'name': release_name}},
|
|
{'id': empty_cluster_id}
|
|
]
|
|
|
|
},
|
|
creation_date=installation_date,
|
|
is_filtered=False
|
|
),
|
|
InstallationStructure(
|
|
master_node_uid=mn_uid_no_clusters,
|
|
structure={'clusters': []},
|
|
creation_date=installation_date,
|
|
is_filtered=False
|
|
)
|
|
]
|
|
for structure in structures:
|
|
db.session.add(structure)
|
|
|
|
with app.test_request_context():
|
|
clusters_version_info = get_clusters_version_info()
|
|
|
|
self.assertIn(mn_uid, clusters_version_info)
|
|
self.assertIn(cluster_id, clusters_version_info[mn_uid])
|
|
self.assertNotIn(empty_cluster_id, clusters_version_info[mn_uid])
|
|
self.assertIn(mn_uid_no_clusters, clusters_version_info)
|
|
|
|
actual_version_info = clusters_version_info[mn_uid][cluster_id]
|
|
self.assertEqual(expected_version_info, actual_version_info)
|
|
self.assertEqual({}, clusters_version_info[mn_uid_no_clusters])
|