openstack
/
fuel-stats
Code Issues Proposed changes

Merge "Filtered stats excluded from CSV reports"

This commit is contained in:
Jenkins 2015-06-30 15:13:48 +00:00 committed by Gerrit Code Review
parent a53f6740cb 22ba99c307
commit 460e20dec7
6 changed files with 80 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -57,3 +57,6 @@ target/
.venv/
.idea
*.swp
analytics/static/bower_components
analytics/static/node_modules
analytics/static/js/libs

15
analytics/fuel_analytics/api/resources/csv_exporter.py
View File

@ -67,13 +67,18 @@ def get_to_date():
def get_inst_structures_query(from_date=None, to_date=None):
"""Composes query for fetching installation structures
info with filtering by from and to dates and ordering by id
"""Composes query for fetching not filtered installation
structures info with filtering by from and to dates and
ordering by id. Installation structure is not filtered
if is_filtered is False or None.
:param from_date: filter from creation or modification date
:param to_date: filter to creation or modification date
:return: SQLAlchemy query
"""
query = db.session.query(IS)
query = query.filter(or_(
IS.is_filtered == bool(False), # workaround for PEP8 error E712
IS.is_filtered.is_(None)))
if from_date is not None:
query = query.filter(or_(IS.creation_date >= from_date,
IS.modification_date >= from_date))
@ -173,9 +178,11 @@ def get_oswls_query(resource_type, from_date=None, to_date=None):
OSWS.created_date.label('stats_on_date'), # for showing in CSV
OSWS.resource_type, OSWS.resource_data,
IS.creation_date.label('installation_created_date'),
IS.modification_date.label('installation_updated_date')).\
IS.modification_date.label('installation_updated_date'),
IS.is_filtered).\
join(IS, IS.master_node_uid == OSWS.master_node_uid).\
filter(OSWS.resource_type == resource_type)
filter(OSWS.resource_type == resource_type).\
filter(or_(IS.is_filtered == bool(False), IS.is_filtered.is_(None)))
if from_date is not None:
query = query.filter(OSWS.created_date >= from_date)
if to_date is not None:

6
analytics/fuel_analytics/test/api/resources/utils/inst_structure_test.py
View File

@ -181,7 +181,8 @@ class InstStructureTest(BaseTest):
modification_date_range=(1, 10), clusters_num_range=(0, 10),
plugins_num_range=(0, 5), releases=("6.0-techpreview", "6.0-ga"),
release_generators=('_fuel_release_gen',
'_fuel_release_gen_2015_04')):
'_fuel_release_gen_2015_04'),
is_filtered_values=(False, None)):
for _ in xrange(installations_num):
mn_uid = '{}'.format(uuid.uuid4())
structure = self.generate_structure(
@ -197,7 +198,8 @@ class InstStructureTest(BaseTest):
master_node_uid=mn_uid,
structure=structure,
creation_date=creation_date,
modification_date=modification_date
modification_date=modification_date,
is_filtered=random.choice(is_filtered_values)
)
yield obj

4
analytics/fuel_analytics/test/api/resources/utils/oswl_test.py
View File

@ -315,7 +315,8 @@ class OswlTest(BaseTest):
def generate_inst_structs(self, oswls,
creation_date_range=(2, 10),
modification_date_range=(2, 5),
is_modified_date_nullable=True):
is_modified_date_nullable=True,
is_filtered_values=(False, None)):
mn_uids = set()
for oswl in oswls:
@ -333,6 +334,7 @@ class OswlTest(BaseTest):
master_node_uid=oswl.master_node_uid,
creation_date=creation_date,
modification_date=modification_date,
is_filtered=random.choice(is_filtered_values),
structure={},
)
mn_uids.add(oswl.master_node_uid)

36
analytics/fuel_analytics/test/api/resources/utils/test_csv_exporter.py
View File

@ -36,6 +36,7 @@ from fuel_analytics.api.resources.csv_exporter import archive_dir
from fuel_analytics.api.resources.csv_exporter import extract_date
from fuel_analytics.api.resources.csv_exporter import get_action_logs_query
from fuel_analytics.api.resources.csv_exporter import get_from_date
from fuel_analytics.api.resources.csv_exporter import get_inst_structures
from fuel_analytics.api.resources.csv_exporter import get_inst_structures_query
from fuel_analytics.api.resources.csv_exporter import get_oswls_query
from fuel_analytics.api.resources.csv_exporter import get_resources_types
@ -155,6 +156,41 @@ class CsvExporterTest(OswlTest, DbTest):
datetime.utcnow().date() - timedelta(days=100)).count()
self.assertEqual(0, count_after)
def test_get_inst_structures_query_not_returns_filtered(self):
# Fetching inst structures count
count_initial = get_inst_structures_query().count()
# Generating filtered inst structures
oswls = self.get_saved_oswls(10, consts.OSWL_RESOURCE_TYPES.vm,
stats_per_mn_range=(1, 1))
self.get_saved_inst_structs(oswls, is_filtered_values=(True,))
# Checking filtered inst structures don't fetched
count_with_filtered = get_inst_structures_query(None, None).count()
self.assertEquals(count_initial, count_with_filtered)
# Generating not filtered inst structures
oswls = self.get_saved_oswls(20, consts.OSWL_RESOURCE_TYPES.vm,
stats_per_mn_range=(1, 1))
inst_structures = self.get_saved_inst_structs(
oswls, is_filtered_values=(None, False))
not_filtered_num = len(inst_structures)
# Checking not filtered inst structures fetched
count_with_not_filtered = get_inst_structures_query(None, None).count()
get_inst_structures_query(None, None).all()
self.assertEquals(count_initial + not_filtered_num,
count_with_not_filtered)
def test_no_filtered_structures(self):
oswls = self.get_saved_oswls(100, consts.OSWL_RESOURCE_TYPES.vm,
stats_per_mn_range=(1, 1))
self.get_saved_inst_structs(
oswls, is_filtered_values=(True, False, None))
with app.test_request_context():
for inst_structure in get_inst_structures():
self.assertNotEqual(True, inst_structure.is_filtered)
def test_get_resources_types(self):
for resource_type in self.RESOURCE_TYPES:
self.get_saved_oswls(1, resource_type)

23
analytics/fuel_analytics/test/api/resources/utils/test_oswl_stats_to_csv.py
View File

@ -540,3 +540,26 @@ class OswlStatsToCsvTest(OswlTest, DbTest):
oswl_keys_paths, resource_keys_paths, csv_keys_paths = \
exporter.get_resource_keys_paths(resource_type)
self.assertNotIn(['volume', 'host'], csv_keys_paths)
def test_is_filtered_oswls_export(self):
for resource_type in self.RESOURCE_TYPES:
# Creating filtered OSWLs
filtered_num = 15
filtered_oswls = self.get_saved_oswls(
filtered_num,
resource_type, current_num_range=(1, 1))
self.get_saved_inst_structs(filtered_oswls,
is_filtered_values=(True,))
# Creating not filtered OSWLs
not_filtered_num = 10
not_filtered_oswls = self.get_saved_oswls(
not_filtered_num,
resource_type, current_num_range=(1, 1))
self.get_saved_inst_structs(not_filtered_oswls)
# Checking only not filtered resources fetched
with app.test_request_context():
oswls = get_oswls_query(resource_type).all()
self.assertEqual(not_filtered_num, len(oswls))
for oswl in oswls:
self.assertIn(oswl.is_filtered, (False, None))