Merge "Filtered stats excluded from CSV reports"
This commit is contained in:
commit
460e20dec7
|
@ -57,3 +57,6 @@ target/
|
|||
.venv/
|
||||
.idea
|
||||
*.swp
|
||||
analytics/static/bower_components
|
||||
analytics/static/node_modules
|
||||
analytics/static/js/libs
|
||||
|
|
|
@ -67,13 +67,18 @@ def get_to_date():
|
|||
|
||||
|
||||
def get_inst_structures_query(from_date=None, to_date=None):
|
||||
"""Composes query for fetching installation structures
|
||||
info with filtering by from and to dates and ordering by id
|
||||
"""Composes query for fetching not filtered installation
|
||||
structures info with filtering by from and to dates and
|
||||
ordering by id. Installation structure is not filtered
|
||||
if is_filtered is False or None.
|
||||
:param from_date: filter from creation or modification date
|
||||
:param to_date: filter to creation or modification date
|
||||
:return: SQLAlchemy query
|
||||
"""
|
||||
query = db.session.query(IS)
|
||||
query = query.filter(or_(
|
||||
IS.is_filtered == bool(False), # workaround for PEP8 error E712
|
||||
IS.is_filtered.is_(None)))
|
||||
if from_date is not None:
|
||||
query = query.filter(or_(IS.creation_date >= from_date,
|
||||
IS.modification_date >= from_date))
|
||||
|
@ -173,9 +178,11 @@ def get_oswls_query(resource_type, from_date=None, to_date=None):
|
|||
OSWS.created_date.label('stats_on_date'), # for showing in CSV
|
||||
OSWS.resource_type, OSWS.resource_data,
|
||||
IS.creation_date.label('installation_created_date'),
|
||||
IS.modification_date.label('installation_updated_date')).\
|
||||
IS.modification_date.label('installation_updated_date'),
|
||||
IS.is_filtered).\
|
||||
join(IS, IS.master_node_uid == OSWS.master_node_uid).\
|
||||
filter(OSWS.resource_type == resource_type)
|
||||
filter(OSWS.resource_type == resource_type).\
|
||||
filter(or_(IS.is_filtered == bool(False), IS.is_filtered.is_(None)))
|
||||
if from_date is not None:
|
||||
query = query.filter(OSWS.created_date >= from_date)
|
||||
if to_date is not None:
|
||||
|
|
|
@ -181,7 +181,8 @@ class InstStructureTest(BaseTest):
|
|||
modification_date_range=(1, 10), clusters_num_range=(0, 10),
|
||||
plugins_num_range=(0, 5), releases=("6.0-techpreview", "6.0-ga"),
|
||||
release_generators=('_fuel_release_gen',
|
||||
'_fuel_release_gen_2015_04')):
|
||||
'_fuel_release_gen_2015_04'),
|
||||
is_filtered_values=(False, None)):
|
||||
for _ in xrange(installations_num):
|
||||
mn_uid = '{}'.format(uuid.uuid4())
|
||||
structure = self.generate_structure(
|
||||
|
@ -197,7 +198,8 @@ class InstStructureTest(BaseTest):
|
|||
master_node_uid=mn_uid,
|
||||
structure=structure,
|
||||
creation_date=creation_date,
|
||||
modification_date=modification_date
|
||||
modification_date=modification_date,
|
||||
is_filtered=random.choice(is_filtered_values)
|
||||
)
|
||||
yield obj
|
||||
|
||||
|
|
|
@ -315,7 +315,8 @@ class OswlTest(BaseTest):
|
|||
def generate_inst_structs(self, oswls,
|
||||
creation_date_range=(2, 10),
|
||||
modification_date_range=(2, 5),
|
||||
is_modified_date_nullable=True):
|
||||
is_modified_date_nullable=True,
|
||||
is_filtered_values=(False, None)):
|
||||
|
||||
mn_uids = set()
|
||||
for oswl in oswls:
|
||||
|
@ -333,6 +334,7 @@ class OswlTest(BaseTest):
|
|||
master_node_uid=oswl.master_node_uid,
|
||||
creation_date=creation_date,
|
||||
modification_date=modification_date,
|
||||
is_filtered=random.choice(is_filtered_values),
|
||||
structure={},
|
||||
)
|
||||
mn_uids.add(oswl.master_node_uid)
|
||||
|
|
|
@ -36,6 +36,7 @@ from fuel_analytics.api.resources.csv_exporter import archive_dir
|
|||
from fuel_analytics.api.resources.csv_exporter import extract_date
|
||||
from fuel_analytics.api.resources.csv_exporter import get_action_logs_query
|
||||
from fuel_analytics.api.resources.csv_exporter import get_from_date
|
||||
from fuel_analytics.api.resources.csv_exporter import get_inst_structures
|
||||
from fuel_analytics.api.resources.csv_exporter import get_inst_structures_query
|
||||
from fuel_analytics.api.resources.csv_exporter import get_oswls_query
|
||||
from fuel_analytics.api.resources.csv_exporter import get_resources_types
|
||||
|
@ -155,6 +156,41 @@ class CsvExporterTest(OswlTest, DbTest):
|
|||
datetime.utcnow().date() - timedelta(days=100)).count()
|
||||
self.assertEqual(0, count_after)
|
||||
|
||||
def test_get_inst_structures_query_not_returns_filtered(self):
|
||||
# Fetching inst structures count
|
||||
count_initial = get_inst_structures_query().count()
|
||||
|
||||
# Generating filtered inst structures
|
||||
oswls = self.get_saved_oswls(10, consts.OSWL_RESOURCE_TYPES.vm,
|
||||
stats_per_mn_range=(1, 1))
|
||||
self.get_saved_inst_structs(oswls, is_filtered_values=(True,))
|
||||
|
||||
# Checking filtered inst structures don't fetched
|
||||
count_with_filtered = get_inst_structures_query(None, None).count()
|
||||
self.assertEquals(count_initial, count_with_filtered)
|
||||
|
||||
# Generating not filtered inst structures
|
||||
oswls = self.get_saved_oswls(20, consts.OSWL_RESOURCE_TYPES.vm,
|
||||
stats_per_mn_range=(1, 1))
|
||||
inst_structures = self.get_saved_inst_structs(
|
||||
oswls, is_filtered_values=(None, False))
|
||||
not_filtered_num = len(inst_structures)
|
||||
|
||||
# Checking not filtered inst structures fetched
|
||||
count_with_not_filtered = get_inst_structures_query(None, None).count()
|
||||
get_inst_structures_query(None, None).all()
|
||||
self.assertEquals(count_initial + not_filtered_num,
|
||||
count_with_not_filtered)
|
||||
|
||||
def test_no_filtered_structures(self):
|
||||
oswls = self.get_saved_oswls(100, consts.OSWL_RESOURCE_TYPES.vm,
|
||||
stats_per_mn_range=(1, 1))
|
||||
self.get_saved_inst_structs(
|
||||
oswls, is_filtered_values=(True, False, None))
|
||||
with app.test_request_context():
|
||||
for inst_structure in get_inst_structures():
|
||||
self.assertNotEqual(True, inst_structure.is_filtered)
|
||||
|
||||
def test_get_resources_types(self):
|
||||
for resource_type in self.RESOURCE_TYPES:
|
||||
self.get_saved_oswls(1, resource_type)
|
||||
|
|
|
@ -540,3 +540,26 @@ class OswlStatsToCsvTest(OswlTest, DbTest):
|
|||
oswl_keys_paths, resource_keys_paths, csv_keys_paths = \
|
||||
exporter.get_resource_keys_paths(resource_type)
|
||||
self.assertNotIn(['volume', 'host'], csv_keys_paths)
|
||||
|
||||
def test_is_filtered_oswls_export(self):
|
||||
for resource_type in self.RESOURCE_TYPES:
|
||||
# Creating filtered OSWLs
|
||||
filtered_num = 15
|
||||
filtered_oswls = self.get_saved_oswls(
|
||||
filtered_num,
|
||||
resource_type, current_num_range=(1, 1))
|
||||
self.get_saved_inst_structs(filtered_oswls,
|
||||
is_filtered_values=(True,))
|
||||
# Creating not filtered OSWLs
|
||||
not_filtered_num = 10
|
||||
not_filtered_oswls = self.get_saved_oswls(
|
||||
not_filtered_num,
|
||||
resource_type, current_num_range=(1, 1))
|
||||
self.get_saved_inst_structs(not_filtered_oswls)
|
||||
|
||||
# Checking only not filtered resources fetched
|
||||
with app.test_request_context():
|
||||
oswls = get_oswls_query(resource_type).all()
|
||||
self.assertEqual(not_filtered_num, len(oswls))
|
||||
for oswl in oswls:
|
||||
self.assertIn(oswl.is_filtered, (False, None))
|
||||
|
|
Loading…
Reference in New Issue