openstack
/
fuel-stats
Code Issues Proposed changes

Filtered stats excluded from CSV reports 

Stats info with is_filtered == True excluded from InstallationInfo
and OpenStack workloads reports.
OSWLs info for filtered InstallationInfo excluded from CSV reports
Value is_filtered is set according to FILTERING_RULES in collector
config file.
Web UI javascript libraries files added to .gitignore.

Change-Id: I9a4e2e9e953e3424139e8fa31e0b0f10d61a394a
Partial-Bug: #1443347
This commit is contained in:
Alexander Kislitsky 2015-06-26 17:53:30 +03:00
parent 4111bd2485
commit 22ba99c307
6 changed files with 80 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -57,3 +57,6 @@ target/
.venv/
.idea
*.swp
analytics/static/bower_components
analytics/static/node_modules
analytics/static/js/libs

15
analytics/fuel_analytics/api/resources/csv_exporter.py
View File

@ -67,13 +67,18 @@ def get_to_date():
def get_inst_structures_query(from_date=None, to_date=None):
"""Composes query for fetching installation structures
info with filtering by from and to dates and ordering by id
"""Composes query for fetching not filtered installation
structures info with filtering by from and to dates and
ordering by id. Installation structure is not filtered
if is_filtered is False or None.
:param from_date: filter from creation or modification date
:param to_date: filter to creation or modification date
:return: SQLAlchemy query
"""
query = db.session.query(IS)
query = query.filter(or_(
IS.is_filtered == bool(False), # workaround for PEP8 error E712
IS.is_filtered.is_(None)))
if from_date is not None:
query = query.filter(or_(IS.creation_date >= from_date,
IS.modification_date >= from_date))
@ -173,9 +178,11 @@ def get_oswls_query(resource_type, from_date=None, to_date=None):
OSWS.created_date.label('stats_on_date'), # for showing in CSV
OSWS.resource_type, OSWS.resource_data,
IS.creation_date.label('installation_created_date'),
IS.modification_date.label('installation_updated_date')).\
IS.modification_date.label('installation_updated_date'),
IS.is_filtered).\
join(IS, IS.master_node_uid == OSWS.master_node_uid).\
filter(OSWS.resource_type == resource_type)
filter(OSWS.resource_type == resource_type).\
filter(or_(IS.is_filtered == bool(False), IS.is_filtered.is_(None)))
if from_date is not None:
query = query.filter(OSWS.created_date >= from_date)
if to_date is not None:

6
analytics/fuel_analytics/test/api/resources/utils/inst_structure_test.py
View File

@ -181,7 +181,8 @@ class InstStructureTest(BaseTest):
modification_date_range=(1, 10), clusters_num_range=(0, 10),
plugins_num_range=(0, 5), releases=("6.0-techpreview", "6.0-ga"),
release_generators=('_fuel_release_gen',
'_fuel_release_gen_2015_04')):
'_fuel_release_gen_2015_04'),
is_filtered_values=(False, None)):
for _ in xrange(installations_num):
mn_uid = '{}'.format(uuid.uuid4())
structure = self.generate_structure(
@ -197,7 +198,8 @@ class InstStructureTest(BaseTest):
master_node_uid=mn_uid,
structure=structure,
creation_date=creation_date,
modification_date=modification_date
modification_date=modification_date,
is_filtered=random.choice(is_filtered_values)
)
yield obj

4
analytics/fuel_analytics/test/api/resources/utils/oswl_test.py
View File

@ -315,7 +315,8 @@ class OswlTest(BaseTest):
def generate_inst_structs(self, oswls,
creation_date_range=(2, 10),
modification_date_range=(2, 5),
is_modified_date_nullable=True):
is_modified_date_nullable=True,
is_filtered_values=(False, None)):
mn_uids = set()
for oswl in oswls:
@ -333,6 +334,7 @@ class OswlTest(BaseTest):
master_node_uid=oswl.master_node_uid,
creation_date=creation_date,
modification_date=modification_date,
is_filtered=random.choice(is_filtered_values),
structure={},
)
mn_uids.add(oswl.master_node_uid)

36
analytics/fuel_analytics/test/api/resources/utils/test_csv_exporter.py
View File

@ -36,6 +36,7 @@ from fuel_analytics.api.resources.csv_exporter import archive_dir
from fuel_analytics.api.resources.csv_exporter import extract_date
from fuel_analytics.api.resources.csv_exporter import get_action_logs_query
from fuel_analytics.api.resources.csv_exporter import get_from_date
from fuel_analytics.api.resources.csv_exporter import get_inst_structures
from fuel_analytics.api.resources.csv_exporter import get_inst_structures_query
from fuel_analytics.api.resources.csv_exporter import get_oswls_query
from fuel_analytics.api.resources.csv_exporter import get_resources_types
@ -155,6 +156,41 @@ class CsvExporterTest(OswlTest, DbTest):
datetime.utcnow().date() - timedelta(days=100)).count()
self.assertEqual(0, count_after)
def test_get_inst_structures_query_not_returns_filtered(self):
# Fetching inst structures count
count_initial = get_inst_structures_query().count()
# Generating filtered inst structures
oswls = self.get_saved_oswls(10, consts.OSWL_RESOURCE_TYPES.vm,
stats_per_mn_range=(1, 1))
self.get_saved_inst_structs(oswls, is_filtered_values=(True,))
# Checking filtered inst structures don't fetched
count_with_filtered = get_inst_structures_query(None, None).count()
self.assertEquals(count_initial, count_with_filtered)
# Generating not filtered inst structures
oswls = self.get_saved_oswls(20, consts.OSWL_RESOURCE_TYPES.vm,
stats_per_mn_range=(1, 1))
inst_structures = self.get_saved_inst_structs(
oswls, is_filtered_values=(None, False))
not_filtered_num = len(inst_structures)
# Checking not filtered inst structures fetched
count_with_not_filtered = get_inst_structures_query(None, None).count()
get_inst_structures_query(None, None).all()
self.assertEquals(count_initial + not_filtered_num,
count_with_not_filtered)
def test_no_filtered_structures(self):
oswls = self.get_saved_oswls(100, consts.OSWL_RESOURCE_TYPES.vm,
stats_per_mn_range=(1, 1))
self.get_saved_inst_structs(
oswls, is_filtered_values=(True, False, None))
with app.test_request_context():
for inst_structure in get_inst_structures():
self.assertNotEqual(True, inst_structure.is_filtered)
def test_get_resources_types(self):
for resource_type in self.RESOURCE_TYPES:
self.get_saved_oswls(1, resource_type)

23
analytics/fuel_analytics/test/api/resources/utils/test_oswl_stats_to_csv.py
View File

@ -540,3 +540,26 @@ class OswlStatsToCsvTest(OswlTest, DbTest):
oswl_keys_paths, resource_keys_paths, csv_keys_paths = \
exporter.get_resource_keys_paths(resource_type)
self.assertNotIn(['volume', 'host'], csv_keys_paths)
def test_is_filtered_oswls_export(self):
for resource_type in self.RESOURCE_TYPES:
# Creating filtered OSWLs
filtered_num = 15
filtered_oswls = self.get_saved_oswls(
filtered_num,
resource_type, current_num_range=(1, 1))
self.get_saved_inst_structs(filtered_oswls,
is_filtered_values=(True,))
# Creating not filtered OSWLs
not_filtered_num = 10
not_filtered_oswls = self.get_saved_oswls(
not_filtered_num,
resource_type, current_num_range=(1, 1))
self.get_saved_inst_structs(not_filtered_oswls)
# Checking only not filtered resources fetched
with app.test_request_context():
oswls = get_oswls_query(resource_type).all()
self.assertEqual(not_filtered_num, len(oswls))
for oswl in oswls:
self.assertIn(oswl.is_filtered, (False, None))