openstack
/
fuel-stats
Code Issues Proposed changes

Url for fetching filtered installation_infos added 

Json export of filtered installation infos implemented.
Limit, offset, total info added into json exporter output for
group operations.
Usage of DB commit decreased in analytics tests.

Change-Id: Ied35132ce3d194b584c8af40576a61e608862c3e
Closes-Bug: #1458528
This commit is contained in:
Alexander Kislitsky 2015-05-22 18:52:12 +03:00
parent a26cc0e399
commit 4111bd2485
5 changed files with 58 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
analytics/fuel_analytics/api/resources/json_exporter.py
View File

@ -70,6 +70,10 @@ def get_installation_info(master_node_uid):
return Response(json.dumps(dict_result), mimetype='application/json')
def _get_db_objs_count(model, sql_clauses):
return db.session.query(model).filter(and_(*sql_clauses)).count()
def _get_db_objs_data(model, sql_clauses, order_by, paging_params):
"""Gets DB objects by sql_clauses
:param model: DB model
@ -104,6 +108,17 @@ def _jsonify_collection(collection_iter):
yield ']'
def _jsonify_paged_collection(collection_iter, paging_params, total):
page_info = paging_params.copy()
page_info['total'] = total
yield '{{"paging_params": {0}, "objs": '.format(json.dumps(page_info))
for item in _jsonify_collection(collection_iter):
yield item
yield '}'
@bp.route('/oswls/<master_node_uid>', methods=['GET'])
def get_oswls(master_node_uid):
paging_params = get_paging_params()
@ -112,7 +127,9 @@ def get_oswls(master_node_uid):
sql_clauses = (OSWL.master_node_uid == master_node_uid,)
oswls_data = _get_db_objs_data(OSWL, sql_clauses,
(OSWL.id.asc(),), paging_params)
jsons_data = _jsonify_collection(oswls_data)
oswls_count = _get_db_objs_count(OSWL, sql_clauses)
jsons_data = _jsonify_paged_collection(oswls_data, paging_params,
oswls_count)
app.logger.debug("Oswl info for: %s, paging params: %s fetched",
master_node_uid, paging_params)
return Response(jsons_data, mimetype='application/json')
@ -128,7 +145,9 @@ def get_oswls_by_resource_type(master_node_uid, resource_type):
oswls_data = _get_db_objs_data(
OSWL, sql_clauses, (OSWL.id.asc(), OSWL.resource_type.asc()),
paging_params)
jsons_data = _jsonify_collection(oswls_data)
oswls_total = _get_db_objs_count(OSWL, sql_clauses)
jsons_data = _jsonify_paged_collection(oswls_data, paging_params,
oswls_total)
app.logger.debug("Oswl info for: %s, %s, paging prams: %s fetched",
master_node_uid, resource_type, paging_params)
return Response(jsons_data, mimetype='application/json')
@ -142,7 +161,24 @@ def get_action_logs(master_node_uid):
sql_clauses = (AL.master_node_uid == master_node_uid,)
action_logs_data = _get_db_objs_data(AL, sql_clauses,
(AL.id.asc(),), paging_params)
jsons_data = _jsonify_collection(action_logs_data)
action_logs_total = _get_db_objs_count(AL, sql_clauses)
jsons_data = _jsonify_paged_collection(action_logs_data, paging_params,
action_logs_total)
app.logger.debug("Action_logs for: %s, paging params: %s fetched",
master_node_uid, paging_params)
return Response(jsons_data, mimetype='application/json')
@bp.route('/installation_infos/filtered', methods=['GET'])
def get_filtered_installation_infos():
paging_params = get_paging_params()
app.logger.debug("Fetching filtered installation_info, paging params: %s",
paging_params)
sql_clauses = (IS.is_filtered == bool(1),) # Workaround for PEP8 E712
inst_infos = _get_db_objs_data(IS, sql_clauses, (IS.id.asc(),),
paging_params)
inst_infos_total = _get_db_objs_count(IS, sql_clauses)
jsons_data = _jsonify_paged_collection(inst_infos, paging_params,
inst_infos_total)
app.logger.debug("Filtered installation_info: %s fetched", paging_params)
return Response(jsons_data, mimetype='application/json')

1
analytics/fuel_analytics/test/api/resources/utils/inst_structure_test.py
View File

@ -207,7 +207,6 @@ class InstStructureTest(BaseTest):
for obj in objs:
db.session.add(obj)
result.append(obj)
db.session.commit()
return result
def get_saved_inst_structures(self, *args, **kwargs):

18
analytics/fuel_analytics/test/api/resources/utils/test_json_exporter.py
View File

@ -176,3 +176,21 @@ class JsonExporterTest(InstStructureTest, OswlTest, DbTest):
getattr(expected, column_name),
getattr(actual, column_name)
)
def test_get_filtered_installation_infos(self):
total_num = 10
filtered_num = 4
structs = self.get_saved_inst_structures(installations_num=total_num)
for struct in structs[:filtered_num]:
struct.is_filtered = True
db.session.flush()
with app.test_request_context():
url = '/api/v1/json/installation_infos/filtered'
resp = self.client.get(url)
self.check_response_ok(resp)
result = json.loads(resp.data)
self.assertEquals(filtered_num, result['paging_params']['total'])
for struct in result['objs']:
self.assertTrue(struct['is_filtered'])

8
analytics/fuel_analytics/test/api/resources/utils/test_oswl_stats_to_csv.py
View File

@ -94,7 +94,6 @@ class OswlStatsToCsvTest(OswlTest, DbTest):
# Saving data for true JSON loading from DB object
db.session.add(oswl)
db.session.commit()
resource_data = oswl.resource_data
added_ids = set(d['id'] for d in resource_data['added'])
modified_ids = set(d['id'] for d in resource_data['modified'])
@ -154,7 +153,6 @@ class OswlStatsToCsvTest(OswlTest, DbTest):
inst_sturcts = self.get_saved_inst_structs(oswls_saved)
inst_struct = inst_sturcts[0]
inst_struct.modification_date = None
db.session.commit()
oswls = get_oswls_query(resource_type).all()
oswl = oswls[0]
@ -164,7 +162,6 @@ class OswlStatsToCsvTest(OswlTest, DbTest):
)
inst_struct.modification_date = datetime.utcnow()
db.session.commit()
oswls = get_oswls_query(resource_type).all()
oswl = oswls[0]
self.assertEquals(
@ -221,7 +218,6 @@ class OswlStatsToCsvTest(OswlTest, DbTest):
# Checking only one record is present
inst_struct.modification_date = None
db.session.add(inst_struct)
db.session.commit()
oswls = get_oswls_query(resource_type).all()
oswl = oswls[0]
self.assertIsNotNone(oswl.installation_created_date)
@ -234,7 +230,6 @@ class OswlStatsToCsvTest(OswlTest, DbTest):
# Checking record is duplicated
inst_struct.modification_date = datetime.utcnow()
db.session.add(inst_struct)
db.session.commit()
oswls = get_oswls_query(resource_type).all()
oswl = oswls[0]
@ -315,7 +310,6 @@ class OswlStatsToCsvTest(OswlTest, DbTest):
)
db.session.add(oswl)
oswls_saved.append(oswl)
db.session.commit()
self.get_saved_inst_structs(oswls_saved,
creation_date_range=(0, 0))
# Checking all resources in seamless oswls
@ -382,7 +376,6 @@ class OswlStatsToCsvTest(OswlTest, DbTest):
for oswl in oswls_saved:
db.session.add(oswl)
self.get_saved_inst_structs(oswls_saved, creation_date_range=(0, 0))
db.session.commit()
with app.test_request_context():
oswls = get_oswls(resource_type)
@ -422,7 +415,6 @@ class OswlStatsToCsvTest(OswlTest, DbTest):
)
db.session.add(oswl)
self.get_saved_inst_structs([oswl], creation_date_range=(0, 0))
db.session.commit()
with app.test_request_context():
with mock.patch.object(

2
analytics/fuel_analytics/test/base.py
View File

@ -58,7 +58,7 @@ class DbTest(BaseTest):
OpenStackWorkloadStats.query.delete()
InstallationStructure.query.delete()
ActionLog.query.delete()
db.session.commit()
db.session.flush()
def tearDown(self):
# rollback - everything that happened with the