openstack
/
fuel-stats
Code Issues Proposed changes

Version info handled for OSWLs 

Version info data handled for OpenStackWorkloadStats. If master node
updated and the version info in InstallationStructure changed we have
correct version info in OSWLs stats report.

Fixes in collector:

- version_info added to DB model
- version_info added to API protocol
- tests for version_info added

Fixes in analytics:

- version_info added to DB model
- version_info fields added to CSV
- field installation structure release info removed from CSV
- version_info building on the fly for old OSWLs added

Partial-Bug: #1525902
Change-Id: I27d9d65517b4d5a7c6125b889fb0d1ba1ea213cf
This commit is contained in:
Alexander Kislitsky 2015-12-15 15:32:13 +03:00
parent 4c6b98d126
commit c1fe2d34d9
9 changed files with 212 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
analytics/fuel_analytics/api/db/model.py
View File

@ -28,6 +28,7 @@ class OpenStackWorkloadStats(db.Model):
resource_type = db.Column(db.Text)
resource_data = db.Column(JSON)
resource_checksum = db.Column(db.Text)
version_info = db.Column(JSON)
class InstallationStructure(db.Model):

3
analytics/fuel_analytics/api/resources/csv_exporter.py
View File

@ -175,9 +175,10 @@ def get_oswls_query(resource_type, from_date=None, to_date=None):
OSWS.created_date, # for checking if row is duplicated in CSV
OSWS.created_date.label('stats_on_date'), # for showing in CSV
OSWS.resource_type, OSWS.resource_data, OSWS.resource_checksum,
OSWS.version_info,
IS.creation_date.label('installation_created_date'),
IS.modification_date.label('installation_updated_date'),
IS.structure['fuel_release'].label('fuel_release'),
IS.structure['fuel_release'].label('fuel_release_from_inst_info'),
IS.is_filtered).\
join(IS, IS.master_node_uid == OSWS.master_node_uid).\
filter(OSWS.resource_type == resource_type).\

24
analytics/fuel_analytics/api/resources/utils/oswl_stats_to_csv.py
View File

@ -95,6 +95,27 @@ class OswlStatsToCsv(object):
return result
def handle_empty_version_info(self, oswl):
"""Handles empty version info in oswl object
For OSWLs with empty version_info data we compose version_info
from InstallationStructure data and assign it to oswl object.
If we extract OpenStack release, os, name from
InstallationStructure.structure.clusters we have performance
degradation on fetching all clusters data in csv_exporter.get_oswls
thus only fuel_release will be used in case of empty version_info.
:param oswl: OSWL DB object
:type oswl: fuel_analytics.api.db.model.OpenStackWorkloadStats
"""
if oswl.version_info:
return
fuel_release = oswl.fuel_release_from_inst_info or {}
oswl.version_info = {
'fuel_release': fuel_release.get('release')
}
def get_flatten_resources(self, resource_type, oswl_keys_paths,
resource_keys_paths, oswls):
"""Gets flatten vms data
@ -107,8 +128,7 @@ class OswlStatsToCsv(object):
app.logger.debug("Getting OSWL flatten %s info started", resource_type)
for oswl in oswls:
try:
fuel_release = oswl.fuel_release or {}
setattr(oswl, 'release', fuel_release.get('release'))
self.handle_empty_version_info(oswl)
flatten_oswl = export_utils.get_flatten_data(oswl_keys_paths,
oswl)
resource_data = oswl.resource_data

8
analytics/fuel_analytics/api/resources/utils/skeleton.py
View File

@ -152,7 +152,13 @@ OSWL_SKELETONS = {
'cluster_id': None,
'stats_on_date': None,
'resource_type': None,
'release': None,
'version_info': {
'fuel_release': None,
'release_version': None,
'release_os': None,
'release_name': None,
'environment_version': None
}
},
consts.OSWL_RESOURCE_TYPES.vm: {
'id': None,

78
analytics/fuel_analytics/test/api/resources/utils/test_oswl_stats_to_csv.py
View File

@ -45,7 +45,14 @@ class OswlStatsToCsvTest(OswlTest, DbTest):
exporter.get_resource_keys_paths(resource_type)
self.assertNotIn(['external_id'], oswl_keys_paths)
self.assertNotIn(['updated_time'], oswl_keys_paths)
self.assertIn(['release'], oswl_keys_paths)
self.assertNotIn(['release'], oswl_keys_paths)
self.assertIn(['version_info', 'fuel_release'], oswl_keys_paths)
self.assertIn(['version_info', 'release_version'],
oswl_keys_paths)
self.assertIn(['version_info', 'release_name'], oswl_keys_paths)
self.assertIn(['version_info', 'release_os'], oswl_keys_paths)
self.assertIn(['version_info', 'environment_version'],
oswl_keys_paths)
self.assertIn([resource_type, 'id'], resource_keys_paths)
self.assertIn([resource_type, 'is_added'], csv_keys_paths)
self.assertIn([resource_type, 'is_modified'], csv_keys_paths)
@ -579,7 +586,8 @@ class OswlStatsToCsvTest(OswlTest, DbTest):
exporter.get_resource_keys_paths(resource_type)
# Checking release value in flatten resources
release_pos = csv_keys_paths.index(['release'])
release_pos = csv_keys_paths.index(
['version_info', 'fuel_release'])
flatten_resources = exporter.get_flatten_resources(
resource_type, oswl_keys_paths, resource_keys_paths, oswls)
for flatten_resource in flatten_resources:
@ -633,3 +641,69 @@ class OswlStatsToCsvTest(OswlTest, DbTest):
# Checking only old oswl in seamless_oswls
for o in oswls_seamless:
self.assertEqual(old_created_date, o.created_date)
def test_version_info_in_flatten_resource(self):
exporter = OswlStatsToCsv()
resource_type = consts.OSWL_RESOURCE_TYPES.vm
oswls_saved = [
OpenStackWorkloadStats(
master_node_uid='x',
external_id=1,
cluster_id=1,
created_date=datetime.utcnow().date(),
updated_time=datetime.utcnow().time(),
resource_type=resource_type,
resource_checksum='no_version_info',
resource_data={'current': [{'id': 1}]}
),
OpenStackWorkloadStats(
master_node_uid='y',
external_id=2,
cluster_id=2,
created_date=datetime.utcnow().date(),
updated_time=datetime.utcnow().time(),
resource_type=resource_type,
resource_checksum='empty_version_info',
resource_data={'current': [{'id': 1}]},
version_info={}
),
OpenStackWorkloadStats(
master_node_uid='z',
external_id=3,
cluster_id=3,
created_date=datetime.utcnow().date(),
updated_time=datetime.utcnow().time(),
resource_type=resource_type,
resource_checksum='with_version_info',
resource_data={'current': [{'id': 1}]},
version_info={
'fuel_release': 'fr',
'release_version': 'osr',
'release_os': 'osos',
'release_name': 'osn',
'environment_version': '7.0'
}
),
]
for oswl in oswls_saved:
db.session.add(oswl)
self.get_saved_inst_structs(oswls_saved, creation_date_range=(0, 0))
with app.test_request_context():
oswls = list(get_oswls(resource_type))
oswl_keys_paths, resource_keys_paths, csv_keys_paths = \
exporter.get_resource_keys_paths(resource_type)
fuel_release_pos = csv_keys_paths.index(
['version_info', 'fuel_release'])
flatten_resources = list(exporter.get_flatten_resources(
resource_type, oswl_keys_paths, resource_keys_paths, oswls))
# Checking all oswls are in flatten resources
external_uid_pos = csv_keys_paths.index(['master_node_uid'])
expected_uids = set([oswl.master_node_uid for oswl in oswls])
actual_uids = set([d[external_uid_pos] for d in flatten_resources])
self.assertEqual(expected_uids, actual_uids)
# Checking every flatten_resources contain fuel_release_info
self.assertTrue(all(d[fuel_release_pos] for d in flatten_resources))

44
collector/collector/api/db/migrations/versions/4f46e2c07565_version_info_added.py Normal file
View File

@ -0,0 +1,44 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""version_info added
Revision ID: 4f46e2c07565
Revises: 278885b460cd
Create Date: 2015-12-15 11:51:56.237567
"""
# revision identifiers, used by Alembic.
revision = '4f46e2c07565'
down_revision = '278885b460cd'
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.add_column(
'oswl_stats',
sa.Column('version_info', postgresql.JSON(), nullable=True)
)
### end Alembic commands ###
def downgrade():
### commands auto generated by Alembic - please adjust! ###
op.drop_column('oswl_stats', 'version_info')
### end Alembic commands ###

1
collector/collector/api/db/model.py
View File

@ -64,3 +64,4 @@ class OpenStackWorkloadStats(db.Model):
)
resource_data = db.Column(JSON, nullable=True)
resource_checksum = db.Column(db.Text, nullable=False)
version_info = db.Column(JSON, nullable=True)

10
collector/collector/api/schemas/oswl.json
View File

@ -34,6 +34,16 @@
},
"required": ["added", "current", "removed", "modified"],
"additionalProperties": false
},
"version_info": {
"type": "object",
"properties": {
"fuel_version": {"type": ["string", "null"]},
"release_version": {"type": ["string", "null"]},
"release_name": {"type": ["string", "null"]},
"release_os": {"type": ["string", "null"]},
"environment_version": {"type": ["string", "null"]}
}
}
},
"required": ["master_node_uid", "id", "cluster_id",

49
collector/collector/test/resources/test_oswl_stats.py
View File

@ -80,6 +80,26 @@ class TestOswlStats(DbTest):
'modified': []
}
},
{
'master_node_uid': 'x',
'cluster_id': 1,
'id': 3,
'resource_type': consts.OSWL_RESOURCE_TYPES.vm,
'resource_checksum': 'xx',
'created_date': datetime.utcnow().date().isoformat(),
'updated_time': datetime.utcnow().time().isoformat(),
'resource_data': {
'added': [{'id': 1, 'time': 343434343}],
'current': [{'id': 'xxx', 'status': 'down'}],
'removed': [],
'modified': []
},
'version_info': {
'fuel_version': '7.0',
'openstack_version': None,
}
}
]]
for oswls in oswls_sets:
@ -112,6 +132,22 @@ class TestOswlStats(DbTest):
}
for i in xrange(oswls_num)]
def generate_oswls_with_version_info(self, oswls_num):
oswls = self.generate_dumb_oswls(oswls_num)
version_info_variants = [
{}, {'fuel_version': None}, {'fuel_version': "7.0"},
{'release_version': None}, {'release_version': "2015-xx-yy"},
{'release_os': None}, {'release_os': "OSos"},
{'release_name': None}, {'release_name': "OSname"},
{'environment_version': None}, {'environment_version': "OSname"},
{'fuel_version': 'w', 'release_version': 'x',
'release_name': 'y', 'release_os': 'z',
'environment_version': '8.0'}
]
for oswl in oswls:
oswl['version_info'] = random.choice(version_info_variants)
return oswls
def test_existed_oswls_filtering(self):
oswls_num = 10
dicts = self.generate_dumb_oswls(oswls_num)
@ -277,3 +313,16 @@ class TestOswlStats(DbTest):
for oswl_stat in resp_data['oswl_stats']:
self.assertNotEqual(oswl_stat['status'],
consts.OSWL_STATUSES.failed)
def test_post_oswls_with_version_info(self):
oswls_num = 30
expected_oswls = self.generate_oswls_with_version_info(oswls_num)
resp = self.post(
'/api/v1/oswl_stats/',
{'oswl_stats': expected_oswls}
)
self.check_response_ok(resp)
resp_data = json.loads(resp.data)
oswls_actual_num = db.session.query(OSWL).count()
self.assertEqual(oswls_num, oswls_actual_num)
self.assertEqual(len(resp_data['oswl_stats']), oswls_actual_num)