Split reports by release
PoC for splitting fuel-stats reports by releases list. Change-Id: I51fdadea161cb37a631a7e3638746caebb1130a1
This commit is contained in:
parent
ce4e26fc12
commit
ebea961467
@ -39,6 +39,7 @@ class InstallationStructure(db.Model):
|
||||
creation_date = db.Column(db.DateTime)
|
||||
modification_date = db.Column(db.DateTime)
|
||||
is_filtered = db.Column(db.Boolean)
|
||||
release = db.Column(db.Text)
|
||||
|
||||
|
||||
class ActionLog(db.Model):
|
||||
|
@ -66,6 +66,13 @@ def get_to_date():
|
||||
default_value=datetime.utcnow().date())
|
||||
|
||||
|
||||
def get_list_values(param_name, delimiter=','):
|
||||
if param_name not in request.args:
|
||||
return None
|
||||
value = request.args.get(param_name).strip()
|
||||
return [v.strip() for v in value.split(delimiter)]
|
||||
|
||||
|
||||
def get_inst_structures_query(from_date=None, to_date=None, fields=()):
|
||||
"""Composes query for fetching not filtered installation
|
||||
structures info with filtering by from and to dates and
|
||||
@ -207,12 +214,14 @@ def plugins_to_csv():
|
||||
return Response(result, mimetype='text/csv', headers=headers)
|
||||
|
||||
|
||||
def get_oswls_query(resource_type, from_date=None, to_date=None):
|
||||
def get_oswls_query(resource_type, from_date=None, to_date=None,
|
||||
release=None):
|
||||
"""Composes query for fetching oswls with installation
|
||||
info creation and update dates with ordering by created_date
|
||||
:param resource_type: resource type
|
||||
:param from_date: filter from date
|
||||
:param to_date: filter to date
|
||||
:param release: filter by release
|
||||
:return: SQLAlchemy query
|
||||
"""
|
||||
query = db.session.query(
|
||||
@ -232,20 +241,22 @@ def get_oswls_query(resource_type, from_date=None, to_date=None):
|
||||
query = query.filter(OSWS.created_date >= from_date)
|
||||
if to_date is not None:
|
||||
query = query.filter(OSWS.created_date <= to_date)
|
||||
if release is not None:
|
||||
query = query.filter(IS.release == release)
|
||||
# For proper handling of paging we must use additional ordering by id.
|
||||
# In other case we will lose some OSWLs form the execution result.
|
||||
query = query.order_by(OSWS.created_date, OSWS.id)
|
||||
return query
|
||||
|
||||
|
||||
def get_oswls(resource_type):
|
||||
def get_oswls(resource_type, release=None):
|
||||
yield_per = app.config['CSV_DB_YIELD_PER']
|
||||
app.logger.debug("Fetching %s oswls with yield per %d",
|
||||
resource_type, yield_per)
|
||||
from_date = get_from_date()
|
||||
to_date = get_to_date()
|
||||
query = get_oswls_query(resource_type, from_date=from_date,
|
||||
to_date=to_date)
|
||||
to_date=to_date, release=release)
|
||||
return query.yield_per(yield_per)
|
||||
|
||||
|
||||
@ -279,27 +290,53 @@ def _add_oswl_to_clusters_versions_cache(inst_structure, clusters_versions):
|
||||
clusters_versions[mn_uid][cluster['id']] = version_info
|
||||
|
||||
|
||||
def get_oswls_reports(resource_type, releases, from_date, to_date,
|
||||
clusters_version_info):
|
||||
|
||||
exporter = OswlStatsToCsv()
|
||||
|
||||
if releases is None:
|
||||
releases = [None]
|
||||
|
||||
for release in releases:
|
||||
app.logger.debug("Getting report '%s' for release: '%s'",
|
||||
resource_type, release)
|
||||
|
||||
oswls = get_oswls_query(resource_type, from_date=from_date,
|
||||
to_date=to_date, release=release)
|
||||
report = exporter.export(resource_type, oswls, to_date,
|
||||
clusters_version_info)
|
||||
if release is None:
|
||||
file_name = '{0}.csv'.format(resource_type)
|
||||
else:
|
||||
file_name = '{0}_{1}.csv'.format(resource_type, release)
|
||||
|
||||
app.logger.debug("Report '%s' for release '%s' got",
|
||||
resource_type, release)
|
||||
yield report, file_name
|
||||
|
||||
|
||||
@bp.route('/<resource_type>', methods=['GET'])
|
||||
def oswl_to_csv(resource_type):
|
||||
app.logger.debug("Handling oswl_to_csv get request for resource %s",
|
||||
resource_type)
|
||||
|
||||
exporter = OswlStatsToCsv()
|
||||
oswls = get_oswls(resource_type)
|
||||
|
||||
releases = get_list_values('releases')
|
||||
from_date = get_from_date()
|
||||
to_date = get_to_date()
|
||||
clusters_version_info = get_clusters_version_info()
|
||||
result = exporter.export(resource_type, oswls, get_to_date(),
|
||||
clusters_version_info)
|
||||
|
||||
# NOTE: result - is generator, but streaming can not work with some
|
||||
# WSGI middlewares: http://flask.pocoo.org/docs/0.10/patterns/streaming/
|
||||
app.logger.debug("Request oswl_to_csv for resource %s handled",
|
||||
resource_type)
|
||||
reports = get_oswls_reports(resource_type, releases,
|
||||
from_date, to_date, clusters_version_info)
|
||||
file_name = '{}_from{}_to{}'.format(resource_type, from_date, to_date)
|
||||
headers = {
|
||||
'Content-Disposition': 'attachment; filename={}.csv'.format(
|
||||
resource_type)
|
||||
'Content-Disposition': 'attachment; filename={}.tar'.format(file_name)
|
||||
}
|
||||
return Response(result, mimetype='text/csv', headers=headers)
|
||||
app.logger.debug("Get oswl_to_csv request for resource %s handled",
|
||||
resource_type)
|
||||
|
||||
return Response(stream_reports_tar(reports),
|
||||
mimetype='application/x-tar', headers=headers)
|
||||
|
||||
|
||||
def get_resources_types():
|
||||
@ -379,6 +416,10 @@ def stream_reports_tar(reports):
|
||||
tar_stream.seek(io.SEEK_SET)
|
||||
yield tar_stream.getvalue()
|
||||
|
||||
import time
|
||||
time.sleep(5)
|
||||
app.logger.debug("HHHHHRRRRRR!!!")
|
||||
|
||||
tar_stream.seek(io.SEEK_SET)
|
||||
tar_stream.truncate()
|
||||
|
||||
|
@ -28,12 +28,10 @@ from fuel_analytics.api.errors import DateExtractionError
|
||||
from fuel_analytics.api.resources import csv_exporter as ce
|
||||
from fuel_analytics.api.resources.csv_exporter import extract_date
|
||||
from fuel_analytics.api.resources.csv_exporter import get_action_logs
|
||||
from fuel_analytics.api.resources.csv_exporter import get_from_date
|
||||
from fuel_analytics.api.resources.csv_exporter import get_inst_structures
|
||||
from fuel_analytics.api.resources.csv_exporter import get_inst_structures_query
|
||||
from fuel_analytics.api.resources.csv_exporter import get_oswls_query
|
||||
from fuel_analytics.api.resources.csv_exporter import get_resources_types
|
||||
from fuel_analytics.api.resources.csv_exporter import get_to_date
|
||||
from fuel_analytics.api.resources.utils.stats_to_csv import ActionLogInfo
|
||||
from fuel_analytics.api.resources.utils.stats_to_csv import StatsToCsv
|
||||
|
||||
@ -74,24 +72,35 @@ class CsvExporterTest(OswlTest, DbTest):
|
||||
with app.test_request_context():
|
||||
expected = datetime.utcnow().date() - \
|
||||
timedelta(days=app.config['CSV_DEFAULT_FROM_DATE_DAYS'])
|
||||
actual = get_from_date()
|
||||
actual = ce.get_from_date()
|
||||
self.assertEqual(expected, actual)
|
||||
|
||||
with app.test_request_context('/?from_date=2015-02-24'):
|
||||
expected = datetime(2015, 2, 24).date()
|
||||
actual = get_from_date()
|
||||
actual = ce.get_from_date()
|
||||
self.assertEqual(expected, actual)
|
||||
|
||||
def test_to_date(self):
|
||||
def test_get_to_date(self):
|
||||
with app.test_request_context():
|
||||
actual = get_to_date()
|
||||
actual = ce.get_to_date()
|
||||
self.assertEqual(datetime.utcnow().date(), actual)
|
||||
|
||||
with app.test_request_context('/?to_date=2015-02-25'):
|
||||
expected = datetime(2015, 2, 25).date()
|
||||
actual = get_to_date()
|
||||
actual = ce.get_to_date()
|
||||
self.assertEqual(expected, actual)
|
||||
|
||||
def test_get_list_values(self):
|
||||
with app.test_request_context():
|
||||
self.assertIsNone(ce.get_list_values('param'))
|
||||
with app.test_request_context('/?param='):
|
||||
self.assertItemsEqual([''], ce.get_list_values('param'))
|
||||
with app.test_request_context('/?param= '):
|
||||
self.assertItemsEqual([''], ce.get_list_values('param'))
|
||||
with app.test_request_context('/?param=a, b , c'):
|
||||
self.assertItemsEqual(['a', 'b', 'c'],
|
||||
ce.get_list_values('param'))
|
||||
|
||||
def test_get_oswls_query_with_dates(self):
|
||||
num = 20
|
||||
for resource_type in self.RESOURCE_TYPES:
|
||||
|
@ -0,0 +1,61 @@
|
||||
# Copyright 2016 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
"""Release column added to installation_structures
|
||||
|
||||
Revision ID: 24081e26a283
|
||||
Revises: 2ec36f35eeaa
|
||||
Create Date: 2016-06-23 18:53:01.431773
|
||||
|
||||
"""
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '24081e26a283'
|
||||
down_revision = '2ec36f35eeaa'
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
def upgrade():
|
||||
### commands auto generated by Alembic - please adjust! ###
|
||||
op.add_column(
|
||||
'installation_structures',
|
||||
sa.Column('release', sa.Text(), nullable=True)
|
||||
)
|
||||
op.create_index(
|
||||
op.f('ix_installation_structures_release'),
|
||||
'installation_structures',
|
||||
['release'],
|
||||
unique=False
|
||||
)
|
||||
|
||||
set_release = sa.sql.text(
|
||||
"UPDATE installation_structures "
|
||||
"SET release = structure->'fuel_release'->>'release'"
|
||||
)
|
||||
connection = op.get_bind()
|
||||
connection.execute(set_release)
|
||||
### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade():
|
||||
### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index(
|
||||
op.f('ix_installation_structures_release'),
|
||||
table_name='installation_structures'
|
||||
)
|
||||
op.drop_column('installation_structures', 'release')
|
||||
### end Alembic commands ###
|
@ -43,6 +43,7 @@ class InstallationStructure(db.Model):
|
||||
creation_date = db.Column(db.DateTime)
|
||||
modification_date = db.Column(db.DateTime)
|
||||
is_filtered = db.Column(db.Boolean, default=False, index=True)
|
||||
release = db.Column(db.Text, index=True)
|
||||
|
||||
|
||||
class OpenStackWorkloadStats(db.Model):
|
||||
|
@ -52,6 +52,7 @@ def post():
|
||||
obj.modification_date = datetime.utcnow()
|
||||
status_code = 200
|
||||
obj.is_filtered = _is_filtered(structure)
|
||||
obj.release = get_release(structure)
|
||||
obj.structure = structure
|
||||
db.session.add(obj)
|
||||
return status_code, {'status': 'ok'}
|
||||
@ -133,3 +134,7 @@ def _is_filtered(structure):
|
||||
packages, filtered_by_packages)
|
||||
|
||||
return filtered_by_build_id or filtered_by_packages
|
||||
|
||||
|
||||
def get_release(structure):
|
||||
return structure.get('fuel_release', {}).get('release')
|
||||
|
@ -507,3 +507,28 @@ class TestInstallationStructure(DbTest):
|
||||
filtering_rules = {tuple(sorted(packages)): from_dt_str}
|
||||
self.assertFalse(_is_filtered_by_build_info(
|
||||
packages, filtering_rules))
|
||||
|
||||
def test_release_column(self):
|
||||
master_node_uid = 'x'
|
||||
release = 'release'
|
||||
struct = {
|
||||
'master_node_uid': master_node_uid,
|
||||
'fuel_release': {
|
||||
'release': release,
|
||||
'feature_groups': [],
|
||||
'api': 'v1'
|
||||
},
|
||||
'allocated_nodes_num': 0,
|
||||
'unallocated_nodes_num': 0,
|
||||
'clusters_num': 0,
|
||||
'clusters': []
|
||||
}
|
||||
resp = self.post(
|
||||
'/api/v1/installation_structure/',
|
||||
{'installation_structure': struct}
|
||||
)
|
||||
self.check_response_ok(resp, codes=(201,))
|
||||
obj = db.session.query(InstallationStructure).filter(
|
||||
InstallationStructure.master_node_uid == master_node_uid).one()
|
||||
self.assertEqual(struct, obj.structure)
|
||||
self.assertEqual(release, obj.release)
|
||||
|
Loading…
Reference in New Issue
Block a user