openstack
/
fuel-web
Code Issues Proposed changes

Introduce new history API 

API url: `/api/transactions/<id>/deployment_history`

This change proposes to store all the information about
particular deployment tasks ever executed for each
particular cluster.

Change-Id: I73010a713ab8592418eb59bb133a427ac4c4a665
Implements: blueprint store-deployment-tasks-history
This commit is contained in:
Vladimir Sharshov (warpc) 2016-03-23 23:47:12 +03:00
parent 48d2e1412a
commit 48a42a860b
12 changed files with 377 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
nailgun/nailgun/api/v1/handlers/deployment_history.py Normal file
View File

@ -0,0 +1,48 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import web
from nailgun.api.v1.handlers import base
from nailgun.api.v1.handlers.base import content
from nailgun import objects
class DeploymentHistoryCollectionHandler(base.CollectionHandler):
collection = objects.DeploymentHistoryCollection
@content
def GET(self, transaction_id):
""":returns: Collection of JSONized DeploymentHistory objects.
:http: * 200 (OK)
* 404 (cluster not found in db)
"""
self.get_object_or_404(objects.Transaction, transaction_id)
node_ids = web.input(nodes=None).nodes
statuses = web.input(statuses=None).statuses
if node_ids:
node_ids = set(node_ids.strip().split(','))
if statuses:
statuses = set(statuses.strip().split(','))
return self.collection.to_json(
self.collection.get_history(
transaction_id,
node_ids,
statuses)
)

5
nailgun/nailgun/api/v1/urls.py
View File

@ -44,6 +44,9 @@ from nailgun.api.v1.handlers.cluster_plugin_link \
from nailgun.api.v1.handlers.cluster_plugin_link \
import ClusterPluginLinkHandler
from nailgun.api.v1.handlers.deployment_history \
import DeploymentHistoryCollectionHandler
from nailgun.api.v1.handlers.vip import ClusterVIPCollectionHandler
from nailgun.api.v1.handlers.vip import ClusterVIPHandler
@ -282,6 +285,8 @@ urls = (
TransactionCollectionHandler,
r'/transactions/(?P<obj_id>\d+)/?$',
TransactionHandler,
r'/transactions/(?P<transaction_id>\d+)/deployment_history/?$',
DeploymentHistoryCollectionHandler,
r'/plugins/(?P<plugin_id>\d+)/links/?$',
PluginLinkCollectionHandler,

8
nailgun/nailgun/consts.py
View File

@ -249,6 +249,14 @@ TASK_STATUSES = Enum(
'error'
)
HISTORY_TASK_STATUSES = Enum(
'pending',
'ready',
'running',
'error',
'skipped'
)
TASK_NAMES = Enum(
'super',

56
nailgun/nailgun/db/migration/alembic_migrations/versions/fuel_9_0.py
View File

@ -126,6 +126,14 @@ bond_modes_new = (
'balance-alb',
)
history_task_statuses = (
'pending',
'ready',
'running',
'error',
'skipped',
)
def upgrade():
add_foreign_key_ondelete()
@ -142,9 +150,11 @@ def upgrade():
upgrade_node_stop_deployment_error_type()
upgrade_bond_modes()
upgrade_task_attributes()
upgrade_store_deployment_history()
def downgrade():
downgrade_store_deployment_history()
downgrade_task_attributes()
downgrade_bond_modes()
downgrade_node_stop_deployment_error_type()
@ -1297,3 +1307,49 @@ def downgrade_task_attributes():
op.drop_column('tasks', 'cluster_settings')
op.drop_column('tasks', 'deployment_info')
op.drop_column('tasks', 'deleted_at')
def upgrade_store_deployment_history():
op.create_table(
'deployment_history',
sa.Column(
'id',
sa.Integer(),
nullable=False,
autoincrement=True,
primary_key=True,
),
sa.Column('task_id', sa.Integer(), nullable=False),
sa.Column('node_id', sa.String()),
sa.Column('deployment_graph_task_name', sa.String(), nullable=False),
sa.Column('time_start', sa.DateTime(), nullable=True),
sa.Column('time_end', sa.DateTime(), nullable=True),
sa.Column(
'status',
sa.Enum(
*history_task_statuses,
name='history_task_statuses'),
nullable=False),
sa.Column(
'custom',
fields.JSON(),
default={},
server_default='{}',
nullable=False),
sa.ForeignKeyConstraint(['task_id'], ['tasks.id'], ),
sa.UniqueConstraint(
'task_id',
'node_id',
'deployment_graph_task_name',
name='_task_id_node_id_deployment_graph_task_name_uc'),
)
op.create_index('deployment_history_task_id_and_status',
'deployment_history', ['task_id', 'status'])
def downgrade_store_deployment_history():
op.drop_table('deployment_history')
drop_enum('history_task_statuses')

1
nailgun/nailgun/db/sqlalchemy/models/__init__.py
View File

@ -57,6 +57,7 @@ from nailgun.db.sqlalchemy.models.cluster_plugin_link import ClusterPluginLink
from nailgun.db.sqlalchemy.models.plugin_link import PluginLink
from nailgun.db.sqlalchemy.models.task import Task
from nailgun.db.sqlalchemy.models.deployment_history import DeploymentHistory
from nailgun.db.sqlalchemy.models.master_node_settings \
import MasterNodeSettings

56
nailgun/nailgun/db/sqlalchemy/models/deployment_history.py Normal file
View File

@ -0,0 +1,56 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sqlalchemy as sa
from sqlalchemy.ext.mutable import MutableDict
from nailgun import consts
from nailgun.db.sqlalchemy.models.base import Base
from nailgun.db.sqlalchemy.models.fields import JSON
class DeploymentHistory(Base):
__tablename__ = 'deployment_history'
__table_args__ = (
sa.Index('deployment_history_task_id_and_status',
'task_id', 'status'),
sa.UniqueConstraint(
'task_id',
'node_id',
'deployment_graph_task_name',
name='_task_id_node_id_deployment_graph_task_name_uc'),
)
id = sa.Column(sa.Integer, primary_key=True)
task_id = sa.Column(
sa.Integer,
sa.ForeignKey('tasks.id', ondelete='CASCADE'),
nullable=False)
deployment_graph_task_name = sa.Column(sa.String, nullable=False)
# String, because history need to be saved tasks for master and None nodes
node_id = sa.Column(sa.String)
time_start = sa.Column(sa.DateTime, nullable=True)
time_end = sa.Column(sa.DateTime, nullable=True)
status = sa.Column(
sa.Enum(*consts.HISTORY_TASK_STATUSES,
name='history_task_statuses'),
nullable=False,
default=consts.HISTORY_TASK_STATUSES.pending)
custom = sa.Column(MutableDict.as_mutable(JSON), default={},
server_default='{}', nullable=False)

3
nailgun/nailgun/db/sqlalchemy/models/task.py
View File

@ -80,6 +80,9 @@ class Task(Base):
cluster_settings = Column(MutableDict.as_mutable(JSON), nullable=True)
network_settings = Column(MutableDict.as_mutable(JSON), nullable=True)
deployment_history = relationship(
"DeploymentHistory", backref="task", cascade="all,delete")
def __repr__(self):
return "<Task '{0}' {1} ({2}) {3}>".format(
self.name,

3
nailgun/nailgun/objects/__init__.py
View File

@ -40,6 +40,9 @@ from nailgun.objects.task import TaskCollection
from nailgun.objects.transaction import Transaction
from nailgun.objects.transaction import TransactionCollection
from nailgun.objects.deployment_history import DeploymentHistory
from nailgun.objects.deployment_history import DeploymentHistoryCollection
from nailgun.objects.notification import Notification
from nailgun.objects.notification import NotificationCollection

142
nailgun/nailgun/objects/deployment_history.py Normal file
View File

@ -0,0 +1,142 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from datetime import datetime
from nailgun.consts import HISTORY_TASK_STATUSES
from nailgun.db import db
from nailgun.db.sqlalchemy import models
from nailgun.objects import NailgunCollection
from nailgun.objects import NailgunObject
from nailgun.objects.serializers.deployment_history \
import DeploymentHistorySerializer
from nailgun.logger import logger
class DeploymentHistory(NailgunObject):
model = models.DeploymentHistory
serializer = DeploymentHistorySerializer
@classmethod
def update_if_exist(cls, task_id, node_id, deployment_graph_task_name,
status, custom):
deployment_history = cls.find_history(task_id, node_id,
deployment_graph_task_name)
if not deployment_history:
return
getattr(cls, 'to_{0}'.format(status))(deployment_history)
@classmethod
def find_history(cls, task_id, node_id, deployment_graph_task_name):
return db().query(cls.model)\
.filter_by(task_id=task_id,
node_id=node_id,
deployment_graph_task_name=deployment_graph_task_name)\
.first()
@classmethod
def to_ready(cls, deployment_history):
if deployment_history.status == HISTORY_TASK_STATUSES.running:
deployment_history.status = HISTORY_TASK_STATUSES.ready
cls._set_time_end(deployment_history)
else:
cls._logging_wrong_status_change(deployment_history.status,
HISTORY_TASK_STATUSES.ready)
@classmethod
def to_error(cls, deployment_history):
if deployment_history.status == HISTORY_TASK_STATUSES.running:
deployment_history.status = HISTORY_TASK_STATUSES.error
cls._set_time_end(deployment_history)
else:
cls._logging_wrong_status_change(deployment_history.status,
HISTORY_TASK_STATUSES.error)
@classmethod
def to_skipped(cls, deployment_history):
if deployment_history.status == HISTORY_TASK_STATUSES.pending:
deployment_history.status = HISTORY_TASK_STATUSES.skipped
cls._set_time_start(deployment_history)
cls._set_time_end(deployment_history)
else:
cls._logging_wrong_status_change(deployment_history.status,
HISTORY_TASK_STATUSES.skipped)
@classmethod
def to_running(cls, deployment_history):
if deployment_history.status == HISTORY_TASK_STATUSES.pending:
deployment_history.status = HISTORY_TASK_STATUSES.running
cls._set_time_start(deployment_history)
else:
cls._logging_wrong_status_change(deployment_history.status,
HISTORY_TASK_STATUSES.running)
@classmethod
def to_pending(cls, deployment_history):
cls._logging_wrong_status_change(deployment_history.status,
HISTORY_TASK_STATUSES.pending)
@classmethod
def _set_time_end(cls, deployment_history):
if not deployment_history.time_end:
deployment_history.time_end = datetime.utcnow()
@classmethod
def _set_time_start(cls, deployment_history):
if not deployment_history.time_start:
deployment_history.time_start = datetime.utcnow()
@classmethod
def _logging_wrong_status_change(cls, from_status, to_status):
logger.warn("Error status transition from %s to %s",
from_status, to_status)
class DeploymentHistoryCollection(NailgunCollection):
single = DeploymentHistory
@classmethod
def create(cls, task, tasks_graph):
entries = []
for node_id in tasks_graph:
for graph_task in tasks_graph[node_id]:
if not graph_task.get('id'):
logger.warn("Task name missing. Ignoring %s", graph_task)
continue
entries.append(cls.single.model(
task_id=task.id,
node_id=node_id,
deployment_graph_task_name=graph_task['id']))
db().bulk_save_objects(entries)
@classmethod
def get_history(cls, transaction_id, node_ids=None, statuses=None):
query = cls.filter_by(None, task_id=transaction_id)
if node_ids:
query = query.filter(cls.single.model.node_id.in_(node_ids))
if statuses:
query = query.filter(cls.single.model.status.in_(statuses))
return query

40
nailgun/nailgun/objects/serializers/deployment_history.py Normal file
View File

@ -0,0 +1,40 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from nailgun.objects.serializers.base import BasicSerializer
class DeploymentHistorySerializer(BasicSerializer):
fields = (
"deployment_graph_task_name",
"node_id",
"time_start",
"time_end",
"status",
"custom"
)
@classmethod
def serialize(cls, instance, fields=None):
data_dict = super(DeploymentHistorySerializer, cls).serialize(
instance,
fields)
if instance.time_start:
data_dict['time_start'] = instance.time_start.isoformat()
if instance.time_end:
data_dict['time_end'] = instance.time_end.isoformat()
return data_dict

12
nailgun/nailgun/rpc/receiver.py
View File

@ -262,6 +262,7 @@ class NailgunReceiver(object):
# message with descriptive error
nodes_by_id = {str(n['uid']): n for n in nodes}
master = nodes_by_id.pop(consts.MASTER_NODE_UID, {})
nodes_by_id.pop('None', {})
if nodes_by_id:
# lock nodes for updating so they can't be deleted
@ -316,6 +317,17 @@ class NailgunReceiver(object):
logger.warning("The following nodes is not found: %s",
",".join(sorted(nodes_by_id)))
for node in nodes:
if node.get('deployment_graph_task_name') \
and node.get('task_status'):
objects.DeploymentHistory.update_if_exist(
task.id,
node['uid'],
node['deployment_graph_task_name'],
node['task_status'],
node.get('custom')
)
if nodes and not progress:
progress = TaskHelper.recalculate_deployment_task_progress(task)

3
nailgun/nailgun/task/task.py
View File

@ -302,6 +302,9 @@ class DeploymentTask(BaseDeploymentTask):
task_ids, tasks_events
)
logger.debug("finish tasks serialization.")
objects.DeploymentHistoryCollection.create(task, graph)
return {
"deployment_info": serialized_cluster,
"tasks_directory": directory,