openstack
/
fuel-web
Code Issues Proposed changes

Fix tags creation and migration for plugins 

Change-Id: I3b037cfaab9fa247ac6bb8e75b5113bb984966a1
Implements: blueprint role-decomposition
This commit is contained in:
Viacheslav Valyavskiy 2016-10-21 16:16:54 +03:00
parent ba61ca7670
commit 104fca2393
9 changed files with 517 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

189
nailgun/nailgun/db/migration/alembic_migrations/versions/fuel_11_0.py Normal file
View File

@ -0,0 +1,189 @@
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Fuel 11.0
Revision ID: dc8bc8751c42
Revises: c6edea552f1e
Create Date: 2016-10-22 02:11:47.708895
"""
from alembic import op
from oslo_serialization import jsonutils
import six
import sqlalchemy as sa
from nailgun import consts
from nailgun.db.sqlalchemy.models import fields
# revision identifiers, used by Alembic.
revision = 'dc8bc8751c42'
down_revision = 'c6edea552f1e'
q_select_release_query = sa.sql.text(
"SELECT id, roles_metadata FROM releases "
"WHERE roles_metadata IS NOT NULL"
)
q_update_release_query = sa.sql.text(
"UPDATE releases SET roles_metadata = :roles_metadata WHERE id = :id")
q_select_plugin_query = sa.sql.text(
"SELECT id, roles_metadata FROM plugins "
"WHERE roles_metadata IS NOT NULL"
)
q_update_plugin_query = sa.sql.text(
"UPDATE plugins SET roles_metadata = :roles_metadata WHERE id = :id")
def upgrade():
upgrade_plugins_tags()
def downgrade():
downgrade_plugins_tags()
def _create_tags(conn, select_query, update_query, owner_type):
tag_create_query = sa.sql.text(
"INSERT INTO tags (tag, owner_id, owner_type, has_primary, read_only) "
"VALUES(:tag, :owner_id, :owner_type, :has_primary, true) RETURNING id"
)
for id, roles_metadata in conn.execute(select_query):
roles_metadata = jsonutils.loads(roles_metadata)
for role_name, role_metadata in six.iteritems(roles_metadata):
conn.execute(
tag_create_query,
tag=role_name,
owner_id=id,
owner_type=owner_type,
has_primary=roles_metadata.get('has_primary', False)
)
conn.execute(
update_query,
id=id,
roles_metadata=jsonutils.dumps(roles_metadata),
)
def _upgrade_tags_assignment(conn, node_query, owner_type):
tag_assign_query = sa.sql.text(
"INSERT INTO node_tags (node_id, tag_id, is_primary) "
"VALUES(:node_id, :tag_id, :is_primary)"
)
tag_select_query = sa.sql.text(
"SELECT id FROM tags WHERE owner_id=:id AND "
"owner_type=:owner_type AND tag=:tag"
)
for id, role, primary_roles, owner_id in conn.execute(node_query):
tag = conn.execute(
tag_select_query,
id=owner_id,
owner_type=owner_type,
tag=role
).fetchone()
if not tag:
continue
conn.execute(
tag_assign_query,
node_id=id,
tag_id=tag.id,
is_primary=role in primary_roles
)
def _upgrade_roles_metadata(conn, select_query, update_query):
for id, roles_metadata in conn.execute(select_query):
roles_metadata = jsonutils.loads(roles_metadata)
for role_name, role_metadata in six.iteritems(roles_metadata):
role_metadata['tags'] = [role_name]
conn.execute(
update_query,
id=id,
roles_metadata=jsonutils.dumps(roles_metadata),
)
def _downgrade_roles_metadata(conn, select_query, update_query):
for id, roles_metadata in conn.execute(select_query):
roles_metadata = jsonutils.loads(roles_metadata)
for role_name, role_metadata in six.iteritems(roles_metadata):
del role_metadata['tags']
conn.execute(
update_query,
id=id,
roles_metadata=jsonutils.dumps(roles_metadata),
)
def upgrade_plugins_tags():
upgrade_plugins_table()
upgrade_tags_existing_nodes()
def upgrade_tags_existing_nodes():
connection = op.get_bind()
node_plugin_query = sa.sql.text(
"SELECT n.id as n_id, unnest(roles || pending_roles) AS role, "
"primary_roles, p.id AS plugin_id FROM nodes n "
"JOIN clusters c ON n.cluster_id=c.id "
"JOIN cluster_plugins cp ON cp.cluster_id=c.id "
"JOIN plugins p ON cp.plugin_id=p.id"
)
# Create tags for all plugins roles
_create_tags(
connection,
q_select_plugin_query,
q_update_plugin_query,
consts.TAG_OWNER_TYPES.plugin
)
# for releases
_upgrade_roles_metadata(connection,
q_select_release_query,
q_update_release_query)
# for plugins
_upgrade_roles_metadata(connection,
q_select_plugin_query,
q_update_plugin_query)
# update tag's assignment for plugin tags
_upgrade_tags_assignment(connection,
node_plugin_query,
consts.TAG_OWNER_TYPES.plugin)
def upgrade_plugins_table():
op.add_column(
'plugins',
sa.Column('tags_metadata',
fields.JSON(),
nullable=False,
server_default='{}'),
)
def downgrade_plugins_tags():
connection = op.get_bind()
# for releases
_downgrade_roles_metadata(connection,
q_select_release_query,
q_update_release_query)
# for plugins
_downgrade_roles_metadata(connection,
q_select_plugin_query,
q_update_plugin_query)
op.drop_column('plugins', 'tags_metadata')

2
nailgun/nailgun/db/sqlalchemy/models/plugins.py
View File

@ -180,6 +180,8 @@ class Plugin(Base):
MutableDict.as_mutable(JSON), server_default='{}', nullable=False)
roles_metadata = Column(
MutableDict.as_mutable(JSON), server_default='{}', nullable=False)
tags_metadata = Column(
MutableDict.as_mutable(JSON), server_default='{}', nullable=False)
network_roles_metadata = Column(
MutableList.as_mutable(JSON), server_default='[]', nullable=False)
nic_attributes_metadata = Column(

2
nailgun/nailgun/objects/node.py
View File

@ -1075,7 +1075,7 @@ class Node(NailgunObject):
@classmethod
def update_tags(cls, instance, new_roles):
roles_metadata = instance.cluster.release.roles_metadata
roles_metadata = Cluster.get_roles(instance.cluster)
current_tags = set()
new_tags = set()

33
nailgun/nailgun/objects/plugin.py
View File

@ -75,10 +75,43 @@ class Plugin(NailgunObject):
cls.update(plugin_obj, plugin_adapter.get_metadata())
cls.create_tags(plugin_obj)
ClusterPlugin.add_compatible_clusters(plugin_obj)
return plugin_obj
@classmethod
def create_tags(cls, instance):
from nailgun.objects import Tag
tags = instance.tags_metadata
roles = instance.roles_metadata
# add creation of so-called tags for roles if tags are not
# present in role's metadata. it's necessary for compatibility
# with plugins without tags feature
for role, meta in six.iteritems(roles):
role_tags = meta.get('tags')
if not role_tags:
tags[role] = {
'tag': role,
'has_primary': meta.get('has_primary', False),
}
# it's necessary for auto adding tag when we are
# assigning the role
meta['tags'] = [role]
roles.mark_dirty()
for name, meta in six.iteritems(tags):
data = {
'owner_id': instance.id,
'owner_type': consts.TAG_OWNER_TYPES.plugin,
'tag': name,
'has_primary': meta.get('has_primary', False),
'read_only': True
}
Tag.create(data)
db().flush()
@classmethod
def update(cls, instance, data):
graphs = {}

2
nailgun/nailgun/objects/serializers/node.py
View File

@ -50,7 +50,7 @@ class NodeSerializer(BasicSerializer):
data_dict = super(NodeSerializer, cls).serialize(instance, fields)
data_dict['fqdn'] = Node.get_node_fqdn(instance)
data_dict['status'] = Node.get_status(instance)
data_dict['tags'] = instance.tag_names
data_dict['tags'] = list(instance.tag_names)
return data_dict

1
nailgun/nailgun/statistics/fuel_statistics/installation_info.py
View File

@ -191,6 +191,7 @@ class InstallationInfo(object):
WhiteListRule(('attributes_metadata',), 'attributes_metadata', None),
WhiteListRule(('volumes_metadata',), 'volumes_metadata', None),
WhiteListRule(('roles_metadata',), 'roles_metadata', None),
WhiteListRule(('tags_metadata',), 'tags_metadata', None),
WhiteListRule(('network_roles_metadata',),
'network_roles_metadata', None),
WhiteListRule(('components_metadata',), 'components_metadata', None),

89
nailgun/nailgun/test/unit/test_downgrade_fuel_11_0.py Normal file
View File

@ -0,0 +1,89 @@
# coding: utf-8
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import alembic
from oslo_serialization import jsonutils
import six
import sqlalchemy as sa
from nailgun.db import db
from nailgun.db import dropdb
from nailgun.db.migration import ALEMBIC_CONFIG
from nailgun.test import base
_prepare_revision = 'dc8bc8751c42'
_test_revision = 'c6edea552f1e'
def setup_module():
dropdb()
alembic.command.upgrade(ALEMBIC_CONFIG, _prepare_revision)
prepare()
db.commit()
alembic.command.downgrade(ALEMBIC_CONFIG, _test_revision)
def prepare():
meta = base.reflect_db_metadata()
db.execute(
meta.tables['plugins'].insert(),
[{
'name': 'test_plugin_a',
'title': 'Test plugin A',
'version': '2.0.0',
'description': 'Test plugin A for Fuel',
'homepage': 'http://fuel_plugins.test_plugin.com',
'package_version': '5.0.0',
'groups': jsonutils.dumps(['tgroup']),
'authors': jsonutils.dumps(['tauthor']),
'licenses': jsonutils.dumps(['tlicense']),
'releases': jsonutils.dumps([
{'repository_path': 'repositories/ubuntu'}
]),
'fuel_version': jsonutils.dumps(['10.0']),
'roles_metadata': jsonutils.dumps({
'role_x': {
'name': 'role_x',
'has_primary': False,
'tags': ['role_x']
},
}),
'tags_metadata': jsonutils.dumps({
'role_x': {
'has_primary': False
},
})
}]
)
class TestPluginTags(base.BaseAlembicMigrationTest):
def test_tag_column_is_absent(self):
plugins = self.meta.tables['plugins']
self.assertNotIn('tags_metadata', plugins.c)
def test_tags_are_absent_in_role_meta(self):
plugins = self.meta.tables['plugins']
q_roles_meta = sa.select([plugins.c.roles_metadata])
for role_meta in db.execute(q_roles_meta):
for role, meta in six.iteritems(jsonutils.loads(role_meta[0])):
self.assertNotIn('tags', meta)

190
nailgun/nailgun/test/unit/test_migration_fuel_11_0.py Normal file
View File

@ -0,0 +1,190 @@
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import alembic
from oslo_serialization import jsonutils
import six
import sqlalchemy as sa
from nailgun.db import db
from nailgun.db import dropdb
from nailgun.db.migration import ALEMBIC_CONFIG
from nailgun.test import base
_prepare_revision = 'c6edea552f1e'
_test_revision = 'dc8bc8751c42'
def setup_module():
dropdb()
alembic.command.upgrade(ALEMBIC_CONFIG, _prepare_revision)
prepare()
alembic.command.upgrade(ALEMBIC_CONFIG, _test_revision)
def prepare():
meta = base.reflect_db_metadata()
result = db.execute(
meta.tables['releases'].insert(),
[{
'name': 'test_name',
'version': '2016.1-11.0',
'operating_system': 'ubuntu',
'state': 'available',
'roles': jsonutils.dumps([
'controller',
]),
'roles_metadata': jsonutils.dumps({
'controller': {
'name': 'Controller',
},
}),
'is_deployable': True
}])
release_id = result.inserted_primary_key[0]
result = db.execute(
meta.tables['clusters'].insert(),
[{
'name': 'test_env1',
'release_id': release_id,
'mode': 'ha_compact',
'status': 'operational',
'net_provider': 'neutron',
'grouping': 'roles',
'fuel_version': '10.0',
}])
cluster_id = result.inserted_primary_key[0]
result = db.execute(
meta.tables['plugins'].insert(),
[{
'name': 'test_plugin_a',
'title': 'Test plugin A',
'version': '2.0.0',
'description': 'Test plugin A for Fuel',
'homepage': 'http://fuel_plugins.test_plugin.com',
'package_version': '5.0.0',
'groups': jsonutils.dumps(['tgroup']),
'authors': jsonutils.dumps(['tauthor']),
'licenses': jsonutils.dumps(['tlicense']),
'releases': jsonutils.dumps([
{'repository_path': 'repositories/ubuntu'}
]),
'fuel_version': jsonutils.dumps(['10.0']),
'roles_metadata': jsonutils.dumps({
'role_x': {
'name': 'role_x',
'has_primary': False
},
})
}]
)
plugin_a_id = result.inserted_primary_key[0]
result = db.execute(
meta.tables['plugins'].insert(),
[{
'name': 'test_plugin_b',
'title': 'Test plugin B',
'version': '2.0.0',
'description': 'Test plugin B for Fuel',
'homepage': 'http://fuel_plugins.test_plugin.com',
'package_version': '5.0.0',
'groups': jsonutils.dumps(['tgroup']),
'authors': jsonutils.dumps(['tauthor']),
'licenses': jsonutils.dumps(['tlicense']),
'releases': jsonutils.dumps([
{'repository_path': 'repositories/ubuntu'}
]),
'fuel_version': jsonutils.dumps(['10.0']),
'roles_metadata': jsonutils.dumps({
'role_y': {
'name': 'role_y',
'has_primary': True
},
})
}]
)
plugin_b_id = result.inserted_primary_key[0]
db.execute(
meta.tables['cluster_plugins'].insert(),
[
{'cluster_id': cluster_id, 'plugin_id': plugin_a_id},
{'cluster_id': cluster_id, 'plugin_id': plugin_b_id}
]
)
result = db.execute(
meta.tables['nodes'].insert(),
[{
'id': 2,
'uuid': 'fcd49872-3917-4a18-98f9-3f5acfe3fdec',
'cluster_id': cluster_id,
'group_id': None,
'status': 'ready',
'roles': ['role_x', 'role_y'],
'primary_roles': ['role_y'],
'meta': '{}',
'mac': 'bb:aa:aa:aa:aa:aa',
'timestamp': datetime.datetime.utcnow(),
}]
)
db.commit()
class TestTags(base.BaseAlembicMigrationTest):
def test_plugins_tags_created_on_upgrade(self):
tags_count = db.execute(
sa.select(
[sa.func.count(self.meta.tables['tags'].c.id)]
)).fetchone()[0]
self.assertEqual(tags_count, 2)
def test_nodes_assigned_tags(self):
tags = self.meta.tables['tags']
node_tags = self.meta.tables['node_tags']
query = sa.select([tags.c.tag, node_tags.c.is_primary]).select_from(
sa.join(
tags, node_tags,
tags.c.id == node_tags.c.tag_id
)
).where(
node_tags.c.node_id == 2
)
res = db.execute(query)
primary_tags = []
tags = []
for tag, is_primary in res:
tags.append(tag)
if is_primary:
primary_tags.append(tag)
self.assertItemsEqual(tags, ['role_x', 'role_y'])
self.assertItemsEqual(primary_tags, ['role_y'])
def test_plugins_role_metadata_changed(self):
plugins = self.meta.tables['plugins']
q_roles_meta = sa.select([plugins.c.roles_metadata])
for role_meta in db.execute(q_roles_meta):
for role, meta in six.iteritems(jsonutils.loads(role_meta[0])):
self.assertEqual(meta['tags'], [role])

11
nailgun/nailgun/test/unit/test_plugin_adapters.py
View File

@ -79,6 +79,17 @@ class TestPluginBase(base.BaseTestCase):
db().flush()
def test_plugins_tags(self):
role = 'role_x'
ClusterPlugin.set_attributes(self.cluster.id,
self.plugin_adapter.plugin.id,
enabled=True)
self.node = self.env.create_node(api=True,
cluster_id=self.cluster.id,
pending_roles=[role],
pending_addition=True)
self.assertItemsEqual(self.node['tags'], [role])
def test_plugin_release_versions(self):
"""Should return set of all versions this plugin is applicable to"""
self.assertEqual(