diff --git a/nailgun/nailgun/db/migration/alembic_migrations/versions/fuel_11_0.py b/nailgun/nailgun/db/migration/alembic_migrations/versions/fuel_11_0.py new file mode 100644 index 0000000000..c75bc1558b --- /dev/null +++ b/nailgun/nailgun/db/migration/alembic_migrations/versions/fuel_11_0.py @@ -0,0 +1,189 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Fuel 11.0 + +Revision ID: dc8bc8751c42 +Revises: c6edea552f1e +Create Date: 2016-10-22 02:11:47.708895 + +""" + +from alembic import op +from oslo_serialization import jsonutils + +import six +import sqlalchemy as sa + +from nailgun import consts +from nailgun.db.sqlalchemy.models import fields + + +# revision identifiers, used by Alembic. +revision = 'dc8bc8751c42' +down_revision = 'c6edea552f1e' + +q_select_release_query = sa.sql.text( + "SELECT id, roles_metadata FROM releases " + "WHERE roles_metadata IS NOT NULL" +) +q_update_release_query = sa.sql.text( + "UPDATE releases SET roles_metadata = :roles_metadata WHERE id = :id") +q_select_plugin_query = sa.sql.text( + "SELECT id, roles_metadata FROM plugins " + "WHERE roles_metadata IS NOT NULL" +) +q_update_plugin_query = sa.sql.text( + "UPDATE plugins SET roles_metadata = :roles_metadata WHERE id = :id") + + +def upgrade(): + upgrade_plugins_tags() + + +def downgrade(): + downgrade_plugins_tags() + + +def _create_tags(conn, select_query, update_query, owner_type): + tag_create_query = sa.sql.text( + "INSERT INTO tags (tag, owner_id, owner_type, has_primary, read_only) " + "VALUES(:tag, :owner_id, :owner_type, :has_primary, true) RETURNING id" + ) + for id, roles_metadata in conn.execute(select_query): + roles_metadata = jsonutils.loads(roles_metadata) + for role_name, role_metadata in six.iteritems(roles_metadata): + conn.execute( + tag_create_query, + tag=role_name, + owner_id=id, + owner_type=owner_type, + has_primary=roles_metadata.get('has_primary', False) + ) + conn.execute( + update_query, + id=id, + roles_metadata=jsonutils.dumps(roles_metadata), + ) + + +def _upgrade_tags_assignment(conn, node_query, owner_type): + tag_assign_query = sa.sql.text( + "INSERT INTO node_tags (node_id, tag_id, is_primary) " + "VALUES(:node_id, :tag_id, :is_primary)" + ) + tag_select_query = sa.sql.text( + "SELECT id FROM tags WHERE owner_id=:id AND " + "owner_type=:owner_type AND tag=:tag" + ) + for id, role, primary_roles, owner_id in conn.execute(node_query): + tag = conn.execute( + tag_select_query, + id=owner_id, + owner_type=owner_type, + tag=role + ).fetchone() + + if not tag: + continue + + conn.execute( + tag_assign_query, + node_id=id, + tag_id=tag.id, + is_primary=role in primary_roles + ) + + +def _upgrade_roles_metadata(conn, select_query, update_query): + for id, roles_metadata in conn.execute(select_query): + roles_metadata = jsonutils.loads(roles_metadata) + for role_name, role_metadata in six.iteritems(roles_metadata): + role_metadata['tags'] = [role_name] + conn.execute( + update_query, + id=id, + roles_metadata=jsonutils.dumps(roles_metadata), + ) + + +def _downgrade_roles_metadata(conn, select_query, update_query): + for id, roles_metadata in conn.execute(select_query): + roles_metadata = jsonutils.loads(roles_metadata) + for role_name, role_metadata in six.iteritems(roles_metadata): + del role_metadata['tags'] + conn.execute( + update_query, + id=id, + roles_metadata=jsonutils.dumps(roles_metadata), + ) + + +def upgrade_plugins_tags(): + upgrade_plugins_table() + upgrade_tags_existing_nodes() + + +def upgrade_tags_existing_nodes(): + connection = op.get_bind() + node_plugin_query = sa.sql.text( + "SELECT n.id as n_id, unnest(roles || pending_roles) AS role, " + "primary_roles, p.id AS plugin_id FROM nodes n " + "JOIN clusters c ON n.cluster_id=c.id " + "JOIN cluster_plugins cp ON cp.cluster_id=c.id " + "JOIN plugins p ON cp.plugin_id=p.id" + ) + + # Create tags for all plugins roles + _create_tags( + connection, + q_select_plugin_query, + q_update_plugin_query, + consts.TAG_OWNER_TYPES.plugin + ) + # for releases + _upgrade_roles_metadata(connection, + q_select_release_query, + q_update_release_query) + # for plugins + _upgrade_roles_metadata(connection, + q_select_plugin_query, + q_update_plugin_query) + # update tag's assignment for plugin tags + _upgrade_tags_assignment(connection, + node_plugin_query, + consts.TAG_OWNER_TYPES.plugin) + + +def upgrade_plugins_table(): + op.add_column( + 'plugins', + sa.Column('tags_metadata', + fields.JSON(), + nullable=False, + server_default='{}'), + ) + + +def downgrade_plugins_tags(): + connection = op.get_bind() + # for releases + _downgrade_roles_metadata(connection, + q_select_release_query, + q_update_release_query) + # for plugins + _downgrade_roles_metadata(connection, + q_select_plugin_query, + q_update_plugin_query) + op.drop_column('plugins', 'tags_metadata') diff --git a/nailgun/nailgun/db/sqlalchemy/models/plugins.py b/nailgun/nailgun/db/sqlalchemy/models/plugins.py index b2037a6470..9489a0862f 100644 --- a/nailgun/nailgun/db/sqlalchemy/models/plugins.py +++ b/nailgun/nailgun/db/sqlalchemy/models/plugins.py @@ -180,6 +180,8 @@ class Plugin(Base): MutableDict.as_mutable(JSON), server_default='{}', nullable=False) roles_metadata = Column( MutableDict.as_mutable(JSON), server_default='{}', nullable=False) + tags_metadata = Column( + MutableDict.as_mutable(JSON), server_default='{}', nullable=False) network_roles_metadata = Column( MutableList.as_mutable(JSON), server_default='[]', nullable=False) nic_attributes_metadata = Column( diff --git a/nailgun/nailgun/objects/node.py b/nailgun/nailgun/objects/node.py index f738c3f11f..4d16e5caeb 100644 --- a/nailgun/nailgun/objects/node.py +++ b/nailgun/nailgun/objects/node.py @@ -1075,7 +1075,7 @@ class Node(NailgunObject): @classmethod def update_tags(cls, instance, new_roles): - roles_metadata = instance.cluster.release.roles_metadata + roles_metadata = Cluster.get_roles(instance.cluster) current_tags = set() new_tags = set() diff --git a/nailgun/nailgun/objects/plugin.py b/nailgun/nailgun/objects/plugin.py index 4938607565..dcbbd434c8 100644 --- a/nailgun/nailgun/objects/plugin.py +++ b/nailgun/nailgun/objects/plugin.py @@ -75,10 +75,43 @@ class Plugin(NailgunObject): cls.update(plugin_obj, plugin_adapter.get_metadata()) + cls.create_tags(plugin_obj) + ClusterPlugin.add_compatible_clusters(plugin_obj) return plugin_obj + @classmethod + def create_tags(cls, instance): + from nailgun.objects import Tag + tags = instance.tags_metadata + roles = instance.roles_metadata + # add creation of so-called tags for roles if tags are not + # present in role's metadata. it's necessary for compatibility + # with plugins without tags feature + for role, meta in six.iteritems(roles): + role_tags = meta.get('tags') + if not role_tags: + tags[role] = { + 'tag': role, + 'has_primary': meta.get('has_primary', False), + } + # it's necessary for auto adding tag when we are + # assigning the role + meta['tags'] = [role] + roles.mark_dirty() + + for name, meta in six.iteritems(tags): + data = { + 'owner_id': instance.id, + 'owner_type': consts.TAG_OWNER_TYPES.plugin, + 'tag': name, + 'has_primary': meta.get('has_primary', False), + 'read_only': True + } + Tag.create(data) + db().flush() + @classmethod def update(cls, instance, data): graphs = {} diff --git a/nailgun/nailgun/objects/serializers/node.py b/nailgun/nailgun/objects/serializers/node.py index 5c887d3011..6a7a081ea3 100644 --- a/nailgun/nailgun/objects/serializers/node.py +++ b/nailgun/nailgun/objects/serializers/node.py @@ -50,7 +50,7 @@ class NodeSerializer(BasicSerializer): data_dict = super(NodeSerializer, cls).serialize(instance, fields) data_dict['fqdn'] = Node.get_node_fqdn(instance) data_dict['status'] = Node.get_status(instance) - data_dict['tags'] = instance.tag_names + data_dict['tags'] = list(instance.tag_names) return data_dict diff --git a/nailgun/nailgun/statistics/fuel_statistics/installation_info.py b/nailgun/nailgun/statistics/fuel_statistics/installation_info.py index 9582bf74ac..09f3cd71a5 100644 --- a/nailgun/nailgun/statistics/fuel_statistics/installation_info.py +++ b/nailgun/nailgun/statistics/fuel_statistics/installation_info.py @@ -191,6 +191,7 @@ class InstallationInfo(object): WhiteListRule(('attributes_metadata',), 'attributes_metadata', None), WhiteListRule(('volumes_metadata',), 'volumes_metadata', None), WhiteListRule(('roles_metadata',), 'roles_metadata', None), + WhiteListRule(('tags_metadata',), 'tags_metadata', None), WhiteListRule(('network_roles_metadata',), 'network_roles_metadata', None), WhiteListRule(('components_metadata',), 'components_metadata', None), diff --git a/nailgun/nailgun/test/unit/test_downgrade_fuel_11_0.py b/nailgun/nailgun/test/unit/test_downgrade_fuel_11_0.py new file mode 100644 index 0000000000..3b79eb8efe --- /dev/null +++ b/nailgun/nailgun/test/unit/test_downgrade_fuel_11_0.py @@ -0,0 +1,89 @@ +# coding: utf-8 + +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import alembic +from oslo_serialization import jsonutils + +import six +import sqlalchemy as sa + +from nailgun.db import db +from nailgun.db import dropdb +from nailgun.db.migration import ALEMBIC_CONFIG +from nailgun.test import base + + +_prepare_revision = 'dc8bc8751c42' +_test_revision = 'c6edea552f1e' + + +def setup_module(): + dropdb() + alembic.command.upgrade(ALEMBIC_CONFIG, _prepare_revision) + + prepare() + db.commit() + + alembic.command.downgrade(ALEMBIC_CONFIG, _test_revision) + + +def prepare(): + meta = base.reflect_db_metadata() + + db.execute( + meta.tables['plugins'].insert(), + [{ + 'name': 'test_plugin_a', + 'title': 'Test plugin A', + 'version': '2.0.0', + 'description': 'Test plugin A for Fuel', + 'homepage': 'http://fuel_plugins.test_plugin.com', + 'package_version': '5.0.0', + 'groups': jsonutils.dumps(['tgroup']), + 'authors': jsonutils.dumps(['tauthor']), + 'licenses': jsonutils.dumps(['tlicense']), + 'releases': jsonutils.dumps([ + {'repository_path': 'repositories/ubuntu'} + ]), + 'fuel_version': jsonutils.dumps(['10.0']), + 'roles_metadata': jsonutils.dumps({ + 'role_x': { + 'name': 'role_x', + 'has_primary': False, + 'tags': ['role_x'] + }, + }), + 'tags_metadata': jsonutils.dumps({ + 'role_x': { + 'has_primary': False + }, + }) + }] + ) + + +class TestPluginTags(base.BaseAlembicMigrationTest): + + def test_tag_column_is_absent(self): + plugins = self.meta.tables['plugins'] + self.assertNotIn('tags_metadata', plugins.c) + + def test_tags_are_absent_in_role_meta(self): + plugins = self.meta.tables['plugins'] + q_roles_meta = sa.select([plugins.c.roles_metadata]) + for role_meta in db.execute(q_roles_meta): + for role, meta in six.iteritems(jsonutils.loads(role_meta[0])): + self.assertNotIn('tags', meta) diff --git a/nailgun/nailgun/test/unit/test_migration_fuel_11_0.py b/nailgun/nailgun/test/unit/test_migration_fuel_11_0.py new file mode 100644 index 0000000000..cb32982b36 --- /dev/null +++ b/nailgun/nailgun/test/unit/test_migration_fuel_11_0.py @@ -0,0 +1,190 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import datetime + +import alembic +from oslo_serialization import jsonutils +import six +import sqlalchemy as sa + +from nailgun.db import db +from nailgun.db import dropdb +from nailgun.db.migration import ALEMBIC_CONFIG +from nailgun.test import base + +_prepare_revision = 'c6edea552f1e' +_test_revision = 'dc8bc8751c42' + + +def setup_module(): + dropdb() + alembic.command.upgrade(ALEMBIC_CONFIG, _prepare_revision) + prepare() + alembic.command.upgrade(ALEMBIC_CONFIG, _test_revision) + + +def prepare(): + meta = base.reflect_db_metadata() + + result = db.execute( + meta.tables['releases'].insert(), + [{ + 'name': 'test_name', + 'version': '2016.1-11.0', + 'operating_system': 'ubuntu', + 'state': 'available', + 'roles': jsonutils.dumps([ + 'controller', + ]), + 'roles_metadata': jsonutils.dumps({ + 'controller': { + 'name': 'Controller', + }, + }), + 'is_deployable': True + }]) + + release_id = result.inserted_primary_key[0] + + result = db.execute( + meta.tables['clusters'].insert(), + [{ + 'name': 'test_env1', + 'release_id': release_id, + 'mode': 'ha_compact', + 'status': 'operational', + 'net_provider': 'neutron', + 'grouping': 'roles', + 'fuel_version': '10.0', + }]) + cluster_id = result.inserted_primary_key[0] + + result = db.execute( + meta.tables['plugins'].insert(), + [{ + 'name': 'test_plugin_a', + 'title': 'Test plugin A', + 'version': '2.0.0', + 'description': 'Test plugin A for Fuel', + 'homepage': 'http://fuel_plugins.test_plugin.com', + 'package_version': '5.0.0', + 'groups': jsonutils.dumps(['tgroup']), + 'authors': jsonutils.dumps(['tauthor']), + 'licenses': jsonutils.dumps(['tlicense']), + 'releases': jsonutils.dumps([ + {'repository_path': 'repositories/ubuntu'} + ]), + 'fuel_version': jsonutils.dumps(['10.0']), + 'roles_metadata': jsonutils.dumps({ + 'role_x': { + 'name': 'role_x', + 'has_primary': False + }, + }) + }] + ) + plugin_a_id = result.inserted_primary_key[0] + + result = db.execute( + meta.tables['plugins'].insert(), + [{ + 'name': 'test_plugin_b', + 'title': 'Test plugin B', + 'version': '2.0.0', + 'description': 'Test plugin B for Fuel', + 'homepage': 'http://fuel_plugins.test_plugin.com', + 'package_version': '5.0.0', + 'groups': jsonutils.dumps(['tgroup']), + 'authors': jsonutils.dumps(['tauthor']), + 'licenses': jsonutils.dumps(['tlicense']), + 'releases': jsonutils.dumps([ + {'repository_path': 'repositories/ubuntu'} + ]), + 'fuel_version': jsonutils.dumps(['10.0']), + 'roles_metadata': jsonutils.dumps({ + 'role_y': { + 'name': 'role_y', + 'has_primary': True + }, + }) + }] + ) + plugin_b_id = result.inserted_primary_key[0] + + db.execute( + meta.tables['cluster_plugins'].insert(), + [ + {'cluster_id': cluster_id, 'plugin_id': plugin_a_id}, + {'cluster_id': cluster_id, 'plugin_id': plugin_b_id} + ] + ) + + result = db.execute( + meta.tables['nodes'].insert(), + [{ + 'id': 2, + 'uuid': 'fcd49872-3917-4a18-98f9-3f5acfe3fdec', + 'cluster_id': cluster_id, + 'group_id': None, + 'status': 'ready', + 'roles': ['role_x', 'role_y'], + 'primary_roles': ['role_y'], + 'meta': '{}', + 'mac': 'bb:aa:aa:aa:aa:aa', + 'timestamp': datetime.datetime.utcnow(), + }] + ) + + db.commit() + + +class TestTags(base.BaseAlembicMigrationTest): + def test_plugins_tags_created_on_upgrade(self): + tags_count = db.execute( + sa.select( + [sa.func.count(self.meta.tables['tags'].c.id)] + )).fetchone()[0] + + self.assertEqual(tags_count, 2) + + def test_nodes_assigned_tags(self): + tags = self.meta.tables['tags'] + node_tags = self.meta.tables['node_tags'] + + query = sa.select([tags.c.tag, node_tags.c.is_primary]).select_from( + sa.join( + tags, node_tags, + tags.c.id == node_tags.c.tag_id + ) + ).where( + node_tags.c.node_id == 2 + ) + + res = db.execute(query) + primary_tags = [] + tags = [] + for tag, is_primary in res: + tags.append(tag) + if is_primary: + primary_tags.append(tag) + self.assertItemsEqual(tags, ['role_x', 'role_y']) + self.assertItemsEqual(primary_tags, ['role_y']) + + def test_plugins_role_metadata_changed(self): + plugins = self.meta.tables['plugins'] + q_roles_meta = sa.select([plugins.c.roles_metadata]) + for role_meta in db.execute(q_roles_meta): + for role, meta in six.iteritems(jsonutils.loads(role_meta[0])): + self.assertEqual(meta['tags'], [role]) diff --git a/nailgun/nailgun/test/unit/test_plugin_adapters.py b/nailgun/nailgun/test/unit/test_plugin_adapters.py index 400376e8f3..458f5e52c1 100644 --- a/nailgun/nailgun/test/unit/test_plugin_adapters.py +++ b/nailgun/nailgun/test/unit/test_plugin_adapters.py @@ -79,6 +79,17 @@ class TestPluginBase(base.BaseTestCase): db().flush() + def test_plugins_tags(self): + role = 'role_x' + ClusterPlugin.set_attributes(self.cluster.id, + self.plugin_adapter.plugin.id, + enabled=True) + self.node = self.env.create_node(api=True, + cluster_id=self.cluster.id, + pending_roles=[role], + pending_addition=True) + self.assertItemsEqual(self.node['tags'], [role]) + def test_plugin_release_versions(self): """Should return set of all versions this plugin is applicable to""" self.assertEqual(