From a80bbf8b6e549f968ed6e5ed9b7ca1b1dd5a6822 Mon Sep 17 00:00:00 2001 From: Ilya Kharin Date: Tue, 28 Jul 2015 05:03:32 +0300 Subject: [PATCH 01/43] Introduce extension to upgrade clusters The patch adds an extension which implements the procedure of upgrading clusters from one major release to another. As a first step of the procedure the extension provides an ability to create a seed cluster with the same settings as the original. Implements blueprint: nailgun-api-env-upgrade-extensions Change-Id: I22d51a3ffd51a7c88bdcbde0eef6f47b65def1c8 --- cluster_upgrade/__init__.py | 0 .../alembic_migrations/__init__.py | 0 .../alembic_migrations/alembic.ini | 54 ++++++++ .../alembic_migrations/migrations/__init__.py | 0 .../alembic_migrations/migrations/env.py | 90 +++++++++++++ .../migrations/script.py.mako | 24 ++++ .../001_add_upgrade_relations_table.py | 51 +++++++ cluster_upgrade/extension.py | 42 ++++++ cluster_upgrade/handlers.py | 51 +++++++ cluster_upgrade/models.py | 31 +++++ cluster_upgrade/objects/__init__.py | 0 cluster_upgrade/objects/adapters.py | 108 +++++++++++++++ cluster_upgrade/objects/relations.py | 48 +++++++ cluster_upgrade/tests/__init__.py | 17 +++ cluster_upgrade/tests/base.py | 49 +++++++ cluster_upgrade/tests/test_db_migrations.py | 47 +++++++ cluster_upgrade/tests/test_extension.py | 29 ++++ cluster_upgrade/tests/test_handlers.py | 69 ++++++++++ cluster_upgrade/tests/test_objects.py | 48 +++++++ cluster_upgrade/tests/test_upgrade.py | 98 ++++++++++++++ cluster_upgrade/tests/test_validators.py | 87 ++++++++++++ cluster_upgrade/upgrade.py | 125 ++++++++++++++++++ cluster_upgrade/validators.py | 83 ++++++++++++ 23 files changed, 1151 insertions(+) create mode 100644 cluster_upgrade/__init__.py create mode 100644 cluster_upgrade/alembic_migrations/__init__.py create mode 100644 cluster_upgrade/alembic_migrations/alembic.ini create mode 100644 cluster_upgrade/alembic_migrations/migrations/__init__.py create mode 100644 cluster_upgrade/alembic_migrations/migrations/env.py create mode 100644 cluster_upgrade/alembic_migrations/migrations/script.py.mako create mode 100644 cluster_upgrade/alembic_migrations/migrations/versions/001_add_upgrade_relations_table.py create mode 100644 cluster_upgrade/extension.py create mode 100644 cluster_upgrade/handlers.py create mode 100644 cluster_upgrade/models.py create mode 100644 cluster_upgrade/objects/__init__.py create mode 100644 cluster_upgrade/objects/adapters.py create mode 100644 cluster_upgrade/objects/relations.py create mode 100644 cluster_upgrade/tests/__init__.py create mode 100644 cluster_upgrade/tests/base.py create mode 100644 cluster_upgrade/tests/test_db_migrations.py create mode 100644 cluster_upgrade/tests/test_extension.py create mode 100644 cluster_upgrade/tests/test_handlers.py create mode 100644 cluster_upgrade/tests/test_objects.py create mode 100644 cluster_upgrade/tests/test_upgrade.py create mode 100644 cluster_upgrade/tests/test_validators.py create mode 100644 cluster_upgrade/upgrade.py create mode 100644 cluster_upgrade/validators.py diff --git a/cluster_upgrade/__init__.py b/cluster_upgrade/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cluster_upgrade/alembic_migrations/__init__.py b/cluster_upgrade/alembic_migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cluster_upgrade/alembic_migrations/alembic.ini b/cluster_upgrade/alembic_migrations/alembic.ini new file mode 100644 index 0000000..c0990fe --- /dev/null +++ b/cluster_upgrade/alembic_migrations/alembic.ini @@ -0,0 +1,54 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = migrations + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +sqlalchemy.url = driver://user:pass@localhost/dbname + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/cluster_upgrade/alembic_migrations/migrations/__init__.py b/cluster_upgrade/alembic_migrations/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cluster_upgrade/alembic_migrations/migrations/env.py b/cluster_upgrade/alembic_migrations/migrations/env.py new file mode 100644 index 0000000..14343db --- /dev/null +++ b/cluster_upgrade/alembic_migrations/migrations/env.py @@ -0,0 +1,90 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import with_statement + +from alembic import context +from logging.config import fileConfig +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + context.configure( + url=config.get_main_option('sqlalchemy.url'), + version_table=config.get_main_option('version_table')) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + engine = engine_from_config( + config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + connection = engine.connect() + + context.configure( + connection=connection, + target_metadata=target_metadata, + version_table=config.get_main_option('version_table')) + + try: + with context.begin_transaction(): + context.run_migrations() + finally: + connection.close() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/cluster_upgrade/alembic_migrations/migrations/script.py.mako b/cluster_upgrade/alembic_migrations/migrations/script.py.mako new file mode 100644 index 0000000..43c0940 --- /dev/null +++ b/cluster_upgrade/alembic_migrations/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/cluster_upgrade/alembic_migrations/migrations/versions/001_add_upgrade_relations_table.py b/cluster_upgrade/alembic_migrations/migrations/versions/001_add_upgrade_relations_table.py new file mode 100644 index 0000000..c55cadd --- /dev/null +++ b/cluster_upgrade/alembic_migrations/migrations/versions/001_add_upgrade_relations_table.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""cluster_upgrade + +Revision ID: 3b5d115d7e49 +Revises: None +Create Date: 2015-07-17 19:46:59.579553 + +""" + +# revision identifiers, used by Alembic. +revision = '3b5d115d7e49' +down_revision = None + + +from alembic import context +from alembic import op + +import sqlalchemy as sa + +table_prefix = context.config.get_main_option('table_prefix') +table_upgrade_relation_name = '{0}relations'.format(table_prefix) + + +def upgrade(): + op.create_table( + table_upgrade_relation_name, + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('orig_cluster_id', sa.Integer(), nullable=False), + sa.Column('seed_cluster_id', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('orig_cluster_id'), + sa.UniqueConstraint('seed_cluster_id')) + + +def downgrade(): + op.drop_table(table_upgrade_relation_name) diff --git a/cluster_upgrade/extension.py b/cluster_upgrade/extension.py new file mode 100644 index 0000000..e2431e4 --- /dev/null +++ b/cluster_upgrade/extension.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +from nailgun import extensions + +from . import handlers + + +class ClusterUpgradeExtension(extensions.BaseExtension): + name = 'cluster_upgrade' + version = '0.0.1' + + urls = [ + {'uri': r'/clusters/(?P\d+)/upgrade/clone/?$', + 'handler': handlers.ClusterUpgradeHandler}, + ] + + @classmethod + def alembic_migrations_path(cls): + return os.path.join(os.path.dirname(__file__), + 'alembic_migrations', 'migrations') + + @classmethod + def on_cluster_delete(cls, cluster): + from .objects import relations + + relations.UpgradeRelationObject.delete_relation(cluster.id) diff --git a/cluster_upgrade/handlers.py b/cluster_upgrade/handlers.py new file mode 100644 index 0000000..a361ca5 --- /dev/null +++ b/cluster_upgrade/handlers.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from nailgun.api.v1.handlers import base +from nailgun import objects + +from . import upgrade +from . import validators +from .objects import adapters + + +class ClusterUpgradeHandler(base.BaseHandler): + single = objects.Cluster + validator = validators.ClusterUpgradeValidator + + @base.content + def POST(self, cluster_id): + """Initialize the upgrade of the cluster. + + Creates a new cluster with specified name and release_id. The + new cluster is created with parameters that are copied from the + cluster with the given cluster_id. The values of the generated + and editable attributes are just copied from one to the other. + + :param cluster_id: ID of the cluster from which parameters would + be copied + :returns: JSON representation of the created cluster + :http: * 200 (OK) + * 400 (upgrade parameters are invalid) + * 404 (node or release not found in db) + """ + orig_cluster = adapters.NailgunClusterAdapter( + self.get_object_or_404(self.single, cluster_id)) + request_data = self.checked_data(cluster=orig_cluster) + new_cluster = upgrade.UpgradeHelper.clone_cluster(orig_cluster, + request_data) + return new_cluster.to_json() diff --git a/cluster_upgrade/models.py b/cluster_upgrade/models.py new file mode 100644 index 0000000..c6f740f --- /dev/null +++ b/cluster_upgrade/models.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import Column +from sqlalchemy import Integer + +from nailgun.db.sqlalchemy.models.base import Base + +from . import extension + + +class UpgradeRelation(Base): + __tablename__ = '{0}relations'.format( + extension.ClusterUpgradeExtension.table_prefix()) + + id = Column(Integer, primary_key=True) + orig_cluster_id = Column(Integer, unique=True, nullable=False) + seed_cluster_id = Column(Integer, unique=True, nullable=False) diff --git a/cluster_upgrade/objects/__init__.py b/cluster_upgrade/objects/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cluster_upgrade/objects/adapters.py b/cluster_upgrade/objects/adapters.py new file mode 100644 index 0000000..f4cf917 --- /dev/null +++ b/cluster_upgrade/objects/adapters.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nailgun import objects + + +class NailgunClusterAdapter(object): + def __init__(self, cluster): + self.cluster = cluster + + @classmethod + def create(cls, data): + cluster = objects.Cluster.create(data) + return cls(cluster) + + @property + def id(self): + return self.cluster.id + + @property + def name(self): + return self.cluster.name + + @property + def net_provider(self): + return self.cluster.net_provider + + @property + def release(self): + return NailgunReleaseAdapter(self.cluster.release) + + @property + def generated_attrs(self): + return self.cluster.attributes.generated + + @generated_attrs.setter + def generated_attrs(self, attrs): + self.cluster.attributes.generated = attrs + + @property + def editable_attrs(self): + return self.cluster.attributes.editable + + @editable_attrs.setter + def editable_attrs(self, attrs): + self.cluster.attributes.editable = attrs + + def get_create_data(self): + return objects.Cluster.get_create_data(self.cluster) + + def get_network_manager(self): + net_manager = objects.Cluster.get_network_manager( + instance=self.cluster) + return NailgunNetworkManager(self.cluster, net_manager) + + def to_json(self): + return objects.Cluster.to_json(self.cluster) + + +class NailgunReleaseAdapter(object): + def __init__(self, release): + self.release = release + + @classmethod + def get_by_uid(cls, uid, fail_if_not_found=False): + release = objects.Release.get_by_uid( + uid, fail_if_not_found=fail_if_not_found) + return release + + @property + def is_deployable(self): + return self.release.is_deployable + + def __cmp__(self, other): + if isinstance(other, NailgunReleaseAdapter): + other = other.release + return self.release.__cmp__(other) + + +class NailgunNetworkManager(object): + def __init__(self, cluster, net_manager): + self.cluster = cluster + self.net_manager = net_manager + + def update(self, network_configuration): + self.net_manager.update(self.cluster, network_configuration) + + def get_assigned_vips(self): + return self.net_manager.get_assigned_vips(self.cluster) + + def assign_vips_for_net_groups(self): + return self.net_manager.assign_vips_for_net_groups(self.cluster) + + def assign_given_vips_for_net_groups(self, vips): + self.net_manager.assign_given_vips_for_net_groups(self.cluster, vips) diff --git a/cluster_upgrade/objects/relations.py b/cluster_upgrade/objects/relations.py new file mode 100644 index 0000000..5e2acc8 --- /dev/null +++ b/cluster_upgrade/objects/relations.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nailgun.db import db + +from .. import models + + +class UpgradeRelationObject(object): + @staticmethod + def _query_cluster_relations(cluster_id): + return db.query(models.UpgradeRelation).filter( + (models.UpgradeRelation.orig_cluster_id == cluster_id) | + (models.UpgradeRelation.seed_cluster_id == cluster_id)) + + @classmethod + def get_cluster_relation(cls, cluster_id): + return cls._query_cluster_relations(cluster_id).first() + + @classmethod + def delete_relation(cls, cluster_id): + cls._query_cluster_relations(cluster_id).delete() + + @classmethod + def is_cluster_in_upgrade(cls, cluster_id): + query = cls._query_cluster_relations(cluster_id).exists() + return db.query(query).scalar() + + @classmethod + def create_relation(cls, orig_cluster_id, seed_cluster_id): + relation = models.UpgradeRelation( + orig_cluster_id=orig_cluster_id, + seed_cluster_id=seed_cluster_id) + db.add(relation) + db.flush() diff --git a/cluster_upgrade/tests/__init__.py b/cluster_upgrade/tests/__init__.py new file mode 100644 index 0000000..86c153e --- /dev/null +++ b/cluster_upgrade/tests/__init__.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- + +# Copyright 2013 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +EXTENSION = "nailgun.extensions.cluster_upgrade." diff --git a/cluster_upgrade/tests/base.py b/cluster_upgrade/tests/base.py new file mode 100644 index 0000000..df35c2a --- /dev/null +++ b/cluster_upgrade/tests/base.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nailgun import consts +from nailgun.test import base as nailgun_test_base + +from .. import upgrade +from ..objects import adapters + + +class BaseCloneClusterTest(nailgun_test_base.BaseIntegrationTest): + helper = upgrade.UpgradeHelper + + def setUp(self): + super(BaseCloneClusterTest, self).setUp() + self.release_61 = self.env.create_release( + operating_system=consts.RELEASE_OS.ubuntu, + version="2014.2.2-6.1", + is_deployable=False, + ) + self.release_70 = self.env.create_release( + operating_system=consts.RELEASE_OS.ubuntu, + version="2015.1.0-7.0", + ) + self.cluster_61_db = self.env.create_cluster( + api=False, + release_id=self.release_61.id, + net_provider=consts.CLUSTER_NET_PROVIDERS.neutron, + net_l23_provider=consts.NEUTRON_L23_PROVIDERS.ovs, + ) + self.cluster_61 = adapters.NailgunClusterAdapter( + self.cluster_61_db) + self.data = { + "name": "cluster-clone-{0}".format(self.cluster_61.id), + "release_id": self.release_70.id, + } diff --git a/cluster_upgrade/tests/test_db_migrations.py b/cluster_upgrade/tests/test_db_migrations.py new file mode 100644 index 0000000..16ce49c --- /dev/null +++ b/cluster_upgrade/tests/test_db_migrations.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import alembic + +from nailgun import db +from nailgun.db.migration import make_alembic_config_from_extension +from nailgun.test import base + +from .. import extension + + +_test_revision = '3b5d115d7e49' + + +def setup_module(module): + alembic_config = make_alembic_config_from_extension( + extension.ClusterUpgradeExtension) + db.dropdb() + alembic.command.upgrade(alembic_config, _test_revision) + + +class TestAddRelations(base.BaseAlembicMigrationTest): + + def test_works_without_core_migrations(self): + columns = [ + t.name for t in + self.meta.tables['cluster_upgrade_relations'].columns] + + self.assertItemsEqual(columns, [ + 'id', + 'orig_cluster_id', + 'seed_cluster_id', + ]) diff --git a/cluster_upgrade/tests/test_extension.py b/cluster_upgrade/tests/test_extension.py new file mode 100644 index 0000000..92bff1d --- /dev/null +++ b/cluster_upgrade/tests/test_extension.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +from nailgun.test.base import BaseTestCase + +from .. import extension +from ..objects import relations + + +class TestExtension(BaseTestCase): + @mock.patch.object(relations.UpgradeRelationObject, "delete_relation") + def test_on_cluster_delete(self, mock_on_cluster_delete): + cluster = mock.Mock(id=42) + extension.ClusterUpgradeExtension.on_cluster_delete(cluster) + mock_on_cluster_delete.assert_called_once_with(42) diff --git a/cluster_upgrade/tests/test_handlers.py b/cluster_upgrade/tests/test_handlers.py new file mode 100644 index 0000000..ef0e27b --- /dev/null +++ b/cluster_upgrade/tests/test_handlers.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_serialization import jsonutils + +from nailgun.utils import reverse + +from . import base as tests_base + + +class TestClusterUpgradeHandler(tests_base.BaseCloneClusterTest): + def test_clone(self): + resp = self.app.post( + reverse("ClusterUpgradeHandler", + kwargs={"cluster_id": self.cluster_61.id}), + jsonutils.dumps(self.data), + headers=self.default_headers) + body = resp.json_body + self.assertEqual(resp.status_code, 200) + self.assertEqual(body["name"], + "cluster-clone-{0}".format(self.cluster_61.id)) + self.assertEqual(body["release_id"], self.release_70.id) + + def test_clone_cluster_not_found_error(self): + resp = self.app.post( + reverse("ClusterUpgradeHandler", + kwargs={"cluster_id": 42}), + jsonutils.dumps(self.data), + headers=self.default_headers, + expect_errors=True) + self.assertEqual(resp.status_code, 404) + self.assertEqual(resp.json_body["message"], "Cluster not found") + + def test_clone_cluster_already_in_upgrade_error(self): + self.app.post( + reverse("ClusterUpgradeHandler", + kwargs={"cluster_id": self.cluster_61.id}), + jsonutils.dumps(self.data), + headers=self.default_headers) + resp = self.app.post( + reverse("ClusterUpgradeHandler", + kwargs={"cluster_id": self.cluster_61.id}), + jsonutils.dumps(self.data), + headers=self.default_headers, + expect_errors=True) + self.assertEqual(resp.status_code, 400) + + def test_clone_cluster_name_already_exists_error(self): + data = dict(self.data, name=self.cluster_61.name) + resp = self.app.post( + reverse("ClusterUpgradeHandler", + kwargs={"cluster_id": self.cluster_61.id}), + jsonutils.dumps(data), + headers=self.default_headers, + expect_errors=True) + self.assertEqual(resp.status_code, 409) diff --git a/cluster_upgrade/tests/test_objects.py b/cluster_upgrade/tests/test_objects.py new file mode 100644 index 0000000..746b9ec --- /dev/null +++ b/cluster_upgrade/tests/test_objects.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nailgun.test.base import BaseIntegrationTest + +from .. import models +from ..objects import relations as objects + + +class TestUpgradeRelationObject(BaseIntegrationTest): + def test_get_and_create_relation(self): + objects.UpgradeRelationObject.create_relation(1, 2) + rel0 = objects.UpgradeRelationObject.get_cluster_relation(1) + self.assertEqual(rel0.orig_cluster_id, 1) + self.assertEqual(rel0.seed_cluster_id, 2) + rel1 = objects.UpgradeRelationObject.get_cluster_relation(2) + self.assertEqual(rel1.orig_cluster_id, 1) + self.assertEqual(rel1.seed_cluster_id, 2) + + def test_is_cluster_in_upgrade(self): + objects.UpgradeRelationObject.create_relation(1, 2) + in_upgrade = objects.UpgradeRelationObject.is_cluster_in_upgrade + self.assertTrue(in_upgrade(1)) + self.assertTrue(in_upgrade(2)) + + def test_is_cluster_not_in_upgrade(self): + self.assertEqual(self.db.query(models.UpgradeRelation).count(), 0) + in_upgrade = objects.UpgradeRelationObject.is_cluster_in_upgrade + self.assertFalse(in_upgrade(1)) + self.assertFalse(in_upgrade(2)) + + def test_delete_relation(self): + objects.UpgradeRelationObject.create_relation(1, 2) + objects.UpgradeRelationObject.delete_relation(1) + self.assertEqual(self.db.query(models.UpgradeRelation).count(), 0) diff --git a/cluster_upgrade/tests/test_upgrade.py b/cluster_upgrade/tests/test_upgrade.py new file mode 100644 index 0000000..1504686 --- /dev/null +++ b/cluster_upgrade/tests/test_upgrade.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import six + +from nailgun import consts +from nailgun.objects.serializers import network_configuration + +from . import base as base_tests +from ..objects import relations + + +class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest): + def test_create_cluster_clone(self): + new_cluster = self.helper.create_cluster_clone(self.cluster_61, + self.data) + cluster_61_data = self.cluster_61.get_create_data() + new_cluster_data = new_cluster.get_create_data() + for key, value in cluster_61_data.items(): + if key in ("name", "release_id"): + continue + self.assertEqual(value, new_cluster_data[key]) + + def test_copy_attributes(self): + new_cluster = self.helper.create_cluster_clone(self.cluster_61, + self.data) + self.assertNotEqual(self.cluster_61.generated_attrs, + new_cluster.generated_attrs) + + # Do some unordinary changes + attrs = copy.deepcopy(self.cluster_61.editable_attrs) + attrs["access"]["user"]["value"] = "operator" + attrs["access"]["password"]["value"] = "secrete" + self.cluster_61.editable_attrs = attrs + + self.helper.copy_attributes(self.cluster_61, new_cluster) + + self.assertEqual(self.cluster_61.generated_attrs, + new_cluster.generated_attrs) + editable_attrs = self.cluster_61.editable_attrs + for section, params in six.iteritems(new_cluster.editable_attrs): + if section == "repo_setup": + continue + for key, value in six.iteritems(params): + if key == "metadata": + continue + self.assertEqual(editable_attrs[section][key]["value"], + value["value"]) + + def test_copy_network_config(self): + new_cluster = self.helper.create_cluster_clone(self.cluster_61, + self.data) + orig_net_manager = self.cluster_61.get_network_manager() + new_net_manager = new_cluster.get_network_manager() + + # Do some unordinary changes + nets = network_configuration.NeutronNetworkConfigurationSerializer.\ + serialize_for_cluster(self.cluster_61.cluster) + nets["networks"][0].update({ + "cidr": "172.16.42.0/24", + "gateway": "172.16.42.1", + "ip_ranges": [["172.16.42.2", "172.16.42.126"]], + }) + orig_net_manager.update(nets) + orig_net_manager.assign_vips_for_net_groups() + + self.helper.copy_network_config(self.cluster_61, new_cluster) + + orig_vips = orig_net_manager.get_assigned_vips() + new_vips = new_net_manager.get_assigned_vips() + for net_name in (consts.NETWORKS.public, + consts.NETWORKS.management): + for vip_type in consts.NETWORK_VIP_TYPES: + self.assertEqual(orig_vips[net_name][vip_type], + new_vips[net_name][vip_type]) + + def test_clone_cluster(self): + orig_net_manager = self.cluster_61.get_network_manager() + orig_net_manager.assign_vips_for_net_groups() + new_cluster = self.helper.clone_cluster(self.cluster_61, self.data) + relation = relations.UpgradeRelationObject.get_cluster_relation( + self.cluster_61.id) + self.assertEqual(relation.orig_cluster_id, self.cluster_61.id) + self.assertEqual(relation.seed_cluster_id, new_cluster.id) diff --git a/cluster_upgrade/tests/test_validators.py b/cluster_upgrade/tests/test_validators.py new file mode 100644 index 0000000..bfb6c83 --- /dev/null +++ b/cluster_upgrade/tests/test_validators.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_serialization import jsonutils + +from nailgun import consts +from nailgun.errors import errors + +from .. import validators +from . import base as tests_base +from ..objects import relations + + +class TestClusterUpgradeValidator(tests_base.BaseCloneClusterTest): + validator = validators.ClusterUpgradeValidator + + def test_validate_release_upgrade(self): + self.validator.validate_release_upgrade(self.release_61, + self.release_70) + + def test_validate_release_upgrade_deprecated_release(self): + release_511 = self.env.create_release( + operating_system=consts.RELEASE_OS.ubuntu, + version="2014.1.3-5.1.1", + is_deployable=False, + ) + msg = "^Upgrade to the given release \({0}\).*is deprecated and " \ + "cannot be installed\.$".format(self.release_61.id) + with self.assertRaisesRegexp(errors.InvalidData, msg): + self.validator.validate_release_upgrade(release_511, + self.release_61) + + def test_validate_release_upgrade_to_older_release(self): + self.release_61.is_deployable = True + msg = "^Upgrade to the given release \({0}\).*release is equal or " \ + "lower than the release of the original cluster\.$" \ + .format(self.release_61.id) + with self.assertRaisesRegexp(errors.InvalidData, msg): + self.validator.validate_release_upgrade(self.release_70, + self.release_61) + + def test_validate_cluster_name(self): + self.validator.validate_cluster_name("cluster-42") + + def test_validate_cluster_name_already_exists(self): + msg = "^Environment with this name '{0}' already exists\.$"\ + .format(self.cluster_61.name) + with self.assertRaisesRegexp(errors.AlreadyExists, msg): + self.validator.validate_cluster_name(self.cluster_61.name) + + def test_validate_cluster_status(self): + self.validator.validate_cluster_status(self.cluster_61) + + def test_validate_cluster_status_invalid(self): + cluster_70 = self.env.create_cluster( + api=False, + release_id=self.release_70.id, + ) + relations.UpgradeRelationObject.create_relation(self.cluster_61.id, + cluster_70.id) + msg = "^Upgrade is not possible because of the original cluster " \ + "\({0}\) is already involved in the upgrade routine\.$" \ + .format(self.cluster_61.id) + with self.assertRaisesRegexp(errors.InvalidData, msg): + self.validator.validate_cluster_status(self.cluster_61) + + def test_validate(self): + data = jsonutils.dumps(self.data) + self.validator.validate(data, self.cluster_61) + + def test_validate_invalid_data(self): + data = "{}" + with self.assertRaises(errors.InvalidData): + self.validator.validate(data, self.cluster_61) diff --git a/cluster_upgrade/upgrade.py b/cluster_upgrade/upgrade.py new file mode 100644 index 0000000..49bd16d --- /dev/null +++ b/cluster_upgrade/upgrade.py @@ -0,0 +1,125 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import six + +from nailgun import consts +from nailgun.objects.serializers import network_configuration +from nailgun import utils + +from .objects import adapters + + +def merge_attributes(a, b): + """Merge values of editable attributes. + + The values of the b attributes have precedence over the values + of the a attributes. + """ + attrs = copy.deepcopy(b) + for section, pairs in six.iteritems(attrs): + if section == "repo_setup" or section not in a: + continue + a_values = a[section] + for key, values in six.iteritems(pairs): + if key != "metadata" and key in a_values: + values["value"] = a_values[key]["value"] + return attrs + + +def merge_nets(a, b): + new_settings = copy.deepcopy(b) + source_networks = dict((n["name"], n) for n in a["networks"]) + for net in new_settings["networks"]: + if net["name"] not in source_networks: + continue + source_net = source_networks[net["name"]] + for key, value in six.iteritems(net): + if (key not in ("cluster_id", "id", "meta", "group_id") and + key in source_net): + net[key] = source_net[key] + networking_params = new_settings["networking_parameters"] + source_params = a["networking_parameters"] + for key, value in six.iteritems(networking_params): + if key not in source_params: + continue + networking_params[key] = source_params[key] + return new_settings + + +class UpgradeHelper(object): + network_serializers = { + consts.CLUSTER_NET_PROVIDERS.neutron: + network_configuration.NeutronNetworkConfigurationSerializer, + consts.CLUSTER_NET_PROVIDERS.nova_network: + network_configuration.NovaNetworkConfigurationSerializer, + } + + @classmethod + def clone_cluster(cls, orig_cluster, data): + from .objects import relations + + new_cluster = cls.create_cluster_clone(orig_cluster, data) + cls.copy_attributes(orig_cluster, new_cluster) + cls.copy_network_config(orig_cluster, new_cluster) + relations.UpgradeRelationObject.create_relation(orig_cluster.id, + new_cluster.id) + return new_cluster + + @classmethod + def create_cluster_clone(cls, orig_cluster, data): + create_data = orig_cluster.get_create_data() + create_data["name"] = data["name"] + create_data["release_id"] = data["release_id"] + new_cluster = adapters.NailgunClusterAdapter.create(create_data) + return new_cluster + + @classmethod + def copy_attributes(cls, orig_cluster, new_cluster): + # TODO(akscram): Attributes should be copied including + # borderline cases when some parameters are + # renamed or moved into plugins. Also, we should + # to keep special steps in copying of parameters + # that know how to translate parameters from one + # version to another. A set of this kind of steps + # should define an upgrade path of a particular + # cluster. + new_cluster.generated_attrs = utils.dict_merge( + new_cluster.generated_attrs, + orig_cluster.generated_attrs) + new_cluster.editable_attrs = merge_attributes( + orig_cluster.editable_attrs, + new_cluster.editable_attrs) + + @classmethod + def copy_network_config(cls, orig_cluster, new_cluster): + nets_serializer = cls.network_serializers[orig_cluster.net_provider] + nets = merge_nets( + nets_serializer.serialize_for_cluster(orig_cluster.cluster), + nets_serializer.serialize_for_cluster(new_cluster.cluster)) + + orig_net_manager = orig_cluster.get_network_manager() + new_net_manager = new_cluster.get_network_manager() + + new_net_manager.update(nets) + vips = orig_net_manager.get_assigned_vips() + for ng_name in vips: + if ng_name not in (consts.NETWORKS.public, + consts.NETWORKS.management): + vips.pop(ng_name) + new_net_manager.assign_given_vips_for_net_groups(vips) + new_net_manager.assign_vips_for_net_groups() diff --git a/cluster_upgrade/validators.py b/cluster_upgrade/validators.py new file mode 100644 index 0000000..e8990b9 --- /dev/null +++ b/cluster_upgrade/validators.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nailgun.api.v1.validators import base +from nailgun.errors import errors +from nailgun import objects + +from .objects import adapters + + +class ClusterUpgradeValidator(base.BasicValidator): + schema = { + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Start upgrade procedure for a cluster", + "description": "Serialized parameters to upgrade a cluster.", + "type": "object", + "properties": { + "name": {"type": "string"}, + "release_id": {"type": "number"}, + }, + "required": ["name", "release_id"], + } + + @classmethod + def validate(cls, data, cluster): + cluster = adapters.NailgunClusterAdapter(cluster) + data = super(ClusterUpgradeValidator, cls).validate(data) + cls.validate_schema(data, cls.schema) + cls.validate_cluster_status(cluster) + cls.validate_cluster_name(data["name"]) + release = adapters.NailgunReleaseAdapter.get_by_uid( + data["release_id"], fail_if_not_found=True) + cls.validate_release_upgrade(cluster.release, release) + return data + + @classmethod + def validate_release_upgrade(cls, orig_release, new_release): + if not new_release.is_deployable: + raise errors.InvalidData( + "Upgrade to the given release ({0}) is not possible because " + "this release is deprecated and cannot be installed." + .format(new_release.id), + log_message=True) + if orig_release >= new_release: + raise errors.InvalidData( + "Upgrade to the given release ({0}) is not possible because " + "this release is equal or lower than the release of the " + "original cluster.".format(new_release.id), + log_message=True) + + @classmethod + def validate_cluster_name(cls, cluster_name): + clusters = objects.ClusterCollection.filter_by(None, + name=cluster_name) + if clusters.first(): + raise errors.AlreadyExists( + "Environment with this name '{0}' already exists." + .format(cluster_name), + log_message=True) + + @classmethod + def validate_cluster_status(cls, cluster): + from .objects.relations import UpgradeRelationObject + + if UpgradeRelationObject.is_cluster_in_upgrade(cluster.id): + raise errors.InvalidData( + "Upgrade is not possible because of the original cluster ({0})" + " is already involved in the upgrade routine." + .format(cluster.id), + log_message=True) From 26b51e7126c8a0aa96aa7a7850c5d8ab1cc18ea2 Mon Sep 17 00:00:00 2001 From: Ilya Kharin Date: Sun, 19 Jul 2015 23:50:40 +0300 Subject: [PATCH 02/43] Directly assign node to an upgrade cluster The patch adds method that assigns a node to an upgrade cluster without deleting it from DB. This allows to keep ID of the node and IP addresses assigned to it. The node is booted into the bootstrap image as soon as it moves to an upgrade cluster. Implements blueprint: nailgun-api-env-upgrade-extensions Co-Authored-By: Artur Svechnikov Change-Id: If10fadd149a32317420778607146d9d12108d3f9 --- cluster_upgrade/extension.py | 2 + cluster_upgrade/handlers.py | 39 ++++++++++ cluster_upgrade/objects/adapters.py | 91 ++++++++++++++++++++++ cluster_upgrade/tests/test_handlers.py | 96 ++++++++++++++++++++++++ cluster_upgrade/tests/test_validators.py | 36 +++++++++ cluster_upgrade/upgrade.py | 30 ++++++++ cluster_upgrade/validators.py | 53 +++++++++++++ 7 files changed, 347 insertions(+) diff --git a/cluster_upgrade/extension.py b/cluster_upgrade/extension.py index e2431e4..d83df48 100644 --- a/cluster_upgrade/extension.py +++ b/cluster_upgrade/extension.py @@ -28,6 +28,8 @@ class ClusterUpgradeExtension(extensions.BaseExtension): urls = [ {'uri': r'/clusters/(?P\d+)/upgrade/clone/?$', 'handler': handlers.ClusterUpgradeHandler}, + {'uri': r'/clusters/(?P\d+)/upgrade/assign/?$', + 'handler': handlers.NodeReassignHandler}, ] @classmethod diff --git a/cluster_upgrade/handlers.py b/cluster_upgrade/handlers.py index a361ca5..1ef1ae1 100644 --- a/cluster_upgrade/handlers.py +++ b/cluster_upgrade/handlers.py @@ -14,9 +14,11 @@ # License for the specific language governing permissions and limitations # under the License. +import six from nailgun.api.v1.handlers import base from nailgun import objects +from nailgun.task import manager from . import upgrade from . import validators @@ -49,3 +51,40 @@ class ClusterUpgradeHandler(base.BaseHandler): new_cluster = upgrade.UpgradeHelper.clone_cluster(orig_cluster, request_data) return new_cluster.to_json() + + +class NodeReassignHandler(base.BaseHandler): + single = objects.Cluster + validator = validators.NodeReassignValidator + task_manager = manager.ProvisioningTaskManager + + def handle_task(self, cluster_id, nodes): + try: + task_manager = self.task_manager(cluster_id=cluster_id) + task = task_manager.execute(nodes) + except Exception as exc: + raise self.http(400, msg=six.text_type(exc)) + + self.raise_task(task) + + @base.content + def POST(self, cluster_id): + """Reassign node to cluster via reinstallation + + :param cluster_id: ID of the cluster which node should be + assigned to. + :returns: None + :http: * 202 (OK) + * 400 (Incorrect node state or problem with task execution) + * 404 (Cluster or node not found) + """ + cluster = adapters.NailgunClusterAdapter( + self.get_object_or_404(self.single, cluster_id)) + + data = self.checked_data(cluster=cluster) + node = adapters.NailgunNodeAdapter( + self.get_object_or_404(objects.Node, data['node_id'])) + + upgrade.UpgradeHelper.assign_node_to_cluster(node, cluster) + + self.handle_task(cluster_id, [node.node, ]) diff --git a/cluster_upgrade/objects/adapters.py b/cluster_upgrade/objects/adapters.py index f4cf917..13a60b4 100644 --- a/cluster_upgrade/objects/adapters.py +++ b/cluster_upgrade/objects/adapters.py @@ -69,6 +69,19 @@ class NailgunClusterAdapter(object): def to_json(self): return objects.Cluster.to_json(self.cluster) + @classmethod + def get_by_uid(cls, cluster_id): + cluster = objects.Cluster.get_by_uid(cluster_id) + return cls(cluster) + + def get_network_groups(self): + return (NailgunNetworkGroupAdapter(ng) + for ng in self.cluster.network_groups) + + def get_admin_network_group(self): + manager = self.get_network_manager() + return manager.get_admin_network_group() + class NailgunReleaseAdapter(object): def __init__(self, release): @@ -106,3 +119,81 @@ class NailgunNetworkManager(object): def assign_given_vips_for_net_groups(self, vips): self.net_manager.assign_given_vips_for_net_groups(self.cluster, vips) + + def get_admin_network_group(self, node_id=None): + ng = self.net_manager.get_admin_network_group(node_id) + return NailgunNetworkGroupAdapter(ng) + + def set_node_netgroups_ids(self, node, mapping): + return self.net_manager.set_node_netgroups_ids(node.node, mapping) + + def set_nic_assignment_netgroups_ids(self, node, mapping): + return self.net_manager.set_nic_assignment_netgroups_ids( + node.node, mapping) + + def set_bond_assignment_netgroups_ids(self, node, mapping): + return self.net_manager.set_bond_assignment_netgroups_ids( + node.node, mapping) + + +class NailgunNodeAdapter(object): + + def __new__(cls, node=None): + if not node: + return None + return super(NailgunNodeAdapter, cls).__new__(cls, node) + + def __init__(self, node): + self.node = node + + @property + def id(self): + return self.node.id + + @property + def cluster_id(self): + return self.node.cluster_id + + @property + def hostname(self): + return self.node.hostname + + @hostname.setter + def hostname(self, hostname): + self.node.hostname = hostname + + @property + def status(self): + return self.node.status + + @property + def error_type(self): + return self.node.error_type + + @classmethod + def get_by_uid(cls, node_id): + return cls(objects.Node.get_by_uid(node_id)) + + @property + def roles(self): + return self.node.roles + + def update_cluster_assignment(self, cluster): + objects.Node.update_cluster_assignment(self.node, cluster) + + def add_pending_change(self, change): + objects.Node.add_pending_change(self.node, change) + + +class NailgunNetworkGroupAdapter(object): + + def __init__(self, network_group): + self.network_group = network_group + + @property + def id(self): + return self.network_group.id + + @property + def name(self): + return self.network_group.name diff --git a/cluster_upgrade/tests/test_handlers.py b/cluster_upgrade/tests/test_handlers.py index ef0e27b..e68458c 100644 --- a/cluster_upgrade/tests/test_handlers.py +++ b/cluster_upgrade/tests/test_handlers.py @@ -14,8 +14,12 @@ # License for the specific language governing permissions and limitations # under the License. +from mock import patch + from oslo_serialization import jsonutils +from nailgun import consts +from nailgun.test import base from nailgun.utils import reverse from . import base as tests_base @@ -67,3 +71,95 @@ class TestClusterUpgradeHandler(tests_base.BaseCloneClusterTest): headers=self.default_headers, expect_errors=True) self.assertEqual(resp.status_code, 409) + + +class TestNodeReassignHandler(base.BaseIntegrationTest): + + @patch('nailgun.task.task.rpc.cast') + def test_node_reassign_handler(self, mcast): + self.env.create( + cluster_kwargs={'api': False}, + nodes_kwargs=[{'status': consts.NODE_STATUSES.ready}]) + self.env.create_cluster() + cluster = self.env.clusters[0] + seed_cluster = self.env.clusters[1] + node_id = cluster.nodes[0]['id'] + + resp = self.app.post( + reverse('NodeReassignHandler', + kwargs={'cluster_id': seed_cluster['id']}), + jsonutils.dumps({'node_id': node_id}), + headers=self.default_headers) + self.assertEqual(202, resp.status_code) + + args, kwargs = mcast.call_args + nodes = args[1]['args']['provisioning_info']['nodes'] + provisioned_uids = [int(n['uid']) for n in nodes] + self.assertEqual([node_id, ], provisioned_uids) + + def test_node_reassign_handler_no_node(self): + self.env.create_cluster() + + cluster = self.env.clusters[0] + + resp = self.app.post( + reverse('NodeReassignHandler', + kwargs={'cluster_id': cluster['id']}), + jsonutils.dumps({'node_id': 42}), + headers=self.default_headers, + expect_errors=True) + self.assertEqual(404, resp.status_code) + self.assertEqual("Node with id 42 was not found.", + resp.json_body['message']) + + def test_node_reassing_handler_wrong_status(self): + self.env.create( + cluster_kwargs={'api': False}, + nodes_kwargs=[{'status': 'discover'}]) + cluster = self.env.clusters[0] + + resp = self.app.post( + reverse('NodeReassignHandler', + kwargs={'cluster_id': cluster['id']}), + jsonutils.dumps({'node_id': cluster.nodes[0]['id']}), + headers=self.default_headers, + expect_errors=True) + self.assertEqual(400, resp.status_code) + self.assertRegexpMatches(resp.json_body['message'], + "^Node should be in one of statuses:") + + def test_node_reassing_handler_wrong_error_type(self): + self.env.create( + cluster_kwargs={'api': False}, + nodes_kwargs=[{'status': 'error', + 'error_type': 'provision'}]) + cluster = self.env.clusters[0] + + resp = self.app.post( + reverse('NodeReassignHandler', + kwargs={'cluster_id': cluster['id']}), + jsonutils.dumps({'node_id': cluster.nodes[0]['id']}), + headers=self.default_headers, + expect_errors=True) + self.assertEqual(400, resp.status_code) + self.assertRegexpMatches(resp.json_body['message'], + "^Node should be in error state") + + def test_node_reassign_handler_to_the_same_cluster(self): + self.env.create( + cluster_kwargs={'api': False}, + nodes_kwargs=[{'status': 'ready'}]) + cluster = self.env.clusters[0] + + cluster_id = cluster['id'] + node_id = cluster.nodes[0]['id'] + resp = self.app.post( + reverse('NodeReassignHandler', + kwargs={'cluster_id': cluster_id}), + jsonutils.dumps({'node_id': node_id}), + headers=self.default_headers, + expect_errors=True) + self.assertEqual(400, resp.status_code) + self.assertEqual("Node {0} is already assigned to cluster {1}". + format(node_id, cluster_id), + resp.json_body['message']) diff --git a/cluster_upgrade/tests/test_validators.py b/cluster_upgrade/tests/test_validators.py index bfb6c83..8ceb42d 100644 --- a/cluster_upgrade/tests/test_validators.py +++ b/cluster_upgrade/tests/test_validators.py @@ -14,13 +14,17 @@ # License for the specific language governing permissions and limitations # under the License. +import mock + from oslo_serialization import jsonutils from nailgun import consts from nailgun.errors import errors +from nailgun.test import base from .. import validators from . import base as tests_base +from . import EXTENSION from ..objects import relations @@ -85,3 +89,35 @@ class TestClusterUpgradeValidator(tests_base.BaseCloneClusterTest): data = "{}" with self.assertRaises(errors.InvalidData): self.validator.validate(data, self.cluster_61) + + +class TestNodeReassignValidator(base.BaseTestCase): + validator = validators.NodeReassignValidator + + @mock.patch(EXTENSION + "validators.adapters.NailgunNodeAdapter." + "get_by_uid") + def test_validate_node_not_found(self, mock_gbu): + mock_gbu.return_value = None + with self.assertRaises(errors.ObjectNotFound): + self.validator.validate_node(42) + + @mock.patch(EXTENSION + "validators.adapters.NailgunNodeAdapter." + "get_by_uid") + def test_validate_node_wrong_status(self, mock_gbu): + mock_gbu.return_value = mock.Mock(status='wrong_state') + with self.assertRaises(errors.InvalidData): + self.validator.validate_node(42) + + @mock.patch(EXTENSION + "validators.adapters.NailgunNodeAdapter." + "get_by_uid") + def test_validate_node_wrong_error_type(self, mock_gbu): + mock_gbu.return_value = mock.Mock(status='error', + error_type='wrong') + with self.assertRaises(errors.InvalidData): + self.validator.validate_node(42) + + def test_validate_node_cluster(self): + node = mock.Mock(id=42, cluster_id=42) + cluster = mock.Mock(id=42) + with self.assertRaises(errors.InvalidData): + self.validator.validate_node_cluster(node, cluster) diff --git a/cluster_upgrade/upgrade.py b/cluster_upgrade/upgrade.py index 49bd16d..30a3c4b 100644 --- a/cluster_upgrade/upgrade.py +++ b/cluster_upgrade/upgrade.py @@ -123,3 +123,33 @@ class UpgradeHelper(object): vips.pop(ng_name) new_net_manager.assign_given_vips_for_net_groups(vips) new_net_manager.assign_vips_for_net_groups() + + @classmethod + def assign_node_to_cluster(cls, node, seed_cluster): + orig_cluster = adapters.NailgunClusterAdapter.get_by_uid( + node.cluster_id) + + orig_manager = orig_cluster.get_network_manager() + seed_manager = seed_cluster.get_network_manager() + + netgroups_id_mapping = cls.get_netgroups_id_mapping( + orig_cluster, seed_cluster) + + node.update_cluster_assignment(seed_cluster) + seed_manager.set_node_netgroups_ids(node, netgroups_id_mapping) + orig_manager.set_nic_assignment_netgroups_ids( + node, netgroups_id_mapping) + orig_manager.set_bond_assignment_netgroups_ids( + node, netgroups_id_mapping) + node.add_pending_change(consts.CLUSTER_CHANGES.interfaces) + + @classmethod + def get_netgroups_id_mapping(self, orig_cluster, seed_cluster): + orig_ng = orig_cluster.get_network_groups() + seed_ng = seed_cluster.get_network_groups() + + seed_ng_dict = dict((ng.name, ng.id) for ng in seed_ng) + mapping = dict((ng.id, seed_ng_dict[ng.name]) for ng in orig_ng) + mapping[orig_cluster.get_admin_network_group().id] = \ + seed_cluster.get_admin_network_group().id + return mapping diff --git a/cluster_upgrade/validators.py b/cluster_upgrade/validators.py index e8990b9..d867567 100644 --- a/cluster_upgrade/validators.py +++ b/cluster_upgrade/validators.py @@ -15,6 +15,7 @@ # under the License. from nailgun.api.v1.validators import base +from nailgun import consts from nailgun.errors import errors from nailgun import objects @@ -81,3 +82,55 @@ class ClusterUpgradeValidator(base.BasicValidator): " is already involved in the upgrade routine." .format(cluster.id), log_message=True) + + +class NodeReassignValidator(base.BasicValidator): + schema = { + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Assign Node Parameters", + "description": "Serialized parameters to assign node", + "type": "object", + "properties": { + "node_id": {"type": "number"}, + }, + } + + @classmethod + def validate(cls, data, cluster): + data = super(NodeReassignValidator, cls).validate(data) + cls.validate_schema(data, cls.schema) + node = cls.validate_node(data['node_id']) + cls.validate_node_cluster(node, cluster) + return data + + @classmethod + def validate_node(cls, node_id): + node = adapters.NailgunNodeAdapter.get_by_uid(node_id) + + if not node: + raise errors.ObjectNotFound("Node with id {0} was not found.". + format(node_id), log_message=True) + + # node can go to error state while upgrade process + allowed_statuses = (consts.NODE_STATUSES.ready, + consts.NODE_STATUSES.provisioned, + consts.NODE_STATUSES.error) + if node.status not in allowed_statuses: + raise errors.InvalidData("Node should be in one of statuses: {0}." + " Currently node has {1} status.". + format(allowed_statuses, node.status), + log_message=True) + if node.status == consts.NODE_STATUSES.error and\ + node.error_type != consts.NODE_ERRORS.deploy: + raise errors.InvalidData("Node should be in error state only with" + "deploy error type. Currently error type" + " of node is {0}".format(node.error_type), + log_message=True) + return node + + @classmethod + def validate_node_cluster(cls, node, cluster): + if node.cluster_id == cluster.id: + raise errors.InvalidData("Node {0} is already assigned to cluster" + " {1}".format(node.id, cluster.id), + log_message=True) From 3a92661957a93e006633fb2f918b6a9db5832670 Mon Sep 17 00:00:00 2001 From: Ilya Kharin Date: Tue, 11 Aug 2015 16:23:08 +0300 Subject: [PATCH 03/43] Rename types of VIPs during upgrade to 7.0 In the 7.0 release the networking templates were introduced. They use the ip.addrs.vip_type column as names of VIPs and these names differ from names of previous releases. To solve this we can renamed VIPs of older releases during upgrade to 7.0 accoring the rules: management: haproxy -> management public: haproxy -> public public: vrouter -> vrouter_pub Change-Id: Ia77d13ea90408a06896f2a49c6e43d44c6af1d0d Closes-Bug: #1482577 --- cluster_upgrade/objects/adapters.py | 4 +++ cluster_upgrade/tests/test_upgrade.py | 25 +++++++++------- cluster_upgrade/upgrade.py | 43 +++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 11 deletions(-) diff --git a/cluster_upgrade/objects/adapters.py b/cluster_upgrade/objects/adapters.py index 13a60b4..3e107bd 100644 --- a/cluster_upgrade/objects/adapters.py +++ b/cluster_upgrade/objects/adapters.py @@ -97,6 +97,10 @@ class NailgunReleaseAdapter(object): def is_deployable(self): return self.release.is_deployable + @property + def environment_version(self): + return self.release.environment_version + def __cmp__(self, other): if isinstance(other, NailgunReleaseAdapter): other = other.release diff --git a/cluster_upgrade/tests/test_upgrade.py b/cluster_upgrade/tests/test_upgrade.py index 1504686..4eeda80 100644 --- a/cluster_upgrade/tests/test_upgrade.py +++ b/cluster_upgrade/tests/test_upgrade.py @@ -17,7 +17,6 @@ import copy import six -from nailgun import consts from nailgun.objects.serializers import network_configuration from . import base as base_tests @@ -65,11 +64,12 @@ class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest): new_cluster = self.helper.create_cluster_clone(self.cluster_61, self.data) orig_net_manager = self.cluster_61.get_network_manager() - new_net_manager = new_cluster.get_network_manager() + serialize_nets = network_configuration.\ + NeutronNetworkConfigurationSerializer.\ + serialize_for_cluster # Do some unordinary changes - nets = network_configuration.NeutronNetworkConfigurationSerializer.\ - serialize_for_cluster(self.cluster_61.cluster) + nets = serialize_nets(self.cluster_61.cluster) nets["networks"][0].update({ "cidr": "172.16.42.0/24", "gateway": "172.16.42.1", @@ -80,13 +80,16 @@ class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest): self.helper.copy_network_config(self.cluster_61, new_cluster) - orig_vips = orig_net_manager.get_assigned_vips() - new_vips = new_net_manager.get_assigned_vips() - for net_name in (consts.NETWORKS.public, - consts.NETWORKS.management): - for vip_type in consts.NETWORK_VIP_TYPES: - self.assertEqual(orig_vips[net_name][vip_type], - new_vips[net_name][vip_type]) + orig_nets = serialize_nets(self.cluster_61_db) + new_nets = serialize_nets(new_cluster.cluster) + self.assertEqual(orig_nets["management_vip"], + new_nets["management_vip"]) + self.assertEqual(orig_nets["management_vrouter_vip"], + new_nets["management_vrouter_vip"]) + self.assertEqual(orig_nets["public_vip"], + new_nets["public_vip"]) + self.assertEqual(orig_nets["public_vrouter_vip"], + new_nets["public_vrouter_vip"]) def test_clone_cluster(self): orig_net_manager = self.cluster_61.get_network_manager() diff --git a/cluster_upgrade/upgrade.py b/cluster_upgrade/upgrade.py index 30a3c4b..5416463 100644 --- a/cluster_upgrade/upgrade.py +++ b/cluster_upgrade/upgrade.py @@ -14,7 +14,9 @@ # License for the specific language governing permissions and limitations # under the License. +import collections import copy +from distutils import version import six from nailgun import consts @@ -105,6 +107,41 @@ class UpgradeHelper(object): orig_cluster.editable_attrs, new_cluster.editable_attrs) + @classmethod + def transform_vips_for_net_groups_70(cls, vips): + """Rename or remove types of VIPs for 7.0 network groups. + + This method renames types of VIPs from older releases (<7.0) to + be compatible with network groups of the 7.0 release according + to the rules: + + management: haproxy -> management + public: haproxy -> public + public: vrouter -> vrouter_pub + + Note, that in the result VIPs are present only those IPs that + correspond to the given rules. + """ + rename_vip_rules = { + "management": { + "haproxy": "management", + "vrouter": "vrouter", + }, + "public": { + "haproxy": "public", + "vrouter": "vrouter_pub", + }, + } + renamed_vips = collections.defaultdict(dict) + for ng_name, vips in six.iteritems(vips): + ng_vip_rules = rename_vip_rules[ng_name] + for vip_type, vip_addr in six.iteritems(vips): + if vip_type not in ng_vip_rules: + continue + new_vip_type = ng_vip_rules[vip_type] + renamed_vips[ng_name][new_vip_type] = vip_addr + return renamed_vips + @classmethod def copy_network_config(cls, orig_cluster, new_cluster): nets_serializer = cls.network_serializers[orig_cluster.net_provider] @@ -121,6 +158,12 @@ class UpgradeHelper(object): if ng_name not in (consts.NETWORKS.public, consts.NETWORKS.management): vips.pop(ng_name) + # NOTE(akscram): In the 7.0 release was introduced networking + # templates that use the vip_type column as + # unique names of VIPs. + if version.LooseVersion(orig_cluster.release.environment_version) < \ + version.LooseVersion("7.0"): + vips = cls.transform_vips_for_net_groups_70(vips) new_net_manager.assign_given_vips_for_net_groups(vips) new_net_manager.assign_vips_for_net_groups() From f29f6dd05fdbd112289b6bb592119b42141f6a19 Mon Sep 17 00:00:00 2001 From: Ilya Kharin Date: Tue, 18 Aug 2015 14:37:25 +0300 Subject: [PATCH 04/43] Set the node_id param as a required property The node_id property is a required property to perform the re-assigning a node with a given ID from one cluster to another. Change-Id: I2442442260f19833db239b01856d0ff4f63ecfbb Closes-Bug: #1483239 --- cluster_upgrade/tests/test_handlers.py | 20 ++++++++++++++++++++ cluster_upgrade/tests/test_validators.py | 18 ++++++++++++++++++ cluster_upgrade/validators.py | 1 + 3 files changed, 39 insertions(+) diff --git a/cluster_upgrade/tests/test_handlers.py b/cluster_upgrade/tests/test_handlers.py index e68458c..c89ca9a 100644 --- a/cluster_upgrade/tests/test_handlers.py +++ b/cluster_upgrade/tests/test_handlers.py @@ -163,3 +163,23 @@ class TestNodeReassignHandler(base.BaseIntegrationTest): self.assertEqual("Node {0} is already assigned to cluster {1}". format(node_id, cluster_id), resp.json_body['message']) + + def test_node_reassign_handler_with_empty_data(self): + cluster = self.env.create_cluster(api=False) + resp = self.app.post( + reverse('NodeReassignHandler', + kwargs={'cluster_id': cluster.id}), + "{}", + headers=self.default_headers, + expect_errors=True) + self.assertEqual(400, resp.status_code) + + def test_node_reassign_handler_with_empty_body(self): + cluster = self.env.create_cluster(api=False) + resp = self.app.post( + reverse('NodeReassignHandler', + kwargs={'cluster_id': cluster.id}), + "", + headers=self.default_headers, + expect_errors=True) + self.assertEqual(400, resp.status_code) diff --git a/cluster_upgrade/tests/test_validators.py b/cluster_upgrade/tests/test_validators.py index 8ceb42d..16ec066 100644 --- a/cluster_upgrade/tests/test_validators.py +++ b/cluster_upgrade/tests/test_validators.py @@ -121,3 +121,21 @@ class TestNodeReassignValidator(base.BaseTestCase): cluster = mock.Mock(id=42) with self.assertRaises(errors.InvalidData): self.validator.validate_node_cluster(node, cluster) + + def test_validate_empty_data(self): + cluster = self.env.create_cluster(api=False) + node = self.env.create_node(cluster_id=cluster.id, + roles=["compute"], + status="ready") + msg = "^'node_id' is a required property" + with self.assertRaisesRegexp(errors.InvalidData, msg): + self.validator.validate("{}", node) + + def test_validate_empty_body(self): + cluster = self.env.create_cluster(api=False) + node = self.env.create_node(cluster_id=cluster.id, + roles=["compute"], + status="ready") + msg = "^Empty request received$" + with self.assertRaisesRegexp(errors.InvalidData, msg): + self.validator.validate("", node) diff --git a/cluster_upgrade/validators.py b/cluster_upgrade/validators.py index d867567..a86201c 100644 --- a/cluster_upgrade/validators.py +++ b/cluster_upgrade/validators.py @@ -93,6 +93,7 @@ class NodeReassignValidator(base.BasicValidator): "properties": { "node_id": {"type": "number"}, }, + "required": ["node_id"], } @classmethod From e62381890442491fc7bc05f49429e7e53242b89b Mon Sep 17 00:00:00 2001 From: Maciej Kwiek Date: Tue, 20 Oct 2015 11:40:25 +0200 Subject: [PATCH 05/43] Replace release.is_deployable with release.state is_deployable doesn't really give any additional information for a release, it is removed. To make change API-backward-compatible, it is retained as a property dependent on release.state. New state is added - manageonly, for environments that are not able to be deployed, but can still be managed. Change-Id: I518a0114730a2f227c9ef035a376f9a90d3d5bbd Closes-bug: #1503303 DocImpact --- cluster_upgrade/objects/adapters.py | 2 +- cluster_upgrade/tests/base.py | 2 +- cluster_upgrade/tests/test_validators.py | 6 ++++-- cluster_upgrade/validators.py | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cluster_upgrade/objects/adapters.py b/cluster_upgrade/objects/adapters.py index 3e107bd..7bc6776 100644 --- a/cluster_upgrade/objects/adapters.py +++ b/cluster_upgrade/objects/adapters.py @@ -95,7 +95,7 @@ class NailgunReleaseAdapter(object): @property def is_deployable(self): - return self.release.is_deployable + return objects.Release.is_deployable(self.release) @property def environment_version(self): diff --git a/cluster_upgrade/tests/base.py b/cluster_upgrade/tests/base.py index df35c2a..e5bbeee 100644 --- a/cluster_upgrade/tests/base.py +++ b/cluster_upgrade/tests/base.py @@ -29,7 +29,7 @@ class BaseCloneClusterTest(nailgun_test_base.BaseIntegrationTest): self.release_61 = self.env.create_release( operating_system=consts.RELEASE_OS.ubuntu, version="2014.2.2-6.1", - is_deployable=False, + state=consts.RELEASE_STATES.manageonly ) self.release_70 = self.env.create_release( operating_system=consts.RELEASE_OS.ubuntu, diff --git a/cluster_upgrade/tests/test_validators.py b/cluster_upgrade/tests/test_validators.py index 16ec066..7f9984a 100644 --- a/cluster_upgrade/tests/test_validators.py +++ b/cluster_upgrade/tests/test_validators.py @@ -20,6 +20,7 @@ from oslo_serialization import jsonutils from nailgun import consts from nailgun.errors import errors +from nailgun.settings import settings from nailgun.test import base from .. import validators @@ -35,11 +36,12 @@ class TestClusterUpgradeValidator(tests_base.BaseCloneClusterTest): self.validator.validate_release_upgrade(self.release_61, self.release_70) + @mock.patch.dict(settings.VERSION, {'feature_groups': ['mirantis']}) def test_validate_release_upgrade_deprecated_release(self): release_511 = self.env.create_release( operating_system=consts.RELEASE_OS.ubuntu, version="2014.1.3-5.1.1", - is_deployable=False, + state=consts.RELEASE_STATES.manageonly ) msg = "^Upgrade to the given release \({0}\).*is deprecated and " \ "cannot be installed\.$".format(self.release_61.id) @@ -48,7 +50,7 @@ class TestClusterUpgradeValidator(tests_base.BaseCloneClusterTest): self.release_61) def test_validate_release_upgrade_to_older_release(self): - self.release_61.is_deployable = True + self.release_61.state = consts.RELEASE_STATES.available msg = "^Upgrade to the given release \({0}\).*release is equal or " \ "lower than the release of the original cluster\.$" \ .format(self.release_61.id) diff --git a/cluster_upgrade/validators.py b/cluster_upgrade/validators.py index a86201c..5a6a735 100644 --- a/cluster_upgrade/validators.py +++ b/cluster_upgrade/validators.py @@ -49,7 +49,7 @@ class ClusterUpgradeValidator(base.BasicValidator): @classmethod def validate_release_upgrade(cls, orig_release, new_release): - if not new_release.is_deployable: + if not objects.Release.is_deployable(new_release): raise errors.InvalidData( "Upgrade to the given release ({0}) is not possible because " "this release is deprecated and cannot be installed." From 1d8bfe5435c9d08b62057dce91d04c6cce0482a7 Mon Sep 17 00:00:00 2001 From: Oleg Gelbukh Date: Fri, 30 Oct 2015 13:54:28 +0000 Subject: [PATCH 06/43] Rename ClusterUpgradeHandler Name ClusterUpgradeHandler should be used for handling uri /cluster//upgrade. Current handler performs clone of environment for upgrading. Thus, rename ClusterUpgradeHandler to ClusterUpgradeCloneHandler. Blueprint: upgrade-major-openstack-environment Change-Id: I6c98e0882c300d587caa32429b49b22baa9ad82f --- cluster_upgrade/extension.py | 2 +- cluster_upgrade/handlers.py | 2 +- cluster_upgrade/tests/test_handlers.py | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cluster_upgrade/extension.py b/cluster_upgrade/extension.py index d83df48..36243c6 100644 --- a/cluster_upgrade/extension.py +++ b/cluster_upgrade/extension.py @@ -27,7 +27,7 @@ class ClusterUpgradeExtension(extensions.BaseExtension): urls = [ {'uri': r'/clusters/(?P\d+)/upgrade/clone/?$', - 'handler': handlers.ClusterUpgradeHandler}, + 'handler': handlers.ClusterUpgradeCloneHandler}, {'uri': r'/clusters/(?P\d+)/upgrade/assign/?$', 'handler': handlers.NodeReassignHandler}, ] diff --git a/cluster_upgrade/handlers.py b/cluster_upgrade/handlers.py index 1ef1ae1..eb5f64c 100644 --- a/cluster_upgrade/handlers.py +++ b/cluster_upgrade/handlers.py @@ -25,7 +25,7 @@ from . import validators from .objects import adapters -class ClusterUpgradeHandler(base.BaseHandler): +class ClusterUpgradeCloneHandler(base.BaseHandler): single = objects.Cluster validator = validators.ClusterUpgradeValidator diff --git a/cluster_upgrade/tests/test_handlers.py b/cluster_upgrade/tests/test_handlers.py index c89ca9a..f2e521a 100644 --- a/cluster_upgrade/tests/test_handlers.py +++ b/cluster_upgrade/tests/test_handlers.py @@ -25,10 +25,10 @@ from nailgun.utils import reverse from . import base as tests_base -class TestClusterUpgradeHandler(tests_base.BaseCloneClusterTest): +class TestClusterUpgradeCloneHandler(tests_base.BaseCloneClusterTest): def test_clone(self): resp = self.app.post( - reverse("ClusterUpgradeHandler", + reverse("ClusterUpgradeCloneHandler", kwargs={"cluster_id": self.cluster_61.id}), jsonutils.dumps(self.data), headers=self.default_headers) @@ -40,7 +40,7 @@ class TestClusterUpgradeHandler(tests_base.BaseCloneClusterTest): def test_clone_cluster_not_found_error(self): resp = self.app.post( - reverse("ClusterUpgradeHandler", + reverse("ClusterUpgradeCloneHandler", kwargs={"cluster_id": 42}), jsonutils.dumps(self.data), headers=self.default_headers, @@ -50,12 +50,12 @@ class TestClusterUpgradeHandler(tests_base.BaseCloneClusterTest): def test_clone_cluster_already_in_upgrade_error(self): self.app.post( - reverse("ClusterUpgradeHandler", + reverse("ClusterUpgradeCloneHandler", kwargs={"cluster_id": self.cluster_61.id}), jsonutils.dumps(self.data), headers=self.default_headers) resp = self.app.post( - reverse("ClusterUpgradeHandler", + reverse("ClusterUpgradeCloneHandler", kwargs={"cluster_id": self.cluster_61.id}), jsonutils.dumps(self.data), headers=self.default_headers, @@ -65,7 +65,7 @@ class TestClusterUpgradeHandler(tests_base.BaseCloneClusterTest): def test_clone_cluster_name_already_exists_error(self): data = dict(self.data, name=self.cluster_61.name) resp = self.app.post( - reverse("ClusterUpgradeHandler", + reverse("ClusterUpgradeCloneHandler", kwargs={"cluster_id": self.cluster_61.id}), jsonutils.dumps(data), headers=self.default_headers, From 192f9e192bb6359fd3bf29c985846dd16eb02c54 Mon Sep 17 00:00:00 2001 From: Andrey Shestakov Date: Mon, 2 Nov 2015 17:08:04 +0200 Subject: [PATCH 07/43] Patch network_roles_metadata for 7.0 release where needed The patching mostly done for tests for nailgun components adhering to the release version. E.g. network serializers. Some tests are forced to create environments of 8.0 version. The change is needed to resolve possible issues with the tests when network roles metadata is changed drastically and is not compatible with mentioned components. Partial-Bug: #1517874 Change-Id: I55607157ae7767ffdfd1d855b872630832706e8e --- cluster_upgrade/tests/base.py | 8 +++++--- cluster_upgrade/tests/test_handlers.py | 2 +- cluster_upgrade/tests/test_validators.py | 10 +++++----- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/cluster_upgrade/tests/base.py b/cluster_upgrade/tests/base.py index e5bbeee..c99838d 100644 --- a/cluster_upgrade/tests/base.py +++ b/cluster_upgrade/tests/base.py @@ -31,10 +31,12 @@ class BaseCloneClusterTest(nailgun_test_base.BaseIntegrationTest): version="2014.2.2-6.1", state=consts.RELEASE_STATES.manageonly ) - self.release_70 = self.env.create_release( + + self.release_80 = self.env.create_release( operating_system=consts.RELEASE_OS.ubuntu, - version="2015.1.0-7.0", + version="2015.1.0-8.0", ) + self.cluster_61_db = self.env.create_cluster( api=False, release_id=self.release_61.id, @@ -45,5 +47,5 @@ class BaseCloneClusterTest(nailgun_test_base.BaseIntegrationTest): self.cluster_61_db) self.data = { "name": "cluster-clone-{0}".format(self.cluster_61.id), - "release_id": self.release_70.id, + "release_id": self.release_80.id, } diff --git a/cluster_upgrade/tests/test_handlers.py b/cluster_upgrade/tests/test_handlers.py index f2e521a..736052c 100644 --- a/cluster_upgrade/tests/test_handlers.py +++ b/cluster_upgrade/tests/test_handlers.py @@ -36,7 +36,7 @@ class TestClusterUpgradeCloneHandler(tests_base.BaseCloneClusterTest): self.assertEqual(resp.status_code, 200) self.assertEqual(body["name"], "cluster-clone-{0}".format(self.cluster_61.id)) - self.assertEqual(body["release_id"], self.release_70.id) + self.assertEqual(body["release_id"], self.release_80.id) def test_clone_cluster_not_found_error(self): resp = self.app.post( diff --git a/cluster_upgrade/tests/test_validators.py b/cluster_upgrade/tests/test_validators.py index 7f9984a..280734e 100644 --- a/cluster_upgrade/tests/test_validators.py +++ b/cluster_upgrade/tests/test_validators.py @@ -34,7 +34,7 @@ class TestClusterUpgradeValidator(tests_base.BaseCloneClusterTest): def test_validate_release_upgrade(self): self.validator.validate_release_upgrade(self.release_61, - self.release_70) + self.release_80) @mock.patch.dict(settings.VERSION, {'feature_groups': ['mirantis']}) def test_validate_release_upgrade_deprecated_release(self): @@ -55,7 +55,7 @@ class TestClusterUpgradeValidator(tests_base.BaseCloneClusterTest): "lower than the release of the original cluster\.$" \ .format(self.release_61.id) with self.assertRaisesRegexp(errors.InvalidData, msg): - self.validator.validate_release_upgrade(self.release_70, + self.validator.validate_release_upgrade(self.release_80, self.release_61) def test_validate_cluster_name(self): @@ -71,12 +71,12 @@ class TestClusterUpgradeValidator(tests_base.BaseCloneClusterTest): self.validator.validate_cluster_status(self.cluster_61) def test_validate_cluster_status_invalid(self): - cluster_70 = self.env.create_cluster( + cluster_80 = self.env.create_cluster( api=False, - release_id=self.release_70.id, + release_id=self.release_80.id, ) relations.UpgradeRelationObject.create_relation(self.cluster_61.id, - cluster_70.id) + cluster_80.id) msg = "^Upgrade is not possible because of the original cluster " \ "\({0}\) is already involved in the upgrade routine\.$" \ .format(self.cluster_61.id) From c5fd9ee16f81b630104810404da8f2457ad52117 Mon Sep 17 00:00:00 2001 From: Oleg Gelbukh Date: Thu, 22 Oct 2015 17:57:23 +0000 Subject: [PATCH 08/43] Change openstack_version to liberty-9.0 in openstack.yaml Set release name to 'Liberty' in display name and description of new release. Set openstack_version setting to 'liberty-9.0' to comply with the existing versions schema. Set opesntack_version to 'liberty-9.0' in tests. Closes-bug: 1503663 Change-Id: Ifef952e18f08b98c430bbff9434984deaa68df81 --- cluster_upgrade/tests/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster_upgrade/tests/base.py b/cluster_upgrade/tests/base.py index c99838d..50b07c7 100644 --- a/cluster_upgrade/tests/base.py +++ b/cluster_upgrade/tests/base.py @@ -34,7 +34,7 @@ class BaseCloneClusterTest(nailgun_test_base.BaseIntegrationTest): self.release_80 = self.env.create_release( operating_system=consts.RELEASE_OS.ubuntu, - version="2015.1.0-8.0", + version="liberty-8.0", ) self.cluster_61_db = self.env.create_cluster( From f0faf014f82212e4fa0bba10954605d9bfa23527 Mon Sep 17 00:00:00 2001 From: Ivan Kliuk Date: Sat, 5 Dec 2015 17:16:38 +0200 Subject: [PATCH 09/43] New VIP-related fields in the database * Add 'is_user_defined' field to 'ip_addrs' table. * Rename 'vip_type' field to 'vip_name' of table 'ip_addrs'. * Add 'vip_namespace' field to 'ip_addrs' table. * Copy vip namespaces from plugin table network roles to 'vip_namespace' field according to the unique vip name. * Add database migrations. * Add unit test for migrations. Change-Id: Ia3e1d7f6e08dbebcb182de75eeaf58ddf6be4a8d Partial-bug: #1482399 --- cluster_upgrade/upgrade.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cluster_upgrade/upgrade.py b/cluster_upgrade/upgrade.py index 5416463..6fe249f 100644 --- a/cluster_upgrade/upgrade.py +++ b/cluster_upgrade/upgrade.py @@ -135,11 +135,11 @@ class UpgradeHelper(object): renamed_vips = collections.defaultdict(dict) for ng_name, vips in six.iteritems(vips): ng_vip_rules = rename_vip_rules[ng_name] - for vip_type, vip_addr in six.iteritems(vips): - if vip_type not in ng_vip_rules: + for vip_name, vip_addr in six.iteritems(vips): + if vip_name not in ng_vip_rules: continue - new_vip_type = ng_vip_rules[vip_type] - renamed_vips[ng_name][new_vip_type] = vip_addr + new_vip_name = ng_vip_rules[vip_name] + renamed_vips[ng_name][new_vip_name] = vip_addr return renamed_vips @classmethod @@ -159,7 +159,7 @@ class UpgradeHelper(object): consts.NETWORKS.management): vips.pop(ng_name) # NOTE(akscram): In the 7.0 release was introduced networking - # templates that use the vip_type column as + # templates that use the vip_name column as # unique names of VIPs. if version.LooseVersion(orig_cluster.release.environment_version) < \ version.LooseVersion("7.0"): From 761d356c22ab134f5a4f501bf8b23a76cf6a6b74 Mon Sep 17 00:00:00 2001 From: Sylwester Brzeczkowski Date: Tue, 22 Dec 2015 11:22:08 +0100 Subject: [PATCH 10/43] Nailgun extensions in stevedore The change introduces Nailgun extensions which use stevedore to plug into nailgun. Stevedore enables (not only) Fuel developers to write extensions and develop them independently in separate repos. Functions for loading and retriving extensions classes was moved to new file `manager.py`. Change-Id: I59015a28f460b1e45312b1c003aadec3cc396ad5 Implements: blueprint stevedore-extensions-discovery --- cluster_upgrade/extension.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster_upgrade/extension.py b/cluster_upgrade/extension.py index 36243c6..f484fa9 100644 --- a/cluster_upgrade/extension.py +++ b/cluster_upgrade/extension.py @@ -24,6 +24,7 @@ from . import handlers class ClusterUpgradeExtension(extensions.BaseExtension): name = 'cluster_upgrade' version = '0.0.1' + description = "Cluster Upgrade Extension" urls = [ {'uri': r'/clusters/(?P\d+)/upgrade/clone/?$', From a2fd459b6bdcf6e500ae7ff896537707bab4d5e8 Mon Sep 17 00:00:00 2001 From: Ryan Moe Date: Tue, 27 Oct 2015 14:54:43 -0700 Subject: [PATCH 11/43] Move all db queries from network manager to objects All network-related database queries are moved into the appropriate object methods. This is being done to make it possible to have an external network management service. Blueprint: network-config-refactoring Change-Id: I4ce965f227c54577659e64f598ff5cdf4c868ed6 --- cluster_upgrade/objects/adapters.py | 11 +++++------ cluster_upgrade/upgrade.py | 4 ++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/cluster_upgrade/objects/adapters.py b/cluster_upgrade/objects/adapters.py index 7bc6776..6890f8e 100644 --- a/cluster_upgrade/objects/adapters.py +++ b/cluster_upgrade/objects/adapters.py @@ -79,8 +79,7 @@ class NailgunClusterAdapter(object): for ng in self.cluster.network_groups) def get_admin_network_group(self): - manager = self.get_network_manager() - return manager.get_admin_network_group() + return objects.NetworkGroup.get_admin_network_group() class NailgunReleaseAdapter(object): @@ -125,18 +124,18 @@ class NailgunNetworkManager(object): self.net_manager.assign_given_vips_for_net_groups(self.cluster, vips) def get_admin_network_group(self, node_id=None): - ng = self.net_manager.get_admin_network_group(node_id) + ng = objects.NetworkGroup.get_admin_network_group(node_id) return NailgunNetworkGroupAdapter(ng) def set_node_netgroups_ids(self, node, mapping): - return self.net_manager.set_node_netgroups_ids(node.node, mapping) + return objects.Node.set_netgroups_ids(node.node, mapping) def set_nic_assignment_netgroups_ids(self, node, mapping): - return self.net_manager.set_nic_assignment_netgroups_ids( + return objects.Node.set_nic_assignment_netgroups_ids( node.node, mapping) def set_bond_assignment_netgroups_ids(self, node, mapping): - return self.net_manager.set_bond_assignment_netgroups_ids( + return objects.Node.set_bond_assignment_netgroups_ids( node.node, mapping) diff --git a/cluster_upgrade/upgrade.py b/cluster_upgrade/upgrade.py index 6fe249f..ddd21d3 100644 --- a/cluster_upgrade/upgrade.py +++ b/cluster_upgrade/upgrade.py @@ -20,6 +20,7 @@ from distutils import version import six from nailgun import consts +from nailgun import objects from nailgun.objects.serializers import network_configuration from nailgun import utils @@ -173,13 +174,12 @@ class UpgradeHelper(object): node.cluster_id) orig_manager = orig_cluster.get_network_manager() - seed_manager = seed_cluster.get_network_manager() netgroups_id_mapping = cls.get_netgroups_id_mapping( orig_cluster, seed_cluster) node.update_cluster_assignment(seed_cluster) - seed_manager.set_node_netgroups_ids(node, netgroups_id_mapping) + objects.Node.set_netgroups_ids(node, netgroups_id_mapping) orig_manager.set_nic_assignment_netgroups_ids( node, netgroups_id_mapping) orig_manager.set_bond_assignment_netgroups_ids( From 73b52c9e655e7c67e847dd506e310a9e50d7df30 Mon Sep 17 00:00:00 2001 From: Fedor Zhadaev Date: Mon, 8 Feb 2016 11:42:00 +0300 Subject: [PATCH 12/43] Remove Mirantis-specific code from fuel-web repo Remove code related to registration in Mirantis tracking system and using Mirantis server to collect statistics. Change-Id: Ie1243a8b12368a0d61cd51da2d5ab6cce3eeea65 Related to blueprint remove-vendor-code Depends-On: Id67d6201cb23e371dc42ec6e818bcbc2ca0fde31 --- cluster_upgrade/tests/test_validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster_upgrade/tests/test_validators.py b/cluster_upgrade/tests/test_validators.py index 280734e..c449a06 100644 --- a/cluster_upgrade/tests/test_validators.py +++ b/cluster_upgrade/tests/test_validators.py @@ -36,7 +36,7 @@ class TestClusterUpgradeValidator(tests_base.BaseCloneClusterTest): self.validator.validate_release_upgrade(self.release_61, self.release_80) - @mock.patch.dict(settings.VERSION, {'feature_groups': ['mirantis']}) + @mock.patch.dict(settings.VERSION, {'feature_groups': []}) def test_validate_release_upgrade_deprecated_release(self): release_511 = self.env.create_release( operating_system=consts.RELEASE_OS.ubuntu, From 3cda80e31af64eb182e2f5bfd976417221eefc9d Mon Sep 17 00:00:00 2001 From: Ilya Kharin Date: Mon, 7 Mar 2016 21:04:04 -0600 Subject: [PATCH 13/43] Use the 9.0 release in tests for cluster_upgrade This patch also renames names of variables in tests of the cluster_upgrade extension from more specific names that contain versions of releases to least specific names. Change-Id: Ic02b4426c1bb0bf38ba06cfeb54255043ebc8481 Closes-Bug: #1555339 --- cluster_upgrade/tests/base.py | 18 +++++------ cluster_upgrade/tests/test_handlers.py | 14 ++++----- cluster_upgrade/tests/test_upgrade.py | 38 ++++++++++++------------ cluster_upgrade/tests/test_validators.py | 38 ++++++++++++------------ 4 files changed, 54 insertions(+), 54 deletions(-) diff --git a/cluster_upgrade/tests/base.py b/cluster_upgrade/tests/base.py index 50b07c7..8d7ce10 100644 --- a/cluster_upgrade/tests/base.py +++ b/cluster_upgrade/tests/base.py @@ -26,26 +26,26 @@ class BaseCloneClusterTest(nailgun_test_base.BaseIntegrationTest): def setUp(self): super(BaseCloneClusterTest, self).setUp() - self.release_61 = self.env.create_release( + self.src_release = self.env.create_release( operating_system=consts.RELEASE_OS.ubuntu, version="2014.2.2-6.1", state=consts.RELEASE_STATES.manageonly ) - self.release_80 = self.env.create_release( + self.dst_release = self.env.create_release( operating_system=consts.RELEASE_OS.ubuntu, - version="liberty-8.0", + version="liberty-9.0", ) - self.cluster_61_db = self.env.create_cluster( + self.src_cluster_db = self.env.create_cluster( api=False, - release_id=self.release_61.id, + release_id=self.src_release.id, net_provider=consts.CLUSTER_NET_PROVIDERS.neutron, net_l23_provider=consts.NEUTRON_L23_PROVIDERS.ovs, ) - self.cluster_61 = adapters.NailgunClusterAdapter( - self.cluster_61_db) + self.src_cluster = adapters.NailgunClusterAdapter( + self.src_cluster_db) self.data = { - "name": "cluster-clone-{0}".format(self.cluster_61.id), - "release_id": self.release_80.id, + "name": "cluster-clone-{0}".format(self.src_cluster.id), + "release_id": self.dst_release.id, } diff --git a/cluster_upgrade/tests/test_handlers.py b/cluster_upgrade/tests/test_handlers.py index 736052c..d960be5 100644 --- a/cluster_upgrade/tests/test_handlers.py +++ b/cluster_upgrade/tests/test_handlers.py @@ -29,14 +29,14 @@ class TestClusterUpgradeCloneHandler(tests_base.BaseCloneClusterTest): def test_clone(self): resp = self.app.post( reverse("ClusterUpgradeCloneHandler", - kwargs={"cluster_id": self.cluster_61.id}), + kwargs={"cluster_id": self.src_cluster.id}), jsonutils.dumps(self.data), headers=self.default_headers) body = resp.json_body self.assertEqual(resp.status_code, 200) self.assertEqual(body["name"], - "cluster-clone-{0}".format(self.cluster_61.id)) - self.assertEqual(body["release_id"], self.release_80.id) + "cluster-clone-{0}".format(self.src_cluster.id)) + self.assertEqual(body["release_id"], self.dst_release.id) def test_clone_cluster_not_found_error(self): resp = self.app.post( @@ -51,22 +51,22 @@ class TestClusterUpgradeCloneHandler(tests_base.BaseCloneClusterTest): def test_clone_cluster_already_in_upgrade_error(self): self.app.post( reverse("ClusterUpgradeCloneHandler", - kwargs={"cluster_id": self.cluster_61.id}), + kwargs={"cluster_id": self.src_cluster.id}), jsonutils.dumps(self.data), headers=self.default_headers) resp = self.app.post( reverse("ClusterUpgradeCloneHandler", - kwargs={"cluster_id": self.cluster_61.id}), + kwargs={"cluster_id": self.src_cluster.id}), jsonutils.dumps(self.data), headers=self.default_headers, expect_errors=True) self.assertEqual(resp.status_code, 400) def test_clone_cluster_name_already_exists_error(self): - data = dict(self.data, name=self.cluster_61.name) + data = dict(self.data, name=self.src_cluster.name) resp = self.app.post( reverse("ClusterUpgradeCloneHandler", - kwargs={"cluster_id": self.cluster_61.id}), + kwargs={"cluster_id": self.src_cluster.id}), jsonutils.dumps(data), headers=self.default_headers, expect_errors=True) diff --git a/cluster_upgrade/tests/test_upgrade.py b/cluster_upgrade/tests/test_upgrade.py index 4eeda80..723f0ee 100644 --- a/cluster_upgrade/tests/test_upgrade.py +++ b/cluster_upgrade/tests/test_upgrade.py @@ -25,32 +25,32 @@ from ..objects import relations class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest): def test_create_cluster_clone(self): - new_cluster = self.helper.create_cluster_clone(self.cluster_61, + new_cluster = self.helper.create_cluster_clone(self.src_cluster, self.data) - cluster_61_data = self.cluster_61.get_create_data() + src_cluster_data = self.src_cluster.get_create_data() new_cluster_data = new_cluster.get_create_data() - for key, value in cluster_61_data.items(): + for key, value in src_cluster_data.items(): if key in ("name", "release_id"): continue self.assertEqual(value, new_cluster_data[key]) def test_copy_attributes(self): - new_cluster = self.helper.create_cluster_clone(self.cluster_61, + new_cluster = self.helper.create_cluster_clone(self.src_cluster, self.data) - self.assertNotEqual(self.cluster_61.generated_attrs, + self.assertNotEqual(self.src_cluster.generated_attrs, new_cluster.generated_attrs) # Do some unordinary changes - attrs = copy.deepcopy(self.cluster_61.editable_attrs) + attrs = copy.deepcopy(self.src_cluster.editable_attrs) attrs["access"]["user"]["value"] = "operator" attrs["access"]["password"]["value"] = "secrete" - self.cluster_61.editable_attrs = attrs + self.src_cluster.editable_attrs = attrs - self.helper.copy_attributes(self.cluster_61, new_cluster) + self.helper.copy_attributes(self.src_cluster, new_cluster) - self.assertEqual(self.cluster_61.generated_attrs, + self.assertEqual(self.src_cluster.generated_attrs, new_cluster.generated_attrs) - editable_attrs = self.cluster_61.editable_attrs + editable_attrs = self.src_cluster.editable_attrs for section, params in six.iteritems(new_cluster.editable_attrs): if section == "repo_setup": continue @@ -61,15 +61,15 @@ class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest): value["value"]) def test_copy_network_config(self): - new_cluster = self.helper.create_cluster_clone(self.cluster_61, + new_cluster = self.helper.create_cluster_clone(self.src_cluster, self.data) - orig_net_manager = self.cluster_61.get_network_manager() + orig_net_manager = self.src_cluster.get_network_manager() serialize_nets = network_configuration.\ NeutronNetworkConfigurationSerializer.\ serialize_for_cluster # Do some unordinary changes - nets = serialize_nets(self.cluster_61.cluster) + nets = serialize_nets(self.src_cluster.cluster) nets["networks"][0].update({ "cidr": "172.16.42.0/24", "gateway": "172.16.42.1", @@ -78,9 +78,9 @@ class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest): orig_net_manager.update(nets) orig_net_manager.assign_vips_for_net_groups() - self.helper.copy_network_config(self.cluster_61, new_cluster) + self.helper.copy_network_config(self.src_cluster, new_cluster) - orig_nets = serialize_nets(self.cluster_61_db) + orig_nets = serialize_nets(self.src_cluster_db) new_nets = serialize_nets(new_cluster.cluster) self.assertEqual(orig_nets["management_vip"], new_nets["management_vip"]) @@ -92,10 +92,10 @@ class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest): new_nets["public_vrouter_vip"]) def test_clone_cluster(self): - orig_net_manager = self.cluster_61.get_network_manager() + orig_net_manager = self.src_cluster.get_network_manager() orig_net_manager.assign_vips_for_net_groups() - new_cluster = self.helper.clone_cluster(self.cluster_61, self.data) + new_cluster = self.helper.clone_cluster(self.src_cluster, self.data) relation = relations.UpgradeRelationObject.get_cluster_relation( - self.cluster_61.id) - self.assertEqual(relation.orig_cluster_id, self.cluster_61.id) + self.src_cluster.id) + self.assertEqual(relation.orig_cluster_id, self.src_cluster.id) self.assertEqual(relation.seed_cluster_id, new_cluster.id) diff --git a/cluster_upgrade/tests/test_validators.py b/cluster_upgrade/tests/test_validators.py index c449a06..ac906f5 100644 --- a/cluster_upgrade/tests/test_validators.py +++ b/cluster_upgrade/tests/test_validators.py @@ -33,8 +33,8 @@ class TestClusterUpgradeValidator(tests_base.BaseCloneClusterTest): validator = validators.ClusterUpgradeValidator def test_validate_release_upgrade(self): - self.validator.validate_release_upgrade(self.release_61, - self.release_80) + self.validator.validate_release_upgrade(self.src_release, + self.dst_release) @mock.patch.dict(settings.VERSION, {'feature_groups': []}) def test_validate_release_upgrade_deprecated_release(self): @@ -44,53 +44,53 @@ class TestClusterUpgradeValidator(tests_base.BaseCloneClusterTest): state=consts.RELEASE_STATES.manageonly ) msg = "^Upgrade to the given release \({0}\).*is deprecated and " \ - "cannot be installed\.$".format(self.release_61.id) + "cannot be installed\.$".format(self.src_release.id) with self.assertRaisesRegexp(errors.InvalidData, msg): self.validator.validate_release_upgrade(release_511, - self.release_61) + self.src_release) def test_validate_release_upgrade_to_older_release(self): - self.release_61.state = consts.RELEASE_STATES.available + self.src_release.state = consts.RELEASE_STATES.available msg = "^Upgrade to the given release \({0}\).*release is equal or " \ "lower than the release of the original cluster\.$" \ - .format(self.release_61.id) + .format(self.src_release.id) with self.assertRaisesRegexp(errors.InvalidData, msg): - self.validator.validate_release_upgrade(self.release_80, - self.release_61) + self.validator.validate_release_upgrade(self.dst_release, + self.src_release) def test_validate_cluster_name(self): self.validator.validate_cluster_name("cluster-42") def test_validate_cluster_name_already_exists(self): msg = "^Environment with this name '{0}' already exists\.$"\ - .format(self.cluster_61.name) + .format(self.src_cluster.name) with self.assertRaisesRegexp(errors.AlreadyExists, msg): - self.validator.validate_cluster_name(self.cluster_61.name) + self.validator.validate_cluster_name(self.src_cluster.name) def test_validate_cluster_status(self): - self.validator.validate_cluster_status(self.cluster_61) + self.validator.validate_cluster_status(self.src_cluster) def test_validate_cluster_status_invalid(self): - cluster_80 = self.env.create_cluster( + dst_cluster = self.env.create_cluster( api=False, - release_id=self.release_80.id, + release_id=self.dst_release.id, ) - relations.UpgradeRelationObject.create_relation(self.cluster_61.id, - cluster_80.id) + relations.UpgradeRelationObject.create_relation(self.src_cluster.id, + dst_cluster.id) msg = "^Upgrade is not possible because of the original cluster " \ "\({0}\) is already involved in the upgrade routine\.$" \ - .format(self.cluster_61.id) + .format(self.src_cluster.id) with self.assertRaisesRegexp(errors.InvalidData, msg): - self.validator.validate_cluster_status(self.cluster_61) + self.validator.validate_cluster_status(self.src_cluster) def test_validate(self): data = jsonutils.dumps(self.data) - self.validator.validate(data, self.cluster_61) + self.validator.validate(data, self.src_cluster) def test_validate_invalid_data(self): data = "{}" with self.assertRaises(errors.InvalidData): - self.validator.validate(data, self.cluster_61) + self.validator.validate(data, self.src_cluster) class TestNodeReassignValidator(base.BaseTestCase): From 405c7db906b72000c95224a365fc8a30be4b56a5 Mon Sep 17 00:00:00 2001 From: Ilya Kharin Date: Sun, 14 Feb 2016 23:15:32 -0600 Subject: [PATCH 14/43] Reassign nodes without reinstallation In some upgrade scenarios when shadow environments are used some of nodes should not be reprovisioned during this procedure. It is useful in combination when control plane nodes are reprovisioned and data plane nodes are updated in place. The update_cluster_assignment method of the objects.Node class was changed to accept roles and pending_roles for node during the reassignment. It allows to specify proper roles by the upgrade extention. The NodeReassignHandler handler accepts two additional parameters in the request body: - reprovision = True (default) - allows to skip the reprovision step - roles = [] (default) - allows to specify new roles or preserve the current roles if empty Two additional methods were added to NailgunClusterAdapter and NailgunReleaseAdapter respectively. Change-Id: Iedb20a904e58f5b9a86eb47de8d8d317dc3cc61b Blueprint: upgrade-major-openstack-environment Closes-Bug: #1558655 --- cluster_upgrade/handlers.py | 31 ++++++++++----- cluster_upgrade/objects/adapters.py | 13 ++++++- cluster_upgrade/tests/test_handlers.py | 46 ++++++++++++++++++++++ cluster_upgrade/tests/test_validators.py | 49 ++++++++++++++++++++++++ cluster_upgrade/upgrade.py | 28 +++++++++++++- cluster_upgrade/validators.py | 22 ++++++++--- 6 files changed, 171 insertions(+), 18 deletions(-) diff --git a/cluster_upgrade/handlers.py b/cluster_upgrade/handlers.py index eb5f64c..21f0bb0 100644 --- a/cluster_upgrade/handlers.py +++ b/cluster_upgrade/handlers.py @@ -69,14 +69,21 @@ class NodeReassignHandler(base.BaseHandler): @base.content def POST(self, cluster_id): - """Reassign node to cluster via reinstallation + """Reassign node to the given cluster. - :param cluster_id: ID of the cluster which node should be - assigned to. - :returns: None - :http: * 202 (OK) - * 400 (Incorrect node state or problem with task execution) - * 404 (Cluster or node not found) + The given node will be assigned from the current cluster to the + given cluster, by default it involves the reprovisioning of this + node. If the 'reprovision' flag is set to False, then the node + will be just reassigned. If the 'roles' list is specified, then + the given roles will be used as 'pending_roles' in case of + the reprovisioning or otherwise as 'roles'. + + :param cluster_id: ID of the cluster node should be assigned to. + :returns: None + :http: * 202 (OK) + * 400 (Incorrect node state, problem with task execution, + conflicting or incorrect roles) + * 404 (Cluster or node not found) """ cluster = adapters.NailgunClusterAdapter( self.get_object_or_404(self.single, cluster_id)) @@ -84,7 +91,13 @@ class NodeReassignHandler(base.BaseHandler): data = self.checked_data(cluster=cluster) node = adapters.NailgunNodeAdapter( self.get_object_or_404(objects.Node, data['node_id'])) + reprovision = data.get('reprovision', True) + given_roles = data.get('roles', []) - upgrade.UpgradeHelper.assign_node_to_cluster(node, cluster) + roles, pending_roles = upgrade.UpgradeHelper.get_node_roles( + reprovision, node.roles, given_roles) + upgrade.UpgradeHelper.assign_node_to_cluster( + node, cluster, roles, pending_roles) - self.handle_task(cluster_id, [node.node, ]) + if reprovision: + self.handle_task(cluster_id, [node.node]) diff --git a/cluster_upgrade/objects/adapters.py b/cluster_upgrade/objects/adapters.py index 6890f8e..a4495a9 100644 --- a/cluster_upgrade/objects/adapters.py +++ b/cluster_upgrade/objects/adapters.py @@ -42,6 +42,10 @@ class NailgunClusterAdapter(object): def release(self): return NailgunReleaseAdapter(self.cluster.release) + @property + def attributes(self): + return self.cluster.attributes + @property def generated_attrs(self): return self.cluster.attributes.generated @@ -100,6 +104,10 @@ class NailgunReleaseAdapter(object): def environment_version(self): return self.release.environment_version + @property + def roles_metadata(self): + return self.release.roles_metadata + def __cmp__(self, other): if isinstance(other, NailgunReleaseAdapter): other = other.release @@ -181,8 +189,9 @@ class NailgunNodeAdapter(object): def roles(self): return self.node.roles - def update_cluster_assignment(self, cluster): - objects.Node.update_cluster_assignment(self.node, cluster) + def update_cluster_assignment(self, cluster, roles, pending_roles): + objects.Node.update_cluster_assignment(self.node, cluster, roles, + pending_roles) def add_pending_change(self, change): objects.Node.add_pending_change(self.node, change) diff --git a/cluster_upgrade/tests/test_handlers.py b/cluster_upgrade/tests/test_handlers.py index d960be5..575f4d3 100644 --- a/cluster_upgrade/tests/test_handlers.py +++ b/cluster_upgrade/tests/test_handlers.py @@ -97,6 +97,52 @@ class TestNodeReassignHandler(base.BaseIntegrationTest): provisioned_uids = [int(n['uid']) for n in nodes] self.assertEqual([node_id, ], provisioned_uids) + @patch('nailgun.task.task.rpc.cast') + def test_node_reassign_handler_with_roles(self, mcast): + cluster = self.env.create( + cluster_kwargs={'api': False}, + nodes_kwargs=[{'status': consts.NODE_STATUSES.ready, + 'roles': ['controller']}]) + node = cluster.nodes[0] + seed_cluster = self.env.create_cluster(api=False) + + # NOTE(akscram): reprovision=True means that the node will be + # re-provisioned during the reassigning. This is + # a default behavior. + data = {'node_id': node.id, + 'reprovision': True, + 'roles': ['compute']} + resp = self.app.post( + reverse('NodeReassignHandler', + kwargs={'cluster_id': seed_cluster.id}), + jsonutils.dumps(data), + headers=self.default_headers) + self.assertEqual(202, resp.status_code) + self.assertEqual(node.roles, []) + self.assertEqual(node.pending_roles, ['compute']) + self.assertTrue(mcast.called) + + @patch('nailgun.task.task.rpc.cast') + def test_node_reassign_handler_without_reprovisioning(self, mcast): + cluster = self.env.create( + cluster_kwargs={'api': False}, + nodes_kwargs=[{'status': consts.NODE_STATUSES.ready, + 'roles': ['controller']}]) + node = cluster.nodes[0] + seed_cluster = self.env.create_cluster(api=False) + + data = {'node_id': node.id, + 'reprovision': False, + 'roles': ['compute']} + resp = self.app.post( + reverse('NodeReassignHandler', + kwargs={'cluster_id': seed_cluster.id}), + jsonutils.dumps(data), + headers=self.default_headers) + self.assertEqual(200, resp.status_code) + self.assertFalse(mcast.called) + self.assertEqual(node.roles, ['compute']) + def test_node_reassign_handler_no_node(self): self.env.create_cluster() diff --git a/cluster_upgrade/tests/test_validators.py b/cluster_upgrade/tests/test_validators.py index ac906f5..9902dca 100644 --- a/cluster_upgrade/tests/test_validators.py +++ b/cluster_upgrade/tests/test_validators.py @@ -141,3 +141,52 @@ class TestNodeReassignValidator(base.BaseTestCase): msg = "^Empty request received$" with self.assertRaisesRegexp(errors.InvalidData, msg): self.validator.validate("", node) + + +class TestNodeReassignNoReinstallValidator(tests_base.BaseCloneClusterTest): + validator = validators.NodeReassignValidator + + def setUp(self): + super(TestNodeReassignNoReinstallValidator, self).setUp() + self.dst_cluster = self.env.create_cluster( + api=False, + release_id=self.dst_release.id, + ) + self.node = self.env.create_node(cluster_id=self.src_cluster.id, + roles=["compute"], status="ready") + + def test_validate_defaults(self): + request = {"node_id": self.node.id} + data = jsonutils.dumps(request) + parsed = self.validator.validate(data, self.dst_cluster) + self.assertEqual(parsed, request) + self.assertEqual(self.node.roles, ['compute']) + + def test_validate_with_roles(self): + request = { + "node_id": self.node.id, + "reprovision": True, + "roles": ['controller'], + } + data = jsonutils.dumps(request) + parsed = self.validator.validate(data, self.dst_cluster) + self.assertEqual(parsed, request) + + def test_validate_not_unique_roles(self): + data = jsonutils.dumps({ + "node_id": self.node.id, + "roles": ['compute', 'compute'], + }) + msg = "has non-unique elements" + with self.assertRaisesRegexp(errors.InvalidData, msg): + self.validator.validate(data, self.dst_cluster) + + def test_validate_no_reprovision_with_conflicts(self): + data = jsonutils.dumps({ + "node_id": self.node.id, + "reprovision": False, + "roles": ['controller', 'compute'], + }) + msg = '^Role "controller" in conflict with role compute$' + with self.assertRaisesRegexp(errors.InvalidData, msg): + self.validator.validate(data, self.dst_cluster) diff --git a/cluster_upgrade/upgrade.py b/cluster_upgrade/upgrade.py index ddd21d3..6278b74 100644 --- a/cluster_upgrade/upgrade.py +++ b/cluster_upgrade/upgrade.py @@ -169,7 +169,31 @@ class UpgradeHelper(object): new_net_manager.assign_vips_for_net_groups() @classmethod - def assign_node_to_cluster(cls, node, seed_cluster): + def get_node_roles(cls, reprovision, current_roles, given_roles): + """Return roles depending on the reprovisioning status. + + In case the node should be re-provisioned, only pending roles + should be set, otherwise for an already provisioned and deployed + node only actual roles should be set. In the both case the + given roles will have precedence over the existing. + + :param reprovision: boolean, if set to True then the node should + be re-provisioned + :param current_roles: a list of current roles of the node + :param given_roles: a list of roles that should be assigned to + the node + :returns: a tuple of a list of roles and a list of pending roles + that will be assigned to the node + """ + roles_to_assign = given_roles if given_roles else current_roles + if reprovision: + roles, pending_roles = [], roles_to_assign + else: + roles, pending_roles = roles_to_assign, [] + return roles, pending_roles + + @classmethod + def assign_node_to_cluster(cls, node, seed_cluster, roles, pending_roles): orig_cluster = adapters.NailgunClusterAdapter.get_by_uid( node.cluster_id) @@ -178,7 +202,7 @@ class UpgradeHelper(object): netgroups_id_mapping = cls.get_netgroups_id_mapping( orig_cluster, seed_cluster) - node.update_cluster_assignment(seed_cluster) + node.update_cluster_assignment(seed_cluster, roles, pending_roles) objects.Node.set_netgroups_ids(node, netgroups_id_mapping) orig_manager.set_nic_assignment_netgroups_ids( node, netgroups_id_mapping) diff --git a/cluster_upgrade/validators.py b/cluster_upgrade/validators.py index 5a6a735..b29dee8 100644 --- a/cluster_upgrade/validators.py +++ b/cluster_upgrade/validators.py @@ -14,6 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. +from nailgun.api.v1.validators import assignment from nailgun.api.v1.validators import base from nailgun import consts from nailgun.errors import errors @@ -84,7 +85,7 @@ class ClusterUpgradeValidator(base.BasicValidator): log_message=True) -class NodeReassignValidator(base.BasicValidator): +class NodeReassignValidator(assignment.NodeAssignmentValidator): schema = { "$schema": "http://json-schema.org/draft-04/schema#", "title": "Assign Node Parameters", @@ -92,17 +93,28 @@ class NodeReassignValidator(base.BasicValidator): "type": "object", "properties": { "node_id": {"type": "number"}, + "reprovision": {"type": "boolean", "default": True}, + "roles": {"type": "array", + "items": {"type": "string"}, + "uniqueItems": True}, }, "required": ["node_id"], } @classmethod def validate(cls, data, cluster): - data = super(NodeReassignValidator, cls).validate(data) - cls.validate_schema(data, cls.schema) - node = cls.validate_node(data['node_id']) + parsed = super(NodeReassignValidator, cls).validate(data) + cls.validate_schema(parsed, cls.schema) + + node = cls.validate_node(parsed['node_id']) cls.validate_node_cluster(node, cluster) - return data + + roles = parsed.get('roles', []) + if roles: + cls.validate_roles(cluster, roles) + else: + cls.validate_roles(cluster, node.roles) + return parsed @classmethod def validate_node(cls, node_id): From ecec1f6d3c68e58edc28616d1f7547119b019426 Mon Sep 17 00:00:00 2001 From: Alexander Kislitsky Date: Tue, 22 Mar 2016 09:31:58 +0300 Subject: [PATCH 15/43] Performance of network manager operation improve We have extra SQLs generated in the NetworkManager when passing node_id instead already loaded SQLAlchemy node object. Additional changes: - Bulk insert used in IPs assiging process. - zip changed on six.moves.zip in the NetworkManager. - Removed unused function get_admin_ips_for_interfaces from NetworkManager. Co-Authored-By: Dmitry Guryanov Partial-Bug: #1498365 Change-Id: I0518a5879c775d568de5652dbdd856a0cede80ce --- cluster_upgrade/objects/adapters.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cluster_upgrade/objects/adapters.py b/cluster_upgrade/objects/adapters.py index a4495a9..7a2854e 100644 --- a/cluster_upgrade/objects/adapters.py +++ b/cluster_upgrade/objects/adapters.py @@ -131,10 +131,6 @@ class NailgunNetworkManager(object): def assign_given_vips_for_net_groups(self, vips): self.net_manager.assign_given_vips_for_net_groups(self.cluster, vips) - def get_admin_network_group(self, node_id=None): - ng = objects.NetworkGroup.get_admin_network_group(node_id) - return NailgunNetworkGroupAdapter(ng) - def set_node_netgroups_ids(self, node, mapping): return objects.Node.set_netgroups_ids(node.node, mapping) From c0876c580e4a9138c7bfe420c459a4dafb9bd4c3 Mon Sep 17 00:00:00 2001 From: Artem Roma Date: Tue, 1 Mar 2016 15:38:37 +0200 Subject: [PATCH 16/43] Add handler for copying of VIPs New handler triggers copying of VIPs from given original cluster to new one. The reason for separation of this action from copying of network configuration is that accordion to [1] copying of existing VIPs/allocation of new ones will not take effect for new cluster unless it has assigned nodes. Thus in context of cluster upgrade procedure VIP transfer must be done after node reassignment, and as long as nodes are being operated on one by one it would be not efficient to call VIP copying method after each such reassignment. Tests updated accordingly. [1]: https://review.openstack.org/#/c/284841/ Change-Id: I33670e8f2561be6fe18cec75bfc7ecc056ae2f6b Closes-Bug: #1552744 --- cluster_upgrade/extension.py | 2 + cluster_upgrade/handlers.py | 38 ++++++++++++ cluster_upgrade/tests/base.py | 14 +++-- cluster_upgrade/tests/test_handlers.py | 33 +++++++++-- cluster_upgrade/tests/test_upgrade.py | 73 ++++++++++++++++++------ cluster_upgrade/tests/test_validators.py | 34 +++++++++++ cluster_upgrade/upgrade.py | 7 ++- cluster_upgrade/validators.py | 15 +++++ 8 files changed, 188 insertions(+), 28 deletions(-) diff --git a/cluster_upgrade/extension.py b/cluster_upgrade/extension.py index f484fa9..66df106 100644 --- a/cluster_upgrade/extension.py +++ b/cluster_upgrade/extension.py @@ -31,6 +31,8 @@ class ClusterUpgradeExtension(extensions.BaseExtension): 'handler': handlers.ClusterUpgradeCloneHandler}, {'uri': r'/clusters/(?P\d+)/upgrade/assign/?$', 'handler': handlers.NodeReassignHandler}, + {'uri': r'/clusters/(?P\d+)/upgrade/vips/?$', + 'handler': handlers.CopyVIPsHandler}, ] @classmethod diff --git a/cluster_upgrade/handlers.py b/cluster_upgrade/handlers.py index 21f0bb0..4f61f2e 100644 --- a/cluster_upgrade/handlers.py +++ b/cluster_upgrade/handlers.py @@ -101,3 +101,41 @@ class NodeReassignHandler(base.BaseHandler): if reprovision: self.handle_task(cluster_id, [node.node]) + + +class CopyVIPsHandler(base.BaseHandler): + single = objects.Cluster + validator = validators.CopyVIPsValidator + + @base.content + def POST(self, cluster_id): + """Copy VIPs from original cluster to new one + + Original cluster object is obtained from existing relation between + clusters that is created on cluster clone operation + + :param cluster_id: id of cluster that VIPs must be copied to + + :http: * 200 (OK) + * 400 (validation failed) + * 404 (seed cluster is not found) + """ + from .objects import relations + + cluster = self.get_object_or_404(self.single, cluster_id) + relation = relations.UpgradeRelationObject.get_cluster_relation( + cluster.id) + + self.checked_data(cluster=cluster, relation=relation) + + # get original cluster object and create adapter with it + orig_cluster_adapter = \ + adapters.NailgunClusterAdapter( + adapters.NailgunClusterAdapter.get_by_uid( + relation.orig_cluster_id) + ) + + seed_cluster_adapter = adapters.NailgunClusterAdapter(cluster) + + upgrade.UpgradeHelper.copy_vips(orig_cluster_adapter, + seed_cluster_adapter) diff --git a/cluster_upgrade/tests/base.py b/cluster_upgrade/tests/base.py index 8d7ce10..497656f 100644 --- a/cluster_upgrade/tests/base.py +++ b/cluster_upgrade/tests/base.py @@ -37,14 +37,18 @@ class BaseCloneClusterTest(nailgun_test_base.BaseIntegrationTest): version="liberty-9.0", ) - self.src_cluster_db = self.env.create_cluster( - api=False, - release_id=self.src_release.id, - net_provider=consts.CLUSTER_NET_PROVIDERS.neutron, - net_l23_provider=consts.NEUTRON_L23_PROVIDERS.ovs, + self.src_cluster_db = self.env.create( + cluster_kwargs={ + 'api': False, + 'release_id': self.src_release.id, + 'net_provider': consts.CLUSTER_NET_PROVIDERS.neutron, + 'net_l23_provider': consts.NEUTRON_L23_PROVIDERS.ovs, + }, + nodes_kwargs=[{'roles': ['controller']}] ) self.src_cluster = adapters.NailgunClusterAdapter( self.src_cluster_db) + self.data = { "name": "cluster-clone-{0}".format(self.src_cluster.id), "release_id": self.dst_release.id, diff --git a/cluster_upgrade/tests/test_handlers.py b/cluster_upgrade/tests/test_handlers.py index 575f4d3..127172d 100644 --- a/cluster_upgrade/tests/test_handlers.py +++ b/cluster_upgrade/tests/test_handlers.py @@ -14,7 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -from mock import patch +import mock from oslo_serialization import jsonutils @@ -75,7 +75,7 @@ class TestClusterUpgradeCloneHandler(tests_base.BaseCloneClusterTest): class TestNodeReassignHandler(base.BaseIntegrationTest): - @patch('nailgun.task.task.rpc.cast') + @mock.patch('nailgun.task.task.rpc.cast') def test_node_reassign_handler(self, mcast): self.env.create( cluster_kwargs={'api': False}, @@ -97,7 +97,7 @@ class TestNodeReassignHandler(base.BaseIntegrationTest): provisioned_uids = [int(n['uid']) for n in nodes] self.assertEqual([node_id, ], provisioned_uids) - @patch('nailgun.task.task.rpc.cast') + @mock.patch('nailgun.task.task.rpc.cast') def test_node_reassign_handler_with_roles(self, mcast): cluster = self.env.create( cluster_kwargs={'api': False}, @@ -122,7 +122,7 @@ class TestNodeReassignHandler(base.BaseIntegrationTest): self.assertEqual(node.pending_roles, ['compute']) self.assertTrue(mcast.called) - @patch('nailgun.task.task.rpc.cast') + @mock.patch('nailgun.task.task.rpc.cast') def test_node_reassign_handler_without_reprovisioning(self, mcast): cluster = self.env.create( cluster_kwargs={'api': False}, @@ -229,3 +229,28 @@ class TestNodeReassignHandler(base.BaseIntegrationTest): headers=self.default_headers, expect_errors=True) self.assertEqual(400, resp.status_code) + + +class TestCopyVipsHandler(base.BaseIntegrationTest): + + def test_copy_vips_called(self): + from ..objects import relations + + orig_cluster = self.env.create_cluster(api=False) + new_cluster = self.env.create_cluster(api=False) + + relations.UpgradeRelationObject.create_relation( + orig_cluster.id, new_cluster.id) + + with mock.patch('nailgun.extensions.cluster_upgrade.handlers' + '.upgrade.UpgradeHelper.copy_vips') as copy_vips_mc: + resp = self.app.post( + reverse( + 'CopyVIPsHandler', + kwargs={'cluster_id': new_cluster.id} + ), + headers=self.default_headers, + ) + + self.assertEqual(resp.status_code, 200) + self.assertTrue(copy_vips_mc.called) diff --git a/cluster_upgrade/tests/test_upgrade.py b/cluster_upgrade/tests/test_upgrade.py index 723f0ee..dac04b5 100644 --- a/cluster_upgrade/tests/test_upgrade.py +++ b/cluster_upgrade/tests/test_upgrade.py @@ -17,13 +17,31 @@ import copy import six +from nailgun import consts from nailgun.objects.serializers import network_configuration from . import base as base_tests +from ..objects import adapters from ..objects import relations class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest): + + def setUp(self): + super(TestUpgradeHelperCloneCluster, self).setUp() + + self.orig_net_manager = self.src_cluster.get_network_manager() + + self.serialize_nets = network_configuration.\ + NeutronNetworkConfigurationSerializer.\ + serialize_for_cluster + + self.public_net_data = { + "cidr": "192.168.42.0/24", + "gateway": "192.168.42.1", + "ip_ranges": [["192.168.42.5", "192.168.42.11"]], + } + def test_create_cluster_clone(self): new_cluster = self.helper.create_cluster_clone(self.src_cluster, self.data) @@ -60,28 +78,48 @@ class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest): self.assertEqual(editable_attrs[section][key]["value"], value["value"]) + def update_public_net_params(self, networks): + pub_net = self._get_pub_net(networks) + pub_net.update(self.public_net_data) + self.orig_net_manager.update(networks) + + def _get_pub_net(self, networks): + return next(net for net in networks['networks'] if + net['name'] == consts.NETWORKS.public) + def test_copy_network_config(self): new_cluster = self.helper.create_cluster_clone(self.src_cluster, self.data) - orig_net_manager = self.src_cluster.get_network_manager() - serialize_nets = network_configuration.\ - NeutronNetworkConfigurationSerializer.\ - serialize_for_cluster - - # Do some unordinary changes - nets = serialize_nets(self.src_cluster.cluster) - nets["networks"][0].update({ - "cidr": "172.16.42.0/24", - "gateway": "172.16.42.1", - "ip_ranges": [["172.16.42.2", "172.16.42.126"]], - }) - orig_net_manager.update(nets) - orig_net_manager.assign_vips_for_net_groups() + # Do some unordinary changes to public network + nets = self.serialize_nets(self.src_cluster.cluster) + self.update_public_net_params(nets) self.helper.copy_network_config(self.src_cluster, new_cluster) - orig_nets = serialize_nets(self.src_cluster_db) - new_nets = serialize_nets(new_cluster.cluster) + new_nets = self.serialize_nets(new_cluster.cluster) + + public_net = self._get_pub_net(new_nets) + + self.assertEqual(public_net['cidr'], self.public_net_data['cidr']) + self.assertEqual(public_net['gateway'], + self.public_net_data['gateway']) + self.assertEqual(public_net['ip_ranges'], + self.public_net_data['ip_ranges']) + + def test_copy_vips(self): + new_cluster = self.helper.clone_cluster(self.src_cluster, self.data) + + # we have to move node to new cluster before VIP assignment + # because there is no point in the operation for a cluster + # w/o nodes + node = adapters.NailgunNodeAdapter(self.src_cluster.cluster.nodes[0]) + self.helper.assign_node_to_cluster(node, new_cluster, node.roles, []) + + self.helper.copy_vips(self.src_cluster, new_cluster) + + orig_nets = self.serialize_nets(self.src_cluster.cluster) + new_nets = self.serialize_nets(new_cluster.cluster) + self.assertEqual(orig_nets["management_vip"], new_nets["management_vip"]) self.assertEqual(orig_nets["management_vrouter_vip"], @@ -92,8 +130,7 @@ class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest): new_nets["public_vrouter_vip"]) def test_clone_cluster(self): - orig_net_manager = self.src_cluster.get_network_manager() - orig_net_manager.assign_vips_for_net_groups() + self.orig_net_manager.assign_vips_for_net_groups() new_cluster = self.helper.clone_cluster(self.src_cluster, self.data) relation = relations.UpgradeRelationObject.get_cluster_relation( self.src_cluster.id) diff --git a/cluster_upgrade/tests/test_validators.py b/cluster_upgrade/tests/test_validators.py index 9902dca..950ef21 100644 --- a/cluster_upgrade/tests/test_validators.py +++ b/cluster_upgrade/tests/test_validators.py @@ -190,3 +190,37 @@ class TestNodeReassignNoReinstallValidator(tests_base.BaseCloneClusterTest): msg = '^Role "controller" in conflict with role compute$' with self.assertRaisesRegexp(errors.InvalidData, msg): self.validator.validate(data, self.dst_cluster) + + +class TestCopyVIPsValidator(base.BaseTestCase): + validator = validators.CopyVIPsValidator + + def test_non_existing_relation_fail(self): + with self.assertRaises(errors.InvalidData) as cm: + self.validator.validate(data=None, cluster=None, relation=None) + + self.assertEqual( + cm.exception.message, + "Relation for given cluster does not exist" + ) + + def test_cluster_is_not_seed(self): + cluster = self.env.create_cluster(api=False) + seed_cluster = self.env.create_cluster(api=False) + + relations.UpgradeRelationObject.create_relation( + orig_cluster_id=cluster.id, + seed_cluster_id=cluster.id, + ) + + relation = relations.UpgradeRelationObject.get_cluster_relation( + cluster.id) + + with self.assertRaises(errors.InvalidData) as cm: + self.validator.validate(data=None, cluster=seed_cluster, + relation=relation) + + self.assertEqual( + cm.exception.message, + "Given cluster is not seed cluster" + ) diff --git a/cluster_upgrade/upgrade.py b/cluster_upgrade/upgrade.py index 6278b74..b84e366 100644 --- a/cluster_upgrade/upgrade.py +++ b/cluster_upgrade/upgrade.py @@ -150,10 +150,15 @@ class UpgradeHelper(object): nets_serializer.serialize_for_cluster(orig_cluster.cluster), nets_serializer.serialize_for_cluster(new_cluster.cluster)) - orig_net_manager = orig_cluster.get_network_manager() new_net_manager = new_cluster.get_network_manager() new_net_manager.update(nets) + + @classmethod + def copy_vips(cls, orig_cluster, new_cluster): + orig_net_manager = orig_cluster.get_network_manager() + new_net_manager = new_cluster.get_network_manager() + vips = orig_net_manager.get_assigned_vips() for ng_name in vips: if ng_name not in (consts.NETWORKS.public, diff --git a/cluster_upgrade/validators.py b/cluster_upgrade/validators.py index b29dee8..3874147 100644 --- a/cluster_upgrade/validators.py +++ b/cluster_upgrade/validators.py @@ -147,3 +147,18 @@ class NodeReassignValidator(assignment.NodeAssignmentValidator): raise errors.InvalidData("Node {0} is already assigned to cluster" " {1}".format(node.id, cluster.id), log_message=True) + + +class CopyVIPsValidator(base.BasicValidator): + + @classmethod + def validate(cls, data, cluster, relation): + if relation is None: + raise errors.InvalidData( + "Relation for given cluster does not exist" + ) + + if cluster.id != relation.seed_cluster_id: + raise errors.InvalidData("Given cluster is not seed cluster") + + return data From 6db717cff27edce20f1727e83b8f30b2c7f05676 Mon Sep 17 00:00:00 2001 From: Artem Roma Date: Thu, 25 Feb 2016 19:45:50 +0200 Subject: [PATCH 17/43] Fix get_common_node_group behavior Now if common nodegroup has not been found for given nodes None as a result is returned instead of default node group. This directly affects procedure of VIP allocation as (e.g.) VIPs will not be allocated when cluster does not have assigned nodes. Tests updated accordingly Change-Id: Iaa94453f3f98cc9238f9810aab7311ffabbfa8b7 Closes-Bug: #1549254 --- cluster_upgrade/tests/test_upgrade.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cluster_upgrade/tests/test_upgrade.py b/cluster_upgrade/tests/test_upgrade.py index dac04b5..7e1dee2 100644 --- a/cluster_upgrade/tests/test_upgrade.py +++ b/cluster_upgrade/tests/test_upgrade.py @@ -107,6 +107,11 @@ class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest): self.public_net_data['ip_ranges']) def test_copy_vips(self): + # save network information before node reassignment to seed cluster + # as after that no VIP will be allocated/serialized due to + # absence of assigned nodes for the source cluster + orig_nets = self.serialize_nets(self.src_cluster.cluster) + new_cluster = self.helper.clone_cluster(self.src_cluster, self.data) # we have to move node to new cluster before VIP assignment @@ -117,7 +122,6 @@ class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest): self.helper.copy_vips(self.src_cluster, new_cluster) - orig_nets = self.serialize_nets(self.src_cluster.cluster) new_nets = self.serialize_nets(new_cluster.cluster) self.assertEqual(orig_nets["management_vip"], From 8f29bf537d9b5006c9532a70a2b7e0a58fc66442 Mon Sep 17 00:00:00 2001 From: Oleg Gelbukh Date: Wed, 9 Mar 2016 18:42:21 +0000 Subject: [PATCH 18/43] Properly iterate through VIPs In the original form, the iterator was changed inside the loop. Iterating through separate list object fixes this issue. Change-Id: I8c2a47b54f1339669f9f20f1a420a028044cb9d8 Closes-bug: 1539039 --- cluster_upgrade/upgrade.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cluster_upgrade/upgrade.py b/cluster_upgrade/upgrade.py index b84e366..3089d82 100644 --- a/cluster_upgrade/upgrade.py +++ b/cluster_upgrade/upgrade.py @@ -159,11 +159,10 @@ class UpgradeHelper(object): orig_net_manager = orig_cluster.get_network_manager() new_net_manager = new_cluster.get_network_manager() - vips = orig_net_manager.get_assigned_vips() - for ng_name in vips: - if ng_name not in (consts.NETWORKS.public, - consts.NETWORKS.management): - vips.pop(ng_name) + vips = {} + assigned_vips = orig_net_manager.get_assigned_vips() + for ng_name in (consts.NETWORKS.public, consts.NETWORKS.management): + vips[ng_name] = assigned_vips[ng_name] # NOTE(akscram): In the 7.0 release was introduced networking # templates that use the vip_name column as # unique names of VIPs. From 99d9afbe2befd97a5fde70eca5eb9ceeafb7f376 Mon Sep 17 00:00:00 2001 From: Ryan Moe Date: Mon, 4 Apr 2016 14:29:58 -0700 Subject: [PATCH 19/43] Move network objects to extension All network-related objects have been moved into the network_manager extension and import paths have been updated. Blueprint: network-manager-extension Change-Id: I6e16df86a58d6192d312e8e8955ed38912d2b059 --- cluster_upgrade/tests/test_upgrade.py | 3 ++- cluster_upgrade/upgrade.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cluster_upgrade/tests/test_upgrade.py b/cluster_upgrade/tests/test_upgrade.py index 7e1dee2..95bf48b 100644 --- a/cluster_upgrade/tests/test_upgrade.py +++ b/cluster_upgrade/tests/test_upgrade.py @@ -18,7 +18,8 @@ import copy import six from nailgun import consts -from nailgun.objects.serializers import network_configuration +from nailgun.extensions.network_manager.objects.serializers import \ + network_configuration from . import base as base_tests from ..objects import adapters diff --git a/cluster_upgrade/upgrade.py b/cluster_upgrade/upgrade.py index 3089d82..fd2917c 100644 --- a/cluster_upgrade/upgrade.py +++ b/cluster_upgrade/upgrade.py @@ -20,8 +20,9 @@ from distutils import version import six from nailgun import consts +from nailgun.extensions.network_manager.objects.serializers import \ + network_configuration from nailgun import objects -from nailgun.objects.serializers import network_configuration from nailgun import utils from .objects import adapters From 5bbaf23795c645b172753ae85011a4669f9dc828 Mon Sep 17 00:00:00 2001 From: Nikita Zubkov Date: Tue, 5 Apr 2016 15:31:37 +0300 Subject: [PATCH 20/43] Add exceptions hierarchy nailgun.errors have a huge set of exceptions but without hierarchy. This patch remove exception generation from dict and make it explicitly with python classes and add some exceptions hierarchy. Now all network errors inherit from NetworkException and same for other exceptions. Change-Id: I9a2c6b358ea02a16711da74562308664ad7aed97 Closes-bug: #1566195 --- cluster_upgrade/tests/test_validators.py | 2 +- cluster_upgrade/validators.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster_upgrade/tests/test_validators.py b/cluster_upgrade/tests/test_validators.py index 950ef21..8af57c1 100644 --- a/cluster_upgrade/tests/test_validators.py +++ b/cluster_upgrade/tests/test_validators.py @@ -19,7 +19,7 @@ import mock from oslo_serialization import jsonutils from nailgun import consts -from nailgun.errors import errors +from nailgun import errors from nailgun.settings import settings from nailgun.test import base diff --git a/cluster_upgrade/validators.py b/cluster_upgrade/validators.py index 3874147..0dcd286 100644 --- a/cluster_upgrade/validators.py +++ b/cluster_upgrade/validators.py @@ -17,7 +17,7 @@ from nailgun.api.v1.validators import assignment from nailgun.api.v1.validators import base from nailgun import consts -from nailgun.errors import errors +from nailgun import errors from nailgun import objects from .objects import adapters From e7230f5824315e964bca3cdc214ca899a99f7c5e Mon Sep 17 00:00:00 2001 From: Dmitry Guryanov Date: Tue, 19 Apr 2016 18:07:31 +0300 Subject: [PATCH 21/43] Don't use self.env.clusters[0] in tests where possible Since self.env.create always return db object now, we can use this returned value instead of self.env.clusters list. It's a refactoring, so no bug or blueprint. Change-Id: If7c84cb7124bcf08ef5ff110542012564190fae1 --- cluster_upgrade/tests/test_handlers.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/cluster_upgrade/tests/test_handlers.py b/cluster_upgrade/tests/test_handlers.py index 127172d..cc82acb 100644 --- a/cluster_upgrade/tests/test_handlers.py +++ b/cluster_upgrade/tests/test_handlers.py @@ -77,12 +77,10 @@ class TestNodeReassignHandler(base.BaseIntegrationTest): @mock.patch('nailgun.task.task.rpc.cast') def test_node_reassign_handler(self, mcast): - self.env.create( + cluster = self.env.create( cluster_kwargs={'api': False}, nodes_kwargs=[{'status': consts.NODE_STATUSES.ready}]) - self.env.create_cluster() - cluster = self.env.clusters[0] - seed_cluster = self.env.clusters[1] + seed_cluster = self.env.create_cluster() node_id = cluster.nodes[0]['id'] resp = self.app.post( @@ -144,9 +142,7 @@ class TestNodeReassignHandler(base.BaseIntegrationTest): self.assertEqual(node.roles, ['compute']) def test_node_reassign_handler_no_node(self): - self.env.create_cluster() - - cluster = self.env.clusters[0] + cluster = self.env.create_cluster() resp = self.app.post( reverse('NodeReassignHandler', @@ -159,10 +155,9 @@ class TestNodeReassignHandler(base.BaseIntegrationTest): resp.json_body['message']) def test_node_reassing_handler_wrong_status(self): - self.env.create( + cluster = self.env.create( cluster_kwargs={'api': False}, nodes_kwargs=[{'status': 'discover'}]) - cluster = self.env.clusters[0] resp = self.app.post( reverse('NodeReassignHandler', @@ -175,11 +170,10 @@ class TestNodeReassignHandler(base.BaseIntegrationTest): "^Node should be in one of statuses:") def test_node_reassing_handler_wrong_error_type(self): - self.env.create( + cluster = self.env.create( cluster_kwargs={'api': False}, nodes_kwargs=[{'status': 'error', 'error_type': 'provision'}]) - cluster = self.env.clusters[0] resp = self.app.post( reverse('NodeReassignHandler', @@ -192,10 +186,9 @@ class TestNodeReassignHandler(base.BaseIntegrationTest): "^Node should be in error state") def test_node_reassign_handler_to_the_same_cluster(self): - self.env.create( + cluster = self.env.create( cluster_kwargs={'api': False}, nodes_kwargs=[{'status': 'ready'}]) - cluster = self.env.clusters[0] cluster_id = cluster['id'] node_id = cluster.nodes[0]['id'] From 93d7fb982b994e8418288b8e98b35d6f0c9b25d1 Mon Sep 17 00:00:00 2001 From: Artur Svechnikov Date: Wed, 4 May 2016 13:29:17 +0300 Subject: [PATCH 22/43] Check nodes roles before deployment Nodes roles should be checked in CheckBeforeDeploymentTask, because it's possible to deploy node with conflicting roles or with incompatible role. Roles release metadata will be used for roles checks, this metadata contains restrictions. Since `depends` is not used anymore, it's changed to `restrictions` in assignment validator. Change-Id: Ibba7951968cbafd59fff0d516e74f9dd9e454edc Closes-Bug: #1573006 --- cluster_upgrade/tests/test_validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster_upgrade/tests/test_validators.py b/cluster_upgrade/tests/test_validators.py index 8af57c1..122fbbc 100644 --- a/cluster_upgrade/tests/test_validators.py +++ b/cluster_upgrade/tests/test_validators.py @@ -187,7 +187,7 @@ class TestNodeReassignNoReinstallValidator(tests_base.BaseCloneClusterTest): "reprovision": False, "roles": ['controller', 'compute'], }) - msg = '^Role "controller" in conflict with role compute$' + msg = "^Role 'controller' in conflict with role 'compute'.$" with self.assertRaisesRegexp(errors.InvalidData, msg): self.validator.validate(data, self.dst_cluster) From 963272a0d577d505195c9b0f5cede75c22d706d5 Mon Sep 17 00:00:00 2001 From: Artur Svechnikov Date: Wed, 11 May 2016 12:43:23 +0300 Subject: [PATCH 23/43] Move models for restrictions to cluster object Since there are a couple of places where models for restrictions is initialized, it's moved to cluster object. Also, comments from previous commit (Ibba7951968cbafd59fff0d516e74f9dd9e454edc) are fixed It's refactoring bug is not needed. Change-Id: Ic499a5deefb12740ebedc630b024dae0b4248ec5 --- cluster_upgrade/tests/test_validators.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cluster_upgrade/tests/test_validators.py b/cluster_upgrade/tests/test_validators.py index 122fbbc..056c1cb 100644 --- a/cluster_upgrade/tests/test_validators.py +++ b/cluster_upgrade/tests/test_validators.py @@ -187,10 +187,14 @@ class TestNodeReassignNoReinstallValidator(tests_base.BaseCloneClusterTest): "reprovision": False, "roles": ['controller', 'compute'], }) - msg = "^Role 'controller' in conflict with role 'compute'.$" - with self.assertRaisesRegexp(errors.InvalidData, msg): + with self.assertRaises(errors.InvalidData) as exc: self.validator.validate(data, self.dst_cluster) + self.assertEqual( + exc.exception.message, + "Role 'controller' in conflict with role 'compute'." + ) + class TestCopyVIPsValidator(base.BaseTestCase): validator = validators.CopyVIPsValidator From f4b285d098baee9cf55a1ee05f9c32cc17eeac2d Mon Sep 17 00:00:00 2001 From: Sergey Abramov Date: Fri, 13 May 2016 17:53:43 +0300 Subject: [PATCH 24/43] Fix cluster attributes dns_list and ntp_list Settings in release change. Cloned cluster should have values valid for its release. dns_list and net_list were text values but in version mitaka-9.0 it changed to text_list Change-Id: Iac0aa42b7c36333e6d9c40b8a27a19df9efe36f5 Closes-Bug: 1572179 --- cluster_upgrade/tests/test_upgrade.py | 40 +++++++++++++++++++++++++++ cluster_upgrade/upgrade.py | 7 +++++ 2 files changed, 47 insertions(+) diff --git a/cluster_upgrade/tests/test_upgrade.py b/cluster_upgrade/tests/test_upgrade.py index 95bf48b..f7926f3 100644 --- a/cluster_upgrade/tests/test_upgrade.py +++ b/cluster_upgrade/tests/test_upgrade.py @@ -141,3 +141,43 @@ class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest): self.src_cluster.id) self.assertEqual(relation.orig_cluster_id, self.src_cluster.id) self.assertEqual(relation.seed_cluster_id, new_cluster.id) + + def _check_dns_and_ntp_list_values(self, new_cluster, dns_list, ntp_list): + self.assertEqual( + new_cluster.editable_attrs["external_ntp"]["ntp_list"]["value"], + ntp_list) + self.assertEqual( + new_cluster.editable_attrs["external_dns"]["dns_list"]["value"], + dns_list) + self.assertEqual( + new_cluster.editable_attrs["external_ntp"]["ntp_list"]["type"], + "text_list") + self.assertEqual( + new_cluster.editable_attrs["external_dns"]["dns_list"]["type"], + "text_list") + + def test_cluster_copy_attrs_with_different_types_dns_and_ntp_lists(self): + attrs = copy.deepcopy(self.src_cluster.editable_attrs) + attrs["external_ntp"]["ntp_list"]["type"] = "text" + attrs["external_ntp"]["ntp_list"]["value"] = "1,2,3" + attrs["external_dns"]["dns_list"]["type"] = "text" + attrs["external_dns"]["dns_list"]["value"] = "4,5,6" + self.src_cluster.editable_attrs = attrs + new_cluster = self.helper.create_cluster_clone( + self.src_cluster, self.data) + self.helper.copy_attributes(self.src_cluster, new_cluster) + self._check_dns_and_ntp_list_values( + new_cluster, ["4", "5", "6"], ["1", "2", "3"]) + + def test_cluster_copy_attrs_with_same_types_dns_and_ntp_lists(self): + attrs = copy.deepcopy(self.src_cluster.editable_attrs) + attrs["external_ntp"]["ntp_list"]["type"] = "text_list" + attrs["external_ntp"]["ntp_list"]["value"] = ["1", "2", "3"] + attrs["external_dns"]["dns_list"]["type"] = "text_list" + attrs["external_dns"]["dns_list"]["value"] = ["4", "5", "6"] + self.src_cluster.editable_attrs = attrs + new_cluster = self.helper.create_cluster_clone( + self.src_cluster, self.data) + self.helper.copy_attributes(self.src_cluster, new_cluster) + self._check_dns_and_ntp_list_values( + new_cluster, ["4", "5", "6"], ["1", "2", "3"]) diff --git a/cluster_upgrade/upgrade.py b/cluster_upgrade/upgrade.py index fd2917c..ada4c40 100644 --- a/cluster_upgrade/upgrade.py +++ b/cluster_upgrade/upgrade.py @@ -42,6 +42,13 @@ def merge_attributes(a, b): for key, values in six.iteritems(pairs): if key != "metadata" and key in a_values: values["value"] = a_values[key]["value"] + # NOTE: In the mitaka-9.0 release types of values dns_list and + # ntp_list were changed from 'text' + # (a string of comma-separated IP-addresses) + # to 'text_list' (a list of strings of IP-addresses). + if a_values[key]['type'] == 'text' and \ + values['type'] == 'text_list': + values["value"] = values['value'].split(',') return attrs From 71148973619caea267a548d5924edecec29650b6 Mon Sep 17 00:00:00 2001 From: Nikita Zubkov Date: Wed, 13 Jul 2016 13:43:16 +0300 Subject: [PATCH 25/43] Add nailgun extension entrypoint to setup.cfg --- setup.cfg | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.cfg b/setup.cfg index 348ece1..3d6beb9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,3 +20,7 @@ classifier = [files] packages = cluster_upgrade + +[entry_points] +nailgun.extensions = + cluster_upgrade = cluster_upgrade.extension:ClusterUpgradeExtension From e46a649e87a64548c826ce1332f8234f44b60ced Mon Sep 17 00:00:00 2001 From: Nikita Zubkov Date: Wed, 13 Jul 2016 13:43:58 +0300 Subject: [PATCH 26/43] Fix package namespace --- cluster_upgrade/tests/__init__.py | 2 +- cluster_upgrade/tests/test_handlers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster_upgrade/tests/__init__.py b/cluster_upgrade/tests/__init__.py index 86c153e..6e70254 100644 --- a/cluster_upgrade/tests/__init__.py +++ b/cluster_upgrade/tests/__init__.py @@ -14,4 +14,4 @@ # License for the specific language governing permissions and limitations # under the License. -EXTENSION = "nailgun.extensions.cluster_upgrade." +EXTENSION = "cluster_upgrade." diff --git a/cluster_upgrade/tests/test_handlers.py b/cluster_upgrade/tests/test_handlers.py index cc82acb..ad7239a 100644 --- a/cluster_upgrade/tests/test_handlers.py +++ b/cluster_upgrade/tests/test_handlers.py @@ -235,7 +235,7 @@ class TestCopyVipsHandler(base.BaseIntegrationTest): relations.UpgradeRelationObject.create_relation( orig_cluster.id, new_cluster.id) - with mock.patch('nailgun.extensions.cluster_upgrade.handlers' + with mock.patch('cluster_upgrade.handlers' '.upgrade.UpgradeHelper.copy_vips') as copy_vips_mc: resp = self.app.post( reverse( From 035a6f7baa5bc4b80b4c065e3904a81943689460 Mon Sep 17 00:00:00 2001 From: Nikita Zubkov Date: Tue, 19 Jul 2016 18:24:42 +0300 Subject: [PATCH 27/43] Switch to upstream fuel-web repository Change-Id: I994304bdc8eaf7e4da175981cb721d41a286fed0 Depends-On: Id0bc78478cf3f40767fed760cd54e487a934fa10 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index fd8d423..631a27b 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ envlist = pep8,py27 skipsdist = True [base] -NAILGUN_REPO = git+https://github.com/zubchick/fuel-web.git +NAILGUN_REPO = git+https://github.com/openstack/fuel-web.git NAILGUN_CONFIG = {toxinidir}/nailgun-test-settings.yaml NAILGUN_BRANCH={env:ZUUL_BRANCH:master} From 651a269e1f4ba4ab4ce0213d44d09d1fc0c8c1bc Mon Sep 17 00:00:00 2001 From: Nikita Zubkov Date: Tue, 19 Jul 2016 17:04:32 +0300 Subject: [PATCH 28/43] Add README Change-Id: I7c875d89f0423c26f28f1a6debf947b5510988fd --- README.rst | 5 +++++ setup.cfg | 1 + 2 files changed, 6 insertions(+) create mode 100644 README.rst diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..8991794 --- /dev/null +++ b/README.rst @@ -0,0 +1,5 @@ +Fuel nailgun extenstion for cluster upgrade +=========================================== + +This extension for Nailgun provides API handlers and logic for +cluster upgrading. This extension used by the fuel-octane project. diff --git a/setup.cfg b/setup.cfg index 3d6beb9..b5b9a81 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,7 @@ [metadata] name = fuel-nailgun-extension-cluster-upgrade summary = Cluster upgrade extension for Fuel +description-file = README.rst author = Mirantis Inc. author-email = product@mirantis.com home-page = http://mirantis.com From fdd2a6226483c67ce8bc7adc8b2d354862125bac Mon Sep 17 00:00:00 2001 From: Anastasiya Date: Fri, 15 Jul 2016 10:24:11 +0300 Subject: [PATCH 29/43] Correction of transformation for text_list * added removing of space in text_list * added test for merge_attributes Change-Id: I5582878fc7c524551593abf21dfd4ea45cd430c9 Closes-bug: 1602607 --- cluster_upgrade/tests/test_upgrade.py | 31 +++++++++++++++++++++++++++ cluster_upgrade/upgrade.py | 4 +++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/cluster_upgrade/tests/test_upgrade.py b/cluster_upgrade/tests/test_upgrade.py index f7926f3..2e5a162 100644 --- a/cluster_upgrade/tests/test_upgrade.py +++ b/cluster_upgrade/tests/test_upgrade.py @@ -21,6 +21,7 @@ from nailgun import consts from nailgun.extensions.network_manager.objects.serializers import \ network_configuration +from .. import upgrade from . import base as base_tests from ..objects import adapters from ..objects import relations @@ -43,6 +44,36 @@ class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest): "ip_ranges": [["192.168.42.5", "192.168.42.11"]], } + def test_merge_attributes(self): + src_editable_attrs = { + "test": + {"metadata": "src_fake", + "key": + {"type": "text", + "value": "fake1, fake2,fake3 , fake4"}, + "src_key": "src_data" + }, + "repo_setup": "src_data" + } + + new_editable_attrs = { + "test": + {"metadata": "new_fake", + "key": + {"type": "text_list", + "value": "fake"}, + "new_key": "new_data" + }, + "repo_setup": "new_data" + } + result = upgrade.merge_attributes( + src_editable_attrs, new_editable_attrs + ) + new_editable_attrs["test"]["key"]["value"] = [ + "fake1", "fake2", "fake3", "fake4" + ] + self.assertEqual(result, new_editable_attrs) + def test_create_cluster_clone(self): new_cluster = self.helper.create_cluster_clone(self.src_cluster, self.data) diff --git a/cluster_upgrade/upgrade.py b/cluster_upgrade/upgrade.py index ada4c40..1f5b28e 100644 --- a/cluster_upgrade/upgrade.py +++ b/cluster_upgrade/upgrade.py @@ -48,7 +48,9 @@ def merge_attributes(a, b): # to 'text_list' (a list of strings of IP-addresses). if a_values[key]['type'] == 'text' and \ values['type'] == 'text_list': - values["value"] = values['value'].split(',') + values["value"] = [ + value.strip() for value in values['value'].split(',') + ] return attrs From 56dc7c1f8fc95eea2cbc5410cbcf75ed99f6909d Mon Sep 17 00:00:00 2001 From: Anastasiya Date: Fri, 8 Jul 2016 10:34:07 +0300 Subject: [PATCH 30/43] Assign node to cluster with network template Don't need to assign nics and bonds to netgroups if network template has been applied to new cluster. Change-Id: Ibd06de87964bf7a2038899d32e3d8a0189b9cbbd Partial-bug: 1584044 --- cluster_upgrade/objects/adapters.py | 4 ++++ cluster_upgrade/upgrade.py | 11 +++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cluster_upgrade/objects/adapters.py b/cluster_upgrade/objects/adapters.py index 7a2854e..3348229 100644 --- a/cluster_upgrade/objects/adapters.py +++ b/cluster_upgrade/objects/adapters.py @@ -58,6 +58,10 @@ class NailgunClusterAdapter(object): def editable_attrs(self): return self.cluster.attributes.editable + @property + def network_template(self): + return self.cluster.network_config.configuration_template + @editable_attrs.setter def editable_attrs(self, attrs): self.cluster.attributes.editable = attrs diff --git a/cluster_upgrade/upgrade.py b/cluster_upgrade/upgrade.py index 1f5b28e..5ad22ab 100644 --- a/cluster_upgrade/upgrade.py +++ b/cluster_upgrade/upgrade.py @@ -218,10 +218,13 @@ class UpgradeHelper(object): node.update_cluster_assignment(seed_cluster, roles, pending_roles) objects.Node.set_netgroups_ids(node, netgroups_id_mapping) - orig_manager.set_nic_assignment_netgroups_ids( - node, netgroups_id_mapping) - orig_manager.set_bond_assignment_netgroups_ids( - node, netgroups_id_mapping) + + if not seed_cluster.network_template: + orig_manager.set_nic_assignment_netgroups_ids( + node, netgroups_id_mapping) + orig_manager.set_bond_assignment_netgroups_ids( + node, netgroups_id_mapping) + node.add_pending_change(consts.CLUSTER_CHANGES.interfaces) @classmethod From 9aa37dfe1095fa16c8f1f967945a23027ac05984 Mon Sep 17 00:00:00 2001 From: Alexander Tsamutali Date: Mon, 1 Aug 2016 15:47:58 +0300 Subject: [PATCH 31/43] Add package spec Change-Id: Id71764dff07a4b32851eb8ccf69c66dca4a7b6ab Related-Bug: #1604492 --- ...uel-nailgun-extension-cluster-upgrade.spec | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 specs/fuel-nailgun-extension-cluster-upgrade.spec diff --git a/specs/fuel-nailgun-extension-cluster-upgrade.spec b/specs/fuel-nailgun-extension-cluster-upgrade.spec new file mode 100644 index 0000000..5b26f60 --- /dev/null +++ b/specs/fuel-nailgun-extension-cluster-upgrade.spec @@ -0,0 +1,38 @@ +Name: fuel-nailgun-extension-cluster-upgrade +Version: 10.0~b1 +Release: 1%{?dist} +Summary: Cluster upgrade extension for Fuel +License: Apache-2.0 +Url: https://git.openstack.org/cgit/openstack/fuel-nailgun-extension-cluster-upgrade/ +Source0: %{name}-%{version}.tar.gz +BuildArch: noarch + +BuildRequires: python-devel +BuildRequires: python-pbr +BuildRequires: python-setuptools + +Requires: fuel-nailgun +Requires: python-pbr + +%description +Cluster upgrade extension for Fuel + +%prep +%setup -q -c -n %{name}-%{version} + +%build +export OSLO_PACKAGE_VERSION=%{version} +%py2_build + +%install +export OSLO_PACKAGE_VERSION=%{version} +%py2_install + +%files +%license LICENSE +%{python2_sitelib}/cluster_upgrade +%{python2_sitelib}/*.egg-info + +%changelog +* Thu Aug 04 2016 Alexander Tsamutali - 10.0~b1-1 +- Initial package. From 9e9ae1fabef46dc654f4c2c353646e66f1cf0742 Mon Sep 17 00:00:00 2001 From: Ilya Kharin Date: Wed, 10 Aug 2016 17:28:17 +0300 Subject: [PATCH 32/43] Add absent __init__.py to migrations/versions Without the versions/__init__.py file versions was not identified as a package and was not included in a distribution. Change-Id: I67f152ebb9234df880c61d79d154b1aabc8828c6 Closes-Bug: #1611793 --- .../alembic_migrations/migrations/versions/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 cluster_upgrade/alembic_migrations/migrations/versions/__init__.py diff --git a/cluster_upgrade/alembic_migrations/migrations/versions/__init__.py b/cluster_upgrade/alembic_migrations/migrations/versions/__init__.py new file mode 100644 index 0000000..e69de29 From dc2e3f930957b2c8af2d6c6a60bfcc6c5e6bb061 Mon Sep 17 00:00:00 2001 From: Anastasiya Date: Mon, 1 Aug 2016 14:39:35 +0300 Subject: [PATCH 33/43] Move change_env_settings function from octane to cluster upgrade extension * change_env_settings function was moved to cluster upgrade extention * merge generated attributes code was written Change-Id: I6d1e27b8b0c01f3251067bc88931cd2354feb5ce Partial-Bug: #1602587 --- cluster_upgrade/tests/test_upgrade.py | 31 +++++++++++++++++++++++++-- cluster_upgrade/upgrade.py | 15 ++++++++++++- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/cluster_upgrade/tests/test_upgrade.py b/cluster_upgrade/tests/test_upgrade.py index 2e5a162..8b6c7b2 100644 --- a/cluster_upgrade/tests/test_upgrade.py +++ b/cluster_upgrade/tests/test_upgrade.py @@ -98,8 +98,16 @@ class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest): self.helper.copy_attributes(self.src_cluster, new_cluster) - self.assertEqual(self.src_cluster.generated_attrs, - new_cluster.generated_attrs) + self.assertNotEqual(new_cluster.generated_attrs.get('provision'), + self.src_cluster.generated_attrs.get('provision')) + + # We make image_data in src_cluster and in new_cluster the same + # to validate that all other generated attributes are equal + generated_attrs = copy.deepcopy(self.src_cluster.generated_attrs) + generated_attrs['provision']['image_data'] = \ + new_cluster.generated_attrs['provision']['image_data'] + + self.assertEqual(generated_attrs, new_cluster.generated_attrs) editable_attrs = self.src_cluster.editable_attrs for section, params in six.iteritems(new_cluster.editable_attrs): if section == "repo_setup": @@ -212,3 +220,22 @@ class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest): self.helper.copy_attributes(self.src_cluster, new_cluster) self._check_dns_and_ntp_list_values( new_cluster, ["4", "5", "6"], ["1", "2", "3"]) + + def test_change_env_settings(self): + new_cluster = self.helper.create_cluster_clone(self.src_cluster, + self.data) + self.helper.copy_attributes(self.src_cluster, new_cluster) + attrs = new_cluster.attributes + self.helper.change_env_settings(self.src_cluster, new_cluster) + self.assertEqual('image', + attrs['editable']['provision']['method']['value']) + + def test_change_env_settings_no_editable_provision(self): + new_cluster = self.helper.create_cluster_clone(self.src_cluster, + self.data) + self.helper.copy_attributes(self.src_cluster, new_cluster) + attrs = new_cluster.attributes + attrs['editable']['provision']['method']['value'] = 'cobbler' + self.helper.change_env_settings(self.src_cluster, new_cluster) + self.assertEqual('image', + attrs['editable']['provision']['method']['value']) diff --git a/cluster_upgrade/upgrade.py b/cluster_upgrade/upgrade.py index 5ad22ab..e0a3aa7 100644 --- a/cluster_upgrade/upgrade.py +++ b/cluster_upgrade/upgrade.py @@ -54,6 +54,13 @@ def merge_attributes(a, b): return attrs +def merge_generated_attrs(new_attrs, orig_attrs): + # skip attributes that should be generated for new cluster + attrs = copy.deepcopy(orig_attrs) + attrs.pop('provision', None) + return utils.dict_merge(new_attrs, attrs) + + def merge_nets(a, b): new_settings = copy.deepcopy(b) source_networks = dict((n["name"], n) for n in a["networks"]) @@ -91,6 +98,7 @@ class UpgradeHelper(object): cls.copy_network_config(orig_cluster, new_cluster) relations.UpgradeRelationObject.create_relation(orig_cluster.id, new_cluster.id) + cls.change_env_settings(orig_cluster, new_cluster) return new_cluster @classmethod @@ -111,13 +119,18 @@ class UpgradeHelper(object): # version to another. A set of this kind of steps # should define an upgrade path of a particular # cluster. - new_cluster.generated_attrs = utils.dict_merge( + new_cluster.generated_attrs = merge_generated_attrs( new_cluster.generated_attrs, orig_cluster.generated_attrs) new_cluster.editable_attrs = merge_attributes( orig_cluster.editable_attrs, new_cluster.editable_attrs) + @classmethod + def change_env_settings(cls, orig_cluster, new_cluster): + attrs = new_cluster.attributes + attrs['editable']['provision']['method']['value'] = 'image' + @classmethod def transform_vips_for_net_groups_70(cls, vips): """Rename or remove types of VIPs for 7.0 network groups. From 6150aaca88088bcc9ff4f204d913f5250b9ecac5 Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Tue, 23 Aug 2016 12:25:30 +0300 Subject: [PATCH 34/43] Replace @content decorator with all that madness Change Ia3da3bd809bcca923d53666eca54def78c995f65 broke our handlers as it incorporated destructive changes to all handlers. Change-Id: I688e833b1fb8b658f01b7f858a140c315fa513a2 --- cluster_upgrade/handlers.py | 12 ++++++++---- cluster_upgrade/objects/adapters.py | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/cluster_upgrade/handlers.py b/cluster_upgrade/handlers.py index 4f61f2e..9a89a61 100644 --- a/cluster_upgrade/handlers.py +++ b/cluster_upgrade/handlers.py @@ -29,7 +29,9 @@ class ClusterUpgradeCloneHandler(base.BaseHandler): single = objects.Cluster validator = validators.ClusterUpgradeValidator - @base.content + @base.handle_errors + @base.validate + @base.serialize def POST(self, cluster_id): """Initialize the upgrade of the cluster. @@ -50,7 +52,7 @@ class ClusterUpgradeCloneHandler(base.BaseHandler): request_data = self.checked_data(cluster=orig_cluster) new_cluster = upgrade.UpgradeHelper.clone_cluster(orig_cluster, request_data) - return new_cluster.to_json() + return new_cluster.to_dict() class NodeReassignHandler(base.BaseHandler): @@ -67,7 +69,8 @@ class NodeReassignHandler(base.BaseHandler): self.raise_task(task) - @base.content + @base.handle_errors + @base.validate def POST(self, cluster_id): """Reassign node to the given cluster. @@ -107,7 +110,8 @@ class CopyVIPsHandler(base.BaseHandler): single = objects.Cluster validator = validators.CopyVIPsValidator - @base.content + @base.handle_errors + @base.validate def POST(self, cluster_id): """Copy VIPs from original cluster to new one diff --git a/cluster_upgrade/objects/adapters.py b/cluster_upgrade/objects/adapters.py index 3348229..cf4751b 100644 --- a/cluster_upgrade/objects/adapters.py +++ b/cluster_upgrade/objects/adapters.py @@ -74,8 +74,8 @@ class NailgunClusterAdapter(object): instance=self.cluster) return NailgunNetworkManager(self.cluster, net_manager) - def to_json(self): - return objects.Cluster.to_json(self.cluster) + def to_dict(self): + return objects.Cluster.to_dict(self.cluster) @classmethod def get_by_uid(cls, cluster_id): From d94df3e42b96cdf08901e3917d70b8fe3b97c989 Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Tue, 23 Aug 2016 12:47:03 +0300 Subject: [PATCH 35/43] Add bindep.txt to shorten test run time Currently we have the generic huge list of packages installed on each test run, fix that to include only necessary packages. Change-Id: Ic218c7640dea8a65999259dda006ed59cd87bfb9 --- bindep.txt | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 bindep.txt diff --git a/bindep.txt b/bindep.txt new file mode 100644 index 0000000..568f269 --- /dev/null +++ b/bindep.txt @@ -0,0 +1,6 @@ +libpq-dev +postgresql +postgresql-client +# We don't use these, but mysql-prep step is in template job +mysql-client +mysql-server From 7cf3fe9b3aa0965514e4370e908909e548ff918e Mon Sep 17 00:00:00 2001 From: Ilya Kharin Date: Tue, 23 Aug 2016 01:04:21 +0300 Subject: [PATCH 36/43] Disallow to change operating system during upgrade Changing of an operating system for clouds nodes is not supported and is not tested at all. That's why this additional validation was added. Change-Id: Ibf6db17f783879eff88e2366dfdb0a2871e2aa0a --- cluster_upgrade/objects/adapters.py | 4 ++++ cluster_upgrade/tests/test_validators.py | 8 ++++++++ cluster_upgrade/validators.py | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/cluster_upgrade/objects/adapters.py b/cluster_upgrade/objects/adapters.py index cf4751b..7a00c0c 100644 --- a/cluster_upgrade/objects/adapters.py +++ b/cluster_upgrade/objects/adapters.py @@ -100,6 +100,10 @@ class NailgunReleaseAdapter(object): uid, fail_if_not_found=fail_if_not_found) return release + @property + def operating_system(self): + return self.release.operating_system + @property def is_deployable(self): return objects.Release.is_deployable(self.release) diff --git a/cluster_upgrade/tests/test_validators.py b/cluster_upgrade/tests/test_validators.py index 056c1cb..cf81cb8 100644 --- a/cluster_upgrade/tests/test_validators.py +++ b/cluster_upgrade/tests/test_validators.py @@ -58,6 +58,14 @@ class TestClusterUpgradeValidator(tests_base.BaseCloneClusterTest): self.validator.validate_release_upgrade(self.dst_release, self.src_release) + def test_validate_release_upgrade_to_different_os(self): + self.dst_release.operating_system = consts.RELEASE_OS.centos + msg = "^Changing of operating system is not possible during upgrade " \ + "\(from {0} to {1}\).$".format("Ubuntu", "CentOS") + with self.assertRaisesRegexp(errors.InvalidData, msg): + self.validator.validate_release_upgrade(self.src_release, + self.dst_release) + def test_validate_cluster_name(self): self.validator.validate_cluster_name("cluster-42") diff --git a/cluster_upgrade/validators.py b/cluster_upgrade/validators.py index 0dcd286..78f959e 100644 --- a/cluster_upgrade/validators.py +++ b/cluster_upgrade/validators.py @@ -62,6 +62,12 @@ class ClusterUpgradeValidator(base.BasicValidator): "this release is equal or lower than the release of the " "original cluster.".format(new_release.id), log_message=True) + if orig_release.operating_system != new_release.operating_system: + raise errors.InvalidData( + "Changing of operating system is not possible during upgrade " + "(from {0} to {1}).".format(orig_release.operating_system, + new_release.operating_system), + log_message=True) @classmethod def validate_cluster_name(cls, cluster_name): From 163ce243fbade3dac05eb535ad2987687a57f87d Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Tue, 23 Aug 2016 22:51:19 +0300 Subject: [PATCH 37/43] Add pluggable transformations for data migration This change introduces new transformation mechanism: - all available transformations are listed in setuptools entry points under namespace like this (for cluster transformations): nailgun.cluster_upgrade.transformations.cluster.9.0 = dns_list = ... ntp_list = ... nailgun.cluster_upgrade.transformations.cluster.8.0 = ... - config file will include section that specifies enabled transformations like this: CLUSTER_UPGRADE_TRANSFORMATIONS: cluster: 9.0: dns_list ntp_list ... 8.0: ... 7.0: ... (only default values are implemented here, actual config support will follow) - when transformations are applied to clone cluster from version X to version Y, first transformations for version X+1 are applied, then X+2, and so on ending with transformations for version Y. Since Nailgun doesn't provide any special extension initialization callback, a Lazy wrapper is implemented to facilitate transformations manager usage in extension. Change-Id: I8ee75b54180106ad46c1df67f8d5937d6bd810a1 --- cluster_upgrade/tests/test_transformations.py | 179 ++++++++++++++++++ cluster_upgrade/transformations/__init__.py | 94 +++++++++ 2 files changed, 273 insertions(+) create mode 100644 cluster_upgrade/tests/test_transformations.py create mode 100644 cluster_upgrade/transformations/__init__.py diff --git a/cluster_upgrade/tests/test_transformations.py b/cluster_upgrade/tests/test_transformations.py new file mode 100644 index 0000000..46e0c77 --- /dev/null +++ b/cluster_upgrade/tests/test_transformations.py @@ -0,0 +1,179 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from distutils import version + +import mock +from nailgun.test import base as nailgun_test_base +import six + +from .. import transformations + + +class TestTransformations(nailgun_test_base.BaseUnitTest): + def test_get_config(self): + config = object() + + class Manager(transformations.Manager): + default_config = config + + self.assertIs(config, Manager.get_config('testname')) + + def setup_extension_manager(self, extensions): + p = mock.patch("stevedore.ExtensionManager", spec=['__call__']) + mock_extman = p.start() + self.addCleanup(p.stop) + + def extman(namespace, *args, **kwargs): + instance = mock.MagicMock(name=namespace) + ext_results = {} + for ver, exts in six.iteritems(extensions): + if namespace.endswith(ver): + ext_results = {name: mock.Mock(name=name, plugin=ext) + for name, ext in six.iteritems(exts)} + break + else: + self.fail("Called with unexpected version in namespace: {}, " + "expected versions: {}".format( + namespace, list(extensions))) + instance.__getitem__.side_effect = ext_results.__getitem__ + return instance + + mock_extman.side_effect = extman + return mock_extman + + def test_load_transformers(self): + config = {'9.0': ['a', 'b']} + extensions = {'9.0': { + 'a': mock.Mock(name='a'), + 'b': mock.Mock(name='b'), + }} + mock_extman = self.setup_extension_manager(extensions) + + res = transformations.Manager.load_transformers('testname', config) + + self.assertEqual(res, [(version.StrictVersion('9.0'), [ + extensions['9.0']['a'], + extensions['9.0']['b'], + ])]) + callback = transformations.reraise_endpoint_load_failure + self.assertEqual(mock_extman.mock_calls, [ + mock.call( + 'nailgun.cluster_upgrade.transformations.testname.9.0', + on_load_failure_callback=callback, + ), + ]) + + def test_load_transformers_empty(self): + config = {} + extensions = {'9.0': { + 'a': mock.Mock(name='a'), + 'b': mock.Mock(name='b'), + }} + mock_extman = self.setup_extension_manager(extensions) + + res = transformations.Manager.load_transformers('testname', config) + + self.assertEqual(res, []) + self.assertEqual(mock_extman.mock_calls, []) + + def test_load_transformers_sorted(self): + config = {'9.0': ['a', 'b'], '8.0': ['c']} + extensions = { + '9.0': { + 'a': mock.Mock(name='a'), + 'b': mock.Mock(name='b'), + }, + '8.0': { + 'c': mock.Mock(name='c'), + 'd': mock.Mock(name='d'), + }, + } + mock_extman = self.setup_extension_manager(extensions) + + orig_iteritems = six.iteritems + iteritems_patch = mock.patch('six.iteritems') + mock_iteritems = iteritems_patch.start() + self.addCleanup(iteritems_patch.stop) + + def sorted_iteritems(d): + return sorted(orig_iteritems(d), reverse=True) + + mock_iteritems.side_effect = sorted_iteritems + + res = transformations.Manager.load_transformers('testname', config) + + self.assertEqual(res, [ + (version.StrictVersion('8.0'), [ + extensions['8.0']['c'], + ]), + (version.StrictVersion('9.0'), [ + extensions['9.0']['a'], + extensions['9.0']['b'], + ]), + ]) + callback = transformations.reraise_endpoint_load_failure + self.assertItemsEqual(mock_extman.mock_calls, [ + mock.call( + 'nailgun.cluster_upgrade.transformations.testname.9.0', + on_load_failure_callback=callback, + ), + mock.call( + 'nailgun.cluster_upgrade.transformations.testname.8.0', + on_load_failure_callback=callback, + ), + ]) + + def test_load_transformers_keyerror(self): + config = {'9.0': ['a', 'b', 'c']} + extensions = {'9.0': { + 'a': mock.Mock(name='a'), + 'b': mock.Mock(name='b'), + }} + mock_extman = self.setup_extension_manager(extensions) + + with self.assertRaisesRegexp(KeyError, 'c'): + transformations.Manager.load_transformers('testname', config) + + callback = transformations.reraise_endpoint_load_failure + self.assertEqual(mock_extman.mock_calls, [ + mock.call( + 'nailgun.cluster_upgrade.transformations.testname.9.0', + on_load_failure_callback=callback, + ), + ]) + + @mock.patch.object(transformations.Manager, 'load_transformers') + def test_apply(self, mock_load): + mock_trans = mock.Mock() + mock_load.return_value = [ + (version.StrictVersion('7.0'), [mock_trans.a, mock_trans.b]), + (version.StrictVersion('8.0'), [mock_trans.c, mock_trans.d]), + (version.StrictVersion('9.0'), [mock_trans.e, mock_trans.f]), + ] + man = transformations.Manager() + res = man.apply('7.0', '9.0', {}) + self.assertEqual(res, mock_trans.f.return_value) + self.assertEqual(mock_trans.mock_calls, [ + mock.call.c({}), + mock.call.d(mock_trans.c.return_value), + mock.call.e(mock_trans.d.return_value), + mock.call.f(mock_trans.e.return_value), + ]) + + +class TestLazy(nailgun_test_base.BaseUnitTest): + def test_lazy(self): + mgr_cls_mock = mock.Mock() + lazy_obj = transformations.Lazy(mgr_cls_mock) + lazy_obj.apply() + self.assertEqual(lazy_obj.apply, mgr_cls_mock.return_value.apply) diff --git a/cluster_upgrade/transformations/__init__.py b/cluster_upgrade/transformations/__init__.py new file mode 100644 index 0000000..64c14d7 --- /dev/null +++ b/cluster_upgrade/transformations/__init__.py @@ -0,0 +1,94 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import distutils.version +import logging +import threading + +import six + +import stevedore + +LOG = logging.getLogger(__name__) + + +def reraise_endpoint_load_failure(manager, endpoint, exc): + LOG.error('Failed to load %s: %s', endpoint.name, exc) + raise # Avoid unexpectedly skipped steps + + +class Manager(object): + default_config = None + name = None + + def __init__(self): + self.config = self.get_config(self.name) + self.transformers = self.load_transformers(self.name, self.config) + + @classmethod + def get_config(cls, name): + # TODO(yorik-sar): merge actual config with defaults + return cls.default_config + + @staticmethod + def load_transformers(name, config): + transformers = [] + for version, names in six.iteritems(config): + extension_manager = stevedore.ExtensionManager( + 'nailgun.cluster_upgrade.transformations.{}.{}'.format( + name, version), + on_load_failure_callback=reraise_endpoint_load_failure, + ) + try: + sorted_extensions = [extension_manager[n].plugin + for n in names] + except KeyError as exc: + LOG.error('%s transformer %s not found for version %s', + name, exc, version) + raise + strict_version = distutils.version.StrictVersion(version) + transformers.append((strict_version, sorted_extensions)) + transformers.sort() + return transformers + + def apply(self, from_version, to_version, data): + strict_from = distutils.version.StrictVersion(from_version) + strict_to = distutils.version.StrictVersion(to_version) + assert strict_from < strict_to, \ + "from_version must be smaller than to_version" + data = copy.deepcopy(data) + for version, transformers in self.transformers: + if version <= strict_from: + continue + if version > strict_to: + break + for transformer in transformers: + LOG.debug("Applying %s transformer %s", + self.name, transformer) + data = transformer(data) + return data + + +class Lazy(object): + def __init__(self, mgr_cls): + self.mgr_cls = mgr_cls + self.mgr = None + self.lock = threading.Lock() + + def apply(self, *args, **kwargs): + if self.mgr is None: + with self.lock: + if self.mgr is None: + self.mgr = self.mgr_cls() + self.apply = self.mgr.apply + return self.mgr.apply(*args, **kwargs) From 95ff3a3598397bb6e3c68793051298aaa006d163 Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Tue, 23 Aug 2016 23:01:16 +0300 Subject: [PATCH 38/43] Add cluster transformations Implement transformations that are applied to cluster attributes during environment cloning. Conversion from text to text_list type has been limited to dns_list and ntp_list keys only to keep predictable behavior. Change-Id: I1ff596f850bd42243697cad1c1c35f0cf1386376 --- cluster_upgrade/tests/test_transformations.py | 42 +++++++++++++++ cluster_upgrade/tests/test_upgrade.py | 5 +- cluster_upgrade/transformations/cluster.py | 52 +++++++++++++++++++ cluster_upgrade/upgrade.py | 47 +++++++---------- setup.cfg | 4 ++ 5 files changed, 118 insertions(+), 32 deletions(-) create mode 100644 cluster_upgrade/transformations/cluster.py diff --git a/cluster_upgrade/tests/test_transformations.py b/cluster_upgrade/tests/test_transformations.py index 46e0c77..880ee2d 100644 --- a/cluster_upgrade/tests/test_transformations.py +++ b/cluster_upgrade/tests/test_transformations.py @@ -17,6 +17,7 @@ from nailgun.test import base as nailgun_test_base import six from .. import transformations +from ..transformations import cluster class TestTransformations(nailgun_test_base.BaseUnitTest): @@ -177,3 +178,44 @@ class TestLazy(nailgun_test_base.BaseUnitTest): lazy_obj = transformations.Lazy(mgr_cls_mock) lazy_obj.apply() self.assertEqual(lazy_obj.apply, mgr_cls_mock.return_value.apply) + + +class TestClusterTransformers(nailgun_test_base.BaseUnitTest): + def setUp(self): + self.data = { + 'editable': { + 'external_dns': { + 'dns_list': {'type': 'text', 'value': 'a,b,\nc, d'}}, + 'external_ntp': { + 'ntp_list': {'type': 'text', 'value': 'a,b,\nc, d'}}, + }, + 'generated': { + 'provision': {}, + }, + } + + def test_dns_list(self): + res = cluster.transform_dns_list(self.data) + self.assertEqual( + res['editable']['external_dns']['dns_list'], + {'type': 'text_list', 'value': ['a', 'b', 'c', 'd']}, + ) + + def test_ntp_list(self): + res = cluster.transform_ntp_list(self.data) + self.assertEqual( + res['editable']['external_ntp']['ntp_list'], + {'type': 'text_list', 'value': ['a', 'b', 'c', 'd']}, + ) + + def test_provision(self): + res = cluster.drop_generated_provision(self.data) + self.assertNotIn('provision', res['generated']) + + def test_manager(self): + man = cluster.Manager() # verify default config and entry points + self.assertEqual(man.transformers, [(version.StrictVersion('9.0'), [ + cluster.transform_dns_list, + cluster.transform_ntp_list, + cluster.drop_generated_provision, + ])]) diff --git a/cluster_upgrade/tests/test_upgrade.py b/cluster_upgrade/tests/test_upgrade.py index 8b6c7b2..2f9ff98 100644 --- a/cluster_upgrade/tests/test_upgrade.py +++ b/cluster_upgrade/tests/test_upgrade.py @@ -50,7 +50,7 @@ class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest): {"metadata": "src_fake", "key": {"type": "text", - "value": "fake1, fake2,fake3 , fake4"}, + "value": "fake"}, "src_key": "src_data" }, "repo_setup": "src_data" @@ -69,9 +69,6 @@ class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest): result = upgrade.merge_attributes( src_editable_attrs, new_editable_attrs ) - new_editable_attrs["test"]["key"]["value"] = [ - "fake1", "fake2", "fake3", "fake4" - ] self.assertEqual(result, new_editable_attrs) def test_create_cluster_clone(self): diff --git a/cluster_upgrade/transformations/cluster.py b/cluster_upgrade/transformations/cluster.py new file mode 100644 index 0000000..61368e0 --- /dev/null +++ b/cluster_upgrade/transformations/cluster.py @@ -0,0 +1,52 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from cluster_upgrade import transformations + +# NOTE: In the mitaka-9.0 release types of values dns_list and +# ntp_list were changed from 'text' +# (a string of comma-separated IP-addresses) +# to 'text_list' (a list of strings of IP-addresses). + + +def transform_to_text_list(data): + if data['type'] == 'text': + data['type'] = 'text_list' + data['value'] = [ + part.strip() for part in data['value'].split(',') + ] + + return data + + +def transform_dns_list(data): + dns_list = data['editable']['external_dns']['dns_list'] + transform_to_text_list(dns_list) + return data + + +def transform_ntp_list(data): + ntp_list = data['editable']['external_ntp']['ntp_list'] + transform_to_text_list(ntp_list) + return data + + +def drop_generated_provision(data): + data['generated'].pop('provision', None) + return data + + +class Manager(transformations.Manager): + default_config = { + '9.0': ['dns_list', 'ntp_list', 'drop_provision'], + } + name = 'cluster' diff --git a/cluster_upgrade/upgrade.py b/cluster_upgrade/upgrade.py index e0a3aa7..e1b9ec8 100644 --- a/cluster_upgrade/upgrade.py +++ b/cluster_upgrade/upgrade.py @@ -25,7 +25,9 @@ from nailgun.extensions.network_manager.objects.serializers import \ from nailgun import objects from nailgun import utils +from . import transformations # That's weird, but that's how hacking likes from .objects import adapters +from .transformations import cluster as cluster_trs def merge_attributes(a, b): @@ -42,25 +44,9 @@ def merge_attributes(a, b): for key, values in six.iteritems(pairs): if key != "metadata" and key in a_values: values["value"] = a_values[key]["value"] - # NOTE: In the mitaka-9.0 release types of values dns_list and - # ntp_list were changed from 'text' - # (a string of comma-separated IP-addresses) - # to 'text_list' (a list of strings of IP-addresses). - if a_values[key]['type'] == 'text' and \ - values['type'] == 'text_list': - values["value"] = [ - value.strip() for value in values['value'].split(',') - ] return attrs -def merge_generated_attrs(new_attrs, orig_attrs): - # skip attributes that should be generated for new cluster - attrs = copy.deepcopy(orig_attrs) - attrs.pop('provision', None) - return utils.dict_merge(new_attrs, attrs) - - def merge_nets(a, b): new_settings = copy.deepcopy(b) source_networks = dict((n["name"], n) for n in a["networks"]) @@ -88,6 +74,7 @@ class UpgradeHelper(object): consts.CLUSTER_NET_PROVIDERS.nova_network: network_configuration.NovaNetworkConfigurationSerializer, } + cluster_transformations = transformations.Lazy(cluster_trs.Manager) @classmethod def clone_cluster(cls, orig_cluster, data): @@ -111,20 +98,24 @@ class UpgradeHelper(object): @classmethod def copy_attributes(cls, orig_cluster, new_cluster): - # TODO(akscram): Attributes should be copied including - # borderline cases when some parameters are - # renamed or moved into plugins. Also, we should - # to keep special steps in copying of parameters - # that know how to translate parameters from one - # version to another. A set of this kind of steps - # should define an upgrade path of a particular - # cluster. - new_cluster.generated_attrs = merge_generated_attrs( + attrs = cls.cluster_transformations.apply( + orig_cluster.release.environment_version, + new_cluster.release.environment_version, + { + 'editable': orig_cluster.editable_attrs, + 'generated': orig_cluster.generated_attrs, + }, + ) + + new_cluster.generated_attrs = utils.dict_merge( new_cluster.generated_attrs, - orig_cluster.generated_attrs) + attrs['generated'], + ) + new_cluster.editable_attrs = merge_attributes( - orig_cluster.editable_attrs, - new_cluster.editable_attrs) + attrs['editable'], + new_cluster.editable_attrs, + ) @classmethod def change_env_settings(cls, orig_cluster, new_cluster): diff --git a/setup.cfg b/setup.cfg index b5b9a81..aa902e4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,3 +25,7 @@ packages = [entry_points] nailgun.extensions = cluster_upgrade = cluster_upgrade.extension:ClusterUpgradeExtension +nailgun.cluster_upgrade.transformations.cluster.9.0 = + dns_list = cluster_upgrade.transformations.cluster:transform_dns_list + ntp_list = cluster_upgrade.transformations.cluster:transform_ntp_list + drop_provision = cluster_upgrade.transformations.cluster:drop_generated_provision From a764ba306dcd0cfb65a82c89f07ec0f3d129c66b Mon Sep 17 00:00:00 2001 From: Nikita Zubkov Date: Tue, 23 Aug 2016 14:05:16 +0300 Subject: [PATCH 39/43] Update README Add instalation section Change-Id: Ib16132ab9c18d757e96e98304fdd8339ddf5497a --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index 8991794..e8a886c 100644 --- a/README.rst +++ b/README.rst @@ -3,3 +3,9 @@ Fuel nailgun extenstion for cluster upgrade This extension for Nailgun provides API handlers and logic for cluster upgrading. This extension used by the fuel-octane project. + +Instalation +----------- +After installing `fuel-nailgun-extension-cluster-upgrade` package run: +1) `nailgun_syncdb` - migrate database +2) restart nailgun service From b3ce0d348cc20ed988b5adf66474836f631eb12d Mon Sep 17 00:00:00 2001 From: Anastasiya Date: Mon, 1 Aug 2016 15:47:13 +0300 Subject: [PATCH 40/43] Add tests for assign_node_to_cluster Change-Id: Iafa1baa6a1ca4d701ec89e49dd9d6f969804c82e --- cluster_upgrade/objects/adapters.py | 14 +++++++-- cluster_upgrade/tests/test_upgrade.py | 42 +++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/cluster_upgrade/objects/adapters.py b/cluster_upgrade/objects/adapters.py index 3348229..41970bc 100644 --- a/cluster_upgrade/objects/adapters.py +++ b/cluster_upgrade/objects/adapters.py @@ -58,13 +58,17 @@ class NailgunClusterAdapter(object): def editable_attrs(self): return self.cluster.attributes.editable + @editable_attrs.setter + def editable_attrs(self, attrs): + self.cluster.attributes.editable = attrs + @property def network_template(self): return self.cluster.network_config.configuration_template - @editable_attrs.setter - def editable_attrs(self, attrs): - self.cluster.attributes.editable = attrs + @network_template.setter + def network_template(self, template): + self.cluster.network_config.configuration_template = template def get_create_data(self): return objects.Cluster.get_create_data(self.cluster) @@ -177,6 +181,10 @@ class NailgunNodeAdapter(object): def status(self): return self.node.status + @property + def nic_interfaces(self): + return self.node.nic_interfaces + @property def error_type(self): return self.node.error_type diff --git a/cluster_upgrade/tests/test_upgrade.py b/cluster_upgrade/tests/test_upgrade.py index 8b6c7b2..7189c35 100644 --- a/cluster_upgrade/tests/test_upgrade.py +++ b/cluster_upgrade/tests/test_upgrade.py @@ -20,6 +20,7 @@ import six from nailgun import consts from nailgun.extensions.network_manager.objects.serializers import \ network_configuration +from nailgun.test.base import fake_tasks from .. import upgrade from . import base as base_tests @@ -239,3 +240,44 @@ class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest): self.helper.change_env_settings(self.src_cluster, new_cluster) self.assertEqual('image', attrs['editable']['provision']['method']['value']) + + def get_assigned_nets(self, node): + assigned_nets = {} + for iface in node.nic_interfaces: + nets = [net.name for net in iface.assigned_networks_list] + assigned_nets[iface.name] = nets + return assigned_nets + + @fake_tasks() + def assign_node_to_cluster(self, template=None): + new_cluster = self.helper.clone_cluster(self.src_cluster, self.data) + node = adapters.NailgunNodeAdapter(self.src_cluster.cluster.nodes[0]) + + orig_assigned_nets = self.get_assigned_nets(node) + + if template: + net_template = self.env.read_fixtures(['network_template_80'])[0] + new_cluster.network_template = net_template + orig_assigned_nets = { + 'eth0': ['fuelweb_admin'], 'eth1': ['public', 'management'] + } + + self.helper.assign_node_to_cluster(node, new_cluster, node.roles, []) + self.db.refresh(new_cluster.cluster) + + self.assertEqual(node.cluster_id, new_cluster.id) + + self.env.clusters.append(new_cluster.cluster) + task = self.env.launch_provisioning_selected(cluster_id=new_cluster.id) + self.assertEqual(task.status, consts.TASK_STATUSES.ready) + for n in new_cluster.cluster.nodes: + self.assertEqual(consts.NODE_STATUSES.provisioned, n.status) + + new_assigned_nets = self.get_assigned_nets(node) + self.assertEqual(orig_assigned_nets, new_assigned_nets) + + def test_assign_node_to_cluster(self): + self.assign_node_to_cluster() + + def test_assign_node_to_cluster_with_template(self): + self.assign_node_to_cluster(template=True) From b54f9d4c29808c4561b20af508b85b9e40fa72b6 Mon Sep 17 00:00:00 2001 From: Nikita Zubkov Date: Wed, 24 Aug 2016 17:51:39 +0300 Subject: [PATCH 41/43] Move partition info transformation to extension octane have some dirty hacks to change volumes attributes using nailgun as a library, such modifications should be done in a extension Change-Id: I422bb368916f3a319e286edcc6103a2834097a87 --- cluster_upgrade/objects/adapters.py | 9 ++++ cluster_upgrade/transformations/__init__.py | 4 +- cluster_upgrade/transformations/volumes.py | 53 +++++++++++++++++++++ cluster_upgrade/upgrade.py | 9 ++++ setup.cfg | 2 + 5 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 cluster_upgrade/transformations/volumes.py diff --git a/cluster_upgrade/objects/adapters.py b/cluster_upgrade/objects/adapters.py index 7a00c0c..cc2d169 100644 --- a/cluster_upgrade/objects/adapters.py +++ b/cluster_upgrade/objects/adapters.py @@ -14,6 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. +from nailgun.extensions.volume_manager import extension as volume_ext from nailgun import objects @@ -200,6 +201,14 @@ class NailgunNodeAdapter(object): def add_pending_change(self, change): objects.Node.add_pending_change(self.node, change) + def get_volumes(self): + return volume_ext.VolumeManagerExtension.get_node_volumes(self.node) + + def set_volumes(self, volumes): + return volume_ext.VolumeManagerExtension.set_node_volumes( + self.node, volumes + ) + class NailgunNetworkGroupAdapter(object): diff --git a/cluster_upgrade/transformations/__init__.py b/cluster_upgrade/transformations/__init__.py index 64c14d7..7e1d861 100644 --- a/cluster_upgrade/transformations/__init__.py +++ b/cluster_upgrade/transformations/__init__.py @@ -64,8 +64,8 @@ class Manager(object): def apply(self, from_version, to_version, data): strict_from = distutils.version.StrictVersion(from_version) strict_to = distutils.version.StrictVersion(to_version) - assert strict_from < strict_to, \ - "from_version must be smaller than to_version" + assert strict_from <= strict_to, \ + "from_version must not be greater than to_version" data = copy.deepcopy(data) for version, transformers in self.transformers: if version <= strict_from: diff --git a/cluster_upgrade/transformations/volumes.py b/cluster_upgrade/transformations/volumes.py new file mode 100644 index 0000000..cb9dca7 --- /dev/null +++ b/cluster_upgrade/transformations/volumes.py @@ -0,0 +1,53 @@ +# coding: utf-8 + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from cluster_upgrade import transformations + + +def transform_node_volumes(volumes): + try: + os_vg = next(vol for vol in volumes + if 'id' in vol and vol['id'] == 'os') + except StopIteration: + return volumes + + other_volumes = [vol for vol in volumes + if 'id' not in vol or vol['id'] != 'os'] + + for disk in other_volumes: + disk_volumes = disk['volumes'] + disk['volumes'] = [] + + for v in disk_volumes: + if v['type'] == 'pv' and v['vg'] == 'os' and v['size'] > 0: + for vv in os_vg['volumes']: + partition = {'name': vv['name'], + 'size': vv['size'], + 'type': 'partition', + 'mount': vv['mount'], + 'file_system': vv['file_system']} + disk['volumes'].append(partition) + else: + if v['type'] == 'lvm_meta_pool' or v['type'] == 'boot': + v['size'] = 0 + disk['volumes'].append(v) + + return volumes + + +class Manager(transformations.Manager): + default_config = { + '6.1': ['node_volumes'] + } + name = 'volumes' diff --git a/cluster_upgrade/upgrade.py b/cluster_upgrade/upgrade.py index e1b9ec8..bd2c580 100644 --- a/cluster_upgrade/upgrade.py +++ b/cluster_upgrade/upgrade.py @@ -28,6 +28,7 @@ from nailgun import utils from . import transformations # That's weird, but that's how hacking likes from .objects import adapters from .transformations import cluster as cluster_trs +from .transformations import volumes as volumes_trs def merge_attributes(a, b): @@ -75,6 +76,7 @@ class UpgradeHelper(object): network_configuration.NovaNetworkConfigurationSerializer, } cluster_transformations = transformations.Lazy(cluster_trs.Manager) + volumes_transformations = transformations.Lazy(volumes_trs.Manager) @classmethod def clone_cluster(cls, orig_cluster, data): @@ -215,6 +217,13 @@ class UpgradeHelper(object): orig_cluster = adapters.NailgunClusterAdapter.get_by_uid( node.cluster_id) + volumes = cls.volumes_transformations.apply( + orig_cluster.release.environment_version, + seed_cluster.release.environment_version, + node.get_volumes(), + ) + node.set_volumes(volumes) + orig_manager = orig_cluster.get_network_manager() netgroups_id_mapping = cls.get_netgroups_id_mapping( diff --git a/setup.cfg b/setup.cfg index aa902e4..ea69fea 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,6 +25,8 @@ packages = [entry_points] nailgun.extensions = cluster_upgrade = cluster_upgrade.extension:ClusterUpgradeExtension +nailgun.cluster_upgrade.transformations.volumes.6.1 = + node_volumes = cluster_upgrade.transformations.volumes:transform_node_volumes nailgun.cluster_upgrade.transformations.cluster.9.0 = dns_list = cluster_upgrade.transformations.cluster:transform_dns_list ntp_list = cluster_upgrade.transformations.cluster:transform_ntp_list From 8de47e0dbf5955b3cbe59a3956e1ca1f4d8679ef Mon Sep 17 00:00:00 2001 From: Nikita Zubkov Date: Wed, 24 Aug 2016 15:42:32 +0300 Subject: [PATCH 42/43] Add VIPs transformer Move upgrade vips code to the transformer Change-Id: Ia6935521b8b90b73fb74ba92859e7febf74c4ced --- cluster_upgrade/transformations/vip.py | 62 ++++++++++++++++++++++++++ cluster_upgrade/upgrade.py | 51 ++++----------------- setup.cfg | 2 + 3 files changed, 72 insertions(+), 43 deletions(-) create mode 100644 cluster_upgrade/transformations/vip.py diff --git a/cluster_upgrade/transformations/vip.py b/cluster_upgrade/transformations/vip.py new file mode 100644 index 0000000..417b5f6 --- /dev/null +++ b/cluster_upgrade/transformations/vip.py @@ -0,0 +1,62 @@ +# coding: utf-8 + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections + +from cluster_upgrade import transformations + + +def transform_vips(data): + """Rename or remove types of VIPs for 7.0 network groups. + + This method renames types of VIPs from older releases (<7.0) to + be compatible with network groups of the 7.0 release according + to the rules: + + management: haproxy -> management + public: haproxy -> public + public: vrouter -> vrouter_pub + + Note, that in the result VIPs are present only those IPs that + correspond to the given rules. + """ + rename_vip_rules = { + "management": { + "haproxy": "management", + "vrouter": "vrouter", + }, + "public": { + "haproxy": "public", + "vrouter": "vrouter_pub", + }, + } + renamed_vips = collections.defaultdict(dict) + for ng_name, vips_obj in data.items(): + + ng_vip_rules = rename_vip_rules[ng_name] + for vip_name, vip_addr in vips_obj.items(): + if vip_name not in ng_vip_rules: + continue + + new_vip_name = ng_vip_rules[vip_name] + renamed_vips[ng_name][new_vip_name] = vip_addr + + return renamed_vips + + +class Manager(transformations.Manager): + default_config = { + '7.0': ['transform_vips'] + } + name = 'vip' diff --git a/cluster_upgrade/upgrade.py b/cluster_upgrade/upgrade.py index bd2c580..dde0606 100644 --- a/cluster_upgrade/upgrade.py +++ b/cluster_upgrade/upgrade.py @@ -14,9 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -import collections import copy -from distutils import version import six from nailgun import consts @@ -28,6 +26,7 @@ from nailgun import utils from . import transformations # That's weird, but that's how hacking likes from .objects import adapters from .transformations import cluster as cluster_trs +from .transformations import vip from .transformations import volumes as volumes_trs @@ -76,6 +75,7 @@ class UpgradeHelper(object): network_configuration.NovaNetworkConfigurationSerializer, } cluster_transformations = transformations.Lazy(cluster_trs.Manager) + vip_transformations = transformations.Lazy(vip.Manager) volumes_transformations = transformations.Lazy(volumes_trs.Manager) @classmethod @@ -124,41 +124,6 @@ class UpgradeHelper(object): attrs = new_cluster.attributes attrs['editable']['provision']['method']['value'] = 'image' - @classmethod - def transform_vips_for_net_groups_70(cls, vips): - """Rename or remove types of VIPs for 7.0 network groups. - - This method renames types of VIPs from older releases (<7.0) to - be compatible with network groups of the 7.0 release according - to the rules: - - management: haproxy -> management - public: haproxy -> public - public: vrouter -> vrouter_pub - - Note, that in the result VIPs are present only those IPs that - correspond to the given rules. - """ - rename_vip_rules = { - "management": { - "haproxy": "management", - "vrouter": "vrouter", - }, - "public": { - "haproxy": "public", - "vrouter": "vrouter_pub", - }, - } - renamed_vips = collections.defaultdict(dict) - for ng_name, vips in six.iteritems(vips): - ng_vip_rules = rename_vip_rules[ng_name] - for vip_name, vip_addr in six.iteritems(vips): - if vip_name not in ng_vip_rules: - continue - new_vip_name = ng_vip_rules[vip_name] - renamed_vips[ng_name][new_vip_name] = vip_addr - return renamed_vips - @classmethod def copy_network_config(cls, orig_cluster, new_cluster): nets_serializer = cls.network_serializers[orig_cluster.net_provider] @@ -179,12 +144,12 @@ class UpgradeHelper(object): assigned_vips = orig_net_manager.get_assigned_vips() for ng_name in (consts.NETWORKS.public, consts.NETWORKS.management): vips[ng_name] = assigned_vips[ng_name] - # NOTE(akscram): In the 7.0 release was introduced networking - # templates that use the vip_name column as - # unique names of VIPs. - if version.LooseVersion(orig_cluster.release.environment_version) < \ - version.LooseVersion("7.0"): - vips = cls.transform_vips_for_net_groups_70(vips) + + vips = cls.vip_transformations.apply( + orig_cluster.release.environment_version, + new_cluster.release.environment_version, + vips + ) new_net_manager.assign_given_vips_for_net_groups(vips) new_net_manager.assign_vips_for_net_groups() diff --git a/setup.cfg b/setup.cfg index ea69fea..c88b57f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,3 +31,5 @@ nailgun.cluster_upgrade.transformations.cluster.9.0 = dns_list = cluster_upgrade.transformations.cluster:transform_dns_list ntp_list = cluster_upgrade.transformations.cluster:transform_ntp_list drop_provision = cluster_upgrade.transformations.cluster:drop_generated_provision +nailgun.cluster_upgrade.transformations.vip.7.0 = + transform_vips = cluster_upgrade.transformations.vip:transform_vips From 17ab5a3aaa7d2c4ae4652d1a4dc0410b31e7c47d Mon Sep 17 00:00:00 2001 From: Sergey Abramov Date: Wed, 24 Aug 2016 18:32:39 +0300 Subject: [PATCH 43/43] Add create upgrade release handler Required for create new release just for upgrade, that have overwrited params. This params are valid for orig cluster release. Change-Id: Ib2387b9c2b74902c7289ee8f69a5f5d323ec82ca --- cluster_upgrade/extension.py | 3 +++ cluster_upgrade/handlers.py | 42 ++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/cluster_upgrade/extension.py b/cluster_upgrade/extension.py index 66df106..79bdca4 100644 --- a/cluster_upgrade/extension.py +++ b/cluster_upgrade/extension.py @@ -33,6 +33,9 @@ class ClusterUpgradeExtension(extensions.BaseExtension): 'handler': handlers.NodeReassignHandler}, {'uri': r'/clusters/(?P\d+)/upgrade/vips/?$', 'handler': handlers.CopyVIPsHandler}, + {'uri': r'/clusters/(?P\d+)/upgrade/clone_release/' + r'(?P\d+)/?$', + 'handler': handlers.CreateUpgradeReleaseHandler}, ] @classmethod diff --git a/cluster_upgrade/handlers.py b/cluster_upgrade/handlers.py index 9a89a61..d4b7f71 100644 --- a/cluster_upgrade/handlers.py +++ b/cluster_upgrade/handlers.py @@ -143,3 +143,45 @@ class CopyVIPsHandler(base.BaseHandler): upgrade.UpgradeHelper.copy_vips(orig_cluster_adapter, seed_cluster_adapter) + + +class CreateUpgradeReleaseHandler(base.BaseHandler): + @staticmethod + def merge_network_roles(base_nets, orig_nets): + """Create network metadata based on two releases. + + Overwrite base default_mapping by orig default_maping values. + """ + orig_network_dict = {n['id']: n for n in orig_nets} + for base_net in base_nets: + orig_net = orig_network_dict.get(base_net['id']) + if orig_net is None: + orig_net = base_net + base_net['default_mapping'] = orig_net['default_mapping'] + return base_net + + @base.serialize + def POST(self, cluster_id, release_id): + """Create release for upgrade purposes. + + Creates a new release with network_roles_metadata based the given + release and re-use network parameters from the given cluster. + + :returns: JSON representation of the created cluster + :http: * 200 (OK) + * 404 (Cluster or release not found.) + """ + base_release = self.get_object_or_404(objects.Release, release_id) + orig_cluster = self.get_object_or_404(objects.Cluster, cluster_id) + orig_release = orig_cluster.release + + network_metadata = self.merge_network_roles( + base_release.network_roles_metadata, + orig_release.network_roles_metadata) + data = objects.Release.to_dict(base_release) + data['network_roles_metadata'] = network_metadata + data['name'] = '{0} Upgrade ({1})'.format( + base_release.name, orig_release.id) + del data['id'] + new_release = objects.Release.create(data) + return new_release.to_dict()