From 61fd42670920d8f39cfc3107796a908bdbee955e Mon Sep 17 00:00:00 2001 From: Ryan Moe Date: Fri, 24 Jan 2014 14:43:49 -0800 Subject: [PATCH] Allow multiple networks to be created per cluster Added a NodeGroup model. NodeGroups can be created and modified via the Fuel API. Clusters are created with a default node group and the cluster networks are tied to that default node group. Upon creation of a node group a set of networks will be generated. When a node group is deleted those networks will be deleted as well. See previous change: Ic30a1b46112599022645b06f7bef2223ff4c6475 DocImpact Change-Id: I8bbe6b8c6338d6586c7e9a61542939ae2a19828b Implements: blueprint multiple-cluster-networks --- fuelclient/fuelclient/cli/actions/__init__.py | 4 +- fuelclient/fuelclient/cli/actions/node.py | 4 +- .../fuelclient/cli/actions/nodegroup.py | 106 +++++++++++ fuelclient/fuelclient/cli/arguments.py | 8 + fuelclient/fuelclient/objects/nodegroup.py | 87 +++++++++ .../api/v1/handlers/network_configuration.py | 4 - nailgun/nailgun/api/v1/handlers/node.py | 17 +- nailgun/nailgun/api/v1/handlers/node_group.py | 101 ++++++++++ nailgun/nailgun/api/v1/urls.py | 8 + .../nailgun/api/v1/validators/node_group.py | 47 +++++ .../alembic_migrations/versions/fuel_6_0.py | 55 ++++++ nailgun/nailgun/db/sqlalchemy/__init__.py | 1 - .../nailgun/db/sqlalchemy/models/__init__.py | 1 + .../nailgun/db/sqlalchemy/models/cluster.py | 29 ++- .../nailgun/db/sqlalchemy/models/network.py | 2 +- nailgun/nailgun/db/sqlalchemy/models/node.py | 14 ++ nailgun/nailgun/fixtures/admin_network.json | 4 +- nailgun/nailgun/network/checker.py | 6 +- nailgun/nailgun/network/manager.py | 180 +++++++++++++----- nailgun/nailgun/objects/__init__.py | 3 + nailgun/nailgun/objects/cluster.py | 1 + nailgun/nailgun/objects/node.py | 39 +++- nailgun/nailgun/objects/node_group.py | 76 ++++++++ .../serializers/network_configuration.py | 17 +- nailgun/nailgun/objects/serializers/node.py | 3 +- .../nailgun/objects/serializers/node_group.py | 26 +++ .../orchestrator/deployment_serializers.py | 52 +++-- .../orchestrator/provisioning_serializers.py | 11 +- nailgun/nailgun/task/manager.py | 10 + nailgun/nailgun/task/task.py | 6 +- nailgun/nailgun/test/base.py | 2 + .../test_cluster_changes_handler.py | 28 ++- .../test_cluster_collection_handlers.py | 15 +- .../test/integration/test_horizon_url.py | 3 +- .../test_mellanox_orchestrator_serializer.py | 3 +- .../integration/test_network_configuration.py | 4 +- .../test/integration/test_network_manager.py | 2 +- .../test_node_collection_handlers.py | 8 + .../test_orchestrator_serializer.py | 5 +- .../test/integration/test_provisioning.py | 1 + .../test/integration/test_rpc_consumer.py | 11 +- .../test/integration/test_stop_deployment.py | 2 +- .../nailgun/test/unit/test_logs_handlers.py | 3 +- .../test/unit/test_node_assignment_handler.py | 5 +- .../nailgun/test/unit/test_node_deletion.py | 2 +- nailgun/nailgun/test/unit/test_node_disks.py | 2 +- nailgun/nailgun/test/unit/test_node_groups.py | 126 ++++++++++++ nailgun/nailgun/test/unit/test_objects.py | 1 + 48 files changed, 1019 insertions(+), 126 deletions(-) create mode 100644 fuelclient/fuelclient/cli/actions/nodegroup.py create mode 100644 fuelclient/fuelclient/objects/nodegroup.py create mode 100644 nailgun/nailgun/api/v1/handlers/node_group.py create mode 100644 nailgun/nailgun/api/v1/validators/node_group.py create mode 100644 nailgun/nailgun/objects/node_group.py create mode 100644 nailgun/nailgun/objects/serializers/node_group.py create mode 100644 nailgun/nailgun/test/unit/test_node_groups.py diff --git a/fuelclient/fuelclient/cli/actions/__init__.py b/fuelclient/fuelclient/cli/actions/__init__.py index d8136e44ad..ea12ac6023 100644 --- a/fuelclient/fuelclient/cli/actions/__init__.py +++ b/fuelclient/fuelclient/cli/actions/__init__.py @@ -25,6 +25,7 @@ from fuelclient.cli.actions.interrupt import ResetAction from fuelclient.cli.actions.interrupt import StopAction from fuelclient.cli.actions.network import NetworkAction from fuelclient.cli.actions.node import NodeAction +from fuelclient.cli.actions.nodegroup import NodeGroupAction from fuelclient.cli.actions.release import ReleaseAction from fuelclient.cli.actions.role import RoleAction from fuelclient.cli.actions.settings import SettingsAction @@ -49,7 +50,8 @@ actions_tuple = ( SnapshotAction, HealthCheckAction, UserAction, - PluginAction + PluginAction, + NodeGroupAction ) actions = dict( diff --git a/fuelclient/fuelclient/cli/actions/node.py b/fuelclient/fuelclient/cli/actions/node.py index 53cad7746e..84579e9201 100644 --- a/fuelclient/fuelclient/cli/actions/node.py +++ b/fuelclient/fuelclient/cli/actions/node.py @@ -33,7 +33,7 @@ class NodeAction(Action): """ action_name = "node" acceptable_keys = ("id", "status", "name", "cluster", "ip", - "mac", "roles", "pending_roles", "online") + "mac", "roles", "pending_roles", "online", "group_id") def __init__(self): super(NodeAction, self).__init__() @@ -49,7 +49,7 @@ class NodeAction(Action): Args.get_delete_from_db_arg( "Delete specific nodes only from fuel db.\n" "User should still delete node from cobbler"), - Args.get_provision_arg("Provision specific nodes.") + Args.get_provision_arg("Provision specific nodes."), ), group( Args.get_default_arg( diff --git a/fuelclient/fuelclient/cli/actions/nodegroup.py b/fuelclient/fuelclient/cli/actions/nodegroup.py new file mode 100644 index 0000000000..2354618dd8 --- /dev/null +++ b/fuelclient/fuelclient/cli/actions/nodegroup.py @@ -0,0 +1,106 @@ +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from fuelclient.cli.actions.base import Action +from fuelclient.cli.actions.base import check_all +import fuelclient.cli.arguments as Args +from fuelclient.cli.arguments import group +from fuelclient.cli.error import ActionException +from fuelclient.cli.formatting import format_table +from fuelclient.objects.node import Node +from fuelclient.objects.nodegroup import NodeGroup +from fuelclient.objects.nodegroup import NodeGroupCollection + + +class NodeGroupAction(Action): + """Show or modify node groups + """ + action_name = "nodegroup" + acceptable_keys = ("id", "cluster", "name") + + def __init__(self): + super(NodeGroupAction, self).__init__() + self.args = ( + Args.get_env_arg(), + Args.get_list_arg("List all node groups."), + Args.get_name_arg("Name of new node group."), + Args.get_group_arg("ID of node group."), + Args.get_node_arg("List of nodes to assign specified group to."), + group( + Args.get_create_arg( + "Create a new node group in the specified environment." + ), + Args.get_assign_arg( + "Download current network configuration."), + Args.get_delete_arg( + "Verify current network configuration."), + ) + ) + self.flag_func_map = ( + ("create", self.create), + ("delete", self.delete), + ("assign", self.assign), + (None, self.list) + ) + + def create(self, params): + """Create a new node group + fuel --env 1 nodegroup --create --name "group 1" + """ + NodeGroup.create(params.name, int(params.env)) + + def delete(self, params): + """Delete the specified node groups + fuel --env 1 nodegroup --delete --group 1 + fuel --env 1 nodegroup --delete --group 2,3,4 + """ + ngs = NodeGroup.get_by_ids(params.group) + for n in ngs: + if n.name == "default": + raise ActionException( + "Default node groups cannot be deleted." + ) + NodeGroup.delete(n.id) + + @check_all("env") + def assign(self, params): + """Assign nodes to specified node group: + fuel --env 1 nodegroup --assign --node 1 --group 1 + fuel --env 1 nodegroup --assign --node 2,3,4 --group 1 + """ + nodes = [n.id for n in map(Node, params.node)] + ngs = map(NodeGroup, params.group) + if len(ngs) > 1: + raise ActionException( + "Nodes can only be assigned to one node group." + ) + NodeGroup.assign(ngs[0].id, nodes) + + def list(self, params): + """To list all available node groups: + fuel nodegroup + + To filter them by environment: + fuel --env-id 1 nodegroup + """ + group_collection = NodeGroupCollection.get_all() + if params.env: + group_collection.filter_by_env_id(int(params.env)) + self.serializer.print_to_output( + group_collection.data, + format_table( + group_collection.data, + acceptable_keys=self.acceptable_keys, + ) + ) diff --git a/fuelclient/fuelclient/cli/arguments.py b/fuelclient/fuelclient/cli/arguments.py index ce8cac6360..d3536beed8 100644 --- a/fuelclient/fuelclient/cli/arguments.py +++ b/fuelclient/fuelclient/cli/arguments.py @@ -337,6 +337,14 @@ def get_delete_arg(help_msg): return get_boolean_arg("delete", help=help_msg) +def get_assign_arg(help_msg): + return get_boolean_arg("assign", help=help_msg) + + +def get_group_arg(help_msg): + return get_set_type_arg("group", help=help_msg) + + def get_release_arg(help_msg, required=False): return get_int_arg( "release", diff --git a/fuelclient/fuelclient/objects/nodegroup.py b/fuelclient/fuelclient/objects/nodegroup.py new file mode 100644 index 0000000000..bd99365f5f --- /dev/null +++ b/fuelclient/fuelclient/objects/nodegroup.py @@ -0,0 +1,87 @@ +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from operator import attrgetter + +from fuelclient.objects.base import BaseObject + + +class NodeGroup(BaseObject): + + class_api_path = "nodegroups/" + instance_api_path = "nodegroups/{0}/" + + @property + def env_id(self): + return self.get_fresh_data()["cluster"] + + @property + def name(self): + return self.get_fresh_data()["name"] + + @classmethod + def create(cls, name, cluster_id): + return cls.connection.post_request( + cls.class_api_path, + {'cluster_id': cluster_id, 'name': name}, + ) + + @classmethod + def delete(cls, group_id): + return cls.connection.delete_request( + cls.instance_api_path.format(group_id) + ) + + @classmethod + def assign(cls, group_id, nodes): + return cls.connection.post_request( + cls.instance_api_path.format(group_id), + nodes + ) + + +class NodeGroupCollection(object): + + def __init__(self, groups): + self.collection = groups + + @classmethod + def init_with_ids(cls, ids): + return cls(map(NodeGroup, ids)) + + @classmethod + def init_with_data(cls, data): + return cls(map(NodeGroup.init_with_data, data)) + + def __str__(self): + return "node groups [{0}]".format( + ", ".join(map(lambda n: str(n.id), self.collection)) + ) + + def __iter__(self): + return iter(self.collection) + + @property + def data(self): + return map(attrgetter("data"), self.collection) + + @classmethod + def get_all(cls): + return cls(NodeGroup.get_all()) + + def filter_by_env_id(self, env_id): + self.collection = filter( + lambda group: group.env_id == env_id, + self.collection + ) diff --git a/nailgun/nailgun/api/v1/handlers/network_configuration.py b/nailgun/nailgun/api/v1/handlers/network_configuration.py index a76687d75e..0273b33152 100644 --- a/nailgun/nailgun/api/v1/handlers/network_configuration.py +++ b/nailgun/nailgun/api/v1/handlers/network_configuration.py @@ -149,10 +149,6 @@ class NeutronNetworkConfigurationHandler(ProviderHandler): @content_json def PUT(self, cluster_id): data = jsonutils.loads(web.data()) - if data.get("networks"): - data["networks"] = [ - n for n in data["networks"] if n.get("name") != "fuelweb_admin" - ] cluster = self.get_object_or_404(objects.Cluster, cluster_id) self.check_net_provider(cluster) diff --git a/nailgun/nailgun/api/v1/handlers/node.py b/nailgun/nailgun/api/v1/handlers/node.py index a898fbbfe0..c40309a2da 100644 --- a/nailgun/nailgun/api/v1/handlers/node.py +++ b/nailgun/nailgun/api/v1/handlers/node.py @@ -20,6 +20,9 @@ Handlers dealing with nodes from datetime import datetime +from netaddr import IPAddress +from netaddr import IPNetwork + import web from nailgun.api.v1.handlers.base import BaseHandler @@ -44,7 +47,6 @@ from nailgun import notifier class NodeHandler(SingleHandler): - single = objects.Node validator = NodeValidator @@ -56,7 +58,8 @@ class NodeCollectionHandler(CollectionHandler): fields = ('id', 'name', 'meta', 'progress', 'roles', 'pending_roles', 'status', 'mac', 'fqdn', 'ip', 'manufacturer', 'platform_name', 'pending_addition', 'pending_deletion', 'os_platform', - 'error_type', 'online', 'cluster', 'uuid', 'network_data') + 'error_type', 'online', 'cluster', 'uuid', 'network_data', + 'group_id') validator = NodeValidator collection = objects.NodeCollection @@ -130,6 +133,16 @@ class NodeAgentHandler(BaseHandler): raise self.http(404, "Can't find node: {0}".format(nd)) node.timestamp = datetime.now() + + if node.group_id is None: + admin_ngs = db().query(NetworkGroup).filter_by( + name="fuelweb_admin") + ip = IPAddress(node.ip) + for ng in admin_ngs: + if ip in IPNetwork(ng.cidr): + node.group_id = ng.group_id + break + if not node.online: node.online = True msg = u"Node '{0}' is back online".format(node.human_readable_name) diff --git a/nailgun/nailgun/api/v1/handlers/node_group.py b/nailgun/nailgun/api/v1/handlers/node_group.py new file mode 100644 index 0000000000..31e9272890 --- /dev/null +++ b/nailgun/nailgun/api/v1/handlers/node_group.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- + +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import web + +from nailgun.api.v1.handlers.base import BaseHandler +from nailgun.api.v1.handlers.base import CollectionHandler +from nailgun.api.v1.handlers.base import SingleHandler + +from nailgun.api.v1.handlers.base import content_json +from nailgun.api.v1.validators.node_group import NodeGroupValidator + +from nailgun.db import db + +from nailgun import objects + +""" +Handlers dealing with node groups +""" + + +class NodeGroupHandler(SingleHandler): + """NodeGroup single handler + """ + single = objects.NodeGroup + validator = NodeGroupValidator + + def DELETE(self, group_id): + node_group = self.get_object_or_404(objects.NodeGroup, group_id) + db().delete(node_group) + db().commit() + raise web.webapi.HTTPError( + status="204 No Content", + data="" + ) + + +class NodeGroupCollectionHandler(CollectionHandler): + """NodeGroup collection handler + """ + + collection = objects.NodeGroupCollection + validator = NodeGroupValidator + + @content_json + def GET(self): + """May receive cluster_id parameter to filter list + of groups + + :returns: Collection of JSONized Task objects. + :http: * 200 (OK) + * 404 (task not found in db) + """ + user_data = web.input(cluster_id=None) + + if user_data.cluster_id is not None: + return self.collection.to_json( + query=self.collection.get_by_cluster_id( + user_data.cluster_id + ) + ) + else: + return self.collection.to_json() + + +class NodeGroupAssignmentHandler(BaseHandler): + """Node group assignment handler + """ + + @content_json + def POST(self, group_id): + """:returns: Http response. + :http: * 201 (nodes are successfully assigned) + * 400 (invalid nodes data specified) + """ + self.get_object_or_404( + objects.NodeGroup, + group_id + ) + data = self.checked_data() + + nodes = self.get_objects_list_or_404( + objects.NodeCollection, + data + ) + + for node in nodes: + objects.Node.update(node, {"group_id": group_id}) diff --git a/nailgun/nailgun/api/v1/urls.py b/nailgun/nailgun/api/v1/urls.py index 312b54a490..ebff320c9e 100644 --- a/nailgun/nailgun/api/v1/urls.py +++ b/nailgun/nailgun/api/v1/urls.py @@ -40,6 +40,8 @@ from nailgun.api.v1.handlers.logs import LogEntryCollectionHandler from nailgun.api.v1.handlers.logs import LogPackageHandler from nailgun.api.v1.handlers.logs import LogSourceByNodeCollectionHandler from nailgun.api.v1.handlers.logs import LogSourceCollectionHandler +from nailgun.api.v1.handlers.node_group import NodeGroupCollectionHandler +from nailgun.api.v1.handlers.node_group import NodeGroupHandler from nailgun.api.v1.handlers.network_configuration \ import NeutronNetworkConfigurationHandler @@ -146,11 +148,17 @@ urls = ( r'/clusters/(?P\d+)/update/?$', ClusterUpdateHandler, + r'/clusters/(?P\d+)/assignment/?$', NodeAssignmentHandler, r'/clusters/(?P\d+)/unassignment/?$', NodeUnassignmentHandler, + r'/nodegroups/?$', + NodeGroupCollectionHandler, + r'/nodegroups/(?P\d+)/?$', + NodeGroupHandler, + r'/nodes/?$', NodeCollectionHandler, r'/nodes/agent/?$', diff --git a/nailgun/nailgun/api/v1/validators/node_group.py b/nailgun/nailgun/api/v1/validators/node_group.py new file mode 100644 index 0000000000..236fc41408 --- /dev/null +++ b/nailgun/nailgun/api/v1/validators/node_group.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nailgun.api.v1.validators.base import BasicValidator +from nailgun import consts +from nailgun.errors import errors +from nailgun import objects + + +class NodeGroupValidator(BasicValidator): + + @classmethod + def validate(cls, data): + data = cls.validate_json(data) + cluster = objects.Cluster.get_by_uid(data['cluster_id']) + if (cluster.net_provider == consts.CLUSTER_NET_PROVIDERS.nova_network + or cluster.network_config.segmentation_type != + consts.NEUTRON_SEGMENT_TYPES.gre): + raise errors.NotAllowed( + "Node groups can only be created when using Neutron GRE." + ) + + return data + + @classmethod + def validate_delete(cls, instance, force=False): + if (instance.nodes or instance.networks) and not force: + raise errors.CannotDelete( + "You cannot delete a node group that contains " + "nodes or networks" + ) + + @classmethod + def validate_update(cls, data, **kwargs): + return cls.validate_json(data) diff --git a/nailgun/nailgun/db/migration/alembic_migrations/versions/fuel_6_0.py b/nailgun/nailgun/db/migration/alembic_migrations/versions/fuel_6_0.py index dfa83c43f2..e2500fa6f2 100644 --- a/nailgun/nailgun/db/migration/alembic_migrations/versions/fuel_6_0.py +++ b/nailgun/nailgun/db/migration/alembic_migrations/versions/fuel_6_0.py @@ -152,6 +152,21 @@ def upgrade_schema(): ['plugin_id'], ['plugins.id'], ondelete='CASCADE'), sa.PrimaryKeyConstraint('id') ) + op.create_table( + 'nodegroups', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('cluster_id', sa.Integer(), nullable=True), + sa.Column('name', sa.String(length=50), nullable=False), + sa.ForeignKeyConstraint(['cluster_id'], ['clusters.id']), + sa.PrimaryKeyConstraint('id') + ) + op.create_unique_constraint(None, 'clusters', ['name']) + op.add_column( + 'network_groups', + sa.Column('group_id', sa.Integer(), nullable=True) + ) + op.drop_column('network_groups', 'cluster_id') + op.add_column('nodes', sa.Column('group_id', sa.Integer(), nullable=True)) def upgrade_releases(): @@ -182,6 +197,7 @@ def upgrade_data(): # do not deploy 5.0.x series upgrade_release_set_deployable_false( connection, ['2014.1', '2014.1.1-5.0.1', '2014.1.1-5.0.2']) + upgrade_node_groups(connection) # In Fuel 5.x default releases do not have filled orchestrator_data, # and defaults one have been used. In Fuel 6.0 we're going to change @@ -204,6 +220,45 @@ def downgrade_schema(): map(drop_enum, ENUMS) op.drop_table('cluster_plugins') op.drop_table('plugins') + op.drop_column(u'nodes', 'group_id') + op.drop_column(u'network_groups', 'group_id') + op.add_column( + 'network_groups', + sa.Column('cluster_id', sa.Integer(), sa.ForeignKey('clusters.id')) + ) + op.drop_column(u'releases', 'wizard_metadata') + op.drop_table('nodegroups') + + +def upgrade_node_groups(connection): + cluster_select = sa.text("SELECT id from clusters") + node_sel = sa.text("SELECT id FROM nodes WHERE cluster_id=:cluster_id") + node_update = sa.text( + """UPDATE nodes + SET group_id=(SELECT id FROM nodegroups WHERE cluster_id=:cluster_id) + WHERE id=:id""") + group_insert = sa.text("""INSERT INTO nodegroups (cluster_id, name) + VALUES(:cluster_id, 'default')""") + net_select = sa.text("""SELECT id FROM network_groups WHERE + cluster_id=:cluster_id""") + net_update = sa.text("""UPDATE network_groups + SET group_id=(SELECT id FROM nodegroups WHERE cluster_id=:cluster_id) + WHERE id=:id""") + + clusters = connection.execute(cluster_select) + + for cluster in clusters: + connection.execute(group_insert, cluster_id=cluster[0]) + + # Assign nodes to the newly created node group + nodes = connection.execute(node_sel, cluster_id=cluster[0]) + for node in nodes: + connection.execute(node_update, cluster_id=cluster[0], id=node[0]) + + # Assign networks to the newly created node group + nets = connection.execute(net_select, cluster_id=cluster[0]) + for net in nets: + connection.execute(net_update, cluster_id=cluster[0], id=net[0]) def downgrade_data(): diff --git a/nailgun/nailgun/db/sqlalchemy/__init__.py b/nailgun/nailgun/db/sqlalchemy/__init__.py index 9408ff16ed..d09af6c205 100644 --- a/nailgun/nailgun/db/sqlalchemy/__init__.py +++ b/nailgun/nailgun/db/sqlalchemy/__init__.py @@ -36,7 +36,6 @@ db_str = "{engine}://{user}:{passwd}@{host}:{port}/{name}".format( engine = create_engine(db_str, client_encoding='utf8') - class NoCacheQuery(Query): """Override for common Query class. Needed for automatic refreshing objects diff --git a/nailgun/nailgun/db/sqlalchemy/models/__init__.py b/nailgun/nailgun/db/sqlalchemy/models/__init__.py index 6799d600a1..97dd5d5add 100644 --- a/nailgun/nailgun/db/sqlalchemy/models/__init__.py +++ b/nailgun/nailgun/db/sqlalchemy/models/__init__.py @@ -31,6 +31,7 @@ from nailgun.db.sqlalchemy.models.node import Role from nailgun.db.sqlalchemy.models.node import NodeAttributes from nailgun.db.sqlalchemy.models.node import NodeBondInterface from nailgun.db.sqlalchemy.models.node import NodeNICInterface +from nailgun.db.sqlalchemy.models.node import NodeGroup from nailgun.db.sqlalchemy.models.network import NetworkGroup from nailgun.db.sqlalchemy.models.network import IPAddr diff --git a/nailgun/nailgun/db/sqlalchemy/models/cluster.py b/nailgun/nailgun/db/sqlalchemy/models/cluster.py index b7c62e4869..e23bd2c837 100644 --- a/nailgun/nailgun/db/sqlalchemy/models/cluster.py +++ b/nailgun/nailgun/db/sqlalchemy/models/cluster.py @@ -31,6 +31,7 @@ from nailgun.db import db from nailgun.db.sqlalchemy.models.base import Base from nailgun.db.sqlalchemy.models.fields import JSON from nailgun.db.sqlalchemy.models.node import Node +from nailgun.db.sqlalchemy.models.node import NodeGroup class ClusterChanges(Base): @@ -88,17 +89,21 @@ class Cluster(Base): # During cluster deletion sqlalchemy engine will set null # into cluster foreign key column of notification entity notifications = relationship("Notification", backref="cluster") - network_groups = relationship( - "NetworkGroup", + node_groups = relationship( + "NodeGroup", backref="cluster", - cascade="delete", - order_by="NetworkGroup.id" + cascade="delete" ) replaced_deployment_info = Column(JSON, default={}) replaced_provisioning_info = Column(JSON, default={}) is_customized = Column(Boolean, default=False) fuel_version = Column(Text, nullable=False) + def create_default_group(self): + ng = NodeGroup(cluster_id=self.id, name="default") + db().add(ng) + db().commit() + @property def changes(self): return [ @@ -128,6 +133,22 @@ class Cluster(Base): return False return True + @property + def default_group(self): + if not self.node_groups: + self.create_default_group() + return [g.id for g in self.node_groups if g.name == "default"][0] + + def get_default_group(self): + return [g for g in self.node_groups if g.name == "default"][0] + + @property + def network_groups(self): + net_list = [] + for ng in self.node_groups: + net_list.extend(ng.networks) + return net_list + class Attributes(Base): __tablename__ = 'attributes' diff --git a/nailgun/nailgun/db/sqlalchemy/models/network.py b/nailgun/nailgun/db/sqlalchemy/models/network.py index 13d1c787ea..c69dd5aca4 100644 --- a/nailgun/nailgun/db/sqlalchemy/models/network.py +++ b/nailgun/nailgun/db/sqlalchemy/models/network.py @@ -65,7 +65,7 @@ class NetworkGroup(Base): # can be nullable only for fuelweb admin net release = Column(Integer, ForeignKey('releases.id')) # can be nullable only for fuelweb admin net - cluster_id = Column(Integer, ForeignKey('clusters.id')) + group_id = Column(Integer, ForeignKey('nodegroups.id'), nullable=True) vlan_start = Column(Integer) cidr = Column(String(25)) gateway = Column(String(25)) diff --git a/nailgun/nailgun/db/sqlalchemy/models/node.py b/nailgun/nailgun/db/sqlalchemy/models/node.py index 13af271054..4fe5d150c9 100755 --- a/nailgun/nailgun/db/sqlalchemy/models/node.py +++ b/nailgun/nailgun/db/sqlalchemy/models/node.py @@ -67,12 +67,26 @@ class Role(Base): name = Column(String(50), nullable=False) +class NodeGroup(Base): + __tablename__ = 'nodegroups' + id = Column(Integer, primary_key=True) + cluster_id = Column(Integer, ForeignKey('clusters.id')) + name = Column(String(50), nullable=False) + nodes = relationship("Node") + networks = relationship( + "NetworkGroup", + backref="nodegroup", + cascade="delete" + ) + + class Node(Base): __tablename__ = 'nodes' id = Column(Integer, primary_key=True) uuid = Column(String(36), nullable=False, default=lambda: str(uuid.uuid4()), unique=True) cluster_id = Column(Integer, ForeignKey('clusters.id')) + group_id = Column(Integer, ForeignKey('nodegroups.id'), nullable=True) name = Column(Unicode(100)) status = Column( Enum(*consts.NODE_STATUSES, name='node_status'), diff --git a/nailgun/nailgun/fixtures/admin_network.json b/nailgun/nailgun/fixtures/admin_network.json index 8b70d9ad50..b922e2e951 100644 --- a/nailgun/nailgun/fixtures/admin_network.json +++ b/nailgun/nailgun/fixtures/admin_network.json @@ -7,8 +7,8 @@ "cidr": "{{settings.ADMIN_NETWORK['cidr']}}", "vlan_start": null, "meta": { - "use_gateway": false, - "notation": "cidr", + "use_gateway": true, + "notation": "ip_ranges", "render_type": null, "render_addr_mask": null, "map_priority": 0, diff --git a/nailgun/nailgun/network/checker.py b/nailgun/nailgun/network/checker.py index 54cc09a2a6..6ee1532e17 100644 --- a/nailgun/nailgun/network/checker.py +++ b/nailgun/nailgun/network/checker.py @@ -66,6 +66,8 @@ class NetworkCheck(object): if data_net.get('meta'): data_net.pop('meta') net.update(data_net) + if data_net.get('name') == 'fuelweb_admin': + net.update(name='admin (PXE)') break else: raise errors.NetworkCheckError( @@ -168,8 +170,8 @@ class NetworkCheck(object): "errors": ["cidr"] }) # Check for intersection with floating ranges - nets_w_cidr = filter(lambda n: n['meta']['notation'] == 'cidr', - self.networks) + nets_w_cidr = [n for n in self.networks + if n.get('cidr') and n['name'] != 'public'] fl_ranges = [netaddr.IPRange(v[0], v[1]) for v in self.network_config['floating_ranges']] for net_vs_range in product(nets_w_cidr, fl_ranges): diff --git a/nailgun/nailgun/network/manager.py b/nailgun/nailgun/network/manager.py index 107d51b0ec..c2c88491c8 100644 --- a/nailgun/nailgun/network/manager.py +++ b/nailgun/nailgun/network/manager.py @@ -25,6 +25,7 @@ from netaddr import IPRange from sqlalchemy.orm import joinedload from sqlalchemy.sql import not_ +from sqlalchemy.sql import or_ from nailgun import objects @@ -36,10 +37,12 @@ from nailgun.db.sqlalchemy.models import NetworkGroup from nailgun.db.sqlalchemy.models import NetworkNICAssignment from nailgun.db.sqlalchemy.models import Node from nailgun.db.sqlalchemy.models import NodeBondInterface +from nailgun.db.sqlalchemy.models import NodeGroup from nailgun.db.sqlalchemy.models import NodeNICInterface from nailgun.errors import errors from nailgun.logger import logger from nailgun.utils.zabbix import ZabbixManager +from nailgun.settings import settings class NetworkManager(object): @@ -61,26 +64,33 @@ class NetworkManager(object): db().commit() @classmethod - def get_admin_network_group_id(cls): + def get_admin_network_group_id(cls, node_id=None): """Method for receiving Admin NetworkGroup ID. :type fail_if_not_found: bool :returns: Admin NetworkGroup ID or None. :raises: errors.AdminNetworkNotFound """ - return cls.get_admin_network_group().id + return cls.get_admin_network_group(node_id=node_id).id @classmethod - def get_admin_network_group(cls): + def get_admin_network_group(cls, node_id=None): """Method for receiving Admin NetworkGroup. :type fail_if_not_found: bool :returns: Admin NetworkGroup or None. :raises: errors.AdminNetworkNotFound """ - admin_ng = db().query(NetworkGroup).filter_by( - name="fuelweb_admin" - ).first() + admin_ng = None + admin_ngs = db().query(NetworkGroup).filter_by( + name="fuelweb_admin", + ) + if node_id: + node_db = db().query(Node).get(node_id) + admin_ng = admin_ngs.filter_by(group_id=node_db.group_id).first() + + admin_ng = admin_ng or admin_ngs.filter_by(group_id=None).first() + if not admin_ng: raise errors.AdminNetworkNotFound() return admin_ng @@ -112,13 +122,12 @@ class NetworkManager(object): :type num: int :returns: None """ - admin_net_id = cls.get_admin_network_group_id() - admin_net = db().query(NetworkGroup).get(admin_net_id) - # Check which nodes need ips nodes_need_ips = [] for node in nodes: node_id = node.id + admin_net = cls.get_admin_network_group(node_id) + admin_net_id = admin_net.id node_admin_ips = db().query(IPAddr).filter_by( node=node_id, network=admin_net_id) logger.debug(u"Trying to assign admin ip: node=%s", node_id) @@ -161,11 +170,10 @@ class NetworkManager(object): ) ) - network = db().query(NetworkGroup).\ - filter(NetworkGroup.cluster_id == cluster_id).\ - filter_by(name=network_name).first() + network_groups = db().query(NetworkGroup).\ + filter_by(name=network_name) - if not network: + if not network_groups: raise errors.AssignIPError( u"Network '%s' for cluster_id=%s not found." % (network_name, cluster_id) @@ -179,6 +187,14 @@ class NetworkManager(object): if network_name == 'public' and \ not objects.Node.should_have_public(node): continue + group_id = node.group_id or cluster.default_group + + network = network_groups.filter( + or_( + NetworkGroup.group_id == group_id, + NetworkGroup.group_id == None # flake8: noqa + ) + ).first() node_ips = imap( lambda i: i.ip_addr, @@ -248,14 +264,22 @@ class NetworkManager(object): if not cluster: raise Exception(u"Cluster id='%s' not found" % cluster_id) + group_id = None + for node in cluster.nodes: + if 'controller' in node.all_roles or \ + 'primary-controller' in node.all_roles: + group_id = node.group_id + break + + if not group_id: + group_id = cluster.default_group + network = db().query(NetworkGroup).\ - filter(NetworkGroup.cluster_id == cluster_id).\ - filter_by(name=network_name).first() + filter_by(name=network_name, group_id=group_id).first() if not network: raise Exception(u"Network '%s' for cluster_id=%s not found." % (network_name, cluster_id)) - admin_net_id = cls.get_admin_network_group_id() cluster_ips = [ne.ip_addr for ne in db().query(IPAddr).filter_by( network=network.id, @@ -365,7 +389,7 @@ class NetworkManager(object): ips = ips.filter_by(network=network_id) try: - admin_net_id = cls.get_admin_network_group_id() + admin_net_id = cls.get_admin_network_group_id(node_id=node_id) except errors.AdminNetworkNotFound: admin_net_id = None if admin_net_id: @@ -393,7 +417,12 @@ class NetworkManager(object): networks metadata """ nics = [] - ngs = node.cluster.network_groups + [cls.get_admin_network_group()] + group_id = node.group_id + if not group_id: + group_id = node.cluster.default_group + + node_group = db().query(NodeGroup).get(group_id) + ngs = node_group.networks + [cls.get_admin_network_group(node.id)] ngs_by_id = dict((ng.id, ng) for ng in ngs) # sort Network Groups ids by map_priority to_assign_ids = list( @@ -403,7 +432,7 @@ class NetworkManager(object): key=lambda x: x[1]))[0] ) ng_ids = set(ng.id for ng in ngs) - ng_wo_admin_ids = ng_ids ^ set([cls.get_admin_network_group_id()]) + ng_wo_admin_ids = ng_ids ^ set([cls.get_admin_network_group_id(node.id)]) for nic in node.nic_interfaces: nic_dict = { "id": nic.id, @@ -483,7 +512,14 @@ class NetworkManager(object): :type node: Node :returns: List of network groups for cluster node belongs to. """ - return node.cluster.network_groups + if node.group_id: + return db().query(NetworkGroup).filter_by( + group_id=node.group_id, + ).filter( + NetworkGroup.name != 'fuelweb_admin' + ).order_by(NetworkGroup.id).all() + else: + return node.cluster.network_groups @classmethod def get_node_networkgroups_ids(cls, node): @@ -494,23 +530,27 @@ class NetworkManager(object): @classmethod def _get_admin_node_network(cls, node): - net = cls.get_admin_network_group() + node_db = db().query(Node).get(node) + net = cls.get_admin_network_group(node) net_cidr = IPNetwork(net.cidr) ip_addr = cls.get_admin_ip_for_node(node) + if ip_addr: + ip_addr = "{0}/{1}".format(ip_addr, net_cidr.prefixlen) + return { 'name': net.name, + 'cidr': net.cidr, 'vlan': net.vlan_start, - 'ip': "{0}/{1}".format(ip_addr, net_cidr.prefixlen), + 'ip': ip_addr, 'netmask': str(net_cidr.netmask), 'brd': str(net_cidr.broadcast), 'gateway': net.gateway, - 'dev': node.admin_interface.name + 'dev': node_db.admin_interface.name } @classmethod def get_node_network_by_netname(cls, node, netname): networks = cls.get_node_networks(node) - networks.append(cls._get_admin_node_network(node)) return filter( lambda n: n['name'] == netname, networks)[0] @@ -529,6 +569,7 @@ class NetworkManager(object): prefix = str(IPNetwork(net.cidr).prefixlen) return { 'name': net.name, + 'cidr': net.cidr, 'vlan': cls.get_network_vlan(net, node_db.cluster), 'ip': ip.ip_addr + '/' + prefix, 'netmask': str(IPNetwork(net.cidr).netmask), @@ -539,6 +580,7 @@ class NetworkManager(object): @classmethod def _get_network_data_wo_ip(cls, node_db, interface, net): return {'name': net.name, + 'cidr': net.cidr, 'vlan': cls.get_network_vlan(net, node_db.cluster), 'dev': interface.name} @@ -548,27 +590,27 @@ class NetworkManager(object): if net.name != 'fuelweb_admin') @classmethod - def get_node_networks(cls, node_db): - cluster_db = node_db.cluster + def get_node_networks(cls, node): + cluster_db = node.cluster if cluster_db is None: # Node doesn't belong to any cluster, so it should not have nets return [] network_data = [] - for interface in node_db.interfaces: + for interface in node.interfaces: networks_wo_admin = cls._get_networks_except_admin( interface.assigned_networks_list) for net in networks_wo_admin: - ip = cls._get_ip_by_network_name(node_db, net.name) + ip = cls._get_ip_by_network_name(node, net.name) if ip is not None: network_data.append(cls._get_network_data_with_ip( - node_db, interface, net, ip)) + node, interface, net, ip)) else: if not cls.fixed_and_vlan_manager(net, cluster_db): network_data.append(cls._get_network_data_wo_ip( - node_db, interface, net)) + node, interface, net)) - network_data.append(cls._get_admin_network(node_db)) + network_data.append(cls._get_admin_node_network(node.id)) return network_data @@ -678,7 +720,7 @@ class NetworkManager(object): admin_interface = None for interface in interfaces: ip_addr = interface.get('ip') - if cls.is_ip_belongs_to_admin_subnet(ip_addr): + if cls.is_ip_belongs_to_admin_subnet(ip_addr, node.id): # Interface was founded admin_interface = interface break @@ -702,8 +744,8 @@ class NetworkManager(object): ) @classmethod - def is_ip_belongs_to_admin_subnet(cls, ip_addr): - admin_cidr = cls.get_admin_network_group().cidr + def is_ip_belongs_to_admin_subnet(cls, ip_addr, node_id=None): + admin_cidr = cls.get_admin_network_group(node_id).cidr if ip_addr and IPAddress(ip_addr) in IPNetwork(admin_cidr): return True return False @@ -760,25 +802,26 @@ class NetworkManager(object): def get_admin_ip_for_node(cls, node): """Returns first admin IP address for node """ - admin_net_id = cls.get_admin_network_group_id() + admin_net_id = cls.get_admin_network_group_id(node_id=node) admin_ip = db().query(IPAddr).order_by( IPAddr.id ).filter_by( - node=node.id + node=node ).filter_by( network=admin_net_id ).first() - return admin_ip.ip_addr + + return getattr(admin_ip, 'ip_addr', None) @classmethod def get_admin_ips_for_interfaces(cls, node): """Returns mapping admin {"inteface name" => "admin ip"} """ - admin_net_id = cls.get_admin_network_group_id() + admin_net_id = cls.get_admin_network_group_id(node) admin_ips = set([ i.ip_addr for i in db().query(IPAddr). order_by(IPAddr.id). - filter_by(node=node.id). + filter_by(node=node). filter_by(network=admin_net_id)]) interfaces_names = sorted(set([ @@ -789,8 +832,14 @@ class NetworkManager(object): @classmethod def _get_admin_network(cls, node): """Returns dict with admin network.""" + + net = cls.get_admin_network_group(node_id=node) return { - 'name': 'admin', + 'id': net.id, + 'cidr': net.cidr, + 'name': net.name, + 'gateway': net.gateway, + 'vlan': net.vlan_start, 'dev': cls.get_admin_interface(node).name } @@ -830,7 +879,8 @@ class NetworkManager(object): @classmethod def _get_ip_by_network_name(cls, node, network_name): for ip in node.ip_addrs: - if ip.network_data.name == network_name: + ng = ip.network_data + if ng.name == network_name and ng.group_id == node.group_id: return ip return None @@ -937,7 +987,23 @@ class NetworkManager(object): db().commit() @classmethod - def create_network_groups(cls, cluster_id, neutron_segment_type): + def create_admin_network_group(cls, cluster_id, group_id): + cluster_db = objects.Cluster.get_by_uid(cluster_id) + admin_ng = cls.get_admin_network_group() + new_admin = NetworkGroup( + release=cluster_db.release.id, + name='fuelweb_admin', + cidr='9.9.9.0/24', + gateway='9.9.9.1', + group_id=group_id, + vlan_start=None, + meta=admin_ng.meta + ) + db().add(new_admin) + db().flush() + + @classmethod + def create_network_groups(cls, cluster_id, neutron_segment_type, gid=None): """Method for creation of network groups for cluster. :param cluster_id: Cluster database ID. @@ -945,6 +1011,7 @@ class NetworkManager(object): :returns: None """ cluster_db = objects.Cluster.get_by_uid(cluster_id) + group_id = gid or cluster_db.default_group networks_metadata = cluster_db.release.networks_metadata networks_list = networks_metadata[cluster_db.net_provider]["networks"] used_nets = [IPNetwork(cls.get_admin_network_group().cidr)] @@ -992,7 +1059,7 @@ class NetworkManager(object): name=net['name'], cidr=str(cidr) if cidr else None, gateway=gw, - cluster_id=cluster_id, + group_id=group_id, vlan_start=vlan_start, meta=net ) @@ -1025,7 +1092,7 @@ class NetworkManager(object): if ng_db.meta.get("notation"): cls.cleanup_network_group(ng_db) - objects.Cluster.add_pending_changes(ng_db.cluster, 'networks') + objects.Cluster.add_pending_changes(cluster, 'networks') @classmethod def update(cls, cluster, network_configuration): @@ -1054,3 +1121,28 @@ class NetworkManager(object): data.get('net_l23_provider')) elif cluster.net_provider == 'nova_network': cls.create_nova_network_config(cluster) + + @classmethod + def get_default_gateway(cls, node_id): + return cls.get_admin_network_group(node_id).gateway \ + or settings.MASTER_IP + + @classmethod + def get_networks_not_on_node(cls, node): + node_net = [(n['name'], n['cidr']) + for n in cls.get_node_networks(node) if n.get('cidr')] + all_nets = [(n.name, n.cidr) + for n in node.cluster.network_groups if n.cidr] + + if node.group_id != node.cluster.default_group: + admin_net = cls.get_admin_network_group() + all_nets.append((admin_net.name, admin_net.cidr)) + + other_nets = set(all_nets) ^ set(node_net) + output = {} + for name, cidr in other_nets: + if name not in output: + output[name] = [] + output[name].append(cidr) + + return output diff --git a/nailgun/nailgun/objects/__init__.py b/nailgun/nailgun/objects/__init__.py index 4225145993..ef75381b02 100644 --- a/nailgun/nailgun/objects/__init__.py +++ b/nailgun/nailgun/objects/__init__.py @@ -41,5 +41,8 @@ from nailgun.objects.capacity import CapacityLog from nailgun.objects.master_node_settings import MasterNodeSettings +from nailgun.objects.node_group import NodeGroup +from nailgun.objects.node_group import NodeGroupCollection + from nailgun.objects.plugin import Plugin from nailgun.objects.plugin import PluginCollection diff --git a/nailgun/nailgun/objects/cluster.py b/nailgun/nailgun/objects/cluster.py index 05eacbb706..5c530bd7b9 100644 --- a/nailgun/nailgun/objects/cluster.py +++ b/nailgun/nailgun/objects/cluster.py @@ -173,6 +173,7 @@ class Cluster(NailgunObject): data["fuel_version"] = settings.VERSION["release"] new_cluster = super(Cluster, cls).create(data) + new_cluster.create_default_group() cls.create_attributes(new_cluster) diff --git a/nailgun/nailgun/objects/node.py b/nailgun/nailgun/objects/node.py index 72e481b604..f12a6b8138 100644 --- a/nailgun/nailgun/objects/node.py +++ b/nailgun/nailgun/objects/node.py @@ -22,6 +22,8 @@ import traceback from datetime import datetime +from netaddr import IPAddress +from netaddr import IPNetwork from sqlalchemy.orm import joinedload from sqlalchemy.orm import subqueryload_all @@ -66,6 +68,7 @@ class Node(NailgunObject): "type": "string", "enum": list(consts.NODE_STATUSES) }, + "group_id": {"type": "number"}, "meta": {"type": "object"}, "mac": {"type": "string"}, "fqdn": {"type": "string"}, @@ -222,6 +225,24 @@ class Node(NailgunObject): cls.create_discover_notification(new_node) return new_node + @classmethod + def assign_group(cls, instance): + if instance.group_id is None and instance.ip: + admin_ngs = db().query(models.NetworkGroup).filter_by( + name="fuelweb_admin") + ip = IPAddress(instance.ip) + + for ng in admin_ngs: + if ip in IPNetwork(ng.cidr): + instance.group_id = ng.group_id + break + + if not instance.group_id: + instance.group_id = instance.cluster.default_group + + db().add(instance) + db().flush() + @classmethod def create_attributes(cls, instance): """Create attributes for Node instance @@ -401,6 +422,10 @@ class Node(NailgunObject): if new_meta: instance.update_meta(new_meta) + # The call to update_interfaces will execute a select query for + # the current instance. This appears to overwrite the object in the + # current session and we lose the meta changes. + db().flush() # smarter check needed cls.update_interfaces(instance) @@ -424,6 +449,15 @@ class Node(NailgunObject): cluster_changed = True cls.add_into_cluster(instance, new_cluster_id) + if "group_id" in data: + new_group_id = data.pop("group_id") + if instance.group_id != new_group_id: + nm = Cluster.get_network_manager(instance.cluster) + nm.clear_assigned_networks(instance) + nm.clear_bond_configuration(instance) + instance.group_id = new_group_id + cls.add_into_cluster(instance, instance.cluster_id) + # calculating flags roles_changed = ( roles is not None and set(roles) != set(instance.roles) @@ -491,7 +525,7 @@ class Node(NailgunObject): #(dshulyak) change this verification to NODE_STATUSES.deploying # after we will reuse ips from dhcp range netmanager = Cluster.get_network_manager() - admin_ng = netmanager.get_admin_network_group() + admin_ng = netmanager.get_admin_network_group(instance.id) if data.get('ip') and not netmanager.is_same_network(data['ip'], admin_ng.cidr): logger.debug( @@ -582,7 +616,7 @@ class Node(NailgunObject): """ instance.cluster_id = cluster_id db().flush() - + cls.assign_group(instance) network_manager = Cluster.get_network_manager(instance.cluster) network_manager.assign_networks_by_default(instance) cls.add_pending_change(instance, consts.CLUSTER_CHANGES.interfaces) @@ -638,6 +672,7 @@ class Node(NailgunObject): cls.update_pending_roles(instance, []) cls.remove_replaced_params(instance) instance.cluster_id = None + instance.group_id = None instance.kernel_params = None instance.reset_name_to_default() db().flush() diff --git a/nailgun/nailgun/objects/node_group.py b/nailgun/nailgun/objects/node_group.py new file mode 100644 index 0000000000..fb8c59c0d0 --- /dev/null +++ b/nailgun/nailgun/objects/node_group.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- + +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from nailgun.objects.serializers.node_group import NodeGroupSerializer + +from nailgun.db import db +from nailgun.db.sqlalchemy.models import NodeGroup as DBNodeGroup +from nailgun.errors import errors +from nailgun.objects import Cluster +from nailgun.objects import NailgunCollection +from nailgun.objects import NailgunObject + + +class NodeGroup(NailgunObject): + + model = DBNodeGroup + serializer = NodeGroupSerializer + + schema = { + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "NodeGroup", + "description": "Serialized NodeGroup object", + "type": "object", + "properties": { + "id": {"type": "number"}, + "cluster_id": {"type": "number"}, + "name": {"type": "string"} + } + } + + @classmethod + def create(cls, data): + new_group = super(NodeGroup, cls).create(data) + try: + cluster = Cluster.get_by_uid(new_group.cluster_id) + nm = Cluster.get_network_manager(cluster) + nst = cluster.network_config.segmentation_type + nm.create_network_groups(new_group.cluster_id, nst, + gid=new_group.id) + nm.create_admin_network_group(new_group.cluster_id, new_group.id) + except ( + errors.OutOfVLANs, + errors.OutOfIPs, + errors.NoSuitableCIDR, + errors.InvalidNetworkPool + ) as exc: + db().delete(new_group) + raise errors.CannotCreate(exc.message) + + db().flush() + return new_group + + +class NodeGroupCollection(NailgunCollection): + + single = NodeGroup + + @classmethod + def get_by_cluster_id(cls, cluster_id): + if cluster_id == '': + return cls.filter_by(cluster_id=None) + return cls.filter_by(cluster_id=cluster_id) diff --git a/nailgun/nailgun/objects/serializers/network_configuration.py b/nailgun/nailgun/objects/serializers/network_configuration.py index 1349cc9845..edc33dee9e 100644 --- a/nailgun/nailgun/objects/serializers/network_configuration.py +++ b/nailgun/nailgun/objects/serializers/network_configuration.py @@ -14,14 +14,13 @@ # License for the specific language governing permissions and limitations # under the License. -from nailgun import objects - +from nailgun.network.manager import NetworkManager from nailgun.objects.serializers.base import BasicSerializer class NetworkConfigurationSerializer(BasicSerializer): - fields = ('id', 'cluster_id', 'name', 'cidr', + fields = ('id', 'group_id', 'name', 'cidr', 'gateway', 'vlan_start', 'meta') @classmethod @@ -39,21 +38,19 @@ class NetworkConfigurationSerializer(BasicSerializer): @classmethod def serialize_net_groups_and_vips(cls, cluster): result = {} - net_manager = objects.Cluster.get_network_manager(cluster) + net_manager = NetworkManager + nets = cluster.network_groups + [net_manager.get_admin_network_group()] + result['networks'] = map( cls.serialize_network_group, - cluster.network_groups - ) - result['networks'].append( - cls.serialize_network_group( - net_manager.get_admin_network_group() - ) + nets ) if cluster.is_ha_mode: for ng in cluster.network_groups: if ng.meta.get("assign_vip"): result['{0}_vip'.format(ng.name)] = \ net_manager.assign_vip(cluster.id, ng.name) + return result @classmethod diff --git a/nailgun/nailgun/objects/serializers/node.py b/nailgun/nailgun/objects/serializers/node.py index 3aad63aec1..9b3fc593eb 100644 --- a/nailgun/nailgun/objects/serializers/node.py +++ b/nailgun/nailgun/objects/serializers/node.py @@ -41,7 +41,8 @@ class NodeSerializer(BasicSerializer): 'error_type', 'online', 'cluster', - 'network_data' + 'network_data', + 'group_id' ) diff --git a/nailgun/nailgun/objects/serializers/node_group.py b/nailgun/nailgun/objects/serializers/node_group.py new file mode 100644 index 0000000000..530e614723 --- /dev/null +++ b/nailgun/nailgun/objects/serializers/node_group.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nailgun.objects.serializers.base import BasicSerializer + + +class NodeGroupSerializer(BasicSerializer): + + fields = ( + "id", + "cluster", + "name" + ) diff --git a/nailgun/nailgun/orchestrator/deployment_serializers.py b/nailgun/nailgun/orchestrator/deployment_serializers.py index 8bf002df59..80c38fbaca 100644 --- a/nailgun/nailgun/orchestrator/deployment_serializers.py +++ b/nailgun/nailgun/orchestrator/deployment_serializers.py @@ -57,7 +57,7 @@ class NetworkDeploymentSerializer(object): def get_common_attrs(cls, cluster, attrs): """Cluster network attributes.""" common = cls.network_provider_cluster_attrs(cluster) - common.update(cls.network_ranges(cluster)) + common.update(cls.network_ranges(cluster.default_group)) common.update({'master_ip': settings.MASTER_IP}) common['nodes'] = deepcopy(attrs['nodes']) @@ -93,11 +93,11 @@ class NetworkDeploymentSerializer(object): raise NotImplementedError() @classmethod - def network_ranges(cls, cluster): + def network_ranges(cls, group_id): """Returns ranges for network groups - except range for public network + except range for public network for each node """ - ng_db = db().query(NetworkGroup).filter_by(cluster_id=cluster.id).all() + ng_db = db().query(NetworkGroup).filter_by(group_id=group_id).all() attrs = {} for net in ng_db: net_name = net.name + '_network_range' @@ -138,11 +138,13 @@ class NetworkDeploymentSerializer(object): def get_admin_ip_w_prefix(node): """Getting admin ip and assign prefix from admin network.""" network_manager = objects.Node.get_network_manager(node) - admin_ip = network_manager.get_admin_ip_for_node(node) + admin_ip = network_manager.get_admin_ip_for_node(node.id) admin_ip = IPNetwork(admin_ip) # Assign prefix from admin network - admin_net = IPNetwork(network_manager.get_admin_network_group().cidr) + admin_net = IPNetwork( + network_manager.get_admin_network_group(node.id).cidr + ) admin_ip.prefixlen = admin_net.prefixlen return str(admin_ip) @@ -231,11 +233,12 @@ class NovaNetworkDeploymentSerializer(NetworkDeploymentSerializer): if network.get('ip'): interface['ipaddr'].append(network.get('ip')) - if network_name == 'admin': + if network_name == 'fuelweb_admin': admin_ip_addr = cls.get_admin_ip_w_prefix(node) interface['ipaddr'].append(admin_ip_addr) elif network_name == 'public' and network.get('gateway'): interface['gateway'] = network['gateway'] + interface['default_gateway'] = True for if_name, if_data in interfaces.iteritems(): if len(if_data['ipaddr']) == 0: @@ -513,9 +516,9 @@ class NeutronNetworkDeploymentSerializer(NetworkDeploymentSerializer): # We have to add them after br-ethXX bridges because it is the way # to provide a right ordering of ifdown/ifup operations with # IP interfaces. - brnames = ['br-mgmt', 'br-storage', 'br-fw-admin'] - if objects.Node.should_have_public(node): - brnames.append('br-ex') + brnames = ['br-ex', 'br-mgmt', 'br-storage', 'br-fw-admin'] + if not objects.Node.should_have_public(node): + brnames.pop(0) for brname in brnames: attrs['transformations'].append({ @@ -532,18 +535,37 @@ class NeutronNetworkDeploymentSerializer(NetworkDeploymentSerializer): if objects.Node.should_have_public(node): netgroup_mapping.append(('public', 'br-ex')) + # Include information about all subnets that don't belong to this node. + # This is used during deployment to configure routes to all other + # networks in the environment. + other_nets = nm.get_networks_not_on_node(node) + netgroups = {} for ngname, brname in netgroup_mapping: # Here we get a dict with network description for this particular # node with its assigned IPs and device names for each network. netgroup = nm.get_node_network_by_netname(node, ngname) - attrs['endpoints'][brname]['IP'] = [netgroup['ip']] + if netgroup.get('ip'): + attrs['endpoints'][brname]['IP'] = [netgroup['ip']] + if netgroup.get('gateway'): + attrs['endpoints'][brname]['gateway'] = netgroup['gateway'] + + attrs['endpoints'][brname]['other_nets'] = \ + other_nets.get(ngname, []) + netgroups[ngname] = netgroup + if objects.Node.should_have_public(node): attrs['endpoints']['br-ex']['gateway'] = \ netgroups['public']['gateway'] else: - attrs['endpoints']['br-fw-admin']['gateway'] = settings.MASTER_IP + gw = nm.get_default_gateway(node.id) + attrs['endpoints']['br-fw-admin']['gateway'] = gw + + for brname in brnames: + if attrs['endpoints'][brname].get('gateway'): + attrs['endpoints'][brname]['default_gateway'] = True + break # Connect interface bridges to network bridges. for ngname, brname in netgroup_mapping: @@ -627,7 +649,7 @@ class NeutronNetworkDeploymentSerializer(NetworkDeploymentSerializer): NetworkGroup.cidr, NetworkGroup.gateway ).filter_by( - cluster_id=cluster.id, + group_id=cluster.default_group, name='public' ).first() join_range = lambda r: (":".join(map(str, r)) if r else None) @@ -945,6 +967,10 @@ class DeploymentMultinodeSerializer(object): node_attrs.update(self.get_net_provider_serializer( node.cluster).get_node_attrs(node)) + node_attrs.update( + self.get_net_provider_serializer(node.cluster). + network_ranges(node.group_id) + ) node_attrs.update(self.get_image_cache_max_size(node)) node_attrs.update(self.generate_test_vm_image_data(node)) return node_attrs diff --git a/nailgun/nailgun/orchestrator/provisioning_serializers.py b/nailgun/nailgun/orchestrator/provisioning_serializers.py index fc92870466..272185e195 100644 --- a/nailgun/nailgun/orchestrator/provisioning_serializers.py +++ b/nailgun/nailgun/orchestrator/provisioning_serializers.py @@ -147,6 +147,13 @@ class ProvisioningSerializer(object): 'mlnx_iser_enabled': cluster_attrs['storage']['iser'], }) + net_manager = objects.Node.get_network_manager(node) + gw = net_manager.get_default_gateway(node.id) + serialized_node['ks_meta'].update({'gw': gw}) + serialized_node['ks_meta'].update( + {'admin_net': net_manager.get_admin_network_group(node.id).cidr} + ) + serialized_node.update(cls.serialize_interfaces(node)) return serialized_node @@ -156,9 +163,9 @@ class ProvisioningSerializer(object): interfaces = {} interfaces_extra = {} net_manager = objects.Node.get_network_manager(node) - admin_ip = net_manager.get_admin_ip_for_node(node) + admin_ip = net_manager.get_admin_ip_for_node(node.id) admin_netmask = str(netaddr.IPNetwork( - net_manager.get_admin_network_group().cidr + net_manager.get_admin_network_group(node.id).cidr ).netmask) for interface in node.nic_interfaces: diff --git a/nailgun/nailgun/task/manager.py b/nailgun/nailgun/task/manager.py index d200d4bf5d..cdcbdf5511 100644 --- a/nailgun/nailgun/task/manager.py +++ b/nailgun/nailgun/task/manager.py @@ -702,6 +702,16 @@ class VerifyNetworksTaskManager(TaskManager): db().commit() return task + if len(self.cluster.node_groups) > 1: + task.status = TASK_STATUSES.error + task.progress = 100 + task.message = ('Network verfiication is disabled for ' + 'environments containing more than one node ' + 'group.') + db().add(task) + db().commit() + return task + if self.cluster.status in self._blocking_statuses: task.status = TASK_STATUSES.error task.progress = 100 diff --git a/nailgun/nailgun/task/task.py b/nailgun/nailgun/task/task.py index c90dc1ca94..fcedd0764b 100644 --- a/nailgun/nailgun/task/task.py +++ b/nailgun/nailgun/task/task.py @@ -212,7 +212,7 @@ class ProvisionTask(object): admin_net_id = objects.Node.get_network_manager( node - ).get_admin_network_group_id() + ).get_admin_network_group_id(node.id) TaskHelper.prepare_syslog_dir(node, admin_net_id) db().commit() @@ -364,7 +364,7 @@ class StopDeploymentTask(object): 'slave_name': objects.Node.make_slave_name(n), 'admin_ip': objects.Node.get_network_manager( n - ).get_admin_ip_for_node(n) + ).get_admin_ip_for_node(n.id) } for n in nodes_to_stop ], "engine": { @@ -453,7 +453,7 @@ class BaseNetworkVerification(object): vlans = [] for ng in assigned_networks: # Handle FuelWeb admin network first. - if not ng.cluster_id: + if ng.group_id is None: vlans.append(0) continue data_ng = filter(lambda i: i['name'] == ng.name, diff --git a/nailgun/nailgun/test/base.py b/nailgun/nailgun/test/base.py index 2d4784739f..f2d64e335a 100644 --- a/nailgun/nailgun/test/base.py +++ b/nailgun/nailgun/test/base.py @@ -183,6 +183,7 @@ class Environment(object): ) else: cluster = Cluster.create(cluster_data) + cluster.create_default_group() db().commit() self.clusters.append(cluster) @@ -211,6 +212,7 @@ class Environment(object): node_data = { 'mac': mac, 'status': 'discover', + 'ip': '10.20.0.130', 'meta': default_metadata } if kwargs: diff --git a/nailgun/nailgun/test/integration/test_cluster_changes_handler.py b/nailgun/nailgun/test/integration/test_cluster_changes_handler.py index 34b49f1560..0be09998ac 100644 --- a/nailgun/nailgun/test/integration/test_cluster_changes_handler.py +++ b/nailgun/nailgun/test/integration/test_cluster_changes_handler.py @@ -80,7 +80,7 @@ class TestHandlers(BaseIntegrationTest): 'management_interface': 'eth0.101', 'fixed_interface': 'eth0.103', - 'admin_interface': 'eth1', + 'fuelweb_admin_interface': 'eth1', 'storage_interface': 'eth0.102', 'public_interface': 'eth0', 'floating_interface': 'eth0', @@ -202,7 +202,8 @@ class TestHandlers(BaseIntegrationTest): 'eth0': { 'interface': 'eth0', 'ipaddr': ['%s/24' % ips['public']], - 'gateway': '172.16.0.1'}, + 'gateway': '172.16.0.1', + 'default_gateway': True}, 'eth0.101': { 'interface': 'eth0.101', 'ipaddr': ['%s/24' % ips['internal']]}, @@ -296,6 +297,10 @@ class TestHandlers(BaseIntegrationTest): 'mlnx_vf_num': "16", 'mlnx_plugin_mode': "disabled", 'mlnx_iser_enabled': False, + 'gw': + self.env.network_manager.get_default_gateway(n.id), + 'admin_net': + self.env.network_manager.get_admin_network_group(n.id).cidr } } orchestrator_data = objects.Release.get_orchestrator_data_dict( @@ -310,7 +315,7 @@ class TestHandlers(BaseIntegrationTest): NetworkManager.assign_admin_ips([n]) - admin_ip = self.env.network_manager.get_admin_ip_for_node(n) + admin_ip = self.env.network_manager.get_admin_ip_for_node(n.id) for i in n.interfaces: if 'interfaces' not in pnd: @@ -609,7 +614,8 @@ class TestHandlers(BaseIntegrationTest): "br-mgmt": {"IP": [ips['management'] + "/24"]}, "br-ex": { "IP": [ips['public'] + "/24"], - "gateway": "172.16.0.1" + "default_gateway": True, + "gateway": "172.16.0.1", }, "br-storage": {"IP": [ips['storage'] + "/24"]}, "br-fw-admin": {"IP": [ips['admin']]}, @@ -636,6 +642,9 @@ class TestHandlers(BaseIntegrationTest): "action": "add-port", "bridge": u"br-eth1", "name": u"eth1"}, + { + "action": "add-br", + "name": "br-ex"}, { "action": "add-br", "name": "br-mgmt"}, @@ -645,9 +654,6 @@ class TestHandlers(BaseIntegrationTest): { "action": "add-br", "name": "br-fw-admin"}, - { - "action": "add-br", - "name": "br-ex"}, { "action": "add-patch", "bridges": [u"br-eth0", "br-storage"], @@ -746,7 +752,11 @@ class TestHandlers(BaseIntegrationTest): 'mlnx_vf_num': "16", 'mlnx_plugin_mode': "disabled", 'mlnx_iser_enabled': False, - 'image_data': cluster_attrs['provision']['image_data'] + 'image_data': cluster_attrs['provision']['image_data'], + 'gw': + self.env.network_manager.get_default_gateway(n.id), + 'admin_net': + self.env.network_manager.get_admin_network_group(n.id).cidr } } orchestrator_data = objects.Release.get_orchestrator_data_dict( @@ -761,7 +771,7 @@ class TestHandlers(BaseIntegrationTest): NetworkManager.assign_admin_ips([n]) - admin_ip = self.env.network_manager.get_admin_ip_for_node(n) + admin_ip = self.env.network_manager.get_admin_ip_for_node(n.id) for i in n.meta.get('interfaces', []): if 'interfaces' not in pnd: diff --git a/nailgun/nailgun/test/integration/test_cluster_collection_handlers.py b/nailgun/nailgun/test/integration/test_cluster_collection_handlers.py index da9f686924..bbafcefdd8 100644 --- a/nailgun/nailgun/test/integration/test_cluster_collection_handlers.py +++ b/nailgun/nailgun/test/integration/test_cluster_collection_handlers.py @@ -70,7 +70,7 @@ class TestHandlers(BaseIntegrationTest): for clstr in (cluster_db, cluster2_db): management_net = self.db.query(NetworkGroup).filter_by( name="management", - cluster_id=clstr.id + group_id=clstr.default_group ).first() NovaNetworkManager.update( clstr, @@ -95,11 +95,14 @@ class TestHandlers(BaseIntegrationTest): cluster2_nets = self._get_cluster_networks(cluster2["id"]) for net1, net2 in zip(cluster1_nets, cluster2_nets): - for f in ('cluster_id', 'id'): + for f in ('group_id', 'id'): del net1[f] del net2[f] - self.assertEqual(cluster1_nets, cluster2_nets) + cluster1_nets = sorted(cluster1_nets, key=lambda n: n['name']) + cluster2_nets = sorted(cluster2_nets, key=lambda n: n['name']) + + self.assertEquals(cluster1_nets, cluster2_nets) def test_cluster_creation_same_networks(self): cluster1_id = self.env.create_cluster(api=True)["id"] @@ -108,12 +111,12 @@ class TestHandlers(BaseIntegrationTest): cluster2_nets = self._get_cluster_networks(cluster2_id) for net1, net2 in zip(cluster1_nets, cluster2_nets): - for f in ('cluster_id', 'id'): + for f in ('group_id', 'id'): del net1[f] del net2[f] - cluster1_nets = sorted(cluster1_nets, key=lambda n: n['vlan_start']) - cluster2_nets = sorted(cluster2_nets, key=lambda n: n['vlan_start']) + cluster1_nets = sorted(cluster1_nets, key=lambda n: n['name']) + cluster2_nets = sorted(cluster2_nets, key=lambda n: n['name']) self.assertEqual(cluster1_nets, cluster2_nets) diff --git a/nailgun/nailgun/test/integration/test_horizon_url.py b/nailgun/nailgun/test/integration/test_horizon_url.py index 23ed8eaf68..cedc308b3f 100644 --- a/nailgun/nailgun/test/integration/test_horizon_url.py +++ b/nailgun/nailgun/test/integration/test_horizon_url.py @@ -41,7 +41,8 @@ class TestHorizonURL(BaseIntegrationTest): self.env.wait_ready(supertask, 60) network = self.db.query(NetworkGroup).\ - filter(NetworkGroup.cluster_id == self.env.clusters[0].id).\ + filter(NetworkGroup.group_id == + self.env.clusters[0].default_group).\ filter_by(name="public").first() lost_ips = self.db.query(IPAddr).filter_by( network=network.id, diff --git a/nailgun/nailgun/test/integration/test_mellanox_orchestrator_serializer.py b/nailgun/nailgun/test/integration/test_mellanox_orchestrator_serializer.py index cfc224708b..77dfce34d8 100644 --- a/nailgun/nailgun/test/integration/test_mellanox_orchestrator_serializer.py +++ b/nailgun/nailgun/test/integration/test_mellanox_orchestrator_serializer.py @@ -57,8 +57,7 @@ class TestMellanox(OrchestratorSerializerTestBase): iser_sttr = editable_attrs.setdefault('storage', {}) iser_sttr.setdefault('iser', {})['value'] = True network_group = self.db().query(NetworkGroup) - storage = network_group.filter_by(name="storage", - cluster_id=self.cluster_id) + storage = network_group.filter_by(name="storage") if iser_vlan: storage.update( {"vlan_start": iser_vlan}, synchronize_session="fetch") diff --git a/nailgun/nailgun/test/integration/test_network_configuration.py b/nailgun/nailgun/test/integration/test_network_configuration.py index 3f8fb52710..e93327ef80 100644 --- a/nailgun/nailgun/test/integration/test_network_configuration.py +++ b/nailgun/nailgun/test/integration/test_network_configuration.py @@ -45,7 +45,7 @@ class TestNovaNetworkConfigurationHandlerMultinode(BaseIntegrationTest): keys = [ 'name', - 'cluster_id', + 'group_id', 'vlan_start', 'cidr', 'id'] @@ -268,7 +268,7 @@ class TestNeutronNetworkConfigurationHandlerMultinode(BaseIntegrationTest): keys = [ 'name', - 'cluster_id', + 'group_id', 'vlan_start', 'cidr', 'id'] diff --git a/nailgun/nailgun/test/integration/test_network_manager.py b/nailgun/nailgun/test/integration/test_network_manager.py index d1c50fa8ce..ebff26c99b 100644 --- a/nailgun/nailgun/test/integration/test_network_manager.py +++ b/nailgun/nailgun/test/integration/test_network_manager.py @@ -58,7 +58,7 @@ class TestNetworkManager(BaseIntegrationTest): management_net = self.db.query(NetworkGroup).\ filter( - NetworkGroup.cluster_id == self.env.clusters[0].id + NetworkGroup.group_id == self.env.clusters[0].default_group ).filter_by( name='management' ).first() diff --git a/nailgun/nailgun/test/integration/test_node_collection_handlers.py b/nailgun/nailgun/test/integration/test_node_collection_handlers.py index 7c610c61ab..d9b0ab8a72 100644 --- a/nailgun/nailgun/test/integration/test_node_collection_handlers.py +++ b/nailgun/nailgun/test/integration/test_node_collection_handlers.py @@ -334,6 +334,14 @@ class TestHandlers(BaseIntegrationTest): # Set IP outside of admin network range on eth1 node.meta['interfaces'][1]['ip'] = '10.21.0.3' + self.app.put( + reverse('NodeAgentHandler'), + jsonutils.dumps({ + 'mac': node.mac, + 'meta': node.meta, + }), + headers=self.default_headers) + self.env.network_manager.update_interfaces_info(node) # node.mac == eth0 mac so eth0 should now be admin interface diff --git a/nailgun/nailgun/test/integration/test_orchestrator_serializer.py b/nailgun/nailgun/test/integration/test_orchestrator_serializer.py index 7921bfe3fb..3b89fccf8a 100644 --- a/nailgun/nailgun/test/integration/test_orchestrator_serializer.py +++ b/nailgun/nailgun/test/integration/test_orchestrator_serializer.py @@ -293,7 +293,8 @@ class TestNovaOrchestratorSerializer(OrchestratorSerializerTestBase): 'ipaddr': ['172.16.0.2/24', '192.168.0.1/24', '192.168.1.1/24'], - 'gateway': '172.16.0.1' + 'gateway': '172.16.0.1', + 'default_gateway': True } } self.datadiff(expected_interfaces, interfaces, ignore_keys=['ipaddr']) @@ -977,7 +978,7 @@ class TestNeutronOrchestratorSerializer(OrchestratorSerializerTestBase): public_ng = self.db.query(NetworkGroup).filter( NetworkGroup.name == 'public' ).filter( - NetworkGroup.cluster_id == cluster.id + NetworkGroup.group_id == cluster.default_group ).first() public_ng.gateway = test_gateway self.db.add(public_ng) diff --git a/nailgun/nailgun/test/integration/test_provisioning.py b/nailgun/nailgun/test/integration/test_provisioning.py index 20192f01ac..ac4266644f 100644 --- a/nailgun/nailgun/test/integration/test_provisioning.py +++ b/nailgun/nailgun/test/integration/test_provisioning.py @@ -67,6 +67,7 @@ class TestProvisioning(BaseIntegrationTest): ) cluster = self.env.clusters[0] objects.Cluster.clear_pending_changes(cluster) + self.env.network_manager.assign_ips(self.env.nodes, 'fuelweb_admin') self.env.network_manager.assign_ips(self.env.nodes, 'management') self.env.network_manager.assign_ips(self.env.nodes, 'storage') self.env.network_manager.assign_ips(self.env.nodes, 'public') diff --git a/nailgun/nailgun/test/integration/test_rpc_consumer.py b/nailgun/nailgun/test/integration/test_rpc_consumer.py index 1e4499181b..a6988eb4d9 100644 --- a/nailgun/nailgun/test/integration/test_rpc_consumer.py +++ b/nailgun/nailgun/test/integration/test_rpc_consumer.py @@ -1148,7 +1148,8 @@ class TestConsumer(BaseIntegrationTest): cluster_id=cluster_id ) networks = self.db.query(NetworkGroup).\ - filter(NetworkGroup.cluster_id == cluster_id).all() + filter(NetworkGroup.group_id == + self.env.clusters[0].default_group).all() vlans = [] for net in networks: @@ -1190,8 +1191,9 @@ class TestConsumer(BaseIntegrationTest): self.assertEqual(len(nots_db), 0) nets_db = self.db.query(NetworkGroup).\ - filter(NetworkGroup.cluster_id == cluster_id).all() - self.assertEqual(len(nets_db), 0) + filter(NetworkGroup.group_id == + self.env.clusters[0].default_group).all() + self.assertEquals(len(nets_db), 0) task_db = self.db.query(Task)\ .filter_by(cluster_id=cluster_id).all() @@ -1247,5 +1249,6 @@ class TestConsumer(BaseIntegrationTest): self.assertNotEqual(len(nots_db), 0) nets_db = self.db.query(NetworkGroup).\ - filter(NetworkGroup.cluster_id == cluster_db.id).all() + filter(NetworkGroup.group_id == + self.env.clusters[0].default_group).all() self.assertNotEqual(len(nets_db), 0) diff --git a/nailgun/nailgun/test/integration/test_stop_deployment.py b/nailgun/nailgun/test/integration/test_stop_deployment.py index e619f17d4d..f7b0594f33 100644 --- a/nailgun/nailgun/test/integration/test_stop_deployment.py +++ b/nailgun/nailgun/test/integration/test_stop_deployment.py @@ -80,7 +80,7 @@ class TestStopDeployment(BaseIntegrationTest): n["admin_ip"], objects.Node.get_network_manager( n_db - ).get_admin_ip_for_node(n_db) + ).get_admin_ip_for_node(n_db.id) ) @fake_tasks(recover_nodes=False, tick_interval=1) diff --git a/nailgun/nailgun/test/unit/test_logs_handlers.py b/nailgun/nailgun/test/unit/test_logs_handlers.py index 8a1d05ace9..d10201e30c 100644 --- a/nailgun/nailgun/test/unit/test_logs_handlers.py +++ b/nailgun/nailgun/test/unit/test_logs_handlers.py @@ -116,7 +116,8 @@ class TestLogs(BaseIntegrationTest): 'text2', ], ] - cluster = self.env.create_cluster(api=False) + self.env.create_cluster() + cluster = self.env.clusters[0] node = self.env.create_node(cluster_id=cluster.id, ip=node_ip) self._create_logfile_for_node(settings.LOGS[0], log_entries) self._create_logfile_for_node(settings.LOGS[1], log_entries, node) diff --git a/nailgun/nailgun/test/unit/test_node_assignment_handler.py b/nailgun/nailgun/test/unit/test_node_assignment_handler.py index e0243660f7..23938d404f 100644 --- a/nailgun/nailgun/test/unit/test_node_assignment_handler.py +++ b/nailgun/nailgun/test/unit/test_node_assignment_handler.py @@ -26,7 +26,10 @@ class TestAssignmentHandlers(BaseIntegrationTest): self.env.create( cluster_kwargs={"api": True}, nodes_kwargs=[ - {"cluster_id": None} + { + "cluster_id": None, + "api": True + } ] ) cluster = self.env.clusters[0] diff --git a/nailgun/nailgun/test/unit/test_node_deletion.py b/nailgun/nailgun/test/unit/test_node_deletion.py index ffd5da13ae..ae67a7f21b 100644 --- a/nailgun/nailgun/test/unit/test_node_deletion.py +++ b/nailgun/nailgun/test/unit/test_node_deletion.py @@ -58,7 +58,7 @@ class TestNodeDeletion(BaseIntegrationTest): self.assertEqual(node_try, None) management_net = self.db.query(NetworkGroup).\ - filter(NetworkGroup.cluster_id == cluster.id).filter_by( + filter(NetworkGroup.group_id == cluster.default_group).filter_by( name='management').first() ipaddrs = self.db.query(IPAddr).\ diff --git a/nailgun/nailgun/test/unit/test_node_disks.py b/nailgun/nailgun/test/unit/test_node_disks.py index 98803207a4..8b95f8f08c 100644 --- a/nailgun/nailgun/test/unit/test_node_disks.py +++ b/nailgun/nailgun/test/unit/test_node_disks.py @@ -373,7 +373,7 @@ class TestNodeDefaultsDisksHandler(BaseIntegrationTest): self.assertEqual(len(disk['volumes']), len(vgs)) def test_get_default_attrs(self): - self.env.create_node(api=True) + self.env.create_node(api=False) node_db = self.env.nodes[0] volumes_from_api = self.get(node_db.id) diff --git a/nailgun/nailgun/test/unit/test_node_groups.py b/nailgun/nailgun/test/unit/test_node_groups.py new file mode 100644 index 0000000000..63e7aa333d --- /dev/null +++ b/nailgun/nailgun/test/unit/test_node_groups.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- + +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json + +from nailgun.db import db +from nailgun.db.sqlalchemy.models import NetworkGroup +from nailgun.test.base import BaseIntegrationTest +from nailgun.test.base import reverse + + +class TestNodeGroups(BaseIntegrationTest): + + def setUp(self): + super(TestNodeGroups, self).setUp() + self.cluster = self.env.create_cluster( + api=False, + net_provider='neutron', + net_segment_type='gre' + ) + + def create_node_group(self): + resp = self.app.post( + reverse('NodeGroupCollectionHandler'), + json.dumps({'cluster_id': self.cluster['id'], 'name': 'test_ng'}), + headers=self.default_headers, + expect_errors=False + ) + + return resp + + def test_nodegroup_creation(self): + resp = self.create_node_group() + + self.assertEquals(resp.status_code, 201) + response = json.loads(resp.body) + self.assertEquals(response['cluster'], self.cluster['id']) + + def test_nodegroup_assignment(self): + self.env.create( + cluster_kwargs={ + 'api': True, + 'net_provider': 'neutron', + 'net_segment_type': 'gre' + }, + nodes_kwargs=[{ + 'roles': [], + 'pending_roles': ['controller'], + 'pending_addition': True, + 'api': True}] + ) + cluster = self.env.clusters[0] + node = self.env.nodes[0] + + resp = self.app.post( + reverse('NodeGroupCollectionHandler'), + json.dumps({'cluster_id': cluster['id'], 'name': 'test_ng'}), + headers=self.default_headers, + expect_errors=False + ) + + response = json.loads(resp.body) + ng_id = response['id'] + + resp = self.app.put( + reverse('NodeHandler', kwargs={'obj_id': node['id']}), + json.dumps({'group_id': ng_id}), + headers=self.default_headers, + expect_errors=False + ) + + response = json.loads(resp.body) + self.assertEquals(resp.status_code, 200) + self.assertEquals(node.group_id, ng_id) + + def test_nodegroup_create_network(self): + resp = self.create_node_group() + + response = json.loads(resp.body) + nets = db().query(NetworkGroup).filter_by(group_id=response['id']) + self.assertEquals(nets.count(), 4) + + def test_nodegroup_deletion(self): + resp = self.create_node_group() + response = json.loads(resp.body) + group_id = response['id'] + + self.app.delete( + reverse( + 'NodeGroupHandler', + kwargs={'obj_id': group_id} + ), + headers=self.default_headers, + expect_errors=False + ) + + nets = db().query(NetworkGroup).filter_by(group_id=response['id']) + self.assertEquals(nets.count(), 0) + + def test_nodegroup_invalid_segmentation_type(self): + cluster = self.env.create_cluster( + api=False, + net_provider='neutron', + net_segment_type='vlan' + ) + resp = self.app.post( + reverse('NodeGroupCollectionHandler'), + json.dumps({'cluster_id': cluster['id'], 'name': 'test_ng'}), + headers=self.default_headers, + expect_errors=True + ) + + self.assertEquals(resp.status_code, 403) diff --git a/nailgun/nailgun/test/unit/test_objects.py b/nailgun/nailgun/test/unit/test_objects.py index 6f52d43b1e..1b09358dd1 100644 --- a/nailgun/nailgun/test/unit/test_objects.py +++ b/nailgun/nailgun/test/unit/test_objects.py @@ -271,6 +271,7 @@ class TestNodeObject(BaseIntegrationTest): self.assertEqual(node_db.pending_roles, []) exclude_fields = [ + "group_id", "id", "mac", "meta",