From 7b19ba7f83036ca1081a05a47ea5f1c2f1120010 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 24 May 2017 07:42:54 -0500 Subject: [PATCH] Add support for defining groups in nodesets If users are expected to be able to use ansible content written for production, it is important to be able to define arbitrary groups of nodes in their inventory. For instance, a playbook to deploy OpenStack may want a groups called controller, compute, ceph-osd and ceph-monitor, but a job to test that playbook may want three nodes, one called compute, one called controller1 and one called controller2. For the test job, I would want to put controller1 in the ceph-osd group and controller1 and controller2 in the ceph-monitor group. nodepool does not need to know anything about these - they are just logical names the user is describing to make it into the inventory. There are currently no tests of the inventory we're writing out. The next patch adds a test to ensure that inventories are written out properly. Change-Id: I5555c86ffa96e6a43df5e46302f4e76840372999 --- zuul/configloader.py | 24 ++++++++++++++++++++++ zuul/executor/client.py | 4 +++- zuul/executor/server.py | 5 +++++ zuul/model.py | 44 ++++++++++++++++++++++++++++++++++++++++- 4 files changed, 75 insertions(+), 2 deletions(-) diff --git a/zuul/configloader.py b/zuul/configloader.py index c0267ede58..3438815ae2 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -47,6 +47,16 @@ class ConfigurationSyntaxError(Exception): pass +class NodeFromGroupNotFoundError(Exception): + def __init__(self, nodeset, node, group): + message = textwrap.dedent("""\ + In nodeset {nodeset} the group {group} contains a + node named {node} which is not defined in the nodeset.""") + message = textwrap.fill(message.format(nodeset=nodeset, + node=node, group=group)) + super(NodeFromGroupNotFoundError, self).__init__(message) + + class ProjectNotFoundError(Exception): def __init__(self, project): message = textwrap.dedent("""\ @@ -169,8 +179,13 @@ class NodeSetParser(object): vs.Required('image'): str, } + group = {vs.Required('name'): str, + vs.Required('nodes'): [str] + } + nodeset = {vs.Required('name'): str, vs.Required('nodes'): [node], + 'groups': [group], '_source_context': model.SourceContext, '_start_mark': yaml.Mark, } @@ -182,9 +197,18 @@ class NodeSetParser(object): with configuration_exceptions('nodeset', conf): NodeSetParser.getSchema()(conf) ns = model.NodeSet(conf['name']) + node_names = [] for conf_node in as_list(conf['nodes']): node = model.Node(conf_node['name'], conf_node['image']) ns.addNode(node) + node_names.append(conf_node['name']) + for conf_group in as_list(conf.get('groups', [])): + for node_name in conf_group['nodes']: + if node_name not in node_names: + raise NodeFromGroupNotFoundError(conf['name'], node_name, + conf_group['name']) + group = model.Group(conf_group['name'], conf_group['nodes']) + ns.addGroup(group) return ns diff --git a/zuul/executor/client.py b/zuul/executor/client.py index f82b7c2330..907cce510f 100644 --- a/zuul/executor/client.py +++ b/zuul/executor/client.py @@ -270,8 +270,9 @@ class ExecutorClient(object): params['post_playbooks'] = [x.toDict() for x in job.post_run] params['roles'] = [x.toDict() for x in job.roles] + nodeset = item.current_build_set.getJobNodeSet(job.name) nodes = [] - for node in item.current_build_set.getJobNodeSet(job.name).getNodes(): + for node in nodeset.getNodes(): nodes.append(dict(name=node.name, image=node.image, az=node.az, host_keys=node.host_keys, @@ -281,6 +282,7 @@ class ExecutorClient(object): public_ipv6=node.public_ipv6, public_ipv4=node.public_ipv4)) params['nodes'] = nodes + params['groups'] = [group.toDict() for group in nodeset.getGroups()] params['vars'] = copy.deepcopy(job.variables) if job.auth: for secret in job.auth.secrets: diff --git a/zuul/executor/server.py b/zuul/executor/server.py index 72c042c93a..a4ae4cd024 100644 --- a/zuul/executor/server.py +++ b/zuul/executor/server.py @@ -1088,6 +1088,11 @@ class AnsibleJob(object): inventory.write('\n') for key in item['host_keys']: keys.append(key) + for group in args['groups']: + inventory.write('[{name}]\n'.format(name=group['name'])) + for node_name in group['nodes']: + inventory.write(node_name) + inventory.write('\n') with open(self.jobdir.known_hosts, 'w') as known_hosts: for key in keys: diff --git a/zuul/model.py b/zuul/model.py index 12ddbda58d..b468bbf1ce 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -410,6 +410,37 @@ class Node(object): self._keys = keys +class Group(object): + """A logical group of nodes for use by a job. + + A Group is a named set of node names that will be provided to + jobs in the inventory to describe logical units where some subset of tasks + run. + """ + + def __init__(self, name, nodes): + self.name = name + self.nodes = nodes + + def __repr__(self): + return '' % (self.name, str(self.nodes)) + + def __ne__(self, other): + return not self.__eq__(other) + + def __eq__(self, other): + if not isinstance(other, Group): + return False + return (self.name == other.name and + self.nodes == other.nodes) + + def toDict(self): + return { + 'name': self.name, + 'nodes': self.nodes + } + + class NodeSet(object): """A set of nodes. @@ -423,6 +454,7 @@ class NodeSet(object): def __init__(self, name=None): self.name = name or '' self.nodes = OrderedDict() + self.groups = OrderedDict() def __ne__(self, other): return not self.__eq__(other) @@ -437,6 +469,8 @@ class NodeSet(object): n = NodeSet(self.name) for name, node in self.nodes.items(): n.addNode(Node(node.name, node.image)) + for name, group in self.groups.items(): + n.addGroup(Group(group.name, group.nodes[:])) return n def addNode(self, node): @@ -447,12 +481,20 @@ class NodeSet(object): def getNodes(self): return list(self.nodes.values()) + def addGroup(self, group): + if group.name in self.groups: + raise Exception("Duplicate group in %s" % (self,)) + self.groups[group.name] = group + + def getGroups(self): + return list(self.groups.values()) + def __repr__(self): if self.name: name = self.name + ' ' else: name = '' - return '' % (name, self.nodes) + return '' % (name, self.nodes, self.groups) class NodeRequest(object):