Merge "Add support for defining groups in nodesets" into feature/zuulv3

zuul/configloader.py

@@ -47,6 +47,16 @@ class ConfigurationSyntaxError(Exception):
     pass
 
 
+class NodeFromGroupNotFoundError(Exception):
+    def __init__(self, nodeset, node, group):
+        message = textwrap.dedent("""\
+        In nodeset {nodeset} the group {group} contains a
+        node named {node} which is not defined in the nodeset.""")
+        message = textwrap.fill(message.format(nodeset=nodeset,
+                                               node=node, group=group))
+        super(NodeFromGroupNotFoundError, self).__init__(message)
+
+
 class ProjectNotFoundError(Exception):
     def __init__(self, project):
         message = textwrap.dedent("""\
@@ -169,8 +179,13 @@ class NodeSetParser(object):
                 vs.Required('image'): str,
                 }
 
+        group = {vs.Required('name'): str,
+                 vs.Required('nodes'): [str]
+                 }
+
         nodeset = {vs.Required('name'): str,
                    vs.Required('nodes'): [node],
+                   'groups': [group],
                    '_source_context': model.SourceContext,
                    '_start_mark': yaml.Mark,
                    }
@@ -182,9 +197,18 @@ class NodeSetParser(object):
         with configuration_exceptions('nodeset', conf):
             NodeSetParser.getSchema()(conf)
         ns = model.NodeSet(conf['name'])
+        node_names = []
         for conf_node in as_list(conf['nodes']):
             node = model.Node(conf_node['name'], conf_node['image'])
             ns.addNode(node)
+            node_names.append(conf_node['name'])
+        for conf_group in as_list(conf.get('groups', [])):
+            for node_name in conf_group['nodes']:
+                if node_name not in node_names:
+                    raise NodeFromGroupNotFoundError(conf['name'], node_name,
+                                                     conf_group['name'])
+            group = model.Group(conf_group['name'], conf_group['nodes'])
+            ns.addGroup(group)
         return ns
 
 

zuul/executor/client.py

@@ -274,8 +274,9 @@ class ExecutorClient(object):
             params['post_playbooks'] = [x.toDict() for x in job.post_run]
             params['roles'] = [x.toDict() for x in job.roles]
 
+        nodeset = item.current_build_set.getJobNodeSet(job.name)
         nodes = []
-        for node in item.current_build_set.getJobNodeSet(job.name).getNodes():
+        for node in nodeset.getNodes():
             nodes.append(dict(name=node.name, image=node.image,
                               az=node.az,
                               host_keys=node.host_keys,
@@ -285,6 +286,7 @@ class ExecutorClient(object):
                               public_ipv6=node.public_ipv6,
                               public_ipv4=node.public_ipv4))
         params['nodes'] = nodes
+        params['groups'] = [group.toDict() for group in nodeset.getGroups()]
         params['vars'] = copy.deepcopy(job.variables)
         if job.auth:
             for secret in job.auth.secrets:

zuul/executor/server.py

@@ -1130,6 +1130,11 @@ class AnsibleJob(object):
                 inventory.write('\n')
                 for key in item['host_keys']:
                     keys.append(key)
+            for group in args['groups']:
+                inventory.write('[{name}]\n'.format(name=group['name']))
+                for node_name in group['nodes']:
+                    inventory.write(node_name)
+                    inventory.write('\n')
 
         with open(self.jobdir.known_hosts, 'w') as known_hosts:
             for key in keys:

zuul/model.py

@@ -410,6 +410,37 @@ class Node(object):
         self._keys = keys
 
 
+class Group(object):
+    """A logical group of nodes for use by a job.
+
+    A Group is a named set of node names that will be provided to
+    jobs in the inventory to describe logical units where some subset of tasks
+    run.
+    """
+
+    def __init__(self, name, nodes):
+        self.name = name
+        self.nodes = nodes
+
+    def __repr__(self):
+        return '<Group %s %s>' % (self.name, str(self.nodes))
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __eq__(self, other):
+        if not isinstance(other, Group):
+            return False
+        return (self.name == other.name and
+                self.nodes == other.nodes)
+
+    def toDict(self):
+        return {
+            'name': self.name,
+            'nodes': self.nodes
+        }
+
+
 class NodeSet(object):
     """A set of nodes.
 
@@ -423,6 +454,7 @@ class NodeSet(object):
     def __init__(self, name=None):
         self.name = name or ''
         self.nodes = OrderedDict()
+        self.groups = OrderedDict()
 
     def __ne__(self, other):
         return not self.__eq__(other)
@@ -437,6 +469,8 @@ class NodeSet(object):
         n = NodeSet(self.name)
         for name, node in self.nodes.items():
             n.addNode(Node(node.name, node.image))
+        for name, group in self.groups.items():
+            n.addGroup(Group(group.name, group.nodes[:]))
         return n
 
     def addNode(self, node):
@@ -447,12 +481,20 @@ class NodeSet(object):
     def getNodes(self):
         return list(self.nodes.values())
 
+    def addGroup(self, group):
+        if group.name in self.groups:
+            raise Exception("Duplicate group in %s" % (self,))
+        self.groups[group.name] = group
+
+    def getGroups(self):
+        return list(self.groups.values())
+
     def __repr__(self):
         if self.name:
             name = self.name + ' '
         else:
             name = ''
-        return '<NodeSet %s%s>' % (name, self.nodes)
+        return '<NodeSet %s%s%s>' % (name, self.nodes, self.groups)
 
 
 class NodeRequest(object):