Add specific setup inventory

By default, Zuul uses runAnsibleSetup on all inventory nodes prior
to running job playbooks.
This translates to doing an 'ansible -m setup' against all nodes, but
this won't work on nodes where Python is not available, like network
nodes.
This change adds a specific setup_inventory.yaml file, which will not contain
nodes where setup module cannot work.

Change-Id: Ieb02a19036854b8d9089bcd4cc9dd0b46e3ce2fc

tests/base.py

@@ -1741,6 +1741,8 @@ class FakeNodepool(object):
             data['username'] = 'fakeuser'
         if 'windows' in node_type:
             data['connection_type'] = 'winrm'
+        if 'network' in node_type:
+            data['connection_type'] = 'network_cli'
 
         data = json.dumps(data).encode('utf8')
         path = self.client.create(path, data,

tests/fixtures/config/inventory/git/common-config/zuul.yaml

@@ -40,6 +40,8 @@
         label: fakeuser-label
       - name: windows
         label: windows-label
+      - name: network
+        label: network-label
 
 - job:
     name: base

tests/unit/test_inventory.py

@@ -37,6 +37,12 @@ class TestInventory(ZuulTestCase):
         inv_path = os.path.join(build.jobdir.root, 'ansible', 'inventory.yaml')
         return yaml.safe_load(open(inv_path, 'r'))
 
+    def _get_setup_inventory(self, name):
+        build = self.getBuildByName(name)
+        setup_inv_path = os.path.join(build.jobdir.root, 'ansible',
+                                      'setup-inventory.yaml')
+        return yaml.safe_load(open(setup_inv_path, 'r'))
+
     def test_single_inventory(self):
 
         inventory = self._get_build_inventory('single-inventory')
@@ -131,3 +137,23 @@ class TestInventory(ZuulTestCase):
 
         self.executor_server.release()
         self.waitUntilSettled()
+
+    def test_setup_inventory(self):
+
+        setup_inventory = self._get_setup_inventory('hostvars-inventory')
+        inventory = self._get_build_inventory('hostvars-inventory')
+
+        self.assertIn('all', inventory)
+        self.assertIn('hosts', inventory['all'])
+
+        self.assertIn('default', setup_inventory['all']['hosts'])
+        self.assertIn('fakeuser', setup_inventory['all']['hosts'])
+        self.assertIn('windows', setup_inventory['all']['hosts'])
+        self.assertNotIn('network', setup_inventory['all']['hosts'])
+        self.assertIn('default', inventory['all']['hosts'])
+        self.assertIn('fakeuser', inventory['all']['hosts'])
+        self.assertIn('windows', inventory['all']['hosts'])
+        self.assertIn('network', inventory['all']['hosts'])
+
+        self.executor_server.release()
+        self.waitUntilSettled()

zuul/executor/server.py

@@ -45,6 +45,7 @@ BUFFER_LINES_FOR_SYNTAX = 200
 COMMANDS = ['stop', 'pause', 'unpause', 'graceful', 'verbose',
             'unverbose', 'keep', 'nokeep']
 DEFAULT_FINGER_PORT = 79
+BLACKLISTED_ANSIBLE_CONNECTION_TYPES = ['network_cli']
 
 
 class StopException(Exception):
@@ -347,6 +348,8 @@ class JobDir(object):
             pass
         self.known_hosts = os.path.join(ssh_dir, 'known_hosts')
         self.inventory = os.path.join(self.ansible_root, 'inventory.yaml')
+        self.setup_inventory = os.path.join(self.ansible_root,
+                                            'setup-inventory.yaml')
         self.logging_json = os.path.join(self.ansible_root, 'logging.json')
         self.playbooks = []  # The list of candidate playbooks
         self.playbook = None  # A pointer to the candidate we have chosen
@@ -493,6 +496,26 @@ def _copy_ansible_files(python_module, target_dir):
                 shutil.copy(os.path.join(library_path, fn), target_dir)
 
 
+def make_setup_inventory_dict(nodes):
+
+    hosts = {}
+    for node in nodes:
+        if (node['host_vars']['ansible_connection'] in
+            BLACKLISTED_ANSIBLE_CONNECTION_TYPES):
+            continue
+
+        for name in node['name']:
+            hosts[name] = node['host_vars']
+
+    inventory = {
+        'all': {
+            'hosts': hosts,
+        }
+    }
+
+    return inventory
+
+
 def make_inventory_dict(nodes, groups, all_vars):
 
     hosts = {}
@@ -1157,8 +1180,13 @@ class AnsibleJob(object):
             result_data_file=self.jobdir.result_data_file)
 
         nodes = self.getHostList(args)
+        setup_inventory = make_setup_inventory_dict(nodes)
         inventory = make_inventory_dict(nodes, args['groups'], all_vars)
 
+        with open(self.jobdir.setup_inventory, 'w') as setup_inventory_yaml:
+            setup_inventory_yaml.write(
+                yaml.safe_dump(setup_inventory, default_flow_style=False))
+
         with open(self.jobdir.inventory, 'w') as inventory_yaml:
             inventory_yaml.write(
                 yaml.safe_dump(inventory, default_flow_style=False))
@@ -1423,6 +1451,7 @@ class AnsibleJob(object):
             verbose = '-v'
 
         cmd = ['ansible', '*', verbose, '-m', 'setup',
+               '-i', self.jobdir.setup_inventory,
                '-a', 'gather_subset=!all']
 
         result, code = self.runAnsible(