Write inventory as yaml not ini

All of the zuul config files are yaml. Ansible supports yaml formatted
inventory files. If we use yaml, then displaying inventories to users
in raw log files should be easier to read for non-Ansible users.
Variables are set under the hostname, rather than as key=value strings
on the same line. We can also stop writing the vars file and just add
the variables to the top level vars of the same inventory file, but
let's do that next.

Construct the dict as an unattached function not a method to allow for
easy interative unittesting as context isn't needed to validate inputs
and outputs.

Change-Id: Ife7a909ea0e54015bddbf5426343dd5c5911953c

tests/base.py

@@ -2436,6 +2436,12 @@ class ZuulTestCase(BaseTestCase):
         jobs = filter(lambda x: x.result == result, jobs)
         return len(list(jobs))
 
+    def getBuildByName(self, name):
+        for build in self.builds:
+            if build.name == name:
+                return build
+        raise Exception("Unable to find build %s" % name)
+
     def getJobFromHistory(self, name, project=None):
         for job in self.history:
             if (job.name == name and

tests/fixtures/config/inventory/git/common-config/playbooks/group-inventory.yaml

@@ -0,0 +1,2 @@
+- hosts: all
+  tasks: []

tests/fixtures/config/inventory/git/common-config/playbooks/single-inventory.yaml

@@ -0,0 +1,2 @@
+- hosts: all
+  tasks: []

tests/fixtures/config/inventory/git/common-config/zuul.yaml

@@ -0,0 +1,42 @@
+- pipeline:
+    name: check
+    manager: independent
+    allow-secrets: true
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        verified: 1
+    failure:
+      gerrit:
+        verified: -1
+
+- nodeset:
+    name: nodeset1
+    nodes:
+      - name: controller
+        image: controller-image
+      - name: compute1
+        image: compute-image
+      - name: compute2
+        image: compute-image
+    groups:
+      - name: ceph-osd
+        nodes:
+          - controller
+      - name: ceph-monitor
+        nodes:
+          - controller
+          - compute1
+          - compute2
+
+- job:
+    name: single-inventory
+    nodes:
+      - name: ubuntu-xenial
+        image: ubuntu-xenial
+
+- job:
+    name: group-inventory
+    nodes: nodeset1

tests/fixtures/config/inventory/git/org_project/.zuul.yaml

@@ -0,0 +1,6 @@
+- project:
+    name: org/project
+    check:
+      jobs:
+        - single-inventory
+        - group-inventory

tests/fixtures/config/inventory/git/org_project/README

@@ -0,0 +1 @@
+test

tests/fixtures/config/inventory/main.yaml

@@ -0,0 +1,8 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project

tests/unit/test_inventory.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+
+import yaml
+
+from tests.base import ZuulTestCase
+
+
+class TestInventory(ZuulTestCase):
+
+    tenant_config_file = 'config/inventory/main.yaml'
+
+    def setUp(self):
+        super(TestInventory, self).setUp()
+        self.executor_server.hold_jobs_in_build = True
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+    def _get_build_inventory(self, name):
+        build = self.getBuildByName(name)
+        inv_path = os.path.join(build.jobdir.root, 'ansible', 'inventory.yaml')
+        return yaml.safe_load(open(inv_path, 'r'))
+
+    def test_single_inventory(self):
+
+        inventory = self._get_build_inventory('single-inventory')
+
+        all_nodes = ('ubuntu-xenial',)
+        self.assertIn('all', inventory)
+        self.assertIn('hosts', inventory['all'])
+        for node_name in all_nodes:
+            self.assertIn(node_name, inventory['all']['hosts'])
+
+        self.executor_server.release()
+        self.waitUntilSettled()
+
+    def test_group_inventory(self):
+
+        inventory = self._get_build_inventory('group-inventory')
+
+        all_nodes = ('controller', 'compute1', 'compute2')
+        self.assertIn('all', inventory)
+        self.assertIn('hosts', inventory['all'])
+        for group_name in ('ceph-osd', 'ceph-monitor'):
+            self.assertIn(group_name, inventory)
+        for node_name in all_nodes:
+            self.assertIn(node_name, inventory['all']['hosts'])
+            self.assertIn(node_name,
+                          inventory['ceph-monitor']['hosts'])
+        self.executor_server.release()
+        self.waitUntilSettled()

zuul/executor/server.py

@@ -183,7 +183,7 @@ class JobDir(object):
         self.ansible_root = os.path.join(self.root, 'ansible')
         os.makedirs(self.ansible_root)
         self.known_hosts = os.path.join(self.ansible_root, 'known_hosts')
-        self.inventory = os.path.join(self.ansible_root, 'inventory')
+        self.inventory = os.path.join(self.ansible_root, 'inventory.yaml')
         self.vars = os.path.join(self.ansible_root, 'vars.yaml')
         self.playbooks = []  # The list of candidate playbooks
         self.playbook = None  # A pointer to the candidate we have chosen
@@ -313,6 +313,30 @@ def _copy_ansible_files(python_module, target_dir):
                 shutil.copy(os.path.join(library_path, fn), target_dir)
 
 
+def make_inventory_dict(nodes, groups):
+
+    hosts = {}
+    for node in nodes:
+        hosts[node['name']] = node['host_vars']
+
+    inventory = {
+        'all': {
+            'hosts': hosts,
+        }
+    }
+
+    for group in groups:
+        group_hosts = {}
+        for node_name in group['nodes']:
+            # children is a dict with None as values because we don't have
+            # and per-group variables. If we did, None would be a dict
+            # with the per-group variables
+            group_hosts[node_name] = None
+        inventory[group['name']] = {'hosts': group_hosts}
+
+    return inventory
+
+
 class ExecutorMergeWorker(gear.TextWorker):
     def __init__(self, executor_server, *args, **kw):
         self.zuul_executor_server = executor_server
@@ -1079,24 +1103,17 @@ class AnsibleJob(object):
             self.jobdir.trusted_roles_path.append(trusted_role_path)
 
     def prepareAnsibleFiles(self, args):
-        keys = []
-        with open(self.jobdir.inventory, 'w') as inventory:
-            for item in self.getHostList(args):
-                inventory.write(item['name'])
-                for k, v in item['host_vars'].items():
-                    inventory.write(' %s="%s"' % (k, v))
-                inventory.write('\n')
-                for key in item['host_keys']:
-                    keys.append(key)
-            for group in args['groups']:
-                inventory.write('[{name}]\n'.format(name=group['name']))
-                for node_name in group['nodes']:
-                    inventory.write(node_name)
-                    inventory.write('\n')
+        nodes = self.getHostList(args)
+        inventory = make_inventory_dict(nodes, args['groups'])
+
+        with open(self.jobdir.inventory, 'w') as inventory_yaml:
+            inventory_yaml.write(
+                yaml.safe_dump(inventory, default_flow_style=False))
 
         with open(self.jobdir.known_hosts, 'w') as known_hosts:
-            for key in keys:
-                known_hosts.write('%s\n' % key)
+            for node in nodes:
+                for key in node['host_keys']:
+                    known_hosts.write('%s\n' % key)
 
         with open(self.jobdir.vars, 'w') as vars_yaml:
             zuul_vars = dict(args['vars'])