Enable multinode scenarios in zuulv3

Another step to migrate to new CI

Change-Id: Id9bf9b4b1114f5be66f6c028a0db16d869efeceb
Depends-On: Iaacdca31b27b7033ec08499bcea7061b9d3ec279

.zuul.yaml

@@ -5,6 +5,9 @@
         - kolla-ansible-centos-source
         - kolla-ansible-ubuntu-source
         - kolla-ansible-oraclelinux-source
+        - kolla-ansible-ubuntu-source-ceph
+        - kolla-ansible-centos-source-ceph
+        - kolla-ansible-oraclelinux-source-ceph
 
 - nodeset:
     name: kolla-ansible-centos
@@ -18,19 +21,57 @@
       - name: primary
         label: ubuntu-xenial
 
+- nodeset:
+    name: kolla-ansible-xenial-multi
+    nodes:
+      - name: primary
+        label: ubuntu-xenial
+      - name: secondary1
+        label: ubuntu-xenial
+      - name: secondary2
+        label: ubuntu-xenial
+    groups:
+      - name: switch
+        nodes:
+          - primary
+      - name: peers
+        nodes:
+          - secondary1
+          - secondary2
+
+- nodeset:
+    name: kolla-ansible-centos-multi
+    nodes:
+      - name: primary
+        label: centos-7
+      - name: secondary1
+        label: centos-7
+      - name: secondary2
+        label: centos-7
+    groups:
+      - name: switch
+        nodes:
+          - primary
+      - name: peers
+        nodes:
+          - secondary1
+          - secondary2
+
 - job:
     name: kolla-ansible-base
     pre-run: tests/pre.yml
     run: tests/run.yml
     post-run: tests/post.yml
     attempts: 1
-    timeout: 10800
+    timeout: 5400
     required-projects:
       - openstack/kolla
       - openstack/requirements
     irrelevant-files:
       - ^.*\.rst$
       - ^doc/.*
+    vars:
+      scenario: aio
 
 - job:
     name: kolla-ansible-centos-source
@@ -85,3 +126,33 @@
     vars:
       base_distro: oraclelinux
       install_type: binary
+
+- job:
+    name: kolla-ansible-ubuntu-source-ceph
+    parent: kolla-ansible-base
+    nodeset: kolla-ansible-xenial-multi
+    voting: false
+    vars:
+      base_distro: ubuntu
+      install_type: source
+      scenario: ceph
+
+- job:
+    name: kolla-ansible-centos-source-ceph
+    parent: kolla-ansible-base
+    nodeset: kolla-ansible-centos-multi
+    voting: false
+    vars:
+      base_distro: centos
+      install_type: source
+      scenario: ceph
+
+- job:
+    name: kolla-ansible-oraclelinux-source-ceph
+    parent: kolla-ansible-base
+    nodeset: kolla-ansible-centos-multi
+    voting: false
+    vars:
+      base_distro: oraclelinux
+      install_type: source
+      scenario: ceph

tests/post.yml

@@ -18,16 +18,25 @@
       debug:
         msg: "{{ get_logs_result.stdout }}"
 
-    - name: Download logs to executor
-      synchronize:
-        dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
-        mode: pull
-        src: "{{ logs_dir }}/"
-      ignore_errors: yes
-
     - name: Download /etc/hosts file to executor
       synchronize:
         src: "/etc/hosts"
         dest: "{{ zuul.executor.log_root }}/{{inventory_hostname }}/"
         mode: pull
       ignore_errors: yes
+
+    - name: Print all facts
+      copy:
+        content: "{{ hostvars[inventory_hostname] | to_nice_json }}"
+        dest: "/tmp/logs/facts.json"
+
+    - name: dump all iptables rules
+      shell: "iptables-save > /tmp/logs/iptables"
+      become: true
+
+    - name: Download /tmp/logs file to executor
+      synchronize:
+        src: "/tmp/logs"
+        dest: "{{ zuul.executor.log_root }}/{{inventory_hostname }}/"
+        mode: pull
+

tests/pre.yml

@@ -29,3 +29,10 @@
       hostname:
         name: "{{ inventory_hostname }}"
       become: true
+
+# TODO(inc0): we're dropping iptables rules but in fact we should create
+# linuxbridge-managed tunnels for control and dataplane
+
+    - name: Drop iptables rules
+      command: "iptables -F"
+      become: true

tests/run.yml

@@ -1,5 +1,5 @@
 ---
-- hosts: all
+- hosts: primary
   vars:
     kolla_inventory_path: "/etc/kolla/inventory"
     logs_dir: "/tmp/logs"
@@ -7,6 +7,8 @@
     kolla_ansible_full_src_dir: "{{ zuul.executor.work_root }}/{{ kolla_ansible_src_dir }}"
     need_build_image: false
   tasks:
+    - setup:
+
     - name: ensure /etc/kolla exists
       file:
         path: "/etc/kolla"
@@ -14,25 +16,12 @@
         mode: 0777
       become: true
 
-    - set_fact:
-        nodes: |
-          {% for host in hostvars %}
-          {{ host }} ansible_host={{ hostvars[host]['nodepool']['private_ipv4'] }} ansible_become=true ansible_user={{ hostvars[host]['ansible_user'] }}
-          {% endfor %}
-
     - name: copy default ansible kolla-ansible inventory
-      copy:
+      template:
-        src: "{{ kolla_ansible_full_src_dir }}/ansible/inventory/all-in-one"
+        src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/inventory.j2"
         dest: "{{ kolla_inventory_path }}"
       delegate_to: "primary"
 
-    - name: replace localhost with IPs
-      replace:
-        path: "{{ kolla_inventory_path }}"
-        regexp: "localhost.*$"
-        replace: "{{ nodes }}"
-      delegate_to: "primary"
-
     # FIXME: in multi node env, api_interface may be different on each node.
     - name: detect api_interface_name variable
       vars:
@@ -68,7 +57,7 @@
       become: true
       delegate_to: "primary"
 
-    - name: generate config overrides
+    - name: generate nova config overrides
       template:
         src: "{{ kolla_ansible_full_src_dir }}/tests/templates/nova-compute-overrides.j2"
         dest: /etc/kolla/config/nova/nova-compute.conf
@@ -105,6 +94,18 @@
       shell: "{{ kolla_ansible_src_dir }}/tools/generate_passwords.py"
       delegate_to: "primary"
 
+    - name: Prepare ceph disks
+      script: "{{ kolla_ansible_full_src_dir }}/tests/setup_ceph_disks.sh"
+      when: scenario == "ceph"
+      become: true
+
+    - name: generate ceph config overrides
+      template:
+        src: "{{ kolla_ansible_full_src_dir }}/tests/templates/ceph-overrides.j2"
+        dest: /etc/kolla/config/ceph.conf
+      when: scenario == "ceph"
+      delegate_to: "primary"
+
     - shell:
         cmd: |
           set -e
@@ -112,7 +113,6 @@
           export BASE_DISTRO={{ base_distro }}
           export INSTALL_TYPE={{ install_type }}
           export NODEPOOL_TARBALLS_MIRROR=http://{{ zuul_site_mirror_fqdn }}:8080/tarballs
-
           export BUILD_IMAGE={{ need_build_image }}
           export KOLLA_SRC_DIR={{ ansible_env.HOME }}/src/git.openstack.org/openstack/kolla
 

tests/templates/ceph-overrides.j2

@@ -0,0 +1,3 @@
+[global]
+osd pool default size = 1
+osd pool default min size = 1

tests/templates/globals-default.j2

@@ -3,8 +3,8 @@ kolla_base_distro: "{{ base_distro }}"
 kolla_install_type: "{{ install_type }}"
 
 {% if hostvars|length > 2 %}
-enable_haproxy: "no"
 kolla_internal_vip_address: "{{ api_interface_address }}"
+enable_haproxy: "no"
 {% else %}
 kolla_internal_vip_address: "169.254.169.10"
 {% endif %}
@@ -18,7 +18,7 @@ docker_registry: "{{ api_interface_address }}:4000"
 {% else %}
 # use docker hub images
 docker_namespace: "kolla"
-openstack_release: "queens"
+openstack_release: "{{ zuul.branch | basename }}"
 {% endif %}
 neutron_external_interface: "fake_interface"
 enable_horizon: "yes"
@@ -26,7 +26,9 @@ enable_heat: "no"
 openstack_logging_debug: "True"
 openstack_service_workers: "1"
 
-{% if 'ceph' in zuul.job %}
+{% if scenario == "ceph" %}
 enable_ceph: "yes"
 enable_cinder: "yes"
+ceph_pool_pg_num: 64
+ceph_pool_pgp_num: 64
 {% endif %}

tests/templates/inventory.j2

@@ -0,0 +1,600 @@
+# These initial groups are the only groups required to be modified. The
+# additional groups are for more control of the environment.
+[control]
+{% for host in hostvars %}
+{{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_become=true ansible_user={{ hostvars[host]['ansible_user'] }}
+{% endfor %}
+
+[network]
+{% for host in hostvars %}
+{{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_become=true ansible_user={{ hostvars[host]['ansible_user'] }}
+{% endfor %}
+
+[compute]
+{% for host in hostvars %}
+{{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_become=true ansible_user={{ hostvars[host]['ansible_user'] }}
+{% endfor %}
+
+[storage]
+{% for host in hostvars %}
+{{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_become=true ansible_user={{ hostvars[host]['ansible_user'] }}
+{% endfor %}
+
+[monitoring]
+{% for host in hostvars %}
+{{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_become=true ansible_user={{ hostvars[host]['ansible_user'] }}
+{% endfor %}
+
+[deployment]
+{% for host in hostvars %}
+{{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_become=true ansible_user={{ hostvars[host]['ansible_user'] }}
+{% endfor %}
+
+# You can explicitly specify which hosts run each project by updating the
+# groups in the sections below. Common services are grouped together.
+[chrony-server:children]
+haproxy
+
+[chrony:children]
+network
+compute
+storage
+monitoring
+
+[collectd:children]
+compute
+
+[baremetal:children]
+control
+
+[grafana:children]
+monitoring
+
+[etcd:children]
+control
+compute
+
+[karbor:children]
+control
+
+[kibana:children]
+control
+
+[telegraf:children]
+compute
+control
+monitoring
+network
+storage
+
+[elasticsearch:children]
+control
+
+[haproxy:children]
+network
+
+[hyperv]
+#hyperv_host
+
+[hyperv:vars]
+#ansible_user=user
+#ansible_password=password
+#ansible_port=5986
+#ansible_connection=winrm
+#ansible_winrm_server_cert_validation=ignore
+
+[mariadb:children]
+control
+
+[rabbitmq:children]
+control
+
+[outward-rabbitmq:children]
+control
+
+[qdrouterd:children]
+control
+
+[mongodb:children]
+control
+
+[keystone:children]
+control
+
+[glance:children]
+control
+
+[nova:children]
+control
+
+[neutron:children]
+network
+
+[openvswitch:children]
+network
+compute
+manila-share
+
+[opendaylight:children]
+network
+
+[cinder:children]
+control
+
+[cloudkitty:children]
+control
+
+[freezer:children]
+control
+
+[memcached:children]
+control
+
+[horizon:children]
+control
+
+[swift:children]
+control
+
+[barbican:children]
+control
+
+[heat:children]
+control
+
+[murano:children]
+control
+
+[ceph:children]
+control
+
+[ironic:children]
+control
+
+[influxdb:children]
+monitoring
+
+[magnum:children]
+control
+
+[sahara:children]
+control
+
+[solum:children]
+control
+
+[mistral:children]
+control
+
+[manila:children]
+control
+
+[panko:children]
+control
+
+[gnocchi:children]
+control
+
+[ceilometer:children]
+control
+
+[aodh:children]
+control
+
+[congress:children]
+control
+
+[tacker:children]
+control
+
+# Tempest
+[tempest:children]
+control
+
+[senlin:children]
+control
+
+[vmtp:children]
+control
+
+[trove:children]
+control
+
+[watcher:children]
+control
+
+[rally:children]
+control
+
+[searchlight:children]
+control
+
+[octavia:children]
+control
+
+[designate:children]
+control
+
+[placement:children]
+control
+
+[bifrost:children]
+deployment
+
+[zun:children]
+control
+
+[skydive:children]
+monitoring
+
+[redis:children]
+control
+
+# Additional control implemented here. These groups allow you to control which
+# services run on which hosts at a per-service level.
+#
+# Word of caution: Some services are required to run on the same host to
+# function appropriately. For example, neutron-metadata-agent must run on the
+# same host as the l3-agent and (depending on configuration) the dhcp-agent.
+
+# Glance
+[glance-api:children]
+glance
+
+[glance-registry:children]
+glance
+
+# Nova
+[nova-api:children]
+nova
+
+[nova-conductor:children]
+nova
+
+[nova-consoleauth:children]
+nova
+
+[nova-novncproxy:children]
+nova
+
+[nova-scheduler:children]
+nova
+
+[nova-spicehtml5proxy:children]
+nova
+
+[nova-compute-ironic:children]
+nova
+
+[nova-serialproxy:children]
+nova
+
+# Neutron
+[neutron-server:children]
+control
+
+[neutron-dhcp-agent:children]
+neutron
+
+[neutron-l3-agent:children]
+neutron
+
+[neutron-lbaas-agent:children]
+neutron
+
+[neutron-metadata-agent:children]
+neutron
+
+[neutron-vpnaas-agent:children]
+neutron
+
+[neutron-bgp-dragent:children]
+neutron
+
+# Ceph
+[ceph-mgr:children]
+ceph
+
+[ceph-mon:children]
+ceph
+
+[ceph-rgw:children]
+ceph
+
+[ceph-osd:children]
+storage
+
+# Cinder
+[cinder-api:children]
+cinder
+
+[cinder-backup:children]
+storage
+
+[cinder-scheduler:children]
+cinder
+
+[cinder-volume:children]
+storage
+
+# Cloudkitty
+[cloudkitty-api:children]
+cloudkitty
+
+[cloudkitty-processor:children]
+cloudkitty
+
+# Freezer
+[freezer-api:children]
+freezer
+
+# iSCSI
+[iscsid:children]
+compute
+storage
+ironic
+
+[tgtd:children]
+storage
+
+# Karbor
+[karbor-api:children]
+karbor
+
+[karbor-protection:children]
+karbor
+
+[karbor-operationengine:children]
+karbor
+
+# Manila
+[manila-api:children]
+manila
+
+[manila-scheduler:children]
+manila
+
+[manila-share:children]
+network
+
+[manila-data:children]
+manila
+
+# Swift
+[swift-proxy-server:children]
+swift
+
+[swift-account-server:children]
+storage
+
+[swift-container-server:children]
+storage
+
+[swift-object-server:children]
+storage
+
+# Barbican
+[barbican-api:children]
+barbican
+
+[barbican-keystone-listener:children]
+barbican
+
+[barbican-worker:children]
+barbican
+
+# Trove
+[trove-api:children]
+trove
+
+[trove-conductor:children]
+trove
+
+[trove-taskmanager:children]
+trove
+
+# Heat
+[heat-api:children]
+heat
+
+[heat-api-cfn:children]
+heat
+
+[heat-engine:children]
+heat
+
+# Murano
+[murano-api:children]
+murano
+
+[murano-engine:children]
+murano
+
+# Ironic
+[ironic-api:children]
+ironic
+
+[ironic-conductor:children]
+ironic
+
+[ironic-inspector:children]
+ironic
+
+[ironic-pxe:children]
+ironic
+
+# Magnum
+[magnum-api:children]
+magnum
+
+[magnum-conductor:children]
+magnum
+
+# Solum
+[solum-api:children]
+solum
+
+[solum-worker:children]
+solum
+
+[solum-deployer:children]
+solum
+
+[solum-conductor:children]
+solum
+
+# Mistral
+[mistral-api:children]
+mistral
+
+[mistral-executor:children]
+mistral
+
+[mistral-engine:children]
+mistral
+
+# Aodh
+[aodh-api:children]
+aodh
+
+[aodh-evaluator:children]
+aodh
+
+[aodh-listener:children]
+aodh
+
+[aodh-notifier:children]
+aodh
+
+# Panko
+[panko-api:children]
+panko
+
+# Gnocchi
+[gnocchi-api:children]
+gnocchi
+
+[gnocchi-statsd:children]
+gnocchi
+
+[gnocchi-metricd:children]
+gnocchi
+
+# Sahara
+[sahara-api:children]
+sahara
+
+[sahara-engine:children]
+sahara
+
+# Ceilometer
+[ceilometer-api:children]
+ceilometer
+
+[ceilometer-central:children]
+ceilometer
+
+[ceilometer-notification:children]
+ceilometer
+
+[ceilometer-collector:children]
+ceilometer
+
+[ceilometer-compute:children]
+compute
+
+# Congress
+[congress-api:children]
+congress
+
+[congress-datasource:children]
+congress
+
+[congress-policy-engine:children]
+congress
+
+# Multipathd
+[multipathd:children]
+compute
+
+# Watcher
+[watcher-api:children]
+watcher
+
+[watcher-engine:children]
+watcher
+
+[watcher-applier:children]
+watcher
+
+# Senlin
+[senlin-api:children]
+senlin
+
+[senlin-engine:children]
+senlin
+
+# Searchlight
+[searchlight-api:children]
+searchlight
+
+[searchlight-listener:children]
+searchlight
+
+# Octavia
+[octavia-api:children]
+octavia
+
+[octavia-health-manager:children]
+octavia
+
+[octavia-housekeeping:children]
+octavia
+
+[octavia-worker:children]
+octavia
+
+# Designate
+[designate-api:children]
+designate
+
+[designate-central:children]
+designate
+
+[designate-mdns:children]
+network
+
+[designate-worker:children]
+designate
+
+[designate-sink:children]
+designate
+
+[designate-backend-bind9:children]
+designate
+
+# Placement
+[placement-api:children]
+placement
+
+# Zun
+[zun-api:children]
+zun
+
+[zun-compute:children]
+compute
+
+# Skydive
+[skydive-analyzer:children]
+skydive
+
+[skydive-agent:children]
+compute
+network
+
+# Tacker
+[tacker-server:children]
+tacker
+
+[tacker-conductor:children]
+tacker

tools/setup_gate.sh

@@ -178,6 +178,7 @@ tools/kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy > /tmp/logs/ansible/pos
 
 # Test OpenStack Environment
 # TODO: use kolla-ansible check when it's ready
+
 sanity_check
 
 # TODO(jeffrey4l): make some configure file change and