Add multi-node integration jobs

Change-Id: I4a81f292acf993c8ab25c7cc36fddf704c485c6c

test-playbooks/multinode/multi-node-bridge.yaml

@@ -0,0 +1,54 @@
+- name: Test the multi-node-bridge role
+  hosts:
+    - switch
+    - peers
+  roles:
+    - multi-node-bridge
+  post_tasks:
+    - become: yes
+      block:
+        - name: openvswitch should be installed
+          package:
+            name: "{{ ovs_package }}"
+            state: installed
+          register: ovs_installed
+
+        - name: openvswitch should be running
+          service:
+            name: "{{ ovs_service }}"
+            state: started
+            enabled: yes
+          register: ovs_running
+
+        - name: bridge should exist
+          openvswitch_bridge:
+            bridge: "{{ bridge_name }}"
+          register: ovs_bridge
+
+        - name: port should exist
+          command: ovs-vsctl show
+          changed_when: false
+          register: ovs_port
+
+        - name: switch should be reachable
+          command: ping -c 4 {{ bridge_address_prefix }}.{{ bridge_address_offset }}
+          changed_when: false
+          failed_when: false
+          register: ovs_ping_switch
+
+        - name: peer should be reachable
+          command: ping -c 4 {{ bridge_address_prefix }}.{{ bridge_address_offset + 1 }}
+          changed_when: false
+          failed_when: false
+          register: ovs_ping_peer
+
+        - name: assert test results
+          assert:
+            that:
+              - ovs_installed is not changed
+              - ovs_running is not changed
+              - ovs_bridge is not changed
+              - ovs_port.rc == 0
+              - "'Port \"br-infra_' in ovs_port.stdout"
+              - ovs_ping_switch.rc == 0
+              - ovs_ping_peer.rc == 0

test-playbooks/multinode/multi-node-firewall.yaml

@@ -0,0 +1,43 @@
+- name: Test the multi-node-firewall role
+  hosts: all
+  roles:
+    - multi-node-firewall
+  post_tasks:
+    - name: switch and peer nodes should be in the ipv4 firewall
+      become: yes
+      command: iptables-save
+      changed_when: false
+      failed_when: false
+      register: iptables_rules
+
+    - name: Validate ipv4 private firewall configuration
+      assert:
+        that:
+          - "'-A INPUT -s {{ hostvars[item]['nodepool']['private_ipv4'] }}/32 -j ACCEPT' in iptables_rules.stdout"
+      with_items: "{{ groups['all'] }}"
+      when:
+        - hostvars[item]['nodepool']['private_ipv4']
+
+    - name: Validate ipv4 public firewall configuration
+      assert:
+        that:
+          - "'-A INPUT -s {{ hostvars[item]['nodepool']['public_ipv4'] }}/32 -j ACCEPT' in iptables_rules.stdout"
+      with_items: "{{ groups['all'] }}"
+      when:
+        - hostvars[item]['nodepool']['public_ipv4']
+
+    # ipv6_addresses is set by the multi-node-firewall role
+    - when: ipv6_addresses | length > 0
+      block:
+        - name: switch and peer nodes should be in the ipv6 firewall
+          become: yes
+          command: ip6tables-save
+          changed_when: false
+          failed_when: false
+          register: ip6tables_rules
+
+        - name: Validate ipv6 firewall configuration
+          assert:
+            that:
+              - "'-A INPUT -s {{ hostvars[item]['nodepool']['public_ipv6'] }}/128 -j ACCEPT' in ip6tables_rules.stdout"
+          with_items: "{{ groups['all'] }}"

test-playbooks/multinode/multi-node-hosts-file.yaml

@@ -0,0 +1,25 @@
+- name: Test the multi-node-hosts-file role
+  hosts: all
+  roles:
+    - multi-node-hosts-file
+  post_tasks:
+    - name: lookup hosts file
+      command: cat /etc/hosts
+      register: hosts_file
+
+    - name: Set up the list of hosts and addresses
+      set_fact:
+        host_addresses: >
+          {% set hosts = [] -%}
+          {% for host, vars in hostvars.items() -%}
+          {% set _ = hosts.append({'host': host, 'address': vars['nodepool']['private_ipv4']}) -%}
+          {% endfor -%}
+          {{- hosts -}}
+
+    - name: assert that hosts are in the hosts file
+      vars:
+        line: "{{ item.address }} {{ item.host }}"
+      assert:
+        that:
+          - "line in hosts_file.stdout"
+      with_list: "{{ host_addresses }}"

test-playbooks/multinode/multi-node-known-hosts.yaml

@@ -0,0 +1,31 @@
+- name: Test the multi-node-known-hosts role
+  hosts: all
+  roles:
+    - multi-node-known-hosts
+  post_tasks:
+    - name: lookup known_hosts file
+      command: cat ~/.ssh/known_hosts
+      register: known_hosts
+
+    - name: Set up host addresses
+      set_fact:
+        host_addresses: >
+          {% set hosts = [] -%}
+          {% for host, vars in hostvars.items() -%}
+          {% if vars['nodepool']['private_ipv4'] != '' -%}
+            {% set _ = hosts.append(vars['nodepool']['private_ipv4']) -%}
+          {% endif -%}
+          {% if vars['nodepool']['public_ipv4'] != '' -%}
+            {% set _ = hosts.append(vars['nodepool']['public_ipv4']) -%}
+          {% endif -%}
+          {% if vars['nodepool']['public_ipv6'] != '' -%}
+            {% set _ = hosts.append(vars['nodepool']['public_ipv6']) -%}
+          {% endif -%}
+          {% endfor -%}
+          {{- hosts | sort | unique -}}
+
+    - name: assert that hosts are in known_hosts
+      assert:
+        that:
+          - "item in known_hosts.stdout"
+      with_items: "{{ host_addresses }}"

test-playbooks/multinode/multinode.yaml

@@ -0,0 +1,10 @@
+# Roles that are part of the 'multinode' job
+
+# If you add new tests, also update the files section in jobs
+# base-integration and multinode-integration in zuul.d/jobs.yaml.
+
+- include: multi-node-known-hosts.yaml
+- include: multi-node-hosts-file.yaml
+- include: multi-node-firewall.yaml
+- include: multi-node-bridge.yaml
+- include: persistent-firewall.yaml

test-playbooks/multinode/multinode_firewall_persistence_vars/Debian.yaml

@@ -0,0 +1,2 @@
+iptables_service:
+  - netfilter-persistent

test-playbooks/multinode/multinode_firewall_persistence_vars/Gentoo.yaml

@@ -0,0 +1,3 @@
+iptables_service:
+  - iptables-restore
+  - ip6tables-restore

test-playbooks/multinode/multinode_firewall_persistence_vars/README.rst

@@ -0,0 +1,21 @@
+multinode_firewall_persistence_vars
+===================================
+
+This directory is meant to contain distribution specific variables used in
+integration tests for the ``multinode_firewall_persistence`` role.
+
+The behavior of the ``with_first_found`` lookup used with the ``include_vars``
+module will make it search for the ``vars`` directory in the "usual" order of
+precedence which means if there is a ``vars`` directory inside the playbook
+directory, it will search there first.
+
+This can result in one of two issues:
+
+1. If you try to prepend ``{{ role_path }}`` to workaround this issue with the
+   variable file paths, Zuul will deny the lookup if you are running an
+   untrusted playbook because the role was prepared in a trusted location and
+   Ansible is trying to search outside the work root as a result.
+2. The variables included are the wrong ones -- the ones from
+   ``playbooks/vars`` are loaded instead of ``path/to/<role>/vars``
+
+This is why this directory is called ``multinode_firewall_persistence_vars``.

test-playbooks/multinode/multinode_firewall_persistence_vars/RedHat.yaml

@@ -0,0 +1,3 @@
+iptables_service:
+  - iptables
+  - ip6tables

test-playbooks/multinode/multinode_firewall_persistence_vars/Suse.yaml

@@ -0,0 +1,2 @@
+iptables_service:
+  - SuSEfirewall2

test-playbooks/multinode/multinode_firewall_persistence_vars/Ubuntu_trusty.yaml

@@ -0,0 +1,2 @@
+iptables_service:
+  - iptables-persistent

test-playbooks/multinode/persistent-firewall.yaml

@@ -0,0 +1,80 @@
+- name: Test the persistent-firewall role
+  hosts: all
+  roles:
+    # We're including multi-node-bridge a second time with the toggle for
+    # enabling firewall rules for the bridge network subnet
+    # By this time, multi-node-firewall has already ran, we don't need to run
+    # it again -- we're testing here that both are persisted properly.
+    - { role: multi-node-bridge, bridge_authorize_internal_traffic: true }
+  post_tasks:
+    # NOTE (dmsimard): Using with_first_found and include_vars can yield
+    # unexpected results, see multinode_firewall_persistence_vars/README.rst
+    - name: Include OS-specific variables
+      include_vars: "{{ item }}"
+      with_first_found:
+        - "multinode_firewall_persistence_vars/{{ ansible_distribution }}_{{ ansible_distribution_release }}.yaml"
+        - "multinode_firewall_persistence_vars/{{ ansible_distribution }}.yaml"
+        - "multinode_firewall_persistence_vars/{{ ansible_os_family }}.yaml"
+        - "multinode_firewall_persistence_vars/default.yaml"
+
+    - name: Flush iptables rules
+      become: yes
+      command: "{{ item }}"
+      with_items:
+        - iptables --flush
+        - ip6tables --flush
+
+    # NOTE (dmsimard): We're using with_items here because RedHat and Gentoo
+    # need to restart both iptables and ip6tables.
+    - name: Restart iptables
+      become: yes
+      service:
+        name: "{{ item }}"
+        state: restarted
+      when: iptables_service is defined
+      with_items: "{{ iptables_service }}"
+
+    - name: switch and peer nodes should be in the ipv4 firewall
+      become: yes
+      command: iptables-save
+      changed_when: false
+      failed_when: false
+      register: iptables_rules
+
+    - name: Validate ipv4 private firewall configuration
+      assert:
+        that:
+          - "'-A INPUT -s {{ hostvars[item]['nodepool']['private_ipv4'] }}/32 -j ACCEPT' in iptables_rules.stdout"
+      with_items: "{{ groups['all'] }}"
+      when:
+        - hostvars[item]['nodepool']['private_ipv4']
+
+    - name: Validate ipv4 public firewall configuration
+      assert:
+        that:
+          - "'-A INPUT -s {{ hostvars[item]['nodepool']['public_ipv4'] }}/32 -j ACCEPT' in iptables_rules.stdout"
+      with_items: "{{ groups['all'] }}"
+      when:
+        - hostvars[item]['nodepool']['public_ipv4']
+
+    - name: Validate ipv4 bridge firewall configuration
+      assert:
+        that:
+          - "'-A INPUT -s {{ bridge_address_prefix }}.0/{{ bridge_address_subnet }} -d {{ bridge_address_prefix }}.0/{{ bridge_address_subnet }} -j ACCEPT' in iptables_rules.stdout"
+      with_items: "{{ groups['all'] }}"
+
+    # ipv6_addresses is set by the multi-node-firewall role
+    - when: ipv6_addresses | length > 0
+      block:
+        - name: switch and peer nodes should be in the ipv6 firewall
+          become: yes
+          command: ip6tables-save
+          changed_when: false
+          failed_when: false
+          register: ip6tables_rules
+
+        - name: Validate ipv6 firewall configuration
+          assert:
+            that:
+              - "'-A INPUT -s {{ hostvars[item]['nodepool']['public_ipv6'] }}/128 -j ACCEPT' in ip6tables_rules.stdout"
+          with_items: "{{ groups['all'] }}"

zuul-tests.d/general-roles-jobs.yaml

@@ -83,6 +83,176 @@
     tags: auto-generated
     nodeset: ubuntu-xenial
 
+- job:
+    name: zuul-jobs-test-multinode-roles
+    description: |
+      Tests multinode setup roles
+
+      These roles are tested together in this job because they
+      interact with each other.
+    tags: all-platforms-multinode
+    abstract: true
+    run: test-playbooks/multinode/multinode.yaml
+    files:
+      - ^roles/multi-node-bridge/.*
+      - ^roles/multi-node-firewall/.*
+      - ^roles/persistent-firewall/.*
+      - ^roles/multi-node-hosts-file/.*
+      - ^roles/multi-node-known-hosts/.*
+      - ^test-playbooks/multinode/.*
+
+- job:
+    name: zuul-jobs-test-multinode-roles-centos-7
+    description: Tests multinode setup roles on centos-7
+    parent: zuul-jobs-test-multinode-roles
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: primary
+          label: centos-7
+        - name: secondary
+          label: centos-7
+      groups:
+        - name: switch
+          nodes:
+            - primary
+        - name: peers
+          nodes:
+            - secondary
+
+- job:
+    name: zuul-jobs-test-multinode-roles-debian-stable
+    description: Tests multinode setup roles on debian-stable
+    parent: zuul-jobs-test-multinode-roles
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: primary
+          label: debian-stretch
+        - name: secondary
+          label: debian-stretch
+      groups:
+        - name: switch
+          nodes:
+            - primary
+        - name: peers
+          nodes:
+            - secondary
+
+- job:
+    name: zuul-jobs-test-multinode-roles-fedora-latest
+    description: Tests multinode setup roles on fedora-latest
+    parent: zuul-jobs-test-multinode-roles
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: primary
+          label: fedora-29
+        - name: secondary
+          label: fedora-29
+      groups:
+        - name: switch
+          nodes:
+            - primary
+        - name: peers
+          nodes:
+            - secondary
+
+- job:
+    name: zuul-jobs-test-multinode-roles-opensuse-15
+    description: Tests multinode setup roles on opensuse-15
+    parent: zuul-jobs-test-multinode-roles
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: primary
+          label: opensuse-15
+        - name: secondary
+          label: opensuse-15
+      groups:
+        - name: switch
+          nodes:
+            - primary
+        - name: peers
+          nodes:
+            - secondary
+
+- job:
+    name: zuul-jobs-test-multinode-roles-opensuse-tumbleweed
+    description: Tests multinode setup roles on opensuse-tumbleweed
+    parent: zuul-jobs-test-multinode-roles
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: primary
+          label: opensuse-tumbleweed
+        - name: secondary
+          label: opensuse-tumbleweed
+      groups:
+        - name: switch
+          nodes:
+            - primary
+        - name: peers
+          nodes:
+            - secondary
+
+- job:
+    name: zuul-jobs-test-multinode-roles-ubuntu-bionic
+    description: Tests multinode setup roles on ubuntu-bionic
+    parent: zuul-jobs-test-multinode-roles
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: primary
+          label: ubuntu-bionic
+        - name: secondary
+          label: ubuntu-bionic
+      groups:
+        - name: switch
+          nodes:
+            - primary
+        - name: peers
+          nodes:
+            - secondary
+
+- job:
+    name: zuul-jobs-test-multinode-roles-ubuntu-trusty
+    description: Tests multinode setup roles on ubuntu-trusty
+    parent: zuul-jobs-test-multinode-roles
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: primary
+          label: ubuntu-trusty
+        - name: secondary
+          label: ubuntu-trusty
+      groups:
+        - name: switch
+          nodes:
+            - primary
+        - name: peers
+          nodes:
+            - secondary
+
+- job:
+    name: zuul-jobs-test-multinode-roles-ubuntu-xenial
+    description: Tests multinode setup roles on ubuntu-xenial
+    parent: zuul-jobs-test-multinode-roles
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: primary
+          label: ubuntu-xenial
+        - name: secondary
+          label: ubuntu-xenial
+      groups:
+        - name: switch
+          nodes:
+            - primary
+        - name: peers
+          nodes:
+            - secondary
+
 - job:
     name: zuul-jobs-test-upload-git-mirror
     description: Test the upload-git-mirror role
@@ -102,6 +272,14 @@
         - zuul-jobs-test-base-roles-ubuntu-bionic
         - zuul-jobs-test-base-roles-ubuntu-trusty
         - zuul-jobs-test-base-roles-ubuntu-xenial
+        - zuul-jobs-test-multinode-roles-centos-7
+        - zuul-jobs-test-multinode-roles-debian-stable
+        - zuul-jobs-test-multinode-roles-fedora-latest
+        - zuul-jobs-test-multinode-roles-opensuse-15
+        - zuul-jobs-test-multinode-roles-opensuse-tumbleweed
+        - zuul-jobs-test-multinode-roles-ubuntu-bionic
+        - zuul-jobs-test-multinode-roles-ubuntu-trusty
+        - zuul-jobs-test-multinode-roles-ubuntu-xenial
         - zuul-jobs-test-upload-git-mirror
     gate:
       jobs: *id001

zuul-tests.d/project.yaml

@@ -7,24 +7,10 @@
       - build-tox-docs
     check:
       jobs:
-        - openstack-infra-multinode-integration-centos-7
-        - openstack-infra-multinode-integration-debian-stable
-        - openstack-infra-multinode-integration-fedora-latest
-        - openstack-infra-multinode-integration-ubuntu-bionic
-        - openstack-infra-multinode-integration-ubuntu-trusty
-        - openstack-infra-multinode-integration-ubuntu-xenial
-        - openstack-infra-multinode-integration-opensuse423
         - tox-py27
         - tox-py35
     gate:
       jobs:
-        - openstack-infra-multinode-integration-centos-7
-        - openstack-infra-multinode-integration-debian-stable
-        - openstack-infra-multinode-integration-fedora-latest
-        - openstack-infra-multinode-integration-ubuntu-bionic
-        - openstack-infra-multinode-integration-ubuntu-trusty
-        - openstack-infra-multinode-integration-ubuntu-xenial
-        - openstack-infra-multinode-integration-opensuse423
         - tox-py27
         - tox-py35
     post: