From 47bbb96b29ab30764d6220cdd43e63a1d2072533 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 25 Feb 2021 09:23:06 +0000 Subject: [PATCH] CI: retry overcloud jobs that land on vexxhost clouds Jobs that test baremetal deployment are currently failing when run on Vexxhost infrastructure. The seed VM job also has trouble on Vexxhost, although does sometimes pass. It's not clear why yet, and we are still investigating. Until we have a solution, fail in the pre phase when we land on vexxhost clouds, in order to trigger the zuul retry mechanism. Also use the normal number (5) of attempts for upgrade jobs, since these no longer deploy the previous release in the pre phase. Finally, incorporate a fix from https://review.opendev.org/c/openstack/kayobe-config-dev/+/777076 to allow this to merge. This part can be reverted once the kayobe-config-dev patch merges. [1] http://lists.openstack.org/pipermail/openstack-discuss/2021-February/020685.html Change-Id: Ie5ca04428855cbaf505adc57de76ea0b2d404080 --- playbooks/kayobe-overcloud-base/ironic.conf | 2 ++ playbooks/kayobe-overcloud-base/pre.yml | 17 +++++++++++++++++ playbooks/kayobe-overcloud-upgrade-base/pre.yml | 6 ++++++ playbooks/kayobe-seed-vm-base/pre.yml | 6 ++++++ zuul.d/jobs.yaml | 6 ------ 5 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 playbooks/kayobe-overcloud-base/ironic.conf diff --git a/playbooks/kayobe-overcloud-base/ironic.conf b/playbooks/kayobe-overcloud-base/ironic.conf new file mode 100644 index 000000000..45f848825 --- /dev/null +++ b/playbooks/kayobe-overcloud-base/ironic.conf @@ -0,0 +1,2 @@ +[DEFAULT] +minimum_required_memory = 256 diff --git a/playbooks/kayobe-overcloud-base/pre.yml b/playbooks/kayobe-overcloud-base/pre.yml index 38ca53d2a..0fa834231 100644 --- a/playbooks/kayobe-overcloud-base/pre.yml +++ b/playbooks/kayobe-overcloud-base/pre.yml @@ -3,6 +3,12 @@ environment: KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}" tasks: + # FIXME(mgoddard): work out what's going wrong here. + - name: Fail if running on Vexxhost infra + fail: + msg: Baremetal tests fail on vexxhost - roll the dice again + when: "'vexxhost' in ansible_hostname" + # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes # precedence over the standard config files. - name: Ensure kayobe-config override config file exists @@ -16,6 +22,17 @@ dest: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/globals.yml" when: tls_enabled + - name: Ensure kolla custom config dir exists + file: + path: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/config" + state: directory + + # TODO(mgoddard): Remove this when it has been added to kayobe-config-dev. + - name: Ensure ironic.conf override config file exists + copy: + src: ironic.conf + dest: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/config/ironic.conf" + # NOTE(mgoddard): The kayobe dev config by default expects a bridge - # breth1 - to exist with an IP address of 192.168.33.3. - name: Ensure all-in-one network bridge interface exists diff --git a/playbooks/kayobe-overcloud-upgrade-base/pre.yml b/playbooks/kayobe-overcloud-upgrade-base/pre.yml index cefbe5df2..bec74e650 100644 --- a/playbooks/kayobe-overcloud-upgrade-base/pre.yml +++ b/playbooks/kayobe-overcloud-upgrade-base/pre.yml @@ -1,6 +1,12 @@ --- - hosts: primary tasks: + # FIXME(mgoddard): work out what's going wrong here. + - name: Fail if running on Vexxhost infra + fail: + msg: Baremetal tests fail on vexxhost - roll the dice again + when: "'vexxhost' in ansible_hostname" + # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes # precedence over the standard config files. - name: Ensure kayobe-config override config file exists diff --git a/playbooks/kayobe-seed-vm-base/pre.yml b/playbooks/kayobe-seed-vm-base/pre.yml index 85635fa53..db722a6e7 100644 --- a/playbooks/kayobe-seed-vm-base/pre.yml +++ b/playbooks/kayobe-seed-vm-base/pre.yml @@ -3,6 +3,12 @@ environment: KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}" tasks: + # FIXME(mgoddard): work out what's going wrong here. + - name: Fail if running on Vexxhost infra + fail: + msg: Seed VM tests fail on vexxhost - roll the dice again + when: "'vexxhost' in ansible_hostname" + # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes # precedence over the standard config files. - name: Ensure kayobe-config override config file exists diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index c7b9bfe53..0655af26f 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -117,9 +117,6 @@ pre-run: playbooks/kayobe-overcloud-upgrade-base/pre.yml run: playbooks/kayobe-overcloud-upgrade-base/run.yml timeout: 9000 - # Use fewer attempts for upgrades, since pre.yml includes deployment of the - # previous release which takes a considerable amount of time. - attempts: 3 - job: name: kayobe-overcloud-upgrade-centos8 @@ -169,9 +166,6 @@ pre-run: playbooks/kayobe-seed-upgrade-base/pre.yml run: playbooks/kayobe-seed-upgrade-base/run.yml timeout: 5400 - # Use fewer attempts for upgrades, since pre.yml includes deployment of the - # previous release which takes a considerable amount of time. - attempts: 3 - job: name: kayobe-seed-upgrade-centos8