From 8d910aca970e738468b1ba5302ecd99a3ffb1de4 Mon Sep 17 00:00:00 2001 From: David Shrewsbury Date: Thu, 16 Nov 2017 12:39:06 -0500 Subject: [PATCH] Convert from legacy to native devstack job Changes nodepool-functional-py35 and nodepool-functional-py35-src to use the native devstack job and not the legacy job. Change-Id: I3d97d83360816001da0f599d95b13eae3abb6c96 --- .zuul.yaml | 45 +++++-- devstack/plugin.sh | 1 + playbooks/nodepool-functional-base/post.yaml | 9 ++ playbooks/nodepool-functional-base/pre.yaml | 11 ++ .../nodepool-functional-py35-src/post.yaml | 15 --- .../nodepool-functional-py35-src/run.yaml | 123 +----------------- playbooks/nodepool-functional-py35/post.yaml | 15 --- playbooks/nodepool-functional-py35/run.yaml | 87 +------------ tools/check_devstack_plugin.sh | 16 +-- 9 files changed, 72 insertions(+), 250 deletions(-) create mode 100644 playbooks/nodepool-functional-base/post.yaml create mode 100644 playbooks/nodepool-functional-base/pre.yaml delete mode 100644 playbooks/nodepool-functional-py35-src/post.yaml delete mode 100644 playbooks/nodepool-functional-py35/post.yaml diff --git a/.zuul.yaml b/.zuul.yaml index 1904788ff..d00e33106 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,21 +1,48 @@ - job: - name: nodepool-functional-py35 - parent: legacy-dsvm-base - run: playbooks/nodepool-functional-py35/run.yaml - post-run: playbooks/nodepool-functional-py35/post.yaml + name: nodepool-functional-base + parent: devstack + pre-run: playbooks/nodepool-functional-base/pre.yaml + post-run: playbooks/nodepool-functional-base/post.yaml timeout: 5400 required-projects: - - openstack-infra/devstack-gate - openstack-infra/nodepool + vars: + devstack_services: + horizon: false + ceilometer-acentral: false + ceilometer-acompute: false + ceilometer-alarm-evaluator: false + ceilometer-alarm-notifier: false + ceilometer-anotification: false + ceilometer-api: false + ceilometer-collector: false + cinder: false + c-bak: false + c-sch: false + c-api: false + c-vol: false + s-account: false + s-container: false + s-object: false + s-proxy: false + devstack_plugins: + nodepool: https://git.openstack.org/openstack-infra/nodepool + +- job: + name: nodepool-functional-py35 + parent: nodepool-functional-base + description: | + Run nodepool functional tests for py35 + run: playbooks/nodepool-functional-py35/run.yaml - job: name: nodepool-functional-py35-src - parent: legacy-dsvm-base + parent: nodepool-functional-base run: playbooks/nodepool-functional-py35-src/run.yaml - post-run: playbooks/nodepool-functional-py35-src/post.yaml - timeout: 5400 + vars: + devstack_localrc: + LIBS_FROM_GIT: shade,glean,diskimage-builder required-projects: - - openstack-infra/devstack-gate - openstack-infra/glean - openstack-infra/nodepool - openstack-infra/shade diff --git a/devstack/plugin.sh b/devstack/plugin.sh index fc559dce1..41609d153 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -69,6 +69,7 @@ function install_nodepool { setup_develop $DEST/nodepool $NODEPOOL_INSTALL/bin/pip install $DEST/nodepool + $NODEPOOL_INSTALL/bin/pbr freeze } # requires some globals from devstack, which *might* not be stable api diff --git a/playbooks/nodepool-functional-base/post.yaml b/playbooks/nodepool-functional-base/post.yaml new file mode 100644 index 000000000..2a784f035 --- /dev/null +++ b/playbooks/nodepool-functional-base/post.yaml @@ -0,0 +1,9 @@ +- hosts: all + vars: + nodepool_log_dir: '{{ ansible_user_dir }}/work/logs/nodepool' + tasks: + - name: 'Copy files from {{ nodepool_log_dir }}' + synchronize: + src: '{{ nodepool_log_dir }}' + dest: '{{ zuul.executor.log_root }}/{{ inventory_hostname }}' + mode: pull diff --git a/playbooks/nodepool-functional-base/pre.yaml b/playbooks/nodepool-functional-base/pre.yaml new file mode 100644 index 000000000..5ac259cff --- /dev/null +++ b/playbooks/nodepool-functional-base/pre.yaml @@ -0,0 +1,11 @@ +- hosts: all + roles: + - run-devstack + - role: bindep + bindep_profile: default + + tasks: + - name: Ensure nodepool output log directory + file: + path: '{{ ansible_user_dir }}/work/logs/nodepool' + state: directory diff --git a/playbooks/nodepool-functional-py35-src/post.yaml b/playbooks/nodepool-functional-py35-src/post.yaml deleted file mode 100644 index e07f5510a..000000000 --- a/playbooks/nodepool-functional-py35-src/post.yaml +++ /dev/null @@ -1,15 +0,0 @@ -- hosts: primary - tasks: - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=/logs/** - - --include=*/ - - --exclude=* - - --prune-empty-dirs diff --git a/playbooks/nodepool-functional-py35-src/run.yaml b/playbooks/nodepool-functional-py35-src/run.yaml index 812532ee1..00b9dd590 100644 --- a/playbooks/nodepool-functional-py35-src/run.yaml +++ b/playbooks/nodepool-functional-py35-src/run.yaml @@ -1,121 +1,6 @@ - hosts: all - name: Autoconverted job legacy-dsvm-nodepool-py35-src from old job gate-dsvm-nodepool-py35-src-nv tasks: - - - name: Ensure legacy workspace directory - file: - path: '{{ ansible_user_dir }}/workspace' - state: directory - - - shell: - cmd: | - set -e - set -x - cat > clonemap.yaml << EOF - clonemap: - - name: openstack-infra/devstack-gate - dest: devstack-gate - EOF - /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \ - git://git.openstack.org \ - openstack-infra/devstack-gate - executable: /bin/bash - chdir: '{{ ansible_user_dir }}/workspace' - environment: '{{ zuul | zuul_legacy_vars }}' - - - shell: - cmd: | - set -e - set -x - export PYTHONUNBUFFERED=true - # Disable tempest as nodepool is talking to the cloud not tempest. - export DEVSTACK_GATE_TEMPEST=0 - # Use neutron as the public clouds in use are neutron based. - export DEVSTACK_GATE_NEUTRON=1 - # The nodepool process needs sudo rights in order to - # perform dib image builds - export DEVSTACK_GATE_REMOVE_STACK_SUDO=0 - # Disable services we do not need for nodepool - export DEVSTACK_LOCAL_CONFIG="disable_service horizon" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-acentral" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-acompute" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-alarm-evaluator" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-alarm-notifier" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-anotification" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-api" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-collector" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service cinder" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service c-bak" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service c-sch" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service c-api" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service c-vol" - - if [ "-py35" == "-py35" ]; then - export DEVSTACK_GATE_USE_PYTHON3=True - - # swift is not ready for python3 yet - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service s-account" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service s-container" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service s-object" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service s-proxy" - fi - - export BRANCH_OVERRIDE=default - if [ "$BRANCH_OVERRIDE" != "default" ] ; then - export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE - fi - # Because we are testing a non standard project, add the - # our project repository. This makes zuul do the right - # reference magic for testing changes. - export PROJECTS="openstack-infra/nodepool $PROJECTS" - # note the actual url here is somewhat irrelevant because it - # caches in nodepool, however make it a valid url for - # documentation purposes. - export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin nodepool git://git.openstack.org/openstack-infra/nodepool" - export DEVSTACK_PROJECT_FROM_GIT="diskimage-builder" - export DEVSTACK_PROJECT_FROM_GIT+=",glean" - export DEVSTACK_PROJECT_FROM_GIT+=",shade" - - # Start with all images disabled. - export NODEPOOL_PAUSE_CENTOS_7_DIB=true - export NODEPOOL_PAUSE_DEBIAN_JESSIE_DIB=true - export NODEPOOL_PAUSE_FEDORA_26_DIB=true - export NODEPOOL_PAUSE_OPENSUSE_423_DIB=true - export NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB=true - export NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB=true - - if [ "" == "" ] ; then - # dsvm-nodepool-src - export NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB=false - elif [ "" == "-debian" ] ; then - # dsvm-nodepool-debian-src - export NODEPOOL_PAUSE_DEBIAN_JESSIE_DIB=false - elif [ "" == "-opensuse" ] ; then - # dsvm-nodepool-opensuse-src - export NODEPOOL_PAUSE_OPENSUSE_423_DIB=false - elif [ "" == "-redhat" ] ; then - # dsvm-nodepool-redhat-src - export NODEPOOL_PAUSE_CENTOS_7_DIB=false - export NODEPOOL_PAUSE_FEDORA_26_DIB=false - elif [ "" == "-ubuntu" ] ; then - # dsvm-nodepool-ubuntu-src - export NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB=false - export NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB=false - fi - export DEVSTACK_LOCAL_CONFIG+=$'\n'"NODEPOOL_PAUSE_CENTOS_7_DIB=$NODEPOOL_PAUSE_CENTOS_7_DIB" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"NODEPOOL_PAUSE_DEBIAN_JESSIE_DIB=$NODEPOOL_PAUSE_DEBIAN_JESSIE_DIB" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"NODEPOOL_PAUSE_FEDORA_26_DIB=$NODEPOOL_PAUSE_FEDORA_26_DIB" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"NODEPOOL_PAUSE_OPENSUSE_423_DIB=$NODEPOOL_PAUSE_OPENSUSE_423_DIB" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB=$NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB=$NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB" - - function post_test_hook { - /opt/stack/new/nodepool/tools/check_devstack_plugin.sh - } - export -f post_test_hook - - cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh - ./safe-devstack-vm-gate-wrap.sh - executable: /bin/bash - chdir: '{{ ansible_user_dir }}/workspace' - environment: '{{ zuul | zuul_legacy_vars }}' + - name: Run nodepool tests + command: tools/check_devstack_plugin.sh {{ ansible_user_dir }}/work/logs/nodepool + args: + chdir: '{{ zuul.project.src_dir }}' diff --git a/playbooks/nodepool-functional-py35/post.yaml b/playbooks/nodepool-functional-py35/post.yaml deleted file mode 100644 index e07f5510a..000000000 --- a/playbooks/nodepool-functional-py35/post.yaml +++ /dev/null @@ -1,15 +0,0 @@ -- hosts: primary - tasks: - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=/logs/** - - --include=*/ - - --exclude=* - - --prune-empty-dirs diff --git a/playbooks/nodepool-functional-py35/run.yaml b/playbooks/nodepool-functional-py35/run.yaml index d62470167..00b9dd590 100644 --- a/playbooks/nodepool-functional-py35/run.yaml +++ b/playbooks/nodepool-functional-py35/run.yaml @@ -1,85 +1,6 @@ - hosts: all - name: Autoconverted job legacy-dsvm-nodepool-py35 from old job gate-dsvm-nodepool-py35-nv tasks: - - - name: Ensure legacy workspace directory - file: - path: '{{ ansible_user_dir }}/workspace' - state: directory - - - shell: - cmd: | - set -e - set -x - cat > clonemap.yaml << EOF - clonemap: - - name: openstack-infra/devstack-gate - dest: devstack-gate - EOF - /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \ - git://git.openstack.org \ - openstack-infra/devstack-gate - executable: /bin/bash - chdir: '{{ ansible_user_dir }}/workspace' - environment: '{{ zuul | zuul_legacy_vars }}' - - - shell: - cmd: | - set -e - set -x - export PYTHONUNBUFFERED=true - # Disable tempest as nodepool is talking to the cloud not tempest. - export DEVSTACK_GATE_TEMPEST=0 - # Use neutron as the public clouds in use are neutron based. - export DEVSTACK_GATE_NEUTRON=1 - # The nodepool process needs sudo rights in order to - # perform dib image builds - export DEVSTACK_GATE_REMOVE_STACK_SUDO=0 - # Disable services we do not need for nodepool - export DEVSTACK_LOCAL_CONFIG="disable_service horizon" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-acentral" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-acompute" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-alarm-evaluator" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-alarm-notifier" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-anotification" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-api" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-collector" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service cinder" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service c-bak" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service c-sch" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service c-api" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service c-vol" - - if [ "-py35" == "-py35" ]; then - export DEVSTACK_GATE_USE_PYTHON3=True - - # swift is not ready for python3 yet - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service s-account" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service s-container" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service s-object" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service s-proxy" - fi - - export BRANCH_OVERRIDE=default - if [ "$BRANCH_OVERRIDE" != "default" ] ; then - export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE - fi - # Because we are testing a non standard project, add the - # our project repository. This makes zuul do the right - # reference magic for testing changes. - export PROJECTS="openstack-infra/nodepool $PROJECTS" - # note the actual url here is somewhat irrelevant because it - # caches in nodepool, however make it a valid url for - # documentation purposes. - export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin nodepool git://git.openstack.org/openstack-infra/nodepool" - - function post_test_hook { - /opt/stack/new/nodepool/tools/check_devstack_plugin.sh - } - export -f post_test_hook - - cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh - ./safe-devstack-vm-gate-wrap.sh - executable: /bin/bash - chdir: '{{ ansible_user_dir }}/workspace' - environment: '{{ zuul | zuul_legacy_vars }}' + - name: Run nodepool tests + command: tools/check_devstack_plugin.sh {{ ansible_user_dir }}/work/logs/nodepool + args: + chdir: '{{ zuul.project.src_dir }}' diff --git a/tools/check_devstack_plugin.sh b/tools/check_devstack_plugin.sh index 285a5ea6b..8b86fa2a0 100755 --- a/tools/check_devstack_plugin.sh +++ b/tools/check_devstack_plugin.sh @@ -1,6 +1,8 @@ #!/bin/bash -ex -NODEPOOL_INSTALL=${NODEPOOL_INSTALL:-/opt/stack/new/nodepool-venv} +LOGDIR=$1 + +NODEPOOL_INSTALL=${NODEPOOL_INSTALL:-/opt/stack/nodepool-venv} NODEPOOL_CONFIG=${NODEPOOL_CONFIG:-/etc/nodepool/nodepool.yaml} NODEPOOL_SECURE=${NODEPOOL_SECURE:-/etc/nodepool/secure.conf} NODEPOOL="$NODEPOOL_INSTALL/bin/nodepool -c $NODEPOOL_CONFIG -s $NODEPOOL_SECURE" @@ -20,10 +22,8 @@ function waitforimage { state='ready' while ! $NODEPOOL image-list | grep $name | grep $state; do - $NODEPOOL image-list > /tmp/.nodepool-image-list.txt - $NODEPOOL list > /tmp/.nodepool-list.txt - sudo mv /tmp/.nodepool-image-list.txt $WORKSPACE/logs/nodepool-image-list.txt - sudo mv /tmp/.nodepool-list.txt $WORKSPACE/logs/nodepool-list.txt + $NODEPOOL image-list > ${LOGDIR}/nodepool-image-list.txt + $NODEPOOL list > ${LOGDIR}/nodepool-list.txt sleep 10 done } @@ -33,10 +33,8 @@ function waitfornode { state='ready' while ! $NODEPOOL list | grep $name | grep $state | grep "unlocked"; do - $NODEPOOL image-list > /tmp/.nodepool-image-list.txt - $NODEPOOL list > /tmp/.nodepool-list.txt - sudo mv /tmp/.nodepool-image-list.txt $WORKSPACE/logs/nodepool-image-list.txt - sudo mv /tmp/.nodepool-list.txt $WORKSPACE/logs/nodepool-list.txt + $NODEPOOL image-list > ${LOGDIR}/nodepool-image-list.txt + $NODEPOOL list > ${LOGDIR}/nodepool-list.txt sleep 10 done }