Implement zuul v2/v3 role fetch shim

In order to be able to simultaneously work with the differing implementations of zuul-cloner in zuul v2 and zuul v3 we implement a detection routine to behave appropriately for each version. This is very much a band-aid just to help us transition because once the transition to zuul v3 is complete we should be able to ditch this whole playbook for something built into zuul v3, and use something very simple for executing tests when not using zuul. Change-Id: Ic824e7ca7c564d5de7a27c77a79014ad274afade (cherry picked from commit b58a2c0287)
2017-10-13 16:40:59 +01:00 · 2017-10-13 16:40:59 +01:00 · 36b82710ea
commit 36b82710ea
parent d980701983
4 changed files with 215 additions and 85 deletions
--- a/get-ansible-role-requirements.yml
+++ b/get-ansible-role-requirements.yml
@ -24,6 +24,31 @@
  gather_facts: false
  tasks:

+    - name: Remove target role directories if they are not git repositories
+      shell: |
+        EXIT_CODE=0
+        {% for role in osa_roles %}
+        {%   if role['scm'] == "git" or role['scm'] is undefined %}
+        ROLE_REPO_PATH="{{ lookup('env', 'ANSIBLE_ROLE_DIR') }}/{{ role['name'] | default(role['src'] | basename) }}"
+        if [[ -e ${ROLE_REPO_PATH} ]] && [[ ! -d "${ROLE_REPO_PATH}/.git" ]]; then
+          echo "${ROLE_REPO_PATH} is not a git repo, deleting..."
+          rm -rf "${ROLE_REPO_PATH}"
+          EXIT_CODE=2
+        fi
+        {%   endif %}
+        {% endfor %}
+        exit ${EXIT_CODE}
+      args:
+        executable: /bin/bash
+      register: existing_dir_cleanup
+      changed_when: existing_dir_cleanup.rc == 2
+      failed_when: existing_dir_cleanup.rc not in [0,2]
+
+    - name: Create the ansible role directory
+      file:
+        dest: "{{ lookup('env', 'ANSIBLE_ROLE_DIR') }}"
+        state: directory
+
    - name: Check whether zuul-cloner is installed and provide the path to it
      shell: |
        which zuul-cloner || { [[ -x /usr/zuul-env/bin/zuul-cloner ]] && echo '/usr/zuul-env/bin/zuul-cloner'; }
@ -33,88 +58,162 @@
      failed_when: false
      register: _zuul_cloner_check

-    - name: Remove target directory if required
-      shell: |
-        if [[ ! -d "{{ item.path | default(homedir + '/.ansible/roles') }}/{{ item.name | default(item.src | basename) }}/.git" ]]; then
-          rm -rf "{{ item.path | default(homedir + '/.ansible/roles') }}/{{ item.name | default(item.src | basename) }}"
-        fi
-      args:
-        executable: /bin/bash
-      when:
-        - item.scm == "git" or item.scm is undefined
-      with_items: "{{ roles }}"
-
-    - name: Prepare git clone list
+    - name: Set fact to decide which version of zuul to cater to, if any
      set_fact:
-        git_roles: >
-          {%- set filtered_role_list = [] %}
-          {%- for role in roles %}
-          {%-   if (not role.src | match(".*git.openstack.org.*")) or
-                   (role.name | match("os_previous_.*")) or
-                   (role.name | match("previous_.*")) %}
-          {%-     set _ = filtered_role_list.append(role) %}
+        zuul_version: >-
+          {#- This tells us that we are running under zuul v2 or v3 #}
+          {%- if _zuul_cloner_check.rc == 0 %}
+          {#-   So now we check for an environment variable that's unique to zuul v2 #}
+          {#-   ref: http://git.openstack.org/cgit/openstack-infra/zuul/tree/zuul/ansible/filter/zuul_filters.py?h=feature/zuulv3#n17 #}
+          {%-   if lookup('env', 'ZUUL_REF') != "" %}
+          {%-     set version_output="2" %}
+          {%-   else %}
+          {%-     set version_output="3" %}
          {%-   endif %}
-          {%- endfor %}
-          {{- filtered_role_list -}}
-        zuul_roles: >
-          {%- set filtered_role_list = [] %}
-          {%- for role in roles %}
-          {%-   if (role.src | match(".*git.openstack.org.*")) and (not role.name | match("os_previous_.*")) %}
-          {%-     set role_src_cleaned = role.src | regex_replace('https://git.openstack.org/', '') %}
-          {%-     set _ = filtered_role_list.append(role_src_cleaned) %}
-          {%-   endif %}
-          {%- endfor %}
-          {{- filtered_role_list -}}
-        openstack_repo_list: >
-          {%- set filtered_repo_list = [] %}
-          {%- set repo_list = lookup('env', 'ZUUL_CHANGES').split('^') %}
-          {%- for repo in repo_list %}
-          {%-   set repo_cleaned = repo | regex_replace(':.*$', '') %}
-          {%-   if not repo_cleaned | match("^openstack/openstack-ansible.*") %}
-          {%-     set _ = filtered_repo_list.append(repo_cleaned) %}
-          {%-   endif %}
-          {%- endfor %}
-          {{- filtered_repo_list -}}
-      when:
-        - _zuul_cloner_check.rc == 0
+          {%- else %}
+          {%-   set version_output="none" %}
+          {%- endif %}
+          {{- version_output }}

-    - name: Clone git repos (with git)
-      git:
-        repo: "{{ item.src }}"
-        dest: "{{ homedir }}/.ansible/roles/{{ item.name | default(item.src | basename) }}"
-        version: "{{ item.version | default('master') }}"
-        update: true
-        force: true
-      when:
-        - item.scm == "git" or item.scm is undefined
-      with_items: "{{ git_roles | default(roles) }}"
+    - name: Clone git repos (no zuul)
+      block:
+        - name: Prepare git clone list
+          set_fact:
+            git_roles: "{{ osa_roles }}"

-    - name: Create clone map
-      copy:
-        content: |
-          clonemap:
-            - name: 'openstack/openstack-ansible-tests'
-              dest: '{{ lookup("env", "WORKING_DIR") }}/tests/common'
-            - name: 'openstack/openstack-ansible-(?!tests)(.*)'
-              dest: '{{ homedir }}/.ansible/roles/\1'
-            - name: 'openstack/(?!openstack-ansible)(.*)'
-              dest: '{{ homedir }}/git/openstack/\1'
-        dest: "{{ homedir }}/.ansible/clonemap.yml"
+        - name: Clone git repos
+          git:
+            repo: "{{ item['src'] }}"
+            dest: "{{ lookup('env', 'ANSIBLE_ROLE_DIR') }}/{{ item['name'] | default(item['src'] | basename) }}"
+            version: "{{ item['version'] | default('master') }}"
+            update: true
+            force: true
+          with_items: "{{ git_roles }}"
+          when:
+            - item['scm'] == "git" or item['scm'] is undefined
      when:
-        - _zuul_cloner_check.rc == 0
+        - zuul_version == "none"

-    - name: Clone git repos (with zuul-cloner)
-      shell: |
-        {{ _zuul_cloner_check.stdout }} \
-                -m {{ homedir }}/.ansible/clonemap.yml \
-                --cache-dir /opt/git \
-                git://git.openstack.org \
-        {% for repo in zuul_roles + openstack_repo_list %}
-                {{ repo }} \
-        {% endfor %}
+    - name: Clone git repos (zuul v2)
+      block:
+        - name: Prepare git clone list
+          set_fact:
+            git_roles: >
+              {%- set filtered_role_list = [] %}
+              {%- for role in osa_roles %}
+              {%-   if (not role.src | match(".*git.openstack.org.*")) or
+                       (role.name | match("os_previous_.*")) or
+                       (role.name | match("previous_.*")) %}
+              {%-     set _ = filtered_role_list.append(role) %}
+              {%-   endif %}
+              {%- endfor %}
+              {{- filtered_role_list -}}
+            zuul_roles: >
+              {%- set filtered_role_list = [] %}
+              {%- for role in osa_roles %}
+              {%-   if (role.src | match(".*git.openstack.org.*")) and (not role.name | match("os_previous_.*")) %}
+              {%-     set role_src_cleaned = role.src | regex_replace('https://git.openstack.org/', '') %}
+              {%-     set _ = filtered_role_list.append(role_src_cleaned) %}
+              {%-   endif %}
+              {%- endfor %}
+              {{- filtered_role_list -}}
+            openstack_repo_list: >
+              {%- set filtered_repo_list = [] %}
+              {%- set repo_list = lookup('env', 'ZUUL_CHANGES').split('^') %}
+              {%- for repo in repo_list %}
+              {%-   set repo_cleaned = repo | regex_replace(':.*$', '') %}
+              {%-   if not repo_cleaned | match("^openstack/openstack-ansible.*") %}
+              {%-     set _ = filtered_repo_list.append(repo_cleaned) %}
+              {%-   endif %}
+              {%- endfor %}
+              {{- filtered_repo_list -}}
+
+        - name: Clone git repos (non openstack)
+          git:
+            repo: "{{ item['src'] }}"
+            dest: "{{ lookup('env', 'ANSIBLE_ROLE_DIR') }}/{{ item['name'] | default(item['src'] | basename) }}"
+            version: "{{ item['version'] | default('master') }}"
+            update: true
+            force: true
+          with_items: "{{ git_roles }}"
+          when:
+            - item['scm'] == "git" or item['scm'] is undefined
+
+        - name: Create clone map
+          copy:
+            content: |
+              clonemap:
+                - name: 'openstack/openstack-ansible-tests'
+                  dest: '{{ lookup("env", "WORKING_DIR") }}/tests/common'
+                - name: 'openstack/openstack-ansible-(?!tests)(.*)'
+                  dest: '{{ lookup("env", "ANSIBLE_ROLE_DIR") }}/\1'
+                - name: 'openstack/(?!openstack-ansible)(.*)'
+                  dest: '{{ homedir }}/git/openstack/\1'
+            dest: "{{ homedir }}/.ansible/clonemap.yml"
+
+        - name: Clone git repos (with zuul-cloner)
+          shell: |
+            {{ _zuul_cloner_check.stdout }} \
+                    -m {{ homedir }}/.ansible/clonemap.yml \
+                    --cache-dir /opt/git \
+                    git://git.openstack.org \
+            {% for repo in zuul_roles + openstack_repo_list %}
+                    {{ repo }} \
+            {% endfor %}
      when:
-        - _zuul_cloner_check.rc == 0
+        - zuul_version == "2"
+
+    - name: Clone git repos (zuul v3)
+      block:
+        - name: Find repositories already cloned by zuul
+          command: "find {{ zuul_git_src_dir }} -type d -maxdepth 2"
+          register: zuul_src_folder
+
+        - name: Simplify the given src repository list
+          set_fact:
+            zuul_src_repo_list: >
+              {%- set filtered_repo_list = [] %}
+              {%- for folder_path in zuul_src_folder['stdout_lines'] %}
+              {%-   if folder_path | match("^" ~ zuul_git_src_dir ~ "/openstack/openstack-ansible-(?!tests).*") %}
+              {%-     set repo_cleaned = folder_path | regex_replace('^' ~ zuul_git_src_dir ~ '/', '') %}
+              {%-     set _ = filtered_repo_list.append(repo_cleaned) %}
+              {%-   endif %}
+              {%- endfor %}
+              {{- filtered_repo_list -}}
+
+        - name: Prepare git clone list
+          set_fact:
+            git_roles: >
+              {%- set filtered_role_list = [] %}
+              {%- for role in osa_roles %}
+              {%-   if role['src'] | regex_replace('https://git.openstack.org/', '') not in zuul_src_repo_list %}
+              {%-     set _ = filtered_role_list.append(role) %}
+              {%-   endif %}
+              {%- endfor %}
+              {{- filtered_role_list -}}
+
+        - name: Link the zuul provided roles
+          file:
+            src: "{{ zuul_git_src_dir }}/{{ item }}"
+            dest: "{{ lookup('env', 'ANSIBLE_ROLE_DIR') }}/{{ item | regex_replace('openstack/openstack-ansible-', '') }}"
+            state: link
+            force: yes
+          with_items: "{{ zuul_src_repo_list }}"
+
+        - name: Clone the remaining git repos
+          git:
+            repo: "{{ item['src'] }}"
+            dest: "{{ lookup('env', 'ANSIBLE_ROLE_DIR') }}/{{ item['name'] | default(item['src'] | basename) }}"
+            version: "{{ item['version'] | default('master') }}"
+            update: true
+            force: true
+          with_items: "{{ git_roles }}"
+          when:
+            - item['scm'] == "git" or item['scm'] is undefined
+      when:
+        - zuul_version == "3"

  vars:
    role_file: "{{ toxinidir }}/tests/ansible-role-requirements.yml"
-    roles: "{{ lookup('file', role_file) | from_yaml }}"
+    osa_roles: "{{ lookup('file', role_file) | from_yaml }}"
+    zuul_git_src_dir: "/home/zuul/src/git.openstack.org"
--- a/test-ansible-env-prep.sh
+++ b/test-ansible-env-prep.sh
@ -94,7 +94,11 @@ mkdir -p "${ANSIBLE_LOG_DIR}"
 # If zuul-cloner is present, use it so that we
 # also include any dependent patches from the
 # plugins repo noted in the commit message.
-if [[ -x /usr/zuul-env/bin/zuul-cloner ]]; then
+# We only want to use zuul-cloner if we detect
+# zuul v2 running, so we check for the presence
+# of the ZUUL_REF environment variable.
+# ref: http://git.openstack.org/cgit/openstack-infra/zuul/tree/zuul/ansible/filter/zuul_filters.py?h=feature/zuulv3#n17
+if [[ -x /usr/zuul-env/bin/zuul-cloner ]] && [[ "${ZUUL_REF:-none}" != "none" ]]; then

    # Prepare the clonemap for zuul-cloner to use
    create_plugins_clonemap
@ -117,8 +121,11 @@ if [[ -x /usr/zuul-env/bin/zuul-cloner ]]; then
 else
    if [[ ! -d "${ANSIBLE_PLUGIN_DIR}" ]]; then
        # The plugins repo doesn't need a clone, we can just
-        # symlink it.
-        if [[ "$(basename ${WORKING_DIR})" == "openstack-ansible-plugins" ]]; then
+        # symlink it. As zuul v3 clones into a folder called
+        # 'workspace' we have to use one of its environment
+        # variables to determine the project name.
+        if [[ "${ZUUL_SHORT_PROJECT_NAME:-none}" == "openstack-ansible-plugins" ]] ||\
+           [[ "$(basename ${WORKING_DIR})" == "openstack-ansible-plugins" ]]; then
            ln -s ${WORKING_DIR} "${ANSIBLE_PLUGIN_DIR}"
        else
            git clone -b stable/pike \
@ -129,8 +136,11 @@ else

    if [[ ! -d "${OSA_OPS_DIR}" ]]; then
        # The ops repo doesn't need a clone, we can just
-        # symlink it.
-        if [[ "$(basename ${WORKING_DIR})" == "openstack-ansible-ops" ]]; then
+        # symlink it. As zuul v3 clones into a folder called
+        # 'workspace' we have to use one of its environment
+        # variables to determine the project name.
+        if [[ "${ZUUL_SHORT_PROJECT_NAME:-none}" == "openstack-ansible-ops" ]] ||\
+           [[ "$(basename ${WORKING_DIR})" == "openstack-ansible-ops" ]]; then
            ln -s ${WORKING_DIR} "${OSA_OPS_DIR}"
        else
            git clone -b stable/pike \
@ -146,7 +156,7 @@ if [ ! -d "${ANSIBLE_ROLE_DIR}" ] && [ -f "${ANSIBLE_ROLE_REQUIREMENTS_PATH}" ];
   ansible-playbook -i ${ANSIBLE_INVENTORY} \
         ${COMMON_TESTS_PATH}/get-ansible-role-requirements.yml \
         -e "toxinidir=${WORKING_DIR} homedir=${TESTING_HOME}" \
-         -v
+         -vvv
 fi


--- a/tests/tests-repo-clone.sh
+++ b/tests/tests-repo-clone.sh
@ -52,7 +52,11 @@ EOF
 # If zuul-cloner is present, use it so that we
 # also include any dependent patches from the
 # tests repo noted in the commit message.
-if [[ -x /usr/zuul-env/bin/zuul-cloner ]]; then
+# We only want to use zuul-cloner if we detect
+# zuul v2 running, so we check for the presence
+# of the ZUUL_REF environment variable.
+# ref: http://git.openstack.org/cgit/openstack-infra/zuul/tree/zuul/ansible/filter/zuul_filters.py?h=feature/zuulv3#n17
+if [[ -x /usr/zuul-env/bin/zuul-cloner ]] && [[ "${ZUUL_REF:-none}" != "none" ]]; then

    # Prepare the clonemap for zuul-cloner to use
    create_tests_clonemap
@ -74,8 +78,11 @@ if [[ -x /usr/zuul-env/bin/zuul-cloner ]]; then
 elif [[ ! -d tests/common ]]; then

    # The tests repo doesn't need a clone, we can just
-    # symlink it.
-    if [[ "$(basename ${WORKING_DIR})" == "openstack-ansible-tests" ]]; then
+    # symlink it. As zuul v3 clones into a folder called
+    # 'workspace' we have to use one of its environment
+    # variables to determine the project name.
+    if [[ "${ZUUL_SHORT_PROJECT_NAME:-none}" == "openstack-ansible-tests" ]] ||\
+       [[ "$(basename ${WORKING_DIR})" == "openstack-ansible-tests" ]]; then
        ln -s ${WORKING_DIR} ${WORKING_DIR}/tests/common
    else
        git clone -b stable/pike \
--- a/tox.ini
+++ b/tox.ini
@ -1,7 +1,7 @@
 [tox]
 minversion = 2.0
 skipsdist = True
-envlist = docs,linters,functional
+envlist = docs,linters,func


 [testenv]
@ -102,6 +102,20 @@ commands =
    bash -c "{toxinidir}/tests/common/test-ansible-lint.sh"


+# This is here due to a bug in the ubuntu-xenial
+# job which does not do the translation from
+# 'func' to 'functional' for us.
+# TODO(odyssey4me):
+# Remove this once https://review.openstack.org/512351
+# has been merged.
+[testenv:func]
+deps =
+    {[testenv:ansible]deps}
+commands =
+    bash -c "{toxinidir}/tests/tests-repo-clone.sh"
+    bash -c "{toxinidir}/tests/common/test-ansible-functional.sh"
+
+
 [testenv:functional]
 deps =
    {[testenv:ansible]deps}