Refactor buildtestpackages for downstream purpose

It includes:
* parameterize rdoinfo_repo_name
* Use zuul dict to parse zuul changes and remove zuul_deps
  ansible plugin as it is no longer needed.
* Added dlrn config template for downstream and should be
  enabled only when osp_release is defined.
* Use latest rdopkg
* It also fixes the DLRN installing from source.
* Moves dlrn command to shell script
* Remove deprecated parts of DLRN-0.8.0
* find out zuul dict only when there is a change_url otherwise
  ignore it.

https://tree.taiga.io/project/tripleo-ci-board/task/1789

Change-Id: I48bcddbc7693375396a99dc0ee79d32f6e4a4863
Signed-off-by: Chandan Kumar (raukadah) <chkumar@redhat.com>

roles/build-test-packages/defaults/main.yml

@@ -1,9 +1,9 @@
 ---
 
 build_repo_dir: "{{ ansible_user_dir }}"
-
+rdoinfo_repo_name: rdoinfo
 artg_dlrn_repo_url: "https://github.com/openstack-packages/DLRN.git"
-artg_rdoinfo_repo_url: "https://github.com/redhat-openstack/rdoinfo"
+artg_rdoinfo_repo_url: https://github.com/redhat-openstack/rdoinfo
 artg_compressed_gating_repo: "/home/stack/gating_repo.tar.gz"
 artg_build_one: true
 artg_rdo_packages: []

roles/build-test-packages/library/zuul_deps.py

@@ -1,83 +0,0 @@
-#!/usr/bin/python
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# see http://docs.ansible.com/developing_modules.html#common-module-boilerplate
-from ansible.module_utils.basic import *
-
-DOCUMENTATION = '''
----
-module: zuul_deps
-version_added: "2.0"
-short_description: Transforms the dependent changes variable from Zuul format into a dictionary
-description:
-    - Transforms the dependent changes variable from Zuul format into a dictionary
-options:
-  host:
-    description:
-      - The content of the ZUUL_URL variable
-    required: True
-  changes:
-    description:
-      - The content of the ZUUL_CHANGES variable
-    required: True
-  branch:
-    description:
-      - Which branch to transform dependent changes on.
-    default: None
-'''
-
-EXAMPLES = '''
-- zuul_deps:
-    host: https://review.opendev.org
-    changes: "openstack/tripleo-heat-templates:master:refs/changes/88/296488/1^openstack/instack-undercloud:master:refs/changes/84/315184/5"
-'''
-
-
-def process(host, changes, branch):
-    """Process the changes from Zuul format"""
-    output = []
-
-    for item in reversed(changes.split("^")):
-        params = item.split(":")
-        if branch and branch != params[1]:
-            continue
-        if params[0] in [i['project'] for i in output]:
-            continue
-        # add URL prefix if only the bare hostname is given
-        if host.find('://') == -1:
-            host = ''.join(['https://', host])
-        output.append({"host": host,
-                       "project": params[0],
-                       "branch": params[1],
-                       "refspec": params[2]})
-    return {'changed': True,
-            'ansible_facts': {'zuul_change_list': output}}
-
-
-def main():
-    module = AnsibleModule(
-        argument_spec=dict(
-            host=dict(required=True, type='str'),
-            changes=dict(required=True, type='str'),
-            branch=dict(default=None, type='str')
-        )
-    )
-    result = process(module.params['host'],
-                     module.params['changes'],
-                     module.params['branch'])
-    module.exit_json(**result)
-
-
-if __name__ == "__main__":
-    main()

roles/build-test-packages/tasks/dlrn-build.yml

@@ -17,6 +17,13 @@
     artg_distro: "{{ artg_release }}-rdo"
   when: artg_release not in ['master', 'mitaka']
 
+- name: Make sure /usr/share/rhn/RHN-ORG-TRUSTED-SSL-CERT exists (RHEL)
+  shell:
+    cmd: |
+      touch /usr/share/rhn/RHN-ORG-TRUSTED-SSL-CERT
+  become: true
+  when: ansible_distribution in ['RedHat']
+
 - name: Use local baseurl when using local mirrors
   set_fact:
     dlrn_baseurl: 'file:///tmp/fake-dlrn-repos'
@@ -33,6 +40,19 @@
     - {regexp: 'source=.*', line: 'source={{ artg_change.branch }}'}
   loop_control:
     loop_var: project_item
+  when: osp_release is not defined
+
+- name: Override projects.ini settings for downstream
+  lineinfile:
+    dest: '{{ build_repo_dir }}/DLRN/projects.ini'
+    regexp: '{{ project_item.regexp }}'
+    line: '{{ project_item.line }}'
+  with_items:
+    - {regexp: 'baseurl=.*', line: 'baseurl={{ dlrn_baseurl }}'}
+    - {regexp: 'source=.*', line: 'source={{ artg_change.branch }}'}
+  loop_control:
+    loop_var: project_item
+  when: osp_release is defined
 
 - block:
     - name: Map project name to DLRN project name
@@ -42,7 +62,10 @@
         cmd: |
           source '{{ build_repo_dir }}/dlrn-venv/bin/activate'
           set -xeo pipefail
-          rdopkg findpkg -s '{{ artg_change.project | basename }}' -l rdoinfo | \
+          # {{ rdoinfo_repo_name }}/{{ rdoinfo_repo_name.split('info')[0] }}-full.yml will
+          # return rdo-full.yml and for downstream is osp-full.yml.
+          rdopkg findpkg -s '{{ artg_change.project | basename }}' \
+            -i {{ rdoinfo_repo_name }}/{{ rdoinfo_repo_name.split('info')[0] }}-full.yml | \
             awk '/^name/{print $2}; {print "findpkg: " $0 > "/dev/stderr"}'
       changed_when: false
       failed_when: project_name_mapped.rc != 0 or not project_name_mapped.stdout
@@ -92,26 +115,17 @@
         - artg_change.distgit is defined
         - artg_repos_dir is not defined or copy_dir is failed
 
+    - name: Generate DLRN Script
+      template:
+        src: run_dlrn.sh.j2
+        dest: "{{ build_repo_dir }}/run_dlrn.sh"
+        mode: 0744
+
     - name: Run DLRN
       shell: >
-        set +e;
+        set -o pipefail &&
-        source {{ build_repo_dir }}/dlrn-venv/bin/activate;
+        {{ build_repo_dir }}/run_dlrn.sh 2>&1 {{ timestamper_cmd }} >> {{ build_repo_dir }}/dlrn.log
-        while true; do
-            dlrn --config-file projects.ini --head-only --package-name {{ project_name_mapped.stdout }} --local --info-repo rdoinfo --dev;
-            if [ $? -eq 0 ]; then
-                # SUCCESS
-                break;
-            elif [ $? -eq 1 ]; then
-                # FAILED
-                exit 1;
-            elif [ $? -eq 2 ]; then
-                # RETRY
-                continue;
-            fi;
-            # Unexpected DLRN return code
-            exit $?;
-        done;
       args:
-        chdir: '{{ build_repo_dir }}/DLRN'
+        chdir: '{{ build_repo_dir }}'
       register: repo_built
       when: artg_build_one|bool

roles/build-test-packages/tasks/main.yml

@@ -83,6 +83,17 @@
     groups: mock
     append: true
 
+- name: Fetch DLRN from source
+  git:
+    repo: '{{ artg_dlrn_repo_url }}'
+    dest: '{{ build_repo_dir }}/DLRN'
+    version: 'master'
+  register: _git_clone
+  until: _git_clone is success
+  retries: 3
+  delay: 5
+  when: dlrn_pre_installed|bool
+
 - name: Ensure DLRN dir is present
   file:
     path: '{{ build_repo_dir }}/DLRN/'
@@ -110,10 +121,11 @@
     extra_args: --upgrade
 
 - name: Pip install rdopkg
+  # noqa 403
   pip:
     name: rdopkg
     virtualenv: "{{ build_repo_dir }}/dlrn-venv"
-    version: 0.47.3
+    state: latest
   when: not dlrn_pre_installed|bool
 
 - name: Pip install DLRN
@@ -127,8 +139,8 @@
 - name: Pip install pre-installed DLRN
   shell: >
     source {{ build_repo_dir }}/dlrn-venv/bin/activate;
-    pip install -r requirements.txt;
+    pip{{ ansible_python.version.major }} install -r requirements.txt;
-    python setup.py install;
+    python{{ ansible_python.version.major }} setup.py install;
   args:
     chdir: '{{ build_repo_dir }}/DLRN'
   when: dlrn_pre_installed|bool
@@ -142,13 +154,21 @@
 - name: Fetch local rdoinfo copy
   git:
     repo: '{{ artg_rdoinfo_repo_url }}'
-    dest: '{{ build_repo_dir }}/DLRN/rdoinfo'
+    dest: '{{ build_repo_dir }}/DLRN/{{ rdoinfo_repo_name }}'
     version: 'master'
   register: _git_clone
   until: _git_clone is success
   retries: 3
   delay: 5
 
+- name: Remove ssh from git url
+  when: osp_release is defined
+  shell: >
+    set -o pipefail;
+    sed -i -e "s|git+ssh|git|g" packages.yml
+  args:
+    chdir: '{{ build_repo_dir }}/DLRN/{{ rdoinfo_repo_name }}'
+
 - name: Use local {{ ansible_distribution }} mirror if available ( centos 7 and RHEL )
   shell: >
     set -o pipefail ;
@@ -159,8 +179,8 @@
     >> {{ build_repo_dir }}/DLRN/scripts/{{ initial_dlrn_config }}-local.cfg;
     find /etc/yum.repos.d -name "delorean*build-deps.repo" -exec sed 's/enabled=.*/enabled=1/g' {} \;
     >> {{ build_repo_dir }}/DLRN/scripts/{{ initial_dlrn_config }}-local.cfg;
-    {% if ansible_distribution|lower == "redhat" and ansible_distribution_major_version == "8" %}
+    {% if ansible_distribution in ["RedHat"] and osp_release is defined %}
-    find /etc/yum.repos.d -name "rh-cloud.repo" -exec sed 's/enabled=.*/enabled=1/g' {} \;
+    find /etc/yum.repos.d -name "osptrunk-deps.repo" -exec sed 's/enabled=.*/enabled=1/g' {} \;
     >> {{ build_repo_dir }}/DLRN/scripts/{{ initial_dlrn_config }}-local.cfg;
     {% endif %}
     echo '"""' >> {{ build_repo_dir }}/DLRN/scripts/{{ initial_dlrn_config }}-local.cfg;
@@ -170,19 +190,6 @@
   when:
     - dlrn_use_local_mirrors|bool
 
-# FIXME(jpena): once DLRN 0.8.0 is released, we can remove this
-- name: Check if DLRN/scripts/build_rpm.sh exists
-  stat:
-    path: '{{ build_repo_dir }}/DLRN/scripts/build_rpm.sh'
-  register: old_dlrn
-
-- name: Remove DLRN RPM install test
-  # noqa 303
-  shell: >
-    sed -i -e 's%--postinstall%%'
-    {{ build_repo_dir }}/DLRN/scripts/build_rpm.sh
-  when: old_dlrn.stat.exists
-
 - name: get env variables for changes and default change list
   set_fact:
     gerrit_host: "{{ lookup('env', 'GERRIT_HOST') }}"
@@ -203,17 +210,25 @@
     artg_change_list: "{{ artg_change_list|union(jenkins_change_list|default([])) }}"
     cacheable: true
 
-- name: Parse Zuul changes
+# Below is the list of cases needs to handle while constructing zuul change lists to process
-  zuul_deps:
+# 1. Gerrit change branch is not equal to project branch
-    host: "{{ lookup('env', 'ZUUL_URL')|default('https://review.opendev.org', true) }}"
+# 2. project name should be present in zuul processed change list
-    changes: "{{ lookup('env', 'ZUUL_CHANGES') }}"
+- name: construct zuul deps list
-    branch: "{{ zuul.override_checkout | default(lookup('env', 'ZUUL_BRANCH')) }}"
-  when: zuul_changes != ""  # noqa 602
-
-- name: Add Zuul changes to the change list
   set_fact:
-    artg_change_list: "{{ artg_change_list|union(zuul_change_list|default([])) }}"
+    artg_change_list: "{{ artg_change_list|default([]) + [{'host': item.change_url.rpartition('/')[0],
+                                                           'project': item.project.name,
+                                                           'branch': item.branch,
+                                                           'refspec': '/'.join(['refs', 'changes',
+                                                                                item.change[-2:],
+                                                                                item.change,
+                                                                                item.patchset]) }] }}"
     cacheable: true
+  with_items: "{{ zuul['items'] }}"
+  when:
+    - zuul is defined
+    - "'change_url' in item"
+    - item.project.name not in artg_change_list|default([]) | map(attribute='project') |list
+    - (item.branch == zuul.override_checkout | default(zuul.branch)) or (osp_release is defined and osp_release in item.branch)
 
 - name: Print out the change list
   debug:
@@ -235,31 +250,24 @@
   when:
     - '"-distgit" not in item.project'
     - item.project not in artg_skipped_projects
-    - release in item.branch or (zuul.project.name | default("") == "rdoinfo")
+    - release in item.branch or (zuul.project.name | default("") == rdoinfo_repo_name) or (osp_release is defined and osp_release in item.branch)
 
-- name: Run DLRN
+- name: Build package using DLRN
-  shell: >
-    set +e;
-    source {{ build_repo_dir }}/dlrn-venv/bin/activate;
-    while true; do
-        dlrn --config-file projects.ini --head-only --package-name {{ artg_rdo_packages|join(' --package-name ') }} --local --info-repo rdoinfo --dev --order;
-        if [ $? -eq 0 ]; then
-            # SUCCESS
-            break;
-        elif [ $? -eq 1 ]; then
-            # FAILED
-            exit 1;
-        elif [ $? -eq 2 ]; then
-            # RETRY
-            continue;
-        fi;
-        # Unexpected DLRN return code
-        exit $?;
-    done;
-  args:
-    chdir: '{{ build_repo_dir }}/DLRN'
-  register: repo_built_multi
   when: not artg_build_one|bool
+  block:
+    - name: Generate DLRN Script
+      template:
+        src: run_dlrn.sh.j2
+        dest: "{{ build_repo_dir }}/run_dlrn.sh"
+        mode: 0744
+
+    - name: Run DLRN
+      shell: >
+        set -o pipefail &&
+        {{ build_repo_dir }}/run_dlrn.sh 2>&1 {{ timestamper_cmd }} >> {{ build_repo_dir }}/dlrn.log
+      args:
+        chdir: '{{ build_repo_dir }}'
+      register: repo_built_multi
 
 - name: Clean up loop devices created by mock
   file:

roles/build-test-packages/templates/projects.ini.j2

@@ -9,7 +9,11 @@ smtpserver=
 reponame=delorean
 #templatedir=$VIRTUAL_ENV/dlrn/templates
 maxretries=3
+{% if osp_release is defined %}
+pkginfo_driver=dlrn.drivers.downstream.DownstreamInfoDriver
+{% else %}
 pkginfo_driver=dlrn.drivers.rdoinfo.RdoInfoDriver
+{% endif %}
 tags=
 #tags=mitaka
 rsyncdest=
@@ -20,6 +24,18 @@ database_connection=sqlite:///commits.sqlite
 fallback_to_master=1
 use_components={{ use_components }}
 
+{% if osp_release is defined %}
+[downstream_driver]
+info_files=osp.yml
+versions_url=https://trunk.rdoproject.org/centos8-master/current/versions.csv
+downstream_distro_branch={{ osp_release }}.0-rhel-8-trunk
+# downstream tag needs to be osp-17.0
+downstream_tag=osp-{{ osp_release.split('-')[1] }}.0
+downstream_distgit_key=osp-distgit
+use_upstream_spec=true
+downstream_spec_replace_list=^%global with_doc.+/%global with_doc 0,^%global rhosp.*/%global rhosp 1
+{% endif %}
+
 [gitrepo_driver]
 # Options to be specified if pkginfo_driver is set to
 # dlrn.drivers.gitrepo.GitRepoDriver

roles/build-test-packages/templates/run_dlrn.sh.j2

@@ -0,0 +1,27 @@
+set +e
+
+cd {{ build_repo_dir }}/DLRN
+source {{ build_repo_dir }}/dlrn-venv/bin/activate;
+{% if artg_build_one|bool %}
+export PKG={{ project_name_mapped.stdout }}
+{% else %}
+export PKG="{{ artg_rdo_packages|join(' --package-name ') }}"
+{% endif %}
+
+while true; do
+    dlrn --config-file projects.ini --head-only --package-name $PKG --local \
+        --info-repo {{ rdoinfo_repo_name }} \
+        {% if not artg_build_one|bool %}--order {% endif %}--dev;
+    if [ $? -eq 0 ]; then
+        # SUCCESS
+        break;
+    elif [ $? -eq 1 ]; then
+        # FAILED
+        exit 1;
+    elif [ $? -eq 2 ]; then
+        # RETRY
+        continue;
+    fi;
+    # Unexpected DLRN return code
+    exit $?;
+done;