openstack/tripleo-quickstart-extras
Code Issues Proposed changes

Refactor buildtestpackages for downstream purpose

Browse Source 
It includes:
* parameterize rdoinfo_repo_name
* Use zuul dict to parse zuul changes and remove zuul_deps
  ansible plugin as it is no longer needed.
* Added dlrn config template for downstream and should be
  enabled only when osp_release is defined.
* Use latest rdopkg
* It also fixes the DLRN installing from source.
* Moves dlrn command to shell script
* Remove deprecated parts of DLRN-0.8.0
* find out zuul dict only when there is a change_url otherwise
  ignore it.

https://tree.taiga.io/project/tripleo-ci-board/task/1789

Change-Id: I48bcddbc7693375396a99dc0ee79d32f6e4a4863
Signed-off-by: Chandan Kumar (raukadah) <chkumar@redhat.com>
This commit is contained in:
Chandan Kumar (raukadah) 2020-05-12 19:28:28 +05:30
parent 08081fafba
commit c3a2d81104
6 changed files with 137 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
roles/build-test-packages/defaults/main.yml
View File

@ -1,9 +1,9 @@
---
build_repo_dir: "{{ ansible_user_dir }}"
rdoinfo_repo_name: rdoinfo
artg_dlrn_repo_url: "https://github.com/openstack-packages/DLRN.git"
artg_rdoinfo_repo_url: "https://github.com/redhat-openstack/rdoinfo"
artg_rdoinfo_repo_url: https://github.com/redhat-openstack/rdoinfo
artg_compressed_gating_repo: "/home/stack/gating_repo.tar.gz"
artg_build_one: true
artg_rdo_packages: []

83
roles/build-test-packages/library/zuul_deps.py
View File

@ -1,83 +0,0 @@
#!/usr/bin/python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# see http://docs.ansible.com/developing_modules.html#common-module-boilerplate
from ansible.module_utils.basic import *
DOCUMENTATION = '''
---
module: zuul_deps
version_added: "2.0"
short_description: Transforms the dependent changes variable from Zuul format into a dictionary
description:
- Transforms the dependent changes variable from Zuul format into a dictionary
options:
host:
description:
- The content of the ZUUL_URL variable
required: True
changes:
description:
- The content of the ZUUL_CHANGES variable
required: True
branch:
description:
- Which branch to transform dependent changes on.
default: None
'''
EXAMPLES = '''
- zuul_deps:
host: https://review.opendev.org
changes: "openstack/tripleo-heat-templates:master:refs/changes/88/296488/1^openstack/instack-undercloud:master:refs/changes/84/315184/5"
'''
def process(host, changes, branch):
"""Process the changes from Zuul format"""
output = []
for item in reversed(changes.split("^")):
params = item.split(":")
if branch and branch != params[1]:
continue
if params[0] in [i['project'] for i in output]:
continue
# add URL prefix if only the bare hostname is given
if host.find('://') == -1:
host = ''.join(['https://', host])
output.append({"host": host,
"project": params[0],
"branch": params[1],
"refspec": params[2]})
return {'changed': True,
'ansible_facts': {'zuul_change_list': output}}
def main():
module = AnsibleModule(
argument_spec=dict(
host=dict(required=True, type='str'),
changes=dict(required=True, type='str'),
branch=dict(default=None, type='str')
)
)
result = process(module.params['host'],
module.params['changes'],
module.params['branch'])
module.exit_json(**result)
if __name__ == "__main__":
main()

52
roles/build-test-packages/tasks/dlrn-build.yml
View File

@ -17,6 +17,13 @@
artg_distro: "{{ artg_release }}-rdo"
when: artg_release not in ['master', 'mitaka']
- name: Make sure /usr/share/rhn/RHN-ORG-TRUSTED-SSL-CERT exists (RHEL)
shell:
cmd: |
touch /usr/share/rhn/RHN-ORG-TRUSTED-SSL-CERT
become: true
when: ansible_distribution in ['RedHat']
- name: Use local baseurl when using local mirrors
set_fact:
dlrn_baseurl: 'file:///tmp/fake-dlrn-repos'
@ -33,6 +40,19 @@
- {regexp: 'source=.*', line: 'source={{ artg_change.branch }}'}
loop_control:
loop_var: project_item
when: osp_release is not defined
- name: Override projects.ini settings for downstream
lineinfile:
dest: '{{ build_repo_dir }}/DLRN/projects.ini'
regexp: '{{ project_item.regexp }}'
line: '{{ project_item.line }}'
with_items:
- {regexp: 'baseurl=.*', line: 'baseurl={{ dlrn_baseurl }}'}
- {regexp: 'source=.*', line: 'source={{ artg_change.branch }}'}
loop_control:
loop_var: project_item
when: osp_release is defined
- block:
- name: Map project name to DLRN project name
@ -42,7 +62,10 @@
cmd: |
source '{{ build_repo_dir }}/dlrn-venv/bin/activate'
set -xeo pipefail
rdopkg findpkg -s '{{ artg_change.project | basename }}' -l rdoinfo | \
# {{ rdoinfo_repo_name }}/{{ rdoinfo_repo_name.split('info')[0] }}-full.yml will
# return rdo-full.yml and for downstream is osp-full.yml.
rdopkg findpkg -s '{{ artg_change.project | basename }}' \
-i {{ rdoinfo_repo_name }}/{{ rdoinfo_repo_name.split('info')[0] }}-full.yml | \
awk '/^name/{print $2}; {print "findpkg: " $0 > "/dev/stderr"}'
changed_when: false
failed_when: project_name_mapped.rc != 0 or not project_name_mapped.stdout
@ -92,26 +115,17 @@
- artg_change.distgit is defined
- artg_repos_dir is not defined or copy_dir is failed
- name: Generate DLRN Script
template:
src: run_dlrn.sh.j2
dest: "{{ build_repo_dir }}/run_dlrn.sh"
mode: 0744
- name: Run DLRN
shell: >
set +e;
source {{ build_repo_dir }}/dlrn-venv/bin/activate;
while true; do
dlrn --config-file projects.ini --head-only --package-name {{ project_name_mapped.stdout }} --local --info-repo rdoinfo --dev;
if [ $? -eq 0 ]; then
# SUCCESS
break;
elif [ $? -eq 1 ]; then
# FAILED
exit 1;
elif [ $? -eq 2 ]; then
# RETRY
continue;
fi;
# Unexpected DLRN return code
exit $?;
done;
set -o pipefail &&
{{ build_repo_dir }}/run_dlrn.sh 2>&1 {{ timestamper_cmd }} >> {{ build_repo_dir }}/dlrn.log
args:
chdir: '{{ build_repo_dir }}/DLRN'
chdir: '{{ build_repo_dir }}'
register: repo_built
when: artg_build_one|bool

110
roles/build-test-packages/tasks/main.yml
View File

@ -83,6 +83,17 @@
groups: mock
append: true
- name: Fetch DLRN from source
git:
repo: '{{ artg_dlrn_repo_url }}'
dest: '{{ build_repo_dir }}/DLRN'
version: 'master'
register: _git_clone
until: _git_clone is success
retries: 3
delay: 5
when: dlrn_pre_installed|bool
- name: Ensure DLRN dir is present
file:
path: '{{ build_repo_dir }}/DLRN/'
@ -110,10 +121,11 @@
extra_args: --upgrade
- name: Pip install rdopkg
# noqa 403
pip:
name: rdopkg
virtualenv: "{{ build_repo_dir }}/dlrn-venv"
version: 0.47.3
state: latest
when: not dlrn_pre_installed|bool
- name: Pip install DLRN
@ -127,8 +139,8 @@
- name: Pip install pre-installed DLRN
shell: >
source {{ build_repo_dir }}/dlrn-venv/bin/activate;
pip install -r requirements.txt;
python setup.py install;
pip{{ ansible_python.version.major }} install -r requirements.txt;
python{{ ansible_python.version.major }} setup.py install;
args:
chdir: '{{ build_repo_dir }}/DLRN'
when: dlrn_pre_installed|bool
@ -142,13 +154,21 @@
- name: Fetch local rdoinfo copy
git:
repo: '{{ artg_rdoinfo_repo_url }}'
dest: '{{ build_repo_dir }}/DLRN/rdoinfo'
dest: '{{ build_repo_dir }}/DLRN/{{ rdoinfo_repo_name }}'
version: 'master'
register: _git_clone
until: _git_clone is success
retries: 3
delay: 5
- name: Remove ssh from git url
when: osp_release is defined
shell: >
set -o pipefail;
sed -i -e "s|git+ssh|git|g" packages.yml
args:
chdir: '{{ build_repo_dir }}/DLRN/{{ rdoinfo_repo_name }}'
- name: Use local {{ ansible_distribution }} mirror if available ( centos 7 and RHEL )
shell: >
set -o pipefail ;
@ -159,8 +179,8 @@
>> {{ build_repo_dir }}/DLRN/scripts/{{ initial_dlrn_config }}-local.cfg;
find /etc/yum.repos.d -name "delorean*build-deps.repo" -exec sed 's/enabled=.*/enabled=1/g' {} \;
>> {{ build_repo_dir }}/DLRN/scripts/{{ initial_dlrn_config }}-local.cfg;
{% if ansible_distribution|lower == "redhat" and ansible_distribution_major_version == "8" %}
find /etc/yum.repos.d -name "rh-cloud.repo" -exec sed 's/enabled=.*/enabled=1/g' {} \;
{% if ansible_distribution in ["RedHat"] and osp_release is defined %}
find /etc/yum.repos.d -name "osptrunk-deps.repo" -exec sed 's/enabled=.*/enabled=1/g' {} \;
>> {{ build_repo_dir }}/DLRN/scripts/{{ initial_dlrn_config }}-local.cfg;
{% endif %}
echo '"""' >> {{ build_repo_dir }}/DLRN/scripts/{{ initial_dlrn_config }}-local.cfg;
@ -170,19 +190,6 @@
when:
- dlrn_use_local_mirrors|bool
# FIXME(jpena): once DLRN 0.8.0 is released, we can remove this
- name: Check if DLRN/scripts/build_rpm.sh exists
stat:
path: '{{ build_repo_dir }}/DLRN/scripts/build_rpm.sh'
register: old_dlrn
- name: Remove DLRN RPM install test
# noqa 303
shell: >
sed -i -e 's%--postinstall%%'
{{ build_repo_dir }}/DLRN/scripts/build_rpm.sh
when: old_dlrn.stat.exists
- name: get env variables for changes and default change list
set_fact:
gerrit_host: "{{ lookup('env', 'GERRIT_HOST') }}"
@ -203,17 +210,25 @@
artg_change_list: "{{ artg_change_list|union(jenkins_change_list|default([])) }}"
cacheable: true
- name: Parse Zuul changes
zuul_deps:
host: "{{ lookup('env', 'ZUUL_URL')|default('https://review.opendev.org', true) }}"
changes: "{{ lookup('env', 'ZUUL_CHANGES') }}"
branch: "{{ zuul.override_checkout | default(lookup('env', 'ZUUL_BRANCH')) }}"
when: zuul_changes != "" # noqa 602
- name: Add Zuul changes to the change list
# Below is the list of cases needs to handle while constructing zuul change lists to process
# 1. Gerrit change branch is not equal to project branch
# 2. project name should be present in zuul processed change list
- name: construct zuul deps list
set_fact:
artg_change_list: "{{ artg_change_list|union(zuul_change_list|default([])) }}"
artg_change_list: "{{ artg_change_list|default([]) + [{'host': item.change_url.rpartition('/')[0],
'project': item.project.name,
'branch': item.branch,
'refspec': '/'.join(['refs', 'changes',
item.change[-2:],
item.change,
item.patchset]) }] }}"
cacheable: true
with_items: "{{ zuul['items'] }}"
when:
- zuul is defined
- "'change_url' in item"
- item.project.name not in artg_change_list|default([]) | map(attribute='project') |list
- (item.branch == zuul.override_checkout | default(zuul.branch)) or (osp_release is defined and osp_release in item.branch)
- name: Print out the change list
debug:
@ -235,31 +250,24 @@
when:
- '"-distgit" not in item.project'
- item.project not in artg_skipped_projects
- release in item.branch or (zuul.project.name | default("") == "rdoinfo")
- release in item.branch or (zuul.project.name | default("") == rdoinfo_repo_name) or (osp_release is defined and osp_release in item.branch)
- name: Run DLRN
shell: >
set +e;
source {{ build_repo_dir }}/dlrn-venv/bin/activate;
while true; do
dlrn --config-file projects.ini --head-only --package-name {{ artg_rdo_packages|join(' --package-name ') }} --local --info-repo rdoinfo --dev --order;
if [ $? -eq 0 ]; then
# SUCCESS
break;
elif [ $? -eq 1 ]; then
# FAILED
exit 1;
elif [ $? -eq 2 ]; then
# RETRY
continue;
fi;
# Unexpected DLRN return code
exit $?;
done;
args:
chdir: '{{ build_repo_dir }}/DLRN'
register: repo_built_multi
- name: Build package using DLRN
when: not artg_build_one|bool
block:
- name: Generate DLRN Script
template:
src: run_dlrn.sh.j2
dest: "{{ build_repo_dir }}/run_dlrn.sh"
mode: 0744
- name: Run DLRN
shell: >
set -o pipefail &&
{{ build_repo_dir }}/run_dlrn.sh 2>&1 {{ timestamper_cmd }} >> {{ build_repo_dir }}/dlrn.log
args:
chdir: '{{ build_repo_dir }}'
register: repo_built_multi
- name: Clean up loop devices created by mock
file:

16
roles/build-test-packages/templates/projects.ini.j2
View File

@ -9,7 +9,11 @@ smtpserver=
reponame=delorean
#templatedir=$VIRTUAL_ENV/dlrn/templates
maxretries=3
{% if osp_release is defined %}
pkginfo_driver=dlrn.drivers.downstream.DownstreamInfoDriver
{% else %}
pkginfo_driver=dlrn.drivers.rdoinfo.RdoInfoDriver
{% endif %}
tags=
#tags=mitaka
rsyncdest=
@ -20,6 +24,18 @@ database_connection=sqlite:///commits.sqlite
fallback_to_master=1
use_components={{ use_components }}
{% if osp_release is defined %}
[downstream_driver]
info_files=osp.yml
versions_url=https://trunk.rdoproject.org/centos8-master/current/versions.csv
downstream_distro_branch={{ osp_release }}.0-rhel-8-trunk
# downstream tag needs to be osp-17.0
downstream_tag=osp-{{ osp_release.split('-')[1] }}.0
downstream_distgit_key=osp-distgit
use_upstream_spec=true
downstream_spec_replace_list=^%global with_doc.+/%global with_doc 0,^%global rhosp.*/%global rhosp 1
{% endif %}
[gitrepo_driver]
# Options to be specified if pkginfo_driver is set to
# dlrn.drivers.gitrepo.GitRepoDriver

27
roles/build-test-packages/templates/run_dlrn.sh.j2 Normal file
View File

@ -0,0 +1,27 @@
set +e
cd {{ build_repo_dir }}/DLRN
source {{ build_repo_dir }}/dlrn-venv/bin/activate;
{% if artg_build_one|bool %}
export PKG={{ project_name_mapped.stdout }}
{% else %}
export PKG="{{ artg_rdo_packages|join(' --package-name ') }}"
{% endif %}
while true; do
dlrn --config-file projects.ini --head-only --package-name $PKG --local \
--info-repo {{ rdoinfo_repo_name }} \
{% if not artg_build_one|bool %}--order {% endif %}--dev;
if [ $? -eq 0 ]; then
# SUCCESS
break;
elif [ $? -eq 1 ]; then
# FAILED
exit 1;
elif [ $? -eq 2 ]; then
# RETRY
continue;
fi;
# Unexpected DLRN return code
exit $?;
done;