diff --git a/roles/build-test-packages/defaults/main.yml b/roles/build-test-packages/defaults/main.yml index 493b54c35..693005302 100644 --- a/roles/build-test-packages/defaults/main.yml +++ b/roles/build-test-packages/defaults/main.yml @@ -1,9 +1,9 @@ --- build_repo_dir: "{{ ansible_user_dir }}" - +rdoinfo_repo_name: rdoinfo artg_dlrn_repo_url: "https://github.com/openstack-packages/DLRN.git" -artg_rdoinfo_repo_url: "https://github.com/redhat-openstack/rdoinfo" +artg_rdoinfo_repo_url: https://github.com/redhat-openstack/rdoinfo artg_compressed_gating_repo: "/home/stack/gating_repo.tar.gz" artg_build_one: true artg_rdo_packages: [] diff --git a/roles/build-test-packages/library/zuul_deps.py b/roles/build-test-packages/library/zuul_deps.py deleted file mode 100755 index 7a8cf25f0..000000000 --- a/roles/build-test-packages/library/zuul_deps.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/python -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# see http://docs.ansible.com/developing_modules.html#common-module-boilerplate -from ansible.module_utils.basic import * - -DOCUMENTATION = ''' ---- -module: zuul_deps -version_added: "2.0" -short_description: Transforms the dependent changes variable from Zuul format into a dictionary -description: - - Transforms the dependent changes variable from Zuul format into a dictionary -options: - host: - description: - - The content of the ZUUL_URL variable - required: True - changes: - description: - - The content of the ZUUL_CHANGES variable - required: True - branch: - description: - - Which branch to transform dependent changes on. - default: None -''' - -EXAMPLES = ''' -- zuul_deps: - host: https://review.opendev.org - changes: "openstack/tripleo-heat-templates:master:refs/changes/88/296488/1^openstack/instack-undercloud:master:refs/changes/84/315184/5" -''' - - -def process(host, changes, branch): - """Process the changes from Zuul format""" - output = [] - - for item in reversed(changes.split("^")): - params = item.split(":") - if branch and branch != params[1]: - continue - if params[0] in [i['project'] for i in output]: - continue - # add URL prefix if only the bare hostname is given - if host.find('://') == -1: - host = ''.join(['https://', host]) - output.append({"host": host, - "project": params[0], - "branch": params[1], - "refspec": params[2]}) - return {'changed': True, - 'ansible_facts': {'zuul_change_list': output}} - - -def main(): - module = AnsibleModule( - argument_spec=dict( - host=dict(required=True, type='str'), - changes=dict(required=True, type='str'), - branch=dict(default=None, type='str') - ) - ) - result = process(module.params['host'], - module.params['changes'], - module.params['branch']) - module.exit_json(**result) - - -if __name__ == "__main__": - main() diff --git a/roles/build-test-packages/tasks/dlrn-build.yml b/roles/build-test-packages/tasks/dlrn-build.yml index 2c1893c6a..b77a9ba91 100644 --- a/roles/build-test-packages/tasks/dlrn-build.yml +++ b/roles/build-test-packages/tasks/dlrn-build.yml @@ -17,6 +17,13 @@ artg_distro: "{{ artg_release }}-rdo" when: artg_release not in ['master', 'mitaka'] +- name: Make sure /usr/share/rhn/RHN-ORG-TRUSTED-SSL-CERT exists (RHEL) + shell: + cmd: | + touch /usr/share/rhn/RHN-ORG-TRUSTED-SSL-CERT + become: true + when: ansible_distribution in ['RedHat'] + - name: Use local baseurl when using local mirrors set_fact: dlrn_baseurl: 'file:///tmp/fake-dlrn-repos' @@ -33,6 +40,19 @@ - {regexp: 'source=.*', line: 'source={{ artg_change.branch }}'} loop_control: loop_var: project_item + when: osp_release is not defined + +- name: Override projects.ini settings for downstream + lineinfile: + dest: '{{ build_repo_dir }}/DLRN/projects.ini' + regexp: '{{ project_item.regexp }}' + line: '{{ project_item.line }}' + with_items: + - {regexp: 'baseurl=.*', line: 'baseurl={{ dlrn_baseurl }}'} + - {regexp: 'source=.*', line: 'source={{ artg_change.branch }}'} + loop_control: + loop_var: project_item + when: osp_release is defined - block: - name: Map project name to DLRN project name @@ -42,7 +62,10 @@ cmd: | source '{{ build_repo_dir }}/dlrn-venv/bin/activate' set -xeo pipefail - rdopkg findpkg -s '{{ artg_change.project | basename }}' -l rdoinfo | \ + # {{ rdoinfo_repo_name }}/{{ rdoinfo_repo_name.split('info')[0] }}-full.yml will + # return rdo-full.yml and for downstream is osp-full.yml. + rdopkg findpkg -s '{{ artg_change.project | basename }}' \ + -i {{ rdoinfo_repo_name }}/{{ rdoinfo_repo_name.split('info')[0] }}-full.yml | \ awk '/^name/{print $2}; {print "findpkg: " $0 > "/dev/stderr"}' changed_when: false failed_when: project_name_mapped.rc != 0 or not project_name_mapped.stdout @@ -92,26 +115,17 @@ - artg_change.distgit is defined - artg_repos_dir is not defined or copy_dir is failed + - name: Generate DLRN Script + template: + src: run_dlrn.sh.j2 + dest: "{{ build_repo_dir }}/run_dlrn.sh" + mode: 0744 + - name: Run DLRN shell: > - set +e; - source {{ build_repo_dir }}/dlrn-venv/bin/activate; - while true; do - dlrn --config-file projects.ini --head-only --package-name {{ project_name_mapped.stdout }} --local --info-repo rdoinfo --dev; - if [ $? -eq 0 ]; then - # SUCCESS - break; - elif [ $? -eq 1 ]; then - # FAILED - exit 1; - elif [ $? -eq 2 ]; then - # RETRY - continue; - fi; - # Unexpected DLRN return code - exit $?; - done; + set -o pipefail && + {{ build_repo_dir }}/run_dlrn.sh 2>&1 {{ timestamper_cmd }} >> {{ build_repo_dir }}/dlrn.log args: - chdir: '{{ build_repo_dir }}/DLRN' + chdir: '{{ build_repo_dir }}' register: repo_built when: artg_build_one|bool diff --git a/roles/build-test-packages/tasks/main.yml b/roles/build-test-packages/tasks/main.yml index f2c81f431..fa1d470ce 100644 --- a/roles/build-test-packages/tasks/main.yml +++ b/roles/build-test-packages/tasks/main.yml @@ -83,6 +83,17 @@ groups: mock append: true +- name: Fetch DLRN from source + git: + repo: '{{ artg_dlrn_repo_url }}' + dest: '{{ build_repo_dir }}/DLRN' + version: 'master' + register: _git_clone + until: _git_clone is success + retries: 3 + delay: 5 + when: dlrn_pre_installed|bool + - name: Ensure DLRN dir is present file: path: '{{ build_repo_dir }}/DLRN/' @@ -110,10 +121,11 @@ extra_args: --upgrade - name: Pip install rdopkg + # noqa 403 pip: name: rdopkg virtualenv: "{{ build_repo_dir }}/dlrn-venv" - version: 0.47.3 + state: latest when: not dlrn_pre_installed|bool - name: Pip install DLRN @@ -127,8 +139,8 @@ - name: Pip install pre-installed DLRN shell: > source {{ build_repo_dir }}/dlrn-venv/bin/activate; - pip install -r requirements.txt; - python setup.py install; + pip{{ ansible_python.version.major }} install -r requirements.txt; + python{{ ansible_python.version.major }} setup.py install; args: chdir: '{{ build_repo_dir }}/DLRN' when: dlrn_pre_installed|bool @@ -142,13 +154,21 @@ - name: Fetch local rdoinfo copy git: repo: '{{ artg_rdoinfo_repo_url }}' - dest: '{{ build_repo_dir }}/DLRN/rdoinfo' + dest: '{{ build_repo_dir }}/DLRN/{{ rdoinfo_repo_name }}' version: 'master' register: _git_clone until: _git_clone is success retries: 3 delay: 5 +- name: Remove ssh from git url + when: osp_release is defined + shell: > + set -o pipefail; + sed -i -e "s|git+ssh|git|g" packages.yml + args: + chdir: '{{ build_repo_dir }}/DLRN/{{ rdoinfo_repo_name }}' + - name: Use local {{ ansible_distribution }} mirror if available ( centos 7 and RHEL ) shell: > set -o pipefail ; @@ -159,8 +179,8 @@ >> {{ build_repo_dir }}/DLRN/scripts/{{ initial_dlrn_config }}-local.cfg; find /etc/yum.repos.d -name "delorean*build-deps.repo" -exec sed 's/enabled=.*/enabled=1/g' {} \; >> {{ build_repo_dir }}/DLRN/scripts/{{ initial_dlrn_config }}-local.cfg; - {% if ansible_distribution|lower == "redhat" and ansible_distribution_major_version == "8" %} - find /etc/yum.repos.d -name "rh-cloud.repo" -exec sed 's/enabled=.*/enabled=1/g' {} \; + {% if ansible_distribution in ["RedHat"] and osp_release is defined %} + find /etc/yum.repos.d -name "osptrunk-deps.repo" -exec sed 's/enabled=.*/enabled=1/g' {} \; >> {{ build_repo_dir }}/DLRN/scripts/{{ initial_dlrn_config }}-local.cfg; {% endif %} echo '"""' >> {{ build_repo_dir }}/DLRN/scripts/{{ initial_dlrn_config }}-local.cfg; @@ -170,19 +190,6 @@ when: - dlrn_use_local_mirrors|bool -# FIXME(jpena): once DLRN 0.8.0 is released, we can remove this -- name: Check if DLRN/scripts/build_rpm.sh exists - stat: - path: '{{ build_repo_dir }}/DLRN/scripts/build_rpm.sh' - register: old_dlrn - -- name: Remove DLRN RPM install test - # noqa 303 - shell: > - sed -i -e 's%--postinstall%%' - {{ build_repo_dir }}/DLRN/scripts/build_rpm.sh - when: old_dlrn.stat.exists - - name: get env variables for changes and default change list set_fact: gerrit_host: "{{ lookup('env', 'GERRIT_HOST') }}" @@ -203,17 +210,25 @@ artg_change_list: "{{ artg_change_list|union(jenkins_change_list|default([])) }}" cacheable: true -- name: Parse Zuul changes - zuul_deps: - host: "{{ lookup('env', 'ZUUL_URL')|default('https://review.opendev.org', true) }}" - changes: "{{ lookup('env', 'ZUUL_CHANGES') }}" - branch: "{{ zuul.override_checkout | default(lookup('env', 'ZUUL_BRANCH')) }}" - when: zuul_changes != "" # noqa 602 - -- name: Add Zuul changes to the change list +# Below is the list of cases needs to handle while constructing zuul change lists to process +# 1. Gerrit change branch is not equal to project branch +# 2. project name should be present in zuul processed change list +- name: construct zuul deps list set_fact: - artg_change_list: "{{ artg_change_list|union(zuul_change_list|default([])) }}" + artg_change_list: "{{ artg_change_list|default([]) + [{'host': item.change_url.rpartition('/')[0], + 'project': item.project.name, + 'branch': item.branch, + 'refspec': '/'.join(['refs', 'changes', + item.change[-2:], + item.change, + item.patchset]) }] }}" cacheable: true + with_items: "{{ zuul['items'] }}" + when: + - zuul is defined + - "'change_url' in item" + - item.project.name not in artg_change_list|default([]) | map(attribute='project') |list + - (item.branch == zuul.override_checkout | default(zuul.branch)) or (osp_release is defined and osp_release in item.branch) - name: Print out the change list debug: @@ -235,31 +250,24 @@ when: - '"-distgit" not in item.project' - item.project not in artg_skipped_projects - - release in item.branch or (zuul.project.name | default("") == "rdoinfo") + - release in item.branch or (zuul.project.name | default("") == rdoinfo_repo_name) or (osp_release is defined and osp_release in item.branch) -- name: Run DLRN - shell: > - set +e; - source {{ build_repo_dir }}/dlrn-venv/bin/activate; - while true; do - dlrn --config-file projects.ini --head-only --package-name {{ artg_rdo_packages|join(' --package-name ') }} --local --info-repo rdoinfo --dev --order; - if [ $? -eq 0 ]; then - # SUCCESS - break; - elif [ $? -eq 1 ]; then - # FAILED - exit 1; - elif [ $? -eq 2 ]; then - # RETRY - continue; - fi; - # Unexpected DLRN return code - exit $?; - done; - args: - chdir: '{{ build_repo_dir }}/DLRN' - register: repo_built_multi +- name: Build package using DLRN when: not artg_build_one|bool + block: + - name: Generate DLRN Script + template: + src: run_dlrn.sh.j2 + dest: "{{ build_repo_dir }}/run_dlrn.sh" + mode: 0744 + + - name: Run DLRN + shell: > + set -o pipefail && + {{ build_repo_dir }}/run_dlrn.sh 2>&1 {{ timestamper_cmd }} >> {{ build_repo_dir }}/dlrn.log + args: + chdir: '{{ build_repo_dir }}' + register: repo_built_multi - name: Clean up loop devices created by mock file: diff --git a/roles/build-test-packages/templates/projects.ini.j2 b/roles/build-test-packages/templates/projects.ini.j2 index 6fba4a078..41fccbda5 100644 --- a/roles/build-test-packages/templates/projects.ini.j2 +++ b/roles/build-test-packages/templates/projects.ini.j2 @@ -9,7 +9,11 @@ smtpserver= reponame=delorean #templatedir=$VIRTUAL_ENV/dlrn/templates maxretries=3 +{% if osp_release is defined %} +pkginfo_driver=dlrn.drivers.downstream.DownstreamInfoDriver +{% else %} pkginfo_driver=dlrn.drivers.rdoinfo.RdoInfoDriver +{% endif %} tags= #tags=mitaka rsyncdest= @@ -20,6 +24,18 @@ database_connection=sqlite:///commits.sqlite fallback_to_master=1 use_components={{ use_components }} +{% if osp_release is defined %} +[downstream_driver] +info_files=osp.yml +versions_url=https://trunk.rdoproject.org/centos8-master/current/versions.csv +downstream_distro_branch={{ osp_release }}.0-rhel-8-trunk +# downstream tag needs to be osp-17.0 +downstream_tag=osp-{{ osp_release.split('-')[1] }}.0 +downstream_distgit_key=osp-distgit +use_upstream_spec=true +downstream_spec_replace_list=^%global with_doc.+/%global with_doc 0,^%global rhosp.*/%global rhosp 1 +{% endif %} + [gitrepo_driver] # Options to be specified if pkginfo_driver is set to # dlrn.drivers.gitrepo.GitRepoDriver diff --git a/roles/build-test-packages/templates/run_dlrn.sh.j2 b/roles/build-test-packages/templates/run_dlrn.sh.j2 new file mode 100644 index 000000000..2ebeee8e9 --- /dev/null +++ b/roles/build-test-packages/templates/run_dlrn.sh.j2 @@ -0,0 +1,27 @@ +set +e + +cd {{ build_repo_dir }}/DLRN +source {{ build_repo_dir }}/dlrn-venv/bin/activate; +{% if artg_build_one|bool %} +export PKG={{ project_name_mapped.stdout }} +{% else %} +export PKG="{{ artg_rdo_packages|join(' --package-name ') }}" +{% endif %} + +while true; do + dlrn --config-file projects.ini --head-only --package-name $PKG --local \ + --info-repo {{ rdoinfo_repo_name }} \ + {% if not artg_build_one|bool %}--order {% endif %}--dev; + if [ $? -eq 0 ]; then + # SUCCESS + break; + elif [ $? -eq 1 ]; then + # FAILED + exit 1; + elif [ $? -eq 2 ]; then + # RETRY + continue; + fi; + # Unexpected DLRN return code + exit $?; +done;