Add Ansible Playbook to build specfiles via Zuul

This initial playbook and roles builds both CentOS and openSUSE
specfiles. The build occurs on OBS and is currently done in a
sandbox area in my home project [0]. The job currently just
starts the OBS job, it does not currently check the build
results, this will come later.

This has been tested with fault so far. The zuul template will
be added to each repo as specfiles are updated.

Added support to automagically build _service files.

[0] https://build.opensuse.org/project/subprojects/home:saulwold

zuul.d/projects needs to use the stx-zuul-jobs-linters to set
ANSIBLE_ROLES_PATH correctly

Change-Id: I6490030894b5887f33a55b1e66e374bcd50960d5
Signed-off-by: Saul Wold <sgw@linux.intel.com>

playbooks/buildproject.yaml

@@ -0,0 +1,234 @@
+- hosts: all
+  vars:
+    project_ver: 1.0
+    src_project: "Cloud:StarlingX:2.0"
+    target_project: "home:saulwold:sandbox-{{ target_distro }}"
+    test_project: "{{ target_project }}-{{ project_ver }}"
+    stx_project_desc: ""
+    stx_url: ""
+    osc_user: "saulwold"
+    osc_register: osc_result
+    obs_projects: []
+    spec_dirs: []
+    sorted_dirs: []
+    osc_key_key: "@ub@FB7OZxs391wt"
+  tasks:
+    - name: Print OS Family
+      debug:
+       msg: "OS distribution: {{ ansible_distribution }}"
+
+    - name: Setup local distro flag CentOS
+      set_fact:
+        target_distro: centos
+      when: ansible_distribution == "CentOS"
+
+    - name: Setup local distro flag openSUSE
+      set_fact:
+        target_distro: opensuse
+      when: ansible_distribution == "openSUSE Leap"
+
+    - name: Setup local directories when not using zuul
+      set_fact:
+        top_dir: /test
+        src_dir: /test
+      when: zuul is not defined
+
+    - name: Setup local directories from zuul data
+      set_fact:
+        top_dir: "{{ zuul.executor.src_root }}"
+        src_dir: "{{ zuul.project.src_dir }}"
+      when: zuul is defined
+
+    - name: Print OS Family
+      debug:
+       msg: "OS distribution: {{ ansible_distribution }} {{ target_distro }}"
+
+    - name: Get OSC Command
+      import_role:
+        name: osc
+        tasks_from: install_osc
+
+    - name: Create temporary directory
+      tempfile:
+        state: directory
+        prefix: stx.
+      register: stx_pkg_tmpdir
+
+    - name: Setup oscrc file
+      import_role:
+        name: osc
+        tasks_from: create_oscrc
+
+    - name: OSC Version
+      import_role:
+        name: osc
+      vars:
+        osc_options: "--version"
+      register: osc_version
+
+    - name: Clean OSC Project
+      import_role:
+        name: osc
+      vars:
+        osc_options: rdelete -r -m 'rpm-packing CI Cleanup' {{ test_project }}
+      ignore_errors: yes
+
+    - name: Create Project
+      import_role:
+        name: osc
+        tasks_from: create_meta
+
+    - name: Freeze Project
+      import_role:
+        name: osc
+        tasks_from: freeze
+
+    - name: Init Project
+      import_role:
+        name: osc
+      vars:
+        osc_options: init {{ test_project }}
+
+    - name: Add prjconf
+      import_role:
+        name: osc
+        tasks_from: create_prjconf
+      when: target_distro == "centos"
+
+    - name: Find local directories with spec files
+      find:
+        paths: "{{ src_dir }}"
+        hidden: false
+        file_type: directory
+        patterns: "{{ target_distro }}"
+        recurse: yes
+      register: specfiles
+
+    - name: Generate Repo name
+      set_fact:
+        repo_name: "{{ specfiles.files.0.path.split('/')[-3] }}"
+
+    - name: Generate Repo Dir
+      set_fact:
+        repo_dir: "{{ src_dir}}/{{ repo_name }}"
+
+    - debug:
+        msg: "{{ repo_name }}: {{ repo_dir }} "
+
+    - meta: end_play
+
+    - name: List spec files
+      debug:
+        msg: "Specfile List: {{ specfiles }}"
+
+    - name: Create Project list
+      set_fact:
+        spec_dirs: "{{ spec_dirs }} + [ '{{ item.path.split('/')[-2] }}' ]"
+      with_items: "{{ specfiles.files }}"
+
+    - name: Sort directories
+      set_fact:
+        sorted_dirs: "{{ spec_dirs | sort | unique }}"
+
+    - name: "list Sorted Directories"
+      debug:
+        msg: "Sorted Directories: {{ item }}"
+      with_items: "{{ sorted_dirs }}"
+
+    - name: OSC mkpac
+      import_role:
+        name: osc
+        tasks_from: mkpac
+
+    - name: OSC detachbranch
+      import_role:
+        name: osc
+        tasks_from: detachbranch
+
+    - name: Create Service file
+      include_role:
+        name: osc
+        tasks_from: create_service
+      vars:
+        package_dir_orig: "{{ package_name }}/{{ lookup('ini', 'SRC_DIR type=properties file=/fault/' + package_name + '/centos/build_srpm.data') | regex_replace('\"', '') | regex_replace('\\/\\.', '') }}"
+      with_items: "{{ sorted_dirs }}"
+      loop_control:
+        loop_var: package_name
+
+    - name: OSC commit
+      import_role:
+        name: osc
+        tasks_from: commit
+
+#        pushd ./out
+#        sleep 5
+#        # Check if there is no change, then pretent success
+#        if [ $($osc_timed ls ${{OBS_TEST_PROJECT}} | wc -l) -lt 1 ]; then
+#            echo "Apparently there is no change... claiming SUCCESS"
+#            exit 0
+#        fi
+#
+#        while true; do
+#            unset pending
+#            unset failed
+#            unset kickscheduler
+#            unset succeeded
+#            res=`osc results --csv -r standard`
+#            if [ $? -ne 0 ]; then
+#                sleep 5
+#                continue
+#            fi
+#            echo "... $(date): "
+#            for r in $res; do
+#                # some failures?
+#                if [[ $r =~ broken$ ]]; then
+#                    echo "ignoring OBS broken state"
+#                    echo $r
+#                    continue
+#                fi
+#                # some failures?
+#                if [[ $r =~ (failed$|unresolvable$) ]]; then
+#                    echo $r
+#                    failed=1
+#                fi
+#                # still pending builds?
+#                if [[ $r =~ (blocked$|scheduled$|dispatching$|building$|signing$) ]]; then
+#                    pending=1
+#                fi
+#                # scheduler stuck?
+#                if [[ $r =~ (finished$|unknown$|outdated$) ]]; then
+#                    kickscheduler=1
+#                fi
+#                # found something?
+#                if [[ $r =~ succeeded$ ]]; then
+#                    succeeded=1
+#                fi
+#            done
+#
+#            if [ -n "$kickscheduler" ]; then
+#                echo "# $(date)" | osc meta prjconf -F - ${{OBS_TEST_PROJECT}}
+#                sleep $((RANDOM%60+30))
+#                echo "kicking scheduler"
+#                continue
+#            fi
+#
+#            if [ -n "$pending" ]; then
+#                sleep $((RANDOM%50+30))
+#                echo ""
+#                continue
+#            fi
+#
+#            if [ -n "$failed" ]; then
+#                echo "Build failed :-("
+#                exit 1
+#            elif [ -n "$succeeded" ];
+#                break
+#            fi
+#        done
+#
+
+        #    - name: Clean-up tmp directory
+        #      file:
+        #        path: "{{ stx_pkg_tmpdir.path }}"
+        #        state: absent
+        #      when: stx_pkg_tmpdir.path is defined

roles/osc/README.rst

@@ -0,0 +1,2 @@
+Ansible tasks and templates for working with the openSUSE Build System (OBS)
+via the command line tool called osc.

roles/osc/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+# defaults file for osc

roles/osc/handlers/main.yml

@@ -0,0 +1,2 @@
+---
+# handlers file for osc

roles/osc/meta/main.yml

@@ -0,0 +1,60 @@
+galaxy_info:
+  author: your name
+  description: your description
+  company: your company (optional)
+
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+
+  # Some suggested licenses:
+  # - BSD (default)
+  # - MIT
+  # - GPLv2
+  # - GPLv3
+  # - Apache
+  # - CC-BY
+  license: license (GPLv2, CC-BY, etc)
+
+  min_ansible_version: 2.4
+
+  # If this a Container Enabled role, provide the minimum Ansible Container version.
+  # min_ansible_container_version:
+
+  # Optionally specify the branch Galaxy will use when accessing the GitHub
+  # repo for this role. During role install, if no tags are available,
+  # Galaxy will use this branch. During import Galaxy will access files on
+  # this branch. If Travis integration is configured, only notifications for this
+  # branch will be accepted. Otherwise, in all cases, the repo's default branch
+  # (usually master) will be used.
+  #github_branch:
+
+  #
+  # Provide a list of supported platforms, and for each platform a list of versions.
+  # If you don't wish to enumerate all versions for a particular platform, use 'all'.
+  # To view available platforms and versions (or releases), visit:
+  # https://galaxy.ansible.com/api/v1/platforms/
+  #
+  # platforms:
+  # - name: Fedora
+  #   versions:
+  #   - all
+  #   - 25
+  # - name: SomePlatform
+  #   versions:
+  #   - all
+  #   - 1.0
+  #   - 7
+  #   - 99.99
+
+  galaxy_tags: []
+    # List tags for your role here, one per line. A tag is a keyword that describes
+    # and categorizes the role. Users find roles by searching for tags. Be sure to
+    # remove the '[]' above, if you add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
+    #       Maximum 20 tags per role.
+
+dependencies: []
+  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
+  # if you add dependencies to this list.

roles/osc/tasks/commit.yaml

@@ -0,0 +1,29 @@
+---
+# Create the _meta template file
+- name: OSC Service
+  command: "osc service run"
+  args:
+    chdir: "{{ stx_pkg_tmpdir.path }}/{{ item }}"
+  with_items: "{{ sorted_dirs }}"
+- name: OSC Addremove
+  command: "osc addremove"
+  args:
+    chdir: "{{ stx_pkg_tmpdir.path }}/{{ item }}"
+  with_items: "{{ sorted_dirs }}"
+- name: OSC Commit
+  command: "osc commit -m CI-Build"
+  args:
+    chdir: "{{ stx_pkg_tmpdir.path }}"
+  ignore_errors: yes
+
+  # def osc_commit_all(workdir, packagename):
+  #    olddir = os.getcwd()
+  #  try:
+  #      os.chdir(os.path.join(workdir, packagename))
+  #      sh.osc('addremove')
+  #      for o in sh.osc('service', 'localrun', 'source_validator'):
+  #          if o.startswith('###ASK'):
+  #              sh.osc('rm', o.strip().split()[1])
+  #      sh.osc('commit', '--noservice', '-n')
+  #  finally:
+  #      os.chdir(olddir)

roles/osc/tasks/create_meta.yaml

@@ -0,0 +1,20 @@
+---
+# Create the _meta template file
+- name: Create temporary _meta file
+  tempfile:
+    state: file
+    prefix: stx_meta.
+  register: stx_meta_file
+- name: Print Debug output
+  debug:
+    msg:
+      - "{{ stx_meta_file.path }}"
+- name: Template _prj
+  template:
+    src: _prj-{{ target_distro }}.j2
+    dest: "{{ stx_meta_file.path }}"
+- name: Create OSC Repo
+  import_role:
+    name: osc
+  vars:
+    osc_options: api -T {{ stx_meta_file.path }} /source/{{ test_project }}/_meta

roles/osc/tasks/create_oscrc.yaml

@@ -0,0 +1,27 @@
+---
+# Create the _meta template file
+- name: oscrc - Zuul Block
+  block:
+  - name: Create osc config directory
+    file:
+      path: "/home/zuul/.config/osc"
+      state: directory
+      mode: 0755
+  - name: Template .oscrc
+    template:
+      src: oscrc.j2
+      dest: "/home/zuul/.config/osc/oscrc"
+  when: zuul is defined
+
+- name: oscrc - Non-Zuul Block
+  block:
+  - name: Create osc config directory
+    file:
+      path: "/root/.config/osc"
+      state: directory
+      mode: 0755
+  - name: Template .oscrc
+    template:
+      src: oscrc.j2
+      dest: "/root/.config/osc/oscrc"
+  when: zuul is not defined

roles/osc/tasks/create_prjconf.yaml

@@ -0,0 +1,18 @@
+---
+# Create the _meta template file
+- name: Create temporary _prjconf file
+  tempfile:
+    state: file
+    prefix: stx_prjconf.
+  register: stx_prjconf_file
+
+- name: Template _prjconf
+  template:
+    src: _prjconf-{{ target_distro }}.j2
+    dest: "{{ stx_prjconf_file.path }}"
+
+- name: Create prjconf via meta command
+  import_role:
+    name: osc
+  vars:
+    osc_options: meta prjconf -F {{ stx_prjconf_file.path }}

roles/osc/tasks/create_service.yaml

@@ -0,0 +1,10 @@
+---
+- name: Fix up package_dir variable
+  set_fact:
+    package_dir: "{{ package_dir_orig | regex_replace('\\/\\.', '') }}"
+
+- name: Template _service
+  template:
+    src: _service.j2
+    dest: "{{ stx_pkg_tmpdir.path }}/{{ package_name }}/_service"
+

roles/osc/tasks/detachbranch.yaml

@@ -0,0 +1,51 @@
+---
+# Create the _meta template file
+#- name: OSC detachbranch
+#  command: "osc detachbranch {{ test_project }} {{ item }}"
+#  args:
+#    chdir: "{{ stx_pkg_tmpdir.path }}"
+#  with_items: "{{ sorted_dirs }}"
+#- name: Remove directory
+#  file:
+#    path: "{{ stx_pkg_tmpdir.path }}/{{ item }}"
+#    state: absent
+#  with_items: "{{ sorted_dirs }}"
+#- name: OSC Checkout
+#  command: "osc checkout {{ item }}"
+#  args:
+#    chdir: "{{ stx_pkg_tmpdir.path }}"
+#  with_items: "{{ sorted_dirs }}"
+- name: Restore meta-data
+  shell: cp {{ src_dir }}/{{ item }}/{{ target_distro }}/*[.cpst]* {{ stx_pkg_tmpdir.path }}/{{ item }}
+  with_items: "{{ sorted_dirs }}"
+#- name: OSC AddRemove
+#  command: "osc addremove"
+#  args:
+#    chdir: "{{ stx_pkg_tmpdir.path }}/{{ item }}"
+#  with_items: "{{ sorted_dirs }}"
+#- name: OSC Commit
+#  command: "osc commit"
+#  args:
+#    chdir: "{{ stx_pkg_tmpdir.path }}/{{ item }}"
+#  with_items: "{{ sorted_dirs }}"
+  #
+  #def osc_detachbranch(workdir, project, pkgname):
+  #    olddir = os.getcwd()
+  #  try:
+  #      os.chdir(os.path.join(workdir))
+  #      sh.osc('detachbranch', project, pkgname)
+  #      os.mkdir(pkgname + '.b')
+  #      for f in glob.glob(os.path.join(pkgname, '*')):
+  #          os.rename(f, os.path.join(pkgname + '.b', os.path.basename(f)))
+  #      sh.rm('-rf', pkgname)
+  #      sh.osc('co', pkgname)
+  #      for f in glob.glob(os.path.join(pkgname + '.b', '*')):
+  #          dst = os.path.basename(f)
+  #          try:
+  #              os.unlink(os.path.join(pkgname, dst))
+  #          except OSError:
+  #              pass
+  #          os.rename(f, os.path.join(pkgname, dst))
+  #      os.rmdir(pkgname + '.b')
+  #  finally:
+  #      os.chdir(olddir)

roles/osc/tasks/freeze.yaml

@@ -0,0 +1,8 @@
+---
+# Run OSC Command
+- name: Create OSC Freeze Link
+  import_role:
+    name: osc
+  vars:
+    osc_options: api -X POST /source/{{ test_project }}?cmd=freezelink
+  until: osc_result.stdout.find("status code=ok") != -1

roles/osc/tasks/install_osc.yaml

@@ -0,0 +1,64 @@
+- name: Install Centos OSC Command
+  block:
+  - name: Add OSC Repo
+    yum_repository:
+      name: opensuse_tools
+      description: openSUSE Tools for OSC and OBS
+      baseurl: http://download.opensuse.org/repositories/openSUSE:/Tools/CentOS_7
+      gpgcheck: no
+
+  - name: Install OSC
+    yum:
+      name:
+       - build
+       - obs-service-obs_scm
+       - obs-service-recompress
+       - obs-service-tar
+       - obs-service-tar_scm
+       - osc
+      state: present
+  when: target_distro == 'centos'
+  become: yes
+
+- name: Install openSUSE OSC
+  block:
+  - name: Install OSC
+    zypper:
+      name:
+       - build
+       - obs-service-obs_scm
+       - obs-service-recompress
+       - obs-service-tar
+       - obs-service-tar_scm
+       - osc
+      state: present
+      disable_recommends: no
+  when: target_distro == 'opensuse'
+  become: yes
+
+- name: Install Ubuntu OSC Command
+  block:
+  - name: Add Ubuntu Tools repository key
+    apt_key:
+      url: "http://download.opensuse.org/repositories/openSUSE:/Tools/xUbuntu_18.10/Release.key"
+
+  - name: Add Ubuntu Tools repo
+    apt_repository:
+      repo: "deb http://download.opensuse.org/repositories/openSUSE:/Tools/xUbuntu_18.10 bionic main"
+      state: present
+      update_cache: yes
+    ignore_errors: yes
+
+  - name: Update apt cache
+    apt:
+      update_cache: yes
+    ignore_errors: yes
+
+  - name: Install required software
+    package:
+      name:
+       - software-properties-common
+       - osc
+      state: present
+  when: target_distro == "ubuntu"
+  become: yes

roles/osc/tasks/main.yaml

@@ -0,0 +1,8 @@
+---
+# Run OSC Command
+- name: Run OSC with options
+  command: osc {{ osc_options }}
+  args:
+    chdir: "{{ stx_pkg_tmpdir.path }}"
+  register: osc_result
+  #  register: "{{ osc_register }}"

roles/osc/tasks/mkpac.yaml

@@ -0,0 +1,10 @@
+---
+# Create the _meta template file
+- name: OSC Makepackages
+  command: "osc mkpac {{ item }}"
+  args:
+    chdir: "{{ stx_pkg_tmpdir.path }}"
+  with_items: "{{ sorted_dirs }}"
+  #- name: Copy meta-data
+  #shell: ls -la {{ top_dir }}/save/{{ item }}/*.[cstp]* {{ stx_pkg_tmpdir.path }}/{{ item }} >> /tmp/list; cp  {{ top_dir }}/save/{{ item }}/*.[cstp]* {{ stx_pkg_tmpdir.path }}/{{ item }}
+  #with_items: "{{ sorted_dirs }}"

roles/osc/templates/_prj-centos.j2

@@ -0,0 +1,23 @@
+<project name="{{ test_project }}">
+  <title>Autogenerated CI project</title>
+  <description>
+{{ stx_project_desc }}
+  </description>
+  <url>
+{{ stx_url }}
+  </url>
+  <person userid="{{ osc_user }}" role="maintainer"/>
+  <publish>
+    <disable/>
+  </publish>
+  <repository name="CentOS_7">
+    <path project="home:saulwold:mytest" repository="CentOS_7"/>
+    <path project="home:gassyfeve:lico-dep" repository="CentOS_7"/>
+    <path project="home:lenovo-lico:lico-dep:5.1:el7" repository="el7"/>
+    <path project="CentOS:CentOS-7" repository="openstack-stein"/>
+    <path project="Fedora:EPEL:7" repository="CentOS"/>
+    <path project="CentOS:CentOS-7" repository="standard"/>
+    <arch>x86_64</arch>
+  </repository>
+</project>
+

roles/osc/templates/_prj-opensuse.j2

@@ -0,0 +1,17 @@
+<project name="{{ test_project }}">
+  <title>Autogenerated CI project</title>
+  <description>
+{{ stx_project_desc }}
+  </description>
+  <url>
+{{ stx_url }}
+  </url>
+  <person userid="{{ osc_user }}" role="maintainer"/>
+  <publish>
+    <disable/>
+  </publish>
+  <repository name="openSUSE_Leap_15.1" rebuild="direct" block="local" linkedbuild="localdep">
+    <path project="{{ src_project }}" repository="openSUSE_Leap_15.1"/>
+    <arch>x86_64</arch>
+  </repository>
+</project>

roles/osc/templates/_prjconf-centos.j2

@@ -0,0 +1,4 @@
+Preinstall: python-backport-macros
+
+Prefer: python2-ipaddress python2-pyparsing python2-pyyaml
+

roles/osc/templates/_service.j2

@@ -0,0 +1,16 @@
+<services>
+  <service name="tar_scm">
+    <param name="scm">git</param>
+    <param name="url">https://opendev.org/starlingx/fault</param>
+    <param name="version">1.0</param>
+    <param name="subdir">{{  package_dir }}</param>
+    <param name="filename">{{  package_name }}</param>
+    <param name="changesgenerate">disable</param>
+  </service>
+  <service name="recompress">
+    <param name="compression">gz</param>
+    <param name="file">*.tar</param>
+  </service>
+  <service name="set_version" mode="disabled"/>
+</services>
+

roles/osc/templates/oscrc.j2

@@ -0,0 +1,12 @@
+[general]
+apiurl = https://api.opensuse.org
+
+[api]
+email = sgw@linux.intel.com
+
+[https://api.opensuse.org]
+user = saulwold
+pass = {{ osc_key_key }}
+email = sgw@linux.intel.com
+trusted_prj=openSUSE:Leap:15.0 SUSE:SLE-12-SP4:GA Cloud:OpenStack:Stein devel:kubic openSUSE:Leap:15.1
+

roles/osc/tests/inventory

@@ -0,0 +1,2 @@
+localhost
+

roles/osc/tests/test.yml

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  remote_user: root
+  roles:
+    - osc

roles/osc/vars/main.yml

@@ -0,0 +1,2 @@
+---
+# vars file for osc

zuul.d/jobs.yaml

@@ -23,3 +23,14 @@
     vars:
       tox_envlist: rpm-packaging-lint
     files: .*\.spec
+
+- job:
+    name: stx-obs-build-opensuse
+    nodeset: opensuse-150
+    run: playbooks/buildproject.yaml
+
+- job:
+    name: stx-obs-build-centos
+    nodeset: centos-7
+    run: playbooks/buildproject.yaml
+

zuul.d/project-templates.yaml

@@ -58,3 +58,14 @@
       jobs:
         - stx-check-specfile:
             voting: false
+
+- project-template:
+    name: stx-build-rpms
+    description: |
+      Buiild RPMs via Ansible Playbook and OBS
+    check:
+      jobs:
+        - stx-obs-build-centos:
+            voting: false
+        - stx-obs-build-opensuse:
+            voting: false

zuul.d/project.yaml

@@ -3,9 +3,7 @@
       - build-openstack-docs-pti
     check:
       jobs:
-        - openstack-tox-linters
-        - openstack-tox-pep8
+        - stx-zuul-jobs-linters
     gate:
       jobs:
-        - openstack-tox-linters
-        - openstack-tox-pep8
+        - stx-zuul-jobs-linters