[ci] Temporary copy-paste docker jobs content from rally

openstack/rally project is not registered as a trusted place for Zuul. It means that we cannot transmit secret and reuse docker job from there. This is temporary workaround that should be removed asap to reduce code duplication. Change-Id: I19d8d40d72ed46d4bf576afab8e93cdbb9df9540
2020-04-22 15:09:11 +03:00 · 2020-04-22 15:09:11 +03:00 · 8189dd4d5d
parent f13406795a
commit 8189dd4d5d
8 changed files with 163 additions and 9 deletions
--- a/.zuul.d/docker-jobs.yaml
+++ b/.zuul.d/docker-jobs.yaml
@ -24,29 +24,38 @@
          pm95hJSPjVDW6LhujIKIWL21swKyeSrJGB9f3MlaYkiQn3vZ1Dy/7rE/fRN9a9Vur/KNU
          eh7c/tXVG3k0n2Y+7O6gMnLpT6lHM18jlbjyNfvEWLSfDVoRWG87L2BDeVTZtY=

+
 - job:
    name: rally-openstack-docker-build
-    parent: rally-docker-build
+    parent: build-docker-image
+    nodeset: ubuntu-bionic
+    run: tests/ci/playbooks/docker-build-and-check.yaml
+    post-run: tests/ci/playbooks/fetch-html-and-json-reports.yaml
+    timeout: 1800
    required-projects:
      - name: openstack/rally
      - name: openstack/rally-openstack
    vars:
-      docker_repository: "xrally/xrally-openstack"
      docker_image_version: latest
+      docker_repository: "xrally/xrally-openstack"
      rally_plugin_name: "Dummy.openstack"
      rally_package_name: "rally-openstack"

 - job:
    name: rally-openstack-docker-build-and-push
-    parent: rally-docker-build-and-push
+    parent: build-docker-image
+    nodeset: ubuntu-bionic
+    run: tests/ci/playbooks/docker-build-check-and-push.yaml
+    post-run: tests/ci/playbooks/fetch-html-and-json-reports.yaml
+    timeout: 1800
    required-projects:
-        - name: openstack/rally
-        - name: openstack/rally-openstack
+      - name: openstack/rally
+      - name: openstack/rally-openstack
+    secrets:
+      name: docker_credentials
+      secret: rally-openstack-dockerhub-credentials
+      pass-to-parent: false
    vars:
      docker_repository: "xrally/xrally-openstack"
      rally_plugin_name: "Dummy.openstack"
      rally_package_name: "rally-openstack"
-    secrets:
-        name: dockerhub_credentials
-        secret: rally-openstack-dockerhub-credentials
-        pass-to-parent: true
--- a/tests/ci/playbooks/docker-build-and-check.yaml
+++ b/tests/ci/playbooks/docker-build-and-check.yaml
@ -0,0 +1,4 @@
+- hosts: all
+  name: Build docker image for Rally and check it (calling the similar workload as `tox -e self`)
+  roles:
+    - docker-build-image
--- a/tests/ci/playbooks/docker-build-check-and-push.yaml
+++ b/tests/ci/playbooks/docker-build-check-and-push.yaml
@ -0,0 +1,5 @@
+- hosts: all
+  name: Build and push docker image for Rally
+  roles:
+    - docker-build-image
+    - docker-push-image
--- a/tests/ci/playbooks/fetch-html-and-json-reports.yaml
+++ b/tests/ci/playbooks/fetch-html-and-json-reports.yaml
@ -0,0 +1,42 @@
+- hosts: all
+  vars:
+    results_dir: "{{ zuul.project.src_dir }}/.test_results/"
+    html_report: "{{ tox_env | default('self') }}_report.html"
+    json_report: "{{ tox_env | default('self') }}_report.json"
+  tasks:
+    - shell: "ls {{ results_dir }}"
+      register: results_dir_stat
+      ignore_errors: True
+
+    - name: Save results
+      become: yes
+      when: results_dir_stat.rc == 0
+      synchronize:
+        src: "{{ results_dir }}"
+        dest: '{{ zuul.executor.log_root }}/'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+        - --include=/**
+        - --include=*/
+        - --exclude=*
+        - --prune-empty-dirs
+
+    - name: Return artifact to Zuul
+      when: html_report in results_dir_stat.stdout
+      zuul_return:
+        data:
+          zuul:
+            artifacts:
+              - name: "HTML report"
+                url: "{{ html_report }}"
+
+    - name: Return artifact to Zuul
+      when: json_report in results_dir_stat.stdout
+      zuul_return:
+        data:
+          zuul:
+            artifacts:
+              - name: "JSON report"
+                url: "{{ json_report }}"
--- a/tests/ci/playbooks/roles/docker-build-image/defaults/main.yaml
+++ b/tests/ci/playbooks/roles/docker-build-image/defaults/main.yaml
@ -0,0 +1,6 @@
+docker_repository: xrally/xrally
+docker_image_version: "{{ zuul.tag }}"
+docker_image_tag: "{{ docker_repository }}:{{ docker_image_version }}"
+rally_project_path: "{{ zuul.projects['opendev.org/openstack/rally'].src_dir }}"
+rally_plugin_name: "Dummy.dummy"
+rally_package_name: ""
--- a/tests/ci/playbooks/roles/docker-build-image/tasks/main.yaml
+++ b/tests/ci/playbooks/roles/docker-build-image/tasks/main.yaml
@ -0,0 +1,42 @@
+- name: "Build {{ docker_image_tag }} docker image"
+  shell:
+    chdir: '{{ zuul.project.src_dir }}'
+    cmd: 'docker build -t {{ docker_image_tag }} ./'
+
+- name: Get image size
+  shell: >
+    set -e
+    SIZE=$(docker image inspect {{ docker_image_tag }} {% raw %}--format='{{.Size}}'{% endraw %} )
+    SIZE=$(expr $SIZE / 1000 / 1000)
+    echo "Size of {{ docker_image_tag }}: "$SIZE" Mbyte"
+
+- name: Prepare directories to mount to the docker container for persistency
+  shell:
+    chdir: '{{ zuul.project.src_dir }}'
+    cmd: |
+      mkdir -p .test_results
+      sudo chown 65500 .test_results
+
+- name: List Rally version
+  shell: docker run {{ docker_image_tag }} --version
+  register: rally_version_info
+
+- name: Check plugin package is discoverable
+  fail:
+    msg: "{{ rally_package_name }} is not discoverable."
+  when:
+   - '"{{ rally_package_name }}" != ""'
+   - '"{{ rally_package_name }}" not in rally_version_info.stdout'
+
+- name: "Check availability of {{ rally_plugin_name }} plugin"
+  shell: docker run {{ docker_image_tag }} plugin show {{ rally_plugin_name }}
+
+- name: Execute the similar wokrloads as `tox -e self`
+  shell:
+    cmd: >
+      python3 {{ rally_project_path }}/tests/ci/rally_self_job.py
+      --task /rally/source/rally-jobs/self-rally.yaml
+      --plugins-path /rally/source/rally-jobs/plugins
+      --rally-cmd 'docker run -v '$(realpath {{ zuul.project.src_dir }})'/.test_results:/home/rally/.rally {{ docker_image_tag }}'
+      --results-dir /home/rally/.rally
+      --without-tmp-sqlite
--- a/tests/ci/playbooks/roles/docker-push-image/defaults/main.yaml
+++ b/tests/ci/playbooks/roles/docker-push-image/defaults/main.yaml
@ -0,0 +1,4 @@
+docker_repository: xrally/xrally
+docker_image_version: latest
+docker_image_tag: "{{ docker_repository }}:{{ docker_image_version }}"
+docker_file: "{{ zuul.project.src_dir }}/DOCKER_README.md"
--- a/tests/ci/playbooks/roles/docker-push-image/tasks/main.yaml
+++ b/tests/ci/playbooks/roles/docker-push-image/tasks/main.yaml
@ -0,0 +1,42 @@
+- name: Log in to dockerhub
+  command: "docker login -u '{{ docker_credentials.username }}' -p '{{ docker_credentials.password }}'"
+  no_log: true
+
+- name: Upload image to dockerhub
+  command: "docker push {{ docker_image_tag }}"
+  register: result
+  until: result.rc == 0
+  retries: 3
+  delay: 30
+
+- name: Read README file for docker
+  command: "cat {{ docker_file }}"
+  register: docker_readme
+
+- name: Get dockerhub JWT token
+  no_log: true
+  uri:
+    url: "https://hub.docker.com/v2/users/login/"
+    body_format: json
+    body:
+      username: '{{ docker_credentials.username }}'
+      password: '{{ docker_credentials.password }}'
+  register: jwt_token
+  delay: 5
+  retries: 3
+  until: jwt_token and jwt_token.status==200
+
+- name: Update README at Docker HUB
+  no_log: true
+  uri:
+    url: "https://hub.docker.com/v2/repositories/{{ docker_repository }}/"
+    method: "PATCH"
+    body_format: json
+    headers:
+      Authorization: "JWT {{ jwt_token.json.token }}"
+    body:
+      full_description: "{{ docker_readme.stdout }}"
+  register: repository_metadata
+  delay: 5
+  retries: 3
+  until: repository_metadata and repository_metadata.status==200