Rework docker build

* put Dockerfile at root level and remove all hooks for autobuilds at
  hub.docker.org
* run the similar wokrload as `tox -e self` using rally docker image
* rewrite workload that is used by `tox -e self` at python
* push docker image at gate pipeline

Change-Id: Iddd4b3ad518dcf46bf9cb1d67e0b161ad1728166

.dockerignore

@@ -1,5 +1,5 @@
-rally-jobs
 tests
 contrib
 test-requirements.txt
 tox.ini
+.tox

.zuul.d/zuul.yaml

@@ -1,3 +1,29 @@
+- secret:
+    name: dockerhub_credentials
+    data:
+      username: !encrypted/pkcs1-oaep
+        - k7GCjmH8P2DqliVc5ihbr+yYwVeylzShrmoP3qIMb1jx4Mkgn8YDY54vf84HT5xEDAuJw
+          y/ndAtvVY/4WF0MsRZ/p4ZaAZx1FyV90ZgNXKNAgizMmvBIZoVEbFN5gimJn8DVnvcm5R
+          vIezNzsgWQoNiOOuxwB3j6Fq+40K6mBg2k1ceuvqX/DAKqXgNdX/V5SPxBYek4WkofxUs
+          gDFdHt3aMIty+Gd/fW4D/ufGAurW4CUbFmiAVyZagXzq8NKXCKUOGFeNvL/tftXhT2bd1
+          PczUz8Nhur1Z7nYeqc5K1TnN35OlNP0IT4KG8vst33nX4ycSkFOpbkdvflnchlysEvn/+
+          Z1NMFKLxQvi/k4rMa2jBv7O6xs9y1LOZMSHwG0Y2U8lV6m1FqrDyi9h22fcBGEtOyp0KV
+          V9DmSFRCNb7lhd7v7vaztzL8yqrGlezrJYELodUvYLy2AwZi+FnkUVFw7dB4D5m/fjMCf
+          ej/LRO4XiUpVpLUL0BOxLNXVHqqrbROwPkp2ekJL3WxMwhAGSxlAuMwvPU1KZzRhrdnG7
+          Ysm12YV5qgd6b6XZ8X7vELxAiqZPCjSN+66FPZW+Q6I5UVX+IxIgndgxgkoHJOgg9Wjwb
+          kKDgYM8EE5Ks4/Rv9jfNXxXvH/cQFqWIMHHbsZfmV6QcLfJxYXDRZ98r6KTzwA=
+      password: !encrypted/pkcs1-oaep
+        - JxlQJhxRBxgyqfHyUPVbS4pagJxkA6sdJyPf5p84iWFa1nWbIDTbpzhGYhYBqMt1YrXFf
+          NWGHzs7rJ/hp4MXSRHWVG4lEi5O6RFx/KEWiru+rGJwr7CyIfyyVORlRGlnABrc/LYyAe
+          LoSsQ5GM98Q1wERbhiZq7zsKSf0xIazFDmf34+bIx6Q9r4nlWdCQuQJ9N07UT4aRsl3WS
+          hOlUp8ekxBiQRtS9lRUPLUmJ8OzHTzXjJ6V0fM3MhDGA3YnH0Tj9r1D7cmwhsTy5ZT36k
+          bt2WCknxJ9tiJND5/NMvzo9T82apgoqeH8HCWDjE6PEdmJdYLiiW4aCk22D2M6a0+qvQI
+          fD8ZeEbrlJLvf58ldGgCJfl1Rpql0FspgnERBkCL4xSlyMQa8+S5R2S/SN6hAsbLRm/jZ
+          zUt3C4n8pJJvBDUBGEK+lqersyJtx5jEiz6xh0FVDVzFxBn/fUbNvQQH+zJAe4V1j1hHw
+          MBu4V2p20Qpjv2+wF2yzwT+tnrCkaeyuBfBLGRVdn4nCz630X1eaOwqh0hu1ZPDiwNBs4
+          hNv/EV7oaaTlk4VftYRwYocdK/pvj1Mi0QCea+hC01vuoqJE8EK05oOD1KThOTrbwXQX5
+          CDzBLc/Vyd4tFnAXPwEJDEKFNo25R7nmDBYTQIuiHSOSZHJis+r5UwvHfbuJPk=
+
 - job:
     name: rally-install-base
     parent: base
@@ -23,12 +49,25 @@
 
 - job:
     name: rally-docker-build
-    parent: base
+    parent: build-docker-image
     nodeset: ubuntu-bionic
-    pre-run: tests/ci/playbooks/rally-docker-build-pre.yaml
-    run: tests/ci/playbooks/rally-docker-build-run.yaml
+    run: tests/ci/playbooks/build-and-check-docker-image.yaml
+    post-run: tests/ci/playbooks/fetch-html-and-json-reports.yaml
     timeout: 1800
 
+
+- job:
+    name: rally-docker-build-and-push
+    parent: build-docker-image
+    nodeset: ubuntu-bionic
+    run: tests/ci/playbooks/build-check-and-push-docker-image.yaml
+    post-run: tests/ci/playbooks/fetch-html-and-json-reports.yaml
+    timeout: 1800
+    secrets:
+      name: docker_credentials
+      secret: dockerhub_credentials
+      pass-to-parent: false
+
 - job:
     name: rally-database-migration
     parent: base
@@ -71,4 +110,4 @@
         - rally-install-ubuntu-bionic
         - rally-install-centos-7
         - rally-install-centos-8
-        - rally-docker-build
+        - rally-docker-build-and-push

Dockerfile

@@ -9,8 +9,7 @@ RUN apt-get update && apt-get install --yes sudo python3-dev python3-pip vim git
     echo "rally ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/00-rally-user && \
     mkdir /rally && chown -R rally:rally /rally
 
-COPY ./src /rally/source
-COPY ./motd /etc/motd
+COPY ./ /rally/source
 WORKDIR /rally/source
 
 RUN pip3 install . --constraint upper-constraints.txt && \
@@ -19,13 +18,15 @@ RUN pip3 install . --constraint upper-constraints.txt && \
     mkdir -p /etc/rally && \
     echo "[database]" > /etc/rally/rally.conf && \
     echo "connection=sqlite:////home/rally/.rally/rally.db" >> /etc/rally/rally.conf
-RUN echo '[ ! -z "$TERM" -a -r /etc/motd ] && cat /etc/motd' >> /etc/bash.bashrc
 # Cleanup pip
 RUN rm -rf /root/.cache/
 
+COPY ./etc/motd_for_docker /etc/motd
+RUN echo '[ ! -z "$TERM" -a -r /etc/motd ] && cat /etc/motd' >> /etc/bash.bashrc
+
 USER rally
 ENV HOME /home/rally
-RUN  mkdir -p /home/rally/.rally && rally db recreate
+RUN mkdir -p /home/rally/.rally && rally db recreate
 
 # Docker volumes have specific behavior that allows this construction to work.
 # Data generated during the image creation is copied to volume only when it's

etc/docker/README_FIRST.rst

@@ -1,11 +0,0 @@
-ReadMe of <rally-repo>/etc/docker dir
-=====================================
-
-We are using automated docker image builds on `Docker Hub
-<https://hub.docker.com/>`_ which allows to reduce time of making new releases.
-
-Docker Hub has one specific feature - each time it builds new image, it
-updates the description of the image. The description it takes from README file
-of the same directory as Dockerfile is located. That is why Dockerfile is
-placed to the separate directory with 2 README files: one for Docker Hub,
-another one (this one) for explanation of situation.

etc/docker/hooks/post_checkout

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-# NOTE(andreykurilin): We have automated builds for new docker images at
-#   Docker Cloud. To publish a custom, docker specific README file, we need
-#   to apply a workaround: change the context directory for building images
-#   from the root repo dir '/' to '/etc/docker/' where we store our Dockerfile.
-#   In this case, Docker Cloud will use local README, i.e /etc/docker/README.md
-#   To make source code available from Dockerfile we need to apply
-#   post_checkout docker cloud hook (see [*] for more details). Our custom hook
-#   copies rally framework code under the directory which is available from
-#   docker build context, i.e from /etc/docker
-#
-#   [*] - https://docs.docker.com/docker-cloud/builds/advanced/#custom-build-phase-hooks
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-
-mkdir ${DIR}/../src/
-rsync -av --progress ${DIR}/../../../ ${DIR}/../src/ --exclude etc/docker

tests/ci/playbooks/build-and-check-docker-image.yaml

@@ -0,0 +1,4 @@
+- hosts: all
+  name: Execute the similar to `tox -e self` job
+  roles:
+    - build_docker_image

tests/ci/playbooks/build-check-and-push-docker-image.yaml

@@ -0,0 +1,5 @@
+- hosts: all
+  name: Execute the similar to `tox -e self` job
+  roles:
+    - build_docker_image
+    - push_docker_image

tests/ci/playbooks/rally-docker-build-pre.yaml

@@ -1,18 +0,0 @@
-- hosts: all
-  name: Execute post checkout hook as like hub.docker.com does
-  tasks:
-    - name: Install docker
-      shell:
-        cmd: |
-          sudo apt install --yes apt-transport-https ca-certificates curl software-properties-common
-          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-          sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable"
-          sudo apt update --yes
-          apt-cache policy docker-ce
-          sudo apt install --yes docker-ce
-          sudo systemctl status docker
-        executable: /bin/bash
-
-    - name: Execute post_checkout hook
-      shell:
-        cmd: "sudo /bin/bash {{ zuul.project.src_dir }}/etc/docker/hooks/post_checkout"

tests/ci/playbooks/rally-docker-build-run.yaml

@@ -1,7 +0,0 @@
-- hosts: all
-  name: a run script for rally-install-* jobs
-  tasks:
-    - name: Build docker image for Rally
-      shell:
-        cmd: "sudo docker build -t rally:dev ./"
-        chdir: '{{ zuul.project.src_dir }}/etc/docker'

tests/ci/playbooks/rally-install-run.yaml

@@ -19,8 +19,18 @@
         chdir: '{{ zuul.project.src_dir }}'
         cmd: "sudo pip3 install --constraint ./upper-constraints.txt ./"
 
+    - name: Create direcotry for html and json reports
+      shell:
+        executable: /bin/bash
+        chdir: '{{ zuul.project.src_dir }}'
+        cmd: mkdir .test_results
+
     - name: Execute the similar script as `tox -e self`
       shell:
-        chdir: '{{ zuul.project.src_dir }}'
         executable: /bin/bash
-        cmd: './tests/ci/rally_self_job.sh ./rally-jobs/self-rally.yaml'
+        chdir: '{{ zuul.project.src_dir }}'
+        cmd: >
+          python3 ./tests/ci/rally_self_job.py
+          --task ./rally-jobs/self-rally.yaml
+          --plugins-path ./rally-jobs/plugins
+          --results-dir ./.test_results

tests/ci/playbooks/roles/build_docker_image/defaults/main.yaml

@@ -0,0 +1,3 @@
+docker_repository: xrally/xrally
+docker_image_version: latest
+docker_image_tag: "{{ docker_repository }}:{{ docker_image_version }}"

tests/ci/playbooks/roles/build_docker_image/tasks/main.yaml

@@ -0,0 +1,24 @@
+- name: Build xrally/xrally docker image
+  shell:
+    chdir: '{{ zuul.project.src_dir }}'
+    cmd: 'docker build -t {{ docker_image_tag }} ./'
+
+- name: List available docker images
+  shell: docker image list
+
+- name: Prepare directories to mount to the docker container for persistency
+  shell:
+    chdir: '{{ zuul.project.src_dir }}'
+    cmd: |
+      mkdir -p .test_results
+      sudo chown 65500 .test_results
+
+- name: Execute the similar wokrloads as `tox -e self`
+  shell:
+    cmd: >
+      python3 {{ zuul.project.src_dir }}/tests/ci/rally_self_job.py
+      --task /rally/source/rally-jobs/self-rally.yaml
+      --plugins-path /rally/source/rally-jobs/plugins
+      --rally-cmd 'docker run -v '$(realpath {{ zuul.project.src_dir }})'/.test_results:/home/rally/.rally {{ docker_image_tag }}'
+      --results-dir /home/rally/.rally
+      --without-tmp-sqlite

tests/ci/playbooks/roles/push_docker_image/defaults/main.yaml

@@ -0,0 +1,4 @@
+docker_repository: xrally/xrally
+docker_image_version: latest
+docker_image_tag: "{{ docker_repository }}:{{ docker_image_version }}"
+docker_file: "{{ zuul.project.src_dir }}/DOCKER_README.md"

tests/ci/playbooks/roles/push_docker_image/tasks/main.yaml

@@ -0,0 +1,42 @@
+- name: Log in to dockerhub
+  command: "docker login -u '{{ docker_credentials.username }}' -p '{{ docker_credentials.password }}'"
+  no_log: true
+
+- name: Upload image to dockerhub
+  command: "docker push {{ docker_image_tag }}"
+  register: result
+  until: result.rc == 0
+  retries: 3
+  delay: 30
+
+- name: Read README file for docker
+  command: "cat {{ docker_file }}"
+  register: docker_readme
+
+- name: Get dockerhub JWT token
+  no_log: true
+  uri:
+    url: "https://hub.docker.com/v2/users/login/"
+    body_format: json
+    body:
+      username: '{{ docker_credentials.username }}'
+      password: '{{ docker_credentials.password }}'
+  register: jwt_token
+  delay: 5
+  retries: 3
+  until: jwt_token and jwt_token.status==200
+
+- name: Update README at Docker HUB
+  no_log: true
+  uri:
+    url: "https://hub.docker.com/v2/repositories/{{ docker_repository }}/"
+    method: "PATCH"
+    body_format: json
+    headers:
+      Authorization: "JWT {{ jwt_token.json.token }}"
+    body:
+      full_description: "{{ docker_readme.stdout }}"
+  register: repository_metadata
+  delay: 5
+  retries: 3
+  until: repository_metadata and repository_metadata.status==200

tests/ci/rally_self_job.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import argparse
+import subprocess
+import sys
+import uuid
+
+
+def create_cfg_for_using_sqlite():
+    """Create a config file for rally with path to temporary sqlite database"""
+    run_id = str(uuid.uuid4())[:8]
+
+    cfg_path = f"/tmp/self_rally_{run_id}.conf"
+
+    cfg = (f"[database]\n"
+           f"connection = sqlite:////tmp/self-rally-{run_id}.sqlite")
+    with open(cfg_path, "w") as f:
+        f.write(cfg)
+
+    return "--config-file", cfg_path
+
+
+def _print_with_wrapper(*messages):
+    print("#" * 80)
+    for m in messages:
+        print(f"# {m}")
+    print("#" * 80)
+
+
+class Rally(object):
+    def __init__(self, rally_cmd):
+        self._rally_cmd = rally_cmd
+
+    def __call__(self, cmd, debug=False, description=None):
+        if not isinstance(cmd, (tuple, list)):
+            cmd = cmd.split(" ")
+
+        final_cmd = self._rally_cmd[:]
+        if debug:
+            final_cmd.append("--debug")
+        final_cmd.extend(cmd)
+
+        messages = [f"Calling: {' '.join(final_cmd)}"]
+        if description:
+            messages.append(f"Description: {description}")
+        _print_with_wrapper(*messages)
+
+        p = subprocess.Popen(
+            final_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+        )
+        for line in iter(p.stdout.readline, b''):
+            print(line.rstrip().decode("utf-8"))
+        p.wait()
+        if p.returncode != 0:
+            _print_with_wrapper(
+                f"Command returned non-zero exit status {p.returncode}.")
+            sys.exit(1)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        "rally_self_job",
+        description="A script that ensures that Rally works after installation"
+    )
+    parser.add_argument(
+        "--results-dir", type=str, default=".test_results",
+        help="A path to directory to place rally results execution."
+    )
+    parser.add_argument(
+        "--task", type=str, required=True,
+        help="A simple task to launch."
+    )
+    parser.add_argument(
+        "--without-tmp-sqlite", action="store_false",
+        dest="prepare_sqlite",
+        help="Do not create a tmp sqlite db for Rally"
+    )
+    parser.add_argument(
+        "--rally-cmd", type=str, default="rally",
+        help="An override 'rally' entry point for all commands."
+    )
+    parser.add_argument(
+        "--plugins-path", type=str, required=True,
+        help="A path to external rally plugins that include "
+             "'FakePlugin.testplugin'"
+    )
+
+    args = parser.parse_args()
+
+    rally_cmd = args.rally_cmd.split(" ")
+    if "--config-file" in rally_cmd:
+        raise Exception(
+            "It is restricted to use '--config-file' at rally cmd.")
+
+    if args.prepare_sqlite:
+        rally_cmd.extend(create_cfg_for_using_sqlite())
+
+    rally_cmd.append(f"--plugin-paths={args.plugins_path}")
+
+    rally = Rally(rally_cmd)
+
+    rally("--version")
+    rally("plugin show --name FakePlugin.testplugin", debug=True,
+          description="Ensure plugins loading")
+
+    rally("db create", description="Initialize temporary sqlite database")
+
+    rally("env create --name=self",
+          description="Create empty environment")
+    rally(f"task start {args.task}")
+
+    results_dir = args.results_dir
+    rally(f"task report --html-static --out {results_dir}/self_report.html")
+    rally(f"task report --json --out {results_dir}/self_report.json")
+    rally(f"task sla-check")
+
+
+if __name__ == "__main__":
+    main()

tests/ci/rally_self_job.sh

@@ -1,53 +0,0 @@
-#!/bin/bash -ex
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-TASK_FILE=$1
-PLUGIN_PATHS=rally-jobs/plugins
-
-mkdir -p .test_results
-HTML_REPORT=.test_results/self_report.html
-JSON_REPORT=.test_results/self_report.json
-
-RND=$(head /dev/urandom | tr -dc a-z0-9 | head -c 5)
-TMP_RALLY_CONF="/tmp/self-rally-$RND.conf"
-TMP_RALLY_DB="/tmp/self-rally-$RND.sqlite"
-DBCONNSTRING="sqlite:///$TMP_RALLY_DB"
-RALLY="rally --config-file $TMP_RALLY_CONF"
-
-# Create temp cfg
-cp etc/rally/rally.conf.sample $TMP_RALLY_CONF
-sed -i.bak "s|#connection =.*|connection = \"$DBCONNSTRING\"|" $TMP_RALLY_CONF
-
-# ensure plugins loading
-$RALLY --debug --plugin-paths=$PLUGIN_PATHS plugin show --name FakePlugin.testplugin
-
-# Create db
-$RALLY db create
-
-# Create self deployment
-$RALLY -d env create --name=self
-
-# Run task
-set +e
-$RALLY -d --plugin-paths=$PLUGIN_PATHS task start $TASK_FILE
-if [ $? -eq 1 ]; then
-    exit 1
-fi
-set -e
-
-$RALLY task report --html-static --out $HTML_REPORT
-$RALLY task report --json --out $JSON_REPORT
-
-# Check sla (this may fail the job)
-$RALLY task sla-check

tests/unit/doc/test_docker_readme.py

@@ -23,7 +23,6 @@ from tests.unit import test
 
 
 ROOT_DIR = os.path.dirname(os.path.dirname(rally.__file__))
-DOCKER_DIR = os.path.join(ROOT_DIR, "etc", "docker")
 
 
 class DockerReadmeTestCase(test.TestCase):
@@ -51,7 +50,7 @@ class DockerReadmeTestCase(test.TestCase):
         return releases
 
     def test_mentioned_latest_version(self):
-        full_path = os.path.join(ROOT_DIR, "etc/docker/README.md")
+        full_path = os.path.join(ROOT_DIR, "DOCKER_README.md")
         with open(full_path) as f:
             readme = f.read()
 

tox.ini

@@ -96,10 +96,16 @@ commands = bindep
 
 [testenv:self-py38]
 basepython = python3.8
-commands = {toxinidir}/tests/ci/rally_self_job.sh {toxinidir}/rally-jobs/self-rally.yaml
+commands =
+  find . -type f -name "*.pyc" -delete
+  mkdir -p .test_results
+  python3 {toxinidir}/tests/ci/rally_self_job.py --task {toxinidir}/rally-jobs/self-rally.yaml --plugins-path {toxinidir}/rally-jobs/plugins
 
 [testenv:self]
-commands = {toxinidir}/tests/ci/rally_self_job.sh {toxinidir}/rally-jobs/self-rally.yaml
+commands = \
+  find . -type f -name "*.pyc" -delete
+  mkdir -p {toxinidir}/.test_results
+  python3 {toxinidir}/tests/ci/rally_self_job.py --task {toxinidir}/rally-jobs/self-rally.yaml --plugins-path {toxinidir}/rally-jobs/plugins
 
 [pytest]
 filterwarnings =