Add zuul jobs

This patch adds the following Zuul CI jobs:

- Python 2.7 unit tests
- Python 3.6 unit tests
- Functional tests with LVM
- Functional tests with Ceph
- Publis documentation
- Release notes

Change-Id: I2f3f34c1db4716b323c48908f51500898c2c6242

.gitignore

@@ -1,6 +1,9 @@
 # Byte-compiled / optimized / DLL files
 .*
 !.gitignore
+!.testr.conf
+!.stestr.conf
+!.zuul.yaml
 !.travis.yml
 .*.sw?
 __pycache__/

.stestr.conf

@@ -0,0 +1,3 @@
+[DEFAULT]
+test_path=${OS_TEST_PATH:-./cinderlib/tests/unit}
+top_dir=./

.zuul.yaml

@@ -0,0 +1,40 @@
+- project:
+    templates:
+      # TODO(geguileo): Uncomment publish-to-pypi once we release Cinder on
+      # PyPi and we can add it to test-requirements instead of the git repo.
+      # - publish-to-pypi
+      # PEP8 + PY27 + Post branch-tarball
+      - openstack-python-jobs
+      - openstack-python36-jobs
+      - publish-openstack-docs-pti
+      - release-notes-jobs-python3
+    check:
+      queue: integrated
+      jobs:
+        - cinderlib-lvm-functional
+        - cinderlib-ceph-functional
+    gate:
+      queue: integrated
+      jobs:
+        - cinderlib-lvm-functional
+        - cinderlib-ceph-functional
+
+- job:
+    name: cinderlib-lvm-functional
+    parent: openstack-tox-functional-with-sudo
+    pre-run: playbooks/setup-lvm.yaml
+    nodeset: centos-7
+
+- job:
+    name: cinderlib-ceph-functional
+    parent: openstack-tox-functional-with-sudo
+    pre-run: playbooks/setup-ceph.yaml
+    nodeset: centos-7
+    vars:
+      tox_environment:
+        CL_FTEST_CFG: "cinderlib/tests/functional/ceph.yaml"
+        CL_FTEST_ROOT_HELPER: sudo
+        # These come from great-great-grandparent tox job
+        NOSE_WITH_HTML_OUTPUT: 1
+        NOSE_HTML_OUT_FILE: nose_results.html
+        NOSE_WITH_XUNIT: 1

cinderlib/tests/functional/ceph.yaml

@@ -0,0 +1,11 @@
+# Logs are way too verbose, so we disable them
+logs: false
+
+# We only define one backend
+backends:
+    - volume_backend_name: ceph
+      volume_driver: cinder.volume.drivers.rbd.RBDDriver
+      rbd_user: admin
+      rbd_pool: rbd
+      rbd_ceph_conf: /etc/ceph/ceph.conf
+      rbd_keyring_conf: /etc/ceph/ceph.client.admin.keyring

doc/source/_static/.placeholder

@@ -0,0 +1 @@
+

playbooks/setup-ceph.yaml

@@ -0,0 +1,76 @@
+# Copyright (c) 2018, Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+#
+---
+#------------------------------------------------------------------------------
+# Setup an Ceph cluster that will be used by cinderlib's functional tests
+#------------------------------------------------------------------------------
+- hosts: all
+  vars:
+    ansible_become: yes
+  tasks:
+      # Tox job has in its pre.yaml the ensure-tox role, which installs tox
+      # from pip, which brings six from pip.  This conflicts with ceph-common's
+      # python-six dependency.
+      - name: Remove pip's six from the system
+        pip:
+          name: six
+          state: absent
+
+      - name: Install ceph-common and epel-release
+        yum:
+          name: ['epel-release', 'ceph-common']
+          state: present
+
+      - name: Install Docker from epel
+        yum:
+          name: 'docker'
+          state: present
+
+      - name: Start Docker
+        service:
+            name: docker
+            state: started
+
+      - name: Start Ceph demo
+        command: |
+          docker run -d
+          --name ceph-demo
+          -e MON_IP=127.0.0.1
+          -e CEPH_PUBLIC_NETWORK=127.0.0.1/0
+          -e DEMO_DAEMONS="osd mds"
+          --net=host
+          --volume /etc/ceph:/etc/ceph
+          --privileged
+          ceph/daemon:latest-luminous
+          demo
+
+      - name: Wait for ceph.conf
+        wait_for:
+            path: /etc/ceph/ceph.conf
+            search_regex: '[global]'
+
+      - name: Set ceph features in config
+        lineinfile:
+            path: /etc/ceph/ceph.conf
+            insertafter: '[global]'
+            line: 'rbd default features = 3'
+            state: present
+
+      - name: Set ceph keyring mode
+        file:
+            path: /etc/ceph/ceph.client.admin.keyring
+            mode: 0644

playbooks/setup-lvm.yaml

@@ -0,0 +1,75 @@
+# Copyright (c) 2018, Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+#
+---
+#------------------------------------------------------------------------------
+# Setup an LVM VG that will be used by cinderlib's functional tests
+#------------------------------------------------------------------------------
+- hosts: all
+  vars:
+    cldir: .
+    vg: cinder-volumes
+    ansible_become: yes
+  tasks:
+      - name: Install LVM package
+        package:
+          name: lvm2
+          state: present
+
+      - name: Start LVM metadata
+        service:
+            name: lvm2-lvmetad
+            state: started
+
+      - name: Create LVM backing file
+        command: "truncate -s 10G {{vg}}"
+        args:
+            creates: "{{cldir}}/{{vg}}"
+
+      - name: Check if VG already exists
+        shell: "losetup -l | awk '/{{vg}}/ {print $1}'"
+        changed_when: false
+        register: existing_loop_device
+
+      - name: "Create loopback device {{vg}}"
+        command: "losetup --show -f {{cldir}}/{{vg}}"
+        register: new_loop_device
+        when: existing_loop_device.stdout == ''
+      # Workaround because Ansible destroys registers when skipped
+      - set_fact: loop_device="{{ new_loop_device.stdout if new_loop_device.changed else existing_loop_device.stdout }}"
+
+      - name: "Create VG {{vg}}"
+        shell: "vgcreate {{vg}} {{loop_device}} && touch {{cldir}}/lvm.vgcreate"
+        args:
+            creates: "{{cldir}}/lvm.vgcreate"
+
+      - command: "vgscan --cache"
+        changed_when: false
+
+      - name: Install iSCSI package
+        package:
+          name: iscsi-initiator-utils
+          state: present
+
+      - name: Create initiator name
+        shell: echo InitiatorName=`iscsi-iname` > /etc/iscsi/initiatorname.iscsi
+        args:
+            creates: /etc/iscsi/initiatorname.iscsi
+
+      - name: Start iSCSI initiator
+        service:
+            name: iscsid
+            state: started

requirements_dev.txt

@@ -1,14 +0,0 @@
-ddt>=1.0.1
-unittest2
-pyyaml
-pip==8.1.2
-bumpversion==0.5.3
-wheel==0.31.1
-watchdog==0.8.3
-flake8==2.6.0
-tox==2.3.1
-coverage==4.1
-Sphinx==1.6.5
-mock==2.0.0
-readme_renderer==21.0
-git+https://github.com/openstack/cinder.git

setup.cfg

@@ -2,7 +2,7 @@
 universal = 1
 
 [flake8]
-exclude = docs
+exclude = .git,.venv,.tox,dist,doc,*egg,build
 
 [metadata]
 name = cinderlib

test-requirements.txt

@@ -2,6 +2,7 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
+pbr!=2.1.0,>=2.0.0 # Apache-2.0
 hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0
 coverage!=4.4,>=4.0 # Apache-2.0
 ddt>=1.0.1 # MIT

tools/cinder-cfg-to-python.py

@@ -1,4 +1,18 @@
 #!/bin/env python
+# Copyright (c) 2017, Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
 """Generate Python code to initialize cinderlib based on Cinder config file
 
 This tool generates Python code to instantiate backends using a cinder.conf

tox.ini

@@ -1,15 +1,15 @@
 [tox]
 minversion = 2.0
-envlist = py27, py36, flake8
+envlist = py27, py36, pep8
 skipsdist = True
 setenv = VIRTUAL_ENV={envdir}
-usedevelop=True
 
 [testenv]
 setenv = OS_STDOUT_CAPTURE=1
          OS_STDERR_CAPTURE=1
          OS_TEST_TIMEOUT=60
          OS_TEST_PATH=./cinderlib/tests/unit
+usedevelop=True
 install_command = pip install {opts} {packages}
 deps= -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt}
       -r{toxinidir}/test-requirements.txt
@@ -25,10 +25,13 @@ whitelist_externals =
 passenv = *_proxy *_PROXY
 
 [testenv:functional]
+usedevelop=True
 basepython=python2.7
 setenv = OS_TEST_PATH=./cinderlib/tests/functional
-         CL_FTEST_CFG={env:CL_FTEST_CFG:{toxinidir}/tools/lvm.yaml}
+         CL_FTEST_CFG={toxinidir}/{env:CL_FTEST_CFG:cinderlib/tests/functional/lvm.yaml}
          CL_FTEST_ROOT_HELPER={env:CL_FTEST_ROOT_HELPER:{toxinidir}/tools/virtualenv-sudo.sh}
+         ADD_PATHS=/usr/local/sbin:/usr/sbin
+
 sitepackages = True
 # Not reusing py27's env due to https://github.com/tox-dev/tox/issues/477
 # envdir = {toxworkdir}/py27
@@ -36,18 +39,20 @@ sitepackages = True
 # Must run serially or test_stats_with_creation may fail occasionally
 commands =
     find . -ignore_readdir_race -type f -name "*.pyc" -delete
-    stestr run --serial {posargs}
+    # Tox has a bug and it ignores the PATH set in setenv, so we work around it
+    bash -i -c "PATH=$PATH:$ADD_PATHS stestr run --serial {posargs}"
     stestr slowest
 
 whitelist_externals =
   bash
   find
 
-[testenv:functional-py35]
+[testenv:functional-py36]
+usedevelop=True
 setenv =
   {[testenv:functional]setenv}
 sitepackages = True
-basepython=python3.5
+basepython=python3.6
 # Not reusing py35's env due to https://github.com/tox-dev/tox/issues/477
 # envdir = {toxworkdir}/py35
 commands = {[testenv:functional]commands}
@@ -95,9 +100,9 @@ commands =
     coverage html -d cover
     coverage xml -o cover/coverage.xml
 
-[testenv:flake8]
+[testenv:pep8]
 basepython=python3
-commands=flake8 cinderlib
+commands=flake8 {posargs} .
 deps=
     flake8
     -r{toxinidir}/test-requirements.txt