Migrate python-mistralclient jobs to Zuul v3

Change-Id: I5423c5460f32bafe2d907230ea7360c0444b5741
Depends-On: I10c06d3fb644c7e3026eac7a00660f0465fb0749

.stestr.conf

@@ -1,3 +1,3 @@
 [DEFAULT]
-test_path=./mistralclient/tests/unit
+test_path=${TEST_PATH:-./mistralclient/tests/unit}
 top_dir=./

.zuul.yaml

@@ -1,21 +1,33 @@
 - job:
-    name: python-mistralclient-devstack-dsvm
+    name: python-mistralclient-functional-devstack
-    parent: legacy-dsvm-base
+    parent: devstack-tox-functional-consumer
-    run: playbooks/legacy/python-mistralclient-devstack-dsvm/run.yaml
+    timeout: 9000
-    post-run: playbooks/legacy/python-mistralclient-devstack-dsvm/post.yaml
+    vars:
-    timeout: 4200
+      devstack_plugins:
+        mistral: https://git.openstack.org/openstack/mistral
+        heat: https://git.openstack.org/openstack/heat
+      devstack_services:
+        heat: True
+        h-api: True
+        h-api-cfn: True
+        h-api-cw: True
+        h-eng: True
+      tox_envlist: functional
+      tox_environment:
+        IDENTITY_API_VERSION: 3
+        PYTHONUNBUFFERED: 'true'
+        MISTRAL_USE_MOD_WSGI: True
+        MISTRAL_RPC_IMPLEMENTATION: oslo
+        MYSQL_ROOT_PW: secretdatabase
     required-projects:
-      - openstack-dev/devstack
-      - openstack-infra/devstack-gate
       - openstack/heat
       - openstack/mistral
-      - openstack/mistral-dashboard
       - openstack/python-mistralclient
 
 - project:
     check:
       jobs:
-        - python-mistralclient-devstack-dsvm
+        - python-mistralclient-functional-devstack
         # TripleO jobs that deploy Mistral.
         # Note we don't use a project-template here, so it's easier
         # to disable voting on one specific job if things go wrong.
@@ -29,8 +41,8 @@
         - openstack-tox-lower-constraints
     gate:
       jobs:
-        - python-mistralclient-devstack-dsvm
+        - python-mistralclient-functional-devstack
         - tripleo-ci-centos-7-scenario003-multinode-oooq
         - tripleo-ci-centos-7-scenario003-multinode-oooq-container
-
         - openstack-tox-lower-constraints
+

lower-constraints.txt

@@ -103,7 +103,7 @@ snowballstemmer==1.2.1
 Sphinx==1.6.2
 sphinxcontrib-websupport==1.0.1
 statsd==3.2.1
-stestr==1.0.0
+stestr==2.0.0
 stevedore==1.20.0
 tempest==17.1.0
 tenacity==3.2.1

mistralclient/tests/functional/cli/base.py

@@ -14,7 +14,7 @@
 
 import os
 
-from six.moves import configparser
+import os_client_config
 from tempest.lib.cli import base
 
 
@@ -22,54 +22,24 @@ CLI_DIR = os.environ.get(
     'OS_MISTRALCLIENT_EXEC_DIR',
     os.path.join(os.path.abspath('.'), '.tox/functional/bin')
 )
-_CREDS_FILE = 'functional_creds.conf'
 
 
-def credentials(group='admin'):
+def credentials(cloud='devstack-admin'):
-    """Retrieves credentials to run functional tests.
+    """Retrieves credentials to run functional tests
 
-    Credentials are either read from the environment or from a config file
+    Credentials are either read via os-client-config from the environment
-    ('functional_creds.conf'). Environment variables override those from the
+    or from a config file ('clouds.yaml'). Environment variables override
-    config file.
+    those from the config file.
-
+    devstack produces a clouds.yaml with two named clouds - one named
-    The 'functional_creds.conf' file is the clean and new way to use (by
+    'devstack' which has user privs and one named 'devstack-admin' which
-    default tox 2.0 does not pass environment variables).
+    has admin privs. This function will default to getting the devstack-admin
+    cloud as that is the current expected behavior.
     """
-    if group == 'admin':
+    return get_cloud_config(cloud=cloud).get_auth_args()
-        username = os.environ.get('OS_USERNAME')
-        password = os.environ.get('OS_PASSWORD')
-        tenant_name = os.environ.get('OS_TENANT_NAME')
-        user_domain = os.environ.get('OS_USER_DOMAIN_NAME')
-        project_domain = os.environ.get('OS_PROJECT_DOMAIN_NAME')
-    else:
-        username = os.environ.get('OS_ALT_USERNAME')
-        password = os.environ.get('OS_ALT_PASSWORD')
-        tenant_name = os.environ.get('OS_ALT_TENANT_NAME')
-        user_domain = os.environ.get('OS_ALT_USER_DOMAIN_NAME')
-        project_domain = os.environ.get('OS_ALT_PROJECT_DOMAIN_NAME')
 
-    auth_url = os.environ.get('OS_AUTH_URL')
 
-    config = configparser.RawConfigParser()
+def get_cloud_config(cloud='devstack-admin'):
-    if config.read(_CREDS_FILE):
+    return os_client_config.OpenStackConfig().get_one_cloud(cloud=cloud)
-        username = username or config.get(group, 'user')
-        password = password or config.get(group, 'pass')
-        tenant_name = tenant_name or config.get(group, 'tenant')
-        auth_url = auth_url or config.get('auth', 'uri')
-        user_domain = user_domain or config.get(group, 'user_domain')
-        project_domain = project_domain or config.get(group, 'project_domain')
-
-    # TODO(ddeja): Default value of OS_AUTH_URL is to provide url to v3 API.
-    # Since tempest openstack client doesn't properly handle it, we switch
-    # it back to v2. Once tempest openstack starts to use v3, this can be
-    # deleted.
-    # https://github.com/openstack/tempest/blob/master/tempest/lib/cli/base.py#L363
-    return {
-        'username': username,
-        'password': password,
-        'tenant_name': tenant_name,
-        'auth_url': auth_url.replace('v3', 'v2.0')
-    }
 
 
 class MistralCLIAuth(base.ClientTestBase):
@@ -82,8 +52,10 @@ class MistralCLIAuth(base.ClientTestBase):
         clients = base.CLIClient(
             username=creds['username'],
             password=creds['password'],
-            tenant_name=creds['tenant_name'],
+            tenant_name=creds['project_name'],
-            project_name=creds['tenant_name'],
+            project_name=creds['project_name'],
+            user_domain_id=creds['user_domain_id'],
+            project_domain_id=creds['project_domain_id'],
             uri=creds['auth_url'],
             cli_dir=CLI_DIR
         )
@@ -96,6 +68,7 @@ class MistralCLIAuth(base.ClientTestBase):
     def mistral(self, action, flags='', params='', fail_ok=False):
         """Executes Mistral command."""
         mistral_url_op = "--os-mistral-url %s" % self._mistral_url
+        flags = "{} --insecure".format(flags)
 
         if 'WITHOUT_AUTH' in os.environ:
             return base.execute(
@@ -116,21 +89,14 @@ class MistralCLIAuth(base.ClientTestBase):
                 fail_ok
             )
 
-    def get_project_id(self, project='admin'):
+    def get_project_id(self, project_name='admin'):
-        project_name = credentials(project)['tenant_name']
-
         admin_clients = self._get_clients()
 
-        # TODO(mfedosin): when bug #1719687 is closed we should provide
-        # domain names in related parameters, not just as abstract flags
-        flags = "--os-user-domain-name default " \
-                "--os-project-domain-name default " \
-                "--os-identity-api-version 3"
         projects = self.parser.listing(
             admin_clients.openstack(
                 'project show',
                 params=project_name,
-                flags=flags
+                flags='--os-identity-api-version 3 --insecure'
             )
         )
 
@@ -142,13 +108,15 @@ class MistralCLIAltAuth(base.ClientTestBase):
     _mistral_url = None
 
     def _get_alt_clients(self):
-        creds = credentials('demo')
+        creds = credentials('devstack-alt')
 
         clients = base.CLIClient(
             username=creds['username'],
             password=creds['password'],
-            project_name=creds['tenant_name'],
+            project_name=creds['project_name'],
-            tenant_name=creds['tenant_name'],
+            tenant_name=creds['project_name'],
+            user_domain_id=creds['user_domain_id'],
+            project_domain_id=creds['project_domain_id'],
             uri=creds['auth_url'],
             cli_dir=CLI_DIR
         )
@@ -161,6 +129,7 @@ class MistralCLIAltAuth(base.ClientTestBase):
     def mistral_alt(self, action, flags='', params='', mode='alt_user'):
         """Executes Mistral command for alt_user from alt_tenant."""
         mistral_url_op = "--os-mistral-url %s" % self._mistral_url
+        flags = "{} --insecure".format(flags)
 
         return self.clients.cmd_with_auth(
             'mistral %s' % mistral_url_op, action, flags, params)

mistralclient/tests/functional/cli/v2/base_v2.py

@@ -169,7 +169,7 @@ class MistralClientTestBase(base.MistralCLIAuth, base.MistralCLIAltAuth):
 
     def workflow_member_create(self, wf_id):
         cmd_param = (
-            '%s workflow %s' % (wf_id, self.get_project_id("demo"))
+            '%s workflow %s' % (wf_id, self.get_project_id("alt_demo"))
         )
         member = self.mistral_admin("member-create", params=cmd_param)
 

mistralclient/tests/functional/cli/v2/test_cli_multi_tenancy.py

@@ -219,7 +219,7 @@ class WorkflowSharingCLITests(base_v2.MistralClientTestBase):
         self.assertEqual('pending', status)
 
         cmd_param = '%s workflow --status %s --member-id %s' % (
-            self.wf[0]["ID"], new_status, self.get_project_id("demo"))
+            self.wf[0]["ID"], new_status, self.get_project_id("alt_demo"))
         member = self.mistral_alt_user("member-update", params=cmd_param)
         status = self.get_field_value(member, 'Status')
 

mistralclient/tests/functional/cli/v2/test_cli_v2.py

@@ -924,7 +924,7 @@ class CronTriggerCLITests(base_v2.MistralClientTestBase):
         trigger = self.mistral_admin(
             'cron-trigger-create',
             params=('trigger %s {} --pattern "5 * * * *" --count 5'
-                    ' --first-time "4242-12-25 13:37"' % self.wf_name)
+                    ' --first-time "4242-12-25 13:37" --utc' % self.wf_name)
         )
 
         self.assertTableStruct(trigger, ['Field', 'Value'])

playbooks/legacy/python-mistralclient-devstack-dsvm/post.yaml

@@ -1,15 +0,0 @@
-- hosts: primary
-  tasks:
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/logs/**
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs

playbooks/legacy/python-mistralclient-devstack-dsvm/run.yaml

@@ -1,65 +0,0 @@
-- hosts: all
-  name: Autoconverted job legacy-python-mistralclient-devstack-dsvm from old job gate-python-mistralclient-devstack-dsvm-ubuntu-xenial
-  tasks:
-
-    - name: Ensure legacy workspace directory
-      file:
-        path: '{{ ansible_user_dir }}/workspace'
-        state: directory
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          cat > clonemap.yaml << EOF
-          clonemap:
-            - name: openstack-infra/devstack-gate
-              dest: devstack-gate
-          EOF
-          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
-              git://git.openstack.org \
-              openstack-infra/devstack-gate
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          export PYTHONUNBUFFERED=true
-          export DEVSTACK_GATE_NEUTRON=1
-
-          if [ "python-mistralclient" = "python-mistralclient" ] ; then
-              # This puts the repo in PROJECTS
-              export DEVSTACK_PROJECT_FROM_GIT="python-mistralclient"
-          else
-              export PROJECTS="openstack/python-mistralclient $PROJECTS"
-          fi
-
-          export ENABLED_SERVICES=heat,h-api,h-api-cfn,h-api-cw,h-eng,tempest
-          export PROJECTS="openstack/heat $PROJECTS"
-          export PROJECTS="openstack/mistral $PROJECTS"
-          export PROJECTS="openstack/mistral-dashboard $PROJECTS"
-          export DEVSTACK_LOCAL_CONFIG="enable_plugin mistral https://git.openstack.org/openstack/mistral"
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin heat git://git.openstack.org/openstack/heat"
-
-          if [ "" == "-non-apache" ]; then
-            export DEVSTACK_LOCAL_CONFIG+=$'\n'"MISTRAL_USE_MOD_WSGI=False"
-          fi
-
-          if [ "" == "-kombu" ]; then
-            export DEVSTACK_LOCAL_CONFIG+=$'\n'"MISTRAL_RPC_IMPLEMENTATION=kombu"
-          fi
-
-          function post_test_hook {
-              cd /opt/stack/new/python-mistralclient/functionaltests
-              ./post_test_hook.sh
-          }
-          export -f post_test_hook
-
-          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
-          ./safe-devstack-vm-gate-wrap.sh
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'

test-requirements.txt

@@ -10,5 +10,5 @@ requests-mock>=1.2.0 # Apache-2.0
 tempest>=17.1.0 # Apache-2.0
 osprofiler>=1.4.0 # Apache-2.0
 reno>=2.5.0 # Apache-2.0
-stestr>=1.0.0 # Apache-2.0
+stestr>=2.0.0 # Apache-2.0
 openstackdocstheme>=1.18.1 # Apache-2.0

tox.ini

@@ -8,28 +8,25 @@ usedevelop = True
 install_command = pip install {opts} {packages}
 setenv =
     VIRTUAL_ENV={envdir}
-    NOSE_WITH_OPENSTACK=1
+    PYTHONDONTWRITEBYTECODE = 1
-    NOSE_OPENSTACK_COLOR=1
+    PYTHONWARNINGS=default::DeprecationWarning
-    NOSE_OPENSTACK_RED=0.05
-    NOSE_OPENSTACK_YELLOW=0.025
-    NOSE_OPENSTACK_SHOW_ELAPSED=1
-    NOSE_OPENSTACK_STDOUT=1
-    NOSE_XUNIT=1
-    DISCOVER_DIRECTORY=mistralclient/tests/unit
 passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
 deps =
     -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt}
     -r{toxinidir}/requirements.txt
     -r{toxinidir}/test-requirements.txt
 commands =
+    rm -f .testrepository/times.dbm
     find . -type f -name "*.pyc" -delete
-    stestr run --concurrency 1 {posargs}
+    stestr run --concurrency 1 --slowest {posargs}
 whitelist_externals = find
                       rm
 [testenv:functional]
 setenv =
-    OS_TEST_PATH = ./mistralclient/tests/functional
+    TEST_PATH = ./mistralclient/tests/functional
-commands = {posargs}
+commands =
+    find . -type f -name "*.pyc" -delete
+    stestr run --concurrency 1 --slowest {posargs}
 
 [testenv:pep8]
 basepython = python3
@@ -66,3 +63,4 @@ deps =
   -c{toxinidir}/lower-constraints.txt
   -r{toxinidir}/test-requirements.txt
   -r{toxinidir}/requirements.txt
+