Run add-ons tests as well

Run tests of all target programs we have.
The patch enables required services in devstack (environment
the tests are executed on in Zuul) and implements a logic when
share config options which can be discovered only by using admin
credentials are passed to the second tempestconf execution
(with demo creds) so that they are set in tempest.conf used to run
refstack-client.

Change-Id: I79b7611f47d7d67d439e24c2427ee59870732172

.zuul.yaml

@@ -25,6 +25,17 @@
       - osf/python-tempestconf
       - openstack/devstack
       - openstack/requirements
+      # NOTE until refstack-client installs plugins from local source
+      # the plugin code can't be gated by this job as the patches in gerrit
+      # are not taken into account
+      - openstack/designate
+      - openstack/designate-tempest-plugin
+      - openstack/heat
+      - openstack/heat-tempest-plugin
+      - openstack/manila
+      - openstack/manila-ui
+      # NOTE the same as for designate plugin above
+      - openstack/manila-tempest-plugin
     roles:
       - zuul: openstack/devstack
     vars:
@@ -33,6 +44,34 @@
         "{{ zuul.projects['opendev.org/osf/refstack-client'].src_dir }}/etc/tempest.conf": logs
         "{{ zuul.projects['opendev.org/osf/refstack-client'].src_dir }}/.tempest/.stestr/": logs
         "{{ zuul.projects['opendev.org/osf/refstack-client'].src_dir }}/.tempest/tempest.log": logs
+      devstack_plugins:
+        heat: https://opendev.org/openstack/heat
+        designate: https://opendev.org/openstack/designate
+        manila: https://opendev.org/openstack/manila
+        manila-ui: https://opendev.org/openstack/manila-ui
+      devstack_localrc:
+        SHARE_DRIVER: manila.share.drivers.lvm.LVMShareDriver
+        MANILA_ENABLED_BACKENDS: london,paris
+        MANILA_BACKEND1_CONFIG_GROUP_NAME: london
+        MANILA_BACKEND2_CONFIG_GROUP_NAME: paris
+        MANILA_SHARE_BACKEND1_NAME: LONDON
+        MANILA_SHARE_BACKEND2_NAME: PARIS
+        MANILA_OPTGROUP_london_driver_handles_share_servers: false
+        MANILA_OPTGROUP_paris_driver_handles_share_servers: false
+        SHARE_BACKING_FILE_SIZE: 32000M
+        MANILA_SHARE_MIGRATION_PERIOD_TASK_INTERVAL: 1
+        MANILA_DEFAULT_SHARE_TYPE_EXTRA_SPECS: 'snapshot_support=True create_share_from_snapshot_support=True revert_to_snapshot_support=True mount_snapshot_support=True'  # yamllint disable-line rule:line-length
+      devstack_services:
+        h-eng: true
+        h-api: true
+        h-api-cfn: true
+        h-api-cw: true
+        designate: true
+        designate-central: true
+        designate-api: true
+        designate-worker: true
+        designate-producer: true
+        designate-mdns: true
     run: playbooks/refstack-client-devstack.yaml
     irrelevant-files:
       - ^.*\.md$

README.md

@@ -29,7 +29,7 @@ deployment is working by passing refstack tests.
 | tempestconf_source           | False    | None                                                                | String | A path to python-tempestconf's source. If not defined, refstack-client's default is applied.            |
 | refstack_client_version      | False    | HEAD                                                                | String | Version of refstack-client cloned from git.                                                             |
 | server                       | False    | https://refstack.openstack.org/api                                  | String | Server url where the results will be uploaded to if upload_results is True.                             |
-| refstack_target_programs     | False    | ['platform', 'compute', 'object']                                   | List | The tests within the specified target programs will be executed.                                          |
+| refstack_target_programs     | False    | ['platform', 'compute', 'object', 'dns', 'orchestration', 'shared_file_system'] | List | The tests within the specified target programs will be executed.                              |
 | tempest_account_concurrency  | False    | 3                                                                   | Int    | A concurrency accounts.yaml file is generated with by tempest.                                          |
 | tempest_config_path          | False    | None                                                                | String | Destination of tempest configuration file to be used for running refstack tests.                        |
 | tempest_tag                  | False    | refstack-client's default                                           | String | Tempest will be cloned and checkouted to this specific tag.                                             |

defaults/main.yaml

@@ -21,6 +21,13 @@ refstack_target_programs:
   - platform
   - compute
   - object
+  - dns
+  - orchestration
+  - shared_file_system
+refstack_tempest_plugins:
+  manila: master
+  designate: master
+  heat: master
 url_cirros_image: "http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img"
 # a concurrency accounts.yaml file is generated with by tempest
 tempest_account_concurrency: 3

infrared_plugin/plugin.spec

@@ -85,7 +85,7 @@ subparsers:
             type: ListValue
             help: |
               Tests within the specified target programs will be executed.
-              Usage: --refstack_target_programs platform,compute
+              Usage: --refstack_target_programs platform,compute,orchestration
           upload_results:
             type: Bool
             help: |

playbooks/refstack-client-devstack.yaml

@@ -27,4 +27,6 @@
         tempestconf_source: "/home/zuul/{{ zuul.projects['opendev.org/osf/python-tempestconf'].src_dir }}"
         source_credentials: "{{ devstack_base_dir }}/devstack/openrc demo demo"
         source_admin_credentials: "{{ devstack_base_dir }}/devstack/openrc admin admin"
-        additional_tempestconf_params: "auth.tempest_roles Member object-storage.operator_role Member"
+        # not all backends support shrinking shares, LVM one which is used in
+        # this job does not support the shrinking feature
+        additional_tempestconf_params: "auth.tempest_roles Member object-storage.operator_role Member share.run_shrink_tests False"

tasks/generate-accounts.yaml

@@ -1,5 +1,5 @@
 ---
-- name: Generate tempest.conf as admin
+- name: Generate tempest_admin.conf as admin
   shell: |
     set -ex
     source .venv/bin/activate
@@ -8,13 +8,13 @@
        --debug \
        -v \
        --create \
-       --out ./etc/tempest.conf
+       --out ./etc/tempest_admin.conf
   args:
     executable: /bin/bash
     chdir: "{{ refstack_client_source }}"
   when: additional_tempestconf_params is not defined
 
-- name: Generate tempest.conf as admin additional params
+- name: Generate tempest_admin.conf as admin additional params
   shell: |
     set -ex
     source .venv/bin/activate
@@ -23,7 +23,7 @@
        --debug \
        -v \
        --create \
-       --out ./etc/tempest.conf \
+       --out ./etc/tempest_admin.conf \
        {{ additional_tempestconf_params }} \
        image.http_image {{ url_cirros_image }}
   args:
@@ -31,14 +31,18 @@
     chdir: "{{ refstack_client_source }}"
   when: additional_tempestconf_params is defined
 
-- name: Cat generated tempest.conf file
+- name: Cat generated tempest_admin.conf file
   shell: |
-    cat ./etc/tempest.conf
+    cat ./etc/tempest_admin.conf
   args:
     executable: /bin/bash
     chdir: "{{ refstack_client_source }}"
   changed_when: false
 
+- name: Set path to tempest_admin.conf
+  set_fact:
+    path_to_admin_tempest_config: "{{ refstack_client_source }}/etc/tempest_admin.conf"
+
 - name: Generate tempest accounts.yaml file
   shell: |
     set -ex
@@ -47,8 +51,9 @@
     source {{ source_admin_credentials }}
     printenv
     tempest account-generator \
-      --config-file ./etc/tempest.conf \
+      --config-file ./etc/tempest_admin.conf \
       --concurrency {{ tempest_account_concurrency }} \
+      {{ '"shared_file_system" in refstack_target_programs' | ternary('--with-admin', '') }} \
       ./etc/accounts.yaml
   args:
     executable: /bin/bash

tasks/generate-tempestconf.sh.j2

@@ -15,6 +15,7 @@ discover-tempest-config \
 validation.run_validation true \
 compute.allow_tenant_isolation true \
 compute-feature-enabled.resize true \
+{{ share_tempestconf_overrides | default([]) | join(' ') }} \
 {% if additional_tempestconf_params is defined %}
 {{ additional_tempestconf_params }} \
 {% endif %}

tasks/generate-tempestconf.yaml

@@ -1,4 +1,25 @@
 ---
+- when: "'shared_file_system' in refstack_target_programs"
+  block:
+    - name: Copy python script for parsing tempest.conf
+      copy:
+        src: parse_ini_get_tempestconf_overrides.py
+        dest: "{{ refstack_client_source }}/parse_ini_get_tempestconf_overrides.py"
+
+    - name: Parse share config from tempest_admin.conf
+      shell: |
+        python3 ./parse_ini_get_tempestconf_overrides.py \
+          backend_names,enable_protocols,multitenancy_enabled,multi_backend \
+          share \
+          etc/tempest_admin.conf
+      register: share_tempestconf_overrides
+      args:
+        chdir: "{{ refstack_client_source }}"
+
+    - name: Form share overrides for tempestconf
+      set_fact:
+        share_tempestconf_overrides: "{{ share_tempestconf_overrides.stdout_lines }}"
+
 - name: Generate tempest configuration script
   template:
     src: generate-tempestconf.sh.j2

tasks/parse_ini_get_tempestconf_overrides.py

@@ -0,0 +1,36 @@
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This script parses given .ini file and returnes value(s) of requested
+# options.
+#   $ get_val_from_ini.py <value,[value2, ...]> <section> <file path>
+
+import configparser
+import sys
+
+
+def main():
+    # parse wanted value(s) of the given section
+    val = sys.argv[1].split(',')
+    sec = sys.argv[2]
+    # load the given file
+    conf = configparser.ConfigParser()
+    conf.read(sys.argv[3])
+    for v in val:
+        sys.stdout.write(sec + "." + v + " " + conf[sec][v] + '\n')
+
+
+if __name__ == '__main__':
+    main()

tasks/post-tasks.yaml

@@ -48,14 +48,14 @@
     - name: Download results file in .json
       fetch:
         src: "{{ ls_out.files[-(item[0] | int + 1) | int].path }}"
-        dest: "{{ dest_dir }}/test_results.json"
+        dest: "{{ dest_dir }}/test_results{{ (item[0] | int + 1) | int }}.json"
         flat: true
       with_indexed_items: "{{ refstack_target_programs }}"
 
     - name: Download results file in subunit
       fetch:
         src: "{{ ls_out.files[-(item[0] | int + 1) | int].path | splitext | first }}"
-        dest: "{{ dest_dir }}/test_results_subunit"
+        dest: "{{ dest_dir }}/test_results_subunit {{ (item[0] | int + 1) | int }}"
         flat: true
       with_indexed_items: "{{ refstack_target_programs }}"
 
@@ -67,6 +67,12 @@
       delegate_to: localhost
       when: upload_results | bool
 
+    - name: Download tempest_admin.conf file
+      fetch:
+        src: "{{ path_to_admin_tempest_config }}"
+        dest: "{{ dest_dir }}/tempest_admin.conf"
+        flat: true
+
     - name: Download tempest.conf file
       fetch:
         src: "{{ path_to_tempest_config }}"

tasks/run-refstack-client.yaml

@@ -33,15 +33,14 @@
   when: test_list is defined
 
 - name: "Run {{ refstack_target_programs }} program tests associated within the {{ guideline }} guideline"
-  shell: |
+  shell: "set -ex;
-    set -ex
+    export PATH=$PATH:/usr/local/sbin:/usr/sbin;
-    export PATH=$PATH:/usr/local/sbin:/usr/sbin
+    source .venv/bin/activate;
-    source .venv/bin/activate
+    printenv;
-    printenv
+    refstack-client test -c {{ path_to_tempest_config }}  -v --test-list \
-    refstack-client test \
+    'https://refstack.openstack.org/api/v1/guidelines/\
-      -c {{ path_to_tempest_config }} \
+    {{ (item in ['dns', 'orchestration', 'shared_file_system']) | ternary(item + '.', '' ) }}\
-      -v \
+    {{ guideline }}/tests?target={{ item }}&type=required&alias=true&flag=false'"
-      --test-list "https://refstack.openstack.org/api/v1/guidelines/{{ guideline }}/tests?target={{ item }}&type=required&alias=true&flag=false"
   args:
     chdir: "{{ refstack_client_source }}"
     executable: /bin/bash