Fix the (experimental) multinode scenario job (spark)

Some changes are related to the way multinode works (after the final
bits have been settled during the PTG) and others are just bug fixes
that were not noticed before because the entire job did not work.
- switch to the orchestrate-devstack role as needed for multinode jobs;
  likewise, explicitly set the execution strategy to linear;
- remove few Swift related settings (services, a configuration
  variable) which are now set by default by devstack;
- increase the size of the default loop disk used by Swift
  and the maximum file size, otherwise the custom-built image
  will not fit;
- fix the variable which points to the sahara configuration
  file so that the the value set is injected in post-config;
- rework the creation of the custom flavor, and actually
  use the created flavor for the job;
- use devstack-admin credentials for now (in the future
  it may be devstack, but it may require more changes);
- remove the "voting: false" attribute because
  non-voting is implicit for experimental jobs.

Story: 2001686
Task: 8659

Change-Id: I51c1628f66beb827f3450f4432db8c985cd38f7f

.zuul.yaml

@@ -13,8 +13,7 @@
         - sahara-tests-tempest
     experimental:
       jobs:
-        - sahara-tests-scenario-multinode-spark:
-            voting: false
+        - sahara-tests-scenario-multinode-spark
         - sahara-tests-scenario-py3:
             voting: false
 
@@ -69,25 +68,29 @@
       - openstack/sahara-image-elements
       - openstack-infra/shade
     run: playbooks/sahara-tests-scenario.yaml
+    host-vars:
+      controller:
+        devstack_plugins:
+          sahara: 'git://git.openstack.org/openstack/sahara'
+          heat: 'git://git.openstack.org/openstack/heat'
+          ceilometer: 'git://git.openstack.org/openstack/ceilometer'
+          shade: 'git://git.openstack.org/openstack-infra/shade'
+    group-vars:
+      subnode:
+        devstack_services:
+          tls-proxy: false
     vars:
+      devstack_services:
+        tls-proxy: false
       devstack_localrc:
-        SWIFT_HASH: 'abcdef0123456789abcdef0123456789'
+        # required to contain (almost any) custom-built image
+        SWIFT_LOOPBACK_DISK_SIZE: '8G'
+        SWIFT_MAX_FILE_SIZE: 8589934592
       devstack_local_conf:
         post-config:
-          "$SAHARA_CONF":
+          "$SAHARA_CONF_FILE":
             DEFAULT:
               min_transient_cluster_active_time: 90
-      devstack_plugins:
-        sahara: 'git://git.openstack.org/openstack/sahara'
-        heat: 'git://git.openstack.org/openstack/heat'
-        ceilometer: 'git://git.openstack.org/openstack/ceilometer'
-        shade: 'git://git.openstack.org/openstack-infra/shade'
-      devstack_services:
-        s-proxy: true
-        s-object: true
-        s-container: true
-        s-account: true
-        tls-proxy: false
       sahara_image_name: 'xenial-server'
       sahara_image_url: 'https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img'
       sahara_plugin: 'fake'
@@ -121,7 +124,7 @@
       sahara_plugin_version: '1.6.0'
       sahara_scenario_test_template: 'spark-1.6.0.yaml.mako'
       sahara_flavors:
-        - name: sahara-flavor
+        'sah1.small':
           id: 20
           ram: 1536
           disk: 20

playbooks/sahara-tests-scenario.yaml

@@ -1,6 +1,7 @@
 - hosts: all
+  strategy: linear
   roles:
-    - run-devstack
+    - orchestrate-devstack
 
 - hosts: controller
   tasks:

roles/run-sahara-scenario/defaults/main.yaml

@@ -1,5 +1,5 @@
 ---
 sahara_tests_src_dir: "{{ zuul.projects['git.openstack.org/openstack/sahara-tests'].src_dir }}"
-sahara_cloud_demo: 'devstack'
+sahara_cloud_demo: 'devstack-admin'
 sahara_scenario_conf: '{{ ansible_user_dir }}/template_vars.ini'
 sahara_scenario_test_template: 'fake.yaml.mako'

roles/setup-sahara-scenario-env/defaults/main.yaml

@@ -1,6 +1,6 @@
 ---
 sahara_cloud_admin: 'devstack-admin'
-sahara_cloud_demo: 'devstack'
+sahara_cloud_demo: 'devstack-admin'
 sahara_plugin: 'fake'
 sahara_plugin_version: '0.1'
 sahara_image_name: 'xenial-server'
@@ -10,11 +10,11 @@ sahara_scenario_conf: '{{ ansible_user_dir }}/template_vars.ini'
 sahara_network_type: 'neutron'
 private_network_name: 'private'
 public_network_name: 'public'
-sahara_flavor_small: 'm1.small'
+sahara_flavor_small: 'sah1.small'
 sahara_cluster_transient: 'False'
 sahara_auto_security_group: 'True'
 sahara_flavors:
-  - name: sahara-flavor
+  'sah1.small':
     id: 20
     ram: 512
     disk: 10

roles/setup-sahara-scenario-env/tasks/main.yaml

@@ -32,12 +32,12 @@
   os_nova_flavor:
     cloud: "{{ sahara_cloud_admin }}"
     state: present
-    name: "{{ item.name }}"
-    ram: "{{ item.ram }}"
-    vcpus: "{{ item.vcpus|default('1') }}"
-    disk: "{{ item.disk|default('10') }}"
-    ephemeral: "{{ item.disk|default(omit) }}"
-  with_items: "{{ sahara_flavors }}"
+    name: "{{ item.key }}"
+    ram: "{{ item.value.ram }}"
+    vcpus: "{{ item.value.vcpus|default('1') }}"
+    disk: "{{ item.value.disk|default('10') }}"
+    ephemeral: "{{ item.value.disk|default(omit) }}"
+  with_dict: "{{ sahara_flavors }}"
 
 - name: generate the configuration file for the scenario test
   template: