Adding ability run several edp jobs flows
Field edp_jobs_flow in "clusters" section support array type and string. You can set several jobs flows Change-Id: Idb736a7a60a6c03611c962119221c433dd6a59a0
This commit is contained in:
Evgeny Sikachev
Alexander Ignatov
parent
1a8f32502e
commit
f5035bd262
@ -172,37 +172,37 @@ Section "clusters"
|
||||
|
||||
This sections is an array-type.
|
||||
|
||||
+-----------------------------+---------+----------+-----------------------------------+---------------------------------------+
|
||||
+-----------------------------+-----------------+----------+-----------------------------------+---------------------------------------+
|
||||
| Fields | Type | Required | Default | Value |
|
||||
+=============================+=========+==========+===================================+=======================================+
|
||||
+=============================+=================+==========+===================================+=======================================+
|
||||
| plugin_name | string | True | | name of plugin |
|
||||
+-----------------------------+---------+----------+-----------------------------------+---------------------------------------+
|
||||
+-----------------------------+-----------------+----------+-----------------------------------+---------------------------------------+
|
||||
| plugin_version | string | True | | version of plugin |
|
||||
+-----------------------------+---------+----------+-----------------------------------+---------------------------------------+
|
||||
+-----------------------------+-----------------+----------+-----------------------------------+---------------------------------------+
|
||||
| image | string | True | | name or id of image |
|
||||
+-----------------------------+---------+----------+-----------------------------------+---------------------------------------+
|
||||
+-----------------------------+-----------------+----------+-----------------------------------+---------------------------------------+
|
||||
| node_group_templates | object | | | see `section "node_group_templates"`_ |
|
||||
+-----------------------------+---------+----------+-----------------------------------+---------------------------------------+
|
||||
+-----------------------------+-----------------+----------+-----------------------------------+---------------------------------------+
|
||||
| cluster_template | object | | | see `section "cluster_template"`_ |
|
||||
+-----------------------------+---------+----------+-----------------------------------+---------------------------------------+
|
||||
+-----------------------------+-----------------+----------+-----------------------------------+---------------------------------------+
|
||||
| cluster | object | | | see `section "cluster"`_ |
|
||||
+-----------------------------+---------+----------+-----------------------------------+---------------------------------------+
|
||||
+-----------------------------+-----------------+----------+-----------------------------------+---------------------------------------+
|
||||
| scaling | object | | | see `section "scaling"`_ |
|
||||
+-----------------------------+---------+----------+-----------------------------------+---------------------------------------+
|
||||
+-----------------------------+-----------------+----------+-----------------------------------+---------------------------------------+
|
||||
| timeout_check_transient | integer | | 300 | timeout for checking transient |
|
||||
+-----------------------------+---------+----------+-----------------------------------+---------------------------------------+
|
||||
+-----------------------------+-----------------+----------+-----------------------------------+---------------------------------------+
|
||||
| timeout_poll_jobs_status | integer | | 1800 | timeout for polling jobs state |
|
||||
+-----------------------------+---------+----------+-----------------------------------+---------------------------------------+
|
||||
+-----------------------------+-----------------+----------+-----------------------------------+---------------------------------------+
|
||||
| timeout_delete_resource | integer | | 300 | timeout for delete resource |
|
||||
+-----------------------------+---------+----------+-----------------------------------+---------------------------------------+
|
||||
+-----------------------------+-----------------+----------+-----------------------------------+---------------------------------------+
|
||||
| timeout_poll_cluster_status | integer | | 1800 | timeout for polling cluster state |
|
||||
+-----------------------------+---------+----------+-----------------------------------+---------------------------------------+
|
||||
+-----------------------------+-----------------+----------+-----------------------------------+---------------------------------------+
|
||||
| scenario | array | | ['run_jobs', 'scale', 'run_jobs'] | array of checks |
|
||||
+-----------------------------+---------+----------+-----------------------------------+---------------------------------------+
|
||||
| edp_jobs_flow | string | | | name of edp job flow |
|
||||
+-----------------------------+---------+----------+-----------------------------------+---------------------------------------+
|
||||
+-----------------------------+-----------------+----------+-----------------------------------+---------------------------------------+
|
||||
| edp_jobs_flow | array or string | | | names of edp jobs flows |
|
||||
+-----------------------------+-----------------+----------+-----------------------------------+---------------------------------------+
|
||||
| retain_resources | boolean | | False | |
|
||||
+-----------------------------+---------+----------+-----------------------------------+---------------------------------------+
|
||||
+-----------------------------+-----------------+----------+-----------------------------------+---------------------------------------+
|
||||
|
||||
|
||||
Section "node_group_templates"
|
||||
@ -320,7 +320,7 @@ This section is an array-type.
|
||||
Section "edp_jobs_flow"
|
||||
-----------------------
|
||||
|
||||
This section has an object with a name from the `section "clusters"`_ field "edp_jobs_flow"
|
||||
This section has an object with a name from the `section "clusters"`_ field "edp_jobs_flows"
|
||||
Object has sections of array-type.
|
||||
Required: type
|
||||
|
||||
|
||||
@ -24,7 +24,7 @@ import tempfile
|
||||
|
||||
from mako import template as mako_template
|
||||
from oslo_utils import fileutils
|
||||
from six.moves import configparser
|
||||
import six
|
||||
import yaml
|
||||
|
||||
from sahara.tests.scenario import validation
|
||||
@ -65,9 +65,13 @@ def set_defaults(config):
|
||||
testcase['plugin_version'].replace('.', '_')])
|
||||
testcase['retain_resources'] = testcase.get('retain_resources', False)
|
||||
testcase['scenario'] = testcase.get('scenario', default_scenario)
|
||||
testcase['edp_jobs_flow'] = (
|
||||
config.get('edp_jobs_flow', {}).get(
|
||||
testcase.get('edp_jobs_flow', None), None))
|
||||
if isinstance(testcase.get('edp_jobs_flow'), six.string_types):
|
||||
testcase['edp_jobs_flow'] = [testcase['edp_jobs_flow']]
|
||||
edp_jobs_flow = []
|
||||
for edp_flow in testcase.get('edp_jobs_flow', []):
|
||||
edp_jobs_flow.extend(config.get('edp_jobs_flow',
|
||||
{}).get(edp_flow))
|
||||
testcase['edp_jobs_flow'] = edp_jobs_flow
|
||||
|
||||
|
||||
def _merge_dicts_sections(dict_with_section, dict_for_merge, section):
|
||||
@ -96,7 +100,7 @@ def recursive_walk(directory):
|
||||
def read_template_variables(variable_file, verbose=False):
|
||||
variables = {}
|
||||
try:
|
||||
cp = configparser.ConfigParser()
|
||||
cp = six.moves.configparser.ConfigParser()
|
||||
# key-sensitive keys
|
||||
cp.optionxform = lambda option: option
|
||||
cp.readfp(open(variable_file))
|
||||
@ -105,7 +109,7 @@ def read_template_variables(variable_file, verbose=False):
|
||||
print("WARNING: the input contains at least one template, but "
|
||||
"the variable configuration file '%s' is not valid: %s" %
|
||||
(variable_file, ioe))
|
||||
except configparser.Error as cpe:
|
||||
except six.moves.configparser.Error as cpe:
|
||||
print("WARNING: the input contains at least one template, but "
|
||||
"the variable configuration file '%s' can not be parsed: "
|
||||
"%s" % (variable_file, cpe))
|
||||
|
||||
@ -294,8 +294,7 @@ SCHEMA = {
|
||||
}
|
||||
},
|
||||
"edp_jobs_flow": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
"type": ["string", "array"]
|
||||
},
|
||||
"retain_resources": {
|
||||
"type": "boolean"
|
||||
|
||||
@ -65,7 +65,7 @@ class RunnerUnitTest(testtools.TestCase):
|
||||
"clusters": [
|
||||
{
|
||||
"image": "sahara-vanilla-2.7.1-ubuntu-14.04",
|
||||
"edp_jobs_flow": None,
|
||||
"edp_jobs_flow": [],
|
||||
"class_name": "vanilla2_7_1",
|
||||
"plugin_name": "vanilla",
|
||||
"scenario": ['run_jobs', 'scale', 'run_jobs'],
|
||||
@ -138,6 +138,20 @@ class RunnerUnitTest(testtools.TestCase):
|
||||
"args": [10, 10]
|
||||
},
|
||||
],
|
||||
"test_flow2": [
|
||||
{
|
||||
"type": "Java",
|
||||
"additional_libs": [
|
||||
{
|
||||
"type": "database",
|
||||
"source": "sahara/tests/integration/tests/"
|
||||
"resources/"
|
||||
}],
|
||||
"configs": "edp.java.main_class: org.apache.hadoop."
|
||||
"examples.QuasiMonteCarlo",
|
||||
"args": [20, 20]
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user