From 2cc9fed940d7cac157ac55e1a30fedce789f4ba3 Mon Sep 17 00:00:00 2001 From: Trevor McKay Date: Wed, 4 Mar 2015 09:59:03 -0500 Subject: [PATCH] Add job-types endpoint This change adds basic support for a job-types endpoint. The abstract methods in provisioning.py will be implemented for each plugin in separate CRs. See the spec (available from the blueprint) for details on supported filtering. Change-Id: Ia6b41b1bab0f78b0b8e35633d09de93cb67bda90 Partial-Implements: blueprint edp-job-types-endpoint --- doc/source/restapi/rest_api_v1.1_EDP.rst | 449 ++++++++++++++++++++++- etc/sahara/policy.json | 4 +- sahara/api/v11.py | 10 + sahara/plugins/provisioning.py | 8 + sahara/service/edp/api.py | 63 ++++ 5 files changed, 518 insertions(+), 16 deletions(-) diff --git a/doc/source/restapi/rest_api_v1.1_EDP.rst b/doc/source/restapi/rest_api_v1.1_EDP.rst index 371d8cfc..8392b2c1 100644 --- a/doc/source/restapi/rest_api_v1.1_EDP.rst +++ b/doc/source/restapi/rest_api_v1.1_EDP.rst @@ -729,21 +729,21 @@ A Job may be run on an existing cluster or a new transient cluster may be create **Job ops** -+-----------------+-------------------------------------------------------------------+-----------------------------------------------------+ -| Verb | URI | Description | -+=================+===================================================================+=====================================================+ -| GET | /v1.1/{tenant_id}/jobs | Lists all created Jobs | -+-----------------+-------------------------------------------------------------------+-----------------------------------------------------+ -| GET | /v1.1/{tenant_id}/jobs/ | Shows info about specified Job by id | -+-----------------+-------------------------------------------------------------------+-----------------------------------------------------+ -| POST | /v1.1/{tenant_id}/jobs | Create a new Job object | -+-----------------+-------------------------------------------------------------------+-----------------------------------------------------+ -| DELETE | /v1.1/{tenant_id}/jobs/ | Removes specified Job | -+-----------------+-------------------------------------------------------------------+-----------------------------------------------------+ -| GET | /v1.1/{tenant_id}/jobs/config-hints/ | Shows default configuration by specified Job type | -+-----------------+-------------------------------------------------------------------+-----------------------------------------------------+ -| POST | /v1.1/{tenant_id}/jobs//execute | Starts Job executing | -+-----------------+-------------------------------------------------------------------+-----------------------------------------------------+ ++-----------------+-------------------------------------------------------------------+--------------------------------------------------------+ +| Verb | URI | Description | ++=================+===================================================================+========================================================+ +| GET | /v1.1/{tenant_id}/jobs | Lists all created Jobs | ++-----------------+-------------------------------------------------------------------+--------------------------------------------------------+ +| GET | /v1.1/{tenant_id}/jobs/ | Shows info about specified Job by id | ++-----------------+-------------------------------------------------------------------+--------------------------------------------------------+ +| POST | /v1.1/{tenant_id}/jobs | Create a new Job object | ++-----------------+-------------------------------------------------------------------+--------------------------------------------------------+ +| DELETE | /v1.1/{tenant_id}/jobs/ | Removes specified Job | ++-----------------+-------------------------------------------------------------------+--------------------------------------------------------+ +| GET | /v1.1/{tenant_id}/jobs/config-hints/ | Shows default configuration for Job type (deprecated) | ++-----------------+-------------------------------------------------------------------+--------------------------------------------------------+ +| POST | /v1.1/{tenant_id}/jobs//execute | Starts Job executing | ++-----------------+-------------------------------------------------------------------+--------------------------------------------------------+ **Examples** @@ -996,6 +996,8 @@ Errors: none This operation returns hints for configuration parameters which can be applied during job execution. +(deprecated) For config-hints, the *job-types* endpoint should be used instead. + This operation does not require a request body. **Note** @@ -1587,3 +1589,420 @@ The following json response represents a Job Execution object returned from Saha "job_configs": {}, "created_at": "2013-10-17 13:51:11.671977" } + + +7 Job Types +=========== + +**Description** + +Each plugin that supports EDP will support specific job types. +Different versions of a plugin may actually support different job types. +Configuration options will vary by plugin, version, and job type. + +This endpoint provides information on which job types are supported by +which plugins and optionally how they may be configured. + +**Job Binary Internal ops** + ++-----------------+------------------------------------------------------+----------------------------------------------------------+ +| Verb | URI | Description | ++=================+======================================================+==========================================================+ +| GET | /v1.1/{tenant_id}/job-types | Lists job types supported by all versions of all plugins | ++-----------------+------------------------------------------------------+----------------------------------------------------------+ +| GET | /v1.1/{tenant_id}/job-types?plugin= | Filter results by plugin name | ++-----------------+------------------------------------------------------+----------------------------------------------------------+ +| GET | /v1.1/{tenant_id}/job-types?version= | Filter results by plugin version | ++-----------------+------------------------------------------------------+----------------------------------------------------------+ +| GET | /v1.1/{tenant_id}/job-types?type= | Filter results by job type | ++-----------------+------------------------------------------------------+----------------------------------------------------------+ +| GET | /v1.1/{tenant_id}/job-types?hints=true | Include configuration hints in results | ++-----------------+------------------------------------------------------+----------------------------------------------------------+ + +Note that multiple search filters may be combined in the query string with *&* (for example ?type=Java&type=Pig&plugin=vanilla). + +**Examples** + +7.1 List all Job Types +---------------------- + +.. http:get:: /v1.1/{tenant_id}/job-types + +Normal Response Code: 200 (OK) + +Errors: none + +For all job types show which versions of which plugins support them + +This operation does not require a request body. + +**Example**: + **request** + + .. sourcecode:: http + + GET http://sahara:8386/v1.1/775181/job-types + + **response** + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + .. sourcecode:: json + + { + "job_types": [ + { + "name": "Hive", + "plugins": [ + { + "description": "The Apache Vanilla plugin.", + "name": "vanilla", + "title": "Vanilla Apache Hadoop", + "versions": { + "1.2.1": {} + } + }, + { + "description": "The Hortonworks Sahara plugin.", + "name": "hdp", + "title": "Hortonworks Data Platform", + "versions": { + "1.3.2": {}, + "2.0.6": {} + } + } + ] + }, + { + "name": "Java", + "plugins": [ + { + "description": "The Apache Vanilla plugin.", + "name": "vanilla", + "title": "Vanilla Apache Hadoop", + "versions": { + "1.2.1": {} + } + }, + { + "description": "The Hortonworks Sahara plugin.", + "name": "hdp", + "title": "Hortonworks Data Platform", + "versions": { + "1.3.2": {}, + "2.0.6": {} + } + } + ] + }, + { + "name": "MapReduce", + "plugins": [ + { + "description": "The Apache Vanilla plugin.", + "name": "vanilla", + "title": "Vanilla Apache Hadoop", + "versions": { + "1.2.1": {} + } + }, + { + "description": "The Hortonworks Sahara plugin.", + "name": "hdp", + "title": "Hortonworks Data Platform", + "versions": { + "1.3.2": {}, + "2.0.6": {} + } + } + ] + }, + { + "name": "MapReduce.Streaming", + "plugins": [ + { + "description": "The Apache Vanilla plugin.", + "name": "vanilla", + "title": "Vanilla Apache Hadoop", + "versions": { + "1.2.1": {} + } + }, + { + "description": "The Hortonworks Sahara plugin.", + "name": "hdp", + "title": "Hortonworks Data Platform", + "versions": { + "1.3.2": {}, + "2.0.6": {} + } + } + ] + }, + { + "name": "Pig", + "plugins": [ + { + "description": "The Apache Vanilla plugin.", + "name": "vanilla", + "title": "Vanilla Apache Hadoop", + "versions": { + "1.2.1": {} + } + }, + { + "description": "The Hortonworks Sahara plugin.", + "name": "hdp", + "title": "Hortonworks Data Platform", + "versions": { + "1.3.2": {}, + "2.0.6": {} + } + } + ] + } + ] + } + +7.2 List a subset of Job Types +------------------------------ + +.. http:get:: /v1.1/{tenant_id}/job-types?type=&type= + +Normal Response Code: 200 (OK) + +Errors: none + +For the specified job types show which versions of which plugins support them + +This operation does not require a request body. + +**Example**: + **request** + + .. sourcecode:: http + + GET http://sahara:8386/v1.1/775181/job-types?type=Hive&type=Java + + **response** + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + .. sourcecode:: json + + { + "job_types": [ + { + "name": "Hive", + "plugins": [ + { + "description": "The Apache Vanilla plugin.", + "name": "vanilla", + "title": "Vanilla Apache Hadoop", + "versions": { + "1.2.1": {} + } + }, + { + "description": "The Hortonworks Sahara plugin.", + "name": "hdp", + "title": "Hortonworks Data Platform", + "versions": { + "1.3.2": {}, + "2.0.6": {} + } + } + ] + }, + { + "name": "Java", + "plugins": [ + { + "description": "The Apache Vanilla plugin.", + "name": "vanilla", + "title": "Vanilla Apache Hadoop", + "versions": { + "1.2.1": {} + } + }, + { + "description": "The Hortonworks Sahara plugin.", + "name": "hdp", + "title": "Hortonworks Data Platform", + "versions": { + "1.3.2": {}, + "2.0.6": {} + } + } + ] + } + ] + } + +7.3 List all Job Types supported by a plugin version +---------------------------------------------------- + +.. http:get:: /v1.1/{tenant_id}/job-types?plugin=&version= + +Normal Response Code: 200 (OK) + +Errors: none + +Show all of the job types that are supported by a specified version +of a specified plugin + +This operation does not require a request body. + +**Example**: + **request** + + .. sourcecode:: http + + GET http://sahara:8386/v1.1/775181/job-types?plugin=hdp&version=2.0.6 + + **response** + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + .. sourcecode:: json + + { + "job_types": [ + { + "name": "Hive", + "plugins": [ + { + "description": "The Hortonworks Sahara plugin.", + "name": "hdp", + "title": "Hortonworks Data Platform", + "versions": { + "2.0.6": {} + } + } + ] + }, + { + "name": "Java", + "plugins": [ + { + "description": "The Hortonworks Sahara plugin.", + "name": "hdp", + "title": "Hortonworks Data Platform", + "versions": { + "2.0.6": {} + } + } + ] + }, + { + "name": "MapReduce", + "plugins": [ + { + "description": "The Hortonworks Sahara plugin.", + "name": "hdp", + "title": "Hortonworks Data Platform", + "versions": { + "2.0.6": {} + } + } + ] + }, + { + "name": "MapReduce.Streaming", + "plugins": [ + { + "description": "The Hortonworks Sahara plugin.", + "name": "hdp", + "title": "Hortonworks Data Platform", + "versions": { + "2.0.6": {} + } + } + ] + }, + { + "name": "Pig", + "plugins": [ + { + "description": "The Hortonworks Sahara plugin.", + "name": "hdp", + "title": "Hortonworks Data Platform", + "versions": { + "2.0.6": {} + } + } + ] + } + ] + } + +7.4 Show configuration hints for a specific Job Type supported by a specific plugin version +------------------------------------------------------------------------------------------- + +.. http:get:: /v1.1/{tenant_id}/job-types?hints=true&plugin=&version=&type= + +Normal Response Code: 200 (OK) + +Errors: none + +Show the configuration hints for a single job type supported by a particular plugin version + +This operation does not require a request body. + +**Example** + **request** + + .. sourcecode:: http + + GET http://sahara/v1.1/775181/job-types?hints=true&plugin=hdp&version=1.3.2&type=Hive + + **response** + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + .. sourcecode:: json + + { + "job_types": [ + { + "name": "Hive", + "plugins": [ + { + "description": "The Hortonworks Sahara plugin.", + "name": "hdp", + "title": "Hortonworks Data Platform", + "versions": { + "1.3.2": { + "job_config": { + "args": {}, + "configs": [ + { + "description": "Reduce tasks.", + "name": "mapred.reduce.tasks", + "value": "-1" + } + ], + "params": {} + } + } + } + } + ] + } + ] + } + + +This is an abbreviated example that shows imaginary config hints. diff --git a/etc/sahara/policy.json b/etc/sahara/policy.json index 3c2c89ca..8915f899 100644 --- a/etc/sahara/policy.json +++ b/etc/sahara/policy.json @@ -60,5 +60,7 @@ "job-binary-internals:create": "", "job-binary-internals:get": "", "job-binary-internals:delete": "", - "job-binary-internals:get_data": "" + "job-binary-internals:get_data": "", + + "job-types:get_all": "" } diff --git a/sahara/api/v11.py b/sahara/api/v11.py index 1f677f13..e06f0b28 100644 --- a/sahara/api/v11.py +++ b/sahara/api/v11.py @@ -151,8 +151,18 @@ def job_config_hints_get(job_type): return u.render(api.get_job_config_hints(job_type)) +@rest.get('/job-types') +@acl.enforce("job-types:get_all") +def job_types_get(): + # We want to use flat=False with to_dict() so that + # the value of each arg is given as a list. This supports + # filters of the form ?type=Pig&type=Java, etc. + return u.render(job_types=api.get_job_types( + **u.get_request_args().to_dict(flat=False))) + # Job binary ops + @rest.post('/job-binaries') @acl.enforce("job-binaries:create") @v.validate(v_j_b.JOB_BINARY_SCHEMA, v_j_b.check_job_binary) diff --git a/sahara/plugins/provisioning.py b/sahara/plugins/provisioning.py index ca1fb667..cb341322 100644 --- a/sahara/plugins/provisioning.py +++ b/sahara/plugins/provisioning.py @@ -65,6 +65,14 @@ class ProvisioningPluginBase(plugins_base.PluginInterface): def get_edp_engine(self, cluster, job_type): pass + @plugins_base.optional + def get_edp_job_types(self, versions=[]): + return {} + + @plugins_base.optional + def get_edp_config_hints(self, job_type, version): + return {} + @plugins_base.required_with_default def get_open_ports(self, node_group): return [] diff --git a/sahara/service/edp/api.py b/sahara/service/edp/api.py index 4820ca63..c633333f 100644 --- a/sahara/service/edp/api.py +++ b/sahara/service/edp/api.py @@ -15,11 +15,14 @@ from oslo_config import cfg from oslo_log import log as logging +import six from sahara import conductor as c from sahara import context from sahara import exceptions as ex from sahara.i18n import _LE +from sahara.plugins import base as plugin_base +from sahara.plugins import provisioning from sahara.service.edp.binary_retrievers import dispatch from sahara.service.edp import job_manager as manager from sahara.utils import edp @@ -40,6 +43,66 @@ def setup_edp_api(ops): OPS = ops +def get_job_types(**kwargs): + # Return a dictionary of all the job types that can be run + # by this instance of Sahara. For each job type, the value + # will be a list of plugins that support the job type. For + # each plugin, include a dictionary of the versions that + # support the job type. + + # All entries in kwargs are expected to have list values + hints = kwargs.get("hints", ["false"])[0].lower() == "true" + + plugin_names = kwargs.get("plugin", []) + all_plugins = plugin_base.PLUGINS.get_plugins( + base=provisioning.ProvisioningPluginBase) + if plugin_names: + plugins = filter(lambda x: x.name in plugin_names, all_plugins) + else: + plugins = all_plugins + + job_types = kwargs.get("type", edp.JOB_TYPES_ALL) + versions = kwargs.get("version", []) + + res = [] + + for job_type in job_types: + # All job types supported by all versions of the plugin. + # This is a dictionary where keys are plugin version + # strings and values are lists of job types + job_entry = {"name": job_type, + "plugins": []} + + for plugin in plugins: + types_for_plugin = plugin.get_edp_job_types(versions) + + # dict returns a new object so we are not modifying the plugin + p = plugin.dict + + # Find only the versions of this plugin that support the job. + # Additionally, instead of a list we want a dictionary of + # plugin versions with corresponding config hints + p["versions"] = {} + + for version, supported_types in six.iteritems(types_for_plugin): + if job_type in supported_types: + if hints: + config_hints = plugin.get_edp_config_hints(job_type, + version) + else: + config_hints = {} + p["versions"][version] = config_hints + + # If we found at least one version of the plugin that + # supports the job type, add the plugin to the result + if p["versions"]: + job_entry["plugins"].append(p) + + if job_entry["plugins"]: + res.append(job_entry) + return res + + def get_job_config_hints(job_type): return manager.get_job_config_hints(job_type)