Model Evaluation

Prediction model needs to be evaluated accuracy.

Add a "meteos model-evaluation" command to allow users
to evaluate their models which they created.

implements blueprint evaluate-model
Change-Id: I4d54d789298f88251ddbd2dabab357318052cc40

meteosclient/api/client.py

@@ -26,6 +26,7 @@ from meteosclient.api import templates
 from meteosclient.api import experiments
 from meteosclient.api import datasets
 from meteosclient.api import models
+from meteosclient.api import model_evaluations
 from meteosclient.api import learnings
 
 
@@ -119,6 +120,7 @@ class Client(object):
         self.experiments = experiments.ExperimentManager(client)
         self.datasets = datasets.DatasetManager(client)
         self.models = models.ModelManager(client)
+        self.model_evaluations = model_evaluations.ModelEvaluationManager(client)
         self.learnings = learnings.LearningManager(client)
 
     def _get_keystone_auth(self, username=None, api_key=None, auth_url=None,

meteosclient/api/model_evaluations.py

@@ -0,0 +1,68 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+from six.moves.urllib import parse
+
+from meteosclient.api import base
+
+
+class ModelEvaluation(base.Resource):
+    resource_name = 'ModelEvaluation'
+
+
+class ModelEvaluationManager(base.ResourceManager):
+    resource_class = ModelEvaluation
+    NotUpdated = base.NotUpdated()
+
+    def create(self, display_name=None, model_id=None,
+               source_dataset_url=None, dataset_format=None,
+               swift_tenant=None, swift_username=None,
+               swift_password=None):
+        """Create a ModelEvaluation."""
+
+        data = {
+            'display_name': display_name,
+            'model_id': model_id,
+            'source_dataset_url': source_dataset_url,
+            'dataset_format': dataset_format,
+            'swift_tenant': swift_tenant,
+            'swift_username': swift_username,
+            'swift_password': swift_password,
+        }
+
+        body = {'model_evaluation': data}
+
+        return self._create('/model_evaluations', body, 'model_evaluation')
+
+    def list(self, search_opts=None, limit=None, marker=None,
+             sort_by=None, reverse=None):
+        """Get a list of ModelEvaluations."""
+        query = base.get_query_string(search_opts, limit=limit, marker=marker,
+                                      sort_by=sort_by, reverse=reverse)
+        url = "/model_evaluations%s" % query
+        return self._page(url, 'model_evaluations', limit)
+
+    def get(self, model_evaluation_id, show_progress=False):
+        """Get information about a ModelEvaluation."""
+        url = ('/model_evaluations/%(model_evaluation_id)s?%(params)s' %
+               {"model_evaluation_id": model_evaluation_id,
+                "params": parse.urlencode({"show_progress": show_progress})})
+
+        return self._get(url, 'model_evaluation')
+
+    def delete(self, model_evaluation_id):
+        """Delete a ModelEvaluation."""
+        self._delete('/model_evaluations/%s' % model_evaluation_id)

meteosclient/api/shell.py

@@ -288,6 +288,61 @@ def do_model_unload(cs, args):
     cs.models.unload(args.id)
 
 
+#
+# Model_Evaluations
+# ~~~~~~~~
+# model_evaluation-list
+#
+# model_evaluation-show <model_evaluation_id>
+#
+# model_evaluation-create [--json <file>]
+#
+# model_evaluation-delete <model_evaluation_id>
+#
+
+def do_model_evaluation_list(cs, args):
+    """Print a list of available model_evaluations."""
+    model_evaluations = cs.model_evaluations.list()
+
+    columns = ('id',
+               'name',
+               'status',
+               'model_id',
+               'model_type',
+               'source_dataset_url',
+               'stdout')
+    utils.print_list(model_evaluations, columns)
+
+
+@utils.arg('id',
+           metavar='<model_evaluation_id>',
+           help='ID of the model_evaluation to show.')
+def do_model_evaluation_show(cs, args):
+    """Show details of a model_evaluation."""
+    model_evaluation = cs.model_evaluations.get(args.id)
+    _show_dict(model_evaluation)
+
+
+@utils.arg('--json',
+           default=sys.stdin,
+           type=argparse.FileType('r'),
+           help='JSON representation of model_evaluation.')
+def do_model_evaluation_create(cs, args):
+    """Create a model_evaluation."""
+    model_evaluation = json.loads(args.json.read())
+
+    _filter_call_args(model_evaluation, cs.model_evaluations.create)
+    _show_dict(cs.model_evaluations.create(**model_evaluation))
+
+
+@utils.arg('id',
+           metavar='<model_evaluation_id>',
+           help='ID of the model_evaluation to delete.')
+def do_model_evaluation_delete(cs, args):
+    """Delete a model_evaluation."""
+    cs.model_evaluations.delete(args.id)
+
+
 #
 # Learnings
 # ~~~~~~~~

sample/json/model_evaluation.json

@@ -0,0 +1,8 @@
+{
+    "display_name": "sample-evaluation",
+    "source_dataset_url": "<YOUR_SOURCE_DATASET_URL>",
+    "model_id": "<YOUR_MODEL_ID>",
+    "swift_tenant": "demo",
+    "swift_username": "demo",
+    "swift_password": "nova"
+}