Refactoring for pep8
implements blueprint unit-testing Change-Id: Ie80a1f26c167c2b36f6c06852e2d625b58f4c614
This commit is contained in:
parent
10d6362b61
commit
786806e5d3
|
@ -21,7 +21,6 @@ WSGI middleware for OpenStack API controllers.
|
||||||
from oslo_log import log
|
from oslo_log import log
|
||||||
import routes
|
import routes
|
||||||
|
|
||||||
from meteos.api.openstack import wsgi
|
|
||||||
from meteos.i18n import _, _LW
|
from meteos.i18n import _, _LW
|
||||||
from meteos import wsgi as base_wsgi
|
from meteos import wsgi as base_wsgi
|
||||||
|
|
||||||
|
|
|
@ -16,13 +16,7 @@
|
||||||
|
|
||||||
"""The datasets api."""
|
"""The datasets api."""
|
||||||
|
|
||||||
import ast
|
|
||||||
import re
|
|
||||||
import string
|
|
||||||
|
|
||||||
from oslo_log import log
|
from oslo_log import log
|
||||||
from oslo_utils import strutils
|
|
||||||
from oslo_utils import uuidutils
|
|
||||||
import six
|
import six
|
||||||
import webob
|
import webob
|
||||||
from webob import exc
|
from webob import exc
|
||||||
|
|
|
@ -16,13 +16,7 @@
|
||||||
|
|
||||||
"""The experiments api."""
|
"""The experiments api."""
|
||||||
|
|
||||||
import ast
|
|
||||||
import re
|
|
||||||
import string
|
|
||||||
|
|
||||||
from oslo_log import log
|
from oslo_log import log
|
||||||
from oslo_utils import strutils
|
|
||||||
from oslo_utils import uuidutils
|
|
||||||
import six
|
import six
|
||||||
import webob
|
import webob
|
||||||
from webob import exc
|
from webob import exc
|
||||||
|
|
|
@ -16,13 +16,7 @@
|
||||||
|
|
||||||
"""The learnings api."""
|
"""The learnings api."""
|
||||||
|
|
||||||
import ast
|
|
||||||
import re
|
|
||||||
import string
|
|
||||||
|
|
||||||
from oslo_log import log
|
from oslo_log import log
|
||||||
from oslo_utils import strutils
|
|
||||||
from oslo_utils import uuidutils
|
|
||||||
import six
|
import six
|
||||||
import webob
|
import webob
|
||||||
from webob import exc
|
from webob import exc
|
||||||
|
|
|
@ -16,13 +16,7 @@
|
||||||
|
|
||||||
"""The model_evaluations api."""
|
"""The model_evaluations api."""
|
||||||
|
|
||||||
import ast
|
|
||||||
import re
|
|
||||||
import string
|
|
||||||
|
|
||||||
from oslo_log import log
|
from oslo_log import log
|
||||||
from oslo_utils import strutils
|
|
||||||
from oslo_utils import uuidutils
|
|
||||||
import six
|
import six
|
||||||
import webob
|
import webob
|
||||||
from webob import exc
|
from webob import exc
|
||||||
|
@ -52,7 +46,8 @@ class ModelEvaluationController(wsgi.Controller, wsgi.AdminActionsMixin):
|
||||||
context = req.environ['meteos.context']
|
context = req.environ['meteos.context']
|
||||||
|
|
||||||
try:
|
try:
|
||||||
model_evaluation = self.engine_api.get_model_evaluation(context, id)
|
model_evaluation = self.engine_api.get_model_evaluation(context,
|
||||||
|
id)
|
||||||
except exception.NotFound:
|
except exception.NotFound:
|
||||||
raise exc.HTTPNotFound()
|
raise exc.HTTPNotFound()
|
||||||
|
|
||||||
|
|
|
@ -16,13 +16,7 @@
|
||||||
|
|
||||||
"""The models api."""
|
"""The models api."""
|
||||||
|
|
||||||
import ast
|
|
||||||
import re
|
|
||||||
import string
|
|
||||||
|
|
||||||
from oslo_log import log
|
from oslo_log import log
|
||||||
from oslo_utils import strutils
|
|
||||||
from oslo_utils import uuidutils
|
|
||||||
import six
|
import six
|
||||||
import webob
|
import webob
|
||||||
from webob import exc
|
from webob import exc
|
||||||
|
|
|
@ -20,12 +20,12 @@ WSGI middleware for OpenStack Learning API v1.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import meteos.api.openstack
|
import meteos.api.openstack
|
||||||
from meteos.api.v1 import learnings
|
|
||||||
from meteos.api.v1 import experiments
|
|
||||||
from meteos.api.v1 import templates
|
|
||||||
from meteos.api.v1 import datasets
|
from meteos.api.v1 import datasets
|
||||||
from meteos.api.v1 import models
|
from meteos.api.v1 import experiments
|
||||||
|
from meteos.api.v1 import learnings
|
||||||
from meteos.api.v1 import model_evaluations
|
from meteos.api.v1 import model_evaluations
|
||||||
|
from meteos.api.v1 import models
|
||||||
|
from meteos.api.v1 import templates
|
||||||
from meteos.api import versions
|
from meteos.api import versions
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -16,13 +16,7 @@
|
||||||
|
|
||||||
"""The templates api."""
|
"""The templates api."""
|
||||||
|
|
||||||
import ast
|
|
||||||
import re
|
|
||||||
import string
|
|
||||||
|
|
||||||
from oslo_log import log
|
from oslo_log import log
|
||||||
from oslo_utils import strutils
|
|
||||||
from oslo_utils import uuidutils
|
|
||||||
import six
|
import six
|
||||||
import webob
|
import webob
|
||||||
from webob import exc
|
from webob import exc
|
||||||
|
|
|
@ -19,7 +19,6 @@ import copy
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
|
|
||||||
from meteos.api import openstack
|
from meteos.api import openstack
|
||||||
from meteos.api.openstack import api_version_request
|
|
||||||
from meteos.api.openstack import wsgi
|
from meteos.api.openstack import wsgi
|
||||||
from meteos.api.views import versions as views_versions
|
from meteos.api.views import versions as views_versions
|
||||||
|
|
||||||
|
|
|
@ -47,8 +47,6 @@ class ViewBuilder(common.ViewBuilder):
|
||||||
|
|
||||||
def detail(self, request, dataset):
|
def detail(self, request, dataset):
|
||||||
"""Detailed view of a single dataset."""
|
"""Detailed view of a single dataset."""
|
||||||
context = request.environ['meteos.context']
|
|
||||||
|
|
||||||
dataset_dict = {
|
dataset_dict = {
|
||||||
'id': dataset.get('id'),
|
'id': dataset.get('id'),
|
||||||
'created_at': dataset.get('created_at'),
|
'created_at': dataset.get('created_at'),
|
||||||
|
|
|
@ -46,8 +46,6 @@ class ViewBuilder(common.ViewBuilder):
|
||||||
|
|
||||||
def detail(self, request, experiment):
|
def detail(self, request, experiment):
|
||||||
"""Detailed view of a single experiment."""
|
"""Detailed view of a single experiment."""
|
||||||
context = request.environ['meteos.context']
|
|
||||||
|
|
||||||
experiment_dict = {
|
experiment_dict = {
|
||||||
'id': experiment.get('id'),
|
'id': experiment.get('id'),
|
||||||
'created_at': experiment.get('created_at'),
|
'created_at': experiment.get('created_at'),
|
||||||
|
|
|
@ -49,8 +49,6 @@ class ViewBuilder(common.ViewBuilder):
|
||||||
|
|
||||||
def detail(self, request, learning):
|
def detail(self, request, learning):
|
||||||
"""Detailed view of a single learning."""
|
"""Detailed view of a single learning."""
|
||||||
context = request.environ['meteos.context']
|
|
||||||
|
|
||||||
learning_dict = {
|
learning_dict = {
|
||||||
'id': learning.get('id'),
|
'id': learning.get('id'),
|
||||||
'created_at': learning.get('created_at'),
|
'created_at': learning.get('created_at'),
|
||||||
|
|
|
@ -49,8 +49,6 @@ class ViewBuilder(common.ViewBuilder):
|
||||||
|
|
||||||
def detail(self, request, model_evaluation):
|
def detail(self, request, model_evaluation):
|
||||||
"""Detailed view of a single model evaluation."""
|
"""Detailed view of a single model evaluation."""
|
||||||
context = request.environ['meteos.context']
|
|
||||||
|
|
||||||
model_evaluation_dict = {
|
model_evaluation_dict = {
|
||||||
'id': model_evaluation.get('id'),
|
'id': model_evaluation.get('id'),
|
||||||
'name': model_evaluation.get('display_name'),
|
'name': model_evaluation.get('display_name'),
|
||||||
|
@ -65,17 +63,19 @@ class ViewBuilder(common.ViewBuilder):
|
||||||
'stderr': model_evaluation.get('stderr'),
|
'stderr': model_evaluation.get('stderr'),
|
||||||
}
|
}
|
||||||
|
|
||||||
self.update_versioned_resource_dict(request, model_evaluation_dict, model_evaluation)
|
self.update_versioned_resource_dict(request,
|
||||||
|
model_evaluation_dict,
|
||||||
|
model_evaluation)
|
||||||
|
|
||||||
return {'model_evaluation': model_evaluation_dict}
|
return {'model_evaluation': model_evaluation_dict}
|
||||||
|
|
||||||
def _list_view(self, func, request, model_evaluations):
|
def _list_view(self, func, request, model_evaluations):
|
||||||
"""Provide a view for a list of model evaluations."""
|
"""Provide a view for a list of model evaluations."""
|
||||||
model_evaluations_list = [func(request, model_evaluation)['model_evaluation']
|
model_evaluations_list = [func(request, model_evaluation)['model_evaluation']
|
||||||
for model_evaluation in model_evaluations]
|
for model_evaluation in model_evaluations]
|
||||||
model_evaluations_links = self._get_collection_links(request,
|
model_evaluations_links = self._get_collection_links(request,
|
||||||
model_evaluations,
|
model_evaluations,
|
||||||
self._collection_name)
|
self._collection_name)
|
||||||
model_evaluations_dict = dict(model_evaluations=model_evaluations_list)
|
model_evaluations_dict = dict(model_evaluations=model_evaluations_list)
|
||||||
|
|
||||||
if model_evaluations_links:
|
if model_evaluations_links:
|
||||||
|
|
|
@ -48,8 +48,6 @@ class ViewBuilder(common.ViewBuilder):
|
||||||
|
|
||||||
def detail(self, request, model):
|
def detail(self, request, model):
|
||||||
"""Detailed view of a single model."""
|
"""Detailed view of a single model."""
|
||||||
context = request.environ['meteos.context']
|
|
||||||
|
|
||||||
model_dict = {
|
model_dict = {
|
||||||
'id': model.get('id'),
|
'id': model.get('id'),
|
||||||
'created_at': model.get('created_at'),
|
'created_at': model.get('created_at'),
|
||||||
|
|
|
@ -48,8 +48,6 @@ class ViewBuilder(common.ViewBuilder):
|
||||||
|
|
||||||
def detail(self, request, template):
|
def detail(self, request, template):
|
||||||
"""Detailed view of a single template."""
|
"""Detailed view of a single template."""
|
||||||
context = request.environ['meteos.context']
|
|
||||||
|
|
||||||
template_dict = {
|
template_dict = {
|
||||||
'id': template.get('id'),
|
'id': template.get('id'),
|
||||||
'created_at': template.get('created_at'),
|
'created_at': template.get('created_at'),
|
||||||
|
|
|
@ -32,35 +32,40 @@
|
||||||
|
|
||||||
import base64
|
import base64
|
||||||
import os
|
import os
|
||||||
|
import socket
|
||||||
import sys
|
import sys
|
||||||
import uuid
|
import uuid
|
||||||
import socket
|
|
||||||
from ast import literal_eval
|
from ast import literal_eval
|
||||||
from numpy import array
|
|
||||||
from math import sqrt
|
from math import sqrt
|
||||||
|
from numpy import array
|
||||||
from pyspark import SparkContext
|
from pyspark import SparkContext
|
||||||
|
|
||||||
from pyspark.mllib.linalg import SparseVector
|
|
||||||
from pyspark.mllib.classification import LogisticRegressionWithSGD
|
|
||||||
from pyspark.mllib.classification import LogisticRegressionModel
|
from pyspark.mllib.classification import LogisticRegressionModel
|
||||||
from pyspark.mllib.clustering import KMeans, KMeansModel
|
from pyspark.mllib.classification import LogisticRegressionWithSGD
|
||||||
|
from pyspark.mllib.clustering import KMeans
|
||||||
|
from pyspark.mllib.clustering import KMeansModel
|
||||||
|
from pyspark.mllib.evaluation import BinaryClassificationMetrics
|
||||||
|
from pyspark.mllib.evaluation import MulticlassMetrics
|
||||||
|
from pyspark.mllib.evaluation import RankingMetrics
|
||||||
|
from pyspark.mllib.evaluation import RegressionMetrics
|
||||||
from pyspark.mllib.feature import Word2Vec
|
from pyspark.mllib.feature import Word2Vec
|
||||||
from pyspark.mllib.feature import Word2VecModel
|
from pyspark.mllib.feature import Word2VecModel
|
||||||
from pyspark.mllib.fpm import FPGrowth
|
from pyspark.mllib.fpm import FPGrowth
|
||||||
from pyspark.mllib.fpm import FPGrowthModel
|
from pyspark.mllib.fpm import FPGrowthModel
|
||||||
from pyspark.mllib.evaluation import BinaryClassificationMetrics
|
from pyspark.mllib.linalg import SparseVector
|
||||||
from pyspark.mllib.evaluation import RegressionMetrics
|
from pyspark.mllib.recommendation import ALS
|
||||||
from pyspark.mllib.evaluation import MulticlassMetrics
|
from pyspark.mllib.recommendation import MatrixFactorizationModel
|
||||||
from pyspark.mllib.evaluation import RankingMetrics
|
from pyspark.mllib.recommendation import Rating
|
||||||
from pyspark.mllib.recommendation import ALS, MatrixFactorizationModel, Rating
|
|
||||||
from pyspark.mllib.regression import LabeledPoint
|
from pyspark.mllib.regression import LabeledPoint
|
||||||
from pyspark.mllib.regression import LinearRegressionWithSGD
|
|
||||||
from pyspark.mllib.regression import LinearRegressionModel
|
from pyspark.mllib.regression import LinearRegressionModel
|
||||||
from pyspark.mllib.tree import DecisionTree, DecisionTreeModel
|
from pyspark.mllib.regression import LinearRegressionWithSGD
|
||||||
|
from pyspark.mllib.tree import DecisionTree
|
||||||
|
from pyspark.mllib.tree import DecisionTreeModel
|
||||||
from pyspark.mllib.util import MLUtils
|
from pyspark.mllib.util import MLUtils
|
||||||
|
|
||||||
|
|
||||||
EXIT_CODE='80577372-9349-463a-bbc3-1ca54f187cc9'
|
EXIT_CODE = '80577372-9349-463a-bbc3-1ca54f187cc9'
|
||||||
|
|
||||||
|
|
||||||
class ModelController(object):
|
class ModelController(object):
|
||||||
|
@ -155,12 +160,11 @@ class RecommendationController(ModelController):
|
||||||
testData = ratings.map(lambda p: (p.user, p.product))
|
testData = ratings.map(lambda p: (p.user, p.product))
|
||||||
|
|
||||||
predictions = model.predictAll(testData)\
|
predictions = model.predictAll(testData)\
|
||||||
.map(lambda r: ((r.user, r.product), r.rating))
|
.map(lambda r: ((r.user, r.product), r.rating))
|
||||||
|
|
||||||
ratingsTuple = ratings.map(lambda r: ((r.user, r.product), r.rating))
|
ratingsTuple = ratings.map(lambda r: ((r.user, r.product), r.rating))
|
||||||
scoreAndLabels = predictions.join(ratingsTuple).map(lambda tup: tup[1])
|
scoreAndLabels = predictions.join(ratingsTuple).map(lambda tup: tup[1])
|
||||||
|
|
||||||
# Instantiate regression metrics to compare predicted and actual ratings
|
|
||||||
metrics = RegressionMetrics(scoreAndLabels)
|
metrics = RegressionMetrics(scoreAndLabels)
|
||||||
|
|
||||||
result = "{}: {}".format("MAE", metrics.meanAbsoluteError) + os.linesep\
|
result = "{}: {}".format("MAE", metrics.meanAbsoluteError) + os.linesep\
|
||||||
|
@ -287,7 +291,7 @@ class DecisionTreeModelController(ModelController):
|
||||||
index = ','.join(index_l)
|
index = ','.join(index_l)
|
||||||
value = ','.join(value_l)
|
value = ','.join(value_l)
|
||||||
|
|
||||||
parsed_str = '(' + param_len + ', [' + index + '],[' + value + '])'
|
parsed_str = '(' + param_len + ', [' + index + '],[' + value + '])'
|
||||||
|
|
||||||
return SparseVector.parse(parsed_str)
|
return SparseVector.parse(parsed_str)
|
||||||
|
|
||||||
|
@ -378,7 +382,7 @@ class FPGrowthModelController(ModelController):
|
||||||
|
|
||||||
transactions = data.map(lambda line: line.strip().split(' '))
|
transactions = data.map(lambda line: line.strip().split(' '))
|
||||||
|
|
||||||
model= FPGrowth.train(transactions,
|
model = FPGrowth.train(transactions,
|
||||||
minSupport=minSupport,
|
minSupport=minSupport,
|
||||||
numPartitions=numPartitions)
|
numPartitions=numPartitions)
|
||||||
|
|
||||||
|
@ -431,7 +435,7 @@ class MeteosSparkController(object):
|
||||||
if collect:
|
if collect:
|
||||||
self.data.collect()
|
self.data.collect()
|
||||||
self.data.saveAsTextFile(self.datapath)
|
self.data.saveAsTextFile(self.datapath)
|
||||||
print self.data.take(10)
|
print(self.data.take(10))
|
||||||
|
|
||||||
def load_data(self):
|
def load_data(self):
|
||||||
|
|
||||||
|
@ -475,7 +479,8 @@ class MeteosSparkController(object):
|
||||||
dataset_format = self.job_args.get('dataset_format')
|
dataset_format = self.job_args.get('dataset_format')
|
||||||
|
|
||||||
if dataset_format == 'libsvm':
|
if dataset_format == 'libsvm':
|
||||||
self.model = self.controller.create_model_libsvm(self.data, list_params)
|
self.model = self.controller.create_model_libsvm(self.data,
|
||||||
|
list_params)
|
||||||
else:
|
else:
|
||||||
self.model = self.controller.create_model(self.data, list_params)
|
self.model = self.controller.create_model(self.data, list_params)
|
||||||
|
|
||||||
|
|
|
@ -18,20 +18,14 @@
|
||||||
Handles all requests relating to volumes + sahara.
|
Handles all requests relating to volumes + sahara.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import copy
|
|
||||||
import os
|
|
||||||
|
|
||||||
from osc_lib import exceptions as sahara_exception
|
|
||||||
from saharaclient import client as sahara_client
|
|
||||||
from keystoneauth1 import loading as ks_loading
|
from keystoneauth1 import loading as ks_loading
|
||||||
|
from osc_lib import exceptions as sahara_exception
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
import six
|
from saharaclient import client as sahara_client
|
||||||
|
|
||||||
from meteos.common import client_auth
|
from meteos.common import client_auth
|
||||||
from meteos.common.config import core_opts
|
from meteos.common.config import core_opts
|
||||||
import meteos.context as ctxt
|
|
||||||
from meteos.db import base
|
from meteos.db import base
|
||||||
from meteos import exception
|
|
||||||
from meteos.i18n import _
|
from meteos.i18n import _
|
||||||
|
|
||||||
SAHARA_GROUP = 'sahara'
|
SAHARA_GROUP = 'sahara'
|
||||||
|
|
|
@ -30,11 +30,7 @@ from oslo_config import cfg
|
||||||
from oslo_log import log
|
from oslo_log import log
|
||||||
from oslo_middleware import cors
|
from oslo_middleware import cors
|
||||||
from oslo_utils import netutils
|
from oslo_utils import netutils
|
||||||
import six
|
|
||||||
|
|
||||||
from meteos.common import constants
|
|
||||||
from meteos import exception
|
|
||||||
from meteos.i18n import _
|
|
||||||
|
|
||||||
CONF = cfg.CONF
|
CONF = cfg.CONF
|
||||||
log.register_options(CONF)
|
log.register_options(CONF)
|
||||||
|
|
|
@ -307,7 +307,8 @@ def model_evaluation_get(context, model_evaluation_id):
|
||||||
return IMPL.model_evaluation_get(context, model_evaluation_id)
|
return IMPL.model_evaluation_get(context, model_evaluation_id)
|
||||||
|
|
||||||
|
|
||||||
def model_evaluation_get_all(context, filters=None, sort_key=None, sort_dir=None):
|
def model_evaluation_get_all(context, filters=None,
|
||||||
|
sort_key=None, sort_dir=None):
|
||||||
"""Get all model_evaluations."""
|
"""Get all model_evaluations."""
|
||||||
return IMPL.model_evaluation_get_all(
|
return IMPL.model_evaluation_get_all(
|
||||||
context, filters=filters, sort_key=sort_key, sort_dir=sort_dir,
|
context, filters=filters, sort_key=sort_key, sort_dir=sort_dir,
|
||||||
|
@ -315,7 +316,7 @@ def model_evaluation_get_all(context, filters=None, sort_key=None, sort_dir=None
|
||||||
|
|
||||||
|
|
||||||
def model_evaluation_get_all_by_project(context, project_id, filters=None,
|
def model_evaluation_get_all_by_project(context, project_id, filters=None,
|
||||||
sort_key=None, sort_dir=None):
|
sort_key=None, sort_dir=None):
|
||||||
"""Returns all model_evaluations with given project ID."""
|
"""Returns all model_evaluations with given project ID."""
|
||||||
return IMPL.model_evaluation_get_all_by_project(
|
return IMPL.model_evaluation_get_all_by_project(
|
||||||
context, project_id, filters=filters,
|
context, project_id, filters=filters,
|
||||||
|
|
|
@ -26,8 +26,8 @@ down_revision = None
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
from oslo_log import log
|
from oslo_log import log
|
||||||
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Text
|
from sqlalchemy import Boolean, Column, DateTime, Text
|
||||||
from sqlalchemy import Integer, MetaData, String, Table, UniqueConstraint
|
from sqlalchemy import Integer, MetaData, String, Table
|
||||||
|
|
||||||
from meteos.i18n import _LE
|
from meteos.i18n import _LE
|
||||||
|
|
||||||
|
|
|
@ -19,10 +19,8 @@
|
||||||
"""Implementation of SQLAlchemy backend."""
|
"""Implementation of SQLAlchemy backend."""
|
||||||
|
|
||||||
import copy
|
import copy
|
||||||
import datetime
|
|
||||||
from functools import wraps
|
from functools import wraps
|
||||||
import sys
|
import sys
|
||||||
import uuid
|
|
||||||
import warnings
|
import warnings
|
||||||
|
|
||||||
# NOTE(uglide): Required to override default oslo_db Query class
|
# NOTE(uglide): Required to override default oslo_db Query class
|
||||||
|
@ -30,21 +28,13 @@ import meteos.db.sqlalchemy.query # noqa
|
||||||
|
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
from oslo_db import api as oslo_db_api
|
from oslo_db import api as oslo_db_api
|
||||||
from oslo_db import exception as db_exception
|
|
||||||
from oslo_db import options as db_options
|
from oslo_db import options as db_options
|
||||||
from oslo_db.sqlalchemy import session
|
from oslo_db.sqlalchemy import session
|
||||||
from oslo_db.sqlalchemy import utils as db_utils
|
from oslo_db.sqlalchemy import utils as db_utils
|
||||||
from oslo_log import log
|
from oslo_log import log
|
||||||
from oslo_utils import timeutils
|
|
||||||
from oslo_utils import uuidutils
|
from oslo_utils import uuidutils
|
||||||
import six
|
|
||||||
from sqlalchemy import and_
|
|
||||||
from sqlalchemy import or_
|
|
||||||
from sqlalchemy.orm import joinedload
|
|
||||||
from sqlalchemy.sql.expression import true
|
|
||||||
from sqlalchemy.sql import func
|
from sqlalchemy.sql import func
|
||||||
|
|
||||||
from meteos.common import constants
|
|
||||||
from meteos.db.sqlalchemy import models
|
from meteos.db.sqlalchemy import models
|
||||||
from meteos import exception
|
from meteos import exception
|
||||||
from meteos.i18n import _, _LE, _LW
|
from meteos.i18n import _, _LE, _LW
|
||||||
|
@ -239,19 +229,6 @@ def ensure_dict_has_id(model_dict):
|
||||||
return model_dict
|
return model_dict
|
||||||
|
|
||||||
|
|
||||||
def _sync_learnings(context, project_id, user_id, session):
|
|
||||||
(learnings, gigs) = learning_data_get_for_project(context,
|
|
||||||
project_id,
|
|
||||||
user_id,
|
|
||||||
session=session)
|
|
||||||
return {'learnings': learnings}
|
|
||||||
|
|
||||||
|
|
||||||
QUOTA_SYNC_FUNCTIONS = {
|
|
||||||
'_sync_learnings': _sync_learnings,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
#
|
#
|
||||||
|
|
||||||
|
|
||||||
|
@ -857,11 +834,14 @@ def model_evaluation_create(context, model_evaluation_values):
|
||||||
model_evaluation_ref.save(session=session)
|
model_evaluation_ref.save(session=session)
|
||||||
|
|
||||||
# NOTE(u_glide): Do so to prevent errors with relationships
|
# NOTE(u_glide): Do so to prevent errors with relationships
|
||||||
return model_evaluation_get(context, model_evaluation_ref['id'], session=session)
|
return model_evaluation_get(context,
|
||||||
|
model_evaluation_ref['id'],
|
||||||
|
session=session)
|
||||||
|
|
||||||
|
|
||||||
def _model_evaluation_get_all_with_filters(context, project_id=None, filters=None,
|
def _model_evaluation_get_all_with_filters(context, project_id=None,
|
||||||
sort_key=None, sort_dir=None):
|
filters=None, sort_key=None,
|
||||||
|
sort_dir=None):
|
||||||
if not sort_key:
|
if not sort_key:
|
||||||
sort_key = 'created_at'
|
sort_key = 'created_at'
|
||||||
if not sort_dir:
|
if not sort_dir:
|
||||||
|
@ -899,7 +879,7 @@ def _model_evaluation_get_all_with_filters(context, project_id=None, filters=Non
|
||||||
|
|
||||||
@require_context
|
@require_context
|
||||||
def model_evaluation_get_all_by_project(context, project_id, filters=None,
|
def model_evaluation_get_all_by_project(context, project_id, filters=None,
|
||||||
sort_key=None, sort_dir=None):
|
sort_key=None, sort_dir=None):
|
||||||
"""Returns list of model_evaluations with given project ID."""
|
"""Returns list of model_evaluations with given project ID."""
|
||||||
query = _model_evaluation_get_all_with_filters(
|
query = _model_evaluation_get_all_with_filters(
|
||||||
context, project_id=project_id, filters=filters,
|
context, project_id=project_id, filters=filters,
|
||||||
|
@ -913,7 +893,9 @@ def model_evaluation_delete(context, model_evaluation_id):
|
||||||
session = get_session()
|
session = get_session()
|
||||||
|
|
||||||
with session.begin():
|
with session.begin():
|
||||||
model_evaluation_ref = model_evaluation_get(context, model_evaluation_id, session)
|
model_evaluation_ref = model_evaluation_get(context,
|
||||||
|
model_evaluation_id,
|
||||||
|
session)
|
||||||
model_evaluation_ref.soft_delete(session=session)
|
model_evaluation_ref.soft_delete(session=session)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -22,10 +22,9 @@ SQLAlchemy models for Meteos data.
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
from oslo_db.sqlalchemy import models
|
from oslo_db.sqlalchemy import models
|
||||||
from oslo_log import log
|
from oslo_log import log
|
||||||
from sqlalchemy import Column, Integer, String, schema
|
from sqlalchemy import Column, Integer, String
|
||||||
from sqlalchemy.ext.declarative import declarative_base
|
from sqlalchemy.ext.declarative import declarative_base
|
||||||
from sqlalchemy import orm
|
from sqlalchemy import DateTime, Boolean, Text
|
||||||
from sqlalchemy import ForeignKey, DateTime, Boolean, Enum, Text
|
|
||||||
|
|
||||||
from meteos.common import constants
|
from meteos.common import constants
|
||||||
|
|
||||||
|
|
|
@ -20,20 +20,16 @@
|
||||||
Handles all requests relating to learnings.
|
Handles all requests relating to learnings.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from oslo_config import cfg
|
|
||||||
from oslo_log import log
|
from oslo_log import log
|
||||||
from oslo_utils import excutils
|
from oslo_utils import excutils
|
||||||
from oslo_utils import strutils
|
|
||||||
from oslo_utils import timeutils
|
|
||||||
import six
|
import six
|
||||||
|
|
||||||
from meteos.common import constants
|
from meteos.common import constants
|
||||||
from meteos.db import base
|
from meteos.db import base
|
||||||
|
from meteos.engine import rpcapi as engine_rpcapi
|
||||||
from meteos import exception
|
from meteos import exception
|
||||||
from meteos.i18n import _, _LE, _LI, _LW
|
from meteos.i18n import _, _LE, _LI, _LW
|
||||||
from meteos import policy
|
from meteos import policy
|
||||||
from meteos.engine import rpcapi as engine_rpcapi
|
|
||||||
from meteos import utils
|
|
||||||
|
|
||||||
LOG = log.getLogger(__name__)
|
LOG = log.getLogger(__name__)
|
||||||
|
|
||||||
|
@ -121,7 +117,7 @@ class API(base.Base):
|
||||||
"statuses": statuses}
|
"statuses": statuses}
|
||||||
raise exception.InvalidLearning(reason=msg)
|
raise exception.InvalidLearning(reason=msg)
|
||||||
|
|
||||||
result = self.engine_rpcapi.delete_template(context, id)
|
self.engine_rpcapi.delete_template(context, id)
|
||||||
|
|
||||||
def get_all_experiments(self, context, search_opts=None,
|
def get_all_experiments(self, context, search_opts=None,
|
||||||
sort_key='created_at', sort_dir='desc'):
|
sort_key='created_at', sort_dir='desc'):
|
||||||
|
@ -199,7 +195,7 @@ class API(base.Base):
|
||||||
"statuses": statuses}
|
"statuses": statuses}
|
||||||
raise exception.InvalidLearning(reason=msg)
|
raise exception.InvalidLearning(reason=msg)
|
||||||
|
|
||||||
result = self.engine_rpcapi.delete_experiment(context, id)
|
self.engine_rpcapi.delete_experiment(context, id)
|
||||||
|
|
||||||
def get_all_datasets(self, context, search_opts=None,
|
def get_all_datasets(self, context, search_opts=None,
|
||||||
sort_key='created_at', sort_dir='desc'):
|
sort_key='created_at', sort_dir='desc'):
|
||||||
|
@ -286,10 +282,10 @@ class API(base.Base):
|
||||||
"statuses": statuses}
|
"statuses": statuses}
|
||||||
raise exception.InvalidLearning(reason=msg)
|
raise exception.InvalidLearning(reason=msg)
|
||||||
|
|
||||||
result = self.engine_rpcapi.delete_dataset(context,
|
self.engine_rpcapi.delete_dataset(context,
|
||||||
dataset['cluster_id'],
|
dataset['cluster_id'],
|
||||||
dataset['job_id'],
|
dataset['job_id'],
|
||||||
id)
|
id)
|
||||||
|
|
||||||
def _enable_load_model(self, context):
|
def _enable_load_model(self, context):
|
||||||
|
|
||||||
|
@ -387,10 +383,10 @@ class API(base.Base):
|
||||||
"statuses": statuses}
|
"statuses": statuses}
|
||||||
raise exception.InvalidLearning(reason=msg)
|
raise exception.InvalidLearning(reason=msg)
|
||||||
|
|
||||||
result = self.engine_rpcapi.delete_model(context,
|
self.engine_rpcapi.delete_model(context,
|
||||||
model['cluster_id'],
|
model['cluster_id'],
|
||||||
model['job_id'],
|
model['job_id'],
|
||||||
id)
|
id)
|
||||||
|
|
||||||
def load_model(self, context, id, dataset_format, model_type,
|
def load_model(self, context, id, dataset_format, model_type,
|
||||||
job_template_id, experiment_id, cluster_id):
|
job_template_id, experiment_id, cluster_id):
|
||||||
|
@ -445,20 +441,22 @@ class API(base.Base):
|
||||||
self.db.model_update(context, id, updates)
|
self.db.model_update(context, id, updates)
|
||||||
|
|
||||||
def get_all_model_evaluations(self, context, search_opts=None,
|
def get_all_model_evaluations(self, context, search_opts=None,
|
||||||
sort_key='created_at', sort_dir='desc'):
|
sort_key='created_at', sort_dir='desc'):
|
||||||
policy.check_policy(context, 'model_evaluation', 'get_all')
|
policy.check_policy(context, 'model_evaluation', 'get_all')
|
||||||
|
|
||||||
if search_opts is None:
|
if search_opts is None:
|
||||||
search_opts = {}
|
search_opts = {}
|
||||||
|
|
||||||
LOG.debug("Searching for model evaluations by: %s", six.text_type(search_opts))
|
LOG.debug("Searching for model evaluations by: %s",
|
||||||
|
six.text_type(search_opts))
|
||||||
|
|
||||||
project_id = context.project_id
|
project_id = context.project_id
|
||||||
|
|
||||||
model_evaluations = self.db.model_evaluation_get_all_by_project(context,
|
model_evaluations = self.db.model_evaluation_get_all_by_project(
|
||||||
project_id,
|
context,
|
||||||
sort_key=sort_key,
|
project_id,
|
||||||
sort_dir=sort_dir)
|
sort_key=sort_key,
|
||||||
|
sort_dir=sort_dir)
|
||||||
|
|
||||||
if search_opts:
|
if search_opts:
|
||||||
results = []
|
results = []
|
||||||
|
@ -474,22 +472,23 @@ class API(base.Base):
|
||||||
return rv
|
return rv
|
||||||
|
|
||||||
def create_model_evaluation(self, context, name, source_dataset_url,
|
def create_model_evaluation(self, context, name, source_dataset_url,
|
||||||
model_id, model_type, dataset_format, template_id,
|
model_id, model_type, dataset_format,
|
||||||
job_template_id, experiment_id, cluster_id,
|
template_id, job_template_id, experiment_id,
|
||||||
swift_tenant, swift_username, swift_password):
|
cluster_id, swift_tenant, swift_username,
|
||||||
|
swift_password):
|
||||||
"""Create a Model Evaluation"""
|
"""Create a Model Evaluation"""
|
||||||
policy.check_policy(context, 'model_evaluation', 'create')
|
policy.check_policy(context, 'model_evaluation', 'create')
|
||||||
|
|
||||||
model_evaluation = {'id': None,
|
model_evaluation = {'id': None,
|
||||||
'display_name': name,
|
'display_name': name,
|
||||||
'model_id': model_id,
|
'model_id': model_id,
|
||||||
'model_type': model_type,
|
'model_type': model_type,
|
||||||
'source_dataset_url': source_dataset_url,
|
'source_dataset_url': source_dataset_url,
|
||||||
'dataset_format': dataset_format,
|
'dataset_format': dataset_format,
|
||||||
'user_id': context.user_id,
|
'user_id': context.user_id,
|
||||||
'project_id': context.project_id,
|
'project_id': context.project_id,
|
||||||
'cluster_id': cluster_id
|
'cluster_id': cluster_id
|
||||||
}
|
}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result = self.db.model_evaluation_create(context, model_evaluation)
|
result = self.db.model_evaluation_create(context, model_evaluation)
|
||||||
|
@ -505,7 +504,8 @@ class API(base.Base):
|
||||||
result['id'],
|
result['id'],
|
||||||
updates)
|
updates)
|
||||||
|
|
||||||
LOG.info(_LI("Accepted creation of model evaluation %s."), result['id'])
|
LOG.info(_LI("Accepted creation of model evaluation %s."),
|
||||||
|
result['id'])
|
||||||
except Exception:
|
except Exception:
|
||||||
with excutils.save_and_reraise_exception():
|
with excutils.save_and_reraise_exception():
|
||||||
self.db.model_evaluation_delete(context, result['id'])
|
self.db.model_evaluation_delete(context, result['id'])
|
||||||
|
@ -530,10 +530,11 @@ class API(base.Base):
|
||||||
raise exception.InvalidLearning(reason=msg)
|
raise exception.InvalidLearning(reason=msg)
|
||||||
|
|
||||||
if model_evaluation.job_id:
|
if model_evaluation.job_id:
|
||||||
self.engine_rpcapi.delete_model_evaluation(context,
|
self.engine_rpcapi\
|
||||||
model_evaluation['cluster_id'],
|
.delete_model_evaluation(context,
|
||||||
model_evaluation['job_id'],
|
model_evaluation['cluster_id'],
|
||||||
id)
|
model_evaluation['job_id'],
|
||||||
|
id)
|
||||||
else:
|
else:
|
||||||
self.db.model_evaluation_delete(context, id)
|
self.db.model_evaluation_delete(context, id)
|
||||||
|
|
||||||
|
|
|
@ -18,15 +18,10 @@ Drivers for learnings.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import six
|
|
||||||
import time
|
|
||||||
|
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
from oslo_log import log
|
from oslo_log import log
|
||||||
|
|
||||||
from meteos import exception
|
|
||||||
from meteos.i18n import _, _LE
|
from meteos.i18n import _, _LE
|
||||||
from meteos import utils
|
|
||||||
|
|
||||||
LOG = log.getLogger(__name__)
|
LOG = log.getLogger(__name__)
|
||||||
|
|
||||||
|
|
|
@ -26,20 +26,16 @@ from oslo_concurrency import processutils
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
from oslo_log import log
|
from oslo_log import log
|
||||||
from oslo_utils import excutils
|
from oslo_utils import excutils
|
||||||
from oslo_utils import importutils
|
|
||||||
from oslo_utils import units
|
|
||||||
import retrying
|
|
||||||
import six
|
|
||||||
|
|
||||||
|
from meteos import cluster
|
||||||
from meteos.common import constants as const
|
from meteos.common import constants as const
|
||||||
from meteos import context
|
from meteos import context
|
||||||
from meteos import exception
|
from meteos import exception
|
||||||
from meteos.i18n import _, _LE, _LI, _LW
|
from meteos.i18n import _, _LE, _LI, _LW
|
||||||
from meteos.engine import driver
|
from meteos.engine import driver
|
||||||
from meteos import utils
|
from meteos import utils
|
||||||
from meteos import cluster
|
|
||||||
|
|
||||||
EXIT_CODE='80577372-9349-463a-bbc3-1ca54f187cc9'
|
EXIT_CODE = '80577372-9349-463a-bbc3-1ca54f187cc9'
|
||||||
LOG = log.getLogger(__name__)
|
LOG = log.getLogger(__name__)
|
||||||
|
|
||||||
learning_opts = [
|
learning_opts = [
|
||||||
|
@ -77,8 +73,9 @@ class GenericLearningDriver(driver.LearningDriver):
|
||||||
|
|
||||||
def __init__(self, *args, **kwargs):
|
def __init__(self, *args, **kwargs):
|
||||||
"""Do initialization."""
|
"""Do initialization."""
|
||||||
super(GenericLearningDriver, self).__init__(
|
super(GenericLearningDriver, self).__init__([False, True],
|
||||||
[False, True], *args, **kwargs)
|
*args,
|
||||||
|
**kwargs)
|
||||||
self.admin_context = context.get_admin_context()
|
self.admin_context = context.get_admin_context()
|
||||||
self.cluster_api = cluster.API()
|
self.cluster_api = cluster.API()
|
||||||
self.sshpool = None
|
self.sshpool = None
|
||||||
|
@ -162,7 +159,8 @@ class GenericLearningDriver(driver.LearningDriver):
|
||||||
job_args['method'] = method
|
job_args['method'] = method
|
||||||
|
|
||||||
# Set parameters of DataSet
|
# Set parameters of DataSet
|
||||||
job_args['source_dataset_url'] = request_specs.get('source_dataset_url')
|
job_args['source_dataset_url'] = request_specs\
|
||||||
|
.get('source_dataset_url')
|
||||||
job_args['dataset_format'] = request_specs.get('dataset_format')
|
job_args['dataset_format'] = request_specs.get('dataset_format')
|
||||||
dataset_args = {'params': request_specs.get('params')}
|
dataset_args = {'params': request_specs.get('params')}
|
||||||
job_args['dataset'] = dataset_args
|
job_args['dataset'] = dataset_args
|
||||||
|
@ -420,10 +418,10 @@ class GenericLearningDriver(driver.LearningDriver):
|
||||||
try:
|
try:
|
||||||
stdout, stderr = self._run_ssh(ip,
|
stdout, stderr = self._run_ssh(ip,
|
||||||
['netstat',
|
['netstat',
|
||||||
'-tnl',
|
'-tnl',
|
||||||
'|',
|
'|',
|
||||||
'grep',
|
'grep',
|
||||||
port])
|
port])
|
||||||
except processutils.ProcessExecutionError:
|
except processutils.ProcessExecutionError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
@ -463,12 +461,12 @@ class GenericLearningDriver(driver.LearningDriver):
|
||||||
ip = self._get_master_ip(context, request_specs['cluster_id'])
|
ip = self._get_master_ip(context, request_specs['cluster_id'])
|
||||||
port = request_specs['port']
|
port = request_specs['port']
|
||||||
|
|
||||||
return self._run_ssh(ip, ['echo',
|
return self._run_ssh(ip, ['echo',
|
||||||
base64.b64encode(EXIT_CODE),
|
base64.b64encode(EXIT_CODE),
|
||||||
'|',
|
'|',
|
||||||
'netcat',
|
'netcat',
|
||||||
'localhost',
|
'localhost',
|
||||||
port])
|
port])
|
||||||
|
|
||||||
self._wait_for_model_to_load(ip, port, unload=True)
|
self._wait_for_model_to_load(ip, port, unload=True)
|
||||||
|
|
||||||
|
@ -504,12 +502,12 @@ class GenericLearningDriver(driver.LearningDriver):
|
||||||
|
|
||||||
LOG.debug("Execute job with args: %s", learning_args)
|
LOG.debug("Execute job with args: %s", learning_args)
|
||||||
|
|
||||||
return self._run_ssh(ip, ['echo',
|
return self._run_ssh(ip, ['echo',
|
||||||
learning_args,
|
learning_args,
|
||||||
'|',
|
'|',
|
||||||
'netcat',
|
'netcat',
|
||||||
'localhost',
|
'localhost',
|
||||||
port])
|
port])
|
||||||
|
|
||||||
def delete_learning(self, context, cluster_id, job_id, id):
|
def delete_learning(self, context, cluster_id, job_id, id):
|
||||||
"""Delete Learning."""
|
"""Delete Learning."""
|
||||||
|
|
|
@ -20,29 +20,16 @@
|
||||||
:learning_driver: Used by :class:`LearningManager`.
|
:learning_driver: Used by :class:`LearningManager`.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import copy
|
|
||||||
import datetime
|
|
||||||
import functools
|
|
||||||
|
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
from oslo_log import log
|
from oslo_log import log
|
||||||
from oslo_serialization import jsonutils
|
|
||||||
from oslo_service import periodic_task
|
|
||||||
from oslo_utils import excutils
|
from oslo_utils import excutils
|
||||||
from oslo_utils import importutils
|
from oslo_utils import importutils
|
||||||
from oslo_utils import strutils
|
|
||||||
from oslo_utils import timeutils
|
from oslo_utils import timeutils
|
||||||
import six
|
|
||||||
|
|
||||||
from meteos.common import constants
|
from meteos.common import constants
|
||||||
from meteos import context
|
import meteos.engine.configuration
|
||||||
from meteos import exception
|
|
||||||
from meteos.i18n import _, _LE, _LI, _LW
|
from meteos.i18n import _, _LE, _LI, _LW
|
||||||
from meteos import manager
|
from meteos import manager
|
||||||
from meteos.engine import api
|
|
||||||
import meteos.engine.configuration
|
|
||||||
from meteos.engine import rpcapi as engine_rpcapi
|
|
||||||
from meteos import utils
|
|
||||||
|
|
||||||
LOG = log.getLogger(__name__)
|
LOG = log.getLogger(__name__)
|
||||||
|
|
||||||
|
@ -86,10 +73,10 @@ class LearningManager(manager.Manager):
|
||||||
|
|
||||||
if stderr:
|
if stderr:
|
||||||
status = constants.STATUS_ERROR
|
status = constants.STATUS_ERROR
|
||||||
LOG.error(_LI("Fail to create %s %s."), resource_name, id)
|
LOG.error(_LI("Fail to create %s."), id)
|
||||||
else:
|
else:
|
||||||
status = constants.STATUS_AVAILABLE
|
status = constants.STATUS_AVAILABLE
|
||||||
LOG.info(_LI("%s %s created successfully."), resource_name, id)
|
LOG.info(_LI("%s created successfully."), id)
|
||||||
|
|
||||||
updates = {
|
updates = {
|
||||||
'status': status,
|
'status': status,
|
||||||
|
@ -124,7 +111,7 @@ class LearningManager(manager.Manager):
|
||||||
response = self.driver.create_template(
|
response = self.driver.create_template(
|
||||||
context, request_spec)
|
context, request_spec)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
with excutils.save_and_reraise_exception():
|
with excutils.save_and_reraise_exception():
|
||||||
LOG.error(_LE("template %s failed on creation."),
|
LOG.error(_LE("template %s failed on creation."),
|
||||||
request_spec['id'])
|
request_spec['id'])
|
||||||
|
@ -150,7 +137,7 @@ class LearningManager(manager.Manager):
|
||||||
template = self.db.template_get(context, id)
|
template = self.db.template_get(context, id)
|
||||||
self.driver.delete_template(context, template)
|
self.driver.delete_template(context, template)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
with excutils.save_and_reraise_exception():
|
with excutils.save_and_reraise_exception():
|
||||||
LOG.error(_LE("Template %s failed on deletion."), id)
|
LOG.error(_LE("Template %s failed on deletion."), id)
|
||||||
self.db.template_update(
|
self.db.template_update(
|
||||||
|
@ -177,7 +164,7 @@ class LearningManager(manager.Manager):
|
||||||
|
|
||||||
experiment = request_spec
|
experiment = request_spec
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
with excutils.save_and_reraise_exception():
|
with excutils.save_and_reraise_exception():
|
||||||
LOG.error(_LE("Experiment %s failed on creation."),
|
LOG.error(_LE("Experiment %s failed on creation."),
|
||||||
request_spec['id'])
|
request_spec['id'])
|
||||||
|
@ -203,7 +190,7 @@ class LearningManager(manager.Manager):
|
||||||
experiment = self.db.experiment_get(context, id)
|
experiment = self.db.experiment_get(context, id)
|
||||||
self.driver.delete_experiment(context, experiment['cluster_id'])
|
self.driver.delete_experiment(context, experiment['cluster_id'])
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
with excutils.save_and_reraise_exception():
|
with excutils.save_and_reraise_exception():
|
||||||
LOG.error(_LE("Experiment %s failed on deletion."), id)
|
LOG.error(_LE("Experiment %s failed on deletion."), id)
|
||||||
self.db.experiment_update(
|
self.db.experiment_update(
|
||||||
|
@ -228,7 +215,7 @@ class LearningManager(manager.Manager):
|
||||||
request_spec['template_id'],
|
request_spec['template_id'],
|
||||||
request_spec['cluster_id'])
|
request_spec['cluster_id'])
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
with excutils.save_and_reraise_exception():
|
with excutils.save_and_reraise_exception():
|
||||||
LOG.error(_LE("Dataset %s failed on creation."),
|
LOG.error(_LE("Dataset %s failed on creation."),
|
||||||
request_spec['id'])
|
request_spec['id'])
|
||||||
|
@ -247,7 +234,7 @@ class LearningManager(manager.Manager):
|
||||||
try:
|
try:
|
||||||
self.driver.delete_dataset(context, cluster_id, job_id, id)
|
self.driver.delete_dataset(context, cluster_id, job_id, id)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
with excutils.save_and_reraise_exception():
|
with excutils.save_and_reraise_exception():
|
||||||
LOG.error(_LE("Dataset %s failed on deletion."), id)
|
LOG.error(_LE("Dataset %s failed on deletion."), id)
|
||||||
self.db.dataset_update(
|
self.db.dataset_update(
|
||||||
|
@ -272,7 +259,7 @@ class LearningManager(manager.Manager):
|
||||||
request_spec['template_id'],
|
request_spec['template_id'],
|
||||||
request_spec['cluster_id'])
|
request_spec['cluster_id'])
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
with excutils.save_and_reraise_exception():
|
with excutils.save_and_reraise_exception():
|
||||||
LOG.error(_LE("Model %s failed on creation."),
|
LOG.error(_LE("Model %s failed on creation."),
|
||||||
request_spec['id'])
|
request_spec['id'])
|
||||||
|
@ -291,7 +278,7 @@ class LearningManager(manager.Manager):
|
||||||
try:
|
try:
|
||||||
self.driver.delete_model(context, cluster_id, job_id, id)
|
self.driver.delete_model(context, cluster_id, job_id, id)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
with excutils.save_and_reraise_exception():
|
with excutils.save_and_reraise_exception():
|
||||||
LOG.error(_LE("Model %s failed on deletion."), id)
|
LOG.error(_LE("Model %s failed on deletion."), id)
|
||||||
self.db.model_update(
|
self.db.model_update(
|
||||||
|
@ -314,7 +301,7 @@ class LearningManager(manager.Manager):
|
||||||
try:
|
try:
|
||||||
self.driver.load_model(context, request_spec)
|
self.driver.load_model(context, request_spec)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
with excutils.save_and_reraise_exception():
|
with excutils.save_and_reraise_exception():
|
||||||
LOG.error(_LE("Model %s failed on loading."),
|
LOG.error(_LE("Model %s failed on loading."),
|
||||||
request_spec['id'])
|
request_spec['id'])
|
||||||
|
@ -325,7 +312,7 @@ class LearningManager(manager.Manager):
|
||||||
|
|
||||||
self.db.model_update(context,
|
self.db.model_update(context,
|
||||||
request_spec['id'],
|
request_spec['id'],
|
||||||
{'status' : constants.STATUS_ACTIVE})
|
{'status': constants.STATUS_ACTIVE})
|
||||||
|
|
||||||
def unload_model(self, context, request_spec=None):
|
def unload_model(self, context, request_spec=None):
|
||||||
"""Unload a Model."""
|
"""Unload a Model."""
|
||||||
|
@ -338,7 +325,7 @@ class LearningManager(manager.Manager):
|
||||||
try:
|
try:
|
||||||
self.driver.unload_model(context, request_spec)
|
self.driver.unload_model(context, request_spec)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
with excutils.save_and_reraise_exception():
|
with excutils.save_and_reraise_exception():
|
||||||
LOG.error(_LE("Model %s failed on unloading."),
|
LOG.error(_LE("Model %s failed on unloading."),
|
||||||
request_spec['id'])
|
request_spec['id'])
|
||||||
|
@ -349,7 +336,7 @@ class LearningManager(manager.Manager):
|
||||||
|
|
||||||
self.db.model_update(context,
|
self.db.model_update(context,
|
||||||
request_spec['id'],
|
request_spec['id'],
|
||||||
{'status' : constants.STATUS_AVAILABLE})
|
{'status': constants.STATUS_AVAILABLE})
|
||||||
|
|
||||||
def create_model_evaluation(self, context, request_spec=None):
|
def create_model_evaluation(self, context, request_spec=None):
|
||||||
"""Create a Model Evaluation."""
|
"""Create a Model Evaluation."""
|
||||||
|
@ -366,7 +353,7 @@ class LearningManager(manager.Manager):
|
||||||
request_spec['template_id'],
|
request_spec['template_id'],
|
||||||
request_spec['cluster_id'])
|
request_spec['cluster_id'])
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
with excutils.save_and_reraise_exception():
|
with excutils.save_and_reraise_exception():
|
||||||
LOG.error(_LE("Model Evaluation %s failed on creation."),
|
LOG.error(_LE("Model Evaluation %s failed on creation."),
|
||||||
model_evaluation_id)
|
model_evaluation_id)
|
||||||
|
@ -378,14 +365,18 @@ class LearningManager(manager.Manager):
|
||||||
self._update_status(context, 'Model Evaluation', model_evaluation_id,
|
self._update_status(context, 'Model Evaluation', model_evaluation_id,
|
||||||
job_id, stdout, stderr)
|
job_id, stdout, stderr)
|
||||||
|
|
||||||
def delete_model_evaluation(self, context, cluster_id=None, job_id=None, id=None):
|
def delete_model_evaluation(self, context, cluster_id=None,
|
||||||
|
job_id=None, id=None):
|
||||||
"""Deletes a Model Evaluation."""
|
"""Deletes a Model Evaluation."""
|
||||||
context = context.elevated()
|
context = context.elevated()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
self.driver.delete_model_evaluation(context, cluster_id, job_id, id)
|
self.driver.delete_model_evaluation(context,
|
||||||
|
cluster_id,
|
||||||
|
job_id,
|
||||||
|
id)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
with excutils.save_and_reraise_exception():
|
with excutils.save_and_reraise_exception():
|
||||||
LOG.error(_LE("Model Evaluation %s failed on deletion."), id)
|
LOG.error(_LE("Model Evaluation %s failed on deletion."), id)
|
||||||
self.db.model_evaluation_update(
|
self.db.model_evaluation_update(
|
||||||
|
@ -411,7 +402,7 @@ class LearningManager(manager.Manager):
|
||||||
request_spec['template_id'],
|
request_spec['template_id'],
|
||||||
request_spec['cluster_id'])
|
request_spec['cluster_id'])
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
with excutils.save_and_reraise_exception():
|
with excutils.save_and_reraise_exception():
|
||||||
LOG.error(_LE("Learning %s failed on creation."),
|
LOG.error(_LE("Learning %s failed on creation."),
|
||||||
learning_id)
|
learning_id)
|
||||||
|
@ -432,9 +423,10 @@ class LearningManager(manager.Manager):
|
||||||
LOG.debug("Create learning with request: %s", request_spec)
|
LOG.debug("Create learning with request: %s", request_spec)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
stdout, stderr = self.driver.create_online_learning(context, request_spec)
|
stdout, stderr = self.driver.create_online_learning(context,
|
||||||
|
request_spec)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
with excutils.save_and_reraise_exception():
|
with excutils.save_and_reraise_exception():
|
||||||
LOG.error(_LE("Learning %s failed on creation."),
|
LOG.error(_LE("Learning %s failed on creation."),
|
||||||
request_spec['id'])
|
request_spec['id'])
|
||||||
|
@ -453,7 +445,7 @@ class LearningManager(manager.Manager):
|
||||||
try:
|
try:
|
||||||
self.driver.delete_learning(context, cluster_id, job_id, id)
|
self.driver.delete_learning(context, cluster_id, job_id, id)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
with excutils.save_and_reraise_exception():
|
with excutils.save_and_reraise_exception():
|
||||||
LOG.error(_LE("Learning %s failed on deletion."), id)
|
LOG.error(_LE("Learning %s failed on deletion."), id)
|
||||||
self.db.learning_update(
|
self.db.learning_update(
|
||||||
|
|
|
@ -171,11 +171,6 @@ class MalformedRequestBody(MeteosException):
|
||||||
message = _("Malformed message body: %(reason)s.")
|
message = _("Malformed message body: %(reason)s.")
|
||||||
|
|
||||||
|
|
||||||
class NotAuthorized(MeteosException):
|
|
||||||
message = _("Not authorized.")
|
|
||||||
code = 403
|
|
||||||
|
|
||||||
|
|
||||||
class AdminRequired(NotAuthorized):
|
class AdminRequired(NotAuthorized):
|
||||||
message = _("User does not have admin privileges.")
|
message = _("User does not have admin privileges.")
|
||||||
|
|
||||||
|
|
|
@ -26,16 +26,16 @@ import oslo_policy.opts
|
||||||
|
|
||||||
import meteos.api.common
|
import meteos.api.common
|
||||||
import meteos.api.middleware.auth
|
import meteos.api.middleware.auth
|
||||||
|
import meteos.cluster.sahara
|
||||||
import meteos.common.config
|
import meteos.common.config
|
||||||
import meteos.db.api
|
import meteos.db.api
|
||||||
import meteos.db.base
|
import meteos.db.base
|
||||||
import meteos.exception
|
|
||||||
import meteos.service
|
|
||||||
import meteos.engine.api
|
import meteos.engine.api
|
||||||
import meteos.engine.driver
|
import meteos.engine.driver
|
||||||
import meteos.engine.drivers.generic
|
import meteos.engine.drivers.generic
|
||||||
import meteos.engine.manager
|
import meteos.engine.manager
|
||||||
import meteos.cluster.sahara
|
import meteos.exception
|
||||||
|
import meteos.service
|
||||||
import meteos.wsgi
|
import meteos.wsgi
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -225,7 +225,6 @@ class Service(service.Service):
|
||||||
def report_state(self):
|
def report_state(self):
|
||||||
"""Update the state of this service in the datastore."""
|
"""Update the state of this service in the datastore."""
|
||||||
ctxt = context.get_admin_context()
|
ctxt = context.get_admin_context()
|
||||||
zone = CONF.storage_availability_zone
|
|
||||||
state_catalog = {}
|
state_catalog = {}
|
||||||
try:
|
try:
|
||||||
try:
|
try:
|
||||||
|
|
|
@ -17,31 +17,22 @@
|
||||||
|
|
||||||
"""Utilities and helper functions."""
|
"""Utilities and helper functions."""
|
||||||
|
|
||||||
import contextlib
|
|
||||||
import errno
|
import errno
|
||||||
import functools
|
import functools
|
||||||
import inspect
|
import inspect
|
||||||
import os
|
import os
|
||||||
import pyclbr
|
import pyclbr
|
||||||
import random
|
|
||||||
import re
|
|
||||||
import shutil
|
|
||||||
import socket
|
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
|
||||||
import time
|
import time
|
||||||
|
|
||||||
from eventlet import pools
|
from eventlet import pools
|
||||||
import netaddr
|
|
||||||
from oslo_concurrency import lockutils
|
from oslo_concurrency import lockutils
|
||||||
from oslo_concurrency import processutils
|
from oslo_concurrency import processutils
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
from oslo_log import log
|
from oslo_log import log
|
||||||
from oslo_utils import importutils
|
from oslo_utils import importutils
|
||||||
from oslo_utils import netutils
|
|
||||||
from oslo_utils import timeutils
|
from oslo_utils import timeutils
|
||||||
import paramiko
|
import paramiko
|
||||||
import retrying
|
|
||||||
import six
|
import six
|
||||||
|
|
||||||
from meteos.common import constants
|
from meteos.common import constants
|
||||||
|
@ -250,7 +241,6 @@ def file_open(filename):
|
||||||
except IOError as e:
|
except IOError as e:
|
||||||
if e.errno != errno.ENOENT:
|
if e.errno != errno.ENOENT:
|
||||||
raise
|
raise
|
||||||
result = False
|
|
||||||
else:
|
else:
|
||||||
data = fd.read()
|
data = fd.read()
|
||||||
fd.close()
|
fd.close()
|
||||||
|
|
39
tox.ini
39
tox.ini
|
@ -34,24 +34,7 @@ basepython = python3.4
|
||||||
commands = oslo_debug_helper {posargs}
|
commands = oslo_debug_helper {posargs}
|
||||||
|
|
||||||
[testenv:pep8]
|
[testenv:pep8]
|
||||||
commands =
|
commands = flake8 {posargs}
|
||||||
flake8 {posargs}
|
|
||||||
# Run bashate during pep8 runs to ensure violations are caught by
|
|
||||||
# the check and gate queues.
|
|
||||||
bashate -i E006,E042,E043 \
|
|
||||||
tools/enable-pre-commit-hook.sh \
|
|
||||||
contrib/ci/pre_test_hook.sh \
|
|
||||||
contrib/ci/post_test_hook.sh \
|
|
||||||
devstack/plugin.sh \
|
|
||||||
devstack/upgrade/from-mitaka/upgrade-meteos \
|
|
||||||
devstack/upgrade/resources.sh \
|
|
||||||
devstack/upgrade/shutdown.sh \
|
|
||||||
devstack/upgrade/upgrade.sh \
|
|
||||||
tools/cover.sh \
|
|
||||||
tools/check_logging.sh \
|
|
||||||
run_tests.sh
|
|
||||||
{toxinidir}/tools/check_exec.py {toxinidir}/meteos
|
|
||||||
{toxinidir}/tools/check_logging.sh {toxinidir}/meteos
|
|
||||||
|
|
||||||
[testenv:genconfig]
|
[testenv:genconfig]
|
||||||
whitelist_externals = bash
|
whitelist_externals = bash
|
||||||
|
@ -64,24 +47,6 @@ commands = {posargs}
|
||||||
[testenv:docs]
|
[testenv:docs]
|
||||||
commands = python setup.py build_sphinx
|
commands = python setup.py build_sphinx
|
||||||
|
|
||||||
[testenv:cover]
|
|
||||||
commands = {toxinidir}/tools/cover.sh {posargs}
|
|
||||||
|
|
||||||
[testenv:fast8]
|
|
||||||
commands =
|
|
||||||
{toxinidir}/tools/fast8.sh
|
|
||||||
|
|
||||||
[testenv:pylint]
|
|
||||||
deps = -r{toxinidir}/requirements.txt
|
|
||||||
pylint==0.26.0
|
|
||||||
whitelist_externals = bash
|
|
||||||
commands = bash tools/lintstack.sh
|
|
||||||
|
|
||||||
[testenv:lint]
|
|
||||||
deps = -r{toxinidir}/requirements.txt
|
|
||||||
pylint==0.26.0
|
|
||||||
commands = python tools/lintstack.py check
|
|
||||||
|
|
||||||
[testenv:api-ref]
|
[testenv:api-ref]
|
||||||
# This environment is called from CI scripts to test and publish
|
# This environment is called from CI scripts to test and publish
|
||||||
# the API Ref to developer.openstack.org.
|
# the API Ref to developer.openstack.org.
|
||||||
|
@ -96,7 +61,7 @@ commands = alembic -c meteos/db/migrations/alembic.ini revision -m ""{posargs}
|
||||||
|
|
||||||
[flake8]
|
[flake8]
|
||||||
# Following checks are ignored on purpose:
|
# Following checks are ignored on purpose:
|
||||||
ignore =
|
ignore = E127,E128,E501,F401
|
||||||
builtins = _
|
builtins = _
|
||||||
exclude = .git,.tox,.testrepository,.venv,build,cover,dist,doc,*egg,api-ref/build,*/source/conf.py
|
exclude = .git,.tox,.testrepository,.venv,build,cover,dist,doc,*egg,api-ref/build,*/source/conf.py
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue