diff --git a/.gitignore b/.gitignore index b57cb6fdf..20cc9750b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,11 +22,12 @@ lib64 pip-log.txt # Unit test / coverage reports -.coverage +.coverage* .tox nosetests.xml .testrepository .venv +.idea # Translations *.mo @@ -57,6 +58,6 @@ sftp-config.json /cover/ .settings/ .eclipse -.project -.pydevproject +cover +/demo/ diff --git a/README.rst b/README.rst index 7969c2269..39113daad 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,6 @@ -=============================== -watcher -=============================== +======= +Watcher +======= Watcher takes advantage of CEP and ML algorithms/metaheuristics to improve physical resources usage through better VM placement. Watcher can improve your cloud optimization by reducing energy footprint and increasing profits. @@ -8,4 +8,4 @@ Watcher takes advantage of CEP and ML algorithms/metaheuristics to improve physi * Wiki: http://wiki.openstack.org/wiki/Watcher * Source: http://git.openstack.org/cgit/stackforge/watcher * Bugs: http://bugs.launchpad.net/watcher - +* Documentation: http://factory.b-com.com/www/watcher/watcher/doc/build/html/ diff --git a/setup.cfg b/setup.cfg index b19c850be..855cb0a3d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [metadata] -name = watcher +name = python-watcher summary = Watcher takes advantage of CEP and ML algorithms/metaheuristics to improve physical resources usage through better VM placement. Watcher can improve your cloud optimization by reducing energy footprint and increasing profits. description-file = README.rst @@ -23,11 +23,13 @@ classifier = [files] packages = watcher +data_files = + etc/ = etc/* [global] setup-hooks = pbr.hooks.setup_hook - + [entry_points] oslo.config.opts = watcher = watcher.opts:list_opts @@ -41,6 +43,12 @@ console_scripts = watcher.database.migration_backend = sqlalchemy = watcher.db.sqlalchemy.migration +watcher_strategies = + basic = watcher.decision_engine.strategies.basic_consolidation:BasicConsolidation + +watcher_metrics_collector = + influxdb = watcher.metrics_engine.framework.datasources.influxdb_collector:InfluxDBCollector + [build_sphinx] source-dir = doc/source build-dir = doc/build @@ -63,4 +71,3 @@ input_file = watcher/locale/watcher.pot keywords = _ gettext ngettext l_ lazy_gettext mapping_file = babel.cfg output_file = watcher/locale/watcher.pot - diff --git a/tox.ini b/tox.ini index b4c430aad..e1ff1b5ef 100644 --- a/tox.ini +++ b/tox.ini @@ -19,7 +19,7 @@ commands = flake8 commands = {posargs} [testenv:cover] -commands = python setup.py testr --coverage --testr-args='{posargs}' +commands = python setup.py testr --coverage --omit="watcher/tests/*,watcher/openstack/*" --testr-args='{posargs}' [testenv:docs] commands = python setup.py build_sphinx @@ -33,7 +33,7 @@ commands = oslo-config-generator --namespace watcher \ --namespace keystonemiddleware.auth_token \ --namespace oslo.db \ - --output-file etc/watcher/watcher.conf + --output-file etc/watcher/watcher.conf.sample [flake8] # E123, E125 skipped as they are invalid PEP-8. @@ -41,4 +41,12 @@ commands = show-source=True ignore=E123,E125,H404,H405,H305 builtins= _ -exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build,*sqlalchemy/alembic/versions/* +exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build,*sqlalchemy/alembic/versions/*,demo/ + +[testenv:pypi] +commands = + python setup.py sdist bdist_wheel + twine upload --config-file .pypirc {posargs} dist/* + +[testenv:wheel] +commands = python setup.py bdist_wheel diff --git a/watcher/__init__.py b/watcher/__init__.py index 604b37029..a920bfb1f 100644 --- a/watcher/__init__.py +++ b/watcher/__init__.py @@ -15,5 +15,4 @@ import pbr.version -__version__ = pbr.version.VersionInfo( - 'watcher').version_string() +__version__ = pbr.version.VersionInfo('python-watcher').version_string() diff --git a/watcher/api/controllers/v1/__init__.py b/watcher/api/controllers/v1/__init__.py index aeb85668c..8f752ec81 100644 --- a/watcher/api/controllers/v1/__init__.py +++ b/watcher/api/controllers/v1/__init__.py @@ -33,6 +33,7 @@ from watcher.api.controllers.v1 import action from watcher.api.controllers.v1 import action_plan from watcher.api.controllers.v1 import audit from watcher.api.controllers.v1 import audit_template +from watcher.api.controllers.v1 import goal class APIBase(wtypes.Base): @@ -155,6 +156,7 @@ class Controller(rest.RestController): audit_templates = audit_template.AuditTemplatesController() actions = action.ActionsController() action_plans = action_plan.ActionPlansController() + goals = goal.GoalsController() @wsme_pecan.wsexpose(V1) def get(self): diff --git a/watcher/api/controllers/v1/audit.py b/watcher/api/controllers/v1/audit.py index 0f964141f..2e2a17b4b 100644 --- a/watcher/api/controllers/v1/audit.py +++ b/watcher/api/controllers/v1/audit.py @@ -48,25 +48,44 @@ class Audit(base.APIBase): between the internal object model and the API representation of a audit. """ _audit_template_uuid = None + _audit_template_name = None + + def _get_audit_template(self, value): + if value == wtypes.Unset: + return None + audit_template = None + try: + if utils.is_uuid_like(value) or utils.is_int_like(value): + audit_template = objects.AuditTemplate.get( + pecan.request.context, value) + else: + audit_template = objects.AuditTemplate.get_by_name( + pecan.request.context, value) + except exception.AuditTemplateNotFound: + pass + if audit_template: + self.audit_template_id = audit_template.id + return audit_template def _get_audit_template_uuid(self): return self._audit_template_uuid def _set_audit_template_uuid(self, value): - if value == wtypes.Unset: - self._audit_template_uuid = wtypes.Unset - elif value and self._audit_template_uuid != value: - try: - if utils.is_uuid_like(value) or utils.is_int_like(value): - audit_template = objects.AuditTemplate.get( - pecan.request.context, value) - else: - audit_template = objects.AuditTemplate.get_by_name( - pecan.request.context, value) + if value and self._audit_template_uuid != value: + self._audit_template_uuid = None + audit_template = self._get_audit_template(value) + if audit_template: self._audit_template_uuid = audit_template.uuid - self.audit_template_id = audit_template.id - except exception.AuditTemplateNotFound: - self._audit_template_uuid = None + + def _get_audit_template_name(self): + return self._audit_template_name + + def _set_audit_template_name(self, value): + if value and self._audit_template_name != value: + self._audit_template_name = None + audit_template = self._get_audit_template(value) + if audit_template: + self._audit_template_name = audit_template.name uuid = types.uuid """Unique UUID for this audit""" @@ -84,7 +103,13 @@ class Audit(base.APIBase): _get_audit_template_uuid, _set_audit_template_uuid, mandatory=True) - """The UUID of the node this port belongs to""" + """The UUID of the audit template this audit refers to""" + + audit_template_name = wsme.wsproperty(wtypes.text, + _get_audit_template_name, + _set_audit_template_name, + mandatory=False) + """The name of the audit template this audit refers to""" links = wsme.wsattr([link.Link], readonly=True) """A list containing a self link and associated audit links""" @@ -92,9 +117,7 @@ class Audit(base.APIBase): def __init__(self, **kwargs): self.fields = [] fields = list(objects.Audit.fields) - # audit_template_uuid is not part of objects.Audit.fields - # because it's an API-only attribute. - fields.append('audit_template_uuid') + for k in fields: # Skip fields we do not expose. if not hasattr(self, k): @@ -103,14 +126,22 @@ class Audit(base.APIBase): setattr(self, k, kwargs.get(k, wtypes.Unset)) self.fields.append('audit_template_id') + + # audit_template_uuid & audit_template_name are not part of + # objects.Audit.fields because they're API-only attributes. + fields.append('audit_template_uuid') setattr(self, 'audit_template_uuid', kwargs.get('audit_template_id', wtypes.Unset)) + fields.append('audit_template_name') + setattr(self, 'audit_template_name', kwargs.get('audit_template_id', + wtypes.Unset)) @staticmethod def _convert_with_links(audit, url, expand=True): if not expand: audit.unset_fields_except(['uuid', 'type', 'deadline', - 'state', 'audit_template_uuid']) + 'state', 'audit_template_uuid', + 'audit_template_name']) # The numeric ID should not be exposed to # the user, it's internal only. @@ -237,7 +268,7 @@ class AuditsController(rest.RestController): :param limit: maximum number of resources to return in a single result. :param sort_key: column to sort results by. Default: id. :param sort_dir: direction to sort. "asc" or "desc". Default: asc. - :param audit_template: Optional UUID or description of an audit + :param audit_template: Optional UUID or name of an audit template, to get only audits for that audit template. """ return self._get_audits_collection(marker, limit, sort_key, diff --git a/watcher/api/controllers/v1/goal.py b/watcher/api/controllers/v1/goal.py new file mode 100644 index 000000000..3f9559a53 --- /dev/null +++ b/watcher/api/controllers/v1/goal.py @@ -0,0 +1,208 @@ +# -*- encoding: utf-8 -*- +# Copyright 2013 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +import pecan +from pecan import rest +import wsme +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from watcher.api.controllers import base +from watcher.api.controllers import link +from watcher.api.controllers.v1 import collection +from watcher.api.controllers.v1 import types +from watcher.api.controllers.v1 import utils as api_utils +from watcher.common import exception + +CONF = cfg.CONF + + +class Goal(base.APIBase): + """API representation of a action. + + This class enforces type checking and value constraints, and converts + between the internal object model and the API representation of a action. + """ + + name = wtypes.text + """Name of the goal""" + + strategy = wtypes.text + """The strategy associated with the goal""" + + uuid = types.uuid + """Unused field""" + + links = wsme.wsattr([link.Link], readonly=True) + """A list containing a self link and associated action links""" + + def __init__(self, **kwargs): + super(Goal, self).__init__() + + self.fields = [] + self.fields.append('name') + self.fields.append('strategy') + setattr(self, 'name', kwargs.get('name', + wtypes.Unset)) + setattr(self, 'strategy', kwargs.get('strategy', + wtypes.Unset)) + + @staticmethod + def _convert_with_links(goal, url, expand=True): + if not expand: + goal.unset_fields_except(['name', 'strategy']) + + goal.links = [link.Link.make_link('self', url, + 'goals', goal.name), + link.Link.make_link('bookmark', url, + 'goals', goal.name, + bookmark=True)] + return goal + + @classmethod + def convert_with_links(cls, goal, expand=True): + goal = Goal(**goal) + return cls._convert_with_links(goal, pecan.request.host_url, expand) + + @classmethod + def sample(cls, expand=True): + sample = cls(name='27e3153e-d5bf-4b7e-b517-fb518e17f34c', + strategy='action description') + return cls._convert_with_links(sample, 'http://localhost:9322', expand) + + +class GoalCollection(collection.Collection): + """API representation of a collection of goals.""" + + goals = [Goal] + """A list containing goals objects""" + + def __init__(self, **kwargs): + self._type = 'goals' + + @staticmethod + def convert_with_links(goals, limit, url=None, expand=False, + **kwargs): + + collection = GoalCollection() + collection.goals = [Goal.convert_with_links(g, expand) for g in goals] + + if 'sort_key' in kwargs: + reverse = False + if kwargs['sort_key'] == 'strategy': + if 'sort_dir' in kwargs: + reverse = True if kwargs['sort_dir'] == 'desc' else False + collection.goals = sorted( + collection.goals, + key=lambda goal: goal.name, + reverse=reverse) + + collection.next = collection.get_next(limit, url=url, **kwargs) + return collection + + @classmethod + def sample(cls): + sample = cls() + sample.actions = [Goal.sample(expand=False)] + return sample + + +class GoalsController(rest.RestController): + """REST controller for Goals.""" + def __init__(self): + super(GoalsController, self).__init__() + + from_goals = False + """A flag to indicate if the requests to this controller are coming + from the top-level resource Goals.""" + + _custom_actions = { + 'detail': ['GET'], + } + + def _get_goals_collection(self, limit, + sort_key, sort_dir, expand=False, + resource_url=None, goal_name=None): + + limit = api_utils.validate_limit(limit) + sort_dir = api_utils.validate_sort_dir(sort_dir) + + goals = [] + + if not goal_name and goal_name in CONF.watcher_goals.goals.keys(): + goals.append({'name': goal_name, 'strategy': goals[goal_name]}) + else: + for name, strategy in CONF.watcher_goals.goals.items(): + goals.append({'name': name, 'strategy': strategy}) + + return GoalCollection.convert_with_links(goals[:limit], limit, + url=resource_url, + expand=expand, + sort_key=sort_key, + sort_dir=sort_dir) + + @wsme_pecan.wsexpose(GoalCollection, int, wtypes.text, wtypes.text) + def get_all(self, limit=None, + sort_key='name', sort_dir='asc'): + """Retrieve a list of goals. + + :param limit: maximum number of resources to return in a single result. + :param sort_key: column to sort results by. Default: id. + :param sort_dir: direction to sort. "asc" or "desc". Default: asc. + to get only actions for that goal. + """ + return self._get_goals_collection(limit, sort_key, sort_dir) + + @wsme_pecan.wsexpose(GoalCollection, wtypes.text, int, + wtypes.text, wtypes.text) + def detail(self, goal_name=None, limit=None, + sort_key='name', sort_dir='asc'): + """Retrieve a list of actions with detail. + + :param goal_name: name of a goal, to get only goals for that + action. + :param limit: maximum number of resources to return in a single result. + :param sort_key: column to sort results by. Default: id. + :param sort_dir: direction to sort. "asc" or "desc". Default: asc. + to get only goals for that goal. + """ + # NOTE(lucasagomes): /detail should only work agaist collections + parent = pecan.request.path.split('/')[:-1][-1] + if parent != "goals": + raise exception.HTTPNotFound + expand = True + resource_url = '/'.join(['goals', 'detail']) + return self._get_goals_collection(limit, sort_key, sort_dir, + expand, resource_url, goal_name) + + @wsme_pecan.wsexpose(Goal, wtypes.text) + def get_one(self, goal_name): + """Retrieve information about the given goal. + + :param goal_name: name of the goal. + """ + if self.from_goals: + raise exception.OperationNotPermitted + + goals = CONF.watcher_goals.goals + goal = {} + if goal_name in goals.keys(): + goal = {'name': goal_name, 'strategy': goals[goal_name]} + + return Goal.convert_with_links(goal) diff --git a/watcher/applier/api/applier.py b/watcher/applier/api/applier.py index d7a07e631..381e96758 100644 --- a/watcher/applier/api/applier.py +++ b/watcher/applier/api/applier.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/watcher/applier/api/command_mapper.py b/watcher/applier/api/command_mapper.py index a457259c2..c536f2e7c 100644 --- a/watcher/applier/api/command_mapper.py +++ b/watcher/applier/api/command_mapper.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/watcher/applier/api/messaging/applier_command.py b/watcher/applier/api/messaging/applier_command.py index 3b4e088e0..93288d249 100644 --- a/watcher/applier/api/messaging/applier_command.py +++ b/watcher/applier/api/messaging/applier_command.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/watcher/applier/api/primitive_command.py b/watcher/applier/api/primitive_command.py index 94a69a65d..d8bdf91a3 100644 --- a/watcher/applier/api/primitive_command.py +++ b/watcher/applier/api/primitive_command.py @@ -1,11 +1,13 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# from watcher.applier.api.promise import Promise diff --git a/watcher/applier/api/promise.py b/watcher/applier/api/promise.py index 3ba1c279d..dbede895e 100644 --- a/watcher/applier/api/promise.py +++ b/watcher/applier/api/promise.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/watcher/applier/framework/command/hypervisor_state_command.py b/watcher/applier/framework/command/hypervisor_state_command.py index 47485d6e3..726eb4187 100644 --- a/watcher/applier/framework/command/hypervisor_state_command.py +++ b/watcher/applier/framework/command/hypervisor_state_command.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/watcher/applier/framework/command/migrate_command.py b/watcher/applier/framework/command/migrate_command.py index fe506559e..1fdb04cae 100644 --- a/watcher/applier/framework/command/migrate_command.py +++ b/watcher/applier/framework/command/migrate_command.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/watcher/applier/framework/command/nop_command.py b/watcher/applier/framework/command/nop_command.py index 6ab3c1794..4f3d9ee98 100644 --- a/watcher/applier/framework/command/nop_command.py +++ b/watcher/applier/framework/command/nop_command.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,6 +16,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # + from watcher.applier.api.primitive_command import PrimitiveCommand from watcher.applier.api.promise import Promise diff --git a/watcher/applier/framework/command/power_state_command.py b/watcher/applier/framework/command/power_state_command.py index ee5bfb394..d88b75621 100644 --- a/watcher/applier/framework/command/power_state_command.py +++ b/watcher/applier/framework/command/power_state_command.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/watcher/applier/framework/command/wrapper/nova_wrapper.py b/watcher/applier/framework/command/wrapper/nova_wrapper.py index 9c66ecb44..c4d1f14a4 100644 --- a/watcher/applier/framework/command/wrapper/nova_wrapper.py +++ b/watcher/applier/framework/command/wrapper/nova_wrapper.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -568,7 +570,7 @@ class NovaWrapper(object): def create_instance(self, hypervisor_id, inst_name="test", image_id=None, flavor_name="m1.tiny", sec_group_list=["default"], - network_names_list=["private"], keypair_name="mykeys", + network_names_list=["demo-net"], keypair_name="mykeys", create_new_floating_ip=True, block_device_mapping_v2=None): """This method creates a new instance. @@ -623,15 +625,16 @@ class NovaWrapper(object): return net_obj = {"net-id": nic_id} net_list.append(net_obj) - s = self.nova.servers - instance = s.create(inst_name, - image, flavor=flavor, - key_name=keypair_name, - security_groups=sec_group_list, - nics=net_list, - block_device_mapping_v2=block_device_mapping_v2, - availability_zone="nova:" + - hypervisor_id) + + instance = self.nova.servers. \ + create(inst_name, + image, flavor=flavor, + key_name=keypair_name, + security_groups=sec_group_list, + nics=net_list, + block_device_mapping_v2=block_device_mapping_v2, + availability_zone="nova:" + + hypervisor_id) # Poll at 5 second intervals, until the status is no longer 'BUILD' if instance: diff --git a/watcher/applier/framework/command_executor.py b/watcher/applier/framework/command_executor.py index 60e5ab811..4133999fa 100644 --- a/watcher/applier/framework/command_executor.py +++ b/watcher/applier/framework/command_executor.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/watcher/applier/framework/default_applier.py b/watcher/applier/framework/default_applier.py index 0aaa2646d..cb4c0baef 100644 --- a/watcher/applier/framework/default_applier.py +++ b/watcher/applier/framework/default_applier.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# from watcher.applier.api.applier import Applier from watcher.applier.framework.command_executor import CommandExecutor from watcher.objects import Action diff --git a/watcher/applier/framework/default_command_mapper.py b/watcher/applier/framework/default_command_mapper.py index e5c1ec178..87b68c182 100644 --- a/watcher/applier/framework/default_command_mapper.py +++ b/watcher/applier/framework/default_command_mapper.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# from watcher.applier.api.command_mapper import CommandMapper diff --git a/watcher/applier/framework/deploy_phase.py b/watcher/applier/framework/deploy_phase.py index 1feeae14f..faf4d86ae 100644 --- a/watcher/applier/framework/deploy_phase.py +++ b/watcher/applier/framework/deploy_phase.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# from watcher.openstack.common import log LOG = log.getLogger(__name__) diff --git a/watcher/applier/framework/manager_applier.py b/watcher/applier/framework/manager_applier.py index 8ba878278..23da68353 100644 --- a/watcher/applier/framework/manager_applier.py +++ b/watcher/applier/framework/manager_applier.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# from concurrent.futures import ThreadPoolExecutor from oslo_config import cfg diff --git a/watcher/applier/framework/messaging/events.py b/watcher/applier/framework/messaging/events.py index 9a1696874..2e186bf21 100644 --- a/watcher/applier/framework/messaging/events.py +++ b/watcher/applier/framework/messaging/events.py @@ -1,11 +1,13 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# from enum import Enum diff --git a/watcher/applier/framework/messaging/launch_action_plan.py b/watcher/applier/framework/messaging/launch_action_plan.py index 8c37698e5..06f99d186 100644 --- a/watcher/applier/framework/messaging/launch_action_plan.py +++ b/watcher/applier/framework/messaging/launch_action_plan.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/watcher/applier/framework/messaging/trigger_action_plan.py b/watcher/applier/framework/messaging/trigger_action_plan.py index e2a23b855..fc53b719d 100644 --- a/watcher/applier/framework/messaging/trigger_action_plan.py +++ b/watcher/applier/framework/messaging/trigger_action_plan.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/watcher/applier/framework/rpcapi.py b/watcher/applier/framework/rpcapi.py index ee6753c33..2d4672931 100644 --- a/watcher/applier/framework/rpcapi.py +++ b/watcher/applier/framework/rpcapi.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/watcher/cmd/decisionengine.py b/watcher/cmd/decisionengine.py index d28575481..7f6d70300 100644 --- a/watcher/cmd/decisionengine.py +++ b/watcher/cmd/decisionengine.py @@ -27,7 +27,10 @@ from watcher.decision_engine.framework.manager_decision_engine import \ DecisionEngineManager from watcher.openstack.common._i18n import _LI from watcher.openstack.common import log as logging - +cfg.CONF.import_opt('hostname', + 'watcher.metrics_engine.framework.' + 'datasources.influxdb_collector', + group='watcher_influxdb_collector') LOG = logging.getLogger(__name__) diff --git a/watcher/common/config.py b/watcher/common/config.py index ff3ea3d9c..c88a4ab87 100644 --- a/watcher/common/config.py +++ b/watcher/common/config.py @@ -24,7 +24,7 @@ from watcher import version def parse_args(argv, default_config_files=None): rpc.set_defaults(control_exchange='watcher') cfg.CONF(argv[1:], - project='watcher', + project='python-watcher', version=version.version_info.release_string(), default_config_files=default_config_files) rpc.init(cfg.CONF) diff --git a/watcher/common/exception.py b/watcher/common/exception.py index aa924bf59..f93ee24a5 100644 --- a/watcher/common/exception.py +++ b/watcher/common/exception.py @@ -29,7 +29,6 @@ from watcher.common.i18n import _ from watcher.common.i18n import _LE from watcher.openstack.common import log as logging - LOG = logging.getLogger(__name__) exc_log_opts = [ @@ -227,6 +226,61 @@ class PatchError(Invalid): # decision engine + +class BaseException(Exception): + + def __init__(self, desc=""): + if (not isinstance(desc, basestring)): + raise IllegalArgumentException( + "Description must be an instance of str") + + desc = desc.strip() + + self._desc = desc + + def get_description(self): + return self._desc + + def get_message(self): + return "An exception occurred without a description." + + def __str__(self): + return self.get_message() + + +class IllegalArgumentException(BaseException): + def __init__(self, desc): + BaseException.__init__(self, desc) + + if self._desc == "": + raise IllegalArgumentException("Description cannot be empty") + + def get_message(self): + return self._desc + + +class NoSuchMetric(BaseException): + def __init__(self, desc): + BaseException.__init__(self, desc) + + if self._desc == "": + raise NoSuchMetric("No such metric") + + def get_message(self): + return self._desc + + +class NoDataFound(BaseException): + def __init__(self, desc): + BaseException.__init__(self, desc) + + if self._desc == "": + raise NoSuchMetric("no rows were returned") + + def get_message(self): + return self._desc + + class ClusterEmpty(WatcherException): message = _("The list of hypervisor(s) in the cluster is empty.'") diff --git a/watcher/decision_engine/api/collector/metrics_resource_collector.py b/watcher/decision_engine/api/collector/metrics_resource_collector.py deleted file mode 100644 index a8426eea1..000000000 --- a/watcher/decision_engine/api/collector/metrics_resource_collector.py +++ /dev/null @@ -1,35 +0,0 @@ -# -*- encoding: utf-8 -*- -# Copyright (c) 2015 b<>com -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -class MetricsResourceCollector(object): - def __init__(self): - pass - - def get_average_usage_vm_cpu(self, uuid): - raise NotImplementedError("Should have implemented this") - - def get_average_usage_vm_memory(self, uuid): - raise NotImplementedError("Should have implemented this") - - def get_virtual_machine_capacity(self, uuid): - raise NotImplementedError("Should have implemented this") - - def get_average_network_incomming(self, uuid): - raise NotImplementedError("Should have implemented this") - - def get_average_network_outcomming(self, uuid): - raise NotImplementedError("Should have implemented this") diff --git a/watcher/decision_engine/api/messaging/decision_engine_command.py b/watcher/decision_engine/api/messaging/decision_engine_command.py index 381d596cc..eac64aaef 100644 --- a/watcher/decision_engine/api/messaging/decision_engine_command.py +++ b/watcher/decision_engine/api/messaging/decision_engine_command.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/watcher/decision_engine/api/messaging/event_consumer.py b/watcher/decision_engine/api/messaging/event_consumer.py index eaa3d2364..9ae57ff7c 100644 --- a/watcher/decision_engine/api/messaging/event_consumer.py +++ b/watcher/decision_engine/api/messaging/event_consumer.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# class EventConsumer(object): diff --git a/watcher/decision_engine/api/planner/planner.py b/watcher/decision_engine/api/planner/planner.py index e899b2e31..a502f84fc 100644 --- a/watcher/decision_engine/api/planner/planner.py +++ b/watcher/decision_engine/api/planner/planner.py @@ -1,11 +1,13 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# class Planner(object): diff --git a/watcher/decision_engine/api/solution/solution.py b/watcher/decision_engine/api/solution/solution.py index 8cf84cdb0..602304213 100644 --- a/watcher/decision_engine/api/solution/solution.py +++ b/watcher/decision_engine/api/solution/solution.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/watcher/decision_engine/api/solution/solution_comparator.py b/watcher/decision_engine/api/solution/solution_comparator.py index b2f4aae63..57484aee8 100644 --- a/watcher/decision_engine/api/solution/solution_comparator.py +++ b/watcher/decision_engine/api/solution/solution_comparator.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/watcher/decision_engine/api/solution/solution_evaluator.py b/watcher/decision_engine/api/solution/solution_evaluator.py index 66b758b89..abe695664 100644 --- a/watcher/decision_engine/api/solution/solution_evaluator.py +++ b/watcher/decision_engine/api/solution/solution_evaluator.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/watcher/decision_engine/api/strategy/meta_action.py b/watcher/decision_engine/api/strategy/meta_action.py index 01d139dff..6f1aecc9e 100644 --- a/watcher/decision_engine/api/strategy/meta_action.py +++ b/watcher/decision_engine/api/strategy/meta_action.py @@ -1,11 +1,13 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# from watcher.decision_engine.api.strategy.strategy import StrategyLevel diff --git a/watcher/decision_engine/api/selector/selector.py b/watcher/decision_engine/api/strategy/selector.py similarity index 78% rename from watcher/decision_engine/api/selector/selector.py rename to watcher/decision_engine/api/strategy/selector.py index 295601169..f06cb3ef9 100644 --- a/watcher/decision_engine/api/selector/selector.py +++ b/watcher/decision_engine/api/strategy/selector.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,7 +15,9 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# class Selector(object): - pass + def define_from_goal(self, goal_name): + raise NotImplementedError("Should have implemented this") diff --git a/watcher/decision_engine/api/strategy/strategy.py b/watcher/decision_engine/api/strategy/strategy.py index b09a9a06f..b3a3af55f 100644 --- a/watcher/decision_engine/api/strategy/strategy.py +++ b/watcher/decision_engine/api/strategy/strategy.py @@ -24,8 +24,6 @@ from watcher.decision_engine.framework.default_solution import DefaultSolution LOG = log.getLogger(__name__) -# todo(jed) add interface - @six.add_metaclass(abc.ABCMeta) class Strategy(object): diff --git a/watcher/decision_engine/api/strategy/strategy_context.py b/watcher/decision_engine/api/strategy/strategy_context.py index ed295b51a..085a94629 100644 --- a/watcher/decision_engine/api/strategy/strategy_context.py +++ b/watcher/decision_engine/api/strategy/strategy_context.py @@ -1,11 +1,13 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# class StrategyContext(object): diff --git a/watcher/decision_engine/api/strategy/strategy_level.py b/watcher/decision_engine/api/strategy/strategy_level.py index 6e6e03c9f..ea79539b4 100644 --- a/watcher/decision_engine/api/strategy/strategy_level.py +++ b/watcher/decision_engine/api/strategy/strategy_level.py @@ -1,11 +1,13 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# from enum import Enum diff --git a/watcher/decision_engine/api/strategy/strategy_state.py b/watcher/decision_engine/api/strategy/strategy_state.py index 98c521b0e..5730581de 100644 --- a/watcher/decision_engine/api/strategy/strategy_state.py +++ b/watcher/decision_engine/api/strategy/strategy_state.py @@ -1,11 +1,13 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# from enum import Enum diff --git a/watcher/decision_engine/framework/command/trigger_audit_command.py b/watcher/decision_engine/framework/command/trigger_audit_command.py index b17ed7e40..6fd49dbc0 100644 --- a/watcher/decision_engine/framework/command/trigger_audit_command.py +++ b/watcher/decision_engine/framework/command/trigger_audit_command.py @@ -45,41 +45,42 @@ class TriggerAuditCommand(DecisionEngineCommand): self.messaging.topic_status.publish_event(event.get_type().name, payload) - # todo(jed) remove params + def update_audit(self, request_context, audit_uuid, state): + LOG.debug("update audit " + str(state)) + audit = Audit.get_by_uuid(request_context, audit_uuid) + audit.state = state + audit.save() + self.notify(audit_uuid, Events.TRIGGER_AUDIT, state) + return audit def execute(self, audit_uuid, request_context): - LOG.debug("Execute TriggerAuditCommand ") + try: + LOG.debug("Execute TriggerAuditCommand ") - # 1 - change status to ONGOING - audit = Audit.get_by_uuid(request_context, audit_uuid) - audit.state = AuditStatus.ONGOING - audit.save() + # 1 - change status to ONGOING + audit = self.update_audit(request_context, audit_uuid, + AuditStatus.ONGOING) - # 2 - notify the others components of the system - self.notify(audit_uuid, Events.TRIGGER_AUDIT, AuditStatus.ONGOING) + # 3 - Retrieve metrics + cluster = self.statedb.get_latest_state_cluster() - # 3 - Retrieve metrics - cluster = self.statedb.get_latest_state_cluster() + # 4 - Select appropriate strategy + audit_template = AuditTemplate.get_by_id(request_context, + audit.audit_template_id) - # 4 - Select appropriate strategy - audit_template = AuditTemplate.get_by_id(request_context, - audit.audit_template_id) + self.strategy_context.set_goal(audit_template.goal) + self.strategy_context.set_metrics_resource_collector( + self.ressourcedb) - self.strategy_context.set_goal(audit_template.goal) - self.strategy_context.set_metrics_resource_collector(self.ressourcedb) + # 5 - compute change requests + solution = self.strategy_context.execute_strategy(cluster) - # 5 - compute change requests - solution = self.strategy_context.execute_strategy(cluster) + # 6 - create an action plan + planner = DefaultPlanner() + planner.schedule(request_context, audit.id, solution) - # 6 - create an action plan - planner = DefaultPlanner() - planner.schedule(request_context, audit.id, solution) - - # 7 - change status to SUCCESS - audit = Audit.get_by_uuid(request_context, audit_uuid) - audit.state = AuditStatus.SUCCESS - audit.save() - - # 8 - notify the others components of the system - self.notify(audit_uuid, Events.TRIGGER_AUDIT, - AuditStatus.SUCCESS) + # 7 - change status to SUCCESS and notify + self.update_audit(request_context, audit_uuid, AuditStatus.SUCCESS) + except Exception as e: + self.update_audit(request_context, audit_uuid, AuditStatus.FAILED) + LOG.error(" " + unicode(e)) diff --git a/watcher/decision_engine/framework/default_planner.py b/watcher/decision_engine/framework/default_planner.py index 97d55558e..ed003948a 100644 --- a/watcher/decision_engine/framework/default_planner.py +++ b/watcher/decision_engine/framework/default_planner.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -88,11 +90,11 @@ class DefaultPlanner(Planner): # TODO(jed) type primitive = self.create_action(action_plan.id, Primitives.LIVE_MIGRATE.value, - action.get_vm().get_uuid(), + action.get_vm().uuid, action.get_source_hypervisor(). - get_uuid(), + uuid, action.get_dest_hypervisor(). - get_uuid(), + uuid, description=str(action) ) @@ -100,18 +102,16 @@ class DefaultPlanner(Planner): primitive = self.create_action(action_plan_id=action_plan.id, action_type=Primitives. POWER_STATE.value, - applies_to=action.target. - get_uuid(), + applies_to=action.target.uuid, parameter=action. - get_power_state(). + powerstate. value, description=str(action)) elif isinstance(action, ChangeHypervisorState): primitive = self.create_action(action_plan_id=action_plan.id, action_type=Primitives. HYPERVISOR_STATE.value, - applies_to=action.target. - get_uuid(), - parameter=action.get_state(). + applies_to=action.target.uuid, + parameter=action.state. value, description=str(action)) diff --git a/watcher/decision_engine/framework/default_solution.py b/watcher/decision_engine/framework/default_solution.py index 010e5eb8d..2824f9c82 100644 --- a/watcher/decision_engine/framework/default_solution.py +++ b/watcher/decision_engine/framework/default_solution.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,7 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. -# jed +# from watcher.decision_engine.api.solution.solution import Solution from watcher.openstack.common import log diff --git a/watcher/decision_engine/framework/manager_decision_engine.py b/watcher/decision_engine/framework/manager_decision_engine.py index 6f76e477b..b55cade6b 100644 --- a/watcher/decision_engine/framework/manager_decision_engine.py +++ b/watcher/decision_engine/framework/manager_decision_engine.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# from concurrent.futures import ThreadPoolExecutor from oslo_config import cfg diff --git a/watcher/decision_engine/framework/messaging/audit_endpoint.py b/watcher/decision_engine/framework/messaging/audit_endpoint.py index 5d446352d..a6950a899 100644 --- a/watcher/decision_engine/framework/messaging/audit_endpoint.py +++ b/watcher/decision_engine/framework/messaging/audit_endpoint.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,24 +15,24 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. - +# from watcher.decision_engine.framework.command.trigger_audit_command import \ TriggerAuditCommand -from watcher.decision_engine.framework.ressourcedb_collector import RessourceDB -from watcher.decision_engine.framework.statedb_collector import NovaCollector +from watcher.metrics_engine.framework.collector_manager import CollectorManager from watcher.openstack.common import log - LOG = log.getLogger(__name__) class AuditEndpoint(object): def __init__(self, de): self.de = de + self.manager = CollectorManager() def do_trigger_audit(self, context, audit_uuid): - statedb = NovaCollector() - ressourcedb = RessourceDB() + statedb = self.manager.get_statedb_collector() + ressourcedb = self.manager.get_metric_collector() + audit = TriggerAuditCommand(self.de, statedb, ressourcedb) audit.execute(audit_uuid, context) diff --git a/watcher/decision_engine/framework/messaging/events.py b/watcher/decision_engine/framework/messaging/events.py index 40e53761a..adcbd2d52 100644 --- a/watcher/decision_engine/framework/messaging/events.py +++ b/watcher/decision_engine/framework/messaging/events.py @@ -1,11 +1,13 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# from enum import Enum diff --git a/watcher/decision_engine/framework/meta_actions/hypervisor_state.py b/watcher/decision_engine/framework/meta_actions/hypervisor_state.py index ebc50e8f1..48468de5e 100644 --- a/watcher/decision_engine/framework/meta_actions/hypervisor_state.py +++ b/watcher/decision_engine/framework/meta_actions/hypervisor_state.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# from watcher.decision_engine.api.strategy.meta_action import MetaAction from watcher.decision_engine.framework.model.hypervisor_state import \ HypervisorState @@ -26,14 +29,24 @@ class ChangeHypervisorState(MetaAction): :param target: :return: ''' - self.target = target - self.state = HypervisorState.ONLINE + self._target = target + self._state = HypervisorState.ONLINE - def set_state(self, state): - self.state = state + @property + def state(self): + return self._state - def get_state(self): - return self.state + @state.setter + def state(self, state): + self._state = state + + @property + def target(self): + return self._target + + @target.setter + def target(self, p): + self._target = p def __str__(self): return MetaAction.__str__(self) + " ChangeHypervisorState" + str( diff --git a/watcher/decision_engine/framework/meta_actions/migrate.py b/watcher/decision_engine/framework/meta_actions/migrate.py index 3b642554a..fc5201561 100644 --- a/watcher/decision_engine/framework/meta_actions/migrate.py +++ b/watcher/decision_engine/framework/meta_actions/migrate.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# from enum import Enum diff --git a/watcher/decision_engine/framework/meta_actions/power_state.py b/watcher/decision_engine/framework/meta_actions/power_state.py index 76f39bb64..ee98f198f 100644 --- a/watcher/decision_engine/framework/meta_actions/power_state.py +++ b/watcher/decision_engine/framework/meta_actions/power_state.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# from watcher.decision_engine.api.strategy.meta_action import MetaAction from watcher.decision_engine.framework.model.power_state import PowerState @@ -25,15 +28,25 @@ class ChangePowerState(MetaAction): :param target: :return: """ - self.target = target - self.power_state = PowerState.g0 + self._target = target + self._power_state = PowerState.g0 - def set_power_state(self, state): - self.power_state = state + @property + def powerstate(self): + return self._power_state - def get_power_state(self): - return self.power_state + @powerstate.setter + def powerstate(self, p): + self._power_state = p + + @property + def target(self): + return self._target + + @target.setter + def target(self, p): + self._target = p def __str__(self): return MetaAction.__str__(self) + "ChangePowerState " + str( - self.target) + " => " + str(self.power_state) + self.target) + " => " + str(self.powerstate) diff --git a/watcher/decision_engine/framework/model/hypervisor.py b/watcher/decision_engine/framework/model/hypervisor.py index 1d01fe074..96c3034c2 100644 --- a/watcher/decision_engine/framework/model/hypervisor.py +++ b/watcher/decision_engine/framework/model/hypervisor.py @@ -21,11 +21,30 @@ from watcher.decision_engine.framework.model.power_state import PowerState class Hypervisor(NamedElement): def __init__(self): - self.state = HypervisorState.ONLINE - self.power_state = PowerState.g0 + self._state = HypervisorState.ONLINE + self._status = HypervisorState.ENABLED + self._power_state = PowerState.g0 - def set_state(self, state): - self.state = state + @property + def state(self): + return self._state - def get_state(self): - return self.state + @state.setter + def state(self, state): + self._state = state + + @property + def status(self): + return self._status + + @status.setter + def status(self, s): + self._status = s + + @property + def powerstate(self): + return self._power_state + + @powerstate.setter + def powerstate(self, p): + self._power_state = p diff --git a/watcher/decision_engine/framework/model/hypervisor_state.py b/watcher/decision_engine/framework/model/hypervisor_state.py index 9ac97e345..9852562e0 100644 --- a/watcher/decision_engine/framework/model/hypervisor_state.py +++ b/watcher/decision_engine/framework/model/hypervisor_state.py @@ -18,5 +18,7 @@ from enum import Enum class HypervisorState(Enum): - ONLINE = 'ONLINE' - OFFLINE = 'OFFLINE' + ONLINE = 'up' + OFFLINE = 'down' + ENABLED = 'enabled' + DISABLED = 'disabled' diff --git a/watcher/decision_engine/framework/model/mapping.py b/watcher/decision_engine/framework/model/mapping.py index f09ad8962..5bef6b1be 100644 --- a/watcher/decision_engine/framework/model/mapping.py +++ b/watcher/decision_engine/framework/model/mapping.py @@ -37,15 +37,15 @@ class Mapping(object): self.lock.acquire() # init first - if hypervisor.get_uuid() not in self._mapping_hypervisors.keys(): - self._mapping_hypervisors[hypervisor.get_uuid()] = [] + if hypervisor.uuid not in self._mapping_hypervisors.keys(): + self._mapping_hypervisors[hypervisor.uuid] = [] # map node => vms - self._mapping_hypervisors[hypervisor.get_uuid()].append( - vm.get_uuid()) + self._mapping_hypervisors[hypervisor.uuid].append( + vm.uuid) # map vm => node - self.mapping_vm[vm.get_uuid()] = hypervisor.get_uuid() + self.mapping_vm[vm.uuid] = hypervisor.uuid finally: self.lock.release() @@ -56,16 +56,23 @@ class Mapping(object): :param hypervisor: the hypervisor :param vm: the virtual machine or instance """ - self.unmap_from_id(hypervisor.get_uuid(), vm.get_uuid()) + self.unmap_from_id(hypervisor.uuid, vm.uuid) def unmap_from_id(self, node_uuid, vm_uuid): + """ + + :rtype : object + """ try: self.lock.acquire() if str(node_uuid) in self._mapping_hypervisors: self._mapping_hypervisors[str(node_uuid)].remove(str(vm_uuid)) + # remove vm + self.mapping_vm.pop(vm_uuid) else: LOG.warn("trying to delete the virtual machine " + str( - vm_uuid) + " but it was not found") + vm_uuid) + " but it was not found on hypervisor" + str( + node_uuid)) finally: self.lock.release() @@ -76,7 +83,7 @@ class Mapping(object): return self.mapping_vm def get_node_from_vm(self, vm): - return self.get_node_from_vm_id(vm.get_uuid()) + return self.get_node_from_vm_id(vm.uuid) def get_node_from_vm_id(self, vm_uuid): """Getting host information from the guest VM @@ -93,7 +100,7 @@ class Mapping(object): :param hypervisor: :return: """ - return self.get_node_vms_from_id(hypervisor.get_uuid()) + return self.get_node_vms_from_id(hypervisor.uuid) def get_node_vms_from_id(self, node_uuid): if str(node_uuid) in self._mapping_hypervisors.keys(): diff --git a/watcher/decision_engine/framework/model/model_root.py b/watcher/decision_engine/framework/model/model_root.py index 1d73e314b..c1dd3af21 100644 --- a/watcher/decision_engine/framework/model/model_root.py +++ b/watcher/decision_engine/framework/model/model_root.py @@ -15,6 +15,7 @@ # limitations under the License. from watcher.common.exception import HypervisorNotFound +from watcher.common.exception import IllegalArgumentException from watcher.common.exception import VMNotFound from watcher.decision_engine.framework.model.hypervisor import Hypervisor from watcher.decision_engine.framework.model.mapping import Mapping @@ -33,26 +34,28 @@ class ModelRoot(object): def assert_hypervisor(self, hypervisor): if not isinstance(hypervisor, Hypervisor): - raise Exception("assert_vm") + raise IllegalArgumentException( + "Hypervisor must be an instance of hypervisor") def assert_vm(self, vm): if not isinstance(vm, VM): - raise Exception("assert_vm") + raise IllegalArgumentException( + "VM must be an instance of VM") def add_hypervisor(self, hypervisor): self.assert_hypervisor(hypervisor) - self._hypervisors[hypervisor.get_uuid()] = hypervisor + self._hypervisors[hypervisor.uuid] = hypervisor def remove_hypervisor(self, hypervisor): self.assert_hypervisor(hypervisor) - if str(hypervisor.get_uuid()) not in self._hypervisors.keys(): - raise HypervisorNotFound(hypervisor.get_uuid()) + if str(hypervisor.uuid) not in self._hypervisors.keys(): + raise HypervisorNotFound(hypervisor.uuid) else: - del self._hypervisors[hypervisor.get_uuid()] + del self._hypervisors[hypervisor.uuid] def add_vm(self, vm): self.assert_vm(vm) - self._vms[vm.get_uuid()] = vm + self._vms[vm.uuid] = vm def get_all_hypervisors(self): return self._hypervisors diff --git a/watcher/decision_engine/framework/model/named_element.py b/watcher/decision_engine/framework/model/named_element.py index 234559fc6..1bcf72b65 100644 --- a/watcher/decision_engine/framework/model/named_element.py +++ b/watcher/decision_engine/framework/model/named_element.py @@ -18,13 +18,24 @@ class NamedElement(object): def __init__(self): - self.uuid = "" + self._uuid = "" + self._human_id = "" - def set_uuid(self, uuid): - self.uuid = uuid + @property + def uuid(self): + return self._uuid - def get_uuid(self): - return self.uuid + @uuid.setter + def uuid(self, u): + self._uuid = u + + @property + def human_id(self): + return self._human_id + + @human_id.setter + def human_id(self, h): + self._human_id = h def __str__(self): return "[" + str(self.uuid) + "]" diff --git a/watcher/decision_engine/framework/model/resource.py b/watcher/decision_engine/framework/model/resource.py index 66d7dac77..3586bf9f4 100644 --- a/watcher/decision_engine/framework/model/resource.py +++ b/watcher/decision_engine/framework/model/resource.py @@ -39,7 +39,7 @@ class Resource(object): return self.name def set_capacity(self, element, value): - self.mapping[element.get_uuid()] = value + self.mapping[element.uuid] = value def get_capacity_from_id(self, uuid): if str(uuid) in self.mapping.keys(): @@ -49,4 +49,4 @@ class Resource(object): return None def get_capacity(self, element): - return self.get_capacity_from_id(element.get_uuid()) + return self.get_capacity_from_id(element.uuid) diff --git a/watcher/decision_engine/framework/model/vm.py b/watcher/decision_engine/framework/model/vm.py index 99d5898e1..18a7785bf 100644 --- a/watcher/decision_engine/framework/model/vm.py +++ b/watcher/decision_engine/framework/model/vm.py @@ -19,10 +19,12 @@ from watcher.decision_engine.framework.model.vm_state import VMState class VM(NamedElement): def __init__(self): - self.state = VMState.INIT + self._state = VMState.ACTIVE.value - def set_state(self, state): - self.state = state + @property + def state(self): + return self._state - def get_state(self): - return self.state + @state.setter + def state(self, state): + self._state = state diff --git a/watcher/decision_engine/framework/model/vm_state.py b/watcher/decision_engine/framework/model/vm_state.py index 758f2d493..f31e1c247 100644 --- a/watcher/decision_engine/framework/model/vm_state.py +++ b/watcher/decision_engine/framework/model/vm_state.py @@ -18,9 +18,17 @@ from enum import Enum class VMState(Enum): - INIT = 1, - READY = 2, - RUNNING = 3, - SLEEPING = 4, - KILLED = 5, - LIVE_MIGRATION = 6 + ACTIVE = 'active' # VM is running + BUILDING = 'building' # VM only exists in DB + PAUSED = 'paused' + SUSPENDED = 'suspended' # VM is suspended to disk. + STOPPED = 'stopped' # VM is powered off, the disk image is still there. + RESCUED = 'rescued' # A rescue image is running with the original VM image + # attached. + RESIZED = 'resized' # a VM with the new size is active. + + SOFT_DELETED = 'soft-delete' + # still available to restore. + DELETED = 'deleted' # VM is permanently deleted. + + ERROR = 'error' diff --git a/watcher/decision_engine/framework/ressourcedb_collector.py b/watcher/decision_engine/framework/ressourcedb_collector.py deleted file mode 100644 index 490aec24d..000000000 --- a/watcher/decision_engine/framework/ressourcedb_collector.py +++ /dev/null @@ -1,117 +0,0 @@ -# -*- encoding: utf-8 -*- -# Copyright (c) 2015 b<>com -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import ceilometerclient.v2 as c_client -import keystoneclient.v3.client as ksclient -from oslo_config import cfg -CONF = cfg.CONF - -from watcher.decision_engine.api.collector.metrics_resource_collector import \ - MetricsResourceCollector - -CONF.import_opt('admin_user', 'keystonemiddleware.auth_token', - group='keystone_authtoken') -CONF.import_opt('admin_tenant_name', 'keystonemiddleware.auth_token', - group='keystone_authtoken') -CONF.import_opt('admin_password', 'keystonemiddleware.auth_token', - group='keystone_authtoken') -CONF.import_opt('auth_uri', 'keystonemiddleware.auth_token', - group='keystone_authtoken') - - -class RessourceDB(MetricsResourceCollector): - def __init__(self): - creds = \ - {'auth_url': CONF.keystone_authtoken.auth_uri, - 'username': CONF.keystone_authtoken.admin_user, - 'password': CONF.keystone_authtoken.admin_password, - 'project_name': CONF.keystone_authtoken.admin_tenant_name, - 'user_domain_name': "default", - 'project_domain_name': "default"} - self.keystone = ksclient.Client(**creds) - - self.ceilometer = c_client.Client( - endpoint=self.get_ceilometer_uri(), - token=self.keystone.auth_token) - - def make_query(user_id=None, tenant_id=None, resource_id=None, - user_ids=None, tenant_ids=None, resource_ids=None): - - """Returns query built form given parameters. - This query can be then used for querying resources, meters and - statistics. - :Parameters: - - `user_id`: user_id, has a priority over list of ids - - `tenant_id`: tenant_id, has a priority over list of ids - - `resource_id`: resource_id, has a priority over list of ids - - `user_ids`: list of user_ids - - `tenant_ids`: list of tenant_ids - - `resource_ids`: list of resource_ids - """ - user_ids = user_ids or [] - tenant_ids = tenant_ids or [] - resource_ids = resource_ids or [] - query = [] - if user_id: - user_ids = [user_id] - for u_id in user_ids: - query.append({"field": "user_id", "op": "eq", "value": u_id}) - if tenant_id: - tenant_ids = [tenant_id] - for t_id in tenant_ids: - query.append({"field": "project_id", "op": "eq", "value": t_id}) - if resource_id: - resource_ids = [resource_id] - for r_id in resource_ids: - query.append({"field": "resource_id", "op": "eq", "value": r_id}) - return query - - def get_ceilometer_uri(self): - a = self.keystone.services.list(**{'type': 'metering'}) - e = self.keystone.endpoints.list() - for s in e: - if s.service_id == a[0].id and s.interface == 'internal': - return s.url - raise Exception("Ceilometer Metering Service internal not defined") - - def get_average_usage_vm_cpu(self, instance_uuid): - """The last VM CPU usage values to average - - :param uuid:00 - :return: - """ - # query influxdb stream - query = self.make_query(resource_id=instance_uuid) - cpu_util_sample = self.ceilometer.samples.list('cpu_util', - q=query) - cpu_usage = 0 - count = len(cpu_util_sample) - for each in cpu_util_sample: - # print each.timestamp, each.counter_name, each.counter_volume - cpu_usage = cpu_usage + each.counter_volume - if count == 0: - return 0 - else: - return cpu_usage / len(cpu_util_sample) - - def get_average_usage_vm_memory(self, uuid): - # Obtaining Memory Usage is not implemented for LibvirtInspector - # waiting for kilo memory.resident - return 1 - - def get_average_usage_vm_disk(self, uuid): - # waiting for kilo disk.usage - return 1 diff --git a/watcher/decision_engine/framework/rpcapi.py b/watcher/decision_engine/framework/rpcapi.py index 4f3daebba..1c16a849a 100644 --- a/watcher/decision_engine/framework/rpcapi.py +++ b/watcher/decision_engine/framework/rpcapi.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# from oslo_config import cfg diff --git a/watcher/decision_engine/framework/statedb_collector.py b/watcher/decision_engine/framework/statedb_collector.py deleted file mode 100644 index 54566a825..000000000 --- a/watcher/decision_engine/framework/statedb_collector.py +++ /dev/null @@ -1,104 +0,0 @@ -# -*- encoding: utf-8 -*- -# Copyright (c) 2015 b<>com -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -from keystoneclient.auth.identity import v3 -from keystoneclient import session - -from watcher.applier.framework.command.wrapper.nova_wrapper import NovaWrapper -from watcher.decision_engine.api.collector.cluster_state_collector import \ - ClusterStateCollector -from watcher.decision_engine.framework.model.hypervisor import Hypervisor -from watcher.decision_engine.framework.model.model_root import ModelRoot -from watcher.decision_engine.framework.model.resource import Resource -from watcher.decision_engine.framework.model.resource import ResourceType -from watcher.decision_engine.framework.model.vm import VM -from watcher.openstack.common import log - -from oslo_config import cfg -CONF = cfg.CONF -LOG = log.getLogger(__name__) - -CONF.import_opt('admin_user', 'keystonemiddleware.auth_token', - group='keystone_authtoken') -CONF.import_opt('admin_tenant_name', 'keystonemiddleware.auth_token', - group='keystone_authtoken') -CONF.import_opt('admin_password', 'keystonemiddleware.auth_token', - group='keystone_authtoken') -CONF.import_opt('auth_uri', 'keystonemiddleware.auth_token', - group='keystone_authtoken') - - -class NovaCollector(ClusterStateCollector): - def get_latest_state_cluster(self): - try: - creds = \ - {'auth_url': CONF.keystone_authtoken.auth_uri, - 'username': CONF.keystone_authtoken.admin_user, - 'password': CONF.keystone_authtoken.admin_password, - 'project_name': CONF.keystone_authtoken.admin_tenant_name, - 'user_domain_name': "default", - 'project_domain_name': "default"} - auth = v3.Password(auth_url=creds['auth_url'], - username=creds['username'], - password=creds['password'], - project_name=creds['project_name'], - user_domain_name=creds[ - 'user_domain_name'], - project_domain_name=creds[ - 'project_domain_name']) - sess = session.Session(auth=auth) - wrapper = NovaWrapper(creds, session=sess) - - cluster = ModelRoot() - mem = Resource(ResourceType.memory) - num_cores = Resource(ResourceType.cpu_cores) - disk = Resource(ResourceType.disk) - cluster.create_resource(mem) - cluster.create_resource(num_cores) - cluster.create_resource(disk) - - flavor_cache = {} - hypervisors = wrapper.get_hypervisors_list() - for h in hypervisors: - i = h.hypervisor_hostname.index('.') - name = h.hypervisor_hostname[0:i] - # create hypervisor in stateDB - hypervisor = Hypervisor() - hypervisor.set_uuid(name) - # set capacity - mem.set_capacity(hypervisor, h.memory_mb) - disk.set_capacity(hypervisor, h.disk_available_least) - num_cores.set_capacity(hypervisor, h.vcpus) - cluster.add_hypervisor(hypervisor) - vms = wrapper.get_vms_by_hypervisor(str(name)) - for v in vms: - # create VM in stateDB - vm = VM() - vm.set_uuid(v.id) - # set capacity - wrapper.get_flavor_instance(v, flavor_cache) - mem.set_capacity(vm, v.flavor['ram']) - disk.set_capacity(vm, v.flavor['disk']) - num_cores.set_capacity(vm, v.flavor['vcpus']) - # print(dir(v)) - cluster.get_mapping().map(hypervisor, vm) - cluster.add_vm(vm) - return cluster - except Exception as e: - LOG.error("nova collector " + unicode(e)) - return None diff --git a/watcher/decision_engine/framework/strategy/strategy_loader.py b/watcher/decision_engine/framework/strategy/strategy_loader.py index 03efc31e1..b9a23bff0 100644 --- a/watcher/decision_engine/framework/strategy/strategy_loader.py +++ b/watcher/decision_engine/framework/strategy/strategy_loader.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,7 +15,9 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# from oslo_config import cfg +from stevedore import driver from watcher.decision_engine.strategies.basic_consolidation import \ BasicConsolidation from watcher.openstack.common import log @@ -52,5 +56,13 @@ class StrategyLoader(object): "Basic offline consolidation") } + def load_driver(self, algo): + _algo = driver.DriverManager( + namespace='watcher_strategies', + name=algo, + invoke_on_load=True, + ) + return _algo + def load(self, model): return self.strategies[model] diff --git a/watcher/decision_engine/framework/strategy/strategy_selector.py b/watcher/decision_engine/framework/strategy/strategy_selector.py index e3ba14ca1..79205489a 100644 --- a/watcher/decision_engine/framework/strategy/strategy_selector.py +++ b/watcher/decision_engine/framework/strategy/strategy_selector.py @@ -15,6 +15,7 @@ # limitations under the License. from oslo_config import cfg +from watcher.decision_engine.api.strategy.selector import Selector from watcher.decision_engine.framework.strategy.strategy_loader import \ StrategyLoader from watcher.objects.audit_template import Goal @@ -40,7 +41,7 @@ CONF.register_group(goals_opt_group) CONF.register_opts(WATCHER_GOALS_OPTS, goals_opt_group) -class StrategySelector(object): +class StrategySelector(Selector): def __init__(self): self.strategy_loader = StrategyLoader() diff --git a/watcher/decision_engine/strategies/basic_consolidation.py b/watcher/decision_engine/strategies/basic_consolidation.py index 3e870edff..22e60395c 100644 --- a/watcher/decision_engine/strategies/basic_consolidation.py +++ b/watcher/decision_engine/strategies/basic_consolidation.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,10 +15,15 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# +from watcher.decision_engine.framework.model.vm_state import VMState +from watcher.metrics_engine.api.metrics_resource_collector import \ + AggregationFunction from watcher.common.exception import ClusterEmpty from watcher.common.exception import ClusteStateNotDefined from watcher.common.exception import MetricCollectorNotDefined +from watcher.common.exception import NoDataFound from watcher.decision_engine.api.strategy.strategy import Strategy from watcher.decision_engine.api.strategy.strategy import StrategyLevel from watcher.decision_engine.framework.meta_actions.hypervisor_state import \ @@ -78,8 +85,8 @@ class BasicConsolidation(Strategy): self.efficiency = 100 # TODO(jed) improve threshold overbooking ?,... - self.threshold_mem = 0.90 - self.threshold_disk = 0.80 + self.threshold_mem = 1 + self.threshold_disk = 1 self.threshold_cores = 1 # TODO(jed) target efficiency @@ -115,6 +122,11 @@ class BasicConsolidation(Strategy): if src_hypervisor == dest_hypervisor: return False + LOG.debug('Migrate VM %s from %s to %s ', + str(src_hypervisor), + str(dest_hypervisor), + str(vm_to_mig)) + total_cores = 0 total_disk = 0 total_mem = 0 @@ -162,6 +174,13 @@ class BasicConsolidation(Strategy): cores_available = cap_cores.get_capacity(dest_hypervisor) disk_available = cap_disk.get_capacity(dest_hypervisor) mem_available = cap_mem.get_capacity(dest_hypervisor) + LOG.debug("VCPU %s/%s ", str(total_cores * self.threshold_cores), + str(cores_available), ) + LOG.debug("DISK %s/%s ", str(total_disk * self.threshold_disk), + str(disk_available), ) + LOG.debug("MEM %s/%s ", str(total_mem * self.threshold_mem), + str(mem_available)) + if cores_available >= total_cores * self.threshold_cores \ and disk_available >= total_disk * self.threshold_disk \ and mem_available >= total_mem * self.threshold_mem: @@ -232,21 +251,25 @@ class BasicConsolidation(Strategy): metrics_collector = self.get_metrics_resource_collector() if metrics_collector is None: raise MetricCollectorNotDefined() - total_cores_used = 0 - total_memory_used = 0 - total_disk_used = 0 - for vm_id in model.get_mapping().get_node_vms(hypervisor): - total_cores_used += metrics_collector.get_average_usage_vm_cpu( - vm_id) - total_memory_used += metrics_collector.get_average_usage_vm_memory( - vm_id) - total_disk_used += metrics_collector.get_average_usage_vm_disk( - vm_id) + cpu_compute_mean_16h = metrics_collector.get_measurement( + metric='compute_cpu_user_percent_gauge', + aggregation_function=AggregationFunction.MEAN, + start_time="16 hours before now", + filters=["resource_id=" + hypervisor.uuid + ""]) + if len(cpu_compute_mean_16h) > 0: + cpu_capacity = model.get_resource_from_id( + ResourceType.cpu_cores).get_capacity(hypervisor) + cpu_utilization = float(cpu_compute_mean_16h[0].value) + total_cores_used = cpu_capacity * (cpu_utilization / 100) + else: + raise NoDataFound( + "No values returned for " + str(hypervisor.uuid) + + " compute_cpu_percent_gauge") return self.calculate_weight(model, hypervisor, total_cores_used, - total_disk_used, - total_memory_used) + 0, + 0) def calculate_migration_efficiency(self): """Calculate migration efficiency @@ -275,15 +298,25 @@ class BasicConsolidation(Strategy): if model is None: raise ClusteStateNotDefined() - vm = model.get_vm_from_id(vm.get_uuid()) - cores_used = metric_collector.get_average_usage_vm_cpu(vm.get_uuid()) - memory_used = metric_collector.get_average_usage_vm_memory( - vm.get_uuid()) - disk_used = metric_collector.get_average_usage_vm_disk(vm.get_uuid()) + vm = model.get_vm_from_id(vm.uuid) + instance_cpu_mean_16 = metric_collector.get_measurement( + metric='instance_cpu_percent_gauge', + aggregation_function=AggregationFunction.MEAN, + start_time="16 hours before now", + filters=["resource_id=" + vm.uuid + ""]) - return self.calculate_weight(model, vm, cores_used, - disk_used, - memory_used) + if len(instance_cpu_mean_16) > 0: + cpu_capacity = model.get_resource_from_id( + ResourceType.cpu_cores).get_capacity(vm) + vm_cpu_utilization = instance_cpu_mean_16[0].value + total_cores_used = cpu_capacity * (vm_cpu_utilization / 100) + else: + raise NoDataFound("No values returned for " + str(vm.uuid) + + " instance_cpu_percent_gauge") + + return self.calculate_weight(model, vm, total_cores_used, + 0, + 0) def print_utilization(self, model): if model is None: @@ -308,13 +341,27 @@ class BasicConsolidation(Strategy): unsuccessful_migration = 0 first = True - self.print_utilization(current_model) size_cluster = len(current_model.get_all_hypervisors()) if size_cluster == 0: raise ClusterEmpty() self.compute_attempts(size_cluster) + for hypevisor_id in current_model.get_all_hypervisors(): + hypervisor = current_model.get_hypervisor_from_id(hypevisor_id) + count = current_model.get_mapping(). \ + get_node_vms_from_id(hypevisor_id) + if len(count) == 0: + change_power = ChangePowerState(hypervisor) + change_power.powerstate = PowerState.g1_S1 + change_power.set_level(StrategyLevel.conservative) + self.solution.add_change_request(change_power) + if hypervisor.state == HypervisorState.ONLINE: + h = ChangeHypervisorState(hypervisor) + h.set_level(StrategyLevel.aggressive) + h.state = HypervisorState.OFFLINE + self.solution.add_change_request(h) + while self.get_allowed_migration_attempts() >= unsuccessful_migration: if first is not True: self.efficiency = self.calculate_migration_efficiency() @@ -325,9 +372,16 @@ class BasicConsolidation(Strategy): ''' calculate score of nodes based on load by VMs ''' for hypevisor_id in current_model.get_all_hypervisors(): - hypevisor = current_model.get_hypervisor_from_id(hypevisor_id) - result = self.calculate_score_node(hypevisor, current_model) - if result != 0: + hypervisor = current_model.get_hypervisor_from_id(hypevisor_id) + count = current_model.get_mapping(). \ + get_node_vms_from_id(hypevisor_id) + if len(count) > 0: + result = self.calculate_score_node(hypervisor, + current_model) + else: + ''' the hypervisor has not VMs ''' + result = 0 + if len(count) > 0: score.append((hypevisor_id, result)) ''' sort compute nodes by Score decreasing ''''' @@ -350,8 +404,9 @@ class BasicConsolidation(Strategy): vm_score = [] for vm_id in vms_to_mig: vm = current_model.get_vm_from_id(vm_id) - vm_score.append( - (vm_id, self.calculate_score_vm(vm, current_model))) + if vm.state == VMState.ACTIVE.value: + vm_score.append( + (vm_id, self.calculate_score_vm(vm, current_model))) ''' sort VM's by Score ''' v = sorted(vm_score, reverse=True, key=lambda x: (x[1])) @@ -392,7 +447,7 @@ class BasicConsolidation(Strategy): # TODO(jed) how to manage strategy level # from conservative to aggressive change_power = ChangePowerState(mig_src_hypervisor) - change_power.set_power_state(PowerState.g1_S1) + change_power.powerstate = PowerState.g1_S1 change_power.set_level( StrategyLevel.conservative) tmp_vm_migration_schedule.append(change_power) @@ -400,7 +455,7 @@ class BasicConsolidation(Strategy): h = ChangeHypervisorState(mig_src_hypervisor) h.set_level(StrategyLevel.aggressive) - h.set_state(HypervisorState.OFFLINE) + h.state = HypervisorState.OFFLINE tmp_vm_migration_schedule.append(h) self.number_of_released_nodes += 1 @@ -414,7 +469,7 @@ class BasicConsolidation(Strategy): self.solution.add_change_request(a) else: unsuccessful_migration += 1 - self.print_utilization(current_model) + # self.print_utilization(current_model) infos = { "number_of_migrations": self.number_of_migrations, "number_of_nodes_released": self.number_of_released_nodes, diff --git a/watcher/decision_engine/strategies/dummy_strategy.py b/watcher/decision_engine/strategies/dummy_strategy.py index 762a0af7d..96ce874b4 100644 --- a/watcher/decision_engine/strategies/dummy_strategy.py +++ b/watcher/decision_engine/strategies/dummy_strategy.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# from watcher.decision_engine.api.strategy.strategy import Strategy from watcher.openstack.common import log diff --git a/watcher/decision_engine/api/collector/__init__.py b/watcher/metrics_engine/__init__.py similarity index 100% rename from watcher/decision_engine/api/collector/__init__.py rename to watcher/metrics_engine/__init__.py diff --git a/watcher/decision_engine/api/selector/__init__.py b/watcher/metrics_engine/api/__init__.py similarity index 100% rename from watcher/decision_engine/api/selector/__init__.py rename to watcher/metrics_engine/api/__init__.py diff --git a/watcher/decision_engine/api/collector/cluster_state_collector.py b/watcher/metrics_engine/api/cluster_state_collector.py similarity index 78% rename from watcher/decision_engine/api/collector/cluster_state_collector.py rename to watcher/metrics_engine/api/cluster_state_collector.py index 2b09c3808..77c8008be 100644 --- a/watcher/decision_engine/api/collector/cluster_state_collector.py +++ b/watcher/metrics_engine/api/cluster_state_collector.py @@ -1,11 +1,13 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -13,10 +15,14 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# +import abc +import six +@six.add_metaclass(abc.ABCMeta) class ClusterStateCollector(object): + + @abc.abstractmethod def get_latest_state_cluster(self): raise NotImplementedError("Should have implemented this") - # todo(jed) think abouts needed interfaces - # todo(jed) stream incremental diff diff --git a/watcher/metrics_engine/api/metrics_resource_collector.py b/watcher/metrics_engine/api/metrics_resource_collector.py new file mode 100644 index 000000000..42afb4763 --- /dev/null +++ b/watcher/metrics_engine/api/metrics_resource_collector.py @@ -0,0 +1,65 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Authors: Jean-Emile DARTOIS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import abc +from enum import Enum +import six + + +class AggregationFunction(Enum): + MEAN = 'mean' + COUNT = 'count' + + +class Measure(object): + def __init__(self, time, value): + self.time = time + self.value = value + + def __str__(self): + return str(self.time) + " " + str(self.value) + + +@six.add_metaclass(abc.ABCMeta) +class MetricsResourceCollector(object): + @abc.abstractmethod + def get_measurement(self, + metric, + callback=None, + start_time=None, + end_time=None, + filters=None, + aggregation_function=None, + intervals=None): + """ + + :param metric: The full name of a metric in the system. + Must be the complete name. Case sensitive + :param callback: Asynchronous Callback Functions to live retrev + :param start_time:Starting time for the query. + This may be an absolute or relative time. + :param end_time: An end time for the query. + If the end time is not supplied, the current time + on the TSD will be used. + :param filters: An optional set of tags for filtering or grouping + :param aggregation_function: A mathematical function + :param intervals: An optional interval and function + to reduce the number of data points returned + :return: + """ + raise NotImplementedError("Should have implemented this") diff --git a/watcher/tests/applier/demo/__init__.py b/watcher/metrics_engine/framework/__init__.py similarity index 100% rename from watcher/tests/applier/demo/__init__.py rename to watcher/metrics_engine/framework/__init__.py diff --git a/watcher/metrics_engine/framework/collector_manager.py b/watcher/metrics_engine/framework/collector_manager.py new file mode 100644 index 000000000..4acde9136 --- /dev/null +++ b/watcher/metrics_engine/framework/collector_manager.py @@ -0,0 +1,83 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Authors: Jean-Emile DARTOIS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from keystoneclient.auth.identity import v3 +from keystoneclient import session + +from oslo_config import cfg +from stevedore import driver +from watcher.applier.framework.command.wrapper.nova_wrapper import NovaWrapper + +from watcher.metrics_engine.framework.statedb_collector import NovaCollector +from watcher.openstack.common import log + +LOG = log.getLogger(__name__) +CONF = cfg.CONF + +WATCHER_METRICS_COLLECTOR_OPTS = [ + cfg.StrOpt('metrics_resource', + default="influxdb", + help='The driver that collect measurements' + 'of the utilization' + 'of the physical and virtual resources') +] +metrics_collector_opt_group = cfg.OptGroup( + name='watcher_collector', + title='Defines Metrics collector available') +CONF.register_group(metrics_collector_opt_group) +CONF.register_opts(WATCHER_METRICS_COLLECTOR_OPTS, metrics_collector_opt_group) + +CONF.import_opt('admin_user', 'keystonemiddleware.auth_token', + group='keystone_authtoken') +CONF.import_opt('admin_tenant_name', 'keystonemiddleware.auth_token', + group='keystone_authtoken') +CONF.import_opt('admin_password', 'keystonemiddleware.auth_token', + group='keystone_authtoken') +CONF.import_opt('auth_uri', 'keystonemiddleware.auth_token', + group='keystone_authtoken') + + +class CollectorManager(object): + def get_metric_collector(self): + manager = driver.DriverManager( + namespace='watcher_metrics_collector', + name=CONF.watcher_collector.metrics_resource, + invoke_on_load=True, + ) + return manager.driver + + def get_statedb_collector(self): + creds = \ + {'auth_url': CONF.keystone_authtoken.auth_uri, + 'username': CONF.keystone_authtoken.admin_user, + 'password': CONF.keystone_authtoken.admin_password, + 'project_name': CONF.keystone_authtoken.admin_tenant_name, + 'user_domain_name': "default", + 'project_domain_name': "default"} + + auth = v3.Password(auth_url=creds['auth_url'], + username=creds['username'], + password=creds['password'], + project_name=creds['project_name'], + user_domain_name=creds[ + 'user_domain_name'], + project_domain_name=creds[ + 'project_domain_name']) + sess = session.Session(auth=auth) + wrapper = NovaWrapper(creds, session=sess) + return NovaCollector(wrapper=wrapper) diff --git a/watcher/metrics_engine/framework/datasources/__init__.py b/watcher/metrics_engine/framework/datasources/__init__.py new file mode 100644 index 000000000..b5de5a85c --- /dev/null +++ b/watcher/metrics_engine/framework/datasources/__init__.py @@ -0,0 +1,20 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +class InvalidQuery(Exception): + pass diff --git a/watcher/metrics_engine/framework/datasources/influxdb_collector.py b/watcher/metrics_engine/framework/datasources/influxdb_collector.py new file mode 100644 index 000000000..44d40ebed --- /dev/null +++ b/watcher/metrics_engine/framework/datasources/influxdb_collector.py @@ -0,0 +1,169 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Authors: Jean-Emile DARTOIS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from concurrent.futures import ThreadPoolExecutor +import datetime +import parsedatetime + +from influxdb import InfluxDBClient +from oslo_config import cfg +from watcher.metrics_engine.api.metrics_resource_collector import \ + AggregationFunction +from watcher.metrics_engine.api.metrics_resource_collector import Measure +from watcher.metrics_engine.api.metrics_resource_collector import \ + MetricsResourceCollector +from watcher.metrics_engine.framework.datasources.sql_ast.build_db_query import \ + DBQuery +from watcher.metrics_engine.framework.datasources.sql_ast.sql_ast import And +from watcher.metrics_engine.framework.datasources.sql_ast.sql_ast import \ + Condition +from watcher.openstack.common import log + +LOG = log.getLogger(__name__) +CONF = cfg.CONF + +WATCHER_INFLUXDB_COLLECTOR_OPTS = [ + cfg.StrOpt('hostname', + default='localhost', + help='The hostname to connect to InfluxDB'), + cfg.IntOpt('port', + default='8086', + help='port to connect to InfluxDB, defaults to 8086'), + cfg.StrOpt('username', + default='root', + help='user to connect, defaults to root'), + cfg.StrOpt('password', + default='root', + help='password of the user, defaults to root'), + cfg.StrOpt('database', + default='indeed', + help='database name to connect to'), + cfg.BoolOpt('param ssl', + default=False, + help='use https instead of http to connect to InfluxDB'), + cfg.IntOpt('timeout', + default='5', + help='number of seconds Requests' + 'will wait for your client to establish a connection'), + cfg.IntOpt('timeout', + default='5', + help='number of seconds Requests' + 'will wait for your client to establish a connection'), + cfg.BoolOpt('use_udp', + default=False, + help='use UDP to connect to InfluxDB'), + cfg.IntOpt('udp_port', + default='4444', + help=' UDP port to connect to InfluxDB') +] + +influxdb_collector_opt_group = cfg.OptGroup( + name='watcher_influxdb_collector', + title='Defines the parameters of the module collector') +CONF.register_group(influxdb_collector_opt_group) +CONF.register_opts(WATCHER_INFLUXDB_COLLECTOR_OPTS, + influxdb_collector_opt_group) + + +class InfluxDBCollector(MetricsResourceCollector): + def __init__(self): + self.executor = ThreadPoolExecutor(max_workers=3) + + def get_client(self): + LOG.debug("InfluxDB " + str(CONF.watcher_influxdb_collector.hostname)) + influx = InfluxDBClient(CONF.watcher_influxdb_collector.hostname, + CONF.watcher_influxdb_collector.port, + CONF.watcher_influxdb_collector.username, + CONF.watcher_influxdb_collector.password, + CONF.watcher_influxdb_collector.database) + if {u'name': u'' + CONF.watcher_influxdb_collector.database + ''} not \ + in influx.get_list_database(): + raise Exception("The selected database does not exist" + "or the user credentials supplied are wrong") + return influx + + def convert(self, time): + cal = parsedatetime.Calendar() + time_struct, result = cal.parse(time) + return datetime.datetime(*time_struct[:6]).ctime() + + def build_query(self, + measurement, + start_time=None, + end_time=None, + filters=None, + aggregation_function=None, + intervals=None): + query = DBQuery(measurement) + conditions = [] + if start_time is not None: + c = Condition('time', '>', self.convert(start_time)) + conditions.append(c) + if end_time is not None: + c = Condition('time', '>', self.convert(end_time)) + conditions.append(c) + if filters is not None: + for f in filters: + c = Condition(f.split('=')[0], '=', f.split('=')[1]) + conditions.append(c) + if aggregation_function is not None: + if aggregation_function == AggregationFunction.MEAN: + query.select("mean(value)") + elif aggregation_function == AggregationFunction.COUNT: + query.select("count(value)") + + if intervals is not None: + query.groupby("time(" + str(intervals) + ")") + + if len(conditions) == 1: + query.where(conditions[0]) + elif len(conditions) != 0: + _where = And(conditions[0], conditions[1]) + for i in range(2, len(conditions)): + _where = And(_where, conditions[i]) + query.where(_where) + LOG.debug(query) + return query + + def get_measurement(self, + metric, + callback=None, + start_time=None, + end_time=None, + filters=None, + aggregation_function=None, + intervals=None): + results = [] + client = self.get_client() + query = self.build_query(metric, start_time, end_time, filters, + aggregation_function, intervals) + + results_from_influx = client.query(query) + + for item in results_from_influx[None]: + time = item.get('time', None) + for field in ['value', 'count', 'min', 'max', 'mean']: + value = item.get(field, None) + if value is not None: + row = Measure(time, value) + if callback is not None: + self.executor.submit(callback, row) + else: + results.append(row) + return results diff --git a/watcher/tests/decision_engine/demo/__init__.py b/watcher/metrics_engine/framework/datasources/sql_ast/__init__.py similarity index 100% rename from watcher/tests/decision_engine/demo/__init__.py rename to watcher/metrics_engine/framework/datasources/sql_ast/__init__.py diff --git a/watcher/metrics_engine/framework/datasources/sql_ast/build_db_query.py b/watcher/metrics_engine/framework/datasources/sql_ast/build_db_query.py new file mode 100644 index 000000000..0e805ddcc --- /dev/null +++ b/watcher/metrics_engine/framework/datasources/sql_ast/build_db_query.py @@ -0,0 +1,59 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Authors: Jean-Emile DARTOIS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from watcher.metrics_engine.framework.datasources.sql_ast.sql_ast import From +from watcher.metrics_engine.framework.datasources.sql_ast.sql_ast import \ + GroupBy +from watcher.metrics_engine.framework.datasources.sql_ast.sql_ast import Limit +from watcher.metrics_engine.framework.datasources.sql_ast.sql_ast import List +from watcher.metrics_engine.framework.datasources.sql_ast.sql_ast import Select +from watcher.metrics_engine.framework.datasources.sql_ast.sql_ast import Where + + +class DBQuery(object): + def __init__(self, _from): + self._select = Select(_from) + self.inline = False + + def select_from(self, _from): + self._select._from = From(_from) + return self + + def where(self, where): + self._select.where = Where(where) + return self + + def groupby(self, g): + self._select.groupby = GroupBy(g) + return self + + def limit(self, limit): + self._select.limit = Limit(limit) + return self + + def select(self, *args): + self._select.what = List(*args) + return self + + def __str__(self): + self._select.inline = self.inline + s = str(self._select) + if not self.inline: + s += ';' + return s diff --git a/watcher/metrics_engine/framework/datasources/sql_ast/sql_ast.py b/watcher/metrics_engine/framework/datasources/sql_ast/sql_ast.py new file mode 100644 index 000000000..9a2de862b --- /dev/null +++ b/watcher/metrics_engine/framework/datasources/sql_ast/sql_ast.py @@ -0,0 +1,157 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Authors: Jean-Emile DARTOIS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +class ASTNode(object): + visited = False + inline = False + + def __init__(self): + pass + + def children(self): + for c in self._children: + yield c + + def __str__(self): + pass + + +class Condition(ASTNode): + def __init__(self, what, operator, _on): + self.what = what + self._on = "'" + str(_on) + "'" + self.operator = operator + + def __str__(self): + s = self.what + ' ' + self.operator + ' ' + self._on + if self.inline: + s = '' + s + '' + return s + + +class BinaryNode(ASTNode): + def __init__(self, left, right): + self.left = left + self.right = right + + def __str__(self): + return '({left} {middle} {right})'.format( + left=self.left, + middle=self.middle, + right=self.right + ) + + +class And(BinaryNode): + middle = 'AND' + + +class Or(BinaryNode): + middle = 'OR' + + +class List(ASTNode): + def __init__(self, *args): + for arg in args: + if hasattr(arg, 'inline'): + arg.inline = True + self.items = args + + def __str__(self): + lst = ', '.join(map(lambda x: str(x), self.items)) + if self.inline: + lst = '(' + lst + ')' + return lst + + +class Set(ASTNode): + def __init__(self, **kwargs): + self.items = kwargs + + def __str__(self): + return ', '.join( + ['{0}={1}'.format(key, val) for key, val in self.items.items()]) + + +class Returning(ASTNode): + def __init__(self, _list='*'): + self._list = _list + + def __str__(self): + return 'RETURNING {_list}'.format(_list=self._list) + + +class Limit(ASTNode): + def __init__(self, limit): + if hasattr(limit, 'inline'): + limit.inline = True + self.limit = limit + + def __str__(self): + return " LIMIT " + str(self.limit) + + +class Where(ASTNode): + def __init__(self, logic): + if hasattr(logic, 'inline'): + logic.inline = True + self.logic = logic + + def __str__(self): + return "WHERE " + str(self.logic) + + +class GroupBy(ASTNode): + def __init__(self, logic): + if hasattr(logic, 'inline'): + logic.inline = True + self.logic = logic + + def __str__(self): + return " group by " + str(self.logic) + + +class From(ASTNode): + def __init__(self, _from): + if hasattr(_from, 'inline'): + _from.inline = True + self._from = _from + + def __str__(self): + return 'FROM {_from}'.format(_from=self._from) + + +class Select(ASTNode): + def __init__(self, _from, what='*', where='', groupby='', + limit=''): + self._from = "\"" + _from + "\"" + self.what = what + self.where = where and Where(where) + self.groupby = groupby + self.limit = limit and Limit(limit) + self.inlint = False + + def __str__(self): + s = 'SELECT ' + str(self.what) + ' FROM ' + str( + self._from) + ' ' + str(self.where) + str(self.groupby) + str( + self.limit) + if self.inline: + s = '(' + s + ')' + return s diff --git a/watcher/metrics_engine/framework/statedb_collector.py b/watcher/metrics_engine/framework/statedb_collector.py new file mode 100644 index 000000000..2f05fe8d6 --- /dev/null +++ b/watcher/metrics_engine/framework/statedb_collector.py @@ -0,0 +1,79 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Authors: Jean-Emile DARTOIS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +from oslo_config import cfg +from watcher.decision_engine.framework.model.hypervisor import Hypervisor +from watcher.decision_engine.framework.model.model_root import ModelRoot +from watcher.decision_engine.framework.model.resource import Resource +from watcher.decision_engine.framework.model.resource import ResourceType +from watcher.decision_engine.framework.model.vm import VM +from watcher.metrics_engine.api.cluster_state_collector import \ + ClusterStateCollector +from watcher.openstack.common import log + +CONF = cfg.CONF +LOG = log.getLogger(__name__) + + +class NovaCollector(ClusterStateCollector): + def __init__(self, wrapper): + self.wrapper = wrapper + + def get_latest_state_cluster(self): + + cluster = ModelRoot() + mem = Resource(ResourceType.memory) + num_cores = Resource(ResourceType.cpu_cores) + disk = Resource(ResourceType.disk) + cluster.create_resource(mem) + cluster.create_resource(num_cores) + cluster.create_resource(disk) + + flavor_cache = {} + hypervisors = self.wrapper.get_hypervisors_list() + for h in hypervisors: + service = self.wrapper.nova.services.find(id=h.service['id']) + # create hypervisor in stateDB + hypervisor = Hypervisor() + hypervisor.uuid = service.host + # set capacity + mem.set_capacity(hypervisor, h.memory_mb) + disk.set_capacity(hypervisor, h.free_disk_gb) + num_cores.set_capacity(hypervisor, h.vcpus) + hypervisor.state = h.state + hypervisor.status = h.status + cluster.add_hypervisor(hypervisor) + vms = self.wrapper.get_vms_by_hypervisor(str(service.host)) + for v in vms: + # create VM in stateDB + vm = VM() + vm.uuid = v.id + # nova/nova/compute/vm_states.py + vm.state = getattr(v, 'OS-EXT-STS:vm_state') + + # set capacity + self.wrapper.get_flavor_instance(v, flavor_cache) + mem.set_capacity(vm, v.flavor['ram']) + disk.set_capacity(vm, v.flavor['disk']) + num_cores.set_capacity(vm, v.flavor['vcpus']) + # print(dir(v)) + cluster.get_mapping().map(hypervisor, vm) + cluster.add_vm(vm) + return cluster diff --git a/watcher/opts.py b/watcher/opts.py index e7f20c5c1..c7fb8a7fe 100644 --- a/watcher/opts.py +++ b/watcher/opts.py @@ -20,10 +20,14 @@ import itertools import watcher.api.app from watcher.applier.framework import manager_applier import watcher.common.messaging.messaging_core + +import watcher.openstack.common.log + from watcher.decision_engine.framework import manager_decision_engine from watcher.decision_engine.framework.strategy import strategy_loader from watcher.decision_engine.framework.strategy import strategy_selector -import watcher.openstack.common.log +from watcher.metrics_engine.framework import collector_manager +from watcher.metrics_engine.framework.datasources import influxdb_collector def list_opts(): @@ -42,5 +46,9 @@ def list_opts(): ('watcher_decision_engine', manager_decision_engine.WATCHER_DECISION_ENGINE_OPTS), ('watcher_applier', - manager_applier.APPLIER_MANAGER_OPTS) + manager_applier.APPLIER_MANAGER_OPTS), + ('watcher_influxdb_collector', + influxdb_collector.WATCHER_INFLUXDB_COLLECTOR_OPTS), + ('watcher_metrics_collector', + collector_manager.WATCHER_METRICS_COLLECTOR_OPTS) ] diff --git a/watcher/service.py b/watcher/service.py index aa66ab131..6beed04ec 100644 --- a/watcher/service.py +++ b/watcher/service.py @@ -33,5 +33,5 @@ cfg.CONF.register_opts(service_opts) def prepare_service(args=None, conf=cfg.CONF): # log.register_options(conf) log.setup(conf, 'watcher') - conf(args, project='watcher') + conf(args, project='python-watcher') conf.log_opt_values(LOG, logging.DEBUG) diff --git a/watcher/tests/api/v1/test_actions.py b/watcher/tests/api/v1/test_actions.py index 1fc27d3a7..ea34fffe7 100644 --- a/watcher/tests/api/v1/test_actions.py +++ b/watcher/tests/api/v1/test_actions.py @@ -31,7 +31,7 @@ from watcher.tests.objects import utils as obj_utils def post_get_test_action(**kw): action = api_utils.action_post_data(**kw) action_plan = db_utils.get_test_action_plan() - action['action_plan_id'] = None + del action['action_plan_id'] action['action_plan_uuid'] = kw.get('action_plan_uuid', action_plan['uuid']) action['next'] = None diff --git a/watcher/tests/api/v1/test_goals.py b/watcher/tests/api/v1/test_goals.py new file mode 100644 index 000000000..fdf41b59e --- /dev/null +++ b/watcher/tests/api/v1/test_goals.py @@ -0,0 +1,63 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg +from watcher.tests.api import base as api_base + +CONF = cfg.CONF + + +class TestListGoal(api_base.FunctionalTest): + + def setUp(self): + super(TestListGoal, self).setUp() + + def _assert_goal_fields(self, goal): + goal_fields = ['name', 'strategy'] + for field in goal_fields: + self.assertIn(field, goal) + + def test_one(self): + response = self.get_json('/goals') + self._assert_goal_fields(response['goals'][0]) + + def test_get_one(self): + goal_name = CONF.watcher_goals.goals.keys()[0] + response = self.get_json('/goals/%s' % goal_name) + self.assertEqual(goal_name, response['name']) + self._assert_goal_fields(response) + + def test_detail(self): + goal_name = CONF.watcher_goals.goals.keys()[0] + response = self.get_json('/goals/detail') + self.assertEqual(goal_name, response['goals'][0]["name"]) + self._assert_goal_fields(response['goals'][0]) + + def test_detail_against_single(self): + goal_name = CONF.watcher_goals.goals.keys()[0] + response = self.get_json('/goals/%s/detail' % goal_name, + expect_errors=True) + self.assertEqual(404, response.status_int) + + def test_many(self): + response = self.get_json('/goals') + self.assertEqual(len(CONF.watcher_goals.goals), + len(response['goals'])) + + def test_collection_links(self): + response = self.get_json('/goals/?limit=2') + self.assertEqual(2, len(response['goals'])) + + def test_collection_links_default_limit(self): + cfg.CONF.set_override('max_limit', 3, 'api') + response = self.get_json('/goals') + self.assertEqual(3, len(response['goals'])) diff --git a/watcher/tests/applier/demo/test_applier.py b/watcher/tests/applier/demo/test_applier.py deleted file mode 100644 index 64a70aea6..000000000 --- a/watcher/tests/applier/demo/test_applier.py +++ /dev/null @@ -1,65 +0,0 @@ -# -*- encoding: utf-8 -*- -# Copyright (c) 2015 b<>com -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -from oslo_config import cfg - -from watcher.applier.framework.default_applier import DefaultApplier - -from watcher.common import utils -from watcher.decision_engine.framework.default_planner import DefaultPlanner -from watcher.decision_engine.strategies.basic_consolidation import \ - BasicConsolidation -from watcher.openstack.common import log -from watcher.tests.db import base -from watcher.tests.db import utils as db_utils -from watcher.tests.decision_engine.faker_cluster_state import \ - FakerStateCollector -from watcher.tests.decision_engine.faker_metrics_collector import \ - FakerMetricsCollector - -from oslo_config import cfg - -CONF = cfg.CONF -"" -class TestApplier(base.DbTestCase): - default_planner = DefaultPlanner() - - def create_solution(self): - metrics = FakerMetricsCollector() - current_state_cluster = FakerStateCollector() - sercon = BasicConsolidation() - sercon.set_metrics_resource_collector(metrics) - return sercon.execute(current_state_cluster.generate_scenario_1()) - - def test_scheduler_w(self): - CONF.debug = True - log.setup('watcher-sercon-demo') - - CONF.keystone_authtoken.auth_uri = "http://10.50.0.105:5000/v3" - CONF.keystone_authtoken.admin_user = "admin" - CONF.keystone_authtoken.admin_password = "openstacktest" - CONF.keystone_authtoken.admin_tenant_name = "test" - - audit = db_utils.create_test_audit(uuid=utils.generate_uuid()) - - action_plan = self.default_planner.schedule(self.context, - audit.id, - self.create_solution()) - - applier = DefaultApplier() - applier.execute(self.context, action_plan.uuid) -""""" diff --git a/watcher/tests/applier/demo/test_migrate.py b/watcher/tests/applier/demo/test_migrate.py deleted file mode 100644 index 1f748cf7b..000000000 --- a/watcher/tests/applier/demo/test_migrate.py +++ /dev/null @@ -1,99 +0,0 @@ -# -*- encoding: utf-8 -*- -# Copyright (c) 2015 b<>com -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -""" -from keystoneclient import session - -from keystoneclient.auth.identity import v3 - -import cinderclient.v2.client as ciclient -import glanceclient.v2.client as glclient -import keystoneclient.v3.client as ksclient -import neutronclient.neutron.client as netclient -import novaclient.v2.client as nvclient - -from watcher.common.utils import CONF -from oslo_config import cfg -from watcher.applier.framework.command.migrate_command import MigrateCommand -from watcher.applier.framework.command.wrapper.nova_wrapper import NovaWrapper -from watcher.decision_engine.framework.default_planner import Primitives -from watcher.openstack.common import log - -cfg.CONF.import_opt('auth_uri', 'keystoneclient.middleware.auth_token', - group='keystone_authtoken') -cfg.CONF.import_opt('admin_user', 'keystoneclient.middleware.auth_token', - group='keystone_authtoken') -cfg.CONF.import_opt('admin_password', 'keystoneclient.middleware.auth_token', - group='keystone_authtoken') -cfg.CONF.import_opt('admin_tenant_name', - 'keystoneclient.middleware.auth_token', - group='keystone_authtoken') - -cfg.CONF.keystone_authtoken.auth_uri = "http://10.50.0.105:5000/v3/" -cfg.CONF.keystone_authtoken.admin_user = "admin" -cfg.CONF.keystone_authtoken.admin_password = "openstacktest" -cfg.CONF.keystone_authtoken.admin_tenant_name = "test" - -try: - cfg.CONF.debug = True - log.setup('watcher-sercon-demo') - creds = \ - {'auth_url': CONF.keystone_authtoken.auth_uri, - 'username': CONF.keystone_authtoken.admin_user, - 'password': CONF.keystone_authtoken.admin_password, - 'project_name': CONF.keystone_authtoken.admin_tenant_name, - 'user_domain_name': "default", - 'project_domain_name': "default"} - auth = v3.Password(auth_url=creds['auth_url'], - username=creds['username'], - password=creds['password'], - project_name=creds['project_name'], - user_domain_name=creds[ - 'user_domain_name'], - project_domain_name=creds[ - 'project_domain_name']) - sess = session.Session(auth=auth) - nova = nvclient.Client("3", session=sess) - neutron = netclient.Client('2.0', session=sess) - neutron.format = 'json' - keystone = ksclient.Client(**creds) - - glance_endpoint = keystone. \ - service_catalog.url_for(service_type='image', - endpoint_type='publicURL') - glance = glclient.Client(glance_endpoint, - token=keystone.auth_token) - - cinder = ciclient.Client('2', session=sess) - wrapper = NovaWrapper(user=creds['username'], nova=nova, - neutron=neutron, glance=glance, - cinder=cinder) - instance = wrapper. \ - create_instance(hypervisor_id='ldev-indeedsrv006', - inst_name="demo_instance_1", - keypair_name='admin', - image_id= - "2b958331-379b-4618-b2ba-fbe8a608b2bb") - - cmd = MigrateCommand(instance.id, Primitives.COLD_MIGRATE, - 'ldev-indeedsrv006', - 'ldev-indeedsrv005') - resu = cmd.execute(cmd) - resu.result() - # wrapper.delete_instance(instance.id) -except Exception as e: - print("rollback " + unicode(e)) -""""" diff --git a/watcher/tests/applier/framework/command/test_launch_action_plan_command.py b/watcher/tests/applier/framework/command/test_launch_action_plan_command.py index 4ad336940..735c76805 100644 --- a/watcher/tests/applier/framework/command/test_launch_action_plan_command.py +++ b/watcher/tests/applier/framework/command/test_launch_action_plan_command.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# from mock import call from mock import MagicMock diff --git a/watcher/tests/applier/framework/command/wrapper/test_nova_wrapper.py b/watcher/tests/applier/framework/command/wrapper/test_nova_wrapper.py index 15cee4ec0..06fe39517 100644 --- a/watcher/tests/applier/framework/command/wrapper/test_nova_wrapper.py +++ b/watcher/tests/applier/framework/command/wrapper/test_nova_wrapper.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/watcher/tests/applier/framework/messaging/test_launch_action_plan_endpoint.py b/watcher/tests/applier/framework/messaging/test_launch_action_plan_endpoint.py index 65e992959..ff4feda59 100644 --- a/watcher/tests/applier/framework/messaging/test_launch_action_plan_endpoint.py +++ b/watcher/tests/applier/framework/messaging/test_launch_action_plan_endpoint.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# from mock import MagicMock diff --git a/watcher/tests/applier/framework/test_applier_manager.py b/watcher/tests/applier/framework/test_applier_manager.py index 54daab5aa..20da715de 100644 --- a/watcher/tests/applier/framework/test_applier_manager.py +++ b/watcher/tests/applier/framework/test_applier_manager.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/watcher/tests/applier/framework/test_command_executor.py b/watcher/tests/applier/framework/test_command_executor.py index 30dfea9d7..780f8d79d 100644 --- a/watcher/tests/applier/framework/test_command_executor.py +++ b/watcher/tests/applier/framework/test_command_executor.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/watcher/tests/applier/framework/test_command_mapper.py b/watcher/tests/applier/framework/test_command_mapper.py index e784b15b9..1f93d5ede 100644 --- a/watcher/tests/applier/framework/test_command_mapper.py +++ b/watcher/tests/applier/framework/test_command_mapper.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/watcher/tests/applier/framework/test_manager.py b/watcher/tests/applier/framework/test_manager.py index 71022df5d..ff876851d 100644 --- a/watcher/tests/applier/framework/test_manager.py +++ b/watcher/tests/applier/framework/test_manager.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# from watcher.decision_engine.framework.manager_decision_engine import \ diff --git a/watcher/tests/applier/framework/test_rpcapi.py b/watcher/tests/applier/framework/test_rpcapi.py index 2be8710ef..2056dfe33 100644 --- a/watcher/tests/applier/framework/test_rpcapi.py +++ b/watcher/tests/applier/framework/test_rpcapi.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# import mock import oslo.messaging as om @@ -42,12 +45,6 @@ class TestApplierAPI(base.TestCase): 'check_api_version', api_version=ApplierAPI().API_VERSION) - def test_execute_action_plan_throw_exception(self): - action_plan_uuid = "uuid" - self.assertRaises(exception.InvalidUuidOrName, - self.api.launch_action_plan, - action_plan_uuid) - def test_execute_audit_without_error(self): with mock.patch.object(om.RPCClient, 'call') as mock_call: action_plan_uuid = utils.generate_uuid() @@ -56,3 +53,9 @@ class TestApplierAPI(base.TestCase): self.context.to_dict(), 'launch_action_plan', action_plan_uuid=action_plan_uuid) + + def test_execute_action_plan_throw_exception(self): + action_plan_uuid = "uuid" + self.assertRaises(exception.InvalidUuidOrName, + self.api.launch_action_plan, + action_plan_uuid) diff --git a/watcher/tests/collector/__init__.py b/watcher/tests/collector/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/watcher/tests/collector/test_influxdb.py b/watcher/tests/collector/test_influxdb.py new file mode 100644 index 000000000..f4a02f9a6 --- /dev/null +++ b/watcher/tests/collector/test_influxdb.py @@ -0,0 +1,85 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Authors: Jean-Emile DARTOIS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import mock +from watcher.metrics_engine.api.metrics_resource_collector import \ + AggregationFunction +from watcher.metrics_engine.framework.datasources.influxdb_collector import \ + InfluxDBCollector + +from watcher.tests import base + + +class TestInfluxDB(base.TestCase): + def get_databases(self): + return {'name': 'indeed'} + + def test_get_measurement(self): + influx = InfluxDBCollector() + influx.get_client = mock.MagicMock() + influx.get_client.get_list_database = self.get_databases + result = influx.get_measurement("") + self.assertEqual(result, []) + + def test_build_query(self): + influx = InfluxDBCollector() + influx.get_client = mock.MagicMock() + query = influx.build_query("cpu_compute") + self.assertEqual(str(query), "SELECT * FROM \"cpu_compute\" ;") + + def test_build_query_aggregate(self): + influx = InfluxDBCollector() + influx.get_client = mock.MagicMock() + query = influx.build_query("cpu_compute", + aggregation_function=AggregationFunction. + COUNT) + self.assertEqual(str(query), + "SELECT count(value) FROM \"cpu_compute\" ;") + + def test_build_query_aggregate_intervals(self): + influx = InfluxDBCollector() + influx.get_client = mock.MagicMock() + query = influx.build_query("cpu_compute", + aggregation_function=AggregationFunction. + COUNT, + intervals="5m") + self.assertEqual(str(query), + "SELECT count(value) FROM \"cpu_compute\" " + "group by time(5m);") + + def test_build_query_aggregate_filters(self): + influx = InfluxDBCollector() + influx.get_client = mock.MagicMock() + filters = ['host=server1'] + query = influx.build_query("cpu_compute", + aggregation_function=AggregationFunction. + COUNT, + intervals="5m", + filters=filters) + self.assertEqual(str(query), 'SELECT count(value) FROM' + ' \"cpu_compute" WHERE' + ' host = \'server1\' group by time(5m);') + + def test_get_qusurement_start(self): + influx = InfluxDBCollector() + influx.get_client = mock.MagicMock() + influx.get_client.get_list_database = self.get_databases + result = influx.get_measurement("cpu_compute", start_time='now', + end_time="now") + self.assertEqual(result, []) diff --git a/watcher/tests/collector/test_nova_collector.py b/watcher/tests/collector/test_nova_collector.py new file mode 100644 index 000000000..31ea3eac5 --- /dev/null +++ b/watcher/tests/collector/test_nova_collector.py @@ -0,0 +1,43 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Authors: Jean-Emile DARTOIS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import mock +from watcher.metrics_engine.framework.statedb_collector import NovaCollector + +from watcher.tests import base + + +class TestNovaCollector(base.TestCase): + @mock.patch('keystoneclient.v3.client.Client') + def setUp(self, mock_ksclient): + super(TestNovaCollector, self).setUp() + self.wrapper = mock.MagicMock() + self.nova_collector = NovaCollector(self.wrapper) + + def test_nova_collector(self): + hypervisor = mock.Mock() + hypervisor.hypervisor_hostname = "rdev-lannion.eu" + hypervisor.service = mock.MagicMock() + service = mock.Mock() + service.host = "" + self.wrapper.get_hypervisors_list.return_value = {hypervisor} + self.wrapper.nova.services.find.get.return_value = service + model = self.nova_collector.get_latest_state_cluster() + self.assertIsNotNone(model) diff --git a/watcher/tests/collector/test_query.py b/watcher/tests/collector/test_query.py new file mode 100644 index 000000000..1de54e2de --- /dev/null +++ b/watcher/tests/collector/test_query.py @@ -0,0 +1,52 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Authors: Jean-Emile DARTOIS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from watcher.metrics_engine.framework.datasources.sql_ast.build_db_query import \ + DBQuery + +from watcher.tests import base + + +class TestDBQuery(base.TestCase): + + def test_query(self): + expected = "SELECT * FROM \"cpu_compute.cpu.user.percent_gauge\" ;" + query = DBQuery("cpu_compute.cpu.user.percent_gauge") + self.assertEqual(str(query), expected) + + def test_query_where(self): + expected = "SELECT * FROM" \ + " \"cpu_compute.cpu.user.percent_gauge\" WHERE host=jed;" + query = DBQuery("cpu_compute.cpu.user.percent_gauge").where( + "host=jed") + self.assertEqual(str(query), expected) + + def test_query_filter(self): + expected = "SELECT mean(value) FROM" \ + " \"cpu_compute.cpu.user.percent_gauge\" WHERE host=jed;" + query = DBQuery("cpu_compute.cpu.user.percent_gauge").where( + "host=jed").select("mean(value)") + self.assertEqual(str(query), expected) + + def test_query_groupby(self): + expected = "SELECT * FROM" \ + " \"cpu_compute.cpu.user.percent_gauge\" " \ + "group by time(5m);" + query = DBQuery("cpu_compute.cpu.user.percent_gauge").groupby( + "time(5m)") + self.assertEqual(str(query), expected) diff --git a/watcher/tests/common/messaging/utils/test_transport_url_builder.py b/watcher/tests/common/messaging/utils/test_transport_url_builder.py index 406febdda..49d17c667 100644 --- a/watcher/tests/common/messaging/utils/test_transport_url_builder.py +++ b/watcher/tests/common/messaging/utils/test_transport_url_builder.py @@ -31,7 +31,6 @@ class TestTransportUrlBuilder(base.TestCase): def test_transport_url_not_none(self): url = TransportUrlBuilder().url - print(url) self.assertIsNotNone(url, "The transport url must not be none") def test_transport_url_valid_pattern(self): diff --git a/watcher/tests/decision_engine/api/__init__.py b/watcher/tests/decision_engine/api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/watcher/tests/decision_engine/api/messaging/__init__.py b/watcher/tests/decision_engine/api/messaging/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/watcher/tests/decision_engine/api/messaging/test_decision_engine_command.py b/watcher/tests/decision_engine/api/messaging/test_decision_engine_command.py new file mode 100755 index 000000000..38c97ba76 --- /dev/null +++ b/watcher/tests/decision_engine/api/messaging/test_decision_engine_command.py @@ -0,0 +1,25 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from watcher.decision_engine.api.messaging.decision_engine_command import \ + DecisionEngineCommand +from watcher.tests import base + + +class TestDecisionEngineCommand(base.TestCase): + def test_execute(self): + DEC = DecisionEngineCommand() + self.assertRaises(NotImplementedError, DEC.execute) diff --git a/watcher/tests/decision_engine/api/messaging/test_event_consumer.py b/watcher/tests/decision_engine/api/messaging/test_event_consumer.py new file mode 100644 index 000000000..029645ac0 --- /dev/null +++ b/watcher/tests/decision_engine/api/messaging/test_event_consumer.py @@ -0,0 +1,30 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from watcher.decision_engine.api.messaging.event_consumer import EventConsumer +from watcher.tests import base + + +class TestEventConsumer(base.TestCase): + def test_set_messaging(self): + messaging = "test message" + EC = EventConsumer() + EC.set_messaging(messaging) + self.assertEqual(EC.messaging, messaging) + + def test_execute(self): + EC = EventConsumer() + self.assertRaises(NotImplementedError, EC.execute, None, None, None) diff --git a/watcher/tests/decision_engine/api/planner/__init__.py b/watcher/tests/decision_engine/api/planner/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/watcher/tests/decision_engine/api/planner/test_planner.py b/watcher/tests/decision_engine/api/planner/test_planner.py new file mode 100755 index 000000000..a95a59f5d --- /dev/null +++ b/watcher/tests/decision_engine/api/planner/test_planner.py @@ -0,0 +1,24 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from watcher.decision_engine.api.planner.planner import Planner +from watcher.tests import base + + +class TestPlanner(base.TestCase): + def test_schedule(self): + pl = Planner() + self.assertRaises(NotImplementedError, pl.schedule, None, None, None) diff --git a/watcher/tests/decision_engine/api/solution/__init__.py b/watcher/tests/decision_engine/api/solution/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/watcher/tests/decision_engine/api/solution/test_solution.py b/watcher/tests/decision_engine/api/solution/test_solution.py new file mode 100644 index 000000000..fa57fa152 --- /dev/null +++ b/watcher/tests/decision_engine/api/solution/test_solution.py @@ -0,0 +1,26 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +'''from watcher.decision_engine.api.solution.solution import Solution +from watcher.decision_engine.framework.model.model_root import ModelRoot +from watcher.tests import base + +class TestSolution(base.TestCase): + def test_get_model(self): + sol = Solution() + current_model = + sol.set_model(current_model) +''' diff --git a/watcher/tests/decision_engine/api/solution/test_solution_comparator.py b/watcher/tests/decision_engine/api/solution/test_solution_comparator.py new file mode 100755 index 000000000..bb119fd60 --- /dev/null +++ b/watcher/tests/decision_engine/api/solution/test_solution_comparator.py @@ -0,0 +1,30 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from watcher.decision_engine.api.solution.solution import Solution as sol +from watcher.decision_engine.api.solution.solution_comparator import Solution +from watcher.tests import base + + +class test_Solution_Comparator(base.TestCase): + def test_compare(self): + sol1 = sol() + sol2 = sol() + solution_comparator = Solution() + self.assertRaises(NotImplementedError, + solution_comparator.compare, + sol1, + sol2) diff --git a/watcher/tests/decision_engine/api/solution/test_solution_evaluator.py b/watcher/tests/decision_engine/api/solution/test_solution_evaluator.py new file mode 100755 index 000000000..8fb393049 --- /dev/null +++ b/watcher/tests/decision_engine/api/solution/test_solution_evaluator.py @@ -0,0 +1,25 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from watcher.decision_engine.api.solution.solution_evaluator import \ + SolutionEvaluator +from watcher.tests import base + + +class TestSolutionEvaluator(base.TestCase): + def test_evaluate(self): + SE = SolutionEvaluator() + self.assertRaises(NotImplementedError, SE.evaluate, None) diff --git a/watcher/tests/decision_engine/api/strategy/__init__.py b/watcher/tests/decision_engine/api/strategy/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/watcher/tests/decision_engine/api/strategy/test_meta_action.py b/watcher/tests/decision_engine/api/strategy/test_meta_action.py new file mode 100644 index 000000000..77c64929b --- /dev/null +++ b/watcher/tests/decision_engine/api/strategy/test_meta_action.py @@ -0,0 +1,30 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from watcher.decision_engine.api.strategy.meta_action import MetaAction +from watcher.tests import base + + +class TestMetaAction(base.TestCase): + def test_get_priority(self): + MA = MetaAction() + MA.set_priority(3) + self.assertEqual(MA.get_priority(), 3) + + def test_get_level(self): + MA = MetaAction() + MA.set_level(5) + self.assertEqual(MA.get_level(), 5) diff --git a/watcher/tests/decision_engine/api/strategy/test_selector.py b/watcher/tests/decision_engine/api/strategy/test_selector.py new file mode 100755 index 000000000..2d01d363c --- /dev/null +++ b/watcher/tests/decision_engine/api/strategy/test_selector.py @@ -0,0 +1,24 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from watcher.decision_engine.api.strategy.selector import Selector +from watcher.tests import base + + +class TestSelector(base.TestCase): + def test_define_from_goal(self): + Sel = Selector() + self.assertRaises(NotImplementedError, Sel.define_from_goal, None) diff --git a/watcher/tests/decision_engine/api/strategy/test_strategy_context.py b/watcher/tests/decision_engine/api/strategy/test_strategy_context.py new file mode 100644 index 000000000..bf020f6c3 --- /dev/null +++ b/watcher/tests/decision_engine/api/strategy/test_strategy_context.py @@ -0,0 +1,25 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from watcher.decision_engine.api.strategy.strategy_context import \ + StrategyContext +from watcher.tests import base + + +class TestStrategyContext(base.TestCase): + def test_execute_strategy(self): + SC = StrategyContext() + self.assertRaises(NotImplementedError, SC.execute_strategy, None) diff --git a/watcher/tests/decision_engine/demo/plot_consolidation_basic.py b/watcher/tests/decision_engine/demo/plot_consolidation_basic.py deleted file mode 100644 index 60a08e093..000000000 --- a/watcher/tests/decision_engine/demo/plot_consolidation_basic.py +++ /dev/null @@ -1,103 +0,0 @@ -# -*- encoding: utf-8 -*- -# Copyright (c) 2015 b<>com -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# FIXME(jed): remove this class due jenkins build failed -# The following librairies are removed from requirement.txt : -# - numpy -# - matplotlib -# These dependencies required a server x, jenkin's server has no -# server x - -# import matplotlib.pyplot as plt -# import numpy as np - - -from watcher.decision_engine.strategies.basic_consolidation import \ - BasicConsolidation -from watcher.tests.decision_engine.faker_cluster_state import \ - FakerStateCollector -from watcher.tests.decision_engine.faker_metrics_collector import \ - FakerMetricsCollector - - -class PlotConsolidationBasic(object): - def plot(self, sercon, orign_model, solution): - pass - -# cluster_size = len(orign_model._hypervisors) -# labels = [] -# before_score = [] -# after_score = [] -# for hypevisor_id in orign_model.get_all_hypervisors(): -# labels.append(hypevisor_id) -# hypevisor = orign_model.get_hypervisor_from_id(hypevisor_id) -# result_before = sercon.calculate_score_node(hypevisor, -# orign_model) -# result_after = sercon.calculate_score_node(hypevisor, -# solution.get_model()) -# before_score.append(float(result_before * 100)) -# if result_after == 0: -# result_after = 0 -# after_score.append(float(result_after * 100)) -# -# ind = np.arange(cluster_size) # the x locations for the groups -# width = 0.35 # the width of the bars -# -# fig, ax = plt.subplots() -# -# rects1 = ax.bar(ind, before_score, width, color='b') -# -# rects2 = ax.bar(ind + width, after_score, width, color='r') -# -# # add some text for labels, title and axes ticks -# ax.set_ylabel( -# 'Score of each hypervisor that represent their \ -# utilization level') -# ax.set_title('Watcher Basic Server consolidation (efficiency ' + str( -# sercon.get_solution().get_efficiency()) + " %)") -# -# ax.set_xticks(ind + width) -# ax.set_xticklabels(labels) -# ax.set_ylim([0, 140]) - -# ax.legend((rects1[0], rects2[0]), -# ('Before Consolidation', 'After Consolidation')) - -# def autolabel(rects): -# # attach some text labels -# for rect in rects: -# height = rect.get_height() -# ax.text(rect.get_x() + rect.get_width() / 2., 1.05 * height, -# '%d' % int(height), -# ha='center', va='bottom') -# -# autolabel(rects1) -# autolabel(rects2) - -# plt.show() - - -cluster = FakerStateCollector() -metrics = FakerMetricsCollector() -sercon = BasicConsolidation() -sercon.set_metrics_resource_collector(metrics) -# try overbooking ? :) 150 % cpu -sercon.set_threshold_cores(1) -model_cluster = cluster.generate_scenario_1() -solution = sercon.execute(model_cluster) -plot = PlotConsolidationBasic() -plot.plot(sercon, cluster.generate_scenario_1(), solution) diff --git a/watcher/tests/decision_engine/demo/test_context_strategy.py b/watcher/tests/decision_engine/demo/test_context_strategy.py deleted file mode 100644 index ae81f56ba..000000000 --- a/watcher/tests/decision_engine/demo/test_context_strategy.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- encoding: utf-8 -*- -# Copyright (c) 2015 b<>com -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from oslo_config import cfg -import time -from watcher.decision_engine.strategies.basic_consolidation import \ - BasicConsolidation - -from watcher.openstack.common import log - -from watcher.tests.decision_engine.faker_cluster_state import \ - FakerStateCollector -from watcher.tests.decision_engine.faker_metrics_collector import \ - FakerMetricsCollector - - -LOG = log.getLogger(__name__) - -cfg.CONF.debug = True -log.setup('metering-controller') - -metrics = FakerMetricsCollector() -current_state_cluster = FakerStateCollector() - -sercon = BasicConsolidation("basic", "Basic offline consolidation") -sercon.set_metrics_resource_collector(metrics) - -start_time = time.clock() -solution = sercon.execute(current_state_cluster.generate_scenario_1()) -print(time.clock() - start_time, "seconds") -print(solution) -# planner = DefaultPlanner() -# planner.schedule(solution) diff --git a/watcher/tests/decision_engine/demo/test_sercon.py b/watcher/tests/decision_engine/demo/test_sercon.py deleted file mode 100644 index dab9ffb4d..000000000 --- a/watcher/tests/decision_engine/demo/test_sercon.py +++ /dev/null @@ -1,43 +0,0 @@ -# -*- encoding: utf-8 -*- -# Copyright (c) 2015 b<>com -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from oslo_config import cfg -import time -from watcher.decision_engine.strategies.basic_consolidation import \ - BasicConsolidation - -from watcher.openstack.common import log - -from watcher.tests.decision_engine.faker_cluster_state import \ - FakerStateCollector -from watcher.tests.decision_engine.faker_metrics_collector import \ - FakerMetricsCollector - - -LOG = log.getLogger(__name__) -# debug on -cfg.CONF.debug = True -log.setup('metering-controller') - -metrics = FakerMetricsCollector() -current_state_cluster = FakerStateCollector() - -sercon = BasicConsolidation() -sercon.set_metrics_resource_collector(metrics) - -start_time = time.clock() -solution = sercon.execute(current_state_cluster.generate_scenario_1()) -print("duration =" + str((time.clock() - start_time)), "seconds") -LOG.debug(solution) diff --git a/watcher/tests/decision_engine/faker_cluster_state.py b/watcher/tests/decision_engine/faker_cluster_state.py index 65109e97e..8e0f0b494 100644 --- a/watcher/tests/decision_engine/faker_cluster_state.py +++ b/watcher/tests/decision_engine/faker_cluster_state.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,16 +15,17 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# import random -from watcher.decision_engine.api.collector.cluster_state_collector import \ - ClusterStateCollector from watcher.decision_engine.framework.model.hypervisor import Hypervisor from watcher.decision_engine.framework.model.model_root import ModelRoot from watcher.decision_engine.framework.model.resource import Resource from watcher.decision_engine.framework.model.resource import ResourceType from watcher.decision_engine.framework.model.vm import VM +from watcher.metrics_engine.api.cluster_state_collector import \ + ClusterStateCollector class FakerStateCollector(ClusterStateCollector): @@ -56,7 +59,7 @@ class FakerStateCollector(ClusterStateCollector): for i in range(0, count_node): node_uuid = "Node_" + str(i) hypervisor = Hypervisor() - hypervisor.set_uuid(node_uuid) + hypervisor.uuid = node_uuid mem.set_capacity(hypervisor, 132) disk.set_capacity(hypervisor, 250) num_cores.set_capacity(hypervisor, 40) @@ -66,7 +69,7 @@ class FakerStateCollector(ClusterStateCollector): for i in range(0, count_vm): vm_uuid = "VM_" + str(i) vm = VM() - vm.set_uuid(vm_uuid) + vm.uuid = vm_uuid # print("create "+str(vm)) mem.set_capacity(vm, 8) disk.set_capacity(vm, 10) @@ -107,7 +110,7 @@ class FakerStateCollector(ClusterStateCollector): for i in range(0, count_node): node_uuid = "Node_" + str(i) node = Hypervisor() - node.set_uuid(node_uuid) + node.uuid = node_uuid mem.set_capacity(node, 132) disk.set_capacity(node, 250) @@ -118,7 +121,7 @@ class FakerStateCollector(ClusterStateCollector): for i in range(0, count_vm): vm_uuid = "VM_" + str(i) vm = VM() - vm.set_uuid(vm_uuid) + vm.uuid = vm_uuid # print("create "+str(vm)) mem.set_capacity(vm, 2) disk.set_capacity(vm, 20) @@ -178,7 +181,7 @@ class FakerStateCollector(ClusterStateCollector): for i in range(0, count_node): node_uuid = "Node_" + str(i) node = Hypervisor() - node.set_uuid(node_uuid) + node.uuid = node_uuid mem.set_capacity(node, 132) disk.set_capacity(node, 250) num_cores.set_capacity(node, 40) @@ -215,7 +218,7 @@ class FakerStateCollector(ClusterStateCollector): for i in range(0, count_node): node_uuid = "Node_" + str(i) node = Hypervisor() - node.set_uuid(node_uuid) + node.uuid = node_uuid mem.set_capacity(node, 132) disk.set_capacity(node, 250) num_cores.set_capacity(node, 40) @@ -225,14 +228,13 @@ class FakerStateCollector(ClusterStateCollector): for i in range(0, count_vm): vm_uuid = "VM_" + str(i) vm = VM() - vm.set_uuid(vm_uuid) + vm.uuid = vm_uuid # print("create "+str(vm)) mem.set_capacity(vm, 10) disk.set_capacity(vm, 25) num_cores.set_capacity(vm, 16) vms.append(vm) current_state_cluster.add_vm(vm) - print(count_vm) indice = 0 for j in range(0, 2): node_uuid = "Node_" + str(j) @@ -253,3 +255,84 @@ class FakerStateCollector(ClusterStateCollector): self.map(current_state_cluster, node_uuid, vm_uuid) return current_state_cluster + + def generate_scenario_4_with_2_hypervisors(self): + vms = [] + + current_state_cluster = ModelRoot() + # number of nodes + count_node = 2 + # number max of vm per node + node_count_vm = 1 + # total number of virtual machine + count_vm = (count_node * node_count_vm) + + # define ressouce ( CPU, MEM disk, ... ) + mem = Resource(ResourceType.memory) + # 2199.954 Mhz + num_cores = Resource(ResourceType.cpu_cores) + disk = Resource(ResourceType.disk) + + current_state_cluster.create_resource(mem) + current_state_cluster.create_resource(num_cores) + current_state_cluster.create_resource(disk) + + for i in range(0, count_node): + node_uuid = "Node_" + str(i) + node = Hypervisor() + node.uuid = node_uuid + + mem.set_capacity(node, 132) + disk.set_capacity(node, 250) + num_cores.set_capacity(node, 40) + # print("create "+str(node)) + current_state_cluster.add_hypervisor(node) + + for i in range(0, count_vm): + vm_uuid = "VM_" + str(i) + vm = VM() + vm.uuid = vm_uuid + # print("create "+str(vm)) + mem.set_capacity(vm, 2) + disk.set_capacity(vm, 20) + num_cores.set_capacity(vm, 10) + vms.append(vm) + current_state_cluster.add_vm(vm) + + current_state_cluster.get_mapping().map( + current_state_cluster.get_hypervisor_from_id("Node_0"), + current_state_cluster.get_vm_from_id("VM_0")) + + current_state_cluster.get_mapping().map( + current_state_cluster.get_hypervisor_from_id("Node_1"), + current_state_cluster.get_vm_from_id("VM_1")) + + return current_state_cluster + + def generate_scenario_5_with_1_hypervisor_no_vm(self): + current_state_cluster = ModelRoot() + # number of nodes + count_node = 1 + + # define ressouce ( CPU, MEM disk, ... ) + mem = Resource(ResourceType.memory) + # 2199.954 Mhz + num_cores = Resource(ResourceType.cpu_cores) + disk = Resource(ResourceType.disk) + + current_state_cluster.create_resource(mem) + current_state_cluster.create_resource(num_cores) + current_state_cluster.create_resource(disk) + + for i in range(0, count_node): + node_uuid = "Node_" + str(i) + node = Hypervisor() + node.uuid = node_uuid + + mem.set_capacity(node, 1) + disk.set_capacity(node, 1) + num_cores.set_capacity(node, 1) + # print("create "+str(node)) + current_state_cluster.add_hypervisor(node) + + return current_state_cluster diff --git a/watcher/tests/decision_engine/faker_metrics_collector.py b/watcher/tests/decision_engine/faker_metrics_collector.py index 96b1b3d5f..6a73bf50b 100644 --- a/watcher/tests/decision_engine/faker_metrics_collector.py +++ b/watcher/tests/decision_engine/faker_metrics_collector.py @@ -1,6 +1,8 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,15 +15,130 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# import random -from watcher.decision_engine.api.collector.metrics_resource_collector import \ + +from watcher.metrics_engine.api.metrics_resource_collector import Measure +from watcher.metrics_engine.api.metrics_resource_collector import \ MetricsResourceCollector class FakerMetricsCollector(MetricsResourceCollector): def __init__(self): - pass + self.emptytype = "" + + def empty_one_metric(self, emptytype): + self.emptytype = emptytype + + def get_measurement(self, + metric, + callback=None, + start_time=None, + end_time=None, + filters=None, + aggregation_function=None, + intervals=None): + + results = [] + if metric == "compute_cpu_user_percent_gauge": + if self.emptytype == "CPU_COMPUTE": + pass + else: + results.append(Measure(0, 5)) + elif metric == "instance_cpu_percent_gauge": + results.append( + self.get_average_usage_vm_cpu(filters[0].split('=')[1])) + elif metric == "instance_memory_resident_used_bytes_gauge": + results.append( + self.get_average_usage_vm_memory(filters[0].split('=')[1])) + elif metric == "instance_disk_used_bytes_gauge": + if self.emptytype == "DISK_COMPUTE": + pass + else: + results.append( + self.get_average_usage_vm_disk(filters[0].split('=')[1])) + elif metric == "compute_memory_used_bytes_gauge": + if self.emptytype == "MEM_COMPUTE": + pass + else: + results.append(self.get_usage_node_cpu( + filters[0].split('=')[1])) + elif metric == "compute_disk_size_used_bytes_gauge": + if self.emptytype == "DISK_COMPUTE": + pass + else: + results.append(self.get_usage_node_disk( + filters[0].split('=')[1])) + else: + results.append(Measure(0, 0)) + return results + + def get_usage_node_disk(self, uuid): + """The last VM CPU usage values to average + + :param uuid:00 + :return: + """ + # query influxdb stream + + # compute in stream + + # Normalize + mock = {} + # node 0 + mock['Node_0'] = Measure(0, 7) + mock['Node_1'] = Measure(0, 100) + # node 1 + mock['Node_2'] = Measure(0, 10) + # node 2 + mock['Node_3'] = Measure(0, 5) + mock['Node_4'] = Measure(0, 5) + mock['Node_5'] = Measure(0, 10) + + # node 3 + mock['Node_6'] = Measure(0, 8) + + # node 4 + mock['VM_7'] = Measure(0, 4) + if uuid not in mock.keys(): + # mock[uuid] = random.randint(1, 4) + mock[uuid] = Measure(0, 8) + + return mock[str(uuid)] + + def get_usage_node_cpu(self, uuid): + """The last VM CPU usage values to average + + :param uuid:00 + :return: + """ + # query influxdb stream + + # compute in stream + + # Normalize + mock = {} + # node 0 + mock['Node_0'] = Measure(0, 7) + mock['Node_1'] = Measure(0, 7) + # node 1 + mock['Node_2'] = Measure(0, 80) + # node 2 + mock['Node_3'] = Measure(0, 5) + mock['Node_4'] = Measure(0, 5) + mock['Node_5'] = Measure(0, 10) + + # node 3 + mock['Node_6'] = Measure(0, 8) + + # node 4 + mock['VM_7'] = Measure(0, 4) + if uuid not in mock.keys(): + # mock[uuid] = random.randint(1, 4) + mock[uuid] = Measure(0, 8) + + return mock[str(uuid)] def get_average_usage_vm_cpu(self, uuid): """The last VM CPU usage values to average @@ -36,70 +153,70 @@ class FakerMetricsCollector(MetricsResourceCollector): # Normalize mock = {} # node 0 - mock['VM_0'] = 7 - mock['VM_1'] = 7 + mock['VM_0'] = Measure(0, 7) + mock['VM_1'] = Measure(0, 7) # node 1 - mock['VM_2'] = 10 + mock['VM_2'] = Measure(0, 10) # node 2 - mock['VM_3'] = 5 - mock['VM_4'] = 5 - mock['VM_5'] = 10 + mock['VM_3'] = Measure(0, 5) + mock['VM_4'] = Measure(0, 5) + mock['VM_5'] = Measure(0, 10) # node 3 - mock['VM_6'] = 8 + mock['VM_6'] = Measure(0, 8) # node 4 - mock['VM_7'] = 4 + mock['VM_7'] = Measure(0, 4) if uuid not in mock.keys(): # mock[uuid] = random.randint(1, 4) - mock[uuid] = 8 + mock[uuid] = Measure(0, 8) return mock[str(uuid)] def get_average_usage_vm_memory(self, uuid): mock = {} # node 0 - mock['VM_0'] = 2 - mock['VM_1'] = 5 + mock['VM_0'] = Measure(0, 2) + mock['VM_1'] = Measure(0, 5) # node 1 - mock['VM_2'] = 5 + mock['VM_2'] = Measure(0, 5) # node 2 - mock['VM_3'] = 8 - mock['VM_4'] = 5 - mock['VM_5'] = 16 + mock['VM_3'] = Measure(0, 8) + mock['VM_4'] = Measure(0, 5) + mock['VM_5'] = Measure(0, 16) # node 3 - mock['VM_6'] = 8 + mock['VM_6'] = Measure(0, 8) # node 4 - mock['VM_7'] = 4 + mock['VM_7'] = Measure(0, 4) if uuid not in mock.keys(): # mock[uuid] = random.randint(1, 4) - mock[uuid] = 10 + mock[uuid] = Measure(0, 10) return mock[str(uuid)] def get_average_usage_vm_disk(self, uuid): mock = {} # node 0 - mock['VM_0'] = 2 - mock['VM_1'] = 2 + mock['VM_0'] = Measure(0, 2) + mock['VM_1'] = Measure(0, 2) # node 1 - mock['VM_2'] = 2 + mock['VM_2'] = Measure(0, 2) # node 2 - mock['VM_3'] = 10 - mock['VM_4'] = 15 - mock['VM_5'] = 20 + mock['VM_3'] = Measure(0, 10) + mock['VM_4'] = Measure(0, 15) + mock['VM_5'] = Measure(0, 20) # node 3 - mock['VM_6'] = 8 + mock['VM_6'] = Measure(0, 8) # node 4 - mock['VM_7'] = 4 + mock['VM_7'] = Measure(0, 4) if uuid not in mock.keys(): # mock[uuid] = random.randint(1, 4) - mock[uuid] = 4 + mock[uuid] = Measure(0, 4) return mock[str(uuid)] diff --git a/watcher/tests/decision_engine/framework/command/test_trigger_audit_command.py b/watcher/tests/decision_engine/framework/command/test_trigger_audit_command.py index 64ce087aa..5a2b2f5e4 100644 --- a/watcher/tests/decision_engine/framework/command/test_trigger_audit_command.py +++ b/watcher/tests/decision_engine/framework/command/test_trigger_audit_command.py @@ -39,14 +39,21 @@ class TestTriggerAuditCommand(DbTestCase): self.context, audit_template_id=self.audit_template.id) - def test_trigger_audit_wihout_errors(self): + def test_trigger_audit_without_errors(self): try: statedb = FakerStateCollector() ressourcedb = FakerMetricsCollector() command = TriggerAuditCommand(MagicMock(), statedb, ressourcedb) command.execute(self.audit.uuid, self.context) except Exception: - self.fail("The audit should be trigged wihtour error") + self.fail("The audit should be trigged without error") + + def test_trigger_audit_with_errors(self): + try: + command = TriggerAuditCommand(MagicMock(), 0, 0) + command.execute(self.audit.uuid, self.context) + except Exception: + self.fail("The audit should be trigged with error") def test_trigger_audit_state_succes(self): statedb = FakerStateCollector() diff --git a/watcher/tests/decision_engine/framework/messaging/test_audit_endpoint.py b/watcher/tests/decision_engine/framework/messaging/test_audit_endpoint.py index 5fef8a92e..8824de58d 100644 --- a/watcher/tests/decision_engine/framework/messaging/test_audit_endpoint.py +++ b/watcher/tests/decision_engine/framework/messaging/test_audit_endpoint.py @@ -13,7 +13,6 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. -""" import mock from mock import MagicMock from watcher.common import utils @@ -21,22 +20,54 @@ from watcher.decision_engine.framework.command.trigger_audit_command import \ TriggerAuditCommand from watcher.decision_engine.framework.messaging.audit_endpoint import \ AuditEndpoint +from watcher.metrics_engine.framework.collector_manager import CollectorManager from watcher.tests import base +from watcher.tests.decision_engine.faker_cluster_state import \ + FakerStateCollector +from watcher.tests.decision_engine.faker_metrics_collector import \ + FakerMetricsCollector + + +class TriggerAuditCommandWithExecutor(TriggerAuditCommand): + def setUp(self): + super(TriggerAuditCommand, self).setUp() + + def executor(self): + pass class TestAuditEndpoint(base.TestCase): - def setUp(self): super(TestAuditEndpoint, self).setUp() self.endpoint = AuditEndpoint(MagicMock()) + def test_do_trigger_audit(self): + audit_uuid = utils.generate_uuid() + statedb = FakerStateCollector() + ressourcedb = FakerMetricsCollector() + command = TriggerAuditCommand(MagicMock(), statedb, ressourcedb) + endpoint = AuditEndpoint(command) + + with mock.patch.object(CollectorManager, 'get_statedb_collector') \ + as mock_call2: + mock_call2.return_value = 0 + + with mock.patch.object(TriggerAuditCommand, 'execute') \ + as mock_call: + mock_call.return_value = 0 + endpoint.do_trigger_audit(command, audit_uuid) + # mock_call.assert_called_once_with() + mock_call2.assert_called_once_with() + def test_trigger_audit(self): audit_uuid = utils.generate_uuid() - # todo() add + statedb = FakerStateCollector() + ressourcedb = FakerMetricsCollector() + command = TriggerAuditCommandWithExecutor(MagicMock(), + statedb, ressourcedb) + endpoint = AuditEndpoint(command) - with mock.patch.object(TriggerAuditCommand, 'execute') as mock_call: - expected_uuid = self.endpoint.trigger_audit( - self.context, audit_uuid) - self.assertEqual(audit_uuid, expected_uuid) - mock_call.assert_called_once_with(audit_uuid, self.context) -""" + with mock.patch.object(TriggerAuditCommandWithExecutor, 'executor') \ + as mock_call: + mock_call.return_value = 0 + endpoint.trigger_audit(command, audit_uuid) diff --git a/watcher/tests/decision_engine/framework/meta_actions/__init__.py b/watcher/tests/decision_engine/framework/meta_actions/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/watcher/tests/decision_engine/framework/meta_actions/test_migrate.py b/watcher/tests/decision_engine/framework/meta_actions/test_migrate.py new file mode 100644 index 000000000..34d085169 --- /dev/null +++ b/watcher/tests/decision_engine/framework/meta_actions/test_migrate.py @@ -0,0 +1,32 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Authors: Jean-Emile DARTOIS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from watcher.decision_engine.framework.meta_actions.migrate import Migrate +from watcher.decision_engine.framework.model.hypervisor import Hypervisor +from watcher.decision_engine.framework.model.vm import VM +from watcher.tests import base + + +class TestMigtrate(base.BaseTestCase): + def test_set_get_bandwidth(self): + vm = VM() + hyp_src = Hypervisor() + hyp_dst = Hypervisor() + mig = Migrate(vm, hyp_src, hyp_dst) + mig.set_bandwidth(2) + self.assertEqual(mig.get_bandwidth(), 2) diff --git a/watcher/tests/decision_engine/framework/model/__init__.py b/watcher/tests/decision_engine/framework/model/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/watcher/tests/decision_engine/framework/model/test_diskinfo.py b/watcher/tests/decision_engine/framework/model/test_diskinfo.py new file mode 100644 index 000000000..143e4198c --- /dev/null +++ b/watcher/tests/decision_engine/framework/model/test_diskinfo.py @@ -0,0 +1,33 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Authors: Jean-Emile DARTOIS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from watcher.decision_engine.framework.model.diskInfo import DiskInfo +from watcher.tests import base + + +class TestDiskInfo(base.BaseTestCase): + def test_all(self): + disk_information = DiskInfo() + disk_information.set_size(1024) + self.assertEqual(disk_information.get_size(), 1024) + + disk_information.set_scheduler = "scheduler_qcq" + + disk_information.set_device_name("nom_qcq") + self.assertEqual(disk_information.get_device_name(), "nom_qcq") diff --git a/watcher/tests/decision_engine/framework/model/test_mapping.py b/watcher/tests/decision_engine/framework/model/test_mapping.py new file mode 100644 index 000000000..b902fc635 --- /dev/null +++ b/watcher/tests/decision_engine/framework/model/test_mapping.py @@ -0,0 +1,106 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Authors: Jean-Emile DARTOIS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import uuid +from watcher.decision_engine.framework.model.hypervisor import Hypervisor +from watcher.decision_engine.framework.model.vm_state import VMState +from watcher.tests import base +from watcher.tests.decision_engine.faker_cluster_state import \ + FakerStateCollector + + +class TestMapping(base.BaseTestCase): + def test_get_node_from_vm(self): + fake_cluster = FakerStateCollector() + model = fake_cluster.generate_scenario_4_with_2_hypervisors() + + vms = model.get_all_vms() + keys = vms.keys() + vm = vms[keys[0]] + if vm.uuid != 'VM_0': + vm = vms[keys[1]] + node = model.mapping.get_node_from_vm(vm) + self.assertEqual(node.uuid, 'Node_0') + + def test_get_node_from_vm_id(self): + fake_cluster = FakerStateCollector() + model = fake_cluster.generate_scenario_4_with_2_hypervisors() + + hyps = model.mapping.get_node_vms_from_id("BLABLABLA") + self.assertEqual(hyps.__len__(), 0) + + def test_get_all_vms(self): + fake_cluster = FakerStateCollector() + model = fake_cluster.generate_scenario_4_with_2_hypervisors() + + vms = model.get_all_vms() + self.assertEqual(vms.__len__(), 2) + self.assertEqual(vms['VM_0'].state, VMState.ACTIVE.value) + self.assertEqual(vms['VM_0'].uuid, 'VM_0') + self.assertEqual(vms['VM_1'].state, VMState.ACTIVE.value) + self.assertEqual(vms['VM_1'].uuid, 'VM_1') + + def test_get_mapping(self): + fake_cluster = FakerStateCollector() + model = fake_cluster.generate_scenario_4_with_2_hypervisors() + + mapping_vm = model.mapping.get_mapping_vm() + self.assertEqual(mapping_vm.__len__(), 2) + self.assertEqual(mapping_vm['VM_0'], 'Node_0') + self.assertEqual(mapping_vm['VM_1'], 'Node_1') + + def test_migrate_vm(self): + fake_cluster = FakerStateCollector() + model = fake_cluster.generate_scenario_4_with_2_hypervisors() + vms = model.get_all_vms() + keys = vms.keys() + vm0 = vms[keys[0]] + hyp0 = model.mapping.get_node_from_vm_id(vm0.uuid) + vm1 = vms[keys[1]] + hyp1 = model.mapping.get_node_from_vm_id(vm1.uuid) + + self.assertEqual(model.mapping.migrate_vm(vm1, hyp1, hyp1), False) + self.assertEqual(model.mapping.migrate_vm(vm1, hyp0, hyp0), False) + self.assertEqual(model.mapping.migrate_vm(vm1, hyp1, hyp0), True) + self.assertEqual(model.mapping.migrate_vm(vm1, hyp0, hyp1), True) + + def test_unmap_from_id_log_warning(self): + fake_cluster = FakerStateCollector() + model = fake_cluster.generate_scenario_4_with_2_hypervisors() + vms = model.get_all_vms() + keys = vms.keys() + vm0 = vms[keys[0]] + id = str(uuid.uuid4()) + hypervisor = Hypervisor() + hypervisor.uuid = id + + model.mapping.unmap_from_id(hypervisor.uuid, vm0.uuid) + # self.assertEqual(len(model.mapping.get_node_vms_from_id( + # hypervisor.uuid)), 1) + + def test_unmap_from_id(self): + fake_cluster = FakerStateCollector() + model = fake_cluster.generate_scenario_4_with_2_hypervisors() + vms = model.get_all_vms() + keys = vms.keys() + vm0 = vms[keys[0]] + hyp0 = model.mapping.get_node_from_vm_id(vm0.uuid) + + model.mapping.unmap_from_id(hyp0.uuid, vm0.uuid) + self.assertEqual(len(model.mapping.get_node_vms_from_id( + hyp0.uuid)), 0) diff --git a/watcher/tests/decision_engine/framework/model/test_model.py b/watcher/tests/decision_engine/framework/model/test_model.py new file mode 100644 index 000000000..3f92532b7 --- /dev/null +++ b/watcher/tests/decision_engine/framework/model/test_model.py @@ -0,0 +1,139 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Authors: Jean-Emile DARTOIS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import uuid +from watcher.common import exception +from watcher.common.exception import IllegalArgumentException +from watcher.decision_engine.framework.model.hypervisor import Hypervisor +from watcher.decision_engine.framework.model.hypervisor_state import \ + HypervisorState +from watcher.decision_engine.framework.model.model_root import ModelRoot +from watcher.tests.decision_engine.faker_cluster_state import \ + FakerStateCollector + +from watcher.tests import base + + +class TestModel(base.BaseTestCase): + def test_model(self): + fake_cluster = FakerStateCollector() + model = fake_cluster.generate_scenario_1() + + self.assertEqual(len(model._hypervisors), 5) + self.assertEqual(len(model._vms), 35) + self.assertEqual(len(model.get_mapping().get_mapping()), 5) + + def test_add_hypervisor(self): + model = ModelRoot() + id = str(uuid.uuid4()) + hypervisor = Hypervisor() + hypervisor.uuid = id + model.add_hypervisor(hypervisor) + self.assertEqual(model.get_hypervisor_from_id(id), hypervisor) + + def test_delete_hypervisor(self): + model = ModelRoot() + id = str(uuid.uuid4()) + hypervisor = Hypervisor() + hypervisor.uuid = id + model.add_hypervisor(hypervisor) + self.assertEqual(model.get_hypervisor_from_id(id), hypervisor) + model.remove_hypervisor(hypervisor) + self.assertRaises(exception.HypervisorNotFound, + model.get_hypervisor_from_id, id) + + def test_get_all_hypervisors(self): + model = ModelRoot() + for i in range(10): + id = str(uuid.uuid4()) + hypervisor = Hypervisor() + hypervisor.uuid = id + model.add_hypervisor(hypervisor) + all_hypervisors = model.get_all_hypervisors() + for id in all_hypervisors: + hyp = model.get_hypervisor_from_id(id) + model.assert_hypervisor(hyp) + + def test_set_get_state_hypervisors(self): + model = ModelRoot() + id = str(uuid.uuid4()) + hypervisor = Hypervisor() + hypervisor.uuid = id + model.add_hypervisor(hypervisor) + + self.assertIsInstance(hypervisor.state, HypervisorState) + + hyp = model.get_hypervisor_from_id(id) + hyp.state = HypervisorState.OFFLINE + self.assertIsInstance(hyp.state, HypervisorState) + + # /watcher/decision_engine/framework/model/hypervisor.py + # set_state accept any char chain. + # verification (IsInstance) should be used in the function + # hyp.set_state('blablabla') + # self.assertEqual(hyp.get_state(), 'blablabla') + # self.assertIsInstance(hyp.get_state(), HypervisorState) + + # def test_get_all_vms(self): + # model = ModelRoot() + # vms = model.get_all_vms() + # self.assert(len(model._vms)) + def test_hypervisor_from_id_raise(self): + model = ModelRoot() + id = str(uuid.uuid4()) + hypervisor = Hypervisor() + hypervisor.uuid = id + model.add_hypervisor(hypervisor) + + id2 = str(uuid.uuid4()) + self.assertRaises(exception.HypervisorNotFound, + model.get_hypervisor_from_id, id2) + + def test_remove_hypervisor_raise(self): + model = ModelRoot() + id = str(uuid.uuid4()) + hypervisor = Hypervisor() + hypervisor.uuid = id + model.add_hypervisor(hypervisor) + + id2 = str(uuid.uuid4()) + hypervisor2 = Hypervisor() + hypervisor2.uuid = id2 + + self.assertRaises(exception.HypervisorNotFound, + model.remove_hypervisor, hypervisor2) + + def test_assert_hypervisor_raise(self): + model = ModelRoot() + id = str(uuid.uuid4()) + hypervisor = Hypervisor() + hypervisor.uuid = id + model.add_hypervisor(hypervisor) + self.assertRaises(IllegalArgumentException, + model.assert_hypervisor, "objet_qcq") + + def test_vm_from_id_raise(self): + fake_cluster = FakerStateCollector() + model = fake_cluster.generate_scenario_1() + self.assertRaises(exception.VMNotFound, + model.get_vm_from_id, "valeur_qcq") + + def test_assert_vm_raise(self): + model = ModelRoot() + self.assertRaises(IllegalArgumentException, + model.assert_vm, "valeur_qcq") diff --git a/watcher/tests/decision_engine/framework/model/test_named_element.py b/watcher/tests/decision_engine/framework/model/test_named_element.py new file mode 100644 index 000000000..0c5d00cd6 --- /dev/null +++ b/watcher/tests/decision_engine/framework/model/test_named_element.py @@ -0,0 +1,32 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Authors: Jean-Emile DARTOIS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from watcher.decision_engine.framework.model.named_element import NamedElement +from watcher.tests import base + + +class TestNamedElement(base.BaseTestCase): + def test_namedelement(self): + id = NamedElement() + id.uuid = "BLABLABLA" + self.assertEqual(id.uuid, "BLABLABLA") + + def test_set_get_human_id(self): + id = NamedElement() + id.human_id = "BLABLABLA" + self.assertEqual(id.human_id, "BLABLABLA") diff --git a/watcher/tests/decision_engine/framework/model/test_vm.py b/watcher/tests/decision_engine/framework/model/test_vm.py new file mode 100644 index 000000000..0c0a4b8a1 --- /dev/null +++ b/watcher/tests/decision_engine/framework/model/test_vm.py @@ -0,0 +1,30 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Authors: Jean-Emile DARTOIS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from watcher.decision_engine.framework.model.vm import VM +from watcher.decision_engine.framework.model.vm_state import VMState +from watcher.tests import base + + +class TestVm(base.BaseTestCase): + def test_namedelement(self): + vm = VM() + vm.state = VMState.ACTIVE + self.assertEqual(vm.state, VMState.ACTIVE) + vm.human_id = "human_05" + self.assertEqual(vm.human_id, "human_05") diff --git a/watcher/tests/decision_engine/framework/strategy/test_strategy_loader.py b/watcher/tests/decision_engine/framework/strategy/test_strategy_loader.py index 92d608b7a..c5a311112 100644 --- a/watcher/tests/decision_engine/framework/strategy/test_strategy_loader.py +++ b/watcher/tests/decision_engine/framework/strategy/test_strategy_loader.py @@ -34,3 +34,7 @@ class TestStrategySelector(base.BaseTestCase): selected_strategy.get_name(), exptected_strategy, 'The default strategy should be basic') + + def test_load_driver(self): + algo = self.strategy_loader.load_driver("basic") + self.assertEqual(algo._names[0], "basic") diff --git a/watcher/decision_engine/framework/client_selector_strategy.py b/watcher/tests/decision_engine/framework/strategy/test_strategy_manager_impl.py similarity index 52% rename from watcher/decision_engine/framework/client_selector_strategy.py rename to watcher/tests/decision_engine/framework/strategy/test_strategy_manager_impl.py index 6532f0dda..dab06397b 100644 --- a/watcher/decision_engine/framework/client_selector_strategy.py +++ b/watcher/tests/decision_engine/framework/strategy/test_strategy_manager_impl.py @@ -13,19 +13,20 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. -from watcher.common.messaging.messaging_core import MessagingCore -from watcher.decision_engine.api.selector.selector import Selector + +from watcher.decision_engine.framework.strategy.StrategyManagerImpl import \ + StrategyContextImpl +from watcher.tests import base -class ClientSelectorStrategy(Selector, MessagingCore): +class FakeStrategy(object): + def __init__(self): + self.name = "BALANCE_LOAD" - """Trigger an audit (a request for optimizing a cluster) - :param goal: the strategy selected by the strategy selector - :param hosts: the list of hypervisors where a nova-compute service - is running (host aggregate) - :return: None - """ - def launch_audit(self, goal): - # TODO(jed): - # client = ClientScheduler() - pass + +class TestStrategyContextImpl(base.BaseTestCase): + def test_add_remove_strategy(self): + strategy = FakeStrategy() + strategy_context = StrategyContextImpl() + strategy_context.add_strategy(strategy) + strategy_context.remove_strategy(strategy) diff --git a/watcher/tests/decision_engine/framework/test_default_planner.py b/watcher/tests/decision_engine/framework/test_default_planner.py index 0f92e474e..87ce98fba 100644 --- a/watcher/tests/decision_engine/framework/test_default_planner.py +++ b/watcher/tests/decision_engine/framework/test_default_planner.py @@ -15,6 +15,7 @@ # limitations under the License. import mock +from watcher.common.exception import MetaActionNotFound from watcher.common import utils from watcher.db import api as db_api from watcher.decision_engine.framework.default_planner import DefaultPlanner @@ -39,6 +40,17 @@ class SolutionFaker(object): return sercon.execute(current_state_cluster.generate_scenario_1()) +class SolutionFakerSingleHyp(object): + @staticmethod + def build(): + metrics = FakerMetricsCollector() + current_state_cluster = FakerStateCollector() + sercon = BasicConsolidation("basic", "Basic offline consolidation") + sercon.set_metrics_resource_collector(metrics) + return sercon.execute( + current_state_cluster.generate_scenario_4_with_2_hypervisors()) + + class TestDefaultPlanner(base.DbTestCase): default_planner = DefaultPlanner() @@ -71,5 +83,25 @@ class TestDefaultPlanner(base.DbTestCase): fake_solution = SolutionFaker.build() action_plan = self.default_planner.schedule(self.context, audit.id, fake_solution) - + self.assertIsNotNone(action_plan.uuid) + + def test_schedule_raise(self): + audit = db_utils.create_test_audit(uuid=utils.generate_uuid()) + fake_solution = SolutionFaker.build() + fake_solution._meta_actions[0] = "valeur_qcq" + self.assertRaises(MetaActionNotFound, self.default_planner.schedule, + self.context, audit.id, fake_solution) + + def test_schedule_scheduled_empty(self): + audit = db_utils.create_test_audit(uuid=utils.generate_uuid()) + fake_solution = SolutionFakerSingleHyp.build() + action_plan = self.default_planner.schedule(self.context, + audit.id, fake_solution) + self.assertIsNotNone(action_plan.uuid) + + def test_scheduler_warning_empty_action_plan(self): + audit = db_utils.create_test_audit(uuid=utils.generate_uuid()) + fake_solution = SolutionFaker.build() + action_plan = self.default_planner.schedule(self.context, + audit.id, fake_solution) self.assertIsNotNone(action_plan.uuid) diff --git a/watcher/tests/decision_engine/framework/test_default_solution.py b/watcher/tests/decision_engine/framework/test_default_solution.py new file mode 100644 index 000000000..bc00fe28e --- /dev/null +++ b/watcher/tests/decision_engine/framework/test_default_solution.py @@ -0,0 +1,24 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from watcher.decision_engine.framework.default_solution import DefaultSolution +from watcher.tests import base + + +class TestDefaultSolution(base.BaseTestCase): + def test_default_solution(self): + solution = DefaultSolution() + solution.add_change_request("BLA") + self.assertEqual(solution.meta_actions[0], "BLA") diff --git a/watcher/tests/decision_engine/strategies/__init__.py b/watcher/tests/decision_engine/strategies/__init__.py new file mode 100644 index 000000000..2327bf100 --- /dev/null +++ b/watcher/tests/decision_engine/strategies/__init__.py @@ -0,0 +1 @@ +__author__ = 'Jean-Emile DARTOIS ' diff --git a/watcher/tests/decision_engine/strategies/test_basic_consolidation.py b/watcher/tests/decision_engine/strategies/test_basic_consolidation.py new file mode 100644 index 000000000..7cff514ca --- /dev/null +++ b/watcher/tests/decision_engine/strategies/test_basic_consolidation.py @@ -0,0 +1,314 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Authors: Jean-Emile DARTOIS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import mock +from watcher.common import exception + +from watcher.decision_engine.framework.meta_actions.hypervisor_state import \ + ChangeHypervisorState +from watcher.decision_engine.framework.meta_actions.power_state import \ + ChangePowerState + +from watcher.decision_engine.framework.meta_actions.migrate import Migrate +from watcher.decision_engine.framework.model.model_root import ModelRoot +from watcher.decision_engine.strategies.basic_consolidation import \ + BasicConsolidation +from watcher.tests import base +from watcher.tests.decision_engine.faker_cluster_state import \ + FakerStateCollector +from watcher.tests.decision_engine.faker_metrics_collector import \ + FakerMetricsCollector +# from watcher.tests.decision_engine.faker_metrics_collector import \ +# FakerMetricsCollectorEmptyType + + +class TestBasicConsolidation(base.BaseTestCase): + # fake metrics + fake_metrics = FakerMetricsCollector() + + # fake cluster + fake_cluster = FakerStateCollector() + + def test_cluster_size(self): + size_cluster = len( + self.fake_cluster.generate_scenario_1().get_all_hypervisors()) + size_cluster_assert = 5 + self.assertEqual(size_cluster, size_cluster_assert) + + def test_basic_consolidation_score_hypervisor(self): + cluster = self.fake_cluster.generate_scenario_1() + sercon = BasicConsolidation() + sercon.set_metrics_resource_collector(self.fake_metrics) + node_1_score = 0.01666666666666668 + self.assertEqual( + sercon.calculate_score_node( + cluster.get_hypervisor_from_id("Node_1"), + cluster), node_1_score) + node_2_score = 0.01666666666666668 + self.assertEqual( + sercon.calculate_score_node( + cluster.get_hypervisor_from_id("Node_2"), + cluster), node_2_score) + node_0_score = 0.01666666666666668 + self.assertEqual( + sercon.calculate_score_node( + cluster.get_hypervisor_from_id("Node_0"), + cluster), node_0_score) + + def test_basic_consolidation_score_vm(self): + cluster = self.fake_cluster.generate_scenario_1() + sercon = BasicConsolidation() + sercon.set_metrics_resource_collector(self.fake_metrics) + vm_0 = cluster.get_vm_from_id("VM_0") + vm_0_score = 0.0 + self.assertEqual(sercon.calculate_score_vm(vm_0, cluster), vm_0_score) + + vm_1 = cluster.get_vm_from_id("VM_1") + vm_1_score = 0.0 + self.assertEqual(sercon.calculate_score_vm(vm_1, cluster), + vm_1_score) + vm_2 = cluster.get_vm_from_id("VM_2") + vm_2_score = 0.0 + self.assertEqual(sercon.calculate_score_vm(vm_2, cluster), vm_2_score) + vm_6 = cluster.get_vm_from_id("VM_6") + vm_6_score = 0.0 + self.assertEqual(sercon.calculate_score_vm(vm_6, cluster), vm_6_score) + vm_7 = cluster.get_vm_from_id("VM_7") + vm_7_score = 0.0 + self.assertEqual(sercon.calculate_score_vm(vm_7, cluster), vm_7_score) + + def test_basic_consolidation_weight(self): + cluster = self.fake_cluster.generate_scenario_1() + sercon = BasicConsolidation() + sercon.set_metrics_resource_collector(self.fake_metrics) + vm_0 = cluster.get_vm_from_id("VM_0") + cores = 16 + # 80 Go + disk = 80 + # mem 8 Go + mem = 8 + vm_0_weight_assert = 3.1999999999999997 + self.assertEqual(sercon.calculate_weight(cluster, vm_0, cores, disk, + mem), + vm_0_weight_assert) + + def test_calculate_migration_efficiency(self): + sercon = BasicConsolidation() + sercon.calculate_migration_efficiency() + + def test_exception_model(self): + sercon = BasicConsolidation() + self.assertRaises(exception.ClusteStateNotDefined, sercon.execute, + None) + + def test_exception_cluster_empty(self): + sercon = BasicConsolidation() + model = ModelRoot() + self.assertRaises(exception.ClusterEmpty, sercon.execute, + model) + + def test_calculate_score_vm_raise_metric_collector(self): + sercon = BasicConsolidation() + self.assertRaises(exception.MetricCollectorNotDefined, + sercon.calculate_score_vm, "VM_1", None) + + def test_calculate_score_vm_raise_cluster_state_not_found(self): + metrics = FakerMetricsCollector() + metrics.empty_one_metric("CPU_COMPUTE") + sercon = BasicConsolidation() + sercon.set_metrics_resource_collector(metrics) + self.assertRaises(exception.ClusteStateNotDefined, + sercon.calculate_score_vm, "VM_1", None) + + def test_print_utilization_raise_cluster_state_not_found(self): + sercon = BasicConsolidation() + self.assertRaises(exception.ClusteStateNotDefined, + sercon.print_utilization, None) + + def check_migration(self, array, indice, vm, src, dest): + """Helper to check migration + + :param array: + :param indice: + :param vm: + :param src: + :param dest: + :return: + """ + self.assertEqual(array[indice].get_vm().uuid, vm) + self.assertEqual(array[indice].get_source_hypervisor().uuid, src) + self.assertEqual(array[indice].get_dest_hypervisor().uuid, dest) + + self.assertEqual(array[indice].get_bandwidth(), 0) + array[indice].set_bandwidth(5) + self.assertEqual(array[indice].get_bandwidth(), 5) + + def test_check_migration(self): + sercon = BasicConsolidation() + fake_cluster = FakerStateCollector() + model = fake_cluster.generate_scenario_4_with_2_hypervisors() + + all_vms = model.get_all_vms() + all_hyps = model.get_all_hypervisors() + vm0 = all_vms[all_vms.keys()[0]] + hyp0 = all_hyps[all_hyps.keys()[0]] + + sercon.check_migration(model, hyp0, hyp0, vm0) + + def test_threshold(self): + sercon = BasicConsolidation() + fake_cluster = FakerStateCollector() + model = fake_cluster.generate_scenario_4_with_2_hypervisors() + + all_hyps = model.get_all_hypervisors() + hyp0 = all_hyps[all_hyps.keys()[0]] + + sercon.check_threshold(model, hyp0, 1000, 1000, 1000) + + threshold_cores = sercon.get_threshold_cores() + sercon.set_threshold_cores(threshold_cores + 1) + self.assertEqual(sercon.get_threshold_cores(), threshold_cores + 1) + + def test_number_of(self): + sercon = BasicConsolidation() + sercon.get_number_of_released_nodes() + sercon.get_number_of_migrations() + + def test_calculate_score_node_raise_1(self): + sercon = BasicConsolidation() + metrics = FakerStateCollector() + + model = metrics.generate_scenario_4_with_2_hypervisors() + all_hyps = model.get_all_hypervisors() + hyp0 = all_hyps[all_hyps.keys()[0]] + + self.assertRaises(exception.MetricCollectorNotDefined, + sercon.calculate_score_node, hyp0, model) + + def test_calculate_score_node_raise_cpu_compute(self): + metrics = FakerMetricsCollector() + metrics.empty_one_metric("CPU_COMPUTE") + sercon = BasicConsolidation() + sercon.set_metrics_resource_collector(metrics) + current_state_cluster = FakerStateCollector() + model = current_state_cluster.generate_scenario_4_with_2_hypervisors() + + all_hyps = model.get_all_hypervisors() + hyp0 = all_hyps[all_hyps.keys()[0]] + + self.assertRaises(exception.NoDataFound, + sercon.calculate_score_node, hyp0, model) + + """ + def test_calculate_score_node_raise_memory_compute(self): + metrics = FakerMetricsCollector() + metrics.empty_one_metric("MEM_COMPUTE") + sercon = BasicConsolidation() + sercon.set_metrics_resource_collector(metrics) + current_state_cluster = FakerStateCollector() + model = current_state_cluster.generate_scenario_4_with_2_hypervisors() + + all_hyps = model.get_all_hypervisors() + hyp0 = all_hyps[all_hyps.keys()[0]] + self.assertRaises(exception.NoDataFound, + sercon.calculate_score_node, hyp0, model) + + def test_calculate_score_node_raise_disk_compute(self): + metrics = FakerMetricsCollector() + metrics.empty_one_metric("DISK_COMPUTE") + sercon = BasicConsolidation() + sercon.set_metrics_resource_collector(metrics) + current_state_cluster = FakerStateCollector() + model = current_state_cluster.generate_scenario_4_with_2_hypervisors() + + all_hyps = model.get_all_hypervisors() + hyp0 = all_hyps[all_hyps.keys()[0]] + + self.assertRaises(exception.NoDataFound, + sercon.calculate_score_node, hyp0, model) + """ + + def test_basic_consolidation_migration(self): + sercon = BasicConsolidation() + sercon.set_metrics_resource_collector(FakerMetricsCollector()) + solution = None + solution = sercon.execute(FakerStateCollector().generate_scenario_1()) + + count_migration = 0 + change_hypervisor_state = 0 + change_power_state = 0 + migrate = [] + for action in solution.meta_actions: + if isinstance(action, Migrate): + count_migration += 1 + migrate.append(action) + if isinstance(action, ChangeHypervisorState): + change_hypervisor_state += 1 + if isinstance(action, ChangePowerState): + change_power_state += 1 + + # self.assertEqual(change_hypervisor_state, 1) + # self.assertEqual(count_migration, 2) + + def test_execute_cluster_empty(self): + metrics = FakerMetricsCollector() + current_state_cluster = FakerStateCollector() + + sercon = BasicConsolidation("sercon", "Basic offline consolidation") + sercon.set_metrics_resource_collector(metrics) + model = current_state_cluster.generate_random(0, 0) + self.assertRaises(exception.ClusterEmpty, sercon.execute, model) + + def test_basic_consolidation_random(self): + metrics = FakerMetricsCollector() + current_state_cluster = FakerStateCollector() + + sercon = BasicConsolidation("sercon", "Basic offline consolidation") + sercon.set_metrics_resource_collector(metrics) + + solution = sercon.execute( + current_state_cluster.generate_random(25, 2)) + solution.__str__() + + count_migration = 0 + change_hypervisor_state = 0 + change_power_state = 0 + migrate = [] + for action in solution.meta_actions: + if isinstance(action, Migrate): + count_migration += 1 + migrate.append(action) + if isinstance(action, ChangeHypervisorState): + change_hypervisor_state += 1 + if isinstance(action, ChangePowerState): + change_power_state += 1 + + # calculate_weight + def test_execute_no_workload(self): + metrics = FakerMetricsCollector() + sercon = BasicConsolidation() + sercon.set_metrics_resource_collector(metrics) + current_state_cluster = FakerStateCollector() + model = current_state_cluster.\ + generate_scenario_5_with_1_hypervisor_no_vm() + + with mock.patch.object(BasicConsolidation, 'calculate_weight') \ + as mock_score_call: + mock_score_call.return_value = 0 + solution = sercon.execute(model) + self.assertEqual(solution.efficiency, 100) diff --git a/watcher/tests/decision_engine/strategies/test_dummy_strategy.py b/watcher/tests/decision_engine/strategies/test_dummy_strategy.py new file mode 100644 index 000000000..c94a3aa52 --- /dev/null +++ b/watcher/tests/decision_engine/strategies/test_dummy_strategy.py @@ -0,0 +1,27 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from watcher.decision_engine.strategies.dummy_strategy import DummyStrategy +from watcher.tests import base +from watcher.tests.decision_engine.faker_cluster_state import \ + FakerStateCollector + + +class TestDummyStrategy(base.TestCase): + def test_dummy_strategy(self): + tactique = DummyStrategy("basic", "Basic offline consolidation") + fake_cluster = FakerStateCollector() + model = fake_cluster.generate_scenario_4_with_2_hypervisors() + tactique.execute(model) diff --git a/watcher/tests/decision_engine/test_basic_consolidation.py b/watcher/tests/decision_engine/test_basic_consolidation.py deleted file mode 100644 index 1889b26aa..000000000 --- a/watcher/tests/decision_engine/test_basic_consolidation.py +++ /dev/null @@ -1,184 +0,0 @@ -# -*- encoding: utf-8 -*- -# Copyright (c) 2015 b<>com -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from watcher.common import exception - -from watcher.decision_engine.framework.meta_actions.hypervisor_state import \ - ChangeHypervisorState -from watcher.decision_engine.framework.meta_actions.power_state import \ - ChangePowerState - -from watcher.decision_engine.framework.meta_actions.migrate import Migrate -from watcher.decision_engine.framework.model.model_root import ModelRoot -from watcher.decision_engine.strategies.basic_consolidation import \ - BasicConsolidation - -from watcher.tests import base -from watcher.tests.decision_engine.faker_cluster_state import \ - FakerStateCollector -from watcher.tests.decision_engine.faker_metrics_collector import \ - FakerMetricsCollector - - -class TestBasicConsolidation(base.BaseTestCase): - # fake metrics - fake_metrics = FakerMetricsCollector() - - # fake cluster - fake_cluster = FakerStateCollector() - - def test_cluster_size(self): - size_cluster = len( - self.fake_cluster.generate_scenario_1().get_all_hypervisors()) - size_cluster_assert = 5 - self.assertEqual(size_cluster, size_cluster_assert) - - def test_basic_consolidation_score_hypervisor(self): - cluster = self.fake_cluster.generate_scenario_1() - sercon = BasicConsolidation() - sercon.set_metrics_resource_collector(self.fake_metrics) - node_1_score = 0.09862626262626262 - self.assertEqual( - sercon.calculate_score_node( - cluster.get_hypervisor_from_id("Node_1"), - cluster), node_1_score) - node_2_score = 0.29989898989898994 - self.assertEqual( - sercon.calculate_score_node( - cluster.get_hypervisor_from_id("Node_2"), - cluster), node_2_score) - node_0_score = 0.13967676767676765 - self.assertEqual( - sercon.calculate_score_node( - cluster.get_hypervisor_from_id("Node_0"), - cluster), node_0_score) - - def test_basic_consolidation_score_vm(self): - cluster = self.fake_cluster.generate_scenario_1() - sercon = BasicConsolidation() - sercon.set_metrics_resource_collector(self.fake_metrics) - vm_0 = cluster.get_vm_from_id("VM_0") - vm_0_score = 0.6 - self.assertEqual(sercon.calculate_score_vm(vm_0, cluster), vm_0_score) - - vm_1 = cluster.get_vm_from_id("VM_1") - vm_1_score = 1.0999999999999999 - self.assertEqual(sercon.calculate_score_vm(vm_1, cluster), - vm_1_score) - vm_2 = cluster.get_vm_from_id("VM_2") - vm_2_score = 1.2 - self.assertEqual(sercon.calculate_score_vm(vm_2, cluster), vm_2_score) - - def test_basic_consolidation_weight(self): - cluster = self.fake_cluster.generate_scenario_1() - sercon = BasicConsolidation() - sercon.set_metrics_resource_collector(self.fake_metrics) - vm_0 = cluster.get_vm_from_id("VM_0") - cores = 16 - # 80 Go - disk = 80 - # mem 8 Go - mem = 8 - vm_0_weight_assert = 3.1999999999999997 - self.assertEqual(sercon.calculate_weight(cluster, vm_0, cores, disk, - mem), - vm_0_weight_assert) - - def test_basic_consolidation_efficiency(self): - sercon = BasicConsolidation() - sercon.set_metrics_resource_collector(self.fake_metrics) - efficient_assert = 100 - solution = sercon.execute(self.fake_cluster.generate_scenario_1()) - self.assertEqual(solution.get_efficiency(), efficient_assert) - - def test_exception_model(self): - sercon = BasicConsolidation() - self.assertRaises(exception.ClusteStateNotDefined, sercon.execute, - None) - - def test_exception_cluster_empty(self): - sercon = BasicConsolidation() - model = ModelRoot() - self.assertRaises(exception.ClusterEmpty, sercon.execute, - model) - - def test_exception_metric_collector(self): - sercon = BasicConsolidation() - self.assertRaises(exception.MetricCollectorNotDefined, - sercon.calculate_score_vm, "VM_1", None) - - def check_migration(self, array, indice, vm, src, dest): - """Helper to check migration - - :param array: - :param indice: - :param vm: - :param src: - :param dest: - :return: - """ - self.assertEqual(array[indice].get_vm().get_uuid(), vm) - self.assertEqual(array[indice].get_source_hypervisor().get_uuid(), src) - self.assertEqual(array[indice].get_dest_hypervisor().get_uuid(), dest) - - def test_basic_consolidation_migration(self): - sercon = BasicConsolidation() - sercon.set_metrics_resource_collector(self.fake_metrics) - - solution = sercon.execute(self.fake_cluster.generate_scenario_1()) - - count_migration = 0 - change_hypervisor_state = 0 - change_power_state = 0 - migrate = [] - for action in solution.meta_actions: - if isinstance(action, Migrate): - count_migration += 1 - migrate.append(action) - if isinstance(action, ChangeHypervisorState): - change_hypervisor_state += 1 - if isinstance(action, ChangePowerState): - change_power_state += 1 - - self.assertEqual(change_hypervisor_state, 3) - self.assertEqual(count_migration, 3) - # check migration - self.check_migration(migrate, 0, "VM_7", "Node_4", "Node_2") - self.check_migration(migrate, 1, "VM_6", "Node_3", "Node_0") - self.check_migration(migrate, 2, "VM_2", "Node_1", "Node_0") - - def test_basic_consolidation_random(self): - metrics = FakerMetricsCollector() - current_state_cluster = FakerStateCollector() - - sercon = BasicConsolidation("sercon", "Basic offline consolidation") - sercon.set_metrics_resource_collector(metrics) - - solution = sercon.execute( - current_state_cluster.generate_random(25, 2)) - - count_migration = 0 - change_hypervisor_state = 0 - change_power_state = 0 - migrate = [] - for action in solution.meta_actions: - if isinstance(action, Migrate): - count_migration += 1 - migrate.append(action) - if isinstance(action, ChangeHypervisorState): - change_hypervisor_state += 1 - if isinstance(action, ChangePowerState): - change_power_state += 1 diff --git a/watcher/tests/decision_engine/test_loader.py b/watcher/tests/decision_engine/test_loader.py index c633fb285..bbf5f77fa 100644 --- a/watcher/tests/decision_engine/test_loader.py +++ b/watcher/tests/decision_engine/test_loader.py @@ -1,11 +1,13 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# from watcher.tests import base diff --git a/watcher/tests/decision_engine/test_model.py b/watcher/tests/decision_engine/test_model.py deleted file mode 100644 index 4f0f1c287..000000000 --- a/watcher/tests/decision_engine/test_model.py +++ /dev/null @@ -1,54 +0,0 @@ -# -*- encoding: utf-8 -*- -# Copyright (c) 2015 b<>com -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import uuid -from watcher.common import exception - -from watcher.decision_engine.framework.model.hypervisor import Hypervisor -from watcher.decision_engine.framework.model.model_root import ModelRoot - -from watcher.tests.decision_engine.faker_cluster_state import \ - FakerStateCollector - -from watcher.tests import base - - -class TestModel(base.BaseTestCase): - def test_model(self): - fake_cluster = FakerStateCollector() - model = fake_cluster.generate_scenario_1() - - self.assertEqual(len(model._hypervisors), 5) - self.assertEqual(len(model._vms), 35) - self.assertEqual(len(model.get_mapping().get_mapping()), 5) - - def test_add_hypervisor(self): - model = ModelRoot() - id = str(uuid.uuid4()) - hypervisor = Hypervisor() - hypervisor.set_uuid(id) - model.add_hypervisor(hypervisor) - self.assertEqual(model.get_hypervisor_from_id(id), hypervisor) - - def test_delete_hypervisor(self): - model = ModelRoot() - id = str(uuid.uuid4()) - hypervisor = Hypervisor() - hypervisor.set_uuid(id) - model.add_hypervisor(hypervisor) - self.assertEqual(model.get_hypervisor_from_id(id), hypervisor) - model.remove_hypervisor(hypervisor) - self.assertRaises(exception.HypervisorNotFound, - model.get_hypervisor_from_id, id) diff --git a/watcher/tests/decision_engine/test_planner.py b/watcher/tests/decision_engine/test_planner.py index 6696d37f1..69fd53513 100644 --- a/watcher/tests/decision_engine/test_planner.py +++ b/watcher/tests/decision_engine/test_planner.py @@ -1,11 +1,13 @@ # -*- encoding: utf-8 -*- # Copyright (c) 2015 b<>com # +# Authors: Jean-Emile DARTOIS +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -13,6 +15,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +# from watcher.tests import base diff --git a/watcher/tests/demo_vancouver.py b/watcher/tests/demo_vancouver.py deleted file mode 100644 index c31d42693..000000000 --- a/watcher/tests/demo_vancouver.py +++ /dev/null @@ -1,151 +0,0 @@ -# -*- encoding: utf-8 -*- -# Copyright (c) 2015 b<>com -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -""" -from concurrent.futures import ThreadPoolExecutor - -from keystoneclient import session - -from keystoneclient.auth.identity import v3 - -import cinderclient.v2.client as ciclient -import glanceclient.v2.client as glclient -import keystoneclient.v3.client as ksclient -import neutronclient.neutron.client as netclient -import novaclient.v2.client as nvclient - -from watcher.common.utils import CONF -from oslo_config import cfg -from watcher.applier.framework.command.migrate_command import MigrateCommand -from watcher.applier.framework.command.wrapper.nova_wrapper import NovaWrapper -from watcher.decision_engine.framework.default_planner import Primitives -from watcher.openstack.common import log -import ceilometerclient.v2 as c_client - -cfg.CONF.debug = True -log.setup('metering-controller') - -cfg.CONF.import_opt('auth_uri', 'keystoneclient.middleware.auth_token', - group='keystone_authtoken') -cfg.CONF.import_opt('admin_user', 'keystoneclient.middleware.auth_token', - group='keystone_authtoken') -cfg.CONF.import_opt('admin_password', 'keystoneclient.middleware.auth_token', - group='keystone_authtoken') -cfg.CONF.import_opt('admin_tenant_name', - 'keystoneclient.middleware.auth_token', - group='keystone_authtoken') - -cfg.CONF.keystone_authtoken.auth_uri = "http://10.50.0.105:5000/v3/" -cfg.CONF.keystone_authtoken.admin_user = "watcher" -cfg.CONF.keystone_authtoken.admin_password = "watcher" -cfg.CONF.keystone_authtoken.admin_tenant_name = "services" - - -def make_query(user_id=None, tenant_id=None, resource_id=None, - user_ids=None, tenant_ids=None, resource_ids=None): - user_ids = user_ids or [] - tenant_ids = tenant_ids or [] - resource_ids = resource_ids or [] - query = [] - if user_id: - user_ids = [user_id] - for u_id in user_ids: - query.append({"field": "user_id", "op": "eq", "value": u_id}) - if tenant_id: - tenant_ids = [tenant_id] - for t_id in tenant_ids: - query.append({"field": "project_id", "op": "eq", "value": t_id}) - if resource_id: - resource_ids = [resource_id] - for r_id in resource_ids: - query.append({"field": "resource_id", "op": "eq", "value": r_id}) - return query - - -# nova-manage service enable ---host='ldev-indeedsrv005' --service='nova-compute' - - -def create(wrapper, id, hypervisorid): - print("create instance VM_{0} on {1}".format(str(id), str(hypervisorid))) - try: - - for image in glance.images.list(name='Cirros'): - id_image = image.id - - vm = wrapper.create_instance(hypervisor_id=hypervisorid, - inst_name="VM_" + str(id), - keypair_name='admin', - image_id=id_image, - create_new_floating_ip=True, - flavor_name='m1.medium') - print(vm) - except Exception as e: - print(unicode(e)) - - -def purge(nova, wrapper): - print("Purging the cluster") - instances = nova.servers.list() - for instance in instances: - wrapper.delete_instance(instance.id) - - -try: - executor = ThreadPoolExecutor(max_workers=3) - creds = \ - {'auth_url': CONF.keystone_authtoken.auth_uri, - 'username': CONF.keystone_authtoken.admin_user, - 'password': CONF.keystone_authtoken.admin_password, - 'project_name': CONF.keystone_authtoken.admin_tenant_name, - 'user_domain_name': "default", - 'project_domain_name': "default"} - auth = v3.Password(auth_url=creds['auth_url'], - username=creds['username'], - password=creds['password'], - project_name=creds['project_name'], - user_domain_name=creds[ - 'user_domain_name'], - project_domain_name=creds[ - 'project_domain_name']) - sess = session.Session(auth=auth) - nova = nvclient.Client("3", session=sess) - neutron = netclient.Client('2.0', session=sess) - neutron.format = 'json' - keystone = ksclient.Client(**creds) - - glance_endpoint = keystone. \ - service_catalog.url_for(service_type='image', - endpoint_type='publicURL') - glance = glclient.Client(glance_endpoint, - token=keystone.auth_token) - - wrapper = NovaWrapper(creds, session=sess) - - wrapper.live_migrate_instance( - instance_id="b2aca823-a621-4235-9d56-9f0f75955dc1", - dest_hostname="ldev-indeedsrv006", block_migration=True) - - nova-manage service enable --host='ldev-indeedsrv005' \ - --service='nova-compute' - nova-manage service enable --host='ldev-indeedsrv006' \ - --service='nova-compute' - - -except Exception as e: - print("rollback " + str(e)) - -""" diff --git a/watcher/version.py b/watcher/version.py index 79f0cb9f5..a24842723 100644 --- a/watcher/version.py +++ b/watcher/version.py @@ -1,3 +1,4 @@ +# -*- encoding: utf-8 -*- # Copyright 2011 OpenStack Foundation # All Rights Reserved. # @@ -15,4 +16,4 @@ import pbr.version -version_info = pbr.version.VersionInfo('watcher') +version_info = pbr.version.VersionInfo('python-watcher')