From f5a7653cf0f1453d74483b48a9ce269fb71ad7ed Mon Sep 17 00:00:00 2001 From: Stamatis Katsaounis Date: Thu, 2 Jan 2020 14:09:15 +0200 Subject: [PATCH] First commit of watcher charm Signed-off-by: Stamatis Katsaounis --- .gitignore | 9 + .stestr.conf | 3 + LICENSE | 202 +++++++++++ rebuild | 5 + requirements.txt | 8 + src/HACKING.md | 9 + src/README.md | 43 +++ src/config.yaml | 139 ++++++++ src/copyright | 6 + src/layer.yaml | 17 + src/lib/__init__.py | 13 + src/lib/charm/__init__.py | 13 + src/lib/charm/openstack/__init__.py | 13 + src/lib/charm/openstack/watcher.py | 197 +++++++++++ src/metadata.yaml | 21 ++ src/reactive/__init__.py | 13 + src/reactive/watcher_handlers.py | 62 ++++ src/templates/train/watcher-api.conf | 37 ++ src/templates/train/watcher.conf | 228 ++++++++++++ src/test-requirements.txt | 11 + src/tox.ini | 44 +++ test-requirements.txt | 14 + tox.ini | 88 +++++ unit_tests/__init__.py | 22 ++ .../test_lib_charm_openstack_watcher.py | 334 ++++++++++++++++++ unit_tests/test_watcher_handlers.py | 90 +++++ 26 files changed, 1641 insertions(+) create mode 100644 .gitignore create mode 100644 .stestr.conf create mode 100644 LICENSE create mode 100644 rebuild create mode 100644 requirements.txt create mode 100644 src/HACKING.md create mode 100644 src/README.md create mode 100644 src/config.yaml create mode 100644 src/copyright create mode 100644 src/layer.yaml create mode 100644 src/lib/__init__.py create mode 100644 src/lib/charm/__init__.py create mode 100644 src/lib/charm/openstack/__init__.py create mode 100644 src/lib/charm/openstack/watcher.py create mode 100644 src/metadata.yaml create mode 100644 src/reactive/__init__.py create mode 100644 src/reactive/watcher_handlers.py create mode 100644 src/templates/train/watcher-api.conf create mode 100644 src/templates/train/watcher.conf create mode 100644 src/test-requirements.txt create mode 100644 src/tox.ini create mode 100644 test-requirements.txt create mode 100644 tox.ini create mode 100644 unit_tests/__init__.py create mode 100644 unit_tests/test_lib_charm_openstack_watcher.py create mode 100644 unit_tests/test_watcher_handlers.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..32ddcf4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +.tox +.stestr +*__pycache__* +*.pyc +build +.idea +.venv +cover +.coverage diff --git a/.stestr.conf b/.stestr.conf new file mode 100644 index 0000000..5fcccac --- /dev/null +++ b/.stestr.conf @@ -0,0 +1,3 @@ +[DEFAULT] +test_path=./unit_tests +top_dir=./ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/rebuild b/rebuild new file mode 100644 index 0000000..fb0c355 --- /dev/null +++ b/rebuild @@ -0,0 +1,5 @@ +# This file is used to trigger rebuilds +# when dependencies of the charm change, +# but nothing in the charm needs to. +# simply change the uuid to something new +2c669f80-1a7e-11ea-92d1-9cb6d0d31a29 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b1d4872 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +# This file is managed centrally by release-tools and should not be modified +# within individual charm repos. See the 'global' dir contents for available +# choices of *requirements.txt files for OpenStack Charms: +# https://github.com/openstack-charmers/release-tools +# +# Build requirements +charm-tools>=2.4.4 +simplejson diff --git a/src/HACKING.md b/src/HACKING.md new file mode 100644 index 0000000..a3b8a24 --- /dev/null +++ b/src/HACKING.md @@ -0,0 +1,9 @@ +# Overview + +This charm is developed by Stamatis Katsaounis and it is based on the official +OpenStack Charms project. + +You can find its source code here: . + + + diff --git a/src/README.md b/src/README.md new file mode 100644 index 0000000..bb58810 --- /dev/null +++ b/src/README.md @@ -0,0 +1,43 @@ +# Overview + +This charm provides the Watcher service for an OpenStack Cloud. It is comprised +of three different services. The API, the Decision Engine and the Applier. This +charm takes care of all three, bundled as a single application. + +# Usage + +The OpenStack Watcher charm requires a running OpenStack deployment and relation +with mysql database and keystone identity service. + +A simple deployment requires only four commands: + + juju deploy --series bionic --config openstack-origin=cloud:bionic-train watcher + juju add-relation watcher mysql + juju add-relation watcher keystone + +The charm also support High Availability by relating it to hacluster charm: + + juju add-relation nova-cloud-controller + +# Bugs + +Please report bugs on [GitHub](https://github.com/grnet/charm-watcher/issues). +For general questions please refer to the OpenStack [Charm Guide](https://docs.openstack.org/charm-guide/latest/). + +# Configuration + +The configuration options will be listed on the charm store, however If you're +making assumptions or opinionated decisions in the charm (like setting a default +administrator password), you should detail that here so the user knows how to +change it immediately, etc. + +# Contact Information + +Though this will be listed in the charm store itself don't assume a user will +know that, so include that information here: + +## OpenStack Watcher + +- [Watcher](https://wiki.openstack.org/wiki/Watcher) +- [Watcher Bugs](https://launchpad.net/watcher) +- Watcher IRC on freenode at #openstack-watcher diff --git a/src/config.yaml b/src/config.yaml new file mode 100644 index 0000000..5df5082 --- /dev/null +++ b/src/config.yaml @@ -0,0 +1,139 @@ +options: + collector-plugins: + default: "compute" + type: string + description: | + A comma separated list of cluster data model plugin names. + . + Available collector-plugins are: compute and storage. + grafana-auth-token: + default: "changeme" + type: string + description: | + The authtoken for access to Grafana datasource. + grafana-base-url: + default: + type: string + description: | + The base url parameter will need to specify the type of http protocol + and the use of plain text http is strongly discouraged due to the + transmission of the access token. + . + Additionally the path to the proxy interface needs to be supplied as + well in case Grafana is placed in a sub directory of the web server. + . + An example would be: https://mygrafana.org/api/datasource/proxy/ + grafana-project-id-map: + default: + type: string + description: | + Mapping of datasource metrics to Grafana project ids. + . + Example: + host_airflow:1,host_cpu_usage:2,host_inlet_temp:3,host_outlet_temp:4, + host_power:5,host_ram_usage:6,instance_cpu_usage:7, + instance_l3_cache_usage:8,instance_ram_allocated:9,instance_ram_usage:10, + instance_root_disk_size:11 + grafana-database-map: + default: + type: string + description: | + Mapping of datasource metrics to Grafana databases. + . + Example: + host_airflow:1,host_cpu_usage:2,host_inlet_temp:3,host_outlet_temp:4, + host_power:5,host_ram_usage:6,instance_cpu_usage:7, + instance_l3_cache_usage:8,instance_ram_allocated:9,instance_ram_usage:10, + instance_root_disk_size:11 + grafana-attribute-map: + default: + type: string + description: | + Mapping of datasource metrics to resource attributes. For a complete list + of available attributes see + https://docs.openstack.org/watcher/latest/datasources/grafana.html#attribute + . + Example: + host_airflow:1,host_cpu_usage:2,host_inlet_temp:3,host_outlet_temp:4, + host_power:5,host_ram_usage:6,instance_cpu_usage:7, + instance_l3_cache_usage:8,instance_ram_allocated:9,instance_ram_usage:10, + instance_root_disk_size:11 + grafana-translator-map: + default: + type: string + description: | + Mapping of datasource metrics to Grafana translators. + . + Example: + host_airflow:1,host_cpu_usage:2,host_inlet_temp:3,host_outlet_temp:4, + host_power:5,host_ram_usage:6,instance_cpu_usage:7, + instance_l3_cache_usage:8,instance_ram_allocated:9,instance_ram_usage:10, + instance_root_disk_size:11 + grafana-query-map: + default: + type: string + description: | + Mapping of datasource metrics to Grafana queries. Values should be + strings for which the .format method will transform it. + The transformation offers five parameters to the query labeled {0} to + {4}. {0} will be replaced with the aggregate, {1} with the resource + attribute, {2} with the period, {3} with the granularity and {4} with + translator specifics for InfluxDB this will be the retention period. + These queries will need to be constructed using tools such as Postman. + Example: SELECT cpu FROM {4}.cpu_percent WHERE host == '{1}' AND + time > now()-{2}s + . + Example: + host_airflow:1,host_cpu_usage:2,host_inlet_temp:3,host_outlet_temp:4, + host_power:5,host_ram_usage:6,instance_cpu_usage:7, + instance_l3_cache_usage:8,instance_ram_allocated:9,instance_ram_usage:10, + instance_root_disk_size:11 + grafana-retention-periods: + default: + type: string + description: | + Keys are the names of retention periods in InfluxDB and the values should + correspond with the maximum time they can retain in seconds. + . + Example: five_years:31556952,one_month:2592000,one_week:604800 + data-model-period: + default: 3600 + type: int + description: | + The time interval (in seconds) between each synchronization of the model + datasources: + default: + type: string + description: | + Datasources to use in order to query the needed metrics. If one of + strategy metric is not available in the first datasource, the next + datasource will be chosen. + . + Available datasources are: gnocchi, ceilometer and grafana. + action-plan-expiry: + default: 24 + type: int + description: | + An expiry timespan (hours). Watcher invalidates any action plan for which + its creation time - whose number of hours has been offset by this + value - is older that the current time. + check-periodic-interval: + default: 1800 + type: int + description: | + Interval (in seconds) for checking action plan expiry. + planner: + default: "weight" + type: string + description: | + The selected planner used to schedule the actions. + . + Available planners are: weight, workload_stabilization, basic and + storage_capacity_balance. + planner-config: + default: + type: string + description: | + User provided planner configuration. Supports a string representation of + a python dictionary where each top-level key represents a value in the + relevant planner section in watcher.conf template. diff --git a/src/copyright b/src/copyright new file mode 100644 index 0000000..97135b3 --- /dev/null +++ b/src/copyright @@ -0,0 +1,6 @@ +Format: http://dep.debian.net/deps/dep5/ + +Files: * +Copyright: Copyright 2020, GRNET SA +License: Apache-2.0 + diff --git a/src/layer.yaml b/src/layer.yaml new file mode 100644 index 0000000..0bf6bc1 --- /dev/null +++ b/src/layer.yaml @@ -0,0 +1,17 @@ +includes: + - layer:leadership + - layer:openstack-api + - interface:mysql-shared + - interface:rabbitmq + - interface:keystone + - interface:hacluster + - interface:openstack-ha +options: + basic: + use_venv: True + include_system_packages: False + packages: [ 'libffi-dev', 'libssl-dev' ] + repo: https://github.com/grnet/charm-watcher +config: + deletes: + - verbose diff --git a/src/lib/__init__.py b/src/lib/__init__.py new file mode 100644 index 0000000..858116c --- /dev/null +++ b/src/lib/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020 GRNET SA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/lib/charm/__init__.py b/src/lib/charm/__init__.py new file mode 100644 index 0000000..858116c --- /dev/null +++ b/src/lib/charm/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020 GRNET SA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/lib/charm/openstack/__init__.py b/src/lib/charm/openstack/__init__.py new file mode 100644 index 0000000..858116c --- /dev/null +++ b/src/lib/charm/openstack/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020 GRNET SA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/lib/charm/openstack/watcher.py b/src/lib/charm/openstack/watcher.py new file mode 100644 index 0000000..a12c02a --- /dev/null +++ b/src/lib/charm/openstack/watcher.py @@ -0,0 +1,197 @@ +# Copyright 2020 GRNET SA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections + +import charms_openstack.adapters as openstack_adapters +import charms_openstack.charm as openstack_charm +import charms_openstack.ip as os_ip + +from charmhelpers.contrib.openstack.utils import config_flags_parser +from charmhelpers.core.hookenv import ( + config, + log, + WARNING +) + +WATCHER_CONF = '/etc/watcher/watcher.conf' +WATCHER_WSGI_CONF = '/etc/apache2/sites-available/watcher-api.conf' + +openstack_charm.use_defaults('charm.default-select-release') + + +@openstack_adapters.config_property +def planner_weights(cls): + conf = config('planner') + if conf in ['weight', 'workload_stabilization']: + conf = config_flags_parser(config('planner-config')) + weights = conf.get('weights', None) + if not weights: + log('Provided planner-config dictionary does not contain key ' + 'weights - ignoring', level=WARNING) + else: + return weights + + +@openstack_adapters.config_property +def planner_parallelization(cls): + conf = config('planner') + if conf == 'weight': + conf = config_flags_parser(config('planner-config')) + parallelization = conf.get('parallelization', None) + if not parallelization: + log('Provided planner-config dictionary does not contain key ' + 'parallelization - ignoring', level=WARNING) + else: + return parallelization + + +@openstack_adapters.config_property +def planner_check_optimize_metadata(cls): + conf = config('planner') + if conf == 'basic': + conf = config_flags_parser(config('planner-config')) + check_optimize_metadata = conf.get('check_optimize_metadata', None) + if not check_optimize_metadata: + log('Provided planner-config dictionary does not contain key ' + 'check_optimize_metadata - ignoring', level=WARNING) + else: + return check_optimize_metadata + + +@openstack_adapters.config_property +def planner_ex_pools(cls): + conf = config('planner') + if conf == 'storage_capacity_balance': + conf = config_flags_parser(config('planner-config')) + ex_pools = conf.get('ex_pools', None) + if not ex_pools: + log('Provided planner-config dictionary does not contain key ' + 'ex_pools - ignoring', level=WARNING) + else: + return ex_pools + + +class WatcherCharm(openstack_charm.HAOpenStackCharm): + service_name = name = 'watcher' + + release = 'train' + + packages = [ + 'watcher-common', 'watcher-api', 'watcher-decision-engine', + 'watcher-applier', 'python3-watcher', 'libapache2-mod-wsgi-py3', + 'python-apt', # NOTE: workaround for hacluster subordinate + ] + + api_ports = { + 'watcher-api': { + os_ip.PUBLIC: 9322, + os_ip.ADMIN: 9322, + os_ip.INTERNAL: 9322, + } + } + + group = 'watcher' + service_type = 'watcher' + default_service = 'watcher-api' + services = ['watcher-api', 'watcher-decision-engine', 'watcher-applier'] + + required_relations = ['shared-db', 'amqp', 'identity-service'] + + restart_map = { + WATCHER_CONF: services, + WATCHER_WSGI_CONF: services, + } + + ha_resources = ['vips', 'haproxy', 'dnsha'] + + release_pkg = 'watcher-common' + + package_codenames = { + 'watcher-common': collections.OrderedDict([ + ('3', 'train'), + ]), + } + + sync_cmd = ['watcher-db-manage', '--config-file', WATCHER_CONF, 'upgrade'] + + def get_amqp_credentials(self): + return 'watcher', 'openstack' + + def get_database_setup(self): + return [ + dict(database='watcher', + username='watcher',) + ] + + def grafana_configuration_complete(self): + """Determine whether sufficient configuration has been provided + via charm config options when Grafana datasource is chosen. + :returns: boolean indicating whether configuration is complete + """ + required_config = [ + self.options.grafana_auth_token, + self.options.grafana_base_url, + self.options.grafana_project_id_map, + self.options.grafana_database_map, + self.options.grafana_attribute_map, + self.options.grafana_translator_map, + self.options.grafana_query_map, + self.options.grafana_retention_periods + ] + + return all(required_config) + + def custom_assess_status_check(self): + """Verify that the configuration provided is valid and thus the service + is ready to go. This will return blocked if the configuration is not + valid for the service. + :returns (status: string, message: string): the status, and message if + there is a problem. Or (None, None) if there are no issues. + """ + datasources = self.options.datasources + if not datasources: + return 'blocked', 'datasources not set' + if not set(datasources.split(',')).issubset( + ['gnocchi', 'ceilometer', 'grafana']): + return ('blocked', + 'Provided datasources {} does not contain valid options' + .format(datasources)) + if 'grafana' in datasources: + if not self.grafana_configuration_complete(): + return ('blocked', + 'grafana datasource requires all grafana related ' + 'options to be set') + planner = self.options.planner + if planner not in [ + 'weight', 'workload_stabilization', 'basic', + 'storage_capacity_balance']: + return ('blocked', + 'Invalid planner: {}. Available options are: ' + 'weights, workload_stabilization, basic, ' + 'storage_capacity_balance'.format(planner)) + planner_config = config_flags_parser(self.options.planner_config) + planner_config_values = { + 'weight': ['weights', 'parallelization'], + 'workload_stabilization': ['weights'], + 'basic': ['check_optimize_metadata'], + 'storage_capacity_balance': ['ex_pools'], + } + if list(planner_config.keys()) != planner_config_values[planner]: + return ('blocked', + 'Provided planner {} must contain only the following ' + 'configuration attributes: {}'.format( + planner, ', '.join(planner_config_values[planner]))) + + return None, None diff --git a/src/metadata.yaml b/src/metadata.yaml new file mode 100644 index 0000000..b106001 --- /dev/null +++ b/src/metadata.yaml @@ -0,0 +1,21 @@ +name: watcher +summary: OpenStack Watcher service +maintainer: Stamatis Katsaounis +description: | + OpenStack Watcher provides a flexible and scalable resource + optimization service for multi-tenant clouds. + . + OpenStack Train or later is required. +tags: + - openstack +series: + - bionic + - eoan +subordinate: false +requires: + shared-db: + interface: mysql-shared + amqp: + interface: rabbitmq + identity-service: + interface: keystone diff --git a/src/reactive/__init__.py b/src/reactive/__init__.py new file mode 100644 index 0000000..858116c --- /dev/null +++ b/src/reactive/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020 GRNET SA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/reactive/watcher_handlers.py b/src/reactive/watcher_handlers.py new file mode 100644 index 0000000..9b19cbe --- /dev/null +++ b/src/reactive/watcher_handlers.py @@ -0,0 +1,62 @@ +# Copyright 2020 GRNET SA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import charms.reactive as reactive + +import charms_openstack.bus +import charms_openstack.charm + +charms_openstack.bus.discover() + + +charms_openstack.charm.use_defaults( + 'charm.installed', + 'amqp.connected', + 'shared-db.connected', + 'identity-service.connected', + 'identity-service.available', + 'config.changed', + 'update-status', + 'upgrade-charm', + 'certificates.available', +) + + +@reactive.when('shared-db.available') +@reactive.when('identity-service.available') +@reactive.when('amqp.available') +def render_config(*args): + with charms_openstack.charm.provide_charm_instance() as watcher_charm: + watcher_charm.render_with_interfaces(args) + watcher_charm.assess_status() + reactive.set_state('config.rendered') + + +@reactive.when_not('db.synced') +@reactive.when('config.rendered') +def init_db(): + """Run initial DB migrations when config is rendered.""" + with charms_openstack.charm.provide_charm_instance() as watcher_charm: + watcher_charm.db_sync() + watcher_charm.restart_all() + reactive.set_state('db.synced') + watcher_charm.assess_status() + + +@reactive.when('ha.connected') +def cluster_connected(hacluster): + """Configure HA resources in corosync""" + with charms_openstack.charm.provide_charm_instance() as watcher_charm: + watcher_charm.configure_ha_resources(hacluster) + watcher_charm.assess_status() diff --git a/src/templates/train/watcher-api.conf b/src/templates/train/watcher-api.conf new file mode 100644 index 0000000..58adda5 --- /dev/null +++ b/src/templates/train/watcher-api.conf @@ -0,0 +1,37 @@ +Listen {{ options.service_listen_info.watcher_api.public_port }} + + + WSGIScriptAlias / /usr/bin/watcher-api + WSGIDaemonProcess watcher processes={{ options.wsgi_worker_context.processes }} threads=2 user=watcher group=watcher display-name=%{GROUP} + WSGIProcessGroup watcher + WSGIApplicationGroup %{GLOBAL} + WSGIPassAuthorization On + LimitRequestBody 114688 + + = 2.4> + ErrorLogFormat "%{cu}t %M" + + + ErrorLog /var/log/apache2/watcher_error.log + CustomLog /var/log/apache2/watcher_access.log combined + + + = 2.4> + Require all granted + + + Order allow,deny + Allow from all + + + + +Alias /watcher /usr/bin/watcher-api + + SetHandler wsgi-script + Options +ExecCGI + + WSGIProcessGroup watcher + WSGIApplicationGroup %{GLOBAL} + WSGIPassAuthorization On + diff --git a/src/templates/train/watcher.conf b/src/templates/train/watcher.conf new file mode 100644 index 0000000..542c952 --- /dev/null +++ b/src/templates/train/watcher.conf @@ -0,0 +1,228 @@ +# Train +[DEFAULT] +debug = {{ options.debug }} +bind_host = {{ options.service_listen_info.watcher_api.ip }} +bind_port = {{ options.service_listen_info.watcher_api.port }} +host_href = {{ options.external_endpoints.watcher_api.url }} + +{% include "parts/section-transport-url" %} + +{% include "parts/database" %} + +{% include "parts/section-keystone-authtoken" %} + +{% include "parts/section-oslo-messaging-rabbit" %} + +{% include "parts/section-oslo-middleware" %} + +[api] +# The port for the watcher API server (port value) +port = {{ options.service_listen_info.watcher_api.port }} + +# The listen IP address for the watcher API server (host address value) +host = {{ options.service_listen_info.watcher_api.ip }} + +# Number of workers for Watcher API service. The default is equal to the number +# of CPUs available if that can be determined, else a default worker count of 1 +# is returned. (integer value) +# Minimum value: 1 +workers = {{ options.workers }} + +[collector] +# The cluster data model plugin names. +collector_plugins = {{ options.collector_plugins }} + +{% if 'grafana' in options.datasources -%} +[grafana_client] +# See https://docs.openstack.org/watcher/latest/datasources/grafana.html for +# details on how these options are used. + +token = {{ options.grafana_auth_token }} +base_url = {{ options.grafana_base_url }} + +# Mapping of datasource metrics to grafana project ids. Dictionary values +# should be positive integers. +project_id_map = {{ options.grafana_project_id_map }} + +# Mapping of datasource metrics to grafana databases. Values should be strings. +database_map = {{ options.grafana_database_map }} + +# Mapping of datasource metrics to resource attributes. Values should be +# strings. +attribute_map = {{ options.grafana_attribute_map }} + +# Mapping of datasource metrics to grafana translators. Values should be +# strings. +translator_map = {{ options.grafana_translator_map }} + +# Mapping of datasource metrics to grafana queries. Values should be strings +# for which the .format method will transform it. +query_map = {{ options.grafana_query_map }} + +[grafana_translators] +# Keys are the names of retention periods in InfluxDB and the values should +# correspond with the maximum time they can retain in seconds. +retention_periods = {{ options.grafana_retention_periods }} +{%- endif %} + +[cinder_client] +# Type of endpoint to use in cinderclient. (string value) +{% if options.use_internal_endpoints -%} +endpoint_type = internalURL +{% else -%} +endpoint_type = publicURL +{%- endif %} + +# Region in Identity service catalog to use for communication with the +# OpenStack service. (string value) +region_name = {{ options.region }} + +[glance_client] +# Type of endpoint to use in glanceclient. (string value) +{% if options.use_internal_endpoints -%} +endpoint_type = internalURL +{% else -%} +endpoint_type = publicURL +{%- endif %} + +# Region in Identity service catalog to use for communication with the +# OpenStack service. (string value) +region_name = {{ options.region }} + +[gnocchi_client] +# Type of endpoint to use in gnocchi client. (string value) +{% if options.use_internal_endpoints -%} +endpoint_type = internal +{% else -%} +endpoint_type = public +{%- endif %} + +# Region in Identity service catalog to use for communication with the +# OpenStack service. (string value) +region_name = {{ options.region }} + +[keystone_client] +# Type of endpoint to use in keystoneclient. (string value) +{% if options.use_internal_endpoints -%} +interface = internal +{% else -%} +interface = public +{%- endif %} + +# Region in Identity service catalog to use for communication with the +# OpenStack service. (string value) +region_name = {{ options.region }} + +[neutron_client] +# Type of endpoint to use in neutronclient. (string value) +{% if options.use_internal_endpoints -%} +endpoint_type = internalURL +{% else -%} +endpoint_type = publicURL +{%- endif %} + +# Region in Identity service catalog to use for communication with the +# OpenStack service. (string value) +region_name = {{ options.region }} + +[nova_client] +# Type of endpoint to use in novaclient. (string value) +{% if options.use_internal_endpoints -%} +endpoint_type = internalURL +{% else -%} +endpoint_type = publicURL +{%- endif %} + +# Region in Identity service catalog to use for communication with the +# OpenStack service. (string value) +region_name = {{ options.region }} + +[placement_client] +# Type of endpoint when using placement service. (string value) +{% if options.use_internal_endpoints -%} +interface = internal +{% else -%} +interface = public +{%- endif %} + +# Region in Identity service catalog to use for communication with the +# OpenStack service. (string value) +region_name = {{ options.region }} + +[watcher_applier] +# Number of workers for applier, default value is 1. (integer value) +# Minimum value: 1 +workers = {{ options.workers }} + +[watcher_clients_auth] +auth_type = password +auth_uri = {{ identity_service.service_protocol }}://{{ identity_service.service_host }}:{{ identity_service.service_port }} +auth_url = {{ identity_service.auth_protocol }}://{{ identity_service.auth_host }}:{{ identity_service.auth_port }} +project_domain_name = {{ identity_service.service_domain }} +user_domain_name = {{ identity_service.service_domain }} +project_name = {{ identity_service.service_tenant }} +username = {{ identity_service.service_username }} +password = {{ identity_service.service_password }} + +[watcher_cluster_data_model_collectors.baremetal] +period = {{ options.data_model_period }} + +[watcher_cluster_data_model_collectors.compute] +period = {{ options.data_model_period }} + +[watcher_cluster_data_model_collectors.storage] +period = {{ options.data_model_period }} + +[watcher_datasources] +# Datasources to use in order to query the needed metrics. If one of strategy +# metric is not available in the first datasource, the next datasource will be +# chosen. This is the default for all strategies unless a strategy has a +# specific override. (list value) +datasources = {{ options.datasources }} + +[watcher_decision_engine] +# The maximum number of threads that can be used to execute strategies. (integer +# value) +max_workers = {{ options.workers }} + +# An expiry timespan (hours). Watcher invalidates any action plan for which its +# creation time - whose number of hours has been offset by this value - is older +# that the current time. (integer value) +action_plan_expiry = {{ options.action_plan_expiry }} + +# Interval (in seconds) for checking action plan expiry. (integer value) +check_periodic_interval = {{ options.check_periodic_interval }} + +[watcher_planner] +# The selected planner used to schedule the actions. (string value) +planner = {{ options.planner }} + +{% if 'weight' == options.planner -%} +[watcher_planners.weight] +# These weights are used to schedule the actions. Action Plan will be build in +# accordance with sets of actions ordered by descending weights. Two action +# types cannot have the same weight. (dict value) +weights = {{ options.weights }} + +# Number of actions to be run in parallel on a per action type basis. (dict +# value) +parallelization = {{ options.parallelization }} +{%- endif %} + +{% if 'workload_stabilization' == options.planner -%} +[watcher_planners.workload_stabilization] +# These weights are used to schedule the actions. (dict value) +weights = {{ options.weights }} +{%- endif %} + +{% if 'basic' == options.planner -%} +[watcher_strategies.basic] +# Check optimize metadata field in instance before migration. (boolean value) +check_optimize_metadata = {{ options.check_optimize_metadata }} +{%- endif %} + +{% if 'storage_capacity_balance' == options.planner -%} +[watcher_strategies.storage_capacity_balance] +# exclude pools (list value) +ex_pools = {{ options.ex_pools }} +{%- endif %} diff --git a/src/test-requirements.txt b/src/test-requirements.txt new file mode 100644 index 0000000..1a31860 --- /dev/null +++ b/src/test-requirements.txt @@ -0,0 +1,11 @@ +# This file is managed centrally. If you find the need to modify this as a +# one-off, please don't. Instead, consult #openstack-charms and ask about +# requirements management in charms via bot-control. Thank you. +charm-tools>=2.4.4 +coverage>=3.6 +mock>=1.2 +flake8>=2.2.4,<=2.4.1 +stestr>=2.2.0 +requests>=2.18.4 +git+https://github.com/openstack-charmers/zaza.git#egg=zaza +git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack diff --git a/src/tox.ini b/src/tox.ini new file mode 100644 index 0000000..00c1134 --- /dev/null +++ b/src/tox.ini @@ -0,0 +1,44 @@ +[tox] +envlist = pep8 +skipsdist = True +# NOTE(beisner): Avoid build/test env pollution by not enabling sitepackages. +sitepackages = False +# NOTE(beisner): Avoid false positives by not skipping missing interpreters. +skip_missing_interpreters = False + +[testenv] +setenv = VIRTUAL_ENV={envdir} + PYTHONHASHSEED=0 +whitelist_externals = juju +passenv = HOME TERM CS_API_* OS_* +deps = -r{toxinidir}/test-requirements.txt +install_command = + pip install {opts} {packages} + +[testenv:pep8] +basepython = python3 +deps=charm-tools +commands = charm-proof + +[testenv:func-noop] +basepython = python3 +commands = + true + +[testenv:func] +basepython = python3 +commands = + functest-run-suite --keep-model + +[testenv:func-smoke] +basepython = python3 +commands = + functest-run-suite --keep-model --smoke + +[testenv:func-target] +basepython = python3 +commands = + functest-run-suite --keep-model --bundle {posargs} + +[testenv:venv] +commands = {posargs} diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..dd5e16a --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,14 @@ +# This file is managed centrally by release-tools and should not be modified +# within individual charm repos. See the 'global' dir contents for available +# choices of *requirements.txt files for OpenStack Charms: +# https://github.com/openstack-charmers/release-tools +# +# Lint and unit test requirements +flake8>=2.2.4,<=2.4.1 +stestr>=2.2.0 +requests>=2.18.4 +charms.reactive +mock>=1.2 +nose>=1.3.7 +coverage>=3.6 +git+https://github.com/openstack/charms.openstack.git#egg=charms.openstack diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..2cc5a50 --- /dev/null +++ b/tox.ini @@ -0,0 +1,88 @@ +# Source charm: ./tox.ini +# This file is managed centrally by release-tools and should not be modified +# within individual charm repos. +[tox] +skipsdist = True +envlist = pep8,py3 +# NOTE(beisner): Avoid build/test env pollution by not enabling sitepackages. +sitepackages = False +# NOTE(beisner): Avoid false positives by not skipping missing interpreters. +skip_missing_interpreters = False + +[testenv] +setenv = VIRTUAL_ENV={envdir} + PYTHONHASHSEED=0 + TERM=linux + LAYER_PATH={toxinidir}/layers + JUJU_REPOSITORY={toxinidir}/build +passenv = http_proxy https_proxy INTERFACE_PATH +install_command = + pip install {opts} {packages} +deps = + -r{toxinidir}/requirements.txt + +[testenv:build] +basepython = python3 +commands = + charm-build --log-level DEBUG -o {toxinidir}/build src {posargs} + +[testenv:py3] +basepython = python3 +deps = -r{toxinidir}/test-requirements.txt +commands = stestr run {posargs} + +[testenv:py35] +basepython = python3.5 +deps = -r{toxinidir}/test-requirements.txt +commands = stestr run {posargs} + +[testenv:py36] +basepython = python3.6 +deps = -r{toxinidir}/test-requirements.txt +commands = stestr run {posargs} + +[testenv:py37] +basepython = python3.7 +deps = -r{toxinidir}/test-requirements.txt +commands = stestr run {posargs} + +[testenv:pep8] +basepython = python3 +deps = -r{toxinidir}/test-requirements.txt +commands = flake8 {posargs} src unit_tests + +[testenv:cover] +# Technique based heavily upon +# https://github.com/openstack/nova/blob/master/tox.ini +basepython = python3 +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +setenv = + {[testenv]setenv} + PYTHON=coverage run +commands = + coverage erase + stestr run {posargs} + coverage combine + coverage html -d cover + coverage xml -o cover/coverage.xml + coverage report + +[coverage:run] +branch = True +concurrency = multiprocessing +parallel = True +source = + . +omit = + .tox/* + */charmhelpers/* + unit_tests/* + +[testenv:venv] +basepython = python3 +commands = {posargs} + +[flake8] +# E402 ignore necessary for path append before sys module import in actions +ignore = E402 diff --git a/unit_tests/__init__.py b/unit_tests/__init__.py new file mode 100644 index 0000000..8f28921 --- /dev/null +++ b/unit_tests/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2020 GRNET SA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +sys.path.append('src') +sys.path.append('src/lib') + +# Mock out charmhelpers so that we can test without it. +import charms_openstack.test_mocks # noqa +charms_openstack.test_mocks.mock_charmhelpers() diff --git a/unit_tests/test_lib_charm_openstack_watcher.py b/unit_tests/test_lib_charm_openstack_watcher.py new file mode 100644 index 0000000..02d56bd --- /dev/null +++ b/unit_tests/test_lib_charm_openstack_watcher.py @@ -0,0 +1,334 @@ +# Copyright 2020 GRNET SA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from __future__ import print_function +from collections import OrderedDict + +from unittest import mock + +import charmhelpers + +import charm.openstack.watcher as watcher + +import charms_openstack.test_utils as test_utils + + +class Helper(test_utils.PatchHelper): + + def setUp(self): + super().setUp() + self.patch_release(watcher.WatcherCharm.release) + + +class TestWatcherCharmConfigProperties(Helper): + def test_planner_weights(self): + cls = mock.MagicMock() + self.patch('charm.openstack.watcher.config', 'config') + self.patch('charm.openstack.watcher.log', 'log') + self.patch( + 'charm.openstack.watcher.config_flags_parser', + 'config_flags_parser') + self.config.side_effect = [ + 'weight', + '{"weights": "change_node_power_state:9,' + 'change_nova_service_state:50"}'] + conf_dict = OrderedDict() + conf_dict['weights'] = \ + 'change_node_power_state:9,change_nova_service_state:50' + self.config_flags_parser.return_value = conf_dict + + self.assertEqual( + watcher.planner_weights(cls), + 'change_node_power_state:9,change_nova_service_state:50') + + def test_planner_weights_invalid(self): + cls = mock.MagicMock() + self.patch('charm.openstack.watcher.config', 'config') + self.patch('charm.openstack.watcher.log', 'log') + self.patch('charm.openstack.watcher.WARNING', 'WARNING') + self.patch( + 'charm.openstack.watcher.config_flags_parser', + 'config_flags_parser') + self.config.side_effect = ['weight', '{}'] + self.config_flags_parser.return_value = OrderedDict() + + self.assertEqual(watcher.planner_weights(cls), None) + self.log.assert_called_once_with( + 'Provided planner-config dictionary does not contain ' + 'key weights - ignoring', level=self.WARNING) + + def test_planner_parallelization(self): + cls = mock.MagicMock() + self.patch('charm.openstack.watcher.config', 'config') + self.patch('charm.openstack.watcher.log', 'log') + self.patch( + 'charm.openstack.watcher.config_flags_parser', + 'config_flags_parser') + self.config.side_effect = [ + 'weight', + '{"parallelization": "change_node_power_state:9,' + 'change_nova_service_state:50"}'] + conf_dict = OrderedDict() + conf_dict['parallelization'] = \ + 'change_node_power_state:9,change_nova_service_state:50' + self.config_flags_parser.return_value = conf_dict + + self.assertEqual( + watcher.planner_parallelization(cls), + 'change_node_power_state:9,change_nova_service_state:50') + + def test_planner_parallelization_invalid(self): + cls = mock.MagicMock() + self.patch('charm.openstack.watcher.config', 'config') + self.patch('charm.openstack.watcher.log', 'log') + self.patch('charm.openstack.watcher.WARNING', 'WARNING') + self.patch( + 'charm.openstack.watcher.config_flags_parser', + 'config_flags_parser') + self.config.side_effect = ['weight', '{}'] + self.config_flags_parser.return_value = OrderedDict() + + self.assertEqual(watcher.planner_parallelization(cls), None) + self.log.assert_called_once_with( + 'Provided planner-config dictionary does not contain ' + 'key parallelization - ignoring', level=self.WARNING) + + def test_planner_check_optimize_metadata(self): + cls = mock.MagicMock() + self.patch('charm.openstack.watcher.config', 'config') + self.patch('charm.openstack.watcher.log', 'log') + self.patch( + 'charm.openstack.watcher.config_flags_parser', + 'config_flags_parser') + self.config.side_effect = [ + 'basic', '{"check_optimize_metadata": "true"}'] + conf_dict = OrderedDict() + conf_dict['check_optimize_metadata'] = 'true' + self.config_flags_parser.return_value = conf_dict + + self.assertEqual( + watcher.planner_check_optimize_metadata(cls), 'true') + + def test_planner_check_optimize_metadata_invalid(self): + cls = mock.MagicMock() + self.patch('charm.openstack.watcher.config', 'config') + self.patch('charm.openstack.watcher.log', 'log') + self.patch('charm.openstack.watcher.WARNING', 'WARNING') + self.patch( + 'charm.openstack.watcher.config_flags_parser', + 'config_flags_parser') + self.config.side_effect = ['basic', '{}'] + self.config_flags_parser.return_value = OrderedDict() + + self.assertEqual(watcher.planner_check_optimize_metadata(cls), None) + self.log.assert_called_once_with( + 'Provided planner-config dictionary does not contain ' + 'key check_optimize_metadata - ignoring', level=self.WARNING) + + def test_planner_ex_pools(self): + cls = mock.MagicMock() + self.patch('charm.openstack.watcher.config', 'config') + self.patch('charm.openstack.watcher.log', 'log') + self.patch( + 'charm.openstack.watcher.config_flags_parser', + 'config_flags_parser') + self.config.side_effect = [ + 'storage_capacity_balance', '{"ex_pools": "local_vstorage"}'] + conf_dict = OrderedDict() + conf_dict['ex_pools'] = 'local_vstorage' + self.config_flags_parser.return_value = conf_dict + + self.assertEqual( + watcher.planner_ex_pools(cls), 'local_vstorage') + + def test_planner_ex_pools_invalid(self): + cls = mock.MagicMock() + self.patch('charm.openstack.watcher.config', 'config') + self.patch('charm.openstack.watcher.log', 'log') + self.patch('charm.openstack.watcher.WARNING', 'WARNING') + self.patch( + 'charm.openstack.watcher.config_flags_parser', + 'config_flags_parser') + self.config.side_effect = ['storage_capacity_balance', '{}'] + self.config_flags_parser.return_value = OrderedDict() + + self.assertEqual(watcher.planner_ex_pools(cls), None) + self.log.assert_called_once_with( + 'Provided planner-config dictionary does not contain key ex_pools ' + '- ignoring', level=self.WARNING) + + +class TestWatcherCharm(Helper): + def _patch_config_and_charm(self, config): + self.patch_object(charmhelpers.core.hookenv, 'config') + + def cf(key=None): + if key is not None: + return config[key] + return config + + self.config.side_effect = cf + c = watcher.WatcherCharm() + return c + + def _patch_get_adapter(self, c): + self.patch_object(c, 'get_adapter') + + def _helper(x): + self.var = x + return self.out + + self.get_adapter.side_effect = _helper + + def test_get_amqp_credentials(self): + c = watcher.WatcherCharm() + result = c.get_amqp_credentials() + + self.assertEqual(result, ('watcher', 'openstack')) + + def test_get_database_setup(self): + c = watcher.WatcherCharm() + result = c.get_database_setup() + + self.assertEqual(result, [{'database': 'watcher', + 'username': 'watcher'}]) + + def test_custom_assess_status_check1(self): + config = { + 'datasources': 'gnocchi', + 'planner': 'weight', + 'planner-config': '{"weights": "42", "parallelization": "42"}' + } + c = self._patch_config_and_charm(config) + self._patch_get_adapter(c) + self.patch( + 'charm.openstack.watcher.config_flags_parser', + 'config_flags_parser') + conf_dict = OrderedDict() + conf_dict['weights'] = '42' + conf_dict['parallelization'] = '42' + self.config_flags_parser.return_value = conf_dict + + self.assertEqual(c.options.datasources, config['datasources']) + self.assertEqual(c.options.planner, config['planner']) + self.assertEqual(c.options.planner_config, config['planner-config']) + self.assertEqual(c.custom_assess_status_check(), (None, None)) + + def test_custom_assess_status_check2(self): + config = { + 'datasources': 'gnocchi', + 'planner': 'invalid', + } + c = self._patch_config_and_charm(config) + self._patch_get_adapter(c) + + self.assertEqual(c.options.datasources, config['datasources']) + self.assertEqual(c.options.planner, config['planner']) + self.assertEqual( + c.custom_assess_status_check(), + ('blocked', + 'Invalid planner: {}. Available options are: ' + 'weights, workload_stabilization, basic, ' + 'storage_capacity_balance'.format(config['planner']))) + + def test_custom_assess_status_check3(self): + config = { + 'datasources': 'gnocchi', + 'planner': 'basic', + 'planner-config': 'invalid' + } + c = self._patch_config_and_charm(config) + self._patch_get_adapter(c) + self.patch( + 'charm.openstack.watcher.config_flags_parser', + 'config_flags_parser') + self.config_flags_parser.return_value = OrderedDict() + + self.assertEqual(c.options.datasources, config['datasources']) + self.assertEqual(c.options.planner, config['planner']) + self.assertEqual(c.options.planner_config, config['planner-config']) + self.assertEqual( + c.custom_assess_status_check(), + ('blocked', + 'Provided planner {} must contain only the following ' + 'configuration attributes: check_optimize_metadata'.format( + config['planner']))) + + def test_custom_assess_status_check4(self): + config = { + 'datasources': None, + } + c = self._patch_config_and_charm(config) + self._patch_get_adapter(c) + + self.assertEqual(c.options.datasources, config['datasources']) + self.assertEqual( + c.custom_assess_status_check(), + ('blocked', + 'datasources not set')) + + def test_custom_assess_status_check5(self): + config = { + 'datasources': 'invalid', + } + c = self._patch_config_and_charm(config) + self._patch_get_adapter(c) + + self.assertEqual(c.options.datasources, config['datasources']) + self.assertEqual( + c.custom_assess_status_check(), + ('blocked', + 'Provided datasources {} does not contain valid options'.format( + config['datasources']))) + + def test_custom_assess_status_check6(self): + config = { + 'datasources': 'grafana', + } + c = self._patch_config_and_charm(config) + self._patch_get_adapter(c) + self.patch_object(c, 'grafana_configuration_complete') + self.grafana_configuration_complete.return_value = False + + self.assertEqual(c.options.datasources, config['datasources']) + self.assertEqual( + c.custom_assess_status_check(), + ('blocked', + 'grafana datasource requires all grafana related options to be ' + 'set')) + + def test_grafana_configuration_complete(self): + config = { + 'grafana-auth-token': 1, + 'grafana-base-url': 2, + 'grafana-project-id-map': 3, + 'grafana-database-map': 4, + 'grafana-attribute-map': 5, + 'grafana-translator-map': 6, + 'grafana-query-map': 7, + 'grafana-retention-periods': 8, + } + c = self._patch_config_and_charm(config) + self._patch_get_adapter(c) + + self.assertEqual(c.options.grafana_auth_token, 1) + self.assertEqual(c.options.grafana_base_url, 2) + self.assertEqual(c.options.grafana_project_id_map, 3) + self.assertEqual(c.options.grafana_database_map, 4) + self.assertEqual(c.options.grafana_attribute_map, 5) + self.assertEqual(c.options.grafana_translator_map, 6) + self.assertEqual(c.options.grafana_query_map, 7) + self.assertEqual(c.options.grafana_retention_periods, 8) + self.assertEqual(c.grafana_configuration_complete(), True) diff --git a/unit_tests/test_watcher_handlers.py b/unit_tests/test_watcher_handlers.py new file mode 100644 index 0000000..6744e7f --- /dev/null +++ b/unit_tests/test_watcher_handlers.py @@ -0,0 +1,90 @@ +# Copyright 2020 GRNET SA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from __future__ import print_function + +import mock + +import reactive.watcher_handlers as handlers + +import charms_openstack.test_utils as test_utils + + +class TestRegisteredHooks(test_utils.TestRegisteredHooks): + + def test_hooks(self): + defaults = [ + 'charm.installed', + 'amqp.connected', + 'shared-db.connected', + 'identity-service.connected', + 'config.changed', + 'update-status', + 'upgrade-charm', + 'certificates.available'] + hook_set = { + 'when': { + 'render_config': ('shared-db.available', + 'identity-service.available', + 'amqp.available',), + 'init_db': ('config.rendered',), + 'cluster_connected': ('ha.connected',), + }, + 'when_not': { + 'init_db': ('db.synced',), + } + } + # test that the hooks were registered via the + # reactive.watcher_handlers + self.registered_hooks_test_helper(handlers, hook_set, defaults) + + +class TestWatcherHandlers(test_utils.PatchHelper): + + def setUp(self): + super().setUp() + self.watcher_charm = mock.MagicMock() + self.patch_object(handlers.charms_openstack.charm, + 'provide_charm_instance', + new=mock.MagicMock()) + self.provide_charm_instance().__enter__.return_value = ( + self.watcher_charm) + self.provide_charm_instance().__exit__.return_value = None + + def test_render_config(self): + self.patch_object(handlers.reactive, 'set_state') + handlers.render_config('arg1', 'arg2') + + self.watcher_charm.render_with_interfaces.assert_called_once_with( + ('arg1', 'arg2')) + self.watcher_charm.assess_status.assert_called_once_with() + self.set_state.assert_called_once_with('config.rendered') + + def test_init_db(self): + self.patch('charms.reactive.set_state', 'set_state') + handlers.init_db() + + self.watcher_charm.db_sync.assert_called_once_with() + self.watcher_charm.restart_all.assert_called_once_with() + self.set_state.assert_called_once_with('db.synced') + self.watcher_charm.assess_status.assert_called_once_with() + + def test_cluster_connected(self): + hacluster = mock.MagicMock() + handlers.cluster_connected(hacluster) + + self.watcher_charm.configure_ha_resources.assert_called_once_with( + hacluster) + self.watcher_charm.assess_status.assert_called_once_with()