From af61ee043af0c796e16d08f1e82faa4f5f648ae2 Mon Sep 17 00:00:00 2001 From: hubian Date: Fri, 2 Dec 2016 10:33:03 +0800 Subject: [PATCH] Insert DB:etcd for BP:multi-podmanager This patch adds etcd DB for multi-podmanager as well as valence. Change-Id: Ie1ef2554cc95a5f9f18341e768f27a02e192b588 Closes-Bug: #1646389 --- README.rst | 18 +++++++- etc/valence/valence.conf.sample | 4 ++ requirements.txt | 2 + setup.cfg | 1 + valence/cmd/db_manager.py | 39 +++++++++++++++++ valence/config.py | 4 ++ valence/db/__init__.py | 0 valence/db/etcd_adapter.py | 73 ++++++++++++++++++++++++++++++++ valence/db/etcd_db.py | 31 ++++++++++++++ valence/db/models/__init__.py | 0 valence/db/models/pod_manager.py | 71 +++++++++++++++++++++++++++++++ 11 files changed, 241 insertions(+), 2 deletions(-) create mode 100644 valence/cmd/db_manager.py create mode 100644 valence/db/__init__.py create mode 100644 valence/db/etcd_adapter.py create mode 100644 valence/db/etcd_db.py create mode 100644 valence/db/models/__init__.py create mode 100644 valence/db/models/pod_manager.py diff --git a/README.rst b/README.rst index 03e8662..95c5581 100644 --- a/README.rst +++ b/README.rst @@ -19,6 +19,16 @@ Download and Installation The following steps capture how to install valence. All installation steps require super user permissions. +******************************************* +Database etcd installation +******************************************* + + Single node installation reference: https://github.com/coreos/etcd/releases + + Distributed installation reference: https://github.com/coreos/etcd/blob/master/Documentation/op-guide/clustering.md + + For development, single node installation is recommended practice. + ******************************************* Valence installation ******************************************* @@ -43,11 +53,15 @@ Valence installation 5. Check the PYTHON_HOME and other variables in /etc/init/valence.conf - 6. Start valence service + 6. Initialize etcd database + + ``$ sudo db_manager init`` + + 7. Start valence service ``$ sudo service valence start`` - 7. Logs are located at /var/logs/valence/ + 8. Logs are located at /var/logs/valence/ **************** GUI installation diff --git a/etc/valence/valence.conf.sample b/etc/valence/valence.conf.sample index fcd11fe..d1ac73a 100644 --- a/etc/valence/valence.conf.sample +++ b/etc/valence/valence.conf.sample @@ -18,3 +18,7 @@ url=https://: user= password= redfish_base_ext=/redfish/v1/ + +[database_etcd] +host = localhost +port = 2379 diff --git a/requirements.txt b/requirements.txt index e85b923..7817635 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,5 @@ pytz==2016.7 requests==2.11.1 six==1.10.0 gunicorn==19.6.0 +python-etcd>=0.4.3 # MIT License +oslo.utils>=3.18.0 # Apache-2.0 diff --git a/setup.cfg b/setup.cfg index 7828023..71f6ac3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -53,3 +53,4 @@ source-dir = releasenotes/source [entry_points] console_scripts = valence = valence.cmd.api:main + db_manager = valence.cmd.db_manager:main \ No newline at end of file diff --git a/valence/cmd/db_manager.py b/valence/cmd/db_manager.py new file mode 100644 index 0000000..e1d476a --- /dev/null +++ b/valence/cmd/db_manager.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python + +# copyright (c) 2016 Intel, Inc. +# +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import sys + +from valence.db.etcd_db import init_etcd_db + +LOG = logging.getLogger(__name__) + + +def init(): + init_etcd_db() + + +def migrate(): + pass + + +def main(argv): + if argv[1] == 'init': + LOG.info("initialize database") + init() + + +if __name__ == '__main__': + main(sys.argv) diff --git a/valence/config.py b/valence/config.py index e0a1bf0..b88fb1e 100644 --- a/valence/config.py +++ b/valence/config.py @@ -67,3 +67,7 @@ podm_url = get_option("podm", "url", "http://127.0.0.1") podm_user = get_option("podm", "user", "admin") podm_password = get_option("podm", "password", "admin") redfish_base_ext = get_option("podm", "redfish_base_ext", "/redfish/v1/") + +# Database etcd Settings +etcd_host = get_option("database_etcd", "host", "localhost") +etcd_port = get_option("database_etcd", "port", 2379) diff --git a/valence/db/__init__.py b/valence/db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/valence/db/etcd_adapter.py b/valence/db/etcd_adapter.py new file mode 100644 index 0000000..e759ffd --- /dev/null +++ b/valence/db/etcd_adapter.py @@ -0,0 +1,73 @@ +# Copyright 2016 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json + +from etcd import EtcdKeyNotFound +from oslo_utils import uuidutils + +from valence.db.etcd_db import etcd_client +from valence.db.etcd_db import etcd_directories +from valence.db.models.pod_manager import PodManager + + +class EtcdAdapter(object): + dir_model_map = { + "/v1/pod_managers": PodManager + } + + def get_model_from_dir(self, directory): + return self.dir_model_map[directory] + + def add_object(self, directory, **object_items): + if directory not in etcd_directories: + return "directory not found, invalid request" + + gen_uuid = uuidutils.generate_uuid() + db_object = self.get_model_from_dir(directory).convert(**object_items) + etcd_client.write(directory + '/' + gen_uuid, db_object) + pod_info = self.get_object_by_uuid(directory, gen_uuid) + return pod_info + + @staticmethod + def update_object(key, **update_items): + db_object = etcd_client.read(key) + # update the object's value properties + db_object_value = json.loads(db_object.value) + for k, v in update_items.items(): + db_object_value[k] = v + # reassignment the object's value then update the whole object + db_object.value = db_object_value + etcd_client.update(db_object) + + @staticmethod + def get_object_list(directory): + if directory not in etcd_directories: + return "directory not found, invalid request" + + try: + object_list = etcd_client.read(directory) + return map(lambda x: x['value'], object_list._children) + except EtcdKeyNotFound: + return [] + + @staticmethod + def get_object_by_uuid(directory, uuid): + if not uuidutils.is_uuid_like(uuid): + return None + try: + object_info = etcd_client.read(directory + '/' + uuid).value + return object_info + except EtcdKeyNotFound: + return None diff --git a/valence/db/etcd_db.py b/valence/db/etcd_db.py new file mode 100644 index 0000000..906e54e --- /dev/null +++ b/valence/db/etcd_db.py @@ -0,0 +1,31 @@ +# Copyright 2016 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import etcd + +from valence import config + + +etcd_directories = [ + "/v1/pod_managers" +] + +etcd_client = etcd.Client(config.etcd_host, int(config.etcd_port)) + + +def init_etcd_db(): + """initialize etcd database""" + for directory in etcd_directories: + etcd_client.write(directory, None, dir=True, append=True) diff --git a/valence/db/models/__init__.py b/valence/db/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/valence/db/models/pod_manager.py b/valence/db/models/pod_manager.py new file mode 100644 index 0000000..9d24e9a --- /dev/null +++ b/valence/db/models/pod_manager.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python + +# copyright (c) 2016 Intel, Inc. +# +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json +from valence.api import base +from valence.api import types + + +class PodManager(base.APIBase): + + fields = { + 'name': { + 'validate': types.Text.validate + }, + 'url': { + 'validate': types.Text.validate + }, + 'auth': { + 'validate': types.Text.validate + }, + 'status': { + 'validate': types.Text.validate + }, + 'description': { + 'validate': types.Text.validate + }, + 'location': { + 'validate': types.Text.validate + }, + 'redfish_link': { + 'validate': types.Text.validate + }, + 'bookmark_link': { + 'validate': types.Text.validate + }, + 'created_at': { + 'validate': types.Text.validate + }, + 'updated_at': { + 'validate': types.Text.validate + }, + } + + @staticmethod + def convert(name=None, url=None, auth=None, status=None, + description=None, location=None, redfish_link=None, + bookmark_link=None, created_at=None, updated_at=None): + pod_obj = PodManager() + pod_obj.name = name + pod_obj.url = url + pod_obj.auth = auth + pod_obj.status = status + pod_obj.description = description + pod_obj.location = location + pod_obj.redfish_link = redfish_link + pod_obj.bookmark_link = bookmark_link + pod_obj.created_at = created_at + pod_obj.updated_at = updated_at + return json.dumps(pod_obj, default=lambda o: o.as_dict())