Insert DB:etcd for BP:multi-podmanager

This patch adds etcd DB for multi-podmanager as well as valence.

Change-Id: Ie1ef2554cc95a5f9f18341e768f27a02e192b588
Closes-Bug: #1646389

README.rst

@@ -19,6 +19,16 @@ Download and Installation
 The following steps capture how to install valence. All installation steps
 require super user permissions.
 
+*******************************************
+Database etcd installation
+*******************************************
+
+ Single node installation reference: https://github.com/coreos/etcd/releases
+
+ Distributed installation reference: https://github.com/coreos/etcd/blob/master/Documentation/op-guide/clustering.md
+
+ For development, single node installation is recommended practice.
+
 *******************************************
 Valence installation
 *******************************************
@@ -43,11 +53,15 @@ Valence installation
 
  5. Check the PYTHON_HOME and other variables in /etc/init/valence.conf
 
- 6. Start valence service
+ 6. Initialize etcd database
+
+    ``$ sudo db_manager init``
+
+ 7. Start valence service
 
     ``$ sudo service valence start``
 
- 7. Logs are located at /var/logs/valence/
+ 8. Logs are located at /var/logs/valence/
 
 ****************
 GUI installation

etc/valence/valence.conf.sample

@@ -18,3 +18,7 @@ url=https://<ip address>:<port>
 user=<podm user>
 password=<podm admin>
 redfish_base_ext=/redfish/v1/
+
+[database_etcd]
+host = localhost
+port = 2379

requirements.txt

@@ -12,3 +12,5 @@ pytz==2016.7
 requests==2.11.1
 six==1.10.0
 gunicorn==19.6.0
+python-etcd>=0.4.3 # MIT License
+oslo.utils>=3.18.0 # Apache-2.0

setup.cfg

@@ -53,3 +53,4 @@ source-dir = releasenotes/source
 [entry_points]
 console_scripts =
     valence = valence.cmd.api:main
+    db_manager = valence.cmd.db_manager:main

valence/cmd/db_manager.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+
+# copyright (c) 2016 Intel, Inc.
+#
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import logging
+import sys
+
+from valence.db.etcd_db import init_etcd_db
+
+LOG = logging.getLogger(__name__)
+
+
+def init():
+    init_etcd_db()
+
+
+def migrate():
+    pass
+
+
+def main(argv):
+    if argv[1] == 'init':
+        LOG.info("initialize database")
+        init()
+
+
+if __name__ == '__main__':
+    main(sys.argv)

valence/config.py

@@ -67,3 +67,7 @@ podm_url = get_option("podm", "url", "http://127.0.0.1")
 podm_user = get_option("podm", "user", "admin")
 podm_password = get_option("podm", "password", "admin")
 redfish_base_ext = get_option("podm", "redfish_base_ext", "/redfish/v1/")
+
+# Database etcd Settings
+etcd_host = get_option("database_etcd", "host", "localhost")
+etcd_port = get_option("database_etcd", "port", 2379)

valence/db/etcd_adapter.py

@@ -0,0 +1,73 @@
+# Copyright 2016 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import json
+
+from etcd import EtcdKeyNotFound
+from oslo_utils import uuidutils
+
+from valence.db.etcd_db import etcd_client
+from valence.db.etcd_db import etcd_directories
+from valence.db.models.pod_manager import PodManager
+
+
+class EtcdAdapter(object):
+    dir_model_map = {
+        "/v1/pod_managers": PodManager
+    }
+
+    def get_model_from_dir(self, directory):
+        return self.dir_model_map[directory]
+
+    def add_object(self, directory, **object_items):
+        if directory not in etcd_directories:
+            return "directory not found, invalid request"
+
+        gen_uuid = uuidutils.generate_uuid()
+        db_object = self.get_model_from_dir(directory).convert(**object_items)
+        etcd_client.write(directory + '/' + gen_uuid, db_object)
+        pod_info = self.get_object_by_uuid(directory, gen_uuid)
+        return pod_info
+
+    @staticmethod
+    def update_object(key, **update_items):
+        db_object = etcd_client.read(key)
+        # update the object's value properties
+        db_object_value = json.loads(db_object.value)
+        for k, v in update_items.items():
+            db_object_value[k] = v
+        # reassignment the object's value then update the whole object
+        db_object.value = db_object_value
+        etcd_client.update(db_object)
+
+    @staticmethod
+    def get_object_list(directory):
+        if directory not in etcd_directories:
+            return "directory not found, invalid request"
+
+        try:
+            object_list = etcd_client.read(directory)
+            return map(lambda x: x['value'], object_list._children)
+        except EtcdKeyNotFound:
+            return []
+
+    @staticmethod
+    def get_object_by_uuid(directory, uuid):
+        if not uuidutils.is_uuid_like(uuid):
+            return None
+        try:
+            object_info = etcd_client.read(directory + '/' + uuid).value
+            return object_info
+        except EtcdKeyNotFound:
+            return None

valence/db/etcd_db.py

@@ -0,0 +1,31 @@
+# Copyright 2016 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+import etcd
+
+from valence import config
+
+
+etcd_directories = [
+    "/v1/pod_managers"
+]
+
+etcd_client = etcd.Client(config.etcd_host, int(config.etcd_port))
+
+
+def init_etcd_db():
+    """initialize etcd database"""
+    for directory in etcd_directories:
+        etcd_client.write(directory, None, dir=True, append=True)

valence/db/models/pod_manager.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+
+# copyright (c) 2016 Intel, Inc.
+#
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import json
+from valence.api import base
+from valence.api import types
+
+
+class PodManager(base.APIBase):
+
+    fields = {
+        'name': {
+            'validate': types.Text.validate
+        },
+        'url': {
+            'validate': types.Text.validate
+        },
+        'auth': {
+            'validate': types.Text.validate
+        },
+        'status': {
+            'validate': types.Text.validate
+        },
+        'description': {
+            'validate': types.Text.validate
+        },
+        'location': {
+            'validate': types.Text.validate
+        },
+        'redfish_link': {
+            'validate': types.Text.validate
+        },
+        'bookmark_link': {
+            'validate': types.Text.validate
+        },
+        'created_at': {
+            'validate': types.Text.validate
+        },
+        'updated_at': {
+            'validate': types.Text.validate
+        },
+    }
+
+    @staticmethod
+    def convert(name=None, url=None, auth=None, status=None,
+                description=None, location=None, redfish_link=None,
+                bookmark_link=None, created_at=None, updated_at=None):
+        pod_obj = PodManager()
+        pod_obj.name = name
+        pod_obj.url = url
+        pod_obj.auth = auth
+        pod_obj.status = status
+        pod_obj.description = description
+        pod_obj.location = location
+        pod_obj.redfish_link = redfish_link
+        pod_obj.bookmark_link = bookmark_link
+        pod_obj.created_at = created_at
+        pod_obj.updated_at = updated_at
+        return json.dumps(pod_obj, default=lambda o: o.as_dict())