deployment module

Change-Id: I1305b74595ab3259e9a813157eb4d8e653ca6f3f

compass/actions/deploy.py

@@ -13,16 +13,21 @@
 # limitations under the License.
 
 """Module to deploy a given cluster
-
-   .. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
 """
-import logging
 
 from compass.actions import util
-from compass.db.api import database
+from compass.db.api import adapter_holder as adapter_db
+from compass.db.api import cluster as cluster_db
+from compass.db.api import host as host_db
+from compass.db.api import user as user_db
+from compass.deployment.deploy_manager import DeployManager
+from compass.deployment.utils import constants as const
 
 
-def deploy(cluster_hosts):
+import logging
+
+
+def deploy(cluster_id, hosts_id_list, username=None):
     """Deploy clusters.
 
     :param cluster_hosts: clusters and hosts in each cluster to deploy.
@@ -35,4 +40,207 @@ def deploy(cluster_hosts):
         if not lock:
             raise Exception('failed to acquire lock to deploy')
 
-        logging.debug('deploy cluster_hosts: %s', cluster_hosts)
+        user = user_db.get_user_object(username)
+
+        cluster_info = ActionHelper.get_cluster_info(cluster_id, user)
+        adapter_id = cluster_info[const.ADAPTER_ID]
+
+        adapter_info = ActionHelper.get_adapter_info(adapter_id, cluster_id,
+                                                     user)
+        hosts_info = ActionHelper.get_hosts_info(hosts_id_list, user)
+
+        logging.debug('[action][deploy]: adapter_info is %s', adapter_info)
+        logging.debug('[action][deploy]: cluster_info is %s', cluster_info)
+        logging.debug('[action][deploy]: hosts_info is %s', hosts_info)
+
+        deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info)
+        #deploy_manager.prepare_for_deploy()
+        deployed_config = deploy_manager.deploy()
+
+        ActionHelper.save_deployed_config(deployed_config, user)
+
+
+def redeploy(cluster_id, hosts_id_list, username=None):
+    """Deploy clusters.
+
+    :param cluster_hosts: clusters and hosts in each cluster to deploy.
+    :type cluster_hosts: dict of int or str to list of int or str
+    """
+    with util.lock('serialized_action') as lock:
+        if not lock:
+            raise Exception('failed to acquire lock to deploy')
+
+        user = user_db.get_user_object(username)
+        cluster_info = ActionHelper.get_cluster_info(cluster_id, user)
+        adapter_id = cluster_info[const.ADAPTER_ID]
+
+        adapter_info = ActionHelper.get_adapter_info(adapter_id,
+                                                     cluster_id,
+                                                     user)
+        hosts_info = ActionHelper.get_hosts_info(cluster_id,
+                                                 hosts_id_list,
+                                                 user)
+
+        deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info)
+        #deploy_manager.prepare_for_deploy()
+        deploy_manager.redeploy()
+
+
+def poweron(host_id):
+    """Power on a list of hosts."""
+    pass
+
+
+def poweroff(host_id):
+    pass
+
+
+def reset(host_id):
+    pass
+
+
+class ActionHelper(object):
+
+    @staticmethod
+    def get_adapter_info(adapter_id, cluster_id, user):
+        """Get adapter information. Return a dictionary as below,
+           {
+              "id": 1,
+              "name": "xxx",
+              "roles": ['xxx', 'yyy', ...],
+              "metadata": {
+                  "os_config": {
+                      ...
+                  },
+                  "package_config": {
+                      ...
+                  }
+              },
+              "os_installer": {
+                  "name": "cobbler",
+                  "settings": {....}
+              },
+              "pk_installer": {
+                  "name": "chef",
+                  "settings": {....}
+              },
+              ...
+           }
+           To view a complete output, please refer to backend doc.
+        """
+        adapter_info = adapter_db.get_adapter(user, adapter_id)
+        metadata = cluster_db.get_cluster_metadata(user, cluster_id)
+        adapter_info.update(metadata)
+
+        roles_info = adapter_info[const.ROLES]
+        roles_list = [role[const.NAME] for role in roles_info]
+        adapter_info[const.ROLES] = roles_list
+
+        return adapter_info
+
+    @staticmethod
+    def get_cluster_info(cluster_id, user):
+        """Get cluster information.Return a dictionary as below,
+           {
+               "cluster": {
+                   "id": 1,
+                   "adapter_id": 1,
+                   "os_version": "CentOS-6.5-x86_64",
+                   "name": "cluster_01",
+                   "os_config": {..},
+                   "package_config": {...},
+                   "deployed_os_config": {},
+                   "deployed_package_config": {},
+                   "owner": "xxx"
+               }
+           }
+        """
+        cluster_info = cluster_db.get_cluster(user, cluster_id)
+        cluster_config = cluster_db.get_cluster_config(user, cluster_id)
+        cluster_info.update(cluster_config)
+
+        deploy_config = cluster_db.get_cluster_deployed_config(user,
+                                                               cluster_id)
+        cluster_info.update(deploy_config)
+
+        return cluster_info
+
+    @staticmethod
+    def get_hosts_info(cluster_id, hosts_id_list, user):
+        """Get hosts information. Return a dictionary as below,
+           {
+               "hosts": {
+                   1($clusterhost_id/host_id): {
+                        "reinstall_os": True,
+                        "mac": "xxx",
+                        "name": "xxx",
+                        },
+                        "networks": {
+                            "eth0": {
+                                "ip": "192.168.1.1",
+                                "netmask": "255.255.255.0",
+                                "is_mgmt": True,
+                                "is_promiscuous": False,
+                                "subnet": "192.168.1.0/24"
+                            },
+                            "eth1": {...}
+                        },
+                        "os_config": {},
+                        "package_config": {},
+                        "deployed_os_config": {},
+                        "deployed_package_config": {}
+                   },
+                   2: {...},
+                   ....
+               }
+           }
+        """
+        hosts_info = {}
+        for clusterhost_id in hosts_id_list:
+            info = cluster_db.get_clusterhost(user, clusterhost_id)
+            host_id = info[const.HOST_ID]
+            temp = host_db.get_host(user, host_id)
+            config = cluster_db.get_cluster_host_config(user, cluster_id,
+                                                        host_id)
+            # Delete 'id' from temp
+            del temp['id']
+            info.update(temp)
+            info.update(config)
+
+            networks = info[const.NETWORKS]
+            networks_dict = {}
+            # Convert networks from list to dictionary format
+            for entry in networks:
+                nic_info = {}
+                nic_info = {
+                    entry[const.NIC]: {
+                        const.IP_ADDR: entry[const.IP_ADDR],
+                        const.NETMASK: entry[const.NETMASK],
+                        const.MGMT_NIC_FLAG: entry[const.MGMT_NIC_FLAG],
+                        const.PROMISCUOUS_FLAG: entry[const.PROMISCUOUS_FLAG],
+                        const.SUBNET: entry[const.SUBNET]
+                    }
+                }
+                networks_dict.update(nic_info)
+
+            info[const.NETWORKS] = networks_dict
+
+            hosts_info[clusterhost_id] = info
+
+        return hosts_info
+
+    @staticmethod
+    def save_deployed_config(self, deployed_config, user):
+        cluster_config = deployed_config[const.CLUSTER]
+        cluster_id = cluster_config[const.ID]
+        del cluster_config[const.ID]
+
+        cluster_db.update_cluster_deployed_config(user, cluster_id,
+                                                  **cluster_config)
+
+        hosts_id_list = deployed_config[const.HOSTS].keys()
+        for clusterhost_id in hosts_id_list:
+            config = deployed_config[const.HOSTS][clusterhost_id]
+            cluster_db.update_clusterhost_deployed_config(user,
+                                                          clusterhost_id,
+                                                          **config)

compass/deployment/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = "Grace Yu (grace.yu@huawei.com)"

compass/deployment/deploy_manager.py

@@ -0,0 +1,164 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = "Grace Yu (grace.yu@huawei.com)"
+
+"""Module to get configs from provider and isntallers and update
+   them to provider and installers.
+"""
+from compass.deployment.installers.installer import OSInstaller
+from compass.deployment.installers.installer import PKInstaller
+from compass.deployment.utils import constants as const
+from compass.utils import util
+
+
+import logging
+
+
+class DeployManager(object):
+    """Deploy manager module."""
+    def __init__(self, adapter_info, cluster_info, hosts_info):
+        os_installer_name = adapter_info[const.OS_INSTALLER][const.NAME]
+        pk_installer_name = adapter_info[const.PK_INSTALLER][const.NAME]
+
+        os_hosts_info = self._get_hosts_for_os_installation(hosts_info)
+
+        self.os_installer = DeployManager._get_installer(OSInstaller,
+                                                         os_installer_name,
+                                                         adapter_info,
+                                                         cluster_info,
+                                                         os_hosts_info)
+        self.pk_installer = DeployManager._get_installer(PKInstaller,
+                                                         pk_installer_name,
+                                                         adapter_info,
+                                                         cluster_info,
+                                                         hosts_info)
+
+    @staticmethod
+    def _get_installer(installer_type, name, adapter_info, cluster_info,
+                       hosts_info):
+        """Get installer instance."""
+        callback = getattr(installer_type, 'get_installer')
+        installer = callback(name, adapter_info, cluster_info, hosts_info)
+
+        return installer
+
+    def clean_progress(self):
+        """Clean previous installation log and progress."""
+        # Clean DB
+        # db_api.cluster.clean_progress(self.cluster_id)
+        # db_api.cluster.clean_progress(self.cluster_id, self.host_id_list)
+
+        # OS installer cleans previous installing progress.
+        if self.os_installer:
+            self.os_installer.clean_progress()
+
+        # Package installer cleans previous installing progress.
+        if self.pk_installer:
+            self.package_installer.clean_progress()
+
+    def prepare_for_deploy(self):
+        self.clean_progress()
+
+    def deploy(self):
+        """Deploy the cluster."""
+        deploy_config = {}
+        pk_instl_confs = {}
+        if self.pk_installer:
+            # generate target system config which will be installed by OS
+            # installer right after OS installation is completed.
+            pk_instl_confs = self.package_installer.generate_installer_config()
+            logging.debug('[DeployManager][deploy] pk_instl_confs is %s',
+                          pk_instl_confs)
+
+        if self.os_installer:
+            logging.info('[DeployManager][deploy]get OS installer %s',
+                         self.os_installer)
+            # Send package installer config info to OS installer.
+            if pk_instl_confs:
+                self.os_installer.set_package_installer_config(pk_instl_confs)
+
+            # start to deploy OS
+            try:
+                os_deploy_config = self.os_installer.deploy()
+                deploy_config = os_deploy_config
+            except Exception as ex:
+                logging.error(ex.message)
+
+        if self.pk_installer:
+            logging.info('DeployManager][deploy]get package installer %s',
+                         self.pk_installer)
+
+            pk_deploy_config = self.package_installer.deploy()
+            util.merge_dict(deploy_config, pk_deploy_config)
+
+        return deploy_config
+
+    def redeploy(self):
+        """Redeploy the cluster without changing configurations."""
+        if self.os_installer:
+            self.os_installer.redeploy()
+
+        if self.package_installer:
+            self.package_installer.redeploy()
+
+    def remove_hosts(self):
+        """Remove hosts from both OS and package installlers server side."""
+        if self.os_installer:
+            self.os_installer.delete_hosts()
+
+        if self.pk_installer:
+            self.pk_installer.delete_hosts()
+
+    def _get_hosts_for_os_installation(self, hosts_info):
+        """Get info of hosts which need to install/reinstall OS."""
+        hosts_list = {}
+        for host_id in hosts_info:
+            reinstall_os_flag = hosts_info[host_id][const.REINSTALL_OS_FLAG]
+            if not reinstall_os_flag:
+                continue
+
+            hosts_list[host_id] = hosts_info[host_id]
+
+        return hosts_list
+
+
+class PowerManager(object):
+    """Manage host to power on, power off, and reset."""
+
+    def __init__(self, adapter_info, cluster_info, hosts_info):
+        os_installer_name = adapter_info[const.OS_INSTALLER][const.NAME]
+        self.os_installer = DeployManager._get_installer(OSInstaller,
+                                                         os_installer_name,
+                                                         adapter_info,
+                                                         cluster_info,
+                                                         hosts_info)
+
+    def poweron(self):
+        if not self.os_installer:
+            logging.info("No OS installer found, cannot power on machine!")
+            return
+        self.os_installer.poweron()
+
+    def poweroff(self):
+        if not self.os_installer:
+            logging.info("No OS installer found, cannot power on machine!")
+            return
+        self.os_installer.poweroff()
+
+    def reset(self):
+        if not self.os_installer:
+            logging.info("No OS installer found, cannot power on machine!")
+            return
+        self.os_installer.reset()

compass/deployment/installers/__init__.py

@@ -0,0 +1,21 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = "Grace Yu (grace.yu@huawei.com)"
+
+
+"""modules to read/write cluster/host config from installers.
+
+   .. moduleauthor:: Grace Yu <grace.yu@huawei.com>
+"""

compass/deployment/installers/config_manager.py

@@ -0,0 +1,460 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = "Grace Yu (grace.yu@huawei.com)"
+
+
+"""Module to manage and access cluster, hosts and adapter config.
+"""
+from copy import deepcopy
+import logging
+
+
+from compass.deployment.utils import constants as const
+from compass.utils import util
+
+
+class BaseConfigManager(object):
+
+    def __init__(self, adapter_info, cluster_info, hosts_info):
+        self.adapter_info = adapter_info
+        self.cluster_info = cluster_info
+        self.hosts_info = hosts_info
+
+    def get_cluster_id(self):
+        if not self.cluster_info:
+            logging.info("cluster config is None or {}")
+            return None
+
+        return self.cluster_info[const.ID]
+
+    def get_clustername(self):
+        if not self.cluster_info:
+            logging.info("cluster config is None or {}")
+            return None
+        return self.cluster_info[const.NAME]
+
+    def get_os_version(self):
+        if not self.cluster_info:
+            logging.info("cluster config is None or {}")
+            return None
+        return self.cluster_info[const.OS_VERSION]
+
+    def get_cluster_baseinfo(self):
+        """Get cluster base information, including cluster_id, os_version,
+           and cluster_name.
+        """
+        if not self.cluster_info:
+            logging.info("cluster config is None or {}")
+            return None
+
+        attr_names = [const.ID, const.NAME, const.OS_VERSION]
+
+        base_info = {}
+        for name in attr_names:
+            base_info[name] = self.cluster_info[name]
+
+        return base_info
+
+    def get_host_id_list(self):
+        if not self.hosts_info:
+            logging.info("hosts config is None or {}")
+            return []
+
+        return self.hosts_info.keys()
+
+    def get_cluster_os_config(self):
+        if not self.cluster_info:
+            logging.info("cluster config is None or {}")
+            return {}
+
+        return deepcopy(self.cluster_info[const.OS_CONFIG])
+
+    def get_cluster_package_config(self):
+        if not self.cluster_info:
+            logging.info("cluster config is None or {}")
+            return {}
+
+        return deepcopy(self.cluster_info[const.PK_CONFIG])
+
+    def get_cluster_network_mapping(self):
+        package_config = self.get_cluster_package_config()
+        if not package_config:
+            logging.info("cluster package_config is None or {}.")
+            return {}
+
+        mapping = package_config.setdefault(const.NETWORK_MAPPING, {})
+        logging.info("Network mapping in the config is '%s'!", mapping)
+
+        return deepcopy(mapping)
+
+    def get_cluster_deployed_os_config(self):
+        if not self.cluster_info:
+            logging.info("cluster config is None or {}")
+            return {}
+
+        config = self.cluster_info.setdefault(const.DEPLOYED_OS_CONFIG, {})
+        return config
+
+    def get_cluster_deployed_package_config(self):
+        if not self.cluster_info:
+            logging.info("cluster config is None or {}")
+            return {}
+
+        config = self.cluster_info.setdefault(const.DEPLOYED_PK_CONFIG, {})
+        return config
+
+    def get_cluster_os_vars_dict(self):
+        deployed_config = self.get_cluster_deployed_os_config()
+        return self.__get_deployed_config_item(deployed_config,
+                                               const.TMPL_VARS_DICT)
+
+    def get_cluster_package_vars_dict(self):
+        deployed_config = self.get_cluster_deployed_package_config()
+        return self.__get_deployed_config_item(deployed_config,
+                                               const.TMPL_VARS_DICT)
+
+    def __get_deployed_config_item(deployed_config, item):
+        if not deployed_config:
+            logging.info("Deploy config is {}")
+            return {}
+
+        value = deployed_config.setdefault(item, {})
+        return deepcopy(value)
+
+    def get_cluster_roles_mapping(self):
+        if not self.cluster_info:
+            logging.info("cluster config is None or {}")
+            return {}
+
+        deploy_config = self.get_cluster_deployed_package_config()
+
+        if const.ROLES_MAPPING not in deploy_config:
+            mapping = self._get_cluster_roles_mapping_helper()
+            deploy_config[const.ROLES_MAPPING] = mapping
+        else:
+            mapping = deploy_config[const.ROLES_MAPPING]
+
+        return mapping
+
+    def _get_host_info(self, host_id):
+        if not self.hosts_info:
+            logging.info("hosts config is None or {}")
+            return {}
+
+        if host_id not in self.hosts_info:
+            logging.info("Cannot find host, ID is '%s'", host_id)
+            return {}
+
+        return self.hosts_info[host_id]
+
+    def get_host_baseinfo(self, host_id):
+        """Get host base information."""
+        host_info = self._get_host_info(host_id)
+        if not host_info:
+            return {}
+
+        attr_names = [const.REINSTALL_OS_FLAG, const.MAC_ADDR, const.NAME,
+                      const.HOSTNAME, const.NETWORKS]
+        base_info = {}
+        for attr in attr_names:
+            temp = host_info[attr]
+            if isinstance(temp, dict) or isinstance(temp, list):
+                base_info[attr] = deepcopy(temp)
+            else:
+                base_info[attr] = temp
+
+        base_info[const.DNS] = self.get_host_dns(host_id)
+
+        return base_info
+
+    def get_host_name(self, host_id):
+        host_info = self._get_host_info(host_id)
+        if not host_info:
+            return None
+
+        return host_info.setdefault(const.NAME, None)
+
+    def get_host_dns(self, host_id):
+        host_info = self._get_host_info(host_id)
+        if not host_info:
+            return None
+
+        if const.DNS not in host_info:
+            name = host_info[const.NAME]
+            domain = self.get_host_domain(host_id)
+            host_info[const.DNS] = '.'.join((name, domain))
+
+        return host_info[const.DNS]
+
+    def get_host_mac_address(self, host_id):
+        host_info = self._get_host_info(host_id)
+        if not host_info:
+            return None
+
+        return host_info.setdefault(const.MAC_ADDR, None)
+
+    def get_hostname(self, host_id):
+        host_info = self._get_host_info(host_id)
+        if not host_info:
+            return None
+
+        return host_info.setdefault(const.HOSTNAME, None)
+
+    def get_host_networks(self, host_id):
+        host_info = self._get_host_info(host_id)
+        if not host_info:
+            return {}
+
+        return deepcopy(host_info.setdefault(const.NETWORKS, {}))
+
+    def get_host_interfaces(self, host_id):
+        networks = self.get_host_networks(host_id)
+        if not networks:
+            return []
+
+        nic_names = networks.keys()
+        return nic_names
+
+    def get_host_interface_config(self, host_id, interface):
+        networks = self.get_host_networks(host_id)
+        if not networks:
+            return {}
+
+        return deepcopy(networks.setdefault(interface, {}))
+
+    def get_host_interface_ip(self, host_id, interface):
+        interface_config = self._get_host_interface_config(host_id, interface)
+        if not interface_config:
+            return None
+
+        return interface_config.setdefault(const.IP_ADDR, None)
+
+    def get_host_interface_netmask(self, host_id, interface):
+        interface_config = self.get_host_interface_config(host_id, interface)
+        if not interface_config:
+            return None
+
+        return interface_config.setdefault(const.NETMASK, None)
+
+    def get_host_interface_subnet(self, host_id, interface):
+        nic_config = self.get_host_interface_config(host_id, interface)
+        if not nic_config:
+            return None
+
+        return nic_config.setdefault(const.SUBNET, None)
+
+    def is_interface_promiscuous(self, host_id, interface):
+        nic_config = self.get_host_interface_config(host_id, interface)
+        if not nic_config:
+            raise Exception("Cannot find interface '%s'", interface)
+
+        return nic_config[const.PROMISCUOUS_FLAG]
+
+    def is_interface_mgmt(self, host_id, interface):
+        nic_config = self.get_host_interface_config(host_id, interface)
+        if not nic_config:
+            raise Exception("Cannot find interface '%s'", interface)
+
+        return nic_config[const.MGMT_NIC_FLAG]
+
+    def get_host_os_config(self, host_id):
+        host_info = self._get_host_info(host_id)
+        if not host_info:
+            return {}
+
+        return deepcopy(host_info.setdefault(const.OS_CONFIG, {}))
+
+    def get_host_domain(self, host_id):
+        os_config = self.get_host_os_config(host_id)
+        os_general_config = os_config.setdefault(const.OS_CONFIG_GENERAL, {})
+        domain = os_general_config.setdefault(const.DOMAIN, None)
+        if domain is None:
+            global_config = self.get_cluster_os_config()
+            global_general = global_config.setdefault(const.OS_CONFIG_GENERAL,
+                                                      {})
+            domain = global_general.setdefault(const.DOMAIN, None)
+
+        return domain
+
+    def get_host_network_mapping(self, host_id):
+        package_config = self.get_host_package_config(host_id)
+        if const.NETWORK_MAPPING not in package_config:
+            network_mapping = self.get_cluster_network_mapping()
+        else:
+            network_mapping = package_config[const.NETWORK_MAPPING]
+
+        return network_mapping
+
+    def get_host_package_config(self, host_id):
+        host_info = self._get_host_info(host_id)
+        if not host_info:
+            return {}
+
+        return deepcopy(host_info.setdefault(const.PK_CONFIG, {}))
+
+    def get_host_deployed_os_config(self, host_id):
+        host_info = self._get_host_info(host_id)
+        if not host_info:
+            return {}
+
+        return host_info.setdefault(const.DEPLOYED_OS_CONFIG, {})
+
+    def get_host_os_vars_dict(self, host_id):
+        deployed_config = self.get_host_deployed_os_config(host_id)
+        return self.__get_deployed_config_item(deployed_config,
+                                               const.TMPL_VARS_DICT)
+
+    def get_host_completed_os_config(self, host_id):
+        deployed_config = self.get_host_deployed_os_config(host_id)
+        config = self.__get_deployed_config_item(deployed_config,
+                                                 const.COMPLETED_OS_CONFIG)
+        if not config:
+            config = self.get_cluster_os_config()
+            util.merge_dict(config, self.get_host_os_config(host_id))
+            deployed_config[const.COMPLETED_OS_CONFIG] = config
+
+        return config
+
+    def get_host_deployed_package_config(self, host_id):
+        host_info = self._get_host_info(host_id)
+        if not host_info:
+            return {}
+
+        return host_info.setdefault(const.DEPLOYED_PK_CONFIG, {})
+
+    def get_host_package_vars_dict(self, host_id):
+        deployed_config = self.get_host_deployed_package_config(host_id)
+        return self.__get_deployed_config_item(deployed_config,
+                                               const.TMPL_VARS_DICT)
+
+    def get_host_roles(self, host_id):
+        config = self.get_host_package_config(host_id)
+        if not config:
+            return []
+
+        return deepcopy(config.setdefault(const.ROLES, []))
+
+    def get_host_roles_mapping(self, host_id):
+        roles_mapping = {}
+        deployed_pk_config = self.get_host_deployed_package_config(host_id)
+        if const.ROLES_MAPPING not in deployed_pk_config:
+            roles_mapping = self._get_host_roles_mapping_helper(host_id)
+        else:
+            roles_mapping = deployed_pk_config[const.ROLES_MAPPING]
+
+        return roles_mapping
+
+    def get_adapter_name(self):
+        if not self.adapter_info:
+            logging.info("Adapter Info is None!")
+            return None
+
+        return self.adapter_info.setdefault(const.NAME, None)
+
+    def get_dist_system_name(self):
+        if not self.adapter_info:
+            logging.info("Adapter Info is None!")
+            return None
+
+        return self.adapter_info.setdefault(const.DIST_SYS_NAME, None)
+
+    def get_adapter_roles(self):
+        if not self.adapter_info:
+            logging.info("Adapter Info is None!")
+            return []
+
+        roles = self.adapter_info.setdefault(const.ROLES, [])
+
+        return deepcopy(roles)
+
+    def get_os_installer_settings(self):
+        if not self.adapter_info:
+            logging.info("Adapter Info is None!")
+            return None
+
+        return self.adapter_info[const.OS_INSTALLER][const.INSTALLER_SETTINGS]
+
+    def get_pk_installer_settings(self):
+        if not self.adapter_info:
+            logging.info("Adapter Info is None!")
+            return None
+
+        return self.adapter_info[const.PK_INSTALLER][const.INSTALLER_SETTINGS]
+
+    def get_os_config_metadata(self):
+        if not self.adapter_info:
+            logging.info("Adapter Info is None!")
+            return None
+
+        return self.adapter_info[const.METADATA][const.OS_CONFIG]
+
+    def get_pk_config_meatadata(self):
+        if not self.adapter_info:
+            logging.info("Adapter Info is None!")
+            return None
+
+        return self.adapter_info[const.METADATA][const.PK_CONFIG]
+
+    def _get_cluster_roles_mapping_helper(self):
+        """The ouput format will be as below, for example:
+           {
+               "controller": {
+                   "management": {
+                       "interface": "eth0",
+                       "ip": "192.168.1.10",
+                       "netmask": "255.255.255.0",
+                       "subnet": "192.168.1.0/24",
+                       "is_mgmt": True,
+                       "is_promiscuous": False
+                   },
+                   ...
+               },
+                   ...
+           }
+        """
+        mapping = {}
+        hosts_id_list = self.get_host_id_list()
+        network_mapping = self.get_cluster_network_mapping()
+        if not network_mapping:
+            return {}
+
+        for host_id in hosts_id_list:
+            roles_mapping = self.get_host_roles_mapping(host_id)
+            for role in roles_mapping:
+                if role not in mapping:
+                    mapping[role] = roles_mapping[role]
+
+        return mapping
+
+    def _get_host_roles_mapping_helper(self, host_id):
+        """The format will be the same as cluster roles mapping."""
+        mapping = {}
+        network_mapping = self.get_host_network_mapping(host_id)
+        if not network_mapping:
+            return {}
+
+        roles = self.get_host_roles(host_id)
+        interfaces = self.get_host_interfaces(host_id)
+        temp = {}
+        for key in network_mapping:
+            nic = network_mapping[key][const.NIC]
+            if nic in interfaces:
+                temp[key] = self.get_host_interface_config(host_id, nic)
+                temp[key][const.NIC] = nic
+
+        for role in roles:
+            role = role.replace("-", "_")
+            mapping[role] = temp
+        return mapping

compass/deployment/installers/installer.py

@@ -0,0 +1,221 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = "Grace Yu (grace.yu@huawei.com)"
+
+
+"""Module to provider installer interface.
+"""
+from Cheetah.Template import Template
+from copy import deepcopy
+import imp
+import logging
+import os
+import simplejson as json
+
+
+CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+class BaseInstaller(object):
+    """Interface for installer."""
+    NAME = 'installer'
+
+    def __repr__(self):
+        return '%r[%r]' % (self.__class__.__name__, self.NAME)
+
+    def deploy(self, **kwargs):
+        """virtual method to start installing process."""
+        raise NotImplementedError
+
+    def clean_progress(self, **kwargs):
+        raise NotImplementedError
+
+    def delete_hosts(self, **kwargs):
+        """Delete hosts from installer server."""
+        raise NotImplementedError
+
+    def redeploy(self, **kwargs):
+        raise NotImplementedError
+
+    def get_tmpl_vars_from_metadata(self, metadata, config):
+        """Get variables dictionary for rendering templates from metadata.
+           :param dict metadata: The metadata dictionary.
+           :param dict config: The
+        """
+        template_vars = {}
+        self._get_tmpl_vars_helper(metadata, config, template_vars)
+
+        return template_vars
+
+    def _get_key_mapping(self, metadata, key, is_regular_key):
+        """Get the keyword which the input key maps to. This keyword will be
+           added to dictionary used to render templates.
+
+           If the key in metadata has a mapping to another keyword which is
+           used for templates, then return this keyword. If the key is started
+           with '$', which is a variable in metadata, return the key itself as
+           the mapping keyword. If the key has no mapping, return None.
+
+           :param dict metadata: metadata/submetadata dictionary.
+           :param str key: The keyword defined in metadata.
+           :param bool is_regular_key: False when the key defined in metadata
+                                       is a variable(starting with '$').
+        """
+        mapping_to = key
+        if is_regular_key:
+            try:
+                mapping_to = metadata['_self']['mapping_to']
+            except Exception:
+                mapping_to = None
+        return mapping_to
+
+    def _get_submeta_by_key(self, metadata, key):
+        """Get submetadata dictionary based on current metadata key. And
+           determines the input key is a regular string keyword or a variable
+           keyword defined in metadata, which starts with '$'.
+
+           :param dict metadata: The metadata dictionary.
+           :param str key: The keyword defined in the metadata.
+        """
+        if key in metadata:
+            return (True, metadata[key])
+
+        temp = deepcopy(metadata)
+        del temp['_self']
+        meta_key = temp.keys()[0]
+        if meta_key.startswith("$"):
+            return (False, metadata[meta_key])
+
+        raise KeyError("'%s' is an invalid metadata!" % key)
+
+    def _get_tmpl_vars_helper(self, metadata, config, output):
+        for key, config_value in config.iteritems():
+            is_regular_key, sub_meta = self._get_submeta_by_key(metadata, key)
+            mapping_to = self._get_key_mapping(sub_meta, key, is_regular_key)
+
+            if isinstance(config_value, dict):
+                if mapping_to:
+                    new_output = output[mapping_to] = {}
+                else:
+                    new_output = output
+
+                self._get_tmpl_vars_helper(sub_meta, config_value,
+                                           new_output)
+            elif mapping_to:
+                output[mapping_to] = config_value
+
+    def get_config_from_template(self, tmpl_dir, vars_dict):
+        if not os.path.exists(tmpl_dir) or not vars_dict:
+            logging.info("Template or variables dict is not specified!")
+            logging.debug("template dir is %s", tmpl_dir)
+            logging.debug("template vars dict is %s", vars_dict)
+            return {}
+
+        tmpl = Template(file=tmpl_dir, searchList=[vars_dict])
+        config = json.loads(tmpl.respond(), encoding='utf-8')
+        config = json.loads(json.dumps(config), encoding='utf-8')
+        return config
+
+    @classmethod
+    def get_installer(cls, name, path, adapter_info, cluster_info, hosts_info):
+        installer = None
+        try:
+            mod_file, path, descr = imp.find_module(name, [path])
+            if mod_file:
+                mod = imp.load_module(name, mod_file, path, descr)
+                return getattr(mod, mod.NAME)(adapter_info, cluster_info,
+                                              hosts_info)
+
+        except ImportError as exc:
+            logging.error('No such module found: %s', name)
+            logging.exception(exc)
+
+        return installer
+
+
+class OSInstaller(BaseInstaller):
+    """Interface for os installer."""
+    NAME = 'OSInstaller'
+    INSTALLER_BASE_DIR = os.path.join(CURRENT_DIR, 'os_installers')
+
+    def get_oses(self):
+        """virtual method to get supported oses.
+
+        :returns: list of str, each is the supported os version.
+        """
+        return []
+
+    @classmethod
+    def get_installer(cls, name, adapter_info, cluster_info, hosts_info):
+        path = os.path.join(cls.INSTALLER_BASE_DIR, name)
+        installer = super(OSInstaller, cls).get_installer(name, path,
+                                                          adapter_info,
+                                                          cluster_info,
+                                                          hosts_info)
+
+        if not isinstance(installer, OSInstaller):
+            logging.info("Installer '%s' is not an OS installer!" % name)
+            return None
+
+        return installer
+
+    def poweron(self, host_id):
+        pass
+
+    def poweroff(self, host_id):
+        pass
+
+    def reset(self, host_id):
+        pass
+
+
+class PKInstaller(BaseInstaller):
+    """Interface for package installer."""
+    NAME = 'PKInstaller'
+    INSTALLER_BASE_DIR = os.path.join(CURRENT_DIR, 'pk_installers')
+
+    def get_target_systems(self):
+        """virtual method to get available target_systems for each os.
+
+        :param oses: supported os versions.
+        :type oses: list of st
+
+        :returns: dict of os_version to target systems as list of str.
+        """
+        return {}
+
+    def get_roles(self, target_system):
+        """virtual method to get all roles of given target system.
+
+        :param target_system: target distributed system such as openstack.
+        :type target_system: str
+
+        :returns: dict of role to role description as str.
+        """
+        return {}
+
+    @classmethod
+    def get_installer(cls, name, adapter_info, cluster_info, hosts_info):
+        path = os.path.join(cls.INSTALLER_BASE_DIR, name)
+        installer = super(PKInstaller, cls).get_installer(name, path,
+                                                          adapter_info,
+                                                          cluster_info,
+                                                          hosts_info)
+
+        if not isinstance(installer, PKInstaller):
+            logging.info("Installer '%s' is not a package installer!" % name)
+            return None
+
+        return installer

compass/deployment/installers/os_installers/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

compass/deployment/installers/os_installers/cobbler/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

compass/deployment/installers/os_installers/cobbler/cobbler.py

@@ -0,0 +1,415 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""os installer cobbler plugin.
+"""
+import logging
+import os
+import shutil
+import xmlrpclib
+
+from compass.deployment.installers.config_manager import BaseConfigManager
+from compass.deployment.installers.installer import OSInstaller
+from compass.deployment.utils import constants as const
+from compass.utils import setting_wrapper as compass_setting
+from compass.utils import util
+
+
+NAME = 'CobblerInstaller'
+
+
+class CobblerInstaller(OSInstaller):
+    """cobbler installer"""
+    CREDENTIALS = "credentials"
+    USERNAME = 'username'
+    PASSWORD = 'password'
+
+    INSTALLER_URL = "cobbler_url"
+    TMPL_DIR = 'tmpl_dir'
+    SYS_TMPL = 'system.tmpl'
+    SYS_TMPL_NAME = 'system.tmpl'
+    PROFILE = 'profile'
+    POWER_TYPE = 'power_type'
+    POWER_ADDR = 'power_address'
+    POWER_USER = 'power_user'
+    POWER_PASS = 'power_pass'
+
+    def __init__(self, adapter_info, cluster_info, hosts_info):
+        super(CobblerInstaller, self).__init__()
+
+        self.config_manager = CobblerConfigManager(adapter_info,
+                                                   cluster_info,
+                                                   hosts_info)
+        installer_settings = self.config_manager.get_os_installer_settings()
+        try:
+            username = installer_settings[self.CREDENTIALS][self.USERNAME]
+            password = installer_settings[self.CREDENTIALS][self.PASSWORD]
+            cobbler_url = installer_settings[self.INSTALLER_URL]
+            self.tmpl_dir = CobblerInstaller.get_tmpl_path()
+
+        except KeyError as ex:
+            raise KeyError(ex.message)
+
+        # The connection is created when cobbler installer is initialized.
+        self.remote = self._get_cobbler_server(cobbler_url)
+        self.token = self._get_token(username, password)
+        self.pk_installer_config = None
+
+        logging.debug('%s instance created', 'CobblerInstaller')
+
+    @classmethod
+    def get_tmpl_path(cls):
+        return os.path.join(compass_setting.TMPL_DIR, 'cobbler')
+
+    def __repr__(self):
+        return '%r[name=%r,remote=%r,token=%r' % (
+            self.__class__.__name__, self.remote, self.token)
+
+    def _get_cobbler_server(self, cobbler_url):
+        if not cobbler_url:
+            logging.error("Cobbler URL is None!")
+            raise Exception("Cobbler URL cannot be None!")
+
+        return xmlrpclib.Server(cobbler_url)
+
+    def _get_token(self, username, password):
+        if self.remote is None:
+            raise Exception("Cobbler remote instance is None!")
+        return self.remote.login(username, password)
+
+    def get_supported_oses(self):
+        """get supported os versions.
+        .. note::
+           In cobbler, we treat profile name as the indicator
+           of os version. It is just a simple indicator
+           and not accurate.
+        """
+        profiles = self.remote.get_profiles()
+        oses = []
+        for profile in profiles:
+            oses.append(profile['name'])
+        return oses
+
+    def deploy(self):
+        """Sync cobbler to catch up the latest update config and start to
+           install OS. Return both cluster and hosts deploy configs. The return
+           format:
+           {
+               "cluster": {
+                   "id": 1,
+                   "deployed_os_config": {},
+               },
+               "hosts": {
+                    1($clusterhost_id): {
+                         "deployed_os_config": {...},
+                    },
+                    ....
+               }
+           }
+        """
+        os_version = self.config_manager.get_os_version()
+        profile = self._get_profile_from_server(os_version)
+
+        global_vars_dict = self._get_cluster_tmpl_vars_dict()
+
+        clusterhost_ids = self.config_manager.get_host_id_list()
+        hosts_deploy_config = {}
+
+        for host_id in clusterhost_ids:
+            fullname = self.config_manager.get_host_name(host_id)
+            vars_dict = self._get_host_tmpl_vars_dict(host_id,
+                                                      global_vars_dict,
+                                                      fullname=fullname,
+                                                      profile=profile)
+
+            self.update_host_config_to_cobbler(host_id, fullname, vars_dict)
+
+            # set host deploy config
+            temp = {}
+            temp = self.config_manager.get_host_deploy_os_config(host_id)
+            temp[const.TMPL_VARS_DICT] = vars_dict
+            host_config = {const.DEPLOYED_OS_CONFIG: temp}
+            hosts_deploy_config[host_id] = host_config
+
+        # set cluster deploy config
+        cluster_deconfig = self.config_manager.get_cluster_deployed_os_config()
+        cluster_deconfig[const.TMPL_VARS_DICT] = global_vars_dict
+
+        # sync to cobbler and trigger installtion.
+        self._sync()
+
+        return {
+            const.CLUSTER: {
+                const.ID: self.config_manager.get_cluster_id(),
+                const.DEPLOYED_OS_CONFIG: cluster_deconfig
+            },
+            const.HOSTS: hosts_deploy_config
+        }
+
+    def clean_progress(self):
+        """clean log files and config for hosts which to deploy."""
+        clusterhost_list = self.config_manager.get_host_id_list()
+        log_dir_prefix = compass_setting.INSTALLATION_LOGDIR[self.NAME]
+
+        for host_id in clusterhost_list:
+            fullname = self.config_manager.get_host_name(host_id)
+            self._clean_log(log_dir_prefix, fullname)
+
+    def redeploy(self):
+        """redeploy hosts."""
+        host_ids = self.config_manager.get_host_id_list()
+        for host_id in host_ids:
+            fullname = self.config_manager.get_host_name(host_id)
+            sys_id = self._get_system_id(fullname)
+            if sys_id:
+                self._netboot_enabled(sys_id)
+
+        self._sync()
+
+    def set_package_installer_config(self, package_configs):
+        """Cobbler can install and configure package installer right after
+           OS installation compelets by setting package_config info provided
+           by package installer.
+
+           :param dict package_configs: The dict of config generated by package
+                                        installer for each clusterhost. The IDs
+                                        of clusterhosts are the keys of
+                                        package_configs.
+        """
+        self.pk_installer_config = package_configs
+
+    def _sync(self):
+        """Sync the updated config to cobbler and trigger installation."""
+        try:
+            self.remote.sync(self.token)
+            logging.debug('sync %s', self)
+            os.system('sudo service rsyslog restart')
+        except Exception as ex:
+            logging.debug("Failed to sync cobbler server! Error:", ex.message)
+            raise ex
+
+    def _get_system_config(self, host_id, vars_dict):
+        """Generate updated system config from the template.
+
+           :param vars_dict: dict of variables for the system template to
+                             generate system config dict.
+        """
+        os_version = self.config_manager.get_os_version()
+
+        system_tmpl_path = os.path.join(os.path.join(self.tmpl_dir,
+                                                     os_version),
+                                        self.SYS_TMPL_NAME)
+        system_config = self.get_config_from_template(system_tmpl_path,
+                                                      vars_dict)
+        # update package config info to cobbler ksmeta
+        if self.pk_installer_config and host_id in self.pk_installer_config:
+            pk_config = self.pk_installer_config[host_id]
+            ksmeta = system_config.setdefault("ksmeta", {})
+            util.merge_dict(ksmeta, pk_config)
+            system_config["ksmeta"] = ksmeta
+
+        return system_config
+
+    def _get_profile_from_server(self, os_version):
+        """Get profile from cobbler server."""
+        result = self.remote.find_profile({'name': os_version})
+        if not result:
+            raise Exception("Cannot find profile for '%s'", os_version)
+
+        profile = result[0]
+        return profile
+
+    def _get_system_id(self, fullname):
+        """get system reference id for the host."""
+        sys_name = fullname
+        sys_id = None
+        system_info = self.remote.find_system({"name": fullname})
+
+        if not system_info:
+            # Create a new system
+            sys_id = self.remote.new_system(self.token)
+            self.remote.modify_system(sys_id, "name", fullname, self.token)
+            logging.debug('create new system %s for %s', sys_id, sys_name)
+        else:
+            sys_id = self.remote.get_system_handle(sys_name, self.token)
+
+        return sys_id
+
+    def _clean_system(self, fullname):
+        """clean system."""
+        sys_name = fullname
+        try:
+            self.remote.remove_system(sys_name, self.token)
+            logging.debug('system %s is removed', sys_name)
+        except Exception:
+            logging.debug('no system %s found to remove', sys_name)
+
+    def _update_system_config(self, sys_id, system_config):
+        """update modify system."""
+        for key, value in system_config.iteritems():
+            self.remote.modify_system(sys_id, str(key), value, self.token)
+
+        self.remote.save_system(sys_id, self.token)
+
+    def _netboot_enabled(self, sys_id):
+        """enable netboot."""
+        self.remote.modify_system(sys_id, 'netboot_enabled', True, self.token)
+        self.remote.save_system(sys_id, self.token)
+
+    def _clean_log(self, log_dir_prefix, system_name):
+        """clean log."""
+        log_dir = os.path.join(log_dir_prefix, system_name)
+        shutil.rmtree(log_dir, True)
+
+    def update_host_config_to_cobbler(self, host_id, fullname, vars_dict):
+        """update host config and upload to cobbler server."""
+        sys_id = self._get_system_id(fullname)
+
+        system_config = self._get_system_config(host_id, vars_dict)
+        logging.debug('%s system config to update: %s', host_id, system_config)
+
+        self._update_system_config(sys_id, system_config)
+        self._netboot_enabled(sys_id)
+
+    def delete_hosts(self):
+        hosts_id_list = self.config_manager.get_host_id_list()
+        for host_id in hosts_id_list:
+            self.delete_single_host(host_id)
+
+    def delete_single_host(self, host_id):
+        """Delete the host from cobbler server and clean up the installation
+           progress.
+        """
+        fullname = self.config_manager.get_host_name(host_id)
+        try:
+            log_dir_prefix = compass_setting.INSTALLATION_LOGDIR[self.NAME]
+            self._clean_system(fullname)
+            self._clean_log(log_dir_prefix, fullname)
+        except Exception as ex:
+            logging.info("Deleting host got exception: %s", ex.message)
+
+    def _get_host_tmpl_vars_dict(self, host_id, global_vars_dict, **kwargs):
+        """Generate template variables dictionary.
+        """
+        vars_dict = {}
+        if global_vars_dict:
+            # Set cluster template vars_dict from cluster os_config.
+            util.merge_dict(vars_dict, global_vars_dict)
+
+        if self.PROFILE in kwargs:
+            profile = kwargs[self.PROFILE]
+        else:
+            os_version = self.config_manager.get_os_version()
+            profile = self._get_profile_from_server(os_version)
+        vars_dict[self.PROFILE] = profile
+
+        # Set fullname, MAC address and hostname, networks, dns and so on.
+        host_baseinfo = self.config_manager.get_host_baseinfo(host_id)
+        util.merge_dict(vars_dict, host_baseinfo)
+
+        os_config_metadata = self.config_manager.get_os_config_metadata()
+        host_os_config = self.config_manager.get_host_os_config(host_id)
+
+        # Get template variables values from host os_config
+        host_vars_dict = self.get_tmpl_vars_from_metadata(os_config_metadata,
+                                                          host_os_config)
+        util.merge_dict(vars_dict, host_vars_dict)
+
+        return {const.HOST: vars_dict}
+
+    def _get_cluster_tmpl_vars_dict(self):
+        os_config_metadata = self.config_manager.get_os_config_metadata()
+        cluster_os_config = self.config_manager.get_cluster_os_config()
+
+        vars_dict = self.get_tmpl_vars_from_metadata(os_config_metadata,
+                                                     cluster_os_config)
+        return vars_dict
+
+    def _check_and_set_system_impi(self, host_id, sys_id):
+        if not sys_id:
+            logging.info("System is None!")
+            return False
+
+        fullname = self.config_manager.get_host_name(host_id)
+        system = self.remote.get_system_as_rendered(fullname)
+        if system[self.POWER_TYPE] != 'ipmilan' or not system[self.POWER_USER]:
+            ipmi_info = self.config_manager.get_host_ipmi_info(host_id)
+            if not ipmi_info:
+                logging.info('No IPMI information found! Failed power on.')
+                return False
+
+            ipmi_ip, ipmi_user, ipmi_pass = ipmi_info
+            power_opts = {}
+            power_opts[self.POWER_TYPE] = 'ipmilan'
+            power_opts[self.POWER_ADDR] = ipmi_ip
+            power_opts[self.POWER_USER] = ipmi_user
+            power_opts[self.POWER_PASS] = ipmi_pass
+
+            self._update_system_config(sys_id, power_opts)
+
+        return True
+
+    def poweron(self, host_id):
+        fullname = self.config_manager.get_host_name(host_id)
+        sys_id = self._get_system_id(fullname)
+        if not self._check_and_set_system_impi(sys_id):
+            return
+
+        self.remote.power_system(sys_id, self.token, power='on')
+        logging.info("Host with ID=%d starts to power on!" % host_id)
+
+    def poweroff(self, host_id):
+        fullname = self.config_manager.get_host_name(host_id)
+        sys_id = self._get_system_id(fullname)
+        if not self._check_and_set_system_impi(sys_id):
+            return
+
+        self.remote.power_system(sys_id, self.token, power='off')
+        logging.info("Host with ID=%d starts to power off!" % host_id)
+
+    def reset(self, host_id):
+        fullname = self.config_manager.get_host_name(host_id)
+        sys_id = self._get_system_id(fullname)
+        if not self._check_and_set_system_impi(sys_id):
+            return
+
+        self.remote.power_system(sys_id, self.token, power='reboot')
+        logging.info("Host with ID=%d starts to reboot!" % host_id)
+
+
+class CobblerConfigManager(BaseConfigManager):
+
+    def __init__(self, adapter_info, cluster_info, hosts_info):
+        super(CobblerConfigManager, self).__init__(adapter_info,
+                                                   cluster_info,
+                                                   hosts_info)
+
+    def get_host_ipmi_info(self, host_id):
+        host_info = self._get_host_info(host_id)
+        if not host_info:
+            return None
+
+        if self.IPMI not in host_info:
+            return None
+
+        ipmi_info = host_info[const.IPMI]
+
+        if not ipmi_info:
+            raise Exception('Cannot find IPMI information!')
+
+        ipmi_ip = ipmi_info[const.IP_ADDR]
+        ipmi_user = ipmi_info[const.IPMI_CREDS][const.USERNAME]
+        ipmi_pass = ipmi_info[const.IPMI_CREDS][const.PASSWORD]
+
+        return (ipmi_ip, ipmi_user, ipmi_pass)

compass/deployment/installers/pk_installers/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

compass/deployment/installers/pk_installers/chef_installer/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

compass/deployment/installers/pk_installers/chef_installer/chef_installer.py

@@ -0,0 +1,472 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = "Grace Yu (grace.yu@huawei.com)"
+
+"""package installer: chef plugin."""
+import logging
+import os
+import shutil
+
+from compass.deployment.installers.config_manager import BaseConfigManager
+from compass.deployment.installers.installer import PKInstaller
+from compass.deployment.utils import constants as const
+from compass.utils import setting_wrapper as compass_setting
+from compass.utils import util
+
+
+NAME = 'ChefInstaller'
+
+
+class ChefInstaller(PKInstaller):
+    """chef package installer."""
+    ENV_TMPL_NAME = 'env.tmpl'
+    NODE_TMPL_DIR = 'node'
+    DATABAG_TMPL_DIR = 'databags'
+    COMMON_NODE_TMPL_NAME = 'node.tmpl'
+
+    def __init__(self, adapter_info, cluster_info, hosts_info):
+        super(ChefInstaller, self).__init__()
+
+        self.config_manager = ChefConfigManager(adapter_info, cluster_info,
+                                                hosts_info)
+        adapter_name = self.config_manager.get_dist_system_name()
+        self.tmpl_dir = ChefInstaller.get_tmpl_path(adapter_name)
+
+        self.installer_url = self.config_manager.get_chef_url()
+        key, client = self.config_manager.get_chef_credentials()
+
+        self.chef_api = self._get_chef_api(key, client)
+        logging.debug('%s instance created', self)
+
+    @classmethod
+    def get_tmpl_path(cls, adapter_name):
+        tmpl_path = os.path.join(os.path.join(compass_setting.TMPL_DIR,
+                                              'chef_installer'),
+                                 adapter_name)
+        return tmpl_path
+
+    def __repr__(self):
+        return '%s[name=%s,installer_url=%s]' % (
+            self.__class__.__name__, self.NAME, self.installer_url)
+
+    def _get_chef_api(self, key=None, client=None):
+        """Initializes chef API client."""
+        import chef
+        chef_api = None
+        try:
+            if key and client:
+                chef_api = chef.ChefAPI(self.installer_url, key, client)
+            else:
+                chef_api = chef.autoconfigure()
+
+        except Exception as ex:
+            err_msg = "Failed to instantiate chef API, error: %s" % ex.message
+            raise Exception(err_msg)
+
+        return chef_api
+
+    def get_env_name(self, dist_sys_name, cluster_id):
+        """Generate environment name."""
+        return "-".join((dist_sys_name, cluster_id))
+
+    def get_databag_name(self):
+        """Get databag name."""
+        return self.config_manager.get_dist_system_name()
+
+    def get_databag(self, databag_name):
+        """Get databag object from chef server. Creating the databag if its
+           doesnot exist.
+        """
+        import chef
+        bag = chef.DataBag(databag_name, api=self.chef_api)
+        bag.save()
+        return bag
+
+    def get_node(self, node_name, env_name=None):
+        """Get chef node if existing, otherwise create one and set its
+           environment.
+
+           :param str node_name: The name for this node.
+           :param str env_name: The environment name for this node.
+        """
+        import chef
+        if not self.chef_api:
+            logging.info("Cannot find ChefAPI object!")
+            raise Exception("Cannot find ChefAPI object!")
+
+        node = chef.Node(node_name, api=self.chef_api)
+        if node not in chef.Node.list(api=self.chef_api):
+            if env_name:
+                node.chef_environment = env_name
+            node.save()
+
+        return node
+
+    def delete_hosts(self):
+        hosts_id_list = self.config_manager.get_host_id_list()
+        for host_id in hosts_id_list:
+            self.delete_node(host_id)
+
+    def delete_node(self, host_id):
+        fullname = self.config_manager.get_host_name(host_id)
+        node = self.get_node(fullname)
+        self._delete_node(node)
+
+    def _delete_node(self, node):
+        """clean node attributes about target system."""
+        import chef
+        if node is None:
+            raise Exception("Node is None, cannot delete a none node.")
+        node_name = node.name
+        client_name = node_name
+
+        # Clean log for this node first
+        log_dir_prefix = compass_setting.INSTALLATION_LOGDIR[self.NAME]
+        self._clean_log(log_dir_prefix, node_name)
+
+        # Delete node and its client on chef server
+        try:
+            node.delete()
+            client = chef.Client(client_name, api=self.chef_api)
+            client.delete()
+            logging.debug('delete node %s', node_name)
+            log_dir_prefix = compass_setting.INSTALLATION_LOGDIR[self.NAME]
+            self._clean_log(log_dir_prefix, node_name)
+        except Exception as error:
+            logging.debug(
+                'failed to delete node %s, error: %s', node_name, error)
+
+    def _add_roles(self, node, roles):
+        """Add roles to the node.
+           :param object node: The node object.
+           :param list roles: The list of roles for this node.
+        """
+        if node is None:
+            raise Exception("Node is None!")
+
+        if not roles:
+            logging.info("Role list is None. Run list will no change.")
+            return
+
+        run_list = node.run_list
+        for role in roles:
+            if role not in run_list:
+                node.run_list.append('role[%s]' % role)
+
+        node.save()
+        logging.debug('Runlist for node %s is %s', node.name, node.run_list)
+
+    def _get_node_attributes(self, roles, vars_dict):
+        """Get node attributes from templates according to its roles. The
+           templates are named by roles without '-'. Return the dictionary
+           of attributes defined in the templates.
+
+           :param list roles: The roles for this node, used to load the
+                              specific template.
+           :param dict vars_dict: The dictionary used in cheetah searchList to
+                                  render attributes from templates.
+        """
+        if not roles:
+            return {}
+
+        node_tmpl_dir = os.path.join(self.tmpl_dir, self.NODE_TMPL_DIR)
+        node_attri = {}
+        for role in roles:
+            role = role.replace('-', '_')
+            tmpl_name = '.'.join((role, 'tmpl'))
+            node_tmpl = os.path.join(node_tmpl_dir, tmpl_name)
+            node_attri = self.get_config_from_template(node_tmpl, vars_dict)
+
+        return node_attri
+
+    def update_node(self, node, roles, vars_dict):
+        """Update node attributes to chef server."""
+        if node is None:
+            raise Exception("Node is None!")
+
+        if not roles:
+            logging.info("The list of roles is None.")
+            return
+
+        # Add roles to node Rolelist on chef server.
+        self._add_roles(node, roles)
+
+        # Update node attributes.
+        node_config = self._get_node_attributes(roles, vars_dict)
+        for attr in node_config:
+            setattr(node, attr, node_config[attr])
+
+        node.save()
+
+    def _get_env_attributes(self, vars_dict):
+        """Get environment attributes from env templates."""
+
+        env_tmpl = os.path.join(self.tmpl_dir, self.ENV_TMPL_NAME)
+        env_attri = self.get_config_from_template(env_tmpl, vars_dict)
+        return env_attri
+
+    def update_environment(self, env_name, vars_dict):
+        """Generate environment attributes based on the template file and
+           upload it to chef server.
+
+           :param str env_name: The environment name.
+           :param dict vars_dict: The dictionary used in cheetah searchList to
+                                  render attributes from templates.
+        """
+        import chef
+        env_config = self._get_env_attributes(vars_dict)
+        env = chef.Environment(env_name, api=self.chef_api)
+        for attr in env_config:
+            if attr in env.attributes:
+                setattr(env, attr, env_config[attr])
+        env.save()
+
+    def _get_databagitem_attributes(self, tmpl_dir, vars_dict):
+        databagitem_attri = self.get_config_from_template(tmpl_dir,
+                                                          vars_dict)
+
+        return databagitem_attri
+
+    def update_databags(self, vars_dict):
+        """Update datbag item attributes.
+
+           :param dict vars_dict: The dictionary used to get attributes from
+                                  templates.
+        """
+        databag_names = self.config_manager.get_chef_databag_names()
+        if not databag_names:
+            return
+
+        import chef
+        databags_dir = os.path.join(self.tmpl_dir, self.DATABAG_TMPL_DIR)
+        for databag_name in databag_names:
+            databag = self.get_databag(databag_name)
+            databag_tmpl = os.paht.join(databags_dir, databag_name)
+            databagitem_attri = self._get_databagitem_attributes(databag_tmpl,
+                                                                 vars_dict)
+
+            for item, item_values in databagitem_attri.iteritems():
+                databagitem = chef.DataBagItem(databag, item,
+                                               api=self.chef_api)
+                for key, value in item_values.iteritems():
+                    databagitem[key] = value
+                databagitem.save()
+
+    def _get_host_tmpl_vars(self, host_id, global_vars_dict):
+        """Get templates variables dictionary for cheetah searchList based
+           on host package config.
+
+           :param int host_id: The host ID.
+           :param dict global_vars_dict: The vars_dict got from cluster level
+                                         package_config.
+        """
+        vars_dict = {}
+        if global_vars_dict:
+            temp = global_vars_dict[const.CLUSTER][const.DEPLOYED_PK_CONFIG]
+            vars_dict[const.DEPLOYED_PK_CONFIG] = temp
+
+        host_baseinfo = self.config_manager.get_host_baseinfo(host_id)
+        util.merge_dict(vars_dict, host_baseinfo)
+
+        pk_config = self.config_manager.get_host_package_config(host_id)
+        if pk_config:
+            # Get host template variables and merge to vars_dict
+            metadata = self.config_manager.get_pk_config_meatadata()
+            host_dict = self.get_tmpl_vars_from_metadata(metadata, pk_config)
+            util.merge_dict(vars_dict[const.DEPLOYED_PK_CONFIG], host_dict)
+
+        # Set role_mapping for host
+        mapping = self.config_manager.get_host_roles_mapping(host_id)
+        vars_dict[const.DEPLOYED_PK_CONFIG][const.ROLES_MAPPING] = mapping
+
+        return {const.HOST: vars_dict}
+
+    def _get_cluster_tmpl_vars(self):
+        vars_dict = {}
+        cluster_baseinfo = self.config_manager.get_cluster_baseinfo()
+        util.merge_dict(vars_dict, cluster_baseinfo)
+
+        pk_metadata = self.config_manager.get_pk_config_meatadata()
+        pk_config = self.config_manager.get_cluster_package_config()
+        meta_dict = self.get_tmpl_vars_from_metadata(pk_metadata, pk_config)
+        vars_dict[const.DEPLOYED_PK_CONFIG] = meta_dict
+
+        mapping = self.config_manager.get_cluster_roles_mapping()
+        vars_dict[const.DEPLOYED_PK_CONFIG][const.ROLES_MAPPING] = mapping
+
+        return {const.CLUSTER: vars_dict}
+
+    def deploy(self):
+        """Start to deploy a distributed system. Return both cluster and hosts
+           deployed configs. The return format:
+           {
+               "cluster": {
+                   "id": 1,
+                   "deployed_package_config": {...}
+               },
+               "hosts": {
+                   1($clusterhost_id): {
+                       "deployed_package_config": {...}
+                   },
+                   ....
+               }
+           }
+        """
+        adapter_name = self.config_manager.get_adapter_name()
+        cluster_id = self.config_manager.get_cluster_id()
+        env_name = self.get_env_name(adapter_name, str(cluster_id))
+
+        global_vars_dict = self._get_cluster_tmpl_vars()
+        # Update environment
+        self.update_environment(env_name, global_vars_dict)
+
+        # Update Databag item
+        self.update_databags(global_vars_dict)
+
+        host_list = self.config_manager.get_host_id_list()
+        hosts_deployed_configs = {}
+
+        for host_id in host_list:
+            node_name = self.config_manager.get_host_name(host_id)
+            roles = self.config_manager.get_host_roles(host_id)
+
+            node = self.get_node(node_name, env_name)
+            vars_dict = self._get_host_tmpl_vars(host_id, global_vars_dict)
+            self.update_node(node, roles, vars_dict)
+
+            # set each host deployed config
+            tmp = self.config_manager.get_host_deployed_package_config(host_id)
+            tmp[const.TMPL_VARS_DICT] = vars_dict
+            hosts_deployed_configs[host_id] = tmp
+
+        # set cluster deployed config
+        cl_config = self.config_manager.get_cluster_deployed_package_config()
+        cl_config[const.TMPL_VARS_DICT] = global_vars_dict
+
+        return {
+            const.CLUSTER: {
+                const.ID: cluster_id,
+                const.DEPLOYED_PK_CONFIG: cl_config
+            },
+            const.HOSTS: hosts_deployed_configs
+        }
+
+    def generate_installer_config(self):
+        """Render chef config file (client.rb) by OS installing right after
+           OS is installed successfully.
+           The output format:
+           {
+              '1'($host_id/clusterhost_id):{
+                  'tool': 'chef',
+                  'chef_url': 'https://xxx',
+                  'chef_client_name': '$host_name',
+                  'chef_node_name': '$host_name'
+              },
+              .....
+           }
+        """
+        host_ids = self.config_manager.get_host_id_list()
+        os_installer_configs = {}
+        for host_id in host_ids:
+            fullname = self.config_manager.get_host_name(host_id)
+            temp = {
+                "tool": "chef",
+                "chef_url": self.installer_url
+            }
+            temp['chef_client_name'] = fullname
+            temp['chef_node_name'] = fullname
+            os_installer_configs[host_id] = temp
+
+        return os_installer_configs
+
+    def clean_progress(self):
+        """Clean all installing log about the node."""
+        log_dir_prefix = compass_setting.INSTALLATION_LOGDIR[self.NAME]
+        hosts_list = self.config_manager.get_host_id_list()
+        for host_id in hosts_list:
+            fullname = self.config_manager.get_host_name()
+            self._clean_log(log_dir_prefix, fullname)
+
+    def _clean_log(self, log_dir_prefix, node_name):
+        log_dir = os.path.join(log_dir_prefix, node_name)
+        shutil.rmtree(log_dir, True)
+
+    def get_supported_dist_systems(self):
+        """get target systems from chef. All target_systems for compass will
+           be stored in the databag called "compass".
+        """
+        databag = self.__get_compass_databag()
+        target_systems = {}
+        for system_name, item in databag:
+            target_systems[system_name] = item
+
+        return target_systems
+
+    def _clean_databag_item(self, databag, item_name):
+        """clean databag item."""
+        import chef
+        if item_name not in chef.DataBagItem.list(api=self.chef_api):
+            logging.info("Databag item '%s' is not found!", item_name)
+            return
+
+        bag_item = databag[item_name]
+        try:
+            bag_item.delete()
+            logging.debug('databag item %s is removed from databag',
+                          item_name)
+        except Exception as error:
+            logging.debug('Failed to delete item  %s from databag! Error: %s',
+                          item_name, error)
+        databag.save()
+
+    def redeploy(self):
+        """reinstall host."""
+        pass
+
+
+class ChefConfigManager(BaseConfigManager):
+    TMPL_DIR = 'tmpl_dir'
+    DATABAGS = "databags"
+    CHEFSERVER_URL = "chef_url"
+    KEY_DIR = "key_dir"
+    CLIENT = "client_name"
+
+    def __init__(self, adapter_info, cluster_info, hosts_info):
+        super(ChefConfigManager, self).__init__(adapter_info,
+                                                cluster_info,
+                                                hosts_info)
+
+    def get_chef_url(self):
+        pk_installer_settings = self.get_pk_installer_settings()
+        if self.CHEFSERVER_URL not in pk_installer_settings:
+            raise KeyError("'%s' must be set in package settings!",
+                           self.CHEFSERVER_URL)
+
+        return pk_installer_settings[self.CHEFSERVER_URL]
+
+    def get_chef_credentials(self):
+        installer_settings = self.get_pk_installer_settings()
+        key_dir = installer_settings.setdefault(self.KEY_DIR, None)
+        client = installer_settings.setdefault(self.CLIENT, None)
+
+        return (key_dir, client)
+
+    def get_chef_databag_names(self):
+        pk_installer_settings = self.get_pk_installer_settings()
+        if self.DATABAGS in pk_installer_settings:
+            logging.info("No databags is set!")
+            return None
+
+        return pk_installer_settings[self.DATABAGS]

compass/deployment/utils/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = "Grace Yu (grace.yu@huawei.com)"

compass/deployment/utils/constants.py

@@ -0,0 +1,75 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = "Grace Yu (grace.yu@huawei.com)"
+
+
+"""All keywords variables in deployment are defined in this module."""
+
+
+# General keywords
+CLUSTER = 'cluster'
+HOST = 'host'
+HOSTS = 'hosts'
+ID = 'id'
+NAME = 'name'
+PASSWORD = 'password'
+USERNAME = 'username'
+
+
+# Adapter info related keywords
+DIST_SYS_NAME = 'distributed_system_name'
+INSTALLER_SETTINGS = 'settings'
+METADATA = 'metadata'
+OS_INSTALLER = 'os_installer'
+PK_INSTALLER = 'pk_installer'
+SUPPORT_OSES = 'supported_oses'
+
+
+# Cluster info related keywords
+ADAPTER_ID = 'adapter_id'
+OS_VERSION = 'os_name'
+
+
+# Host info related keywords
+DNS = 'dns'
+DOMAIN = 'domain'
+HOST_ID = 'host_id'
+HOSTNAME = 'hostname'
+IP_ADDR = 'ip'
+IPMI = 'ipmi'
+IPMI_CREDS = 'ipmi_credentials'
+MAC_ADDR = 'mac'
+MGMT_NIC_FLAG = 'is_mgmt'
+NETMASK = 'netmask'
+NETWORKS = 'networks'
+NIC = 'interface'
+ORIGIN_CLUSTER_ID = 'origin_cluster_id'
+PROMISCUOUS_FLAG = 'is_promiscuous'
+REINSTALL_OS_FLAG = 'reinstall_os'
+SUBNET = 'subnet'
+
+
+# Cluster/host config related keywords
+COMPLETED_PK_CONFIG = 'completed_package_config'
+COMPLETED_OS_CONFIG = 'completed_os_config'
+DEPLOYED_OS_CONFIG = 'deployed_os_config'
+DEPLOYED_PK_CONFIG = 'deployed_package_config'
+NETWORK_MAPPING = 'network_mapping'
+OS_CONFIG = 'os_config'
+OS_CONFIG_GENERAL = 'general'
+PK_CONFIG = 'package_config'
+ROLES = 'roles'
+ROLES_MAPPING = 'roles_mapping'
+TMPL_VARS_DICT = 'vars_dict'

compass/tasks/tasks.py

@@ -19,11 +19,9 @@
 import logging
 
 from celery.signals import celeryd_init
-from celery.signals import setup_logging
 
 from compass.actions import deploy
 from compass.actions import poll_switch
-from compass.actions import reinstall
 from compass.actions import update_progress
 from compass.db.api import adapter_holder as adapter_api
 from compass.db.api import database
@@ -78,7 +76,10 @@ def deploy_cluster(deployer_email, cluster_id, clusterhost_ids):
     :param cluster_hosts: the cluster and hosts of each cluster to deploy.
     :type cluster_hosts: dict of int to list of int
     """
-    pass
+    try:
+        deploy.deploy(cluster_id, clusterhost_ids, deployer_email)
+    except Exception as error:
+        logging.exception(error)
 
 
 @celery.task(name='compass.tasks.reinstall_cluster')

compass/tests/actions/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

compass/tests/actions/deploy/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

compass/tests/actions/deploy/test_deploy.py

@@ -0,0 +1,151 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""Test deploy action module."""
+
+
+from mock import patch
+import os
+import unittest2
+
+
+os.environ['COMPASS_IGNORE_SETTING'] = 'true'
+
+
+from compass.actions import deploy
+from compass.utils import setting_wrapper as setting
+reload(setting)
+
+
+class TestDeployAction(unittest2.TestCase):
+    """Test deploy moudle functions in actions."""
+    def setUp(self):
+        super(TestDeployAction, self).setUp()
+
+    def tearDown(self):
+        super(TestDeployAction, self).tearDown()
+
+    @patch('compass.db.api.cluster.get_cluster_metadata')
+    @patch('compass.db.api.adapter_holder.get_adapter')
+    def test_get_adatper_info(self, mock_get_adapter, mock_get_cluster_meta):
+        mock_get_adapter.return_value = {
+            "id": 1,
+            "name": "test_adapter",
+            "roles": [
+                {
+                    "name": "test-role-1",
+                    "display_name": "test role 1"
+                },
+                {
+                    "name": "test-role-2",
+                    "display_name": "test role 2"
+                }
+            ],
+            "os_installer": {
+                "name": "test_os_installer",
+                "settings": {
+                    "url": "http://127.0.0.1"
+                }
+            },
+            "pk_installer": {
+                "name": "test_pk_installer",
+                "settings": {
+                    "url": "http://127.0.0.1"
+                }
+            }
+        }
+        mock_get_cluster_meta.return_value = {
+            "os_config": {},
+            "package_config": {}
+        }
+        expected_output = {
+            "id": 1,
+            "name": "test_adapter",
+            "roles": ["test-role-1", "test-role-2"],
+            "os_installer": {
+                "name": "test_os_installer",
+                "settings": {
+                    "url": "http://127.0.0.1"
+                }
+            },
+            "pk_installer": {
+                "name": "test_pk_installer",
+                "settings": {
+                    "url": "http://127.0.0.1"
+                }
+            },
+            "os_config": {},
+            "package_config": {}
+        }
+        output = deploy.ActionHelper.get_adapter_info(1, 1, None)
+        self.maxDiff = None
+        self.assertDictEqual(expected_output, output)
+
+    @patch('compass.db.api.cluster.get_cluster_host_config')
+    @patch('compass.db.api.host.get_host')
+    @patch('compass.db.api.cluster.get_clusterhost')
+    def test_get_hosts_info(self, mock_get_clusterhost, mock_get_host,
+                            mock_get_cluster_host_config):
+        mock_get_clusterhost.return_value = {
+            "id": 1,
+            "host_id": 10,
+            "name": "test"
+        }
+        mock_get_host.return_value = {
+            "id": 10,
+            "mac": "00:89:23:a1:e9:10",
+            "hostname": "server01",
+            "networks": [
+                {
+                    "interface": "eth0",
+                    "ip": "127.0.0.1",
+                    "netmask": "255.255.255.0",
+                    "is_mgmt": True,
+                    "subnet": "127.0.0.0/24",
+                    "is_promiscuous": False
+                }
+            ]
+        }
+        mock_get_cluster_host_config.return_value = {
+            "os_config": {},
+            "package_config": {},
+            "deployed_os_config": {},
+            "deployed_package_config": {}
+        }
+        expected_output = {
+            1: {
+                "id": 1,
+                "host_id": 10,
+                "name": "test",
+                "mac": "00:89:23:a1:e9:10",
+                "hostname": "server01",
+                "networks": {
+                    "eth0": {
+                        "ip": "127.0.0.1",
+                        "netmask": "255.255.255.0",
+                        "is_mgmt": True,
+                        "subnet": "127.0.0.0/24",
+                        "is_promiscuous": False
+                    }
+                },
+                "os_config": {},
+                "package_config": {},
+                "deployed_os_config": {},
+                "deployed_package_config": {}
+            }
+        }
+        output = deploy.ActionHelper.get_hosts_info(1, [1], None)
+        self.maxDiff = None
+        self.assertDictEqual(expected_output, output)

compass/tests/deployment/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

compass/tests/deployment/installers/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

compass/tests/deployment/installers/os_installers/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

compass/tests/deployment/installers/os_installers/cobbler/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

compass/tests/deployment/installers/os_installers/cobbler/test_cobbler.py

@@ -0,0 +1,173 @@
+#!/usr/bin/python
+#
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = "Grace Yu (grace.yu@huawei.com)"
+
+
+"""Test cobbler installer module."""
+
+from copy import deepcopy
+from mock import Mock
+import os
+import unittest2
+
+
+os.environ['COMPASS_IGNORE_SETTING'] = 'true'
+
+
+from compass.deployment.installers.os_installers.cobbler.cobbler \
+    import CobblerInstaller
+from compass.tests.deployment.test_data import config_data
+from compass.utils import setting_wrapper as compass_setting
+reload(compass_setting)
+
+
+class TestCobblerInstaller(unittest2.TestCase):
+    """Test CobblerInstaller methods."""
+    def setUp(self):
+        super(TestCobblerInstaller, self).setUp()
+        self.test_cobbler = self._get_cobbler_installer()
+        self.expected_host_vars_dict = {
+            "host": {
+                "mac": "00:0c:29:3e:60:e9",
+                "name": "server01.test",
+                "profile": "Ubuntu-12.04-x86_64",
+                "hostname": "server01",
+                "dns": "server01.test.ods.com",
+                "reinstall_os": True,
+                "networks": {
+                    "vnet0": {
+                        "ip": "12.234.32.100",
+                        "netmask": "255.255.255.0",
+                        "is_mgmt": True,
+                        "is_promiscuous": False,
+                        "subnet": "12.234.32.0/24"
+                    },
+                    "vnet1": {
+                        "ip": "172.16.1.1",
+                        "netmask": "255.255.255.0",
+                        "is_mgmt": False,
+                        "is_promiscuous": False,
+                        "subnet": "172.16.1.0/24"
+                    }
+                },
+                "partition": {
+                    "/var": {
+                        "vol_size": 30,
+                        "vol_percentage": 30
+                    },
+                    "/home": {
+                        "vol_size": 50,
+                        "vol_percentage": 40
+                    },
+                    "/test": {
+                        "vol_size": 10,
+                        "vol_percentage": 10
+                    }
+                },
+                "server_credentials": {
+                    "username": "root",
+                    "password": "huawei"
+                },
+                "language": "EN",
+                "timezone": "UTC",
+                "http_proxy": "http://127.0.0.1:3128",
+                "https_proxy": "",
+                "ntp_server": "127.0.0.1",
+                "nameservers": ["127.0.0.1"],
+                "search_path": ["1.ods.com", "ods.com"],
+                "gateway": "10.145.88.1"
+            }
+        }
+
+    def tearDown(self):
+        super(TestCobblerInstaller, self).tearDown()
+        del self.test_cobbler
+
+    def _get_cobbler_installer(self):
+        adapter_info = deepcopy(config_data.adapter_test_config)
+        cluster_info = deepcopy(config_data.cluster_test_config)
+        hosts_info = deepcopy(config_data.hosts_test_config)
+        # In config_data, only hosts with ID 1 and 2 needs to install OS.
+        del hosts_info[3]
+
+        CobblerInstaller._get_cobbler_server = Mock()
+        CobblerInstaller._get_cobbler_server.return_value = "mock_server"
+        CobblerInstaller._get_token = Mock()
+        CobblerInstaller._get_token.return_value = "mock_token"
+
+        CobblerInstaller.get_tmpl_path = Mock()
+        test_tmpl_dir = os.path.join(config_data.test_tmpl_dir, 'cobbler')
+        CobblerInstaller.get_tmpl_path.return_value = test_tmpl_dir
+        return CobblerInstaller(adapter_info, cluster_info, hosts_info)
+
+    def test_get_host_tmpl_vars_dict(self):
+        host_id = 1
+        profile = 'Ubuntu-12.04-x86_64'
+        global_vars_dict = self.test_cobbler._get_cluster_tmpl_vars_dict()
+        output = self.test_cobbler._get_host_tmpl_vars_dict(host_id,
+                                                            global_vars_dict,
+                                                            profile=profile)
+        self.maxDiff = None
+        self.assertDictEqual(self.expected_host_vars_dict, output)
+
+    def test_get_system_config(self):
+        expected_system_config = {
+            "name": "server01.test",
+            "hostname": "server01",
+            "profile": "Ubuntu-12.04-x86_64",
+            "gateway": "10.145.88.1",
+            "name_servers": ["127.0.0.1"],
+            "name_servers_search": "1.ods.com ods.com",
+            "proxy": "http://127.0.0.1:3128",
+            "modify_interface": {
+                "ipaddress-vnet0": "12.234.32.100",
+                "netmask-vnet0": "255.255.255.0",
+                "management-vnet0": True,
+                "macaddress-vnet0": "00:0c:29:3e:60:e9",
+                "dns-vnet0": "server01.test.ods.com",
+                "static-vnet0": True,
+                "ipaddress-vnet1": "172.16.1.1",
+                "netmask-vnet1": "255.255.255.0",
+                "management-vnet1": False,
+                "static-vnet1": True
+            },
+            "ksmeta": {
+                "promisc_nics": "",
+                "timezone": "UTC",
+                "partition": "/test 10%;/var 30%;/home 40%",
+                "https_proxy": "",
+                "ntp_server": "127.0.0.1",
+                "chef_url": "https://127.0.0.1",
+                "chef_client_name": "server01.test",
+                "chef_node_name": "server01.test",
+                "tool": "chef"
+            }
+        }
+        package_config = {
+            1: {
+                "chef_url": "https://127.0.0.1",
+                "chef_client_name": "server01.test",
+                "chef_node_name": "server01.test",
+                "tool": "chef"
+            }
+        }
+        host_id = 1
+        self.test_cobbler.set_package_installer_config(package_config)
+        output = self.test_cobbler._get_system_config(
+            host_id, self.expected_host_vars_dict)
+        self.maxDiff = None
+        self.assertEqual(expected_system_config, output)

compass/tests/deployment/installers/pk_installers/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

compass/tests/deployment/installers/pk_installers/chef_installer/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

compass/tests/deployment/installers/pk_installers/chef_installer/test_chef.py

@@ -0,0 +1,131 @@
+#!/usr/bin/python
+#
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = "Grace Yu (grace.yu@huawei.com)"
+
+
+"""Test Chef installer module.
+"""
+
+from copy import deepcopy
+from mock import Mock
+import os
+import unittest2
+
+
+os.environ['COMPASS_IGNORE_SETTING'] = 'true'
+
+from compass.tests.deployment.test_data import config_data
+from compass.utils import setting_wrapper as compass_setting
+reload(compass_setting)
+
+
+from compass.deployment.installers.pk_installers.chef_installer.chef_installer\
+    import ChefInstaller
+
+
+class TestChefInstaller(unittest2.TestCase):
+    """Test installer functionality."""
+    def setUp(self):
+        super(TestChefInstaller, self).setUp()
+        self.test_chef = self._get_chef_installer()
+
+    def tearDown(self):
+        super(TestChefInstaller, self).tearDown()
+
+    def _get_chef_installer(self):
+        adapter_info = deepcopy(config_data.adapter_test_config)
+        cluster_info = deepcopy(config_data.cluster_test_config)
+        hosts_info = deepcopy(config_data.hosts_test_config)
+
+        ChefInstaller.get_tmpl_path = Mock()
+        test_tmpl_dir = os.path.join(os.path.join(config_data.test_tmpl_dir,
+                                                  'chef_installer'),
+                                     'openstack_icehouse')
+        ChefInstaller.get_tmpl_path.return_value = test_tmpl_dir
+
+        ChefInstaller._get_chef_api = Mock()
+        ChefInstaller._get_chef_api.return_value = 'mock_server'
+        chef_installer = ChefInstaller(adapter_info, cluster_info, hosts_info)
+        return chef_installer
+
+    def test_get_tmpl_vars(self):
+        pass
+
+    def test_get_node_attributes(self):
+        cluster_dict = self.test_chef._get_cluster_tmpl_vars()
+        vars_dict = self.test_chef._get_host_tmpl_vars(2, cluster_dict)
+        expected_node_attr = {
+            "override_attributes": {
+                "endpoints": {
+                    "compute-vnc-bind": {
+                        "host": "12.234.32.101"
+                    }
+                }
+            }
+        }
+        output = self.test_chef._get_node_attributes(['os-compute'], vars_dict)
+        self.maxDiff = None
+        self.assertDictEqual(expected_node_attr, output)
+
+    def test_get_env_attributes(self):
+        expected_env = {
+            "name": "testing",
+            "description": "Environment",
+            "cookbook_versions": {
+            },
+            "json_class": "Chef::Environment",
+            "chef_type": "environment",
+            "default_attributes": {
+            },
+            "override_attributes": {
+                "compute": {
+                    "syslog": {
+                        "use": False
+                    },
+                    "libvirt": {
+                        "bind_interface": "eth0"
+                    },
+                    "novnc_proxy": {
+                        "bind_interface": "vnet0"
+                    },
+                    "xvpvnc_proxy": {
+                        "bind_interface": "eth0"
+                    }
+                },
+                "db": {
+                    "bind_interface": "vnet0",
+                    "compute": {
+                        "host": "12.234.32.100"
+                    },
+                    "identity": {
+                        "host": "12.234.32.100"
+                    }
+                },
+                "mq": {
+                    "user": "guest",
+                    "password": "test",
+                    "vhost": "/nova",
+                    "network": {
+                        "service_type": "rabbitmq"
+                    }
+                }
+            }
+        }
+        vars_dict = self.test_chef._get_cluster_tmpl_vars()
+        output = self.test_chef._get_env_attributes(vars_dict)
+        self.maxDiff = None
+        self.assertDictEqual(expected_env, output)

compass/tests/deployment/installers/test_config_manager.py

@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

compass/tests/deployment/installers/test_installer.py

@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

compass/tests/deployment/test_data/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

compass/tests/deployment/test_data/config_data.py

@@ -0,0 +1,366 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = "Grace Yu (grace.yu@huawei.com)"
+
+import os
+
+
+curr_dir = os.path.dirname(os.path.realpath(__file__))
+test_tmpl_dir = os.path.join(curr_dir, 'templates')
+
+
+adapter_test_config = {
+    "name": "openstack_icehouse",
+    "distributed_system_name": "openstack_icehouse",
+    "roles": ["os-controller", "os-compute-worker", "os-network"],
+    "os_installer": {
+        "name": "cobbler",
+        "settings": {
+            "cobbler_url": "http://127.0.0.1/cobbler_api",
+            "credentials": {
+                "username": "cobbler",
+                "password": "cobbler"
+            }
+        }
+    },
+    "pk_installer": {
+        "name": "chef_installer",
+        "settings": {
+            "chef_url": "https://127.0.0.1",
+            "key_dir": "xxx",
+            "client_name": "xxx"
+        }
+    },
+    "metadata": {
+        "os_config": {
+            "_self": {},
+            "general": {
+                "_self": {"mapping_to": ""},
+                "language": {
+                    "_self": {
+                        "mapping_to": "language",
+                    },
+                },
+                "timezone": {
+                    "_self": {
+                        "mapping_to": "timezone"
+                    }
+                },
+                "default_gateway": {
+                    "_self": {
+                        "mapping_to": "gateway"
+                    }
+                },
+                "domain": {
+                    "_self": {"mapping_to": ""}
+                },
+                "http_proxy": {
+                    "_self": {
+                        "mapping_to": "http_proxy"
+                    }
+                },
+                "ntp_server": {
+                    "_self": {"mapping_to": "ntp_server"}
+                },
+                "dns_servers": {
+                    "_self": {"mapping_to": "nameservers"}
+                },
+                "search_path": {
+                    "_self": {"mapping_to": "search_path"}
+                },
+                "https_proxy": {
+                    "_self": {"mapping_to": "https_proxy"}
+                }
+            },
+            "partition": {
+                "_self": {
+                    "mapping_to": "partition"
+                },
+                "$path": {
+                    "_self": {"mapping_to": ""},
+                    "max_size": {
+                        "_self": {"mapping_to": "vol_size"}
+                    },
+                    "size_percentage": {
+                        "_self": {"mapping_to": "vol_percentage"}
+                    }
+                }
+            },
+            "server_credentials": {
+                "_self": {
+                    "mapping_to": "server_credentials"
+                },
+                "username": {
+                    "_self": {"mapping_to": "username"}
+                },
+                "password": {
+                    "_self": {"mapping_to": "password"}
+                }
+            }
+        },
+        "package_config": {
+            "_self": {},
+            "security": {
+                "_self": {},
+                "service_credentials": {
+                    "_self": {
+                        "mapping_to": "service_credentials"
+                    },
+                    "rabbit_mq": {
+                        "_self": {
+                            "mapping_to": "mq"
+                        },
+                        "username": {
+                            "_self": {
+                                "mapping_to": "username"
+                            }
+                        },
+                        "password": {
+                            "_self": {
+                                "mapping_to": "password"
+                            }
+                        }
+                    }
+                }
+            },
+            "network_mapping": {
+                "_self": {},
+                "management": {
+                    "_self": {},
+                    "interface": {
+                        "_self": {}
+                    }
+                },
+                "public": {
+                    "_self": {},
+                    "interface": {
+                        "_self": {}
+                    }
+                },
+                "tenant": {
+                    "_self": {},
+                    "interface": {
+                        "_self": {}
+                    }
+                }
+            },
+            "roles": {
+                "_self": {}
+            }
+        }
+    }
+}
+
+
+cluster_test_config = {
+    "id": 1,
+    "os_name": "Ubuntu-12.04-x86_64",
+    "name": "test",
+    "os_config": {
+        "general": {
+            "language": "EN",
+            "timezone": "UTC",
+            "default_gateway": "12.234.32.1",
+            "domain": "ods.com",
+            "http_proxy": "http://127.0.0.1:3128",
+            "https_proxy": "",
+            "ntp_server": "127.0.0.1",
+            "dns_servers": ["127.0.0.1"],
+            "search_path": ["1.ods.com", "ods.com"]
+        },
+        "partition": {
+            "/var": {
+                "max_size": 20,
+                "size_percentage": 20
+            },
+            "/home": {
+                "max_size": 50,
+                "size_percentage": 40
+            }
+        },
+        "server_credentials": {
+            "username": "root",
+            "password": "huawei"
+        }
+    },
+    "package_config": {
+        "security": {
+            "service_credentials": {
+                "rabbit_mq": {
+                    "username": "guest",
+                    "password": "test"
+                }
+            }
+        },
+        "network_mapping": {
+            "management": {
+                "interface": "eth0"
+            },
+            "public": {
+                "interface": "eth2"
+            },
+            "tenant": {
+                "interface": "eth1"
+            }
+        }
+    }
+}
+
+hosts_test_config = {
+    1: {
+        "host_id": 1,
+        "reinstall_os": True,
+        "mac": "00:0c:29:3e:60:e9",
+        "name": "server01.test",
+        "hostname": "server01",
+        "networks": {
+            "vnet0": {
+                "ip": "12.234.32.100",
+                "netmask": "255.255.255.0",
+                "is_mgmt": True,
+                "is_promiscuous": False,
+                "subnet": "12.234.32.0/24"
+            },
+            "vnet1": {
+                "ip": "172.16.1.1",
+                "netmask": "255.255.255.0",
+                "is_mgmt": False,
+                "is_promiscuous": False,
+                "subnet": "172.16.1.0/24"
+            }
+        },
+        "os_config": {
+            "general": {
+                "default_gateway": "10.145.88.1",
+            },
+            "partition": {
+                "/var": {
+                    "max_size": 30,
+                    "size_percentage": 30
+                },
+                "/test": {
+                    "max_size": 10,
+                    "size_percentage": 10
+                }
+            }
+        },
+        "package_config": {
+            "network_mapping": {
+                "management": {
+                    "interface": "vnet0"
+                },
+                "tenant": {
+                    "interface": "vnet1"
+                }
+            },
+            "roles": ["os-controller"]
+        }
+    },
+    2: {
+        "host_id": 2,
+        "reinstall_os": True,
+        "mac": "00:0c:29:3e:60:a1",
+        "name": "server02.test",
+        "hostname": "server02",
+        "networks": {
+            "eth0": {
+                "ip": "12.234.32.101",
+                "netmask": "255.255.255.0",
+                "is_mgmt": True,
+                "is_promiscuous": False,
+                "subnet": "12.234.32.0/24"
+            },
+            "eth1": {
+                "ip": "172.16.1.2",
+                "netmask": "255.255.255.0",
+                "is_mgmt": False,
+                "is_promiscuous": False,
+                "subnet": "172.16.1.0/24"
+            }
+        },
+        "os_config": {
+            "general": {
+                "language": "EN",
+                "timezone": "UTC",
+                "domain": "ods.com"
+            },
+            "partition": {
+                "/test": {
+                    "max_size": 10,
+                    "size_percentage": 20
+                }
+            }
+        },
+        "package_config": {
+            "roles": ["os-compute"]
+        }
+    },
+    3: {
+        "host_id": 10,
+        "reinstall_os": False,
+        "mac_address": "00:0c:29:3e:60:a2",
+        "name": "server03.test",
+        "hostname": "server03",
+        "networks": {
+            "eth0": {
+                "ip": "12.234.32.103",
+                "netmask": "255.255.255.0",
+                "is_mgmt": True,
+                "is_promiscuous": False,
+                "subnet": "12.234.32.0/24"
+            },
+            "eth1": {
+                "ip": "172.16.1.3",
+                "netmask": "255.255.255.0",
+                "is_mgmt": False,
+                "is_promiscuous": False,
+                "subnet": "172.16.1.0/24"
+            },
+            "eth2": {
+                "ip": "10.0.0.1",
+                "netmask": "255.255.255.0",
+                "is_mgmt": False,
+                "is_promiscuous": True,
+                "subnet": "10.0.0.0/24"
+            }
+        },
+        "os_config": {
+            "general": {
+                "language": "EN",
+                "timezone": "UTC",
+                "default_gateway": "12.234.32.1",
+                "domain": "ods.com",
+                "http_proxy": "http://10.145.88.211:3128",
+                "https_proxy": "",
+                "ntp_server": "10.145.88.211",
+                "dns_servers": "10.145.88.211",
+                "search_path": "1.ods.com ods.com"
+            },
+            "partition": {
+                "/var": {
+                    "max_size": 20,
+                    "size_percentage": 20
+                },
+                "/home": {
+                    "max_size": 50,
+                    "size_percentage": 40
+                }
+            }
+        },
+        "package_config": {
+            "roles": ["os-network"]
+        }
+    }
+}

compass/tests/deployment/test_data/templates/chef_installer/openstack_icehouse/env.tmpl

@@ -0,0 +1,46 @@
+#set config = $cluster.deployed_package_config
+#set mappings = $config.roles_mapping
+#set credentials = $config.service_credentials
+{
+  "name": "testing",
+  "description": "Environment",
+  "cookbook_versions": {
+  },
+  "json_class": "Chef::Environment",
+  "chef_type": "environment",
+  "default_attributes": {
+  },
+  "override_attributes": {
+      "compute": {
+          "syslog": {
+            "use": false
+          },
+          "libvirt": {
+            "bind_interface": "$mappings.os_compute.management.interface"
+          },
+          "novnc_proxy": {
+            "bind_interface": "$mappings.os_controller.management.interface"
+          },
+          "xvpvnc_proxy": {
+            "bind_interface": "eth0"
+          }
+      },
+      "db": {
+        "bind_interface": "$mappings.os_controller.management.interface",
+        "compute": {
+          "host": "$mappings.os_controller.management.ip"
+        },
+        "identity": {
+          "host": "$mappings.os_controller.management.ip"
+        }
+      },
+      "mq": {
+        "user": "$credentials.mq.username",
+        "password": "$credentials.mq.password",
+        "vhost": "/nova",
+        "network": {
+          "service_type": "rabbitmq"
+        }
+      }
+    }
+}

compass/tests/deployment/test_data/templates/chef_installer/openstack_icehouse/node/os_compute.tmpl

@@ -0,0 +1,9 @@
+{
+   "override_attributes": {
+       "endpoints": {
+           "compute-vnc-bind": {
+               "host":"$host.deployed_package_config.roles_mapping.os_compute.management.ip"
+           }
+       }
+   } 
+}

compass/tests/deployment/test_data/templates/cobbler/Ubuntu-12.04-x86_64/system.tmpl

@@ -0,0 +1,57 @@
+{
+    "name": "$host.name",
+    "hostname": "$host.hostname",
+    "profile": "$host.profile",
+    "gateway": "$host.gateway",
+    #import simplejson as json
+    #set nameservers = json.dumps($host.nameservers, encoding='utf-8')
+    "name_servers": $nameservers,
+    #set search_path = ' '.join($host.search_path)
+    "name_servers_search": "$search_path",
+    "proxy": "$host.http_proxy",
+    "modify_interface":
+        #set networks = $host.networks
+        #set rekeyed = {}
+        #set promicnic = ""
+        #for $nic, $val in $networks.iteritems():
+            #set ip_key = '-'.join(('ipaddress', $nic))
+            #set netmask_key = '-'.join(('netmask', $nic))
+            #set mgmt_key = '-'.join(('management', $nic))
+            #set static_key = '-'.join(('static', $nic))
+            #set $rekeyed[$ip_key] = $val.ip
+            #set $rekeyed[$netmask_key] = $val.netmask
+            #set $rekeyed[$mgmt_key] = $val.is_mgmt
+            #set $rekeyed[$static_key] = True
+
+            #if $val.is_promiscuous:
+                #set promicnic = $nic
+            #end if
+            #if $val.is_mgmt:
+                #set mac_key = '-'.join(('macaddress', $nic))
+                #set dns_key = '-'.join(('dns', $nic))
+                #set $rekeyed[$mac_key] = $host.mac
+                #set $rekeyed[$dns_key] = $host.dns
+            #end if
+        #end for
+        #set nic_info = json.dumps($rekeyed, encoding='utf-8')
+        $nic_info
+    ,
+    "ksmeta":{
+        #set partition_config = ''
+        #for k, v in $host.partition.iteritems():
+            #set path = ''
+            #if v['vol_percentage']:
+                #set $path = k + ' ' + str(v['vol_percentage']) + '%'
+            #else:
+                #set $path = k + ' ' + str(v['vol_size'])
+            #end if
+            #set partition_config = ';'.join((partition_config, $path))
+        #end for
+        #set partition_config = partition_config[1:]
+        "promisc_nics": "$promicnic",
+        "partition": "$partition_config",
+        "https_proxy": "$host.https_proxy",
+        "ntp_server": "$host.ntp_server",
+        "timezone": "$host.timezone"
+    }
+}

compass/tests/deployment/test_deploy_manager.py

@@ -0,0 +1,62 @@
+#!/usr/bin/python
+#
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = "Grace Yu (grace.yu@huawei.com)"
+
+
+"""Test deploy_manager module."""
+
+from mock import Mock
+import os
+import unittest2
+
+
+os.environ['COMPASS_IGNORE_SETTING'] = 'true'
+
+
+from compass.utils import setting_wrapper as setting
+from copy import deepcopy
+reload(setting)
+
+
+from compass.deployment.deploy_manager import DeployManager
+from compass.tests.deployment.test_data import config_data
+
+
+class TestDeployManager(unittest2.TestCase):
+    """Test DeployManager methods."""
+    def setUp(self):
+        super(TestDeployManager, self).setUp()
+
+    def tearDown(self):
+        super(TestDeployManager, self).tearDown()
+
+    def test_init_DeployManager(self):
+        adapter_info = deepcopy(config_data.adapter_test_config)
+        cluster_info = deepcopy(config_data.cluster_test_config)
+        hosts_info = deepcopy(config_data.hosts_test_config)
+
+        DeployManager._get_installer = Mock()
+        DeployManager._get_installer.return_value = "mock_installer"
+
+        test_manager = DeployManager(adapter_info, cluster_info, hosts_info)
+        self.assertIsNotNone(test_manager)
+
+        # Test hepler function _get_hosts_for_os_installation return correct
+        # number of hosts config for os deployment. In config_data, two out of
+        # three hosts need to install OS.
+        hosts_list = test_manager._get_hosts_for_os_installation(hosts_info)
+        self.assertEqual(2, len(hosts_list))

compass/utils/setting_wrapper.py

@@ -70,6 +70,7 @@ OS_FIELD_DIR = '/etc/compass/os_field'
 PACKAGE_FIELD_DIR = '/etc/compass/package_field'
 ADAPTER_ROLE_DIR = '/etc/compass/role'
 VALIDATOR_DIR = '/etc/compass/validator'
+TMPL_DIR = '/etc/compass/templates'
 if (
     'COMPASS_IGNORE_SETTING' in os.environ and
     os.environ['COMPASS_IGNORE_SETTING']

conf/adapter/chef_openstack.conf

@@ -1,7 +1,7 @@
-NAME = 'openstack(chef)'
-DISPLAY_NAME = 'openstack(chef)'
+NAME = 'openstack_icehouse'
+DISPLAY_NAME = 'OpenStack Icehouse'
 PARENT = 'openstack'
-PACKAGE_INSTALLER = 'chef(icehouse)'
+PACKAGE_INSTALLER = 'chef_installer'
 OS_INSTALLER = 'cobbler'
 SUPPORTED_OS_PATTERNS = ['(?i)centos.*', '(?i)ubuntu.*']
 DEPLOYABLE = True

conf/os_installer/cobbler.conf

@@ -1,6 +1,9 @@
 NAME = 'cobbler'
 INSTANCE_NAME = 'cobbler'
 SETTINGS = {
-    'url': 'http://127.0.0.1/cobbler_api',
-    'token': ('cobbler', 'cobbler') 
+    'cobbler_url': 'http://127.0.0.1/cobbler_api',
+    'credentials': {
+        'username': 'cobbler',
+        'password': 'cobbler'
+    }
 }

conf/os_metadata/general.conf

@@ -9,12 +9,13 @@ METADATA = {
                 'field': 'general',
                 'default_value': 'EN',
                 'options': ['EN', 'CN'],
+                'mapping_to': 'language'
             }
         },
         'timezone': {
             '_self': {
                 'field': 'general',
-                'default_value': 'GMT -8:00',
+                'default_value': 'UTC',
                 'options': [
                     'GMT -12:00', 'GMT -11:00', 'GMT -10:00', 'GMT -9:00',
                     'GMT -8:00', 'GMT -7:00', 'GMT -6:00', 'GMT -5:00',
@@ -22,8 +23,9 @@ METADATA = {
                     'GMT 0:00', 'GMT +1:00', 'GMT +2:00', 'GMT +3:00',
                     'GMT +4:00', 'GMT +5:00', 'GMT +6:00', 'GMT +7:00',
                     'GMT +8:00', 'GMT +9:00', 'GMT +10:00', 'GMT +11:00',
-                    'GMT +12:00'
+                    'GMT +12:00', UTC
                 ],
+                'mapping_to': 'timezone'
             }
         },
         'http_proxy': {
@@ -33,6 +35,7 @@ METADATA = {
                 'options': [
                     'http://10.145.88.211:3128'
                 ],
+                'mapping_to': 'http_proxy'
             }
         },
         'https_proxy': {
@@ -42,6 +45,7 @@ METADATA = {
                 'options': [
                     'http://10.145.88.211:3128'
                 ],
+                'mapping_to': 'https_proxy'
             }
         },
         'no_proxy': {
@@ -56,7 +60,8 @@ METADATA = {
                     '127.0.0.1',
                     'compass',
                     '10.145.88.211'
-                ]
+                ],
+                'mapping_to': 'no_proxy'
             }
         },
         'ntp_server': {
@@ -66,7 +71,8 @@ METADATA = {
                 'default_value': '10.145.88.211',
                 'options': [
                     '10.145.88.211'
-                ]
+                ],
+                'mapping_to': 'ntp_server'
             }
         },
         'dns_servers': {
@@ -78,7 +84,8 @@ METADATA = {
                 ],
                 'options': [
                     '10.145.88.211'
-                ]
+                ],
+                'mapping_to': 'nameservers'
             }
         },
         'domain': {
@@ -95,7 +102,8 @@ METADATA = {
                 'default_value': [
                     'ods.com'
                 ],
-                'options': ['ods.com']
+                'options': ['ods.com'],
+                'mapping_to': 'search_path'
             }
         },
         'default_gateway': {
@@ -103,23 +111,27 @@ METADATA = {
                 'is_required': True,
                 'field': 'ip',
                 'default_value': '10.145.88.1',
+                'mapping_to': 'gateway'
             }
         }
     },
     'server_credentials': {
         '_self': {
             'required_in_whole_config': True,
+            'mapping_to': 'server_credentials'
         },
         'username': {
             '_self': {
                 'is_required': True,
                 'field': 'username',
+                'mapping_to': 'username'
             }
         },
         'password': {
             '_self': {
                 'is_required': True,
-                'field': 'password'
+                'field': 'password',
+                'mapping_to': 'password'
             }
         }
     },
@@ -127,6 +139,7 @@ METADATA = {
         '_self': {
             'required_in_whole_config': True,
             'options': ['/boot', 'swap', '/var', '/home'],
+            'mapping_to': 'partition'
         },
         '$partition': {
             '_self': {
@@ -134,17 +147,20 @@ METADATA = {
             },
             'max_size': {
                 '_self': {
-                    'field': 'size'
+                    'field': 'size',
+                    'mapping_to': 'max_vol_size'
                 },
             },
             'percentage': {
                 '_self': {
                     'field': 'percentage',
+                    'mapping_to': 'vol_percentage'
                 }
             },
             'size': {
                 '_self': {
-                    'field': 'size'
+                    'field': 'size',
+                    'mapping_to': 'vol_size'
                 },
             }
         }

conf/package_installer/chef-icehouse.conf

@@ -1,5 +1,7 @@
-NAME = 'chef'
-INSTANCE_NAME = 'chef(icehouse)'
+NAME = 'chef_installer'
+INSTANCE_NAME = 'chef_installer'
 SETTINGS = {
-    'url': 'https://127.0.0.1'
+    'chef_url': 'https://10.145.88.150',
+    'key_dir': '/root/grace.pem',
+    'client_name': 'grace' 
 }

conf/package_metadata/openstack.conf

@@ -4,18 +4,41 @@ METADATA = {
         '_self': {
             'required_in_whole_config': True,
         },
-        '$credential_type': {
-            '$credential': {
+        'service_credentials': {
+            '_self': {
+                'mapping_to': 'service_credentials'    
+            },
+            '$service': {
                 'username': {
                     '_self': {
                         'is_required': True,
                         'field': 'username',
+                        'mapping_to': 'username'
                     }
                 },
                 'password': {
                     '_self': {
                         'is_required': True,
-                        'field': 'password'
+                        'field': 'password',
+                        'mapping_to': 'password'
+                    }
+                }
+            }
+        },
+        'console_credentials': {
+            '$console': {
+                'username': {
+                    '_self': {
+                        'is_required': True,
+                        'field': 'username',
+                        'mapping_to': 'username'
+                    }
+                },
+                'password': {
+                    '_self': {
+                        'is_required': True,
+                        'field': 'password',
+                        'mapping_to': 'password'
                     }
                 }
             }

conf/setting

@@ -26,3 +26,4 @@ PROGRESS_UPDATE_INTERVAL=30
 POLLSWITCH_INTERVAL=60
 SWITCHES = [
 ]
+TMPL_DIR = '/etc/compass/templates'

requirements.txt

@@ -4,6 +4,7 @@ flask-restful
 flask-sqlalchemy
 flask-login
 celery
+cheetah
 netaddr
 paramiko==1.7.5
 simplejson

templates/chef_installer/openstack_icehouse/databags/db_passwords.tmpl

@@ -0,0 +1,11 @@
+{
+    "nova": {
+        "nova": "mypass"    
+    },
+    "horizon": {
+        "horizon": "horizon"    
+    },
+    "keystone": {
+        
+    }
+}

templates/chef_installer/openstack_icehouse/env.tmpl

@@ -0,0 +1,317 @@
+#set config = $cluster.deployed_package_config
+#set mappings = $config.roles_mapping
+#set credentials = $config.service_credentials
+#set allinone_compute_mgmt_nic = $mappings.allinone_compute.management.interface
+#set allinone_compute_mgmt_ip = $mappings.allinone_compute.management.ip
+#set allinone_compute_mgmt_nic = $mappings.allinone_compute.management.interface
+{
+  "name": "testing",
+  "description": "Environment used in testing the upstream cookbooks and reference Chef repository",
+  "cookbook_versions": {
+  },
+  "json_class": "Chef::Environment",
+  "chef_type": "environment",
+  "default_attributes": {
+  },
+  "override_attributes": {
+    "mysql": {
+      "server_root_password": "test",
+      "server_debian_password": "root",
+      "server_repl_password": "root",
+      "allow_remote_root": true,
+      "root_network_acl": "%"
+    },
+    "openstack": {
+      "auth": {
+        "validate_certs": false
+      },
+      "block-storage": {
+        "syslog": {
+          "use": false
+        },
+        "api": {
+          "ratelimit": "False"
+        },
+        "debug": true,
+        "image_api_chef_role": "os-image",
+        "identity_service_chef_role": "os-identity",
+        "rabbit_server_chef_role": "os-ops-messaging"
+      },
+      "dashboard": {
+        "use_ssl": "false"
+      },
+      "compute": {
+        "syslog": {
+          "use": false
+        },
+        "libvirt": {
+          "bind_interface": "$allinone_compute_mgmt_nic"
+        },
+        "novnc_proxy": {
+          "bind_interface": "$allinone_compute_mgmt_nic"
+        },
+        "xvpvnc_proxy": {
+          "bind_interface": "$allinone_compute_mgmt_nic"
+        },
+        "image_api_chef_role": "os-image",
+        "identity_service_chef_role": "os-identity",
+        "nova_setup_chef_role": "os-compute-api",
+        "rabbit_server_chef_role": "os-ops-messaging",
+        "ratelimit": {
+          "api": {
+            "enabled": false
+          },
+          "volume": {
+            "enabled": false
+          }
+        },
+        "network": {
+          "service_type": "neutron",
+          "fixed_range": "10.0.0.0/8"
+        }
+      },
+      "network": {
+        "verbose": "True",
+        "debug": "True",
+        "service_plugins": [
+          "router"
+        ],
+        "ml2": {
+          "type_drivers": "local,flat,vlan,gre",
+          "tenant_network_types": "vlan,gre",
+          "network_vlan_ranges": "physnet1:1000:2999",
+          "enable_security_group": "True"
+        },
+        "openvswitch": {
+          "tenant_network_type": "vlan",
+          "network_vlan_ranges": "physnet1:1000:2999",
+          "bridge_mappings": "physnet1:br-eth1",
+          "fw_driver": "neutron.agent.firewall.NoopFirewallDriver"
+        }
+      },
+      "db": {
+        "bind_interface": "$allinone_compute_mgmt_nic",
+        "compute": {
+          "host": "$allinone_compute_mgmt_ip"
+        },
+        "identity": {
+          "host": "$allinone_compute_mgmt_ip"
+        },
+        "image": {
+          "host": "$allinone_compute_mgmt_ip"
+        },
+        "network": {
+          "host": "$allinone_compute_mgmt_ip"
+        },
+        "volume": {
+          "host": "$allinone_compute_mgmt_ip"
+        },
+        "dashboard": {
+          "host": "$allinone_compute_mgmt_ip"
+        },
+        "telemetry": {
+          "host": "$allinone_compute_mgmt_ip"
+        },
+        "orchestration": {
+          "host": "$allinone_compute_mgmt_ip"
+        }
+      },
+      "developer_mode": true,
+      "endpoints": {
+        "db": {
+          "host": "$allinone_compute_mgmt_ip"
+        },
+        "mq": {
+          "host": "$allinone_compute_mgmt_ip"
+        },
+        "compute-api": {
+          "host": "$allinone_compute_mgmt_ip",
+          "scheme": "http",
+          "port": "8774",
+          "path": "/v2/%(tenant_id)s"
+        },
+        "compute-api-bind": {
+          "bind_interface": "$allinone_compute_mgmt_nic"
+        },
+        "compute-ec2-admin": {
+          "host": "$allinone_compute_mgmt_ip",
+          "scheme": "http",
+          "port": "8773",
+          "path": "/services/Admin"
+        },
+        "compute-ec2-api": {
+          "host": "$allinone_compute_mgmt_ip",
+          "scheme": "http",
+          "port": "8773",
+          "path": "/services/Cloud"
+        },
+        "compute-xvpvnc": {
+          "host": "$allinone_compute_mgmt_ip",
+          "scheme": "http",
+          "port": "6081",
+          "path": "/console"
+        },
+        "compute-novnc": {
+          "host": "$allinone_compute_mgmt_ip",
+          "scheme": "http",
+          "port": "6080",
+          "path": "/vnc_auto.html"
+        },
+        "compute-novnc-bind": {
+          "bind_interface": "$allinone_compute_mgmt_nic"
+        },
+        "vnc_bind": {
+          "bind_interface": "$allinone_compute_mgmt_nic"
+        },
+        "image-api": {
+          "host": "$allinone_compute_mgmt_ip",
+          "scheme": "http",
+          "port": "9292",
+          "path": "/v2"
+        },
+        "image-api-bind": {
+          "bind_interface": "$allinone_compute_mgmt_nic"
+        },
+        "image-registry": {
+          "host": "$allinone_compute_mgmt_ip",
+          "scheme": "http",
+          "port": "9191",
+          "path": "/v2"
+        },
+        "image-registry-bind": {
+          "bind_interface": "$allinone_compute_mgmt_nic"
+        },
+        "identity-bind": {
+          "bind_interface": "$allinone_compute_mgmt_nic"
+        },
+        "identity-api": {
+          "host": "$allinone_compute_mgmt_ip",
+          "scheme": "http",
+          "port": "5000",
+          "path": "/v2.0"
+        },
+        "identity-admin": {
+          "host": "$allinone_compute_mgmt_ip",
+          "scheme": "http",
+          "port": "35357",
+          "path": "/v2.0"
+        },
+        "block-storage-api": {
+          "host": "$allinone_compute_mgmt_ip",
+          "scheme": "http",
+          "port": "8776",
+          "path": "/v1/%(tenant_id)s"
+        },
+        "block-storage-api-bind": {
+          "bind_interface": "$allinone_compute_mgmt_nic"
+        },
+        "telemetry-api": {
+          "host": "$allinone_compute_mgmt_ip",
+          "scheme": "http",
+          "port": "8777",
+          "path": "/v1"
+        },
+        "network-api": {
+          "host": "$allinone_compute_mgmt_ip",
+          "scheme": "http",
+          "port": "9696",
+          "path": ""
+        },
+        "network-api-bind": {
+          "bind_interface": "$allinone_compute_mgmt_nic"
+        },
+        "network-openvswitch": {
+          "bind_interface": "$allinone_compute_mgmt_nic"
+        },
+        "orchestration-api": {
+          "host": "$allinone_compute_mgmt_ip",
+          "scheme": "http",
+          "port": "8004",
+          "path": "/v1/%(tenant_id)s"
+        },
+        "orchestration-api-cfn": {
+          "host": "$allinone_compute_mgmt_ip",
+          "scheme": "http",
+          "port": "8000",
+          "path": "/v1"
+        }
+      },
+      "identity": {
+        "admin_user": "admin",
+        "bind_interface": "$allinone_compute_mgmt_nic",
+        "catalog": {
+          "backend": "sql"
+        },
+        "debug": true,
+        "rabbit_server_chef_role": "os-ops-messaging",
+        "roles": [
+          "admin",
+          "keystone_admin",
+          "keystone_service_admin",
+          "member",
+          "netadmin",
+          "sysadmin"
+        ],
+        "syslog": {
+          "use": false
+        },
+        "tenants": [
+          "admin",
+          "service",
+          "demo"
+        ],
+        "token": {
+          "backend": "sql"
+        },
+        "users": {
+          "admin": {
+            "password": "admin",
+            "default_tenant": "admin",
+            "roles": {
+              "admin": [
+                "admin"
+              ],
+              "keystone_admin": [
+                "admin"
+              ],
+              "keystone_service_admin": [
+                "admin"
+              ]
+            }
+          }
+        }
+      },
+      "image": {
+        "api": {
+          "bind_interface": "eth0"
+        },
+        "debug": true,
+        "identity_service_chef_role": "os-identity",
+        "rabbit_server_chef_role": "os-ops-messaging",
+        "registry": {
+          "bind_interface": "eth0"
+        },
+        "syslog": {
+          "use": false
+        },
+        "upload_image": {
+          "cirros": "http://download.cirros-cloud.net/0.3.2/cirros-0.3.2-x86_64-disk.img"
+        },
+        "upload_images": [
+          "cirros"
+        ]
+      },
+      "memcached_servers": [
+        "$allinone_compute_mgmt_nic:11211"
+      ],
+      "mq": {
+        "user": "$credentials.rabbitmq.username",
+        "password": "$credentials.rabbitmq.password",
+        "vhost": "/nova",
+        "network": {
+          "service_type": "rabbitmq"
+        }
+      }
+    }
+  }
+}

templates/chef_installer/openstack_icehouse/env.tmpl.multiroles

@@ -0,0 +1,317 @@
+#set config = $cluster.deploy_package_config
+#set mappings = $config.roles_mapping
+#set credentials = $config.service_credentials
+#set compute_mgmt_nic = $mappings.os_compute.management.interface
+#set controller_mgmt_ip = $mappings.os_controller.management.ip
+#set controller_mgmt_nic = $mappings.os_controller.management.interface
+{
+  "name": "testing",
+  "description": "Environment used in testing the upstream cookbooks and reference Chef repository",
+  "cookbook_versions": {
+  },
+  "json_class": "Chef::Environment",
+  "chef_type": "environment",
+  "default_attributes": {
+  },
+  "override_attributes": {
+    "mysql": {
+      "server_root_password": "test",
+      "server_debian_password": "root",
+      "server_repl_password": "root",
+      "allow_remote_root": true,
+      "root_network_acl": "%"
+    },
+    "openstack": {
+      "auth": {
+        "validate_certs": false
+      },
+      "block-storage": {
+        "syslog": {
+          "use": false
+        },
+        "api": {
+          "ratelimit": "False"
+        },
+        "debug": true,
+        "image_api_chef_role": "os-image",
+        "identity_service_chef_role": "os-identity",
+        "rabbit_server_chef_role": "os-ops-messaging"
+      },
+      "dashboard": {
+        "use_ssl": "false"
+      },
+      "compute": {
+        "syslog": {
+          "use": false
+        },
+        "libvirt": {
+          "bind_interface": "$compute_mgmt_nic"
+        },
+        "novnc_proxy": {
+          "bind_interface": "$controller_mgmt_nic"
+        },
+        "xvpvnc_proxy": {
+          "bind_interface": "$controller_mgmt_nic"
+        },
+        "image_api_chef_role": "os-image",
+        "identity_service_chef_role": "os-identity",
+        "nova_setup_chef_role": "os-compute-api",
+        "rabbit_server_chef_role": "os-ops-messaging",
+        "ratelimit": {
+          "api": {
+            "enabled": false
+          },
+          "volume": {
+            "enabled": false
+          }
+        },
+        "network": {
+          "service_type": "neutron",
+          "fixed_range": "10.0.0.0/8"
+        }
+      },
+      "network": {
+        "verbose": "True",
+        "debug": "True",
+        "service_plugins": [
+          "router"
+        ],
+        "ml2": {
+          "type_drivers": "local,flat,vlan,gre",
+          "tenant_network_types": "vlan,gre",
+          "network_vlan_ranges": "physnet1:1000:2999",
+          "enable_security_group": "True"
+        },
+        "openvswitch": {
+          "tenant_network_type": "vlan",
+          "network_vlan_ranges": "physnet1:1000:2999",
+          "bridge_mappings": "physnet1:br-eth1",
+          "fw_driver": "neutron.agent.firewall.NoopFirewallDriver"
+        }
+      },
+      "db": {
+        "bind_interface": "$controller_mgmt_nic",
+        "compute": {
+          "host": "$controller_mgmt_ip"
+        },
+        "identity": {
+          "host": "$controller_mgmt_ip"
+        },
+        "image": {
+          "host": "$controller_mgmt_ip"
+        },
+        "network": {
+          "host": "$controller_mgmt_ip"
+        },
+        "volume": {
+          "host": "$controller_mgmt_ip"
+        },
+        "dashboard": {
+          "host": "$controller_mgmt_ip"
+        },
+        "telemetry": {
+          "host": "$controller_mgmt_ip"
+        },
+        "orchestration": {
+          "host": "$controller_mgmt_ip"
+        }
+      },
+      "developer_mode": true,
+      "endpoints": {
+        "db": {
+          "host": "$controller_mgmt_ip"
+        },
+        "mq": {
+          "host": "$controller_mgmt_ip"
+        },
+        "compute-api": {
+          "host": "$controller_mgmt_ip",
+          "scheme": "http",
+          "port": "8774",
+          "path": "/v2/%(tenant_id)s"
+        },
+        "compute-api-bind": {
+          "bind_interface": "$controller_mgmt_nic"
+        },
+        "compute-ec2-admin": {
+          "host": "$controller_mgmt_ip",
+          "scheme": "http",
+          "port": "8773",
+          "path": "/services/Admin"
+        },
+        "compute-ec2-api": {
+          "host": "$controller_mgmt_ip",
+          "scheme": "http",
+          "port": "8773",
+          "path": "/services/Cloud"
+        },
+        "compute-xvpvnc": {
+          "host": "$controller_mgmt_ip",
+          "scheme": "http",
+          "port": "6081",
+          "path": "/console"
+        },
+        "compute-novnc": {
+          "host": "$controller_mgmt_ip",
+          "scheme": "http",
+          "port": "6080",
+          "path": "/vnc_auto.html"
+        },
+        "compute-novnc-bind": {
+          "bind_interface": "$controller_mgmt_nic"
+        },
+        "vnc_bind": {
+          "bind_interface": "$controller_mgmt_nic"
+        },
+        "image-api": {
+          "host": "$controller_mgmt_ip",
+          "scheme": "http",
+          "port": "9292",
+          "path": "/v2"
+        },
+        "image-api-bind": {
+          "bind_interface": "$controller_mgmt_nic"
+        },
+        "image-registry": {
+          "host": "$controller_mgmt_ip",
+          "scheme": "http",
+          "port": "9191",
+          "path": "/v2"
+        },
+        "image-registry-bind": {
+          "bind_interface": "$controller_mgmt_nic"
+        },
+        "identity-bind": {
+          "bind_interface": "$controller_mgmt_nic"
+        },
+        "identity-api": {
+          "host": "$controller_mgmt_ip",
+          "scheme": "http",
+          "port": "5000",
+          "path": "/v2.0"
+        },
+        "identity-admin": {
+          "host": "$controller_mgmt_ip",
+          "scheme": "http",
+          "port": "35357",
+          "path": "/v2.0"
+        },
+        "block-storage-api": {
+          "host": "$controller_mgmt_ip",
+          "scheme": "http",
+          "port": "8776",
+          "path": "/v1/%(tenant_id)s"
+        },
+        "block-storage-api-bind": {
+          "bind_interface": "$controller_mgmt_nic"
+        },
+        "telemetry-api": {
+          "host": "$controller_mgmt_ip",
+          "scheme": "http",
+          "port": "8777",
+          "path": "/v1"
+        },
+        "network-api": {
+          "host": "$controller_mgmt_ip",
+          "scheme": "http",
+          "port": "9696",
+          "path": ""
+        },
+        "network-api-bind": {
+          "bind_interface": "$controller_mgmt_nic"
+        },
+        "network-openvswitch": {
+          "bind_interface": "$controller_mgmt_nic"
+        },
+        "orchestration-api": {
+          "host": "$controller_mgmt_ip",
+          "scheme": "http",
+          "port": "8004",
+          "path": "/v1/%(tenant_id)s"
+        },
+        "orchestration-api-cfn": {
+          "host": "$controller_mgmt_ip",
+          "scheme": "http",
+          "port": "8000",
+          "path": "/v1"
+        }
+      },
+      "identity": {
+        "admin_user": "admin",
+        "bind_interface": "$controller_mgmt_nic",
+        "catalog": {
+          "backend": "sql"
+        },
+        "debug": true,
+        "rabbit_server_chef_role": "os-ops-messaging",
+        "roles": [
+          "admin",
+          "keystone_admin",
+          "keystone_service_admin",
+          "member",
+          "netadmin",
+          "sysadmin"
+        ],
+        "syslog": {
+          "use": false
+        },
+        "tenants": [
+          "admin",
+          "service",
+          "demo"
+        ],
+        "token": {
+          "backend": "sql"
+        },
+        "users": {
+          "admin": {
+            "password": "admin",
+            "default_tenant": "admin",
+            "roles": {
+              "admin": [
+                "admin"
+              ],
+              "keystone_admin": [
+                "admin"
+              ],
+              "keystone_service_admin": [
+                "admin"
+              ]
+            }
+          }
+        }
+      },
+      "image": {
+        "api": {
+          "bind_interface": "eth0"
+        },
+        "debug": true,
+        "identity_service_chef_role": "os-identity",
+        "rabbit_server_chef_role": "os-ops-messaging",
+        "registry": {
+          "bind_interface": "eth0"
+        },
+        "syslog": {
+          "use": false
+        },
+        "upload_image": {
+          "cirros": "http://download.cirros-cloud.net/0.3.2/cirros-0.3.2-x86_64-disk.img"
+        },
+        "upload_images": [
+          "cirros"
+        ]
+      },
+      "memcached_servers": [
+        "$controller_mgmt_nic:11211"
+      ],
+      "mq": {
+        "user": "$credentials.mq.username",
+        "password": "$credentials.mq.password",
+        "vhost": "/nova",
+        "network": {
+          "service_type": "rabbitmq"
+        }
+      }
+    }
+  }
+}

templates/chef_installer/openstack_icehouse/node/os_compute.tmpl

@@ -0,0 +1,9 @@
+{
+   "override_attributes": {
+       "endpoints": {
+           "compute-vnc-bind": {
+               "host":"$host.deployed_package_config.roles_mapping.os_compute.management.ip"
+           }
+       }
+   } 
+}

templates/cobbler/CentOS-6.5-x86_64/system.tmpl

@@ -0,0 +1,62 @@
+{
+    "name": "$host.name",
+    "hostname": "$host.hostname",
+    "profile": "$host.profile",
+    "gateway": "$host.gateway",
+    #import simplejson as json
+    #set nameservers = json.dumps($host.nameservers, encoding='utf-8')
+    "name_servers": $nameservers,
+    #set search_path = ' '.join($host.search_path)
+    "name_servers_search": "$search_path",
+    "proxy": "$host.http_proxy",
+    "modify_interface":
+        #set networks = $host.networks
+        #set rekeyed = {}
+        #set promicnic = ""
+        #for $nic, $val in $networks.iteritems():
+            #set ip_key = '-'.join(('ipaddress', $nic))
+            #set netmask_key = '-'.join(('netmask', $nic))
+            #set mgmt_key = '-'.join(('management', $nic))
+            #set static_key = '-'.join(('static', $nic))
+            #set $rekeyed[$ip_key] = $val.ip
+            #set $rekeyed[$netmask_key] = $val.netmask
+            #set $rekeyed[$mgmt_key] = $val.is_mgmt
+            #set $rekeyed[$static_key] = True
+
+            #if $val.is_promiscuous:
+                #set promicnic = $nic
+            #end if
+            #if $val.is_mgmt:
+                #set mac_key = '-'.join(('macaddress', $nic))
+                #set dns_key = '-'.join(('dns', $nic))
+                #set $rekeyed[$mac_key] = $host.mac
+                #set $rekeyed[$dns_key] = $host.dns
+            #end if
+        #end for
+        #set nic_info = json.dumps($rekeyed, encoding='utf-8')
+        $nic_info
+    ,
+    "ksmeta":{
+        #set partition_config = ''
+        #for k, v in $host.partition.iteritems():
+            #set path = ''
+            #if v['vol_percentage']:
+                #set $path = k + ' ' + str(v['vol_percentage']) + '%'
+            #else:
+                #set $path = k + ' ' + str(v['vol_size'])
+            #end if
+            #set partition_config = ';'.join((partition_config, $path))
+        #end for
+        #set partition_config = partition_config[1:]
+        #set $password = crypt.crypt($host.server_credentials.password, "az")
+        #set no_proxy = ','.join($host.no_proxy)
+        "username": "$host.server_credentials.username",
+        "password": "$password",
+        "promisc_nics": "$promicnic",
+        "partition": "$partition_config",
+        "https_proxy": "$host.https_proxy",
+        "ntp_server": "$host.ntp_server",
+        "timezone": "$host.timezone",
+        "ignore_proxy": "$no_proxy"
+    }
+}

templates/cobbler/Ubuntu-12.04-x86_64/system.tmpl

@@ -0,0 +1,36 @@
+{
+    "name": "$host.fullname",
+    "hostname": "$host.hostname",
+    "profile": "$host.profile",
+    "gateway": "$host.gateway",
+    "name_servers": "$host.name_servers",
+    "name_servers_search": "$host.search_path",
+    "proxy": "$host.http_proxy",
+    "modify_interface":
+        #import simplejson
+        #set interfaces = $host.networks.interfaces
+        #set rekeyed = {}
+        #for $nic, $val in $interfaces.iteritems():
+            #set $rekeyed = { "ipaddress-vnet0" : $val.ip, "netmask-vnet0": $val.netmask, "management-vnet0": $val.is_mgmt, "static-vnet0" : True }
+        #if $val.is_mgmt:
+          #set $rekeyed["macaddress-vnet0"] = $host.mac_address
+          #set $rekeyed["dnsname-vnet0"] = $host.dns
+            #end if
+        #end for
+        #set $str=simplejson.dumps($rekeyed, encoding='utf-8')
+        $str
+    ,
+    "ksmeta":{
+        "username": "$host.server_credentials.username",
+        #import crypt
+        #set $password = crypt.crypt($host.server_credentials.password, "az")
+        "password": "$password",
+        #set f=[(k + ' ' + v['vol_percentage'] or v['vol_size']) for k,v in $host.partition.iteritems()]
+        #set $partition_config = '; '.join($f)
+        "partition": "$partition_config",
+        "https_proxy": "$host.https_proxy",
+        "ignore_proxy": "$host.ignore_proxy",
+        "ntp_server": "$host.ntp_server"
+    }
+}
+