diff --git a/compass/actions/deploy.py b/compass/actions/deploy.py index 0e8a5988..9f8cc09a 100644 --- a/compass/actions/deploy.py +++ b/compass/actions/deploy.py @@ -13,16 +13,21 @@ # limitations under the License. """Module to deploy a given cluster - - .. moduleauthor:: Xiaodong Wang """ -import logging from compass.actions import util -from compass.db.api import database +from compass.db.api import adapter_holder as adapter_db +from compass.db.api import cluster as cluster_db +from compass.db.api import host as host_db +from compass.db.api import user as user_db +from compass.deployment.deploy_manager import DeployManager +from compass.deployment.utils import constants as const -def deploy(cluster_hosts): +import logging + + +def deploy(cluster_id, hosts_id_list, username=None): """Deploy clusters. :param cluster_hosts: clusters and hosts in each cluster to deploy. @@ -35,4 +40,207 @@ def deploy(cluster_hosts): if not lock: raise Exception('failed to acquire lock to deploy') - logging.debug('deploy cluster_hosts: %s', cluster_hosts) + user = user_db.get_user_object(username) + + cluster_info = ActionHelper.get_cluster_info(cluster_id, user) + adapter_id = cluster_info[const.ADAPTER_ID] + + adapter_info = ActionHelper.get_adapter_info(adapter_id, cluster_id, + user) + hosts_info = ActionHelper.get_hosts_info(hosts_id_list, user) + + logging.debug('[action][deploy]: adapter_info is %s', adapter_info) + logging.debug('[action][deploy]: cluster_info is %s', cluster_info) + logging.debug('[action][deploy]: hosts_info is %s', hosts_info) + + deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info) + #deploy_manager.prepare_for_deploy() + deployed_config = deploy_manager.deploy() + + ActionHelper.save_deployed_config(deployed_config, user) + + +def redeploy(cluster_id, hosts_id_list, username=None): + """Deploy clusters. + + :param cluster_hosts: clusters and hosts in each cluster to deploy. + :type cluster_hosts: dict of int or str to list of int or str + """ + with util.lock('serialized_action') as lock: + if not lock: + raise Exception('failed to acquire lock to deploy') + + user = user_db.get_user_object(username) + cluster_info = ActionHelper.get_cluster_info(cluster_id, user) + adapter_id = cluster_info[const.ADAPTER_ID] + + adapter_info = ActionHelper.get_adapter_info(adapter_id, + cluster_id, + user) + hosts_info = ActionHelper.get_hosts_info(cluster_id, + hosts_id_list, + user) + + deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info) + #deploy_manager.prepare_for_deploy() + deploy_manager.redeploy() + + +def poweron(host_id): + """Power on a list of hosts.""" + pass + + +def poweroff(host_id): + pass + + +def reset(host_id): + pass + + +class ActionHelper(object): + + @staticmethod + def get_adapter_info(adapter_id, cluster_id, user): + """Get adapter information. Return a dictionary as below, + { + "id": 1, + "name": "xxx", + "roles": ['xxx', 'yyy', ...], + "metadata": { + "os_config": { + ... + }, + "package_config": { + ... + } + }, + "os_installer": { + "name": "cobbler", + "settings": {....} + }, + "pk_installer": { + "name": "chef", + "settings": {....} + }, + ... + } + To view a complete output, please refer to backend doc. + """ + adapter_info = adapter_db.get_adapter(user, adapter_id) + metadata = cluster_db.get_cluster_metadata(user, cluster_id) + adapter_info.update(metadata) + + roles_info = adapter_info[const.ROLES] + roles_list = [role[const.NAME] for role in roles_info] + adapter_info[const.ROLES] = roles_list + + return adapter_info + + @staticmethod + def get_cluster_info(cluster_id, user): + """Get cluster information.Return a dictionary as below, + { + "cluster": { + "id": 1, + "adapter_id": 1, + "os_version": "CentOS-6.5-x86_64", + "name": "cluster_01", + "os_config": {..}, + "package_config": {...}, + "deployed_os_config": {}, + "deployed_package_config": {}, + "owner": "xxx" + } + } + """ + cluster_info = cluster_db.get_cluster(user, cluster_id) + cluster_config = cluster_db.get_cluster_config(user, cluster_id) + cluster_info.update(cluster_config) + + deploy_config = cluster_db.get_cluster_deployed_config(user, + cluster_id) + cluster_info.update(deploy_config) + + return cluster_info + + @staticmethod + def get_hosts_info(cluster_id, hosts_id_list, user): + """Get hosts information. Return a dictionary as below, + { + "hosts": { + 1($clusterhost_id/host_id): { + "reinstall_os": True, + "mac": "xxx", + "name": "xxx", + }, + "networks": { + "eth0": { + "ip": "192.168.1.1", + "netmask": "255.255.255.0", + "is_mgmt": True, + "is_promiscuous": False, + "subnet": "192.168.1.0/24" + }, + "eth1": {...} + }, + "os_config": {}, + "package_config": {}, + "deployed_os_config": {}, + "deployed_package_config": {} + }, + 2: {...}, + .... + } + } + """ + hosts_info = {} + for clusterhost_id in hosts_id_list: + info = cluster_db.get_clusterhost(user, clusterhost_id) + host_id = info[const.HOST_ID] + temp = host_db.get_host(user, host_id) + config = cluster_db.get_cluster_host_config(user, cluster_id, + host_id) + # Delete 'id' from temp + del temp['id'] + info.update(temp) + info.update(config) + + networks = info[const.NETWORKS] + networks_dict = {} + # Convert networks from list to dictionary format + for entry in networks: + nic_info = {} + nic_info = { + entry[const.NIC]: { + const.IP_ADDR: entry[const.IP_ADDR], + const.NETMASK: entry[const.NETMASK], + const.MGMT_NIC_FLAG: entry[const.MGMT_NIC_FLAG], + const.PROMISCUOUS_FLAG: entry[const.PROMISCUOUS_FLAG], + const.SUBNET: entry[const.SUBNET] + } + } + networks_dict.update(nic_info) + + info[const.NETWORKS] = networks_dict + + hosts_info[clusterhost_id] = info + + return hosts_info + + @staticmethod + def save_deployed_config(self, deployed_config, user): + cluster_config = deployed_config[const.CLUSTER] + cluster_id = cluster_config[const.ID] + del cluster_config[const.ID] + + cluster_db.update_cluster_deployed_config(user, cluster_id, + **cluster_config) + + hosts_id_list = deployed_config[const.HOSTS].keys() + for clusterhost_id in hosts_id_list: + config = deployed_config[const.HOSTS][clusterhost_id] + cluster_db.update_clusterhost_deployed_config(user, + clusterhost_id, + **config) diff --git a/compass/deployment/__init__.py b/compass/deployment/__init__.py new file mode 100644 index 00000000..cbd36e06 --- /dev/null +++ b/compass/deployment/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = "Grace Yu (grace.yu@huawei.com)" diff --git a/compass/deployment/deploy_manager.py b/compass/deployment/deploy_manager.py new file mode 100644 index 00000000..d209c460 --- /dev/null +++ b/compass/deployment/deploy_manager.py @@ -0,0 +1,164 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = "Grace Yu (grace.yu@huawei.com)" + +"""Module to get configs from provider and isntallers and update + them to provider and installers. +""" +from compass.deployment.installers.installer import OSInstaller +from compass.deployment.installers.installer import PKInstaller +from compass.deployment.utils import constants as const +from compass.utils import util + + +import logging + + +class DeployManager(object): + """Deploy manager module.""" + def __init__(self, adapter_info, cluster_info, hosts_info): + os_installer_name = adapter_info[const.OS_INSTALLER][const.NAME] + pk_installer_name = adapter_info[const.PK_INSTALLER][const.NAME] + + os_hosts_info = self._get_hosts_for_os_installation(hosts_info) + + self.os_installer = DeployManager._get_installer(OSInstaller, + os_installer_name, + adapter_info, + cluster_info, + os_hosts_info) + self.pk_installer = DeployManager._get_installer(PKInstaller, + pk_installer_name, + adapter_info, + cluster_info, + hosts_info) + + @staticmethod + def _get_installer(installer_type, name, adapter_info, cluster_info, + hosts_info): + """Get installer instance.""" + callback = getattr(installer_type, 'get_installer') + installer = callback(name, adapter_info, cluster_info, hosts_info) + + return installer + + def clean_progress(self): + """Clean previous installation log and progress.""" + # Clean DB + # db_api.cluster.clean_progress(self.cluster_id) + # db_api.cluster.clean_progress(self.cluster_id, self.host_id_list) + + # OS installer cleans previous installing progress. + if self.os_installer: + self.os_installer.clean_progress() + + # Package installer cleans previous installing progress. + if self.pk_installer: + self.package_installer.clean_progress() + + def prepare_for_deploy(self): + self.clean_progress() + + def deploy(self): + """Deploy the cluster.""" + deploy_config = {} + pk_instl_confs = {} + if self.pk_installer: + # generate target system config which will be installed by OS + # installer right after OS installation is completed. + pk_instl_confs = self.package_installer.generate_installer_config() + logging.debug('[DeployManager][deploy] pk_instl_confs is %s', + pk_instl_confs) + + if self.os_installer: + logging.info('[DeployManager][deploy]get OS installer %s', + self.os_installer) + # Send package installer config info to OS installer. + if pk_instl_confs: + self.os_installer.set_package_installer_config(pk_instl_confs) + + # start to deploy OS + try: + os_deploy_config = self.os_installer.deploy() + deploy_config = os_deploy_config + except Exception as ex: + logging.error(ex.message) + + if self.pk_installer: + logging.info('DeployManager][deploy]get package installer %s', + self.pk_installer) + + pk_deploy_config = self.package_installer.deploy() + util.merge_dict(deploy_config, pk_deploy_config) + + return deploy_config + + def redeploy(self): + """Redeploy the cluster without changing configurations.""" + if self.os_installer: + self.os_installer.redeploy() + + if self.package_installer: + self.package_installer.redeploy() + + def remove_hosts(self): + """Remove hosts from both OS and package installlers server side.""" + if self.os_installer: + self.os_installer.delete_hosts() + + if self.pk_installer: + self.pk_installer.delete_hosts() + + def _get_hosts_for_os_installation(self, hosts_info): + """Get info of hosts which need to install/reinstall OS.""" + hosts_list = {} + for host_id in hosts_info: + reinstall_os_flag = hosts_info[host_id][const.REINSTALL_OS_FLAG] + if not reinstall_os_flag: + continue + + hosts_list[host_id] = hosts_info[host_id] + + return hosts_list + + +class PowerManager(object): + """Manage host to power on, power off, and reset.""" + + def __init__(self, adapter_info, cluster_info, hosts_info): + os_installer_name = adapter_info[const.OS_INSTALLER][const.NAME] + self.os_installer = DeployManager._get_installer(OSInstaller, + os_installer_name, + adapter_info, + cluster_info, + hosts_info) + + def poweron(self): + if not self.os_installer: + logging.info("No OS installer found, cannot power on machine!") + return + self.os_installer.poweron() + + def poweroff(self): + if not self.os_installer: + logging.info("No OS installer found, cannot power on machine!") + return + self.os_installer.poweroff() + + def reset(self): + if not self.os_installer: + logging.info("No OS installer found, cannot power on machine!") + return + self.os_installer.reset() diff --git a/compass/deployment/installers/__init__.py b/compass/deployment/installers/__init__.py new file mode 100644 index 00000000..0296be54 --- /dev/null +++ b/compass/deployment/installers/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = "Grace Yu (grace.yu@huawei.com)" + + +"""modules to read/write cluster/host config from installers. + + .. moduleauthor:: Grace Yu +""" diff --git a/compass/deployment/installers/config_manager.py b/compass/deployment/installers/config_manager.py new file mode 100644 index 00000000..1d412be9 --- /dev/null +++ b/compass/deployment/installers/config_manager.py @@ -0,0 +1,460 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = "Grace Yu (grace.yu@huawei.com)" + + +"""Module to manage and access cluster, hosts and adapter config. +""" +from copy import deepcopy +import logging + + +from compass.deployment.utils import constants as const +from compass.utils import util + + +class BaseConfigManager(object): + + def __init__(self, adapter_info, cluster_info, hosts_info): + self.adapter_info = adapter_info + self.cluster_info = cluster_info + self.hosts_info = hosts_info + + def get_cluster_id(self): + if not self.cluster_info: + logging.info("cluster config is None or {}") + return None + + return self.cluster_info[const.ID] + + def get_clustername(self): + if not self.cluster_info: + logging.info("cluster config is None or {}") + return None + return self.cluster_info[const.NAME] + + def get_os_version(self): + if not self.cluster_info: + logging.info("cluster config is None or {}") + return None + return self.cluster_info[const.OS_VERSION] + + def get_cluster_baseinfo(self): + """Get cluster base information, including cluster_id, os_version, + and cluster_name. + """ + if not self.cluster_info: + logging.info("cluster config is None or {}") + return None + + attr_names = [const.ID, const.NAME, const.OS_VERSION] + + base_info = {} + for name in attr_names: + base_info[name] = self.cluster_info[name] + + return base_info + + def get_host_id_list(self): + if not self.hosts_info: + logging.info("hosts config is None or {}") + return [] + + return self.hosts_info.keys() + + def get_cluster_os_config(self): + if not self.cluster_info: + logging.info("cluster config is None or {}") + return {} + + return deepcopy(self.cluster_info[const.OS_CONFIG]) + + def get_cluster_package_config(self): + if not self.cluster_info: + logging.info("cluster config is None or {}") + return {} + + return deepcopy(self.cluster_info[const.PK_CONFIG]) + + def get_cluster_network_mapping(self): + package_config = self.get_cluster_package_config() + if not package_config: + logging.info("cluster package_config is None or {}.") + return {} + + mapping = package_config.setdefault(const.NETWORK_MAPPING, {}) + logging.info("Network mapping in the config is '%s'!", mapping) + + return deepcopy(mapping) + + def get_cluster_deployed_os_config(self): + if not self.cluster_info: + logging.info("cluster config is None or {}") + return {} + + config = self.cluster_info.setdefault(const.DEPLOYED_OS_CONFIG, {}) + return config + + def get_cluster_deployed_package_config(self): + if not self.cluster_info: + logging.info("cluster config is None or {}") + return {} + + config = self.cluster_info.setdefault(const.DEPLOYED_PK_CONFIG, {}) + return config + + def get_cluster_os_vars_dict(self): + deployed_config = self.get_cluster_deployed_os_config() + return self.__get_deployed_config_item(deployed_config, + const.TMPL_VARS_DICT) + + def get_cluster_package_vars_dict(self): + deployed_config = self.get_cluster_deployed_package_config() + return self.__get_deployed_config_item(deployed_config, + const.TMPL_VARS_DICT) + + def __get_deployed_config_item(deployed_config, item): + if not deployed_config: + logging.info("Deploy config is {}") + return {} + + value = deployed_config.setdefault(item, {}) + return deepcopy(value) + + def get_cluster_roles_mapping(self): + if not self.cluster_info: + logging.info("cluster config is None or {}") + return {} + + deploy_config = self.get_cluster_deployed_package_config() + + if const.ROLES_MAPPING not in deploy_config: + mapping = self._get_cluster_roles_mapping_helper() + deploy_config[const.ROLES_MAPPING] = mapping + else: + mapping = deploy_config[const.ROLES_MAPPING] + + return mapping + + def _get_host_info(self, host_id): + if not self.hosts_info: + logging.info("hosts config is None or {}") + return {} + + if host_id not in self.hosts_info: + logging.info("Cannot find host, ID is '%s'", host_id) + return {} + + return self.hosts_info[host_id] + + def get_host_baseinfo(self, host_id): + """Get host base information.""" + host_info = self._get_host_info(host_id) + if not host_info: + return {} + + attr_names = [const.REINSTALL_OS_FLAG, const.MAC_ADDR, const.NAME, + const.HOSTNAME, const.NETWORKS] + base_info = {} + for attr in attr_names: + temp = host_info[attr] + if isinstance(temp, dict) or isinstance(temp, list): + base_info[attr] = deepcopy(temp) + else: + base_info[attr] = temp + + base_info[const.DNS] = self.get_host_dns(host_id) + + return base_info + + def get_host_name(self, host_id): + host_info = self._get_host_info(host_id) + if not host_info: + return None + + return host_info.setdefault(const.NAME, None) + + def get_host_dns(self, host_id): + host_info = self._get_host_info(host_id) + if not host_info: + return None + + if const.DNS not in host_info: + name = host_info[const.NAME] + domain = self.get_host_domain(host_id) + host_info[const.DNS] = '.'.join((name, domain)) + + return host_info[const.DNS] + + def get_host_mac_address(self, host_id): + host_info = self._get_host_info(host_id) + if not host_info: + return None + + return host_info.setdefault(const.MAC_ADDR, None) + + def get_hostname(self, host_id): + host_info = self._get_host_info(host_id) + if not host_info: + return None + + return host_info.setdefault(const.HOSTNAME, None) + + def get_host_networks(self, host_id): + host_info = self._get_host_info(host_id) + if not host_info: + return {} + + return deepcopy(host_info.setdefault(const.NETWORKS, {})) + + def get_host_interfaces(self, host_id): + networks = self.get_host_networks(host_id) + if not networks: + return [] + + nic_names = networks.keys() + return nic_names + + def get_host_interface_config(self, host_id, interface): + networks = self.get_host_networks(host_id) + if not networks: + return {} + + return deepcopy(networks.setdefault(interface, {})) + + def get_host_interface_ip(self, host_id, interface): + interface_config = self._get_host_interface_config(host_id, interface) + if not interface_config: + return None + + return interface_config.setdefault(const.IP_ADDR, None) + + def get_host_interface_netmask(self, host_id, interface): + interface_config = self.get_host_interface_config(host_id, interface) + if not interface_config: + return None + + return interface_config.setdefault(const.NETMASK, None) + + def get_host_interface_subnet(self, host_id, interface): + nic_config = self.get_host_interface_config(host_id, interface) + if not nic_config: + return None + + return nic_config.setdefault(const.SUBNET, None) + + def is_interface_promiscuous(self, host_id, interface): + nic_config = self.get_host_interface_config(host_id, interface) + if not nic_config: + raise Exception("Cannot find interface '%s'", interface) + + return nic_config[const.PROMISCUOUS_FLAG] + + def is_interface_mgmt(self, host_id, interface): + nic_config = self.get_host_interface_config(host_id, interface) + if not nic_config: + raise Exception("Cannot find interface '%s'", interface) + + return nic_config[const.MGMT_NIC_FLAG] + + def get_host_os_config(self, host_id): + host_info = self._get_host_info(host_id) + if not host_info: + return {} + + return deepcopy(host_info.setdefault(const.OS_CONFIG, {})) + + def get_host_domain(self, host_id): + os_config = self.get_host_os_config(host_id) + os_general_config = os_config.setdefault(const.OS_CONFIG_GENERAL, {}) + domain = os_general_config.setdefault(const.DOMAIN, None) + if domain is None: + global_config = self.get_cluster_os_config() + global_general = global_config.setdefault(const.OS_CONFIG_GENERAL, + {}) + domain = global_general.setdefault(const.DOMAIN, None) + + return domain + + def get_host_network_mapping(self, host_id): + package_config = self.get_host_package_config(host_id) + if const.NETWORK_MAPPING not in package_config: + network_mapping = self.get_cluster_network_mapping() + else: + network_mapping = package_config[const.NETWORK_MAPPING] + + return network_mapping + + def get_host_package_config(self, host_id): + host_info = self._get_host_info(host_id) + if not host_info: + return {} + + return deepcopy(host_info.setdefault(const.PK_CONFIG, {})) + + def get_host_deployed_os_config(self, host_id): + host_info = self._get_host_info(host_id) + if not host_info: + return {} + + return host_info.setdefault(const.DEPLOYED_OS_CONFIG, {}) + + def get_host_os_vars_dict(self, host_id): + deployed_config = self.get_host_deployed_os_config(host_id) + return self.__get_deployed_config_item(deployed_config, + const.TMPL_VARS_DICT) + + def get_host_completed_os_config(self, host_id): + deployed_config = self.get_host_deployed_os_config(host_id) + config = self.__get_deployed_config_item(deployed_config, + const.COMPLETED_OS_CONFIG) + if not config: + config = self.get_cluster_os_config() + util.merge_dict(config, self.get_host_os_config(host_id)) + deployed_config[const.COMPLETED_OS_CONFIG] = config + + return config + + def get_host_deployed_package_config(self, host_id): + host_info = self._get_host_info(host_id) + if not host_info: + return {} + + return host_info.setdefault(const.DEPLOYED_PK_CONFIG, {}) + + def get_host_package_vars_dict(self, host_id): + deployed_config = self.get_host_deployed_package_config(host_id) + return self.__get_deployed_config_item(deployed_config, + const.TMPL_VARS_DICT) + + def get_host_roles(self, host_id): + config = self.get_host_package_config(host_id) + if not config: + return [] + + return deepcopy(config.setdefault(const.ROLES, [])) + + def get_host_roles_mapping(self, host_id): + roles_mapping = {} + deployed_pk_config = self.get_host_deployed_package_config(host_id) + if const.ROLES_MAPPING not in deployed_pk_config: + roles_mapping = self._get_host_roles_mapping_helper(host_id) + else: + roles_mapping = deployed_pk_config[const.ROLES_MAPPING] + + return roles_mapping + + def get_adapter_name(self): + if not self.adapter_info: + logging.info("Adapter Info is None!") + return None + + return self.adapter_info.setdefault(const.NAME, None) + + def get_dist_system_name(self): + if not self.adapter_info: + logging.info("Adapter Info is None!") + return None + + return self.adapter_info.setdefault(const.DIST_SYS_NAME, None) + + def get_adapter_roles(self): + if not self.adapter_info: + logging.info("Adapter Info is None!") + return [] + + roles = self.adapter_info.setdefault(const.ROLES, []) + + return deepcopy(roles) + + def get_os_installer_settings(self): + if not self.adapter_info: + logging.info("Adapter Info is None!") + return None + + return self.adapter_info[const.OS_INSTALLER][const.INSTALLER_SETTINGS] + + def get_pk_installer_settings(self): + if not self.adapter_info: + logging.info("Adapter Info is None!") + return None + + return self.adapter_info[const.PK_INSTALLER][const.INSTALLER_SETTINGS] + + def get_os_config_metadata(self): + if not self.adapter_info: + logging.info("Adapter Info is None!") + return None + + return self.adapter_info[const.METADATA][const.OS_CONFIG] + + def get_pk_config_meatadata(self): + if not self.adapter_info: + logging.info("Adapter Info is None!") + return None + + return self.adapter_info[const.METADATA][const.PK_CONFIG] + + def _get_cluster_roles_mapping_helper(self): + """The ouput format will be as below, for example: + { + "controller": { + "management": { + "interface": "eth0", + "ip": "192.168.1.10", + "netmask": "255.255.255.0", + "subnet": "192.168.1.0/24", + "is_mgmt": True, + "is_promiscuous": False + }, + ... + }, + ... + } + """ + mapping = {} + hosts_id_list = self.get_host_id_list() + network_mapping = self.get_cluster_network_mapping() + if not network_mapping: + return {} + + for host_id in hosts_id_list: + roles_mapping = self.get_host_roles_mapping(host_id) + for role in roles_mapping: + if role not in mapping: + mapping[role] = roles_mapping[role] + + return mapping + + def _get_host_roles_mapping_helper(self, host_id): + """The format will be the same as cluster roles mapping.""" + mapping = {} + network_mapping = self.get_host_network_mapping(host_id) + if not network_mapping: + return {} + + roles = self.get_host_roles(host_id) + interfaces = self.get_host_interfaces(host_id) + temp = {} + for key in network_mapping: + nic = network_mapping[key][const.NIC] + if nic in interfaces: + temp[key] = self.get_host_interface_config(host_id, nic) + temp[key][const.NIC] = nic + + for role in roles: + role = role.replace("-", "_") + mapping[role] = temp + return mapping diff --git a/compass/deployment/installers/installer.py b/compass/deployment/installers/installer.py new file mode 100644 index 00000000..d1347a0b --- /dev/null +++ b/compass/deployment/installers/installer.py @@ -0,0 +1,221 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = "Grace Yu (grace.yu@huawei.com)" + + +"""Module to provider installer interface. +""" +from Cheetah.Template import Template +from copy import deepcopy +import imp +import logging +import os +import simplejson as json + + +CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) + + +class BaseInstaller(object): + """Interface for installer.""" + NAME = 'installer' + + def __repr__(self): + return '%r[%r]' % (self.__class__.__name__, self.NAME) + + def deploy(self, **kwargs): + """virtual method to start installing process.""" + raise NotImplementedError + + def clean_progress(self, **kwargs): + raise NotImplementedError + + def delete_hosts(self, **kwargs): + """Delete hosts from installer server.""" + raise NotImplementedError + + def redeploy(self, **kwargs): + raise NotImplementedError + + def get_tmpl_vars_from_metadata(self, metadata, config): + """Get variables dictionary for rendering templates from metadata. + :param dict metadata: The metadata dictionary. + :param dict config: The + """ + template_vars = {} + self._get_tmpl_vars_helper(metadata, config, template_vars) + + return template_vars + + def _get_key_mapping(self, metadata, key, is_regular_key): + """Get the keyword which the input key maps to. This keyword will be + added to dictionary used to render templates. + + If the key in metadata has a mapping to another keyword which is + used for templates, then return this keyword. If the key is started + with '$', which is a variable in metadata, return the key itself as + the mapping keyword. If the key has no mapping, return None. + + :param dict metadata: metadata/submetadata dictionary. + :param str key: The keyword defined in metadata. + :param bool is_regular_key: False when the key defined in metadata + is a variable(starting with '$'). + """ + mapping_to = key + if is_regular_key: + try: + mapping_to = metadata['_self']['mapping_to'] + except Exception: + mapping_to = None + return mapping_to + + def _get_submeta_by_key(self, metadata, key): + """Get submetadata dictionary based on current metadata key. And + determines the input key is a regular string keyword or a variable + keyword defined in metadata, which starts with '$'. + + :param dict metadata: The metadata dictionary. + :param str key: The keyword defined in the metadata. + """ + if key in metadata: + return (True, metadata[key]) + + temp = deepcopy(metadata) + del temp['_self'] + meta_key = temp.keys()[0] + if meta_key.startswith("$"): + return (False, metadata[meta_key]) + + raise KeyError("'%s' is an invalid metadata!" % key) + + def _get_tmpl_vars_helper(self, metadata, config, output): + for key, config_value in config.iteritems(): + is_regular_key, sub_meta = self._get_submeta_by_key(metadata, key) + mapping_to = self._get_key_mapping(sub_meta, key, is_regular_key) + + if isinstance(config_value, dict): + if mapping_to: + new_output = output[mapping_to] = {} + else: + new_output = output + + self._get_tmpl_vars_helper(sub_meta, config_value, + new_output) + elif mapping_to: + output[mapping_to] = config_value + + def get_config_from_template(self, tmpl_dir, vars_dict): + if not os.path.exists(tmpl_dir) or not vars_dict: + logging.info("Template or variables dict is not specified!") + logging.debug("template dir is %s", tmpl_dir) + logging.debug("template vars dict is %s", vars_dict) + return {} + + tmpl = Template(file=tmpl_dir, searchList=[vars_dict]) + config = json.loads(tmpl.respond(), encoding='utf-8') + config = json.loads(json.dumps(config), encoding='utf-8') + return config + + @classmethod + def get_installer(cls, name, path, adapter_info, cluster_info, hosts_info): + installer = None + try: + mod_file, path, descr = imp.find_module(name, [path]) + if mod_file: + mod = imp.load_module(name, mod_file, path, descr) + return getattr(mod, mod.NAME)(adapter_info, cluster_info, + hosts_info) + + except ImportError as exc: + logging.error('No such module found: %s', name) + logging.exception(exc) + + return installer + + +class OSInstaller(BaseInstaller): + """Interface for os installer.""" + NAME = 'OSInstaller' + INSTALLER_BASE_DIR = os.path.join(CURRENT_DIR, 'os_installers') + + def get_oses(self): + """virtual method to get supported oses. + + :returns: list of str, each is the supported os version. + """ + return [] + + @classmethod + def get_installer(cls, name, adapter_info, cluster_info, hosts_info): + path = os.path.join(cls.INSTALLER_BASE_DIR, name) + installer = super(OSInstaller, cls).get_installer(name, path, + adapter_info, + cluster_info, + hosts_info) + + if not isinstance(installer, OSInstaller): + logging.info("Installer '%s' is not an OS installer!" % name) + return None + + return installer + + def poweron(self, host_id): + pass + + def poweroff(self, host_id): + pass + + def reset(self, host_id): + pass + + +class PKInstaller(BaseInstaller): + """Interface for package installer.""" + NAME = 'PKInstaller' + INSTALLER_BASE_DIR = os.path.join(CURRENT_DIR, 'pk_installers') + + def get_target_systems(self): + """virtual method to get available target_systems for each os. + + :param oses: supported os versions. + :type oses: list of st + + :returns: dict of os_version to target systems as list of str. + """ + return {} + + def get_roles(self, target_system): + """virtual method to get all roles of given target system. + + :param target_system: target distributed system such as openstack. + :type target_system: str + + :returns: dict of role to role description as str. + """ + return {} + + @classmethod + def get_installer(cls, name, adapter_info, cluster_info, hosts_info): + path = os.path.join(cls.INSTALLER_BASE_DIR, name) + installer = super(PKInstaller, cls).get_installer(name, path, + adapter_info, + cluster_info, + hosts_info) + + if not isinstance(installer, PKInstaller): + logging.info("Installer '%s' is not a package installer!" % name) + return None + + return installer diff --git a/compass/deployment/installers/os_installers/__init__.py b/compass/deployment/installers/os_installers/__init__.py new file mode 100644 index 00000000..5e42ae96 --- /dev/null +++ b/compass/deployment/installers/os_installers/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/deployment/installers/os_installers/cobbler/__init__.py b/compass/deployment/installers/os_installers/cobbler/__init__.py new file mode 100644 index 00000000..5e42ae96 --- /dev/null +++ b/compass/deployment/installers/os_installers/cobbler/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/deployment/installers/os_installers/cobbler/cobbler.py b/compass/deployment/installers/os_installers/cobbler/cobbler.py new file mode 100644 index 00000000..f3ede0eb --- /dev/null +++ b/compass/deployment/installers/os_installers/cobbler/cobbler.py @@ -0,0 +1,415 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""os installer cobbler plugin. +""" +import logging +import os +import shutil +import xmlrpclib + +from compass.deployment.installers.config_manager import BaseConfigManager +from compass.deployment.installers.installer import OSInstaller +from compass.deployment.utils import constants as const +from compass.utils import setting_wrapper as compass_setting +from compass.utils import util + + +NAME = 'CobblerInstaller' + + +class CobblerInstaller(OSInstaller): + """cobbler installer""" + CREDENTIALS = "credentials" + USERNAME = 'username' + PASSWORD = 'password' + + INSTALLER_URL = "cobbler_url" + TMPL_DIR = 'tmpl_dir' + SYS_TMPL = 'system.tmpl' + SYS_TMPL_NAME = 'system.tmpl' + PROFILE = 'profile' + POWER_TYPE = 'power_type' + POWER_ADDR = 'power_address' + POWER_USER = 'power_user' + POWER_PASS = 'power_pass' + + def __init__(self, adapter_info, cluster_info, hosts_info): + super(CobblerInstaller, self).__init__() + + self.config_manager = CobblerConfigManager(adapter_info, + cluster_info, + hosts_info) + installer_settings = self.config_manager.get_os_installer_settings() + try: + username = installer_settings[self.CREDENTIALS][self.USERNAME] + password = installer_settings[self.CREDENTIALS][self.PASSWORD] + cobbler_url = installer_settings[self.INSTALLER_URL] + self.tmpl_dir = CobblerInstaller.get_tmpl_path() + + except KeyError as ex: + raise KeyError(ex.message) + + # The connection is created when cobbler installer is initialized. + self.remote = self._get_cobbler_server(cobbler_url) + self.token = self._get_token(username, password) + self.pk_installer_config = None + + logging.debug('%s instance created', 'CobblerInstaller') + + @classmethod + def get_tmpl_path(cls): + return os.path.join(compass_setting.TMPL_DIR, 'cobbler') + + def __repr__(self): + return '%r[name=%r,remote=%r,token=%r' % ( + self.__class__.__name__, self.remote, self.token) + + def _get_cobbler_server(self, cobbler_url): + if not cobbler_url: + logging.error("Cobbler URL is None!") + raise Exception("Cobbler URL cannot be None!") + + return xmlrpclib.Server(cobbler_url) + + def _get_token(self, username, password): + if self.remote is None: + raise Exception("Cobbler remote instance is None!") + return self.remote.login(username, password) + + def get_supported_oses(self): + """get supported os versions. + .. note:: + In cobbler, we treat profile name as the indicator + of os version. It is just a simple indicator + and not accurate. + """ + profiles = self.remote.get_profiles() + oses = [] + for profile in profiles: + oses.append(profile['name']) + return oses + + def deploy(self): + """Sync cobbler to catch up the latest update config and start to + install OS. Return both cluster and hosts deploy configs. The return + format: + { + "cluster": { + "id": 1, + "deployed_os_config": {}, + }, + "hosts": { + 1($clusterhost_id): { + "deployed_os_config": {...}, + }, + .... + } + } + """ + os_version = self.config_manager.get_os_version() + profile = self._get_profile_from_server(os_version) + + global_vars_dict = self._get_cluster_tmpl_vars_dict() + + clusterhost_ids = self.config_manager.get_host_id_list() + hosts_deploy_config = {} + + for host_id in clusterhost_ids: + fullname = self.config_manager.get_host_name(host_id) + vars_dict = self._get_host_tmpl_vars_dict(host_id, + global_vars_dict, + fullname=fullname, + profile=profile) + + self.update_host_config_to_cobbler(host_id, fullname, vars_dict) + + # set host deploy config + temp = {} + temp = self.config_manager.get_host_deploy_os_config(host_id) + temp[const.TMPL_VARS_DICT] = vars_dict + host_config = {const.DEPLOYED_OS_CONFIG: temp} + hosts_deploy_config[host_id] = host_config + + # set cluster deploy config + cluster_deconfig = self.config_manager.get_cluster_deployed_os_config() + cluster_deconfig[const.TMPL_VARS_DICT] = global_vars_dict + + # sync to cobbler and trigger installtion. + self._sync() + + return { + const.CLUSTER: { + const.ID: self.config_manager.get_cluster_id(), + const.DEPLOYED_OS_CONFIG: cluster_deconfig + }, + const.HOSTS: hosts_deploy_config + } + + def clean_progress(self): + """clean log files and config for hosts which to deploy.""" + clusterhost_list = self.config_manager.get_host_id_list() + log_dir_prefix = compass_setting.INSTALLATION_LOGDIR[self.NAME] + + for host_id in clusterhost_list: + fullname = self.config_manager.get_host_name(host_id) + self._clean_log(log_dir_prefix, fullname) + + def redeploy(self): + """redeploy hosts.""" + host_ids = self.config_manager.get_host_id_list() + for host_id in host_ids: + fullname = self.config_manager.get_host_name(host_id) + sys_id = self._get_system_id(fullname) + if sys_id: + self._netboot_enabled(sys_id) + + self._sync() + + def set_package_installer_config(self, package_configs): + """Cobbler can install and configure package installer right after + OS installation compelets by setting package_config info provided + by package installer. + + :param dict package_configs: The dict of config generated by package + installer for each clusterhost. The IDs + of clusterhosts are the keys of + package_configs. + """ + self.pk_installer_config = package_configs + + def _sync(self): + """Sync the updated config to cobbler and trigger installation.""" + try: + self.remote.sync(self.token) + logging.debug('sync %s', self) + os.system('sudo service rsyslog restart') + except Exception as ex: + logging.debug("Failed to sync cobbler server! Error:", ex.message) + raise ex + + def _get_system_config(self, host_id, vars_dict): + """Generate updated system config from the template. + + :param vars_dict: dict of variables for the system template to + generate system config dict. + """ + os_version = self.config_manager.get_os_version() + + system_tmpl_path = os.path.join(os.path.join(self.tmpl_dir, + os_version), + self.SYS_TMPL_NAME) + system_config = self.get_config_from_template(system_tmpl_path, + vars_dict) + # update package config info to cobbler ksmeta + if self.pk_installer_config and host_id in self.pk_installer_config: + pk_config = self.pk_installer_config[host_id] + ksmeta = system_config.setdefault("ksmeta", {}) + util.merge_dict(ksmeta, pk_config) + system_config["ksmeta"] = ksmeta + + return system_config + + def _get_profile_from_server(self, os_version): + """Get profile from cobbler server.""" + result = self.remote.find_profile({'name': os_version}) + if not result: + raise Exception("Cannot find profile for '%s'", os_version) + + profile = result[0] + return profile + + def _get_system_id(self, fullname): + """get system reference id for the host.""" + sys_name = fullname + sys_id = None + system_info = self.remote.find_system({"name": fullname}) + + if not system_info: + # Create a new system + sys_id = self.remote.new_system(self.token) + self.remote.modify_system(sys_id, "name", fullname, self.token) + logging.debug('create new system %s for %s', sys_id, sys_name) + else: + sys_id = self.remote.get_system_handle(sys_name, self.token) + + return sys_id + + def _clean_system(self, fullname): + """clean system.""" + sys_name = fullname + try: + self.remote.remove_system(sys_name, self.token) + logging.debug('system %s is removed', sys_name) + except Exception: + logging.debug('no system %s found to remove', sys_name) + + def _update_system_config(self, sys_id, system_config): + """update modify system.""" + for key, value in system_config.iteritems(): + self.remote.modify_system(sys_id, str(key), value, self.token) + + self.remote.save_system(sys_id, self.token) + + def _netboot_enabled(self, sys_id): + """enable netboot.""" + self.remote.modify_system(sys_id, 'netboot_enabled', True, self.token) + self.remote.save_system(sys_id, self.token) + + def _clean_log(self, log_dir_prefix, system_name): + """clean log.""" + log_dir = os.path.join(log_dir_prefix, system_name) + shutil.rmtree(log_dir, True) + + def update_host_config_to_cobbler(self, host_id, fullname, vars_dict): + """update host config and upload to cobbler server.""" + sys_id = self._get_system_id(fullname) + + system_config = self._get_system_config(host_id, vars_dict) + logging.debug('%s system config to update: %s', host_id, system_config) + + self._update_system_config(sys_id, system_config) + self._netboot_enabled(sys_id) + + def delete_hosts(self): + hosts_id_list = self.config_manager.get_host_id_list() + for host_id in hosts_id_list: + self.delete_single_host(host_id) + + def delete_single_host(self, host_id): + """Delete the host from cobbler server and clean up the installation + progress. + """ + fullname = self.config_manager.get_host_name(host_id) + try: + log_dir_prefix = compass_setting.INSTALLATION_LOGDIR[self.NAME] + self._clean_system(fullname) + self._clean_log(log_dir_prefix, fullname) + except Exception as ex: + logging.info("Deleting host got exception: %s", ex.message) + + def _get_host_tmpl_vars_dict(self, host_id, global_vars_dict, **kwargs): + """Generate template variables dictionary. + """ + vars_dict = {} + if global_vars_dict: + # Set cluster template vars_dict from cluster os_config. + util.merge_dict(vars_dict, global_vars_dict) + + if self.PROFILE in kwargs: + profile = kwargs[self.PROFILE] + else: + os_version = self.config_manager.get_os_version() + profile = self._get_profile_from_server(os_version) + vars_dict[self.PROFILE] = profile + + # Set fullname, MAC address and hostname, networks, dns and so on. + host_baseinfo = self.config_manager.get_host_baseinfo(host_id) + util.merge_dict(vars_dict, host_baseinfo) + + os_config_metadata = self.config_manager.get_os_config_metadata() + host_os_config = self.config_manager.get_host_os_config(host_id) + + # Get template variables values from host os_config + host_vars_dict = self.get_tmpl_vars_from_metadata(os_config_metadata, + host_os_config) + util.merge_dict(vars_dict, host_vars_dict) + + return {const.HOST: vars_dict} + + def _get_cluster_tmpl_vars_dict(self): + os_config_metadata = self.config_manager.get_os_config_metadata() + cluster_os_config = self.config_manager.get_cluster_os_config() + + vars_dict = self.get_tmpl_vars_from_metadata(os_config_metadata, + cluster_os_config) + return vars_dict + + def _check_and_set_system_impi(self, host_id, sys_id): + if not sys_id: + logging.info("System is None!") + return False + + fullname = self.config_manager.get_host_name(host_id) + system = self.remote.get_system_as_rendered(fullname) + if system[self.POWER_TYPE] != 'ipmilan' or not system[self.POWER_USER]: + ipmi_info = self.config_manager.get_host_ipmi_info(host_id) + if not ipmi_info: + logging.info('No IPMI information found! Failed power on.') + return False + + ipmi_ip, ipmi_user, ipmi_pass = ipmi_info + power_opts = {} + power_opts[self.POWER_TYPE] = 'ipmilan' + power_opts[self.POWER_ADDR] = ipmi_ip + power_opts[self.POWER_USER] = ipmi_user + power_opts[self.POWER_PASS] = ipmi_pass + + self._update_system_config(sys_id, power_opts) + + return True + + def poweron(self, host_id): + fullname = self.config_manager.get_host_name(host_id) + sys_id = self._get_system_id(fullname) + if not self._check_and_set_system_impi(sys_id): + return + + self.remote.power_system(sys_id, self.token, power='on') + logging.info("Host with ID=%d starts to power on!" % host_id) + + def poweroff(self, host_id): + fullname = self.config_manager.get_host_name(host_id) + sys_id = self._get_system_id(fullname) + if not self._check_and_set_system_impi(sys_id): + return + + self.remote.power_system(sys_id, self.token, power='off') + logging.info("Host with ID=%d starts to power off!" % host_id) + + def reset(self, host_id): + fullname = self.config_manager.get_host_name(host_id) + sys_id = self._get_system_id(fullname) + if not self._check_and_set_system_impi(sys_id): + return + + self.remote.power_system(sys_id, self.token, power='reboot') + logging.info("Host with ID=%d starts to reboot!" % host_id) + + +class CobblerConfigManager(BaseConfigManager): + + def __init__(self, adapter_info, cluster_info, hosts_info): + super(CobblerConfigManager, self).__init__(adapter_info, + cluster_info, + hosts_info) + + def get_host_ipmi_info(self, host_id): + host_info = self._get_host_info(host_id) + if not host_info: + return None + + if self.IPMI not in host_info: + return None + + ipmi_info = host_info[const.IPMI] + + if not ipmi_info: + raise Exception('Cannot find IPMI information!') + + ipmi_ip = ipmi_info[const.IP_ADDR] + ipmi_user = ipmi_info[const.IPMI_CREDS][const.USERNAME] + ipmi_pass = ipmi_info[const.IPMI_CREDS][const.PASSWORD] + + return (ipmi_ip, ipmi_user, ipmi_pass) diff --git a/compass/deployment/installers/pk_installers/__init__.py b/compass/deployment/installers/pk_installers/__init__.py new file mode 100644 index 00000000..5e42ae96 --- /dev/null +++ b/compass/deployment/installers/pk_installers/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/deployment/installers/pk_installers/chef_installer/__init__.py b/compass/deployment/installers/pk_installers/chef_installer/__init__.py new file mode 100644 index 00000000..5e42ae96 --- /dev/null +++ b/compass/deployment/installers/pk_installers/chef_installer/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/deployment/installers/pk_installers/chef_installer/chef_installer.py b/compass/deployment/installers/pk_installers/chef_installer/chef_installer.py new file mode 100644 index 00000000..00871b2b --- /dev/null +++ b/compass/deployment/installers/pk_installers/chef_installer/chef_installer.py @@ -0,0 +1,472 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = "Grace Yu (grace.yu@huawei.com)" + +"""package installer: chef plugin.""" +import logging +import os +import shutil + +from compass.deployment.installers.config_manager import BaseConfigManager +from compass.deployment.installers.installer import PKInstaller +from compass.deployment.utils import constants as const +from compass.utils import setting_wrapper as compass_setting +from compass.utils import util + + +NAME = 'ChefInstaller' + + +class ChefInstaller(PKInstaller): + """chef package installer.""" + ENV_TMPL_NAME = 'env.tmpl' + NODE_TMPL_DIR = 'node' + DATABAG_TMPL_DIR = 'databags' + COMMON_NODE_TMPL_NAME = 'node.tmpl' + + def __init__(self, adapter_info, cluster_info, hosts_info): + super(ChefInstaller, self).__init__() + + self.config_manager = ChefConfigManager(adapter_info, cluster_info, + hosts_info) + adapter_name = self.config_manager.get_dist_system_name() + self.tmpl_dir = ChefInstaller.get_tmpl_path(adapter_name) + + self.installer_url = self.config_manager.get_chef_url() + key, client = self.config_manager.get_chef_credentials() + + self.chef_api = self._get_chef_api(key, client) + logging.debug('%s instance created', self) + + @classmethod + def get_tmpl_path(cls, adapter_name): + tmpl_path = os.path.join(os.path.join(compass_setting.TMPL_DIR, + 'chef_installer'), + adapter_name) + return tmpl_path + + def __repr__(self): + return '%s[name=%s,installer_url=%s]' % ( + self.__class__.__name__, self.NAME, self.installer_url) + + def _get_chef_api(self, key=None, client=None): + """Initializes chef API client.""" + import chef + chef_api = None + try: + if key and client: + chef_api = chef.ChefAPI(self.installer_url, key, client) + else: + chef_api = chef.autoconfigure() + + except Exception as ex: + err_msg = "Failed to instantiate chef API, error: %s" % ex.message + raise Exception(err_msg) + + return chef_api + + def get_env_name(self, dist_sys_name, cluster_id): + """Generate environment name.""" + return "-".join((dist_sys_name, cluster_id)) + + def get_databag_name(self): + """Get databag name.""" + return self.config_manager.get_dist_system_name() + + def get_databag(self, databag_name): + """Get databag object from chef server. Creating the databag if its + doesnot exist. + """ + import chef + bag = chef.DataBag(databag_name, api=self.chef_api) + bag.save() + return bag + + def get_node(self, node_name, env_name=None): + """Get chef node if existing, otherwise create one and set its + environment. + + :param str node_name: The name for this node. + :param str env_name: The environment name for this node. + """ + import chef + if not self.chef_api: + logging.info("Cannot find ChefAPI object!") + raise Exception("Cannot find ChefAPI object!") + + node = chef.Node(node_name, api=self.chef_api) + if node not in chef.Node.list(api=self.chef_api): + if env_name: + node.chef_environment = env_name + node.save() + + return node + + def delete_hosts(self): + hosts_id_list = self.config_manager.get_host_id_list() + for host_id in hosts_id_list: + self.delete_node(host_id) + + def delete_node(self, host_id): + fullname = self.config_manager.get_host_name(host_id) + node = self.get_node(fullname) + self._delete_node(node) + + def _delete_node(self, node): + """clean node attributes about target system.""" + import chef + if node is None: + raise Exception("Node is None, cannot delete a none node.") + node_name = node.name + client_name = node_name + + # Clean log for this node first + log_dir_prefix = compass_setting.INSTALLATION_LOGDIR[self.NAME] + self._clean_log(log_dir_prefix, node_name) + + # Delete node and its client on chef server + try: + node.delete() + client = chef.Client(client_name, api=self.chef_api) + client.delete() + logging.debug('delete node %s', node_name) + log_dir_prefix = compass_setting.INSTALLATION_LOGDIR[self.NAME] + self._clean_log(log_dir_prefix, node_name) + except Exception as error: + logging.debug( + 'failed to delete node %s, error: %s', node_name, error) + + def _add_roles(self, node, roles): + """Add roles to the node. + :param object node: The node object. + :param list roles: The list of roles for this node. + """ + if node is None: + raise Exception("Node is None!") + + if not roles: + logging.info("Role list is None. Run list will no change.") + return + + run_list = node.run_list + for role in roles: + if role not in run_list: + node.run_list.append('role[%s]' % role) + + node.save() + logging.debug('Runlist for node %s is %s', node.name, node.run_list) + + def _get_node_attributes(self, roles, vars_dict): + """Get node attributes from templates according to its roles. The + templates are named by roles without '-'. Return the dictionary + of attributes defined in the templates. + + :param list roles: The roles for this node, used to load the + specific template. + :param dict vars_dict: The dictionary used in cheetah searchList to + render attributes from templates. + """ + if not roles: + return {} + + node_tmpl_dir = os.path.join(self.tmpl_dir, self.NODE_TMPL_DIR) + node_attri = {} + for role in roles: + role = role.replace('-', '_') + tmpl_name = '.'.join((role, 'tmpl')) + node_tmpl = os.path.join(node_tmpl_dir, tmpl_name) + node_attri = self.get_config_from_template(node_tmpl, vars_dict) + + return node_attri + + def update_node(self, node, roles, vars_dict): + """Update node attributes to chef server.""" + if node is None: + raise Exception("Node is None!") + + if not roles: + logging.info("The list of roles is None.") + return + + # Add roles to node Rolelist on chef server. + self._add_roles(node, roles) + + # Update node attributes. + node_config = self._get_node_attributes(roles, vars_dict) + for attr in node_config: + setattr(node, attr, node_config[attr]) + + node.save() + + def _get_env_attributes(self, vars_dict): + """Get environment attributes from env templates.""" + + env_tmpl = os.path.join(self.tmpl_dir, self.ENV_TMPL_NAME) + env_attri = self.get_config_from_template(env_tmpl, vars_dict) + return env_attri + + def update_environment(self, env_name, vars_dict): + """Generate environment attributes based on the template file and + upload it to chef server. + + :param str env_name: The environment name. + :param dict vars_dict: The dictionary used in cheetah searchList to + render attributes from templates. + """ + import chef + env_config = self._get_env_attributes(vars_dict) + env = chef.Environment(env_name, api=self.chef_api) + for attr in env_config: + if attr in env.attributes: + setattr(env, attr, env_config[attr]) + env.save() + + def _get_databagitem_attributes(self, tmpl_dir, vars_dict): + databagitem_attri = self.get_config_from_template(tmpl_dir, + vars_dict) + + return databagitem_attri + + def update_databags(self, vars_dict): + """Update datbag item attributes. + + :param dict vars_dict: The dictionary used to get attributes from + templates. + """ + databag_names = self.config_manager.get_chef_databag_names() + if not databag_names: + return + + import chef + databags_dir = os.path.join(self.tmpl_dir, self.DATABAG_TMPL_DIR) + for databag_name in databag_names: + databag = self.get_databag(databag_name) + databag_tmpl = os.paht.join(databags_dir, databag_name) + databagitem_attri = self._get_databagitem_attributes(databag_tmpl, + vars_dict) + + for item, item_values in databagitem_attri.iteritems(): + databagitem = chef.DataBagItem(databag, item, + api=self.chef_api) + for key, value in item_values.iteritems(): + databagitem[key] = value + databagitem.save() + + def _get_host_tmpl_vars(self, host_id, global_vars_dict): + """Get templates variables dictionary for cheetah searchList based + on host package config. + + :param int host_id: The host ID. + :param dict global_vars_dict: The vars_dict got from cluster level + package_config. + """ + vars_dict = {} + if global_vars_dict: + temp = global_vars_dict[const.CLUSTER][const.DEPLOYED_PK_CONFIG] + vars_dict[const.DEPLOYED_PK_CONFIG] = temp + + host_baseinfo = self.config_manager.get_host_baseinfo(host_id) + util.merge_dict(vars_dict, host_baseinfo) + + pk_config = self.config_manager.get_host_package_config(host_id) + if pk_config: + # Get host template variables and merge to vars_dict + metadata = self.config_manager.get_pk_config_meatadata() + host_dict = self.get_tmpl_vars_from_metadata(metadata, pk_config) + util.merge_dict(vars_dict[const.DEPLOYED_PK_CONFIG], host_dict) + + # Set role_mapping for host + mapping = self.config_manager.get_host_roles_mapping(host_id) + vars_dict[const.DEPLOYED_PK_CONFIG][const.ROLES_MAPPING] = mapping + + return {const.HOST: vars_dict} + + def _get_cluster_tmpl_vars(self): + vars_dict = {} + cluster_baseinfo = self.config_manager.get_cluster_baseinfo() + util.merge_dict(vars_dict, cluster_baseinfo) + + pk_metadata = self.config_manager.get_pk_config_meatadata() + pk_config = self.config_manager.get_cluster_package_config() + meta_dict = self.get_tmpl_vars_from_metadata(pk_metadata, pk_config) + vars_dict[const.DEPLOYED_PK_CONFIG] = meta_dict + + mapping = self.config_manager.get_cluster_roles_mapping() + vars_dict[const.DEPLOYED_PK_CONFIG][const.ROLES_MAPPING] = mapping + + return {const.CLUSTER: vars_dict} + + def deploy(self): + """Start to deploy a distributed system. Return both cluster and hosts + deployed configs. The return format: + { + "cluster": { + "id": 1, + "deployed_package_config": {...} + }, + "hosts": { + 1($clusterhost_id): { + "deployed_package_config": {...} + }, + .... + } + } + """ + adapter_name = self.config_manager.get_adapter_name() + cluster_id = self.config_manager.get_cluster_id() + env_name = self.get_env_name(adapter_name, str(cluster_id)) + + global_vars_dict = self._get_cluster_tmpl_vars() + # Update environment + self.update_environment(env_name, global_vars_dict) + + # Update Databag item + self.update_databags(global_vars_dict) + + host_list = self.config_manager.get_host_id_list() + hosts_deployed_configs = {} + + for host_id in host_list: + node_name = self.config_manager.get_host_name(host_id) + roles = self.config_manager.get_host_roles(host_id) + + node = self.get_node(node_name, env_name) + vars_dict = self._get_host_tmpl_vars(host_id, global_vars_dict) + self.update_node(node, roles, vars_dict) + + # set each host deployed config + tmp = self.config_manager.get_host_deployed_package_config(host_id) + tmp[const.TMPL_VARS_DICT] = vars_dict + hosts_deployed_configs[host_id] = tmp + + # set cluster deployed config + cl_config = self.config_manager.get_cluster_deployed_package_config() + cl_config[const.TMPL_VARS_DICT] = global_vars_dict + + return { + const.CLUSTER: { + const.ID: cluster_id, + const.DEPLOYED_PK_CONFIG: cl_config + }, + const.HOSTS: hosts_deployed_configs + } + + def generate_installer_config(self): + """Render chef config file (client.rb) by OS installing right after + OS is installed successfully. + The output format: + { + '1'($host_id/clusterhost_id):{ + 'tool': 'chef', + 'chef_url': 'https://xxx', + 'chef_client_name': '$host_name', + 'chef_node_name': '$host_name' + }, + ..... + } + """ + host_ids = self.config_manager.get_host_id_list() + os_installer_configs = {} + for host_id in host_ids: + fullname = self.config_manager.get_host_name(host_id) + temp = { + "tool": "chef", + "chef_url": self.installer_url + } + temp['chef_client_name'] = fullname + temp['chef_node_name'] = fullname + os_installer_configs[host_id] = temp + + return os_installer_configs + + def clean_progress(self): + """Clean all installing log about the node.""" + log_dir_prefix = compass_setting.INSTALLATION_LOGDIR[self.NAME] + hosts_list = self.config_manager.get_host_id_list() + for host_id in hosts_list: + fullname = self.config_manager.get_host_name() + self._clean_log(log_dir_prefix, fullname) + + def _clean_log(self, log_dir_prefix, node_name): + log_dir = os.path.join(log_dir_prefix, node_name) + shutil.rmtree(log_dir, True) + + def get_supported_dist_systems(self): + """get target systems from chef. All target_systems for compass will + be stored in the databag called "compass". + """ + databag = self.__get_compass_databag() + target_systems = {} + for system_name, item in databag: + target_systems[system_name] = item + + return target_systems + + def _clean_databag_item(self, databag, item_name): + """clean databag item.""" + import chef + if item_name not in chef.DataBagItem.list(api=self.chef_api): + logging.info("Databag item '%s' is not found!", item_name) + return + + bag_item = databag[item_name] + try: + bag_item.delete() + logging.debug('databag item %s is removed from databag', + item_name) + except Exception as error: + logging.debug('Failed to delete item %s from databag! Error: %s', + item_name, error) + databag.save() + + def redeploy(self): + """reinstall host.""" + pass + + +class ChefConfigManager(BaseConfigManager): + TMPL_DIR = 'tmpl_dir' + DATABAGS = "databags" + CHEFSERVER_URL = "chef_url" + KEY_DIR = "key_dir" + CLIENT = "client_name" + + def __init__(self, adapter_info, cluster_info, hosts_info): + super(ChefConfigManager, self).__init__(adapter_info, + cluster_info, + hosts_info) + + def get_chef_url(self): + pk_installer_settings = self.get_pk_installer_settings() + if self.CHEFSERVER_URL not in pk_installer_settings: + raise KeyError("'%s' must be set in package settings!", + self.CHEFSERVER_URL) + + return pk_installer_settings[self.CHEFSERVER_URL] + + def get_chef_credentials(self): + installer_settings = self.get_pk_installer_settings() + key_dir = installer_settings.setdefault(self.KEY_DIR, None) + client = installer_settings.setdefault(self.CLIENT, None) + + return (key_dir, client) + + def get_chef_databag_names(self): + pk_installer_settings = self.get_pk_installer_settings() + if self.DATABAGS in pk_installer_settings: + logging.info("No databags is set!") + return None + + return pk_installer_settings[self.DATABAGS] diff --git a/compass/deployment/utils/__init__.py b/compass/deployment/utils/__init__.py new file mode 100644 index 00000000..cbd36e06 --- /dev/null +++ b/compass/deployment/utils/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = "Grace Yu (grace.yu@huawei.com)" diff --git a/compass/deployment/utils/constants.py b/compass/deployment/utils/constants.py new file mode 100644 index 00000000..c5a85117 --- /dev/null +++ b/compass/deployment/utils/constants.py @@ -0,0 +1,75 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = "Grace Yu (grace.yu@huawei.com)" + + +"""All keywords variables in deployment are defined in this module.""" + + +# General keywords +CLUSTER = 'cluster' +HOST = 'host' +HOSTS = 'hosts' +ID = 'id' +NAME = 'name' +PASSWORD = 'password' +USERNAME = 'username' + + +# Adapter info related keywords +DIST_SYS_NAME = 'distributed_system_name' +INSTALLER_SETTINGS = 'settings' +METADATA = 'metadata' +OS_INSTALLER = 'os_installer' +PK_INSTALLER = 'pk_installer' +SUPPORT_OSES = 'supported_oses' + + +# Cluster info related keywords +ADAPTER_ID = 'adapter_id' +OS_VERSION = 'os_name' + + +# Host info related keywords +DNS = 'dns' +DOMAIN = 'domain' +HOST_ID = 'host_id' +HOSTNAME = 'hostname' +IP_ADDR = 'ip' +IPMI = 'ipmi' +IPMI_CREDS = 'ipmi_credentials' +MAC_ADDR = 'mac' +MGMT_NIC_FLAG = 'is_mgmt' +NETMASK = 'netmask' +NETWORKS = 'networks' +NIC = 'interface' +ORIGIN_CLUSTER_ID = 'origin_cluster_id' +PROMISCUOUS_FLAG = 'is_promiscuous' +REINSTALL_OS_FLAG = 'reinstall_os' +SUBNET = 'subnet' + + +# Cluster/host config related keywords +COMPLETED_PK_CONFIG = 'completed_package_config' +COMPLETED_OS_CONFIG = 'completed_os_config' +DEPLOYED_OS_CONFIG = 'deployed_os_config' +DEPLOYED_PK_CONFIG = 'deployed_package_config' +NETWORK_MAPPING = 'network_mapping' +OS_CONFIG = 'os_config' +OS_CONFIG_GENERAL = 'general' +PK_CONFIG = 'package_config' +ROLES = 'roles' +ROLES_MAPPING = 'roles_mapping' +TMPL_VARS_DICT = 'vars_dict' diff --git a/compass/tasks/tasks.py b/compass/tasks/tasks.py index 9001f12b..b2c6cbbb 100644 --- a/compass/tasks/tasks.py +++ b/compass/tasks/tasks.py @@ -19,11 +19,9 @@ import logging from celery.signals import celeryd_init -from celery.signals import setup_logging from compass.actions import deploy from compass.actions import poll_switch -from compass.actions import reinstall from compass.actions import update_progress from compass.db.api import adapter_holder as adapter_api from compass.db.api import database @@ -78,7 +76,10 @@ def deploy_cluster(deployer_email, cluster_id, clusterhost_ids): :param cluster_hosts: the cluster and hosts of each cluster to deploy. :type cluster_hosts: dict of int to list of int """ - pass + try: + deploy.deploy(cluster_id, clusterhost_ids, deployer_email) + except Exception as error: + logging.exception(error) @celery.task(name='compass.tasks.reinstall_cluster') diff --git a/compass/tests/actions/__init__.py b/compass/tests/actions/__init__.py new file mode 100644 index 00000000..4ee55a4c --- /dev/null +++ b/compass/tests/actions/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/tests/actions/deploy/__init__.py b/compass/tests/actions/deploy/__init__.py new file mode 100644 index 00000000..4ee55a4c --- /dev/null +++ b/compass/tests/actions/deploy/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/tests/actions/deploy/test_deploy.py b/compass/tests/actions/deploy/test_deploy.py new file mode 100644 index 00000000..39166d16 --- /dev/null +++ b/compass/tests/actions/deploy/test_deploy.py @@ -0,0 +1,151 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Test deploy action module.""" + + +from mock import patch +import os +import unittest2 + + +os.environ['COMPASS_IGNORE_SETTING'] = 'true' + + +from compass.actions import deploy +from compass.utils import setting_wrapper as setting +reload(setting) + + +class TestDeployAction(unittest2.TestCase): + """Test deploy moudle functions in actions.""" + def setUp(self): + super(TestDeployAction, self).setUp() + + def tearDown(self): + super(TestDeployAction, self).tearDown() + + @patch('compass.db.api.cluster.get_cluster_metadata') + @patch('compass.db.api.adapter_holder.get_adapter') + def test_get_adatper_info(self, mock_get_adapter, mock_get_cluster_meta): + mock_get_adapter.return_value = { + "id": 1, + "name": "test_adapter", + "roles": [ + { + "name": "test-role-1", + "display_name": "test role 1" + }, + { + "name": "test-role-2", + "display_name": "test role 2" + } + ], + "os_installer": { + "name": "test_os_installer", + "settings": { + "url": "http://127.0.0.1" + } + }, + "pk_installer": { + "name": "test_pk_installer", + "settings": { + "url": "http://127.0.0.1" + } + } + } + mock_get_cluster_meta.return_value = { + "os_config": {}, + "package_config": {} + } + expected_output = { + "id": 1, + "name": "test_adapter", + "roles": ["test-role-1", "test-role-2"], + "os_installer": { + "name": "test_os_installer", + "settings": { + "url": "http://127.0.0.1" + } + }, + "pk_installer": { + "name": "test_pk_installer", + "settings": { + "url": "http://127.0.0.1" + } + }, + "os_config": {}, + "package_config": {} + } + output = deploy.ActionHelper.get_adapter_info(1, 1, None) + self.maxDiff = None + self.assertDictEqual(expected_output, output) + + @patch('compass.db.api.cluster.get_cluster_host_config') + @patch('compass.db.api.host.get_host') + @patch('compass.db.api.cluster.get_clusterhost') + def test_get_hosts_info(self, mock_get_clusterhost, mock_get_host, + mock_get_cluster_host_config): + mock_get_clusterhost.return_value = { + "id": 1, + "host_id": 10, + "name": "test" + } + mock_get_host.return_value = { + "id": 10, + "mac": "00:89:23:a1:e9:10", + "hostname": "server01", + "networks": [ + { + "interface": "eth0", + "ip": "127.0.0.1", + "netmask": "255.255.255.0", + "is_mgmt": True, + "subnet": "127.0.0.0/24", + "is_promiscuous": False + } + ] + } + mock_get_cluster_host_config.return_value = { + "os_config": {}, + "package_config": {}, + "deployed_os_config": {}, + "deployed_package_config": {} + } + expected_output = { + 1: { + "id": 1, + "host_id": 10, + "name": "test", + "mac": "00:89:23:a1:e9:10", + "hostname": "server01", + "networks": { + "eth0": { + "ip": "127.0.0.1", + "netmask": "255.255.255.0", + "is_mgmt": True, + "subnet": "127.0.0.0/24", + "is_promiscuous": False + } + }, + "os_config": {}, + "package_config": {}, + "deployed_os_config": {}, + "deployed_package_config": {} + } + } + output = deploy.ActionHelper.get_hosts_info(1, [1], None) + self.maxDiff = None + self.assertDictEqual(expected_output, output) diff --git a/compass/tests/deployment/__init__.py b/compass/tests/deployment/__init__.py new file mode 100644 index 00000000..5e42ae96 --- /dev/null +++ b/compass/tests/deployment/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/tests/deployment/installers/__init__.py b/compass/tests/deployment/installers/__init__.py new file mode 100644 index 00000000..5e42ae96 --- /dev/null +++ b/compass/tests/deployment/installers/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/tests/deployment/installers/os_installers/__init__.py b/compass/tests/deployment/installers/os_installers/__init__.py new file mode 100644 index 00000000..5e42ae96 --- /dev/null +++ b/compass/tests/deployment/installers/os_installers/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/tests/deployment/installers/os_installers/cobbler/__init__.py b/compass/tests/deployment/installers/os_installers/cobbler/__init__.py new file mode 100644 index 00000000..5e42ae96 --- /dev/null +++ b/compass/tests/deployment/installers/os_installers/cobbler/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/tests/deployment/installers/os_installers/cobbler/test_cobbler.py b/compass/tests/deployment/installers/os_installers/cobbler/test_cobbler.py new file mode 100644 index 00000000..807c5b2b --- /dev/null +++ b/compass/tests/deployment/installers/os_installers/cobbler/test_cobbler.py @@ -0,0 +1,173 @@ +#!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = "Grace Yu (grace.yu@huawei.com)" + + +"""Test cobbler installer module.""" + +from copy import deepcopy +from mock import Mock +import os +import unittest2 + + +os.environ['COMPASS_IGNORE_SETTING'] = 'true' + + +from compass.deployment.installers.os_installers.cobbler.cobbler \ + import CobblerInstaller +from compass.tests.deployment.test_data import config_data +from compass.utils import setting_wrapper as compass_setting +reload(compass_setting) + + +class TestCobblerInstaller(unittest2.TestCase): + """Test CobblerInstaller methods.""" + def setUp(self): + super(TestCobblerInstaller, self).setUp() + self.test_cobbler = self._get_cobbler_installer() + self.expected_host_vars_dict = { + "host": { + "mac": "00:0c:29:3e:60:e9", + "name": "server01.test", + "profile": "Ubuntu-12.04-x86_64", + "hostname": "server01", + "dns": "server01.test.ods.com", + "reinstall_os": True, + "networks": { + "vnet0": { + "ip": "12.234.32.100", + "netmask": "255.255.255.0", + "is_mgmt": True, + "is_promiscuous": False, + "subnet": "12.234.32.0/24" + }, + "vnet1": { + "ip": "172.16.1.1", + "netmask": "255.255.255.0", + "is_mgmt": False, + "is_promiscuous": False, + "subnet": "172.16.1.0/24" + } + }, + "partition": { + "/var": { + "vol_size": 30, + "vol_percentage": 30 + }, + "/home": { + "vol_size": 50, + "vol_percentage": 40 + }, + "/test": { + "vol_size": 10, + "vol_percentage": 10 + } + }, + "server_credentials": { + "username": "root", + "password": "huawei" + }, + "language": "EN", + "timezone": "UTC", + "http_proxy": "http://127.0.0.1:3128", + "https_proxy": "", + "ntp_server": "127.0.0.1", + "nameservers": ["127.0.0.1"], + "search_path": ["1.ods.com", "ods.com"], + "gateway": "10.145.88.1" + } + } + + def tearDown(self): + super(TestCobblerInstaller, self).tearDown() + del self.test_cobbler + + def _get_cobbler_installer(self): + adapter_info = deepcopy(config_data.adapter_test_config) + cluster_info = deepcopy(config_data.cluster_test_config) + hosts_info = deepcopy(config_data.hosts_test_config) + # In config_data, only hosts with ID 1 and 2 needs to install OS. + del hosts_info[3] + + CobblerInstaller._get_cobbler_server = Mock() + CobblerInstaller._get_cobbler_server.return_value = "mock_server" + CobblerInstaller._get_token = Mock() + CobblerInstaller._get_token.return_value = "mock_token" + + CobblerInstaller.get_tmpl_path = Mock() + test_tmpl_dir = os.path.join(config_data.test_tmpl_dir, 'cobbler') + CobblerInstaller.get_tmpl_path.return_value = test_tmpl_dir + return CobblerInstaller(adapter_info, cluster_info, hosts_info) + + def test_get_host_tmpl_vars_dict(self): + host_id = 1 + profile = 'Ubuntu-12.04-x86_64' + global_vars_dict = self.test_cobbler._get_cluster_tmpl_vars_dict() + output = self.test_cobbler._get_host_tmpl_vars_dict(host_id, + global_vars_dict, + profile=profile) + self.maxDiff = None + self.assertDictEqual(self.expected_host_vars_dict, output) + + def test_get_system_config(self): + expected_system_config = { + "name": "server01.test", + "hostname": "server01", + "profile": "Ubuntu-12.04-x86_64", + "gateway": "10.145.88.1", + "name_servers": ["127.0.0.1"], + "name_servers_search": "1.ods.com ods.com", + "proxy": "http://127.0.0.1:3128", + "modify_interface": { + "ipaddress-vnet0": "12.234.32.100", + "netmask-vnet0": "255.255.255.0", + "management-vnet0": True, + "macaddress-vnet0": "00:0c:29:3e:60:e9", + "dns-vnet0": "server01.test.ods.com", + "static-vnet0": True, + "ipaddress-vnet1": "172.16.1.1", + "netmask-vnet1": "255.255.255.0", + "management-vnet1": False, + "static-vnet1": True + }, + "ksmeta": { + "promisc_nics": "", + "timezone": "UTC", + "partition": "/test 10%;/var 30%;/home 40%", + "https_proxy": "", + "ntp_server": "127.0.0.1", + "chef_url": "https://127.0.0.1", + "chef_client_name": "server01.test", + "chef_node_name": "server01.test", + "tool": "chef" + } + } + package_config = { + 1: { + "chef_url": "https://127.0.0.1", + "chef_client_name": "server01.test", + "chef_node_name": "server01.test", + "tool": "chef" + } + } + host_id = 1 + self.test_cobbler.set_package_installer_config(package_config) + output = self.test_cobbler._get_system_config( + host_id, self.expected_host_vars_dict) + self.maxDiff = None + self.assertEqual(expected_system_config, output) diff --git a/compass/tests/deployment/installers/pk_installers/__init__.py b/compass/tests/deployment/installers/pk_installers/__init__.py new file mode 100644 index 00000000..5e42ae96 --- /dev/null +++ b/compass/tests/deployment/installers/pk_installers/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/tests/deployment/installers/pk_installers/chef_installer/__init__.py b/compass/tests/deployment/installers/pk_installers/chef_installer/__init__.py new file mode 100644 index 00000000..5e42ae96 --- /dev/null +++ b/compass/tests/deployment/installers/pk_installers/chef_installer/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/tests/deployment/installers/pk_installers/chef_installer/test_chef.py b/compass/tests/deployment/installers/pk_installers/chef_installer/test_chef.py new file mode 100644 index 00000000..695058da --- /dev/null +++ b/compass/tests/deployment/installers/pk_installers/chef_installer/test_chef.py @@ -0,0 +1,131 @@ +#!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = "Grace Yu (grace.yu@huawei.com)" + + +"""Test Chef installer module. +""" + +from copy import deepcopy +from mock import Mock +import os +import unittest2 + + +os.environ['COMPASS_IGNORE_SETTING'] = 'true' + +from compass.tests.deployment.test_data import config_data +from compass.utils import setting_wrapper as compass_setting +reload(compass_setting) + + +from compass.deployment.installers.pk_installers.chef_installer.chef_installer\ + import ChefInstaller + + +class TestChefInstaller(unittest2.TestCase): + """Test installer functionality.""" + def setUp(self): + super(TestChefInstaller, self).setUp() + self.test_chef = self._get_chef_installer() + + def tearDown(self): + super(TestChefInstaller, self).tearDown() + + def _get_chef_installer(self): + adapter_info = deepcopy(config_data.adapter_test_config) + cluster_info = deepcopy(config_data.cluster_test_config) + hosts_info = deepcopy(config_data.hosts_test_config) + + ChefInstaller.get_tmpl_path = Mock() + test_tmpl_dir = os.path.join(os.path.join(config_data.test_tmpl_dir, + 'chef_installer'), + 'openstack_icehouse') + ChefInstaller.get_tmpl_path.return_value = test_tmpl_dir + + ChefInstaller._get_chef_api = Mock() + ChefInstaller._get_chef_api.return_value = 'mock_server' + chef_installer = ChefInstaller(adapter_info, cluster_info, hosts_info) + return chef_installer + + def test_get_tmpl_vars(self): + pass + + def test_get_node_attributes(self): + cluster_dict = self.test_chef._get_cluster_tmpl_vars() + vars_dict = self.test_chef._get_host_tmpl_vars(2, cluster_dict) + expected_node_attr = { + "override_attributes": { + "endpoints": { + "compute-vnc-bind": { + "host": "12.234.32.101" + } + } + } + } + output = self.test_chef._get_node_attributes(['os-compute'], vars_dict) + self.maxDiff = None + self.assertDictEqual(expected_node_attr, output) + + def test_get_env_attributes(self): + expected_env = { + "name": "testing", + "description": "Environment", + "cookbook_versions": { + }, + "json_class": "Chef::Environment", + "chef_type": "environment", + "default_attributes": { + }, + "override_attributes": { + "compute": { + "syslog": { + "use": False + }, + "libvirt": { + "bind_interface": "eth0" + }, + "novnc_proxy": { + "bind_interface": "vnet0" + }, + "xvpvnc_proxy": { + "bind_interface": "eth0" + } + }, + "db": { + "bind_interface": "vnet0", + "compute": { + "host": "12.234.32.100" + }, + "identity": { + "host": "12.234.32.100" + } + }, + "mq": { + "user": "guest", + "password": "test", + "vhost": "/nova", + "network": { + "service_type": "rabbitmq" + } + } + } + } + vars_dict = self.test_chef._get_cluster_tmpl_vars() + output = self.test_chef._get_env_attributes(vars_dict) + self.maxDiff = None + self.assertDictEqual(expected_env, output) diff --git a/compass/tests/deployment/installers/test_config_manager.py b/compass/tests/deployment/installers/test_config_manager.py new file mode 100644 index 00000000..5e42ae96 --- /dev/null +++ b/compass/tests/deployment/installers/test_config_manager.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/tests/deployment/installers/test_installer.py b/compass/tests/deployment/installers/test_installer.py new file mode 100644 index 00000000..5e42ae96 --- /dev/null +++ b/compass/tests/deployment/installers/test_installer.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/tests/deployment/test_data/__init__.py b/compass/tests/deployment/test_data/__init__.py new file mode 100644 index 00000000..5e42ae96 --- /dev/null +++ b/compass/tests/deployment/test_data/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/tests/deployment/test_data/config_data.py b/compass/tests/deployment/test_data/config_data.py new file mode 100644 index 00000000..f4766787 --- /dev/null +++ b/compass/tests/deployment/test_data/config_data.py @@ -0,0 +1,366 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = "Grace Yu (grace.yu@huawei.com)" + +import os + + +curr_dir = os.path.dirname(os.path.realpath(__file__)) +test_tmpl_dir = os.path.join(curr_dir, 'templates') + + +adapter_test_config = { + "name": "openstack_icehouse", + "distributed_system_name": "openstack_icehouse", + "roles": ["os-controller", "os-compute-worker", "os-network"], + "os_installer": { + "name": "cobbler", + "settings": { + "cobbler_url": "http://127.0.0.1/cobbler_api", + "credentials": { + "username": "cobbler", + "password": "cobbler" + } + } + }, + "pk_installer": { + "name": "chef_installer", + "settings": { + "chef_url": "https://127.0.0.1", + "key_dir": "xxx", + "client_name": "xxx" + } + }, + "metadata": { + "os_config": { + "_self": {}, + "general": { + "_self": {"mapping_to": ""}, + "language": { + "_self": { + "mapping_to": "language", + }, + }, + "timezone": { + "_self": { + "mapping_to": "timezone" + } + }, + "default_gateway": { + "_self": { + "mapping_to": "gateway" + } + }, + "domain": { + "_self": {"mapping_to": ""} + }, + "http_proxy": { + "_self": { + "mapping_to": "http_proxy" + } + }, + "ntp_server": { + "_self": {"mapping_to": "ntp_server"} + }, + "dns_servers": { + "_self": {"mapping_to": "nameservers"} + }, + "search_path": { + "_self": {"mapping_to": "search_path"} + }, + "https_proxy": { + "_self": {"mapping_to": "https_proxy"} + } + }, + "partition": { + "_self": { + "mapping_to": "partition" + }, + "$path": { + "_self": {"mapping_to": ""}, + "max_size": { + "_self": {"mapping_to": "vol_size"} + }, + "size_percentage": { + "_self": {"mapping_to": "vol_percentage"} + } + } + }, + "server_credentials": { + "_self": { + "mapping_to": "server_credentials" + }, + "username": { + "_self": {"mapping_to": "username"} + }, + "password": { + "_self": {"mapping_to": "password"} + } + } + }, + "package_config": { + "_self": {}, + "security": { + "_self": {}, + "service_credentials": { + "_self": { + "mapping_to": "service_credentials" + }, + "rabbit_mq": { + "_self": { + "mapping_to": "mq" + }, + "username": { + "_self": { + "mapping_to": "username" + } + }, + "password": { + "_self": { + "mapping_to": "password" + } + } + } + } + }, + "network_mapping": { + "_self": {}, + "management": { + "_self": {}, + "interface": { + "_self": {} + } + }, + "public": { + "_self": {}, + "interface": { + "_self": {} + } + }, + "tenant": { + "_self": {}, + "interface": { + "_self": {} + } + } + }, + "roles": { + "_self": {} + } + } + } +} + + +cluster_test_config = { + "id": 1, + "os_name": "Ubuntu-12.04-x86_64", + "name": "test", + "os_config": { + "general": { + "language": "EN", + "timezone": "UTC", + "default_gateway": "12.234.32.1", + "domain": "ods.com", + "http_proxy": "http://127.0.0.1:3128", + "https_proxy": "", + "ntp_server": "127.0.0.1", + "dns_servers": ["127.0.0.1"], + "search_path": ["1.ods.com", "ods.com"] + }, + "partition": { + "/var": { + "max_size": 20, + "size_percentage": 20 + }, + "/home": { + "max_size": 50, + "size_percentage": 40 + } + }, + "server_credentials": { + "username": "root", + "password": "huawei" + } + }, + "package_config": { + "security": { + "service_credentials": { + "rabbit_mq": { + "username": "guest", + "password": "test" + } + } + }, + "network_mapping": { + "management": { + "interface": "eth0" + }, + "public": { + "interface": "eth2" + }, + "tenant": { + "interface": "eth1" + } + } + } +} + +hosts_test_config = { + 1: { + "host_id": 1, + "reinstall_os": True, + "mac": "00:0c:29:3e:60:e9", + "name": "server01.test", + "hostname": "server01", + "networks": { + "vnet0": { + "ip": "12.234.32.100", + "netmask": "255.255.255.0", + "is_mgmt": True, + "is_promiscuous": False, + "subnet": "12.234.32.0/24" + }, + "vnet1": { + "ip": "172.16.1.1", + "netmask": "255.255.255.0", + "is_mgmt": False, + "is_promiscuous": False, + "subnet": "172.16.1.0/24" + } + }, + "os_config": { + "general": { + "default_gateway": "10.145.88.1", + }, + "partition": { + "/var": { + "max_size": 30, + "size_percentage": 30 + }, + "/test": { + "max_size": 10, + "size_percentage": 10 + } + } + }, + "package_config": { + "network_mapping": { + "management": { + "interface": "vnet0" + }, + "tenant": { + "interface": "vnet1" + } + }, + "roles": ["os-controller"] + } + }, + 2: { + "host_id": 2, + "reinstall_os": True, + "mac": "00:0c:29:3e:60:a1", + "name": "server02.test", + "hostname": "server02", + "networks": { + "eth0": { + "ip": "12.234.32.101", + "netmask": "255.255.255.0", + "is_mgmt": True, + "is_promiscuous": False, + "subnet": "12.234.32.0/24" + }, + "eth1": { + "ip": "172.16.1.2", + "netmask": "255.255.255.0", + "is_mgmt": False, + "is_promiscuous": False, + "subnet": "172.16.1.0/24" + } + }, + "os_config": { + "general": { + "language": "EN", + "timezone": "UTC", + "domain": "ods.com" + }, + "partition": { + "/test": { + "max_size": 10, + "size_percentage": 20 + } + } + }, + "package_config": { + "roles": ["os-compute"] + } + }, + 3: { + "host_id": 10, + "reinstall_os": False, + "mac_address": "00:0c:29:3e:60:a2", + "name": "server03.test", + "hostname": "server03", + "networks": { + "eth0": { + "ip": "12.234.32.103", + "netmask": "255.255.255.0", + "is_mgmt": True, + "is_promiscuous": False, + "subnet": "12.234.32.0/24" + }, + "eth1": { + "ip": "172.16.1.3", + "netmask": "255.255.255.0", + "is_mgmt": False, + "is_promiscuous": False, + "subnet": "172.16.1.0/24" + }, + "eth2": { + "ip": "10.0.0.1", + "netmask": "255.255.255.0", + "is_mgmt": False, + "is_promiscuous": True, + "subnet": "10.0.0.0/24" + } + }, + "os_config": { + "general": { + "language": "EN", + "timezone": "UTC", + "default_gateway": "12.234.32.1", + "domain": "ods.com", + "http_proxy": "http://10.145.88.211:3128", + "https_proxy": "", + "ntp_server": "10.145.88.211", + "dns_servers": "10.145.88.211", + "search_path": "1.ods.com ods.com" + }, + "partition": { + "/var": { + "max_size": 20, + "size_percentage": 20 + }, + "/home": { + "max_size": 50, + "size_percentage": 40 + } + } + }, + "package_config": { + "roles": ["os-network"] + } + } +} diff --git a/compass/tests/deployment/test_data/templates/chef_installer/openstack_icehouse/env.tmpl b/compass/tests/deployment/test_data/templates/chef_installer/openstack_icehouse/env.tmpl new file mode 100644 index 00000000..f59f686d --- /dev/null +++ b/compass/tests/deployment/test_data/templates/chef_installer/openstack_icehouse/env.tmpl @@ -0,0 +1,46 @@ +#set config = $cluster.deployed_package_config +#set mappings = $config.roles_mapping +#set credentials = $config.service_credentials +{ + "name": "testing", + "description": "Environment", + "cookbook_versions": { + }, + "json_class": "Chef::Environment", + "chef_type": "environment", + "default_attributes": { + }, + "override_attributes": { + "compute": { + "syslog": { + "use": false + }, + "libvirt": { + "bind_interface": "$mappings.os_compute.management.interface" + }, + "novnc_proxy": { + "bind_interface": "$mappings.os_controller.management.interface" + }, + "xvpvnc_proxy": { + "bind_interface": "eth0" + } + }, + "db": { + "bind_interface": "$mappings.os_controller.management.interface", + "compute": { + "host": "$mappings.os_controller.management.ip" + }, + "identity": { + "host": "$mappings.os_controller.management.ip" + } + }, + "mq": { + "user": "$credentials.mq.username", + "password": "$credentials.mq.password", + "vhost": "/nova", + "network": { + "service_type": "rabbitmq" + } + } + } +} diff --git a/compass/tests/deployment/test_data/templates/chef_installer/openstack_icehouse/node/os_compute.tmpl b/compass/tests/deployment/test_data/templates/chef_installer/openstack_icehouse/node/os_compute.tmpl new file mode 100644 index 00000000..c59a730a --- /dev/null +++ b/compass/tests/deployment/test_data/templates/chef_installer/openstack_icehouse/node/os_compute.tmpl @@ -0,0 +1,9 @@ +{ + "override_attributes": { + "endpoints": { + "compute-vnc-bind": { + "host":"$host.deployed_package_config.roles_mapping.os_compute.management.ip" + } + } + } +} diff --git a/compass/tests/deployment/test_data/templates/cobbler/Ubuntu-12.04-x86_64/system.tmpl b/compass/tests/deployment/test_data/templates/cobbler/Ubuntu-12.04-x86_64/system.tmpl new file mode 100644 index 00000000..694874b5 --- /dev/null +++ b/compass/tests/deployment/test_data/templates/cobbler/Ubuntu-12.04-x86_64/system.tmpl @@ -0,0 +1,57 @@ +{ + "name": "$host.name", + "hostname": "$host.hostname", + "profile": "$host.profile", + "gateway": "$host.gateway", + #import simplejson as json + #set nameservers = json.dumps($host.nameservers, encoding='utf-8') + "name_servers": $nameservers, + #set search_path = ' '.join($host.search_path) + "name_servers_search": "$search_path", + "proxy": "$host.http_proxy", + "modify_interface": + #set networks = $host.networks + #set rekeyed = {} + #set promicnic = "" + #for $nic, $val in $networks.iteritems(): + #set ip_key = '-'.join(('ipaddress', $nic)) + #set netmask_key = '-'.join(('netmask', $nic)) + #set mgmt_key = '-'.join(('management', $nic)) + #set static_key = '-'.join(('static', $nic)) + #set $rekeyed[$ip_key] = $val.ip + #set $rekeyed[$netmask_key] = $val.netmask + #set $rekeyed[$mgmt_key] = $val.is_mgmt + #set $rekeyed[$static_key] = True + + #if $val.is_promiscuous: + #set promicnic = $nic + #end if + #if $val.is_mgmt: + #set mac_key = '-'.join(('macaddress', $nic)) + #set dns_key = '-'.join(('dns', $nic)) + #set $rekeyed[$mac_key] = $host.mac + #set $rekeyed[$dns_key] = $host.dns + #end if + #end for + #set nic_info = json.dumps($rekeyed, encoding='utf-8') + $nic_info + , + "ksmeta":{ + #set partition_config = '' + #for k, v in $host.partition.iteritems(): + #set path = '' + #if v['vol_percentage']: + #set $path = k + ' ' + str(v['vol_percentage']) + '%' + #else: + #set $path = k + ' ' + str(v['vol_size']) + #end if + #set partition_config = ';'.join((partition_config, $path)) + #end for + #set partition_config = partition_config[1:] + "promisc_nics": "$promicnic", + "partition": "$partition_config", + "https_proxy": "$host.https_proxy", + "ntp_server": "$host.ntp_server", + "timezone": "$host.timezone" + } +} diff --git a/compass/tests/deployment/test_deploy_manager.py b/compass/tests/deployment/test_deploy_manager.py new file mode 100644 index 00000000..325e6a36 --- /dev/null +++ b/compass/tests/deployment/test_deploy_manager.py @@ -0,0 +1,62 @@ +#!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = "Grace Yu (grace.yu@huawei.com)" + + +"""Test deploy_manager module.""" + +from mock import Mock +import os +import unittest2 + + +os.environ['COMPASS_IGNORE_SETTING'] = 'true' + + +from compass.utils import setting_wrapper as setting +from copy import deepcopy +reload(setting) + + +from compass.deployment.deploy_manager import DeployManager +from compass.tests.deployment.test_data import config_data + + +class TestDeployManager(unittest2.TestCase): + """Test DeployManager methods.""" + def setUp(self): + super(TestDeployManager, self).setUp() + + def tearDown(self): + super(TestDeployManager, self).tearDown() + + def test_init_DeployManager(self): + adapter_info = deepcopy(config_data.adapter_test_config) + cluster_info = deepcopy(config_data.cluster_test_config) + hosts_info = deepcopy(config_data.hosts_test_config) + + DeployManager._get_installer = Mock() + DeployManager._get_installer.return_value = "mock_installer" + + test_manager = DeployManager(adapter_info, cluster_info, hosts_info) + self.assertIsNotNone(test_manager) + + # Test hepler function _get_hosts_for_os_installation return correct + # number of hosts config for os deployment. In config_data, two out of + # three hosts need to install OS. + hosts_list = test_manager._get_hosts_for_os_installation(hosts_info) + self.assertEqual(2, len(hosts_list)) diff --git a/compass/utils/setting_wrapper.py b/compass/utils/setting_wrapper.py index 3e382656..5de5c1a2 100644 --- a/compass/utils/setting_wrapper.py +++ b/compass/utils/setting_wrapper.py @@ -70,6 +70,7 @@ OS_FIELD_DIR = '/etc/compass/os_field' PACKAGE_FIELD_DIR = '/etc/compass/package_field' ADAPTER_ROLE_DIR = '/etc/compass/role' VALIDATOR_DIR = '/etc/compass/validator' +TMPL_DIR = '/etc/compass/templates' if ( 'COMPASS_IGNORE_SETTING' in os.environ and os.environ['COMPASS_IGNORE_SETTING'] diff --git a/conf/adapter/chef_openstack.conf b/conf/adapter/chef_openstack.conf index 3a32af04..b523815c 100644 --- a/conf/adapter/chef_openstack.conf +++ b/conf/adapter/chef_openstack.conf @@ -1,7 +1,7 @@ -NAME = 'openstack(chef)' -DISPLAY_NAME = 'openstack(chef)' +NAME = 'openstack_icehouse' +DISPLAY_NAME = 'OpenStack Icehouse' PARENT = 'openstack' -PACKAGE_INSTALLER = 'chef(icehouse)' +PACKAGE_INSTALLER = 'chef_installer' OS_INSTALLER = 'cobbler' SUPPORTED_OS_PATTERNS = ['(?i)centos.*', '(?i)ubuntu.*'] DEPLOYABLE = True diff --git a/conf/os_installer/cobbler.conf b/conf/os_installer/cobbler.conf index 92b0f795..240c490f 100644 --- a/conf/os_installer/cobbler.conf +++ b/conf/os_installer/cobbler.conf @@ -1,6 +1,9 @@ NAME = 'cobbler' INSTANCE_NAME = 'cobbler' SETTINGS = { - 'url': 'http://127.0.0.1/cobbler_api', - 'token': ('cobbler', 'cobbler') + 'cobbler_url': 'http://127.0.0.1/cobbler_api', + 'credentials': { + 'username': 'cobbler', + 'password': 'cobbler' + } } diff --git a/conf/os_metadata/general.conf b/conf/os_metadata/general.conf index 20cddb8e..b0939a07 100644 --- a/conf/os_metadata/general.conf +++ b/conf/os_metadata/general.conf @@ -9,12 +9,13 @@ METADATA = { 'field': 'general', 'default_value': 'EN', 'options': ['EN', 'CN'], + 'mapping_to': 'language' } }, 'timezone': { '_self': { 'field': 'general', - 'default_value': 'GMT -8:00', + 'default_value': 'UTC', 'options': [ 'GMT -12:00', 'GMT -11:00', 'GMT -10:00', 'GMT -9:00', 'GMT -8:00', 'GMT -7:00', 'GMT -6:00', 'GMT -5:00', @@ -22,8 +23,9 @@ METADATA = { 'GMT 0:00', 'GMT +1:00', 'GMT +2:00', 'GMT +3:00', 'GMT +4:00', 'GMT +5:00', 'GMT +6:00', 'GMT +7:00', 'GMT +8:00', 'GMT +9:00', 'GMT +10:00', 'GMT +11:00', - 'GMT +12:00' + 'GMT +12:00', UTC ], + 'mapping_to': 'timezone' } }, 'http_proxy': { @@ -33,6 +35,7 @@ METADATA = { 'options': [ 'http://10.145.88.211:3128' ], + 'mapping_to': 'http_proxy' } }, 'https_proxy': { @@ -42,6 +45,7 @@ METADATA = { 'options': [ 'http://10.145.88.211:3128' ], + 'mapping_to': 'https_proxy' } }, 'no_proxy': { @@ -56,7 +60,8 @@ METADATA = { '127.0.0.1', 'compass', '10.145.88.211' - ] + ], + 'mapping_to': 'no_proxy' } }, 'ntp_server': { @@ -66,7 +71,8 @@ METADATA = { 'default_value': '10.145.88.211', 'options': [ '10.145.88.211' - ] + ], + 'mapping_to': 'ntp_server' } }, 'dns_servers': { @@ -78,7 +84,8 @@ METADATA = { ], 'options': [ '10.145.88.211' - ] + ], + 'mapping_to': 'nameservers' } }, 'domain': { @@ -95,7 +102,8 @@ METADATA = { 'default_value': [ 'ods.com' ], - 'options': ['ods.com'] + 'options': ['ods.com'], + 'mapping_to': 'search_path' } }, 'default_gateway': { @@ -103,23 +111,27 @@ METADATA = { 'is_required': True, 'field': 'ip', 'default_value': '10.145.88.1', + 'mapping_to': 'gateway' } } }, 'server_credentials': { '_self': { 'required_in_whole_config': True, + 'mapping_to': 'server_credentials' }, 'username': { '_self': { 'is_required': True, 'field': 'username', + 'mapping_to': 'username' } }, 'password': { '_self': { 'is_required': True, - 'field': 'password' + 'field': 'password', + 'mapping_to': 'password' } } }, @@ -127,6 +139,7 @@ METADATA = { '_self': { 'required_in_whole_config': True, 'options': ['/boot', 'swap', '/var', '/home'], + 'mapping_to': 'partition' }, '$partition': { '_self': { @@ -134,17 +147,20 @@ METADATA = { }, 'max_size': { '_self': { - 'field': 'size' + 'field': 'size', + 'mapping_to': 'max_vol_size' }, }, 'percentage': { '_self': { 'field': 'percentage', + 'mapping_to': 'vol_percentage' } }, 'size': { '_self': { - 'field': 'size' + 'field': 'size', + 'mapping_to': 'vol_size' }, } } diff --git a/conf/package_installer/chef-icehouse.conf b/conf/package_installer/chef-icehouse.conf index c932f8dd..e72b6c13 100644 --- a/conf/package_installer/chef-icehouse.conf +++ b/conf/package_installer/chef-icehouse.conf @@ -1,5 +1,7 @@ -NAME = 'chef' -INSTANCE_NAME = 'chef(icehouse)' +NAME = 'chef_installer' +INSTANCE_NAME = 'chef_installer' SETTINGS = { - 'url': 'https://127.0.0.1' + 'chef_url': 'https://10.145.88.150', + 'key_dir': '/root/grace.pem', + 'client_name': 'grace' } diff --git a/conf/package_metadata/openstack.conf b/conf/package_metadata/openstack.conf index c27246e2..362a7aba 100644 --- a/conf/package_metadata/openstack.conf +++ b/conf/package_metadata/openstack.conf @@ -4,18 +4,41 @@ METADATA = { '_self': { 'required_in_whole_config': True, }, - '$credential_type': { - '$credential': { + 'service_credentials': { + '_self': { + 'mapping_to': 'service_credentials' + }, + '$service': { 'username': { '_self': { 'is_required': True, 'field': 'username', + 'mapping_to': 'username' } }, 'password': { '_self': { 'is_required': True, - 'field': 'password' + 'field': 'password', + 'mapping_to': 'password' + } + } + } + }, + 'console_credentials': { + '$console': { + 'username': { + '_self': { + 'is_required': True, + 'field': 'username', + 'mapping_to': 'username' + } + }, + 'password': { + '_self': { + 'is_required': True, + 'field': 'password', + 'mapping_to': 'password' } } } diff --git a/conf/setting b/conf/setting index dc3f4213..2e34eacf 100644 --- a/conf/setting +++ b/conf/setting @@ -26,3 +26,4 @@ PROGRESS_UPDATE_INTERVAL=30 POLLSWITCH_INTERVAL=60 SWITCHES = [ ] +TMPL_DIR = '/etc/compass/templates' diff --git a/requirements.txt b/requirements.txt index f2a9e631..9c339c2a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ flask-restful flask-sqlalchemy flask-login celery +cheetah netaddr paramiko==1.7.5 simplejson diff --git a/templates/chef_installer/openstack_icehouse/databags/db_passwords.tmpl b/templates/chef_installer/openstack_icehouse/databags/db_passwords.tmpl new file mode 100644 index 00000000..38acc5f4 --- /dev/null +++ b/templates/chef_installer/openstack_icehouse/databags/db_passwords.tmpl @@ -0,0 +1,11 @@ +{ + "nova": { + "nova": "mypass" + }, + "horizon": { + "horizon": "horizon" + }, + "keystone": { + + } +} diff --git a/templates/chef_installer/openstack_icehouse/databags/secrets.tmpl b/templates/chef_installer/openstack_icehouse/databags/secrets.tmpl new file mode 100644 index 00000000..e69de29b diff --git a/templates/chef_installer/openstack_icehouse/databags/service_passwords.tmpl b/templates/chef_installer/openstack_icehouse/databags/service_passwords.tmpl new file mode 100644 index 00000000..e69de29b diff --git a/templates/chef_installer/openstack_icehouse/databags/user_passwords.tmpl b/templates/chef_installer/openstack_icehouse/databags/user_passwords.tmpl new file mode 100644 index 00000000..e69de29b diff --git a/templates/chef_installer/openstack_icehouse/env.tmpl b/templates/chef_installer/openstack_icehouse/env.tmpl new file mode 100644 index 00000000..c1ba235c --- /dev/null +++ b/templates/chef_installer/openstack_icehouse/env.tmpl @@ -0,0 +1,317 @@ +#set config = $cluster.deployed_package_config +#set mappings = $config.roles_mapping +#set credentials = $config.service_credentials +#set allinone_compute_mgmt_nic = $mappings.allinone_compute.management.interface +#set allinone_compute_mgmt_ip = $mappings.allinone_compute.management.ip +#set allinone_compute_mgmt_nic = $mappings.allinone_compute.management.interface +{ + "name": "testing", + "description": "Environment used in testing the upstream cookbooks and reference Chef repository", + "cookbook_versions": { + }, + "json_class": "Chef::Environment", + "chef_type": "environment", + "default_attributes": { + }, + "override_attributes": { + "mysql": { + "server_root_password": "test", + "server_debian_password": "root", + "server_repl_password": "root", + "allow_remote_root": true, + "root_network_acl": "%" + }, + "openstack": { + "auth": { + "validate_certs": false + }, + "block-storage": { + "syslog": { + "use": false + }, + "api": { + "ratelimit": "False" + }, + "debug": true, + "image_api_chef_role": "os-image", + "identity_service_chef_role": "os-identity", + "rabbit_server_chef_role": "os-ops-messaging" + }, + "dashboard": { + "use_ssl": "false" + }, + "compute": { + "syslog": { + "use": false + }, + "libvirt": { + "bind_interface": "$allinone_compute_mgmt_nic" + }, + "novnc_proxy": { + "bind_interface": "$allinone_compute_mgmt_nic" + }, + "xvpvnc_proxy": { + "bind_interface": "$allinone_compute_mgmt_nic" + }, + "image_api_chef_role": "os-image", + "identity_service_chef_role": "os-identity", + "nova_setup_chef_role": "os-compute-api", + "rabbit_server_chef_role": "os-ops-messaging", + "ratelimit": { + "api": { + "enabled": false + }, + "volume": { + "enabled": false + } + }, + "network": { + "service_type": "neutron", + "fixed_range": "10.0.0.0/8" + } + }, + "network": { + "verbose": "True", + "debug": "True", + "service_plugins": [ + "router" + ], + "ml2": { + "type_drivers": "local,flat,vlan,gre", + "tenant_network_types": "vlan,gre", + "network_vlan_ranges": "physnet1:1000:2999", + "enable_security_group": "True" + }, + "openvswitch": { + "tenant_network_type": "vlan", + "network_vlan_ranges": "physnet1:1000:2999", + "bridge_mappings": "physnet1:br-eth1", + "fw_driver": "neutron.agent.firewall.NoopFirewallDriver" + } + }, + "db": { + "bind_interface": "$allinone_compute_mgmt_nic", + "compute": { + "host": "$allinone_compute_mgmt_ip" + }, + "identity": { + "host": "$allinone_compute_mgmt_ip" + }, + "image": { + "host": "$allinone_compute_mgmt_ip" + }, + "network": { + "host": "$allinone_compute_mgmt_ip" + }, + "volume": { + "host": "$allinone_compute_mgmt_ip" + }, + "dashboard": { + "host": "$allinone_compute_mgmt_ip" + }, + "telemetry": { + "host": "$allinone_compute_mgmt_ip" + }, + "orchestration": { + "host": "$allinone_compute_mgmt_ip" + } + }, + "developer_mode": true, + "endpoints": { + "db": { + "host": "$allinone_compute_mgmt_ip" + }, + "mq": { + "host": "$allinone_compute_mgmt_ip" + }, + "compute-api": { + "host": "$allinone_compute_mgmt_ip", + "scheme": "http", + "port": "8774", + "path": "/v2/%(tenant_id)s" + }, + "compute-api-bind": { + "bind_interface": "$allinone_compute_mgmt_nic" + }, + "compute-ec2-admin": { + "host": "$allinone_compute_mgmt_ip", + "scheme": "http", + "port": "8773", + "path": "/services/Admin" + }, + "compute-ec2-api": { + "host": "$allinone_compute_mgmt_ip", + "scheme": "http", + "port": "8773", + "path": "/services/Cloud" + }, + "compute-xvpvnc": { + "host": "$allinone_compute_mgmt_ip", + "scheme": "http", + "port": "6081", + "path": "/console" + }, + "compute-novnc": { + "host": "$allinone_compute_mgmt_ip", + "scheme": "http", + "port": "6080", + "path": "/vnc_auto.html" + }, + "compute-novnc-bind": { + "bind_interface": "$allinone_compute_mgmt_nic" + }, + "vnc_bind": { + "bind_interface": "$allinone_compute_mgmt_nic" + }, + "image-api": { + "host": "$allinone_compute_mgmt_ip", + "scheme": "http", + "port": "9292", + "path": "/v2" + }, + "image-api-bind": { + "bind_interface": "$allinone_compute_mgmt_nic" + }, + "image-registry": { + "host": "$allinone_compute_mgmt_ip", + "scheme": "http", + "port": "9191", + "path": "/v2" + }, + "image-registry-bind": { + "bind_interface": "$allinone_compute_mgmt_nic" + }, + "identity-bind": { + "bind_interface": "$allinone_compute_mgmt_nic" + }, + "identity-api": { + "host": "$allinone_compute_mgmt_ip", + "scheme": "http", + "port": "5000", + "path": "/v2.0" + }, + "identity-admin": { + "host": "$allinone_compute_mgmt_ip", + "scheme": "http", + "port": "35357", + "path": "/v2.0" + }, + "block-storage-api": { + "host": "$allinone_compute_mgmt_ip", + "scheme": "http", + "port": "8776", + "path": "/v1/%(tenant_id)s" + }, + "block-storage-api-bind": { + "bind_interface": "$allinone_compute_mgmt_nic" + }, + "telemetry-api": { + "host": "$allinone_compute_mgmt_ip", + "scheme": "http", + "port": "8777", + "path": "/v1" + }, + "network-api": { + "host": "$allinone_compute_mgmt_ip", + "scheme": "http", + "port": "9696", + "path": "" + }, + "network-api-bind": { + "bind_interface": "$allinone_compute_mgmt_nic" + }, + "network-openvswitch": { + "bind_interface": "$allinone_compute_mgmt_nic" + }, + "orchestration-api": { + "host": "$allinone_compute_mgmt_ip", + "scheme": "http", + "port": "8004", + "path": "/v1/%(tenant_id)s" + }, + "orchestration-api-cfn": { + "host": "$allinone_compute_mgmt_ip", + "scheme": "http", + "port": "8000", + "path": "/v1" + } + }, + "identity": { + "admin_user": "admin", + "bind_interface": "$allinone_compute_mgmt_nic", + "catalog": { + "backend": "sql" + }, + "debug": true, + "rabbit_server_chef_role": "os-ops-messaging", + "roles": [ + "admin", + "keystone_admin", + "keystone_service_admin", + "member", + "netadmin", + "sysadmin" + ], + "syslog": { + "use": false + }, + "tenants": [ + "admin", + "service", + "demo" + ], + "token": { + "backend": "sql" + }, + "users": { + "admin": { + "password": "admin", + "default_tenant": "admin", + "roles": { + "admin": [ + "admin" + ], + "keystone_admin": [ + "admin" + ], + "keystone_service_admin": [ + "admin" + ] + } + } + } + }, + "image": { + "api": { + "bind_interface": "eth0" + }, + "debug": true, + "identity_service_chef_role": "os-identity", + "rabbit_server_chef_role": "os-ops-messaging", + "registry": { + "bind_interface": "eth0" + }, + "syslog": { + "use": false + }, + "upload_image": { + "cirros": "http://download.cirros-cloud.net/0.3.2/cirros-0.3.2-x86_64-disk.img" + }, + "upload_images": [ + "cirros" + ] + }, + "memcached_servers": [ + "$allinone_compute_mgmt_nic:11211" + ], + "mq": { + "user": "$credentials.rabbitmq.username", + "password": "$credentials.rabbitmq.password", + "vhost": "/nova", + "network": { + "service_type": "rabbitmq" + } + } + } + } +} diff --git a/templates/chef_installer/openstack_icehouse/env.tmpl.multiroles b/templates/chef_installer/openstack_icehouse/env.tmpl.multiroles new file mode 100644 index 00000000..d62ad7c9 --- /dev/null +++ b/templates/chef_installer/openstack_icehouse/env.tmpl.multiroles @@ -0,0 +1,317 @@ +#set config = $cluster.deploy_package_config +#set mappings = $config.roles_mapping +#set credentials = $config.service_credentials +#set compute_mgmt_nic = $mappings.os_compute.management.interface +#set controller_mgmt_ip = $mappings.os_controller.management.ip +#set controller_mgmt_nic = $mappings.os_controller.management.interface +{ + "name": "testing", + "description": "Environment used in testing the upstream cookbooks and reference Chef repository", + "cookbook_versions": { + }, + "json_class": "Chef::Environment", + "chef_type": "environment", + "default_attributes": { + }, + "override_attributes": { + "mysql": { + "server_root_password": "test", + "server_debian_password": "root", + "server_repl_password": "root", + "allow_remote_root": true, + "root_network_acl": "%" + }, + "openstack": { + "auth": { + "validate_certs": false + }, + "block-storage": { + "syslog": { + "use": false + }, + "api": { + "ratelimit": "False" + }, + "debug": true, + "image_api_chef_role": "os-image", + "identity_service_chef_role": "os-identity", + "rabbit_server_chef_role": "os-ops-messaging" + }, + "dashboard": { + "use_ssl": "false" + }, + "compute": { + "syslog": { + "use": false + }, + "libvirt": { + "bind_interface": "$compute_mgmt_nic" + }, + "novnc_proxy": { + "bind_interface": "$controller_mgmt_nic" + }, + "xvpvnc_proxy": { + "bind_interface": "$controller_mgmt_nic" + }, + "image_api_chef_role": "os-image", + "identity_service_chef_role": "os-identity", + "nova_setup_chef_role": "os-compute-api", + "rabbit_server_chef_role": "os-ops-messaging", + "ratelimit": { + "api": { + "enabled": false + }, + "volume": { + "enabled": false + } + }, + "network": { + "service_type": "neutron", + "fixed_range": "10.0.0.0/8" + } + }, + "network": { + "verbose": "True", + "debug": "True", + "service_plugins": [ + "router" + ], + "ml2": { + "type_drivers": "local,flat,vlan,gre", + "tenant_network_types": "vlan,gre", + "network_vlan_ranges": "physnet1:1000:2999", + "enable_security_group": "True" + }, + "openvswitch": { + "tenant_network_type": "vlan", + "network_vlan_ranges": "physnet1:1000:2999", + "bridge_mappings": "physnet1:br-eth1", + "fw_driver": "neutron.agent.firewall.NoopFirewallDriver" + } + }, + "db": { + "bind_interface": "$controller_mgmt_nic", + "compute": { + "host": "$controller_mgmt_ip" + }, + "identity": { + "host": "$controller_mgmt_ip" + }, + "image": { + "host": "$controller_mgmt_ip" + }, + "network": { + "host": "$controller_mgmt_ip" + }, + "volume": { + "host": "$controller_mgmt_ip" + }, + "dashboard": { + "host": "$controller_mgmt_ip" + }, + "telemetry": { + "host": "$controller_mgmt_ip" + }, + "orchestration": { + "host": "$controller_mgmt_ip" + } + }, + "developer_mode": true, + "endpoints": { + "db": { + "host": "$controller_mgmt_ip" + }, + "mq": { + "host": "$controller_mgmt_ip" + }, + "compute-api": { + "host": "$controller_mgmt_ip", + "scheme": "http", + "port": "8774", + "path": "/v2/%(tenant_id)s" + }, + "compute-api-bind": { + "bind_interface": "$controller_mgmt_nic" + }, + "compute-ec2-admin": { + "host": "$controller_mgmt_ip", + "scheme": "http", + "port": "8773", + "path": "/services/Admin" + }, + "compute-ec2-api": { + "host": "$controller_mgmt_ip", + "scheme": "http", + "port": "8773", + "path": "/services/Cloud" + }, + "compute-xvpvnc": { + "host": "$controller_mgmt_ip", + "scheme": "http", + "port": "6081", + "path": "/console" + }, + "compute-novnc": { + "host": "$controller_mgmt_ip", + "scheme": "http", + "port": "6080", + "path": "/vnc_auto.html" + }, + "compute-novnc-bind": { + "bind_interface": "$controller_mgmt_nic" + }, + "vnc_bind": { + "bind_interface": "$controller_mgmt_nic" + }, + "image-api": { + "host": "$controller_mgmt_ip", + "scheme": "http", + "port": "9292", + "path": "/v2" + }, + "image-api-bind": { + "bind_interface": "$controller_mgmt_nic" + }, + "image-registry": { + "host": "$controller_mgmt_ip", + "scheme": "http", + "port": "9191", + "path": "/v2" + }, + "image-registry-bind": { + "bind_interface": "$controller_mgmt_nic" + }, + "identity-bind": { + "bind_interface": "$controller_mgmt_nic" + }, + "identity-api": { + "host": "$controller_mgmt_ip", + "scheme": "http", + "port": "5000", + "path": "/v2.0" + }, + "identity-admin": { + "host": "$controller_mgmt_ip", + "scheme": "http", + "port": "35357", + "path": "/v2.0" + }, + "block-storage-api": { + "host": "$controller_mgmt_ip", + "scheme": "http", + "port": "8776", + "path": "/v1/%(tenant_id)s" + }, + "block-storage-api-bind": { + "bind_interface": "$controller_mgmt_nic" + }, + "telemetry-api": { + "host": "$controller_mgmt_ip", + "scheme": "http", + "port": "8777", + "path": "/v1" + }, + "network-api": { + "host": "$controller_mgmt_ip", + "scheme": "http", + "port": "9696", + "path": "" + }, + "network-api-bind": { + "bind_interface": "$controller_mgmt_nic" + }, + "network-openvswitch": { + "bind_interface": "$controller_mgmt_nic" + }, + "orchestration-api": { + "host": "$controller_mgmt_ip", + "scheme": "http", + "port": "8004", + "path": "/v1/%(tenant_id)s" + }, + "orchestration-api-cfn": { + "host": "$controller_mgmt_ip", + "scheme": "http", + "port": "8000", + "path": "/v1" + } + }, + "identity": { + "admin_user": "admin", + "bind_interface": "$controller_mgmt_nic", + "catalog": { + "backend": "sql" + }, + "debug": true, + "rabbit_server_chef_role": "os-ops-messaging", + "roles": [ + "admin", + "keystone_admin", + "keystone_service_admin", + "member", + "netadmin", + "sysadmin" + ], + "syslog": { + "use": false + }, + "tenants": [ + "admin", + "service", + "demo" + ], + "token": { + "backend": "sql" + }, + "users": { + "admin": { + "password": "admin", + "default_tenant": "admin", + "roles": { + "admin": [ + "admin" + ], + "keystone_admin": [ + "admin" + ], + "keystone_service_admin": [ + "admin" + ] + } + } + } + }, + "image": { + "api": { + "bind_interface": "eth0" + }, + "debug": true, + "identity_service_chef_role": "os-identity", + "rabbit_server_chef_role": "os-ops-messaging", + "registry": { + "bind_interface": "eth0" + }, + "syslog": { + "use": false + }, + "upload_image": { + "cirros": "http://download.cirros-cloud.net/0.3.2/cirros-0.3.2-x86_64-disk.img" + }, + "upload_images": [ + "cirros" + ] + }, + "memcached_servers": [ + "$controller_mgmt_nic:11211" + ], + "mq": { + "user": "$credentials.mq.username", + "password": "$credentials.mq.password", + "vhost": "/nova", + "network": { + "service_type": "rabbitmq" + } + } + } + } +} diff --git a/templates/chef_installer/openstack_icehouse/node/os_compute.tmpl b/templates/chef_installer/openstack_icehouse/node/os_compute.tmpl new file mode 100644 index 00000000..c59a730a --- /dev/null +++ b/templates/chef_installer/openstack_icehouse/node/os_compute.tmpl @@ -0,0 +1,9 @@ +{ + "override_attributes": { + "endpoints": { + "compute-vnc-bind": { + "host":"$host.deployed_package_config.roles_mapping.os_compute.management.ip" + } + } + } +} diff --git a/templates/cobbler/CentOS-6.5-x86_64/system.tmpl b/templates/cobbler/CentOS-6.5-x86_64/system.tmpl new file mode 100644 index 00000000..bb498dc3 --- /dev/null +++ b/templates/cobbler/CentOS-6.5-x86_64/system.tmpl @@ -0,0 +1,62 @@ +{ + "name": "$host.name", + "hostname": "$host.hostname", + "profile": "$host.profile", + "gateway": "$host.gateway", + #import simplejson as json + #set nameservers = json.dumps($host.nameservers, encoding='utf-8') + "name_servers": $nameservers, + #set search_path = ' '.join($host.search_path) + "name_servers_search": "$search_path", + "proxy": "$host.http_proxy", + "modify_interface": + #set networks = $host.networks + #set rekeyed = {} + #set promicnic = "" + #for $nic, $val in $networks.iteritems(): + #set ip_key = '-'.join(('ipaddress', $nic)) + #set netmask_key = '-'.join(('netmask', $nic)) + #set mgmt_key = '-'.join(('management', $nic)) + #set static_key = '-'.join(('static', $nic)) + #set $rekeyed[$ip_key] = $val.ip + #set $rekeyed[$netmask_key] = $val.netmask + #set $rekeyed[$mgmt_key] = $val.is_mgmt + #set $rekeyed[$static_key] = True + + #if $val.is_promiscuous: + #set promicnic = $nic + #end if + #if $val.is_mgmt: + #set mac_key = '-'.join(('macaddress', $nic)) + #set dns_key = '-'.join(('dns', $nic)) + #set $rekeyed[$mac_key] = $host.mac + #set $rekeyed[$dns_key] = $host.dns + #end if + #end for + #set nic_info = json.dumps($rekeyed, encoding='utf-8') + $nic_info + , + "ksmeta":{ + #set partition_config = '' + #for k, v in $host.partition.iteritems(): + #set path = '' + #if v['vol_percentage']: + #set $path = k + ' ' + str(v['vol_percentage']) + '%' + #else: + #set $path = k + ' ' + str(v['vol_size']) + #end if + #set partition_config = ';'.join((partition_config, $path)) + #end for + #set partition_config = partition_config[1:] + #set $password = crypt.crypt($host.server_credentials.password, "az") + #set no_proxy = ','.join($host.no_proxy) + "username": "$host.server_credentials.username", + "password": "$password", + "promisc_nics": "$promicnic", + "partition": "$partition_config", + "https_proxy": "$host.https_proxy", + "ntp_server": "$host.ntp_server", + "timezone": "$host.timezone", + "ignore_proxy": "$no_proxy" + } +} diff --git a/templates/cobbler/Ubuntu-12.04-x86_64/system.tmpl b/templates/cobbler/Ubuntu-12.04-x86_64/system.tmpl new file mode 100644 index 00000000..57fc437d --- /dev/null +++ b/templates/cobbler/Ubuntu-12.04-x86_64/system.tmpl @@ -0,0 +1,36 @@ +{ + "name": "$host.fullname", + "hostname": "$host.hostname", + "profile": "$host.profile", + "gateway": "$host.gateway", + "name_servers": "$host.name_servers", + "name_servers_search": "$host.search_path", + "proxy": "$host.http_proxy", + "modify_interface": + #import simplejson + #set interfaces = $host.networks.interfaces + #set rekeyed = {} + #for $nic, $val in $interfaces.iteritems(): + #set $rekeyed = { "ipaddress-vnet0" : $val.ip, "netmask-vnet0": $val.netmask, "management-vnet0": $val.is_mgmt, "static-vnet0" : True } + #if $val.is_mgmt: + #set $rekeyed["macaddress-vnet0"] = $host.mac_address + #set $rekeyed["dnsname-vnet0"] = $host.dns + #end if + #end for + #set $str=simplejson.dumps($rekeyed, encoding='utf-8') + $str + , + "ksmeta":{ + "username": "$host.server_credentials.username", + #import crypt + #set $password = crypt.crypt($host.server_credentials.password, "az") + "password": "$password", + #set f=[(k + ' ' + v['vol_percentage'] or v['vol_size']) for k,v in $host.partition.iteritems()] + #set $partition_config = '; '.join($f) + "partition": "$partition_config", + "https_proxy": "$host.https_proxy", + "ignore_proxy": "$host.ignore_proxy", + "ntp_server": "$host.ntp_server" + } +} +