Seperate chef installer as a plugin (move chef tests to plugin directory)

Closes-Bug: #1506700

This is the 4th change list of a series of changes that will eventually
convert chef installer as a plugin.
It moves chef installer related tests to plugins directory
extract a method get_plugins_config_files from load_config in util.py,
this function might be used directly, add a unit test for this function

Change-Id: Idbd54ce11ab4489b356b402d176c9df2f688acc2
This commit is contained in:
Carl Li 2015-10-31 23:02:55 -07:00
parent 02d07a8ad1
commit a165bbe6bc
28 changed files with 600 additions and 111 deletions

View File

@ -1,4 +1,4 @@
[DEFAULT]
test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 ${PYTHON:-python} -m subunit.run discover -t ./ ./compass/tests $LISTOPT $IDOPTION
test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 ${PYTHON:-python} -m subunit.run discover -t ./ ./ $LISTOPT $IDOPTION
test_id_option=--load-list $IDFILE
test_list_option=--list

View File

@ -1,13 +0,0 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

View File

@ -24,12 +24,6 @@ reload(compass_setting)
curr_dir = os.path.dirname(os.path.realpath(__file__))
test_tmpl_dir = os.path.join(curr_dir, 'templates')
test_plugins_dir = curr_dir
test_chef_url = compass_setting.TEST_CHEF_URL
test_client_key = compass_setting.TEST_CLIENT_KEY_PATH
test_client = compass_setting.TEST_CLIENT_NAME
adapter_test_config = {
"name": "openstack_icehouse",

View File

@ -0,0 +1,8 @@
NAME = 'openstack_icehouse'
DISPLAY_NAME = 'OpenStack Icehouse'
PARENT = 'openstack'
PACKAGE_INSTALLER = 'chef_installer'
OS_INSTALLER = 'cobbler'
SUPPORTED_OS_PATTERNS = ['(?i)centos.*', '(?i)ubuntu.*', '(?i)sles.*']
DEPLOYABLE = True
HEALTH_CHECK_COMMAND = 'docker run -i -d $cluster_name python /opt/compass/rally/check_health.py'

View File

@ -0,0 +1,7 @@
NAME = 'ceph_firefly'
DISPLAY_NAME = 'Ceph Firefly'
PARENT = 'ceph'
PACKAGE_INSTALLER = 'chef_installer'
OS_INSTALLER = 'cobbler'
SUPPORTED_OS_PATTERNS = ['(?i)centos-6\.5.*', '(?i)ubuntu-12\.04.*']
DEPLOYABLE = True

View File

@ -0,0 +1,23 @@
ADAPTER_NAME = 'openstack_juno'
FLAVORS = [{
'flavor': 'allinone',
'display_name': 'All-In-One',
'template': 'allinone.tmpl',
'roles': ['allinone-compute'],
}, {
'flavor': 'single-controller',
'display_name': 'Single Controller',
'template': 'single-controller.tmpl',
'roles': [
'controller', 'compute', 'network', 'storage'
],
}, {
'flavor': 'multinodes',
'display_name': 'Multi-nodes',
'template': 'multinodes.tmpl',
'roles': [
'compute-controller', 'compute-worker', 'network-server',
'network-worker', 'database', 'messaging', 'image',
'dashboard', 'identity', 'storage-controller', 'storage-volume'
],
}]

View File

@ -0,0 +1,7 @@
NAME = 'ceph_openstack_icehouse'
DISPLAY_NAME = 'Ceph + OpenStack Icehouse'
PARENT = 'openstack'
PACKAGE_INSTALLER = 'chef_installer'
OS_INSTALLER = 'cobbler'
SUPPORTED_OS_PATTERNS = ['(?i)centos-6\.5.*', '(?i)ubuntu-12\.04.*']
DEPLOYABLE = True

View File

@ -0,0 +1,7 @@
NAME = 'ceph_firefly'
DISPLAY_NAME = 'Ceph Firefly'
PARENT = 'ceph'
PACKAGE_INSTALLER = 'chef_installer'
OS_INSTALLER = 'cobbler'
SUPPORTED_OS_PATTERNS = ['(?i)centos-6\.5.*', '(?i)ubuntu-12\.04.*']
DEPLOYABLE = True

View File

@ -209,6 +209,32 @@ class TestParseTimeInterval(unittest2.TestCase):
super(TestParseTimeInterval, self).tearDown()
class TestGetPluginsConfigFiles(unittest2.TestCase):
"""Test get plugins config files."""
def setUp(self):
super(TestGetPluginsConfigFiles, self).setUp()
self.TEST_UTIL_HOME = os.path.dirname(os.path.realpath(__file__))
def tearDown(self):
super(TestGetPluginsConfigFiles, self).tearDown()
def test_get_plugins_config_files(self):
setting.PLUGINS_DIR = self.TEST_UTIL_HOME + '/data/plugins'
loaded = util.get_plugins_config_files(
'adapter'
)
expected = [
setting.PLUGINS_DIR + '/test_installer1/adapter/test.conf',
setting.PLUGINS_DIR + '/test_installer1/adapter/test1.conf',
setting.PLUGINS_DIR + '/test_installer2/adapter/test.conf',
setting.PLUGINS_DIR + '/test_installer2/adapter/test2.conf',
]
loaded.sort()
expected.sort()
self.assertEqual(loaded, expected)
class TestLoadConfigs(unittest2.TestCase):
"""Test load configs."""

View File

@ -1,13 +0,0 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

View File

@ -1,13 +0,0 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

View File

@ -1,13 +0,0 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

View File

@ -1,13 +0,0 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

View File

@ -1,13 +0,0 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

View File

@ -70,11 +70,6 @@ COMPASS_SUPPORTED_DOMAINS = []
COMPASS_SUPPORTED_DEFAULT_GATEWAY = '127.0.0.1'
COMPASS_SUPPORTED_LOCAL_REPO = 'http://127.0.0.1'
# For test chef server. please replace these config info with your own.
TEST_CHEF_URL = "https://api.opscode.com/organizations/compasscheftest"
TEST_CLIENT_KEY_PATH = "/etc/compass/client.pem"
TEST_CLIENT_NAME = "graceyu"
PROGRESS_UPDATE_PID_FILE = '/var/run/progress_update.pid'
PROXY_URL_PREFIX = 'http://10.145.81.205:5000'
@ -176,4 +171,5 @@ if not FLAVOR_MAPPING_DIR:
FLAVOR_MAPPING_DIR = os.path.join(CONFIG_DIR, 'flavor_mapping')
if not PLUGINS_DIR:
PLUGINS_DIR = os.path.join(CONFIG_DIR, 'plugins')
PLUGINS_DIR = os.environ.get('COMPASS_PLUGINS_DIR',
os.path.join(CONFIG_DIR, 'plugins'))

View File

@ -213,6 +213,24 @@ def parse_time_interval(time_interval_str):
) / 1e6
def get_plugins_config_files(name, suffix=".conf"):
"""walk through each of plugin to find all the config files in the"""
"""name directory"""
plugins_path = setting.PLUGINS_DIR
files = []
if os.path.exists(plugins_path):
for plugin in os.listdir(plugins_path):
plugin_path = os.path.join(plugins_path, plugin)
plugin_config = os.path.join(plugin_path, name)
if os.path.exists(plugin_config):
for component in os.listdir(plugin_config):
if not component.endswith(suffix):
continue
files.append(os.path.join(plugin_config, component))
return files
def load_configs(
config_dir, config_name_suffix='.conf',
env_globals={}, env_locals={}
@ -239,16 +257,9 @@ def load_configs(
"""search for plugins config_dir"""
index = config_dir.rfind("/")
plugins_path = setting.PLUGINS_DIR
if os.path.exists(plugins_path):
for plugin in os.listdir(plugins_path):
plugin_path = os.path.join(plugins_path, plugin)
plugin_config = os.path.join(plugin_path, config_dir[index + 1:])
if os.path.exists(plugin_config):
for component in os.listdir(plugin_config):
if not component.endswith(config_name_suffix):
continue
config_files.append(os.path.join(plugin_config, component))
config_files.extend(get_plugins_config_files(config_dir[index + 1:],
config_name_suffix))
if not config_files:
logging.error('path %s and plugins does not exist', config_dir)

View File

@ -0,0 +1,479 @@
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
__author__ = "Grace Yu (grace.yu@huawei.com)"
import os
os.environ['COMPASS_IGNORE_SETTING'] = 'true'
from compass.utils import setting_wrapper as compass_setting
reload(compass_setting)
curr_dir = os.path.dirname(os.path.realpath(__file__))
test_tmpl_dir = os.path.join(curr_dir, 'templates')
test_plugins_dir = curr_dir
adapter_test_config = {
"name": "openstack_icehouse",
"flavors": [
{
"falvor_name": "test_flavor",
"roles": ["os-controller", "os-compute-worker", "os-network"],
"template": "multinodes.tmpl"
}
],
"os_installer": {
"name": "cobbler",
"settings": {
"cobbler_url": "http://127.0.0.1/cobbler_api",
"credentials": {
"username": "cobbler",
"password": "cobbler"
}
}
},
"package_installer": {
"name": "chef_installer",
"settings": {
"chef_url": "https://127.0.0.1",
"chef_server_ip": "127.0.0.1",
"chef_server_dns": "test_chef",
"key_dir": "xxx",
"client_name": "xxx",
"databags": ["user_passwords", "db_passwords"]
}
},
"metadata": {
"os_config": {
"_self": {},
"general": {
"_self": {"mapping_to": ""},
"language": {
"_self": {
"mapping_to": "language",
},
},
"timezone": {
"_self": {
"mapping_to": "timezone"
}
},
"default_gateway": {
"_self": {
"mapping_to": "gateway"
}
},
"domain": {
"_self": {"mapping_to": ""}
},
"http_proxy": {
"_self": {
"mapping_to": "http_proxy"
}
},
"ntp_server": {
"_self": {"mapping_to": "ntp_server"}
},
"dns_servers": {
"_self": {"mapping_to": "nameservers"}
},
"search_path": {
"_self": {"mapping_to": "search_path"}
},
"https_proxy": {
"_self": {"mapping_to": "https_proxy"}
}
},
"partition": {
"_self": {
"mapping_to": "partition"
},
"$path": {
"_self": {"mapping_to": ""},
"max_size": {
"_self": {"mapping_to": "vol_size"}
},
"size_percentage": {
"_self": {"mapping_to": "vol_percentage"}
}
}
},
"server_credentials": {
"_self": {
"mapping_to": "server_credentials"
},
"username": {
"_self": {"mapping_to": "username"}
},
"password": {
"_self": {"mapping_to": "password"}
}
}
},
"package_config": {
"_self": {},
"security": {
"_self": {},
"service_credentials": {
"_self": {
"mapping_to": "service_credentials"
},
"rabbit_mq": {
"_self": {
"mapping_to": "mq"
},
"username": {
"_self": {
"mapping_to": "username"
}
},
"password": {
"_self": {
"mapping_to": "password"
}
}
}
}
},
"network_mapping": {
"_self": {},
"management": {
"_self": {},
"interface": {
"_self": {}
}
},
"public": {
"_self": {},
"interface": {
"_self": {}
}
},
"tenant": {
"_self": {},
"interface": {
"_self": {}
}
}
},
"roles": {
"_self": {}
}
}
}
}
cluster_test_config = {
"id": 1,
"os_name": "Ubuntu-12.04-x86_64",
"name": "test",
"flavor": {
"falvor_name": "test_flavor",
"roles": ["os-controller", "os-compute-worker", "os-network"],
"template": "multinodes.tmpl"
},
"os_config": {
"general": {
"language": "EN",
"timezone": "UTC",
"default_gateway": "12.234.32.1",
"domain": "ods.com",
"http_proxy": "http://127.0.0.1:3128",
"https_proxy": "",
"ntp_server": "127.0.0.1",
"dns_servers": ["127.0.0.1"],
"search_path": ["ods.com"]
},
"partition": {
"/var": {
"max_size": 20,
"size_percentage": 20
},
"/home": {
"max_size": 50,
"size_percentage": 40
}
},
"server_credentials": {
"username": "root",
"password": "huawei"
}
},
"package_config": {
"security": {
"service_credentials": {
"rabbit_mq": {
"username": "guest",
"password": "test"
}
}
},
"network_mapping": {
"management": {
"interface": "eth0"
},
"public": {
"interface": "eth2"
},
"tenant": {
"interface": "eth1"
}
}
}
}
hosts_test_config = {
1: {
"host_id": 1,
"reinstall_os": True,
"mac": "00:0c:29:3e:60:e9",
"name": "server01.test",
"hostname": "server01",
"roles": ["os-controller"],
"networks": {
"vnet0": {
"ip": "12.234.32.100",
"netmask": "255.255.255.0",
"is_mgmt": True,
"is_promiscuous": False,
"subnet": "12.234.32.0/24"
},
"vnet1": {
"ip": "172.16.1.1",
"netmask": "255.255.255.0",
"is_mgmt": False,
"is_promiscuous": False,
"subnet": "172.16.1.0/24"
}
},
"os_config": {
"general": {
"default_gateway": "10.145.88.1",
},
"partition": {
"/var": {
"max_size": 30,
"size_percentage": 30
},
"/test": {
"max_size": 10,
"size_percentage": 10
}
}
},
"package_config": {
"network_mapping": {
"management": {
"interface": "vnet0"
},
"tenant": {
"interface": "vnet1"
}
}
}
},
2: {
"host_id": 2,
"reinstall_os": True,
"mac": "00:0c:29:3e:60:a1",
"name": "server02.test",
"hostname": "server02",
"roles": ["os-compute-worker"],
"networks": {
"eth0": {
"ip": "12.234.32.101",
"netmask": "255.255.255.0",
"is_mgmt": True,
"is_promiscuous": False,
"subnet": "12.234.32.0/24"
},
"eth1": {
"ip": "172.16.1.2",
"netmask": "255.255.255.0",
"is_mgmt": False,
"is_promiscuous": False,
"subnet": "172.16.1.0/24"
}
},
"os_config": {
"general": {
"language": "EN",
"timezone": "UTC",
"domain": "ods.com"
},
"partition": {
"/test": {
"max_size": 10,
"size_percentage": 20
}
}
},
"package_config": {
}
},
3: {
"host_id": 10,
"reinstall_os": False,
"mac": "00:0c:29:3e:60:a2",
"name": "server03.test",
"hostname": "server03",
"roles": ["os-network", "os-compute-worker"],
"networks": {
"eth0": {
"ip": "12.234.32.103",
"netmask": "255.255.255.0",
"is_mgmt": True,
"is_promiscuous": False,
"subnet": "12.234.32.0/24"
},
"eth1": {
"ip": "172.16.1.3",
"netmask": "255.255.255.0",
"is_mgmt": False,
"is_promiscuous": False,
"subnet": "172.16.1.0/24"
},
"eth2": {
"ip": "10.0.0.1",
"netmask": "255.255.255.0",
"is_mgmt": False,
"is_promiscuous": True,
"subnet": "10.0.0.0/24"
}
},
"ipmi_credentials": {
"ip": "172.16.100.104",
"username": "admin",
"password": "admin"
},
"os_config": {
"general": {
"language": "EN",
"timezone": "UTC",
"default_gateway": "12.234.32.1",
"domain": "ods.com",
"http_proxy": "http://10.145.88.211:3128",
"https_proxy": "",
"ntp_server": "10.145.88.211",
"dns_servers": "10.145.88.211",
"search_path": "ods.com"
},
"partition": {
"/var": {
"max_size": 20,
"size_percentage": 20
},
"/home": {
"max_size": 50,
"size_percentage": 40
}
}
},
"package_config": {
}
}
}
metadata_test_cases = [
{
"metadata": {
"general": {
"_self": {},
"language": {
"_self": {"mapping_to": "lan"}
},
"timezone": {
"_self": {"mapping_to": "timezone"}
}
}
},
"config": {
"general": {
"language": "EN",
"timezone": "UTC"
}
},
"expected_output": {
"lan": "EN",
"timezone": "UTC"
}
},
{
"metadata": {
"security": {
"_self": {"mapping_to": "security"},
"$credentials": {
"_self": {},
"$service": {
"username": {
"_self": {"mapping_to": "user"}
},
"password": {
"_self": {"mapping_to": "pass"}
}
}
}
},
"test": {
"_self": {"mapping_to": "test_section"},
"item1": {
"_self": {"mapping_to": "itema"}
},
"item2": {
"_self": {"mapping_to": "itemb"}
}
}
},
"config": {
"security": {
"service_credentials": {
"glance": {"username": "glance", "password": "glance"},
"identity": {"username": "keystone",
"password": "keystone"},
"dash": {"username": "dash", "password": "dash"}
},
"db_credentials": {
"mysql": {"username": "root", "password": "root"},
"rabbit_mq": {"username": "guest", "password": "guest"}
}
},
"test": {
"item1": "a",
"item2": "b"
}
},
"expected_output": {
"security": {
"service_credentials": {
"glance": {"user": "glance", "pass": "glance"},
"identity": {"user": "keystone", "pass": "keystone"},
"dash": {"user": "dash", "pass": "dash"}
},
"db_credentials": {
"mysql": {"user": "root", "pass": "root"},
"rabbit_mq": {"user": "guest", "pass": "guest"}
}
},
"test_section": {
"itema": "a",
"itemb": "b"
}
}
}
]

View File

@ -31,13 +31,13 @@ os.environ['COMPASS_IGNORE_SETTING'] = 'true'
from compass.utils import setting_wrapper as compass_setting
reload(compass_setting)
import config_data
from compass.deployment.installers.config_manager import BaseConfigManager
from compass.tests.deployment.test_data import config_data
from compass.utils import flags
from compass.utils import logsetting
# TODO(Carl) move this test to plugins
from plugins.chef_installer.implementation.chef_installer import ChefInstaller
@ -61,7 +61,7 @@ class TestChefInstaller(unittest2.TestCase):
ChefInstaller.get_tmpl_path = Mock()
test_tmpl_dir = os.path.join(
os.path.join(config_data.test_plugins_dir,
'chef_installer/templates'),
'templates'),
'openstack_icehouse'
)
ChefInstaller.get_tmpl_path.return_value = test_tmpl_dir

View File

@ -27,17 +27,26 @@ import unittest2
os.environ['COMPASS_IGNORE_SETTING'] = 'true'
from compass.tests.deployment.test_data import config_data
from compass.utils import setting_wrapper as compass_setting
reload(compass_setting)
import config_data
from compass.deployment.installers.config_manager import BaseConfigManager
# TODO(Carl) move this test to plugins
from plugins.chef_installer.implementation.chef_installer import ChefInstaller
"""It requires Chef server installed, in that case, this test can be run by
removing skip annontation
"""
# For test chef server. please replace these config info with your own.
TEST_CHEF_URL = "https://api.opscode.com/organizations/compasscheftest"
TEST_CLIENT_KEY_PATH = "/etc/compass/client.pem"
TEST_CLIENT_NAME = "graceyu"
@unittest2.skip("showing class skipping")
class TestChefInstaller(unittest2.TestCase):
"""Test installer functionality."""
def setUp(self):
@ -66,9 +75,9 @@ class TestChefInstaller(unittest2.TestCase):
def _get_testchefapi(self):
import chef
return chef.ChefAPI(config_data.test_chef_url,
config_data.test_client_key,
config_data.test_client)
return chef.ChefAPI(TEST_CHEF_URL,
TEST_CLIENT_KEY_PATH,
TEST_CLIENT_NAME)
def _register(self, obj):
self.objects.append(obj)
@ -83,7 +92,7 @@ class TestChefInstaller(unittest2.TestCase):
ChefInstaller.get_tmpl_path = Mock()
test_tmpl_dir = os.path.join(os.path.join(config_data.test_plugins_dir,
'chef_installer/templates'),
'templates'),
'openstack_icehouse')
ChefInstaller.get_tmpl_path.return_value = test_tmpl_dir