openstack/deb-sahara
Code Issues Proposed changes

Oozie + MySQL configuration

Browse Source 
* oozie-site.xml MySQL configuration
* MySQL starts on oozie node group
* MySQL starts if MySQL is enabled by config
* Oozie DB schema creating in MySQL
* Applying multiple general configs
* Added test for general configs

Implements blueprint edp-oozie-for-vanilla-plugin

Change-Id: I77342be5523f8584a5c6ba0d9fa2794d2702d27d
This commit is contained in:
Nikolay Mahotkin 2013-08-14 12:37:24 +04:00
parent 6699fe9943
commit 27928eac5f
7 changed files with 155 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
savanna/plugins/vanilla/config_helper.py
View File

@ -58,6 +58,12 @@ ENABLE_SWIFT = p.Config('Enable Swift', 'general', 'cluster',
config_type="bool", priority=1,
default_value=True, is_optional=True)
ENABLE_MYSQL = p.Config('Enable MySQL', 'general', 'cluster',
config_type="bool", priority=1,
default_value=True, is_optional=True)
GENERAL_CONFS = {}
HIDDEN_CONFS = ['fs.default.name', 'dfs.name.dir', 'dfs.data.dir',
'mapred.job.tracker', 'mapred.system.dir', 'mapred.local.dir',
'hadoop.proxyuser.hadoop.hosts',
@ -117,6 +123,7 @@ def _initialise_configs():
config_type="int"))
configs.append(ENABLE_SWIFT)
configs.append(ENABLE_MYSQL)
return configs
@ -128,8 +135,37 @@ def get_plugin_configs():
return PLUGIN_CONFIGS
def set_general_configs():
GENERAL_CONFS.update({
ENABLE_SWIFT.name: {
'default_value': ENABLE_SWIFT.default_value,
'conf': extract_name_values(swift.get_swift_configs())
},
ENABLE_MYSQL.name: {
'default_value': ENABLE_MYSQL.default_value,
'conf': o_h.get_oozie_mysql_configs()
}
})
def generate_cfg_from_general(cfg, configs, general_config,
rest_excluded=False):
if 'general' in configs:
for nm in general_config:
if nm not in configs['general'] and not rest_excluded:
configs['general'][nm] = general_config[nm]['default_value']
for name, value in configs['general'].items():
if value:
cfg = _set_config(cfg, general_config, name)
LOG.info("Applying config: %s" % name)
else:
cfg = _set_config(cfg, general_config)
return cfg
def generate_xml_configs(configs, storage_path, nn_hostname,
jt_hostname, oozie_hostname):
set_general_configs()
# inserting common configs depends on provisioned VMs and HDFS placement
# TODO(aignatov): should be moved to cluster context
cfg = {
@ -169,18 +205,8 @@ def generate_xml_configs(configs, storage_path, nn_hostname,
cfg[key] = value
# applying swift configs if user enabled it
swift_xml_confs = []
#TODO(aignatov): should be changed. General configs not only Swift
swift_in_config = False
if ('general' in configs and
ENABLE_SWIFT.name in configs['general']):
swift_in_config = True
if ((swift_in_config and configs['general'][ENABLE_SWIFT.name]) or
(not swift_in_config and ENABLE_SWIFT.default_value)):
swift_xml_confs = swift.get_swift_configs()
cfg.update(extract_name_values(swift_xml_confs))
LOG.info("Swift integration is enabled")
swift_xml_confs = swift.get_swift_configs()
cfg = generate_cfg_from_general(cfg, configs, GENERAL_CONFS)
# invoking applied configs to appropriate xml files
xml_configs = {
'core-site': x.create_hadoop_xml(cfg, CORE_DEFAULT + swift_xml_confs),
@ -272,3 +298,20 @@ def determine_cluster_config(cluster, config_name):
for conf in all_conf:
if conf.name == config_name:
return conf.default_value
def _set_config(cfg, gen_cfg, name=None):
if name in gen_cfg:
cfg.update(gen_cfg[name]['conf'])
if name is None:
for name in gen_cfg:
cfg.update(gen_cfg[name]['conf'])
return cfg
def is_mysql_enable(cluster):
for ng in cluster.node_groups:
conf = ng.configuration
if 'general' in conf and ENABLE_MYSQL.name in conf['general']:
return conf['general'][ENABLE_MYSQL.name]
return ENABLE_MYSQL.default_value

11
savanna/plugins/vanilla/oozie_helper.py
View File

@ -55,3 +55,14 @@ def append_oozie_setup(setup_script, env_configs):
setup_script.append(
"cat /opt/oozie/conf/oozie-env.sh >> /tmp/oozie-env.sh")
setup_script.append("cp /tmp/oozie-env.sh /opt/oozie/conf/oozie-env.sh")
def get_oozie_mysql_configs():
return {
'oozie.service.JPAService.jdbc.driver':
'com.mysql.jdbc.Driver',
'oozie.service.JPAService.jdbc.url':
'jdbc:mysql://localhost:3306/oozie',
'oozie.service.JPAService.jdbc.username': 'oozie',
'oozie.service.JPAService.jdbc.password': 'oozie'
}

12
savanna/plugins/vanilla/plugin.py
View File

@ -23,6 +23,7 @@ from savanna.plugins.vanilla import config_helper as c_helper
from savanna.plugins.vanilla import run_scripts as run
from savanna.plugins.vanilla import scaling as sc
from savanna.utils import crypto
from savanna.utils import files as f
from savanna.utils import remote
@ -124,8 +125,12 @@ class VanillaProvider(p.ProvisioningPluginBase):
LOG.info("MapReduce service at '%s' has been started",
jt_instance.hostname)
#TODO(nmakhotkin) Add start MySQL on hive_server if oozie != hive
if oozie:
with remote.get_remote(oozie) as r:
if c_helper.is_mysql_enable(cluster):
run.mysql_start(r, oozie)
run.oozie_create_db(r)
run.oozie_share_lib(r, nn_instance.hostname)
run.start_oozie(r)
LOG.info("Oozie service at '%s' has been started",
@ -252,6 +257,13 @@ class VanillaProvider(p.ProvisioningPluginBase):
r.write_file_to('/opt/oozie/conf/oozie-site.xml',
extra[oozie.node_group.id]
['xml']['oozie-site'])
if c_helper.is_mysql_enable(cluster):
sql_script = f.get_file_text(
'plugins/vanilla/resources/create_oozie_db.sql')
files = {
'/tmp/create_oozie_db.sql': sql_script
}
remote.get_remote(oozie).write_files_to(files)
def _set_cluster_info(self, cluster):
nn = utils.get_namenode(cluster)

4
savanna/plugins/vanilla/resources/create_oozie_db.sql Normal file
View File

@ -0,0 +1,4 @@
create database oozie;
grant all privileges on oozie.* to 'oozie'@'localhost' identified by 'oozie';
grant all privileges on oozie.* to 'oozie'@'%' identified by 'oozie';
exit

10
savanna/plugins/vanilla/run_scripts.py
View File

@ -44,6 +44,16 @@ def oozie_share_lib(remote, nn_hostname):
'-run Validate DB Connection" hadoop')
def mysql_start(remote, mysql_instance):
LOG.debug("Starting mysql at %s" % mysql_instance.hostname)
remote.execute_command("/opt/start-mysql.sh")
def oozie_create_db(remote):
LOG.debug("Creating Oozie DB Schema...")
remote.execute_command("mysql -u root < /tmp/create_oozie_db.sql")
def start_oozie(remote):
remote.execute_command(
'sudo su - -c "/opt/oozie/bin/oozied.sh start" hadoop')

39
savanna/tests/unit/plugins/vanilla/test_plugin.py
View File

@ -114,3 +114,42 @@ class VanillaPluginTest(unittest2.TestCase):
('dfs.replication', 3),
('mapred.reduce.tasks', 2),
('io.sort.factor', 10)])
def test_general_configs(self):
gen_config = {
c_h.ENABLE_SWIFT.name: {
'default_value': c_h.ENABLE_SWIFT.default_value,
'conf': {
'fs.swift.enabled': True
}
},
c_h.ENABLE_MYSQL.name: {
'default_value': c_h.ENABLE_MYSQL.default_value,
'conf': {
'oozie.service.JPAService.jdbc.username': 'oozie'
}
}
}
all_configured = {
'fs.swift.enabled': True,
'oozie.service.JPAService.jdbc.username': 'oozie'
}
configs = {
'general': {
'Enable Swift': True
}
}
cfg = c_h.generate_cfg_from_general({}, configs, gen_config)
self.assertDictEqual(cfg, all_configured)
configs['general'].update({'Enable MySQL': False})
cfg = c_h.generate_cfg_from_general({}, configs, gen_config)
self.assertDictEqual(cfg, {'fs.swift.enabled': True})
configs['general'].update({
'Enable Swift': False,
'Enable MySQL': False
})
cfg = c_h.generate_cfg_from_general({}, configs, gen_config)
self.assertDictEqual(cfg, {})
configs = {}
cfg = c_h.generate_cfg_from_general({}, configs, gen_config)
self.assertDictEqual(cfg, all_configured)

24
savanna/utils/files.py Normal file
View File

@ -0,0 +1,24 @@
# Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pkg_resources as pkg
from savanna import version
def get_file_text(file_name):
full_name = pkg.resource_filename(
version.version_info.package, file_name)
return open(full_name).read()