Oozie + MySQL configuration
* oozie-site.xml MySQL configuration * MySQL starts on oozie node group * MySQL starts if MySQL is enabled by config * Oozie DB schema creating in MySQL * Applying multiple general configs * Added test for general configs Implements blueprint edp-oozie-for-vanilla-plugin Change-Id: I77342be5523f8584a5c6ba0d9fa2794d2702d27d
This commit is contained in:
parent
6699fe9943
commit
27928eac5f
@ -58,6 +58,12 @@ ENABLE_SWIFT = p.Config('Enable Swift', 'general', 'cluster',
|
|||||||
config_type="bool", priority=1,
|
config_type="bool", priority=1,
|
||||||
default_value=True, is_optional=True)
|
default_value=True, is_optional=True)
|
||||||
|
|
||||||
|
ENABLE_MYSQL = p.Config('Enable MySQL', 'general', 'cluster',
|
||||||
|
config_type="bool", priority=1,
|
||||||
|
default_value=True, is_optional=True)
|
||||||
|
|
||||||
|
GENERAL_CONFS = {}
|
||||||
|
|
||||||
HIDDEN_CONFS = ['fs.default.name', 'dfs.name.dir', 'dfs.data.dir',
|
HIDDEN_CONFS = ['fs.default.name', 'dfs.name.dir', 'dfs.data.dir',
|
||||||
'mapred.job.tracker', 'mapred.system.dir', 'mapred.local.dir',
|
'mapred.job.tracker', 'mapred.system.dir', 'mapred.local.dir',
|
||||||
'hadoop.proxyuser.hadoop.hosts',
|
'hadoop.proxyuser.hadoop.hosts',
|
||||||
@ -117,6 +123,7 @@ def _initialise_configs():
|
|||||||
config_type="int"))
|
config_type="int"))
|
||||||
|
|
||||||
configs.append(ENABLE_SWIFT)
|
configs.append(ENABLE_SWIFT)
|
||||||
|
configs.append(ENABLE_MYSQL)
|
||||||
|
|
||||||
return configs
|
return configs
|
||||||
|
|
||||||
@ -128,8 +135,37 @@ def get_plugin_configs():
|
|||||||
return PLUGIN_CONFIGS
|
return PLUGIN_CONFIGS
|
||||||
|
|
||||||
|
|
||||||
|
def set_general_configs():
|
||||||
|
GENERAL_CONFS.update({
|
||||||
|
ENABLE_SWIFT.name: {
|
||||||
|
'default_value': ENABLE_SWIFT.default_value,
|
||||||
|
'conf': extract_name_values(swift.get_swift_configs())
|
||||||
|
},
|
||||||
|
ENABLE_MYSQL.name: {
|
||||||
|
'default_value': ENABLE_MYSQL.default_value,
|
||||||
|
'conf': o_h.get_oozie_mysql_configs()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def generate_cfg_from_general(cfg, configs, general_config,
|
||||||
|
rest_excluded=False):
|
||||||
|
if 'general' in configs:
|
||||||
|
for nm in general_config:
|
||||||
|
if nm not in configs['general'] and not rest_excluded:
|
||||||
|
configs['general'][nm] = general_config[nm]['default_value']
|
||||||
|
for name, value in configs['general'].items():
|
||||||
|
if value:
|
||||||
|
cfg = _set_config(cfg, general_config, name)
|
||||||
|
LOG.info("Applying config: %s" % name)
|
||||||
|
else:
|
||||||
|
cfg = _set_config(cfg, general_config)
|
||||||
|
return cfg
|
||||||
|
|
||||||
|
|
||||||
def generate_xml_configs(configs, storage_path, nn_hostname,
|
def generate_xml_configs(configs, storage_path, nn_hostname,
|
||||||
jt_hostname, oozie_hostname):
|
jt_hostname, oozie_hostname):
|
||||||
|
set_general_configs()
|
||||||
# inserting common configs depends on provisioned VMs and HDFS placement
|
# inserting common configs depends on provisioned VMs and HDFS placement
|
||||||
# TODO(aignatov): should be moved to cluster context
|
# TODO(aignatov): should be moved to cluster context
|
||||||
cfg = {
|
cfg = {
|
||||||
@ -169,18 +205,8 @@ def generate_xml_configs(configs, storage_path, nn_hostname,
|
|||||||
cfg[key] = value
|
cfg[key] = value
|
||||||
|
|
||||||
# applying swift configs if user enabled it
|
# applying swift configs if user enabled it
|
||||||
swift_xml_confs = []
|
swift_xml_confs = swift.get_swift_configs()
|
||||||
#TODO(aignatov): should be changed. General configs not only Swift
|
cfg = generate_cfg_from_general(cfg, configs, GENERAL_CONFS)
|
||||||
swift_in_config = False
|
|
||||||
if ('general' in configs and
|
|
||||||
ENABLE_SWIFT.name in configs['general']):
|
|
||||||
swift_in_config = True
|
|
||||||
if ((swift_in_config and configs['general'][ENABLE_SWIFT.name]) or
|
|
||||||
(not swift_in_config and ENABLE_SWIFT.default_value)):
|
|
||||||
swift_xml_confs = swift.get_swift_configs()
|
|
||||||
cfg.update(extract_name_values(swift_xml_confs))
|
|
||||||
LOG.info("Swift integration is enabled")
|
|
||||||
|
|
||||||
# invoking applied configs to appropriate xml files
|
# invoking applied configs to appropriate xml files
|
||||||
xml_configs = {
|
xml_configs = {
|
||||||
'core-site': x.create_hadoop_xml(cfg, CORE_DEFAULT + swift_xml_confs),
|
'core-site': x.create_hadoop_xml(cfg, CORE_DEFAULT + swift_xml_confs),
|
||||||
@ -272,3 +298,20 @@ def determine_cluster_config(cluster, config_name):
|
|||||||
for conf in all_conf:
|
for conf in all_conf:
|
||||||
if conf.name == config_name:
|
if conf.name == config_name:
|
||||||
return conf.default_value
|
return conf.default_value
|
||||||
|
|
||||||
|
|
||||||
|
def _set_config(cfg, gen_cfg, name=None):
|
||||||
|
if name in gen_cfg:
|
||||||
|
cfg.update(gen_cfg[name]['conf'])
|
||||||
|
if name is None:
|
||||||
|
for name in gen_cfg:
|
||||||
|
cfg.update(gen_cfg[name]['conf'])
|
||||||
|
return cfg
|
||||||
|
|
||||||
|
|
||||||
|
def is_mysql_enable(cluster):
|
||||||
|
for ng in cluster.node_groups:
|
||||||
|
conf = ng.configuration
|
||||||
|
if 'general' in conf and ENABLE_MYSQL.name in conf['general']:
|
||||||
|
return conf['general'][ENABLE_MYSQL.name]
|
||||||
|
return ENABLE_MYSQL.default_value
|
||||||
|
@ -55,3 +55,14 @@ def append_oozie_setup(setup_script, env_configs):
|
|||||||
setup_script.append(
|
setup_script.append(
|
||||||
"cat /opt/oozie/conf/oozie-env.sh >> /tmp/oozie-env.sh")
|
"cat /opt/oozie/conf/oozie-env.sh >> /tmp/oozie-env.sh")
|
||||||
setup_script.append("cp /tmp/oozie-env.sh /opt/oozie/conf/oozie-env.sh")
|
setup_script.append("cp /tmp/oozie-env.sh /opt/oozie/conf/oozie-env.sh")
|
||||||
|
|
||||||
|
|
||||||
|
def get_oozie_mysql_configs():
|
||||||
|
return {
|
||||||
|
'oozie.service.JPAService.jdbc.driver':
|
||||||
|
'com.mysql.jdbc.Driver',
|
||||||
|
'oozie.service.JPAService.jdbc.url':
|
||||||
|
'jdbc:mysql://localhost:3306/oozie',
|
||||||
|
'oozie.service.JPAService.jdbc.username': 'oozie',
|
||||||
|
'oozie.service.JPAService.jdbc.password': 'oozie'
|
||||||
|
}
|
||||||
|
@ -23,6 +23,7 @@ from savanna.plugins.vanilla import config_helper as c_helper
|
|||||||
from savanna.plugins.vanilla import run_scripts as run
|
from savanna.plugins.vanilla import run_scripts as run
|
||||||
from savanna.plugins.vanilla import scaling as sc
|
from savanna.plugins.vanilla import scaling as sc
|
||||||
from savanna.utils import crypto
|
from savanna.utils import crypto
|
||||||
|
from savanna.utils import files as f
|
||||||
from savanna.utils import remote
|
from savanna.utils import remote
|
||||||
|
|
||||||
|
|
||||||
@ -124,8 +125,12 @@ class VanillaProvider(p.ProvisioningPluginBase):
|
|||||||
LOG.info("MapReduce service at '%s' has been started",
|
LOG.info("MapReduce service at '%s' has been started",
|
||||||
jt_instance.hostname)
|
jt_instance.hostname)
|
||||||
|
|
||||||
|
#TODO(nmakhotkin) Add start MySQL on hive_server if oozie != hive
|
||||||
if oozie:
|
if oozie:
|
||||||
with remote.get_remote(oozie) as r:
|
with remote.get_remote(oozie) as r:
|
||||||
|
if c_helper.is_mysql_enable(cluster):
|
||||||
|
run.mysql_start(r, oozie)
|
||||||
|
run.oozie_create_db(r)
|
||||||
run.oozie_share_lib(r, nn_instance.hostname)
|
run.oozie_share_lib(r, nn_instance.hostname)
|
||||||
run.start_oozie(r)
|
run.start_oozie(r)
|
||||||
LOG.info("Oozie service at '%s' has been started",
|
LOG.info("Oozie service at '%s' has been started",
|
||||||
@ -252,6 +257,13 @@ class VanillaProvider(p.ProvisioningPluginBase):
|
|||||||
r.write_file_to('/opt/oozie/conf/oozie-site.xml',
|
r.write_file_to('/opt/oozie/conf/oozie-site.xml',
|
||||||
extra[oozie.node_group.id]
|
extra[oozie.node_group.id]
|
||||||
['xml']['oozie-site'])
|
['xml']['oozie-site'])
|
||||||
|
if c_helper.is_mysql_enable(cluster):
|
||||||
|
sql_script = f.get_file_text(
|
||||||
|
'plugins/vanilla/resources/create_oozie_db.sql')
|
||||||
|
files = {
|
||||||
|
'/tmp/create_oozie_db.sql': sql_script
|
||||||
|
}
|
||||||
|
remote.get_remote(oozie).write_files_to(files)
|
||||||
|
|
||||||
def _set_cluster_info(self, cluster):
|
def _set_cluster_info(self, cluster):
|
||||||
nn = utils.get_namenode(cluster)
|
nn = utils.get_namenode(cluster)
|
||||||
|
4
savanna/plugins/vanilla/resources/create_oozie_db.sql
Normal file
4
savanna/plugins/vanilla/resources/create_oozie_db.sql
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
create database oozie;
|
||||||
|
grant all privileges on oozie.* to 'oozie'@'localhost' identified by 'oozie';
|
||||||
|
grant all privileges on oozie.* to 'oozie'@'%' identified by 'oozie';
|
||||||
|
exit
|
@ -44,6 +44,16 @@ def oozie_share_lib(remote, nn_hostname):
|
|||||||
'-run Validate DB Connection" hadoop')
|
'-run Validate DB Connection" hadoop')
|
||||||
|
|
||||||
|
|
||||||
|
def mysql_start(remote, mysql_instance):
|
||||||
|
LOG.debug("Starting mysql at %s" % mysql_instance.hostname)
|
||||||
|
remote.execute_command("/opt/start-mysql.sh")
|
||||||
|
|
||||||
|
|
||||||
|
def oozie_create_db(remote):
|
||||||
|
LOG.debug("Creating Oozie DB Schema...")
|
||||||
|
remote.execute_command("mysql -u root < /tmp/create_oozie_db.sql")
|
||||||
|
|
||||||
|
|
||||||
def start_oozie(remote):
|
def start_oozie(remote):
|
||||||
remote.execute_command(
|
remote.execute_command(
|
||||||
'sudo su - -c "/opt/oozie/bin/oozied.sh start" hadoop')
|
'sudo su - -c "/opt/oozie/bin/oozied.sh start" hadoop')
|
||||||
|
@ -114,3 +114,42 @@ class VanillaPluginTest(unittest2.TestCase):
|
|||||||
('dfs.replication', 3),
|
('dfs.replication', 3),
|
||||||
('mapred.reduce.tasks', 2),
|
('mapred.reduce.tasks', 2),
|
||||||
('io.sort.factor', 10)])
|
('io.sort.factor', 10)])
|
||||||
|
|
||||||
|
def test_general_configs(self):
|
||||||
|
gen_config = {
|
||||||
|
c_h.ENABLE_SWIFT.name: {
|
||||||
|
'default_value': c_h.ENABLE_SWIFT.default_value,
|
||||||
|
'conf': {
|
||||||
|
'fs.swift.enabled': True
|
||||||
|
}
|
||||||
|
},
|
||||||
|
c_h.ENABLE_MYSQL.name: {
|
||||||
|
'default_value': c_h.ENABLE_MYSQL.default_value,
|
||||||
|
'conf': {
|
||||||
|
'oozie.service.JPAService.jdbc.username': 'oozie'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
all_configured = {
|
||||||
|
'fs.swift.enabled': True,
|
||||||
|
'oozie.service.JPAService.jdbc.username': 'oozie'
|
||||||
|
}
|
||||||
|
configs = {
|
||||||
|
'general': {
|
||||||
|
'Enable Swift': True
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cfg = c_h.generate_cfg_from_general({}, configs, gen_config)
|
||||||
|
self.assertDictEqual(cfg, all_configured)
|
||||||
|
configs['general'].update({'Enable MySQL': False})
|
||||||
|
cfg = c_h.generate_cfg_from_general({}, configs, gen_config)
|
||||||
|
self.assertDictEqual(cfg, {'fs.swift.enabled': True})
|
||||||
|
configs['general'].update({
|
||||||
|
'Enable Swift': False,
|
||||||
|
'Enable MySQL': False
|
||||||
|
})
|
||||||
|
cfg = c_h.generate_cfg_from_general({}, configs, gen_config)
|
||||||
|
self.assertDictEqual(cfg, {})
|
||||||
|
configs = {}
|
||||||
|
cfg = c_h.generate_cfg_from_general({}, configs, gen_config)
|
||||||
|
self.assertDictEqual(cfg, all_configured)
|
||||||
|
24
savanna/utils/files.py
Normal file
24
savanna/utils/files.py
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
# Copyright (c) 2013 Mirantis Inc.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
# implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
import pkg_resources as pkg
|
||||||
|
|
||||||
|
from savanna import version
|
||||||
|
|
||||||
|
|
||||||
|
def get_file_text(file_name):
|
||||||
|
full_name = pkg.resource_filename(
|
||||||
|
version.version_info.package, file_name)
|
||||||
|
return open(full_name).read()
|
Loading…
Reference in New Issue
Block a user