diff --git a/savanna/plugins/vanilla/config_helper.py b/savanna/plugins/vanilla/config_helper.py index e6faab4478..f7cda215b1 100644 --- a/savanna/plugins/vanilla/config_helper.py +++ b/savanna/plugins/vanilla/config_helper.py @@ -58,6 +58,12 @@ ENABLE_SWIFT = p.Config('Enable Swift', 'general', 'cluster', config_type="bool", priority=1, default_value=True, is_optional=True) +ENABLE_MYSQL = p.Config('Enable MySQL', 'general', 'cluster', + config_type="bool", priority=1, + default_value=True, is_optional=True) + +GENERAL_CONFS = {} + HIDDEN_CONFS = ['fs.default.name', 'dfs.name.dir', 'dfs.data.dir', 'mapred.job.tracker', 'mapred.system.dir', 'mapred.local.dir', 'hadoop.proxyuser.hadoop.hosts', @@ -117,6 +123,7 @@ def _initialise_configs(): config_type="int")) configs.append(ENABLE_SWIFT) + configs.append(ENABLE_MYSQL) return configs @@ -128,8 +135,37 @@ def get_plugin_configs(): return PLUGIN_CONFIGS +def set_general_configs(): + GENERAL_CONFS.update({ + ENABLE_SWIFT.name: { + 'default_value': ENABLE_SWIFT.default_value, + 'conf': extract_name_values(swift.get_swift_configs()) + }, + ENABLE_MYSQL.name: { + 'default_value': ENABLE_MYSQL.default_value, + 'conf': o_h.get_oozie_mysql_configs() + } + }) + + +def generate_cfg_from_general(cfg, configs, general_config, + rest_excluded=False): + if 'general' in configs: + for nm in general_config: + if nm not in configs['general'] and not rest_excluded: + configs['general'][nm] = general_config[nm]['default_value'] + for name, value in configs['general'].items(): + if value: + cfg = _set_config(cfg, general_config, name) + LOG.info("Applying config: %s" % name) + else: + cfg = _set_config(cfg, general_config) + return cfg + + def generate_xml_configs(configs, storage_path, nn_hostname, jt_hostname, oozie_hostname): + set_general_configs() # inserting common configs depends on provisioned VMs and HDFS placement # TODO(aignatov): should be moved to cluster context cfg = { @@ -169,18 +205,8 @@ def generate_xml_configs(configs, storage_path, nn_hostname, cfg[key] = value # applying swift configs if user enabled it - swift_xml_confs = [] - #TODO(aignatov): should be changed. General configs not only Swift - swift_in_config = False - if ('general' in configs and - ENABLE_SWIFT.name in configs['general']): - swift_in_config = True - if ((swift_in_config and configs['general'][ENABLE_SWIFT.name]) or - (not swift_in_config and ENABLE_SWIFT.default_value)): - swift_xml_confs = swift.get_swift_configs() - cfg.update(extract_name_values(swift_xml_confs)) - LOG.info("Swift integration is enabled") - + swift_xml_confs = swift.get_swift_configs() + cfg = generate_cfg_from_general(cfg, configs, GENERAL_CONFS) # invoking applied configs to appropriate xml files xml_configs = { 'core-site': x.create_hadoop_xml(cfg, CORE_DEFAULT + swift_xml_confs), @@ -272,3 +298,20 @@ def determine_cluster_config(cluster, config_name): for conf in all_conf: if conf.name == config_name: return conf.default_value + + +def _set_config(cfg, gen_cfg, name=None): + if name in gen_cfg: + cfg.update(gen_cfg[name]['conf']) + if name is None: + for name in gen_cfg: + cfg.update(gen_cfg[name]['conf']) + return cfg + + +def is_mysql_enable(cluster): + for ng in cluster.node_groups: + conf = ng.configuration + if 'general' in conf and ENABLE_MYSQL.name in conf['general']: + return conf['general'][ENABLE_MYSQL.name] + return ENABLE_MYSQL.default_value diff --git a/savanna/plugins/vanilla/oozie_helper.py b/savanna/plugins/vanilla/oozie_helper.py index ca6c9f18c2..5ac21b6537 100644 --- a/savanna/plugins/vanilla/oozie_helper.py +++ b/savanna/plugins/vanilla/oozie_helper.py @@ -55,3 +55,14 @@ def append_oozie_setup(setup_script, env_configs): setup_script.append( "cat /opt/oozie/conf/oozie-env.sh >> /tmp/oozie-env.sh") setup_script.append("cp /tmp/oozie-env.sh /opt/oozie/conf/oozie-env.sh") + + +def get_oozie_mysql_configs(): + return { + 'oozie.service.JPAService.jdbc.driver': + 'com.mysql.jdbc.Driver', + 'oozie.service.JPAService.jdbc.url': + 'jdbc:mysql://localhost:3306/oozie', + 'oozie.service.JPAService.jdbc.username': 'oozie', + 'oozie.service.JPAService.jdbc.password': 'oozie' + } diff --git a/savanna/plugins/vanilla/plugin.py b/savanna/plugins/vanilla/plugin.py index 821a5087e3..5041ec0645 100644 --- a/savanna/plugins/vanilla/plugin.py +++ b/savanna/plugins/vanilla/plugin.py @@ -23,6 +23,7 @@ from savanna.plugins.vanilla import config_helper as c_helper from savanna.plugins.vanilla import run_scripts as run from savanna.plugins.vanilla import scaling as sc from savanna.utils import crypto +from savanna.utils import files as f from savanna.utils import remote @@ -124,8 +125,12 @@ class VanillaProvider(p.ProvisioningPluginBase): LOG.info("MapReduce service at '%s' has been started", jt_instance.hostname) + #TODO(nmakhotkin) Add start MySQL on hive_server if oozie != hive if oozie: with remote.get_remote(oozie) as r: + if c_helper.is_mysql_enable(cluster): + run.mysql_start(r, oozie) + run.oozie_create_db(r) run.oozie_share_lib(r, nn_instance.hostname) run.start_oozie(r) LOG.info("Oozie service at '%s' has been started", @@ -252,6 +257,13 @@ class VanillaProvider(p.ProvisioningPluginBase): r.write_file_to('/opt/oozie/conf/oozie-site.xml', extra[oozie.node_group.id] ['xml']['oozie-site']) + if c_helper.is_mysql_enable(cluster): + sql_script = f.get_file_text( + 'plugins/vanilla/resources/create_oozie_db.sql') + files = { + '/tmp/create_oozie_db.sql': sql_script + } + remote.get_remote(oozie).write_files_to(files) def _set_cluster_info(self, cluster): nn = utils.get_namenode(cluster) diff --git a/savanna/plugins/vanilla/resources/create_oozie_db.sql b/savanna/plugins/vanilla/resources/create_oozie_db.sql new file mode 100644 index 0000000000..ad45051829 --- /dev/null +++ b/savanna/plugins/vanilla/resources/create_oozie_db.sql @@ -0,0 +1,4 @@ +create database oozie; +grant all privileges on oozie.* to 'oozie'@'localhost' identified by 'oozie'; +grant all privileges on oozie.* to 'oozie'@'%' identified by 'oozie'; +exit \ No newline at end of file diff --git a/savanna/plugins/vanilla/run_scripts.py b/savanna/plugins/vanilla/run_scripts.py index cd2e6c66bf..d2a11b9f20 100644 --- a/savanna/plugins/vanilla/run_scripts.py +++ b/savanna/plugins/vanilla/run_scripts.py @@ -44,6 +44,16 @@ def oozie_share_lib(remote, nn_hostname): '-run Validate DB Connection" hadoop') +def mysql_start(remote, mysql_instance): + LOG.debug("Starting mysql at %s" % mysql_instance.hostname) + remote.execute_command("/opt/start-mysql.sh") + + +def oozie_create_db(remote): + LOG.debug("Creating Oozie DB Schema...") + remote.execute_command("mysql -u root < /tmp/create_oozie_db.sql") + + def start_oozie(remote): remote.execute_command( 'sudo su - -c "/opt/oozie/bin/oozied.sh start" hadoop') diff --git a/savanna/tests/unit/plugins/vanilla/test_plugin.py b/savanna/tests/unit/plugins/vanilla/test_plugin.py index 8249683277..a0cee8cc01 100644 --- a/savanna/tests/unit/plugins/vanilla/test_plugin.py +++ b/savanna/tests/unit/plugins/vanilla/test_plugin.py @@ -114,3 +114,42 @@ class VanillaPluginTest(unittest2.TestCase): ('dfs.replication', 3), ('mapred.reduce.tasks', 2), ('io.sort.factor', 10)]) + + def test_general_configs(self): + gen_config = { + c_h.ENABLE_SWIFT.name: { + 'default_value': c_h.ENABLE_SWIFT.default_value, + 'conf': { + 'fs.swift.enabled': True + } + }, + c_h.ENABLE_MYSQL.name: { + 'default_value': c_h.ENABLE_MYSQL.default_value, + 'conf': { + 'oozie.service.JPAService.jdbc.username': 'oozie' + } + } + } + all_configured = { + 'fs.swift.enabled': True, + 'oozie.service.JPAService.jdbc.username': 'oozie' + } + configs = { + 'general': { + 'Enable Swift': True + } + } + cfg = c_h.generate_cfg_from_general({}, configs, gen_config) + self.assertDictEqual(cfg, all_configured) + configs['general'].update({'Enable MySQL': False}) + cfg = c_h.generate_cfg_from_general({}, configs, gen_config) + self.assertDictEqual(cfg, {'fs.swift.enabled': True}) + configs['general'].update({ + 'Enable Swift': False, + 'Enable MySQL': False + }) + cfg = c_h.generate_cfg_from_general({}, configs, gen_config) + self.assertDictEqual(cfg, {}) + configs = {} + cfg = c_h.generate_cfg_from_general({}, configs, gen_config) + self.assertDictEqual(cfg, all_configured) diff --git a/savanna/utils/files.py b/savanna/utils/files.py new file mode 100644 index 0000000000..f1e7be7723 --- /dev/null +++ b/savanna/utils/files.py @@ -0,0 +1,24 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pkg_resources as pkg + +from savanna import version + + +def get_file_text(file_name): + full_name = pkg.resource_filename( + version.version_info.package, file_name) + return open(full_name).read()