diff --git a/.stestr.conf b/.stestr.conf index d81c293..1f3c729 100644 --- a/.stestr.conf +++ b/.stestr.conf @@ -1,3 +1,3 @@ [DEFAULT] -test_path=./sahara/tests/unit +test_path=./sahara_plugin_vanilla/tests/unit top_dir=./ diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..92f6253 --- /dev/null +++ b/README.rst @@ -0,0 +1,35 @@ +======================== +Team and repository tags +======================== + +.. image:: https://governance.openstack.org/tc/badges/sahara.svg + :target: https://governance.openstack.org/tc/reference/tags/index.html + +.. Change things from this point on + +OpenStack Data Processing ("Sahara") project +============================================ + +Sahara at wiki.openstack.org: https://wiki.openstack.org/wiki/Sahara + +Storyboard project: https://storyboard.openstack.org/#!/project/935 + +Sahara docs site: https://docs.openstack.org/sahara/latest/ + +Roadmap: https://wiki.openstack.org/wiki/Sahara/Roadmap + +Quickstart guide: https://docs.openstack.org/sahara/latest/user/quickstart.html + +How to participate: https://docs.openstack.org/sahara/latest/contributor/how-to-participate.html + +Source: https://git.openstack.org/cgit/openstack/sahara + +Bugs and feature requests: https://storyboard.openstack.org/#!/project/935 + +Release notes: https://docs.openstack.org/releasenotes/sahara/ + +License +------- + +Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0 + diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 0000000..9cff004 --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,9 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +openstackdocstheme>=1.18.1 # Apache-2.0 +os-api-ref>=1.4.0 # Apache-2.0 +reno>=2.5.0 # Apache-2.0 +sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD +sphinxcontrib-httpdomain>=1.3.0 # BSD +whereto>=0.3.0 # Apache-2.0 diff --git a/errors.txt b/errors.txt new file mode 100644 index 0000000..6809594 --- /dev/null +++ b/errors.txt @@ -0,0 +1,34 @@ +py27 develop-inst-nodeps: /home/tenobreg/coding/upstream/sahara/sahara +py27 installed: alabaster==0.7.11,alembic==1.0.0,amqp==2.3.2,appdirs==1.4.3,asn1crypto==0.24.0,astroid==1.3.8,Babel==2.6.0,bandit==1.5.0,bashate==0.6.0,bcrypt==3.1.4,botocore==1.10.62,cachetools==2.1.0,castellan==0.18.0,certifi==2018.4.16,cffi==1.11.5,chardet==3.0.4,click==6.7,cliff==2.13.0,cmd2==0.8.8,contextlib2==0.5.5,coverage==4.5.1,cryptography==2.3,debtcollector==1.20.0,decorator==4.3.0,deprecation==2.0.5,doc8==0.8.0,docutils==0.14,dogpile.cache==0.6.6,dulwich==0.19.5,enum-compat==0.0.2,enum34==1.1.6,eventlet==0.20.0,extras==1.0.0,fasteners==0.14.1,fixtures==3.0.0,flake8==2.5.5,Flask==1.0.2,funcsigs==1.0.2,functools32==3.2.3.post2,future==0.16.0,futures==3.2.0,futurist==1.7.0,gitdb2==2.0.4,GitPython==2.1.11,greenlet==0.4.13,hacking==0.12.0,idna==2.7,imagesize==1.0.0,ipaddress==1.0.22,iso8601==0.1.12,itsdangerous==0.24,Jinja2==2.10,jmespath==0.9.3,jsonpatch==1.23,jsonpointer==2.0,jsonschema==2.6.0,keystoneauth1==3.10.0,keystonemiddleware==5.2.0,kombu==4.2.1,linecache2==1.0.0,logilab-common==1.4.2,Mako==1.0.7,MarkupSafe==1.0,mccabe==0.2.1,mock==2.0.0,monotonic==1.5,mox3==0.26.0,msgpack==0.5.6,munch==2.3.2,netaddr==0.7.19,netifaces==0.10.7,openstackdocstheme==1.22.0,openstacksdk==0.17.2,os-api-ref==1.5.0,os-client-config==1.31.2,os-service-types==1.3.0,os-testr==1.0.0,osc-lib==1.11.1,oslo.cache==1.30.1,oslo.concurrency==3.27.0,oslo.config==6.4.0,oslo.context==2.21.0,oslo.db==4.40.0,oslo.i18n==3.21.0,oslo.log==3.39.0,oslo.messaging==8.1.0,oslo.middleware==3.36.0,oslo.policy==1.38.1,oslo.rootwrap==5.14.1,oslo.serialization==2.27.0,oslo.service==1.31.3,oslo.utils==3.36.4,oslotest==3.6.0,packaging==17.1,paramiko==2.4.1,Paste==2.0.3,PasteDeploy==1.5.2,pbr==4.2.0,pep8==1.5.7,prettytable==0.7.2,psycopg2==2.7.5,pyasn1==0.4.3,pycadf==2.8.0,pycparser==2.18,pyflakes==0.8.1,Pygments==2.2.0,pyinotify==0.9.6,pylint==1.4.5,PyMySQL==0.9.2,PyNaCl==1.2.1,pyOpenSSL==18.0.0,pyparsing==2.2.0,pyperclip==1.6.4,python-barbicanclient==4.7.0,python-cinderclient==4.0.1,python-dateutil==2.7.3,python-editor==1.0.3,python-glanceclient==2.12.1,python-heatclient==1.16.1,python-keystoneclient==3.17.0,python-manilaclient==1.24.1,python-mimeparse==1.6.0,python-neutronclient==6.9.0,python-novaclient==11.0.0,python-openstackclient==3.16.0,python-saharaclient==2.0.0,python-subunit==1.3.0,python-swiftclient==3.6.0,pytz==2018.5,PyYAML==3.13,reno==2.9.2,repoze.lru==0.7,requests==2.19.1,requestsexceptions==1.4.0,restructuredtext-lint==1.1.3,rfc3986==1.1.0,Routes==2.4.1,-e git+https://github.com/openstack/sahara.git@efb05b3624044f307168d0b5da888132f51aebb7#egg=sahara,simplejson==3.16.0,six==1.11.0,smmap2==2.0.4,snowballstemmer==1.2.1,Sphinx==1.7.6,sphinxcontrib-httpdomain==1.7.0,sphinxcontrib-websupport==1.1.0,SQLAlchemy==1.2.10,sqlalchemy-migrate==0.11.0,sqlparse==0.2.4,statsd==3.2.2,stestr==2.1.0,stevedore==1.29.0,subprocess32==3.5.2,Tempita==0.5.2,tenacity==4.12.0,testresources==2.0.1,testscenarios==0.5.0,testtools==2.3.0,tooz==1.62.0,traceback2==1.4.0,typing==3.6.4,unicodecsv==0.14.1,unittest2==1.1.0,urllib3==1.23,vine==1.1.4,voluptuous==0.11.1,warlock==1.3.0,wcwidth==0.1.7,WebOb==1.8.2,Werkzeug==0.14.1,wrapt==1.10.11 +py27 runtests: PYTHONHASHSEED='839100177' +py27 runtests: commands[0] | ostestr + +========================= +Failures during discovery +========================= +--- import errors --- +Failed to import test module: sahara.tests.unit.service.edp.spark.test_shell +Traceback (most recent call last): + File "/home/tenobreg/coding/upstream/sahara/sahara/.tox/py27/lib/python2.7/site-packages/unittest2/loader.py", line 456, in _find_test_path + module = self._get_module_from_name(name) + File "/home/tenobreg/coding/upstream/sahara/sahara/.tox/py27/lib/python2.7/site-packages/unittest2/loader.py", line 395, in _get_module_from_name + __import__(name) + File "sahara/tests/unit/service/edp/spark/test_shell.py", line 18, in + from sahara.plugins.spark import shell_engine +ImportError: No module named spark + +Failed to import test module: sahara.tests.unit.service.edp.spark.test_spark +Traceback (most recent call last): + File "/home/tenobreg/coding/upstream/sahara/sahara/.tox/py27/lib/python2.7/site-packages/unittest2/loader.py", line 456, in _find_test_path + module = self._get_module_from_name(name) + File "/home/tenobreg/coding/upstream/sahara/sahara/.tox/py27/lib/python2.7/site-packages/unittest2/loader.py", line 395, in _get_module_from_name + __import__(name) + File "sahara/tests/unit/service/edp/spark/test_spark.py", line 17, in + from sahara.plugins.spark import edp_engine as spark_edp +ImportError: No module named spark + +================================================================================ +The above traceback was encountered during test discovery which imports all the found test modules in the specified test_path. +ERROR: InvocationError: '/home/tenobreg/coding/upstream/sahara/sahara/.tox/py27/bin/ostestr' +___________________________________ summary ____________________________________ +ERROR: py27: commands failed diff --git a/lower-constraints.txt b/lower-constraints.txt new file mode 100644 index 0000000..efaf3f8 --- /dev/null +++ b/lower-constraints.txt @@ -0,0 +1,162 @@ +alabaster==0.7.10 +alembic==0.8.10 +amqp==2.2.2 +appdirs==1.4.3 +asn1crypto==0.24.0 +astroid==1.3.8 +Babel==2.3.4 +bandit==1.1.0 +bashate==0.5.1 +bcrypt==3.1.4 +botocore==1.5.1 +cachetools==2.0.1 +castellan==0.16.0 +certifi==2018.1.18 +cffi==1.11.5 +chardet==3.0.4 +click==6.7 +cliff==2.11.0 +cmd2==0.8.1 +contextlib2==0.5.5 +coverage==4.0 +cryptography==2.1.4 +debtcollector==1.19.0 +decorator==4.2.1 +deprecation==2.0 +doc8==0.6.0 +docutils==0.14 +dogpile.cache==0.6.5 +dulwich==0.19.0 +enum-compat==0.0.2 +eventlet==0.18.2 +extras==1.0.0 +fasteners==0.14.1 +fixtures==3.0.0 +flake8==2.6.2 +Flask==1.0.2 +future==0.16.0 +futurist==1.6.0 +gitdb2==2.0.3 +GitPython==2.1.8 +greenlet==0.4.13 +hacking==1.1.0 +idna==2.6 +imagesize==1.0.0 +iso8601==0.1.11 +itsdangerous==0.24 +Jinja2==2.10 +jmespath==0.9.3 +jsonpatch==1.21 +jsonpointer==2.0 +jsonschema==2.6.0 +keystoneauth1==3.4.0 +keystonemiddleware==4.17.0 +kombu==4.1.0 +linecache2==1.0.0 +logilab-common==1.4.1 +Mako==1.0.7 +MarkupSafe==1.0 +mccabe==0.2.1 +mock==2.0.0 +monotonic==1.4 +mox3==0.25.0 +msgpack==0.5.6 +munch==2.2.0 +netaddr==0.7.19 +netifaces==0.10.6 +openstackdocstheme==1.18.1 +openstacksdk==0.12.0 +os-api-ref==1.4.0 +os-client-config==1.29.0 +os-service-types==1.2.0 +osc-lib==1.10.0 +oslo.cache==1.29.0 +oslo.concurrency==3.26.0 +oslo.config==5.2.0 +oslo.context==2.19.2 +oslo.db==4.27.0 +oslo.i18n==3.15.3 +oslo.log==3.36.0 +oslo.messaging==5.29.0 +oslo.middleware==3.31.0 +oslo.policy==1.30.0 +oslo.rootwrap==5.8.0 +oslo.serialization==2.18.0 +oslo.service==1.24.0 +oslo.upgradecheck==0.1.0 +oslo.utils==3.33.0 +oslotest==3.2.0 +packaging==17.1 +paramiko==2.0.0 +Paste==2.0.3 +PasteDeploy==1.5.2 +pbr==2.0.0 +pika-pool==0.1.3 +pika==0.10.0 +prettytable==0.7.2 +psycopg2==2.6.2 +pyasn1==0.4.2 +pycadf==2.7.0 +pycparser==2.18 +pycodestyle==2.4.0 +pyflakes==0.8.1 +Pygments==2.2.0 +pyinotify==0.9.6 +pylint==1.4.5 +PyMySQL==0.7.6 +PyNaCl==1.2.1 +pyOpenSSL==17.5.0 +pyparsing==2.2.0 +pyperclip==1.6.0 +python-barbicanclient==4.6.0 +python-cinderclient==3.3.0 +python-dateutil==2.7.0 +python-editor==1.0.3 +python-glanceclient==2.8.0 +python-heatclient==1.10.0 +python-keystoneclient==3.8.0 +python-manilaclient==1.16.0 +python-mimeparse==1.6.0 +python-neutronclient==6.7.0 +python-novaclient==9.1.0 +python-openstackclient==3.14.0 +python-saharaclient==1.4.0 +python-subunit==1.2.0 +python-swiftclient==3.2.0 +pytz==2018.3 +PyYAML==3.12 +reno==2.5.0 +repoze.lru==0.7 +requests==2.14.2 +requestsexceptions==1.4.0 +restructuredtext-lint==1.1.3 +rfc3986==1.1.0 +Routes==2.4.1 +simplejson==3.13.2 +six==1.10.0 +smmap2==2.0.3 +snowballstemmer==1.2.1 +Sphinx==1.6.2 +sphinxcontrib-httpdomain==1.3.0 +sphinxcontrib-websupport==1.0.1 +sqlalchemy-migrate==0.11.0 +SQLAlchemy==1.0.10 +sqlparse==0.2.4 +statsd==3.2.2 +stestr==1.0.0 +stevedore==1.20.0 +Tempita==0.5.2 +tenacity==4.9.0 +testresources==2.0.0 +testscenarios==0.4 +testtools==2.2.0 +tooz==1.58.0 +traceback2==1.4.0 +unittest2==1.1.0 +urllib3==1.22 +vine==1.1.4 +voluptuous==0.11.1 +warlock==1.3.0 +WebOb==1.7.1 +Werkzeug==0.14.1 +wrapt==1.10.11 diff --git a/requirements.txt b/requirements.txt index e052228..4cdb2b2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,6 +39,7 @@ python-swiftclient>=3.2.0 # Apache-2.0 python-neutronclient>=6.7.0 # Apache-2.0 python-heatclient>=1.10.0 # Apache-2.0 python-glanceclient>=2.8.0 # Apache-2.0 +sahara six>=1.10.0 # MIT stevedore>=1.20.0 # Apache-2.0 SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=1.0.10 # MIT diff --git a/sahara/__init__.py b/sahara_plugin_vanilla/__init__.py similarity index 100% rename from sahara/__init__.py rename to sahara_plugin_vanilla/__init__.py diff --git a/sahara/i18n.py b/sahara_plugin_vanilla/i18n.py similarity index 100% rename from sahara/i18n.py rename to sahara_plugin_vanilla/i18n.py diff --git a/sahara/plugins/__init__.py b/sahara_plugin_vanilla/plugins/__init__.py similarity index 100% rename from sahara/plugins/__init__.py rename to sahara_plugin_vanilla/plugins/__init__.py diff --git a/sahara/plugins/vanilla/__init__.py b/sahara_plugin_vanilla/plugins/vanilla/__init__.py similarity index 100% rename from sahara/plugins/vanilla/__init__.py rename to sahara_plugin_vanilla/plugins/vanilla/__init__.py diff --git a/sahara/plugins/vanilla/abstractversionhandler.py b/sahara_plugin_vanilla/plugins/vanilla/abstractversionhandler.py similarity index 100% rename from sahara/plugins/vanilla/abstractversionhandler.py rename to sahara_plugin_vanilla/plugins/vanilla/abstractversionhandler.py diff --git a/sahara/plugins/vanilla/confighints_helper.py b/sahara_plugin_vanilla/plugins/vanilla/confighints_helper.py similarity index 75% rename from sahara/plugins/vanilla/confighints_helper.py rename to sahara_plugin_vanilla/plugins/vanilla/confighints_helper.py index 17a900c..b80dd5d 100644 --- a/sahara/plugins/vanilla/confighints_helper.py +++ b/sahara_plugin_vanilla/plugins/vanilla/confighints_helper.py @@ -13,14 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from sahara.service.edp.oozie.workflow_creator import workflow_factory -from sahara.utils import xmlutils +from sahara.plugins import edp +from sahara.plugins import utils def get_possible_hive_config_from(file_name): '''Return the possible configs, args, params for a Hive job.''' config = { - 'configs': xmlutils.load_hadoop_xml_defaults(file_name), + 'configs': utils.load_hadoop_xml_defaults(file_name, + 'sahara_plugin_vanilla'), 'params': {} } return config @@ -31,14 +32,15 @@ def get_possible_mapreduce_config_from(file_name): config = { 'configs': get_possible_pig_config_from(file_name).get('configs') } - config['configs'] += workflow_factory.get_possible_mapreduce_configs() + config['configs'] += edp.get_possible_mapreduce_configs() return config def get_possible_pig_config_from(file_name): '''Return the possible configs, args, params for a Pig job.''' config = { - 'configs': xmlutils.load_hadoop_xml_defaults(file_name), + 'configs': utils.load_hadoop_xml_defaults(file_name, + 'sahara_plugin_vanilla'), 'args': [], 'params': {} } diff --git a/sahara/plugins/vanilla/edp_engine.py b/sahara_plugin_vanilla/plugins/vanilla/edp_engine.py similarity index 89% rename from sahara/plugins/vanilla/edp_engine.py rename to sahara_plugin_vanilla/plugins/vanilla/edp_engine.py index 58989a0..5a61330 100644 --- a/sahara/plugins/vanilla/edp_engine.py +++ b/sahara_plugin_vanilla/plugins/vanilla/edp_engine.py @@ -13,13 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from sahara.plugins import edp from sahara.plugins import exceptions as ex from sahara.plugins import utils as u -from sahara.plugins.vanilla import utils as vu -from sahara.service.edp.oozie import engine as edp_engine +from sahara_plugin_vanilla.plugins.vanilla import utils as vu -class EdpOozieEngine(edp_engine.OozieJobEngine): +class EdpOozieEngine(edp.PluginsOozieJobEngine): def get_hdfs_user(self): return 'hadoop' diff --git a/sahara/plugins/vanilla/hadoop2/__init__.py b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/__init__.py similarity index 100% rename from sahara/plugins/vanilla/hadoop2/__init__.py rename to sahara_plugin_vanilla/plugins/vanilla/hadoop2/__init__.py diff --git a/sahara/plugins/vanilla/hadoop2/config.py b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/config.py similarity index 86% rename from sahara/plugins/vanilla/hadoop2/config.py rename to sahara_plugin_vanilla/plugins/vanilla/hadoop2/config.py index 08d10b4..042aa9d 100644 --- a/sahara/plugins/vanilla/hadoop2/config.py +++ b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/config.py @@ -18,21 +18,16 @@ from oslo_config import cfg from oslo_log import log as logging import six -from sahara import context -from sahara.i18n import _ +from sahara.plugins import castellan_utils as key_manager +from sahara.plugins import context +from sahara.plugins import swift_helper as swift +from sahara.plugins import topology_helper as th from sahara.plugins import utils -from sahara.plugins.vanilla.hadoop2 import config_helper as c_helper -from sahara.plugins.vanilla.hadoop2 import oozie_helper as o_helper -from sahara.plugins.vanilla.hadoop2 import utils as u -from sahara.plugins.vanilla import utils as vu -from sahara.service.castellan import utils as key_manager -from sahara.swift import swift_helper as swift -from sahara.topology import topology_helper as th -from sahara.utils import cluster_progress_ops as cpo -from sahara.utils import configs as s_cfg -from sahara.utils import files as f -from sahara.utils import proxy -from sahara.utils import xmlutils as x +from sahara_plugin_vanilla.i18n import _ +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config_helper +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import oozie_helper +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import utils as u +from sahara_plugin_vanilla.plugins.vanilla import utils as vu CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -61,8 +56,8 @@ def configure_cluster(pctx, cluster): LOG.debug("Configuring cluster") if (CONF.use_identity_api_v3 and CONF.use_domain_for_proxy_users and vu.get_hiveserver(cluster) and - c_helper.is_swift_enabled(pctx, cluster)): - cluster = proxy.create_proxy_user_for_cluster(cluster) + config_helper.is_swift_enabled(pctx, cluster)): + cluster = utils.create_proxy_user_for_cluster(cluster) instances = utils.get_instances(cluster) configure_instances(pctx, instances) @@ -74,7 +69,7 @@ def configure_cluster(pctx, cluster): def configure_zookeeper(cluster, instances=None): zk_servers = vu.get_zk_servers(cluster) if zk_servers: - zk_conf = c_helper.generate_zk_basic_config(cluster) + zk_conf = config_helper.generate_zk_basic_config(cluster) zk_conf += _form_zk_servers_to_quorum(cluster, instances) _push_zk_configs_to_nodes(cluster, zk_conf, instances) @@ -126,7 +121,7 @@ def _push_spark_configs_to_node(cluster, extra): def _push_spark_configs_to_existing_node(spark_master, cluster, extra): - sp_home = c_helper.get_spark_home(cluster) + sp_home = config_helper.get_spark_home(cluster) files = { os.path.join(sp_home, 'conf/spark-env.sh'): extra['sp_master'], @@ -160,13 +155,13 @@ def _extract_spark_configs_to_extra(cluster): config_master = '' if sp_master is not None: - config_master = c_helper.generate_spark_env_configs(cluster) + config_master = config_helper.generate_spark_env_configs(cluster) # Any node that might be used to run spark-submit will need # these libs for swift integration - config_defaults = c_helper.generate_spark_executor_classpath(cluster) + config_defaults = config_helper.generate_spark_executor_classpath(cluster) - extra['job_cleanup'] = c_helper.generate_job_cleanup_config(cluster) + extra['job_cleanup'] = config_helper.generate_job_cleanup_config(cluster) extra['sp_master'] = config_master extra['sp_defaults'] = config_defaults @@ -177,7 +172,7 @@ def configure_instances(pctx, instances): if len(instances) == 0: return - cpo.add_provisioning_step( + utils.add_provisioning_step( instances[0].cluster_id, _("Configure instances"), len(instances)) for instance in instances: @@ -185,7 +180,7 @@ def configure_instances(pctx, instances): _configure_instance(pctx, instance) -@cpo.event_wrapper(True) +@utils.event_wrapper(True) def _configure_instance(pctx, instance): _provisioning_configs(pctx, instance) _post_configuration(pctx, instance) @@ -201,8 +196,8 @@ def _generate_configs(pctx, instance): hadoop_xml_confs = _get_hadoop_configs(pctx, instance) user_xml_confs, user_env_confs = _get_user_configs( pctx, instance.node_group) - xml_confs = s_cfg.merge_configs(user_xml_confs, hadoop_xml_confs) - env_confs = s_cfg.merge_configs(pctx['env_confs'], user_env_confs) + xml_confs = utils.merge_configs(user_xml_confs, hadoop_xml_confs) + env_confs = utils.merge_configs(pctx['env_confs'], user_env_confs) return xml_confs, env_confs @@ -249,20 +244,21 @@ def _get_hadoop_configs(pctx, instance): } confs['Hadoop'].update(hadoop_cfg) - oozie_cfg = o_helper.get_oozie_required_xml_configs(HADOOP_CONF_DIR) - if c_helper.is_mysql_enabled(pctx, cluster): - oozie_cfg.update(o_helper.get_oozie_mysql_configs(cluster)) + oozie_cfg = oozie_helper.get_oozie_required_xml_configs( + HADOOP_CONF_DIR) + if config_helper.is_mysql_enabled(pctx, cluster): + oozie_cfg.update(oozie_helper.get_oozie_mysql_configs(cluster)) confs['JobFlow'] = oozie_cfg - if c_helper.is_swift_enabled(pctx, cluster): + if config_helper.is_swift_enabled(pctx, cluster): swift_configs = {} for config in swift.get_swift_configs(): swift_configs[config['name']] = config['value'] confs['Hadoop'].update(swift_configs) - if c_helper.is_data_locality_enabled(pctx, cluster): + if config_helper.is_data_locality_enabled(pctx, cluster): confs['Hadoop'].update(th.TOPOLOGY_CONFIG) confs['Hadoop'].update({"topology.script.file.name": HADOOP_CONF_DIR + "/topology.sh"}) @@ -277,7 +273,7 @@ def _get_hadoop_configs(pctx, instance): 'jdbc:derby:;databaseName=/opt/hive/metastore_db;create=true' } - if c_helper.is_mysql_enabled(pctx, cluster): + if config_helper.is_mysql_enabled(pctx, cluster): hive_cfg.update({ 'javax.jdo.option.ConnectionURL': 'jdbc:mysql://%s/metastore' % hive_hostname, @@ -291,7 +287,7 @@ def _get_hadoop_configs(pctx, instance): }) proxy_configs = cluster.cluster_configs.get('proxy_configs') - if proxy_configs and c_helper.is_swift_enabled(pctx, cluster): + if proxy_configs and config_helper.is_swift_enabled(pctx, cluster): hive_cfg.update({ swift.HADOOP_SWIFT_USERNAME: proxy_configs['proxy_username'], swift.HADOOP_SWIFT_PASSWORD: key_manager.get_secret( @@ -311,8 +307,8 @@ def _get_user_configs(pctx, node_group): cl_xml_confs, cl_env_confs = _separate_configs( node_group.cluster.cluster_configs, pctx['env_confs']) - xml_confs = s_cfg.merge_configs(cl_xml_confs, ng_xml_confs) - env_confs = s_cfg.merge_configs(cl_env_confs, ng_env_confs) + xml_confs = utils.merge_configs(cl_xml_confs, ng_xml_confs) + env_confs = utils.merge_configs(cl_env_confs, ng_env_confs) return xml_confs, env_confs @@ -336,7 +332,7 @@ def _separate_configs(configs, all_env_configs): def _generate_xml(configs): xml_confs = {} for service, confs in six.iteritems(configs): - xml_confs[service] = x.create_hadoop_xml(confs) + xml_confs[service] = utils.create_hadoop_xml(confs) return xml_confs @@ -417,8 +413,9 @@ def _post_configuration(pctx, instance): 'hadoop_secure_dn_log_dir': dirs['hadoop_secure_dn_log_dir'], 'yarn_log_dir': dirs['yarn_log_dir'] } - post_conf_script = f.get_file_text( - 'plugins/vanilla/hadoop2/resources/post_conf.template') + post_conf_script = utils.get_file_text( + 'plugins/vanilla/hadoop2/resources/post_conf.template', + 'sahara_plugin_vanilla') post_conf_script = post_conf_script.format(**args) with instance.remote() as r: @@ -426,12 +423,11 @@ def _post_configuration(pctx, instance): r.execute_command('chmod +x /tmp/post_conf.sh') r.execute_command('sudo /tmp/post_conf.sh') - if c_helper.is_data_locality_enabled(pctx, - instance.cluster): + if config_helper.is_data_locality_enabled(pctx, instance.cluster): t_script = HADOOP_CONF_DIR + '/topology.sh' - r.write_file_to(t_script, f.get_file_text( - 'plugins/vanilla/hadoop2/resources/topology.sh'), - run_as_root=True) + r.write_file_to(t_script, utils.get_file_text( + 'plugins/vanilla/hadoop2/resources/topology.sh', + 'sahara_plugin_vanilla'), run_as_root=True) r.execute_command('chmod +x ' + t_script, run_as_root=True) @@ -456,10 +452,10 @@ def _make_hadoop_paths(paths, hadoop_dir): return [path + hadoop_dir for path in paths] -@cpo.event_wrapper( +@utils.event_wrapper( True, step=_("Configure topology data"), param=('cluster', 1)) def configure_topology_data(pctx, cluster): - if c_helper.is_data_locality_enabled(pctx, cluster): + if config_helper.is_data_locality_enabled(pctx, cluster): LOG.warning("Node group awareness is not implemented in YARN yet " "so enable_hypervisor_awareness set to False explicitly") tpl_map = th.generate_topology_map(cluster, is_node_awareness=False) diff --git a/sahara/plugins/vanilla/hadoop2/config_helper.py b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/config_helper.py similarity index 95% rename from sahara/plugins/vanilla/hadoop2/config_helper.py rename to sahara_plugin_vanilla/plugins/vanilla/hadoop2/config_helper.py index 007ac94..3fc3398 100644 --- a/sahara/plugins/vanilla/hadoop2/config_helper.py +++ b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/config_helper.py @@ -1,4 +1,4 @@ -# Copyright (c) 2014 Mirantis Inc. +# Copyright (c) 2014 Mirantis, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,12 +16,10 @@ from oslo_config import cfg import six -from sahara import exceptions as ex -from sahara.i18n import _ +from sahara.plugins import exceptions as ex from sahara.plugins import provisioning as p from sahara.plugins import utils -from sahara.utils import files as f -from sahara.utils import types +from sahara_plugin_vanilla.i18n import _ CONF = cfg.CONF CONF.import_opt("enable_data_locality", "sahara.topology.topology_helper") @@ -165,7 +163,7 @@ def init_xml_configs(xml_confs): if cfg.default_value in ["true", "false"]: cfg.config_type = "bool" cfg.default_value = (cfg.default_value == 'true') - elif types.is_int(cfg.default_value): + elif utils.is_int(cfg.default_value): cfg.config_type = "int" cfg.default_value = int(cfg.default_value) if config['name'] in CLUSTER_WIDE_CONFS: @@ -245,7 +243,7 @@ def get_config_value(pctx, service, name, cluster=None): if c.applicable_target == service and c.name == name: return c.default_value - raise ex.NotFoundException( + raise ex.PluginNotFoundException( {"name": name, "service": service}, _("Unable to get parameter '%(name)s' from service %(service)s")) @@ -303,10 +301,12 @@ def generate_job_cleanup_config(cluster): (args['minimum_cleanup_megabytes'] > 0 and args['minimum_cleanup_seconds'] > 0))} if job_conf['valid']: - job_conf['cron'] = f.get_file_text( - 'plugins/vanilla/hadoop2/resources/spark-cleanup.cron'), - job_cleanup_script = f.get_file_text( - 'plugins/vanilla/hadoop2/resources/tmp-cleanup.sh.template') + job_conf['cron'] = utils.get_file_text( + 'plugins/vanilla/hadoop2/resources/spark-cleanup.cron', + 'sahara_plugin_vanilla'), + job_cleanup_script = utils.get_file_text( + 'plugins/vanilla/hadoop2/resources/tmp-cleanup.sh.template', + 'sahara_plugin_vanilla') job_conf['script'] = job_cleanup_script.format(**args) return job_conf @@ -324,6 +324,7 @@ def generate_zk_basic_config(cluster): 'synclimit': utils.get_config_value_or_default( "ZooKeeper", "syncLimit", cluster) } - zoo_cfg = f.get_file_text( - 'plugins/vanilla/hadoop2/resources/zoo_sample.cfg') + zoo_cfg = utils.get_file_text( + 'plugins/vanilla/hadoop2/resources/zoo_sample.cfg', + 'sahara_plugin_vanilla') return zoo_cfg.format(**args) diff --git a/sahara/plugins/vanilla/hadoop2/edp_engine.py b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/edp_engine.py similarity index 82% rename from sahara/plugins/vanilla/hadoop2/edp_engine.py rename to sahara_plugin_vanilla/plugins/vanilla/hadoop2/edp_engine.py index 8698db7..916e2f2 100644 --- a/sahara/plugins/vanilla/hadoop2/edp_engine.py +++ b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/edp_engine.py @@ -13,14 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from sahara.plugins.vanilla import edp_engine -from sahara.service.edp import hdfs_helper +from sahara.plugins import edp +from sahara_plugin_vanilla.plugins.vanilla import edp_engine class EdpOozieEngine(edp_engine.EdpOozieEngine): def create_hdfs_dir(self, remote, dir_name): - hdfs_helper.create_dir_hadoop2(remote, dir_name, self.get_hdfs_user()) + edp.create_dir_hadoop2(remote, dir_name, self.get_hdfs_user()) def get_resource_manager_uri(self, cluster): return cluster['info']['YARN']['ResourceManager'] diff --git a/sahara/plugins/vanilla/hadoop2/keypairs.py b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/keypairs.py similarity index 90% rename from sahara/plugins/vanilla/hadoop2/keypairs.py rename to sahara_plugin_vanilla/plugins/vanilla/hadoop2/keypairs.py index 465c4cf..32a4d4f 100644 --- a/sahara/plugins/vanilla/hadoop2/keypairs.py +++ b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/keypairs.py @@ -18,12 +18,10 @@ from castellan import key_manager from oslo_log import log as logging -from sahara import conductor -from sahara import context -from sahara.utils import cluster as utils -from sahara.utils import crypto +from sahara.plugins import conductor +from sahara.plugins import context +from sahara.plugins import utils -cond = conductor.API LOG = logging.getLogger(__name__) @@ -72,15 +70,15 @@ def provision_keypairs(cluster, instances=None): # cluster created before mitaka, skipping provisioning return if not keypair: - private, public = crypto.generate_key_pair() + private, public = utils.generate_key_pair() keypair = {'public': public, 'private': private} extra['vanilla_keypair'] = keypair extra['vanilla_keypair']['private'] = _store_secret( keypair['private']) - cond.cluster_update(context.ctx(), cluster, {'extra': extra}) + conductor.cluster_update(context.ctx(), cluster, {'extra': extra}) else: keypair['private'] = _get_secret(keypair['private']) - with context.ThreadGroup() as tg: + with context.PluginsThreadGroup() as tg: for instance in instances: tg.spawn( 'provision-key-%s' % instance.instance_name, diff --git a/sahara/plugins/vanilla/hadoop2/oozie_helper.py b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/oozie_helper.py similarity index 96% rename from sahara/plugins/vanilla/hadoop2/oozie_helper.py rename to sahara_plugin_vanilla/plugins/vanilla/hadoop2/oozie_helper.py index 38e47b1..6be74ab 100644 --- a/sahara/plugins/vanilla/hadoop2/oozie_helper.py +++ b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/oozie_helper.py @@ -12,7 +12,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. -from sahara.plugins.vanilla.hadoop2 import utils as u +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import utils as u def get_oozie_required_xml_configs(hadoop_conf_dir): diff --git a/sahara/plugins/vanilla/hadoop2/recommendations_utils.py b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/recommendations_utils.py similarity index 100% rename from sahara/plugins/vanilla/hadoop2/recommendations_utils.py rename to sahara_plugin_vanilla/plugins/vanilla/hadoop2/recommendations_utils.py diff --git a/sahara/plugins/vanilla/hadoop2/resources/create_oozie_db.sql b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/resources/create_oozie_db.sql similarity index 100% rename from sahara/plugins/vanilla/hadoop2/resources/create_oozie_db.sql rename to sahara_plugin_vanilla/plugins/vanilla/hadoop2/resources/create_oozie_db.sql diff --git a/sahara/plugins/vanilla/hadoop2/resources/post_conf.template b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/resources/post_conf.template similarity index 100% rename from sahara/plugins/vanilla/hadoop2/resources/post_conf.template rename to sahara_plugin_vanilla/plugins/vanilla/hadoop2/resources/post_conf.template diff --git a/sahara/plugins/vanilla/hadoop2/resources/spark-cleanup.cron b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/resources/spark-cleanup.cron similarity index 100% rename from sahara/plugins/vanilla/hadoop2/resources/spark-cleanup.cron rename to sahara_plugin_vanilla/plugins/vanilla/hadoop2/resources/spark-cleanup.cron diff --git a/sahara/plugins/vanilla/hadoop2/resources/tmp-cleanup.sh.template b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/resources/tmp-cleanup.sh.template similarity index 100% rename from sahara/plugins/vanilla/hadoop2/resources/tmp-cleanup.sh.template rename to sahara_plugin_vanilla/plugins/vanilla/hadoop2/resources/tmp-cleanup.sh.template diff --git a/sahara/plugins/vanilla/hadoop2/resources/topology.sh b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/resources/topology.sh similarity index 100% rename from sahara/plugins/vanilla/hadoop2/resources/topology.sh rename to sahara_plugin_vanilla/plugins/vanilla/hadoop2/resources/topology.sh diff --git a/sahara/plugins/vanilla/hadoop2/resources/zoo_sample.cfg b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/resources/zoo_sample.cfg similarity index 100% rename from sahara/plugins/vanilla/hadoop2/resources/zoo_sample.cfg rename to sahara_plugin_vanilla/plugins/vanilla/hadoop2/resources/zoo_sample.cfg diff --git a/sahara/plugins/vanilla/hadoop2/run_scripts.py b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/run_scripts.py similarity index 81% rename from sahara/plugins/vanilla/hadoop2/run_scripts.py rename to sahara_plugin_vanilla/plugins/vanilla/hadoop2/run_scripts.py index 48e61c5..31f9504 100644 --- a/sahara/plugins/vanilla/hadoop2/run_scripts.py +++ b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/run_scripts.py @@ -17,34 +17,31 @@ import os from oslo_log import log as logging -from sahara import context -from sahara.i18n import _ -from sahara.plugins import utils as pu -from sahara.plugins.vanilla.hadoop2 import config_helper as c_helper -from sahara.plugins.vanilla.hadoop2 import oozie_helper -from sahara.plugins.vanilla.hadoop2 import utils as u -from sahara.plugins.vanilla import utils as vu -from sahara.utils import cluster_progress_ops as cpo -from sahara.utils import edp -from sahara.utils import files -from sahara.utils import poll_utils +from sahara.plugins import context +from sahara.plugins import edp +from sahara.plugins import utils +from sahara_plugin_vanilla.i18n import _ +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config_helper +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import oozie_helper +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import utils as u +from sahara_plugin_vanilla.plugins.vanilla import utils as vu LOG = logging.getLogger(__name__) def start_dn_nm_processes(instances): filternames = ['datanode', 'nodemanager'] - instances = pu.instances_with_services(instances, filternames) + instances = utils.instances_with_services(instances, filternames) if len(instances) == 0: return - cpo.add_provisioning_step( + utils.add_provisioning_step( instances[0].cluster_id, - pu.start_process_event_message("DataNodes, NodeManagers"), + utils.start_process_event_message("DataNodes, NodeManagers"), len(instances)) - with context.ThreadGroup() as tg: + with context.PluginsThreadGroup() as tg: for instance in instances: with context.set_current_instance_id(instance.instance_id): processes = set(instance.node_group.node_processes) @@ -53,7 +50,7 @@ def start_dn_nm_processes(instances): _start_processes, instance, list(processes)) -@cpo.event_wrapper(True) +@utils.event_wrapper(True) def _start_processes(instance, processes): with instance.remote() as r: if 'datanode' in processes: @@ -74,21 +71,23 @@ def start_yarn_process(instance, process): 'sudo su - -c "yarn-daemon.sh start %s" hadoop' % process) -@cpo.event_wrapper(True, step=pu.start_process_event_message("HistoryServer")) +@utils.event_wrapper( + True, step=utils.start_process_event_message("HistoryServer")) def start_historyserver(instance): instance.remote().execute_command( 'sudo su - -c "mr-jobhistory-daemon.sh start historyserver" hadoop') -@cpo.event_wrapper(True, step=pu.start_process_event_message("Oozie")) +@utils.event_wrapper(True, step=utils.start_process_event_message("Oozie")) def start_oozie_process(pctx, instance): with context.set_current_instance_id(instance.instance_id): with instance.remote() as r: - if c_helper.is_mysql_enabled(pctx, instance.cluster): + if config_helper.is_mysql_enabled(pctx, instance.cluster): _start_mysql(r) LOG.debug("Creating Oozie DB Schema") - sql_script = files.get_file_text( - 'plugins/vanilla/hadoop2/resources/create_oozie_db.sql') + sql_script = utils.get_file_text( + 'plugins/vanilla/hadoop2/resources/create_oozie_db.sql', + 'sahara_plugin_vanilla') password = oozie_helper.get_oozie_mysql_configs( instance.cluster)[ @@ -105,10 +104,10 @@ def start_oozie_process(pctx, instance): _start_oozie(r) -@cpo.event_wrapper( - True, step=pu.start_process_event_message("Spark History Server")) +@utils.event_wrapper( + True, step=utils.start_process_event_message("Spark History Server")) def start_spark_history_server(master): - sp_home = c_helper.get_spark_home(master.cluster) + sp_home = config_helper.get_spark_home(master.cluster) with context.set_current_instance_id(master.instance_id): with master.remote() as r: r.execute_command('sudo su - -c "bash %s" hadoop' % os.path.join( @@ -116,12 +115,12 @@ def start_spark_history_server(master): def start_zk_server(instances): - cpo.add_provisioning_step( + utils.add_provisioning_step( instances[0].cluster_id, - pu.start_process_event_message("ZooKeeper"), + utils.start_process_event_message("ZooKeeper"), len(instances)) - with context.ThreadGroup() as tg: + with context.PluginsThreadGroup() as tg: for instance in instances: with context.set_current_instance_id(instance.instance_id): tg.spawn('ZK-start-processes-%s' % instance.instance_name, @@ -135,19 +134,19 @@ def refresh_zk_servers(cluster, to_delete_instances=None): if instance in instances: instances.remove(instance) - cpo.add_provisioning_step( + utils.add_provisioning_step( cluster.id, - pu.start_process_event_message("ZooKeeper"), + utils.start_process_event_message("ZooKeeper"), len(instances)) - with context.ThreadGroup() as tg: + with context.PluginsThreadGroup() as tg: for instance in instances: with context.set_current_instance_id(instance.instance_id): tg.spawn('ZK-restart-processes-%s' % instance.instance_name, _start_zk_processes, instance, 'restart') -@cpo.event_wrapper(True) +@utils.event_wrapper(True) def _start_zk_processes(instance, operation): with instance.remote() as r: r.execute_command( @@ -160,15 +159,16 @@ def format_namenode(instance): 'sudo su - -c "hdfs namenode -format" hadoop') -@cpo.event_wrapper( - True, step=pu.start_process_event_message("Oozie"), param=('cluster', 0)) +@utils.event_wrapper( + True, + step=utils.start_process_event_message("Oozie"), param=('cluster', 0)) def refresh_hadoop_nodes(cluster): nn = vu.get_namenode(cluster) nn.remote().execute_command( 'sudo su - -c "hdfs dfsadmin -refreshNodes" hadoop') -@cpo.event_wrapper( +@utils.event_wrapper( True, step=_("Refresh %s nodes") % "YARN", param=('cluster', 0)) def refresh_yarn_nodes(cluster): rm = vu.get_resourcemanager(cluster) @@ -210,7 +210,7 @@ def _start_oozie(remote): 'sudo su - -c "/opt/oozie/bin/oozied.sh start" hadoop') -@cpo.event_wrapper( +@utils.event_wrapper( True, step=_("Await %s start up") % "DataNodes", param=('cluster', 0)) def await_datanodes(cluster): datanodes_count = len(vu.get_datanodes(cluster)) @@ -219,9 +219,9 @@ def await_datanodes(cluster): l_message = _("Waiting on %s datanodes to start up") % datanodes_count with vu.get_namenode(cluster).remote() as r: - poll_utils.plugin_option_poll( + utils.plugin_option_poll( cluster, _check_datanodes_count, - c_helper.DATANODES_STARTUP_TIMEOUT, l_message, 1, { + config_helper.DATANODES_STARTUP_TIMEOUT, l_message, 1, { 'remote': r, 'count': datanodes_count}) @@ -265,7 +265,8 @@ def _hive_metastore_start(remote): " --service metastore > /dev/null &' hadoop") -@cpo.event_wrapper(True, step=pu.start_process_event_message("HiveServer")) +@utils.event_wrapper( + True, step=utils.start_process_event_message("HiveServer")) def start_hiveserver_process(pctx, instance): with context.set_current_instance_id(instance.instance_id): with instance.remote() as r: @@ -273,16 +274,15 @@ def start_hiveserver_process(pctx, instance): _hive_copy_shared_conf( r, edp.get_hive_shared_conf_path('hadoop')) - if c_helper.is_mysql_enabled(pctx, instance.cluster): + if config_helper.is_mysql_enabled(pctx, instance.cluster): oozie = vu.get_oozie(instance.node_group.cluster) if not oozie or instance.hostname() != oozie.hostname(): _start_mysql(r) version = instance.cluster.hadoop_version - sql_script = files.get_file_text( + sql_script = utils.get_file_text( 'plugins/vanilla/v{}/resources/create_hive_db.sql'.format( - version.replace('.', '_')) - ) + version.replace('.', '_')), 'sahara_plugin_vanilla') sql_script = sql_script.replace( '{{password}}', u.get_hive_password(instance.cluster)) diff --git a/sahara/plugins/vanilla/hadoop2/scaling.py b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/scaling.py similarity index 79% rename from sahara/plugins/vanilla/hadoop2/scaling.py rename to sahara_plugin_vanilla/plugins/vanilla/hadoop2/scaling.py index 3911dc4..ea2b420 100644 --- a/sahara/plugins/vanilla/hadoop2/scaling.py +++ b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/scaling.py @@ -13,17 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from sahara.i18n import _ -from sahara.plugins import utils as u -from sahara.plugins.vanilla.hadoop2 import config -from sahara.plugins.vanilla.hadoop2 import config_helper as c_helper -from sahara.plugins.vanilla.hadoop2 import run_scripts as run -from sahara.plugins.vanilla.hadoop2 import utils as pu -from sahara.plugins.vanilla import utils as vu -from sahara.swift import swift_helper -from sahara.utils import cluster_progress_ops as cpo -from sahara.utils import poll_utils - +from sahara.plugins import swift_helper +from sahara.plugins import utils +from sahara_plugin_vanilla.i18n import _ +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config_helper +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import run_scripts as run +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import utils as pu +from sahara_plugin_vanilla.plugins.vanilla import utils as vu HADOOP_CONF_DIR = config.HADOOP_CONF_DIR @@ -48,19 +45,20 @@ def _get_instances_with_service(instances, service): if service in instance.node_group.node_processes] -@cpo.event_wrapper(True, step=_("Update include files"), param=('cluster', 0)) +@utils.event_wrapper( + True, step=_("Update include files"), param=('cluster', 0)) def _update_include_files(cluster, dec_instances=None): dec_instances = dec_instances or [] dec_instances_ids = [instance.id for instance in dec_instances] - instances = u.get_instances(cluster) + instances = utils.get_instances(cluster) inst_filter = lambda inst: inst.id not in dec_instances_ids datanodes = filter(inst_filter, vu.get_datanodes(cluster)) nodemanagers = filter(inst_filter, vu.get_nodemanagers(cluster)) - dn_hosts = u.generate_fqdn_host_names(datanodes) - nm_hosts = u.generate_fqdn_host_names(nodemanagers) + dn_hosts = utils.generate_fqdn_host_names(datanodes) + nm_hosts = utils.generate_fqdn_host_names(nodemanagers) for instance in instances: with instance.remote() as r: r.execute_command( @@ -97,9 +95,9 @@ def decommission_nodes(pctx, cluster, instances): def _update_exclude_files(cluster, instances): datanodes = _get_instances_with_service(instances, 'datanode') nodemanagers = _get_instances_with_service(instances, 'nodemanager') - dn_hosts = u.generate_fqdn_host_names(datanodes) - nm_hosts = u.generate_fqdn_host_names(nodemanagers) - for instance in u.get_instances(cluster): + dn_hosts = utils.generate_fqdn_host_names(datanodes) + nm_hosts = utils.generate_fqdn_host_names(nodemanagers) + for instance in utils.get_instances(cluster): with instance.remote() as r: r.execute_command( 'sudo su - -c "echo \'%s\' > %s/dn-exclude" hadoop' % ( @@ -110,7 +108,7 @@ def _update_exclude_files(cluster, instances): def _clear_exclude_files(cluster): - for instance in u.get_instances(cluster): + for instance in utils.get_instances(cluster): with instance.remote() as r: r.execute_command( 'sudo su - -c "echo > %s/dn-exclude" hadoop' % HADOOP_CONF_DIR) @@ -127,21 +125,21 @@ def is_decommissioned(cluster, check_func, instances): def _check_decommission(cluster, instances, check_func, option): - poll_utils.plugin_option_poll( + utils.plugin_option_poll( cluster, is_decommissioned, option, _("Wait for decommissioning"), 5, {'cluster': cluster, 'check_func': check_func, 'instances': instances}) -@cpo.event_wrapper( +@utils.event_wrapper( True, step=_("Decommission %s") % "NodeManagers", param=('cluster', 0)) def _check_nodemanagers_decommission(cluster, instances): _check_decommission(cluster, instances, pu.get_nodemanagers_status, - c_helper.NODEMANAGERS_DECOMMISSIONING_TIMEOUT) + config_helper.NODEMANAGERS_DECOMMISSIONING_TIMEOUT) -@cpo.event_wrapper( +@utils.event_wrapper( True, step=_("Decommission %s") % "DataNodes", param=('cluster', 0)) def _check_datanodes_decommission(cluster, instances): _check_decommission(cluster, instances, pu.get_datanodes_status, - c_helper.DATANODES_DECOMMISSIONING_TIMEOUT) + config_helper.DATANODES_DECOMMISSIONING_TIMEOUT) diff --git a/sahara/plugins/vanilla/hadoop2/starting_scripts.py b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/starting_scripts.py similarity index 90% rename from sahara/plugins/vanilla/hadoop2/starting_scripts.py rename to sahara_plugin_vanilla/plugins/vanilla/hadoop2/starting_scripts.py index 91b1a1a..2045f68 100644 --- a/sahara/plugins/vanilla/hadoop2/starting_scripts.py +++ b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/starting_scripts.py @@ -14,9 +14,8 @@ # limitations under the License. from sahara.plugins import utils -from sahara.plugins.vanilla.hadoop2 import run_scripts as run -from sahara.plugins.vanilla import utils as vu -from sahara.utils import cluster_progress_ops as cpo +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import run_scripts as run +from sahara_plugin_vanilla.plugins.vanilla import utils as vu def start_namenode(cluster): @@ -24,7 +23,7 @@ def start_namenode(cluster): _start_namenode(nn) -@cpo.event_wrapper( +@utils.event_wrapper( True, step=utils.start_process_event_message('NameNode')) def _start_namenode(nn): run.format_namenode(nn) @@ -37,7 +36,7 @@ def start_secondarynamenode(cluster): _start_secondarynamenode(snn) -@cpo.event_wrapper( +@utils.event_wrapper( True, step=utils.start_process_event_message("SecondaryNameNodes")) def _start_secondarynamenode(snn): run.start_hadoop_process(snn, 'secondarynamenode') @@ -49,7 +48,7 @@ def start_resourcemanager(cluster): _start_resourcemanager(rm) -@cpo.event_wrapper( +@utils.event_wrapper( True, step=utils.start_process_event_message('ResourceManager')) def _start_resourcemanager(snn): run.start_yarn_process(snn, 'resourcemanager') diff --git a/sahara/plugins/vanilla/hadoop2/utils.py b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/utils.py similarity index 93% rename from sahara/plugins/vanilla/hadoop2/utils.py rename to sahara_plugin_vanilla/plugins/vanilla/hadoop2/utils.py index 8994695..ecfc062 100644 --- a/sahara/plugins/vanilla/hadoop2/utils.py +++ b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/utils.py @@ -17,12 +17,11 @@ import re from oslo_log import log as logging -from sahara import conductor as cond -from sahara import context -from sahara.plugins.vanilla import utils as u -from sahara.service.castellan import utils as castellan +from sahara.plugins import castellan_utils as castellan +from sahara.plugins import conductor +from sahara.plugins import context +from sahara_plugin_vanilla.plugins.vanilla import utils as u -conductor = cond.API LOG = logging.getLogger(__name__) diff --git a/sahara/plugins/vanilla/hadoop2/validation.py b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/validation.py similarity index 95% rename from sahara/plugins/vanilla/hadoop2/validation.py rename to sahara_plugin_vanilla/plugins/vanilla/hadoop2/validation.py index 400f5f4..0872840 100644 --- a/sahara/plugins/vanilla/hadoop2/validation.py +++ b/sahara_plugin_vanilla/plugins/vanilla/hadoop2/validation.py @@ -13,12 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from sahara.i18n import _ from sahara.plugins import exceptions as ex from sahara.plugins import utils as u -from sahara.plugins.vanilla.hadoop2 import config_helper as cu -from sahara.plugins.vanilla import utils as vu -from sahara.utils import general as gu +from sahara_plugin_vanilla.i18n import _ +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config_helper as cu +from sahara_plugin_vanilla.plugins.vanilla import utils as vu def validate_cluster_creating(pctx, cluster): @@ -95,7 +94,7 @@ def validate_additional_ng_scaling(cluster, additional): scalable_processes = _get_scalable_processes() for ng_id in additional: - ng = gu.get_by_id(cluster.node_groups, ng_id) + ng = u.get_by_id(cluster.node_groups, ng_id) if not set(ng.node_processes).issubset(scalable_processes): msg = _("Vanilla plugin cannot scale nodegroup with processes: %s") raise ex.NodeGroupCannotBeScaled(ng.name, @@ -140,7 +139,7 @@ def validate_zookeeper_node_count(zk_ng, existing, additional): zk_amount += ng.count for ng_id in additional: - ng = gu.get_by_id(zk_ng, ng_id) + ng = u.get_by_id(zk_ng, ng_id) if "zookeeper" in ng.node_processes: zk_amount += ng.count diff --git a/sahara/plugins/vanilla/plugin.py b/sahara_plugin_vanilla/plugins/vanilla/plugin.py similarity index 97% rename from sahara/plugins/vanilla/plugin.py rename to sahara_plugin_vanilla/plugins/vanilla/plugin.py index 3e4492f..03f2726 100644 --- a/sahara/plugins/vanilla/plugin.py +++ b/sahara_plugin_vanilla/plugins/vanilla/plugin.py @@ -15,9 +15,9 @@ import copy -from sahara.i18n import _ from sahara.plugins import provisioning as p -from sahara.plugins.vanilla import versionfactory as vhf +from sahara_plugin_vanilla.i18n import _ +from sahara_plugin_vanilla.plugins.vanilla import versionfactory as vhf class VanillaProvider(p.ProvisioningPluginBase): diff --git a/sahara/plugins/vanilla/utils.py b/sahara_plugin_vanilla/plugins/vanilla/utils.py similarity index 96% rename from sahara/plugins/vanilla/utils.py rename to sahara_plugin_vanilla/plugins/vanilla/utils.py index f18d188..75746b2 100644 --- a/sahara/plugins/vanilla/utils.py +++ b/sahara_plugin_vanilla/plugins/vanilla/utils.py @@ -16,8 +16,8 @@ from oslo_utils import uuidutils +from sahara.plugins import castellan_utils as castellan from sahara.plugins import utils as u -from sahara.service.castellan import utils as castellan def get_namenode(cluster): diff --git a/sahara/plugins/vanilla/v2_7_1/__init__.py b/sahara_plugin_vanilla/plugins/vanilla/v2_7_1/__init__.py similarity index 100% rename from sahara/plugins/vanilla/v2_7_1/__init__.py rename to sahara_plugin_vanilla/plugins/vanilla/v2_7_1/__init__.py diff --git a/sahara/plugins/vanilla/v2_7_1/config_helper.py b/sahara_plugin_vanilla/plugins/vanilla/v2_7_1/config_helper.py similarity index 74% rename from sahara/plugins/vanilla/v2_7_1/config_helper.py rename to sahara_plugin_vanilla/plugins/vanilla/v2_7_1/config_helper.py index 3c1f060..9d91872 100644 --- a/sahara/plugins/vanilla/v2_7_1/config_helper.py +++ b/sahara_plugin_vanilla/plugins/vanilla/v2_7_1/config_helper.py @@ -19,34 +19,40 @@ from oslo_config import cfg import six from sahara.plugins import provisioning as p -from sahara.plugins.vanilla.hadoop2 import config_helper as c_helper -from sahara.utils import xmlutils as x +from sahara.plugins import utils +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config_helper CONF = cfg.CONF CONF.import_opt("enable_data_locality", "sahara.topology.topology_helper") -CORE_DEFAULT = x.load_hadoop_xml_defaults( - 'plugins/vanilla/v2_7_1/resources/core-default.xml') +CORE_DEFAULT = utils.load_hadoop_xml_defaults( + 'plugins/vanilla/v2_7_1/resources/core-default.xml', + 'sahara_plugin_vanilla') -HDFS_DEFAULT = x.load_hadoop_xml_defaults( - 'plugins/vanilla/v2_7_1/resources/hdfs-default.xml') +HDFS_DEFAULT = utils.load_hadoop_xml_defaults( + 'plugins/vanilla/v2_7_1/resources/hdfs-default.xml', + 'sahara_plugin_vanilla') -MAPRED_DEFAULT = x.load_hadoop_xml_defaults( - 'plugins/vanilla/v2_7_1/resources/mapred-default.xml') +MAPRED_DEFAULT = utils.load_hadoop_xml_defaults( + 'plugins/vanilla/v2_7_1/resources/mapred-default.xml', + 'sahara_plugin_vanilla') -YARN_DEFAULT = x.load_hadoop_xml_defaults( - 'plugins/vanilla/v2_7_1/resources/yarn-default.xml') +YARN_DEFAULT = utils.load_hadoop_xml_defaults( + 'plugins/vanilla/v2_7_1/resources/yarn-default.xml', + 'sahara_plugin_vanilla') -OOZIE_DEFAULT = x.load_hadoop_xml_defaults( - 'plugins/vanilla/v2_7_1/resources/oozie-default.xml') +OOZIE_DEFAULT = utils.load_hadoop_xml_defaults( + 'plugins/vanilla/v2_7_1/resources/oozie-default.xml', + 'sahara_plugin_vanilla') -HIVE_DEFAULT = x.load_hadoop_xml_defaults( - 'plugins/vanilla/v2_7_1/resources/hive-default.xml') +HIVE_DEFAULT = utils.load_hadoop_xml_defaults( + 'plugins/vanilla/v2_7_1/resources/hive-default.xml', + 'sahara_plugin_vanilla') _default_executor_classpath = ":".join( ['/opt/hadoop/share/hadoop/tools/lib/hadoop-openstack-2.7.1.jar']) -SPARK_CONFS = copy.deepcopy(c_helper.SPARK_CONFS) +SPARK_CONFS = copy.deepcopy(config_helper.SPARK_CONFS) SPARK_CONFS['Spark']['OPTIONS'].append( { @@ -87,15 +93,15 @@ ENV_CONFS = { } # Initialise plugin Hadoop configurations -PLUGIN_XML_CONFIGS = c_helper.init_xml_configs(XML_CONFS) -PLUGIN_ENV_CONFIGS = c_helper.init_env_configs(ENV_CONFS) +PLUGIN_XML_CONFIGS = config_helper.init_xml_configs(XML_CONFS) +PLUGIN_ENV_CONFIGS = config_helper.init_env_configs(ENV_CONFS) def _init_all_configs(): configs = [] configs.extend(PLUGIN_XML_CONFIGS) configs.extend(PLUGIN_ENV_CONFIGS) - configs.extend(c_helper.PLUGIN_GENERAL_CONFIGS) + configs.extend(config_helper.PLUGIN_GENERAL_CONFIGS) configs.extend(_get_spark_configs()) configs.extend(_get_zookeeper_configs()) return configs @@ -124,7 +130,7 @@ def _get_spark_configs(): def _get_zookeeper_configs(): zk_configs = [] - for service, config_items in six.iteritems(c_helper.ZOOKEEPER_CONFS): + for service, config_items in six.iteritems(config_helper.ZOOKEEPER_CONFS): for item in config_items['OPTIONS']: cfg = p.Config(name=item["name"], description=item["description"], diff --git a/sahara/plugins/vanilla/v2_7_1/edp_engine.py b/sahara_plugin_vanilla/plugins/vanilla/v2_7_1/edp_engine.py similarity index 79% rename from sahara/plugins/vanilla/v2_7_1/edp_engine.py rename to sahara_plugin_vanilla/plugins/vanilla/v2_7_1/edp_engine.py index 5ca1dd2..8e5cb42 100644 --- a/sahara/plugins/vanilla/v2_7_1/edp_engine.py +++ b/sahara_plugin_vanilla/plugins/vanilla/v2_7_1/edp_engine.py @@ -14,34 +14,33 @@ # limitations under the License. import os -from sahara import exceptions as ex -from sahara.i18n import _ +from sahara.plugins import edp +from sahara.plugins import exceptions as ex from sahara.plugins import utils as plugin_utils -from sahara.plugins.vanilla import confighints_helper as ch_helper -from sahara.plugins.vanilla.hadoop2 import edp_engine -from sahara.plugins.vanilla import utils as v_utils -from sahara.service.edp.spark import engine as edp_spark_engine -from sahara.utils import edp +from sahara_plugin_vanilla.i18n import _ +from sahara_plugin_vanilla.plugins.vanilla import confighints_helper as chh +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import edp_engine +from sahara_plugin_vanilla.plugins.vanilla import utils as v_utils class EdpOozieEngine(edp_engine.EdpOozieEngine): @staticmethod def get_possible_job_config(job_type): if edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE): - return {'job_config': ch_helper.get_possible_hive_config_from( + return {'job_config': chh.get_possible_hive_config_from( 'plugins/vanilla/v2_7_1/resources/hive-default.xml')} if edp.compare_job_type(job_type, edp.JOB_TYPE_MAPREDUCE, edp.JOB_TYPE_MAPREDUCE_STREAMING): - return {'job_config': ch_helper.get_possible_mapreduce_config_from( + return {'job_config': chh.get_possible_mapreduce_config_from( 'plugins/vanilla/v2_7_1/resources/mapred-default.xml')} if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG): - return {'job_config': ch_helper.get_possible_pig_config_from( + return {'job_config': chh.get_possible_pig_config_from( 'plugins/vanilla/v2_7_1/resources/mapred-default.xml')} return edp_engine.EdpOozieEngine.get_possible_job_config(job_type) -class EdpSparkEngine(edp_spark_engine.SparkJobEngine): +class EdpSparkEngine(edp.PluginsSparkJobEngine): edp_base_version = "2.7.1" @@ -68,13 +67,13 @@ class EdpSparkEngine(edp_spark_engine.SparkJobEngine): @staticmethod def job_type_supported(job_type): return (job_type in - edp_spark_engine.SparkJobEngine.get_supported_job_types()) + edp.PluginsSparkJobEngine.get_supported_job_types()) def validate_job_execution(self, cluster, job, data): if (not self.edp_supported(cluster.hadoop_version) or not v_utils.get_spark_history_server(cluster)): - raise ex.InvalidDataException( + raise ex.PluginInvalidDataException( _('Spark {base} or higher required to run {type} jobs').format( base=EdpSparkEngine.edp_base_version, type=job.type)) diff --git a/sahara/plugins/vanilla/v2_7_1/resources/README.rst b/sahara_plugin_vanilla/plugins/vanilla/v2_7_1/resources/README.rst similarity index 100% rename from sahara/plugins/vanilla/v2_7_1/resources/README.rst rename to sahara_plugin_vanilla/plugins/vanilla/v2_7_1/resources/README.rst diff --git a/sahara/plugins/vanilla/v2_7_1/resources/core-default.xml b/sahara_plugin_vanilla/plugins/vanilla/v2_7_1/resources/core-default.xml similarity index 100% rename from sahara/plugins/vanilla/v2_7_1/resources/core-default.xml rename to sahara_plugin_vanilla/plugins/vanilla/v2_7_1/resources/core-default.xml diff --git a/sahara/plugins/vanilla/v2_7_1/resources/create_hive_db.sql b/sahara_plugin_vanilla/plugins/vanilla/v2_7_1/resources/create_hive_db.sql similarity index 100% rename from sahara/plugins/vanilla/v2_7_1/resources/create_hive_db.sql rename to sahara_plugin_vanilla/plugins/vanilla/v2_7_1/resources/create_hive_db.sql diff --git a/sahara/plugins/vanilla/v2_7_1/resources/hdfs-default.xml b/sahara_plugin_vanilla/plugins/vanilla/v2_7_1/resources/hdfs-default.xml similarity index 100% rename from sahara/plugins/vanilla/v2_7_1/resources/hdfs-default.xml rename to sahara_plugin_vanilla/plugins/vanilla/v2_7_1/resources/hdfs-default.xml diff --git a/sahara/plugins/vanilla/v2_7_1/resources/hive-default.xml b/sahara_plugin_vanilla/plugins/vanilla/v2_7_1/resources/hive-default.xml similarity index 100% rename from sahara/plugins/vanilla/v2_7_1/resources/hive-default.xml rename to sahara_plugin_vanilla/plugins/vanilla/v2_7_1/resources/hive-default.xml diff --git a/sahara/plugins/vanilla/v2_7_1/resources/mapred-default.xml b/sahara_plugin_vanilla/plugins/vanilla/v2_7_1/resources/mapred-default.xml similarity index 100% rename from sahara/plugins/vanilla/v2_7_1/resources/mapred-default.xml rename to sahara_plugin_vanilla/plugins/vanilla/v2_7_1/resources/mapred-default.xml diff --git a/sahara/plugins/vanilla/v2_7_1/resources/oozie-default.xml b/sahara_plugin_vanilla/plugins/vanilla/v2_7_1/resources/oozie-default.xml similarity index 100% rename from sahara/plugins/vanilla/v2_7_1/resources/oozie-default.xml rename to sahara_plugin_vanilla/plugins/vanilla/v2_7_1/resources/oozie-default.xml diff --git a/sahara/plugins/vanilla/v2_7_1/resources/yarn-default.xml b/sahara_plugin_vanilla/plugins/vanilla/v2_7_1/resources/yarn-default.xml similarity index 100% rename from sahara/plugins/vanilla/v2_7_1/resources/yarn-default.xml rename to sahara_plugin_vanilla/plugins/vanilla/v2_7_1/resources/yarn-default.xml diff --git a/sahara/plugins/vanilla/v2_8_2/versionhandler.py b/sahara_plugin_vanilla/plugins/vanilla/v2_7_1/versionhandler.py similarity index 72% rename from sahara/plugins/vanilla/v2_8_2/versionhandler.py rename to sahara_plugin_vanilla/plugins/vanilla/v2_7_1/versionhandler.py index 8cc377b..44cb87b 100644 --- a/sahara/plugins/vanilla/v2_8_2/versionhandler.py +++ b/sahara_plugin_vanilla/plugins/vanilla/v2_7_1/versionhandler.py @@ -15,34 +15,32 @@ from oslo_config import cfg -from sahara import conductor -from sahara import context +from sahara.plugins import conductor +from sahara.plugins import context +from sahara.plugins import swift_helper from sahara.plugins import utils -from sahara.plugins.vanilla import abstractversionhandler as avm -from sahara.plugins.vanilla.hadoop2 import config as c -from sahara.plugins.vanilla.hadoop2 import keypairs -from sahara.plugins.vanilla.hadoop2 import recommendations_utils as ru -from sahara.plugins.vanilla.hadoop2 import run_scripts as run -from sahara.plugins.vanilla.hadoop2 import scaling as sc -from sahara.plugins.vanilla.hadoop2 import starting_scripts as s_scripts -from sahara.plugins.vanilla.hadoop2 import utils as u -from sahara.plugins.vanilla.hadoop2 import validation as vl -from sahara.plugins.vanilla import utils as vu -from sahara.plugins.vanilla.v2_8_2 import config_helper as c_helper -from sahara.plugins.vanilla.v2_8_2 import edp_engine -from sahara.swift import swift_helper -from sahara.utils import cluster as cluster_utils +from sahara_plugin_vanilla.plugins.vanilla import abstractversionhandler as avm +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config as c +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import keypairs +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import recommendations_utils +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import run_scripts as run +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import scaling as sc +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import starting_scripts +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import utils as u +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import validation as vl +from sahara_plugin_vanilla.plugins.vanilla import utils as vu +from sahara_plugin_vanilla.plugins.vanilla.v2_7_1 import config_helper +from sahara_plugin_vanilla.plugins.vanilla.v2_7_1 import edp_engine -conductor = conductor.API CONF = cfg.CONF class VersionHandler(avm.AbstractVersionHandler): def __init__(self): self.pctx = { - 'env_confs': c_helper.get_env_configs(), - 'all_confs': c_helper.get_plugin_configs() + 'env_confs': config_helper.get_env_configs(), + 'all_confs': config_helper.get_plugin_configs() } def get_plugin_configs(self): @@ -72,22 +70,22 @@ class VersionHandler(avm.AbstractVersionHandler): def start_cluster(self, cluster): keypairs.provision_keypairs(cluster) - s_scripts.start_namenode(cluster) - s_scripts.start_secondarynamenode(cluster) - s_scripts.start_resourcemanager(cluster) + starting_scripts.start_namenode(cluster) + starting_scripts.start_secondarynamenode(cluster) + starting_scripts.start_resourcemanager(cluster) run.start_dn_nm_processes(utils.get_instances(cluster)) run.await_datanodes(cluster) - s_scripts.start_historyserver(cluster) - s_scripts.start_oozie(self.pctx, cluster) - s_scripts.start_hiveserver(self.pctx, cluster) - s_scripts.start_zookeeper(cluster) + starting_scripts.start_historyserver(cluster) + starting_scripts.start_oozie(self.pctx, cluster) + starting_scripts.start_hiveserver(self.pctx, cluster) + starting_scripts.start_zookeeper(cluster) - swift_helper.install_ssl_certs(cluster_utils.get_instances(cluster)) + swift_helper.install_ssl_certs(utils.get_instances(cluster)) self._set_cluster_info(cluster) - s_scripts.start_spark(cluster) + starting_scripts.start_spark(cluster) def decommission_nodes(self, cluster, instances): sc.decommission_nodes(self.pctx, cluster, instances) @@ -167,4 +165,6 @@ class VersionHandler(avm.AbstractVersionHandler): return c.get_open_ports(node_group) def recommend_configs(self, cluster, scaling): - ru.recommend_configs(cluster, self.get_plugin_configs(), scaling) + recommendations_utils.recommend_configs(cluster, + self.get_plugin_configs(), + scaling) diff --git a/sahara/plugins/vanilla/v2_7_5/__init__.py b/sahara_plugin_vanilla/plugins/vanilla/v2_7_5/__init__.py similarity index 100% rename from sahara/plugins/vanilla/v2_7_5/__init__.py rename to sahara_plugin_vanilla/plugins/vanilla/v2_7_5/__init__.py diff --git a/sahara/plugins/vanilla/v2_7_5/config_helper.py b/sahara_plugin_vanilla/plugins/vanilla/v2_7_5/config_helper.py similarity index 74% rename from sahara/plugins/vanilla/v2_7_5/config_helper.py rename to sahara_plugin_vanilla/plugins/vanilla/v2_7_5/config_helper.py index fb65cad..4198240 100644 --- a/sahara/plugins/vanilla/v2_7_5/config_helper.py +++ b/sahara_plugin_vanilla/plugins/vanilla/v2_7_5/config_helper.py @@ -19,34 +19,40 @@ from oslo_config import cfg import six from sahara.plugins import provisioning as p -from sahara.plugins.vanilla.hadoop2 import config_helper as c_helper -from sahara.utils import xmlutils as x +from sahara.plugins import utils +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config_helper CONF = cfg.CONF CONF.import_opt("enable_data_locality", "sahara.topology.topology_helper") -CORE_DEFAULT = x.load_hadoop_xml_defaults( - 'plugins/vanilla/v2_7_5/resources/core-default.xml') +CORE_DEFAULT = utils.load_hadoop_xml_defaults( + 'plugins/vanilla/v2_7_5/resources/core-default.xml', + 'sahara_plugin_vanilla') -HDFS_DEFAULT = x.load_hadoop_xml_defaults( - 'plugins/vanilla/v2_7_5/resources/hdfs-default.xml') +HDFS_DEFAULT = utils.load_hadoop_xml_defaults( + 'plugins/vanilla/v2_7_5/resources/hdfs-default.xml', + 'sahara_plugin_vanilla') -MAPRED_DEFAULT = x.load_hadoop_xml_defaults( - 'plugins/vanilla/v2_7_5/resources/mapred-default.xml') +MAPRED_DEFAULT = utils.load_hadoop_xml_defaults( + 'plugins/vanilla/v2_7_5/resources/mapred-default.xml', + 'sahara_plugin_vanilla') -YARN_DEFAULT = x.load_hadoop_xml_defaults( - 'plugins/vanilla/v2_7_5/resources/yarn-default.xml') +YARN_DEFAULT = utils.load_hadoop_xml_defaults( + 'plugins/vanilla/v2_7_5/resources/yarn-default.xml', + 'sahara_plugin_vanilla') -OOZIE_DEFAULT = x.load_hadoop_xml_defaults( - 'plugins/vanilla/v2_7_5/resources/oozie-default.xml') +OOZIE_DEFAULT = utils.load_hadoop_xml_defaults( + 'plugins/vanilla/v2_7_5/resources/oozie-default.xml', + 'sahara_plugin_vanilla') -HIVE_DEFAULT = x.load_hadoop_xml_defaults( - 'plugins/vanilla/v2_7_5/resources/hive-default.xml') +HIVE_DEFAULT = utils.load_hadoop_xml_defaults( + 'plugins/vanilla/v2_7_5/resources/hive-default.xml', + 'sahara_plugin_vanilla') _default_executor_classpath = ":".join( ['/opt/hadoop/share/hadoop/tools/lib/hadoop-openstack-2.7.5.jar']) -SPARK_CONFS = copy.deepcopy(c_helper.SPARK_CONFS) +SPARK_CONFS = copy.deepcopy(config_helper.SPARK_CONFS) SPARK_CONFS['Spark']['OPTIONS'].append( { @@ -87,15 +93,15 @@ ENV_CONFS = { } # Initialise plugin Hadoop configurations -PLUGIN_XML_CONFIGS = c_helper.init_xml_configs(XML_CONFS) -PLUGIN_ENV_CONFIGS = c_helper.init_env_configs(ENV_CONFS) +PLUGIN_XML_CONFIGS = config_helper.init_xml_configs(XML_CONFS) +PLUGIN_ENV_CONFIGS = config_helper.init_env_configs(ENV_CONFS) def _init_all_configs(): configs = [] configs.extend(PLUGIN_XML_CONFIGS) configs.extend(PLUGIN_ENV_CONFIGS) - configs.extend(c_helper.PLUGIN_GENERAL_CONFIGS) + configs.extend(config_helper.PLUGIN_GENERAL_CONFIGS) configs.extend(_get_spark_configs()) configs.extend(_get_zookeeper_configs()) return configs @@ -124,7 +130,7 @@ def _get_spark_configs(): def _get_zookeeper_configs(): zk_configs = [] - for service, config_items in six.iteritems(c_helper.ZOOKEEPER_CONFS): + for service, config_items in six.iteritems(config_helper.ZOOKEEPER_CONFS): for item in config_items['OPTIONS']: cfg = p.Config(name=item["name"], description=item["description"], diff --git a/sahara/plugins/vanilla/v2_7_5/edp_engine.py b/sahara_plugin_vanilla/plugins/vanilla/v2_7_5/edp_engine.py similarity index 78% rename from sahara/plugins/vanilla/v2_7_5/edp_engine.py rename to sahara_plugin_vanilla/plugins/vanilla/v2_7_5/edp_engine.py index c52e3b5..d273e44 100644 --- a/sahara/plugins/vanilla/v2_7_5/edp_engine.py +++ b/sahara_plugin_vanilla/plugins/vanilla/v2_7_5/edp_engine.py @@ -14,34 +14,37 @@ # limitations under the License. import os -from sahara import exceptions as ex -from sahara.i18n import _ +from sahara.plugins import edp +from sahara.plugins import exceptions as ex from sahara.plugins import utils as plugin_utils -from sahara.plugins.vanilla import confighints_helper as ch_helper -from sahara.plugins.vanilla.hadoop2 import edp_engine -from sahara.plugins.vanilla import utils as v_utils -from sahara.service.edp.spark import engine as edp_spark_engine -from sahara.utils import edp +from sahara_plugin_vanilla.i18n import _ +from sahara_plugin_vanilla.plugins.vanilla import confighints_helper +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import edp_engine +from sahara_plugin_vanilla.plugins.vanilla import utils as v_utils class EdpOozieEngine(edp_engine.EdpOozieEngine): @staticmethod def get_possible_job_config(job_type): if edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE): - return {'job_config': ch_helper.get_possible_hive_config_from( + return { + 'job_config': confighints_helper.get_possible_hive_config_from( 'plugins/vanilla/v2_7_5/resources/hive-default.xml')} if edp.compare_job_type(job_type, edp.JOB_TYPE_MAPREDUCE, edp.JOB_TYPE_MAPREDUCE_STREAMING): - return {'job_config': ch_helper.get_possible_mapreduce_config_from( + return { + 'job_config': + confighints_helper.get_possible_mapreduce_config_from( 'plugins/vanilla/v2_7_5/resources/mapred-default.xml')} if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG): - return {'job_config': ch_helper.get_possible_pig_config_from( + return { + 'job_config': confighints_helper.get_possible_pig_config_from( 'plugins/vanilla/v2_7_5/resources/mapred-default.xml')} return edp_engine.EdpOozieEngine.get_possible_job_config(job_type) -class EdpSparkEngine(edp_spark_engine.SparkJobEngine): +class EdpSparkEngine(edp.PluginsSparkJobEngine): edp_base_version = "2.7.5" @@ -68,7 +71,7 @@ class EdpSparkEngine(edp_spark_engine.SparkJobEngine): @staticmethod def job_type_supported(job_type): return (job_type in - edp_spark_engine.SparkJobEngine.get_supported_job_types()) + edp.PluginsSparkJobEngine.get_supported_job_types()) def validate_job_execution(self, cluster, job, data): if (not self.edp_supported(cluster.hadoop_version) or diff --git a/sahara/plugins/vanilla/v2_7_5/resources/README.rst b/sahara_plugin_vanilla/plugins/vanilla/v2_7_5/resources/README.rst similarity index 100% rename from sahara/plugins/vanilla/v2_7_5/resources/README.rst rename to sahara_plugin_vanilla/plugins/vanilla/v2_7_5/resources/README.rst diff --git a/sahara/plugins/vanilla/v2_7_5/resources/core-default.xml b/sahara_plugin_vanilla/plugins/vanilla/v2_7_5/resources/core-default.xml similarity index 100% rename from sahara/plugins/vanilla/v2_7_5/resources/core-default.xml rename to sahara_plugin_vanilla/plugins/vanilla/v2_7_5/resources/core-default.xml diff --git a/sahara/plugins/vanilla/v2_7_5/resources/create_hive_db.sql b/sahara_plugin_vanilla/plugins/vanilla/v2_7_5/resources/create_hive_db.sql similarity index 100% rename from sahara/plugins/vanilla/v2_7_5/resources/create_hive_db.sql rename to sahara_plugin_vanilla/plugins/vanilla/v2_7_5/resources/create_hive_db.sql diff --git a/sahara/plugins/vanilla/v2_7_5/resources/hdfs-default.xml b/sahara_plugin_vanilla/plugins/vanilla/v2_7_5/resources/hdfs-default.xml similarity index 100% rename from sahara/plugins/vanilla/v2_7_5/resources/hdfs-default.xml rename to sahara_plugin_vanilla/plugins/vanilla/v2_7_5/resources/hdfs-default.xml diff --git a/sahara/plugins/vanilla/v2_7_5/resources/hive-default.xml b/sahara_plugin_vanilla/plugins/vanilla/v2_7_5/resources/hive-default.xml similarity index 100% rename from sahara/plugins/vanilla/v2_7_5/resources/hive-default.xml rename to sahara_plugin_vanilla/plugins/vanilla/v2_7_5/resources/hive-default.xml diff --git a/sahara/plugins/vanilla/v2_7_5/resources/mapred-default.xml b/sahara_plugin_vanilla/plugins/vanilla/v2_7_5/resources/mapred-default.xml similarity index 100% rename from sahara/plugins/vanilla/v2_7_5/resources/mapred-default.xml rename to sahara_plugin_vanilla/plugins/vanilla/v2_7_5/resources/mapred-default.xml diff --git a/sahara/plugins/vanilla/v2_7_5/resources/oozie-default.xml b/sahara_plugin_vanilla/plugins/vanilla/v2_7_5/resources/oozie-default.xml similarity index 100% rename from sahara/plugins/vanilla/v2_7_5/resources/oozie-default.xml rename to sahara_plugin_vanilla/plugins/vanilla/v2_7_5/resources/oozie-default.xml diff --git a/sahara/plugins/vanilla/v2_7_5/resources/yarn-default.xml b/sahara_plugin_vanilla/plugins/vanilla/v2_7_5/resources/yarn-default.xml similarity index 100% rename from sahara/plugins/vanilla/v2_7_5/resources/yarn-default.xml rename to sahara_plugin_vanilla/plugins/vanilla/v2_7_5/resources/yarn-default.xml diff --git a/sahara/plugins/vanilla/v2_7_5/versionhandler.py b/sahara_plugin_vanilla/plugins/vanilla/v2_7_5/versionhandler.py similarity index 72% rename from sahara/plugins/vanilla/v2_7_5/versionhandler.py rename to sahara_plugin_vanilla/plugins/vanilla/v2_7_5/versionhandler.py index f8a5363..1fd621c 100644 --- a/sahara/plugins/vanilla/v2_7_5/versionhandler.py +++ b/sahara_plugin_vanilla/plugins/vanilla/v2_7_5/versionhandler.py @@ -15,34 +15,32 @@ from oslo_config import cfg -from sahara import conductor -from sahara import context +from sahara.plugins import conductor +from sahara.plugins import context +from sahara.plugins import swift_helper from sahara.plugins import utils -from sahara.plugins.vanilla import abstractversionhandler as avm -from sahara.plugins.vanilla.hadoop2 import config as c -from sahara.plugins.vanilla.hadoop2 import keypairs -from sahara.plugins.vanilla.hadoop2 import recommendations_utils as ru -from sahara.plugins.vanilla.hadoop2 import run_scripts as run -from sahara.plugins.vanilla.hadoop2 import scaling as sc -from sahara.plugins.vanilla.hadoop2 import starting_scripts as s_scripts -from sahara.plugins.vanilla.hadoop2 import utils as u -from sahara.plugins.vanilla.hadoop2 import validation as vl -from sahara.plugins.vanilla import utils as vu -from sahara.plugins.vanilla.v2_7_5 import config_helper as c_helper -from sahara.plugins.vanilla.v2_7_5 import edp_engine -from sahara.swift import swift_helper -from sahara.utils import cluster as cluster_utils +from sahara_plugin_vanilla.plugins.vanilla import abstractversionhandler as avm +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config as c +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import keypairs +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import recommendations_utils +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import run_scripts as run +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import scaling as sc +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import starting_scripts +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import utils as u +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import validation as vl +from sahara_plugin_vanilla.plugins.vanilla import utils as vu +from sahara_plugin_vanilla.plugins.vanilla.v2_7_5 import config_helper +from sahara_plugin_vanilla.plugins.vanilla.v2_7_5 import edp_engine -conductor = conductor.API CONF = cfg.CONF class VersionHandler(avm.AbstractVersionHandler): def __init__(self): self.pctx = { - 'env_confs': c_helper.get_env_configs(), - 'all_confs': c_helper.get_plugin_configs() + 'env_confs': config_helper.get_env_configs(), + 'all_confs': config_helper.get_plugin_configs() } def get_plugin_configs(self): @@ -72,22 +70,22 @@ class VersionHandler(avm.AbstractVersionHandler): def start_cluster(self, cluster): keypairs.provision_keypairs(cluster) - s_scripts.start_namenode(cluster) - s_scripts.start_secondarynamenode(cluster) - s_scripts.start_resourcemanager(cluster) + starting_scripts.start_namenode(cluster) + starting_scripts.start_secondarynamenode(cluster) + starting_scripts.start_resourcemanager(cluster) run.start_dn_nm_processes(utils.get_instances(cluster)) run.await_datanodes(cluster) - s_scripts.start_historyserver(cluster) - s_scripts.start_oozie(self.pctx, cluster) - s_scripts.start_hiveserver(self.pctx, cluster) - s_scripts.start_zookeeper(cluster) + starting_scripts.start_historyserver(cluster) + starting_scripts.start_oozie(self.pctx, cluster) + starting_scripts.start_hiveserver(self.pctx, cluster) + starting_scripts.start_zookeeper(cluster) - swift_helper.install_ssl_certs(cluster_utils.get_instances(cluster)) + swift_helper.install_ssl_certs(utils.get_instances(cluster)) self._set_cluster_info(cluster) - s_scripts.start_spark(cluster) + starting_scripts.start_spark(cluster) def decommission_nodes(self, cluster, instances): sc.decommission_nodes(self.pctx, cluster, instances) @@ -167,4 +165,6 @@ class VersionHandler(avm.AbstractVersionHandler): return c.get_open_ports(node_group) def recommend_configs(self, cluster, scaling): - ru.recommend_configs(cluster, self.get_plugin_configs(), scaling) + recommendations_utils.recommend_configs(cluster, + self.get_plugin_configs(), + scaling) diff --git a/sahara/plugins/vanilla/v2_8_2/__init__.py b/sahara_plugin_vanilla/plugins/vanilla/v2_8_2/__init__.py similarity index 100% rename from sahara/plugins/vanilla/v2_8_2/__init__.py rename to sahara_plugin_vanilla/plugins/vanilla/v2_8_2/__init__.py diff --git a/sahara/plugins/vanilla/v2_8_2/config_helper.py b/sahara_plugin_vanilla/plugins/vanilla/v2_8_2/config_helper.py similarity index 74% rename from sahara/plugins/vanilla/v2_8_2/config_helper.py rename to sahara_plugin_vanilla/plugins/vanilla/v2_8_2/config_helper.py index 280330a..3b24b23 100644 --- a/sahara/plugins/vanilla/v2_8_2/config_helper.py +++ b/sahara_plugin_vanilla/plugins/vanilla/v2_8_2/config_helper.py @@ -19,34 +19,40 @@ from oslo_config import cfg import six from sahara.plugins import provisioning as p -from sahara.plugins.vanilla.hadoop2 import config_helper as c_helper -from sahara.utils import xmlutils as x +from sahara.plugins import utils +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config_helper CONF = cfg.CONF CONF.import_opt("enable_data_locality", "sahara.topology.topology_helper") -CORE_DEFAULT = x.load_hadoop_xml_defaults( - 'plugins/vanilla/v2_8_2/resources/core-default.xml') +CORE_DEFAULT = utils.load_hadoop_xml_defaults( + 'plugins/vanilla/v2_8_2/resources/core-default.xml', + 'sahara_plugin_vanilla') -HDFS_DEFAULT = x.load_hadoop_xml_defaults( - 'plugins/vanilla/v2_8_2/resources/hdfs-default.xml') +HDFS_DEFAULT = utils.load_hadoop_xml_defaults( + 'plugins/vanilla/v2_8_2/resources/hdfs-default.xml', + 'sahara_plugin_vanilla') -MAPRED_DEFAULT = x.load_hadoop_xml_defaults( - 'plugins/vanilla/v2_8_2/resources/mapred-default.xml') +MAPRED_DEFAULT = utils.load_hadoop_xml_defaults( + 'plugins/vanilla/v2_8_2/resources/mapred-default.xml', + 'sahara_plugin_vanilla') -YARN_DEFAULT = x.load_hadoop_xml_defaults( - 'plugins/vanilla/v2_8_2/resources/yarn-default.xml') +YARN_DEFAULT = utils.load_hadoop_xml_defaults( + 'plugins/vanilla/v2_8_2/resources/yarn-default.xml', + 'sahara_plugin_vanilla') -OOZIE_DEFAULT = x.load_hadoop_xml_defaults( - 'plugins/vanilla/v2_8_2/resources/oozie-default.xml') +OOZIE_DEFAULT = utils.load_hadoop_xml_defaults( + 'plugins/vanilla/v2_8_2/resources/oozie-default.xml', + 'sahara_plugin_vanilla') -HIVE_DEFAULT = x.load_hadoop_xml_defaults( - 'plugins/vanilla/v2_8_2/resources/hive-default.xml') +HIVE_DEFAULT = utils.load_hadoop_xml_defaults( + 'plugins/vanilla/v2_8_2/resources/hive-default.xml', + 'sahara_plugin_vanilla') _default_executor_classpath = ":".join( ['/opt/hadoop/share/hadoop/tools/lib/hadoop-openstack-2.8.2.jar']) -SPARK_CONFS = copy.deepcopy(c_helper.SPARK_CONFS) +SPARK_CONFS = copy.deepcopy(config_helper.SPARK_CONFS) SPARK_CONFS['Spark']['OPTIONS'].append( { @@ -87,15 +93,15 @@ ENV_CONFS = { } # Initialise plugin Hadoop configurations -PLUGIN_XML_CONFIGS = c_helper.init_xml_configs(XML_CONFS) -PLUGIN_ENV_CONFIGS = c_helper.init_env_configs(ENV_CONFS) +PLUGIN_XML_CONFIGS = config_helper.init_xml_configs(XML_CONFS) +PLUGIN_ENV_CONFIGS = config_helper.init_env_configs(ENV_CONFS) def _init_all_configs(): configs = [] configs.extend(PLUGIN_XML_CONFIGS) configs.extend(PLUGIN_ENV_CONFIGS) - configs.extend(c_helper.PLUGIN_GENERAL_CONFIGS) + configs.extend(config_helper.PLUGIN_GENERAL_CONFIGS) configs.extend(_get_spark_configs()) configs.extend(_get_zookeeper_configs()) return configs @@ -124,7 +130,7 @@ def _get_spark_configs(): def _get_zookeeper_configs(): zk_configs = [] - for service, config_items in six.iteritems(c_helper.ZOOKEEPER_CONFS): + for service, config_items in six.iteritems(config_helper.ZOOKEEPER_CONFS): for item in config_items['OPTIONS']: cfg = p.Config(name=item["name"], description=item["description"], diff --git a/sahara/plugins/vanilla/v2_8_2/edp_engine.py b/sahara_plugin_vanilla/plugins/vanilla/v2_8_2/edp_engine.py similarity index 77% rename from sahara/plugins/vanilla/v2_8_2/edp_engine.py rename to sahara_plugin_vanilla/plugins/vanilla/v2_8_2/edp_engine.py index 3b7d174..a5c4ad1 100644 --- a/sahara/plugins/vanilla/v2_8_2/edp_engine.py +++ b/sahara_plugin_vanilla/plugins/vanilla/v2_8_2/edp_engine.py @@ -14,34 +14,37 @@ # limitations under the License. import os -from sahara import exceptions as ex -from sahara.i18n import _ +from sahara.plugins import edp +from sahara.plugins import exceptions as ex from sahara.plugins import utils as plugin_utils -from sahara.plugins.vanilla import confighints_helper as ch_helper -from sahara.plugins.vanilla.hadoop2 import edp_engine -from sahara.plugins.vanilla import utils as v_utils -from sahara.service.edp.spark import engine as edp_spark_engine -from sahara.utils import edp +from sahara_plugin_vanilla.i18n import _ +from sahara_plugin_vanilla.plugins.vanilla import confighints_helper +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import edp_engine +from sahara_plugin_vanilla.plugins.vanilla import utils as v_utils class EdpOozieEngine(edp_engine.EdpOozieEngine): @staticmethod def get_possible_job_config(job_type): if edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE): - return {'job_config': ch_helper.get_possible_hive_config_from( + return { + 'job_config': confighints_helper.get_possible_hive_config_from( 'plugins/vanilla/v2_8_2/resources/hive-default.xml')} if edp.compare_job_type(job_type, edp.JOB_TYPE_MAPREDUCE, edp.JOB_TYPE_MAPREDUCE_STREAMING): - return {'job_config': ch_helper.get_possible_mapreduce_config_from( + return { + 'job_config': + confighints_helper.get_possible_mapreduce_config_from( 'plugins/vanilla/v2_8_2/resources/mapred-default.xml')} if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG): - return {'job_config': ch_helper.get_possible_pig_config_from( + return { + 'job_config': confighints_helper.get_possible_pig_config_from( 'plugins/vanilla/v2_8_2/resources/mapred-default.xml')} return edp_engine.EdpOozieEngine.get_possible_job_config(job_type) -class EdpSparkEngine(edp_spark_engine.SparkJobEngine): +class EdpSparkEngine(edp.PluginsSparkJobEngine): edp_base_version = "2.8.2" @@ -68,13 +71,13 @@ class EdpSparkEngine(edp_spark_engine.SparkJobEngine): @staticmethod def job_type_supported(job_type): return (job_type in - edp_spark_engine.SparkJobEngine.get_supported_job_types()) + edp.PluginsSparkJobEngine.get_supported_job_types()) def validate_job_execution(self, cluster, job, data): if (not self.edp_supported(cluster.hadoop_version) or not v_utils.get_spark_history_server(cluster)): - raise ex.InvalidDataException( + raise ex.PluginInvalidDataException( _('Spark {base} or higher required to run {type} jobs').format( base=EdpSparkEngine.edp_base_version, type=job.type)) diff --git a/sahara/plugins/vanilla/v2_8_2/resources/README.rst b/sahara_plugin_vanilla/plugins/vanilla/v2_8_2/resources/README.rst similarity index 100% rename from sahara/plugins/vanilla/v2_8_2/resources/README.rst rename to sahara_plugin_vanilla/plugins/vanilla/v2_8_2/resources/README.rst diff --git a/sahara/plugins/vanilla/v2_8_2/resources/core-default.xml b/sahara_plugin_vanilla/plugins/vanilla/v2_8_2/resources/core-default.xml similarity index 100% rename from sahara/plugins/vanilla/v2_8_2/resources/core-default.xml rename to sahara_plugin_vanilla/plugins/vanilla/v2_8_2/resources/core-default.xml diff --git a/sahara/plugins/vanilla/v2_8_2/resources/create_hive_db.sql b/sahara_plugin_vanilla/plugins/vanilla/v2_8_2/resources/create_hive_db.sql similarity index 100% rename from sahara/plugins/vanilla/v2_8_2/resources/create_hive_db.sql rename to sahara_plugin_vanilla/plugins/vanilla/v2_8_2/resources/create_hive_db.sql diff --git a/sahara/plugins/vanilla/v2_8_2/resources/hdfs-default.xml b/sahara_plugin_vanilla/plugins/vanilla/v2_8_2/resources/hdfs-default.xml similarity index 100% rename from sahara/plugins/vanilla/v2_8_2/resources/hdfs-default.xml rename to sahara_plugin_vanilla/plugins/vanilla/v2_8_2/resources/hdfs-default.xml diff --git a/sahara/plugins/vanilla/v2_8_2/resources/hive-default.xml b/sahara_plugin_vanilla/plugins/vanilla/v2_8_2/resources/hive-default.xml similarity index 100% rename from sahara/plugins/vanilla/v2_8_2/resources/hive-default.xml rename to sahara_plugin_vanilla/plugins/vanilla/v2_8_2/resources/hive-default.xml diff --git a/sahara/plugins/vanilla/v2_8_2/resources/mapred-default.xml b/sahara_plugin_vanilla/plugins/vanilla/v2_8_2/resources/mapred-default.xml similarity index 100% rename from sahara/plugins/vanilla/v2_8_2/resources/mapred-default.xml rename to sahara_plugin_vanilla/plugins/vanilla/v2_8_2/resources/mapred-default.xml diff --git a/sahara/plugins/vanilla/v2_8_2/resources/oozie-default.xml b/sahara_plugin_vanilla/plugins/vanilla/v2_8_2/resources/oozie-default.xml similarity index 100% rename from sahara/plugins/vanilla/v2_8_2/resources/oozie-default.xml rename to sahara_plugin_vanilla/plugins/vanilla/v2_8_2/resources/oozie-default.xml diff --git a/sahara/plugins/vanilla/v2_8_2/resources/yarn-default.xml b/sahara_plugin_vanilla/plugins/vanilla/v2_8_2/resources/yarn-default.xml similarity index 100% rename from sahara/plugins/vanilla/v2_8_2/resources/yarn-default.xml rename to sahara_plugin_vanilla/plugins/vanilla/v2_8_2/resources/yarn-default.xml diff --git a/sahara/plugins/vanilla/v2_7_1/versionhandler.py b/sahara_plugin_vanilla/plugins/vanilla/v2_8_2/versionhandler.py similarity index 72% rename from sahara/plugins/vanilla/v2_7_1/versionhandler.py rename to sahara_plugin_vanilla/plugins/vanilla/v2_8_2/versionhandler.py index 478f4e4..78a4ed0 100644 --- a/sahara/plugins/vanilla/v2_7_1/versionhandler.py +++ b/sahara_plugin_vanilla/plugins/vanilla/v2_8_2/versionhandler.py @@ -15,34 +15,32 @@ from oslo_config import cfg -from sahara import conductor -from sahara import context +from sahara.plugins import conductor +from sahara.plugins import context +from sahara.plugins import swift_helper from sahara.plugins import utils -from sahara.plugins.vanilla import abstractversionhandler as avm -from sahara.plugins.vanilla.hadoop2 import config as c -from sahara.plugins.vanilla.hadoop2 import keypairs -from sahara.plugins.vanilla.hadoop2 import recommendations_utils as ru -from sahara.plugins.vanilla.hadoop2 import run_scripts as run -from sahara.plugins.vanilla.hadoop2 import scaling as sc -from sahara.plugins.vanilla.hadoop2 import starting_scripts as s_scripts -from sahara.plugins.vanilla.hadoop2 import utils as u -from sahara.plugins.vanilla.hadoop2 import validation as vl -from sahara.plugins.vanilla import utils as vu -from sahara.plugins.vanilla.v2_7_1 import config_helper as c_helper -from sahara.plugins.vanilla.v2_7_1 import edp_engine -from sahara.swift import swift_helper -from sahara.utils import cluster as cluster_utils +from sahara_plugin_vanilla.plugins.vanilla import abstractversionhandler as avm +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config as c +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import keypairs +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import recommendations_utils +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import run_scripts as run +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import scaling as sc +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import starting_scripts +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import utils as u +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import validation as vl +from sahara_plugin_vanilla.plugins.vanilla import utils as vu +from sahara_plugin_vanilla.plugins.vanilla.v2_8_2 import config_helper +from sahara_plugin_vanilla.plugins.vanilla.v2_8_2 import edp_engine -conductor = conductor.API CONF = cfg.CONF class VersionHandler(avm.AbstractVersionHandler): def __init__(self): self.pctx = { - 'env_confs': c_helper.get_env_configs(), - 'all_confs': c_helper.get_plugin_configs() + 'env_confs': config_helper.get_env_configs(), + 'all_confs': config_helper.get_plugin_configs() } def get_plugin_configs(self): @@ -72,22 +70,22 @@ class VersionHandler(avm.AbstractVersionHandler): def start_cluster(self, cluster): keypairs.provision_keypairs(cluster) - s_scripts.start_namenode(cluster) - s_scripts.start_secondarynamenode(cluster) - s_scripts.start_resourcemanager(cluster) + starting_scripts.start_namenode(cluster) + starting_scripts.start_secondarynamenode(cluster) + starting_scripts.start_resourcemanager(cluster) run.start_dn_nm_processes(utils.get_instances(cluster)) run.await_datanodes(cluster) - s_scripts.start_historyserver(cluster) - s_scripts.start_oozie(self.pctx, cluster) - s_scripts.start_hiveserver(self.pctx, cluster) - s_scripts.start_zookeeper(cluster) + starting_scripts.start_historyserver(cluster) + starting_scripts.start_oozie(self.pctx, cluster) + starting_scripts.start_hiveserver(self.pctx, cluster) + starting_scripts.start_zookeeper(cluster) - swift_helper.install_ssl_certs(cluster_utils.get_instances(cluster)) + swift_helper.install_ssl_certs(utils.get_instances(cluster)) self._set_cluster_info(cluster) - s_scripts.start_spark(cluster) + starting_scripts.start_spark(cluster) def decommission_nodes(self, cluster, instances): sc.decommission_nodes(self.pctx, cluster, instances) @@ -167,4 +165,6 @@ class VersionHandler(avm.AbstractVersionHandler): return c.get_open_ports(node_group) def recommend_configs(self, cluster, scaling): - ru.recommend_configs(cluster, self.get_plugin_configs(), scaling) + recommendations_utils.recommend_configs(cluster, + self.get_plugin_configs(), + scaling) diff --git a/sahara/plugins/vanilla/versionfactory.py b/sahara_plugin_vanilla/plugins/vanilla/versionfactory.py similarity index 87% rename from sahara/plugins/vanilla/versionfactory.py rename to sahara_plugin_vanilla/plugins/vanilla/versionfactory.py index 61e2676..c625866 100644 --- a/sahara/plugins/vanilla/versionfactory.py +++ b/sahara_plugin_vanilla/plugins/vanilla/versionfactory.py @@ -16,7 +16,7 @@ import os import re -from sahara.utils import general +from sahara.plugins import utils class VersionFactory(object): @@ -33,13 +33,13 @@ class VersionFactory(object): for name in os.listdir(src_dir) if (os.path.isdir(os.path.join(src_dir, name)) and re.match(r'^v\d+_\d+_\d+$', name))]) - versions.sort(key=general.natural_sort_key) + versions.sort(key=utils.natural_sort_key) VersionFactory.versions = versions VersionFactory.modules = {} for version in VersionFactory.versions: - module_name = 'sahara.plugins.vanilla.v%s.versionhandler' % ( - version.replace('.', '_')) + module_name = ('sahara_plugin_vanilla.plugins.vanilla.v%s.' + 'versionhandler' % (version.replace('.', '_'))) module_class = getattr( __import__(module_name, fromlist=['sahara']), 'VersionHandler') diff --git a/sahara/tests/__init__.py b/sahara_plugin_vanilla/tests/__init__.py similarity index 94% rename from sahara/tests/__init__.py rename to sahara_plugin_vanilla/tests/__init__.py index 90fe6c0..40224b0 100644 --- a/sahara/tests/__init__.py +++ b/sahara_plugin_vanilla/tests/__init__.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from sahara.utils import patches +from sahara_plugin_vanilla.utils import patches patches.patch_all() import oslo_i18n diff --git a/sahara/tests/unit/__init__.py b/sahara_plugin_vanilla/tests/unit/__init__.py similarity index 100% rename from sahara/tests/unit/__init__.py rename to sahara_plugin_vanilla/tests/unit/__init__.py diff --git a/sahara/tests/unit/base.py b/sahara_plugin_vanilla/tests/unit/base.py similarity index 67% rename from sahara/tests/unit/base.py rename to sahara_plugin_vanilla/tests/unit/base.py index 962f220..364f04a 100644 --- a/sahara/tests/unit/base.py +++ b/sahara_plugin_vanilla/tests/unit/base.py @@ -13,13 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock from oslotest import base -from sahara import context -from sahara.db import api as db_api -from sahara import main -from sahara.utils import rpc +from sahara.plugins import context +from sahara.plugins import db as db_api +from sahara.plugins import main +from sahara.plugins import utils class SaharaTestCase(base.BaseTestCase): @@ -27,7 +26,7 @@ class SaharaTestCase(base.BaseTestCase): def setUp(self): super(SaharaTestCase, self).setUp() self.setup_context() - rpc.setup('all-in-one') + utils.rpc_setup('all-in-one') def setup_context(self, username="test_user", tenant_id="tenant_1", auth_token="test_auth_token", tenant_name='test_tenant', @@ -35,14 +34,14 @@ class SaharaTestCase(base.BaseTestCase): self.addCleanup(context.set_ctx, context.ctx() if context.has_ctx() else None) - context.set_ctx(context.Context( + context.set_ctx(context.PluginsContext( username=username, tenant_id=tenant_id, auth_token=auth_token, service_catalog=service_catalog or {}, tenant_name=tenant_name, **kwargs)) def override_config(self, name, override, group=None): - main.CONF.set_override(name, override, group) - self.addCleanup(main.CONF.clear_override, name, group) + main.set_override(name, override, group) + self.addCleanup(main.clear_override, name, group) class SaharaWithDbTestCase(SaharaTestCase): @@ -52,22 +51,3 @@ class SaharaWithDbTestCase(SaharaTestCase): self.override_config('connection', "sqlite://", group='database') db_api.setup_db() self.addCleanup(db_api.drop_db) - - -class _ConsecutiveThreadGroup(context.ThreadGroup): - def __init__(self, _thread_pool_size=1000): - pass - - def spawn(self, thread_description, func, *args, **kwargs): - func(*args, **kwargs) - - def __enter__(self): - return self - - def __exit__(self, *ex): - pass - - -def mock_thread_group(func): - return mock.patch('sahara.context.ThreadGroup', - new=_ConsecutiveThreadGroup)(func) diff --git a/sahara/tests/unit/plugins/__init__.py b/sahara_plugin_vanilla/tests/unit/plugins/__init__.py similarity index 100% rename from sahara/tests/unit/plugins/__init__.py rename to sahara_plugin_vanilla/tests/unit/plugins/__init__.py diff --git a/sahara/tests/unit/plugins/vanilla/__init__.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/__init__.py similarity index 100% rename from sahara/tests/unit/plugins/vanilla/__init__.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/__init__.py diff --git a/sahara/tests/unit/plugins/vanilla/hadoop2/__init__.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/__init__.py similarity index 100% rename from sahara/tests/unit/plugins/vanilla/hadoop2/__init__.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/__init__.py diff --git a/sahara/tests/unit/plugins/vanilla/hadoop2/resources/dfs-report.txt b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/resources/dfs-report.txt similarity index 100% rename from sahara/tests/unit/plugins/vanilla/hadoop2/resources/dfs-report.txt rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/resources/dfs-report.txt diff --git a/sahara/tests/unit/plugins/vanilla/hadoop2/resources/yarn-report.txt b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/resources/yarn-report.txt similarity index 100% rename from sahara/tests/unit/plugins/vanilla/hadoop2/resources/yarn-report.txt rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/resources/yarn-report.txt diff --git a/sahara/tests/unit/plugins/vanilla/hadoop2/test_config_helper.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_config_helper.py similarity index 72% rename from sahara/tests/unit/plugins/vanilla/hadoop2/test_config_helper.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_config_helper.py index a105272..c8a2ef4 100644 --- a/sahara/tests/unit/plugins/vanilla/hadoop2/test_config_helper.py +++ b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_config_helper.py @@ -16,15 +16,15 @@ import mock from oslo_config import cfg -from sahara import exceptions as ex +from sahara.plugins import exceptions as ex from sahara.plugins import provisioning as p -from sahara.plugins.vanilla.hadoop2 import config_helper as c_helper -from sahara.tests.unit import base +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config_helper +from sahara_plugin_vanilla.tests.unit import base class TestConfigHelper(base.SaharaTestCase): - plugin_path = 'sahara.plugins.vanilla.hadoop2.' + plugin_path = 'sahara_plugin_vanilla.plugins.vanilla.hadoop2.' def setUp(self): super(TestConfigHelper, self).setUp() @@ -54,21 +54,22 @@ class TestConfigHelper(base.SaharaTestCase): 'Oozie Heap Size': 1024 } } - configs = c_helper.init_env_configs(ENV_CONFS) + configs = config_helper.init_env_configs(ENV_CONFS) for config in configs: self.assertIsInstance(config, p.Config) def test_init_general_configs(self): - sample_configs = [c_helper.ENABLE_SWIFT, c_helper.ENABLE_MYSQL, - c_helper.DATANODES_STARTUP_TIMEOUT, - c_helper.DATANODES_DECOMMISSIONING_TIMEOUT, - c_helper.NODEMANAGERS_DECOMMISSIONING_TIMEOUT] + sample_configs = [config_helper.ENABLE_SWIFT, + config_helper.ENABLE_MYSQL, + config_helper.DATANODES_STARTUP_TIMEOUT, + config_helper.DATANODES_DECOMMISSIONING_TIMEOUT, + config_helper.NODEMANAGERS_DECOMMISSIONING_TIMEOUT] self.CONF.enable_data_locality = False - self.assertEqual(c_helper._init_general_configs(), sample_configs) + self.assertEqual(config_helper._init_general_configs(), sample_configs) - sample_configs.append(c_helper.ENABLE_DATA_LOCALITY) + sample_configs.append(config_helper.ENABLE_DATA_LOCALITY) self.CONF.enable_data_locality = True - self.assertEqual(c_helper._init_general_configs(), sample_configs) + self.assertEqual(config_helper._init_general_configs(), sample_configs) def test_get_config_value(self): cluster = mock.Mock() @@ -78,8 +79,8 @@ class TestConfigHelper(base.SaharaTestCase): cl = 'test' ng.configuration.return_value.get.return_value.get.return_value = cl cluster.node_groups = [ng] - cl_param = c_helper.get_config_value('pctx', 'service', - 'name', cluster) + cl_param = config_helper.get_config_value('pctx', 'service', + 'name', cluster) self.assertEqual(cl, cl_param) all_confs = mock.Mock() @@ -87,46 +88,48 @@ class TestConfigHelper(base.SaharaTestCase): all_confs.name = 'name' all_confs.default_value = 'default' pctx = {'all_confs': [all_confs]} - value = c_helper.get_config_value(pctx, 'service', 'name') + value = config_helper.get_config_value(pctx, 'service', 'name') self.assertEqual(value, 'default') pctx = {'all_confs': []} - self.assertRaises(ex.NotFoundException, c_helper.get_config_value, - pctx, 'service', 'name') + self.assertRaises(ex.PluginNotFoundException, + config_helper.get_config_value, pctx, 'service', + 'name') @mock.patch(plugin_path + 'config_helper.get_config_value') def test_is_swift_enabled(self, get_config_value): - target = c_helper.ENABLE_SWIFT.applicable_target - name = c_helper.ENABLE_SWIFT.name - c_helper.is_swift_enabled(self.pctx, self.cluster) + target = config_helper.ENABLE_SWIFT.applicable_target + name = config_helper.ENABLE_SWIFT.name + config_helper.is_swift_enabled(self.pctx, self.cluster) get_config_value.assert_called_once_with(self.pctx, target, name, self.cluster) @mock.patch(plugin_path + 'config_helper.get_config_value') def test_is_mysql_enabled(self, get_config_value): - target = c_helper.ENABLE_MYSQL.applicable_target - name = c_helper.ENABLE_MYSQL.name - c_helper.is_mysql_enabled(self.pctx, self.cluster) + target = config_helper.ENABLE_MYSQL.applicable_target + name = config_helper.ENABLE_MYSQL.name + config_helper.is_mysql_enabled(self.pctx, self.cluster) get_config_value.assert_called_once_with(self.pctx, target, name, self.cluster) @mock.patch(plugin_path + 'config_helper.get_config_value') def test_is_data_locality_enabled(self, get_config_value): self.CONF.enable_data_locality = False - enabled = c_helper.is_data_locality_enabled(self.pctx, self.cluster) + enabled = config_helper.is_data_locality_enabled(self.pctx, + self.cluster) self.assertEqual(enabled, False) self.CONF.enable_data_locality = True - target = c_helper.ENABLE_DATA_LOCALITY.applicable_target - name = c_helper.ENABLE_DATA_LOCALITY.name - c_helper.is_data_locality_enabled(self.pctx, self.cluster) + target = config_helper.ENABLE_DATA_LOCALITY.applicable_target + name = config_helper.ENABLE_DATA_LOCALITY.name + config_helper.is_data_locality_enabled(self.pctx, self.cluster) get_config_value.assert_called_once_with(self.pctx, target, name, self.cluster) def test_generate_spark_env_configs(self): configs = 'HADOOP_CONF_DIR=/opt/hadoop/etc/hadoop\n' \ 'YARN_CONF_DIR=/opt/hadoop/etc/hadoop' - ret = c_helper.generate_spark_env_configs(self.cluster) + ret = config_helper.generate_spark_env_configs(self.cluster) self.assertEqual(ret, configs) @mock.patch('sahara.plugins.utils.get_config_value_or_default') @@ -134,17 +137,17 @@ class TestConfigHelper(base.SaharaTestCase): get_config_value_or_default): get_config_value_or_default.return_value = None path = 'Executor extra classpath' - ret = c_helper.generate_spark_executor_classpath(self.cluster) + ret = config_helper.generate_spark_executor_classpath(self.cluster) get_config_value_or_default.assert_called_once_with('Spark', path, self.cluster) self.assertEqual(ret, '\n') get_config_value_or_default.return_value = 'test' - ret = c_helper.generate_spark_executor_classpath(self.cluster) + ret = config_helper.generate_spark_executor_classpath(self.cluster) self.assertEqual(ret, 'spark.executor.extraClassPath test') - @mock.patch('sahara.utils.files.get_file_text') + @mock.patch('sahara.plugins.utils.get_file_text') @mock.patch('sahara.plugins.utils.get_config_value_or_default') def test_generate_job_cleanup_config(self, get_config_value_or_default, @@ -160,26 +163,26 @@ class TestConfigHelper(base.SaharaTestCase): 'script': script} get_file_text.return_value = cron get_config_value_or_default.return_value = 1 - ret = c_helper.generate_job_cleanup_config(self.cluster) + ret = config_helper.generate_job_cleanup_config(self.cluster) self.assertEqual(get_config_value_or_default.call_count, 3) self.assertEqual(get_file_text.call_count, 2) self.assertEqual(ret, job_conf) job_conf = {'valid': False} get_config_value_or_default.return_value = 0 - ret = c_helper.generate_job_cleanup_config(self.cluster) + ret = config_helper.generate_job_cleanup_config(self.cluster) self.assertEqual(get_config_value_or_default.call_count, 6) self.assertEqual(ret, job_conf) @mock.patch('sahara.plugins.utils.get_config_value_or_default') def test_get_spark_home(self, get_config_value_or_default): get_config_value_or_default.return_value = 1 - self.assertEqual(c_helper.get_spark_home(self.cluster), 1) + self.assertEqual(config_helper.get_spark_home(self.cluster), 1) get_config_value_or_default.assert_called_once_with('Spark', 'Spark home', self.cluster) - @mock.patch('sahara.utils.files.get_file_text') + @mock.patch('sahara.plugins.utils.get_file_text') @mock.patch('sahara.plugins.utils.get_config_value_or_default') def test_generate_zk_basic_config(self, get_config_value_or_default, get_file_text): @@ -191,6 +194,6 @@ class TestConfigHelper(base.SaharaTestCase): get_config_value_or_default.return_value = 5 get_file_text.return_value = key - ret = c_helper.generate_zk_basic_config(self.cluster) + ret = config_helper.generate_zk_basic_config(self.cluster) self.assertEqual(get_config_value_or_default.call_count, 3) self.assertEqual(ret, actual) diff --git a/sahara/tests/unit/plugins/vanilla/hadoop2/test_configs.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_configs.py similarity index 92% rename from sahara/tests/unit/plugins/vanilla/hadoop2/test_configs.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_configs.py index cbfa6cd..112a219 100644 --- a/sahara/tests/unit/plugins/vanilla/hadoop2/test_configs.py +++ b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_configs.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from sahara.plugins.vanilla.hadoop2 import config as c -from sahara.tests.unit import base +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config as c +from sahara_plugin_vanilla.tests.unit import base class VanillaTwoConfigTestCase(base.SaharaTestCase): diff --git a/sahara/tests/unit/plugins/vanilla/hadoop2/test_edp_engine.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_edp_engine.py similarity index 84% rename from sahara/tests/unit/plugins/vanilla/hadoop2/test_edp_engine.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_edp_engine.py index 720aab9..9dd1ec7 100644 --- a/sahara/tests/unit/plugins/vanilla/hadoop2/test_edp_engine.py +++ b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_edp_engine.py @@ -15,19 +15,22 @@ import mock +from sahara.plugins import base as pb from sahara.plugins import exceptions as ex -from sahara.plugins.vanilla.hadoop2 import edp_engine -from sahara.tests.unit import base as sahara_base +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import edp_engine +from sahara_plugin_vanilla.tests.unit import base as sahara_base class EdpOozieEngineTest(sahara_base.SaharaTestCase): - engine_path = 'sahara.service.edp.oozie.engine.' + engine_path = 'sahara.plugins.edp.' def setUp(self): super(EdpOozieEngineTest, self).setUp() self.cluster = mock.Mock() - with mock.patch('sahara.service.edp.job_utils.get_plugin', + self.override_config("plugins", ["vanilla"]) + pb.setup_plugins() + with mock.patch('sahara.plugins.edp.get_plugin', return_value='test_plugins'): self.engine = edp_engine.EdpOozieEngine(self.cluster) @@ -48,14 +51,14 @@ class EdpOozieEngineTest(sahara_base.SaharaTestCase): ret = self.engine.get_oozie_server_uri(cluster) self.assertEqual(ret, 'test_url/oozie/') - @mock.patch('sahara.plugins.vanilla.utils.get_oozie') + @mock.patch('sahara_plugin_vanilla.plugins.vanilla.utils.get_oozie') def test_get_oozie_server(self, get_oozie): get_oozie.return_value = 'bingo' ret = self.engine.get_oozie_server(self.cluster) get_oozie.assert_called_once_with(self.cluster) self.assertEqual(ret, 'bingo') - @mock.patch(engine_path + 'OozieJobEngine.validate_job_execution') + @mock.patch(engine_path + 'PluginsOozieJobEngine.validate_job_execution') @mock.patch('sahara.plugins.utils.get_instances_count') def test_validate_job_execution(self, get_instances_count, @@ -72,7 +75,7 @@ class EdpOozieEngineTest(sahara_base.SaharaTestCase): validate_job_execution.assert_called_once_with(self.cluster, job, data) - @mock.patch('sahara.service.edp.hdfs_helper.create_dir_hadoop2') + @mock.patch('sahara.plugins.edp.create_dir_hadoop2') def test_create_hdfs_dir(self, create_dir_hadoop2): self.engine.get_hdfs_user = mock.Mock(return_value='test_user') remote = mock.Mock() diff --git a/sahara/tests/unit/plugins/vanilla/hadoop2/test_oozie_helper.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_oozie_helper.py similarity index 87% rename from sahara/tests/unit/plugins/vanilla/hadoop2/test_oozie_helper.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_oozie_helper.py index c870c3b..72debda 100644 --- a/sahara/tests/unit/plugins/vanilla/hadoop2/test_oozie_helper.py +++ b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_oozie_helper.py @@ -15,8 +15,8 @@ import mock -from sahara.plugins.vanilla.hadoop2 import oozie_helper as o_helper -from sahara.tests.unit import base +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import oozie_helper +from sahara_plugin_vanilla.tests.unit import base class TestOozieHelper(base.SaharaTestCase): @@ -46,10 +46,11 @@ class TestOozieHelper(base.SaharaTestCase): 'oozie.service.HadoopAccessorService.hadoop.configurations': '*=/root' } - ret = o_helper.get_oozie_required_xml_configs(hadoop_conf_dir) + ret = oozie_helper.get_oozie_required_xml_configs(hadoop_conf_dir) self.assertEqual(ret, configs) - @mock.patch('sahara.plugins.vanilla.hadoop2.utils.get_oozie_password') + @mock.patch('sahara_plugin_vanilla.plugins.vanilla.hadoop2.' + 'utils.get_oozie_password') def test_get_oozie_mysql_configs(self, get_oozie_password): get_oozie_password.return_value = '123' configs = { @@ -61,6 +62,6 @@ class TestOozieHelper(base.SaharaTestCase): 'oozie.service.JPAService.jdbc.password': '123' } cluster = mock.Mock() - ret = o_helper.get_oozie_mysql_configs(cluster) + ret = oozie_helper.get_oozie_mysql_configs(cluster) get_oozie_password.assert_called_once_with(cluster) self.assertEqual(ret, configs) diff --git a/sahara/tests/unit/plugins/vanilla/hadoop2/test_plugin.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_plugin.py similarity index 84% rename from sahara/tests/unit/plugins/vanilla/hadoop2/test_plugin.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_plugin.py index 94a82a1..c0d3869 100644 --- a/sahara/tests/unit/plugins/vanilla/hadoop2/test_plugin.py +++ b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_plugin.py @@ -15,22 +15,20 @@ import mock -from sahara import conductor as cond -from sahara import context from sahara.plugins import base as pb -from sahara.tests.unit import base -from sahara.utils import edp - - -conductor = cond.API +from sahara.plugins import conductor +from sahara.plugins import context +from sahara.plugins import edp +from sahara_plugin_vanilla.tests.unit import base class VanillaPluginTest(base.SaharaWithDbTestCase): def setUp(self): super(VanillaPluginTest, self).setUp() + self.override_config("plugins", ["vanilla"]) pb.setup_plugins() - @mock.patch('sahara.service.edp.hdfs_helper.create_dir_hadoop2') + @mock.patch('sahara.plugins.edp.create_dir_hadoop2') def test_edp_calls_hadoop2_create_dir(self, create_dir): for version in ['2.7.1']: cluster_dict = { diff --git a/sahara/tests/unit/plugins/vanilla/hadoop2/test_recommendation_utils.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_recommendation_utils.py similarity index 93% rename from sahara/tests/unit/plugins/vanilla/hadoop2/test_recommendation_utils.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_recommendation_utils.py index 424fd85..c8b0bf1 100644 --- a/sahara/tests/unit/plugins/vanilla/hadoop2/test_recommendation_utils.py +++ b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_recommendation_utils.py @@ -16,7 +16,7 @@ import mock import testtools -from sahara.plugins.vanilla.hadoop2 import recommendations_utils as ru +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import recommendations_utils CONFIGURATION_SCHEMA = { 'cluster_configs': { @@ -54,7 +54,7 @@ class TestVersionHandler(testtools.TestCase): 'HadoopAutoConfigsProvider') def test_recommend_configs(self, provider): f_cluster, f_configs = mock.Mock(), mock.Mock() - ru.recommend_configs(f_cluster, f_configs, False) + recommendations_utils.recommend_configs(f_cluster, f_configs, False) self.assertEqual([ mock.call(CONFIGURATION_SCHEMA, f_configs, f_cluster, False) ], provider.call_args_list) diff --git a/sahara/tests/unit/plugins/vanilla/hadoop2/test_run_scripts.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_run_scripts.py similarity index 82% rename from sahara/tests/unit/plugins/vanilla/hadoop2/test_run_scripts.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_run_scripts.py index 8ffb4be..0eb1bd5 100644 --- a/sahara/tests/unit/plugins/vanilla/hadoop2/test_run_scripts.py +++ b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_run_scripts.py @@ -13,20 +13,30 @@ # See the License for the specific language governing permissions and # limitations under the License. +from functools import wraps import mock -from sahara.i18n import _ + +def mock_event_wrapper(*args, **kwargs): + def decorator(f): + @wraps(f) + def decorated_function(*args, **kwargs): + return f(*args, **kwargs) + return decorated_function + return decorator + +from sahara.plugins import edp from sahara.plugins import utils as pu -from sahara.plugins.vanilla.hadoop2 import config_helper as c_helper -from sahara.plugins.vanilla.hadoop2 import run_scripts as rs -from sahara.tests.unit import base -from sahara.utils import edp -from sahara.utils import files +mock.patch('sahara.plugins.utils.event_wrapper', mock_event_wrapper).start() +from sahara_plugin_vanilla.i18n import _ +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config_helper +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import run_scripts as rs +from sahara_plugin_vanilla.tests.unit import base class RunScriptsTest(base.SaharaTestCase): - PLUGINS_PATH = 'sahara.plugins.vanilla.hadoop2.' + PLUGINS_PATH = 'sahara_plugin_vanilla.plugins.vanilla.hadoop2.' def setUp(self): super(RunScriptsTest, self).setUp() @@ -36,10 +46,11 @@ class RunScriptsTest(base.SaharaTestCase): self.remote.__enter__ = self.remote self.remote.__exit__ = mock.Mock() self.instance.remote.return_value = self.remote + pu.event_wrapper = mock_event_wrapper @mock.patch(PLUGINS_PATH + 'run_scripts._start_processes') - @mock.patch('sahara.context.set_current_instance_id') - @mock.patch('sahara.utils.cluster_progress_ops.add_provisioning_step') + @mock.patch('sahara.plugins.context.set_current_instance_id') + @mock.patch('sahara.plugins.utils.add_provisioning_step') @mock.patch('sahara.plugins.utils.instances_with_services') def test_start_dn_nm_processes(self, instances_with_services, add_provisioning_step, @@ -60,21 +71,21 @@ class RunScriptsTest(base.SaharaTestCase): set_current_instance_id.assert_called_once_with('123') _start_processes.assert_called_once_with(ins, ['datanode']) - @mock.patch('sahara.utils.cluster.check_cluster_exists') + @mock.patch('sahara.plugins.utils.check_cluster_exists') def test_start_processes_datanode(self, check_cluster_exists): processes = ['datanode'] rs._start_processes(self.instance, processes) self.r.execute_command.assert_called_once_with( 'sudo su - -c "hadoop-daemon.sh start datanode" hadoop') - @mock.patch('sahara.utils.cluster.check_cluster_exists') + @mock.patch('sahara.plugins.utils.check_cluster_exists') def test_start_processes_nodemanager(self, check_cluster_exists): processes = ['nodemanager'] rs._start_processes(self.instance, processes) self.r.execute_command.assert_called_once_with( 'sudo su - -c "yarn-daemon.sh start nodemanager" hadoop') - @mock.patch('sahara.utils.cluster.check_cluster_exists') + @mock.patch('sahara.plugins.utils.check_cluster_exists') def test_start_processes_both(self, check_cluster_exists): processes = ['datanode', 'nodemanager'] rs._start_processes(self.instance, processes) @@ -95,8 +106,8 @@ class RunScriptsTest(base.SaharaTestCase): self.remote.execute_command.assert_called_once_with( 'sudo su - -c "yarn-daemon.sh start %s" hadoop' % process) - @mock.patch('sahara.utils.cluster.check_cluster_exists') - @mock.patch('sahara.utils.cluster_progress_ops.add_provisioning_step') + @mock.patch('sahara.plugins.utils.check_cluster_exists') + @mock.patch('sahara.plugins.utils.add_provisioning_step') def test_start_historyserver(self, add_provisioning_step, check_cluster_exists): rs.start_historyserver(self.instance) @@ -109,9 +120,9 @@ class RunScriptsTest(base.SaharaTestCase): @mock.patch(PLUGINS_PATH + 'run_scripts._start_mysql') @mock.patch(PLUGINS_PATH + 'config_helper.is_mysql_enabled') @mock.patch(PLUGINS_PATH + 'utils.get_oozie_password') - @mock.patch('sahara.context.set_current_instance_id') - @mock.patch('sahara.utils.cluster.check_cluster_exists') - @mock.patch('sahara.utils.cluster_progress_ops.add_provisioning_step') + @mock.patch('sahara.plugins.context.set_current_instance_id') + @mock.patch('sahara.plugins.utils.check_cluster_exists') + @mock.patch('sahara.plugins.utils.add_provisioning_step') def test_start_oozie_process(self, add_provisioning_step, check_cluster_exists, set_current_instance_id, get_oozie_password, @@ -120,8 +131,9 @@ class RunScriptsTest(base.SaharaTestCase): self.instance.instance_id = '112233' pctx = mock.Mock() is_mysql_enabled.return_value = True - sql_script = files.get_file_text( - 'plugins/vanilla/hadoop2/resources/create_oozie_db.sql') + sql_script = pu.get_file_text( + 'plugins/vanilla/hadoop2/resources/create_oozie_db.sql', + 'sahara_plugin_vanilla') get_oozie_password.return_value = '123' pwd_script = sql_script.replace('password', '123') rs.start_oozie_process(pctx, self.instance) @@ -138,9 +150,9 @@ class RunScriptsTest(base.SaharaTestCase): _start_oozie.assert_called_once_with(self.r) @mock.patch(PLUGINS_PATH + 'config_helper.get_spark_home') - @mock.patch('sahara.context.set_current_instance_id') - @mock.patch('sahara.utils.cluster.check_cluster_exists') - @mock.patch('sahara.utils.cluster_progress_ops.add_provisioning_step') + @mock.patch('sahara.plugins.context.set_current_instance_id') + @mock.patch('sahara.plugins.utils.check_cluster_exists') + @mock.patch('sahara.plugins.utils.add_provisioning_step') def test_start_spark_history_server(self, add_provisioning_step, check_cluster_exists, set_current_instance_id, @@ -158,9 +170,9 @@ class RunScriptsTest(base.SaharaTestCase): self.remote.execute_command.assert_called_once_with( 'sudo su - -c "hdfs namenode -format" hadoop') - @mock.patch('sahara.plugins.vanilla.utils.get_namenode') - @mock.patch('sahara.utils.cluster.check_cluster_exists') - @mock.patch('sahara.utils.cluster_progress_ops.add_provisioning_step') + @mock.patch('sahara_plugin_vanilla.plugins.vanilla.utils.get_namenode') + @mock.patch('sahara.plugins.utils.check_cluster_exists') + @mock.patch('sahara.plugins.utils.add_provisioning_step') def test_refresh_hadoop_nodes(self, add_provisioning_step, check_cluster_exists, get_namenode): cluster = mock.Mock() @@ -170,9 +182,10 @@ class RunScriptsTest(base.SaharaTestCase): self.remote.execute_command.assert_called_once_with( 'sudo su - -c "hdfs dfsadmin -refreshNodes" hadoop') - @mock.patch('sahara.plugins.vanilla.utils.get_resourcemanager') - @mock.patch('sahara.utils.cluster.check_cluster_exists') - @mock.patch('sahara.utils.cluster_progress_ops.add_provisioning_step') + @mock.patch('sahara_plugin_vanilla.plugins.vanilla.utils.' + 'get_resourcemanager') + @mock.patch('sahara.plugins.utils.check_cluster_exists') + @mock.patch('sahara.plugins.utils.add_provisioning_step') def test_refresh_yarn_nodes(self, add_provisioning_step, check_cluster_exists, get_resourcemanager): cluster = mock.Mock() @@ -207,11 +220,11 @@ class RunScriptsTest(base.SaharaTestCase): self.r.execute_command.assert_called_once_with( 'sudo su - -c "/opt/oozie/bin/oozied.sh start" hadoop') - @mock.patch('sahara.plugins.vanilla.utils.get_namenode') - @mock.patch('sahara.plugins.vanilla.utils.get_datanodes') - @mock.patch('sahara.utils.cluster.check_cluster_exists') - @mock.patch('sahara.utils.cluster_progress_ops.add_provisioning_step') - @mock.patch('sahara.utils.poll_utils.plugin_option_poll') + @mock.patch('sahara_plugin_vanilla.plugins.vanilla.utils.get_namenode') + @mock.patch('sahara_plugin_vanilla.plugins.vanilla.utils.get_datanodes') + @mock.patch('sahara.plugins.utils.check_cluster_exists') + @mock.patch('sahara.plugins.utils.add_provisioning_step') + @mock.patch('sahara.plugins.utils.plugin_option_poll') def test_await_datanodes(self, plugin_option_poll, add_provisioning_step, check_cluster_exists, get_datanodes, get_namenode): @@ -226,7 +239,7 @@ class RunScriptsTest(base.SaharaTestCase): get_namenode.return_value = namenode mess = _('Waiting on 1 datanodes to start up') test_data = {'remote': r, 'count': 1} - timeout = c_helper.DATANODES_STARTUP_TIMEOUT + timeout = config_helper.DATANODES_STARTUP_TIMEOUT rs.await_datanodes(cluster) get_datanodes.assert_called_once_with(cluster) get_namenode.assert_called_once_with(cluster) @@ -276,10 +289,10 @@ class RunScriptsTest(base.SaharaTestCase): @mock.patch(PLUGINS_PATH + 'run_scripts._start_mysql') @mock.patch(PLUGINS_PATH + 'run_scripts._hive_copy_shared_conf') @mock.patch(PLUGINS_PATH + 'run_scripts._hive_create_warehouse_dir') - @mock.patch('sahara.plugins.vanilla.utils.get_oozie') - @mock.patch('sahara.context.set_current_instance_id') - @mock.patch('sahara.utils.cluster.check_cluster_exists') - @mock.patch('sahara.utils.cluster_progress_ops.add_provisioning_step') + @mock.patch('sahara_plugin_vanilla.plugins.vanilla.utils.get_oozie') + @mock.patch('sahara.plugins.context.set_current_instance_id') + @mock.patch('sahara.plugins.utils.check_cluster_exists') + @mock.patch('sahara.plugins.utils.add_provisioning_step') def test_start_hiveserver_process(self, add_provisioning_step, check_cluster_exists, set_current_instance_id, get_oozie, @@ -294,8 +307,9 @@ class RunScriptsTest(base.SaharaTestCase): self.instance.cluster.hadoop_version = '2.7.1' ng_cluster = self.instance.node_group.cluster get_oozie.return_value = None - sql_script = files.get_file_text( - 'plugins/vanilla/v2_7_1/resources/create_hive_db.sql') + sql_script = pu.get_file_text( + 'plugins/vanilla/v2_7_1/resources/create_hive_db.sql', + 'sahara_plugin_vanilla') get_hive_password.return_value = '123' pwd_script = sql_script.replace('{{password}}', '123') rs.start_hiveserver_process(pctx, self.instance) diff --git a/sahara/tests/unit/plugins/vanilla/hadoop2/test_scaling.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_scaling.py similarity index 89% rename from sahara/tests/unit/plugins/vanilla/hadoop2/test_scaling.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_scaling.py index ab49e58..69146a0 100644 --- a/sahara/tests/unit/plugins/vanilla/hadoop2/test_scaling.py +++ b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_scaling.py @@ -15,16 +15,16 @@ import mock -from sahara.i18n import _ -from sahara.plugins.vanilla.hadoop2 import config_helper as c_helper -from sahara.plugins.vanilla.hadoop2 import scaling -from sahara.plugins.vanilla.hadoop2 import utils as pu -from sahara.tests.unit import base +from sahara_plugin_vanilla.i18n import _ +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import config_helper +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import scaling +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import utils as pu +from sahara_plugin_vanilla.tests.unit import base class ScalingTest(base.SaharaTestCase): - PLUGINS_PATH = 'sahara.plugins.vanilla.hadoop2.' + PLUGINS_PATH = 'sahara_plugin_vanilla.plugins.vanilla.hadoop2.' def setUp(self): super(ScalingTest, self).setUp() @@ -37,8 +37,9 @@ class ScalingTest(base.SaharaTestCase): return_value=self.r) self.instance.remote.return_value.__exit__ = mock.Mock() - @mock.patch('sahara.swift.swift_helper.install_ssl_certs') - @mock.patch('sahara.plugins.vanilla.utils.get_resourcemanager') + @mock.patch('sahara.plugins.swift_helper.install_ssl_certs') + @mock.patch('sahara_plugin_vanilla.plugins.vanilla.utils.' + 'get_resourcemanager') @mock.patch(PLUGINS_PATH + 'run_scripts.refresh_zk_servers') @mock.patch(PLUGINS_PATH + 'config.configure_zookeeper') @mock.patch(PLUGINS_PATH + 'run_scripts.start_dn_nm_processes') @@ -81,10 +82,10 @@ class ScalingTest(base.SaharaTestCase): ret = scaling._get_instances_with_service(instances, service) self.assertEqual(ret, [ins_1]) - @mock.patch('sahara.plugins.vanilla.utils.get_nodemanagers') - @mock.patch('sahara.plugins.vanilla.utils.get_datanodes') - @mock.patch('sahara.utils.cluster.check_cluster_exists') - @mock.patch('sahara.utils.cluster_progress_ops.add_provisioning_step') + @mock.patch('sahara_plugin_vanilla.plugins.vanilla.utils.get_nodemanagers') + @mock.patch('sahara_plugin_vanilla.plugins.vanilla.utils.get_datanodes') + @mock.patch('sahara.plugins.utils.check_cluster_exists') + @mock.patch('sahara.plugins.utils.add_provisioning_step') @mock.patch('sahara.plugins.utils.generate_fqdn_host_names') @mock.patch('sahara.plugins.utils.get_instances') def test_update_include_files(self, get_instances, @@ -120,7 +121,8 @@ class ScalingTest(base.SaharaTestCase): host, DIR))] self.r.execute_command.assert_has_calls(command_calls, any_order=True) - @mock.patch('sahara.plugins.vanilla.utils.get_resourcemanager') + @mock.patch('sahara_plugin_vanilla.plugins.vanilla.utils.' + 'get_resourcemanager') @mock.patch(PLUGINS_PATH + 'run_scripts.refresh_zk_servers') @mock.patch(PLUGINS_PATH + 'config.configure_zookeeper') @mock.patch(PLUGINS_PATH + 'config.configure_topology_data') @@ -217,7 +219,7 @@ class ScalingTest(base.SaharaTestCase): ret = scaling.is_decommissioned(cluster, check_func, instances) self.assertEqual(ret, False) - @mock.patch('sahara.utils.poll_utils.plugin_option_poll') + @mock.patch('sahara.plugins.utils.plugin_option_poll') def test_check_decommission(self, plugin_option_poll): check_func = mock.Mock() option = mock.Mock() @@ -233,12 +235,12 @@ class ScalingTest(base.SaharaTestCase): sample_dict) @mock.patch(PLUGINS_PATH + 'scaling._check_decommission') - @mock.patch('sahara.utils.cluster.check_cluster_exists') - @mock.patch('sahara.utils.cluster_progress_ops.add_provisioning_step') + @mock.patch('sahara.plugins.utils.check_cluster_exists') + @mock.patch('sahara.plugins.utils.add_provisioning_step') def test_check_nodemanagers_decommission(self, add_provisioning_step, check_cluster_exists, _check_decommission): - timeout = c_helper.NODEMANAGERS_DECOMMISSIONING_TIMEOUT + timeout = config_helper.NODEMANAGERS_DECOMMISSIONING_TIMEOUT status = pu.get_nodemanagers_status scaling._check_nodemanagers_decommission(self.cluster, self.instances) _check_decommission.assert_called_once_with(self.cluster, @@ -246,12 +248,12 @@ class ScalingTest(base.SaharaTestCase): status, timeout) @mock.patch(PLUGINS_PATH + 'scaling._check_decommission') - @mock.patch('sahara.utils.cluster.check_cluster_exists') - @mock.patch('sahara.utils.cluster_progress_ops.add_provisioning_step') + @mock.patch('sahara.plugins.utils.check_cluster_exists') + @mock.patch('sahara.plugins.utils.add_provisioning_step') def test_check_datanodes_decommission(self, add_provisioning_step, check_cluster_exists, _check_decommission): - timeout = c_helper.DATANODES_DECOMMISSIONING_TIMEOUT + timeout = config_helper.DATANODES_DECOMMISSIONING_TIMEOUT status = pu.get_datanodes_status scaling._check_datanodes_decommission(self.cluster, self.instances) _check_decommission.assert_called_once_with(self.cluster, diff --git a/sahara/tests/unit/plugins/vanilla/hadoop2/test_starting_scripts.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_starting_scripts.py similarity index 81% rename from sahara/tests/unit/plugins/vanilla/hadoop2/test_starting_scripts.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_starting_scripts.py index 45a062b..5b60519 100644 --- a/sahara/tests/unit/plugins/vanilla/hadoop2/test_starting_scripts.py +++ b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_starting_scripts.py @@ -15,13 +15,13 @@ import mock -from sahara.plugins.vanilla.hadoop2 import starting_scripts as s_scripts -from sahara.tests.unit import base +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import starting_scripts +from sahara_plugin_vanilla.tests.unit import base class StartingScriptsTest(base.SaharaTestCase): - plugins_path = 'sahara.plugins.vanilla.' + plugins_path = 'sahara_plugin_vanilla.plugins.vanilla.' def setUp(self): super(StartingScriptsTest, self).setUp() @@ -32,11 +32,11 @@ class StartingScriptsTest(base.SaharaTestCase): def test_start_namenode(self, _start_namenode, get_namenode): namenode = mock.Mock() get_namenode.return_value = namenode - s_scripts.start_namenode(self.cluster) + starting_scripts.start_namenode(self.cluster) get_namenode.assert_called_once_with(self.cluster) _start_namenode.assert_called_once_with(namenode) - @mock.patch('sahara.utils.cluster.check_cluster_exists') + @mock.patch('sahara.plugins.utils.check_cluster_exists') @mock.patch(plugins_path + 'hadoop2.run_scripts.start_hadoop_process') @mock.patch(plugins_path + 'hadoop2.run_scripts.format_namenode') def test__start_namenode(self, format_namenode, @@ -44,7 +44,7 @@ class StartingScriptsTest(base.SaharaTestCase): check_cluster_exists): check_cluster_exists.return_value = None nn = mock.Mock() - s_scripts._start_namenode(nn) + starting_scripts._start_namenode(nn) format_namenode.assert_called_once_with(nn) start_hadoop_process.assert_called_once_with(nn, 'namenode') @@ -54,21 +54,21 @@ class StartingScriptsTest(base.SaharaTestCase): def test_start_secondarynamenode(self, get_secondarynamenode, _start_secondarynamenode): get_secondarynamenode.return_value = 0 - s_scripts.start_secondarynamenode(self.cluster) + starting_scripts.start_secondarynamenode(self.cluster) get_secondarynamenode.assert_called_once_with(self.cluster) get_secondarynamenode.return_value = 1 - s_scripts.start_secondarynamenode(self.cluster) + starting_scripts.start_secondarynamenode(self.cluster) _start_secondarynamenode.assert_called_once_with(1) self.assertEqual(get_secondarynamenode.call_count, 2) - @mock.patch('sahara.utils.cluster.check_cluster_exists') + @mock.patch('sahara.plugins.utils.check_cluster_exists') @mock.patch(plugins_path + 'hadoop2.run_scripts.start_hadoop_process') def test__start_secondarynamenode(self, start_hadoop_process, check_cluster_exists): check_cluster_exists.return_value = None snn = mock.Mock() - s_scripts._start_secondarynamenode(snn) + starting_scripts._start_secondarynamenode(snn) start_hadoop_process.assert_called_once_with(snn, 'secondarynamenode') @@ -78,21 +78,21 @@ class StartingScriptsTest(base.SaharaTestCase): def test_start_resourcemanager(self, get_resourcemanager, _start_resourcemanager): get_resourcemanager.return_value = 0 - s_scripts.start_resourcemanager(self.cluster) + starting_scripts.start_resourcemanager(self.cluster) get_resourcemanager.assert_called_once_with(self.cluster) get_resourcemanager.return_value = 1 - s_scripts.start_resourcemanager(self.cluster) + starting_scripts.start_resourcemanager(self.cluster) _start_resourcemanager.assert_called_once_with(1) self.assertEqual(get_resourcemanager.call_count, 2) - @mock.patch('sahara.utils.cluster.check_cluster_exists') + @mock.patch('sahara.plugins.utils.check_cluster_exists') @mock.patch(plugins_path + 'hadoop2.run_scripts.start_yarn_process') def test__start_resourcemanager(self, start_yarn_process, check_cluster_exists): check_cluster_exists.return_value = None snn = mock.Mock() - s_scripts._start_resourcemanager(snn) + starting_scripts._start_resourcemanager(snn) start_yarn_process.assert_called_once_with(snn, 'resourcemanager') @@ -101,11 +101,11 @@ class StartingScriptsTest(base.SaharaTestCase): def test_start_historyserver(self, get_historyserver, start_historyserver): get_historyserver.return_value = 0 - s_scripts.start_historyserver(self.cluster) + starting_scripts.start_historyserver(self.cluster) get_historyserver.assert_called_once_with(self.cluster) get_historyserver.return_value = 1 - s_scripts.start_historyserver(self.cluster) + starting_scripts.start_historyserver(self.cluster) start_historyserver.assert_called_once_with(1) self.assertEqual(get_historyserver.call_count, 2) @@ -114,11 +114,11 @@ class StartingScriptsTest(base.SaharaTestCase): def test_start_oozie(self, get_oozie, start_oozie_process): pctx = mock.Mock() get_oozie.return_value = 0 - s_scripts.start_oozie(pctx, self.cluster) + starting_scripts.start_oozie(pctx, self.cluster) get_oozie.assert_called_once_with(self.cluster) get_oozie.return_value = 1 - s_scripts.start_oozie(pctx, self.cluster) + starting_scripts.start_oozie(pctx, self.cluster) start_oozie_process.assert_called_once_with(pctx, 1) self.assertEqual(get_oozie.call_count, 2) @@ -129,11 +129,11 @@ class StartingScriptsTest(base.SaharaTestCase): start_hiveserver_process): pctx = mock.Mock() get_hiveserver.return_value = 0 - s_scripts.start_hiveserver(pctx, self.cluster) + starting_scripts.start_hiveserver(pctx, self.cluster) get_hiveserver.assert_called_once_with(self.cluster) get_hiveserver.return_value = 1 - s_scripts.start_hiveserver(pctx, self.cluster) + starting_scripts.start_hiveserver(pctx, self.cluster) start_hiveserver_process.assert_called_once_with(pctx, 1) self.assertEqual(get_hiveserver.call_count, 2) @@ -143,10 +143,10 @@ class StartingScriptsTest(base.SaharaTestCase): def test_start_spark(self, get_spark_history_server, start_spark_history_server): get_spark_history_server.return_value = 0 - s_scripts.start_spark(self.cluster) + starting_scripts.start_spark(self.cluster) get_spark_history_server.assert_called_once_with(self.cluster) get_spark_history_server.return_value = 1 - s_scripts.start_spark(self.cluster) + starting_scripts.start_spark(self.cluster) start_spark_history_server.assert_called_once_with(1) self.assertEqual(get_spark_history_server.call_count, 2) diff --git a/sahara/tests/unit/plugins/vanilla/hadoop2/test_utils.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_utils.py similarity index 78% rename from sahara/tests/unit/plugins/vanilla/hadoop2/test_utils.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_utils.py index 98b4d32..a39d31c 100644 --- a/sahara/tests/unit/plugins/vanilla/hadoop2/test_utils.py +++ b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_utils.py @@ -15,16 +15,17 @@ import mock -from sahara.plugins.vanilla.hadoop2 import utils as u -from sahara.tests.unit import base -from sahara.utils import files +from sahara.plugins import utils +from sahara_plugin_vanilla.plugins.vanilla.hadoop2 import utils as u +from sahara_plugin_vanilla.tests.unit import base class UtilsTestCase(base.SaharaTestCase): - @mock.patch('sahara.plugins.vanilla.utils.get_namenode') + @mock.patch('sahara_plugin_vanilla.plugins.vanilla.utils.get_namenode') def test_datanodes_status(self, nn): - report = files.get_file_text( - 'tests/unit/plugins/vanilla/hadoop2/resources/dfs-report.txt') + report = utils.get_file_text( + 'tests/unit/plugins/vanilla/hadoop2/resources/dfs-report.txt', + 'sahara_plugin_vanilla') nn.return_value = self._get_instance(report) statuses = u.get_datanodes_status(None) @@ -38,10 +39,12 @@ class UtilsTestCase(base.SaharaTestCase): self.assertEqual(expected, statuses) - @mock.patch('sahara.plugins.vanilla.utils.get_resourcemanager') + @mock.patch('sahara_plugin_vanilla.plugins.vanilla.utils.' + 'get_resourcemanager') def test_nodemanagers_status(self, rm): - report = files.get_file_text( - 'tests/unit/plugins/vanilla/hadoop2/resources/yarn-report.txt') + report = utils.get_file_text( + 'tests/unit/plugins/vanilla/hadoop2/resources/yarn-report.txt', + 'sahara_plugin_vanilla') rm.return_value = self._get_instance(report) statuses = u.get_nodemanagers_status(None) @@ -65,11 +68,11 @@ class UtilsTestCase(base.SaharaTestCase): return inst - @mock.patch('sahara.conductor.API.cluster_get') - @mock.patch('sahara.service.castellan.utils.get_secret') - @mock.patch('sahara.service.castellan.utils.store_secret') - @mock.patch('sahara.plugins.vanilla.utils') - @mock.patch('sahara.conductor.API.cluster_update') + @mock.patch('sahara.plugins.conductor.cluster_get') + @mock.patch('sahara.plugins.castellan_utils.get_secret') + @mock.patch('sahara.plugins.castellan_utils.store_secret') + @mock.patch('sahara_plugin_vanilla.plugins.vanilla.utils') + @mock.patch('sahara.plugins.conductor.cluster_update') def test_oozie_password(self, cluster_update, vu, store_secret, get_secret, conductor): cluster = mock.MagicMock() @@ -91,7 +94,7 @@ class UtilsTestCase(base.SaharaTestCase): result = u.get_oozie_password(cluster) self.assertEqual('oozie_pass', result) - @mock.patch('sahara.service.castellan.utils.delete_secret') + @mock.patch('sahara.plugins.castellan_utils.delete_secret') def test_delete_oozie_password(self, delete_secret): cluster = mock.MagicMock() cluster.extra.to_dict = mock.MagicMock() @@ -104,10 +107,10 @@ class UtilsTestCase(base.SaharaTestCase): u.delete_oozie_password(cluster) delete_secret.assert_called_once_with("31415926") - @mock.patch('sahara.conductor.API.cluster_get') - @mock.patch('sahara.service.castellan.utils.get_secret') - @mock.patch('sahara.service.castellan.utils.store_secret') - @mock.patch('sahara.conductor.API.cluster_update') + @mock.patch('sahara.plugins.conductor.cluster_get') + @mock.patch('sahara.plugins.castellan_utils.get_secret') + @mock.patch('sahara.plugins.castellan_utils.store_secret') + @mock.patch('sahara.plugins.conductor.cluster_update') def test_get_hive_password(self, cluster_update, store_secret, get_secret, conductor): cluster = mock.MagicMock() @@ -127,7 +130,7 @@ class UtilsTestCase(base.SaharaTestCase): result = u.get_hive_password(cluster) self.assertEqual('hive_pass', result) - @mock.patch('sahara.service.castellan.utils.delete_secret') + @mock.patch('sahara.plugins.castellan_utils.delete_secret') def test_delete_hive_password(self, delete_secret): cluster = mock.MagicMock() diff --git a/sahara/tests/unit/plugins/vanilla/hadoop2/test_validation.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_validation.py similarity index 96% rename from sahara/tests/unit/plugins/vanilla/hadoop2/test_validation.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_validation.py index ad90c3b..2e71db9 100644 --- a/sahara/tests/unit/plugins/vanilla/hadoop2/test_validation.py +++ b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/hadoop2/test_validation.py @@ -16,9 +16,9 @@ import testtools from sahara.plugins import exceptions as ex -from sahara.plugins.vanilla import plugin as p -from sahara.tests.unit import base -from sahara.tests.unit import testutils as tu +from sahara.plugins import testutils as tu +from sahara_plugin_vanilla.plugins.vanilla import plugin as p +from sahara_plugin_vanilla.tests.unit import base class ValidationTest(base.SaharaTestCase): diff --git a/sahara/tests/unit/plugins/vanilla/test_confighints_helper.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/test_confighints_helper.py similarity index 60% rename from sahara/tests/unit/plugins/vanilla/test_confighints_helper.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/test_confighints_helper.py index 1eafb33..effbdf5 100644 --- a/sahara/tests/unit/plugins/vanilla/test_confighints_helper.py +++ b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/test_confighints_helper.py @@ -15,36 +15,40 @@ import mock -from sahara.plugins.vanilla import confighints_helper as ch_helper -from sahara.tests.unit import base as sahara_base +from sahara_plugin_vanilla.plugins.vanilla import confighints_helper +from sahara_plugin_vanilla.tests.unit import base as sahara_base class ConfigHintsHelperTest(sahara_base.SaharaTestCase): - @mock.patch('sahara.utils.xmlutils.load_hadoop_xml_defaults', + @mock.patch('sahara.plugins.utils.load_hadoop_xml_defaults', return_value=[]) def test_get_possible_hive_config_from(self, load_hadoop_xml_defaults): expected_config = { 'configs': [], 'params': {} } - actual_config = ch_helper.get_possible_hive_config_from( + actual_config = confighints_helper.get_possible_hive_config_from( 'sample-config.xml') - load_hadoop_xml_defaults.assert_called_once_with('sample-config.xml') + load_hadoop_xml_defaults.assert_called_once_with( + 'sample-config.xml', 'sahara_plugin_vanilla') self.assertEqual(expected_config, actual_config) - @mock.patch('sahara.utils.xmlutils.load_hadoop_xml_defaults', + @mock.patch('sahara.plugins.utils.load_hadoop_xml_defaults', + return_value=[]) + @mock.patch('sahara.plugins.edp.get_possible_mapreduce_configs', return_value=[]) def test_get_possible_mapreduce_config_from( - self, load_hadoop_xml_defaults): + self, get_possible_mapreduce_configs, load_hadoop_xml_defaults): expected_config = { 'configs': [], } - actual_config = ch_helper.get_possible_mapreduce_config_from( + actual_config = confighints_helper.get_possible_mapreduce_config_from( 'sample-config.xml') - load_hadoop_xml_defaults.assert_any_call('sample-config.xml') + load_hadoop_xml_defaults.assert_any_call('sample-config.xml', + 'sahara_plugin_vanilla') self.assertEqual(expected_config, actual_config) - @mock.patch('sahara.utils.xmlutils.load_hadoop_xml_defaults', + @mock.patch('sahara.plugins.utils.load_hadoop_xml_defaults', return_value=[]) def test_get_possible_pig_config_from( self, load_hadoop_xml_defaults): @@ -53,7 +57,8 @@ class ConfigHintsHelperTest(sahara_base.SaharaTestCase): 'args': [], 'params': {} } - actual_config = ch_helper.get_possible_pig_config_from( + actual_config = confighints_helper.get_possible_pig_config_from( 'sample-config.xml') - load_hadoop_xml_defaults.assert_called_once_with('sample-config.xml') + load_hadoop_xml_defaults.assert_called_once_with( + 'sample-config.xml', 'sahara_plugin_vanilla') self.assertEqual(expected_config, actual_config) diff --git a/sahara/tests/unit/plugins/vanilla/test_utils.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/test_utils.py similarity index 95% rename from sahara/tests/unit/plugins/vanilla/test_utils.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/test_utils.py index e65d449..86566a2 100644 --- a/sahara/tests/unit/plugins/vanilla/test_utils.py +++ b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/test_utils.py @@ -13,10 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from sahara.plugins.vanilla import plugin as p -from sahara.plugins.vanilla import utils as u -from sahara.tests.unit import base -from sahara.tests.unit import testutils as tu +from sahara.plugins import testutils as tu +from sahara_plugin_vanilla.plugins.vanilla import plugin as p +from sahara_plugin_vanilla.plugins.vanilla import utils as u +from sahara_plugin_vanilla.tests.unit import base class TestUtils(base.SaharaWithDbTestCase): diff --git a/sahara/tests/unit/plugins/vanilla/v2_7_1/__init__.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_7_1/__init__.py similarity index 100% rename from sahara/tests/unit/plugins/vanilla/v2_7_1/__init__.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_7_1/__init__.py diff --git a/sahara/tests/unit/plugins/vanilla/v2_7_1/test_config_helper.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_7_1/test_config_helper.py similarity index 73% rename from sahara/tests/unit/plugins/vanilla/v2_7_1/test_config_helper.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_7_1/test_config_helper.py index 70f80c5..2d4799a 100644 --- a/sahara/tests/unit/plugins/vanilla/v2_7_1/test_config_helper.py +++ b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_7_1/test_config_helper.py @@ -16,14 +16,14 @@ import mock from sahara.plugins import provisioning as p -from sahara.plugins.vanilla.v2_7_1 import config_helper as v_helper -from sahara.tests.unit import base +from sahara_plugin_vanilla.plugins.vanilla.v2_7_1 import config_helper +from sahara_plugin_vanilla.tests.unit import base class TestConfigHelper(base.SaharaTestCase): - plugin_path = 'sahara.plugins.vanilla.v2_7_1.' - plugin_hadoop_path = 'sahara.plugins.vanilla.hadoop2.' + plugin_path = 'sahara_plugin_vanilla.plugins.vanilla.v2_7_1.' + plugin_hadoop_path = 'sahara_plugin_vanilla.plugins.vanilla.hadoop2.' def setUp(self): super(TestConfigHelper, self).setUp() @@ -45,29 +45,29 @@ class TestConfigHelper(base.SaharaTestCase): configs.extend(PLUGIN_GENERAL_CONFIGS) configs.extend(_get_spark_configs()) configs.extend(_get_zk_configs()) - init_configs = v_helper._init_all_configs() + init_configs = config_helper._init_all_configs() self.assertEqual(init_configs, configs) def test_get_spark_opt_default(self): opt_name = 'Executor extra classpath' _default_executor_classpath = ":".join( ['/opt/hadoop/share/hadoop/tools/lib/hadoop-openstack-2.7.1.jar']) - default = v_helper._get_spark_opt_default(opt_name) + default = config_helper._get_spark_opt_default(opt_name) self.assertEqual(default, _default_executor_classpath) def test_get_spark_configs(self): - spark_configs = v_helper._get_spark_configs() + spark_configs = config_helper._get_spark_configs() for i in spark_configs: self.assertIsInstance(i, p.Config) def test_get_plugin_configs(self): - self.assertEqual(v_helper.get_plugin_configs(), - v_helper.PLUGIN_CONFIGS) + self.assertEqual(config_helper.get_plugin_configs(), + config_helper.PLUGIN_CONFIGS) def test_get_xml_configs(self): - self.assertEqual(v_helper.get_xml_configs(), - v_helper.PLUGIN_XML_CONFIGS) + self.assertEqual(config_helper.get_xml_configs(), + config_helper.PLUGIN_XML_CONFIGS) def test_get_env_configs(self): - self.assertEqual(v_helper.get_env_configs(), - v_helper.ENV_CONFS) + self.assertEqual(config_helper.get_env_configs(), + config_helper.ENV_CONFS) diff --git a/sahara/tests/unit/plugins/vanilla/v2_7_1/test_edp_engine.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_7_1/test_edp_engine.py similarity index 85% rename from sahara/tests/unit/plugins/vanilla/v2_7_1/test_edp_engine.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_7_1/test_edp_engine.py index 40187ba..144a339 100644 --- a/sahara/tests/unit/plugins/vanilla/v2_7_1/test_edp_engine.py +++ b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_7_1/test_edp_engine.py @@ -15,14 +15,14 @@ import mock -from sahara.plugins.vanilla.v2_7_1 import edp_engine -from sahara.tests.unit import base as sahara_base -from sahara.utils import edp +from sahara.plugins import edp +from sahara_plugin_vanilla.plugins.vanilla.v2_7_1 import edp_engine +from sahara_plugin_vanilla.tests.unit import base as sahara_base class Vanilla2ConfigHintsTest(sahara_base.SaharaTestCase): @mock.patch( - 'sahara.plugins.vanilla.confighints_helper.' + 'sahara_plugin_vanilla.plugins.vanilla.confighints_helper.' 'get_possible_hive_config_from', return_value={}) def test_get_possible_job_config_hive( @@ -34,7 +34,8 @@ class Vanilla2ConfigHintsTest(sahara_base.SaharaTestCase): 'plugins/vanilla/v2_7_1/resources/hive-default.xml') self.assertEqual(expected_config, actual_config) - @mock.patch('sahara.plugins.vanilla.hadoop2.edp_engine.EdpOozieEngine') + @mock.patch('sahara_plugin_vanilla.plugins.vanilla.hadoop2.edp_engine.' + 'EdpOozieEngine') def test_get_possible_job_config_java(self, BaseVanillaEdpOozieEngine): expected_config = {'job_config': {}} BaseVanillaEdpOozieEngine.get_possible_job_config.return_value = ( @@ -46,7 +47,7 @@ class Vanilla2ConfigHintsTest(sahara_base.SaharaTestCase): self.assertEqual(expected_config, actual_config) @mock.patch( - 'sahara.plugins.vanilla.confighints_helper.' + 'sahara_plugin_vanilla.plugins.vanilla.confighints_helper.' 'get_possible_mapreduce_config_from', return_value={}) def test_get_possible_job_config_mapreduce( @@ -59,7 +60,7 @@ class Vanilla2ConfigHintsTest(sahara_base.SaharaTestCase): self.assertEqual(expected_config, actual_config) @mock.patch( - 'sahara.plugins.vanilla.confighints_helper.' + 'sahara_plugin_vanilla.plugins.vanilla.confighints_helper.' 'get_possible_mapreduce_config_from', return_value={}) def test_get_possible_job_config_mapreduce_streaming( @@ -72,7 +73,7 @@ class Vanilla2ConfigHintsTest(sahara_base.SaharaTestCase): self.assertEqual(expected_config, actual_config) @mock.patch( - 'sahara.plugins.vanilla.confighints_helper.' + 'sahara_plugin_vanilla.plugins.vanilla.confighints_helper.' 'get_possible_pig_config_from', return_value={}) def test_get_possible_job_config_pig( @@ -84,7 +85,8 @@ class Vanilla2ConfigHintsTest(sahara_base.SaharaTestCase): 'plugins/vanilla/v2_7_1/resources/mapred-default.xml') self.assertEqual(expected_config, actual_config) - @mock.patch('sahara.plugins.vanilla.hadoop2.edp_engine.EdpOozieEngine') + @mock.patch('sahara_plugin_vanilla.plugins.vanilla.hadoop2.edp_engine.' + 'EdpOozieEngine') def test_get_possible_job_config_shell(self, BaseVanillaEdpOozieEngine): expected_config = {'job_config': {}} BaseVanillaEdpOozieEngine.get_possible_job_config.return_value = ( diff --git a/sahara/tests/unit/plugins/vanilla/v2_7_1/test_versionhandler.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_7_1/test_versionhandler.py similarity index 90% rename from sahara/tests/unit/plugins/vanilla/v2_7_1/test_versionhandler.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_7_1/test_versionhandler.py index 763278e..119ffab 100644 --- a/sahara/tests/unit/plugins/vanilla/v2_7_1/test_versionhandler.py +++ b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_7_1/test_versionhandler.py @@ -17,13 +17,16 @@ import mock import six import testtools -from sahara.conductor import resource as r +from sahara.plugins import base as pb from sahara.plugins import exceptions as ex -from sahara.plugins.vanilla.v2_7_1.edp_engine import EdpOozieEngine -from sahara.plugins.vanilla.v2_7_1.edp_engine import EdpSparkEngine -from sahara.plugins.vanilla.v2_7_1 import versionhandler as v_h -from sahara.tests.unit import base -from sahara.tests.unit import testutils +from sahara.plugins import resource as r +from sahara.plugins import testutils +from sahara_plugin_vanilla.plugins.vanilla.v2_7_1.edp_engine import \ + EdpOozieEngine +from sahara_plugin_vanilla.plugins.vanilla.v2_7_1.edp_engine import \ + EdpSparkEngine +from sahara_plugin_vanilla.plugins.vanilla.v2_7_1 import versionhandler as v_h +from sahara_plugin_vanilla.tests.unit import base class TestConfig(object): @@ -34,13 +37,15 @@ class TestConfig(object): class VersionHandlerTest(base.SaharaTestCase): - plugin_path = 'sahara.plugins.vanilla.' - plugin_hadoop2_path = 'sahara.plugins.vanilla.hadoop2.' + plugin_path = 'sahara_plugin_vanilla.plugins.vanilla.' + plugin_hadoop2_path = 'sahara_plugin_vanilla.plugins.vanilla.hadoop2.' def setUp(self): super(VersionHandlerTest, self).setUp() self.cluster = mock.Mock() self.vh = v_h.VersionHandler() + self.override_config("plugins", ["vanilla"]) + pb.setup_plugins() def test_get_plugin_configs(self): self.vh.pctx['all_confs'] = 'haha' @@ -74,11 +79,11 @@ class VersionHandlerTest(base.SaharaTestCase): configure_cluster.assert_called_once_with(self.vh.pctx, self.cluster) @mock.patch(plugin_path + 'v2_7_1.versionhandler.run') - @mock.patch(plugin_path + 'v2_7_1.versionhandler.s_scripts') - @mock.patch('sahara.swift.swift_helper.install_ssl_certs') + @mock.patch(plugin_path + 'v2_7_1.versionhandler.starting_scripts') + @mock.patch('sahara.plugins.swift_helper.install_ssl_certs') @mock.patch(plugin_hadoop2_path + 'keypairs.provision_keypairs') @mock.patch('sahara.plugins.utils.get_instances') - @mock.patch('sahara.utils.cluster.get_instances') + @mock.patch('sahara.plugins.utils.cluster_get_instances') def test_start_cluster(self, c_get_instances, u_get_instances, provision_keypairs, install_ssl_certs, s_scripts, run): @@ -113,7 +118,7 @@ class VersionHandlerTest(base.SaharaTestCase): cluster, instances) - @mock.patch('sahara.utils.general.get_by_id') + @mock.patch('sahara.plugins.utils.general.get_by_id') @mock.patch(plugin_hadoop2_path + 'validation.validate_additional_ng_scaling') @mock.patch(plugin_hadoop2_path + @@ -141,7 +146,7 @@ class VersionHandlerTest(base.SaharaTestCase): with testtools.ExpectedException(ex.ClusterCannotBeScaled): self.vh.validate_scaling(cluster, existing, {}) - get_by_id.return_value = r.NodeGroupResource(ng5) + get_by_id.return_value = r.create_node_group_resource(ng5) with testtools.ExpectedException(ex.ClusterCannotBeScaled): self.vh.validate_scaling(cluster, {}, additional) @@ -158,8 +163,8 @@ class VersionHandlerTest(base.SaharaTestCase): self.cluster, instances) - @mock.patch("sahara.conductor.API.cluster_update") - @mock.patch("sahara.context.ctx") + @mock.patch("sahara.plugins.conductor.cluster_update") + @mock.patch("sahara.plugins.context.ctx") @mock.patch(plugin_path + 'utils.get_namenode') @mock.patch(plugin_path + 'utils.get_resourcemanager') @mock.patch(plugin_path + 'utils.get_historyserver') @@ -202,7 +207,7 @@ class VersionHandlerTest(base.SaharaTestCase): cluster_update.assert_called_once_with(ctx(), self.cluster, {'info': info}) - @mock.patch("sahara.service.edp.job_utils.get_plugin") + @mock.patch("sahara.plugins.edp.get_plugin") @mock.patch('sahara.plugins.utils.get_instance') @mock.patch('os.path.join') def test_get_edp_engine(self, join, get_instance, get_plugin): diff --git a/sahara/tests/unit/plugins/vanilla/v2_7_5/__init__.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_7_5/__init__.py similarity index 100% rename from sahara/tests/unit/plugins/vanilla/v2_7_5/__init__.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_7_5/__init__.py diff --git a/sahara/tests/unit/plugins/vanilla/v2_7_5/test_config_helper.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_7_5/test_config_helper.py similarity index 73% rename from sahara/tests/unit/plugins/vanilla/v2_7_5/test_config_helper.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_7_5/test_config_helper.py index 5a185ed..211b724 100644 --- a/sahara/tests/unit/plugins/vanilla/v2_7_5/test_config_helper.py +++ b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_7_5/test_config_helper.py @@ -16,14 +16,14 @@ import mock from sahara.plugins import provisioning as p -from sahara.plugins.vanilla.v2_7_5 import config_helper as v_helper -from sahara.tests.unit import base +from sahara_plugin_vanilla.plugins.vanilla.v2_7_5 import config_helper +from sahara_plugin_vanilla.tests.unit import base class TestConfigHelper(base.SaharaTestCase): - plugin_path = 'sahara.plugins.vanilla.v2_7_5.' - plugin_hadoop_path = 'sahara.plugins.vanilla.hadoop2.' + plugin_path = 'sahara_plugin_vanilla.plugins.vanilla.v2_7_5.' + plugin_hadoop_path = 'sahara_plugin_vanilla.plugins.vanilla.hadoop2.' def setUp(self): super(TestConfigHelper, self).setUp() @@ -45,29 +45,29 @@ class TestConfigHelper(base.SaharaTestCase): configs.extend(PLUGIN_GENERAL_CONFIGS) configs.extend(_get_spark_configs()) configs.extend(_get_zk_configs()) - init_configs = v_helper._init_all_configs() + init_configs = config_helper._init_all_configs() self.assertEqual(init_configs, configs) def test_get_spark_opt_default(self): opt_name = 'Executor extra classpath' _default_executor_classpath = ":".join( ['/opt/hadoop/share/hadoop/tools/lib/hadoop-openstack-2.7.5.jar']) - default = v_helper._get_spark_opt_default(opt_name) + default = config_helper._get_spark_opt_default(opt_name) self.assertEqual(default, _default_executor_classpath) def test_get_spark_configs(self): - spark_configs = v_helper._get_spark_configs() + spark_configs = config_helper._get_spark_configs() for i in spark_configs: self.assertIsInstance(i, p.Config) def test_get_plugin_configs(self): - self.assertEqual(v_helper.get_plugin_configs(), - v_helper.PLUGIN_CONFIGS) + self.assertEqual(config_helper.get_plugin_configs(), + config_helper.PLUGIN_CONFIGS) def test_get_xml_configs(self): - self.assertEqual(v_helper.get_xml_configs(), - v_helper.PLUGIN_XML_CONFIGS) + self.assertEqual(config_helper.get_xml_configs(), + config_helper.PLUGIN_XML_CONFIGS) def test_get_env_configs(self): - self.assertEqual(v_helper.get_env_configs(), - v_helper.ENV_CONFS) + self.assertEqual(config_helper.get_env_configs(), + config_helper.ENV_CONFS) diff --git a/sahara/tests/unit/plugins/vanilla/v2_7_5/test_edp_engine.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_7_5/test_edp_engine.py similarity index 85% rename from sahara/tests/unit/plugins/vanilla/v2_7_5/test_edp_engine.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_7_5/test_edp_engine.py index 417c7fb..5da1fac 100644 --- a/sahara/tests/unit/plugins/vanilla/v2_7_5/test_edp_engine.py +++ b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_7_5/test_edp_engine.py @@ -15,14 +15,14 @@ import mock -from sahara.plugins.vanilla.v2_7_5 import edp_engine -from sahara.tests.unit import base as sahara_base -from sahara.utils import edp +from sahara.plugins import edp +from sahara_plugin_vanilla.plugins.vanilla.v2_7_5 import edp_engine +from sahara_plugin_vanilla.tests.unit import base as sahara_base class Vanilla2ConfigHintsTest(sahara_base.SaharaTestCase): @mock.patch( - 'sahara.plugins.vanilla.confighints_helper.' + 'sahara_plugin_vanilla.plugins.vanilla.confighints_helper.' 'get_possible_hive_config_from', return_value={}) def test_get_possible_job_config_hive( @@ -34,7 +34,8 @@ class Vanilla2ConfigHintsTest(sahara_base.SaharaTestCase): 'plugins/vanilla/v2_7_5/resources/hive-default.xml') self.assertEqual(expected_config, actual_config) - @mock.patch('sahara.plugins.vanilla.hadoop2.edp_engine.EdpOozieEngine') + @mock.patch('sahara_plugin_vanilla.plugins.vanilla.hadoop2.edp_engine.' + 'EdpOozieEngine') def test_get_possible_job_config_java(self, BaseVanillaEdpOozieEngine): expected_config = {'job_config': {}} BaseVanillaEdpOozieEngine.get_possible_job_config.return_value = ( @@ -46,7 +47,7 @@ class Vanilla2ConfigHintsTest(sahara_base.SaharaTestCase): self.assertEqual(expected_config, actual_config) @mock.patch( - 'sahara.plugins.vanilla.confighints_helper.' + 'sahara_plugin_vanilla.plugins.vanilla.confighints_helper.' 'get_possible_mapreduce_config_from', return_value={}) def test_get_possible_job_config_mapreduce( @@ -59,7 +60,7 @@ class Vanilla2ConfigHintsTest(sahara_base.SaharaTestCase): self.assertEqual(expected_config, actual_config) @mock.patch( - 'sahara.plugins.vanilla.confighints_helper.' + 'sahara_plugin_vanilla.plugins.vanilla.confighints_helper.' 'get_possible_mapreduce_config_from', return_value={}) def test_get_possible_job_config_mapreduce_streaming( @@ -72,7 +73,7 @@ class Vanilla2ConfigHintsTest(sahara_base.SaharaTestCase): self.assertEqual(expected_config, actual_config) @mock.patch( - 'sahara.plugins.vanilla.confighints_helper.' + 'sahara_plugin_vanilla.plugins.vanilla.confighints_helper.' 'get_possible_pig_config_from', return_value={}) def test_get_possible_job_config_pig( @@ -84,7 +85,8 @@ class Vanilla2ConfigHintsTest(sahara_base.SaharaTestCase): 'plugins/vanilla/v2_7_5/resources/mapred-default.xml') self.assertEqual(expected_config, actual_config) - @mock.patch('sahara.plugins.vanilla.hadoop2.edp_engine.EdpOozieEngine') + @mock.patch('sahara_plugin_vanilla.plugins.vanilla.hadoop2.edp_engine.' + 'EdpOozieEngine') def test_get_possible_job_config_shell(self, BaseVanillaEdpOozieEngine): expected_config = {'job_config': {}} BaseVanillaEdpOozieEngine.get_possible_job_config.return_value = ( diff --git a/sahara/tests/unit/plugins/vanilla/v2_7_5/test_versionhandler.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_7_5/test_versionhandler.py similarity index 90% rename from sahara/tests/unit/plugins/vanilla/v2_7_5/test_versionhandler.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_7_5/test_versionhandler.py index 3c78c70..71ca403 100644 --- a/sahara/tests/unit/plugins/vanilla/v2_7_5/test_versionhandler.py +++ b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_7_5/test_versionhandler.py @@ -17,13 +17,16 @@ import mock import six import testtools -from sahara.conductor import resource as r +from sahara.plugins import base as pb from sahara.plugins import exceptions as ex -from sahara.plugins.vanilla.v2_7_5.edp_engine import EdpOozieEngine -from sahara.plugins.vanilla.v2_7_5.edp_engine import EdpSparkEngine -from sahara.plugins.vanilla.v2_7_5 import versionhandler as v_h -from sahara.tests.unit import base -from sahara.tests.unit import testutils +from sahara.plugins import resource as r +from sahara.plugins import testutils +from sahara_plugin_vanilla.plugins.vanilla.v2_7_5.edp_engine import \ + EdpOozieEngine +from sahara_plugin_vanilla.plugins.vanilla.v2_7_5.edp_engine import \ + EdpSparkEngine +from sahara_plugin_vanilla.plugins.vanilla.v2_7_5 import versionhandler as v_h +from sahara_plugin_vanilla.tests.unit import base class TestConfig(object): @@ -34,13 +37,15 @@ class TestConfig(object): class VersionHandlerTest(base.SaharaTestCase): - plugin_path = 'sahara.plugins.vanilla.' - plugin_hadoop2_path = 'sahara.plugins.vanilla.hadoop2.' + plugin_path = 'sahara_plugin_vanilla.plugins.vanilla.' + plugin_hadoop2_path = 'sahara_plugin_vanilla.plugins.vanilla.hadoop2.' def setUp(self): super(VersionHandlerTest, self).setUp() self.cluster = mock.Mock() self.vh = v_h.VersionHandler() + self.override_config("plugins", ["vanilla"]) + pb.setup_plugins() def test_get_plugin_configs(self): self.vh.pctx['all_confs'] = 'haha' @@ -74,11 +79,11 @@ class VersionHandlerTest(base.SaharaTestCase): configure_cluster.assert_called_once_with(self.vh.pctx, self.cluster) @mock.patch(plugin_path + 'v2_7_5.versionhandler.run') - @mock.patch(plugin_path + 'v2_7_5.versionhandler.s_scripts') - @mock.patch('sahara.swift.swift_helper.install_ssl_certs') + @mock.patch(plugin_path + 'v2_7_5.versionhandler.starting_scripts') + @mock.patch('sahara.plugins.swift_helper.install_ssl_certs') @mock.patch(plugin_hadoop2_path + 'keypairs.provision_keypairs') @mock.patch('sahara.plugins.utils.get_instances') - @mock.patch('sahara.utils.cluster.get_instances') + @mock.patch('sahara.plugins.utils.cluster_get_instances') def test_start_cluster(self, c_get_instances, u_get_instances, provision_keypairs, install_ssl_certs, s_scripts, run): @@ -113,7 +118,7 @@ class VersionHandlerTest(base.SaharaTestCase): cluster, instances) - @mock.patch('sahara.utils.general.get_by_id') + @mock.patch('sahara.plugins.utils.general.get_by_id') @mock.patch(plugin_hadoop2_path + 'validation.validate_additional_ng_scaling') @mock.patch(plugin_hadoop2_path + @@ -141,7 +146,7 @@ class VersionHandlerTest(base.SaharaTestCase): with testtools.ExpectedException(ex.ClusterCannotBeScaled): self.vh.validate_scaling(cluster, existing, {}) - get_by_id.return_value = r.NodeGroupResource(ng5) + get_by_id.return_value = r.create_node_group_resource(ng5) with testtools.ExpectedException(ex.ClusterCannotBeScaled): self.vh.validate_scaling(cluster, {}, additional) @@ -158,8 +163,8 @@ class VersionHandlerTest(base.SaharaTestCase): self.cluster, instances) - @mock.patch("sahara.conductor.API.cluster_update") - @mock.patch("sahara.context.ctx") + @mock.patch("sahara.plugins.conductor.cluster_update") + @mock.patch("sahara.plugins.context.ctx") @mock.patch(plugin_path + 'utils.get_namenode') @mock.patch(plugin_path + 'utils.get_resourcemanager') @mock.patch(plugin_path + 'utils.get_historyserver') @@ -202,7 +207,7 @@ class VersionHandlerTest(base.SaharaTestCase): cluster_update.assert_called_once_with(ctx(), self.cluster, {'info': info}) - @mock.patch("sahara.service.edp.job_utils.get_plugin") + @mock.patch("sahara.plugins.edp.get_plugin") @mock.patch('sahara.plugins.utils.get_instance') @mock.patch('os.path.join') def test_get_edp_engine(self, join, get_instance, get_plugin): diff --git a/sahara/tests/unit/plugins/vanilla/v2_8_2/__init__.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_8_2/__init__.py similarity index 100% rename from sahara/tests/unit/plugins/vanilla/v2_8_2/__init__.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_8_2/__init__.py diff --git a/sahara/tests/unit/plugins/vanilla/v2_8_2/test_config_helper.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_8_2/test_config_helper.py similarity index 73% rename from sahara/tests/unit/plugins/vanilla/v2_8_2/test_config_helper.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_8_2/test_config_helper.py index f0d67b2..0387e67 100644 --- a/sahara/tests/unit/plugins/vanilla/v2_8_2/test_config_helper.py +++ b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_8_2/test_config_helper.py @@ -16,14 +16,14 @@ import mock from sahara.plugins import provisioning as p -from sahara.plugins.vanilla.v2_8_2 import config_helper as v_helper -from sahara.tests.unit import base +from sahara_plugin_vanilla.plugins.vanilla.v2_8_2 import config_helper +from sahara_plugin_vanilla.tests.unit import base class TestConfigHelper(base.SaharaTestCase): - plugin_path = 'sahara.plugins.vanilla.v2_8_2.' - plugin_hadoop_path = 'sahara.plugins.vanilla.hadoop2.' + plugin_path = 'sahara_plugin_vanilla.plugins.vanilla.v2_8_2.' + plugin_hadoop_path = 'sahara_plugin_vanilla.plugins.vanilla.hadoop2.' def setUp(self): super(TestConfigHelper, self).setUp() @@ -45,29 +45,29 @@ class TestConfigHelper(base.SaharaTestCase): configs.extend(PLUGIN_GENERAL_CONFIGS) configs.extend(_get_spark_configs()) configs.extend(_get_zk_configs()) - init_configs = v_helper._init_all_configs() + init_configs = config_helper._init_all_configs() self.assertEqual(init_configs, configs) def test_get_spark_opt_default(self): opt_name = 'Executor extra classpath' _default_executor_classpath = ":".join( ['/opt/hadoop/share/hadoop/tools/lib/hadoop-openstack-2.8.2.jar']) - default = v_helper._get_spark_opt_default(opt_name) + default = config_helper._get_spark_opt_default(opt_name) self.assertEqual(default, _default_executor_classpath) def test_get_spark_configs(self): - spark_configs = v_helper._get_spark_configs() + spark_configs = config_helper._get_spark_configs() for i in spark_configs: self.assertIsInstance(i, p.Config) def test_get_plugin_configs(self): - self.assertEqual(v_helper.get_plugin_configs(), - v_helper.PLUGIN_CONFIGS) + self.assertEqual(config_helper.get_plugin_configs(), + config_helper.PLUGIN_CONFIGS) def test_get_xml_configs(self): - self.assertEqual(v_helper.get_xml_configs(), - v_helper.PLUGIN_XML_CONFIGS) + self.assertEqual(config_helper.get_xml_configs(), + config_helper.PLUGIN_XML_CONFIGS) def test_get_env_configs(self): - self.assertEqual(v_helper.get_env_configs(), - v_helper.ENV_CONFS) + self.assertEqual(config_helper.get_env_configs(), + config_helper.ENV_CONFS) diff --git a/sahara/tests/unit/plugins/vanilla/v2_8_2/test_edp_engine.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_8_2/test_edp_engine.py similarity index 86% rename from sahara/tests/unit/plugins/vanilla/v2_8_2/test_edp_engine.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_8_2/test_edp_engine.py index d77cb81..427b40f 100644 --- a/sahara/tests/unit/plugins/vanilla/v2_8_2/test_edp_engine.py +++ b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_8_2/test_edp_engine.py @@ -15,14 +15,14 @@ import mock -from sahara.plugins.vanilla.v2_8_2 import edp_engine +from sahara.plugins import edp from sahara.tests.unit import base as sahara_base -from sahara.utils import edp +from sahara_plugin_vanilla.plugins.vanilla.v2_8_2 import edp_engine class Vanilla2ConfigHintsTest(sahara_base.SaharaTestCase): @mock.patch( - 'sahara.plugins.vanilla.confighints_helper.' + 'sahara_plugin_vanilla.plugins.vanilla.confighints_helper.' 'get_possible_hive_config_from', return_value={}) def test_get_possible_job_config_hive( @@ -34,7 +34,8 @@ class Vanilla2ConfigHintsTest(sahara_base.SaharaTestCase): 'plugins/vanilla/v2_8_2/resources/hive-default.xml') self.assertEqual(expected_config, actual_config) - @mock.patch('sahara.plugins.vanilla.hadoop2.edp_engine.EdpOozieEngine') + @mock.patch('sahara_plugin_vanilla.plugins.vanilla.hadoop2.edp_engine.' + 'EdpOozieEngine') def test_get_possible_job_config_java(self, BaseVanillaEdpOozieEngine): expected_config = {'job_config': {}} BaseVanillaEdpOozieEngine.get_possible_job_config.return_value = ( @@ -46,7 +47,7 @@ class Vanilla2ConfigHintsTest(sahara_base.SaharaTestCase): self.assertEqual(expected_config, actual_config) @mock.patch( - 'sahara.plugins.vanilla.confighints_helper.' + 'sahara_plugin_vanilla.plugins.vanilla.confighints_helper.' 'get_possible_mapreduce_config_from', return_value={}) def test_get_possible_job_config_mapreduce( @@ -59,7 +60,7 @@ class Vanilla2ConfigHintsTest(sahara_base.SaharaTestCase): self.assertEqual(expected_config, actual_config) @mock.patch( - 'sahara.plugins.vanilla.confighints_helper.' + 'sahara_plugin_vanilla.plugins.vanilla.confighints_helper.' 'get_possible_mapreduce_config_from', return_value={}) def test_get_possible_job_config_mapreduce_streaming( @@ -72,7 +73,7 @@ class Vanilla2ConfigHintsTest(sahara_base.SaharaTestCase): self.assertEqual(expected_config, actual_config) @mock.patch( - 'sahara.plugins.vanilla.confighints_helper.' + 'sahara_plugin_vanilla.plugins.vanilla.confighints_helper.' 'get_possible_pig_config_from', return_value={}) def test_get_possible_job_config_pig( @@ -84,7 +85,8 @@ class Vanilla2ConfigHintsTest(sahara_base.SaharaTestCase): 'plugins/vanilla/v2_8_2/resources/mapred-default.xml') self.assertEqual(expected_config, actual_config) - @mock.patch('sahara.plugins.vanilla.hadoop2.edp_engine.EdpOozieEngine') + @mock.patch('sahara_plugin_vanilla.plugins.vanilla.hadoop2.edp_engine.' + 'EdpOozieEngine') def test_get_possible_job_config_shell(self, BaseVanillaEdpOozieEngine): expected_config = {'job_config': {}} BaseVanillaEdpOozieEngine.get_possible_job_config.return_value = ( diff --git a/sahara/tests/unit/plugins/vanilla/v2_8_2/test_versionhandler.py b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_8_2/test_versionhandler.py similarity index 91% rename from sahara/tests/unit/plugins/vanilla/v2_8_2/test_versionhandler.py rename to sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_8_2/test_versionhandler.py index 55f3199..98c15d7 100644 --- a/sahara/tests/unit/plugins/vanilla/v2_8_2/test_versionhandler.py +++ b/sahara_plugin_vanilla/tests/unit/plugins/vanilla/v2_8_2/test_versionhandler.py @@ -17,13 +17,15 @@ import mock import six import testtools -from sahara.conductor import resource as r from sahara.plugins import exceptions as ex -from sahara.plugins.vanilla.v2_8_2.edp_engine import EdpOozieEngine -from sahara.plugins.vanilla.v2_8_2.edp_engine import EdpSparkEngine -from sahara.plugins.vanilla.v2_8_2 import versionhandler as v_h -from sahara.tests.unit import base -from sahara.tests.unit import testutils +from sahara.plugins import resource as r +from sahara.plugins import testutils +from sahara_plugin_vanilla.plugins.vanilla.v2_8_2.edp_engine import \ + EdpOozieEngine +from sahara_plugin_vanilla.plugins.vanilla.v2_8_2.edp_engine import \ + EdpSparkEngine +from sahara_plugin_vanilla.plugins.vanilla.v2_8_2 import versionhandler as v_h +from sahara_plugin_vanilla.tests.unit import base class TestConfig(object): @@ -34,8 +36,8 @@ class TestConfig(object): class VersionHandlerTest(base.SaharaTestCase): - plugin_path = 'sahara.plugins.vanilla.' - plugin_hadoop2_path = 'sahara.plugins.vanilla.hadoop2.' + plugin_path = 'sahara_plugin_vanilla.plugins.vanilla.' + plugin_hadoop2_path = 'sahara_plugin_vanilla.plugins.vanilla.hadoop2.' def setUp(self): super(VersionHandlerTest, self).setUp() @@ -74,11 +76,11 @@ class VersionHandlerTest(base.SaharaTestCase): configure_cluster.assert_called_once_with(self.vh.pctx, self.cluster) @mock.patch(plugin_path + 'v2_8_2.versionhandler.run') - @mock.patch(plugin_path + 'v2_8_2.versionhandler.s_scripts') - @mock.patch('sahara.swift.swift_helper.install_ssl_certs') + @mock.patch(plugin_path + 'v2_8_2.versionhandler.starting_scripts') + @mock.patch('sahara.plugins.swift_helper.install_ssl_certs') @mock.patch(plugin_hadoop2_path + 'keypairs.provision_keypairs') @mock.patch('sahara.plugins.utils.get_instances') - @mock.patch('sahara.utils.cluster.get_instances') + @mock.patch('sahara.plugins.utils.cluster_get_instances') def test_start_cluster(self, c_get_instances, u_get_instances, provision_keypairs, install_ssl_certs, s_scripts, run): @@ -113,7 +115,7 @@ class VersionHandlerTest(base.SaharaTestCase): cluster, instances) - @mock.patch('sahara.utils.general.get_by_id') + @mock.patch('sahara.plugins.utils.general.get_by_id') @mock.patch(plugin_hadoop2_path + 'validation.validate_additional_ng_scaling') @mock.patch(plugin_hadoop2_path + @@ -141,7 +143,7 @@ class VersionHandlerTest(base.SaharaTestCase): with testtools.ExpectedException(ex.ClusterCannotBeScaled): self.vh.validate_scaling(cluster, existing, {}) - get_by_id.return_value = r.NodeGroupResource(ng5) + get_by_id.return_value = r.create_node_group_resource(ng5) with testtools.ExpectedException(ex.ClusterCannotBeScaled): self.vh.validate_scaling(cluster, {}, additional) @@ -158,8 +160,8 @@ class VersionHandlerTest(base.SaharaTestCase): self.cluster, instances) - @mock.patch("sahara.conductor.API.cluster_update") - @mock.patch("sahara.context.ctx") + @mock.patch("sahara.plugins.conductor.cluster_update") + @mock.patch("sahara.plugins.context.ctx") @mock.patch(plugin_path + 'utils.get_namenode') @mock.patch(plugin_path + 'utils.get_resourcemanager') @mock.patch(plugin_path + 'utils.get_historyserver') @@ -202,7 +204,7 @@ class VersionHandlerTest(base.SaharaTestCase): cluster_update.assert_called_once_with(ctx(), self.cluster, {'info': info}) - @mock.patch("sahara.service.edp.job_utils.get_plugin") + @mock.patch("sahara.plugins.edp.get_plugin") @mock.patch('sahara.plugins.utils.get_instance') @mock.patch('os.path.join') def test_get_edp_engine(self, join, get_instance, get_plugin): diff --git a/sahara/utils/__init__.py b/sahara_plugin_vanilla/utils/__init__.py similarity index 100% rename from sahara/utils/__init__.py rename to sahara_plugin_vanilla/utils/__init__.py diff --git a/sahara/utils/patches.py b/sahara_plugin_vanilla/utils/patches.py similarity index 100% rename from sahara/utils/patches.py rename to sahara_plugin_vanilla/utils/patches.py diff --git a/setup.cfg b/setup.cfg index fc4cc89..a369266 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] -name = sahara -summary = Sahara project +name = sahara_plugin_vanilla +summary = Vanilla Plugin for Sahara Project description-file = README.rst license = Apache Software License classifiers = @@ -23,80 +23,17 @@ setup-hooks = pbr.hooks.setup_hook [files] packages = - sahara - -data_files = - etc/sahara = - etc/sahara/api-paste.ini - etc/sahara/rootwrap.conf - etc/sahara/rootwrap.d = etc/sahara/rootwrap.d/* + sahara_plugin_vanilla [entry_points] -console_scripts = - sahara-all = sahara.cli.sahara_all:main - sahara-api = sahara.cli.sahara_api:main - sahara-engine = sahara.cli.sahara_engine:main - sahara-db-manage = sahara.db.migration.cli:main - sahara-rootwrap = oslo_rootwrap.cmd:main - _sahara-subprocess = sahara.cli.sahara_subprocess:main - sahara-templates = sahara.db.templates.cli:main - sahara-image-pack = sahara.cli.image_pack.cli:main - sahara-status = sahara.cli.sahara_status:main - -wsgi_scripts = - sahara-wsgi-api = sahara.cli.sahara_api:setup_api - sahara.cluster.plugins = - vanilla = sahara.plugins.vanilla.plugin:VanillaProvider - ambari = sahara.plugins.ambari.plugin:AmbariPluginProvider - mapr = sahara.plugins.mapr.plugin:MapRPlugin - cdh = sahara.plugins.cdh.plugin:CDHPluginProvider - fake = sahara.plugins.fake.plugin:FakePluginProvider - spark = sahara.plugins.spark.plugin:SparkProvider - storm = sahara.plugins.storm.plugin:StormProvider + vanilla = sahara_plugin_vanilla.plugins.vanilla.plugin:VanillaProvider -sahara.data_source.types = - hdfs = sahara.service.edp.data_sources.hdfs.implementation:HDFSType - manila = sahara.service.edp.data_sources.manila.implementation:ManilaType - maprfs = sahara.service.edp.data_sources.maprfs.implementation:MapRFSType - swift = sahara.service.edp.data_sources.swift.implementation:SwiftType - s3 = sahara.service.edp.data_sources.s3.implementation:S3Type +[build_sphinx] +all_files = 1 +build-dir = doc/build +source-dir = doc/source +warning-is-error = 1 -sahara.job_binary.types = - internal-db = sahara.service.edp.job_binaries.internal_db.implementation:InternalDBType - manila = sahara.service.edp.job_binaries.manila.implementation:ManilaType - swift = sahara.service.edp.job_binaries.swift.implementation:SwiftType - s3 = sahara.service.edp.job_binaries.s3.implementation:S3Type - -sahara.infrastructure.engine = - heat = sahara.service.heat.heat_engine:HeatEngine - -sahara.remote = - ssh = sahara.utils.ssh_remote:SshRemoteDriver - -sahara.run.mode = - all-in-one = sahara.service.ops:LocalOps - distributed = sahara.service.ops:RemoteOps - -oslo.config.opts = - sahara.config = sahara.config:list_opts - -oslo.config.opts.defaults = - sahara.config = sahara.common.config:set_cors_middleware_defaults - -oslo.policy.policies = - sahara = sahara.common.policies:list_rules - -[extract_messages] -keywords = _ gettext ngettext l_ lazy_gettext -mapping_file = babel.cfg -output_file = sahara/locale/sahara.pot - -[compile_catalog] -directory = sahara/locale -domain = sahara - -[update_catalog] -domain = sahara -output_dir = sahara/locale -input_file = sahara/locale/sahara.pot +[wheel] +universal = 1 diff --git a/tox.ini b/tox.ini index 1ca7fa3..6d2f34b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] -envlist = py35,py27,pep8,genpolicy -minversion = 1.6 +envlist = py36,py35,py27,pep8 +minversion = 2.0 skipsdist = True [testenv] @@ -8,7 +8,7 @@ usedevelop = True install_command = pip install {opts} {packages} setenv = VIRTUAL_ENV={envdir} - DISCOVER_DIRECTORY=sahara/tests/unit + DISCOVER_DIRECTORY=sahara_plugin_vanilla/tests/unit deps = -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} -r{toxinidir}/requirements.txt @@ -16,19 +16,13 @@ deps = commands = stestr run {posargs} passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY -[testenv:cover] -basepython = python3 -setenv = - PACKAGE_NAME=sahara -commands = {toxinidir}/tools/cover.sh {posargs} - [testenv:debug-py27] basepython = python2.7 -commands = oslo_debug_helper -t sahara/tests/unit {posargs} +commands = oslo_debug_helper -t sahara_plugin_vanilla/tests/unit {posargs} [testenv:debug-py35] basepython = python3.5 -commands = oslo_debug_helper -t sahara/tests/unit {posargs} +commands = oslo_debug_helper -t sahara_plugin_vanilla/tests/unit {posargs} [testenv:pep8] basepython = python3 @@ -41,23 +35,12 @@ commands = flake8 {posargs} doc8 doc/source # Run bashate checks - bash -c "find sahara -iname '*.sh' -print0 | xargs -0 bashate -v" - bash -c "find devstack -not -name \*.template -and -not -name README.rst -and -not -name \*.json -type f -print0 | xargs -0 bashate -v" - # Run security linter - bandit -c bandit.yaml -r sahara -n5 -p sahara_default -x tests - -[testenv:genpolicy] -basepython = python3 -commands = oslopolicy-sample-generator --config-file tools/config/sahara-policy-generator.conf + bash -c "find sahara_plugin_vanilla -iname '*.sh' -print0 | xargs -0 bashate -v" [testenv:venv] basepython = python3 commands = {posargs} -[testenv:images] -sitepackages = True -commands = {posargs} - [testenv:docs] basepython = python3 deps = @@ -65,35 +48,9 @@ deps = -r{toxinidir}/doc/requirements.txt commands = rm -rf doc/html doc/build - rm -rf api-ref/build api-ref/html - rm -rf doc/source/apidoc doc/source/api sphinx-build -W -b html doc/source doc/build/html - sphinx-build -W -b html -d api-ref/build/doctrees api-ref/source api-ref/build/html - whereto doc/source/_extra/.htaccess doc/test/redirect-tests.txt -whitelist_externals = rm - -[testenv:api-ref] -basepython = python3 -deps = - -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} - -r{toxinidir}/doc/requirements.txt -install_command = pip install -U --force-reinstall {opts} {packages} -commands = - rm -rf api-ref/build api-ref/html - sphinx-build -W -b html -d api-ref/build/doctrees api-ref/source api-ref/build/html -whitelist_externals = rm - - -[testenv:pylint] -basepython = python3 -setenv = VIRTUAL_ENV={envdir} -commands = bash tools/lintstack.sh - -[testenv:genconfig] -basepython = python3 -commands = - oslo-config-generator --config-file tools/config/config-generator.sahara.conf \ - --output-file etc/sahara/sahara.conf.sample +whitelist_externals = + rm [testenv:releasenotes] basepython = python3 @@ -107,16 +64,11 @@ whitelist_externals = rm [testenv:debug] basepython = python3 -# It runs tests from the specified dir (default is sahara/tests) +# It runs tests from the specified dir (default is sahara_plugin_vanilla/tests) # in interactive mode, so, you could use pbr for tests debug. -# Example usage: tox -e debug -- -t sahara/tests/unit some.test.path +# Example usage: tox -e debug -- -t sahara_plugin_vanilla/tests/unit some.test.path # https://docs.openstack.org/oslotest/latest/features.html#debugging-with-oslo-debug-helper -commands = oslo_debug_helper -t sahara/tests/unit {posargs} - -[testenv:bandit] -basepython = python3 -deps = -r{toxinidir}/test-requirements-bandit.txt -commands = bandit -c bandit.yaml -r sahara -n5 -p sahara_default -x tests +commands = oslo_debug_helper -t sahara_plugin_vanilla/tests/unit {posargs} [flake8] show-source = true @@ -128,19 +80,12 @@ exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools # [H204] Use assert(Not)Equal to check for equality # [H205] Use assert(Greater|Less)(Equal) for comparison enable-extensions=H904,H106,H203,H204,H205 - -[hacking] -import_exceptions = sahara.i18n -local-check-factory = sahara.utils.hacking.checks.factory - -[testenv:bindep] -basepython = python3 -# Do not install any requirements. We want this to be fast and work even if -# system dependencies are missing, since it's used to tell you what system -# dependencies are missing! This also means that bindep must be installed -# separately, outside of the requirements files. -deps = bindep -commands = bindep test +# [E123] Closing bracket does not match indentation of opening bracket's line +# [E226] Missing whitespace around arithmetic operator +# [E402] Module level import not at top of file +# [E731] Do not assign a lambda expression, use a def +# [W503] Line break occurred before a binary operator +ignore=E123,E226,E402,E731,W503 [testenv:lower-constraints] basepython = python3 @@ -148,3 +93,4 @@ deps = -c{toxinidir}/lower-constraints.txt -r{toxinidir}/test-requirements.txt -r{toxinidir}/requirements.txt +