diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 8b60ef07..00000000 --- a/.gitignore +++ /dev/null @@ -1,30 +0,0 @@ -*.py[co] -*.egg -*.egg-info -dist -build -eggs -parts -var -sdist -develop-eggs -.installed.cfg -pip-log.txt -.tox -*.mo -.mr.developer.cfg -.DS_Store -Thumbs.db -.venv -.idea -out -target -*.iml -*.ipr -*.iws -*.db -.coverage -ChangeLog -AUTHORS -*.qcow2 -diskimage-create.* diff --git a/.mailmap b/.mailmap deleted file mode 100644 index 978490e4..00000000 --- a/.mailmap +++ /dev/null @@ -1,5 +0,0 @@ -# Format is: -# -# -Ivan Berezovskiy -Sergey Lukjanov diff --git a/.zuul.yaml b/.zuul.yaml deleted file mode 100644 index 331f8dd1..00000000 --- a/.zuul.yaml +++ /dev/null @@ -1,73 +0,0 @@ -- project: - templates: - - check-requirements - check: - jobs: - - openstack-tox-pep8 - - sahara-image-elements-buildimages-ambari: - voting: false - - sahara-image-elements-buildimages-cloudera: - voting: false - - sahara-image-elements-buildimages-mapr: - voting: false - - sahara-image-elements-buildimages-plain: - voting: false - - sahara-image-elements-buildimages-spark: - voting: false - - sahara-image-elements-buildimages-storm: - voting: false - - sahara-image-elements-buildimages-vanilla: - voting: false - gate: - jobs: - - openstack-tox-pep8 - post: - jobs: - - publish-openstack-python-branch-tarball - -- job: - name: sahara-image-elements-buildimages-basejob - timeout: 5400 - run: playbooks/buildimages/run.yaml - -- job: - name: sahara-image-elements-buildimages-ambari - parent: sahara-image-elements-buildimages-basejob - vars: - sahara_plugin: ambari - -- job: - name: sahara-image-elements-buildimages-cloudera - parent: sahara-image-elements-buildimages-basejob - vars: - sahara_plugin: cloudera - -- job: - name: sahara-image-elements-buildimages-mapr - parent: sahara-image-elements-buildimages-basejob - vars: - sahara_plugin: mapr - -- job: - name: sahara-image-elements-buildimages-plain - parent: sahara-image-elements-buildimages-basejob - vars: - sahara_plugin: plain - -- job: - name: sahara-image-elements-buildimages-spark - parent: sahara-image-elements-buildimages-basejob - vars: - sahara_plugin: spark - -- job: - name: sahara-image-elements-buildimages-storm - parent: sahara-image-elements-buildimages-basejob - vars: - sahara_plugin: storm - -- job: - name: sahara-image-elements-buildimages-vanilla - parent: sahara-image-elements-buildimages-basejob - vars: - sahara_plugin: vanilla diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst deleted file mode 100644 index 90c4b029..00000000 --- a/CONTRIBUTING.rst +++ /dev/null @@ -1,19 +0,0 @@ -The source repository for this project can be found at: - - https://opendev.org/openstack/sahara-image-elements - -Pull requests submitted through GitHub are not monitored. - -To start contributing to OpenStack, follow the steps in the contribution guide -to set up and use Gerrit: - - https://docs.openstack.org/contributors/code-and-documentation/quick-start.html - -Bugs should be filed on Storyboard: - - https://storyboard.openstack.org/#!/project/openstack/sahara-image-elements - -For more specific information about contributing to this repository, see the -sahara contributor guide: - - https://docs.openstack.org/sahara/latest/contributor/contributing.html diff --git a/HACKING.rst b/HACKING.rst deleted file mode 100644 index 7a93779f..00000000 --- a/HACKING.rst +++ /dev/null @@ -1,12 +0,0 @@ -Sahara Style Commandments -========================== - -- Step 1: Read the OpenStack Style Commandments - https://docs.openstack.org/hacking/latest/ -- Step 2: Read on - -Sahara Specific Commandments ------------------------------ - -None so far - diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 67db8588..00000000 --- a/LICENSE +++ /dev/null @@ -1,175 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index a664f166..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,2 +0,0 @@ -include README.md -graft elements diff --git a/README.rst b/README.rst index 1c8caaa9..4ee2c5f1 100644 --- a/README.rst +++ b/README.rst @@ -1,21 +1,10 @@ -======================== -Team and repository tags -======================== +This project is no longer maintained. -.. image:: https://governance.openstack.org/tc/badges/sahara-image-elements.svg - :target: https://governance.openstack.org/tc/reference/tags/index.html +The contents of this repository are still available in the Git +source code management system. To see the contents of this +repository before it reached its end of life, please check out the +previous commit with "git checkout HEAD^1". -.. Change things from this point on - -Sahara image elements project -============================== - -This repo is a place for Sahara-related for diskimage-builder elements. - -Script for creating Fedora and Ubuntu cloud images with our elements and default parameters. You should only need to run this command: - -.. sourcecode:: bash - - tox -e venv -- sahara-image-create - -Note: More information about script `diskimage-create `_ +For any further questions, please email +openstack-discuss@lists.openstack.org or join #openstack-dev on +OFTC. diff --git a/bin/sahara-image-create b/bin/sahara-image-create deleted file mode 100755 index 0ba1f6e6..00000000 --- a/bin/sahara-image-create +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -eu -set -o pipefail - -SCRIPT_HOME=$(dirname $(readlink -f $0)) -if [ -d $SCRIPT_HOME/../share/sahara-elements ]; then - _PREFIX=$SCRIPT_HOME/../share/sahara-elements -elif [ -d $SCRIPT_HOME/../../../elements ]; then - _PREFIX=$SCRIPT_HOME/../../.. -else - _PREFIX=$SCRIPT_HOME/.. -fi -export ELEMENTS_PATH=$_PREFIX/elements - -diskimage-create.sh $@ diff --git a/diskimage-create/README.rst b/diskimage-create/README.rst deleted file mode 100644 index 33c2659d..00000000 --- a/diskimage-create/README.rst +++ /dev/null @@ -1,137 +0,0 @@ -Diskimage-builder script for creation cloud images -================================================== - -This script builds Ubuntu, Fedora, CentOS cloud images for use in Sahara. -By default the all plugin are targeted, all images will be built. The '-p' -option can be used to select plugin (vanilla, spark, cloudera, storm, mapr, -ambari, or plain). The '-i' option can be used to select image type (ubuntu, -fedora, or centos7). The '-v' option can be used to select hadoop -version. - -For users: - -1. Use your environment (export / setenv) to alter the scripts behavior. -Environment variables the script accepts are 'DIB_HADOOP_VERSION_2_7_1', -'JAVA_DOWNLOAD_URL', 'JAVA_TARGET_LOCATION', 'OOZIE_DOWNLOAD_URL', -'HIVE_VERSION', -'[ubuntu|fedora|centos7]_vanilla_hadoop_2_7_1_image_name', -'ubuntu_spark_image_name', 'ubuntu_storm_image_name', -'ambari_[ubuntu|centos7]_image_name', -'cloudera_[5_7|5_9|5_11]_[ubuntu|centos7]_image_name', -'mapr_[ubuntu|centos7]_image_name', -'plain_[ubuntu|fedora|centos7]_image_name'. - -2. For creating all images just clone this repository and run script. - -.. sourcecode:: bash - - tox -e venv -- sahara-image-create - -3. If you want to use your local mirrors, you should specify http urls for -Fedora, CentOS and Ubuntu mirrors using parameters 'FEDORA_MIRROR', -'CENTOS_MIRROR' and 'UBUNTU_MIRROR' like this: - -.. sourcecode:: bash - - USE_MIRRORS=true FEDORA_MIRROR="url_for_fedora_mirror" \ - CENTOS_MIRROR="url_for_centos_mirror" \ - UBUNTU_MIRROR="url_for_ubuntu_mirror" tox -e venv -- sahara-image-create - -If you want to use your local image, you can specify path of image file using -parameters 'DIB_LOCAL_IMAGE', which defined in project `[diskimage-builder] -(https://github.com/openstack/diskimage-builder)`, like this: - -.. sourcecode:: bash - - DIB_LOCAL_IMAGE="path_of_image" tox -e venv -- sahara-image-create - -NOTE: Do not create all images for all plugins with the same mirrors. -Different plugins use different OS version. - -4. To select which plugin to target use the '-p' commandline option like this: - -.. sourcecode:: bash - - tox -e venv -- sahara-image-create -p [vanilla|spark|cloudera|storm|mapr|ambari|plain] - -5. To select which hadoop version to target use the '-v' commandline option -like this: - -.. sourcecode:: bash - - tox -e venv -- sahara-image-create -v [2.7.1|5.5|5.7|5.9] - -Also, if you are planning to select which ambari version to target use the -'-v' commandline option like this: - -.. sourcecode:: bash - - tox -e venv -- sahara-image-create -v [2.2.0.0|2.2.1.0|2.4.2.0] - -6. To select which operating system to target use the '-i' commandline option -like this: - -.. sourcecode:: bash - - tox -e venv -- sahara-image-create -i [ubuntu|fedora|centos7] - -7. To select which Spark version to target use the '-s' commandline option -like this: - -.. sourcecode:: bash - - tox -e venv -- sahara-image-create -p spark -s [1.3.1|1.6.0|2.1.0|2.2.0] # spark standalone - tox -e venv -- sahara-image-create -p vanilla -v 2.7.1 -s [1.6.0|2.1.0|2.2.0] # spark on vanilla - -8. To select which MapR version to target use the '-r' commandline option like -this: - -.. sourcecode:: bash - - tox -e venv -- sahara-image-create -p mapr -r [5.1.0|5.2.0] - -9. If the host system is missing packages required for sahara-image-create, -the '-u' commandline option will instruct the script to install them without -prompt. - -NOTE for 4, 5, 6: - -For Vanilla you can create ubuntu, fedora and centos7 cloud image with 2.x.x -versions. Use environment variables 'DIB_HADOOP_VERSION_2' to change defaults. -For Spark you can create only ubuntu images, so you shouldn't specify an image -type. The default Spark and HDFS versions included in the build are tested and -known working together with the Sahara Spark plugin, other combinations should -be used only for evaluation or testing purposes. You can select a different -Spark version with commandline option '-s' and Hadoop HDFS version with '-v', -but only Cludera CDH versions are available for now. For Cloudera you can -create ubuntu and centos7 images with preinstalled cloudera hadoop. You -shouldn't specify hadoop version. You can create centos7, ubuntu, fedora images -without hadoop ('plain' image) - -NOTE for CentOS images (for vanilla, ambari, and cloudera plugins): - -Resizing disk space during firstboot on that images fails with errors -(https://storyboard.openstack.org/#!/story/1304100). So, you will get an instance -that will have a small available disk space. To solve this problem we build -images with 10G available disk space as default. If you need in more available -disk space you should export parameter DIB_IMAGE_SIZE: - -.. sourcecode:: bash - - DIB_IMAGE_SIZE=40 tox -e venv -- sahara-image-create -i centos7 - -For all other images parameter DIB_IMAGE_SIZE will be unset. - -`DIB_CLOUD_INIT_DATASOURCES` contains a growing collection of data source -modules and most are enabled by default. This causes cloud-init to query each -data source on first boot. This can cause delays or even boot problems -depending on your environment. You must define `DIB_CLOUD_INIT_DATASOURCES` as -a comma-separated list of valid data sources to limit the data sources that -will be queried for metadata on first boot. - - -For developers: - -If you want to add your element to this repository, you should edit this -script in your commit (you should export variables for your element and add -name of element to variables 'element_sequence'). diff --git a/diskimage-create/diskimage-create.sh b/diskimage-create/diskimage-create.sh deleted file mode 100755 index beb09f06..00000000 --- a/diskimage-create/diskimage-create.sh +++ /dev/null @@ -1,864 +0,0 @@ -#!/bin/bash - -# dummy change -set -e - -export IMAGE_SIZE=$DIB_IMAGE_SIZE -# This will unset parameter DIB_IMAGE_SIZE for Ubuntu and Fedora vanilla images -unset DIB_IMAGE_SIZE - -# DEBUG_MODE is set by the -d flag, debug is enabled if the value is "true" -DEBUG_MODE="false" - -# The default version for a MapR plugin -DIB_DEFAULT_MAPR_VERSION="5.2.0" - -# The default version for Spark plugin -DIB_DEFAULT_SPARK_VERSION="2.3.0" - -# The default version for Storm plugin -DIB_DEFAULT_STORM_VERSION="1.2.1" - -# Bare metal image generation is enabled with the -b flag, it is off by default -SIE_BAREMETAL="false" - -# Default list of datasource modules for ubuntu. Workaround for bug #1375645 -export CLOUD_INIT_DATASOURCES=${DIB_CLOUD_INIT_DATASOURCES:-"NoCloud, ConfigDrive, OVF, MAAS, Ec2"} - -# Tracing control -TRACING= - -usage() { - echo - echo "Usage: $(basename $0)" - echo " [-p vanilla|spark|cloudera|storm|mapr|ambari|plain]" - echo " [-i ubuntu|fedora|centos7]" - echo " [-v 2.7.1|2.7.5|2.8.2|3.0.1|5.5|5.7|5.9|5.11|2.2.0.0|2.2.1.0|2.4.2.0]" - echo " [-r 5.1.0|5.2.0]" - echo " [-s 1.6.0|2.1.0|2.2.0|2.3.0]" - echo " [-t 1.0.1|1.1.0|1.1.1|1.2.0|1.2.1]" - echo " [-f qcow2|raw]" - echo " [-d]" - echo " [-u]" - echo " [-j openjdk|oracle-java]" - echo " [-x]" - echo " [-h]" - echo " '-p' is plugin version (default: all plugins)" - echo " '-i' is operating system of the base image (default: all non-deprecated" - echo " by plugin)." - echo " '-v' is hadoop version (default: all supported by plugin)" - echo " '-r' is MapR Version (default: ${DIB_DEFAULT_MAPR_VERSION})" - echo " '-s' is Spark version (default: ${DIB_DEFAULT_SPARK_VERSION})" - echo " '-f' is the image format (default: qcow2)" - echo " '-d' enable debug mode, root account will have password 'hadoop'" - echo " '-u' install missing packages necessary for building" - echo " '-j' is java distribution (default: openjdk)" - echo " '-x' turns on tracing" - echo " '-b' generate a bare metal image" - echo " '-h' display this message" - echo - echo "You shouldn't specify image type for spark plugin" - echo "You shouldn't specify hadoop version for plain images" - echo "Debug mode should only be enabled for local debugging purposes, not for production systems" - echo "By default all images for all plugins will be created" - echo -} - -while getopts "p:i:v:f:dur:s:t:j:xhb" opt; do - case $opt in - p) - PLUGIN=$OPTARG - ;; - i) - BASE_IMAGE_OS=$OPTARG - ;; - v) - HADOOP_VERSION=$OPTARG - ;; - d) - DEBUG_MODE="true" - ;; - r) - DIB_MAPR_VERSION=$OPTARG - ;; - s) - DIB_SPARK_VERSION=$OPTARG - ;; - t) - DIB_STORM_VERSION=$OPTARG - ;; - f) - IMAGE_FORMAT="-t $OPTARG" - ;; - u) - DIB_UPDATE_REQUESTED=true - ;; - j) - JAVA_ELEMENT=$OPTARG - ;; - x) - TRACING="$TRACING -x" - set -x - ;; - b) - SIE_BAREMETAL="true" - ;; - h) - usage - exit 0 - ;; - *) - usage - exit 1 - ;; - esac -done - -shift $((OPTIND-1)) -if [ "$1" ]; then - usage - exit 1 -fi - -JAVA_ELEMENT=${JAVA_ELEMENT:-"openjdk"} - -if [ -e /etc/os-release ]; then - platform=$(cat /etc/os-release | awk -F= '/^ID=/ {print tolower($2);}') - # remove eventual quotes around ID=... - platform=$(echo $platform | sed -e 's,^",,;s,"$,,') -elif [ -e /etc/system-release ]; then - case "$(head -1 /etc/system-release)" in - "Red Hat Enterprise Linux Server"*) - platform=rhel - ;; - "CentOS"*) - platform=centos - ;; - *) - echo -e "Unknown value in /etc/system-release. Impossible to build images.\nAborting" - exit 2 - ;; - esac -else - echo -e "Unknown host OS. Impossible to build images.\nAborting" - exit 2 -fi - -# Checks of input -if [ "$DEBUG_MODE" = "true" -a "$platform" != 'ubuntu' ]; then - if [ "$(getenforce)" != "Disabled" ]; then - echo "Debug mode cannot be used from this platform while SELinux is enabled, see https://storyboard.openstack.org/#!/story/1292614" - exit 1 - fi -fi - -check_spark_version () { - case "$DIB_SPARK_VERSION" in - "1.6.0" | "2.1.0" | "2.2.0" | "2.3.0");; - "") - echo "Spark version not specified" - echo "Spark ${DIB_DEFAULT_SPARK_VERSION} will be used" - DIB_SPARK_VERSION=${DIB_DEFAULT_SPARK_VERSION} - ;; - *) - echo -e "Unknown Spark version selected.\nAborting" - exit 1 - ;; - esac -} - -case "$PLUGIN" in - "");; - "vanilla") - case "$HADOOP_VERSION" in - "" | "2.7.1" | "2.7.5" | "2.8.2" | "3.0.1");; - *) - echo -e "Unknown hadoop version selected.\nAborting" - exit 1 - ;; - esac - case "$BASE_IMAGE_OS" in - "" | "ubuntu" | "fedora" | "centos7");; - *) - echo -e "'$BASE_IMAGE_OS' image type is not supported by '$PLUGIN'.\nAborting" - exit 1 - ;; - esac - check_spark_version - ;; - "cloudera") - case "$BASE_IMAGE_OS" in - "" | "ubuntu" | "centos7");; - *) - echo -e "'$BASE_IMAGE_OS' image type is not supported by '$PLUGIN'.\nAborting" - exit 1 - ;; - esac - - case "$HADOOP_VERSION" in - "" | "5.5" | "5.7" | "5.9" | "5.11");; - *) - echo -e "Unknown hadoop version selected.\nAborting" - exit 1 - ;; - esac - - if [ "$BASE_IMAGE_OS" = "centos7" ]; then - if [ ! -z "$HADOOP_VERSION" -a ! "$HADOOP_VERSION" = "5.5" -a ! "$HADOOP_VERSION" = "5.7" -a ! "$HADOOP_VERSION" = "5.9" -a ! "$HADOOP_VERSION" = "5.11" ]; then - echo -e "Unsupported version combination, Centos 7 can only be used with CDH 5.5 or higher version" - exit 1 - fi - fi - - if [ -n "$DIB_CDH_MINOR_VERSION" ]; then - echo -e "Continuing image building with custom CDH version: \"$DIB_CDH_MINOR_VERSION\".\n" - fi - ;; - "spark") - case "$BASE_IMAGE_OS" in - "" | "ubuntu");; - *) - echo -e "'$BASE_IMAGE_OS' image type is not supported by '$PLUGIN'.\nAborting" - exit 1 - ;; - esac - check_spark_version - ;; - "storm") - case "$BASE_IMAGE_OS" in - "" | "ubuntu");; - *) - echo -e "'$BASE_IMAGE_OS' image type is not supported by '$PLUGIN'.\nAborting" - exit 1 - ;; - esac - - case "$DIB_STORM_VERSION" in - "1.0.1" | "1.1.0" | "1.1.1" | "1.2.0" | "1.2.1");; - "") - echo "Storm version not specified" - echo "Storm ${DIB_DEFAULT_STORM_VERSION} will be used" - DIB_STORM_VERSION=${DIB_DEFAULT_STORM_VERSION} - ;; - *) - echo -e "Unknown Storm version selected.\nAborting" - exit 1 - ;; - esac - - if [ -n "$HADOOP_VERSION" ]; then - echo -e "You shouldn't specify hadoop version for '$PLUGIN'.\nAborting" - exit 1 - fi - ;; - "ambari") - case "$BASE_IMAGE_OS" in - "" | "centos7" | "ubuntu" ) - ;; - * ) - echo "\"$BASE_IMAGE_OS\" image type is not supported by \"$PLUGIN\".\nAborting" - exit 1 - ;; - esac - - case "$HADOOP_VERSION" in - "" | "2.2.0.0" | "2.2.1.0" | "2.4.2.0");; - *) - echo -e "Continuing image building with custom ambari version \"$HADOOP_VERSION\"\n" - ;; - esac - ;; - "mapr") - case "$BASE_IMAGE_OS" in - "" | "ubuntu" | "centos7");; - *) - echo -e "'$BASE_IMAGE_OS' image type is not supported by '$PLUGIN'.\nAborting" - exit 1 - ;; - esac - - if [ -n "$HADOOP_VERSION" ]; then - echo -e "You shouldn't specify hadoop version for 'mapr'.\nAborting" - exit 1 - fi - - case "$DIB_MAPR_VERSION" in - "") - echo "MapR version is not specified" - echo "${DIB_DEFAULT_MAPR_VERSION} version would be used" - DIB_MAPR_VERSION=${DIB_DEFAULT_MAPR_VERSION} - ;; - "5.1.0" | "5.2.0");; - *) - echo -e "Unknown MapR version.\nExit" - exit 1 - ;; - esac - ;; - "plain") - case "$BASE_IMAGE_OS" in - "" | "ubuntu" | "fedora" | "centos7");; - *) - echo -e "'$BASE_IMAGE_OS' image type is not supported by '$PLUGIN'.\nAborting" - exit 1 - ;; - esac - - if [ -n "$HADOOP_VERSION" ]; then - echo -e "You shouldn't specify hadoop version for '$PLUGIN'.\nAborting" - exit 1 - fi - ;; - *) - echo -e "Unknown plugin selected.\nAborting" - exit 1 -esac - -if [ "$PLUGIN" != "mapr" -a -n "$DIB_MAPR_VERSION" ]; then - echo -e "'-r' parameter should be used only with 'mapr' plugin.\nAborting" - exit 1 -fi - - -if [ "$JAVA_ELEMENT" != "openjdk" -a "$JAVA_ELEMENT" != "oracle-java" ]; then - echo "Unknown java distro" - exit 1 -fi - -################# - -is_installed() { - if [ "$platform" = 'ubuntu' -o "$platform" = 'debian' ]; then - dpkg -s "$1" &> /dev/null - else - # centos, fedora, opensuse, or rhel - if ! rpm -q "$1" &> /dev/null; then - rpm -q "$(rpm -q --whatprovides "$1")" - fi - fi -} - -need_required_packages() { - case "$platform" in - "ubuntu" | "debian") - package_list="qemu-utils kpartx git" - ;; - "fedora") - package_list="qemu-img kpartx git" - ;; - "opensuse") - package_list="qemu kpartx git-core" - ;; - "rhel" | "centos") - package_list="qemu-kvm qemu-img kpartx git" - ;; - *) - echo -e "Unknown platform '$platform' for the package list.\nAborting" - exit 2 - ;; - esac - - for p in `echo $package_list`; do - if ! is_installed $p; then - echo "Package $p is not installed on the system." - return 0 - fi - done - return 1 -} - -if need_required_packages; then - # install required packages if requested - if [ -n "$DIB_UPDATE_REQUESTED" ]; then - case "$platform" in - "ubuntu" | "debian") - sudo apt-get update - sudo apt-get install $package_list -y - ;; - "opensuse") - sudo zypper --non-interactive --gpg-auto-import-keys in $package_list - ;; - *) - echo -e "Unknown platform '$platform' for installing packages.\nAborting" - exit 2 - ;; - esac - else - echo "Missing one of the following packages: $package_list" - echo "Please install manually or rerun with the update option (-u)." - exit 1 - fi -fi - -if [ "$DEBUG_MODE" = "true" ]; then - echo "Using Image Debug Mode, using root-pwd in images, NOT FOR PRODUCTION USAGE." - # Each image has a root login, password is "hadoop" - export DIB_PASSWORD="hadoop" -fi - -################# - -# Common helper for invoking disk-image-create, adding all the common -# elements and arguments, and setting common environment variables. -# -# Usage: -# image_create DISTRO OUTPUT [args...] -# - DISTRO is the main element of the distribution -# - OUTPUT is the output name for the image -# - any other argument is passed directly to disk-image-create -image_create() { - local distro=$1 - shift - local output=$1 - shift - - # the base elements and args, used in *all* the images - local elements="sahara-version ntp xfs-tools" - if [ $SIE_BAREMETAL = "true" ]; then - elements="grub2 baremetal dhcp-all-interfaces $elements" - else - elements="vm $elements" - fi - local args="" - - # debug mode handling - if [ "$DEBUG_MODE" = "true" ]; then - elements="$elements root-passwd" - fi - # mirror handling - if [ -n "$USE_MIRRORS" ]; then - case "$distro" in - ubuntu) elements="$elements apt-mirror" ;; - fedora) elements="$elements fedora-mirror" ;; - centos7) elements="$elements centos-mirror" ;; - esac - fi - - disk-image-create $IMAGE_FORMAT $TRACING -o "$output" $args "$distro" $elements "$@" -} - -set_hive_version() { - if [ -z "${HIVE_VERSION:-}" ]; then - case "$DIB_HADOOP_VERSION" in - "2.7.1" ) - export HIVE_VERSION="0.11.0" - ;; - "2.7.5" ) - export HIVE_VERSION="2.3.2" - ;; - "2.8.2" ) - export HIVE_VERSION="2.3.2" - ;; - "3.0.1" ) - export HIVE_VERSION="3.0.0" - ;; - *) - echo -e "Unknown Hadoop version, therefore cannot choose Hive version.\nAborting." - exit 1 - ;; - esac - fi -} - - -############################# -# Images for Vanilla plugin # -############################# - -if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then - export HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.7.1.tar.gz"} - export HADOOP_V2_7_5_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_7_5_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.7.5.tar.gz"} - export HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.8.2.tar.gz"} - export HADOOP_V3_0_1_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V3_0_1_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-3.0.1.tar.gz"} - export OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL=${OOZIE_HADOOP_V2_7_1_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.2.0-hadoop-2.7.1.tar.gz"} - export OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL=${OOZIE_HADOOP_V2_7_5_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.3.0-hadoop-2.7.5.tar.gz"} - export OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL=${OOZIE_HADOOP_V2_8_2_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.3.0-hadoop-2.8.2.tar.gz"} - export OOZIE_HADOOP_V3_0_1_DOWNLOAD_URL=${OOZIE_HADOOP_V3_0_1_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-5.0.0-hadoop-3.0.1.tar.gz"} - export DIB_HDFS_LIB_DIR="/opt/hadoop/share/hadoop/tools/lib" - export plugin_type="vanilla" - - export DIB_SPARK_VERSION - - if [ "$DIB_SPARK_VERSION" = "1.6.0" ]; then - export SPARK_HADOOP_DL=hadoop2.6 - else - export SPARK_HADOOP_DL=hadoop2.7 - fi - - ubuntu_elements_sequence="hadoop oozie mysql hive $JAVA_ELEMENT swift_hadoop spark s3_hadoop" - fedora_elements_sequence="hadoop oozie mysql disable-firewall hive $JAVA_ELEMENT swift_hadoop spark s3_hadoop" - centos7_elements_sequence="hadoop oozie mysql disable-firewall hive $JAVA_ELEMENT swift_hadoop spark nc s3_hadoop" - - # Workaround for https://bugs.launchpad.net/diskimage-builder/+bug/1204824 - # https://storyboard.openstack.org/#!/story/1252684 - if [ "$platform" = 'ubuntu' ]; then - echo "**************************************************************" - echo "WARNING: As a workaround for DIB bug 1204824, you are about to" - echo " create a Fedora and CentOS images that has SELinux " - echo " disabled. Do not use these images in production. " - echo "**************************************************************" - fedora_elements_sequence="$fedora_elements_sequence selinux-permissive" - centos7_elements_sequence="$centos7_elements_sequence selinux-permissive" - suffix=".selinux-permissive" - fi - # Ubuntu cloud image - if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then - export DIB_CLOUD_INIT_DATASOURCES=$CLOUD_INIT_DATASOURCES - - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.1" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_1:-"2.7.1"} - export ubuntu_image_name=${ubuntu_vanilla_hadoop_2_7_1_image_name:-"ubuntu_sahara_vanilla_hadoop_2_7_1_latest"} - export DIB_RELEASE=${DIB_RELEASE:-xenial} - set_hive_version - image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence - unset DIB_RELEASE - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.5" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_5:-"2.7.5"} - export ubuntu_image_name=${ubuntu_vanilla_hadoop_2_7_5_image_name:-"ubuntu_sahara_vanilla_hadoop_2_7_5_latest"} - export DIB_RELEASE=${DIB_RELEASE:-xenial} - set_hive_version - image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence - unset DIB_RELEASE - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.8.2" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_8_2:-"2.8.2"} - export ubuntu_image_name=${ubuntu_vanilla_hadoop_2_8_2_image_name:-"ubuntu_sahara_vanilla_hadoop_2_8_2_latest"} - export DIB_RELEASE=${DIB_RELEASE:-xenial} - set_hive_version - image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence - unset DIB_RELEASE - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "3.0.1" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_3_0_1:-"3.0.1"} - export ubuntu_image_name=${ubuntu_vanilla_hadoop_3_0_1_image_name:-"ubuntu_sahara_vanilla_hadoop_3_0_1_latest"} - export DIB_RELEASE=${DIB_RELEASE:-xenial} - set_hive_version - image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence - unset DIB_RELEASE - fi - unset DIB_CLOUD_INIT_DATASOURCES - fi - - # Fedora cloud image - if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "fedora" ]; then - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.1" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_1:-"2.7.1"} - export fedora_image_name=${fedora_vanilla_hadoop_2_7_1_image_name:-"fedora_sahara_vanilla_hadoop_2_7_1_latest$suffix"} - set_hive_version - image_create fedora $fedora_image_name $fedora_elements_sequence - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.5" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_5:-"2.7.5"} - export fedora_image_name=${fedora_vanilla_hadoop_2_7_5_image_name:-"fedora_sahara_vanilla_hadoop_2_7_5_latest$suffix"} - set_hive_version - image_create fedora $fedora_image_name $fedora_elements_sequence - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.8.2" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_8_2:-"2.8.2"} - export fedora_image_name=${fedora_vanilla_hadoop_2_8_2_image_name:-"fedora_sahara_vanilla_hadoop_2_8_2_latest$suffix"} - set_hive_version - image_create fedora $fedora_image_name $fedora_elements_sequence - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "3.0.1" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_3_0_1:-"3.0.1"} - export fedora_image_name=${fedora_vanilla_hadoop_3_0_1_image_name:-"fedora_sahara_vanilla_hadoop_3_0_1_latest$suffix"} - set_hive_version - image_create fedora $fedora_image_name $fedora_elements_sequence - fi - fi - - # CentOS 7 cloud image - if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos7" ]; then - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.1" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_1:-"2.7.1"} - export centos7_image_name=${centos7_vanilla_hadoop_2_7_1_image_name:-"centos7_sahara_vanilla_hadoop_2_7_1_latest$suffix"} - set_hive_version - image_create centos7 $centos7_image_name $centos7_elements_sequence - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.5" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_5:-"2.7.5"} - export centos7_image_name=${centos7_vanilla_hadoop_2_7_5_image_name:-"centos7_sahara_vanilla_hadoop_2_7_5_latest$suffix"} - set_hive_version - image_create centos7 $centos7_image_name $centos7_elements_sequence - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.8.2" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_8_2:-"2.8.2"} - export centos7_image_name=${centos7_vanilla_hadoop_2_8_2_image_name:-"centos7_sahara_vanilla_hadoop_2_8_2_latest$suffix"} - set_hive_version - image_create centos7 $centos7_image_name $centos7_elements_sequence - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "3.0.1" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_3_0_1:-"3.0.1"} - export centos7_image_name=${centos7_vanilla_hadoop_3_0_1_image_name:-"centos7_sahara_vanilla_hadoop_3_0_1_latest$suffix"} - set_hive_version - image_create centos7 $centos7_image_name $centos7_elements_sequence - fi - fi - - unset plugin_type - unset DIB_HDFS_LIB_DIR - unset DIB_SPARK_VERSION - unset SPARK_HADOOP_DL -fi - -########################### -# Images for Spark plugin # -########################### - -if [ -z "$PLUGIN" -o "$PLUGIN" = "spark" ]; then - export DIB_HDFS_LIB_DIR="/usr/lib/hadoop-mapreduce" - export DIB_CLOUD_INIT_DATASOURCES=$CLOUD_INIT_DATASOURCES - export DIB_SPARK_VERSION - export plugin_type="spark" - - if [ "$DIB_SPARK_VERSION" = "2.2.0" -o "$DIB_SPARK_VERSION" = "2.3.0" ]; then - export DIB_CDH_VERSION="5.11" - export DIB_RELEASE=${DIB_RELEASE:-xenial} - else - export DIB_RELEASE=${DIB_RELEASE:-trusty} - export DIB_CDH_VERSION="5.5" - fi - if [ "$DIB_SPARK_VERSION" = "1.6.0" ]; then - export SPARK_HADOOP_DL=hadoop2.6 - else - export SPARK_HADOOP_DL=hadoop2.7 - fi - # Tell the cloudera element to install only hdfs - export DIB_CDH_HDFS_ONLY=1 - - ubuntu_elements_sequence="$JAVA_ELEMENT swift_hadoop spark hadoop-cloudera s3_hadoop" - export ubuntu_image_name=${ubuntu_spark_image_name:-"ubuntu_sahara_spark_latest"} - - # Creating Ubuntu cloud image - image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence - unset SPARK_HADOOP_DL - unset DIB_CLOUD_INIT_DATASOURCES - unset DIB_HDFS_LIB_DIR - unset DIB_CDH_HDFS_ONLY - unset DIB_CDH_VERSION - unset DIB_SPARK_VERSION - unset DIB_HADOOP_VERSION - unset DIB_RELEASE - unset plugin_type -fi - - -########################## -# Image for Storm plugin # -########################## - -if [ -z "$PLUGIN" -o "$PLUGIN" = "storm" ]; then - export DIB_CLOUD_INIT_DATASOURCES=$CLOUD_INIT_DATASOURCES - - export DIB_STORM_VERSION - export ubuntu_image_name=${ubuntu_storm_image_name:-"ubuntu_sahara_storm_latest_$DIB_STORM_VERSION"} - - ubuntu_elements_sequence="$JAVA_ELEMENT zookeeper storm" - - # Creating Ubuntu cloud image - export DIB_RELEASE=${DIB_RELEASE:-xenial} - image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence - unset DIB_RELEASE - unset DIB_CLOUD_INIT_DATASOURCES -fi - -############################ -# Images for Ambari plugin # -############################ - -if [ -z "$PLUGIN" -o "$PLUGIN" = "ambari" ]; then - export DIB_AMBARI_VERSION="$HADOOP_VERSION" - export plugin_type="ambari" - # set the temporary folder for hadoop-openstack.jar file - export DIB_HDFS_LIB_DIR="/opt" - - if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then - ambari_ubuntu_image_name=${ambari_ubuntu_image_name:-ubuntu_sahara_ambari} - ambari_element_sequence="ambari $JAVA_ELEMENT swift_hadoop kdc" - export DIB_RELEASE="trusty" - image_create ubuntu $ambari_ubuntu_image_name $ambari_element_sequence - unset DIB_RELEASE - fi - if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos7" ]; then - ambari_centos7_image_name=${ambari_centos7_image_name:-"centos7-sahara-ambari"} - ambari_element_sequence="disable-selinux ambari $JAVA_ELEMENT disable-firewall swift_hadoop kdc nc" - image_create centos7 $ambari_centos7_image_name $ambari_element_sequence - fi - - unset DIB_HDFS_LIB_DIR - unset plugin_type - unset DIB_AMBARI_VERSION -fi - -######################### -# Images for CDH plugin # -######################### - -if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then - # Cloudera installation requires additional space - export DIB_MIN_TMPFS=5 - export plugin_type="cloudera" - export DIB_HDFS_LIB_DIR="/usr/lib/hadoop-mapreduce" - - if [ -n "$DIB_CDH_MINOR_VERSION" ]; then - # cut minor version number, e.g. from 5.7.1 to 5.7 - # this is needed if user specified minor version but didn't specify - # hadoop version by '-v' parameter - HADOOP_VERSION=${DIB_CDH_MINOR_VERSION%.*} - fi - - cloudera_elements_sequence="hadoop-cloudera swift_hadoop kdc" - if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.5" ]; then - export DIB_CDH_VERSION="5.5" - cloudera_5_5_ubuntu_image_name=${cloudera_5_5_ubuntu_image_name:-ubuntu_sahara_cloudera_5_5_0} - - # Cloudera supports 14.04 Ubuntu in 5.5 - export DIB_RELEASE="trusty" - image_create ubuntu $cloudera_5_5_ubuntu_image_name $cloudera_elements_sequence - unset DIB_CDH_VERSION DIB_RELEASE - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.7" ]; then - export DIB_CDH_VERSION="5.7" - export DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$DIB_CDH_VERSION.0} - cloudera_5_7_ubuntu_image_name=${cloudera_5_7_ubuntu_image_name:-ubuntu_sahara_cloudera_$DIB_CDH_MINOR_VERSION} - - export DIB_RELEASE="trusty" - image_create ubuntu $cloudera_5_7_ubuntu_image_name $cloudera_elements_sequence - unset DIB_CDH_VERSION DIB_RELEASE DIB_CDH_MINOR_VERSION - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.9" ]; then - export DIB_CDH_VERSION="5.9" - export DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$DIB_CDH_VERSION.0} - cloudera_5_9_ubuntu_image_name=${cloudera_5_9_ubuntu_image_name:-ubuntu_sahara_cloudera_$DIB_CDH_MINOR_VERSION} - - export DIB_RELEASE="trusty" - image_create ubuntu $cloudera_5_9_ubuntu_image_name $cloudera_elements_sequence - unset DIB_CDH_VERSION DIB_RELEASE DIB_CDH_MINOR_VERSION - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.11" ]; then - export DIB_CDH_VERSION="5.11" - export DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$DIB_CDH_VERSION.0} - cloudera_5_11_ubuntu_image_name=${cloudera_5_11_ubuntu_image_name:-ubuntu_sahara_cloudera_$DIB_CDH_MINOR_VERSION} - - export DIB_RELEASE="xenial" - image_create ubuntu $cloudera_5_11_ubuntu_image_name $cloudera_elements_sequence - unset DIB_CDH_VERSION DIB_RELEASE DIB_CDH_MINOR_VERSION - fi - fi - - if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos7" ]; then - centos7_cloudera_elements_sequence="selinux-permissive disable-firewall nc" - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.5" ]; then - export DIB_CDH_VERSION="5.5" - - cloudera_5_5_centos7_image_name=${cloudera_5_5_centos7_image_name:-centos7_sahara_cloudera_5_5_0} - image_create centos7 $cloudera_5_5_centos7_image_name $cloudera_elements_sequence $centos7_cloudera_elements_sequence - - unset DIB_CDH_VERSION - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.7" ]; then - export DIB_CDH_VERSION="5.7" - export DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$DIB_CDH_VERSION.0} - - cloudera_5_7_centos7_image_name=${cloudera_5_7_centos7_image_name:-centos7_sahara_cloudera_$DIB_CDH_MINOR_VERSION} - image_create centos7 $cloudera_5_7_centos7_image_name $cloudera_elements_sequence $centos7_cloudera_elements_sequence - - unset DIB_CDH_VERSION DIB_CDH_MINOR_VERSION - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.9" ]; then - export DIB_CDH_VERSION="5.9" - export DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$DIB_CDH_VERSION.0} - - cloudera_5_9_centos7_image_name=${cloudera_5_9_centos7_image_name:-centos7_sahara_cloudera_$DIB_CDH_MINOR_VERSION} - image_create centos7 $cloudera_5_9_centos7_image_name $cloudera_elements_sequence $centos7_cloudera_elements_sequence - - unset DIB_CDH_VERSION DIB_CDH_MINOR_VERSION - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.11" ]; then - export DIB_CDH_VERSION="5.11" - export DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$DIB_CDH_VERSION.0} - - cloudera_5_11_centos7_image_name=${cloudera_5_11_centos7_image_name:-centos7_sahara_cloudera_$DIB_CDH_MINOR_VERSION} - image_create centos7 $cloudera_5_11_centos7_image_name $cloudera_elements_sequence $centos7_cloudera_elements_sequence - - unset DIB_CDH_VERSION DIB_CDH_MINOR_VERSION - fi - fi - - unset DIB_CDH_MINOR_VERSION - unset DIB_HDFS_LIB_DIR - unset DIB_MIN_TMPFS - unset plugin_type -fi - -########################## -# Images for MapR plugin # -########################## -if [ -z "$PLUGIN" -o "$PLUGIN" = "mapr" ]; then - export DIB_MAPR_VERSION=${DIB_MAPR_VERSION:-${DIB_DEFAULT_MAPR_VERSION}} - export plugin_type="mapr" - - export DIB_CLOUD_INIT_DATASOURCES=$CLOUD_INIT_DATASOURCES - - export DIB_IMAGE_SIZE=${IMAGE_SIZE:-"10"} - #MapR repository requires additional space - export DIB_MIN_TMPFS=10 - - mapr_ubuntu_elements_sequence="ssh hadoop-mapr $JAVA_ELEMENT" - mapr_centos_elements_sequence="ssh hadoop-mapr selinux-permissive $JAVA_ELEMENT disable-firewall nc" - - if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then - export DIB_RELEASE=${DIB_RELEASE:-trusty} - - mapr_ubuntu_image_name=${mapr_ubuntu_image_name:-ubuntu_${DIB_RELEASE}_mapr_${DIB_MAPR_VERSION}_latest} - - image_create ubuntu $mapr_ubuntu_image_name $mapr_ubuntu_elements_sequence - - unset DIB_RELEASE - fi - - if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos7" ]; then - mapr_centos7_image_name=${mapr_centos7_image_name:-centos_7_mapr_${DIB_MAPR_VERSION}_latest} - - image_create centos7 $mapr_centos7_image_name $mapr_centos_elements_sequence - - unset DIB_CLOUD_INIT_DATASOURCES - fi - unset plugin_type - -fi - -################ -# Plain images # -################ -if [ -z "$PLUGIN" -o "$PLUGIN" = "plain" ]; then - # generate plain (no Hadoop components) images for testing - - common_elements="ssh" - - ubuntu_elements_sequence="$common_elements" - fedora_elements_sequence="$common_elements" - centos7_elements_sequence="$common_elements disable-firewall disable-selinux nc" - - if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then - plain_image_name=${plain_ubuntu_image_name:-ubuntu_plain} - - export DIB_RELEASE=${DIB_RELEASE:-xenial} - image_create ubuntu $plain_image_name $ubuntu_elements_sequence - unset DIB_RELEASE - fi - - if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "fedora" ]; then - plain_image_name=${plain_fedora_image_name:-fedora_plain} - - image_create fedora $plain_image_name $fedora_elements_sequence - fi - - if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos7" ]; then - plain_image_name=${plain_centos7_image_name:-centos7_plain} - - image_create centos7 $plain_image_name $centos7_elements_sequence - fi -fi diff --git a/elements/.gitignore b/elements/.gitignore deleted file mode 100644 index b25c15b8..00000000 --- a/elements/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*~ diff --git a/elements/README.rst b/elements/README.rst deleted file mode 100644 index 96469d4f..00000000 --- a/elements/README.rst +++ /dev/null @@ -1,69 +0,0 @@ -Diskimage-builder tools for creation cloud images -================================================= - -Steps how to create cloud image with Apache Hadoop installed using diskimage-builder project: - -1. Clone the repository "https://github.com/openstack/diskimage-builder" locally. Note: Make sure you have commit 43b96d91 in your clone, it provides a mapping for default-jre. - -.. sourcecode:: bash - - git clone https://github.com/openstack/diskimage-builder - -2. Add ~/diskimage-builder/bin/ directory to your path (for example, PATH=$PATH:/home/$USER/diskimage-builder/bin/ ). - -3. Export the following variable ELEMENTS_PATH=/home/$USER/diskimage-builder/elements/ to your .bashrc. Then source it. - -4. Copy file "img-build-sudoers" from ~/disk-image-builder/sudoers.d/ to your /etc/sudoers.d/. - -.. sourcecode:: bash - - chmod 440 /etc/sudoers.d/img-build-sudoers - chown root:root /etc/sudoers.d/img-build-sudoers - -5. Export sahara-elements commit id variable (from sahara-extra directory): - -.. sourcecode:: bash - - export SAHARA_ELEMENTS_COMMIT_ID=`git show --format=%H | head -1` - -6. Move elements/ directory to disk-image-builder/elements/ - -.. sourcecode:: bash - - mv elements/* /path_to_disk_image_builder/diskimage-builder/elements/ - -7. Export DIB commit id variable (from DIB directory): - -.. sourcecode:: bash - - export DIB_COMMIT_ID=`git show --format=%H | head -1` - -8. Call the following command to create cloud image is able to run on OpenStack: - -8.1. Ubuntu cloud image - -.. sourcecode:: bash - - JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_HADOOP_VERSION=1.2.1 OOZIE_FILE=oozie-4.0.0.tar.gz disk-image-create base vm hadoop oozie ubuntu root-passwd -o ubuntu_hadoop_1_2_1 - -8.2. Fedora cloud image - -.. sourcecode:: bash - - JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_HADOOP_VERSION=1.2.1 OOZIE_FILE=oozie-4.0.0.tar.gz DIB_IMAGE_SIZE=10 disk-image-create base vm fedora hadoop root-passwd oozie -o fedora_hadoop_1_2_1 - -Note: If you are building this image from Ubuntu or Fedora 18 OS host, you should add element 'selinux-permissive'. - -.. sourcecode:: bash - - JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_HADOOP_VERSION=1.2.1 OOZIE_FILE=oozie-4.0.0.tar.gz DIB_IMAGE_SIZE=10 disk-image-create base vm fedora hadoop root-passwd oozie selinux-permissive -o fedora_hadoop_1_2_1 - -In this command 'DIB_HADOOP_VERSION' parameter is version of hadoop needs to be installed. -You can use 'JAVA_DOWNLOAD_URL' parameter to specify download link for JDK (tarball or bin). -'DIB_IMAGE_SIZE' is parameter that specifes a volume of hard disk of instance. You need to specify it because Fedora and CentOS don't use all available volume. -If you have already downloaded the jdk package, move it to "elements/hadoop/install.d/" and use its filename as 'JAVA_FILE' parameter. -In order of working EDP components with Sahara DIB images you need pre-installed Oozie libs. -Use OOZIE_DOWNLOAD_URL to specify link to Oozie archive (tar.gz). For example the Oozie libraries -for Hadoop 2.7.1 are available from: -https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.2.0-hadoop-2.7.1.tar.gz -If you have already downloaded archive, move it to "elements/oozie/install.d/" and use its filename as 'OOZIE_FILE' parameter. diff --git a/elements/ambari/README.rst b/elements/ambari/README.rst deleted file mode 100644 index e59f7aea..00000000 --- a/elements/ambari/README.rst +++ /dev/null @@ -1,14 +0,0 @@ -====== -ambari -====== - -Installs Ambari Management Console - -Environment Variables ---------------------- - -AMBARI_VERSION - :Required: No - :Default: 2.4.2.0 - :Description: Version of Ambari Management Console to install - :Example: ``AMBARI_VERSION="2.2.1.0"`` installs Ambari 2.2.1.0 diff --git a/elements/ambari/element-deps b/elements/ambari/element-deps deleted file mode 100644 index e6180587..00000000 --- a/elements/ambari/element-deps +++ /dev/null @@ -1,3 +0,0 @@ -java -mysql -package-installs diff --git a/elements/ambari/package-installs.yaml b/elements/ambari/package-installs.yaml deleted file mode 100644 index 645fc03d..00000000 --- a/elements/ambari/package-installs.yaml +++ /dev/null @@ -1,19 +0,0 @@ -wget: - phase: pre-install.d -ambari-agent: -ambari-log4j: -ambari-metrics-hadoop-sink: -ambari-metrics-monitor: -ambari-server: -ambari-metrics-collector: -redhat-lsb: -nc: -unzip: -curl: -tar: -zip: -rpcbind: -fuse-libs: -fuse: -snappy-devel: -rng-tools: diff --git a/elements/ambari/pkg-map b/elements/ambari/pkg-map deleted file mode 100644 index 3adf2b09..00000000 --- a/elements/ambari/pkg-map +++ /dev/null @@ -1,20 +0,0 @@ -{ - "family": { - "debian": { - "nc": "netcat", - "redhat-lsb": "", - "snappy-devel": "", - "fuse-libs": "fuse", - "ambari-metrics-monitor": "ambari-metrics-assembly", - "ambari-metrics-collector": "ambari-metrics-assembly", - "ambari-metrics-hadoop-sink": "ambari-metrics-assembly", - "ambari-log4j": "" - }, - "redhat": { - "nc": "nc", - "redhat-lsb": "redhat-lsb", - "snappy-devel": "snappy-devel", - "fuse-libs": "fuse-libs" - } - } -} diff --git a/elements/ambari/post-install.d/10-disable-ambari b/elements/ambari/post-install.d/10-disable-ambari deleted file mode 100755 index e7841f74..00000000 --- a/elements/ambari/post-install.d/10-disable-ambari +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -case "${DISTRO_NAME}" in - rhel7 | centos7 ) - chkconfig ambari-server off - chkconfig ambari-agent off - ;; -esac diff --git a/elements/ambari/post-install.d/20-disable-ca-certificate b/elements/ambari/post-install.d/20-disable-ca-certificate deleted file mode 100755 index 711f1f21..00000000 --- a/elements/ambari/post-install.d/20-disable-ca-certificate +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -case "${DISTRO_NAME}" in - rhel7 | centos7 ) - config=/etc/python/cert-verification.cfg - [ -e $config ] && sed -i "s%^\(verify=\s*\).*$%verify=disable%" $config - ;; -esac diff --git a/elements/ambari/pre-install.d/90-ambari-repo-conf b/elements/ambari/pre-install.d/90-ambari-repo-conf deleted file mode 100755 index 46d5d483..00000000 --- a/elements/ambari/pre-install.d/90-ambari-repo-conf +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -DIB_AMBARI_VERSION="${DIB_AMBARI_VERSION:-2.4.2.0}" - -case "${DISTRO_NAME}" in - rhel7 | centos7 ) - wget http://public-repo-1.hortonworks.com/ambari/centos7/2.x/updates/$DIB_AMBARI_VERSION/ambari.repo -O /etc/yum.repos.d/ambari.repo - ;; - ubuntu ) - wget http://public-repo-1.hortonworks.com/ambari/ubuntu14/2.x/updates/$DIB_AMBARI_VERSION/ambari.list -O /etc/apt/sources.list.d/ambari.list - apt-key adv --recv-keys --keyserver keyserver.ubuntu.com B9733A7A07513CAD - ;; -esac diff --git a/elements/apt-mirror/README.rst b/elements/apt-mirror/README.rst deleted file mode 100644 index 0454cc88..00000000 --- a/elements/apt-mirror/README.rst +++ /dev/null @@ -1,13 +0,0 @@ -========== -apt-mirror -========== - -This element sets up the mirror for updating the Ubuntu cloud image. -Using a mirror improves the speed of the image building. - -Environment Variables ---------------------- - -UBUNTU_MIRROR - :Required: Yes - :Description: URL to the Ubuntu mirror. diff --git a/elements/apt-mirror/finalise.d/99-setup b/elements/apt-mirror/finalise.d/99-setup deleted file mode 100755 index 91ce9815..00000000 --- a/elements/apt-mirror/finalise.d/99-setup +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -rm /etc/apt/apt.conf.d/01proxy diff --git a/elements/apt-mirror/pre-install.d/00-apt b/elements/apt-mirror/pre-install.d/00-apt deleted file mode 100755 index e73d437d..00000000 --- a/elements/apt-mirror/pre-install.d/00-apt +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -mkdir -p /etc/apt/apt.conf.d/ -touch /etc/apt/apt.conf.d/01proxy -echo -e "Acquire::http { Proxy \"$UBUNTU_MIRROR\"; };" > /etc/apt/apt.conf.d/01proxy diff --git a/elements/apt-mirror/root.d/0-check b/elements/apt-mirror/root.d/0-check deleted file mode 100755 index a87746e1..00000000 --- a/elements/apt-mirror/root.d/0-check +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -if [ -z "$UBUNTU_MIRROR" ]; then - echo "You should specify parameter 'UBUNTU_MIRROR'" - exit 2 -fi diff --git a/elements/centos-mirror/README.rst b/elements/centos-mirror/README.rst deleted file mode 100644 index 242d00e3..00000000 --- a/elements/centos-mirror/README.rst +++ /dev/null @@ -1,13 +0,0 @@ -============= -centos-mirror -============= - -This element sets up the mirror for updating the CentOS cloud image. -Using a mirror improves the speed of the image building. - -Environment Variables ---------------------- - -CENTOS_MIRROR - :Required: Yes - :Description: URL to the CentOS mirror. diff --git a/elements/centos-mirror/finalise.d/99-setup b/elements/centos-mirror/finalise.d/99-setup deleted file mode 100755 index bbc84deb..00000000 --- a/elements/centos-mirror/finalise.d/99-setup +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -mv /opt/repos/* /etc/yum.repos.d/ diff --git a/elements/centos-mirror/pre-install.d/00-yum b/elements/centos-mirror/pre-install.d/00-yum deleted file mode 100755 index ef658883..00000000 --- a/elements/centos-mirror/pre-install.d/00-yum +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -mkdir -p /opt/repos -mv /etc/yum.repos.d/* /opt/repos/ -cat >> /etc/yum.repos.d/centos.repo <> /etc/yum.repos.d/fedora.repo < /dev/null; then - addgroup hadoop - fi - if ! getent passwd hadoop > /dev/null; then - adduser --ingroup hadoop --disabled-password --gecos GECOS hadoop - fi - adduser hadoop sudo - ;; -esac - -echo "Pre-configuring Hadoop" - -cat >> /home/hadoop/.bashrc <> /etc/hadoop/hadoop-env.sh <> /etc/apt/preferences.d/cloudera << EOF -Package: * -Pin: origin "archive.cloudera.com" -Pin-Priority: 800 -EOF - - dpkg -i /tmp/cdh4-repository_1.0_all.deb - rm /tmp/cdh4-repository_1.0_all.deb - wget -O - http://archive.cloudera.com/cdh4/ubuntu/precise/amd64/cdh/archive.key | sudo apt-key add - - ;; - *) - echo "Distro $DISTRO_NAME not supported by CDH. Exiting." - exit 1 - ;; -esac diff --git a/elements/hadoop-cdh/root.d/0-check-hadoop-cdh b/elements/hadoop-cdh/root.d/0-check-hadoop-cdh deleted file mode 100755 index e7ad22d4..00000000 --- a/elements/hadoop-cdh/root.d/0-check-hadoop-cdh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -echo "##############################################################" -echo "## ##" -echo "## WARNING: this element (hadoop-cdh) has been deprecated ##" -echo "## Please use the hadoop-cloudera element ##" -echo "## ##" -echo "##############################################################" - -if [ -z "${JAVA_DOWNLOAD_URL:-}" ]; then - if [ -z "${JAVA_FILE:-}" ]; then - echo "JAVA_FILE and JAVA_DOWNLOAD_URL are not set. Proceeding with distro native Java." - fi -fi -if [ -z "$DIB_CDH_VERSION" ]; then - echo "DIB_CDH_VERSION is not set. Impossible to install hadoop. Exit" - exit 1 -fi -if [ $DIB_CDH_VERSION != "CDH4" ]; then - echo "CDH version $DIB_CDH_VERSION not supported. Exiting." - exit 1 -fi diff --git a/elements/hadoop-cloudera/README.rst b/elements/hadoop-cloudera/README.rst deleted file mode 100644 index 3b0e5c0b..00000000 --- a/elements/hadoop-cloudera/README.rst +++ /dev/null @@ -1,37 +0,0 @@ -=============== -hadoop-cloudera -=============== - -Installs cloudera (cloudera-manager-agent cloudera-manager-daemons -cloudera-manager-server cloudera-manager-server-db-2 hadoop-hdfs-namenode -hadoop-hdfs-secondarynamenode hadoop-hdfs-datanode hadoop-yarn-resourcemanager -hadoop-yarn-nodemanager hadoop-mapreduce hadoop-mapreduce-historyserver) and -Java (oracle-j2sdk1.7) packages from cloudera repositories: -`cdh5 `_ and -`cm5 `_. - -Also installs Cloudera distribution of Apache Kafka for CDH version >= 5.5 from -Cloudera repository: `kafka `_. - -In order to create the Cloudera images with ``diskimage-create.sh``, use the -following syntax to select the ``cloudera`` plugin: - -.. sourcecode:: bash - - diskimage-create.sh -p cloudera - -Environment Variables ---------------------- - -The element can be configured by exporting variables using a -`environment.d` script. - -DIB_CDH_HDFS_ONLY - :Required: No - :Description: If set will install only the namenode and datanode - packages with their dependencies. - -DIB_CDH_MINOR_VERSION - :Required: No - :Description: If set will install minor version of CDH. Available minor - versions are 5.7.x. diff --git a/elements/hadoop-cloudera/element-deps b/elements/hadoop-cloudera/element-deps deleted file mode 100644 index 52ac5063..00000000 --- a/elements/hadoop-cloudera/element-deps +++ /dev/null @@ -1,4 +0,0 @@ -extjs -java -package-installs -ssh diff --git a/elements/hadoop-cloudera/environment.d/00-configure-extjs b/elements/hadoop-cloudera/environment.d/00-configure-extjs deleted file mode 100644 index f8d84197..00000000 --- a/elements/hadoop-cloudera/environment.d/00-configure-extjs +++ /dev/null @@ -1 +0,0 @@ -export EXTJS_DESTINATION_DIR="/var/lib/oozie" diff --git a/elements/hadoop-cloudera/environment.d/01-set-java-distro b/elements/hadoop-cloudera/environment.d/01-set-java-distro deleted file mode 100644 index 85cb17bd..00000000 --- a/elements/hadoop-cloudera/environment.d/01-set-java-distro +++ /dev/null @@ -1 +0,0 @@ -export DIB_JAVA_DISTRO="cloudera-jdk" diff --git a/elements/hadoop-cloudera/install.d/34-java b/elements/hadoop-cloudera/install.d/34-java deleted file mode 100755 index 5f479c4f..00000000 --- a/elements/hadoop-cloudera/install.d/34-java +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then - echo "Installing Oracle JDK for Cloudera" - - pkg="oracle-j2sdk1.7" - install-packages $pkg - - JDK_HOME="" - case $DISTRO_NAME in - centos7 | rhel7 ) - JDK_HOME=$(find /usr/java -name 'jdk*cloudera' -print) - ;; - ubuntu ) - JDK_HOME="/usr/lib/jvm/java-7-oracle-cloudera" - ;; - esac - - JRE_HOME=$JDK_HOME/jre - - setup-java-home $JRE_HOME $JDK_HOME -else - echo "Skip this step. JDK has already been installed for Spark." -fi diff --git a/elements/hadoop-cloudera/install.d/50-install-cloudera b/elements/hadoop-cloudera/install.d/50-install-cloudera deleted file mode 100755 index beac78c7..00000000 --- a/elements/hadoop-cloudera/install.d/50-install-cloudera +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -if [ "$DISTRO_NAME" = "ubuntu" ]; then - export DEBIAN_FRONTEND=noninteractive - export RUNLEVEL=1 -fi - -# Install the rest of CDH unless a limited HDFS install was requested -if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then - install-packages \ - cloudera-manager-agent \ - cloudera-manager-daemons \ - cloudera-manager-server \ - cloudera-manager-server-db-2 \ - hadoop-hdfs-secondarynamenode \ - hadoop-mapreduce \ - hadoop-mapreduce-historyserver \ - hadoop-yarn-nodemanager \ - hadoop-yarn-resourcemanager \ - hbase \ - hive-hcatalog \ - hive-metastore \ - hive-server2 \ - hive-webhcat-server \ - hue \ - oozie \ - spark-core \ - zookeeper \ - flume-ng \ - hadoop-kms \ - hbase-solr \ - impala \ - impala-server \ - impala-state-store \ - impala-catalog \ - impala-shell \ - keytrustee-keyprovider \ - sentry \ - solr-server \ - solr-doc \ - search \ - spark-history-server \ - sqoop2 \ - kafka \ - kafka-server -fi - -DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$DIB_CDH_VERSION.0} - -case "$DIB_CDH_VERSION" in - 5.7) - HADOOP_OPENSTACK_5_7_x_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.6.0-cdh$DIB_CDH_MINOR_VERSION/hadoop-openstack-2.6.0-cdh$DIB_CDH_MINOR_VERSION.jar" - ;; - 5.9) - HADOOP_OPENSTACK_5_9_x_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.6.0-cdh$DIB_CDH_MINOR_VERSION/hadoop-openstack-2.6.0-cdh$DIB_CDH_MINOR_VERSION.jar" - ;; - 5.11) - HADOOP_OPENSTACK_5_11_x_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.6.0-cdh$DIB_CDH_MINOR_VERSION/hadoop-openstack-2.6.0-cdh$DIB_CDH_MINOR_VERSION.jar" - ;; -esac -HADOOP_OPENSTACK_5_5_0_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.6.0-cdh5.5.0/hadoop-openstack-2.6.0-cdh5.5.0.jar" - -dest=/usr/lib/hadoop-mapreduce/hadoop-openstack.jar -case "$DIB_CDH_VERSION" in - 5.5) - wget -O $dest $HADOOP_OPENSTACK_5_5_0_URL - if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then - # Create links to keep backward version support. - ln -s /usr/lib/oozie/oozie-sharelib-yarn /usr/lib/oozie/oozie-sharelib-yarn.tar.gz - ln -s /usr/lib/oozie/oozie-sharelib-mr1 /usr/lib/oozie/oozie-sharelib-mr1.tar.gz - ln -s /usr/lib/oozie/oozie-sharelib-yarn.tar.gz /usr/lib/oozie/oozie-sharelib.tar.gz - fi - ;; - 5.7) - wget -O $dest $HADOOP_OPENSTACK_5_7_x_URL - if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then - # Create links to keep backward version support. - ln -s /usr/lib/oozie/oozie-sharelib-yarn /usr/lib/oozie/oozie-sharelib-yarn.tar.gz - ln -s /usr/lib/oozie/oozie-sharelib-mr1 /usr/lib/oozie/oozie-sharelib-mr1.tar.gz - ln -s /usr/lib/oozie/oozie-sharelib-yarn.tar.gz /usr/lib/oozie/oozie-sharelib.tar.gz - fi - ;; - 5.9) - wget -O $dest $HADOOP_OPENSTACK_5_9_x_URL - if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then - # Create links to keep backward version support. - ln -s /usr/lib/oozie/oozie-sharelib-yarn /usr/lib/oozie/oozie-sharelib-yarn.tar.gz - ln -s /usr/lib/oozie/oozie-sharelib-mr1 /usr/lib/oozie/oozie-sharelib-mr1.tar.gz - ln -s /usr/lib/oozie/oozie-sharelib-yarn.tar.gz /usr/lib/oozie/oozie-sharelib.tar.gz - fi - ;; - 5.11) - wget -O $dest $HADOOP_OPENSTACK_5_11_x_URL - if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then - # Create links to keep backward version support. - ln -s /usr/lib/oozie/oozie-sharelib-yarn /usr/lib/oozie/oozie-sharelib-yarn.tar.gz - ln -s /usr/lib/oozie/oozie-sharelib-mr1 /usr/lib/oozie/oozie-sharelib-mr1.tar.gz - ln -s /usr/lib/oozie/oozie-sharelib-yarn.tar.gz /usr/lib/oozie/oozie-sharelib.tar.gz - fi - ;; - *) - echo "Unhandled version $DIB_CDH_VERSION for hadoop-openstack.jar" - exit 1 -esac - -if [ "$DISTRO_NAME" = "ubuntu" ]; then - unset RUNLEVEL -fi diff --git a/elements/hadoop-cloudera/package-installs.yaml b/elements/hadoop-cloudera/package-installs.yaml deleted file mode 100644 index 0a12a18f..00000000 --- a/elements/hadoop-cloudera/package-installs.yaml +++ /dev/null @@ -1,6 +0,0 @@ -wget: - phase: pre-install.d -ntp: -hadoop-hdfs-datanode: -hadoop-hdfs-namenode: -# other packages are installed conditionally in install.d/50-install-cloudera diff --git a/elements/hadoop-cloudera/post-install.d/70-turn-services-off b/elements/hadoop-cloudera/post-install.d/70-turn-services-off deleted file mode 100755 index 62e2874b..00000000 --- a/elements/hadoop-cloudera/post-install.d/70-turn-services-off +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -for i in cloudera-scm-agent \ -cloudera-scm-server \ -cloudera-scm-server-db \ -hadoop-hdfs-datanode \ -hadoop-hdfs-namenode \ -hadoop-hdfs-secondarynamenode \ -hadoop-mapreduce-historyserver \ -hadoop-yarn-nodemanager \ -hadoop-yarn-resourcemanager \ -hive-metastore \ -hive-server2 \ -hive-webhcat-server \ -hue \ -oozie \ -postgresql -do - if [ "$DISTRO_NAME" = "ubuntu" ]; then - update-rc.d -f $i remove - else - chkconfig $i off - fi -done - -if [ $DIB_CDH_VERSION != "5.0" ]; then - for i in impala-catalog \ - impala-server \ - impala-state-store \ - solr-server \ - spark-history-server - do - if [ "$DISTRO_NAME" = "ubuntu" ]; then - update-rc.d -f $i remove - else - chkconfig $i off - fi - done -fi diff --git a/elements/hadoop-cloudera/pre-install.d/10-add-mirror b/elements/hadoop-cloudera/pre-install.d/10-add-mirror deleted file mode 100755 index 7b5c84c8..00000000 --- a/elements/hadoop-cloudera/pre-install.d/10-add-mirror +++ /dev/null @@ -1,304 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -function add_repo_5_5_0 { - case $DISTRO_NAME in - ubuntu ) - # Add repository with postgresql package (it's dependency of cloudera packages) - # Base image doesn't contain this repo - echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ trusty universe multiverse main' >> /etc/apt/sources.list - - # Cloudera repositories - echo 'deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh trusty-cdh5.5.0 contrib' > /etc/apt/sources.list.d/cdh5.list - echo 'deb-src http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh trusty-cdh5.5.0 contrib' >> /etc/apt/sources.list.d/cdh5.list - - wget -qO - http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh/archive.key | apt-key add - - - echo 'deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm trusty-cm5.5.0 contrib' > /etc/apt/sources.list.d/cm5.list - echo 'deb-src http://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm trusty-cm5.5.0 contrib' >> /etc/apt/sources.list.d/cm5.list - - wget -qO - http://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm/archive.key | apt-key add - - - wget -O /etc/apt/sources.list.d/kms.list http://archive.cloudera.com/navigator-keytrustee5/ubuntu/trusty/amd64/navigator-keytrustee/cloudera.list - wget -qO - http://archive.cloudera.com/navigator-keytrustee5/ubuntu/trusty/amd64/navigator-keytrustee/archive.key | apt-key add - - - # add Kafka repository - echo -e 'deb http://archive.cloudera.com/kafka/ubuntu/trusty/amd64/kafka/ trusty-kafka2.0.2 contrib' >> /etc/apt/sources.list - wget -qO - https://archive.cloudera.com/kafka/ubuntu/trusty/amd64/kafka/archive.key | apt-key add - - - #change repository priority - echo -e 'Package: zookeeper\nPin: origin "archive.cloudera.com"\nPin-Priority: 1001' > /etc/apt/preferences.d/cloudera-pin - - apt-get update - ;; - centos7 | rhel7 ) - releasever=7 - echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo - echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo "baseurl=http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/5.5.0/" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo "gpgkey = http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo - - echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo - echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo - echo "baseurl=http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/5.5.0/" >> /etc/yum.repos.d/cloudera-manager.repo - echo "gpgkey = http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-manager.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo - - echo '[navigator-keytrustee]' > /etc/yum.repos.d/kms.repo - echo "name=Cloudera's Distribution for navigator-Keytrustee, Version 5" >> /etc/yum.repos.d/kms.repo - - RETURN_CODE="$(curl -s -o /dev/null -w "%{http_code}" http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_MINOR_VERSION/)" - if [ "$RETURN_CODE" == "404" ]; then - echo "baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_VERSION/" >> /etc/yum.repos.d/kms.repo - else - echo "baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/kms.repo - fi - - echo "gpgkey = http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/kms.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/kms.repo - - echo "[cloudera-kafka]" > /etc/yum.repos.d/cloudera-kafka.repo - echo "name=Cloudera's Distribution for kafka, Version 2.0.2" >> /etc/yum.repos.d/cloudera-kafka.repo - echo "baseurl=http://archive.cloudera.com/kafka/redhat/$releasever/x86_64/kafka/2.0.2/" >> /etc/yum.repos.d/cloudera-kafka.repo - echo "gpgkey = http://archive.cloudera.com/kafka/redhat/$releasever/x86_64/kafka/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-kafka.repo - echo "gpgcheck = 1" >> /etc/yum.repos.d/cloudera-kafka.repo - - yum clean all - ;; - esac -} - -function add_repo_5_7_x { - case $DISTRO_NAME in - ubuntu ) - # Add repository with postgresql package (it's dependency of cloudera packages) - # Base image doesn't contain this repo - echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ trusty universe multiverse main' >> /etc/apt/sources.list - - # Cloudera repositories - echo "deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh trusty-cdh$DIB_CDH_MINOR_VERSION contrib" > /etc/apt/sources.list.d/cdh5.list - echo "deb-src http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh trusty-cdh$DIB_CDH_MINOR_VERSION contrib" >> /etc/apt/sources.list.d/cdh5.list - - wget -qO - http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh/archive.key | apt-key add - - - echo "deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm trusty-cm$DIB_CDH_MINOR_VERSION contrib" > /etc/apt/sources.list.d/cm5.list - echo "deb-src http://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm trusty-cm$DIB_CDH_MINOR_VERSION contrib" >> /etc/apt/sources.list.d/cm5.list - - wget -qO - http://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm/archive.key | apt-key add - - - wget -O /etc/apt/sources.list.d/kms.list http://archive.cloudera.com/navigator-keytrustee5/ubuntu/trusty/amd64/navigator-keytrustee/cloudera.list - wget -qO - http://archive.cloudera.com/navigator-keytrustee5/ubuntu/trusty/amd64/navigator-keytrustee/archive.key | apt-key add - - - # add Kafka repository - echo -e 'deb http://archive.cloudera.com/kafka/ubuntu/trusty/amd64/kafka/ trusty-kafka2.0.2 contrib' >> /etc/apt/sources.list - wget -qO - https://archive.cloudera.com/kafka/ubuntu/trusty/amd64/kafka/archive.key | apt-key add - - - #change repository priority - echo -e 'Package: zookeeper\nPin: origin "archive.cloudera.com"\nPin-Priority: 1001' > /etc/apt/preferences.d/cloudera-pin - - apt-get update - ;; - centos7 | rhel7 ) - releasever=7 - echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo - echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo "baseurl=http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo "gpgkey = http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo - - echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo - echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo - echo "baseurl=http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/cloudera-manager.repo - echo "gpgkey = http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-manager.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo - - echo '[navigator-keytrustee]' > /etc/yum.repos.d/kms.repo - echo "name=Cloudera's Distribution for navigator-Keytrustee, Version 5" >> /etc/yum.repos.d/kms.repo - - RETURN_CODE="$(curl -s -o /dev/null -w "%{http_code}" http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_MINOR_VERSION/)" - if [ "$RETURN_CODE" == "404" ]; then - echo "baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_VERSION/" >> /etc/yum.repos.d/kms.repo - else - echo "baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/kms.repo - fi - - echo "gpgkey = http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/kms.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/kms.repo - - echo "[cloudera-kafka]" > /etc/yum.repos.d/cloudera-kafka.repo - echo "name=Cloudera's Distribution for kafka, Version 2.0.2" >> /etc/yum.repos.d/cloudera-kafka.repo - echo "baseurl=http://archive.cloudera.com/kafka/redhat/$releasever/x86_64/kafka/2.0.2/" >> /etc/yum.repos.d/cloudera-kafka.repo - echo "gpgkey = http://archive.cloudera.com/kafka/redhat/$releasever/x86_64/kafka/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-kafka.repo - echo "gpgcheck = 1" >> /etc/yum.repos.d/cloudera-kafka.repo - - yum clean all - ;; - esac -} - -function add_repo_5_9_x { - case $DISTRO_NAME in - ubuntu ) - # Add repository with postgresql package (it's dependency of cloudera packages) - # Base image doesn't contain this repo - echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ trusty universe multiverse main' >> /etc/apt/sources.list - - # Cloudera repositories - echo "deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh trusty-cdh$DIB_CDH_MINOR_VERSION contrib" > /etc/apt/sources.list.d/cdh5.list - echo "deb-src http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh trusty-cdh$DIB_CDH_MINOR_VERSION contrib" >> /etc/apt/sources.list.d/cdh5.list - - wget -qO - http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh/archive.key | apt-key add - - - echo "deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm trusty-cm$DIB_CDH_MINOR_VERSION contrib" > /etc/apt/sources.list.d/cm5.list - echo "deb-src http://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm trusty-cm$DIB_CDH_MINOR_VERSION contrib" >> /etc/apt/sources.list.d/cm5.list - - wget -qO - http://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm/archive.key | apt-key add - - - wget -O /etc/apt/sources.list.d/kms.list http://archive.cloudera.com/navigator-keytrustee5/ubuntu/trusty/amd64/navigator-keytrustee/cloudera.list - wget -qO - http://archive.cloudera.com/navigator-keytrustee5/ubuntu/trusty/amd64/navigator-keytrustee/archive.key | apt-key add - - - # add Kafka repository - echo -e 'deb http://archive.cloudera.com/kafka/ubuntu/trusty/amd64/kafka/ trusty-kafka2.0.2 contrib' >> /etc/apt/sources.list - wget -qO - https://archive.cloudera.com/kafka/ubuntu/trusty/amd64/kafka/archive.key | apt-key add - - - #change repository priority - echo -e 'Package: zookeeper\nPin: origin "archive.cloudera.com"\nPin-Priority: 1001' > /etc/apt/preferences.d/cloudera-pin - - apt-get update - ;; - centos7 | rhel7 ) - releasever=7 - echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo - echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo "baseurl=http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo "gpgkey = http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo - - echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo - echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo - echo "baseurl=http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/cloudera-manager.repo - echo "gpgkey = http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-manager.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo - - echo '[navigator-keytrustee]' > /etc/yum.repos.d/kms.repo - echo "name=Cloudera's Distribution for navigator-Keytrustee, Version 5" >> /etc/yum.repos.d/kms.repo - - RETURN_CODE="$(curl -s -o /dev/null -w "%{http_code}" http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_MINOR_VERSION/)" - if [ "$RETURN_CODE" == "404" ]; then - echo "baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_VERSION/" >> /etc/yum.repos.d/kms.repo - else - echo "baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/kms.repo - fi - - echo "gpgkey = http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/kms.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/kms.repo - - echo "[cloudera-kafka]" > /etc/yum.repos.d/cloudera-kafka.repo - echo "name=Cloudera's Distribution for kafka, Version 2.0.2" >> /etc/yum.repos.d/cloudera-kafka.repo - echo "baseurl=http://archive.cloudera.com/kafka/redhat/$releasever/x86_64/kafka/2.0.2/" >> /etc/yum.repos.d/cloudera-kafka.repo - echo "gpgkey = http://archive.cloudera.com/kafka/redhat/$releasever/x86_64/kafka/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-kafka.repo - echo "gpgcheck = 1" >> /etc/yum.repos.d/cloudera-kafka.repo - - yum clean all - ;; - esac -} - -function add_repo_5_11_x { - case $DISTRO_NAME in - ubuntu ) - # Add repository with postgresql package (it's dependency of cloudera packages) - # Base image doesn't contain this repo - echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ xenial universe multiverse main' >> /etc/apt/sources.list - - # Cloudera repositories - echo "deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/xenial/amd64/cdh xenial-cdh$DIB_CDH_MINOR_VERSION contrib" > /etc/apt/sources.list.d/cdh5.list - echo "deb-src http://archive.cloudera.com/cdh5/ubuntu/xenial/amd64/cdh xenial-cdh$DIB_CDH_MINOR_VERSION contrib" >> /etc/apt/sources.list.d/cdh5.list - - wget -qO - http://archive.cloudera.com/cdh5/ubuntu/xenial/amd64/cdh/archive.key | apt-key add - - - echo "deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/xenial/amd64/cm xenial-cm$DIB_CDH_MINOR_VERSION contrib" > /etc/apt/sources.list.d/cm5.list - echo "deb-src http://archive.cloudera.com/cm5/ubuntu/xenial/amd64/cm xenial-cm$DIB_CDH_MINOR_VERSION contrib" >> /etc/apt/sources.list.d/cm5.list - - wget -qO - http://archive.cloudera.com/cm5/ubuntu/xenial/amd64/cm/archive.key | apt-key add - - - wget -O /etc/apt/sources.list.d/kms.list http://archive.cloudera.com/navigator-keytrustee5/ubuntu/xenial/amd64/navigator-keytrustee/cloudera.list - wget -qO - http://archive.cloudera.com/navigator-keytrustee5/ubuntu/xenial/amd64/navigator-keytrustee/archive.key | apt-key add - - - # add Kafka repository - echo -e 'deb http://archive.cloudera.com/kafka/ubuntu/xenial/amd64/kafka/ xenial-kafka2.2.0 contrib' >> /etc/apt/sources.list - wget -qO - https://archive.cloudera.com/kafka/ubuntu/xenial/amd64/kafka/archive.key | apt-key add - - - #change repository priority - echo -e 'Package: zookeeper\nPin: origin "archive.cloudera.com"\nPin-Priority: 1001' > /etc/apt/preferences.d/cloudera-pin - - apt-get update - ;; - centos7 | rhel7 ) - releasever=7 - echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo - echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo "baseurl=http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo "gpgkey = http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo - - echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo - echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo - echo "baseurl=http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/cloudera-manager.repo - echo "gpgkey = http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-manager.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo - - echo '[navigator-keytrustee]' > /etc/yum.repos.d/kms.repo - echo "name=Cloudera's Distribution for navigator-Keytrustee, Version 5" >> /etc/yum.repos.d/kms.repo - - RETURN_CODE="$(curl -s -o /dev/null -w "%{http_code}" http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_MINOR_VERSION/)" - if [ "$RETURN_CODE" == "404" ]; then - echo "baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_VERSION/" >> /etc/yum.repos.d/kms.repo - else - echo "baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/kms.repo - fi - - echo "gpgkey = http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/kms.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/kms.repo - - echo "[cloudera-kafka]" > /etc/yum.repos.d/cloudera-kafka.repo - echo "name=Cloudera's Distribution for kafka, Version 2.2.0" >> /etc/yum.repos.d/cloudera-kafka.repo - echo "baseurl=http://archive.cloudera.com/kafka/redhat/$releasever/x86_64/kafka/2.2.0/" >> /etc/yum.repos.d/cloudera-kafka.repo - echo "gpgkey = http://archive.cloudera.com/kafka/redhat/$releasever/x86_64/kafka/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-kafka.repo - echo "gpgcheck = 1" >> /etc/yum.repos.d/cloudera-kafka.repo - - yum clean all - ;; - esac -} - -DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$DIB_CDH_VERSION.0} - -# Call version-specific script to install the desired version of CDH -case "$DIB_CDH_VERSION" in - 5.5) - echo "Installing CDH Version $DIB_CDH_VERSION..." - add_repo_5_5_0 - ;; - 5.7) - echo "Installing CDH Version $DIB_CDH_MINOR_VERSION..." - add_repo_5_7_x - ;; - 5.9) - echo "Installing CDH Version $DIB_CDH_MINOR_VERSION..." - add_repo_5_9_x - ;; - 5.11) - echo "Installing CDH Version $DIB_CDH_MINOR_VERSION..." - add_repo_5_11_x - ;; - *) - echo "Invalid CDH Version : $DIB_CDH_VERSION specified, exiting install." - exit 1 -esac diff --git a/elements/hadoop-mapr/README.rst b/elements/hadoop-mapr/README.rst deleted file mode 100644 index 1a4534b9..00000000 --- a/elements/hadoop-mapr/README.rst +++ /dev/null @@ -1,44 +0,0 @@ -=========== -hadoop-mapr -=========== - -Creates images with local mirrors of MapR repositories: -`core `_ and -`ecosystem `_. -Installs `OpenJDK `_ and -`Scala `_. - -In order to create the MapR images with ``diskimage-create.sh``, use the -following syntax to select the ``MapR`` plugin: - -.. sourcecode:: bash - - diskimage-create.sh -p mapr [-i ubuntu|centos7|rhel7] [-r 5.1.0 | 5.2.0] - -In order to speed up image creation process you can download archives with MapR -repositories and specify environment variables: -``DIB_MAPR_CORE_DEB_REPO``, ``DIB_MAPR_CORE_RPM_REPO``, -``DIB_MAPR_ECO_DEB_REPO``, ``DIB_MAPR_ECO_RPM_REPO``. - -For example: - -.. sourcecode:: bash - - export DIB_MAPR_CORE_DEB_REPO="file:///mapr-v5.2.0GA.deb.tgz" - export DIB_MAPR_CORE_RPM_REPO="file:///mapr-v5.2.0GA.rpm.tgz" - export DIB_MAPR_ECO_DEB_REPO="http:///mapr-ecosystem.deb.tgz" - export DIB_MAPR_ECO_RPM_REPO="http:///mapr-ecosystem.rpm.tgz" - diskimage-create.sh -p mapr -r 5.2.0 - -Environment Variables ---------------------- - -DIB_MAPR_VERSION - :Required: Yes - :Description: Version of MapR to install. - :Example: ``DIB_MAPR_VERSION=5.2.0`` - -SCALA_VERSION - :Required: No - :Description: Force a specific version of Scala (disable autodetection). - :Example: ``SCALA_VERSION=2.12.2`` diff --git a/elements/hadoop-mapr/element-deps b/elements/hadoop-mapr/element-deps deleted file mode 100644 index 5dcf86aa..00000000 --- a/elements/hadoop-mapr/element-deps +++ /dev/null @@ -1,4 +0,0 @@ -extjs -java -mysql -package-installs diff --git a/elements/hadoop-mapr/environment.d/00-configure-extjs b/elements/hadoop-mapr/environment.d/00-configure-extjs deleted file mode 100644 index 10de1c84..00000000 --- a/elements/hadoop-mapr/environment.d/00-configure-extjs +++ /dev/null @@ -1,2 +0,0 @@ -export EXTJS_DESTINATION_DIR="/opt/mapr-repository/" -export EXTJS_NO_UNPACK=1 diff --git a/elements/hadoop-mapr/install.d/40-mapr-dependencies b/elements/hadoop-mapr/install.d/40-mapr-dependencies deleted file mode 100755 index 0a5d53ac..00000000 --- a/elements/hadoop-mapr/install.d/40-mapr-dependencies +++ /dev/null @@ -1,155 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -echo "START: installing MapR core dependencies" - -if [ "$DISTRO_NAME" = "ubuntu" ]; then - # Required for libicu48 - cat >> /etc/apt/sources.list.d/security_repo.list << EOF -deb http://security.ubuntu.com/ubuntu precise-security main -EOF - # Required for libmysqlclient16 - cat >> /etc/apt/sources.list.d/security_repo.list << EOF -deb http://old-releases.ubuntu.com/ubuntu lucid-security main -EOF - apt-get update - install-packages binutils \ - daemon \ - dpkg-dev \ - dpkg-repack \ - gcc \ - gcc-4.8 \ - gcc-doc \ - gcc-multilib \ - iputils-arping \ - libasan0 \ - libatomic1 \ - libc-dev-bin \ - libc6 \ - libc6-dev \ - libcrypt-passwdmd5-perl \ - libgcc-4.8-dev \ - libgomp1 \ - libgssglue1 \ - libicu48 \ - libitm1 \ - libmysqlclient-dev \ - libmysqlclient16 \ - libmysqlclient18 \ - libnfsidmap2 \ - libquadmath0 \ - libsasl2-dev \ - libsasl2-modules-gssapi-mit \ - libssl0.9.8 \ - libtirpc1 \ - libtsan0 \ - libxslt1.1 \ - linux-libc-dev \ - manpages-dev \ - mtools \ - mysql-common \ - nfs-common \ - open-iscsi \ - openjdk-6-jre \ - rpcbind \ - sdparm \ - syslinux \ - syslinux-common \ - unzip \ - watchdog \ - zlib1g-dev \ - zip -elif [ "$DISTRO_NAME" = "centos7" -o "$DISTRO_NAME" = "rhel7" ]; then - install-packages cdparanoia-libs \ - cups \ - cups-libs \ - createrepo \ - cvs \ - cyrus-sasl-gssapi \ - cyrus-sasl-plain \ - db4-cxx \ - db4-devel \ - foomatic \ - foomatic-db \ - foomatic-db-filesystem \ - foomatic-db-ppds \ - gdbm-devel \ - gettext \ - ghostscript \ - ghostscript-fonts \ - glibc \ - glibc-common \ - glibc-devel \ - glibc-headers \ - gstreamer \ - gstreamer-plugins-base \ - gstreamer-tools \ - hdparm \ - irqbalance \ - iso-codes \ - kernel-headers \ - lcms-libs \ - libXt \ - libXv \ - libXxf86vm \ - libgomp \ - libgssglue \ - libgudev1 \ - libicu \ - libmng \ - liboil \ - libtheora \ - libtirpc \ - libvisual \ - libxslt \ - mesa-dri-drivers \ - mesa-dri-filesystem \ - mesa-dri1-drivers \ - mesa-libGL \ - mesa-libGLU \ - mesa-private-llvm \ - mtools \ - nc \ - numactl \ - openjpeg-libs \ - patch \ - pax \ - perl-CGI \ - perl-ExtUtils-MakeMaker \ - perl-ExtUtils-ParseXS \ - perl-Test-Harness \ - perl-Test-Simple \ - perl-devel \ - phonon-backend-gstreamer \ - poppler \ - poppler-data \ - poppler-utils \ - portreserve \ - qt \ - qt-sqlite \ - qt-x11 \ - qt3 \ - redhat-lsb \ - redhat-lsb-compat \ - redhat-lsb-core \ - redhat-lsb-graphics \ - redhat-lsb-printing \ - rpcbind \ - sdparm \ - soci \ - soci-mysql \ - syslinux \ - syslinux-nonlinux \ - unzip \ - urw-fonts \ - yum-utils \ - xml-common \ - zip -fi - -echo "END: installing MapR core dependencies" diff --git a/elements/hadoop-mapr/install.d/41-scala b/elements/hadoop-mapr/install.d/41-scala deleted file mode 100755 index ea83e91f..00000000 --- a/elements/hadoop-mapr/install.d/41-scala +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -echo "START: installing Scala" - -#Current available version -DEF_VERSION="2.11.6" - -RETURN_CODE="$(curl -s -L -o /dev/null -w "%{http_code}" https://www.scala-lang.org/)" - -if [ "$RETURN_CODE" != "200" ]; then - echo "https://www.scala-lang.org is unreachable" && exit 1 -fi - -if [ -n "${SCALA_VERSION:-}" ]; then - VERSION=${SCALA_VERSION} -elif [ "trusty" == "${DIB_RELEASE:-}" ]; then - # scale >= 2.12 for ubuntu depends on openjdk-8, not available on trusty - VERSION=${DEF_VERSION} -else - VERSION="$(curl -s -L --fail https://www.scala-lang.org| tr -d '\n' | sed 's/^.*[^0-9]\+\([0-9\.\?]\+\)<.\+$/\1/')" - - if [ $? != 0 -o -z "${VERSION}" ]; then - echo "Installing default version $DEF_VERSION" - VERSION=${DEF_VERSION} - fi -fi - -PKG=scala-${VERSION} - -URL="https://downloads.lightbend.com/scala/${VERSION}" - -if [ "$DISTRO_NAME" = "ubuntu" ]; then - wget -N ${URL}/${PKG}.deb - dpkg -i ${PKG}.deb - rm ${PKG}.deb -elif [ "$DISTRO_NAME" = "centos7" -o "$DISTRO_NAME" = "rhel7" ]; then - rpm -Uhv ${URL}/${PKG}.rpm -fi - -echo "END: installing Scala" - diff --git a/elements/hadoop-mapr/install.d/43-mapr-core-repository b/elements/hadoop-mapr/install.d/43-mapr-core-repository deleted file mode 100755 index b23033ec..00000000 --- a/elements/hadoop-mapr/install.d/43-mapr-core-repository +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -source "$(dirname $0)/../resources/package_utils.sh" - -echo "START: installing MapR core repository" - -get_repo_url() { - local repo_url="" - - if [ "$DISTRO_NAME" = "ubuntu" ]; then - repo_url="${DIB_MAPR_CORE_DEB_REPO:-http://package.mapr.com/releases/v${DIB_MAPR_VERSION}/ubuntu/mapr-v${DIB_MAPR_VERSION}GA.deb.tgz}" - elif [ "$DISTRO_NAME" = "centos7" -o "$DISTRO_NAME" = "rhel7" ]; then - repo_url="${DIB_MAPR_CORE_RPM_REPO:-http://package.mapr.com/releases/v${DIB_MAPR_VERSION}/redhat/mapr-v${DIB_MAPR_VERSION}GA.rpm.tgz}" - fi - - echo "$repo_url" -} - -MAPR_REPO_URL="$(get_repo_url)" -MAPR_REPO_DIR="/opt/mapr-repository/core" -MAPR_REPO_NAME="mapr_core" - -echo "Downloading MapR repository archive" -mkdir -p "$MAPR_REPO_DIR" && curl "$MAPR_REPO_URL" | tar -xz -C "$MAPR_REPO_DIR" - -echo "Creating local repository" -create_repo "$MAPR_REPO_DIR" - -echo "Adding MapR repository" -add_local_repo "$MAPR_REPO_NAME" "$MAPR_REPO_DIR" - -echo "END: installing MapR core repository" diff --git a/elements/hadoop-mapr/install.d/44-mapr-eco-repository b/elements/hadoop-mapr/install.d/44-mapr-eco-repository deleted file mode 100755 index 013ab54e..00000000 --- a/elements/hadoop-mapr/install.d/44-mapr-eco-repository +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -VERSIONS_PY="$(dirname $0)/../resources/versions.py" - -source "$(dirname $0)/../resources/package_utils.sh" - -echo "START: installing MapR ecosystem repository" - -get_repo_url() { - local repo_url="" - - if [ "$DISTRO_NAME" = "ubuntu" ]; then - case "$DIB_MAPR_VERSION" in - "5.1.0") - repo_url="${DIB_MAPR_ECO_DEB_REPO:-http://package.mapr.com/releases/ecosystem-5.x/ubuntu binary/}" - ;; - "5.2.0") - repo_url="${DIB_MAPR_ECO_DEB_REPO:-http://package.mapr.com/releases/MEP/MEP-2.0.0/ubuntu/ binary trusty}" - ;; - esac - elif [ "$DISTRO_NAME" = "centos7" -o "$DISTRO_NAME" = "rhel7" ]; then - case "$DIB_MAPR_VERSION" in - "5.1.0") - repo_url="${DIB_MAPR_ECO_RPM_REPO:-http://package.mapr.com/releases/ecosystem-5.x/redhat}" - ;; - "5.2.0") - repo_url="${DIB_MAPR_ECO_RPM_REPO:-http://package.mapr.com/releases/MEP/MEP-2.0.0/redhat}" - ;; - esac - fi - - echo "$repo_url" -} - -RPM_REPO="${DIB_MAPR_ECO_RPM_REPO:-}" -DEB_REPO="${DIB_MAPR_ECO_DEB_REPO:-}" - -MAPR_REPO_DIR="/opt/mapr-repository/ecosystem" -MAPR_REPO_URL="$(get_repo_url)" -MAPR_REPO_NAME="mapr_ecosystem" -MAPR_PKG_GROUPS="$(dirname $0)/../resources/packages.json" - -if [ -f $(dirname $0)/../resources/spec_${DIB_MAPR_VERSION}_${DISTRO_NAME}.json ]; then - MAPR_SPEC="$(dirname $0)/../resources/spec_${DIB_MAPR_VERSION}_${DISTRO_NAME}.json" -else - MAPR_SPEC="$(dirname $0)/../resources/spec_${DIB_MAPR_VERSION}.json" -fi - -echo "Creating local MapR ecosystem repository" -localize_repo "$MAPR_REPO_NAME" "$MAPR_REPO_URL" "$MAPR_PKG_GROUPS" "$MAPR_SPEC" "$MAPR_REPO_DIR" -echo $MAPR_SPEC - -echo "END: installing MapR ecosystem repository" diff --git a/elements/hadoop-mapr/install.d/45-epel-dependencies b/elements/hadoop-mapr/install.d/45-epel-dependencies deleted file mode 100755 index 3f671a54..00000000 --- a/elements/hadoop-mapr/install.d/45-epel-dependencies +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -if [ "$DISTRO_NAME" = "rhel7" -o "$DISTRO_NAME" = "centos7" ]; then - yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm - yum install -y soci soci-mysql - yum remove -y epel-release -fi diff --git a/elements/hadoop-mapr/package-installs.yaml b/elements/hadoop-mapr/package-installs.yaml deleted file mode 100644 index a3aacbce..00000000 --- a/elements/hadoop-mapr/package-installs.yaml +++ /dev/null @@ -1,2 +0,0 @@ -curl: -wget: diff --git a/elements/hadoop-mapr/post-install.d/99-requiretty b/elements/hadoop-mapr/post-install.d/99-requiretty deleted file mode 100755 index c961661c..00000000 --- a/elements/hadoop-mapr/post-install.d/99-requiretty +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -if [ "$DISTRO_NAME" = "centos7" -o "$DISTRO_NAME" = "rhel7" ]; then - sed '/^Defaults requiretty*/ s/^/#/' -i /etc/sudoers -fi - diff --git a/elements/hadoop-mapr/post-install.d/99-update-repository b/elements/hadoop-mapr/post-install.d/99-update-repository deleted file mode 100755 index f39496a4..00000000 --- a/elements/hadoop-mapr/post-install.d/99-update-repository +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -if [ "$DISTRO_NAME" = "ubuntu" ]; then - apt-get update -elif [ "$DISTRO_NAME" = "centos7" -o "$DISTRO_NAME" = "rhel7" ]; then - yum clean all && yum repolist -fi diff --git a/elements/hadoop-mapr/resources/package_utils.sh b/elements/hadoop-mapr/resources/package_utils.sh deleted file mode 100644 index 8ebd2599..00000000 --- a/elements/hadoop-mapr/resources/package_utils.sh +++ /dev/null @@ -1,226 +0,0 @@ -# execute_in_directory -execute_in_directory() { - local directory="$(readlink -f "$1")"; shift - local cmd="$*" - - pushd "$directory" && eval "$cmd" && popd -} - -# get_distro -get_distro() { - echo "$DISTRO_NAME" -} - -# download_apt_package [version] [directory] -download_apt_package() { - local package="$1" - local version="${2:-}" - local directory="${3:-$(pwd)}" - local package_spec="$package${version:+=$version*}" - - execute_in_directory "$directory" apt-get --allow-unauthenticated download "$package_spec" -} - -# download_yum_package [version] [directory] -download_yum_package() { - local package="$1" - local version="${2:-}" - local directory="${3:-$(pwd)}" - local package_spec="$package${version:+-$version*}" - - yumdownloader --destdir "$directory" "$package_spec" -} - -# download_package [version] [directory] [distro] -download_package() { - local package="$1" - local version="${2:-}" - local directory="${3:-$(pwd)}" - local distro="${4:-$(get_distro)}" - - if [[ "$distro" == "ubuntu" ]]; then - download_apt_package "$package" "$version" "$directory" - elif [[ "$distro" == "centos7" || "$distro" == "rhel7" ]]; then - download_yum_package "$package" "$version" "$directory" - fi -} - -# get_packages [version_separator] -get_packages() { - local package_groups_file="$1" - local spec_file="$2" - local version_separator="${3:-:}" - - python "$VERSIONS_PY" --separator "$version_separator" "$package_groups_file" "$spec_file" -} - -# download_packages [directory] [distro] -download_packages() { - local package_groups_file="$1" - local spec_file="$2" - local directory="${3:-$(pwd)}" - local distro="${4:-$(get_distro)}" - local version_separator=":" - - local packages="$(get_packages "$package_groups_file" "$spec_file" "$version_separator")" - for package in $packages; do - IFS="$version_separator" read -ra package_version <<< "$package" - download_package "${package_version[@]}" "$directory" "$distro" - done -} - -# create_apt_repo -create_apt_repo() { - local directory="$(readlink -f "$1")" - local binary_dir="$directory/binary" - local packages_gz="$binary_dir/Packages.gz" - - mkdir -p "$binary_dir" - execute_in_directory "$directory" "dpkg-scanpackages -m . /dev/null | gzip -9c > $packages_gz" -} - -# create_yum_repo -create_yum_repo() { - local directory="$(readlink -f "$1")" - - createrepo "$directory" -} - -# create_repo [distro] -create_repo() { - local directory="$(readlink -f "$1")" - local distro="${2:-$(get_distro)}" - - if [[ "$distro" == "ubuntu" ]]; then - create_apt_repo "$directory" - elif [[ "$distro" == "centos7" || "$distro" == "rhel7" ]]; then - create_yum_repo "$directory" - fi -} - -# add_apt_repo -add_apt_repo() { - local repo_name="$1" - local repo_url="$2" - local repo="deb $repo_url" - local repo_path="/etc/apt/sources.list.d/$repo_name.list" - - echo "$repo" > "$repo_path" && apt-get update -} - -# add_yum_repo -add_yum_repo() { - local repo_name="$1" - local repo_url="$2" - local repo_path="/etc/yum.repos.d/$repo_name.repo" - - cat > "$repo_path" << EOF -[$repo_name] -name=$repo_name -baseurl=$repo_url -enabled=1 -gpgcheck=0 -protect=1 -EOF - yum clean all && rm -rf /var/cache/yum/* && yum check-update -} - -# add_repo [distro] -add_repo() { - local repo_name="$1" - local repo_url="$2" - local distro="${3:-$(get_distro)}" - - if [[ "$distro" == "ubuntu" ]]; then - add_apt_repo "$repo_name" "$repo_url" - elif [[ "$distro" == "centos7" || "$distro" == "rhel7" ]]; then - add_yum_repo "$repo_name" "$repo_url" - fi -} - -# add_local_apt_repo -add_local_apt_repo() { - local repo_name="$1" - local directory="$(readlink -f "$2")" - local repo_url="file:$directory binary/" - - add_apt_repo "$repo_name" "$repo_url" -} - -# add_local_yum_repo -add_local_yum_repo() { - local repo_name="$1" - local directory="$(readlink -f "$2")" - local repo_url="file://$directory" - - add_yum_repo "$repo_name" "$repo_url" -} - -# add_local_repo [distro] -add_local_repo() { - local repo_name="$1" - local directory="$(readlink -f "$2")" - local distro="${3:-$(get_distro)}" - - if [[ "$distro" == "ubuntu" ]]; then - add_local_apt_repo "$repo_name" "$directory" - elif [[ "$distro" == "centos7" || "$distro" == "rhel7" ]]; then - add_local_yum_repo "$repo_name" "$directory" - fi -} - -# remove_apt_repo -remove_apt_repo() { - local repo_name="$1" - local repo_path="/etc/apt/sources.list.d/$repo_name.list" - - rm "$repo_path" && apt-get update -} - -# remove_yum_repo -remove_yum_repo() { - local repo_name="$1" - local repo_path="/etc/yum.repos.d/$repo_name.repo" - - rm "$repo_path" -} - -# remove_repo [distro] -remove_repo() { - local repo_name="$1" - local distro="${2:-$(get_distro)}" - - if [[ "$distro" == "ubuntu" ]]; then - remove_apt_repo "$repo_name" - elif [[ "$distro" == "centos7" || "$distro" == "rhel7" ]]; then - remove_yum_repo "$repo_name" - fi -} - -# create_local_repo -create_local_repo() { - local repo_name="$1" - local repo_url="$2" - local package_groups_file="$3" - local spec_file="$4" - local directory="$5" - - add_repo "$repo_name" "$repo_url" - mkdir -p "$directory" && directory="$(readlink -f "$directory")" - download_packages "$package_groups_file" "$spec_file" "$directory" - remove_repo "$repo_name" - create_repo "$directory" -} - -# localize_repo -localize_repo() { - local repo_name="$1" - local repo_url="$2" - local package_groups_file="$3" - local spec_file="$4" - local directory="$5" - - mkdir -p "$directory" && directory="$(readlink -f "$directory")" - create_local_repo "$repo_name" "$repo_url" "$package_groups_file" "$spec_file" "$directory" - add_local_repo "$repo_name" "$directory" -} diff --git a/elements/hadoop-mapr/resources/packages.json b/elements/hadoop-mapr/resources/packages.json deleted file mode 100644 index b1cb5d0f..00000000 --- a/elements/hadoop-mapr/resources/packages.json +++ /dev/null @@ -1,140 +0,0 @@ -{ - "asynchbase": { - "all": [ - "mapr-asynchbase" - ] - }, - "drill": { - "all": [ - "mapr-drill" - ] - }, - "flume": { - "all": [ - "mapr-flume" - ] - }, - "hbase": { - "all": [ - "mapr-hbase", - "mapr-hbase-internal", - "mapr-hbase-master", - "mapr-hbase-regionserver", - "mapr-hbasethrift", - "mapr-hbase-rest" - ], - "0.98.12": [ - "mapr-hbase", - "mapr-hbase-internal", - "mapr-hbase-master", - "mapr-hbase-regionserver", - "mapr-hbasethrift", - "mapr-libhbase", - "mapr-hbase-rest" - ], - "1.1.1": [ - "mapr-hbase", - "mapr-hbase-internal", - "mapr-hbase-master", - "mapr-hbase-regionserver", - "mapr-hbasethrift", - "mapr-libhbase", - "mapr-hbase-rest" - ] - }, - "hive": { - "all": [ - "mapr-hive", - "mapr-hivemetastore", - "mapr-hiveserver2" - ] - }, - "httpfs": { - "all": [ - "mapr-httpfs" - ] - }, - "hue": { - "all": [ - "mapr-hue", - "mapr-hue-base", - "mapr-hue-livy" - ], - "3.10.0": [ - "mapr-hue", - "mapr-hue-livy" - ] - }, - "impala": { - "all": [ - "mapr-impala", - "mapr-impala-catalog", - "mapr-impala-server", - "mapr-impala-statestore", - "mapr-impala-udf" - ] - }, - "mahout": { - "all": [ - "mapr-mahout" - ] - }, - "oozie": { - "all": [ - "mapr-oozie", - "mapr-oozie-internal" - ] - }, - "pig": { - "all": [ - "mapr-pig" - ] - }, - "sentry": { - "all": [ - "mapr-sentry" - ] - }, - "spark": { - "all": [ - "mapr-spark", - "mapr-spark-historyserver", - "mapr-spark-master" - ] - }, - "sqoop": { - "all": [ - "mapr-sqoop2-client", - "mapr-sqoop2-server" - ] - }, - "storm": { - "all": [ - "mapr-storm", - "mapr-storm-ui", - "mapr-storm-nimbus", - "mapr-storm-supervisor" - ] - }, - "tez": { - "all": [ - "mapr-tez" - ] - }, - "kafka": { - "all": [ - "mapr-kafka" - ] - }, - "kafka-connect": { - "all": [ - "mapr-kafka-connect-hdfs", - "mapr-kafka-connect-jdbc" - ] - }, - "kafka-rest": { - "all": [ - "mapr-kafka-rest" - ] - } -} diff --git a/elements/hadoop-mapr/resources/spec_5.1.0.json b/elements/hadoop-mapr/resources/spec_5.1.0.json deleted file mode 100644 index 11ce5be0..00000000 --- a/elements/hadoop-mapr/resources/spec_5.1.0.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "drill": [ - "1.1.0", - "1.2.0", - "1.4.0" - ], - "flume": [ - "1.5.0", - "1.6.0" - ], - "hbase": [ - "0.98.9", - "0.98.12" - ], - "hive": [ - "0.13", - "1.0", - "1.2" - ], - "httpfs": [ - "1.0" - ], - "hue": [ - "3.8.1", - "3.9.0" - ], - "impala": [ - "1.4.1" - ], - "mahout": [ - "0.10.0" - ], - "oozie": [ - "4.2.0" - ], - "pig": [ - "0.14", - "0.15" - ], - "sqoop": [ - "2.0.0" - ], - "spark": [ - "1.5.2" - ] -} diff --git a/elements/hadoop-mapr/resources/spec_5.2.0.json b/elements/hadoop-mapr/resources/spec_5.2.0.json deleted file mode 100644 index 744971bb..00000000 --- a/elements/hadoop-mapr/resources/spec_5.2.0.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "drill": [ - "1.9.0" - ], - "flume": [ - "1.6.0" - ], - "hbase": [ - "1.1.1" - ], - "hive": [ - "1.2" - ], - "httpfs": [ - "1.0" - ], - "hue": [ - "3.10.0" - ], - "impala": [ - "2.5.0" - ], - "mahout": [ - "0.12.0" - ], - "oozie": [ - "4.2.0" - ], - "pig": [ - "0.16" - ], - "sqoop": [ - "2.0.0" - ], - "spark": [ - "2.0.1" - ], - "sentry": [ - "1.6.0" - ], - "kafka": [ - "0.9.0" - ], - "kafka-connect": [ - "2.0.1" - ], - "kafka-rest": [ - "2.0.1" - ] -} \ No newline at end of file diff --git a/elements/hadoop-mapr/resources/spec_5.2.0_ubuntu.json b/elements/hadoop-mapr/resources/spec_5.2.0_ubuntu.json deleted file mode 100644 index 43270512..00000000 --- a/elements/hadoop-mapr/resources/spec_5.2.0_ubuntu.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "drill": [ - "1.9.0" - ], - "flume": [ - "1.6.0" - ], - "hbase": [ - "1.1.1" - ], - "hive": [ - "1.2" - ], - "httpfs": [ - "1.0" - ], - "hue": [ - "3.10.0" - ], - "mahout": [ - "0.12.0" - ], - "oozie": [ - "4.2.0" - ], - "pig": [ - "0.16" - ], - "sqoop": [ - "2.0.0" - ], - "spark": [ - "2.0.1" - ], - "sentry": [ - "1.6.0" - ], - "kafka": [ - "0.9.0" - ], - "kafka-connect": [ - "2.0.1" - ], - "kafka-rest": [ - "2.0.1" - ] -} diff --git a/elements/hadoop-mapr/resources/versions.py b/elements/hadoop-mapr/resources/versions.py deleted file mode 100644 index 8a5589c8..00000000 --- a/elements/hadoop-mapr/resources/versions.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (c) 2015, MapR Technologies -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -import argparse -import json -import sys - -_GROUP_VERSION_SEPARATOR = "," -_ALL_GROUP_VERSION = "all" - - -def _build_parser(): - parser = argparse.ArgumentParser() - - parser.add_argument("packages", help="path to the packages.json") - parser.add_argument("spec", help="path to the spec.json") - parser.add_argument("--separator", default=":", - help="separator between package name" - " and version in output") - - return parser - - -def _load_json(path): - with open(path) as json_file: - return json.load(json_file) - - -def _version_matches(version, group_version): - for gv in group_version.split(_GROUP_VERSION_SEPARATOR): - if version.startswith(gv): - return True - - return False - - -def _get_packages(version, group_spec): - for group_version in group_spec: - if _version_matches(version, group_version): - return group_spec[group_version] - - return group_spec[_ALL_GROUP_VERSION] - - -def _get_package_versions(spec, package_groups): - return [(package, version) - for pg_name, versions in spec.items() - for version in versions - for package in _get_packages(version, package_groups[pg_name])] - - -parser = _build_parser() - - -def main(args=None): - args = parser.parse_args(args or sys.argv[1:]) - - spec = _load_json(args.spec) - package_groups = _load_json(args.packages) - separator = args.separator - - package_versions = _get_package_versions(spec, package_groups) - package_format = "%s" + separator + "%s\n" - package_versions = map(lambda pv: package_format % pv, package_versions) - - sys.stdout.writelines(package_versions) - - -if __name__ == "__main__": - main() diff --git a/elements/hadoop-mapr/root.d/0-mapr-check-version b/elements/hadoop-mapr/root.d/0-mapr-check-version deleted file mode 100755 index 026b27c9..00000000 --- a/elements/hadoop-mapr/root.d/0-mapr-check-version +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -echo "MapR version ${DIB_MAPR_VERSION}" - -if [ -z ${DIB_MAPR_VERSION} ]; then - echo "DIB_MAPR_VERSION not set.\nExit" - exit 1 -fi diff --git a/elements/hadoop/README.rst b/elements/hadoop/README.rst deleted file mode 100644 index 3d3a0682..00000000 --- a/elements/hadoop/README.rst +++ /dev/null @@ -1,36 +0,0 @@ -====== -hadoop -====== - -Installs Java and Hadoop, configures SSH. - -HOWTO build Hadoop Native Libs ------------------------------- - -- Install: *jdk >= 6*, *maven*, *cmake* and *protobuf >= 2.5.0* - -- Get Hadoop source code: - - .. code:: bash - - wget http://archive.apache.org/dist/hadoop/core/hadoop-2.7.1/hadoop-2.7.1-src.tar.gz - -- Unpack source: - - .. code:: bash - - tar xvf hadoop-2.7.1-src.tar.gz - -- Build Hadoop: - - .. code:: bash - - cd hadoop-2.7.1-src - mvn package -Pdist,native -DskipTests - -- Create tarball with Hadoop Native Libs: - - .. code:: bash - - cd hadoop-dist/target/hadoop-2.7.1/lib - tar -czvf hadoop-native-libs-2.7.1.tar.gz native diff --git a/elements/hadoop/element-deps b/elements/hadoop/element-deps deleted file mode 100644 index 217cbbe3..00000000 --- a/elements/hadoop/element-deps +++ /dev/null @@ -1,5 +0,0 @@ -cache-url -java -package-installs -sahara-version -ssh diff --git a/elements/hadoop/install.d/40-setup-hadoop b/elements/hadoop/install.d/40-setup-hadoop deleted file mode 100755 index e58a2822..00000000 --- a/elements/hadoop/install.d/40-setup-hadoop +++ /dev/null @@ -1,137 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -function install_hadoop_v2 { - case "$DIB_HADOOP_VERSION" in - "2.7.1") - hadoop_native_libs_url="${HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL}" - ;; - "2.7.5") - hadoop_native_libs_url="${HADOOP_V2_7_5_NATIVE_LIBS_DOWNLOAD_URL}" - ;; - "2.8.2") - hadoop_native_libs_url="${HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL}" - ;; - "3.0.1") - hadoop_native_libs_url="${HADOOP_V3_0_1_NATIVE_LIBS_DOWNLOAD_URL}" - ;; - *) - echo "Invalid DIB_HADOOP_VERSION: $DIB_HADOOP_VERSION" - exit 1 - ;; - esac - - package="hadoop-$DIB_HADOOP_VERSION.tar.gz" - - echo "Installing hadoop" - - INSTALL_DIR="/opt" - HADOOP_HOME="/opt/hadoop" - mkdir -p "$INSTALL_DIR" - tar xvf "$tmp_dir/$package" -C "$INSTALL_DIR" - ln -s "$INSTALL_DIR/hadoop-$DIB_HADOOP_VERSION" "$HADOOP_HOME" - chown -R hadoop:hadoop "$INSTALL_DIR/hadoop-$DIB_HADOOP_VERSION" - chown -R hadoop:hadoop "$HADOOP_HOME" - rm -r $tmp_dir - - echo "Inject Hadoop native libs" - rm -r "$HADOOP_HOME/lib/native" - wget "$hadoop_native_libs_url" - native_libs_filename=$(basename "$hadoop_native_libs_url") - tar xvf "$native_libs_filename" -C "$HADOOP_HOME/lib" - rm "$native_libs_filename" - - echo "Pre-configuring Hadoop" - - HADOOP_PID_DIR="/var/run/hadoop" - - cat >> /etc/profile.d/hadoop.sh <> $HADOOP_HOME/etc/hadoop/hadoop-env.sh - - sed -i -e "s,YARN_LOG_DIR=.*,YARN_LOG_DIR=/mnt/log/hadoop/yarn," \ - $HADOOP_HOME/etc/hadoop/yarn-env.sh - echo "source $JAVA_RC" >> $HADOOP_HOME/etc/hadoop/yarn-env.sh - - # enable swiftfs - # Sahara expects the jar to be in both folders, even though only one is - # the "official" classpath. Until images and Sahara become more strictly - # coupled we must maintain this legacy behavior. - ln -s ${HADOOP_HOME}/share/hadoop/tools/lib/hadoop-openstack-${DIB_HADOOP_VERSION}.jar ${HADOOP_HOME}/share/hadoop/common/lib/ - -} - -case "$DISTRO_NAME" in - fedora | ubuntu | rhel7 | centos7 ) - ;; - *) - echo "Unknown distro: $DISTRO_NAME. Exiting." - exit 1 - ;; -esac - -echo "Hadoop setup begins for $DISTRO_NAME" -tmp_dir=/tmp/hadoop - -echo "Creating hadoop user & group" -case "$DISTRO_NAME" in - ubuntu ) - addgroup hadoop - adduser --ingroup hadoop --disabled-password --gecos GECOS hadoop - adduser hadoop sudo - ;; - fedora | rhel7 | centos7 ) - adduser -G adm,wheel hadoop - ;; -esac - -install_hadoop_v2 - -echo "Applying firstboot script" - -RC_SCRIPT_DIR="" -if [ "$DISTRO_NAME" == "ubuntu" ]; then - # File '/etc/rc.local' may not exist - if [ -f "/etc/rc.local" ]; then - mv /etc/rc.local /etc/rc.local.old - fi - RC_SCRIPT_DIR="/etc" -else - # File '/etc/rc.d/rc.local' may not exist - if [ -f "/etc/rc.d/rc.local" ]; then - mv /etc/rc.d/rc.local /etc/rc.d/rc.local.old - fi - RC_SCRIPT_DIR="/etc/rc.d" -fi - -# Make sure that rc-local.service starts after cloud-init.service, -# so that the cloud-init user (centos, ubuntu, cloud-user, ...) -# is available. See also: -# https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/1468103 -# All version of distributions affected by this change use systemd. -mkdir -p /etc/systemd/system/rc-local.service.d/ -cat >/etc/systemd/system/rc-local.service.d/override.conf <> /home/hadoop/.bashrc < " -} - -if [[ $# != 2 ]]; then - usage - exit 1 -fi - -JRE_HOME=$1 -JDK_HOME=$2 - -echo "export JAVA_HOME=$JRE_HOME" >> $JAVA_RC -chmod +x $JAVA_RC - -echo "export PATH=$JRE_HOME/bin:\$PATH" >> $JAVA_BIN_RC -echo "export PATH=$JDK_HOME/bin:\$PATH" >> $JAVA_BIN_RC -chmod +x $JAVA_BIN_RC - -echo "Setting up alternatives for Java" - -case "$DISTRO_NAME" in - ubuntu ) - update-alternatives --install "/usr/bin/java" "java" "$JRE_HOME/bin/java" 1 - update-alternatives --install "/usr/bin/javac" "javac" "$JDK_HOME/bin/javac" 1 - - update-alternatives --set java $JRE_HOME/bin/java - update-alternatives --set javac $JDK_HOME/bin/javac - ;; - fedora | rhel7 | centos7 ) - alternatives --install /usr/bin/java java $JRE_HOME/bin/java 200000 - alternatives --install /usr/bin/javac javac $JDK_HOME/bin/javac 200000 - ;; -esac diff --git a/elements/java/cleanup.d/00-clean-java-vars b/elements/java/cleanup.d/00-clean-java-vars deleted file mode 100755 index 54e048f1..00000000 --- a/elements/java/cleanup.d/00-clean-java-vars +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -unset DIB_JAVA_DISTRO -unset JAVA_BIN_RC -unset JAVA_RC diff --git a/elements/java/environment.d/00-java-rc b/elements/java/environment.d/00-java-rc deleted file mode 100644 index f2ecc947..00000000 --- a/elements/java/environment.d/00-java-rc +++ /dev/null @@ -1,2 +0,0 @@ -export JAVA_RC="/etc/profile.d/99-java.sh" -export JAVA_BIN_RC="/etc/profile.d/98-java-bin.sh" diff --git a/elements/java/pre-install.d/00-check-java b/elements/java/pre-install.d/00-check-java deleted file mode 100755 index 44075584..00000000 --- a/elements/java/pre-install.d/00-check-java +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -if [[ -z $DIB_JAVA_DISTRO ]]; then - echo "Java distribution is not selected." - exit 1 -fi diff --git a/elements/kdc/README.rst b/elements/kdc/README.rst deleted file mode 100644 index 1bcf53d3..00000000 --- a/elements/kdc/README.rst +++ /dev/null @@ -1,13 +0,0 @@ -=== -kdc -=== - -Installs all needed artifacts for KDC server and Kerberos authentication - -Environment Variables ---------------------- - -DIB_UNLIMITED_SECURITY_LOCATION - :Required: No - :Default: https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/ - :Description: Place where UnlimitedSecurity polices are located diff --git a/elements/kdc/element-deps b/elements/kdc/element-deps deleted file mode 100644 index 7076aba9..00000000 --- a/elements/kdc/element-deps +++ /dev/null @@ -1 +0,0 @@ -package-installs diff --git a/elements/kdc/package-installs.yaml b/elements/kdc/package-installs.yaml deleted file mode 100644 index f9279ebc..00000000 --- a/elements/kdc/package-installs.yaml +++ /dev/null @@ -1,7 +0,0 @@ -wget: - phase: pre-install.d -krb-server: -krb-libs: -ldap-utils: -krb-client: -rng-tools: \ No newline at end of file diff --git a/elements/kdc/pkg-map b/elements/kdc/pkg-map deleted file mode 100644 index 45bcd31a..00000000 --- a/elements/kdc/pkg-map +++ /dev/null @@ -1,18 +0,0 @@ -{ - "family": { - "debian": { - "krb-server": "krb5-admin-server", - "krb-libs": "libpam-krb5", - "ldap-utils": "ldap-utils", - "krb-client": "krb5-user", - "rng-tools": "rng-tools" - }, - "redhat": { - "krb-server": "krb5-server", - "ldap-utils": "", - "krb-libs": "krb5-libs", - "krb-client": "krb5-workstation", - "rng-tools": "rng-tools" - } - } -} diff --git a/elements/kdc/pre-install.d/76-unlimited-security-artifacts b/elements/kdc/pre-install.d/76-unlimited-security-artifacts deleted file mode 100755 index 985cabb9..00000000 --- a/elements/kdc/pre-install.d/76-unlimited-security-artifacts +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail -DIB_UNLIMITED_SECURITY_LOCATION="${DIB_UNLIMITED_SECURITY_LOCATION:-https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/}" - -mkdir /tmp/UnlimitedPolicy/ -wget $DIB_UNLIMITED_SECURITY_LOCATION/local_policy.jar -O /tmp/UnlimitedPolicy/local_policy.jar -wget $DIB_UNLIMITED_SECURITY_LOCATION/US_export_policy.jar -O /tmp/UnlimitedPolicy/US_export_policy.jar diff --git a/elements/mysql/README.rst b/elements/mysql/README.rst deleted file mode 100644 index 387c3b8d..00000000 --- a/elements/mysql/README.rst +++ /dev/null @@ -1,8 +0,0 @@ -===== -mysql -===== - -This element sets up the basic components of MySQL. - -It is light version of original MySQL element -(https://github.com/openstack/tripleo-image-elements/tree/master/elements/mysql). diff --git a/elements/mysql/element-deps b/elements/mysql/element-deps deleted file mode 100644 index 7076aba9..00000000 --- a/elements/mysql/element-deps +++ /dev/null @@ -1 +0,0 @@ -package-installs diff --git a/elements/mysql/install.d/10-mysql b/elements/mysql/install.d/10-mysql deleted file mode 100755 index e8281d6d..00000000 --- a/elements/mysql/install.d/10-mysql +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -case "$DISTRO_NAME" in - fedora | rhel7 | centos7 ) - mkdir -p /etc/mysql/conf.d - ;; -esac - -rm -rf /var/lib/mysql/ib_logfile* diff --git a/elements/mysql/install.d/99-disable-mysql-on-boot b/elements/mysql/install.d/99-disable-mysql-on-boot deleted file mode 100755 index cd534a64..00000000 --- a/elements/mysql/install.d/99-disable-mysql-on-boot +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# Disable startup MySQL on boot in Ubuntu -# Service mysqld doesn't start on boot in Fedora and CentOS -# Delete config property 'bind-address' for remote mode (0.0.0.0) - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -if [ "$DISTRO_NAME" = "ubuntu" ]; then - if [ -e "/etc/init/mysql.conf" ]; then - sed -i "s/start on runlevel \[.*\]/start on never runlevel [2345]/g" /etc/init/mysql.conf - # The mysql configuration file name and directory changed in Ubuntu Xenial. - # TODO(shuyingya): delete this if statement when we not support trusty - if [ "$DIB_RELEASE" = "trusty" ]; then - sed -i '/bind-address/d' /etc/mysql/my.cnf - elif [ "$DIB_RELEASE" = "xenial" ]; then - sed -i '/bind-address/d' /etc/mysql/mysql.conf.d/mysqld.cnf - fi - else - update-rc.d -f mysql remove - fi -fi - -# Script for starting mysql - -install -D -g root -o root -m 0755 $(dirname $0)/start-mysql.sh /opt/start-mysql.sh diff --git a/elements/mysql/install.d/start-mysql.sh b/elements/mysql/install.d/start-mysql.sh deleted file mode 100644 index 6a8c5fc9..00000000 --- a/elements/mysql/install.d/start-mysql.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# dib-lint: disable=executable - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -DISTRO="$(lsb_release -is)" - -case "$DISTRO" in - Ubuntu ) - sudo service mysql start - ;; - Fedora | RedHatEnterpriseServer ) - sudo service mysqld start - ;; - CentOS ) - case "$(lsb_release -rs)" in - 7.*) - sudo service mariadb start - ;; - 6.*) - sudo service mysqld start - ;; - esac - ;; -esac diff --git a/elements/mysql/package-installs.yaml b/elements/mysql/package-installs.yaml deleted file mode 100644 index 8f8117d2..00000000 --- a/elements/mysql/package-installs.yaml +++ /dev/null @@ -1,4 +0,0 @@ -mysql_package: -mysql_libs_package: -mysql_server_package: -mysql_java_package: diff --git a/elements/mysql/pkg-map b/elements/mysql/pkg-map deleted file mode 100644 index b286f72a..00000000 --- a/elements/mysql/pkg-map +++ /dev/null @@ -1,46 +0,0 @@ -{ - "distro": { - "fedora": { - "mysql_package": "community-mysql", - "mysql_libs_package": "community-mysql-libs", - "mysql_server_package": "community-mysql-server", - "mysql_java_package": "mysql-connector-java" - }, - "centos7": { - "mysql_package": "mariadb", - "mysql_libs_package": "mariadb-libs", - "mysql_server_package": "mariadb-server", - "mysql_java_package": "mysql-connector-java" - }, - "rhel7": { - "mysql_package": "mariadb", - "mysql_libs_package": "mariadb-libs", - "mysql_server_package": "mariadb-server", - "mysql_java_package": "mysql-connector-java" - } - }, - "release": { - "ubuntu": { - "trusty": { - "mysql_package": "mysql-client-5.5", - "mysql_libs_package": "", - "mysql_server_package": "mysql-server-5.5", - "mysql_java_package": "libmysql-java" - }, - "xenial": { - "mysql_package": "mysql-client-5.7", - "mysql_libs_package": "", - "mysql_server_package": "mysql-server-5.7", - "mysql_java_package": "libmysql-java" - } - } - }, - "family": { - "redhat": { - "mysql_package": "mysql", - "mysql_libs_package": "mysql-libs", - "mysql_server_package": "mysql-server", - "mysql_java_package": "mysql-connector-java" - } - } -} diff --git a/elements/nc/README.rst b/elements/nc/README.rst deleted file mode 100644 index e25d36da..00000000 --- a/elements/nc/README.rst +++ /dev/null @@ -1,5 +0,0 @@ -== -nc -== - -This element installs nc/netcat. diff --git a/elements/nc/elements-deps b/elements/nc/elements-deps deleted file mode 100644 index 7076aba9..00000000 --- a/elements/nc/elements-deps +++ /dev/null @@ -1 +0,0 @@ -package-installs diff --git a/elements/nc/package-installs.yaml b/elements/nc/package-installs.yaml deleted file mode 100644 index ea627711..00000000 --- a/elements/nc/package-installs.yaml +++ /dev/null @@ -1 +0,0 @@ -nc: diff --git a/elements/nc/pkg-map b/elements/nc/pkg-map deleted file mode 100644 index c1d943e0..00000000 --- a/elements/nc/pkg-map +++ /dev/null @@ -1,10 +0,0 @@ -{ - "family": { - "debian": { - "nc": "netcat" - }, - "redhat": { - "nc": "nc" - } - } -} diff --git a/elements/nfs-shares/README.rst b/elements/nfs-shares/README.rst deleted file mode 100644 index 4b4946b2..00000000 --- a/elements/nfs-shares/README.rst +++ /dev/null @@ -1,5 +0,0 @@ -========== -nfs-shares -========== - -This element installs the daemon for the kernel NFS server. diff --git a/elements/nfs-shares/element-deps b/elements/nfs-shares/element-deps deleted file mode 100644 index 73015c24..00000000 --- a/elements/nfs-shares/element-deps +++ /dev/null @@ -1,2 +0,0 @@ -package-installs -pkg-map diff --git a/elements/nfs-shares/package-installs.yaml b/elements/nfs-shares/package-installs.yaml deleted file mode 100644 index 4a8acdbf..00000000 --- a/elements/nfs-shares/package-installs.yaml +++ /dev/null @@ -1 +0,0 @@ -nfs-utilities: diff --git a/elements/nfs-shares/pkg-map b/elements/nfs-shares/pkg-map deleted file mode 100644 index 115baeb9..00000000 --- a/elements/nfs-shares/pkg-map +++ /dev/null @@ -1,10 +0,0 @@ -{ - "family": { - "debian": { - "nfs-utilities": "nfs-common" - }, - "redhat": { - "nfs-utilities": "nfs-utils" - } - } -} diff --git a/elements/ntp/README.rst b/elements/ntp/README.rst deleted file mode 100644 index 99afcd33..00000000 --- a/elements/ntp/README.rst +++ /dev/null @@ -1,5 +0,0 @@ -=== -ntp -=== - -Installs ntp daemon for sync time on VM diff --git a/elements/ntp/elements-deps b/elements/ntp/elements-deps deleted file mode 100644 index 7076aba9..00000000 --- a/elements/ntp/elements-deps +++ /dev/null @@ -1 +0,0 @@ -package-installs diff --git a/elements/ntp/package-installs.yaml b/elements/ntp/package-installs.yaml deleted file mode 100644 index ec81c28e..00000000 --- a/elements/ntp/package-installs.yaml +++ /dev/null @@ -1 +0,0 @@ -ntp: diff --git a/elements/oozie/README.rst b/elements/oozie/README.rst deleted file mode 100644 index 2f028026..00000000 --- a/elements/oozie/README.rst +++ /dev/null @@ -1,5 +0,0 @@ -===== -oozie -===== - -Oozie deployment. diff --git a/elements/oozie/element-deps b/elements/oozie/element-deps deleted file mode 100644 index 619589f1..00000000 --- a/elements/oozie/element-deps +++ /dev/null @@ -1,4 +0,0 @@ -cache-url -extjs -package-installs -sahara-version diff --git a/elements/oozie/environment.d/00-configure-extjs b/elements/oozie/environment.d/00-configure-extjs deleted file mode 100644 index d3be741a..00000000 --- a/elements/oozie/environment.d/00-configure-extjs +++ /dev/null @@ -1,2 +0,0 @@ -export EXTJS_DESTINATION_DIR="/opt/oozie/libtools/" -export EXTJS_NO_UNPACK=1 diff --git a/elements/oozie/install.d/50-setup-oozie b/elements/oozie/install.d/50-setup-oozie deleted file mode 100755 index 7ad6a764..00000000 --- a/elements/oozie/install.d/50-setup-oozie +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -echo "Oozie setup" - -tmp_dir=/tmp/oozie - -if [[ "$DIB_HADOOP_VERSION" == "2.7.1" ]]; then - if [ -z "${OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL:-}" ]; then - OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_7_1_FILE) - else - OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL) - fi -elif [[ "$DIB_HADOOP_VERSION" == "2.7.5" ]]; then - if [ -z "${OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL:-}" ]; then - OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_7_5_FILE) - else - OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL) - fi -elif [[ "$DIB_HADOOP_VERSION" == "2.8.2" ]]; then - if [ -z "${OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL:-}" ]; then - OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_8_2_FILE) - else - OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL) - fi -elif [[ "$DIB_HADOOP_VERSION" == "3.0.1" ]]; then - if [ -z "${OOZIE_HADOOP_V3_0_1_DOWNLOAD_URL:-}" ]; then - OOZIE_FILE=$(basename $OOZIE_HADOOP_V3_0_1_FILE) - else - OOZIE_FILE=$(basename $OOZIE_HADOOP_V3_0_1_DOWNLOAD_URL) - fi -else - echo "Unknown Hadoop version selected. Aborting." - exit 1 -fi - -cd $tmp_dir -tar xzf $OOZIE_FILE -rm $OOZIE_FILE - -if [ $(ls | wc -l) == "1" ]; then - mv $(ls) /opt/oozie -else - mkdir /opt/oozie - mv * /opt/oozie/ -fi - -rm -rf $tmp_dir diff --git a/elements/oozie/package-installs.yaml b/elements/oozie/package-installs.yaml deleted file mode 100644 index 90c87a70..00000000 --- a/elements/oozie/package-installs.yaml +++ /dev/null @@ -1,2 +0,0 @@ -zip: -tar: diff --git a/elements/oozie/post-install.d/90-oozie-war b/elements/oozie/post-install.d/90-oozie-war deleted file mode 100755 index c3da6a4d..00000000 --- a/elements/oozie/post-install.d/90-oozie-war +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -ln -s /usr/share/java/mysql-connector-java.jar /opt/oozie/libtools/mysql.jar -if [ "$DIB_HADOOP_VERSION" = "3.0.1" ]; then - ln -s /usr/share/java/mysql-connector-java.jar /opt/oozie/embedded-oozie-server/dependency/mysql.jar - /opt/oozie/bin/oozie-setup.sh -d /opt/oozie/libtools -else - ln -s /usr/share/java/mysql-connector-java.jar /opt/oozie/oozie-server/lib/mysql.jar - /opt/oozie/bin/oozie-setup.sh prepare-war -d /opt/oozie/libtools -fi - -chown -R hadoop:hadoop /opt/oozie diff --git a/elements/oozie/root.d/0-check-oozie b/elements/oozie/root.d/0-check-oozie deleted file mode 100755 index 51746089..00000000 --- a/elements/oozie/root.d/0-check-oozie +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -if [ "$DIB_HADOOP_VERSION" == "2.7.1" ]; then - if [ -z "${OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL:-}" -a -z "${OOZIE_HADOOP_V2_7_1_FILE:-}" ]; then - echo "OOZIE_HADOOP_V2_7_1_FILE and OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit" - exit 1 - fi -elif [ "$DIB_HADOOP_VERSION" == "2.7.5" ]; then - if [ -z "${OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL:-}" -a -z "${OOZIE_HADOOP_V2_7_5_FILE:-}" ]; then - echo "OOZIE_HADOOP_V2_7_5_FILE and OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit" - exit 1 - fi -elif [ "$DIB_HADOOP_VERSION" == "2.8.2" ]; then - if [ -z "${OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL:-}" -a -z "${OOZIE_HADOOP_V2_8_2_FILE:-}" ]; then - echo "OOZIE_HADOOP_V2_8_2_FILE and OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit" - exit 1 - fi -elif [ "$DIB_HADOOP_VERSION" == "3.0.1" ]; then - if [ -z "${OOZIE_HADOOP_V3_0_1_DOWNLOAD_URL:-}" -a -z "${OOZIE_HADOOP_V3_0_1_FILE:-}" ]; then - echo "OOZIE_HADOOP_V3_0_1_FILE and OOZIE_HADOOP_V3_0_1_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit" - exit 1 - fi -else - echo "Unknown Hadoop version selected. Aborting" - exit 1 -fi diff --git a/elements/oozie/root.d/50-download-oozie b/elements/oozie/root.d/50-download-oozie deleted file mode 100755 index 63da988a..00000000 --- a/elements/oozie/root.d/50-download-oozie +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -tmp_dir=$TARGET_ROOT/tmp/oozie -mkdir -p $tmp_dir - -if [[ "$DIB_HADOOP_VERSION" == "2.7.1" ]]; then - OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL:-}" - OOZIE_FILE="${OOZIE_HADOOP_V2_7_1_FILE:-}" -elif [[ "$DIB_HADOOP_VERSION" == "2.7.5" ]]; then - OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL:-}" - OOZIE_FILE="${OOZIE_HADOOP_V2_7_5_FILE:-}" -elif [[ "$DIB_HADOOP_VERSION" == "2.8.2" ]]; then - OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL:-}" - OOZIE_FILE="${OOZIE_HADOOP_V2_8_2_FILE:-}" -elif [[ "$DIB_HADOOP_VERSION" == "3.0.1" ]]; then - OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V3_0_1_DOWNLOAD_URL:-}" - OOZIE_FILE="${OOZIE_HADOOP_V3_0_1_FILE:-}" -else - echo "Unknown Hadoop version selected. Aborting." - exit 1 -fi - -if [ -n "${OOZIE_DOWNLOAD_URL:-}" ]; then - OOZIE_FILE=$(basename $OOZIE_DOWNLOAD_URL) - cached_tar="$DIB_IMAGE_CACHE/$OOZIE_FILE" - $TMP_HOOKS_PATH/bin/cache-url "$OOZIE_DOWNLOAD_URL" $cached_tar - sudo install -D -g root -o root -m 0755 $cached_tar $tmp_dir -elif [ -n "$OOZIE_FILE" ]; then - sudo install -D -g root -o root -m 0755 $(dirname $0)/$OOZIE_FILE $tmp_dir -fi diff --git a/elements/openjdk/README.rst b/elements/openjdk/README.rst deleted file mode 100644 index dfbf28f4..00000000 --- a/elements/openjdk/README.rst +++ /dev/null @@ -1,5 +0,0 @@ -======= -openjdk -======= - -This element installs the correct version of OpenJDK. diff --git a/elements/openjdk/element-deps b/elements/openjdk/element-deps deleted file mode 100644 index 1daac749..00000000 --- a/elements/openjdk/element-deps +++ /dev/null @@ -1,2 +0,0 @@ -java -pkg-map diff --git a/elements/openjdk/environment.d/00-set-java-distro b/elements/openjdk/environment.d/00-set-java-distro deleted file mode 100644 index 168ebf31..00000000 --- a/elements/openjdk/environment.d/00-set-java-distro +++ /dev/null @@ -1 +0,0 @@ -export DIB_JAVA_DISTRO="openjdk" diff --git a/elements/openjdk/install.d/33-java b/elements/openjdk/install.d/33-java deleted file mode 100755 index 26920694..00000000 --- a/elements/openjdk/install.d/33-java +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -echo "Installing OpenJDK" - -default_jdk_version_for_distro () { - case "$DISTRO_NAME" in - fedora ) - if [ $(lsb_release -rs) -ge '21' ]; then - echo "8" - else - echo "7" - fi - ;; - centos7 | rhel7 ) - echo "8" - ;; - ubuntu ) - if [ "$DIB_RELEASE" = "trusty" ]; then - echo "7" - elif [ "$DIB_RELEASE" = "xenial" ]; then - echo "8" - fi - ;; - *) - echo "Unknown OpenJDK distro: $DISTRO_NAME" >&2 - exit 1 - ;; - esac -} - -pkg=openjdk-$(default_jdk_version_for_distro) -pkg=$(pkg-map --element openjdk $pkg) -install-packages $pkg - -case "$DISTRO_NAME" in - ubuntu ) - JRE_PATH=$(update-alternatives --list java) - JRE_HOME=${JRE_PATH%/bin/java} - JDK_PATH=$(update-alternatives --list javac) - JDK_HOME=${JDK_PATH%/bin/javac} - ;; - fedora | rhel7 | centos7 ) - # NOTE: rpm -E "%{java_home}" command returns wrong java home - # on CentOS, Fedora and RHEL - JRE_HOME="/usr/lib/jvm/java-openjdk/jre" - JDK_HOME="/usr/lib/jvm/java-openjdk" - ;; -esac - -setup-java-home $JRE_HOME $JDK_HOME - -echo "OpenJDK has been installed" diff --git a/elements/openjdk/pkg-map b/elements/openjdk/pkg-map deleted file mode 100644 index 35ec2ccd..00000000 --- a/elements/openjdk/pkg-map +++ /dev/null @@ -1,12 +0,0 @@ -{ - "family": { - "debian": { - "openjdk-7": "openjdk-7-jdk", - "openjdk-8": "openjdk-8-jdk" - }, - "redhat": { - "openjdk-7": "java-1.7.0-openjdk-devel", - "openjdk-8": "java-1.8.0-openjdk-devel" - } - } -} diff --git a/elements/oracle-java/README.rst b/elements/oracle-java/README.rst deleted file mode 100644 index bb62cbe4..00000000 --- a/elements/oracle-java/README.rst +++ /dev/null @@ -1,21 +0,0 @@ -=========== -oracle-java -=========== - -This element installs a Java Virtual Machine into the image. There are -three options for selecting what version of the JVM is installed - - -0. Provide no input, the package manager in your image will be used to -install the natively packaged JVM - -1. Provide JAVA_FILE via the environment, it should be a .tar.gz or -.bin and will be installed under JAVA_TARGET_LOCATION, which will default -to "/usr/java" (see 3. for more information on JAVA_TARGET_LOCATION). - -2. Provide JAVA_DOWNLOAD_URL via the environment, it should be a url -pointing to a file that will be placed in JAVA_FILE (see 1.) - -3. In addition to selecting the JDK to install, this element can be -configured to install to a specific location. Set the JAVA_TARGET_LOCATION -variable in order to customize the top-level directory that will -contain the JDK install. By default, this variable is set to "/usr/java". diff --git a/elements/oracle-java/element-deps b/elements/oracle-java/element-deps deleted file mode 100644 index f3d360b1..00000000 --- a/elements/oracle-java/element-deps +++ /dev/null @@ -1 +0,0 @@ -java diff --git a/elements/oracle-java/environment.d/00-set-java-distro b/elements/oracle-java/environment.d/00-set-java-distro deleted file mode 100644 index ff5585f4..00000000 --- a/elements/oracle-java/environment.d/00-set-java-distro +++ /dev/null @@ -1 +0,0 @@ -export DIB_JAVA_DISTRO="oracle-java" diff --git a/elements/oracle-java/install.d/33-java b/elements/oracle-java/install.d/33-java deleted file mode 100755 index d89514d9..00000000 --- a/elements/oracle-java/install.d/33-java +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -echo "Begin: installation of Java" - -# NOTE: $(dirname $0) is read-only, use space under $TARGET_ROOT -JAVA_LOCATION=${JAVA_TARGET_LOCATION:-"/usr/java"} -JAVA_NAME="oracle-jdk" -JAVA_HOME=$JAVA_LOCATION/$JAVA_NAME -JAVA_DOWNLOAD_URL=${JAVA_DOWNLOAD_URL:-"http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-linux-x64.tar.gz"} - -mkdir -p $JAVA_LOCATION - -if [ -n "$JAVA_DOWNLOAD_URL" ]; then - install-packages wget - JAVA_FILE=$(basename $JAVA_DOWNLOAD_URL) - wget --no-check-certificate --no-cookies -c \ - --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" \ - -O $JAVA_LOCATION/$JAVA_FILE $JAVA_DOWNLOAD_URL -elif [ -n "$JAVA_FILE" ]; then - install -D -g root -o root -m 0755 $(dirname $0)/$JAVA_FILE $JAVA_LOCATION -fi - -cd $JAVA_LOCATION - -echo "Decompressing Java archive" -echo -e "\n" | tar -zxf $JAVA_FILE -echo "Setting up $JAVA_NAME" -chown -R root:root $JAVA_LOCATION -JAVA_DIR=`ls -1 $JAVA_LOCATION | grep -v tar.gz` -ln -s $JAVA_LOCATION/$JAVA_DIR $JAVA_HOME - -setup-java-home $JAVA_HOME $JAVA_HOME - -rm $JAVA_FILE - -echo "End: installation of Java" diff --git a/elements/root-passwd/README.rst b/elements/root-passwd/README.rst deleted file mode 100644 index 4ce6aefc..00000000 --- a/elements/root-passwd/README.rst +++ /dev/null @@ -1,15 +0,0 @@ -=========== -root-passwd -=========== - -Assign a password to root. - -This is useful when booting outside of a cloud environment (e.g. manually via -kvm). - -Environment Variables ---------------------- - -DIB_PASSWORD - :Required: Yes - :Description: The password for the root user. diff --git a/elements/root-passwd/element-deps b/elements/root-passwd/element-deps deleted file mode 100644 index 7076aba9..00000000 --- a/elements/root-passwd/element-deps +++ /dev/null @@ -1 +0,0 @@ -package-installs diff --git a/elements/root-passwd/package-installs.yaml b/elements/root-passwd/package-installs.yaml deleted file mode 100644 index b520b92d..00000000 --- a/elements/root-passwd/package-installs.yaml +++ /dev/null @@ -1,6 +0,0 @@ -augeas-tools: - phase: post-install.d -openssh-server: - phase: post-install.d -openssh-client: - phase: post-install.d diff --git a/elements/root-passwd/post-install.d/99-setup b/elements/root-passwd/post-install.d/99-setup deleted file mode 100755 index 38053199..00000000 --- a/elements/root-passwd/post-install.d/99-setup +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -if [ -z "$DIB_PASSWORD" ]; then - echo "Error during setup password for root" - exit 1 -fi -sed -i "s/disable_root: true/disable_root: false/" /etc/cloud/cloud.cfg -augtool -s set /files/etc/ssh/sshd_config/PasswordAuthentication yes -augtool -s set /files/etc/ssh/sshd_config/PermitRootLogin yes -augtool -s set /files/etc/ssh/ssh_config/PasswordAuthentication yes -echo -e "$DIB_PASSWORD\n$DIB_PASSWORD\n" | passwd diff --git a/elements/s3_hadoop/README.rst b/elements/s3_hadoop/README.rst deleted file mode 100644 index 4280b160..00000000 --- a/elements/s3_hadoop/README.rst +++ /dev/null @@ -1,10 +0,0 @@ -========= -s3_hadoop -========= - -Copy the Hadoop S3 connector libraries into the Hadoop and Spark classpaths. - -Environment Variables ---------------------- - -None. diff --git a/elements/s3_hadoop/element-deps b/elements/s3_hadoop/element-deps deleted file mode 100644 index 7076aba9..00000000 --- a/elements/s3_hadoop/element-deps +++ /dev/null @@ -1 +0,0 @@ -package-installs diff --git a/elements/s3_hadoop/package-installs.yaml b/elements/s3_hadoop/package-installs.yaml deleted file mode 100644 index cc77790e..00000000 --- a/elements/s3_hadoop/package-installs.yaml +++ /dev/null @@ -1,2 +0,0 @@ -wget: - phase: post-install.d diff --git a/elements/s3_hadoop/post-install.d/89-manipulate-s3 b/elements/s3_hadoop/post-install.d/89-manipulate-s3 deleted file mode 100755 index 3ceda808..00000000 --- a/elements/s3_hadoop/post-install.d/89-manipulate-s3 +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - - -case "$plugin_type" in - "vanilla" | "spark" ) - ;; - "cloudera" ) - echo -n "The s3_hadoop element is not supported on CDH," - echo " because the relevant libraries need no manipulation." - # NOTE: actually the above statement is only true on CDH>=5.9 - exit 1 - ;; - *) - # TODO: Investigate if some changes are in fact needed for HDP, MapR - echo "The s3_hadoop element is only relevant to Vanilla and Spark." - exit 1 -esac - -SPARK_JARS_DIR_PATH="/opt/spark/jars" -HADOOP_TOOLS_DIR_PATH="/opt/hadoop/share/hadoop/tools/lib" -HADOOP_COMMON_DIR_PATH="/opt/hadoop/share/hadoop/common/lib" - -if [ "$plugin_type" = "vanilla" ]; then - if [ "$DIB_HADOOP_VERSION" = "2.7.1" -o "$DIB_HADOOP_VERSION" = "2.7.5" ]; then - # These versions need a patched hadoop-aws jar - wget https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-aws-$DIB_HADOOP_VERSION.jar -O $HADOOP_TOOLS_DIR_PATH/hadoop-aws-$DIB_HADOOP_VERSION.jar - fi - - # NOTE: It's easier just to copy, than to mess with YARN - cp $HADOOP_TOOLS_DIR_PATH/*aws*.jar $HADOOP_COMMON_DIR_PATH - if [ "$DIB_HADOOP_VERSION" = "2.7.1" -o "$DIB_HADOOP_VERSION" = "2.7.5" -o "$DIB_HADOOP_VERSION" = "2.8.2" ]; then - # Hadoop-aws older than 2.9.0 needs these too - cp $HADOOP_TOOLS_DIR_PATH/joda-time*.jar $HADOOP_COMMON_DIR_PATH - # The following jars are also on-disk, but under the wrong namespace - wget https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-core/2.5.3/jackson-core-2.5.3.jar -O $HADOOP_COMMON_DIR_PATH/jackson-core.jar - wget https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-databind/2.5.3/jackson-databind-2.5.3.jar -O $HADOOP_COMMON_DIR_PATH/jackson-databind.jar - wget https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-annotations/2.5.3/jackson-annotations-2.5.3.jar -O $HADOOP_COMMON_DIR_PATH/jackson-annotations.jar - fi -fi - -# For both Spark and Vanilla plugins: -# (The s3a driver in hadoop-aws 2.6.5 is too buggy to be redeemed) -if [ "$SPARK_HADOOP_DL" != "hadoop2.6" ]; then - # The hadoop-aws and aws-java-sdk libraries are missing here, but we - # cannot copy them from the Hadoop folder on-disk due to - # version/patching issues - wget https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-aws-2.7.3.jar -O $SPARK_JARS_DIR_PATH/hadoop-aws.jar - wget https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk/1.7.4/aws-java-sdk-1.7.4.jar -O $SPARK_JARS_DIR_PATH/aws-java-sdk.jar -fi diff --git a/elements/sahara-version/README.rst b/elements/sahara-version/README.rst deleted file mode 100644 index ec76f903..00000000 --- a/elements/sahara-version/README.rst +++ /dev/null @@ -1,5 +0,0 @@ -============== -sahara-version -============== - -This element saves the version of a few components in a file. diff --git a/elements/sahara-version/root.d/90-sahara-version b/elements/sahara-version/root.d/90-sahara-version deleted file mode 100755 index 231e7df9..00000000 --- a/elements/sahara-version/root.d/90-sahara-version +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -function _get_version_package { - if type -p pbr > /dev/null; then - local version=$(pbr info $1 | awk '{print $2}') - echo "${version}" - fi -} - -cat << EOF | sudo tee "${TARGET_ROOT}/etc/sahara-extra.version" -sahara-image-elements version: $(_get_version_package "sahara-image-elements") -dib-utils version: $(_get_version_package "dib-utils") -diskimage-builder version: $(_get_version_package "diskimage-builder") -EOF diff --git a/elements/spark/README.rst b/elements/spark/README.rst deleted file mode 100644 index 6786ecc6..00000000 --- a/elements/spark/README.rst +++ /dev/null @@ -1,48 +0,0 @@ -===== -spark -===== - -Installs Spark. Requires Hadoop. - -This element will install Spark. It tries to guess the -correct file to download based on the ``DIB_SPARK_VERSION`` and -``DIB_CDH_VERSION``, but this behaviour can be overridden by using -``SPARK_DOWNLOAD_URL`` to specify a download URL for a pre-built -Spark tar.gz file. -See http://spark.apache.org/downloads.html for more download options. - -Versions --------- - -This element is able to generate images containing any valid Spark version, -compiled against one version of Hadoop HDFS libraries. - -Only some combinations of Spark and Hadoop versions are possible, depending on -the availability of a pre-compiled binary and only few of them are tested with -the Sahara Spark plugin. - -The ``diskimage-create.sh`` script will use tested defaults. Those defaults -generate an image supported by the Sahara Spark plugin. Other combinations -should be used only for evaluation or testing purposes. Refer to the Sahara -Spark plugin wiki page (https://wiki.openstack.org/wiki/Sahara/SparkPlugin) -for more information about tested and supported versions. - -Environment Variables ---------------------- - -DIB_SPARK_VERSION - :Required: Yes, if ``SPARK_DOWNLOAD_URL`` is not set. - :Description: Version of the Spark package to download. - :Example: ``DIB_SPARK_VERSION=1.3.1`` - -DIB_CDH_VERSION - :Required: Required only for images for Spark Plugin and - if ``SPARK_DOWNLOAD_URL`` is not set. - :Description: Version of the CDH platform to use for Hadoop compatibility. - CDH version 5.3 is known to work well. - :Example: ``DIB_CDH_VERSION=5.3`` - -SPARK_DOWNLOAD_URL - :Required: No, if set overrides ``DIB_CDH_VERSION`` and ``DIB_SPARK_VERSION`` - :Description: Download URL of a tgz Spark package to override the automatic - selection from the Apache repositories. diff --git a/elements/spark/element-deps b/elements/spark/element-deps deleted file mode 100644 index 0e0a242d..00000000 --- a/elements/spark/element-deps +++ /dev/null @@ -1,2 +0,0 @@ -cache-url -package-installs diff --git a/elements/spark/install.d/60-spark b/elements/spark/install.d/60-spark deleted file mode 100755 index adfcb8ef..00000000 --- a/elements/spark/install.d/60-spark +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# This script installs Spark -# More documentation in the README.md file - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -tmp_dir=/tmp/spark -pushd $tmp_dir - - -echo "Extracting SPARK" -spark_url=$(cat spark_url.txt) -spark_file=$(basename $spark_url) -extract_folder=$(tar tzf $spark_file | sed -e 's@/.*@@' | uniq) -echo "Decompressing Spark..." -tar xzf $spark_file -rm $spark_file - -echo "Moving SPARK to /opt/" -# Placing spark in /opt/spark -mv $extract_folder /opt/spark/ -mv spark_url.txt /opt/spark/ - -popd -rm -Rf $tmp_dir diff --git a/elements/spark/package-installs.yaml b/elements/spark/package-installs.yaml deleted file mode 100644 index 56fd8234..00000000 --- a/elements/spark/package-installs.yaml +++ /dev/null @@ -1 +0,0 @@ -tar: diff --git a/elements/spark/post-install.d/20-spark b/elements/spark/post-install.d/20-spark deleted file mode 100755 index 48db9a54..00000000 --- a/elements/spark/post-install.d/20-spark +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -# We have to chown the Spark directories to make it usable for the default user -# Unfortunately the Ubuntu user does not exists when DIB is creating the image -# and we need to execute this code the first time the VM boots. - -firstboot_script_name="/opt/spark/firstboot.sh" -if [ "$DISTRO_NAME" == "ubuntu" ]; then - sed -i -e "s,^exit 0$,[ -f $firstboot_script_name ] \&\& sh $firstboot_script_name; exit 0," /etc/rc.local - if [ "$plugin_type" == "vanilla" ]; then - user_and_group_names="hadoop:hadoop" - elif [ "$plugin_type" == "spark" ]; then - user_and_group_names="ubuntu:ubuntu" - fi -else - sed -i -e "s,^exit 0$,[ -f $firstboot_script_name ] \&\& sh $firstboot_script_name; exit 0," /etc/rc.d/rc.local - user_and_group_names="hadoop:hadoop" -fi - -cat >> $firstboot_script_name < $tmp_dir/spark_url.txt diff --git a/elements/ssh/README.rst b/elements/ssh/README.rst deleted file mode 100644 index 4d4b451a..00000000 --- a/elements/ssh/README.rst +++ /dev/null @@ -1,6 +0,0 @@ -=== -ssh -=== - -This element installs an SSH server then configures it to be suitable -for use with Sahara. diff --git a/elements/ssh/element-deps b/elements/ssh/element-deps deleted file mode 100644 index 7076aba9..00000000 --- a/elements/ssh/element-deps +++ /dev/null @@ -1 +0,0 @@ -package-installs diff --git a/elements/ssh/install.d/33-ssh b/elements/ssh/install.d/33-ssh deleted file mode 100755 index 5582c6e5..00000000 --- a/elements/ssh/install.d/33-ssh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -ue -set -o pipefail -echo "Begin: install and configure SSH" - -case "$DISTRO_NAME" in - fedora ) - # the main config file is empty in F25+ - SSH_CONFIG_FILE="/etc/ssh/ssh_config.d/05-redhat.conf" - ;; - * ) - SSH_CONFIG_FILE="/etc/ssh/ssh_config" - ;; -esac - -augtool <> /opt/zookeeper/firstboot.sh <=2.0.0 # Apache-2.0 - -dib-utils>=0.0.8 # Apache-2.0 -diskimage-builder!=1.6.0,!=1.7.0,!=1.7.1,>=1.1.2 # Apache-2.0 diff --git a/roles/build-sahara-images-dib/README.rst b/roles/build-sahara-images-dib/README.rst deleted file mode 100644 index b5d6c51e..00000000 --- a/roles/build-sahara-images-dib/README.rst +++ /dev/null @@ -1,13 +0,0 @@ -Build Sahara Images with DIB - -**Role Variables** - -.. zuul:rolevar:: sahara_build_directory - :default: /var/tmp/sahara-image-build - - The base directory used for the build process. - -.. zuul:rolevar:: sahara_plugin - :default: vanilla - - The plugin whose images will be built. diff --git a/roles/build-sahara-images-dib/defaults/main.yaml b/roles/build-sahara-images-dib/defaults/main.yaml deleted file mode 100644 index 2a51014b..00000000 --- a/roles/build-sahara-images-dib/defaults/main.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -sahara_image_elements_src_dir: "{{ zuul.projects['opendev.org/openstack/sahara-image-elements'].src_dir }}" -sahara_build_directory: /var/tmp/sahara-image-build -sahara_plugin: vanilla -sahara_plugin_version: '' -sahara_plugin_distro: '' -tox_executable: 'tox' diff --git a/roles/build-sahara-images-dib/tasks/main.yaml b/roles/build-sahara-images-dib/tasks/main.yaml deleted file mode 100644 index 2bd3c2b4..00000000 --- a/roles/build-sahara-images-dib/tasks/main.yaml +++ /dev/null @@ -1,4 +0,0 @@ -- name: Build all the images for the selected plugin - command: ./tools/gate/build-images "{{ sahara_plugin }}" "{{ sahara_plugin_version }}" "{{ sahara_plugin_distro }}" "{{ tox_executable }}" - args: - chdir: "{{ sahara_image_elements_src_dir }}" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index e5117ad1..00000000 --- a/setup.cfg +++ /dev/null @@ -1,22 +0,0 @@ -[metadata] -name = sahara-image-elements -summary = Image elements for Sahara -description_file = README.rst -license = Apache Software License -python_requires = >=3.6 -classifiers = - Intended Audience :: Information Technology - Intended Audience :: System Administrators - License :: OSI Approved :: Apache Software License - Operating System :: POSIX :: Linux -author = OpenStack -author_email = openstack-discuss@lists.openstack.org -home_page = https://docs.openstack.org/sahara/latest - -[files] -scripts = - bin/sahara-image-create - diskimage-create/diskimage-create.sh - -data_files = - share/sahara-elements/elements = elements/* diff --git a/setup.py b/setup.py deleted file mode 100644 index 5019c359..00000000 --- a/setup.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import setuptools - -setuptools.setup( - setup_requires=['pbr>=2.0.0'], - pbr=True, - py_modules=[]) diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index 8567cf5a..00000000 --- a/test-requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. - -bashate>=2.1.0 # Apache-2.0 -hacking>=3.0.1,<3.1.0 # Apache-2.0 diff --git a/tools/gate/build-images b/tools/gate/build-images deleted file mode 100755 index e9e344a1..00000000 --- a/tools/gate/build-images +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash -xe - -PLUGIN=$1 -PLUGIN_VERSION=${2:-''} -PLUGIN_DISTRO=${3:-''} -TOX_EXECUTABLE=${4:-'tox'} - -export DIB_MIN_TMPFS=10 - -# build a specific version if requested -if [ -n "${PLUGIN}" ] && [ -n "${PLUGIN_VERSION}" ] && [ -n "${PLUGIN_DISTRO}" ]; then - case ${PLUGIN} in - vanilla|cloudera|ambari) - VERSION_FLAG="-v ${PLUGIN_VERSION}" - ;; - mapr) - VERSION_FLAG="-r ${PLUGIN_VERSION}" - ;; - spark) - VERSION_FLAG="-s ${PLUGIN_VERSION}" - ;; - storm) - VERSION_FLAG="-t ${PLUGIN_VERSION}" - ;; - *) - VERSION_FLAG="" - ;; - esac - ${TOX_EXECUTABLE} -e venv -- sahara-image-create -x -u -p ${PLUGIN} -i ${PLUGIN_DISTRO} ${VERSION_FLAG} - exit -fi - -# build (almost) all versions for a specified plugin -case "$PLUGIN" in - "vanilla") - # temporary don't test building fedora image - for distro in ubuntu centos7; do - # limit to 4 releases - for version in 2.7.1 2.7.5 2.8.2 3.0.1; do - ${TOX_EXECUTABLE} -e venv -- sahara-image-create -x -u -p $PLUGIN -i $distro -v $version - done - done - ;; - "cloudera") - # manually call each distro for cloudera, so we can cleanup - # cache and generated images from the previous run - # (saving space a bit) - for distro in ubuntu centos7; do - # NOTE(vgridnev): Number of versions are growing; testing only 2 latest - # releases of CDH - for version in 5.9 5.11; do - ${TOX_EXECUTABLE} -e venv -- sahara-image-create -x -u -p $PLUGIN -i $distro -v $version - find . -maxdepth 1 -name '*cloudera*.qcow2' -delete - sudo rm -rf ~/.cache/image-create - done - done - ;; - *) - ${TOX_EXECUTABLE} -e venv -- sahara-image-create -x -u -p $PLUGIN - ;; -esac diff --git a/tools/run_bashate.sh b/tools/run_bashate.sh deleted file mode 100755 index e48bc540..00000000 --- a/tools/run_bashate.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -xe - -find ./ -not -wholename \*.tox/\* -and \( -name \*.sh -or -wholename \*.d/\* -and -not -name \*.md -and -not -name \*.rst -and -not -name \*.py \) -print0 | xargs -0 bashate -v -i E006 diff --git a/tox.ini b/tox.ini deleted file mode 100644 index b58af680..00000000 --- a/tox.ini +++ /dev/null @@ -1,46 +0,0 @@ -[tox] -envlist = pep8 -minversion = 3.18.0 -skipsdist = True -ignore_basepython_conflict = True - -[testenv] -basepython = python3 -usedevelop = True -setenv = - VIRTUAL_ENV={envdir} -deps = - -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} - -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -passenv = - http_proxy - https_proxy - no_proxy - -[testenv:bashate] -whitelist_externals = bash -commands = bash -c "find {toxinidir} -not -wholename \*.tox/\* -and \( -name \*.sh -or -wholename \*.d/\* -and -not -name \*.md -and -not -name \*.rst -and -not -name \*.py \) -print0 | xargs -0 bashate -v" -allowlist_externals = bash - -[testenv:pep8] -setenv = - ELEMENTS_DIR=elements -commands = - flake8 {posargs} - {toxinidir}/tools/run_bashate.sh - dib-lint -allowlist_externals = /bin/bash - {toxinidir}/tools/run_bashate.sh - -[testenv:venv] -commands = {posargs} -passenv = * - -[dib-lint] -ignore=safe_sudo - -[flake8] -show-source = true -builtins = _ -exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools