From 9dad58b05acafbf51da0aea7ec71e2c9a35c9c50 Mon Sep 17 00:00:00 2001 From: Ghanshyam Mann Date: Fri, 10 May 2024 17:33:46 -0700 Subject: [PATCH] Retire Sahara: remove repo content Sahara project is retiring - https://review.opendev.org/c/openstack/governance/+/919374 this commit remove the content of this project repo Depends-On: https://review.opendev.org/c/openstack/project-config/+/919376 Change-Id: I46e97281ceab80a0a0ecbebd861f85ef4deaac65 --- .gitignore | 30 - .mailmap | 5 - .zuul.yaml | 73 -- CONTRIBUTING.rst | 19 - HACKING.rst | 12 - LICENSE | 175 ---- MANIFEST.in | 2 - README.rst | 27 +- bin/sahara-image-create | 16 - diskimage-create/README.rst | 137 --- diskimage-create/diskimage-create.sh | 864 ------------------ elements/.gitignore | 1 - elements/README.rst | 69 -- elements/ambari/README.rst | 14 - elements/ambari/element-deps | 3 - elements/ambari/package-installs.yaml | 19 - elements/ambari/pkg-map | 20 - .../ambari/post-install.d/10-disable-ambari | 14 - .../post-install.d/20-disable-ca-certificate | 14 - .../ambari/pre-install.d/90-ambari-repo-conf | 19 - elements/apt-mirror/README.rst | 13 - elements/apt-mirror/finalise.d/99-setup | 9 - elements/apt-mirror/pre-install.d/00-apt | 11 - elements/apt-mirror/root.d/0-check | 12 - elements/centos-mirror/README.rst | 13 - elements/centos-mirror/finalise.d/99-setup | 9 - elements/centos-mirror/pre-install.d/00-yum | 19 - elements/centos-mirror/root.d/0-check | 12 - elements/disable-firewall/README.rst | 11 - .../post-install.d/30-disable-firewalld | 18 - .../post-install.d/31-disable-ip6tables | 25 - .../post-install.d/32-disable-iptables | 25 - elements/extjs/README.rst | 30 - elements/extjs/element-deps | 2 - elements/extjs/install.d/80-install-extjs | 19 - elements/extjs/package-installs.yaml | 1 - elements/extjs/root.d/00-check-extjs | 9 - elements/extjs/root.d/50-download-extjs | 14 - elements/fedora-mirror/README.rst | 13 - elements/fedora-mirror/finalise.d/99-setup | 9 - elements/fedora-mirror/pre-install.d/00-yum | 19 - elements/fedora-mirror/root.d/0-check | 12 - elements/hadoop-cdh/README.rst | 21 - elements/hadoop-cdh/element-deps | 4 - elements/hadoop-cdh/package-installs.yaml | 4 - .../hadoop-cdh/post-install.d/40-setup-hadoop | 34 - .../hadoop-cdh/post-install.d/50-cdh-fixes | 28 - elements/hadoop-cdh/pre-install.d/40-cdh-repo | 29 - elements/hadoop-cdh/root.d/0-check-hadoop-cdh | 28 - elements/hadoop-cloudera/README.rst | 37 - elements/hadoop-cloudera/element-deps | 4 - .../environment.d/00-configure-extjs | 1 - .../environment.d/01-set-java-distro | 1 - elements/hadoop-cloudera/install.d/34-java | 30 - .../install.d/50-install-cloudera | 114 --- .../hadoop-cloudera/package-installs.yaml | 6 - .../post-install.d/70-turn-services-off | 45 - .../pre-install.d/10-add-mirror | 304 ------ elements/hadoop-mapr/README.rst | 44 - elements/hadoop-mapr/element-deps | 4 - .../environment.d/00-configure-extjs | 2 - .../install.d/40-mapr-dependencies | 155 ---- elements/hadoop-mapr/install.d/41-scala | 47 - .../install.d/43-mapr-core-repository | 38 - .../install.d/44-mapr-eco-repository | 59 -- .../install.d/45-epel-dependencies | 13 - elements/hadoop-mapr/package-installs.yaml | 2 - .../hadoop-mapr/post-install.d/99-requiretty | 12 - .../post-install.d/99-update-repository | 13 - .../hadoop-mapr/resources/package_utils.sh | 226 ----- elements/hadoop-mapr/resources/packages.json | 140 --- .../hadoop-mapr/resources/spec_5.1.0.json | 46 - .../hadoop-mapr/resources/spec_5.2.0.json | 50 - .../resources/spec_5.2.0_ubuntu.json | 47 - elements/hadoop-mapr/resources/versions.py | 82 -- .../hadoop-mapr/root.d/0-mapr-check-version | 14 - elements/hadoop/README.rst | 36 - elements/hadoop/element-deps | 5 - elements/hadoop/install.d/40-setup-hadoop | 137 --- elements/hadoop/install.d/firstboot | 71 -- elements/hadoop/package-installs.yaml | 3 - elements/hadoop/pkg-map | 10 - elements/hadoop/root.d/0-check-hadoop | 24 - elements/hadoop/root.d/50-download-hadoop | 33 - elements/hdp-local-mirror/README.rst | 18 - elements/hdp-local-mirror/element-deps | 3 - .../99-set-up-default-apache-dir | 12 - .../install.d/10-enable-apache | 13 - .../hdp-local-mirror/package-installs.yaml | 1 - elements/hdp-local-mirror/pkg-map | 10 - .../root.d/00-check-hdp-local-mirror | 10 - .../source-repository-hdp-stack | 1 - .../source-repository-hdp-utils | 1 - elements/hive/README.rst | 21 - elements/hive/element-deps | 4 - elements/hive/package-installs.yaml | 2 - elements/hive/post-install.d/60-hive | 43 - elements/hive/root.d/0-check-hive | 15 - elements/hive/root.d/50-download-hive | 24 - elements/java/README.rst | 6 - elements/java/bin/setup-java-home | 43 - elements/java/cleanup.d/00-clean-java-vars | 11 - elements/java/environment.d/00-java-rc | 2 - elements/java/pre-install.d/00-check-java | 12 - elements/kdc/README.rst | 13 - elements/kdc/element-deps | 1 - elements/kdc/package-installs.yaml | 7 - elements/kdc/pkg-map | 18 - .../76-unlimited-security-artifacts | 12 - elements/mysql/README.rst | 8 - elements/mysql/element-deps | 1 - elements/mysql/install.d/10-mysql | 15 - .../mysql/install.d/99-disable-mysql-on-boot | 30 - elements/mysql/install.d/start-mysql.sh | 29 - elements/mysql/package-installs.yaml | 4 - elements/mysql/pkg-map | 46 - elements/nc/README.rst | 5 - elements/nc/elements-deps | 1 - elements/nc/package-installs.yaml | 1 - elements/nc/pkg-map | 10 - elements/nfs-shares/README.rst | 5 - elements/nfs-shares/element-deps | 2 - elements/nfs-shares/package-installs.yaml | 1 - elements/nfs-shares/pkg-map | 10 - elements/ntp/README.rst | 5 - elements/ntp/elements-deps | 1 - elements/ntp/package-installs.yaml | 1 - elements/oozie/README.rst | 5 - elements/oozie/element-deps | 4 - .../oozie/environment.d/00-configure-extjs | 2 - elements/oozie/install.d/50-setup-oozie | 53 -- elements/oozie/package-installs.yaml | 2 - elements/oozie/post-install.d/90-oozie-war | 18 - elements/oozie/root.d/0-check-oozie | 32 - elements/oozie/root.d/50-download-oozie | 36 - elements/openjdk/README.rst | 5 - elements/openjdk/element-deps | 2 - .../openjdk/environment.d/00-set-java-distro | 1 - elements/openjdk/install.d/33-java | 58 -- elements/openjdk/pkg-map | 12 - elements/oracle-java/README.rst | 21 - elements/oracle-java/element-deps | 1 - .../environment.d/00-set-java-distro | 1 - elements/oracle-java/install.d/33-java | 42 - elements/root-passwd/README.rst | 15 - elements/root-passwd/element-deps | 1 - elements/root-passwd/package-installs.yaml | 6 - elements/root-passwd/post-install.d/99-setup | 17 - elements/s3_hadoop/README.rst | 10 - elements/s3_hadoop/element-deps | 1 - elements/s3_hadoop/package-installs.yaml | 2 - .../s3_hadoop/post-install.d/89-manipulate-s3 | 54 -- elements/sahara-version/README.rst | 5 - .../sahara-version/root.d/90-sahara-version | 20 - elements/spark/README.rst | 48 - elements/spark/element-deps | 2 - elements/spark/install.d/60-spark | 29 - elements/spark/package-installs.yaml | 1 - elements/spark/post-install.d/20-spark | 32 - elements/spark/root.d/0-check-spark | 36 - elements/spark/root.d/50-download-spark | 41 - elements/ssh/README.rst | 6 - elements/ssh/element-deps | 1 - elements/ssh/install.d/33-ssh | 65 -- elements/ssh/package-installs.yaml | 5 - elements/storm/README.rst | 5 - elements/storm/element-deps | 3 - elements/storm/install.d/60-storm | 68 -- elements/storm/package-installs.yaml | 9 - elements/storm/root.d/0-check-storm | 12 - elements/storm/root.d/50-download-storm | 35 - elements/swift_hadoop/README.rst | 25 - elements/swift_hadoop/element-deps | 1 - elements/swift_hadoop/package-installs.yaml | 2 - .../swift_hadoop/post-install.d/81-add-jar | 62 -- elements/xfs-tools/README.rst | 5 - elements/xfs-tools/package-installs.yaml | 1 - elements/zookeeper/README.rst | 5 - elements/zookeeper/element-deps | 1 - elements/zookeeper/install.d/40-zookeeper | 24 - elements/zookeeper/package-installs.yaml | 1 - .../zookeeper/post-install.d/50-zookeeper | 21 - playbooks/buildimages/run.yaml | 4 - requirements.txt | 12 - roles/build-sahara-images-dib/README.rst | 13 - .../defaults/main.yaml | 7 - roles/build-sahara-images-dib/tasks/main.yaml | 4 - setup.cfg | 22 - setup.py | 21 - test-requirements.txt | 6 - tools/gate/build-images | 61 -- tools/run_bashate.sh | 3 - tox.ini | 46 - 193 files changed, 8 insertions(+), 5482 deletions(-) delete mode 100644 .gitignore delete mode 100644 .mailmap delete mode 100644 .zuul.yaml delete mode 100644 CONTRIBUTING.rst delete mode 100644 HACKING.rst delete mode 100644 LICENSE delete mode 100644 MANIFEST.in delete mode 100755 bin/sahara-image-create delete mode 100644 diskimage-create/README.rst delete mode 100755 diskimage-create/diskimage-create.sh delete mode 100644 elements/.gitignore delete mode 100644 elements/README.rst delete mode 100644 elements/ambari/README.rst delete mode 100644 elements/ambari/element-deps delete mode 100644 elements/ambari/package-installs.yaml delete mode 100644 elements/ambari/pkg-map delete mode 100755 elements/ambari/post-install.d/10-disable-ambari delete mode 100755 elements/ambari/post-install.d/20-disable-ca-certificate delete mode 100755 elements/ambari/pre-install.d/90-ambari-repo-conf delete mode 100644 elements/apt-mirror/README.rst delete mode 100755 elements/apt-mirror/finalise.d/99-setup delete mode 100755 elements/apt-mirror/pre-install.d/00-apt delete mode 100755 elements/apt-mirror/root.d/0-check delete mode 100644 elements/centos-mirror/README.rst delete mode 100755 elements/centos-mirror/finalise.d/99-setup delete mode 100755 elements/centos-mirror/pre-install.d/00-yum delete mode 100755 elements/centos-mirror/root.d/0-check delete mode 100644 elements/disable-firewall/README.rst delete mode 100755 elements/disable-firewall/post-install.d/30-disable-firewalld delete mode 100755 elements/disable-firewall/post-install.d/31-disable-ip6tables delete mode 100755 elements/disable-firewall/post-install.d/32-disable-iptables delete mode 100644 elements/extjs/README.rst delete mode 100644 elements/extjs/element-deps delete mode 100755 elements/extjs/install.d/80-install-extjs delete mode 100644 elements/extjs/package-installs.yaml delete mode 100755 elements/extjs/root.d/00-check-extjs delete mode 100755 elements/extjs/root.d/50-download-extjs delete mode 100644 elements/fedora-mirror/README.rst delete mode 100755 elements/fedora-mirror/finalise.d/99-setup delete mode 100755 elements/fedora-mirror/pre-install.d/00-yum delete mode 100755 elements/fedora-mirror/root.d/0-check delete mode 100644 elements/hadoop-cdh/README.rst delete mode 100644 elements/hadoop-cdh/element-deps delete mode 100644 elements/hadoop-cdh/package-installs.yaml delete mode 100755 elements/hadoop-cdh/post-install.d/40-setup-hadoop delete mode 100755 elements/hadoop-cdh/post-install.d/50-cdh-fixes delete mode 100755 elements/hadoop-cdh/pre-install.d/40-cdh-repo delete mode 100755 elements/hadoop-cdh/root.d/0-check-hadoop-cdh delete mode 100644 elements/hadoop-cloudera/README.rst delete mode 100644 elements/hadoop-cloudera/element-deps delete mode 100644 elements/hadoop-cloudera/environment.d/00-configure-extjs delete mode 100644 elements/hadoop-cloudera/environment.d/01-set-java-distro delete mode 100755 elements/hadoop-cloudera/install.d/34-java delete mode 100755 elements/hadoop-cloudera/install.d/50-install-cloudera delete mode 100644 elements/hadoop-cloudera/package-installs.yaml delete mode 100755 elements/hadoop-cloudera/post-install.d/70-turn-services-off delete mode 100755 elements/hadoop-cloudera/pre-install.d/10-add-mirror delete mode 100644 elements/hadoop-mapr/README.rst delete mode 100644 elements/hadoop-mapr/element-deps delete mode 100644 elements/hadoop-mapr/environment.d/00-configure-extjs delete mode 100755 elements/hadoop-mapr/install.d/40-mapr-dependencies delete mode 100755 elements/hadoop-mapr/install.d/41-scala delete mode 100755 elements/hadoop-mapr/install.d/43-mapr-core-repository delete mode 100755 elements/hadoop-mapr/install.d/44-mapr-eco-repository delete mode 100755 elements/hadoop-mapr/install.d/45-epel-dependencies delete mode 100644 elements/hadoop-mapr/package-installs.yaml delete mode 100755 elements/hadoop-mapr/post-install.d/99-requiretty delete mode 100755 elements/hadoop-mapr/post-install.d/99-update-repository delete mode 100644 elements/hadoop-mapr/resources/package_utils.sh delete mode 100644 elements/hadoop-mapr/resources/packages.json delete mode 100644 elements/hadoop-mapr/resources/spec_5.1.0.json delete mode 100644 elements/hadoop-mapr/resources/spec_5.2.0.json delete mode 100644 elements/hadoop-mapr/resources/spec_5.2.0_ubuntu.json delete mode 100644 elements/hadoop-mapr/resources/versions.py delete mode 100755 elements/hadoop-mapr/root.d/0-mapr-check-version delete mode 100644 elements/hadoop/README.rst delete mode 100644 elements/hadoop/element-deps delete mode 100755 elements/hadoop/install.d/40-setup-hadoop delete mode 100644 elements/hadoop/install.d/firstboot delete mode 100644 elements/hadoop/package-installs.yaml delete mode 100644 elements/hadoop/pkg-map delete mode 100755 elements/hadoop/root.d/0-check-hadoop delete mode 100755 elements/hadoop/root.d/50-download-hadoop delete mode 100644 elements/hdp-local-mirror/README.rst delete mode 100644 elements/hdp-local-mirror/element-deps delete mode 100644 elements/hdp-local-mirror/environment.d/99-set-up-default-apache-dir delete mode 100755 elements/hdp-local-mirror/install.d/10-enable-apache delete mode 100644 elements/hdp-local-mirror/package-installs.yaml delete mode 100644 elements/hdp-local-mirror/pkg-map delete mode 100755 elements/hdp-local-mirror/root.d/00-check-hdp-local-mirror delete mode 100644 elements/hdp-local-mirror/source-repository-hdp-stack delete mode 100644 elements/hdp-local-mirror/source-repository-hdp-utils delete mode 100644 elements/hive/README.rst delete mode 100644 elements/hive/element-deps delete mode 100644 elements/hive/package-installs.yaml delete mode 100755 elements/hive/post-install.d/60-hive delete mode 100755 elements/hive/root.d/0-check-hive delete mode 100755 elements/hive/root.d/50-download-hive delete mode 100644 elements/java/README.rst delete mode 100755 elements/java/bin/setup-java-home delete mode 100755 elements/java/cleanup.d/00-clean-java-vars delete mode 100644 elements/java/environment.d/00-java-rc delete mode 100755 elements/java/pre-install.d/00-check-java delete mode 100644 elements/kdc/README.rst delete mode 100644 elements/kdc/element-deps delete mode 100644 elements/kdc/package-installs.yaml delete mode 100644 elements/kdc/pkg-map delete mode 100755 elements/kdc/pre-install.d/76-unlimited-security-artifacts delete mode 100644 elements/mysql/README.rst delete mode 100644 elements/mysql/element-deps delete mode 100755 elements/mysql/install.d/10-mysql delete mode 100755 elements/mysql/install.d/99-disable-mysql-on-boot delete mode 100644 elements/mysql/install.d/start-mysql.sh delete mode 100644 elements/mysql/package-installs.yaml delete mode 100644 elements/mysql/pkg-map delete mode 100644 elements/nc/README.rst delete mode 100644 elements/nc/elements-deps delete mode 100644 elements/nc/package-installs.yaml delete mode 100644 elements/nc/pkg-map delete mode 100644 elements/nfs-shares/README.rst delete mode 100644 elements/nfs-shares/element-deps delete mode 100644 elements/nfs-shares/package-installs.yaml delete mode 100644 elements/nfs-shares/pkg-map delete mode 100644 elements/ntp/README.rst delete mode 100644 elements/ntp/elements-deps delete mode 100644 elements/ntp/package-installs.yaml delete mode 100644 elements/oozie/README.rst delete mode 100644 elements/oozie/element-deps delete mode 100644 elements/oozie/environment.d/00-configure-extjs delete mode 100755 elements/oozie/install.d/50-setup-oozie delete mode 100644 elements/oozie/package-installs.yaml delete mode 100755 elements/oozie/post-install.d/90-oozie-war delete mode 100755 elements/oozie/root.d/0-check-oozie delete mode 100755 elements/oozie/root.d/50-download-oozie delete mode 100644 elements/openjdk/README.rst delete mode 100644 elements/openjdk/element-deps delete mode 100644 elements/openjdk/environment.d/00-set-java-distro delete mode 100755 elements/openjdk/install.d/33-java delete mode 100644 elements/openjdk/pkg-map delete mode 100644 elements/oracle-java/README.rst delete mode 100644 elements/oracle-java/element-deps delete mode 100644 elements/oracle-java/environment.d/00-set-java-distro delete mode 100755 elements/oracle-java/install.d/33-java delete mode 100644 elements/root-passwd/README.rst delete mode 100644 elements/root-passwd/element-deps delete mode 100644 elements/root-passwd/package-installs.yaml delete mode 100755 elements/root-passwd/post-install.d/99-setup delete mode 100644 elements/s3_hadoop/README.rst delete mode 100644 elements/s3_hadoop/element-deps delete mode 100644 elements/s3_hadoop/package-installs.yaml delete mode 100755 elements/s3_hadoop/post-install.d/89-manipulate-s3 delete mode 100644 elements/sahara-version/README.rst delete mode 100755 elements/sahara-version/root.d/90-sahara-version delete mode 100644 elements/spark/README.rst delete mode 100644 elements/spark/element-deps delete mode 100755 elements/spark/install.d/60-spark delete mode 100644 elements/spark/package-installs.yaml delete mode 100755 elements/spark/post-install.d/20-spark delete mode 100755 elements/spark/root.d/0-check-spark delete mode 100755 elements/spark/root.d/50-download-spark delete mode 100644 elements/ssh/README.rst delete mode 100644 elements/ssh/element-deps delete mode 100755 elements/ssh/install.d/33-ssh delete mode 100644 elements/ssh/package-installs.yaml delete mode 100644 elements/storm/README.rst delete mode 100644 elements/storm/element-deps delete mode 100755 elements/storm/install.d/60-storm delete mode 100644 elements/storm/package-installs.yaml delete mode 100755 elements/storm/root.d/0-check-storm delete mode 100755 elements/storm/root.d/50-download-storm delete mode 100644 elements/swift_hadoop/README.rst delete mode 100644 elements/swift_hadoop/element-deps delete mode 100644 elements/swift_hadoop/package-installs.yaml delete mode 100755 elements/swift_hadoop/post-install.d/81-add-jar delete mode 100644 elements/xfs-tools/README.rst delete mode 100644 elements/xfs-tools/package-installs.yaml delete mode 100644 elements/zookeeper/README.rst delete mode 100644 elements/zookeeper/element-deps delete mode 100755 elements/zookeeper/install.d/40-zookeeper delete mode 100644 elements/zookeeper/package-installs.yaml delete mode 100755 elements/zookeeper/post-install.d/50-zookeeper delete mode 100644 playbooks/buildimages/run.yaml delete mode 100644 requirements.txt delete mode 100644 roles/build-sahara-images-dib/README.rst delete mode 100644 roles/build-sahara-images-dib/defaults/main.yaml delete mode 100644 roles/build-sahara-images-dib/tasks/main.yaml delete mode 100644 setup.cfg delete mode 100644 setup.py delete mode 100644 test-requirements.txt delete mode 100755 tools/gate/build-images delete mode 100755 tools/run_bashate.sh delete mode 100644 tox.ini diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 8b60ef07..00000000 --- a/.gitignore +++ /dev/null @@ -1,30 +0,0 @@ -*.py[co] -*.egg -*.egg-info -dist -build -eggs -parts -var -sdist -develop-eggs -.installed.cfg -pip-log.txt -.tox -*.mo -.mr.developer.cfg -.DS_Store -Thumbs.db -.venv -.idea -out -target -*.iml -*.ipr -*.iws -*.db -.coverage -ChangeLog -AUTHORS -*.qcow2 -diskimage-create.* diff --git a/.mailmap b/.mailmap deleted file mode 100644 index 978490e4..00000000 --- a/.mailmap +++ /dev/null @@ -1,5 +0,0 @@ -# Format is: -# -# -Ivan Berezovskiy -Sergey Lukjanov diff --git a/.zuul.yaml b/.zuul.yaml deleted file mode 100644 index 331f8dd1..00000000 --- a/.zuul.yaml +++ /dev/null @@ -1,73 +0,0 @@ -- project: - templates: - - check-requirements - check: - jobs: - - openstack-tox-pep8 - - sahara-image-elements-buildimages-ambari: - voting: false - - sahara-image-elements-buildimages-cloudera: - voting: false - - sahara-image-elements-buildimages-mapr: - voting: false - - sahara-image-elements-buildimages-plain: - voting: false - - sahara-image-elements-buildimages-spark: - voting: false - - sahara-image-elements-buildimages-storm: - voting: false - - sahara-image-elements-buildimages-vanilla: - voting: false - gate: - jobs: - - openstack-tox-pep8 - post: - jobs: - - publish-openstack-python-branch-tarball - -- job: - name: sahara-image-elements-buildimages-basejob - timeout: 5400 - run: playbooks/buildimages/run.yaml - -- job: - name: sahara-image-elements-buildimages-ambari - parent: sahara-image-elements-buildimages-basejob - vars: - sahara_plugin: ambari - -- job: - name: sahara-image-elements-buildimages-cloudera - parent: sahara-image-elements-buildimages-basejob - vars: - sahara_plugin: cloudera - -- job: - name: sahara-image-elements-buildimages-mapr - parent: sahara-image-elements-buildimages-basejob - vars: - sahara_plugin: mapr - -- job: - name: sahara-image-elements-buildimages-plain - parent: sahara-image-elements-buildimages-basejob - vars: - sahara_plugin: plain - -- job: - name: sahara-image-elements-buildimages-spark - parent: sahara-image-elements-buildimages-basejob - vars: - sahara_plugin: spark - -- job: - name: sahara-image-elements-buildimages-storm - parent: sahara-image-elements-buildimages-basejob - vars: - sahara_plugin: storm - -- job: - name: sahara-image-elements-buildimages-vanilla - parent: sahara-image-elements-buildimages-basejob - vars: - sahara_plugin: vanilla diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst deleted file mode 100644 index 90c4b029..00000000 --- a/CONTRIBUTING.rst +++ /dev/null @@ -1,19 +0,0 @@ -The source repository for this project can be found at: - - https://opendev.org/openstack/sahara-image-elements - -Pull requests submitted through GitHub are not monitored. - -To start contributing to OpenStack, follow the steps in the contribution guide -to set up and use Gerrit: - - https://docs.openstack.org/contributors/code-and-documentation/quick-start.html - -Bugs should be filed on Storyboard: - - https://storyboard.openstack.org/#!/project/openstack/sahara-image-elements - -For more specific information about contributing to this repository, see the -sahara contributor guide: - - https://docs.openstack.org/sahara/latest/contributor/contributing.html diff --git a/HACKING.rst b/HACKING.rst deleted file mode 100644 index 7a93779f..00000000 --- a/HACKING.rst +++ /dev/null @@ -1,12 +0,0 @@ -Sahara Style Commandments -========================== - -- Step 1: Read the OpenStack Style Commandments - https://docs.openstack.org/hacking/latest/ -- Step 2: Read on - -Sahara Specific Commandments ------------------------------ - -None so far - diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 67db8588..00000000 --- a/LICENSE +++ /dev/null @@ -1,175 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index a664f166..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,2 +0,0 @@ -include README.md -graft elements diff --git a/README.rst b/README.rst index 1c8caaa9..4ee2c5f1 100644 --- a/README.rst +++ b/README.rst @@ -1,21 +1,10 @@ -======================== -Team and repository tags -======================== +This project is no longer maintained. -.. image:: https://governance.openstack.org/tc/badges/sahara-image-elements.svg - :target: https://governance.openstack.org/tc/reference/tags/index.html +The contents of this repository are still available in the Git +source code management system. To see the contents of this +repository before it reached its end of life, please check out the +previous commit with "git checkout HEAD^1". -.. Change things from this point on - -Sahara image elements project -============================== - -This repo is a place for Sahara-related for diskimage-builder elements. - -Script for creating Fedora and Ubuntu cloud images with our elements and default parameters. You should only need to run this command: - -.. sourcecode:: bash - - tox -e venv -- sahara-image-create - -Note: More information about script `diskimage-create `_ +For any further questions, please email +openstack-discuss@lists.openstack.org or join #openstack-dev on +OFTC. diff --git a/bin/sahara-image-create b/bin/sahara-image-create deleted file mode 100755 index 0ba1f6e6..00000000 --- a/bin/sahara-image-create +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -eu -set -o pipefail - -SCRIPT_HOME=$(dirname $(readlink -f $0)) -if [ -d $SCRIPT_HOME/../share/sahara-elements ]; then - _PREFIX=$SCRIPT_HOME/../share/sahara-elements -elif [ -d $SCRIPT_HOME/../../../elements ]; then - _PREFIX=$SCRIPT_HOME/../../.. -else - _PREFIX=$SCRIPT_HOME/.. -fi -export ELEMENTS_PATH=$_PREFIX/elements - -diskimage-create.sh $@ diff --git a/diskimage-create/README.rst b/diskimage-create/README.rst deleted file mode 100644 index 33c2659d..00000000 --- a/diskimage-create/README.rst +++ /dev/null @@ -1,137 +0,0 @@ -Diskimage-builder script for creation cloud images -================================================== - -This script builds Ubuntu, Fedora, CentOS cloud images for use in Sahara. -By default the all plugin are targeted, all images will be built. The '-p' -option can be used to select plugin (vanilla, spark, cloudera, storm, mapr, -ambari, or plain). The '-i' option can be used to select image type (ubuntu, -fedora, or centos7). The '-v' option can be used to select hadoop -version. - -For users: - -1. Use your environment (export / setenv) to alter the scripts behavior. -Environment variables the script accepts are 'DIB_HADOOP_VERSION_2_7_1', -'JAVA_DOWNLOAD_URL', 'JAVA_TARGET_LOCATION', 'OOZIE_DOWNLOAD_URL', -'HIVE_VERSION', -'[ubuntu|fedora|centos7]_vanilla_hadoop_2_7_1_image_name', -'ubuntu_spark_image_name', 'ubuntu_storm_image_name', -'ambari_[ubuntu|centos7]_image_name', -'cloudera_[5_7|5_9|5_11]_[ubuntu|centos7]_image_name', -'mapr_[ubuntu|centos7]_image_name', -'plain_[ubuntu|fedora|centos7]_image_name'. - -2. For creating all images just clone this repository and run script. - -.. sourcecode:: bash - - tox -e venv -- sahara-image-create - -3. If you want to use your local mirrors, you should specify http urls for -Fedora, CentOS and Ubuntu mirrors using parameters 'FEDORA_MIRROR', -'CENTOS_MIRROR' and 'UBUNTU_MIRROR' like this: - -.. sourcecode:: bash - - USE_MIRRORS=true FEDORA_MIRROR="url_for_fedora_mirror" \ - CENTOS_MIRROR="url_for_centos_mirror" \ - UBUNTU_MIRROR="url_for_ubuntu_mirror" tox -e venv -- sahara-image-create - -If you want to use your local image, you can specify path of image file using -parameters 'DIB_LOCAL_IMAGE', which defined in project `[diskimage-builder] -(https://github.com/openstack/diskimage-builder)`, like this: - -.. sourcecode:: bash - - DIB_LOCAL_IMAGE="path_of_image" tox -e venv -- sahara-image-create - -NOTE: Do not create all images for all plugins with the same mirrors. -Different plugins use different OS version. - -4. To select which plugin to target use the '-p' commandline option like this: - -.. sourcecode:: bash - - tox -e venv -- sahara-image-create -p [vanilla|spark|cloudera|storm|mapr|ambari|plain] - -5. To select which hadoop version to target use the '-v' commandline option -like this: - -.. sourcecode:: bash - - tox -e venv -- sahara-image-create -v [2.7.1|5.5|5.7|5.9] - -Also, if you are planning to select which ambari version to target use the -'-v' commandline option like this: - -.. sourcecode:: bash - - tox -e venv -- sahara-image-create -v [2.2.0.0|2.2.1.0|2.4.2.0] - -6. To select which operating system to target use the '-i' commandline option -like this: - -.. sourcecode:: bash - - tox -e venv -- sahara-image-create -i [ubuntu|fedora|centos7] - -7. To select which Spark version to target use the '-s' commandline option -like this: - -.. sourcecode:: bash - - tox -e venv -- sahara-image-create -p spark -s [1.3.1|1.6.0|2.1.0|2.2.0] # spark standalone - tox -e venv -- sahara-image-create -p vanilla -v 2.7.1 -s [1.6.0|2.1.0|2.2.0] # spark on vanilla - -8. To select which MapR version to target use the '-r' commandline option like -this: - -.. sourcecode:: bash - - tox -e venv -- sahara-image-create -p mapr -r [5.1.0|5.2.0] - -9. If the host system is missing packages required for sahara-image-create, -the '-u' commandline option will instruct the script to install them without -prompt. - -NOTE for 4, 5, 6: - -For Vanilla you can create ubuntu, fedora and centos7 cloud image with 2.x.x -versions. Use environment variables 'DIB_HADOOP_VERSION_2' to change defaults. -For Spark you can create only ubuntu images, so you shouldn't specify an image -type. The default Spark and HDFS versions included in the build are tested and -known working together with the Sahara Spark plugin, other combinations should -be used only for evaluation or testing purposes. You can select a different -Spark version with commandline option '-s' and Hadoop HDFS version with '-v', -but only Cludera CDH versions are available for now. For Cloudera you can -create ubuntu and centos7 images with preinstalled cloudera hadoop. You -shouldn't specify hadoop version. You can create centos7, ubuntu, fedora images -without hadoop ('plain' image) - -NOTE for CentOS images (for vanilla, ambari, and cloudera plugins): - -Resizing disk space during firstboot on that images fails with errors -(https://storyboard.openstack.org/#!/story/1304100). So, you will get an instance -that will have a small available disk space. To solve this problem we build -images with 10G available disk space as default. If you need in more available -disk space you should export parameter DIB_IMAGE_SIZE: - -.. sourcecode:: bash - - DIB_IMAGE_SIZE=40 tox -e venv -- sahara-image-create -i centos7 - -For all other images parameter DIB_IMAGE_SIZE will be unset. - -`DIB_CLOUD_INIT_DATASOURCES` contains a growing collection of data source -modules and most are enabled by default. This causes cloud-init to query each -data source on first boot. This can cause delays or even boot problems -depending on your environment. You must define `DIB_CLOUD_INIT_DATASOURCES` as -a comma-separated list of valid data sources to limit the data sources that -will be queried for metadata on first boot. - - -For developers: - -If you want to add your element to this repository, you should edit this -script in your commit (you should export variables for your element and add -name of element to variables 'element_sequence'). diff --git a/diskimage-create/diskimage-create.sh b/diskimage-create/diskimage-create.sh deleted file mode 100755 index beb09f06..00000000 --- a/diskimage-create/diskimage-create.sh +++ /dev/null @@ -1,864 +0,0 @@ -#!/bin/bash - -# dummy change -set -e - -export IMAGE_SIZE=$DIB_IMAGE_SIZE -# This will unset parameter DIB_IMAGE_SIZE for Ubuntu and Fedora vanilla images -unset DIB_IMAGE_SIZE - -# DEBUG_MODE is set by the -d flag, debug is enabled if the value is "true" -DEBUG_MODE="false" - -# The default version for a MapR plugin -DIB_DEFAULT_MAPR_VERSION="5.2.0" - -# The default version for Spark plugin -DIB_DEFAULT_SPARK_VERSION="2.3.0" - -# The default version for Storm plugin -DIB_DEFAULT_STORM_VERSION="1.2.1" - -# Bare metal image generation is enabled with the -b flag, it is off by default -SIE_BAREMETAL="false" - -# Default list of datasource modules for ubuntu. Workaround for bug #1375645 -export CLOUD_INIT_DATASOURCES=${DIB_CLOUD_INIT_DATASOURCES:-"NoCloud, ConfigDrive, OVF, MAAS, Ec2"} - -# Tracing control -TRACING= - -usage() { - echo - echo "Usage: $(basename $0)" - echo " [-p vanilla|spark|cloudera|storm|mapr|ambari|plain]" - echo " [-i ubuntu|fedora|centos7]" - echo " [-v 2.7.1|2.7.5|2.8.2|3.0.1|5.5|5.7|5.9|5.11|2.2.0.0|2.2.1.0|2.4.2.0]" - echo " [-r 5.1.0|5.2.0]" - echo " [-s 1.6.0|2.1.0|2.2.0|2.3.0]" - echo " [-t 1.0.1|1.1.0|1.1.1|1.2.0|1.2.1]" - echo " [-f qcow2|raw]" - echo " [-d]" - echo " [-u]" - echo " [-j openjdk|oracle-java]" - echo " [-x]" - echo " [-h]" - echo " '-p' is plugin version (default: all plugins)" - echo " '-i' is operating system of the base image (default: all non-deprecated" - echo " by plugin)." - echo " '-v' is hadoop version (default: all supported by plugin)" - echo " '-r' is MapR Version (default: ${DIB_DEFAULT_MAPR_VERSION})" - echo " '-s' is Spark version (default: ${DIB_DEFAULT_SPARK_VERSION})" - echo " '-f' is the image format (default: qcow2)" - echo " '-d' enable debug mode, root account will have password 'hadoop'" - echo " '-u' install missing packages necessary for building" - echo " '-j' is java distribution (default: openjdk)" - echo " '-x' turns on tracing" - echo " '-b' generate a bare metal image" - echo " '-h' display this message" - echo - echo "You shouldn't specify image type for spark plugin" - echo "You shouldn't specify hadoop version for plain images" - echo "Debug mode should only be enabled for local debugging purposes, not for production systems" - echo "By default all images for all plugins will be created" - echo -} - -while getopts "p:i:v:f:dur:s:t:j:xhb" opt; do - case $opt in - p) - PLUGIN=$OPTARG - ;; - i) - BASE_IMAGE_OS=$OPTARG - ;; - v) - HADOOP_VERSION=$OPTARG - ;; - d) - DEBUG_MODE="true" - ;; - r) - DIB_MAPR_VERSION=$OPTARG - ;; - s) - DIB_SPARK_VERSION=$OPTARG - ;; - t) - DIB_STORM_VERSION=$OPTARG - ;; - f) - IMAGE_FORMAT="-t $OPTARG" - ;; - u) - DIB_UPDATE_REQUESTED=true - ;; - j) - JAVA_ELEMENT=$OPTARG - ;; - x) - TRACING="$TRACING -x" - set -x - ;; - b) - SIE_BAREMETAL="true" - ;; - h) - usage - exit 0 - ;; - *) - usage - exit 1 - ;; - esac -done - -shift $((OPTIND-1)) -if [ "$1" ]; then - usage - exit 1 -fi - -JAVA_ELEMENT=${JAVA_ELEMENT:-"openjdk"} - -if [ -e /etc/os-release ]; then - platform=$(cat /etc/os-release | awk -F= '/^ID=/ {print tolower($2);}') - # remove eventual quotes around ID=... - platform=$(echo $platform | sed -e 's,^",,;s,"$,,') -elif [ -e /etc/system-release ]; then - case "$(head -1 /etc/system-release)" in - "Red Hat Enterprise Linux Server"*) - platform=rhel - ;; - "CentOS"*) - platform=centos - ;; - *) - echo -e "Unknown value in /etc/system-release. Impossible to build images.\nAborting" - exit 2 - ;; - esac -else - echo -e "Unknown host OS. Impossible to build images.\nAborting" - exit 2 -fi - -# Checks of input -if [ "$DEBUG_MODE" = "true" -a "$platform" != 'ubuntu' ]; then - if [ "$(getenforce)" != "Disabled" ]; then - echo "Debug mode cannot be used from this platform while SELinux is enabled, see https://storyboard.openstack.org/#!/story/1292614" - exit 1 - fi -fi - -check_spark_version () { - case "$DIB_SPARK_VERSION" in - "1.6.0" | "2.1.0" | "2.2.0" | "2.3.0");; - "") - echo "Spark version not specified" - echo "Spark ${DIB_DEFAULT_SPARK_VERSION} will be used" - DIB_SPARK_VERSION=${DIB_DEFAULT_SPARK_VERSION} - ;; - *) - echo -e "Unknown Spark version selected.\nAborting" - exit 1 - ;; - esac -} - -case "$PLUGIN" in - "");; - "vanilla") - case "$HADOOP_VERSION" in - "" | "2.7.1" | "2.7.5" | "2.8.2" | "3.0.1");; - *) - echo -e "Unknown hadoop version selected.\nAborting" - exit 1 - ;; - esac - case "$BASE_IMAGE_OS" in - "" | "ubuntu" | "fedora" | "centos7");; - *) - echo -e "'$BASE_IMAGE_OS' image type is not supported by '$PLUGIN'.\nAborting" - exit 1 - ;; - esac - check_spark_version - ;; - "cloudera") - case "$BASE_IMAGE_OS" in - "" | "ubuntu" | "centos7");; - *) - echo -e "'$BASE_IMAGE_OS' image type is not supported by '$PLUGIN'.\nAborting" - exit 1 - ;; - esac - - case "$HADOOP_VERSION" in - "" | "5.5" | "5.7" | "5.9" | "5.11");; - *) - echo -e "Unknown hadoop version selected.\nAborting" - exit 1 - ;; - esac - - if [ "$BASE_IMAGE_OS" = "centos7" ]; then - if [ ! -z "$HADOOP_VERSION" -a ! "$HADOOP_VERSION" = "5.5" -a ! "$HADOOP_VERSION" = "5.7" -a ! "$HADOOP_VERSION" = "5.9" -a ! "$HADOOP_VERSION" = "5.11" ]; then - echo -e "Unsupported version combination, Centos 7 can only be used with CDH 5.5 or higher version" - exit 1 - fi - fi - - if [ -n "$DIB_CDH_MINOR_VERSION" ]; then - echo -e "Continuing image building with custom CDH version: \"$DIB_CDH_MINOR_VERSION\".\n" - fi - ;; - "spark") - case "$BASE_IMAGE_OS" in - "" | "ubuntu");; - *) - echo -e "'$BASE_IMAGE_OS' image type is not supported by '$PLUGIN'.\nAborting" - exit 1 - ;; - esac - check_spark_version - ;; - "storm") - case "$BASE_IMAGE_OS" in - "" | "ubuntu");; - *) - echo -e "'$BASE_IMAGE_OS' image type is not supported by '$PLUGIN'.\nAborting" - exit 1 - ;; - esac - - case "$DIB_STORM_VERSION" in - "1.0.1" | "1.1.0" | "1.1.1" | "1.2.0" | "1.2.1");; - "") - echo "Storm version not specified" - echo "Storm ${DIB_DEFAULT_STORM_VERSION} will be used" - DIB_STORM_VERSION=${DIB_DEFAULT_STORM_VERSION} - ;; - *) - echo -e "Unknown Storm version selected.\nAborting" - exit 1 - ;; - esac - - if [ -n "$HADOOP_VERSION" ]; then - echo -e "You shouldn't specify hadoop version for '$PLUGIN'.\nAborting" - exit 1 - fi - ;; - "ambari") - case "$BASE_IMAGE_OS" in - "" | "centos7" | "ubuntu" ) - ;; - * ) - echo "\"$BASE_IMAGE_OS\" image type is not supported by \"$PLUGIN\".\nAborting" - exit 1 - ;; - esac - - case "$HADOOP_VERSION" in - "" | "2.2.0.0" | "2.2.1.0" | "2.4.2.0");; - *) - echo -e "Continuing image building with custom ambari version \"$HADOOP_VERSION\"\n" - ;; - esac - ;; - "mapr") - case "$BASE_IMAGE_OS" in - "" | "ubuntu" | "centos7");; - *) - echo -e "'$BASE_IMAGE_OS' image type is not supported by '$PLUGIN'.\nAborting" - exit 1 - ;; - esac - - if [ -n "$HADOOP_VERSION" ]; then - echo -e "You shouldn't specify hadoop version for 'mapr'.\nAborting" - exit 1 - fi - - case "$DIB_MAPR_VERSION" in - "") - echo "MapR version is not specified" - echo "${DIB_DEFAULT_MAPR_VERSION} version would be used" - DIB_MAPR_VERSION=${DIB_DEFAULT_MAPR_VERSION} - ;; - "5.1.0" | "5.2.0");; - *) - echo -e "Unknown MapR version.\nExit" - exit 1 - ;; - esac - ;; - "plain") - case "$BASE_IMAGE_OS" in - "" | "ubuntu" | "fedora" | "centos7");; - *) - echo -e "'$BASE_IMAGE_OS' image type is not supported by '$PLUGIN'.\nAborting" - exit 1 - ;; - esac - - if [ -n "$HADOOP_VERSION" ]; then - echo -e "You shouldn't specify hadoop version for '$PLUGIN'.\nAborting" - exit 1 - fi - ;; - *) - echo -e "Unknown plugin selected.\nAborting" - exit 1 -esac - -if [ "$PLUGIN" != "mapr" -a -n "$DIB_MAPR_VERSION" ]; then - echo -e "'-r' parameter should be used only with 'mapr' plugin.\nAborting" - exit 1 -fi - - -if [ "$JAVA_ELEMENT" != "openjdk" -a "$JAVA_ELEMENT" != "oracle-java" ]; then - echo "Unknown java distro" - exit 1 -fi - -################# - -is_installed() { - if [ "$platform" = 'ubuntu' -o "$platform" = 'debian' ]; then - dpkg -s "$1" &> /dev/null - else - # centos, fedora, opensuse, or rhel - if ! rpm -q "$1" &> /dev/null; then - rpm -q "$(rpm -q --whatprovides "$1")" - fi - fi -} - -need_required_packages() { - case "$platform" in - "ubuntu" | "debian") - package_list="qemu-utils kpartx git" - ;; - "fedora") - package_list="qemu-img kpartx git" - ;; - "opensuse") - package_list="qemu kpartx git-core" - ;; - "rhel" | "centos") - package_list="qemu-kvm qemu-img kpartx git" - ;; - *) - echo -e "Unknown platform '$platform' for the package list.\nAborting" - exit 2 - ;; - esac - - for p in `echo $package_list`; do - if ! is_installed $p; then - echo "Package $p is not installed on the system." - return 0 - fi - done - return 1 -} - -if need_required_packages; then - # install required packages if requested - if [ -n "$DIB_UPDATE_REQUESTED" ]; then - case "$platform" in - "ubuntu" | "debian") - sudo apt-get update - sudo apt-get install $package_list -y - ;; - "opensuse") - sudo zypper --non-interactive --gpg-auto-import-keys in $package_list - ;; - *) - echo -e "Unknown platform '$platform' for installing packages.\nAborting" - exit 2 - ;; - esac - else - echo "Missing one of the following packages: $package_list" - echo "Please install manually or rerun with the update option (-u)." - exit 1 - fi -fi - -if [ "$DEBUG_MODE" = "true" ]; then - echo "Using Image Debug Mode, using root-pwd in images, NOT FOR PRODUCTION USAGE." - # Each image has a root login, password is "hadoop" - export DIB_PASSWORD="hadoop" -fi - -################# - -# Common helper for invoking disk-image-create, adding all the common -# elements and arguments, and setting common environment variables. -# -# Usage: -# image_create DISTRO OUTPUT [args...] -# - DISTRO is the main element of the distribution -# - OUTPUT is the output name for the image -# - any other argument is passed directly to disk-image-create -image_create() { - local distro=$1 - shift - local output=$1 - shift - - # the base elements and args, used in *all* the images - local elements="sahara-version ntp xfs-tools" - if [ $SIE_BAREMETAL = "true" ]; then - elements="grub2 baremetal dhcp-all-interfaces $elements" - else - elements="vm $elements" - fi - local args="" - - # debug mode handling - if [ "$DEBUG_MODE" = "true" ]; then - elements="$elements root-passwd" - fi - # mirror handling - if [ -n "$USE_MIRRORS" ]; then - case "$distro" in - ubuntu) elements="$elements apt-mirror" ;; - fedora) elements="$elements fedora-mirror" ;; - centos7) elements="$elements centos-mirror" ;; - esac - fi - - disk-image-create $IMAGE_FORMAT $TRACING -o "$output" $args "$distro" $elements "$@" -} - -set_hive_version() { - if [ -z "${HIVE_VERSION:-}" ]; then - case "$DIB_HADOOP_VERSION" in - "2.7.1" ) - export HIVE_VERSION="0.11.0" - ;; - "2.7.5" ) - export HIVE_VERSION="2.3.2" - ;; - "2.8.2" ) - export HIVE_VERSION="2.3.2" - ;; - "3.0.1" ) - export HIVE_VERSION="3.0.0" - ;; - *) - echo -e "Unknown Hadoop version, therefore cannot choose Hive version.\nAborting." - exit 1 - ;; - esac - fi -} - - -############################# -# Images for Vanilla plugin # -############################# - -if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then - export HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.7.1.tar.gz"} - export HADOOP_V2_7_5_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_7_5_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.7.5.tar.gz"} - export HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.8.2.tar.gz"} - export HADOOP_V3_0_1_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V3_0_1_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-3.0.1.tar.gz"} - export OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL=${OOZIE_HADOOP_V2_7_1_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.2.0-hadoop-2.7.1.tar.gz"} - export OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL=${OOZIE_HADOOP_V2_7_5_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.3.0-hadoop-2.7.5.tar.gz"} - export OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL=${OOZIE_HADOOP_V2_8_2_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.3.0-hadoop-2.8.2.tar.gz"} - export OOZIE_HADOOP_V3_0_1_DOWNLOAD_URL=${OOZIE_HADOOP_V3_0_1_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-5.0.0-hadoop-3.0.1.tar.gz"} - export DIB_HDFS_LIB_DIR="/opt/hadoop/share/hadoop/tools/lib" - export plugin_type="vanilla" - - export DIB_SPARK_VERSION - - if [ "$DIB_SPARK_VERSION" = "1.6.0" ]; then - export SPARK_HADOOP_DL=hadoop2.6 - else - export SPARK_HADOOP_DL=hadoop2.7 - fi - - ubuntu_elements_sequence="hadoop oozie mysql hive $JAVA_ELEMENT swift_hadoop spark s3_hadoop" - fedora_elements_sequence="hadoop oozie mysql disable-firewall hive $JAVA_ELEMENT swift_hadoop spark s3_hadoop" - centos7_elements_sequence="hadoop oozie mysql disable-firewall hive $JAVA_ELEMENT swift_hadoop spark nc s3_hadoop" - - # Workaround for https://bugs.launchpad.net/diskimage-builder/+bug/1204824 - # https://storyboard.openstack.org/#!/story/1252684 - if [ "$platform" = 'ubuntu' ]; then - echo "**************************************************************" - echo "WARNING: As a workaround for DIB bug 1204824, you are about to" - echo " create a Fedora and CentOS images that has SELinux " - echo " disabled. Do not use these images in production. " - echo "**************************************************************" - fedora_elements_sequence="$fedora_elements_sequence selinux-permissive" - centos7_elements_sequence="$centos7_elements_sequence selinux-permissive" - suffix=".selinux-permissive" - fi - # Ubuntu cloud image - if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then - export DIB_CLOUD_INIT_DATASOURCES=$CLOUD_INIT_DATASOURCES - - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.1" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_1:-"2.7.1"} - export ubuntu_image_name=${ubuntu_vanilla_hadoop_2_7_1_image_name:-"ubuntu_sahara_vanilla_hadoop_2_7_1_latest"} - export DIB_RELEASE=${DIB_RELEASE:-xenial} - set_hive_version - image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence - unset DIB_RELEASE - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.5" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_5:-"2.7.5"} - export ubuntu_image_name=${ubuntu_vanilla_hadoop_2_7_5_image_name:-"ubuntu_sahara_vanilla_hadoop_2_7_5_latest"} - export DIB_RELEASE=${DIB_RELEASE:-xenial} - set_hive_version - image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence - unset DIB_RELEASE - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.8.2" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_8_2:-"2.8.2"} - export ubuntu_image_name=${ubuntu_vanilla_hadoop_2_8_2_image_name:-"ubuntu_sahara_vanilla_hadoop_2_8_2_latest"} - export DIB_RELEASE=${DIB_RELEASE:-xenial} - set_hive_version - image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence - unset DIB_RELEASE - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "3.0.1" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_3_0_1:-"3.0.1"} - export ubuntu_image_name=${ubuntu_vanilla_hadoop_3_0_1_image_name:-"ubuntu_sahara_vanilla_hadoop_3_0_1_latest"} - export DIB_RELEASE=${DIB_RELEASE:-xenial} - set_hive_version - image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence - unset DIB_RELEASE - fi - unset DIB_CLOUD_INIT_DATASOURCES - fi - - # Fedora cloud image - if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "fedora" ]; then - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.1" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_1:-"2.7.1"} - export fedora_image_name=${fedora_vanilla_hadoop_2_7_1_image_name:-"fedora_sahara_vanilla_hadoop_2_7_1_latest$suffix"} - set_hive_version - image_create fedora $fedora_image_name $fedora_elements_sequence - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.5" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_5:-"2.7.5"} - export fedora_image_name=${fedora_vanilla_hadoop_2_7_5_image_name:-"fedora_sahara_vanilla_hadoop_2_7_5_latest$suffix"} - set_hive_version - image_create fedora $fedora_image_name $fedora_elements_sequence - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.8.2" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_8_2:-"2.8.2"} - export fedora_image_name=${fedora_vanilla_hadoop_2_8_2_image_name:-"fedora_sahara_vanilla_hadoop_2_8_2_latest$suffix"} - set_hive_version - image_create fedora $fedora_image_name $fedora_elements_sequence - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "3.0.1" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_3_0_1:-"3.0.1"} - export fedora_image_name=${fedora_vanilla_hadoop_3_0_1_image_name:-"fedora_sahara_vanilla_hadoop_3_0_1_latest$suffix"} - set_hive_version - image_create fedora $fedora_image_name $fedora_elements_sequence - fi - fi - - # CentOS 7 cloud image - if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos7" ]; then - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.1" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_1:-"2.7.1"} - export centos7_image_name=${centos7_vanilla_hadoop_2_7_1_image_name:-"centos7_sahara_vanilla_hadoop_2_7_1_latest$suffix"} - set_hive_version - image_create centos7 $centos7_image_name $centos7_elements_sequence - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.5" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_5:-"2.7.5"} - export centos7_image_name=${centos7_vanilla_hadoop_2_7_5_image_name:-"centos7_sahara_vanilla_hadoop_2_7_5_latest$suffix"} - set_hive_version - image_create centos7 $centos7_image_name $centos7_elements_sequence - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.8.2" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_8_2:-"2.8.2"} - export centos7_image_name=${centos7_vanilla_hadoop_2_8_2_image_name:-"centos7_sahara_vanilla_hadoop_2_8_2_latest$suffix"} - set_hive_version - image_create centos7 $centos7_image_name $centos7_elements_sequence - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "3.0.1" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_3_0_1:-"3.0.1"} - export centos7_image_name=${centos7_vanilla_hadoop_3_0_1_image_name:-"centos7_sahara_vanilla_hadoop_3_0_1_latest$suffix"} - set_hive_version - image_create centos7 $centos7_image_name $centos7_elements_sequence - fi - fi - - unset plugin_type - unset DIB_HDFS_LIB_DIR - unset DIB_SPARK_VERSION - unset SPARK_HADOOP_DL -fi - -########################### -# Images for Spark plugin # -########################### - -if [ -z "$PLUGIN" -o "$PLUGIN" = "spark" ]; then - export DIB_HDFS_LIB_DIR="/usr/lib/hadoop-mapreduce" - export DIB_CLOUD_INIT_DATASOURCES=$CLOUD_INIT_DATASOURCES - export DIB_SPARK_VERSION - export plugin_type="spark" - - if [ "$DIB_SPARK_VERSION" = "2.2.0" -o "$DIB_SPARK_VERSION" = "2.3.0" ]; then - export DIB_CDH_VERSION="5.11" - export DIB_RELEASE=${DIB_RELEASE:-xenial} - else - export DIB_RELEASE=${DIB_RELEASE:-trusty} - export DIB_CDH_VERSION="5.5" - fi - if [ "$DIB_SPARK_VERSION" = "1.6.0" ]; then - export SPARK_HADOOP_DL=hadoop2.6 - else - export SPARK_HADOOP_DL=hadoop2.7 - fi - # Tell the cloudera element to install only hdfs - export DIB_CDH_HDFS_ONLY=1 - - ubuntu_elements_sequence="$JAVA_ELEMENT swift_hadoop spark hadoop-cloudera s3_hadoop" - export ubuntu_image_name=${ubuntu_spark_image_name:-"ubuntu_sahara_spark_latest"} - - # Creating Ubuntu cloud image - image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence - unset SPARK_HADOOP_DL - unset DIB_CLOUD_INIT_DATASOURCES - unset DIB_HDFS_LIB_DIR - unset DIB_CDH_HDFS_ONLY - unset DIB_CDH_VERSION - unset DIB_SPARK_VERSION - unset DIB_HADOOP_VERSION - unset DIB_RELEASE - unset plugin_type -fi - - -########################## -# Image for Storm plugin # -########################## - -if [ -z "$PLUGIN" -o "$PLUGIN" = "storm" ]; then - export DIB_CLOUD_INIT_DATASOURCES=$CLOUD_INIT_DATASOURCES - - export DIB_STORM_VERSION - export ubuntu_image_name=${ubuntu_storm_image_name:-"ubuntu_sahara_storm_latest_$DIB_STORM_VERSION"} - - ubuntu_elements_sequence="$JAVA_ELEMENT zookeeper storm" - - # Creating Ubuntu cloud image - export DIB_RELEASE=${DIB_RELEASE:-xenial} - image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence - unset DIB_RELEASE - unset DIB_CLOUD_INIT_DATASOURCES -fi - -############################ -# Images for Ambari plugin # -############################ - -if [ -z "$PLUGIN" -o "$PLUGIN" = "ambari" ]; then - export DIB_AMBARI_VERSION="$HADOOP_VERSION" - export plugin_type="ambari" - # set the temporary folder for hadoop-openstack.jar file - export DIB_HDFS_LIB_DIR="/opt" - - if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then - ambari_ubuntu_image_name=${ambari_ubuntu_image_name:-ubuntu_sahara_ambari} - ambari_element_sequence="ambari $JAVA_ELEMENT swift_hadoop kdc" - export DIB_RELEASE="trusty" - image_create ubuntu $ambari_ubuntu_image_name $ambari_element_sequence - unset DIB_RELEASE - fi - if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos7" ]; then - ambari_centos7_image_name=${ambari_centos7_image_name:-"centos7-sahara-ambari"} - ambari_element_sequence="disable-selinux ambari $JAVA_ELEMENT disable-firewall swift_hadoop kdc nc" - image_create centos7 $ambari_centos7_image_name $ambari_element_sequence - fi - - unset DIB_HDFS_LIB_DIR - unset plugin_type - unset DIB_AMBARI_VERSION -fi - -######################### -# Images for CDH plugin # -######################### - -if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then - # Cloudera installation requires additional space - export DIB_MIN_TMPFS=5 - export plugin_type="cloudera" - export DIB_HDFS_LIB_DIR="/usr/lib/hadoop-mapreduce" - - if [ -n "$DIB_CDH_MINOR_VERSION" ]; then - # cut minor version number, e.g. from 5.7.1 to 5.7 - # this is needed if user specified minor version but didn't specify - # hadoop version by '-v' parameter - HADOOP_VERSION=${DIB_CDH_MINOR_VERSION%.*} - fi - - cloudera_elements_sequence="hadoop-cloudera swift_hadoop kdc" - if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.5" ]; then - export DIB_CDH_VERSION="5.5" - cloudera_5_5_ubuntu_image_name=${cloudera_5_5_ubuntu_image_name:-ubuntu_sahara_cloudera_5_5_0} - - # Cloudera supports 14.04 Ubuntu in 5.5 - export DIB_RELEASE="trusty" - image_create ubuntu $cloudera_5_5_ubuntu_image_name $cloudera_elements_sequence - unset DIB_CDH_VERSION DIB_RELEASE - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.7" ]; then - export DIB_CDH_VERSION="5.7" - export DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$DIB_CDH_VERSION.0} - cloudera_5_7_ubuntu_image_name=${cloudera_5_7_ubuntu_image_name:-ubuntu_sahara_cloudera_$DIB_CDH_MINOR_VERSION} - - export DIB_RELEASE="trusty" - image_create ubuntu $cloudera_5_7_ubuntu_image_name $cloudera_elements_sequence - unset DIB_CDH_VERSION DIB_RELEASE DIB_CDH_MINOR_VERSION - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.9" ]; then - export DIB_CDH_VERSION="5.9" - export DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$DIB_CDH_VERSION.0} - cloudera_5_9_ubuntu_image_name=${cloudera_5_9_ubuntu_image_name:-ubuntu_sahara_cloudera_$DIB_CDH_MINOR_VERSION} - - export DIB_RELEASE="trusty" - image_create ubuntu $cloudera_5_9_ubuntu_image_name $cloudera_elements_sequence - unset DIB_CDH_VERSION DIB_RELEASE DIB_CDH_MINOR_VERSION - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.11" ]; then - export DIB_CDH_VERSION="5.11" - export DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$DIB_CDH_VERSION.0} - cloudera_5_11_ubuntu_image_name=${cloudera_5_11_ubuntu_image_name:-ubuntu_sahara_cloudera_$DIB_CDH_MINOR_VERSION} - - export DIB_RELEASE="xenial" - image_create ubuntu $cloudera_5_11_ubuntu_image_name $cloudera_elements_sequence - unset DIB_CDH_VERSION DIB_RELEASE DIB_CDH_MINOR_VERSION - fi - fi - - if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos7" ]; then - centos7_cloudera_elements_sequence="selinux-permissive disable-firewall nc" - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.5" ]; then - export DIB_CDH_VERSION="5.5" - - cloudera_5_5_centos7_image_name=${cloudera_5_5_centos7_image_name:-centos7_sahara_cloudera_5_5_0} - image_create centos7 $cloudera_5_5_centos7_image_name $cloudera_elements_sequence $centos7_cloudera_elements_sequence - - unset DIB_CDH_VERSION - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.7" ]; then - export DIB_CDH_VERSION="5.7" - export DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$DIB_CDH_VERSION.0} - - cloudera_5_7_centos7_image_name=${cloudera_5_7_centos7_image_name:-centos7_sahara_cloudera_$DIB_CDH_MINOR_VERSION} - image_create centos7 $cloudera_5_7_centos7_image_name $cloudera_elements_sequence $centos7_cloudera_elements_sequence - - unset DIB_CDH_VERSION DIB_CDH_MINOR_VERSION - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.9" ]; then - export DIB_CDH_VERSION="5.9" - export DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$DIB_CDH_VERSION.0} - - cloudera_5_9_centos7_image_name=${cloudera_5_9_centos7_image_name:-centos7_sahara_cloudera_$DIB_CDH_MINOR_VERSION} - image_create centos7 $cloudera_5_9_centos7_image_name $cloudera_elements_sequence $centos7_cloudera_elements_sequence - - unset DIB_CDH_VERSION DIB_CDH_MINOR_VERSION - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.11" ]; then - export DIB_CDH_VERSION="5.11" - export DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$DIB_CDH_VERSION.0} - - cloudera_5_11_centos7_image_name=${cloudera_5_11_centos7_image_name:-centos7_sahara_cloudera_$DIB_CDH_MINOR_VERSION} - image_create centos7 $cloudera_5_11_centos7_image_name $cloudera_elements_sequence $centos7_cloudera_elements_sequence - - unset DIB_CDH_VERSION DIB_CDH_MINOR_VERSION - fi - fi - - unset DIB_CDH_MINOR_VERSION - unset DIB_HDFS_LIB_DIR - unset DIB_MIN_TMPFS - unset plugin_type -fi - -########################## -# Images for MapR plugin # -########################## -if [ -z "$PLUGIN" -o "$PLUGIN" = "mapr" ]; then - export DIB_MAPR_VERSION=${DIB_MAPR_VERSION:-${DIB_DEFAULT_MAPR_VERSION}} - export plugin_type="mapr" - - export DIB_CLOUD_INIT_DATASOURCES=$CLOUD_INIT_DATASOURCES - - export DIB_IMAGE_SIZE=${IMAGE_SIZE:-"10"} - #MapR repository requires additional space - export DIB_MIN_TMPFS=10 - - mapr_ubuntu_elements_sequence="ssh hadoop-mapr $JAVA_ELEMENT" - mapr_centos_elements_sequence="ssh hadoop-mapr selinux-permissive $JAVA_ELEMENT disable-firewall nc" - - if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then - export DIB_RELEASE=${DIB_RELEASE:-trusty} - - mapr_ubuntu_image_name=${mapr_ubuntu_image_name:-ubuntu_${DIB_RELEASE}_mapr_${DIB_MAPR_VERSION}_latest} - - image_create ubuntu $mapr_ubuntu_image_name $mapr_ubuntu_elements_sequence - - unset DIB_RELEASE - fi - - if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos7" ]; then - mapr_centos7_image_name=${mapr_centos7_image_name:-centos_7_mapr_${DIB_MAPR_VERSION}_latest} - - image_create centos7 $mapr_centos7_image_name $mapr_centos_elements_sequence - - unset DIB_CLOUD_INIT_DATASOURCES - fi - unset plugin_type - -fi - -################ -# Plain images # -################ -if [ -z "$PLUGIN" -o "$PLUGIN" = "plain" ]; then - # generate plain (no Hadoop components) images for testing - - common_elements="ssh" - - ubuntu_elements_sequence="$common_elements" - fedora_elements_sequence="$common_elements" - centos7_elements_sequence="$common_elements disable-firewall disable-selinux nc" - - if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then - plain_image_name=${plain_ubuntu_image_name:-ubuntu_plain} - - export DIB_RELEASE=${DIB_RELEASE:-xenial} - image_create ubuntu $plain_image_name $ubuntu_elements_sequence - unset DIB_RELEASE - fi - - if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "fedora" ]; then - plain_image_name=${plain_fedora_image_name:-fedora_plain} - - image_create fedora $plain_image_name $fedora_elements_sequence - fi - - if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos7" ]; then - plain_image_name=${plain_centos7_image_name:-centos7_plain} - - image_create centos7 $plain_image_name $centos7_elements_sequence - fi -fi diff --git a/elements/.gitignore b/elements/.gitignore deleted file mode 100644 index b25c15b8..00000000 --- a/elements/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*~ diff --git a/elements/README.rst b/elements/README.rst deleted file mode 100644 index 96469d4f..00000000 --- a/elements/README.rst +++ /dev/null @@ -1,69 +0,0 @@ -Diskimage-builder tools for creation cloud images -================================================= - -Steps how to create cloud image with Apache Hadoop installed using diskimage-builder project: - -1. Clone the repository "https://github.com/openstack/diskimage-builder" locally. Note: Make sure you have commit 43b96d91 in your clone, it provides a mapping for default-jre. - -.. sourcecode:: bash - - git clone https://github.com/openstack/diskimage-builder - -2. Add ~/diskimage-builder/bin/ directory to your path (for example, PATH=$PATH:/home/$USER/diskimage-builder/bin/ ). - -3. Export the following variable ELEMENTS_PATH=/home/$USER/diskimage-builder/elements/ to your .bashrc. Then source it. - -4. Copy file "img-build-sudoers" from ~/disk-image-builder/sudoers.d/ to your /etc/sudoers.d/. - -.. sourcecode:: bash - - chmod 440 /etc/sudoers.d/img-build-sudoers - chown root:root /etc/sudoers.d/img-build-sudoers - -5. Export sahara-elements commit id variable (from sahara-extra directory): - -.. sourcecode:: bash - - export SAHARA_ELEMENTS_COMMIT_ID=`git show --format=%H | head -1` - -6. Move elements/ directory to disk-image-builder/elements/ - -.. sourcecode:: bash - - mv elements/* /path_to_disk_image_builder/diskimage-builder/elements/ - -7. Export DIB commit id variable (from DIB directory): - -.. sourcecode:: bash - - export DIB_COMMIT_ID=`git show --format=%H | head -1` - -8. Call the following command to create cloud image is able to run on OpenStack: - -8.1. Ubuntu cloud image - -.. sourcecode:: bash - - JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_HADOOP_VERSION=1.2.1 OOZIE_FILE=oozie-4.0.0.tar.gz disk-image-create base vm hadoop oozie ubuntu root-passwd -o ubuntu_hadoop_1_2_1 - -8.2. Fedora cloud image - -.. sourcecode:: bash - - JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_HADOOP_VERSION=1.2.1 OOZIE_FILE=oozie-4.0.0.tar.gz DIB_IMAGE_SIZE=10 disk-image-create base vm fedora hadoop root-passwd oozie -o fedora_hadoop_1_2_1 - -Note: If you are building this image from Ubuntu or Fedora 18 OS host, you should add element 'selinux-permissive'. - -.. sourcecode:: bash - - JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_HADOOP_VERSION=1.2.1 OOZIE_FILE=oozie-4.0.0.tar.gz DIB_IMAGE_SIZE=10 disk-image-create base vm fedora hadoop root-passwd oozie selinux-permissive -o fedora_hadoop_1_2_1 - -In this command 'DIB_HADOOP_VERSION' parameter is version of hadoop needs to be installed. -You can use 'JAVA_DOWNLOAD_URL' parameter to specify download link for JDK (tarball or bin). -'DIB_IMAGE_SIZE' is parameter that specifes a volume of hard disk of instance. You need to specify it because Fedora and CentOS don't use all available volume. -If you have already downloaded the jdk package, move it to "elements/hadoop/install.d/" and use its filename as 'JAVA_FILE' parameter. -In order of working EDP components with Sahara DIB images you need pre-installed Oozie libs. -Use OOZIE_DOWNLOAD_URL to specify link to Oozie archive (tar.gz). For example the Oozie libraries -for Hadoop 2.7.1 are available from: -https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.2.0-hadoop-2.7.1.tar.gz -If you have already downloaded archive, move it to "elements/oozie/install.d/" and use its filename as 'OOZIE_FILE' parameter. diff --git a/elements/ambari/README.rst b/elements/ambari/README.rst deleted file mode 100644 index e59f7aea..00000000 --- a/elements/ambari/README.rst +++ /dev/null @@ -1,14 +0,0 @@ -====== -ambari -====== - -Installs Ambari Management Console - -Environment Variables ---------------------- - -AMBARI_VERSION - :Required: No - :Default: 2.4.2.0 - :Description: Version of Ambari Management Console to install - :Example: ``AMBARI_VERSION="2.2.1.0"`` installs Ambari 2.2.1.0 diff --git a/elements/ambari/element-deps b/elements/ambari/element-deps deleted file mode 100644 index e6180587..00000000 --- a/elements/ambari/element-deps +++ /dev/null @@ -1,3 +0,0 @@ -java -mysql -package-installs diff --git a/elements/ambari/package-installs.yaml b/elements/ambari/package-installs.yaml deleted file mode 100644 index 645fc03d..00000000 --- a/elements/ambari/package-installs.yaml +++ /dev/null @@ -1,19 +0,0 @@ -wget: - phase: pre-install.d -ambari-agent: -ambari-log4j: -ambari-metrics-hadoop-sink: -ambari-metrics-monitor: -ambari-server: -ambari-metrics-collector: -redhat-lsb: -nc: -unzip: -curl: -tar: -zip: -rpcbind: -fuse-libs: -fuse: -snappy-devel: -rng-tools: diff --git a/elements/ambari/pkg-map b/elements/ambari/pkg-map deleted file mode 100644 index 3adf2b09..00000000 --- a/elements/ambari/pkg-map +++ /dev/null @@ -1,20 +0,0 @@ -{ - "family": { - "debian": { - "nc": "netcat", - "redhat-lsb": "", - "snappy-devel": "", - "fuse-libs": "fuse", - "ambari-metrics-monitor": "ambari-metrics-assembly", - "ambari-metrics-collector": "ambari-metrics-assembly", - "ambari-metrics-hadoop-sink": "ambari-metrics-assembly", - "ambari-log4j": "" - }, - "redhat": { - "nc": "nc", - "redhat-lsb": "redhat-lsb", - "snappy-devel": "snappy-devel", - "fuse-libs": "fuse-libs" - } - } -} diff --git a/elements/ambari/post-install.d/10-disable-ambari b/elements/ambari/post-install.d/10-disable-ambari deleted file mode 100755 index e7841f74..00000000 --- a/elements/ambari/post-install.d/10-disable-ambari +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -case "${DISTRO_NAME}" in - rhel7 | centos7 ) - chkconfig ambari-server off - chkconfig ambari-agent off - ;; -esac diff --git a/elements/ambari/post-install.d/20-disable-ca-certificate b/elements/ambari/post-install.d/20-disable-ca-certificate deleted file mode 100755 index 711f1f21..00000000 --- a/elements/ambari/post-install.d/20-disable-ca-certificate +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -case "${DISTRO_NAME}" in - rhel7 | centos7 ) - config=/etc/python/cert-verification.cfg - [ -e $config ] && sed -i "s%^\(verify=\s*\).*$%verify=disable%" $config - ;; -esac diff --git a/elements/ambari/pre-install.d/90-ambari-repo-conf b/elements/ambari/pre-install.d/90-ambari-repo-conf deleted file mode 100755 index 46d5d483..00000000 --- a/elements/ambari/pre-install.d/90-ambari-repo-conf +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -DIB_AMBARI_VERSION="${DIB_AMBARI_VERSION:-2.4.2.0}" - -case "${DISTRO_NAME}" in - rhel7 | centos7 ) - wget http://public-repo-1.hortonworks.com/ambari/centos7/2.x/updates/$DIB_AMBARI_VERSION/ambari.repo -O /etc/yum.repos.d/ambari.repo - ;; - ubuntu ) - wget http://public-repo-1.hortonworks.com/ambari/ubuntu14/2.x/updates/$DIB_AMBARI_VERSION/ambari.list -O /etc/apt/sources.list.d/ambari.list - apt-key adv --recv-keys --keyserver keyserver.ubuntu.com B9733A7A07513CAD - ;; -esac diff --git a/elements/apt-mirror/README.rst b/elements/apt-mirror/README.rst deleted file mode 100644 index 0454cc88..00000000 --- a/elements/apt-mirror/README.rst +++ /dev/null @@ -1,13 +0,0 @@ -========== -apt-mirror -========== - -This element sets up the mirror for updating the Ubuntu cloud image. -Using a mirror improves the speed of the image building. - -Environment Variables ---------------------- - -UBUNTU_MIRROR - :Required: Yes - :Description: URL to the Ubuntu mirror. diff --git a/elements/apt-mirror/finalise.d/99-setup b/elements/apt-mirror/finalise.d/99-setup deleted file mode 100755 index 91ce9815..00000000 --- a/elements/apt-mirror/finalise.d/99-setup +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -rm /etc/apt/apt.conf.d/01proxy diff --git a/elements/apt-mirror/pre-install.d/00-apt b/elements/apt-mirror/pre-install.d/00-apt deleted file mode 100755 index e73d437d..00000000 --- a/elements/apt-mirror/pre-install.d/00-apt +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -mkdir -p /etc/apt/apt.conf.d/ -touch /etc/apt/apt.conf.d/01proxy -echo -e "Acquire::http { Proxy \"$UBUNTU_MIRROR\"; };" > /etc/apt/apt.conf.d/01proxy diff --git a/elements/apt-mirror/root.d/0-check b/elements/apt-mirror/root.d/0-check deleted file mode 100755 index a87746e1..00000000 --- a/elements/apt-mirror/root.d/0-check +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -if [ -z "$UBUNTU_MIRROR" ]; then - echo "You should specify parameter 'UBUNTU_MIRROR'" - exit 2 -fi diff --git a/elements/centos-mirror/README.rst b/elements/centos-mirror/README.rst deleted file mode 100644 index 242d00e3..00000000 --- a/elements/centos-mirror/README.rst +++ /dev/null @@ -1,13 +0,0 @@ -============= -centos-mirror -============= - -This element sets up the mirror for updating the CentOS cloud image. -Using a mirror improves the speed of the image building. - -Environment Variables ---------------------- - -CENTOS_MIRROR - :Required: Yes - :Description: URL to the CentOS mirror. diff --git a/elements/centos-mirror/finalise.d/99-setup b/elements/centos-mirror/finalise.d/99-setup deleted file mode 100755 index bbc84deb..00000000 --- a/elements/centos-mirror/finalise.d/99-setup +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -mv /opt/repos/* /etc/yum.repos.d/ diff --git a/elements/centos-mirror/pre-install.d/00-yum b/elements/centos-mirror/pre-install.d/00-yum deleted file mode 100755 index ef658883..00000000 --- a/elements/centos-mirror/pre-install.d/00-yum +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -mkdir -p /opt/repos -mv /etc/yum.repos.d/* /opt/repos/ -cat >> /etc/yum.repos.d/centos.repo <> /etc/yum.repos.d/fedora.repo < /dev/null; then - addgroup hadoop - fi - if ! getent passwd hadoop > /dev/null; then - adduser --ingroup hadoop --disabled-password --gecos GECOS hadoop - fi - adduser hadoop sudo - ;; -esac - -echo "Pre-configuring Hadoop" - -cat >> /home/hadoop/.bashrc <> /etc/hadoop/hadoop-env.sh <> /etc/apt/preferences.d/cloudera << EOF -Package: * -Pin: origin "archive.cloudera.com" -Pin-Priority: 800 -EOF - - dpkg -i /tmp/cdh4-repository_1.0_all.deb - rm /tmp/cdh4-repository_1.0_all.deb - wget -O - http://archive.cloudera.com/cdh4/ubuntu/precise/amd64/cdh/archive.key | sudo apt-key add - - ;; - *) - echo "Distro $DISTRO_NAME not supported by CDH. Exiting." - exit 1 - ;; -esac diff --git a/elements/hadoop-cdh/root.d/0-check-hadoop-cdh b/elements/hadoop-cdh/root.d/0-check-hadoop-cdh deleted file mode 100755 index e7ad22d4..00000000 --- a/elements/hadoop-cdh/root.d/0-check-hadoop-cdh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -echo "##############################################################" -echo "## ##" -echo "## WARNING: this element (hadoop-cdh) has been deprecated ##" -echo "## Please use the hadoop-cloudera element ##" -echo "## ##" -echo "##############################################################" - -if [ -z "${JAVA_DOWNLOAD_URL:-}" ]; then - if [ -z "${JAVA_FILE:-}" ]; then - echo "JAVA_FILE and JAVA_DOWNLOAD_URL are not set. Proceeding with distro native Java." - fi -fi -if [ -z "$DIB_CDH_VERSION" ]; then - echo "DIB_CDH_VERSION is not set. Impossible to install hadoop. Exit" - exit 1 -fi -if [ $DIB_CDH_VERSION != "CDH4" ]; then - echo "CDH version $DIB_CDH_VERSION not supported. Exiting." - exit 1 -fi diff --git a/elements/hadoop-cloudera/README.rst b/elements/hadoop-cloudera/README.rst deleted file mode 100644 index 3b0e5c0b..00000000 --- a/elements/hadoop-cloudera/README.rst +++ /dev/null @@ -1,37 +0,0 @@ -=============== -hadoop-cloudera -=============== - -Installs cloudera (cloudera-manager-agent cloudera-manager-daemons -cloudera-manager-server cloudera-manager-server-db-2 hadoop-hdfs-namenode -hadoop-hdfs-secondarynamenode hadoop-hdfs-datanode hadoop-yarn-resourcemanager -hadoop-yarn-nodemanager hadoop-mapreduce hadoop-mapreduce-historyserver) and -Java (oracle-j2sdk1.7) packages from cloudera repositories: -`cdh5 `_ and -`cm5 `_. - -Also installs Cloudera distribution of Apache Kafka for CDH version >= 5.5 from -Cloudera repository: `kafka `_. - -In order to create the Cloudera images with ``diskimage-create.sh``, use the -following syntax to select the ``cloudera`` plugin: - -.. sourcecode:: bash - - diskimage-create.sh -p cloudera - -Environment Variables ---------------------- - -The element can be configured by exporting variables using a -`environment.d` script. - -DIB_CDH_HDFS_ONLY - :Required: No - :Description: If set will install only the namenode and datanode - packages with their dependencies. - -DIB_CDH_MINOR_VERSION - :Required: No - :Description: If set will install minor version of CDH. Available minor - versions are 5.7.x. diff --git a/elements/hadoop-cloudera/element-deps b/elements/hadoop-cloudera/element-deps deleted file mode 100644 index 52ac5063..00000000 --- a/elements/hadoop-cloudera/element-deps +++ /dev/null @@ -1,4 +0,0 @@ -extjs -java -package-installs -ssh diff --git a/elements/hadoop-cloudera/environment.d/00-configure-extjs b/elements/hadoop-cloudera/environment.d/00-configure-extjs deleted file mode 100644 index f8d84197..00000000 --- a/elements/hadoop-cloudera/environment.d/00-configure-extjs +++ /dev/null @@ -1 +0,0 @@ -export EXTJS_DESTINATION_DIR="/var/lib/oozie" diff --git a/elements/hadoop-cloudera/environment.d/01-set-java-distro b/elements/hadoop-cloudera/environment.d/01-set-java-distro deleted file mode 100644 index 85cb17bd..00000000 --- a/elements/hadoop-cloudera/environment.d/01-set-java-distro +++ /dev/null @@ -1 +0,0 @@ -export DIB_JAVA_DISTRO="cloudera-jdk" diff --git a/elements/hadoop-cloudera/install.d/34-java b/elements/hadoop-cloudera/install.d/34-java deleted file mode 100755 index 5f479c4f..00000000 --- a/elements/hadoop-cloudera/install.d/34-java +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then - echo "Installing Oracle JDK for Cloudera" - - pkg="oracle-j2sdk1.7" - install-packages $pkg - - JDK_HOME="" - case $DISTRO_NAME in - centos7 | rhel7 ) - JDK_HOME=$(find /usr/java -name 'jdk*cloudera' -print) - ;; - ubuntu ) - JDK_HOME="/usr/lib/jvm/java-7-oracle-cloudera" - ;; - esac - - JRE_HOME=$JDK_HOME/jre - - setup-java-home $JRE_HOME $JDK_HOME -else - echo "Skip this step. JDK has already been installed for Spark." -fi diff --git a/elements/hadoop-cloudera/install.d/50-install-cloudera b/elements/hadoop-cloudera/install.d/50-install-cloudera deleted file mode 100755 index beac78c7..00000000 --- a/elements/hadoop-cloudera/install.d/50-install-cloudera +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -if [ "$DISTRO_NAME" = "ubuntu" ]; then - export DEBIAN_FRONTEND=noninteractive - export RUNLEVEL=1 -fi - -# Install the rest of CDH unless a limited HDFS install was requested -if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then - install-packages \ - cloudera-manager-agent \ - cloudera-manager-daemons \ - cloudera-manager-server \ - cloudera-manager-server-db-2 \ - hadoop-hdfs-secondarynamenode \ - hadoop-mapreduce \ - hadoop-mapreduce-historyserver \ - hadoop-yarn-nodemanager \ - hadoop-yarn-resourcemanager \ - hbase \ - hive-hcatalog \ - hive-metastore \ - hive-server2 \ - hive-webhcat-server \ - hue \ - oozie \ - spark-core \ - zookeeper \ - flume-ng \ - hadoop-kms \ - hbase-solr \ - impala \ - impala-server \ - impala-state-store \ - impala-catalog \ - impala-shell \ - keytrustee-keyprovider \ - sentry \ - solr-server \ - solr-doc \ - search \ - spark-history-server \ - sqoop2 \ - kafka \ - kafka-server -fi - -DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$DIB_CDH_VERSION.0} - -case "$DIB_CDH_VERSION" in - 5.7) - HADOOP_OPENSTACK_5_7_x_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.6.0-cdh$DIB_CDH_MINOR_VERSION/hadoop-openstack-2.6.0-cdh$DIB_CDH_MINOR_VERSION.jar" - ;; - 5.9) - HADOOP_OPENSTACK_5_9_x_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.6.0-cdh$DIB_CDH_MINOR_VERSION/hadoop-openstack-2.6.0-cdh$DIB_CDH_MINOR_VERSION.jar" - ;; - 5.11) - HADOOP_OPENSTACK_5_11_x_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.6.0-cdh$DIB_CDH_MINOR_VERSION/hadoop-openstack-2.6.0-cdh$DIB_CDH_MINOR_VERSION.jar" - ;; -esac -HADOOP_OPENSTACK_5_5_0_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.6.0-cdh5.5.0/hadoop-openstack-2.6.0-cdh5.5.0.jar" - -dest=/usr/lib/hadoop-mapreduce/hadoop-openstack.jar -case "$DIB_CDH_VERSION" in - 5.5) - wget -O $dest $HADOOP_OPENSTACK_5_5_0_URL - if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then - # Create links to keep backward version support. - ln -s /usr/lib/oozie/oozie-sharelib-yarn /usr/lib/oozie/oozie-sharelib-yarn.tar.gz - ln -s /usr/lib/oozie/oozie-sharelib-mr1 /usr/lib/oozie/oozie-sharelib-mr1.tar.gz - ln -s /usr/lib/oozie/oozie-sharelib-yarn.tar.gz /usr/lib/oozie/oozie-sharelib.tar.gz - fi - ;; - 5.7) - wget -O $dest $HADOOP_OPENSTACK_5_7_x_URL - if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then - # Create links to keep backward version support. - ln -s /usr/lib/oozie/oozie-sharelib-yarn /usr/lib/oozie/oozie-sharelib-yarn.tar.gz - ln -s /usr/lib/oozie/oozie-sharelib-mr1 /usr/lib/oozie/oozie-sharelib-mr1.tar.gz - ln -s /usr/lib/oozie/oozie-sharelib-yarn.tar.gz /usr/lib/oozie/oozie-sharelib.tar.gz - fi - ;; - 5.9) - wget -O $dest $HADOOP_OPENSTACK_5_9_x_URL - if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then - # Create links to keep backward version support. - ln -s /usr/lib/oozie/oozie-sharelib-yarn /usr/lib/oozie/oozie-sharelib-yarn.tar.gz - ln -s /usr/lib/oozie/oozie-sharelib-mr1 /usr/lib/oozie/oozie-sharelib-mr1.tar.gz - ln -s /usr/lib/oozie/oozie-sharelib-yarn.tar.gz /usr/lib/oozie/oozie-sharelib.tar.gz - fi - ;; - 5.11) - wget -O $dest $HADOOP_OPENSTACK_5_11_x_URL - if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then - # Create links to keep backward version support. - ln -s /usr/lib/oozie/oozie-sharelib-yarn /usr/lib/oozie/oozie-sharelib-yarn.tar.gz - ln -s /usr/lib/oozie/oozie-sharelib-mr1 /usr/lib/oozie/oozie-sharelib-mr1.tar.gz - ln -s /usr/lib/oozie/oozie-sharelib-yarn.tar.gz /usr/lib/oozie/oozie-sharelib.tar.gz - fi - ;; - *) - echo "Unhandled version $DIB_CDH_VERSION for hadoop-openstack.jar" - exit 1 -esac - -if [ "$DISTRO_NAME" = "ubuntu" ]; then - unset RUNLEVEL -fi diff --git a/elements/hadoop-cloudera/package-installs.yaml b/elements/hadoop-cloudera/package-installs.yaml deleted file mode 100644 index 0a12a18f..00000000 --- a/elements/hadoop-cloudera/package-installs.yaml +++ /dev/null @@ -1,6 +0,0 @@ -wget: - phase: pre-install.d -ntp: -hadoop-hdfs-datanode: -hadoop-hdfs-namenode: -# other packages are installed conditionally in install.d/50-install-cloudera diff --git a/elements/hadoop-cloudera/post-install.d/70-turn-services-off b/elements/hadoop-cloudera/post-install.d/70-turn-services-off deleted file mode 100755 index 62e2874b..00000000 --- a/elements/hadoop-cloudera/post-install.d/70-turn-services-off +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -for i in cloudera-scm-agent \ -cloudera-scm-server \ -cloudera-scm-server-db \ -hadoop-hdfs-datanode \ -hadoop-hdfs-namenode \ -hadoop-hdfs-secondarynamenode \ -hadoop-mapreduce-historyserver \ -hadoop-yarn-nodemanager \ -hadoop-yarn-resourcemanager \ -hive-metastore \ -hive-server2 \ -hive-webhcat-server \ -hue \ -oozie \ -postgresql -do - if [ "$DISTRO_NAME" = "ubuntu" ]; then - update-rc.d -f $i remove - else - chkconfig $i off - fi -done - -if [ $DIB_CDH_VERSION != "5.0" ]; then - for i in impala-catalog \ - impala-server \ - impala-state-store \ - solr-server \ - spark-history-server - do - if [ "$DISTRO_NAME" = "ubuntu" ]; then - update-rc.d -f $i remove - else - chkconfig $i off - fi - done -fi diff --git a/elements/hadoop-cloudera/pre-install.d/10-add-mirror b/elements/hadoop-cloudera/pre-install.d/10-add-mirror deleted file mode 100755 index 7b5c84c8..00000000 --- a/elements/hadoop-cloudera/pre-install.d/10-add-mirror +++ /dev/null @@ -1,304 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -function add_repo_5_5_0 { - case $DISTRO_NAME in - ubuntu ) - # Add repository with postgresql package (it's dependency of cloudera packages) - # Base image doesn't contain this repo - echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ trusty universe multiverse main' >> /etc/apt/sources.list - - # Cloudera repositories - echo 'deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh trusty-cdh5.5.0 contrib' > /etc/apt/sources.list.d/cdh5.list - echo 'deb-src http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh trusty-cdh5.5.0 contrib' >> /etc/apt/sources.list.d/cdh5.list - - wget -qO - http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh/archive.key | apt-key add - - - echo 'deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm trusty-cm5.5.0 contrib' > /etc/apt/sources.list.d/cm5.list - echo 'deb-src http://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm trusty-cm5.5.0 contrib' >> /etc/apt/sources.list.d/cm5.list - - wget -qO - http://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm/archive.key | apt-key add - - - wget -O /etc/apt/sources.list.d/kms.list http://archive.cloudera.com/navigator-keytrustee5/ubuntu/trusty/amd64/navigator-keytrustee/cloudera.list - wget -qO - http://archive.cloudera.com/navigator-keytrustee5/ubuntu/trusty/amd64/navigator-keytrustee/archive.key | apt-key add - - - # add Kafka repository - echo -e 'deb http://archive.cloudera.com/kafka/ubuntu/trusty/amd64/kafka/ trusty-kafka2.0.2 contrib' >> /etc/apt/sources.list - wget -qO - https://archive.cloudera.com/kafka/ubuntu/trusty/amd64/kafka/archive.key | apt-key add - - - #change repository priority - echo -e 'Package: zookeeper\nPin: origin "archive.cloudera.com"\nPin-Priority: 1001' > /etc/apt/preferences.d/cloudera-pin - - apt-get update - ;; - centos7 | rhel7 ) - releasever=7 - echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo - echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo "baseurl=http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/5.5.0/" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo "gpgkey = http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo - - echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo - echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo - echo "baseurl=http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/5.5.0/" >> /etc/yum.repos.d/cloudera-manager.repo - echo "gpgkey = http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-manager.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo - - echo '[navigator-keytrustee]' > /etc/yum.repos.d/kms.repo - echo "name=Cloudera's Distribution for navigator-Keytrustee, Version 5" >> /etc/yum.repos.d/kms.repo - - RETURN_CODE="$(curl -s -o /dev/null -w "%{http_code}" http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_MINOR_VERSION/)" - if [ "$RETURN_CODE" == "404" ]; then - echo "baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_VERSION/" >> /etc/yum.repos.d/kms.repo - else - echo "baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/kms.repo - fi - - echo "gpgkey = http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/kms.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/kms.repo - - echo "[cloudera-kafka]" > /etc/yum.repos.d/cloudera-kafka.repo - echo "name=Cloudera's Distribution for kafka, Version 2.0.2" >> /etc/yum.repos.d/cloudera-kafka.repo - echo "baseurl=http://archive.cloudera.com/kafka/redhat/$releasever/x86_64/kafka/2.0.2/" >> /etc/yum.repos.d/cloudera-kafka.repo - echo "gpgkey = http://archive.cloudera.com/kafka/redhat/$releasever/x86_64/kafka/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-kafka.repo - echo "gpgcheck = 1" >> /etc/yum.repos.d/cloudera-kafka.repo - - yum clean all - ;; - esac -} - -function add_repo_5_7_x { - case $DISTRO_NAME in - ubuntu ) - # Add repository with postgresql package (it's dependency of cloudera packages) - # Base image doesn't contain this repo - echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ trusty universe multiverse main' >> /etc/apt/sources.list - - # Cloudera repositories - echo "deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh trusty-cdh$DIB_CDH_MINOR_VERSION contrib" > /etc/apt/sources.list.d/cdh5.list - echo "deb-src http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh trusty-cdh$DIB_CDH_MINOR_VERSION contrib" >> /etc/apt/sources.list.d/cdh5.list - - wget -qO - http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh/archive.key | apt-key add - - - echo "deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm trusty-cm$DIB_CDH_MINOR_VERSION contrib" > /etc/apt/sources.list.d/cm5.list - echo "deb-src http://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm trusty-cm$DIB_CDH_MINOR_VERSION contrib" >> /etc/apt/sources.list.d/cm5.list - - wget -qO - http://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm/archive.key | apt-key add - - - wget -O /etc/apt/sources.list.d/kms.list http://archive.cloudera.com/navigator-keytrustee5/ubuntu/trusty/amd64/navigator-keytrustee/cloudera.list - wget -qO - http://archive.cloudera.com/navigator-keytrustee5/ubuntu/trusty/amd64/navigator-keytrustee/archive.key | apt-key add - - - # add Kafka repository - echo -e 'deb http://archive.cloudera.com/kafka/ubuntu/trusty/amd64/kafka/ trusty-kafka2.0.2 contrib' >> /etc/apt/sources.list - wget -qO - https://archive.cloudera.com/kafka/ubuntu/trusty/amd64/kafka/archive.key | apt-key add - - - #change repository priority - echo -e 'Package: zookeeper\nPin: origin "archive.cloudera.com"\nPin-Priority: 1001' > /etc/apt/preferences.d/cloudera-pin - - apt-get update - ;; - centos7 | rhel7 ) - releasever=7 - echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo - echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo "baseurl=http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo "gpgkey = http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo - - echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo - echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo - echo "baseurl=http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/cloudera-manager.repo - echo "gpgkey = http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-manager.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo - - echo '[navigator-keytrustee]' > /etc/yum.repos.d/kms.repo - echo "name=Cloudera's Distribution for navigator-Keytrustee, Version 5" >> /etc/yum.repos.d/kms.repo - - RETURN_CODE="$(curl -s -o /dev/null -w "%{http_code}" http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_MINOR_VERSION/)" - if [ "$RETURN_CODE" == "404" ]; then - echo "baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_VERSION/" >> /etc/yum.repos.d/kms.repo - else - echo "baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/kms.repo - fi - - echo "gpgkey = http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/kms.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/kms.repo - - echo "[cloudera-kafka]" > /etc/yum.repos.d/cloudera-kafka.repo - echo "name=Cloudera's Distribution for kafka, Version 2.0.2" >> /etc/yum.repos.d/cloudera-kafka.repo - echo "baseurl=http://archive.cloudera.com/kafka/redhat/$releasever/x86_64/kafka/2.0.2/" >> /etc/yum.repos.d/cloudera-kafka.repo - echo "gpgkey = http://archive.cloudera.com/kafka/redhat/$releasever/x86_64/kafka/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-kafka.repo - echo "gpgcheck = 1" >> /etc/yum.repos.d/cloudera-kafka.repo - - yum clean all - ;; - esac -} - -function add_repo_5_9_x { - case $DISTRO_NAME in - ubuntu ) - # Add repository with postgresql package (it's dependency of cloudera packages) - # Base image doesn't contain this repo - echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ trusty universe multiverse main' >> /etc/apt/sources.list - - # Cloudera repositories - echo "deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh trusty-cdh$DIB_CDH_MINOR_VERSION contrib" > /etc/apt/sources.list.d/cdh5.list - echo "deb-src http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh trusty-cdh$DIB_CDH_MINOR_VERSION contrib" >> /etc/apt/sources.list.d/cdh5.list - - wget -qO - http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh/archive.key | apt-key add - - - echo "deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm trusty-cm$DIB_CDH_MINOR_VERSION contrib" > /etc/apt/sources.list.d/cm5.list - echo "deb-src http://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm trusty-cm$DIB_CDH_MINOR_VERSION contrib" >> /etc/apt/sources.list.d/cm5.list - - wget -qO - http://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm/archive.key | apt-key add - - - wget -O /etc/apt/sources.list.d/kms.list http://archive.cloudera.com/navigator-keytrustee5/ubuntu/trusty/amd64/navigator-keytrustee/cloudera.list - wget -qO - http://archive.cloudera.com/navigator-keytrustee5/ubuntu/trusty/amd64/navigator-keytrustee/archive.key | apt-key add - - - # add Kafka repository - echo -e 'deb http://archive.cloudera.com/kafka/ubuntu/trusty/amd64/kafka/ trusty-kafka2.0.2 contrib' >> /etc/apt/sources.list - wget -qO - https://archive.cloudera.com/kafka/ubuntu/trusty/amd64/kafka/archive.key | apt-key add - - - #change repository priority - echo -e 'Package: zookeeper\nPin: origin "archive.cloudera.com"\nPin-Priority: 1001' > /etc/apt/preferences.d/cloudera-pin - - apt-get update - ;; - centos7 | rhel7 ) - releasever=7 - echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo - echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo "baseurl=http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo "gpgkey = http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo - - echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo - echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo - echo "baseurl=http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/cloudera-manager.repo - echo "gpgkey = http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-manager.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo - - echo '[navigator-keytrustee]' > /etc/yum.repos.d/kms.repo - echo "name=Cloudera's Distribution for navigator-Keytrustee, Version 5" >> /etc/yum.repos.d/kms.repo - - RETURN_CODE="$(curl -s -o /dev/null -w "%{http_code}" http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_MINOR_VERSION/)" - if [ "$RETURN_CODE" == "404" ]; then - echo "baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_VERSION/" >> /etc/yum.repos.d/kms.repo - else - echo "baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/kms.repo - fi - - echo "gpgkey = http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/kms.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/kms.repo - - echo "[cloudera-kafka]" > /etc/yum.repos.d/cloudera-kafka.repo - echo "name=Cloudera's Distribution for kafka, Version 2.0.2" >> /etc/yum.repos.d/cloudera-kafka.repo - echo "baseurl=http://archive.cloudera.com/kafka/redhat/$releasever/x86_64/kafka/2.0.2/" >> /etc/yum.repos.d/cloudera-kafka.repo - echo "gpgkey = http://archive.cloudera.com/kafka/redhat/$releasever/x86_64/kafka/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-kafka.repo - echo "gpgcheck = 1" >> /etc/yum.repos.d/cloudera-kafka.repo - - yum clean all - ;; - esac -} - -function add_repo_5_11_x { - case $DISTRO_NAME in - ubuntu ) - # Add repository with postgresql package (it's dependency of cloudera packages) - # Base image doesn't contain this repo - echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ xenial universe multiverse main' >> /etc/apt/sources.list - - # Cloudera repositories - echo "deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/xenial/amd64/cdh xenial-cdh$DIB_CDH_MINOR_VERSION contrib" > /etc/apt/sources.list.d/cdh5.list - echo "deb-src http://archive.cloudera.com/cdh5/ubuntu/xenial/amd64/cdh xenial-cdh$DIB_CDH_MINOR_VERSION contrib" >> /etc/apt/sources.list.d/cdh5.list - - wget -qO - http://archive.cloudera.com/cdh5/ubuntu/xenial/amd64/cdh/archive.key | apt-key add - - - echo "deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/xenial/amd64/cm xenial-cm$DIB_CDH_MINOR_VERSION contrib" > /etc/apt/sources.list.d/cm5.list - echo "deb-src http://archive.cloudera.com/cm5/ubuntu/xenial/amd64/cm xenial-cm$DIB_CDH_MINOR_VERSION contrib" >> /etc/apt/sources.list.d/cm5.list - - wget -qO - http://archive.cloudera.com/cm5/ubuntu/xenial/amd64/cm/archive.key | apt-key add - - - wget -O /etc/apt/sources.list.d/kms.list http://archive.cloudera.com/navigator-keytrustee5/ubuntu/xenial/amd64/navigator-keytrustee/cloudera.list - wget -qO - http://archive.cloudera.com/navigator-keytrustee5/ubuntu/xenial/amd64/navigator-keytrustee/archive.key | apt-key add - - - # add Kafka repository - echo -e 'deb http://archive.cloudera.com/kafka/ubuntu/xenial/amd64/kafka/ xenial-kafka2.2.0 contrib' >> /etc/apt/sources.list - wget -qO - https://archive.cloudera.com/kafka/ubuntu/xenial/amd64/kafka/archive.key | apt-key add - - - #change repository priority - echo -e 'Package: zookeeper\nPin: origin "archive.cloudera.com"\nPin-Priority: 1001' > /etc/apt/preferences.d/cloudera-pin - - apt-get update - ;; - centos7 | rhel7 ) - releasever=7 - echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo - echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo "baseurl=http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo "gpgkey = http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo - - echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo - echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo - echo "baseurl=http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/cloudera-manager.repo - echo "gpgkey = http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-manager.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo - - echo '[navigator-keytrustee]' > /etc/yum.repos.d/kms.repo - echo "name=Cloudera's Distribution for navigator-Keytrustee, Version 5" >> /etc/yum.repos.d/kms.repo - - RETURN_CODE="$(curl -s -o /dev/null -w "%{http_code}" http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_MINOR_VERSION/)" - if [ "$RETURN_CODE" == "404" ]; then - echo "baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_VERSION/" >> /etc/yum.repos.d/kms.repo - else - echo "baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/kms.repo - fi - - echo "gpgkey = http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/kms.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/kms.repo - - echo "[cloudera-kafka]" > /etc/yum.repos.d/cloudera-kafka.repo - echo "name=Cloudera's Distribution for kafka, Version 2.2.0" >> /etc/yum.repos.d/cloudera-kafka.repo - echo "baseurl=http://archive.cloudera.com/kafka/redhat/$releasever/x86_64/kafka/2.2.0/" >> /etc/yum.repos.d/cloudera-kafka.repo - echo "gpgkey = http://archive.cloudera.com/kafka/redhat/$releasever/x86_64/kafka/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-kafka.repo - echo "gpgcheck = 1" >> /etc/yum.repos.d/cloudera-kafka.repo - - yum clean all - ;; - esac -} - -DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$DIB_CDH_VERSION.0} - -# Call version-specific script to install the desired version of CDH -case "$DIB_CDH_VERSION" in - 5.5) - echo "Installing CDH Version $DIB_CDH_VERSION..." - add_repo_5_5_0 - ;; - 5.7) - echo "Installing CDH Version $DIB_CDH_MINOR_VERSION..." - add_repo_5_7_x - ;; - 5.9) - echo "Installing CDH Version $DIB_CDH_MINOR_VERSION..." - add_repo_5_9_x - ;; - 5.11) - echo "Installing CDH Version $DIB_CDH_MINOR_VERSION..." - add_repo_5_11_x - ;; - *) - echo "Invalid CDH Version : $DIB_CDH_VERSION specified, exiting install." - exit 1 -esac diff --git a/elements/hadoop-mapr/README.rst b/elements/hadoop-mapr/README.rst deleted file mode 100644 index 1a4534b9..00000000 --- a/elements/hadoop-mapr/README.rst +++ /dev/null @@ -1,44 +0,0 @@ -=========== -hadoop-mapr -=========== - -Creates images with local mirrors of MapR repositories: -`core `_ and -`ecosystem `_. -Installs `OpenJDK `_ and -`Scala `_. - -In order to create the MapR images with ``diskimage-create.sh``, use the -following syntax to select the ``MapR`` plugin: - -.. sourcecode:: bash - - diskimage-create.sh -p mapr [-i ubuntu|centos7|rhel7] [-r 5.1.0 | 5.2.0] - -In order to speed up image creation process you can download archives with MapR -repositories and specify environment variables: -``DIB_MAPR_CORE_DEB_REPO``, ``DIB_MAPR_CORE_RPM_REPO``, -``DIB_MAPR_ECO_DEB_REPO``, ``DIB_MAPR_ECO_RPM_REPO``. - -For example: - -.. sourcecode:: bash - - export DIB_MAPR_CORE_DEB_REPO="file:///mapr-v5.2.0GA.deb.tgz" - export DIB_MAPR_CORE_RPM_REPO="file:///mapr-v5.2.0GA.rpm.tgz" - export DIB_MAPR_ECO_DEB_REPO="http:///mapr-ecosystem.deb.tgz" - export DIB_MAPR_ECO_RPM_REPO="http:///mapr-ecosystem.rpm.tgz" - diskimage-create.sh -p mapr -r 5.2.0 - -Environment Variables ---------------------- - -DIB_MAPR_VERSION - :Required: Yes - :Description: Version of MapR to install. - :Example: ``DIB_MAPR_VERSION=5.2.0`` - -SCALA_VERSION - :Required: No - :Description: Force a specific version of Scala (disable autodetection). - :Example: ``SCALA_VERSION=2.12.2`` diff --git a/elements/hadoop-mapr/element-deps b/elements/hadoop-mapr/element-deps deleted file mode 100644 index 5dcf86aa..00000000 --- a/elements/hadoop-mapr/element-deps +++ /dev/null @@ -1,4 +0,0 @@ -extjs -java -mysql -package-installs diff --git a/elements/hadoop-mapr/environment.d/00-configure-extjs b/elements/hadoop-mapr/environment.d/00-configure-extjs deleted file mode 100644 index 10de1c84..00000000 --- a/elements/hadoop-mapr/environment.d/00-configure-extjs +++ /dev/null @@ -1,2 +0,0 @@ -export EXTJS_DESTINATION_DIR="/opt/mapr-repository/" -export EXTJS_NO_UNPACK=1 diff --git a/elements/hadoop-mapr/install.d/40-mapr-dependencies b/elements/hadoop-mapr/install.d/40-mapr-dependencies deleted file mode 100755 index 0a5d53ac..00000000 --- a/elements/hadoop-mapr/install.d/40-mapr-dependencies +++ /dev/null @@ -1,155 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -echo "START: installing MapR core dependencies" - -if [ "$DISTRO_NAME" = "ubuntu" ]; then - # Required for libicu48 - cat >> /etc/apt/sources.list.d/security_repo.list << EOF -deb http://security.ubuntu.com/ubuntu precise-security main -EOF - # Required for libmysqlclient16 - cat >> /etc/apt/sources.list.d/security_repo.list << EOF -deb http://old-releases.ubuntu.com/ubuntu lucid-security main -EOF - apt-get update - install-packages binutils \ - daemon \ - dpkg-dev \ - dpkg-repack \ - gcc \ - gcc-4.8 \ - gcc-doc \ - gcc-multilib \ - iputils-arping \ - libasan0 \ - libatomic1 \ - libc-dev-bin \ - libc6 \ - libc6-dev \ - libcrypt-passwdmd5-perl \ - libgcc-4.8-dev \ - libgomp1 \ - libgssglue1 \ - libicu48 \ - libitm1 \ - libmysqlclient-dev \ - libmysqlclient16 \ - libmysqlclient18 \ - libnfsidmap2 \ - libquadmath0 \ - libsasl2-dev \ - libsasl2-modules-gssapi-mit \ - libssl0.9.8 \ - libtirpc1 \ - libtsan0 \ - libxslt1.1 \ - linux-libc-dev \ - manpages-dev \ - mtools \ - mysql-common \ - nfs-common \ - open-iscsi \ - openjdk-6-jre \ - rpcbind \ - sdparm \ - syslinux \ - syslinux-common \ - unzip \ - watchdog \ - zlib1g-dev \ - zip -elif [ "$DISTRO_NAME" = "centos7" -o "$DISTRO_NAME" = "rhel7" ]; then - install-packages cdparanoia-libs \ - cups \ - cups-libs \ - createrepo \ - cvs \ - cyrus-sasl-gssapi \ - cyrus-sasl-plain \ - db4-cxx \ - db4-devel \ - foomatic \ - foomatic-db \ - foomatic-db-filesystem \ - foomatic-db-ppds \ - gdbm-devel \ - gettext \ - ghostscript \ - ghostscript-fonts \ - glibc \ - glibc-common \ - glibc-devel \ - glibc-headers \ - gstreamer \ - gstreamer-plugins-base \ - gstreamer-tools \ - hdparm \ - irqbalance \ - iso-codes \ - kernel-headers \ - lcms-libs \ - libXt \ - libXv \ - libXxf86vm \ - libgomp \ - libgssglue \ - libgudev1 \ - libicu \ - libmng \ - liboil \ - libtheora \ - libtirpc \ - libvisual \ - libxslt \ - mesa-dri-drivers \ - mesa-dri-filesystem \ - mesa-dri1-drivers \ - mesa-libGL \ - mesa-libGLU \ - mesa-private-llvm \ - mtools \ - nc \ - numactl \ - openjpeg-libs \ - patch \ - pax \ - perl-CGI \ - perl-ExtUtils-MakeMaker \ - perl-ExtUtils-ParseXS \ - perl-Test-Harness \ - perl-Test-Simple \ - perl-devel \ - phonon-backend-gstreamer \ - poppler \ - poppler-data \ - poppler-utils \ - portreserve \ - qt \ - qt-sqlite \ - qt-x11 \ - qt3 \ - redhat-lsb \ - redhat-lsb-compat \ - redhat-lsb-core \ - redhat-lsb-graphics \ - redhat-lsb-printing \ - rpcbind \ - sdparm \ - soci \ - soci-mysql \ - syslinux \ - syslinux-nonlinux \ - unzip \ - urw-fonts \ - yum-utils \ - xml-common \ - zip -fi - -echo "END: installing MapR core dependencies" diff --git a/elements/hadoop-mapr/install.d/41-scala b/elements/hadoop-mapr/install.d/41-scala deleted file mode 100755 index ea83e91f..00000000 --- a/elements/hadoop-mapr/install.d/41-scala +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -echo "START: installing Scala" - -#Current available version -DEF_VERSION="2.11.6" - -RETURN_CODE="$(curl -s -L -o /dev/null -w "%{http_code}" https://www.scala-lang.org/)" - -if [ "$RETURN_CODE" != "200" ]; then - echo "https://www.scala-lang.org is unreachable" && exit 1 -fi - -if [ -n "${SCALA_VERSION:-}" ]; then - VERSION=${SCALA_VERSION} -elif [ "trusty" == "${DIB_RELEASE:-}" ]; then - # scale >= 2.12 for ubuntu depends on openjdk-8, not available on trusty - VERSION=${DEF_VERSION} -else - VERSION="$(curl -s -L --fail https://www.scala-lang.org| tr -d '\n' | sed 's/^.*[^0-9]\+\([0-9\.\?]\+\)<.\+$/\1/')" - - if [ $? != 0 -o -z "${VERSION}" ]; then - echo "Installing default version $DEF_VERSION" - VERSION=${DEF_VERSION} - fi -fi - -PKG=scala-${VERSION} - -URL="https://downloads.lightbend.com/scala/${VERSION}" - -if [ "$DISTRO_NAME" = "ubuntu" ]; then - wget -N ${URL}/${PKG}.deb - dpkg -i ${PKG}.deb - rm ${PKG}.deb -elif [ "$DISTRO_NAME" = "centos7" -o "$DISTRO_NAME" = "rhel7" ]; then - rpm -Uhv ${URL}/${PKG}.rpm -fi - -echo "END: installing Scala" - diff --git a/elements/hadoop-mapr/install.d/43-mapr-core-repository b/elements/hadoop-mapr/install.d/43-mapr-core-repository deleted file mode 100755 index b23033ec..00000000 --- a/elements/hadoop-mapr/install.d/43-mapr-core-repository +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -source "$(dirname $0)/../resources/package_utils.sh" - -echo "START: installing MapR core repository" - -get_repo_url() { - local repo_url="" - - if [ "$DISTRO_NAME" = "ubuntu" ]; then - repo_url="${DIB_MAPR_CORE_DEB_REPO:-http://package.mapr.com/releases/v${DIB_MAPR_VERSION}/ubuntu/mapr-v${DIB_MAPR_VERSION}GA.deb.tgz}" - elif [ "$DISTRO_NAME" = "centos7" -o "$DISTRO_NAME" = "rhel7" ]; then - repo_url="${DIB_MAPR_CORE_RPM_REPO:-http://package.mapr.com/releases/v${DIB_MAPR_VERSION}/redhat/mapr-v${DIB_MAPR_VERSION}GA.rpm.tgz}" - fi - - echo "$repo_url" -} - -MAPR_REPO_URL="$(get_repo_url)" -MAPR_REPO_DIR="/opt/mapr-repository/core" -MAPR_REPO_NAME="mapr_core" - -echo "Downloading MapR repository archive" -mkdir -p "$MAPR_REPO_DIR" && curl "$MAPR_REPO_URL" | tar -xz -C "$MAPR_REPO_DIR" - -echo "Creating local repository" -create_repo "$MAPR_REPO_DIR" - -echo "Adding MapR repository" -add_local_repo "$MAPR_REPO_NAME" "$MAPR_REPO_DIR" - -echo "END: installing MapR core repository" diff --git a/elements/hadoop-mapr/install.d/44-mapr-eco-repository b/elements/hadoop-mapr/install.d/44-mapr-eco-repository deleted file mode 100755 index 013ab54e..00000000 --- a/elements/hadoop-mapr/install.d/44-mapr-eco-repository +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -VERSIONS_PY="$(dirname $0)/../resources/versions.py" - -source "$(dirname $0)/../resources/package_utils.sh" - -echo "START: installing MapR ecosystem repository" - -get_repo_url() { - local repo_url="" - - if [ "$DISTRO_NAME" = "ubuntu" ]; then - case "$DIB_MAPR_VERSION" in - "5.1.0") - repo_url="${DIB_MAPR_ECO_DEB_REPO:-http://package.mapr.com/releases/ecosystem-5.x/ubuntu binary/}" - ;; - "5.2.0") - repo_url="${DIB_MAPR_ECO_DEB_REPO:-http://package.mapr.com/releases/MEP/MEP-2.0.0/ubuntu/ binary trusty}" - ;; - esac - elif [ "$DISTRO_NAME" = "centos7" -o "$DISTRO_NAME" = "rhel7" ]; then - case "$DIB_MAPR_VERSION" in - "5.1.0") - repo_url="${DIB_MAPR_ECO_RPM_REPO:-http://package.mapr.com/releases/ecosystem-5.x/redhat}" - ;; - "5.2.0") - repo_url="${DIB_MAPR_ECO_RPM_REPO:-http://package.mapr.com/releases/MEP/MEP-2.0.0/redhat}" - ;; - esac - fi - - echo "$repo_url" -} - -RPM_REPO="${DIB_MAPR_ECO_RPM_REPO:-}" -DEB_REPO="${DIB_MAPR_ECO_DEB_REPO:-}" - -MAPR_REPO_DIR="/opt/mapr-repository/ecosystem" -MAPR_REPO_URL="$(get_repo_url)" -MAPR_REPO_NAME="mapr_ecosystem" -MAPR_PKG_GROUPS="$(dirname $0)/../resources/packages.json" - -if [ -f $(dirname $0)/../resources/spec_${DIB_MAPR_VERSION}_${DISTRO_NAME}.json ]; then - MAPR_SPEC="$(dirname $0)/../resources/spec_${DIB_MAPR_VERSION}_${DISTRO_NAME}.json" -else - MAPR_SPEC="$(dirname $0)/../resources/spec_${DIB_MAPR_VERSION}.json" -fi - -echo "Creating local MapR ecosystem repository" -localize_repo "$MAPR_REPO_NAME" "$MAPR_REPO_URL" "$MAPR_PKG_GROUPS" "$MAPR_SPEC" "$MAPR_REPO_DIR" -echo $MAPR_SPEC - -echo "END: installing MapR ecosystem repository" diff --git a/elements/hadoop-mapr/install.d/45-epel-dependencies b/elements/hadoop-mapr/install.d/45-epel-dependencies deleted file mode 100755 index 3f671a54..00000000 --- a/elements/hadoop-mapr/install.d/45-epel-dependencies +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -if [ "$DISTRO_NAME" = "rhel7" -o "$DISTRO_NAME" = "centos7" ]; then - yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm - yum install -y soci soci-mysql - yum remove -y epel-release -fi diff --git a/elements/hadoop-mapr/package-installs.yaml b/elements/hadoop-mapr/package-installs.yaml deleted file mode 100644 index a3aacbce..00000000 --- a/elements/hadoop-mapr/package-installs.yaml +++ /dev/null @@ -1,2 +0,0 @@ -curl: -wget: diff --git a/elements/hadoop-mapr/post-install.d/99-requiretty b/elements/hadoop-mapr/post-install.d/99-requiretty deleted file mode 100755 index c961661c..00000000 --- a/elements/hadoop-mapr/post-install.d/99-requiretty +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -if [ "$DISTRO_NAME" = "centos7" -o "$DISTRO_NAME" = "rhel7" ]; then - sed '/^Defaults requiretty*/ s/^/#/' -i /etc/sudoers -fi - diff --git a/elements/hadoop-mapr/post-install.d/99-update-repository b/elements/hadoop-mapr/post-install.d/99-update-repository deleted file mode 100755 index f39496a4..00000000 --- a/elements/hadoop-mapr/post-install.d/99-update-repository +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -if [ "$DISTRO_NAME" = "ubuntu" ]; then - apt-get update -elif [ "$DISTRO_NAME" = "centos7" -o "$DISTRO_NAME" = "rhel7" ]; then - yum clean all && yum repolist -fi diff --git a/elements/hadoop-mapr/resources/package_utils.sh b/elements/hadoop-mapr/resources/package_utils.sh deleted file mode 100644 index 8ebd2599..00000000 --- a/elements/hadoop-mapr/resources/package_utils.sh +++ /dev/null @@ -1,226 +0,0 @@ -# execute_in_directory -execute_in_directory() { - local directory="$(readlink -f "$1")"; shift - local cmd="$*" - - pushd "$directory" && eval "$cmd" && popd -} - -# get_distro -get_distro() { - echo "$DISTRO_NAME" -} - -# download_apt_package [version] [directory] -download_apt_package() { - local package="$1" - local version="${2:-}" - local directory="${3:-$(pwd)}" - local package_spec="$package${version:+=$version*}" - - execute_in_directory "$directory" apt-get --allow-unauthenticated download "$package_spec" -} - -# download_yum_package [version] [directory] -download_yum_package() { - local package="$1" - local version="${2:-}" - local directory="${3:-$(pwd)}" - local package_spec="$package${version:+-$version*}" - - yumdownloader --destdir "$directory" "$package_spec" -} - -# download_package [version] [directory] [distro] -download_package() { - local package="$1" - local version="${2:-}" - local directory="${3:-$(pwd)}" - local distro="${4:-$(get_distro)}" - - if [[ "$distro" == "ubuntu" ]]; then - download_apt_package "$package" "$version" "$directory" - elif [[ "$distro" == "centos7" || "$distro" == "rhel7" ]]; then - download_yum_package "$package" "$version" "$directory" - fi -} - -# get_packages [version_separator] -get_packages() { - local package_groups_file="$1" - local spec_file="$2" - local version_separator="${3:-:}" - - python "$VERSIONS_PY" --separator "$version_separator" "$package_groups_file" "$spec_file" -} - -# download_packages [directory] [distro] -download_packages() { - local package_groups_file="$1" - local spec_file="$2" - local directory="${3:-$(pwd)}" - local distro="${4:-$(get_distro)}" - local version_separator=":" - - local packages="$(get_packages "$package_groups_file" "$spec_file" "$version_separator")" - for package in $packages; do - IFS="$version_separator" read -ra package_version <<< "$package" - download_package "${package_version[@]}" "$directory" "$distro" - done -} - -# create_apt_repo -create_apt_repo() { - local directory="$(readlink -f "$1")" - local binary_dir="$directory/binary" - local packages_gz="$binary_dir/Packages.gz" - - mkdir -p "$binary_dir" - execute_in_directory "$directory" "dpkg-scanpackages -m . /dev/null | gzip -9c > $packages_gz" -} - -# create_yum_repo -create_yum_repo() { - local directory="$(readlink -f "$1")" - - createrepo "$directory" -} - -# create_repo [distro] -create_repo() { - local directory="$(readlink -f "$1")" - local distro="${2:-$(get_distro)}" - - if [[ "$distro" == "ubuntu" ]]; then - create_apt_repo "$directory" - elif [[ "$distro" == "centos7" || "$distro" == "rhel7" ]]; then - create_yum_repo "$directory" - fi -} - -# add_apt_repo -add_apt_repo() { - local repo_name="$1" - local repo_url="$2" - local repo="deb $repo_url" - local repo_path="/etc/apt/sources.list.d/$repo_name.list" - - echo "$repo" > "$repo_path" && apt-get update -} - -# add_yum_repo -add_yum_repo() { - local repo_name="$1" - local repo_url="$2" - local repo_path="/etc/yum.repos.d/$repo_name.repo" - - cat > "$repo_path" << EOF -[$repo_name] -name=$repo_name -baseurl=$repo_url -enabled=1 -gpgcheck=0 -protect=1 -EOF - yum clean all && rm -rf /var/cache/yum/* && yum check-update -} - -# add_repo [distro] -add_repo() { - local repo_name="$1" - local repo_url="$2" - local distro="${3:-$(get_distro)}" - - if [[ "$distro" == "ubuntu" ]]; then - add_apt_repo "$repo_name" "$repo_url" - elif [[ "$distro" == "centos7" || "$distro" == "rhel7" ]]; then - add_yum_repo "$repo_name" "$repo_url" - fi -} - -# add_local_apt_repo -add_local_apt_repo() { - local repo_name="$1" - local directory="$(readlink -f "$2")" - local repo_url="file:$directory binary/" - - add_apt_repo "$repo_name" "$repo_url" -} - -# add_local_yum_repo -add_local_yum_repo() { - local repo_name="$1" - local directory="$(readlink -f "$2")" - local repo_url="file://$directory" - - add_yum_repo "$repo_name" "$repo_url" -} - -# add_local_repo [distro] -add_local_repo() { - local repo_name="$1" - local directory="$(readlink -f "$2")" - local distro="${3:-$(get_distro)}" - - if [[ "$distro" == "ubuntu" ]]; then - add_local_apt_repo "$repo_name" "$directory" - elif [[ "$distro" == "centos7" || "$distro" == "rhel7" ]]; then - add_local_yum_repo "$repo_name" "$directory" - fi -} - -# remove_apt_repo -remove_apt_repo() { - local repo_name="$1" - local repo_path="/etc/apt/sources.list.d/$repo_name.list" - - rm "$repo_path" && apt-get update -} - -# remove_yum_repo -remove_yum_repo() { - local repo_name="$1" - local repo_path="/etc/yum.repos.d/$repo_name.repo" - - rm "$repo_path" -} - -# remove_repo [distro] -remove_repo() { - local repo_name="$1" - local distro="${2:-$(get_distro)}" - - if [[ "$distro" == "ubuntu" ]]; then - remove_apt_repo "$repo_name" - elif [[ "$distro" == "centos7" || "$distro" == "rhel7" ]]; then - remove_yum_repo "$repo_name" - fi -} - -# create_local_repo -create_local_repo() { - local repo_name="$1" - local repo_url="$2" - local package_groups_file="$3" - local spec_file="$4" - local directory="$5" - - add_repo "$repo_name" "$repo_url" - mkdir -p "$directory" && directory="$(readlink -f "$directory")" - download_packages "$package_groups_file" "$spec_file" "$directory" - remove_repo "$repo_name" - create_repo "$directory" -} - -# localize_repo -localize_repo() { - local repo_name="$1" - local repo_url="$2" - local package_groups_file="$3" - local spec_file="$4" - local directory="$5" - - mkdir -p "$directory" && directory="$(readlink -f "$directory")" - create_local_repo "$repo_name" "$repo_url" "$package_groups_file" "$spec_file" "$directory" - add_local_repo "$repo_name" "$directory" -} diff --git a/elements/hadoop-mapr/resources/packages.json b/elements/hadoop-mapr/resources/packages.json deleted file mode 100644 index b1cb5d0f..00000000 --- a/elements/hadoop-mapr/resources/packages.json +++ /dev/null @@ -1,140 +0,0 @@ -{ - "asynchbase": { - "all": [ - "mapr-asynchbase" - ] - }, - "drill": { - "all": [ - "mapr-drill" - ] - }, - "flume": { - "all": [ - "mapr-flume" - ] - }, - "hbase": { - "all": [ - "mapr-hbase", - "mapr-hbase-internal", - "mapr-hbase-master", - "mapr-hbase-regionserver", - "mapr-hbasethrift", - "mapr-hbase-rest" - ], - "0.98.12": [ - "mapr-hbase", - "mapr-hbase-internal", - "mapr-hbase-master", - "mapr-hbase-regionserver", - "mapr-hbasethrift", - "mapr-libhbase", - "mapr-hbase-rest" - ], - "1.1.1": [ - "mapr-hbase", - "mapr-hbase-internal", - "mapr-hbase-master", - "mapr-hbase-regionserver", - "mapr-hbasethrift", - "mapr-libhbase", - "mapr-hbase-rest" - ] - }, - "hive": { - "all": [ - "mapr-hive", - "mapr-hivemetastore", - "mapr-hiveserver2" - ] - }, - "httpfs": { - "all": [ - "mapr-httpfs" - ] - }, - "hue": { - "all": [ - "mapr-hue", - "mapr-hue-base", - "mapr-hue-livy" - ], - "3.10.0": [ - "mapr-hue", - "mapr-hue-livy" - ] - }, - "impala": { - "all": [ - "mapr-impala", - "mapr-impala-catalog", - "mapr-impala-server", - "mapr-impala-statestore", - "mapr-impala-udf" - ] - }, - "mahout": { - "all": [ - "mapr-mahout" - ] - }, - "oozie": { - "all": [ - "mapr-oozie", - "mapr-oozie-internal" - ] - }, - "pig": { - "all": [ - "mapr-pig" - ] - }, - "sentry": { - "all": [ - "mapr-sentry" - ] - }, - "spark": { - "all": [ - "mapr-spark", - "mapr-spark-historyserver", - "mapr-spark-master" - ] - }, - "sqoop": { - "all": [ - "mapr-sqoop2-client", - "mapr-sqoop2-server" - ] - }, - "storm": { - "all": [ - "mapr-storm", - "mapr-storm-ui", - "mapr-storm-nimbus", - "mapr-storm-supervisor" - ] - }, - "tez": { - "all": [ - "mapr-tez" - ] - }, - "kafka": { - "all": [ - "mapr-kafka" - ] - }, - "kafka-connect": { - "all": [ - "mapr-kafka-connect-hdfs", - "mapr-kafka-connect-jdbc" - ] - }, - "kafka-rest": { - "all": [ - "mapr-kafka-rest" - ] - } -} diff --git a/elements/hadoop-mapr/resources/spec_5.1.0.json b/elements/hadoop-mapr/resources/spec_5.1.0.json deleted file mode 100644 index 11ce5be0..00000000 --- a/elements/hadoop-mapr/resources/spec_5.1.0.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "drill": [ - "1.1.0", - "1.2.0", - "1.4.0" - ], - "flume": [ - "1.5.0", - "1.6.0" - ], - "hbase": [ - "0.98.9", - "0.98.12" - ], - "hive": [ - "0.13", - "1.0", - "1.2" - ], - "httpfs": [ - "1.0" - ], - "hue": [ - "3.8.1", - "3.9.0" - ], - "impala": [ - "1.4.1" - ], - "mahout": [ - "0.10.0" - ], - "oozie": [ - "4.2.0" - ], - "pig": [ - "0.14", - "0.15" - ], - "sqoop": [ - "2.0.0" - ], - "spark": [ - "1.5.2" - ] -} diff --git a/elements/hadoop-mapr/resources/spec_5.2.0.json b/elements/hadoop-mapr/resources/spec_5.2.0.json deleted file mode 100644 index 744971bb..00000000 --- a/elements/hadoop-mapr/resources/spec_5.2.0.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "drill": [ - "1.9.0" - ], - "flume": [ - "1.6.0" - ], - "hbase": [ - "1.1.1" - ], - "hive": [ - "1.2" - ], - "httpfs": [ - "1.0" - ], - "hue": [ - "3.10.0" - ], - "impala": [ - "2.5.0" - ], - "mahout": [ - "0.12.0" - ], - "oozie": [ - "4.2.0" - ], - "pig": [ - "0.16" - ], - "sqoop": [ - "2.0.0" - ], - "spark": [ - "2.0.1" - ], - "sentry": [ - "1.6.0" - ], - "kafka": [ - "0.9.0" - ], - "kafka-connect": [ - "2.0.1" - ], - "kafka-rest": [ - "2.0.1" - ] -} \ No newline at end of file diff --git a/elements/hadoop-mapr/resources/spec_5.2.0_ubuntu.json b/elements/hadoop-mapr/resources/spec_5.2.0_ubuntu.json deleted file mode 100644 index 43270512..00000000 --- a/elements/hadoop-mapr/resources/spec_5.2.0_ubuntu.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "drill": [ - "1.9.0" - ], - "flume": [ - "1.6.0" - ], - "hbase": [ - "1.1.1" - ], - "hive": [ - "1.2" - ], - "httpfs": [ - "1.0" - ], - "hue": [ - "3.10.0" - ], - "mahout": [ - "0.12.0" - ], - "oozie": [ - "4.2.0" - ], - "pig": [ - "0.16" - ], - "sqoop": [ - "2.0.0" - ], - "spark": [ - "2.0.1" - ], - "sentry": [ - "1.6.0" - ], - "kafka": [ - "0.9.0" - ], - "kafka-connect": [ - "2.0.1" - ], - "kafka-rest": [ - "2.0.1" - ] -} diff --git a/elements/hadoop-mapr/resources/versions.py b/elements/hadoop-mapr/resources/versions.py deleted file mode 100644 index 8a5589c8..00000000 --- a/elements/hadoop-mapr/resources/versions.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (c) 2015, MapR Technologies -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -import argparse -import json -import sys - -_GROUP_VERSION_SEPARATOR = "," -_ALL_GROUP_VERSION = "all" - - -def _build_parser(): - parser = argparse.ArgumentParser() - - parser.add_argument("packages", help="path to the packages.json") - parser.add_argument("spec", help="path to the spec.json") - parser.add_argument("--separator", default=":", - help="separator between package name" - " and version in output") - - return parser - - -def _load_json(path): - with open(path) as json_file: - return json.load(json_file) - - -def _version_matches(version, group_version): - for gv in group_version.split(_GROUP_VERSION_SEPARATOR): - if version.startswith(gv): - return True - - return False - - -def _get_packages(version, group_spec): - for group_version in group_spec: - if _version_matches(version, group_version): - return group_spec[group_version] - - return group_spec[_ALL_GROUP_VERSION] - - -def _get_package_versions(spec, package_groups): - return [(package, version) - for pg_name, versions in spec.items() - for version in versions - for package in _get_packages(version, package_groups[pg_name])] - - -parser = _build_parser() - - -def main(args=None): - args = parser.parse_args(args or sys.argv[1:]) - - spec = _load_json(args.spec) - package_groups = _load_json(args.packages) - separator = args.separator - - package_versions = _get_package_versions(spec, package_groups) - package_format = "%s" + separator + "%s\n" - package_versions = map(lambda pv: package_format % pv, package_versions) - - sys.stdout.writelines(package_versions) - - -if __name__ == "__main__": - main() diff --git a/elements/hadoop-mapr/root.d/0-mapr-check-version b/elements/hadoop-mapr/root.d/0-mapr-check-version deleted file mode 100755 index 026b27c9..00000000 --- a/elements/hadoop-mapr/root.d/0-mapr-check-version +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -echo "MapR version ${DIB_MAPR_VERSION}" - -if [ -z ${DIB_MAPR_VERSION} ]; then - echo "DIB_MAPR_VERSION not set.\nExit" - exit 1 -fi diff --git a/elements/hadoop/README.rst b/elements/hadoop/README.rst deleted file mode 100644 index 3d3a0682..00000000 --- a/elements/hadoop/README.rst +++ /dev/null @@ -1,36 +0,0 @@ -====== -hadoop -====== - -Installs Java and Hadoop, configures SSH. - -HOWTO build Hadoop Native Libs ------------------------------- - -- Install: *jdk >= 6*, *maven*, *cmake* and *protobuf >= 2.5.0* - -- Get Hadoop source code: - - .. code:: bash - - wget http://archive.apache.org/dist/hadoop/core/hadoop-2.7.1/hadoop-2.7.1-src.tar.gz - -- Unpack source: - - .. code:: bash - - tar xvf hadoop-2.7.1-src.tar.gz - -- Build Hadoop: - - .. code:: bash - - cd hadoop-2.7.1-src - mvn package -Pdist,native -DskipTests - -- Create tarball with Hadoop Native Libs: - - .. code:: bash - - cd hadoop-dist/target/hadoop-2.7.1/lib - tar -czvf hadoop-native-libs-2.7.1.tar.gz native diff --git a/elements/hadoop/element-deps b/elements/hadoop/element-deps deleted file mode 100644 index 217cbbe3..00000000 --- a/elements/hadoop/element-deps +++ /dev/null @@ -1,5 +0,0 @@ -cache-url -java -package-installs -sahara-version -ssh diff --git a/elements/hadoop/install.d/40-setup-hadoop b/elements/hadoop/install.d/40-setup-hadoop deleted file mode 100755 index e58a2822..00000000 --- a/elements/hadoop/install.d/40-setup-hadoop +++ /dev/null @@ -1,137 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -function install_hadoop_v2 { - case "$DIB_HADOOP_VERSION" in - "2.7.1") - hadoop_native_libs_url="${HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL}" - ;; - "2.7.5") - hadoop_native_libs_url="${HADOOP_V2_7_5_NATIVE_LIBS_DOWNLOAD_URL}" - ;; - "2.8.2") - hadoop_native_libs_url="${HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL}" - ;; - "3.0.1") - hadoop_native_libs_url="${HADOOP_V3_0_1_NATIVE_LIBS_DOWNLOAD_URL}" - ;; - *) - echo "Invalid DIB_HADOOP_VERSION: $DIB_HADOOP_VERSION" - exit 1 - ;; - esac - - package="hadoop-$DIB_HADOOP_VERSION.tar.gz" - - echo "Installing hadoop" - - INSTALL_DIR="/opt" - HADOOP_HOME="/opt/hadoop" - mkdir -p "$INSTALL_DIR" - tar xvf "$tmp_dir/$package" -C "$INSTALL_DIR" - ln -s "$INSTALL_DIR/hadoop-$DIB_HADOOP_VERSION" "$HADOOP_HOME" - chown -R hadoop:hadoop "$INSTALL_DIR/hadoop-$DIB_HADOOP_VERSION" - chown -R hadoop:hadoop "$HADOOP_HOME" - rm -r $tmp_dir - - echo "Inject Hadoop native libs" - rm -r "$HADOOP_HOME/lib/native" - wget "$hadoop_native_libs_url" - native_libs_filename=$(basename "$hadoop_native_libs_url") - tar xvf "$native_libs_filename" -C "$HADOOP_HOME/lib" - rm "$native_libs_filename" - - echo "Pre-configuring Hadoop" - - HADOOP_PID_DIR="/var/run/hadoop" - - cat >> /etc/profile.d/hadoop.sh <> $HADOOP_HOME/etc/hadoop/hadoop-env.sh - - sed -i -e "s,YARN_LOG_DIR=.*,YARN_LOG_DIR=/mnt/log/hadoop/yarn," \ - $HADOOP_HOME/etc/hadoop/yarn-env.sh - echo "source $JAVA_RC" >> $HADOOP_HOME/etc/hadoop/yarn-env.sh - - # enable swiftfs - # Sahara expects the jar to be in both folders, even though only one is - # the "official" classpath. Until images and Sahara become more strictly - # coupled we must maintain this legacy behavior. - ln -s ${HADOOP_HOME}/share/hadoop/tools/lib/hadoop-openstack-${DIB_HADOOP_VERSION}.jar ${HADOOP_HOME}/share/hadoop/common/lib/ - -} - -case "$DISTRO_NAME" in - fedora | ubuntu | rhel7 | centos7 ) - ;; - *) - echo "Unknown distro: $DISTRO_NAME. Exiting." - exit 1 - ;; -esac - -echo "Hadoop setup begins for $DISTRO_NAME" -tmp_dir=/tmp/hadoop - -echo "Creating hadoop user & group" -case "$DISTRO_NAME" in - ubuntu ) - addgroup hadoop - adduser --ingroup hadoop --disabled-password --gecos GECOS hadoop - adduser hadoop sudo - ;; - fedora | rhel7 | centos7 ) - adduser -G adm,wheel hadoop - ;; -esac - -install_hadoop_v2 - -echo "Applying firstboot script" - -RC_SCRIPT_DIR="" -if [ "$DISTRO_NAME" == "ubuntu" ]; then - # File '/etc/rc.local' may not exist - if [ -f "/etc/rc.local" ]; then - mv /etc/rc.local /etc/rc.local.old - fi - RC_SCRIPT_DIR="/etc" -else - # File '/etc/rc.d/rc.local' may not exist - if [ -f "/etc/rc.d/rc.local" ]; then - mv /etc/rc.d/rc.local /etc/rc.d/rc.local.old - fi - RC_SCRIPT_DIR="/etc/rc.d" -fi - -# Make sure that rc-local.service starts after cloud-init.service, -# so that the cloud-init user (centos, ubuntu, cloud-user, ...) -# is available. See also: -# https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/1468103 -# All version of distributions affected by this change use systemd. -mkdir -p /etc/systemd/system/rc-local.service.d/ -cat >/etc/systemd/system/rc-local.service.d/override.conf <> /home/hadoop/.bashrc < " -} - -if [[ $# != 2 ]]; then - usage - exit 1 -fi - -JRE_HOME=$1 -JDK_HOME=$2 - -echo "export JAVA_HOME=$JRE_HOME" >> $JAVA_RC -chmod +x $JAVA_RC - -echo "export PATH=$JRE_HOME/bin:\$PATH" >> $JAVA_BIN_RC -echo "export PATH=$JDK_HOME/bin:\$PATH" >> $JAVA_BIN_RC -chmod +x $JAVA_BIN_RC - -echo "Setting up alternatives for Java" - -case "$DISTRO_NAME" in - ubuntu ) - update-alternatives --install "/usr/bin/java" "java" "$JRE_HOME/bin/java" 1 - update-alternatives --install "/usr/bin/javac" "javac" "$JDK_HOME/bin/javac" 1 - - update-alternatives --set java $JRE_HOME/bin/java - update-alternatives --set javac $JDK_HOME/bin/javac - ;; - fedora | rhel7 | centos7 ) - alternatives --install /usr/bin/java java $JRE_HOME/bin/java 200000 - alternatives --install /usr/bin/javac javac $JDK_HOME/bin/javac 200000 - ;; -esac diff --git a/elements/java/cleanup.d/00-clean-java-vars b/elements/java/cleanup.d/00-clean-java-vars deleted file mode 100755 index 54e048f1..00000000 --- a/elements/java/cleanup.d/00-clean-java-vars +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -unset DIB_JAVA_DISTRO -unset JAVA_BIN_RC -unset JAVA_RC diff --git a/elements/java/environment.d/00-java-rc b/elements/java/environment.d/00-java-rc deleted file mode 100644 index f2ecc947..00000000 --- a/elements/java/environment.d/00-java-rc +++ /dev/null @@ -1,2 +0,0 @@ -export JAVA_RC="/etc/profile.d/99-java.sh" -export JAVA_BIN_RC="/etc/profile.d/98-java-bin.sh" diff --git a/elements/java/pre-install.d/00-check-java b/elements/java/pre-install.d/00-check-java deleted file mode 100755 index 44075584..00000000 --- a/elements/java/pre-install.d/00-check-java +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -if [[ -z $DIB_JAVA_DISTRO ]]; then - echo "Java distribution is not selected." - exit 1 -fi diff --git a/elements/kdc/README.rst b/elements/kdc/README.rst deleted file mode 100644 index 1bcf53d3..00000000 --- a/elements/kdc/README.rst +++ /dev/null @@ -1,13 +0,0 @@ -=== -kdc -=== - -Installs all needed artifacts for KDC server and Kerberos authentication - -Environment Variables ---------------------- - -DIB_UNLIMITED_SECURITY_LOCATION - :Required: No - :Default: https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/ - :Description: Place where UnlimitedSecurity polices are located diff --git a/elements/kdc/element-deps b/elements/kdc/element-deps deleted file mode 100644 index 7076aba9..00000000 --- a/elements/kdc/element-deps +++ /dev/null @@ -1 +0,0 @@ -package-installs diff --git a/elements/kdc/package-installs.yaml b/elements/kdc/package-installs.yaml deleted file mode 100644 index f9279ebc..00000000 --- a/elements/kdc/package-installs.yaml +++ /dev/null @@ -1,7 +0,0 @@ -wget: - phase: pre-install.d -krb-server: -krb-libs: -ldap-utils: -krb-client: -rng-tools: \ No newline at end of file diff --git a/elements/kdc/pkg-map b/elements/kdc/pkg-map deleted file mode 100644 index 45bcd31a..00000000 --- a/elements/kdc/pkg-map +++ /dev/null @@ -1,18 +0,0 @@ -{ - "family": { - "debian": { - "krb-server": "krb5-admin-server", - "krb-libs": "libpam-krb5", - "ldap-utils": "ldap-utils", - "krb-client": "krb5-user", - "rng-tools": "rng-tools" - }, - "redhat": { - "krb-server": "krb5-server", - "ldap-utils": "", - "krb-libs": "krb5-libs", - "krb-client": "krb5-workstation", - "rng-tools": "rng-tools" - } - } -} diff --git a/elements/kdc/pre-install.d/76-unlimited-security-artifacts b/elements/kdc/pre-install.d/76-unlimited-security-artifacts deleted file mode 100755 index 985cabb9..00000000 --- a/elements/kdc/pre-install.d/76-unlimited-security-artifacts +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail -DIB_UNLIMITED_SECURITY_LOCATION="${DIB_UNLIMITED_SECURITY_LOCATION:-https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/}" - -mkdir /tmp/UnlimitedPolicy/ -wget $DIB_UNLIMITED_SECURITY_LOCATION/local_policy.jar -O /tmp/UnlimitedPolicy/local_policy.jar -wget $DIB_UNLIMITED_SECURITY_LOCATION/US_export_policy.jar -O /tmp/UnlimitedPolicy/US_export_policy.jar diff --git a/elements/mysql/README.rst b/elements/mysql/README.rst deleted file mode 100644 index 387c3b8d..00000000 --- a/elements/mysql/README.rst +++ /dev/null @@ -1,8 +0,0 @@ -===== -mysql -===== - -This element sets up the basic components of MySQL. - -It is light version of original MySQL element -(https://github.com/openstack/tripleo-image-elements/tree/master/elements/mysql). diff --git a/elements/mysql/element-deps b/elements/mysql/element-deps deleted file mode 100644 index 7076aba9..00000000 --- a/elements/mysql/element-deps +++ /dev/null @@ -1 +0,0 @@ -package-installs diff --git a/elements/mysql/install.d/10-mysql b/elements/mysql/install.d/10-mysql deleted file mode 100755 index e8281d6d..00000000 --- a/elements/mysql/install.d/10-mysql +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -case "$DISTRO_NAME" in - fedora | rhel7 | centos7 ) - mkdir -p /etc/mysql/conf.d - ;; -esac - -rm -rf /var/lib/mysql/ib_logfile* diff --git a/elements/mysql/install.d/99-disable-mysql-on-boot b/elements/mysql/install.d/99-disable-mysql-on-boot deleted file mode 100755 index cd534a64..00000000 --- a/elements/mysql/install.d/99-disable-mysql-on-boot +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# Disable startup MySQL on boot in Ubuntu -# Service mysqld doesn't start on boot in Fedora and CentOS -# Delete config property 'bind-address' for remote mode (0.0.0.0) - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -if [ "$DISTRO_NAME" = "ubuntu" ]; then - if [ -e "/etc/init/mysql.conf" ]; then - sed -i "s/start on runlevel \[.*\]/start on never runlevel [2345]/g" /etc/init/mysql.conf - # The mysql configuration file name and directory changed in Ubuntu Xenial. - # TODO(shuyingya): delete this if statement when we not support trusty - if [ "$DIB_RELEASE" = "trusty" ]; then - sed -i '/bind-address/d' /etc/mysql/my.cnf - elif [ "$DIB_RELEASE" = "xenial" ]; then - sed -i '/bind-address/d' /etc/mysql/mysql.conf.d/mysqld.cnf - fi - else - update-rc.d -f mysql remove - fi -fi - -# Script for starting mysql - -install -D -g root -o root -m 0755 $(dirname $0)/start-mysql.sh /opt/start-mysql.sh diff --git a/elements/mysql/install.d/start-mysql.sh b/elements/mysql/install.d/start-mysql.sh deleted file mode 100644 index 6a8c5fc9..00000000 --- a/elements/mysql/install.d/start-mysql.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# dib-lint: disable=executable - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -DISTRO="$(lsb_release -is)" - -case "$DISTRO" in - Ubuntu ) - sudo service mysql start - ;; - Fedora | RedHatEnterpriseServer ) - sudo service mysqld start - ;; - CentOS ) - case "$(lsb_release -rs)" in - 7.*) - sudo service mariadb start - ;; - 6.*) - sudo service mysqld start - ;; - esac - ;; -esac diff --git a/elements/mysql/package-installs.yaml b/elements/mysql/package-installs.yaml deleted file mode 100644 index 8f8117d2..00000000 --- a/elements/mysql/package-installs.yaml +++ /dev/null @@ -1,4 +0,0 @@ -mysql_package: -mysql_libs_package: -mysql_server_package: -mysql_java_package: diff --git a/elements/mysql/pkg-map b/elements/mysql/pkg-map deleted file mode 100644 index b286f72a..00000000 --- a/elements/mysql/pkg-map +++ /dev/null @@ -1,46 +0,0 @@ -{ - "distro": { - "fedora": { - "mysql_package": "community-mysql", - "mysql_libs_package": "community-mysql-libs", - "mysql_server_package": "community-mysql-server", - "mysql_java_package": "mysql-connector-java" - }, - "centos7": { - "mysql_package": "mariadb", - "mysql_libs_package": "mariadb-libs", - "mysql_server_package": "mariadb-server", - "mysql_java_package": "mysql-connector-java" - }, - "rhel7": { - "mysql_package": "mariadb", - "mysql_libs_package": "mariadb-libs", - "mysql_server_package": "mariadb-server", - "mysql_java_package": "mysql-connector-java" - } - }, - "release": { - "ubuntu": { - "trusty": { - "mysql_package": "mysql-client-5.5", - "mysql_libs_package": "", - "mysql_server_package": "mysql-server-5.5", - "mysql_java_package": "libmysql-java" - }, - "xenial": { - "mysql_package": "mysql-client-5.7", - "mysql_libs_package": "", - "mysql_server_package": "mysql-server-5.7", - "mysql_java_package": "libmysql-java" - } - } - }, - "family": { - "redhat": { - "mysql_package": "mysql", - "mysql_libs_package": "mysql-libs", - "mysql_server_package": "mysql-server", - "mysql_java_package": "mysql-connector-java" - } - } -} diff --git a/elements/nc/README.rst b/elements/nc/README.rst deleted file mode 100644 index e25d36da..00000000 --- a/elements/nc/README.rst +++ /dev/null @@ -1,5 +0,0 @@ -== -nc -== - -This element installs nc/netcat. diff --git a/elements/nc/elements-deps b/elements/nc/elements-deps deleted file mode 100644 index 7076aba9..00000000 --- a/elements/nc/elements-deps +++ /dev/null @@ -1 +0,0 @@ -package-installs diff --git a/elements/nc/package-installs.yaml b/elements/nc/package-installs.yaml deleted file mode 100644 index ea627711..00000000 --- a/elements/nc/package-installs.yaml +++ /dev/null @@ -1 +0,0 @@ -nc: diff --git a/elements/nc/pkg-map b/elements/nc/pkg-map deleted file mode 100644 index c1d943e0..00000000 --- a/elements/nc/pkg-map +++ /dev/null @@ -1,10 +0,0 @@ -{ - "family": { - "debian": { - "nc": "netcat" - }, - "redhat": { - "nc": "nc" - } - } -} diff --git a/elements/nfs-shares/README.rst b/elements/nfs-shares/README.rst deleted file mode 100644 index 4b4946b2..00000000 --- a/elements/nfs-shares/README.rst +++ /dev/null @@ -1,5 +0,0 @@ -========== -nfs-shares -========== - -This element installs the daemon for the kernel NFS server. diff --git a/elements/nfs-shares/element-deps b/elements/nfs-shares/element-deps deleted file mode 100644 index 73015c24..00000000 --- a/elements/nfs-shares/element-deps +++ /dev/null @@ -1,2 +0,0 @@ -package-installs -pkg-map diff --git a/elements/nfs-shares/package-installs.yaml b/elements/nfs-shares/package-installs.yaml deleted file mode 100644 index 4a8acdbf..00000000 --- a/elements/nfs-shares/package-installs.yaml +++ /dev/null @@ -1 +0,0 @@ -nfs-utilities: diff --git a/elements/nfs-shares/pkg-map b/elements/nfs-shares/pkg-map deleted file mode 100644 index 115baeb9..00000000 --- a/elements/nfs-shares/pkg-map +++ /dev/null @@ -1,10 +0,0 @@ -{ - "family": { - "debian": { - "nfs-utilities": "nfs-common" - }, - "redhat": { - "nfs-utilities": "nfs-utils" - } - } -} diff --git a/elements/ntp/README.rst b/elements/ntp/README.rst deleted file mode 100644 index 99afcd33..00000000 --- a/elements/ntp/README.rst +++ /dev/null @@ -1,5 +0,0 @@ -=== -ntp -=== - -Installs ntp daemon for sync time on VM diff --git a/elements/ntp/elements-deps b/elements/ntp/elements-deps deleted file mode 100644 index 7076aba9..00000000 --- a/elements/ntp/elements-deps +++ /dev/null @@ -1 +0,0 @@ -package-installs diff --git a/elements/ntp/package-installs.yaml b/elements/ntp/package-installs.yaml deleted file mode 100644 index ec81c28e..00000000 --- a/elements/ntp/package-installs.yaml +++ /dev/null @@ -1 +0,0 @@ -ntp: diff --git a/elements/oozie/README.rst b/elements/oozie/README.rst deleted file mode 100644 index 2f028026..00000000 --- a/elements/oozie/README.rst +++ /dev/null @@ -1,5 +0,0 @@ -===== -oozie -===== - -Oozie deployment. diff --git a/elements/oozie/element-deps b/elements/oozie/element-deps deleted file mode 100644 index 619589f1..00000000 --- a/elements/oozie/element-deps +++ /dev/null @@ -1,4 +0,0 @@ -cache-url -extjs -package-installs -sahara-version diff --git a/elements/oozie/environment.d/00-configure-extjs b/elements/oozie/environment.d/00-configure-extjs deleted file mode 100644 index d3be741a..00000000 --- a/elements/oozie/environment.d/00-configure-extjs +++ /dev/null @@ -1,2 +0,0 @@ -export EXTJS_DESTINATION_DIR="/opt/oozie/libtools/" -export EXTJS_NO_UNPACK=1 diff --git a/elements/oozie/install.d/50-setup-oozie b/elements/oozie/install.d/50-setup-oozie deleted file mode 100755 index 7ad6a764..00000000 --- a/elements/oozie/install.d/50-setup-oozie +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -echo "Oozie setup" - -tmp_dir=/tmp/oozie - -if [[ "$DIB_HADOOP_VERSION" == "2.7.1" ]]; then - if [ -z "${OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL:-}" ]; then - OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_7_1_FILE) - else - OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL) - fi -elif [[ "$DIB_HADOOP_VERSION" == "2.7.5" ]]; then - if [ -z "${OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL:-}" ]; then - OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_7_5_FILE) - else - OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL) - fi -elif [[ "$DIB_HADOOP_VERSION" == "2.8.2" ]]; then - if [ -z "${OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL:-}" ]; then - OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_8_2_FILE) - else - OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL) - fi -elif [[ "$DIB_HADOOP_VERSION" == "3.0.1" ]]; then - if [ -z "${OOZIE_HADOOP_V3_0_1_DOWNLOAD_URL:-}" ]; then - OOZIE_FILE=$(basename $OOZIE_HADOOP_V3_0_1_FILE) - else - OOZIE_FILE=$(basename $OOZIE_HADOOP_V3_0_1_DOWNLOAD_URL) - fi -else - echo "Unknown Hadoop version selected. Aborting." - exit 1 -fi - -cd $tmp_dir -tar xzf $OOZIE_FILE -rm $OOZIE_FILE - -if [ $(ls | wc -l) == "1" ]; then - mv $(ls) /opt/oozie -else - mkdir /opt/oozie - mv * /opt/oozie/ -fi - -rm -rf $tmp_dir diff --git a/elements/oozie/package-installs.yaml b/elements/oozie/package-installs.yaml deleted file mode 100644 index 90c87a70..00000000 --- a/elements/oozie/package-installs.yaml +++ /dev/null @@ -1,2 +0,0 @@ -zip: -tar: diff --git a/elements/oozie/post-install.d/90-oozie-war b/elements/oozie/post-install.d/90-oozie-war deleted file mode 100755 index c3da6a4d..00000000 --- a/elements/oozie/post-install.d/90-oozie-war +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -ln -s /usr/share/java/mysql-connector-java.jar /opt/oozie/libtools/mysql.jar -if [ "$DIB_HADOOP_VERSION" = "3.0.1" ]; then - ln -s /usr/share/java/mysql-connector-java.jar /opt/oozie/embedded-oozie-server/dependency/mysql.jar - /opt/oozie/bin/oozie-setup.sh -d /opt/oozie/libtools -else - ln -s /usr/share/java/mysql-connector-java.jar /opt/oozie/oozie-server/lib/mysql.jar - /opt/oozie/bin/oozie-setup.sh prepare-war -d /opt/oozie/libtools -fi - -chown -R hadoop:hadoop /opt/oozie diff --git a/elements/oozie/root.d/0-check-oozie b/elements/oozie/root.d/0-check-oozie deleted file mode 100755 index 51746089..00000000 --- a/elements/oozie/root.d/0-check-oozie +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -if [ "$DIB_HADOOP_VERSION" == "2.7.1" ]; then - if [ -z "${OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL:-}" -a -z "${OOZIE_HADOOP_V2_7_1_FILE:-}" ]; then - echo "OOZIE_HADOOP_V2_7_1_FILE and OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit" - exit 1 - fi -elif [ "$DIB_HADOOP_VERSION" == "2.7.5" ]; then - if [ -z "${OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL:-}" -a -z "${OOZIE_HADOOP_V2_7_5_FILE:-}" ]; then - echo "OOZIE_HADOOP_V2_7_5_FILE and OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit" - exit 1 - fi -elif [ "$DIB_HADOOP_VERSION" == "2.8.2" ]; then - if [ -z "${OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL:-}" -a -z "${OOZIE_HADOOP_V2_8_2_FILE:-}" ]; then - echo "OOZIE_HADOOP_V2_8_2_FILE and OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit" - exit 1 - fi -elif [ "$DIB_HADOOP_VERSION" == "3.0.1" ]; then - if [ -z "${OOZIE_HADOOP_V3_0_1_DOWNLOAD_URL:-}" -a -z "${OOZIE_HADOOP_V3_0_1_FILE:-}" ]; then - echo "OOZIE_HADOOP_V3_0_1_FILE and OOZIE_HADOOP_V3_0_1_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit" - exit 1 - fi -else - echo "Unknown Hadoop version selected. Aborting" - exit 1 -fi diff --git a/elements/oozie/root.d/50-download-oozie b/elements/oozie/root.d/50-download-oozie deleted file mode 100755 index 63da988a..00000000 --- a/elements/oozie/root.d/50-download-oozie +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -tmp_dir=$TARGET_ROOT/tmp/oozie -mkdir -p $tmp_dir - -if [[ "$DIB_HADOOP_VERSION" == "2.7.1" ]]; then - OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL:-}" - OOZIE_FILE="${OOZIE_HADOOP_V2_7_1_FILE:-}" -elif [[ "$DIB_HADOOP_VERSION" == "2.7.5" ]]; then - OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL:-}" - OOZIE_FILE="${OOZIE_HADOOP_V2_7_5_FILE:-}" -elif [[ "$DIB_HADOOP_VERSION" == "2.8.2" ]]; then - OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL:-}" - OOZIE_FILE="${OOZIE_HADOOP_V2_8_2_FILE:-}" -elif [[ "$DIB_HADOOP_VERSION" == "3.0.1" ]]; then - OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V3_0_1_DOWNLOAD_URL:-}" - OOZIE_FILE="${OOZIE_HADOOP_V3_0_1_FILE:-}" -else - echo "Unknown Hadoop version selected. Aborting." - exit 1 -fi - -if [ -n "${OOZIE_DOWNLOAD_URL:-}" ]; then - OOZIE_FILE=$(basename $OOZIE_DOWNLOAD_URL) - cached_tar="$DIB_IMAGE_CACHE/$OOZIE_FILE" - $TMP_HOOKS_PATH/bin/cache-url "$OOZIE_DOWNLOAD_URL" $cached_tar - sudo install -D -g root -o root -m 0755 $cached_tar $tmp_dir -elif [ -n "$OOZIE_FILE" ]; then - sudo install -D -g root -o root -m 0755 $(dirname $0)/$OOZIE_FILE $tmp_dir -fi diff --git a/elements/openjdk/README.rst b/elements/openjdk/README.rst deleted file mode 100644 index dfbf28f4..00000000 --- a/elements/openjdk/README.rst +++ /dev/null @@ -1,5 +0,0 @@ -======= -openjdk -======= - -This element installs the correct version of OpenJDK. diff --git a/elements/openjdk/element-deps b/elements/openjdk/element-deps deleted file mode 100644 index 1daac749..00000000 --- a/elements/openjdk/element-deps +++ /dev/null @@ -1,2 +0,0 @@ -java -pkg-map diff --git a/elements/openjdk/environment.d/00-set-java-distro b/elements/openjdk/environment.d/00-set-java-distro deleted file mode 100644 index 168ebf31..00000000 --- a/elements/openjdk/environment.d/00-set-java-distro +++ /dev/null @@ -1 +0,0 @@ -export DIB_JAVA_DISTRO="openjdk" diff --git a/elements/openjdk/install.d/33-java b/elements/openjdk/install.d/33-java deleted file mode 100755 index 26920694..00000000 --- a/elements/openjdk/install.d/33-java +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -echo "Installing OpenJDK" - -default_jdk_version_for_distro () { - case "$DISTRO_NAME" in - fedora ) - if [ $(lsb_release -rs) -ge '21' ]; then - echo "8" - else - echo "7" - fi - ;; - centos7 | rhel7 ) - echo "8" - ;; - ubuntu ) - if [ "$DIB_RELEASE" = "trusty" ]; then - echo "7" - elif [ "$DIB_RELEASE" = "xenial" ]; then - echo "8" - fi - ;; - *) - echo "Unknown OpenJDK distro: $DISTRO_NAME" >&2 - exit 1 - ;; - esac -} - -pkg=openjdk-$(default_jdk_version_for_distro) -pkg=$(pkg-map --element openjdk $pkg) -install-packages $pkg - -case "$DISTRO_NAME" in - ubuntu ) - JRE_PATH=$(update-alternatives --list java) - JRE_HOME=${JRE_PATH%/bin/java} - JDK_PATH=$(update-alternatives --list javac) - JDK_HOME=${JDK_PATH%/bin/javac} - ;; - fedora | rhel7 | centos7 ) - # NOTE: rpm -E "%{java_home}" command returns wrong java home - # on CentOS, Fedora and RHEL - JRE_HOME="/usr/lib/jvm/java-openjdk/jre" - JDK_HOME="/usr/lib/jvm/java-openjdk" - ;; -esac - -setup-java-home $JRE_HOME $JDK_HOME - -echo "OpenJDK has been installed" diff --git a/elements/openjdk/pkg-map b/elements/openjdk/pkg-map deleted file mode 100644 index 35ec2ccd..00000000 --- a/elements/openjdk/pkg-map +++ /dev/null @@ -1,12 +0,0 @@ -{ - "family": { - "debian": { - "openjdk-7": "openjdk-7-jdk", - "openjdk-8": "openjdk-8-jdk" - }, - "redhat": { - "openjdk-7": "java-1.7.0-openjdk-devel", - "openjdk-8": "java-1.8.0-openjdk-devel" - } - } -} diff --git a/elements/oracle-java/README.rst b/elements/oracle-java/README.rst deleted file mode 100644 index bb62cbe4..00000000 --- a/elements/oracle-java/README.rst +++ /dev/null @@ -1,21 +0,0 @@ -=========== -oracle-java -=========== - -This element installs a Java Virtual Machine into the image. There are -three options for selecting what version of the JVM is installed - - -0. Provide no input, the package manager in your image will be used to -install the natively packaged JVM - -1. Provide JAVA_FILE via the environment, it should be a .tar.gz or -.bin and will be installed under JAVA_TARGET_LOCATION, which will default -to "/usr/java" (see 3. for more information on JAVA_TARGET_LOCATION). - -2. Provide JAVA_DOWNLOAD_URL via the environment, it should be a url -pointing to a file that will be placed in JAVA_FILE (see 1.) - -3. In addition to selecting the JDK to install, this element can be -configured to install to a specific location. Set the JAVA_TARGET_LOCATION -variable in order to customize the top-level directory that will -contain the JDK install. By default, this variable is set to "/usr/java". diff --git a/elements/oracle-java/element-deps b/elements/oracle-java/element-deps deleted file mode 100644 index f3d360b1..00000000 --- a/elements/oracle-java/element-deps +++ /dev/null @@ -1 +0,0 @@ -java diff --git a/elements/oracle-java/environment.d/00-set-java-distro b/elements/oracle-java/environment.d/00-set-java-distro deleted file mode 100644 index ff5585f4..00000000 --- a/elements/oracle-java/environment.d/00-set-java-distro +++ /dev/null @@ -1 +0,0 @@ -export DIB_JAVA_DISTRO="oracle-java" diff --git a/elements/oracle-java/install.d/33-java b/elements/oracle-java/install.d/33-java deleted file mode 100755 index d89514d9..00000000 --- a/elements/oracle-java/install.d/33-java +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -echo "Begin: installation of Java" - -# NOTE: $(dirname $0) is read-only, use space under $TARGET_ROOT -JAVA_LOCATION=${JAVA_TARGET_LOCATION:-"/usr/java"} -JAVA_NAME="oracle-jdk" -JAVA_HOME=$JAVA_LOCATION/$JAVA_NAME -JAVA_DOWNLOAD_URL=${JAVA_DOWNLOAD_URL:-"http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-linux-x64.tar.gz"} - -mkdir -p $JAVA_LOCATION - -if [ -n "$JAVA_DOWNLOAD_URL" ]; then - install-packages wget - JAVA_FILE=$(basename $JAVA_DOWNLOAD_URL) - wget --no-check-certificate --no-cookies -c \ - --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" \ - -O $JAVA_LOCATION/$JAVA_FILE $JAVA_DOWNLOAD_URL -elif [ -n "$JAVA_FILE" ]; then - install -D -g root -o root -m 0755 $(dirname $0)/$JAVA_FILE $JAVA_LOCATION -fi - -cd $JAVA_LOCATION - -echo "Decompressing Java archive" -echo -e "\n" | tar -zxf $JAVA_FILE -echo "Setting up $JAVA_NAME" -chown -R root:root $JAVA_LOCATION -JAVA_DIR=`ls -1 $JAVA_LOCATION | grep -v tar.gz` -ln -s $JAVA_LOCATION/$JAVA_DIR $JAVA_HOME - -setup-java-home $JAVA_HOME $JAVA_HOME - -rm $JAVA_FILE - -echo "End: installation of Java" diff --git a/elements/root-passwd/README.rst b/elements/root-passwd/README.rst deleted file mode 100644 index 4ce6aefc..00000000 --- a/elements/root-passwd/README.rst +++ /dev/null @@ -1,15 +0,0 @@ -=========== -root-passwd -=========== - -Assign a password to root. - -This is useful when booting outside of a cloud environment (e.g. manually via -kvm). - -Environment Variables ---------------------- - -DIB_PASSWORD - :Required: Yes - :Description: The password for the root user. diff --git a/elements/root-passwd/element-deps b/elements/root-passwd/element-deps deleted file mode 100644 index 7076aba9..00000000 --- a/elements/root-passwd/element-deps +++ /dev/null @@ -1 +0,0 @@ -package-installs diff --git a/elements/root-passwd/package-installs.yaml b/elements/root-passwd/package-installs.yaml deleted file mode 100644 index b520b92d..00000000 --- a/elements/root-passwd/package-installs.yaml +++ /dev/null @@ -1,6 +0,0 @@ -augeas-tools: - phase: post-install.d -openssh-server: - phase: post-install.d -openssh-client: - phase: post-install.d diff --git a/elements/root-passwd/post-install.d/99-setup b/elements/root-passwd/post-install.d/99-setup deleted file mode 100755 index 38053199..00000000 --- a/elements/root-passwd/post-install.d/99-setup +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -if [ -z "$DIB_PASSWORD" ]; then - echo "Error during setup password for root" - exit 1 -fi -sed -i "s/disable_root: true/disable_root: false/" /etc/cloud/cloud.cfg -augtool -s set /files/etc/ssh/sshd_config/PasswordAuthentication yes -augtool -s set /files/etc/ssh/sshd_config/PermitRootLogin yes -augtool -s set /files/etc/ssh/ssh_config/PasswordAuthentication yes -echo -e "$DIB_PASSWORD\n$DIB_PASSWORD\n" | passwd diff --git a/elements/s3_hadoop/README.rst b/elements/s3_hadoop/README.rst deleted file mode 100644 index 4280b160..00000000 --- a/elements/s3_hadoop/README.rst +++ /dev/null @@ -1,10 +0,0 @@ -========= -s3_hadoop -========= - -Copy the Hadoop S3 connector libraries into the Hadoop and Spark classpaths. - -Environment Variables ---------------------- - -None. diff --git a/elements/s3_hadoop/element-deps b/elements/s3_hadoop/element-deps deleted file mode 100644 index 7076aba9..00000000 --- a/elements/s3_hadoop/element-deps +++ /dev/null @@ -1 +0,0 @@ -package-installs diff --git a/elements/s3_hadoop/package-installs.yaml b/elements/s3_hadoop/package-installs.yaml deleted file mode 100644 index cc77790e..00000000 --- a/elements/s3_hadoop/package-installs.yaml +++ /dev/null @@ -1,2 +0,0 @@ -wget: - phase: post-install.d diff --git a/elements/s3_hadoop/post-install.d/89-manipulate-s3 b/elements/s3_hadoop/post-install.d/89-manipulate-s3 deleted file mode 100755 index 3ceda808..00000000 --- a/elements/s3_hadoop/post-install.d/89-manipulate-s3 +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - - -case "$plugin_type" in - "vanilla" | "spark" ) - ;; - "cloudera" ) - echo -n "The s3_hadoop element is not supported on CDH," - echo " because the relevant libraries need no manipulation." - # NOTE: actually the above statement is only true on CDH>=5.9 - exit 1 - ;; - *) - # TODO: Investigate if some changes are in fact needed for HDP, MapR - echo "The s3_hadoop element is only relevant to Vanilla and Spark." - exit 1 -esac - -SPARK_JARS_DIR_PATH="/opt/spark/jars" -HADOOP_TOOLS_DIR_PATH="/opt/hadoop/share/hadoop/tools/lib" -HADOOP_COMMON_DIR_PATH="/opt/hadoop/share/hadoop/common/lib" - -if [ "$plugin_type" = "vanilla" ]; then - if [ "$DIB_HADOOP_VERSION" = "2.7.1" -o "$DIB_HADOOP_VERSION" = "2.7.5" ]; then - # These versions need a patched hadoop-aws jar - wget https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-aws-$DIB_HADOOP_VERSION.jar -O $HADOOP_TOOLS_DIR_PATH/hadoop-aws-$DIB_HADOOP_VERSION.jar - fi - - # NOTE: It's easier just to copy, than to mess with YARN - cp $HADOOP_TOOLS_DIR_PATH/*aws*.jar $HADOOP_COMMON_DIR_PATH - if [ "$DIB_HADOOP_VERSION" = "2.7.1" -o "$DIB_HADOOP_VERSION" = "2.7.5" -o "$DIB_HADOOP_VERSION" = "2.8.2" ]; then - # Hadoop-aws older than 2.9.0 needs these too - cp $HADOOP_TOOLS_DIR_PATH/joda-time*.jar $HADOOP_COMMON_DIR_PATH - # The following jars are also on-disk, but under the wrong namespace - wget https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-core/2.5.3/jackson-core-2.5.3.jar -O $HADOOP_COMMON_DIR_PATH/jackson-core.jar - wget https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-databind/2.5.3/jackson-databind-2.5.3.jar -O $HADOOP_COMMON_DIR_PATH/jackson-databind.jar - wget https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-annotations/2.5.3/jackson-annotations-2.5.3.jar -O $HADOOP_COMMON_DIR_PATH/jackson-annotations.jar - fi -fi - -# For both Spark and Vanilla plugins: -# (The s3a driver in hadoop-aws 2.6.5 is too buggy to be redeemed) -if [ "$SPARK_HADOOP_DL" != "hadoop2.6" ]; then - # The hadoop-aws and aws-java-sdk libraries are missing here, but we - # cannot copy them from the Hadoop folder on-disk due to - # version/patching issues - wget https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-aws-2.7.3.jar -O $SPARK_JARS_DIR_PATH/hadoop-aws.jar - wget https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk/1.7.4/aws-java-sdk-1.7.4.jar -O $SPARK_JARS_DIR_PATH/aws-java-sdk.jar -fi diff --git a/elements/sahara-version/README.rst b/elements/sahara-version/README.rst deleted file mode 100644 index ec76f903..00000000 --- a/elements/sahara-version/README.rst +++ /dev/null @@ -1,5 +0,0 @@ -============== -sahara-version -============== - -This element saves the version of a few components in a file. diff --git a/elements/sahara-version/root.d/90-sahara-version b/elements/sahara-version/root.d/90-sahara-version deleted file mode 100755 index 231e7df9..00000000 --- a/elements/sahara-version/root.d/90-sahara-version +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -function _get_version_package { - if type -p pbr > /dev/null; then - local version=$(pbr info $1 | awk '{print $2}') - echo "${version}" - fi -} - -cat << EOF | sudo tee "${TARGET_ROOT}/etc/sahara-extra.version" -sahara-image-elements version: $(_get_version_package "sahara-image-elements") -dib-utils version: $(_get_version_package "dib-utils") -diskimage-builder version: $(_get_version_package "diskimage-builder") -EOF diff --git a/elements/spark/README.rst b/elements/spark/README.rst deleted file mode 100644 index 6786ecc6..00000000 --- a/elements/spark/README.rst +++ /dev/null @@ -1,48 +0,0 @@ -===== -spark -===== - -Installs Spark. Requires Hadoop. - -This element will install Spark. It tries to guess the -correct file to download based on the ``DIB_SPARK_VERSION`` and -``DIB_CDH_VERSION``, but this behaviour can be overridden by using -``SPARK_DOWNLOAD_URL`` to specify a download URL for a pre-built -Spark tar.gz file. -See http://spark.apache.org/downloads.html for more download options. - -Versions --------- - -This element is able to generate images containing any valid Spark version, -compiled against one version of Hadoop HDFS libraries. - -Only some combinations of Spark and Hadoop versions are possible, depending on -the availability of a pre-compiled binary and only few of them are tested with -the Sahara Spark plugin. - -The ``diskimage-create.sh`` script will use tested defaults. Those defaults -generate an image supported by the Sahara Spark plugin. Other combinations -should be used only for evaluation or testing purposes. Refer to the Sahara -Spark plugin wiki page (https://wiki.openstack.org/wiki/Sahara/SparkPlugin) -for more information about tested and supported versions. - -Environment Variables ---------------------- - -DIB_SPARK_VERSION - :Required: Yes, if ``SPARK_DOWNLOAD_URL`` is not set. - :Description: Version of the Spark package to download. - :Example: ``DIB_SPARK_VERSION=1.3.1`` - -DIB_CDH_VERSION - :Required: Required only for images for Spark Plugin and - if ``SPARK_DOWNLOAD_URL`` is not set. - :Description: Version of the CDH platform to use for Hadoop compatibility. - CDH version 5.3 is known to work well. - :Example: ``DIB_CDH_VERSION=5.3`` - -SPARK_DOWNLOAD_URL - :Required: No, if set overrides ``DIB_CDH_VERSION`` and ``DIB_SPARK_VERSION`` - :Description: Download URL of a tgz Spark package to override the automatic - selection from the Apache repositories. diff --git a/elements/spark/element-deps b/elements/spark/element-deps deleted file mode 100644 index 0e0a242d..00000000 --- a/elements/spark/element-deps +++ /dev/null @@ -1,2 +0,0 @@ -cache-url -package-installs diff --git a/elements/spark/install.d/60-spark b/elements/spark/install.d/60-spark deleted file mode 100755 index adfcb8ef..00000000 --- a/elements/spark/install.d/60-spark +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# This script installs Spark -# More documentation in the README.md file - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -tmp_dir=/tmp/spark -pushd $tmp_dir - - -echo "Extracting SPARK" -spark_url=$(cat spark_url.txt) -spark_file=$(basename $spark_url) -extract_folder=$(tar tzf $spark_file | sed -e 's@/.*@@' | uniq) -echo "Decompressing Spark..." -tar xzf $spark_file -rm $spark_file - -echo "Moving SPARK to /opt/" -# Placing spark in /opt/spark -mv $extract_folder /opt/spark/ -mv spark_url.txt /opt/spark/ - -popd -rm -Rf $tmp_dir diff --git a/elements/spark/package-installs.yaml b/elements/spark/package-installs.yaml deleted file mode 100644 index 56fd8234..00000000 --- a/elements/spark/package-installs.yaml +++ /dev/null @@ -1 +0,0 @@ -tar: diff --git a/elements/spark/post-install.d/20-spark b/elements/spark/post-install.d/20-spark deleted file mode 100755 index 48db9a54..00000000 --- a/elements/spark/post-install.d/20-spark +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -# We have to chown the Spark directories to make it usable for the default user -# Unfortunately the Ubuntu user does not exists when DIB is creating the image -# and we need to execute this code the first time the VM boots. - -firstboot_script_name="/opt/spark/firstboot.sh" -if [ "$DISTRO_NAME" == "ubuntu" ]; then - sed -i -e "s,^exit 0$,[ -f $firstboot_script_name ] \&\& sh $firstboot_script_name; exit 0," /etc/rc.local - if [ "$plugin_type" == "vanilla" ]; then - user_and_group_names="hadoop:hadoop" - elif [ "$plugin_type" == "spark" ]; then - user_and_group_names="ubuntu:ubuntu" - fi -else - sed -i -e "s,^exit 0$,[ -f $firstboot_script_name ] \&\& sh $firstboot_script_name; exit 0," /etc/rc.d/rc.local - user_and_group_names="hadoop:hadoop" -fi - -cat >> $firstboot_script_name < $tmp_dir/spark_url.txt diff --git a/elements/ssh/README.rst b/elements/ssh/README.rst deleted file mode 100644 index 4d4b451a..00000000 --- a/elements/ssh/README.rst +++ /dev/null @@ -1,6 +0,0 @@ -=== -ssh -=== - -This element installs an SSH server then configures it to be suitable -for use with Sahara. diff --git a/elements/ssh/element-deps b/elements/ssh/element-deps deleted file mode 100644 index 7076aba9..00000000 --- a/elements/ssh/element-deps +++ /dev/null @@ -1 +0,0 @@ -package-installs diff --git a/elements/ssh/install.d/33-ssh b/elements/ssh/install.d/33-ssh deleted file mode 100755 index 5582c6e5..00000000 --- a/elements/ssh/install.d/33-ssh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -ue -set -o pipefail -echo "Begin: install and configure SSH" - -case "$DISTRO_NAME" in - fedora ) - # the main config file is empty in F25+ - SSH_CONFIG_FILE="/etc/ssh/ssh_config.d/05-redhat.conf" - ;; - * ) - SSH_CONFIG_FILE="/etc/ssh/ssh_config" - ;; -esac - -augtool <> /opt/zookeeper/firstboot.sh <=2.0.0 # Apache-2.0 - -dib-utils>=0.0.8 # Apache-2.0 -diskimage-builder!=1.6.0,!=1.7.0,!=1.7.1,>=1.1.2 # Apache-2.0 diff --git a/roles/build-sahara-images-dib/README.rst b/roles/build-sahara-images-dib/README.rst deleted file mode 100644 index b5d6c51e..00000000 --- a/roles/build-sahara-images-dib/README.rst +++ /dev/null @@ -1,13 +0,0 @@ -Build Sahara Images with DIB - -**Role Variables** - -.. zuul:rolevar:: sahara_build_directory - :default: /var/tmp/sahara-image-build - - The base directory used for the build process. - -.. zuul:rolevar:: sahara_plugin - :default: vanilla - - The plugin whose images will be built. diff --git a/roles/build-sahara-images-dib/defaults/main.yaml b/roles/build-sahara-images-dib/defaults/main.yaml deleted file mode 100644 index 2a51014b..00000000 --- a/roles/build-sahara-images-dib/defaults/main.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -sahara_image_elements_src_dir: "{{ zuul.projects['opendev.org/openstack/sahara-image-elements'].src_dir }}" -sahara_build_directory: /var/tmp/sahara-image-build -sahara_plugin: vanilla -sahara_plugin_version: '' -sahara_plugin_distro: '' -tox_executable: 'tox' diff --git a/roles/build-sahara-images-dib/tasks/main.yaml b/roles/build-sahara-images-dib/tasks/main.yaml deleted file mode 100644 index 2bd3c2b4..00000000 --- a/roles/build-sahara-images-dib/tasks/main.yaml +++ /dev/null @@ -1,4 +0,0 @@ -- name: Build all the images for the selected plugin - command: ./tools/gate/build-images "{{ sahara_plugin }}" "{{ sahara_plugin_version }}" "{{ sahara_plugin_distro }}" "{{ tox_executable }}" - args: - chdir: "{{ sahara_image_elements_src_dir }}" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index e5117ad1..00000000 --- a/setup.cfg +++ /dev/null @@ -1,22 +0,0 @@ -[metadata] -name = sahara-image-elements -summary = Image elements for Sahara -description_file = README.rst -license = Apache Software License -python_requires = >=3.6 -classifiers = - Intended Audience :: Information Technology - Intended Audience :: System Administrators - License :: OSI Approved :: Apache Software License - Operating System :: POSIX :: Linux -author = OpenStack -author_email = openstack-discuss@lists.openstack.org -home_page = https://docs.openstack.org/sahara/latest - -[files] -scripts = - bin/sahara-image-create - diskimage-create/diskimage-create.sh - -data_files = - share/sahara-elements/elements = elements/* diff --git a/setup.py b/setup.py deleted file mode 100644 index 5019c359..00000000 --- a/setup.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import setuptools - -setuptools.setup( - setup_requires=['pbr>=2.0.0'], - pbr=True, - py_modules=[]) diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index 8567cf5a..00000000 --- a/test-requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. - -bashate>=2.1.0 # Apache-2.0 -hacking>=3.0.1,<3.1.0 # Apache-2.0 diff --git a/tools/gate/build-images b/tools/gate/build-images deleted file mode 100755 index e9e344a1..00000000 --- a/tools/gate/build-images +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash -xe - -PLUGIN=$1 -PLUGIN_VERSION=${2:-''} -PLUGIN_DISTRO=${3:-''} -TOX_EXECUTABLE=${4:-'tox'} - -export DIB_MIN_TMPFS=10 - -# build a specific version if requested -if [ -n "${PLUGIN}" ] && [ -n "${PLUGIN_VERSION}" ] && [ -n "${PLUGIN_DISTRO}" ]; then - case ${PLUGIN} in - vanilla|cloudera|ambari) - VERSION_FLAG="-v ${PLUGIN_VERSION}" - ;; - mapr) - VERSION_FLAG="-r ${PLUGIN_VERSION}" - ;; - spark) - VERSION_FLAG="-s ${PLUGIN_VERSION}" - ;; - storm) - VERSION_FLAG="-t ${PLUGIN_VERSION}" - ;; - *) - VERSION_FLAG="" - ;; - esac - ${TOX_EXECUTABLE} -e venv -- sahara-image-create -x -u -p ${PLUGIN} -i ${PLUGIN_DISTRO} ${VERSION_FLAG} - exit -fi - -# build (almost) all versions for a specified plugin -case "$PLUGIN" in - "vanilla") - # temporary don't test building fedora image - for distro in ubuntu centos7; do - # limit to 4 releases - for version in 2.7.1 2.7.5 2.8.2 3.0.1; do - ${TOX_EXECUTABLE} -e venv -- sahara-image-create -x -u -p $PLUGIN -i $distro -v $version - done - done - ;; - "cloudera") - # manually call each distro for cloudera, so we can cleanup - # cache and generated images from the previous run - # (saving space a bit) - for distro in ubuntu centos7; do - # NOTE(vgridnev): Number of versions are growing; testing only 2 latest - # releases of CDH - for version in 5.9 5.11; do - ${TOX_EXECUTABLE} -e venv -- sahara-image-create -x -u -p $PLUGIN -i $distro -v $version - find . -maxdepth 1 -name '*cloudera*.qcow2' -delete - sudo rm -rf ~/.cache/image-create - done - done - ;; - *) - ${TOX_EXECUTABLE} -e venv -- sahara-image-create -x -u -p $PLUGIN - ;; -esac diff --git a/tools/run_bashate.sh b/tools/run_bashate.sh deleted file mode 100755 index e48bc540..00000000 --- a/tools/run_bashate.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -xe - -find ./ -not -wholename \*.tox/\* -and \( -name \*.sh -or -wholename \*.d/\* -and -not -name \*.md -and -not -name \*.rst -and -not -name \*.py \) -print0 | xargs -0 bashate -v -i E006 diff --git a/tox.ini b/tox.ini deleted file mode 100644 index b58af680..00000000 --- a/tox.ini +++ /dev/null @@ -1,46 +0,0 @@ -[tox] -envlist = pep8 -minversion = 3.18.0 -skipsdist = True -ignore_basepython_conflict = True - -[testenv] -basepython = python3 -usedevelop = True -setenv = - VIRTUAL_ENV={envdir} -deps = - -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} - -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -passenv = - http_proxy - https_proxy - no_proxy - -[testenv:bashate] -whitelist_externals = bash -commands = bash -c "find {toxinidir} -not -wholename \*.tox/\* -and \( -name \*.sh -or -wholename \*.d/\* -and -not -name \*.md -and -not -name \*.rst -and -not -name \*.py \) -print0 | xargs -0 bashate -v" -allowlist_externals = bash - -[testenv:pep8] -setenv = - ELEMENTS_DIR=elements -commands = - flake8 {posargs} - {toxinidir}/tools/run_bashate.sh - dib-lint -allowlist_externals = /bin/bash - {toxinidir}/tools/run_bashate.sh - -[testenv:venv] -commands = {posargs} -passenv = * - -[dib-lint] -ignore=safe_sudo - -[flake8] -show-source = true -builtins = _ -exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools