Added Elasticearch and its deployment.

Part of ELK stack. Includes Dockerfiles for both Centos and Ubuntu.

Change-Id: I9f76adf084cd4f68e29326112b76ffd02b5adada
Partially-implements: blueprint central-logging-service

ansible/group_vars/all.yml

@@ -126,6 +126,7 @@ rgw_port: "6780"
 mistral_api_port: "8989"
 
 kibana_port: "5601"
+
 elasticsearch_port: "9200"
 
 ####################

ansible/inventory/all-in-one

@@ -18,6 +18,9 @@ localhost       ansible_connection=local
 [kibana:children]
 control
 
+[elasticsearch:children]
+control
+
 [haproxy:children]
 network
 

ansible/inventory/multinode

@@ -30,6 +30,9 @@ storage01
 [kibana:children]
 control
 
+[elasticsearch:children]
+control
+
 [haproxy:children]
 network
 

ansible/roles/elasticsearch/defaults/main.yml

@@ -0,0 +1,9 @@
+---
+####################
+# Elasticsearch
+####################
+elasticsearch_port: "{{ elasticsearch_port }}"
+elasticsearch_host: "{{ kolla_internal_address }}"
+elasticsearch_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-elasticsearch"
+elasticsearch_tag: "{{ openstack_release }}"
+elasticsearch_image_full: "{{ elasticsearch_image }}:{{ elasticsearch_tag }}"

ansible/roles/elasticsearch/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - { role: common }

ansible/roles/elasticsearch/tasks/config.yml

@@ -0,0 +1,22 @@
+---
+- name: Ensuring config directories exist
+  file:
+    path: "{{ node_config_directory }}/{{ item }}"
+    state: "directory"
+    recurse: yes
+  with_items:
+    - "elasticsearch"
+
+- name: Copying over config.json files for services
+  template:
+    src: "{{ item }}.json.j2"
+    dest: "{{ node_config_directory }}/{{ item }}/config.json"
+  with_items:
+    - "elasticsearch"
+
+- name: Copying over elasticsearch.yml
+  template:
+    src: "elasticsearch.yml.j2"
+    dest: "{{ node_config_directory }}/{{ item }}/{{ item }}.yml"
+  with_items:
+    - "elasticsearch"

ansible/roles/elasticsearch/tasks/deploy.yml

@@ -0,0 +1,4 @@
+---
+- include: config.yml
+
+- include: start.yml

ansible/roles/elasticsearch/tasks/main.yml

@@ -0,0 +1,2 @@
+---
+- include: "{{ action }}.yml"

ansible/roles/elasticsearch/tasks/pull.yml

@@ -0,0 +1,6 @@
+---
+- name: Pulling elasticsearch image
+  kolla_docker:
+    action: "pull_image"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ elasticsearch_image_full }}"

ansible/roles/elasticsearch/tasks/start.yml

@@ -0,0 +1,10 @@
+---
+- name: Starting Elasticsearch container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ elasticsearch_image_full }}"
+    name: "elasticsearch"
+    volumes:
+      - "{{ node_config_directory }}/elasticsearch/:{{ container_config_directory }}/"
+      - "elasticsearch:/var/lib/elasticsearch/data"

ansible/roles/elasticsearch/templates/elasticsearch.json.j2

@@ -0,0 +1,11 @@
+{
+    "command": "/usr/share/elasticsearch/bin/elasticsearch",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/elasticsearch.yml",
+            "dest": "/usr/share/elasticsearch/config/elasticsearch.yml",
+            "owner": "elasticsearch",
+            "perm": "0600"
+        }
+    ]
+}

ansible/roles/elasticsearch/templates/elasticsearch.yml.j2

@@ -0,0 +1,5 @@
+network.host: {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}
+path.conf: "/etc/elasticsearch"
+path.data: "/var/lib/elasticsearch/data"
+path.logs: "/var/log/elasticsearch"
+path.scripts: "/etc/elasticsearch/scripts"

ansible/site.yml

@@ -32,6 +32,12 @@
         tags: kibana,
         when: enable_elk | bool }
 
+- hosts: elasticsearch
+  roles:
+    - { role: elasticsearch,
+        tags: elasticsearch,
+        when: enable_elk | bool }
+
 - hosts: memcached
   roles:
     - { role: memcached,

docker/base/Dockerfile.j2

@@ -18,6 +18,8 @@ ENV KOLLA_INSTALL_METATYPE {{ install_metatype }}
 {% if base_distro in ['fedora', 'centos', 'oraclelinux', 'rhel'] %}
 
 #### BEGIN REPO ENABLEMENT
+# Turns on Elasticsearch repos
+COPY elasticsearch.yum.repo /etc/yum.repos.d/elasticsearch.repo
 
 # Turns on MariaDB repos throughout the RPM build
 COPY mariadb.yum.repo /etc/yum.repos.d/MariaDB.repo
@@ -28,7 +30,8 @@ RUN yum -y install http://repo.percona.com/release/7/RPMS/x86_64/percona-release
 
 RUN rpm --import http://yum.mariadb.org/RPM-GPG-KEY-MariaDB \
     && rpm --import https://download.ceph.com/keys/release.asc \
-    && rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-Percona
+    && rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-Percona \
+    && rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch
 
     {% if install_metatype in ['rdo', 'mixed'] %}
 
@@ -153,6 +156,7 @@ RUN apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 199369E540
     && apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 391A9AA2147192839E9DB0315EDB1B62EC4926EA \
     && apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 430BDF5C56E7C94E848EE60C1C4CBDCDCD2EFD2A \
     && apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 08B73419AC32B4E966C1A330E84AC2C0460F3994 \
+    && apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 46095ACC8548582C1A2699A9D27D666CD88E42B4 \
     && apt-get update \
     && apt-get upgrade -y \
     && apt-get dist-upgrade -y \

docker/base/elasticsearch.yum.repo

@@ -0,0 +1,6 @@
+[elasticsearch-2.x]
+name=Elasticsearch repository for 2.x packages
+baseurl=http://packages.elastic.co/elasticsearch/2.x/centos
+gpgcheck=1
+gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch
+enabled=1

docker/base/sources.list

@@ -18,3 +18,6 @@ deb http://repo.percona.com/apt trusty main
 
 # Ceph repo
 deb http://download.ceph.com/debian-hammer/ trusty main
+
+# Elastic repo
+deb http://packages.elastic.co/elasticsearch/2.x/debian stable main

docker/elasticsearch/Dockerfile.j2

@@ -0,0 +1,40 @@
+FROM {{ namespace }}/{{ image_prefix }}base:{{ tag }}
+MAINTAINER {{ maintainer }}
+
+
+{% if base_distro in ['centos', 'fedora', 'oraclelinux', 'rhel'] %}
+
+ENV JAVA_HOME /usr/lib/jvm/jre-1.7.0-openjdk/
+
+RUN yum -y install \
+        java-1.7.0-openjdk \
+        elasticsearch-2.1.1 \
+    && yum clean all
+
+{% elif base_distro in ['ubuntu', 'debian'] %}
+
+ENV JAVA_HOME /usr/lib/jvm/java-1.7.0-openjdk-amd64/
+
+RUN apt-get install -y --no-install-recommends \
+       openjdk-7-jre \
+       elasticsearch=2.1.1 \
+    && apt-get clean
+
+{% endif %}
+
+COPY elasticsearch_sudoers /etc/sudoers.d/elasticsearch_sudoers
+COPY extend_start.sh /usr/local/bin/kolla_extend_start
+RUN chmod 755 /usr/local/bin/kolla_extend_start \
+    && chmod 750 /etc/sudoers.d \
+    && chmod 440 /etc/sudoers.d/elasticsearch_sudoers \
+    && usermod -a -G kolla elasticsearch
+
+
+# by default elasticsearch shell is /bin/false, we need
+# /bin/bash to run elasticsearch as non-root
+# https://discuss.elastic.co/t/running-as-non-root-user-service-wrapper-has-changed/7863
+RUN usermod -s /bin/bash elasticsearch -d /usr/share/elasticsearch
+
+{{ include_footer }}
+
+USER elasticsearch

docker/elasticsearch/elasticsearch_sudoers

@@ -0,0 +1 @@
+%kolla ALL=(root) NOPASSWD: /bin/chown elasticsearch\: /var/lib/elasticsearch/data, /usr/bin/chown elasticsearch\: /var/lib/elasticsearch/data

docker/elasticsearch/extend_start.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Only update permissions if permissions need to be updated
+if [[ $(stat -c %U:%G /var/lib/elasticsearch/data) != "elasticsearch:elasticsearch" ]]; then
+    sudo chown elasticsearch: /var/lib/elasticsearch/data
+fi