openstack-ansible-ops/elk_metrics_6x/installLogstash.yml

---
- name: Install Logstash
  hosts: elastic-logstash
  serial: "50%"
  become: true
  vars_files:
    - vars/variables.yml

  vars:
    temp_dir: /var/lib/logstash/tmp
    logstash_pipelines: "{{lookup('template', 'templates/logstash-pipelines.yml.j2') }}"

  environment: "{{ deployment_environment_variables | default({}) }}"

  pre_tasks:
    - include_tasks: common_task_data_node_hosts.yml
      tags:
        - always

  tasks:
    - name: Set quarter memory fact
      set_fact:
        q_mem: "{{ (ansible_memtotal_mb | int) // 3 }}"
      when:
        - q_mem is not defined
      tags:
        - always

    - name: Set processor cores fact
      set_fact:
        q_storage: "{{ (ansible_processor_cores | int) * (ansible_processor_threads_per_core | int) * 2 }}"
      when:
        - q_storage is not defined
      tags:
        - always

    - name: Set logstash facts
      set_fact:
        elastic_heap_size: "{{ ((q_mem | int) > 30720) | ternary(30720, q_mem) }}"
        logstash_queue_size: "{{ ((((q_storage | int) >= 2) | ternary(q_storage, 2) | int) * 1024) // ((logstash_pipelines | from_yaml) | length) }}"
        elastic_log_rotate_path: "/var/log/logstash"
      tags:
        - always

    - include_tasks: common_task_install_elk_repo.yml

    - name: Configure systcl vm.max_map_count=262144 on container hosts
      sysctl:
        name: "vm.max_map_count"
        value: "262144"
        state: "present"
        reload: "yes"
      delegate_to: "{{ physical_host }}"
      tags:
        - sysctl

    - name: Ensure mount directories exists
      file:
        path: "/openstack/{{ inventory_hostname }}/logstash"
        state: "directory"
      delegate_to: "{{ physical_host }}"

    - name: logstash datapath bind mount
      lxc_container:
        name: "{{ inventory_hostname }}"
        container_command: |
          [[ ! -d "/var/lib/logstash" ]] && mkdir -p "/var/lib/logstash"          
        container_config:
          - "lxc.mount.entry=/openstack/{{ inventory_hostname }}/logstash var/lib/logstash none bind 0 0"
      delegate_to: "{{ physical_host }}"
      when:
        - physical_host != inventory_hostname
        - container_tech | default('lxc') == 'lxc'

    - name: Ensure Java is installed
      apt:
        name: openjdk-8-jre
        state: "{{ elk_package_state | default('present') }}"
        install_recommends: yes
        update_cache: yes
      register: _apt_task
      until: _apt_task is success
      retries: 3
      delay: 2
      tags:
        - package_install

    - name: Ensure Logstash is installed
      apt:
        name: "{{ item }}"
        state: "{{ elk_package_state | default('present') }}"
        update_cache: yes
      with_items:
        - logrotate
        - logstash
      register: _apt_task
      until: _apt_task is success
      retries: 3
      delay: 2
      notify:
        - Enable and restart logstash
      tags:
        - package_install

    - name: exit playbook after uninstall
      meta: end_play
      when:
        - (elk_package_state | default('present')) == 'absent'

  post_tasks:
    - name: Create logstash systemd service config dir
      file:
        path: "/etc/systemd/system/logstash.service.d"
        state: "directory"
        group: "root"
        owner: "root"
        mode: "0755"

    - name: Apply systemd options
      template:
        src: "{{ item.src }}"
        dest: "/etc/systemd/system/logstash.service.d/{{ item.dest }}"
        mode: "0644"
      with_items:
        - { src: "systemd.general-overrides.conf.j2", dest: "logstash-overrides.conf" }
      notify:
        - Enable and restart logstash

    - name: Create patterns directory
      file:
        name: "/opt/logstash/patterns"
        owner: "logstash"
        group: "logstash"
        state: directory
      tags:
        - logstash-patterns

    - name: Logstash Extra Patterns
      template:
        src: "{{ item }}"
        dest: "/opt/logstash/patterns/{{ item }}"
        owner: "logstash"
        group: "logstash"
      with_items:
        - extras
      when:
        - logstash_deploy_filters
      notify:
        - Enable and restart logstash
      tags:
        - logstash-filters
        - config

    - name: Run kafka output block
      block:
        - name: Copy kafka keystore into place
          copy:
            src: "{{ logstash_kafka_ssl_keystore_location }}"
            dest: "/var/lib/logstash/{{ logstash_kafka_ssl_keystore_location | basename }}"
          when:
            - logstash_kafka_ssl_keystore_location is defined

        - name: Copy kafka truststore into place
          copy:
            src: "{{ logstash_kafka_ssl_truststore_location }}"
            dest: "/var/lib/logstash/{{ logstash_kafka_ssl_truststore_location | basename }}"
          when:
            - logstash_kafka_ssl_truststore_location is defined

      when:
        - logstash_kafka_options is defined

    - name: Drop logstash conf file(s)
      template:
        src: "{{ item.src }}"
        dest: "{{ item.dest }}"
      with_items:
        - src: templates/jvm.options.j2
          dest: /etc/logstash/jvm.options
        - src: templates/logstash.yml.j2
          dest: /etc/logstash/logstash.yml
        - src: templates/logstash-pipelines.yml.j2
          dest: /etc/logstash/pipelines.yml
        - src: "templates/logrotate.j2"
          dest: "/etc/logrotate.d/logstash"
      notify:
        - Enable and restart logstash
      tags:
        - config

    - name: Ensure logstash ownership
      file:
        path: /var/lib/logstash
        owner: logstash
        group: logstash
        recurse: true
      register: l_perms
      until: l_perms is success
      retries: 3
      delay: 1

    - name: Ensure logstash tmp dir
      file:
        path: "/var/lib/logstash/tmp"
        state: directory
        owner: "logstash"
        group: "logstash"
        mode: "0750"

    - name: Run arcsight output block
      block:
        - name: Initialise arcsight local facts
          ini_file:
            dest: "/etc/ansible/facts.d/logstash.fact"
            section: arcsight
            option: "initialise"
            value: true

        - name: refresh local facts
          setup:
            filter: ansible_local
            gather_subset: "!all"
          tags:
            - nova-config

        - name: Setup arcsight smart connector
          shell: >-
            /usr/local/bin/logstash --modules arcsight {{ (ansible_local['arcsight'][item.host] is defined) | ternary('', '--setup') }}
              -M "arcsight.var.input.smartconnector.bootstrap_servers={{ item.host }}:{{ item.port }}"
              -M "arcsight.var.elasticsearch.hosts=localhost:{{ elastic_port }}"
              -M "arcsight.var.kibana.host={{ hostvars[groups['kibana'][0]]['ansible_host'] }}:{{ kibana_port }}"            
          with_items: "{{ logstash_arcsight_smart_connectors }}"
          run_once: true
          register: smart_connector
          until: smart_connector is success
          retries: 5
          delay: 5

        - name: Setup arcsight event broker
          shell: >-
            /usr/local/bin/logstash --modules arcsight {{ (ansible_local['arcsight'][item.host] is defined) | ternary('', '--setup') }}
              -M "arcsight.var.input.eventbroker.bootstrap_servers={{ item.host }}:{{ item.port }}"
              -M "arcsight.var.elasticsearch.hosts=localhost:{{ elastic_port }}"
              -M "arcsight.var.kibana.host={{ hostvars[groups['kibana'][0]]['ansible_host'] }}:{{ kibana_port }}"            
          with_items: "{{ logstash_arcsight_event_brokers }}"
          run_once: true
          register: event_broker
          until: event_broker is success
          retries: 5
          delay: 5

        - name: Set arcsight local facts
          ini_file:
            dest: "/etc/ansible/facts.d/logstash.fact"
            section: arcsight
            option: "{{ item.host }}"
            value: "{{ item.port }}"
          with_items: "{{ logstash_arcsight_smart_connectors | union(logstash_arcsight_event_brokers) }}"
      when:
        - logstash_arcsight_smart_connectors or
          logstash_arcsight_event_brokers

  handlers:
    - name: Enable and restart logstash
      systemd:
        name: "logstash"
        enabled: true
        state: restarted
        daemon_reload: true
      when:
        - (elk_package_state | default('present')) != 'absent'
      tags:
        - config

  tags:
    - server-install