From afca2222b295326cd758f9e3cb31f241b6cc2e46 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Evrard Date: Tue, 24 Apr 2018 21:36:21 +0000 Subject: [PATCH] Add Graylog Central Logging This adds a central logging using graylog2 with little to no code. Change-Id: I63e59e249329ececf0598552b95329b38b4ed32c --- graylog/README.rst | 119 ++++++++++++++++++++++++++ graylog/ansible-role-requirements.yml | 16 ++++ graylog/ansible.cfg | 2 + graylog/graylog-forward-logs.yml | 44 ++++++++++ graylog/graylog2-install.yml | 76 ++++++++++++++++ 5 files changed, 257 insertions(+) create mode 100644 graylog/README.rst create mode 100644 graylog/ansible-role-requirements.yml create mode 100644 graylog/ansible.cfg create mode 100644 graylog/graylog-forward-logs.yml create mode 100644 graylog/graylog2-install.yml diff --git a/graylog/README.rst b/graylog/README.rst new file mode 100644 index 00000000..965bfee6 --- /dev/null +++ b/graylog/README.rst @@ -0,0 +1,119 @@ +Central Logging with Graylog2 +============================= + +Introduction +------------ + +This part of the ops repo is in charge of: + +* Setting up Graylog2 into the ``graylog_hosts`` group +* Shipping all your hosts logs into Graylog2 using graylog native format (GELF) +* Configuring haproxy for Graylog2 + +Current limitations +------------------- + +The upstream Graylog2 ansible role doesn't currently support deploying in a cluster +setup, and therefore the deploy needs to be restricted to one backend for now: +https://github.com/Graylog2/graylog-ansible-role/issues/89. It is all due to the +fact the authentication sessions have to be shared on a mongoDB cluster, and no +role is available to build the mongo cluster. Patches welcomed! + +Fetching the roles +------------------ + +To install Graylog2 you need to make sure all the necessary roles are in your environment, +if you don't have them already. + +You can re-use the bootstrap-ansible script with this ansible-role-requirement file +(see the OpenStack-Ansible reference documentation), or, simply run:: + + ansible-galaxy install -r ansible-role-requirements.yml + + +Installing Graylog2 on graylog_hosts +------------------------------------ + +Add a file in /etc/openstack_deploy/user_graylog.yml, with the following content:: + + graylog_password_secret: "" # The output of `pwgen -N 1 -s 96` + graylog_root_username: "admin" + graylog_root_password_sha2: "" # The output of `echo -n yourpassword | shasum -a 256` + haproxy_extra_services: + - service: + haproxy_service_name: graylog + haproxy_backend_nodes: "{{ [groups['graylog_hosts'][0]] | default([]) }}" + haproxy_ssl: "{{ haproxy_ssl }}" + haproxy_port: 9000 + haproxy_balance_type: http + +See more Graylog2 deploy variables in +https://github.com/Graylog2/graylog-ansible-role/blob/e1159ec2712199f2da5768187cee84d1359bbd55/defaults/main.yml + +If you want the ``graylog_hosts`` group to match the existing ``log_hosts`` group, +add the following in your ``/etc/openstack_deploy/inventory.ini``:: + + [graylog_hosts:children] + log_hosts + +To deploy Graylog2, simply run the install playbook:: + + openstack-ansible graylog2-install.yml + +To point haproxy to your new Graylog2 instance, re-run the ``haproxy-install.yml`` playbook. + +Note: If running Graylog2 on the same host as the load balancer, you'll hit an issue with an already +taken port. In that case, either don't configure haproxy, or configure it to run on an interface not yet +bound. For example, you can use the following line in your ``user_graylog.yml`` haproxy service section +to bind only on the external lb vip address:: + + haproxy_bind: "{{ [external_lb_vip_address] }}" + +Note: You can optionally add a series of headers in your haproxy to help on the web interface +redirection, if you have a specific network configuration. + + http-request set-header X-Graylog-Server-URL https://{{ external_lb_vip_address }}:9000/api + +Configuration of Graylog2 +------------------------- + +Connect as the interface on your loadbalancer address, port 9000, with the user ``admin``, and the +previously defined password whose shasum was given into ``graylog_root_password_sha2``. + +In the web interface, add the inputs you need. + +If you want to configure your your nodes with the provided playbook, you will need to +create a new GELF UDP input on at least one of your Graylog2 nodes (select ``global`` if you want to +listen on all the nodes). + +For the exercise, we are defining the port to listen to as UDP 12201. + +Sending logs to Graylog2 +------------------------ + +Graylog2 can receive data with different protocols, but there is an efficient native format for it, GELF. + +All of this is configured in a single playbook: ``graylog-forward-logs.yml``. + +There are many packages to forward the journal into Graylog2, like the official `journal2gelf`_. +The ``graylog-ship-logs.yml`` playbook uses a fork of `journal2gelf` using `gelfclient`_. +It's lightweight and easy to install. + +This script needs to know where to forward to, and depends on how you configured Graylog2 at the +previous step. + +In the example above, the following variables need to be set in +``/etc/openstack_deploy/user_graylog.yml``:: + + graylog_targets: + - "{{ groups['graylog_hosts'][0] }}:12201" + +If you are shipping journals directly from containers to the host, there is no need to run this playbook +on the full list of nodes. Instead, use the ansible ``--limit`` directive to restrict on which host +this playbook should run. + +That's all folks! + +.. _journal2gelf: https://github.com/systemd/journal2gelf +.. _gelfclient: https://github.com/nailgun/journal2gelf + diff --git a/graylog/ansible-role-requirements.yml b/graylog/ansible-role-requirements.yml new file mode 100644 index 00000000..abca0149 --- /dev/null +++ b/graylog/ansible-role-requirements.yml @@ -0,0 +1,16 @@ +--- +- name: elastic.elasticsearch + src: https://github.com/elastic/ansible-elasticsearch.git + version: 3bdcd8fe4d0afdc2da5e12475b2093bb2bb3326b + +- name: jdauphant.nginx + src: https://github.com/jdauphant/ansible-role-nginx.git + version: 'v2.7.4' + +- name: geerlingguy.java + src: https://github.com/geerlingguy/ansible-role-java + version: ebe72b1b52fe0053bb156fd1b29d044f2048556b + +- name: Graylog2.graylog-ansible-role + src: https://github.com/Graylog2/graylog-ansible-role.git + version: e1159ec2712199f2da5768187cee84d1359bbd55 diff --git a/graylog/ansible.cfg b/graylog/ansible.cfg new file mode 100644 index 00000000..ae190d32 --- /dev/null +++ b/graylog/ansible.cfg @@ -0,0 +1,2 @@ +[defaults] +roles_path = /etc/ansible/roles diff --git a/graylog/graylog-forward-logs.yml b/graylog/graylog-forward-logs.yml new file mode 100644 index 00000000..ac787e94 --- /dev/null +++ b/graylog/graylog-forward-logs.yml @@ -0,0 +1,44 @@ +--- +- hosts: all:!log_hosts + gather_facts: no + vars: + graylog_forwarder_system_packages: + - python-systemd + graylog_forwarder_pip_packages: + - gelfclient==0.0.7 + - journal2gelf==2.0.0 + tasks: + #- name: Gather variables for each operating system + # include_vars: "{{ item }}" + # with_first_found: + # - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + # - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + # - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + # - "{{ ansible_distribution | lower }}.yml" + # - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + # - "{{ ansible_os_family | lower }}.yml" + # tags: + # - always + + - name: Install graylog forwarder package requirements + package: + name: "{{ graylog_forwarder_system_packages }}" + state: present + # Graylog wasn't build in repo, and requires running "isolated" + - name: Install graylog forwarder requirements + pip: + name: "{{ graylog_forwarder_pip_packages }}" + state: present + - name: Install the log forwarder service. + include_role: + name: systemd_service + with_items: "{{ graylog_targets }}" + loop_control: + loop_var: graylog_target + vars: + systemd_services: + - service_name: gelf-forwarder + state: started + execstarts: + - "/usr/local/bin/journal2gelf {{ graylog_target }}" + diff --git a/graylog/graylog2-install.yml b/graylog/graylog2-install.yml new file mode 100644 index 00000000..b6384fc4 --- /dev/null +++ b/graylog/graylog2-install.yml @@ -0,0 +1,76 @@ +--- +- name: Ensure sysctl + hosts: graylog_hosts + gather_facts: true + tasks: + - name: Setup sysctl + sysctl: + name: vm.max_map_count + value: "262144" + state: present + delegate_to: "{{ physical_host }}" + +- name: Install java from openjdk + hosts: graylog_hosts + tasks: + # TODO: Replace this with a group var to log_hosts to use openstack_hosts role. + - name: installing repo for Java 8 in Ubuntu 16.04 + apt_repository: repo='ppa:openjdk-r/ppa' + when: ansible_distribution_release == 'xenial' + + - name: Install Java for Ubuntu + include_role: + name: geerlingguy.java + when: + - ansible_os_family | lower == 'debian' + - ansible_distribution_release == 'xenial' + vars: + java_packages: + - openjdk-8-jdk + + # TODO: Add SUSE support + - name: Install Java on CentOS + package: + name: java-1.8.0-openjdk-headless.x86_64 + state: present + when: ansible_os_family | lower == 'redhat' + +- name: Install graylog + hosts: graylog_hosts + vars: + # Graylog is compatible with elasticsearch 5.x since version 2.3.0, so ensure to use the right combination for your installation + # Also use the right branch of the Elasticsearch Ansible role, master supports 5.x. + es_java_install: False + es_java: openjdk-8-jre-headless + es_major_version: "5.x" + es_version: "5.6.7" + es_instance_name: 'graylog' + es_scripts: False + es_templates: False + es_version_lock: False + es_heap_size: 1g + es_config: { + node.name: "graylog", + cluster.name: "graylog", + http.port: 9200, + transport.tcp.port: 9300, + network.host: "{{ es_bind_address | default('0.0.0.0') }}", + node.data: true, + node.master: "{{ inventory_hostname == groups['graylog_hosts'][0] }}", + } + + graylog_install_java: False + graylog_install_mongodb: True + graylog_install_nginx: False #Will be behind your LB + graylog_web_endpoint_uri: "https://{{ external_lb_vip_address }}:9000/api/" + graylog_is_master: "{{ inventory_hostname == groups['graylog_hosts'][0] }}" + graylog_elasticsearch_hosts: "{{ groups['graylog_hosts'] | map('extract', hostvars, 'ansible_host') | map('regex_replace', '^(.*)$', 'http://\\1:9200') | join(', ') }}" + graylog_web_listen_uri: "http://{{ ansible_host }}:9000/" + graylog_rest_listen_uri: "http://{{ ansible_host }}:9000/api/" + # TODO(evrardjp): Replace this with a proper test when + # https://github.com/Graylog2/graylog-ansible-role/pull/88 has merged + graylog_not_testing: False + roles: + # TODO: Contribute to the role for SUSE support + - role: Graylog2.graylog-ansible-role + tags: graylog