From f87b238db52353884dffd60b86dea78c5139ebd4 Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Fri, 23 Feb 2018 17:36:27 +0000 Subject: [PATCH] Add support for deploying ZooKeeper This picks up the abandoned review: https://review.openstack.org/#/c/412423 Co-Authored-By: Sam Yaple Partially-Implements: blueprint monasca-roles Change-Id: Idc01afcb125271181ee5e15d327d0929f07b49e9 --- ansible/group_vars/all.yml | 5 ++ ansible/inventory/all-in-one | 3 + ansible/inventory/multinode | 3 + ansible/roles/common/tasks/config.yml | 2 + .../conf/filter/00-record_transformer.conf.j2 | 7 ++ .../templates/conf/input/06-zookeeper.conf.j2 | 11 +++ .../cron-logrotate-zookeeper.conf.j2 | 3 + ansible/roles/common/templates/cron.json.j2 | 1 + .../roles/common/templates/fluentd.json.j2 | 6 ++ ansible/roles/zookeeper/defaults/main.yml | 28 +++++++ ansible/roles/zookeeper/handlers/main.yml | 24 ++++++ ansible/roles/zookeeper/meta/main.yml | 3 + ansible/roles/zookeeper/tasks/check.yml | 1 + ansible/roles/zookeeper/tasks/config.yml | 76 +++++++++++++++++++ ansible/roles/zookeeper/tasks/deploy.yml | 5 ++ ansible/roles/zookeeper/tasks/main.yml | 2 + ansible/roles/zookeeper/tasks/precheck.yml | 21 +++++ ansible/roles/zookeeper/tasks/pull.yml | 10 +++ ansible/roles/zookeeper/tasks/reconfigure.yml | 2 + ansible/roles/zookeeper/tasks/upgrade.yml | 5 ++ ansible/roles/zookeeper/templates/myid.j2 | 5 ++ .../zookeeper/templates/zookeeper.cfg.j2 | 8 ++ .../zookeeper/templates/zookeeper.json.j2 | 29 +++++++ ansible/site.yml | 9 +++ etc/kolla/globals.yml | 1 + .../add-zookeeper-role-9eb474f26035ec77.yaml | 5 ++ 26 files changed, 275 insertions(+) create mode 100644 ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2 create mode 100644 ansible/roles/common/templates/cron-logrotate-zookeeper.conf.j2 create mode 100644 ansible/roles/zookeeper/defaults/main.yml create mode 100644 ansible/roles/zookeeper/handlers/main.yml create mode 100644 ansible/roles/zookeeper/meta/main.yml create mode 100644 ansible/roles/zookeeper/tasks/check.yml create mode 100644 ansible/roles/zookeeper/tasks/config.yml create mode 100644 ansible/roles/zookeeper/tasks/deploy.yml create mode 100644 ansible/roles/zookeeper/tasks/main.yml create mode 100644 ansible/roles/zookeeper/tasks/precheck.yml create mode 100644 ansible/roles/zookeeper/tasks/pull.yml create mode 100644 ansible/roles/zookeeper/tasks/reconfigure.yml create mode 100644 ansible/roles/zookeeper/tasks/upgrade.yml create mode 100644 ansible/roles/zookeeper/templates/myid.j2 create mode 100644 ansible/roles/zookeeper/templates/zookeeper.cfg.j2 create mode 100644 ansible/roles/zookeeper/templates/zookeeper.json.j2 create mode 100644 releasenotes/notes/add-zookeeper-role-9eb474f26035ec77.yaml diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index b5ccbb3d5e..81b8165391 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -303,6 +303,10 @@ trove_api_port: "8779" watcher_api_port: "9322" +zookeeper_client_port: "2181" +zookeeper_peer_port: "2888" +zookeeper_quorum_port: "3888" + zun_api_port: "9517" opendaylight_clustering_port: "2550" @@ -477,6 +481,7 @@ enable_trove: "no" enable_vitrage: "no" enable_vmtp: "no" enable_watcher: "no" +enable_zookeeper: "no" enable_zun: "no" ovs_datapath: "{{ 'netdev' if enable_ovs_dpdk | bool else 'system' }}" diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one index 31d1c0db43..fd50b41025 100644 --- a/ansible/inventory/all-in-one +++ b/ansible/inventory/all-in-one @@ -222,6 +222,9 @@ control [bifrost:children] deployment +[zookeeper:children] +control + [zun:children] control diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode index 29932f31f7..d2ba5b8b4c 100644 --- a/ansible/inventory/multinode +++ b/ansible/inventory/multinode @@ -241,6 +241,9 @@ control [bifrost:children] deployment +[zookeeper:children] +control + [zun:children] control diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml index 2dba74b3ba..89531bb730 100644 --- a/ansible/roles/common/tasks/config.yml +++ b/ansible/roles/common/tasks/config.yml @@ -51,6 +51,7 @@ - "03-rabbitmq" - "04-openstack-wsgi" - "05-libvirt" + - "06-zookeeper" notify: - Restart fluentd container @@ -217,6 +218,7 @@ - { name: "trove", enabled: "{{ enable_trove }}" } - { name: "vitrage", enabled: "{{ enable_vitrage }}" } - { name: "watcher", enabled: "{{ enable_watcher }}" } + - { name: "zookeeper", enabled: "{{ enable_zookeeper }}" } - { name: "zun", enabled: "{{ enable_zun }}" } notify: - Restart cron container diff --git a/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2 b/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2 index 12b9b566c9..37ea4c19e6 100644 --- a/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2 +++ b/ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2 @@ -7,6 +7,13 @@ + + @type record_transformer + + Logger ${tag_parts[4]} + + + @type record_transformer diff --git a/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2 b/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2 new file mode 100644 index 0000000000..e80b8a3c32 --- /dev/null +++ b/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2 @@ -0,0 +1,11 @@ +{% set fluentd_dir = 'td-agent' if kolla_base_distro in ['ubuntu', 'debian'] else 'fluentd' %} + + @type tail + path /var/log/kolla/zookeeper/zookeeper.log + pos_file /var/run/{{ fluentd_dir }}/zookeeper.pos + tag infra.* + format multiline + format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} \S+ \S+ \S+ .*$/ + format1 /^(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}) \[(?\S+)\] \S+ (?\S+) (?.*)$/ + time_key Timestamp + diff --git a/ansible/roles/common/templates/cron-logrotate-zookeeper.conf.j2 b/ansible/roles/common/templates/cron-logrotate-zookeeper.conf.j2 new file mode 100644 index 0000000000..b2d07766e2 --- /dev/null +++ b/ansible/roles/common/templates/cron-logrotate-zookeeper.conf.j2 @@ -0,0 +1,3 @@ +"/var/log/kolla/zookeeper/*.log" +{ +} diff --git a/ansible/roles/common/templates/cron.json.j2 b/ansible/roles/common/templates/cron.json.j2 index 8601208172..bfe3c4c1c4 100644 --- a/ansible/roles/common/templates/cron.json.j2 +++ b/ansible/roles/common/templates/cron.json.j2 @@ -53,6 +53,7 @@ ( 'trove', enable_trove ), ( 'vitrage', enable_vitrage ), ( 'watcher', enable_watcher ), + ( 'zookeeper', enable_zookeeper ), ( 'zun', enable_zun ) ] %} { diff --git a/ansible/roles/common/templates/fluentd.json.j2 b/ansible/roles/common/templates/fluentd.json.j2 index f711699444..c622b0a664 100644 --- a/ansible/roles/common/templates/fluentd.json.j2 +++ b/ansible/roles/common/templates/fluentd.json.j2 @@ -47,6 +47,12 @@ "owner": "{{ fluentd_user }}", "perm": "0600" }, + { + "source": "{{ container_config_directory }}/input/06-zookeeper.conf", + "dest": "{{ fluentd_dir }}/input/06-zookeeper.conf", + "owner": "{{ fluentd_user }}", + "perm": "0600" + }, {# Copy all configuration files in filter/ directory to include #} {# custom filter configs. #} { diff --git a/ansible/roles/zookeeper/defaults/main.yml b/ansible/roles/zookeeper/defaults/main.yml new file mode 100644 index 0000000000..125165e729 --- /dev/null +++ b/ansible/roles/zookeeper/defaults/main.yml @@ -0,0 +1,28 @@ +--- +zookeeper_services: + zookeeper: + container_name: zookeeper + group: zookeeper + enabled: true + image: "{{ zookeeper_image_full }}" + environment: + ZOO_LOG_DIR: /var/log/kolla/zookeeper + ZOO_LOG4J_PROP: "{{ zookeeper_log_settings }}" + volumes: + - "{{ node_config_directory }}/zookeeper/:{{ container_config_directory }}/" + - "/etc/localtime:/etc/localtime:ro" + - "zookeeper:/var/lib/zookeeper/data" + - "kolla_logs:/var/log/kolla/" + +#################### +# Zookeeper +#################### +zookeeper_log_settings: 'INFO,ROLLINGFILE' + +#################### +# Docker +#################### +zookeeper_install_type: "{{ kolla_install_type }}" +zookeeper_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ zookeeper_install_type }}-zookeeper" +zookeeper_tag: "{{ openstack_release }}" +zookeeper_image_full: "{{ zookeeper_image }}:{{ zookeeper_tag }}" diff --git a/ansible/roles/zookeeper/handlers/main.yml b/ansible/roles/zookeeper/handlers/main.yml new file mode 100644 index 0000000000..3ba27a0c15 --- /dev/null +++ b/ansible/roles/zookeeper/handlers/main.yml @@ -0,0 +1,24 @@ +--- +- name: Restart zookeeper container + vars: + service_name: "zookeeper" + service: "{{ zookeeper_services[service_name] }}" + config_json: "{{ zookeeper_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + zookeeper_conf: "{{ zookeeper_confs.results|selectattr('item.key', 'equalto', service_name)|first }}" + zookeeper_instance_id: "{{ zookeeper_instance_id.results|selectattr('item.key', 'equalto', service_name)|first }}" + zookeeper_container: "{{ check_zookeeper_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + environment: "{{ service.environment }}" + volumes: "{{ service.volumes }}" + when: + - action != "config" + - inventory_hostname in groups[service.group] + - service.enabled | bool + - config_json.changed | bool + or zookeeper_conf.changed | bool + or zookeeper_instance_id.changed | bool + or zookeeper_container.changed | bool diff --git a/ansible/roles/zookeeper/meta/main.yml b/ansible/roles/zookeeper/meta/main.yml new file mode 100644 index 0000000000..6b4fff8fef --- /dev/null +++ b/ansible/roles/zookeeper/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - { role: common } diff --git a/ansible/roles/zookeeper/tasks/check.yml b/ansible/roles/zookeeper/tasks/check.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/ansible/roles/zookeeper/tasks/check.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/roles/zookeeper/tasks/config.yml b/ansible/roles/zookeeper/tasks/config.yml new file mode 100644 index 0000000000..b69c5b19e5 --- /dev/null +++ b/ansible/roles/zookeeper/tasks/config.yml @@ -0,0 +1,76 @@ +--- +- name: Ensuring config directories exist + file: + path: "{{ node_config_directory }}/{{ item.key }}" + state: "directory" + owner: "{{ config_owner_user }}" + group: "{{ config_owner_group }}" + mode: "0770" + recurse: yes + become: true + when: + - inventory_hostname in groups[item.value.group] + - item.value.enabled | bool + with_dict: "{{ zookeeper_services }}" + +- name: Copying over config.json files for services + template: + src: "{{ item.key }}.json.j2" + dest: "{{ node_config_directory }}/{{ item.key }}/config.json" + mode: "0660" + become: true + register: zookeeper_config_jsons + when: + - inventory_hostname in groups[item.value.group] + - item.value.enabled | bool + with_dict: "{{ zookeeper_services }}" + notify: + - Restart zookeeper container + +- name: Copying over zookeeper configuration + merge_configs: + sources: + - "{{ role_path }}/templates/{{ item.key }}.cfg.j2" + - "{{ node_custom_config }}/{{ item.key }}.cfg" + - "{{ node_custom_config }}/{{ item.key }}/{{ inventory_hostname }}/{{ item.key }}.cfg" + dest: "{{ node_config_directory }}/{{ item.key }}/{{ item.key }}.cfg" + mode: "0660" + become: true + register: zookeeper_confs + when: + - inventory_hostname in groups[item.value.group] + - item.value.enabled | bool + with_dict: "{{ zookeeper_services }}" + notify: + - Restart zookeeper container + +- name: Copying over zookeeper instance id + template: + src: "myid.j2" + dest: "{{ node_config_directory }}/{{ item.key }}/myid" + mode: "0660" + become: true + register: zookeeper_instance_id + when: + - inventory_hostname in groups[item.value.group] + - item.value.enabled | bool + with_dict: "{{ zookeeper_services }}" + notify: + - Restart zookeeper container + +- name: Check zookeeper containers + kolla_docker: + action: "compare_container" + common_options: "{{ docker_common_options }}" + name: "{{ item.value.container_name }}" + image: "{{ item.value.image }}" + volumes: "{{ item.value.volumes }}" + environment: "{{ item.value.environment }}" + register: check_zookeeper_containers + when: + - action != "config" + - inventory_hostname in groups[item.value.group] + - item.value.enabled | bool + with_dict: "{{ zookeeper_services }}" + notify: + - Restart zookeeper container diff --git a/ansible/roles/zookeeper/tasks/deploy.yml b/ansible/roles/zookeeper/tasks/deploy.yml new file mode 100644 index 0000000000..dd26ecc34d --- /dev/null +++ b/ansible/roles/zookeeper/tasks/deploy.yml @@ -0,0 +1,5 @@ +--- +- include: config.yml + +- name: Flush handlers + meta: flush_handlers diff --git a/ansible/roles/zookeeper/tasks/main.yml b/ansible/roles/zookeeper/tasks/main.yml new file mode 100644 index 0000000000..b017e8b4ad --- /dev/null +++ b/ansible/roles/zookeeper/tasks/main.yml @@ -0,0 +1,2 @@ +--- +- include: "{{ action }}.yml" diff --git a/ansible/roles/zookeeper/tasks/precheck.yml b/ansible/roles/zookeeper/tasks/precheck.yml new file mode 100644 index 0000000000..38aeeb95c8 --- /dev/null +++ b/ansible/roles/zookeeper/tasks/precheck.yml @@ -0,0 +1,21 @@ +--- +- name: Get container facts + kolla_container_facts: + name: + - zookeeper + register: container_facts + +- name: Checking zookeeper ports are available + wait_for: + host: "{{ api_interface_address }}" + port: "{{ item }}" + connect_timeout: 1 + timeout: 1 + state: stopped + with_items: + - "{{ zookeeper_client_port }}" + - "{{ zookeeper_peer_port }}" + - "{{ zookeeper_quorum_port }}" + when: + - container_facts['zookeeper'] is not defined + - inventory_hostname in groups['zookeeper'] diff --git a/ansible/roles/zookeeper/tasks/pull.yml b/ansible/roles/zookeeper/tasks/pull.yml new file mode 100644 index 0000000000..f5614aca34 --- /dev/null +++ b/ansible/roles/zookeeper/tasks/pull.yml @@ -0,0 +1,10 @@ +--- +- name: Pulling zookeeper images + kolla_docker: + action: "pull_image" + common_options: "{{ docker_common_options }}" + image: "{{ item.value.image }}" + when: + - inventory_hostname in groups[item.value.group] + - item.value.enabled | bool + with_dict: "{{ zookeeper_services }}" diff --git a/ansible/roles/zookeeper/tasks/reconfigure.yml b/ansible/roles/zookeeper/tasks/reconfigure.yml new file mode 100644 index 0000000000..e078ef1318 --- /dev/null +++ b/ansible/roles/zookeeper/tasks/reconfigure.yml @@ -0,0 +1,2 @@ +--- +- include: deploy.yml diff --git a/ansible/roles/zookeeper/tasks/upgrade.yml b/ansible/roles/zookeeper/tasks/upgrade.yml new file mode 100644 index 0000000000..dd26ecc34d --- /dev/null +++ b/ansible/roles/zookeeper/tasks/upgrade.yml @@ -0,0 +1,5 @@ +--- +- include: config.yml + +- name: Flush handlers + meta: flush_handlers diff --git a/ansible/roles/zookeeper/templates/myid.j2 b/ansible/roles/zookeeper/templates/myid.j2 new file mode 100644 index 0000000000..4492f393cb --- /dev/null +++ b/ansible/roles/zookeeper/templates/myid.j2 @@ -0,0 +1,5 @@ +{% for host in groups['zookeeper'] -%} +{% if hostvars[host]['ansible_hostname'] == ansible_hostname -%} +{{ loop.index }} +{%- endif %} +{%- endfor %} diff --git a/ansible/roles/zookeeper/templates/zookeeper.cfg.j2 b/ansible/roles/zookeeper/templates/zookeeper.cfg.j2 new file mode 100644 index 0000000000..f7ab93d647 --- /dev/null +++ b/ansible/roles/zookeeper/templates/zookeeper.cfg.j2 @@ -0,0 +1,8 @@ +tickTime=2000 +initLimit=10 +syncLimit=5 +dataDir=/var/lib/zookeeper/data +clientPort={{ zookeeper_client_port }} +{% for host in groups['zookeeper'] %} +server.{{ loop.index }}={{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ zookeeper_peer_port }}:{{ zookeeper_quorum_port }} +{% endfor %} diff --git a/ansible/roles/zookeeper/templates/zookeeper.json.j2 b/ansible/roles/zookeeper/templates/zookeeper.json.j2 new file mode 100644 index 0000000000..9d9d609901 --- /dev/null +++ b/ansible/roles/zookeeper/templates/zookeeper.json.j2 @@ -0,0 +1,29 @@ +{ + "command": "/opt/zookeeper/bin/zkServer.sh start-foreground /etc/zookeeper/conf/zoo.cfg", + "config_files": [ + { + "source": "{{ container_config_directory }}/myid", + "dest": "/var/lib/zookeeper/data/myid", + "owner": "zookeeper", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/zookeeper.cfg", + "dest": "/etc/zookeeper/conf/zoo.cfg", + "owner": "zookeeper", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/lib/zookeeper", + "owner": "zookeeper:zookeeper", + "recurse": true + }, + { + "path": "/var/log/kolla/zookeeper", + "owner": "zookeeper:zookeeper", + "recurse": true + } + ] +} diff --git a/ansible/site.yml b/ansible/site.yml index 4c17417920..a599338171 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -75,6 +75,15 @@ tags: collectd, when: enable_collectd | bool } +- name: Apply role zookeeper + gather_facts: false + hosts: zookeeper + serial: '{{ serial|default("0") }}' + roles: + - { role: zookeeper, + tags: zookeeper, + when: enable_zookeeper | bool } + - name: Apply role elasticsearch gather_facts: false hosts: elasticsearch diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index 1b57792752..067739de7a 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -234,6 +234,7 @@ kolla_internal_vip_address: "10.10.10.254" #enable_vitrage: "no" #enable_vmtp: "no" #enable_watcher: "no" +#enable_zookeeper: "no" #enable_zun: "no" ############## diff --git a/releasenotes/notes/add-zookeeper-role-9eb474f26035ec77.yaml b/releasenotes/notes/add-zookeeper-role-9eb474f26035ec77.yaml new file mode 100644 index 0000000000..84f0cd38ba --- /dev/null +++ b/releasenotes/notes/add-zookeeper-role-9eb474f26035ec77.yaml @@ -0,0 +1,5 @@ +--- +features: + - Add a role for deploying Apache Zookeeper for the purpose of + supporting Apache Kafka. See https://zookeeper.apache.org/ + for more details.