ADD venus for kolla-ansible

This project [1] can provide a one-stop solution to log collection,
cleaning, indexing, analysis, alarm, visualization, report generation
and other needs, which involves helping operator or maintainer to
quickly solve retrieve problems, grasp the operational health of the
platform, and improve the level of platform management.

[1] https://wiki.openstack.org/wiki/Venus

Change-Id: If3562bbed6181002b76831bab54f863041c5a885
This commit is contained in:
jinyuanliu 2021-09-17 05:59:43 -04:00
parent f26b9cd8ad
commit 3ccb176f13
33 changed files with 540 additions and 0 deletions

View File

@ -517,6 +517,8 @@ tacker_server_port: "9890"
trove_api_port: "8779"
venus_api_port: "10010"
watcher_api_port: "9322"
zookeeper_client_port: "2181"
@ -710,6 +712,7 @@ enable_tacker: "no"
enable_telegraf: "no"
enable_trove: "no"
enable_trove_singletenant: "no"
enable_venus: "no"
enable_vitrage: "no"
enable_watcher: "no"
enable_zookeeper: "{{ enable_kafka | bool or enable_storm | bool }}"

View File

@ -735,3 +735,12 @@ ovn-database
[ovn-sb-db:children]
ovn-database
[venus:children]
monitoring
[venus-api:children]
venus
[venus-manager:children]
venus

View File

@ -753,3 +753,12 @@ ovn-database
[ovn-sb-db:children]
ovn-database
[venus:children]
monitoring
[venus-api:children]
venus
[venus-manager:children]
venus

View File

@ -201,6 +201,7 @@
- { name: "swift", enabled: "{{ enable_swift | bool }}" }
- { name: "tacker", enabled: "{{ enable_tacker | bool }}" }
- { name: "trove", enabled: "{{ enable_trove | bool }}" }
- { name: "venus", enabled: "{{ enable_venus | bool }}" }
- { name: "vitrage", enabled: "{{ enable_vitrage | bool }}" }
- { name: "watcher", enabled: "{{ enable_watcher | bool }}" }
- { name: "zookeeper", enabled: "{{ enable_zookeeper | bool }}" }

View File

@ -166,6 +166,11 @@
pattern ^(masakari-engine|masakari-api)$
tag openstack_python
</rule>
<rule>
key programname
pattern ^(venus-api|venus-manager)$
tag openstack_python
</rule>
<rule>
key programname
pattern .+

View File

@ -0,0 +1,3 @@
"/var/log/kolla/venus/*.log"
{
}

View File

@ -0,0 +1,138 @@
---
venus_services:
venus-api:
container_name: venus_api
group: venus-api
enabled: true
image: "{{ venus_api_image_full }}"
volumes: "{{ venus_api_default_volumes + venus_api_extra_volumes }}"
dimensions: "{{ venus_api_dimensions }}"
healthcheck: "{{ venus_api_healthcheck }}"
haproxy:
venus_api:
enabled: "{{ enable_venus }}"
mode: "http"
external: false
port: "{{ venus_api_port }}"
venus_api_external:
enabled: "{{ enable_venus }}"
mode: "http"
external: true
port: "{{ venus_api_port }}"
venus-manager:
container_name: venus_manager
group: venus-manager
enabled: true
image: "{{ venus_manager_image_full }}"
volumes: "{{ venus_manager_default_volumes + venus_manager_extra_volumes }}"
dimensions: "{{ venus_manager_dimensions }}"
####################
# Database
####################
venus_database_name: "venus"
venus_database_user: "{% if use_preconfigured_databases | bool and use_common_mariadb_user | bool %}{{ database_user }}{% else %}venus{% endif %}"
venus_database_address: "{{ database_address | put_address_in_context('url') }}:{{ database_port }}"
####################
# Docker
####################
venus_install_type: "{{ kolla_install_type }}"
venus_tag: "{{ openstack_tag }}"
venus_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ venus_install_type }}-venus-api"
venus_api_tag: "{{ venus_tag }}"
venus_api_image_full: "{{ venus_api_image }}:{{ venus_api_tag }}"
venus_manager_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ venus_install_type }}-venus-manager"
venus_manager_tag: "{{ venus_tag }}"
venus_manager_image_full: "{{ venus_manager_image }}:{{ venus_manager_tag }}"
venus_api_dimensions: "{{ default_container_dimensions }}"
venus_manager_dimensions: "{{ default_container_dimensions }}"
venus_api_enable_healthchecks: "{{ enable_container_healthchecks }}"
venus_api_healthcheck_interval: "{{ default_container_healthcheck_interval }}"
venus_api_healthcheck_retries: "{{ default_container_healthcheck_retries }}"
venus_api_healthcheck_start_period: "{{ default_container_healthcheck_start_period }}"
venus_api_healthcheck_test: ["CMD-SHELL", "healthcheck_curl http://{{ api_interface_address | put_address_in_context('url') }}:{{ venus_api_port }}"]
venus_api_healthcheck_timeout: "{{ default_container_healthcheck_timeout }}"
venus_api_healthcheck:
interval: "{{ venus_api_healthcheck_interval }}"
retries: "{{ venus_api_healthcheck_retries }}"
start_period: "{{ venus_api_healthcheck_start_period }}"
test: "{% if venus_api_enable_healthchecks | bool %}{{ venus_api_healthcheck_test }}{% else %}NONE{% endif %}"
timeout: "{{ venus_api_healthcheck_timeout }}"
venus_api_default_volumes:
- "{{ node_config_directory }}/venus-api/:{{ container_config_directory }}/:ro"
- "/etc/localtime:/etc/localtime:ro"
- "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
- "kolla_logs:/var/log/kolla/"
- "{{ kolla_dev_repos_directory ~ '/venus/venus:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/venus' if venus_dev_mode | bool else '' }}"
- "venus:/var/lib/venus/"
venus_manager_default_volumes:
- "{{ node_config_directory }}/venus-manager/:{{ container_config_directory }}/:ro"
- "/etc/localtime:/etc/localtime:ro"
- "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
- "kolla_logs:/var/log/kolla/"
- "{{ kolla_dev_repos_directory ~ '/venus/venus:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/venus' if venus_dev_mode | bool else '' }}"
- "venus:/var/lib/venus/"
venus_extra_volumes: "{{ default_extra_volumes }}"
venus_api_extra_volumes: "{{ venus_extra_volumes }}"
venus_manager_extra_volumes: "{{ venus_extra_volumes }}"
####################
# OpenStack
####################
venus_internal_endpoint: "{{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ venus_api_port }}/v1.0/%(tenant_id)s"
venus_public_endpoint: "{{ public_protocol }}://{{ kolla_external_fqdn | put_address_in_context('url') }}:{{ venus_api_port }}/v1.0/%(tenant_id)s"
venus_logging_debug: "{{ openstack_logging_debug }}"
venus_keystone_user: "venus"
openstack_venus_auth: "{{ openstack_auth }}"
####################
# Kolla
####################
venus_git_repository: "https://opendev.org/inspur/venus"
venus_dev_repos_pull: "{{ kolla_dev_repos_pull }}"
venus_dev_mode: "{{ kolla_dev_mode }}"
venus_source_version: "{{ kolla_source_version }}"
####################
# logging
####################
openstack_logging_default_format_string: "%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [- req-None - - - - -] %(instance)s%(message)s"
openstack_logging_context_format_string: "%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(user_identity)s] %(instance)s%(message)s"
####################
# Notifications
####################
venus_notification_topics:
- name: notifications
enabled: "{{ enable_ceilometer | bool }}"
venus_enabled_notification_topics: "{{ venus_notification_topics | selectattr('enabled', 'equalto', true) | list }}"
####################
# Keystone
####################
venus_ks_services:
- name: "venus"
type: "LMS"
description: "Log Manager Service"
endpoints:
- {'interface': 'internal', 'url': '{{ venus_internal_endpoint }}'}
- {'interface': 'public', 'url': '{{ venus_public_endpoint }}'}
venus_ks_users:
- project: "service"
user: "{{ venus_keystone_user }}"
password: "{{ venus_keystone_password }}"
role: "admin"

View File

@ -0,0 +1,32 @@
---
- name: Restart venus-api container
vars:
service_name: "venus-api"
service: "{{ venus_services[service_name] }}"
become: true
kolla_docker:
action: "recreate_or_restart_container"
common_options: "{{ docker_common_options }}"
name: "{{ service.container_name }}"
image: "{{ service.image }}"
volumes: "{{ service.volumes|reject('equalto', '')|list }}"
dimensions: "{{ service.dimensions }}"
healthcheck: "{{ service.healthcheck | default(omit) }}"
when:
- kolla_action != "config"
- name: Restart venus-manager container
vars:
service_name: "venus-manager"
service: "{{ venus_services[service_name] }}"
become: true
kolla_docker:
action: "recreate_or_restart_container"
common_options: "{{ docker_common_options }}"
name: "{{ service.container_name }}"
image: "{{ service.image }}"
volumes: "{{ service.volumes|reject('equalto', '')|list }}"
dimensions: "{{ service.dimensions }}"
healthcheck: "{{ service.healthcheck | default(omit) }}"
when:
- kolla_action != "config"

View File

@ -0,0 +1,34 @@
---
- name: Creating venus database
become: true
kolla_toolbox:
module_name: mysql_db
module_args:
login_host: "{{ database_address }}"
login_port: "{{ database_port }}"
login_user: "{{ database_user }}"
login_password: "{{ database_password }}"
name: "{{ venus_database_name }}"
run_once: True
delegate_to: "{{ groups['venus-api'][0] }}"
when:
- not use_preconfigured_databases | bool
- name: Creating venus database user and setting permissions
become: true
kolla_toolbox:
module_name: mysql_user
module_args:
login_host: "{{ database_address }}"
login_port: "{{ database_port }}"
login_user: "{{ database_user }}"
login_password: "{{ database_password }}"
name: "{{ venus_database_user }}"
password: "{{ venus_database_password }}"
host: "%"
priv: "{{ venus_database_name }}.*:ALL"
append_privs: "yes"
run_once: True
delegate_to: "{{ groups['venus-api'][0] }}"
when:
- not use_preconfigured_databases | bool

View File

@ -0,0 +1,17 @@
---
- name: Check venus containers
become: true
kolla_docker:
action: "compare_container"
common_options: "{{ docker_common_options }}"
name: "{{ item.value.container_name }}"
image: "{{ item.value.image }}"
volumes: "{{ item.value.volumes|reject('equalto', '')|list }}"
dimensions: "{{ item.value.dimensions }}"
healthcheck: "{{ item.value.healthcheck | default(omit) }}"
when:
- inventory_hostname in groups[item.value.group]
- item.value.enabled | bool
with_dict: "{{ venus_services }}"
notify:
- "Restart {{ item.key }} container"

View File

@ -0,0 +1 @@
---

View File

@ -0,0 +1,8 @@
---
- name: Cloning venus source repository for development
become: true
git:
repo: "{{ venus_git_repository }}"
dest: "{{ kolla_dev_repos_directory }}/{{ project_name }}"
update: "{{ venus_dev_repos_pull }}"
version: "{{ venus_source_version }}"

View File

@ -0,0 +1,81 @@
---
- name: Ensuring config directories exist
file:
path: "{{ node_config_directory }}/{{ item.key }}"
state: "directory"
owner: "{{ config_owner_user }}"
group: "{{ config_owner_group }}"
mode: "0770"
become: true
when:
- inventory_hostname in groups[item.value.group]
- item.value.enabled | bool
with_dict: "{{ venus_services }}"
- name: Check if policies shall be overwritten
stat:
path: "{{ item }}"
run_once: True
delegate_to: localhost
register: venus_policy
with_first_found:
- files: "{{ supported_policy_format_list }}"
paths:
- "{{ node_custom_config }}/venus/"
skip: true
- name: Set venus policy file
set_fact:
venus_policy_file: "{{ venus_policy.results.0.stat.path | basename }}"
venus_policy_file_path: "{{ venus_policy.results.0.stat.path }}"
when:
- venus_policy.results
- include_tasks: copy-certs.yml
when:
- kolla_copy_ca_into_containers | bool
- name: Copying over config.json files for services
template:
src: "{{ item.key }}.json.j2"
dest: "{{ node_config_directory }}/{{ item.key }}/config.json"
mode: "0660"
become: true
when:
- inventory_hostname in groups[item.value.group]
- item.value.enabled | bool
with_dict: "{{ venus_services }}"
notify:
- "Restart {{ item.key }} container"
- name: Copying over venus.conf
vars:
service_name: "{{ item.key }}"
merge_configs:
sources:
- "{{ role_path }}/templates/venus.conf.j2"
- "{{ node_custom_config }}/global.conf"
- "{{ node_custom_config }}/venus.conf"
- "{{ node_custom_config }}/venus/{{ item.key }}.conf"
- "{{ node_custom_config }}/venus/{{ inventory_hostname }}/venus.conf"
dest: "{{ node_config_directory }}/{{ item.key }}/venus.conf"
mode: "0660"
become: true
when:
- inventory_hostname in groups[item.value.group]
- item.value.enabled | bool
with_dict: "{{ venus_services }}"
notify:
- "Restart {{ item.key }} container"
- name: Copying over existing policy file
template:
src: "{{ venus_policy_file_path }}"
dest: "{{ node_config_directory }}/{{ item.key }}/{{ venus_policy_file }}"
mode: "0660"
when:
- venus_policy_file is defined
- inventory_hostname in groups[item.value.group]
with_dict: "{{ venus_services }}"
notify:
- Restart {{ item.key }} container

View File

@ -0,0 +1,6 @@
---
- name: "Copy certificates and keys for {{ project_name }}"
import_role:
role: service-cert-copy
vars:
project_services: "{{ venus_services }}"

View File

@ -0,0 +1,2 @@
---
- import_tasks: check-containers.yml

View File

@ -0,0 +1,14 @@
---
- import_tasks: register.yml
- import_tasks: config.yml
- import_tasks: check-containers.yml
- include_tasks: clone.yml
when: venus_dev_mode | bool
- import_tasks: bootstrap.yml
- name: Flush handlers
meta: flush_handlers

View File

@ -0,0 +1,7 @@
---
- name: "Configure haproxy for {{ project_name }}"
import_role:
name: haproxy-config
vars:
project_services: "{{ venus_services }}"
tags: always

View File

@ -0,0 +1,2 @@
---
- include_tasks: "{{ kolla_action }}.yml"

View File

@ -0,0 +1,24 @@
---
- import_role:
name: service-precheck
vars:
service_precheck_services: "{{ venus_services }}"
service_name: "{{ project_name }}"
- name: Get container facts
become: true
kolla_container_facts:
name:
- venus_api
register: container_facts
- name: Checking free port for Venus API
wait_for:
host: "{{ api_interface_address }}"
port: "{{ venus_api_port }}"
connect_timeout: 1
timeout: 1
state: stopped
when:
- container_facts['venus_api'] is not defined
- inventory_hostname in groups['venus-api']

View File

@ -0,0 +1,11 @@
---
- name: Pulling venus images
become: true
kolla_docker:
action: "pull_image"
common_options: "{{ docker_common_options }}"
image: "{{ item.value.image }}"
when:
- inventory_hostname in groups[item.value.group]
- item.value.enabled | bool
with_dict: "{{ venus_services }}"

View File

@ -0,0 +1,2 @@
---
- import_tasks: deploy.yml

View File

@ -0,0 +1,7 @@
---
- import_role:
name: service-ks-register
vars:
service_ks_register_auth: "{{ openstack_venus_auth }}"
service_ks_register_services: "{{ venus_ks_services }}"
service_ks_register_users: "{{ venus_ks_users }}"

View File

@ -0,0 +1,6 @@
---
- import_role:
name: service-stop
vars:
project_services: "{{ venus_services }}"
service_name: "{{ project_name }}"

View File

@ -0,0 +1,7 @@
---
- import_tasks: config.yml
- import_tasks: check-containers.yml
- name: Flush handlers
meta: flush_handlers

View File

@ -0,0 +1,19 @@
{
"command": "venus-api --config-file /etc/venus/venus.conf",
"config_files": [
{
"source": "{{ container_config_directory }}/venus.conf",
"dest": "/etc/venus/venus.conf",
"owner": "venus",
"perm": "0644"
}
],
"permissions": [
{
"path":"/var/log/kolla/venus/venus-api.log",
"owner": "venus:venus",
"recurse": true
}
]
}

View File

@ -0,0 +1,19 @@
{
"command": "venus-manager --config-file /etc/venus/venus.conf task start",
"config_files": [
{
"source": "{{ container_config_directory }}/venus.conf",
"dest": "/etc/venus/venus.conf",
"owner": "venus",
"perm": "0644"
}
],
"permissions": [
{
"path":"/var/log/kolla/venus/venus-manager.log",
"owner": "venus:venus",
"recurse": true
}
]
}

View File

@ -0,0 +1,35 @@
[DEFAULT]
my_ip = {{ api_interface_address }}
periodic_interval = 60
rootwrap_config = /etc/venus/rootwrap.conf
api_paste_config = /etc/venus/api-paste.ini
log_dir = /var/log/kolla/venus/
debug = {{ venus_logging_debug }}
auth_strategy = keystone
os_region_name = {{ openstack_region_name }}
osapi_venus_listen = {{ api_interface_address }}
osapi_venus_listen_port = {{ venus_api_port }}
logging_default_format_string = {{ openstack_logging_default_format_string }}
logging_context_format_string = {{ openstack_logging_context_format_string }}
transport_url = {{ rpc_transport_url }}
[database]
connection = mysql+pymysql://{{ venus_database_user }}:{{ venus_database_password }}@{{ venus_database_address }}/{{ venus_database_name }}?charset=utf8
[keystone_authtoken]
cafile = {{ openstack_cacert }}
project_name = service
password = {{ venus_keystone_password }}
username = {{ venus_keystone_user }}
auth_url = {{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_admin_port }}
project_domain_id = {{ default_project_domain_id }}
user_domain_id = {{ default_user_domain_id }}
auth_type = password
memcached_servers = {% for host in groups['memcached'] %}{{ 'api' | kolla_address(host) | put_address_in_context('memcache') }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}
{% if enable_elasticsearch | bool %}
[elasticsearch]
url = {{ elasticsearch_internal_endpoint }}
{% endif %}

View File

@ -0,0 +1,2 @@
---
project_name: "venus"

View File

@ -74,6 +74,7 @@
- enable_tacker_{{ enable_tacker | bool }}
- enable_telegraf_{{ enable_telegraf | bool }}
- enable_trove_{{ enable_trove | bool }}
- enable_venus_{{ enable_venus | bool }}
- enable_vitrage_{{ enable_vitrage | bool }}
- enable_watcher_{{ enable_watcher | bool }}
- enable_zookeeper_{{ enable_zookeeper | bool }}
@ -326,6 +327,11 @@
tasks_from: loadbalancer
tags: trove
when: enable_trove | bool
- include_role:
name: venus
tasks_from: loadbalancer
tags: venus
when: enable_venus | bool
- include_role:
name: vitrage
tasks_from: loadbalancer
@ -1121,3 +1127,15 @@
- { role: masakari,
tags: masakari,
when: enable_masakari | bool }
- name: Apply role venus
gather_facts: false
hosts:
- venus-api
- venus-manager
- '&enable_venus_True'
serial: '{{ kolla_serial|default("0") }}'
roles:
- { role: venus,
tags: venus,
when: enable_venus | bool }

View File

@ -401,6 +401,7 @@
#enable_telegraf: "no"
#enable_trove: "no"
#enable_trove_singletenant: "no"
#enable_venus: "no"
#enable_vitrage: "no"
#enable_watcher: "no"
#enable_zookeeper: "{{ enable_kafka | bool or enable_storm | bool }}"

View File

@ -168,6 +168,9 @@ zun_keystone_password:
vitrage_database_password:
vitrage_keystone_password:
venus_database_password:
venus_keystone_password:
masakari_database_password:
masakari_keystone_password:

View File

@ -0,0 +1,5 @@
---
features:
- |
Add Venus deployment support.
The project provides a solution to log collection.

View File

@ -810,3 +810,12 @@ ovn-database
[ovn-sb-db:children]
ovn-database
[venus:children]
monitoring
[venus-api:children]
venus
[venus-manager:children]
venus