Merge "ADD venus for kolla-ansible"

ansible/group_vars/all.yml

@@ -518,6 +518,8 @@ tacker_server_port: "9890"
 
 trove_api_port: "8779"
 
+venus_api_port: "10010"
+
 watcher_api_port: "9322"
 
 zookeeper_client_port: "2181"
@@ -711,6 +713,7 @@ enable_tacker: "no"
 enable_telegraf: "no"
 enable_trove: "no"
 enable_trove_singletenant: "no"
+enable_venus: "no"
 enable_vitrage: "no"
 enable_watcher: "no"
 enable_zookeeper: "{{ enable_kafka | bool or enable_storm | bool }}"

ansible/inventory/all-in-one

@@ -735,3 +735,12 @@ ovn-database
 
 [ovn-sb-db:children]
 ovn-database
+
+[venus:children]
+monitoring
+
+[venus-api:children]
+venus
+
+[venus-manager:children]
+venus

ansible/inventory/multinode

@@ -753,3 +753,12 @@ ovn-database
 
 [ovn-sb-db:children]
 ovn-database
+
+[venus:children]
+monitoring
+
+[venus-api:children]
+venus
+
+[venus-manager:children]
+venus

ansible/roles/common/tasks/config.yml

@@ -201,6 +201,7 @@
       - { name: "swift", enabled: "{{ enable_swift | bool }}" }
       - { name: "tacker", enabled: "{{ enable_tacker | bool }}" }
       - { name: "trove", enabled: "{{ enable_trove | bool }}" }
+      - { name: "venus", enabled: "{{ enable_venus | bool }}" }
       - { name: "vitrage", enabled: "{{ enable_vitrage | bool }}" }
       - { name: "watcher", enabled: "{{ enable_watcher | bool }}" }
       - { name: "zookeeper", enabled: "{{ enable_zookeeper | bool }}" }

ansible/roles/common/templates/conf/filter/01-rewrite.conf.j2

@@ -166,6 +166,11 @@
     pattern ^(masakari-engine|masakari-api)$
     tag openstack_python
   </rule>
+  <rule>
+    key     programname
+    pattern ^(venus-api|venus-manager)$
+    tag openstack_python
+  </rule>
   <rule>
     key     programname
     pattern .+

ansible/roles/common/templates/cron-logrotate-venus.conf.j2

@@ -0,0 +1,3 @@
+"/var/log/kolla/venus/*.log"
+{
+}

ansible/roles/venus/defaults/main.yml

@@ -0,0 +1,138 @@
+---
+venus_services:
+  venus-api:
+    container_name: venus_api
+    group: venus-api
+    enabled: true
+    image: "{{ venus_api_image_full }}"
+    volumes: "{{ venus_api_default_volumes + venus_api_extra_volumes }}"
+    dimensions: "{{ venus_api_dimensions }}"
+    healthcheck: "{{ venus_api_healthcheck }}"
+    haproxy:
+      venus_api:
+        enabled: "{{ enable_venus }}"
+        mode: "http"
+        external: false
+        port: "{{ venus_api_port }}"
+      venus_api_external:
+        enabled: "{{ enable_venus }}"
+        mode: "http"
+        external: true
+        port: "{{ venus_api_port }}"
+  venus-manager:
+    container_name: venus_manager
+    group: venus-manager
+    enabled: true
+    image: "{{ venus_manager_image_full }}"
+    volumes: "{{ venus_manager_default_volumes + venus_manager_extra_volumes }}"
+    dimensions: "{{ venus_manager_dimensions }}"
+
+####################
+# Database
+####################
+venus_database_name: "venus"
+venus_database_user: "{% if use_preconfigured_databases | bool and use_common_mariadb_user | bool %}{{ database_user }}{% else %}venus{% endif %}"
+venus_database_address: "{{ database_address | put_address_in_context('url') }}:{{ database_port }}"
+
+
+####################
+# Docker
+####################
+venus_install_type: "{{ kolla_install_type }}"
+venus_tag: "{{ openstack_tag }}"
+
+venus_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ venus_install_type }}-venus-api"
+venus_api_tag: "{{ venus_tag }}"
+venus_api_image_full: "{{ venus_api_image }}:{{ venus_api_tag }}"
+
+venus_manager_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ venus_install_type }}-venus-manager"
+venus_manager_tag: "{{ venus_tag }}"
+venus_manager_image_full: "{{ venus_manager_image }}:{{ venus_manager_tag }}"
+
+venus_api_dimensions: "{{ default_container_dimensions }}"
+venus_manager_dimensions: "{{ default_container_dimensions }}"
+
+venus_api_enable_healthchecks: "{{ enable_container_healthchecks }}"
+venus_api_healthcheck_interval: "{{ default_container_healthcheck_interval }}"
+venus_api_healthcheck_retries: "{{ default_container_healthcheck_retries }}"
+venus_api_healthcheck_start_period: "{{ default_container_healthcheck_start_period }}"
+venus_api_healthcheck_test: ["CMD-SHELL", "healthcheck_curl http://{{ api_interface_address |  put_address_in_context('url') }}:{{ venus_api_port }}"]
+venus_api_healthcheck_timeout: "{{ default_container_healthcheck_timeout }}"
+venus_api_healthcheck:
+  interval: "{{ venus_api_healthcheck_interval }}"
+  retries: "{{ venus_api_healthcheck_retries }}"
+  start_period: "{{ venus_api_healthcheck_start_period }}"
+  test: "{% if venus_api_enable_healthchecks | bool %}{{ venus_api_healthcheck_test }}{% else %}NONE{% endif %}"
+  timeout: "{{ venus_api_healthcheck_timeout }}"
+
+venus_api_default_volumes:
+  - "{{ node_config_directory }}/venus-api/:{{ container_config_directory }}/:ro"
+  - "/etc/localtime:/etc/localtime:ro"
+  - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
+  - "kolla_logs:/var/log/kolla/"
+  - "{{ kolla_dev_repos_directory ~ '/venus/venus:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/venus' if venus_dev_mode | bool else '' }}"
+  - "venus:/var/lib/venus/"
+venus_manager_default_volumes:
+  - "{{ node_config_directory }}/venus-manager/:{{ container_config_directory }}/:ro"
+  - "/etc/localtime:/etc/localtime:ro"
+  - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
+  - "kolla_logs:/var/log/kolla/"
+  - "{{ kolla_dev_repos_directory ~ '/venus/venus:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/venus' if venus_dev_mode | bool else '' }}"
+  - "venus:/var/lib/venus/"
+
+venus_extra_volumes: "{{ default_extra_volumes }}"
+venus_api_extra_volumes: "{{ venus_extra_volumes }}"
+venus_manager_extra_volumes: "{{ venus_extra_volumes }}"
+
+####################
+# OpenStack
+####################
+venus_internal_endpoint: "{{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ venus_api_port }}/v1.0/%(tenant_id)s"
+venus_public_endpoint: "{{ public_protocol }}://{{ kolla_external_fqdn | put_address_in_context('url') }}:{{ venus_api_port }}/v1.0/%(tenant_id)s"
+
+venus_logging_debug: "{{ openstack_logging_debug }}"
+
+venus_keystone_user: "venus"
+
+openstack_venus_auth: "{{ openstack_auth }}"
+
+
+####################
+# Kolla
+####################
+venus_git_repository: "https://opendev.org/inspur/venus"
+venus_dev_repos_pull: "{{ kolla_dev_repos_pull }}"
+venus_dev_mode: "{{ kolla_dev_mode }}"
+venus_source_version: "{{ kolla_source_version }}"
+
+####################
+# logging
+####################
+openstack_logging_default_format_string: "%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [- req-None - - - - -] %(instance)s%(message)s"
+openstack_logging_context_format_string: "%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(user_identity)s] %(instance)s%(message)s"
+
+####################
+# Notifications
+####################
+venus_notification_topics:
+  - name: notifications
+    enabled: "{{ enable_ceilometer | bool }}"
+
+venus_enabled_notification_topics: "{{ venus_notification_topics | selectattr('enabled', 'equalto', true) | list }}"
+
+####################
+# Keystone
+####################
+venus_ks_services:
+  - name: "venus"
+    type: "LMS"
+    description: "Log Manager Service"
+    endpoints:
+      - {'interface': 'internal', 'url': '{{ venus_internal_endpoint }}'}
+      - {'interface': 'public', 'url': '{{ venus_public_endpoint }}'}
+
+venus_ks_users:
+  - project: "service"
+    user: "{{ venus_keystone_user }}"
+    password: "{{ venus_keystone_password }}"
+    role: "admin"

ansible/roles/venus/handlers/main.yml

@@ -0,0 +1,32 @@
+---
+- name: Restart venus-api container
+  vars:
+    service_name: "venus-api"
+    service: "{{ venus_services[service_name] }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ service.dimensions }}"
+    healthcheck: "{{ service.healthcheck | default(omit) }}"
+  when:
+    - kolla_action != "config"
+
+- name: Restart venus-manager container
+  vars:
+    service_name: "venus-manager"
+    service: "{{ venus_services[service_name] }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ service.dimensions }}"
+    healthcheck: "{{ service.healthcheck | default(omit) }}"
+  when:
+    - kolla_action != "config"

ansible/roles/venus/tasks/bootstrap.yml

@@ -0,0 +1,34 @@
+---
+- name: Creating venus database
+  become: true
+  kolla_toolbox:
+    module_name: mysql_db
+    module_args:
+      login_host: "{{ database_address }}"
+      login_port: "{{ database_port }}"
+      login_user: "{{ database_user }}"
+      login_password: "{{ database_password }}"
+      name: "{{ venus_database_name }}"
+  run_once: True
+  delegate_to: "{{ groups['venus-api'][0] }}"
+  when:
+    - not use_preconfigured_databases | bool
+
+- name: Creating venus database user and setting permissions
+  become: true
+  kolla_toolbox:
+    module_name: mysql_user
+    module_args:
+      login_host: "{{ database_address }}"
+      login_port: "{{ database_port }}"
+      login_user: "{{ database_user }}"
+      login_password: "{{ database_password }}"
+      name: "{{ venus_database_user }}"
+      password: "{{ venus_database_password }}"
+      host: "%"
+      priv: "{{ venus_database_name }}.*:ALL"
+      append_privs: "yes"
+  run_once: True
+  delegate_to: "{{ groups['venus-api'][0] }}"
+  when:
+    - not use_preconfigured_databases | bool

ansible/roles/venus/tasks/check-containers.yml

@@ -0,0 +1,17 @@
+---
+- name: Check venus containers
+  become: true
+  kolla_docker:
+    action: "compare_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ item.value.container_name }}"
+    image: "{{ item.value.image }}"
+    volumes: "{{ item.value.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ item.value.dimensions }}"
+    healthcheck: "{{ item.value.healthcheck | default(omit) }}"
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ venus_services }}"
+  notify:
+    - "Restart {{ item.key }} container"

ansible/roles/venus/tasks/check.yml

@@ -0,0 +1 @@
+---

ansible/roles/venus/tasks/clone.yml

@@ -0,0 +1,8 @@
+---
+- name: Cloning venus source repository for development
+  become: true
+  git:
+    repo: "{{ venus_git_repository }}"
+    dest: "{{ kolla_dev_repos_directory }}/{{ project_name }}"
+    update: "{{ venus_dev_repos_pull }}"
+    version: "{{ venus_source_version }}"

ansible/roles/venus/tasks/config.yml

@@ -0,0 +1,81 @@
+---
+- name: Ensuring config directories exist
+  file:
+    path: "{{ node_config_directory }}/{{ item.key }}"
+    state: "directory"
+    owner: "{{ config_owner_user }}"
+    group: "{{ config_owner_group }}"
+    mode: "0770"
+  become: true
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ venus_services }}"
+
+- name: Check if policies shall be overwritten
+  stat:
+    path: "{{ item }}"
+  run_once: True
+  delegate_to: localhost
+  register: venus_policy
+  with_first_found:
+    - files: "{{ supported_policy_format_list }}"
+      paths:
+        - "{{ node_custom_config }}/venus/"
+      skip: true
+
+- name: Set venus policy file
+  set_fact:
+    venus_policy_file: "{{ venus_policy.results.0.stat.path | basename }}"
+    venus_policy_file_path: "{{ venus_policy.results.0.stat.path }}"
+  when:
+    - venus_policy.results
+
+- include_tasks: copy-certs.yml
+  when:
+    - kolla_copy_ca_into_containers | bool
+
+- name: Copying over config.json files for services
+  template:
+    src: "{{ item.key }}.json.j2"
+    dest: "{{ node_config_directory }}/{{ item.key }}/config.json"
+    mode: "0660"
+  become: true
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ venus_services }}"
+  notify:
+    - "Restart {{ item.key }} container"
+
+- name: Copying over venus.conf
+  vars:
+    service_name: "{{ item.key }}"
+  merge_configs:
+    sources:
+      - "{{ role_path }}/templates/venus.conf.j2"
+      - "{{ node_custom_config }}/global.conf"
+      - "{{ node_custom_config }}/venus.conf"
+      - "{{ node_custom_config }}/venus/{{ item.key }}.conf"
+      - "{{ node_custom_config }}/venus/{{ inventory_hostname }}/venus.conf"
+    dest: "{{ node_config_directory }}/{{ item.key }}/venus.conf"
+    mode: "0660"
+  become: true
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ venus_services }}"
+  notify:
+    - "Restart {{ item.key }} container"
+
+- name: Copying over existing policy file
+  template:
+    src: "{{ venus_policy_file_path }}"
+    dest: "{{ node_config_directory }}/{{ item.key }}/{{ venus_policy_file }}"
+    mode: "0660"
+  when:
+    - venus_policy_file is defined
+    - inventory_hostname in groups[item.value.group]
+  with_dict: "{{ venus_services }}"
+  notify:
+    - Restart {{ item.key }} container

ansible/roles/venus/tasks/copy-certs.yml

@@ -0,0 +1,6 @@
+---
+- name: "Copy certificates and keys for {{ project_name }}"
+  import_role:
+    role: service-cert-copy
+  vars:
+    project_services: "{{ venus_services }}"

ansible/roles/venus/tasks/deploy-containers.yml

@@ -0,0 +1,2 @@
+---
+- import_tasks: check-containers.yml

ansible/roles/venus/tasks/deploy.yml

@@ -0,0 +1,14 @@
+---
+- import_tasks: register.yml
+
+- import_tasks: config.yml
+
+- import_tasks: check-containers.yml
+
+- include_tasks: clone.yml
+  when: venus_dev_mode | bool
+
+- import_tasks: bootstrap.yml
+
+- name: Flush handlers
+  meta: flush_handlers

ansible/roles/venus/tasks/loadbalancer.yml

@@ -0,0 +1,7 @@
+---
+- name: "Configure haproxy for {{ project_name }}"
+  import_role:
+    name: haproxy-config
+  vars:
+    project_services: "{{ venus_services }}"
+  tags: always

ansible/roles/venus/tasks/main.yml

@@ -0,0 +1,2 @@
+---
+- include_tasks: "{{ kolla_action }}.yml"

ansible/roles/venus/tasks/precheck.yml

@@ -0,0 +1,24 @@
+---
+- import_role:
+    name: service-precheck
+  vars:
+    service_precheck_services: "{{ venus_services }}"
+    service_name: "{{ project_name }}"
+
+- name: Get container facts
+  become: true
+  kolla_container_facts:
+    name:
+      - venus_api
+  register: container_facts
+
+- name: Checking free port for Venus API
+  wait_for:
+    host: "{{ api_interface_address }}"
+    port: "{{ venus_api_port }}"
+    connect_timeout: 1
+    timeout: 1
+    state: stopped
+  when:
+    - container_facts['venus_api'] is not defined
+    - inventory_hostname in groups['venus-api']

ansible/roles/venus/tasks/pull.yml

@@ -0,0 +1,11 @@
+---
+- name: Pulling venus images
+  become: true
+  kolla_docker:
+    action: "pull_image"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ item.value.image }}"
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ venus_services }}"

ansible/roles/venus/tasks/reconfigure.yml

@@ -0,0 +1,2 @@
+---
+- import_tasks: deploy.yml

ansible/roles/venus/tasks/register.yml

@@ -0,0 +1,7 @@
+---
+- import_role:
+    name: service-ks-register
+  vars:
+    service_ks_register_auth: "{{ openstack_venus_auth }}"
+    service_ks_register_services: "{{ venus_ks_services }}"
+    service_ks_register_users: "{{ venus_ks_users }}"

ansible/roles/venus/tasks/stop.yml

@@ -0,0 +1,6 @@
+---
+- import_role:
+    name: service-stop
+  vars:
+    project_services: "{{ venus_services }}"
+    service_name: "{{ project_name }}"

ansible/roles/venus/tasks/upgrade.yml

@@ -0,0 +1,7 @@
+---
+- import_tasks: config.yml
+
+- import_tasks: check-containers.yml
+
+- name: Flush handlers
+  meta: flush_handlers

ansible/roles/venus/templates/venus-api.json.j2

@@ -0,0 +1,19 @@
+{
+    "command": "venus-api --config-file /etc/venus/venus.conf",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/venus.conf",
+            "dest": "/etc/venus/venus.conf",
+            "owner": "venus",
+            "perm": "0644"
+        }
+    ],
+   "permissions": [
+        {
+            "path":"/var/log/kolla/venus/venus-api.log",
+            "owner": "venus:venus",
+            "recurse": true
+        }
+    ]
+}
+

ansible/roles/venus/templates/venus-manager.json.j2

@@ -0,0 +1,19 @@
+{
+    "command": "venus-manager --config-file /etc/venus/venus.conf task start",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/venus.conf",
+            "dest": "/etc/venus/venus.conf",
+            "owner": "venus",
+            "perm": "0644"
+        }
+    ],
+   "permissions": [
+        {
+            "path":"/var/log/kolla/venus/venus-manager.log",
+            "owner": "venus:venus",
+            "recurse": true
+        }
+    ]
+}
+

ansible/roles/venus/templates/venus.conf.j2

@@ -0,0 +1,35 @@
+[DEFAULT]
+my_ip = {{ api_interface_address }}
+periodic_interval = 60
+rootwrap_config = /etc/venus/rootwrap.conf
+api_paste_config = /etc/venus/api-paste.ini
+log_dir = /var/log/kolla/venus/
+debug = {{ venus_logging_debug }}
+auth_strategy = keystone
+os_region_name = {{ openstack_region_name }}
+osapi_venus_listen = {{ api_interface_address }}
+osapi_venus_listen_port = {{ venus_api_port }}
+
+logging_default_format_string = {{ openstack_logging_default_format_string }}
+logging_context_format_string = {{ openstack_logging_context_format_string }}
+
+transport_url = {{ rpc_transport_url }}
+
+[database]
+connection = mysql+pymysql://{{ venus_database_user }}:{{ venus_database_password }}@{{ venus_database_address }}/{{ venus_database_name }}?charset=utf8
+
+[keystone_authtoken]
+cafile = {{ openstack_cacert }}
+project_name = service
+password = {{ venus_keystone_password }}
+username = {{ venus_keystone_user }}
+auth_url = {{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_admin_port }}
+project_domain_id = {{ default_project_domain_id }}
+user_domain_id = {{ default_user_domain_id }}
+auth_type = password
+memcached_servers = {% for host in groups['memcached'] %}{{ 'api' | kolla_address(host) | put_address_in_context('memcache') }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}
+
+{% if enable_elasticsearch | bool %}
+[elasticsearch]
+url = {{ elasticsearch_internal_endpoint }}
+{% endif %}

ansible/roles/venus/vars/main.yml

@@ -0,0 +1,2 @@
+---
+project_name: "venus"

ansible/site.yml

@@ -74,6 +74,7 @@
         - enable_tacker_{{ enable_tacker | bool }}
         - enable_telegraf_{{ enable_telegraf | bool }}
         - enable_trove_{{ enable_trove | bool }}
+        - enable_venus_{{ enable_venus | bool }}
         - enable_vitrage_{{ enable_vitrage | bool }}
         - enable_watcher_{{ enable_watcher | bool }}
         - enable_zookeeper_{{ enable_zookeeper | bool }}
@@ -326,6 +327,11 @@
             tasks_from: loadbalancer
           tags: trove
           when: enable_trove | bool
+        - include_role:
+            name: venus
+            tasks_from: loadbalancer
+          tags: venus
+          when: enable_venus | bool
         - include_role:
             name: vitrage
             tasks_from: loadbalancer
@@ -1121,3 +1127,15 @@
     - { role: masakari,
         tags: masakari,
         when: enable_masakari | bool }
+
+- name: Apply role venus
+  gather_facts: false
+  hosts:
+    - venus-api
+    - venus-manager
+    - '&enable_venus_True'
+  serial: '{{ kolla_serial|default("0") }}'
+  roles:
+    - { role: venus,
+        tags: venus,
+        when: enable_venus | bool }

etc/kolla/globals.yml

@@ -401,6 +401,7 @@
 #enable_telegraf: "no"
 #enable_trove: "no"
 #enable_trove_singletenant: "no"
+#enable_venus: "no"
 #enable_vitrage: "no"
 #enable_watcher: "no"
 #enable_zookeeper: "{{ enable_kafka | bool or enable_storm | bool }}"

etc/kolla/passwords.yml

@@ -168,6 +168,9 @@ zun_keystone_password:
 vitrage_database_password:
 vitrage_keystone_password:
 
+venus_database_password:
+venus_keystone_password:
+
 masakari_database_password:
 masakari_keystone_password:
 

releasenotes/notes/add-venus-80b98b2d3bea348e.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Add Venus deployment support.
+    The project provides a solution to log collection.

tests/templates/inventory.j2

@@ -810,3 +810,12 @@ ovn-database
 
 [ovn-sb-db:children]
 ovn-database
+
+[venus:children]
+monitoring
+
+[venus-api:children]
+venus
+
+[venus-manager:children]
+venus