From d6198cdd32053e9e14ba7d163e31b5cbed2cdb10 Mon Sep 17 00:00:00 2001 From: Satish Patel Date: Wed, 2 Jun 2021 16:14:44 +0000 Subject: [PATCH] Add ovn clustering support This patch will add ovn clustering support, Basically it will use first node to start cluster and then new nodes will use leader node to join cluster. Change-Id: I4b11d3484c99e538ecd6f7d05570486b5f59c782 --- defaults/main.yml | 1 + handlers/main.yml | 11 ++++ tasks/providers/ovn_cluster_setup.yml | 86 +++++++++++++++++++++++++++ tasks/providers/setup_ovs_ovn.yml | 18 +----- templates/ovn-central.j2 | 17 ++++++ 5 files changed, 118 insertions(+), 15 deletions(-) create mode 100644 tasks/providers/ovn_cluster_setup.yml create mode 100644 templates/ovn-central.j2 diff --git a/defaults/main.yml b/defaults/main.yml index ae193e8d..81ab2c3c 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -439,6 +439,7 @@ calico_felix_sha256: ae0bed304702097cee0ad5d9b4abb07b263deeb46ac21f2bcb0118d5bf4 calico_felix_validate_certs: yes # OVN Defaults +neutron_ovn_primary_cluster_node: "{{ groups[neutron_services['neutron-ovn-northd']['group']] | first }}" neutron_ovn_northd_service_name: ovn-northd neutron_ovn_controller_service_name: ovn-controller neutron_ovn_l3_scheduler: leastloaded diff --git a/handlers/main.yml b/handlers/main.yml index 23d0e4c6..2b1d9a28 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -104,3 +104,14 @@ listen: - "Restart neutron services" - "venv changed" + +- name: start ovn-central service + service: + name: ovn-central + state: started + +# (NOTE) Restarting twice to cleanup some pid. +- name: restart ovn-central service + service: + name: ovn-central + state: restarted diff --git a/tasks/providers/ovn_cluster_setup.yml b/tasks/providers/ovn_cluster_setup.yml new file mode 100644 index 00000000..92cb186a --- /dev/null +++ b/tasks/providers/ovn_cluster_setup.yml @@ -0,0 +1,86 @@ +--- +# (c) 2021, Satish Patel +# +# Copyright +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# checking if ovn db are clustered or not, if not then this is fresh node. +- name: Check if ovn db is clustered + command: ovsdb-tool db-is-clustered /var/lib/ovn/ovnnb_db.db + ignore_errors: true + failed_when: false + register: _check_cluster_db + tags: + - skip_ansible_lint + +# We need to clean existing ovn db/lock file before cluster join. +- name: Clean up db/lock files before creating ovn cluster + when: + - _check_cluster_db.rc != 0 + block: + - name: Stop ovn services + service: + name: "{{ item }}" + state: stopped + with_items: + - ovn-northd + - ovn-central + + - name: Clean up ovn db directory + file: + path: /var/lib/ovn/ + state: absent + +# We are finding leader node so new nodes use leader to join cluster. +- name: Find leader node in ovn cluster + shell: ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound | sed 's/ //g' | grep -oP '(?<=Role:).*' + args: + executable: /bin/bash + ignore_errors: true + delegate_to: "{{ item }}" + with_items: "{{ groups['neutron_ovn_northd'] }}" + run_once: true + failed_when: false + register: _find_leader + tags: + - skip_ansible_lint + +# set leader_node variable +- name: Set leader_node fact + set_fact: + leader_node: "{{ (_find_leader.results | selectattr('stdout', 'search', 'leader')) | map(attribute='item') | list }}" + +# This play only run first time to build cluster using primary node. +- name: Setup ovn cluster using primary node. + template: + src: ovn-central.j2 + dest: "/etc/default/ovn-central" + when: + - "inventory_hostname == neutron_ovn_primary_cluster_node" + - _check_cluster_db.rc != 0 + - not leader_node + notify: + - start ovn-central service + - restart ovn-central service + +# This play will add nodes in existing cluster using leader_node var. +- name: Join new nodes to ovn cluster using leader node + template: + src: ovn-central.j2 + dest: "/etc/default/ovn-central" + when: + - _check_cluster_db.rc != 0 + notify: + - start ovn-central service + - restart ovn-central service diff --git a/tasks/providers/setup_ovs_ovn.yml b/tasks/providers/setup_ovs_ovn.yml index 9ad89e0a..3ec10656 100644 --- a/tasks/providers/setup_ovs_ovn.yml +++ b/tasks/providers/setup_ovs_ovn.yml @@ -20,21 +20,6 @@ - (neutron_services['neutron-ovn-northd']['group'] in group_names) or (neutron_services['neutron-ovn-controller']['group'] in group_names) -- name: Create ovsdb-server Listener - command: "ovs-vsctl set-manager ptcp:6640" - when: - - neutron_services['neutron-ovn-northd']['group'] in group_names - -- name: Configure OVN Northbound Listener - command: "ovn-nbctl set-connection ptcp:6641:{{ openstack_service_bind_address }} -- set connection . inactivity_probe=60000" - when: - - neutron_services['neutron-ovn-northd']['group'] in group_names - -- name: Configure OVN Southbound Listener - command: "ovn-sbctl set-connection ptcp:6642:{{ openstack_service_bind_address }} -- set connection . inactivity_probe=60000" - when: - - neutron_services['neutron-ovn-northd']['group'] in group_names - # (NOTE) This makes all computes eligible to be gateway nodes - name: Set CMS Options for Gateway Scheduling command: "ovs-vsctl set open . external-ids:ovn-cms-options=enable-chassis-as-gw" @@ -99,3 +84,6 @@ - neutron_provider_networks.network_mappings is defined - neutron_services['neutron-ovn-controller']['group'] in group_names +- include_tasks: ovn_cluster_setup.yml + when: + - neutron_services['neutron-ovn-northd']['group'] in group_names diff --git a/templates/ovn-central.j2 b/templates/ovn-central.j2 new file mode 100644 index 00000000..ef13b0bc --- /dev/null +++ b/templates/ovn-central.j2 @@ -0,0 +1,17 @@ +# {{ ansible_managed }} + +# OVN cluster parameters +OVN_CTL_OPTS=" \ + --db-nb-create-insecure-remote=yes \ + --db-sb-create-insecure-remote=yes \ + --db-nb-addr={{ ansible_host }} \ + --db-sb-addr={{ ansible_host }} \ + --db-nb-cluster-local-addr={{ ansible_host }} \ + --db-sb-cluster-local-addr={{ ansible_host }} \ + {% if leader_node %} + --db-nb-cluster-remote-addr={% for item in leader_node %}{{ item }} {% endfor %} \ + --db-sb-cluster-remote-addr={% for item in leader_node %}{{ item }} {% endfor %} \ + {% endif %} + --ovn-northd-nb-db=tcp:{{ groups['neutron_ovn_northd'] | map('extract', hostvars, ['ansible_host']) | join(':6641,tcp:') }}:6641 \ + --ovn-northd-sb-db=tcp:{{ groups['neutron_ovn_northd'] | map('extract', hostvars, ['ansible_host']) | join(':6642,tcp:') }}:6642 \ +"