Fix Clustering after a rebase to Ussuri + OVN

The previous work included incorrect handling of
configuration for the multi-node case in terms of
OVN configuration.

This change addresses that in addition to other
minor fixes related to the clustering setup.

Change-Id: Ibf04af95271d1746f59192d11831d6129ba5b8d0

snap-overlay/bin/set-default-config

@@ -11,6 +11,10 @@ snapctl set \
         config.post-setup=true \
         ;
 
+snapctl set \
+        config.keystone.region-name=microstack \
+        ;
+
 # Networking related settings.
 snapctl set \
         config.network.dns-servers=1.1.1.1 \
@@ -25,6 +29,8 @@ snapctl set \
         config.network.ports.dashboard=80 \
         config.network.ports.mysql=3306 \
         config.network.ports.rabbit=5672 \
+	config.network.external-bridge-name=br-ex \
+        config.network.physnet-name=physnet1 \
         ;
 
 # Passwords, certs, etc.

snap-overlay/bin/setup-br-ex

@@ -11,19 +11,15 @@ set -ex
 
 extcidr=$(snapctl get config.network.ext-cidr)
 controlip=$(snapctl get config.network.control-ip)
+external_bridge_name=$(snapctl get config.network.external-bridge-name)
+physnet_name=$(snapctl get config.network.physnet-name)
 
 # NOTE(dmitriis): this needs to be reworked to allow for OVN + direct exit of traffic to
 # the provider network from a compute node.
 
 # Create an external bridge in the system datapath.
-ovs-vsctl --retry --may-exist add-br br-ex -- set bridge br-ex datapath_type=system protocols=OpenFlow13,OpenFlow15
-ovs-vsctl set open . external-ids:ovn-bridge-mappings=physnet1:br-ex
-ovs-vsctl set open . external-ids:ovn-cms-options="enable-chassis-as-gw"
-
-# Configure the settings used by self-configuration of ovn-controller.
-ovs-vsctl set open . external-ids:ovn-encap-type=geneve -- set open . external-ids:ovn-encap-ip=$controlip
-# Leave SB database connection details for ovn-controller to pick up.
-ovs-vsctl set open . external-ids:ovn-remote='unix:/var/snap/microstack/common/run/ovn/ovnsb_db.sock'
+ovs-vsctl --retry --may-exist add-br $external_bridge_name -- set bridge $external_bridge_name datapath_type=system protocols=OpenFlow13,OpenFlow15
+ovs-vsctl set open . external-ids:ovn-bridge-mappings=$physnet_name:$external_bridge_name
 
 # NOTE: system-id is a randomly-generated UUID (see the --system-id=random option for ovs-ctl)
 # As it is generated automatically, we do not set it here.

snap-overlay/snap-openstack.yaml

@@ -82,6 +82,7 @@ setup:
     "{snap_common}/etc/microstack.rc": 0644
     "{snap_common}/etc/microstack.json": 0644
   snap-config-keys:
+    region_name: 'config.keystone.region-name'
     ospassword: 'config.credentials.os-password'
     nova_password: 'config.credentials.nova-password'
     cinder_password: 'config.credentials.cinder-password'
@@ -213,16 +214,6 @@ entry_points:
       - "{snap_common}/etc/neutron/plugins/ml2/ml2_conf.ini"
     config-dirs:
       - "{snap_common}/etc/neutron/neutron.conf.d"
-  neutron-openvswitch-agent:
-    binary: "{snap}/bin/neutron-openvswitch-agent"
-    config-files:
-      - "{snap}/etc/neutron/neutron.conf"
-      - "{snap}/etc/neutron/plugins/ml2/openvswitch_agent.ini"
-    config-files-override:
-      - "{snap_common}/etc/neutron/neutron.conf"
-      - "{snap_common}/etc/neutron/plugins/ml2/openvswitch_agent.ini"
-    config-dirs:
-      - "{snap_common}/etc/neutron/neutron.conf.d"
   neutron-ovs-cleanup:
     binary: "{snap}/bin/neutron-ovs-cleanup"
     config-files:

snap-overlay/templates/cinder.keystone.conf.j2

@@ -4,7 +4,7 @@ auth_strategy = keystone
 [keystone_authtoken]
 auth_uri = http://{{ control_ip }}:5000
 auth_url = http://{{ control_ip }}:5000
-memcached_servers = {{ control_ip }}:11211
+memcached_servers = {{ compute_ip }}:11211
 auth_type = password
 project_domain_name = default
 user_domain_name = default

snap-overlay/templates/glance.conf.d.keystone.conf.j2

@@ -1,7 +1,7 @@
 [keystone_authtoken]
 auth_uri = http://{{ control_ip }}:5000
 auth_url = http://{{ control_ip }}:5000
-memcached_servers = {{ control_ip }}:11211
+memcached_servers = {{ compute_ip }}:11211
 auth_type = password
 project_domain_name = default
 user_domain_name = default

snap-overlay/templates/neutron.keystone.conf.j2

@@ -4,7 +4,7 @@ auth_strategy = keystone
 [keystone_authtoken]
 auth_uri = http://{{ control_ip }}:5000
 auth_url = http://{{ control_ip }}:5000
-memcached_servers = {{ control_ip }}:11211
+memcached_servers = {{ compute_ip }}:11211
 auth_type = password
 project_domain_name = default
 user_domain_name = default

snap-overlay/templates/neutron.nova.conf.j2

@@ -7,7 +7,7 @@ auth_url = http://{{ control_ip }}:5000
 auth_type = password
 project_domain_name = default
 user_domain_name = default
-region_name = microstack
+region_name = {{ region_name }}
 project_name = service
 username = nova
 password = {{ nova_password }}

snap-overlay/templates/nova-snap.conf.j2

@@ -20,4 +20,4 @@ lock_path = {{ snap_common }}/lock
 ovsdb_connection = unix:{{ snap_common }}/run/openvswitch/db.sock
 
 [cinder]
-os_region_name = microstack
+os_region_name = {{ region_name }}

snap-overlay/templates/nova.conf.d.keystone.conf.j2

@@ -1,7 +1,7 @@
 [keystone_authtoken]
 auth_uri = http://{{ control_ip }}:5000
 auth_url = http://{{ control_ip }}:5000
-memcached_servers = {{ control_ip }}:11211
+memcached_servers = {{ compute_ip }}:11211
 auth_type = password
 project_domain_name = default
 user_domain_name = default

snap-overlay/templates/nova.conf.d.neutron.conf.j2

@@ -1,11 +1,11 @@
 [neutron]
 url = http://{{ control_ip }}:9696
 auth_url = http://{{ control_ip }}:5000
-memcached_servers = {{ control_ip }}:11211
+memcached_servers = {{ compute_ip }}:11211
 auth_type = password
 project_domain_name = default
 user_domain_name = default
-region_name = microstack
+region_name = {{ region_name }}
 project_name = service
 username = neutron
 password = {{ neutron_password }}

snap-overlay/templates/nova.conf.d.placement.conf.j2

@@ -1,11 +1,11 @@
 [placement]
 auth_uri = http://{{ control_ip }}:5000
 auth_url = http://{{ control_ip }}:5000
-memcached_servers = {{ control_ip }}:11211
+memcached_servers = {{ compute_ip }}:11211
 auth_type = password
 project_domain_name = default
 user_domain_name = default
 project_name = service
 username = nova
 password = {{ nova_password }}
-os_region_name = RegionOne
+region_name = {{ region_name }}

snap-overlay/templates/placement.conf.d.keystone.conf.j2

@@ -1,7 +1,7 @@
 [keystone_authtoken]
 auth_uri = http://{{ control_ip }}:5000
 auth_url = http://{{ control_ip }}:5000
-memcached_servers = {{ control_ip }}:11211
+memcached_servers = {{ compute_ip }}:11211
 auth_type = password
 project_domain_name = default
 user_domain_name = default

snap/hooks/install

@@ -15,6 +15,7 @@ set-default-config
 #    an operator (connecting openvswitch-support loads the openvswitch kernel module
 #    but auto-connection is not enabled for openvswitch-support).
 snapctl stop --disable $SNAP_INSTANCE_NAME.ovsdb-server
+snapctl stop --disable $SNAP_INSTANCE_NAME.neutron-ovn-metadata-agent
 snapctl stop --disable $SNAP_INSTANCE_NAME.ovn-ovsdb-server-sb
 snapctl stop --disable $SNAP_INSTANCE_NAME.ovn-ovsdb-server-nb
 snapctl stop --disable $SNAP_INSTANCE_NAME.ovs-vswitchd

tools/cluster/cluster/client.py

@@ -4,7 +4,7 @@ import json
 
 import requests
 
-from cluster.shell import check, check_output, write_tunnel_config
+from cluster.shell import check, check_output
 
 
 def join():
@@ -30,10 +30,6 @@ def join():
     # TODO: add better error handling to the below
     os_password = resp['config']['credentials']['os-password']
 
-    # Write out tunnel config and restart neutron openvswitch agent.
-    write_tunnel_config(my_ip)
-    check('snapctl', 'restart', 'microstack.neutron-openvswitch-agent')
-
     # Set passwords and such
     check('snapctl', 'set', 'config.credentials.os-password={}'.format(
         os_password))

tools/cluster/cluster/daemon.py

@@ -2,7 +2,7 @@ import json
 
 from flask import Flask, request
 
-from cluster.shell import check, check_output, write_tunnel_config
+from cluster.shell import check_output
 
 
 app = Flask(__name__)
@@ -22,10 +22,6 @@ def join_info(password, ip_address):
     # TODO: be selective about what we return. For now, we just get everything.
     config = json.loads(check_output('snapctl', 'get', 'config'))
 
-    # Write out tunnel config and restart neutron openvswitch agent.
-    write_tunnel_config(config['network']['control-ip'])
-    check('snapctl', 'restart', 'microstack.neutron-openvswitch-agent')
-
     info = {'config': config}
     return info
 

tools/cluster/cluster/shell.py

@@ -36,15 +36,3 @@ def check(*args):
 
     """
     return subprocess.check_call(args, env=os.environ)
-
-
-def write_tunnel_config(local_ip):
-    """Write tunnel config file for neutron agent."""
-
-    path_ = '{SNAP_COMMON}/etc/neutron/neutron.conf.d/tunnel.conf'.format(
-        **os.environ)
-    with open(path_, 'w') as file_:
-        file_.write("""\
-[OVS]
-local_ip = {local_ip}
-""".format(local_ip=local_ip))

tools/init/init/questions/__init__.py

@@ -189,18 +189,30 @@ class NetworkSettings(Question):
         network.ExtGateway().ask()
         network.ExtCidr().ask()
 
+        control_ip = check_output('snapctl', 'get',
+                                  'config.network.control-ip')
         if role == 'control':
             nb_conn = 'unix:{SNAP_COMMON}/run/ovn/ovnnb_db.sock'.format(**_env)
             sb_conn = 'unix:{SNAP_COMMON}/run/ovn/ovnsb_db.sock'.format(**_env)
+            check_output('ovs-vsctl', 'set', 'open', '.',
+                         f'external-ids:ovn-encap-ip={control_ip}')
         elif role == 'compute':
-            control_ip = check_output('snapctl', 'get',
-                                      'config.network.control-ip')
             sb_conn = f'tcp:{control_ip}:6642'
             # Not used by any compute node services.
             nb_conn = ''
+            compute_ip = check_output('snapctl', 'get',
+                                      'config.network.compute-ip')
+            # Set the IP address to be used for a tunnel endpoint.
+            check_output('ovs-vsctl', 'set', 'open', '.',
+                         f'external-ids:ovn-encap-ip={compute_ip}')
         else:
             raise Exception(f'Unexpected node role: {role}')
 
+        # ovn-controller does not start unless both the ovn-encap-ip and the
+        # ovn-encap-type are set.
+        check_output('ovs-vsctl', 'set', 'open', '.',
+                     'external-ids:ovn-encap-type=geneve')
+
         # Configure OVN SB and NB sockets based on the role node. For
         # single-node deployments there is no need to use a TCP socket.
         check_output('snapctl', 'set',
@@ -208,6 +220,16 @@ class NetworkSettings(Question):
         check_output('snapctl', 'set',
                      f'config.network.ovn-sb-connection={sb_conn}')
 
+        # Set SB database connection details for ovn-controller to pick up.
+        check_output(
+                'ovs-vsctl', 'set', 'open', '.',
+                f'external-ids:ovn-remote={sb_conn}'
+        )
+        check_output(
+                'ovs-vsctl', 'set', 'open', '.',
+                'external-ids:ovn-cms-options=enable-chassis-as-gw'
+        )
+
         # Now that we have default or overriden values, setup the
         # bridge and write all the proper values into our config
         # files.
@@ -215,6 +237,7 @@ class NetworkSettings(Question):
         check('snap-openstack', 'setup')
 
         if role == 'control':
+
             enable('ovn-northd')
             enable('ovn-controller')