Add ctlplane networking for routed networks

* Add a new post install software deployment which runs
a python script to configure the undercloud control
plane network. Replaces section in post shell script.

Change-Id: I1cd594564d1628a6e1fccb9eadf18b716ccc5c72

environments/undercloud.yaml

@@ -90,3 +90,10 @@ parameter_defaults:
   #     - ip_netmask: 192.168.26.0/24
   #       next_hop: 192.168.24.1
   ControlPlaneStaticRoutes: []
+  UndercloudCtlplaneSubnets:
+    ctlplane-subnet:
+      NetworkCidr: '192.168.24.0/24'
+      NetworkGateway: '192.168.24.1'
+      DhcpRangeStart: '192.168.24.5'
+      DhcpRangeEnd: '192.168.24.24'
+  UndercloudCtlplaneLocalSubnet: 'ctlplane-subnet'

extraconfig/post_deploy/undercloud_ctlplane_network.py

@@ -0,0 +1,273 @@
+#!/usr/bin/python
+
+import json
+import netaddr
+import os
+import os_client_config
+import subprocess
+
+CTLPLANE_NETWORK_NAME = 'ctlplane'
+
+AUTH_URL = os.environ['auth_url']
+ADMIN_PASSWORD = os.environ['admin_password']
+CONF = json.loads(os.environ['config'])
+
+
+def _run_command(args, env=None, name=None):
+    """Run the command defined by args and return its output
+
+    :param args: List of arguments for the command to be run.
+    :param env: Dict defining the environment variables. Pass None to use
+        the current environment.
+    :param name: User-friendly name for the command being run. A value of
+        None will cause args[0] to be used.
+    """
+    if name is None:
+        name = args[0]
+
+    if env is None:
+        env = os.environ
+    env = env.copy()
+
+    # When running a localized python script, we need to tell it that we're
+    # using utf-8 for stdout, otherwise it can't tell because of the pipe.
+    env['PYTHONIOENCODING'] = 'utf8'
+
+    try:
+        return subprocess.check_output(args,
+                                       stderr=subprocess.STDOUT,
+                                       env=env).decode('utf-8')
+    except subprocess.CalledProcessError as ex:
+        print('ERROR: %s failed: %s' % (name, ex.output))
+        raise
+
+
+def _ensure_neutron_network(sdk):
+    try:
+        network = list(sdk.network.networks(name=CTLPLANE_NETWORK_NAME))
+        if not network:
+            network = sdk.network.create_network(
+                name=CTLPLANE_NETWORK_NAME,
+                provider_network_type='flat',
+                provider_physical_network=CONF['physical_network'],
+                mtu=CONF['mtu'])
+            print('INFO: Network created %s' % network)
+            # (hjensas) Delete the default segment, we create a new segment
+            # per subnet later.
+            segments = list(sdk.network.segments(network_id=network.id))
+            sdk.network.delete_segment(segments[0].id)
+            print('INFO: Default segment on network %s deleted.' %
+                  network.name)
+        else:
+            network = sdk.network.update_network(
+                network[0].id,
+                name=CTLPLANE_NETWORK_NAME,
+                mtu=CONF['mtu'])
+            print('INFO: Network updated %s' % network)
+    except Exception:
+        print('ERROR: Network create/update failed.')
+        raise
+
+    return network
+
+
+def _neutron_subnet_create(sdk, network_id, cidr, gateway, host_routes,
+                           allocation_pool, name, segment_id, dns_nameservers):
+    try:
+        if netaddr.IPNetwork(cidr).version == 6:
+            subnet = sdk.network.create_subnet(
+                name=name,
+                cidr=cidr,
+                gateway_ip=gateway,
+                enable_dhcp=True,
+                ip_version='6',
+                ipv6_address_mode='dhcpv6-stateless',
+                ipv6_ra_mode='dhcpv6-stateless',
+                allocation_pools=allocation_pool,
+                network_id=network_id,
+                segment_id=segment_id,
+                dns_nameservers=dns_nameservers)
+        else:
+            subnet = sdk.network.create_subnet(
+                name=name,
+                cidr=cidr,
+                gateway_ip=gateway,
+                host_routes=host_routes,
+                enable_dhcp=True,
+                ip_version='4',
+                allocation_pools=allocation_pool,
+                network_id=network_id,
+                segment_id=segment_id,
+                dns_nameservers=dns_nameservers)
+            print('INFO: Subnet created %s' % subnet)
+    except Exception:
+        print('ERROR: Create subnet %s failed.' % name)
+        raise
+
+    return subnet
+
+
+def _neutron_subnet_update(sdk, subnet_id, cidr, gateway, host_routes,
+                           allocation_pool, name, dns_nameservers):
+    try:
+        if netaddr.IPNetwork(cidr).version == 6:
+            subnet = sdk.network.update_subnet(
+                subnet_id,
+                name=name,
+                gateway_ip=gateway,
+                allocation_pools=allocation_pool,
+                dns_nameservers=dns_nameservers)
+        else:
+            subnet = sdk.network.update_subnet(
+                subnet_id,
+                name=name,
+                gateway_ip=gateway,
+                host_routes=host_routes,
+                allocation_pools=allocation_pool,
+                dns_nameservers=dns_nameservers)
+        print('INFO: Subnet updated %s' % subnet)
+    except Exception:
+        print('ERROR: Update of subnet %s failed.' % name)
+        raise
+
+
+def _neutron_segment_create(sdk, name, network_id, phynet):
+    try:
+        segment = sdk.network.create_segment(
+            name=name,
+            network_id=network_id,
+            physical_network=phynet,
+            network_type='flat')
+        print('INFO: Neutron Segment created %s' % segment)
+    except Exception as ex:
+        print('ERROR: Neutron Segment %s create failed.' % name)
+        raise
+
+    return segment
+
+
+def _neutron_segment_update(sdk, segment_id, name):
+    try:
+        segment = sdk.network.update_segment(segment_id, name=name)
+        print('INFO: Neutron Segment updated %s', segment)
+    except Exception:
+        print('ERROR: Neutron Segment %s update failed.' % name)
+        raise
+
+
+def _ensure_neutron_router(sdk, name, subnet_id):
+    try:
+        router = sdk.network.create_router(name=name, admin_state_up='true')
+        sdk.network.add_interface_to_router(router.id, subnet_id=subnet_id)
+    except Exception:
+        print('ERROR: Create router for subnet %s failed.' % name)
+        raise
+
+
+def _get_subnet(sdk, cidr, network_id):
+    try:
+        subnet = list(sdk.network.subnets(cidr=cidr, network_id=network_id))
+    except Exception as ex:
+        print('ERROR: Get subnet with cidr %s failed.' % cidr)
+        raise
+
+    return False if not subnet else subnet[0]
+
+
+def _get_segment(sdk, phy, network_id):
+    try:
+        segment = list(sdk.network.segments(physical_network=phy,
+                                            network_id=network_id))
+    except Exception:
+        print('ERROR: Get segment for physical_network %s on network_id %s '
+              'failed.' % (phy, network_id))
+        raise
+
+    return False if not segment else segment[0]
+
+
+def config_neutron_segments_and_subnets(sdk, ctlplane_id):
+    s = CONF['subnets'][CONF['local_subnet']]
+    subnet = _get_subnet(sdk, s['NetworkCidr'], ctlplane_id)
+    if subnet and not subnet.segment_id:
+        print('WARNING: Local subnet %s already exists and is not associated '
+              'with a network segment. Any additional subnets will be '
+              'ignored.' % CONF['local_subnet'])
+        host_routes = [{'destination': '169.254.169.254/32',
+                        'nexthop': CONF['local_ip']}]
+        allocation_pool = [{'start': s['DhcpRangeStart'],
+                            'end': s['DhcpRangeEnd']}]
+        _neutron_subnet_update(
+            sdk, subnet.id, s['NetworkCidr'], s['NetworkGateway'], host_routes,
+            allocation_pool, CONF['local_subnet'], CONF['nameservers'])
+        # If the subnet is IPv6 we need to start a router so that router
+        # advertisments are sent out for stateless IP addressing to work.
+        if netaddr.IPNetwork(s['NetworkCidr']).version == 6:
+            _ensure_neutron_router(sdk, CONF['local_subnet'], subnet.id)
+    else:
+        for name in CONF['subnets']:
+            s = CONF['subnets'][name]
+
+            phynet = name
+            metadata_nexthop = s['NetworkGateway']
+            if name == CONF['local_subnet']:
+                phynet = CONF['physical_network']
+                metadata_nexthop = CONF['local_ip']
+
+            host_routes = [{'destination': '169.254.169.254/32',
+                            'nexthop': metadata_nexthop}]
+            allocation_pool = [{'start': s['DhcpRangeStart'],
+                                'end': s['DhcpRangeEnd']}]
+
+            subnet = _get_subnet(sdk, s['NetworkCidr'], ctlplane_id)
+            segment = _get_segment(sdk, phynet, ctlplane_id)
+
+            if name == CONF['local_subnet']:
+                if ((subnet and not segment) or
+                        (subnet and segment and
+                         subnet.segment_id != segment.id)):
+                    raise RuntimeError(
+                        'The cidr: %s of the local subnet is already used in '
+                        'subnet: %s which is associated with segment_id: %s.' %
+                        (s['NetworkCidr'], subnet.id, subnet.segment_id))
+
+            if subnet:
+                _neutron_segment_update(sdk, subnet.segment_id, name)
+                _neutron_subnet_update(
+                    sdk, subnet.id, s['NetworkCidr'], s['NetworkGateway'],
+                    host_routes, allocation_pool, name, CONF['nameservers'])
+            else:
+                if segment:
+                    _neutron_segment_update(sdk, segment.id, name)
+                else:
+                    segment = _neutron_segment_create(sdk, name,
+                                                      ctlplane_id, phynet)
+
+                if CONF['enable_routed_networks']:
+                    subnet = _neutron_subnet_create(
+                        sdk, ctlplane_id, s['NetworkCidr'],
+                        s['NetworkGateway'], host_routes, allocation_pool,
+                        name, segment.id, CONF['nameservers'])
+                else:
+                    subnet = _neutron_subnet_create(
+                        sdk, ctlplane_id, s['NetworkCidr'],
+                        s['NetworkGateway'], host_routes, allocation_pool,
+                        name, None, CONF['nameservers'])
+
+            # If the subnet is IPv6 we need to start a router so that router
+            # advertisments are sent out for stateless IP addressing to work.
+            if netaddr.IPNetwork(s['NetworkCidr']).version == 6:
+                _ensure_neutron_router(sdk, name, subnet.id)
+
+
+
+if _run_command(['hiera', 'neutron_api_enabled'], name='hiera'):
+    sdk = os_client_config.make_sdk(auth_url=AUTH_URL,
+                                    project_name='admin',
+                                    username='admin',
+                                    password=ADMIN_PASSWORD,
+                                    project_domain_name='Default',
+                                    user_domain_name='Default')
+
+    network = _ensure_neutron_network(sdk)
+    config_neutron_segments_and_subnets(sdk, network.id)

extraconfig/post_deploy/undercloud_post.sh

@@ -64,57 +64,6 @@ if ! grep "$(cat $HOMEDIR/.ssh/id_rsa.pub)" $HOMEDIR/.ssh/authorized_keys; then
 fi
 chown -R "$USERNAME:$GROUPNAME" "$HOMEDIR/.ssh"
 
-if [ "$(hiera neutron_api_enabled)" = "true" ]; then
-    PHYSICAL_NETWORK=ctlplane
-
-    ctlplane_id=$(openstack network list -f csv -c ID -c Name --quote none | tail -n +2 | grep ctlplane | cut -d, -f1)
-    subnet_ids=$(openstack subnet list -f csv -c ID --quote none | tail -n +2)
-    subnet_id=
-
-    for subnet_id in $subnet_ids; do
-        network_id=$(openstack subnet show -f value -c network_id $subnet_id)
-        if [ "$network_id" = "$ctlplane_id" ]; then
-            break
-        fi
-    done
-
-    net_create=1
-    if [ -n "$subnet_id" ]; then
-        cidr=$(openstack subnet show $subnet_id -f value -c cidr)
-        if [ "$cidr" = "$undercloud_network_cidr" ]; then
-            net_create=0
-        else
-            echo "New cidr $undercloud_network_cidr does not equal old cidr $cidr"
-            echo "Will attempt to delete and recreate subnet $subnet_id"
-        fi
-    fi
-
-    if [ "$net_create" -eq "1" ]; then
-        # Delete the subnet and network to make sure it doesn't already exist
-        if openstack subnet list | grep start; then
-            openstack subnet delete $(openstack subnet list | grep start | awk '{print $4}')
-        fi
-        if openstack network show ctlplane; then
-            openstack network delete ctlplane
-        fi
-
-
-        NETWORK_ID=$(openstack network create --provider-network-type=flat --provider-physical-network=ctlplane ctlplane | grep " id " | awk '{print $4}')
-
-        NAMESERVER_ARG=""
-        if [ -n "${undercloud_nameserver:-}" ]; then
-            NAMESERVER_ARG="--dns-nameserver $undercloud_nameserver"
-        fi
-
-        openstack subnet create --network=$NETWORK_ID \
-            --gateway=$undercloud_network_gateway \
-            --subnet-range=$undercloud_network_cidr \
-            --allocation-pool start=$undercloud_dhcp_start,end=$undercloud_dhcp_end \
-            --host-route destination=169.254.169.254/32,gateway=$local_ip \
-            $NAMESERVER_ARG ctlplane-subnet
-    fi
-fi
-
 if [ "$(hiera nova_api_enabled)" = "true" ]; then
     # Disable nova quotas
     openstack quota set --cores -1 --instances -1 --ram -1 $(openstack project show admin | awk '$2=="id" {print $4}')

extraconfig/post_deploy/undercloud_post.yaml

@@ -13,21 +13,6 @@ parameters:
     description: The HOME directory where the stackrc and ssh credentials for the Undercloud will be installed. Set to /home/<user> to customize the location.
     type: string
     default: '/root'
-  UndercloudDhcpRangeStart:
-    type: string
-    default: '192.168.24.5'
-  UndercloudDhcpRangeEnd:
-    type: string
-    default: '192.168.24.24'
-  UndercloudNetworkCidr:
-    type: string
-    default: '192.168.24.0/24'
-  UndercloudNetworkGateway:
-    type: string
-    default: '192.168.24.1'
-  UndercloudNameserver:
-    type: string
-    default: ''
   AdminPassword: #supplied by tripleo-undercloud-passwords.yaml
     type: string
     description: The password for the keystone admin account, used for monitoring, querying neutron etc.
@@ -52,6 +37,33 @@ parameters:
     description: >
       Whether the TripleO validations are enabled.
     type: boolean
+  DnsServers: # Override this via parameter_defaults
+    default: []
+    description: A list of DNS servers (2 max for some implementations) that will be added to resolv.conf.
+    type: comma_delimited_list
+  CtlplaneLocalPhysicalNetwork:
+    default: ctlplane
+    type: string
+    description: Physical network name for the ctlplane network local to the undercloud
+  UndercloudCtlplaneSubnets:
+    description: >
+      Dictionary of subnets to configure on the Undercloud ctlplan network
+    default: {}
+    type: json
+  UndercloudCtlplaneLocalSubnet:
+    description: The subnet local to the undercloud on the ctlplane network
+    default: ctlplane-subnet
+    type: string
+  UndercloudEnableRoutedNetworks:
+    description: Enable support for routed ctlplane networks.
+    default: False
+    type: boolean
+  UndercloudLocalMtu: # Override this via parameter_defaults
+    default: 1500
+    description: MTU to use for the Undercloud local_interface.
+    type: number
+    constraints:
+      - range: { min: 1000, max: 65536 }
 
 conditions:
 
@@ -74,12 +86,6 @@ resources:
       group: script
       inputs:
         - name: deploy_identifier
-        - name: local_ip
-        - name: undercloud_dhcp_start
-        - name: undercloud_dhcp_end
-        - name: undercloud_network_cidr
-        - name: undercloud_network_gateway
-        - name: undercloud_nameserver
         - name: admin_password
         - name: auth_url
         - name: snmp_readonly_user_password
@@ -93,12 +99,6 @@ resources:
       servers: {get_param: servers}
       config: {get_resource: UndercloudPostConfig}
       input_values:
-        local_ip: {get_param: [DeployedServerPortMap, 'control_virtual_ip', fixed_ips, 0, ip_address]}
-        undercloud_dhcp_start: {get_param: UndercloudDhcpRangeStart}
-        undercloud_dhcp_end: {get_param: UndercloudDhcpRangeEnd}
-        undercloud_network_cidr: {get_param: UndercloudNetworkCidr}
-        undercloud_network_gateway: {get_param: UndercloudNetworkGateway}
-        undercloud_nameserver: {get_param: UndercloudNameserver}
         ssl_certificate: {get_param: SSLCertificate}
         homedir: {get_param: UndercloudHomeDir}
         admin_password: {get_param: AdminPassword}
@@ -118,3 +118,48 @@ resources:
               host: {get_param: [DeployedServerPortMap, 'control_virtual_ip', fixed_ips, 0, ip_address]}
               port: 5000
               path: /
+
+  UndercloudCtlplaneNetworkConfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      inputs:
+        - name: admin_password
+        - name: auth_url
+        - name: config
+      config: {get_file: ./undercloud_ctlplane_network.py}
+
+  UndercloudCtlplaneNetworkDeployment:
+    type: OS::Heat::SoftwareDeployments
+    properties:
+      name: UndercloudCtlplaneNetworkDeployment
+      servers: {get_param: servers}
+      config: {get_resource: UndercloudCtlplaneNetworkConfig}
+      input_values:
+        admin_password: {get_param: AdminPassword}
+        # if SSL is enabled we use the public virtual ip as the stackrc endpoint
+        auth_url:
+          if:
+          - tls_enabled
+          - make_url:
+              scheme: https
+              host: {get_param: [DeployedServerPortMap, 'public_virtual_ip', fixed_ips, 0, ip_address]}
+              port: 13000
+              path: /
+          - make_url:
+              scheme: http
+              host: {get_param: [DeployedServerPortMap, 'control_virtual_ip', fixed_ips, 0, ip_address]}
+              port: 5000
+              path: /
+        config:
+          str_replace:
+            template: JSON
+            params:
+              JSON:
+                local_ip: {get_param: [DeployedServerPortMap, 'control_virtual_ip', fixed_ips, 0, ip_address]}
+                local_subnet: {get_param: UndercloudCtlplaneLocalSubnet}
+                nameservers: {get_param: DnsServers}
+                physical_network: {get_param: CtlplaneLocalPhysicalNetwork}
+                subnets: {get_param: UndercloudCtlplaneSubnets}
+                enable_routed_networks: {get_param: UndercloudEnableRoutedNetworks}
+                mtu: {get_param: UndercloudLocalMtu}

net-config-undercloud.j2.yaml

@@ -29,6 +29,8 @@ parameters:
     default: 1500
     description: MTU to use for the Undercloud local_interface.
     type: number
+    constraints:
+      - range: { min: 1000, max: 65536 }
 resources:
   OsNetConfigImpl:
     type: OS::Heat::SoftwareConfig