Override network_scheme to skip interfaces used by DPDK vrouter

* Depends on new l23network features
  * Overrides br-mesh to vhost0
  * vhost0 pre-up line has to be moved from interfaces.d file
  * Gateway is set if multirack configuration is used
  * Supervisor config is generated in the same way as in virouter init
    script. This prevents unnecessary restarts during task execution
  * Private interface mac is generated by facter instead of a function
  * openstack-network-common-config is skipped for the dpdk role

Change-Id: Ia2eeb5641729261ec4e58f33fe7db69526ffa5a0
Signed-off-by: Illia Polliul <ipolliul@mirantis.com>

deployment_scripts/puppet/manifests/compute-hiera.pp

@@ -20,3 +20,12 @@ file { '/etc/hiera/plugins/contrail.yaml':
   ensure  => file,
   content => 'use_ovs: false',
 }
+
+if roles_include('dpdk') {
+  file_line {'contrail-vrouter-override_ns':
+    path  => '/etc/hiera.yaml',
+    line  => '    - plugins/contrail-vrouter-override_ns',
+    after => '  !ruby/sym hierarchy:',
+  }
+}
+

deployment_scripts/puppet/manifests/contrail-analytics.pp

@@ -16,3 +16,4 @@ notice('MODULAR: contrail/contrail-analytics.pp')
 
 include contrail
 class { 'contrail::analytics': }
+

deployment_scripts/puppet/manifests/contrail-compute-netconfig-override.pp

@@ -0,0 +1,18 @@
+#    Copyright 2016 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+notice('MODULAR: contrail/contrail-compute-netconfig-override.pp')
+
+include contrail
+class { 'contrail::compute::compute_netconfig_override': }

deployment_scripts/puppet/modules/contrail/files/pre-up-contrail-vrouter

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+[ "$IFACE" != "vhost0" ] && exit 0
+. /opt/contrail/bin/if-vhost0

deployment_scripts/puppet/modules/contrail/lib/facter/mac_from_vrouter.rb

@@ -0,0 +1,43 @@
+#    Copyright 2016 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+require 'hiera'
+
+Facter.add("mac_from_vrouter") do
+  setcode do
+    output=`vif --list`
+    hiera = Hiera.new(:config => '/etc/hiera.yaml')
+    network_scheme = hiera.lookup('network_scheme', {}, {}, nil, :hash)
+
+    phys_dev = vlan = ''
+    network_scheme['transformations'].each do |trans|
+      if trans['bridge'] == network_scheme['roles']['neutron/mesh']
+        phys_dev, vlan = trans['name'].split('.')
+        break
+      end
+    end
+
+    mac = `cat /sys/class/net/#{phys_dev}/address`.chomp
+    if $?.success?
+       output.split('vif').each do |iface|
+         if iface.start_with?( '0/0')
+           mac = iface.split[8][7..-1]
+         end
+       end
+     end
+     mac
+   end
+end
+

deployment_scripts/puppet/modules/contrail/lib/facter/supervisor_params.rb

@@ -0,0 +1,109 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+# This facter returns the version and build for the python-contrail package.
+# It may be used to detect a version of contrail used in the environment.
+
+require 'hiera'
+
+Facter.add("supervisor_params") do
+  setcode do
+    res = []
+    vrouter_config = '/etc/contrail/supervisord_vrouter_files/contrail-vrouter-dpdk.ini'
+    mac_from_config = nil
+    if File.exist?(vrouter_config)
+      config_vrouter_params = File.readlines(vrouter_config).find { |line| line.include?('command=')}.split('--no-daemon')[-1].strip
+      mac_from_config = config_vrouter_params.split.find { |param| param.include?('mac') }
+      mac_from_config = mac_from_config.split(',').find { |param| param.include?('mac') }.split('=')
+    end
+    if `status supervisor-vrouter`.include?('stop/waiting')
+      # NOTE (dukov) We do not need to gather data from system in case vrouter has started
+      # Moreover these data may differ from data without vrouter service started
+      bond_policy_map = {
+        'layer2'   => 'l2',
+        'layer3+4' => 'l34',
+        'layer2+3' => 'l23',
+        'encap2+3' => 'l23',
+        'encap3+4' => 'l34',
+      }
+      hiera = Hiera.new(:config => '/etc/hiera.yaml')
+      network_scheme = hiera.lookup('network_scheme', {}, {}, nil, :hash)
+
+      phys_dev = vlan = ''
+      network_scheme['transformations'].each do |trans|
+        if trans['bridge'] == network_scheme['roles']['neutron/mesh']
+          phys_dev, vlan = trans['name'].split('.')
+          break
+        end
+      end
+
+      bond_dir = "/sys/class/net/#{phys_dev}/bonding"
+      add_vdev = false
+      if File.exist?(bond_dir)
+        # NOTE(dukov) This chunk of code will return a Hash with bond slaves info
+        # Example
+        #   {
+        #    "enp67s0f1"=> {
+        #      "numa_node"=>"1",
+        #      "slave_pci"=>"0000:43:00.1"},
+        #    "enp68s0f1"=> {
+        #      "numa_node"=>"1",
+        #      "slave_pci"=>"0000:44:00.1"}
+        #   }
+        bond_slaves = Hash[`cat #{bond_dir}/slaves`.split.sort.collect do |slave|
+          slave_pci = `basename $(readlink /sys/class/net/#{slave}/device)`.chomp
+          numa_node = `cat /sys/class/net/#{slave}/device/numa_node`.chomp
+          [slave, {'numa_node' => numa_node == '-1' ? 0 : numa_node, 'slave_pci' => slave_pci}]
+        end]
+
+        add_vdev = !bond_slaves.values[-1]['numa_node'].empty?
+      end
+
+      # vdev
+      if add_vdev
+        res << '--vdev'
+        vdev_values = []
+        vdev_values << "eth_bond_#{phys_dev}"
+        vdev_values << "mode=#{`cat #{bond_dir}/mode`.split()[1]}"
+
+        bond_policy = `cat #{bond_dir}/xmit_hash_policy`.split()[0]
+        vdev_values << "xmit_policy=#{bond_policy_map[bond_policy]}"
+        vdev_values << "socket_id=#{bond_slaves.values[-1]['numa_node']}"
+
+        # NOTE(dukov) we should not change mac here to avoid vrouter restart
+        # which is why we need to grab this from Supervisor config
+        mac = !!mac_from_config ? mac_from_config[1] : `cat /sys/class/net/#{bond_slaves.keys[-1]}/address`.chomp
+
+        vdev_values << "mac=#{mac}"
+        vdev_values << bond_slaves.values.map {|slave_info| "slave=#{slave_info['slave_pci']}"}.join(',')
+        res << "\"#{vdev_values.join(',')}\""
+      end
+
+      # vlan_tci and vlan_fwd_intf_name
+      if !!vlan
+        res << "--vlan_tci \"#{vlan}\""
+        res << "--vlan_fwd_intf_name \"#{phys_dev}\""
+      end
+
+      # socket-mem
+      socket_mem = Dir["/sys/devices/system/node/node*/hugepages"].map{ |pages| 1024}.join(',')
+      res << "--socket-mem #{socket_mem}" unless socket_mem.empty?
+    else
+      # NOTE(dukov) Let's get data from Supervisor configfile
+      res << config_vrouter_params
+    end
+
+    res.join(' ')
+  end
+end

deployment_scripts/puppet/modules/contrail/lib/puppet/parser/functions/get_private_ifname.rb

@@ -12,8 +12,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-require 'yaml'
-
 module Puppet::Parser::Functions
 newfunction(:get_private_ifname, :type => :rvalue, :doc => <<-EOS
     Returns interface selected as "Private network" in web UI

deployment_scripts/puppet/modules/contrail/lib/puppet/parser/functions/vrouter_override_network_scheme.rb

@@ -0,0 +1,67 @@
+#    Copyright 2016 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+Puppet::Parser::Functions::newfunction( :vrouter_override_network_scheme,
+                                        :type => :rvalue, :arity => 3, :doc => <<-EOS
+    Override network_scheme to skip interfaces used by DPDK vrouter
+    1. Get real nic name
+    2. Check if interface is a bond
+    3. Override bond creation if it is
+    4. Override add-port to br-mesh
+    5. Override add-br for br-mesh
+    6. Override endpoint for br-mesh
+    EOS
+  ) do |argv|
+    override       = {'transformations' => [],
+                      'endpoints' => {},
+                      'roles' => {},
+                      'interfaces' => {}}
+    orig_ns, interface, dpdk_enabled = argv
+    interface_real = interface.split('.').first
+    private_bridge = function_get_network_role_property(['neutron/mesh', 'interface'])
+
+    # Overriding transformations
+    orig_ns['transformations'].each do |tr|
+      if tr['action'] == 'add-bond' and tr['name'] == interface_real and dpdk_enabled
+        override['transformations'] << {'action'          => 'override',
+                                        'override'        => interface_real,
+                                        'override-action' => 'noop'}
+      elsif tr['action'] == 'add-port' and tr['name'] == interface
+        override['transformations'] << {'action'          => 'override',
+                                        'override'        => interface,
+                                        'override-action' => 'noop'}
+      elsif tr['action'] == 'add-br' and tr['name'] == private_bridge
+        override['transformations'] << {'action'          => 'override',
+                                        'override'        => private_bridge,
+                                        'override-action' => 'noop'}
+      end
+    end
+
+    # Overriding 'br-mesh' endpoint
+    orig_ns['endpoints'].each do |ep_name, ep_data|
+      if ep_name == private_bridge
+        override['endpoints'][private_bridge] = ''
+        override['endpoints']['vhost0'] = orig_ns['endpoints'][private_bridge]
+      end
+    end
+
+    # Overriding network roles
+    override['roles']['contrail/vhost0'] = 'vhost0'
+
+    # Overriding interfaces
+    override['interfaces']['vhost0'] = {}
+
+    {'network_scheme' => override}
+end
+# vim: set ts=2 sw=2 et :

deployment_scripts/puppet/modules/contrail/manifests/compute/compute_netconfig_override.pp

@@ -0,0 +1,33 @@
+#    Copyright 2016 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+class contrail::compute::compute_netconfig_override {
+
+  if roles_include('dpdk') {
+    # Override network_scheme to skip interfaces used by the vrouter
+    $settings = hiera_hash('contrail', {})
+    $network_scheme = hiera_hash('network_scheme')
+
+    prepare_network_config($network_scheme)
+    $override_ns = vrouter_override_network_scheme($network_scheme,
+                                                  $contrail::phys_dev,
+                                                  $contrail::compute_dpdk_enabled)
+
+    file { '/etc/hiera/plugins/contrail-vrouter-override_ns.yaml':
+      ensure  => file,
+      content => inline_template('<%= YAML.dump @override_ns %>'),
+      replace => false,
+    }
+  }
+}

deployment_scripts/puppet/modules/contrail/manifests/compute/network.pp

@@ -13,11 +13,10 @@
 #    under the License.
 
 class contrail::compute::network {
-  $node_role = 'compute'
+  $address        = $contrail::address
-  $address = $contrail::address
+  $ifname         = $contrail::phys_dev
-  $ifname = $contrail::phys_dev
+  $netmask        = $contrail::netmask_short
-  $netmask = $contrail::netmask_short
+  $default_gw     = undef
-  $default_gw = undef
 
   $br_file = $::operatingsystem ? {
     'Ubuntu' => '/etc/network/interfaces.d/ifcfg-br-mesh',

deployment_scripts/puppet/modules/contrail/manifests/compute/vrouter.pp

@@ -26,8 +26,9 @@ class contrail::compute::vrouter {
 
   if $contrail::compute_dpdk_enabled {
 
-    if empty($dev_mac) {
+    $dpdk_mac = $::mac_from_vrouter
-      $dpdk_dev_mac = get_mac_from_vrouter()
+    if $dpdk_mac {
+      $dpdk_dev_mac = $dpdk_mac
     } else {
       $dpdk_dev_mac = $dev_mac
     }
@@ -38,9 +39,9 @@ class contrail::compute::vrouter {
     if ( 'bond' in $raw_phys_dev) {
       file_line { 'permanent_mac':
         ensure => present,
-        line   => "hwaddress ${dev_mac}",
+        line => "hwaddress ${dpdk_dev_mac}",
-        path   => "/etc/network/interfaces.d/ifcfg-${raw_phys_dev}",
+        path => "/etc/network/interfaces.d/ifcfg-${raw_phys_dev}",
-        after  => "iface ${raw_phys_dev} inet manual",
+        after => "iface ${raw_phys_dev} inet manual",
       }
     }
 
@@ -48,7 +49,7 @@ class contrail::compute::vrouter {
     $delete_packages  = ['openvswitch-common','openvswitch-datapath-dkms','openvswitch-datapath-lts-saucy-dkms','openvswitch-switch','nova-network','nova-api']
 
     contrail_vrouter_dpdk_ini_config {
-      'program:contrail-vrouter-dpdk/command':                 value => "taskset ${contrail::vrouter_core_mask} /usr/bin/contrail-vrouter-dpdk --no-daemon";
+      'program:contrail-vrouter-dpdk/command':                 value => "taskset ${contrail::vrouter_core_mask} /usr/bin/contrail-vrouter-dpdk --no-daemon ${::supervisor_params}";
       'program:contrail-vrouter-dpdk/priority':                value => '410';
       'program:contrail-vrouter-dpdk/loglevel':                value => 'debug';
       'program:contrail-vrouter-dpdk/autostart':               value => true;

deployment_scripts/puppet/modules/contrail/manifests/init.pp

@@ -33,18 +33,20 @@ class contrail {
 
   # Network configuration
   prepare_network_config($network_scheme)
-  $interface         = get_network_role_property('neutron/mesh', 'interface')
+  $interface         = pick(get_network_role_property('neutron/mesh', 'interface'), 'br-mesh')
-  $routes            = pick($network_scheme['endpoints'][$interface]['routes'], false)
 
-  if  $routes {
+  $iface = pick($network_scheme['endpoints'][$interface], {})
+  $routes = pick($iface['routes'], false)
+
+  if $routes {
     $gateway = $routes[0]['via']
   } else {
     $gateway = false
   }
 
-  $address           = get_network_role_property('neutron/mesh', 'ipaddr')
+  $address           = pick(get_network_role_property('neutron/mesh', 'ipaddr'), get_network_role_property('contrail/vhost0', 'ipaddr'))
-  $cidr              = get_network_role_property('neutron/mesh', 'cidr')
+  $cidr              = pick(get_network_role_property('neutron/mesh', 'cidr'), get_network_role_property('contrail/vhost0', 'cidr'))
-  $netmask           = get_network_role_property('neutron/mesh', 'netmask')
+  $netmask           = pick(get_network_role_property('neutron/mesh', 'netmask'), get_network_role_property('contrail/vhost0', 'netmask'))
   $netmask_short     = netmask_to_cidr($netmask)
   $phys_dev          = get_private_ifname($interface, $network_scheme)
   $phys_dev_pci      = get_dev_pci_addr($phys_dev, $network_scheme)

deployment_tasks.yaml

@@ -515,6 +515,32 @@
   type: skipped
   version: 2.0.0
 
+# Redefined task to skip it on DPDK computes
+- id: openstack-network-common-config
+  type: puppet
+  version: 2.1.0
+  groups: [primary-controller,controller,compute]
+  required_for: [openstack-network-end]
+  requires: [openstack-network-start]
+  condition:
+    yaql_exp: >
+      (changedAny($.get('openstack_network'), $.get('verbose'), $.debug,
+      $.quantum_settings, $.neutron_advanced_configuration, $.rabbit,
+      $.ceilometer, $.network_scheme, $.get('use_syslog'),
+      $.get('use_stderr'), $.get('syslog_log_facility_neutron'),
+      $.network_metadata.nodes.values().where(
+        $.node_roles.any($.matches('controller'))).network_roles.select(
+          $.get('mgmt/messaging')),
+      $.get('amqp_hosts'), $.get('kombu_compression')) and not 'dpdk' in $.roles)
+  parameters:
+    puppet_manifest: /etc/puppet/modules/openstack_tasks/examples/openstack-network/common-config.pp
+    puppet_modules: /etc/puppet/modules
+    timeout: 1800
+
+##################
+# Configuration for Nova, Neutron, Heat, Ceilometer on OpenStack Controllers
+##################
+
 - id: openstack-controller-contrail
   type: puppet
   version: 2.0.0
@@ -709,6 +735,20 @@
   version: 2.0.0
   #
 
+# Overrides bridges settings to make netconfig idempotent with DPDK
+- id: contrail-compute-netconfig-override
+  groups: [compute]
+  type: puppet
+  version: 2.1.0
+  condition:
+    yaql_exp: ('dpdk' in $.roles)
+  required_for: [deploy_end, contrail-override-repository]
+  requires: [deploy_start, netconfig]
+  parameters:
+    puppet_manifest: puppet/manifests/contrail-compute-netconfig-override.pp
+    puppet_modules: puppet/modules:/etc/puppet/modules
+    timeout: 120
+
 # Set apt pin for packeges that need to be override
 - id: contrail-override-repository
   type: puppet