From 2badc837fc6ce5c31bfa984e3275781b968dabe3 Mon Sep 17 00:00:00 2001 From: Gil Meir Date: Sun, 29 Mar 2015 01:55:24 +0300 Subject: [PATCH] Adding Ethernet flow * The pre_deployment stage runs after OS provisioning, and before Openstack is installed. It does the following: - Installs OFED - Sets SR-IOV related configurations in the driver & FW - Renames the iSER interface if iSER is used * The post_deployment stage runs after the Openstack installation. It does the following: - Runs puppet manifests to configure each role: controller, cinder, compute - Replaces the existing TestVM image with the Mellanox-Cirros image Change-Id: Icd31cd38079f8ffa93f4dc34df88107b318a5df3 Signed-off-by: Gil Meir --- .gitignore | 5 + README.md | 2 +- deployment_scripts/common | 49 +++ deployment_scripts/delete_images.rb | 100 ++++++ deployment_scripts/install_ofed.sh | 133 ++++++++ deployment_scripts/link_astute_file.sh | 48 +++ deployment_scripts/log_stage.sh | 18 + deployment_scripts/mellanox_settings.py | 308 ++++++++++++++++++ deployment_scripts/puppet/manifests/cinder.pp | 7 + .../puppet/manifests/compute.pp | 8 + .../puppet/manifests/controller.pp | 9 + .../puppet/manifests/iser_rename.pp | 8 + deployment_scripts/puppet/manifests/testvm.pp | 3 + .../puppet/modules/mellanox_openstack/LICENSE | 201 ++++++++++++ .../modules/mellanox_openstack/Modulefile | 6 + .../modules/mellanox_openstack/README.md | 31 ++ .../modules/mellanox_openstack/Rakefile | 18 + .../mellanox_openstack/files/network.filters | 94 ++++++ .../mellanox_agent_config/ini_setting.rb | 22 ++ .../mellanox_eswitchd_config/ini_setting.rb | 22 ++ .../lib/puppet/type/mellanox_agent_config.rb | 19 ++ .../puppet/type/mellanox_eswitchd_config.rb | 19 ++ .../mellanox_openstack/manifests/agent.pp | 70 ++++ .../mellanox_openstack/manifests/cinder.pp | 33 ++ .../mellanox_openstack/manifests/compute.pp | 33 ++ .../manifests/controller.pp | 22 ++ .../mellanox_openstack/manifests/eswitchd.pp | 31 ++ .../mellanox_openstack/manifests/init.pp | 3 + .../manifests/iser_rename.pp | 35 ++ .../mellanox_openstack/manifests/params.pp | 23 ++ .../modules/mellanox_openstack/metadata.json | 17 + .../mellanox_openstack/spec/spec_helper.rb | 17 + .../templates/iser_rename.erb | 98 ++++++ .../modules/mellanox_openstack/tests/init.pp | 12 + deployment_scripts/replace_cirros.sh | 37 +++ deployment_scripts/sriov.sh | 205 ++++++++++++ environment_config.yaml | 26 +- metadata.yaml | 8 +- pre_build_hook | 51 ++- specs/post.spec | 6 +- tasks.yaml | 95 +++++- 41 files changed, 1931 insertions(+), 21 deletions(-) create mode 100644 .gitignore create mode 100755 deployment_scripts/common create mode 100755 deployment_scripts/delete_images.rb create mode 100755 deployment_scripts/install_ofed.sh create mode 100755 deployment_scripts/link_astute_file.sh create mode 100755 deployment_scripts/log_stage.sh create mode 100755 deployment_scripts/mellanox_settings.py create mode 100644 deployment_scripts/puppet/manifests/cinder.pp create mode 100644 deployment_scripts/puppet/manifests/compute.pp create mode 100644 deployment_scripts/puppet/manifests/controller.pp create mode 100644 deployment_scripts/puppet/manifests/iser_rename.pp create mode 100644 deployment_scripts/puppet/manifests/testvm.pp create mode 100644 deployment_scripts/puppet/modules/mellanox_openstack/LICENSE create mode 100644 deployment_scripts/puppet/modules/mellanox_openstack/Modulefile create mode 100644 deployment_scripts/puppet/modules/mellanox_openstack/README.md create mode 100644 deployment_scripts/puppet/modules/mellanox_openstack/Rakefile create mode 100644 deployment_scripts/puppet/modules/mellanox_openstack/files/network.filters create mode 100644 deployment_scripts/puppet/modules/mellanox_openstack/lib/puppet/provider/mellanox_agent_config/ini_setting.rb create mode 100644 deployment_scripts/puppet/modules/mellanox_openstack/lib/puppet/provider/mellanox_eswitchd_config/ini_setting.rb create mode 100644 deployment_scripts/puppet/modules/mellanox_openstack/lib/puppet/type/mellanox_agent_config.rb create mode 100644 deployment_scripts/puppet/modules/mellanox_openstack/lib/puppet/type/mellanox_eswitchd_config.rb create mode 100644 deployment_scripts/puppet/modules/mellanox_openstack/manifests/agent.pp create mode 100644 deployment_scripts/puppet/modules/mellanox_openstack/manifests/cinder.pp create mode 100644 deployment_scripts/puppet/modules/mellanox_openstack/manifests/compute.pp create mode 100644 deployment_scripts/puppet/modules/mellanox_openstack/manifests/controller.pp create mode 100644 deployment_scripts/puppet/modules/mellanox_openstack/manifests/eswitchd.pp create mode 100644 deployment_scripts/puppet/modules/mellanox_openstack/manifests/init.pp create mode 100644 deployment_scripts/puppet/modules/mellanox_openstack/manifests/iser_rename.pp create mode 100644 deployment_scripts/puppet/modules/mellanox_openstack/manifests/params.pp create mode 100644 deployment_scripts/puppet/modules/mellanox_openstack/metadata.json create mode 100644 deployment_scripts/puppet/modules/mellanox_openstack/spec/spec_helper.rb create mode 100644 deployment_scripts/puppet/modules/mellanox_openstack/templates/iser_rename.erb create mode 100644 deployment_scripts/puppet/modules/mellanox_openstack/tests/init.pp create mode 100755 deployment_scripts/replace_cirros.sh create mode 100755 deployment_scripts/sriov.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9656622 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.build +bootstrap/* +repositories/* +*.rpm +*.deb diff --git a/README.md b/README.md index d7ce6bd..04fc267 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -mellanox_plugin +mellanox-plugin ================ Plugin description diff --git a/deployment_scripts/common b/deployment_scripts/common new file mode 100755 index 0000000..d333d0e --- /dev/null +++ b/deployment_scripts/common @@ -0,0 +1,49 @@ +#!/bin/bash +# Copyright 2015 Mellanox Technologies, Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +readonly SCRIPT_NAME=`basename $0` + +function logger_print () { + priority=$1 + msg=$2 + logger -t $SCRIPT_NAME "${priority}: ${msg}" + echo $(date +"%Y-%m-%d %H:%M:%S") $SCRIPT_NAME ${priority}: ${msg} >> /var/log/mellanox-plugin.log +} + +function get_mlnx_param () { + param="$1" + val=$(ruby -r hiera -r yaml -e "hiera = Hiera.new(:config => '/etc/puppet/hiera.yaml'); mlnx = hiera.lookup 'mellanox-plugin', [], {}; puts mlnx['"$param"']") + echo $val +} + +function get_distro () { + if [[ -f /etc/redhat-release ]]; then + dist='redhat'; + elif [[ -f /etc/debian_version ]]; then + dist='ubuntu'; + else + logger_print error "Could not detect linux distribution" + fi + echo $dist +} + +readonly DISTRO=`get_distro` +readonly DRIVER=`get_mlnx_param driver` +readonly SRIOV=`get_mlnx_param sriov` +readonly USER_NUM_OF_VFS=`get_mlnx_param num_of_vfs` +readonly ISER=`get_mlnx_param iser` +readonly MAX_VFS=64 +readonly MIN_VFS=1 diff --git a/deployment_scripts/delete_images.rb b/deployment_scripts/delete_images.rb new file mode 100755 index 0000000..8c83656 --- /dev/null +++ b/deployment_scripts/delete_images.rb @@ -0,0 +1,100 @@ +#!/usr/bin/env ruby +## Copyright 2015 Mellanox Technologies, Ltd +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +## implied. +## See the License for the specific language governing permissions and +## limitations under the License. + +require 'hiera' +ENV['LANG'] = 'C' +LOG_FILE="/var/log/mellanox-plugin.log" + +def log(level, msg) + current_time = Time.now.strftime("%Y-%m-%d %H:%M:%S") + File.open(LOG_FILE, 'a') { |f| + f.puts "#{current_time} delete_images.rb #{level}: #{msg}" + } +end + +def image_list + stdout = `. /root/openrc && glance image-list` + return_code = $?.exitstatus + [ stdout, return_code ] +end + +def images_ids + stdout, return_code = image_list + if return_code != 0 + raise 'Failed retrieving image-list' + end + ids = [] + stdout.split("\n").each do |line| + fields = line.split('|').map { |f| f.chomp.strip } + next if fields[1] == 'ID' + next unless fields[1] + ids << fields[1] + end + {:ids => ids, :exit_code => return_code} +end + + +def delete_image(id) + command = ". /root/openrc && /usr/bin/glance image-delete #{id}" + stdout = `#{command}` + return_code = $?.exitstatus + [ stdout, return_code ] +end + +def wait_for_glance + 5.times.each do |retries| + sleep 10 if retries > 0 + _, return_code = image_list + return if return_code == 0 + end + raise 'Could not get a list of glance images!' +end + +def main + log("info", "Waiting for glance response") + wait_for_glance + + log("info", "Fetching list of current images") + ids = images_ids + if ids[:exit_code] != 0 + raise 'Failed retrieving existing images ids' + end + + succeed = true + ids[:ids].each do |id| + stdout, return_code = delete_image(id) + if return_code != 0 + log("error", "Failed deleting image with ID = '#{id}'" + stdout) + succeed = false + end + end + if ! succeed + log("error", "Some images weren't deleted, this may cause errors when "+ + "uploading images to glance or when creating an instance") + end +end + +######################## + +begin + main +rescue + log("error", "Some images weren't deleted, this may cause errors when "+ + "uploading images to glance or when creating an instance") + exit 1 +else + log("info", "Successfully deleted all existing images from glance") +end diff --git a/deployment_scripts/install_ofed.sh b/deployment_scripts/install_ofed.sh new file mode 100755 index 0000000..e3fa4d6 --- /dev/null +++ b/deployment_scripts/install_ofed.sh @@ -0,0 +1,133 @@ +#!/bin/bash +# Copyright 2015 Mellanox Technologies, Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +readonly SCRIPT_DIR=$(dirname "$0") +source $SCRIPT_DIR/common + +readonly KERNEL_VERSION="$(uname -r)" +readonly OFED_PACKAGE_NAME="mlnx-ofed-fuel" +readonly OFED_BASE_DIR="/opt/ofed" +readonly OFED_SRC_DIR="${OFED_BASE_DIR}/MLNX_OFED" +readonly OFED_SUCCESS_FILE="${OFED_BASE_DIR}/.success" +readonly OFED_INFO="/usr/bin/ofed_info" +OFED_DIR=$OFED_SRC_DIR +OFED_INSTALL_SCRIPT_CMD="${OFED_DIR}/mlnxofedinstall" + +function is_ofed_installed () { + if [ -f ${OFED_SUCCESS_FILE} ] && [ -f ${OFED_INFO} ] && ( ${OFED_INFO} 2>&1 >/dev/null ); then + installed_ofed_version=`${OFED_INFO} -s` + logger_print info "OFED is already installed: ${installed_ofed_version}" + return 0 + fi +} + +function install_mlnx_ofed_src () { + logger_print info "Installing ${OFED_PACKAGE_NAME} source code" + if [ "$DISTRO" == "redhat" ]; then + yum install ${OFED_PACKAGE_NAME} -y + elif [ "$DISTRO" == "ubuntu" ]; then + apt-get install ${OFED_PACKAGE_NAME} -y + fi + if [ $? -ne 0 ]; then + logger_print error "Failed installing ${OFED_PACKAGE_NAME} package" + exit 1 + fi +} + +function add_kernel_support () { + # ubuntu doesn't require recompilation in case of kernel change, it supports dkms + if [ "$DISTRO" == "ubuntu" ]; then + return + fi + OFED_ADD_KERNEL_SUPPORT_SCRIPT="${OFED_DIR}/mlnx_add_kernel_support.sh" + if [ ! -x $OFED_ADD_KERNEL_SUPPORT_SCRIPT ] ; then + logger_print error "Failed to find $OFED_ADD_KERNEL_SUPPORT_SCRIPT" + exit 1 + fi + OFED_VERSION=$(cat ${OFED_DIR}/.mlnx) + OFED_ARCH=$(cat ${OFED_DIR}/.arch) + OFED_DISTRO=$(cat ${OFED_DIR}/distro) + RECOMPILED_OFED_NAME="MLNX_OFED_LINUX-${OFED_VERSION}-${OFED_DISTRO}-${OFED_ARCH}-ext" + RECOMPILED_OFED_DIR="${OFED_BASE_DIR}/${RECOMPILED_OFED_NAME}" + + # Recompile OFED in case original OFED at $OFED_DIR doesn't support the existing kernel + if ( ! grep -Fxq ${KERNEL_VERSION} ${OFED_DIR}/.supported_kernels ); then + if [ ! -d ${RECOMPILED_OFED_DIR} ] || ( ! grep -Fxq ${KERNEL_VERSION} ${RECOMPILED_OFED_DIR}/.supported_kernels); then + logger_print info "Recompiling OFED for kernel ${KERNEL_VERSION}" + ${OFED_DIR}/mlnx_add_kernel_support.sh --force --yes --make-tgz --mlnx_ofed ${OFED_DIR} + recompiled_ofed_archive=/tmp/${RECOMPILED_OFED_NAME}.tgz + tar zxf $recompiled_ofed_archive -C ${OFED_BASE_DIR} + rm -f $recompiled_ofed_archive + fi + OFED_DIR=$RECOMPILED_OFED_DIR + fi +} + +function install_ofed_without_fw_update () { + OFED_INSTALL_SCRIPT="${OFED_DIR}/mlnxofedinstall" + if [ ! -x $OFED_INSTALL_SCRIPT ] ; then + logger_print error "Failed to find $OFED_INSTALL_SCRIPT" + exit 1 + fi + + logger_print info "Installing OFED drivers" + OFED_INSTALL_SCRIPT_CMD="/usr/bin/perl ${OFED_INSTALL_SCRIPT}" + ${OFED_INSTALL_SCRIPT_CMD} --force --enable-sriov --without-fw-update + rc=$? + if [ $rc -ne 0 ] ;then + logger_print error "Failed execute ${OFED_INSTALL_SCRIPT_CMD} error code ${rc}" + exit 1 + else + touch ${OFED_SUCCESS_FILE} + fi +} + +function update_fw_if_not_oem () { + BUS_ID=`lspci | grep -m 1 Mellanox | cut -d' ' -f1` + if [ -z $BUS_ID ]; then + logger_print info "Didn't find bus, skipping firmware upgrade" + exit 0 + fi + + mstflint -d ${BUS_ID} q | grep -i PSID | grep MT_ + if [ $? -ne 0 ] ;then + logger_print info "Not Mellanox Card, skipping firmware upgrade" + exit 0 + fi + + logger_print info "Updating FW on Mellanox HCA with BUS ID = ${BUS_ID}" + ${OFED_INSTALL_SCRIPT_CMD} --fw-update-only + if [ $? -ne 0 ] ;then + logger_print error "Failed execute ${OFED_INSTALL_SCRIPT_CMD} error code $?" + exit 1 + fi +} + +if ! is_ofed_installed; then + # Install mlnx-ofed-fuel rpm/deb package which extracts OFED installation dir + install_mlnx_ofed_src + + # Add support for the current kernel in case the OFED included in Fuel + # wasn't compiled for the current kernel + add_kernel_support + + # First install OFED without SR-IOV and FW upgrade + install_ofed_without_fw_update +fi + +# OEM cards require a different dedicated OFED build, this build doesn't +# support them. +update_fw_if_not_oem diff --git a/deployment_scripts/link_astute_file.sh b/deployment_scripts/link_astute_file.sh new file mode 100755 index 0000000..414eb03 --- /dev/null +++ b/deployment_scripts/link_astute_file.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Copyright 2015 Mellanox Technologies, Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source ./common +ROLES="primary-controller controller compute cinder" +ASTUTE_FILE=/etc/astute.yaml + +function check_symlink () { + symlink_source=$(readlink ${ASTUTE_FILE}) +} + +# Check if a symlink already exists +check_symlink +if [ $? -eq 0 ]; then + logger_print info "Symbolic link already exists: ${ASTUTE_FILE} --> ${symlink_source}" + exit 0 +fi + +# Create astute.yaml symlink to any of the .yaml files +for role in $ROLES; do + role_file=/etc/"$role".yaml + if [ -f $role_file ]; then + ln -s -f $role_file ${ASTUTE_FILE} + break + fi +done + +check_symlink +if [ $? -ne 0 ]; then + logger_print error "Failed creating a symbolic link for ${ASTUTE_FILE}" + exit 1 +else + logger_print info "Symbolic link ${ASTUTE_FILE} --> ${symlink_source} was created" + exit 0 +fi diff --git a/deployment_scripts/log_stage.sh b/deployment_scripts/log_stage.sh new file mode 100755 index 0000000..f0719dd --- /dev/null +++ b/deployment_scripts/log_stage.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright 2015 Mellanox Technologies, Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source ./common +logger_print info "===== Running plugin $1 stage =====" diff --git a/deployment_scripts/mellanox_settings.py b/deployment_scripts/mellanox_settings.py new file mode 100755 index 0000000..9d2bcce --- /dev/null +++ b/deployment_scripts/mellanox_settings.py @@ -0,0 +1,308 @@ +#!/usr/bin/python +# Copyright 2015 Mellanox Technologies, Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import yaml +import glob +import traceback + +MLNX_SECTION = 'mellanox-plugin' +SETTINGS_FILE = '/etc/astute.yaml' +PLUGIN_OVERRIDE_FILE = '/etc/hiera/override/plugins.yaml' +MLNX_DRIVERS_LIST = ('mlx4_en', 'eth_ipoib') +ISER_IFC_NAME = 'eth_iser0' + + +class MellanoxSettingsException(Exception): + pass + +class MellanoxSettings(object): + + data = None + + @classmethod + def get_mlnx_section(cls): + if cls.data is None: + raise MellanoxSettingsException("No YAML file loaded") + if MLNX_SECTION not in cls.data: + raise MellanoxSettingsException( + "Couldn't find section '{0}'".format(MLNX_SECTION) + ) + return cls.data[MLNX_SECTION] + + @classmethod + def get_bridge_for_network(cls, network): + network_to_bridge = { + 'private': 'prv', + 'management': 'mgmt', + 'storage': 'storage', + } + return 'br-{0}'.format(network_to_bridge[network]) + + @classmethod + def get_interface_by_network(cls, network): + if network not in ('management', 'storage', 'private'): + raise MellanoxSettingsException("Unknown network: {0}".format(network)) + br_name = cls.get_bridge_for_network(network) + endpoints = cls.get_endpoints_section() + ifc = endpoints[br_name]['vendor_specific']['phy_interfaces'][0] + return ifc + + @classmethod + def add_driver(cls): + interfaces = cls.get_interfaces_section() + mlnx = cls.get_mlnx_section() + + # validation that no more than 1 mellanox driver is used + interfaces_drivers = {} + for ifc in cls.get_physical_interfaces(): + if ('driver' not in interfaces[ifc]['vendor_specific']) : + raise MellanoxSettingsException( + "Couldn't find 'driver' for interface '{0}'".format(ifc) + ) + interfaces_drivers[ifc] = interfaces[ifc]['vendor_specific']['driver'] + mlnx_drivers = dict( + (ifc, drv) for (ifc, drv) in interfaces_drivers.iteritems() + if drv in MLNX_DRIVERS_LIST + ) + if len(set(mlnx_drivers.values())) > 1: + raise MellanoxSettingsException( + "Found mismatching Mellanox drivers on different interfaces: " + "{0}".format(mlnx_drivers) + ) + + # add the driver to the yaml + mlnx['driver'] = mlnx_drivers.values().pop() + + @classmethod + def add_physical_port(cls): + interfaces = cls.get_interfaces_section() + mlnx = cls.get_mlnx_section() + + private_ifc = cls.get_interface_by_network('private') + if mlnx['driver'] == 'eth_ipoib': + if 'bus_info' not in interfaces[private_ifc]['vendor_specific']: + raise MellanoxSettingsException( + "Couldn't find 'bus_info' for interface " + "{0}".format(private_ifc) + ) + mlnx['physical_port'] = interfaces[private_ifc]['vendor_specific']['bus_info'] + elif mlnx['driver'] == 'mlx4_en': + mlnx['physical_port'] = private_ifc + + @classmethod + def add_storage_vlan(cls): + mlnx = cls.get_mlnx_section() + endpoints = cls.get_endpoints_section() + try: + vlan = int(endpoints['br-storage']['vendor_specific']['vlans']) + except ValueError: + raise MellanoxSettingsException( + "Failed reading vlan for br-storage" + ) + mlnx['storage_vlan'] = vlan + + @classmethod + def add_storage_parent(cls): + mlnx = cls.get_mlnx_section() + storage_ifc = cls.get_interface_by_network('storage') + mlnx['storage_parent'] = storage_ifc + + @classmethod + def add_iser_interface_name(cls): + mlnx = cls.get_mlnx_section() + storage_ifc = cls.get_interface_by_network('storage') + if mlnx['driver'] == 'mlx4_en': + mlnx['iser_ifc_name'] = ISER_IFC_NAME + elif mlnx['driver'] == 'eth_ipoib': + interfaces = cls.get_interfaces_section() + mlnx['iser_ifc_name'] = interfaces[storage_ifc]['vendor_specific']['bus_info'] + else: + raise MellanoxSettingsException("Could not find 'driver' in " + "{0} section".format(MLNX_SECTION)) + + @classmethod + def set_storage_networking_scheme(cls): + endpoints = cls.get_endpoints_section() + interfaces = cls.get_interfaces_section() + transformations = cls.data['network_scheme']['transformations'] + mlnx = cls.get_mlnx_section() + + # Handle iSER interface with and w/o vlan tagging + storage_vlan = mlnx['storage_vlan'] + if storage_vlan: + vlan_name = "{0}.{1}".format(ISER_IFC_NAME, storage_vlan) + # Set storage rule to iSER interface vlan interface + cls.data['network_scheme']['roles']['storage'] = vlan_name + # Set iSER interface vlan interface + transf_add = { + 'action': 'add-port', + 'name': vlan_name, + 'vlan_id': int(storage_vlan), + 'vlan_dev': ISER_IFC_NAME + } + if transf_add not in transformations: + transformations.append(transf_add) + transformations_to_delete = [ + { 'action': 'add-port', + 'name': "{0}.{1}".format( + cls.get_interface_by_network('storage'), + storage_vlan + ), + 'bridge': 'br-storage' } , + { 'action': 'add-br', + 'name': 'br-storage' } + ] + endpoints['br-storage']['vendor_specific']['phy_interfaces'] = [ ISER_IFC_NAME ] + endpoints[vlan_name] = ( + endpoints.pop('br-storage', {}) + ) + for transf_del in transformations_to_delete: + transformations.remove(transf_del) + else: + # Set storage rule to iSER port + cls.data['network_scheme']['roles']['storage'] = ISER_IFC_NAME + # Set iSER endpoint with br-storage parameters + endpoints[ISER_IFC_NAME] = ( + endpoints.pop('br-storage', {}) + ) + interfaces[ISER_IFC_NAME] = {} + + @classmethod + def get_endpoints_section(cls): + return cls.data['network_scheme']['endpoints'] + + @classmethod + def get_physical_interfaces(cls): + endpoints = cls.get_endpoints_section() + interfaces = cls.get_interfaces_section() + mlnx_phys_ifcs = [] + for ep in endpoints: + # skip non physical interfaces + if ('vendor_specific' not in endpoints[ep] or + 'phy_interfaces' not in endpoints[ep]['vendor_specific']): + continue + phys_ifc = endpoints[ep]['vendor_specific']['phy_interfaces'][0] + if ('vendor_specific' not in interfaces[phys_ifc] or + 'driver' not in interfaces[phys_ifc]['vendor_specific']): + raise MellanoxSettingsException( + "Missing 'vendor_specific' or 'driver' " + "in {0}".format(phys_ifc) + ) + if (interfaces[phys_ifc]['vendor_specific']['driver'] in + MLNX_DRIVERS_LIST): + mlnx_phys_ifcs.append(phys_ifc) + return list(set(mlnx_phys_ifcs)) + + @classmethod + def get_interfaces_section(cls): + return cls.data['network_scheme']['interfaces'] + + @classmethod + def is_iser_enabled(cls): + return cls.get_mlnx_section()['iser'] + + @classmethod + def update_role_settings(cls): + # realize the driver in use (eth/ib) + cls.add_driver() + # decide the physical function for SR-IOV + cls.add_physical_port() + # set iSER parameters + if cls.is_iser_enabled(): + cls.add_storage_parent() + cls.add_storage_vlan() + cls.add_iser_interface_name() + cls.set_storage_networking_scheme() + + @classmethod + def read_from_yaml(cls, settings_file): + try: + fd = open(settings_file, 'r') + except IOError: + raise MellanoxSettingsException("Given YAML file {0} doesn't " + "exist".format(settings_file)) + try: + data = yaml.load(fd) + except yaml.YAMLError, exc: + if hasattr(exc, 'problem_mark'): + mark = exc.problem_mark + raise MellanoxSettingsException( + "Faild parsing YAML file {0}: error position " + "({2},{3})".format(mark.line+1, mark.column+1) + ) + finally: + fd.close() + cls.data = data + + @classmethod + def write_to_yaml(cls, settings_file): + # choose only the edited sections + data = {} + data['network_scheme'] = cls.data['network_scheme'] + data[MLNX_SECTION] = cls.data[MLNX_SECTION] + # create containing adir + try: + settings_dir = os.path.dirname(settings_file) + if not os.path.isdir(settings_dir): + os.makedirs(settings_dir) + except OSError: + raise MellanoxSettingsException( + "Failed creating directory: {0}".format(settings_dir) + ) + try: + fd = open(settings_file, 'w') + yaml.dump(data, fd, default_flow_style=False) + except IOError: + raise MellanoxSettingsException("Failed writing changes to " + "{0}".format(settings_file)) + finally: + if fd: + fd.close() + + @classmethod + def update_settings(cls): + # define input yaml file + try: + cls.read_from_yaml(SETTINGS_FILE) + cls.update_role_settings() + cls.write_to_yaml(PLUGIN_OVERRIDE_FILE) + except MellanoxSettingsException, exc: + sys.stderr.write("Couldn't add Mellanox settings to " + "{0}: {1}\n".format(settings_file, exc)) + raise MellanoxSettingsException("Failed updating one or more " + "setting files") + +def main(): + try: + settings = MellanoxSettings() + settings.update_settings() + except MellanoxSettingsException, exc: + sys.stderr.write("Failed adding Mellanox settings: {0}\n".format(exc)) + sys.exit(1) + except Exception, exc: + sys.stderr.write("An unknown error has occured while adding " + "Mellanox settings: {0}\n".format( + traceback.format_exc() + ) + ) + sys.exit(1) + sys.stdout.write("Done adding Mellanox settings\n") + sys.exit(0) + +if __name__ == '__main__': + main() diff --git a/deployment_scripts/puppet/manifests/cinder.pp b/deployment_scripts/puppet/manifests/cinder.pp new file mode 100644 index 0000000..58940ac --- /dev/null +++ b/deployment_scripts/puppet/manifests/cinder.pp @@ -0,0 +1,7 @@ +$mlnx = hiera('mellanox-plugin') +$storage_address = hiera('storage_address') + +class { 'mellanox_openstack::cinder' : + iser => $mlnx['iser'], + iser_ip_address => $storage_address, +} diff --git a/deployment_scripts/puppet/manifests/compute.pp b/deployment_scripts/puppet/manifests/compute.pp new file mode 100644 index 0000000..5ab010f --- /dev/null +++ b/deployment_scripts/puppet/manifests/compute.pp @@ -0,0 +1,8 @@ +$network_scheme = hiera('network_scheme') +$quantum_settings = hiera('quantum_settings') +$mlnx = hiera('mellanox-plugin') + +class { 'mellanox_openstack::compute' : + physnet => $quantum_settings['predefined_networks']['net04']['L2']['physnet'], + physifc => $mlnx['physical_port'], +} diff --git a/deployment_scripts/puppet/manifests/controller.pp b/deployment_scripts/puppet/manifests/controller.pp new file mode 100644 index 0000000..462dbc6 --- /dev/null +++ b/deployment_scripts/puppet/manifests/controller.pp @@ -0,0 +1,9 @@ +$eswitch_vnic_type = 'hostdev' +$eswitch_apply_profile_patch = 'True' +$mechanism_drivers = 'openvswitch' + +class { 'mellanox_openstack::controller' : + eswitch_vnic_type => $eswitch_vnic_type, + eswitch_apply_profile_patch => $eswitch_apply_profile_patch, + mechanism_drivers => $mechanism_drivers, +} diff --git a/deployment_scripts/puppet/manifests/iser_rename.pp b/deployment_scripts/puppet/manifests/iser_rename.pp new file mode 100644 index 0000000..56e8cbc --- /dev/null +++ b/deployment_scripts/puppet/manifests/iser_rename.pp @@ -0,0 +1,8 @@ +$mlnx = hiera('mellanox-plugin') + +if ($mlnx['iser']) { + class { 'mellanox_openstack::iser_rename' : + storage_parent => $mlnx['storage_parent'], + iser_interface_name => $mlnx['iser_ifc_name'], + } +} diff --git a/deployment_scripts/puppet/manifests/testvm.pp b/deployment_scripts/puppet/manifests/testvm.pp new file mode 100644 index 0000000..9aef76a --- /dev/null +++ b/deployment_scripts/puppet/manifests/testvm.pp @@ -0,0 +1,3 @@ +package { 'cirros-testvm-mellanox' : + ensure => installed, +} diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/LICENSE b/deployment_scripts/puppet/modules/mellanox_openstack/LICENSE new file mode 100644 index 0000000..8d968b6 --- /dev/null +++ b/deployment_scripts/puppet/modules/mellanox_openstack/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/Modulefile b/deployment_scripts/puppet/modules/mellanox_openstack/Modulefile new file mode 100644 index 0000000..8347260 --- /dev/null +++ b/deployment_scripts/puppet/modules/mellanox_openstack/Modulefile @@ -0,0 +1,6 @@ +name 'mellanox-mellanox_openstack' +version '1.1.0' +author 'mellanox' +license 'Apache License, Version 2.0' +summary 'Mellanox module for Mirantis Fuel Openstack deployment' +description 'This is a Puppet module to install Mellanox Openstack components over Mirantis Fuel' diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/README.md b/deployment_scripts/puppet/modules/mellanox_openstack/README.md new file mode 100644 index 0000000..7238562 --- /dev/null +++ b/deployment_scripts/puppet/modules/mellanox_openstack/README.md @@ -0,0 +1,31 @@ +# mellanox_openstack + +## Overview + +The mellanox_openstack module is designated for configuring the Mellanox plugin +for openstack installed using Fuel. + + +## Module Description + +This module was written for integrating Mellanox into Fuel (https://launchpad.net/fuel). +The plugin supports SR-IOV for networking and iSER protocol for storage features over Mellanox hardware. +* SR-IOV - ml2 is configured to work with Mellanox plugin, including Eswitchd service + for managing virtual functions, and Mellanox Neutron Agent for networking properties. +* iSER - Cinder is set to work with iSER protocol. + + +### Setup Requirements + +The module is designed to be used by Fuel (an openstack installer by Mirantis). +It assumes an Openstack environment using Neutron with VLAN segmentation & KVM. + + +## Release Notes/Contributors/Etc + +Contributors: +Gil Meir, gilmeir@mellanox.com +Aviram Bar-Haim, aviramb@mellanox.com + +Support: +Mellanox support - support@mellanox.com diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/Rakefile b/deployment_scripts/puppet/modules/mellanox_openstack/Rakefile new file mode 100644 index 0000000..d1e11f7 --- /dev/null +++ b/deployment_scripts/puppet/modules/mellanox_openstack/Rakefile @@ -0,0 +1,18 @@ +require 'rubygems' +require 'puppetlabs_spec_helper/rake_tasks' +require 'puppet-lint/tasks/puppet-lint' +PuppetLint.configuration.send('disable_80chars') +PuppetLint.configuration.ignore_paths = ["spec/**/*.pp", "pkg/**/*.pp"] + +desc "Validate manifests, templates, and ruby files" +task :validate do + Dir['manifests/**/*.pp'].each do |manifest| + sh "puppet parser validate --noop #{manifest}" + end + Dir['spec/**/*.rb','lib/**/*.rb'].each do |ruby_file| + sh "ruby -c #{ruby_file}" unless ruby_file =~ /spec\/fixtures/ + end + Dir['templates/**/*.erb'].each do |template| + sh "erb -P -x -T '-' #{template} | ruby -c" + end +end diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/files/network.filters b/deployment_scripts/puppet/modules/mellanox_openstack/files/network.filters new file mode 100644 index 0000000..568e8d4 --- /dev/null +++ b/deployment_scripts/puppet/modules/mellanox_openstack/files/network.filters @@ -0,0 +1,94 @@ +# nova-rootwrap command filters for network nodes +# This file should be owned by (and only-writeable by) the root user + +[Filters] +# nova/virt/libvirt/vif.py: 'ip', 'tuntap', 'add', dev, 'mode', 'tap' +# nova/virt/libvirt/vif.py: 'ip', 'link', 'set', dev, 'up' +# nova/virt/libvirt/vif.py: 'ip', 'link', 'delete', dev +# nova/network/linux_net.py: 'ip', 'addr', 'add', str(floating_ip)+'/32'i.. +# nova/network/linux_net.py: 'ip', 'addr', 'del', str(floating_ip)+'/32'.. +# nova/network/linux_net.py: 'ip', 'addr', 'add', '169.254.169.254/32',.. +# nova/network/linux_net.py: 'ip', 'addr', 'show', 'dev', dev, 'scope',.. +# nova/network/linux_net.py: 'ip', 'addr', 'del/add', ip_params, dev) +# nova/network/linux_net.py: 'ip', 'addr', 'del', params, fields[-1] +# nova/network/linux_net.py: 'ip', 'addr', 'add', params, bridge +# nova/network/linux_net.py: 'ip', '-f', 'inet6', 'addr', 'change', .. +# nova/network/linux_net.py: 'ip', 'link', 'set', 'dev', dev, 'promisc',.. +# nova/network/linux_net.py: 'ip', 'link', 'add', 'link', bridge_if ... +# nova/network/linux_net.py: 'ip', 'link', 'set', interface, address,.. +# nova/network/linux_net.py: 'ip', 'link', 'set', interface, 'up' +# nova/network/linux_net.py: 'ip', 'link', 'set', bridge, 'up' +# nova/network/linux_net.py: 'ip', 'addr', 'show', 'dev', interface, .. +# nova/network/linux_net.py: 'ip', 'link', 'set', dev, address, .. +# nova/network/linux_net.py: 'ip', 'link', 'set', dev, 'up' +# nova/network/linux_net.py: 'ip', 'route', 'add', .. +# nova/network/linux_net.py: 'ip', 'route', 'del', . +# nova/network/linux_net.py: 'ip', 'route', 'show', 'dev', dev +ip: CommandFilter, ip, root + +# nova/virt/libvirt/vif.py: 'ovs-vsctl', ... +# nova/virt/libvirt/vif.py: 'ovs-vsctl', 'del-port', ... +# nova/network/linux_net.py: 'ovs-vsctl', .... +ovs-vsctl: CommandFilter, ovs-vsctl, root + +# nova/network/linux_net.py: 'ovs-ofctl', .... +ovs-ofctl: CommandFilter, ovs-ofctl, root + +# nova/virt/libvirt/vif.py: 'ivs-ctl', ... +# nova/virt/libvirt/vif.py: 'ivs-ctl', 'del-port', ... +# nova/network/linux_net.py: 'ivs-ctl', .... +ivs-ctl: CommandFilter, ivs-ctl, root + +# nova/virt/libvirt/vif.py: 'ifc_ctl', ... +ifc_ctl: CommandFilter, /opt/pg/bin/ifc_ctl, root + +# nova/virt/libvirt/vif.py: 'ebrctl', ... +ebrctl: CommandFilter, ebrctl, root + +# nova/virt/libvirt/vif.py: 'mm-ctl', ... +mm-ctl: CommandFilter, mm-ctl, root + +# nova/network/linux_net.py: 'ebtables', '-D' ... +# nova/network/linux_net.py: 'ebtables', '-I' ... +ebtables: CommandFilter, ebtables, root +ebtables_usr: CommandFilter, ebtables, root + +# nova/network/linux_net.py: 'ip[6]tables-save' % (cmd, '-t', ... +iptables-save: CommandFilter, iptables-save, root +ip6tables-save: CommandFilter, ip6tables-save, root + +# nova/network/linux_net.py: 'ip[6]tables-restore' % (cmd,) +iptables-restore: CommandFilter, iptables-restore, root +ip6tables-restore: CommandFilter, ip6tables-restore, root + +# nova/network/linux_net.py: 'arping', '-U', floating_ip, '-A', '-I', ... +# nova/network/linux_net.py: 'arping', '-U', network_ref['dhcp_server'],.. +arping: CommandFilter, arping, root + +# nova/network/linux_net.py: 'dhcp_release', dev, address, mac_address +dhcp_release: CommandFilter, dhcp_release, root + +# nova/network/linux_net.py: 'kill', '-9', pid +# nova/network/linux_net.py: 'kill', '-HUP', pid +kill_dnsmasq: KillFilter, root, /usr/sbin/dnsmasq, -9, -HUP + +# nova/network/linux_net.py: 'kill', pid +kill_radvd: KillFilter, root, /usr/sbin/radvd + +# nova/network/linux_net.py: dnsmasq call +dnsmasq: EnvFilter, env, root, CONFIG_FILE=, NETWORK_ID=, dnsmasq + +# nova/network/linux_net.py: 'radvd', '-C', '%s' % _ra_file(dev, 'conf'.. +radvd: CommandFilter, radvd, root + +# nova/network/linux_net.py: 'brctl', 'addbr', bridge +# nova/network/linux_net.py: 'brctl', 'setfd', bridge, 0 +# nova/network/linux_net.py: 'brctl', 'stp', bridge, 'off' +# nova/network/linux_net.py: 'brctl', 'addif', bridge, interface +brctl: CommandFilter, brctl, root + +# nova/network/linux_net.py: 'sysctl', .... +sysctl: CommandFilter, sysctl, root + +# nova/network/linux_net.py: 'conntrack' +conntrack: CommandFilter, conntrack, root diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/lib/puppet/provider/mellanox_agent_config/ini_setting.rb b/deployment_scripts/puppet/modules/mellanox_openstack/lib/puppet/provider/mellanox_agent_config/ini_setting.rb new file mode 100644 index 0000000..766f90b --- /dev/null +++ b/deployment_scripts/puppet/modules/mellanox_openstack/lib/puppet/provider/mellanox_agent_config/ini_setting.rb @@ -0,0 +1,22 @@ +Puppet::Type.type(:mellanox_agent_config).provide( + :ini_setting, + :parent => Puppet::Type.type(:ini_setting).provider(:ruby) +) do + + def section + resource[:name].split('/', 2).first + end + + def setting + resource[:name].split('/', 2).last + end + + def separator + '=' + end + + def file_path + '/etc/neutron/plugins/mlnx/mlnx_conf.ini' + end + +end diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/lib/puppet/provider/mellanox_eswitchd_config/ini_setting.rb b/deployment_scripts/puppet/modules/mellanox_openstack/lib/puppet/provider/mellanox_eswitchd_config/ini_setting.rb new file mode 100644 index 0000000..d265405 --- /dev/null +++ b/deployment_scripts/puppet/modules/mellanox_openstack/lib/puppet/provider/mellanox_eswitchd_config/ini_setting.rb @@ -0,0 +1,22 @@ +Puppet::Type.type(:mellanox_eswitchd_config).provide( + :ini_setting, + :parent => Puppet::Type.type(:ini_setting).provider(:ruby) +) do + + def section + resource[:name].split('/', 2).first + end + + def setting + resource[:name].split('/', 2).last + end + + def separator + '=' + end + + def file_path + '/etc/eswitchd/eswitchd.conf' + end + +end diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/lib/puppet/type/mellanox_agent_config.rb b/deployment_scripts/puppet/modules/mellanox_openstack/lib/puppet/type/mellanox_agent_config.rb new file mode 100644 index 0000000..97db1c2 --- /dev/null +++ b/deployment_scripts/puppet/modules/mellanox_openstack/lib/puppet/type/mellanox_agent_config.rb @@ -0,0 +1,19 @@ +Puppet::Type.newtype(:mellanox_agent_config) do + + ensurable + + newparam(:name, :namevar => true) do + desc 'Section/setting name to manage from /etc/neutron/plugins/mlnx/mlnx_conf.ini' + newvalues(/\S+\/\S+/) + end + + newproperty(:value) do + desc 'The value of the setting to be defined.' + munge do |value| + value = value.to_s.strip + value.capitalize! if value =~ /^(true|false)$/i + value + end + end + +end diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/lib/puppet/type/mellanox_eswitchd_config.rb b/deployment_scripts/puppet/modules/mellanox_openstack/lib/puppet/type/mellanox_eswitchd_config.rb new file mode 100644 index 0000000..ca38fac --- /dev/null +++ b/deployment_scripts/puppet/modules/mellanox_openstack/lib/puppet/type/mellanox_eswitchd_config.rb @@ -0,0 +1,19 @@ +Puppet::Type.newtype(:mellanox_eswitchd_config) do + + ensurable + + newparam(:name, :namevar => true) do + desc 'Section/setting name to manage from /etc/eswitchd/eswitchd.conf' + newvalues(/\S+\/\S+/) + end + + newproperty(:value) do + desc 'The value of the setting to be defined.' + munge do |value| + value = value.to_s.strip + value.capitalize! if value =~ /^(true|false)$/i + value + end + end + +end diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/manifests/agent.pp b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/agent.pp new file mode 100644 index 0000000..ba0d490 --- /dev/null +++ b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/agent.pp @@ -0,0 +1,70 @@ +class mellanox_openstack::agent ( + $physnet, + $physifc, +) { + include mellanox_openstack::params + + $package = $::mellanox_openstack::params::neutron_mlnx_packages + $agent = $::mellanox_openstack::params::agent_service + $filters_dir = $::mellanox_openstack::params::filters_dir + $filters_file = $::mellanox_openstack::params::filters_file + $compute_service_name = $::mellanox_openstack::params::compute_service_name + $mlnx_agent_conf = $::mellanox_openstack::params::mlnx_agent_conf + + # Only relevant for Debian since no package provides network.filters file + if $::osfamily == 'Debian' { + File { + owner => 'root', + group => 'root', + } + + file { $filters_dir : + ensure => directory, + mode => '0755', + } + + file { $filters_file : + ensure => present, + mode => '0644', + source => 'puppet:///modules/mellanox_openstack/network.filters', + } + + service { $compute_service_name : + ensure => running + } + + File <| title == '/etc/nova/nova.conf' |> -> + File[$filters_dir] -> + File[$filters_file] ~> + Service[$compute_service_name] + } + + file { $mlnx_agent_conf : + owner => 'neutron' + } + + mellanox_agent_config { + 'agent/rpc_support_old_agents' : value => true; + 'eswitch/physical_interface_mappings' : value => "${physnet}:${physifc}"; + } + + package { $package : + ensure => installed, + } + + service { $agent : + ensure => running, + enable => true, + hasstatus => true, + hasrestart => true, + } + + Package[$package] -> + File[$mlnx_agent_conf] -> + Mellanox_agent_config <||> ~> + Service[$agent] + + Package[$package] ~> + Service[$agent] + +} diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/manifests/cinder.pp b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/cinder.pp new file mode 100644 index 0000000..6c732dd --- /dev/null +++ b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/cinder.pp @@ -0,0 +1,33 @@ +class mellanox_openstack::cinder ( + $iser, + $iser_ip_address, +) { + include cinder::params + + class { 'mellanox_openstack::cinder::cinder_conf' : + iser => $iser, + iser_ip_address => $iser_ip_address, + } ~> + service { $cinder::params::volume_service : + ensure => running + } +} + +class mellanox_openstack::cinder::cinder_conf ( + $iser, + $iser_ip_address, +) { + include cinder::params + include mellanox_openstack::params + + if $iser { + cinder_config { 'DEFAULT/volume_driver' : + value => 'cinder.volume.drivers.lvm.LVMISERDriver' + } + cinder_config { 'DEFAULT/iser_ip_address' : + value => $iser_ip_address + } + } +} + + diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/manifests/compute.pp b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/compute.pp new file mode 100644 index 0000000..5f40c02 --- /dev/null +++ b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/compute.pp @@ -0,0 +1,33 @@ +class mellanox_openstack::compute ( + $physnet, + $physifc, +) { + + include nova::params + $libvirt_service_name = $nova::params::libvirt_service_name + $libvirt_package_name = $nova::params::libvirt_package_name + + class { 'mellanox_openstack::eswitchd' : + physnet => $physnet, + physifc => $physifc, + } + + class { 'mellanox_openstack::agent' : + physnet => $physnet, + physifc => $physifc, + } + + package { $libvirt_package_name : + ensure => installed + } + + service { $libvirt_service_name : + ensure => running + } + + Package[$libvirt_package_name] -> + Service[$libvirt_service_name] -> + Class['mellanox_openstack::eswitchd'] ~> + Class['mellanox_openstack::agent'] + +} diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/manifests/controller.pp b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/controller.pp new file mode 100644 index 0000000..45a5c5e --- /dev/null +++ b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/controller.pp @@ -0,0 +1,22 @@ +class mellanox_openstack::controller ( + $eswitch_vnic_type, + $eswitch_apply_profile_patch, + $mechanism_drivers, +) { + + include neutron::params + $server_service = $neutron::params::server_service + + neutron_plugin_ml2 { + 'eswitch/vnic_type': value => $eswitch_vnic_type; + 'eswitch/apply_profile_patch': value => $eswitch_apply_profile_patch; + 'ml2/mechanism_drivers': value => "mlnx,${mechanism_drivers}"; + } + + service { $server_service : + ensure => running + } + + Neutron_plugin_ml2 <||> ~> + Service[$server_service] +} diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/manifests/eswitchd.pp b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/eswitchd.pp new file mode 100644 index 0000000..2dd9f64 --- /dev/null +++ b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/eswitchd.pp @@ -0,0 +1,31 @@ +class mellanox_openstack::eswitchd ( + $physnet, + $physifc, +) { + include mellanox_openstack::params + + $package = $::mellanox_openstack::params::eswitchd_package + + mellanox_eswitchd_config { + 'DAEMON/fabrics': value => "${physnet}:${physifc}"; + } + + package { $package : + ensure => installed, + } + + service { 'eswitchd' : + ensure => running, + enable => true, + hasstatus => true, + hasrestart => true, + } + + Package[$package] -> + Mellanox_eswitchd_config <||> ~> + Service['eswitchd'] + + Package[$package] ~> + Service['eswitchd'] + +} diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/manifests/init.pp b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/init.pp new file mode 100644 index 0000000..4a4dde2 --- /dev/null +++ b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/init.pp @@ -0,0 +1,3 @@ +class mellanox_openstack { + +} diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/manifests/iser_rename.pp b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/iser_rename.pp new file mode 100644 index 0000000..0bfec44 --- /dev/null +++ b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/iser_rename.pp @@ -0,0 +1,35 @@ +class mellanox_openstack::iser_rename ( + $storage_parent, + $iser_interface_name, +) { + + $interfaces_path = '/sys/class/net/' + $iser_script_dir = '/opt/iser' + $iser_rename_script = "$iser_script_dir/iser_rename.sh" + + + notify { $storage_parent : } + notify { $iser_interface_name : } + + file { $iser_script_dir: + ensure => directory, + } + + file { $iser_rename_script: + ensure => file, + owner => 'root', + group => 'root', + mode => '500', + content => template('mellanox_openstack/iser_rename.erb'), + } + + exec { 'iser_rename': + command => "bash $iser_rename_script", + unless => "test -f $interfaces_path/$iser_interface_name", + path => ['/usr/bin','/usr/sbin','/bin','/sbin','/usr/local/bin'], + logoutput => true, + require => File[$iser_rename_script], + } + +} + diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/manifests/params.pp b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/params.pp new file mode 100644 index 0000000..6b238ae --- /dev/null +++ b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/params.pp @@ -0,0 +1,23 @@ +class mellanox_openstack::params { + + $eswitchd_package = 'eswitchd' + $filters_dir = '/etc/nova/rootwrap.d' + $filters_file = "${filters_dir}/network.filters" + $mlnx_agent_conf = '/etc/neutron/plugins/mlnx/mlnx_conf.ini' + + case $::osfamily { + 'RedHat': { + $neutron_mlnx_packages = ['openstack-neutron-mellanox'] + $agent_service = 'neutron-mlnx-agent' + $compute_service_name = 'openstack-nova-compute' + $openvswitch_mgmt_service = 'openvswitch' + } + 'Debian': { + $neutron_mlnx_packages = ['neutron-plugin-mlnx','neutron-plugin-mlnx-agent'] + $agent_service = 'neutron-plugin-mlnx-agent' + $compute_service_name = 'nova-compute' + $openvswitch_mgmt_service = 'openvswitch-switch' + } + } + +} diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/metadata.json b/deployment_scripts/puppet/modules/mellanox_openstack/metadata.json new file mode 100644 index 0000000..37f852c --- /dev/null +++ b/deployment_scripts/puppet/modules/mellanox_openstack/metadata.json @@ -0,0 +1,17 @@ +{ + "name": "mellanox-mellanox_openstack", + "version": "1.1.0", + "author": "mellanox", + "summary": "Mellanox module for supporting Mellanox hardware on Mirantis Fuel openstack deployment", + "license": "Apache 2.0", + "source": "", + "issues_url": null, + "project_page": null, + "dependencies": [ + { + "version_range": ">= 1.0.0", + "name": "puppetlabs-stdlib" + } + ] +} + diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/spec/spec_helper.rb b/deployment_scripts/puppet/modules/mellanox_openstack/spec/spec_helper.rb new file mode 100644 index 0000000..5fda588 --- /dev/null +++ b/deployment_scripts/puppet/modules/mellanox_openstack/spec/spec_helper.rb @@ -0,0 +1,17 @@ +dir = File.expand_path(File.dirname(__FILE__)) +$LOAD_PATH.unshift File.join(dir, 'lib') + +require 'mocha' +require 'puppet' +require 'rspec' +require 'spec/autorun' + +Spec::Runner.configure do |config| + config.mock_with :mocha +end + +# We need this because the RAL uses 'should' as a method. This +# allows us the same behaviour but with a different method name. +class Object + alias :must :should +end diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/templates/iser_rename.erb b/deployment_scripts/puppet/modules/mellanox_openstack/templates/iser_rename.erb new file mode 100644 index 0000000..0b127d3 --- /dev/null +++ b/deployment_scripts/puppet/modules/mellanox_openstack/templates/iser_rename.erb @@ -0,0 +1,98 @@ +#!/usr/bin/bash +# Copyright 2015 Mellanox Technologies, Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Constants +readonly SCOPE=`basename $0` +readonly UDEV_FILE='/etc/udev/rules.d/70-persistent-net.rules' + +# Variables +STORAGE_PORT='<%=@storage_parent%>' +ISER_NAME='<%=@iser_interface_name%>' +PARENT_FIRST_VF="/sys/class/net/$STORAGE_PORT/device/virtfn0" + +# This functions print logs to /var/log/messages +function logger_print () { + priority=$1 + msg=$2 + logger -t $SCOPE "$priority: $msg" +} + +# Check that a first probe VF exists +if [ ! -d $PARENT_FIRST_VF ]; then + logger_print error "Did not find probed ports of ${STORAGE_PORT}, skipping rename." + exit 1 +fi + +DEVICES='/sys/class/net/*/device' +SON_BUS=`basename $(readlink /sys/class/net/$STORAGE_PORT/device/virtfn0)` +STORAGE_PORT_NUMBER=`cat /sys/class/net/$STORAGE_PORT/dev_id` + +# Find the probe VF port that fits the storage ports number and BUS +for dev in $DEVICES; do + # Check for correct bus + CANDIDATE_BUS=`readlink -nq $dev`; + if [[ $CANDIDATE_BUS != *$SON_BUS* ]]; then + continue; + fi + + # Check for correct dev_id + CANDIDATE_DIRNAME=`dirname $dev` + PORT_NUMBER=`cat $CANDIDATE_DIRNAME/dev_id` + if [ $PORT_NUMBER = $STORAGE_PORT_NUMBER ]; then + PROBED_DIRNAME=`dirname $dev` + PROBED_PORT_NAME=`basename $PROBED_DIRNAME` + fi +done + +# Verify that we find the appropriate virtual port +if [ -z "$PROBED_PORT_NAME" ]; then + logger_print error "Did not find $STORAGE_PORT_NUMBER probed ports of $STORAGE_PORT, exiting." + exit 1 +fi + +# Verify that udev file exists +if [ ! -r "$UDEV_FILE" ]; then + logger_print error "Did not find $UDEV_FILE to rename iser port." + exit 1 +fi + +# Persistantly rename the matched probed port +if [ $PROBED_PORT_NAME != $ISER_NAME ]; then + #Prepare line for udev + UDEV_LINE="SUBSYSTEM==\"net\", ACTION==\"add\", " + UDEV_LINE+="ATTR{dev_id}==\"$STORAGE_PORT_NUMBER\", KERNELS==\"$SON_BUS\", " + UDEV_LINE+="ATTR{type}==\"1\", KERNEL==\"eth*\", NAME=\"$ISER_NAME\"" + + # Change/add line in udev file + grep $PROBED_PORT_NAME $UDEV_FILE > /dev/null 2>&1 + if [ $? -eq 0 ]; then + OLD_LINE_NUMBER=`grep -n $PROBED_PORT_NAME $UDEV_FILE | cut -d : -f 1` + eval "sed '"$OLD_LINE_NUMBER"d' -i $UDEV_FILE" + fi + echo $UDEV_LINE >> $UDEV_FILE + + # restart OFED modules for udev changes to take effect + modprobe -r mlx4_en && modprobe mlx4_en + if [ $? -ne 0 ]; then + logger_print error "Mellanox drivers restart failed." + exit 1 + fi + logger_print info "Changed probed port name from $PROBED_PORT_NAME to $ISER_NAME." +else + logger_print info "Probed port name is configured properly to $ISER_NAME." +fi + +exit 0 diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/tests/init.pp b/deployment_scripts/puppet/modules/mellanox_openstack/tests/init.pp new file mode 100644 index 0000000..f0cc8e2 --- /dev/null +++ b/deployment_scripts/puppet/modules/mellanox_openstack/tests/init.pp @@ -0,0 +1,12 @@ +# The baseline for module testing used by Puppet Labs is that each manifest +# should have a corresponding test manifest that declares that class or defined +# type. +# +# Tests are then run by using puppet apply --noop (to check for compilation +# errors and view a log of events) or by fully applying the test in a virtual +# environment (to compare the resulting system state to the desired state). +# +# Learn more about module testing here: +# http://docs.puppetlabs.com/guides/tests_smoke.html +# +include mellanox_openstack diff --git a/deployment_scripts/replace_cirros.sh b/deployment_scripts/replace_cirros.sh new file mode 100755 index 0000000..517d140 --- /dev/null +++ b/deployment_scripts/replace_cirros.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Copyright 2015 Mellanox Technologies, Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source ./common +CIRROS_PACKAGE_NAME='cirros-testvm-mellanox' + +function install_cirros() { + if [ $DISTRO == 'redhat' ]; then + yum install -y $CIRROS_PACKAGE_NAME + elif [ $DISTRO == 'ubuntu' ]; then + apt-get install -y $CIRROS_PACKAGE_NAME + fi +} + +ruby ./delete_images.rb && +install_cirros && +ruby /etc/puppet/modules/osnailyfacter/modular/astute/upload_cirros.rb 2>/dev/null +if [ $? -ne 0 ]; then + logger_print error "Replacing Cirros image with Mellanox-Cirros image failed" + exit 1 +else + logger_print info "Cirros image was successfully replaced with Mellanox-Cirros image" + exit 0 +fi diff --git a/deployment_scripts/sriov.sh b/deployment_scripts/sriov.sh new file mode 100755 index 0000000..df34646 --- /dev/null +++ b/deployment_scripts/sriov.sh @@ -0,0 +1,205 @@ +#!/bin/bash -x +# Copyright 2015 Mellanox Technologies, Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +readonly SCRIPT_DIR=$(dirname "$0") +source $SCRIPT_DIR/common +readonly SCRIPT_MODE=$1 +readonly FALLBACK_NUM_VFS=16 +readonly NEW_KERNEL_PARAM="intel_iommu=on" + +function get_port_type() { + if [ $DRIVER == 'mlx4_en' ]; then + port_type=2 + elif [ $DRIVER == 'eth_ipoib' ]; then + port_type=1 + fi + echo $port_type +} + +function get_num_probe_vfs () { + if [ $ISER == true ] && [ $DRIVER == 'mlx4_en' ]; then + probe_vfs=1 + else + probe_vfs=0 + fi + echo $probe_vfs +} + +function calculate_total_vfs () { + # validate num of vfs is an integer, 0 <= num <= 64 + if [ "${USER_NUM_OF_VFS}" -ne "${USER_NUM_OF_VFS}" ] 2>/dev/null || + [ "${USER_NUM_OF_VFS}" -gt ${MAX_VFS} ] || + [ "${USER_NUM_OF_VFS}" -lt ${MIN_VFS} ]; then + logger_print error "Illegal number of VFs ${USER_NUM_OF_VFS}, value + should be an integer between ${MIN_VFS},${MAX_VFS}" + return 1 + fi + num_of_vfs=${USER_NUM_OF_VFS} + if [ $((${USER_NUM_OF_VFS} % 2)) -eq 1 ]; then + let num_of_vfs="${USER_NUM_OF_VFS} + 1" # number of vfs is odd and <= 64, then +1 is legal + fi + echo ${num_of_vfs} +} + +function set_modprobe_file () { + PROBE_VFS=`get_num_probe_vfs` + MLX4_CORE_FILE="/etc/modprobe.d/mlx4_core.conf" + PORT_TYPE=`get_port_type` + MLX4_CORE_STR="options mlx4_core + enable_64b_cqe_eqe=0 + log_num_mgm_entry_size=-1 + port_type_array=${PORT_TYPE},${PORT_TYPE}" + TOTAL_VFS=$1 + if [[ $TOTAL_VFS -gt 0 ]]; then + MLX4_CORE_STR="${MLX4_CORE_STR} num_vfs=${TOTAL_VFS}" + if [[ $PROBE_VFS -gt 0 ]]; then + MLX4_CORE_STR="${MLX4_CORE_STR} probe_vf=${PROBE_VFS}" + fi + fi + echo ${MLX4_CORE_STR} > ${MLX4_CORE_FILE} +} + +function set_kernel_params () { + grub_file=`get_grub_file` + if [ "$DISTRO" == "redhat" ]; then + grub_file='/boot/grub/grub.conf' + kernel_line=`egrep 'kernel\s+/vmlinuz' ${grub_file} | grep -v '#'` + elif [ "$DISTRO" == "ubuntu" ]; then + grub_file='/boot/grub/grub.cfg' + kernel_line=$(echo "$(egrep 'linux\s+/vmlinuz' ${grub_file} | grep -v '#')" | head -1) + fi + + if [[ $? -ne 0 ]]; then + echo "Couldn't find kernel line in grub file" >&2 && return 1 + fi + if ! grep -q ${NEW_KERNEL_PARAM} ${grub_file} ; then + line_num=$(echo "$(grep -n "${kernel_line}" ${grub_file} |cut -f1 -d: )" | head -1) + new_kernel_line="${kernel_line} ${NEW_KERNEL_PARAM}" + # delete original line + sed -i "${line_num}d" ${grub_file} + # insert the corrected line on the same line number + sed -i "${line_num}i\ ${new_kernel_line}" ${grub_file} + fi +} + +function burn_vfs_in_fw () { + total_vfs=$1 + # required for mlxconfig to discover mlnx devices + service openibd start &>/dev/null + service mst start &>/dev/null + devices=$(mst status | grep pciconf | awk '{print $1}') + for dev in $devices; do + logger_print debug "device=$dev" + flint -d $dev dc | grep -i sriov | grep -i -q true &> /dev/null + sriov_enabled=$? + current_num_of_vfs=`flint -d $dev dc | grep -i total_vfs | awk '{print $3}'` + if [ $sriov_enabled -eq 0 ] 2>/dev/null; then + logger_print debug "Detected SR-IOV is already enabled" + else + logger_print debug "Detected SR-IOV is disabled" + fi + if [ "$total_vfs" -ne "$current_num_of_vfs" ] 2>/dev/null; then + logger_print debug "Current allowed number of VFs is ${current_num_of_vfs}, required number is ${total_vfs}" + mlxconfig -y -d $dev s SRIOV_EN=1 NUM_OF_VFS=$total_vfs 2>&1 >/dev/null + if [ $? -ne 0 ]; then + logger_print error "Failed changing number of VFs in FW for HCA ${dev}" + fi + else + logger_print debug "Current number of VFs is correctly set to ${current_num_of_vfs} in FW." + fi + done + service mst stop &>/dev/null +} + +function is_sriov_required () { + [ $SRIOV == true ] || + ( [ $ISER == true ] && [ $DRIVER == 'mlx4_en' ] ) + return $? +} + +function configure_sriov () { + if is_sriov_required; then + # Calculate the total amount of virtual functions, based on user seclection + total_vfs=`calculate_total_vfs` + if [ -z ${total_vfs} ]; then + exit 1 + fi + logger_print info "Configuring ${total_vfs} virtual functions + (only even number is currently supported)" + set_modprobe_file $total_vfs && + set_kernel_params && + burn_vfs_in_fw $total_vfs + return $? + else + logger_print info "Skipping SR-IOV configuration" + return 0 + fi +} + +function validate_sriov () { + logger_print info "Validating SR-IOV is enabled, and the required + amount of virtual functions exist" + # get number of VFs + current_num_vfs=`lspci | grep -i mellanox | grep -i virtual | wc -l` + total_vfs=`calculate_total_vfs` + if [ -z ${total_vfs} ]; then + exit 1 + fi + # check if kernel was loaded with the new parameter + grub_file=`get_grub_file` + grep ${NEW_KERNEL_PARAM} /proc/cmdline + has_kernel_param_status=$? + if [ $has_kernel_param_status -eq 0 ]; then + if [ $current_num_vfs -eq $total_vfs ]; then + logger_print info "Successfully verified SR-IOV is enabled with ${current_num_vfs} VFs" + return 0 + fi + else + logger_print error "Kernel did not come up with the kernel parameter: ${NEW_KERNEL_PARAM}, + SR-IOV configuration failed" + return 1 + fi + + # fallback only if kernel param exists and amount of vfs is not as expcted + logger_print error "Failed , trying to fallback to ${FALLBACK_NUM_VFS}" + set_modprobe_file $FALLBACK_NUM_VFS + service openibd restart &> /dev/null + current_num_vfs=`lspci | grep -i mellanox | grep -i virtual | wc -l` + if [ $current_num_vfs -eq $FALLBACK_NUM_VFS ]; then + logger_print info "Fallback to ${FALLBACK_NUM_VFS} succeeded" + return 0 + else + logger_print error "Failed to configure SR-IOV" + return 1 + fi +} + +################# + +case $SCRIPT_MODE in + 'configure') + configure_sriov + ;; + 'validate') + validate_sriov + ;; + *) + logger_print error "Unsupported execution mode ${SCRIPT_MODE}" + exit 1 + ;; +esac + +exit $? diff --git a/environment_config.yaml b/environment_config.yaml index e88e0c0..c182607 100644 --- a/environment_config.yaml +++ b/environment_config.yaml @@ -1,6 +1,24 @@ attributes: - test: - value: "test" - label: "test" - weight: 1 + sriov: + value: false + label: "Neutron SR-IOV plugin" + weight: 60 + type: "checkbox" + restrictions: + - "settings:common.libvirt_type.value != 'kvm' or not (cluster:net_provider == 'neutron' and networking_parameters:segmentation_type == 'vlan')" + iser: + value: false + label: "iSER protocol for volumes (Cinder)" + description: "High performance block storage: Cinder volumes over iSER protocol (iSCSI over RDMA). This feature requires SR-IOV capabilities in the NIC, and will use a dedicated virtual function for the storage network." + weight: 11 + type: "checkbox" + restrictions: + - "settings:storage.volumes_lvm.value != true or settings:common.libvirt_type.value != 'kvm'" + num_of_vfs: + value: "16" + label: "Number of virtual NICs" + description: "Note that one virtual function will be reserved to the storage network, in case of choosing iSER." + weight: 70 type: "text" + restrictions: + - "settings:mellanox-plugin.sriov.value == false" diff --git a/metadata.yaml b/metadata.yaml index 15bb09b..6cec363 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -1,11 +1,11 @@ # Plugin name -name: mellanox_plugin +name: mellanox-plugin # Human-readable name for your plugin title: Mellanox Openstack features # Plugin version -version: 0.1.1 +version: 0.1.6 # Description description: Enable features over Mellanox hardware @@ -29,12 +29,12 @@ groups: ['storage::cinder', 'hypervisor'] releases: - os: ubuntu version: 2014.2-6.1 - mode: ['ha', 'multinode'] + mode: ['ha'] deployment_scripts_path: deployment_scripts/ repository_path: repositories/ubuntu - os: centos version: 2014.2-6.1 - mode: ['ha', 'multinode'] + mode: ['ha'] deployment_scripts_path: deployment_scripts/ repository_path: repositories/centos diff --git a/pre_build_hook b/pre_build_hook index 0d2b310..ad155ef 100755 --- a/pre_build_hook +++ b/pre_build_hook @@ -29,17 +29,54 @@ function download { PREFIX_URL=mellanox_plugin/bootstrap BUILD_DIR=bootstrap ;; + 'rpm') + PREFIX_URL=mellanox_plugin/repositories/centos/Packages + BUILD_DIR=repositories/centos/Packages + ;; + 'deb') + PREFIX_URL=mellanox_plugin/repositories/ubuntu + BUILD_DIR=repositories/ubuntu + ;; *) echo "Can't download ${FILE_NAME}. File of type ${FILE_TYPE} is not supported." exit 1 esac - wget http://${REPOSITORY_HOST}/${PREFIX_URL}/${FILE_NAME} -P ${PLUGIN_DIR}/${BUILD_DIR} + URL="http://${REPOSITORY_HOST}/${PREFIX_URL}/${FILE_NAME}" + DEST_DIR="${PLUGIN_DIR}/${BUILD_DIR}" + wget ${URL} -P ${DEST_DIR} + if [ $? -ne 0 ]; then + echo "Failed fetching package from: ${URL} to ${DEST_DIR}" >&2 + fi } -# download bootstrap files -rm -rf ${PLUGIN_DIR}/bootstrap/* -for f in `cat ${PLUGIN_DIR}/requirements-bootstrap.txt`; do - download bootstrap $f -done -exit 0 +function get_packages() { + file_type=$1 + directory=$2 + files=$3 + rm -rf $directory + mkdir $directory + for f in $files; do + download $file_type $f + done +} + +rpm_dir="${PLUGIN_DIR}/repositories/centos/Packages/" +rpm_files="cirros-testvm-mellanox-0.3.2-3.el6.x86_64.rpm + eswitchd-0.11-3.el6.x86_64.rpm + mlnx-ofed-fuel-2.3-2.0.6.el6.x86_64.rpm + redhat-rpm-config-9.0.3-42.el6.centos.noarch.rpm" +get_packages "rpm" "$rpm_dir" "$rpm_files" + +deb_dir="${PLUGIN_DIR}/repositories/ubuntu/" +deb_files="cirros-testvm-mellanox_0.3.2-ubuntu3_amd64.deb + eswitchd_0.10-3_amd64.deb + mlnx-ofed-fuel_2.3-2.0.6_amd64.deb" +get_packages "deb" "$deb_dir" "$deb_files" + +bootstrap_dir="${PLUGIN_DIR}/bootstrap/" +bootstrap_files="initramfs.img + linux + .ofed + .kernel" +get_packages "bootstrap" "$bootstrap_dir" "$bootstrap_files" diff --git a/specs/post.spec b/specs/post.spec index 36d8b51..b4d690e 100644 --- a/specs/post.spec +++ b/specs/post.spec @@ -5,13 +5,13 @@ if [ -d "/var/www/nailgun/bootstrap/" ]; then mkdir -p /opt/old_bootstrap_image/ mv /var/www/nailgun/bootstrap/* /opt/old_bootstrap_image/ fi - \cp $(ls /var/www/nailgun/plugins/mellanox_plugin*/bootstrap/initramfs.img) /var/www/nailgun/bootstrap/ - \cp $(ls /var/www/nailgun/plugins/mellanox_plugin*/bootstrap/linux) /var/www/nailgun/bootstrap/ + \cp $(ls /var/www/nailgun/plugins/mellanox-plugin*/bootstrap/initramfs.img) /var/www/nailgun/bootstrap/ + \cp $(ls /var/www/nailgun/plugins/mellanox-plugin*/bootstrap/linux) /var/www/nailgun/bootstrap/ command -v dockerctl >/dev/null 2>&1 if [ $? -eq 0 ];then dockerctl copy /var/www/nailgun/bootstrap/initramfs.img cobbler:/var/lib/tftpboot/images/bootstrap/initramfs.img dockerctl copy /var/www/nailgun/bootstrap/linux cobbler:/var/lib/tftpboot/images/bootstrap/linux - \cp $(ls /var/www/nailgun/plugins/mellanox_plugin*/scripts/reboot_bootstrap_nodes) /sbin/ + \cp $(ls /var/www/nailgun/plugins/mellanox-plugin*/scripts/reboot_bootstrap_nodes) /sbin/ echo " `tput bold`Bootstrap discovery image has been replaced for detecting Mellanox Infiniband HW." echo " please reboot your old bootstrap nodes ('reboot_bootstrap_nodes [environment_id|--help]' can be used).`tput sgr0`" fi diff --git a/tasks.yaml b/tasks.yaml index 7e96396..7b4bad2 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -1,6 +1,97 @@ +# Log a notice about pre_deployment start - role: '*' stage: pre_deployment type: shell parameters: - cmd: echo all > /tmp/plugin.all - timeout: 42 + cmd: ./log_stage.sh pre_deployment + timeout: 5 +# This is a workaround: during the plugin pre_deployment stage +# there is no symbolic link from astute.yaml to .yaml. +# Since the data that the plugin uses is common to all .yaml files, +# this script links astute.yaml to any .yaml on each node. +- role: '*' + stage: pre_deployment + type: shell + parameters: + cmd: ./link_astute_file.sh + timeout: 5 +# Add relevant settings for Mellanox manifests to mellanox plugin section in +# hiera, to make the data easily accessible and independent of astute.yaml +- role: '*' + stage: pre_deployment + type: shell + parameters: + cmd: ./mellanox_settings.py + timeout: 10 +# Install OFED + FW upgrade +- role: '*' + stage: pre_deployment + type: shell + parameters: + cmd: ./install_ofed.sh + timeout: 1200 +# Configure number of VFs according to the user decision: +# change modprobe file + IOMMU in grub file + change VFs num in FW +- role: '*' + stage: pre_deployment + type: shell + parameters: + cmd: ./sriov.sh configure + timeout: 60 +# Reboot due to OFED installation / IOMMU configuration +- role: '*' + stage: pre_deployment + type: reboot + parameters: + timeout: 420 +# change modprobe file + IOMMU in grub file + change VFs num in FW +- role: '*' + stage: pre_deployment + type: shell + parameters: + cmd: ./sriov.sh validate + timeout: 60 +# Rename iSER interface +- role: '*' + stage: pre_deployment + type: puppet + parameters: + puppet_manifest: puppet/manifests/iser_rename.pp + puppet_modules: puppet/modules:/etc/puppet/modules + timeout: 60 +# Log a notice about post_deployment start +- role: '*' + stage: post_deployment + type: shell + parameters: + cmd: ./log_stage.sh post_deployment + timeout: 5 +# Execute post_deployment manifest for each role +- role: ['controller', 'primary-controller'] + stage: post_deployment + type: puppet + parameters: + puppet_manifest: puppet/manifests/controller.pp + puppet_modules: puppet/modules:/etc/puppet/modules + timeout: 360 +- role: ['compute'] + stage: post_deployment + type: puppet + parameters: + puppet_manifest: puppet/manifests/compute.pp + puppet_modules: puppet/modules:/etc/puppet/modules + timeout: 360 +- role: ['cinder'] + stage: post_deployment + type: puppet + parameters: + puppet_manifest: puppet/manifests/cinder.pp + puppet_modules: puppet/modules:/etc/puppet/modules + timeout: 360 +# Override the testvm with Mellanox Cirros TestVM +- role: ['controller', 'primary-controller'] + stage: post_deployment + type: shell + parameters: + cmd: ./replace_cirros.sh + timeout: 180