network validation to ping test each interface

This patches wires in a new "all nodes" validation resource
that can be used to add validations that occur early on
during the deployment process. This occurs after the nodes
have been brought online and the initial networks
have been configured but before any "post" (puppet, etc.)
sort of configuration has been executed.

A initial validation script has been added to ping test network IPs
on each network. When using network isolation this will ensure
network connectivity (vlans, etc) are working on each
node and if not the heat stack will fail early, allowing
time to fix the network connections and retry the
stack creation via an update.

Change-Id: I63cf95b27e8ad2aed48718cf84df5f324780e597
Co-Authored-By: Ian Main <imain@redhat.com>
Co-Authored-By: Ryan Hallisey <rhallise@redhat.com>
This commit is contained in:
Dan Prince 2015-07-22 18:22:50 -04:00
parent 386fc60890
commit a6861730bd
5 changed files with 110 additions and 0 deletions

27
all-nodes-validation.yaml Normal file
View File

@ -0,0 +1,27 @@
heat_template_version: 2015-04-30
description: >
Software Config to drive validations that occur on all nodes.
Note, you need the heat-config-script element built into your
images, due to the script group below.
parameters:
PingTestIps:
default: ''
description: A string containing a space separated list of IP addresses used to ping test each available network interface.
type: string
resources:
AllNodesValidationsImpl:
type: OS::Heat::SoftwareConfig
properties:
group: script
inputs:
- name: ping_test_ips
default: {get_param: PingTestIps}
config: {get_file: ./validation-scripts/all-nodes.sh}
outputs:
OS::stack_id:
description: The ID of the AllNodesValidationsImpl resource.
value: {get_resource: AllNodesValidationsImpl}

View File

@ -74,5 +74,8 @@ resource_registry:
# Port assignments for service virtual IPs for the controller role
OS::TripleO::Controller::Ports::RedisVipPort: network/ports/ctlplane_vip.yaml
# validation resources
OS::TripleO::AllNodes::Validation: all-nodes-validation.yaml
parameter_defaults:
EnablePackageInstall: false

View File

@ -64,3 +64,6 @@ resource_registry:
# Port assignments for service virtual IPs for the controller role
OS::TripleO::Controller::Ports::RedisVipPort: network/ports/noop.yaml
# validation resources
OS::TripleO::AllNodes::Validation: all-nodes-validation.yaml

View File

@ -1226,6 +1226,54 @@ resources:
config: {get_attr: [allNodesConfig, config_id]}
servers: {get_attr: [CephStorage, attributes, nova_server_resource]}
# All Nodes Validations
AllNodesValidationConfig:
type: OS::TripleO::AllNodes::Validation
properties:
PingTestIps:
list_join:
- ' '
- - {get_attr: [Controller, resource.0.external_ip_address]}
- {get_attr: [Controller, resource.0.internal_api_ip_address]}
- {get_attr: [Controller, resource.0.storage_ip_address]}
- {get_attr: [Controller, resource.0.storage_mgmt_ip_address]}
- {get_attr: [Controller, resource.0.tenant_ip_address]}
ControllerAllNodesValidationDeployment:
type: OS::Heat::StructuredDeployments
depends_on: ControllerAllNodesDeployment
properties:
config: {get_resource: AllNodesValidationConfig}
servers: {get_attr: [Controller, attributes, nova_server_resource]}
ComputeAllNodesValidationDeployment:
type: OS::Heat::StructuredDeployments
depends_on: ComputeAllNodesDeployment
properties:
config: {get_resource: AllNodesValidationConfig}
servers: {get_attr: [Compute, attributes, nova_server_resource]}
BlockStorageAllNodesValidationDeployment:
type: OS::Heat::StructuredDeployments
depends_on: BlockStorageAllNodesDeployment
properties:
config: {get_resource: AllNodesValidationConfig}
servers: {get_attr: [BlockStorage, attributes, nova_server_resource]}
ObjectStorageAllNodesValidationDeployment:
type: OS::Heat::StructuredDeployments
depends_on: ObjectStorageAllNodesDeployment
properties:
config: {get_resource: AllNodesValidationConfig}
servers: {get_attr: [ObjectStorage, attributes, nova_server_resource]}
CephStorageAllNodesValidationDeployment:
type: OS::Heat::StructuredDeployments
depends_on: CephStorageAllNodesDeployment
properties:
config: {get_resource: AllNodesValidationConfig}
servers: {get_attr: [CephStorage, attributes, nova_server_resource]}
# Nested stack deployment runs after all other controller deployments
ControllerNodesPostDeployment:
type: OS::TripleO::ControllerPostDeployment

View File

@ -0,0 +1,29 @@
#!/bin/bash
# For each unique remote IP (specified via Heat) we check to
# see if one of the locally configured networks matches and if so we
# attempt a ping test on that networks remote IP.
function ping_controller_ips() {
local REMOTE_IPS=$1
for REMOTE_IP in $(echo $REMOTE_IPS | sed -e "s| |\n|g" | sort -u); do
for LOCAL_NETWORK in $(ip r | grep -v default | cut -d " " -f 1); do
local LOCAL_CIDR=$(echo $LOCAL_NETWORK | cut -d "/" -f 2)
local LOCAL_NETMASK=$(ipcalc -m $LOCAL_NETWORK | grep NETMASK | cut -d "=" -f 2)
local REMOTE_NETWORK=$(ipcalc -np $REMOTE_IP $LOCAL_NETMASK | grep NETWORK | cut -d "=" -f 2)
if [ $REMOTE_NETWORK/$LOCAL_CIDR == $LOCAL_NETWORK ]; then
echo -n "Trying to ping $REMOTE_IP for local network $LOCAL_NETWORK..."
if ! ping -c 1 $REMOTE_IP &> /dev/null; then
echo "FAILURE"
echo "$REMOTE_IP is not pingable. Local Network: $LOCAL_NETWORK" >&2
exit 1
fi
echo "SUCCESS"
fi
done
done
}
ping_controller_ips "$ping_test_ips"