From d5533d3deebc80d432726fb2e83e31145e9c8748 Mon Sep 17 00:00:00 2001 From: Steve Baker Date: Tue, 3 Nov 2015 10:24:14 +1300 Subject: [PATCH] Allow a single BMC to manage multiple BMs This change creates a single BMC server and uses that to manage multiple baremetal servers. It does this by using a naming convention on the BM and BMC neutron ports and using that to match a BMC port to a BM nova server. In the BMC boot script a separate systemd unit is created per network interface, and the IP address of that interface is used to bind to instead of ::. This requires a small change to pyghmi which has been proposed upstream Ice82bf788269a31819e4495e15b9ce19a1dd057b. Until this lands upstream pyghmi is installed from my github fork. Other approaches were considered: - multiple neutron ports, single openstackbmc bound to ::, this would require switching the pyghmi socket recv function to recvmsg to get the receiving IP address, but recvmsg isn't available in python 2.7 - single neutron port, single openstackbmc, modified pyghmi to select based on different bmc username/password. This looks impractical based on the assumptions the pyghmi auth code makes. - single neutron port, multiple openstackbmc units each bound to a different UDP port other than 623. Impractical without many changes to Ironic which assumes 623. --- bin/build-nodes-json | 35 ++++++----- bin/install_openstackbmc.sh | 58 ++++++++++++++++++ bin/openstackbmc | 9 ++- templates/virtual-baremetal-servers.yaml | 77 ++++++++---------------- templates/virtual-baremetal.yaml | 22 +++++++ 5 files changed, 132 insertions(+), 69 deletions(-) create mode 100644 bin/install_openstackbmc.sh diff --git a/bin/build-nodes-json b/bin/build-nodes-json index bd197f4..b0a1c62 100755 --- a/bin/build-nodes-json +++ b/bin/build-nodes-json @@ -5,6 +5,7 @@ import json import os import sys +from neutronclient.v2_0 import client as neutronclient from novaclient import client as novaclient def main(): @@ -51,25 +52,31 @@ def main(): } if not username or not password or not tenant or not auth_url: - print 'Source an appropriate rc file first' + print('Source an appropriate rc file first') sys.exit(1) nova = novaclient.Client(2, username, password, tenant, auth_url) + neutron = neutronclient.Client( + username=username, + password=password, + tenant_name=tenant, + auth_url=auth_url + ) - bmcs = nova.servers.list(search_opts={'name': bmc_base + '_.*'}) - baremetals = nova.servers.list(search_opts={'name': baremetal_base + '_.*'}) - bmcs = sorted(bmcs, key=lambda x: x.name) - baremetals = sorted(baremetals, key=lambda x: x.name) - if len(bmcs) != len(baremetals): - raise RuntimeException('Found different numbers of baremetal and ' - 'bmc vms.') + all_ports = sorted(neutron.list_ports()['ports'], key=lambda x: x['name']) + bmc_ports = list([p for p in all_ports + if p['name'].startswith(bmc_base)]) + bm_ports = list([p for p in all_ports + if p['name'].startswith(baremetal_base)]) + if len(bmc_ports) != len(bm_ports): + raise RuntimeError('Found different numbers of baremetal and ' + 'bmc ports.') nodes = [] - for pair in zip(bmcs, baremetals): - bmc = pair[0] - baremetal = pair[1] + for bmc_port, baremetal_port in zip(bmc_ports, bm_ports): + baremetal = nova.servers.get(baremetal_port['device_id']) node = dict(node_template) - node['pm_addr'] = bmc.addresses[private_net][0]['addr'] + node['pm_addr'] = bmc_port['fixed_ips'][0]['ip_address'] node['mac'] = [baremetal.addresses[provision_net][0]['OS-EXT-IPS-MAC:mac_addr']] flavor = nova.flavors.get(baremetal.flavor['id']) node['cpu'] = flavor.vcpus @@ -78,9 +85,9 @@ def main(): nodes.append(node) with open('nodes.json', 'w') as node_file: - contents = json.dumps({'nodes': nodes}) + contents = json.dumps({'nodes': nodes}, indent=2) node_file.write(contents) - print contents + print(contents) if __name__ == '__main__': main() diff --git a/bin/install_openstackbmc.sh b/bin/install_openstackbmc.sh new file mode 100644 index 0000000..c3ae3e7 --- /dev/null +++ b/bin/install_openstackbmc.sh @@ -0,0 +1,58 @@ +#!/bin/bash +set -x + +yum -y update centos-release # required for rdo-release install to work +yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm +yum install -y https://rdo.fedorapeople.org/rdo-release.rpm +yum install -y python-pip python-crypto os-net-config python-novaclient python-neutronclient git jq +pip install --no-deps git+https://github.com/steveb/pyghmi.git@custom-bind-address + +cat </usr/local/bin/openstackbmc +$openstackbmc_script +EOF +chmod +x /usr/local/bin/openstackbmc + +mkdir /etc/os-net-config +echo "network_config:" > /etc/os-net-config/config.yaml + +for i in $(seq 2 $bm_node_count) +do + nic="eth$(($i-1))" + echo "- {name: $nic, type: interface, use_dhcp: true, defroute: no}" >> /etc/os-net-config/config.yaml +done +os-net-config --verbose + +export OS_USERNAME=$os_user +export OS_TENANT_NAME=$os_tenant +export OS_PASSWORD=$os_password +export OS_AUTH_URL=$os_auth_url + +for i in $(seq 1 $bm_node_count) +do + bm_port="$bm_prefix_$(($i-1))" + bm_instance=$(neutron port-show $bm_port -c device_id -f value) + bmc_port="$bmc_prefix_$(($i-1))" + bmc_ip=$(neutron port-show $bmc_port -c fixed_ips -f value | jq -r .ip_address) + unit="openstack-bmc-$bm_port.service" + + cat </usr/lib/systemd/system/$unit +[Unit] +Description=openstack-bmc Service + +[Service] +ExecStart=/usr/local/bin/openstackbmc --os-user $os_user --os-password $os_password --os-tenant $os_tenant --os-auth-url $os_auth_url --instance $bm_instance --address $bmc_ip + +User=root +StandardOutput=kmsg+console +StandardError=inherit + +[Install] +WantedBy=multi-user.target +Alias=openstack-bmc.service +EOF + + systemctl enable $unit + systemctl start $unit + systemctl status $unit +done + diff --git a/bin/openstackbmc b/bin/openstackbmc index 4becacb..5e3c4e7 100755 --- a/bin/openstackbmc +++ b/bin/openstackbmc @@ -33,9 +33,9 @@ import pyghmi.ipmi.bmc as bmc class OpenStackBmc(bmc.Bmc): - def __init__(self, authdata, port, instance, user, password, tenant, + def __init__(self, authdata, port, address, instance, user, password, tenant, auth_url): - super(OpenStackBmc, self).__init__(authdata, port) + super(OpenStackBmc, self).__init__(authdata, port=port, address=address) self.novaclient = novaclient.Client(2, user, password, tenant, auth_url) self.instance = None @@ -135,6 +135,10 @@ if __name__ == '__main__': type=int, default=623, help='Port to listen on; defaults to 623') + parser.add_argument('--address', + dest='address', + default='::', + help='Address to bind to; defaults to ::') parser.add_argument('--instance', dest='instance', required=True, @@ -157,6 +161,7 @@ if __name__ == '__main__': help='The OpenStack Keystone auth url') args = parser.parse_args() mybmc = OpenStackBmc({'admin': 'password'}, port=args.port, + address='::ffff:%s' % args.address, instance=args.instance, user=args.user, password=args.password, diff --git a/templates/virtual-baremetal-servers.yaml b/templates/virtual-baremetal-servers.yaml index cde52f2..553f6a8 100644 --- a/templates/virtual-baremetal-servers.yaml +++ b/templates/virtual-baremetal-servers.yaml @@ -57,79 +57,50 @@ parameters: type: string resources: - OpenStackBMCServer: - type: OS::Nova::Server + + bmc_port: + type: OS::Neutron::Port properties: - flavor: {get_param: bmc_flavor} - image: {get_param: bmc_image} - key_name: {get_param: key_name} - security_groups: - - {get_param: private_sg} - networks: - - network: {get_param: private_net} name: list_join: - '' - - {get_param: bmc_prefix} - {get_param: suffix} - user_data_format: RAW - user_data: - str_replace: - params: - $bm_instance: {get_resource: OpenStackBaremetalServer} - $os_user: {get_param: os_user} - $os_password: {get_param: os_password} - $os_tenant: {get_param: os_tenant} - $os_auth_url: {get_param: os_auth_url} - $openstackbmc_script: {get_file: ../bin/openstackbmc} - template: | - #!/bin/bash - set -x - cat </usr/lib/systemd/system/openstack-bmc.service - [Unit] - Description=openstack-bmc Service + network: {get_param: private_net} + security_groups: + - {get_param: private_sg} - [Service] - ExecStart=/usr/local/bin/openstackbmc --os-user $os_user --os-password $os_password --os-tenant $os_tenant --os-auth-url $os_auth_url --instance $bm_instance - User=root - StandardOutput=kmsg+console - StandardError=inherit + baremetal_port: + type: OS::Neutron::Port + properties: + name: + list_join: + - '' + - - {get_param: baremetal_prefix} + - {get_param: suffix} + network: {get_param: provision_net} + security_groups: + - {get_param: provision_sg} - [Install] - WantedBy=multi-user.target - Alias=openstack-bmc.service - EOF - - cat </usr/local/bin/openstackbmc - $openstackbmc_script - EOF - chmod +x /usr/local/bin/openstackbmc - - yum -y update centos-release # required for rdo-release install to work - yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm - yum install -y https://rdo.fedorapeople.org/rdo-release.rpm - yum install -y python-pip python-crypto python-novaclient - pip install --no-deps "pyghmi==0.8.0" - systemctl enable openstack-bmc - systemctl start openstack-bmc - systemctl status openstack-bmc - - OpenStackBaremetalServer: + baremetal_server: type: OS::Nova::Server properties: flavor: {get_param: baremetal_flavor} image: {get_param: baremetal_image} config_drive: false key_name: {get_param: key_name} - security_groups: - - {get_param: provision_sg} + # security_groups: # TODO(bnemec): Reattach these when it doesn't wreak havoc on TripleO to have multiple nics. #- {get_param: public_sg} networks: - - network: {get_param: provision_net} + - port: {get_resource: baremetal_port} #- network: {get_param: public_net} name: list_join: - '' - - {get_param: baremetal_prefix} - {get_param: suffix} + +outputs: + bmc_nic: + value: {port: {get_resource: bmc_port}} diff --git a/templates/virtual-baremetal.yaml b/templates/virtual-baremetal.yaml index feeb78e..52ebea9 100644 --- a/templates/virtual-baremetal.yaml +++ b/templates/virtual-baremetal.yaml @@ -134,6 +134,28 @@ resources: port_range_min: 623 port_range_max: 623 + bmc_server: + type: OS::Nova::Server + properties: + flavor: {get_param: bmc_flavor} + image: {get_param: bmc_image} + key_name: {get_param: key_name} + networks: {get_attr: [openstack_baremetal_servers, bmc_nic]} + name: {get_param: bmc_prefix} + user_data_format: RAW + user_data: + str_replace: + params: + $os_user: {get_param: os_user} + $os_password: {get_param: os_password} + $os_tenant: {get_param: os_tenant} + $os_auth_url: {get_param: os_auth_url} + $bm_node_count: {get_param: node_count} + $bmc_prefix: {get_param: bmc_prefix} + $bm_prefix: {get_param: baremetal_prefix} + $openstackbmc_script: {get_file: ../bin/openstackbmc} + template: {get_file: ../bin/install_openstackbmc.sh} + openstack_baremetal_servers: type: OS::Heat::ResourceGroup properties: