Allow a single BMC to manage multiple BMs

This change creates a single BMC server and uses that to manage multiple baremetal servers. It does this by using a naming convention on the BM and BMC neutron ports and using that to match a BMC port to a BM nova server. In the BMC boot script a separate systemd unit is created per network interface, and the IP address of that interface is used to bind to instead of ::. This requires a small change to pyghmi which has been proposed upstream Ice82bf788269a31819e4495e15b9ce19a1dd057b. Until this lands upstream pyghmi is installed from my github fork. Other approaches were considered: - multiple neutron ports, single openstackbmc bound to ::, this would require switching the pyghmi socket recv function to recvmsg to get the receiving IP address, but recvmsg isn't available in python 2.7 - single neutron port, single openstackbmc, modified pyghmi to select based on different bmc username/password. This looks impractical based on the assumptions the pyghmi auth code makes. - single neutron port, multiple openstackbmc units each bound to a different UDP port other than 623. Impractical without many changes to Ironic which assumes 623.
2015-11-03 10:24:14 +13:00 · 2015-11-03 10:24:14 +13:00 · d5533d3dee
commit d5533d3dee
parent 14b715e109
5 changed files with 132 additions and 69 deletions
--- a/bin/build-nodes-json
+++ b/bin/build-nodes-json
@ -5,6 +5,7 @@ import json
 import os
 import sys

+from neutronclient.v2_0 import client as neutronclient
 from novaclient import client as novaclient

 def main():
@ -51,25 +52,31 @@ def main():
        }

    if not username or not password or not tenant or not auth_url:
-        print 'Source an appropriate rc file first'
+        print('Source an appropriate rc file first')
        sys.exit(1)

    nova = novaclient.Client(2, username, password, tenant, auth_url)
+    neutron = neutronclient.Client(
+        username=username,
+        password=password,
+        tenant_name=tenant,
+        auth_url=auth_url
+    )

-    bmcs = nova.servers.list(search_opts={'name': bmc_base + '_.*'})
-    baremetals = nova.servers.list(search_opts={'name': baremetal_base + '_.*'})
-    bmcs = sorted(bmcs, key=lambda x: x.name)
-    baremetals = sorted(baremetals, key=lambda x: x.name)
-    if len(bmcs) != len(baremetals):
-        raise RuntimeException('Found different numbers of baremetal and '
-                               'bmc vms.')
+    all_ports = sorted(neutron.list_ports()['ports'], key=lambda x: x['name'])
+    bmc_ports = list([p for p in all_ports
+                     if p['name'].startswith(bmc_base)])
+    bm_ports = list([p for p in all_ports
+                    if p['name'].startswith(baremetal_base)])
+    if len(bmc_ports) != len(bm_ports):
+        raise RuntimeError('Found different numbers of baremetal and '
+                           'bmc ports.')
    nodes = []

-    for pair in zip(bmcs, baremetals):
-        bmc = pair[0]
-        baremetal = pair[1]
+    for bmc_port, baremetal_port in zip(bmc_ports, bm_ports):
+        baremetal = nova.servers.get(baremetal_port['device_id'])
        node = dict(node_template)
-        node['pm_addr'] = bmc.addresses[private_net][0]['addr']
+        node['pm_addr'] = bmc_port['fixed_ips'][0]['ip_address']
        node['mac'] = [baremetal.addresses[provision_net][0]['OS-EXT-IPS-MAC:mac_addr']]
        flavor = nova.flavors.get(baremetal.flavor['id'])
        node['cpu'] = flavor.vcpus
@ -78,9 +85,9 @@ def main():
        nodes.append(node)

    with open('nodes.json', 'w') as node_file:
-        contents = json.dumps({'nodes': nodes})
+        contents = json.dumps({'nodes': nodes}, indent=2)
        node_file.write(contents)
-        print contents
+        print(contents)

 if __name__ == '__main__':
    main()
--- a/bin/install_openstackbmc.sh
+++ b/bin/install_openstackbmc.sh
@ -0,0 +1,58 @@
+#!/bin/bash
+set -x
+
+yum -y update centos-release # required for rdo-release install to work
+yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+yum install -y https://rdo.fedorapeople.org/rdo-release.rpm
+yum install -y python-pip python-crypto os-net-config python-novaclient python-neutronclient git jq
+pip install --no-deps git+https://github.com/steveb/pyghmi.git@custom-bind-address
+
+cat <<EOF >/usr/local/bin/openstackbmc
+$openstackbmc_script
+EOF
+chmod +x /usr/local/bin/openstackbmc
+
+mkdir /etc/os-net-config
+echo "network_config:" > /etc/os-net-config/config.yaml
+
+for i in $(seq 2 $bm_node_count)
+do
+    nic="eth$(($i-1))"
+    echo "- {name: $nic, type: interface, use_dhcp: true, defroute: no}" >> /etc/os-net-config/config.yaml
+done
+os-net-config --verbose
+
+export OS_USERNAME=$os_user
+export OS_TENANT_NAME=$os_tenant
+export OS_PASSWORD=$os_password
+export OS_AUTH_URL=$os_auth_url
+
+for i in $(seq 1 $bm_node_count)
+do
+    bm_port="$bm_prefix_$(($i-1))"
+    bm_instance=$(neutron port-show $bm_port -c device_id -f value)
+    bmc_port="$bmc_prefix_$(($i-1))"
+    bmc_ip=$(neutron port-show $bmc_port -c fixed_ips -f value | jq -r .ip_address)
+    unit="openstack-bmc-$bm_port.service"
+
+    cat <<EOF >/usr/lib/systemd/system/$unit
+[Unit]
+Description=openstack-bmc Service
+
+[Service]
+ExecStart=/usr/local/bin/openstackbmc  --os-user $os_user --os-password $os_password --os-tenant $os_tenant --os-auth-url $os_auth_url --instance $bm_instance --address $bmc_ip
+
+User=root
+StandardOutput=kmsg+console
+StandardError=inherit
+
+[Install]
+WantedBy=multi-user.target
+Alias=openstack-bmc.service
+EOF
+
+    systemctl enable $unit
+    systemctl start $unit
+    systemctl status $unit
+done
+
--- a/bin/openstackbmc
+++ b/bin/openstackbmc
@ -33,9 +33,9 @@ import pyghmi.ipmi.bmc as bmc


 class OpenStackBmc(bmc.Bmc):
-    def __init__(self, authdata, port, instance, user, password, tenant,
+    def __init__(self, authdata, port, address, instance, user, password, tenant,
                 auth_url):
-        super(OpenStackBmc, self).__init__(authdata, port)
+        super(OpenStackBmc, self).__init__(authdata, port=port, address=address)
        self.novaclient = novaclient.Client(2, user, password,
                                            tenant, auth_url)
        self.instance = None
@ -135,6 +135,10 @@ if __name__ == '__main__':
                        type=int,
                        default=623,
                        help='Port to listen on; defaults to 623')
+    parser.add_argument('--address',
+                        dest='address',
+                        default='::',
+                        help='Address to bind to; defaults to ::')
    parser.add_argument('--instance',
                        dest='instance',
                        required=True,
@ -157,6 +161,7 @@ if __name__ == '__main__':
                        help='The OpenStack Keystone auth url')
    args = parser.parse_args()
    mybmc = OpenStackBmc({'admin': 'password'}, port=args.port,
+                         address='::ffff:%s' % args.address,
                         instance=args.instance,
                         user=args.user,
                         password=args.password,
--- a/templates/virtual-baremetal-servers.yaml
+++ b/templates/virtual-baremetal-servers.yaml
@ -57,79 +57,50 @@ parameters:
    type: string

 resources:
-  OpenStackBMCServer:
-    type: OS::Nova::Server
+
+  bmc_port:
+    type: OS::Neutron::Port
    properties:
-      flavor: {get_param: bmc_flavor}
-      image: {get_param: bmc_image}
-      key_name: {get_param: key_name}
-      security_groups:
-      - {get_param: private_sg}
-      networks:
-      - network: {get_param: private_net}
      name:
        list_join:
        - ''
        - - {get_param: bmc_prefix}
          - {get_param: suffix}
-      user_data_format: RAW
-      user_data:
-        str_replace:
-          params:
-            $bm_instance: {get_resource: OpenStackBaremetalServer}
-            $os_user: {get_param: os_user}
-            $os_password: {get_param: os_password}
-            $os_tenant: {get_param: os_tenant}
-            $os_auth_url: {get_param: os_auth_url}
-            $openstackbmc_script: {get_file: ../bin/openstackbmc}
-          template: |
-            #!/bin/bash
-            set -x
-            cat <<EOF >/usr/lib/systemd/system/openstack-bmc.service
-            [Unit]
-            Description=openstack-bmc Service
+      network: {get_param: private_net}
+      security_groups:
+      - {get_param: private_sg}

-            [Service]
-            ExecStart=/usr/local/bin/openstackbmc  --os-user $os_user --os-password $os_password --os-tenant $os_tenant --os-auth-url $os_auth_url --instance $bm_instance
-            User=root
-            StandardOutput=kmsg+console
-            StandardError=inherit
+  baremetal_port:
+    type: OS::Neutron::Port
+    properties:
+      name:
+        list_join:
+        - ''
+        - - {get_param: baremetal_prefix}
+          - {get_param: suffix}
+      network: {get_param: provision_net}
+      security_groups:
+      - {get_param: provision_sg}

-            [Install]
-            WantedBy=multi-user.target
-            Alias=openstack-bmc.service
-            EOF
-
-            cat <<EOF >/usr/local/bin/openstackbmc
-            $openstackbmc_script
-            EOF
-            chmod +x /usr/local/bin/openstackbmc
-
-            yum -y update centos-release # required for rdo-release install to work
-            yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-            yum install -y https://rdo.fedorapeople.org/rdo-release.rpm
-            yum install -y python-pip python-crypto python-novaclient
-            pip install --no-deps "pyghmi==0.8.0"
-            systemctl enable openstack-bmc
-            systemctl start openstack-bmc
-            systemctl status openstack-bmc
-
-  OpenStackBaremetalServer:
+  baremetal_server:
    type: OS::Nova::Server
    properties:
      flavor: {get_param: baremetal_flavor}
      image: {get_param: baremetal_image}
      config_drive: false
      key_name: {get_param: key_name}
-      security_groups:
-      - {get_param: provision_sg}
+      # security_groups:
      # TODO(bnemec): Reattach these when it doesn't wreak havoc on TripleO to have multiple nics.
      #- {get_param: public_sg}
      networks:
-      - network: {get_param: provision_net}
+      - port: {get_resource: baremetal_port}
      #- network: {get_param: public_net}
      name:
        list_join:
        - ''
        - - {get_param: baremetal_prefix}
          - {get_param: suffix}
+
+outputs:
+  bmc_nic:
+    value: {port: {get_resource: bmc_port}}
--- a/templates/virtual-baremetal.yaml
+++ b/templates/virtual-baremetal.yaml
@ -134,6 +134,28 @@ resources:
        port_range_min: 623
        port_range_max: 623

+  bmc_server:
+    type: OS::Nova::Server
+    properties:
+      flavor: {get_param: bmc_flavor}
+      image: {get_param: bmc_image}
+      key_name: {get_param: key_name}
+      networks: {get_attr: [openstack_baremetal_servers, bmc_nic]}
+      name: {get_param: bmc_prefix}
+      user_data_format: RAW
+      user_data:
+        str_replace:
+          params:
+            $os_user: {get_param: os_user}
+            $os_password: {get_param: os_password}
+            $os_tenant: {get_param: os_tenant}
+            $os_auth_url: {get_param: os_auth_url}
+            $bm_node_count: {get_param: node_count}
+            $bmc_prefix: {get_param: bmc_prefix}
+            $bm_prefix: {get_param: baremetal_prefix}
+            $openstackbmc_script: {get_file: ../bin/openstackbmc}
+          template: {get_file: ../bin/install_openstackbmc.sh}
+
  openstack_baremetal_servers:
    type: OS::Heat::ResourceGroup
    properties: