airship/vino
Code Issues Proposed changes

support single libvirt network

Browse Source 
Change-Id: If632341dba45a8cc063658ce443ea676579ffe4e
This commit is contained in:
Kostiantyn Kalynovskyi 2021-06-09 12:48:44 +00:00
parent 5d70de0f3d
commit 728741dbd1
18 changed files with 289 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
config/crd/bases/airship.airshipit.org_vinoes.yaml
View File

@ -90,6 +90,12 @@ spec:
items:
type: string
type: array
instanceSubnet:
type: string
libvirtTemplate:
description: LibvirtTemplate identifies which libvirt template
to be used to create a network
type: string
macPrefix:
description: MACPrefix defines the zero-padded MAC prefix to
use for VM mac addresses, and is the first address that will
@ -100,6 +106,10 @@ spec:
name:
description: Network Parameter defined
type: string
physicalInterface:
description: PhysicalInterface identifies interface into which
to plug in libvirt network
type: string
routes:
items:
description: VMRoutes defined
@ -147,6 +157,10 @@ spec:
that will be created These labels will override keys from
k8s node, that are specified in vino.NodeLabelKeysToCopy
type: object
bootInterfaceName:
description: BootInterfaceName interface name to use to boot
virtual machines
type: string
count:
type: integer
diskDrives:

14
config/manager/flavor-templates.yaml
View File

@ -18,7 +18,7 @@ flavorTemplates:
</memoryBacking>
{% endif %}
<vcpu placement="static">{{ flavors.master.vcpus }}</vcpu>
{% if node_core_map[domain.name] is defined %}
{% if domain.name in node_core_map %}
# function to produce list of cpus, in same numa (controled by bool), state will need to be tracked via file on hypervisor host. gotpl psudo:
<cputune>
<shares>8192</shares>
@ -68,17 +68,11 @@ flavorTemplates:
<alias name="ide"/>
</controller>
<interface type='network'>
<source network='pxe'/>
<mac address='{{ domain.bootMACAddress }}'/>
<model type='virtio'/>
</interface>
# for each interface defined in vino, e.g.
{% for interface in domain.interfaces %}
<interface type='bridge'>
<interface type='{{ interface.type }}'>
<mac address='{{ interface.macAddress }}'/>
<source bridge='{{ interface.network }}'/>
<source {{ interface.type }}='{{ interface.network }}'/>
<model type='virtio'/>
</interface>
{% endfor %}
@ -138,7 +132,7 @@ flavorTemplates:
</memoryBacking>
{% endif %}
<vcpu placement="static">{{ flavors.worker.vcpus }}</vcpu>
{% if node_core_map[domain.name] is defined %}
{% if domain.name in node_core_map %}
# function to produce list of cpus, in same numa (controled by bool), state will need to be tracked via file on hypervisor host. gotpl psudo:
<cputune>
<shares>8192</shares>

12
config/manager/network-templates.yaml
View File

@ -1,14 +1,14 @@
libvirtNetworks:
- name: pxe
management:
libvirtTemplate: |
<network>
<name>pxe</name>
<forward mode='nat'/>
<bridge name='pxe' stp='off' delay='0'/>
<ip address='10.153.241.1' netmask='255.255.255.0'>
<name>{{ network.name }}</name>
<forward mode='route'/>
<bridge name='vm-infra-bridge' stp='off' delay='0' {% if network.physicalInterface is defined %} dev='{{ network.physicalInterface }}' {% endif %}/>
<ip address='{{ ipam.bridge_ip | default(omit) }}' netmask='{{ ipam.bridge_subnet_netmask }}'>
<!-- <tftp root='/srv/tftp'/> -->
<dhcp>
<range start='10.153.241.2' end='10.153.241.254'/>
<range start='{{ ipam.instance_ips[0] }}' end='{{ ipam.instance_ips[-1] }}'/>
<bootp file='http://{{ pxeBootImageHost | default(ansible_default_ipv4.address) }}:{{ pxeBootImageHostPort | default(80) }}/dualboot.ipxe'/>
</dhcp>
</ip>

13
config/samples/vino_cr.yaml
View File

@ -4,7 +4,7 @@ metadata:
name: vino-test-cr
labels: {}
spec:
vmBridge: vm-infra
vmBridge: vm-infra-bridge
nodeLabelKeysToCopy:
- "airshipit.org/server"
- "airshipit.org/rack"
@ -14,17 +14,16 @@ spec:
configuration:
cpuExclude: 0-1
networks:
- name: vm-infra
- name: management
libvirtTemplate: management
subnet: 192.168.2.0/20
instanceSubnet: 192.168.2.0/22
type: bridge
allocationStart: 192.168.2.10
allocationStop: 192.168.2.14 # docs should specify that the range should = number of vms (to permit future expansion over multiple vino crs etc)
routes:
- network: 10.0.0.0
netmask: 255.255.255.0
gateway: $vinobridge # vino will need to populate this from the nodelabel value `airshipit.org/vino.nodebridgegw`
dns_servers: ["135.188.34.124"]
macPrefix: "52:54:00:06:00:00"
physicalInterface: enp3s7
nodes:
- name: master
count: 1
@ -37,7 +36,7 @@ spec:
networkInterfaces:
- name: management
type: network
network: vm-infra
network: management
mtu: 1500
bmcCredentials:
username: admin

13
config/samples/vino_cr_4_workers_1_cp.yaml
View File

@ -4,7 +4,7 @@ metadata:
name: vino-test-cr
labels: {}
spec:
vmBridge: vm-infra
vmBridge: vm-infra-bridge
nodeLabelKeysToCopy:
- "airshipit.org/server"
- "airshipit.org/rack"
@ -14,8 +14,10 @@ spec:
configuration:
cpuExclude: 0-1
networks:
- name: vm-infra
- name: management
libvirtTemplate: management
subnet: 192.168.2.0/20
instanceSubnet: 192.168.2.0/22
type: ipv4
allocationStart: 192.168.2.10
allocationStop: 192.168.2.14 # docs should specify that the range should = number of vms (to permit future expansion over multiple vino crs etc)
@ -25,6 +27,7 @@ spec:
gateway: $vinobridge # vino will need to populate this from the nodelabel value `airshipit.org/vino.nodebridgegw`
dns_servers: ["135.188.34.124"]
macPrefix: "52:54:00:06:00:00"
physicalInterface: enp3s7
nodes:
- name: master
count: 1
@ -33,10 +36,11 @@ spec:
networkDataTemplate:
name: "test-template"
namespace: "default"
bootInterfaceName: management
networkInterfaces:
- name: management
type: network
network: vm-infra
network: management
mtu: 1500
- name: worker
count: 4
@ -45,10 +49,11 @@ spec:
networkDataTemplate:
name: "test-template"
namespace: "default"
bootInterfaceName: management
networkInterfaces:
- name: management
type: network
network: vm-infra
network: management
mtu: 1500
bmcCredentials:
username: admin

69
docs/api/vino.md
View File

@ -159,18 +159,6 @@ int
</tr>
<tr>
<td>
<code>nodes</code><br>
<em>
<a href="#airship.airshipit.org/v1.NodeSet">
[]NodeSet
</a>
</em>
</td>
<td>
</td>
</tr>
<tr>
<td>
<code>configuration</code><br>
<em>
<a href="#airship.airshipit.org/v1.CPUConfiguration">
@ -194,6 +182,16 @@ CPUConfiguration
<td>
</td>
</tr>
<tr>
<td>
<code>nodeCount</code><br>
<em>
int
</em>
</td>
<td>
</td>
</tr>
</tbody>
</table>
</div>
@ -307,6 +305,9 @@ NetworkInterface
</em>
</td>
<td>
<p>
(Members of <code>NetworkInterface</code> are embedded into this type.)
</p>
</td>
</tr>
</tbody>
@ -810,6 +811,16 @@ string
</tr>
<tr>
<td>
<code>instanceSubnet</code><br>
<em>
string
</em>
</td>
<td>
</td>
</tr>
<tr>
<td>
<code>type</code><br>
<em>
string
@ -876,6 +887,28 @@ The prefix should be specified in full MAC notation, e.g.
06:42:42:00:00:00</p>
</td>
</tr>
<tr>
<td>
<code>physicalInterface</code><br>
<em>
string
</em>
</td>
<td>
<p>PhysicalInterface identifies interface into which to plug in libvirt network</p>
</td>
</tr>
<tr>
<td>
<code>libvirtTemplate</code><br>
<em>
string
</em>
</td>
<td>
<p>LibvirtTemplate identifies which libvirt template to be used to create a network</p>
</td>
</tr>
</tbody>
</table>
</div>
@ -989,7 +1022,6 @@ map[string]string
</h3>
<p>
(<em>Appears on:</em>
<a href="#airship.airshipit.org/v1.Builder">Builder</a>,
<a href="#airship.airshipit.org/v1.VinoSpec">VinoSpec</a>)
</p>
<p>NodeSet node definitions</p>
@ -1097,6 +1129,17 @@ string
default is /dev/vda</p>
</td>
</tr>
<tr>
<td>
<code>bootInterfaceName</code><br>
<em>
string
</em>
</td>
<td>
<p>BootInterfaceName interface name to use to boot virtual machines</p>
</td>
</tr>
</tbody>
</table>
</div>

2
pkg/api/v1/vino_builder.go
View File

@ -23,10 +23,10 @@ type Builder struct {
PXEBootImageHostPort int `json:"pxeBootImageHostPort,omitempty"`
Networks []Network `json:"networks,omitempty"`
Nodes []NodeSet `json:"nodes,omitempty"`
// (TODO) change json tag to cpuConfiguration when vino-builder has these chanages as well
CPUConfiguration CPUConfiguration `json:"configuration,omitempty"`
Domains []BuilderDomain `json:"domains,omitempty"`
NodeCount int `json:"nodeCount,omitempty"`
}
type BuilderNetworkInterface struct {

7
pkg/api/v1/vino_types.go
View File

@ -101,6 +101,7 @@ type Network struct {
//Network Parameter defined
Name string `json:"name,omitempty"`
SubNet string `json:"subnet,omitempty"`
InstanceSubnet string `json:"instanceSubnet,omitempty"`
Type string `json:"type,omitempty"`
AllocationStart string `json:"allocationStart,omitempty"`
AllocationStop string `json:"allocationStop,omitempty"`
@ -113,6 +114,10 @@ type Network struct {
// The prefix should be specified in full MAC notation, e.g.
// 06:42:42:00:00:00
MACPrefix string `json:"macPrefix,omitempty"`
// PhysicalInterface identifies interface into which to plug in libvirt network
PhysicalInterface string `json:"physicalInterface,omitempty"`
// LibvirtTemplate identifies which libvirt template to be used to create a network
LibvirtTemplate string `json:"libvirtTemplate,omitempty"`
}
// VMRoutes defined
@ -138,6 +143,8 @@ type NodeSet struct {
// RootDeviceName is the root device for underlying VM, /dev/vda for example
// default is /dev/vda
RootDeviceName string `json:"rootDeviceName,omitempty"`
// BootInterfaceName interface name to use to boot virtual machines
BootInterfaceName string `json:"bootInterfaceName,omitempty"`
}
// NamespacedName to be used to spawn VMs

7
pkg/api/v1/zz_generated.deepcopy.go
View File

@ -65,13 +65,6 @@ func (in *Builder) DeepCopyInto(out *Builder) {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.Nodes != nil {
in, out := &in.Nodes, &out.Nodes
*out = make([]NodeSet, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
out.CPUConfiguration = in.CPUConfiguration
if in.Domains != nil {
in, out := &in.Domains, &out.Domains

43
pkg/managers/bmh.go
View File

@ -160,24 +160,7 @@ func (r *BMHManager) requestVMs(ctx context.Context) error {
}
func (r *BMHManager) createIpamNetworks(ctx context.Context, vino *vinov1.Vino) error {
// TODO (kkalynovskyi) these needs to be propagated into network template, and be configurable
// TODO (kkalynovskyi) develop generic network templates that would allow to handle all networks
// in single generic way.
// Bootnetwork needs to be handled spearately because it needs to be created by libvirt
// And have different configuration.
if r.BootNetwork == nil {
r.BootNetwork = &vinov1.Network{
SubNet: "10.153.241.0/24",
AllocationStart: "10.153.241.2",
AllocationStop: "10.153.241.254",
Name: "pxe-boot",
MACPrefix: "52:54:00:32:00:00",
}
}
networks := vino.Spec.Networks
// Append bootnetwork to be created in IPAM
networks = append(networks, *r.BootNetwork)
for _, network := range networks {
for _, network := range vino.Spec.Networks {
if err := r.createIpamNetwork(ctx, network); err != nil {
return err
}
@ -280,9 +263,9 @@ func (r *BMHManager) setBMHs(ctx context.Context, pod corev1.Pod) error {
PXEBootImageHost: r.ViNO.Spec.PXEBootImageHost,
PXEBootImageHostPort: r.ViNO.Spec.PXEBootImageHostPort,
Networks: r.ViNO.Spec.Networks,
Nodes: r.ViNO.Spec.Nodes,
CPUConfiguration: r.ViNO.Spec.CPUConfiguration,
Domains: domains,
NodeCount: len(r.ViNO.Spec.Nodes),
}
return r.annotateNode(ctx, k8sNode, vinoBuilder)
}
@ -312,7 +295,7 @@ func (r *BMHManager) domainSpecificNetValues(
node vinov1.NodeSet,
networks []vinov1.Network) (networkTemplateValues, error) {
// Allocate an IP for each of this BMH's network interfaces
bootMAC := ""
domainInterfaces := []vinov1.BuilderNetworkInterface{}
for _, iface := range node.NetworkInterfaces {
networkName := iface.NetworkName
@ -345,12 +328,11 @@ func (r *BMHManager) domainSpecificNetValues(
r.Logger.Info("Got MAC and IP for the network and node",
"MAC", macAddress, "IP", ipAddress, "bmh name", bmhName)
if iface.Name == node.BootInterfaceName {
bootMAC = macAddress
}
}
// Handle bootMAC separately
bootMAC, err := r.generatePXEBootMAC(ctx, bmhName)
if err != nil {
return networkTemplateValues{}, err
}
r.Logger.Info("Got bootMAC address for BMH node", "bmh name", bmhName, "bootMAC", bootMAC)
return networkTemplateValues{
Node: node,
@ -363,17 +345,6 @@ func (r *BMHManager) domainSpecificNetValues(
}, nil
}
func (r *BMHManager) generatePXEBootMAC(ctx context.Context, bmhName string) (string, error) {
subnetRange, err := ipam.NewRange(r.BootNetwork.AllocationStart, r.BootNetwork.AllocationStop)
if err != nil {
return "", err
}
ipAllocatedTo := fmt.Sprintf("%s/%s", bmhName, "pxe-boot")
_, mac, err := r.Ipam.AllocateIP(ctx, r.BootNetwork.SubNet, subnetRange, ipAllocatedTo)
return mac, err
}
func (r *BMHManager) annotateNode(ctx context.Context, k8sNode *corev1.Node, vinoBuilder vinov1.Builder) error {
b, err := yaml.Marshal(vinoBuilder)
if err != nil {

9
tools/deployment/configure-bridges.sh
View File

@ -20,10 +20,15 @@ PXE_NET="172.3.3.0/24"
export DEBCONF_NONINTERACTIVE_SEEN=true
export DEBIAN_FRONTEND=noninteractive
sudo modprobe dummy
sudo ip link add enp3s7 type dummy
sudo ip addr add dev enp3s7 "192.168.2.1/24"
sudo ip link set up dev enp3s7
sudo -E apt-get update
sudo -E apt-get install -y bridge-utils
echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward
create_bridge ${VM_INFRA_BRIDGE} ${VM_INFRA_BRIDGE_IP}
create_bridge ${VM_PXE_BRIDGE} ${VM_PXE_BRIDGE_IP}
create_bridge ${VM_PXE_BRIDGE} ${VM_PXE_BRIDGE_IP}

96
vino-builder/assets/playbooks/roles/ipam/library/ipam.py Normal file
View File

@ -0,0 +1,96 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# generate_baremetal_macs method ripped from
# openstack/tripleo-incubator/scripts/configure-vm
import math
import random
import sys
import fnmatch
import os
import socket
import struct
from itertools import chain
import json
import netaddr
DOCUMENTATION = '''
---
module: ipam
version_added: "1.0"
short_description: Help with IPAM allocation
description:
- Generate IPs for instances, ensuring they're unique on every node
'''
# we don't support specifying subnet_bridge or instances yet
def allocate_ips(nodes, physical_primary_ip, physical_node_count=1, subnet_bridge='192.168.0.0/24', subnet_instances='192.168.4.0/22'):
"""Return IP assignments"""
# calculate some stuff
vm_instance_count = len(nodes)
last_octet = physical_primary_ip.split('.')[-1]
node_index = int(last_octet) % int(physical_node_count)
bridge_ip = netaddr.IPNetwork(subnet_bridge)[node_index+1]
# generate an ip for every vm in the entire environment
ip_buckets=[None] * physical_node_count
vm_ip_list = list(netaddr.IPNetwork(subnet_instances))
vm_ip_list.reverse()
# throw away 0, .1, .2, .3 - assumes we won't exceed .255
vm_ip_list.pop()
vm_ip_list.pop()
vm_ip_list.pop()
vm_ip_list.pop()
# now take IPs from this list - enough for all the VMs
# we need to create and place them into groups
# one for each physical node
for physnode in range(0, physical_node_count):
ip_buckets[physnode] = {}
ip_list = []
for vmidx in range(0, vm_instance_count):
ip_list.append(vm_ip_list.pop().__str__())
ip_buckets[physnode] = ip_list
bridge_subnet_netmask = cidr_to_netmask(subnet_bridge)
return {
'node_index': node_index,
'bridge_ip': bridge_ip.__str__(),
'instance_ips': ip_buckets[node_index],
'bridge_subnet_netmask': bridge_subnet_netmask,
}
def cidr_to_netmask(cidr):
_, net_bits = cidr.split('/')
host_bits = 32 - int(net_bits)
netmask = socket.inet_ntoa(struct.pack('!I', (1 << 32) - (1 << host_bits)))
return netmask
def main():
module = AnsibleModule(
argument_spec=dict(
nodes=dict(required=True, type='list'),
physical_node_count=dict(required=True, type='int'),
primary_ipaddress=dict(required=True, type='str'),
subnet_bridge=dict(required=True, type='str'),
subnet_instances=dict(required=True, type='str'),
)
)
result = allocate_ips(module.params["nodes"],
module.params["primary_ipaddress"],
module.params["physical_node_count"],
module.params["subnet_bridge"],
module.params["subnet_instances"])
module.exit_json(**result)
# see http://docs.ansible.com/developing_modules.html#common-module-boilerplate
from ansible.module_utils.basic import AnsibleModule # noqa
if __name__ == '__main__':
main()

38
vino-builder/assets/playbooks/roles/ipam/tasks/main.yaml Normal file
View File

@ -0,0 +1,38 @@
##################################################
# REMOVE ONCE OPERATOR IS SUPPLYING THIS DATA ##
##################################################
- name: initialize ipam
set_fact:
ipam: {}
# get the subnet in yaml - the network named 'management'
# is special
- name: discover management subnet from network definitions
set_fact:
management_subnet: "{{ item }}"
when: item.name == 'management'
loop: "{{ networks }}"
- name: print value of management subnet
debug:
msg: "Value of management subnet is {{ management_subnet }}"
# get our ip from admin interface which is always our default route
- name: discover assigned address of this machines management interface
set_fact:
primary_ip: "{{ ansible_default_ipv4.address|default(ansible_all_ipv4_addresses[0]) }}"
- name: ipam allocation
ipam:
nodes: "{{ domains }}"
primary_ipaddress: "{{ primary_ip }}"
physical_node_count: "{{ nodeCount | int }}"
subnet_bridge: "{{ management_subnet.subnet }}"
subnet_instances: "{{ management_subnet.instanceSubnet }}"
register: ipam
when: domains
- name: debug ipam result
debug:
msg: "IPAM Result {{ ipam }}"

3
vino-builder/assets/playbooks/roles/libvirt/defaults/main.yml
View File

@ -1 +1,2 @@
libvirt_uri: qemu:///system
libvirt_uri: qemu:///system
node_core_map: {}

55
vino-builder/assets/playbooks/roles/libvirt/library/core_allocation.py
View File

@ -96,40 +96,39 @@ def allocate_cores(nodes, flavors, exclude_cpu):
# address the case where previous != desired - delete previous, re-run
for node in nodes:
flavor = node['bmhLabels']['airshipit.org/k8s-role']
flavor = node["role"]
vcpus = flavors[flavor]['vcpus']
for num_node in range(0, node['count']):
# generate a unique name such as master-0, master-1
node_name = node['name'] + '-' + str(num_node)
# generate a unique name such as master-0, master-1
node_name = node["name"]
# extract the core count
core_count = int(vcpus)
# extract the core count
core_count = int(vcpus)
# discover any previous allocation
if 'assignments' in core_state:
if node_name in core_state['assignments']:
if len(core_state['assignments'][node_name]) == core_count:
continue
else:
# TODO: support releasing the cores and adding them back
# to available
raise Exception("Existing assignment exists for node %s but does not match current core count needed" % node_name)
# allocate the cores
allocated=False
for numa in core_state['available']:
if core_count <= len(core_state['available'][numa]):
allocated=True
cores_to_use = core_state['available'][numa][:core_count]
core_state['assignments'][node_name] = cores_to_use
core_state['available'][numa] = core_state['available'][numa][core_count:]
break
else:
# discover any previous allocation
if 'assignments' in core_state:
if node_name in core_state['assignments']:
if len(core_state['assignments'][node_name]) == core_count:
continue
if not allocated:
raise Exception("Unable to find sufficient cores (%s) for node %s (available was %r)" % (core_count, node_name, core_state['available']))
else:
# TODO: support releasing the cores and adding them back
# to available
raise Exception("Existing assignment exists for node %s but does not match current core count needed" % node_name)
# allocate the cores
allocated=False
for numa in core_state['available']:
if core_count <= len(core_state['available'][numa]):
allocated=True
cores_to_use = core_state['available'][numa][:core_count]
core_state['assignments'][node_name] = cores_to_use
core_state['available'][numa] = core_state['available'][numa][core_count:]
break
else:
continue
if not allocated:
raise Exception("Unable to find sufficient cores (%s) for node %s (available was %r)" % (core_count, node_name, core_state['available']))
# return a dict of nodes: cores
# or error if insufficient

2
vino-builder/assets/playbooks/roles/libvirt/tasks/create-network.yaml
View File

@ -17,7 +17,7 @@
state: present
# looks like setting name here is a redundant, the name is anyways taken from the template xml file, but should set it to make virt_pool module happy.
name: "{{ network.name }}"
xml: "{{ network.libvirtTemplate }}"
xml: "{{ libvirtNetworks[network.libvirtTemplate].libvirtTemplate }}"
uri: "{{ libvirt_uri }}"
when: "network.name not in ansible_libvirt_networks"

7
vino-builder/assets/playbooks/roles/libvirt/tasks/main.yml
View File

@ -12,7 +12,7 @@
- name: create network
include_tasks: create-network.yaml
loop: "{{ libvirtNetworks }}"
loop: "{{ networks }}"
loop_control:
loop_var: network
@ -22,15 +22,14 @@
- name: allocate domain cores
core_allocation:
nodes: "{{ nodes }}"
nodes: "{{ domains }}"
flavors: "{{ flavors }}"
exclude_cpu: "{{ configuration.cpuExclude }}"
register: node_core_map
when: nodes
- name: debug print node_core_map
debug:
msg: "node_core_map = {{ node_core_map }}"
var: node_core_map
- name: define domain outer loop
include_tasks: create-domain.yaml

6
vino-builder/assets/playbooks/vino-builder.yaml
View File

@ -37,9 +37,13 @@
---
- hosts: localhost
tasks:
# generate libvirt definitions for storage, networks, and domains
- name: generate management network ip addresses
include_role:
name: ipam
# generate libvirt definitions for storage, networks, and domains
- name: process libvirt definitions
include_role: