+ {{ network.name }}
+
+
+
-
+
diff --git a/config/samples/vino_cr.yaml b/config/samples/vino_cr.yaml
index 4509a44..dfb0b4c 100644
--- a/config/samples/vino_cr.yaml
+++ b/config/samples/vino_cr.yaml
@@ -4,7 +4,7 @@ metadata:
name: vino-test-cr
labels: {}
spec:
- vmBridge: vm-infra
+ vmBridge: vm-infra-bridge
nodeLabelKeysToCopy:
- "airshipit.org/server"
- "airshipit.org/rack"
@@ -14,17 +14,16 @@ spec:
configuration:
cpuExclude: 0-1
networks:
- - name: vm-infra
+ - name: management
+ libvirtTemplate: management
subnet: 192.168.2.0/20
+ instanceSubnet: 192.168.2.0/22
type: bridge
allocationStart: 192.168.2.10
allocationStop: 192.168.2.14 # docs should specify that the range should = number of vms (to permit future expansion over multiple vino crs etc)
- routes:
- - network: 10.0.0.0
- netmask: 255.255.255.0
- gateway: $vinobridge # vino will need to populate this from the nodelabel value `airshipit.org/vino.nodebridgegw`
dns_servers: ["135.188.34.124"]
macPrefix: "52:54:00:06:00:00"
+ physicalInterface: enp3s7
nodes:
- name: master
count: 1
@@ -37,7 +36,7 @@ spec:
networkInterfaces:
- name: management
type: network
- network: vm-infra
+ network: management
mtu: 1500
bmcCredentials:
username: admin
diff --git a/config/samples/vino_cr_4_workers_1_cp.yaml b/config/samples/vino_cr_4_workers_1_cp.yaml
index dfe7be8..19b5b26 100644
--- a/config/samples/vino_cr_4_workers_1_cp.yaml
+++ b/config/samples/vino_cr_4_workers_1_cp.yaml
@@ -4,7 +4,7 @@ metadata:
name: vino-test-cr
labels: {}
spec:
- vmBridge: vm-infra
+ vmBridge: vm-infra-bridge
nodeLabelKeysToCopy:
- "airshipit.org/server"
- "airshipit.org/rack"
@@ -14,8 +14,10 @@ spec:
configuration:
cpuExclude: 0-1
networks:
- - name: vm-infra
+ - name: management
+ libvirtTemplate: management
subnet: 192.168.2.0/20
+ instanceSubnet: 192.168.2.0/22
type: ipv4
allocationStart: 192.168.2.10
allocationStop: 192.168.2.14 # docs should specify that the range should = number of vms (to permit future expansion over multiple vino crs etc)
@@ -25,6 +27,7 @@ spec:
gateway: $vinobridge # vino will need to populate this from the nodelabel value `airshipit.org/vino.nodebridgegw`
dns_servers: ["135.188.34.124"]
macPrefix: "52:54:00:06:00:00"
+ physicalInterface: enp3s7
nodes:
- name: master
count: 1
@@ -33,10 +36,11 @@ spec:
networkDataTemplate:
name: "test-template"
namespace: "default"
+ bootInterfaceName: management
networkInterfaces:
- name: management
type: network
- network: vm-infra
+ network: management
mtu: 1500
- name: worker
count: 4
@@ -45,10 +49,11 @@ spec:
networkDataTemplate:
name: "test-template"
namespace: "default"
+ bootInterfaceName: management
networkInterfaces:
- name: management
type: network
- network: vm-infra
+ network: management
mtu: 1500
bmcCredentials:
username: admin
diff --git a/docs/api/vino.md b/docs/api/vino.md
index 4c554e0..1f48a5c 100644
--- a/docs/api/vino.md
+++ b/docs/api/vino.md
@@ -159,18 +159,6 @@ int
-nodes
-
-
-[]NodeSet
-
-
- |
-
- |
-
-
-
configuration
@@ -194,6 +182,16 @@ CPUConfiguration
|
|
+
+
+nodeCount
+
+int
+
+ |
+
+ |
+
@@ -307,6 +305,9 @@ NetworkInterface
+
+(Members of NetworkInterface are embedded into this type.)
+
|
@@ -810,6 +811,16 @@ string
+instanceSubnet
+
+string
+
+ |
+
+ |
+
+
+
type
string
@@ -876,6 +887,28 @@ The prefix should be specified in full MAC notation, e.g.
06:42:42:00:00:00
|
+
+
+physicalInterface
+
+string
+
+ |
+
+ PhysicalInterface identifies interface into which to plug in libvirt network
+ |
+
+
+
+libvirtTemplate
+
+string
+
+ |
+
+ LibvirtTemplate identifies which libvirt template to be used to create a network
+ |
+
@@ -989,7 +1022,6 @@ map[string]string
(Appears on:
-Builder,
VinoSpec)
NodeSet node definitions
@@ -1097,6 +1129,17 @@ string
default is /dev/vda
+
+
+bootInterfaceName
+
+string
+
+ |
+
+ BootInterfaceName interface name to use to boot virtual machines
+ |
+
diff --git a/pkg/api/v1/vino_builder.go b/pkg/api/v1/vino_builder.go
index 30b4a53..7c95b45 100644
--- a/pkg/api/v1/vino_builder.go
+++ b/pkg/api/v1/vino_builder.go
@@ -23,10 +23,10 @@ type Builder struct {
PXEBootImageHostPort int `json:"pxeBootImageHostPort,omitempty"`
Networks []Network `json:"networks,omitempty"`
- Nodes []NodeSet `json:"nodes,omitempty"`
// (TODO) change json tag to cpuConfiguration when vino-builder has these chanages as well
CPUConfiguration CPUConfiguration `json:"configuration,omitempty"`
Domains []BuilderDomain `json:"domains,omitempty"`
+ NodeCount int `json:"nodeCount,omitempty"`
}
type BuilderNetworkInterface struct {
diff --git a/pkg/api/v1/vino_types.go b/pkg/api/v1/vino_types.go
index fc15ee2..1f59075 100644
--- a/pkg/api/v1/vino_types.go
+++ b/pkg/api/v1/vino_types.go
@@ -101,6 +101,7 @@ type Network struct {
//Network Parameter defined
Name string `json:"name,omitempty"`
SubNet string `json:"subnet,omitempty"`
+ InstanceSubnet string `json:"instanceSubnet,omitempty"`
Type string `json:"type,omitempty"`
AllocationStart string `json:"allocationStart,omitempty"`
AllocationStop string `json:"allocationStop,omitempty"`
@@ -113,6 +114,10 @@ type Network struct {
// The prefix should be specified in full MAC notation, e.g.
// 06:42:42:00:00:00
MACPrefix string `json:"macPrefix,omitempty"`
+ // PhysicalInterface identifies interface into which to plug in libvirt network
+ PhysicalInterface string `json:"physicalInterface,omitempty"`
+ // LibvirtTemplate identifies which libvirt template to be used to create a network
+ LibvirtTemplate string `json:"libvirtTemplate,omitempty"`
}
// VMRoutes defined
@@ -138,6 +143,8 @@ type NodeSet struct {
// RootDeviceName is the root device for underlying VM, /dev/vda for example
// default is /dev/vda
RootDeviceName string `json:"rootDeviceName,omitempty"`
+ // BootInterfaceName interface name to use to boot virtual machines
+ BootInterfaceName string `json:"bootInterfaceName,omitempty"`
}
// NamespacedName to be used to spawn VMs
diff --git a/pkg/api/v1/zz_generated.deepcopy.go b/pkg/api/v1/zz_generated.deepcopy.go
index e9c3571..b6a2a02 100644
--- a/pkg/api/v1/zz_generated.deepcopy.go
+++ b/pkg/api/v1/zz_generated.deepcopy.go
@@ -65,13 +65,6 @@ func (in *Builder) DeepCopyInto(out *Builder) {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
- if in.Nodes != nil {
- in, out := &in.Nodes, &out.Nodes
- *out = make([]NodeSet, len(*in))
- for i := range *in {
- (*in)[i].DeepCopyInto(&(*out)[i])
- }
- }
out.CPUConfiguration = in.CPUConfiguration
if in.Domains != nil {
in, out := &in.Domains, &out.Domains
diff --git a/pkg/managers/bmh.go b/pkg/managers/bmh.go
index bef2f14..ca26f62 100644
--- a/pkg/managers/bmh.go
+++ b/pkg/managers/bmh.go
@@ -160,24 +160,7 @@ func (r *BMHManager) requestVMs(ctx context.Context) error {
}
func (r *BMHManager) createIpamNetworks(ctx context.Context, vino *vinov1.Vino) error {
- // TODO (kkalynovskyi) these needs to be propagated into network template, and be configurable
- // TODO (kkalynovskyi) develop generic network templates that would allow to handle all networks
- // in single generic way.
- // Bootnetwork needs to be handled spearately because it needs to be created by libvirt
- // And have different configuration.
- if r.BootNetwork == nil {
- r.BootNetwork = &vinov1.Network{
- SubNet: "10.153.241.0/24",
- AllocationStart: "10.153.241.2",
- AllocationStop: "10.153.241.254",
- Name: "pxe-boot",
- MACPrefix: "52:54:00:32:00:00",
- }
- }
- networks := vino.Spec.Networks
- // Append bootnetwork to be created in IPAM
- networks = append(networks, *r.BootNetwork)
- for _, network := range networks {
+ for _, network := range vino.Spec.Networks {
if err := r.createIpamNetwork(ctx, network); err != nil {
return err
}
@@ -280,9 +263,9 @@ func (r *BMHManager) setBMHs(ctx context.Context, pod corev1.Pod) error {
PXEBootImageHost: r.ViNO.Spec.PXEBootImageHost,
PXEBootImageHostPort: r.ViNO.Spec.PXEBootImageHostPort,
Networks: r.ViNO.Spec.Networks,
- Nodes: r.ViNO.Spec.Nodes,
CPUConfiguration: r.ViNO.Spec.CPUConfiguration,
Domains: domains,
+ NodeCount: len(r.ViNO.Spec.Nodes),
}
return r.annotateNode(ctx, k8sNode, vinoBuilder)
}
@@ -312,7 +295,7 @@ func (r *BMHManager) domainSpecificNetValues(
node vinov1.NodeSet,
networks []vinov1.Network) (networkTemplateValues, error) {
// Allocate an IP for each of this BMH's network interfaces
-
+ bootMAC := ""
domainInterfaces := []vinov1.BuilderNetworkInterface{}
for _, iface := range node.NetworkInterfaces {
networkName := iface.NetworkName
@@ -345,12 +328,11 @@ func (r *BMHManager) domainSpecificNetValues(
r.Logger.Info("Got MAC and IP for the network and node",
"MAC", macAddress, "IP", ipAddress, "bmh name", bmhName)
+ if iface.Name == node.BootInterfaceName {
+ bootMAC = macAddress
+ }
}
- // Handle bootMAC separately
- bootMAC, err := r.generatePXEBootMAC(ctx, bmhName)
- if err != nil {
- return networkTemplateValues{}, err
- }
+
r.Logger.Info("Got bootMAC address for BMH node", "bmh name", bmhName, "bootMAC", bootMAC)
return networkTemplateValues{
Node: node,
@@ -363,17 +345,6 @@ func (r *BMHManager) domainSpecificNetValues(
}, nil
}
-func (r *BMHManager) generatePXEBootMAC(ctx context.Context, bmhName string) (string, error) {
- subnetRange, err := ipam.NewRange(r.BootNetwork.AllocationStart, r.BootNetwork.AllocationStop)
- if err != nil {
- return "", err
- }
-
- ipAllocatedTo := fmt.Sprintf("%s/%s", bmhName, "pxe-boot")
- _, mac, err := r.Ipam.AllocateIP(ctx, r.BootNetwork.SubNet, subnetRange, ipAllocatedTo)
- return mac, err
-}
-
func (r *BMHManager) annotateNode(ctx context.Context, k8sNode *corev1.Node, vinoBuilder vinov1.Builder) error {
b, err := yaml.Marshal(vinoBuilder)
if err != nil {
diff --git a/tools/deployment/configure-bridges.sh b/tools/deployment/configure-bridges.sh
index c6c634d..8ad5504 100755
--- a/tools/deployment/configure-bridges.sh
+++ b/tools/deployment/configure-bridges.sh
@@ -20,10 +20,15 @@ PXE_NET="172.3.3.0/24"
export DEBCONF_NONINTERACTIVE_SEEN=true
export DEBIAN_FRONTEND=noninteractive
+sudo modprobe dummy
+
+sudo ip link add enp3s7 type dummy
+sudo ip addr add dev enp3s7 "192.168.2.1/24"
+sudo ip link set up dev enp3s7
+
sudo -E apt-get update
sudo -E apt-get install -y bridge-utils
echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward
-create_bridge ${VM_INFRA_BRIDGE} ${VM_INFRA_BRIDGE_IP}
-create_bridge ${VM_PXE_BRIDGE} ${VM_PXE_BRIDGE_IP}
+create_bridge ${VM_PXE_BRIDGE} ${VM_PXE_BRIDGE_IP}
\ No newline at end of file
diff --git a/vino-builder/assets/playbooks/roles/ipam/library/ipam.py b/vino-builder/assets/playbooks/roles/ipam/library/ipam.py
new file mode 100644
index 0000000..d4d3435
--- /dev/null
+++ b/vino-builder/assets/playbooks/roles/ipam/library/ipam.py
@@ -0,0 +1,96 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# generate_baremetal_macs method ripped from
+# openstack/tripleo-incubator/scripts/configure-vm
+
+import math
+import random
+import sys
+import fnmatch
+import os
+import socket
+import struct
+from itertools import chain
+import json
+import netaddr
+DOCUMENTATION = '''
+---
+module: ipam
+version_added: "1.0"
+short_description: Help with IPAM allocation
+description:
+ - Generate IPs for instances, ensuring they're unique on every node
+'''
+# we don't support specifying subnet_bridge or instances yet
+def allocate_ips(nodes, physical_primary_ip, physical_node_count=1, subnet_bridge='192.168.0.0/24', subnet_instances='192.168.4.0/22'):
+ """Return IP assignments"""
+ # calculate some stuff
+ vm_instance_count = len(nodes)
+ last_octet = physical_primary_ip.split('.')[-1]
+ node_index = int(last_octet) % int(physical_node_count)
+ bridge_ip = netaddr.IPNetwork(subnet_bridge)[node_index+1]
+ # generate an ip for every vm in the entire environment
+ ip_buckets=[None] * physical_node_count
+ vm_ip_list = list(netaddr.IPNetwork(subnet_instances))
+ vm_ip_list.reverse()
+
+ # throw away 0, .1, .2, .3 - assumes we won't exceed .255
+ vm_ip_list.pop()
+ vm_ip_list.pop()
+ vm_ip_list.pop()
+ vm_ip_list.pop()
+
+ # now take IPs from this list - enough for all the VMs
+ # we need to create and place them into groups
+ # one for each physical node
+ for physnode in range(0, physical_node_count):
+ ip_buckets[physnode] = {}
+ ip_list = []
+ for vmidx in range(0, vm_instance_count):
+ ip_list.append(vm_ip_list.pop().__str__())
+ ip_buckets[physnode] = ip_list
+
+ bridge_subnet_netmask = cidr_to_netmask(subnet_bridge)
+ return {
+ 'node_index': node_index,
+ 'bridge_ip': bridge_ip.__str__(),
+ 'instance_ips': ip_buckets[node_index],
+ 'bridge_subnet_netmask': bridge_subnet_netmask,
+ }
+
+def cidr_to_netmask(cidr):
+ _, net_bits = cidr.split('/')
+ host_bits = 32 - int(net_bits)
+ netmask = socket.inet_ntoa(struct.pack('!I', (1 << 32) - (1 << host_bits)))
+ return netmask
+
+def main():
+ module = AnsibleModule(
+ argument_spec=dict(
+ nodes=dict(required=True, type='list'),
+ physical_node_count=dict(required=True, type='int'),
+ primary_ipaddress=dict(required=True, type='str'),
+ subnet_bridge=dict(required=True, type='str'),
+ subnet_instances=dict(required=True, type='str'),
+ )
+ )
+ result = allocate_ips(module.params["nodes"],
+ module.params["primary_ipaddress"],
+ module.params["physical_node_count"],
+ module.params["subnet_bridge"],
+ module.params["subnet_instances"])
+ module.exit_json(**result)
+# see http://docs.ansible.com/developing_modules.html#common-module-boilerplate
+from ansible.module_utils.basic import AnsibleModule # noqa
+if __name__ == '__main__':
+ main()
diff --git a/vino-builder/assets/playbooks/roles/ipam/tasks/main.yaml b/vino-builder/assets/playbooks/roles/ipam/tasks/main.yaml
new file mode 100644
index 0000000..6316d44
--- /dev/null
+++ b/vino-builder/assets/playbooks/roles/ipam/tasks/main.yaml
@@ -0,0 +1,38 @@
+
+ ##################################################
+ # REMOVE ONCE OPERATOR IS SUPPLYING THIS DATA ##
+ ##################################################
+ - name: initialize ipam
+ set_fact:
+ ipam: {}
+ # get the subnet in yaml - the network named 'management'
+ # is special
+
+ - name: discover management subnet from network definitions
+ set_fact:
+ management_subnet: "{{ item }}"
+ when: item.name == 'management'
+ loop: "{{ networks }}"
+
+ - name: print value of management subnet
+ debug:
+ msg: "Value of management subnet is {{ management_subnet }}"
+ # get our ip from admin interface which is always our default route
+
+ - name: discover assigned address of this machines management interface
+ set_fact:
+ primary_ip: "{{ ansible_default_ipv4.address|default(ansible_all_ipv4_addresses[0]) }}"
+
+ - name: ipam allocation
+ ipam:
+ nodes: "{{ domains }}"
+ primary_ipaddress: "{{ primary_ip }}"
+ physical_node_count: "{{ nodeCount | int }}"
+ subnet_bridge: "{{ management_subnet.subnet }}"
+ subnet_instances: "{{ management_subnet.instanceSubnet }}"
+ register: ipam
+ when: domains
+
+ - name: debug ipam result
+ debug:
+ msg: "IPAM Result {{ ipam }}"
diff --git a/vino-builder/assets/playbooks/roles/libvirt/defaults/main.yml b/vino-builder/assets/playbooks/roles/libvirt/defaults/main.yml
index d2a4f49..d6fdf8d 100644
--- a/vino-builder/assets/playbooks/roles/libvirt/defaults/main.yml
+++ b/vino-builder/assets/playbooks/roles/libvirt/defaults/main.yml
@@ -1 +1,2 @@
-libvirt_uri: qemu:///system
\ No newline at end of file
+libvirt_uri: qemu:///system
+node_core_map: {}
\ No newline at end of file
diff --git a/vino-builder/assets/playbooks/roles/libvirt/library/core_allocation.py b/vino-builder/assets/playbooks/roles/libvirt/library/core_allocation.py
index eb25684..352cff2 100644
--- a/vino-builder/assets/playbooks/roles/libvirt/library/core_allocation.py
+++ b/vino-builder/assets/playbooks/roles/libvirt/library/core_allocation.py
@@ -96,40 +96,39 @@ def allocate_cores(nodes, flavors, exclude_cpu):
# address the case where previous != desired - delete previous, re-run
for node in nodes:
- flavor = node['bmhLabels']['airshipit.org/k8s-role']
+ flavor = node["role"]
vcpus = flavors[flavor]['vcpus']
- for num_node in range(0, node['count']):
- # generate a unique name such as master-0, master-1
- node_name = node['name'] + '-' + str(num_node)
+ # generate a unique name such as master-0, master-1
+ node_name = node["name"]
- # extract the core count
- core_count = int(vcpus)
+ # extract the core count
+ core_count = int(vcpus)
- # discover any previous allocation
- if 'assignments' in core_state:
- if node_name in core_state['assignments']:
- if len(core_state['assignments'][node_name]) == core_count:
- continue
- else:
- # TODO: support releasing the cores and adding them back
- # to available
- raise Exception("Existing assignment exists for node %s but does not match current core count needed" % node_name)
-
- # allocate the cores
- allocated=False
- for numa in core_state['available']:
- if core_count <= len(core_state['available'][numa]):
- allocated=True
- cores_to_use = core_state['available'][numa][:core_count]
- core_state['assignments'][node_name] = cores_to_use
- core_state['available'][numa] = core_state['available'][numa][core_count:]
- break
- else:
+ # discover any previous allocation
+ if 'assignments' in core_state:
+ if node_name in core_state['assignments']:
+ if len(core_state['assignments'][node_name]) == core_count:
continue
- if not allocated:
- raise Exception("Unable to find sufficient cores (%s) for node %s (available was %r)" % (core_count, node_name, core_state['available']))
+ else:
+ # TODO: support releasing the cores and adding them back
+ # to available
+ raise Exception("Existing assignment exists for node %s but does not match current core count needed" % node_name)
+
+ # allocate the cores
+ allocated=False
+ for numa in core_state['available']:
+ if core_count <= len(core_state['available'][numa]):
+ allocated=True
+ cores_to_use = core_state['available'][numa][:core_count]
+ core_state['assignments'][node_name] = cores_to_use
+ core_state['available'][numa] = core_state['available'][numa][core_count:]
+ break
+ else:
+ continue
+ if not allocated:
+ raise Exception("Unable to find sufficient cores (%s) for node %s (available was %r)" % (core_count, node_name, core_state['available']))
# return a dict of nodes: cores
# or error if insufficient
diff --git a/vino-builder/assets/playbooks/roles/libvirt/tasks/create-network.yaml b/vino-builder/assets/playbooks/roles/libvirt/tasks/create-network.yaml
index 91e19d8..acd9674 100644
--- a/vino-builder/assets/playbooks/roles/libvirt/tasks/create-network.yaml
+++ b/vino-builder/assets/playbooks/roles/libvirt/tasks/create-network.yaml
@@ -17,7 +17,7 @@
state: present
# looks like setting name here is a redundant, the name is anyways taken from the template xml file, but should set it to make virt_pool module happy.
name: "{{ network.name }}"
- xml: "{{ network.libvirtTemplate }}"
+ xml: "{{ libvirtNetworks[network.libvirtTemplate].libvirtTemplate }}"
uri: "{{ libvirt_uri }}"
when: "network.name not in ansible_libvirt_networks"
diff --git a/vino-builder/assets/playbooks/roles/libvirt/tasks/main.yml b/vino-builder/assets/playbooks/roles/libvirt/tasks/main.yml
index b99b198..d80ab88 100644
--- a/vino-builder/assets/playbooks/roles/libvirt/tasks/main.yml
+++ b/vino-builder/assets/playbooks/roles/libvirt/tasks/main.yml
@@ -12,7 +12,7 @@
- name: create network
include_tasks: create-network.yaml
- loop: "{{ libvirtNetworks }}"
+ loop: "{{ networks }}"
loop_control:
loop_var: network
@@ -22,15 +22,14 @@
- name: allocate domain cores
core_allocation:
- nodes: "{{ nodes }}"
+ nodes: "{{ domains }}"
flavors: "{{ flavors }}"
exclude_cpu: "{{ configuration.cpuExclude }}"
register: node_core_map
- when: nodes
- name: debug print node_core_map
debug:
- msg: "node_core_map = {{ node_core_map }}"
+ var: node_core_map
- name: define domain outer loop
include_tasks: create-domain.yaml
diff --git a/vino-builder/assets/playbooks/vino-builder.yaml b/vino-builder/assets/playbooks/vino-builder.yaml
index 6ba068c..5084bfe 100644
--- a/vino-builder/assets/playbooks/vino-builder.yaml
+++ b/vino-builder/assets/playbooks/vino-builder.yaml
@@ -37,9 +37,13 @@
---
- hosts: localhost
-
tasks:
+ # generate libvirt definitions for storage, networks, and domains
+ - name: generate management network ip addresses
+ include_role:
+ name: ipam
+
# generate libvirt definitions for storage, networks, and domains
- name: process libvirt definitions
include_role: