Generate unique subranges for k8s nodes

The ranges will be stored in IPPool object and will never change
once assigned.

Change-Id: Ie3184f2a9de405c00223367939f8c3e3112f0e68

config/crd/bases/airship.airshipit.org_ippools.yaml

@@ -55,6 +55,21 @@ spec:
                   - mac
                   type: object
                 type: array
+              allocatedRanges:
+                items:
+                  properties:
+                    allocatedTo:
+                      type: string
+                    start:
+                      type: string
+                    stop:
+                      type: string
+                  required:
+                  - allocatedTo
+                  - start
+                  - stop
+                  type: object
+                type: array
               macPrefix:
                 description: MACPrefix defines the MAC prefix to use for VM mac addresses
                 type: string

config/crd/bases/airship.airshipit.org_vinoes.yaml

@@ -82,20 +82,24 @@ spec:
                 items:
                   description: Network defines libvirt networks
                   properties:
-                    allocationStart:
-                      type: string
-                    allocationStop:
-                      type: string
                     bridgeName:
                       description: BridgeName is the name of the bridge to be created
                         as libvirt network. works if AllocateNodeIP is sepcified
                       type: string
+                    dhcpAllocationStart:
+                      description: DHCPAllocationStart must be inside the SubNet range
+                      type: string
+                    dhcpAllocationStop:
+                      description: DHCPAllocationStop must be inside the SubNet range
+                      type: string
                     dns_servers:
                       items:
                         type: string
                       type: array
-                    instanceSubnet:
+                    instanceSubnetBitStep:
-                      type: string
+                      description: InstanceSubnetBitStep indicates how many bites
+                        to allocate for each node DHCP range
+                      type: integer
                     libvirtTemplate:
                       description: LibvirtTemplate identifies which libvirt template
                         to be used to create a network
@@ -126,6 +130,10 @@ spec:
                             type: string
                         type: object
                       type: array
+                    staticAllocationStart:
+                      type: string
+                    staticAllocationStop:
+                      type: string
                     subnet:
                       type: string
                     type:

config/manager/network-templates.yaml

@@ -5,10 +5,10 @@ libvirtNetworks:
         <name>{{ network.name }}</name>
         <forward mode='route'/>
         <bridge name='{{ network.bridgeName | default('vm-infra-bridge') }}' stp='off' delay='0'/>
-        <ip address='{{ network.bridgeIP | default(omit) }}' netmask='{{ ipam.bridge_subnet_netmask }}'>
+        <ip address='{{ network.bridgeIP | default(omit) }}' netmask='{{ network.subnet | ansible.netcommon.ipaddr('netmask') }}'>
           <!-- <tftp root='/srv/tftp'/> -->
           <dhcp>
-            <range start='{{ ipam.instance_ips[0] }}' end='{{ ipam.instance_ips[-1] }}'/>
+            <range start='{{ network.range.start }}' end='{{ network.range.stop }}'/>
             <bootp file='http://{{ pxeBootImageHost | default(ansible_default_ipv4.address) }}:{{ pxeBootImageHostPort | default(80) }}/dualboot.ipxe'/>
           </dhcp>
         </ip>

config/samples/vino_cr.yaml

@@ -16,10 +16,12 @@ spec:
     - name: management
       libvirtTemplate: management
       subnet: 192.168.2.0/20
-      instanceSubnet: 192.168.4.0/22
+      dhcpAllocationStart: 192.168.4.0
+      dhcpAllocationStop: 192.168.7.255
+      instanceSubnetBitStep: 4
       type: bridge
-      allocationStart: 192.168.2.10
+      staticAllocationStart: 192.168.2.10
-      allocationStop: 192.168.2.24 # docs should specify that the range should = number of vms (to permit future expansion over multiple vino crs etc)
+      staticAllocationStop: 192.168.2.24
       dns_servers: ["135.188.34.124"]
       macPrefix: "52:54:00:06:00:00"
       physicalInterface: enp3s7

config/samples/vino_cr_4_workers_1_cp.yaml

@@ -16,14 +16,16 @@ spec:
     - name: management
       libvirtTemplate: management
       subnet: 192.168.2.0/20
-      instanceSubnet: 192.168.4.0/22
+      dhcpAllocationStart: 192.168.4.0
-      type: ipv4
+      dhcpAllocationStop: 192.168.7.255
-      allocationStart: 192.168.2.10
+      instanceSubnetBitStep: 6
-      allocationStop: 192.168.2.25 # docs should specify that the range should = number of vms (to permit future expansion over multiple vino crs etc)
+      type: bridge
+      staticAllocationStart: 192.168.2.10
+      staticAllocationStop: 192.168.2.24
       routes:
         - network: 10.0.0.0
           netmask: 255.255.255.0
-          gateway: $vinobridge # vino will need to populate this from the nodelabel value `airshipit.org/vino.nodebridgegw`
+          gateway: $vinobridge
       dns_servers: ["135.188.34.124"]
       macPrefix: "52:54:00:06:00:00"
       physicalInterface: enp3s7

docs/api/vino.md

@@ -60,6 +60,51 @@ string
 </table>
 </div>
 </div>
+<h3 id="airship.airshipit.org/v1.AllocatedRange">AllocatedRange
+</h3>
+<p>
+(<em>Appears on:</em>
+<a href="#airship.airshipit.org/v1.IPPoolSpec">IPPoolSpec</a>)
+</p>
+<div class="md-typeset__scrollwrap">
+<div class="md-typeset__table">
+<table>
+<thead>
+<tr>
+<th>Field</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>
+<code>allocatedTo</code><br>
+<em>
+string
+</em>
+</td>
+<td>
+</td>
+</tr>
+<tr>
+<td>
+<code>Range</code><br>
+<em>
+<a href="#airship.airshipit.org/v1.Range">
+Range
+</a>
+</em>
+</td>
+<td>
+<p>
+(Members of <code>Range</code> are embedded into this type.)
+</p>
+</td>
+</tr>
+</tbody>
+</table>
+</div>
+</div>
 <h3 id="airship.airshipit.org/v1.BMCCredentials">BMCCredentials
 </h3>
 <p>
@@ -685,6 +730,18 @@ string
 </tr>
 <tr>
 <td>
+<code>allocatedRanges</code><br>
+<em>
+<a href="#airship.airshipit.org/v1.AllocatedRange">
+[]AllocatedRange
+</a>
+</em>
+</td>
+<td>
+</td>
+</tr>
+<tr>
+<td>
 <code>allocatedIPs</code><br>
 <em>
 <a href="#airship.airshipit.org/v1.AllocatedIP">
@@ -781,6 +838,18 @@ string
 </tr>
 <tr>
 <td>
+<code>allocatedRanges</code><br>
+<em>
+<a href="#airship.airshipit.org/v1.AllocatedRange">
+[]AllocatedRange
+</a>
+</em>
+</td>
+<td>
+</td>
+</tr>
+<tr>
+<td>
 <code>allocatedIPs</code><br>
 <em>
 <a href="#airship.airshipit.org/v1.AllocatedIP">
@@ -908,12 +977,35 @@ string
 </tr>
 <tr>
 <td>
-<code>instanceSubnet</code><br>
+<code>dhcpAllocationStart</code><br>
 <em>
 string
 </em>
 </td>
 <td>
+<p>DHCPAllocationStart must be inside the SubNet range</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>dhcpAllocationStop</code><br>
+<em>
+string
+</em>
+</td>
+<td>
+<p>DHCPAllocationStop must be inside the SubNet range</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>instanceSubnetBitStep</code><br>
+<em>
+int
+</em>
+</td>
+<td>
+<p>InstanceSubnetBitStep indicates how many bites to allocate for each node DHCP range</p>
 </td>
 </tr>
 <tr>
@@ -928,7 +1020,7 @@ string
 </tr>
 <tr>
 <td>
-<code>allocationStart</code><br>
+<code>staticAllocationStart</code><br>
 <em>
 string
 </em>
@@ -938,7 +1030,7 @@ string
 </tr>
 <tr>
 <td>
-<code>allocationStop</code><br>
+<code>staticAllocationStop</code><br>
 <em>
 string
 </em>
@@ -1268,6 +1360,7 @@ bool
 </h3>
 <p>
 (<em>Appears on:</em>
+<a href="#airship.airshipit.org/v1.AllocatedRange">AllocatedRange</a>, 
 <a href="#airship.airshipit.org/v1.BuilderNetwork">BuilderNetwork</a>, 
 <a href="#airship.airshipit.org/v1.IPPoolSpec">IPPoolSpec</a>)
 </p>

pkg/api/v1/ippool_types.go

@@ -26,9 +26,10 @@ import (
 // within the subnet from which IPs can be allocated by IPAM,
 // and a set of IPs that are currently allocated already.
 type IPPoolSpec struct {
-	Subnet       string        `json:"subnet"`
+	Subnet          string           `json:"subnet"`
-	Ranges       []Range       `json:"ranges"`
+	Ranges          []Range          `json:"ranges"`
-	AllocatedIPs []AllocatedIP `json:"allocatedIPs"`
+	AllocatedRanges []AllocatedRange `json:"allocatedRanges,omitempty"`
+	AllocatedIPs    []AllocatedIP    `json:"allocatedIPs"`
 	// MACPrefix defines the MAC prefix to use for VM mac addresses
 	MACPrefix string `json:"macPrefix"`
 	// NextMAC indicates the next MAC address (in sequence) that
@@ -43,6 +44,11 @@ type AllocatedIP struct {
 	AllocatedTo string `json:"allocatedTo"`
 }
 
+type AllocatedRange struct {
+	AllocatedTo string `json:"allocatedTo"`
+	Range       `json:",inline"`
+}
+
 // Range has (inclusive) bounds within a subnet from which IPs can be allocated
 type Range struct {
 	Start string `json:"start"`

pkg/api/v1/vino_types.go

@@ -41,6 +41,8 @@ const (
 	VinoNetworkDataTemplateDefaultKey = "template"
 	// VinoDefaultRootDeviceName is default root device for the underlying libvirt VM
 	VinoDefaultRootDeviceName = "/dev/vda"
+	// VinoDefaultInstanceSubnetBitStep is the value for InstanceSubnetBitStep
+	VinoDefaultInstanceSubnetBitStep = 4
 )
 
 // Constants for BasicAuth
@@ -97,14 +99,19 @@ type CPUConfiguration struct {
 // Network defines libvirt networks
 type Network struct {
 	//Network Parameter defined
-	Name            string     `json:"name,omitempty"`
+	Name   string `json:"name,omitempty"`
-	SubNet          string     `json:"subnet,omitempty"`
+	SubNet string `json:"subnet,omitempty"`
-	InstanceSubnet  string     `json:"instanceSubnet,omitempty"`
+	// DHCPAllocationStart must be inside the SubNet range
-	Type            string     `json:"type,omitempty"`
+	DHCPAllocationStart string `json:"dhcpAllocationStart,omitempty"`
-	AllocationStart string     `json:"allocationStart,omitempty"`
+	// DHCPAllocationStop must be inside the SubNet range
-	AllocationStop  string     `json:"allocationStop,omitempty"`
+	DHCPAllocationStop string `json:"dhcpAllocationStop,omitempty"`
-	DNSServers      []string   `json:"dns_servers,omitempty"`
+	// InstanceSubnetBitStep indicates how many bites to allocate for each node DHCP range
-	Routes          []VMRoutes `json:"routes,omitempty"`
+	InstanceSubnetBitStep int        `json:"instanceSubnetBitStep,omitempty"`
+	Type                  string     `json:"type,omitempty"`
+	StaticAllocationStart string     `json:"staticAllocationStart,omitempty"`
+	StaticAllocationStop  string     `json:"staticAllocationStop,omitempty"`
+	DNSServers            []string   `json:"dns_servers,omitempty"`
+	Routes                []VMRoutes `json:"routes,omitempty"`
 	// MACPrefix defines the zero-padded MAC prefix to use for
 	// VM mac addresses, and is the first address that will be
 	// allocated sequentially to VMs in this network.

pkg/api/v1/zz_generated.deepcopy.go

@@ -40,6 +40,22 @@ func (in *AllocatedIP) DeepCopy() *AllocatedIP {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AllocatedRange) DeepCopyInto(out *AllocatedRange) {
+	*out = *in
+	out.Range = in.Range
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AllocatedRange.
+func (in *AllocatedRange) DeepCopy() *AllocatedRange {
+	if in == nil {
+		return nil
+	}
+	out := new(AllocatedRange)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *BMCCredentials) DeepCopyInto(out *BMCCredentials) {
 	*out = *in
@@ -273,6 +289,11 @@ func (in *IPPoolSpec) DeepCopyInto(out *IPPoolSpec) {
 		*out = make([]Range, len(*in))
 		copy(*out, *in)
 	}
+	if in.AllocatedRanges != nil {
+		in, out := &in.AllocatedRanges, &out.AllocatedRanges
+		*out = make([]AllocatedRange, len(*in))
+		copy(*out, *in)
+	}
 	if in.AllocatedIPs != nil {
 		in, out := &in.AllocatedIPs, &out.AllocatedIPs
 		*out = make([]AllocatedIP, len(*in))

pkg/ipam/ipam.go

@@ -17,6 +17,7 @@ package ipam
 import (
 	"context"
 	"fmt"
+	"math"
 	"net"
 	"regexp"
 	"strings"
@@ -378,3 +379,100 @@ func (i *Ipam) getIPPools(ctx context.Context) (map[string]*vinov1.IPPoolSpec, e
 	}
 	return ippools, nil
 }
+
+func (i *Ipam) AllocateRange(ctx context.Context,
+	bitStep int,
+	host, macPrefix, start, stop, subnet string) (vinov1.Range, error) {
+	ipPool, err := i.getIPPoolWithRanges(ctx, bitStep, macPrefix, start, stop, subnet)
+	if err != nil {
+		return vinov1.Range{}, err
+	}
+
+	result, err := chooseRange(host, ipPool)
+	if err != nil {
+		return vinov1.Range{}, err
+	}
+	return result, i.applyIPPool(ctx, *ipPool)
+}
+
+func chooseRange(host string, ipPool *vinov1.IPPoolSpec) (vinov1.Range, error) {
+	const unallocated = -1
+	freeIndex := unallocated
+	for i, r := range ipPool.AllocatedRanges {
+		if r.AllocatedTo == host {
+			return r.Range, nil
+		} else if r.AllocatedTo == "" && freeIndex == unallocated {
+			freeIndex = i
+		}
+	}
+	if freeIndex != unallocated {
+		ipPool.AllocatedRanges[freeIndex].AllocatedTo = host
+	} else {
+		return vinov1.Range{}, fmt.Errorf("No free ranges available for host %s", host)
+	}
+	return ipPool.AllocatedRanges[freeIndex].Range, nil
+}
+
+func (i *Ipam) getIPPoolWithRanges(ctx context.Context, bitStep int,
+	macPrefix, start, stop, subnet string) (*vinov1.IPPoolSpec, error) {
+	ippools, err := i.getIPPools(ctx)
+	if err != nil {
+		return &vinov1.IPPoolSpec{}, err
+	}
+	logger := i.Log.WithValues("subnet", subnet)
+
+	ippool, exists := ippools[subnet]
+	if !exists {
+		logger.Info("IPAM creating subnet")
+		_, err = macStringToInt(macPrefix) // mac format validation
+		if err != nil {
+			return &vinov1.IPPoolSpec{}, err
+		}
+		ippool = &vinov1.IPPoolSpec{
+			Subnet:       subnet,
+			Ranges:       []vinov1.Range{},
+			AllocatedIPs: []vinov1.AllocatedIP{},
+			MACPrefix:    macPrefix,
+			NextMAC:      macPrefix,
+		}
+		ippools[subnet] = ippool
+	}
+	if len(ippool.AllocatedRanges) != 0 {
+		return ippool, nil
+	}
+
+	ranges, err := generateRanges(start, stop, bitStep)
+	if err != nil {
+		return nil, err
+	}
+	ippool.AllocatedRanges = ranges
+	return ippool, nil
+}
+
+func generateRanges(start, stop string, bitStep int) ([]vinov1.AllocatedRange, error) {
+	firstNetIPInt, err := ipStringToInt(start)
+	if err != nil {
+		return nil, err
+	}
+
+	// support only IPv4, use 32 netmask
+	subnetEnd, err := ipStringToInt(stop)
+	if err != nil {
+		return nil, err
+	}
+	ranges := []vinov1.AllocatedRange{}
+	shift := uint64(math.Pow(2, float64(bitStep)))
+
+	for start, end := firstNetIPInt, firstNetIPInt+shift; end-1 <= subnetEnd; {
+		fmt.Printf("start is %s, end is %s\n", intToIPv4String(start), intToIPv4String(subnetEnd))
+		ranges = append(ranges, vinov1.AllocatedRange{
+			Range: vinov1.Range{
+				Start: intToIPv4String(start),
+				Stop:  intToIPv4String(end - 1),
+			},
+		})
+		start += shift
+		end += shift
+	}
+	return ranges, nil
+}

pkg/managers/bmh.go

@@ -169,7 +169,7 @@ func (r *BMHManager) createIpamNetworks(ctx context.Context, vino *vinov1.Vino)
 }
 
 func (r *BMHManager) createIpamNetwork(ctx context.Context, network vinov1.Network) error {
-	subnetRange, err := ipam.NewRange(network.AllocationStart, network.AllocationStop)
+	subnetRange, err := ipam.NewRange(network.StaticAllocationStart, network.StaticAllocationStop)
 	if err != nil {
 		return err
 	}
@@ -295,6 +295,21 @@ func (r *BMHManager) nodeNetworks(ctx context.Context,
 				builderNetwork.Network.Routes[routeIndex].Gateway = bridgeIP
 			}
 		}
+		bitStep := network.InstanceSubnetBitStep
+		if bitStep == 0 {
+			bitStep = vinov1.VinoDefaultInstanceSubnetBitStep
+		}
+		r, err := r.Ipam.AllocateRange(ctx,
+			bitStep,
+			k8sNode.Name,
+			network.MACPrefix,
+			network.DHCPAllocationStart,
+			network.DHCPAllocationStop,
+			network.SubNet)
+		if err != nil {
+			return []vinov1.BuilderNetwork{}, err
+		}
+		builderNetwork.Range = r
 		builderNetworks = append(builderNetworks, builderNetwork)
 	}
 	return builderNetworks, nil
@@ -316,7 +331,7 @@ func (r *BMHManager) domainSpecificNetValues(
 		for _, network := range networks {
 			if network.Name == networkName {
 				subnet = network.SubNet
-				subnetRange, err = ipam.NewRange(network.AllocationStart, network.AllocationStop)
+				subnetRange, err = ipam.NewRange(network.StaticAllocationStart, network.StaticAllocationStop)
 				if err != nil {
 					return networkTemplateValues{}, err
 				}
@@ -381,7 +396,7 @@ func (r *BMHManager) annotateNode(ctx context.Context, k8sNode *corev1.Node, vin
 func (r *BMHManager) getBridgeIPandMAC(ctx context.Context,
 	network vinov1.Network,
 	k8sNode *corev1.Node) (string, string, error) {
-	subnetRange, err := ipam.NewRange(network.AllocationStart, network.AllocationStop)
+	subnetRange, err := ipam.NewRange(network.StaticAllocationStart, network.StaticAllocationStop)
 	if err != nil {
 		return "", "", err
 	}

vino-builder/assets/playbooks/vino-builder.yaml

@@ -39,11 +39,6 @@
 - hosts: localhost
   tasks:
 
-  # generate libvirt definitions for storage, networks, and domains
-  - name: generate management network ip addresses
-    include_role:
-      name: ipam
-
   # generate libvirt definitions for storage, networks, and domains
   - name: process libvirt definitions
     include_role: