From 7602aa53240517c94127ee0c1db1cecfd7e80f87 Mon Sep 17 00:00:00 2001 From: Kostiantyn Kalynovskyi Date: Mon, 14 Jun 2021 22:51:46 +0000 Subject: [PATCH] Generate unique subranges for k8s nodes The ranges will be stored in IPPool object and will never change once assigned. Change-Id: Ie3184f2a9de405c00223367939f8c3e3112f0e68 --- .../bases/airship.airshipit.org_ippools.yaml | 15 +++ .../bases/airship.airshipit.org_vinoes.yaml | 20 ++-- config/manager/network-templates.yaml | 4 +- config/samples/vino_cr.yaml | 8 +- config/samples/vino_cr_4_workers_1_cp.yaml | 12 ++- docs/api/vino.md | 99 ++++++++++++++++++- pkg/api/v1/ippool_types.go | 12 ++- pkg/api/v1/vino_types.go | 23 +++-- pkg/api/v1/zz_generated.deepcopy.go | 21 ++++ pkg/ipam/ipam.go | 98 ++++++++++++++++++ pkg/managers/bmh.go | 21 +++- .../assets/playbooks/vino-builder.yaml | 5 - 12 files changed, 300 insertions(+), 38 deletions(-) diff --git a/config/crd/bases/airship.airshipit.org_ippools.yaml b/config/crd/bases/airship.airshipit.org_ippools.yaml index a504e47..f7ff5b7 100644 --- a/config/crd/bases/airship.airshipit.org_ippools.yaml +++ b/config/crd/bases/airship.airshipit.org_ippools.yaml @@ -55,6 +55,21 @@ spec: - mac type: object type: array + allocatedRanges: + items: + properties: + allocatedTo: + type: string + start: + type: string + stop: + type: string + required: + - allocatedTo + - start + - stop + type: object + type: array macPrefix: description: MACPrefix defines the MAC prefix to use for VM mac addresses type: string diff --git a/config/crd/bases/airship.airshipit.org_vinoes.yaml b/config/crd/bases/airship.airshipit.org_vinoes.yaml index 6953b21..0fee682 100644 --- a/config/crd/bases/airship.airshipit.org_vinoes.yaml +++ b/config/crd/bases/airship.airshipit.org_vinoes.yaml @@ -82,20 +82,24 @@ spec: items: description: Network defines libvirt networks properties: - allocationStart: - type: string - allocationStop: - type: string bridgeName: description: BridgeName is the name of the bridge to be created as libvirt network. works if AllocateNodeIP is sepcified type: string + dhcpAllocationStart: + description: DHCPAllocationStart must be inside the SubNet range + type: string + dhcpAllocationStop: + description: DHCPAllocationStop must be inside the SubNet range + type: string dns_servers: items: type: string type: array - instanceSubnet: - type: string + instanceSubnetBitStep: + description: InstanceSubnetBitStep indicates how many bites + to allocate for each node DHCP range + type: integer libvirtTemplate: description: LibvirtTemplate identifies which libvirt template to be used to create a network @@ -126,6 +130,10 @@ spec: type: string type: object type: array + staticAllocationStart: + type: string + staticAllocationStop: + type: string subnet: type: string type: diff --git a/config/manager/network-templates.yaml b/config/manager/network-templates.yaml index d3f13c8..e8d5e7d 100644 --- a/config/manager/network-templates.yaml +++ b/config/manager/network-templates.yaml @@ -5,10 +5,10 @@ libvirtNetworks: {{ network.name }} - + - + diff --git a/config/samples/vino_cr.yaml b/config/samples/vino_cr.yaml index 69b1b1b..4f96d9f 100644 --- a/config/samples/vino_cr.yaml +++ b/config/samples/vino_cr.yaml @@ -16,10 +16,12 @@ spec: - name: management libvirtTemplate: management subnet: 192.168.2.0/20 - instanceSubnet: 192.168.4.0/22 + dhcpAllocationStart: 192.168.4.0 + dhcpAllocationStop: 192.168.7.255 + instanceSubnetBitStep: 4 type: bridge - allocationStart: 192.168.2.10 - allocationStop: 192.168.2.24 # docs should specify that the range should = number of vms (to permit future expansion over multiple vino crs etc) + staticAllocationStart: 192.168.2.10 + staticAllocationStop: 192.168.2.24 dns_servers: ["135.188.34.124"] macPrefix: "52:54:00:06:00:00" physicalInterface: enp3s7 diff --git a/config/samples/vino_cr_4_workers_1_cp.yaml b/config/samples/vino_cr_4_workers_1_cp.yaml index 64106aa..44b5f37 100644 --- a/config/samples/vino_cr_4_workers_1_cp.yaml +++ b/config/samples/vino_cr_4_workers_1_cp.yaml @@ -16,14 +16,16 @@ spec: - name: management libvirtTemplate: management subnet: 192.168.2.0/20 - instanceSubnet: 192.168.4.0/22 - type: ipv4 - allocationStart: 192.168.2.10 - allocationStop: 192.168.2.25 # docs should specify that the range should = number of vms (to permit future expansion over multiple vino crs etc) + dhcpAllocationStart: 192.168.4.0 + dhcpAllocationStop: 192.168.7.255 + instanceSubnetBitStep: 6 + type: bridge + staticAllocationStart: 192.168.2.10 + staticAllocationStop: 192.168.2.24 routes: - network: 10.0.0.0 netmask: 255.255.255.0 - gateway: $vinobridge # vino will need to populate this from the nodelabel value `airshipit.org/vino.nodebridgegw` + gateway: $vinobridge dns_servers: ["135.188.34.124"] macPrefix: "52:54:00:06:00:00" physicalInterface: enp3s7 diff --git a/docs/api/vino.md b/docs/api/vino.md index 52b273c..8a8c032 100644 --- a/docs/api/vino.md +++ b/docs/api/vino.md @@ -60,6 +60,51 @@ string +

AllocatedRange +

+

+(Appears on: +IPPoolSpec) +

+
+
+ + + + + + + + + + + + + + + + + +
FieldDescription
+allocatedTo
+ +string + +		 +
+Range
+ + +Range + + +		 +

+(Members of Range are embedded into this type.) +

+
+
+

BMCCredentials

@@ -685,6 +730,18 @@ string +allocatedRanges
+ + +[]AllocatedRange + + + + + + + + allocatedIPs
@@ -781,6 +838,18 @@ string +allocatedRanges
+ + +[]AllocatedRange + + + + + + + + allocatedIPs
@@ -908,12 +977,35 @@ string -instanceSubnet
+dhcpAllocationStart
string +

DHCPAllocationStart must be inside the SubNet range

+ + + + +dhcpAllocationStop
+ +string + + + +

DHCPAllocationStop must be inside the SubNet range

+ + + + +instanceSubnetBitStep
+ +int + + + +

InstanceSubnetBitStep indicates how many bites to allocate for each node DHCP range

@@ -928,7 +1020,7 @@ string -allocationStart
+staticAllocationStart
string @@ -938,7 +1030,7 @@ string -allocationStop
+staticAllocationStop
string @@ -1268,6 +1360,7 @@ bool

(Appears on: +AllocatedRange, BuilderNetwork, IPPoolSpec)

diff --git a/pkg/api/v1/ippool_types.go b/pkg/api/v1/ippool_types.go index 491b2b4..c9227c4 100644 --- a/pkg/api/v1/ippool_types.go +++ b/pkg/api/v1/ippool_types.go @@ -26,9 +26,10 @@ import ( // within the subnet from which IPs can be allocated by IPAM, // and a set of IPs that are currently allocated already. type IPPoolSpec struct { - Subnet string `json:"subnet"` - Ranges []Range `json:"ranges"` - AllocatedIPs []AllocatedIP `json:"allocatedIPs"` + Subnet string `json:"subnet"` + Ranges []Range `json:"ranges"` + AllocatedRanges []AllocatedRange `json:"allocatedRanges,omitempty"` + AllocatedIPs []AllocatedIP `json:"allocatedIPs"` // MACPrefix defines the MAC prefix to use for VM mac addresses MACPrefix string `json:"macPrefix"` // NextMAC indicates the next MAC address (in sequence) that @@ -43,6 +44,11 @@ type AllocatedIP struct { AllocatedTo string `json:"allocatedTo"` } +type AllocatedRange struct { + AllocatedTo string `json:"allocatedTo"` + Range `json:",inline"` +} + // Range has (inclusive) bounds within a subnet from which IPs can be allocated type Range struct { Start string `json:"start"` diff --git a/pkg/api/v1/vino_types.go b/pkg/api/v1/vino_types.go index d0a9bd7..9eec06d 100644 --- a/pkg/api/v1/vino_types.go +++ b/pkg/api/v1/vino_types.go @@ -41,6 +41,8 @@ const ( VinoNetworkDataTemplateDefaultKey = "template" // VinoDefaultRootDeviceName is default root device for the underlying libvirt VM VinoDefaultRootDeviceName = "/dev/vda" + // VinoDefaultInstanceSubnetBitStep is the value for InstanceSubnetBitStep + VinoDefaultInstanceSubnetBitStep = 4 ) // Constants for BasicAuth @@ -97,14 +99,19 @@ type CPUConfiguration struct { // Network defines libvirt networks type Network struct { //Network Parameter defined - Name string `json:"name,omitempty"` - SubNet string `json:"subnet,omitempty"` - InstanceSubnet string `json:"instanceSubnet,omitempty"` - Type string `json:"type,omitempty"` - AllocationStart string `json:"allocationStart,omitempty"` - AllocationStop string `json:"allocationStop,omitempty"` - DNSServers []string `json:"dns_servers,omitempty"` - Routes []VMRoutes `json:"routes,omitempty"` + Name string `json:"name,omitempty"` + SubNet string `json:"subnet,omitempty"` + // DHCPAllocationStart must be inside the SubNet range + DHCPAllocationStart string `json:"dhcpAllocationStart,omitempty"` + // DHCPAllocationStop must be inside the SubNet range + DHCPAllocationStop string `json:"dhcpAllocationStop,omitempty"` + // InstanceSubnetBitStep indicates how many bites to allocate for each node DHCP range + InstanceSubnetBitStep int `json:"instanceSubnetBitStep,omitempty"` + Type string `json:"type,omitempty"` + StaticAllocationStart string `json:"staticAllocationStart,omitempty"` + StaticAllocationStop string `json:"staticAllocationStop,omitempty"` + DNSServers []string `json:"dns_servers,omitempty"` + Routes []VMRoutes `json:"routes,omitempty"` // MACPrefix defines the zero-padded MAC prefix to use for // VM mac addresses, and is the first address that will be // allocated sequentially to VMs in this network. diff --git a/pkg/api/v1/zz_generated.deepcopy.go b/pkg/api/v1/zz_generated.deepcopy.go index fac2701..bf80608 100644 --- a/pkg/api/v1/zz_generated.deepcopy.go +++ b/pkg/api/v1/zz_generated.deepcopy.go @@ -40,6 +40,22 @@ func (in *AllocatedIP) DeepCopy() *AllocatedIP { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AllocatedRange) DeepCopyInto(out *AllocatedRange) { + *out = *in + out.Range = in.Range +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AllocatedRange. +func (in *AllocatedRange) DeepCopy() *AllocatedRange { + if in == nil { + return nil + } + out := new(AllocatedRange) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BMCCredentials) DeepCopyInto(out *BMCCredentials) { *out = *in @@ -273,6 +289,11 @@ func (in *IPPoolSpec) DeepCopyInto(out *IPPoolSpec) { *out = make([]Range, len(*in)) copy(*out, *in) } + if in.AllocatedRanges != nil { + in, out := &in.AllocatedRanges, &out.AllocatedRanges + *out = make([]AllocatedRange, len(*in)) + copy(*out, *in) + } if in.AllocatedIPs != nil { in, out := &in.AllocatedIPs, &out.AllocatedIPs *out = make([]AllocatedIP, len(*in)) diff --git a/pkg/ipam/ipam.go b/pkg/ipam/ipam.go index 8b61a39..f60e4a9 100644 --- a/pkg/ipam/ipam.go +++ b/pkg/ipam/ipam.go @@ -17,6 +17,7 @@ package ipam import ( "context" "fmt" + "math" "net" "regexp" "strings" @@ -378,3 +379,100 @@ func (i *Ipam) getIPPools(ctx context.Context) (map[string]*vinov1.IPPoolSpec, e } return ippools, nil } + +func (i *Ipam) AllocateRange(ctx context.Context, + bitStep int, + host, macPrefix, start, stop, subnet string) (vinov1.Range, error) { + ipPool, err := i.getIPPoolWithRanges(ctx, bitStep, macPrefix, start, stop, subnet) + if err != nil { + return vinov1.Range{}, err + } + + result, err := chooseRange(host, ipPool) + if err != nil { + return vinov1.Range{}, err + } + return result, i.applyIPPool(ctx, *ipPool) +} + +func chooseRange(host string, ipPool *vinov1.IPPoolSpec) (vinov1.Range, error) { + const unallocated = -1 + freeIndex := unallocated + for i, r := range ipPool.AllocatedRanges { + if r.AllocatedTo == host { + return r.Range, nil + } else if r.AllocatedTo == "" && freeIndex == unallocated { + freeIndex = i + } + } + if freeIndex != unallocated { + ipPool.AllocatedRanges[freeIndex].AllocatedTo = host + } else { + return vinov1.Range{}, fmt.Errorf("No free ranges available for host %s", host) + } + return ipPool.AllocatedRanges[freeIndex].Range, nil +} + +func (i *Ipam) getIPPoolWithRanges(ctx context.Context, bitStep int, + macPrefix, start, stop, subnet string) (*vinov1.IPPoolSpec, error) { + ippools, err := i.getIPPools(ctx) + if err != nil { + return &vinov1.IPPoolSpec{}, err + } + logger := i.Log.WithValues("subnet", subnet) + + ippool, exists := ippools[subnet] + if !exists { + logger.Info("IPAM creating subnet") + _, err = macStringToInt(macPrefix) // mac format validation + if err != nil { + return &vinov1.IPPoolSpec{}, err + } + ippool = &vinov1.IPPoolSpec{ + Subnet: subnet, + Ranges: []vinov1.Range{}, + AllocatedIPs: []vinov1.AllocatedIP{}, + MACPrefix: macPrefix, + NextMAC: macPrefix, + } + ippools[subnet] = ippool + } + if len(ippool.AllocatedRanges) != 0 { + return ippool, nil + } + + ranges, err := generateRanges(start, stop, bitStep) + if err != nil { + return nil, err + } + ippool.AllocatedRanges = ranges + return ippool, nil +} + +func generateRanges(start, stop string, bitStep int) ([]vinov1.AllocatedRange, error) { + firstNetIPInt, err := ipStringToInt(start) + if err != nil { + return nil, err + } + + // support only IPv4, use 32 netmask + subnetEnd, err := ipStringToInt(stop) + if err != nil { + return nil, err + } + ranges := []vinov1.AllocatedRange{} + shift := uint64(math.Pow(2, float64(bitStep))) + + for start, end := firstNetIPInt, firstNetIPInt+shift; end-1 <= subnetEnd; { + fmt.Printf("start is %s, end is %s\n", intToIPv4String(start), intToIPv4String(subnetEnd)) + ranges = append(ranges, vinov1.AllocatedRange{ + Range: vinov1.Range{ + Start: intToIPv4String(start), + Stop: intToIPv4String(end - 1), + }, + }) + start += shift + end += shift + } + return ranges, nil +} diff --git a/pkg/managers/bmh.go b/pkg/managers/bmh.go index 75b4ffa..e301b10 100644 --- a/pkg/managers/bmh.go +++ b/pkg/managers/bmh.go @@ -169,7 +169,7 @@ func (r *BMHManager) createIpamNetworks(ctx context.Context, vino *vinov1.Vino) } func (r *BMHManager) createIpamNetwork(ctx context.Context, network vinov1.Network) error { - subnetRange, err := ipam.NewRange(network.AllocationStart, network.AllocationStop) + subnetRange, err := ipam.NewRange(network.StaticAllocationStart, network.StaticAllocationStop) if err != nil { return err } @@ -295,6 +295,21 @@ func (r *BMHManager) nodeNetworks(ctx context.Context, builderNetwork.Network.Routes[routeIndex].Gateway = bridgeIP } } + bitStep := network.InstanceSubnetBitStep + if bitStep == 0 { + bitStep = vinov1.VinoDefaultInstanceSubnetBitStep + } + r, err := r.Ipam.AllocateRange(ctx, + bitStep, + k8sNode.Name, + network.MACPrefix, + network.DHCPAllocationStart, + network.DHCPAllocationStop, + network.SubNet) + if err != nil { + return []vinov1.BuilderNetwork{}, err + } + builderNetwork.Range = r builderNetworks = append(builderNetworks, builderNetwork) } return builderNetworks, nil @@ -316,7 +331,7 @@ func (r *BMHManager) domainSpecificNetValues( for _, network := range networks { if network.Name == networkName { subnet = network.SubNet - subnetRange, err = ipam.NewRange(network.AllocationStart, network.AllocationStop) + subnetRange, err = ipam.NewRange(network.StaticAllocationStart, network.StaticAllocationStop) if err != nil { return networkTemplateValues{}, err } @@ -381,7 +396,7 @@ func (r *BMHManager) annotateNode(ctx context.Context, k8sNode *corev1.Node, vin func (r *BMHManager) getBridgeIPandMAC(ctx context.Context, network vinov1.Network, k8sNode *corev1.Node) (string, string, error) { - subnetRange, err := ipam.NewRange(network.AllocationStart, network.AllocationStop) + subnetRange, err := ipam.NewRange(network.StaticAllocationStart, network.StaticAllocationStop) if err != nil { return "", "", err } diff --git a/vino-builder/assets/playbooks/vino-builder.yaml b/vino-builder/assets/playbooks/vino-builder.yaml index 5084bfe..f72708e 100644 --- a/vino-builder/assets/playbooks/vino-builder.yaml +++ b/vino-builder/assets/playbooks/vino-builder.yaml @@ -39,11 +39,6 @@ - hosts: localhost tasks: - # generate libvirt definitions for storage, networks, and domains - - name: generate management network ip addresses - include_role: - name: ipam - # generate libvirt definitions for storage, networks, and domains - name: process libvirt definitions include_role: