diff --git a/config/crd/bases/airship.airshipit.org_vinoes.yaml b/config/crd/bases/airship.airshipit.org_vinoes.yaml index d59b06d..6953b21 100644 --- a/config/crd/bases/airship.airshipit.org_vinoes.yaml +++ b/config/crd/bases/airship.airshipit.org_vinoes.yaml @@ -86,6 +86,10 @@ spec: type: string allocationStop: type: string + bridgeName: + description: BridgeName is the name of the bridge to be created + as libvirt network. works if AllocateNodeIP is sepcified + type: string dns_servers: items: type: string @@ -239,13 +243,8 @@ spec: description: PXEBootImageHostPort will be used to download the PXE boot image type: integer - vmBridge: - description: VMBridge defines the single interface name to be used - as a bridge for VMs - type: string required: - bmcCredentials - - vmBridge type: object status: description: VinoStatus defines the observed state of Vino diff --git a/config/manager/daemonset-template.yaml b/config/manager/daemonset-template.yaml index 0e584c8..7c7b5ef 100644 --- a/config/manager/daemonset-template.yaml +++ b/config/manager/daemonset-template.yaml @@ -108,14 +108,14 @@ spec: # host: 127.0.0.1 # initialDelaySeconds: 30 # periodSeconds: 30 - - name: labeler - image: quay.io/airshipit/nodelabeler - imagePullPolicy: IfNotPresent - env: - - name: NODE - valueFrom: - fieldRef: - fieldPath: spec.nodeName + # - name: labeler + # image: quay.io/airshipit/nodelabeler + # imagePullPolicy: IfNotPresent + # env: + # - name: NODE + # valueFrom: + # fieldRef: + # fieldPath: spec.nodeName - name: vino-builder readinessProbe: initialDelaySeconds: 20 diff --git a/config/manager/network-templates.yaml b/config/manager/network-templates.yaml index 8414113..d3f13c8 100644 --- a/config/manager/network-templates.yaml +++ b/config/manager/network-templates.yaml @@ -4,8 +4,8 @@ libvirtNetworks: {{ network.name }} - - + + diff --git a/config/samples/vino_cr.yaml b/config/samples/vino_cr.yaml index 7079976..69b1b1b 100644 --- a/config/samples/vino_cr.yaml +++ b/config/samples/vino_cr.yaml @@ -4,7 +4,6 @@ metadata: name: vino-test-cr labels: {} spec: - vmBridge: vm-infra-bridge nodeLabelKeysToCopy: - "airshipit.org/server" - "airshipit.org/rack" @@ -17,13 +16,14 @@ spec: - name: management libvirtTemplate: management subnet: 192.168.2.0/20 - instanceSubnet: 192.168.2.0/22 + instanceSubnet: 192.168.4.0/22 type: bridge allocationStart: 192.168.2.10 - allocationStop: 192.168.2.14 # docs should specify that the range should = number of vms (to permit future expansion over multiple vino crs etc) + allocationStop: 192.168.2.24 # docs should specify that the range should = number of vms (to permit future expansion over multiple vino crs etc) dns_servers: ["135.188.34.124"] macPrefix: "52:54:00:06:00:00" physicalInterface: enp3s7 + bridgeName: vm-infra nodes: - name: master count: 1 diff --git a/config/samples/vino_cr_4_workers_1_cp.yaml b/config/samples/vino_cr_4_workers_1_cp.yaml index 7d72c29..64106aa 100644 --- a/config/samples/vino_cr_4_workers_1_cp.yaml +++ b/config/samples/vino_cr_4_workers_1_cp.yaml @@ -4,7 +4,6 @@ metadata: name: vino-test-cr labels: {} spec: - vmBridge: vm-infra-bridge nodeLabelKeysToCopy: - "airshipit.org/server" - "airshipit.org/rack" @@ -17,10 +16,10 @@ spec: - name: management libvirtTemplate: management subnet: 192.168.2.0/20 - instanceSubnet: 192.168.2.0/22 + instanceSubnet: 192.168.4.0/22 type: ipv4 allocationStart: 192.168.2.10 - allocationStop: 192.168.2.14 # docs should specify that the range should = number of vms (to permit future expansion over multiple vino crs etc) + allocationStop: 192.168.2.25 # docs should specify that the range should = number of vms (to permit future expansion over multiple vino crs etc) routes: - network: 10.0.0.0 netmask: 255.255.255.0 @@ -28,6 +27,7 @@ spec: dns_servers: ["135.188.34.124"] macPrefix: "52:54:00:06:00:00" physicalInterface: enp3s7 + bridgeName: vm-infra nodes: - name: master count: 1 diff --git a/docs/api/vino.md b/docs/api/vino.md index 1f48a5c..ae0ae1c 100644 --- a/docs/api/vino.md +++ b/docs/api/vino.md @@ -149,8 +149,8 @@ int networks
- -[]Network + +[]BuilderNetwork @@ -245,6 +245,26 @@ string +enableVNC
+ +bool + + + + + + + +vncPassword
+ +string + + + + + + + interfaces
@@ -259,6 +279,73 @@ string +

BuilderNetwork +

+

+(Appears on: +Builder) +

+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+bridgeIP
+ +string + +		 +
+bridgeMAC
+ +string + +		 +
+range
+ + +Range + + +		 +
+Network
+ + +Network + + +		 +

+(Members of Network are embedded into this type.) +

+
+
+

BuilderNetworkInterface

@@ -774,7 +861,7 @@ string

(Appears on: -Builder, +BuilderNetwork, VinoSpec)

Network defines libvirt networks

@@ -909,6 +996,18 @@ string

LibvirtTemplate identifies which libvirt template to be used to create a network

+ + +bridgeName
+ +string + + + +

BridgeName is the name of the bridge to be created as libvirt network. +works if AllocateNodeIP is sepcified

+ + @@ -1140,6 +1239,17 @@ string

BootInterfaceName interface name to use to boot virtual machines

+ + +enableVNC
+ +bool + + + +

EnableVNC create VNC for graphical interaction with the VM that will be created.

+ + @@ -1148,6 +1258,7 @@ string

(Appears on: +BuilderNetwork, IPPoolSpec)

Range has (inclusive) bounds within a subnet from which IPs can be allocated

@@ -1343,17 +1454,6 @@ DaemonSetOptions -vmBridge
- -string - - - -

VMBridge defines the single interface name to be used as a bridge for VMs

- - - - bmcCredentials
@@ -1503,17 +1603,6 @@ DaemonSetOptions -vmBridge
- -string - - - -

VMBridge defines the single interface name to be used as a bridge for VMs

- - - - bmcCredentials
 diff --git a/pkg/api/v1/vino_builder.go b/pkg/api/v1/vino_builder.go index 2aff3e9..939853f 100644 --- a/pkg/api/v1/vino_builder.go +++ b/pkg/api/v1/vino_builder.go @@ -22,7 +22,7 @@ type Builder struct { PXEBootImageHost string `json:"pxeBootImageHost,omitempty"` PXEBootImageHostPort int `json:"pxeBootImageHostPort,omitempty"` - Networks []Network `json:"networks,omitempty"` + Networks []BuilderNetwork `json:"networks,omitempty"` // (TODO) change json tag to cpuConfiguration when vino-builder has these chanages as well CPUConfiguration CPUConfiguration `json:"configuration,omitempty"` Domains []BuilderDomain `json:"domains,omitempty"` @@ -35,6 +35,13 @@ type BuilderNetworkInterface struct { NetworkInterface `json:",inline"` } +type BuilderNetwork struct { + BridgeIP string `json:"bridgeIP,omitempty"` + BridgeMAC string `json:"bridgeMAC,omitempty"` + Range Range `json:"range,omitempty"` + Network `json:",inline"` +} + // BuilderDomain represents a VINO libvirt domain type BuilderDomain struct { Name string `json:"name,omitempty"` diff --git a/pkg/api/v1/vino_types.go b/pkg/api/v1/vino_types.go index bedb74f..d0a9bd7 100644 --- a/pkg/api/v1/vino_types.go +++ b/pkg/api/v1/vino_types.go @@ -63,8 +63,6 @@ type VinoSpec struct { Nodes []NodeSet `json:"nodes,omitempty"` // DaemonSetOptions defines how vino will spawn daemonset on nodes DaemonSetOptions DaemonSetOptions `json:"daemonSetOptions,omitempty"` - // VMBridge defines the single interface name to be used as a bridge for VMs - VMBridge string `json:"vmBridge"` // BMCCredentials contain credentials that will be used to create BMH nodes // sushy tools will use these credentials as well, to set up authentication BMCCredentials BMCCredentials `json:"bmcCredentials"` @@ -118,6 +116,9 @@ type Network struct { PhysicalInterface string `json:"physicalInterface,omitempty"` // LibvirtTemplate identifies which libvirt template to be used to create a network LibvirtTemplate string `json:"libvirtTemplate,omitempty"` + // BridgeName is the name of the bridge to be created as libvirt network. + // works if AllocateNodeIP is sepcified + BridgeName string `json:"bridgeName,omitempty"` } // VMRoutes defined diff --git a/pkg/api/v1/zz_generated.deepcopy.go b/pkg/api/v1/zz_generated.deepcopy.go index b6a2a02..fac2701 100644 --- a/pkg/api/v1/zz_generated.deepcopy.go +++ b/pkg/api/v1/zz_generated.deepcopy.go @@ -60,7 +60,7 @@ func (in *Builder) DeepCopyInto(out *Builder) { *out = *in if in.Networks != nil { in, out := &in.Networks, &out.Networks - *out = make([]Network, len(*in)) + *out = make([]BuilderNetwork, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -107,6 +107,23 @@ func (in *BuilderDomain) DeepCopy() *BuilderDomain { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BuilderNetwork) DeepCopyInto(out *BuilderNetwork) { + *out = *in + out.Range = in.Range + in.Network.DeepCopyInto(&out.Network) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BuilderNetwork. +func (in *BuilderNetwork) DeepCopy() *BuilderNetwork { + if in == nil { + return nil + } + out := new(BuilderNetwork) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BuilderNetworkInterface) DeepCopyInto(out *BuilderNetworkInterface) { *out = *in diff --git a/pkg/controllers/vino_controller.go b/pkg/controllers/vino_controller.go index 61fe9b3..3202ae4 100644 --- a/pkg/controllers/vino_controller.go +++ b/pkg/controllers/vino_controller.go @@ -229,11 +229,6 @@ func (r *VinoReconciler) decorateDaemonSet(ctx context.Context, ds *appsv1.Daemo ds.Namespace = getRuntimeNamespace() ds.Name = r.getDaemonSetName(vino) - // TODO develop logic to derive all required ENV variables from VINO CR, and pass them - // to setENV function instead - if vino.Spec.VMBridge != "" { - setEnv(ctx, ds, vinov1.EnvVarVMInterfaceName, vino.Spec.VMBridge) - } setEnv(ctx, ds, vinov1.EnvVarBasicAuthUsername, vino.Spec.BMCCredentials.Username) setEnv(ctx, ds, vinov1.EnvVarBasicAuthPassword, vino.Spec.BMCCredentials.Password) diff --git a/pkg/managers/bmh.go b/pkg/managers/bmh.go index bc0ccc5..0fb0709 100644 --- a/pkg/managers/bmh.go +++ b/pkg/managers/bmh.go @@ -19,7 +19,6 @@ import ( "context" "fmt" "text/template" - "time" "github.com/Masterminds/sprig" "github.com/go-logr/logr" @@ -45,7 +44,7 @@ type networkTemplateValues struct { BMHName string Node vinov1.NodeSet // the specific node type to be templated - Networks []vinov1.Network + Networks []vinov1.BuilderNetwork vinov1.BuilderDomain } @@ -264,7 +263,7 @@ func (r *BMHManager) setBMHs(ctx context.Context, pod corev1.Pod, nodeCount int) vinoBuilder := vinov1.Builder{ PXEBootImageHost: r.ViNO.Spec.PXEBootImageHost, PXEBootImageHostPort: r.ViNO.Spec.PXEBootImageHostPort, - Networks: r.ViNO.Spec.Networks, + Networks: nodeNetworks, CPUConfiguration: r.ViNO.Spec.CPUConfiguration, Domains: domains, NodeCount: nodeCount, @@ -275,27 +274,36 @@ func (r *BMHManager) setBMHs(ctx context.Context, pod corev1.Pod, nodeCount int) // nodeNetworks returns a copy of node network with a unique per node values func (r *BMHManager) nodeNetworks(ctx context.Context, globalNetworks []vinov1.Network, - k8sNode *corev1.Node) ([]vinov1.Network, error) { - for netIndex, network := range globalNetworks { - for routeIndex, route := range network.Routes { + k8sNode *corev1.Node) ([]vinov1.BuilderNetwork, error) { + builderNetworks := []vinov1.BuilderNetwork{} + for _, network := range globalNetworks { + builderNetwork := vinov1.BuilderNetwork{ + Network: network, + } + + r.Logger.Info("Getting GW bridge IP from node", "node", k8sNode.Name) + bridgeIP, mac, err := r.getBridgeIPandMAC(ctx, network, k8sNode) + if err != nil { + return []vinov1.BuilderNetwork{}, err + } + builderNetwork.BridgeIP = bridgeIP + builderNetwork.BridgeMAC = mac + + for routeIndex, route := range builderNetwork.Network.Routes { if route.Gateway == "$vinobridge" { - r.Logger.Info("Getting GW bridge IP from node", "node", k8sNode.Name) - bridgeIP, err := r.getBridgeIP(ctx, k8sNode) - if err != nil { - return []vinov1.Network{}, err - } - globalNetworks[netIndex].Routes[routeIndex].Gateway = bridgeIP + builderNetwork.Network.Routes[routeIndex].Gateway = bridgeIP } } + builderNetworks = append(builderNetworks, builderNetwork) } - return globalNetworks, nil + return builderNetworks, nil } func (r *BMHManager) domainSpecificNetValues( ctx context.Context, bmhName string, node vinov1.NodeSet, - networks []vinov1.Network) (networkTemplateValues, error) { + networks []vinov1.BuilderNetwork) (networkTemplateValues, error) { // Allocate an IP for each of this BMH's network interfaces bootMAC := "" domainInterfaces := []vinov1.BuilderNetworkInterface{} @@ -364,31 +372,14 @@ func (r *BMHManager) annotateNode(ctx context.Context, k8sNode *corev1.Node, vin return r.Update(ctx, k8sNode) } -func (r *BMHManager) getBridgeIP(ctx context.Context, k8sNode *corev1.Node) (string, error) { - ctxTimeout, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - - for { - select { - case <-ctxTimeout.Done(): - return "", ctx.Err() - default: - node := &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: k8sNode.Name, - }, - } - if err := r.Get(ctx, client.ObjectKeyFromObject(node), node); err != nil { - return "", err - } - - ip, exist := k8sNode.Labels[vinov1.VinoDefaultGatewayBridgeLabel] - if exist { - return ip, nil - } - time.Sleep(10 * time.Second) - } +func (r *BMHManager) getBridgeIPandMAC(ctx context.Context, + network vinov1.Network, + k8sNode *corev1.Node) (string, string, error) { + subnetRange, err := ipam.NewRange(network.AllocationStart, network.AllocationStop) + if err != nil { + return "", "", err } + return r.Ipam.AllocateIP(ctx, network.SubNet, subnetRange, k8sNode.Name) } func (r *BMHManager) getNode(ctx context.Context, pod corev1.Pod) (*corev1.Node, error) { @@ -397,8 +388,7 @@ func (r *BMHManager) getNode(ctx context.Context, pod corev1.Pod) (*corev1.Node, Name: pod.Spec.NodeName, }, } - err := r.Get(ctx, client.ObjectKeyFromObject(node), node) - return node, err + return node, r.Get(ctx, client.ObjectKeyFromObject(node), node) } func (r *BMHManager) getBMHNodePrefix(pod corev1.Pod) string {