diff --git a/config/crd/bases/airship.airshipit.org_sipclusters.yaml b/config/crd/bases/airship.airshipit.org_sipclusters.yaml index d6498a7..d2333af 100644 --- a/config/crd/bases/airship.airshipit.org_sipclusters.yaml +++ b/config/crd/bases/airship.airshipit.org_sipclusters.yaml @@ -178,10 +178,36 @@ spec: - nodeSSHPrivateKeys type: object type: array - loadBalancer: + loadBalancerControlPlane: description: LoadBalancer defines the sub-cluster load balancer services. items: + description: LoadBalancerServiceControlPlane is an infrastructure + service type that represents the sub-cluster load balancer service. + properties: + clusterIP: + type: string + image: + type: string + nodeInterfaceId: + type: string + nodeLabels: + additionalProperties: + type: string + type: object + nodePort: + type: integer + required: + - image + - nodePort + type: object + type: array + loadBalancerWorker: + description: ' LoadBalancer defines the sub-cluster load balancer + services.' + items: + description: LoadBalancerServiceWorker is an infrastructure service + type that represents the sub-cluster load balancer service. properties: clusterIP: type: string diff --git a/config/rbac/sipcluster_scheduler_role.yaml b/config/rbac/sipcluster_scheduler_role.yaml index 5f575e3..710d157 100644 --- a/config/rbac/sipcluster_scheduler_role.yaml +++ b/config/rbac/sipcluster_scheduler_role.yaml @@ -17,8 +17,10 @@ rules: - update - apiGroups: - "" + - apps resources: - secrets + - deployments verbs: - get - list diff --git a/config/samples/airship_v1beta1_sipcluster.yaml b/config/samples/airship_v1beta1_sipcluster.yaml index d1b3c54..ab614ae 100644 --- a/config/samples/airship_v1beta1_sipcluster.yaml +++ b/config/samples/airship_v1beta1_sipcluster.yaml @@ -8,19 +8,19 @@ metadata: spec: nodes: ControlPlane: - labelSelector: - vino.airshipit.org/flavor: control-plane + labelSelector: + vino.airshipit.org/flavor: control-plane topologyKey: vino.airshipit.org/rack - count: - active: 1 - standby: 1 + count: + active: 1 + standby: 1 Worker: - labelSelector: - vino.airshipit.org/flavor: worker - topologyKey: vino.airshipit.org/host - count: - active: 1 - standby: 1 # Slew for upgrades + labelSelector: + vino.airshipit.org/flavor: worker + topologyKey: vino.airshipit.org/host + count: + active: 1 + standby: 1 # Slew for upgrades services: # NOTE: The auth service has not yet been implemented. # auth: @@ -36,7 +36,7 @@ spec: # NOTE: nodeLabels not yet implemented. # nodeLabels: # kubernetes.io/os: linux - nodePort: 30001 + nodePort: 30000 nodeInterfaceId: oam-ipv4 # NOTE: clusterIP has not yet been implemented. # clusterIP: 1.2.3.4 # IP of the base cluster VIP @@ -46,13 +46,21 @@ spec: - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCyaozS8kZRw2a1d0O4YXhxtJlDPThqIZilGCsXLbukIFOyMUmMTwQAtwWp5epwU1+5ponC2uBENB6xCCj3cl5Rd43d2/B6HxyAPQGKo6/zKYGAKW2nzYDxSWMl6NUSsiJAyXUA7ZlNZQe0m8PmaferlkQyLLZo3NJpizz6U6ZCtxvj43vEl7NYWnLUEIzGP9zMqltIGnD4vYrU9keVKKXSsp+DkApnbrDapeigeGATCammy2xRrUQDuOvGHsfnQbXr2j0onpTIh0PiLrXLQAPDg8UJRgVB+ThX+neI3rQ320djzRABckNeE6e4Kkwzn+QdZsmA2SDvM9IU7boK1jVQlgUPp7zF5q3hbb8Rx7AadyTarBayUkCgNlrMqth+tmTMWttMqCPxJRGnhhvesAHIl55a28Kzz/2Oqa3J9zwzbyDIwlEXho0eAq3YXEPeBhl34k+7gOt/5Zdbh+yacFoxDh0LrshQgboAijcVVaXPeN0LsHEiVvYIzugwIvCkoFMPWoPj/kEGzPY6FCkVneDA7VoLTCoG8dlrN08Lf05/BGC7Wllm66pTNZC/cKXP+cjpQn1iEuiuPxnPldlMHx9sx2y/BRoft6oT/GzqkNy1NTY/xI+MfmxXnF5kwSbcTbzZQ9fZ8xjh/vmpPBgDNrxOEAT4N6OG7GQIhb9HEhXQCQ== example-key - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwpOyZjZ4gB0OTvmofH3llh6cBCWaEiEmHZWSkDXr8Bih6HcXVOtYMcFi/ZnUVGUBPw3ATNQBZUaVCYKeF+nDfKTJ9hmnlsyHxV2LeMsVg1o15Pb6f+QJuavEqtE6HI7mHyId4Z1quVTJXDWDW8OZEG7M3VktauqAn/e9UJvlL0bGmTFD1XkNcbRsWMRWkQgt2ozqlgrpPtvrg2/+bNucxX++VUjnsn+fGgAT07kbnrZwppGnAfjbYthxhv7GeSD0+Z0Lf1kiKy/bhUqXsZIuexOfF0YrRyUH1KBl8GCX2OLBYvXHyusByqsrOPiROqRdjX5PsK6HSAS0lk0niTt1p example-key-2 nodeSSHPrivateKeys: ssh-private-keys - loadBalancer: + loadBalancerControlPlane: - image: haproxy:2.3.2 # NOTE: nodeLabels not yet implemented. # nodeLabels: - # kubernetes.io/os: linux - nodePort: 30000 + # kubernetes.io/os + nodePort: 30001 + nodeInterfaceId: oam-ipv4 + # NOTE: clusterIP has not yet been implemented. + # clusterIP: 1.2.3.4 # IP of the base cluster VIP + loadBalancerWorker: + - image: haproxy:2.3.2 + # NOTE: nodeLabels not yet implemented. + # nodeLabels: + # kubernetes.io/os + nodePort: 30002 nodeInterfaceId: oam-ipv4 # NOTE: clusterIP has not yet been implemented. # clusterIP: 1.2.3.4 # IP of the base cluster VIP - diff --git a/docs/api/sipcluster.md b/docs/api/sipcluster.md index 7acf490..9fadddd 100644 --- a/docs/api/sipcluster.md +++ b/docs/api/sipcluster.md @@ -116,6 +116,78 @@ directory, and then configured as identity files in the SSH config file of the d +

LoadBalancerServiceControlPlane +

+

+(Appears on: +SIPClusterServices) +

+

LoadBalancerServiceControlPlane is an infrastructure service type that represents the sub-cluster load balancer service.

+
+
+ + + + + + + + + + + + + +
FieldDescription
+SIPClusterService
+ + +SIPClusterService + + +		 +

+(Members of SIPClusterService are embedded into this type.) +

+
+
+
+

LoadBalancerServiceWorker +

+

+(Appears on: +SIPClusterServices) +

+

LoadBalancerServiceWorker is an infrastructure service type that represents the sub-cluster load balancer service.

+
+
+ + + + + + + + + + + + + +
FieldDescription
+SIPClusterService
+ + +SIPClusterService + + +		 +

+(Members of SIPClusterService are embedded into this type.) +

+
+
+

NodeCount

@@ -317,6 +389,8 @@ SIPClusterStatus

(Appears on: JumpHostService, +LoadBalancerServiceControlPlane, +LoadBalancerServiceWorker, SIPClusterServices)

@@ -402,10 +476,23 @@ string -loadBalancer
+loadBalancerControlPlane
- -[]SIPClusterService + +[]LoadBalancerServiceControlPlane + + + + +

LoadBalancer defines the sub-cluster load balancer services.

+ + + + +loadBalancerWorker
+ + +[]LoadBalancerServiceWorker diff --git a/pkg/api/v1/sipcluster_types.go b/pkg/api/v1/sipcluster_types.go index b87b6f0..f97c8ce 100644 --- a/pkg/api/v1/sipcluster_types.go +++ b/pkg/api/v1/sipcluster_types.go @@ -53,7 +53,10 @@ type SIPClusterSpec struct { // SIPClusterServices defines the services that are deployed when a SIPCluster is provisioned. type SIPClusterServices struct { // LoadBalancer defines the sub-cluster load balancer services. - LoadBalancer []SIPClusterService `json:"loadBalancer,omitempty"` + LoadBalancerControlPlane []LoadBalancerServiceControlPlane `json:"loadBalancerControlPlane,omitempty"` + // LoadBalancer defines the sub-cluster load balancer services. + LoadBalancerWorker []LoadBalancerServiceWorker `json:"loadBalancerWorker,omitempty"` + // Auth defines the sub-cluster authentication services. Auth []SIPClusterService `json:"auth,omitempty"` // JumpHost defines the sub-cluster jump host services. @@ -62,8 +65,11 @@ type SIPClusterServices struct { func (s SIPClusterServices) GetAll() []SIPClusterService { all := []SIPClusterService{} - for _, s := range s.LoadBalancer { - all = append(all, s) + for _, s := range s.LoadBalancerControlPlane { + all = append(all, s.SIPClusterService) + } + for _, s := range s.LoadBalancerWorker { + all = append(all, s.SIPClusterService) } for _, s := range s.Auth { all = append(all, s) @@ -86,6 +92,18 @@ type JumpHostService struct { NodeSSHPrivateKeys string `json:"nodeSSHPrivateKeys"` } +/* +LoadBalancerServiceControlPlane is an infrastructure service type that represents the sub-cluster load balancer service. +*/ +type LoadBalancerServiceControlPlane struct { + SIPClusterService `json:",inline"` +} + +// LoadBalancerServiceWorker is an infrastructure service type that represents the sub-cluster load balancer service. +type LoadBalancerServiceWorker struct { + SIPClusterService `json:",inline"` +} + // SIPClusterStatus defines the observed state of SIPCluster type SIPClusterStatus struct { Conditions []metav1.Condition `json:"conditions,omitempty"` diff --git a/pkg/api/v1/zz_generated.deepcopy.go b/pkg/api/v1/zz_generated.deepcopy.go index 6a731f9..6438ea9 100644 --- a/pkg/api/v1/zz_generated.deepcopy.go +++ b/pkg/api/v1/zz_generated.deepcopy.go @@ -66,6 +66,38 @@ func (in *JumpHostService) DeepCopy() *JumpHostService { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancerServiceControlPlane) DeepCopyInto(out *LoadBalancerServiceControlPlane) { + *out = *in + in.SIPClusterService.DeepCopyInto(&out.SIPClusterService) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerServiceControlPlane. +func (in *LoadBalancerServiceControlPlane) DeepCopy() *LoadBalancerServiceControlPlane { + if in == nil { + return nil + } + out := new(LoadBalancerServiceControlPlane) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancerServiceWorker) DeepCopyInto(out *LoadBalancerServiceWorker) { + *out = *in + in.SIPClusterService.DeepCopyInto(&out.SIPClusterService) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerServiceWorker. +func (in *LoadBalancerServiceWorker) DeepCopy() *LoadBalancerServiceWorker { + if in == nil { + return nil + } + out := new(LoadBalancerServiceWorker) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeCount) DeepCopyInto(out *NodeCount) { *out = *in @@ -191,9 +223,16 @@ func (in *SIPClusterService) DeepCopy() *SIPClusterService { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SIPClusterServices) DeepCopyInto(out *SIPClusterServices) { *out = *in - if in.LoadBalancer != nil { - in, out := &in.LoadBalancer, &out.LoadBalancer - *out = make([]SIPClusterService, len(*in)) + if in.LoadBalancerControlPlane != nil { + in, out := &in.LoadBalancerControlPlane, &out.LoadBalancerControlPlane + *out = make([]LoadBalancerServiceControlPlane, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.LoadBalancerWorker != nil { + in, out := &in.LoadBalancerWorker, &out.LoadBalancerWorker + *out = make([]LoadBalancerServiceWorker, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } diff --git a/pkg/bmh/bmh_test.go b/pkg/bmh/bmh_test.go index c1fcece..32d316d 100644 --- a/pkg/bmh/bmh_test.go +++ b/pkg/bmh/bmh_test.go @@ -128,14 +128,28 @@ var _ = Describe("MachineList", func() { sipCluster, nodeSSHPrivateKeys := testutil.CreateSIPCluster("subcluster-1", "default", 1, 3) sipCluster.Spec.Services = airshipv1.SIPClusterServices{ - LoadBalancer: []airshipv1.SIPClusterService{ + LoadBalancerControlPlane: []airshipv1.LoadBalancerServiceControlPlane{ { - Image: "haproxy:latest", - NodeLabels: map[string]string{ - "test": "true", + SIPClusterService: airshipv1.SIPClusterService{ + Image: "haproxy:latest", + NodeLabels: map[string]string{ + "test": "true", + }, + NodePort: 30001, + NodeInterface: "oam-ipv4", + }, + }, + }, + LoadBalancerWorker: []airshipv1.LoadBalancerServiceWorker{ + { + SIPClusterService: airshipv1.SIPClusterService{ + Image: "haproxy:latest", + NodeLabels: map[string]string{ + "test": "true", + }, + NodePort: 30002, + NodeInterface: "oam-ipv4", }, - NodePort: 30000, - NodeInterface: "oam-ipv4", }, }, } @@ -179,14 +193,28 @@ var _ = Describe("MachineList", func() { sipCluster, nodeSSHPrivateKeys := testutil.CreateSIPCluster("subcluster-1", "default", 1, 3) sipCluster.Spec.Services = airshipv1.SIPClusterServices{ - LoadBalancer: []airshipv1.SIPClusterService{ + LoadBalancerControlPlane: []airshipv1.LoadBalancerServiceControlPlane{ { - Image: "haproxy:latest", - NodeLabels: map[string]string{ - "test": "true", + SIPClusterService: airshipv1.SIPClusterService{ + Image: "haproxy:latest", + NodeLabels: map[string]string{ + "test": "true", + }, + NodePort: 30001, + NodeInterface: "oam-ipv4", + }, + }, + }, + LoadBalancerWorker: []airshipv1.LoadBalancerServiceWorker{ + { + SIPClusterService: airshipv1.SIPClusterService{ + Image: "haproxy:latest", + NodeLabels: map[string]string{ + "test": "true", + }, + NodePort: 30002, + NodeInterface: "oam-ipv4", }, - NodePort: 30000, - NodeInterface: "oam-ipv4", }, }, } @@ -228,14 +256,28 @@ var _ = Describe("MachineList", func() { sipCluster, nodeSSHPrivateKeys := testutil.CreateSIPCluster("subcluster-1", "default", 1, 3) sipCluster.Spec.Services = airshipv1.SIPClusterServices{ - LoadBalancer: []airshipv1.SIPClusterService{ + LoadBalancerControlPlane: []airshipv1.LoadBalancerServiceControlPlane{ { - Image: "haproxy:latest", - NodeLabels: map[string]string{ - "test": "true", + SIPClusterService: airshipv1.SIPClusterService{ + Image: "haproxy:latest", + NodeLabels: map[string]string{ + "test": "true", + }, + NodePort: 30001, + NodeInterface: "oam-ipv4", + }, + }, + }, + LoadBalancerWorker: []airshipv1.LoadBalancerServiceWorker{ + { + SIPClusterService: airshipv1.SIPClusterService{ + Image: "haproxy:latest", + NodeLabels: map[string]string{ + "test": "true", + }, + NodePort: 30002, + NodeInterface: "oam-ipv4", }, - NodePort: 30000, - NodeInterface: "oam-ipv4", }, }, } @@ -281,14 +323,28 @@ var _ = Describe("MachineList", func() { sipCluster, nodeSSHPrivateKeys := testutil.CreateSIPCluster("subcluster-1", "default", 1, 3) sipCluster.Spec.Services = airshipv1.SIPClusterServices{ - LoadBalancer: []airshipv1.SIPClusterService{ + LoadBalancerControlPlane: []airshipv1.LoadBalancerServiceControlPlane{ { - Image: "haproxy:latest", - NodeLabels: map[string]string{ - "test": "true", + SIPClusterService: airshipv1.SIPClusterService{ + Image: "haproxy:latest", + NodeLabels: map[string]string{ + "test": "true", + }, + NodePort: 30001, + NodeInterface: "oam-ipv4", + }, + }, + }, + LoadBalancerWorker: []airshipv1.LoadBalancerServiceWorker{ + { + SIPClusterService: airshipv1.SIPClusterService{ + Image: "haproxy:latest", + NodeLabels: map[string]string{ + "test": "true", + }, + NodePort: 30002, + NodeInterface: "oam-ip4", }, - NodePort: 30000, - NodeInterface: "oam-ipv4", }, }, } @@ -328,14 +384,28 @@ var _ = Describe("MachineList", func() { sipCluster, nodeSSHPrivateKeys := testutil.CreateSIPCluster("subcluster-1", "default", 1, 3) sipCluster.Spec.Services = airshipv1.SIPClusterServices{ - LoadBalancer: []airshipv1.SIPClusterService{ + LoadBalancerControlPlane: []airshipv1.LoadBalancerServiceControlPlane{ { - Image: "haproxy:latest", - NodeLabels: map[string]string{ - "test": "true", + SIPClusterService: airshipv1.SIPClusterService{ + Image: "haproxy:latest", + NodeLabels: map[string]string{ + "test": "true", + }, + NodePort: 30001, + NodeInterface: "oam-ipv4", + }, + }, + }, + LoadBalancerWorker: []airshipv1.LoadBalancerServiceWorker{ + { + SIPClusterService: airshipv1.SIPClusterService{ + Image: "haproxy:latest", + NodeLabels: map[string]string{ + "test": "true", + }, + NodePort: 30002, + NodeInterface: "oam-ipv4", }, - NodePort: 30000, - NodeInterface: "oam-ipv4", }, }, } @@ -374,14 +444,28 @@ var _ = Describe("MachineList", func() { sipCluster, nodeSSHPrivateKeys := testutil.CreateSIPCluster("subcluster-1", "default", 1, 3) sipCluster.Spec.Services = airshipv1.SIPClusterServices{ - LoadBalancer: []airshipv1.SIPClusterService{ + LoadBalancerControlPlane: []airshipv1.LoadBalancerServiceControlPlane{ { - Image: "haproxy:latest", - NodeLabels: map[string]string{ - "test": "true", + SIPClusterService: airshipv1.SIPClusterService{ + Image: "haproxy:latest", + NodeLabels: map[string]string{ + "test": "true", + }, + NodePort: 30001, + NodeInterface: "oam-ipv4", + }, + }, + }, + LoadBalancerWorker: []airshipv1.LoadBalancerServiceWorker{ + { + SIPClusterService: airshipv1.SIPClusterService{ + Image: "haproxy:latest", + NodeLabels: map[string]string{ + "test": "true", + }, + NodePort: 30002, + NodeInterface: "oam-ipv4", }, - NodePort: 30000, - NodeInterface: "oam-ipv4", }, }, } diff --git a/pkg/services/loadbalancer.go b/pkg/services/loadbalancer.go index da15139..0165b7b 100644 --- a/pkg/services/loadbalancer.go +++ b/pkg/services/loadbalancer.go @@ -16,6 +16,7 @@ package services import ( "bytes" + "strings" "html/template" airshipv1 "sipcluster/pkg/api/v1" @@ -44,7 +45,7 @@ func (lb loadBalancer) Deploy() error { lb.config.Image = DefaultBalancerImage } - instance := LoadBalancerServiceName + "-" + lb.sipName.Name + instance := LoadBalancerServiceName + "-" + strings.ToLower(string(lb.bmhRole)) + "-" + lb.sipName.Name labels := map[string]string{ // See https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/#labels "app.kubernetes.io/part-of": "sip", @@ -149,7 +150,7 @@ func (lb loadBalancer) generateSecret(instance string) (*corev1.Secret, error) { Backends: make([]backend, 0), } for _, machine := range lb.machines.Machines { - if machine.BMHRole == airshipv1.RoleControlPlane { + if machine.BMHRole == lb.bmhRole { name := machine.BMH.Name namespace := machine.BMH.Namespace ip, exists := machine.Data.IPOnInterface[lb.config.NodeInterface] @@ -163,7 +164,7 @@ func (lb loadBalancer) generateSecret(instance string) (*corev1.Secret, error) { p.Backends = append(p.Backends, backend{IP: ip, Name: machine.BMH.Name, Port: 6443}) } } - secretData, err := generateTemplate(p) + secretData, err := lb.generateTemplate(p) if err != nil { return nil, err } @@ -216,22 +217,59 @@ type loadBalancer struct { logger logr.Logger config airshipv1.SIPClusterService machines *bmh.MachineList + bmhRole airshipv1.BMHRole + template string } -func newLB(name, namespace string, +type loadBalancerControlPlane struct { + loadBalancer + config airshipv1.LoadBalancerServiceControlPlane +} + +type loadBalancerWorker struct { + loadBalancer + config airshipv1.LoadBalancerServiceWorker +} + +func newLBControlPlane(name, namespace string, logger logr.Logger, - config airshipv1.SIPClusterService, + config airshipv1.LoadBalancerServiceControlPlane, machines *bmh.MachineList, - client client.Client) loadBalancer { - return loadBalancer{ + client client.Client) loadBalancerControlPlane { + return loadBalancerControlPlane{loadBalancer{ sipName: types.NamespacedName{ Name: name, Namespace: namespace, }, logger: logger, - config: config, + config: config.SIPClusterService, machines: machines, client: client, + bmhRole: airshipv1.RoleControlPlane, + template: templateControlPlane, + }, + config, + } +} + +func newLBWorker(name, namespace string, + logger logr.Logger, + config airshipv1.LoadBalancerServiceWorker, + machines *bmh.MachineList, + client client.Client) loadBalancerWorker { + return loadBalancerWorker{loadBalancer{ + sipName: types.NamespacedName{ + Name: name, + Namespace: namespace, + }, + logger: logger, + config: config.SIPClusterService, + machines: machines, + client: client, + bmhRole: airshipv1.RoleWorker, + template: templateWorker, + }, + config, } } @@ -240,8 +278,8 @@ func (lb loadBalancer) Finalize() error { return nil } -func generateTemplate(p proxy) ([]byte, error) { - tmpl, err := template.New("haproxy-config").Parse(defaultTemplate) +func (lb loadBalancer) generateTemplate(p proxy) ([]byte, error) { + tmpl, err := template.New("haproxy-config").Parse(lb.template) if err != nil { return nil, err } @@ -255,7 +293,7 @@ func generateTemplate(p proxy) ([]byte, error) { return rendered, nil } -var defaultTemplate = `global +var templateControlPlane = `global log stdout format raw local0 notice daemon @@ -305,3 +343,52 @@ backend kube-apiservers {{- $backEnd := . }} server {{ $backEnd.Name }} {{ $backEnd.IP }}:{{ $backEnd.Port }} {{ end -}}` + +// TODO Update this template to work for workload services, as it currently references api server(control plane) +var templateWorker = `global + log stdout format raw local0 notice + daemon +defaults + mode http + log global + option httplog + option dontlognull + retries 1 + # Configures the timeout for a connection request to be left pending in a queue + # (connection requests are queued once the maximum number of connections is reached). + timeout queue 30s + # Configures the timeout for a connection to a backend server to be established. + timeout connect 30s + # Configures the timeout for inactivity during periods when we would expect + # the client to be speaking. For usability of 'kubectl exec', the timeout should + # be long enough to cover inactivity due to idleness of interactive sessions. + timeout client 600s + # Configures the timeout for inactivity during periods when we would expect + # the server to be speaking. For usability of 'kubectl log -f', the timeout should + # be long enough to cover inactivity due to the lack of new logs. + timeout server 600s +#--------------------------------------------------------------------- +# apiserver frontend which proxys to the masters +#--------------------------------------------------------------------- +frontend apiserver + bind *:{{ .FrontPort }} + mode tcp + option tcplog + default_backend kube-apiservers +#--------------------------------------------------------------------- +# round robin balancing for apiserver +#--------------------------------------------------------------------- +backend kube-apiservers + mode tcp + balance roundrobin + option httpchk GET /readyz + http-check expect status 200 + option log-health-checks + # Observed apiserver returns 500 for around 10s when 2nd cp node joins. + # downinter 2s makes it check more frequently to recover from that state sooner. + # Also changing fall to 4 so that it takes longer (4 failures) for it to take down a backend. + default-server check check-ssl verify none inter 5s downinter 2s fall 4 on-marked-down shutdown-sessions +{{- range .Backends }} +{{- $backEnd := . }} + server {{ $backEnd.Name }} {{ $backEnd.IP }}:{{ $backEnd.Port }} +{{ end -}}` diff --git a/pkg/services/services_test.go b/pkg/services/services_test.go index dd26a12..7bae90d 100644 --- a/pkg/services/services_test.go +++ b/pkg/services/services_test.go @@ -3,6 +3,7 @@ package services_test import ( "context" "encoding/json" + "strings" airshipv1 "sipcluster/pkg/api/v1" @@ -106,7 +107,7 @@ var _ = Describe("Service Set", func() { set := services.NewServiceSet(logger, *sipCluster, machineList, k8sClient) serviceList, err := set.ServiceList() - Expect(serviceList).To(HaveLen(2)) + Expect(serviceList).To(HaveLen(3)) Expect(err).To(Succeed()) for _, svc := range serviceList { err := svc.Deploy() @@ -121,7 +122,8 @@ var _ = Describe("Service Set", func() { It("Does not deploy a Jump Host when an invalid SSH key is provided", func() { sip, _ := testutil.CreateSIPCluster("default", "default", 1, 1) sip.Spec.Services.Auth = []airshipv1.SIPClusterService{} - sip.Spec.Services.LoadBalancer = []airshipv1.SIPClusterService{} + sip.Spec.Services.LoadBalancerControlPlane = []airshipv1.LoadBalancerServiceControlPlane{} + sip.Spec.Services.LoadBalancerWorker = []airshipv1.LoadBalancerServiceWorker{} sip.Spec.Services.JumpHost[0].SSHAuthorizedKeys = []string{ "sshrsaAAAAAAAAAAAAAAAAAAAAAinvalidkey", } @@ -141,29 +143,62 @@ var _ = Describe("Service Set", func() { }) func testDeployment(sip *airshipv1.SIPCluster, machineList bmh.MachineList) error { - loadBalancerDeployment := &appsv1.Deployment{} + loadBalancerControlPlaneDeployment := &appsv1.Deployment{} err := k8sClient.Get(context.Background(), types.NamespacedName{ Namespace: "default", - Name: services.LoadBalancerServiceName + "-" + sip.GetName(), - }, loadBalancerDeployment) + Name: services.LoadBalancerServiceName + "-" + strings.ToLower(string(airshipv1.RoleControlPlane)) + "-" + + sip.GetName(), + }, loadBalancerControlPlaneDeployment) if err != nil { return err } - loadBalancerSecret := &corev1.Secret{} + loadBalancerWorkerDeployment := &appsv1.Deployment{} err = k8sClient.Get(context.Background(), types.NamespacedName{ Namespace: "default", - Name: services.LoadBalancerServiceName + "-" + sip.GetName(), - }, loadBalancerSecret) + Name: services.LoadBalancerServiceName + "-" + strings.ToLower(string(airshipv1.RoleWorker)) + "-" + + sip.GetName(), + }, loadBalancerWorkerDeployment) if err != nil { return err } - loadBalancerService := &corev1.Service{} + loadBalancerControlPlaneSecret := &corev1.Secret{} err = k8sClient.Get(context.Background(), types.NamespacedName{ Namespace: "default", - Name: services.LoadBalancerServiceName + "-" + sip.GetName(), - }, loadBalancerService) + Name: services.LoadBalancerServiceName + "-" + strings.ToLower(string(airshipv1.RoleControlPlane)) + "-" + + sip.GetName(), + }, loadBalancerControlPlaneSecret) + if err != nil { + return err + } + + loadBalancerWorkerSecret := &corev1.Secret{} + err = k8sClient.Get(context.Background(), types.NamespacedName{ + Namespace: "default", + Name: services.LoadBalancerServiceName + "-" + strings.ToLower(string(airshipv1.RoleWorker)) + "-" + + sip.GetName(), + }, loadBalancerWorkerSecret) + if err != nil { + return err + } + + loadBalancerControlPlaneService := &corev1.Service{} + err = k8sClient.Get(context.Background(), types.NamespacedName{ + Namespace: "default", + Name: services.LoadBalancerServiceName + "-" + strings.ToLower(string(airshipv1.RoleControlPlane)) + "-" + + sip.GetName(), + }, loadBalancerControlPlaneService) + if err != nil { + return err + } + + loadBalancerWorkerService := &corev1.Service{} + err = k8sClient.Get(context.Background(), types.NamespacedName{ + Namespace: "default", + Name: services.LoadBalancerServiceName + "-" + strings.ToLower(string(airshipv1.RoleWorker)) + "-" + + sip.GetName(), + }, loadBalancerWorkerService) if err != nil { return err } diff --git a/pkg/services/set.go b/pkg/services/set.go index 92759da..2ea4392 100644 --- a/pkg/services/set.go +++ b/pkg/services/set.go @@ -64,9 +64,18 @@ func (ss ServiceSet) Finalize() error { func (ss ServiceSet) ServiceList() ([]InfraService, error) { serviceList := []InfraService{} services := ss.sip.Spec.Services - for _, svc := range services.LoadBalancer { + for _, svc := range services.LoadBalancerControlPlane { serviceList = append(serviceList, - newLB(ss.sip.GetName(), + newLBControlPlane(ss.sip.GetName(), + ss.sip.GetNamespace(), + ss.logger, + svc, + ss.machines, + ss.client)) + } + for _, svc := range services.LoadBalancerWorker { + serviceList = append(serviceList, + newLBWorker(ss.sip.GetName(), ss.sip.GetNamespace(), ss.logger, svc, diff --git a/testutil/testutil.go b/testutil/testutil.go index abaec7d..b9f4c23 100644 --- a/testutil/testutil.go +++ b/testutil/testutil.go @@ -261,17 +261,11 @@ func CreateSIPCluster(name string, namespace string, controlPlanes int, workers }, }, Services: airshipv1.SIPClusterServices{ - LoadBalancer: []airshipv1.SIPClusterService{ - { - NodeInterface: "eno3", - NodePort: 30000, - }, - }, JumpHost: []airshipv1.JumpHostService{ { SIPClusterService: airshipv1.SIPClusterService{ Image: "quay.io/airshipit/jump-host", - NodePort: 30001, + NodePort: 30000, NodeInterface: "eno3", }, SSHAuthorizedKeys: []string{ @@ -281,6 +275,22 @@ func CreateSIPCluster(name string, namespace string, controlPlanes int, workers NodeSSHPrivateKeys: sshPrivateKeySecretName, }, }, + LoadBalancerControlPlane: []airshipv1.LoadBalancerServiceControlPlane{ + { + SIPClusterService: airshipv1.SIPClusterService{ + NodeInterface: "eno3", + NodePort: 30001, + }, + }, + }, + LoadBalancerWorker: []airshipv1.LoadBalancerServiceWorker{ + { + SIPClusterService: airshipv1.SIPClusterService{ + NodeInterface: "eno3", + NodePort: 30002, + }, + }, + }, }, }, Status: airshipv1.SIPClusterStatus{},