airship/sip
Code Issues Proposed changes

Wrap service pods in Deployment/Service

Browse Source 
This wraps the service pods with deployments to allow for
self healing if a pod crashes. This also adds a NodePort Service
for the jumphost service, and it removes the node port range
check, in favor of relying on the validation of the Service
itself by the kubernetes control plane, and allowing for
auto-allocation if the node port is not set.

Signed-off-by: Sean Eagan <seaneagan1@gmail.com>
Change-Id: I78b0b651765a43f585a790e96c4472b841fc950d
This commit is contained in:
Sean Eagan
2021-02-12 16:36:01 -06:00
parent e81e120dbf
commit 116bddc80d
6 changed files with 169 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
config/rbac/sipcluster_scheduler_role.yaml
View File

@@ -50,7 +50,7 @@ rules:
- ""
resources:
- namespaces
- pods
- deployments
- secrets
verbs:
- create

100
pkg/services/jumphost.go
View File

@@ -15,10 +15,8 @@
package services
import (
"context"
"fmt"
"github.com/go-logr/logr"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
@@ -29,8 +27,7 @@ import (
)
const (
jumpHostContainerName = "ssh"
jumpHostPodNameTemplate = "%s-jump-pod"
JumpHostServiceName = "jumphost"
)
// JumpHost is an InfrastructureService that provides SSH capabilities to access a sub-cluster.
@@ -58,36 +55,89 @@ func newJumpHost(name, namespace string, logger logr.Logger, config airshipv1.Ju
// Deploy creates a JumpHost service in the base cluster.
func (jh jumpHost) Deploy() error {
jh.logger.Info("deploying jump host", "sub-cluster", jh.sipName.Name)
instance := JumpHostServiceName + "-" + jh.sipName.Name
labels := map[string]string{
// See https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/#labels
"app.kubernetes.io/part-of": "sip",
"app.kubernetes.io/name": JumpHostServiceName,
"app.kubernetes.io/component": JumpHostServiceName,
"app.kubernetes.io/instance": instance,
}
jumpHostPodName := fmt.Sprintf(jumpHostPodNameTemplate, jh.sipName.Name)
pod := corev1.Pod{
TypeMeta: metav1.TypeMeta{
APIVersion: corev1.SchemeGroupVersion.String(),
Kind: "Pod",
},
// TODO: Validate Deployment becomes ready.
deployment := jh.generateDeployment(instance, labels)
jh.logger.Info("Applying deployment", "deployment", deployment.GetNamespace()+"/"+deployment.GetName())
err := applyRuntimeObject(client.ObjectKey{Name: deployment.GetName(), Namespace: deployment.GetNamespace()},
deployment, jh.client)
if err != nil {
return err
}
// TODO: Validate Service becomes ready.
service := jh.generateService(instance, labels)
jh.logger.Info("Applying service", "service", service.GetNamespace()+"/"+service.GetName())
err = applyRuntimeObject(client.ObjectKey{Name: service.GetName(), Namespace: service.GetNamespace()},
service, jh.client)
if err != nil {
return err
}
return nil
}
func (jh jumpHost) generateDeployment(instance string, labels map[string]string) *appsv1.Deployment {
return &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: jumpHostPodName,
Name: instance,
Namespace: jh.sipName.Namespace,
Labels: labels,
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: jumpHostContainerName,
Image: jh.config.Image,
Spec: appsv1.DeploymentSpec{
Replicas: int32Ptr(1),
Selector: &metav1.LabelSelector{
MatchLabels: labels,
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: labels,
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: JumpHostServiceName,
Image: jh.config.Image,
Ports: []corev1.ContainerPort{
{
Name: "ssh",
ContainerPort: 22,
},
},
},
},
},
},
},
}
}
if err := jh.client.Create(context.Background(), &pod); err != nil {
return err
func (jh jumpHost) generateService(instance string, labels map[string]string) *corev1.Service {
return &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: instance,
Namespace: jh.sipName.Namespace,
},
Spec: corev1.ServiceSpec{
Ports: []corev1.ServicePort{
{
Name: "ssh",
Port: 22,
NodePort: int32(jh.config.NodePort),
},
},
Selector: labels,
Type: corev1.ServiceTypeNodePort,
},
}
jh.logger.Info("successfully deployed jump host", "sub-cluster", jh.sipName.Name, "jump host pod name",
jumpHostPodName, "namespace", jh.sipName.Namespace)
return nil
}
// Finalize removes a deployed JumpHost service.

109
pkg/services/loadbalancer.go
View File

@@ -24,6 +24,7 @@ import (
"github.com/go-logr/logr"
"k8s.io/apimachinery/pkg/types"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -34,19 +35,25 @@ const (
/* #nosec */
ConfigSecretName = "haproxy-config"
// DefaultBalancerImage is the image that will be used as load balancer
DefaultBalancerImage = "haproxy:2.3.2"
DefaultBalancerImage = "haproxy:2.3.2"
LoadBalancerServiceName = "loadbalancer"
)
func (lb loadBalancer) Deploy() error {
if lb.config.Image == "" {
lb.config.Image = DefaultBalancerImage
}
if lb.config.NodePort < 30000 || lb.config.NodePort > 32767 {
lb.logger.Info("Either NodePort is not defined in the CR or NodePort is not in the required range of 30000-32767")
return nil
instance := LoadBalancerServiceName + "-" + lb.sipName.Name
labels := map[string]string{
// See https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/#labels
"app.kubernetes.io/part-of": "sip",
"app.kubernetes.io/component": LoadBalancerServiceName,
"app.kubernetes.io/name": "haproxy",
"app.kubernetes.io/instance": instance,
}
pod, secret, err := lb.generatePodAndSecret()
deployment, secret, err := lb.generateDeploymentAndSecret(instance, labels)
if err != nil {
return err
}
@@ -57,13 +64,16 @@ func (lb loadBalancer) Deploy() error {
return err
}
lb.logger.Info("Applying loadbalancer pod", "pod", pod.GetNamespace()+"/"+pod.GetName())
err = applyRuntimeObject(client.ObjectKey{Name: pod.GetName(), Namespace: pod.GetNamespace()}, pod, lb.client)
// TODO: Validate Deployment becomes ready.
lb.logger.Info("Applying loadbalancer deployment", "deployment", deployment.GetNamespace()+"/"+deployment.GetName())
err = applyRuntimeObject(client.ObjectKey{Name: deployment.GetName(), Namespace: deployment.GetNamespace()},
deployment, lb.client)
if err != nil {
return err
}
lbService := lb.generateService()
// TODO: Validate Service becomes ready.
lbService := lb.generateService(instance, labels)
lb.logger.Info("Applying loadbalancer service", "service", lbService.GetNamespace()+"/"+lbService.GetName())
err = applyRuntimeObject(client.ObjectKey{Name: lbService.GetName(), Namespace: lbService.GetNamespace()},
lbService, lb.client)
@@ -73,53 +83,66 @@ func (lb loadBalancer) Deploy() error {
return nil
}
func (lb loadBalancer) generatePodAndSecret() (*corev1.Pod, *corev1.Secret, error) {
secret, err := lb.generateSecret()
func (lb loadBalancer) generateDeploymentAndSecret(instance string, labels map[string]string) (*appsv1.Deployment,
*corev1.Secret, error) {
secret, err := lb.generateSecret(instance)
if err != nil {
return nil, nil, err
}
pod := &corev1.Pod{
deployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: lb.sipName.Name + "-load-balancer",
Name: instance,
Namespace: lb.sipName.Namespace,
Labels: map[string]string{"lb-name": lb.sipName.Namespace + "-haproxy"},
Labels: labels,
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "balancer",
Image: lb.config.Image,
Ports: []corev1.ContainerPort{
{
Name: "http",
ContainerPort: 6443,
},
},
VolumeMounts: []corev1.VolumeMount{
{
Name: ConfigSecretName,
MountPath: "/usr/local/etc/haproxy",
},
},
},
Spec: appsv1.DeploymentSpec{
Replicas: int32Ptr(1),
Selector: &metav1.LabelSelector{
MatchLabels: labels,
},
Volumes: []corev1.Volume{
{
Name: ConfigSecretName,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: secret.GetName(),
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: labels,
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: LoadBalancerServiceName,
Image: lb.config.Image,
Ports: []corev1.ContainerPort{
{
Name: "http",
ContainerPort: 6443,
},
},
VolumeMounts: []corev1.VolumeMount{
{
Name: ConfigSecretName,
MountPath: "/usr/local/etc/haproxy",
},
},
},
},
Volumes: []corev1.Volume{
{
Name: ConfigSecretName,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: secret.GetName(),
},
},
},
},
},
},
},
}
return pod, secret, nil
return deployment, secret, nil
}
func (lb loadBalancer) generateSecret() (*corev1.Secret, error) {
func (lb loadBalancer) generateSecret(instance string) (*corev1.Secret, error) {
p := proxy{
FrontPort: 6443,
Backends: make([]backend, 0),
@@ -145,7 +168,7 @@ func (lb loadBalancer) generateSecret() (*corev1.Secret, error) {
}
return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: lb.sipName.Name + "-load-balancer",
Name: instance,
Namespace: lb.sipName.Namespace,
},
Type: corev1.SecretTypeOpaque,
@@ -155,10 +178,10 @@ func (lb loadBalancer) generateSecret() (*corev1.Secret, error) {
}, nil
}
func (lb loadBalancer) generateService() *corev1.Service {
func (lb loadBalancer) generateService(instance string, labels map[string]string) *corev1.Service {
return &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: lb.sipName.Name + "-load-balancer-service",
Name: instance,
Namespace: lb.sipName.Namespace,
},
Spec: corev1.ServiceSpec{
@@ -169,7 +192,7 @@ func (lb loadBalancer) generateService() *corev1.Service {
NodePort: int32(lb.config.NodePort),
},
},
Selector: map[string]string{"lb-name": lb.sipName.Namespace + "-haproxy"},
Selector: labels,
Type: corev1.ServiceTypeNodePort,
},
}

38
pkg/services/services_test.go
View File

@@ -7,6 +7,7 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
@@ -65,20 +66,11 @@ var _ = Describe("Service Set", func() {
})
func testDeployment(sip *airshipv1.SIPCluster) error {
jumpHostPod := &corev1.Pod{}
loadBalancerDeployment := &appsv1.Deployment{}
err := k8sClient.Get(context.Background(), types.NamespacedName{
Namespace: "default",
Name: sip.GetName() + "-jump-pod",
}, jumpHostPod)
if err != nil {
return err
}
loadBalancerPod := &corev1.Pod{}
err = k8sClient.Get(context.Background(), types.NamespacedName{
Namespace: "default",
Name: sip.GetName() + "-load-balancer",
}, loadBalancerPod)
Name: services.LoadBalancerServiceName + "-" + sip.GetName(),
}, loadBalancerDeployment)
if err != nil {
return err
}
@@ -86,7 +78,7 @@ func testDeployment(sip *airshipv1.SIPCluster) error {
loadBalancerSecret := &corev1.Secret{}
err = k8sClient.Get(context.Background(), types.NamespacedName{
Namespace: "default",
Name: sip.GetName() + "-load-balancer",
Name: services.LoadBalancerServiceName + "-" + sip.GetName(),
}, loadBalancerSecret)
if err != nil {
return err
@@ -95,11 +87,29 @@ func testDeployment(sip *airshipv1.SIPCluster) error {
loadBalancerService := &corev1.Service{}
err = k8sClient.Get(context.Background(), types.NamespacedName{
Namespace: "default",
Name: sip.GetName() + "-load-balancer-service",
Name: services.LoadBalancerServiceName + "-" + sip.GetName(),
}, loadBalancerService)
if err != nil {
return err
}
jumpHostDeployment := &appsv1.Deployment{}
err = k8sClient.Get(context.Background(), types.NamespacedName{
Namespace: "default",
Name: services.JumpHostServiceName + "-" + sip.GetName(),
}, jumpHostDeployment)
if err != nil {
return err
}
jumpHostService := &corev1.Service{}
err = k8sClient.Get(context.Background(), types.NamespacedName{
Namespace: "default",
Name: services.JumpHostServiceName + "-" + sip.GetName(),
}, jumpHostService)
if err != nil {
return err
}
return nil
}

2
pkg/services/set.go
View File

@@ -130,3 +130,5 @@ func applyRuntimeObject(key client.ObjectKey, obj client.Object, c client.Client
return err
}
}
func int32Ptr(i int32) *int32 { return &i }

2
testutil/testutil.go
View File

@@ -245,7 +245,7 @@ func CreateSIPCluster(name string, namespace string, controlPlanes int, workers
{
SIPClusterService: airshipv1.SIPClusterService{
Image: "ubuntu:20.04",
NodePort: 7022,
NodePort: 30001,
NodeInterface: "eno3",
},
SSHKey: "",