From cc5d82883c5a074c87228b064e384f591d85da94 Mon Sep 17 00:00:00 2001 From: Mohammed Naser Date: Sun, 22 Mar 2020 20:28:40 -0400 Subject: [PATCH] Add builder pattern This builder pattern refactors things so that we can use the same code to generate manifests/etc. This means that if we make sure that we exclusively use those, we can do testing there and keep something common. Change-Id: Ibc39f9b9e3e21b18fb255ba2a67d2d8ba3b5c585 --- Dockerfile | 2 + builders/config_map.go | 37 ++++++ builders/container.go | 148 +++++++++++++++++++++ builders/deployment.go | 59 +++++++++ builders/pod_spec.go | 74 +++++++++++ builders/pod_template_spec.go | 46 +++++++ builders/security_context.go | 42 ++++++ builders/service.go | 50 +++++++ builders/volume.go | 39 ++++++ config/manager/kustomization.yaml | 6 + controllers/mcrouter_controller.go | 199 +++++++--------------------- controllers/memcached_controller.go | 146 ++++++-------------- controllers/suite_test.go | 1 - go.mod | 1 + images/mcrouter/Dockerfile | 10 +- playbooks/functional/post.yaml | 3 +- utils/kubernetes.go | 35 +++++ 17 files changed, 633 insertions(+), 265 deletions(-) create mode 100755 builders/config_map.go create mode 100755 builders/container.go create mode 100755 builders/deployment.go create mode 100755 builders/pod_spec.go create mode 100755 builders/pod_template_spec.go create mode 100755 builders/security_context.go create mode 100755 builders/service.go create mode 100755 builders/volume.go mode change 100644 => 100755 playbooks/functional/post.yaml create mode 100644 utils/kubernetes.go diff --git a/Dockerfile b/Dockerfile index 9e39e304..37bb4e07 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,9 @@ RUN go mod download # Copy the go source COPY main.go main.go COPY api/ api/ +COPY builders/ builders/ COPY controllers/ controllers/ +COPY utils/ utils/ COPY version/ version/ # Build diff --git a/builders/config_map.go b/builders/config_map.go new file mode 100755 index 00000000..7604ced6 --- /dev/null +++ b/builders/config_map.go @@ -0,0 +1,37 @@ +package builders + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +// ConfigMapBuilder defines the interface to build a ConfigMap +type ConfigMapBuilder struct { + obj *corev1.ConfigMap + owner metav1.Object + scheme *runtime.Scheme +} + +// ConfigMap returns a new service builder +func ConfigMap(existing *corev1.ConfigMap, owner metav1.Object, scheme *runtime.Scheme) *ConfigMapBuilder { + existing.Data = map[string]string{} + + return &ConfigMapBuilder{ + obj: existing, + owner: owner, + scheme: scheme, + } +} + +// Data sets a key inside this ConfigMap +func (cm *ConfigMapBuilder) Data(key, value string) *ConfigMapBuilder { + cm.obj.Data[key] = value + return cm +} + +// Build returns a complete ConfigMap object +func (cm *ConfigMapBuilder) Build() error { + return controllerutil.SetControllerReference(cm.owner, cm.obj, cm.scheme) +} diff --git a/builders/container.go b/builders/container.go new file mode 100755 index 00000000..9880acd5 --- /dev/null +++ b/builders/container.go @@ -0,0 +1,148 @@ +package builders + +import ( + "errors" + + "github.com/alecthomas/units" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// ContainerBuilder provides an interface to build containers +type ContainerBuilder struct { + obj *corev1.Container + securityContext *SecurityContextBuilder +} + +// Container returns a new container builder +func Container(name string, image string) *ContainerBuilder { + container := &corev1.Container{ + Name: name, + Image: image, + ImagePullPolicy: corev1.PullAlways, + TerminationMessagePath: "/dev/termination-log", + TerminationMessagePolicy: corev1.TerminationMessageReadFile, + } + + return &ContainerBuilder{ + obj: container, + } +} + +// Args sets the arguments for that container +func (c *ContainerBuilder) Args(args ...string) *ContainerBuilder { + c.obj.Args = args + return c +} + +// SecurityContext sets the SecurityContext for that container +func (c *ContainerBuilder) SecurityContext(SecurityContext *SecurityContextBuilder) *ContainerBuilder { + c.securityContext = SecurityContext + return c +} + +// Port appends a port to the container +func (c *ContainerBuilder) Port(name string, port int32) *ContainerBuilder { + c.obj.Ports = append(c.obj.Ports, v1.ContainerPort{ + Name: name, + ContainerPort: port, + Protocol: corev1.ProtocolTCP, + }) + return c +} + +// Volume appends a volume to the container +func (c *ContainerBuilder) Volume(name string, path string) *ContainerBuilder { + c.obj.VolumeMounts = append(c.obj.VolumeMounts, v1.VolumeMount{ + Name: name, + MountPath: path, + }) + return c +} + +// Resources defines the resource configuration for the container +func (c *ContainerBuilder) Resources(cpu int64, memory int64, storage int64, factor float64) *ContainerBuilder { + memory = memory * int64(units.Mebibyte) + storage = storage * int64(units.Megabyte) + + cpuLimit := int64(float64(cpu) * factor) + memoryLimit := int64(float64(memory) * factor) + storageLimit := int64(float64(storage) * factor) + + c.obj.Resources = v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(cpuLimit, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(memoryLimit, resource.BinarySI), + v1.ResourceEphemeralStorage: *resource.NewQuantity(storageLimit, resource.DecimalSI), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(cpu, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(memory, resource.BinarySI), + v1.ResourceEphemeralStorage: *resource.NewQuantity(storage, resource.DecimalSI), + }, + } + + return c +} + +// HTTPProbe creates both a readiness and liveness probe with provided intervals +func (c *ContainerBuilder) HTTPProbe(port string, path string, readyInterval int32, liveInterval int32) *ContainerBuilder { + handler := v1.Handler{ + HTTPGet: &v1.HTTPGetAction{ + Path: path, + Port: intstr.FromString(port), + Scheme: v1.URISchemeHTTP, + }, + } + + return c.Probe(handler, readyInterval, liveInterval) +} + +// PortProbe creates both a readiness and liveness probe with provided intervals +func (c *ContainerBuilder) PortProbe(port string, readyInterval int32, liveInterval int32) *ContainerBuilder { + handler := v1.Handler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.FromString(port), + }, + } + + return c.Probe(handler, readyInterval, liveInterval) +} + +// Probe creates both a readiness and liveness probe based on a handler provided +func (c *ContainerBuilder) Probe(handler v1.Handler, readyInterval int32, liveInterval int32) *ContainerBuilder { + c.obj.ReadinessProbe = &v1.Probe{ + Handler: handler, + InitialDelaySeconds: 0, + PeriodSeconds: readyInterval, + TimeoutSeconds: 1, + SuccessThreshold: 1, + FailureThreshold: 3, + } + c.obj.LivenessProbe = &v1.Probe{ + Handler: handler, + InitialDelaySeconds: 0, + PeriodSeconds: liveInterval, + TimeoutSeconds: 1, + SuccessThreshold: 1, + FailureThreshold: 3, + } + + return c +} + +// Build returns the object after making certain assertions +func (c *ContainerBuilder) Build() (corev1.Container, error) { + if c.securityContext == nil { + return corev1.Container{}, errors.New("missing security context") + } + securityContext, err := c.securityContext.Build() + + if err != nil { + return corev1.Container{}, err + } + c.obj.SecurityContext = &securityContext + return *c.obj, nil +} diff --git a/builders/deployment.go b/builders/deployment.go new file mode 100755 index 00000000..71b5e870 --- /dev/null +++ b/builders/deployment.go @@ -0,0 +1,59 @@ +package builders + +import ( + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +// DeploymentBuilder defines the interface to build a deployment +type DeploymentBuilder struct { + obj *appsv1.Deployment + podTemplateSpec *PodTemplateSpecBuilder + owner metav1.Object + scheme *runtime.Scheme + labels map[string]string +} + +// Deployment returns a new deployment builder +func Deployment(existing *appsv1.Deployment, owner metav1.Object, scheme *runtime.Scheme) *DeploymentBuilder { + return &DeploymentBuilder{ + obj: existing, + labels: map[string]string{}, + owner: owner, + scheme: scheme, + } +} + +// Labels specifies labels for the deployment +func (d *DeploymentBuilder) Labels(labels map[string]string) *DeploymentBuilder { + d.labels = labels + d.obj.ObjectMeta.Labels = d.labels + d.obj.Spec.Selector = &metav1.LabelSelector{MatchLabels: d.labels} + return d +} + +// Replicas defines the number of replicas +func (d *DeploymentBuilder) Replicas(replicas int32) *DeploymentBuilder { + d.obj.Spec.Replicas = pointer.Int32Ptr(replicas) + return d +} + +// PodTemplateSpec defines a builder for the pod template spec +func (d *DeploymentBuilder) PodTemplateSpec(podTemplateSpec *PodTemplateSpecBuilder) *DeploymentBuilder { + d.podTemplateSpec = podTemplateSpec + return d +} + +// Build creates a final deployment objet +func (d *DeploymentBuilder) Build() error { + podTemplateSpec, err := d.podTemplateSpec.Labels(d.labels).Build() + if err != nil { + return err + } + + d.obj.Spec.Template = podTemplateSpec + return controllerutil.SetControllerReference(d.owner, d.obj, d.scheme) +} diff --git a/builders/pod_spec.go b/builders/pod_spec.go new file mode 100755 index 00000000..07c0fc13 --- /dev/null +++ b/builders/pod_spec.go @@ -0,0 +1,74 @@ +package builders + +import ( + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" +) + +// PodSpecBuilder is an interface for building a PodSpec +type PodSpecBuilder struct { + obj *corev1.PodSpec + containers []*ContainerBuilder + volumes []*VolumeBuilder +} + +// PodSpec returns a builder object for a PodSpec +func PodSpec() *PodSpecBuilder { + podSpec := &corev1.PodSpec{ + DNSPolicy: corev1.DNSClusterFirst, + RestartPolicy: corev1.RestartPolicyAlways, + SchedulerName: "default-scheduler", + // SecurityContext: &v1.PodSecurityContext{ + // RunAsNonRoot: pointer.BoolPtr(true), + // }, + TerminationGracePeriodSeconds: pointer.Int64Ptr(10), + } + + return &PodSpecBuilder{ + obj: podSpec, + } +} + +// Containers appends a container builder to the PodSpec +func (ps *PodSpecBuilder) Containers(c ...*ContainerBuilder) *PodSpecBuilder { + ps.containers = c + return ps +} + +// Volumes appends a volume builder to the PodSpec +func (ps *PodSpecBuilder) Volumes(v ...*VolumeBuilder) *PodSpecBuilder { + ps.volumes = v + return ps +} + +// NodeSelector defines a NodeSelector for PodSpec +func (ps *PodSpecBuilder) NodeSelector(selector map[string]string) *PodSpecBuilder { + ps.obj.NodeSelector = selector + return ps +} + +// Tolerations defines tolerations for PodSpec +func (ps *PodSpecBuilder) Tolerations(tolerations []v1.Toleration) *PodSpecBuilder { + ps.obj.Tolerations = tolerations + return ps +} + +// Build generates an object ensuring that all sub-objects work +func (ps *PodSpecBuilder) Build() (corev1.PodSpec, error) { + for _, c := range ps.containers { + container, err := c.Build() + if err != nil { + return corev1.PodSpec{}, err + } + + ps.obj.Containers = append(ps.obj.Containers, container) + } + + for _, v := range ps.volumes { + volume := v.Build() + ps.obj.Volumes = append(ps.obj.Volumes, volume) + } + + return *ps.obj, nil +} diff --git a/builders/pod_template_spec.go b/builders/pod_template_spec.go new file mode 100755 index 00000000..9ee939a5 --- /dev/null +++ b/builders/pod_template_spec.go @@ -0,0 +1,46 @@ +package builders + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// PodTemplateSpecBuilder is an interface for building a PodTemplateSpecBuilder +type PodTemplateSpecBuilder struct { + obj *corev1.PodTemplateSpec + podSpec *PodSpecBuilder +} + +// PodTemplateSpec returns a builder object for a PodTemplateSpec +func PodTemplateSpec() *PodTemplateSpecBuilder { + podTemplateSpec := &corev1.PodTemplateSpec{} + + return &PodTemplateSpecBuilder{ + obj: podTemplateSpec, + } +} + +// Labels sets up the labels for a PodTemplateSpec +func (pts *PodTemplateSpecBuilder) Labels(labels map[string]string) *PodTemplateSpecBuilder { + pts.obj.ObjectMeta = metav1.ObjectMeta{ + Labels: labels, + } + return pts +} + +// PodSpec points this builder to PodSpec builder +func (pts *PodTemplateSpecBuilder) PodSpec(podSpec *PodSpecBuilder) *PodTemplateSpecBuilder { + pts.podSpec = podSpec + return pts +} + +// Build generates an object ensuring that all sub-objects work +func (pts *PodTemplateSpecBuilder) Build() (corev1.PodTemplateSpec, error) { + podSpec, err := pts.podSpec.Build() + if err != nil { + return corev1.PodTemplateSpec{}, err + } + + pts.obj.Spec = podSpec + return *pts.obj, nil +} diff --git a/builders/security_context.go b/builders/security_context.go new file mode 100755 index 00000000..b97af485 --- /dev/null +++ b/builders/security_context.go @@ -0,0 +1,42 @@ +package builders + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" +) + +// SecurityContextBuilder defines the interface to build a securityContext +type SecurityContextBuilder struct { + obj *corev1.SecurityContext +} + +// SecurityContext returns a new SecurityContext builder +func SecurityContext() *SecurityContextBuilder { + securityContext := &corev1.SecurityContext{} + return &SecurityContextBuilder{ + obj: securityContext, + } +} + +// RunAsUser sets the RunAsUser inside this SecurityContext +func (sc *SecurityContextBuilder) RunAsUser(userID int64) *SecurityContextBuilder { + sc.obj.RunAsUser = pointer.Int64Ptr(userID) + return sc +} + +// RunAsGroup sets the RunAsGroup inside this SecurityContext +func (sc *SecurityContextBuilder) RunAsGroup(groupID int64) *SecurityContextBuilder { + sc.obj.RunAsGroup = pointer.Int64Ptr(groupID) + return sc +} + +// RunAsNonRoot sets the RunAsNonRoot inside this SecurityContext +func (sc *SecurityContextBuilder) RunAsNonRoot(flag bool) *SecurityContextBuilder { + sc.obj.RunAsNonRoot = pointer.BoolPtr(flag) + return sc +} + +// Build returns a complete ConfigMap object +func (sc *SecurityContextBuilder) Build() (corev1.SecurityContext, error) { + return *sc.obj, nil +} diff --git a/builders/service.go b/builders/service.go new file mode 100755 index 00000000..c76d7aba --- /dev/null +++ b/builders/service.go @@ -0,0 +1,50 @@ +package builders + +import ( + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +// ServiceBuilder defines the interface to build a service +type ServiceBuilder struct { + obj *corev1.Service + owner metav1.Object + scheme *runtime.Scheme +} + +// Service returns a new service builder +func Service(existing *corev1.Service, owner metav1.Object, scheme *runtime.Scheme) *ServiceBuilder { + existing.Spec.Ports = []corev1.ServicePort{} + + return &ServiceBuilder{ + obj: existing, + owner: owner, + scheme: scheme, + } +} + +// Port appends a port to the service +func (s *ServiceBuilder) Port(name string, port int32) *ServiceBuilder { + s.obj.Spec.Ports = append(s.obj.Spec.Ports, corev1.ServicePort{ + Name: name, + Protocol: v1.ProtocolTCP, + Port: port, + TargetPort: intstr.FromString(name), + }) + return s +} + +// Selector defines the service selectors +func (s *ServiceBuilder) Selector(labels map[string]string) *ServiceBuilder { + s.obj.Spec.Selector = labels + return s +} + +// Build returns a complete Service object +func (s *ServiceBuilder) Build() error { + return controllerutil.SetControllerReference(s.owner, s.obj, s.scheme) +} diff --git a/builders/volume.go b/builders/volume.go new file mode 100755 index 00000000..0672b858 --- /dev/null +++ b/builders/volume.go @@ -0,0 +1,39 @@ +package builders + +import ( + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" +) + +// VolumeBuilder provides an interface to build volumes +type VolumeBuilder struct { + obj *corev1.Volume +} + +// Volume returns a new volume builder +func Volume(name string) *VolumeBuilder { + volume := &corev1.Volume{ + Name: name, + } + + return &VolumeBuilder{ + obj: volume, + } +} + +// FromConfigMap sets the source of the volume from a ConfigMap +func (v *VolumeBuilder) FromConfigMap(name string) *VolumeBuilder { + v.obj.VolumeSource = corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{Name: name}, + DefaultMode: pointer.Int32Ptr(420), + }, + } + return v +} + +// Build returns the object after checking assertions +func (v *VolumeBuilder) Build() corev1.Volume { + return *v.obj +} diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 5c5f0b84..ad13e96b 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,2 +1,8 @@ resources: - manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: controller + newTag: latest diff --git a/controllers/mcrouter_controller.go b/controllers/mcrouter_controller.go index 0c0acabc..bf2c908d 100755 --- a/controllers/mcrouter_controller.go +++ b/controllers/mcrouter_controller.go @@ -5,22 +5,17 @@ import ( "encoding/json" "fmt" - "github.com/alecthomas/units" "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" infrastructurev1alpha1 "opendev.org/vexxhost/openstack-operator/api/v1alpha1" - "opendev.org/vexxhost/openstack-operator/version" + "opendev.org/vexxhost/openstack-operator/builders" + "opendev.org/vexxhost/openstack-operator/utils" ) // McrouterReconciler reconciles a Mcrouter object @@ -57,20 +52,17 @@ func (r *McrouterReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { ObjectMeta: metav1.ObjectMeta{ Namespace: req.Namespace, Name: fmt.Sprintf("mcrouter-%s", req.Name), - Labels: labels, }, } - op, err := controllerutil.CreateOrUpdate(ctx, r, configMap, func() error { + op, err := utils.CreateOrUpdate(ctx, r, configMap, func() error { b, err := json.Marshal(mcrouter.Spec) if err != nil { return err } - configMap.Data = map[string]string{ - "config.json": string(b), - } - - return controllerutil.SetControllerReference(&mcrouter, configMap, r.Scheme) + return builders.ConfigMap(configMap, &mcrouter, r.Scheme). + Data("config.json", string(b)). + Build() }) if err != nil { return ctrl.Result{}, err @@ -82,133 +74,44 @@ func (r *McrouterReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { ObjectMeta: metav1.ObjectMeta{ Namespace: req.Namespace, Name: fmt.Sprintf("mcrouter-%s", req.Name), - Labels: labels, }, } - op, err = controllerutil.CreateOrUpdate(ctx, r, deployment, func() error { - if deployment.ObjectMeta.CreationTimestamp.IsZero() { - deployment.Spec.Selector = &metav1.LabelSelector{ - MatchLabels: labels, - } - } - - deployment.Spec.Replicas = pointer.Int32Ptr(2) - deployment.Spec.Template = corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "mcrouter", - Image: fmt.Sprintf("vexxhost/mcrouter:%s", version.Revision), - Args: []string{"-p", "11211", "-f", "/data/config.json"}, - Ports: []v1.ContainerPort{ - { - Name: "mcrouter", - ContainerPort: int32(11211), - }, - }, - VolumeMounts: []v1.VolumeMount{ - { - Name: "config", - MountPath: "/data", - }, - }, - Resources: v1.ResourceRequirements{ - Limits: v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(1000, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(int64(units.Mebibyte)*256, resource.BinarySI), - v1.ResourceEphemeralStorage: *resource.NewQuantity(int64(units.MB)*1000, resource.DecimalSI), - }, - Requests: v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(int64(units.Mebibyte)*128, resource.BinarySI), - v1.ResourceEphemeralStorage: *resource.NewQuantity(int64(units.MB)*500, resource.DecimalSI), - }, - }, - - StartupProbe: &v1.Probe{}, - ReadinessProbe: &v1.Probe{ - Handler: v1.Handler{ - TCPSocket: &v1.TCPSocketAction{ - Port: intstr.FromString("mcrouter"), - }, - }, - PeriodSeconds: int32(10), - }, - LivenessProbe: &v1.Probe{ - Handler: v1.Handler{ - TCPSocket: &v1.TCPSocketAction{ - Port: intstr.FromString("mcrouter"), - }, - }, - InitialDelaySeconds: int32(15), - PeriodSeconds: int32(30), - }, - }, - { - Name: "exporter", - Image: fmt.Sprintf("vexxhost/mcrouter_exporter:%s", version.Revision), - Args: []string{"-mcrouter.address", "localhost:11211"}, - Ports: []v1.ContainerPort{ - { - Name: "metrics", - ContainerPort: int32(9442), - }, - }, - Resources: v1.ResourceRequirements{ - Limits: v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(1000, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(int64(units.Mebibyte)*256, resource.BinarySI), - v1.ResourceEphemeralStorage: *resource.NewQuantity(int64(units.MB)*1000, resource.DecimalSI), - }, - Requests: v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(int64(units.Mebibyte)*128, resource.BinarySI), - v1.ResourceEphemeralStorage: *resource.NewQuantity(int64(units.MB)*500, resource.DecimalSI), - }, - }, - - StartupProbe: &v1.Probe{}, - ReadinessProbe: &v1.Probe{ - Handler: v1.Handler{ - HTTPGet: &v1.HTTPGetAction{ - Path: string("/metrics"), - Port: intstr.FromString("metrics"), - }, - }, - InitialDelaySeconds: int32(5), - PeriodSeconds: int32(10), - }, - LivenessProbe: &v1.Probe{ - Handler: v1.Handler{ - HTTPGet: &v1.HTTPGetAction{ - Path: string("/metrics"), - Port: intstr.FromString("metrics"), - }, - }, - InitialDelaySeconds: int32(15), - PeriodSeconds: int32(30), - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: v1.LocalObjectReference{Name: configMap.GetName()}, - }, - }, - }, - }, - NodeSelector: mcrouter.Spec.NodeSelector, - Tolerations: mcrouter.Spec.Tolerations, - }, - } - - return controllerutil.SetControllerReference(&mcrouter, deployment, r.Scheme) + op, err = utils.CreateOrUpdate(ctx, r, deployment, func() error { + return builders.Deployment(deployment, &mcrouter, r.Scheme). + Labels(labels). + Replicas(2). + PodTemplateSpec( + builders.PodTemplateSpec(). + PodSpec( + builders.PodSpec(). + NodeSelector(mcrouter.Spec.NodeSelector). + Tolerations(mcrouter.Spec.Tolerations). + Containers( + builders.Container("mcrouter", "vexxhost/mcrouter:latest"). + Args("-p", "11211", "-f", "/data/config.json"). + Port("mcrouter", 11211).PortProbe("mcrouter", 10, 30). + Resources(500, 128, 500, 2). + Volume("config", "/data"). + SecurityContext( + builders.SecurityContext(). + RunAsUser(999). + RunAsGroup(999), + ), + builders.Container("exporter", "vexxhost/mcrouter_exporter:latest"). + Args("-mcrouter.address", "localhost:11211"). + Port("metrics", 9442).HTTPProbe("metrics", "/metrics", 10, 30). + Resources(500, 128, 500, 2). + SecurityContext( + builders.SecurityContext(). + RunAsUser(1001), + ), + ). + Volumes( + builders.Volume("config").FromConfigMap(configMap.GetName()), + ), + ), + ). + Build() }) if err != nil { return ctrl.Result{}, err @@ -220,21 +123,13 @@ func (r *McrouterReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { ObjectMeta: metav1.ObjectMeta{ Namespace: req.Namespace, Name: fmt.Sprintf("mcrouter-%s", req.Name), - Labels: labels, }, } - op, err = controllerutil.CreateOrUpdate(ctx, r, service, func() error { - service.Spec.Type = corev1.ServiceTypeClusterIP - service.Spec.Ports = []v1.ServicePort{ - { - Name: "mcrouter", - Port: int32(11211), - TargetPort: intstr.FromString("mcrouter"), - }, - } - service.Spec.Selector = labels - - return controllerutil.SetControllerReference(&mcrouter, service, r.Scheme) + op, err = utils.CreateOrUpdate(ctx, r, service, func() error { + return builders.Service(service, &mcrouter, r.Scheme). + Port("mcrouter", 11211). + Selector(labels). + Build() }) if err != nil { return ctrl.Result{}, err diff --git a/controllers/memcached_controller.go b/controllers/memcached_controller.go index eed45cb2..544d7099 100755 --- a/controllers/memcached_controller.go +++ b/controllers/memcached_controller.go @@ -19,24 +19,21 @@ package controllers import ( "context" "fmt" + "sort" "strconv" - "github.com/alecthomas/units" "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" infrastructurev1alpha1 "opendev.org/vexxhost/openstack-operator/api/v1alpha1" - "opendev.org/vexxhost/openstack-operator/version" + "opendev.org/vexxhost/openstack-operator/builders" + "opendev.org/vexxhost/openstack-operator/utils" ) // MemcachedReconciler reconciles a Memcached object @@ -79,108 +76,37 @@ func (r *MemcachedReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { Labels: labels, }, } - op, err := controllerutil.CreateOrUpdate(ctx, r, deployment, func() error { - if deployment.ObjectMeta.CreationTimestamp.IsZero() { - deployment.Spec.Selector = &metav1.LabelSelector{ - MatchLabels: labels, - } - } - - deployment.Spec.Replicas = pointer.Int32Ptr(2) - deployment.Spec.Template = corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "memcached", - Image: fmt.Sprintf("vexxhost/memcached:%s", version.Revision), - Args: []string{"-m", strconv.Itoa(size)}, - Ports: []v1.ContainerPort{ - { - Name: "memcached", - ContainerPort: int32(11211), - }, - }, - Resources: v1.ResourceRequirements{ - Limits: v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(1000, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(int64(size)*int64(units.MiB)+int64(size)*102*int64(units.KiB), resource.BinarySI), - v1.ResourceEphemeralStorage: *resource.NewQuantity(int64(units.MB)*1000, resource.DecimalSI), - }, - Requests: v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(int64(size)*int64(units.MiB), resource.BinarySI), - v1.ResourceEphemeralStorage: *resource.NewQuantity(int64(units.MB)*500, resource.DecimalSI), - }, - }, - StartupProbe: &v1.Probe{}, - ReadinessProbe: &v1.Probe{ - Handler: v1.Handler{ - TCPSocket: &v1.TCPSocketAction{ - Port: intstr.FromString("memcached"), - }, - }, - PeriodSeconds: int32(10), - }, - LivenessProbe: &v1.Probe{ - Handler: v1.Handler{ - TCPSocket: &v1.TCPSocketAction{ - Port: intstr.FromString("memcached"), - }, - }, - InitialDelaySeconds: int32(15), - PeriodSeconds: int32(30), - }, - }, - { - Name: "exporter", - Image: fmt.Sprintf("vexxhost/memcached_exporter:%s", version.Revision), - Ports: []v1.ContainerPort{ - { - Name: "metrics", - ContainerPort: int32(9150), - }, - }, - Resources: v1.ResourceRequirements{ - Limits: v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(1000, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(int64(units.Mebibyte)*256, resource.BinarySI), - v1.ResourceEphemeralStorage: *resource.NewQuantity(int64(units.MB)*1000, resource.DecimalSI), - }, - Requests: v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(int64(units.Mebibyte)*128, resource.BinarySI), - v1.ResourceEphemeralStorage: *resource.NewQuantity(int64(units.MB)*500, resource.DecimalSI), - }, - }, - StartupProbe: &v1.Probe{}, - ReadinessProbe: &v1.Probe{ - Handler: v1.Handler{ - HTTPGet: &v1.HTTPGetAction{ - Path: string("/metrics"), - Port: intstr.FromString("metrics"), - }, - }, - PeriodSeconds: int32(10), - }, - LivenessProbe: &v1.Probe{ - Handler: v1.Handler{ - HTTPGet: &v1.HTTPGetAction{ - Path: string("/metrics"), - Port: intstr.FromString("metrics"), - }, - }, - InitialDelaySeconds: int32(15), - PeriodSeconds: int32(20), - }, - }, - }, - }, - } - - return controllerutil.SetControllerReference(&memcached, deployment, r.Scheme) + op, err := utils.CreateOrUpdate(ctx, r, deployment, func() error { + return builders.Deployment(deployment, &memcached, r.Scheme). + Labels(labels). + Replicas(2). + PodTemplateSpec( + builders.PodTemplateSpec(). + Labels(labels). + PodSpec( + builders.PodSpec(). + NodeSelector(memcached.Spec.NodeSelector). + Tolerations(memcached.Spec.Tolerations). + Containers( + builders.Container("memcached", "vexxhost/memcached:latest"). + Args("-m", strconv.Itoa(size)). + Port("memcached", 11211).PortProbe("memcached", 10, 30). + Resources(1000, int64(size), 500, 1.10). + SecurityContext( + builders.SecurityContext(). + RunAsUser(1001), + ), + builders.Container("exporter", "vexxhost/memcached_exporter:latest"). + Port("metrics", 9150).HTTPProbe("metrics", "/metrics", 10, 30). + Resources(500, 128, 500, 2). + SecurityContext( + builders.SecurityContext(). + RunAsUser(1001), + ), + ), + ), + ). + Build() }) if err != nil { return ctrl.Result{}, err @@ -211,6 +137,9 @@ func (r *MemcachedReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { return ctrl.Result{Requeue: true}, nil } + // Make sure that they're sorted so we're idempotent + sort.Strings(servers) + // Mcrouter mcrouter := &infrastructurev1alpha1.Mcrouter{ ObjectMeta: metav1.ObjectMeta{ @@ -245,5 +174,6 @@ func (r *MemcachedReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&infrastructurev1alpha1.Memcached{}). Owns(&appsv1.Deployment{}). + Owns(&infrastructurev1alpha1.Mcrouter{}). Complete(r) } diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 3f9da50f..0d049b95 100755 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -72,7 +72,6 @@ var _ = BeforeSuite(func(done Done) { Expect(err).NotTo(HaveOccurred()) // +kubebuilder:scaffold:scheme - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) Expect(err).ToNot(HaveOccurred()) Expect(k8sClient).ToNot(BeNil()) diff --git a/go.mod b/go.mod index eebf63e9..cde54e39 100755 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.13 require ( github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf github.com/go-logr/logr v0.1.0 + github.com/google/go-cmp v0.3.0 github.com/onsi/ginkgo v1.11.0 github.com/onsi/gomega v1.8.1 k8s.io/api v0.17.2 diff --git a/images/mcrouter/Dockerfile b/images/mcrouter/Dockerfile index e66ffe9c..7b8f1e08 100644 --- a/images/mcrouter/Dockerfile +++ b/images/mcrouter/Dockerfile @@ -1,5 +1,6 @@ FROM ubuntu:bionic - +RUN groupadd -r mcrouter \ + && useradd -r -g mcrouter mcrouter RUN apt update && \ apt install -y --no-install-recommends ca-certificates wget gnupg && \ wget -O - https://facebook.github.io/mcrouter/debrepo/bionic/PUBLIC.KEY | apt-key add && \ @@ -9,5 +10,8 @@ RUN apt update && \ apt remove -y wget gnupg && \ apt autoremove -y && \ apt clean all - -ENTRYPOINT ["/usr/bin/mcrouter"] \ No newline at end of file +RUN chown -R mcrouter:mcrouter /var/spool/mcrouter +RUN chown -R mcrouter:mcrouter /var/mcrouter +RUN chown -R mcrouter:mcrouter /usr/bin/mcrouter +USER mcrouter +ENTRYPOINT ["/usr/bin/mcrouter"] diff --git a/playbooks/functional/post.yaml b/playbooks/functional/post.yaml old mode 100644 new mode 100755 index 6a8735c4..5b366979 --- a/playbooks/functional/post.yaml +++ b/playbooks/functional/post.yaml @@ -1,3 +1,4 @@ - hosts: all roles: - - collect-container-logs \ No newline at end of file + - collect-container-logs + - collect-kubernetes-logs \ No newline at end of file diff --git a/utils/kubernetes.go b/utils/kubernetes.go new file mode 100644 index 00000000..f8f63c5f --- /dev/null +++ b/utils/kubernetes.go @@ -0,0 +1,35 @@ +package utils + +import ( + "context" + "fmt" + + "github.com/google/go-cmp/cmp" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +// CreateOrUpdate wraps the function provided by controller-runtime to include +// some additional logging and common functionality across all resources. +func CreateOrUpdate(ctx context.Context, c client.Client, obj runtime.Object, f controllerutil.MutateFn) (controllerutil.OperationResult, error) { + return controllerutil.CreateOrUpdate(ctx, c, obj, func() error { + original := obj.DeepCopyObject() + + err := f() + if err != nil { + return err + } + + generateObjectDiff(original, obj) + return nil + }) +} + +func generateObjectDiff(original runtime.Object, modified runtime.Object) { + diff := cmp.Diff(original, modified) + + if len(diff) != 0 { + fmt.Println(diff) + } +}