alertrule

Create PrometheusRule objects for the alerting.
One rule is created per instance while one podmonitor is created per type.

Change-Id: I43d436fd46ecd1ad92547dec972fdab3fd2ae05e

api/monitoring/v1/monitor_types.go

@@ -11,12 +11,15 @@ const (
 	PodMonitorsKind   = "PodMonitor"
 	PodMonitorName    = "podmonitors"
 	PodMonitorKindKey = "podmonitor"
+
+	PrometheusRuleKind    = "PrometheusRule"
+	PrometheusRuleName    = "prometheusrules"
+	PrometheusRuleKindKey = "prometheusrule"
 )
 
 // PodMonitor defines monitoring for a set of pods.
 // +genclient
 // +k8s:openapi-gen=true
-
 // +kubebuilder:object:root=true
 type PodMonitor struct {
 	metav1.TypeMeta   `json:",inline"`
@@ -44,7 +47,6 @@ type PodMonitorSpec struct {
 
 // PodMonitorList is a list of PodMonitors.
 // +k8s:openapi-gen=true
-
 // +kubebuilder:object:root=true
 type PodMonitorList struct {
 	metav1.TypeMeta `json:",inline"`
@@ -125,9 +127,67 @@ type NamespaceSelector struct {
 	// implementation to support label selections.
 }
 
+// PrometheusRuleList is a list of PrometheusRules.
+// +k8s:openapi-gen=true
+// +kubebuilder:object:root=true
+type PrometheusRuleList struct {
+	metav1.TypeMeta `json:",inline"`
+	// Standard list metadata
+	// More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata
+	metav1.ListMeta `json:"metadata,omitempty"`
+	// List of Rules
+	Items []*PrometheusRule `json:"items"`
+}
+
+// PrometheusRule defines alerting rules for a Prometheus instance
+// +genclient
+// +k8s:openapi-gen=true
+// +kubebuilder:object:root=true
+type PrometheusRule struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+	// Specification of desired alerting rule definitions for Prometheus.
+	Spec PrometheusRuleSpec `json:"spec"`
+}
+
+// PrometheusRuleSpec contains specification parameters for a Rule.
+// +k8s:openapi-gen=true
+type PrometheusRuleSpec struct {
+	// Content of Prometheus rule file
+	Groups []RuleGroup `json:"groups,omitempty"`
+}
+
+// RuleGroup and Rule are copied instead of vendored because the
+// upstream Prometheus struct definitions don't have json struct tags.
+
+// RuleGroup is a list of sequentially evaluated recording and alerting rules.
+// Note: PartialResponseStrategy is only used by ThanosRuler and will
+// be ignored by Prometheus instances.  Valid values for this field are 'warn'
+// or 'abort'.  More info: https://github.com/thanos-io/thanos/blob/master/docs/components/rule.md#partial-response
+// +k8s:openapi-gen=true
+type RuleGroup struct {
+	Name                    string `json:"name"`
+	Interval                string `json:"interval,omitempty"`
+	Rules                   []Rule `json:"rules"`
+	PartialResponseStrategy string `json:"partial_response_strategy,omitempty"`
+}
+
+// Rule describes an alerting or recording rule.
+// +k8s:openapi-gen=true
+type Rule struct {
+	Record      string             `json:"record,omitempty"`
+	Alert       string             `json:"alert,omitempty"`
+	Expr        intstr.IntOrString `json:"expr"`
+	For         string             `json:"for,omitempty"`
+	Labels      map[string]string  `json:"labels,omitempty"`
+	Annotations map[string]string  `json:"annotations,omitempty"`
+}
+
 // +kubebuilder:rbac:groups=monitoring.coreos.com,resources=podmonitors,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=monitoring.coreos.com,resources=podmonitors/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=prometheusrules,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=prometheusrules/status,verbs=get;update;patch
 
 func init() {
-	SchemeBuilder.Register(&PodMonitor{}, &PodMonitorList{})
+	SchemeBuilder.Register(&PodMonitor{}, &PodMonitorList{}, &PrometheusRule{}, &PrometheusRuleList{})
 }

api/monitoring/v1/zz_generated.deepcopy.go

@@ -203,6 +203,90 @@ func (in *PodMonitorSpec) DeepCopy() *PodMonitorSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PrometheusRule) DeepCopyInto(out *PrometheusRule) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrometheusRule.
+func (in *PrometheusRule) DeepCopy() *PrometheusRule {
+	if in == nil {
+		return nil
+	}
+	out := new(PrometheusRule)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *PrometheusRule) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PrometheusRuleList) DeepCopyInto(out *PrometheusRuleList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]*PrometheusRule, len(*in))
+		for i := range *in {
+			if (*in)[i] != nil {
+				in, out := &(*in)[i], &(*out)[i]
+				*out = new(PrometheusRule)
+				(*in).DeepCopyInto(*out)
+			}
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrometheusRuleList.
+func (in *PrometheusRuleList) DeepCopy() *PrometheusRuleList {
+	if in == nil {
+		return nil
+	}
+	out := new(PrometheusRuleList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *PrometheusRuleList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PrometheusRuleSpec) DeepCopyInto(out *PrometheusRuleSpec) {
+	*out = *in
+	if in.Groups != nil {
+		in, out := &in.Groups, &out.Groups
+		*out = make([]RuleGroup, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrometheusRuleSpec.
+func (in *PrometheusRuleSpec) DeepCopy() *PrometheusRuleSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(PrometheusRuleSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *RelabelConfig) DeepCopyInto(out *RelabelConfig) {
 	*out = *in
@@ -222,3 +306,55 @@ func (in *RelabelConfig) DeepCopy() *RelabelConfig {
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Rule) DeepCopyInto(out *Rule) {
+	*out = *in
+	out.Expr = in.Expr
+	if in.Labels != nil {
+		in, out := &in.Labels, &out.Labels
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.Annotations != nil {
+		in, out := &in.Annotations, &out.Annotations
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Rule.
+func (in *Rule) DeepCopy() *Rule {
+	if in == nil {
+		return nil
+	}
+	out := new(Rule)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *RuleGroup) DeepCopyInto(out *RuleGroup) {
+	*out = *in
+	if in.Rules != nil {
+		in, out := &in.Rules, &out.Rules
+		*out = make([]Rule, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuleGroup.
+func (in *RuleGroup) DeepCopy() *RuleGroup {
+	if in == nil {
+		return nil
+	}
+	out := new(RuleGroup)
+	in.DeepCopyInto(out)
+	return out
+}

builders/prometheus_rule.go

@@ -0,0 +1,63 @@
+package builders
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	monitoringv1 "opendev.org/vexxhost/openstack-operator/api/monitoring/v1"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+)
+
+// PrometheusRuleBuilder provides an interface to build PrometheusRules
+type PrometheusRuleBuilder struct {
+	obj        *monitoringv1.PrometheusRule
+	ruleGroups []*RuleGroupBuilder
+	owner      metav1.Object
+	scheme     *runtime.Scheme
+}
+
+// PrometheusRule returns a new PrometheusRule builder
+func PrometheusRule(existing *monitoringv1.PrometheusRule, owner metav1.Object, scheme *runtime.Scheme) *PrometheusRuleBuilder {
+	return &PrometheusRuleBuilder{
+		obj:    existing,
+		owner:  owner,
+		scheme: scheme,
+	}
+}
+
+// RuleGroups returns the ruleGroups
+func (pm *PrometheusRuleBuilder) RuleGroups(ruleGroups ...*RuleGroupBuilder) *PrometheusRuleBuilder {
+	pm.ruleGroups = ruleGroups
+	return pm
+}
+
+// Build returns the object after making certain assertions
+func (pm *PrometheusRuleBuilder) Build() error {
+	pm.obj.Spec.Groups = []monitoringv1.RuleGroup{}
+	for _, rgBuilder := range pm.ruleGroups {
+		ruleGroup, err := rgBuilder.Build()
+		if err != nil {
+			return err
+		}
+
+		pm.obj.Spec.Groups = append(pm.obj.Spec.Groups, ruleGroup)
+	}
+	if !pm.isOwnedByOthers() {
+		return controllerutil.SetControllerReference(pm.owner, pm.obj, pm.scheme)
+	}
+	return nil
+}
+
+// isOwnedByOthers checks if this podMonitor has been possessed by an another object already.
+func (pm *PrometheusRuleBuilder) isOwnedByOthers() bool {
+	ownerName := pm.owner.GetName()
+
+	existingRefs := pm.obj.GetOwnerReferences()
+	for _, r := range existingRefs {
+		if r.Name == ownerName {
+			return false
+		} else if r.Controller != nil && *r.Controller {
+			return true
+		}
+	}
+	return false
+}

builders/rule.go

@@ -0,0 +1,53 @@
+package builders
+
+import (
+	"strconv"
+
+	"k8s.io/apimachinery/pkg/util/intstr"
+	monitoringv1 "opendev.org/vexxhost/openstack-operator/api/monitoring/v1"
+)
+
+// RuleBuilder provides an interface to build rule
+type RuleBuilder struct {
+	obj *monitoringv1.Rule
+}
+
+// Rule returns a new podmonitor builder
+func Rule() *RuleBuilder {
+	Rule := &monitoringv1.Rule{
+		Annotations: map[string]string{},
+	}
+	return &RuleBuilder{
+		obj: Rule,
+	}
+}
+
+func (r *RuleBuilder) Alert(alert string) *RuleBuilder {
+	r.obj.Alert = alert
+	return r
+}
+
+func (r *RuleBuilder) Expr(expr string) *RuleBuilder {
+	r.obj.Expr = intstr.FromString(expr)
+	return r
+}
+
+func (r *RuleBuilder) For(duration string) *RuleBuilder {
+	r.obj.For = duration
+	return r
+}
+
+func (r *RuleBuilder) Priority(p int) *RuleBuilder {
+	r.obj.Annotations["priority"] = "P" + strconv.Itoa(p)
+	return r
+}
+
+func (r *RuleBuilder) Message(m string) *RuleBuilder {
+	r.obj.Annotations["message"] = m
+	return r
+}
+
+// Build returns the object after making certain assertions
+func (r *RuleBuilder) Build() (monitoringv1.Rule, error) {
+	return *r.obj, nil
+}

builders/rule_group.go

@@ -0,0 +1,55 @@
+package builders
+
+import (
+	monitoringv1 "opendev.org/vexxhost/openstack-operator/api/monitoring/v1"
+)
+
+// RuleGroupBuilder provides an interface to build RuleGroup
+type RuleGroupBuilder struct {
+	obj   *monitoringv1.RuleGroup
+	rules []*RuleBuilder
+}
+
+// RuleGroup returns a new rulegroup builder
+func RuleGroup() *RuleGroupBuilder {
+	RuleGroup := &monitoringv1.RuleGroup{}
+	return &RuleGroupBuilder{
+		obj: RuleGroup,
+	}
+}
+
+func (r *RuleGroupBuilder) Name(Name string) *RuleGroupBuilder {
+	r.obj.Name = Name
+	return r
+}
+
+func (r *RuleGroupBuilder) Interval(Interval string) *RuleGroupBuilder {
+	r.obj.Interval = Interval
+	return r
+}
+
+func (r *RuleGroupBuilder) Rules(Rules ...*RuleBuilder) *RuleGroupBuilder {
+	r.rules = Rules
+	return r
+}
+
+func (r *RuleGroupBuilder) PartialResponseStrategy(prs string) *RuleGroupBuilder {
+	r.obj.PartialResponseStrategy = prs
+	return r
+}
+
+// Build returns the object after making certain assertions
+func (r *RuleGroupBuilder) Build() (monitoringv1.RuleGroup, error) {
+
+	r.obj.Rules = []monitoringv1.Rule{}
+	for _, rBuilder := range r.rules {
+		rule, err := rBuilder.Build()
+		if err != nil {
+			return monitoringv1.RuleGroup{}, err
+		}
+
+		r.obj.Rules = append(r.obj.Rules, rule)
+	}
+
+	return *r.obj, nil
+}

chart/crds/monitoring.coreos.com_podmonitors.yaml

@@ -17,6 +17,7 @@ spec:
   scope: Namespaced
   validation:
     openAPIV3Schema:
+      description: PodMonitor defines monitoring for a set of pods.
       properties:
         apiVersion:
           description: 'APIVersion defines the versioned schema of this representation

chart/crds/monitoring.coreos.com_prometheusrules.yaml

@@ -0,0 +1,97 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.2.5
+  creationTimestamp: null
+  name: prometheusrules.monitoring.coreos.com
+spec:
+  group: monitoring.coreos.com
+  names:
+    kind: PrometheusRule
+    listKind: PrometheusRuleList
+    plural: prometheusrules
+    singular: prometheusrule
+  scope: Namespaced
+  validation:
+    openAPIV3Schema:
+      description: PrometheusRule defines alerting rules for a Prometheus instance
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+          type: string
+        metadata:
+          type: object
+        spec:
+          description: Specification of desired alerting rule definitions for Prometheus.
+          properties:
+            groups:
+              description: Content of Prometheus rule file
+              items:
+                description: 'RuleGroup is a list of sequentially evaluated recording
+                  and alerting rules. Note: PartialResponseStrategy is only used by
+                  ThanosRuler and will be ignored by Prometheus instances.  Valid
+                  values for this field are ''warn'' or ''abort''.  More info: https://github.com/thanos-io/thanos/blob/master/docs/components/rule.md#partial-response'
+                properties:
+                  interval:
+                    type: string
+                  name:
+                    type: string
+                  partial_response_strategy:
+                    type: string
+                  rules:
+                    items:
+                      description: Rule describes an alerting or recording rule.
+                      properties:
+                        alert:
+                          type: string
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        expr:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          x-kubernetes-int-or-string: true
+                        for:
+                          type: string
+                        labels:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        record:
+                          type: string
+                      required:
+                      - expr
+                      type: object
+                    type: array
+                required:
+                - name
+                - rules
+                type: object
+              type: array
+          type: object
+      required:
+      - spec
+      type: object
+  version: v1
+  versions:
+  - name: v1
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

chart/templates/clusterrole.yaml

@@ -103,3 +103,23 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - monitoring.coreos.com
+  resources:
+  - prometheusrules
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - monitoring.coreos.com
+  resources:
+  - prometheusrules/status
+  verbs:
+  - get
+  - patch
+  - update

config/crd/bases/monitoring.coreos.com_podmonitors.yaml

@@ -17,6 +17,7 @@ spec:
   scope: Namespaced
   validation:
     openAPIV3Schema:
+      description: PodMonitor defines monitoring for a set of pods.
       properties:
         apiVersion:
           description: 'APIVersion defines the versioned schema of this representation

config/crd/bases/monitoring.coreos.com_prometheusrules.yaml

@@ -0,0 +1,97 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.2.5
+  creationTimestamp: null
+  name: prometheusrules.monitoring.coreos.com
+spec:
+  group: monitoring.coreos.com
+  names:
+    kind: PrometheusRule
+    listKind: PrometheusRuleList
+    plural: prometheusrules
+    singular: prometheusrule
+  scope: Namespaced
+  validation:
+    openAPIV3Schema:
+      description: PrometheusRule defines alerting rules for a Prometheus instance
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+          type: string
+        metadata:
+          type: object
+        spec:
+          description: Specification of desired alerting rule definitions for Prometheus.
+          properties:
+            groups:
+              description: Content of Prometheus rule file
+              items:
+                description: 'RuleGroup is a list of sequentially evaluated recording
+                  and alerting rules. Note: PartialResponseStrategy is only used by
+                  ThanosRuler and will be ignored by Prometheus instances.  Valid
+                  values for this field are ''warn'' or ''abort''.  More info: https://github.com/thanos-io/thanos/blob/master/docs/components/rule.md#partial-response'
+                properties:
+                  interval:
+                    type: string
+                  name:
+                    type: string
+                  partial_response_strategy:
+                    type: string
+                  rules:
+                    items:
+                      description: Rule describes an alerting or recording rule.
+                      properties:
+                        alert:
+                          type: string
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        expr:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          x-kubernetes-int-or-string: true
+                        for:
+                          type: string
+                        labels:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        record:
+                          type: string
+                      required:
+                      - expr
+                      type: object
+                    type: array
+                required:
+                - name
+                - rules
+                type: object
+              type: array
+          type: object
+      required:
+      - spec
+      type: object
+  version: v1
+  versions:
+  - name: v1
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

config/crd/kustomization.yaml

@@ -5,6 +5,7 @@ resources:
 - bases/infrastructure.vexxhost.cloud_mcrouters.yaml
 - bases/infrastructure.vexxhost.cloud_memcacheds.yaml
 - bases/monitoring.coreos.com_podmonitors.yaml
+- bases/monitoring.coreos.com_prometheusrules.yaml
 # +kubebuilder:scaffold:crdkustomizeresource
 
 patchesStrategicMerge:

config/rbac/role.yaml

@@ -103,3 +103,23 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - monitoring.coreos.com
+  resources:
+  - prometheusrules
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - monitoring.coreos.com
+  resources:
+  - prometheusrules/status
+  verbs:
+  - get
+  - patch
+  - update

controllers/mcrouter_controller.go

@@ -28,7 +28,7 @@ type McrouterReconciler struct {
 
 // +kubebuilder:rbac:groups=infrastructure.vexxhost.cloud,resources=mcrouters,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=infrastructure.vexxhost.cloud,resources=mcrouters/status,verbs=get;update;patch
-
+// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=prometheusrules,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=monitoring.coreos.com,resources=podmonitors,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=configmaps;services,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
@@ -47,6 +47,7 @@ func (r *McrouterReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 	labels := map[string]string{
 		"app.kubernetes.io/name":       "mcrouter",
 		"app.kubernetes.io/instance":   req.Name,
+		"app.kubernetes.io/managed-by": "openstack-operator",
 	}
 
 	// ConfigMap
@@ -58,6 +59,7 @@ func (r *McrouterReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 	}
 	op, err := utils.CreateOrUpdate(ctx, r, configMap, func() error {
 		b, err := json.Marshal(mcrouter.Spec)
+
 		if err != nil {
 			return err
 		}
@@ -131,6 +133,7 @@ func (r *McrouterReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 			Name:      fmt.Sprintf("mcrouter-podmonitor"),
 			Labels: map[string]string{
 				"app.kubernetes.io/name":       "mcrouter",
+				"app.kubernetes.io/managed-by": "openstack-operator",
 			},
 		},
 	}
@@ -151,7 +154,40 @@ func (r *McrouterReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 	if err != nil {
 		return ctrl.Result{}, err
 	}
-	log.WithValues("resource", "podmonitor").WithValues("op", op).Info("Reconciled")
+	log.WithValues("resource", "mcrouter-podmonitor").WithValues("op", op).Info("Reconciled")
+
+	// Alertrule
+	alertRule := &monitoringv1.PrometheusRule{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: req.Namespace,
+			Name:      fmt.Sprintf("mcrouter-alertrule"),
+		},
+	}
+	op, err = utils.CreateOrUpdate(ctx, r, alertRule, func() error {
+
+		return builders.PrometheusRule(alertRule, &mcrouter, r.Scheme).
+			RuleGroups(builders.RuleGroup().
+				Name("mcrouter-rule").
+				Rules(
+
+					builders.Rule().
+						Alert("McrouterBackendDown").
+						Message("Backend Memcached servers are down.").
+						Priority(1).
+						Expr("mcrouter_servers{state='down'}!=0"),
+					builders.Rule().
+						Alert("McrouterBackendTimeout").
+						Message("Backend Memcached servers are timeout.").
+						Priority(1).
+						Expr("mcrouter_server_memcached_timeout_count>0"),
+				).
+				Interval("1m")).
+			Build()
+	})
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	log.WithValues("resource", "mcrouter-alertrule").WithValues("op", op).Info("Reconciled")
 
 	// Service
 	service := &corev1.Service{

controllers/memcached_controller.go

@@ -47,7 +47,7 @@ type MemcachedReconciler struct {
 
 // +kubebuilder:rbac:groups=infrastructure.vexxhost.cloud,resources=memcacheds,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=infrastructure.vexxhost.cloud,resources=memcacheds/status,verbs=get;update;patch
-
+// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=prometheusrules,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=monitoring.coreos.com,resources=podmonitors,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
@@ -69,6 +69,7 @@ func (r *MemcachedReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 	labels := map[string]string{
 		"app.kubernetes.io/name":       "memcached",
 		"app.kubernetes.io/instance":   req.Name,
+		"app.kubernetes.io/managed-by": "openstack-operator",
 	}
 
 	// Deployment
@@ -129,6 +130,7 @@ func (r *MemcachedReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 			Name:      fmt.Sprintf("memcached-podmonitor"),
 			Labels: map[string]string{
 				"app.kubernetes.io/name":       "memcached",
+				"app.kubernetes.io/managed-by": "openstack-operator",
 			},
 		},
 	}
@@ -175,6 +177,34 @@ func (r *MemcachedReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 		return ctrl.Result{Requeue: true}, nil
 	}
 
+	// Alertrule
+	alertRule := &monitoringv1.PrometheusRule{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: req.Namespace,
+			Name:      fmt.Sprintf("memcached-alertrule"),
+		},
+	}
+	op, err = utils.CreateOrUpdate(ctx, r, alertRule, func() error {
+
+		return builders.PrometheusRule(alertRule, &memcached, r.Scheme).
+			RuleGroups(builders.RuleGroup().
+				Name("memcached-rule").
+				Rules(
+
+					builders.Rule().
+						Alert("MemcachedConnectionLimit").
+						Message("This memcached connection is over max.").
+						Priority(1).
+						Expr("memcached_current_connections/memcached_max_connections*100 >90"),
+				).
+				Interval("1m")).
+			Build()
+	})
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	log.WithValues("resource", "memcached-alertrule").WithValues("op", op).Info("Reconciled")
+
 	// Make sure that they're sorted so we're idempotent
 	sort.Strings(servers)