diff --git a/api/monitoring/v1/monitor_types.go b/api/monitoring/v1/monitor_types.go index 77f11169..bd84d3a3 100755 --- a/api/monitoring/v1/monitor_types.go +++ b/api/monitoring/v1/monitor_types.go @@ -11,12 +11,15 @@ const ( PodMonitorsKind = "PodMonitor" PodMonitorName = "podmonitors" PodMonitorKindKey = "podmonitor" + + PrometheusRuleKind = "PrometheusRule" + PrometheusRuleName = "prometheusrules" + PrometheusRuleKindKey = "prometheusrule" ) // PodMonitor defines monitoring for a set of pods. // +genclient // +k8s:openapi-gen=true - // +kubebuilder:object:root=true type PodMonitor struct { metav1.TypeMeta `json:",inline"` @@ -44,7 +47,6 @@ type PodMonitorSpec struct { // PodMonitorList is a list of PodMonitors. // +k8s:openapi-gen=true - // +kubebuilder:object:root=true type PodMonitorList struct { metav1.TypeMeta `json:",inline"` @@ -125,9 +127,67 @@ type NamespaceSelector struct { // implementation to support label selections. } +// PrometheusRuleList is a list of PrometheusRules. +// +k8s:openapi-gen=true +// +kubebuilder:object:root=true +type PrometheusRuleList struct { + metav1.TypeMeta `json:",inline"` + // Standard list metadata + // More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata + metav1.ListMeta `json:"metadata,omitempty"` + // List of Rules + Items []*PrometheusRule `json:"items"` +} + +// PrometheusRule defines alerting rules for a Prometheus instance +// +genclient +// +k8s:openapi-gen=true +// +kubebuilder:object:root=true +type PrometheusRule struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + // Specification of desired alerting rule definitions for Prometheus. + Spec PrometheusRuleSpec `json:"spec"` +} + +// PrometheusRuleSpec contains specification parameters for a Rule. +// +k8s:openapi-gen=true +type PrometheusRuleSpec struct { + // Content of Prometheus rule file + Groups []RuleGroup `json:"groups,omitempty"` +} + +// RuleGroup and Rule are copied instead of vendored because the +// upstream Prometheus struct definitions don't have json struct tags. + +// RuleGroup is a list of sequentially evaluated recording and alerting rules. +// Note: PartialResponseStrategy is only used by ThanosRuler and will +// be ignored by Prometheus instances. Valid values for this field are 'warn' +// or 'abort'. More info: https://github.com/thanos-io/thanos/blob/master/docs/components/rule.md#partial-response +// +k8s:openapi-gen=true +type RuleGroup struct { + Name string `json:"name"` + Interval string `json:"interval,omitempty"` + Rules []Rule `json:"rules"` + PartialResponseStrategy string `json:"partial_response_strategy,omitempty"` +} + +// Rule describes an alerting or recording rule. +// +k8s:openapi-gen=true +type Rule struct { + Record string `json:"record,omitempty"` + Alert string `json:"alert,omitempty"` + Expr intstr.IntOrString `json:"expr"` + For string `json:"for,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` +} + // +kubebuilder:rbac:groups=monitoring.coreos.com,resources=podmonitors,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=monitoring.coreos.com,resources=podmonitors/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=prometheusrules,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=prometheusrules/status,verbs=get;update;patch func init() { - SchemeBuilder.Register(&PodMonitor{}, &PodMonitorList{}) + SchemeBuilder.Register(&PodMonitor{}, &PodMonitorList{}, &PrometheusRule{}, &PrometheusRuleList{}) } diff --git a/api/monitoring/v1/zz_generated.deepcopy.go b/api/monitoring/v1/zz_generated.deepcopy.go index 8bae4be0..0f157ebf 100755 --- a/api/monitoring/v1/zz_generated.deepcopy.go +++ b/api/monitoring/v1/zz_generated.deepcopy.go @@ -203,6 +203,90 @@ func (in *PodMonitorSpec) DeepCopy() *PodMonitorSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PrometheusRule) DeepCopyInto(out *PrometheusRule) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrometheusRule. +func (in *PrometheusRule) DeepCopy() *PrometheusRule { + if in == nil { + return nil + } + out := new(PrometheusRule) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PrometheusRule) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PrometheusRuleList) DeepCopyInto(out *PrometheusRuleList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]*PrometheusRule, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(PrometheusRule) + (*in).DeepCopyInto(*out) + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrometheusRuleList. +func (in *PrometheusRuleList) DeepCopy() *PrometheusRuleList { + if in == nil { + return nil + } + out := new(PrometheusRuleList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PrometheusRuleList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PrometheusRuleSpec) DeepCopyInto(out *PrometheusRuleSpec) { + *out = *in + if in.Groups != nil { + in, out := &in.Groups, &out.Groups + *out = make([]RuleGroup, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrometheusRuleSpec. +func (in *PrometheusRuleSpec) DeepCopy() *PrometheusRuleSpec { + if in == nil { + return nil + } + out := new(PrometheusRuleSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RelabelConfig) DeepCopyInto(out *RelabelConfig) { *out = *in @@ -222,3 +306,55 @@ func (in *RelabelConfig) DeepCopy() *RelabelConfig { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Rule) DeepCopyInto(out *Rule) { + *out = *in + out.Expr = in.Expr + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Rule. +func (in *Rule) DeepCopy() *Rule { + if in == nil { + return nil + } + out := new(Rule) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RuleGroup) DeepCopyInto(out *RuleGroup) { + *out = *in + if in.Rules != nil { + in, out := &in.Rules, &out.Rules + *out = make([]Rule, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuleGroup. +func (in *RuleGroup) DeepCopy() *RuleGroup { + if in == nil { + return nil + } + out := new(RuleGroup) + in.DeepCopyInto(out) + return out +} diff --git a/builders/prometheus_rule.go b/builders/prometheus_rule.go new file mode 100755 index 00000000..d3cc4283 --- /dev/null +++ b/builders/prometheus_rule.go @@ -0,0 +1,63 @@ +package builders + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + monitoringv1 "opendev.org/vexxhost/openstack-operator/api/monitoring/v1" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +// PrometheusRuleBuilder provides an interface to build PrometheusRules +type PrometheusRuleBuilder struct { + obj *monitoringv1.PrometheusRule + ruleGroups []*RuleGroupBuilder + owner metav1.Object + scheme *runtime.Scheme +} + +// PrometheusRule returns a new PrometheusRule builder +func PrometheusRule(existing *monitoringv1.PrometheusRule, owner metav1.Object, scheme *runtime.Scheme) *PrometheusRuleBuilder { + return &PrometheusRuleBuilder{ + obj: existing, + owner: owner, + scheme: scheme, + } +} + +// RuleGroups returns the ruleGroups +func (pm *PrometheusRuleBuilder) RuleGroups(ruleGroups ...*RuleGroupBuilder) *PrometheusRuleBuilder { + pm.ruleGroups = ruleGroups + return pm +} + +// Build returns the object after making certain assertions +func (pm *PrometheusRuleBuilder) Build() error { + pm.obj.Spec.Groups = []monitoringv1.RuleGroup{} + for _, rgBuilder := range pm.ruleGroups { + ruleGroup, err := rgBuilder.Build() + if err != nil { + return err + } + + pm.obj.Spec.Groups = append(pm.obj.Spec.Groups, ruleGroup) + } + if !pm.isOwnedByOthers() { + return controllerutil.SetControllerReference(pm.owner, pm.obj, pm.scheme) + } + return nil +} + +// isOwnedByOthers checks if this podMonitor has been possessed by an another object already. +func (pm *PrometheusRuleBuilder) isOwnedByOthers() bool { + ownerName := pm.owner.GetName() + + existingRefs := pm.obj.GetOwnerReferences() + for _, r := range existingRefs { + if r.Name == ownerName { + return false + } else if r.Controller != nil && *r.Controller { + return true + } + } + return false +} diff --git a/builders/rule.go b/builders/rule.go new file mode 100755 index 00000000..d8bcf96a --- /dev/null +++ b/builders/rule.go @@ -0,0 +1,53 @@ +package builders + +import ( + "strconv" + + "k8s.io/apimachinery/pkg/util/intstr" + monitoringv1 "opendev.org/vexxhost/openstack-operator/api/monitoring/v1" +) + +// RuleBuilder provides an interface to build rule +type RuleBuilder struct { + obj *monitoringv1.Rule +} + +// Rule returns a new podmonitor builder +func Rule() *RuleBuilder { + Rule := &monitoringv1.Rule{ + Annotations: map[string]string{}, + } + return &RuleBuilder{ + obj: Rule, + } +} + +func (r *RuleBuilder) Alert(alert string) *RuleBuilder { + r.obj.Alert = alert + return r +} + +func (r *RuleBuilder) Expr(expr string) *RuleBuilder { + r.obj.Expr = intstr.FromString(expr) + return r +} + +func (r *RuleBuilder) For(duration string) *RuleBuilder { + r.obj.For = duration + return r +} + +func (r *RuleBuilder) Priority(p int) *RuleBuilder { + r.obj.Annotations["priority"] = "P" + strconv.Itoa(p) + return r +} + +func (r *RuleBuilder) Message(m string) *RuleBuilder { + r.obj.Annotations["message"] = m + return r +} + +// Build returns the object after making certain assertions +func (r *RuleBuilder) Build() (monitoringv1.Rule, error) { + return *r.obj, nil +} diff --git a/builders/rule_group.go b/builders/rule_group.go new file mode 100755 index 00000000..a7fffc66 --- /dev/null +++ b/builders/rule_group.go @@ -0,0 +1,55 @@ +package builders + +import ( + monitoringv1 "opendev.org/vexxhost/openstack-operator/api/monitoring/v1" +) + +// RuleGroupBuilder provides an interface to build RuleGroup +type RuleGroupBuilder struct { + obj *monitoringv1.RuleGroup + rules []*RuleBuilder +} + +// RuleGroup returns a new rulegroup builder +func RuleGroup() *RuleGroupBuilder { + RuleGroup := &monitoringv1.RuleGroup{} + return &RuleGroupBuilder{ + obj: RuleGroup, + } +} + +func (r *RuleGroupBuilder) Name(Name string) *RuleGroupBuilder { + r.obj.Name = Name + return r +} + +func (r *RuleGroupBuilder) Interval(Interval string) *RuleGroupBuilder { + r.obj.Interval = Interval + return r +} + +func (r *RuleGroupBuilder) Rules(Rules ...*RuleBuilder) *RuleGroupBuilder { + r.rules = Rules + return r +} + +func (r *RuleGroupBuilder) PartialResponseStrategy(prs string) *RuleGroupBuilder { + r.obj.PartialResponseStrategy = prs + return r +} + +// Build returns the object after making certain assertions +func (r *RuleGroupBuilder) Build() (monitoringv1.RuleGroup, error) { + + r.obj.Rules = []monitoringv1.Rule{} + for _, rBuilder := range r.rules { + rule, err := rBuilder.Build() + if err != nil { + return monitoringv1.RuleGroup{}, err + } + + r.obj.Rules = append(r.obj.Rules, rule) + } + + return *r.obj, nil +} diff --git a/chart/crds/monitoring.coreos.com_podmonitors.yaml b/chart/crds/monitoring.coreos.com_podmonitors.yaml index 59706ba6..396cc26d 100644 --- a/chart/crds/monitoring.coreos.com_podmonitors.yaml +++ b/chart/crds/monitoring.coreos.com_podmonitors.yaml @@ -17,6 +17,7 @@ spec: scope: Namespaced validation: openAPIV3Schema: + description: PodMonitor defines monitoring for a set of pods. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation diff --git a/chart/crds/monitoring.coreos.com_prometheusrules.yaml b/chart/crds/monitoring.coreos.com_prometheusrules.yaml new file mode 100644 index 00000000..d3cd1b53 --- /dev/null +++ b/chart/crds/monitoring.coreos.com_prometheusrules.yaml @@ -0,0 +1,97 @@ + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + creationTimestamp: null + name: prometheusrules.monitoring.coreos.com +spec: + group: monitoring.coreos.com + names: + kind: PrometheusRule + listKind: PrometheusRuleList + plural: prometheusrules + singular: prometheusrule + scope: Namespaced + validation: + openAPIV3Schema: + description: PrometheusRule defines alerting rules for a Prometheus instance + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Specification of desired alerting rule definitions for Prometheus. + properties: + groups: + description: Content of Prometheus rule file + items: + description: 'RuleGroup is a list of sequentially evaluated recording + and alerting rules. Note: PartialResponseStrategy is only used by + ThanosRuler and will be ignored by Prometheus instances. Valid + values for this field are ''warn'' or ''abort''. More info: https://github.com/thanos-io/thanos/blob/master/docs/components/rule.md#partial-response' + properties: + interval: + type: string + name: + type: string + partial_response_strategy: + type: string + rules: + items: + description: Rule describes an alerting or recording rule. + properties: + alert: + type: string + annotations: + additionalProperties: + type: string + type: object + expr: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + for: + type: string + labels: + additionalProperties: + type: string + type: object + record: + type: string + required: + - expr + type: object + type: array + required: + - name + - rules + type: object + type: array + type: object + required: + - spec + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/chart/templates/clusterrole.yaml b/chart/templates/clusterrole.yaml index 7d5ce46c..127b2753 100644 --- a/chart/templates/clusterrole.yaml +++ b/chart/templates/clusterrole.yaml @@ -103,3 +103,23 @@ rules: - get - patch - update +- apiGroups: + - monitoring.coreos.com + resources: + - prometheusrules + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - monitoring.coreos.com + resources: + - prometheusrules/status + verbs: + - get + - patch + - update diff --git a/config/crd/bases/monitoring.coreos.com_podmonitors.yaml b/config/crd/bases/monitoring.coreos.com_podmonitors.yaml index 59706ba6..396cc26d 100644 --- a/config/crd/bases/monitoring.coreos.com_podmonitors.yaml +++ b/config/crd/bases/monitoring.coreos.com_podmonitors.yaml @@ -17,6 +17,7 @@ spec: scope: Namespaced validation: openAPIV3Schema: + description: PodMonitor defines monitoring for a set of pods. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation diff --git a/config/crd/bases/monitoring.coreos.com_prometheusrules.yaml b/config/crd/bases/monitoring.coreos.com_prometheusrules.yaml new file mode 100644 index 00000000..d3cd1b53 --- /dev/null +++ b/config/crd/bases/monitoring.coreos.com_prometheusrules.yaml @@ -0,0 +1,97 @@ + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + creationTimestamp: null + name: prometheusrules.monitoring.coreos.com +spec: + group: monitoring.coreos.com + names: + kind: PrometheusRule + listKind: PrometheusRuleList + plural: prometheusrules + singular: prometheusrule + scope: Namespaced + validation: + openAPIV3Schema: + description: PrometheusRule defines alerting rules for a Prometheus instance + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Specification of desired alerting rule definitions for Prometheus. + properties: + groups: + description: Content of Prometheus rule file + items: + description: 'RuleGroup is a list of sequentially evaluated recording + and alerting rules. Note: PartialResponseStrategy is only used by + ThanosRuler and will be ignored by Prometheus instances. Valid + values for this field are ''warn'' or ''abort''. More info: https://github.com/thanos-io/thanos/blob/master/docs/components/rule.md#partial-response' + properties: + interval: + type: string + name: + type: string + partial_response_strategy: + type: string + rules: + items: + description: Rule describes an alerting or recording rule. + properties: + alert: + type: string + annotations: + additionalProperties: + type: string + type: object + expr: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + for: + type: string + labels: + additionalProperties: + type: string + type: object + record: + type: string + required: + - expr + type: object + type: array + required: + - name + - rules + type: object + type: array + type: object + required: + - spec + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index bdc18be0..156e9073 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -5,6 +5,7 @@ resources: - bases/infrastructure.vexxhost.cloud_mcrouters.yaml - bases/infrastructure.vexxhost.cloud_memcacheds.yaml - bases/monitoring.coreos.com_podmonitors.yaml +- bases/monitoring.coreos.com_prometheusrules.yaml # +kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index c2d3da4b..e188c5b3 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -103,3 +103,23 @@ rules: - get - patch - update +- apiGroups: + - monitoring.coreos.com + resources: + - prometheusrules + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - monitoring.coreos.com + resources: + - prometheusrules/status + verbs: + - get + - patch + - update diff --git a/controllers/mcrouter_controller.go b/controllers/mcrouter_controller.go index 1f44abee..7eacb0bd 100755 --- a/controllers/mcrouter_controller.go +++ b/controllers/mcrouter_controller.go @@ -28,7 +28,7 @@ type McrouterReconciler struct { // +kubebuilder:rbac:groups=infrastructure.vexxhost.cloud,resources=mcrouters,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=infrastructure.vexxhost.cloud,resources=mcrouters/status,verbs=get;update;patch - +// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=prometheusrules,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=monitoring.coreos.com,resources=podmonitors,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=core,resources=configmaps;services,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete @@ -45,8 +45,9 @@ func (r *McrouterReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { // Labels labels := map[string]string{ - "app.kubernetes.io/name": "mcrouter", - "app.kubernetes.io/instance": req.Name, + "app.kubernetes.io/name": "mcrouter", + "app.kubernetes.io/instance": req.Name, + "app.kubernetes.io/managed-by": "openstack-operator", } // ConfigMap @@ -58,6 +59,7 @@ func (r *McrouterReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { } op, err := utils.CreateOrUpdate(ctx, r, configMap, func() error { b, err := json.Marshal(mcrouter.Spec) + if err != nil { return err } @@ -130,7 +132,8 @@ func (r *McrouterReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { Namespace: req.Namespace, Name: fmt.Sprintf("mcrouter-podmonitor"), Labels: map[string]string{ - "app.kubernetes.io/name": "mcrouter", + "app.kubernetes.io/name": "mcrouter", + "app.kubernetes.io/managed-by": "openstack-operator", }, }, } @@ -151,7 +154,40 @@ func (r *McrouterReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { if err != nil { return ctrl.Result{}, err } - log.WithValues("resource", "podmonitor").WithValues("op", op).Info("Reconciled") + log.WithValues("resource", "mcrouter-podmonitor").WithValues("op", op).Info("Reconciled") + + // Alertrule + alertRule := &monitoringv1.PrometheusRule{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: req.Namespace, + Name: fmt.Sprintf("mcrouter-alertrule"), + }, + } + op, err = utils.CreateOrUpdate(ctx, r, alertRule, func() error { + + return builders.PrometheusRule(alertRule, &mcrouter, r.Scheme). + RuleGroups(builders.RuleGroup(). + Name("mcrouter-rule"). + Rules( + + builders.Rule(). + Alert("McrouterBackendDown"). + Message("Backend Memcached servers are down."). + Priority(1). + Expr("mcrouter_servers{state='down'}!=0"), + builders.Rule(). + Alert("McrouterBackendTimeout"). + Message("Backend Memcached servers are timeout."). + Priority(1). + Expr("mcrouter_server_memcached_timeout_count>0"), + ). + Interval("1m")). + Build() + }) + if err != nil { + return ctrl.Result{}, err + } + log.WithValues("resource", "mcrouter-alertrule").WithValues("op", op).Info("Reconciled") // Service service := &corev1.Service{ diff --git a/controllers/memcached_controller.go b/controllers/memcached_controller.go index 2c345b7a..0f908023 100755 --- a/controllers/memcached_controller.go +++ b/controllers/memcached_controller.go @@ -47,7 +47,7 @@ type MemcachedReconciler struct { // +kubebuilder:rbac:groups=infrastructure.vexxhost.cloud,resources=memcacheds,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=infrastructure.vexxhost.cloud,resources=memcacheds/status,verbs=get;update;patch - +// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=prometheusrules,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=monitoring.coreos.com,resources=podmonitors,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete @@ -67,8 +67,9 @@ func (r *MemcachedReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { // Labels labels := map[string]string{ - "app.kubernetes.io/name": "memcached", - "app.kubernetes.io/instance": req.Name, + "app.kubernetes.io/name": "memcached", + "app.kubernetes.io/instance": req.Name, + "app.kubernetes.io/managed-by": "openstack-operator", } // Deployment @@ -128,7 +129,8 @@ func (r *MemcachedReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { Namespace: req.Namespace, Name: fmt.Sprintf("memcached-podmonitor"), Labels: map[string]string{ - "app.kubernetes.io/name": "memcached", + "app.kubernetes.io/name": "memcached", + "app.kubernetes.io/managed-by": "openstack-operator", }, }, } @@ -175,6 +177,34 @@ func (r *MemcachedReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { return ctrl.Result{Requeue: true}, nil } + // Alertrule + alertRule := &monitoringv1.PrometheusRule{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: req.Namespace, + Name: fmt.Sprintf("memcached-alertrule"), + }, + } + op, err = utils.CreateOrUpdate(ctx, r, alertRule, func() error { + + return builders.PrometheusRule(alertRule, &memcached, r.Scheme). + RuleGroups(builders.RuleGroup(). + Name("memcached-rule"). + Rules( + + builders.Rule(). + Alert("MemcachedConnectionLimit"). + Message("This memcached connection is over max."). + Priority(1). + Expr("memcached_current_connections/memcached_max_connections*100 >90"), + ). + Interval("1m")). + Build() + }) + if err != nil { + return ctrl.Result{}, err + } + log.WithValues("resource", "memcached-alertrule").WithValues("op", op).Info("Reconciled") + // Make sure that they're sorted so we're idempotent sort.Strings(servers)