OSH-Infra: Update monitoring chart configs

This updates chart configuration overrides for the monitoring
services, as well as adds missing secrets, charts and service
accounts for other exporters that have now been enabled

Change-Id: Ia1ed7bba38d7c262e85de8162d53012cdadf487e

global/software/charts/osh-infra/osh-infra-dashboards/grafana.yaml

@@ -68,6 +68,12 @@ metadata:
         path: .osh_infra.grafana.oslo_db
       dest:
         path: .values.endpoints.oslo_db.auth.user
+    - src:
+        schema: pegleg/AccountCatalogue/v1
+        name: osh_infra_service_accounts
+        path: .osh_infra.prometheus.admin
+      dest:
+        path: .values.endpoints.monitoring.auth.admin
     - src:
         schema: pegleg/AccountCatalogue/v1
         name: osh_infra_service_accounts
@@ -120,6 +126,12 @@ metadata:
         schema: deckhand/Passphrase/v1
         name: osh_infra_oslo_db_admin_password
         path: .
+    - dest:
+        path: .values.endpoints.monitoring.auth.admin.password
+      src:
+        schema: deckhand/Passphrase/v1
+        name: osh_infra_prometheus_admin_password
+        path: .
 
     # LDAP Configuration Details
     - src:
@@ -211,6 +223,10 @@ data:
         node_selector_key: openstack-control-plane
         node_selector_value: enabled
     conf:
+      provisioning:
+        datasources:
+          monitoring:
+            url: http://prom-metrics.osh-infra.svc.cluster.local:80/
       ldap:
         config:
           base_dns:

global/software/charts/osh-infra/osh-infra-mariadb/mariadb.yaml

@@ -31,6 +31,13 @@ metadata:
         path: .osh_infra.oslo_db
       dest:
         path: .values.endpoints.olso_db
+    - src:
+        schema: pegleg/EndpointCatalogue/v1
+        name: osh_infra_endpoints
+        path: .osh_infra.prometheus_mysql_exporter
+      dest:
+        path: .values.endpoints.prometheus_mysql_exporter
+
     # Accounts
     - src:
         schema: pegleg/AccountCatalogue/v1
@@ -38,6 +45,12 @@ metadata:
         path: .osh_infra.oslo_db.admin
       dest:
         path: .values.endpoints.oslo_db.auth.admin
+    - src:
+        schema: pegleg/AccountCatalogue/v1
+        name: osh_infra_service_accounts
+        path: .osh_infra.prometheus_mysql_exporter.user
+      dest:
+        path: .values.endpoints.prometheus_mysql_exporter.auth.user
 
     # Secrets
     - dest:
@@ -46,7 +59,12 @@ metadata:
         schema: deckhand/Passphrase/v1
         name: osh_infra_oslo_db_admin_password
         path: .
-
+    - dest:
+        path: .values.endpoints.oslo_db.auth.exporter.password
+      src:
+        schema: deckhand/Passphrase/v1
+        name: osh_infra_oslo_db_exporter_password
+        path: .
 data:
   chart_name: osh-infra-mariadb
   release: osh-infra-mariadb
@@ -72,6 +90,9 @@ data:
       prometheus_mysql_exporter:
         node_selector_key: openstack-control-plane
         node_selector_value: enabled
+    monitoring:
+      prometheus:
+        enabled: true
   dependencies:
     - osh-helm-toolkit
 ...

global/software/charts/osh-infra/osh-infra-monitoring/chart-group.yaml

@@ -13,5 +13,6 @@ data:
     - prometheus
     - prometheus-alertmanager
     - prometheus-node-exporter
+    - prometheus-process-exporter
     - prometheus-kube-state-metrics
     - nagios

global/software/charts/osh-infra/osh-infra-monitoring/nagios.yaml

@@ -37,6 +37,12 @@ metadata:
         path: .osh_infra.monitoring
       dest:
         path: .values.endpoints.monitoring
+    - src:
+        schema: pegleg/EndpointCatalogue/v1
+        name: osh_infra_endpoints
+        path: .osh_infra.elasticsearch
+      dest:
+        path: .values.endpoints.elasticsearch
     - src:
         schema: pegleg/EndpointCatalogue/v1
         name: osh_infra_endpoints
@@ -51,6 +57,18 @@ metadata:
         path: .osh_infra.nagios.admin
       dest:
         path: .values.endpoints.nagios.auth.admin
+    - src:
+        schema: pegleg/AccountCatalogue/v1
+        name: osh_infra_service_accounts
+        path: .osh_infra.prometheus.admin
+      dest:
+        path: .values.endpoints.monitoring.auth.admin
+    - src:
+        schema: pegleg/AccountCatalogue/v1
+        name: osh_infra_service_accounts
+        path: .osh_infra.elasticsearch.admin
+      dest:
+        path: .values.endpoints.elasticsearch.auth.admin
 
     # Secrets
     - dest:
@@ -59,6 +77,18 @@ metadata:
         schema: deckhand/Passphrase/v1
         name: osh_infra_nagios_admin_password
         path: .
+    - dest:
+        path: .values.endpoints.elasticsearch.auth.admin.password
+      src:
+        schema: deckhand/Passphrase/v1
+        name: osh_infra_elasticsearch_admin_password
+        path: .
+    - dest:
+        path: .values.endpoints.monitoring.auth.admin.password
+      src:
+        schema: deckhand/Passphrase/v1
+        name: osh_infra_prometheus_admin_password
+        path: .
 
     # LDAP Details
     - src:

global/software/charts/osh-infra/osh-infra-monitoring/prometheus-process-exporter.yaml

@@ -0,0 +1,65 @@
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: prometheus-process-exporter
+  layeringDefinition:
+    abstract: false
+    layer: global
+  storagePolicy: cleartext
+  substitutions:
+    # Chart source
+    - src:
+        schema: pegleg/SoftwareVersions/v1
+        name: software-versions
+        path: .charts.osh_infra.prometheus_process_exporter
+      dest:
+        path: .source
+
+    # Images
+    - src:
+        schema: pegleg/SoftwareVersions/v1
+        name: software-versions
+        path: .images.osh_infra.prometheus_process_exporter
+      dest:
+        path: .values.images.tags
+
+    # Endpoints
+    - src:
+        schema: pegleg/EndpointCatalogue/v1
+        name: osh_infra_endpoints
+        path: .osh_infra.process_exporter_metrics
+      dest:
+        path: .values.endpoints.process_exporter_metrics
+
+data:
+  chart_name: prometheus-process-exporter
+  release: prometheus-process-exporter
+  namespace: kube-system
+  wait:
+    timeout: 900
+    labels:
+      release_group: airship-prometheus-process-exporter
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+        - type: job
+          labels:
+            release_group: airship-prometheus-process-exporter
+      create: []
+    post:
+      create: []
+  values:
+    labels:
+      node_exporter:
+        node_selector_key: node-exporter
+        node_selector_value: enabled
+      job:
+        node_selector_key: openstack-control-plane
+        node_selector_value: enabled
+  dependencies:
+    - osh-infra-helm-toolkit
+...

global/software/charts/osh/openstack-mariadb/mariadb.yaml

@@ -31,6 +31,13 @@ metadata:
         path: .osh.oslo_db
       dest:
         path: .values.endpoints.olso_db
+    - src:
+        schema: pegleg/EndpointCatalogue/v1
+        name: osh_endpoints
+        path: .osh.prometheus_mysql_exporter
+      dest:
+        path: .values.endpoints.prometheus_mysql_exporter
+
     # Accounts
     - src:
         schema: pegleg/AccountCatalogue/v1
@@ -38,6 +45,12 @@ metadata:
         path: .osh.oslo_db.admin
       dest:
         path: .values.endpoints.oslo_db.auth.admin
+    - src:
+        schema: pegleg/AccountCatalogue/v1
+        name: osh_service_accounts
+        path: .osh.prometheus_mysql_exporter.user
+      dest:
+        path: .values.endpoints.prometheus_mysql_exporter.auth.user
 
     # Secrets
     - dest:
@@ -46,6 +59,12 @@ metadata:
         schema: deckhand/Passphrase/v1
         name: osh_oslo_db_admin_password
         path: .
+    - dest:
+        path: .values.endpoints.oslo_db.auth.exporter.password
+      src:
+        schema: deckhand/Passphrase/v1
+        name: osh_oslo_db_exporter_password
+        path: .
 
 data:
   chart_name: openstack-mariadb
@@ -72,6 +91,9 @@ data:
       prometheus_mysql_exporter:
         node_selector_key: openstack-control-plane
         node_selector_value: enabled
+    monitoring:
+      prometheus:
+        enabled: true
   dependencies:
     - osh-helm-toolkit
 ...

global/software/charts/ucp/ceph/ceph-mon.yaml

@@ -130,8 +130,6 @@ data:
         namespace: ceph
       ceph_mon:
         namespace: ceph
-      fluentd:
-        namespace: osh-infra
     deployment:
       ceph: true
       storage_secrets: true

global/software/charts/ucp/ceph/ceph-osd.yaml

@@ -122,8 +122,6 @@ data:
         namespace: ceph
       ceph_mon:
         namespace: ceph
-      fluentd:
-        namespace: osh-infra
     bootstrap:
       enabled: true
     conf:

global/software/charts/ucp/ucp-openstack-exporter/chart-group.yaml

@@ -0,0 +1,13 @@
+---
+schema: armada/ChartGroup/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp-prometheus-openstack-exporter
+  layeringDefinition:
+    abstract: false
+    layer: global
+  storagePolicy: cleartext
+data:
+  description: Prometheus OpenStack Exporter for UCP Components
+  chart_group:
+    - ucp-prometheus-openstack-exporter

global/software/charts/ucp/ucp-openstack-exporter/prometheus-openstack-exporter.yaml

@@ -0,0 +1,95 @@
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp-prometheus-openstack-exporter
+  layeringDefinition:
+    abstract: false
+    layer: global
+  storagePolicy: cleartext
+  substitutions:
+    # Chart source
+    - src:
+        schema: pegleg/SoftwareVersions/v1
+        name: software-versions
+        path: .charts.osh_infra.prometheus_openstack_exporter
+      dest:
+        path: .source
+
+    # Images
+    - src:
+        schema: pegleg/SoftwareVersions/v1
+        name: software-versions
+        path: .images.osh_infra.prometheus_openstack_exporter
+      dest:
+        path: .values.images.tags
+
+    # Endpoints
+    - src:
+        schema: pegleg/EndpointCatalogue/v1
+        name: ucp_endpoints
+        path: .ucp.prometheus_openstack_exporter
+      dest:
+        path: .values.endpoints.prometheus_openstack_exporter
+    - src:
+        schema: pegleg/EndpointCatalogue/v1
+        name: ucp_endpoints
+        path: .ucp.identity
+      dest:
+        path: .values.endpoints.identity
+
+    # Accounts
+    - src:
+        schema: pegleg/AccountCatalogue/v1
+        name: ucp_service_accounts
+        path: .ucp.keystone.admin
+      dest:
+        path: .values.endpoints.identity.auth.admin
+    - src:
+        schema: pegleg/AccountCatalogue/v1
+        name: ucp_service_accounts
+        path: .ucp.prometheus_openstack_exporter.user
+      dest:
+        path: .values.endpoints.identity.auth.user
+
+    # Secrets
+    - dest:
+        path: .values.endpoints.identity.auth.admin.password
+      src:
+        schema: deckhand/Passphrase/v1
+        name: ucp_keystone_admin_password
+        path: .
+    - dest:
+        path: .values.endpoints.identity.auth.user.password
+      src:
+        schema: deckhand/Passphrase/v1
+        name: ucp_openstack_exporter_keystone_password
+        path: .
+data:
+  chart_name: ucp-prometheus-openstack-exporter
+  release: ucp-prometheus-openstack-exporter
+  namespace: ucp
+  wait:
+    timeout: 900
+    labels:
+      release_group: airship-ucp-prometheus-openstack-exporter
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+        - type: job
+          labels:
+            release_group: airship-ucp-prometheus-openstack-exporter
+  values:
+    labels:
+      openstack_exporter:
+        node_selector_key: openstack-control-plane
+        node_selector_value: enabled
+      job:
+        node_selector_key: openstack-control-plane
+        node_selector_value: enabled
+  dependencies:
+    - osh-infra-helm-toolkit
+...

global/software/config/versions.yaml

@@ -230,6 +230,11 @@ data:
         reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
         subpath: prometheus-node-exporter
         type: git
+      prometheus_process_exporter:
+        location: https://git.openstack.org/openstack/openstack-helm-infra
+        reference: 922d7d3d26b15d1d25e40ba1528337002ae15b0c
+        subpath: prometheus-process-exporter
+        type: git
       prometheus_openstack_exporter:
         location: https://git.openstack.org/openstack/openstack-helm-infra
         reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
@@ -528,6 +533,7 @@ data:
       prometheus_kube_state_metrics: {}
       prometheus_node_exporter: {}
       prometheus_openstack_exporter: {}
+      prometheus_process_exporter: {}
     ucp:
       armada:
         api: quay.io/airshipit/armada:32baf9851e9cd29f4b8670e73a9409a0e7a59e35

global/software/manifests/full-site.yaml

@@ -29,6 +29,7 @@ data:
     - ucp-drydock
     - ucp-promenade
     - ucp-shipyard
+    - ucp-prometheus-openstack-exporter
     - osh-infra-ingress-controller
     - osh-infra-ceph-config
     - osh-infra-radosgw

site/airship-seaworthy/secrets/passphrases/osh_infra_oslo_db_exporter_password.yaml

@@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: osh_infra_oslo_db_exporter_password
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...

site/airship-seaworthy/secrets/passphrases/osh_infra_prometheus_admin_password.yaml

@@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: osh_infra_prometheus_admin_password
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...

site/airship-seaworthy/secrets/passphrases/osh_infra_rgw_s3_admin_access_key.yaml

@@ -7,5 +7,5 @@ metadata:
     abstract: false
     layer: site
   storagePolicy: cleartext
-data: password123
+data: admin_access_key
 ...

site/airship-seaworthy/secrets/passphrases/osh_infra_rgw_s3_admin_secret_key.yaml

@@ -7,5 +7,5 @@ metadata:
     abstract: false
     layer: site
   storagePolicy: cleartext
-data: password123
+data: admin_secret_key
 ...

site/airship-seaworthy/secrets/passphrases/osh_infra_rgw_s3_elasticsearch_access_key.yaml

@@ -7,5 +7,5 @@ metadata:
     abstract: false
     layer: site
   storagePolicy: cleartext
-data: password123
+data: elastic_access_key
 ...

site/airship-seaworthy/secrets/passphrases/osh_infra_rgw_s3_elasticsearch_secret_key.yaml

@@ -7,5 +7,5 @@ metadata:
     abstract: false
     layer: site
   storagePolicy: cleartext
-data: password123
+data: elastic_secret_key
 ...

site/airship-seaworthy/secrets/passphrases/osh_oslo_db_exporter_password.yaml

@@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: osh_oslo_db_exporter_password
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...

site/airship-seaworthy/secrets/passphrases/ucp_openstack_exporter_keystone_password.yaml

@@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp_openstack_exporter_keystone_password
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...

site/airship-seaworthy/software/config/endpoints.yaml

@@ -260,6 +260,19 @@ data:
         default: "http"
       host_fqdn_override:
         default: null
+    prometheus_openstack_exporter:
+      namespace: ucp
+      hosts:
+        default: openstack-metrics
+      host_fqdn_override:
+        default: null
+      path:
+        default: null
+      scheme:
+        default: "http"
+      port:
+        exporter:
+          default: 9103
   ceph:
     object_store:
       name: swift
@@ -499,6 +512,19 @@ data:
           default: 3306
         wsrep:
           default: 4567
+    prometheus_mysql_exporter:
+      namespace: openstack
+      hosts:
+        default: mysql-exporter
+      host_fqdn_override:
+        default: null
+      path:
+        default: /metrics
+      scheme:
+        default: 'http'
+      port:
+        metrics:
+          default: 9104
     keystone_oslo_messaging:
       namespace: openstack
       hosts:
@@ -1098,6 +1124,19 @@ data:
       port:
         mysql:
           default: 3306
+    prometheus_mysql_exporter:
+      namespace: osh-infra
+      hosts:
+        default: mysql-exporter
+      host_fqdn_override:
+        default: null
+      path:
+        default: /metrics
+      scheme:
+        default: 'http'
+      port:
+        metrics:
+          default: 9104
     grafana:
       name: grafana
       namespace: osh-infra
@@ -1132,7 +1171,8 @@ data:
       port:
         api:
           default: 9090
-          public: 80
+        http:
+          default: 80
     kibana:
       name: kibana
       namespace: osh-infra
@@ -1209,6 +1249,19 @@ data:
           default: 9100
         prometheus_port:
           default: 9100
+    process_exporter_metrics:
+      namespace: kube-system
+      hosts:
+        default: process-exporter
+      host_fqdn_override:
+        default: null
+      path:
+        default: null
+      scheme:
+        default: "http"
+      port:
+        metrics:
+          default: 9256
     prometheus_openstack_exporter:
       namespace: openstack
       hosts:

site/airship-seaworthy/software/config/service_accounts.yaml

@@ -122,6 +122,14 @@ data:
             postgres:
                 username: deckhand
                 database: deckhand
+        prometheus_openstack_exporter:
+            user:
+                region_name: RegionOne
+                role: admin
+                username: prometheus-openstack-exporter
+                project_name: service
+                user_domain_name: default
+                project_domain_name: default
     ceph:
         swift:
             keystone:
@@ -303,6 +311,9 @@ data:
     oslo_db:
       admin:
         username: root
+    prometheus_mysql_exporter:
+      user:
+        username: osh-oslodb-exporter
     neutron:
       neutron:
         role: admin
@@ -406,6 +417,9 @@ data:
     oslo_db:
       admin:
         username: root
+    prometheus_mysql_exporter:
+      user:
+        username: osh-infra-oslodb-exporter
     prometheus_openstack_exporter:
       user:
         role: admin
@@ -416,6 +430,9 @@ data:
     nagios:
       admin:
         username: nagios
+    prometheus:
+      admin:
+        username: prometheus
     ldap:
       admin:
         # NEWSITE-CHANGEME: Replace with the site's LDAP account used to