OSH-Infra: Update chart configs

This updates chart configuration overrides for elasticsearch, fluent-logging, and prometheus. This also adds the service user for Prometheus basic auth. These configuration overrides help increase the performance of the services listed above Change-Id: Ia1ed7bba38d7c262e85de8162d53012cdadf487e
2018-09-17 08:13:25 -05:00 · 2018-09-17 08:13:25 -05:00 · e9034afeae
parent 9ac56a7157
commit e9034afeae
25 changed files with 2554 additions and 57 deletions
--- a/global/profiles/host/cp.yaml
+++ b/global/profiles/host/cp.yaml
@ -105,4 +105,5 @@ data:
      kube-ingress: enabled
      beta.kubernetes.io/fluentd-ds-ready: 'true'
      node-exporter: enabled
+      fluentbit: enabled
 ...
--- a/global/profiles/host/dp.yaml
+++ b/global/profiles/host/dp.yaml
@ -57,4 +57,5 @@ data:
      openstack-libvirt: kernel
      beta.kubernetes.io/fluentd-ds-ready: 'true'
      node-exporter: enabled
+      fluentbit: enabled
 ...
--- a/global/software/charts/osh-infra/osh-infra-dashboards/grafana.yaml
+++ b/global/software/charts/osh-infra/osh-infra-dashboards/grafana.yaml
@ -68,6 +68,12 @@ metadata:
        path: .osh_infra.grafana.oslo_db
      dest:
        path: .values.endpoints.oslo_db.auth.user
+    - src:
+        schema: pegleg/AccountCatalogue/v1
+        name: osh_infra_service_accounts
+        path: .osh_infra.prometheus.admin
+      dest:
+        path: .values.endpoints.prometheus.auth.admin
    - src:
        schema: pegleg/AccountCatalogue/v1
        name: osh_infra_service_accounts
@ -120,6 +126,12 @@ metadata:
        schema: deckhand/Passphrase/v1
        name: osh_infra_oslo_db_admin_password
        path: .
+    - dest:
+        path: .values.endpoints.prometheus.auth.admin.password
+      src:
+        schema: deckhand/Passphrase/v1
+        name: osh_infra_prometheus_admin_password
+        path: .

    # LDAP Configuration Details
    - src:
--- a/global/software/charts/osh-infra/osh-infra-logging/elasticsearch.yaml
+++ b/global/software/charts/osh-infra/osh-infra-logging/elasticsearch.yaml
@ -54,7 +54,18 @@ metadata:
        path: .osh_infra.elasticsearch.admin
      dest:
        path: .values.endpoints.elasticsearch.auth.admin
-
+    - src:
+        schema: pegleg/AccountCatalogue/v1
+        name: osh_infra_service_accounts
+        path: .osh_infra.ceph_object_store.admin
+      dest:
+        path: .values.endpoints.ceph_object_store.auth.admin
+    - src:
+        schema: pegleg/AccountCatalogue/v1
+        name: osh_infra_service_accounts
+        path: .osh_infra.ceph_object_store.elasticsearch
+      dest:
+        path: .values.endpoints.ceph_object_store.auth.elasticsearch
    # Secrets
    - dest:
        path: .values.endpoints.elasticsearch.auth.admin.password
@ -62,6 +73,30 @@ metadata:
        schema: deckhand/Passphrase/v1
        name: osh_infra_elasticsearch_admin_password
        path: .
+    - dest:
+        path: .values.endpoints.ceph_object_store.auth.admin.access_key
+      src:
+        schema: deckhand/Passphrase/v1
+        name: osh_infra_rgw_s3_admin_access_key
+        path: .
+    - dest:
+        path: .values.endpoints.ceph_object_store.auth.admin.secret_key
+      src:
+        schema: deckhand/Passphrase/v1
+        name: osh_infra_rgw_s3_admin_secret_key
+        path: .
+    - dest:
+        path: .values.endpoints.ceph_object_store.auth.elasticsearch.access_key
+      src:
+        schema: deckhand/Passphrase/v1
+        name: osh_infra_rgw_s3_elasticsearch_access_key
+        path: .
+    - dest:
+        path: .values.endpoints.ceph_object_store.auth.elasticsearch.secret_key
+      src:
+        schema: deckhand/Passphrase/v1
+        name: osh_infra_rgw_s3_elasticsearch_secret_key
+        path: .

    # LDAP Details
    - src:
@ -97,6 +132,75 @@ data:
    post:
      create: []
  values:
+    pod:
+      replicas:
+        client: 5
+      resources:
+        enabled: true
+        apache_proxy:
+          limits:
+            memory: "1024Mi"
+            cpu: "2000m"
+          requests:
+            memory: "0"
+            cpu: "0"
+        client:
+          requests:
+            memory: "8Gi"
+            cpu: "1000m"
+          limits:
+            memory: "16Gi"
+            cpu: "2000m"
+        master:
+          requests:
+            memory: "8Gi"
+            cpu: "1000m"
+          limits:
+            memory: "16Gi"
+            cpu: "2000m"
+        data:
+          requests:
+            memory: "8Gi"
+            cpu: "1000m"
+          limits:
+            memory: "16Gi"
+            cpu: "2000m"
+        prometheus_elasticsearch_exporter:
+          requests:
+            memory: "0"
+            cpu: "0"
+          limits:
+            memory: "1024Mi"
+            cpu: "2000m"
+        jobs:
+          curator:
+            requests:
+              memory: "0"
+              cpu: "0"
+            limits:
+              memory: "1024Mi"
+              cpu: "2000m"
+          image_repo_sync:
+            requests:
+              memory: "0"
+              cpu: "0"
+            limits:
+              memory: "1024Mi"
+              cpu: "2000m"
+          snapshot_repository:
+            requests:
+              memory: "0"
+              cpu: "0"
+            limits:
+              memory: "1024Mi"
+              cpu: "2000m"
+          tests:
+            requests:
+              memory: "0"
+              cpu: "0"
+            limits:
+              memory: "1024Mi"
+              cpu: "2000m"
    labels:
      elasticsearch:
        node_selector_key: openstack-control-plane
@ -108,27 +212,95 @@ data:
      prometheus:
        enabled: true
    conf:
-      apache:
-        host: |
-          <VirtualHost *:80>
-            <Location />
-                ProxyPass http://localhost:{{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/
-                ProxyPassReverse http://localhost:{{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/
-            </Location>
-            <Proxy *>
-                AuthName "Elasticsearch"
-                AuthType Basic
-                AuthBasicProvider file ldap
-                AuthUserFile /usr/local/apache2/conf/.htpasswd
-                AuthLDAPBindDN {{ .Values.endpoints.ldap.auth.admin.bind }}
-                AuthLDAPBindPassword {{ .Values.endpoints.ldap.auth.admin.password }}
-                AuthLDAPURL {{ tuple "ldap" "public" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
-                Require valid-user
-            </Proxy>
-          </VirtualHost>
+      httpd: |
+        ServerRoot "/usr/local/apache2"
+        Listen 80
+        LoadModule mpm_event_module modules/mod_mpm_event.so
+        LoadModule authn_file_module modules/mod_authn_file.so
+        LoadModule authn_core_module modules/mod_authn_core.so
+        LoadModule authz_host_module modules/mod_authz_host.so
+        LoadModule authz_groupfile_module modules/mod_authz_groupfile.so
+        LoadModule authz_user_module modules/mod_authz_user.so
+        LoadModule authz_core_module modules/mod_authz_core.so
+        LoadModule access_compat_module modules/mod_access_compat.so
+        LoadModule auth_basic_module modules/mod_auth_basic.so
+        LoadModule ldap_module modules/mod_ldap.so
+        LoadModule authnz_ldap_module modules/mod_authnz_ldap.so
+        LoadModule reqtimeout_module modules/mod_reqtimeout.so
+        LoadModule filter_module modules/mod_filter.so
+        LoadModule proxy_html_module modules/mod_proxy_html.so
+        LoadModule log_config_module modules/mod_log_config.so
+        LoadModule env_module modules/mod_env.so
+        LoadModule headers_module modules/mod_headers.so
+        LoadModule setenvif_module modules/mod_setenvif.so
+        LoadModule version_module modules/mod_version.so
+        LoadModule proxy_module modules/mod_proxy.so
+        LoadModule proxy_connect_module modules/mod_proxy_connect.so
+        LoadModule proxy_http_module modules/mod_proxy_http.so
+        LoadModule proxy_balancer_module modules/mod_proxy_balancer.so
+        LoadModule slotmem_shm_module modules/mod_slotmem_shm.so
+        LoadModule slotmem_plain_module modules/mod_slotmem_plain.so
+        LoadModule unixd_module modules/mod_unixd.so
+        LoadModule status_module modules/mod_status.so
+        LoadModule autoindex_module modules/mod_autoindex.so
+        <IfModule unixd_module>
+        User daemon
+        Group daemon
+        </IfModule>
+        <Directory />
+            AllowOverride none
+            Require all denied
+        </Directory>
+        <Files ".ht*">
+            Require all denied
+        </Files>
+        ErrorLog /dev/stderr
+        LogLevel warn
+        <IfModule log_config_module>
+            LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
+            LogFormat "%h %l %u %t \"%r\" %>s %b" common
+            <IfModule logio_module>
+              LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
+            </IfModule>
+            CustomLog /dev/stdout common
+            CustomLog /dev/stdout combined
+        </IfModule>
+        <Directory "/usr/local/apache2/cgi-bin">
+            AllowOverride None
+            Options None
+            Require all granted
+        </Directory>
+        <IfModule headers_module>
+            RequestHeader unset Proxy early
+        </IfModule>
+        <IfModule proxy_html_module>
+        Include conf/extra/proxy-html.conf
+        </IfModule>
+        <VirtualHost *:80>
+          <Location />
+              ProxyPass http://localhost:{{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/
+              ProxyPassReverse http://localhost:{{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/
+          </Location>
+          <Proxy *>
+              AuthName "Elasticsearch"
+              AuthType Basic
+              AuthBasicProvider file ldap
+              AuthUserFile /usr/local/apache2/conf/.htpasswd
+              AuthLDAPBindDN {{ .Values.endpoints.ldap.auth.admin.bind }}
+              AuthLDAPBindPassword {{ .Values.endpoints.ldap.auth.admin.password }}
+              AuthLDAPURL {{ tuple "ldap" "public" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }}
+              Require valid-user
+          </Proxy>
+        </VirtualHost>
      elasticsearch:
+        config:
+          http:
+            max_content_length: 2gb
+            pipelining: false
        env:
-          java_opts: "-Xms5g -Xmx5g"
+          java_opts: "-Xms8g -Xmx8g"
+        snapshots:
+          enabled: true
      curator:
        #run every 6th hour
        schedule:  "0 */6 * * *"
--- a/global/software/charts/osh-infra/osh-infra-logging/fluent-logging.yaml
+++ b/global/software/charts/osh-infra/osh-infra-logging/fluent-logging.yaml
@ -82,12 +82,51 @@ data:
    post:
      create: []
  values:
+    pod:
+      resources:
+        enabled: true
+        fluentbit:
+          limits:
+            memory: '4Gi'
+            cpu: '2000m'
+          requests:
+            memory: '2Gi'
+            cpu: '1000m'
+        fluentd:
+          limits:
+            memory: '4Gi'
+            cpu: '2000m'
+          requests:
+            memory: '2Gi'
+            cpu: '1000m'
+        prometheus_fluentd_exporter:
+          limits:
+            memory: '1024Mi'
+            cpu: '2000m'
+          requests:
+            memory: '0'
+            cpu: '0'
+        jobs:
+          image_repo_sync:
+            requests:
+              memory: '0'
+              cpu: '0'
+            limits:
+              memory: '1024Mi'
+              cpu: '2000m'
+          tests:
+            requests:
+              memory: '0'
+              cpu: '0'
+            limits:
+              memory: '1024Mi'
+              cpu: '2000m'
    labels:
      fluentd:
-        node_selector_key: openstack-control-plane
+        node_selector_key: fluentd
        node_selector_value: enabled
      fluentbit:
-        node_selector_key: openstack-control-plane
+        node_selector_key: fluentbit
        node_selector_value: enabled
      prometheus_fluentd_exporter:
        node_selector_key: openstack-control-plane
@ -95,20 +134,6 @@ data:
      job:
        node_selector_key: openstack-control-plane
        node_selector_value: enabled
-    dependencies:
-      static:
-        fluentbit:
-          jobs: ""
-          services:
-            - endpoint: internal
-              service: fluentd
-        fluentd:
-          jobs: ""
-          services:
-            - endpoint: internal
-              service: elasticsearch
-    manifests:
-      job_elasticsearch_template: false
    conf:
      fluentbit:
        - service:
@ -117,6 +142,28 @@ data:
            Daemon: Off
            Log_Level: info
            Parsers_File: parsers.conf
+        - kubelet:
+            header: input
+            Name: systemd
+            Path: /var/log/journal
+            Tag: syslog.*
+            Systemd_Filter: _SYSTEMD_UNIT=kubelet.service
+            DB: /var/log/kubelet.db
+            Mem_Buf_Limit: 5MB
+            DB.Sync: Normal
+            Buffer_Chunk_Size: 1M
+            Buffer_Max_Size: 1M
+        - docker_daemon:
+            header: input
+            Name: systemd
+            Path: /var/log/journal
+            Tag: syslog.*
+            Systemd_Filter: _SYSTEMD_UNIT=docker.service
+            DB: /var/log/docker.db
+            Mem_Buf_Limit: 5MB
+            DB.Sync: Normal
+            Buffer_Chunk_Size: 1M
+            Buffer_Max_Size: 1M
        - containers_tail:
            header: input
            Name: tail
@ -124,10 +171,14 @@ data:
            Path: /var/log/containers/*.log
            Parser: docker
            DB: /var/log/flb_kube.db
+            Mem_Buf_Limit: 5MB
            DB.Sync: Normal
            Buffer_Chunk_Size: 1M
            Buffer_Max_Size: 1M
-            Mem_Buf_Limit: 5MB
+        - drop_fluentd_logs:
+            header: output
+            Name: "null"
+            Match: "**.fluentd**"
        - kube_filter:
            header: filter
            Name: kubernetes
@ -137,8 +188,16 @@ data:
            header: output
            Name: forward
            Match: "*"
-            Host: ${FLUENTD_HOST}
-            Port: ${FLUENTD_PORT}
+            Host: fluentd-logging.osh-infra
+            Port: 24224
+      parsers:
+        - docker:
+            header: parser
+            Name: docker
+            Format: json
+            Time_Key: time
+            Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
+            Time_Keep: On
      td_agent:
        - metrics_agent:
            header: source
@ -150,22 +209,268 @@ data:
            type: forward
            port: "#{ENV['FLUENTD_PORT']}"
            bind: 0.0.0.0
-        - elasticsearch:
+        - drop_fluent_logs:
+            header: match
+            type: "null"
+            expression: "fluent.*"
+        - add_container_name:
+            header: filter
+            type: record_transformer
+            expression: "kube.**"
+            enable_ruby: true
+            record:
+              -
+                - header: record
+                  container_name: ${record["kubernetes"]["container_name"]}
+        - remove_openstack_pod_logged_events:
+            header: filter
+            type: grep
+            expression: "kube.**"
+            exclude:
+              -
+                - header: exclude
+                  key: container_name
+                  pattern: ^(cinder-api|cinder-scheduler|cinder-volume|cinder-backup|glance-api|glance-registry|heat-api|heat-cfn|heat-engine|keystone-api|neutron-dhcp-agent|neutron-l3-agent|neutron-server|nova-osapi|nova-api|nova-compute|nova-conductor|nova-consoleauth|nova-novncproxy|nova-scheduler)$
+          # NOTE(srwilkers): Look for specific keywords in the log key to determine
+          # log level of event
+        - tag_kubernetes_log_level:
+            header: match
+            type: rewrite_tag_filter
+            expression: "kube.var.log.containers.**.log"
+            rule:
+              -
+                - header: rule
+                  key: log
+                  pattern: /info/i
+                  tag: info.${tag}
+                - header: rule
+                  key: log
+                  pattern: /warn/i
+                  tag: warn.${tag}
+                - header: rule
+                  key: log
+                  pattern: /error/i
+                  tag: error.${tag}
+                - header: rule
+                  key: log
+                  pattern: /critical/i
+                  tag: critical.${tag}
+                - header: rule
+                  key: log
+                  pattern: (.+)
+                  tag: info.${tag}
+          # NOTE(srwilkers): Create new key for log level, and use the tag prefix
+          # added previously
+        - add_kubernetes_log_level_and_application_key:
+            header: filter
+            type: record_transformer
+            enable_ruby: true
+            expression: "**.kube.var.log.containers.**.log"
+            record:
+              -
+                - header: record
+                  level: ${tag_parts[0]}
+                  application: ${record["kubernetes"]["labels"]["application"]}
+        - add_openstack_application_key:
+            header: filter
+            type: record_transformer
+            expression: "openstack.**"
+            record:
+              -
+                - header: record
+                  application: ${tag_parts[1]}
+          #NOTE(srwilkers): This prefixes the tag for oslo.log entries from the
+          # fluent handler/formatter with the log level, allowing for lookups on
+          # openstack logs with a particular log level (ie: error.openstack.keystone)
+        - tag_openstack_log_level:
+            header: match
+            type: rewrite_tag_filter
+            expression: "openstack.**"
+            rule:
+              -
+                - header: rule
+                  key: level
+                  pattern: INFO
+                  tag: info.${tag}
+                - header: rule
+                  key: level
+                  pattern: WARN
+                  tag: warn.${tag}
+                - header: rule
+                  key: level
+                  pattern: ERROR
+                  tag: error.${tag}
+                - header: rule
+                  key: level
+                  pattern: CRITICAL
+                  tag: critical.${tag}
+        - syslog_elasticsearch:
            header: match
            type: elasticsearch
            user: "#{ENV['ELASTICSEARCH_USERNAME']}"
            password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
-            expression: "**"
+            expression: "syslog.**"
            include_tag_key: true
            host: "#{ENV['ELASTICSEARCH_HOST']}"
            port: "#{ENV['ELASTICSEARCH_PORT']}"
            logstash_format: true
+            logstash_prefix: syslog
+            buffer_type: memory
            buffer_chunk_limit: 10M
-            buffer_queue_limit: 32
-            flush_interval: 20s
+            buffer_queue_limit: 512
+            flush_interval: 10
            max_retry_wait: 300
+            request_timeout: 60
            disable_retry_limit: ""
            num_threads: 8
+            type_name: syslog
+        - ceph_elasticsearch:
+            header: match
+            type: elasticsearch
+            user: "#{ENV['ELASTICSEARCH_USERNAME']}"
+            password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
+            expression: "ceph-**.log"
+            include_tag_key: true
+            host: "#{ENV['ELASTICSEARCH_HOST']}"
+            port: "#{ENV['ELASTICSEARCH_PORT']}"
+            logstash_format: true
+            logstash_prefix: ceph
+            buffer_chunk_limit: 10M
+            buffer_queue_limit: 512
+            flush_interval: 10
+            max_retry_wait: 300
+            request_timeout: 60
+            disable_retry_limit: ""
+            num_threads: 8
+            type_name: ceph_logs
+        - oslo_fluentd_elasticsearch:
+            header: match
+            type: elasticsearch
+            user: "#{ENV['ELASTICSEARCH_USERNAME']}"
+            password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
+            expression: "**.openstack.*"
+            include_tag_key: true
+            host: "#{ENV['ELASTICSEARCH_HOST']}"
+            port: "#{ENV['ELASTICSEARCH_PORT']}"
+            logstash_format: true
+            logstash_prefix: openstack
+            buffer_type: memory
+            buffer_chunk_limit: 10M
+            buffer_queue_limit: 512
+            flush_interval: 10
+            max_retry_wait: 300
+            request_timeout: 60
+            disable_retry_limit: ""
+            num_threads: 8
+            type_name: oslo_openstack_fluentd
+        - docker_fluentd_elasticsearch:
+            header: match
+            type: elasticsearch
+            user: "#{ENV['ELASTICSEARCH_USERNAME']}"
+            password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
+            expression: "**.kube.**.log"
+            include_tag_key: true
+            host: "#{ENV['ELASTICSEARCH_HOST']}"
+            port: "#{ENV['ELASTICSEARCH_PORT']}"
+            logstash_format: true
+            buffer_type: memory
+            buffer_chunk_limit: 10M
+            buffer_queue_limit: 512
+            flush_interval: 10
+            max_retry_wait: 300
+            request_timeout: 60
+            disable_retry_limit: ""
+            num_threads: 8
+            type_name: docker_fluentd
+      fluentd_exporter:
+        log:
+          format: "logger:stdout?json=true"
+          level: "info"
+      templates:
+        syslog:
+          template: "syslog-*"
+          index_patterns: "syslog-*"
+          settings:
+            number_of_shards: 1
+          mappings:
+            syslog:
+              properties:
+                cluster:
+                  type: keyword
+                app:
+                  type: keyword
+                pid:
+                  type: integer
+                host:
+                  type: keyword
+                log:
+                  type: text
+        ceph_logs:
+          template: "ceph-*"
+          index_patterns: "ceph-*"
+          settings:
+            number_of_shards: 1
+          mappings:
+            ceph_logs:
+              properties:
+                log:
+                  type: text
+        oslo_openstack_fluentd:
+          template: "openstack-*"
+          index_patterns: "openstack-*"
+          settings:
+            number_of_shards: 1
+          mappings:
+            oslo_openstack_fluentd:
+              properties:
+                extra:
+                  properties:
+                    project:
+                      type: text
+                      norms: false
+                    version:
+                      type: text
+                      norms: false
+                filename:
+                  type: text
+                  norms: false
+                funcname:
+                  type: text
+                  norms: false
+                message:
+                  type: text
+                  norms: false
+                process_name:
+                  type: keyword
+                  index: false
+        docker_fluentd:
+          template: "logstash-*"
+          index_patterns: "logstash-*"
+          settings:
+            number_of_shards: 1
+          mappings:
+            docker_fluentd:
+              properties:
+                kubernetes:
+                  properties:
+                    container_name:
+                      type: keyword
+                      index: false
+                    docker_id:
+                      type: keyword
+                      index: false
+                    host:
+                      type: keyword
+                      index: false
+                    namespace_name:
+                      type: keyword
+                      index: false
+                    pod_id:
+                      type: keyword
+                      index: false
+                    pod_name:
+                      type: keyword
+                      index: false
  dependencies:
    - osh-infra-helm-toolkit
 ...
--- a/global/software/charts/osh-infra/osh-infra-mariadb/mariadb.yaml
+++ b/global/software/charts/osh-infra/osh-infra-mariadb/mariadb.yaml
@ -31,6 +31,13 @@ metadata:
        path: .osh_infra.oslo_db
      dest:
        path: .values.endpoints.olso_db
+    - src:
+        schema: pegleg/EndpointCatalogue/v1
+        name: osh_infra_endpoints
+        path: .osh_infra.prometheus_mysql_exporter
+      dest:
+        path: .values.endpoints.prometheus_mysql_exporter
+
    # Accounts
    - src:
        schema: pegleg/AccountCatalogue/v1
@ -38,6 +45,12 @@ metadata:
        path: .osh_infra.oslo_db.admin
      dest:
        path: .values.endpoints.oslo_db.auth.admin
+    - src:
+        schema: pegleg/AccountCatalogue/v1
+        name: osh_infra_service_accounts
+        path: .osh_infra.prometheus_mysql_exporter.user
+      dest:
+        path: .values.endpoints.prometheus_mysql_exporter.auth.user

    # Secrets
    - dest:
@ -46,7 +59,12 @@ metadata:
        schema: deckhand/Passphrase/v1
        name: osh_infra_oslo_db_admin_password
        path: .
-
+    - dest:
+        path: .values.endpoints.oslo_db.auth.exporter.password
+      src:
+        schema: deckhand/Passphrase/v1
+        name: osh_infra_oslo_db_exporter_password
+        path: .
 data:
  chart_name: osh-infra-mariadb
  release: osh-infra-mariadb
@ -72,6 +90,9 @@ data:
      prometheus_mysql_exporter:
        node_selector_key: openstack-control-plane
        node_selector_value: enabled
+    monitoring:
+      prometheus:
+        enabled: true
  dependencies:
    - osh-helm-toolkit
 ...
--- a/global/software/charts/osh-infra/osh-infra-monitoring/chart-group.yaml
+++ b/global/software/charts/osh-infra/osh-infra-monitoring/chart-group.yaml
@ -13,5 +13,6 @@ data:
    - prometheus
    - prometheus-alertmanager
    - prometheus-node-exporter
+    - prometheus-process-exporter
    - prometheus-kube-state-metrics
    - nagios
--- a/global/software/charts/osh-infra/osh-infra-monitoring/nagios.yaml
+++ b/global/software/charts/osh-infra/osh-infra-monitoring/nagios.yaml
@ -37,6 +37,12 @@ metadata:
        path: .osh_infra.monitoring
      dest:
        path: .values.endpoints.monitoring
+    - src:
+        schema: pegleg/EndpointCatalogue/v1
+        name: osh_infra_endpoints
+        path: .osh_infra.elasticsearch
+      dest:
+        path: .values.endpoints.elasticsearch
    - src:
        schema: pegleg/EndpointCatalogue/v1
        name: osh_infra_endpoints
@ -51,6 +57,18 @@ metadata:
        path: .osh_infra.nagios.admin
      dest:
        path: .values.endpoints.nagios.auth.admin
+    - src:
+        schema: pegleg/AccountCatalogue/v1
+        name: osh_infra_service_accounts
+        path: .osh_infra.prometheus.admin
+      dest:
+        path: .values.endpoints.prometheus.auth.admin
+    - src:
+        schema: pegleg/AccountCatalogue/v1
+        name: osh_infra_service_accounts
+        path: .osh_infra.elasticsearch.admin
+      dest:
+        path: .values.endpoints.elasticsearch.auth.admin

    # Secrets
    - dest:
@ -59,6 +77,18 @@ metadata:
        schema: deckhand/Passphrase/v1
        name: osh_infra_nagios_admin_password
        path: .
+    - dest:
+        path: .values.endpoints.elasticsearch.auth.admin.password
+      src:
+        schema: deckhand/Passphrase/v1
+        name: osh_infra_elasticsearch_admin_password
+        path: .
+    - dest:
+        path: .values.endpoints.prometheus.auth.admin.password
+      src:
+        schema: deckhand/Passphrase/v1
+        name: osh_infra_prometheus_admin_password
+        path: .

    # LDAP Details
    - src:
--- a/global/software/charts/osh-infra/osh-infra-monitoring/prometheus-process-exporter.yaml
+++ b/global/software/charts/osh-infra/osh-infra-monitoring/prometheus-process-exporter.yaml
@ -0,0 +1,65 @@
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: prometheus-process-exporter
+  layeringDefinition:
+    abstract: false
+    layer: global
+  storagePolicy: cleartext
+  substitutions:
+    # Chart source
+    - src:
+        schema: pegleg/SoftwareVersions/v1
+        name: software-versions
+        path: .charts.osh_infra.prometheus_process_exporter
+      dest:
+        path: .source
+
+    # Images
+    - src:
+        schema: pegleg/SoftwareVersions/v1
+        name: software-versions
+        path: .images.osh_infra.prometheus_process_exporter
+      dest:
+        path: .values.images.tags
+
+    # Endpoints
+    - src:
+        schema: pegleg/EndpointCatalogue/v1
+        name: osh_infra_endpoints
+        path: .osh_infra.process_exporter_metrics
+      dest:
+        path: .values.endpoints.process_exporter_metrics
+
+data:
+  chart_name: prometheus-process-exporter
+  release: prometheus-process-exporter
+  namespace: kube-system
+  wait:
+    timeout: 900
+    labels:
+      release_group: airship-prometheus-process-exporter
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+        - type: job
+          labels:
+            release_group: airship-prometheus-process-exporter
+      create: []
+    post:
+      create: []
+  values:
+    labels:
+      node_exporter:
+        node_selector_key: node-exporter
+        node_selector_value: enabled
+      job:
+        node_selector_key: openstack-control-plane
+        node_selector_value: enabled
+  dependencies:
+    - osh-infra-helm-toolkit
+...
--- a/global/software/charts/osh-infra/osh-infra-monitoring/prometheus.yaml
+++ b/global/software/charts/osh-infra/osh-infra-monitoring/prometheus.yaml
--- a/global/software/charts/osh-infra/osh-infra-radosgw/chart-group.yaml
+++ b/global/software/charts/osh-infra/osh-infra-radosgw/chart-group.yaml
@ -0,0 +1,13 @@
+---
+schema: armada/ChartGroup/v1
+metadata:
+  schema: metadata/Document/v1
+  name: osh-infra-radosgw
+  layeringDefinition:
+    abstract: false
+    layer: global
+  storagePolicy: cleartext
+data:
+  description: Deploy Radosgw for OSH-Infra
+  chart_group:
+    - osh-infra-radosgw
--- a/global/software/charts/osh-infra/osh-infra-radosgw/radosgw.yaml
+++ b/global/software/charts/osh-infra/osh-infra-radosgw/radosgw.yaml
@ -0,0 +1,118 @@
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: osh-infra-radosgw
+  layeringDefinition:
+    abstract: false
+    layer: global
+  storagePolicy: cleartext
+  substitutions:
+    # Chart source
+    - src:
+        schema: pegleg/SoftwareVersions/v1
+        name: software-versions
+        path: .charts.ucp.ceph-rgw
+      dest:
+        path: .source
+
+    # Images
+    - src:
+        schema: pegleg/SoftwareVersions/v1
+        name: software-versions
+        path: .images.ceph.ceph-rgw
+      dest:
+        path: .values.images.tags
+
+    # IP addresses
+    - src:
+        schema: pegleg/CommonAddresses/v1
+        name: common-addresses
+        path: .storage.ceph.public_cidr
+      dest:
+        path: .values.network.public
+    - src:
+        schema: pegleg/CommonAddresses/v1
+        name: common-addresses
+        path: .storage.ceph.cluster_cidr
+      dest:
+        path: .values.network.cluster
+
+    # Endpoints
+    - src:
+        schema: pegleg/EndpointCatalogue/v1
+        name: osh_infra_endpoints
+        path: .osh_infra.ceph_object_store
+      dest:
+        path: .values.endpoints.ceph_object_store
+    - src:
+        schema: pegleg/EndpointCatalogue/v1
+        name: ucp_endpoints
+        path: .ceph.ceph_mon
+      dest:
+        path: .values.endpoints.ceph_mon
+
+    # Credentials
+    - src:
+        schema: pegleg/AccountCatalogue/v1
+        name: osh_infra_service_accounts
+        path: .osh_infra.ceph_object_store.admin
+      dest:
+        path: .values.endpoints.ceph_object_store.auth.admin
+
+    # Secrets
+    - dest:
+        path: .values.endpoints.ceph_object_store.auth.admin.access_key
+      src:
+        schema: deckhand/Passphrase/v1
+        name: osh_infra_rgw_s3_admin_access_key
+        path: .
+    - dest:
+        path: .values.endpoints.ceph_object_store.auth.admin.secret_key
+      src:
+        schema: deckhand/Passphrase/v1
+        name: osh_infra_rgw_s3_admin_secret_key
+        path: .
+
+data:
+  chart_name: osh-infra-radosgw
+  release: osh-infra-radosgw
+  namespace: osh-infra
+  wait:
+    timeout: 900
+    labels:
+      release_group: clcp-osh-infra-radosgw
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+        - type: job
+          labels:
+            release_group: clcp-osh-infra-radosgw
+  values:
+    labels:
+      job:
+        node_selector_key: openstack-control-plane
+        node_selector_value: enabled
+      rgw:
+        node_selector_key: ceph-rgw
+        node_selector_value: enabled
+    deployment:
+      storage_secrets: false
+      ceph: true
+      rbd_provisioner: false
+      cephfs_provisioner: false
+      client_secrets: false
+      rgw_keystone_user_and_endpoints: false
+    bootstrap:
+      enabled: false
+    conf:
+      rgw_s3:
+        enabled: true
+    ceph_client:
+      configmap: ceph-etc
+  dependencies:
+    - osh-infra-helm-toolkit
+...
--- a/global/software/charts/osh/openstack-mariadb/mariadb.yaml
+++ b/global/software/charts/osh/openstack-mariadb/mariadb.yaml
@ -31,6 +31,13 @@ metadata:
        path: .osh.oslo_db
      dest:
        path: .values.endpoints.olso_db
+    - src:
+        schema: pegleg/EndpointCatalogue/v1
+        name: osh_endpoints
+        path: .osh.prometheus_mysql_exporter
+      dest:
+        path: .values.endpoints.prometheus_mysql_exporter
+
    # Accounts
    - src:
        schema: pegleg/AccountCatalogue/v1
@ -38,6 +45,12 @@ metadata:
        path: .osh.oslo_db.admin
      dest:
        path: .values.endpoints.oslo_db.auth.admin
+    - src:
+        schema: pegleg/AccountCatalogue/v1
+        name: osh_service_accounts
+        path: .osh.prometheus_mysql_exporter.user
+      dest:
+        path: .values.endpoints.prometheus_mysql_exporter.auth.user

    # Secrets
    - dest:
@ -46,6 +59,12 @@ metadata:
        schema: deckhand/Passphrase/v1
        name: osh_oslo_db_admin_password
        path: .
+    - dest:
+        path: .values.endpoints.oslo_db.auth.exporter.password
+      src:
+        schema: deckhand/Passphrase/v1
+        name: osh_oslo_db_exporter_password
+        path: .

 data:
  chart_name: openstack-mariadb
@ -72,6 +91,9 @@ data:
      prometheus_mysql_exporter:
        node_selector_key: openstack-control-plane
        node_selector_value: enabled
+    monitoring:
+      prometheus:
+        enabled: true
  dependencies:
    - osh-helm-toolkit
 ...
--- a/global/software/config/versions.yaml
+++ b/global/software/config/versions.yaml
@ -182,57 +182,62 @@ data:
    osh_infra:
      elasticsearch:
        location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 4f4e9c5838e9cdf25c453c6a5b85bfc1ce12ad91
+        reference: bc1afb87d7aa529a4ed5321d889cdfe2f1af8a44
        subpath: elasticsearch
        type: git
      fluent_logging:
        location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 4f4e9c5838e9cdf25c453c6a5b85bfc1ce12ad91
+        reference: bc1afb87d7aa529a4ed5321d889cdfe2f1af8a44
        subpath: fluent-logging
        type: git
      grafana:
        location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 4f4e9c5838e9cdf25c453c6a5b85bfc1ce12ad91
+        reference: bc1afb87d7aa529a4ed5321d889cdfe2f1af8a44
        subpath: grafana
        type: git
      helm_toolkit:
        location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 4f4e9c5838e9cdf25c453c6a5b85bfc1ce12ad91
+        reference: bc1afb87d7aa529a4ed5321d889cdfe2f1af8a44
        subpath: helm-toolkit
        type: git
      kibana:
        location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 4f4e9c5838e9cdf25c453c6a5b85bfc1ce12ad91
+        reference: bc1afb87d7aa529a4ed5321d889cdfe2f1af8a44
        subpath: kibana
        type: git
      nagios:
        location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 4f4e9c5838e9cdf25c453c6a5b85bfc1ce12ad91
+        reference: bc1afb87d7aa529a4ed5321d889cdfe2f1af8a44
        subpath: nagios
        type: git
      prometheus:
        location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 4f4e9c5838e9cdf25c453c6a5b85bfc1ce12ad91
+        reference: bc1afb87d7aa529a4ed5321d889cdfe2f1af8a44
        subpath: prometheus
        type: git
      prometheus_alertmanager:
        location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 4f4e9c5838e9cdf25c453c6a5b85bfc1ce12ad91
+        reference: bc1afb87d7aa529a4ed5321d889cdfe2f1af8a44
        subpath: prometheus-alertmanager
        type: git
      prometheus_kube_state_metrics:
        location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 4f4e9c5838e9cdf25c453c6a5b85bfc1ce12ad91
+        reference: bc1afb87d7aa529a4ed5321d889cdfe2f1af8a44
        subpath: prometheus-kube-state-metrics
        type: git
      prometheus_node_exporter:
        location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 4f4e9c5838e9cdf25c453c6a5b85bfc1ce12ad91
+        reference: bc1afb87d7aa529a4ed5321d889cdfe2f1af8a44
        subpath: prometheus-node-exporter
        type: git
+      prometheus_process_exporter:
+        location: https://git.openstack.org/openstack/openstack-helm-infra
+        reference: bc1afb87d7aa529a4ed5321d889cdfe2f1af8a44
+        subpath: prometheus-process-exporter
+        type: git
      prometheus_openstack_exporter:
        location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 4f4e9c5838e9cdf25c453c6a5b85bfc1ce12ad91
+        reference: bc1afb87d7aa529a4ed5321d889cdfe2f1af8a44
        subpath: prometheus-openstack-exporter
        type: git
    ucp:
@ -661,6 +666,9 @@ data:
        curator: docker.io/bobrik/curator:5.2.0
        dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
        elasticsearch: docker.io/elasticsearch:5.6.4
+        ceph_key_placement: docker.io/port/ceph-config-helper:v1.10.3
+        s3_bucket: docker.io/port/ceph-config-helper:v1.10.3
+        s3_user: docker.io/port/ceph-config-helper:v1.10.3
        helm_tests: docker.io/openstackhelm/heat:ocata
        image_repo_sync: docker.io/docker:17.07.0
        memory_init: docker.io/openstackhelm/heat:ocata
@ -713,7 +721,7 @@ data:
        ks_endpoints: docker.io/openstackhelm/heat:ocata
        ks_service: docker.io/openstackhelm/heat:ocata
        ks_user: docker.io/openstackhelm/heat:ocata
-        prometheus_openstack_exporter: quay.io/attcomdev/prometheus-openstack-exporter:3231f14419f0c47547ce2551b7d884cd222104e6
+        prometheus_openstack_exporter: quay.io/attcomdev/prometheus-openstack-exporter:5010c3a532471d4940471a189ca8456bc4db46cb
    ucp:
      armada:
        api: quay.io/airshipit/armada:90618f549c1f6d7741b11dc5c4898f3c6d536895
--- a/global/software/manifests/full-site.yaml
+++ b/global/software/manifests/full-site.yaml
@ -31,6 +31,7 @@ data:
    - ucp-shipyard
    - osh-infra-ingress-controller
    - osh-infra-ceph-config
+    - osh-infra-radosgw
    - osh-infra-logging
    - osh-infra-monitoring
    - osh-infra-mariadb
--- a/site/airship-seaworthy/secrets/passphrases/osh_infra_oslo_db_exporter_password.yaml
+++ b/site/airship-seaworthy/secrets/passphrases/osh_infra_oslo_db_exporter_password.yaml
@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: osh_infra_oslo_db_exporter_password
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...
--- a/site/airship-seaworthy/secrets/passphrases/osh_infra_prometheus_admin_password.yaml
+++ b/site/airship-seaworthy/secrets/passphrases/osh_infra_prometheus_admin_password.yaml
@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: osh_infra_prometheus_admin_password
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...
--- a/site/airship-seaworthy/secrets/passphrases/osh_infra_rgw_s3_admin_access_key.yaml
+++ b/site/airship-seaworthy/secrets/passphrases/osh_infra_rgw_s3_admin_access_key.yaml
@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: osh_infra_rgw_s3_admin_access_key
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...
--- a/site/airship-seaworthy/secrets/passphrases/osh_infra_rgw_s3_admin_secret_key.yaml
+++ b/site/airship-seaworthy/secrets/passphrases/osh_infra_rgw_s3_admin_secret_key.yaml
@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: osh_infra_rgw_s3_admin_secret_key
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...
--- a/site/airship-seaworthy/secrets/passphrases/osh_infra_rgw_s3_elasticsearch_access_key.yaml
+++ b/site/airship-seaworthy/secrets/passphrases/osh_infra_rgw_s3_elasticsearch_access_key.yaml
@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: osh_infra_rgw_s3_elasticsearch_access_key
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...
--- a/site/airship-seaworthy/secrets/passphrases/osh_infra_rgw_s3_elasticsearch_secret_key.yaml
+++ b/site/airship-seaworthy/secrets/passphrases/osh_infra_rgw_s3_elasticsearch_secret_key.yaml
@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: osh_infra_rgw_s3_elasticsearch_secret_key
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...
--- a/site/airship-seaworthy/secrets/passphrases/osh_oslo_db_exporter_password.yaml
+++ b/site/airship-seaworthy/secrets/passphrases/osh_oslo_db_exporter_password.yaml
@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: osh_oslo_db_exporter_password
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...
--- a/site/airship-seaworthy/software/config/endpoints.yaml
+++ b/site/airship-seaworthy/software/config/endpoints.yaml
@ -715,6 +715,19 @@ data:
          default: 3306
        wsrep:
          default: 4567
+    prometheus_mysql_exporter:
+      namespace: openstack
+      hosts:
+        default: mysql-exporter
+      host_fqdn_override:
+        default: null
+      path:
+        default: /metrics
+      scheme:
+        default: 'http'
+      port:
+        metrics:
+          default: 9104
    keystone_oslo_messaging:
      namespace: openstack
      hosts:
@ -1257,6 +1270,22 @@ metadata:
  #       pattern: AUTH_PATH
 data:
  osh_infra:
+    ceph_object_store:
+      name: radosgw
+      namespace: osh-infra
+      hosts:
+        default: ceph-rgw
+        public: radosgw
+      host_fqdn_override:
+        default: null
+      path:
+        default: null
+      scheme:
+        default: "http"
+      port:
+        api:
+          default: 8088
+          public: 80
    elasticsearch:
      name: elasticsearch
      namespace: osh-infra
@ -1272,8 +1301,12 @@ data:
      scheme:
        default: "http"
      port:
+        client:
+          default: 9200
        http:
          default: 80
+        discovery:
+          default: 9300
    prometheus_elasticsearch_exporter:
      namespace: null
      hosts:
@ -1327,6 +1360,19 @@ data:
      port:
        mysql:
          default: 3306
+    prometheus_mysql_exporter:
+      namespace: openstack
+      hosts:
+        default: mysql-exporter
+      host_fqdn_override:
+        default: null
+      path:
+        default: /metrics
+      scheme:
+        default: 'http'
+      port:
+        metrics:
+          default: 9104
    grafana:
      name: grafana
      namespace: osh-infra
@ -1345,6 +1391,7 @@ data:
      port:
        grafana:
          default: 3000
+          public: 80
          # public: 443
    monitoring:
      name: prometheus
@ -1361,7 +1408,8 @@ data:
      port:
        api:
          default: 9090
-          public: 80
+        http:
+          default: 80
    kibana:
      name: kibana
      namespace: osh-infra
@ -1380,6 +1428,8 @@ data:
      port:
        kibana:
          default: 5601
+        http:
+          default: 80
          # public: 443
    alerts:
      name: alertmanager
@ -1438,6 +1488,19 @@ data:
          default: 9100
        prometheus_port:
          default: 9100
+    process_exporter_metrics:
+      namespace: kube-system
+      hosts:
+        default: process-exporter
+      host_fqdn_override:
+        default: null
+      path:
+        default: null
+      scheme:
+        default: "http"
+      port:
+        metrics:
+          default: 9256
    prometheus_openstack_exporter:
      namespace: openstack
      hosts:
--- a/site/airship-seaworthy/software/config/service_accounts.yaml
+++ b/site/airship-seaworthy/software/config/service_accounts.yaml
@ -303,6 +303,9 @@ data:
    oslo_db:
      admin:
        username: root
+    prometheus_mysql_exporter:
+      user:
+        username: osh-oslodb-exporter
    neutron:
      neutron:
        role: admin
@ -383,6 +386,11 @@ metadata:
        path: .osh_infra.prometheus_openstack_exporter.user.region_name
 data:
  osh_infra:
+    ceph_object_store:
+      admin:
+        username: s3_admin
+      elasticsearch:
+        username: elasticsearch
    grafana:
      admin:
        username: grafana
@ -401,6 +409,9 @@ data:
    oslo_db:
      admin:
        username: root
+    prometheus_mysql_exporter:
+      user:
+        username: osh-infra-oslodb-exporter
    prometheus_openstack_exporter:
      user:
        role: admin
@ -411,6 +422,9 @@ data:
    nagios:
      admin:
        username: nagios
+    prometheus:
+      admin:
+        username: prometheus
    ldap:
      admin:
        # NEWSITE-CHANGEME: Replace with the site's LDAP account used to
--- a/site/airship-seaworthy/software/manifests/full-site.yaml
+++ b/site/airship-seaworthy/software/manifests/full-site.yaml
@ -37,6 +37,7 @@ data:
    - ucp-shipyard
    - osh-infra-ingress-controller
    - osh-infra-ceph-config
+    - osh-infra-radosgw
    - osh-infra-logging
    - osh-infra-monitoring
    - osh-infra-mariadb