Browse Source

Update Elasticsearch and Fluent-Logging configurations

This updates the Elasticsearch and Fluent-logging charts to use
the most recent configuration keys in their values overrides, and
also introduces support for the ceph-rgw s3 api for use for
Elasticsearch snapshot repositories

Change-Id: Ia998db9006350a22fcc7dc3052301d7a5b8259f4
changes/50/615650/1
Steve Wilkerson 3 years ago
parent
commit
0a1ba88004
  1. 1
      global/profiles/host/cp.yaml
  2. 1
      global/profiles/host/dp.yaml
  3. 211
      global/software/charts/osh-infra/osh-infra-logging/elasticsearch.yaml
  4. 195
      global/software/charts/osh-infra/osh-infra-logging/fluent-logging.yaml
  5. 13
      global/software/charts/osh-infra/osh-infra-radosgw/chart-group.yaml
  6. 118
      global/software/charts/osh-infra/osh-infra-radosgw/radosgw.yaml
  7. 1
      global/software/manifests/full-site.yaml
  8. 11
      site/airship-seaworthy/secrets/passphrases/osh_infra_rgw_s3_admin_access_key.yaml
  9. 11
      site/airship-seaworthy/secrets/passphrases/osh_infra_rgw_s3_admin_secret_key.yaml
  10. 11
      site/airship-seaworthy/secrets/passphrases/osh_infra_rgw_s3_elasticsearch_access_key.yaml
  11. 11
      site/airship-seaworthy/secrets/passphrases/osh_infra_rgw_s3_elasticsearch_secret_key.yaml
  12. 20
      site/airship-seaworthy/software/config/endpoints.yaml
  13. 5
      site/airship-seaworthy/software/config/service_accounts.yaml

1
global/profiles/host/cp.yaml

@ -102,6 +102,7 @@ data:
prometheus-server: enabled
prometheus-client: enabled
fluentd: enabled
fluentbit: enabled
influxdb: enabled
kibana: enabled
elasticsearch-client: enabled

1
global/profiles/host/dp.yaml

@ -61,4 +61,5 @@ data:
openstack-libvirt: kernel
beta.kubernetes.io/fluentd-ds-ready: 'true'
node-exporter: enabled
fluentbit: enabled
...

211
global/software/charts/osh-infra/osh-infra-logging/elasticsearch.yaml

@ -54,6 +54,18 @@ metadata:
path: .osh_infra.elasticsearch.admin
dest:
path: .values.endpoints.elasticsearch.auth.admin
- src:
schema: pegleg/AccountCatalogue/v1
name: osh_infra_service_accounts
path: .osh_infra.ceph_object_store.admin
dest:
path: .values.endpoints.ceph_object_store.auth.admin
- src:
schema: pegleg/AccountCatalogue/v1
name: osh_infra_service_accounts
path: .osh_infra.ceph_object_store.elasticsearch
dest:
path: .values.endpoints.ceph_object_store.auth.elasticsearch
# Secrets
- dest:
@ -62,6 +74,30 @@ metadata:
schema: deckhand/Passphrase/v1
name: osh_infra_elasticsearch_admin_password
path: .
- dest:
path: .values.endpoints.ceph_object_store.auth.admin.access_key
src:
schema: deckhand/Passphrase/v1
name: osh_infra_rgw_s3_admin_access_key
path: .
- dest:
path: .values.endpoints.ceph_object_store.auth.admin.secret_key
src:
schema: deckhand/Passphrase/v1
name: osh_infra_rgw_s3_admin_secret_key
path: .
- dest:
path: .values.endpoints.ceph_object_store.auth.elasticsearch.access_key
src:
schema: deckhand/Passphrase/v1
name: osh_infra_rgw_s3_elasticsearch_access_key
path: .
- dest:
path: .values.endpoints.ceph_object_store.auth.elasticsearch.secret_key
src:
schema: deckhand/Passphrase/v1
name: osh_infra_rgw_s3_elasticsearch_secret_key
path: .
# LDAP Details
- src:
@ -97,6 +133,75 @@ data:
post:
create: []
values:
pod:
replicas:
client: 5
resources:
enabled: true
apache_proxy:
limits:
memory: "1024Mi"
cpu: "2000m"
requests:
memory: "0"
cpu: "0"
client:
requests:
memory: "8Gi"
cpu: "1000m"
limits:
memory: "16Gi"
cpu: "2000m"
master:
requests:
memory: "8Gi"
cpu: "1000m"
limits:
memory: "16Gi"
cpu: "2000m"
data:
requests:
memory: "8Gi"
cpu: "1000m"
limits:
memory: "16Gi"
cpu: "2000m"
prometheus_elasticsearch_exporter:
requests:
memory: "0"
cpu: "0"
limits:
memory: "1024Mi"
cpu: "2000m"
jobs:
curator:
requests:
memory: "0"
cpu: "0"
limits:
memory: "1024Mi"
cpu: "2000m"
image_repo_sync:
requests:
memory: "0"
cpu: "0"
limits:
memory: "1024Mi"
cpu: "2000m"
snapshot_repository:
requests:
memory: "0"
cpu: "0"
limits:
memory: "1024Mi"
cpu: "2000m"
tests:
requests:
memory: "0"
cpu: "0"
limits:
memory: "1024Mi"
cpu: "2000m"
labels:
elasticsearch:
node_selector_key: openstack-control-plane
@ -108,27 +213,95 @@ data:
prometheus:
enabled: true
conf:
apache:
host: |
<VirtualHost *:80>
<Location />
ProxyPass http://localhost:{{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/
ProxyPassReverse http://localhost:{{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/
</Location>
<Proxy *>
AuthName "Elasticsearch"
AuthType Basic
AuthBasicProvider file ldap
AuthUserFile /usr/local/apache2/conf/.htpasswd
AuthLDAPBindDN {{ .Values.endpoints.ldap.auth.admin.bind }}
AuthLDAPBindPassword {{ .Values.endpoints.ldap.auth.admin.password }}
AuthLDAPURL {{ tuple "ldap" "public" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
Require valid-user
</Proxy>
</VirtualHost>
httpd: |
ServerRoot "/usr/local/apache2"
Listen 80
LoadModule mpm_event_module modules/mod_mpm_event.so
LoadModule authn_file_module modules/mod_authn_file.so
LoadModule authn_core_module modules/mod_authn_core.so
LoadModule authz_host_module modules/mod_authz_host.so
LoadModule authz_groupfile_module modules/mod_authz_groupfile.so
LoadModule authz_user_module modules/mod_authz_user.so
LoadModule authz_core_module modules/mod_authz_core.so
LoadModule access_compat_module modules/mod_access_compat.so
LoadModule auth_basic_module modules/mod_auth_basic.so
LoadModule ldap_module modules/mod_ldap.so
LoadModule authnz_ldap_module modules/mod_authnz_ldap.so
LoadModule reqtimeout_module modules/mod_reqtimeout.so
LoadModule filter_module modules/mod_filter.so
LoadModule proxy_html_module modules/mod_proxy_html.so
LoadModule log_config_module modules/mod_log_config.so
LoadModule env_module modules/mod_env.so
LoadModule headers_module modules/mod_headers.so
LoadModule setenvif_module modules/mod_setenvif.so
LoadModule version_module modules/mod_version.so
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_connect_module modules/mod_proxy_connect.so
LoadModule proxy_http_module modules/mod_proxy_http.so
LoadModule proxy_balancer_module modules/mod_proxy_balancer.so
LoadModule slotmem_shm_module modules/mod_slotmem_shm.so
LoadModule slotmem_plain_module modules/mod_slotmem_plain.so
LoadModule unixd_module modules/mod_unixd.so
LoadModule status_module modules/mod_status.so
LoadModule autoindex_module modules/mod_autoindex.so
<IfModule unixd_module>
User daemon
Group daemon
</IfModule>
<Directory />
AllowOverride none
Require all denied
</Directory>
<Files ".ht*">
Require all denied
</Files>
ErrorLog /dev/stderr
LogLevel warn
<IfModule log_config_module>
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
LogFormat "%h %l %u %t \"%r\" %>s %b" common
<IfModule logio_module>
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
</IfModule>
CustomLog /dev/stdout common
CustomLog /dev/stdout combined
</IfModule>
<Directory "/usr/local/apache2/cgi-bin">
AllowOverride None
Options None
Require all granted
</Directory>
<IfModule headers_module>
RequestHeader unset Proxy early
</IfModule>
<IfModule proxy_html_module>
Include conf/extra/proxy-html.conf
</IfModule>
<VirtualHost *:80>
<Location />
ProxyPass http://localhost:{{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/
ProxyPassReverse http://localhost:{{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/
</Location>
<Proxy *>
AuthName "Elasticsearch"
AuthType Basic
AuthBasicProvider file ldap
AuthUserFile /usr/local/apache2/conf/.htpasswd
AuthLDAPBindDN {{ .Values.endpoints.ldap.auth.admin.bind }}
AuthLDAPBindPassword {{ .Values.endpoints.ldap.auth.admin.password }}
AuthLDAPURL {{ tuple "ldap" "public" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }}
Require valid-user
</Proxy>
</VirtualHost>
elasticsearch:
config:
http:
max_content_length: 2gb
pipelining: false
env:
java_opts: "-Xms5g -Xmx5g"
java_opts: "-Xms8g -Xmx8g"
snapshots:
enabled: true
curator:
#run every 6th hour
schedule: "0 */6 * * *"

195
global/software/charts/osh-infra/osh-infra-logging/fluent-logging.yaml

@ -82,12 +82,54 @@ data:
post:
create: []
values:
monitoring:
prometheus:
enabled: true
pod:
resources:
enabled: true
fluentbit:
limits:
memory: '4Gi'
cpu: '2000m'
requests:
memory: '2Gi'
cpu: '1000m'
fluentd:
limits:
memory: '4Gi'
cpu: '2000m'
requests:
memory: '2Gi'
cpu: '1000m'
prometheus_fluentd_exporter:
limits:
memory: '1024Mi'
cpu: '2000m'
requests:
memory: '0'
cpu: '0'
jobs:
image_repo_sync:
requests:
memory: '0'
cpu: '0'
limits:
memory: '1024Mi'
cpu: '2000m'
tests:
requests:
memory: '0'
cpu: '0'
limits:
memory: '1024Mi'
cpu: '2000m'
labels:
fluentd:
node_selector_key: openstack-control-plane
node_selector_key: fluentd
node_selector_value: enabled
fluentbit:
node_selector_key: openstack-control-plane
node_selector_key: fluentbit
node_selector_value: enabled
prometheus_fluentd_exporter:
node_selector_key: openstack-control-plane
@ -95,20 +137,6 @@ data:
job:
node_selector_key: openstack-control-plane
node_selector_value: enabled
dependencies:
static:
fluentbit:
jobs: ""
services:
- endpoint: internal
service: fluentd
fluentd:
jobs: ""
services:
- endpoint: internal
service: elasticsearch
manifests:
job_elasticsearch_template: false
conf:
fluentbit:
- service:
@ -117,11 +145,67 @@ data:
Daemon: Off
Log_Level: info
Parsers_File: parsers.conf
- kernel_messages:
header: input
Name: tail
Tag: kernel
Path: /var/log/kern.log
DB: /var/log/kern.db
Mem_Buf_Limit: 5MB
DB.Sync: Normal
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
- kubelet:
header: input
Name: systemd
Tag: journal.*
Path: ${JOURNAL_PATH}
Systemd_Filter: _SYSTEMD_UNIT=kubelet.service
DB: /var/log/kubelet.db
Mem_Buf_Limit: 5MB
DB.Sync: Normal
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
- docker_daemon:
header: input
Name: systemd
Tag: journal.*
Path: ${JOURNAL_PATH}
Systemd_Filter: _SYSTEMD_UNIT=docker.service
DB: /var/log/docker.db
Mem_Buf_Limit: 5MB
DB.Sync: Normal
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
- kernel_record_modifier:
header: filter
Name: record_modifier
Match: kernel
Record: hostname ${HOSTNAME}
- systemd_modify_fields:
header: filter
Name: modify
Match: journal.**
Rename:
_BOOT_ID: BOOT_ID
_CAP_EFFECTIVE: CAP_EFFECTIVE
_CMDLINE: CMDLINE
_COMM: COMM
_EXE: EXE
_GID: GID
_HOSTNAME: HOSTNAME
_MACHINE_ID: MACHINE_ID
_PID: PID
_SYSTEMD_CGROUP: SYSTEMD_CGROUP
_SYSTEMD_SLICE: SYSTEMD_SLICE
_SYSTEMD_UNIT: SYSTEMD_UNIT
_UID: UID
_TRANSPORT: TRANSPORT
- ceph_cluster_logs:
header: input
Name: tail
Tag: ceph.cluster.*
Path: /var/log/ceph/ceph.log
Path: /var/log/ceph/airship-ucp-ceph-mon/ceph.log
DB: /var/log/ceph.db
Parsers: syslog
Mem_Buf_Limit: 5MB
@ -132,7 +216,7 @@ data:
header: input
Name: tail
Tag: ceph.audit.*
Path: /var/log/ceph/ceph.audit.log
Path: /var/log/ceph/airship-ucp-ceph-mon/ceph.audit.log
DB: /var/log/ceph.db
Parsers: syslog
Mem_Buf_Limit: 5MB
@ -143,7 +227,7 @@ data:
header: input
Name: tail
Tag: ceph.mon.*
Path: /var/log/ceph/ceph-mon**.log
Path: /var/log/ceph/airship-ucp-ceph-mon/ceph-mon**.log
DB: /var/log/ceph.db
Parsers: syslog
Mem_Buf_Limit: 5MB
@ -154,7 +238,7 @@ data:
header: input
Name: tail
Tag: ceph.osd.*
Path: /var/log/ceph/ceph-osd**.log
Path: /var/log/ceph/airship-ucp-ceph-osd/ceph-osd**.log
DB: /var/log/ceph.db
Parsers: syslog
Mem_Buf_Limit: 5MB
@ -172,6 +256,10 @@ data:
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
Mem_Buf_Limit: 5MB
- drop_fluentd_logs:
header: output
Name: "null"
Match: "**.fluentd**"
- kube_filter:
header: filter
Name: kubernetes
@ -183,7 +271,7 @@ data:
Match: "*"
Host: ${FLUENTD_HOST}
Port: ${FLUENTD_PORT}
td_agent:
fluentd:
- metrics_agent:
header: source
type: monitor_agent
@ -194,12 +282,48 @@ data:
type: forward
port: "#{ENV['FLUENTD_PORT']}"
bind: 0.0.0.0
- filter_fluentd_logs:
header: match
expression: "fluent.**"
type: "null"
- journal_elasticsearch:
header: match
type: elasticsearch
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
expression: "journal.**"
include_tag_key: true
host: "#{ENV['ELASTICSEARCH_HOST']}"
port: "#{ENV['ELASTICSEARCH_PORT']}"
logstash_format: true
logstash_prefix: journal
buffer_chunk_limit: 2M
buffer_queue_limit: 8
flush_interval: "10"
max_retry_wait: 300
disable_retry_limit: ""
- kernel_elasticsearch:
header: match
type: elasticsearch
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
expression: "kernel"
include_tag_key: true
host: "#{ENV['ELASTICSEARCH_HOST']}"
port: "#{ENV['ELASTICSEARCH_PORT']}"
logstash_format: true
logstash_prefix: kernel
buffer_chunk_limit: 2M
buffer_queue_limit: 8
flush_interval: "10"
max_retry_wait: 300
disable_retry_limit: ""
- ceph_elasticsearch:
header: match
type: elasticsearch
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
expression: "ceph**"
expression: "**ceph-**.log"
include_tag_key: true
host: "#{ENV['ELASTICSEARCH_HOST']}"
port: "#{ENV['ELASTICSEARCH_PORT']}"
@ -207,11 +331,30 @@ data:
logstash_prefix: ceph
buffer_chunk_limit: 10M
buffer_queue_limit: 32
flush_interval: "20"
flush_interval: "10"
max_retry_wait: 300
disable_retry_limit: ""
num_threads: 8
type_name: ceph_logs
- oslo_fluentd_elasticsearch:
header: match
type: elasticsearch
user: "#{ENV['ELASTICSEARCH_USERNAME']}"
password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
expression: "**.openstack.*"
include_tag_key: true
host: "#{ENV['ELASTICSEARCH_HOST']}"
port: "#{ENV['ELASTICSEARCH_PORT']}"
logstash_format: true
logstash_prefix: openstack
buffer_type: memory
buffer_chunk_limit: 10M
buffer_queue_limit: 512
flush_interval: "10"
max_retry_wait: 300
request_timeout: 60
disable_retry_limit: ""
num_threads: 8
type_name: oslo_openstack_fluentd
- elasticsearch:
header: match
type: elasticsearch
@ -223,8 +366,8 @@ data:
port: "#{ENV['ELASTICSEARCH_PORT']}"
logstash_format: true
buffer_chunk_limit: 10M
buffer_queue_limit: 32
flush_interval: "20"
buffer_queue_limit: 32g
flush_interval: "10"
max_retry_wait: 300
disable_retry_limit: ""
num_threads: 8

13
global/software/charts/osh-infra/osh-infra-radosgw/chart-group.yaml

@ -0,0 +1,13 @@
---
schema: armada/ChartGroup/v1
metadata:
schema: metadata/Document/v1
name: osh-infra-radosgw
layeringDefinition:
abstract: false
layer: global
storagePolicy: cleartext
data:
description: Deploy Radosgw for OSH-Infra
chart_group:
- osh-infra-radosgw

118
global/software/charts/osh-infra/osh-infra-radosgw/radosgw.yaml

@ -0,0 +1,118 @@
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: osh-infra-radosgw
layeringDefinition:
abstract: false
layer: global
storagePolicy: cleartext
substitutions:
# Chart source
- src:
schema: pegleg/SoftwareVersions/v1
name: software-versions
path: .charts.ucp.ceph-rgw
dest:
path: .source
# Images
- src:
schema: pegleg/SoftwareVersions/v1
name: software-versions
path: .images.ceph.ceph-rgw
dest:
path: .values.images.tags
# IP addresses
- src:
schema: pegleg/CommonAddresses/v1
name: common-addresses
path: .storage.ceph.public_cidr
dest:
path: .values.network.public
- src:
schema: pegleg/CommonAddresses/v1
name: common-addresses
path: .storage.ceph.cluster_cidr
dest:
path: .values.network.cluster
# Endpoints
- src:
schema: pegleg/EndpointCatalogue/v1
name: osh_infra_endpoints
path: .osh_infra.ceph_object_store
dest:
path: .values.endpoints.ceph_object_store
- src:
schema: pegleg/EndpointCatalogue/v1
name: ucp_endpoints
path: .ceph.ceph_mon
dest:
path: .values.endpoints.ceph_mon
# Credentials
- src:
schema: pegleg/AccountCatalogue/v1
name: osh_infra_service_accounts
path: .osh_infra.ceph_object_store.admin
dest:
path: .values.endpoints.ceph_object_store.auth.admin
# Secrets
- dest:
path: .values.endpoints.ceph_object_store.auth.admin.access_key
src:
schema: deckhand/Passphrase/v1
name: osh_infra_rgw_s3_admin_access_key
path: .
- dest:
path: .values.endpoints.ceph_object_store.auth.admin.secret_key
src:
schema: deckhand/Passphrase/v1
name: osh_infra_rgw_s3_admin_secret_key
path: .
data:
chart_name: osh-infra-radosgw
release: osh-infra-radosgw
namespace: osh-infra
wait:
timeout: 900
labels:
release_group: clcp-osh-infra-radosgw
install:
no_hooks: false
upgrade:
no_hooks: false
pre:
delete:
- type: job
labels:
release_group: clcp-osh-infra-radosgw
values:
labels:
job:
node_selector_key: openstack-control-plane
node_selector_value: enabled
rgw:
node_selector_key: ceph-rgw
node_selector_value: enabled
deployment:
storage_secrets: false
ceph: true
rbd_provisioner: false
cephfs_provisioner: false
client_secrets: false
rgw_keystone_user_and_endpoints: false
bootstrap:
enabled: false
conf:
rgw_s3:
enabled: true
ceph_client:
configmap: ceph-etc
dependencies:
- osh-infra-helm-toolkit
...

1
global/software/manifests/full-site.yaml

@ -31,6 +31,7 @@ data:
- ucp-shipyard
- osh-infra-ingress-controller
- osh-infra-ceph-config
- osh-infra-radosgw
- osh-infra-logging
- osh-infra-monitoring
- osh-infra-mariadb

11
site/airship-seaworthy/secrets/passphrases/osh_infra_rgw_s3_admin_access_key.yaml

@ -0,0 +1,11 @@
---
schema: deckhand/Passphrase/v1
metadata:
schema: metadata/Document/v1
name: osh_infra_rgw_s3_admin_access_key
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data: password123
...

11
site/airship-seaworthy/secrets/passphrases/osh_infra_rgw_s3_admin_secret_key.yaml

@ -0,0 +1,11 @@
---
schema: deckhand/Passphrase/v1
metadata:
schema: metadata/Document/v1
name: osh_infra_rgw_s3_admin_secret_key
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data: password123
...

11
site/airship-seaworthy/secrets/passphrases/osh_infra_rgw_s3_elasticsearch_access_key.yaml

@ -0,0 +1,11 @@
---
schema: deckhand/Passphrase/v1
metadata:
schema: metadata/Document/v1
name: osh_infra_rgw_s3_elasticsearch_access_key
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data: password123
...

11
site/airship-seaworthy/secrets/passphrases/osh_infra_rgw_s3_elasticsearch_secret_key.yaml

@ -0,0 +1,11 @@
---
schema: deckhand/Passphrase/v1
metadata:
schema: metadata/Document/v1
name: osh_infra_rgw_s3_elasticsearch_secret_key
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data: password123
...

20
site/airship-seaworthy/software/config/endpoints.yaml

@ -1008,6 +1008,22 @@ metadata:
pattern: AUTH_PATH
data:
osh_infra:
ceph_object_store:
name: radosgw
namespace: osh-infra
hosts:
default: ceph-rgw
public: radosgw
host_fqdn_override:
default: null
path:
default: null
scheme:
default: "http"
port:
api:
default: 8088
public: 80
elasticsearch:
name: elasticsearch
namespace: osh-infra
@ -1023,8 +1039,12 @@ data:
scheme:
default: "http"
port:
client:
default: 9200
http:
default: 80
discovery:
default: 9300
prometheus_elasticsearch_exporter:
namespace: null
hosts:

5
site/airship-seaworthy/software/config/service_accounts.yaml

@ -383,6 +383,11 @@ metadata:
path: .osh_infra.prometheus_openstack_exporter.user.region_name
data:
osh_infra:
ceph_object_store:
admin:
username: s3_admin
elasticsearch:
username: elasticsearch
grafana:
admin:
username: grafana

Loading…
Cancel
Save