Armada metric output for genesis
This change has passed the Promenade resiliency gate. Armada recently added support for generating metric output [0] [1]. This adds support for capturing this output during genesis Armada runs, and storing it in a configurable path on the genesis node, so that it can later be consumed by for example by the node exporter text file collector [2]. The number of Armada attempts to collect metrics for is also configurable. [0]: https://airship-armada.readthedocs.io/en/latest/operations/metrics.html#exporting [1]: https://review.opendev.org/#/c/668980/ [2]: https://github.com/prometheus/node_exporter#textfile-collector Change-Id: Ifd5b9d351204541595b1aadf1f06b16ab54308b0
This commit is contained in:
parent
bffa2b0cfd
commit
4168418719
@ -25,6 +25,9 @@ Here is a complete sample document:
|
|||||||
ip: 192.168.77.10
|
ip: 192.168.77.10
|
||||||
armada:
|
armada:
|
||||||
target_manifest: cluster-bootstrap
|
target_manifest: cluster-bootstrap
|
||||||
|
metrics:
|
||||||
|
output_dir: /var/log/armada/metrics
|
||||||
|
max_attempts: 5
|
||||||
tiller:
|
tiller:
|
||||||
listen: 24134
|
listen: 24134
|
||||||
probe_listen: 24135
|
probe_listen: 24135
|
||||||
@ -57,10 +60,30 @@ Here is a complete sample document:
|
|||||||
Armada
|
Armada
|
||||||
------
|
------
|
||||||
|
|
||||||
This section contains particular configuration options for bootstrapping with
|
Configuration options for bootstrapping with Armada.
|
||||||
Armada. It currently only supports a single option: ``target_manifest``, which
|
|
||||||
specifies which ``armada/Manifest/v1`` to be used during Genesis.
|
|
||||||
|
|
||||||
|
+-----------------+----------+---------------------------------------------------------------------------------------+
|
||||||
|
| keyword | type | action |
|
||||||
|
+=================+==========+=======================================================================================+
|
||||||
|
| target_manifest | string | Specifies the ``armada/Manifest/v1`` to use during Genesis. |
|
||||||
|
+-----------------+----------+---------------------------------------------------------------------------------------+
|
||||||
|
| metrics | object | See `Metrics`_. |
|
||||||
|
+-----------------+----------+---------------------------------------------------------------------------------------+
|
||||||
|
|
||||||
|
Metrics
|
||||||
|
^^^^^^^
|
||||||
|
|
||||||
|
Configuration for Armada bootstrap metric collection.
|
||||||
|
|
||||||
|
+-----------------+----------+---------------------------------------------------------------------------------------+
|
||||||
|
| keyword | type | action |
|
||||||
|
+=================+==========+=======================================================================================+
|
||||||
|
| output_dir | string | (optional, default `/var/log/node-exporter-textfiles`) The directory path in which to |
|
||||||
|
| | | output Armada metric data. |
|
||||||
|
+-----------------+----------+---------------------------------------------------------------------------------------+
|
||||||
|
| max_attempts | integer | (optional, default 10) The maximum Armada attempts to collect metrics for. |
|
||||||
|
| | | Can be set to 0 to disable metrics collection. |
|
||||||
|
+-----------------+----------+---------------------------------------------------------------------------------------+
|
||||||
|
|
||||||
Bootstrapping Images
|
Bootstrapping Images
|
||||||
--------------------
|
--------------------
|
||||||
|
@ -62,6 +62,14 @@ data:
|
|||||||
properties:
|
properties:
|
||||||
target_manifest:
|
target_manifest:
|
||||||
type: string
|
type: string
|
||||||
|
metrics:
|
||||||
|
type: object
|
||||||
|
properties:
|
||||||
|
output_dir:
|
||||||
|
type: string
|
||||||
|
max_attempts:
|
||||||
|
type: integer
|
||||||
|
additionalProperties: false
|
||||||
additionalProperties: false
|
additionalProperties: false
|
||||||
|
|
||||||
apiserver:
|
apiserver:
|
||||||
|
@ -69,6 +69,8 @@ spec:
|
|||||||
- |-
|
- |-
|
||||||
set -x
|
set -x
|
||||||
|
|
||||||
|
declare -i attempt=1
|
||||||
|
|
||||||
while true; do
|
while true; do
|
||||||
sleep 10
|
sleep 10
|
||||||
if armada \
|
if armada \
|
||||||
@ -76,15 +78,21 @@ spec:
|
|||||||
--target-manifest {{ config.get_path('Genesis:armada.target_manifest', 'cluster-bootstrap') }} \
|
--target-manifest {{ config.get_path('Genesis:armada.target_manifest', 'cluster-bootstrap') }} \
|
||||||
--tiller-host 127.0.0.1 \
|
--tiller-host 127.0.0.1 \
|
||||||
--tiller-port {{ tiller_listen }} \
|
--tiller-port {{ tiller_listen }} \
|
||||||
|
$([[ $attempt -le $ARMADA_METRICS_MAX_ATTEMPTS ]] && echo --metrics-output "${ARMADA_METRICS_OUTPUT_DIR}/armada-bootstrap-${attempt}.prom") \
|
||||||
/etc/genesis/armada/assets/manifest.yaml &>> "${ARMADA_LOGFILE}"; then
|
/etc/genesis/armada/assets/manifest.yaml &>> "${ARMADA_LOGFILE}"; then
|
||||||
break
|
break
|
||||||
fi
|
fi
|
||||||
|
attempt+=1
|
||||||
done
|
done
|
||||||
touch /ipc/armada-done
|
touch /ipc/armada-done
|
||||||
sleep 10000
|
sleep 10000
|
||||||
env:
|
env:
|
||||||
- name: ARMADA_LOGFILE
|
- name: ARMADA_LOGFILE
|
||||||
value: /tmp/log/bootstrap-armada.log
|
value: /tmp/log/bootstrap-armada.log
|
||||||
|
- name: ARMADA_METRICS_OUTPUT_DIR
|
||||||
|
value: /tmp/metrics
|
||||||
|
- name: ARMADA_METRICS_MAX_ATTEMPTS
|
||||||
|
value: "{{ config.get_path('Genesis:armada.metrics.max_attempts', 10) }}"
|
||||||
{%- if config['KubernetesNetwork:proxy.url'] is defined %}
|
{%- if config['KubernetesNetwork:proxy.url'] is defined %}
|
||||||
- name: HTTP_PROXY
|
- name: HTTP_PROXY
|
||||||
value: {{ config['KubernetesNetwork:proxy.url'] }}
|
value: {{ config['KubernetesNetwork:proxy.url'] }}
|
||||||
@ -108,6 +116,8 @@ spec:
|
|||||||
mountPath: /ipc
|
mountPath: /ipc
|
||||||
- name: log
|
- name: log
|
||||||
mountPath: /tmp/log
|
mountPath: /tmp/log
|
||||||
|
- name: metrics
|
||||||
|
mountPath: /tmp/metrics
|
||||||
- name: monitor
|
- name: monitor
|
||||||
image: {{ config['HostSystem:images.monitoring_image'] }}
|
image: {{ config['HostSystem:images.monitoring_image'] }}
|
||||||
command:
|
command:
|
||||||
@ -161,6 +171,9 @@ spec:
|
|||||||
- name: log
|
- name: log
|
||||||
hostPath:
|
hostPath:
|
||||||
path: /var/log/armada
|
path: /var/log/armada
|
||||||
|
- name: metrics
|
||||||
|
hostPath:
|
||||||
|
path: {{ config.get_path('Genesis:armada.metrics.output_dir', '/var/log/node-exporter-textfiles') }}
|
||||||
- name: config
|
- name: config
|
||||||
hostPath:
|
hostPath:
|
||||||
path: /etc/genesis/apiserver
|
path: /etc/genesis/apiserver
|
||||||
|
@ -8,6 +8,9 @@ mkdir -p /var/log/armada
|
|||||||
touch /var/log/armada/bootstrap-armada.log
|
touch /var/log/armada/bootstrap-armada.log
|
||||||
chmod 777 /var/log/armada/bootstrap-armada.log
|
chmod 777 /var/log/armada/bootstrap-armada.log
|
||||||
|
|
||||||
|
{% set metrics_output_dir = config.get_path('Genesis:armada.metrics.output_dir', '/var/log/node-exporter-textfiles') %}
|
||||||
|
install -d -m 755 {{ metrics_output_dir }}
|
||||||
|
|
||||||
chmod -R 600 /etc/genesis
|
chmod -R 600 /etc/genesis
|
||||||
|
|
||||||
set +x
|
set +x
|
||||||
|
Loading…
Reference in New Issue
Block a user