---
apiVersion: v1
kind: Pod
metadata:
  name: auxiliary-etcd
  namespace: kube-system
  labels:
    application: kubernetes
    component: auxiliary-etcd
    promenade: genesis
spec:
  hostNetwork: true
  containers:
{%- with etcd_name = 'auxiliary-0', client_port = 12379, peer_port = 12380  %}
{% include "genesis-etcd/server-container.yaml" with context %}
{%- endwith %}

{%- with etcd_name = 'auxiliary-1', client_port = 22379, peer_port = 22380  %}
{% include "genesis-etcd/server-container.yaml" with context %}
{%- endwith %}

    - name: monitor
      image: {{ config['Genesis:images.kubernetes.etcdctl'] }}
      command:
        - /bin/sh
        - -c
        - |-
          set -x

          external_member_count () {
              etcdctl member list \
                  | grep '\bstarted\b' \
                  | grep -Ev "\\b(auxiliary-0|auxiliary-1)\\b" \
                  | wc -l
          }

          remove_if_possible () {
              MEMBER_NAME=$1
              MEMBER_ID=$(etcdctl member list | grep "${MEMBER_NAME}" | awk -F ', ' '{ print $1 }')
              if [ -n "${MEMBER_ID}" ]; then
                  etcdctl member remove $MEMBER_ID
                  sleep 10 
              fi
          }

          auxiliary_threshold="{{ config.get_first('Genesis:etcd.auxiliary_threshold', default=3) }}"
          # NOTE(sh8121att): If there are enough (a fully resilient contigent) non-auxiliary members,
          # then we are ready to remove the auxiliary members.  Otherwise, wait.
          while [ ! "$(external_member_count)" -ge "$auxiliary_threshold" ]; do
              sleep 30
          done

          # NOTE(mark-burnett): Failures beyond this point are unexpected, but
          # should be recovered by restarting this container.
          set -e

          sleep 600
          # make a switch to permanent etcd cluster
          ENDPOINTS=$(etcdctl member list | awk -F ',' '$3 ~! /auxiliary/ { if (output != "") output=output","$5; else output=$5} END {gsub(/ /, "", output); print output}' | sed 's/\//\\\//g')
          sed -i "/- name: ETCD_ENDPOINTS/{n;s/value: .*/value: \"$ENDPOINTS\"/}" /manifests/kubernetes-apiserver.yaml

          remove_if_possible auxiliary-0
          remove_if_possible auxiliary-1

          rm -rf \
              /var/lib/etcd/auxiliary-0 \
              /var/lib/etcd/auxiliary-1 \
              /manifests/auxiliary-kubernetes-etcd.yaml

          sleep 10000
      env:
        - name: ETCDCTL_API
          value: '3'
        - name: ETCDCTL_DIAL_TIMEOUT
          value: 3s
        - name: ETCDCTL_ENDPOINTS
          value: https://127.0.0.1:2379
        - name: ETCDCTL_CACERT
          value: /etc/etcd/pki/client-ca.pem
        - name: ETCDCTL_CERT
          value: /etc/etcd/pki/etcd-client.pem
        - name: ETCDCTL_KEY
          value: /etc/etcd/pki/etcd-client-key.pem
      volumeMounts:
        - name: all-etcd-data
          mountPath: /var/lib/etcd
        - name: pki-auxiliary-0
          mountPath: /etc/etcd/pki
        - name: manifest
          mountPath: /manifests
  volumes:
{%- with etcd_name = 'auxiliary-0', client_port = 12379, peer_port = 12380, volume_name = 'auxiliary-0' %}
{% include "genesis-etcd/common-volumes.yaml" with context %}
{%- endwith %}
{%- with etcd_name = 'auxiliary-1', client_port = 22379, peer_port = 22380, volume_name = 'auxiliary-1' %}
{% include "genesis-etcd/common-volumes.yaml" with context %}
{%- endwith %}
    - name: manifest
      hostPath:
        path: /etc/kubernetes/manifests
    - name: all-etcd-data
      hostPath:
        path: /var/lib/etcd
...