---
#
# Copyright (c) 2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# SUB-TASKS DESCRIPTION:
#   Bring up Kubernetes master
#   - Update iptables
#   - Create daemon.json for insecure unified registry if applicable
#   - Create manifest directory
#   - Enable kubelet service (with default/custom registry)
#   - Run kubeadm init
#   - Prepare admin.conf
#   - Set k8s environment variable for new shell
#   - Prepare Calico config and activate Calico networking
#   - Prepare Multus config and activate Multus networking
#   - Prepare SRIOV config and activate SRIOV networking
#   - Prepare SRIOV device plugin config and activate SRIOV device plugin
#   - Restrict coredns to master node and set anti-affnity (duplex system)
#   - Restrict coredns to 1 pod (simplex system)
#   - Remove taint from master node
#   - Add kubelet service override
#   - Register kubelet with pmond
#   - Reload systemd
#

- name: Setup iptables for Kubernetes
  lineinfile:
    path: /etc/sysctl.d/k8s.conf
    line: "{{ item }}"
    create: yes
  with_items:
    - net.bridge.bridge-nf-call-ip6tables = 1
    - net.bridge.bridge-nf-call-iptables = 1
    - net.ipv4.ip_forward = 1
    - net.ipv4.conf.default.rp_filter = 0
    - net.ipv4.conf.all.rp_filter = 0
    - net.ipv6.conf.all.forwarding = 1

- block:
  - block:
    - name: Create daemon.json file for insecure registry
      copy:
        src: "{{ insecure_docker_registry_template }}"
        dest: /etc/docker/daemon.json
        remote_src: yes
        mode: 0644

    - name: Update daemon.json with registry IP
      command: "sed -i -e 's|<%= @insecure_registries %>|\"$DOCKER_REGISTRY_IP\"|g' /etc/docker/daemon.json"
      args:
        warn: false

    - name: Restart docker
      systemd:
        name: docker
        state: restarted
    when: not is_secure_registry

  environment:
    DOCKER_REGISTRY_IP: "{{ docker_registry }}"
  when: use_unified_registry

- name: Update kernel parameters for iptables
  command: sysctl --system &>/dev/null

- name: Create manifests directory required by kubelet
  file:
    path: /etc/kubernetes/manifests
    state: directory
    mode: 0700

- name: Clear pki directory for kubernetes certificates
  file:
    path: "{{ kubeadm_pki_dir }}"
    state: absent

- name: Setup dictionary of kubernetes certificates to install
  set_fact:
    k8s_pki_files: { ca.crt: "{{k8s_root_ca_cert}}", ca.key: "{{k8s_root_ca_key}}" }
  when: (k8s_root_ca_cert)

- block:
  - name: Create pki directory for kubernetes certificates
    file:
      path: "{{ kubeadm_pki_dir }}"
      state: directory
      mode: 0700

  - name: Copy kubernetes certificates
    copy:
      src: "{{ item.value }}"
      dest: "{{ kubeadm_pki_dir }}/{{item.key}}"
    with_dict: "{{ k8s_pki_files }}"

  when: k8s_pki_files is defined

- name: Set kubelet node configuration
  set_fact:
    node_ip: "{{ controller_0_cluster_host }}"

- name: Create kubelet override config file
  template:
    src: "kubelet.conf.j2"
    dest: /etc/sysconfig/kubelet

- name: Enable kubelet
  systemd:
    name: kubelet
    enabled: yes

- name: Create Kube admin yaml
  copy:
    src: "{{ kube_admin_yaml_template }}"
    dest: /etc/kubernetes/kubeadm.yaml
    remote_src: yes

- name: Set loopback ip for kubeadm configuration
  set_fact:
    loopback_ip: "{{ '127.0.0.1' if ipv6_addressing == False else '::1' }}"

- name: Set apiserver SAN list
  set_fact:
    apiserver_cert_list: "{{ [ cluster_floating_address, loopback_ip ] + apiserver_cert_sans }}"

- name: Update Kube admin yaml with network info
  command: "{{ item }}"
  args:
    warn: false
  with_items:
    - "sed -i -e 's|<%= @apiserver_advertise_address %>|'$CLUSTER_IP'|g' /etc/kubernetes/kubeadm.yaml"
    - "sed -i -e 's|<%= @etcd_endpoint %>|'$ETCD_ENDPOINT'|g' /etc/kubernetes/kubeadm.yaml"
    - "sed -i -e 's|<%= @service_domain %>|'cluster.local'|g' /etc/kubernetes/kubeadm.yaml"
    - "sed -i -e 's|<%= @pod_network_cidr %>|'$POD_NETWORK_CIDR'|g' /etc/kubernetes/kubeadm.yaml"
    - "sed -i -e 's|<%= @service_network_cidr %>|'$SERVICE_NETWORK_CIDR'|g' /etc/kubernetes/kubeadm.yaml"
    - "sed -i -e 's|<%= @k8s_registry %>|'$K8S_REGISTRY'|g' /etc/kubernetes/kubeadm.yaml"
  environment:
    CLUSTER_IP: "{{ cluster_floating_address }}"
    ETCD_ENDPOINT: "http://{{ cluster_floating_address | ipwrap }}:2379"
    POD_NETWORK_CIDR: "{{ cluster_pod_subnet }}"
    SERVICE_NETWORK_CIDR: "{{ cluster_service_subnet }}"
    K8S_REGISTRY: "{{ k8s_registry }}"

- name: Add apiserver certificate SANs to kubeadm
  replace:
    path: /etc/kubernetes/kubeadm.yaml
    regexp: "^<% @apiserver_certsans(.*[\n])*?<% end -%>"
    replace: "{{ apiserver_cert_list | to_nice_yaml(width=512) | indent(2, indentfirst=True) }}"

- name: Update Kube admin yaml with OpenID Connect info
  command: "{{ item }}"
  args:
    warn: false
  with_items:
    - "sed -i -e 's|<%= @apiserver_oidc_client_id %>|'$OIDC_CLIENT_ID'|g' /etc/kubernetes/kubeadm.yaml"
    - "sed -i -e 's|<%= @apiserver_oidc_issuer_url %>|'$OIDC_ISSUER_URL'|g' /etc/kubernetes/kubeadm.yaml"
    - "sed -i -e 's|<%= @apiserver_oidc_username_claim %>|'$OIDC_USERNAME_CLAIM'|g' /etc/kubernetes/kubeadm.yaml"
  environment:
    OIDC_CLIENT_ID: "{{ apiserver_oidc.client_id }}"
    OIDC_ISSUER_URL: "{{ apiserver_oidc.issuer_url }}"
    OIDC_USERNAME_CLAIM: "{{ apiserver_oidc.username_claim }}"
  when: apiserver_oidc | length != 0

- name: Delete Kube admin yaml OpenID Connect entries if required config parameters are not present
  command: "{{ item }}"
  args:
    warn: false
  with_items:
    - "sed -i -e '/<%= @apiserver_oidc_client_id %>/d' /etc/kubernetes/kubeadm.yaml"
    - "sed -i -e '/<%= @apiserver_oidc_issuer_url %>/d' /etc/kubernetes/kubeadm.yaml"
    - "sed -i -e '/<%= @apiserver_oidc_username_claim %>/d' /etc/kubernetes/kubeadm.yaml"
  when: apiserver_oidc | length == 0

- name: Initializing Kubernetes master
  command: kubeadm init --config=/etc/kubernetes/kubeadm.yaml

- name: Update kube admin.conf file mode and owner
  file:
    path: /etc/kubernetes/admin.conf
    mode: 0640
    group: sys_protected

- name: Set up k8s environment variable
  copy:
    src: /usr/share/puppet/modules/platform/files/kubeconfig.sh
    dest: /etc/profile.d/kubeconfig.sh
    remote_src: yes

- name: Set Calico cluster configuration
  set_fact:
    cluster_network_ipv4: "{{ cluster_pod_subnet | ipv4 }}"
    cluster_network_ipv6: "{{ cluster_pod_subnet | ipv6 }}"

# Configure calico networking using the Kubernetes API datastore.
- name: Create Calico config file
  template:
    src: "calico-cni.yaml.j2"
    dest: /etc/kubernetes/calico.yaml

- name: Activate Calico Networking
  command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/calico.yaml"

- name: Create Multus config file
  template:
    src: "multus-cni.yaml.j2"
    dest: /etc/kubernetes/multus.yaml

- name: Activate Multus Networking
  command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/multus.yaml"

- name: Create SRIOV Networking config file
  template:
    src: "sriov-cni.yaml.j2"
    dest: /etc/kubernetes/sriov-cni.yaml

- name: Activate SRIOV Networking
  command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/sriov-cni.yaml"

- name: Create SRIOV device plugin config file
  template:
    src: "sriov-plugin.yaml.j2"
    dest: /etc/kubernetes/sriovdp-daemonset.yaml

- name: Activate SRIOV device plugin
  command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/sriovdp-daemonset.yaml"

  # Restrict coredns to master node and use anti-affinity for core dns for duplex systems
- block:
  - name: Restrict coredns to master node
    command: >-
      kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system patch deployment coredns -p
      '{"spec":{"template":{"spec":{"nodeSelector":{"node-role.kubernetes.io/master":""}}}}}'

  - name: Use anti-affinity for coredns pods
    command: >-
      kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system patch deployment coredns -p
      '{"spec":{"template":{"spec":{"affinity":{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchExpressions":[{"key":"k8s-app","operator":"In","values":["kube-dns"]}]},"topologyKey":"kubernetes.io/hostname"}]}}}}}}'
  when: system_mode != 'simplex'

- name: Restrict coredns to 1 pod for simplex
  command: kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system scale --replicas=1 deployment coredns
  when: system_mode == 'simplex'

- name: Remove taint from master node
  shell: "kubectl --kubeconfig=/etc/kubernetes/admin.conf taint node controller-0 node-role.kubernetes.io/master- || true"

- name: Add kubelet service override
  copy:
    src: "{{ kubelet_override_template }}"
    dest: /etc/systemd/system/kubelet.service.d/kube-stx-override.conf
    mode: preserve
    remote_src: yes

- name: Register kubelet with pmond
  copy:
    src: "{{ kubelet_pmond_template }}"
    dest: /etc/pmon.d/kubelet.conf
    mode: preserve
    remote_src: yes

- name: Reload systemd
  command: systemctl daemon-reload

- name: Create persistent certificate directory
  file:
    path: "{{ config_permdir }}/kubernetes/pki/"
    state: directory
    mode: 0700

- name: Copy certificates
  copy:
    src: "{{ kubeadm_pki_dir }}/{{ item }}"
    dest: "{{ config_permdir }}/kubernetes/pki/"
    remote_src: yes
    force: yes
    mode: 0700
  with_items:
    - ca.crt
    - ca.key
    - sa.pub
    - sa.key

- name: Mark Kubernetes config complete
  file:
    path: /etc/platform/.initial_k8s_config_complete
    state: touch