390 lines
14 KiB
YAML
390 lines
14 KiB
YAML
---
|
|
#
|
|
# Copyright (c) 2019 Wind River Systems, Inc.
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
# SUB-TASKS DESCRIPTION:
|
|
# Bring up Kubernetes master
|
|
# - Update iptables
|
|
# - Create daemon.json for insecure unified registry if applicable
|
|
# - Create manifest directory
|
|
# - Enable kubelet service (with default/custom registry)
|
|
# - Run kubeadm init
|
|
# - Prepare admin.conf
|
|
# - Set k8s environment variable for new shell
|
|
# - Prepare Calico config and activate Calico networking
|
|
# - Prepare Multus config and activate Multus networking
|
|
# - Prepare SRIOV config and activate SRIOV networking
|
|
# - Prepare SRIOV device plugin config and activate SRIOV device plugin
|
|
# - Restrict coredns to master node and set anti-affnity (duplex system)
|
|
# - Restrict coredns to 1 pod (simplex system)
|
|
# - Remove taint from master node
|
|
# - Add kubelet service override
|
|
# - Register kubelet with pmond
|
|
# - Reload systemd
|
|
#
|
|
|
|
- name: Setup iptables for Kubernetes
|
|
lineinfile:
|
|
path: /etc/sysctl.d/k8s.conf
|
|
line: "{{ item }}"
|
|
create: yes
|
|
with_items:
|
|
- net.bridge.bridge-nf-call-ip6tables = 1
|
|
- net.bridge.bridge-nf-call-iptables = 1
|
|
- net.ipv4.ip_forward = 1
|
|
- net.ipv4.conf.default.rp_filter = 0
|
|
- net.ipv4.conf.all.rp_filter = 0
|
|
- net.ipv6.conf.all.forwarding = 1
|
|
|
|
- block:
|
|
- block:
|
|
- name: Create daemon.json file for insecure registry
|
|
copy:
|
|
src: "{{ insecure_docker_registry_template }}"
|
|
dest: /etc/docker/daemon.json
|
|
remote_src: yes
|
|
mode: 0644
|
|
|
|
- name: Update daemon.json with registry IP
|
|
command: "sed -i -e 's|<%= @insecure_registries %>|\"$DOCKER_REGISTRY_IP\"|g' /etc/docker/daemon.json"
|
|
args:
|
|
warn: false
|
|
|
|
- name: Restart docker
|
|
systemd:
|
|
name: docker
|
|
state: restarted
|
|
when: not is_secure_registry
|
|
|
|
environment:
|
|
DOCKER_REGISTRY_IP: "{{ docker_registry.url }}"
|
|
when: use_unified_registry
|
|
|
|
- name: Update kernel parameters for iptables
|
|
command: sysctl --system &>/dev/null
|
|
|
|
- name: Create manifests directory required by kubelet
|
|
file:
|
|
path: /etc/kubernetes/manifests
|
|
state: directory
|
|
mode: 0700
|
|
|
|
- name: Clear pki directory for kubernetes certificates
|
|
file:
|
|
path: "{{ kubeadm_pki_dir }}"
|
|
state: absent
|
|
|
|
- name: Setup dictionary of kubernetes certificates to install
|
|
set_fact:
|
|
k8s_pki_files: { ca.crt: "{{k8s_root_ca_cert}}", ca.key: "{{k8s_root_ca_key}}" }
|
|
when: (k8s_root_ca_cert)
|
|
|
|
- block:
|
|
- name: Create pki directory for kubernetes certificates
|
|
file:
|
|
path: "{{ kubeadm_pki_dir }}"
|
|
state: directory
|
|
mode: 0700
|
|
|
|
- name: Copy kubernetes certificates
|
|
copy:
|
|
src: "{{ item.value }}"
|
|
dest: "{{ kubeadm_pki_dir }}/{{item.key}}"
|
|
with_dict: "{{ k8s_pki_files }}"
|
|
|
|
when: k8s_pki_files is defined
|
|
|
|
- name: Set kubelet node configuration
|
|
set_fact:
|
|
node_ip: "{{ controller_0_cluster_host }}"
|
|
|
|
- name: Create kubelet override config file
|
|
template:
|
|
src: "kubelet.conf.j2"
|
|
dest: /etc/sysconfig/kubelet
|
|
|
|
- name: Enable kubelet
|
|
systemd:
|
|
name: kubelet
|
|
enabled: yes
|
|
|
|
- name: Create Kube admin yaml
|
|
copy:
|
|
src: "{{ kube_admin_yaml_template }}"
|
|
dest: /etc/kubernetes/kubeadm.yaml
|
|
remote_src: yes
|
|
|
|
- name: Set loopback ip for kubeadm configuration
|
|
set_fact:
|
|
loopback_ip: "{{ '127.0.0.1' if ipv6_addressing == False else '::1' }}"
|
|
|
|
- name: Set apiserver SAN list
|
|
set_fact:
|
|
apiserver_cert_list: "{{ [ cluster_floating_address, loopback_ip ] + apiserver_cert_sans }}"
|
|
|
|
- name: Update Kube admin yaml with network info
|
|
command: "{{ item }}"
|
|
args:
|
|
warn: false
|
|
with_items:
|
|
- "sed -i -e 's|<%= @apiserver_advertise_address %>|'$CLUSTER_IP'|g' /etc/kubernetes/kubeadm.yaml"
|
|
- "sed -i -e 's|<%= @etcd_endpoint %>|'$ETCD_ENDPOINT'|g' /etc/kubernetes/kubeadm.yaml"
|
|
- "sed -i -e 's|<%= @service_domain %>|'cluster.local'|g' /etc/kubernetes/kubeadm.yaml"
|
|
- "sed -i -e 's|<%= @pod_network_cidr %>|'$POD_NETWORK_CIDR'|g' /etc/kubernetes/kubeadm.yaml"
|
|
- "sed -i -e 's|<%= @service_network_cidr %>|'$SERVICE_NETWORK_CIDR'|g' /etc/kubernetes/kubeadm.yaml"
|
|
- "sed -i -e 's|<%= @k8s_registry %>|'$K8S_REGISTRY'|g' /etc/kubernetes/kubeadm.yaml"
|
|
environment:
|
|
CLUSTER_IP: "{{ cluster_floating_address }}"
|
|
ETCD_ENDPOINT: "http://{{ cluster_floating_address | ipwrap }}:2379"
|
|
POD_NETWORK_CIDR: "{{ cluster_pod_subnet }}"
|
|
SERVICE_NETWORK_CIDR: "{{ cluster_service_subnet }}"
|
|
K8S_REGISTRY: "{{ k8s_registry.url }}"
|
|
|
|
- name: Add apiserver certificate SANs to kubeadm
|
|
replace:
|
|
path: /etc/kubernetes/kubeadm.yaml
|
|
regexp: "^<% @apiserver_certsans(.*[\n])*?<% end -%>"
|
|
replace: "{{ apiserver_cert_list | to_nice_yaml(width=512) | indent(2, indentfirst=True) }}"
|
|
|
|
- name: Update Kube admin yaml with OpenID Connect info
|
|
command: "{{ item }}"
|
|
args:
|
|
warn: false
|
|
with_items:
|
|
- "sed -i -e 's|<%= @apiserver_oidc_client_id %>|'$OIDC_CLIENT_ID'|g' /etc/kubernetes/kubeadm.yaml"
|
|
- "sed -i -e 's|<%= @apiserver_oidc_issuer_url %>|'$OIDC_ISSUER_URL'|g' /etc/kubernetes/kubeadm.yaml"
|
|
- "sed -i -e 's|<%= @apiserver_oidc_username_claim %>|'$OIDC_USERNAME_CLAIM'|g' /etc/kubernetes/kubeadm.yaml"
|
|
environment:
|
|
OIDC_CLIENT_ID: "{{ apiserver_oidc.client_id }}"
|
|
OIDC_ISSUER_URL: "{{ apiserver_oidc.issuer_url }}"
|
|
OIDC_USERNAME_CLAIM: "{{ apiserver_oidc.username_claim }}"
|
|
when: apiserver_oidc | length != 0
|
|
|
|
- name: Delete Kube admin yaml OpenID Connect entries if required config parameters are not present
|
|
command: "{{ item }}"
|
|
args:
|
|
warn: false
|
|
with_items:
|
|
- "sed -i -e '/<%= @apiserver_oidc_client_id %>/d' /etc/kubernetes/kubeadm.yaml"
|
|
- "sed -i -e '/<%= @apiserver_oidc_issuer_url %>/d' /etc/kubernetes/kubeadm.yaml"
|
|
- "sed -i -e '/<%= @apiserver_oidc_username_claim %>/d' /etc/kubernetes/kubeadm.yaml"
|
|
when: apiserver_oidc | length == 0
|
|
|
|
- name: log in to k8s registry if credentials exist
|
|
docker_login:
|
|
registry: "{{ k8s_registry['url'] }}"
|
|
username: "{{ k8s_registry['username'] }}"
|
|
password: "{{ k8s_registry['password'] }}"
|
|
when: k8s_registry.username is defined
|
|
|
|
- name: log in to gcr registry if credentials exist
|
|
docker_login:
|
|
registry: "{{ gcr_registry['url'] }}"
|
|
username: "{{ gcr_registry['username'] }}"
|
|
password: "{{ gcr_registry['password'] }}"
|
|
when: gcr_registry.username is defined
|
|
|
|
- name: log in to quay registry if credentials exist
|
|
docker_login:
|
|
registry: "{{ quay_registry['url'] }}"
|
|
username: "{{ quay_registry['username'] }}"
|
|
password: "{{ quay_registry['password'] }}"
|
|
when: quay_registry.username is defined
|
|
|
|
- name: log in to docker registry if credentials exist
|
|
docker_login:
|
|
registry: "{{ docker_registry['url'] }}"
|
|
username: "{{ docker_registry['username'] }}"
|
|
password: "{{ docker_registry['password'] }}"
|
|
when: docker_registry.username is defined
|
|
|
|
- name: prepull kubernetes images
|
|
command: kubeadm config images pull --config=/etc/kubernetes/kubeadm.yaml
|
|
|
|
- name: log out of k8s registry if credentials exist
|
|
docker_login:
|
|
registry: "{{ k8s_registry['url'] }}"
|
|
state: absent
|
|
when: k8s_registry.username is defined
|
|
|
|
- name: log out of gcr registry if credentials exist
|
|
docker_login:
|
|
registry: "{{ gcr_registry['url'] }}"
|
|
state: absent
|
|
when: gcr_registry.username is defined
|
|
|
|
- name: log out of quay registry if credentials exist
|
|
docker_login:
|
|
registry: "{{ quay_registry['url'] }}"
|
|
state: absent
|
|
when: quay_registry.username is defined
|
|
|
|
- name: log out of docker registry if credentials exist
|
|
docker_login:
|
|
registry: "{{ docker_registry['url'] }}"
|
|
state: absent
|
|
when: docker_registry.username is defined
|
|
|
|
- name: Initializing Kubernetes master
|
|
command: kubeadm init --config=/etc/kubernetes/kubeadm.yaml
|
|
|
|
- name: Update kube admin.conf file mode and owner
|
|
file:
|
|
path: /etc/kubernetes/admin.conf
|
|
mode: 0640
|
|
group: sys_protected
|
|
|
|
- name: Set up k8s environment variable
|
|
copy:
|
|
src: /usr/share/puppet/modules/platform/files/kubeconfig.sh
|
|
dest: /etc/profile.d/kubeconfig.sh
|
|
remote_src: yes
|
|
|
|
- name: Patch pull secret into kube-proxy service account
|
|
command: >
|
|
kubectl --kubeconfig=/etc/kubernetes/admin.conf patch serviceaccount
|
|
kube-proxy -p '{"imagePullSecrets": [{"name": "k8s-registry-secret"}]}' -n kube-system
|
|
|
|
- name: Find old Kubernetes registry secrets
|
|
shell: "{{ item }}"
|
|
with_items:
|
|
- "kubectl --kubeconfig=/etc/kubernetes/admin.conf get secrets -n kube-system |
|
|
grep k8s-registry-secret | awk '{print $1}'"
|
|
- "kubectl --kubeconfig=/etc/kubernetes/admin.conf get secrets -n kube-system |
|
|
grep gcr-registry-secret | awk '{print $1}'"
|
|
- "kubectl --kubeconfig=/etc/kubernetes/admin.conf get secrets -n kube-system |
|
|
grep quay-registry-secret | awk '{print $1}'"
|
|
- "kubectl --kubeconfig=/etc/kubernetes/admin.conf get secrets -n kube-system |
|
|
grep docker-registry-secret | awk '{print $1}'"
|
|
register: old_kubernetes_secrets
|
|
|
|
- name: Delete old Kubernetes registry secrets
|
|
shell: "kubectl --kubeconfig=/etc/kubernetes/admin.conf delete secret -n kube-system {{ item }}"
|
|
with_items:
|
|
- "{{ old_kubernetes_secrets.results | map(attribute='stdout_lines') | flatten }}"
|
|
|
|
- name: Create k8s registry pull secret
|
|
command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf create secret docker-registry k8s-registry-secret
|
|
--docker-server={{ k8s_registry['url'] }} --docker-username={{ k8s_registry['username'] }}
|
|
--docker-password={{ k8s_registry['password'] }} -n kube-system"
|
|
when: k8s_registry['username'] is defined
|
|
|
|
- name: Create gcr registry pull secret
|
|
command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf create secret docker-registry gcr-registry-secret
|
|
--docker-server={{ gcr_registry['url'] }} --docker-username={{ gcr_registry['username'] }}
|
|
--docker-password={{ gcr_registry['password'] }} -n kube-system"
|
|
when: gcr_registry['username'] is defined
|
|
|
|
- name: Create quay registry pull secret
|
|
command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf create secret docker-registry quay-registry-secret
|
|
--docker-server={{ quay_registry['url'] }} --docker-username={{ quay_registry['username'] }}
|
|
--docker-password={{ quay_registry['password'] }} -n kube-system"
|
|
when: quay_registry['username'] is defined
|
|
|
|
- name: Create docker registry pull secret
|
|
command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf create secret docker-registry docker-registry-secret
|
|
--docker-server={{ docker_registry['url'] }} --docker-username={{ docker_registry['username'] }}
|
|
--docker-password={{ docker_registry['password'] }} -n kube-system"
|
|
when: docker_registry['username'] is defined
|
|
|
|
- name: Set Calico cluster configuration
|
|
set_fact:
|
|
cluster_network_ipv4: "{{ cluster_pod_subnet | ipv4 }}"
|
|
cluster_network_ipv6: "{{ cluster_pod_subnet | ipv6 }}"
|
|
|
|
# Configure calico networking using the Kubernetes API datastore.
|
|
- name: Create Calico config file
|
|
template:
|
|
src: "calico-cni.yaml.j2"
|
|
dest: /etc/kubernetes/calico.yaml
|
|
|
|
- name: Activate Calico Networking
|
|
command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/calico.yaml"
|
|
|
|
- name: Create Multus config file
|
|
template:
|
|
src: "multus-cni.yaml.j2"
|
|
dest: /etc/kubernetes/multus.yaml
|
|
|
|
- name: Activate Multus Networking
|
|
command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/multus.yaml"
|
|
|
|
- name: Create SRIOV Networking config file
|
|
template:
|
|
src: "sriov-cni.yaml.j2"
|
|
dest: /etc/kubernetes/sriov-cni.yaml
|
|
|
|
- name: Activate SRIOV Networking
|
|
command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/sriov-cni.yaml"
|
|
|
|
- name: Create SRIOV device plugin config file
|
|
template:
|
|
src: "sriov-plugin.yaml.j2"
|
|
dest: /etc/kubernetes/sriovdp-daemonset.yaml
|
|
|
|
- name: Activate SRIOV device plugin
|
|
command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/sriovdp-daemonset.yaml"
|
|
|
|
# Restrict coredns to master node and use anti-affinity for core dns for duplex systems
|
|
- block:
|
|
- name: Restrict coredns to master node
|
|
command: >-
|
|
kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system patch deployment coredns -p
|
|
'{"spec":{"template":{"spec":{"nodeSelector":{"node-role.kubernetes.io/master":""}}}}}'
|
|
|
|
- name: Use anti-affinity for coredns pods
|
|
command: >-
|
|
kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system patch deployment coredns -p
|
|
'{"spec":{"template":{"spec":{"affinity":{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchExpressions":[{"key":"k8s-app","operator":"In","values":["kube-dns"]}]},"topologyKey":"kubernetes.io/hostname"}]}}}}}}'
|
|
when: system_mode != 'simplex'
|
|
|
|
- name: Restrict coredns to 1 pod for simplex
|
|
command: kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system scale --replicas=1 deployment coredns
|
|
when: system_mode == 'simplex'
|
|
|
|
- name: Remove taint from master node
|
|
shell: "kubectl --kubeconfig=/etc/kubernetes/admin.conf taint node controller-0 node-role.kubernetes.io/master- || true"
|
|
|
|
- name: Add kubelet service override
|
|
copy:
|
|
src: "{{ kubelet_override_template }}"
|
|
dest: /etc/systemd/system/kubelet.service.d/kube-stx-override.conf
|
|
mode: preserve
|
|
remote_src: yes
|
|
|
|
- name: Register kubelet with pmond
|
|
copy:
|
|
src: "{{ kubelet_pmond_template }}"
|
|
dest: /etc/pmon.d/kubelet.conf
|
|
mode: preserve
|
|
remote_src: yes
|
|
|
|
- name: Reload systemd
|
|
command: systemctl daemon-reload
|
|
|
|
- name: Create persistent certificate directory
|
|
file:
|
|
path: "{{ config_permdir }}/kubernetes/pki/"
|
|
state: directory
|
|
mode: 0700
|
|
|
|
- name: Copy certificates
|
|
copy:
|
|
src: "{{ kubeadm_pki_dir }}/{{ item }}"
|
|
dest: "{{ config_permdir }}/kubernetes/pki/"
|
|
remote_src: yes
|
|
force: yes
|
|
mode: 0700
|
|
with_items:
|
|
- ca.crt
|
|
- ca.key
|
|
- sa.pub
|
|
- sa.key
|
|
|
|
- name: Mark Kubernetes config complete
|
|
file:
|
|
path: /etc/platform/.initial_k8s_config_complete
|
|
state: touch
|