From 1ecac43e0d0868d6e7467e48446cb75086bb2c6f Mon Sep 17 00:00:00 2001 From: Andre Kantek Date: Tue, 4 Jun 2024 15:09:46 -0300 Subject: [PATCH] Kubernetes Dual-Stack Runtime Configuration New classes and scripts are added to handle sm, kubelet, kubeadm, kube-proxy, calico, multus, and certsans. New variables exported to the hiera are used in this change, in conjunction with the existing ones: platform::kubernetes::params::node_ip_secondary platform::network::cluster_pod::params::* platform::network::cluster_pod::ipv4::params:: platform::network::cluster_pod::ipv6::params::* platform::network::cluster_service::params::* platform::network::cluster_service::ipv4::params:: platform::network::cluster_service::ipv6::params::* Test Plan: ========== In all tests pods were brought up and the datapath was validated directly and through a service [PASS] in AIO-DX IPv4, configure dual-stack and back to single-stack [PASS] in AIO-DX IPv6, configure dual-stack and back to single-stack [PASS] in AIO-SX IPv4, configure dual-stack and back to single-stack [PASS] in AIO-SX IPv6, configure dual-stack and back to single-stack [PASS] in Standard IPv4, configure dual-stack and back to single-stack [PASS] in Standard IPv6, configure dual-stack and back to single-stack Story: 2011027 Task: 50203 Change-Id: Ifb908c097960f90c5eabeca8cc02d2f60ae4d731 Signed-off-by: Andre Kantek --- puppet-manifests/src/Makefile | 5 + puppet-manifests/src/bin/dual-stack-calico.py | 188 ++++++++ .../src/bin/dual-stack-kubeadm.py | 195 ++++++++ .../src/bin/dual-stack-kubelet.py | 141 ++++++ .../src/bin/dual-stack-kubeproxy.py | 130 ++++++ puppet-manifests/src/bin/dual-stack-multus.py | 112 +++++ .../modules/platform/manifests/kubernetes.pp | 420 +++++++++++++++++- .../src/modules/platform/manifests/network.pp | 110 +++++ .../src/modules/platform/manifests/sm.pp | 81 ++++ .../templates/callico_ippool.yaml.erb | 11 + .../platform/templates/kubelet.conf.erb | 5 + test-requirements.txt | 1 + 12 files changed, 1387 insertions(+), 12 deletions(-) create mode 100644 puppet-manifests/src/bin/dual-stack-calico.py create mode 100644 puppet-manifests/src/bin/dual-stack-kubeadm.py create mode 100644 puppet-manifests/src/bin/dual-stack-kubelet.py create mode 100644 puppet-manifests/src/bin/dual-stack-kubeproxy.py create mode 100644 puppet-manifests/src/bin/dual-stack-multus.py create mode 100644 puppet-manifests/src/modules/platform/templates/callico_ippool.yaml.erb diff --git a/puppet-manifests/src/Makefile b/puppet-manifests/src/Makefile index cd4d7225b..6148ff0b5 100644 --- a/puppet-manifests/src/Makefile +++ b/puppet-manifests/src/Makefile @@ -27,6 +27,11 @@ endif install -m 755 -D bin/kubelet-cleanup-orphaned-volumes.sh $(BINDIR)/kubelet-cleanup-orphaned-volumes.sh install -m 755 -D bin/check_ipv6_tentative_addresses.py $(BINDIR)/check_ipv6_tentative_addresses.py install -m 755 -D bin/manage_partitions_pre_script.sh $(BINDIR)/manage_partitions_pre_script.sh + install -m 755 -D bin/dual-stack-kubelet.py $(BINDIR)/dual-stack-kubelet.py + install -m 755 -D bin/dual-stack-kubeadm.py $(BINDIR)/dual-stack-kubeadm.py + install -m 755 -D bin/dual-stack-kubeproxy.py $(BINDIR)/dual-stack-kubeproxy.py + install -m 755 -D bin/dual-stack-calico.py $(BINDIR)/dual-stack-calico.py + install -m 755 -D bin/dual-stack-multus.py $(BINDIR)/dual-stack-multus.py install -d -m 0755 $(CONFIGDIR) ifdef hiera_v5 diff --git a/puppet-manifests/src/bin/dual-stack-calico.py b/puppet-manifests/src/bin/dual-stack-calico.py new file mode 100644 index 000000000..1d92a87a9 --- /dev/null +++ b/puppet-manifests/src/bin/dual-stack-calico.py @@ -0,0 +1,188 @@ +#!/usr/bin/python3 +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +''' This script updates the calico config to handle single or dual-stack +''' + +import sys +import subprocess +import yaml +import time +import re +import netaddr + +from datetime import datetime + + +calico_config_map_file = "/tmp/calico-configmap.yaml" +calico_daemonset_file = "/tmp/calico-daemonset.yaml" + +kubectl_config = "--kubeconfig=/etc/kubernetes/admin.conf" + + +def get_yaml_data(cmd, attempts=15): + data = dict() + for i in range(0, attempts): + res = subprocess.run(cmd, check=False, stdout=subprocess.PIPE) + if res.returncode != 0: + if i == attempts: + print(f"An error occurred getting data, attempt={i}: {res}") + sys.exit(1) + else: + time.sleep(5) + else: + print("yaml data was collected") + data = yaml.load(res.stdout, Loader=yaml.Loader) + break + return data + + +def is_valid_ip(address): + try: + if netaddr.valid_ipv4(address): + return True + if netaddr.valid_ipv6(address): + return True + except netaddr.AddrFormatError: + pass + return False + + +def prepend_timestamp_line(file_name): + timestamp_str = datetime.now().strftime(format="%Y-%m-%d %H:%M:%S") + with open(file_name, 'r') as read_file: + lines = read_file.readlines() + lines.insert(0, f"# generated at {timestamp_str}" + "\n") # Add newline character + with open(file_name, 'w') as write_file: + write_file.writelines(lines) + + +def save_configmap(cmd, config_map_file): + + for i in range(0, 9): + res = subprocess.run(cmd, check=False, capture_output=True) + if res.returncode == 0: + output = res.stdout.decode() + with open(config_map_file, "wb") as output_file: + output_file.write(output.encode()) + print(f"Successfully saved configmap to {config_map_file}") + break + else: + if i == 9: + print(f"An error occurred getting data, attempt={i}: {res}") + sys.exit(1) + else: + time.sleep(5) + + +if __name__ == "__main__": + if len(sys.argv) < 5: + print("Usage: dual-stack-calico.py ") + sys.exit(1) + + protocol = sys.argv[1].lower() + if protocol not in ["ipv4", "ipv6"]: + print("invalid IP protocol") + sys.exit(1) + + state = sys.argv[2].lower() + if state not in ["true", "false"]: + print("invalid state for protocol") + sys.exit(1) + + c0_address = sys.argv[3] + if not is_valid_ip(c0_address): + if state == "true": + print(f"Error: invalid node_ip '{c0_address}', exit") + sys.exit(1) + else: + c0_address = None + + wait = 0 + try: + wait = int(sys.argv[4]) + except ValueError: + print(f"Error: restart_wait='{sys.argv[4]}' cannot be converted to an integer.") + sys.exit(1) + + print(f"dual-stack-calico {protocol} {state} {c0_address} {wait}") + + print("execute: kubectl get cm -n kube-system calico-config -o yaml") + command = ["kubectl", kubectl_config, "-n", "kube-system", + "get", "cm", "calico-config", "-o", "yaml"] + save_configmap(command, calico_config_map_file) + + modified_data = str() + with open(calico_config_map_file, "r") as file: + calico_config_map = file.read() + if protocol == "ipv4": + match = re.search(r'"assign_ipv4": "(true|false)"', calico_config_map) + if match: + new_val = f'\"assign_ipv4\": \"{state}\"' + modified_data = calico_config_map.replace(match.group(), new_val) + elif protocol == "ipv6": + match = re.search(r'"assign_ipv6": "(true|false)"', calico_config_map) + if match: + new_val = f'\"assign_ipv6\": \"{state}\"' + modified_data = calico_config_map.replace(match.group(), new_val) + + with open(calico_config_map_file, "w") as file: + file.write(modified_data) + prepend_timestamp_line(calico_config_map_file) + + print(f"execute: kubectl apply -f {calico_config_map_file}") + result = subprocess.run(["kubectl", kubectl_config, "apply", "-f", calico_config_map_file], + check=False, stdout=subprocess.PIPE) + print(f"update calico configmap result={result}") + if result.returncode != 0: + sys.exit(1) + + print("execute: kubectl -n kube-system get daemonset calico-node -o yaml") + command = ["kubectl", kubectl_config, "-n", "kube-system", + "get", "daemonset", "calico-node", "-o", "yaml"] + calico_ds_data = get_yaml_data(command, 9) + + for container in calico_ds_data['spec']['template']['spec']['containers']: + ipv4_autodetect = False + ipv6_autodetect = False + for env in container['env']: + if protocol == "ipv4": + if env["name"] == "IP": + env["value"] = "autodetect" if state == "true" else "none" + if env["name"] == "IP_AUTODETECTION_METHOD": + ipv4_autodetect = True + if protocol == "ipv6": + if env["name"] == "IP6": + env["value"] = "autodetect" if state == "true" else "none" + if env["name"] == "IP6_AUTODETECTION_METHOD": + ipv6_autodetect = True + if not ipv4_autodetect and protocol == "ipv4" and state == "true": + container['env'].append({"name": "IP_AUTODETECTION_METHOD", + "value": f"can-reach={c0_address}"}) + if not ipv6_autodetect and protocol == "ipv6" and state == "true": + container['env'].append({"name": "IP6_AUTODETECTION_METHOD", + "value": f"can-reach={c0_address}"}) + if ipv4_autodetect and protocol == "ipv4" and state == "false": + container['env'] = [env for env in container['env'] + if not env["name"] == "IP_AUTODETECTION_METHOD"] + if ipv6_autodetect and protocol == "ipv6" and state == "false": + container['env'] = [env for env in container['env'] + if not env["name"] == "IP6_AUTODETECTION_METHOD"] + + with open(calico_daemonset_file, 'w') as config_file: + yaml.dump(calico_ds_data, config_file, default_flow_style=False) + prepend_timestamp_line(calico_daemonset_file) + + print(f"execute: kubectl apply -f {calico_daemonset_file}") + result = subprocess.run(["kubectl", kubectl_config, "apply", "-f", calico_daemonset_file], + check=False, stdout=subprocess.PIPE) + print(f"update calico daemonset result={result}") + if result.returncode != 0: + sys.exit(1) + +print(f"wait {wait} seconds for the restarts") +time.sleep(wait) +sys.exit(0) diff --git a/puppet-manifests/src/bin/dual-stack-kubeadm.py b/puppet-manifests/src/bin/dual-stack-kubeadm.py new file mode 100644 index 000000000..3dddcd6e4 --- /dev/null +++ b/puppet-manifests/src/bin/dual-stack-kubeadm.py @@ -0,0 +1,195 @@ +#!/usr/bin/python3 +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +''' This script updates the kubeadm config to handle single or dual-stack +''' + +import sys +import subprocess +import yaml +import time +import os +import netaddr + +from datetime import datetime + + +config_map_file = "/tmp/kubeadm-config.yaml" +cluster_config_file = "/tmp/kubeadm-config-cluster.yaml" +kubectl_config = "--kubeconfig=/etc/kubernetes/admin.conf" +active_controller_puppet_path = '/opt/platform/puppet/' +INITCONFIG_TEMPLATE = '''--- +apiVersion: kubeadm.k8s.io/v1beta3 +kind: InitConfiguration +localAPIEndpoint: + advertiseAddress: {}''' + + +def get_yaml_data(cmd, attempts=15): + data = dict() + for i in range(0, attempts): + res = subprocess.run(cmd, check=False, stdout=subprocess.PIPE) + if res.returncode != 0: + if i == attempts: + print(f"An error occurred getting data, attempt={i}: {res}") + sys.exit(1) + else: + time.sleep(5) + else: + print("yaml data was collected") + data = yaml.load(res.stdout, Loader=yaml.Loader) + break + return data + + +def is_valid_ip(address): + try: + if netaddr.valid_ipv4(address): + return True + if netaddr.valid_ipv6(address): + return True + except netaddr.AddrFormatError: + pass + return False + + +def prepend_timestamp_line(file_name): + timestamp_str = datetime.now().strftime(format="%Y-%m-%d %H:%M:%S") + with open(file_name, 'r') as read_file: + lines = read_file.readlines() + lines.insert(0, f"# generated at {timestamp_str}" + "\n") # Add newline character + with open(file_name, 'w') as write_file: + write_file.writelines(lines) + + +def is_valid_network(address): + """ + This function checks if the provided string is a valid network address using netaddr. + """ + try: + netaddr.IPNetwork(address) + return True + except netaddr.AddrFormatError as ex: + print(f"exception {str(ex)}") + return False + + +if __name__ == "__main__": + if len(sys.argv) < 7: + print("Usage: dual-stack-kubeadm.py " + " " + " ") + sys.exit(1) + + pod_prim_subnet = sys.argv[1] + svc_prim_subnet = sys.argv[2] + pod_sec_subnet = sys.argv[3] + svc_sec_subnet = sys.argv[4] + advertise_address = sys.argv[6] + + if not is_valid_network(pod_prim_subnet): + print(f"Error: invalid pod_prim_subnet '{pod_prim_subnet}', exit") + sys.exit(1) + + if not is_valid_network(svc_prim_subnet): + print(f"Error: invalid svc_prim_subnet '{svc_prim_subnet}', exit") + sys.exit(1) + + if not is_valid_ip(advertise_address): + print(f"Error: invalid advertise_address '{advertise_address}', exit") + sys.exit(1) + + if not is_valid_network(pod_sec_subnet): + pod_sec_subnet = None + + if not is_valid_network(svc_sec_subnet): + svc_sec_subnet = None + + wait = 0 + try: + wait = int(sys.argv[5]) + except ValueError: + print(f"Error: restart_wait='{sys.argv[5]}' cannot be converted to an integer.") + sys.exit(1) + + print(f"dual-stack-kubeadm {pod_prim_subnet} {svc_prim_subnet}" + f" {pod_sec_subnet} {svc_sec_subnet} {wait}") + + print("execute: kubectl -n kube-system get configmap kubeadm-config -o yaml") + command = ["kubectl", kubectl_config, "-n", "kube-system", + "get", "configmap", "kubeadm-config", "-o", "yaml"] + yaml_data = get_yaml_data(command, 9) + cluster_cfg_yaml = yaml_data["data"]["ClusterConfiguration"] + cluster_cfg = yaml.load(cluster_cfg_yaml, Loader=yaml.Loader) + configmap_reconfig = False + if pod_prim_subnet and pod_sec_subnet: + if cluster_cfg["networking"]["podSubnet"] != f"{pod_prim_subnet},{pod_sec_subnet}": + cluster_cfg["networking"]["podSubnet"] = f"{pod_prim_subnet},{pod_sec_subnet}" + configmap_reconfig = True + elif pod_prim_subnet and not pod_sec_subnet: + if cluster_cfg["networking"]["podSubnet"] != f"{pod_prim_subnet}": + cluster_cfg["networking"]["podSubnet"] = f"{pod_prim_subnet}" + configmap_reconfig = True + if svc_prim_subnet and svc_sec_subnet: + if cluster_cfg["networking"]["serviceSubnet"] != f"{svc_prim_subnet},{svc_sec_subnet}": + cluster_cfg["networking"]["serviceSubnet"] = f"{svc_prim_subnet},{svc_sec_subnet}" + configmap_reconfig = True + elif svc_prim_subnet and not svc_sec_subnet: + if cluster_cfg["networking"]["serviceSubnet"] != f"{svc_prim_subnet}": + cluster_cfg["networking"]["serviceSubnet"] = f"{svc_prim_subnet}" + configmap_reconfig = True + + with open(cluster_config_file, 'w') as config_file: + yaml.dump(cluster_cfg, config_file, default_flow_style=False) + + cluster_config_str = str() + with open(cluster_config_file, "r") as file: + cluster_config_lines = file.readlines() + for line in cluster_config_lines: + cluster_config_str = cluster_config_str + line + yaml_data["data"]["ClusterConfiguration"] = cluster_config_str + + with open(config_map_file, 'w') as config_file: + yaml.dump(yaml_data, config_file, default_flow_style=False) + prepend_timestamp_line(config_map_file) + + if configmap_reconfig: + if os.path.exists(active_controller_puppet_path): + print(f"execute: kubectl apply -f {config_map_file}") + result = subprocess.run(["kubectl", kubectl_config, "apply", "-f", config_map_file], + check=False, stdout=subprocess.PIPE) + print(f"update configmap result={result}") + if result.returncode != 0: + sys.exit(1) + else: + print("configmap kubeadm-config already updated") + + print("execute: kubeadm init phase control-plane controller-manager --config" + f" {cluster_config_file}") + result = subprocess.run(["kubeadm", "init", "phase", "control-plane", "controller-manager", + "--config", cluster_config_file], + check=False, stdout=subprocess.PIPE) + print(result) + if result.returncode != 0: + sys.exit(1) + + with open(cluster_config_file, 'a') as file: + file.write(INITCONFIG_TEMPLATE.format(advertise_address)) + prepend_timestamp_line(cluster_config_file) + + # kubeadm init phase control-plane apiserver --config /tmp/kubeadm-config-cluster.yaml + print(f"execute: kubeadm init phase control-plane apiserver --config {cluster_config_file}" + f" --apiserver-advertise-address {advertise_address}") + result = subprocess.run(["kubeadm", "init", "phase", "control-plane", "apiserver", + "--config", cluster_config_file], + check=False, stdout=subprocess.PIPE) + print(result) + if result.returncode != 0: + sys.exit(1) + +print(f"wait {wait} seconds for the restarts") +time.sleep(wait) +sys.exit(0) diff --git a/puppet-manifests/src/bin/dual-stack-kubelet.py b/puppet-manifests/src/bin/dual-stack-kubelet.py new file mode 100644 index 000000000..5e9708303 --- /dev/null +++ b/puppet-manifests/src/bin/dual-stack-kubelet.py @@ -0,0 +1,141 @@ +#!/usr/bin/python3 +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +''' This script updates the kubelet config to handle single or dual-stack +''' +import sys +import subprocess +import time +import yaml +import netaddr + + +filename = "/etc/default/kubelet" + + +def get_yaml_data(cmd, attempts=15): + data = dict() + for i in range(0, attempts): + res = subprocess.run(cmd, check=False, stdout=subprocess.PIPE) + if res.returncode != 0: + if i == attempts: + print(f"An error occurred getting data, attempt={i}: {res}") + sys.exit(1) + else: + time.sleep(5) + else: + print("yaml data was collected") + data = yaml.load(res.stdout, Loader=yaml.Loader) + break + return data + + +def is_valid_ip(address): + try: + if netaddr.valid_ipv4(address): + return True + if netaddr.valid_ipv6(address): + return True + except netaddr.AddrFormatError: + pass + return False + + +if __name__ == "__main__": + if len(sys.argv) < 5: + print("Usage: dual-stack-kubelet.py " + " ") + sys.exit(1) + + node_ip = sys.argv[1] + node_ip_secondary = sys.argv[2] + + if not is_valid_ip(node_ip): + print(f"Error: invalid node_ip '{node_ip}', exit") + sys.exit(1) + + if not is_valid_ip(node_ip_secondary): + node_ip_secondary = None + + wait = 0 + try: + wait = int(sys.argv[3]) + except ValueError: + print(f"Error: restart_wait='{sys.argv[3]}' cannot be converted to an integer.") + sys.exit(1) + + kubectl_config = '' + if sys.argv[4]: + kubectl_config = f"--kubeconfig={sys.argv[4]}" + else: + print(f"Error: invalid kubectl config='{kubectl_config}'") + sys.exit(1) + + print(f"dual-stack-kubelet {node_ip} {node_ip_secondary} {wait} {kubectl_config}") + + # execute get to test availability of kube-api server + print("execute: kubectl -n kube-system get configmap kubeadm-config -o yaml") + command = ["kubectl", kubectl_config, "-n", "kube-system", + "get", "configmap", "kubeadm-config", "-o", "yaml"] + get_yaml_data(command, 20) + + try: + # Open the file for reading + with open(filename, "r") as file: + # Read all lines from the file + lines = file.readlines() + except FileNotFoundError: + print(f"Error: File '{filename}' not found.") + sys.exit(1) + # Check for empty line + if not lines: + print(f"filename {filename} is empty") + sys.exit(1) + line = str() + for value in lines: + if "KUBELET_EXTRA_ARGS" in value: + line = value + if not line: + print(f"filename {filename} do not contain KUBELET_EXTRA_ARGS") + sys.exit(1) + kubelet = line.split('=', 1) + kubelet_args = kubelet[1].split() + need_reconfig = False + modified_strings = [] + for arg in kubelet_args: + if "--node-ip" in arg: + if node_ip_secondary and arg != f"--node-ip={node_ip},{node_ip_secondary}": + modified_strings.append(f"--node-ip={node_ip},{node_ip_secondary}") + need_reconfig = True + elif not node_ip_secondary and arg != f"--node-ip={node_ip}": + modified_strings.append(f"--node-ip={node_ip}") + need_reconfig = True + else: + modified_strings.append(arg) + print(f"need_reconfig={need_reconfig}") + if (need_reconfig): + kubelet_args = modified_strings + output_kubelet = str("# Overrides config file for kubelet\nKUBELET_EXTRA_ARGS=") + output_kubelet += ' '.join(kubelet_args) + "\n" + print(output_kubelet) + with open(filename, "w") as file: + file.writelines(output_kubelet) + print("execute: kubeadm upgrade node phase kubelet-config") + result = subprocess.run(['kubeadm', 'upgrade', 'node', 'phase', 'kubelet-config'], + check=False, stdout=subprocess.PIPE) + print(result) + if result.returncode != 0: + sys.exit(1) + print("execute: /usr/local/sbin/pmon-restart kubelet") + result = subprocess.run(['/usr/local/sbin/pmon-restart', 'kubelet'], + check=False, stdout=subprocess.PIPE) + print(result) + if result.returncode != 0: + sys.exit(1) + +print(f"wait {wait} seconds for the restarts") +time.sleep(wait) +sys.exit(0) diff --git a/puppet-manifests/src/bin/dual-stack-kubeproxy.py b/puppet-manifests/src/bin/dual-stack-kubeproxy.py new file mode 100644 index 000000000..6ad605ed4 --- /dev/null +++ b/puppet-manifests/src/bin/dual-stack-kubeproxy.py @@ -0,0 +1,130 @@ +#!/usr/bin/python3 +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +''' This script updates the kube-proxy config to handle single or dual-stack +''' + +import sys +import subprocess +import yaml +import time +import netaddr + +from datetime import datetime + + +config_map_file = "/tmp/kube-proxy-config.yaml" +proxy_config_file = "/tmp/kube-proxy-config-data.yaml" + +kubectl_config = "--kubeconfig=/etc/kubernetes/admin.conf" + + +def get_yaml_data(cmd, attempts=15): + data = dict() + for i in range(0, attempts): + res = subprocess.run(cmd, check=False, stdout=subprocess.PIPE) + if res.returncode != 0: + if i == attempts: + print(f"An error occurred getting data, attempt={i}: {res}") + sys.exit(1) + else: + time.sleep(5) + else: + print("yaml data was collected") + data = yaml.load(res.stdout, Loader=yaml.Loader) + break + return data + + +def prepend_timestamp_line(file_name): + timestamp_str = datetime.now().strftime(format="%Y-%m-%d %H:%M:%S") + with open(file_name, 'r') as read_file: + lines = read_file.readlines() + lines.insert(0, f"# generated at {timestamp_str}" + "\n") # Add newline character + with open(file_name, 'w') as write_file: + write_file.writelines(lines) + + +def is_valid_network(address): + """ + This function checks if the provided string is a valid network address using netaddr. + """ + try: + netaddr.IPNetwork(address) + return True + except netaddr.AddrFormatError as ex: + print(f"exception {str(ex)}") + return False + + +if __name__ == "__main__": + if len(sys.argv) < 4: + print("Usage: dual-stack-kubeproxy.py ") + sys.exit(1) + + pod_prim_subnet = sys.argv[1] + pod_sec_subnet = sys.argv[2] + + if not is_valid_network(pod_prim_subnet): + print(f"Error: invalid pod_prim_subnet '{pod_prim_subnet}', exit") + sys.exit(1) + + if not is_valid_network(pod_sec_subnet): + pod_sec_subnet = None + + wait = 0 + try: + wait = int(sys.argv[3]) + except ValueError: + print(f"Error: restart_wait='{sys.argv[3]}' cannot be converted to an integer.") + sys.exit(1) + + print(f"dual-stack-kubeproxy {pod_prim_subnet} {pod_sec_subnet} {wait}") + + print("execute: kubectl -n kube-system get configmap kube-proxy -o yaml") + command = ["kubectl", kubectl_config, "-n", "kube-system", + "get", "configmap", "kube-proxy", "-o", "yaml"] + yaml_data = get_yaml_data(command, 15) + proxy_cfg_yaml = yaml_data["data"]["config.conf"] + proxy_cfg = yaml.load(proxy_cfg_yaml, Loader=yaml.Loader) + if pod_prim_subnet and pod_sec_subnet: + proxy_cfg["clusterCIDR"] = f"{pod_prim_subnet},{pod_sec_subnet}" + elif pod_prim_subnet and not pod_sec_subnet: + proxy_cfg["clusterCIDR"] = f"{pod_prim_subnet}" + + with open(proxy_config_file, 'w') as config_file: + yaml.dump(proxy_cfg, config_file, default_flow_style=False) + + proxy_config_str = str() + with open(proxy_config_file, "r") as file: + proxy_config_lines = file.readlines() + for line in proxy_config_lines: + proxy_config_str = proxy_config_str + line + yaml_data["data"]["config.conf"] = proxy_config_str + prepend_timestamp_line(proxy_config_file) + + with open(config_map_file, 'w') as config_file: + yaml.dump(yaml_data, config_file, default_flow_style=False) + prepend_timestamp_line(config_map_file) + + print(f"execute: kubectl apply -f {config_map_file}") + result = subprocess.run(["kubectl", kubectl_config, "apply", "-f", config_map_file], + check=False, stdout=subprocess.PIPE) + print(f"update configmap result={result}") + if result.returncode != 0: + sys.exit(1) + + print("execute: kubectl rollout restart daemonset -n kube-system kube-proxy") + result = subprocess.run(["kubectl", kubectl_config, "-n", "kube-system", + "rollout", "restart", "daemonset", "kube-proxy"], + check=False, stdout=subprocess.PIPE) + print(result) + if result.returncode != 0: + sys.exit(1) + +print(f"wait {wait} seconds for the restarts") +time.sleep(wait) +sys.exit(0) diff --git a/puppet-manifests/src/bin/dual-stack-multus.py b/puppet-manifests/src/bin/dual-stack-multus.py new file mode 100644 index 000000000..15cf38c21 --- /dev/null +++ b/puppet-manifests/src/bin/dual-stack-multus.py @@ -0,0 +1,112 @@ +#!/usr/bin/python3 +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +''' This script updates the multus config to handle single or dual-stack +''' + +import sys +import subprocess +import time +import re +from datetime import datetime + +multus_config_map_file = "/tmp/multus-configmap.yaml" +kubectl_config = "--kubeconfig=/etc/kubernetes/admin.conf" + + +def prepend_timestamp_line(file_name): + timestamp_str = datetime.now().strftime(format="%Y-%m-%d %H:%M:%S") + with open(file_name, 'r') as read_file: + lines = read_file.readlines() + lines.insert(0, f"# generated at {timestamp_str}" + "\n") # Add newline character + with open(file_name, 'w') as write_file: + write_file.writelines(lines) + + +def save_configmap(cmd, config_map_file): + + for i in range(0, 9): + res = subprocess.run(cmd, check=False, capture_output=True) + if res.returncode == 0: + output = res.stdout.decode() + with open(config_map_file, "wb") as output_file: + output_file.write(output.encode()) + print(f"Successfully saved configmap to {config_map_file}") + break + else: + if i == 9: + print(f"An error occurred getting data, attempt={i}: {res}") + sys.exit(1) + else: + time.sleep(5) + + +if __name__ == "__main__": + if len(sys.argv) < 4: + print("Usage: dual-stack-multus.py ") + sys.exit(1) + + protocol = sys.argv[1].lower() + if protocol not in ["ipv4", "ipv6"]: + print("invalid IP protocol") + sys.exit(1) + + state = sys.argv[2].lower() + if state not in ["true", "false"]: + print("invalid state for protocol") + sys.exit(1) + + wait = 0 + try: + wait = int(sys.argv[3]) + except ValueError: + print(f"Error: restart_wait='{sys.argv[3]}' cannot be converted to an integer.") + sys.exit(1) + + print(f"dual-stack-multus {protocol} {state} {wait}") + + print("execute: kubectl get cm -n kube-system multus-cni-config.v1 -o yaml") + command = ["kubectl", kubectl_config, "-n", "kube-system", + "get", "cm", "multus-cni-config.v1", "-o", "yaml"] + save_configmap(command, multus_config_map_file) + + modified_data = str() + with open(multus_config_map_file, "r") as file: + multus_config_map = file.read() + if protocol == "ipv4": + match = re.search(r'"assign_ipv4": "(true|false)"', multus_config_map) + if match: + new_val = f'\"assign_ipv4\": \"{state}\"' + modified_data = multus_config_map.replace(match.group(), new_val) + elif protocol == "ipv6": + match = re.search(r'"assign_ipv6": "(true|false)"', multus_config_map) + if match: + new_val = f'\"assign_ipv6\": \"{state}\"' + modified_data = multus_config_map.replace(match.group(), new_val) + + with open(multus_config_map_file, "w") as file: + file.write(modified_data) + prepend_timestamp_line(multus_config_map_file) + + print(f"execute: kubectl apply -f {multus_config_map_file}") + result = subprocess.run(["kubectl", kubectl_config, + "apply", "-f", multus_config_map_file], + check=False, stdout=subprocess.PIPE) + print(f"update multus configmap result={result}") + if result.returncode != 0: + sys.exit(1) + + print("execute: kubectl rollout restart daemonset/multus -n kube-system") + result = subprocess.run(["kubectl", kubectl_config, "-n", "kube-system", + "rollout", "restart", "daemonset", "kube-multus-ds-amd64"], + check=False, stdout=subprocess.PIPE) + print(result) + if result.returncode != 0: + sys.exit(1) + +print(f"wait {wait} seconds for the restarts") +time.sleep(wait) +sys.exit(0) diff --git a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp index 81cbfe476..0b3de7e82 100644 --- a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp +++ b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp @@ -7,6 +7,7 @@ class platform::kubernetes::params ( $kubeadm_version = undef, $kubelet_version = undef, $node_ip = undef, + $node_ip_secondary = undef, $service_domain = undef, $apiserver_cluster_ip = undef, $dns_service_ip = undef, @@ -266,6 +267,7 @@ class platform::kubernetes::kubeadm { require platform::kubernetes::symlinks $node_ip = $::platform::kubernetes::params::node_ip + $node_ip_secondary = $::platform::kubernetes::params::node_ip_secondary $host_labels = $::platform::kubernetes::params::host_labels $k8s_platform_cpuset = $::platform::kubernetes::params::k8s_platform_cpuset $k8s_reserved_mem = $::platform::kubernetes::params::k8s_reserved_mem @@ -1105,30 +1107,134 @@ class platform::kubernetes::certsans::runtime inherits ::platform::kubernetes::params { include ::platform::params include ::platform::network::mgmt::params + include ::platform::network::mgmt::ipv4::params + include ::platform::network::mgmt::ipv6::params include ::platform::network::oam::params + include ::platform::network::oam::ipv4::params + include ::platform::network::oam::ipv6::params include ::platform::network::cluster_host::params + include ::platform::network::cluster_host::ipv4::params + include ::platform::network::cluster_host::ipv6::params - if $::platform::network::mgmt::params::subnet_version == $::platform::params::ipv6 { + $ipv4_val = $::platform::params::ipv4 + $ipv6_val = $::platform::params::ipv6 + + $prim_mgmt_subnet_ver = $::platform::network::mgmt::params::subnet_version + $ipv4_mgmt_subnet_ver = $::platform::network::mgmt::ipv4::params::subnet_version + $ipv6_mgmt_subnet_ver = $::platform::network::mgmt::ipv6::params::subnet_version + if $prim_mgmt_subnet_ver == $ipv6_val and $ipv4_mgmt_subnet_ver != undef { + $sec_mgmt_subnet_ver = $ipv4_mgmt_subnet_ver + } elsif $prim_mgmt_subnet_ver == $ipv4_val and $ipv6_mgmt_subnet_ver != undef { + $sec_mgmt_subnet_ver = $ipv6_mgmt_subnet_ver + } else { + $sec_mgmt_subnet_ver = undef + } + + $prim_cluster_host_subnet_ver = $::platform::network::cluster_host::params::subnet_version + $ipv4_cluster_host_subnet_ver = $::platform::network::cluster_host::ipv4::params::subnet_version + $ipv6_cluster_host_subnet_ver = $::platform::network::cluster_host::ipv6::params::subnet_version + if $prim_cluster_host_subnet_ver == $ipv6_val and $ipv4_cluster_host_subnet_ver != undef { + $sec_cluster_host_subnet_ver = $ipv4_cluster_host_subnet_ver + } elsif $prim_cluster_host_subnet_ver == $ipv4_val and $ipv6_cluster_host_subnet_ver != undef { + $sec_cluster_host_subnet_ver = $ipv6_cluster_host_subnet_ver + } else { + $sec_cluster_host_subnet_ver = undef + } + + $prim_oam_subnet_ver = $::platform::network::oam::params::subnet_version + $ipv4_oam_subnet_ver = $::platform::network::oam::ipv4::params::subnet_version + $ipv6_oam_subnet_ver = $::platform::network::oam::ipv6::params::subnet_version + if $prim_oam_subnet_ver == $ipv6_val and $ipv4_oam_subnet_ver != undef { + $sec_oam_subnet_ver = $ipv4_oam_subnet_ver + } elsif $prim_oam_subnet_ver == $ipv4_val and $ipv6_oam_subnet_ver != undef { + $sec_oam_subnet_ver = $ipv6_oam_subnet_ver + } else { + $sec_oam_subnet_ver = undef + } + + if $::platform::network::mgmt::params::subnet_version == $ipv6_val { $localhost_address = '::1' } else { $localhost_address = '127.0.0.1' } + if $sec_mgmt_subnet_ver != undef { + if $sec_mgmt_subnet_ver == $ipv4_val { + $certsans_sec_localhost = ',127.0.0.1' + } elsif $sec_mgmt_subnet_ver == $ipv6_val { + $certsans_sec_localhost = ',::1' + } + } else { + $certsans_sec_localhost = '' + } if $::platform::params::system_mode == 'simplex' { - $certsans = "\"${platform::network::cluster_host::params::controller_address}, \ - ${platform::network::cluster_host::params::controller0_address}, \ - ${localhost_address}, \ - ${platform::network::oam::params::controller_address}\"" + $certsans_prim = "${platform::network::cluster_host::params::controller_address}, \ + ${platform::network::cluster_host::params::controller0_address}, \ + ${localhost_address}, \ + ${platform::network::oam::params::controller_address}" + + if $sec_oam_subnet_ver == $ipv4_val { + $certsans_oam_sec = ",${platform::network::oam::ipv4::params::controller_address}" + } elsif $sec_oam_subnet_ver == $ipv6_val { + $certsans_oam_sec = ",${platform::network::oam::ipv6::params::controller_address}" + } else { + $certsans_oam_sec = '' + } + + if $sec_cluster_host_subnet_ver == $ipv4_val { + $certsans_cluster_host_sec = ",${platform::network::cluster_host::ipv4::params::controller_address}, \ + ${platform::network::cluster_host::ipv4::params::controller0_address}" + } elsif $sec_cluster_host_subnet_ver == $ipv6_val { + $certsans_cluster_host_sec = ",${platform::network::cluster_host::ipv6::params::controller_address}, \ + ${platform::network::cluster_host::ipv6::params::controller0_address}" + } else { + $certsans_cluster_host_sec = '' + } + + $certsans_sec_hosts = "${certsans_oam_sec}${certsans_cluster_host_sec}" + + $certsans_sec = "${certsans_sec_hosts}${certsans_sec_localhost}" + } else { - $certsans = "\"${platform::network::cluster_host::params::controller_address}, \ - ${platform::network::cluster_host::params::controller0_address}, \ - ${platform::network::cluster_host::params::controller1_address}, \ - ${localhost_address}, \ - ${platform::network::oam::params::controller_address}, \ - ${platform::network::oam::params::controller0_address}, \ - ${platform::network::oam::params::controller1_address}\"" + $certsans_prim = "${platform::network::cluster_host::params::controller_address}, \ + ${platform::network::cluster_host::params::controller0_address}, \ + ${platform::network::cluster_host::params::controller1_address}, \ + ${localhost_address}, \ + ${platform::network::oam::params::controller_address}, \ + ${platform::network::oam::params::controller0_address}, \ + ${platform::network::oam::params::controller1_address}" + + if $sec_oam_subnet_ver == $ipv4_val { + $certsans_oam_sec = ",${platform::network::oam::ipv4::params::controller_address}, \ + ${platform::network::oam::ipv4::params::controller0_address}, \ + ${platform::network::oam::ipv4::params::controller1_address}" + } elsif $sec_oam_subnet_ver == $ipv6_val { + $certsans_oam_sec = ",${platform::network::oam::ipv6::params::controller_address}, \ + ${platform::network::oam::ipv6::params::controller0_address}, \ + ${platform::network::oam::ipv6::params::controller1_address}" + } else { + $certsans_oam_sec = '' + } + + if $sec_cluster_host_subnet_ver == $ipv4_val { + $certsans_cluster_host_sec = ",${platform::network::cluster_host::ipv4::params::controller_address}, \ + ${platform::network::cluster_host::ipv4::params::controller0_address}, \ + ${platform::network::cluster_host::ipv4::params::controller1_address}" + } elsif $sec_cluster_host_subnet_ver == $ipv6_val { + $certsans_cluster_host_sec = ",${platform::network::cluster_host::ipv6::params::controller_address}, \ + ${platform::network::cluster_host::ipv6::params::controller0_address}, \ + ${platform::network::cluster_host::ipv6::params::controller1_address}" + } else { + $certsans_cluster_host_sec = '' + } + + $certsans_sec_hosts = "${certsans_oam_sec}${certsans_cluster_host_sec}" + + $certsans_sec = "${certsans_sec_hosts}${certsans_sec_localhost}" } + $certsans = "\"${certsans_prim}${certsans_sec}\"" + exec { 'update kube-apiserver certSANs': provider => shell, command => template('platform/kube-apiserver-update-certSANs.erb') @@ -1856,3 +1962,293 @@ class platform::kubernetes::upgrade_abort_recovery command => "kubectl --kubeconfig=/etc/kubernetes/admin.conf uncordon ${::platform::params::hostname}", } } + +class platform::kubernetes::kubelet::update_node_ip::runtime + inherits ::platform::kubernetes::params { + # lint:ignore:140chars + if $::personality == 'worker' or $::personality == 'controller' { + + $node_ip = $::platform::kubernetes::params::node_ip + if $::platform::kubernetes::params::node_ip_secondary { + $node_ip_secondary = $::platform::kubernetes::params::node_ip_secondary + } else { + $node_ip_secondary = 'undef' + } + $restart_wait = '5' + + if $::personality == 'worker' { + $cfgf = '/etc/kubernetes/kubelet.conf' + } elsif $::personality == 'controller' { + $cfgf = '/etc/kubernetes/admin.conf' + } + + exec { 'kubelet-update-node-ip': + path => '/usr/bin:/usr/sbin:/bin:/usr/local/bin', + command => "dual-stack-kubelet.py ${node_ip} ${node_ip_secondary} ${restart_wait} ${cfgf}", + logoutput => true, + } + } + # lint:endignore:140chars +} + +class platform::kubernetes::kubeadm::dual_stack::ipv4::runtime { + # lint:ignore:140chars + include ::platform::network::cluster_pod::params + include ::platform::network::cluster_pod::ipv4::params + include ::platform::network::cluster_service::params + include ::platform::network::cluster_service::ipv4::params + include ::platform::network::cluster_host::ipv4::params + if $::personality == 'controller' { + $restart_wait = '5' + + $pod_prim_network = $::platform::network::cluster_pod::params::subnet_network + $pod_prim_prefixlen = $::platform::network::cluster_pod::params::subnet_prefixlen + $pod_prim_subnet = "${pod_prim_network}/${pod_prim_prefixlen}" + + if $platform::network::cluster_pod::ipv4::params::subnet_version == $::platform::params::ipv4 { + $pod_sec_network = $::platform::network::cluster_pod::ipv4::params::subnet_network + $pod_sec_prefixlen = $::platform::network::cluster_pod::ipv4::params::subnet_prefixlen + $pod_sec_subnet = "${pod_sec_network}/${pod_sec_prefixlen}" + } else { + $pod_sec_subnet = 'undef' + } + + $svc_prim_network = $::platform::network::cluster_service::params::subnet_network + $svc_prim_prefixlen = $::platform::network::cluster_service::params::subnet_prefixlen + $svc_prim_subnet = "${svc_prim_network}/${svc_prim_prefixlen}" + + if $platform::network::cluster_service::ipv4::params::subnet_version == $::platform::params::ipv4 { + $svc_sec_network = $::platform::network::cluster_service::ipv4::params::subnet_network + $svc_sec_prefixlen = $::platform::network::cluster_service::ipv4::params::subnet_prefixlen + $svc_sec_subnet = "${svc_sec_network}/${svc_sec_prefixlen}" + } else { + $svc_sec_subnet = 'undef' + } + + if $::platform::params::hostname == 'controller-0' { + $cluster_host_addr = $::platform::network::cluster_host::params::controller0_address + } else { + $cluster_host_addr = $::platform::network::cluster_host::params::controller1_address + } + + exec { 'update kubeadm pod and service secondary IPv6 subnets': + path => '/usr/bin:/usr/sbin:/bin:/usr/local/bin', + command => "dual-stack-kubeadm.py ${pod_prim_subnet} ${svc_prim_subnet} ${pod_sec_subnet} ${svc_sec_subnet} ${restart_wait} ${cluster_host_addr}", + logoutput => true, + } + } + # lint:endignore:140chars +} + +class platform::kubernetes::kubeadm::dual_stack::ipv6::runtime { + # lint:ignore:140chars + include ::platform::network::cluster_pod::params + include ::platform::network::cluster_pod::ipv6::params + include ::platform::network::cluster_service::params + include ::platform::network::cluster_service::ipv6::params + include ::platform::network::cluster_host::ipv6::params + if $::personality == 'controller' { + $restart_wait = '5' + + $pod_prim_network = $::platform::network::cluster_pod::params::subnet_network + $pod_prim_prefixlen = $::platform::network::cluster_pod::params::subnet_prefixlen + $pod_prim_subnet = "${pod_prim_network}/${pod_prim_prefixlen}" + + if $platform::network::cluster_pod::ipv6::params::subnet_version == $::platform::params::ipv6 { + $pod_sec_network = $::platform::network::cluster_pod::ipv6::params::subnet_network + $pod_sec_prefixlen = $::platform::network::cluster_pod::ipv6::params::subnet_prefixlen + $pod_sec_subnet = "${pod_sec_network}/${pod_sec_prefixlen}" + } else { + $pod_sec_subnet = 'undef' + } + + $svc_prim_network = $::platform::network::cluster_service::params::subnet_network + $svc_prim_prefixlen = $::platform::network::cluster_service::params::subnet_prefixlen + $svc_prim_subnet = "${svc_prim_network}/${svc_prim_prefixlen}" + + if $platform::network::cluster_service::ipv6::params::subnet_version == $::platform::params::ipv6 { + $svc_sec_network = $::platform::network::cluster_service::ipv6::params::subnet_network + $svc_sec_prefixlen = $::platform::network::cluster_service::ipv6::params::subnet_prefixlen + $svc_sec_subnet = "${svc_sec_network}/${svc_sec_prefixlen}" + } else { + $svc_sec_subnet = 'undef' + } + + if $::platform::params::hostname == 'controller-0' { + $cluster_host_addr = $::platform::network::cluster_host::params::controller0_address + } else { + $cluster_host_addr = $::platform::network::cluster_host::params::controller1_address + } + + exec { 'update kubeadm pod and service secondary IPv6 subnets': + path => '/usr/bin:/usr/sbin:/bin:/usr/local/bin', + command => "dual-stack-kubeadm.py ${pod_prim_subnet} ${svc_prim_subnet} ${pod_sec_subnet} ${svc_sec_subnet} ${restart_wait} ${cluster_host_addr}", + logoutput => true, + } + } + # lint:endignore:140chars +} + +class platform::kubernetes::dual_stack::ipv4::runtime { + # lint:ignore:140chars + # adds/removes secondary IPv4 subnets to pod and service + include ::platform::network::cluster_pod::params + include ::platform::network::cluster_pod::ipv4::params + include ::platform::network::cluster_service::params + include ::platform::network::cluster_service::ipv4::params + include ::platform::network::cluster_host::ipv4::params + + $protocol = 'ipv4' + $def_pool_filename = "/tmp/def_pool_${protocol}.yaml" + $kubeconfig = '--kubeconfig=/etc/kubernetes/admin.conf' + $restart_wait = '5' + + $pod_prim_network = $::platform::network::cluster_pod::params::subnet_network + $pod_prim_prefixlen = $::platform::network::cluster_pod::params::subnet_prefixlen + $pod_prim_subnet = "${pod_prim_network}/${pod_prim_prefixlen}" + + if $platform::network::cluster_pod::ipv4::params::subnet_version == $::platform::params::ipv4 { + $pod_sec_network = $::platform::network::cluster_pod::ipv4::params::subnet_network + $pod_sec_prefixlen = $::platform::network::cluster_pod::ipv4::params::subnet_prefixlen + $pod_sec_subnet = "${pod_sec_network}/${pod_sec_prefixlen}" + $c0_addr = $::platform::network::cluster_host::ipv4::params::controller0_address + $state = true + } else { + $pod_sec_subnet = 'undef' + $state = false + $c0_addr = '::' + } + + $svc_prim_network = $::platform::network::cluster_service::params::subnet_network + $svc_prim_prefixlen = $::platform::network::cluster_service::params::subnet_prefixlen + $svc_prim_subnet = "${svc_prim_network}/${svc_prim_prefixlen}" + + if $platform::network::cluster_service::ipv4::params::subnet_version == $::platform::params::ipv4 { + $svc_sec_network = $::platform::network::cluster_service::ipv4::params::subnet_network + $svc_sec_prefixlen = $::platform::network::cluster_service::ipv4::params::subnet_prefixlen + $svc_sec_subnet = "${svc_sec_network}/${svc_sec_prefixlen}" + } else { + $svc_sec_subnet = 'undef' + } + + exec { 'update kube-proxy pod secondary IPv6 subnet': + path => '/usr/bin:/usr/sbin:/bin:/usr/local/bin', + command => "dual-stack-kubeproxy.py ${pod_prim_subnet} ${pod_sec_subnet} ${restart_wait}", + logoutput => true, + } + -> exec { 'update calico node pod secondary IPv6 subnet': + path => '/usr/bin:/usr/sbin:/bin:/usr/local/bin', + command => "dual-stack-calico.py ${protocol} ${state} ${c0_addr} ${restart_wait}", + logoutput => true, + } + if $state == true { + $ipip_mode = 'Always' + file { $def_pool_filename: + ensure => file, + content => template('platform/callico_ippool.yaml.erb'), + owner => 'root', + group => 'root', + mode => '0640', + } + -> exec { "create default-${protocol}-ippool": + path => '/usr/bin:/usr/sbin:/bin:/usr/local/bin', + command => "kubectl ${kubeconfig} apply -f ${def_pool_filename}", + logoutput => true + } + } else { + exec { "delete default-${protocol}-ippool": + path => '/usr/bin:/usr/sbin:/bin:/usr/local/bin', + command => "kubectl ${kubeconfig} delete ippools.crd.projectcalico.org default-${protocol}-ippool", + logoutput => true, + onlyif => "kubectl ${kubeconfig} get ippools.crd.projectcalico.org default-${protocol}-ippool ", + } + } + exec { 'update multus to support IPv4': + path => '/usr/bin:/usr/sbin:/bin:/usr/local/bin', + command => "dual-stack-multus.py ${protocol} ${state} ${restart_wait}", + logoutput => true, + } + # lint:endignore:140chars +} + +class platform::kubernetes::dual_stack::ipv6::runtime { + # lint:ignore:140chars + # adds/removes secondary IPv6 subnets to pod and service + include ::platform::network::cluster_pod::params + include ::platform::network::cluster_pod::ipv6::params + include ::platform::network::cluster_service::params + include ::platform::network::cluster_service::ipv6::params + include ::platform::network::cluster_host::ipv6::params + + $protocol = 'ipv6' + $def_pool_filename = "/tmp/def_pool_${protocol}.yaml" + $kubeconfig = '--kubeconfig=/etc/kubernetes/admin.conf' + $restart_wait = '10' + + $pod_prim_network = $::platform::network::cluster_pod::params::subnet_network + $pod_prim_prefixlen = $::platform::network::cluster_pod::params::subnet_prefixlen + $pod_prim_subnet = "${pod_prim_network}/${pod_prim_prefixlen}" + + if $platform::network::cluster_pod::ipv6::params::subnet_version == $::platform::params::ipv6 { + $pod_sec_network = $::platform::network::cluster_pod::ipv6::params::subnet_network + $pod_sec_prefixlen = $::platform::network::cluster_pod::ipv6::params::subnet_prefixlen + $pod_sec_subnet = "${pod_sec_network}/${pod_sec_prefixlen}" + $c0_addr = $::platform::network::cluster_host::ipv6::params::controller0_address + $state = true + } else { + $pod_sec_subnet = 'undef' + $state = false + $c0_addr = '::' + } + + $svc_prim_network = $::platform::network::cluster_service::params::subnet_network + $svc_prim_prefixlen = $::platform::network::cluster_service::params::subnet_prefixlen + $svc_prim_subnet = "${svc_prim_network}/${svc_prim_prefixlen}" + + if $platform::network::cluster_service::ipv6::params::subnet_version == $::platform::params::ipv6 { + $svc_sec_network = $::platform::network::cluster_service::ipv6::params::subnet_network + $svc_sec_prefixlen = $::platform::network::cluster_service::ipv6::params::subnet_prefixlen + $svc_sec_subnet = "${svc_sec_network}/${svc_sec_prefixlen}" + } else { + $svc_sec_subnet = 'undef' + } + + exec { 'update kube-proxy pod secondary IPv6 subnet': + path => '/usr/bin:/usr/sbin:/bin:/usr/local/bin', + command => "dual-stack-kubeproxy.py ${pod_prim_subnet} ${pod_sec_subnet} ${restart_wait}", + logoutput => true, + } + -> exec { 'update calico node pod secondary IPv6 subnet': + path => '/usr/bin:/usr/sbin:/bin:/usr/local/bin', + command => "dual-stack-calico.py ${protocol} ${state} ${c0_addr} ${restart_wait}", + logoutput => true, + } + if $state == true { + $ipip_mode = 'Never' + file { $def_pool_filename: + ensure => file, + content => template('platform/callico_ippool.yaml.erb'), + owner => 'root', + group => 'root', + mode => '0640', + } + -> exec { "create default-${protocol}-ippool": + path => '/usr/bin:/usr/sbin:/bin:/usr/local/bin', + command => "kubectl ${kubeconfig} apply -f ${def_pool_filename}", + logoutput => true, + } + } else { + exec { "delete default-${protocol}-ippool": + path => '/usr/bin:/usr/sbin:/bin:/usr/local/bin', + command => "kubectl ${kubeconfig} delete ippools.crd.projectcalico.org default-${protocol}-ippool", + logoutput => true, + onlyif => "kubectl ${kubeconfig} get ippools.crd.projectcalico.org default-${protocol}-ippool " + } + } + exec { 'update multus to support IPv6': + path => '/usr/bin:/usr/sbin:/bin:/usr/local/bin', + command => "dual-stack-multus.py ${protocol} ${state} ${restart_wait}", + logoutput => true, + } + # lint:endignore:140chars +} diff --git a/puppet-manifests/src/modules/platform/manifests/network.pp b/puppet-manifests/src/modules/platform/manifests/network.pp index f341c7b7d..0c6f15f59 100644 --- a/puppet-manifests/src/modules/platform/manifests/network.pp +++ b/puppet-manifests/src/modules/platform/manifests/network.pp @@ -223,6 +223,116 @@ class platform::network::cluster_host::params( $mtu = 1500, ) { } +class platform::network::cluster_pod::ipv4::params( + # shared parameters with base class - required for auto hiera parameter lookup + $interface_address = undef, + $subnet_version = undef, + $subnet_network = undef, + $subnet_network_url = undef, + $subnet_prefixlen = undef, + $subnet_netmask = undef, + $subnet_start = undef, + $subnet_end = undef, + $gateway_address = undef, + $controller_address = undef, # controller floating + $controller_address_url = undef, # controller floating url address + $controller0_address = undef, # controller unit0 + $controller1_address = undef, # controller unit1 +) { } + +class platform::network::cluster_pod::ipv6::params( + # shared parameters with base class - required for auto hiera parameter lookup + $interface_address = undef, + $subnet_version = undef, + $subnet_network = undef, + $subnet_network_url = undef, + $subnet_prefixlen = undef, + $subnet_netmask = undef, + $subnet_start = undef, + $subnet_end = undef, + $gateway_address = undef, + $controller_address = undef, # controller floating + $controller_address_url = undef, # controller floating url address + $controller0_address = undef, # controller unit0 + $controller1_address = undef, # controller unit1 +) { } + +class platform::network::cluster_pod::params( + # this class contains the primary pool (ipv4 or ipv6) addresses for compatibility + # shared parameters with base class - required for auto hiera parameter lookup + $interface_name = undef, + $interface_address = undef, + $interface_devices = [], + $subnet_version = undef, + $subnet_network = undef, + $subnet_network_url = undef, + $subnet_prefixlen = undef, + $subnet_netmask = undef, + $subnet_start = undef, + $subnet_end = undef, + $gateway_address = undef, + $controller_address = undef, # controller floating + $controller_address_url = undef, # controller floating url address + $controller0_address = undef, # controller unit0 + $controller1_address = undef, # controller unit1 + $mtu = 1500, +) { } + +class platform::network::cluster_service::ipv4::params( + # shared parameters with base class - required for auto hiera parameter lookup + $interface_address = undef, + $subnet_version = undef, + $subnet_network = undef, + $subnet_network_url = undef, + $subnet_prefixlen = undef, + $subnet_netmask = undef, + $subnet_start = undef, + $subnet_end = undef, + $gateway_address = undef, + $controller_address = undef, # controller floating + $controller_address_url = undef, # controller floating url address + $controller0_address = undef, # controller unit0 + $controller1_address = undef, # controller unit1 +) { } + +class platform::network::cluster_service::ipv6::params( + # shared parameters with base class - required for auto hiera parameter lookup + $interface_address = undef, + $subnet_version = undef, + $subnet_network = undef, + $subnet_network_url = undef, + $subnet_prefixlen = undef, + $subnet_netmask = undef, + $subnet_start = undef, + $subnet_end = undef, + $gateway_address = undef, + $controller_address = undef, # controller floating + $controller_address_url = undef, # controller floating url address + $controller0_address = undef, # controller unit0 + $controller1_address = undef, # controller unit1 +) { } + +class platform::network::cluster_service::params( + # this class contains the primary pool (ipv4 or ipv6) addresses for compatibility + # shared parameters with base class - required for auto hiera parameter lookup + $interface_name = undef, + $interface_address = undef, + $interface_devices = [], + $subnet_version = undef, + $subnet_network = undef, + $subnet_network_url = undef, + $subnet_prefixlen = undef, + $subnet_netmask = undef, + $subnet_start = undef, + $subnet_end = undef, + $gateway_address = undef, + $controller_address = undef, # controller floating + $controller_address_url = undef, # controller floating url address + $controller0_address = undef, # controller unit0 + $controller1_address = undef, # controller unit1 + $mtu = 1500, +) { } + class platform::network::ironic::ipv4::params( # shared parameters with base class - required for auto hiera parameter lookup $interface_address = undef, diff --git a/puppet-manifests/src/modules/platform/manifests/sm.pp b/puppet-manifests/src/modules/platform/manifests/sm.pp index aebdee852..b788e696a 100644 --- a/puppet-manifests/src/modules/platform/manifests/sm.pp +++ b/puppet-manifests/src/modules/platform/manifests/sm.pp @@ -1526,6 +1526,7 @@ class platform::sm::enable_admin_config::runtime { include ::platform::network::admin::params include ::platform::network::admin::ipv4::params include ::platform::network::admin::ipv6::params + $admin_ip_interface = $::platform::network::admin::params::interface_name if $::platform::network::admin::ipv4::params::subnet_version == $::platform::params::ipv4 { exec { 'Manage admin-ipv4 service': @@ -1579,6 +1580,7 @@ class platform::sm::update_admin_config::runtime { include ::platform::network::admin::params include ::platform::network::admin::ipv4::params include ::platform::network::admin::ipv6::params + $admin_ip_interface = $::platform::network::admin::params::interface_name $admin_ip_param_ip = $::platform::network::admin::params::controller_address $admin_ip_param_mask = $::platform::network::admin::params::subnet_prefixlen @@ -1628,6 +1630,85 @@ class platform::sm::update_admin_config::runtime { # lint:endignore:140chars } +class platform::sm::cluster_host::add_ip_config::ipv4::runtime { + # lint:ignore:140chars + include ::platform::network::cluster_host::ipv4::params + + if $::personality == 'controller' { + if $::platform::network::cluster_host::ipv4::params::subnet_version == $::platform::params::ipv4 { + $interface = $::platform::network::cluster_host::params::interface_name + $ip = $::platform::network::cluster_host::ipv4::params::controller_address + $mask = $::platform::network::cluster_host::ipv4::params::subnet_prefixlen + exec { 'Reconfigure Cluster-Host IPv4 service_instance': + command => "sm-configure service_instance cluster-host-ipv4 cluster_host-ipv4 \"ip=${ip},cidr_netmask=${mask},nic=${interface},arp_count=7\" --apply", + } + -> exec { 'Manage cluster_host-ipv4 service': + command => 'sm-manage service cluster-host-ipv4' + } + -> exec { 'Provision cluster-host-ipv4 service': + command => 'sm-provision service-group-member controller-services cluster-host-ipv4 --apply' + } + } + } + # lint:endignore:140chars +} + +class platform::sm::cluster_host::remove_ip_config::ipv4::runtime { + # lint:ignore:140chars + if $::personality == 'controller' { + exec { 'Unmanage cluster-host-ipv4 service': + command => 'sm-unmanage service cluster-host-ipv4', + onlyif => 'sm-dump | grep cluster-host-ipv4', + } + -> exec { 'Deprovision cluster-host-ipv4 service': + command => 'sm-deprovision service-group-member controller-services cluster-host-ipv4 --apply', + onlyif => 'sm-dump | grep cluster-host-ipv4', + } + } + # lint:endignore:140chars +} + +class platform::sm::cluster_host::add_ip_config::ipv6::runtime { + # lint:ignore:140chars + include ::platform::network::cluster_host::ipv6::params + + if $::personality == 'controller' { + if $::platform::network::cluster_host::ipv6::params::subnet_version == $::platform::params::ipv6 { + $interface = $::platform::network::cluster_host::params::interface_name + $ip = $::platform::network::cluster_host::ipv6::params::controller_address + $mask = $::platform::network::cluster_host::ipv6::params::subnet_prefixlen + exec { 'Reconfigure Cluster-Host IPv6 service_instance': + command => "sm-configure service_instance cluster-host-ipv6 cluster_host-ipv6 \"ip=${ip},cidr_netmask=${mask},nic=${interface},arp_count=7\" --apply", + } + -> exec { 'Manage cluster_host-ipv6 service': + command => 'sm-manage service cluster-host-ipv6' + } + -> exec { 'Provision cluster-host-ipv6 service': + command => 'sm-provision service-group-member controller-services cluster-host-ipv6 --apply' + } + } + } + # lint:endignore:140chars +} + +class platform::sm::cluster_host::remove_ip_config::ipv6::runtime { + # lint:ignore:140chars + include ::platform::network::cluster_host::params + include ::platform::network::cluster_host::ipv6::params + + if $::personality == 'controller' { + exec { 'Unmanage cluster-host-ipv6 service': + command => 'sm-unmanage service cluster-host-ipv6', + onlyif => 'sm-dump | grep cluster-host-ipv6', + } + -> exec { 'Deprovision cluster-host-ipv6 service': + command => 'sm-deprovision service-group-member controller-services cluster-host-ipv6 --apply', + onlyif => 'sm-dump | grep cluster-host-ipv6', + } + } + # lint:endignore:140chars +} + define platform::sm::restart { exec {"sm-restart-${name}": command => "sm-restart-safe service ${name}", diff --git a/puppet-manifests/src/modules/platform/templates/callico_ippool.yaml.erb b/puppet-manifests/src/modules/platform/templates/callico_ippool.yaml.erb new file mode 100644 index 000000000..2af7c2bbf --- /dev/null +++ b/puppet-manifests/src/modules/platform/templates/callico_ippool.yaml.erb @@ -0,0 +1,11 @@ +# Calico default ip pool + +apiVersion: crd.projectcalico.org/v1 +kind: IPPool +metadata: + name: default-<%= @protocol %>-ippool +spec: + cidr: <%= @pod_sec_subnet %> + ipipMode: <%= @ipip_mode %> + natOutgoing: true + vxlanMode: Never diff --git a/puppet-manifests/src/modules/platform/templates/kubelet.conf.erb b/puppet-manifests/src/modules/platform/templates/kubelet.conf.erb index 0920edbd9..576cc0d8d 100644 --- a/puppet-manifests/src/modules/platform/templates/kubelet.conf.erb +++ b/puppet-manifests/src/modules/platform/templates/kubelet.conf.erb @@ -1,2 +1,7 @@ # Overrides config file for kubelet +<%- if defined?(@node_ip_secondary) -%> +KUBELET_EXTRA_ARGS=--node-ip=<%= @node_ip %>,<%= @node_ip_secondary %> --volume-plugin-dir=<%= @k8s_vol_plugin_dir %> <%= @k8s_cpu_manager_opts %> --container-runtime-endpoint=unix:///var/run/containerd/containerd.sock --pod-max-pids <%= @k8s_pod_max_pids %> +<%- else -%> KUBELET_EXTRA_ARGS=--node-ip=<%= @node_ip %> --volume-plugin-dir=<%= @k8s_vol_plugin_dir %> <%= @k8s_cpu_manager_opts %> --container-runtime-endpoint=unix:///var/run/containerd/containerd.sock --pod-max-pids <%= @k8s_pod_max_pids %> +<%- end -%> + diff --git a/test-requirements.txt b/test-requirements.txt index ec28eb63b..b1fba817e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,3 +3,4 @@ hacking < 4.0.1 bashate >= 0.2 bandit!=1.6.0,>=1.1.0,<2.0.0;python_version>="3.0" # GPLv2 shellcheck-py;python_version>="3.0" # MIT +netaddr >= 0.7.19