Kubernetes Dual-Stack Runtime Configuration

New classes and scripts are added to handle sm, kubelet, kubeadm,
kube-proxy, calico, multus, and certsans.

New variables exported to the hiera are used in this change, in
conjunction with the existing ones:
platform::kubernetes::params::node_ip_secondary
platform::network::cluster_pod::params::*
platform::network::cluster_pod::ipv4::params::
platform::network::cluster_pod::ipv6::params::*
platform::network::cluster_service::params::*
platform::network::cluster_service::ipv4::params::
platform::network::cluster_service::ipv6::params::*

Test Plan:
==========

In all tests pods were brought up and the datapath was validated
directly and through a service

[PASS] in AIO-DX IPv4, configure dual-stack and back to single-stack
[PASS] in AIO-DX IPv6, configure dual-stack and back to single-stack
[PASS] in AIO-SX IPv4, configure dual-stack and back to single-stack
[PASS] in AIO-SX IPv6, configure dual-stack and back to single-stack
[PASS] in Standard IPv4, configure dual-stack and back to single-stack
[PASS] in Standard IPv6, configure dual-stack and back to single-stack

Story: 2011027
Task: 50203

Change-Id: Ifb908c097960f90c5eabeca8cc02d2f60ae4d731
Signed-off-by: Andre Kantek <andrefernandozanella.kantek@windriver.com>

puppet-manifests/src/Makefile

@@ -27,6 +27,11 @@ endif
 	install -m 755 -D bin/kubelet-cleanup-orphaned-volumes.sh $(BINDIR)/kubelet-cleanup-orphaned-volumes.sh
 	install -m 755 -D bin/check_ipv6_tentative_addresses.py $(BINDIR)/check_ipv6_tentative_addresses.py
 	install -m 755 -D bin/manage_partitions_pre_script.sh $(BINDIR)/manage_partitions_pre_script.sh
+	install -m 755 -D bin/dual-stack-kubelet.py $(BINDIR)/dual-stack-kubelet.py
+	install -m 755 -D bin/dual-stack-kubeadm.py $(BINDIR)/dual-stack-kubeadm.py
+	install -m 755 -D bin/dual-stack-kubeproxy.py $(BINDIR)/dual-stack-kubeproxy.py
+	install -m 755 -D bin/dual-stack-calico.py $(BINDIR)/dual-stack-calico.py
+	install -m 755 -D bin/dual-stack-multus.py $(BINDIR)/dual-stack-multus.py
 
 	install -d -m 0755 $(CONFIGDIR)
 ifdef hiera_v5

puppet-manifests/src/bin/dual-stack-calico.py

@@ -0,0 +1,188 @@
+#!/usr/bin/python3
+#
+# Copyright (c) 2024 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+''' This script updates the calico config to handle single or dual-stack
+'''
+
+import sys
+import subprocess
+import yaml
+import time
+import re
+import netaddr
+
+from datetime import datetime
+
+
+calico_config_map_file = "/tmp/calico-configmap.yaml"
+calico_daemonset_file = "/tmp/calico-daemonset.yaml"
+
+kubectl_config = "--kubeconfig=/etc/kubernetes/admin.conf"
+
+
+def get_yaml_data(cmd, attempts=15):
+    data = dict()
+    for i in range(0, attempts):
+        res = subprocess.run(cmd, check=False, stdout=subprocess.PIPE)
+        if res.returncode != 0:
+            if i == attempts:
+                print(f"An error occurred getting data, attempt={i}: {res}")
+                sys.exit(1)
+            else:
+                time.sleep(5)
+        else:
+            print("yaml data was collected")
+            data = yaml.load(res.stdout, Loader=yaml.Loader)
+            break
+    return data
+
+
+def is_valid_ip(address):
+    try:
+        if netaddr.valid_ipv4(address):
+            return True
+        if netaddr.valid_ipv6(address):
+            return True
+    except netaddr.AddrFormatError:
+        pass
+    return False
+
+
+def prepend_timestamp_line(file_name):
+    timestamp_str = datetime.now().strftime(format="%Y-%m-%d %H:%M:%S")
+    with open(file_name, 'r') as read_file:
+        lines = read_file.readlines()
+    lines.insert(0, f"# generated at {timestamp_str}" + "\n")  # Add newline character
+    with open(file_name, 'w') as write_file:
+        write_file.writelines(lines)
+
+
+def save_configmap(cmd, config_map_file):
+
+    for i in range(0, 9):
+        res = subprocess.run(cmd, check=False, capture_output=True)
+        if res.returncode == 0:
+            output = res.stdout.decode()
+            with open(config_map_file, "wb") as output_file:
+                output_file.write(output.encode())
+                print(f"Successfully saved configmap to {config_map_file}")
+                break
+        else:
+            if i == 9:
+                print(f"An error occurred getting data, attempt={i}: {res}")
+                sys.exit(1)
+            else:
+                time.sleep(5)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 5:
+        print("Usage: dual-stack-calico.py <protocol> <state> <c0_address> <restart_wait>")
+        sys.exit(1)
+
+    protocol = sys.argv[1].lower()
+    if protocol not in ["ipv4", "ipv6"]:
+        print("invalid IP protocol")
+        sys.exit(1)
+
+    state = sys.argv[2].lower()
+    if state not in ["true", "false"]:
+        print("invalid state for protocol")
+        sys.exit(1)
+
+    c0_address = sys.argv[3]
+    if not is_valid_ip(c0_address):
+        if state == "true":
+            print(f"Error: invalid node_ip '{c0_address}', exit")
+            sys.exit(1)
+        else:
+            c0_address = None
+
+    wait = 0
+    try:
+        wait = int(sys.argv[4])
+    except ValueError:
+        print(f"Error: restart_wait='{sys.argv[4]}' cannot be converted to an integer.")
+        sys.exit(1)
+
+    print(f"dual-stack-calico {protocol} {state} {c0_address} {wait}")
+
+    print("execute: kubectl get cm -n kube-system calico-config -o yaml")
+    command = ["kubectl", kubectl_config, "-n", "kube-system",
+               "get", "cm", "calico-config", "-o", "yaml"]
+    save_configmap(command, calico_config_map_file)
+
+    modified_data = str()
+    with open(calico_config_map_file, "r") as file:
+        calico_config_map = file.read()
+        if protocol == "ipv4":
+            match = re.search(r'"assign_ipv4": "(true|false)"', calico_config_map)
+            if match:
+                new_val = f'\"assign_ipv4\": \"{state}\"'
+                modified_data = calico_config_map.replace(match.group(), new_val)
+        elif protocol == "ipv6":
+            match = re.search(r'"assign_ipv6": "(true|false)"', calico_config_map)
+            if match:
+                new_val = f'\"assign_ipv6\": \"{state}\"'
+                modified_data = calico_config_map.replace(match.group(), new_val)
+
+    with open(calico_config_map_file, "w") as file:
+        file.write(modified_data)
+    prepend_timestamp_line(calico_config_map_file)
+
+    print(f"execute: kubectl apply -f {calico_config_map_file}")
+    result = subprocess.run(["kubectl", kubectl_config, "apply", "-f", calico_config_map_file],
+                            check=False, stdout=subprocess.PIPE)
+    print(f"update calico configmap result={result}")
+    if result.returncode != 0:
+        sys.exit(1)
+
+    print("execute: kubectl -n kube-system get daemonset calico-node -o yaml")
+    command = ["kubectl", kubectl_config, "-n", "kube-system",
+               "get", "daemonset", "calico-node", "-o", "yaml"]
+    calico_ds_data = get_yaml_data(command, 9)
+
+    for container in calico_ds_data['spec']['template']['spec']['containers']:
+        ipv4_autodetect = False
+        ipv6_autodetect = False
+        for env in container['env']:
+            if protocol == "ipv4":
+                if env["name"] == "IP":
+                    env["value"] = "autodetect" if state == "true" else "none"
+                if env["name"] == "IP_AUTODETECTION_METHOD":
+                    ipv4_autodetect = True
+            if protocol == "ipv6":
+                if env["name"] == "IP6":
+                    env["value"] = "autodetect" if state == "true" else "none"
+                if env["name"] == "IP6_AUTODETECTION_METHOD":
+                    ipv6_autodetect = True
+        if not ipv4_autodetect and protocol == "ipv4" and state == "true":
+            container['env'].append({"name": "IP_AUTODETECTION_METHOD",
+                                     "value": f"can-reach={c0_address}"})
+        if not ipv6_autodetect and protocol == "ipv6" and state == "true":
+            container['env'].append({"name": "IP6_AUTODETECTION_METHOD",
+                                     "value": f"can-reach={c0_address}"})
+        if ipv4_autodetect and protocol == "ipv4" and state == "false":
+            container['env'] = [env for env in container['env']
+                                if not env["name"] == "IP_AUTODETECTION_METHOD"]
+        if ipv6_autodetect and protocol == "ipv6" and state == "false":
+            container['env'] = [env for env in container['env']
+                                if not env["name"] == "IP6_AUTODETECTION_METHOD"]
+
+    with open(calico_daemonset_file, 'w') as config_file:
+        yaml.dump(calico_ds_data, config_file, default_flow_style=False)
+    prepend_timestamp_line(calico_daemonset_file)
+
+    print(f"execute: kubectl apply -f {calico_daemonset_file}")
+    result = subprocess.run(["kubectl", kubectl_config, "apply", "-f", calico_daemonset_file],
+                            check=False, stdout=subprocess.PIPE)
+    print(f"update calico daemonset result={result}")
+    if result.returncode != 0:
+        sys.exit(1)
+
+print(f"wait {wait} seconds for the restarts")
+time.sleep(wait)
+sys.exit(0)

puppet-manifests/src/bin/dual-stack-kubeadm.py

@@ -0,0 +1,195 @@
+#!/usr/bin/python3
+#
+# Copyright (c) 2024 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+''' This script updates the kubeadm config to handle single or dual-stack
+'''
+
+import sys
+import subprocess
+import yaml
+import time
+import os
+import netaddr
+
+from datetime import datetime
+
+
+config_map_file = "/tmp/kubeadm-config.yaml"
+cluster_config_file = "/tmp/kubeadm-config-cluster.yaml"
+kubectl_config = "--kubeconfig=/etc/kubernetes/admin.conf"
+active_controller_puppet_path = '/opt/platform/puppet/'
+INITCONFIG_TEMPLATE = '''---
+apiVersion: kubeadm.k8s.io/v1beta3
+kind: InitConfiguration
+localAPIEndpoint:
+  advertiseAddress: {}'''
+
+
+def get_yaml_data(cmd, attempts=15):
+    data = dict()
+    for i in range(0, attempts):
+        res = subprocess.run(cmd, check=False, stdout=subprocess.PIPE)
+        if res.returncode != 0:
+            if i == attempts:
+                print(f"An error occurred getting data, attempt={i}: {res}")
+                sys.exit(1)
+            else:
+                time.sleep(5)
+        else:
+            print("yaml data was collected")
+            data = yaml.load(res.stdout, Loader=yaml.Loader)
+            break
+    return data
+
+
+def is_valid_ip(address):
+    try:
+        if netaddr.valid_ipv4(address):
+            return True
+        if netaddr.valid_ipv6(address):
+            return True
+    except netaddr.AddrFormatError:
+        pass
+    return False
+
+
+def prepend_timestamp_line(file_name):
+    timestamp_str = datetime.now().strftime(format="%Y-%m-%d %H:%M:%S")
+    with open(file_name, 'r') as read_file:
+        lines = read_file.readlines()
+    lines.insert(0, f"# generated at {timestamp_str}" + "\n")  # Add newline character
+    with open(file_name, 'w') as write_file:
+        write_file.writelines(lines)
+
+
+def is_valid_network(address):
+    """
+    This function checks if the provided string is a valid network address using netaddr.
+    """
+    try:
+        netaddr.IPNetwork(address)
+        return True
+    except netaddr.AddrFormatError as ex:
+        print(f"exception {str(ex)}")
+    return False
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 7:
+        print("Usage: dual-stack-kubeadm.py <pod_prim_subnet>"
+              " <svc_prim_subnet> <pod_sec_subnet> <svc_sec_subnet> <restart_wait>"
+              " <advertise_address>")
+        sys.exit(1)
+
+    pod_prim_subnet = sys.argv[1]
+    svc_prim_subnet = sys.argv[2]
+    pod_sec_subnet = sys.argv[3]
+    svc_sec_subnet = sys.argv[4]
+    advertise_address = sys.argv[6]
+
+    if not is_valid_network(pod_prim_subnet):
+        print(f"Error: invalid pod_prim_subnet '{pod_prim_subnet}', exit")
+        sys.exit(1)
+
+    if not is_valid_network(svc_prim_subnet):
+        print(f"Error: invalid svc_prim_subnet '{svc_prim_subnet}', exit")
+        sys.exit(1)
+
+    if not is_valid_ip(advertise_address):
+        print(f"Error: invalid advertise_address '{advertise_address}', exit")
+        sys.exit(1)
+
+    if not is_valid_network(pod_sec_subnet):
+        pod_sec_subnet = None
+
+    if not is_valid_network(svc_sec_subnet):
+        svc_sec_subnet = None
+
+    wait = 0
+    try:
+        wait = int(sys.argv[5])
+    except ValueError:
+        print(f"Error: restart_wait='{sys.argv[5]}' cannot be converted to an integer.")
+        sys.exit(1)
+
+    print(f"dual-stack-kubeadm {pod_prim_subnet} {svc_prim_subnet}"
+          f" {pod_sec_subnet} {svc_sec_subnet} {wait}")
+
+    print("execute: kubectl -n kube-system get configmap kubeadm-config -o yaml")
+    command = ["kubectl", kubectl_config, "-n", "kube-system",
+               "get", "configmap", "kubeadm-config", "-o", "yaml"]
+    yaml_data = get_yaml_data(command, 9)
+    cluster_cfg_yaml = yaml_data["data"]["ClusterConfiguration"]
+    cluster_cfg = yaml.load(cluster_cfg_yaml, Loader=yaml.Loader)
+    configmap_reconfig = False
+    if pod_prim_subnet and pod_sec_subnet:
+        if cluster_cfg["networking"]["podSubnet"] != f"{pod_prim_subnet},{pod_sec_subnet}":
+            cluster_cfg["networking"]["podSubnet"] = f"{pod_prim_subnet},{pod_sec_subnet}"
+            configmap_reconfig = True
+    elif pod_prim_subnet and not pod_sec_subnet:
+        if cluster_cfg["networking"]["podSubnet"] != f"{pod_prim_subnet}":
+            cluster_cfg["networking"]["podSubnet"] = f"{pod_prim_subnet}"
+            configmap_reconfig = True
+    if svc_prim_subnet and svc_sec_subnet:
+        if cluster_cfg["networking"]["serviceSubnet"] != f"{svc_prim_subnet},{svc_sec_subnet}":
+            cluster_cfg["networking"]["serviceSubnet"] = f"{svc_prim_subnet},{svc_sec_subnet}"
+            configmap_reconfig = True
+    elif svc_prim_subnet and not svc_sec_subnet:
+        if cluster_cfg["networking"]["serviceSubnet"] != f"{svc_prim_subnet}":
+            cluster_cfg["networking"]["serviceSubnet"] = f"{svc_prim_subnet}"
+            configmap_reconfig = True
+
+    with open(cluster_config_file, 'w') as config_file:
+        yaml.dump(cluster_cfg, config_file, default_flow_style=False)
+
+    cluster_config_str = str()
+    with open(cluster_config_file, "r") as file:
+        cluster_config_lines = file.readlines()
+        for line in cluster_config_lines:
+            cluster_config_str = cluster_config_str + line
+    yaml_data["data"]["ClusterConfiguration"] = cluster_config_str
+
+    with open(config_map_file, 'w') as config_file:
+        yaml.dump(yaml_data, config_file, default_flow_style=False)
+    prepend_timestamp_line(config_map_file)
+
+    if configmap_reconfig:
+        if os.path.exists(active_controller_puppet_path):
+            print(f"execute: kubectl apply -f {config_map_file}")
+            result = subprocess.run(["kubectl", kubectl_config, "apply", "-f", config_map_file],
+                                    check=False, stdout=subprocess.PIPE)
+            print(f"update configmap result={result}")
+            if result.returncode != 0:
+                sys.exit(1)
+    else:
+        print("configmap kubeadm-config already updated")
+
+    print("execute: kubeadm init phase control-plane controller-manager --config"
+          f" {cluster_config_file}")
+    result = subprocess.run(["kubeadm", "init", "phase", "control-plane", "controller-manager",
+                            "--config", cluster_config_file],
+                            check=False, stdout=subprocess.PIPE)
+    print(result)
+    if result.returncode != 0:
+        sys.exit(1)
+
+    with open(cluster_config_file, 'a') as file:
+        file.write(INITCONFIG_TEMPLATE.format(advertise_address))
+    prepend_timestamp_line(cluster_config_file)
+
+    # kubeadm init phase control-plane apiserver --config /tmp/kubeadm-config-cluster.yaml
+    print(f"execute: kubeadm init phase control-plane apiserver --config {cluster_config_file}"
+          f" --apiserver-advertise-address {advertise_address}")
+    result = subprocess.run(["kubeadm", "init", "phase", "control-plane", "apiserver",
+                             "--config", cluster_config_file],
+                            check=False, stdout=subprocess.PIPE)
+    print(result)
+    if result.returncode != 0:
+        sys.exit(1)
+
+print(f"wait {wait} seconds for the restarts")
+time.sleep(wait)
+sys.exit(0)

puppet-manifests/src/bin/dual-stack-kubelet.py

@@ -0,0 +1,141 @@
+#!/usr/bin/python3
+#
+# Copyright (c) 2024 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+''' This script updates the kubelet config to handle single or dual-stack
+'''
+import sys
+import subprocess
+import time
+import yaml
+import netaddr
+
+
+filename = "/etc/default/kubelet"
+
+
+def get_yaml_data(cmd, attempts=15):
+    data = dict()
+    for i in range(0, attempts):
+        res = subprocess.run(cmd, check=False, stdout=subprocess.PIPE)
+        if res.returncode != 0:
+            if i == attempts:
+                print(f"An error occurred getting data, attempt={i}: {res}")
+                sys.exit(1)
+            else:
+                time.sleep(5)
+        else:
+            print("yaml data was collected")
+            data = yaml.load(res.stdout, Loader=yaml.Loader)
+            break
+    return data
+
+
+def is_valid_ip(address):
+    try:
+        if netaddr.valid_ipv4(address):
+            return True
+        if netaddr.valid_ipv6(address):
+            return True
+    except netaddr.AddrFormatError:
+        pass
+    return False
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 5:
+        print("Usage: dual-stack-kubelet.py <node_ip> <node_ip_secondary>"
+              " <restart_wait> <kube-config>")
+        sys.exit(1)
+
+    node_ip = sys.argv[1]
+    node_ip_secondary = sys.argv[2]
+
+    if not is_valid_ip(node_ip):
+        print(f"Error: invalid node_ip '{node_ip}', exit")
+        sys.exit(1)
+
+    if not is_valid_ip(node_ip_secondary):
+        node_ip_secondary = None
+
+    wait = 0
+    try:
+        wait = int(sys.argv[3])
+    except ValueError:
+        print(f"Error: restart_wait='{sys.argv[3]}' cannot be converted to an integer.")
+        sys.exit(1)
+
+    kubectl_config = ''
+    if sys.argv[4]:
+        kubectl_config = f"--kubeconfig={sys.argv[4]}"
+    else:
+        print(f"Error: invalid kubectl config='{kubectl_config}'")
+        sys.exit(1)
+
+    print(f"dual-stack-kubelet {node_ip} {node_ip_secondary} {wait} {kubectl_config}")
+
+    # execute get to test availability of kube-api server
+    print("execute: kubectl -n kube-system get configmap kubeadm-config -o yaml")
+    command = ["kubectl", kubectl_config, "-n", "kube-system",
+               "get", "configmap", "kubeadm-config", "-o", "yaml"]
+    get_yaml_data(command, 20)
+
+    try:
+        # Open the file for reading
+        with open(filename, "r") as file:
+            # Read all lines from the file
+            lines = file.readlines()
+    except FileNotFoundError:
+        print(f"Error: File '{filename}' not found.")
+        sys.exit(1)
+    # Check for empty line
+    if not lines:
+        print(f"filename {filename} is empty")
+        sys.exit(1)
+    line = str()
+    for value in lines:
+        if "KUBELET_EXTRA_ARGS" in value:
+            line = value
+    if not line:
+        print(f"filename {filename} do not contain KUBELET_EXTRA_ARGS")
+        sys.exit(1)
+    kubelet = line.split('=', 1)
+    kubelet_args = kubelet[1].split()
+    need_reconfig = False
+    modified_strings = []
+    for arg in kubelet_args:
+        if "--node-ip" in arg:
+            if node_ip_secondary and arg != f"--node-ip={node_ip},{node_ip_secondary}":
+                modified_strings.append(f"--node-ip={node_ip},{node_ip_secondary}")
+                need_reconfig = True
+            elif not node_ip_secondary and arg != f"--node-ip={node_ip}":
+                modified_strings.append(f"--node-ip={node_ip}")
+                need_reconfig = True
+        else:
+            modified_strings.append(arg)
+    print(f"need_reconfig={need_reconfig}")
+    if (need_reconfig):
+        kubelet_args = modified_strings
+        output_kubelet = str("# Overrides config file for kubelet\nKUBELET_EXTRA_ARGS=")
+        output_kubelet += ' '.join(kubelet_args) + "\n"
+        print(output_kubelet)
+        with open(filename, "w") as file:
+            file.writelines(output_kubelet)
+        print("execute: kubeadm upgrade node phase kubelet-config")
+        result = subprocess.run(['kubeadm', 'upgrade', 'node', 'phase', 'kubelet-config'],
+                                check=False, stdout=subprocess.PIPE)
+        print(result)
+        if result.returncode != 0:
+            sys.exit(1)
+        print("execute: /usr/local/sbin/pmon-restart kubelet")
+        result = subprocess.run(['/usr/local/sbin/pmon-restart', 'kubelet'],
+                                check=False, stdout=subprocess.PIPE)
+        print(result)
+        if result.returncode != 0:
+            sys.exit(1)
+
+print(f"wait {wait} seconds for the restarts")
+time.sleep(wait)
+sys.exit(0)

puppet-manifests/src/bin/dual-stack-kubeproxy.py

@@ -0,0 +1,130 @@
+#!/usr/bin/python3
+#
+# Copyright (c) 2024 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+''' This script updates the kube-proxy config to handle single or dual-stack
+'''
+
+import sys
+import subprocess
+import yaml
+import time
+import netaddr
+
+from datetime import datetime
+
+
+config_map_file = "/tmp/kube-proxy-config.yaml"
+proxy_config_file = "/tmp/kube-proxy-config-data.yaml"
+
+kubectl_config = "--kubeconfig=/etc/kubernetes/admin.conf"
+
+
+def get_yaml_data(cmd, attempts=15):
+    data = dict()
+    for i in range(0, attempts):
+        res = subprocess.run(cmd, check=False, stdout=subprocess.PIPE)
+        if res.returncode != 0:
+            if i == attempts:
+                print(f"An error occurred getting data, attempt={i}: {res}")
+                sys.exit(1)
+            else:
+                time.sleep(5)
+        else:
+            print("yaml data was collected")
+            data = yaml.load(res.stdout, Loader=yaml.Loader)
+            break
+    return data
+
+
+def prepend_timestamp_line(file_name):
+    timestamp_str = datetime.now().strftime(format="%Y-%m-%d %H:%M:%S")
+    with open(file_name, 'r') as read_file:
+        lines = read_file.readlines()
+    lines.insert(0, f"# generated at {timestamp_str}" + "\n")  # Add newline character
+    with open(file_name, 'w') as write_file:
+        write_file.writelines(lines)
+
+
+def is_valid_network(address):
+    """
+    This function checks if the provided string is a valid network address using netaddr.
+    """
+    try:
+        netaddr.IPNetwork(address)
+        return True
+    except netaddr.AddrFormatError as ex:
+        print(f"exception {str(ex)}")
+    return False
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 4:
+        print("Usage: dual-stack-kubeproxy.py <pod_prim_subnet> <pod_sec_subnet> <restart_wait>")
+        sys.exit(1)
+
+    pod_prim_subnet = sys.argv[1]
+    pod_sec_subnet = sys.argv[2]
+
+    if not is_valid_network(pod_prim_subnet):
+        print(f"Error: invalid pod_prim_subnet '{pod_prim_subnet}', exit")
+        sys.exit(1)
+
+    if not is_valid_network(pod_sec_subnet):
+        pod_sec_subnet = None
+
+    wait = 0
+    try:
+        wait = int(sys.argv[3])
+    except ValueError:
+        print(f"Error: restart_wait='{sys.argv[3]}' cannot be converted to an integer.")
+        sys.exit(1)
+
+    print(f"dual-stack-kubeproxy {pod_prim_subnet} {pod_sec_subnet} {wait}")
+
+    print("execute: kubectl -n kube-system get configmap kube-proxy -o yaml")
+    command = ["kubectl", kubectl_config, "-n", "kube-system",
+               "get", "configmap", "kube-proxy", "-o", "yaml"]
+    yaml_data = get_yaml_data(command, 15)
+    proxy_cfg_yaml = yaml_data["data"]["config.conf"]
+    proxy_cfg = yaml.load(proxy_cfg_yaml, Loader=yaml.Loader)
+    if pod_prim_subnet and pod_sec_subnet:
+        proxy_cfg["clusterCIDR"] = f"{pod_prim_subnet},{pod_sec_subnet}"
+    elif pod_prim_subnet and not pod_sec_subnet:
+        proxy_cfg["clusterCIDR"] = f"{pod_prim_subnet}"
+
+    with open(proxy_config_file, 'w') as config_file:
+        yaml.dump(proxy_cfg, config_file, default_flow_style=False)
+
+    proxy_config_str = str()
+    with open(proxy_config_file, "r") as file:
+        proxy_config_lines = file.readlines()
+        for line in proxy_config_lines:
+            proxy_config_str = proxy_config_str + line
+    yaml_data["data"]["config.conf"] = proxy_config_str
+    prepend_timestamp_line(proxy_config_file)
+
+    with open(config_map_file, 'w') as config_file:
+        yaml.dump(yaml_data, config_file, default_flow_style=False)
+    prepend_timestamp_line(config_map_file)
+
+    print(f"execute: kubectl apply -f {config_map_file}")
+    result = subprocess.run(["kubectl", kubectl_config, "apply", "-f", config_map_file],
+                            check=False, stdout=subprocess.PIPE)
+    print(f"update configmap result={result}")
+    if result.returncode != 0:
+        sys.exit(1)
+
+    print("execute: kubectl rollout restart daemonset -n kube-system kube-proxy")
+    result = subprocess.run(["kubectl", kubectl_config, "-n", "kube-system",
+                             "rollout", "restart", "daemonset", "kube-proxy"],
+                            check=False, stdout=subprocess.PIPE)
+    print(result)
+    if result.returncode != 0:
+        sys.exit(1)
+
+print(f"wait {wait} seconds for the restarts")
+time.sleep(wait)
+sys.exit(0)

puppet-manifests/src/bin/dual-stack-multus.py

@@ -0,0 +1,112 @@
+#!/usr/bin/python3
+#
+# Copyright (c) 2024 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+''' This script updates the multus config to handle single or dual-stack
+'''
+
+import sys
+import subprocess
+import time
+import re
+from datetime import datetime
+
+multus_config_map_file = "/tmp/multus-configmap.yaml"
+kubectl_config = "--kubeconfig=/etc/kubernetes/admin.conf"
+
+
+def prepend_timestamp_line(file_name):
+    timestamp_str = datetime.now().strftime(format="%Y-%m-%d %H:%M:%S")
+    with open(file_name, 'r') as read_file:
+        lines = read_file.readlines()
+    lines.insert(0, f"# generated at {timestamp_str}" + "\n")  # Add newline character
+    with open(file_name, 'w') as write_file:
+        write_file.writelines(lines)
+
+
+def save_configmap(cmd, config_map_file):
+
+    for i in range(0, 9):
+        res = subprocess.run(cmd, check=False, capture_output=True)
+        if res.returncode == 0:
+            output = res.stdout.decode()
+            with open(config_map_file, "wb") as output_file:
+                output_file.write(output.encode())
+                print(f"Successfully saved configmap to {config_map_file}")
+                break
+        else:
+            if i == 9:
+                print(f"An error occurred getting data, attempt={i}: {res}")
+                sys.exit(1)
+            else:
+                time.sleep(5)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 4:
+        print("Usage: dual-stack-multus.py <protocol> <state> <restart_wait>")
+        sys.exit(1)
+
+    protocol = sys.argv[1].lower()
+    if protocol not in ["ipv4", "ipv6"]:
+        print("invalid IP protocol")
+        sys.exit(1)
+
+    state = sys.argv[2].lower()
+    if state not in ["true", "false"]:
+        print("invalid state for protocol")
+        sys.exit(1)
+
+    wait = 0
+    try:
+        wait = int(sys.argv[3])
+    except ValueError:
+        print(f"Error: restart_wait='{sys.argv[3]}' cannot be converted to an integer.")
+        sys.exit(1)
+
+    print(f"dual-stack-multus {protocol} {state} {wait}")
+
+    print("execute: kubectl get cm -n kube-system multus-cni-config.v1 -o yaml")
+    command = ["kubectl", kubectl_config, "-n", "kube-system",
+               "get", "cm", "multus-cni-config.v1", "-o", "yaml"]
+    save_configmap(command, multus_config_map_file)
+
+    modified_data = str()
+    with open(multus_config_map_file, "r") as file:
+        multus_config_map = file.read()
+        if protocol == "ipv4":
+            match = re.search(r'"assign_ipv4": "(true|false)"', multus_config_map)
+            if match:
+                new_val = f'\"assign_ipv4\": \"{state}\"'
+                modified_data = multus_config_map.replace(match.group(), new_val)
+        elif protocol == "ipv6":
+            match = re.search(r'"assign_ipv6": "(true|false)"', multus_config_map)
+            if match:
+                new_val = f'\"assign_ipv6\": \"{state}\"'
+                modified_data = multus_config_map.replace(match.group(), new_val)
+
+    with open(multus_config_map_file, "w") as file:
+        file.write(modified_data)
+    prepend_timestamp_line(multus_config_map_file)
+
+    print(f"execute: kubectl apply -f {multus_config_map_file}")
+    result = subprocess.run(["kubectl", kubectl_config,
+                             "apply", "-f", multus_config_map_file],
+                            check=False, stdout=subprocess.PIPE)
+    print(f"update multus configmap result={result}")
+    if result.returncode != 0:
+        sys.exit(1)
+
+    print("execute: kubectl rollout restart daemonset/multus -n kube-system")
+    result = subprocess.run(["kubectl", kubectl_config, "-n", "kube-system",
+                             "rollout", "restart", "daemonset", "kube-multus-ds-amd64"],
+                            check=False, stdout=subprocess.PIPE)
+    print(result)
+    if result.returncode != 0:
+        sys.exit(1)
+
+print(f"wait {wait} seconds for the restarts")
+time.sleep(wait)
+sys.exit(0)

puppet-manifests/src/modules/platform/manifests/kubernetes.pp

@@ -7,6 +7,7 @@ class platform::kubernetes::params (
   $kubeadm_version = undef,
   $kubelet_version = undef,
   $node_ip = undef,
+  $node_ip_secondary = undef,
   $service_domain = undef,
   $apiserver_cluster_ip = undef,
   $dns_service_ip = undef,
@@ -266,6 +267,7 @@ class platform::kubernetes::kubeadm {
   require platform::kubernetes::symlinks
 
   $node_ip = $::platform::kubernetes::params::node_ip
+  $node_ip_secondary = $::platform::kubernetes::params::node_ip_secondary
   $host_labels = $::platform::kubernetes::params::host_labels
   $k8s_platform_cpuset = $::platform::kubernetes::params::k8s_platform_cpuset
   $k8s_reserved_mem = $::platform::kubernetes::params::k8s_reserved_mem
@@ -1105,30 +1107,134 @@ class platform::kubernetes::certsans::runtime
   inherits ::platform::kubernetes::params {
   include ::platform::params
   include ::platform::network::mgmt::params
+  include ::platform::network::mgmt::ipv4::params
+  include ::platform::network::mgmt::ipv6::params
   include ::platform::network::oam::params
+  include ::platform::network::oam::ipv4::params
+  include ::platform::network::oam::ipv6::params
   include ::platform::network::cluster_host::params
+  include ::platform::network::cluster_host::ipv4::params
+  include ::platform::network::cluster_host::ipv6::params
 
-  if $::platform::network::mgmt::params::subnet_version == $::platform::params::ipv6 {
+  $ipv4_val = $::platform::params::ipv4
+  $ipv6_val = $::platform::params::ipv6
+
+  $prim_mgmt_subnet_ver = $::platform::network::mgmt::params::subnet_version
+  $ipv4_mgmt_subnet_ver = $::platform::network::mgmt::ipv4::params::subnet_version
+  $ipv6_mgmt_subnet_ver = $::platform::network::mgmt::ipv6::params::subnet_version
+  if $prim_mgmt_subnet_ver == $ipv6_val and $ipv4_mgmt_subnet_ver != undef {
+    $sec_mgmt_subnet_ver = $ipv4_mgmt_subnet_ver
+  } elsif $prim_mgmt_subnet_ver == $ipv4_val and $ipv6_mgmt_subnet_ver != undef {
+    $sec_mgmt_subnet_ver = $ipv6_mgmt_subnet_ver
+  } else {
+    $sec_mgmt_subnet_ver = undef
+  }
+
+  $prim_cluster_host_subnet_ver = $::platform::network::cluster_host::params::subnet_version
+  $ipv4_cluster_host_subnet_ver = $::platform::network::cluster_host::ipv4::params::subnet_version
+  $ipv6_cluster_host_subnet_ver = $::platform::network::cluster_host::ipv6::params::subnet_version
+  if $prim_cluster_host_subnet_ver == $ipv6_val and $ipv4_cluster_host_subnet_ver != undef {
+    $sec_cluster_host_subnet_ver = $ipv4_cluster_host_subnet_ver
+  } elsif $prim_cluster_host_subnet_ver == $ipv4_val and $ipv6_cluster_host_subnet_ver != undef {
+    $sec_cluster_host_subnet_ver = $ipv6_cluster_host_subnet_ver
+  } else {
+    $sec_cluster_host_subnet_ver = undef
+  }
+
+  $prim_oam_subnet_ver = $::platform::network::oam::params::subnet_version
+  $ipv4_oam_subnet_ver = $::platform::network::oam::ipv4::params::subnet_version
+  $ipv6_oam_subnet_ver = $::platform::network::oam::ipv6::params::subnet_version
+  if $prim_oam_subnet_ver == $ipv6_val and $ipv4_oam_subnet_ver != undef {
+    $sec_oam_subnet_ver = $ipv4_oam_subnet_ver
+  } elsif $prim_oam_subnet_ver == $ipv4_val and $ipv6_oam_subnet_ver != undef {
+    $sec_oam_subnet_ver = $ipv6_oam_subnet_ver
+  } else {
+    $sec_oam_subnet_ver = undef
+  }
+
+  if $::platform::network::mgmt::params::subnet_version == $ipv6_val {
     $localhost_address = '::1'
   } else {
     $localhost_address = '127.0.0.1'
   }
+  if $sec_mgmt_subnet_ver != undef {
+    if $sec_mgmt_subnet_ver == $ipv4_val {
+      $certsans_sec_localhost = ',127.0.0.1'
+    } elsif $sec_mgmt_subnet_ver == $ipv6_val {
+      $certsans_sec_localhost = ',::1'
+    }
+  } else {
+    $certsans_sec_localhost = ''
+  }
 
   if $::platform::params::system_mode == 'simplex' {
-    $certsans = "\"${platform::network::cluster_host::params::controller_address}, \
+    $certsans_prim = "${platform::network::cluster_host::params::controller_address}, \
-                   ${platform::network::cluster_host::params::controller0_address}, \
+                  ${platform::network::cluster_host::params::controller0_address}, \
-                   ${localhost_address}, \
+                  ${localhost_address}, \
-                   ${platform::network::oam::params::controller_address}\""
+                  ${platform::network::oam::params::controller_address}"
+
+    if $sec_oam_subnet_ver == $ipv4_val {
+      $certsans_oam_sec = ",${platform::network::oam::ipv4::params::controller_address}"
+    } elsif $sec_oam_subnet_ver == $ipv6_val {
+      $certsans_oam_sec = ",${platform::network::oam::ipv6::params::controller_address}"
+    } else {
+      $certsans_oam_sec = ''
+    }
+
+    if $sec_cluster_host_subnet_ver == $ipv4_val {
+      $certsans_cluster_host_sec = ",${platform::network::cluster_host::ipv4::params::controller_address}, \
+                                     ${platform::network::cluster_host::ipv4::params::controller0_address}"
+    } elsif $sec_cluster_host_subnet_ver == $ipv6_val {
+      $certsans_cluster_host_sec = ",${platform::network::cluster_host::ipv6::params::controller_address}, \
+                                     ${platform::network::cluster_host::ipv6::params::controller0_address}"
+    } else {
+      $certsans_cluster_host_sec = ''
+    }
+
+    $certsans_sec_hosts = "${certsans_oam_sec}${certsans_cluster_host_sec}"
+
+    $certsans_sec = "${certsans_sec_hosts}${certsans_sec_localhost}"
+
   } else {
-    $certsans = "\"${platform::network::cluster_host::params::controller_address}, \
+    $certsans_prim = "${platform::network::cluster_host::params::controller_address}, \
-                   ${platform::network::cluster_host::params::controller0_address}, \
+                  ${platform::network::cluster_host::params::controller0_address}, \
-                   ${platform::network::cluster_host::params::controller1_address}, \
+                  ${platform::network::cluster_host::params::controller1_address}, \
-                   ${localhost_address}, \
+                  ${localhost_address}, \
-                   ${platform::network::oam::params::controller_address}, \
+                  ${platform::network::oam::params::controller_address}, \
-                   ${platform::network::oam::params::controller0_address}, \
+                  ${platform::network::oam::params::controller0_address}, \
-                   ${platform::network::oam::params::controller1_address}\""
+                  ${platform::network::oam::params::controller1_address}"
+
+    if $sec_oam_subnet_ver == $ipv4_val {
+      $certsans_oam_sec = ",${platform::network::oam::ipv4::params::controller_address}, \
+                            ${platform::network::oam::ipv4::params::controller0_address}, \
+                            ${platform::network::oam::ipv4::params::controller1_address}"
+    } elsif $sec_oam_subnet_ver == $ipv6_val {
+      $certsans_oam_sec = ",${platform::network::oam::ipv6::params::controller_address}, \
+                            ${platform::network::oam::ipv6::params::controller0_address}, \
+                            ${platform::network::oam::ipv6::params::controller1_address}"
+    } else {
+      $certsans_oam_sec = ''
+    }
+
+    if $sec_cluster_host_subnet_ver == $ipv4_val {
+      $certsans_cluster_host_sec = ",${platform::network::cluster_host::ipv4::params::controller_address}, \
+                                     ${platform::network::cluster_host::ipv4::params::controller0_address}, \
+                                     ${platform::network::cluster_host::ipv4::params::controller1_address}"
+    } elsif $sec_cluster_host_subnet_ver == $ipv6_val {
+      $certsans_cluster_host_sec = ",${platform::network::cluster_host::ipv6::params::controller_address}, \
+                                     ${platform::network::cluster_host::ipv6::params::controller0_address}, \
+                                     ${platform::network::cluster_host::ipv6::params::controller1_address}"
+    } else {
+      $certsans_cluster_host_sec = ''
+    }
+
+    $certsans_sec_hosts = "${certsans_oam_sec}${certsans_cluster_host_sec}"
+
+    $certsans_sec = "${certsans_sec_hosts}${certsans_sec_localhost}"
   }
 
+  $certsans = "\"${certsans_prim}${certsans_sec}\""
+
   exec { 'update kube-apiserver certSANs':
     provider => shell,
     command  => template('platform/kube-apiserver-update-certSANs.erb')
@@ -1856,3 +1962,293 @@ class platform::kubernetes::upgrade_abort_recovery
       command   => "kubectl --kubeconfig=/etc/kubernetes/admin.conf uncordon ${::platform::params::hostname}",
   }
 }
+
+class platform::kubernetes::kubelet::update_node_ip::runtime
+  inherits ::platform::kubernetes::params {
+  # lint:ignore:140chars
+  if $::personality == 'worker' or $::personality == 'controller' {
+
+    $node_ip = $::platform::kubernetes::params::node_ip
+    if $::platform::kubernetes::params::node_ip_secondary {
+      $node_ip_secondary = $::platform::kubernetes::params::node_ip_secondary
+    } else {
+      $node_ip_secondary = 'undef'
+    }
+    $restart_wait = '5'
+
+    if $::personality == 'worker' {
+      $cfgf = '/etc/kubernetes/kubelet.conf'
+    } elsif $::personality == 'controller' {
+      $cfgf = '/etc/kubernetes/admin.conf'
+    }
+
+    exec { 'kubelet-update-node-ip':
+      path      => '/usr/bin:/usr/sbin:/bin:/usr/local/bin',
+      command   => "dual-stack-kubelet.py ${node_ip} ${node_ip_secondary} ${restart_wait} ${cfgf}",
+      logoutput => true,
+    }
+  }
+  # lint:endignore:140chars
+}
+
+class platform::kubernetes::kubeadm::dual_stack::ipv4::runtime {
+  # lint:ignore:140chars
+  include ::platform::network::cluster_pod::params
+  include ::platform::network::cluster_pod::ipv4::params
+  include ::platform::network::cluster_service::params
+  include ::platform::network::cluster_service::ipv4::params
+  include ::platform::network::cluster_host::ipv4::params
+  if $::personality == 'controller' {
+    $restart_wait = '5'
+
+    $pod_prim_network = $::platform::network::cluster_pod::params::subnet_network
+    $pod_prim_prefixlen = $::platform::network::cluster_pod::params::subnet_prefixlen
+    $pod_prim_subnet = "${pod_prim_network}/${pod_prim_prefixlen}"
+
+    if $platform::network::cluster_pod::ipv4::params::subnet_version == $::platform::params::ipv4  {
+      $pod_sec_network = $::platform::network::cluster_pod::ipv4::params::subnet_network
+      $pod_sec_prefixlen = $::platform::network::cluster_pod::ipv4::params::subnet_prefixlen
+      $pod_sec_subnet = "${pod_sec_network}/${pod_sec_prefixlen}"
+    } else {
+      $pod_sec_subnet = 'undef'
+    }
+
+    $svc_prim_network = $::platform::network::cluster_service::params::subnet_network
+    $svc_prim_prefixlen = $::platform::network::cluster_service::params::subnet_prefixlen
+    $svc_prim_subnet = "${svc_prim_network}/${svc_prim_prefixlen}"
+
+    if $platform::network::cluster_service::ipv4::params::subnet_version == $::platform::params::ipv4 {
+      $svc_sec_network = $::platform::network::cluster_service::ipv4::params::subnet_network
+      $svc_sec_prefixlen = $::platform::network::cluster_service::ipv4::params::subnet_prefixlen
+      $svc_sec_subnet = "${svc_sec_network}/${svc_sec_prefixlen}"
+    } else {
+      $svc_sec_subnet = 'undef'
+    }
+
+    if $::platform::params::hostname == 'controller-0' {
+      $cluster_host_addr = $::platform::network::cluster_host::params::controller0_address
+    } else {
+      $cluster_host_addr = $::platform::network::cluster_host::params::controller1_address
+    }
+
+    exec { 'update kubeadm pod and service secondary IPv6 subnets':
+      path      => '/usr/bin:/usr/sbin:/bin:/usr/local/bin',
+      command   => "dual-stack-kubeadm.py ${pod_prim_subnet} ${svc_prim_subnet} ${pod_sec_subnet} ${svc_sec_subnet} ${restart_wait} ${cluster_host_addr}",
+      logoutput => true,
+    }
+  }
+  # lint:endignore:140chars
+}
+
+class platform::kubernetes::kubeadm::dual_stack::ipv6::runtime {
+  # lint:ignore:140chars
+  include ::platform::network::cluster_pod::params
+  include ::platform::network::cluster_pod::ipv6::params
+  include ::platform::network::cluster_service::params
+  include ::platform::network::cluster_service::ipv6::params
+  include ::platform::network::cluster_host::ipv6::params
+  if $::personality == 'controller' {
+    $restart_wait = '5'
+
+    $pod_prim_network = $::platform::network::cluster_pod::params::subnet_network
+    $pod_prim_prefixlen = $::platform::network::cluster_pod::params::subnet_prefixlen
+    $pod_prim_subnet = "${pod_prim_network}/${pod_prim_prefixlen}"
+
+    if $platform::network::cluster_pod::ipv6::params::subnet_version == $::platform::params::ipv6  {
+      $pod_sec_network = $::platform::network::cluster_pod::ipv6::params::subnet_network
+      $pod_sec_prefixlen = $::platform::network::cluster_pod::ipv6::params::subnet_prefixlen
+      $pod_sec_subnet = "${pod_sec_network}/${pod_sec_prefixlen}"
+    } else {
+      $pod_sec_subnet = 'undef'
+    }
+
+    $svc_prim_network = $::platform::network::cluster_service::params::subnet_network
+    $svc_prim_prefixlen = $::platform::network::cluster_service::params::subnet_prefixlen
+    $svc_prim_subnet = "${svc_prim_network}/${svc_prim_prefixlen}"
+
+    if $platform::network::cluster_service::ipv6::params::subnet_version == $::platform::params::ipv6 {
+      $svc_sec_network = $::platform::network::cluster_service::ipv6::params::subnet_network
+      $svc_sec_prefixlen = $::platform::network::cluster_service::ipv6::params::subnet_prefixlen
+      $svc_sec_subnet = "${svc_sec_network}/${svc_sec_prefixlen}"
+    } else {
+      $svc_sec_subnet = 'undef'
+    }
+
+    if $::platform::params::hostname == 'controller-0' {
+      $cluster_host_addr = $::platform::network::cluster_host::params::controller0_address
+    } else {
+      $cluster_host_addr = $::platform::network::cluster_host::params::controller1_address
+    }
+
+    exec { 'update kubeadm pod and service secondary IPv6 subnets':
+      path      => '/usr/bin:/usr/sbin:/bin:/usr/local/bin',
+      command   => "dual-stack-kubeadm.py ${pod_prim_subnet} ${svc_prim_subnet} ${pod_sec_subnet} ${svc_sec_subnet} ${restart_wait} ${cluster_host_addr}",
+      logoutput => true,
+    }
+  }
+  # lint:endignore:140chars
+}
+
+class platform::kubernetes::dual_stack::ipv4::runtime {
+  # lint:ignore:140chars
+  # adds/removes secondary IPv4 subnets to pod and service
+  include ::platform::network::cluster_pod::params
+  include ::platform::network::cluster_pod::ipv4::params
+  include ::platform::network::cluster_service::params
+  include ::platform::network::cluster_service::ipv4::params
+  include ::platform::network::cluster_host::ipv4::params
+
+  $protocol = 'ipv4'
+  $def_pool_filename = "/tmp/def_pool_${protocol}.yaml"
+  $kubeconfig = '--kubeconfig=/etc/kubernetes/admin.conf'
+  $restart_wait = '5'
+
+  $pod_prim_network = $::platform::network::cluster_pod::params::subnet_network
+  $pod_prim_prefixlen = $::platform::network::cluster_pod::params::subnet_prefixlen
+  $pod_prim_subnet = "${pod_prim_network}/${pod_prim_prefixlen}"
+
+  if $platform::network::cluster_pod::ipv4::params::subnet_version == $::platform::params::ipv4  {
+    $pod_sec_network = $::platform::network::cluster_pod::ipv4::params::subnet_network
+    $pod_sec_prefixlen = $::platform::network::cluster_pod::ipv4::params::subnet_prefixlen
+    $pod_sec_subnet = "${pod_sec_network}/${pod_sec_prefixlen}"
+    $c0_addr = $::platform::network::cluster_host::ipv4::params::controller0_address
+    $state = true
+  } else {
+    $pod_sec_subnet = 'undef'
+    $state = false
+    $c0_addr = '::'
+  }
+
+  $svc_prim_network = $::platform::network::cluster_service::params::subnet_network
+  $svc_prim_prefixlen = $::platform::network::cluster_service::params::subnet_prefixlen
+  $svc_prim_subnet = "${svc_prim_network}/${svc_prim_prefixlen}"
+
+  if $platform::network::cluster_service::ipv4::params::subnet_version == $::platform::params::ipv4 {
+    $svc_sec_network = $::platform::network::cluster_service::ipv4::params::subnet_network
+    $svc_sec_prefixlen = $::platform::network::cluster_service::ipv4::params::subnet_prefixlen
+    $svc_sec_subnet = "${svc_sec_network}/${svc_sec_prefixlen}"
+  } else {
+    $svc_sec_subnet = 'undef'
+  }
+
+  exec { 'update kube-proxy pod secondary IPv6 subnet':
+    path      => '/usr/bin:/usr/sbin:/bin:/usr/local/bin',
+    command   => "dual-stack-kubeproxy.py ${pod_prim_subnet} ${pod_sec_subnet} ${restart_wait}",
+    logoutput => true,
+  }
+  -> exec { 'update calico node pod secondary IPv6 subnet':
+    path      => '/usr/bin:/usr/sbin:/bin:/usr/local/bin',
+    command   => "dual-stack-calico.py ${protocol} ${state} ${c0_addr} ${restart_wait}",
+    logoutput => true,
+  }
+  if $state == true {
+    $ipip_mode = 'Always'
+    file { $def_pool_filename:
+      ensure  => file,
+      content => template('platform/callico_ippool.yaml.erb'),
+      owner   => 'root',
+      group   => 'root',
+      mode    => '0640',
+    }
+    -> exec { "create default-${protocol}-ippool":
+      path      => '/usr/bin:/usr/sbin:/bin:/usr/local/bin',
+      command   => "kubectl ${kubeconfig} apply -f ${def_pool_filename}",
+      logoutput => true
+    }
+  } else {
+    exec { "delete default-${protocol}-ippool":
+      path      => '/usr/bin:/usr/sbin:/bin:/usr/local/bin',
+      command   => "kubectl ${kubeconfig} delete ippools.crd.projectcalico.org default-${protocol}-ippool",
+      logoutput => true,
+      onlyif    => "kubectl ${kubeconfig} get ippools.crd.projectcalico.org default-${protocol}-ippool ",
+    }
+  }
+  exec { 'update multus to support IPv4':
+    path      => '/usr/bin:/usr/sbin:/bin:/usr/local/bin',
+    command   => "dual-stack-multus.py ${protocol} ${state} ${restart_wait}",
+    logoutput => true,
+  }
+  # lint:endignore:140chars
+}
+
+class platform::kubernetes::dual_stack::ipv6::runtime {
+  # lint:ignore:140chars
+  # adds/removes secondary IPv6 subnets to pod and service
+  include ::platform::network::cluster_pod::params
+  include ::platform::network::cluster_pod::ipv6::params
+  include ::platform::network::cluster_service::params
+  include ::platform::network::cluster_service::ipv6::params
+  include ::platform::network::cluster_host::ipv6::params
+
+  $protocol = 'ipv6'
+  $def_pool_filename = "/tmp/def_pool_${protocol}.yaml"
+  $kubeconfig = '--kubeconfig=/etc/kubernetes/admin.conf'
+  $restart_wait = '10'
+
+  $pod_prim_network = $::platform::network::cluster_pod::params::subnet_network
+  $pod_prim_prefixlen = $::platform::network::cluster_pod::params::subnet_prefixlen
+  $pod_prim_subnet = "${pod_prim_network}/${pod_prim_prefixlen}"
+
+  if $platform::network::cluster_pod::ipv6::params::subnet_version == $::platform::params::ipv6  {
+    $pod_sec_network = $::platform::network::cluster_pod::ipv6::params::subnet_network
+    $pod_sec_prefixlen = $::platform::network::cluster_pod::ipv6::params::subnet_prefixlen
+    $pod_sec_subnet = "${pod_sec_network}/${pod_sec_prefixlen}"
+    $c0_addr = $::platform::network::cluster_host::ipv6::params::controller0_address
+    $state = true
+  } else {
+    $pod_sec_subnet = 'undef'
+    $state = false
+    $c0_addr = '::'
+  }
+
+  $svc_prim_network = $::platform::network::cluster_service::params::subnet_network
+  $svc_prim_prefixlen = $::platform::network::cluster_service::params::subnet_prefixlen
+  $svc_prim_subnet = "${svc_prim_network}/${svc_prim_prefixlen}"
+
+  if $platform::network::cluster_service::ipv6::params::subnet_version == $::platform::params::ipv6 {
+    $svc_sec_network = $::platform::network::cluster_service::ipv6::params::subnet_network
+    $svc_sec_prefixlen = $::platform::network::cluster_service::ipv6::params::subnet_prefixlen
+    $svc_sec_subnet = "${svc_sec_network}/${svc_sec_prefixlen}"
+  } else {
+    $svc_sec_subnet = 'undef'
+  }
+
+  exec { 'update kube-proxy pod secondary IPv6 subnet':
+    path      => '/usr/bin:/usr/sbin:/bin:/usr/local/bin',
+    command   => "dual-stack-kubeproxy.py ${pod_prim_subnet} ${pod_sec_subnet} ${restart_wait}",
+    logoutput => true,
+  }
+  -> exec { 'update calico node pod secondary IPv6 subnet':
+    path      => '/usr/bin:/usr/sbin:/bin:/usr/local/bin',
+    command   => "dual-stack-calico.py ${protocol} ${state} ${c0_addr} ${restart_wait}",
+    logoutput => true,
+  }
+  if $state == true {
+    $ipip_mode = 'Never'
+    file { $def_pool_filename:
+      ensure  => file,
+      content => template('platform/callico_ippool.yaml.erb'),
+      owner   => 'root',
+      group   => 'root',
+      mode    => '0640',
+    }
+    -> exec { "create default-${protocol}-ippool":
+      path      => '/usr/bin:/usr/sbin:/bin:/usr/local/bin',
+      command   => "kubectl ${kubeconfig} apply -f ${def_pool_filename}",
+      logoutput => true,
+    }
+  } else {
+    exec { "delete default-${protocol}-ippool":
+      path      => '/usr/bin:/usr/sbin:/bin:/usr/local/bin',
+      command   => "kubectl ${kubeconfig} delete ippools.crd.projectcalico.org default-${protocol}-ippool",
+      logoutput => true,
+      onlyif    => "kubectl ${kubeconfig} get ippools.crd.projectcalico.org default-${protocol}-ippool "
+    }
+  }
+  exec { 'update multus to support IPv6':
+    path      => '/usr/bin:/usr/sbin:/bin:/usr/local/bin',
+    command   => "dual-stack-multus.py ${protocol} ${state} ${restart_wait}",
+    logoutput => true,
+  }
+  # lint:endignore:140chars
+}

puppet-manifests/src/modules/platform/manifests/network.pp

@@ -223,6 +223,116 @@ class platform::network::cluster_host::params(
   $mtu = 1500,
 ) { }
 
+class platform::network::cluster_pod::ipv4::params(
+  # shared parameters with base class - required for auto hiera parameter lookup
+  $interface_address = undef,
+  $subnet_version = undef,
+  $subnet_network = undef,
+  $subnet_network_url = undef,
+  $subnet_prefixlen = undef,
+  $subnet_netmask = undef,
+  $subnet_start = undef,
+  $subnet_end = undef,
+  $gateway_address = undef,
+  $controller_address = undef,  # controller floating
+  $controller_address_url = undef,  # controller floating url address
+  $controller0_address = undef, # controller unit0
+  $controller1_address = undef, # controller unit1
+) { }
+
+class platform::network::cluster_pod::ipv6::params(
+  # shared parameters with base class - required for auto hiera parameter lookup
+  $interface_address = undef,
+  $subnet_version = undef,
+  $subnet_network = undef,
+  $subnet_network_url = undef,
+  $subnet_prefixlen = undef,
+  $subnet_netmask = undef,
+  $subnet_start = undef,
+  $subnet_end = undef,
+  $gateway_address = undef,
+  $controller_address = undef,  # controller floating
+  $controller_address_url = undef,  # controller floating url address
+  $controller0_address = undef, # controller unit0
+  $controller1_address = undef, # controller unit1
+) { }
+
+class platform::network::cluster_pod::params(
+  # this class contains the primary pool (ipv4 or ipv6) addresses for compatibility
+  # shared parameters with base class - required for auto hiera parameter lookup
+  $interface_name = undef,
+  $interface_address = undef,
+  $interface_devices = [],
+  $subnet_version = undef,
+  $subnet_network = undef,
+  $subnet_network_url = undef,
+  $subnet_prefixlen = undef,
+  $subnet_netmask = undef,
+  $subnet_start = undef,
+  $subnet_end = undef,
+  $gateway_address = undef,
+  $controller_address = undef,  # controller floating
+  $controller_address_url = undef,  # controller floating url address
+  $controller0_address = undef, # controller unit0
+  $controller1_address = undef, # controller unit1
+  $mtu = 1500,
+) { }
+
+class platform::network::cluster_service::ipv4::params(
+  # shared parameters with base class - required for auto hiera parameter lookup
+  $interface_address = undef,
+  $subnet_version = undef,
+  $subnet_network = undef,
+  $subnet_network_url = undef,
+  $subnet_prefixlen = undef,
+  $subnet_netmask = undef,
+  $subnet_start = undef,
+  $subnet_end = undef,
+  $gateway_address = undef,
+  $controller_address = undef,  # controller floating
+  $controller_address_url = undef,  # controller floating url address
+  $controller0_address = undef, # controller unit0
+  $controller1_address = undef, # controller unit1
+) { }
+
+class platform::network::cluster_service::ipv6::params(
+  # shared parameters with base class - required for auto hiera parameter lookup
+  $interface_address = undef,
+  $subnet_version = undef,
+  $subnet_network = undef,
+  $subnet_network_url = undef,
+  $subnet_prefixlen = undef,
+  $subnet_netmask = undef,
+  $subnet_start = undef,
+  $subnet_end = undef,
+  $gateway_address = undef,
+  $controller_address = undef,  # controller floating
+  $controller_address_url = undef,  # controller floating url address
+  $controller0_address = undef, # controller unit0
+  $controller1_address = undef, # controller unit1
+) { }
+
+class platform::network::cluster_service::params(
+  # this class contains the primary pool (ipv4 or ipv6) addresses for compatibility
+  # shared parameters with base class - required for auto hiera parameter lookup
+  $interface_name = undef,
+  $interface_address = undef,
+  $interface_devices = [],
+  $subnet_version = undef,
+  $subnet_network = undef,
+  $subnet_network_url = undef,
+  $subnet_prefixlen = undef,
+  $subnet_netmask = undef,
+  $subnet_start = undef,
+  $subnet_end = undef,
+  $gateway_address = undef,
+  $controller_address = undef,  # controller floating
+  $controller_address_url = undef,  # controller floating url address
+  $controller0_address = undef, # controller unit0
+  $controller1_address = undef, # controller unit1
+  $mtu = 1500,
+) { }
+
 class platform::network::ironic::ipv4::params(
   # shared parameters with base class - required for auto hiera parameter lookup
   $interface_address = undef,

puppet-manifests/src/modules/platform/manifests/sm.pp

@@ -1526,6 +1526,7 @@ class platform::sm::enable_admin_config::runtime {
   include ::platform::network::admin::params
   include ::platform::network::admin::ipv4::params
   include ::platform::network::admin::ipv6::params
+
   $admin_ip_interface  = $::platform::network::admin::params::interface_name
   if $::platform::network::admin::ipv4::params::subnet_version == $::platform::params::ipv4 {
     exec { 'Manage admin-ipv4 service':
@@ -1579,6 +1580,7 @@ class platform::sm::update_admin_config::runtime {
   include ::platform::network::admin::params
   include ::platform::network::admin::ipv4::params
   include ::platform::network::admin::ipv6::params
+
   $admin_ip_interface  = $::platform::network::admin::params::interface_name
   $admin_ip_param_ip   = $::platform::network::admin::params::controller_address
   $admin_ip_param_mask = $::platform::network::admin::params::subnet_prefixlen
@@ -1628,6 +1630,85 @@ class platform::sm::update_admin_config::runtime {
   # lint:endignore:140chars
 }
 
+class platform::sm::cluster_host::add_ip_config::ipv4::runtime {
+  # lint:ignore:140chars
+  include ::platform::network::cluster_host::ipv4::params
+
+  if $::personality == 'controller' {
+    if $::platform::network::cluster_host::ipv4::params::subnet_version == $::platform::params::ipv4 {
+      $interface = $::platform::network::cluster_host::params::interface_name
+      $ip        = $::platform::network::cluster_host::ipv4::params::controller_address
+      $mask      = $::platform::network::cluster_host::ipv4::params::subnet_prefixlen
+      exec { 'Reconfigure Cluster-Host IPv4 service_instance':
+        command => "sm-configure service_instance cluster-host-ipv4 cluster_host-ipv4 \"ip=${ip},cidr_netmask=${mask},nic=${interface},arp_count=7\" --apply",
+      }
+      -> exec { 'Manage cluster_host-ipv4 service':
+          command => 'sm-manage service cluster-host-ipv4'
+      }
+      -> exec { 'Provision cluster-host-ipv4 service':
+        command => 'sm-provision service-group-member controller-services cluster-host-ipv4 --apply'
+      }
+    }
+  }
+  # lint:endignore:140chars
+}
+
+class platform::sm::cluster_host::remove_ip_config::ipv4::runtime {
+  # lint:ignore:140chars
+  if $::personality == 'controller' {
+    exec { 'Unmanage cluster-host-ipv4 service':
+      command => 'sm-unmanage service cluster-host-ipv4',
+      onlyif  => 'sm-dump | grep cluster-host-ipv4',
+    }
+    -> exec { 'Deprovision cluster-host-ipv4 service':
+      command => 'sm-deprovision service-group-member controller-services cluster-host-ipv4 --apply',
+      onlyif  => 'sm-dump | grep cluster-host-ipv4',
+    }
+  }
+  # lint:endignore:140chars
+}
+
+class platform::sm::cluster_host::add_ip_config::ipv6::runtime {
+  # lint:ignore:140chars
+  include ::platform::network::cluster_host::ipv6::params
+
+  if $::personality == 'controller' {
+    if $::platform::network::cluster_host::ipv6::params::subnet_version == $::platform::params::ipv6 {
+      $interface = $::platform::network::cluster_host::params::interface_name
+      $ip        = $::platform::network::cluster_host::ipv6::params::controller_address
+      $mask      = $::platform::network::cluster_host::ipv6::params::subnet_prefixlen
+      exec { 'Reconfigure Cluster-Host IPv6 service_instance':
+        command => "sm-configure service_instance cluster-host-ipv6 cluster_host-ipv6 \"ip=${ip},cidr_netmask=${mask},nic=${interface},arp_count=7\" --apply",
+      }
+      -> exec { 'Manage cluster_host-ipv6 service':
+          command => 'sm-manage service cluster-host-ipv6'
+      }
+      -> exec { 'Provision cluster-host-ipv6 service':
+        command => 'sm-provision service-group-member controller-services cluster-host-ipv6 --apply'
+      }
+    }
+  }
+  # lint:endignore:140chars
+}
+
+class platform::sm::cluster_host::remove_ip_config::ipv6::runtime {
+  # lint:ignore:140chars
+  include ::platform::network::cluster_host::params
+  include ::platform::network::cluster_host::ipv6::params
+
+  if $::personality == 'controller' {
+    exec { 'Unmanage cluster-host-ipv6 service':
+      command => 'sm-unmanage service cluster-host-ipv6',
+      onlyif  => 'sm-dump | grep cluster-host-ipv6',
+    }
+    -> exec { 'Deprovision cluster-host-ipv6 service':
+      command => 'sm-deprovision service-group-member controller-services cluster-host-ipv6 --apply',
+      onlyif  => 'sm-dump | grep cluster-host-ipv6',
+    }
+  }
+  # lint:endignore:140chars
+}
+
 define platform::sm::restart {
   exec {"sm-restart-${name}":
     command => "sm-restart-safe service ${name}",

puppet-manifests/src/modules/platform/templates/callico_ippool.yaml.erb

@@ -0,0 +1,11 @@
+# Calico default ip pool
+
+apiVersion: crd.projectcalico.org/v1
+kind: IPPool
+metadata:
+  name: default-<%= @protocol %>-ippool
+spec:
+  cidr: <%= @pod_sec_subnet %>
+  ipipMode: <%= @ipip_mode %>
+  natOutgoing: true
+  vxlanMode: Never

puppet-manifests/src/modules/platform/templates/kubelet.conf.erb

@@ -1,2 +1,7 @@
 # Overrides config file for kubelet
+<%- if defined?(@node_ip_secondary) -%>
+KUBELET_EXTRA_ARGS=--node-ip=<%= @node_ip %>,<%= @node_ip_secondary %> --volume-plugin-dir=<%= @k8s_vol_plugin_dir %> <%= @k8s_cpu_manager_opts %> --container-runtime-endpoint=unix:///var/run/containerd/containerd.sock --pod-max-pids <%= @k8s_pod_max_pids %>
+<%- else -%>
 KUBELET_EXTRA_ARGS=--node-ip=<%= @node_ip %> --volume-plugin-dir=<%= @k8s_vol_plugin_dir %> <%= @k8s_cpu_manager_opts %> --container-runtime-endpoint=unix:///var/run/containerd/containerd.sock --pod-max-pids <%= @k8s_pod_max_pids %>
+<%- end -%>
+

test-requirements.txt

@@ -3,3 +3,4 @@ hacking < 4.0.1
 bashate >= 0.2
 bandit!=1.6.0,>=1.1.0,<2.0.0;python_version>="3.0" # GPLv2
 shellcheck-py;python_version>="3.0" # MIT
+netaddr >= 0.7.19