[DNM] Changes in Treasuremap to handle vrrp shift to live API Server on Controlplane Node

Change-Id: I74aa58014511d6ab529f43544d53a177d047f804
2021-10-13 19:35:17 +05:30 · 2021-10-13 19:35:17 +05:30 · 3f6daa66be
parent f819f6c510
commit 3f6daa66be
13 changed files with 110 additions and 30 deletions
--- a/manifests/function/k8scontrol-ha/replacements/networking-ha.yaml
+++ b/manifests/function/k8scontrol-ha/replacements/networking-ha.yaml
@ -17,6 +17,16 @@ replacements:
      kind: KubeadmControlPlane
    fieldrefs:
      -  "spec.kubeadmConfigSpec.files[path=/etc/keepalived/keepalived.conf].content%REPLACEMENT_VIP_1%"
+- source:
+    objref:
+      kind: VariableCatalogue
+      name: networking-ha
+    fieldref: vrrp.kubernetes.apiserver_port
+  target:
+    objref:
+      kind: KubeadmControlPlane
+    fieldrefs:
+      -  "spec.kubeadmConfigSpec.files[path=/etc/keepalived/check_apiserver.sh].content%APISERVER_DEST_PORT%"
 - source:
    objref:
      kind: VariableCatalogue
--- a/manifests/function/k8scontrol-ha/vrrp_keepalived_patch.yaml
+++ b/manifests/function/k8scontrol-ha/vrrp_keepalived_patch.yaml
@ -6,6 +6,13 @@
         ! Configuration File for keepalived
         global_defs {
         }
+         vrrp_script check_apiserver {
+          script "/etc/keepalived/check_apiserver.sh"
+          interval 3
+          weight -2
+          fall 10
+          rise 2
+         }
         vrrp_instance KUBERNETES {
             state BACKUP
             interface REPLACEMENT_INTERFACE_1
@ -15,7 +22,10 @@
             virtual_ipaddress {
                REPLACEMENT_VIP_1
             }
-         }
+             track_script {
+              check_apiserver
+             }
+          }
         vrrp_instance INGRESS {
             state BACKUP
             interface REPLACEMENT_INTERFACE_2
@ -25,7 +35,22 @@
             virtual_ipaddress {
                REPLACEMENT_VIP_2
             }
-         }
+          }
+- op: add
+  path: "/spec/kubeadmConfigSpec/files/-"
+  value:
+    path: /etc/keepalived/check_apiserver.sh
+    content: |
+      #!/bin/sh
+      errorExit() {
+        echo "*** $*" 1>&2
+        exit 1
+      }
+
+      curl --silent --max-time 2 --insecure https://localhost:${6443}/ -o /dev/null || errorExit "Error GET https://localhost:${6443}/"
+      if ip addr | grep -q ${REPLACEMENT_VIP_1}; then
+        curl --silent --max-time 2 --insecure https://${REPLACEMENT_VIP_1}:${6443}/ -o /dev/null || errorExit "Error GET https://${REPLACEMENT_VIP_1}:${6443}/"
+      fi
 - op: add
  path: "/spec/kubeadmConfigSpec/preKubeadmCommands/-"
  value:
--- a/manifests/site/virtual-airship-core/ephemeral/controlplane/hostgenerator/host-generation.yaml
+++ b/manifests/site/virtual-airship-core/ephemeral/controlplane/hostgenerator/host-generation.yaml
@ -8,3 +8,5 @@ metadata:
 hosts:
  m3:
    - node01
+    - node04
+    - node05
--- a/manifests/site/virtual-airship-core/ephemeral/controlplane/kustomization.yaml
+++ b/manifests/site/virtual-airship-core/ephemeral/controlplane/kustomization.yaml
@ -9,5 +9,8 @@ resources:

 namespace: target-infra

+patchesStrategicMerge:
+  - patch_controlplane.yaml
+
 transformers:
  - ../../../../type/airship-core/ephemeral/controlplane/replacements
--- a/manifests/site/virtual-airship-core/ephemeral/controlplane/patch_controlplane.yaml
+++ b/manifests/site/virtual-airship-core/ephemeral/controlplane/patch_controlplane.yaml
@ -0,0 +1,6 @@
+kind: KubeadmControlPlane
+apiVersion: controlplane.cluster.x-k8s.io/v1alpha3
+metadata:
+  name: cluster-controlplane
+spec:
+  replicas: 3
--- a/manifests/site/virtual-airship-core/host-inventory/hostgenerator/host-generation.yaml
+++ b/manifests/site/virtual-airship-core/host-inventory/hostgenerator/host-generation.yaml
@ -12,3 +12,5 @@ hosts:
    - node01
    - node02
    - node03
+    - node04
+    - node05
--- a/manifests/site/virtual-airship-core/kubeconfig/kubeconfig.yaml
+++ b/manifests/site/virtual-airship-core/kubeconfig/kubeconfig.yaml
@ -9,7 +9,7 @@ config:
  clusters:
  - cluster:
      certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURWRENDQWp5Z0F3SUJBZ0lVTUNwc09vRXhyRzdnRTVMOVJSamdnT01UOG53d0RRWUpLb1pJaHZjTkFRRUwKQlFBd0dURVhNQlVHQTFVRUF3d09TM1ZpWlhKdVpYUmxjeUJCVUVrd0hoY05NakF3T1RFMU1ERXdORE0zV2hjTgpNekF3T1RFek1ERXdORE0zV2pBWk1SY3dGUVlEVlFRRERBNUxkV0psY201bGRHVnpJRUZRU1RDQ0FTSXdEUVlKCktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQUtBZFo0UWJHZmlLTExpTXNHcFJKS3d5ZkRGWVI5U0MKbGtVb3hlTU1BZVBkeVNNU0paTTlFMFBOaDM5TUtTVjNSZDRIZWt1eGdHK3J4em83WmcrZU1aY1hyNFk3ektQMwo1SW0vaERkMm1TYThsMEkxZTRwV3B0Z25vZjdvRWJpSXVIU2YxQmRhMU4wWm1EUUdtckxyQnFOZFE3c1BVenNWCllPejZVUFZlamNIeEFjMXBvMWZsQXYrWVNZejVXa28wRVRnTXZYRGtxT0hrWFc1WnhPcHBVbiszOVpvWTZMK3gKVmUwUHFQdHlmSVZ1M3dtcnZFNGd4SmxtWEk3dUxmdzZONHpwS2RuK0k0K1RJRWF5aE1EMWRRenNwQzRMM0IrcApYcHFPMWNWM2ZKMlBycS9mNU14SnIxWTVHUTZlQlZyTGVod1ZWTEhEMzF3ZWFpZ3UzeStyM3RVQ0F3RUFBYU9CCmt6Q0JrREFkQmdOVkhRNEVGZ1FVT1d5YTNFd2J5c25UUy9ZajFWTEtjMGh4aDRvd1ZBWURWUjBqQkUwd1M0QVUKT1d5YTNFd2J5c25UUy9ZajFWTEtjMGh4aDRxaEhhUWJNQmt4RnpBVkJnTlZCQU1NRGt0MVltVnlibVYwWlhNZwpRVkJKZ2hRd0ttdzZnVEdzYnVBVGt2MUZHT0NBNHhQeWZEQU1CZ05WSFJNRUJUQURBUUgvTUFzR0ExVWREd1FFCkF3SUJCakFOQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBTVp1U2tJbTdQdlA4MW5HSjlYOVZFOFVZTVdDSU5GMEEKYit1UURFaHRGc0dxdnZFZHhQcURUWUpwdlF1SUJlOVd0cmlWRzh0MENIL1NnZ0g2TlJod0wyYkJwMm5WaEFVVwphK3hZL1RpTmMzUEl5RHNFeEY3VHVENGJzaW1BQUJTZ2ZtbXRxV1dqajRyOStodS9vZ09jLzQyYk9JT0JWbHNkCi9VNzBiR3dZQjU5QXgvL2dIWVJmVDl3L3p0VHBvY2tzdEhhSjZsVDd5SFlqYUkzaU5EWnZNSnFRSWNxME4vTEMKcVBjWjBWQXBMUTZRUHRpMWpVSzBGM1VlZEF6TVc3ZFF4NkV3Qjd5UHo4NWdZS3ZJdWdyaStrc2YwbGMyeHVDRwpXTGg2YjFNWk9Cc1NZNkppVHpSUUpYdXNCRUdaTGN5VkRJSEU3Y0Q4NWhOQmZpdDAvejFmZlE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t
-      server: https://10.23.25.102:6443
+      server: https://10.23.25.201:6443
    name: target-cluster
  - cluster:
      certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRFNU1USXlOakE0TWpneU5Gb1hEVEk1TVRJeU16QTRNamd5TkZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTTFSClM0d3lnajNpU0JBZjlCR0JUS1p5VTFwYmdDaGQ2WTdJektaZWRoakM2K3k1ZEJpWm81ZUx6Z2tEc2gzOC9YQ1MKenFPS2V5cE5RcDN5QVlLdmJKSHg3ODZxSFZZNjg1ZDVYVDNaOHNyVVRzVDR5WmNzZHAzV3lHdDM0eXYzNi9BSQoxK1NlUFErdU5JemN6bzNEdWhXR0ZoQjk3VjZwRitFUTBlVWN5bk05c2hkL3AwWVFzWDR1ZlhxaENENVpzZnZUCnBka3UvTWkyWnVGUldUUUtNeGpqczV3Z2RBWnBsNnN0L2ZkbmZwd1Q5cC9WTjRuaXJnMEsxOURTSFFJTHVrU2MKb013bXNBeDJrZmxITWhPazg5S3FpMEloL2cyczRFYTRvWURZemt0Y2JRZ24wd0lqZ2dmdnVzM3pRbEczN2lwYQo4cVRzS2VmVGdkUjhnZkJDNUZNQ0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFJek9BL00xWmRGUElzd2VoWjFuemJ0VFNURG4KRHMyVnhSV0VnclFFYzNSYmV3a1NkbTlBS3MwVGR0ZHdEbnBEL2tRYkNyS2xEeFF3RWg3NFZNSFZYYkFadDdsVwpCSm90T21xdXgxYThKYklDRTljR0FHRzFvS0g5R29jWERZY0JzOTA3ckxIdStpVzFnL0xVdG5hN1dSampqZnBLCnFGelFmOGdJUHZIM09BZ3B1RVVncUx5QU8ya0VnelZwTjZwQVJxSnZVRks2TUQ0YzFmMnlxWGxwNXhrN2dFSnIKUzQ4WmF6d0RmWUVmV3Jrdld1YWdvZ1M2SktvbjVEZ0Z1ZHhINXM2Snl6R3lPVnZ0eG1TY2FvOHNxaCs3UXkybgoyLzFVcU5ZK0hlN0x4d04rYkhwYkIxNUtIMTU5ZHNuS3BRbjRORG1jSTZrVnJ3MDVJMUg5ZGRBbGF0bz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
--- a/manifests/site/virtual-airship-core/target/catalogues/shareable/hosts.yaml
+++ b/manifests/site/virtual-airship-core/target/catalogues/shareable/hosts.yaml
@ -1,16 +1,13 @@
-# Site-level host catalogue.  This info feeds the Templater
-# kustomize plugin config in the hostgenerator-m3 function.
 apiVersion: airshipit.org/v1alpha1
 kind: VariableCatalogue
 metadata:
  name: host-catalogue
  labels:
    airshipit.org/deploy-k8s: "false"
-
 hosts:
  m3:
    node01:
-      bootMode: UEFI
+      bootMode: legacy
      macAddress: 52:54:00:b6:ed:31
      bmcAddress: redfish+http://10.23.25.1:8000/redfish/v1/Systems/air-target-1
      bmcUsername: root
@ -24,7 +21,7 @@ hosts:
        pxe: 52:54:00:b6:ed:31
      hardwareProfile: default # defined in the hostgenerator-m3 function
    node02:
-      bootMode: UEFI
+      bootMode: legacy
      macAddress: 52:54:00:b6:ed:02
      bmcAddress: redfish+https://10.23.25.1:8443/redfish/v1/Systems/air-ephemeral
      bmcUsername: username
@ -38,29 +35,44 @@ hosts:
        pxe: 52:54:00:b6:ed:02
      hardwareProfile: default
    node03:
-      bootMode: UEFI
+      bootMode: legacy
      macAddress: 52:54:00:b6:ed:23
      bmcAddress: redfish+http://10.23.25.1:8000/redfish/v1/Systems/air-worker-1
      bmcUsername: username
      bmcPassword: password
      disableCertificateVerification: false
      ipAddresses:
-        oam-ipv4: 10.23.25.103
-        pxe-ipv4: 10.23.24.103
+        oam-ipv4: 10.23.25.105
+        pxe-ipv4: 10.23.24.105
      macAddresses:
        oam: 52:54:00:9b:27:07
        pxe: 52:54:00:b6:ed:23
      hardwareProfile: default # defined in the hardwareprofile-example function
    node04:
-      bootMode: UEFI
+      bootMode: legacy
      macAddress: 52:54:00:36:5e:e3
-      bmcAddress: redfish+http://10.23.25.2:8000/redfish/v1/Systems/air-target-2
+      bmcAddress: redfish+http://10.23.25.1:8000/redfish/v1/Systems/air-target-2
      bmcUsername: username
      bmcPassword: password
+      ipAddresses:
+        oam-ipv4: 10.23.25.103
+        pxe-ipv4: 10.23.24.103
+      macAddresses:
+        oam: 52:54:00:dc:ab:04
+        pxe: 52:54:00:36:5e:e3
+      hardwareProfile: default # defined in the hardwareprofile-example function
+    node05:
+      bootMode: legacy
+      macAddress: 52:56:00:b6:ed:23
+      bmcAddress: redfish+http://10.23.25.1:8000/redfish/v1/Systems/air-target-3
+      bmcUsername: username
+      bmcPassword: password
+      disableCertificateVerification: false
      ipAddresses:
        oam-ipv4: 10.23.25.104
        pxe-ipv4: 10.23.24.104
      macAddresses:
-        oam: 52:54:00:dc:ab:04
-        pxe: 52:54:00:51:0b:e4
+        oam: 52:56:00:9b:27:07
+        pxe: 52:56:00:b6:ed:23
      hardwareProfile: default # defined in the hardwareprofile-example function
+
--- a/manifests/site/virtual-airship-core/target/catalogues/shareable/networking-ha.yaml
+++ b/manifests/site/virtual-airship-core/target/catalogues/shareable/networking-ha.yaml
@ -7,9 +7,29 @@ metadata:
  labels:
    airshipit.org/deploy-k8s: "false"
 vrrp:
+  # kubernetes:
+  #   interface: bond.41
+  #   virtual_ipaddress: 10.23.25.103
+  # ingress:
+  #   interface: bond.41
+  #   virtual_ipaddress: 10.23.25.104
  kubernetes:
-    interface: bond.41
-    virtual_ipaddress: 10.23.25.103
+    interface: oam
+    virtual_ipaddress: 10.23.25.201
+    apiserver_port: 6443
  ingress:
-    interface: bond.41
-    virtual_ipaddress: 10.23.25.104
+    interface: oam
+    virtual_ipaddress: 10.23.25.202
+    oam_cidr: 10.23.25.151/32
+    destination:
+      ports:
+      - 2378
+      - 4149
+      - 6443
+      - 6553
+      - 6666
+      - 6667
+      - 9099
+      - 10250
+      - 10255
+      - 10256
--- a/manifests/site/virtual-airship-core/target/catalogues/shareable/networking.yaml
+++ b/manifests/site/virtual-airship-core/target/catalogues/shareable/networking.yaml
@ -11,8 +11,8 @@ spec:
  # clusters, e.g. ephemeral vs target vs tenant
  kubernetes:
    controlPlaneEndpoint:
-      host: "10.23.25.102"
-    apiserverCertSANs: "[10.23.25.102, 10.23.24.102]"
+      host: "10.23.25.201"
+    apiserverCertSANs: "[10.23.25.201, 10.23.24.102]"
  ironic:
    provisioningIp: "10.23.24.102"
    dhcpRange: "10.23.24.200,10.23.24.250"
--- a/playbooks/airship-treasuremap-build-gate.yaml
+++ b/playbooks/airship-treasuremap-build-gate.yaml
@ -93,7 +93,7 @@
            target_vm_cfg: "{{ target_vm_cfg }}"
            target_vm_memory_mb: 7168
            target_vm_vcpus: 2
-            target_vms_count: 1
+            target_vms_count: 3
            worker_disk_size: 35G
            worker_vm_cfg: "{{ worker_vm_cfg }}"
            worker_vm_memory_mb: 7168
--- a/playbooks/get-vm-config.yaml
+++ b/playbooks/get-vm-config.yaml
@ -9,10 +9,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 # TODO(drewwalters96): Move this logic to an airshipctl role so breaking changes
 # do not impact Treasuremap.
-
 - name: get BareMetalHost objects
  shell: |
    set -e
@ -20,15 +18,12 @@
      {{ airship_config_manifest_directory }}/{{ airship_config_site_path }}/{{ path }} 2>/dev/null |
    kustomize cfg grep  "kind=BareMetalHost"
  register: bmh_command
-  failed_when: "bmh_command.stdout == ''"
  environment:
    KUSTOMIZE_PLUGIN_HOME: "/tmp"
    KUSTOMIZE_ENABLE_ALPHA_COMMANDS: "true"
    SOPS_IMPORT_PGP: "{{ airship_config_pgp }}"
-
 - set_fact:
    bmh: "{{ bmh_command.stdout | from_yaml_all | list }}"
-
 - name: get network configuration for BareMetalHost objects
  shell: |
    set -e
@ -42,7 +37,7 @@
    KUSTOMIZE_ENABLE_ALPHA_COMMANDS: "true"
    SOPS_IMPORT_PGP: "{{ airship_config_pgp }}"
  with_items: "{{ bmh }}"
-
+  when: bmh != ""
 - name: get links from network data per BareMetalHost object
  set_fact:
    links: |
@ -52,6 +47,7 @@
          map(attribute='stringData.networkData') | map('from_yaml') |
          map(attribute='links') | list
      }}
+  when: bmh != ""
 - name: define list of VM mac addresses and VM boot mode
  set_fact:
    vm_cfg: "{{ dict(['boot_mode', 'nat_mac', 'provision_mac'] | zip([item.spec.bootMode, nat_mac_list[0], item.spec.bootMACAddress])) }}"
@ -68,6 +64,10 @@
  loop_control:
    index_var: idx
  register: vm_cfg_fact
-
+  when: bmh != ""
+- set_fact:
+    "{{ name }}_vm_cfg": "{{  (lookup('vars', name + '_vm_cfg') |list)  + (vm_cfg_fact.results | map(attribute='ansible_facts.vm_cfg') | list) }}"
+  when: "{{ name + '_vm_cfg' }} is defined and bmh != ''"
 - set_fact:
    "{{ name }}_vm_cfg": "{{ vm_cfg_fact.results | map(attribute='ansible_facts.vm_cfg') | list }}"
+  when: "{{ name + '_vm_cfg' }} is not defined and bmh != ''"
--- a/tools/gate/10_build_gate.sh
+++ b/tools/gate/10_build_gate.sh
@ -24,5 +24,5 @@ ANSIBLE_HOSTS=${ANSIBLE_HOSTS:-"${TMP_DIR}/ansible_hosts"}
 PLAYBOOK_CONFIG=${PLAYBOOK_CONFIG:-"${TMP_DIR}/config.yaml"}

 sudo -E ansible-playbook -i "$ANSIBLE_HOSTS" \
-	playbooks/airship-treasuremap-build-gate.yaml \
+	playbooks/airship-treasuremap-build-gate.yaml -v \
 	-e @"$PLAYBOOK_CONFIG"