From 854e25355baa1b8dcfe05a8a00a3c1a277c7455b Mon Sep 17 00:00:00 2001
From: Rajat Sharma <rajatcu143@gmail.com>
Date: Tue, 10 Nov 2020 16:09:10 +0530
Subject: [PATCH] Incorporate HWCC in deployment gate.

Added sample HCC profiles and deployment script to apply profiles.

Change-Id: If8152a0afda142c40a53ddf9a4bd6866b435b0e5
Co-Authored-By: Rajat Sharma <rajatcu143@gmail.com>
Relates-To-Issue: #326
Depends-On: #738410
---
 manifests/function/hwcc/README.md             |  5 +--
 manifests/function/hwcc/manager/manager.yaml  |  2 +-
 manifests/phases/phases.yaml                  | 13 ++++++
 manifests/phases/plan.yaml                    |  1 +
 .../ephemeral/initinfra/kustomization.yaml    |  1 +
 .../target/initinfra/kustomization.yaml       |  1 +
 .../target/workers/kustomization.yaml         |  7 ---
 .../kubeadmconfigtemplate.yaml                |  0
 .../workers/provision/kustomization.yaml      | 11 +++++
 .../{ => provision}/machinedeployment.yaml    |  0
 .../metal3machinetemplate.yaml                |  0
 .../gating/hwccprofiles/kustomization.yaml    |  5 +++
 .../type/gating/hwccprofiles/profile1.yaml    | 26 +++++++++++
 .../type/gating/hwccprofiles/profile2.yaml    | 26 +++++++++++
 playbooks/airshipctl-gate-runner.yaml         |  1 +
 tools/deployment/34_deploy_worker_node.sh     | 43 +++++++++++--------
 tools/deployment/36_verify_hwcc_profiles.sh   | 43 +++++++++++++++++++
 zuul.d/jobs.yaml                              |  1 +
 18 files changed, 158 insertions(+), 28 deletions(-)
 rename manifests/site/test-site/target/workers/{ => provision}/kubeadmconfigtemplate.yaml (100%)
 create mode 100644 manifests/site/test-site/target/workers/provision/kustomization.yaml
 rename manifests/site/test-site/target/workers/{ => provision}/machinedeployment.yaml (100%)
 rename manifests/site/test-site/target/workers/{ => provision}/metal3machinetemplate.yaml (100%)
 create mode 100644 manifests/type/gating/hwccprofiles/kustomization.yaml
 create mode 100644 manifests/type/gating/hwccprofiles/profile1.yaml
 create mode 100644 manifests/type/gating/hwccprofiles/profile2.yaml
 create mode 100755 tools/deployment/36_verify_hwcc_profiles.sh

diff --git a/manifests/function/hwcc/README.md b/manifests/function/hwcc/README.md
index e690034e8..e5413d121 100644
--- a/manifests/function/hwcc/README.md
+++ b/manifests/function/hwcc/README.md
@@ -19,9 +19,8 @@ HWCC will label matched hosts.
 
    `hardwareclassification.metal3.io/<PROFILE-NAME>=<LABEL>`
 
-HWCC also label hosts which are in error state, e.g.
-
-`hardwareclassification-error=registration-error`
+HWCC also label hosts which are in error state if
+`hardwareclassification-error=All` label is given in workload profile.
 
 HWCC status shows multiple items w.r.t applied profile :
  * Name of the profile
diff --git a/manifests/function/hwcc/manager/manager.yaml b/manifests/function/hwcc/manager/manager.yaml
index 74be0001c..3fa69ea48 100644
--- a/manifests/function/hwcc/manager/manager.yaml
+++ b/manifests/function/hwcc/manager/manager.yaml
@@ -20,7 +20,7 @@ spec:
         - /manager
         args:
         - --enable-leader-election
-        image: quay.io/hwcc/airship-hcc:v1
+        image: quay.io/dell_sp/hwcc:v1.3
         name: manager
         resources:
           limits:
diff --git a/manifests/phases/phases.yaml b/manifests/phases/phases.yaml
index cf4a6e5e4..7ce5dc4cc 100644
--- a/manifests/phases/phases.yaml
+++ b/manifests/phases/phases.yaml
@@ -73,6 +73,19 @@ config:
 ---
 apiVersion: airshipit.org/v1alpha1
 kind: Phase
+metadata:
+  name: workers-classification
+  clusterName: target-cluster
+config:
+  cluster: target-cluster
+  executorRef:
+    apiVersion: airshipit.org/v1alpha1
+    kind: KubernetesApply
+    name: kubernetes-apply
+  documentEntryPoint: target/workers/provision
+---
+apiVersion: airshipit.org/v1alpha1
+kind: Phase
 metadata:
   name: clusterctl-init-ephemeral
   clusterName: ephemeral-cluster
diff --git a/manifests/phases/plan.yaml b/manifests/phases/plan.yaml
index d07d7d630..59a81b63a 100644
--- a/manifests/phases/plan.yaml
+++ b/manifests/phases/plan.yaml
@@ -10,4 +10,5 @@ phaseGroups:
       - name: controlplane-ephemeral
       - name: initinfra-target
       - name: workers-target
+      - name: workers-classification
       - name: workload-target
diff --git a/manifests/site/test-site/ephemeral/initinfra/kustomization.yaml b/manifests/site/test-site/ephemeral/initinfra/kustomization.yaml
index 12fa7caf4..35fbf7d39 100644
--- a/manifests/site/test-site/ephemeral/initinfra/kustomization.yaml
+++ b/manifests/site/test-site/ephemeral/initinfra/kustomization.yaml
@@ -1,5 +1,6 @@
 resources:
   - ../../../../function/baremetal-operator
+  - ../../../../function/hwcc
   - ../../../../function/clusterctl
   - ../../../../composite/infra
   - ../catalogues
diff --git a/manifests/site/test-site/target/initinfra/kustomization.yaml b/manifests/site/test-site/target/initinfra/kustomization.yaml
index 5bdab6d5a..408f08cd8 100644
--- a/manifests/site/test-site/target/initinfra/kustomization.yaml
+++ b/manifests/site/test-site/target/initinfra/kustomization.yaml
@@ -1,5 +1,6 @@
 resources:
   - ../../../../function/baremetal-operator
+  - ../../../../function/hwcc
   - ../../../../function/clusterctl
   - ../../../../composite/flux-helm
   - ../../../../composite/infra
diff --git a/manifests/site/test-site/target/workers/kustomization.yaml b/manifests/site/test-site/target/workers/kustomization.yaml
index 0c94f53b2..e168397b1 100644
--- a/manifests/site/test-site/target/workers/kustomization.yaml
+++ b/manifests/site/test-site/target/workers/kustomization.yaml
@@ -2,10 +2,3 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - nodes
-  - kubeadmconfigtemplate.yaml
-  - metal3machinetemplate.yaml
-  - machinedeployment.yaml
-  - ../../../../function/airshipctl-base-catalogues
-
-transformers:
-  - replacements
diff --git a/manifests/site/test-site/target/workers/kubeadmconfigtemplate.yaml b/manifests/site/test-site/target/workers/provision/kubeadmconfigtemplate.yaml
similarity index 100%
rename from manifests/site/test-site/target/workers/kubeadmconfigtemplate.yaml
rename to manifests/site/test-site/target/workers/provision/kubeadmconfigtemplate.yaml
diff --git a/manifests/site/test-site/target/workers/provision/kustomization.yaml b/manifests/site/test-site/target/workers/provision/kustomization.yaml
new file mode 100644
index 000000000..35e72ee55
--- /dev/null
+++ b/manifests/site/test-site/target/workers/provision/kustomization.yaml
@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../../../../../function/airshipctl-base-catalogues
+  - ../../../../../type/gating/hwccprofiles
+  - kubeadmconfigtemplate.yaml
+  - metal3machinetemplate.yaml
+  - machinedeployment.yaml
+
+transformers:
+  - ../replacements
diff --git a/manifests/site/test-site/target/workers/machinedeployment.yaml b/manifests/site/test-site/target/workers/provision/machinedeployment.yaml
similarity index 100%
rename from manifests/site/test-site/target/workers/machinedeployment.yaml
rename to manifests/site/test-site/target/workers/provision/machinedeployment.yaml
diff --git a/manifests/site/test-site/target/workers/metal3machinetemplate.yaml b/manifests/site/test-site/target/workers/provision/metal3machinetemplate.yaml
similarity index 100%
rename from manifests/site/test-site/target/workers/metal3machinetemplate.yaml
rename to manifests/site/test-site/target/workers/provision/metal3machinetemplate.yaml
diff --git a/manifests/type/gating/hwccprofiles/kustomization.yaml b/manifests/type/gating/hwccprofiles/kustomization.yaml
new file mode 100644
index 000000000..efc32b1b6
--- /dev/null
+++ b/manifests/type/gating/hwccprofiles/kustomization.yaml
@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - profile1.yaml
+  - profile2.yaml
diff --git a/manifests/type/gating/hwccprofiles/profile1.yaml b/manifests/type/gating/hwccprofiles/profile1.yaml
new file mode 100644
index 000000000..e156a21e2
--- /dev/null
+++ b/manifests/type/gating/hwccprofiles/profile1.yaml
@@ -0,0 +1,26 @@
+apiVersion: metal3.io/v1alpha1
+kind: HardwareClassification
+metadata:
+  name: hardwareclassification-profile1
+  namespace: default
+  labels:
+    hardwareclassification-profile1: matches
+    hardwareclassification-error: All
+spec:
+  hardwareCharacteristics:
+      cpu:
+         minimumCount: 1
+         maximumCount: 72
+         minimumSpeedMHz: 1000
+         maximumSpeedMHz: 3600
+      disk:
+         minimumCount: 1
+         maximumCount: 8
+         minimumIndividualSizeGB: 1
+         maximumIndividualSizeGB: 3000
+      ram:
+         minimumSizeGB: 1
+         maximumSizeGB: 180
+      nic:
+         minimumCount: 1
+         maximumCount: 7
diff --git a/manifests/type/gating/hwccprofiles/profile2.yaml b/manifests/type/gating/hwccprofiles/profile2.yaml
new file mode 100644
index 000000000..26b642074
--- /dev/null
+++ b/manifests/type/gating/hwccprofiles/profile2.yaml
@@ -0,0 +1,26 @@
+apiVersion: metal3.io/v1alpha1
+kind: HardwareClassification
+metadata:
+  name: hardwareclassification-profile2
+  namespace: default
+  labels:
+    hardwareclassification-profile2: matches
+    hardwareclassification-error: All
+spec:
+  hardwareCharacteristics:
+      cpu:
+         minimumCount: 80
+         maximumCount: 100
+         minimumSpeedMHz: 2000
+         maximumSpeedMHz: 4000
+      disk:
+         minimumCount: 20
+         maximumCount: 80
+         minimumIndividualSizeGB: 2000
+         maximumIndividualSizeGB: 3000
+      ram:
+         minimumSizeGB: 400
+         maximumSizeGB: 800
+      nic:
+         minimumCount: 50
+         maximumCount: 70
diff --git a/playbooks/airshipctl-gate-runner.yaml b/playbooks/airshipctl-gate-runner.yaml
index c849d4caa..ba6ff0878 100644
--- a/playbooks/airshipctl-gate-runner.yaml
+++ b/playbooks/airshipctl-gate-runner.yaml
@@ -28,6 +28,7 @@
           - ./tools/deployment/33_cluster_move_target_node.sh
           - ./tools/deployment/34_deploy_worker_node.sh
           - ./tools/deployment/35_deploy_workload.sh
+          - ./tools/deployment/36_verify_hwcc_profiles.sh
 
     - name: "Run gate scripts"
       include_role:
diff --git a/tools/deployment/34_deploy_worker_node.sh b/tools/deployment/34_deploy_worker_node.sh
index 5469bf6e0..39778723b 100755
--- a/tools/deployment/34_deploy_worker_node.sh
+++ b/tools/deployment/34_deploy_worker_node.sh
@@ -22,24 +22,33 @@ export KUBECONFIG_TARGET_CONTEXT=${KUBECONFIG_TARGET_CONTEXT:-"target-cluster"}
 echo "Stop ephemeral node"
 sudo virsh destroy air-ephemeral
 
+node_timeout () {
+    end=$(($(date +%s) + $TIMEOUT))
+    while true; do
+        if (kubectl --request-timeout 20s --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT get $1 node03 | grep -qw $2) ; then
+            echo -e "\nGet $1 status"
+            kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT get $1
+            break
+        else
+            now=$(date +%s)
+            if [ $now -gt $end ]; then
+                echo -e "\n$1 is not ready before TIMEOUT."
+                exit 1
+            fi
+            echo -n .
+            sleep 15
+        fi
+    done
+}
+
 echo "Deploy worker node"
 airshipctl phase run  workers-target --debug
 
-#Wait till node is created
-end=$(($(date +%s) + $TIMEOUT))
+echo "Waiting $TIMEOUT seconds for bmh to be in ready state."
+node_timeout bmh ready
+
+echo "Classify and provision worker node"
+airshipctl phase run  workers-classification --debug
+
 echo "Waiting $TIMEOUT seconds for node to be provisioned."
-while true; do
-    if (kubectl --request-timeout 20s --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT get node node03 | grep -qw Ready) ; then
-        echo -e "\nGet node status"
-        kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT get node
-        break
-    else
-        now=$(date +%s)
-        if [ $now -gt $end ]; then
-            echo -e "\nWorker node is not ready before TIMEOUT."
-            exit 1
-        fi
-        echo -n .
-        sleep 15
-    fi
-done
+node_timeout node Ready
diff --git a/tools/deployment/36_verify_hwcc_profiles.sh b/tools/deployment/36_verify_hwcc_profiles.sh
new file mode 100755
index 000000000..e5ec47135
--- /dev/null
+++ b/tools/deployment/36_verify_hwcc_profiles.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+
+export KUBECONFIG=${KUBECONFIG:-"$HOME/.airship/kubeconfig"}
+export KUBECONFIG_TARGET_CONTEXT=${KUBECONFIG_TARGET_CONTEXT:-"target-cluster"}
+declare -A PROFILES
+
+PROFILES[hardwareclassification-profile1]=1
+PROFILES[hardwareclassification-profile2]=0
+
+declare -a ERRORS
+
+# HWCC need BMH in Ready state.
+for i in "${!PROFILES[@]}"
+do
+    nodes=$(kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT get bmh --no-headers=true -l hardwareclassification.metal3.io/$i 2>/dev/null | wc -l)
+
+    if [ $nodes != ${PROFILES[$i]} ]
+    then
+        ERRORS+=($i)
+    fi
+done
+
+if [ ${#ERRORS[@]} != 0 ]
+then
+    echo FAILURE error with ${ERRORS[@]}
+    exit 1
+else
+    echo "SUCCESS"
+fi
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
index 25508fc12..150687b77 100644
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@@ -141,6 +141,7 @@
         - ./tools/deployment/33_cluster_move_target_node.sh
         - ./tools/deployment/34_deploy_worker_node.sh
         - ./tools/deployment/35_deploy_workload.sh
+        - ./tools/deployment/36_verify_hwcc_profiles.sh
       serve_dir: /srv/iso
       serve_port: 8099
       log_roles: