Integrate Sonobuoy Conformance Test Scripts

Add jobs in experimental pipeline to do the following:

  - install Sonobuoy
  - run CNCF Conformace Tests
  - run CIS Benchmarks Tests

Conformance tests include:

  - CNCF Compliance: uses sonobuoy end-to-end (e2e) and systemd-logs
                     plugins

  - CIS Benchmarks: utilizes the kube-bench implementation
                    of the CIS security benchmarks plugin

Pipeline can be triggered by comment
   - "check experimental"
Change-Id: I7d08ae42512dc4c83e2f550c4809ce1f8ddccc7b

Change-Id: I2e6469f5b8e229828532ce5499498da639d23fe6

tools/deployment/sonobuoy/01-install_sonobuoy.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xe
+
+: ${SONOBUOY_VERSION:="0.18.2"}
+: ${KUBECONFIG:="$HOME/.airship/kubeconfig"}
+URL="https://github.com/vmware-tanzu/sonobuoy/releases/download/v${SONOBUOY_VERSION}/sonobuoy_${SONOBUOY_VERSION}_linux_amd64.tar.gz"
+rm -rf /tmp/sonobuoy
+mkdir /tmp/sonobuoy
+sudo -E curl -sSLo "/tmp/sonobuoy/sonobuoy_${SONOBUOY_VERSION}_linux_amd64.tar.gz" ${URL}
+tar xvf /tmp/sonobuoy/sonobuoy_${SONOBUOY_VERSION}_linux_amd64.tar.gz -C /tmp/sonobuoy/
+sudo install -m 755 -o root /tmp/sonobuoy/sonobuoy /usr/local/bin
+echo ${KUBECONFIG}
+sonobuoy version --kubeconfig ${KUBECONFIG}

tools/deployment/sonobuoy/02-run_default.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xe
+: ${KUBECONFIG:="$HOME/.airship/kubeconfig"}
+# Available Modes: quick, certified-conformance, non-disruptive-conformance.
+# (default quick)
+: ${CONFORMANCE_MODE:="quick"}
+: ${KUBE_CONFORMANCE_IMAGE_VERSION:="v1.18.6"}
+: ${TIMEOUT:=10800}
+: ${TARGET_CLUSTER_CONTEXT:="target-cluster"}
+
+mkdir -p /tmp/sonobuoy_snapshots/e2e
+cd /tmp/sonobuoy_snapshots/e2e
+
+# Run aggregator, and default plugins e2e and systemd-logs
+sonobuoy run --plugin e2e --plugin systemd-logs -m ${CONFORMANCE_MODE} \
+--context "$TARGET_CLUSTER_CONTEXT" \
+--kube-conformance-image gcr.io/google-containers/conformance:${KUBE_CONFORMANCE_IMAGE_VERSION} \
+--kubeconfig ${KUBECONFIG} \
+--wait --timeout ${TIMEOUT} \
+--log_dir /tmp/sonobuoy_snapshots/e2e
+
+# Get information on pods
+kubectl get all -n sonobuoy --kubeconfig ${KUBECONFIG} --context "$TARGET_CLUSTER_CONTEXT"
+
+# Check sonobuoy status
+sonobuoy status --kubeconfig ${KUBECONFIG} --context "$TARGET_CLUSTER_CONTEXT"
+
+# Get logs
+sonobuoy logs
+
+# Store Results
+results=$(sonobuoy retrieve --kubeconfig ${KUBECONFIG} --context $TARGET_CLUSTER_CONTEXT)
+echo "Results: ${results}"
+
+# Display Results
+sonobuoy results $results
+ls -ltr /tmp/sonobuoy_snapshots/e2e

tools/deployment/sonobuoy/03-kubebench.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xe
+: ${KUBECONFIG:="$HOME/.airship/kubeconfig"}
+: ${KUBEBENCH_MASTER_PLUGIN:="https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-master-plugin.yaml"}
+: ${KUBEBENCH_WORKER_PLUGIN:="https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-plugin.yaml"}
+: ${TARGET_CLUSTER_CONTEXT:="target-cluster"}
+# This shouldnot include minor version
+: ${KUBEBENCH_K8S_VERSION:=1.18}
+: ${TIMEOUT:=300}
+
+mkdir -p /tmp/sonobuoy_snapshots/kubebench
+cd /tmp/sonobuoy_snapshots/kubebench
+
+# Run aggregator, and default plugins e2e and systemd-logs
+sonobuoy run \
+--kubeconfig ${KUBECONFIG} \
+--context ${TARGET_CLUSTER_CONTEXT} \
+--plugin ${KUBEBENCH_MASTER_PLUGIN} \
+--plugin ${KUBEBENCH_WORKER_PLUGIN} \
+--plugin-env kube-bench-master.KUBERNETES_VERSION=${KUBEBENCH_K8S_VERSION} \
+--plugin-env kube-bench-master.KUBERNETES_VERSION=${KUBEBENCH_K8S_VERSION} \
+--wait --timeout ${TIMEOUT} \
+--log_dir /tmp/sonobuoy_snapshots/kubebench
+
+# Get information on pods
+kubectl get all -n sonobuoy --kubeconfig ${KUBECONFIG} --context ${TARGET_CLUSTER_CONTEXT}
+
+# Check sonobuoy status
+sonobuoy status --kubeconfig ${KUBECONFIG} --context ${TARGET_CLUSTER_CONTEXT}
+
+# Get logs
+sonobuoy logs
+
+# Store Results
+results=$(sonobuoy retrieve --kubeconfig ${KUBECONFIG} --context ${TARGET_CLUSTER_CONTEXT})
+echo "Results: ${results}"
+
+# Display Results
+sonobuoy results $results
+ls -ltr /tmp/sonobuoy_snapshots/kubebench

zuul.d/jobs.yaml

@@ -175,7 +175,7 @@
         soft: true
     vars:
       site_name: docker-test-site
-      gate_scripts:
+      gate_scripts: &docker_gate_scripts
         - ./tools/deployment/21_systemwide_executable.sh
         - ./tools/deployment/01_install_kubectl.sh
         - ./tools/deployment/provider_common/01_install_kind.sh
@@ -188,6 +188,62 @@
         - ./tools/deployment/provider_common/33_cluster_move_target_node.sh
         - WORKERS_COUNT=2 KUBECONFIG=/tmp/target-cluster.kubeconfig SITE=docker-test-site ./tools/deployment/provider_common/34_deploy_worker_node.sh
     voting: false
+- job:
+    name: airship-airshipctl-docker-kubebench-conformance
+    attempts: 1
+    timeout: 10800
+    pre-run: playbooks/airship-airshipctl-deploy-docker.yaml
+    run: playbooks/airshipctl-gate-runner.yaml
+    nodeset: airship-airshipctl-single-node
+    irrelevant-files: *noncodefiles
+    dependencies:
+      - name: openstack-tox-docs
+        soft: true
+      - name: airship-airshipctl-lint
+        soft: true
+      - name: airship-airshipctl-unit
+        soft: true
+      - name: airship-airshipctl-golint
+        soft: true
+      - name: airship-airshipctl-build-image
+        soft: true
+      - name: airship-airshipctl-validate-site-docs
+        soft: true
+    vars:
+      site_name: docker-test-site
+      gate_scripts:
+        - *docker_gate_scripts
+        - KUBECONFIG=/tmp/target-cluster.kubeconfig TARGET_CLUSTER_CONTEXT=target-cluster ./tools/deployment/sonobuoy/01-install_sonobuoy.sh
+        - KUBECONFIG=/tmp/target-cluster.kubeconfig TARGET_CLUSTER_CONTEXT=target-cluster ./tools/deployment/sonobuoy/03-kubebench.sh
+    voting: false
+- job:
+    name: airship-airshipctl-docker-cncf-conformance
+    attempts: 1
+    timeout: 10800
+    pre-run: playbooks/airship-airshipctl-deploy-docker.yaml
+    run: playbooks/airshipctl-gate-runner.yaml
+    nodeset: airship-airshipctl-single-node
+    irrelevant-files: *noncodefiles
+    dependencies:
+      - name: openstack-tox-docs
+        soft: true
+      - name: airship-airshipctl-lint
+        soft: true
+      - name: airship-airshipctl-unit
+        soft: true
+      - name: airship-airshipctl-golint
+        soft: true
+      - name: airship-airshipctl-build-image
+        soft: true
+      - name: airship-airshipctl-validate-site-docs
+        soft: true
+    vars:
+      site_name: docker-test-site
+      gate_scripts:
+        - *docker_gate_scripts
+        - KUBECONFIG=/tmp/target-cluster.kubeconfig TARGET_CLUSTER_CONTEXT=target-cluster ./tools/deployment/sonobuoy/01-install_sonobuoy.sh
+        - KUBECONFIG=/tmp/target-cluster.kubeconfig TARGET_CLUSTER_CONTEXT=target-cluster CONFORMANCE_MODE=certified-conformance ./tools/deployment/sonobuoy/02-run_default.sh
+    voting: false
 - job:
     name: airship-airshipctl-publish-image
     parent: airship-airshipctl-build-image

zuul.d/projects.yaml

@@ -35,6 +35,8 @@
         - airship-airshipctl-gate-script-runner-docker
     experimental:
       jobs:
+        - airship-airshipctl-docker-kubebench-conformance
+        - airship-airshipctl-docker-cncf-conformance
         - airship-airshipctl-gate-script-runner
     gate:
       jobs: