(zuul) Add basic gating

- Add gate for PEP8 linting
- Add gate for Helm chart linting
- Add gate for Bandit security scanning
- Add gate for document generation
- Add gate for unit tests

Change-Id: I7a9358a021dd3268eeede134fbcd68f791b83472

.zuul.yaml

@@ -13,27 +13,91 @@
 - project:
     check:
       jobs:
-        - airship-promenade-linter
-        - airship-docker-build-gate
+        - airship-promenade-lint-ws
+        - airship-promenade-docker-build-gate
+        - airship-promenade-doc-build
+        - airship-promenade-lint-pep8
+        - airship-promenade-lint-chart
+        - airship-promenade-unit-py35
+        - airship-promenade-security-bandit
 
     gate:
       jobs:
-        - airship-promenade-linter
-        - airship-docker-build-gate
+        - airship-promenade-lint-ws
+        - airship-promenade-docker-build-gate
+        - airship-promenade-doc-build
+        - airship-promenade-lint-pep8
+        - airship-promenade-lint-chart
+        - airship-promenade-unit-py35
+        - airship-promenade-security-bandit
 
     post:
       jobs:
         - airship-docker-build-post
 
-- job:
-    name: airship-promenade-linter
-    run: tools/gate/playbooks/zuul-linter.yaml
-    nodeset: openstack-helm-single-node
+- nodeset:
+    name: airship-promenade-single-node
+    nodes:
+      - name: primary
+        label: ubuntu-xenial
 
 - job:
-    name: airship-docker-build-gate
-    run: tools/gate/playbooks/docker-image-build.yaml
-    nodeset: ubuntu-xenial
+    name: airship-promenade-lint-ws
+    description: |
+      Lints all files for trailing whitespace
+    run: tools/zuul/playbooks/zuul-linter.yaml
+    timeout: 300
+    nodeset: airship-promenade-single-node
+
+- job:
+    name: airship-promenade-lint-pep8
+    description: |
+      Lints Python files against the PEP8 standard
+    run: tools/zuul/playbooks/pep8-linter.yaml
+    timeout: 300
+    nodeset: airship-promenade-single-node
+
+- job:
+    name: airship-promenade-lint-chart
+    description: |
+      Lints Helm charts for validity
+    run: tools/zuul/playbooks/helm-linter.yaml
+    timeout: 300
+    nodeset: airship-promenade-single-node
+
+- job:
+    name: airship-promenade-unit-py35
+    description: |
+      Executes unit tests under Python 3.5
+    run: tools/zuul/playbooks/unit-py35.yaml
+    timeout: 300
+    nodeset: airship-promenade-single-node
+
+- job:
+    name: airship-promenade-security-bandit
+    description: |
+      Executes the Bandit security scanner against Python files
+    run: tools/zuul/playbooks/security-bandit.yaml
+    timeout: 300
+    nodeset: airship-promenade-single-node
+
+- job:
+    name: airship-promenade-doc-build
+    description: |
+      Locally build the documentation to check for errors
+    run: tools/zuul/playbooks/doc-build.yaml
+    timeout: 300
+    nodeset: airship-promenade-single-node
+
+- job:
+    name: airship-promenade-linter
+    run: tools/zuul/playbooks/zuul-linter.yaml
+    nodeset: airship-promenade-single-node
+
+- job:
+    name: airship-promenade-docker-build-gate
+    run: tools/zuul/playbooks/docker-image-build.yaml
+    nodeset: airship-promenade-single-node
     vars:
       publish: false
       tags:
@@ -42,7 +106,7 @@
 
 - job:
     name: airship-docker-build-post
-    run: tools/gate/playbooks/docker-image-build.yaml
+    run: tools/zuul/playbooks/docker-image-build.yaml
     nodeset: ubuntu-xenial
     secrets:
       - airship_dockerhub_creds

Makefile

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+BUILD_DIR := $(shell mktemp -d)
+HELM := $(BUILD_DIR)/helm
 HELM ?= helm
 HELM_PIDFILE ?= $(abspath ./.helm-pid)
 
@@ -24,6 +26,22 @@ all: charts lint
 tests: gate-lint
 	tox
 
+.PHONY: tests-security
+tests-security:
+	tox -e bandit
+
+.PHONY: docs
+docs:
+	tox -e docs
+
+.PHONY: tests-unit
+tests-unit:
+	tox -e unit
+
+.PHONY: tests-pep8
+tests-pep8:
+	tox -e lint
+
 chartbanner:
 	@echo Building charts: $(CHARTS)
 
@@ -54,7 +72,7 @@ gate-lint-deps:
 helm-lint: $(addprefix helm-lint-,$(CHARTS))
 
 .PHONY: helm-lint-%
-helm-lint-%: helm-init-%
+helm-lint-%: helm-install helm-init-%
 	@echo Linting chart $*
 	cd charts;$(HELM) lint $*
 
@@ -71,7 +89,7 @@ $(CHARTS): $(addprefix dry-run-,$(CHARTS)) chartbanner
 	$(HELM) package -d charts charts/$@
 
 .PHONY: helm-serve
-helm-serve:
+helm-serve: helm-install
 	./tools/helm_tk.sh $(HELM) $(HELM_PIDFILE)
 
 .PHONY: clean
@@ -79,3 +97,8 @@ clean:
 	rm -f charts/*.tgz
 	rm -f charts/*/requirements.lock
 	rm -rf charts/*/charts
+
+# Install helm binary
+.PHONY: helm-install
+helm-install:
+	tools/helm_install.sh $(HELM)

tools/helm_install.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -x
+
+HELM=$1
+HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://storage.googleapis.com/kubernetes-helm/helm-v2.7.2-linux-amd64.tar.gz"}
+
+
+function install_helm_binary {
+  if [[ -z "${HELM}" ]]
+  then
+    echo "No Helm binary target location."
+    exit -1
+  fi
+
+  if [[ -w "$(dirname ${HELM})" ]]
+  then
+    TMP_DIR=${BUILD_DIR:-$(mktemp -d)}
+    curl -o "${TMP_DIR}/helm.tar.gz" "${HELM_ARTIFACT_URL}"
+    cd ${TMP_DIR}
+    tar -xvzf helm.tar.gz
+    cp "${TMP_DIR}/linux-amd64/helm" "${HELM}"
+  else
+    echo "Cannot write to ${HELM}"
+    exit -1
+  fi
+}
+
+install_helm_binary

tools/helm_tk.sh

@@ -56,6 +56,13 @@ fi
 
 ${HELM} repo add local http://localhost:8879/charts
 
+
+#OSH Makefile is bugged, so ensure helm is in the path
+if [[ ${HELM} != "helm" ]]
+then
+  export PATH=${PATH}:$(dirname ${HELM})
+fi
+
 {
     cd "${SERVE_DIR}"
     git clone --depth 1 https://git.openstack.org/openstack/openstack-helm-infra.git || true

tools/zuul/playbooks/doc-build.yaml

@@ -0,0 +1,20 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: primary
+  tasks:
+    - name: Build documents locally
+      make:
+        chdir: "{{ zuul.project.src_dir }}"
+        target: docs
+      register: result
+      failed_when: result.failed

tools/zuul/playbooks/helm-linter.yaml

@@ -0,0 +1,21 @@
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: primary
+  tasks:
+    - name: Execute the make target for Helm chart linting
+      make:
+        chdir: "{{ zuul.project.src_dir }}"
+        target: helm-lint
+      register: result
+      failed_when: result.failed

tools/zuul/playbooks/pep8-linter.yaml

@@ -0,0 +1,20 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: primary
+  tasks:
+    - name: Execute the make target for PEP8 linting
+      make:
+        chdir: "{{ zuul.project.src_dir }}"
+        target: tests-pep8
+      register: result
+      failed_when: result.failed

tools/zuul/playbooks/security-bandit.yaml

@@ -0,0 +1,20 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: primary
+  tasks:
+    - name: Execute the make target for security scanning
+      make:
+        chdir: "{{ zuul.project.src_dir }}"
+        target: tests-security
+      register: result
+      failed_when: result.failed

tools/zuul/playbooks/unit-py35.yaml

@@ -0,0 +1,20 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: primary
+  tasks:
+    - name: Execute the make target for unit testing
+      make:
+        chdir: "{{ zuul.project.src_dir }}"
+        target: tests-unit
+      register: result
+      failed_when: result.failed

tools/zuul/playbooks/vars.yaml

@@ -0,0 +1,15 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+null: null

tools/zuul/playbooks/zuul-linter.yaml

@@ -15,6 +15,6 @@
 - hosts: primary
   tasks:
     - name: Execute a Whitespace Linter check
-      command: find . -not -path "*/\.*" -not -path "*/doc/build/*" -not -name "*.tgz" -type f -exec egrep -l " +$" {} \;
+      command: find . -not -path "*/\.*" -not -path "*/docs/build/*" -not -name "*.tgz"  -type f -exec egrep -l " +$" {} \;
       register: result
-      failed_when: result.stdout != ""
+      failed_when: result.stdout != ""