From 45b209ac798667ea6601d71fd14f217c23781aea Mon Sep 17 00:00:00 2001 From: Vladimir Kozhukalov Date: Wed, 25 Oct 2023 16:28:42 -0500 Subject: [PATCH] Create osh-bandit role The motivation is to reduce the code base and get rid of unnecessary duplications. This PR is moves bandit tasks from the osh-infra-bandit.yaml playbook to the osh-bandit role. Then we can use this role for the same job in OSH. Change-Id: I9489a8c414e6679186e6c399243a7c0838df812a --- playbooks/mount-volumes.yaml | 17 ++++++++++ playbooks/osh-infra-bandit.yaml | 27 +--------------- playbooks/prepare-hosts.yaml | 1 - roles/osh-bandit/defaults/main.yaml | 17 ++++++++++ roles/osh-bandit/tasks/main.yaml | 50 +++++++++++++++++++++++++++++ tools/gate/template-python.sh | 16 --------- zuul.d/jobs.yaml | 1 + 7 files changed, 86 insertions(+), 43 deletions(-) create mode 100644 playbooks/mount-volumes.yaml create mode 100644 roles/osh-bandit/defaults/main.yaml create mode 100644 roles/osh-bandit/tasks/main.yaml delete mode 100755 tools/gate/template-python.sh diff --git a/playbooks/mount-volumes.yaml b/playbooks/mount-volumes.yaml new file mode 100644 index 000000000..0049da194 --- /dev/null +++ b/playbooks/mount-volumes.yaml @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- hosts: all + roles: + - mount-extra-volume +... diff --git a/playbooks/osh-infra-bandit.yaml b/playbooks/osh-infra-bandit.yaml index 1a118e92f..b77fa586b 100644 --- a/playbooks/osh-infra-bandit.yaml +++ b/playbooks/osh-infra-bandit.yaml @@ -15,30 +15,5 @@ roles: - ensure-python - ensure-pip - tasks: - - name: Install Helm - shell: | - TMP_DIR=$(mktemp -d) - curl -sSL https://get.helm.sh/helm-{{ helm_version }}-linux-amd64.tar.gz | tar -zxv --strip-components=1 -C ${TMP_DIR} - mv "${TMP_DIR}"/helm /usr/local/bin/helm - rm -rf "${TMP_DIR}" - sudo -H pip3 install --upgrade yq bandit=={{ bandit_version }} setuptools - environment: - zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}" - args: - chdir: "{{ zuul.project.src_dir }}" - - - name: Template out python files - shell: | - set -xe; - make all - mkdir -p python-files - ./tools/gate/template-python.sh - args: - chdir: "{{ zuul.project.src_dir }}" - - - name: Run bandit against python files - shell: bandit -r ./python-files - args: - chdir: "{{ zuul.project.src_dir }}" + - osh-bandit ... diff --git a/playbooks/prepare-hosts.yaml b/playbooks/prepare-hosts.yaml index 17ff03ee7..c64aa0d65 100644 --- a/playbooks/prepare-hosts.yaml +++ b/playbooks/prepare-hosts.yaml @@ -14,5 +14,4 @@ - hosts: all roles: - start-zuul-console - - mount-extra-volume ... diff --git a/roles/osh-bandit/defaults/main.yaml b/roles/osh-bandit/defaults/main.yaml new file mode 100644 index 000000000..3d6852845 --- /dev/null +++ b/roles/osh-bandit/defaults/main.yaml @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +work_dir: "{{ zuul.project.src_dir }}" +helm_version: "v3.6.3" +bandit_version: "1.7.1" +... diff --git a/roles/osh-bandit/tasks/main.yaml b/roles/osh-bandit/tasks/main.yaml new file mode 100644 index 000000000..961024b06 --- /dev/null +++ b/roles/osh-bandit/tasks/main.yaml @@ -0,0 +1,50 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Install Helm + shell: | + TMP_DIR=$(mktemp -d) + curl -sSL https://get.helm.sh/helm-{{ helm_version }}-linux-amd64.tar.gz | tar -zxv --strip-components=1 -C ${TMP_DIR} + mv "${TMP_DIR}"/helm /usr/local/bin/helm + rm -rf "${TMP_DIR}" + sudo -H pip3 install --upgrade yq bandit=={{ bandit_version }} setuptools + args: + chdir: "{{ work_dir }}" + +- name: Template out python files + shell: | + set -xe; + make all + mkdir -p python-files + EXCLUDES="helm-toolkit doc tests tools logs tmp roles playbooks releasenotes zuul.d python-files" + DIRS=`ls -d */ | cut -f1 -d'/'` + + for EX in $EXCLUDES; do + DIRS=`echo $DIRS | sed "s/\b$EX\b//g"` + done + + for DIR in $DIRS; do + PYFILES=$(helm template $DIR | yq 'select(.data != null) | .data | to_entries | map(select(.key | test(".*\\.py"))) | select(length > 0) | values[] | {(.key) : (.value)}' | jq -s add) + PYKEYS=$(echo "$PYFILES" | jq -r 'select(. != null) | keys[]') + for KEY in $PYKEYS; do + echo "$PYFILES" | jq -r --arg KEY "$KEY" '.[$KEY]' > ./python-files/"$DIR-$KEY" + done + done + args: + chdir: "{{ work_dir }}" + +- name: Run bandit against python files + shell: bandit -r ./python-files + args: + chdir: "{{ work_dir }}" +... diff --git a/tools/gate/template-python.sh b/tools/gate/template-python.sh deleted file mode 100755 index 19ef3a932..000000000 --- a/tools/gate/template-python.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -EXCLUDES="helm-toolkit doc tests tools logs tmp roles playbooks releasenotes zuul.d python-files" -DIRS=`ls -d */ | cut -f1 -d'/'` - -for EX in $EXCLUDES; do - DIRS=`echo $DIRS | sed "s/\b$EX\b//g"` -done - -for DIR in $DIRS; do - PYFILES=$(helm template $DIR | yq 'select(.data != null) | .data | to_entries | map(select(.key | test(".*\\.py"))) | select(length > 0) | values[] | {(.key) : (.value)}' | jq -s add) - PYKEYS=$(echo "$PYFILES" | jq -r 'select(. != null) | keys[]') - for KEY in $PYKEYS; do - echo "$PYFILES" | jq -r --arg KEY "$KEY" '.[$KEY]' > ./python-files/"$DIR-$KEY" - done -done diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 00d53720e..f99e3332e 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -78,6 +78,7 @@ timeout: 7200 pre-run: - playbooks/prepare-hosts.yaml + - playbooks/mount-volumes.yaml post-run: playbooks/osh-infra-collect-logs.yaml run: - playbooks/deploy-env.yaml