From 8992254b3fcfb92581b51f5ad91cf3b79fb5a50a Mon Sep 17 00:00:00 2001 From: Tin Lam Date: Fri, 5 Feb 2021 18:19:31 -0600 Subject: [PATCH] feat(pipeline): enables images value override This patch set allows the ability to override the application image tag key so it takes in the correct image information during the deployment stage of Jarvis. This patch set rebases on [0] to leverage a multichart configmap paradigm rather than a single chart. [0] https://review.opendev.org/c/airship/charts/+/772135 Signed-off-by: Tin Lam Change-Id: Idbf76bf27460fdd49fa9acf0cd64fb440202c20f --- .../config_map.yaml.example | 94 +++++++++++-------- .../templates/task-functional.yaml | 15 ++- tools/images/standard-container/Dockerfile | 1 + .../roles/functional/tasks/main.yaml | 31 +++++- 4 files changed, 96 insertions(+), 45 deletions(-) diff --git a/charts/development-pipeline/config_map.yaml.example b/charts/development-pipeline/config_map.yaml.example index 41f9c9a6..1f825819 100644 --- a/charts/development-pipeline/config_map.yaml.example +++ b/charts/development-pipeline/config_map.yaml.example @@ -5,37 +5,37 @@ metadata: data: cluster.json: | { - "use_existing_cluster": "true" + "use_existing_cluster": "true" } image.json: | [ { - "build_from_source": false, - "image_name": "mongodb-sharded", - "project": "mongodb", - "repo": "mongodb-sharded", - "tag": "4.4.3", - "image_fullname": "mongodb/mongodb-sharded:4.4.3", - "path": "", - "build": { + "build_from_source": false, + "image_name": "mongodb-sharded", + "project": "mongodb", + "repo": "mongodb-sharded", + "tag": "4.4.3", + "image_fullname": "mongodb/mongodb-sharded:4.4.3", + "path": "", + "build": { "repo": "docker.io/bitnami/mongodb-sharded", "tag": "4.4.3-debian-10-r44" - } + } }, { - "build_from_source": true, - "image_name": "mongodb", - "project": "mongodb", - "repo": "mongodb", - "tag": "1.built", - "image_fullname": "mongodb/mongodb:1.0", - "path": "tools/gate/jarvis/5G-SA-core/mongodb/images/mongodb", - "build": { + "build_from_source": true, + "image_name": "mongodb", + "project": "mongodb", + "repo": "mongodb", + "tag": "1.built", + "image_fullname": "mongodb/mongodb:1.0", + "path": "tools/gate/jarvis/5G-SA-core/mongodb/images/mongodb", + "build": { "repo": "https://review.opendev.org/airship/charts", "target_loc": "/src/checkout/mongodb", "refspec": "master", "version": "refs/changes/*:refs/changes/*" - } + } } ] chart.json: | @@ -54,13 +54,18 @@ data: }, "namespace": "development-pipeline", "release_name": "mongodb-bitnami", - "images": { - "applications": { - "mongodb-sharded": { - "tag": "1.0", - "name": "mongodb", - "repo": "mongodb" - } + "sources": { + "image_map": { + "tag": ["releases", "image", "tag"], + "tmp_name": ["releases", "image", "repository"], + "tmp_repo": ["releases", "image", "registry"] + } + }, + "releases": { + "image": { + "registry": "docker.io", + "repository": "bitnami/mongodb-sharded", + "tag": "4.4.3-debian-10-r44" } } }, @@ -78,12 +83,21 @@ data: }, "namespace": "development-pipeline", "release_name": "mongodb", - "images": { - "applications": { - "mongodb": { - "tag": "1.0", - "name": "mongodb", - "repo": "mongodb" + "sources": { + "image_map": { + "tag": ["releases", "images", "applications", "mongodb", "tag"], + "tmp_name": ["releases", "images", "applications", "mongodb", "name"], + "tmp_repo": ["releases", "images", "applications", "mongodb", "repo"] + } + }, + "releases": { + "images": { + "applications": { + "mongodb": { + "tag": "1.0", + "name": "mongodb", + "repo": "mongodb" + } } } } @@ -97,15 +111,15 @@ data: "noproxy": "", "enabled": false }, - "chart_registry_url": "harbor-core.jarvis.local/chartrepo", - "chart_repository": "jarvis-harbor", - "docker_registry": "harbor-core.jarvis.local", - "harbor_secret_mounted_path": "/workspace/helm-creds", - "clone_async_batch_size": 20, - "chart_async_batch_size": 20, - "image_async_batch_size": 2 + "chart_registry_url": "harbor-core.jarvis.local/chartrepo", + "chart_repository": "jarvis-harbor", + "docker_registry": "harbor-core.jarvis.local", + "harbor_secret_mounted_path": "/workspace/helm-creds", + "clone_async_batch_size": 20, + "chart_async_batch_size": 20, + "image_async_batch_size": 2 } cleanup.json: | { - "remove_artifacts": "true" + "remove_artifacts": "true" } diff --git a/charts/development-pipeline/templates/task-functional.yaml b/charts/development-pipeline/templates/task-functional.yaml index c3d5f6ca..5fabfe2c 100644 --- a/charts/development-pipeline/templates/task-functional.yaml +++ b/charts/development-pipeline/templates/task-functional.yaml @@ -23,13 +23,22 @@ spec: script: | #!/bin/sh update-ca-certificates - ansible-playbook -vvv {{ $.Values.tasks.functional.functionalPlaybook }} -i hosts -e '{"stage":"deploy"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/cluster.json" -e 'loop_source="$(workspaces.development_pipeline_data.path)/chart.json"' + ansible-playbook -vvv {{ $.Values.tasks.functional.functionalPlaybook }} -i hosts \ + -e '{"stage":"deploy"}' \ + -e @"$(workspaces.development_pipeline_data.path)/default.json" \ + -e @"$(workspaces.development_pipeline_data.path)/cluster.json" \ + -e 'loop_source="$(workspaces.development_pipeline_data.path)/chart.json"' \ + -e 'datapath="$(workspaces.development_pipeline_data.path)"' - name: run-helm-tests image: {{ $.Values.tasks.functional.functionalTestImage }} script: | #!/bin/sh - ansible-playbook -vvv {{ $.Values.tasks.functional.functionalPlaybook }} -i hosts -e '{"stage":"test"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/cluster.json" -e 'loop_source="$(workspaces.development_pipeline_data.path)/chart.json"' + ansible-playbook -vvv {{ $.Values.tasks.functional.functionalPlaybook }} -i hosts \ + -e '{"stage":"test"}' \ + -e @"$(workspaces.development_pipeline_data.path)/default.json" \ + -e @"$(workspaces.development_pipeline_data.path)/cluster.json" \ + -e 'loop_source="$(workspaces.development_pipeline_data.path)/chart.json"' volumes: - name: helm-publish-creds secret: @@ -39,4 +48,4 @@ spec: secretName: harbor-ca - name: kubeconfig secret: - secretName: kubeconfig-secret \ No newline at end of file + secretName: kubeconfig-secret diff --git a/tools/images/standard-container/Dockerfile b/tools/images/standard-container/Dockerfile index 69468452..047f8a4d 100644 --- a/tools/images/standard-container/Dockerfile +++ b/tools/images/standard-container/Dockerfile @@ -21,6 +21,7 @@ RUN apt-get update ;\ ca-certificates \ gnupg-agent \ software-properties-common \ + moreutils \ gettext-base ;\ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - ;\ add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" ;\ diff --git a/tools/images/standard-container/assets/playbooks/roles/functional/tasks/main.yaml b/tools/images/standard-container/assets/playbooks/roles/functional/tasks/main.yaml index 4bb86052..ff00b7a0 100644 --- a/tools/images/standard-container/assets/playbooks/roles/functional/tasks/main.yaml +++ b/tools/images/standard-container/assets/playbooks/roles/functional/tasks/main.yaml @@ -6,9 +6,36 @@ when: ("{{ stage }}" == "deploy") block: #Deploy CNF - - name: Deploy charts - shell: helm upgrade --install --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" "{{ chart.release_name }}" "{{ chart.project }}-staging/{{ chart.chart_name }}" --version="{{ chart.version }}" --namespace="{{ chart.namespace }}" + shell: | + set -xe ; + CHARTNAME={{ chart.chart_name }} ; + REPO={{ docker_registry }} ; + NAME={{ chart.project }}-staging/{{ chart.chart_name }} ; + + jq -c --arg c $CHARTNAME --arg v $REPO 'map(if .repo == $c then .["tmp_repo"] = $v else . end)' {{ datapath }}/image.json | sponge {{ datapath }}/image.json ; + jq -c --arg c $CHARTNAME --arg v $NAME 'map(if .repo == $c then .["tmp_name"] = $v else . end)' {{ datapath }}/image.json | sponge {{ datapath }}/image.json ; + + M=$(jq -c --arg c $CHARTNAME 'map(select(.chart_name == $c)) | first | .sources.image_map' {{ datapath }}/chart.json) ; + if [ "$M" != "null" ] && [ -n "$M" ] ; then \ + echo $M | jq -r 'keys[]' | while IFS= read -r k; do \ + v=$(echo $M | jq -c --arg k $k '.[$k]') ;\ + filter=".$k" ;\ + value=$(jq -c --arg c $CHARTNAME 'map(select(.image_name == $c)) | first' {{ datapath}}/image.json | jq -r $filter) ;\ + jq -r --arg c $CHARTNAME --argjson k $v --arg v $value 'map(if .chart_name == $c then getpath($k) = $v else . end)' {{ datapath }}/chart.json | sponge {{ datapath }}/chart.json ;\ + done ; \ + fi ; + jq -c --arg c $CHARTNAME 'map(select(.chart_name == $c)) | first | .releases' \ + {{ datapath }}/chart.json > {{ datapath }}/{{ chart.chart_name }}-overrides.json ; + cat {{ datapath }}/{{ chart.chart_name }}-overrides.json ; + helm upgrade --install \ + --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" \ + "{{ chart.release_name }}" "{{ chart.project }}-staging/{{ chart.chart_name }}" \ + --version="{{ chart.version }}" \ + --values="{{ datapath }}/{{ chart.chart_name }}-overrides.json" \ + --namespace="{{ chart.namespace }}" + args: + executable: /bin/bash loop: "{{ charts }}" loop_control: loop_var: "chart"