From b32e9b669bb390e1031ecabc7398b5e67d041669 Mon Sep 17 00:00:00 2001 From: Ian Howell Date: Wed, 1 Sep 2021 13:10:52 -0500 Subject: [PATCH] AIAP: Utilize document pull Closes: #623 Change-Id: I559c160eb33710c3e3105e5454d035a833ce4abf --- tools/airship-in-a-pod/Makefile | 8 +-- .../artifact-setup/Dockerfile | 16 ++---- .../artifact-setup/assets/entrypoint.sh | 24 ++------ .../examples/airshipctl/patchset.yaml | 23 +++++--- .../examples/base/airship-in-a-pod.yaml | 18 +++--- .../examples/treasuremap/patchset.yaml | 27 +++++---- .../runner/assets/entrypoint.sh | 57 ++++++------------- tools/deployment/22_test_configs.sh | 2 +- 8 files changed, 70 insertions(+), 105 deletions(-) diff --git a/tools/airship-in-a-pod/Makefile b/tools/airship-in-a-pod/Makefile index 8bdd53496..2b3139321 100644 --- a/tools/airship-in-a-pod/Makefile +++ b/tools/airship-in-a-pod/Makefile @@ -1,7 +1,7 @@ DOCKER_REGISTRY ?= quay.io DOCKER_IMAGE_PREFIX ?= airshipit DOCKER_IMAGE_TAG ?= latest -IMAGES ?= infra-builder runner +IMAGES ?= infra-builder runner artifact-setup PUBLISH ?= false .PHONY: help base libvirt artifact-setup $(IMAGES) images test @@ -17,12 +17,6 @@ images: base images: libvirt images: $(IMAGES) ## Build the containers. -artifact-setup: - docker build --tag $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_PREFIX)/aiap-artifact-setup:$(DOCKER_IMAGE_TAG) --build-arg AIRSHIPCTL_REF=$(AIRSHIPCTL_REF) ./artifact-setup -ifeq (true, $(PUBLISH)) - docker push $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_PREFIX)/aiap-artifact-setup:$(DOCKER_IMAGE_TAG) -endif - base: docker build --tag $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_PREFIX)/aiap-base:$(DOCKER_IMAGE_TAG) --build-arg BASE_IMAGE=ubuntu:20.04 ./base ifeq (true, $(PUBLISH)) diff --git a/tools/airship-in-a-pod/artifact-setup/Dockerfile b/tools/airship-in-a-pod/artifact-setup/Dockerfile index 330529f01..b3322aaa7 100644 --- a/tools/airship-in-a-pod/artifact-setup/Dockerfile +++ b/tools/airship-in-a-pod/artifact-setup/Dockerfile @@ -5,19 +5,13 @@ SHELL ["bash", "-exc"] ENV DEBIAN_FRONTEND noninteractive ARG USE_CACHED_ARTIFACTS="false" -ENV USE_CACHED_ARTIFACTS="false" +ENV USE_CACHED_ARTIFACTS=$USE_CACHED_ARTIFACTS -ARG MANIFEST_REPO_URL=https://review.opendev.org/airship/airshipctl -ENV MANIFEST_REPO_URL=$MANIFEST_REPO_URL +ARG AIRSHIPCTL_REPO_URL=https://opendev.org/airship/airshipctl +ENV AIRSHIPCTL_REPO_URL=$AIRSHIPCTL_REPO_URL -ARG MANIFEST_REPO_REF=master -ENV MANIFEST_REPO_REF=$MANIFEST_REPO_REF - -ARG MANIFEST_REPO_NAME=airshipctl -ENV MANIFEST_REPO_NAME=$MANIFEST_REPO_NAME - -ARG AIRSHIPCTL_REF= -ENV AIRSHIPCTL_REF=$AIRSHIPCTL_REF +ARG AIRSHIPCTL_REPO_REF=master +ENV AIRSHIPCTL_REPO_REF=$AIRSHIPCTL_REPO_REF # Update distro and install ansible RUN apt-get update ;\ diff --git a/tools/airship-in-a-pod/artifact-setup/assets/entrypoint.sh b/tools/airship-in-a-pod/artifact-setup/assets/entrypoint.sh index 2f4a404c2..819db6760 100755 --- a/tools/airship-in-a-pod/artifact-setup/assets/entrypoint.sh +++ b/tools/airship-in-a-pod/artifact-setup/assets/entrypoint.sh @@ -14,20 +14,17 @@ set -ex - function cloneRepo(){ - repo_name=$1 + repo_dir=$1 repo_url=$2 repo_ref=$3 - repo_dir="$ARTIFACTS_DIR/$repo_name" mkdir -p "$repo_dir" cd "$repo_dir" git init git fetch "$repo_url" "$repo_ref" git checkout FETCH_HEAD - } if [[ "$USE_CACHED_ARTIFACTS" = "true" ]] @@ -38,22 +35,13 @@ else printf "Waiting 30 seconds for the libvirt and docker services to be ready\n" sleep 30 - cloneRepo $MANIFEST_REPO_NAME $MANIFEST_REPO_URL $MANIFEST_REPO_REF + repo_dir="$ARTIFACTS_DIR/airshipctl" + cloneRepo "$repo_dir" "$AIRSHIPCTL_REPO_URL" "$AIRSHIPCTL_REPO_REF" - if [[ "$MANIFEST_REPO_NAME" != "airshipctl" ]] - then - cloneRepo airshipctl https://github.com/airshipit/airshipctl $AIRSHIPCTL_REF - fi - cd $ARTIFACTS_DIR/$MANIFEST_REPO_NAME - - if [[ "$MANIFEST_REPO_NAME" == "airshipctl" ]] - then - ./tools/deployment/21_systemwide_executable.sh - else - ./tools/deployment/airship-core/21_systemwide_executable.sh - fi + cd "$repo_dir" + ./tools/deployment/21_systemwide_executable.sh mkdir -p bin - cp "$(which airshipctl)" bin + cp "$(command -v airshipctl)" bin fi /signal_complete artifact-setup diff --git a/tools/airship-in-a-pod/examples/airshipctl/patchset.yaml b/tools/airship-in-a-pod/examples/airshipctl/patchset.yaml index 8d8d8ecb8..493ec12a9 100644 --- a/tools/airship-in-a-pod/examples/airshipctl/patchset.yaml +++ b/tools/airship-in-a-pod/examples/airshipctl/patchset.yaml @@ -10,22 +10,31 @@ # See the License for the specific language governing permissions and # limitations under the License. +# This is the location from which to clone the airshipctl binary repo - op: replace path: "/spec/containers/4/env/3/value" - value: https://review.opendev.org/airship/airshipctl + value: https://opendev.org/airship/airshipctl +# This is the ref to checkout for the airshipctl binary repo - op: replace path: "/spec/containers/4/env/4/value" value: master -- op: replace - path: "/spec/containers/4/env/5/value" - value: airshipctl - -# runner container +# This is the location from which to clone the manifest documents repo - op: replace path: "/spec/containers/6/env/3/value" - value: airshipctl + value: https://opendev.org/airship/airshipctl + +# This is the branch or sha to checkout for the manifest documents repo +- op: replace + path: "/spec/containers/6/env/4/value" + value: master + +# This is the ref to checkout for the manifest documents repo +# Note that this will take precedence over the branch if specified above +- op: replace + path: "/spec/containers/6/env/5/value" + value: # for local testing #- op: add diff --git a/tools/airship-in-a-pod/examples/base/airship-in-a-pod.yaml b/tools/airship-in-a-pod/examples/base/airship-in-a-pod.yaml index fc0cb4c80..112b7c3e4 100644 --- a/tools/airship-in-a-pod/examples/base/airship-in-a-pod.yaml +++ b/tools/airship-in-a-pod/examples/base/airship-in-a-pod.yaml @@ -216,14 +216,10 @@ spec: value: "false" - name: ARTIFACTS_DIR value: /opt/aiap-artifacts - - name: MANIFEST_REPO_URL - value: https://review.opendev.org/airship/airshipctl - - name: MANIFEST_REPO_REF + - name: AIRSHIPCTL_REPO_URL + value: https://opendev.org/airship/airshipctl + - name: AIRSHIPCTL_REPO_REF value: master - - name: MANIFEST_REPO_NAME - value: airshipctl - - name: AIRSHIPCTL_REF - value: - name: USE_PROXY value: - name: no_proxy @@ -320,8 +316,12 @@ spec: value: /opt/aiap-artifacts - name: USE_CACHED_ISO value: "false" - - name: MANIFEST_REPO_NAME - value: airshipctl + - name: AIRSHIP_CONFIG_PHASE_REPO_URL + value: https://review.opendev.org/airship/airshipctl + - name: AIRSHIP_CONFIG_PHASE_REPO_BRANCH + value: master + - name: AIRSHIP_CONFIG_PHASE_REPO_REF + value: - name: USE_PROXY value: - name: no_proxy diff --git a/tools/airship-in-a-pod/examples/treasuremap/patchset.yaml b/tools/airship-in-a-pod/examples/treasuremap/patchset.yaml index ef1aeeaf1..651fab9db 100644 --- a/tools/airship-in-a-pod/examples/treasuremap/patchset.yaml +++ b/tools/airship-in-a-pod/examples/treasuremap/patchset.yaml @@ -10,26 +10,31 @@ # See the License for the specific language governing permissions and # limitations under the License. +# This is the location from which to clone the airshipctl binary repo - op: replace path: "/spec/containers/4/env/3/value" - value: https://review.opendev.org/airship/treasuremap + value: https://opendev.org/airship/airshipctl +# This is the ref to checkout for the airshipctl binary repo - op: replace path: "/spec/containers/4/env/4/value" value: master -- op: replace - path: "/spec/containers/4/env/5/value" - value: treasuremap - -- op: replace - path: "/spec/containers/4/env/6/value" - value: 63c1faf718fd3341fc5bd975e575e3cf41647206 - -# runner container +# This is the location from which to clone the manifest documents repo - op: replace path: "/spec/containers/6/env/3/value" - value: treasuremap + value: https://opendev.org/airship/treasuremap + +# This is the branch or sha to checkout for the manifest documents repo +- op: replace + path: "/spec/containers/6/env/4/value" + value: 63c1faf718fd3341fc5bd975e575e3cf41647206 + +# This is the ref to checkout for the manifest documents repo +# Note that this will take precedence over the branch if specified above +- op: replace + path: "/spec/containers/6/env/5/value" + value: # for local testing #- op: add diff --git a/tools/airship-in-a-pod/runner/assets/entrypoint.sh b/tools/airship-in-a-pod/runner/assets/entrypoint.sh index 4a20317d8..1ced21172 100755 --- a/tools/airship-in-a-pod/runner/assets/entrypoint.sh +++ b/tools/airship-in-a-pod/runner/assets/entrypoint.sh @@ -28,16 +28,6 @@ kustomize_download_url="https://github.com/kubernetes-sigs/kustomize/releases/do curl -sSL "$kustomize_download_url" | tar -C /tmp -xzf - install /tmp/kustomize /usr/local/bin -cp "$ARTIFACTS_DIR/$MANIFEST_REPO_NAME/bin/airshipctl" /usr/local/bin/airshipctl -if [ $MANIFEST_REPO_NAME != "airshipctl" ] -then - export AIRSHIP_CONFIG_PHASE_REPO_URL="https://opendev.org/airship/treasuremap" - cp -r $ARTIFACTS_DIR/airshipctl/ /opt/airshipctl -fi - -cp -r $ARTIFACTS_DIR/$MANIFEST_REPO_NAME/ /opt/$MANIFEST_REPO_NAME -cd /opt/$MANIFEST_REPO_NAME - curl -fsSL -o /sops-key.asc https://raw.githubusercontent.com/mozilla/sops/master/pgp/sops_functional_tests_key.asc SOPS_PGP_FP="FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4" SOPS_IMPORT_PGP="$(cat /sops-key.asc)" @@ -46,50 +36,35 @@ export SOPS_PGP_FP echo 'export SOPS_IMPORT_PGP="$(cat /sops-key.asc)"' >> ~/.bashrc echo "export SOPS_PGP_FP=${SOPS_PGP_FP}" >> ~/.bashrc -export AIRSHIP_CONFIG_MANIFEST_DIRECTORY="/tmp/airship" +install "$ARTIFACTS_DIR/airshipctl/bin/airshipctl" /usr/local/bin +cd "$ARTIFACTS_DIR/airshipctl" -# By default, don't build airshipctl - use the binary from the shared volume instead -# ./tools/deployment/21_systemwide_executable.sh -if [ "$MANIFEST_REPO_NAME" == "airshipctl" ] -then - ./tools/deployment/22_test_configs.sh - # `airshipctl document pull` doesn't support pull patchsets yet - #./tools/deployment/23_pull_documents.sh - mkdir /tmp/airship - cp -rp /opt/airshipctl /tmp/airship/airshipctl - ./tools/deployment/23_generate_secrets.sh +export AIRSHIP_CONFIG_MANIFEST_DIRECTORY="$ARTIFACTS_DIR/manifests" +./tools/deployment/22_test_configs.sh +if [[ -n "$AIRSHIP_CONFIG_PHASE_REPO_REF" ]]; then + export NO_CHECKOUT="false" else - ./tools/deployment/airship-core/22_test_configs.sh - ./tools/deployment/airship-core/23_pull_documents.sh - ./tools/deployment/airship-core/23_generate_secrets.sh + export NO_CHECKOUT="true" fi +./tools/deployment/23_pull_documents.sh +./tools/deployment/23_generate_secrets.sh echo "export KUBECONFIG=$HOME/.airship/kubeconfig" >> ~/.bashrc -sed -i -e 's#bmcAddress: redfish+http://\([0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+\):8000#bmcAddress: redfish+https://10.23.25.1:8443#' "/tmp/airship/$MANIFEST_REPO_NAME/manifests/site/test-site/target/catalogues/hosts.yaml" -sed -i -e 's#root#username#' "/tmp/airship/$MANIFEST_REPO_NAME/manifests/site/test-site/target/catalogues/hosts.yaml" -sed -i -e 's#r00tme#password#' "/tmp/airship/$MANIFEST_REPO_NAME/manifests/site/test-site/target/catalogues/hosts.yaml" -sed -i -e 's#disableCertificateVerification: false#disableCertificateVerification: true#' "/tmp/airship/$MANIFEST_REPO_NAME/manifests/site/test-site/target/catalogues/hosts.yaml" +repo_name=$(yq -r .manifests.dummy_manifest.repositories.primary.url /root/.airship/config | awk 'BEGIN {FS="/"} {print $NF}') +sed -i -e 's#bmcAddress: redfish+http://\([0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+\):8000#bmcAddress: redfish+https://10.23.25.1:8443#' "$AIRSHIP_CONFIG_MANIFEST_DIRECTORY/$repo_name/manifests/site/test-site/target/catalogues/hosts.yaml" +sed -i -e 's#root#username#' "$AIRSHIP_CONFIG_MANIFEST_DIRECTORY/$repo_name/manifests/site/test-site/target/catalogues/hosts.yaml" +sed -i -e 's#r00tme#password#' "$AIRSHIP_CONFIG_MANIFEST_DIRECTORY/$repo_name/manifests/site/test-site/target/catalogues/hosts.yaml" +sed -i -e 's#disableCertificateVerification: false#disableCertificateVerification: true#' "$AIRSHIP_CONFIG_MANIFEST_DIRECTORY/$repo_name/manifests/site/test-site/target/catalogues/hosts.yaml" if [[ "$USE_CACHED_ISO" = "true" ]]; then mkdir -p /srv/images tar -xzf "$CACHE_DIR/iso.tar.gz" --directory /srv/images else - if [ "$MANIFEST_REPO_NAME" == "airshipctl" ] - then - ./tools/deployment/24_build_images.sh - else - ./tools/deployment/airship-core/24_build_images.sh - fi - + ./tools/deployment/24_build_images.sh tar -czf "$ARTIFACTS_DIR/iso.tar.gz" --directory=/srv/images . fi -if [ "$MANIFEST_REPO_NAME" == "airshipctl" ] -then - ./tools/deployment/25_deploy_gating.sh -else - ./tools/deployment/airship-core/25_deploy_gating.sh -fi +./tools/deployment/25_deploy_gating.sh /signal_complete runner diff --git a/tools/deployment/22_test_configs.sh b/tools/deployment/22_test_configs.sh index e0be3b9d6..2c074987b 100755 --- a/tools/deployment/22_test_configs.sh +++ b/tools/deployment/22_test_configs.sh @@ -36,7 +36,7 @@ export SYSTEM_ACTION_RETRIES=30 export SYSTEM_REBOOT_DELAY=30 # Branch and ref options are mutually exclusive. Ref gets preference since it's more specific if [[ -z "$AIRSHIP_CONFIG_PHASE_REPO_REF" ]]; then - export AIRSHIP_CONFIG_PHASE_REPO_BRANCH=${BRANCH:-"master"} + export AIRSHIP_CONFIG_PHASE_REPO_BRANCH=${AIRSHIP_CONFIG_PHASE_REPO_BRANCH:-"master"} export AIRSHIP_CONFIG_PHASE_REPO_REMOTE_REF="" else export AIRSHIP_CONFIG_PHASE_REPO_BRANCH=""