AIAP: Utilize document pull

Closes: #623 Change-Id: I559c160eb33710c3e3105e5454d035a833ce4abf
2021-09-01 13:10:52 -05:00 · 2021-09-01 13:10:52 -05:00 · b32e9b669b
parent 3a537cc859
commit b32e9b669b
8 changed files with 70 additions and 105 deletions
--- a/tools/airship-in-a-pod/Makefile
+++ b/tools/airship-in-a-pod/Makefile
@ -1,7 +1,7 @@
 DOCKER_REGISTRY     ?= quay.io
 DOCKER_IMAGE_PREFIX ?= airshipit
 DOCKER_IMAGE_TAG    ?= latest
-IMAGES              ?= infra-builder runner
+IMAGES              ?= infra-builder runner artifact-setup
 PUBLISH             ?= false

 .PHONY: help base libvirt artifact-setup $(IMAGES) images test
@ -17,12 +17,6 @@ images: base
 images: libvirt
 images: $(IMAGES) ## Build the containers.

-artifact-setup:
-	docker build --tag $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_PREFIX)/aiap-artifact-setup:$(DOCKER_IMAGE_TAG) --build-arg AIRSHIPCTL_REF=$(AIRSHIPCTL_REF) ./artifact-setup
-ifeq (true, $(PUBLISH))
-	docker push $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_PREFIX)/aiap-artifact-setup:$(DOCKER_IMAGE_TAG)
-endif
-
 base:
 	docker build --tag $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_PREFIX)/aiap-base:$(DOCKER_IMAGE_TAG) --build-arg BASE_IMAGE=ubuntu:20.04 ./base
 ifeq (true, $(PUBLISH))
--- a/tools/airship-in-a-pod/artifact-setup/Dockerfile
+++ b/tools/airship-in-a-pod/artifact-setup/Dockerfile
@ -5,19 +5,13 @@ SHELL ["bash", "-exc"]
 ENV DEBIAN_FRONTEND noninteractive

 ARG USE_CACHED_ARTIFACTS="false"
-ENV USE_CACHED_ARTIFACTS="false"
+ENV USE_CACHED_ARTIFACTS=$USE_CACHED_ARTIFACTS

-ARG MANIFEST_REPO_URL=https://review.opendev.org/airship/airshipctl
-ENV MANIFEST_REPO_URL=$MANIFEST_REPO_URL
+ARG AIRSHIPCTL_REPO_URL=https://opendev.org/airship/airshipctl
+ENV AIRSHIPCTL_REPO_URL=$AIRSHIPCTL_REPO_URL

-ARG MANIFEST_REPO_REF=master
-ENV MANIFEST_REPO_REF=$MANIFEST_REPO_REF
-
-ARG MANIFEST_REPO_NAME=airshipctl
-ENV MANIFEST_REPO_NAME=$MANIFEST_REPO_NAME
-
-ARG AIRSHIPCTL_REF=
-ENV AIRSHIPCTL_REF=$AIRSHIPCTL_REF
+ARG AIRSHIPCTL_REPO_REF=master
+ENV AIRSHIPCTL_REPO_REF=$AIRSHIPCTL_REPO_REF

 # Update distro and install ansible
 RUN apt-get update ;\
--- a/tools/airship-in-a-pod/artifact-setup/assets/entrypoint.sh
+++ b/tools/airship-in-a-pod/artifact-setup/assets/entrypoint.sh
@ -14,20 +14,17 @@

 set -ex

-
 function cloneRepo(){
-  repo_name=$1
+  repo_dir=$1
  repo_url=$2
  repo_ref=$3

-  repo_dir="$ARTIFACTS_DIR/$repo_name"
  mkdir -p "$repo_dir"
  cd "$repo_dir"

  git init
  git fetch "$repo_url" "$repo_ref"
  git checkout FETCH_HEAD
-
 }

 if [[ "$USE_CACHED_ARTIFACTS" = "true" ]]
@ -38,22 +35,13 @@ else
  printf "Waiting 30 seconds for the libvirt and docker services to be ready\n"
  sleep 30

-  cloneRepo $MANIFEST_REPO_NAME $MANIFEST_REPO_URL $MANIFEST_REPO_REF
+  repo_dir="$ARTIFACTS_DIR/airshipctl"
+  cloneRepo "$repo_dir" "$AIRSHIPCTL_REPO_URL" "$AIRSHIPCTL_REPO_REF"

-  if [[ "$MANIFEST_REPO_NAME" !=  "airshipctl" ]]
-  then
-	  cloneRepo airshipctl https://github.com/airshipit/airshipctl $AIRSHIPCTL_REF
-  fi
-  cd $ARTIFACTS_DIR/$MANIFEST_REPO_NAME
-
-  if [[ "$MANIFEST_REPO_NAME" ==  "airshipctl" ]]
-  then
-    ./tools/deployment/21_systemwide_executable.sh
-  else
-    ./tools/deployment/airship-core/21_systemwide_executable.sh
-  fi
+  cd "$repo_dir"
+  ./tools/deployment/21_systemwide_executable.sh
  mkdir -p bin
-  cp "$(which airshipctl)" bin
+  cp "$(command -v airshipctl)" bin
 fi

 /signal_complete artifact-setup
--- a/tools/airship-in-a-pod/examples/airshipctl/patchset.yaml
+++ b/tools/airship-in-a-pod/examples/airshipctl/patchset.yaml
@ -10,22 +10,31 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+# This is the location from which to clone the airshipctl binary repo
 - op: replace
  path: "/spec/containers/4/env/3/value"
-  value: https://review.opendev.org/airship/airshipctl
+  value: https://opendev.org/airship/airshipctl

+# This is the ref to checkout for the airshipctl binary repo
 - op: replace
  path: "/spec/containers/4/env/4/value"
  value: master

- op: replace
-  path: "/spec/containers/4/env/5/value"
-  value: airshipctl
-
-# runner container
+# This is the location from which to clone the manifest documents repo
 - op: replace
  path: "/spec/containers/6/env/3/value"
-  value: airshipctl
+  value: https://opendev.org/airship/airshipctl
+
+# This is the branch or sha to checkout for the manifest documents repo
+- op: replace
+  path: "/spec/containers/6/env/4/value"
+  value: master
+
+# This is the ref to checkout for the manifest documents repo
+# Note that this will take precedence over the branch if specified above
+- op: replace
+  path: "/spec/containers/6/env/5/value"
+  value:

 # for local testing
 #- op: add
--- a/tools/airship-in-a-pod/examples/base/airship-in-a-pod.yaml
+++ b/tools/airship-in-a-pod/examples/base/airship-in-a-pod.yaml
@ -216,14 +216,10 @@ spec:
        value: "false"
      - name: ARTIFACTS_DIR
        value: /opt/aiap-artifacts
-      - name: MANIFEST_REPO_URL
-        value: https://review.opendev.org/airship/airshipctl
-      - name: MANIFEST_REPO_REF
+      - name: AIRSHIPCTL_REPO_URL
+        value: https://opendev.org/airship/airshipctl
+      - name: AIRSHIPCTL_REPO_REF
        value: master
-      - name: MANIFEST_REPO_NAME
-        value: airshipctl
-      - name: AIRSHIPCTL_REF
-        value:
      - name: USE_PROXY
        value:
      - name: no_proxy
@ -320,8 +316,12 @@ spec:
        value: /opt/aiap-artifacts
      - name: USE_CACHED_ISO
        value: "false"
-      - name: MANIFEST_REPO_NAME
-        value: airshipctl
+      - name: AIRSHIP_CONFIG_PHASE_REPO_URL
+        value: https://review.opendev.org/airship/airshipctl
+      - name: AIRSHIP_CONFIG_PHASE_REPO_BRANCH
+        value: master
+      - name: AIRSHIP_CONFIG_PHASE_REPO_REF
+        value:
      - name: USE_PROXY
        value:
      - name: no_proxy
--- a/tools/airship-in-a-pod/examples/treasuremap/patchset.yaml
+++ b/tools/airship-in-a-pod/examples/treasuremap/patchset.yaml
@ -10,26 +10,31 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+# This is the location from which to clone the airshipctl binary repo
 - op: replace
  path: "/spec/containers/4/env/3/value"
-  value: https://review.opendev.org/airship/treasuremap
+  value: https://opendev.org/airship/airshipctl

+# This is the ref to checkout for the airshipctl binary repo
 - op: replace
  path: "/spec/containers/4/env/4/value"
  value: master

- op: replace
-  path: "/spec/containers/4/env/5/value"
-  value: treasuremap
-
- op: replace
-  path: "/spec/containers/4/env/6/value"
-  value: 63c1faf718fd3341fc5bd975e575e3cf41647206
-
-# runner container
+# This is the location from which to clone the manifest documents repo
 - op: replace
  path: "/spec/containers/6/env/3/value"
-  value: treasuremap
+  value: https://opendev.org/airship/treasuremap
+
+# This is the branch or sha to checkout for the manifest documents repo
+- op: replace
+  path: "/spec/containers/6/env/4/value"
+  value: 63c1faf718fd3341fc5bd975e575e3cf41647206
+
+# This is the ref to checkout for the manifest documents repo
+# Note that this will take precedence over the branch if specified above
+- op: replace
+  path: "/spec/containers/6/env/5/value"
+  value:

 # for local testing
 #- op: add
--- a/tools/airship-in-a-pod/runner/assets/entrypoint.sh
+++ b/tools/airship-in-a-pod/runner/assets/entrypoint.sh
@ -28,16 +28,6 @@ kustomize_download_url="https://github.com/kubernetes-sigs/kustomize/releases/do
 curl -sSL "$kustomize_download_url" | tar -C /tmp -xzf -
 install /tmp/kustomize /usr/local/bin

-cp "$ARTIFACTS_DIR/$MANIFEST_REPO_NAME/bin/airshipctl" /usr/local/bin/airshipctl
-if [ $MANIFEST_REPO_NAME != "airshipctl" ]
-then
-  export AIRSHIP_CONFIG_PHASE_REPO_URL="https://opendev.org/airship/treasuremap"
-  cp -r $ARTIFACTS_DIR/airshipctl/ /opt/airshipctl
-fi
-
-cp  -r $ARTIFACTS_DIR/$MANIFEST_REPO_NAME/ /opt/$MANIFEST_REPO_NAME
-cd /opt/$MANIFEST_REPO_NAME
-
 curl -fsSL -o /sops-key.asc https://raw.githubusercontent.com/mozilla/sops/master/pgp/sops_functional_tests_key.asc
 SOPS_PGP_FP="FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4"
 SOPS_IMPORT_PGP="$(cat /sops-key.asc)"
@ -46,50 +36,35 @@ export SOPS_PGP_FP
 echo 'export SOPS_IMPORT_PGP="$(cat /sops-key.asc)"' >> ~/.bashrc
 echo "export SOPS_PGP_FP=${SOPS_PGP_FP}" >> ~/.bashrc

-export AIRSHIP_CONFIG_MANIFEST_DIRECTORY="/tmp/airship"
+install "$ARTIFACTS_DIR/airshipctl/bin/airshipctl" /usr/local/bin
+cd "$ARTIFACTS_DIR/airshipctl"

-# By default, don't build airshipctl - use the binary from the shared volume instead
-# ./tools/deployment/21_systemwide_executable.sh
-if [ "$MANIFEST_REPO_NAME" == "airshipctl" ]
-then
-  ./tools/deployment/22_test_configs.sh
-  # `airshipctl document pull` doesn't support pull patchsets yet
-  #./tools/deployment/23_pull_documents.sh
-  mkdir /tmp/airship
-  cp -rp /opt/airshipctl /tmp/airship/airshipctl
-  ./tools/deployment/23_generate_secrets.sh
+export AIRSHIP_CONFIG_MANIFEST_DIRECTORY="$ARTIFACTS_DIR/manifests"
+./tools/deployment/22_test_configs.sh
+if [[ -n "$AIRSHIP_CONFIG_PHASE_REPO_REF" ]]; then
+  export NO_CHECKOUT="false"
 else
-  ./tools/deployment/airship-core/22_test_configs.sh
-  ./tools/deployment/airship-core/23_pull_documents.sh
-  ./tools/deployment/airship-core/23_generate_secrets.sh
+  export NO_CHECKOUT="true"
 fi
+./tools/deployment/23_pull_documents.sh
+./tools/deployment/23_generate_secrets.sh

 echo "export KUBECONFIG=$HOME/.airship/kubeconfig" >> ~/.bashrc

-sed -i -e 's#bmcAddress: redfish+http://\([0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+\):8000#bmcAddress: redfish+https://10.23.25.1:8443#' "/tmp/airship/$MANIFEST_REPO_NAME/manifests/site/test-site/target/catalogues/hosts.yaml"
-sed -i -e 's#root#username#' "/tmp/airship/$MANIFEST_REPO_NAME/manifests/site/test-site/target/catalogues/hosts.yaml"
-sed -i -e 's#r00tme#password#' "/tmp/airship/$MANIFEST_REPO_NAME/manifests/site/test-site/target/catalogues/hosts.yaml"
-sed -i -e 's#disableCertificateVerification: false#disableCertificateVerification: true#' "/tmp/airship/$MANIFEST_REPO_NAME/manifests/site/test-site/target/catalogues/hosts.yaml"
+repo_name=$(yq -r .manifests.dummy_manifest.repositories.primary.url /root/.airship/config | awk 'BEGIN {FS="/"} {print $NF}')
+sed -i -e 's#bmcAddress: redfish+http://\([0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+\):8000#bmcAddress: redfish+https://10.23.25.1:8443#' "$AIRSHIP_CONFIG_MANIFEST_DIRECTORY/$repo_name/manifests/site/test-site/target/catalogues/hosts.yaml"
+sed -i -e 's#root#username#' "$AIRSHIP_CONFIG_MANIFEST_DIRECTORY/$repo_name/manifests/site/test-site/target/catalogues/hosts.yaml"
+sed -i -e 's#r00tme#password#' "$AIRSHIP_CONFIG_MANIFEST_DIRECTORY/$repo_name/manifests/site/test-site/target/catalogues/hosts.yaml"
+sed -i -e 's#disableCertificateVerification: false#disableCertificateVerification: true#' "$AIRSHIP_CONFIG_MANIFEST_DIRECTORY/$repo_name/manifests/site/test-site/target/catalogues/hosts.yaml"

 if [[ "$USE_CACHED_ISO" = "true" ]]; then
  mkdir -p /srv/images
  tar -xzf "$CACHE_DIR/iso.tar.gz" --directory /srv/images
 else
-  if [ "$MANIFEST_REPO_NAME" == "airshipctl" ]
-  then
-    ./tools/deployment/24_build_images.sh
-  else
-    ./tools/deployment/airship-core/24_build_images.sh
-  fi
-
+  ./tools/deployment/24_build_images.sh
  tar -czf "$ARTIFACTS_DIR/iso.tar.gz" --directory=/srv/images .
 fi

-if [ "$MANIFEST_REPO_NAME" == "airshipctl" ]
-then
-  ./tools/deployment/25_deploy_gating.sh
-else
-  ./tools/deployment/airship-core/25_deploy_gating.sh
-fi
+./tools/deployment/25_deploy_gating.sh

 /signal_complete runner
--- a/tools/deployment/22_test_configs.sh
+++ b/tools/deployment/22_test_configs.sh
@ -36,7 +36,7 @@ export SYSTEM_ACTION_RETRIES=30
 export SYSTEM_REBOOT_DELAY=30
 # Branch and ref options are mutually exclusive. Ref gets preference since it's more specific
 if [[ -z "$AIRSHIP_CONFIG_PHASE_REPO_REF" ]]; then
-  export AIRSHIP_CONFIG_PHASE_REPO_BRANCH=${BRANCH:-"master"}
+  export AIRSHIP_CONFIG_PHASE_REPO_BRANCH=${AIRSHIP_CONFIG_PHASE_REPO_BRANCH:-"master"}
  export AIRSHIP_CONFIG_PHASE_REPO_REMOTE_REF=""
 else
  export AIRSHIP_CONFIG_PHASE_REPO_BRANCH=""