AIAP: Utilize document pull

Closes: #623

Change-Id: I559c160eb33710c3e3105e5454d035a833ce4abf

tools/airship-in-a-pod/Makefile

@@ -1,7 +1,7 @@
 DOCKER_REGISTRY     ?= quay.io
 DOCKER_IMAGE_PREFIX ?= airshipit
 DOCKER_IMAGE_TAG    ?= latest
-IMAGES              ?= infra-builder runner
+IMAGES              ?= infra-builder runner artifact-setup
 PUBLISH             ?= false
 
 .PHONY: help base libvirt artifact-setup $(IMAGES) images test
@@ -17,12 +17,6 @@ images: base
 images: libvirt
 images: $(IMAGES) ## Build the containers.
 
-artifact-setup:
-	docker build --tag $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_PREFIX)/aiap-artifact-setup:$(DOCKER_IMAGE_TAG) --build-arg AIRSHIPCTL_REF=$(AIRSHIPCTL_REF) ./artifact-setup
-ifeq (true, $(PUBLISH))
-	docker push $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_PREFIX)/aiap-artifact-setup:$(DOCKER_IMAGE_TAG)
-endif
-
 base:
 	docker build --tag $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_PREFIX)/aiap-base:$(DOCKER_IMAGE_TAG) --build-arg BASE_IMAGE=ubuntu:20.04 ./base
 ifeq (true, $(PUBLISH))

tools/airship-in-a-pod/artifact-setup/Dockerfile

@@ -5,19 +5,13 @@ SHELL ["bash", "-exc"]
 ENV DEBIAN_FRONTEND noninteractive
 
 ARG USE_CACHED_ARTIFACTS="false"
-ENV USE_CACHED_ARTIFACTS="false"
+ENV USE_CACHED_ARTIFACTS=$USE_CACHED_ARTIFACTS
 
-ARG MANIFEST_REPO_URL=https://review.opendev.org/airship/airshipctl
-ENV MANIFEST_REPO_URL=$MANIFEST_REPO_URL
+ARG AIRSHIPCTL_REPO_URL=https://opendev.org/airship/airshipctl
+ENV AIRSHIPCTL_REPO_URL=$AIRSHIPCTL_REPO_URL
 
-ARG MANIFEST_REPO_REF=master
-ENV MANIFEST_REPO_REF=$MANIFEST_REPO_REF
-
-ARG MANIFEST_REPO_NAME=airshipctl
-ENV MANIFEST_REPO_NAME=$MANIFEST_REPO_NAME
-
-ARG AIRSHIPCTL_REF=
-ENV AIRSHIPCTL_REF=$AIRSHIPCTL_REF
+ARG AIRSHIPCTL_REPO_REF=master
+ENV AIRSHIPCTL_REPO_REF=$AIRSHIPCTL_REPO_REF
 
 # Update distro and install ansible
 RUN apt-get update ;\

tools/airship-in-a-pod/artifact-setup/assets/entrypoint.sh

@@ -14,20 +14,17 @@
 
 set -ex
 
-
 function cloneRepo(){
-  repo_name=$1
+  repo_dir=$1
   repo_url=$2
   repo_ref=$3
 
-  repo_dir="$ARTIFACTS_DIR/$repo_name"
   mkdir -p "$repo_dir"
   cd "$repo_dir"
 
   git init
   git fetch "$repo_url" "$repo_ref"
   git checkout FETCH_HEAD
-
 }
 
 if [[ "$USE_CACHED_ARTIFACTS" = "true" ]]
@@ -38,22 +35,13 @@ else
   printf "Waiting 30 seconds for the libvirt and docker services to be ready\n"
   sleep 30
 
-  cloneRepo $MANIFEST_REPO_NAME $MANIFEST_REPO_URL $MANIFEST_REPO_REF
+  repo_dir="$ARTIFACTS_DIR/airshipctl"
+  cloneRepo "$repo_dir" "$AIRSHIPCTL_REPO_URL" "$AIRSHIPCTL_REPO_REF"
 
-  if [[ "$MANIFEST_REPO_NAME" !=  "airshipctl" ]]
-  then
-	  cloneRepo airshipctl https://github.com/airshipit/airshipctl $AIRSHIPCTL_REF
-  fi
-  cd $ARTIFACTS_DIR/$MANIFEST_REPO_NAME
-
-  if [[ "$MANIFEST_REPO_NAME" ==  "airshipctl" ]]
-  then
-    ./tools/deployment/21_systemwide_executable.sh
-  else
-    ./tools/deployment/airship-core/21_systemwide_executable.sh
-  fi
+  cd "$repo_dir"
+  ./tools/deployment/21_systemwide_executable.sh
   mkdir -p bin
-  cp "$(which airshipctl)" bin
+  cp "$(command -v airshipctl)" bin
 fi
 
 /signal_complete artifact-setup

tools/airship-in-a-pod/examples/airshipctl/patchset.yaml

@@ -10,22 +10,31 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# This is the location from which to clone the airshipctl binary repo
 - op: replace
   path: "/spec/containers/4/env/3/value"
-  value: https://review.opendev.org/airship/airshipctl
+  value: https://opendev.org/airship/airshipctl
 
+# This is the ref to checkout for the airshipctl binary repo
 - op: replace
   path: "/spec/containers/4/env/4/value"
   value: master
 
+# This is the location from which to clone the manifest documents repo
 - op: replace
-  path: "/spec/containers/4/env/5/value"
-  value: airshipctl
+  path: "/spec/containers/6/env/3/value"
+  value: https://opendev.org/airship/airshipctl
 
-# runner container
+# This is the branch or sha to checkout for the manifest documents repo
 - op: replace
-  path: "/spec/containers/6/env/3/value"
-  value: airshipctl
+  path: "/spec/containers/6/env/4/value"
+  value: master
+
+# This is the ref to checkout for the manifest documents repo
+# Note that this will take precedence over the branch if specified above
+- op: replace
+  path: "/spec/containers/6/env/5/value"
+  value:
 
 # for local testing
 #- op: add

tools/airship-in-a-pod/examples/base/airship-in-a-pod.yaml

@@ -216,14 +216,10 @@ spec:
         value: "false"
       - name: ARTIFACTS_DIR
         value: /opt/aiap-artifacts
-      - name: MANIFEST_REPO_URL
-        value: https://review.opendev.org/airship/airshipctl
-      - name: MANIFEST_REPO_REF
+      - name: AIRSHIPCTL_REPO_URL
+        value: https://opendev.org/airship/airshipctl
+      - name: AIRSHIPCTL_REPO_REF
         value: master
-      - name: MANIFEST_REPO_NAME
-        value: airshipctl
-      - name: AIRSHIPCTL_REF
-        value:
       - name: USE_PROXY
         value:
       - name: no_proxy
@@ -320,8 +316,12 @@ spec:
         value: /opt/aiap-artifacts
       - name: USE_CACHED_ISO
         value: "false"
-      - name: MANIFEST_REPO_NAME
-        value: airshipctl
+      - name: AIRSHIP_CONFIG_PHASE_REPO_URL
+        value: https://review.opendev.org/airship/airshipctl
+      - name: AIRSHIP_CONFIG_PHASE_REPO_BRANCH
+        value: master
+      - name: AIRSHIP_CONFIG_PHASE_REPO_REF
+        value:
       - name: USE_PROXY
         value:
       - name: no_proxy

tools/airship-in-a-pod/examples/treasuremap/patchset.yaml

@@ -10,26 +10,31 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# This is the location from which to clone the airshipctl binary repo
 - op: replace
   path: "/spec/containers/4/env/3/value"
-  value: https://review.opendev.org/airship/treasuremap
+  value: https://opendev.org/airship/airshipctl
 
+# This is the ref to checkout for the airshipctl binary repo
 - op: replace
   path: "/spec/containers/4/env/4/value"
   value: master
 
+# This is the location from which to clone the manifest documents repo
 - op: replace
-  path: "/spec/containers/4/env/5/value"
-  value: treasuremap
+  path: "/spec/containers/6/env/3/value"
+  value: https://opendev.org/airship/treasuremap
 
+# This is the branch or sha to checkout for the manifest documents repo
 - op: replace
-  path: "/spec/containers/4/env/6/value"
+  path: "/spec/containers/6/env/4/value"
   value: 63c1faf718fd3341fc5bd975e575e3cf41647206
 
-# runner container
+# This is the ref to checkout for the manifest documents repo
+# Note that this will take precedence over the branch if specified above
 - op: replace
-  path: "/spec/containers/6/env/3/value"
-  value: treasuremap
+  path: "/spec/containers/6/env/5/value"
+  value:
 
 # for local testing
 #- op: add

tools/airship-in-a-pod/runner/assets/entrypoint.sh

@@ -28,16 +28,6 @@ kustomize_download_url="https://github.com/kubernetes-sigs/kustomize/releases/do
 curl -sSL "$kustomize_download_url" | tar -C /tmp -xzf -
 install /tmp/kustomize /usr/local/bin
 
-cp "$ARTIFACTS_DIR/$MANIFEST_REPO_NAME/bin/airshipctl" /usr/local/bin/airshipctl
-if [ $MANIFEST_REPO_NAME != "airshipctl" ]
-then
-  export AIRSHIP_CONFIG_PHASE_REPO_URL="https://opendev.org/airship/treasuremap"
-  cp -r $ARTIFACTS_DIR/airshipctl/ /opt/airshipctl
-fi
-
-cp  -r $ARTIFACTS_DIR/$MANIFEST_REPO_NAME/ /opt/$MANIFEST_REPO_NAME
-cd /opt/$MANIFEST_REPO_NAME
-
 curl -fsSL -o /sops-key.asc https://raw.githubusercontent.com/mozilla/sops/master/pgp/sops_functional_tests_key.asc
 SOPS_PGP_FP="FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4"
 SOPS_IMPORT_PGP="$(cat /sops-key.asc)"
@@ -46,50 +36,35 @@ export SOPS_PGP_FP
 echo 'export SOPS_IMPORT_PGP="$(cat /sops-key.asc)"' >> ~/.bashrc
 echo "export SOPS_PGP_FP=${SOPS_PGP_FP}" >> ~/.bashrc
 
-export AIRSHIP_CONFIG_MANIFEST_DIRECTORY="/tmp/airship"
+install "$ARTIFACTS_DIR/airshipctl/bin/airshipctl" /usr/local/bin
+cd "$ARTIFACTS_DIR/airshipctl"
 
-# By default, don't build airshipctl - use the binary from the shared volume instead
-# ./tools/deployment/21_systemwide_executable.sh
-if [ "$MANIFEST_REPO_NAME" == "airshipctl" ]
-then
-  ./tools/deployment/22_test_configs.sh
-  # `airshipctl document pull` doesn't support pull patchsets yet
-  #./tools/deployment/23_pull_documents.sh
-  mkdir /tmp/airship
-  cp -rp /opt/airshipctl /tmp/airship/airshipctl
-  ./tools/deployment/23_generate_secrets.sh
+export AIRSHIP_CONFIG_MANIFEST_DIRECTORY="$ARTIFACTS_DIR/manifests"
+./tools/deployment/22_test_configs.sh
+if [[ -n "$AIRSHIP_CONFIG_PHASE_REPO_REF" ]]; then
+  export NO_CHECKOUT="false"
 else
-  ./tools/deployment/airship-core/22_test_configs.sh
-  ./tools/deployment/airship-core/23_pull_documents.sh
-  ./tools/deployment/airship-core/23_generate_secrets.sh
+  export NO_CHECKOUT="true"
 fi
+./tools/deployment/23_pull_documents.sh
+./tools/deployment/23_generate_secrets.sh
 
 echo "export KUBECONFIG=$HOME/.airship/kubeconfig" >> ~/.bashrc
 
-sed -i -e 's#bmcAddress: redfish+http://\([0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+\):8000#bmcAddress: redfish+https://10.23.25.1:8443#' "/tmp/airship/$MANIFEST_REPO_NAME/manifests/site/test-site/target/catalogues/hosts.yaml"
-sed -i -e 's#root#username#' "/tmp/airship/$MANIFEST_REPO_NAME/manifests/site/test-site/target/catalogues/hosts.yaml"
-sed -i -e 's#r00tme#password#' "/tmp/airship/$MANIFEST_REPO_NAME/manifests/site/test-site/target/catalogues/hosts.yaml"
-sed -i -e 's#disableCertificateVerification: false#disableCertificateVerification: true#' "/tmp/airship/$MANIFEST_REPO_NAME/manifests/site/test-site/target/catalogues/hosts.yaml"
+repo_name=$(yq -r .manifests.dummy_manifest.repositories.primary.url /root/.airship/config | awk 'BEGIN {FS="/"} {print $NF}')
+sed -i -e 's#bmcAddress: redfish+http://\([0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+\):8000#bmcAddress: redfish+https://10.23.25.1:8443#' "$AIRSHIP_CONFIG_MANIFEST_DIRECTORY/$repo_name/manifests/site/test-site/target/catalogues/hosts.yaml"
+sed -i -e 's#root#username#' "$AIRSHIP_CONFIG_MANIFEST_DIRECTORY/$repo_name/manifests/site/test-site/target/catalogues/hosts.yaml"
+sed -i -e 's#r00tme#password#' "$AIRSHIP_CONFIG_MANIFEST_DIRECTORY/$repo_name/manifests/site/test-site/target/catalogues/hosts.yaml"
+sed -i -e 's#disableCertificateVerification: false#disableCertificateVerification: true#' "$AIRSHIP_CONFIG_MANIFEST_DIRECTORY/$repo_name/manifests/site/test-site/target/catalogues/hosts.yaml"
 
 if [[ "$USE_CACHED_ISO" = "true" ]]; then
   mkdir -p /srv/images
   tar -xzf "$CACHE_DIR/iso.tar.gz" --directory /srv/images
 else
-  if [ "$MANIFEST_REPO_NAME" == "airshipctl" ]
-  then
-    ./tools/deployment/24_build_images.sh
-  else
-    ./tools/deployment/airship-core/24_build_images.sh
-  fi
-
+  ./tools/deployment/24_build_images.sh
   tar -czf "$ARTIFACTS_DIR/iso.tar.gz" --directory=/srv/images .
 fi
 
-if [ "$MANIFEST_REPO_NAME" == "airshipctl" ]
-then
-  ./tools/deployment/25_deploy_gating.sh
-else
-  ./tools/deployment/airship-core/25_deploy_gating.sh
-fi
+./tools/deployment/25_deploy_gating.sh
 
 /signal_complete runner

tools/deployment/22_test_configs.sh

@@ -36,7 +36,7 @@ export SYSTEM_ACTION_RETRIES=30
 export SYSTEM_REBOOT_DELAY=30
 # Branch and ref options are mutually exclusive. Ref gets preference since it's more specific
 if [[ -z "$AIRSHIP_CONFIG_PHASE_REPO_REF" ]]; then
-  export AIRSHIP_CONFIG_PHASE_REPO_BRANCH=${BRANCH:-"master"}
+  export AIRSHIP_CONFIG_PHASE_REPO_BRANCH=${AIRSHIP_CONFIG_PHASE_REPO_BRANCH:-"master"}
   export AIRSHIP_CONFIG_PHASE_REPO_REMOTE_REF=""
 else
   export AIRSHIP_CONFIG_PHASE_REPO_BRANCH=""