204 lines
9.4 KiB
Bash
204 lines
9.4 KiB
Bash
#!/bin/bash
|
|
#
|
|
# Copyright (c) 2020 Wind River Systems, Inc.
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
|
|
# This migration script is used for upgrading platform applications during the
|
|
# activate stage of a platform upgrade. It will:
|
|
# - Ignore any new applications that are installed in the To-Release and rely on
|
|
# any platform-managed application logic to upload/apply it after the upgrade
|
|
# has completed.
|
|
# - Attempt to delete and upload any apps that were in the uploaded state in the
|
|
# From-Release if the version has changed in the To-Release
|
|
# - Attempt to update any app that was in the applied state in the From-Release
|
|
# if the version has changed in the To-Release
|
|
|
|
NAME=$(basename $0)
|
|
|
|
# The migration scripts are passed these parameters:
|
|
FROM_RELEASE=$1
|
|
TO_RELEASE=$2
|
|
ACTION=$3
|
|
|
|
PLATFORM_APPLICATION_PATH='/usr/local/share/applications/helm'
|
|
RECOVER_RESULT_SLEEP=30
|
|
RECOVER_RESULT_ATTEMPTS=30 # ~15 min to recover app
|
|
DELETE_RESULT_SLEEP=10
|
|
DELETE_RESULT_ATTEMPTS=6 # ~1 min to delete app
|
|
UPLOAD_RESULT_SLEEP=10
|
|
UPLOAD_RESULT_ATTEMPTS=24 # ~4 min to upload app
|
|
UPDATE_RESULT_SLEEP=30
|
|
UPDATE_RESULT_ATTEMPTS=30 # ~15 min to update app
|
|
|
|
source /etc/platform/openrc
|
|
source /etc/platform/platform.conf
|
|
|
|
# This will log to /var/log/platform.log
|
|
function log {
|
|
logger -p local1.info $1
|
|
}
|
|
|
|
function verify_apps_are_not_recovering {
|
|
# Scrape app names. Skip header and footer.
|
|
APPS=$(system application-list --nowrap | head -n-1 | tail -n+4 | awk '{print $2}')
|
|
for a in ${APPS}; do
|
|
APP_STATUS=$(system application-show $a --column status --format value)
|
|
if [[ "${APP_STATUS}" =~ ^(applying|restore-requested)$ ]]; then
|
|
if [ ${system_type} == 'All-in-one' ] && [ ${system_mode} == 'simplex' ]; then
|
|
log "$NAME: $a is in a recovering state: ${APP_STATUS}. Waiting for all applications to be uploaded or applied."
|
|
return 1
|
|
else
|
|
log "$NAME: $a is in an unexpected state: ${APP_STATUS}. Exiting for manual intervention..."
|
|
fi
|
|
exit 1
|
|
fi
|
|
done
|
|
return 0
|
|
}
|
|
|
|
log "$NAME: Starting Kubernetes application updates from release $FROM_RELEASE to $TO_RELEASE with action $ACTION"
|
|
|
|
if [ "$FROM_RELEASE" == "20.06" ] && [ "$ACTION" == "activate" ]; then
|
|
for tries in $(seq 1 $RECOVER_RESULT_ATTEMPTS); do
|
|
if verify_apps_are_not_recovering; then
|
|
break
|
|
elif [ $tries == $RECOVER_RESULT_ATTEMPTS ]; then
|
|
log "$NAME: Exceeded maximum application recovery time of $(date -u -d @"$((RECOVER_RESULT_ATTEMPTS*RECOVER_RESULT_SLEEP))" +"%Mm%Ss"). Execute upgrade-activate again when all applications are uploaded or applied."
|
|
exit 1
|
|
fi
|
|
sleep $RECOVER_RESULT_SLEEP
|
|
done
|
|
|
|
# Get the list of applications installed in the new release
|
|
for fqpn_app in $PLATFORM_APPLICATION_PATH/*; do
|
|
# Extract the app name and version from the tarball name: app_name-version.tgz
|
|
re='^(.*)-([0-9]+\.[0-9]+-[0-9]+).tgz'
|
|
[[ "$(basename $fqpn_app)" =~ $re ]]
|
|
UPGRADE_APP_NAME=${BASH_REMATCH[1]}
|
|
UPGRADE_APP_VERSION=${BASH_REMATCH[2]}
|
|
log "$NAME: Found application ${UPGRADE_APP_NAME}, version ${UPGRADE_APP_VERSION} at $fqpn_app"
|
|
|
|
# Confirm application is upgradable
|
|
if [[ "${UPGRADE_APP_NAME}" =~ ^(cert-manager|nginx-ingress-controller|oidc-auth-apps|platform-integ-apps)$ ]]; then
|
|
log "$NAME: ${UPGRADE_APP_NAME} is a supported platform application."
|
|
else
|
|
log "$NAME: ${UPGRADE_APP_NAME} is not a supported platform application. skipping..."
|
|
continue
|
|
fi
|
|
|
|
# Confirm application is loaded.
|
|
EXISTING_APP_NAME=$(system application-show $UPGRADE_APP_NAME --column name --format value)
|
|
if [ -z "${EXISTING_APP_NAME}" ]; then
|
|
log "$NAME: ${UPGRADE_APP_NAME} is currently not uploaded in the system. skipping..."
|
|
continue
|
|
fi
|
|
|
|
# Get the existing application details
|
|
EXISTING_APP_VERSION=$(system application-show $EXISTING_APP_NAME --column app_version --format value)
|
|
EXISTING_APP_STATUS=$(system application-show $EXISTING_APP_NAME --column status --format value)
|
|
log "$NAME: $EXISTING_APP_NAME, version $EXISTING_APP_VERSION, is currently in the state: $EXISTING_APP_STATUS"
|
|
|
|
if [ "x${UPGRADE_APP_VERSION}" == "x${EXISTING_APP_VERSION}" ]; then
|
|
# If the app is in uploaded or applied state, then we continue with next iteration.
|
|
# Else, the code execution proceeds and the script would exit with an unexpected state.
|
|
if [[ "${EXISTING_APP_STATUS}" =~ ^(uploaded|applied)$ ]]; then
|
|
log "$NAME: ${UPGRADE_APP_NAME}, version ${EXISTING_APP_VERSION}, is already present. skipping..."
|
|
continue
|
|
fi
|
|
fi
|
|
|
|
# All applications should be in an 'applied' or 'uploaded' state. Any other state is unexpected
|
|
case "${EXISTING_APP_STATUS}" in
|
|
|
|
# States that are upgradable
|
|
uploaded)
|
|
log "$NAME: Deleting ${EXISTING_APP_NAME}, version ${EXISTING_APP_VERSION}"
|
|
system application-delete ${EXISTING_APP_NAME}
|
|
|
|
# Wait on the delete, should be quick
|
|
for tries in $(seq 1 $DELETE_RESULT_ATTEMPTS); do
|
|
EXISTING_APP_STATUS=$(system application-show $EXISTING_APP_NAME --column status --format value)
|
|
if [ -z "${EXISTING_APP_STATUS}" ]; then
|
|
log "$NAME: ${EXISTING_APP_NAME} has been deleted."
|
|
break
|
|
fi
|
|
sleep $DELETE_RESULT_SLEEP
|
|
done
|
|
|
|
if [ $tries == $DELETE_RESULT_ATTEMPTS ]; then
|
|
log "$NAME: ${EXISTING_APP_NAME}, version ${EXISTING_APP_VERSION}, was not deleted in the alloted time. Exiting for manual intervention..."
|
|
exit 1
|
|
fi
|
|
|
|
log "$NAME: Uploading ${UPGRADE_APP_NAME}, version ${UPGRADE_APP_VERSION} from $fqpn_app"
|
|
system application-upload $fqpn_app
|
|
# Wait on the upload, should be quick
|
|
for tries in $(seq 1 $UPLOAD_RESULT_ATTEMPTS); do
|
|
UPGRADE_APP_STATUS=$(system application-show $UPGRADE_APP_NAME --column status --format value)
|
|
if [ "${UPGRADE_APP_STATUS}" == 'uploaded' ]; then
|
|
log "$NAME: ${UPGRADE_APP_NAME} has been uploaded."
|
|
break
|
|
fi
|
|
sleep $UPLOAD_RESULT_SLEEP
|
|
done
|
|
|
|
if [ $tries == $UPLOAD_RESULT_ATTEMPTS ]; then
|
|
log "$NAME: ${UPGRADE_APP_NAME}, version ${UPGRADE_APP_VERSION}, was not uploaded in the alloted time. Exiting for manual intervention..."
|
|
exit 1
|
|
fi
|
|
|
|
;;
|
|
|
|
applied)
|
|
log "$NAME: Updating ${EXISTING_APP_NAME}, from version ${EXISTING_APP_VERSION} to version ${UPGRADE_APP_VERSION} from $fqpn_app"
|
|
system application-update $fqpn_app
|
|
# Wait on the upload, should be quick
|
|
for tries in $(seq 1 $UPDATE_RESULT_ATTEMPTS); do
|
|
UPDATING_APP_NAME=$(system application-show $UPGRADE_APP_NAME --column name --format value)
|
|
UPDATING_APP_VERSION=$(system application-show $UPGRADE_APP_NAME --column app_version --format value)
|
|
UPDATING_APP_STATUS=$(system application-show $UPGRADE_APP_NAME --column status --format value)
|
|
|
|
if [ "${UPDATING_APP_NAME}" == "${UPGRADE_APP_NAME}" ] && \
|
|
[ "${UPDATING_APP_VERSION}" == "${UPGRADE_APP_VERSION}" ] && \
|
|
[ "${UPDATING_APP_STATUS}" == "applied" ]; then
|
|
log "$NAME: ${UPGRADE_APP_NAME} has been updated to version ${UPGRADE_APP_VERSION} from version ${EXISTING_APP_VERSION}"
|
|
break
|
|
fi
|
|
sleep $UPDATE_RESULT_SLEEP
|
|
done
|
|
|
|
if [ $tries == $UPDATE_RESULT_ATTEMPTS ]; then
|
|
log "$NAME: ${UPGRADE_APP_NAME}, version ${UPGRADE_APP_VERSION}, was not updated in the alloted time. Exiting for manual intervention..."
|
|
exit 1
|
|
fi
|
|
|
|
if [ $tries != $UPDATE_RESULT_ATTEMPTS ] && [ "${UPDATING_APP_VERSION}" == "${EXISTING_APP_VERSION}" ] ; then
|
|
log "$NAME: ${UPGRADE_APP_NAME}, version ${UPGRADE_APP_VERSION}, update failed and was rolled back. Exiting for manual intervention..."
|
|
exit 1
|
|
fi
|
|
;;
|
|
|
|
# States that are unexpected
|
|
uploading | upload-failed | applying | apply-failed | removing | remove-failed | restore-requested | updating | recovering )
|
|
log "$NAME: ${EXISTING_APP_NAME}, version ${EXISTING_APP_VERSION}, is in an unexpected state: ${EXISTING_APP_STATUS}. Exiting for manual intervention..."
|
|
exit 1
|
|
;;
|
|
|
|
*)
|
|
log "$NAME: ${EXISTING_APP_NAME}, version ${EXISTING_APP_VERSION}, is in an unknown state: ${EXISTING_APP_STATUS}. Exiting for manual intervention..."
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
done
|
|
|
|
log "$NAME: Completed Kubernetes application updates for release $FROM_RELEASE to $TO_RELEASE with action $ACTION"
|
|
else
|
|
log "$NAME: No actions required for from release $FROM_RELEASE to $TO_RELEASE with action $ACTION"
|
|
fi
|
|
|
|
|
|
exit 0
|