tools/import-stx
Davlet Panech 4187e73f86 Commands to reset the build environment
* stx script:
- New command "stx control is-started" to complement start/stop
- New option "stx control {start,stop} --wait"

* stx-init-env:
- new option --reset: delete chroots + restart pods
- new option --reset-hard: stop pods, delete local workspaces,
  chroots, aptly, docker & minikube profile
- rename option "--nuke" to "--delete-minikube-profile"; old spelling
  is still accepted with a warning
- renamed & refactored some functions

* import-stx:
- new env var STX_RM_METHOD: may be optionally set to "docker" for
  deleting root-owned files via "docker run", rather than "sudo"

TESTS
=========================
* Misc sanity checks using minikube & k8s
* Manually tested blacklist checks in safe_rm()
* rm via "sudo" vs "docker run"
* Using minikube:
- stx-init-env
- stx-init-env --rebuild
- stx start, build all packages, --reset, build all packages
- stx start, build all packages, --reset-hard, stx-init-env,
  build all packages

Story: 2011038
Task: 49549

Signed-off-by: Davlet Panech <davlet.panech@windriver.com>
Change-Id: Ife4172ae9fa7b58332ac7ad65beb99525bc2a1a3
2024-02-14 13:19:31 -05:00

249 lines
7.5 KiB
Plaintext

# bash
# vim:syn=sh
#
# ENVIRONMENT
# ===========
#
# These variables may be defined before sourcing this script:
#
# PROJECT (required)
# unique (within k8s) name of your prooject. Used as part of
# pod names etc.
#
# USER (required)
# usually defined by the shell; used in file names as well as
# the user that runs the build inside the builder container
#
# STX_PLATFORM
# "minikube" or "kubernetes" (for vanilla k8s)
# Default: "minikube"
#
# STX_BUILD_HOME
# root directory of all files used by this project
# Default: /localdisk/designer/$USER/$PROJECT
#
# MINIKUBE_HOME
# Minikube saves its profile information here. Must be on a local
# file system, not NFS.
# Default: $HOME
#
# MINIKUBEMEMORY
# Max amount of RAM for minikube in mebibytes (1024-based)
# Default: 16000
#
# STX_BUILD_CPUS
# How may CPU cores to allocate to PODs.
# Default: 6
#
# MINIKUBENAME
# Minikube's profile name
# Default: minikube-$USER-upstream
#
# KUBECONFIG
# Location of kubectl config file
# Default: $MINIKUBE_HOME/.kube/config or $HOME/.kube/config
# depending on $STX_PLATFORM
#
# STX_K8S_NAMESPACE (required on vanilla k8s)
# K8s namespace for builder pods & services. For vanilla k8s only.
# Minikube always uses the default namespace.
#
# STX_INSECURE_DOCKER_REGISTRIES
# Space-separated list of docker registries for which we want to disable
# SSL certificate validation. Only affects docker running in builder pods.
# Requires pod restart when changed.
#
# STX_CONTAINER_MTU
# Assume container network's MTU is this value. Currently this will be
# passed to docker-in-docker's command line. This must be <= to the k8s
# container network's MTU.
#
# STX_BUILDER_IMAGE_TAG
# Assign this tag to builder images generated or downloaded by stx-init-env,
# and reference them in builder helm charts.
# Default: $USER-$PROJECT-$DOCKER_TAG_VERSION
#
# STX_PREBUILT_BUILDER_IMAGE_TAG
# Download pre-built images with this tag. This is used by "stx-init-env"
# without the "--rebuild" flag.
# Default: master-debian-latest
#
# STX_PREBUILT_BUILDER_IMAGE_PREFIX
# Download pre-built images from this registry/prefix. This is used by "stx-init-env"
# without the "--rebuild" flag. If not empty, this must end with "/".
# Default:starlingx/
#
# STX_RM_METHOD
# stx-init-env --reset* may need to delete root-owned files. By default
# we delete them via sudo. If you set STX_RM_METHOD to "docker", we will
# delete such files via a docker container with STX_BUILD_HOME mounted inside.
#
notice_warn () {
local tty_on tty_off
if [[ -t 2 ]] ; then
tty_on=$'\033[1;33m'
tty_off=$'\033[0m'
fi
echo >&2 "${tty_on}$*${tty_off}"
}
number_of_users () {
local count
count=$(users | tr ' ' '\n' | sort --uniq | wc -l)
# Add in non-login users that might trigger a parallel build
# based on a timer, or other trigger.
if getent passwd | grep -q jenkins; then
count=$((count+1))
fi
# Always return at least one. i.e. someone is
# running this script.
if [ $count -le 0 ]; then
count=1
fi
echo $count
}
number_of_cpus () {
/usr/bin/nproc
}
sqrt () {
echo -e "sqrt($1)" | bc -q -i | head -2 | tail -1
}
if [ -z "$PROJECT" ]; then
notice_warn "\$PROJECT needs to be defined, this will be your project name."
notice_warn "It will be used on the docker image tagging to support multiusers."
return 1
fi
# Host side path, exports STX lib to user's PATH
export PRJDIR=$(pwd)
export PATH=$PRJDIR/stx/bin:$PATH
if [[ -n "$STX_BUILDER_IMAGE_TAG" ]] ; then
export DOCKER_TAG_LOCAL="$STX_BUILDER_IMAGE_TAG"
else
DOCKER_TAG_VERSION="v0.1.0"
export DOCKER_TAG_LOCAL="${USER}-${PROJECT}-${DOCKER_TAG_VERSION}"
fi
if [[ -z "$STX_PREBUILT_BUILDER_IMAGE_TAG" ]] ; then
STX_PREBUILT_BUILDER_IMAGE_TAG="master-debian-latest"
fi
export STX_PREBUILT_BUILDER_IMAGE_TAG
# Platform 'minikube' or 'kubernetes'
export STX_PLATFORM="${STX_PLATFORM:-minikube}"
# Max cpus for the build parallel jobs, replaces MINIKUBECPUS env var
export STX_BUILD_CPUS=${STX_BUILD_CPUS:-6}
STX_BUILD_HOME_DEFAULT_v1="/localdisk/$USER"
STX_BUILD_HOME_DEFAULT_v2="/localdisk/designer/$USER/$PROJECT"
if [ ! -f "stx.conf" ]; then
cp stx.conf.sample stx.conf
fi
# Platform specifics
if [ "$STX_PLATFORM" = "minikube" ]; then
# MINIKUBE Settings
if [ -z "$STX_BUILD_HOME" ]; then
# Verify default build home
if [ -d "${STX_BUILD_HOME_DEFAULT_v1}/localdisk/designer/$USER" ]; then
STX_BUILD_HOME="${STX_BUILD_HOME_DEFAULT_v1}"
else
STX_BUILD_HOME="${STX_BUILD_HOME_DEFAULT_v2}"
fi
export STX_BUILD_HOME
fi
if [ -z "$MINIKUBE_HOME" ]; then
MINIKUBE_HOME=$HOME
else
if [ ! -d "$MINIKUBE_HOME" ]; then
echo "The directory defined by \$MINIKUBE_HOME doesn't exist"
return 1
fi
fi
FSTYPE=$(stat -f -L -c %T $MINIKUBE_HOME)
if [ x"$FSTYPE" == x"nfs" ]; then
echo ""
echo "Warning: stx minikube doesn't allow \$MINIKUBE_HOME or \$HOME directory as nfs mount point!!!"
echo " Please set non-nfs MINIKUBE_HOME with the command 'export MINIKUBE_HOME=XXX/YYY'"
echo ""
unset MINIKUBE_HOME
return 1
fi
export MINIKUBEMEMORY=${MINIKUBEMEMORY:-16000}
export MINIKUBENAME=${MINIKUBENAME:-minikube-$USER-upstream}
export KUBECONFIG=$MINIKUBE_HOME/.kube/config
# Consider many users are just working with code and not actually building.
NUM_USERS=$(sqrt $(number_of_users))
ABSOLUTE_MAX_CPUS=$(($(number_of_cpus)/$NUM_USERS))
MAX_CPUS=$(number_of_cpus)
if [ "$MAX_CPUS" == "" ] || [ "$MAX_CPUS" == "0" ]; then
MAX_CPUS=1
fi
if [ $MAX_CPUS -gt $ABSOLUTE_MAX_CPUS ]; then
MAX_CPUS=$ABSOLUTE_MAX_CPUS
fi
if [ $STX_BUILD_CPUS -gt $MAX_CPUS ]; then
notice_warn "\$STX_BUILD_CPUS setting:$STX_BUILD_CPUS is more than MAX_CPUS: $MAX_CPUS."
notice_warn "Limit the minikube cluster with MAX_CPUS."
export STX_BUILD_CPUS=$MAX_CPUS
fi
MAX_MEMORY=`expr $(cat /proc/meminfo |grep MemTotal | awk '{print $2}') / 1024`
if [ "$MAX_MEMORY" == "" ] || [ "$MAX_MEMORY" == "0" ]; then
MAX_MEMORY=2048
fi
if [ $MINIKUBEMEMORY -gt $MAX_MEMORY ]; then
notice_warn "MINIKUBEMEMORY setting:$MINIKUBEMEMORY is more than system MAX_MEMORY: $MAX_MEMORY M."
notice_warn "Limit the minikube cluster with MAX_MEMORY."
export MINIKUBEMEMORY=$MAX_MEMORY
fi
elif [ "$STX_PLATFORM" = "kubernetes" ]; then
# Host side path STX_BUILD_HOME
export STX_BUILD_HOME="${STX_BUILD_HOME:-${STX_BUILD_HOME_DEFAULT_v2}}"
if [ -z "$STX_K8S_NAMESPACE" ]; then
notice_warn "\$STX_K8S_NAMESPACE needs to be defined, this will be your namespace name"
return 1
fi
if [ -z "$KUBECONFIG" ]; then
# Kubeconfig default location inside STX_BUILD_HOME
export KUBECONFIG=$STX_BUILD_HOME/.kube/config
fi
if [ ! -f "$KUBECONFIG" ]; then
notice_warn "KUBECONFIG: $KUBECONFIG not found"
notice_warn "Fix the kube config and try again."
return 1
fi
if ! kubectl get namespace 2>/dev/null | grep -q $STX_K8S_NAMESPACE; then
notice_warn "namespace $STX_K8S_NAMESPACE not found"
return 1
fi
else
notice_warn "\$STX_PLATFORM not specified, valid options are: 'minikube' or 'kubernetes'"
return 1
fi