anvil/smithy
Ivan A. Melnikov 8f8913f19a Put RPMS build on bootstrap to anvil-deps repo
RPMs build on bootstrap (before any python code gets a chance to run)
are now put to temporary but well-defined locations in ./.bootstrap
directory, and then moved to appropriate place at the beginning of
prepare action, when openstack root dir is already known.

So, said RPMs are not lost, but put in place from which they could be
re-installed or installed to different system.

Implements: blueprint bootstrap-pip-rpm
Change-Id: I41dded0ecd8d887b540aea87d0226be03508c3cb
2013-08-21 19:39:26 +04:00

371 lines
9.9 KiB
Bash
Executable File

#!/bin/bash
locale -a | grep -q -e "^en_US$" && export LANG="en_US"
shopt -s nocasematch
SMITHY_NAME=$(readlink -f "$0")
cd "$(dirname "$0")"
VERBOSE="${VERBOSE:-0}"
PY2RPM_CMD="$PWD/tools/py2rpm"
YUMFIND_CMD="$PWD/tools/yumfind"
PIPDOWNLOAD_CMD="$PWD/tools/pip-download"
YUM_OPTS="--assumeyes --nogpgcheck"
RPM_OPTS=""
CURL_OPTS=""
# Colors supported??
COLOR_SUPPORT=`tput colors`
if [ $COLOR_SUPPORT -ge 8 ]; then
ESC_SEQ="\x1b["
COL_RESET=$ESC_SEQ"39;49;00m"
COL_GREEN=$ESC_SEQ"32;01m"
COL_RED=$ESC_SEQ"31;01m"
COL_YELLOW=$ESC_SEQ"33;01m"
else
ESC_SEQ=""
COL_RESET=""
COL_GREEN=""
COL_RED=""
COL_YELLOW=""
fi
if [ "$VERBOSE" == "0" ]; then
YUM_OPTS="$YUM_OPTS -q"
RPM_OPTS="-q"
CURL_OPTS="-s"
fi
# Source in our variables (or overrides)
source ".anvilrc"
if [ -n "$SUDO_USER" ]; then
USER_HOME=$(getent passwd $SUDO_USER | cut -d: -f6)
if [ -n "$USER_HOME" ]; then
HOME_RC="${USER_HOME}/.anvilrc"
if [ -f "$HOME_RC" ]; then
source "$HOME_RC"
fi
fi
fi
if [ -z "$BOOT_FILES" ]; then
BOOT_FN=".anvil_bootstrapped"
BOOT_FILES="${PWD}/$BOOT_FN"
fi
clean_pip()
{
# https://github.com/pypa/pip/issues/982
if [ -n "$SUDO_USER" ]; then
rm -rf /tmp/pip-build-$SUDO_USER
fi
}
rpm_is_installed()
{
local name="$(basename "$1")"
rpm $RPM_OPTS "${name%.rpm}" &>/dev/null
return $?
}
cache_and_install_rpm_url()
{
url=${1:?"Error: rpm uri is undefined!"}
cachedir=${RPM_CACHEDIR:-'/tmp'}
rpm=$(basename $url)
if rpm_is_installed "$rpm"; then
return
fi
if [ ! -f "$cachedir/$rpm" ]; then
echo -e "Downloading ${COL_GREEN}${rpm}${COL_RESET} to ${COL_GREEN}${cachedir}${COL_RESET}"
curl $CURL_OPTS $url -o "$cachedir/$rpm" || return 1
fi
echo -e "Installing ${COL_GREEN}$cachedir/$rpm${COL_RESET}"
yum install $YUM_OPTS "$cachedir/$rpm"
return $?
}
bootstrap_epel()
{
[ -z "$EPEL_RPM_URL" ] && return 0
cache_and_install_rpm_url $EPEL_RPM_URL
return $?
}
dump_list()
{
for var in "$@"; do
for name in $var; do
echo " - $name"
done
done
}
bootstrap_rpm_packages()
{
# NOTE(aababilov): the latter operations require some packages,
# so, begin from installation
if [ -n "$REQUIRES" ]; then
echo -e "Installing ${COL_GREEN}system${COL_RESET} packages:"
dump_list "$REQUIRES"
yum install $YUM_OPTS $REQUIRES
if [ "$?" != "0" ]; then
echo -e "${COL_RED}Failed installing!${COL_RESET}"
return 1
fi
fi
# Remove any known conflicting packages
CONFLICTS=$(python -c "import yaml
packages = set()
try:
for i in yaml.safe_load(open('$DISTRO_CONFIG'))['components'].itervalues():
for j in i.get('conflicts', []):
packages.add(j.get('name'))
except KeyError:
pass
for pkg in packages:
if pkg:
print pkg
")
if [ -n "$CONFLICTS" ]; then
echo -e "Removing ${COL_YELLOW}conflicting${COL_RESET} packages:"
dump_list "$CONFLICTS"
yum erase $YUM_OPTS $CONFLICTS
fi
}
bootstrap_selinux()
{
# See if selinux is on.
if [ `getenforce` == "Enforcing" ]; then
# Ensure all yum api interacting binaries are ok to be used
echo "Enabling selinux for yum like binaries."
chcon -h "system_u:object_r:rpm_exec_t:s0" "$YUMFIND_CMD"
chcon -h "system_u:object_r:rpm_exec_t:s0" "$PWD/tools/yyoom"
fi
}
bootstrap_python_rpms()
{
echo -e "Bootstrapping ${COL_GREEN}python${COL_RESET} rpms."
local package_map=$(python -c "import yaml
try:
for k, v in yaml.safe_load(open('$DISTRO_CONFIG'))['dependency_handler']['package_map'].iteritems():
print '%s==%s' % (k, v)
except KeyError:
pass
")
local python_names=$(cat requirements.txt test-requirements.txt | sed -r -e 's/#.*$//' | sort -u)
local bootstrap_dir="$(readlink -f ./.bootstrap/)"
local missing_packages=""
local found_packages=""
for name in $python_names; do
local pkg_name=$("$PY2RPM_CMD" --package-map $package_map --convert "$name" | while read req pack; do echo $pack; done | head -n1 | tr -s ' ' | cut -d' ' -f1)
local yum_name=$("$YUMFIND_CMD" -p "$pkg_name,$name")
if [ -n "$yum_name" ]; then
found_packages="$found_packages $yum_name"
else
missing_packages="$missing_packages $name"
fi
done
if [ -n "$found_packages" ]; then
echo -e "Installing ${COL_GREEN}available${COL_RESET} python requirements found as packages:"
dump_list "$found_packages"
yum install $YUM_OPTS $found_packages
if [ "$?" != "0" ]; then
echo -e "${COL_RED}Failed installing!${COL_RESET}"
return 1
fi
fi
if [ -z "$missing_packages" ]; then
return 0
fi
echo -e "Building ${COL_YELLOW}missing${COL_RESET} python requirements:"
dump_list "$missing_packages"
local pip_tmp_dir="$bootstrap_dir/pip-download"
mkdir -p "$pip_tmp_dir"
echo "Downloading..."
$PIPDOWNLOAD_CMD -d "$pip_tmp_dir" $missing_packages | grep "^Saved"
echo "Building RPMs..."
local rpm_names=$("$PY2RPM_CMD" --package-map $package_map --scripts-dir "conf/templates/packaging/scripts" --rpm-base "$bootstrap_dir/rpmbuild" -- "$pip_tmp_dir/"* 2>/dev/null |
awk '/^Wrote: /{ print $2 }' | grep -v '.src.rpm' | sort -u)
if [ -z "$rpm_names" ]; then
echo -e "${COL_RED}No binary RPMs were built!${COL_RESET}"
return 1
fi
local rpm_base_names=""
for rpm in $rpm_names; do
rpm_base_names="$rpm_base_names $(basename $rpm)"
done
echo -e "Installing ${COL_YELLOW}missing${COL_RESET} python requirement packages:"
dump_list "$rpm_base_names"
yum install $YUM_OPTS $rpm_names
if [ "$?" != "0" ]; then
echo -e "${COL_RED}Failed installing!${COL_RESET}"
return 1
fi
rm -rf "$pip_tmp_dir"
rm -rf "$bootstrap_dir/rpmbuild/"{BUILD,SOURCES,SPECS,BUILDROOT}
return 0
}
needs_bootstrap()
{
$BOOTSTRAP && return 0
checksums=$(get_checksums)
for i in $BOOT_FILES; do
if [ -f $i ]; then
contents=`cat $i`
[ "$contents" = "$checksums" ] && return 1
fi
done
return 0
}
get_checksums()
{
if [ ! -f "$BSCONF_FILE" ]; then
return 1
fi
# Used to tell if the file have changed
echo $(md5sum "$BSCONF_FILE")
}
run_smithy()
{
PYTHON=`which python`
exec $PYTHON anvil $ARGS
}
puke()
{
cleaned_force=$(echo $FORCE | sed -e 's/\([A-Z]\)/\L\1/g;s/\s//g')
if [[ "$cleaned_force" == "yes" ]]; then
run_smithy
else
echo -e "To run anyway set ${COL_YELLOW}FORCE=yes${COL_RESET} and rerun." >&2
exit 1
fi
}
## Identify which bootstrap configuration file to use: either set
## explicitly (BSCONF_FILE) or determined based on the os distribution:
BSCONF_DIR=${BSCONF_DIR:-$(dirname $(readlink -f "$0"))/tools/bootstrap}
get_os_info(){
OS=`uname`
if [ "${OS}" = "Linux" ] ; then
if [ -f /etc/redhat-release ] ; then
PKG="rpm"
OSNAME=`cat /etc/redhat-release`
OSDIST=`cat /etc/redhat-release | sed -e 's/release.*$//g;s/\s//g'`
PSUEDONAME=`cat /etc/redhat-release | sed s/.*\(// | sed s/\)//`
RELEASE=`cat /etc/redhat-release | sed s/.*release\ // | sed s/\ .*//`
elif [ -f /etc/debian_version ] ; then
PKG="deb"
OSDIST=`cat /etc/lsb-release | grep '^DISTRIB_ID' | awk -F= '{ print $2 }'`
PSUEDONAME=`cat /etc/lsb-release | grep '^DISTRIB_CODENAME' | awk -F= '{ print $2 }'`
RELEASE=`cat /etc/lsb-release | grep '^DISTRIB_RELEASE' | awk -F= '{ print $2 }'`
OSNAME="$OSDIST $RELEASE ($PSUEDONAME)"
fi
fi
}
get_os_info
if [ -z "$BSCONF_FILE" ]; then
BSCONF_FILE="$BSCONF_DIR/$OSDIST"
fi
ARGS=""
BOOTSTRAP=false
# Ad-hoc getopt to handle long opts.
#
# Smithy opts are consumed while those to anvil are copied through.
while [ ! -z $1 ]; do
case "$1" in
'--bootstrap')
BOOTSTRAP=true
shift
;;
'--force')
FORCE=yes
shift
;;
*)
ARGS="$ARGS $1"
shift
;;
esac
done
# Source immediately so that we can export the needed variables.
if [ -f "$BSCONF_FILE" ]; then
source $BSCONF_FILE
export REQUIRED_PACKAGES="$REQUIRES"
fi
if ! needs_bootstrap; then
clean_pip
run_smithy
clean_pip
elif ! $BOOTSTRAP; then
echo "This system needs to be updated in order to run anvil!" >&2
echo "Running 'sudo $SMITHY_NAME --bootstrap' will attempt to do so." >&2
puke
fi
## Bootstrap smithy
if [ "$(id -u)" != "0" ]; then
echo "You must run '$SMITHY_NAME --bootstrap' with root privileges!" >&2
exit 1
fi
if [ ! -f "$BSCONF_FILE" ]; then
echo "Anvil has not been tested on distribution '$OSNAME'" >&2
puke
fi
MIN_RELEASE=${MIN_RELEASE:?"Error: MIN_RELEASE is undefined!"}
SHORTNAME=${SHORTNAME:?"Error: SHORTNAME is undefined!"}
BC_OK=$(echo "$RELEASE < $MIN_RELEASE" | bc)
if [ "$BC_OK" == "1" ]; then
echo "This script must be run on $SHORTNAME $MIN_RELEASE+ and not $SHORTNAME $RELEASE." >&2
puke
fi
echo "Bootstrapping $SHORTNAME $RELEASE"
echo "Please wait..."
clean_pip
for step in ${STEPS:?"Error: STEPS is undefined!"}; do
bootstrap_${step}
if [ $? != 0 ]; then
echo "Bootstrapping $SHORTNAME $RELEASE failed." >&2
exit 1
fi
done
clean_pip
# Write the checksums of the bootstrap file
# which if new requirements are added will cause new checksums
# and a new dependency install...
checksum=$(get_checksums)
for i in $BOOT_FILES; do
echo -e $checksum > $i
done
mkdir -p -v /etc/anvil /usr/share/anvil
if [ -n "$SUDO_UID" -a -n "SUDO_GID" ]; then
chown -c "$SUDO_UID:$SUDO_GID" /etc/anvil /usr/share/anvil
[ -d .bootstrap ] && chown -R "$SUDO_UID:$SUDO_GID" .bootstrap
fi
echo "Bootstrapped for $SHORTNAME $RELEASE"
ARGS="-a moo"
run_smithy