Refactor build scripts
* Move reusable snippets to the library script helpers.sh as functions * Use separate scripts for building requirements and project images Signed-off-by: Vladimir Kozhukalov <kozhukalov@gmail.com> Change-Id: Ib4ba2d0d1eb568898c904922ec8d75e1731f953c
This commit is contained in:
Vladimir Kozhukalov
@@ -30,6 +30,7 @@ ARG KEEP_ALL_WHEELS="False"
|
||||
ARG HORIZON_EXTRA_PANELS
|
||||
ARG PROJECT_PIP_EXTRAS
|
||||
ARG APT_MIRROR="https://archive.ubuntu.com/ubuntu/"
|
||||
ARG SOURCES_DIR="/tmp"
|
||||
|
||||
# NOTE: This option is only applicable to apt/dpkg systems. The value is noop
|
||||
# for rpm based systems. This will not show up in the final image regardless.
|
||||
@@ -58,7 +59,7 @@ ARG SKYLINE_CONSOLE_REF=master
|
||||
|
||||
ARG PIP_VERSION_CONSTRAINT=""
|
||||
ARG SETUPTOOL_CONSTRAINT=""
|
||||
ARG WHEEL_CONSTRAIN=""
|
||||
ARG WHEEL_CONSTRAINT=""
|
||||
|
||||
# End virtualenv argumens
|
||||
|
||||
@@ -67,7 +68,7 @@ COPY scripts /opt/loci/scripts
|
||||
ADD bindep.txt pydep.txt $EXTRA_BINDEP $EXTRA_PYDEP /opt/loci/
|
||||
|
||||
FROM common AS requirements
|
||||
RUN /opt/loci/scripts/install.sh
|
||||
RUN /opt/loci/scripts/requirements.sh
|
||||
|
||||
FROM common AS project
|
||||
RUN --mount=type=bind,from=wheels,target=${WHEELS_PATH} /opt/loci/scripts/install.sh
|
||||
RUN --mount=type=bind,from=wheels,target=${WHEELS_PATH} /opt/loci/scripts/project.sh
|
||||
|
||||
@@ -4,23 +4,7 @@ FROM ${FROM}
|
||||
ARG CEPH_REPO
|
||||
ARG CEPH_KEY="https://download.ceph.com/keys/release.asc"
|
||||
ARG ALLOW_UNAUTHENTICATED=false
|
||||
ARG APT_MIRROR="https://archive.ubuntu.com/ubuntu/"
|
||||
|
||||
RUN echo "APT::Get::AllowUnauthenticated \"${ALLOW_UNAUTHENTICATED}\";\n\
|
||||
Acquire::AllowInsecureRepositories \"${ALLOW_UNAUTHENTICATED}\";\n\
|
||||
Acquire::AllowDowngradeToInsecureRepositories \"${ALLOW_UNAUTHENTICATED}\";" \
|
||||
>> /etc/apt/apt.conf.d/allow-unathenticated
|
||||
|
||||
RUN export DEBIAN_FRONTEND=noninteractive && \
|
||||
apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
apt-transport-https \
|
||||
ca-certificates \
|
||||
gnupg2 \
|
||||
lsb-release \
|
||||
wget && \
|
||||
wget -q -O- ${CEPH_KEY} | apt-key add - && \
|
||||
if [ -n "${CEPH_REPO}" ]; then \
|
||||
echo "${CEPH_REPO}" | tee /etc/apt/sources.list.d/ceph.list; \
|
||||
fi && \
|
||||
apt-get clean && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
COPY scripts/base.sh scripts/helpers.sh /
|
||||
RUN /base.sh && rm -f /base.sh /helpers.sh
|
||||
|
||||
52
scripts/base.sh
Executable file
52
scripts/base.sh
Executable file
@@ -0,0 +1,52 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -xeo pipefail
|
||||
|
||||
source "$(dirname $0)/helpers.sh"
|
||||
|
||||
export LC_CTYPE=C.UTF-8
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
cat <<EOF >> /etc/apt/apt.conf.d/allow-unathenticated
|
||||
APT::Get::AllowUnauthenticated "${ALLOW_UNAUTHENTICATED}";
|
||||
Acquire::AllowInsecureRepositories "${ALLOW_UNAUTHENTICATED}";
|
||||
Acquire::AllowDowngradeToInsecureRepositories "${ALLOW_UNAUTHENTICATED}";
|
||||
EOF
|
||||
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends \
|
||||
apt-transport-https \
|
||||
ca-certificates \
|
||||
gnupg2 \
|
||||
lsb-release \
|
||||
wget
|
||||
|
||||
configure_apt_sources "${APT_MIRROR}"
|
||||
|
||||
wget -q -O- "${CEPH_KEY}" | apt-key add -
|
||||
if [ -n "${CEPH_REPO}" ]; then
|
||||
echo "${CEPH_REPO}" | tee /etc/apt/sources.list.d/ceph.list
|
||||
fi
|
||||
|
||||
apt-get update
|
||||
apt-get upgrade -y
|
||||
apt-get install -y --no-install-recommends \
|
||||
git \
|
||||
netbase \
|
||||
patch \
|
||||
sudo \
|
||||
bind9-host \
|
||||
python3 \
|
||||
python3-venv
|
||||
|
||||
if [[ -n $(apt-cache search ^python3-distutils$ 2>/dev/null) ]]; then
|
||||
apt-get install -y --no-install-recommends python3-distutils
|
||||
fi
|
||||
|
||||
apt-get install -y --no-install-recommends \
|
||||
libpython3.$(python3 -c 'import sys; print(sys.version_info.minor);')
|
||||
|
||||
revert_apt_sources
|
||||
|
||||
apt-get clean
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
@@ -1,23 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
apt-get purge -y --auto-remove \
|
||||
git \
|
||||
patch \
|
||||
python3-virtualenv \
|
||||
virtualenv
|
||||
if [ -f /etc/apt/sources.list.bak ]; then
|
||||
mv /etc/apt/sources.list.bak /etc/apt/sources.list
|
||||
fi
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Changing this option allows python to use libraries outside of the
|
||||
# virtualenv > 20 if they do not exist inside the venv. This is a requirement
|
||||
# for using python-rbd which is not pip installable and is only available
|
||||
# in packaged form.
|
||||
sed -i 's/\(include-system-site-packages\).*/\1 = true/g' /var/lib/openstack/pyvenv.cfg
|
||||
rm -rf /tmp/* /root/.cache /etc/machine-id
|
||||
find /usr/ /var/ \( -name "*.pyc" -o -name "__pycache__" \) -delete
|
||||
# Remove sources added to image
|
||||
rm -rf /opt/loci/data/*
|
||||
@@ -1,11 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -eux
|
||||
|
||||
if [[ ! -d /tmp/${PROJECT} ]]; then
|
||||
git clone ${PROJECT_REPO} /tmp/${PROJECT}
|
||||
pushd /tmp/${PROJECT}
|
||||
git fetch ${PROJECT_REPO} ${PROJECT_REF}
|
||||
git checkout FETCH_HEAD
|
||||
popd
|
||||
fi
|
||||
@@ -1,23 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
INFO_DIR="/etc/image_info"
|
||||
mkdir -p $INFO_DIR
|
||||
PACKAGES_INFO="${INFO_DIR}/packages.txt"
|
||||
PIP_INFO="${INFO_DIR}/pip.txt"
|
||||
PROJECT_INFO="${INFO_DIR}/project.txt"
|
||||
|
||||
dpkg -l > $PACKAGES_INFO
|
||||
|
||||
pip freeze > $PIP_INFO
|
||||
cat > ${PROJECT_INFO} <<EOF
|
||||
PROJECT=${PROJECT}
|
||||
PROJECT_REPO=${PROJECT_REPO}
|
||||
PROJECT_REF=${PROJECT_REF}
|
||||
PROJECT_RELEASE=${PROJECT_RELEASE}
|
||||
EOF
|
||||
pushd /tmp/${PROJECT}
|
||||
echo "========"
|
||||
git log -1 >> ${PROJECT_INFO}
|
||||
popd
|
||||
@@ -1,10 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
: ${COPY_DEFAULT_CONFIG_FILES:="no"}
|
||||
|
||||
if [[ $COPY_DEFAULT_CONFIG_FILES == "yes" ]] && [[ ! ${PROJECT} =~ ^requirements$ ]]; then
|
||||
mkdir -p "/etc/${PROJECT}/"
|
||||
cp -r "/var/lib/openstack/etc/${PROJECT}"/* "/etc/${PROJECT}/" || true
|
||||
fi
|
||||
@@ -1,14 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
groupadd -g ${GID} ${PROJECT}
|
||||
if [[ "${PROJECT}" == "nova" ]];then
|
||||
# NOTE: bash needed for nova to support instance migration
|
||||
useradd -u ${UID} -g ${PROJECT} -M -d /var/lib/${PROJECT} -s /bin/rbash -c "${PROJECT} user" ${PROJECT}
|
||||
else
|
||||
useradd -u ${UID} -g ${PROJECT} -M -d /var/lib/${PROJECT} -s /usr/sbin/nologin -c "${PROJECT} user" ${PROJECT}
|
||||
fi
|
||||
|
||||
mkdir -p /etc/${PROJECT} /var/log/${PROJECT} /var/lib/${PROJECT} /var/cache/${PROJECT}
|
||||
chown ${PROJECT}:${PROJECT} /etc/${PROJECT} /var/log/${PROJECT} /var/lib/${PROJECT} /var/cache/${PROJECT}
|
||||
244
scripts/helpers.sh
Normal file
244
scripts/helpers.sh
Normal file
@@ -0,0 +1,244 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Common helper functions shared by build scripts.
|
||||
|
||||
get_bindep_system_packages() {
|
||||
# Resolve bindep packages for a project/profile set.
|
||||
# Args:
|
||||
# $1: project name
|
||||
# $2...: PROFILES list (space-separated)
|
||||
source /etc/lsb-release
|
||||
|
||||
local project="$1"
|
||||
shift
|
||||
local profiles=("$@")
|
||||
|
||||
local packages=()
|
||||
for file in /opt/loci/bindep*; do
|
||||
packages+=($(bindep -f "$file" -b -l newline "${project}" "${profiles[@]}" "${DISTRIB_CODENAME}" || :))
|
||||
done
|
||||
|
||||
echo "${packages[@]}"
|
||||
}
|
||||
|
||||
install_system_packages() {
|
||||
# Install distribution packages provided as arguments.
|
||||
# Args:
|
||||
# $@ packages to install
|
||||
local packages=("$@")
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
if [[ ${#packages[@]} -gt 0 ]]; then
|
||||
apt-get update && \
|
||||
apt-get install -y --no-install-recommends "${packages[@]}"
|
||||
fi
|
||||
}
|
||||
|
||||
get_pkg_name() {
|
||||
local folder=$1
|
||||
local name
|
||||
pushd "$folder" > /dev/null
|
||||
name=$(python3 setup.py --name 2>/dev/null | grep -v '^\[pbr\]')
|
||||
popd > /dev/null
|
||||
echo "$name"
|
||||
}
|
||||
|
||||
get_pkg_version() {
|
||||
local folder=$1
|
||||
local version
|
||||
pushd "$folder" > /dev/null
|
||||
version=$(python3 setup.py --version 2>/dev/null | grep -v '^\[pbr\]')
|
||||
popd > /dev/null
|
||||
echo "$version"
|
||||
}
|
||||
|
||||
get_pipy_name_by_project_name() {
|
||||
local project_name=$1
|
||||
while read _folder_name _pipy_name _pkg_name; do
|
||||
if [[ "${_pkg_name}" == "${project_name}" ]]; then
|
||||
echo "${_pipy_name}"
|
||||
return
|
||||
fi
|
||||
done < /opt/loci/scripts/python-custom-name-mapping.txt
|
||||
echo "$project_name"
|
||||
}
|
||||
|
||||
configure_apt_sources() {
|
||||
# Configure apt sources for Ubuntu based on the release version.
|
||||
# Args:
|
||||
# $1: apt mirror URL (defaults to APT_MIRROR env or Ubuntu archive)
|
||||
local apt_mirror="${1}"
|
||||
|
||||
source /etc/lsb-release
|
||||
|
||||
if [[ ${DISTRIB_RELEASE%%.*} -ge 24 ]]; then # Ubuntu 24.04 and newer
|
||||
trusted=""
|
||||
if echo "${apt_mirror}" | grep -q -E '\[trusted=yes\]'; then
|
||||
apt_mirror=$(echo "${apt_mirror}" | awk '{print $2}')
|
||||
trusted="yes"
|
||||
fi
|
||||
mv /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list.d/ubuntu.sources.orig
|
||||
cat > /etc/apt/sources.list.d/ubuntu.sources <<EOF
|
||||
Types: deb
|
||||
URIs: ${apt_mirror}
|
||||
Suites: ${DISTRIB_CODENAME} ${DISTRIB_CODENAME}-updates ${DISTRIB_CODENAME}-backports ${DISTRIB_CODENAME}-security
|
||||
Components: main universe restricted multiverse
|
||||
EOF
|
||||
if [[ -n "${trusted}" ]]; then
|
||||
echo "Trusted: ${trusted}" >> /etc/apt/sources.list.d/ubuntu.sources
|
||||
fi
|
||||
cat /etc/apt/sources.list.d/ubuntu.sources
|
||||
else
|
||||
mv /etc/apt/sources.list /etc/apt/sources.list.orig
|
||||
cat > /etc/apt/sources.list <<EOF
|
||||
deb ${apt_mirror} ${DISTRIB_CODENAME} main universe
|
||||
deb ${apt_mirror} ${DISTRIB_CODENAME}-updates main universe
|
||||
deb ${apt_mirror} ${DISTRIB_CODENAME}-security main universe
|
||||
deb ${apt_mirror} ${DISTRIB_CODENAME}-backports main universe
|
||||
EOF
|
||||
cat /etc/apt/sources.list
|
||||
fi
|
||||
}
|
||||
|
||||
revert_apt_sources() {
|
||||
# Restore original apt sources files if backups exist.
|
||||
if [[ -f /etc/apt/sources.list.d/ubuntu.sources.orig ]]; then
|
||||
mv /etc/apt/sources.list.d/ubuntu.sources{.orig,}
|
||||
fi
|
||||
if [[ -f /etc/apt/sources.list.orig ]]; then
|
||||
mv /etc/apt/sources.list{.orig,}
|
||||
fi
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
apt-get update
|
||||
apt-get purge -y --auto-remove \
|
||||
git \
|
||||
patch \
|
||||
python3-virtualenv \
|
||||
virtualenv || true
|
||||
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Allow python to use system site packages when missing from the venv
|
||||
sed -i 's/\(include-system-site-packages\).*/\1 = true/g' /var/lib/openstack/pyvenv.cfg
|
||||
rm -rf /tmp/* /root/.cache/pip /etc/machine-id
|
||||
find /usr/ /var/ \( -name "*.pyc" -o -name "__pycache__" \) -delete
|
||||
# Remove sources added to image
|
||||
rm -rf /opt/loci/*
|
||||
}
|
||||
|
||||
collect_info() {
|
||||
# Gather pip and project metadata into INFO_DIR.
|
||||
local INFO_DIR="/etc/image_info"
|
||||
mkdir -p "${INFO_DIR}"
|
||||
local pip_info="${INFO_DIR}/pip.txt"
|
||||
local project_info="${INFO_DIR}/project.txt"
|
||||
|
||||
pip freeze > "${pip_info}"
|
||||
|
||||
cat > "${project_info}" <<EOF
|
||||
PROJECT=${PROJECT}
|
||||
PROJECT_REPO=${PROJECT_REPO}
|
||||
PROJECT_REF=${PROJECT_REF}
|
||||
PROJECT_RELEASE=${PROJECT_RELEASE}
|
||||
EOF
|
||||
pushd "${SOURCES_DIR}/${PROJECT}"
|
||||
echo "========"
|
||||
git log -1 >> "${project_info}"
|
||||
popd
|
||||
}
|
||||
|
||||
configure_packages() {
|
||||
# Optionally copy default config files from venv/etc into /etc/PROJECT.
|
||||
local copy_default_config_files="${1:-${COPY_DEFAULT_CONFIG_FILES:-no}}"
|
||||
if [[ "${copy_default_config_files}" == "yes" ]]; then
|
||||
mkdir -p "/etc/${PROJECT}/"
|
||||
cp -r "/var/lib/openstack/etc/${PROJECT}"/* "/etc/${PROJECT}/" || true
|
||||
fi
|
||||
}
|
||||
|
||||
clone_project() {
|
||||
# Clone the project defined by provided parameters into /tmp and checkout the ref.
|
||||
# Args:
|
||||
# $1: project name
|
||||
# $2: project repo
|
||||
# $3: project ref
|
||||
local project_name="$1"
|
||||
local project_repo="$2"
|
||||
local project_ref="$3"
|
||||
|
||||
if [[ ! -d "${SOURCES_DIR}/${project_name}" ]]; then
|
||||
git clone "${project_repo}" "${SOURCES_DIR}/${project_name}"
|
||||
pushd "${SOURCES_DIR}/${project_name}"
|
||||
git fetch "${project_repo}" "${project_ref}"
|
||||
git checkout FETCH_HEAD
|
||||
popd
|
||||
fi
|
||||
}
|
||||
|
||||
create_user() {
|
||||
# Create the system user/group for the project.
|
||||
# Args:
|
||||
# $1: GID
|
||||
# $2: UID
|
||||
# $3: PROJECT name
|
||||
local gid="$1"
|
||||
local uid="$2"
|
||||
local project="$3"
|
||||
|
||||
groupadd -g "${gid}" "${project}"
|
||||
if [[ "${project}" == "nova" ]]; then
|
||||
# NOTE: bash needed for nova to support instance migration
|
||||
useradd -u "${uid}" -g "${project}" -M -d "/var/lib/${project}" -s /bin/rbash -c "${project} user" "${project}"
|
||||
else
|
||||
useradd -u "${uid}" -g "${project}" -M -d "/var/lib/${project}" -s /usr/sbin/nologin -c "${project} user" "${project}"
|
||||
fi
|
||||
|
||||
mkdir -p "/etc/${project}" "/var/log/${project}" "/var/lib/${project}" "/var/cache/${project}"
|
||||
chown "${project}:${project}" "/etc/${project}" "/var/log/${project}" "/var/lib/${project}" "/var/cache/${project}"
|
||||
}
|
||||
|
||||
setup_venv() {
|
||||
# Create and prime a virtualenv with optional toolchain constraints.
|
||||
# Uses PIP_VERSION_CONSTRAINT, SETUPTOOL_CONSTRAINT, WHEEL_CONSTRAINT if set.
|
||||
local venv_path="${1:-/var/lib/openstack}"
|
||||
|
||||
python3 -m venv "${venv_path}"
|
||||
# shellcheck source=/dev/null
|
||||
source "${venv_path}/bin/activate"
|
||||
|
||||
pip install --upgrade "pip${PIP_VERSION_CONSTRAINT}"
|
||||
pip install --upgrade "setuptools${SETUPTOOL_CONSTRAINT}"
|
||||
pip install --upgrade "wheel${WHEEL_CONSTRAINT}"
|
||||
pip install --upgrade bindep pkginfo uv
|
||||
}
|
||||
|
||||
honor_local_sources() {
|
||||
# Update constraints to prefer local checkouts in SOURCES_DIR.
|
||||
# List of packages to be built will be placed in UPPER_CONSTRAINTS_BUILD.
|
||||
# Args:
|
||||
# $1: sources dir
|
||||
# $2: UPPER_CONSTRAINTS path
|
||||
# $3: UPPER_CONSTRAINTS_BUILD path
|
||||
local sources_dir="$1"
|
||||
local constraints="$2"
|
||||
local constraints_build="$3"
|
||||
local constraints_tmp="${constraints}.tmp"
|
||||
|
||||
cp "${constraints}" "${constraints_tmp}"
|
||||
cp "${constraints}" "${constraints_build}"
|
||||
pushd "${sources_dir}"
|
||||
for repo in $(ls -1 "${sources_dir}"); do
|
||||
if [[ ! -f "${repo}/setup.cfg" ]]; then
|
||||
continue
|
||||
fi
|
||||
echo "Making build constraint for ${repo}"
|
||||
pkg_name=$(get_pkg_name "${repo}")
|
||||
pkg_version=$(get_pkg_version "${repo}")
|
||||
pipy_name=$(get_pipy_name_by_project_name "${pkg_name}")
|
||||
sed -i "s|^${pipy_name}===.*|file://${sources_dir}/${repo}#egg=${pkg_name}|g" "${constraints_build}"
|
||||
sed -i "s|^${pipy_name}===.*|${pipy_name}===${pkg_version}|g" "${constraints_tmp}"
|
||||
done
|
||||
popd
|
||||
mv "${constraints_tmp}" "${constraints}"
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
source /etc/lsb-release
|
||||
|
||||
export LC_CTYPE=C.UTF-8
|
||||
# This overrides the base image configuration
|
||||
echo 'APT::Get::AllowUnauthenticated "true";' > /etc/apt/apt.conf.d/99allow-unauthenticated
|
||||
mv /etc/apt/sources.list /etc/apt/sources.list.bak
|
||||
cat > /etc/apt/sources.list <<EOF
|
||||
deb ${APT_MIRROR} ${DISTRIB_CODENAME} main universe
|
||||
deb ${APT_MIRROR} ${DISTRIB_CODENAME}-updates main universe
|
||||
deb ${APT_MIRROR} ${DISTRIB_CODENAME}-security main universe
|
||||
deb ${APT_MIRROR} ${DISTRIB_CODENAME}-backports main universe
|
||||
EOF
|
||||
apt-get update
|
||||
apt-get upgrade -y
|
||||
apt-get install -y --no-install-recommends \
|
||||
git \
|
||||
netbase \
|
||||
patch \
|
||||
sudo \
|
||||
bind9-host \
|
||||
python3 \
|
||||
python3-venv
|
||||
if [[ ! -z "$(apt-cache search ^python3-distutils$)" ]]; then
|
||||
apt-get install -y --no-install-recommends python3-distutils
|
||||
fi
|
||||
apt-get install -y --no-install-recommends \
|
||||
libpython3.$(python3 -c 'import sys; print(sys.version_info.minor);')
|
||||
|
||||
if [[ "${PROJECT}" == "requirements" ]]; then
|
||||
$(dirname $0)/requirements.sh
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "${KEEP_ALL_WHEELS}" != "False" ]; then
|
||||
NO_INDEX=--no-index
|
||||
fi
|
||||
|
||||
if [[ "${PLUGIN}" == "no" ]]; then
|
||||
$(dirname $0)/create_user.sh
|
||||
$(dirname $0)/setup_pip.sh
|
||||
$(dirname $0)/pip_install.sh bindep
|
||||
$(dirname $0)/install_packages.sh
|
||||
for file in /opt/loci/pydep*; do
|
||||
PYDEP_PACKAGES+=($(bindep -f $file -b -l newline ${PROJECT} ${PROJECT_RELEASE} ${PROFILES} || :))
|
||||
done
|
||||
$(dirname $0)/pip_install.sh ${PYDEP_PACKAGES[@]}
|
||||
fi
|
||||
|
||||
$(dirname $0)/clone_project.sh
|
||||
$(dirname $0)/install_packages.sh
|
||||
|
||||
extra_projects_path=""
|
||||
for pr in $EXTRA_PROJECTS; do
|
||||
extra_projects_path="$extra_projects_path /tmp/${pr}"
|
||||
done
|
||||
project_cmd=/tmp/${PROJECT}
|
||||
if [[ -n ${PROJECT_PIP_EXTRAS} ]]; then
|
||||
project_cmd="${project_cmd}[${PROJECT_PIP_EXTRAS}]"
|
||||
fi
|
||||
|
||||
$(dirname $0)/pip_install.sh ${NO_INDEX} ${project_cmd} ${extra_projects_path} ${PIP_PACKAGES}
|
||||
for project_script in $(ls $(dirname $0)/project_specific/${PROJECT}); do
|
||||
echo "Running $PROJECT specific script $project_script"
|
||||
$(dirname $0)/project_specific/${PROJECT}/$project_script
|
||||
done
|
||||
$(dirname $0)/configure_packages.sh
|
||||
$(dirname $0)/collect_info.sh
|
||||
$(dirname $0)/cleanup.sh
|
||||
@@ -1,13 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
source /etc/lsb-release
|
||||
|
||||
for file in /opt/loci/bindep*; do
|
||||
PACKAGES+=($(bindep -f $file -b -l newline ${PROJECT} ${PROFILES} ${DISTRIB_CODENAME} || :))
|
||||
done
|
||||
|
||||
if [[ ! -z ${PACKAGES} ]]; then
|
||||
apt-get install -y --no-install-recommends ${PACKAGES[@]} ${DIST_PACKAGES}
|
||||
fi
|
||||
@@ -1,14 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
packages=$@
|
||||
|
||||
# Presence of constraint for project we build
|
||||
# in upper constraints breaks project installation
|
||||
# with unsatisfied constraints error.
|
||||
# This line ensures that such constraint is absent.
|
||||
cp ${WHEELS_PATH}/upper-constraints.txt /tmp/upper-constraints.txt
|
||||
sed -i "/^${PROJECT}===/d" /tmp/upper-constraints.txt
|
||||
|
||||
pip install --no-cache-dir --only-binary :all: --no-compile -c ${WHEELS_PATH}/global-requirements.txt -c /tmp/upper-constraints.txt --find-links ${WHEELS_PATH} --ignore-installed ${PIP_ARGS} ${packages}
|
||||
68
scripts/project.sh
Executable file
68
scripts/project.sh
Executable file
@@ -0,0 +1,68 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
source $(dirname $0)/helpers.sh
|
||||
|
||||
export LC_CTYPE=C.UTF-8
|
||||
|
||||
setup_venv
|
||||
|
||||
read -r -a extra_packages <<<"${DIST_PACKAGES}"
|
||||
read -r -a bindep_packages <<<"$(get_bindep_system_packages "${PROJECT}" ${PROFILES})"
|
||||
install_system_packages "${bindep_packages[@]}" "${extra_packages[@]}"
|
||||
|
||||
if [[ "${PLUGIN}" == "no" ]]; then
|
||||
create_user "${GID}" "${UID}" "${PROJECT}"
|
||||
fi
|
||||
|
||||
for file in /opt/loci/pydep*; do
|
||||
PYDEP_PACKAGES+=($(bindep -f $file -b -l newline ${PROJECT} ${PROJECT_RELEASE} ${PROFILES} || :))
|
||||
done
|
||||
pip install --no-cache-dir --only-binary :all: --no-compile \
|
||||
-c ${WHEELS_PATH}/global-requirements.txt -c ${WHEELS_PATH}/upper-constraints.txt \
|
||||
--find-links ${WHEELS_PATH} --ignore-installed \
|
||||
${PIP_ARGS} \
|
||||
${PYDEP_PACKAGES[@]}
|
||||
|
||||
clone_project "${PROJECT}" "${PROJECT_REPO}" "${PROJECT_REF}"
|
||||
|
||||
extra_projects_path=""
|
||||
for pr in $EXTRA_PROJECTS; do
|
||||
extra_projects_path="$extra_projects_path /tmp/${pr}"
|
||||
done
|
||||
|
||||
project_cmd=${SOURCES_DIR}/${PROJECT}
|
||||
if [[ -n ${PROJECT_PIP_EXTRAS} ]]; then
|
||||
project_cmd="${project_cmd}[${PROJECT_PIP_EXTRAS}]"
|
||||
fi
|
||||
|
||||
# Presence of constraint for project we build
|
||||
# in upper constraints breaks project installation
|
||||
# with unsatisfied constraints error.
|
||||
# This line ensures that such constraint is absent.
|
||||
cp ${WHEELS_PATH}/upper-constraints.txt /tmp/upper-constraints.txt
|
||||
sed -i "/^${PROJECT}===/d" /tmp/upper-constraints.txt
|
||||
|
||||
if [[ "${KEEP_ALL_WHEELS}" != "False" ]]; then
|
||||
# The requirements image contains not only those wheels
|
||||
# that we built from upper-constraints.txt but also all wheels
|
||||
# that were pulled as dependencies during that build.
|
||||
# We can use them as source for dependencies.
|
||||
NO_INDEX="--no-index"
|
||||
fi
|
||||
|
||||
pip install --no-cache-dir --only-binary :all: --no-compile \
|
||||
-c ${WHEELS_PATH}/global-requirements.txt -c /tmp/upper-constraints.txt \
|
||||
--find-links ${WHEELS_PATH} --ignore-installed \
|
||||
${NO_INDEX} ${PIP_ARGS} \
|
||||
${project_cmd} ${extra_projects_path} ${PIP_PACKAGES}
|
||||
|
||||
for project_script in $(ls $(dirname $0)/project_specific/${PROJECT}); do
|
||||
echo "Running $PROJECT specific script $project_script"
|
||||
$(dirname $0)/project_specific/${PROJECT}/$project_script
|
||||
done
|
||||
|
||||
configure_packages
|
||||
collect_info
|
||||
cleanup
|
||||
@@ -2,65 +2,18 @@
|
||||
|
||||
set -eux
|
||||
|
||||
source $(dirname $0)/helpers.sh
|
||||
|
||||
SOURCES_DIR=/tmp
|
||||
UPPER_CONSTRAINTS=/upper-constraints.txt
|
||||
UPPER_CONSTRAINTS_BUILD=/build-upper-constraints.txt
|
||||
UPPER_CONSTRAINTS_DEV=/dev-upper-constraints.txt
|
||||
|
||||
$(dirname $0)/setup_pip.sh
|
||||
pip install ${PIP_ARGS} bindep pkginfo
|
||||
setup_venv
|
||||
|
||||
$(dirname $0)/install_packages.sh
|
||||
$(dirname $0)/clone_project.sh
|
||||
mv /tmp/requirements/{global-requirements.txt,upper-constraints.txt} /
|
||||
read -r -a bindep_packages <<<"$(get_bindep_system_packages "${PROJECT}")"
|
||||
install_system_packages "${bindep_packages[@]}"
|
||||
|
||||
function get_pkg_name {
|
||||
local folder=$1
|
||||
local name
|
||||
pushd $folder > /dev/null
|
||||
name=$(python3 setup.py --name 2>/dev/null | grep -v '^\[pbr\]')
|
||||
popd > /dev/null
|
||||
echo $name
|
||||
}
|
||||
|
||||
function get_pkg_version {
|
||||
local folder=$1
|
||||
local vesion
|
||||
pushd $folder > /dev/null
|
||||
version=$(python3 setup.py --version 2>/dev/null | grep -v '^\[pbr\]')
|
||||
popd > /dev/null
|
||||
echo $version
|
||||
}
|
||||
|
||||
function get_pipy_name_by_project_name {
|
||||
local project_name=$1
|
||||
while read _folder_name _pipy_name _pkg_name; do
|
||||
if [[ "${_pkg_name}" == "${project_name}" ]]; then
|
||||
echo "${_pipy_name}"
|
||||
return
|
||||
fi
|
||||
done < /opt/loci/scripts/python-custom-name-mapping.txt
|
||||
echo "$project_name"
|
||||
}
|
||||
|
||||
function make_build_constraints {
|
||||
cp $UPPER_CONSTRAINTS $UPPER_CONSTRAINTS_DEV
|
||||
cp $UPPER_CONSTRAINTS $UPPER_CONSTRAINTS_BUILD
|
||||
pushd $SOURCES_DIR
|
||||
for repo in $(ls -1 $SOURCES_DIR); do
|
||||
if [[ ! -f $repo/setup.cfg ]]; then
|
||||
continue
|
||||
fi
|
||||
echo "Making build constraint for $repo"
|
||||
pkg_name=$(get_pkg_name $repo)
|
||||
pkg_version=$(get_pkg_version $repo)
|
||||
pipy_name=$(get_pipy_name_by_project_name $pkg_name)
|
||||
sed -i "s|^${pipy_name}===.*|file://${SOURCES_DIR}/${repo}#egg=${pkg_name}|g" $UPPER_CONSTRAINTS_BUILD
|
||||
sed -i "s|^${pipy_name}===.*|${pipy_name}===${pkg_version}|g" $UPPER_CONSTRAINTS_DEV
|
||||
done
|
||||
popd
|
||||
}
|
||||
clone_project "${PROJECT}" "${PROJECT_REPO}" "${PROJECT_REF}"
|
||||
mv ${SOURCES_DIR}/requirements/{global-requirements.txt,upper-constraints.txt} /
|
||||
|
||||
# Setuptools from constraints is not compatible with other constrainted packages
|
||||
[[ "${PROJECT_REF}" == "master" ]] && sed -i '/setuptools/d' /upper-constraints.txt
|
||||
@@ -78,15 +31,7 @@ fi
|
||||
# Drop ceilometer from constraints.
|
||||
sed -i '/ceilometer===.*/d' /upper-constraints.txt
|
||||
|
||||
# NOTE: This is to install python-openstackclient from the HEAD of the stable/2025.1 branch
|
||||
# This was merged after 7.4.0 which is pinned in the 2025.1 constraints.
|
||||
# https://review.opendev.org/c/openstack/python-openstackclient/+/944991
|
||||
if [[ ${PROJECT_REF} == "stable/2025.1" ]]; then
|
||||
sed -i "s|^python-openstackclient===.*|git+https://opendev.org/openstack/python-openstackclient.git@stable/2025.1#egg=python-openstackclient|g" /upper-constraints.txt
|
||||
fi
|
||||
|
||||
# NOTE: This is to install tap-as-a-service the stable/2025.2 branch
|
||||
# It has been removed from the upper-constraints.txt.
|
||||
# NOTE: This is to build tap-as-a-service. It was removed from the upper-constraints.txt in stable/2025.2.
|
||||
if [[ ${PROJECT_REF} == "stable/2025.2" ]]; then
|
||||
echo "git+https://opendev.org/openstack/tap-as-a-service.git@stable/2025.2#egg=tap-as-a-service" >> /upper-constraints.txt
|
||||
fi
|
||||
@@ -128,9 +73,7 @@ export CPUCOUNT=1
|
||||
export UPPER_CONSTRAINTS_BUILD
|
||||
|
||||
# Construct upper-constraints with honor of locally downloaded projects
|
||||
make_build_constraints
|
||||
rm -rf $UPPER_CONSTRAINTS
|
||||
mv $UPPER_CONSTRAINTS_DEV $UPPER_CONSTRAINTS
|
||||
honor_local_sources "${SOURCES_DIR}" "${UPPER_CONSTRAINTS}" "${UPPER_CONSTRAINTS_BUILD}"
|
||||
|
||||
echo "DEBUG: ${UPPER_CONSTRAINTS_BUILD}"
|
||||
cat ${UPPER_CONSTRAINTS_BUILD}
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
PIP_VERSION_CONSTRAINT=${PIP_VERSION_CONSTRAINT}
|
||||
SETUPTOOL_CONSTRAINT=${SETUPTOOL_CONSTRAINT}
|
||||
WHEEL_CONSTRAIN=${WHEEL_CONSTRAIN}
|
||||
|
||||
# Create the virtualenv with the updated toolchain for openstack service
|
||||
# for using python-rbd which is not pip installable and is only available
|
||||
# in packaged form.
|
||||
python3 -m venv --system-site-packages /var/lib/openstack
|
||||
|
||||
source /var/lib/openstack/bin/activate
|
||||
|
||||
pip install --upgrade pip${PIP_VERSION_CONSTRAINT}
|
||||
pip install --upgrade setuptools${SETUPTOOL_CONSTRAINT}
|
||||
pip install --upgrade wheel${WHEEL_CONSTRAIN}
|
||||
Reference in New Issue
Block a user