Pull embedded docker images with ctr
Instead of pulling images on the build host with docker, saving them to a tarball, and loading them into the guest chroot with dockerd, pull images directly into the image's containerd store during the guest image build. This avoids redundant disk writes and removes the build host docker dependency. Retry with --plain-http when HTTPS pull fails so HTTP local registries work without breaking pulls from HTTPS registries such as quay.io. Co-authored-by: Cursor <cursoragent@cursor.com> Change-Id: I07b9622c0788edee91d39dd9e5cf1030864d44ca Signed-off-by: wu.chunyang <wchy1001@gmail.com>
This commit is contained in:
wu.chunyangandCursor committed
1 parent
4cdc0cdadc
commit
f5f2728eb6
6 files changed
+66
-150
No files matched your search
+5
-4
@@ -459,10 +459,11 @@ function configure_tempest_for_trove {
|
||||
|
||||
# Compute the docker image references to embed in the guest image and
|
||||
# export them as DIB_TROVE_DOCKER_IMAGES for the guest-agent element, which
|
||||
# fetches and tags them (see extra-data.d/70-trove-docker-images). Entries
|
||||
# are [source=]target references: the guest-visible target names (e.g. the
|
||||
# local registry) can't be pulled at build time, so the element pulls the
|
||||
# source and tags it as the target.
|
||||
# pulls them directly into the image's containerd store at build time (see
|
||||
# post-install.d/35-preload-docker-images). Entries are [source=]target
|
||||
# references: the guest-visible target names (e.g. the local registry) can't
|
||||
# be pulled at build time, so the element pulls the source and tags it as the
|
||||
# target.
|
||||
function prepare_embedded_docker_images_vars {
|
||||
local quay_alias=${TROVE_DATASTORE_TYPE}
|
||||
[[ "${quay_alias}" == "postgresql" ]] && quay_alias="postgres"
|
||||
|
||||
@@ -162,14 +162,16 @@ Guest image variables
|
||||
``docker_image``/``backup_docker_image`` values the guest agent is
|
||||
configured with; when a source is given, it is pulled and tagged as
|
||||
the target (used when the target name cannot be pulled at build time,
|
||||
e.g. a local registry that only the guest instances can reach). The
|
||||
devstack plugin sets this automatically from the configured datastore
|
||||
when ``TROVE_EMBED_DATASTORE_IMAGES`` is enabled; when building
|
||||
images with ``trovestack build-image`` directly, export it manually.
|
||||
When the variable is set, embedding must succeed: any failure (an
|
||||
image that cannot be fetched, or the extraction into the image
|
||||
failing) fails the build, rather than silently producing an image
|
||||
that pulls at boot. Leave it unset to build without embedded images.
|
||||
e.g. a local registry that only the guest instances can reach). Images
|
||||
are pulled directly from the registry into the image's containerd store
|
||||
during the guest image build (no intermediate tarball on the build
|
||||
host). The devstack plugin sets this automatically from the configured
|
||||
datastore when ``TROVE_EMBED_DATASTORE_IMAGES`` is enabled; when
|
||||
building images with ``trovestack build-image`` directly, export it
|
||||
manually. When the variable is set, embedding must succeed: any failure
|
||||
(an image that cannot be fetched, or the pull into the image failing)
|
||||
fails the build, rather than silently producing an image that pulls at
|
||||
boot. Leave it unset to build without embedded images.
|
||||
|
||||
Reinstall and cleanup
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
@@ -6,8 +6,9 @@ repositories
|
||||
Environment variables:
|
||||
|
||||
``DIB_TROVE_DOCKER_IMAGES``
|
||||
Optional, space separated list of ``[source=]target`` docker image
|
||||
references to embed, fully extracted, into the image's containerd store
|
||||
at build time, so instances don't pull images at create time. See the
|
||||
"Guest image variables" section of
|
||||
``doc/source/install/install-devstack.rst`` for details.
|
||||
Optional, space separated list of ``[source=]target`` docker image
|
||||
references to embed, fully extracted, into the image's containerd store
|
||||
at build time (pulled directly with ``ctr`` during image build), so
|
||||
instances don't pull images at create time. See the "Guest image
|
||||
variables" section of ``doc/source/install/install-devstack.rst`` for
|
||||
details.
|
||||
@@ -1,58 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Fetch the docker images listed in DIB_TROVE_DOCKER_IMAGES with the build
|
||||
# host's docker daemon and save them into a tarball in the hooks directory.
|
||||
# post-install.d extracts it into the image's containerd image store, so
|
||||
# guest instances don't have to pull the images over the network (and,
|
||||
# worse, decompress them on an emulated CPU in the gate) at instance
|
||||
# create time. The tarball itself never enters the image.
|
||||
#
|
||||
# Entries are space separated [source=]target references. The target is
|
||||
# the name the guest agent is configured to use, which is what gets saved;
|
||||
# when a source is given, it is pulled and tagged as the target (used when
|
||||
# the target name is not pullable at build time, e.g. a local registry
|
||||
# that only the guests can reach).
|
||||
#
|
||||
# When DIB_TROVE_DOCKER_IMAGES is set, embedding is expected to work: any
|
||||
# failure here fails the build rather than silently producing an image
|
||||
# that falls back to pulling at boot. Leave the variable unset to build
|
||||
# without embedded images.
|
||||
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
[ -z "${DIB_TROVE_DOCKER_IMAGES:-}" ] && exit 0
|
||||
|
||||
if ! sudo docker info >/dev/null 2>&1; then
|
||||
echo "ERROR: DIB_TROVE_DOCKER_IMAGES is set but docker is not usable" \
|
||||
"on the build host."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
targets=""
|
||||
for entry in ${DIB_TROVE_DOCKER_IMAGES}; do
|
||||
source=${entry%%=*}
|
||||
target=${entry##*=}
|
||||
|
||||
if ! sudo docker image inspect "${target}" >/dev/null 2>&1; then
|
||||
if ! sudo docker image inspect "${source}" >/dev/null 2>&1; then
|
||||
# docker retries failed layer downloads internally, so no
|
||||
# extra retrying here.
|
||||
if ! sudo docker pull "${source}"; then
|
||||
echo "ERROR: unable to pull ${source} for embedding."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
if [ "${source}" != "${target}" ]; then
|
||||
sudo docker tag "${source}" "${target}"
|
||||
fi
|
||||
fi
|
||||
targets="${targets} ${target}"
|
||||
done
|
||||
|
||||
echo "Embedding docker images in the guest image:${targets}"
|
||||
sudo docker save -o ${TMP_HOOKS_PATH}/trove-docker-images.tar ${targets}
|
||||
sudo chmod 0644 ${TMP_HOOKS_PATH}/trove-docker-images.tar
|
||||
+45
-69
@@ -1,26 +1,21 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Extract the docker image tarball produced by
|
||||
# extra-data.d/70-trove-docker-images into the image's containerd image
|
||||
# store, by running the image's own containerd and dockerd inside the
|
||||
# chroot (as plain binaries - systemd is not running here). Instances then
|
||||
# boot with the images fully extracted: no docker pull at instance create
|
||||
# time, which in the gate runs on an emulated CPU and dominates instance
|
||||
# provisioning time.
|
||||
# Pull docker images listed in DIB_TROVE_DOCKER_IMAGES directly into the
|
||||
# image's containerd store using ctr, by running containerd inside the
|
||||
# chroot (systemd is not running here). Instances then boot with the images
|
||||
# fully extracted: no docker pull at instance create time, which in the
|
||||
# gate runs on an emulated CPU and dominates instance provisioning time.
|
||||
#
|
||||
# containerd is started first and dockerd is pointed at it, mirroring how
|
||||
# the guest runs at boot (containerd.service + docker.service): docker
|
||||
# stores images in the containerd instance it is connected to, so a
|
||||
# dockerd-managed containerd here would strand the images under
|
||||
# /var/lib/docker/containerd where the booted guest would never see them.
|
||||
# Images are pulled into containerd's "moby" namespace, which is what Docker
|
||||
# CE uses at boot (containerd.service + docker.service).
|
||||
#
|
||||
# The dib chroot needs accommodations, all undone on every exit path:
|
||||
# a cgroup2 hierarchy at /sys/fs/cgroup (dockerd refuses to start without
|
||||
# one: "Devices cgroup isn't mounted"; cgroup2 is a single hierarchy that
|
||||
# may be mounted at multiple points, so this doesn't disturb the build
|
||||
# host), and self bind-mounts of the containerd and docker state roots
|
||||
# (the chroot root is not a mountpoint, so containerd's mountinfo lookups
|
||||
# fail without them).
|
||||
# Entries are space separated [source=]target references. When a source is
|
||||
# given it is pulled and tagged as the target (used when the target name is
|
||||
# not pullable at build time, e.g. a local registry only guests can reach).
|
||||
#
|
||||
# The dib chroot needs a self bind-mount of the containerd state root (the
|
||||
# chroot root is not a mountpoint, so containerd's mountinfo lookups fail
|
||||
# without it); undone on every exit path.
|
||||
#
|
||||
# When embedding was requested, it must work: any failure here fails the
|
||||
# build. A silent fallback would produce an image that pulls at boot,
|
||||
@@ -32,21 +27,13 @@ fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
# The tarball lives in the bind-mounted hooks directory and never enters
|
||||
# the image itself. Absent tarball means embedding was not requested (the
|
||||
# extra-data.d hook fails the build if it was requested and could not be
|
||||
# prepared).
|
||||
TARBALL=/tmp/in_target.d/trove-docker-images.tar
|
||||
[ -f ${TARBALL} ] || exit 0
|
||||
[ -z "${DIB_TROVE_DOCKER_IMAGES:-}" ] && exit 0
|
||||
|
||||
CONTAINERD_SOCK=/run/containerd/containerd.sock
|
||||
SOCK=unix:///run/dib-docker.sock
|
||||
LOG=/tmp/dib-dockerd.log
|
||||
CLOG=/tmp/dib-containerd.log
|
||||
DOCKERD_PID=""
|
||||
CONTAINERD_PID=""
|
||||
CGROUP_MOUNTED=""
|
||||
BIND_MOUNTS=""
|
||||
CTR="ctr -n moby"
|
||||
|
||||
stop_daemon() {
|
||||
local pid=$1
|
||||
@@ -57,18 +44,13 @@ stop_daemon() {
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
stop_daemon "${DOCKERD_PID}"; DOCKERD_PID=""
|
||||
stop_daemon "${CONTAINERD_PID}"; CONTAINERD_PID=""
|
||||
rm -f /run/dib-docker.pid /run/dib-docker.sock ${CONTAINERD_SOCK}
|
||||
rm -f ${CONTAINERD_SOCK}
|
||||
local m
|
||||
for m in ${BIND_MOUNTS}; do
|
||||
umount ${m} || true
|
||||
done
|
||||
BIND_MOUNTS=""
|
||||
if [ -n "${CGROUP_MOUNTED}" ]; then
|
||||
umount /sys/fs/cgroup || true
|
||||
CGROUP_MOUNTED=""
|
||||
fi
|
||||
:
|
||||
}
|
||||
trap cleanup EXIT
|
||||
@@ -77,24 +59,15 @@ fail() {
|
||||
echo "ERROR: $*"
|
||||
echo "--- containerd log ---"
|
||||
cat ${CLOG} 2>/dev/null || true
|
||||
echo "--- dockerd log ---"
|
||||
cat ${LOG} 2>/dev/null || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
setup_chroot_env() {
|
||||
if ! mountpoint -q /sys/fs/cgroup; then
|
||||
mkdir -p /sys/fs/cgroup
|
||||
mount -t cgroup2 cgroup2 /sys/fs/cgroup
|
||||
CGROUP_MOUNTED=1
|
||||
fi
|
||||
|
||||
local d
|
||||
for d in /var/lib/containerd /var/lib/docker; do
|
||||
mkdir -p ${d}
|
||||
mount --bind ${d} ${d}
|
||||
BIND_MOUNTS="${d} ${BIND_MOUNTS}"
|
||||
done
|
||||
# containerd's mountinfo lookups fail when the chroot root is not a
|
||||
# mountpoint; a self bind-mount makes /var/lib/containerd visible there.
|
||||
mkdir -p /var/lib/containerd
|
||||
mount --bind /var/lib/containerd /var/lib/containerd
|
||||
BIND_MOUNTS="/var/lib/containerd"
|
||||
}
|
||||
|
||||
start_containerd() {
|
||||
@@ -111,28 +84,31 @@ start_containerd() {
|
||||
return 1
|
||||
}
|
||||
|
||||
start_dockerd() {
|
||||
dockerd --host=${SOCK} \
|
||||
--pidfile=/run/dib-docker.pid \
|
||||
--containerd=${CONTAINERD_SOCK} \
|
||||
--bridge=none --iptables=false --ip6tables=false \
|
||||
> ${LOG} 2>&1 &
|
||||
DOCKERD_PID=$!
|
||||
|
||||
local i
|
||||
for i in $(seq 60); do
|
||||
if docker -H ${SOCK} info >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
# Bail out early if dockerd already died.
|
||||
kill -0 ${DOCKERD_PID} 2>/dev/null || return 1
|
||||
sleep 1
|
||||
done
|
||||
return 1
|
||||
pull_image() {
|
||||
local ref=$1
|
||||
if ${CTR} images pull "${ref}"; then
|
||||
return 0
|
||||
fi
|
||||
echo "Retrying ${ref} with --plain-http (insecure local registry)"
|
||||
${CTR} images pull --plain-http "${ref}"
|
||||
}
|
||||
|
||||
setup_chroot_env
|
||||
start_containerd || fail "containerd did not start in the chroot"
|
||||
start_dockerd || fail "dockerd did not start in the chroot"
|
||||
docker -H ${SOCK} load -i ${TARBALL} || fail "docker load failed"
|
||||
|
||||
for entry in ${DIB_TROVE_DOCKER_IMAGES}; do
|
||||
source=${entry%%=*}
|
||||
target=${entry##*=}
|
||||
|
||||
echo "Embedding docker image ${source} as ${target}"
|
||||
if ! pull_image "${source}"; then
|
||||
fail "unable to pull ${source} for embedding"
|
||||
fi
|
||||
if [ "${source}" != "${target}" ]; then
|
||||
if ! ${CTR} images tag "${source}" "${target}"; then
|
||||
fail "unable to tag ${source} as ${target}"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Preloaded embedded docker images into the containerd image store"
|
||||
@@ -769,12 +769,6 @@ function cmd_build_image() {
|
||||
else
|
||||
sudo $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS install qemu-utils git kpartx debootstrap squashfs-tools python3-pip python3-setuptools python3-venv
|
||||
fi
|
||||
# Embedding docker images in the guest image (DIB_TROVE_DOCKER_IMAGES)
|
||||
# needs a working docker daemon on the build host.
|
||||
if [[ -n "${DIB_TROVE_DOCKER_IMAGES:-}" ]] && ! which docker >/dev/null 2>&1; then
|
||||
cmd_install_docker
|
||||
fi
|
||||
|
||||
python3 -m venv ~/diskimage-builder-env
|
||||
sudo $HTTP_PROXY ~/diskimage-builder-env/bin/pip3 install setuptools diskimage-builder
|
||||
source ~/diskimage-builder-env/bin/activate
|
||||
|
||||
Reference in new issue
Block a user