simplify compose create

This commit does several things at once:

- Drop the usage of bdebstrap, it doesnt seem to be available
  as a Debian package going forward, and it makes the start of
  building unit tests easier.
- Use the "unshare" mode by default. This allows us to build
  a rootfs as a regular user and doesnt create device files.
  Which allows us to to drop some workarounds.
- Add debian/trixie support. This was created by doing a straight
  copy of debian/bookworm.

Story: 2010867
Task: 48556

Test Plan
PASS Create ostree branch

Change-Id: Idd38c6417769296d7ab4d6640624a18d5de74da4
Signed-off-by: Charles Short <charles.short@windriver.com>
This commit is contained in:
Charles Short 2024-01-02 14:09:21 +00:00
parent dd24168756
commit 3f2fbfb24f
30 changed files with 573 additions and 21 deletions

View File

@ -9,3 +9,4 @@ rules:
ignore: |
.zuul.yaml
config/debian/bookworm/image/image.yaml
config/debian/trixie/image/image.yaml

View File

@ -10,11 +10,13 @@ import logging
import os
import shutil
import sys
import yaml
import apt
from rich.console import Console
from apt_ostree.constants import excluded_packages
from apt_ostree import exceptions
from apt_ostree.ostree import Ostree
from apt_ostree.utils import run_command
@ -52,18 +54,87 @@ class Bootstrap:
self.logging.info("Found working directory from "
"previous run...removing.")
shutil.rmtree(workdir)
workdir.mkdir(parents=True, exist_ok=True)
rootfs.mkdir(parents=True, exist_ok=True)
self.logging.info("Running bdebstrap, please wait.")
verbosity = "-q"
shutil.copytree(self.state.base, workdir)
cfg = workdir.joinpath("bootstrap.yaml")
if not cfg.exists():
msg = "Unable to find bootstrap.yaml in %s." % workdir
raise exceptions.ConfigError(msg)
with open(cfg, "r") as f:
try:
config = yaml.safe_load(f)
except yaml.YAMLError as exc:
if hasattr(exc, 'problem_mark'):
mark = exc.problem_mark
line = mark.line+1
col = mark.column+1
msg = \
f"Error in bootstrap.yaml at ({line}:{col})"
raise exceptions.ConError(msg)
msg = f"Failed to parse bootstrap yaml {exc}"
raise exceptions.ConfigError(msg)
config = config.get("mmdebstrap", None)
if config is None:
msg = \
"Error reading bootstrap.yaml. Mmdebstrap section missing."
raise exceptions.ConfigError(msg)
if not shutil.which("mmdebstrap"):
raise exceptions.CommandError(
"Command not found: mmdebstrap.")
cmd = ["mmdebstrap"]
if self.state.debug:
verbosity = "-v"
run_command(
["bdebstrap", "-c", "bootstrap.yaml", verbosity,
"--force", "--name", str(self.state.branch),
"--target", str(rootfs),
"--output", str(workdir)], cwd=self.state.base)
cmd += ["--debug"]
else:
cmd += ["-v"]
# Doesnt create a "/dev"
cmd += ["--mode=unshare"]
# If the suite is not specified then it will use the
# current running Debian version.
suite = config.get("suite")
if suite:
cmd += [suite]
# If the architecture is not specified it will use the
# same as the host.
architecture = config.get("architectures", None)
if architecture:
cmd += [f"--architectures={architecture}"]
# Add additional archive pockets.
components = config.get("components", None)
if components is None:
raise exceptions.ConfigError(
"Unable to determine package archive components.")
cmd += [f"--components={','.join(components)}"]
# Generate the target rootfs
rootfs = workdir.joinpath("rootfs")
cmd += [str(rootfs)]
# Include additional Debian packages
packages = config.get("packages", None)
if packages:
cmd += [f"--include={','.join(packages)}"]
# Run addtional scripts or copy addtional files into
# target.
setup_hooks = config.get("setup-hooks", None)
if setup_hooks:
cmd += [f"--setup-hook={hook}" for hook in setup_hooks]
extract_hooks = config.get("extract-hook", None)
if extract_hooks:
cmd += [f"--extract-hook={hook}" for hook in extract_hooks]
customize_hooks = config.get("customize-hooks", None)
if customize_hooks:
cmd += [f"--customize-hook={hook}" for hook in customize_hooks]
self.logging.info("Running mmdebstrap.")
run_command(cmd, cwd=workdir)
self.ostree.init()
self.logging.info(f"Found ostree branch: {self.state.branch}")
@ -98,11 +169,6 @@ class Bootstrap:
with self.console.status(f"Converting {rootdir} to ostree."):
dir_perm = 0o755
# Emptying /dev
self.logging.info("Emptying /dev.")
shutil.rmtree(rootdir.joinpath("dev"))
os.mkdir(rootdir.joinpath("dev"), dir_perm)
# Copying /var
self.sanitize_usr_symlinks(rootdir)
self.logging.info("Moving /var to /usr/rootdirs.")

View File

@ -17,6 +17,11 @@ class AptError(Exception):
return self.message or ""
class ConfigError(AptError):
"""Configuration file error."""
pass
class CommandError(AptError):
"""Command execution error."""
pass

View File

@ -6,8 +6,12 @@
---
mmdebstrap:
suite: bookworm
architectures: [amd64]
components: [main, contrib, non-free, non-free-firmware]
architectures: amd64
components:
- main
- contrib
- non-free
- non-free-firmware
# Customizations that happen before bootstrapping.
setup-hooks:
@ -60,7 +64,7 @@ mmdebstrap:
- python3-gi
- python3-click
- python3-systemd
- bdebstrap
- mmdebstrap
- gobject-introspection
- gir1.2-ostree-1.0
- software-properties-common

View File

@ -6,8 +6,12 @@
---
mmdebstrap:
suite: bullseye
architectures: [amd64]
components: [main, contrib, non-free]
architectures: amd64
components:
- main
- contrib
- non-free
- non-free-firmware
vairant: minbase
setup-hooks:

View File

@ -0,0 +1,66 @@
---
mmdebstrap:
suite: trixie
architectures: amd64
components:
- main
- contrib
- non-free
- non-free-firmware
# Customizations that happen before bootstrapping.
setup-hooks:
- 'sync-in overlay/debian/ /'
- 'sync-in overlay/ostree/ /'
- 'sync-in overlay/auto-login/ /'
- 'sync-in overlay/firstboot/ /'
# Customizations that happen after boostrapping.
customize-hooks:
- 'sync-in overlay/adduser/ /'
- echo 'root:root' | chroot "$1" chpasswd
- chroot $1 echo "localhost" > $1/etc/hostname
- systemctl disable --root=$1 apt-daily.timer cron
- systemctl enable --root="$1" firstboot systemd-networkd
- chroot $1 cp -alf /lib/systemd/resolv.conf /etc/resolv.conf
- chroot $1 cp -alf /usr/libexec/libostree/grub2-15_ostree /etc/grub.d/15_ostree
- chroot $1 cp -alf /usr/lib/os-release /etc/os-release
# Packages to install after bootstrapping.
packages:
- linux-image-amd64
- sudo
- ostree
- ostree-boot
- libostree-1-1
- libostree-dev
- systemd-boot
- mmdebstrap
- dracut
- grub-efi-amd64
- vim
- net-tools
- openssh-client
- openssh-server
- procps
- less
- dbus
- policykit-1
- libnss-resolve
- curl
- wget
- git-core
- rsync
- bubblewrap
- python3-apt
- python3-all
- python3-pip
- python3-pbr
- python3-gi
- python3-click
- python3-systemd
- gobject-introspection
- gir1.2-ostree-1.0
- software-properties-common
- libnss-systemd

View File

@ -0,0 +1,60 @@
{{/* Copyright (c) 2023 Wind River Systems, Inc. */}}
{{/* SPDX-License-Identifier: Apache-2.0 */}}
{{- $architecture := or .architecture "amd64" -}}
{{- $image := or .image (printf "debian-ostree-qemu-uefi-%s.img" $architecture) -}}
{{- $cmdline := or .cmdline "console=tty0 console=ttyS0,115200n8 rootwait rw fsck.mode=auto fsck.repair=yes systemd.gpt_auto=false" -}}
{{- $branch := or .branch "debian/trixie" -}}
{{- $repo := or .repo "ostree_repo" -}}
{{- $size := or .size "20G" -}}
architecture: {{ $architecture }}
actions:
- action: image-partition
imagename: {{ $image }}
imagesize: {{ $size }}
partitiontype: gpt
mountpoints:
- mountpoint: /
partition: system
- mountpoint: /boot/efi
partition: EFI
partitions:
- name: EFI
fs: vfat
start: 0%
end: 256M
flags: [boot]
- name: system
fs: ext4
start: 266M
end: 100%
# Reset the rootfs to allow to deploy OSTree from a clean rootfs
- action: run
description: Reset rootfs before deploying OSTree
chroot: false
command: find ${ROOTDIR} -maxdepth 1 -mindepth 1 -exec rm -rf {} \;
- action: ostree-deploy
repository: ostree_repo
branch: {{ $branch }}
os: debian
append-kernel-cmdline: {{ $cmdline }}
- action: run
description: enable signature verification
chroot: false
command: ostree --repo="${ROOTDIR}/ostree/repo" config set 'remote "origin"'.sign-verify "true"
- action: run
description: enable update bundle verification
chroot: false
command: ostree --repo="${ROOTDIR}/ostree/repo" config set core.sign-verify-deltas "true"
- action: run
description: install bootloader
chroot: false
script: scripts/setup-uefi-bootloader.sh debian

View File

@ -0,0 +1,52 @@
#!/bin/sh
# Based on original script from OStree upstream:
# https://github.com/ostreedev/ostree/blob/master/src/switchroot/switchroot.sh
# Copyright (C) 2013, 2016 Collabora Ltd
# Copyright (C) 2023 Wind River Systems, Inc.
set -eu
sysroot=${ROOTDIR}
osname=$1
bootconf=$sysroot/boot/loader/entries/ostree-1-${osname}.conf
if [ ! -f "$bootconf" ]; then
echo "OStree setup is not available!"
exit 1
fi
ostree=$(grep -o 'ostree=[/.[:alnum:]]\+' $bootconf)
ostree=${ostree#*=}
# Do symlink
ostree=$(realpath $sysroot$ostree)
mkdir -p $ostree/boot/efi
## /etc/machine-id must be writable to allow to work systemd-nspawn
# but original `machine-id` file must stay empty in build time.
touch /tmp/machine-id
mount --bind /tmp/machine-id $ostree/etc/machine-id
# NB: The native resolv.conf management is supported only from systemd v.232.
systemd-nspawn --resolv-conf=off -D $ostree systemd-machine-id-setup
# install EFI
systemd-nspawn \
--resolv-conf=off \
--bind $sysroot/boot:/boot \
--bind $sysroot/boot/efi:/boot/efi \
--bind $sysroot/ostree/deploy/${osname}/var:/var \
-D $ostree bootctl --path=/boot/efi install
umount $ostree/etc/machine-id
rmdir $ostree/boot/efi
# Copy config, kernel and initrd
# Change the name of config to unify with patch added in T4469
rsync -Pav $sysroot/boot/ostree $sysroot/boot/efi/
mkdir $sysroot/boot/efi/loader/entries/
cp $bootconf $sysroot/boot/efi/loader/entries/ostree-0-1.conf
rm -f $sysroot/boot/efi/loader/loader.conf

View File

@ -0,0 +1,107 @@
# /etc/adduser.conf: `adduser' configuration.
# See adduser(8) and adduser.conf(5) for full documentation.
# A commented out setting indicates that this is the default in the
# code. If you need to change those settings, remove the comment and
# make your intended change.
# STDERRMSGLEVEL, STDOUTMSGLEVEL, and LOGMSGLEVEL set the minimum
# priority for messages logged to syslog/journal and the console,
# respectively.
# Values are trace, debug, info, warn, err, and fatal.
# Messages with the set priority or higher get logged to the
# respective medium.
#STDERRMSGLEVEL=warn
#STDOUTMSGLEVEL=info
#SYSLOGLEVEL=info
# The login shell to be used for all new users.
# Default: DSHELL=/bin/bash
#DSHELL=/bin/bash
# The directory in which new home directories should be created.
# Default: DHOME=/home
DHOME=/var/home
# The directory from which skeletal user configuration files
# will be copied.
# Default: SKEL=/etc/skel
#SKEL=/etc/skel
# Specify inclusive ranges of UIDs and GIDs from which UIDs and GIDs
# for system users, system groups, non-system users and non-system groups
# can be dynamically allocated.
# Default: FIRST_SYSTEM_UID=100, LAST_SYSTEM_UID=999
#FIRST_SYSTEM_UID=100
#LAST_SYSTEM_UID=999
# Default: FIRST_SYSTEM_GID=100, LAST_SYSTEM_GID=999
#FIRST_SYSTEM_GID=100
#LAST_SYSTEM_GID=999
# Default: FIRST_UID=1000, LAST_UID=59999
#FIRST_UID=1000
#LAST_UID=59999
# Default: FIRST_GID=1000, LAST_GID=59999
#FIRST_GID=1000
#LAST_GID=59999
# Specify a file or a directory containing UID and GID pool.
#UID_POOL=/etc/adduser-pool.conf
#UID_POOL=/etc/adduser-pool.d/
#GID_POOL=/etc/adduser-pool.conf
#GID_POOL=/etc/adduser-pool.d/
# Specify whether each created non-system user will be
# given their own group to use.
# Default: USERGROUPS=yes
#USERGROUPS=yes
# Defines the groupname or GID of the group all newly-created
# non-system users are placed into.
# It is a configuration error to define both variables
# even if the values are consistent.
# Default: USERS_GID=undefined, USERS_GROUP=users
#USERS_GID=100
#USERS_GROUP=users
# The permissions mode for home directories of non-system users.
# Default: DIR_MODE=0700
#DIR_MODE=0700
# The permissions mode for home directories of system users.
# Default: SYS_DIR_MODE=0755
#SYS_DIR_MODE=0755
# If set to a nonempty value, new users will have quotas copied
# from that user with `edquota -p QUOTAUSER newuser'
# Default: QUOTAUSER=""
#QUOTAUSER=""
# Non-system user- and groupnames are checked against this regular
# expression.
# Default: NAME_REGEX="^[a-z][-a-z0-9_]*\$?$"
#NAME_REGEX="^[a-z][-a-z0-9_]*\$?$"
# System user- and groupnames are checked against this regular
# expression.
# Default: SYS_NAME_REGEX="^[a-z_][-a-z0-9_]*\$?$"
#SYS_NAME_REGEX="^[a-z_][-a-z0-9_]*\$?$"
# When populating the newly created home directory of a non-system user,
# files in SKEL matching this regex are not copied.
# Default: SKEL_IGNORE_REGEX="\.(dpkg|ucf)-(old|new|dist|save)$"
#SKEL_IGNORE_REGEX="\.(dpkg|ucf)-(old|new|dist|save)$"
# list of groups that new non-system users will be added to
# if ADD_EXTRA_GROUPS is non-zero or set on the command line.
# Default: EXTRA_GROUPS="users"
#EXTRA_GROUPS="users"
# Setting this to something other than 0 will cause adduser to add
# newly created non-system users to the list of groups defined by
# EXTRA_GROUPS.
# Default: ADD_EXTRA_GROUPS=0
#ADD_EXTRA_GROUPS=0

View File

@ -0,0 +1,37 @@
# Default values for useradd(8)
#
# The SHELL variable specifies the default login shell on your
# system.
# Similar to DSHELL in adduser. However, we use "sh" here because
# useradd is a low level utility and should be as general
# as possible
SHELL=/bin/sh
#
# The default group for users
# 100=users on Debian systems
# Same as USERS_GID in adduser
# This argument is used when the -n flag is specified.
# The default behavior (when -n and -g are not specified) is to create a
# primary user group with the same name as the user being added to the
# system.
# GROUP=100
#
# The default home directory. Same as DHOME for adduser
HOME=/var/home
#
# The number of days after a password expires until the account
# is permanently disabled
# INACTIVE=-1
#
# The default expire date
# EXPIRE=
#
# The SKEL variable specifies the directory containing "skeletal" user
# files; in other words, files such as a sample .profile that will be
# copied to the new user's home directory when it is created.
# SKEL=/etc/skel
#
# Defines whether the mail spool should be created while
# creating the account
# CREATE_MAIL_SPOOL=no

View File

@ -0,0 +1 @@
localhost

View File

@ -0,0 +1,5 @@
[Match]
Name=en*
[Network]
DHCP=yes

View File

@ -0,0 +1,3 @@
[Service]
ExecStart=
ExecStart=-/sbin/agetty -o '-p -f -- \\u' --noclear --keep-baud --autologin user - 115200,38400,9600 $TERM

View File

@ -0,0 +1,4 @@
[Service]
Type=simple
ExecStart=
ExecStart=-/sbin/agetty --autologin user --noclear %I 38400 linux

View File

@ -0,0 +1 @@
APT::Update::Post-Invoke-Success {"touch /var/lib/apt/periodic/update-success-stamp 2>/dev/null || true";};

View File

@ -0,0 +1,4 @@
# Work around issues with the APT downloader corrupting files and
# causing "Hash Sum Mismatch" errors:
# https://phabricator.collabora.com/T15071
Acquire::http::Pipeline-Depth 0;

View File

@ -0,0 +1,45 @@
# If you change this file, run 'update-grub' afterwards to update
# /boot/grub/grub.cfg.
# For full documentation of the options in this file, see:
# info -f grub -n 'Simple configuration'
GRUB_DEFAULT=0
GRUB_TIMEOUT=5
GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian`
GRUB_CMDLINE_LINUX_DEFAULT="quiet"
GRUB_CMDLINE_LINUX=""
# Kernel console on both serial and kvm/local console
GRUB_CMDLINE_LINUX="console=tty0 console=ttyS0,115200n8 rootdelay=60"
# Show grub menu on both serial and kvm/local console
GRUB_TERMINAL="console serial"
GRUB_SERIAL_COMMAND="serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1"
# If your computer has multiple operating systems installed, then you
# probably want to run os-prober. However, if your computer is a host
# for guest OSes installed via LVM or raw disk devices, running
# os-prober can cause damage to those guest OSes as it mounts
# filesystems to look for things.
#GRUB_DISABLE_OS_PROBER=false
# Uncomment to enable BadRAM filtering, modify to suit your needs
# This works with Linux (no patch required) and with any kernel that obtains
# the memory map information from GRUB (GNU Mach, kernel of FreeBSD ...)
#GRUB_BADRAM="0x01234567,0xfefefefe,0x89abcdef,0xefefefef"
# Uncomment to disable graphical terminal
#GRUB_TERMINAL=console
# The resolution used on graphical terminal
# note that you can use only modes which your graphic card supports via VBE
# you can see them in real GRUB with the command `vbeinfo'
#GRUB_GFXMODE=640x480
# Uncomment if you don't want GRUB to pass "root=UUID=xxx" parameter to Linux
#GRUB_DISABLE_LINUX_UUID=true
# Uncomment to disable generation of recovery mode menu entries
#GRUB_DISABLE_RECOVERY="true"
# Uncomment to get a beep at grub start
#GRUB_INIT_TUNE="480 440 1"

View File

@ -0,0 +1 @@
force-unsafe-io

View File

@ -0,0 +1,9 @@
# Exclude all kind of docs
path-exclude=/usr/share/doc/*
path-include=/usr/share/doc/*/copyright_report*
path-include=/usr/share/doc/*/*.json*
path-exclude=/usr/share/man/*
path-exclude=/usr/share/info/*
path-exclude=/usr/share/lintian/*
path-exclude=/usr/share/linda/*
path-exclude=/var/cache/man/*

View File

@ -0,0 +1 @@
localhost

View File

@ -0,0 +1,2 @@
[global]
break-system-packages = true

View File

@ -0,0 +1,5 @@
[Match]
Name=en*
[Network]
DHCP=yes

View File

@ -0,0 +1,6 @@
[Match]
Name=e*
KernelCommandLine=!nfsroot
[Network]
DHCP=yes

View File

@ -0,0 +1,3 @@
[Service]
ExecStart=
ExecStart=-/sbin/agetty -o '-p -f -- \\u' --noclear --keep-baud --autologin user - 115200,38400,9600 $TERM

View File

@ -0,0 +1,4 @@
[Service]
Type=simple
ExecStart=
ExecStart=-/sbin/agetty --autologin user --noclear %I 38400 linux

View File

@ -0,0 +1,23 @@
#!/bin/bash
if [ -f /var/.firstboot ]; then
exit 0
fi
logger "Setting up admin user"
adduser --gecos User user
adduser user sudo
echo "user:user" | chpasswd
sed -i "s/\#PermitRootLogin prohibit-password/\#PermitRootLogin prohibit-password\nPermitRootLogin Yes\n/" /etc/ssh/sshd_config
systemctl restart ssh.service
grub-install --uefi-secure-boot --target="x86_64-efi" --no-nvram --removable
grub-install --uefi-secure-boot --target="x86_64-efi" --no-nvram
update-grub
cp -rp /boot/ostree/* /ostree/
touch /var/.firstboot
shutdown -r now

View File

@ -0,0 +1,26 @@
#!/bin/bash
if [ -f /var/.firstboot ]; then
exit 0
fi
logger "Setting up admin user"
adduser --gecos User user
adduser user sudo
echo "user:user" | chpasswd
mkdir -p /var/lib/apt/lists
apt-get update
sed -i "s/\#PermitRootLogin prohibit-password/\#PermitRootLogin prohibit-password\nPermitRootLogin Yes\n/" /etc/ssh/sshd_config
systemctl restart ssh.service
grub-install --uefi-secure-boot --target="x86_64-efi" --no-nvram --removable
grub-install --uefi-secure-boot --target="x86_64-efi" --no-nvram
update-grub
cp -rp /boot/ostree/* /ostree/
touch /var/.firstboot
shutdown -r now

View File

@ -0,0 +1,7 @@
[Match]
Name=e*
KernelCommandLine=!nfsroot
[Network]
DHCP=yes

View File

@ -6,7 +6,7 @@ FROM debian:bullseye
RUN apt-get update && \
apt-get install -y \
bdebstrap \
mmdebstrap \
debos \
ostree \
python3 \