Image building tools for OpenStack
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

common-functions 17KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524
  1. #!/bin/bash
  2. # Copyright 2012 Hewlett-Packard Development Company, L.P.
  3. # All Rights Reserved.
  4. #
  5. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  6. # not use this file except in compliance with the License. You may obtain
  7. # a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  13. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  14. # License for the specific language governing permissions and limitations
  15. # under the License.
  16. # This is the "internal" verison of dib-run-parts. If you modify
  17. # this, be aware that it needs to run both inside and outside the
  18. # chroot environment, so it needs to be very generic.
  19. DIB_RUN_PARTS=${_LIB}/dib-run-parts
  20. function tmpfs_check() {
  21. local echo_message=${1:-1}
  22. [ "$DIB_NO_TMPFS" == "0" ] || return 1
  23. [ -r /proc/meminfo ] || return 1
  24. total_kB=$(awk '/^MemTotal/ { print $2 }' /proc/meminfo)
  25. # tmpfs uses by default 50% of the available RAM, so the RAM should be at least
  26. # the double of the minimum tmpfs size required
  27. RAM_NEEDED=$(($DIB_MIN_TMPFS * 2))
  28. [ $total_kB -lt $(($RAM_NEEDED*1024*1024)) ] || return 0
  29. if [ $echo_message == '1' ]; then
  30. echo "WARNING: Not enough RAM to use tmpfs for build. Using ${TMP_DIR:-/tmp}. ($total_kB < ${RAM_NEEDED}G)"
  31. fi
  32. return 1
  33. }
  34. function mk_build_dir () {
  35. TMP_BUILD_DIR=$(mktemp -t -d --tmpdir=${TMP_DIR:-/tmp} dib_build.XXXXXXXX)
  36. TMP_IMAGE_DIR=$(mktemp -t -d --tmpdir=${TMP_DIR:-/tmp} dib_image.XXXXXXXX)
  37. [ $? -eq 0 ] || die "Failed to create tmp directory"
  38. export TMP_BUILD_DIR
  39. if tmpfs_check ; then
  40. sudo mount -t tmpfs tmpfs $TMP_BUILD_DIR
  41. sudo mount -t tmpfs tmpfs $TMP_IMAGE_DIR
  42. sudo chown $(id -u):$(id -g) $TMP_BUILD_DIR $TMP_IMAGE_DIR
  43. fi
  44. trap trap_cleanup EXIT
  45. echo Building in $TMP_BUILD_DIR
  46. export TMP_IMAGE_DIR
  47. export OUT_IMAGE_PATH=$TMP_IMAGE_PATH
  48. export TMP_HOOKS_PATH=$TMP_BUILD_DIR/hooks
  49. }
  50. function finish_image () {
  51. if [ -f $1 -a ${OVERWRITE_OLD_IMAGE:-0} -eq 0 ]; then
  52. old_image="${1%.*}"-$(date +%Y.%m.%d-%H.%M.%S).${1##*.}
  53. echo "Old image found. Renaming it to $old_image"
  54. mv "$1" "$old_image"
  55. if [ -f "$1.md5" ]; then
  56. mv "$1.md5" "$old_image.md5"
  57. fi
  58. if [ -f "$1.sha256" ]; then
  59. mv "$1.sha256" "$old_image.sha256"
  60. fi
  61. fi
  62. mv $OUT_IMAGE_PATH $1
  63. if [ "$DIB_CHECKSUM" == "1" ]; then
  64. # NOTE(pabelanger): Read image into memory once and generate both checksum
  65. # files.
  66. md5sum $1 > $1.md5 & sha256sum $1 > $1.sha256 & wait
  67. fi
  68. echo "Image file $1 created..."
  69. }
  70. function save_image () {
  71. finish_image $1
  72. }
  73. function copy_hooks_not_overwrite () {
  74. _DIR=$(basename $1)
  75. test -d $TMP_HOOKS_PATH/$_DIR || mkdir $TMP_HOOKS_PATH/$_DIR
  76. for _HOOK in $(ls $1); do
  77. if [ ! -f $TMP_HOOKS_PATH/$_DIR/$_HOOK ]; then
  78. echo "Copying hooks $1/$_HOOK"
  79. cp -t $TMP_HOOKS_PATH/$_DIR -a $1/$_HOOK
  80. else
  81. echo "There is a duplicated hook in your elements: $_ELEMENT/$_DIR/$_HOOK"
  82. exit 1
  83. fi
  84. done
  85. }
  86. function generate_hooks () {
  87. local xtrace
  88. xtrace=$(set +o | grep xtrace)
  89. set +o xtrace
  90. local dir
  91. local file
  92. eval declare -A image_elements=($(get_image_element_array))
  93. mkdir -p $TMP_HOOKS_PATH
  94. for i in "${!image_elements[@]}"; do
  95. local element=$i
  96. local element_dir=${image_elements[$i]}
  97. echo "Copying hooks for ${element}"
  98. for dir in $(find $element_dir \
  99. -follow -mindepth 1 -maxdepth 1 \
  100. -type d \
  101. -not -name tests \
  102. -not -name __pycache__); do
  103. copy_hooks_not_overwrite $dir
  104. done
  105. for file in $(find $element_dir \
  106. -follow -maxdepth 1 \
  107. -type f \
  108. -not -name '*.pyc'); do
  109. cp -t $TMP_HOOKS_PATH -a $file
  110. done
  111. done
  112. $xtrace
  113. }
  114. # Call the supplied break-in routine if the named point is listed in the break
  115. # list.
  116. # $1 the break point.
  117. # $2.. what to call if a break is needed
  118. function check_break () {
  119. if echo "${break:-}" | egrep -e "(,|^)$1(,|$)" -q; then
  120. echo "Starting debug shell. Exit to resume building." >&2
  121. echo At stage $1 >&2
  122. shift
  123. "$@"
  124. echo "Resuming" >&2
  125. fi
  126. }
  127. # Check that a real element has been chosen (prevents foot-guns)
  128. function check_element () {
  129. [ -d $TMP_HOOKS_PATH ] || generate_hooks
  130. }
  131. # Run a hook, looking for a regex in its stdout, and eval the matched lines.
  132. # $1 is the hook to run
  133. # $2 is the regex to look for
  134. function eval_run_d () {
  135. local run_output=$(mktemp)
  136. trap "rm -f $run_output; check_break after-error ${break_cmd:-bash}" ERR
  137. run_d $1 $run_output
  138. if grep -q "$2" $run_output; then
  139. local temp=$(grep "$2" $run_output)
  140. eval "$temp"
  141. fi
  142. rm $run_output
  143. trap - ERR
  144. }
  145. # Get any process that appears to be running in $TMP_BUILD_DIR
  146. function _get_chroot_processes () {
  147. # Deselect kernel threads, and use a python script to avoid
  148. # forking lots and lots of readlink / grep processes on a busy
  149. # system.
  150. ps --ppid 2 -p 2 --deselect -o pid= | xargs ${DIB_PYTHON_EXEC:-python} -c '
  151. import os
  152. import sys
  153. for pid in sys.argv[2:]:
  154. try:
  155. root = os.readlink("/proc/%s/root" % pid)
  156. except:
  157. continue
  158. if sys.argv[1] in root:
  159. print("%s" % pid)
  160. ' $TMP_BUILD_DIR
  161. }
  162. function kill_chroot_processes () {
  163. local xtrace
  164. xtrace=$(set +o | grep xtrace)
  165. set +o xtrace
  166. local pidname
  167. if [ -z "${1}" ]; then
  168. echo "ERROR: no chroot directory specified"
  169. exit 1
  170. fi
  171. for pid in $(_get_chroot_processes); do
  172. # If there are open files from the chroot, just kill the process using
  173. # these files. This is racy, but good enough
  174. pidname=$(cat $piddir/comm 2>/dev/null || echo "unknown")
  175. echo "Killing chroot process: '${pidname}($pid)'"
  176. sudo kill $pid
  177. done
  178. $xtrace
  179. }
  180. function cleanup_build_dir () {
  181. if ! timeout 10 sh -c " while ! sudo rm -rf $TMP_BUILD_DIR/built; do sleep 1; done"; then
  182. echo "ERROR: unable to cleanly remove $TMP_BUILD_DIR/built"
  183. exit 1
  184. fi
  185. sudo rm -rf $TMP_BUILD_DIR/mnt
  186. kill_chroot_processes $TMP_BUILD_DIR
  187. if tmpfs_check 0; then
  188. # If kill_chroot_processes did not succeed then we have to wait for
  189. # init to reap the orphaned chroot processes
  190. if ! timeout 120 sh -c "while ! sudo umount -f $TMP_BUILD_DIR; do sleep 1; done"; then
  191. echo "ERROR: failed to umount the $TMP_BUILD_DIR tmpfs mount point"
  192. exit 1
  193. fi
  194. fi
  195. rm -rf --one-file-system $TMP_BUILD_DIR
  196. }
  197. function cleanup_image_dir () {
  198. kill_chroot_processes $TMP_IMAGE_DIR
  199. if tmpfs_check 0; then
  200. if ! timeout 120 sh -c "while ! sudo umount -f $TMP_IMAGE_DIR; do sleep 1; done"; then
  201. echo "ERROR: failed to umount the $TMP_IMAGE_DIR tmpfs mount point"
  202. exit 1
  203. fi
  204. fi
  205. rm -rf --one-file-system $TMP_IMAGE_DIR
  206. }
  207. # Run a directory of hooks outside the target (that is, no chrooting).
  208. function run_d() {
  209. check_element
  210. check_break before-$1 ${break_cmd:-bash}
  211. if [ -d ${TMP_HOOKS_PATH}/$1.d ] ; then
  212. echo "Running hooks from ${TMP_HOOKS_PATH}/$1.d"
  213. if [ -n "$2" ]; then
  214. ${DIB_RUN_PARTS} ${TMP_HOOKS_PATH}/$1.d | tee $2
  215. if [[ ${PIPESTATUS[0]} != 0 ]]; then
  216. return 1
  217. fi
  218. else
  219. ${DIB_RUN_PARTS} ${TMP_HOOKS_PATH}/$1.d
  220. fi
  221. fi
  222. check_break after-$1 bash
  223. }
  224. function _arg_defaults_hack() {
  225. # The block-device configuration looks in all elements for a
  226. # "block-device-default.yaml" file. The "vm" element used to
  227. # provide the default block-device, which was fine when there was
  228. # only one option; but now we have mbr, gpt & efi versions.
  229. #
  230. # So now the vm element has a dependency on the block-device
  231. # element, which several different elements can provide. However,
  232. # for backwards compatability we need to ensure you can still
  233. # build without specifying it. Thus if we see the vm element, but
  234. # no block-device-* element, we will automatically add the old
  235. # default MBR.
  236. #
  237. # Note that you can still override this by setting
  238. # DIB_BLOCK_DEVICE_CONFIG; any value there will be taken over the
  239. # element defaults. In this case you'd have "block-device-mbr" as
  240. # an element, but it wouldn't actually be used for configuration.
  241. #
  242. # XXX: if this is becoming a common problem, we could have some
  243. # sort of "element-defaults" that maps a "element-deps" entry to a
  244. # default.
  245. local vm_seen
  246. local blockdev_seen
  247. local elements
  248. for arg do
  249. if [[ "$arg" =~ "vm" ]]; then
  250. vm_seen=1
  251. elif [[ "$arg" =~ "block-device-" ]]; then
  252. blockdev_seen=1
  253. fi
  254. elements="$elements $arg"
  255. done
  256. if [[ -n "${vm_seen}" && -z "${blockdev_seen}" ]]; then
  257. elements="$elements block-device-mbr"
  258. fi
  259. echo $elements
  260. }
  261. function arg_to_elements() {
  262. for arg do IMAGE_ELEMENT="$IMAGE_ELEMENT $arg" ; done
  263. IMAGE_ELEMENT="$(_arg_defaults_hack $IMAGE_ELEMENT)"
  264. if [ "$SKIP_BASE" != "1" ]; then
  265. IMAGE_ELEMENT="base $IMAGE_ELEMENT"
  266. fi
  267. if [ "$IS_RAMDISK" == "1" ]; then
  268. IMAGE_ELEMENT="$RAMDISK_ELEMENT $IMAGE_ELEMENT"
  269. fi
  270. echo "Building elements: $IMAGE_ELEMENT"
  271. export IMAGE_ELEMENT
  272. # element-info will output bash code to create
  273. # * IMAGE_ELEMENT
  274. # legacy list of elements
  275. #
  276. # * IMAGE_ELEMENT_YAML
  277. # YAML dictionary with key=element, value=path
  278. #
  279. # import os
  280. # import yaml
  281. # yaml.load(os.getenv('IMAGE_ELEMENT_YAML')
  282. #
  283. # * function get_image_element_array
  284. # Function to create Bash associative-array. Since bash can not
  285. # export array variables, we provide a function to populate the
  286. # variables.
  287. #
  288. # # we need the eval, it expands the string for the array create
  289. # eval declare -A image_elements=($(get_image_element_array))
  290. # for i in ${!image_elements[@]}; do
  291. # element=$i
  292. # path=${image_elements[$i]
  293. # done
  294. elinfo_out="$(element-info --env $IMAGE_ELEMENT)"
  295. if [ $? -ne 0 ]; then
  296. echo "ERROR: element-info failed to expand elements."
  297. exit 1
  298. fi
  299. eval "$elinfo_out"
  300. echo "Expanded element dependencies to: $IMAGE_ELEMENT"
  301. }
  302. function create_base () {
  303. mkdir $TMP_BUILD_DIR/mnt
  304. # Make sure the / inside the chroot is owned by root
  305. # If it is not owned by root, some Ubuntu bionic packages will fail
  306. # path validation at install time.
  307. sudo chown root.root $TMP_BUILD_DIR/mnt
  308. export TMP_MOUNT_PATH=$TMP_BUILD_DIR/mnt
  309. # Copy data in to the root.
  310. TARGET_ROOT=$TMP_MOUNT_PATH run_d root
  311. if [ -z "$(ls $TMP_MOUNT_PATH | grep -v '^lost+found\|tmp$')" ] ; then
  312. # No root element copied in. Note the test above allows
  313. # root.d elements to put things in /tmp
  314. echo "Failed to deploy the root element."
  315. exit 1
  316. fi
  317. # Configure Image
  318. # Save resolv.conf as created by the initial install. Note the
  319. # .ORIG file is an exported interface -- it may be modified and we
  320. # will copy it back in during finalisation of the image.
  321. # Note that we use -L and -f to test here as test (and bash [[)
  322. # return false with -e if the link target does not exist.
  323. if [ -L $TMP_MOUNT_PATH/etc/resolv.conf ] || [ -f $TMP_MOUNT_PATH/etc/resolv.conf ] ; then
  324. sudo mv $TMP_MOUNT_PATH/etc/resolv.conf $TMP_MOUNT_PATH/etc/resolv.conf.ORIG
  325. fi
  326. # Recreate resolv.conf
  327. sudo touch $TMP_MOUNT_PATH/etc/resolv.conf
  328. sudo chmod 777 $TMP_MOUNT_PATH/etc/resolv.conf
  329. # use system configured resolv.conf if available to support internal proxy resolving
  330. if [ -e /etc/resolv.conf ]; then
  331. cat /etc/resolv.conf > $TMP_MOUNT_PATH/etc/resolv.conf
  332. else
  333. echo nameserver 8.8.8.8 > $TMP_MOUNT_PATH/etc/resolv.conf
  334. fi
  335. mount_proc_dev_sys
  336. }
  337. # Get mount options for mounting /dev/pts
  338. # Kernel commit https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=eedf265aa003b4781de24cfed40a655a664457e6
  339. # introduced in v4.7 allows multiple instances of devpts. However,
  340. # some distributions are running older kernels so we need to take
  341. # care on what options we use to mount a new instance of devpts
  342. # filesystem since it's not completely independent. The best thing
  343. # to do is to simply use the existing mount options.
  344. function mount_dev_pts_options() {
  345. echo "-o $(findmnt --first-only /dev/pts --noheadings --output OPTIONS)"
  346. }
  347. function mount_proc_dev_sys () {
  348. # supporting kernel file systems
  349. sudo mount -t proc none $TMP_MOUNT_PATH/proc
  350. sudo mount --bind /dev $TMP_MOUNT_PATH/dev
  351. sudo mount -t devpts $(mount_dev_pts_options) devpts $TMP_MOUNT_PATH/dev/pts
  352. sudo mount -t sysfs none $TMP_MOUNT_PATH/sys
  353. }
  354. # Recursively unmount directories under a given directory DIR
  355. # usage:
  356. # unmount_dir DIR
  357. function unmount_dir {
  358. local dir="$1"
  359. local real_dir
  360. local mnts
  361. local split_mounts
  362. local found_mountpoint
  363. if [ ! -d $dir ]; then
  364. echo "*** $dir is not a directory"
  365. return 0
  366. fi
  367. # get rid of any symlink elements in the incoming path, because
  368. # /proc/mounts is the real path
  369. real_dir=$(readlink -e $dir)
  370. # populate the exported mountpoints
  371. IFS='|' read -ra split_mounts <<< "$DIB_MOUNTPOINTS"
  372. # note the "/" on real_dir ... we are just looking for things
  373. # mounted *underneath* this directory.
  374. mnts=$(awk '{print $2}' < /proc/mounts | grep "^$real_dir/" | sort -r)
  375. for m in $mnts; do
  376. # check if suffix is in array
  377. found_mountpoint=false
  378. for mountpoint in "${split_mounts[@]}"; do
  379. if [[ "$mountpoint" != "/" ]]; then
  380. if [[ "$m" == *$mountpoint ]]; then
  381. echo "Mountpoint $m managed by block device; skipping"
  382. found_mountpoint=true
  383. break
  384. fi
  385. fi
  386. done
  387. if [ $found_mountpoint == false ]; then
  388. # unmount the directory as it is not managed by block device
  389. echo "Unmount $m"
  390. sudo umount -fl $m || true
  391. fi
  392. done
  393. }
  394. # Create YAML config file for the block device layer
  395. # The order here is: use the one the user provides - if there is
  396. # none provided, fall back to the possible one element which
  397. # defines a fallback configuration.
  398. # Parameters:
  399. # - name of the to be created config file
  400. function block_device_create_config_file {
  401. # nosiy; we manually trace
  402. local xtrace
  403. xtrace=$(set +o | grep xtrace)
  404. set +o xtrace
  405. local config_yaml="$1"
  406. if [[ ${DIB_BLOCK_DEVICE_CONFIG:-} == file://* ]]; then
  407. cp $(echo ${DIB_BLOCK_DEVICE_CONFIG} | cut -c 8-) ${config_yaml}
  408. echo "Using file-based block-device config: ${DIB_BLOCK_DEVICE_CONFIG}"
  409. $xtrace
  410. return
  411. fi
  412. if [ -n "${DIB_BLOCK_DEVICE_CONFIG:-}" ]; then
  413. printf "%s" "${DIB_BLOCK_DEVICE_CONFIG}" >${config_yaml}
  414. echo "User specified block-device config from DIB_BLOCK_DEVICE_CONFIG"
  415. $xtrace
  416. return
  417. fi
  418. # Search the elements for a matching block-device config.
  419. # XXX: first match wins?
  420. echo "Searching elements for block-device[-${ARCH}].yaml ..."
  421. eval declare -A image_elements=($(get_image_element_array))
  422. for i in ${!image_elements[@]}; do
  423. local cfg
  424. # look for arch specific version first, then default
  425. if [[ "ppc64le ppc64el" =~ $ARCH ]] ; then
  426. # NOTE(tonyb): ppc64el and ppc64le are the same archttechture, it's
  427. # just different distro's have different names. So if we're either
  428. # of them pick the block-device-ppc64el.yaml file
  429. cfg=${image_elements[$i]}/block-device-ppc64el.yaml
  430. else
  431. cfg=${image_elements[$i]}/block-device-${ARCH}.yaml
  432. fi
  433. if [ -e ${cfg} ]; then
  434. cp ${cfg} ${config_yaml}
  435. echo "Using block-device config: ${cfg}"
  436. $xtrace
  437. return
  438. else
  439. cfg=${image_elements[$i]}/block-device-default.yaml
  440. if [ -e ${cfg} ]; then
  441. cp ${cfg} ${config_yaml}
  442. echo "Using block-device config: ${cfg}"
  443. $xtrace
  444. return
  445. fi
  446. fi
  447. done
  448. echo "... done"
  449. # how did this get here?
  450. if [ -e ${config_yaml} ]; then
  451. die "${config_yaml} exists?"
  452. fi
  453. echo "Using default block-device fallback config"
  454. # If no config is there (until now) use the default config
  455. cat >${config_yaml} <<EOF
  456. - local_loop:
  457. name: image0
  458. mkfs:
  459. name: mkfs_root
  460. mount:
  461. mount_point: /
  462. fstab:
  463. options: "defaults"
  464. fsck-passno: 1
  465. EOF
  466. $xtrace
  467. }