StarlingX build tools
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

download_mirror.sh 14KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422
  1. #!/bin/bash -e
  2. #
  3. # SPDX-License-Identifier: Apache-2.0
  4. #
  5. usage() {
  6. echo "$0 [-n] [-c <yum.conf>] [-g] [-s|-S|-u|-U]"
  7. echo ""
  8. echo "Options:"
  9. echo " -n: Do not use sudo when performing operations (option passed on to"
  10. echo " subscripts when appropriate)"
  11. echo " -c: Use an alternate yum.conf rather than the system file (option passed"
  12. echo " on to subscripts when appropriate)"
  13. echo " -g: do not change group IDs of downloaded artifacts"
  14. echo " -s: Download from StarlingX mirror only"
  15. echo " -S: Download from StarlingX mirror, upstream as backup (default)"
  16. echo " -u: Download from original upstream sources only"
  17. echo " -U: Download from original upstream sources, StarlingX mirror as backup"
  18. echo ""
  19. }
  20. generate_log_name() {
  21. filename=$1
  22. level=$2
  23. base=$(basename $filename .lst)
  24. echo $LOGSDIR"/"$base"_download_"$level".log"
  25. }
  26. need_file(){
  27. for f in $*; do
  28. if [ ! -f $f ]; then
  29. echo "ERROR: File $f does not exist."
  30. exit 1
  31. fi
  32. done
  33. }
  34. need_dir(){
  35. for d in $*; do
  36. if [ ! -d $d ]; then
  37. echo "ERROR: Directory $d does not exist."
  38. exit 1
  39. fi
  40. done
  41. }
  42. # Downloader scripts
  43. rpm_downloader="./dl_rpms.sh"
  44. tarball_downloader="./dl_tarball.sh"
  45. other_downloader="./dl_other_from_centos_repo.sh"
  46. make_stx_mirror_yum_conf="./make_stx_mirror_yum_conf.sh"
  47. # track optional arguments
  48. change_group_ids=1
  49. use_system_yum_conf=1
  50. alternate_yum_conf=""
  51. alternate_repo_dir=""
  52. rpm_downloader_extra_args=""
  53. tarball_downloader_extra_args=""
  54. distro="centos"
  55. # lst files to use as input
  56. rpms_from_3rd_parties="./rpms_3rdparties.lst"
  57. rpms_from_centos_repo="./rpms_centos.lst"
  58. rpms_from_centos_3rd_parties="./rpms_centos3rdparties.lst"
  59. other_downloads="./other_downloads.lst"
  60. # Overall success
  61. success=1
  62. # Permitted values of dl_source
  63. dl_from_stx_mirror="stx_mirror"
  64. dl_from_upstream="upstream"
  65. dl_from_stx_then_upstream="$dl_from_stx_mirror $dl_from_upstream"
  66. dl_from_upstream_then_stx="$dl_from_upstream $dl_from_stx_mirror"
  67. # Download from what source?
  68. # dl_from_stx_mirror = StarlingX mirror only
  69. # dl_from_upstream = Original upstream source only
  70. # dl_from_stx_then_upstream = Either source, STX prefered (default)"
  71. # dl_from_upstream_then_stx = Either source, UPSTREAM prefered"
  72. dl_source="$dl_from_stx_then_upstream"
  73. dl_flag=""
  74. dl_from_stx () {
  75. local re="\\b$dl_from_stx_mirror\\b"
  76. [[ "$dl_source" =~ $re ]]
  77. }
  78. dl_from_upstream () {
  79. local re="\\b$dl_from_upstream\\b"
  80. [[ "$dl_source" =~ $re ]]
  81. }
  82. MULTIPLE_DL_FLAG_ERROR_MSG="Error: Please use only one of: -s,-S,-u,-U"
  83. multiple_dl_flag_check () {
  84. if [ "$dl_flag" != "" ]; then
  85. echo "$MULTIPLE_DL_FLAG_ERROR_MSG"
  86. usage
  87. exit 1
  88. fi
  89. }
  90. # Parse out optional arguments
  91. while getopts "c:nghsSuU" o; do
  92. case "${o}" in
  93. n)
  94. # Pass -n ("no-sudo") to rpm downloader
  95. rpm_downloader_extra_args="${rpm_downloader_extra_args} -n"
  96. ;;
  97. c)
  98. # Pass -c ("use alternate yum.conf") to rpm downloader
  99. use_system_yum_conf=0
  100. alternate_yum_conf="${OPTARG}"
  101. ;;
  102. g)
  103. # Do not attempt to change group IDs on downloaded packages
  104. change_group_ids=0
  105. ;;
  106. s)
  107. # Download from StarlingX mirror only. Do not use upstream sources.
  108. multiple_dl_flag_check
  109. dl_source="$dl_from_stx_mirror"
  110. dl_flag="-s"
  111. ;;
  112. S)
  113. # Download from StarlingX mirror only. Do not use upstream sources.
  114. multiple_dl_flag_check
  115. dl_source="$dl_from_stx_then_upstream"
  116. dl_flag="-S"
  117. ;;
  118. u)
  119. # Download from upstream only. Do not use StarlingX mirror.
  120. multiple_dl_flag_check
  121. dl_source="$dl_from_upstream"
  122. dl_flag="-u"
  123. ;;
  124. U)
  125. # Download from upstream only. Do not use StarlingX mirror.
  126. multiple_dl_flag_check
  127. dl_source="$dl_from_upstream_then_stx"
  128. dl_flag="-U"
  129. ;;
  130. h)
  131. # Help
  132. usage
  133. exit 0
  134. ;;
  135. *)
  136. usage
  137. exit 1
  138. ;;
  139. esac
  140. done
  141. shift $((OPTIND-1))
  142. echo "--------------------------------------------------------------"
  143. echo "WARNING: this script HAS TO access internet (http/https/ftp),"
  144. echo "so please make sure your network working properly!!"
  145. LOGSDIR="logs"
  146. mkdir -p $LOGSDIR
  147. # Check extistence of prerequisites files
  148. need_file ${rpm_downloader} ${other_downloader} ${tarball_downloader}
  149. need_file ${rpms_from_3rd_parties}
  150. need_file ${rpms_from_centos_3rd_parties}
  151. need_file ${rpms_from_centos_repo}
  152. need_file ${other_downloads}
  153. need_file tarball-dl.lst
  154. #download RPMs/SRPMs from 3rd_party websites (not CentOS repos) by "wget"
  155. echo "step #1: start downloading RPMs/SRPMs from 3rd-party websites..."
  156. if [ ${use_system_yum_conf} -ne 0 ]; then
  157. # Restore StarlingX_3rd repos from backup
  158. REPO_SOURCE_DIR=/localdisk/yum.repos.d
  159. REPO_DIR=/etc/yum.repos.d
  160. if [ -d $REPO_SOURCE_DIR ] && [ -d $REPO_DIR ]; then
  161. \cp -f $REPO_SOURCE_DIR/*.repo $REPO_DIR/
  162. fi
  163. fi
  164. if [ $use_system_yum_conf -eq 0 ]; then
  165. need_file "${alternate_yum_conf}"
  166. if [ "$alternate_repo_dir" == "" ]; then
  167. alternate_repo_dir=$(grep '^reposdir=' "${alternate_yum_conf}" | cut -d '=' -f 2)
  168. if [ "$alternate_repo_dir" == "" ]; then
  169. alternate_repo_dir="$(dirname "${alternate_yum_conf}"/yum.repos.d)"
  170. fi
  171. need_dir "${alternate_repo_dir}"
  172. fi
  173. fi
  174. TEMP_DIR=""
  175. rpm_downloader_extra_args="${rpm_downloader_extra_args} -D $distro"
  176. if [ "$dl_flag" != "" ]; then
  177. # Pass dl_flag on to the rpm_downloader script
  178. rpm_downloader_extra_args="${rpm_downloader_extra_args} $dl_flag"
  179. fi
  180. if ! dl_from_stx; then
  181. # Not using stx mirror
  182. if [ $use_system_yum_conf -eq 0 ]; then
  183. # Use provided yum.conf unaltered.
  184. rpm_downloader_extra_args="${rpm_downloader_extra_args} -c ${alternate_yum_conf}"
  185. fi
  186. else
  187. # We want to use stx mirror, so we need to create a new, modified yum.conf and yum.repos.d.
  188. # The modifications will add or substitute repos pointing to the StralingX mirror.
  189. TEMP_DIR=$(mktemp -d /tmp/stx_mirror_XXXXXX)
  190. TEMP_CONF="$TEMP_DIR/yum.conf"
  191. need_file ${make_stx_mirror_yum_conf}
  192. need_dir ${TEMP_DIR}
  193. if [ $use_system_yum_conf -eq 0 ]; then
  194. # Modify user provided yum.conf. We expect ir to have a 'reposdir=' entry to
  195. # point to the repos that need to be modified as well.
  196. if dl_from_upstream; then
  197. # add
  198. ${make_stx_mirror_yum_conf} -R -d $TEMP_DIR -y $alternate_yum_conf -r $alternate_repo_dir -D $distro
  199. else
  200. # substitute
  201. ${make_stx_mirror_yum_conf} -d $TEMP_DIR -y $alternate_yum_conf -r $alternate_repo_dir -D $distro
  202. fi
  203. else
  204. # Modify system yum.conf and yum.repos.d. Remember that we expect to run this
  205. # inside a container, and the system yum.conf has like been modified else where
  206. # in these scripts.
  207. if dl_from_upstream; then
  208. # add
  209. ${make_stx_mirror_yum_conf} -R -d $TEMP_DIR -y /etc/yum.conf -r /etc/yum.repos.d -D $distro
  210. else
  211. # substitute
  212. ${make_stx_mirror_yum_conf} -d $TEMP_DIR -y /etc/yum.conf -r /etc/yum.repos.d -D $distro
  213. fi
  214. fi
  215. rpm_downloader_extra_args="${rpm_downloader_extra_args} -c $TEMP_CONF"
  216. fi
  217. list=${rpms_from_3rd_parties}
  218. level=L1
  219. logfile=$(generate_log_name $list $level)
  220. $rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile
  221. retcode=${PIPESTATUS[0]}
  222. if [ $retcode -ne 0 ];then
  223. echo "ERROR: Something wrong with downloading files listed in $list."
  224. echo " Please check the log at $(pwd)/$logfile !"
  225. echo ""
  226. success=0
  227. fi
  228. # download RPMs/SRPMs from 3rd_party repos by "yumdownloader"
  229. list=${rpms_from_centos_3rd_parties}
  230. level=L1
  231. logfile=$(generate_log_name $list $level)
  232. $rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile
  233. retcode=${PIPESTATUS[0]}
  234. if [ $retcode -ne 0 ];then
  235. echo "ERROR: Something wrong with downloading files listed in $list."
  236. echo " Please check the log at $(pwd)/$logfile !"
  237. echo ""
  238. success=0
  239. fi
  240. if [ ${use_system_yum_conf} -eq 1 ]; then
  241. # deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo.
  242. \rm -f $REPO_DIR/StarlingX_3rd*.repo
  243. if [ "$TEMP_DIR" != "" ]; then
  244. \rm -f $TEMP_DIR/yum.repos.d/StarlingX_3rd*.repo
  245. fi
  246. fi
  247. echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..."
  248. #download RPMs/SRPMs from CentOS repos by "yumdownloader"
  249. list=${rpms_from_centos_repo}
  250. level=L1
  251. logfile=$(generate_log_name $list $level)
  252. $rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile
  253. retcode=${PIPESTATUS[0]}
  254. K1_logfile=$(generate_log_name ${rpms_from_centos_repo} K1)
  255. if [ $retcode -ne 1 ]; then
  256. # K1 step not needed. Clear any K1 logs from previous download attempts.
  257. $rpm_downloader -x $LOGSDIR/L1_rpms_missing_centos.log K1 |& tee $K1_logfile
  258. fi
  259. if [ $retcode -eq 0 ]; then
  260. echo "finish 1st round of RPM downloading successfully!"
  261. elif [ $retcode -eq 1 ]; then
  262. echo "finish 1st round of RPM downloading with missing files!"
  263. if [ -e "$LOGSDIR/L1_rpms_missing_centos.log" ]; then
  264. echo "start 2nd round of downloading Binary RPMs with K1 match criteria..."
  265. $rpm_downloader ${rpm_downloader_extra_args} $LOGSDIR/L1_rpms_missing_centos.log K1 centos |& tee $K1_logfile
  266. retcode=${PIPESTATUS[0]}
  267. if [ $retcode -eq 0 ]; then
  268. echo "finish 2nd round of RPM downloading successfully!"
  269. elif [ $retcode -eq 1 ]; then
  270. echo "finish 2nd round of RPM downloading with missing files!"
  271. if [ -e "$LOGSDIR/rpms_missing_K1.log" ]; then
  272. echo "WARNING: missing RPMs listed in $LOGSDIR/centos_rpms_missing_K1.log !"
  273. fi
  274. fi
  275. # Remove files found by K1 download from L1_rpms_missing_centos.txt to prevent
  276. # false reporting of missing files.
  277. grep -v -x -F -f $LOGSDIR/K1_rpms_found_centos.log $LOGSDIR/L1_rpms_missing_centos.log > $LOGSDIR/L1_rpms_missing_centos.tmp
  278. mv -f $LOGSDIR/L1_rpms_missing_centos.tmp $LOGSDIR/L1_rpms_missing_centos.log
  279. missing_num=`wc -l $LOGSDIR/K1_rpms_missing_centos.log | cut -d " " -f1-1`
  280. if [ "$missing_num" != "0" ];then
  281. echo "ERROR: -------RPMs missing: $missing_num ---------------"
  282. retcode=1
  283. fi
  284. fi
  285. if [ -e "$LOGSDIR/L1_srpms_missing_centos.log" ]; then
  286. missing_num=`wc -l $LOGSDIR/L1_srpms_missing_centos.log | cut -d " " -f1-1`
  287. if [ "$missing_num" != "0" ];then
  288. echo "ERROR: --------- SRPMs missing: $missing_num ---------------"
  289. retcode=1
  290. fi
  291. fi
  292. fi
  293. if [ $retcode -ne 0 ]; then
  294. echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_repo}."
  295. echo " Please check the logs at $(pwd)/$logfile"
  296. echo " and $(pwd)/logs/$K1_logfile !"
  297. echo ""
  298. success=0
  299. fi
  300. ## verify all RPMs SRPMs we download for the GPG keys
  301. find ./output -type f -name "*.rpm" | xargs rpm -K | grep -i "MISSING KEYS" > $LOGSDIR/rpm-gpg-key-missing.txt
  302. # remove all i686.rpms to avoid pollute the chroot dep chain
  303. find ./output -name "*.i686.rpm" | tee $LOGSDIR/all_i686.txt
  304. find ./output -name "*.i686.rpm" | xargs rm -f
  305. line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1`
  306. line2=`wc -l ${rpms_from_centos_repo} | cut -d " " -f1-1`
  307. line3=`wc -l ${rpms_from_centos_3rd_parties} | cut -d " " -f1-1`
  308. let total_line=$line1+$line2+$line3
  309. echo "We expected to download $total_line RPMs."
  310. num_of_downloaded_rpms=`find ./output -type f -name "*.rpm" | wc -l | cut -d" " -f1-1`
  311. echo "There are $num_of_downloaded_rpms RPMs in output directory."
  312. if [ "$total_line" != "$num_of_downloaded_rpms" ]; then
  313. echo "WARNING: Not the same number of RPMs in output as RPMs expected to be downloaded, need to check outputs and logs."
  314. fi
  315. if [ $change_group_ids -eq 1 ]; then
  316. # change "./output" and sub-folders to 751 (cgcs) group
  317. chown 751:751 -R ./output
  318. fi
  319. echo "step #3: start downloading other files ..."
  320. logfile=$LOGSDIR"/otherfiles_centos_download.log"
  321. ${other_downloader} ${dl_flag} -D "$distro" ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ |& tee $logfile
  322. retcode=${PIPESTATUS[0]}
  323. if [ $retcode -eq 0 ];then
  324. echo "step #3: done successfully"
  325. else
  326. echo "step #3: finished with errors"
  327. echo "ERROR: Something wrong with downloading from ${other_downloads}."
  328. echo " Please check the log at $(pwd)/$logfile!"
  329. echo ""
  330. success=0
  331. fi
  332. # StarlingX requires a group of source code pakages, in this section
  333. # they will be downloaded.
  334. echo "step #4: start downloading tarball compressed files"
  335. logfile=$LOGSDIR"/tarballs_download.log"
  336. ${tarball_downloader} ${dl_flag} -D "$distro" ${tarball_downloader_extra_args} |& tee $logfile
  337. retcode=${PIPESTATUS[0]}
  338. if [ $retcode -eq 0 ];then
  339. echo "step #4: done successfully"
  340. else
  341. echo "step #4: finished with errors"
  342. echo "ERROR: Something wrong with downloading tarballs."
  343. echo " Please check the log at $(pwd)/$logfile !"
  344. echo ""
  345. success=0
  346. fi
  347. #
  348. # Clean up the mktemp directory, if required.
  349. #
  350. if [ "$TEMP_DIR" != "" ]; then
  351. \rm -rf "$TEMP_DIR"
  352. fi
  353. echo "IMPORTANT: The following 3 files are just bootstrap versions. Based"
  354. echo "on them, the workable images for StarlingX could be generated by"
  355. echo "running \"update-pxe-network-installer\" command after \"build-iso\""
  356. echo " - out/stx-r1/CentOS/pike/Binary/LiveOS/squashfs.img"
  357. echo " - out/stx-r1/CentOS/pike/Binary/images/pxeboot/initrd.img"
  358. echo " - out/stx-r1/CentOS/pike/Binary/images/pxeboot/vmlinuz"
  359. echo ""
  360. if [ $success -ne 1 ]; then
  361. echo "Warning: Not all download steps succeeded. You are likely missing files."
  362. exit 1
  363. fi
  364. echo "Success"
  365. exit 0