Merge "StarlingX: Tar Compressed files repository"
This commit is contained in:
Zuul
Gerrit Code Review
commit
f240ab8b52
@ -105,6 +105,11 @@ if [ $? == 0 ];then
|
||||
echo "step #3: done successfully"
|
||||
fi
|
||||
|
||||
# StarlingX requires a group of source code pakages, in this section
|
||||
# they will be downloaded.
|
||||
echo "step #4: start downloading tarball compressed files"
|
||||
./tarball-dl.sh
|
||||
|
||||
echo "IMPORTANT: The following 3 files are just bootstrap versions. Based"
|
||||
echo "on them, the workable images for StarlingX could be generated by"
|
||||
echo "running \"update-pxe-network-installer\" command after \"build-iso\""
|
||||
|
||||
1251
centos-mirror-tools/mvn-artifacts.lst
Normal file
1251
centos-mirror-tools/mvn-artifacts.lst
Normal file
File diff suppressed because it is too large
Load Diff
44
centos-mirror-tools/tarball-dl.lst
Normal file
44
centos-mirror-tools/tarball-dl.lst
Normal file
@ -0,0 +1,44 @@
|
||||
dpdk-17.05.2.tar.gz#dpdk-stable-17.05.2#https://fast.dpdk.org/rel/dpdk-17.05.2.tar.gz
|
||||
dpkg_1.18.24.tar.xz#dpkg-1.18.24#http://http.debian.net/debian/pool/main/d/dpkg/dpkg_1.18.24.tar.xz
|
||||
drbd-8.4.3.tar.gz#drbd-8.4.3#http://www.linbit.com/downloads/drbd/8.4/archive/drbd-8.4.3.tar.gz
|
||||
drbd-8.4.7-1.tar.gz#drbd-8.4.7-1#http://www.linbit.com/downloads/drbd/8.4/drbd-8.4.7-1.tar.gz
|
||||
dtc-1.4.4.tar.gz#dtc-1.4.4#https://www.kernel.org/pub/software/utils/dtc/dtc-1.4.4.tar.gz
|
||||
e1000e-3.3.6.tar.gz#e1000e-3.3.6#https://sourceforge.net/projects/e1000/files/e1000e%20stable/3.3.6/e1000e-3.3.6.tar.gz
|
||||
expect-lite_4.9.0.tar.gz#expect-lite.proj#https://sourceforge.net/projects/expect-lite/files/expect-lite/expect-lite_4.9.0/expect-lite_4.9.0.tar.gz
|
||||
gnulib-ffc927e.tar.gz#gnulib-ffc927e#http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-ffc927eef29016a5219cd969daad8928af6a1f4d.tar.gz
|
||||
i40e-2.4.3.tar.gz#i40e-2.4.3#https://sourceforge.net/projects/e1000/files/i40e%20stable/2.4.3/i40e-2.4.3.tar.gz/download
|
||||
i40evf-3.4.2.tar.gz#i40evf-3.4.2#https://sourceforge.net/projects/e1000/files/i40evf%20stable/3.4.2/i40evf-3.4.2.tar.gz/download
|
||||
ibsh-0.3e.tar.gz#cgcs-users-1.0#https://sourceforge.net/projects/ibsh/files/ibsh/ibsh-0.3e/ibsh-0.3e.tar.gz/download
|
||||
!integrity-kmod-668a8270.tar.gz#integrity#http://git.infradead.org/users/jjs/linux-tpmdd.git/snapshot/fd5c78694f3f1c875e293de7a641ba8a3d60d00d.tar.gz
|
||||
ixgbe-5.2.3.tar.gz#ixgbe-5.2.3#https://sourceforge.net/projects/e1000/files/ixgbe%20stable/5.2.3/ixgbe-5.2.3.tar.gz/download
|
||||
ixgbevf-4.2.1.tar.gz#ixgbevf-4.2.1#https://sourceforge.net/projects/e1000/files/ixgbevf%20stable/4.2.1/ixgbevf-4.2.1.tar.gz/download
|
||||
keycodemapdb-8370ba8.tar.gz#keycodemapdb#https://github.com/CendioOssman/keycodemapdb/tarball/8370ba868ef76f816842ba934a2e57d43c882a00
|
||||
kvm-unit-tests.git-4ea7633.tar.bz2#kvm-unit-tests#https://git.kernel.org/pub/scm/virt/kvm/kvm-unit-tests.git/snapshot/kvm-unit-tests-20171020.tar.gz
|
||||
ldapscripts-2.0.8.tgz#ldapscripts-2.0.8#https://sourceforge.net/projects/ldapscripts/files/ldapscripts/ldapscripts-2.0.8/ldapscripts-2.0.8.tgz/download
|
||||
libtpms-0.6.0-4f0d59d.tar.gz#libtpms-0.6.0#https://github.com/stefanberger/libtpms/tarball/c421ca0f4d00c0caceeda8d62c1efb2b7e47db04
|
||||
lldpd-0.9.0.tar.gz#lldpd-0.9.0#https://media.luffy.cx/files/lldpd/lldpd-0.9.0.tar.gz
|
||||
!mariadb-10.1.28.tar.gz#mariadb-10.1.28#https://github.com/MariaDB/server/archive/mariadb-10.1.28.tar.gz
|
||||
!MLNX_OFED_LINUX-4.2-1.2.0.0-rhel7.4-x86_64.tgz#MLNX_OFED_LINUX-4.2-1.2.0.0-rhel7.4-x86_64#http://www.mellanox.com/downloads/ofed/MLNX_OFED-4.2-1.2.0.0/MLNX_OFED_LINUX-4.2-1.2.0.0-rhel7.4-x86_64.tgz
|
||||
!MLNX_OFED_LINUX-4.3-1.0.1.0-rhel7.4-x86_64.tgz#MLNX_OFED_LINUX-4.3-1.0.1.0-rhel7.4-x86_64#http://www.mellanox.com/downloads/ofed/MLNX_OFED-4.3-1.0.1.0/MLNX_OFED_LINUX-4.3-1.0.1.0-rhel7.4-x86_64.tgz
|
||||
!mvn.repo.tgz#mvn#https://repo.maven.apache.org/maven2
|
||||
openvswitch-2.8.1.tar.gz#openvswitch-2.8.1#http://openvswitch.org/releases/openvswitch-2.8.1.tar.gz
|
||||
python-cephclient-v0.1.0.5.tar.gz#python-cephclient-0.1.0.5#https://github.com/dmsimard/python-cephclient/archive/v0.1.0.5.tar.gz
|
||||
python-setuptools-v38.5.1.tar.gz#setuptools-38.5.1#https://github.com/pypa/setuptools/archive/v38.5.1.tar.gz
|
||||
python-smartpm-1.4.1.tar.gz#smart-1.4.1#http://launchpad.net/smart/trunk/1.4.1/+download/smart-1.4.1.tar.bz2
|
||||
!qat1.7.upstream.l.1.0.3-42.tar.gz#quickassist#https://01.org/sites/default/files/downloads/intelr-quickassist-technology/qat1.7.upstream.l.1.0.3-42.tar.gz
|
||||
requests-toolbelt-0.5.1.tar.gz#requests-toolbelt-0.5.1#https://github.com/requests/toolbelt/archive/0.5.1.tar.gz
|
||||
rpm-4.14.0.tar.bz2#rpm-4.14.0#https://ftp.osuosl.org/pub/rpm/releases/rpm-4.14.x/rpm-4.14.0.tar.bz2
|
||||
swtpm-0.1.0-253eac5.tar.gz#swtpm-0.1.0#https://github.com/stefanberger/swtpm/tarball/1303be7d03294fb02204cb8242f366cbf0da076d
|
||||
!tpm-kmod-668a8270.tar.gz#tpm#http://git.infradead.org/users/jjs/linux-tpmdd.git/snapshot/fd5c78694f3f1c875e293de7a641ba8a3d60d00d.tar.gz
|
||||
!tss2-930.tar.gz#tss2-930#https://sourceforge.net/code-snapshots/git/i/ib/ibmtpm20tss/tss.git/ibmtpm20tss-tss-52539cb81c811c973b26ed23fafd28a700b7cc78.zip
|
||||
puppet-boolean-22b726dd78b0a60a224cc7054aebbf28e9306f62.tar.gz#puppet-boolean#https://github.com/voxpupuli/puppet-boolean/tarball/22b726dd78b0a60a224cc7054aebbf28e9306f62
|
||||
puppet-dnsmasq-cff07e90890662972c97684a2baee964f68ff3ed.tar.gz#packstack/puppet/modules/dnsmasq/#https://github.com/procore/puppet-dnsmasq/tarball/cff07e90890662972c97684a2baee964f68ff3ed
|
||||
puppet-filemapper-9b53310278e76827bbe12a36cc6470d77071abb2.tar.gz#packstack/puppet/modules/filemapper#https://github.com/voxpupuli/puppet-filemapper/tarball/9b53310278e76827bbe12a36cc6470d77071abb2
|
||||
puppetlabs-create_resources-4639819a7f3a4fa9310d2ba583c63e467df7e2c3.tar.gz#packstack/puppet/modules/create_resources#https://github.com/puppetlabs/puppetlabs-create_resources/tarball/4639819a7f3a4fa9310d2ba583c63e467df7e2c3
|
||||
puppetlabs-drbd-496b3ba9cd74a2d12636f9e90a718739a5451169.tar.gz#puppetlabs-drbd#https://github.com/voxpupuli/puppet-drbd/tarball/496b3ba9cd74a2d12636f9e90a718739a5451169
|
||||
puppetlabs-lvm-d0283da637ae24550fb4ba109a48ef8d5d8c8b84.tar.gz#packstack/puppet/modules/lvm#https://github.com/puppetlabs/puppetlabs-lvm/tarball/d0283da637ae24550fb4ba109a48ef8d5d8c8b84
|
||||
puppetlabs-postgresql-d022a56b28b2174456fc0f6adc51a4b54493afad.tar.gz#puppetlabs-postgresql#https://github.com/puppetlabs/puppetlabs-postgresql/tarball/d022a56b28b2174456fc0f6adc51a4b54493afad
|
||||
puppet-ldap-480f13af6d17d1d3fcf0dc7b4bd04b49fa4099e1.tar.gz#puppet-ldap-master#https://github.com/torian/puppet-ldap/tarball/480f13af6d17d1d3fcf0dc7b4bd04b49fa4099e1
|
||||
puppet-network-7deacd5fdc22c0543455878a8d1872f2f5417c1d.tar.gz#packstack/puppet/modules/network#https://github.com/voxpupuli/puppet-network/tarball/7deacd5fdc22c0543455878a8d1872f2f5417c1d
|
||||
puppet-nslcd-b8c19b1ada89865f2e50758e054583798ad8011a.tar.gz#packstack/puppet/modules/nslcd#https://github.com/jlyheden/puppet-nslcd/tarball/b8c19b1ada89865f2e50758e054583798ad8011a
|
||||
puppi-c1c47f4edfd761d1bbde32a75da0c3fa7cc93a81.tar.gz#puppi-master#https://github.com/example42/puppi/tarball/c1c47f4edfd761d1bbde32a75da0c3fa7cc93a81
|
||||
195
centos-mirror-tools/tarball-dl.sh
Executable file
195
centos-mirror-tools/tarball-dl.sh
Executable file
@ -0,0 +1,195 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# The build of StarlingX relies, besides RPM Binaries and Sources, in this
|
||||
# repository which is a collection of packages in the form of Tar Compressed
|
||||
# files and 3 RPMs obtained from a Tar Compressed file. This script and a text
|
||||
# file containing a list of packages enable their download and the creation
|
||||
# of the repository based in common and specific requirements dictated
|
||||
# by the StarlingX building system recipes.
|
||||
|
||||
# input files:
|
||||
# The files tarball-dl.lst and mvn-artifacts.lst contain the list of packages
|
||||
# and artifacts for building this sub-mirror.
|
||||
script_path="$(dirname $(readlink -f $0))"
|
||||
tarball_file="$script_path/tarball-dl.lst"
|
||||
mvn_artf_file="$script_path/mvn-artifacts.lst"
|
||||
|
||||
if [ ! -e $tarball_file -o ! -e $mvn_artf_file ];then
|
||||
echo "$download_list does not exist, please have a check!"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
# The 2 categories we can divide the list of packages in the output directory:
|
||||
# - General hosted under "downloads" output directory.
|
||||
# - Puppet hosted under "downloads/puppet" output directory.
|
||||
# to be populated under $MY_REPO/addons/wr-cgcs/layers/cgcs/downloads/puppet
|
||||
|
||||
output_main="output"
|
||||
output_path=$output_main/stx-r1/CentOS/pike
|
||||
output_tarball=$output_path/downloads
|
||||
output_puppet=$output_tarball/puppet
|
||||
|
||||
mkdir -p $output_tarball
|
||||
mkdir -p $output_puppet
|
||||
|
||||
# Download function using wget command
|
||||
|
||||
download_package() {
|
||||
wget --spider $1
|
||||
if [[ $? != 0 ]]; then
|
||||
echo "$1 is broken"
|
||||
else
|
||||
wget -t 5 --wait=15 $1
|
||||
fi
|
||||
}
|
||||
|
||||
# This script will iterate over the tarball.lst text file and execute specific
|
||||
# tasks based on the name of the package:
|
||||
|
||||
for line in $(cat $tarball_file); do
|
||||
|
||||
# A line from the text file starting with "#" character is ignored
|
||||
|
||||
if [[ "$line" =~ ^'#' ]]; then
|
||||
echo "Skip $line"
|
||||
continue
|
||||
fi
|
||||
|
||||
# The text file contains 3 columns separated by a character "#"
|
||||
# - Column 1, name of package including extensions as it is referenced
|
||||
# by the build system recipe, character "!" at the beginning of the name package
|
||||
# denotes special handling is required tarball_name=`echo $line | cut -d"#" -f1-1`
|
||||
# - Column 2, name of the directory path after it is decompressed as it is
|
||||
# referenced in the build system recipe.
|
||||
# - Column 3, the URL for the package download
|
||||
|
||||
tarball_name=$(echo $line | cut -d"#" -f1-1)
|
||||
directory_name=$(echo $line | cut -d"#" -f2-2)
|
||||
tarball_url=$(echo $line | cut -d"#" -f3-3)
|
||||
|
||||
# We have 6 packages from the text file starting with the character "!":
|
||||
# they require special handling besides the common process: remove directory,
|
||||
# remove text from some files, clone a git repository, etc.
|
||||
|
||||
if [[ "$line" =~ ^'!' ]]; then
|
||||
tarball_name="${tarball_name//!/}"
|
||||
echo $tarball_name
|
||||
pushd $output_tarball
|
||||
if [ "$tarball_name" = "integrity-kmod-668a8270.tar.gz" ]; then
|
||||
download_package $tarball_url
|
||||
tar xf fd5c78694f3f1c875e293de7a641ba8a3d60d00d.tar.gz
|
||||
mv linux-tpmdd-fd5c786/security/integrity/ $directory_name
|
||||
tar czvf $tarball_name $directory_name
|
||||
rm -rf linux-tpmdd-fd5c786
|
||||
rm fd5c78694f3f1c875e293de7a641ba8a3d60d00d.tar.gz
|
||||
elif [ "$tarball_name" = "mariadb-10.1.28.tar.gz" ]; then
|
||||
download_package $tarball_url
|
||||
mkdir $directory_name
|
||||
tar xf $tarball_name --strip-components 1 -C $directory_name
|
||||
rm $tarball_name
|
||||
pushd $directory_name
|
||||
rm -rf storage/tokudb
|
||||
rm ./man/tokuft_logdump.1 ./man/tokuftdump.1
|
||||
sed -e s/tokuft_logdump.1//g -i man/CMakeLists.txt
|
||||
sed -e s/tokuftdump.1//g -i man/CMakeLists.txt
|
||||
popd
|
||||
tar czvf $tarball_name $directory_name
|
||||
rm -rf $directory_name
|
||||
# The mvn.repo.tgz tarball will be created downloading a serie of
|
||||
# of maven artifacts described in mvn-artifacts file.
|
||||
elif [ "$tarball_name" = "mvn.repo.tgz" ]; then
|
||||
mkdir -p "$directory_name"
|
||||
if [ ! -f "$mvn_artf_file" ]; then
|
||||
echo "$mvn_artf_file no found" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
while read -r artf; do
|
||||
echo "download: $(basename $artf)"
|
||||
wget "$tarball_url/$artf" -P "$directory_name/$(dirname $artf)"
|
||||
done < "$mvn_artf_file"
|
||||
|
||||
# Create tarball
|
||||
tar -zcvf "$tarball_name" -C "$directory_name"/ .
|
||||
rm -rf "$directory_name"
|
||||
mv "$tarball_name" "$download_path"
|
||||
elif [[ "$tarball_name" =~ ^'MLNX_OFED_LINUX' ]]; then
|
||||
pkg_version=$(echo "$tarball_name" | cut -d "-" -f2-3)
|
||||
srpm_path="MLNX_OFED_SRC-${pkg_version}/SRPMS/"
|
||||
download_package "$tarball_url"
|
||||
tar -xf "$tarball_name"
|
||||
tar -xf "$directory_name/src/MLNX_OFED_SRC-${pkg_version}.tgz"
|
||||
# This section of code gets specific SRPMs versions according
|
||||
# to the OFED tarbal version,
|
||||
if [ "$pkg_version" = "4.2-1.2.0.0" ]; then
|
||||
cp "$srpm_path/libibverbs-41mlnx1-OFED.4.2.1.0.6.42120.src.rpm" .
|
||||
elif [ "$pkg_version" = "4.3-1.0.1.0" ]; then
|
||||
cp "$srpm_path/mlnx-ofa_kernel-4.3-OFED.4.3.1.0.1.1.g8509e41.src.rpm" .
|
||||
cp "$srpm_path/rdma-core-43mlnx1-1.43101.src.rpm" .
|
||||
else
|
||||
echo "$pkg_version : unknown version"
|
||||
fi
|
||||
rm -f "$tarball_name"
|
||||
rm -rf "MLNX_OFED_SRC-${pkg_version}"
|
||||
rm -rf "$directory_name"
|
||||
elif [ "$tarball_name" = "qat1.7.upstream.l.1.0.3-42.tar.gz" ]; then
|
||||
download_package $tarball_url
|
||||
elif [ "$tarball_name" = "tpm-kmod-668a8270.tar.gz" ]; then
|
||||
download_package $tarball_url
|
||||
tar xf fd5c78694f3f1c875e293de7a641ba8a3d60d00d.tar.gz
|
||||
mv linux-tpmdd-fd5c786/drivers/char/tpm $directory_name
|
||||
tar czvf $tarball_name $directory_name
|
||||
rm -rf linux-tpmdd-fd5c786
|
||||
rm -rf $directory_name
|
||||
rm fd5c78694f3f1c875e293de7a641ba8a3d60d00d.tar.gz
|
||||
elif [ "$tarball_name" = "tss2-930.tar.gz" ]; then
|
||||
git clone https://git.code.sf.net/p/ibmtpm20tss/tss ibmtpm20tss-tss
|
||||
pushd ibmtpm20tss-tss
|
||||
git checkout v930
|
||||
rm -rf .git
|
||||
popd
|
||||
mv ibmtpm20tss-tss $directory_name
|
||||
tar czvf $tarball_name $directory_name
|
||||
rm -rf $directory_name
|
||||
fi
|
||||
popd
|
||||
continue
|
||||
fi
|
||||
|
||||
# - For the General category and the Puppet category:
|
||||
# - Packages have a common process: download, decompressed,
|
||||
# change the directory path and compressed.
|
||||
|
||||
if [[ "$line" =~ ^pupp* ]]; then
|
||||
download_path=$output_puppet/$tarball_name
|
||||
download_directory=$output_puppet
|
||||
else
|
||||
download_path=$output_tarball/$tarball_name
|
||||
download_directory=$output_tarball
|
||||
fi
|
||||
|
||||
download_cmd="wget -t 5 --wait=15 $tarball_url -O $download_path"
|
||||
|
||||
if [ ! -e $download_path ]; then
|
||||
if $download_cmd ; then
|
||||
echo "Ok: $download_path"
|
||||
pushd $download_directory
|
||||
directory_name_original=$(tar -tf $tarball_name | head -1 | cut -f1 -d"/")
|
||||
if [ "$directory_name" != "$directory_name_original" ]; then
|
||||
mkdir -p $directory_name
|
||||
tar xf $tarball_name --strip-components 1 -C $directory_name
|
||||
tar -czf $tarball_name $directory_name
|
||||
rm -r $directory_name
|
||||
fi
|
||||
popd
|
||||
else
|
||||
echo "Error: Failed to download $tarball_url"
|
||||
fi
|
||||
|
||||
else
|
||||
echo "Already have $download_path"
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
# End of file
|
||||
|
||||
Loading…
Reference in New Issue
Block a user