From abbbf317ce5c848951091689a6dd946b449e0873 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Mon, 18 Jan 2016 17:15:37 +1100 Subject: [PATCH] Cleanup the early devstack clone script This script is unnecessarily complex for what it is doing, which is essentially pre-seeding the source-repositories cache for devstack. As we can see from I41e81d6bac98875eecde2376e0865784626e11a8 it's very confusing having large parts of the source-repositories script copy-pasted as a separate element and has led several of us down the wrong path. Strip this script back to the simple thing it is doing, which is checking out/updating the source-repositories devstack cache. I have tested this by building an image with a warm cache and with a cold cache, in both cases the checkout was found and the list of images to cache in 55-cache-devstack-repos was found. Change-Id: I6c686312de102cbe438585e26bf6986e06b6f41c --- .../extra-data.d/50-early-source-repo | 241 ++++-------------- 1 file changed, 56 insertions(+), 185 deletions(-) diff --git a/nodepool/elements/cache-devstack/extra-data.d/50-early-source-repo b/nodepool/elements/cache-devstack/extra-data.d/50-early-source-repo index a45070656c..4aab7cd0f8 100755 --- a/nodepool/elements/cache-devstack/extra-data.d/50-early-source-repo +++ b/nodepool/elements/cache-devstack/extra-data.d/50-early-source-repo @@ -16,202 +16,73 @@ # See the License for the specific language governing permissions and # limitations under the License. -# We need a copy of the devstack repo so that we can read it so that -# we can generate the list of repos that we need to get. We'd like to -# use the normal repo fetching and caching routines for that, so use -# a modified version. - -# The bulk of this code is copied from -# elements/source-repositories/extra-data.d/98-source-repositories -# Most of it should be removed when we can source and call the -# functions directly - # dib-lint: disable=setpipefail if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then set -x fi set -eu -GIT_BASE=${GIT_BASE:-git://git.openstack.org} +# We need a copy of the devstack repo to query for the list of image +# files to cached by the source-repositories element (see +# 55-cache-devstack-repos) +# +# This script is a hack to use the source-repositories repo cache to +# get early access to the devstack checkout (rather than doing a whole +# separate clone of the devstack tree). In essence, this script knows +# the path of the devstack in the source-repositories cache and +# creates/updates it. To get the gist of where this comes from, see +# dib's +# elements/source-repositories/extra-data.d/98-source-repositories +# +# This could be removed if we could ever source and call the +# clone/update functions asynchronously from the source-repositories +# dib element. -# If the old cache exists, move it to the new name -function make_new_cache(){ - local OLD_CACHE_BASE=$1 - local CACHE_BASE=$2 +function get_devstack_from_cache { - # If the old cache name exists, move it to the new cache name - if [ -e "$OLD_CACHE_BASE" ] ; then - if [ ! -e "$CACHE_BASE" ] ; then - mv -n $OLD_CACHE_BASE $CACHE_BASE - else - echo "Not replacing new cache location with old cache" - fi + local REPONAME=devstack + local REPOTYPE=git + local REPOPATH=/opt/git/openstack-dev/devstack + local REPOLOCATION=git://git.openstack.org/openstack-dev/devstack.git + local REPOREF=master + + local REPO_DEST=$TMP_MOUNT_PATH$REPOPATH + local REPO_SUB_DIRECTORY=$(dirname $REPO_DEST) + sudo mkdir -p $REPO_SUB_DIRECTORY + + # build the same cache name as source-repositories + CACHE_NAME=$(echo "${REPOTYPE}_${REPOLOCATION}" | sha1sum | awk '{ print $1 }' ) + CACHE_NAME=${REPONAME//[^A-Za-z0-9]/_}_${CACHE_NAME} + CACHE_PATH=${CACHE_BASE}/$CACHE_NAME + + # clone repo if not in cache + if [ ! -e "$CACHE_PATH" ] ; then + echo "early-source-repo: Caching $REPONAME from $REPOLOCATION in $CACHE_PATH" + git clone $REPOLOCATION $CACHE_PATH.tmp + mv ${CACHE_PATH}{.tmp,} fi + + # update cached repo + if [ -z "$DIB_OFFLINE" ] ; then + echo "early-source-repo: Updating cache of $REPOLOCATION in $CACHE_PATH with ref $REPOREF" + # note, we could update *all* refs here (+refs/*:refs/* and + # create a really full mirror. but that brings in a lot of + # change refs from gerrit which traditionally we haven't used. + # Also, we only really care about refs/remotes/* as the + # devstack caching script goes through these branches to + # decide what to checkout. We keep refs/heads/* for + # historical reasons...) + git --git-dir=$CACHE_PATH/.git fetch --prune --update-head-ok \ + $REPOLOCATION \ + +refs/heads/*:refs/heads/* +refs/remotes/*:refs/remotes/* + fi + + # clone the updated repo to our destination + sudo git clone $CACHE_PATH $REPO_DEST } -# Gets repositories or individual files listed in the a repository file -# and places them in the specified destination path. -# The format of the repository file is one or more lines matching -# [] -function get_repos_for_element(){ - local REPO_SOURCES=$1 - local CACHE_URL=$TMP_HOOKS_PATH/bin/cache-url - - local REGEX="^([^ ]+) (git|tar|file|package) ?(/[^ ]+)? ?([^ ]+)? ?([^ ]*)$" - - while read line; do - # expand variables - line=$(eval echo $line) - - # ignore blank lines and lines beginning in '#' - [[ "$line" == \#* ]] || [[ -z "$line" ]] && continue - - if [[ "$line" =~ $REGEX ]] ; then - local REPONAME=${BASH_REMATCH[1]} - local REPOTYPE=${BASH_REMATCH[2]} - local REPOPATH=${BASH_REMATCH[3]} - local REPOLOCATION=${BASH_REMATCH[4]} - local REPO_ORIG_LOCATION=$REPOLOCATION - local REPOREF=${BASH_REMATCH[5]:-master} - - local REPO_DEST=$TMP_MOUNT_PATH$REPOPATH - local REPO_SUB_DIRECTORY=$(dirname $REPO_DEST) - - # REPOTYPE can be overridden with DIB_REPOTYPE_{name} - local REPOTYPE_OVERRIDE=DIB_REPOTYPE_${REPONAME//[^A-Za-z0-9]/_} - REPOTYPE=${!REPOTYPE_OVERRIDE:-$REPOTYPE} - - # REPOLOCATION can be overridden with DIB_REPOLOCATION_{name} - local REPOLOCATION_OVERRIDE=DIB_REPOLOCATION_${REPONAME//[^A-Za-z0-9]/_} - REPOLOCATION=${!REPOLOCATION_OVERRIDE:-$REPOLOCATION} - - # REPOREF can be overridden with DIB_REPOREF_{name} - local REPOREF_OVERRIDE=DIB_REPOREF_${REPONAME//[^A-Za-z0-9]/_} - REPOREF=${!REPOREF_OVERRIDE:-$REPOREF} - - # Determine a unique cache path for this repo - CACHE_NAME=$(echo "${REPOTYPE}_${REPOLOCATION}" | sha1sum | awk '{ print $1 }' ) - OLD_CACHE_PATH=${CACHE_BASE}/${CACHE_NAME} - # Add the repo name to the sha1sum for readability - CACHE_NAME=${REPONAME//[^A-Za-z0-9]/_}_${CACHE_NAME} - CACHE_PATH=${CACHE_BASE}/$CACHE_NAME - make_new_cache $OLD_CACHE_PATH $CACHE_PATH - - # Return if install type is not source - local INSTALL_TYPE_VAR=DIB_INSTALLTYPE_${REPONAME//[^A-Za-z0-9]/_} - local INSTALL_TYPE=${!INSTALL_TYPE_VAR:-source} - if [ ! $INSTALL_TYPE = "source" ]; then - echo "$REPONAME install type not set to source" - continue - fi - - case $REPOTYPE in - git) - if [ -z "${!REPOLOCATION_OVERRIDE:-""}" -a -n "${DIB_GITREPOBASE:-""}" ] ; then - # Transform the current repo base to the new one - local NEW_REPOLOCATION=$(echo $REPOLOCATION |\ - sed "s,^[^:]\+://[^/]\+/\(~[^/]\+\)\?\(.*\)$,${DIB_GITREPOBASE}\2,g") - echo "Transformed ${REPOLOCATION} to ${NEW_REPOLOCATION}" - REPOLOCATION=$NEW_REPOLOCATION - # Also update the cache location - CACHE_NAME=$(echo "${REPOTYPE}_${REPOLOCATION}" | sha1sum | awk '{ print $1 }' ) - CACHE_PATH=~/.cache/image-create/repository-sources/$CACHE_NAME - fi - sudo mkdir -p $REPO_SUB_DIRECTORY - - if [ ! -e "$CACHE_PATH" ] ; then - echo "Caching $REPONAME from $REPOLOCATION in $CACHE_PATH" - git clone $REPOLOCATION $CACHE_PATH.tmp - mv ${CACHE_PATH}{.tmp,} - fi - - HAS_REF=$(git --git-dir=$CACHE_PATH/.git name-rev $REPOREF 2>/dev/null || true) - if [ -z "$DIB_OFFLINE" -o -z "$HAS_REF" ] ; then - echo "Updating cache of $REPOLOCATION in $CACHE_PATH with ref $REPOREF" - git --git-dir=$CACHE_PATH/.git fetch --update-head-ok --prune \ - $REPOLOCATION +refs/heads/*:refs/heads/* \ - +refs/remotes/*:refs/remotes/* - fi - - echo "Cloning from $REPONAME cache and applying ref $REPOREF" - sudo git clone $CACHE_PATH $REPO_DEST - pushd $REPO_DEST - sudo git fetch $CACHE_PATH $REPOREF - sudo git reset --hard FETCH_HEAD - # Get the reference in use - git_ref=$(git rev-parse FETCH_HEAD) - popd - - # Write the reference being used into the source-repositories manifest - echo "$REPONAME git $REPOPATH $REPOLOCATION $git_ref" >> $GIT_MANIFEST - ;; - tar) - # The top level directory of the tarball mightn't have a fixed name i.e. - # it could contain version numbers etc... so we write it to a tmpdir - # the then move the contents into the directory we want it in, this does - # assume the tarball only contains a single top level directory - local tmpdir=$(mktemp --tmpdir=$TMP_MOUNT_PATH/tmp -d) - if [ -n "$CACHE_PATH" ] ; then - echo "Caching $REPONAME tarball from $REPOLOCATION in $CACHE_PATH" - if [ ! -f "$CACHE_PATH" -o -z "$DIB_OFFLINE" ] ; then - $CACHE_URL $REPOLOCATION $CACHE_PATH - fi - tar -C $tmpdir -xzf $CACHE_PATH - else - echo "Fetching $REPONAME tarball from $REPOLOCATION" - curl $REPOLOCATION | tar -C $tmpdir -xzf - - fi - sudo mkdir -p $REPO_DEST - sudo mv $tmpdir/*/* $REPO_DEST - rm -rf $tmpdir - ;; - file) - sudo mkdir -p $REPO_SUB_DIRECTORY - if [ -n "$CACHE_PATH" ] ; then - echo "Caching $REPONAME file from $REPOLOCATION in $CACHE_PATH" - if [ ! -f "$CACHE_PATH" -o -z "$DIB_OFFLINE" ] ; then - $CACHE_URL $REPOLOCATION $CACHE_PATH - fi - sudo cp $CACHE_PATH $REPO_DEST - else - echo "Fetching $REPONAME file from $REPOLOCATION" - sudo curl $REPOLOCATION -o $REPO_DEST - fi - ;; - *) - echo "Unsupported repository type: $REPOTYPE" - return 1 - ;; - esac - - # Capture the in-instance repository path for later review / other - # elements (like a pypi dependency cache). - echo "$REPOPATH" | sudo dd of=$TMP_MOUNT_PATH/etc/dib-source-repositories oflag=append conv=notrunc - - else - echo "Couldn't parse '$line' as a source repository" - return 1 - fi - done < $REPO_SOURCES -} - +# in case source-repositories hasn't run yet, make the cache CACHE_BASE=$DIB_IMAGE_CACHE/source-repositories -OLD_CACHE_BASE=$DIB_IMAGE_CACHE/repository-sources -make_new_cache $OLD_CACHE_BASE $CACHE_BASE mkdir -p $CACHE_BASE -# Use the IMAGE_NAME from the calling script, and make it unique with the temporary path -GIT_MANIFEST_NAME=dib-manifest-git-$(basename ${IMAGE_NAME}) -GIT_MANIFEST_CACHE_NAME=${GIT_MANIFEST_NAME}_$(dirname ${TMP_MOUNT_PATH##*.}) -GIT_MANIFEST=$CACHE_BASE/${GIT_MANIFEST_CACHE_NAME} -rm -f $GIT_MANIFEST -# Get source repositories for the target -echo "devstack git /opt/git/openstack-dev/devstack $GIT_BASE/openstack-dev/devstack.git" > $TMP_HOOKS_PATH/early-source-repository-config -for _SOURCEREPO in $(find $TMP_HOOKS_PATH -maxdepth 1 -name "early-source-repository-*" -not -name '*~'); do - get_repos_for_element $_SOURCEREPO -done - -# Copy the manifest into the image if it exists (there may be no git repositories used) -if [ -e "$GIT_MANIFEST" ] ; then - sudo cp $GIT_MANIFEST $TMP_MOUNT_PATH/${DIB_MANIFEST_IMAGE_DIR}/$GIT_MANIFEST_NAME -fi +get_devstack_from_cache