diff --git a/pipelines/monolithic.Jenkinsfile b/pipelines/monolithic.Jenkinsfile index e6dda99..600125f 100644 --- a/pipelines/monolithic.Jenkinsfile +++ b/pipelines/monolithic.Jenkinsfile @@ -200,6 +200,23 @@ pipeline { string ( name: 'JENKINS_SCRIPTS_BRANCH' ) + text ( + name: 'PATCH_LIST', + description: '''\ +
List of Gerrit URLs to apply before running the build, one per line "[PATH] URL REF", eg:
+
+ https://review.opendev.org/starlingx/config refs/changes/71/859571/4
+ https://review.opendev.org/starlingx/stx-puppet refs/changes/75/859575/1
+ https://review.opendev.org/starlingx/tools refs/changes/76/859576/2
+
+or with paths relative to repo root:
+
+ cgcs-root/stx/config https://review.opendev.org/starlingx/config refs/changes/71/859571/4
+ cgcs-root/stx/stx-puppet https://review.opendev.org/starlingx/stx-puppet refs/changes/75/859575/1
+ stx-tools https://review.opendev.org/starlingx/tools refs/changes/76/859576/2
+
+ '''
+ )
}
stages {
diff --git a/pipelines/parts/clone-source.Jenkinsfile b/pipelines/parts/clone-source.Jenkinsfile
index 985f7c8..cd4b6ac 100644
--- a/pipelines/parts/clone-source.Jenkinsfile
+++ b/pipelines/parts/clone-source.Jenkinsfile
@@ -43,6 +43,9 @@ pipeline {
string (
name: 'REFRESH_SOURCE'
)
+ text (
+ name: 'PATCH_LIST'
+ )
}
stages {
stage ("clone-source") {
diff --git a/scripts/clone-source.sh b/scripts/clone-source.sh
index 3ca2366..b956019 100755
--- a/scripts/clone-source.sh
+++ b/scripts/clone-source.sh
@@ -31,43 +31,212 @@ ln -sfn "$WORKSPACE_ROOT_SUBDIR" "$WORKSPACE_ROOT"
shell() {
if [[ "$1" == "--dry-run" ]] ; then
echo ">>> (dry) running:" >&2
- echo "$2" >&2
+ echo "$2" | sed -r 's#^#\t#' >&2
return
fi
echo ">>> running" >&2
- echo "$1" >&2
+ echo "$1" | sed -r 's#^#\t#' >&2
( eval "$1" ; )
}
-# clone sources
cd "$REPO_ROOT"
-if [[ -f ".repo-init-done" ]] && ! $REFRESH_SOURCE ; then
- notice "repo already initialized, exiting"
- exit 0
-fi
-if $DRY_RUN && [[ -f ".repo-init-done" ]] ; then
- dry_run_arg="--dry-run"
+# If repo doesn't exist or REFRESH_SOURCE==true: discard all local changes
+# and re-pull all repos
+if [[ ! -d ".repo" || ! -f ".repo-init-done" ]] || $REFRESH_SOURCE ; then
+ notice "Cloning sources"
+
+ if [[ ! -d ".repo" || ! -f ".repo-init-done" ]] ; then
+ dry_run=false
+ dry_run_arg=
+ else
+ dry_run=$DRY_RUN
+ dry_run_arg=$DRY_RUN_ARG
+ fi
+
+ # Don't change this file name: the builder container relies on it!
+ $dry_run || rm -f ".repo-init-done"
+
+ # clone manifest
+ with_default_retries shell $dry_run_arg "repo init -u \"$MANIFEST_URL\" -b \"$MANIFEST_BRANCH\" -m \"$MANIFEST\""
+
+ # clean up gits: abort in-progress "rebase" & "am" commands &
+ # discard modified files
+ for d in $(repo forall -c 'echo $REPO_PATH' 2>/dev/null) ; do
+ [[ -d "$d" ]] || continue
+ shell $dry_run_arg "\
+set -e ;
+cd \"$d\"
+git rebase --abort >/dev/null 2>&1 || :
+git am --abort >/dev/null 2>&1 || :
+git cherry-pick --abort >/dev/null 2>&1 || :
+git clean -d -f
+git checkout .
+"
+ done
+ # pull gits
+ with_default_retries shell $dry_run_arg "repo sync --force-sync --force-remove-dirty -j4"
+ touch ".repo-init-done"
else
- dry_run_arg=
+ info "repo already cloned"
fi
-# We can't dry run, since we need the sources
-dry_run_arg=
+# Apply patches in "$PATCH_LIST" parameter given one per line:
+# [PATH] URL REF
+if [[ -n "$PATCH_LIST" ]] ; then
+ notice "Applying patches"
-shell $dry_run_arg "repo init -u \"$MANIFEST_URL\" -b \"$MANIFEST_BRANCH\" -m \"$MANIFEST\""
-for d in $(repo forall -c 'echo $REPO_PATH' 2>/dev/null) ; do
- [[ -d "$d" ]] || continue
- shell $dry_run_arg "
- set -e ;
- cd \"$d\"
- git rebase --abort >/dev/null 2>&1 || :
- git am --abort >/dev/null 2>&1 || :
- git clean -d -f
- git checkout .
- "
-done
-with_default_retries shell $dry_run_arg "repo sync --force-sync --force-remove-dirty -j4"
-# prevent "stx build prepare" from doing another "repo sync"
-shell $dry_run_arg "touch .repo-init-done"
+ ALL_PROJECTS="$(repo forall -c 'echo $REPO_PROJECT $REPO_PATH')"
+ # Helper func to determine whether a patch URL refers to the manifest repo
+ is_manifest_url() {
+ local path="$1"
+ local url="$2"
+ [[ -z "$path" ]] || path="$(realpath -m -s "$path" --relative-to=.)" || return 1
+
+ # path given
+ if [[ "$path" == ".repo/manifests" ]] ; then
+ return 0
+ fi
+
+ # url equals MANIFEST_URL exactly
+ if [[ "$url" == "$MANIFEST_URL" ]] ; then
+ return 0
+ fi
+
+ # url matches any of the manifest repo remote URLs on disk
+ all_urls=$( (cd .repo/manifests && git config --list | sed -r -n 's#^remote[.][^.]+[.]url=##gp' | sort -u) 2>/dev/null || :)
+ if [[ -n "$all_urls" ]] && in_list "$url" ${all_urls} ; then
+ return 0
+ fi
+
+ # give up
+ return 1
+ }
+
+ # Helper func to find a sub-project by path & url
+ # If user provided a path, look for the matching project
+ # Else, find project that matches URL's basename
+ # Prints the line "PROJECT PATH"
+ find_project() {
+ local path="$1"
+ local url="$2"
+ local spec
+
+ # path not provided: look for project that matches the URL's basename
+ # with or without .git at the end
+ if [[ -z "$path" ]] ; then
+ local url_base="${url##*/}"
+ local url_base_norm="${url_base%.git}"
+ local url_base_git="${url_base_norm}".git
+ local fail=0
+ while read spec_project spec_project_path ; do
+ [[ -n "$spec_project" ]] || continue
+ local spec_project_norm="${spec_project%.git}"
+ local spec_project_git="${spec_project_norm}.git"
+ if [[ "$spec_project_norm" == "$url_base_norm" || "$spec_project_git" == "$url_base_git" ]] ; then
+ if [[ -n "$path" ]] ; then
+ error "patch url \"$url\" matches more than one project:" \
+ " ${spec_project_path}" \
+ " ${path}" \
+ "Please specify project path explicitly in front of the URL, eg:" \
+ " ${path} ${url} ${ref}"
+ fail=1
+ continue
+ fi
+ echo "$spec_project $spec_project_path"
+ return 0
+ fi
+ done <<<"$ALL_PROJECTS"$'\n'
+ # Errors: return false
+ [[ $fail -eq 0 ]] || return 1
+
+ # path provided: look for project matching this path
+ else
+ path="$(realpath -m -s "$path" --relative-to=.)" || return 1
+ while read spec_project spec_project_path ; do
+ [[ -n "$spec_project" ]] || continue
+ spec_project_path="$(realpath -m -s "$spec_project_path" --relative-to=.)"
+ if [[ "$path" == "$spec_project_path" ]] ; then
+ echo "$spec_project $spec_project_path"
+ return 0
+ fi
+ done <<<"$ALL_PROJECTS"$'\n'
+ fi
+
+ # if we reach here, we couldn't find the project
+ error "unable to find project matching $path $url $ref"
+ return 1
+ }
+
+ # All patches, each element is a string:
+ # PROJECT PATH URL REF
+ PATCH_SPECS="$(
+ fail=0
+ while read line ; do
+ # skip empty lines
+ if [[ -z "$line" || "${line[0]}" == "#" ]] ; then
+ continue
+ fi
+ # split by whitespace
+ declare -a parts=($line)
+ if [[ "${#parts[@]}" -eq 2 ]] ; then
+ path=
+ url="${parts[0]}"
+ ref="${parts[1]}"
+ elif [[ "${#parts[@]}" -eq 3 ]] ; then
+ path="${parts[0]}"
+ url="${parts[1]}"
+ ref="${parts[2]}"
+ else
+ error "Invalid patch spec \"$line\"" \
+ "Expecting [PATH] URL REF"
+ fail=1
+ continue
+ fi
+
+ # manifest ?
+ if is_manifest_url "$path" "$url" ; then
+ project="-"
+ path=".repo/manifests"
+ else
+ # guess project and/or path
+ tmp="$(find_project "$path" "$url")" || return 1
+ read project path <<<"$tmp" || return 1
+ fi
+
+ echo "$project $path $url $ref"
+ done <<<"$PATCH_LIST"$'\n'
+ [[ $fail -eq 0 ]] || exit 1
+ )" || exit 1
+
+ # Reset each project that requires patching. We need to do this
+ # first in case PATCH_LIST includes multiple patches for the same repo,
+ # and in case we didn't run "repo sync" due to REFRESH_SOURCE=false
+ while read project path ; do
+ [[ -n "$project" ]] || continue
+ # abort in-progress "rebase" etc
+ info "" "--- $path: resetting git checkout" ""
+ shell $DRY_RUN_ARH "\
+set -e
+cd \"$path\"
+git rebase --abort >/dev/null 2>&1 || :
+git am --abort >/dev/null 2>&1 || :
+git cherry-pick --abort >/dev/null 2>&1 || :
+git checkout .
+"
+ # manifest: checkout "default" branch
+ if [[ $path == ".repo/manifests" ]] ; then
+ shell $DRY_RUN_ARG "cd \"$path\" && git checkout default"
+ # Other projects: checkout whatever manifest points to
+ else
+ shell $DRY_RUN_ARG "cd \"$path\" && repo sync --no-manifest-update --local-only --detach $project"
+ fi
+ done <<<"$(echo "$PATCH_SPECS" | awk '{print $1, $2}' | sort -u)"
+
+ # Apply patches one at a time
+ while read project path url ref ; do
+ info "" "--- $path: applying patch $url $ref" ""
+ shell $DRY_RUN_ARG "cd \"$path\" && git fetch \"$url\" \"$ref\" && git cherry-pick FETCH_HEAD"
+ done <<<"$(echo "$PATCH_SPECS")"
+fi