Merge "Improve pipline resiliency against failing mirrors"

image-builder/Makefile

@@ -47,6 +47,8 @@ BOOT_TIMEOUT             ?= 300
 #it doesn't matter - we're not going to publish it, because it's useless without jsons
 KRM_IMAGE_TAG            ?= latest
 KRM_BASE_IMAGE           ?= ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/image-profile-krm:${KRM_IMAGE_TAG}
+# Space delimited list of apt mirror FQDN regex matches that will fallback to upstream ubuntu mirror if specified mirror is not working.
+APT_MIRROR_PROBLEM_LIST  ?= "citycloud"
 
 .PHONY: help build images cut_image package_qcow krm_base_image run clean docker_build tag push
 
@@ -60,6 +62,9 @@ images: build generate_iso package_qcow clean
 
 build:
 	set -ex
+	# Use upstream mirrors as a backup in case local apt mirrors are not working.
+	# Only applies to mirrors on the problem list
+	sudo -E ./tools/fix_mirrors.sh $(APT_MIRROR_PROBLEM_LIST)
 ifneq ($(PROFILE), )
 	# Apply any user-defined profiles overrides to playbooks
 	rsync -rc ./../profiles/$(PROFILE)/manifests/ $(WORKDIR)/

image-builder/tools/fix_mirrors.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# This script is to workaround very unreliable mirrors in opendev infra.
+# Note that proxy support is not needed, as this problem only exists for
+# upstream mirrors not behind any proxies.
+
+for bad_mirror in $@; do
+    # See if troublesome mirror is configured here
+    if apt-cache policy | grep "$bad_mirror"; then
+	echo "Detected troubled apt-cache policy:"
+	apt-cache policy
+	# Replace troublesome mirror with working mirror if it's not reachable
+	# Try installing a small package
+	apt -y install cpu-checker || (
+	    bad_mirror_uri="$(apt-cache policy | grep archive | head -1 | awk '{print $2}')"
+	    sed -i "s@$bad_mirror_uri@http://us.archive.ubuntu.com/ubuntu@g" /etc/apt/sources.list
+	    sed -i "s@$bad_mirror_uri@http://us.archive.ubuntu.com/ubuntu@g" /etc/apt/sources.list.d/*
+	    echo "Modified apt-cache policy:"
+	    apt-cache policy
+	    echo "Update apt sources list:"
+	    apt -y update
+	    )
+        break
+    fi
+done