From 414873bb861f73fd089db125fad63d28ab89db9e Mon Sep 17 00:00:00 2001
From: "Anderson, Craig (ca846m)" <ca846m@att.com>
Date: Mon, 1 Nov 2021 12:48:11 -0700
Subject: [PATCH] Improve pipline resiliency against failing mirrors

This change allows fallback to upstream ubuntu mirrors when the apt
mirrors supplied by the cloud provider are not working as expected.

Change-Id: I4391799531e39253f435904c28bda2bebc26f795
---
 image-builder/Makefile             |  5 +++++
 image-builder/tools/fix_mirrors.sh | 24 ++++++++++++++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100755 image-builder/tools/fix_mirrors.sh

diff --git a/image-builder/Makefile b/image-builder/Makefile
index 7395186..1048ca7 100644
--- a/image-builder/Makefile
+++ b/image-builder/Makefile
@@ -47,6 +47,8 @@ BOOT_TIMEOUT             ?= 300
 #it doesn't matter - we're not going to publish it, because it's useless without jsons
 KRM_IMAGE_TAG            ?= latest
 KRM_BASE_IMAGE           ?= ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/image-profile-krm:${KRM_IMAGE_TAG}
+# Space delimited list of apt mirror FQDN regex matches that will fallback to upstream ubuntu mirror if specified mirror is not working.
+APT_MIRROR_PROBLEM_LIST  ?= "citycloud"
 
 .PHONY: help build images cut_image package_qcow krm_base_image run clean docker_build tag push
 
@@ -60,6 +62,9 @@ images: build generate_iso package_qcow clean
 
 build:
 	set -ex
+	# Use upstream mirrors as a backup in case local apt mirrors are not working.
+	# Only applies to mirrors on the problem list
+	sudo -E ./tools/fix_mirrors.sh $(APT_MIRROR_PROBLEM_LIST)
 ifneq ($(PROFILE), )
 	# Apply any user-defined profiles overrides to playbooks
 	rsync -rc ./../profiles/$(PROFILE)/manifests/ $(WORKDIR)/
diff --git a/image-builder/tools/fix_mirrors.sh b/image-builder/tools/fix_mirrors.sh
new file mode 100755
index 0000000..c2d558a
--- /dev/null
+++ b/image-builder/tools/fix_mirrors.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+# This script is to workaround very unreliable mirrors in opendev infra.
+# Note that proxy support is not needed, as this problem only exists for
+# upstream mirrors not behind any proxies.
+
+for bad_mirror in $@; do
+    # See if troublesome mirror is configured here
+    if apt-cache policy | grep "$bad_mirror"; then
+	echo "Detected troubled apt-cache policy:"
+	apt-cache policy
+	# Replace troublesome mirror with working mirror if it's not reachable
+	# Try installing a small package
+	apt -y install cpu-checker || (
+	    bad_mirror_uri="$(apt-cache policy | grep archive | head -1 | awk '{print $2}')"
+	    sed -i "s@$bad_mirror_uri@http://us.archive.ubuntu.com/ubuntu@g" /etc/apt/sources.list
+	    sed -i "s@$bad_mirror_uri@http://us.archive.ubuntu.com/ubuntu@g" /etc/apt/sources.list.d/*
+	    echo "Modified apt-cache policy:"
+	    apt-cache policy
+	    echo "Update apt sources list:"
+	    apt -y update
+	    )
+        break
+    fi
+done