Merge "Add proxy setting for container image build scripts"

build-tools/build-docker-images/build-stx-images.sh

@@ -26,8 +26,10 @@ IMAGE_VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp
 PREFIX=dev
 LATEST_PREFIX=""
 PUSH=no
-PROXY=""
 CONFIG_FILE=""
+HTTP_PROXY=""
+HTTPS_PROXY=""
+NO_PROXY=""
 DOCKER_USER=${USER}
 DOCKER_REGISTRY=
 BASE=
@@ -57,7 +59,9 @@ Options:
     --wheels:     Specify path to wheels tarball or image, URL or docker tag (required option)
     --wheels-alternate: Specify path to alternate wheels tarball or image, URL or docker tag
     --push:       Push to docker repo
-    --proxy:      Set proxy <URL>:<PORT>
+    --http_proxy: Set proxy <URL>:<PORT>, urls splitted with ","
+    --https_proxy: Set proxy <URL>:<PORT>, urls splitted with ","
+    --no_proxy: Set proxy <URL>, urls splitted with ","
     --user:       Docker repo userid
     --registry:   Docker registry
     --prefix:     Prefix on the image tag (default: dev)
@@ -287,7 +291,20 @@ function post_build {
 
 
     if [ -n "${CUSTOMIZATION}" ]; then
-        docker run --entrypoint /bin/bash --name ${USER}_update_img ${build_image_name} -c "${CUSTOMIZATION}"
+        local -a PROXY_ARGS=
+        if [ ! -z "$HTTP_PROXY" ]; then
+            PROXY_ARGS+=(--env http_proxy=$HTTP_PROXY)
+        fi
+
+        if [ ! -z "$HTTPS_PROXY" ]; then
+            PROXY_ARGS+=(--env https_proxy=$HTTPS_PROXY)
+        fi
+
+        if [ ! -z "$NO_PROXY" ]; then
+            PROXY_ARGS+=(--env no_proxy=$NO_PROXY)
+        fi
+
+        docker run "${PROXY_ARGS[@]}" --entrypoint /bin/bash --name ${USER}_update_img ${build_image_name} -c "${CUSTOMIZATION}"
         if [ $? -ne 0 ]; then
             echo "Failed to add customization for ${LABEL}... Aborting"
             RESULTS_FAILED+=(${LABEL})
@@ -402,8 +419,16 @@ function build_image_loci {
         BUILD_ARGS+=(--build-arg WHEELS=${WHEELS})
     fi
 
-    if [ ! -z "$PROXY" ]; then
+    if [ ! -z "$HTTP_PROXY" ]; then
-        BUILD_ARGS+=(--build-arg http_proxy=$PROXY)
+        BUILD_ARGS+=(--build-arg http_proxy=$HTTP_PROXY)
+    fi
+
+    if [ ! -z "$HTTPS_PROXY" ]; then
+        BUILD_ARGS+=(--build-arg https_proxy=$HTTPS_PROXY)
+    fi
+
+    if [ ! -z "$NO_PROXY" ]; then
+        BUILD_ARGS+=(--build-arg no_proxy=$NO_PROXY)
     fi
 
     if [ -n "${PROJECT_REF}" ]; then
@@ -554,9 +579,18 @@ function build_image_docker {
     BASE_BUILD_ARGS+=(${real_docker_context} --no-cache)
     BASE_BUILD_ARGS+=(--file ${real_docker_file})
     BASE_BUILD_ARGS+=(--build-arg "BASE=${BASE}")
-    if [ ! -z "$PROXY" ]; then
+    if [ ! -z "$HTTP_PROXY" ]; then
-        BASE_BUILD_ARGS+=(--build-arg http_proxy=$PROXY)
+        BASE_BUILD_ARGS+=(--build-arg http_proxy=$HTTP_PROXY)
     fi
+
+    if [ ! -z "$HTTPS_PROXY" ]; then
+        BASE_BUILD_ARGS+=(--build-arg https_proxy=$HTTPS_PROXY)
+    fi
+
+    if [ ! -z "$NO_PROXY" ]; then
+        BASE_BUILD_ARGS+=(--build-arg no_proxy=$NO_PROXY)
+    fi
+
     BASE_BUILD_ARGS+=(--tag ${build_image_name})
     with_retries ${MAX_ATTEMPTS} docker build ${BASE_BUILD_ARGS[@]} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log
 
@@ -639,7 +673,7 @@ function build_image_script {
     cp $(dirname ${image_build_file})/${SCRIPT} ${SCRIPT}
     local build_image_name="${USER}/${LABEL}:${IMAGE_TAG_BUILD}"
 
-    with_retries ${MAX_ATTEMPTS} ${COMMAND} ${SCRIPT} ${ARGS} ${build_image_name} $PROXY 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log
+    with_retries ${MAX_ATTEMPTS} ${COMMAND} ${SCRIPT} ${ARGS} ${build_image_name} $HTTP_PROXY $HTTPS_PROXY $NO_PROXY 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log
 
     if [ ${PIPESTATUS[0]} -ne 0 ]; then
         echo "Failed to build ${LABEL}... Aborting"
@@ -681,7 +715,7 @@ function build_image {
     esac
 }
 
-OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,proxy:,user:,registry:,base:,wheels:,wheels-alternate:,only:,skip:,prefix:,latest,latest-prefix:,clean,attempts:,config-file: -- "$@")
+OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,http_proxy:,https_proxy:,no_proxy:,user:,registry:,base:,wheels:,wheels-alternate:,only:,skip:,prefix:,latest,latest-prefix:,clean,attempts:,config-file: -- "$@")
 if [ $? -ne 0 ]; then
     usage
     exit 1
@@ -736,8 +770,16 @@ while true; do
             PUSH=yes
             shift
             ;;
-        --proxy)
+        --http_proxy)
-            PROXY=$2
+            HTTP_PROXY=$2
+            shift 2
+            ;;
+        --https_proxy)
+            HTTPS_PROXY=$2
+            shift 2
+            ;;
+        --no_proxy)
+            NO_PROXY=$2
             shift 2
             ;;
         --user)

build-tools/build-docker-images/update-stx-image.sh

@@ -18,7 +18,9 @@ if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
 fi
 
 PUSH=no
-PROXY=""
+HTTP_PROXY=""
+HTTPS_PROXY=""
+NO_PROXY=""
 DOCKER_USER=
 DOCKER_REGISTRY=
 FILE_BASEDIR=${PWD}
@@ -59,7 +61,9 @@ Options:
     --customize:  Customization script
     --extra:      Extra file (to be accessible to customization script)
     --push:       Push to docker repo
-    --proxy:      Set proxy <URL>:<PORT>
+    --http_proxy:      Set http proxy <URL>:<PORT>, urls splitted with ","
+    --https_proxy:     Set https proxy <URL>:<PORT>, urls splitted with ","
+    --no_proxy:        set bypass list for proxy <URL> urls splitted with ","
     --user:       Docker repo userid
     --registry:   Docker registry
     --clean:      Remove image(s) from local registry
@@ -184,7 +188,7 @@ function read_params_from_file {
     FILE_BASEDIR=$(dirname ${FILE})
 }
 
-OPTS=$(getopt -o h -l help,file:,from:,wheel:,module-src:,pkg:,customize:,extra:,push,proxy:,user:,registry:,clean,attempts:,update-id: -- "$@")
+OPTS=$(getopt -o h -l help,file:,from:,wheel:,module-src:,pkg:,customize:,extra:,push,http_proxy:,https_proxy:,no_proxy:,user:,registry:,clean,attempts:,update-id: -- "$@")
 if [ $? -ne 0 ]; then
     usage
     exit 1
@@ -235,8 +239,16 @@ while true; do
             PUSH=yes
             shift
             ;;
-        --proxy)
+        --http_proxy)
-            PROXY=$2
+            HTTP_PROXY=$2
+            shift 2
+            ;;
+        --https_proxy)
+            HTTPS_PROXY=$2
+            shift 2
+            ;;
+        --no_proxy)
+            NO_PROXY=$2
             shift 2
             ;;
         --user)

build-tools/build-wheels/build-base-wheels.sh

@@ -24,7 +24,9 @@ KEEP_CONTAINER=no
 OS=centos
 OS_VERSION=7.5.1804
 BUILD_STREAM=stable
-PROXY=""
+HTTP_PROXY=""
+HTTPS_PROXY=""
+NO_PROXY=""
 declare -i MAX_ATTEMPTS=1
 
 function usage {
@@ -37,14 +39,16 @@ Options:
     --os-version:     Specify OS version
     --keep-image:     Skip deletion of the wheel build image in docker
     --keep-container: Skip deletion of container used for the build
-    --proxy:          Set proxy <URL>:<PORT>
+    --http_proxy:     Set http proxy <URL>:<PORT>, urls splitted by ","
+    --https_proxy:    Set https proxy <URL>:<PORT>, urls splitted by ","
+    --no_proxy:       Set bypass list for proxy <URL>, urls splitted by ","
     --stream:         Build stream, stable or dev (default: stable)
     --attempts:       Max attempts, in case of failure (default: 1)
 
 EOF
 }
 
-OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,proxy:,attempts: -- "$@")
+OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,http_proxy:,https_proxy:,no_proxy:,attempts: -- "$@")
 if [ $? -ne 0 ]; then
     usage
     exit 1
@@ -83,8 +87,16 @@ while true; do
             BUILD_STREAM=$2
             shift 2
             ;;
-        --proxy)
+        --http_proxy)
-            PROXY=$2
+            HTTP_PROXY=$2
+            shift 2
+            ;;
+        --https_proxy)
+            HTTPS_PROXY=$2
+            shift 2
+            ;;
+        --no_proxy)
+            NO_PROXY=$2
             shift 2
             ;;
         --attempts)
@@ -212,10 +224,18 @@ BASE_IMAGE_PRESENT=$?
 declare -a BUILD_ARGS
 BUILD_ARGS+=(--build-arg RELEASE=${OS_VERSION})
 BUILD_ARGS+=(--build-arg BUILD_STREAM=${BUILD_STREAM})
-if [ ! -z "$PROXY" ]; then
+if [ ! -z "$HTTP_PROXY" ]; then
-    BUILD_ARGS+=(--build-arg http_proxy=$PROXY)
+    BUILD_ARGS+=(--build-arg http_proxy=$HTTP_PROXY)
-    BUILD_ARGS+=(--build-arg https_proxy=$PROXY)
 fi
+
+if [ ! -z "$HTTPS_PROXY" ]; then
+    BUILD_ARGS+=(--build-arg https_proxy=$HTTPS_PROXY)
+fi
+
+if [ ! -z "$NO_PROXY" ]; then
+    BUILD_ARGS+=(--build-arg no_proxy=$NO_PROXY)
+fi
+
 BUILD_ARGS+=(-t ${BUILD_IMAGE_NAME})
 BUILD_ARGS+=(-f ${DOCKER_PATH}/${OS}-dockerfile ${DOCKER_PATH})
 
@@ -233,10 +253,16 @@ if [ "${KEEP_CONTAINER}" = "no" ]; then
 fi
 
 declare -a RUN_ARGS
-if [ ! -z "$PROXY" ]; then
+if [ ! -z "$HTTP_PROXY" ]; then
-    RUN_ARGS+=(--env http_proxy=$PROXY)
+    RUN_ARGS+=(--env http_proxy=$HTTP_PROXY)
-    RUN_ARGS+=(--env https_proxy=$PROXY)
 fi
+if [ ! -z "$HTTPS_PROXY" ]; then
+    RUN_ARGS+=(--env https_proxy=$HTTPS_PROXY)
+fi
+if [ ! -z "$NO_PROXY" ]; then
+    RUN_ARGS+=(--env no_proxy=$NO_PROXY)
+fi
+
 RUN_ARGS+=(${RM_OPT} -v ${BUILD_OUTPUT_PATH}:/wheels ${BUILD_IMAGE_NAME} /docker-build-wheel.sh)
 
 # Run container to build wheels

build-tools/build-wheels/build-wheel-tarball.sh

@@ -24,7 +24,9 @@ BUILD_STREAM=stable
 CURRENT_STABLE_OPENSTACK=ussuri
 VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp
 PUSH=no
-PROXY=""
+HTTP_PROXY=""
+HTTPS_PROXY=""
+NO_PROXY=""
 CLEAN=no
 DOCKER_USER=${USER}
 declare -i MAX_ATTEMPTS=1
@@ -56,7 +58,9 @@ Options:
     --os-version: Specify OS version
     --stream:     Build stream, stable or dev (default: stable)
     --push:       Push to docker repo
-    --proxy:      Set proxy <URL>:<PORT>
+    --http_proxy: Set http proxy <URL>:<PORT>, urls splitted by ","
+    --https_proxy: Set https proxy <URL>:<PORT>, urls splitted by ","
+    --no_proxy:   Set bypass list for proxy <URL>, urls splitted by ","
     --user:       Docker repo userid
     --version:    Version for pushed image (if used with --push)
     --attempts:   Max attempts, in case of failure (default: 1)
@@ -64,7 +68,7 @@ Options:
 EOF
 }
 
-OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,proxy:,version:,attempts: -- "$@")
+OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,http_proxy:,https_proxy:,no_proxy:,version:,attempts: -- "$@")
 if [ $? -ne 0 ]; then
     usage
     exit 1
@@ -107,8 +111,16 @@ while true; do
             BUILD_STREAM=$2
             shift 2
             ;;
-        --proxy)
+        --http_proxy)
-            PROXY=$2
+            HTTP_PROXY=$2
+            shift 2
+            ;;
+        --https_proxy)
+            HTTPS_PROXY=$2
+            shift 2
+            ;;
+        --no_proxy)
+            NO_PROXY=$2
             shift 2
             ;;
         --version)
@@ -147,8 +159,16 @@ fi
 # Build the base wheels and retrieve the StarlingX wheels
 declare -a BUILD_BASE_WL_ARGS
 BUILD_BASE_WL_ARGS+=(--os ${OS} --os-version ${OS_VERSION} --stream ${BUILD_STREAM})
-if [ ! -z "$PROXY" ]; then
+if [ ! -z "$HTTP_PROXY" ]; then
-    BUILD_BASE_WL_ARGS+=(--proxy ${PROXY})
+    BUILD_BASE_WL_ARGS+=(--http_proxy ${HTTP_PROXY})
+fi
+
+if [ ! -z "$HTTPS_PROXY" ]; then
+    BUILD_BASE_WL_ARGS+=(--https_proxy ${HTTPS_PROXY})
+fi
+
+if [ ! -z "$NO_PROXY" ]; then
+    BUILD_BASE_WL_ARGS+=(--no_proxy ${NO_PROXY})
 fi
 
 ${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]} --attempts ${MAX_ATTEMPTS}