Introduce minikube-aio image
Image contains all binaries and saved images required for minikube deployment. Together with [0] it will allow to save up to 6 minutes on each minikube deployment in osh(|-infra) functional jobs. [0] https://review.opendev.org/744561 Change-Id: If76781c4398d4ce3415167d2132700a61f2f1178 Signed-off-by: Andrii Ostapenko <andrii.ostapenko@att.com>
This commit is contained in:
Andrii Ostapenko
parent
3a7d79a4de
commit
7976f3887a
|
|
@ -0,0 +1,82 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
FROM docker.io/ubuntu:bionic
|
||||
MAINTAINER andrii.ostapenko@att.com
|
||||
|
||||
ARG UBUNTU_URL=http://archive.ubuntu.com/ubuntu/
|
||||
ARG ALLOW_UNAUTHENTICATED=false
|
||||
|
||||
COPY ./sources.list /etc/apt/
|
||||
RUN sed -i \
|
||||
-e "s|%%UBUNTU_URL%%|${UBUNTU_URL}|g" \
|
||||
/etc/apt/sources.list ;\
|
||||
echo "APT::Get::AllowUnauthenticated \"${ALLOW_UNAUTHENTICATED}\";\n\
|
||||
Acquire::AllowInsecureRepositories \"${ALLOW_UNAUTHENTICATED}\";\n\
|
||||
Acquire::AllowDowngradeToInsecureRepositories \"${ALLOW_UNAUTHENTICATED}\";" \
|
||||
>> /etc/apt/apt.conf.d/allow-unathenticated
|
||||
|
||||
ARG KUBE_VERSION="v1.16.2"
|
||||
ARG MINIKUBE_VERSION="v1.3.1"
|
||||
ARG CALICO_VERSION="v3.9"
|
||||
ARG HELM_VERSION="v2.14.1"
|
||||
|
||||
COPY images.txt install.sh /opt/
|
||||
|
||||
RUN set -ex; \
|
||||
apt-get update; \
|
||||
apt-get install -y \
|
||||
ca-certificates \
|
||||
curl \
|
||||
gnupg \
|
||||
;\
|
||||
echo "export KUBE_VERSION=${KUBE_VERSION}" > /opt/versions.txt ;\
|
||||
echo "export MINIKUBE_VERSION=${MINIKUBE_VERSION}" >> /opt/versions.txt ;\
|
||||
echo "export CALICO_VERSION=${CALICO_VERSION}" >> /opt/versions.txt ;\
|
||||
echo "export HELM_VERSION=${HELM_VERSION}" >> /opt/versions.txt ;\
|
||||
GOOGLE_REPO_URL=https://storage.googleapis.com ;\
|
||||
MINIKUBE_REPO_URL=${GOOGLE_REPO_URL}/minikube/releases/${MINIKUBE_VERSION} ;\
|
||||
GOOGLE_KUBERNETES_REPO_URL=${GOOGLE_REPO_URL}/kubernetes-release/release/${KUBE_VERSION}/bin/linux/amd64 ;\
|
||||
GOOGLE_HELM_REPO_URL=${GOOGLE_REPO_URL}/kubernetes-helm ;\
|
||||
CNI_REPO_URL=https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION} ;\
|
||||
BINARY_DIR=/opt/binaries ;\
|
||||
mkdir ${BINARY_DIR}; \
|
||||
curl -sSLo ${BINARY_DIR}/minikube ${MINIKUBE_REPO_URL}/minikube-linux-amd64 ;\
|
||||
for BINARY in kubectl kubeadm kubelet; do \
|
||||
curl -sSLo ${BINARY_DIR}/${BINARY} ${GOOGLE_KUBERNETES_REPO_URL}/${BINARY} ;\
|
||||
done ;\
|
||||
TMP_DIR=$(mktemp -d) ;\
|
||||
curl -sSL ${GOOGLE_HELM_REPO_URL}/helm-${HELM_VERSION}-linux-amd64.tar.gz | tar -zxv --strip-components=1 -C ${TMP_DIR} ;\
|
||||
mv ${TMP_DIR}/helm ${BINARY_DIR} ;\
|
||||
rm -rf ${TMP_DIR} ;\
|
||||
chmod +x ${BINARY_DIR}/* ;\
|
||||
curl https://docs.projectcalico.org/"${CALICO_VERSION}"/manifests/calico.yaml -o /opt/calico.yaml; \
|
||||
for CALICO_IMAGE in $(grep -oP '(?<=image:)\s*calico/.+$' /opt/calico.yaml); do \
|
||||
echo ${CALICO_IMAGE} >> /opt/images.txt ;\
|
||||
done ;\
|
||||
printf "kubernetesVersion: ${KUBE_VERSION}\napiVersion: kubeadm.k8s.io/v1beta1\nkind: ClusterConfiguration" >> /tmp/kubeadm.yaml ;\
|
||||
$BINARY_DIR/kubeadm config images list --config /tmp/kubeadm.yaml | tee -a /opt/images.txt ;\
|
||||
. /etc/os-release ;\
|
||||
echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/x${NAME}_${VERSION_ID}/ /" > \
|
||||
/etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list ;\
|
||||
curl -sSL https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/x${NAME}_${VERSION_ID}/Release.key | apt-key add - ;\
|
||||
apt-get update ;\
|
||||
apt-get install -y skopeo gettext ;\
|
||||
mkdir /opt/images ;\
|
||||
echo "$(envsubst < /opt/images.txt | sort | uniq)" > /opt/images.txt ;\
|
||||
for IMAGE in $(cat /opt/images.txt); do \
|
||||
path=/opt/images/$(echo ${IMAGE} | sed 's#[/:]#_#g').tar ;\
|
||||
skopeo copy docker://${IMAGE} docker-archive:${path}:${IMAGE} ;\
|
||||
done
|
||||
|
||||
FROM scratch
|
||||
COPY --from=0 /opt /
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
gcr.io/k8s-minikube/storage-provisioner:v1.8.1
|
||||
gcr.io/kubernetes-helm/tiller:${HELM_VERSION}
|
||||
k8s.gcr.io/kube-addon-manager:v9.0
|
||||
|
|
@ -0,0 +1,221 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
set -xe
|
||||
|
||||
SCRIPT_DIR=$(dirname $0)
|
||||
|
||||
function extract {
|
||||
cd "${SCRIPT_DIR}"
|
||||
source versions.txt
|
||||
MINIKUBE_CACHE_DIR=${HOME}/.minikube/cache/${KUBE_VERSION}/
|
||||
mkdir -p "${MINIKUBE_CACHE_DIR}"
|
||||
chmod +x binaries/*
|
||||
for binary in minikube kubectl helm; do
|
||||
sudo mv binaries/${binary} /usr/local/bin/${binary}
|
||||
done
|
||||
for binary in kubeadm kubelet; do
|
||||
mv binaries/${binary} "${MINIKUBE_CACHE_DIR}"
|
||||
done
|
||||
for image in images/*; do
|
||||
sudo docker load < ${image}
|
||||
done
|
||||
cp calico.yaml /tmp/
|
||||
sudo docker images | sort | uniq | tee /tmp/loaded_images
|
||||
cd -
|
||||
}
|
||||
|
||||
function configure_resolvconf {
|
||||
# Setup resolv.conf to use the k8s api server, which is required for the
|
||||
# kubelet to resolve cluster services.
|
||||
sudo mv /etc/resolv.conf /etc/resolv.conf.backup
|
||||
|
||||
# Create symbolic link to the resolv.conf file managed by systemd-resolved, as
|
||||
# the kubelet.resolv-conf extra-config flag is automatically executed by the
|
||||
# minikube start command, regardless of being passed in here
|
||||
sudo ln -s /run/systemd/resolve/resolv.conf /etc/resolv.conf
|
||||
|
||||
sudo bash -c "echo 'nameserver 10.96.0.10' >> /etc/resolv.conf"
|
||||
|
||||
# NOTE(drewwalters96): Use the Google DNS servers to prevent local addresses in
|
||||
# the resolv.conf file unless using a proxy, then use the existing DNS servers,
|
||||
# as custom DNS nameservers are commonly required when using a proxy server.
|
||||
if [ -z "${HTTP_PROXY}" ]; then
|
||||
sudo bash -c "echo 'nameserver 8.8.8.8' >> /etc/resolv.conf"
|
||||
sudo bash -c "echo 'nameserver 8.8.4.4' >> /etc/resolv.conf"
|
||||
else
|
||||
sed -ne "s/nameserver //p" /etc/resolv.conf.backup | while read -r ns; do
|
||||
sudo bash -c "echo 'nameserver ${ns}' >> /etc/resolv.conf"
|
||||
done
|
||||
fi
|
||||
|
||||
sudo bash -c "echo 'search svc.cluster.local cluster.local' >> /etc/resolv.conf"
|
||||
sudo bash -c "echo 'options ndots:5 timeout:1 attempts:1' >> /etc/resolv.conf"
|
||||
|
||||
sudo rm /etc/resolv.conf.backup
|
||||
}
|
||||
|
||||
# NOTE: Clean Up hosts file
|
||||
sudo sed -i '/^127.0.0.1/c\127.0.0.1 localhost localhost.localdomain localhost4localhost4.localdomain4' /etc/hosts
|
||||
sudo sed -i '/^::1/c\::1 localhost6 localhost6.localdomain6' /etc/hosts
|
||||
|
||||
extract
|
||||
configure_resolvconf
|
||||
|
||||
# Prepare tmpfs for etcd
|
||||
sudo mkdir -p /data
|
||||
sudo mount -t tmpfs -o size=512m tmpfs /data
|
||||
|
||||
# NOTE: Deploy kubenetes using minikube. A CNI that supports network policy is
|
||||
# required for validation; use calico for simplicity.
|
||||
sudo -E minikube config set kubernetes-version "${KUBE_VERSION}"
|
||||
sudo -E minikube config set vm-driver none
|
||||
sudo -E minikube config set embed-certs true
|
||||
|
||||
# NOTE(aostapenko) Minikube still tries to pull images with kubeadm config imagepull
|
||||
# https://github.com/kubernetes/minikube/blob/v1.3.1/pkg/minikube/bootstrapper/kubeadm/kubeadm.go#L417
|
||||
# so we make it to fail fast and continue with existing images saving precious time
|
||||
sudo sed -i 's/127.0.0.1.*/\0 k8s.gcr.io/g' /etc/hosts
|
||||
|
||||
export CHANGE_MINIKUBE_NONE_USER=true
|
||||
export MINIKUBE_IN_STYLE=false
|
||||
sudo -E minikube start \
|
||||
--docker-env HTTP_PROXY="${HTTP_PROXY}" \
|
||||
--docker-env HTTPS_PROXY="${HTTPS_PROXY}" \
|
||||
--docker-env NO_PROXY="${NO_PROXY},10.96.0.0/12" \
|
||||
--network-plugin=cni \
|
||||
--extra-config=controller-manager.allocate-node-cidrs=true \
|
||||
--extra-config=controller-manager.cluster-cidr=192.168.0.0/16
|
||||
|
||||
sudo sed -i 's/k8s.gcr.io//g' /etc/hosts
|
||||
|
||||
kubectl apply -f /tmp/calico.yaml
|
||||
|
||||
# Note: Patch calico daemonset to enable Prometheus metrics and annotations
|
||||
tee /tmp/calico-node.yaml << EOF
|
||||
spec:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9091"
|
||||
spec:
|
||||
containers:
|
||||
- name: calico-node
|
||||
env:
|
||||
- name: FELIX_PROMETHEUSMETRICSENABLED
|
||||
value: "true"
|
||||
- name: FELIX_PROMETHEUSMETRICSPORT
|
||||
value: "9091"
|
||||
EOF
|
||||
kubectl patch daemonset calico-node -n kube-system --patch "$(cat /tmp/calico-node.yaml)"
|
||||
|
||||
# NOTE: Wait for dns to be running.
|
||||
END=$(($(date +%s) + 240))
|
||||
until kubectl --namespace=kube-system \
|
||||
get pods -l k8s-app=kube-dns --no-headers -o name | grep -q "^pod/coredns"; do
|
||||
NOW=$(date +%s)
|
||||
[ "${NOW}" -gt "${END}" ] && exit 1
|
||||
echo "still waiting for dns"
|
||||
sleep 10
|
||||
done
|
||||
kubectl --namespace=kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=kube-dns
|
||||
|
||||
# Deploy helm/tiller into the cluster
|
||||
kubectl create -n kube-system serviceaccount helm-tiller
|
||||
cat <<EOF | kubectl apply -f -
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: helm-tiller
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: cluster-admin
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: helm-tiller
|
||||
namespace: kube-system
|
||||
EOF
|
||||
|
||||
# NOTE(srwilkers): Required due to tiller deployment spec using extensions/v1beta1
|
||||
# which has been removed in Kubernetes 1.16.0.
|
||||
# See: https://github.com/helm/helm/issues/6374
|
||||
helm init --service-account helm-tiller --output yaml \
|
||||
| sed 's@apiVersion: extensions/v1beta1@apiVersion: apps/v1@' \
|
||||
| sed 's@ replicas: 1@ replicas: 1\n selector: {"matchLabels": {"app": "helm", "name": "tiller"}}@' \
|
||||
| kubectl apply -f -
|
||||
|
||||
# Patch tiller-deploy service to expose metrics port
|
||||
tee /tmp/tiller-deploy.yaml << EOF
|
||||
metadata:
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "44135"
|
||||
spec:
|
||||
ports:
|
||||
- name: http
|
||||
port: 44135
|
||||
targetPort: http
|
||||
EOF
|
||||
|
||||
kubectl patch service tiller-deploy -n kube-system --patch "$(cat /tmp/tiller-deploy.yaml)"
|
||||
kubectl --namespace=kube-system wait --timeout=240s --for=condition=Ready pod -l app=helm,name=tiller
|
||||
|
||||
helm init --client-only
|
||||
|
||||
# Set up local helm server
|
||||
sudo -E tee /etc/systemd/system/helm-serve.service << EOF
|
||||
[Unit]
|
||||
Description=Helm Server
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
User=$(id -un 2>&1)
|
||||
Restart=always
|
||||
ExecStart=/usr/local/bin/helm serve
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
sudo chmod 0640 /etc/systemd/system/helm-serve.service
|
||||
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl restart helm-serve
|
||||
sudo systemctl enable helm-serve
|
||||
|
||||
# Remove stable repo, if present, to improve build time
|
||||
helm repo remove stable || true
|
||||
|
||||
# Set up local helm repo
|
||||
helm repo add local http://localhost:8879/charts
|
||||
helm repo update
|
||||
|
||||
# Set required labels on host(s)
|
||||
kubectl label nodes --all openstack-control-plane=enabled
|
||||
kubectl label nodes --all openstack-compute-node=enabled
|
||||
kubectl label nodes --all openvswitch=enabled
|
||||
kubectl label nodes --all linuxbridge=enabled
|
||||
kubectl label nodes --all ceph-mon=enabled
|
||||
kubectl label nodes --all ceph-osd=enabled
|
||||
kubectl label nodes --all ceph-mds=enabled
|
||||
kubectl label nodes --all ceph-rgw=enabled
|
||||
kubectl label nodes --all ceph-mgr=enabled
|
||||
|
||||
# Add labels to the core namespaces
|
||||
kubectl label --overwrite namespace default name=default
|
||||
kubectl label --overwrite namespace kube-system name=kube-system
|
||||
kubectl label --overwrite namespace kube-public name=kube-public
|
||||
sudo docker images | sort | uniq | tee /tmp/images_after_installation
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
deb %%UBUNTU_URL%% bionic main universe
|
||||
deb %%UBUNTU_URL%% bionic-updates main universe
|
||||
deb %%UBUNTU_URL%% bionic-backports main universe
|
||||
deb %%UBUNTU_URL%% bionic-security main universe
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
---
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- project:
|
||||
check:
|
||||
jobs:
|
||||
- openstack-helm-images-build-minikube-aio
|
||||
gate:
|
||||
jobs:
|
||||
- openstack-helm-images-upload-minikube-aio
|
||||
promote:
|
||||
jobs:
|
||||
- openstack-helm-images-promote-minikube-aio
|
||||
periodic:
|
||||
jobs:
|
||||
- openstack-helm-images-build-minikube-aio
|
||||
|
||||
- job:
|
||||
name: openstack-helm-images-build-minikube-aio
|
||||
parent: openstack-helm-images-build
|
||||
provides: common-images
|
||||
description: Build minikube-aio image for OSH gates
|
||||
vars: &minikube_aio_vars
|
||||
currentdate: "{{ now(utc=True,fmt='%Y%m%d') }}"
|
||||
docker_images:
|
||||
- context: minikube-aio
|
||||
repository: openstackhelm/minikube-aio
|
||||
tags:
|
||||
- latest-ubuntu_bionic
|
||||
- "ubuntu_bionic-{{ currentdate }}"
|
||||
files: &minikube_aio_files
|
||||
- minikube-aio/.*
|
||||
- zuul.d/minikube-aio.yaml
|
||||
|
||||
- job:
|
||||
name: openstack-helm-images-upload-minikube-aio
|
||||
parent: openstack-helm-images-upload
|
||||
provides: common-images
|
||||
description: Build and upload minikube-aio image
|
||||
vars: *minikube_aio_vars
|
||||
files: *minikube_aio_files
|
||||
|
||||
- job:
|
||||
name: openstack-helm-images-promote-minikube-aio
|
||||
parent: openstack-helm-images-promote
|
||||
description: Promote previously built minikube-aio image
|
||||
vars: *minikube_aio_vars
|
||||
files: *minikube_aio_files
|
||||
Loading…
Reference in New Issue