Rewrite smoke jobs to one using simple deployment script

Change-Id: Ib2f76e416fbbe243693bdd7eb170385294cb0986
This commit is contained in:
Artur Zarzycki 2016-11-22 10:51:56 +01:00
parent 8910b7c873
commit 86ad6d50e9
3 changed files with 346 additions and 74 deletions

411
jenkins-tp/jobs/builders/fuel-ccp-simple-deployment.sh Normal file → Executable file
View File

@ -6,7 +6,6 @@
# The script expects fuel-ccp cloned into fuel-ccp/ and fuel-ccp-installer
# cloned into fuel-ccp-installer/ directory (e.g. by Jenkins SCM plugin).
# CONFIGURATION:
######################################################
: ${FUEL_DEVOPS_ENV_NAME:="fuel-ccp-${COMPONENT}-${VERSION}-deployment"}
@ -21,21 +20,27 @@ export REGISTRY_IP=`ip -4 addr show eth0 | grep -oP '(?<=inet\s)\d+(\.\d+){3}'`
export REGISTRY_PORT=5000
JENKINS_GID=`getent group jenkins | cut -d":" -f3`
REGISTRY_HASH=`docker inspect --format "{{.Id}}" registry`
export DOCKER_REGISTRY_HOST="registry.mcp.fuel-infra.org"
export DOCKER_REGISTRY="${DOCKER_REGISTRY_HOST}:6000"
export DOCKER_REGISTRY_IP="`getent hosts ${DOCKER_REGISTRY_HOST} | awk '{print $1}'`"
export IMAGES_NAMESPACE="ccp"
export REGISTRY_NAMESPACE="mcp"
# Prepare K8s env:
source "${FUEL_DEVOPS_INSTALLATION_DIR}"/bin/activate
NEED_TO_SETUP_ENV=false
dos.py revert "${FUEL_DEVOPS_ENV_NAME}" "${FUEL_DEVOPS_SNAPSHOT_NAME}" || NEED_TO_SETUP_ENV=true
dos.py resume "${FUEL_DEVOPS_ENV_NAME}" || NEED_TO_SETUP_ENV=true
if [ "${NEED_TO_SETUP_ENV}" = "true" ]; then
echo "Couldn't revert fuel-devops snapshot, recreating environment."
function prepare_k8s_env {
# Prepare K8s env:
source "${FUEL_DEVOPS_INSTALLATION_DIR}"/bin/activate
NEED_TO_SETUP_ENV=false
dos.py revert "${FUEL_DEVOPS_ENV_NAME}" "${FUEL_DEVOPS_SNAPSHOT_NAME}" || NEED_TO_SETUP_ENV=true
dos.py resume "${FUEL_DEVOPS_ENV_NAME}" || NEED_TO_SETUP_ENV=true
if [ "${NEED_TO_SETUP_ENV}" = "true" ]; then
echo "Couldn't revert fuel-devops snapshot, recreating environment."
# Try to delete old envs to ensure clean host:
dos.py destroy "${FUEL_DEVOPS_ENV_NAME}" || true
dos.py erase "${FUEL_DEVOPS_ENV_NAME}" || true
# Try to delete old envs to ensure clean host:
dos.py destroy "${FUEL_DEVOPS_ENV_NAME}" || true
dos.py erase "${FUEL_DEVOPS_ENV_NAME}" || true
# Create kargo deployment script:
cat > k8s_deploy.sh << EOF
# Create kargo deployment script:
cat > k8s_deploy.sh << EOF
#!/bin/bash -ex
export ENV_NAME=${FUEL_DEVOPS_ENV_NAME}
@ -48,30 +53,282 @@ export FUEL_DEVOPS_INSTALLATION_DIR=${FUEL_DEVOPS_INSTALLATION_DIR}
export CUSTOM_YAML='hyperkube_image_repo: "${HYPERKUBE_REPO}"
hyperkube_image_tag: "${HYPERKUBE_TAG}"
kube_version: "${HYPERKUBE_VERSION}"
docker_options: "--insecure-registry=${REGISTRY_IP}:${REGISTRY_PORT}"'
docker_options: "--insecure-registry=${REGISTRY_IP}:${REGISTRY_PORT} --insecure-registry=DOCKER_REGISTRY"'
echo "Running on \${NODE_NAME}: \${ENV_NAME}"
source \${FUEL_DEVOPS_INSTALLATION_DIR}/bin/activate
echo "Use image:"
ls -la \${IMAGE_PATH}
env
pushd fuel-ccp-installer
bash -ex "./utils/jenkins/run_k8s_deploy_test.sh"
popd
EOF
chmod +x k8s_deploy.sh
chmod +x k8s_deploy.sh
# Now deploy the cluster:
./k8s_deploy.sh
# Now deploy the cluster:
./k8s_deploy.sh
# Create fresh snapshot:
dos.py suspend "${FUEL_DEVOPS_ENV_NAME}"
dos.py snapshot "${FUEL_DEVOPS_ENV_NAME}" "${FUEL_DEVOPS_SNAPSHOT_NAME}"
# Create fresh snapshot:
dos.py suspend "${FUEL_DEVOPS_ENV_NAME}"
dos.py snapshot "${FUEL_DEVOPS_ENV_NAME}" "${FUEL_DEVOPS_SNAPSHOT_NAME}"
# Resume from snapshot to deploy CCP later on in the script:
dos.py resume "${FUEL_DEVOPS_ENV_NAME}"
fi
# Resume from snapshot to deploy CCP later on in the script:
dos.py resume "${FUEL_DEVOPS_ENV_NAME}"
fi
}
function fix_restored_env {
# After restore snapshot ntp service doesn't work and need restart on all nodes
${SSH_COMMAND} "sudo service ntp restart"
${SSH_COMMAND} "ssh -o StrictHostKeyChecking=no node2 sudo service ntp restart"
${SSH_COMMAND} "ssh -o StrictHostKeyChecking=no node3 sudo service ntp restart"
# Dirty hack for workaround network problems on CI envs.
# When we deploy env some time after (few minutes) it change resolv.conf into broken one
# From this reason after bring up env we restart network and and kill dhclient,
# we also restart docker and kubelet to make sure that all net-host containers are in good shape
cat > fix_dns.sh << EOF
sudo service networking restart
sudo pkill -9 dhclient
sudo service kubelet restart
sudo service docker restart
EOF
chmod +x fix_dns.sh
${SCP_COMMAND} fix_dns.sh vagrant@"${ADMIN_IP}":~/
${SSH_COMMAND} "scp -o StrictHostKeyChecking=no fix_dns.sh vagrant@node2:~/"
${SSH_COMMAND} "scp -o StrictHostKeyChecking=no fix_dns.sh vagrant@node3:~/"
${SSH_COMMAND} "sudo ./fix_dns.sh"
${SSH_COMMAND} "ssh -o StrictHostKeyChecking=no node2 sudo ./fix_dns.sh"
${SSH_COMMAND} "ssh -o StrictHostKeyChecking=no node3 sudo ./fix_dns.sh"
}
function copy_code_to_vm {
# Prepare env on "admin" VM:
if [ ${COMPONENT} == "full" ]; then
${SCP_COMMAND} -r fuel-ccp/ vagrant@"${ADMIN_IP}":~/
elif [ ${COMPONENT} == "smoke" ]; then
${SCP_COMMAND} -r fuel-ccp/ vagrant@"${ADMIN_IP}":~/
else
${SCP_COMMAND} -r fuel-ccp/ vagrant@"${ADMIN_IP}":~/
# set +x is just for security reasons to avoid publishing internal IP
set +x
getent hosts zuul.mcp.fuel-infra.org | ${SSH_COMMAND} "sudo tee -a /etc/hosts"
set -x
${SCP_COMMAND} -r containers/openstack/ vagrant@"${ADMIN_IP}":/tmp/ccp-repos
fi
}
function prepare_ccp_config {
cat > ccp.yml << EOF
builder:
push: True
workers: 1
registry:
address: "${REGISTRY_IP}:${REGISTRY_PORT}"
timeout: 1500
replicas:
database: 3
rpc: 3
repositories:
path: /tmp/ccp-repos
skip_empty: True
entrypoint_repo_name: entrypoint
repos:
- git_url: https://git.openstack.org/openstack/fuel-ccp-debian-base
name: debian-base
- git_url: https://git.openstack.org/openstack/fuel-ccp-entrypoint
name: entrypoint
- git_url: https://git.openstack.org/openstack/fuel-ccp-etcd
name: etcd
- git_url: https://git.openstack.org/openstack/fuel-ccp-galera
name: galera
- git_url: https://git.openstack.org/openstack/fuel-ccp-glance
name: glance
- git_url: https://git.openstack.org/openstack/fuel-ccp-heat
name: heat
- git_url: https://git.openstack.org/openstack/fuel-ccp-horizon
name: horizon
- git_url: https://git.openstack.org/openstack/fuel-ccp-keystone
name: keystone
- git_url: https://git.openstack.org/openstack/fuel-ccp-memcached
name: memcached
- git_url: https://git.openstack.org/openstack/fuel-ccp-neutron
name: neutron
- git_url: https://git.openstack.org/openstack/fuel-ccp-nova
name: nova
- git_url: https://git.openstack.org/openstack/fuel-ccp-openstack-base
name: openstack-base
- git_url: https://git.openstack.org/openstack/fuel-ccp-rabbitmq
name: rabbitmq
- git_url: https://git.openstack.org/openstack/fuel-ccp-ceph
name: ceph
- git_url: https://git.openstack.org/openstack/fuel-ccp-ironic
name: ironic
configs:
private_interface: ens3
nodes:
node[1-3]:
roles:
- controller-net-bridge
node1:
roles:
- db
- messaging
- openvswitch
- controller-net-host
node[2-3]:
roles:
- db
- messaging
- openvswitch
- compute
sources:
openstack/cinder:
git_url: https://git.openstack.org/openstack/cinder.git
git_ref: ${BRANCH}
openstack/ironic:
git_url: https://git.openstack.org/openstack/ironic.git
git_ref: ${BRANCH}
openstack/murano:
git_url: https://git.openstack.org/openstack/murano.git
git_ref: ${BRANCH}
openstack/murano-dashboard:
git_url: https://git.openstack.org/openstack/murano-dashboard.git
git_ref: ${BRANCH}
openstack/sahara:
git_url: https://git.openstack.org/openstack/sahara.git
git_ref: ${BRANCH}
openstack/sahara-dashboard:
git_url: https://git.openstack.org/openstack/sahara-dashboard.git
git_ref: ${BRANCH}
openstack/keystone:
git_url: https://git.openstack.org/openstack/keystone.git
git_ref: ${BRANCH}
openstack/horizon:
git_url: https://git.openstack.org/openstack/horizon.git
git_ref: ${BRANCH}
openstack/nova:
git_url: https://git.openstack.org/openstack/nova.git
git_ref: ${BRANCH}
openstack/neutron:
git_url: https://git.openstack.org/openstack/neutron.git
git_ref: ${BRANCH}
openstack/heat:
git_url: https://git.openstack.org/openstack/heat.git
git_ref: ${BRANCH}
openstack/glance:
git_url: https://git.openstack.org/openstack/glance.git
git_ref: ${BRANCH}
openstack/requirements:
git_url: https://git.openstack.org/openstack/requirements.git
git_ref: ${BRANCH}
services:
database:
service_def: galera
rpc:
service_def: rabbitmq
notifications:
service_def: rabbitmq
roles:
db:
- database
messaging:
- rpc
- notifications
controller-net-host:
- neutron-dhcp-agent
- neutron-l3-agent
- neutron-metadata-agent
controller-net-bridge:
- etcd
- glance-api
- glance-registry
- heat-api-cfn
- heat-api
- heat-engine
- horizon
- keystone
- memcached
- neutron-server
- nova-api
- nova-conductor
- nova-consoleauth
- nova-novncproxy
- nova-scheduler
- rabbitmq
compute:
- nova-compute
- nova-libvirt
openvswitch:
- neutron-openvswitch-agent
- openvswitch-db
- openvswitch-vswitchd
EOF
}
ccp_wait_for_deployment_to_finish () {
cnt=0
until [[ `${SSH_COMMAND} ccp status -s -f value -c status` == "ok" ]]; do
echo "Waiting for OpenStack deployment to finish..."
sleep 5
cnt=$((cnt + 1))
if [ ${cnt} -eq $1 ]; then
echo "Max time exceeded"
${SSH_COMMAND} ccp status
${SSH_COMMAND} fuel-ccp/tools/diagnostic-snapshot.sh -n ccp -c ccp.yml
${SCP_COMMAND} vagrant@"${ADMIN_IP}":/tmp/ccp-diag/*.tar.gz .
return 1
fi
done
echo "...................................."
echo "Jobs and pods in namespace: ccp"
${SSH_COMMAND} kubectl --namespace ccp get jobs
${SSH_COMMAND} kubectl --namespace ccp get pods
echo "openrc file: openrc-ccp"
${SSH_COMMAND} cat openrc-ccp
echo "...................................."
}
function ccp_install {
${SSH_COMMAND} "sudo -H pip install -r fuel-ccp/requirements.txt"
${SSH_COMMAND} "sudo -H pip install fuel-ccp/"
}
function deploy_ccp {
pwd
${SCP_COMMAND} ccp.yml vagrant@"${ADMIN_IP}":~/
${SSH_COMMAND} "ccp -vvv --debug --config-file ~/ccp.yml build -c etcd memcached rabbitmq galera"
${SSH_COMMAND} "ccp -vvv --debug --config-file ~/ccp.yml deploy -c etcd memcached database"
ccp_wait_for_deployment_to_finish 50
if [ $? -ne 0 ]; then
return 1
fi
${SSH_COMMAND} "ccp -vvv --debug --config-file ~/ccp.yml build -c keystone"
${SSH_COMMAND} "ccp -vvv --debug --config-file ~/ccp.yml deploy -c keystone"
ccp_wait_for_deployment_to_finish 30
if [ $? -ne 0 ]; then
return 1
fi
${SSH_COMMAND} "ccp -vvv --debug --config-file ~/ccp.yml build"
${SSH_COMMAND} "ccp -vvv --debug --config-file ~/ccp.yml deploy"
ccp_wait_for_deployment_to_finish 200
if [ $? -ne 0 ]; then
return 1
fi
}
prepare_k8s_env
# Get IP address of first node in the cluster:
@ -88,67 +345,77 @@ sleep 5
# Store info about Jenkins job on VM:
echo "${BUILD_TAG}" | ${SSH_COMMAND} "tee -a JENKINS_INFO.TXT"
# After restore snapshot ntp service doesn't work and need restart on all nodes
${SSH_COMMAND} "sudo service ntp restart"
${SSH_COMMAND} "ssh -o StrictHostKeyChecking=no node2 sudo service ntp restart"
${SSH_COMMAND} "ssh -o StrictHostKeyChecking=no node3 sudo service ntp restart"
fix_restored_env
copy_code_to_vm
# Dirty hack for workaround network problems on CI envs.
# When we deploy env some time after (few minutes) it change resolv.conf into broken one
# From this reason after bring up env we restart network and and kill dhclient,
# we also restart docker and kubelet to make sure that all net-host containers are in good shape
cat > fix_dns.sh << EOF
sudo service networking restart
sudo pkill -9 dhclient
sudo service kubelet restart
sudo service docker restart
EOF
chmod +x fix_dns.sh
${SCP_COMMAND} fix_dns.sh vagrant@"${ADMIN_IP}":~/
${SSH_COMMAND} "scp -o StrictHostKeyChecking=no fix_dns.sh vagrant@node2:~/"
${SSH_COMMAND} "scp -o StrictHostKeyChecking=no fix_dns.sh vagrant@node3:~/"
${SSH_COMMAND} "sudo ./fix_dns.sh"
${SSH_COMMAND} "ssh -o StrictHostKeyChecking=no node2 sudo ./fix_dns.sh"
${SSH_COMMAND} "ssh -o StrictHostKeyChecking=no node3 sudo ./fix_dns.sh"
# Change registry ip address to slave and set tag
sed -i 's/127.0.0.1:31500/'${REGISTRY_IP}':'${REGISTRY_PORT}'/g' fuel-ccp/tools/ccp-multi-deploy/config/ccp-configs-common.yaml
cat >> fuel-ccp/tools/ccp-multi-deploy/config/ccp-configs-common.yaml << EOF
images:
tag: "${ZUUL_CHANGE}"
EOF
# Prepare env on "admin" VM:
if [ ${COMPONENT} == "full" ];then
${SCP_COMMAND} -r fuel-ccp/ vagrant@"${ADMIN_IP}":~/
# set version of openstack
if [ "${VERSION}" == "master" ];then
export IMAGES_TAG="latest"
export BRANCH="master"
elif [ "${VERSION}" == "ocata" ];then
export IMAGES_TAG="ocata"
export BRANCH="stable/ocata"
elif [ "${VERSION}" == "newton" ]; then
export IMAGES_TAG="newton"
export BRANCH="stable/newton"
else
${SCP_COMMAND} -r fuel-ccp/ vagrant@"${ADMIN_IP}":~/
# set +x is just for security reasons to avoid publishing internal IP
set +x
getent hosts zuul.mcp.fuel-infra.org | ${SSH_COMMAND} "sudo tee -a /etc/hosts"
set -x
${SCP_COMMAND} -r containers/openstack/ vagrant@"${ADMIN_IP}":/tmp/ccp-repos
export IMAGES_TAG="mitaka"
export BRANCH="stable/mitaka"
fi
# Run CCP deployment and OpenStack tests:
${SSH_COMMAND} "pushd fuel-ccp && APT_CACHE_SERVER=http://${APT_CACHE_SERVER_IP}:${APT_CACHE_SERVER_PORT} tox -e multi-deploy -- --openstack-version ${VERSION} --number-of-envs 1 -d"
prepare_ccp_config
docker exec "${REGISTRY_HASH}" chgrp -R "${JENKINS_GID}" /var/lib/registry
docker exec "${REGISTRY_HASH}" chmod -R g+w /var/lib/registry
ccp_install
if [ ${COMPONENT} == "smoke" ]; then
set +e
# Run CCP deployment and OpenStack tests:
deploy_ccp
DEPLOY_STATUS=$?
sshpass -p vagrant ssh -o StrictHostKeyChecking=no vagrant@"${ADMIN_IP}" "echo ${DOCKER_REGISTRY_IP} ${DOCKER_REGISTRY_HOST} |sudo tee -a /etc/hosts"
set -e
#set tag dependent from test result
if [[ "${DEPLOY_STATUS}" == 0 ]]; then
DOCKER_TAG="${IMAGES_TAG}"
else
DOCKER_TAG="${IMAGES_TAG}-unstable"
fi
IMG=`sshpass -p vagrant ssh -o StrictHostKeyChecking=no vagrant@${ADMIN_IP} docker images --format "{{.Repository}}" | awk -F'/' -v search=/${IMAGES_NAMESPACE}/ '$0 ~ search {print $3}'`
# we need docker config file to authentication in remote repository
sshpass -p vagrant ssh -o StrictHostKeyChecking=no vagrant@"${ADMIN_IP}" mkdir -p /home/vagrant/.docker/
sshpass -p vagrant scp -o StrictHostKeyChecking=no /home/jenkins/.docker/config.json vagrant@"${ADMIN_IP}":~/.docker/
for f in ${IMG}; do
sshpass -p vagrant ssh -o StrictHostKeyChecking=no vagrant@"${ADMIN_IP}" \
"docker tag ${REGISTRY_IP}:${REGISTRY_PORT}/${IMAGES_NAMESPACE}/$f:latest ${DOCKER_REGISTRY}/${REGISTRY_NAMESPACE}/${f}:${DOCKER_TAG} && docker push ${DOCKER_REGISTRY}/${REGISTRY_NAMESPACE}/${f}:${DOCKER_TAG}"
if [ "${IMAGES_TAG}" == "ocata" ]; then
sshpass -p vagrant ssh -o StrictHostKeyChecking=no vagrant@"${ADMIN_IP}" \
"docker tag ${REGISTRY_IP}:${REGISTRY_PORT}/${IMAGES_NAMESPACE}/${f}:latest ${DOCKER_REGISTRY}/${REGISTRY_NAMESPACE}/${f}:latest \
&& docker push ${DOCKER_REGISTRY}/${REGISTRY_NAMESPACE}/${f}:latest"
fi
done
else
set +e
# Run CCP deployment and OpenStack tests:
deploy_ccp
DEPLOY_STATUS=$?
echo "Deploy status: ${DEPLOY_STATUS}"
set -e
fi
set +e
# download snapshot if exists
${SCP_COMMAND} vagrant@"${ADMIN_IP}":/tmp/ccp-diag/*.tar.gz .
# remove images from local registry
for i in `curl http://${REGISTRY_IP}:${REGISTRY_PORT}/v2/_catalog | jq -r '.repositories[]'`; do
REGISTRY_DATA_DIR=/home/jenkins/registry/data/docker/registry/v2/ /home/jenkins/registry/delete_docker_registry_image.py --image "${i}":"${ZUUL_CHANGE}"
REGISTRY_DATA_DIR=/home/jenkins/registry/data/docker/registry/v2/ /home/jenkins/registry/delete_docker_registry_image.py --image "${i}":"${ZUUL_CHANGE}"
done
set -e
# Revert to fresh to decrease image size
dos.py revert "${FUEL_DEVOPS_ENV_NAME}" "${FUEL_DEVOPS_SNAPSHOT_NAME}"

View File

@ -60,7 +60,10 @@
version:
- master
- newton
- mitaka
- ocata
component:
- smoke
- full
- debian-base
- etcd

View File

@ -197,8 +197,10 @@ projects:
post:
- update-jenkins-jobs
periodic:
- 'build-newton-images-smoke'
- 'build-ocata-images-smoke'
- 'fuel-ccp-smoke-master-deployment'
- 'fuel-ccp-smoke-newton-deployment'
- 'fuel-ccp-smoke-ocata-deployment'
- 'fuel-ccp-smoke-mitaka-deployment'
jobs:
- name: ^(fuel-ccp).*(newton-deployment)