Add LOCI to OSH images

This adds the LOCI build progress into OSH-images.

Change-Id: Iffc12af6ef8fff82ca1338c528ea427ef2fb0e8a
This commit is contained in:
Jean-Philippe Evrard 2018-11-30 17:14:14 +01:00
parent 1eb2e14491
commit b1132137fe
10 changed files with 522 additions and 6 deletions

View File

@ -8,10 +8,24 @@ the LOCI images by default.
Mechanism used
==============
The images are built by passing an extra series of bindep
in the LOCI build system.
Currently, we are passing arguments to the loci build,
which is enough to customize the build system.
Code
====
LOCI build process is a relatively staged process:
The code to pass and build the image can be found in <TODO>
1. Build (or re-use) a base image
2. Build a requirements image, building wheels.
3. Build the 'project' image, re-using requirements.
Code and parameters
===================
OpenStack-Helm-Images can build multiple OpenStack images based on LOCI.
By default, OpenStack-Helm-Image has one `build.sh` script, in the
`openstack/loci/` folder.
For convenience, default overrides per OpenStack branch are provided in
the same folder:
`build-newton.sh` builds an OpenStack newton image, `build-ocata.sh` builds
an ocata image, and so on.

44
openstack/loci/build-newton.sh Executable file
View File

@ -0,0 +1,44 @@
#!/bin/bash
SCRIPT=`realpath $0`
SCRIPT_DIR=`dirname ${SCRIPT}`
## Only build from main folder
# Replace with Registry URI with your registry like your
# dockerhub user. Example: "docker.io/openstackhelm"
VERSION=${VERSION:-"latest"}
OPENSTACK_VERSION="newton-eol"
#Defaults
requirements_project_ref="stable/newton"
keystone_profiles=${keystone_profiles:-"'apache ldap'"}
keystone_pip_packages=${keystone_pip_packages:-"'pycrypto python-openstackclient'"}
heat_profiles=${heat_profiles:-"'apache'"}
heat_pip_packages=${heat_pip_packages:-"pycrypto"}
barbican_pip_packages=${barbican_pip_packages:-"pycrypto"}
glance_profiles=${glance_profiles:-"'glance ceph'"}
glance_pip_packages=${glance_pip_packages:-"'pycrypto python-swiftclient'"}
cinder_profiles=${cinder_profiles:-"'cinder lvm ceph qemu'"}
cinder_pip_packages=${cinder_pip_packages:-"'pycrypto python-swiftclient'"}
neutron_profiles=${neutron_profiles:-"'neutron linuxbridge openvswitch'"}
neutron_pip_packages=${neutron_pip_packages:-"pycrypto"}
nova_profiles=${nova_profiles:-"'nova ceph linuxbridge openvswitch configdrive qemu apache'"}
nova_pip_packages=${nova_pip_packages:-"pycrypto"}
horizon_profiles=${horizon_profiles:-"'horizon apache'"}
horizon_pip_packages=${horizon_pip_packages:-"pycrypto"}
senlin_profiles=${senlin_profiles:-"'senlin'"}
senlin_pip_packages=${senlin_pip_packages:-"pycrypto"}
congress_profiles=${congress_profiles:-"'congress'"}
congress_pip_packages=${congress_pip_packages:-"'pycrypto python-congressclient'"}
magnum_profiles=${magnum_profiles:-"'magnum'"}
magnum_pip_packages=${magnum_pip_packages:-"pycrypto"}
ironic_profiles=${ironic_profiles:-"'ironic ipxe ipmi qemu tftp'"}
ironic_pip_packages=${ironic_pip_packages:-"pycrypto"}
ironic_dist_packages=${ironic_dist_packages:-"iproute2"}
neutron_sriov_from=${neutron_sriov_from:-${neutron_sriov_from:-"docker.io/ubuntu:18.04"}}
neutron_sriov_project=${neutron_sriov_project:-"neutron"}
neutron_sriov_profiles=${neutron_sriov_profiles:-"'neutron linuxbridge openvswitch'"}
neutron_sriov_pip_packages=${neutron_sriov_pip_packages:-"pycrypto"}
neutron_sriov_dist_packages=${neutron_sriov_dist_packages:-"'ethtool lshw'"}
neutron_sriov_extra_tag=${neutron_sriov_extra_tag:-'-sriov-1804'}
BUILD_PROJECTS=${BUILD_PROJECTS:-'requirements keystone heat barbican glance cinder neutron neutron_sriov nova horizon senlin congress magnum ironic'}
source ${SCRIPT_DIR}/build.sh

10
openstack/loci/build-ocata.sh Executable file
View File

@ -0,0 +1,10 @@
#!/bin/bash
SCRIPT=`realpath $0`
SCRIPT_DIR=`dirname ${SCRIPT}`
## Only build from main folder
# Replace with Registry URI with your registry like your
# dockerhub user. Example: "docker.io/openstackhelm"
VERSION=${VERSION:-"latest"}
OPENSTACK_VERSION="stable/ocata"
source ${SCRIPT_DIR}/build.sh

10
openstack/loci/build-pike.sh Executable file
View File

@ -0,0 +1,10 @@
#!/bin/bash
SCRIPT=`realpath $0`
SCRIPT_DIR=`dirname ${SCRIPT}`
## Only build from main folder
# Replace with Registry URI with your registry like your
# dockerhub user. Example: "docker.io/openstackhelm"
VERSION=${VERSION:-"latest"}
OPENSTACK_VERSION="stable/pike"
source ${SCRIPT_DIR}/build.sh

10
openstack/loci/build-queens.sh Executable file
View File

@ -0,0 +1,10 @@
#!/bin/bash
SCRIPT=`realpath $0`
SCRIPT_DIR=`dirname ${SCRIPT}`
## Only build from main folder
# Replace with Registry URI with your registry like your
# dockerhub user. Example: "docker.io/openstackhelm"
VERSION=${VERSION:-"latest"}
OPENSTACK_VERSION="stable/queens"
source ${SCRIPT_DIR}/build.sh

274
openstack/loci/build.sh Executable file
View File

@ -0,0 +1,274 @@
#!/bin/bash
export PYTHONUNBUFFERED=1
SCRIPT=`realpath $0`
SCRIPT_DIR=`dirname ${SCRIPT}`
## Only build from main folder
cd ${SCRIPT_DIR}/../..
set -e
function get_loci {
#LOCI_SRC_DIR should be set in upstream gates.
#This allows Depends-On patches to LOCI to be built here.
if [[ -z ${LOCI_SRC_DIR+x} ]]; then
echo "LOCI_SRC_DIR unset, cloning in temp folder"
temp_dir=$(mktemp -d)
LOCI_SRC_DIR=${temp_dir}/loci
git clone ${LOCI_CLONE_LOCATION:-"https://git.openstack.org/openstack/loci.git"} ${LOCI_SRC_DIR}
else
echo "LOCI_SRC_DIR set, reusing LOCI folder"
fi
}
function fetch_loci_version {
if [[ ! -z ${LOCI_SHA+x} ]]; then
pushd ${LOCI_SRC_DIR}
git fetch ${FETCH_REPO:-"https://git.openstack.org/openstack/loci.git"} ${FETCH_REFSPEC:-"master"}
git checkout FETCH_HEAD
popd
fi
}
function build_loci_base_image {
base_img_tag=$1
pushd ${LOCI_SRC_DIR}
docker build --network=host -t ${base_img_tag} ${base_extra_build_args} dockerfiles/${BASE_IMAGE}
popd
}
function fetch_base_image {
docker pull ${BASE_IMAGE}
}
function get_project_image_build_arguments {
project=$1
echo "Building $project"
#Evaluate all LOCI arguments to pass, in the
#LOCI documentation order.
#Some projects might want to override FROM image
local this_from="${project}_from"
if [[ -n ${!this_from} ]]; then
build_args="--build-arg FROM=${!this_from}"
else
build_args="--build-arg FROM=${LOCI_ARG_FROM}"
fi
#Assuming you want to build multiple images for an upstream project
#you can define multiple 'projects' and override default upstream
#project name. See for example neutron and neutron_sriov.
local this_project="${project}_project"
if [[ -n ${!this_project} ]]; then
echo "Override of projectname found"
local projectname=${!this_project}
else
local projectname=${project}
fi
build_args="${build_args} --build-arg PROJECT=${projectname}"
#Add PROJECT_REF argument if <project>_project_ref env var is defined
#Or, use default openstack branch reference "OPENSTACK_VERSION"
local this_project_ref="${project}_project_ref"
if [[ -n ${!this_project_ref} ]]; then
PROJECT_REF=${!this_project_ref}
else
PROJECT_REF=${OPENSTACK_VERSION}
fi
#Remove stable/ from the tags, as '/' should not be in tag, and
#'stable' doesn't bring any extra information
TAGGED_PROJECT_REF=${PROJECT_REF/stable\//}
build_args="${build_args} --build-arg PROJECT_REF=${PROJECT_REF}"
#Add PROJECT_REPO argument if <project>_project_repo env var is defined
local this_project_repo="${project}_project_repo"
if [[ -n ${!this_project_repo} ]]; then
build_args="${build_args} --build-arg PROJECT_REPO=${!this_project_repo}"
fi
if [[ "$project" == "requirements" ]]; then
# Ensure all the next builds can refer to a global
# "REQUIREMENTS_TAGGED_PROJECT_REF"
REQUIREMENTS_TAGGED_PROJECT_REF=${TAGGED_PROJECT_REF}
else
#Add uid argument if <project>_uid env var is defined
local this_uid="${project}_uid"
if [[ -n ${!this_uid} ]]; then
build_args="${build_args} --build-arg UID=${!this_uid}"
fi
#Add gid argument if <project>_gid env var is defined
local this_gid="${project}_gid"
if [[ -n ${!this_gid} ]]; then
build_args="${build_args} --build-arg GID=${!this_gid}"
fi
#Point to requirement wheels, or use <project>_wheels
# if defined.
local this_wheels="${project}_wheels"
if [[ -n ${!this_wheels} ]]; then
build_args="${build_args} --build-arg WHEELS=${!this_wheels}"
else
build_args="${build_args} --build-arg WHEELS=${REGISTRY_URI}requirements:${VERSION}-${REQUIREMENTS_TAGGED_PROJECT_REF}-${DISTRO}${requirements_extra_tags:-}"
fi
#Add profiles argument if <project>_profiles env var is defined
local this_profiles="${project}_profiles"
if [[ -n ${!this_profiles} ]]; then
build_args="${build_args} --build-arg PROFILES=${!this_profiles}"
fi
#Add pip_packages argument if <project>_pip_packages env var is defined
local this_pip_packages="${project}_pip_packages"
if [[ -n ${!this_pip_packages} ]]; then
build_args="${build_args} --build-arg PIP_PACKAGES=${!this_pip_packages}"
fi
#Add pip_args argument if <project>_pip_args env var is defined
local this_pip_args="${project}_pip_args"
if [[ -n ${!this_pip_args} ]]; then
build_args="${build_args} --build-arg PIP_ARGS=${!this_pip_args}"
fi
#Add dist_packages argument if <project>_dist_packages env var is defined
local this_dist_packages="${project}_dist_packages"
if [[ -n ${!this_dist_packages} ]]; then
build_args="${build_args} --build-arg DIST_PACKAGES=${!this_dist_packages}"
fi
fi
#Add extra_build_args argument if <project>_extra_build_args env var is defined
local this_extra_build_args="${project}_extra_build_args"
if [[ -n ${!this_extra_build_args} ]]; then
build_args="${build_args} ${!this_extra_build_args}"
fi
#Prepare tag
local this_extra_tags="${project}_extra_tag"
tag="${REGISTRY_URI}${projectname}:${VERSION}-${TAGGED_PROJECT_REF}-${DISTRO}${!this_extra_tags}"
docker_build_cmd="docker build --network=host ${default_project_extra_build_args} ${build_args} --tag $tag ."
}
# Default script behavior
#
# BASE_IMAGE represents LOCI's "base" image name.
# Use ubuntu|leap15|centos|debian to build base image from LOCI's Dockerfiles.
BASE_IMAGE=${BASE_IMAGE:-"gcr.io/google_containers/ubuntu-slim:0.14"}
# Replace with Registry URI with your registry like your
# dockerhub user. Example: "docker.io/openstackhelm"
REGISTRY_URI=${REGISTRY_URI:-"172.17.0.1:5000/openstackhelm/"}
# The image tag used.
VERSION=${VERSION:-"latest"}
# The openstack branch to build, if no per project branch is given.
OPENSTACK_VERSION=${OPENSTACK_VERSION:-"master"}
# extra build arguments for the base image. See loci's dockerfiles for
# arguments that could be used for example.
base_extra_build_args=${base_extra_build_args:-"--force-rm --pull --no-cache"}
# you can use default_project_extra_build_args for proxies.
default_project_extra_build_args=${default_project_extra_build_args:-"--force-rm --pull --no-cache"}
#Log location
LOG_PREFIX="/tmp/loci-log-"
#Defaults for projects
keystone_profiles=${keystone_profiles:-"'fluent apache ldap'"}
keystone_pip_packages=${keystone_pip_packages:-"'pycrypto python-openstackclient'"}
heat_profiles=${heat_profiles:-"'fluent apache'"}
heat_pip_packages=${heat_pip_packages:-"pycrypto"}
barbican_profiles=${barbican_profiles:-"fluent"}
barbican_pip_packages=${barbican_pip_packages:-"pycrypto"}
glance_profiles=${glance_profiles:-"'fluent glance ceph'"}
glance_pip_packages=${glance_pip_packages:-"'pycrypto python-swiftclient'"}
cinder_profiles=${cinder_profiles:-"'fluent cinder lvm ceph qemu'"}
cinder_pip_packages=${cinder_pip_packages:-"'pycrypto python-swiftclient'"}
neutron_profiles=${neutron_profiles:-"'fluent neutron linuxbridge openvswitch'"}
neutron_pip_packages=${neutron_pip_packages:-"pycrypto"}
nova_profiles=${nova_profiles:-"'fluent nova ceph linuxbridge openvswitch configdrive qemu apache'"}
nova_pip_packages=${nova_pip_packages:-"pycrypto"}
horizon_profiles=${horizon_profiles:-"'fluent horizon apache'"}
horizon_pip_packages=${horizon_pip_packages:-"pycrypto"}
senlin_profiles=${senlin_profiles:-"'fluent senlin'"}
senlin_pip_packages=${senlin_pip_packages:-"pycrypto"}
congress_profiles=${congress_profiles:-"'fluent congress'"}
congress_pip_packages=${congress_pip_packages:-"'pycrypto python-congressclient'"}
magnum_profiles=${magnum_profiles:-"'fluent magnum'"}
magnum_pip_packages=${magnum_pip_packages:-"pycrypto"}
ironic_profiles=${ironic_profiles:-"'fluent ironic ipxe ipmi qemu tftp'"}
ironic_pip_packages=${ironic_pip_packages:-"pycrypto"}
ironic_dist_packages=${ironic_dist_packages:-"iproute2"}
neutron_sriov_from=${neutron_sriov_from:-${neutron_sriov_from:-"docker.io/ubuntu:18.04"}}
neutron_sriov_project=${neutron_sriov_project:-"neutron"}
neutron_sriov_profiles=${neutron_sriov_profiles:-"'fluent neutron linuxbridge openvswitch'"}
neutron_sriov_pip_packages=${neutron_sriov_pip_packages:-"pycrypto"}
neutron_sriov_dist_packages=${neutron_sriov_dist_packages:-"'ethtool lshw'"}
neutron_sriov_extra_tag=${neutron_sriov_extra_tag:-'-sriov-1804'}
####################
# Action starts here
####################
# Ensure path to registry ends with /
if [[ "${REGISTRY_URI}" != */ ]]; then
REGISTRY_URI="$REGISTRY_URI/"
fi
get_loci
fetch_loci_version
# The BASE_IMAGE provided by the user may require
# building and re-use LOCI.
# Test if BASE_IMAGE should be built from LOCI dockerfiles.
case ${BASE_IMAGE} in
ubuntu)
#Mark the need to build image from LOCI Dockerfiles
BUILD_IMAGE="yes"
#Makes sure the name of the distro is consistent with other OSH-images.
DISTRO="ubuntu_xenial"
;;
leap15)
BUILD_IMAGE="yes"
DISTRO="suse_15"
;;
centos)
BUILD_IMAGE="yes"
DISTRO="centos_7"
;;
debian)
BUILD_IMAGE="yes"
DISTRO="debian"
;;
*)
BUILD_IMAGE="no"
DISTRO="ubuntu_xenial"
;;
esac
if [[ "${BUILD_IMAGE}" == "yes" ]]; then
LOCI_ARG_FROM="${REGISTRY_URI}base:${VERSION}-${DISTRO}"
build_loci_base_image $LOCI_ARG_FROM
docker push $LOCI_ARG_FROM
else
fetch_base_image
LOCI_ARG_FROM="${BASE_IMAGE}"
fi
BUILD_PROJECTS=${BUILD_PROJECTS:-'requirements keystone heat barbican glance cinder neutron neutron_sriov nova horizon senlin congress magnum ironic'}
projects=( ${BUILD_PROJECTS} )
pushd ${LOCI_SRC_DIR}
# The first project should be requirements, if requirements is built.
# This one should not be run in parallel.
get_project_image_build_arguments ${projects[0]}
eval "${docker_build_cmd}"
docker push ${tag}
unset projects[0]
# Run the rest of the projects with parallel
for project in ${projects[@]}; do
get_project_image_build_arguments $project
echo "${docker_build_cmd} && docker push ${tag}" >> ${LOG_PREFIX}actions
done
parallel --group -a ${LOG_PREFIX}actions
popd
# Return to user folder
cd -

View File

@ -0,0 +1,89 @@
---
# Copyright 2018, SUSE LINUX GmbH.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- project:
check:
jobs:
- openstack-helm-images-openstack-loci-newton-ubuntu_xenial:
voting: false
- openstack-helm-images-openstack-loci-ocata-ubuntu_xenial
- openstack-helm-images-openstack-loci-pike-ubuntu_xenial
gate:
jobs:
- openstack-helm-images-openstack-loci-ocata-ubuntu_xenial
- openstack-helm-images-openstack-loci-pike-ubuntu_xenial
experimental:
jobs:
- openstack-helm-images-openstack-loci-queens-ubuntu_xenial
# - openstack-helm-infra-five-ubuntu
- job:
name: openstack-helm-images-openstack-loci
parent: openstack-helm-images-base
abstract: true
vars:
use_tmpfs: False
files:
- ^openstack/loci/.*
- zuul.d/openstack-loci.yaml
required-projects:
- name: openstack/loci
pre-run:
- zuul.d/playbooks/pre-loci-build.yml
run: zuul.d/playbooks/loci-build.yml
- job:
name: openstack-helm-images-openstack-loci-newton
parent: openstack-helm-images-openstack-loci
abstract: true
vars:
build_command: './build-newton.sh'
- job:
name: openstack-helm-images-openstack-loci-newton-ubuntu_xenial
parent: openstack-helm-images-openstack-loci-newton
- job:
name: openstack-helm-images-openstack-loci-ocata
parent: openstack-helm-images-openstack-loci
abstract: true
vars:
build_command: './build-ocata.sh'
- job:
name: openstack-helm-images-openstack-loci-ocata-ubuntu_xenial
parent: openstack-helm-images-openstack-loci-ocata
- job:
name: openstack-helm-images-openstack-loci-pike
parent: openstack-helm-images-openstack-loci
abstract: true
vars:
build_command: './build-pike.sh'
- job:
name: openstack-helm-images-openstack-loci-pike-ubuntu_xenial
parent: openstack-helm-images-openstack-loci-pike
- job:
name: openstack-helm-images-openstack-loci-queens
parent: openstack-helm-images-openstack-loci
abstract: true
vars:
build_command: './build-queens.sh'
- job:
name: openstack-helm-images-openstack-loci-queens-ubuntu_xenial
parent: openstack-helm-images-openstack-loci-queens

View File

@ -0,0 +1,17 @@
---
# This play will only get consumed in osh-images.
# For depends-on to osh-images and osh changes triggering a rebuild of
# osh-infra or osh, this play won't be used, as we won't know what to build, and
# therefore will build everything for a 'standard' distribution.
# No need to change zuul.project.src_dir to static paths then.
- hosts: all[0]
vars:
default_environment:
LOCI_SRC_DIR: "{{ ansible_user_dir }}/src/git.openstack.org/openstack/loci"
tasks:
- name: Build image
shell: "{{ build_command | default('./build.sh') }}"
args:
chdir: "{{ zuul.project.src_dir }}/openstack/loci/"
executable: /bin/bash
environment: "{{ (build_environment | default({})) | combine(default_environment) }}"

View File

@ -0,0 +1,47 @@
---
- hosts: all[0]
vars:
dockerconf_path: "/etc/docker/daemon.json"
dockerconf_extra_config:
debug: True
insecure-registries:
- 172.17.0.1:5000
become: true
tasks:
# LOCI needs a local registry -- Add docker insecure mode
# First two tasks can be removed when the feature is added
# in upstream role: https://review.openstack.org/#/c/624484/3
- name: Get docker configuration
slurp:
src: "{{ dockerconf_path }}"
register: dockerconf
- name: Add insecure registries into dockerconf
copy:
content: "{{ ( ((dockerconf.content | b64decode) | from_json) | combine(dockerconf_extra_config) )| to_json }}"
dest: "{{ dockerconf_path }}"
- name: Ensure docker service is restarted
service:
name: docker.service
state: restarted
- name: Run docker registry
docker_container:
name: registry
image: registry:2
state: started
published_ports:
- 5000:5000
- name: Allow connections from containers to registry
iptables:
action: insert
chain: INPUT
in_interface: docker0
jump: ACCEPT
- name: Ensure parallel is installed
package:
name: parallel
state: present

View File

@ -14,6 +14,7 @@
opts: size=25g
state: mounted
become: True
when: (use_tmpfs | default(True)) | bool
roles:
- role: install-docker
post_tasks:
@ -28,4 +29,4 @@
become: true
service:
name: docker.service
state: started
state: started