Merge "New CLI option to extract hyperkube"
This commit is contained in:
commit
c326f87c25
32
doc/source/distribution.rst
Normal file
32
doc/source/distribution.rst
Normal file
@ -0,0 +1,32 @@
|
||||
Distribution
|
||||
============
|
||||
|
||||
Promenade is using Hyperkube for all Kubernetes components: kubelet, kubectl, etc.
|
||||
By default Hyperkube binary should be extracted from the image before running Promenade.
|
||||
This is done by external scripts and is not integrated into Promenade source code.
|
||||
The other way is to let Promenade do the job and extract binary. This one is more complicated,
|
||||
needs to share Docker socket inside Promenade container and is optional.
|
||||
|
||||
Default behavior
|
||||
----------------
|
||||
|
||||
IMAGE_HYPERKUBE should be exported and set to appropriate value.
|
||||
Before running build-all CLI for Promenade need to run utility container which will copy binary from image to a shared location.
|
||||
See tools/g2/stages/build-scripts.sh for reference.
|
||||
|
||||
|
||||
Integrated solution
|
||||
-------------------
|
||||
|
||||
To let Promenade extract binary need to provide more env vars and shared locations for Promenade container.
|
||||
Also need to enable option --extract-hyperkube in Promenade CLI.
|
||||
|
||||
Define var for Docker socket(it should be available for user to read/write):
|
||||
DOCKER_SOCK="/var/run/docker.sock"
|
||||
|
||||
Provide it for container:
|
||||
-v "${DOCKER_SOCK}:${DOCKER_SOCK}"
|
||||
-e "DOCKER_HOST=unix:/${DOCKER_SOCK}"
|
||||
|
||||
Provide additional var(it's for internal operations):
|
||||
-e "PROMENADE_TMP_LOCAL=/${PROMENADE_TMP_LOCAL}"
|
@ -32,6 +32,7 @@ Promenade Configuration Guide
|
||||
|
||||
developer-onboarding
|
||||
design
|
||||
distribution
|
||||
getting-started
|
||||
configuration/index
|
||||
troubleshooting/index
|
||||
|
@ -48,7 +48,7 @@ class Builder:
|
||||
if 'content' in file_spec:
|
||||
data = file_spec['content']
|
||||
elif 'docker_image' in file_spec:
|
||||
data = _fetch_image_content(self.config.container_info,
|
||||
data = _fetch_image_content(self.config,
|
||||
file_spec['docker_image'],
|
||||
file_spec['file_path'])
|
||||
elif 'symlink' in file_spec:
|
||||
@ -71,7 +71,6 @@ class Builder:
|
||||
self.config.get_path('Genesis:files', []))
|
||||
|
||||
def build_all(self, *, output_dir):
|
||||
self.config.get_container_info()
|
||||
self.build_genesis(output_dir=output_dir)
|
||||
for node_document in self.config.iterate(
|
||||
schema='promenade/KubernetesNode/v1'):
|
||||
@ -179,6 +178,7 @@ def _encrypt(cfg_dict, data):
|
||||
|
||||
|
||||
# The following environment variables should be used
|
||||
# to extract hyperkube from image:
|
||||
# export DOCKER_HOST="unix://var/run/docker.sock"
|
||||
# export PROMENADE_TMP="tmp_dir_on_host"
|
||||
# export PROMENADE_TMP_LOCAL="tmp_dir_inside_container"
|
||||
@ -187,20 +187,25 @@ def _encrypt(cfg_dict, data):
|
||||
@CACHE.cache('fetch_image', expire=72 * 3600)
|
||||
def _fetch_image_content(config, image_url, file_path):
|
||||
file_name = os.path.basename(file_path)
|
||||
if config is None:
|
||||
result_path = os.path.join(TMP_CACHE, file_name)
|
||||
if not os.path.isfile(result_path):
|
||||
raise Exception(
|
||||
'ERROR: there is no container info and no file in cache')
|
||||
else:
|
||||
result_path = os.path.join(config['dir_local'], file_name)
|
||||
client = config['client']
|
||||
vol = {config['dir']: {'bind': config['dir_local'], 'mode': 'rw'}}
|
||||
cmd = 'cp -v {} {}'.format(file_path, config['dir_local'])
|
||||
if config.extract_hyperkube:
|
||||
container_info = config.get_container_info()
|
||||
result_path = os.path.join(container_info['dir_local'], file_name)
|
||||
client = container_info['client']
|
||||
vol = {
|
||||
container_info['dir']: {
|
||||
'bind': container_info['dir_local'],
|
||||
'mode': 'rw'
|
||||
}
|
||||
}
|
||||
cmd = 'cp -v {} {}'.format(file_path, container_info['dir_local'])
|
||||
image = client.images.pull(image_url)
|
||||
output = client.containers.run(
|
||||
image, command=cmd, auto_remove=True, volumes=vol)
|
||||
LOG.debug(output)
|
||||
else:
|
||||
result_path = os.path.join(TMP_CACHE, file_name)
|
||||
if not os.path.isfile(result_path):
|
||||
raise Exception('ERROR: there is no hyperkube in cache')
|
||||
f = open(result_path, 'rb')
|
||||
return f.read()
|
||||
|
||||
|
@ -27,18 +27,25 @@ def promenade(*, verbose):
|
||||
required=True,
|
||||
help='Location to write complete cluster configuration.')
|
||||
@click.option('--validators', is_flag=True, help='Generate validation scripts')
|
||||
@click.option(
|
||||
'--extract-hyperkube',
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help='Extract hyperkube binary from image')
|
||||
@click.option(
|
||||
'--leave-kubectl',
|
||||
is_flag=True,
|
||||
help='Leave behind kubectl on joined nodes')
|
||||
@click.argument('config_files', nargs=-1, type=click.File('rb'))
|
||||
def build_all(*, config_files, leave_kubectl, output_dir, validators):
|
||||
def build_all(*, config_files, extract_hyperkube, leave_kubectl, output_dir,
|
||||
validators):
|
||||
debug = _debug()
|
||||
try:
|
||||
c = config.Configuration.from_streams(
|
||||
debug=debug,
|
||||
substitute=True,
|
||||
allow_missing_substitutions=False,
|
||||
extract_hyperkube=extract_hyperkube,
|
||||
leave_kubectl=leave_kubectl,
|
||||
streams=config_files)
|
||||
b = builder.Builder(c, validators=validators)
|
||||
|
@ -21,6 +21,7 @@ class Configuration:
|
||||
debug=False,
|
||||
substitute=True,
|
||||
allow_missing_substitutions=True,
|
||||
extract_hyperkube=True,
|
||||
leave_kubectl=False,
|
||||
validate=True):
|
||||
LOG.info("Parsing document schemas.")
|
||||
@ -39,9 +40,9 @@ class Configuration:
|
||||
raise exceptions.DeckhandException(str(e))
|
||||
|
||||
LOG.info("Deckhand engine returned %d documents." % len(documents))
|
||||
self.container_info = None
|
||||
self.debug = debug
|
||||
self.documents = documents
|
||||
self.extract_hyperkube = extract_hyperkube
|
||||
self.leave_kubectl = leave_kubectl
|
||||
|
||||
if validate:
|
||||
@ -116,6 +117,8 @@ class Configuration:
|
||||
for doc in self.iterate(*args, **kwargs):
|
||||
return doc
|
||||
|
||||
# try to use docker socket from ENV
|
||||
# supported the same way like for docker client
|
||||
def get_container_info(self):
|
||||
LOG.debug(
|
||||
'Getting access to Docker via socket and getting mount points')
|
||||
@ -123,7 +126,7 @@ class Configuration:
|
||||
try:
|
||||
client.ping()
|
||||
except:
|
||||
return
|
||||
raise Exception('Docker is not responding, check ENV vars')
|
||||
tmp_dir = os.getenv('PROMENADE_TMP')
|
||||
if tmp_dir is None:
|
||||
raise Exception('ERROR: undefined PROMENADE_TMP')
|
||||
@ -132,7 +135,7 @@ class Configuration:
|
||||
raise Exception('ERROR: undefined PROMENADE_TMP_LOCAL')
|
||||
if not os.path.exists(tmp_dir_local):
|
||||
raise Exception('ERROR: {} not found'.format(tmp_dir_local))
|
||||
self.container_info = {
|
||||
return {
|
||||
'client': client,
|
||||
'dir': tmp_dir,
|
||||
'dir_local': tmp_dir_local,
|
||||
@ -150,6 +153,7 @@ class Configuration:
|
||||
return Configuration(
|
||||
debug=self.debug,
|
||||
documents=documents,
|
||||
extract_hyperkube=self.extract_hyperkube,
|
||||
leave_kubectl=self.leave_kubectl,
|
||||
substitute=False,
|
||||
validate=False)
|
||||
@ -173,6 +177,7 @@ class Configuration:
|
||||
return Configuration(
|
||||
debug=self.debug,
|
||||
documents=documents,
|
||||
extract_hyperkube=self.extract_hyperkube,
|
||||
leave_kubectl=self.leave_kubectl,
|
||||
substitute=False,
|
||||
validate=False)
|
||||
|
@ -49,10 +49,13 @@ class JoinScriptsResource(BaseResource):
|
||||
|
||||
join_ips = _get_join_ips()
|
||||
|
||||
# extract_hyperkube is False for join script because hyperkube should
|
||||
# be extracted in the init container before running promenade
|
||||
try:
|
||||
config = Configuration.from_design_ref(
|
||||
design_ref,
|
||||
allow_missing_substitutions=False,
|
||||
extract_hyperkube=False,
|
||||
leave_kubectl=leave_kubectl)
|
||||
except exceptions.DeckhandException:
|
||||
LOG.exception('Caught Deckhand render error for configuration')
|
||||
|
@ -2,11 +2,12 @@ export TEMP_DIR=${TEMP_DIR:-$(mktemp -d)}
|
||||
export BASE_IMAGE_SIZE=${BASE_IMAGE_SIZE:-68719476736}
|
||||
export BASE_IMAGE_URL=${BASE_IMAGE_URL:-https://cloud-images.ubuntu.com/releases/16.04/release/ubuntu-16.04-server-cloudimg-amd64-disk1.img}
|
||||
export IMAGE_PROMENADE=${IMAGE_PROMENADE:-quay.io/airshipit/promenade:master}
|
||||
export IMAGE_HYPERKUBE=${IMAGE_HYPERKUBE:-gcr.io/google_containers/hyperkube-amd64:v1.11.6}
|
||||
export NGINX_DIR="${TEMP_DIR}/nginx"
|
||||
export NGINX_URL="http://192.168.77.1:7777"
|
||||
export PROMENADE_BASE_URL="http://promenade-api.ucp.svc.cluster.local"
|
||||
export PROMENADE_DEBUG=${PROMENADE_DEBUG:-0}
|
||||
export PROMENADE_TMP_LOCAL=${PROMENADE_TMP_LOCAL:-tmp_bin}
|
||||
export PROMENADE_TMP_LOCAL=${PROMENADE_TMP_LOCAL:-cache}
|
||||
export PROMENADE_ENCRYPTION_KEY=${PROMENADE_ENCRYPTION_KEY:-testkey}
|
||||
export REGISTRY_DATA_DIR=${REGISTRY_DATA_DIR:-/mnt/registry}
|
||||
export VIRSH_POOL=${VIRSH_POOL:-promenade}
|
||||
|
@ -13,18 +13,17 @@ PROMENADE_TMP="${TEMP_DIR}/${PROMENADE_TMP_LOCAL}"
|
||||
mkdir -p "$PROMENADE_TMP"
|
||||
chmod 777 "$PROMENADE_TMP"
|
||||
|
||||
DOCKER_SOCK="/var/run/docker.sock"
|
||||
sudo chmod o+rw $DOCKER_SOCK
|
||||
log Prepare hyperkube
|
||||
docker run --rm -t \
|
||||
-v "${PROMENADE_TMP}:/tmp/${PROMENADE_TMP_LOCAL}" \
|
||||
"${IMAGE_HYPERKUBE}" \
|
||||
cp /hyperkube "/tmp/${PROMENADE_TMP_LOCAL}"
|
||||
|
||||
log Building scripts
|
||||
docker run --rm -t \
|
||||
-w /target \
|
||||
-v "${TEMP_DIR}:/target" \
|
||||
-v "${PROMENADE_TMP}:/${PROMENADE_TMP_LOCAL}" \
|
||||
-v "${DOCKER_SOCK}:${DOCKER_SOCK}" \
|
||||
-e "DOCKER_HOST=unix:/${DOCKER_SOCK}" \
|
||||
-e "PROMENADE_TMP=${PROMENADE_TMP}" \
|
||||
-e "PROMENADE_TMP_LOCAL=/${PROMENADE_TMP_LOCAL}" \
|
||||
-v "${PROMENADE_TMP}:/tmp/${PROMENADE_TMP_LOCAL}" \
|
||||
-e "PROMENADE_DEBUG=${PROMENADE_DEBUG}" \
|
||||
-e "PROMENADE_ENCRYPTION_KEY=${PROMENADE_ENCRYPTION_KEY}" \
|
||||
"${IMAGE_PROMENADE}" \
|
||||
@ -33,5 +32,3 @@ docker run --rm -t \
|
||||
--validators \
|
||||
-o scripts \
|
||||
config/*.yaml
|
||||
|
||||
sudo chmod o-rw $DOCKER_SOCK
|
||||
|
@ -3,6 +3,7 @@
|
||||
set -eux
|
||||
|
||||
IMAGE_PROMENADE=${IMAGE_PROMENADE:-quay.io/airshipit/promenade:master}
|
||||
IMAGE_HYPERKUBE=${IMAGE_HYPERKUBE:-gcr.io/google_containers/hyperkube-amd64:v1.11.6}
|
||||
PROMENADE_DEBUG=${PROMENADE_DEBUG:-0}
|
||||
|
||||
SCRIPT_DIR=$(realpath $(dirname $0))
|
||||
@ -29,9 +30,6 @@ PROMENADE_TMP="${SCRIPT_DIR}/${PROMENADE_TMP_LOCAL}"
|
||||
mkdir -p "$PROMENADE_TMP"
|
||||
chmod 777 "$PROMENADE_TMP"
|
||||
|
||||
DOCKER_SOCK="/var/run/docker.sock"
|
||||
sudo chmod o+rw $DOCKER_SOCK
|
||||
|
||||
cp "${CONFIG_SOURCE}"/*.yaml ${BUILD_DIR}
|
||||
|
||||
if [ ${REPLACE} == 'replace' ]
|
||||
@ -59,6 +57,12 @@ docker run --rm -t \
|
||||
fi
|
||||
|
||||
if [[ -z $1 ]] || [[ $1 = build-all ]]; then
|
||||
echo === Prepare hyperkube ===
|
||||
docker run --rm -t \
|
||||
-v "${PROMENADE_TMP}:/tmp/${PROMENADE_TMP_LOCAL}" \
|
||||
"${IMAGE_HYPERKUBE}" \
|
||||
cp /hyperkube "/tmp/${PROMENADE_TMP_LOCAL}"
|
||||
|
||||
echo === Building bootstrap scripts ===
|
||||
docker run --rm -t \
|
||||
-w /target \
|
||||
@ -66,11 +70,7 @@ docker run --rm -t \
|
||||
-e http_proxy=${HTTP_PROXY} \
|
||||
-e https_proxy=${HTTPS_PROXY} \
|
||||
-e no_proxy=${NO_PROXY} \
|
||||
-v "${PROMENADE_TMP}:/${PROMENADE_TMP_LOCAL}" \
|
||||
-v "${DOCKER_SOCK}:${DOCKER_SOCK}" \
|
||||
-e "DOCKER_HOST=unix:/${DOCKER_SOCK}" \
|
||||
-e "PROMENADE_TMP=${PROMENADE_TMP}" \
|
||||
-e "PROMENADE_TMP_LOCAL=/${PROMENADE_TMP_LOCAL}" \
|
||||
-v "${PROMENADE_TMP}:/tmp/${PROMENADE_TMP_LOCAL}" \
|
||||
-v ${BUILD_DIR}:/target \
|
||||
${IMAGE_PROMENADE} \
|
||||
promenade \
|
||||
@ -80,6 +80,4 @@ docker run --rm -t \
|
||||
$(ls ${BUILD_DIR})
|
||||
fi
|
||||
|
||||
sudo chmod o-rw $DOCKER_SOCK
|
||||
|
||||
echo === Done ===
|
||||
|
@ -16,7 +16,7 @@
|
||||
HTTP_PROXY: ""
|
||||
HTTPS_PROXY: ""
|
||||
NO_PROXY: ""
|
||||
PROMENADE_TMP_LOCAL: "tmp_bin"
|
||||
PROMENADE_TMP_LOCAL: "cache"
|
||||
become: true
|
||||
tasks:
|
||||
- name: Install docker
|
||||
|
Loading…
Reference in New Issue
Block a user