x
/
pbrx
Code Issues Proposed changes

Merge "Fix the alpine build"

This commit is contained in:
Zuul 2018-07-16 20:37:40 +00:00 committed by Gerrit Code Review
parent 3394531fab 1f0bf4659d
commit af52fe659c
1 changed files with 74 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
pbrx/container_images.py
View File

@ -53,7 +53,7 @@ class ContainerContext(object):
self._base = base self._base = base
self._volumes = volumes or [] self._volumes = volumes or []
self.run_id = self.create() self.run_id = self.create()
self._cont = sh.docker.bake("exec", self.run_id, "bash", "-c") self._cont = sh.docker.bake("exec", self.run_id, "sh", "-c")
def create(self): def create(self):
vargs = [ vargs = [
@ -64,15 +64,12 @@ class ContainerContext(object):
"{}:/usr/src".format(os.path.abspath(os.curdir)), "{}:/usr/src".format(os.path.abspath(os.curdir)),
"-w", "-w",
"/usr/src", "/usr/src",
"-v",
"{}:/root/.cache/pip/wheels".format(
os.path.expanduser("~/.cache/pip/wheels")),
] ]
for vol in self._volumes: for vol in self._volumes:
vargs.append("-v") vargs.append("-v")
vargs.append(vol) vargs.append(vol)
vargs.append(self._base) vargs.append(self._base)
vargs.append("bash") vargs.append("sh")
container_id = sh.docker(*vargs).strip() container_id = sh.docker(*vargs).strip()
return sh.docker('start', container_id).strip() return sh.docker('start', container_id).strip()
@ -102,9 +99,6 @@ def docker_container(base, tag=None, prefix=None, comment=None, volumes=None):
container = ContainerContext(base, volumes) container = ContainerContext(base, volumes)
yield container yield container
# Make sure wheels made in the container are owned by the current user
container.run("chown -R {uid} /root/.cache/pip/wheels".format(
uid=os.getuid()))
if tag: if tag:
container.commit(tag, prefix=prefix, comment=comment) container.commit(tag, prefix=prefix, comment=comment)
sh.docker.rm("-f", container.run_id) sh.docker.rm("-f", container.run_id)
@ -142,6 +136,7 @@ def build(args):
"bindep", "bindep",
"-b", "-b",
) )
log.debug(packages)
except sh.ErrorReturnCode_1 as e: except sh.ErrorReturnCode_1 as e:
packages = e.stdout.decode('utf-8').strip() packages = e.stdout.decode('utf-8').strip()
@ -156,6 +151,7 @@ def build(args):
"-b", "-b",
"compile", "compile",
) )
log.debug(compile_packages)
except sh.ErrorReturnCode_1 as e: except sh.ErrorReturnCode_1 as e:
compile_packages = e.stdout.decode('utf-8').strip() compile_packages = e.stdout.decode('utf-8').strip()
packages = packages.replace("\r", "\n").replace("\n", " ") packages = packages.replace("\r", "\n").replace("\n", " ")
@ -165,16 +161,36 @@ def build(args):
with tempfile.TemporaryDirectory( with tempfile.TemporaryDirectory(
dir=os.path.abspath(os.curdir) dir=os.path.abspath(os.curdir)
) as tmpdir: ) as tmpdir:
tmp_volume = "{tmpdir}:/tmp/output".format(tmpdir=tmpdir) # Pass in the directory into ~/.cache/pip so that the built wheel
# cache will persist into the next container. The real wheel cache
# will go there- but we'll also put our built wheel into place there.
tmp_volume = "{tmpdir}:/root/.cache/pip".format(tmpdir=tmpdir)
# Make temporary container that installs all deps to build wheel # Make temporary container that installs all deps to build wheel
# This container also needs git installed for pbr # This container also needs git installed for pbr
log.info("Build wheels in python-base container") log.info("Build wheels in python-base container")
with docker_container("python-base", volumes=[tmp_volume]) as cont: with docker_container("python-base", volumes=[tmp_volume]) as cont:
# Make sure wheel cache dir is owned by container user
cont.run("chown -R $(whoami) /root/.cache/pip")
# Add the compile dependencies
cont.run("apk add {compile_packages} git".format( cont.run("apk add {compile_packages} git".format(
compile_packages=compile_packages)) compile_packages=compile_packages))
cont.run("python setup.py bdist_wheel -d /tmp/output")
cont.run("chmod -R ugo+w /tmp/output") # Build a wheel so that we have an install target.
# pip install . in the container context with the mounted
# source dir gets ... exciting.
cont.run("python setup.py bdist_wheel -d /root/.cache/pip")
log.debug(cont.run("find /root/.cache/pip"))
log.debug(cont.run("ls -ltra /root/.cache"))
# Install with all container-related extras so that we populate
# the wheel cache as needed.
cont.run(
"pip install"
" $(echo /root/.cache/pip/*.whl)[{base},{scripts}]".format(
base=info.base_container.replace('-', '_'),
scripts=','.join(info.scripts).replace('-', '_')))
# Build the final base container. Use dumb-init as the entrypoint so # Build the final base container. Use dumb-init as the entrypoint so
# that signals and subprocesses work properly. # that signals and subprocesses work properly.
@ -184,37 +200,59 @@ def build(args):
tag=info.base_container, tag=info.base_container,
prefix=args.prefix, prefix=args.prefix,
volumes=[tmp_volume], volumes=[tmp_volume],
comment='ENTRYPOINT ["/usr/local/bin/dumb-init", "--"]', comment='ENTRYPOINT ["/usr/bin/dumb-init", "--"]',
) as cont: ) as cont:
try: try:
cont.run( cont.run(
"apk add {packages} dumb-init".format(packages=packages) "apk add {packages} dumb-init".format(packages=packages)
) )
cont.run("pip install -r requirements.txt") wheel_list = cont.run("ls /root/.cache/pip/*whl")
log.debug("Wheels: {wheel_list}".format(wheel_list=wheel_list))
log.debug(cont.run("find /root/.cache/pip"))
cont.run(
"pip install"
" $(echo /root/.cache/pip/*.whl)[{base}]".format(
base=info.base_container.replace('-', '_')))
# chown wheel cache back so the temp dir can delete it
cont.run("chown -R {uid} /root/.cache/pip".format(
uid=os.getuid()))
except Exception as e: except Exception as e:
print(e.stdout) print(e.stdout)
raise raise
# Build a container for each program. # Build a container for each program.
# In the simple-case, it's just an entrypoint commit setting CMD. # In the simple-case, it's just an entrypoint commit setting CMD.
# If a Dockerfile exists for the program, use it instead. # If a Dockerfile exists for the program, use it instead.
# Such a Dockerfile should use: # Such a Dockerfile should use:
# FROM {{ base_container }}-base # FROM {{ base_container }}-base
# This is useful for things like zuul-executor where the full story is not # This is useful for things like zuul-executor where the full story is
# possible to express otherwise. # not possible to express otherwise.
for script in info.scripts: for script in info.scripts:
dockerfile = "Dockerfile.{script}".format(script=script) dockerfile = "Dockerfile.{script}".format(script=script)
if os.path.exists(dockerfile): if os.path.exists(dockerfile):
log.info("Building container for {script} from Dockerfile".format( log.info(
script=script)) "Building container for {script} from Dockerfile".format(
sh.docker.build("-f", dockerfile, "-t", script, ".") script=script))
else: sh.docker.build("-f", dockerfile, "-t", script, ".")
log.info("Building container for {script}".format(script=script)) else:
with docker_container( log.info(
info.base_container, "Building container for {script}".format(script=script))
prefix=args.prefix, with docker_container(
comment='CMD ["/usr/local/bin/{script}"]'.format( info.base_container,
script=script prefix=args.prefix,
), volumes=[tmp_volume],
): comment='CMD ["/usr/local/bin/{script}"]'.format(
pass script=script
),
) as cont:
cont.run(
"pip install"
" $(echo /root/.cache/pip/*.whl)[{script}]".format(
script=script.replace('-', '_')))
# chown wheel cache back so the temp dir can delete it
with docker_container(
"python-base",
volumes=[tmp_volume],
) as cont:
cont.run("chown -R {uid} /root/.cache/pip".format(uid=os.getuid()))