Merge "Add python-builder docker image"

.zuul.yaml

@@ -226,6 +226,35 @@
     vars: *gerrit_vars
     files: *gerrit_files
 
+# python-builder jobs
+- job:
+    name: system-config-build-image-python-builder
+    description: Build a python-builder image.
+    parent: system-config-build-image
+    vars: &python-builder_vars
+      docker_images:
+        - context: docker/python-builder
+          target: python-builder
+          repository: opendevorg/python-builder
+        - context: docker/python-builder
+          target: python-base
+          repository: opendevorg/python-base
+    files: &python-builder_files
+      - docker/python-builder/.*
+
+- job:
+    name: system-config-upload-image-python-builder
+    description: Build and upload a python-builder image.
+    parent: system-config-upload-image
+    vars: *python-builder_vars
+    files: *python-builder_files
+
+- job:
+    name: system-config-promote-image-python-builder
+    description: Promote a previously published python-builder image to latest.
+    parent: system-config-promote-image
+    vars: *python-builder_vars
+    files: *python-builder_files
 # Role integration jobs.  These test the top-level generic roles/*
 # under Zuul.  The range of platforms should be the same as those for
 # openstack-zuul-jobs.
@@ -512,6 +541,7 @@
         - system-config-build-image-gitea-init
         - system-config-build-image-gitea
         - system-config-build-image-gerrit
+        - system-config-build-image-python-builder
     gate:
       jobs:
         - tox-linters
@@ -529,9 +559,11 @@
         - system-config-upload-image-gitea-init
         - system-config-upload-image-gitea
         - system-config-upload-image-gerrit
+        - system-config-upload-image-python-builder
     promote:
       jobs:
         - system-config-promote-image-jinja-init
         - system-config-promote-image-gitea-init
         - system-config-promote-image-gitea
         - system-config-promote-image-gerrit
+        - system-config-promote-image-python-builder

docker/python-builder/Dockerfile

@@ -0,0 +1,39 @@
+# Copyright (c) 2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM python:slim as fake-python
+
+COPY python3-dev.control /tmp/python3-dev.control
+WORKDIR /tmp
+RUN apt-get update && apt-get install -y equivs
+RUN equivs-build /tmp/python3-dev.control
+
+FROM python:slim as python-builder
+
+COPY --from=fake-python /tmp/python3-dev_4.0.0_all.deb /tmp/python3-dev_4.0.0_all.deb
+COPY scripts/assemble /usr/local/bin/assemble
+COPY scripts/get-extras-packages /usr/local/bin/get-extras-packages
+COPY scripts/install-from-bindep /output/install-from-bindep
+RUN dpkg -i /tmp/python3-dev_4.0.0_all.deb \
+  && rm /tmp/python3-dev_4.0.0_all.deb \
+  && pip install bindep
+
+FROM python:slim as python-base
+
+RUN apt-get update \
+  && apt-get install -y dumb-init \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]

docker/python-builder/python3-dev.control

@@ -0,0 +1,17 @@
+# The python:slim base image already has python headers
+# installed, but they are installed from source. This will
+# be used to create a dummy package so that things that
+# say they need python3-dev won't actually install an
+# entire python that is unneeded.
+Section: misc
+Priority: optional
+Standards-Version: 2.3.3
+
+Package: python3-dev
+# Set the version to 4.0.0 so that it'll always be a greater
+# version than any version of python3-dev in the archive.
+Version: 4.0.0
+Section: mail
+Maintainer: Some Body
+Architecture: all
+Description: Dummy python3-dev package

docker/python-builder/scripts/assemble

@@ -0,0 +1,72 @@
+#!/bin/bash
+# Copyright (c) 2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Make a list of bindep dependencies and a collection of built binary
+# wheels for the repo in question as well as its python dependencies.
+# Install javascript tools as well to support python that needs javascript
+# at build time.
+set -ex
+
+mkdir -p /output/bindep
+mkdir -p /output/wheels
+
+cd /tmp/src
+
+apt-get update
+
+# Protect from the bindep builder image use of the assemble script
+# to produce a wheel
+if [ -f bindep.txt -o -f other-requirements.txt ] ; then
+  bindep -l newline > /output/bindep/run.txt || true
+  compile_packages=$(bindep -b compile || true)
+  if [ ! -z "$compile_packages" ] ; then
+    apt-get install -y ${compile_packages}
+  fi
+fi
+
+# pbr needs git installed, else nothing will work
+apt-get install -y git
+
+# Build a wheel so that we have an install target.
+# pip install . in the container context with the mounted
+# source dir gets ... exciting.
+# We run sdist first to trigger code generation steps such
+# as are found in zuul, since the sequencing otherwise
+# happens in a way that makes wheel content copying unhappy.
+# pip wheel isn't used here because it puts all of the output
+# in the output dir and not the wheel cache, so it's not
+# possible to tell what is the wheel for the project and
+# what is the wheel cache.
+python setup.py sdist bdist_wheel -d /output/wheels
+
+# Use a virtualenv for the next install steps in case to prevent
+# things from the current environment from making us not build a
+# wheel.
+python -m venv /tmp/venv
+/tmp/venv/bin/pip install -U pip wheel
+
+# Install everything so that the wheel cache is populated
+# with transitive depends.
+/tmp/venv/bin/pip install --cache-dir=/output/wheels /output/wheels/*whl
+
+# Install each of the extras so that we collect all possibly
+# needed wheels in the wheel cache. get-extras-packages also
+# writes out the req files into /output/$extra/requirements.txt.
+for req in $(get-extras-packages) ; do
+    /tmp/venv/bin/pip install --cache-dir=/output/wheels "$req"
+done
+
+rm -rf /tmp/venv

docker/python-builder/scripts/get-extras-packages

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+# Copyright (c) 2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import configparser
+import os
+import sys
+
+
+def get_extras_packages(path):
+    config = configparser.ConfigParser()
+    config.read('setup.cfg')
+    config = dict(dict(config.items()).get('extras', {}).items())
+
+    extras = set()
+    for name, packages in config.items():
+        for package in packages.split('\n'):
+            package = package.strip()
+            if not package:
+                continue
+            extras.add(package)
+        outpath = os.path.join(path, name, 'requirements.txt')
+        os.mkdir(os.path.dirname(outpath))
+        with open(outpath, 'w') as outfile:
+            outfile.write(packages)
+
+    return " ".join(extras)
+
+
+if __name__ == '__main__':
+    path = '/output'
+    if len(sys.argv) > 1:
+        path = sys.argv[1]
+    print(get_extras_packages(path))

docker/python-builder/scripts/install-from-bindep

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Copyright (c) 2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+apt-get update
+apt-get -y install $(cat /output/bindep/run.txt)
+pip install --cache-dir=/output/wheels /output/wheels/*.whl
+
+# clean up after ourselves
+apt-get clean
+rm -rf /var/lib/apt/lists/*