Build docker images

Images builder which takes the dockerfiles (both raw and jinja2
templated), generated the dependency tree between them and
the builds them in appropriate sequence.

Change-Id: Iedcb32c91d2d779d8dcc6cc4b777da932086dcbd

microservices/build.py

@@ -0,0 +1,119 @@
+import contextlib
+import json
+import os
+import sys
+import threading
+
+import docker
+from oslo_config import cfg
+from oslo_log import log as logging
+import six
+
+
+CONF = cfg.CONF
+CONF.import_group('builder', 'microservices.config.builder')
+CONF.import_group('images', 'microservices.config.images')
+CONF.import_group('repositories', 'microservices.config.repositories')
+
+LOG = logging.getLogger(__name__)
+
+
+def find_dockerfiles(component):
+    dockerfiles = {}
+    component_dir = os.path.join(CONF.repositories.path, component)
+    for root, __, files in os.walk(component_dir):
+        if 'Dockerfile.j2' in files:
+            path = os.path.join(root, 'Dockerfile.j2')
+            is_jinja2 = True
+        elif 'Dockerfile' in files:
+            path = os.path.join(root, 'Dockerfile')
+            is_jinja2 = False
+        else:
+            continue
+        name = os.path.basename(os.path.dirname(path))
+        dockerfiles[name] = {
+            'name': name,
+            'path': path,
+            'is_jinja2': is_jinja2,
+            'parent': None,
+            'children': []
+        }
+
+    if len(dockerfiles) == 0:
+        msg = 'No dockerfile for %s found'
+        if CONF.repositories.skip_empty:
+            LOG.warn(msg, component)
+        else:
+            LOG.error(msg, component)
+            sys.exit(1)
+
+    return dockerfiles
+
+
+def find_dependencies(dockerfiles):
+    for name, dockerfile in dockerfiles.items():
+        with open(dockerfile['path']) as f:
+            content = f.read()
+
+        parent = content.split(' ')[1].split('\n')[0]
+        if CONF.images.namespace not in parent:
+            continue
+        if '/' in parent:
+            parent = parent.split('/')[-1]
+        if ':' in parent:
+            parent = parent.split(':')[0]
+
+        dockerfile['parent'] = dockerfiles[parent]
+        dockerfiles[parent]['children'].append(dockerfile)
+
+
+def create_initial_queue(dockerfiles):
+    queue = six.moves.queue.Queue()
+    for dockerfile in dockerfiles.values():
+        if dockerfile['parent'] is None:
+            queue.put(dockerfile)
+    return queue
+
+
+def build_dockerfile(queue):
+    dockerfile = queue.get()
+    with contextlib.closing(docker.Client()) as dc:
+        if dockerfile['is_jinja2']:
+            # TODO(mrostecki): Write jinja2 templating functionality.
+            pass
+
+        for line in dc.build(fileobj=open(dockerfile['path'], 'r'), rm=True,
+                             tag='%s/%s:%s' % (CONF.images.namespace,
+                                               dockerfile['name'],
+                                               CONF.images.tag)):
+            build_data = json.loads(line)
+            if 'stream' in build_data:
+                LOG.info(build_data['stream'].rstrip())
+            if 'errorDetail' in build_data:
+                LOG.error(build_data['errorDetail']['message'])
+    for child in dockerfile['children']:
+        queue.put(child)
+    queue.task_done()
+
+
+def build_repositories(components=None):
+    if components is None:
+        components = CONF.repositories.components
+
+    dockerfiles = {}
+    for component in components:
+        dockerfiles.update(find_dockerfiles(component))
+
+    find_dependencies(dockerfiles)
+    # TODO(mrostecki): Try to use multiprocessing there.
+    # NOTE(mrostecki): Unfortunately, just using multiprocessing pool
+    # with multiprocessing.Queue, while keeping the same logic, doesn't
+    # work well with docker-py - each process exits before the image build
+    # is done.
+    queue = create_initial_queue(dockerfiles)
+    threads = [threading.Thread(target=build_dockerfile, args=(queue,))
+               for __ in range(CONF.builder.workers)]
+    for thread in threads:
+        thread.daemon = True
+        thread.start()
+    queue.join()

microservices/cli.py

@@ -3,6 +3,7 @@ import sys
 from oslo_config import cfg
 from oslo_log import log as logging
 
+from microservices import build
 from microservices import fetch
 
 
@@ -12,9 +13,10 @@ CONF.import_opt('action', 'microservices.config.cli')
 
 
 def do_build():
+    components = CONF.action.components
     if CONF.repositories.clone:
-        fetch.fetch_repositories(components=CONF.action.components)
-    # TODO(mrostecki): implement build
+        fetch.fetch_repositories(components=components)
+    build.build_repositories(components=components)
 
 
 def do_fetch():

microservices/config/builder.py

@@ -0,0 +1,16 @@
+import multiprocessing
+
+from oslo_config import cfg
+
+
+CONF = cfg.CONF
+builder_opts = [
+    cfg.IntOpt('workers',
+               default=multiprocessing.cpu_count(),
+               help='Number of workers which build docker images')
+]
+builder_opt_group = cfg.OptGroup(name='builder',
+                                 title='Images builder')
+CONF.register_group(builder_opt_group)
+CONF.register_cli_opts(builder_opts, builder_opt_group)
+CONF.register_opts(builder_opts, builder_opt_group)

microservices/config/images.py

@@ -0,0 +1,17 @@
+from oslo_config import cfg
+
+
+CONF = cfg.CONF
+images_opts = [
+    cfg.StrOpt('namespace',
+               default='mcp',
+               help='Namespace for Docker images'),
+    cfg.StrOpt('tag',
+               default='latest',
+               help='Tag for Docker images')
+]
+images_opt_group = cfg.OptGroup(name='images',
+                                title='Docker images')
+CONF.register_group(images_opt_group)
+CONF.register_cli_opts(images_opts, images_opt_group)
+CONF.register_opts(images_opts, images_opt_group)

microservices/config/repositories.py

@@ -8,6 +8,10 @@ repositories_opts = [
     cfg.BoolOpt('clone',
                 default=True,
                 help='Automatic cloning of microservices repositories'),
+    cfg.BoolOpt('skip-empty',
+                default=True,
+                help='Skip repositories not containing Dockerfiles without '
+                     'error'),
     cfg.StrOpt('path',
                default=os.path.expanduser('~/microservices-repos/'),
                help='Path where the microservice repositories are cloned'),

microservices/tests/test_build.py

@@ -0,0 +1,55 @@
+import collections
+import io
+import mock
+
+from microservices import build
+from microservices.tests import base
+
+
+BASE_DOCKERFILE = u"""FROM debian:jessie
+MAINTAINER Foo Bar <foo@bar.org>
+
+RUN apt-get install \
+      curl \
+    && apt-get clean"""
+COMPONENT_DOCKERFILE = u"""FROM mcp/ms-debian-base
+MAINTAINER Foo Bar <foo@bar.org>
+
+RUN apt-get -y install \
+      mysql-server \
+    && apt-get clean"""
+
+
+class TestBuild(base.TestCase):
+
+    def test_find_dependencies(self):
+        m_open = mock.mock_open()
+        m_open.side_effect = [
+            io.StringIO(BASE_DOCKERFILE),
+            io.StringIO(COMPONENT_DOCKERFILE)
+        ]
+        with mock.patch('microservices.build.open', m_open, create=True):
+            dockerfiles = collections.OrderedDict([
+                ('ms-debian-base', {
+                    'name': 'ms-debian-base',
+                    'path': '/home/test/microservices/ms-base/docker/'
+                            'Dockerfile',
+                    'is_jinja2': False,
+                    'parent': None,
+                    'children': []
+                },),
+                ('ms-mysql', {
+                    'name': 'ms-mysql',
+                    'path': '/home/test/microservices/ms-mysql/docker/'
+                            'Dockerfile',
+                    'is_jinja2': False,
+                    'parent': None,
+                    'children': []
+                },)
+            ])
+            build.find_dependencies(dockerfiles)
+
+        self.assertListEqual([dockerfiles['ms-mysql']],
+                             dockerfiles['ms-debian-base']['children'])
+        self.assertDictEqual(dockerfiles['ms-debian-base'],
+                             dockerfiles['ms-mysql']['parent'])

requirements.txt

@@ -4,6 +4,7 @@
 
 pbr>=1.6
 
+docker-py>=1.6.0,<1.8.0  # Apache-2.0
 GitPython>=1.0.1 # BSD License (3 clause)
 oslo.config>=3.9.0  # Apache-2.0
 oslo.log>=1.14.0  # Apache-2.0