Add podman for image building

Add podman as an option to choose for container engine for kolla-build (--engine podman) Signed-off-by: Konstantin Yarovoy <konstantin.yarovoy@tietoevry.com> Co-Authored-By: Michal Arbet <michal.arbet@ultimum.io> Change-Id: I068c906df97745e397408d8c3ef6af47ee037638
2023-09-12 02:28:08 +02:00 · 2023-09-12 02:28:08 +02:00 · b76bd4c6ef
commit b76bd4c6ef
parent 4871ca4d6b
7 changed files with 117 additions and 29 deletions
--- a/kolla/common/config.py
+++ b/kolla/common/config.py
@ -246,8 +246,10 @@ _CLI_OPTS = [
               help='Prefix prepended to image names'),
    cfg.StrOpt('repos-yaml', default='',
               help='Path to alternative repos.yaml file'),
-    cfg.StrOpt('engine', default='docker', choices=['docker'],
-               help='Container engine to build images on.')
+    cfg.StrOpt('engine', default='docker', choices=['docker', 'podman'],
+               help='Container engine to build images on.'),
+    cfg.StrOpt('podman_base_url', default='unix:///run/podman/podman.sock',
+               help='Path to podman socket.')
 ]

 _BASE_OPTS = [
--- a/kolla/engine_adapter/engine.py
+++ b/kolla/engine_adapter/engine.py
@ -18,10 +18,17 @@ try:
 except (ImportError):
    LOG.debug("Docker python library was not found")

+try:
+    import podman
+except (ImportError, ModuleNotFoundError):
+    LOG.debug("Podman python library was not found")
+    pass
+

 class Engine(Enum):

    DOCKER = "docker"
+    PODMAN = "podman"


 class UnsupportedEngineError(ValueError):
@ -37,6 +44,9 @@ class UnsupportedEngineError(ValueError):
 def getEngineException(conf):
    if conf.engine == Engine.DOCKER.value:
        return (docker.errors.DockerException)
+    elif conf.engine == Engine.PODMAN.value:
+        return (podman.errors.exceptions.APIError,
+                podman.errors.exceptions.PodmanError)
    else:
        raise UnsupportedEngineError(conf.engine)

@ -45,5 +55,15 @@ def getEngineClient(conf):
    if conf.engine == Engine.DOCKER.value:
        kwargs_env = docker.utils.kwargs_from_env()
        return docker.DockerClient(version='auto', **kwargs_env)
+    elif conf.engine == Engine.PODMAN.value:
+        client = podman.PodmanClient(base_url=conf.podman_base_url)
+        try:
+            client.version()
+        except podman.errors.exceptions.APIError as e:
+            e.explanation += (". Check if podman service is active and "
+                              "the address to podman.sock is set correctly "
+                              "through --podman_base_url")
+            raise e
+        return client
    else:
        raise UnsupportedEngineError(conf.engine)
--- a/kolla/image/build.py
+++ b/kolla/image/build.py
@ -106,7 +106,8 @@ def run_build():
    if conf.debug:
        LOG.setLevel(logging.DEBUG)

-    if conf.engine not in (engine.Engine.DOCKER.value,):
+    if conf.engine not in (engine.Engine.DOCKER.value,
+                           engine.Engine.PODMAN.value):
        LOG.error(f'Unsupported engine name "{conf.engine}", exiting.')
        sys.exit(1)
    LOG.info(f'Using engine: {conf.engine}')
@ -123,12 +124,21 @@ def run_build():
        except AttributeError:
            LOG.error("Error, Docker Python library is too old, "
                      "Try running 'pip install docker --upgrade'")
-
        if conf.squash:
            squash_version = utils.get_docker_squash_version()
            LOG.info('Image squash is enabled and "docker-squash" version '
                     'is %s', squash_version)

+    if conf.engine == engine.Engine.PODMAN.value:
+        try:
+            import podman
+            podman.__version__
+        except ImportError:
+            LOG.error("Error, you have set podman as container engine, "
+                      "but library is not found."
+                      "Try running pip install podman")
+            exit(1)
+
    kolla = KollaWorker(conf)
    kolla.setup_working_dir()
    kolla.find_dockerfiles()
--- a/kolla/image/tasks.py
+++ b/kolla/image/tasks.py
@ -12,11 +12,16 @@

 import datetime
 import errno
+import json
 import os
 import shutil
 import tarfile

 import docker.errors
+try:
+    import podman.errors.exceptions
+except (ImportError, ModuleNotFoundError):
+    pass
 import git
 import requests
 from requests import exceptions as requests_exc
@ -367,6 +372,20 @@ class BuildTask(EngineTask):
        pull = self.conf.pull if image.parent is None else False

        buildargs = self.update_buildargs()
+
+        kwargs = {}
+        if self.conf.engine == engine.Engine.PODMAN.value:
+            # TODO(kevko): dockerfile path is a workaround,
+            # should be removed as soon as it will be fixed in podman-py
+            # https://github.com/containers/podman-py/issues/177
+            kwargs["dockerfile"] = image.path + '/Dockerfile'
+            # Podman squash is different by default
+            # https://github.com/containers/buildah/issues/1234
+            if self.conf.squash:
+                kwargs["squash"] = False
+                kwargs["layers"] = False
+            else:
+                kwargs["layers"] = True
        try:
            for stream in self.engine_client.images.build(
                    path=image.path,
@ -376,7 +395,10 @@ class BuildTask(EngineTask):
                    network_mode=self.conf.network_mode,
                    pull=pull,
                    forcerm=self.forcerm,
-                    buildargs=buildargs)[1]:
+                    buildargs=buildargs,
+                    **kwargs)[1]:
+                if self.conf.engine == engine.Engine.PODMAN.value:
+                    stream = json.loads(stream)
                if 'stream' in stream:
                    for line in stream['stream'].split('\n'):
                        if line:
@ -399,6 +421,11 @@ class BuildTask(EngineTask):
                for line in e.build_log:
                    if 'stream' in line:
                        self.logger.error(line['stream'].strip())
+            if isinstance(e, podman.errors.exceptions.BuildError):
+                for line in e.build_log:
+                    line = json.loads(line)
+                    if 'stream' in line:
+                        self.logger.error(line['stream'].strip())
            self.logger.exception('Unknown container engine '
                                  'error when building')

--- a/kolla/tests/test_build.py
+++ b/kolla/tests/test_build.py
@ -50,6 +50,9 @@ FAKE_IMAGE_GRANDCHILD = Image(
    'image-grandchild', 'image-grandchild:latest',
    '/fake/path6', parent_name='image-child',
    parent=FAKE_IMAGE_CHILD, status=utils.Status.MATCHED)
+engine_client = "docker.DockerClient"
+if os.getenv("TEST_ENGINE") == "podman":
+    engine_client = "podman.PodmanClient"


 class TasksTest(base.TestCase):
@ -63,9 +66,19 @@ class TasksTest(base.TestCase):
        # NOTE(mandre) we want the local copy of FAKE_IMAGE as the parent
        self.imageChild.parent = self.image
        self.imageChild.path = self.useFixture(fixtures.TempDir()).path
+        if "podman" in engine_client:
+            self.conf.set_override('engine', 'podman')
+            self.build_kwargs["dockerfile"] = self.image.path + '/Dockerfile'
+            # NOTE(kevko): Squash implementation is different in podman
+            #
+            # - Check kolla/image/tasks.py for podman
+            # - Check https://github.com/containers/buildah/issues/1234
+            self.build_kwargs["squash"] = False
+        else:
+            self.build_kwargs = {}

    @mock.patch.dict(os.environ, clear=True)
-    @mock.patch('docker.DockerClient')
+    @mock.patch(engine_client)
    def test_push_image(self, mock_client):
        self.engine_client = mock_client
        pusher = tasks.PushTask(self.conf, self.image)
@ -75,7 +88,7 @@ class TasksTest(base.TestCase):
        self.assertTrue(pusher.success)

    @mock.patch.dict(os.environ, clear=True)
-    @mock.patch('docker.DockerClient')
+    @mock.patch(engine_client)
    def test_push_image_failure(self, mock_client):
        """failure on connecting Docker API"""
        self.engine_client = mock_client
@ -88,7 +101,7 @@ class TasksTest(base.TestCase):
        self.assertEqual(utils.Status.PUSH_ERROR, self.image.status)

    @mock.patch.dict(os.environ, clear=True)
-    @mock.patch('docker.DockerClient')
+    @mock.patch(engine_client)
    def test_push_image_failure_retry(self, mock_client):
        """failure on connecting Docker API, success on retry"""
        self.engine_client = mock_client
@ -108,7 +121,7 @@ class TasksTest(base.TestCase):
        self.assertEqual(utils.Status.BUILT, self.image.status)

    @mock.patch.dict(os.environ, clear=True)
-    @mock.patch('docker.DockerClient')
+    @mock.patch(engine_client)
    def test_push_image_failure_error(self, mock_client):
        """Docker connected, failure to push"""
        self.engine_client = mock_client
@ -122,7 +135,7 @@ class TasksTest(base.TestCase):
        self.assertEqual(utils.Status.PUSH_ERROR, self.image.status)

    @mock.patch.dict(os.environ, clear=True)
-    @mock.patch('docker.DockerClient')
+    @mock.patch(engine_client)
    def test_push_image_failure_error_retry(self, mock_client):
        """Docker connected, failure to push, success on retry"""
        self.engine_client = mock_client
@ -145,7 +158,7 @@ class TasksTest(base.TestCase):
        self.assertEqual(utils.Status.BUILT, self.image.status)

    @mock.patch.dict(os.environ, clear=True)
-    @mock.patch('docker.DockerClient')
+    @mock.patch(engine_client)
    def test_build_image(self, mock_client):
        self.engine_client = mock_client
        push_queue = mock.Mock()
@ -155,12 +168,12 @@ class TasksTest(base.TestCase):
        mock_client().images.build.assert_called_once_with(
            path=self.image.path, tag=self.image.canonical_name,
            network_mode='host', nocache=False, rm=True, pull=True,
-            forcerm=True, buildargs=None)
+            forcerm=True, buildargs=None, **self.build_kwargs)

        self.assertTrue(builder.success)

    @mock.patch.dict(os.environ, clear=True)
-    @mock.patch('docker.DockerClient')
+    @mock.patch(engine_client)
    def test_build_image_with_network_mode(self, mock_client):
        self.engine_client = mock_client
        push_queue = mock.Mock()
@ -172,12 +185,12 @@ class TasksTest(base.TestCase):
        mock_client().images.build.assert_called_once_with(
            path=self.image.path, tag=self.image.canonical_name,
            network_mode='bridge', nocache=False, rm=True, pull=True,
-            forcerm=True, buildargs=None)
+            forcerm=True, buildargs=None, **self.build_kwargs)

        self.assertTrue(builder.success)

    @mock.patch.dict(os.environ, clear=True)
-    @mock.patch('docker.DockerClient')
+    @mock.patch(engine_client)
    def test_build_image_with_build_arg(self, mock_client):
        self.engine_client = mock_client
        build_args = {
@ -192,13 +205,13 @@ class TasksTest(base.TestCase):
        mock_client().images.build.assert_called_once_with(
            path=self.image.path, tag=self.image.canonical_name,
            network_mode='host', nocache=False, rm=True, pull=True,
-            forcerm=True, buildargs=build_args)
+            forcerm=True, buildargs=build_args, **self.build_kwargs)

        self.assertTrue(builder.success)

    @mock.patch.dict(os.environ, {'http_proxy': 'http://FROM_ENV:8080'},
                     clear=True)
-    @mock.patch('docker.DockerClient')
+    @mock.patch(engine_client)
    def test_build_arg_from_env(self, mock_client):
        push_queue = mock.Mock()
        self.engine_client = mock_client
@ -211,13 +224,13 @@ class TasksTest(base.TestCase):
        mock_client().images.build.assert_called_once_with(
            path=self.image.path, tag=self.image.canonical_name,
            network_mode='host', nocache=False, rm=True, pull=True,
-            forcerm=True, buildargs=build_args)
+            forcerm=True, buildargs=build_args, **self.build_kwargs)

        self.assertTrue(builder.success)

    @mock.patch.dict(os.environ, {'http_proxy': 'http://FROM_ENV:8080'},
                     clear=True)
-    @mock.patch('docker.DockerClient')
+    @mock.patch(engine_client)
    def test_build_arg_precedence(self, mock_client):
        self.engine_client = mock_client
        build_args = {
@ -232,11 +245,11 @@ class TasksTest(base.TestCase):
        mock_client().images.build.assert_called_once_with(
            path=self.image.path, tag=self.image.canonical_name,
            network_mode='host', nocache=False, rm=True, pull=True,
-            forcerm=True, buildargs=build_args)
+            forcerm=True, buildargs=build_args, **self.build_kwargs)

        self.assertTrue(builder.success)

-    @mock.patch('docker.DockerClient')
+    @mock.patch(engine_client)
    @mock.patch('requests.get')
    def test_requests_get_timeout(self, mock_get, mock_client):
        self.engine_client = mock_client
@ -261,7 +274,7 @@ class TasksTest(base.TestCase):
    @mock.patch('os.utime')
    @mock.patch('shutil.copyfile')
    @mock.patch('shutil.rmtree')
-    @mock.patch('docker.DockerClient')
+    @mock.patch(engine_client)
    @mock.patch('requests.get')
    def test_process_source(self, mock_get, mock_client,
                            mock_rmtree, mock_copyfile, mock_utime):
@ -293,7 +306,7 @@ class TasksTest(base.TestCase):
                self.assertIsNotNone(get_result)

    @mock.patch.dict(os.environ, clear=True)
-    @mock.patch('docker.DockerClient')
+    @mock.patch(engine_client)
    def test_local_directory(self, mock_client):
        tmpdir = tempfile.mkdtemp()
        file_name = 'test.txt'
@ -326,7 +339,7 @@ class TasksTest(base.TestCase):
            self.assertTrue(builder.success)

    @mock.patch.dict(os.environ, clear=True)
-    @mock.patch('docker.DockerClient')
+    @mock.patch(engine_client)
    def test_malicious_tar(self, mock_client):
        tmpdir = tempfile.mkdtemp()
        file_name = 'test.txt'
@ -366,7 +379,7 @@ class TasksTest(base.TestCase):
            os.rmdir(tmpdir)

    @mock.patch.dict(os.environ, clear=True)
-    @mock.patch('docker.DockerClient')
+    @mock.patch(engine_client)
    def test_malicious_tar_gz(self, mock_client):
        tmpdir = tempfile.mkdtemp()
        file_name = 'test.txt'
@ -428,7 +441,7 @@ class TasksTest(base.TestCase):
        self.assertFalse(builder.success)
        self.assertIsNone(get_result)

-    @mock.patch('docker.DockerClient')
+    @mock.patch(engine_client)
    def test_followups_docker_image(self, mock_client):
        self.imageChild.source = {
            'source': 'http://fake/source',
@ -463,7 +476,7 @@ class KollaWorkerTest(base.TestCase):

        self.images = [image, image_child, image_unmatched,
                       image_error, image_built]
-        patcher = mock.patch('docker.DockerClient')
+        patcher = mock.patch(engine_client)
        self.addCleanup(patcher.stop)
        self.mock_client = patcher.start()

@ -767,7 +780,7 @@ class MainTest(base.TestCase):
        self.assertEqual(1, result)

    @mock.patch('sys.argv')
-    @mock.patch('docker.DockerClient')
+    @mock.patch(engine_client)
    def test_run_build(self, mock_client, mock_sys):
        result = build.run_build()
        self.assertTrue(result)
--- a/releasenotes/notes/add-docker-engine-7402d3fe7349f1f3.yaml
+++ b/releasenotes/notes/add-docker-engine-7402d3fe7349f1f3.yaml
@ -0,0 +1,7 @@
+---
+features:
+  - A new engine command-line option has been added to
+    the client, to allow the user to specify a container
+    engine to be used for image build, instead of the default
+    docker container engine. Kolla now supports both docker
+    and podman container engines.
--- a/tox.ini
+++ b/tox.ini
@ -1,7 +1,7 @@
 [tox]
 minversion = 3.18
 skipsdist = True
-envlist = pep8,py38
+envlist = pep8,py310-{docker,podman}
 ignore_basepython_conflict = True

 [testenv]
@ -15,12 +15,21 @@ deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/up
 passenv = http_proxy,HTTP_PROXY,https_proxy,HTTPS_PROXY,no_proxy,NO_PROXY
          OS_STDOUT_CAPTURE,OS_STDERR_CAPTURE,OS_LOG_CAPTURE,OS_TEST_TIMEOUT
          PYTHON,OS_TEST_PATH,LISTOPT,IDOPTION
+
 commands =
  find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
  find . -type d -name "__pycache__" -delete
  stestr run {posargs}
  stestr slowest

+[testenv:docker]
+setenv =
+    TEST_ENGINE=docker
+
+[testenv:podman]
+setenv =
+    TEST_ENGINE=podman
+
 [testenv:debug]
 commands = oslo_debug_helper -t kolla/tests {posargs}