Revert "docker-cmd hook switch to the paunch library"

This reverts commit 33241a84c1

The requirements check fails since
Ia4e086162e65a51af417a8b381ae898c95966a09 removed paunch as valid
dependency.

Since Paunch is retired, this patch propose to switch to use docker cli
for docker cmd hook and remove any dependency to Paunch.

Change-Id: I9c3839e551259fb85b191a27fa054c605964f30e

doc/source/install/hooks/docker-cmd.rst

@@ -2,7 +2,7 @@
 docker-cmd
 ==========
 
-A hook which uses the ``docker`` command via `paunch`_ to deploy containers.
+A hook which uses the `docker` command to deploy containers.
 
 The hook currently supports specifying containers in the `docker-compose v1
 format`_. The intention is for this hook to also support the kubernetes pod
@@ -12,5 +12,4 @@ A dedicated os-refresh-config script will remove running containers if a
 deployment is removed or changed, then the docker-cmd hook will run any
 containers in new or updated deployments.
 
-.. _paunch: https://docs.openstack.org/paunch/latest/
 .. _docker-compose v1 format: https://docs.docker.com/compose/compose-file/#/version-1

heat-config-docker-cmd/install.d/hook-docker-cmd.py

@@ -12,13 +12,14 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import collections
 import json
 import logging
 import os
+import random
+import string
+import subprocess
 import sys
 
-import paunch
 import yaml
 
 DOCKER_CMD = os.environ.get('HEAT_DOCKER_CMD', 'docker')
@@ -35,7 +36,138 @@ def build_response(deploy_stdout, deploy_stderr, deploy_status_code):
     }
 
 
+def docker_run_args(cmd, container, config):
+    cconfig = config[container]
+    if cconfig.get('detach', True):
+        cmd.append('--detach=true')
+    if 'env_file' in cconfig:
+        if isinstance(cconfig['env_file'], list):
+            for f in cconfig.get('env_file', []):
+                if f:
+                    cmd.append('--env-file=%s' % f)
+        else:
+            cmd.append('--env-file=%s' % cconfig['env_file'])
+    for v in cconfig.get('environment', []):
+        if v:
+            cmd.append('--env=%s' % v)
+    if 'net' in cconfig:
+        cmd.append('--net=%s' % cconfig['net'])
+    if 'pid' in cconfig:
+        cmd.append('--pid=%s' % cconfig['pid'])
+    if 'privileged' in cconfig:
+        cmd.append('--privileged=%s' % str(cconfig['privileged']).lower())
+    if 'restart' in cconfig:
+        cmd.append('--restart=%s' % cconfig['restart'])
+    if 'user' in cconfig:
+        cmd.append('--user=%s' % cconfig['user'])
+    for v in cconfig.get('volumes', []):
+        if v:
+            cmd.append('--volume=%s' % v)
+    for v in cconfig.get('volumes_from', []):
+        if v:
+            cmd.append('--volumes_from=%s' % v)
+
+    cmd.append(cconfig.get('image', ''))
+    cmd.extend(command_argument(cmd, cconfig.get('command')))
+
+
+def docker_exec_args(cmd, container, config, cid):
+    cconfig = config[container]
+    if 'privileged' in cconfig:
+        cmd.append('--privileged=%s' % str(cconfig['privileged']).lower())
+    if 'user' in cconfig:
+        cmd.append('--user=%s' % cconfig['user'])
+    command = command_argument(cmd, cconfig.get('command'))
+    # for exec, the first argument is the container name,
+    # make sure the correct one is used
+    command[0] = discover_container_name(command[0], cid)
+    cmd.extend(command)
+
+
+def command_argument(cmd, command):
+    if not command:
+        return []
+    if not isinstance(command, list):
+        return command.split()
+    return command
+
+
+def execute(cmd):
+    log.debug("execute command: %s" % cmd)
+    subproc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+    cmd_stdout, cmd_stderr = subproc.communicate()
+    log.debug(cmd_stdout)
+    log.debug(cmd_stderr)
+    return cmd_stdout, cmd_stderr, subproc.returncode
+
+
+def label_arguments(cmd, container, cid, iv):
+    cmd.extend([
+        '--label',
+        'deploy_stack_id=%s' % iv.get('deploy_stack_id'),
+        '--label',
+        'deploy_resource_name=%s' % iv.get('deploy_resource_name'),
+        '--label',
+        'config_id=%s' % cid,
+        '--label',
+        'container_name=%s' % container,
+        '--label',
+        'managed_by=docker-cmd'
+    ])
+
+
+def inspect(container, format=None):
+    cmd = [DOCKER_CMD, 'inspect']
+    if format:
+        cmd.append('--format')
+        cmd.append(format)
+    cmd.append(container)
+    (cmd_stdout, cmd_stderr, returncode) = execute(cmd)
+    if returncode != 0:
+        return
+    try:
+        if format:
+            return cmd_stdout
+        else:
+            return json.loads(cmd_stdout)[0]
+    except Exception as e:
+        log.error('Problem parsing docker inspect: %s' % e)
+
+
+def unique_container_name(container):
+    container_name = container
+    while inspect(container_name, format='exists'):
+        suffix = ''.join(random.choice(
+            string.ascii_lowercase + string.digits) for i in range(8))
+        container_name = '%s-%s' % (container, suffix)
+    return container_name
+
+
+def discover_container_name(container, cid):
+    cmd = [
+        DOCKER_CMD,
+        'ps',
+        '-a',
+        '--filter',
+        'label=container_name=%s' % container,
+        '--filter',
+        'label=config_id=%s' % cid,
+        '--format',
+        '{{.Names}}'
+    ]
+    (cmd_stdout, cmd_stderr, returncode) = execute(cmd)
+    if returncode != 0:
+        return container
+    names = cmd_stdout.split()
+    if names:
+        return names[0]
+    return container
+
+
 def main(argv=sys.argv, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr):
+    cmd_stderrs = []
+    cmd_stdouts = []
     global log
     log = logging.getLogger('heat-config')
     handler = logging.StreamHandler(stderr)
@@ -68,21 +200,43 @@ def main(argv=sys.argv, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr):
     if not isinstance(config, dict):
         config = yaml.safe_load(config)
 
-    labels = collections.OrderedDict()
+    def key_fltr(key):
-    labels['deploy_stack_id'] = input_values.get('deploy_stack_id')
+        return config[key].get('start_order', 0)
-    labels['deploy_resource_name'] = input_values.get('deploy_resource_name')
+    for container in sorted(config, key=key_fltr):
-    apply_stdout, apply_stderr, deploy_status_code = paunch.apply(
+        log.debug("Running container: %s" % container)
-        cid,
+        action = config[container].get('action', 'run')
-        config,
+        exit_codes = config[container].get('exit_codes', [0])
-        'docker-cmd',
-        labels,
-        DOCKER_CMD
-    )
 
-    json.dump(build_response(
+        if action == 'run':
-        '\n'.join(apply_stdout),
+            cmd = [
-        '\n'.join(apply_stderr),
+                DOCKER_CMD,
-        deploy_status_code), stdout)
+                'run',
+                '--name',
+                unique_container_name(container)
+            ]
+            label_arguments(cmd, container, cid, input_values)
+            docker_run_args(cmd, container, config)
+        elif action == 'exec':
+            cmd = [DOCKER_CMD, 'exec']
+            docker_exec_args(cmd, container, config, cid)
+
+        (cmd_stdout, cmd_stderr, returncode) = execute(cmd)
+        if cmd_stdout:
+            out_str = cmd_stdout.decode('utf-8')
+            stdout.write(out_str)
+            cmd_stdouts.append(out_str)
+        if cmd_stderr:
+            err_str = cmd_stderr.decode('utf-8')
+            stderr.write(err_str)
+            cmd_stderrs.append(err_str)
+
+        if returncode not in exit_codes:
+            log.error("Error running %s. [%s]\n" % (cmd, returncode))
+            deploy_status_code = returncode
+        else:
+            log.debug('Completed %s' % cmd)
+    json.dump(build_response('\n'.join(cmd_stdouts), '\n'.join(cmd_stderrs),
+              deploy_status_code), sys.stdout)
 
 
 if __name__ == '__main__':

heat-config-docker-cmd/os-refresh-config/configure.d/50-heat-config-docker-cmd

@@ -17,7 +17,7 @@ import logging
 import os
 import sys
 
-import paunch
+import subprocess
 
 CONF_FILE = os.environ.get('HEAT_SHELL_CONFIG',
                            '/var/run/heat-config/heat-config')
@@ -51,11 +51,102 @@ def main(argv=sys.argv):
     cmd_config_ids = [c['id'] for c in configs
                       if c['group'] == 'docker-cmd']
 
-    paunch.cleanup(
+    try:
-        cmd_config_ids,
+        delete_missing_configs(cmd_config_ids)
-        'docker-cmd',
+    except Exception as e:
-        DOCKER_CMD
+        log.exception(e)
-    )
+    try:
+        rename_containers()
+    except Exception as e:
+        log.exception(e)
+
+
+def delete_missing_configs(config_ids):
+    for conf_id in current_config_ids():
+        if type(conf_id) is bytes:
+            conf_id = conf_id.decode('utf-8')
+        if conf_id not in config_ids:
+            log.debug('%s no longer exists, deleting containers' % conf_id)
+            remove_containers(conf_id)
+
+
+def execute(cmd):
+    log.debug("execute command: %s" % cmd)
+    subproc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+    cmd_stdout, cmd_stderr = subproc.communicate()
+    return cmd_stdout, cmd_stderr, subproc.returncode
+
+
+def current_config_ids():
+    # List all config_id labels for containers managed by docker-cmd
+    cmd = [
+        DOCKER_CMD, 'ps', '-a',
+        '--filter', 'label=managed_by=docker-cmd',
+        '--format', '{{.Label "config_id"}}'
+    ]
+    cmd_stdout, cmd_stderr, returncode = execute(cmd)
+    if returncode != 0:
+        return set()
+    return set(cmd_stdout.split())
+
+
+def remove_containers(conf_id):
+    cmd = [
+        DOCKER_CMD, 'ps', '-q', '-a',
+        '--filter', 'label=managed_by=docker-cmd',
+        '--filter', 'label=config_id=%s' % conf_id
+    ]
+    cmd_stdout, cmd_stderr, returncode = execute(cmd)
+    if returncode == 0:
+        for container in cmd_stdout.split():
+            remove_container(container)
+
+
+def remove_container(container):
+    cmd = [DOCKER_CMD, 'rm', '-f', container]
+    cmd_stdout, cmd_stderr, returncode = execute(cmd)
+    if returncode != 0:
+        log.error('Error removing container: %s' % container)
+        log.error(cmd_stderr)
+
+
+def rename_containers():
+    # list every container name, and its container_name label
+    cmd = [
+        DOCKER_CMD, 'ps', '-a',
+        '--format', '{{.Names}} {{.Label "container_name"}}'
+    ]
+    cmd_stdout, cmd_stderr, returncode = execute(cmd)
+    if returncode != 0:
+        return
+
+    lines = cmd_stdout.split(b"\n")
+    current_containers = []
+    need_renaming = {}
+    for line in lines:
+        entry = line.split()
+        if not entry:
+            continue
+        current_containers.append(entry[0])
+
+        # ignore if container_name label not set
+        if len(entry) < 2:
+            continue
+
+        # ignore if desired name is already actual name
+        if entry[0] == entry[-1]:
+            continue
+
+        need_renaming[entry[0]] = entry[-1]
+
+    for current, desired in sorted(need_renaming.items()):
+        if desired in current_containers:
+            log.info('Cannot rename "%s" since "%s" still exists' % (
+                current, desired))
+        else:
+            cmd = [DOCKER_CMD, 'rename', current, desired]
+            execute(cmd)
 
 
 if __name__ == '__main__':

lower-constraints.txt

@@ -37,7 +37,6 @@ oslo.config==5.2.0
 oslo.i18n==3.15.3
 oslo.serialization==2.18.0
 oslo.utils==3.33.0
-paunch==4.0.0
 pbr==2.0.0
 positional==1.2.1
 prettytable==0.7.2

releasenotes/notes/switch-docker-cmd-hook-to-docker-cli-5d8f6deee2125d65.yaml

@@ -0,0 +1,6 @@
+---
+upgrade:
+  - |
+    paunch is retired, hance we now switch docker-cmd hook to use docker
+    client. Aware that when upgrade to current version of heat-agents hooks,
+    the dependency for docker-cmd hook will change accordingly.

test-requirements.txt

@@ -5,10 +5,9 @@ coverage!=4.4,>=4.0 # Apache-2.0
 fixtures>=3.0.0 # Apache-2.0/BSD
 # Hacking already pins down pep8, pyflakes and flake8
 hacking>=3.0.1,<3.1.0 # Apache-2.0
-paunch>=4.0.0 # Apache-2.0
 requests>=2.14.2 # Apache-2.0
 requests-mock>=1.1.0 # Apache-2.0
-salt>=2017.7.4 # Apache-2.0
+salt>=2017.7.4;python_version!='3.8' # Apache-2.0
 testrepository>=0.0.18 # Apache-2.0/BSD
 testscenarios>=0.4 # Apache-2.0/BSD
 testtools>=2.2.0 # MIT

tests/test_hook_docker_cmd.py

@@ -11,13 +11,14 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import copy
 import json
+import os
+import tempfile
 
-from six.moves import cStringIO as StringIO
+import fixtures
-from unittest import mock
 
 from tests import common
-from tests import hook_docker_cmd
 
 
 class HookDockerCmdTest(common.RunScriptTest):
@@ -71,93 +72,751 @@ class HookDockerCmdTest(common.RunScriptTest):
         }
     }
 
-    @mock.patch('paunch.apply', autospec=True)
+    data_exit_code = {
-    def test_hook(self, mock_apply):
+        "name": "abcdef001",
-        mock_apply.return_value = (['it', 'done'], ['one', 'two', 'three'], 0)
+        "group": "docker-cmd",
-        stdin = StringIO(json.dumps(self.data))
+        "id": "abc123",
-        stdout = StringIO()
+        "config": {
-        stderr = StringIO()
+            "web-ls": {
-        hook_docker_cmd.main(
+                "action": "exec",
-            ['/path/to/hook-docker-cmd'], stdin, stdout, stderr)
+                "command": ["web", "/bin/ls", "-l"],
-        mock_apply.assert_called_once_with(
+                "exit_codes": [0, 1]
-            'abc123',
-            self.data['config'],
-            'docker-cmd',
-            {
-                'deploy_stack_id': 'the_stack',
-                'deploy_resource_name': 'the_deployment'
-            },
-            'docker'
-        )
-
-        resp = json.loads(stdout.getvalue())
-
-        self.assertEqual({
-            'deploy_status_code': 0,
-            'deploy_stderr': 'one\ntwo\nthree',
-            'deploy_stdout': 'it\ndone'
-        }, resp)
-
-    @mock.patch('paunch.apply', autospec=True)
-    def test_missing_config(self, mock_apply):
-        data = {
-            "name": "abcdef001",
-            "group": "docker-cmd",
-            "id": "abc123",
-        }
-        stdin = StringIO(json.dumps(data))
-        stdout = StringIO()
-        stderr = StringIO()
-        hook_docker_cmd.main(
-            ['/path/to/hook-docker-cmd'], stdin, stdout, stderr)
-        mock_apply.assert_not_called()
-
-        resp = json.loads(stdout.getvalue())
-
-        self.assertEqual({
-            'deploy_status_code': 0,
-            'deploy_stderr': '',
-            'deploy_stdout': ''
-        }, resp)
-
-    @mock.patch('paunch.apply', autospec=True)
-    def test_action_delete(self, mock_apply):
-        data = {
-            "name": "abcdef001",
-            "group": "docker-cmd",
-            "id": "abc123",
-            "inputs": [{
-                "name": "deploy_action",
-                "value": "DELETE"
-            }, {
-                "name": "deploy_stack_id",
-                "value": "the_stack",
-            }, {
-                "name": "deploy_resource_name",
-                "value": "the_deployment",
-            }],
-            "config": {
-                "db": {
-                    "name": "x",
-                    "image": "xxx",
-                    "privileged": False,
-                    "environment": ["foo=bar"],
-                    "env_file": "env.file",
-                    "start_order": 0
-                }
             }
         }
-        stdin = StringIO(json.dumps(data))
+    }
-        stdout = StringIO()
-        stderr = StringIO()
-        hook_docker_cmd.main(
-            ['/path/to/hook-docker-cmd'], stdin, stdout, stderr)
-        mock_apply.assert_not_called()
 
-        resp = json.loads(stdout.getvalue())
+    def setUp(self):
+        super(HookDockerCmdTest, self).setUp()
+        self.hook_path = self.relative_path(
+            __file__,
+            '..',
+            'heat-config-docker-cmd/install.d/hook-docker-cmd.py')
+
+        self.cleanup_path = self.relative_path(
+            __file__,
+            '..',
+            'heat-config-docker-cmd/',
+            'os-refresh-config/configure.d/50-heat-config-docker-cmd')
+
+        self.fake_tool_path = self.relative_path(
+            __file__,
+            'config-tool-fake.py')
+
+        self.working_dir = self.useFixture(fixtures.TempDir())
+        self.outputs_dir = self.useFixture(fixtures.TempDir())
+        self.test_state_path = self.outputs_dir.join('test_state.json')
+
+        self.env = os.environ.copy()
+        self.env.update({
+            'HEAT_DOCKER_CMD': self.fake_tool_path,
+            'TEST_STATE_PATH': self.test_state_path,
+        })
+
+    def test_hook(self):
+
+        self.env.update({
+            'TEST_RESPONSE': json.dumps([{
+                'stderr': 'Error: No such image, container or task: db',
+                'returncode': 1
+            }, {
+                'stdout': '',
+                'stderr': 'Creating db...'
+            }, {
+                'stderr': 'Error: No such image, container or task: web',
+                'returncode': 1
+            }, {
+                'stdout': '',
+                'stderr': 'Creating web...'
+            }, {
+                'stdout': 'web',
+            }, {
+
+                'stdout': '',
+                'stderr': 'one.txt\ntwo.txt\nthree.txt'
+            }])
+        })
+        returncode, stdout, stderr = self.run_cmd(
+            [self.hook_path], self.env, json.dumps(self.data))
+
+        self.assertEqual(0, returncode, stderr)
 
         self.assertEqual({
-            'deploy_status_code': 0,
+            'deploy_stdout': '',
-            'deploy_stderr': '',
+            'deploy_stderr': 'Creating db...\n'
-            'deploy_stdout': ''
+                             'Creating web...\n'
-        }, resp)
+                             'one.txt\ntwo.txt\nthree.txt',
+            'deploy_status_code': 0
+        }, json.loads(stdout))
+
+        state = list(self.json_from_files(self.test_state_path, 6))
+        self.assertEqual([
+            self.fake_tool_path,
+            'inspect',
+            '--format',
+            'exists',
+            'db',
+        ], state[0]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'run',
+            '--name',
+            'db',
+            '--label',
+            'deploy_stack_id=the_stack',
+            '--label',
+            'deploy_resource_name=the_deployment',
+            '--label',
+            'config_id=abc123',
+            '--label',
+            'container_name=db',
+            '--label',
+            'managed_by=docker-cmd',
+            '--detach=true',
+            '--env-file=env.file',
+            '--env=foo=bar',
+            '--privileged=false',
+            'xxx'
+            ''
+        ], state[1]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'inspect',
+            '--format',
+            'exists',
+            'web',
+        ], state[2]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'run',
+            '--name',
+            'web',
+            '--label',
+            'deploy_stack_id=the_stack',
+            '--label',
+            'deploy_resource_name=the_deployment',
+            '--label',
+            'config_id=abc123',
+            '--label',
+            'container_name=web',
+            '--label',
+            'managed_by=docker-cmd',
+            '--detach=true',
+            '--env-file=foo.env',
+            '--env-file=bar.conf',
+            '--env=KOLLA_CONFIG_STRATEGY=COPY_ALWAYS',
+            '--env=FOO=BAR',
+            '--net=host',
+            '--privileged=true',
+            '--restart=always',
+            '--user=root',
+            '--volume=/run:/run',
+            '--volume=db:/var/lib/db',
+            'yyy',
+            '/bin/webserver',
+            'start'
+        ], state[3]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'ps',
+            '-a',
+            '--filter',
+            'label=container_name=web',
+            '--filter',
+            'label=config_id=abc123',
+            '--format',
+            '{{.Names}}',
+        ], state[4]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'exec',
+            'web',
+            '/bin/ls',
+            '-l'
+        ], state[5]['args'])
+
+    def test_hook_exit_codes(self):
+
+        self.env.update({
+            'TEST_RESPONSE': json.dumps([{
+                'stdout': 'web',
+            }, {
+                'stdout': '',
+                'stderr': 'Warning: custom exit code',
+                'returncode': 1
+            }])
+        })
+        returncode, stdout, stderr = self.run_cmd(
+            [self.hook_path], self.env, json.dumps(self.data_exit_code))
+
+        self.assertEqual({
+            'deploy_stdout': '',
+            'deploy_stderr': 'Warning: custom exit code',
+            'deploy_status_code': 0
+        }, json.loads(stdout))
+
+        state = list(self.json_from_files(self.test_state_path, 2))
+        self.assertEqual([
+            self.fake_tool_path,
+            'ps',
+            '-a',
+            '--filter',
+            'label=container_name=web',
+            '--filter',
+            'label=config_id=abc123',
+            '--format',
+            '{{.Names}}',
+        ], state[0]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'exec',
+            'web',
+            '/bin/ls',
+            '-l'
+        ], state[1]['args'])
+
+    def test_hook_failed(self):
+
+        self.env.update({
+            'TEST_RESPONSE': json.dumps([{
+                'stderr': 'Error: No such image, container or task: db',
+                'returncode': 1
+            }, {
+                'stdout': '',
+                'stderr': 'Creating db...'
+            }, {
+                'stderr': 'Error: No such image, container or task: web',
+                'returncode': 1
+            }, {
+                'stdout': '',
+                'stderr': 'Creating web...'
+            }, {
+                'stdout': 'web',
+            }, {
+                'stdout': '',
+                'stderr': 'No such file or directory',
+                'returncode': 2
+            }])
+        })
+        returncode, stdout, stderr = self.run_cmd(
+            [self.hook_path], self.env, json.dumps(self.data))
+
+        self.assertEqual({
+            'deploy_stdout': '',
+            'deploy_stderr': 'Creating db...\n'
+                             'Creating web...\n'
+                             'No such file or directory',
+            'deploy_status_code': 2
+        }, json.loads(stdout))
+
+        state = list(self.json_from_files(self.test_state_path, 6))
+        self.assertEqual([
+            self.fake_tool_path,
+            'inspect',
+            '--format',
+            'exists',
+            'db',
+        ], state[0]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'run',
+            '--name',
+            'db',
+            '--label',
+            'deploy_stack_id=the_stack',
+            '--label',
+            'deploy_resource_name=the_deployment',
+            '--label',
+            'config_id=abc123',
+            '--label',
+            'container_name=db',
+            '--label',
+            'managed_by=docker-cmd',
+            '--detach=true',
+            '--env-file=env.file',
+            '--env=foo=bar',
+            '--privileged=false',
+            'xxx'
+        ], state[1]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'inspect',
+            '--format',
+            'exists',
+            'web',
+        ], state[2]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'run',
+            '--name',
+            'web',
+            '--label',
+            'deploy_stack_id=the_stack',
+            '--label',
+            'deploy_resource_name=the_deployment',
+            '--label',
+            'config_id=abc123',
+            '--label',
+            'container_name=web',
+            '--label',
+            'managed_by=docker-cmd',
+            '--detach=true',
+            '--env-file=foo.env',
+            '--env-file=bar.conf',
+            '--env=KOLLA_CONFIG_STRATEGY=COPY_ALWAYS',
+            '--env=FOO=BAR',
+            '--net=host',
+            '--privileged=true',
+            '--restart=always',
+            '--user=root',
+            '--volume=/run:/run',
+            '--volume=db:/var/lib/db',
+            'yyy',
+            '/bin/webserver',
+            'start'
+        ], state[3]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'ps',
+            '-a',
+            '--filter',
+            'label=container_name=web',
+            '--filter',
+            'label=config_id=abc123',
+            '--format',
+            '{{.Names}}',
+        ], state[4]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'exec',
+            'web',
+            '/bin/ls',
+            '-l'
+        ], state[5]['args'])
+
+    def test_hook_unique_names(self):
+
+        self.env.update({
+            'TEST_RESPONSE': json.dumps([{
+                'stdout': 'exists\n',
+                'returncode': 0
+            }, {
+                'stderr': 'Error: No such image, container or task: db-blah',
+                'returncode': 1
+            }, {
+                'stdout': '',
+                'stderr': 'Creating db...'
+            }, {
+                'stdout': 'exists\n',
+                'returncode': 0
+            }, {
+                'stderr': 'Error: No such image, container or task: web-blah',
+                'returncode': 1
+            }, {
+                'stdout': '',
+                'stderr': 'Creating web...'
+            }, {
+                'stdout': 'web-asdf1234',
+            }, {
+                'stdout': '',
+                'stderr': 'one.txt\ntwo.txt\nthree.txt'
+            }])
+        })
+        returncode, stdout, stderr = self.run_cmd(
+            [self.hook_path], self.env, json.dumps(self.data))
+
+        self.assertEqual(0, returncode, stderr)
+
+        self.assertEqual({
+            'deploy_stdout': '',
+            'deploy_stderr': 'Creating db...\n'
+                             'Creating web...\n'
+                             'one.txt\ntwo.txt\nthree.txt',
+            'deploy_status_code': 0
+        }, json.loads(stdout))
+
+        state = list(self.json_from_files(self.test_state_path, 8))
+        db_container_name = state[1]['args'][4]
+        web_container_name = state[4]['args'][4]
+        self.assertRegex(db_container_name, 'db-[0-9a-z]{8}')
+        self.assertRegex(web_container_name, 'web-[0-9a-z]{8}')
+        self.assertEqual([
+            self.fake_tool_path,
+            'inspect',
+            '--format',
+            'exists',
+            'db',
+        ], state[0]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'inspect',
+            '--format',
+            'exists',
+            db_container_name,
+        ], state[1]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'run',
+            '--name',
+            db_container_name,
+            '--label',
+            'deploy_stack_id=the_stack',
+            '--label',
+            'deploy_resource_name=the_deployment',
+            '--label',
+            'config_id=abc123',
+            '--label',
+            'container_name=db',
+            '--label',
+            'managed_by=docker-cmd',
+            '--detach=true',
+            '--env-file=env.file',
+            '--env=foo=bar',
+            '--privileged=false',
+            'xxx'
+        ], state[2]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'inspect',
+            '--format',
+            'exists',
+            'web',
+        ], state[3]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'inspect',
+            '--format',
+            'exists',
+            web_container_name,
+        ], state[4]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'run',
+            '--name',
+            web_container_name,
+            '--label',
+            'deploy_stack_id=the_stack',
+            '--label',
+            'deploy_resource_name=the_deployment',
+            '--label',
+            'config_id=abc123',
+            '--label',
+            'container_name=web',
+            '--label',
+            'managed_by=docker-cmd',
+            '--detach=true',
+            '--env-file=foo.env',
+            '--env-file=bar.conf',
+            '--env=KOLLA_CONFIG_STRATEGY=COPY_ALWAYS',
+            '--env=FOO=BAR',
+            '--net=host',
+            '--privileged=true',
+            '--restart=always',
+            '--user=root',
+            '--volume=/run:/run',
+            '--volume=db:/var/lib/db',
+            'yyy',
+            '/bin/webserver',
+            'start'
+        ], state[5]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'ps',
+            '-a',
+            '--filter',
+            'label=container_name=web',
+            '--filter',
+            'label=config_id=abc123',
+            '--format',
+            '{{.Names}}',
+        ], state[6]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'exec',
+            'web-asdf1234',
+            '/bin/ls',
+            '-l'
+        ], state[7]['args'])
+
+    def test_cleanup_deleted(self):
+        self.env.update({
+            'TEST_RESPONSE': json.dumps([{
+                # first run, no running containers
+                'stdout': '\n'
+            }, {
+                # list name and container_name label for all containers
+                'stdout': '\n'
+            }])
+        })
+        conf_dir = self.useFixture(fixtures.TempDir()).join()
+        with tempfile.NamedTemporaryFile(dir=conf_dir, delete=False) as f:
+            f.write(json.dumps([self.data]).encode('utf-8', 'replace'))
+            f.flush()
+            self.env['HEAT_SHELL_CONFIG'] = f.name
+
+            returncode, stdout, stderr = self.run_cmd(
+                [self.cleanup_path], self.env)
+
+        # on the first run, no docker rm calls made
+        state = list(self.json_from_files(self.test_state_path, 2))
+        self.assertEqual([
+            self.fake_tool_path,
+            'ps',
+            '-a',
+            '--filter',
+            'label=managed_by=docker-cmd',
+            '--format',
+            '{{.Label "config_id"}}'
+        ], state[0]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'ps',
+            '-a',
+            '--format',
+            '{{.Names}} {{.Label "container_name"}}'
+        ], state[1]['args'])
+
+        self.env.update({
+            'TEST_RESPONSE': json.dumps([{
+                # list config_id labels, 3 containers same config
+                'stdout': 'abc123\nabc123\nabc123\n'
+            }, {
+                # list containers with config_id
+                'stdout': '111\n222\n333\n'
+            }, {
+                'stdout': '111 deleted'
+            }, {
+                'stdout': '222 deleted'
+            }, {
+                'stdout': '333 deleted'
+            }, {
+                # list name and container_name label for all containers
+                'stdout': '\n'
+            }])
+        })
+
+        # run again with empty config data
+        with tempfile.NamedTemporaryFile(dir=conf_dir, delete=False) as f:
+            f.write(json.dumps([]).encode('utf-8', 'replace'))
+            f.flush()
+            self.env['HEAT_SHELL_CONFIG'] = f.name
+
+            returncode, stdout, stderr = self.run_cmd(
+                [self.cleanup_path], self.env)
+
+        # on the second run, abc123 is deleted,
+        # docker rm is run on all containers
+        state = list(self.json_from_files(self.test_state_path, 6))
+        self.assertEqual([
+            self.fake_tool_path,
+            'ps',
+            '-a',
+            '--filter',
+            'label=managed_by=docker-cmd',
+            '--format',
+            '{{.Label "config_id"}}'
+        ], state[0]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'ps',
+            '-q',
+            '-a',
+            '--filter',
+            'label=managed_by=docker-cmd',
+            '--filter',
+            'label=config_id=abc123'
+        ], state[1]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'rm',
+            '-f',
+            '111',
+        ], state[2]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'rm',
+            '-f',
+            '222',
+        ], state[3]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'rm',
+            '-f',
+            '333',
+        ], state[4]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'ps',
+            '-a',
+            '--format',
+            '{{.Names}} {{.Label "container_name"}}'
+        ], state[5]['args'])
+
+    def test_cleanup_changed(self):
+        self.env.update({
+            'TEST_RESPONSE': json.dumps([{
+                # list config_id labels, 3 containers same config
+                'stdout': 'abc123\nabc123\nabc123\n'
+            }, {
+                # list name and container_name label for all containers
+                'stdout': '111 111\n'
+                          '222 222\n'
+                          '333\n'
+            }])
+        })
+        conf_dir = self.useFixture(fixtures.TempDir()).join()
+        with tempfile.NamedTemporaryFile(dir=conf_dir, delete=False) as f:
+            f.write(json.dumps([self.data]).encode('utf-8', 'replace'))
+            f.flush()
+            self.env['HEAT_SHELL_CONFIG'] = f.name
+
+            returncode, stdout, stderr = self.run_cmd(
+                [self.cleanup_path], self.env)
+
+        # on the first run, no docker rm calls made
+        state = list(self.json_from_files(self.test_state_path, 2))
+        self.assertEqual([
+            self.fake_tool_path,
+            'ps',
+            '-a',
+            '--filter',
+            'label=managed_by=docker-cmd',
+            '--format',
+            '{{.Label "config_id"}}'
+        ], state[0]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'ps',
+            '-a',
+            '--format',
+            '{{.Names}} {{.Label "container_name"}}'
+        ], state[1]['args'])
+
+        # run again with changed config data
+        self.env.update({
+            'TEST_RESPONSE': json.dumps([{
+                # list config_id labels, 3 containers same config
+                'stdout': 'abc123\nabc123\nabc123\n'
+            }, {
+                # list containers with config_id
+                'stdout': '111\n222\n333\n'
+            }, {
+                'stdout': '111 deleted'
+            }, {
+                'stdout': '222 deleted'
+            }, {
+                'stdout': '333 deleted'
+            }, {
+                # list name and container_name label for all containers
+                'stdout': 'abc123 abc123\n'
+            }])
+        })
+        new_data = copy.deepcopy(self.data)
+        new_data['config']['web']['image'] = 'yyy'
+        new_data['id'] = 'def456'
+        with tempfile.NamedTemporaryFile(dir=conf_dir, delete=False) as f:
+            f.write(json.dumps([new_data]).encode('utf-8', 'replace'))
+            f.flush()
+            self.env['HEAT_SHELL_CONFIG'] = f.name
+
+            returncode, stdout, stderr = self.run_cmd(
+                [self.cleanup_path], self.env)
+
+        # on the second run, abc123 is deleted,
+        # docker rm is run on all containers
+        state = list(self.json_from_files(self.test_state_path, 6))
+        self.assertEqual([
+            self.fake_tool_path,
+            'ps',
+            '-a',
+            '--filter',
+            'label=managed_by=docker-cmd',
+            '--format',
+            '{{.Label "config_id"}}'
+        ], state[0]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'ps',
+            '-q',
+            '-a',
+            '--filter',
+            'label=managed_by=docker-cmd',
+            '--filter',
+            'label=config_id=abc123'
+        ], state[1]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'rm',
+            '-f',
+            '111',
+        ], state[2]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'rm',
+            '-f',
+            '222',
+        ], state[3]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'rm',
+            '-f',
+            '333',
+        ], state[4]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'ps',
+            '-a',
+            '--format',
+            '{{.Names}} {{.Label "container_name"}}'
+        ], state[5]['args'])
+
+    def test_cleanup_rename(self):
+        self.env.update({
+            'TEST_RESPONSE': json.dumps([{
+                # list config_id labels, 3 containers same config
+                'stdout': 'abc123\nabc123\nabc123\n'
+            }, {
+                # list name and container_name label for all containers
+                'stdout': '111 111-s84nf83h\n'
+                          '222 222\n'
+                          '333 333-3nd83nfi\n'
+            }])
+        })
+        conf_dir = self.useFixture(fixtures.TempDir()).join()
+        with tempfile.NamedTemporaryFile(dir=conf_dir, delete=False) as f:
+            f.write(json.dumps([self.data]).encode('utf-8', 'replace'))
+            f.flush()
+            self.env['HEAT_SHELL_CONFIG'] = f.name
+
+            returncode, stdout, stderr = self.run_cmd(
+                [self.cleanup_path], self.env)
+
+        # on the first run, no docker rm calls made
+        state = list(self.json_from_files(self.test_state_path, 4))
+        self.assertEqual([
+            self.fake_tool_path,
+            'ps',
+            '-a',
+            '--filter',
+            'label=managed_by=docker-cmd',
+            '--format',
+            '{{.Label "config_id"}}'
+        ], state[0]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'ps',
+            '-a',
+            '--format',
+            '{{.Names}} {{.Label "container_name"}}'
+        ], state[1]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'rename',
+            '111',
+            '111-s84nf83h'
+        ], state[2]['args'])
+        self.assertEqual([
+            self.fake_tool_path,
+            'rename',
+            '333',
+            '333-3nd83nfi'
+        ], state[3]['args'])