Merge "Add env_file support for docker-compose hook"

hot/software-config/elements/heat-config-docker-compose/README.rst

@@ -1 +1,16 @@
-A hook which uses 'docker-compose' to deploy containers.
+A hook which uses `docker-compose` to deploy containers.
+
+A special input 'env_files' can be used with SoftwareConfig and
+StructuredConfig for docker-compose `env_file` key(s).
+
+if env_file keys specified in the `docker-compose.yml`, do not
+exist in input_values supplied, docker-compose will throw an
+error, as it can't find these files.
+
+Also, `-Pf` option can be used to pass env files from client.
+
+Example:
+
+$ heat stack-create test_stack -f example-docker-compose-template.yaml \
+    -Pf env_file_0=./common.env -Pf env_file_1=./apps/web.env \
+    -Pf env_file_2=./test.env -Pf env_file_3=./busybox.env

hot/software-config/elements/heat-config-docker-compose/install.d/50-heat-config-hook-docker-compose

@@ -12,6 +12,6 @@ elif [ -f /etc/redhat-release ]; then
     systemctl enable docker.service
 fi
 
-pip install -U docker-compose
+pip install -U dpath docker-compose
 
 install -D -g root -o root -m 0755 ${SCRIPTDIR}/hook-docker-compose.py /var/lib/heat-config/hooks/docker-compose

hot/software-config/elements/heat-config-docker-compose/install.d/hook-docker-compose.py

@@ -12,11 +12,14 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import ast
+import dpath
 import json
 import logging
 import os
 import subprocess
 import sys
+import yaml
 
 
 WORKING_DIR = os.environ.get('HEAT_DOCKER_COMPOSE_WORKING',
@@ -31,6 +34,21 @@ def prepare_dir(path):
         os.makedirs(path, 0o700)
 
 
+def write_input_file(file_path, content):
+    prepare_dir(os.path.dirname(file_path))
+    with os.fdopen(os.open(
+            file_path, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f:
+        f.write(content.encode('utf-8'))
+
+
+def build_response(deploy_stdout, deploy_stderr, deploy_status_code):
+    return {
+        'deploy_stdout': deploy_stdout,
+        'deploy_stderr': deploy_stderr,
+        'deploy_status_code': deploy_status_code,
+    }
+
+
 def main(argv=sys.argv):
     log = logging.getLogger('heat-config')
     handler = logging.StreamHandler(sys.stderr)
@@ -50,25 +68,38 @@ def main(argv=sys.argv):
     stdout, stderr = {}, {}
 
     if input_values.get('deploy_action') == 'DELETE':
-        response = {
+        json.dump(build_response(stdout, stderr, 0), sys.stdout)
-            'deploy_stdout': stdout,
-            'deploy_stderr': stderr,
-            'deploy_status_code': 0,
-        }
-        json.dump(response, sys.stdout)
         return
 
     config = c.get('config', '')
     if not config:
         log.debug("No 'config' input found, nothing to do.")
-        response = {
+        json.dump(build_response(stdout, stderr, 0), sys.stdout)
-            'deploy_stdout': stdout,
-            'deploy_stderr': stderr,
-            'deploy_status_code': 0,
-        }
-        json.dump(response, sys.stdout)
         return
 
+    #convert config to dict
+    if not isinstance(config, dict):
+        config = ast.literal_eval(json.dumps(yaml.load(config)))
+
+    os.chdir(proj)
+
+    compose_env_files = []
+    for value in dpath.util.values(config, '*/env_file'):
+        if isinstance(value, list):
+            compose_env_files.extend(value)
+        elif isinstance(value, basestring):
+            compose_env_files.extend([value])
+
+    input_env_files = {}
+    if input_values.get('env_files'):
+        input_env_files = dict(
+            (i['file_name'], i['content'])
+            for i in ast.literal_eval(input_values.get('env_files')))
+
+    for file in compose_env_files:
+        if file in input_env_files.keys():
+            write_input_file(file, input_env_files.get(file))
+
     cmd = [
         DOCKER_COMPOSE_CMD,
         'up',
@@ -78,8 +109,6 @@ def main(argv=sys.argv):
 
     log.debug('Running %s' % cmd)
 
-    os.chdir(proj)
-
     subproc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE)
     stdout, stderr = subproc.communicate()
@@ -92,15 +121,7 @@ def main(argv=sys.argv):
     else:
         log.debug('Completed %s' % cmd)
 
-    response = {}
+    json.dump(build_response(stdout, stderr, subproc.returncode), sys.stdout)
-
-    response.update({
-        'deploy_stdout': stdout,
-        'deploy_stderr': stderr,
-        'deploy_status_code': subproc.returncode,
-    })
-
-    json.dump(response, sys.stdout)
 
 if __name__ == '__main__':
     sys.exit(main(sys.argv))

hot/software-config/example-templates/config-scripts/example-docker-compose.yml

@@ -0,0 +1,6 @@
+busybox:
+  env_file: ./busybox.env
+  image: busybox
+  command: ['nc', '-p', '8080', '-l', '-l', '-e', 'echo', 'hello world!']
+  ports:
+  - 8080:8080

hot/software-config/example-templates/example-docker-compose-template.yaml

@@ -15,6 +15,15 @@ parameters:
   public_net:
     type: string
     default: public
+  env_file_0:
+     type: string
+  env_file_1:
+     type: string
+  env_file_2:
+     type: string
+  env_file_3:
+     type: string
+
 
 resources:
   the_sg:
@@ -35,16 +44,31 @@ resources:
     type: OS::Heat::StructuredConfig
     properties:
       group: docker-compose
+      inputs:
+      - name: env_files
       config:
         db:
           image: redis
+          env_file:
+          - ./common.env
+          - ./apps/web.env
         web:
           image: nginx
+          env_file: ./test.env
           links:
           - db
           ports:
           - 80:8000
 
+  other_config:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: docker-compose
+      inputs:
+      - name: env_files
+      config:
+        get_file: config-scripts/example-docker-compose.yml
+
   deployment:
     type: OS::Heat::StructuredDeployment
     properties:
@@ -53,6 +77,27 @@ resources:
         get_resource: config
       server:
         get_resource: server
+      input_values:
+        env_files:
+        - file_name: ./common.env
+          content: {get_param: env_file_0}
+        - file_name: ./apps/web.env
+          content: {get_param: env_file_1}
+        - file_name: ./test.env
+          content: {get_param: env_file_2}
+
+  other_deployment:
+    type: OS::Heat::SoftwareDeployment
+    properties:
+      name: other_deployment
+      config:
+        get_resource: other_config
+      server:
+        get_resource: server
+      input_values:
+        env_files:
+        - file_name: ./busybox.env
+          content: {get_param: env_file_3}
 
   server:
     type: OS::Nova::Server

test-requirements.txt

@@ -1,5 +1,6 @@
 coverage>=3.6
 discover
+dpath>=1.3.2
 fixtures>=0.3.14
 # Hacking already pins down pep8, pyflakes and flake8
 hacking>=0.8.0,<0.9

tests/software_config/test_hook_docker_compose.py

@@ -23,19 +23,47 @@ class HookDockerComposeTest(common.RunScriptTest):
     data = {
         "id": "abcdef001",
         "group": "docker-compose",
-        "inputs": {},
+        "inputs": [
+            {
+                "name": "env_files",
+                "value": u'[ { "file_name": "./common.env", '
+                         u'"content": "xxxxx" }, '
+                         u'{ "file_name": "./test.env", '
+                         u'"content": "yyyy" }, '
+                         u'{ "file_name": "./test1.env", '
+                         u'"content": "zzz" } ]'
+            }
+        ],
         "config": {
             "web": {
-                "image": "nginx",
+                "name": "x",
-                "links": [
+                "env_file": [
-                    "db"
+                    "./common.env",
-                ],
+                    "./test.env"
-                "ports": [
-                    "8000:8000"
                 ]
             },
             "db": {
-                "image": "redis"
+                "name": "y",
+                "env_file": "./test1.env"
+            }
+        }
+    }
+
+    data_without_input = {
+        "id": "abcdef001",
+        "group": "docker-compose",
+        "inputs": [],
+        "config": {
+            "web": {
+                "name": "x",
+                "env_file": [
+                    "./common.env",
+                    "./test.env"
+                ]
+            },
+            "db": {
+                "name": "y",
+                "env_file": "./test1.env"
             }
         }
     }
@@ -92,6 +120,34 @@ class HookDockerComposeTest(common.RunScriptTest):
             ],
             state['args'])
 
+    def test_hook_without_inputs(self):
+
+        self.env.update({
+            'TEST_RESPONSE': json.dumps({
+                'stdout': '',
+                'stderr': 'env_file_not found...',
+                'returncode': 1
+            })
+        })
+        returncode, stdout, stderr = self.run_cmd(
+            [self.hook_path], self.env, json.dumps(self.data_without_input))
+
+        self.assertEqual({
+            'deploy_stdout': '',
+            'deploy_stderr': 'env_file_not found...',
+            'deploy_status_code': 1
+        }, json.loads(stdout))
+
+        state = self.json_from_file(self.test_state_path)
+        self.assertEqual(
+            [
+                self.fake_tool_path,
+                'up',
+                '-d',
+                '--no-build',
+            ],
+            state['args'])
+
     def test_hook_failed(self):
 
         self.env.update({
@@ -104,8 +160,6 @@ class HookDockerComposeTest(common.RunScriptTest):
         returncode, stdout, stderr = self.run_cmd(
             [self.hook_path], self.env, json.dumps(self.data))
 
-        self.assertEqual(0, returncode, stderr)
-
         self.assertEqual({
             'deploy_stdout': '',
             'deploy_stderr': 'Error: image library/xxx:latest not found',