Merge branch 'x-fixes' into x

x/db.py

@@ -11,3 +11,9 @@ def resource_add(key, value):
 
 def get_resource(key):
     return RESOURCE_DB.get(key, None)
+
+
+def clear():
+    global RESOURCE_DB
+
+    RESOURCE_DB = {}

x/deployment.py

@@ -17,9 +17,8 @@ def deploy(filename):
     resource_save_path = os.path.join(workdir, config['resource-save-path'])
 
     # Clean stuff first
-    clients_file = os.path.join(workdir, 'clients.json')
-    if os.path.exists(clients_file):
-        os.remove(clients_file)
+    db.clear()
+    xs.clear()
     shutil.rmtree(resource_save_path, ignore_errors=True)
     os.makedirs(resource_save_path)
 

x/handlers/ansible.py

@@ -1,6 +1,7 @@
 # -*- coding: UTF-8 -*-
 import os
 import subprocess
+import yaml
 
 from x.handlers.base import BaseHandler
 
@@ -11,7 +12,9 @@ class Ansible(BaseHandler):
         playbook_file = self._create_playbook(resource, action_name)
         print 'inventory_file', inventory_file
         print 'playbook_file', playbook_file
-        subprocess.call(['ansible-playbook', '-i', inventory_file, playbook_file])
+        call_args = ['ansible-playbook', '-i', inventory_file, playbook_file]
+        print 'EXECUTING: ', ' '.join(call_args)
+        subprocess.call(call_args)
 
     #def _get_connection(self, resource):
     #    return {'ssh_user': '',

x/resource.py

@@ -19,7 +19,7 @@ class Resource(object):
         self.metadata = metadata
         self.actions = metadata['actions'].keys() if metadata['actions'] else None
         self.requires = metadata['input'].keys()
-        self._validate_args(args)
+        self._validate_args(args, metadata['input'])
         self.args = args
         self.metadata['input'] = args
         self.input_types = metadata.get('input-types', {})
@@ -63,10 +63,15 @@ class Resource(object):
         else:
             raise Exception('Uuups, action is not available')
 
-    def _validate_args(self, args):
+    def _validate_args(self, args, inputs):
         for req in self.requires:
             if req not in args:
-                raise Exception('Requirement `{0}` is missing in args'.format(req))
+                # If metadata input is filled with a value, use it as default
+                # and don't report an error
+                if inputs.get(req):
+                    args[req] = inputs[req]
+                else:
+                    raise Exception('Requirement `{0}` is missing in args'.format(req))
 
     # TODO: versioning
     def save(self):

x/resources/docker_container/actions/run.yml

@@ -2,5 +2,23 @@
 - hosts: [{{ ip }}]
   sudo: yes
   tasks:
-    - shell: docker run -d --net="host" --privileged \
-        --name {{ name }} {{ image }}
+    - apt: name=python-pip state=present
+    - shell: pip install docker-py
+    - service: name=docker state=started
+    - docker:
+       name: {{ name }}
+       image: {{ image }}
+       state: running
+       ports:
+         {% for name, port in ports.items() %}
+         # {{ name }}
+         - {{ port }}:{{ port }}
+         {% endfor %}
+       volumes:
+         # TODO: host_binds might need more work
+         # Currently it's not that trivial to pass custom src: dst here
+         # (when a config variable is passed here from other resource)
+         # so we mount it to the same directory as on host
+         {% for emitter, bind in host_binds.items() %}
+         - {{ bind }}:{{ bind }}
+         {% endfor %}

x/resources/docker_container/meta.yaml

@@ -3,9 +3,13 @@ handler: ansible
 version: 1.0.0
 input:
     ip:
-    image: 
+    image:
+    ports:
     host_binds:
     volume_binds:
+    ssh_user:
+    ssh_key:
 input-types:
+    ports:
     host_binds: list
     volume_binds: list

x/resources/haproxy/actions/run.yml

@@ -1,6 +1,23 @@
 # TODO
 - hosts: [{{ ip }}]
   sudo: yes
+  vars:
+    config_dir: {{ config_dir }}
+    haproxy_ip: {{ ip }}
+    haproxy_services:
+      {% for service, servers in configs.items() %}
+      - name: {{ service }}
+        listen_port: {{ listen_ports[service] }}
+        servers:
+          {% for name, ip in servers.items() %}
+          - name: {{ name }}
+            ip: {{ ip }}
+            port: {{ configs_ports[service][name] }}
+          {% endfor %}
+      {% endfor %}
   tasks:
-    - shell: docker run -d --net="host" --privileged \
-        --name {{ name }} {{ image }}
+    - apt: name=python-pip state=present
+    - shell: pip install docker-py
+    - service: name=docker state=started
+    - file: path=/etc/haproxy/haproxy.cfg state=touch
+    - template: src=/vagrant/haproxy.cfg dest=/etc/haproxy/haproxy.cfg

x/resources/haproxy/meta.yaml

@@ -3,6 +3,13 @@ handler: ansible
 version: 1.0.0
 input:
     ip:
+    config_dir: /etc/haproxy
+    listen_ports:
     configs:
+    configs_ports:
+    ssh_user:
+    ssh_key:
 input-types:
+    listen_ports: list
     configs: list
+    configs_ports: list

x/resources/haproxy_config/meta.yaml

@@ -2,6 +2,9 @@ id: haproxy_config
 handler: ansible
 version: 1.0.0
 input:
+    listen_port:
+    ports:
     servers:
 input-types:
+    ports: list
     servers: list

x/resources/keystone/meta.yaml

@@ -3,4 +3,5 @@ handler: ansible
 version: 1.0.0
 input:
     ip:
+    port: 5000
     image: garland/docker-openstack-keystone

x/resources/nova/meta.yaml

@@ -3,4 +3,5 @@ handler: ansible
 version: 1.0.0
 input:
     ip:
+    port: 8774
     image: # TODO

x/signals.py

@@ -2,6 +2,7 @@
 from collections import defaultdict
 import itertools
 import networkx as nx
+import os
 
 import db
 
@@ -12,6 +13,16 @@ CLIENTS_CONFIG_KEY = 'clients-data-file'
 CLIENTS = utils.read_config_file(CLIENTS_CONFIG_KEY)
 
 
+def clear():
+    global CLIENTS
+
+    CLIENTS = {}
+
+    path = utils.read_config()[CLIENTS_CONFIG_KEY]
+    if os.path.exists(path):
+        os.remove(path)
+
+
 def guess_mapping(emitter, receiver):
     """Guess connection mapping between emitter and receiver.
 
@@ -155,3 +166,17 @@ def connection_graph():
         )
 
     return g
+
+
+def detailed_connection_graph():
+    g = nx.MultiDiGraph()
+
+    for emitter_name, destination_values in CLIENTS.items():
+        for emitter_input, receivers in CLIENTS[emitter_name].items():
+            for receiver_name, receiver_input in receivers:
+                label = emitter_input
+                if emitter_input != receiver_input:
+                    label = '{}:{}'.format(emitter_input, receiver_input)
+                g.add_edge(emitter_name, receiver_name, label=label)
+
+    return g

x/test/__init__.py

@@ -0,0 +1 @@
+__author__ = 'przemek'

x/test/base.py

@@ -0,0 +1,36 @@
+import os
+import shutil
+import tempfile
+import unittest
+import yaml
+
+from x import db
+from x import resource as xr
+from x import signals as xs
+
+
+class BaseResourceTest(unittest.TestCase):
+    def setUp(self):
+        self.storage_dir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.storage_dir)
+        db.clear()
+        xs.clear()
+
+    def make_resource_meta(self, meta_yaml):
+        meta = yaml.load(meta_yaml)
+
+        path = os.path.join(self.storage_dir, meta['id'])
+        os.makedirs(path)
+        with open(os.path.join(path, 'meta.yaml'), 'w') as f:
+            f.write(meta_yaml)
+
+        return path
+
+    def create_resource(self, name, src, args):
+        dst = os.path.join(self.storage_dir, 'rs', name)
+        os.makedirs(dst)
+
+        return xr.create(name, src, dst, args)
+

x/test/test_signals.py

@@ -0,0 +1,140 @@
+import unittest
+
+import base
+
+from x import signals as xs
+
+
+class TestBaseInput(base.BaseResourceTest):
+    def test_input_dict_type(self):
+        sample_meta_dir = self.make_resource_meta("""
+id: sample
+handler: ansible
+version: 1.0.0
+input:
+  values: {}
+        """)
+
+        sample1 = self.create_resource(
+            'sample1', sample_meta_dir, {'values': {'a': 1, 'b': 2}}
+        )
+        sample2 = self.create_resource(
+            'sample2', sample_meta_dir, {'values': None}
+        )
+        xs.connect(sample1, sample2)
+        self.assertItemsEqual(
+            sample1.args['values'],
+            sample2.args['values'],
+        )
+
+
+class TestListInput(base.BaseResourceTest):
+    def test_list_input_single(self):
+        sample_meta_dir = self.make_resource_meta("""
+id: sample
+handler: ansible
+version: 1.0.0
+input:
+  ip:
+        """)
+        list_input_single_meta_dir = self.make_resource_meta("""
+id: list-input-single
+handler: ansible
+version: 1.0.0
+input:
+  ips:
+input-types:
+  ips: list
+        """)
+
+        sample1 = self.create_resource(
+            'sample1', sample_meta_dir, {'ip': '10.0.0.1'}
+        )
+        sample2 = self.create_resource(
+            'sample2', sample_meta_dir, {'ip': '10.0.0.2'}
+        )
+        list_input_single = self.create_resource(
+            'list-input-single', list_input_single_meta_dir, {'ips': {}}
+        )
+
+        xs.connect(sample1, list_input_single, mapping={'ip': 'ips'})
+        self.assertItemsEqual(
+            list_input_single.args['ips'],
+            {
+                'sample1': sample1.args['ip'],
+            }
+        )
+
+        xs.connect(sample2, list_input_single, mapping={'ip': 'ips'})
+        self.assertItemsEqual(
+            list_input_single.args['ips'],
+            {
+                'sample1': sample1.args['ip'],
+                'sample2': sample2.args['ip'],
+            }
+        )
+
+    def test_list_input_multi(self):
+        sample_meta_dir = self.make_resource_meta("""
+id: sample
+handler: ansible
+version: 1.0.0
+input:
+  ip:
+  port:
+        """)
+        list_input_multi_meta_dir = self.make_resource_meta("""
+id: list-input-multi
+handler: ansible
+version: 1.0.0
+input:
+  ips:
+  ports:
+input-types:
+  ips: list
+  ports: list
+        """)
+
+        sample1 = self.create_resource(
+            'sample1', sample_meta_dir, {'ip': '10.0.0.1', 'port': '1000'}
+        )
+        sample2 = self.create_resource(
+            'sample2', sample_meta_dir, {'ip': '10.0.0.2', 'port': '1001'}
+        )
+        list_input_multi = self.create_resource(
+            'list-input-multi', list_input_multi_meta_dir, {'ips': {}, 'ports': {}}
+        )
+
+        xs.connect(sample1, list_input_multi, mapping={'ip': 'ips', 'port': 'ports'})
+        self.assertItemsEqual(
+            list_input_multi.args['ips'],
+            {
+                'sample1': sample1.args['ip'],
+            }
+        )
+        self.assertItemsEqual(
+            list_input_multi.args['ports'],
+            {
+                'sample1': sample1.args['port'],
+            }
+        )
+
+        xs.connect(sample2, list_input_multi, mapping={'ip': 'ips', 'port': 'ports'})
+        self.assertItemsEqual(
+            list_input_multi.args['ips'],
+            {
+                'sample1': sample1.args['ip'],
+                'sample2': sample2.args['ip'],
+            }
+        )
+        self.assertItemsEqual(
+            list_input_multi.args['ports'],
+            {
+                'sample1': sample1.args['port'],
+                'sample2': sample2.args['port'],
+            }
+        )
+
+
+if __name__ == '__main__':
+    unittest.main()