diff --git a/.gitignore b/.gitignore index cb4402c7..83a9627c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ # vagrant .vagrant +tmp/ diff --git a/schema/actions.yml b/schema/actions.yml deleted file mode 100644 index afc89f99..00000000 --- a/schema/actions.yml +++ /dev/null @@ -1,13 +0,0 @@ -actions: - - id: erase_node - tasks: - - shell: erase - - shell: reboot - - id: provision_centos - tasks: - - shell: fuel-agent --image - - shell: reboot - - id: deploy_compute - tasks: - - shell: deploy_compute - diff --git a/schema/ha.yml b/schema/ha.yml index 9414e752..3ebbe236 100644 --- a/schema/ha.yml +++ b/schema/ha.yml @@ -2,3 +2,7 @@ # how to control other services by pacemaker/corosync? if they will # be installed in containers + + +# how we will manage primary and non-primary analog or resources? +# maybe they should be just assigned by user ? diff --git a/schema/resources/compose.yml b/schema/resources/compose.yml deleted file mode 100644 index f390b8a3..00000000 --- a/schema/resources/compose.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- - -id: compose -type: resource -handler: playbook -version: v1 -run: - - shell: docker-compose -f {{compose.item}} -d -parameters: - item: - required: true - default: /some/path.yml diff --git a/schema/resources/docker.yml b/schema/resources/docker.yml index a5482666..86af9522 100644 --- a/schema/resources/docker.yml +++ b/schema/resources/docker.yml @@ -19,9 +19,11 @@ actions: register: docker_version - shell: curl -sSL https://get.docker.com/ubuntu/ | sudo sh when: docker_version|failed - - shell: docker pull debian:jessie + - shell: docker pull {{ docker.base_image }} remove: - hosts: [docker] sudo: yes tasks: - - shell: apt-get remove -y docker + - shell: apt-get remove -y lxc-docker +input: + base_image: ubuntu diff --git a/schema/resources/docker_compose.yml b/schema/resources/docker_compose.yml new file mode 100644 index 00000000..044bb6c6 --- /dev/null +++ b/schema/resources/docker_compose.yml @@ -0,0 +1,23 @@ +--- + + +id: docker_compose +type: resource +handler: ansible +version: v1 +actions: + run: + - hosts: [docker_compose] + sudo: yes + tasks: + - shell: docker-compose --version + register: compose + ignore_errors: true + - shell: curl -L https://github.com/docker/compose/releases/download/1.1.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + when: compose|failed + - shell: chmod +x /usr/local/bin/docker-compose + remove: + - hosts: [docker_compose] + sudo: yes + tasks: + - shell: rm -rf /usr/local/bin/docker-compose diff --git a/schema/resources/images.yml b/schema/resources/images.yml deleted file mode 100644 index df487e02..00000000 --- a/schema/resources/images.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- - -id: images -type: resource -driver: playbook -version: v1 -run: - - shell: docker pull {{item}} - with_items: images.list -parameters: - list: - required: true - another: - default: 1 diff --git a/schema/resources/networks.yml b/schema/resources/networks.yml deleted file mode 100644 index 6cbd647c..00000000 --- a/schema/resources/networks.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- - -id: networks -type: resource -handler: custom_network_schema -version: v1 - -# run part will be generated - -# what kind of parameters? - diff --git a/schema/resources/node.yml b/schema/resources/node.yml new file mode 100644 index 00000000..d6955b22 --- /dev/null +++ b/schema/resources/node.yml @@ -0,0 +1,12 @@ +--- + +id: node +type: resource +handler: ansible +version: v1 +input: + ssh_host: 127.0.0.1 + ssh_port: 2222 + name: first + user: vagrant + key: .vagrant/machines/default/virtualbox/private_key diff --git a/schema/resources/puppet.yml b/schema/resources/puppet.yml deleted file mode 100644 index 1139300b..00000000 --- a/schema/resources/puppet.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- - - -id: network_eth -handler: network_schema -parameters: - networks: - - name: bbc01 - cidr: 10.0.0.4/24 - ip_ranges: [] -provides: [network.ips] - ----- -id: network_infiniband -handler: playbook // infiniband_schema - - ----- - -id: network_check -handler: playbook -run: - - shell: ping {{item}} - with_items: network.ips - diff --git a/schema/resources/replace_nick.yml b/schema/resources/replace_nick.yml deleted file mode 100644 index dd8bd650..00000000 --- a/schema/resources/replace_nick.yml +++ /dev/null @@ -1,10 +0,0 @@ - -- id: nic - parameters: - first: - default: 1 - events: - - run: - shell: echo 3 - -fuel execute --res nic --action run --node 1,2,3 diff --git a/schema/resources/some_service.yml b/schema/resources/some_service.yml new file mode 100644 index 00000000..c8928bff --- /dev/null +++ b/schema/resources/some_service.yml @@ -0,0 +1,25 @@ +--- + + +id: some_service +type: resource +handler: ansible +version: v1 +actions: + run: + - hosts: [some_service] + sudo: yes + tasks: + - shell: docker run --name {{ some_service.name }} --net="host" + --privileged -d {{ some_service.image }} + nc -l {{ some_service.port }} + remove: + - hosts: [some_service] + sudo: yes + tasks: + - shell: docker stop {{ some_service.name }} + - shell: docker rm {{some_service.name}} +input: + image: "{{ docker.base_image }}" + port: 8800 + name: shining_star diff --git a/solar/requirements.txt b/solar/requirements.txt index d5738ba7..60fc0932 100644 --- a/solar/requirements.txt +++ b/solar/requirements.txt @@ -1,2 +1,3 @@ six>=1.9.0 - +pyyaml +jinja2 diff --git a/solar/solar/cli.py b/solar/solar/cli.py index ac99edf8..f3bbcc83 100644 --- a/solar/solar/cli.py +++ b/solar/solar/cli.py @@ -13,5 +13,53 @@ # under the License. +import sys + +import argparse +import yaml + +from solar.core import ansible +from solar.interfaces.db import Storage + + +def parse(): + + parser = argparse.ArgumentParser() + + parser.add_argument( + '-a', + '--action', + help='action to execute', + required=True) + + parser.add_argument( + '-r', + '--resources', + help='list of resources', + nargs='+', + required=True) + + return parser.parse_args() + + def main(): - print 'Solar' + args = parse() + + print 'ACTION %s' % args.action + print 'RESOURCES %s' % args.resources + + storage = Storage.from_files('./schema/resources') + orch = ansible.AnsibleOrchestration( + [storage.get(r) for r in args.resources]) + + + with open('tmp/hosts', 'w') as f: + f.write(orch.inventory) + + with open('tmp/group_vars/all', 'w') as f: + f.write(yaml.dump(orch.vars, default_flow_style=False)) + + with open('tmp/main.yml', 'w') as f: + f.write( + yaml.dump(getattr(orch, args.action)(), + default_flow_style=False)) diff --git a/solar/solar/core/ansible.py b/solar/solar/core/ansible.py new file mode 100644 index 00000000..1ed35511 --- /dev/null +++ b/solar/solar/core/ansible.py @@ -0,0 +1,58 @@ + +import yaml + +from solar.extensions import resource + +from jinja2 import Template + + +ANSIBLE_INVENTORY = """ +{% for node in nodes %} +{{node.node.name}} ansible_ssh_host={{node.node.ssh_host}} ansible_ssh_port={{node.node.ssh_port}} + +{% endfor %} + +{% for res in resources %} +[{{ res.uid }}] +{% for node in nodes %} {{node.node.name}} {% endfor %} {% endfor %} +""" + + +class AnsibleOrchestration(object): + + def __init__(self, resources): + self.resources = [resource(r) for r in resources + if r['id'] != 'node'] + self.nodes = [resource(r) for r in resources + if r['id'] == 'node'] + + @property + def inventory(self): + temp = Template(ANSIBLE_INVENTORY) + node_data = [n.inventory for n in self.nodes] + return temp.render(nodes=node_data, resources=self.resources) + + @property + def vars(self): + result = {} + + for res in self.resources: + + compiled = Template(yaml.dump(res.inventory)) + compiled = yaml.load(compiled.render(**result)) + + result.update(compiled) + + return result + + def run(self, action='run'): + all_playbooks = [] + + for res in self.resources: + + all_playbooks.extend(res.execute(action)) + + return all_playbooks + + def remove(self): + return list(reversed(self.run(action='remove'))) diff --git a/solar/solar/extensions/__init__.py b/solar/solar/extensions/__init__.py index e69de29b..fe3eaee2 100644 --- a/solar/solar/extensions/__init__.py +++ b/solar/solar/extensions/__init__.py @@ -0,0 +1,6 @@ + +from solar.extensions import playbook + + +def resource(config): + return playbook.Playbook(config) diff --git a/tool/tool/resource_handlers/base.py b/solar/solar/extensions/base.py similarity index 62% rename from tool/tool/resource_handlers/base.py rename to solar/solar/extensions/base.py index 505f7b96..6d59bf57 100644 --- a/tool/tool/resource_handlers/base.py +++ b/solar/solar/extensions/base.py @@ -1,9 +1,7 @@ - - class BaseResource(object): - def __init__(self, config, hosts='all'): + def __init__(self, config): """ config - data described in configuration files hosts - can be overwritten if resource is inside of the role, @@ -11,24 +9,15 @@ class BaseResource(object): """ self.config = config self.uid = config['id'] - self.hosts = hosts def prepare(self): """Make some changes in database state.""" + @property def inventory(self): """Return data that will be used for inventory""" - if 'parameters' in self.config: - params = self.config.get('parameters', {}) + return {self.uid: self.config.get('input', {})} - res = {} - - for param, values in self.config.parameters.items(): - res[param] = values.get('value') or values.get('default') - - return res - - - def run(self): + def execute(self, action): """Return data that will be used by orchestration framework""" raise NotImplemented('Mandatory to overwrite') diff --git a/solar/solar/extensions/playbook.py b/solar/solar/extensions/playbook.py new file mode 100644 index 00000000..535329dd --- /dev/null +++ b/solar/solar/extensions/playbook.py @@ -0,0 +1,8 @@ + +from solar.extensions import base + + +class Playbook(base.BaseResource): + + def execute(self, action): + return self.config.get('actions', {}).get(action, []) diff --git a/solar/solar/interfaces/db/__init__.py b/solar/solar/interfaces/db/__init__.py index e69de29b..ee1c7533 100644 --- a/solar/solar/interfaces/db/__init__.py +++ b/solar/solar/interfaces/db/__init__.py @@ -0,0 +1,39 @@ + +import os +from fnmatch import fnmatch + +import yaml + + +def get_files(path, pattern): + for root, dirs, files in os.walk(path): + for file_name in files: + if fnmatch(file_name, pattern): + yield os.path.join(root, file_name) + + +class Storage(object): + + def __init__(self): + self.entities = {} + + def add(self, resource): + if 'id' in resource: + self.entities[resource['id']] = resource + + def add_resource(self, resource): + if 'id' in resource: + self.entities[resource['id']] = resource + + def get(self, resource_id): + return self.entities[resource_id] + + @classmethod + def from_files(cls, path): + store = cls() + for file_path in get_files(path, '*.yml'): + with open(file_path) as f: + entity = yaml.load(f) + + store.add_resource(entity) + return store diff --git a/tool/setup.py b/tool/setup.py deleted file mode 100644 index c4ccb337..00000000 --- a/tool/setup.py +++ /dev/null @@ -1,13 +0,0 @@ - - -setup( - name='tool', - version='0.1', - license='Apache License 2.0', - include_package_data=True, - install_requires=['ansible', 'networkx', 'pyyaml', 'argparse'], - entry_points=""" - [console_scripts] - tool=tool.main:main - """, -) diff --git a/tool/tool/__init__.py b/tool/tool/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tool/tool/main.py b/tool/tool/main.py deleted file mode 100644 index a221bfcc..00000000 --- a/tool/tool/main.py +++ /dev/null @@ -1,35 +0,0 @@ - -import sys - -import argparse -import yaml - -from tool.profile_handlers import process - - -def parse(): - - parser = argparse.ArgumentParser() - - parser.add_argument( - '-p', - '--profile', - help='profile file', - required=True) - - parser.add_argument( - '-r', - '--resources', - help='resources dir', - required=True) - - return parser.parse_args() - - -def main(): - args = parse() - - with open(args.config) as f: - profile = yaml.load(f) - - return process(profile, resources) diff --git a/tool/tool/profile_handlers/__init__.py b/tool/tool/profile_handlers/__init__.py deleted file mode 100644 index 732a0803..00000000 --- a/tool/tool/profile_handlers/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ - - -from tool.profile_handlers import ansible - - -def process(profile, resources): - - # it should be a fabric - return ansible.process(profile, resources) diff --git a/tool/tool/profile_handlers/ansible.py b/tool/tool/profile_handlers/ansible.py deleted file mode 100644 index 771f9715..00000000 --- a/tool/tool/profile_handlers/ansible.py +++ /dev/null @@ -1,16 +0,0 @@ - -from tool.resource_handlers import playbook -from tool.service_handlers import base - - -class AnsibleProfile(object): - """This profile should just serialize - """ - - def __init__(self, storage, config): - self.config = config - - def inventory(self): - pass - - diff --git a/tool/tool/profile_handlers/ansible_graph.py b/tool/tool/profile_handlers/ansible_graph.py deleted file mode 100644 index c26defa5..00000000 --- a/tool/tool/profile_handlers/ansible_graph.py +++ /dev/null @@ -1,32 +0,0 @@ - -"""The point of different ansible graph handlers is that graph data model -allows to decidy which tasks to run in background. -""" - - -import networkx as nx - - -class ProfileGraph(nx.DiGraph): - - def __init__(self, profile): - super(ProfileGraph, self).__init__() - - def add_resources(self, entity): - resources = entity.get('resources', []) - for res in resources: - self.add_resource(res) - - def add_resource(self, resource): - self.add_node(resource['id']) - for dep in resource.get('requires', []): - self.add_edge(dep, resource['id']) - for dep in resource.get('required_for', []): - self.add_edge(resource['id'], dep) - - -def process(profile, resources): - # here we should know how to traverse profile data model - # that is specific to ansible - - graph = nx.DiGraph(profile) diff --git a/tool/tool/resource_handlers/__init__.py b/tool/tool/resource_handlers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tool/tool/resource_handlers/custom_network_schema.py b/tool/tool/resource_handlers/custom_network_schema.py deleted file mode 100644 index 69b75206..00000000 --- a/tool/tool/resource_handlers/custom_network_schema.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -This handler required for custom modification for the networks resource. - -It will create all required tasks for things like ovs/linux network -entities. -""" - -from tool.resoure_handlers import base - - -class NetworkSchema(base.BaseResource): - - def __init__(self, parameters): - pass - - def add_bridge(self, bridge): - return 'shell: ovs-vsctl add-br {0}'.format(bridge) - - def add_port(self, bridge, port): - return 'shell: ovs-vsctl add-port {0} {1}'.format(bridge, port) diff --git a/tool/tool/resource_handlers/playbook.py b/tool/tool/resource_handlers/playbook.py deleted file mode 100644 index 90e8a3b3..00000000 --- a/tool/tool/resource_handlers/playbook.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -Just find or create existing playbook. -""" - -from tool.resource_handlers import base - - -class Playbook(base.BaseResource): - - def run(self): - return { - 'hosts': self.hosts, - 'tasks': self.config.get('run', []), - 'sudo': 'yes'} diff --git a/tool/tool/storage.py b/tool/tool/storage.py deleted file mode 100644 index 73f7868e..00000000 --- a/tool/tool/storage.py +++ /dev/null @@ -1,54 +0,0 @@ - - -import os -from fnmatch import fnmatch - -import yaml - - -def get_files(path, pattern): - for root, dirs, files in os.walk(path): - for file_name in files: - if fnmatch(file_name, file_pattern): - yield os.path.join(root, file_name) - - -class Storage(object): - - def __init__(self): - self.entities = {} - - def add(self, resource): - if 'id' in resource: - self.entities[resource['id']] = resource - - def add_profile(self, profile): - self.entities[profile['id']] = profile - for res in profile.get('resources', []): - self.add_resource(res) - - def add_resource(self, resource): - if 'id' in resource: - self.entities[resource['id']] = resource - - def add_service(self, service): - if 'id' in service: - self.entities[service['id']] = service - for resource in service.get('resources', []): - self.add_resource(resource) - - def get(self, resource_id): - return self.entities[resource_id] - - @classmethod - def from_files(self, path): - for file_path in get_files(path, '*.yml'): - with open(file_path) as f: - entity = yaml.load(f) - - if entity['type'] == 'profile': - self.add_profile(entity) - elif entity['type'] == 'resource': - self.add_resource(entity) - elif entity['type'] == 'service': - self.add_service(entity)