diff --git a/doc/source/generate_inventory.rst b/doc/source/generate_inventory.rst index 643f628..eef2981 100644 --- a/doc/source/generate_inventory.rst +++ b/doc/source/generate_inventory.rst @@ -17,7 +17,7 @@ save this inventory file to the path `$ADMIN_WORKSPACE/inventory`. Below you can find a few examples on how generate Ansible inventory that can be used for deployment. -Using Fuel CCP's simple inventory generator +Using Kargo's simple inventory generator ------------------------------------------- If you run kargo_deploy.sh with a predefined list of nodes, it will generate Ansible inventory for you automatically. Below is an example: diff --git a/doc/source/inventory_repo.rst b/doc/source/inventory_repo.rst index db7cea2..4699aed 100644 --- a/doc/source/inventory_repo.rst +++ b/doc/source/inventory_repo.rst @@ -18,7 +18,7 @@ the repo root directory: * ``inventory.cfg`` - a mandatory inventory file. It must be created manually or generated based on ``$SLAVE_IPS`` provided with the - `helper script `_. + `helper script `_. * ``kargo_default_common.yaml`` - a mandatory vars file, overrides the kargo defaults in the ``$ADMIN_WORKSPACE/kargo/inventory/group_vars/all.yml``) and defaults for roles. diff --git a/test-requirements.txt b/test-requirements.txt index eca58ee..6d5294e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,5 +4,3 @@ oslosphinx>=2.5.0,!=3.4.0 # Apache-2.0 sphinx>=1.2.1,!=1.3b1,<1.3 # BSD hacking>=0.10.2 -pytest>=2.8.0 -mock>=1.3.0 diff --git a/tests/test_inventory.py b/tests/test_inventory.py deleted file mode 100644 index aba68dc..0000000 --- a/tests/test_inventory.py +++ /dev/null @@ -1,212 +0,0 @@ -# Copyright 2016 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import mock -import unittest - -from collections import OrderedDict -import sys - -path = "./utils/kargo" -if path not in sys.path: - sys.path.append(path) - -import inventory - - -class TestInventory(unittest.TestCase): - @mock.patch('inventory.sys') - def setUp(self, sys_mock): - sys_mock.exit = mock.Mock() - super(TestInventory, self).setUp() - self.data = ['10.90.3.2', '10.90.3.3', '10.90.3.4'] - self.inv = inventory.KargoInventory() - - def test_get_ip_from_opts(self): - optstring = "ansible_host=10.90.3.2 ip=10.90.3.2" - expected = "10.90.3.2" - result = self.inv.get_ip_from_opts(optstring) - self.assertEqual(expected, result) - - def test_get_ip_from_opts_invalid(self): - optstring = "notanaddr=value something random!chars:D" - self.assertRaisesRegexp(ValueError, "IP parameter not found", - self.inv.get_ip_from_opts, optstring) - - def test_ensure_required_groups(self): - groups = ['group1', 'group2'] - self.inv.ensure_required_groups(groups) - for group in groups: - self.assertTrue(group in self.inv.config.sections()) - - def test_get_host_id(self): - hostnames = ['node99', 'no99de01', '01node01', 'node1.domain', - 'node3.xyz123.aaa'] - expected = [99, 1, 1, 1, 3] - for hostname, expected in zip(hostnames, expected): - result = self.inv.get_host_id(hostname) - self.assertEqual(expected, result) - - def test_get_host_id_invalid(self): - bad_hostnames = ['node', 'no99de', '01node', 'node.111111'] - for hostname in bad_hostnames: - self.assertRaisesRegexp(ValueError, "Host name must end in an", - self.inv.get_host_id, hostname) - - def test_build_hostnames_add_one(self): - changed_hosts = ['10.90.0.2'] - expected = OrderedDict([('node1', - 'ansible_host=10.90.0.2 ip=10.90.0.2')]) - result = self.inv.build_hostnames(changed_hosts) - self.assertEqual(expected, result) - - def test_build_hostnames_add_duplicate(self): - changed_hosts = ['10.90.0.2'] - expected = OrderedDict([('node1', - 'ansible_host=10.90.0.2 ip=10.90.0.2')]) - self.inv.config['all'] = expected - result = self.inv.build_hostnames(changed_hosts) - self.assertEqual(expected, result) - - def test_build_hostnames_add_two(self): - changed_hosts = ['10.90.0.2', '10.90.0.3'] - expected = OrderedDict([ - ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), - ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) - self.inv.config['all'] = OrderedDict() - result = self.inv.build_hostnames(changed_hosts) - self.assertEqual(expected, result) - - def test_build_hostnames_delete_first(self): - changed_hosts = ['-10.90.0.2'] - existing_hosts = OrderedDict([ - ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), - ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) - self.inv.config['all'] = existing_hosts - expected = OrderedDict([ - ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) - result = self.inv.build_hostnames(changed_hosts) - self.assertEqual(expected, result) - - def test_exists_hostname_positive(self): - hostname = 'node1' - expected = True - existing_hosts = OrderedDict([ - ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), - ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) - result = self.inv.exists_hostname(existing_hosts, hostname) - self.assertEqual(expected, result) - - def test_exists_hostname_negative(self): - hostname = 'node99' - expected = False - existing_hosts = OrderedDict([ - ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), - ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) - result = self.inv.exists_hostname(existing_hosts, hostname) - self.assertEqual(expected, result) - - def test_exists_ip_positive(self): - ip = '10.90.0.2' - expected = True - existing_hosts = OrderedDict([ - ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), - ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) - result = self.inv.exists_ip(existing_hosts, ip) - self.assertEqual(expected, result) - - def test_exists_ip_negative(self): - ip = '10.90.0.200' - expected = False - existing_hosts = OrderedDict([ - ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), - ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) - result = self.inv.exists_ip(existing_hosts, ip) - self.assertEqual(expected, result) - - def test_delete_host_by_ip_positive(self): - ip = '10.90.0.2' - expected = OrderedDict([ - ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) - existing_hosts = OrderedDict([ - ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), - ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) - self.inv.delete_host_by_ip(existing_hosts, ip) - self.assertEqual(expected, existing_hosts) - - def test_delete_host_by_ip_negative(self): - ip = '10.90.0.200' - existing_hosts = OrderedDict([ - ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), - ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) - self.assertRaisesRegexp(ValueError, "Unable to find host", - self.inv.delete_host_by_ip, existing_hosts, ip) - - def test_purge_invalid_hosts(self): - proper_hostnames = ['node1', 'node2'] - bad_host = 'doesnotbelong2' - existing_hosts = OrderedDict([ - ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), - ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3'), - ('doesnotbelong2', 'whateveropts=ilike')]) - self.inv.config['all'] = existing_hosts - self.inv.purge_invalid_hosts(proper_hostnames) - self.assertTrue(bad_host not in self.inv.config['all'].keys()) - - def test_add_host_to_group(self): - group = 'etcd' - host = 'node1' - opts = 'ip=10.90.0.2' - - self.inv.add_host_to_group(group, host, opts) - self.assertEqual(self.inv.config[group].get(host), opts) - - def test_set_kube_master(self): - group = 'kube-master' - host = 'node1' - - self.inv.set_kube_master([host]) - self.assertTrue(host in self.inv.config[group]) - - def test_set_all(self): - group = 'all' - hosts = OrderedDict([ - ('node1', 'opt1'), - ('node2', 'opt2')]) - - self.inv.set_all(hosts) - for host, opt in hosts.items(): - self.assertEqual(self.inv.config[group].get(host), opt) - - def test_set_k8s_cluster(self): - group = 'k8s-cluster:children' - expected_hosts = ['kube-node', 'kube-master'] - - self.inv.set_k8s_cluster() - for host in expected_hosts: - self.assertTrue(host in self.inv.config[group]) - - def test_set_kube_node(self): - group = 'kube-node' - host = 'node1' - - self.inv.set_kube_node([host]) - self.assertTrue(host in self.inv.config[group]) - - def test_set_etcd(self): - group = 'etcd' - host = 'node1' - - self.inv.set_etcd([host]) - self.assertTrue(host in self.inv.config[group]) diff --git a/tox.ini b/tox.ini index 0718d18..1ca27bf 100644 --- a/tox.ini +++ b/tox.ini @@ -1,17 +1,12 @@ [tox] minversion = 1.6 skipsdist = True -envlist = bashate, pep8, py27 +envlist = bashate, pep8 [testenv] -whitelist_externals = py.test -usedevelop = True deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt -setenv = VIRTUAL_ENV={envdir} -passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY -commands = py.test -vv #{posargs:tests} [testenv:doc8] commands = doc8 doc diff --git a/utils/jenkins/kargo_deploy.sh b/utils/jenkins/kargo_deploy.sh index 3395c0a..8565c76 100755 --- a/utils/jenkins/kargo_deploy.sh +++ b/utils/jenkins/kargo_deploy.sh @@ -263,7 +263,7 @@ elif admin_node_command test -e $ADMIN_WORKSPACE/inventory/custom.yaml; then fi if [ "${SLAVE_IPS}" ]; then - admin_node_command CONFIG_FILE=$ADMIN_WORKSPACE/inventory/inventory.cfg python3 $ADMIN_WORKSPACE/utils/kargo/inventory.py ${SLAVE_IPS[@]} + admin_node_command CONFIG_FILE=$ADMIN_WORKSPACE/inventory/inventory.cfg python3 $ADMIN_WORKSPACE/kargo/contrib/inventory_builder/inventory.py ${SLAVE_IPS[@]} fi # Data committed to the inventory has the highest priority, then installer defaults @@ -281,7 +281,7 @@ fi # Try to get IPs from inventory first if [ -z "${SLAVE_IPS}" ]; then if admin_node_command stat $ADMIN_WORKSPACE/inventory/inventory.cfg; then - SLAVE_IPS=($(admin_node_command CONFIG_FILE=$ADMIN_WORKSPACE/inventory/inventory.cfg python3 $ADMIN_WORKSPACE/utils/kargo/inventory.py print_ips)) + SLAVE_IPS=($(admin_node_command CONFIG_FILE=$ADMIN_WORKSPACE/inventory/inventory.cfg python3 $ADMIN_WORKSPACE/kargo/contrib/inventory_builder/inventory.py print_ips)) else echo "No slave nodes available. Unable to proceed!" exit_gracefully 1 diff --git a/utils/kargo/inventory.py b/utils/kargo/inventory.py deleted file mode 100644 index 318da06..0000000 --- a/utils/kargo/inventory.py +++ /dev/null @@ -1,226 +0,0 @@ -#!/usr/bin/python3 -# Usage: kargo_inventory.py ip1 [ip2 ...] -# Examples: kargo_inventory.py 10.10.1.3 10.10.1.4 10.10.1.5 -# -# Advanced usage: -# Add another host after initial creation: kargo_inventory.py 10.10.1.5 -# Delete a host: kargo_inventory.py -10.10.1.3 -# Delete a host by id: kargo_inventory.py -node1 - -from collections import OrderedDict -try: - import configparser -except ImportError: - import ConfigParser as configparser - -import os -import re -import sys - -ROLES = ['kube-master', 'all', 'k8s-cluster:children', 'kube-node', 'etcd'] -PROTECTED_NAMES = ROLES -AVAILABLE_COMMANDS = ['help', 'print_cfg', 'print_ips'] -_boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True, - '0': False, 'no': False, 'false': False, 'off': False} - - -def get_var_as_bool(name, default): - value = os.environ.get(name, '') - return _boolean_states.get(value.lower(), default) - -CONFIG_FILE = os.environ.get("CONFIG_FILE", "./inventory.cfg") -DEBUG = get_var_as_bool("DEBUG", True) -HOST_PREFIX = os.environ.get("HOST_PREFIX", "node") - - -class KargoInventory(object): - - def __init__(self, changed_hosts=None, config_file=None): - self.config = configparser.ConfigParser(allow_no_value=True, - delimiters=('\t', ' ')) - if config_file: - self.config.read(config_file) - - if changed_hosts and changed_hosts[0] in AVAILABLE_COMMANDS: - self.parse_command(changed_hosts[0], changed_hosts[1:]) - sys.exit(0) - - self.ensure_required_groups(ROLES) - - if changed_hosts: - self.hosts = self.build_hostnames(changed_hosts) - self.purge_invalid_hosts(self.hosts.keys(), PROTECTED_NAMES) - self.set_kube_master(list(self.hosts.keys())[:2]) - self.set_all(self.hosts) - self.set_k8s_cluster() - self.set_kube_node(self.hosts.keys()) - self.set_etcd(list(self.hosts.keys())[:3]) - else: # Show help if no options - self.show_help() - sys.exit(0) - - if config_file: - with open(config_file, 'w') as f: - self.config.write(f) - - def debug(self, msg): - if DEBUG: - print("DEBUG: {0}".format(msg)) - - def get_ip_from_opts(self, optstring): - opts = optstring.split(' ') - for opt in opts: - if '=' not in opt: - continue - k, v = opt.split('=') - if k == "ip": - return v - raise ValueError("IP parameter not found in options") - - def ensure_required_groups(self, groups): - for group in groups: - try: - self.config.add_section(group) - except configparser.DuplicateSectionError: - pass - - def get_host_id(self, host): - '''Returns integer host ID (without padding) from a given hostname.''' - try: - short_hostname = host.split('.')[0] - return int(re.findall("\d+$", short_hostname)[-1]) - except IndexError: - raise ValueError("Host name must end in an integer") - - def build_hostnames(self, changed_hosts): - existing_hosts = OrderedDict() - highest_host_id = 0 - try: - for host, opts in self.config.items('all'): - existing_hosts[host] = opts - host_id = self.get_host_id(host) - if host_id > highest_host_id: - highest_host_id = host_id - except configparser.NoSectionError: - pass - - # FIXME(mattymo): Fix condition where delete then add reuses highest id - next_host_id = highest_host_id + 1 - - all_hosts = existing_hosts.copy() - for host in changed_hosts: - if host[0] == "-": - realhost = host[1:] - if self.exists_hostname(all_hosts, realhost): - self.debug("Marked {0} for deletion.".format(realhost)) - all_hosts.pop(realhost) - elif self.exists_ip(all_hosts, realhost): - self.debug("Marked {0} for deletion.".format(realhost)) - self.delete_host_by_ip(all_hosts, realhost) - elif host[0].isdigit(): - if self.exists_hostname(all_hosts, host): - self.debug("Skipping existing host {0}.".format(host)) - continue - elif self.exists_ip(all_hosts, host): - self.debug("Skipping existing host {0}.".format(host)) - continue - - next_host = "{0}{1}".format(HOST_PREFIX, next_host_id) - next_host_id += 1 - all_hosts[next_host] = "ansible_host={0} ip={1}".format( - host, host) - elif host[0].isalpha(): - raise Exception("Adding hosts by hostname is not supported.") - - return all_hosts - - def exists_hostname(self, existing_hosts, hostname): - return hostname in existing_hosts.keys() - - def exists_ip(self, existing_hosts, ip): - for host_opts in existing_hosts.values(): - if ip == self.get_ip_from_opts(host_opts): - return True - return False - - def delete_host_by_ip(self, existing_hosts, ip): - for hostname, host_opts in existing_hosts.items(): - if ip == self.get_ip_from_opts(host_opts): - del existing_hosts[hostname] - return - raise ValueError("Unable to find host by IP: {0}".format(ip)) - - def purge_invalid_hosts(self, hostnames, protected_names=[]): - for role in self.config.sections(): - for host, _ in self.config.items(role): - if host not in hostnames and host not in protected_names: - self.debug("Host {0} removed from role {1}".format(host, - role)) - self.config.remove_option(role, host) - - def add_host_to_group(self, group, host, opts=""): - self.debug("adding host {0} to group {1}".format(host, group)) - self.config.set(group, host, opts) - - def set_kube_master(self, hosts): - for host in hosts: - self.add_host_to_group('kube-master', host) - - def set_all(self, hosts): - for host, opts in hosts.items(): - self.add_host_to_group('all', host, opts) - - def set_k8s_cluster(self): - self.add_host_to_group('k8s-cluster:children', 'kube-node') - self.add_host_to_group('k8s-cluster:children', 'kube-master') - - def set_kube_node(self, hosts): - for host in hosts: - self.add_host_to_group('kube-node', host) - - def set_etcd(self, hosts): - for host in hosts: - self.add_host_to_group('etcd', host) - - def parse_command(self, command, args=None): - if command == 'help': - self.show_help() - elif command == 'print_cfg': - self.print_config() - elif command == 'print_ips': - self.print_ips() - else: - raise Exception("Invalid command specified.") - - def show_help(self): - help_text = '''Usage: inventory.py ip1 [ip2 ...] -Examples: inventory.py 10.10.1.3 10.10.1.4 10.10.1.5 - -Available commands: -help - Display this message -print_cfg - Write inventory file to stdout -print_ips - Write a space-delimited list of IPs from "all" group - -Advanced usage: -Add another host after initial creation: inventory.py 10.10.1.5 -Delete a host: inventory.py -10.10.1.3 -Delete a host by id: inventory.py -node1''' - print(help_text) - - def print_config(self): - self.config.write(sys.stdout) - - def print_ips(self): - ips = [] - for host, opts in self.config.items('all'): - ips.append(self.get_ip_from_opts(opts)) - print(' '.join(ips)) - - -def main(argv=None): - if not argv: - argv = sys.argv[1:] - KargoInventory(argv, CONFIG_FILE) - -if __name__ == "__main__": - sys.exit(main())