From 2cf49b7bc4b823f7d4b238822710efe5588f5ea3 Mon Sep 17 00:00:00 2001 From: Steve Noyes Date: Fri, 6 Nov 2015 11:07:01 -0500 Subject: [PATCH 01/25] add doc for hosts rest api --- doc/source/api/reference/hosts.rst | 307 +++++++++++++++++++++++++++++ 1 file changed, 307 insertions(+) create mode 100644 doc/source/api/reference/hosts.rst diff --git a/doc/source/api/reference/hosts.rst b/doc/source/api/reference/hosts.rst new file mode 100644 index 0000000..20591b8 --- /dev/null +++ b/doc/source/api/reference/hosts.rst @@ -0,0 +1,307 @@ +*********************** +HOSTS API - Reference +*********************** + + +.. warning:: + + This hosts documentation is work in progress and may change in near future. + + +Hosts API +=============== + +.. _get_hosts_all: + +GET /v1/hosts +########################## +Retrieve all hosts in the inventory. + + +Request/Response: +************************************ + +.. code-block:: none + + Request: + + GET /v1/hosts + Headers: + X-Auth-Token: {token_id} + + Response: + + HTTP/1.1 200 OK + { + "hosts":[ + { + "hostname":{host_name1}, + "groups":[ + {group_name1}, + {group_name2}, + ..... + ], + }, + ..... + ] + } + + +HTTP Status Codes +***************** + ++------+-----------------------------------------------------------------------------+ +| Code | Description | ++======+=============================================================================+ +| 200 | Successful request. | ++------+-----------------------------------------------------------------------------+ +| 401 | Missing or Invalid X-Auth-Token. Authentication required. | ++------+-----------------------------------------------------------------------------+ + +.. _get_host: + +GET /v1/hosts/{host_name} +########################## +Retrieve the host for the given host name. + + +Request/Response: +************************************ + +.. code-block:: none + + Request: + + GET /v1/hosts/{host_name} + Headers: + X-Auth-Token: {token_id} + + Response: + + HTTP/1.1 200 OK + { + "hostname":{host_name}, + "groups":[ + {group_name1}, + {group_name2}, + ], + ..... + } + + +HTTP Status Codes +***************** + ++------+-----------------------------------------------------------------------------+ +| Code | Description | ++======+=============================================================================+ +| 200 | Successful request. | ++------+-----------------------------------------------------------------------------+ +| 401 | Missing or Invalid X-Auth-Token. Authentication required. | ++------+-----------------------------------------------------------------------------+ +| 404 | Host does not exist in inventory | ++------+-----------------------------------------------------------------------------+ + + +.. _post_host: + +POST /v1/hosts +########################## +Create new host. + +This call is used to create a new host and add it to the inventory. + +Request/Response (create or replace host): +************************************** + +.. code-block:: none + + Request: + + POST /v1/hosts/{host_name} + Headers: + Content-Type: application/json + X-Auth-Token: {token_id} + + { + "hostname":{host_name}, + "groups": [ + {group_name1}, + {group_name2}, + ..... + ], + } + + Response: + + HTTP/1.1 200 OK + + +HTTP Status Codes +***************** + ++------+-----------------------------------------------------------------------------+ +| Code | Description | ++======+=============================================================================+ +| 200 | Successfully created host. | ++------+-----------------------------------------------------------------------------+ +| 400 | Bad Request. | ++------+-----------------------------------------------------------------------------+ +| 401 | Missing or Invalid X-Auth-Token. Authentication required. | ++------+-----------------------------------------------------------------------------+ +| 404 | Group does not exist in inventory | ++------+-----------------------------------------------------------------------------+ +| 409 | Host already exists in inventory | ++------+-----------------------------------------------------------------------------+ + + +.. _delete_host: + +DELETE /v1/hosts/{host_name} +############################## + +Delete host from the inventory. + +Request/Response: +***************** + +.. code-block:: none + + DELETE /v1/host/{host_name} + Headers: + X-Auth-Token: {token_id} + + Response: + HTTP/1.1 200 OK + + +HTTP Status Codes +***************** + ++------+-----------------------------------------------------------------------------+ +| Code | Description | ++======+=============================================================================+ +| 200 | Successfully deleted host. | ++------+-----------------------------------------------------------------------------+ +| 401 | Missing or Invalid X-Auth-Token. Authentication required. | ++------+-----------------------------------------------------------------------------+ + +.. _check_host: + +POST /v1/hosts/{host_name}/actions +############################## + +Check verifies that the host has its ssh keys set up correctly (can be accessed without a +password from the deploy host). If the host check failed, the reason will be provided in +the response message. + +Request/Response: +***************** + +.. code-block:: none + + POST /v1/hosts/{host_name}/actions + Headers: + Content-Type: application/json + X-Auth-Token: {token_id} + { + "check": { + "host-name": {host_name}, + } + } + +Response: +********* + +.. code-block:: none + + 200 OK + + { + "message":{message_string} + } + + +HTTP Status Codes +***************** + ++------+-----------------------------------------------------------------------------+ +| Code | Description | ++======+=============================================================================+ +| 200 | Host check was successful | ++------+-----------------------------------------------------------------------------+ +| 400 | Bad Request | ++------+-----------------------------------------------------------------------------+ +| 401 | Invalid X-Auth-Token or the token doesn't have permissions to this resource | ++------+-----------------------------------------------------------------------------+ +| 404 | Host does not exist in inventory | ++------+-----------------------------------------------------------------------------+ +| 405 | Host check failed | ++------+-----------------------------------------------------------------------------+ + +.. _setup_host: + +POST /v1/hosts/actions +############################## + +Host setup distributes the ssh keys into the appropriate directory/file on the host. +This assumes docker has been installed and is running on the host. Setup can be done +for a single host or multiple hosts. + +If a single host is to be setup, the host-name and host-password attributes must be +supplied. If multiple hosts are to be setup, the hosts-file-path must be +supplied. + +Either the host-name/password or hosts-file-path must be supplied. If both are supplied, +then all the hosts specified will be setup. + +If the host setup failed, the reason will be provided in +the response message. + +Request/Response: +***************** + +.. code-block:: none + + POST /v1/hosts/actions + Headers: + Content-Type: application/json + X-Auth-Token: {token_id} + + { + "setup": { + "host-name": {host_name}, + "host-password": {password}, + "hosts-file-path": {hosts_file_path} + } + } + +Response: +********* + +.. code-block:: none + + 200 OK + + { + "message":{message_string} + } + + +HTTP Status Codes +***************** + ++------+-----------------------------------------------------------------------------+ +| Code | Description | ++======+=============================================================================+ +| 200 | Host setup was successful | ++------+-----------------------------------------------------------------------------+ +| 400 | Bad Request | ++------+-----------------------------------------------------------------------------+ +| 401 | Invalid X-Auth-Token or the token doesn't have permissions to this resource | ++------+-----------------------------------------------------------------------------+ +| 404 | Host does not exist in inventory | ++------+-----------------------------------------------------------------------------+ +| 405 | Host setup failed | ++------+-----------------------------------------------------------------------------+ + From 538cfddf602b64321d8066ab89ed650c7239f264 Mon Sep 17 00:00:00 2001 From: Steve Noyes Date: Fri, 6 Nov 2015 16:18:45 -0500 Subject: [PATCH 02/25] py3 (1) - update utils to py3 compatibilty - use osli.utils for string decoding - add get_ansible_command function so ansible can run on py3 default system - use six to replace unicode function --- buildrpm/openstack-kollacli.spec | 4 +++ kollacli/utils.py | 51 +++++++++++++++++++++++++++----- requirements.txt | 1 + tox.ini | 2 +- 4 files changed, 50 insertions(+), 8 deletions(-) diff --git a/buildrpm/openstack-kollacli.spec b/buildrpm/openstack-kollacli.spec index 1b87034..2734c8a 100644 --- a/buildrpm/openstack-kollacli.spec +++ b/buildrpm/openstack-kollacli.spec @@ -43,6 +43,7 @@ Requires: python-cliff >= 1.13.0 Requires: python-cliff-tablib >= 1.1 Requires: python-jsonpickle >= 0.9.2 Requires: python-oslo-i18n >= 2.5.0 +Requires: python-oslo-utils >= 2.7.0 Requires: python-paramiko >= 1.15.1 Requires: python-pbr >= 1.6.0 Requires: python-six >= 1.9.0 @@ -154,6 +155,9 @@ esac %changelog +* Fri Nov 06 2015 - Steve Noyes +- add python-oslo-utils requirement + * Mon Oct 26 2015 - Steve Noyes - Remove obsolete json_generator diff --git a/kollacli/utils.py b/kollacli/utils.py index db33715..0754070 100644 --- a/kollacli/utils.py +++ b/kollacli/utils.py @@ -16,8 +16,13 @@ import logging import os import pexpect import pwd +import six +import sys import yaml +from kollacli.exceptions import CommandError +from oslo_utils.encodeutils import safe_decode + def get_kolla_home(): return os.environ.get("KOLLA_HOME", "/usr/share/kolla/") @@ -48,7 +53,14 @@ def get_admin_uids(): def get_kolla_log_file_size(): - return os.environ.get('KOLLA_LOG_FILE_SIZE', 500000) + envvar = 'KOLLA_LOG_FILE_SIZE' + size_str = os.environ.get(envvar, '500000') + try: + size = int(size_str) + except Exception: + raise CommandError('Environmental variable ' + + '(%s) is not an integer' % (envvar, size_str)) + return size def get_admin_user(): @@ -85,18 +97,41 @@ def save_etc_yaml(fileName, contents): f.write(yaml.dump(contents)) +def get_ansible_command(playbook=False): + """get a python2 ansible command + + Ansible cannot run yet with python3. If the current default + python is py3, prefix the ansible command with a py2 + interpreter. + """ + cmd = 'ansible' + if playbook: + cmd = 'ansible-playbook' + if sys.version_info[0] >= 3: + # running with py3, find a py2 interpreter for ansible + py2_path = None + usr_bin = os.path.join('/', 'usr', 'bin') + for file in os.listdir(usr_bin): + if (file.startswith('python2.') and + os.path.isfile(os.path.join(usr_bin, file))): + suffix = file.split('.')[1] + if suffix.isdigit(): + py2_path = os.path.join(usr_bin, file) + break + if py2_path is None: + raise Exception('ansible-playbook requires python2 and no ' + 'python2 interpreter found in %s' % usr_bin) + cmd = '%s %s' % (py2_path, os.path.join(usr_bin, cmd)) + return cmd + + def convert_to_unicode(the_string): """convert string to unicode. This is used to fixup extended ascii chars in strings. these chars cause errors in json pickle/unpickle. """ - uni_string = '' - try: - uni_string = unicode(the_string) - except UnicodeDecodeError: - uni_string = the_string.decode('utf-8') - return uni_string + return six.u(the_string) def run_cmd(cmd, print_output=True): @@ -118,6 +153,7 @@ def run_cmd(cmd, print_output=True): try: child = pexpect.spawn(cmd) sniff = child.read(len(pwd_prompt)) + sniff = safe_decode(sniff) if sniff == pwd_prompt: output = sniff + '\n' raise Exception( @@ -125,6 +161,7 @@ def run_cmd(cmd, print_output=True): child.maxsize = 1 child.timeout = 86400 for line in child: + line = safe_decode(line) outline = sniff + line.rstrip() sniff = '' output = ''.join([output, outline, '\n']) diff --git a/requirements.txt b/requirements.txt index d63b26c..1fa2d93 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ cliff-tablib>=1.1 docker-py>=1.3.1 jsonpickle>=0.9 oslo.i18n>=1.3.0 # Apache-2.0 +oslo.utils>-2.7.0 paramiko>=1.15 pbr>=0.10 pexpect>=2.3,<3.3 diff --git a/tox.ini b/tox.ini index 8002794..f9b90b7 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,7 @@ deps = -r{toxinidir}/requirements.txt [testenv:py27] commands = /usr/bin/find . -type f -name "*.pyc" -delete - {envpython} setup.py test -s tests + {envpython} -m unittest discover -s tests -p "*.*" [testenv:pep8] commands = flake8 From d1e67d88ce56027d641feb5ff5b2f49018f3f41f Mon Sep 17 00:00:00 2001 From: Steve Noyes Date: Fri, 6 Nov 2015 18:07:09 -0500 Subject: [PATCH 03/25] py3 (2) - update host.py to py3 compatibility - py3 is more particular about external package imports; specify kollacli utils explicitly - fix typo on oslo.utils in requirements.txt - use standard spec for oslo utils - !=2.6.0,>=2.4.0 --- buildrpm/openstack-kollacli.spec | 2 +- kollacli/host.py | 13 ++++++------- requirements.txt | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/buildrpm/openstack-kollacli.spec b/buildrpm/openstack-kollacli.spec index 2734c8a..b07fbab 100644 --- a/buildrpm/openstack-kollacli.spec +++ b/buildrpm/openstack-kollacli.spec @@ -43,7 +43,7 @@ Requires: python-cliff >= 1.13.0 Requires: python-cliff-tablib >= 1.1 Requires: python-jsonpickle >= 0.9.2 Requires: python-oslo-i18n >= 2.5.0 -Requires: python-oslo-utils >= 2.7.0 +Requires: python-oslo-utils !=2.6.0, >=2.4.0 Requires: python-paramiko >= 1.15.1 Requires: python-pbr >= 1.6.0 Requires: python-six >= 1.9.0 diff --git a/kollacli/host.py b/kollacli/host.py index 1abcb05..813091e 100644 --- a/kollacli/host.py +++ b/kollacli/host.py @@ -16,7 +16,6 @@ import getpass import logging import os import traceback -import utils import yaml from kollacli.ansible.inventory import Inventory @@ -50,7 +49,7 @@ class HostAdd(Command): def take_action(self, parsed_args): try: hostname = parsed_args.hostname.strip() - hostname = utils.convert_to_unicode(hostname) + hostname = convert_to_unicode(hostname) inventory = Inventory.load() inventory.add_host(hostname) @@ -129,7 +128,7 @@ class HostRemove(Command): def take_action(self, parsed_args): try: hostname = parsed_args.hostname.strip() - hostname = utils.convert_to_unicode(hostname) + hostname = convert_to_unicode(hostname) inventory = Inventory.load() inventory.remove_host(hostname) Inventory.save(inventory) @@ -158,7 +157,7 @@ class HostList(Lister): hostname = None if parsed_args.hostname: hostname = parsed_args.hostname.strip() - hostname = utils.convert_to_unicode(hostname) + hostname = convert_to_unicode(hostname) inventory = Inventory.load() @@ -197,7 +196,7 @@ class HostCheck(Command): def take_action(self, parsed_args): try: hostname = parsed_args.hostname.strip() - hostname = utils.convert_to_unicode(hostname) + hostname = convert_to_unicode(hostname) inventory = Inventory.load() if not inventory.get_host(hostname): _host_not_found(self.log, hostname) @@ -241,7 +240,7 @@ class HostSetup(Command): else: # single host setup hostname = parsed_args.hostname.strip() - hostname = utils.convert_to_unicode(hostname) + hostname = convert_to_unicode(hostname) if not inventory.get_host(hostname): _host_not_found(self.log, hostname) @@ -257,7 +256,7 @@ class HostSetup(Command): setup_user = get_setup_user() password = getpass.getpass('%s password for %s: ' % (setup_user, hostname)) - password = utils.convert_to_unicode(password) + password = convert_to_unicode(password) inventory.setup_host(hostname, password) except CommandError as e: diff --git a/requirements.txt b/requirements.txt index 1fa2d93..2132591 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ cliff-tablib>=1.1 docker-py>=1.3.1 jsonpickle>=0.9 oslo.i18n>=1.3.0 # Apache-2.0 -oslo.utils>-2.7.0 +oslo.utils!=2.6.0,>=2.4.0 # Apache-2.0 paramiko>=1.15 pbr>=0.10 pexpect>=2.3,<3.3 From 8c012186fe15c995915980b170dbbd4ddb0787f3 Mon Sep 17 00:00:00 2001 From: Steve Noyes Date: Mon, 9 Nov 2015 11:27:58 -0500 Subject: [PATCH 04/25] py3 (3) - update inventory.py to py3 compatibility - use list(dict.keys) - use new ansible command - fix use of reserved name (file) in utils --- kollacli/ansible/inventory.py | 8 +++++--- kollacli/utils.py | 8 ++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/kollacli/ansible/inventory.py b/kollacli/ansible/inventory.py index 5d9b895..c98330f 100644 --- a/kollacli/ansible/inventory.py +++ b/kollacli/ansible/inventory.py @@ -24,6 +24,7 @@ from kollacli import utils from kollacli.exceptions import CommandError from kollacli.sshutils import ssh_setup_host from kollacli.utils import get_admin_user +from kollacli.utils import get_ansible_command ANSIBLE_SSH_USER = 'ansible_ssh_user' ANSIBLE_CONNECTION = 'ansible_connection' @@ -334,7 +335,7 @@ class Inventory(object): return self._hosts.values() def get_hostnames(self): - return self._hosts.keys() + return list(self._hosts.keys()) def get_host(self, hostname): host = None @@ -446,7 +447,8 @@ class Inventory(object): return True def check_host(self, hostname, result_only=False): - command_string = '/usr/bin/sudo -u %s ansible ' % get_admin_user() + command_string = '/usr/bin/sudo -u %s %s ' % \ + (get_admin_user(), get_ansible_command()) gen_file_path = self.create_json_gen_file() err_msg = None output = None @@ -510,7 +512,7 @@ class Inventory(object): return group def get_groupnames(self): - return self._groups.keys() + return list(self._groups.keys()) def get_groups(self, host=None): """return all groups containing host diff --git a/kollacli/utils.py b/kollacli/utils.py index 0754070..6e104dd 100644 --- a/kollacli/utils.py +++ b/kollacli/utils.py @@ -111,12 +111,12 @@ def get_ansible_command(playbook=False): # running with py3, find a py2 interpreter for ansible py2_path = None usr_bin = os.path.join('/', 'usr', 'bin') - for file in os.listdir(usr_bin): - if (file.startswith('python2.') and - os.path.isfile(os.path.join(usr_bin, file))): + for fname in os.listdir(usr_bin): + if (fname.startswith('python2.') and + os.path.isfile(os.path.join(usr_bin, fname))): suffix = file.split('.')[1] if suffix.isdigit(): - py2_path = os.path.join(usr_bin, file) + py2_path = os.path.join(usr_bin, fname) break if py2_path is None: raise Exception('ansible-playbook requires python2 and no ' From 9bfeb311c39af29e628b2b3f2838e4467bde049d Mon Sep 17 00:00:00 2001 From: Steve Noyes Date: Mon, 9 Nov 2015 13:27:19 -0500 Subject: [PATCH 05/25] fix build spec for oslo_utils --- buildrpm/openstack-kollacli.spec | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildrpm/openstack-kollacli.spec b/buildrpm/openstack-kollacli.spec index b07fbab..51714e4 100644 --- a/buildrpm/openstack-kollacli.spec +++ b/buildrpm/openstack-kollacli.spec @@ -43,7 +43,7 @@ Requires: python-cliff >= 1.13.0 Requires: python-cliff-tablib >= 1.1 Requires: python-jsonpickle >= 0.9.2 Requires: python-oslo-i18n >= 2.5.0 -Requires: python-oslo-utils !=2.6.0, >=2.4.0 +Requires: python-oslo-utils >= 2.4.0 Requires: python-paramiko >= 1.15.1 Requires: python-pbr >= 1.6.0 Requires: python-six >= 1.9.0 @@ -51,6 +51,7 @@ Requires: PyYAML >= 3.10 Requires: /usr/bin/ssh-keygen +Conflicts: python-oslo-utils = 2.6.0 %description The KollaCLI simplifies OpenStack Kolla deployments. From 891e3a136555d00abab1ebd71a4e0b9fa09afa36 Mon Sep 17 00:00:00 2001 From: Steve Noyes Date: Mon, 9 Nov 2015 13:36:17 -0500 Subject: [PATCH 06/25] py3 (4) - update playbook.py to py3 compatibility --- kollacli/ansible/playbook.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kollacli/ansible/playbook.py b/kollacli/ansible/playbook.py index 3c60687..c4466ab 100644 --- a/kollacli/ansible/playbook.py +++ b/kollacli/ansible/playbook.py @@ -19,6 +19,7 @@ import traceback from kollacli.ansible.inventory import Inventory from kollacli.exceptions import CommandError from kollacli.utils import get_admin_user +from kollacli.utils import get_ansible_command from kollacli.utils import get_kolla_etc from kollacli.utils import run_cmd @@ -49,9 +50,10 @@ class AnsiblePlaybook(object): if self.verbose_level > 1: flag = '-vvv' + ansible_cmd = get_ansible_command(playbook=True) admin_user = get_admin_user() - command_string = ('/usr/bin/sudo -u %s ansible-playbook %s' - % (admin_user, flag)) + command_string = ('/usr/bin/sudo -u %s %s %s' + % (admin_user, ansible_cmd, flag)) inventory = Inventory.load() inventory_filter = {} if self.hosts: From 51622fcd0d60fab7d148fd33ecda22d7b0ed5603 Mon Sep 17 00:00:00 2001 From: Steve Noyes Date: Mon, 9 Nov 2015 13:58:11 -0500 Subject: [PATCH 07/25] py3 (5) - update properties.py to py3 compatibility --- kollacli/ansible/properties.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/kollacli/ansible/properties.py b/kollacli/ansible/properties.py index 40c871d..cd1c505 100644 --- a/kollacli/ansible/properties.py +++ b/kollacli/ansible/properties.py @@ -13,6 +13,7 @@ # under the License. import logging import os +import six import yaml from kollacli.utils import change_property @@ -115,15 +116,16 @@ class AnsibleProperties(object): return sorted(unique_list, key=lambda x: x.name) def filter_jinja2(self, contents): + new_contents = {} for key, value in contents.items(): - if isinstance(value, basestring) is False: - self.log.debug('removing non-string: %s' % str(value)) - del contents[key] + if not isinstance(value, six.string_types): + self.log.debug('removing non-string: %s' % value) continue - if '{{' in value and '}}' in value: + if value and '{{' in value and '}}' in value: self.log.debug('removing jinja2 value: %s' % value) - del contents[key] - return contents + continue + new_contents[key] = value + return new_contents def set_property(self, property_key, property_value): # We only manipulate values in the globals.yml file so look up the key From ee57dcae8a47831ad4e7311ed92caa62327a70af Mon Sep 17 00:00:00 2001 From: Borne Mace Date: Mon, 9 Nov 2015 11:56:53 -0800 Subject: [PATCH 08/25] added support for doing bandit checks. --- kollacli/ansible/inventory.py | 2 +- kollacli/ansible/playbook.py | 4 ++-- kollacli/ansible/properties.py | 6 +++--- kollacli/host.py | 2 +- kollacli/sshutils.py | 6 +++--- kollacli/utils.py | 18 ------------------ test-requirements.txt | 1 + tests/common.py | 2 +- tox.ini | 6 +++++- 9 files changed, 17 insertions(+), 30 deletions(-) diff --git a/kollacli/ansible/inventory.py b/kollacli/ansible/inventory.py index c98330f..ee0128f 100644 --- a/kollacli/ansible/inventory.py +++ b/kollacli/ansible/inventory.py @@ -782,5 +782,5 @@ class Inventory(object): json_gen_file.write("print('%s')" % json_out) # set executable by group - os.chmod(json_gen_path, 0o555) + os.chmod(json_gen_path, 0o555) # nosec return json_gen_path diff --git a/kollacli/ansible/playbook.py b/kollacli/ansible/playbook.py index c4466ab..a449d9e 100644 --- a/kollacli/ansible/playbook.py +++ b/kollacli/ansible/playbook.py @@ -13,7 +13,7 @@ # under the License. import logging import os -import subprocess +import subprocess # nosec import traceback from kollacli.ansible.inventory import Inventory @@ -123,7 +123,7 @@ class AnsiblePlaybook(object): # log the inventory dbg_gen = inventory_path (inv, _) = \ - subprocess.Popen(dbg_gen.split(' '), + subprocess.Popen(dbg_gen.split(' '), # nosec stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() self.log.debug(inv) diff --git a/kollacli/ansible/properties.py b/kollacli/ansible/properties.py index 40c871d..53cf504 100644 --- a/kollacli/ansible/properties.py +++ b/kollacli/ansible/properties.py @@ -58,7 +58,7 @@ class AnsibleProperties(object): ANSIBLE_DEFAULTS_PATH) if os.path.isfile(file_name): with open(file_name) as service_file: - service_contents = yaml.load(service_file) + service_contents = yaml.safe_load(service_file) self.file_contents[file_name] = service_contents service_contents = self.filter_jinja2(service_contents) prop_file_name = service_name + ':main.yml' @@ -73,7 +73,7 @@ class AnsibleProperties(object): try: self.allvars_path = os.path.join(kolla_home, ALLVARS_PATH) with open(self.allvars_path) as allvars_file: - allvars_contents = yaml.load(allvars_file) + allvars_contents = yaml.safe_load(allvars_file) self.file_contents[self.allvars_path] = allvars_contents allvars_contents = self.filter_jinja2(allvars_contents) for key, value in allvars_contents.items(): @@ -87,7 +87,7 @@ class AnsibleProperties(object): try: self.globals_path = os.path.join(kolla_etc, GLOBALS_FILENAME) globals_data = sync_read_file(self.globals_path) - globals_contents = yaml.load(globals_data) + globals_contents = yaml.safe_load(globals_data) self.file_contents[self.globals_path] = globals_contents globals_contents = self.filter_jinja2(globals_contents) for key, value in globals_contents.items(): diff --git a/kollacli/host.py b/kollacli/host.py index 813091e..3c19d38 100644 --- a/kollacli/host.py +++ b/kollacli/host.py @@ -272,7 +272,7 @@ class HostSetup(Command): with open(yml_path, 'r') as hosts_file: file_data = hosts_file.read() - hosts_info = yaml.load(file_data) + hosts_info = yaml.safe_load(file_data) if not hosts_info: raise CommandError('%s is empty' % yml_path) return hosts_info diff --git a/kollacli/sshutils.py b/kollacli/sshutils.py index 15a7786..5b007d4 100644 --- a/kollacli/sshutils.py +++ b/kollacli/sshutils.py @@ -107,7 +107,7 @@ def _post_setup_checks(net_addr, log): try: # a basic test - ssh_client.exec_command('ls') + ssh_client.exec_command('ls') # nosec except Exception as e: raise CommandError("remote command 'ls' failed : %s" % e) @@ -120,13 +120,13 @@ def _close_ssh_client(ssh_client): if ssh_client: try: ssh_client.close() - except Exception: + except Exception: # nosec pass def _exec_ssh_cmd(cmd, ssh_client, log): log.debug(cmd) - _, stdout, stderr = ssh_client.exec_command(cmd, get_pty=True) + _, stdout, stderr = ssh_client.exec_command(cmd, get_pty=True) # nosec msg = stdout.read() errmsg = stderr.read() log.debug('%s : %s' % (msg, errmsg)) diff --git a/kollacli/utils.py b/kollacli/utils.py index 6e104dd..06fd319 100644 --- a/kollacli/utils.py +++ b/kollacli/utils.py @@ -18,7 +18,6 @@ import pexpect import pwd import six import sys -import yaml from kollacli.exceptions import CommandError from oslo_utils.encodeutils import safe_decode @@ -80,23 +79,6 @@ def get_pk_bits(): return 1024 -def load_etc_yaml(fileName): - contents = {} - try: - with open(get_kollacli_etc() + fileName, 'r') as f: - contents = yaml.load(f) - except Exception: - # TODO(bmace) if file doesn't exist on a load we don't - # want to blow up, some better behavior here? - pass - return contents or {} - - -def save_etc_yaml(fileName, contents): - with open(get_kollacli_etc() + fileName, 'w') as f: - f.write(yaml.dump(contents)) - - def get_ansible_command(playbook=False): """get a python2 ansible command diff --git a/test-requirements.txt b/test-requirements.txt index ec1f529..8eb1e68 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,6 +4,7 @@ # Hacking already pins down pep8, pyflakes and flake8 hacking>=0.10.2,<0.11 +bandit>=0.13.2 coverage>=3.6 discover fixtures>=0.3.14 diff --git a/tests/common.py b/tests/common.py index bf8255a..e35a1f3 100644 --- a/tests/common.py +++ b/tests/common.py @@ -258,7 +258,7 @@ class TestConfig(object): with open(path, 'r+') as cfg_file: yml_data = cfg_file.read() - test_cfg = yaml.load(yml_data) + test_cfg = yaml.safe_load(yml_data) hosts_info = test_cfg['hosts'] if hosts_info: diff --git a/tox.ini b/tox.ini index f9b90b7..a8d9bb8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] minversion = 1.6 skipsdist = True -envlist = py27,pep8 +envlist = py27,pep8,bandit [testenv] usedevelop = True @@ -25,3 +25,7 @@ commands = {posargs} [flake8] show-source = True exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build + +[testenv:bandit] +deps = -r{toxinidir}/test-requirements.txt +commands = bandit -r kollacli From 80ff79a882bfc49251caa02e0ed99fdfe1c88934 Mon Sep 17 00:00:00 2001 From: Steve Noyes Date: Mon, 9 Nov 2015 16:56:46 -0500 Subject: [PATCH 09/25] py3 (6) - update unit tests to py3 compatibility --- tests/common.py | 10 +++++++--- tests/host.py | 2 +- tox.ini | 7 ++++++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/common.py b/tests/common.py index e35a1f3..0340a03 100644 --- a/tests/common.py +++ b/tests/common.py @@ -17,10 +17,11 @@ import os import pxssh import subprocess import sys +import testtools import traceback import yaml -import testtools +from oslo_utils.encodeutils import safe_decode import kollacli.utils as utils @@ -47,6 +48,7 @@ class KollaCliTest(testtools.TestCase): % self._testMethodName) # switch to test path + self.log.info('running python: %s/%s' % (sys.executable, sys.version)) etc_path = utils.get_kollacli_etc() self.log.debug('etc for tests: %s' % etc_path) @@ -86,11 +88,12 @@ class KollaCliTest(testtools.TestCase): msg = e.output except Exception as e: - retval = e.errno + retval = -1 msg = ('Unexpected exception: %s, cmd: %s' % (traceback.format_exc(), cmd)) # the py dev debugger adds a string at the line start, remove it + msg = safe_decode(msg) if msg.startswith('pydev debugger'): msg = msg.split('\n', 1)[1] return (retval, msg) @@ -233,7 +236,7 @@ class TestConfig(object): self.hosts_info[name]['groups'].remove(group) def get_hostnames(self): - return self.hosts_info.keys() + return list(self.hosts_info.keys()) def set_username(self, name, username): self.hosts_info[name]['username'] = username @@ -298,6 +301,7 @@ class TestConfig(object): session.sendline(cmd) session.prompt() out = session.before + out = safe_decode(out) self.log.info(out) session.logout() return out diff --git a/tests/host.py b/tests/host.py index daf07da..841ae98 100644 --- a/tests/host.py +++ b/tests/host.py @@ -91,7 +91,7 @@ class TestFunctional(KollaCliTest): # check if host is not set-up timeout = time.time() + 75 - while time.time <= timeout: + while time.time() <= timeout: msg = self.run_cli_cmd('host check %s' % hostname, True) if 'ERROR:' not in msg: self.log.info('waiting for ansible ssh session to timeout') diff --git a/tox.ini b/tox.ini index a8d9bb8..6974753 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] minversion = 1.6 skipsdist = True -envlist = py27,pep8,bandit +envlist = py27,py34,pep8,bandit [testenv] usedevelop = True @@ -16,6 +16,11 @@ commands = /usr/bin/find . -type f -name "*.pyc" -delete {envpython} -m unittest discover -s tests -p "*.*" +[testenv:py34] +commands = + /usr/bin/find . -type f -name "*.pyc" -delete + {envpython} -m unittest discover -s tests -p "*.*" + [testenv:pep8] commands = flake8 From 4691834f7d0a277e4bf0de0a23f7ca8fb6b3ff0e Mon Sep 17 00:00:00 2001 From: Steve Noyes Date: Mon, 9 Nov 2015 17:48:05 -0500 Subject: [PATCH 10/25] py3 (6) - update log_collector.py to py3 compatibility --- tools/log_collector.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/tools/log_collector.py b/tools/log_collector.py index 197e9bd..9cf8f72 100755 --- a/tools/log_collector.py +++ b/tools/log_collector.py @@ -22,36 +22,42 @@ import tempfile from kollacli.ansible.inventory import Inventory from kollacli.ansible import properties from kollacli.utils import get_admin_user -from kollacli.utils import get_kollacli_home +from kollacli.utils import get_ansible_command + +from oslo_utils.encodeutils import safe_decode tar_file_descr = None def run_ansible_cmd(cmd, host): - # sudo -u kolla ansible ol7-c4 -i - # /usr/share/kolla/kollacli/tools/json_generator.py -a "cmd" + # sudo -u kolla ansible ol7-c4 -i inv_path -a "cmd" out = None user = get_admin_user() - kollacli_home = get_kollacli_home() - inv_path = os.path.join(kollacli_home, 'tools', 'json_generator.py') + inv = Inventory.load() + inv_path = inv.create_json_gen_file() - acmd = ('/usr/bin/sudo -u %s ansible %s -i %s -a "%s"' - % (user, host, inv_path, cmd)) + ansible_verb = get_ansible_command() + ansible_cmd = ('/usr/bin/sudo -u %s %s %s -i %s -a "%s"' + % (user, ansible_verb, host, inv_path, cmd)) try: - (out, _) = subprocess.Popen(acmd, shell=True, + (out, _) = subprocess.Popen(ansible_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() except Exception as e: print('%s\nCannot communicate with host: %s, skipping' % (e, host)) + finally: + os.remove(inv_path) if not out: print('Host %s is not accessible, skipping' % host) - elif '>>' not in out: - print('Ansible command: %s' % acmd) - print('Host: %s. \nInvalid ansible return data: [%s]. skipping' - % (host, out)) - out = None + else: + out = safe_decode(out) + if '>>' not in out: + print('Ansible command: %s' % ansible_cmd) + print('Host: %s. \nInvalid ansible return data: [%s]. skipping' + % (host, out)) + out = None return out @@ -165,6 +171,7 @@ def main(): (_, err) = subprocess.Popen('kollacli dump'.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() + err = safe_decode(err) if '/' in err: dump_path = '/' + err.strip().split('/', 1)[1] if os.path.isfile(dump_path): From 0136c2741079640d733adc7fd69c25dde22f8e66 Mon Sep 17 00:00:00 2001 From: Steve Noyes Date: Tue, 10 Nov 2015 15:00:31 -0500 Subject: [PATCH 11/25] dump bug fixes - dump was still using the obsolete json_generator. replace that with the dynamically generated json inventory file - a few other fixes in dump - file checking added to dump unittest - capture error output in log collector when no output is received --- kollacli/common.py | 55 ++++++++++++++++++++++-------------------- tests/deploy.py | 39 ++++++++++++++++++++++++------ tools/log_collector.py | 10 ++++---- 3 files changed, 65 insertions(+), 39 deletions(-) diff --git a/kollacli/common.py b/kollacli/common.py index ad56884..aebc67c 100644 --- a/kollacli/common.py +++ b/kollacli/common.py @@ -26,7 +26,6 @@ from kollacli.utils import get_kolla_etc from kollacli.utils import get_kolla_home from kollacli.utils import get_kolla_log_dir from kollacli.utils import get_kollacli_etc -from kollacli.utils import get_kollacli_home from kollacli.utils import run_cmd from cliff.command import Command @@ -122,11 +121,10 @@ class Dump(Command): kolla_logs = get_kolla_log_dir() kolla_ansible = os.path.join(kolla_home, 'ansible') kolla_docs = os.path.join(kolla_home, 'docs') - kolla_templates = os.path.join(kolla_home, 'templates') kolla_etc = get_kolla_etc() kolla_config = os.path.join(kolla_etc, 'config') kolla_globals = os.path.join(kolla_etc, 'globals.yml') - kollacli_etc = get_kollacli_etc() + kollacli_etc = get_kollacli_etc().rstrip('/') ketc = 'kolla/etc/' kshare = 'kolla/share/' fd, dump_path = tempfile.mkstemp(prefix='kollacli_dump_', @@ -136,21 +134,18 @@ class Dump(Command): # Can't blanket add kolla_home because the .ssh dir is # accessible by the kolla user only (not kolla group) tar.add(kolla_ansible, - arcname=ketc + os.path.basename(kolla_ansible)) + arcname=kshare + os.path.basename(kolla_ansible)) tar.add(kolla_docs, - arcname=ketc + os.path.basename(kolla_docs)) - if os.path.isdir(kolla_templates): - tar.add(kolla_templates, - arcname=ketc + os.path.basename(kolla_templates)) + arcname=kshare + os.path.basename(kolla_docs)) # Can't blanket add kolla_etc because the passwords.yml # file is accessible by the kolla user only (not kolla group) tar.add(kolla_config, - arcname=kshare + os.path.basename(kolla_config)) + arcname=ketc + os.path.basename(kolla_config)) tar.add(kolla_globals, - arcname=kshare + os.path.basename(kolla_globals)) + arcname=ketc + os.path.basename(kolla_globals)) tar.add(kollacli_etc, - arcname=kshare + os.path.basename(kollacli_etc)) + arcname=ketc + os.path.basename(kollacli_etc)) # add kolla log files if os.path.isdir(kolla_logs): @@ -174,22 +169,30 @@ class Dump(Command): 'kollacli password list'] # collect the json inventory output - json_cmd = os.path.join(get_kollacli_home(), 'tools', - 'json_generator.py') - cmds.append(json_cmd) - fd, path = tempfile.mkstemp(suffix='.tmp') - os.close(fd) - with open(path, 'w') as tmp_file: - for cmd in cmds: - err_msg, output = run_cmd(cmd, False) - tmp_file.write('\n\n$ %s\n' % cmd) - if err_msg: - tmp_file.write('Error message: %s\n' % err_msg) - for line in output: - tmp_file.write(line + '\n') + inventory = Inventory.load() + inv_path = inventory.create_json_gen_file() + cmds.append(inv_path) - tar.add(path, arcname=os.path.join('kolla', 'cmds_output')) - os.remove(path) + try: + fd, path = tempfile.mkstemp(suffix='.tmp') + os.close(fd) + with open(path, 'w') as tmp_file: + for cmd in cmds: + err_msg, output = run_cmd(cmd, False) + tmp_file.write('\n\n$ %s\n' % cmd) + if err_msg: + tmp_file.write('Error message: %s\n' % err_msg) + for line in output: + tmp_file.write(line + '\n') + + tar.add(path, arcname=os.path.join('kolla', 'cmds_output')) + except Exception as e: + raise e + finally: + if path: + os.remove(path) + if inv_path: + os.remove(inv_path) return diff --git a/tests/deploy.py b/tests/deploy.py index 1b9caa0..1846233 100644 --- a/tests/deploy.py +++ b/tests/deploy.py @@ -19,6 +19,7 @@ from kollacli.ansible.inventory import SERVICES import json import os +import tarfile import unittest @@ -136,16 +137,38 @@ class TestFunctional(KollaCliTest): self.run_cli_cmd('deploy --groups=control') def test_dump(self): - # quick check of kollacli dump + check_files = [ + 'var/log/kolla/kolla.log', + 'kolla/etc/globals.yml', + 'kolla/etc/config/nova/nova-api.conf', + 'kolla/etc/kollacli/ansible/inventory.json', + 'kolla/share/ansible/site.yml', + 'kolla/share/docs/ansible-deployment.rst', + ] + # dump success output is: # dump successful to /tmp/kollacli_dump_Umxu6d.tgz - msg = self.run_cli_cmd('dump') - self.assertIn('/', msg, 'path not found in dump output: %s' % msg) + dump_path = None + try: + msg = self.run_cli_cmd('dump') + self.assertIn('/', msg, 'path not found in dump output: %s' % msg) - dump_path = '/' + msg.strip().split('/', 1)[1] - is_file = os.path.isfile(dump_path) - self.assertTrue(is_file, - 'dump file not found at %s' % dump_path) - os.remove(dump_path) + dump_path = '/' + msg.strip().split('/', 1)[1] + is_file = os.path.isfile(dump_path) + self.assertTrue(is_file, + 'dump file not found at %s' % dump_path) + file_paths = [] + with tarfile.open(dump_path, 'r') as tar: + file_paths = tar.getnames() + + for check_file in check_files: + self.assertIn(check_file, file_paths, + 'dump: check file: %s, not in files:\n%s' + % (check_file, file_paths)) + except Exception as e: + raise e + finally: + if dump_path and os.path.exists(dump_path): + os.remove(dump_path) def check_json(self, msg, groups, hosts, included_groups, included_hosts): err_msg = ('included groups: %s\n' % included_groups + diff --git a/tools/log_collector.py b/tools/log_collector.py index 9cf8f72..a04e6d4 100755 --- a/tools/log_collector.py +++ b/tools/log_collector.py @@ -38,19 +38,19 @@ def run_ansible_cmd(cmd, host): ansible_verb = get_ansible_command() ansible_cmd = ('/usr/bin/sudo -u %s %s %s -i %s -a "%s"' - % (user, ansible_verb, host, inv_path, cmd)) + % (user, ansible_verb, host, inv_path, cmd)) try: - (out, _) = subprocess.Popen(ansible_cmd, shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE).communicate() + (out, err) = subprocess.Popen(ansible_cmd, shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() except Exception as e: print('%s\nCannot communicate with host: %s, skipping' % (e, host)) finally: os.remove(inv_path) if not out: - print('Host %s is not accessible, skipping' % host) + print('Host %s is not accessible: %s, skipping' % (host, err)) else: out = safe_decode(out) if '>>' not in out: From 0ba87e4eac2baf0ebf77f8323f927c86f0c91d11 Mon Sep 17 00:00:00 2001 From: Borne Mace Date: Tue, 10 Nov 2015 17:32:39 -0800 Subject: [PATCH 12/25] fixed property edit error causing spurious blank lines in globals.yml --- kollacli/utils.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/kollacli/utils.py b/kollacli/utils.py index 06fd319..c8da13c 100644 --- a/kollacli/utils.py +++ b/kollacli/utils.py @@ -177,23 +177,24 @@ def change_property(file_path, property_key, property_value, clear=False): new_contents = [] read_data = sync_read_file(file_path) lines = read_data.split('\n') - new_line = '%s: "%s"\n' % (property_key, property_value) + new_line = '%s: "%s"' % (property_key, property_value) property_key_found = False for line in lines: + line = line.rstrip() # make sure to swallow any newlines if line[0:len(property_key)] == property_key: property_key_found = True if clear: - # clear existing property - line = '' + # clear existing property by not adding it to output + continue else: # edit existing property line = new_line - new_contents.append(line + '\n') + new_contents.append(line) if not property_key_found and not clear: # add new property to file new_contents.append(new_line) - write_data = ''.join(new_contents) + write_data = '\n'.join(new_contents) sync_write_file(file_path, write_data) except Exception as e: From c71328d0c684e333114808ae4cba2a7c5e54177f Mon Sep 17 00:00:00 2001 From: Borne Mace Date: Wed, 11 Nov 2015 12:41:26 -0800 Subject: [PATCH 13/25] added logic to remove reundant empty lines in the globals.yml --- kollacli/utils.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/kollacli/utils.py b/kollacli/utils.py index c8da13c..9e23319 100644 --- a/kollacli/utils.py +++ b/kollacli/utils.py @@ -179,16 +179,25 @@ def change_property(file_path, property_key, property_value, clear=False): lines = read_data.split('\n') new_line = '%s: "%s"' % (property_key, property_value) property_key_found = False + last_line_empty = False for line in lines: - line = line.rstrip() # make sure to swallow any newlines + line = line.rstrip() + + # yank spurious empty lines + if line: + last_line_empty = False + else: + if last_line_empty: + continue + last_line_empty = True + if line[0:len(property_key)] == property_key: property_key_found = True if clear: - # clear existing property by not adding it to output + # clear existing property continue - else: - # edit existing property - line = new_line + # edit existing property + line = new_line new_contents.append(line) if not property_key_found and not clear: # add new property to file From 22c8f1eb4f1f468fba4489b0facbb87addc7b7fb Mon Sep 17 00:00:00 2001 From: Steve Noyes Date: Wed, 11 Nov 2015 16:59:03 -0500 Subject: [PATCH 14/25] update property/password tests to check file size --- tests/password.py | 14 ++++++++++++++ tests/property.py | 15 +++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/tests/password.py b/tests/password.py index d08f892..016aed0 100644 --- a/tests/password.py +++ b/tests/password.py @@ -13,12 +13,21 @@ # under the License. # from common import KollaCliTest +import os import unittest +from kollacli.utils import get_kolla_etc + class TestFunctional(KollaCliTest): def test_password_set_clear(self): + + # This test should leave the passwords.yml file unchanged + # after the test completes. + pwds_path = os.path.join(get_kolla_etc(), 'passwords.yml') + size_start = os.path.getsize(pwds_path) + # test list msg = self.run_cli_cmd('password list') key = 'database_password' @@ -57,6 +66,11 @@ class TestFunctional(KollaCliTest): '(%s/%s) not in output: %s' % (key, value, msg)) + # check that passwords.yml file size didn't change + size_end = os.path.getsize(pwds_path) + self.assertEqual(size_start, size_end, 'passwords.yml size changed ' + + 'from %s to %s' % (size_start, size_end)) + def _password_value_exists(self, key, value, cli_output): """Verify cli data against model data""" # check for any host in cli output that shouldn't be there diff --git a/tests/property.py b/tests/property.py index 1a94165..144ee29 100644 --- a/tests/property.py +++ b/tests/property.py @@ -13,13 +13,23 @@ # under the License. # from common import KollaCliTest + +import os import unittest +from kollacli.utils import get_kolla_etc + class TestFunctional(KollaCliTest): def test_property_set_clear(self): # test list + + # This test should leave the globals.yml file unchanged + # after the test completes. + globals_path = os.path.join(get_kolla_etc(), 'globals.yml') + size_start = os.path.getsize(globals_path) + msg = self.run_cli_cmd('property list') key = 'kolla_base_distro' value = 'ol' @@ -53,6 +63,11 @@ class TestFunctional(KollaCliTest): self.assertFalse(ok, 'clear failed property in output: %s, %s' % (key, value)) + # check that globals.yml file size didn't change + size_end = os.path.getsize(globals_path) + self.assertEqual(size_start, size_end, 'globals.yml size changed ' + + 'from %s to %s' % (size_start, size_end)) + def _property_value_exists(self, key, value, cli_output): """Verify cli data against model data""" # check for any host in cli output that shouldn't be there From af22f6e5f0e817abe56f7d221d436f613547b40f Mon Sep 17 00:00:00 2001 From: Steve Noyes Date: Thu, 12 Nov 2015 17:21:31 -0500 Subject: [PATCH 15/25] update host.py to i18n - update log, help, and exception strings for i18n - fix bug in py3 util --- kollacli/host.py | 74 +++++++++++++++++++++++------------------------ kollacli/utils.py | 2 +- 2 files changed, 37 insertions(+), 39 deletions(-) diff --git a/kollacli/host.py b/kollacli/host.py index 3c19d38..8944af2 100644 --- a/kollacli/host.py +++ b/kollacli/host.py @@ -18,6 +18,9 @@ import os import traceback import yaml +from kollacli.i18n import _ +from kollacli.i18n import _LI + from kollacli.ansible.inventory import Inventory from kollacli.ansible.playbook import AnsiblePlaybook from kollacli.ansible import properties @@ -29,21 +32,21 @@ from kollacli.utils import get_setup_user from cliff.command import Command from cliff.lister import Lister +LOG = logging.getLogger(__name__) -def _host_not_found(log, hostname): + +def _host_not_found(hostname): raise CommandError( - 'Host (%s) not found. ' % hostname + - 'Please add it with "host add"') + _('Host ({host}) not found. Please ' + 'add it with "host add".').format(host=hostname)) class HostAdd(Command): """Add host to open-stack-kolla""" - log = logging.getLogger(__name__) - def get_parser(self, prog_name): parser = super(HostAdd, self).get_parser(prog_name) parser.add_argument('hostname', metavar='', - help='host name or ip address') + help=_('Host name or ip address')) return parser def take_action(self, parsed_args): @@ -66,14 +69,13 @@ class HostDestroy(Command): Stops and removes all kolla related docker containers on either the specified host or if no host is specified, on all hosts. """ - log = logging.getLogger(__name__) def get_parser(self, prog_name): parser = super(HostDestroy, self).get_parser(prog_name) parser.add_argument('hostname', metavar='', - help='host name or ip address or "all"') + help=_('Host name or ip address or "all"')) parser.add_argument('--stop', action='store_true', - help='stop rather than kill') + help=_('Stop rather than kill')) return parser def take_action(self, parsed_args): @@ -86,13 +88,13 @@ class HostDestroy(Command): inventory = Inventory.load() host = inventory.get_host(hostname) if not host: - _host_not_found(self.log, hostname) + _host_not_found(hostname) destroy_type = 'kill' if parsed_args.stop: destroy_type = 'stop' - self.log.info('please be patient as this may take a while.') + LOG.info(_LI('Please be patient as this may take a while.')) ansible_properties = properties.AnsibleProperties() base_distro = \ ansible_properties.get_property('kolla_base_distro') @@ -118,11 +120,10 @@ class HostDestroy(Command): class HostRemove(Command): """Remove host from openstack-kolla""" - log = logging.getLogger(__name__) - def get_parser(self, prog_name): parser = super(HostRemove, self).get_parser(prog_name) - parser.add_argument('hostname', metavar='', help='host name') + parser.add_argument('hostname', metavar='', + help=_('Host name')) return parser def take_action(self, parsed_args): @@ -144,12 +145,10 @@ class HostList(Lister): If a hostname is provided, only list information about that host. """ - log = logging.getLogger(__name__) - def get_parser(self, prog_name): parser = super(HostList, self).get_parser(prog_name) parser.add_argument('hostname', nargs='?', metavar='[hostname]', - help='hostname') + help=_('Host name')) return parser def take_action(self, parsed_args): @@ -164,7 +163,7 @@ class HostList(Lister): if hostname: host = inventory.get_host(hostname) if not host: - _host_not_found(self.log, hostname) + _host_not_found(hostname) data = [] host_groups = inventory.get_host_groups() @@ -186,11 +185,10 @@ class HostList(Lister): class HostCheck(Command): """Check if openstack-kollacli is setup""" - log = logging.getLogger(__name__) - def get_parser(self, prog_name): parser = super(HostCheck, self).get_parser(prog_name) - parser.add_argument('hostname', metavar='', help='hostname') + parser.add_argument('hostname', metavar='', + help=_('Host name')) return parser def take_action(self, parsed_args): @@ -199,7 +197,7 @@ class HostCheck(Command): hostname = convert_to_unicode(hostname) inventory = Inventory.load() if not inventory.get_host(hostname): - _host_not_found(self.log, hostname) + _host_not_found(hostname) inventory.check_host(hostname) except CommandError as e: @@ -211,26 +209,24 @@ class HostCheck(Command): class HostSetup(Command): """Setup openstack-kollacli on host""" - log = logging.getLogger(__name__) - def get_parser(self, prog_name): parser = super(HostSetup, self).get_parser(prog_name) parser.add_argument('hostname', nargs='?', - metavar='', help='hostname') + metavar='', help=_('Host name')) parser.add_argument('--insecure', nargs='?', help=argparse.SUPPRESS) parser.add_argument('--file', '-f', nargs='?', metavar='', - help='hosts info file absolute path') + help=_('Absolute path to hosts info file ')) return parser def take_action(self, parsed_args): try: if not parsed_args.hostname and not parsed_args.file: - raise CommandError('Hostname or hosts info file path ' + - 'is required') + raise CommandError(_('Host name or hosts info file path ' + 'is required.')) if parsed_args.hostname and parsed_args.file: - raise CommandError('Hostname and hosts info file path ' + - 'cannot both be present') + raise CommandError(_('Host name and hosts info file path ' + 'cannot both be present.')) inventory = Inventory.load() if parsed_args.file: @@ -242,20 +238,21 @@ class HostSetup(Command): hostname = parsed_args.hostname.strip() hostname = convert_to_unicode(hostname) if not inventory.get_host(hostname): - _host_not_found(self.log, hostname) + _host_not_found(hostname) check_ok = inventory.check_host(hostname, True) if check_ok: - self.log.info('Skipping setup of host (%s) as check is ok' - % hostname) + LOG.info(_LI('Skipping setup of host ({host}) as ' + 'check is ok.').format(host=hostname)) return True if parsed_args.insecure: password = parsed_args.insecure.strip() else: setup_user = get_setup_user() - password = getpass.getpass('%s password for %s: ' % - (setup_user, hostname)) + password = getpass.getpass(_( + '{user} password for {host}: ').format(user=setup_user, + host=hostname)) password = convert_to_unicode(password) inventory.setup_host(hostname, password) @@ -266,13 +263,14 @@ class HostSetup(Command): def get_yml_data(self, yml_path): if not os.path.isfile(yml_path): - raise CommandError('No file exists at %s. ' % yml_path + - 'An absolute file path is required.') + raise CommandError(_( + 'No file exists at {path}. ' + 'An absolute file path is required.').format(path=yml_path)) with open(yml_path, 'r') as hosts_file: file_data = hosts_file.read() hosts_info = yaml.safe_load(file_data) if not hosts_info: - raise CommandError('%s is empty' % yml_path) + raise CommandError(_('{path} is empty.').format(path=yml_path)) return hosts_info diff --git a/kollacli/utils.py b/kollacli/utils.py index 9e23319..48a1197 100644 --- a/kollacli/utils.py +++ b/kollacli/utils.py @@ -96,7 +96,7 @@ def get_ansible_command(playbook=False): for fname in os.listdir(usr_bin): if (fname.startswith('python2.') and os.path.isfile(os.path.join(usr_bin, fname))): - suffix = file.split('.')[1] + suffix = fname.split('.')[1] if suffix.isdigit(): py2_path = os.path.join(usr_bin, fname) break From d2d746f5c7349d8ac3665920ad441c53edda561e Mon Sep 17 00:00:00 2001 From: Steve Noyes Date: Fri, 13 Nov 2015 15:24:14 -0500 Subject: [PATCH 16/25] substitute rabbitmq for mysqlclster in destroy test to speed it up --- tests/destroy.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/destroy.py b/tests/destroy.py index 89db748..33286b4 100644 --- a/tests/destroy.py +++ b/tests/destroy.py @@ -15,16 +15,14 @@ from common import KollaCliTest from common import TestConfig -from kollacli.ansible import inventory - import unittest DISABLED_SERVICES = [ - 'cinder', 'glance', 'haproxy', 'heat', 'rabbitmq' + 'cinder', 'glance', 'haproxy', 'heat', 'msqlcluster', 'horizon', 'keystone', 'murano', 'neutron', 'nova', ] ENABLED_SERVICES = [ - 'mysqlcluster' + 'rabbitmq' ] UNKNOWN_HOST = 'Name or service not known' @@ -60,9 +58,12 @@ class TestFunctional(KollaCliTest): self.assertIn(UNKNOWN_HOST, '%s' % e, 'Unexpected exception in host setup: %s' % e) - # add host to all deploy groups - for group in inventory.DEPLOY_GROUPS: - self.run_cli_cmd('group addhost %s %s' % (group, hostname)) + # add host to a new deploy group + group_name = 'test_group' + self.run_cli_cmd('group add %s' % group_name) + self.run_cli_cmd('group addhost %s %s' % (group_name, hostname)) + for service in ENABLED_SERVICES: + self.run_cli_cmd('service addgroup %s %s' % (service, group_name)) # destroy services, initialize server try: From 78d0cc608a1052b6217a42be283dd6286646b980 Mon Sep 17 00:00:00 2001 From: Steve Noyes Date: Fri, 13 Nov 2015 15:34:42 -0500 Subject: [PATCH 17/25] update all files to i18n - update all files for i18n - sshutils: - remove no longer used pre_setup and post_setup methods - preserve log level of paramiko log --- kollacli/ansible/inventory.py | 89 ++++++++++++++++++++++------------- kollacli/ansible/passwords.py | 6 ++- kollacli/ansible/playbook.py | 16 ++++--- kollacli/common.py | 46 +++++++++--------- kollacli/exceptions.py | 3 +- kollacli/group.py | 43 ++++++----------- kollacli/host.py | 56 +++++++++++----------- kollacli/password.py | 17 +++---- kollacli/property.py | 17 +++---- kollacli/service.py | 23 ++++----- kollacli/shell.py | 33 ++++++++----- kollacli/sshutils.py | 68 ++++---------------------- kollacli/utils.py | 20 +++++--- 13 files changed, 205 insertions(+), 232 deletions(-) diff --git a/kollacli/ansible/inventory.py b/kollacli/ansible/inventory.py index ee0128f..1aa3a7d 100644 --- a/kollacli/ansible/inventory.py +++ b/kollacli/ansible/inventory.py @@ -18,6 +18,8 @@ import os import tempfile import traceback +import kollacli.i18n as u + from kollacli import exceptions from kollacli import utils @@ -292,8 +294,9 @@ class Inventory(object): else: inventory = Inventory() except Exception: - raise CommandError('loading inventory failed: %s' - % traceback.format_exc()) + raise CommandError( + u._('Loading inventory failed. : {error}') + .format(error=traceback.format_exc())) return inventory @staticmethod @@ -308,7 +311,9 @@ class Inventory(object): utils.sync_write_file(inventory_path, pretty_data) except Exception as e: - raise CommandError('saving inventory failed: %s' % e) + raise CommandError( + u._('Saving inventory failed. : {error}') + .format(error=str(e))) def _create_default_inventory(self): @@ -350,16 +355,19 @@ class Inventory(object): if group name is not none, add host to group """ if groupname and groupname not in self._groups: - raise CommandError('Group name (%s) does not exist' - % groupname) + raise CommandError( + u._('Group name ({group}) does not exist.') + .format(group=groupname)) if groupname and hostname not in self._hosts: - raise CommandError('Host name (%s) does not exist' - % hostname) + raise CommandError( + u._('Host name ({host}) does not exist.') + .format(host=hostname)) if not groupname and not self.remote_mode and len(self._hosts) >= 1: - raise CommandError('Cannot have more than one host when in ' + - 'local deploy mode') + raise CommandError( + u._('Cannot have more than one host when in local deploy ' + 'mode.')) # create new host if it doesn't exist host = Host(hostname) @@ -380,8 +388,9 @@ class Inventory(object): if group name is not none, remove host from group """ if groupname and groupname not in self._groups: - raise CommandError('Group name (%s) does not exist' - % groupname) + raise CommandError( + u._('Group name ({group}) does not exist.') + .format(group=groupname)) if hostname not in self._hosts: return @@ -410,10 +419,10 @@ class Inventory(object): for hostname, host_info in hosts_info.items(): host = self.get_host(hostname) if not host: - failed_hosts[hostname] = "Host doesn't exist" + failed_hosts[hostname] = u._("Host doesn't exist.") continue if not host_info or 'password' not in host_info: - failed_hosts[hostname] = 'No password in yml file' + failed_hosts[hostname] = u._('No password in yml file.') continue passwd = host_info['password'] uname = None @@ -427,23 +436,27 @@ class Inventory(object): summary = '\n' for hostname, err in failed_hosts.items(): summary = summary + '- %s: %s\n' % (hostname, err) - raise CommandError('Not all hosts were set up: %s' % summary) + raise CommandError( + u._('Not all hosts were set up. : {reasons}') + .format(reasons=summary)) else: - self.log.info('All hosts were successfully set up') + self.log.info(u._LI('All hosts were successfully set up.')) def setup_host(self, hostname, password, uname=None): try: - self.log.info('Starting setup of host (%s)' - % hostname) + self.log.info( + u._LI('Starting setup of host ({host}).') + .format(host=hostname)) ssh_setup_host(hostname, password, uname) check_ok = self.check_host(hostname, True) if not check_ok: - raise Exception('Post setup check failed') - self.log.info('Host (%s) setup succeeded' % hostname) + raise Exception(u._('Post setup check failed.')) + self.log.info(u._LI('Host ({host}) setup succeeded.') + .format(host=hostname)) except Exception as e: raise exceptions.CommandError( - 'Host (%s) setup failed : %s' - % (hostname, e)) + u._('Host ({host}) setup failed : {error}') + .format(host=hostname, error=str(e))) return True def check_host(self, hostname, result_only=False): @@ -467,19 +480,22 @@ class Inventory(object): return False else: raise exceptions.CommandError( - 'Host (%s) check failed : %s %s' - % (hostname, err_msg, output)) + u._('Host ({host}) check failed. : {error} {message}') + .format(host=hostname, error=err_msg, message=output)) else: if not result_only: - self.log.info('Host (%s) check succeeded' % hostname) + self.log.info( + u._LI('Host ({host}) check succeeded.') + .format(host=hostname)) return True def add_group(self, groupname): # Group names cannot overlap with service names: if groupname in self._services or groupname in self._sub_services: - raise CommandError('Invalid group name. A service name ' - 'cannot be used for a group name.') + raise CommandError( + u._('Invalid group name. A service name ' + 'cannot be used for a group name.')) if groupname not in self._groups: self._groups[groupname] = HostGroup(groupname) @@ -492,8 +508,9 @@ class Inventory(object): def remove_group(self, groupname): if groupname in PROTECTED_GROUPS: - raise CommandError('Cannot remove %s group. ' % groupname + - 'It is required by kolla.') + raise CommandError( + u._('Cannot remove {group} group. It is required by kolla.') + .format(group=groupname)) # remove group from services & subservices for service in self._services.values(): @@ -589,7 +606,8 @@ class Inventory(object): def add_group_to_service(self, groupname, servicename): if groupname not in self._groups: - raise CommandError('Group (%s) not found.' % groupname) + raise CommandError(u._('Group ({group}) not found.') + .format(group=groupname)) if servicename in self._services: service = self.get_service(servicename) service.add_groupname(groupname) @@ -597,11 +615,13 @@ class Inventory(object): sub_service = self.get_sub_service(servicename) sub_service.add_groupname(groupname) else: - raise CommandError('Service (%s) not found.' % servicename) + raise CommandError(u._('Service ({service})) not found.') + .format(service=servicename)) def remove_group_from_service(self, groupname, servicename): if groupname not in self._groups: - raise CommandError('Group (%s) not found.' % groupname) + raise CommandError(u._('Group ({group}) not found.') + .format(group=groupname)) if servicename in self._services: service = self.get_service(servicename) service.remove_groupname(groupname) @@ -609,7 +629,8 @@ class Inventory(object): sub_service = self.get_sub_service(servicename) sub_service.remove_groupname(groupname) else: - raise CommandError('Service (%s) not found.' % servicename) + raise CommandError(u._('Service ({service})) not found.') + .format(service=servicename)) def create_sub_service(self, sub_servicename): if sub_servicename not in self._sub_services: @@ -659,8 +680,8 @@ class Inventory(object): def set_deploy_mode(self, remote_flag): if not remote_flag and len(self._hosts) > 1: - raise CommandError('Cannot set local deploy mode when multiple ' + - 'hosts exist') + raise CommandError( + u._('Cannot set local deploy mode when multiple hosts exist.')) self.remote_mode = remote_flag for group in self.get_groups(): diff --git a/kollacli/ansible/passwords.py b/kollacli/ansible/passwords.py index 0677de8..cd39d9e 100644 --- a/kollacli/ansible/passwords.py +++ b/kollacli/ansible/passwords.py @@ -13,6 +13,8 @@ # under the License. import os +import kollacli.i18n as u + from kollacli.exceptions import CommandError from kollacli import utils @@ -29,7 +31,9 @@ def set_password(pwd_key, pwd_value): cmd = '%s -k %s -v %s' % (_get_cmd_prefix(), pwd_key, pwd_value) err_msg, output = utils.run_cmd(cmd, print_output=False) if err_msg: - raise CommandError('%s %s' % (err_msg, output)) + raise CommandError( + u._('Password set failed. {error} {message}') + .format(error=err_msg, message=output)) def clear_password(pwd_key): diff --git a/kollacli/ansible/playbook.py b/kollacli/ansible/playbook.py index a449d9e..8f36873 100644 --- a/kollacli/ansible/playbook.py +++ b/kollacli/ansible/playbook.py @@ -16,6 +16,8 @@ import os import subprocess # nosec import traceback +import kollacli.i18n as u + from kollacli.ansible.inventory import Inventory from kollacli.exceptions import CommandError from kollacli.utils import get_admin_user @@ -60,15 +62,15 @@ class AnsiblePlaybook(object): for hostname in self.hosts: host = inventory.get_host(hostname) if not host: - raise CommandError( - 'Host (%s) not found. ' % hostname) + raise CommandError(u._('Host ({host}) not found.') + .format(host=hostname)) inventory_filter['deploy_hosts'] = self.hosts elif self.groups: for groupname in self.groups: group = inventory.get_group(groupname) if not group: - raise CommandError( - 'Group (%s) not found. ' % groupname) + raise CommandError(u._('Group ({group}) not found.') + .format(group=groupname)) inventory_filter['deploy_groups'] = self.groups inventory_path = inventory.create_json_gen_file(inventory_filter) @@ -103,8 +105,8 @@ class AnsiblePlaybook(object): for service in self.services: valid_service = inventory.get_service(service) if not valid_service: - raise CommandError( - 'Service (%s) not found. ' % service) + raise CommandError(u._('Service ({srvc}) not found.') + .format(srvc=service)) if not first: service_string = service_string + ',' else: @@ -136,7 +138,7 @@ class AnsiblePlaybook(object): err_msg = '%s %s' % (err_msg, output) raise CommandError(err_msg) - self.log.info('Success') + self.log.info(u._('Success')) except CommandError as e: raise e except Exception: diff --git a/kollacli/common.py b/kollacli/common.py index aebc67c..c319724 100644 --- a/kollacli/common.py +++ b/kollacli/common.py @@ -17,6 +17,8 @@ import tarfile import tempfile import traceback +import kollacli.i18n as u + from kollacli.ansible.inventory import Inventory from kollacli.ansible.playbook import AnsiblePlaybook from kollacli.ansible.properties import AnsibleProperties @@ -30,32 +32,32 @@ from kollacli.utils import run_cmd from cliff.command import Command +LOG = logging.getLogger(__name__) + class Deploy(Command): """Deploy""" - - log = logging.getLogger(__name__) - def get_parser(self, prog_name): parser = super(Deploy, self).get_parser(prog_name) parser.add_argument('--hosts', nargs='?', metavar='', - help='deployment host list') + help=u._('Deployment host list')) parser.add_argument('--groups', nargs='?', metavar='', - help='deployment group list') + help=u._('Deployment group list')) parser.add_argument('--services', nargs='?', metavar='', - help='deployment service list') + help=u._('Deployment service list')) parser.add_argument('--serial', action='store_true', - help='deploy serially') + help=u._('Deploy serially')) return parser def take_action(self, parsed_args): try: if parsed_args.hosts and parsed_args.groups: - raise CommandError('Hosts and Groups arguments cannot both ' + - 'be present at the same time.') + raise CommandError( + u._('Hosts and Groups arguments cannot ' + 'both be present at the same time.')) self._run_rules() @@ -98,11 +100,12 @@ class Deploy(Command): for expected_file in expected_files: path = os.path.join(path_pre, expected_file) if not os.path.isfile(path): - msg = ('Deploy failed. ' + - 'Swift is enabled but ring buffers have ' + - 'not yet been set up. Please see the ' + - 'documentation for swift configuration ' + - 'instructions.') + msg = u._( + 'Deploy failed. ' + 'Swift is enabled but ring buffers have ' + 'not yet been set up. Please see the ' + 'documentation for swift configuration ' + 'instructions.') raise CommandError(msg) @@ -113,8 +116,6 @@ class Dump(Command): tar file so be given to support / development to help with debugging problems. """ - log = logging.getLogger(__name__) - def take_action(self, parsed_args): try: kolla_home = get_kolla_home() @@ -154,13 +155,15 @@ class Dump(Command): # add output of various commands self._add_cmd_info(tar) - self.log.info('dump successful to %s' % dump_path) + LOG.info( + u._LI('dump successful to {path}').format(path=dump_path)) except Exception: raise Exception(traceback.format_exc()) def _add_cmd_info(self, tar): # run all the kollacli list commands - cmds = ['kollacli service listgroups', + cmds = ['kollacli --version', + 'kollacli service listgroups', 'kollacli service list', 'kollacli group listservices', 'kollacli group listhosts', @@ -206,7 +209,7 @@ class Setdeploy(Command): def get_parser(self, prog_name): parser = super(Setdeploy, self).get_parser(prog_name) parser.add_argument('mode', metavar='', - help='mode=') + help=u._('mode=')) return parser def take_action(self, parsed_args): @@ -216,8 +219,9 @@ class Setdeploy(Command): if mode == 'remote': remote_flag = True elif mode != 'local': - raise CommandError('Invalid deploy mode. Mode must be ' + - 'either "local" or "remote"') + raise CommandError( + u._('Invalid deploy mode. Mode must be ' + 'either "local" or "remote".')) inventory = Inventory.load() inventory.set_deploy_mode(remote_flag) Inventory.save(inventory) diff --git a/kollacli/exceptions.py b/kollacli/exceptions.py index 79f773b..f37ef06 100644 --- a/kollacli/exceptions.py +++ b/kollacli/exceptions.py @@ -12,9 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. """Exception definitions.""" +import kollacli.i18n as u class CommandError(Exception): def __init__(self, message, *args): - message = 'ERROR: %s' % message + message = u._('ERROR: {message}').format(message=message) super(CommandError, self).__init__(message, *args) diff --git a/kollacli/group.py b/kollacli/group.py index 30ca49e..809adf2 100644 --- a/kollacli/group.py +++ b/kollacli/group.py @@ -11,9 +11,10 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -import logging import traceback +import kollacli.i18n as u + from kollacli.ansible.inventory import Inventory from kollacli.exceptions import CommandError from kollacli import utils @@ -24,12 +25,10 @@ from cliff.lister import Lister class GroupAdd(Command): """Add group to open-stack-kolla""" - log = logging.getLogger(__name__) - def get_parser(self, prog_name): parser = super(GroupAdd, self).get_parser(prog_name) parser.add_argument('groupname', metavar='', - help='group') + help=u._('Group name')) return parser def take_action(self, parsed_args): @@ -49,12 +48,10 @@ class GroupAdd(Command): class GroupRemove(Command): """Remove group from openstack-kolla""" - log = logging.getLogger(__name__) - def get_parser(self, prog_name): parser = super(GroupRemove, self).get_parser(prog_name) parser.add_argument('groupname', metavar='', - help='group name') + help=u._('Group name')) return parser def take_action(self, parsed_args): @@ -72,14 +69,12 @@ class GroupRemove(Command): class GroupAddhost(Command): """Add host to group""" - log = logging.getLogger(__name__) - def get_parser(self, prog_name): parser = super(GroupAddhost, self).get_parser(prog_name) parser.add_argument('groupname', metavar='', - help='group') + help=u._('Group name')) parser.add_argument('hostname', metavar='', - help='host') + help=u._('Host name')) return parser def take_action(self, parsed_args): @@ -100,14 +95,12 @@ class GroupAddhost(Command): class GroupRemovehost(Command): """Remove host group from group""" - log = logging.getLogger(__name__) - def get_parser(self, prog_name): parser = super(GroupRemovehost, self).get_parser(prog_name) parser.add_argument('groupname', metavar='', - help='group') + help=u._('Group name')) parser.add_argument('hostname', metavar='', - help='host') + help=u._('Host name')) return parser def take_action(self, parsed_args): @@ -129,8 +122,6 @@ class GroupRemovehost(Command): class GroupListhosts(Lister): """List all groups and their hosts""" - log = logging.getLogger(__name__) - def take_action(self, parsed_args): try: inventory = Inventory.load() @@ -142,7 +133,7 @@ class GroupListhosts(Lister): data.append((groupname, hostnames)) else: data.append(('', '')) - return (('Group', 'Hosts'), sorted(data)) + return ((u._('Group'), u._('Hosts')), sorted(data)) except CommandError as e: raise e except Exception as e: @@ -151,14 +142,12 @@ class GroupListhosts(Lister): class GroupAddservice(Command): """Add service to group""" - log = logging.getLogger(__name__) - def get_parser(self, prog_name): parser = super(GroupAddservice, self).get_parser(prog_name) parser.add_argument('groupname', metavar='', - help='group') + help=u._('Group name')) parser.add_argument('servicename', metavar='', - help='service') + help=u._('Service name')) return parser def take_action(self, parsed_args): @@ -180,14 +169,12 @@ class GroupAddservice(Command): class GroupRemoveservice(Command): """Remove service group from group""" - log = logging.getLogger(__name__) - def get_parser(self, prog_name): parser = super(GroupRemoveservice, self).get_parser(prog_name) parser.add_argument('groupname', metavar='', - help='group') + help=u._('Group name')) parser.add_argument('servicename', metavar='', - help='service') + help=u._('Service name')) return parser def take_action(self, parsed_args): @@ -209,8 +196,6 @@ class GroupRemoveservice(Command): class GroupListservices(Lister): """List all groups and their services""" - log = logging.getLogger(__name__) - def take_action(self, parsed_args): try: inventory = Inventory.load() @@ -222,7 +207,7 @@ class GroupListservices(Lister): data.append((groupname, sorted(servicenames))) else: data.append(('', '')) - return (('Group', 'Services'), sorted(data)) + return ((u._('Group'), u._('Services')), sorted(data)) except CommandError as e: raise e except Exception as e: diff --git a/kollacli/host.py b/kollacli/host.py index 8944af2..65570df 100644 --- a/kollacli/host.py +++ b/kollacli/host.py @@ -18,8 +18,7 @@ import os import traceback import yaml -from kollacli.i18n import _ -from kollacli.i18n import _LI +import kollacli.i18n as u from kollacli.ansible.inventory import Inventory from kollacli.ansible.playbook import AnsiblePlaybook @@ -37,16 +36,17 @@ LOG = logging.getLogger(__name__) def _host_not_found(hostname): raise CommandError( - _('Host ({host}) not found. Please ' - 'add it with "host add".').format(host=hostname)) + u._('Host ({host}) not found. Please add it with "host add".') + .format(host=hostname)) class HostAdd(Command): """Add host to open-stack-kolla""" + def get_parser(self, prog_name): parser = super(HostAdd, self).get_parser(prog_name) parser.add_argument('hostname', metavar='', - help=_('Host name or ip address')) + help=u._('Host name or ip address')) return parser def take_action(self, parsed_args): @@ -73,9 +73,9 @@ class HostDestroy(Command): def get_parser(self, prog_name): parser = super(HostDestroy, self).get_parser(prog_name) parser.add_argument('hostname', metavar='', - help=_('Host name or ip address or "all"')) + help=u._('Host name or ip address or "all"')) parser.add_argument('--stop', action='store_true', - help=_('Stop rather than kill')) + help=u._('Stop rather than kill')) return parser def take_action(self, parsed_args): @@ -94,7 +94,7 @@ class HostDestroy(Command): if parsed_args.stop: destroy_type = 'stop' - LOG.info(_LI('Please be patient as this may take a while.')) + LOG.info(u._LI('Please be patient as this may take a while.')) ansible_properties = properties.AnsibleProperties() base_distro = \ ansible_properties.get_property('kolla_base_distro') @@ -123,7 +123,7 @@ class HostRemove(Command): def get_parser(self, prog_name): parser = super(HostRemove, self).get_parser(prog_name) parser.add_argument('hostname', metavar='', - help=_('Host name')) + help=u._('Host name')) return parser def take_action(self, parsed_args): @@ -148,7 +148,7 @@ class HostList(Lister): def get_parser(self, prog_name): parser = super(HostList, self).get_parser(prog_name) parser.add_argument('hostname', nargs='?', metavar='[hostname]', - help=_('Host name')) + help=u._('Host name')) return parser def take_action(self, parsed_args): @@ -175,7 +175,7 @@ class HostList(Lister): data.append((hostname, groupnames)) else: data.append(('', '')) - return (('Host', 'Groups'), sorted(data)) + return ((u._('Host'), u._('Groups')), sorted(data)) except CommandError as e: raise e except Exception as e: @@ -188,7 +188,7 @@ class HostCheck(Command): def get_parser(self, prog_name): parser = super(HostCheck, self).get_parser(prog_name) parser.add_argument('hostname', metavar='', - help=_('Host name')) + help=u._('Host name')) return parser def take_action(self, parsed_args): @@ -212,21 +212,22 @@ class HostSetup(Command): def get_parser(self, prog_name): parser = super(HostSetup, self).get_parser(prog_name) parser.add_argument('hostname', nargs='?', - metavar='', help=_('Host name')) + metavar='', help=u._('Host name')) parser.add_argument('--insecure', nargs='?', help=argparse.SUPPRESS) parser.add_argument('--file', '-f', nargs='?', metavar='', - help=_('Absolute path to hosts info file ')) + help=u._('Absolute path to hosts info file ')) return parser def take_action(self, parsed_args): try: if not parsed_args.hostname and not parsed_args.file: - raise CommandError(_('Host name or hosts info file path ' - 'is required.')) + raise CommandError( + u._('Host name or hosts info file path is required.')) if parsed_args.hostname and parsed_args.file: - raise CommandError(_('Host name and hosts info file path ' - 'cannot both be present.')) + raise CommandError( + u._('Host name and hosts info file path ' + 'cannot both be present.')) inventory = Inventory.load() if parsed_args.file: @@ -242,17 +243,18 @@ class HostSetup(Command): check_ok = inventory.check_host(hostname, True) if check_ok: - LOG.info(_LI('Skipping setup of host ({host}) as ' - 'check is ok.').format(host=hostname)) + LOG.info( + u._LI('Skipping setup of host ({host}) as ' + 'check is ok.').format(host=hostname)) return True if parsed_args.insecure: password = parsed_args.insecure.strip() else: setup_user = get_setup_user() - password = getpass.getpass(_( - '{user} password for {host}: ').format(user=setup_user, - host=hostname)) + password = getpass.getpass( + u._('{user} password for {host}: ') + .format(user=setup_user, host=hostname)) password = convert_to_unicode(password) inventory.setup_host(hostname, password) @@ -263,14 +265,14 @@ class HostSetup(Command): def get_yml_data(self, yml_path): if not os.path.isfile(yml_path): - raise CommandError(_( - 'No file exists at {path}. ' - 'An absolute file path is required.').format(path=yml_path)) + raise CommandError( + u._('No file exists at {path}. An absolute file path is ' + 'required.').format(path=yml_path)) with open(yml_path, 'r') as hosts_file: file_data = hosts_file.read() hosts_info = yaml.safe_load(file_data) if not hosts_info: - raise CommandError(_('{path} is empty.').format(path=yml_path)) + raise CommandError(u._('{path} is empty.').format(path=yml_path)) return hosts_info diff --git a/kollacli/password.py b/kollacli/password.py index e76af25..37a6ba5 100644 --- a/kollacli/password.py +++ b/kollacli/password.py @@ -13,9 +13,10 @@ # under the License. import argparse import getpass -import logging import traceback +import kollacli.i18n as u + from cliff.command import Command from cliff.lister import Lister @@ -27,12 +28,10 @@ from kollacli.ansible.passwords import set_password class PasswordSet(Command): "Password Set" - log = logging.getLogger(__name__) - def get_parser(self, prog_name): parser = super(PasswordSet, self).get_parser(prog_name) parser.add_argument('passwordname', metavar='', - help='passwordname') + help=u._('Password name')) parser.add_argument('--insecure', nargs='?', help=argparse.SUPPRESS) return parser @@ -42,7 +41,7 @@ class PasswordSet(Command): if parsed_args.insecure: password = parsed_args.insecure.strip() else: - password = getpass.getpass('Password: ').strip() + password = getpass.getpass(u._('Password: ')).strip() set_password(password_name, password) @@ -53,12 +52,10 @@ class PasswordSet(Command): class PasswordClear(Command): "Password Clear" - log = logging.getLogger(__name__) - def get_parser(self, prog_name): parser = super(PasswordClear, self).get_parser(prog_name) parser.add_argument('passwordname', metavar='', - help='passwordname') + help=u._('Password name')) return parser def take_action(self, parsed_args): @@ -72,8 +69,6 @@ class PasswordClear(Command): class PasswordList(Lister): """List all password names""" - log = logging.getLogger(__name__) - def take_action(self, parsed_args): password_names = get_password_names() password_names = sorted(password_names) @@ -82,4 +77,4 @@ class PasswordList(Lister): for password_name in password_names: data.append((password_name, '-')) - return (('Password Name', 'Password'), data) + return ((u._('Password Name'), u._('Password')), data) diff --git a/kollacli/property.py b/kollacli/property.py index 4d4007a..c78e8b7 100644 --- a/kollacli/property.py +++ b/kollacli/property.py @@ -11,9 +11,10 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -import logging import traceback +import kollacli.i18n as u + from kollacli.ansible import properties from cliff.command import Command @@ -23,14 +24,12 @@ from cliff.lister import Lister class PropertySet(Command): "Property Set" - log = logging.getLogger(__name__) - def get_parser(self, prog_name): parser = super(PropertySet, self).get_parser(prog_name) parser.add_argument('propertyname', metavar='', - help='propertyname') + help=u._('Property name')) parser.add_argument('propertyvalue', metavar='> %s"' % (admin_user, public_key, key_dir)) - _exec_ssh_cmd(cmd, ssh_client, log) + _exec_ssh_cmd(cmd, ssh_client) # TODO(bmace) verify ssh connection to the new account except Exception as e: @@ -67,55 +66,6 @@ def ssh_setup_host(net_addr, password, setup_user=None): _close_ssh_client(ssh_client) -def _pre_setup_checks(ssh_client, log): - cmd = 'docker --version' - msg, errmsg = _exec_ssh_cmd(cmd, ssh_client, log) - if errmsg: - raise CommandError("'%s' failed. Is docker installed? : %s" - % (cmd, errmsg)) - if 'Docker version' not in msg: - raise CommandError("'%s' failed. Is docker installed? : %s" - % (cmd, msg)) - - version = msg.split('version ')[1].split(',')[0] - if StrictVersion(version) < StrictVersion(MIN_DOCKER_VERSION): - raise CommandError('docker version (%s) below minimum (%s)' - % (version, msg)) - - # docker is installed, now check if it is running - cmd = 'docker info' - _, errmsg = _exec_ssh_cmd(cmd, ssh_client, log) - # docker info can return warning messages in stderr, ignore them - if errmsg and 'WARNING' not in errmsg: - raise CommandError("'%s' failed. Is docker running? : %s" - % (cmd, errmsg)) - - # check for docker-py - cmd = 'python -c "import docker"' - msg, errmsg = _exec_ssh_cmd(cmd, ssh_client, log) - if errmsg: - raise CommandError('host check failed. ' + - 'Is docker-py installed?') - - -def _post_setup_checks(net_addr, log): - try: - ssh_client = ssh_connect(net_addr, get_admin_user(), '') - - except Exception as e: - raise CommandError("remote login failed : %s" % e) - - try: - # a basic test - ssh_client.exec_command('ls') # nosec - - except Exception as e: - raise CommandError("remote command 'ls' failed : %s" % e) - - finally: - _close_ssh_client(ssh_client) - - def _close_ssh_client(ssh_client): if ssh_client: try: @@ -124,14 +74,16 @@ def _close_ssh_client(ssh_client): pass -def _exec_ssh_cmd(cmd, ssh_client, log): - log.debug(cmd) +def _exec_ssh_cmd(cmd, ssh_client): + LOG.debug(cmd) _, stdout, stderr = ssh_client.exec_command(cmd, get_pty=True) # nosec msg = stdout.read() errmsg = stderr.read() - log.debug('%s : %s' % (msg, errmsg)) + LOG.debug('%s : %s' % (msg, errmsg)) if errmsg: - log.warn('WARNING: command (%s) message : %s' % (cmd, errmsg.strip())) + LOG.warn( + u._LW('WARNING: command : {command})\nmessage : {message}') + .format(command=cmd, message=errmsg.strip())) return msg, errmsg diff --git a/kollacli/utils.py b/kollacli/utils.py index 48a1197..00d68be 100644 --- a/kollacli/utils.py +++ b/kollacli/utils.py @@ -19,6 +19,8 @@ import pwd import six import sys +import kollacli.i18n as u + from kollacli.exceptions import CommandError from oslo_utils.encodeutils import safe_decode @@ -57,8 +59,10 @@ def get_kolla_log_file_size(): try: size = int(size_str) except Exception: - raise CommandError('Environmental variable ' + - '(%s) is not an integer' % (envvar, size_str)) + raise CommandError( + u._('Environmental variable ({env_var}) is not an ' + 'integer ({log_size}).') + .format(env_var=envvar, log_size=size_str)) return size @@ -101,8 +105,10 @@ def get_ansible_command(playbook=False): py2_path = os.path.join(usr_bin, fname) break if py2_path is None: - raise Exception('ansible-playbook requires python2 and no ' - 'python2 interpreter found in %s' % usr_bin) + raise Exception( + u._('ansible-playbook requires python2 and no ' + 'python2 interpreter found in {path}.') + .format(path=usr_bin)) cmd = '%s %s' % (py2_path, os.path.join(usr_bin, cmd)) return cmd @@ -139,7 +145,8 @@ def run_cmd(cmd, print_output=True): if sniff == pwd_prompt: output = sniff + '\n' raise Exception( - 'Insufficient permissions to run command "%s"' % cmd) + u._('Insufficient permissions to run command "{command}".') + .format(command=cmd)) child.maxsize = 1 child.timeout = 86400 for line in child: @@ -156,7 +163,8 @@ def run_cmd(cmd, print_output=True): if child: child.close() if child.exitstatus != 0: - err_msg = 'Command Failed %s' % err_msg + err_msg = (u._('Command failed. : {error}') + .format(error=err_msg)) return err_msg, output From 6f2bef054e935a94f0c56ba21504e2bc753b42f1 Mon Sep 17 00:00:00 2001 From: Steve Noyes Date: Wed, 18 Nov 2015 17:40:12 -0500 Subject: [PATCH 18/25] add initial oslo config framework this adds just enough infrastructure to run the oslo-config generator command to generate a sample config file. --- buildrpm/openstack-kollacli.spec | 1 + kollacli/common/__init__.py | 0 kollacli/common/config.py | 85 ++++++++++++++++++++++++++++++++ kollacli/{common.py => misc.py} | 0 kollacli/version.py | 19 +++++++ requirements.txt | 1 + setup.cfg | 9 ++-- tox.ini | 5 ++ 8 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 kollacli/common/__init__.py create mode 100644 kollacli/common/config.py rename kollacli/{common.py => misc.py} (100%) create mode 100644 kollacli/version.py diff --git a/buildrpm/openstack-kollacli.spec b/buildrpm/openstack-kollacli.spec index 51714e4..60bcbb2 100644 --- a/buildrpm/openstack-kollacli.spec +++ b/buildrpm/openstack-kollacli.spec @@ -42,6 +42,7 @@ Requires: python-babel >= 2.0 Requires: python-cliff >= 1.13.0 Requires: python-cliff-tablib >= 1.1 Requires: python-jsonpickle >= 0.9.2 +Requires: python-oslo-config >= 2.3.0 Requires: python-oslo-i18n >= 2.5.0 Requires: python-oslo-utils >= 2.4.0 Requires: python-paramiko >= 1.15.1 diff --git a/kollacli/common/__init__.py b/kollacli/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/kollacli/common/config.py b/kollacli/common/config.py new file mode 100644 index 0000000..c0fb220 --- /dev/null +++ b/kollacli/common/config.py @@ -0,0 +1,85 @@ +# Copyright 2010-2011 OpenStack LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Configuration setup for kollacli. +""" + +import logging +import os + +from oslo_config import cfg + +from kollacli import i18n as u +from kollacli import version + +MAX_LOG_SIZE = 500000 + +log_opts = [ + + cfg.IntOpt('max_size_of_log_in_bytes', + + default=MAX_LOG_SIZE, + help=u._('The maximum log size in bytes.')), +] + + +def parse_args(conf, args=None, usage=None, default_config_files=None): + conf(args=args if args else [], + project='kollacli', + prog='kollacli', + version=version.__version__, + usage=usage, + default_config_files=default_config_files) + + conf.pydev_debug_host = os.environ.get('PYDEV_DEBUG_HOST') + conf.pydev_debug_port = os.environ.get('PYDEV_DEBUG_PORT') + + +def list_opts(): + yield None, log_opts + + +def new_config(): + conf = cfg.ConfigOpts() + conf.register_opts(log_opts) + return conf + + +def setup_remote_pydev_debug(): + """Required setup for remote debugging.""" + if CONF.pydev_debug_host and CONF.pydev_debug_port: + try: + try: + from pydev import pydevd + except ImportError: + import pydevd + + pydevd.settrace(CONF.pydev_debug_host, + port=int(CONF.pydev_debug_port), + stdoutToServer=True, + stderrToServer=True) + + except Exception: + LOG.exception('Unable to join debugger, please ' + 'make sure that the debugger processes is ' + 'listening on debug-host \'%s\' debug-port \'%s\'.', + CONF.pydev_debug_host, CONF.pydev_debug_port) + raise + + +CONF = new_config() +LOG = logging.getLogger(__name__) +parse_args(CONF) diff --git a/kollacli/common.py b/kollacli/misc.py similarity index 100% rename from kollacli/common.py rename to kollacli/misc.py diff --git a/kollacli/version.py b/kollacli/version.py new file mode 100644 index 0000000..9d0ebcb --- /dev/null +++ b/kollacli/version.py @@ -0,0 +1,19 @@ +# Copyright 2010-2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pbr.version + +version_info = pbr.version.VersionInfo('kollacli') +__version__ = version_info.release_string() diff --git a/requirements.txt b/requirements.txt index 2132591..ef6aa95 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ cliff>=1.13.0 # Apache-2.0 cliff-tablib>=1.1 docker-py>=1.3.1 jsonpickle>=0.9 +oslo.config>=2.3.0 # Apache-2.0 oslo.i18n>=1.3.0 # Apache-2.0 oslo.utils!=2.6.0,>=2.4.0 # Apache-2.0 paramiko>=1.15 diff --git a/setup.cfg b/setup.cfg index f336e3e..cd12912 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,9 +30,12 @@ packages = console_scripts = kollacli = kollacli.shell:main +oslo.config.opts = + kollacli.common.config = kollacli.common.config:list_opts + kolla.cli = - deploy = kollacli.common:Deploy - dump = kollacli.common:Dump + deploy = kollacli.misc:Deploy + dump = kollacli.misc:Dump group_add = kollacli.group:GroupAdd group_addhost = kollacli.group:GroupAddhost group_listhosts = kollacli.group:GroupListhosts @@ -55,7 +58,7 @@ kolla.cli = service_list = kollacli.service:ServiceList service_listgroups = kollacli.service:ServiceListGroups service_removegroup = kollacli.service:ServiceRemoveGroup - setdeploy = kollacli.common:Setdeploy + setdeploy = kollacli.misc:Setdeploy [extract_messages] keywords = _ gettext ngettext l_ lazy_gettext diff --git a/tox.ini b/tox.ini index 6974753..d2b43bd 100644 --- a/tox.ini +++ b/tox.ini @@ -34,3 +34,8 @@ exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build [testenv:bandit] deps = -r{toxinidir}/test-requirements.txt commands = bandit -r kollacli + +[testenv:genconfig] +commands = + oslo-config-generator --namespace kollacli.common.config + From 64dfd34c388d41c29814deef608c613059f3f2fc Mon Sep 17 00:00:00 2001 From: Steve Noyes Date: Mon, 23 Nov 2015 11:22:51 -0500 Subject: [PATCH 19/25] reverting config changes (for kollacli v2.1 stream) --- buildrpm/openstack-kollacli.spec | 1 - kollacli/common/__init__.py | 0 kollacli/common/config.py | 85 -------------------------------- kollacli/version.py | 19 ------- requirements.txt | 1 - setup.cfg | 3 -- tox.ini | 5 -- 7 files changed, 114 deletions(-) delete mode 100644 kollacli/common/__init__.py delete mode 100644 kollacli/common/config.py delete mode 100644 kollacli/version.py diff --git a/buildrpm/openstack-kollacli.spec b/buildrpm/openstack-kollacli.spec index 60bcbb2..51714e4 100644 --- a/buildrpm/openstack-kollacli.spec +++ b/buildrpm/openstack-kollacli.spec @@ -42,7 +42,6 @@ Requires: python-babel >= 2.0 Requires: python-cliff >= 1.13.0 Requires: python-cliff-tablib >= 1.1 Requires: python-jsonpickle >= 0.9.2 -Requires: python-oslo-config >= 2.3.0 Requires: python-oslo-i18n >= 2.5.0 Requires: python-oslo-utils >= 2.4.0 Requires: python-paramiko >= 1.15.1 diff --git a/kollacli/common/__init__.py b/kollacli/common/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/kollacli/common/config.py b/kollacli/common/config.py deleted file mode 100644 index c0fb220..0000000 --- a/kollacli/common/config.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2010-2011 OpenStack LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Configuration setup for kollacli. -""" - -import logging -import os - -from oslo_config import cfg - -from kollacli import i18n as u -from kollacli import version - -MAX_LOG_SIZE = 500000 - -log_opts = [ - - cfg.IntOpt('max_size_of_log_in_bytes', - - default=MAX_LOG_SIZE, - help=u._('The maximum log size in bytes.')), -] - - -def parse_args(conf, args=None, usage=None, default_config_files=None): - conf(args=args if args else [], - project='kollacli', - prog='kollacli', - version=version.__version__, - usage=usage, - default_config_files=default_config_files) - - conf.pydev_debug_host = os.environ.get('PYDEV_DEBUG_HOST') - conf.pydev_debug_port = os.environ.get('PYDEV_DEBUG_PORT') - - -def list_opts(): - yield None, log_opts - - -def new_config(): - conf = cfg.ConfigOpts() - conf.register_opts(log_opts) - return conf - - -def setup_remote_pydev_debug(): - """Required setup for remote debugging.""" - if CONF.pydev_debug_host and CONF.pydev_debug_port: - try: - try: - from pydev import pydevd - except ImportError: - import pydevd - - pydevd.settrace(CONF.pydev_debug_host, - port=int(CONF.pydev_debug_port), - stdoutToServer=True, - stderrToServer=True) - - except Exception: - LOG.exception('Unable to join debugger, please ' - 'make sure that the debugger processes is ' - 'listening on debug-host \'%s\' debug-port \'%s\'.', - CONF.pydev_debug_host, CONF.pydev_debug_port) - raise - - -CONF = new_config() -LOG = logging.getLogger(__name__) -parse_args(CONF) diff --git a/kollacli/version.py b/kollacli/version.py deleted file mode 100644 index 9d0ebcb..0000000 --- a/kollacli/version.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2010-2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import pbr.version - -version_info = pbr.version.VersionInfo('kollacli') -__version__ = version_info.release_string() diff --git a/requirements.txt b/requirements.txt index ef6aa95..2132591 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,6 @@ cliff>=1.13.0 # Apache-2.0 cliff-tablib>=1.1 docker-py>=1.3.1 jsonpickle>=0.9 -oslo.config>=2.3.0 # Apache-2.0 oslo.i18n>=1.3.0 # Apache-2.0 oslo.utils!=2.6.0,>=2.4.0 # Apache-2.0 paramiko>=1.15 diff --git a/setup.cfg b/setup.cfg index cd12912..bdd9278 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,9 +30,6 @@ packages = console_scripts = kollacli = kollacli.shell:main -oslo.config.opts = - kollacli.common.config = kollacli.common.config:list_opts - kolla.cli = deploy = kollacli.misc:Deploy dump = kollacli.misc:Dump diff --git a/tox.ini b/tox.ini index d2b43bd..6974753 100644 --- a/tox.ini +++ b/tox.ini @@ -34,8 +34,3 @@ exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build [testenv:bandit] deps = -r{toxinidir}/test-requirements.txt commands = bandit -r kollacli - -[testenv:genconfig] -commands = - oslo-config-generator --namespace kollacli.common.config - From 4dbbc62f3e630a3b4d90c1888965342e105343d1 Mon Sep 17 00:00:00 2001 From: Steve Noyes Date: Mon, 23 Nov 2015 17:00:25 -0500 Subject: [PATCH 20/25] initial move of business logic in cli to common code - rename kollacli/ansible to kollacli/common - remove business logic from deploy and destroy commands --- kollacli/{ansible => common}/__init__.py | 0 kollacli/common/ansible/__init__.py | 0 kollacli/common/ansible/actions.py | 104 +++++++++++++++++++++ kollacli/{ => common}/ansible/playbook.py | 8 +- kollacli/{ansible => common}/inventory.py | 0 kollacli/{ansible => common}/passwords.py | 0 kollacli/{ansible => common}/properties.py | 0 kollacli/group.py | 2 +- kollacli/host.py | 33 ++----- kollacli/misc.py | 62 +++--------- kollacli/password.py | 6 +- kollacli/property.py | 2 +- kollacli/service.py | 2 +- kollacli/shell.py | 2 +- tests/deploy.py | 4 +- tests/service.py | 6 +- 16 files changed, 142 insertions(+), 89 deletions(-) rename kollacli/{ansible => common}/__init__.py (100%) create mode 100644 kollacli/common/ansible/__init__.py create mode 100644 kollacli/common/ansible/actions.py rename kollacli/{ => common}/ansible/playbook.py (97%) rename kollacli/{ansible => common}/inventory.py (100%) rename kollacli/{ansible => common}/passwords.py (100%) rename kollacli/{ansible => common}/properties.py (100%) diff --git a/kollacli/ansible/__init__.py b/kollacli/common/__init__.py similarity index 100% rename from kollacli/ansible/__init__.py rename to kollacli/common/__init__.py diff --git a/kollacli/common/ansible/__init__.py b/kollacli/common/ansible/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/kollacli/common/ansible/actions.py b/kollacli/common/ansible/actions.py new file mode 100644 index 0000000..e12a14f --- /dev/null +++ b/kollacli/common/ansible/actions.py @@ -0,0 +1,104 @@ +# Copyright(c) 2015, Oracle and/or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import logging +import os + +import kollacli.i18n as u + +from kollacli.common.ansible.playbook import AnsiblePlaybook +from kollacli.common import properties +from kollacli.common.properties import AnsibleProperties +from kollacli.exceptions import CommandError +from kollacli.utils import get_kolla_etc +from kollacli.utils import get_kolla_home +from kollacli.utils import get_kollacli_home + +LOG = logging.getLogger(__name__) + + +def destroy_hosts(hostname, destroy_type, verbose_level=1): + '''destroy containers on a host (or all hosts). + + If hostname == 'all', then containers on all hosts will be + stopped. Otherwise, the containers on the specified host + will be stopped. + + The destroy type can either be 'stop' or 'kill'. + ''' + if destroy_type not in ['stop', 'kill']: + raise CommandError( + u._('Invalid destroy type ({type}). Must be either ' + '"stop" or "kill".').format(type=destroy_type)) + + LOG.info(u._LI('Please be patient as this may take a while.')) + ansible_properties = properties.AnsibleProperties() + base_distro = \ + ansible_properties.get_property('kolla_base_distro') + install_type = \ + ansible_properties.get_property('kolla_install_type') + container_prefix = base_distro + '-' + install_type + kollacli_home = get_kollacli_home() + playbook = AnsiblePlaybook() + playbook.playbook_path = os.path.join(kollacli_home, + 'ansible/host_destroy.yml') + playbook.extra_vars = 'hosts=' + hostname + \ + ' prefix=' + container_prefix + \ + ' destroy_type=' + destroy_type + playbook.print_output = False + playbook.verbose_level = verbose_level + playbook.run() + + +def deploy(hostnames=[], groupnames=[], servicenames=[], + serial_flag=False, verbose_level=1): + if hostnames and groupnames: + raise CommandError( + u._('Hosts and Groups arguments cannot ' + 'both be present at the same time.')) + + _run_deploy_rules() + + playbook = AnsiblePlaybook() + kolla_home = get_kolla_home() + playbook.playbook_path = os.path.join(kolla_home, + 'ansible/site.yml') + playbook.hosts = hostnames + playbook.groups = groupnames + playbook.services = servicenames + playbook.serial = serial_flag + + playbook.verbose_level = verbose_level + playbook.run() + + +def _run_deploy_rules(): + # check that ring files are in /etc/kolla/config/swift if + # swift is enabled + expected_files = ['account.ring.gz', + 'container.ring.gz', + 'object.ring.gz'] + properties = AnsibleProperties() + is_enabled = properties.get_property('enable_swift') + if is_enabled == 'yes': + path_pre = os.path.join(get_kolla_etc(), 'config', 'swift') + for expected_file in expected_files: + path = os.path.join(path_pre, expected_file) + if not os.path.isfile(path): + msg = u._( + 'Deploy failed. ' + 'Swift is enabled but ring buffers have ' + 'not yet been set up. Please see the ' + 'documentation for swift configuration ' + 'instructions.') + raise CommandError(msg) diff --git a/kollacli/ansible/playbook.py b/kollacli/common/ansible/playbook.py similarity index 97% rename from kollacli/ansible/playbook.py rename to kollacli/common/ansible/playbook.py index 8f36873..04c7f3d 100644 --- a/kollacli/ansible/playbook.py +++ b/kollacli/common/ansible/playbook.py @@ -18,13 +18,14 @@ import traceback import kollacli.i18n as u -from kollacli.ansible.inventory import Inventory from kollacli.exceptions import CommandError from kollacli.utils import get_admin_user from kollacli.utils import get_ansible_command from kollacli.utils import get_kolla_etc from kollacli.utils import run_cmd +from kollacli.common.inventory import Inventory + class AnsiblePlaybook(object): playbook_path = '' @@ -56,8 +57,8 @@ class AnsiblePlaybook(object): admin_user = get_admin_user() command_string = ('/usr/bin/sudo -u %s %s %s' % (admin_user, ansible_cmd, flag)) - inventory = Inventory.load() inventory_filter = {} + inventory = Inventory.load() if self.hosts: for hostname in self.hosts: host = inventory.get_host(hostname) @@ -73,7 +74,8 @@ class AnsiblePlaybook(object): .format(group=groupname)) inventory_filter['deploy_groups'] = self.groups - inventory_path = inventory.create_json_gen_file(inventory_filter) + inventory_path = \ + inventory.create_json_gen_file(inventory_filter) inventory_string = '-i ' + inventory_path cmd = (command_string + ' ' + inventory_string) diff --git a/kollacli/ansible/inventory.py b/kollacli/common/inventory.py similarity index 100% rename from kollacli/ansible/inventory.py rename to kollacli/common/inventory.py diff --git a/kollacli/ansible/passwords.py b/kollacli/common/passwords.py similarity index 100% rename from kollacli/ansible/passwords.py rename to kollacli/common/passwords.py diff --git a/kollacli/ansible/properties.py b/kollacli/common/properties.py similarity index 100% rename from kollacli/ansible/properties.py rename to kollacli/common/properties.py diff --git a/kollacli/group.py b/kollacli/group.py index 809adf2..e7d66a1 100644 --- a/kollacli/group.py +++ b/kollacli/group.py @@ -15,7 +15,7 @@ import traceback import kollacli.i18n as u -from kollacli.ansible.inventory import Inventory +from kollacli.common.inventory import Inventory from kollacli.exceptions import CommandError from kollacli import utils diff --git a/kollacli/host.py b/kollacli/host.py index 65570df..79368b5 100644 --- a/kollacli/host.py +++ b/kollacli/host.py @@ -20,12 +20,10 @@ import yaml import kollacli.i18n as u -from kollacli.ansible.inventory import Inventory -from kollacli.ansible.playbook import AnsiblePlaybook -from kollacli.ansible import properties +from kollacli.common.ansible.actions import destroy_hosts +from kollacli.common.inventory import Inventory from kollacli.exceptions import CommandError from kollacli.utils import convert_to_unicode -from kollacli.utils import get_kollacli_home from kollacli.utils import get_setup_user from cliff.command import Command @@ -84,33 +82,14 @@ class HostDestroy(Command): hostname = parsed_args.hostname.strip() hostname = convert_to_unicode(hostname) - if hostname != 'all': - inventory = Inventory.load() - host = inventory.get_host(hostname) - if not host: - _host_not_found(hostname) - destroy_type = 'kill' if parsed_args.stop: destroy_type = 'stop' - LOG.info(u._LI('Please be patient as this may take a while.')) - ansible_properties = properties.AnsibleProperties() - base_distro = \ - ansible_properties.get_property('kolla_base_distro') - install_type = \ - ansible_properties.get_property('kolla_install_type') - container_prefix = base_distro + '-' + install_type - kollacli_home = get_kollacli_home() - playbook = AnsiblePlaybook() - playbook.playbook_path = os.path.join(kollacli_home, - 'ansible/host_destroy.yml') - playbook.extra_vars = 'hosts=' + hostname + \ - ' prefix=' + container_prefix + \ - ' destroy_type=' + destroy_type - playbook.print_output = False - playbook.verbose_level = self.app.options.verbose_level - playbook.run() + verbose_level = self.app.options.verbose_level + + destroy_hosts(hostname, destroy_type, verbose_level) + except CommandError as e: raise e except Exception as e: diff --git a/kollacli/misc.py b/kollacli/misc.py index c319724..99b6925 100644 --- a/kollacli/misc.py +++ b/kollacli/misc.py @@ -19,9 +19,8 @@ import traceback import kollacli.i18n as u -from kollacli.ansible.inventory import Inventory -from kollacli.ansible.playbook import AnsiblePlaybook -from kollacli.ansible.properties import AnsibleProperties +from kollacli.common.ansible.actions import deploy +from kollacli.common.inventory import Inventory from kollacli.exceptions import CommandError from kollacli.utils import convert_to_unicode from kollacli.utils import get_kolla_etc @@ -53,61 +52,30 @@ class Deploy(Command): return parser def take_action(self, parsed_args): + hosts = None + groups = None + services = None + serial_flag = False + verbose_level = self.app.options.verbose_level try: - if parsed_args.hosts and parsed_args.groups: - raise CommandError( - u._('Hosts and Groups arguments cannot ' - 'both be present at the same time.')) - - self._run_rules() - - playbook = AnsiblePlaybook() - kolla_home = get_kolla_home() - playbook.playbook_path = os.path.join(kolla_home, - 'ansible/site.yml') if parsed_args.hosts: host_list = parsed_args.hosts.strip() - host_list = convert_to_unicode(host_list) - playbook.hosts = host_list.split(',') + hosts = convert_to_unicode(host_list).split(',') if parsed_args.groups: group_list = parsed_args.groups.strip() - group_list = convert_to_unicode(group_list) - playbook.groups = group_list.split(',') + groups = convert_to_unicode(group_list).split(',') if parsed_args.services: - tag_list = parsed_args.services.strip() - tag_list = convert_to_unicode(tag_list) - playbook.services = tag_list.split(',') + service_list = parsed_args.services.strip() + services = convert_to_unicode(service_list).split(',') if parsed_args.serial: - playbook.serial = True + serial_flag = True + + deploy(hosts, groups, services, serial_flag, + verbose_level) - playbook.verbose_level = self.app.options.verbose_level - playbook.run() - except CommandError as e: - raise e except Exception: raise Exception(traceback.format_exc()) - def _run_rules(self): - # check that ring files are in /etc/kolla/config/swift if - # swift is enabled - expected_files = ['account.ring.gz', - 'container.ring.gz', - 'object.ring.gz'] - properties = AnsibleProperties() - is_enabled = properties.get_property('enable_swift') - if is_enabled == 'yes': - path_pre = os.path.join(get_kolla_etc(), 'config', 'swift') - for expected_file in expected_files: - path = os.path.join(path_pre, expected_file) - if not os.path.isfile(path): - msg = u._( - 'Deploy failed. ' - 'Swift is enabled but ring buffers have ' - 'not yet been set up. Please see the ' - 'documentation for swift configuration ' - 'instructions.') - raise CommandError(msg) - class Dump(Command): """Dumps configuration data for debugging diff --git a/kollacli/password.py b/kollacli/password.py index 37a6ba5..1219064 100644 --- a/kollacli/password.py +++ b/kollacli/password.py @@ -20,9 +20,9 @@ import kollacli.i18n as u from cliff.command import Command from cliff.lister import Lister -from kollacli.ansible.passwords import clear_password -from kollacli.ansible.passwords import get_password_names -from kollacli.ansible.passwords import set_password +from kollacli.common.passwords import clear_password +from kollacli.common.passwords import get_password_names +from kollacli.common.passwords import set_password class PasswordSet(Command): diff --git a/kollacli/property.py b/kollacli/property.py index c78e8b7..8fd08f7 100644 --- a/kollacli/property.py +++ b/kollacli/property.py @@ -15,7 +15,7 @@ import traceback import kollacli.i18n as u -from kollacli.ansible import properties +from kollacli.common import properties from cliff.command import Command from cliff.lister import Lister diff --git a/kollacli/service.py b/kollacli/service.py index d306eca..54620c8 100644 --- a/kollacli/service.py +++ b/kollacli/service.py @@ -15,7 +15,7 @@ import traceback import kollacli.i18n as u -from kollacli.ansible.inventory import Inventory +from kollacli.common.inventory import Inventory from kollacli.exceptions import CommandError from kollacli import utils diff --git a/kollacli/shell.py b/kollacli/shell.py index d790d33..37b6d6e 100755 --- a/kollacli/shell.py +++ b/kollacli/shell.py @@ -21,7 +21,7 @@ from cliff.commandmanager import CommandManager import kollacli.i18n as u -from kollacli.ansible.inventory import INVENTORY_PATH +from kollacli.common.inventory import INVENTORY_PATH from kollacli.exceptions import CommandError from kollacli.utils import get_kolla_log_dir from kollacli.utils import get_kolla_log_file_size diff --git a/tests/deploy.py b/tests/deploy.py index 1846233..ada6965 100644 --- a/tests/deploy.py +++ b/tests/deploy.py @@ -14,8 +14,8 @@ # from common import KollaCliTest -from kollacli.ansible.inventory import Inventory -from kollacli.ansible.inventory import SERVICES +from kollacli.common.inventory import Inventory +from kollacli.common.inventory import SERVICES import json import os diff --git a/tests/service.py b/tests/service.py index 604a27d..5b64e8b 100644 --- a/tests/service.py +++ b/tests/service.py @@ -17,9 +17,9 @@ from common import KollaCliTest import json import unittest -from kollacli.ansible.inventory import DEFAULT_GROUPS -from kollacli.ansible.inventory import DEFAULT_OVERRIDES -from kollacli.ansible.inventory import SERVICES +from kollacli.common.inventory import DEFAULT_GROUPS +from kollacli.common.inventory import DEFAULT_OVERRIDES +from kollacli.common.inventory import SERVICES class TestFunctional(KollaCliTest): From dae53cabc87c722c6e39fa969291de66a2d8c359 Mon Sep 17 00:00:00 2001 From: Steve Noyes Date: Tue, 24 Nov 2015 15:04:24 -0500 Subject: [PATCH 21/25] restructuring of directories to separate cli from common code - add new commands directory under kollacli for cli commands - split misc commands into support and deploy - clean up imports - move sshutils and utils under common directory --- kollacli/commands/__init__.py | 0 kollacli/commands/deploy.py | 101 ++++++++++++++++++++++ kollacli/{ => commands}/group.py | 22 ++--- kollacli/{ => commands}/host.py | 4 +- kollacli/{ => commands}/password.py | 0 kollacli/{ => commands}/property.py | 0 kollacli/{ => commands}/service.py | 10 +-- kollacli/{misc.py => commands/support.py} | 88 ++----------------- kollacli/common/ansible/actions.py | 6 +- kollacli/common/ansible/playbook.py | 8 +- kollacli/common/inventory.py | 28 +++--- kollacli/common/passwords.py | 2 +- kollacli/common/properties.py | 8 +- kollacli/{ => common}/sshutils.py | 7 +- kollacli/{ => common}/utils.py | 0 kollacli/shell.py | 6 +- setup.cfg | 50 +++++------ tests/common.py | 2 +- tests/password.py | 2 +- tests/property.py | 2 +- tools/passwd_editor.py | 7 +- 21 files changed, 190 insertions(+), 163 deletions(-) create mode 100644 kollacli/commands/__init__.py create mode 100644 kollacli/commands/deploy.py rename kollacli/{ => commands}/group.py (91%) rename kollacli/{ => commands}/host.py (98%) rename kollacli/{ => commands}/password.py (100%) rename kollacli/{ => commands}/property.py (100%) rename kollacli/{ => commands}/service.py (94%) rename kollacli/{misc.py => commands/support.py} (58%) rename kollacli/{ => common}/sshutils.py (95%) rename kollacli/{ => common}/utils.py (100%) diff --git a/kollacli/commands/__init__.py b/kollacli/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/kollacli/commands/deploy.py b/kollacli/commands/deploy.py new file mode 100644 index 0000000..7d54124 --- /dev/null +++ b/kollacli/commands/deploy.py @@ -0,0 +1,101 @@ +# Copyright(c) 2015, Oracle and/or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import logging +import traceback + +import kollacli.i18n as u + +from kollacli.common.ansible.actions import deploy +from kollacli.common.inventory import Inventory +from kollacli.common.utils import convert_to_unicode +from kollacli.exceptions import CommandError + +from cliff.command import Command + +LOG = logging.getLogger(__name__) + + +class Deploy(Command): + """Deploy""" + def get_parser(self, prog_name): + parser = super(Deploy, self).get_parser(prog_name) + parser.add_argument('--hosts', nargs='?', + metavar='', + help=u._('Deployment host list')) + parser.add_argument('--groups', nargs='?', + metavar='', + help=u._('Deployment group list')) + parser.add_argument('--services', nargs='?', + metavar='', + help=u._('Deployment service list')) + parser.add_argument('--serial', action='store_true', + help=u._('Deploy serially')) + return parser + + def take_action(self, parsed_args): + hosts = None + groups = None + services = None + serial_flag = False + verbose_level = self.app.options.verbose_level + try: + if parsed_args.hosts: + host_list = parsed_args.hosts.strip() + hosts = convert_to_unicode(host_list).split(',') + if parsed_args.groups: + group_list = parsed_args.groups.strip() + groups = convert_to_unicode(group_list).split(',') + if parsed_args.services: + service_list = parsed_args.services.strip() + services = convert_to_unicode(service_list).split(',') + if parsed_args.serial: + serial_flag = True + + deploy(hosts, groups, services, serial_flag, + verbose_level) + + except Exception: + raise Exception(traceback.format_exc()) + + +class Setdeploy(Command): + """Set deploy mode + + Set deploy mode to either local or remote. Local indicates + that the openstack deployment will be to the local host. + Remote means that the deployment is on remote hosts. + """ + def get_parser(self, prog_name): + parser = super(Setdeploy, self).get_parser(prog_name) + parser.add_argument('mode', metavar='', + help=u._('mode=')) + return parser + + def take_action(self, parsed_args): + try: + mode = parsed_args.mode.strip() + remote_flag = False + if mode == 'remote': + remote_flag = True + elif mode != 'local': + raise CommandError( + u._('Invalid deploy mode. Mode must be ' + 'either "local" or "remote".')) + inventory = Inventory.load() + inventory.set_deploy_mode(remote_flag) + Inventory.save(inventory) + except CommandError as e: + raise e + except Exception: + raise Exception(traceback.format_exc()) diff --git a/kollacli/group.py b/kollacli/commands/group.py similarity index 91% rename from kollacli/group.py rename to kollacli/commands/group.py index e7d66a1..1f588f6 100644 --- a/kollacli/group.py +++ b/kollacli/commands/group.py @@ -16,8 +16,8 @@ import traceback import kollacli.i18n as u from kollacli.common.inventory import Inventory +from kollacli.common.utils import convert_to_unicode from kollacli.exceptions import CommandError -from kollacli import utils from cliff.command import Command from cliff.lister import Lister @@ -34,7 +34,7 @@ class GroupAdd(Command): def take_action(self, parsed_args): try: groupname = parsed_args.groupname.strip() - groupname = utils.convert_to_unicode(groupname) + groupname = convert_to_unicode(groupname) inventory = Inventory.load() inventory.add_group(groupname) @@ -57,7 +57,7 @@ class GroupRemove(Command): def take_action(self, parsed_args): try: groupname = parsed_args.groupname.strip() - groupname = utils.convert_to_unicode(groupname) + groupname = convert_to_unicode(groupname) inventory = Inventory.load() inventory.remove_group(groupname) Inventory.save(inventory) @@ -80,9 +80,9 @@ class GroupAddhost(Command): def take_action(self, parsed_args): try: groupname = parsed_args.groupname.strip() - groupname = utils.convert_to_unicode(groupname) + groupname = convert_to_unicode(groupname) hostname = parsed_args.hostname.strip() - hostname = utils.convert_to_unicode(hostname) + hostname = convert_to_unicode(hostname) inventory = Inventory.load() inventory.add_host(hostname, groupname) Inventory.save(inventory) @@ -106,9 +106,9 @@ class GroupRemovehost(Command): def take_action(self, parsed_args): try: groupname = parsed_args.groupname.strip() - groupname = utils.convert_to_unicode(groupname) + groupname = convert_to_unicode(groupname) hostname = parsed_args.hostname.strip() - hostname = utils.convert_to_unicode(hostname) + hostname = convert_to_unicode(hostname) inventory = Inventory.load() inventory.remove_host(hostname, groupname) @@ -153,9 +153,9 @@ class GroupAddservice(Command): def take_action(self, parsed_args): try: groupname = parsed_args.groupname.strip() - groupname = utils.convert_to_unicode(groupname) + groupname = convert_to_unicode(groupname) servicename = parsed_args.servicename.strip() - servicename = utils.convert_to_unicode(servicename) + servicename = convert_to_unicode(servicename) inventory = Inventory.load() inventory.add_group_to_service(groupname, servicename) @@ -180,9 +180,9 @@ class GroupRemoveservice(Command): def take_action(self, parsed_args): try: groupname = parsed_args.groupname.strip() - groupname = utils.convert_to_unicode(groupname) + groupname = convert_to_unicode(groupname) servicename = parsed_args.servicename.strip() - servicename = utils.convert_to_unicode(servicename) + servicename = convert_to_unicode(servicename) inventory = Inventory.load() inventory.remove_group_from_service(groupname, servicename) diff --git a/kollacli/host.py b/kollacli/commands/host.py similarity index 98% rename from kollacli/host.py rename to kollacli/commands/host.py index 79368b5..6836a1b 100644 --- a/kollacli/host.py +++ b/kollacli/commands/host.py @@ -22,9 +22,9 @@ import kollacli.i18n as u from kollacli.common.ansible.actions import destroy_hosts from kollacli.common.inventory import Inventory +from kollacli.common.utils import convert_to_unicode +from kollacli.common.utils import get_setup_user from kollacli.exceptions import CommandError -from kollacli.utils import convert_to_unicode -from kollacli.utils import get_setup_user from cliff.command import Command from cliff.lister import Lister diff --git a/kollacli/password.py b/kollacli/commands/password.py similarity index 100% rename from kollacli/password.py rename to kollacli/commands/password.py diff --git a/kollacli/property.py b/kollacli/commands/property.py similarity index 100% rename from kollacli/property.py rename to kollacli/commands/property.py diff --git a/kollacli/service.py b/kollacli/commands/service.py similarity index 94% rename from kollacli/service.py rename to kollacli/commands/service.py index 54620c8..bdb2a04 100644 --- a/kollacli/service.py +++ b/kollacli/commands/service.py @@ -16,8 +16,8 @@ import traceback import kollacli.i18n as u from kollacli.common.inventory import Inventory +from kollacli.common.utils import convert_to_unicode from kollacli.exceptions import CommandError -from kollacli import utils from cliff.command import Command from cliff.lister import Lister @@ -41,9 +41,9 @@ class ServiceAddGroup(Command): def take_action(self, parsed_args): try: groupname = parsed_args.groupname.strip() - groupname = utils.convert_to_unicode(groupname) + groupname = convert_to_unicode(groupname) servicename = parsed_args.servicename.strip() - servicename = utils.convert_to_unicode(servicename) + servicename = convert_to_unicode(servicename) inventory = Inventory.load() @@ -70,9 +70,9 @@ class ServiceRemoveGroup(Command): def take_action(self, parsed_args): try: groupname = parsed_args.groupname.strip() - groupname = utils.convert_to_unicode(groupname) + groupname = convert_to_unicode(groupname) servicename = parsed_args.servicename.strip() - servicename = utils.convert_to_unicode(servicename) + servicename = convert_to_unicode(servicename) inventory = Inventory.load() diff --git a/kollacli/misc.py b/kollacli/commands/support.py similarity index 58% rename from kollacli/misc.py rename to kollacli/commands/support.py index 99b6925..51b8ea1 100644 --- a/kollacli/misc.py +++ b/kollacli/commands/support.py @@ -19,64 +19,18 @@ import traceback import kollacli.i18n as u -from kollacli.common.ansible.actions import deploy from kollacli.common.inventory import Inventory -from kollacli.exceptions import CommandError -from kollacli.utils import convert_to_unicode -from kollacli.utils import get_kolla_etc -from kollacli.utils import get_kolla_home -from kollacli.utils import get_kolla_log_dir -from kollacli.utils import get_kollacli_etc -from kollacli.utils import run_cmd +from kollacli.common.utils import get_kolla_etc +from kollacli.common.utils import get_kolla_home +from kollacli.common.utils import get_kolla_log_dir +from kollacli.common.utils import get_kollacli_etc +from kollacli.common.utils import run_cmd from cliff.command import Command LOG = logging.getLogger(__name__) -class Deploy(Command): - """Deploy""" - def get_parser(self, prog_name): - parser = super(Deploy, self).get_parser(prog_name) - parser.add_argument('--hosts', nargs='?', - metavar='', - help=u._('Deployment host list')) - parser.add_argument('--groups', nargs='?', - metavar='', - help=u._('Deployment group list')) - parser.add_argument('--services', nargs='?', - metavar='', - help=u._('Deployment service list')) - parser.add_argument('--serial', action='store_true', - help=u._('Deploy serially')) - return parser - - def take_action(self, parsed_args): - hosts = None - groups = None - services = None - serial_flag = False - verbose_level = self.app.options.verbose_level - try: - if parsed_args.hosts: - host_list = parsed_args.hosts.strip() - hosts = convert_to_unicode(host_list).split(',') - if parsed_args.groups: - group_list = parsed_args.groups.strip() - groups = convert_to_unicode(group_list).split(',') - if parsed_args.services: - service_list = parsed_args.services.strip() - services = convert_to_unicode(service_list).split(',') - if parsed_args.serial: - serial_flag = True - - deploy(hosts, groups, services, serial_flag, - verbose_level) - - except Exception: - raise Exception(traceback.format_exc()) - - class Dump(Command): """Dumps configuration data for debugging @@ -165,35 +119,3 @@ class Dump(Command): if inv_path: os.remove(inv_path) return - - -class Setdeploy(Command): - """Set deploy mode - - Set deploy mode to either local or remote. Local indicates - that the openstack deployment will be to the local host. - Remote means that the deployment is on remote hosts. - """ - def get_parser(self, prog_name): - parser = super(Setdeploy, self).get_parser(prog_name) - parser.add_argument('mode', metavar='', - help=u._('mode=')) - return parser - - def take_action(self, parsed_args): - try: - mode = parsed_args.mode.strip() - remote_flag = False - if mode == 'remote': - remote_flag = True - elif mode != 'local': - raise CommandError( - u._('Invalid deploy mode. Mode must be ' - 'either "local" or "remote".')) - inventory = Inventory.load() - inventory.set_deploy_mode(remote_flag) - Inventory.save(inventory) - except CommandError as e: - raise e - except Exception: - raise Exception(traceback.format_exc()) diff --git a/kollacli/common/ansible/actions.py b/kollacli/common/ansible/actions.py index e12a14f..3aab05a 100644 --- a/kollacli/common/ansible/actions.py +++ b/kollacli/common/ansible/actions.py @@ -19,10 +19,10 @@ import kollacli.i18n as u from kollacli.common.ansible.playbook import AnsiblePlaybook from kollacli.common import properties from kollacli.common.properties import AnsibleProperties +from kollacli.common.utils import get_kolla_etc +from kollacli.common.utils import get_kolla_home +from kollacli.common.utils import get_kollacli_home from kollacli.exceptions import CommandError -from kollacli.utils import get_kolla_etc -from kollacli.utils import get_kolla_home -from kollacli.utils import get_kollacli_home LOG = logging.getLogger(__name__) diff --git a/kollacli/common/ansible/playbook.py b/kollacli/common/ansible/playbook.py index 04c7f3d..ebc8e11 100644 --- a/kollacli/common/ansible/playbook.py +++ b/kollacli/common/ansible/playbook.py @@ -18,11 +18,11 @@ import traceback import kollacli.i18n as u +from kollacli.common.utils import get_admin_user +from kollacli.common.utils import get_ansible_command +from kollacli.common.utils import get_kolla_etc +from kollacli.common.utils import run_cmd from kollacli.exceptions import CommandError -from kollacli.utils import get_admin_user -from kollacli.utils import get_ansible_command -from kollacli.utils import get_kolla_etc -from kollacli.utils import run_cmd from kollacli.common.inventory import Inventory diff --git a/kollacli/common/inventory.py b/kollacli/common/inventory.py index 1aa3a7d..d9aaad6 100644 --- a/kollacli/common/inventory.py +++ b/kollacli/common/inventory.py @@ -20,13 +20,15 @@ import traceback import kollacli.i18n as u -from kollacli import exceptions -from kollacli import utils +from kollacli.common.sshutils import ssh_setup_host +from kollacli.common.utils import get_admin_user +from kollacli.common.utils import get_ansible_command +from kollacli.common.utils import get_kollacli_etc +from kollacli.common.utils import run_cmd +from kollacli.common.utils import sync_read_file +from kollacli.common.utils import sync_write_file from kollacli.exceptions import CommandError -from kollacli.sshutils import ssh_setup_host -from kollacli.utils import get_admin_user -from kollacli.utils import get_ansible_command ANSIBLE_SSH_USER = 'ansible_ssh_user' ANSIBLE_CONNECTION = 'ansible_connection' @@ -158,7 +160,7 @@ class HostGroup(object): self.set_var(ANSIBLE_BECOME, 'yes') if remote_flag: # set the ssh info for all the servers in the group - self.set_var(ANSIBLE_SSH_USER, utils.get_admin_user()) + self.set_var(ANSIBLE_SSH_USER, get_admin_user()) self.clear_var(ANSIBLE_CONNECTION) else: # remove ssh info, add local connection type @@ -279,11 +281,11 @@ class Inventory(object): @staticmethod def load(): """load the inventory from a pickle file""" - inventory_path = os.path.join(utils.get_kollacli_etc(), INVENTORY_PATH) + inventory_path = os.path.join(get_kollacli_etc(), INVENTORY_PATH) data = '' try: if os.path.exists(inventory_path): - data = utils.sync_read_file(inventory_path) + data = sync_read_file(inventory_path) if data.strip(): inventory = jsonpickle.decode(data) @@ -302,13 +304,13 @@ class Inventory(object): @staticmethod def save(inventory): """Save the inventory in a pickle file""" - inventory_path = os.path.join(utils.get_kollacli_etc(), INVENTORY_PATH) + inventory_path = os.path.join(get_kollacli_etc(), INVENTORY_PATH) try: # multiple trips thru json to render a readable inventory file data = jsonpickle.encode(inventory) data_str = json.loads(data) pretty_data = json.dumps(data_str, indent=4) - utils.sync_write_file(inventory_path, pretty_data) + sync_write_file(inventory_path, pretty_data) except Exception as e: raise CommandError( @@ -454,7 +456,7 @@ class Inventory(object): self.log.info(u._LI('Host ({host}) setup succeeded.') .format(host=hostname)) except Exception as e: - raise exceptions.CommandError( + raise CommandError( u._('Host ({host}) setup failed : {error}') .format(host=hostname, error=str(e))) return True @@ -469,7 +471,7 @@ class Inventory(object): inventory_string = '-i ' + gen_file_path ping_string = ' %s %s' % (hostname, '-m ping') cmd = (command_string + inventory_string + ping_string) - err_msg, output = utils.run_cmd(cmd, False) + err_msg, output = run_cmd(cmd, False) except Exception as e: raise e finally: @@ -479,7 +481,7 @@ class Inventory(object): if result_only: return False else: - raise exceptions.CommandError( + raise CommandError( u._('Host ({host}) check failed. : {error} {message}') .format(host=hostname, error=err_msg, message=output)) else: diff --git a/kollacli/common/passwords.py b/kollacli/common/passwords.py index cd39d9e..d74b2ed 100644 --- a/kollacli/common/passwords.py +++ b/kollacli/common/passwords.py @@ -15,8 +15,8 @@ import os import kollacli.i18n as u +from kollacli.common import utils from kollacli.exceptions import CommandError -from kollacli import utils PWDS_FILENAME = 'passwords.yml' PWD_EDITOR_FILENAME = 'passwd_editor.py' diff --git a/kollacli/common/properties.py b/kollacli/common/properties.py index 19fb103..599ecc3 100644 --- a/kollacli/common/properties.py +++ b/kollacli/common/properties.py @@ -16,10 +16,10 @@ import os import six import yaml -from kollacli.utils import change_property -from kollacli.utils import get_kolla_etc -from kollacli.utils import get_kolla_home -from kollacli.utils import sync_read_file +from kollacli.common.utils import change_property +from kollacli.common.utils import get_kolla_etc +from kollacli.common.utils import get_kolla_home +from kollacli.common.utils import sync_read_file ALLVARS_PATH = 'ansible/group_vars/all.yml' GLOBALS_FILENAME = 'globals.yml' diff --git a/kollacli/sshutils.py b/kollacli/common/sshutils.py similarity index 95% rename from kollacli/sshutils.py rename to kollacli/common/sshutils.py index 7e2f59a..47f9553 100644 --- a/kollacli/sshutils.py +++ b/kollacli/common/sshutils.py @@ -16,11 +16,12 @@ import os.path import paramiko import traceback +from kollacli.common.utils import get_admin_user +from kollacli.common.utils import get_kollacli_etc +from kollacli.common.utils import get_setup_user + import kollacli.i18n as u -from kollacli.utils import get_admin_user -from kollacli.utils import get_kollacli_etc -from kollacli.utils import get_setup_user MIN_DOCKER_VERSION = '1.8.1' diff --git a/kollacli/utils.py b/kollacli/common/utils.py similarity index 100% rename from kollacli/utils.py rename to kollacli/common/utils.py diff --git a/kollacli/shell.py b/kollacli/shell.py index 37b6d6e..b43efa1 100755 --- a/kollacli/shell.py +++ b/kollacli/shell.py @@ -22,10 +22,10 @@ from cliff.commandmanager import CommandManager import kollacli.i18n as u from kollacli.common.inventory import INVENTORY_PATH +from kollacli.common.utils import get_kolla_log_dir +from kollacli.common.utils import get_kolla_log_file_size +from kollacli.common.utils import get_kollacli_etc from kollacli.exceptions import CommandError -from kollacli.utils import get_kolla_log_dir -from kollacli.utils import get_kolla_log_file_size -from kollacli.utils import get_kollacli_etc LOG = logging.getLogger(__name__) diff --git a/setup.cfg b/setup.cfg index bdd9278..0878ea3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,31 +31,31 @@ console_scripts = kollacli = kollacli.shell:main kolla.cli = - deploy = kollacli.misc:Deploy - dump = kollacli.misc:Dump - group_add = kollacli.group:GroupAdd - group_addhost = kollacli.group:GroupAddhost - group_listhosts = kollacli.group:GroupListhosts - group_listservices = kollacli.group:GroupListservices - group_remove = kollacli.group:GroupRemove - group_removehost = kollacli.group:GroupRemovehost - host_add = kollacli.host:HostAdd - host_check = kollacli.host:HostCheck - host_destroy = kollacli.host:HostDestroy - host_list = kollacli.host:HostList - host_remove = kollacli.host:HostRemove - host_setup = kollacli.host:HostSetup - password_clear = kollacli.password:PasswordClear - password_list = kollacli.password:PasswordList - password_set = kollacli.password:PasswordSet - property_clear = kollacli.property:PropertyClear - property_list = kollacli.property:PropertyList - property_set = kollacli.property:PropertySet - service_addgroup = kollacli.service:ServiceAddGroup - service_list = kollacli.service:ServiceList - service_listgroups = kollacli.service:ServiceListGroups - service_removegroup = kollacli.service:ServiceRemoveGroup - setdeploy = kollacli.misc:Setdeploy + deploy = kollacli.commands.deploy:Deploy + dump = kollacli.commands.support:Dump + group_add = kollacli.commands.group:GroupAdd + group_addhost = kollacli.commands.group:GroupAddhost + group_listhosts = kollacli.commands.group:GroupListhosts + group_listservices = kollacli.commands.group:GroupListservices + group_remove = kollacli.commands.group:GroupRemove + group_removehost = kollacli.commands.group:GroupRemovehost + host_add = kollacli.commands.host:HostAdd + host_check = kollacli.commands.host:HostCheck + host_destroy = kollacli.commands.host:HostDestroy + host_list = kollacli.commands.host:HostList + host_remove = kollacli.commands.host:HostRemove + host_setup = kollacli.commands.host:HostSetup + password_clear = kollacli.commands.password:PasswordClear + password_list = kollacli.commands.password:PasswordList + password_set = kollacli.commands.password:PasswordSet + property_clear = kollacli.commands.property:PropertyClear + property_list = kollacli.commands.property:PropertyList + property_set = kollacli.commands.property:PropertySet + service_addgroup = kollacli.commands.service:ServiceAddGroup + service_list = kollacli.commands.service:ServiceList + service_listgroups = kollacli.commands.service:ServiceListGroups + service_removegroup = kollacli.commands.service:ServiceRemoveGroup + setdeploy = kollacli.commands.deploy:Setdeploy [extract_messages] keywords = _ gettext ngettext l_ lazy_gettext diff --git a/tests/common.py b/tests/common.py index 0340a03..d49694c 100644 --- a/tests/common.py +++ b/tests/common.py @@ -23,7 +23,7 @@ import yaml from oslo_utils.encodeutils import safe_decode -import kollacli.utils as utils +import kollacli.common.utils as utils TEST_SUFFIX = 'test/' VENV_PY_PATH = '.venv/bin/python' diff --git a/tests/password.py b/tests/password.py index 016aed0..9050410 100644 --- a/tests/password.py +++ b/tests/password.py @@ -16,7 +16,7 @@ from common import KollaCliTest import os import unittest -from kollacli.utils import get_kolla_etc +from kollacli.common.utils import get_kolla_etc class TestFunctional(KollaCliTest): diff --git a/tests/property.py b/tests/property.py index 144ee29..ee936dc 100644 --- a/tests/property.py +++ b/tests/property.py @@ -17,7 +17,7 @@ from common import KollaCliTest import os import unittest -from kollacli.utils import get_kolla_etc +from kollacli.common.utils import get_kolla_etc class TestFunctional(KollaCliTest): diff --git a/tools/passwd_editor.py b/tools/passwd_editor.py index f982c92..e64067f 100755 --- a/tools/passwd_editor.py +++ b/tools/passwd_editor.py @@ -15,13 +15,14 @@ import getopt import sys -from kollacli import utils +from kollacli.common.utils import change_property +from kollacli.common.utils import sync_read_file def _print_pwd_keys(path): pwd_keys = '' prefix = '' - pwd_data = utils.sync_read_file(path) + pwd_data = sync_read_file(path) for line in pwd_data.split('\n'): if line.startswith('#'): # skip commented lines @@ -67,7 +68,7 @@ def main(): _print_pwd_keys(path) else: # edit a password - utils.change_property(path, pwd_key, pwd_value, clear_flag) + change_property(path, pwd_key, pwd_value, clear_flag) if __name__ == '__main__': From 72db18d96781c41a617ca2a3332dc02cdeb37f97 Mon Sep 17 00:00:00 2001 From: Steve Noyes Date: Wed, 25 Nov 2015 12:51:45 -0500 Subject: [PATCH 22/25] move dump logic out of cli --- kollacli/commands/support.py | 98 +------------------------- kollacli/common/support.py | 129 +++++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+), 96 deletions(-) create mode 100644 kollacli/common/support.py diff --git a/kollacli/commands/support.py b/kollacli/commands/support.py index 51b8ea1..21c8caf 100644 --- a/kollacli/commands/support.py +++ b/kollacli/commands/support.py @@ -11,25 +11,10 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -import logging -import os -import tarfile -import tempfile -import traceback - -import kollacli.i18n as u - -from kollacli.common.inventory import Inventory -from kollacli.common.utils import get_kolla_etc -from kollacli.common.utils import get_kolla_home -from kollacli.common.utils import get_kolla_log_dir -from kollacli.common.utils import get_kollacli_etc -from kollacli.common.utils import run_cmd +from kollacli.common.support import dump from cliff.command import Command -LOG = logging.getLogger(__name__) - class Dump(Command): """Dumps configuration data for debugging @@ -39,83 +24,4 @@ class Dump(Command): debugging problems. """ def take_action(self, parsed_args): - try: - kolla_home = get_kolla_home() - kolla_logs = get_kolla_log_dir() - kolla_ansible = os.path.join(kolla_home, 'ansible') - kolla_docs = os.path.join(kolla_home, 'docs') - kolla_etc = get_kolla_etc() - kolla_config = os.path.join(kolla_etc, 'config') - kolla_globals = os.path.join(kolla_etc, 'globals.yml') - kollacli_etc = get_kollacli_etc().rstrip('/') - ketc = 'kolla/etc/' - kshare = 'kolla/share/' - fd, dump_path = tempfile.mkstemp(prefix='kollacli_dump_', - suffix='.tgz') - os.close(fd) # avoid fd leak - with tarfile.open(dump_path, 'w:gz') as tar: - # Can't blanket add kolla_home because the .ssh dir is - # accessible by the kolla user only (not kolla group) - tar.add(kolla_ansible, - arcname=kshare + os.path.basename(kolla_ansible)) - tar.add(kolla_docs, - arcname=kshare + os.path.basename(kolla_docs)) - - # Can't blanket add kolla_etc because the passwords.yml - # file is accessible by the kolla user only (not kolla group) - tar.add(kolla_config, - arcname=ketc + os.path.basename(kolla_config)) - tar.add(kolla_globals, - arcname=ketc + os.path.basename(kolla_globals)) - tar.add(kollacli_etc, - arcname=ketc + os.path.basename(kollacli_etc)) - - # add kolla log files - if os.path.isdir(kolla_logs): - tar.add(kolla_logs) - - # add output of various commands - self._add_cmd_info(tar) - - LOG.info( - u._LI('dump successful to {path}').format(path=dump_path)) - except Exception: - raise Exception(traceback.format_exc()) - - def _add_cmd_info(self, tar): - # run all the kollacli list commands - cmds = ['kollacli --version', - 'kollacli service listgroups', - 'kollacli service list', - 'kollacli group listservices', - 'kollacli group listhosts', - 'kollacli host list', - 'kollacli property list', - 'kollacli password list'] - - # collect the json inventory output - inventory = Inventory.load() - inv_path = inventory.create_json_gen_file() - cmds.append(inv_path) - - try: - fd, path = tempfile.mkstemp(suffix='.tmp') - os.close(fd) - with open(path, 'w') as tmp_file: - for cmd in cmds: - err_msg, output = run_cmd(cmd, False) - tmp_file.write('\n\n$ %s\n' % cmd) - if err_msg: - tmp_file.write('Error message: %s\n' % err_msg) - for line in output: - tmp_file.write(line + '\n') - - tar.add(path, arcname=os.path.join('kolla', 'cmds_output')) - except Exception as e: - raise e - finally: - if path: - os.remove(path) - if inv_path: - os.remove(inv_path) - return + dump() diff --git a/kollacli/common/support.py b/kollacli/common/support.py new file mode 100644 index 0000000..617b63b --- /dev/null +++ b/kollacli/common/support.py @@ -0,0 +1,129 @@ +# Copyright(c) 2015, Oracle and/or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import logging +import os +import tarfile +import tempfile +import traceback + +import kollacli.i18n as u + +from kollacli.common.inventory import Inventory +from kollacli.common.utils import get_kolla_etc +from kollacli.common.utils import get_kolla_home +from kollacli.common.utils import get_kolla_log_dir +from kollacli.common.utils import get_kollacli_etc +from kollacli.common.utils import run_cmd + +LOG = logging.getLogger(__name__) + + +def dump(): + """Dumps configuration data for debugging + + Dumps most files in /etc/kolla and /usr/share/kolla into a + tar file so be given to support / development to help with + debugging problems. + """ + try: + msg = None + return_code = 0 + kolla_home = get_kolla_home() + kolla_logs = get_kolla_log_dir() + kolla_ansible = os.path.join(kolla_home, 'ansible') + kolla_docs = os.path.join(kolla_home, 'docs') + kolla_etc = get_kolla_etc() + kolla_config = os.path.join(kolla_etc, 'config') + kolla_globals = os.path.join(kolla_etc, 'globals.yml') + kollacli_etc = get_kollacli_etc().rstrip('/') + ketc = 'kolla/etc/' + kshare = 'kolla/share/' + fd, dump_path = tempfile.mkstemp(prefix='kollacli_dump_', + suffix='.tgz') + os.close(fd) # avoid fd leak + with tarfile.open(dump_path, 'w:gz') as tar: + # Can't blanket add kolla_home because the .ssh dir is + # accessible by the kolla user only (not kolla group) + tar.add(kolla_ansible, + arcname=kshare + os.path.basename(kolla_ansible)) + tar.add(kolla_docs, + arcname=kshare + os.path.basename(kolla_docs)) + + # Can't blanket add kolla_etc because the passwords.yml + # file is accessible by the kolla user only (not kolla group) + tar.add(kolla_config, + arcname=ketc + os.path.basename(kolla_config)) + tar.add(kolla_globals, + arcname=ketc + os.path.basename(kolla_globals)) + tar.add(kollacli_etc, + arcname=ketc + os.path.basename(kollacli_etc)) + + # add kolla log files + if os.path.isdir(kolla_logs): + tar.add(kolla_logs) + + # add output of various commands + _add_cmd_info(tar) + + msg = u._LI('dump successful to {path}').format(path=dump_path) + LOG.info(msg) + + except Exception: + msg = (u._LI('dump failed: {reason}') + .format(reason=traceback.format_exc())) + LOG.error(msg) + return_code = -1 + + return return_code, msg + + +def _add_cmd_info(tar): + # run all the kollacli list commands + cmds = ['kollacli --version', + 'kollacli service listgroups', + 'kollacli service list', + 'kollacli group listservices', + 'kollacli group listhosts', + 'kollacli host list', + 'kollacli property list', + 'kollacli password list'] + + # collect the json inventory output + inventory = Inventory.load() + inv_path = inventory.create_json_gen_file() + cmds.append(inv_path) + + try: + fd, path = tempfile.mkstemp(suffix='.tmp') + os.close(fd) + with open(path, 'w') as tmp_file: + for cmd in cmds: + err_msg, output = run_cmd(cmd, False) + tmp_file.write('\n\n$ %s\n' % cmd) + if err_msg: + tmp_file.write('Error message: %s\n' % err_msg) + for line in output: + tmp_file.write(line + '\n') + + tar.add(path, arcname=os.path.join('kolla', 'cmds_output')) + except Exception as e: + raise e + finally: + if path: + os.remove(path) + if inv_path: + os.remove(inv_path) + return From c466e2bad7b60eaa1368daa3b2f3961531bcb1d3 Mon Sep 17 00:00:00 2001 From: Borne Mace Date: Tue, 8 Dec 2015 11:57:19 -0800 Subject: [PATCH 23/25] Forward port of data preservation for host destroy Jira-Issue: OPENSTACK-533 --- ansible/host_destroy.yml | 2 +- ansible/host_destroy_no_data.yml | 9 ++++++++ kollacli/commands/host.py | 7 +++++- kollacli/common/ansible/actions.py | 8 +++++-- tests/destroy.py | 37 +++++++++++++++++++++++++++--- 5 files changed, 56 insertions(+), 7 deletions(-) create mode 100644 ansible/host_destroy_no_data.yml diff --git a/ansible/host_destroy.yml b/ansible/host_destroy.yml index 786b6a7..1e97725 100644 --- a/ansible/host_destroy.yml +++ b/ansible/host_destroy.yml @@ -1,7 +1,7 @@ --- - hosts: '{{ hosts }}' tasks: - - name: destroy kolla containers + - name: destroy all kolla containers shell: '{{ item }}' ignore_errors: yes with_items: diff --git a/ansible/host_destroy_no_data.yml b/ansible/host_destroy_no_data.yml new file mode 100644 index 0000000..9f049aa --- /dev/null +++ b/ansible/host_destroy_no_data.yml @@ -0,0 +1,9 @@ +--- +- hosts: '{{ hosts }}' + tasks: + - name: destroy non-data kolla containers + shell: '{{ item }}' + ignore_errors: yes + with_items: + - docker '{{ destroy_type | default("kill") }}' $(docker ps -a | grep '{{ prefix }}' | grep -v '\-data' | awk '{print $1}') + - docker rm $(docker ps -a | grep '{{ prefix }}' | grep -v '\-data' | awk '{print $1}') diff --git a/kollacli/commands/host.py b/kollacli/commands/host.py index 6836a1b..abf162b 100644 --- a/kollacli/commands/host.py +++ b/kollacli/commands/host.py @@ -74,6 +74,8 @@ class HostDestroy(Command): help=u._('Host name or ip address or "all"')) parser.add_argument('--stop', action='store_true', help=u._('Stop rather than kill')) + parser.add_argument('--includedata', action='store_true', + help=u._('Destroy data containers')) return parser def take_action(self, parsed_args): @@ -85,10 +87,13 @@ class HostDestroy(Command): destroy_type = 'kill' if parsed_args.stop: destroy_type = 'stop' + include_data = False + if parsed_args.includedata: + include_data = True verbose_level = self.app.options.verbose_level - destroy_hosts(hostname, destroy_type, verbose_level) + destroy_hosts(hostname, destroy_type, verbose_level, include_data) except CommandError as e: raise e diff --git a/kollacli/common/ansible/actions.py b/kollacli/common/ansible/actions.py index 3aab05a..242af3d 100644 --- a/kollacli/common/ansible/actions.py +++ b/kollacli/common/ansible/actions.py @@ -27,7 +27,7 @@ from kollacli.exceptions import CommandError LOG = logging.getLogger(__name__) -def destroy_hosts(hostname, destroy_type, verbose_level=1): +def destroy_hosts(hostname, destroy_type, verbose_level=1, include_data=False): '''destroy containers on a host (or all hosts). If hostname == 'all', then containers on all hosts will be @@ -41,6 +41,10 @@ def destroy_hosts(hostname, destroy_type, verbose_level=1): u._('Invalid destroy type ({type}). Must be either ' '"stop" or "kill".').format(type=destroy_type)) + playbook_name = 'host_destroy_no_data.yml' + if include_data: + playbook_name = 'host_destroy.yml' + LOG.info(u._LI('Please be patient as this may take a while.')) ansible_properties = properties.AnsibleProperties() base_distro = \ @@ -51,7 +55,7 @@ def destroy_hosts(hostname, destroy_type, verbose_level=1): kollacli_home = get_kollacli_home() playbook = AnsiblePlaybook() playbook.playbook_path = os.path.join(kollacli_home, - 'ansible/host_destroy.yml') + 'ansible/' + playbook_name) playbook.extra_vars = 'hosts=' + hostname + \ ' prefix=' + container_prefix + \ ' destroy_type=' + destroy_type diff --git a/tests/destroy.py b/tests/destroy.py index 33286b4..a845863 100644 --- a/tests/destroy.py +++ b/tests/destroy.py @@ -24,6 +24,9 @@ DISABLED_SERVICES = [ ENABLED_SERVICES = [ 'rabbitmq' ] +ENABLED_DATA_SERVICES = [ + 'rabbitmq_data' + ] UNKNOWN_HOST = 'Name or service not known' @@ -67,7 +70,7 @@ class TestFunctional(KollaCliTest): # destroy services, initialize server try: - self.run_cli_cmd('host destroy %s' % hostname) + self.run_cli_cmd('host destroy %s --includedata' % hostname) except Exception as e: self.assertFalse(is_physical_host, '1st destroy exception: %s' % e) self.assertIn(UNKNOWN_HOST, '%s' % e, @@ -108,7 +111,8 @@ class TestFunctional(KollaCliTest): 'is not running on host: %s ' % hostname + 'after deploy.') - # destroy services (via --stop flag) + # destroy non-data services (via --stop flag) + # this should leave only data containers running try: self.run_cli_cmd('host destroy %s --stop' % hostname) except Exception as e: @@ -124,10 +128,37 @@ class TestFunctional(KollaCliTest): 'is running on host: %s ' % hostname + 'after destroy.') + for enabled_service in ENABLED_DATA_SERVICES: + self.assertIn(enabled_service, docker_ps, + 'enabled service: %s ' % enabled_service + + 'is not running on host: %s ' % hostname + + 'after no-data destroy.') + + try: + self.run_cli_cmd('host destroy %s --includedata --stop' % hostname) + except Exception as e: + self.assertFalse(is_physical_host, '3rd destroy exception: %s' % e) + self.assertIn(UNKNOWN_HOST, '%s' % e, + 'Unexpected exception in 3rd destroy: %s' % e) + + if is_physical_host: + docker_ps = test_config.run_remote_cmd('docker ps', hostname) + for disabled_service in DISABLED_SERVICES: + self.assertNotIn(disabled_service, docker_ps, + 'disabled service: %s ' % disabled_service + + 'is running on host: %s ' % hostname + + 'after destroy.') + + for enabled_service in ENABLED_DATA_SERVICES: + self.assertNotIn(enabled_service, docker_ps, + 'enabled service: %s ' % enabled_service + + 'is running on host: %s ' % hostname + + 'after destroy.') + for enabled_service in ENABLED_SERVICES: self.assertNotIn(enabled_service, docker_ps, 'enabled service: %s ' % enabled_service + - 'is still running on host: %s ' % hostname + + 'is running on host: %s ' % hostname + 'after destroy.') def tearDown(self): From 5ff4aaaaed0c915a601eb72f3424e1964c0f503e Mon Sep 17 00:00:00 2001 From: Steve Noyes Date: Fri, 11 Dec 2015 11:26:56 -0500 Subject: [PATCH 24/25] Jira-Issue:OPENSTACK-545 fix order of hosts in json inventory generator --- kollacli/common/inventory.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/kollacli/common/inventory.py b/kollacli/common/inventory.py index d9aaad6..f88fb7c 100644 --- a/kollacli/common/inventory.py +++ b/kollacli/common/inventory.py @@ -780,10 +780,13 @@ class Inventory(object): return json.dumps(jdict) def _filter_hosts(self, initial_hostnames, deploy_hostnames): - """filter out hosts not in deploy hosts""" + """filter out hosts not in deploy hosts + + Must preserve the ordering of hosts in the group. + """ filtered_hostnames = [] - for hostname in deploy_hostnames: - if hostname in initial_hostnames: + for hostname in initial_hostnames: + if hostname in deploy_hostnames: filtered_hostnames.append(hostname) return filtered_hostnames From 5995838d10a02b1c93b83b1b3f4cb4235854ab48 Mon Sep 17 00:00:00 2001 From: Borne Mace Date: Tue, 15 Dec 2015 15:33:58 -0800 Subject: [PATCH 25/25] Changed default property values shown and added --all and --long flags. Jira-Issue: OPENSTACK-558 --- kollacli/commands/property.py | 52 +++++++++++++++++++++++++++++++---- kollacli/common/properties.py | 27 ++++++++++++++---- kollacli/common/utils.py | 13 +++++++++ requirements.txt | 2 +- 4 files changed, 82 insertions(+), 12 deletions(-) diff --git a/kollacli/commands/property.py b/kollacli/commands/property.py index 8fd08f7..f303d6e 100644 --- a/kollacli/commands/property.py +++ b/kollacli/commands/property.py @@ -11,15 +11,19 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +import logging import traceback import kollacli.i18n as u from kollacli.common import properties +from kollacli.common import utils from cliff.command import Command from cliff.lister import Lister +LOG = logging.getLogger(__name__) + class PropertySet(Command): "Property Set" @@ -65,14 +69,52 @@ class PropertyClear(Command): class PropertyList(Lister): """List all properties""" + def get_parser(self, prog_name): + parser = super(PropertyList, self).get_parser(prog_name) + parser.add_argument('--all', action='store_true', + help=u._('List all properties')) + parser.add_argument('--long', action='store_true', + help=u._('Show all property attributes')) + return parser + def take_action(self, parsed_args): ansible_properties = properties.AnsibleProperties() property_list = ansible_properties.get_all_unique() + + list_all = False + if parsed_args.all: + list_all = True + + list_long = False + if parsed_args.long: + list_long = True + + property_length = utils.get_property_list_length() data = [] if property_list: - for value in property_list: - data.append((value.name, value.value)) - else: - data.append(('', '')) + for prop in property_list: + include_prop = False + if (prop.value is not None and + len(str(prop.value)) > property_length): + if list_all: + include_prop = True + else: + include_prop = True - return ((u._('Property Name'), u._('Property Value')), data) + if include_prop: + if list_long: + data.append((prop.name, prop.value, prop.overrides, + prop.orig_value)) + else: + data.append((prop.name, prop.value)) + else: + if list_long: + data.append(('', '', '', '')) + else: + data.append(('', '')) + + if list_long: + return ((u._('Property Name'), u._('Property Value'), + u._('Overrides'), u._('Original Value')), data) + else: + return ((u._('Property Name'), u._('Property Value')), data) diff --git a/kollacli/common/properties.py b/kollacli/common/properties.py index 599ecc3..c86f633 100644 --- a/kollacli/common/properties.py +++ b/kollacli/common/properties.py @@ -61,7 +61,6 @@ class AnsibleProperties(object): with open(file_name) as service_file: service_contents = yaml.safe_load(service_file) self.file_contents[file_name] = service_contents - service_contents = self.filter_jinja2(service_contents) prop_file_name = service_name + ':main.yml' for key, value in service_contents.items(): ansible_property = AnsibleProperty(key, value, @@ -76,10 +75,15 @@ class AnsibleProperties(object): with open(self.allvars_path) as allvars_file: allvars_contents = yaml.safe_load(allvars_file) self.file_contents[self.allvars_path] = allvars_contents - allvars_contents = self.filter_jinja2(allvars_contents) for key, value in allvars_contents.items(): + overrides = False + orig_value = None + if key in self.unique_properties: + overrides = True + orig_value = self.unique_properties[key].value ansible_property = AnsibleProperty(key, value, - 'group_vars/all.yml') + 'group_vars/all.yml', + overrides, orig_value) self.properties.append(ansible_property) self.unique_properties[key] = ansible_property except Exception as e: @@ -90,10 +94,15 @@ class AnsibleProperties(object): globals_data = sync_read_file(self.globals_path) globals_contents = yaml.safe_load(globals_data) self.file_contents[self.globals_path] = globals_contents - globals_contents = self.filter_jinja2(globals_contents) for key, value in globals_contents.items(): + overrides = False + orig_value = None + if key in self.unique_properties: + overrides = True + orig_value = self.unique_properties[key].value ansible_property = AnsibleProperty(key, value, - GLOBALS_FILENAME) + GLOBALS_FILENAME, + overrides, orig_value) self.properties.append(ansible_property) self.unique_properties[key] = ansible_property except Exception as e: @@ -115,6 +124,9 @@ class AnsibleProperties(object): unique_list.append(value) return sorted(unique_list, key=lambda x: x.name) + # TODO -- if this isn't used for 2.1.x it should be removed + # property listing is still being tweaked so leaving for + # the time being in case we want to use it def filter_jinja2(self, contents): new_contents = {} for key, value in contents.items(): @@ -150,7 +162,10 @@ class AnsibleProperties(object): class AnsibleProperty(object): - def __init__(self, name, value, file_name): + def __init__(self, name, value, file_name, overrides=False, + orig_value=None): self.name = name self.value = value self.file_name = file_name + self.overrides = overrides + self.orig_value = orig_value diff --git a/kollacli/common/utils.py b/kollacli/common/utils.py index 00d68be..e1cd395 100644 --- a/kollacli/common/utils.py +++ b/kollacli/common/utils.py @@ -66,6 +66,19 @@ def get_kolla_log_file_size(): return size +def get_property_list_length(): + envvar = 'KOLLA_PROP_LIST_LENGTH' + length_str = os.environ.get(envvar, '50') + try: + length = int(length_str) + except Exception: + raise CommandError( + u._('Environmental variable ({env_var}) is not an ' + 'integer ({prop_length}).') + .format(env_var=envvar, prop_length=length_str)) + return length + + def get_admin_user(): return os.environ.get("KOLLA_CLI_ADMIN_USER", "kolla") diff --git a/requirements.txt b/requirements.txt index 2132591..081aa12 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ ansible>=1.9.2 Babel>=0.9.6 cliff>=1.13.0 # Apache-2.0 cliff-tablib>=1.1 -docker-py>=1.3.1 +docker-py==1.3.1 jsonpickle>=0.9 oslo.i18n>=1.3.0 # Apache-2.0 oslo.utils!=2.6.0,>=2.4.0 # Apache-2.0