synchronize access to files to avoid corruption

- add new read and write file methods to synchronous accesss to files to avoid simultaneous reads and writes.
- fix issue with pexpect prompt change in last check-in. caused deploy to fail in some cases.
synchronize access to files to avoid corruption
- better errors on ansible playbook errors

kollacli/ansible/inventory.py

@@ -15,12 +15,9 @@ import json
 import jsonpickle
 import logging
 import os
-import shutil
 import tempfile
 import traceback
 
-from tempfile import mkstemp
-
 from kollacli import exceptions
 from kollacli import utils
 
@@ -322,8 +319,7 @@ class Inventory(object):
         data = ''
         try:
             if os.path.exists(inventory_path):
-                with open(inventory_path, 'rb') as inv_file:
+                data = utils.sync_read_file(inventory_path)
-                    data = inv_file.read()
 
             if data.strip():
                 inventory = jsonpickle.decode(data)
@@ -343,32 +339,14 @@ class Inventory(object):
         """Save the inventory in a pickle file"""
         inventory_path = os.path.join(utils.get_kollacli_etc(), INVENTORY_PATH)
         try:
-            # the file handle returned from mkstemp must be closed or else
-            # if this is called many times you will have an unpleasant
-            # file handle leak
-            tmp_filehandle, tmp_path = mkstemp()
-
             # multiple trips thru json to render a readable inventory file
             data = jsonpickle.encode(inventory)
             data_str = json.loads(data)
             pretty_data = json.dumps(data_str, indent=4)
-            with open(tmp_path, 'w') as tmp_file:
+            utils.sync_write_file(inventory_path, pretty_data)
-                tmp_file.write(pretty_data)
+
-            shutil.copyfile(tmp_path, inventory_path)
-            os.remove(tmp_path)
         except Exception as e:
             raise CommandError('saving inventory failed: %s' % e)
-        finally:
-            try:
-                os.close(tmp_filehandle)
-            except Exception:
-                pass
-
-            if tmp_filehandle is not None:
-                try:
-                    os.close(tmp_filehandle)
-                except Exception:
-                    pass
 
     def _create_default_inventory(self):
 

kollacli/ansible/playbook.py

@@ -96,9 +96,10 @@ class AnsiblePlaybook(object):
                                          stderr=subprocess.PIPE).communicate()
                     self.log.debug(inv)
 
-            err_flag, _ = run_cmd(cmd, self.print_output)
+            err_flag, msg = run_cmd(cmd, self.print_output)
             if err_flag:
-                raise Exception('Failure')
+                raise CommandError('Ansible command failed \n%s\n%s'
+                                   % (cmd, msg))
 
             self.log.info('Success')
         except CommandError as e:

kollacli/ansible/properties.py

@@ -18,6 +18,7 @@ import yaml
 from kollacli.utils import change_property
 from kollacli.utils import get_kolla_etc
 from kollacli.utils import get_kolla_home
+from kollacli.utils import sync_read_file
 
 ALLVARS_PATH = 'ansible/group_vars/all.yml'
 GLOBALS_FILENAME = 'globals.yml'
@@ -85,15 +86,15 @@ class AnsibleProperties(object):
 
         try:
             self.globals_path = os.path.join(kolla_etc, GLOBALS_FILENAME)
-            with open(self.globals_path) as globals_file:
+            globals_data = sync_read_file(self.globals_path)
-                globals_contents = yaml.load(globals_file)
+            globals_contents = yaml.load(globals_data)
-                self.file_contents[self.globals_path] = globals_contents
+            self.file_contents[self.globals_path] = globals_contents
-                globals_contents = self.filter_jinja2(globals_contents)
+            globals_contents = self.filter_jinja2(globals_contents)
-                for key, value in globals_contents.items():
+            for key, value in globals_contents.items():
-                    ansible_property = AnsibleProperty(key, value,
+                ansible_property = AnsibleProperty(key, value,
-                                                       GLOBALS_FILENAME)
+                                                   GLOBALS_FILENAME)
-                    self.properties.append(ansible_property)
+                self.properties.append(ansible_property)
-                    self.unique_properties[key] = ansible_property
+                self.unique_properties[key] = ansible_property
         except Exception as e:
             raise e
 

kollacli/utils.py

@@ -14,8 +14,11 @@
 import logging
 import os
 import pexpect
+import tempfile
 import yaml
 
+from lockfile import LockFile
+
 
 def get_kolla_home():
     return os.environ.get("KOLLA_HOME", "/usr/share/kolla/")
@@ -98,19 +101,22 @@ def run_cmd(cmd, print_output=True):
     If the command is an ansible playbook command, record the
     output in an ansible log file.
     """
+    pwd_prompt = '[sudo] password'
     log = logging.getLogger(__name__)
     err_flag = False
     output = []
     try:
         child = pexpect.spawn(cmd)
-        index = child.expect([pexpect.EOF, '[sudo] password'])
+        sniff = child.read(len(pwd_prompt))
-        if index == 1:
+        if sniff == pwd_prompt:
+            output.append(sniff + '\n')
             raise Exception(
                 'Insufficient permissions to run command "%s"' % cmd)
         child.maxsize = 1
         child.timeout = 86400
         for line in child:
-            outline = line.rstrip()
+            outline = sniff + line.rstrip()
+            sniff = ''
             output.append(outline)
             if print_output:
                 log.info(outline)
@@ -140,29 +146,61 @@ def change_property(file_path, property_key, property_value, clear=False):
     If not clear, and key is found, edit property in place.
     """
     try:
-        file_contents = []
+        new_contents = []
-        with open(file_path, 'r+') as property_file:
+        read_data = sync_read_file(file_path)
-            new_line = '%s: "%s"\n' % (property_key, property_value)
+        lines = read_data.split('\n')
-            property_key_found = False
+        new_line = '%s: "%s"\n' % (property_key, property_value)
-            for line in property_file:
+        property_key_found = False
-                if line[0:len(property_key)] == property_key:
+        for line in lines:
-                    property_key_found = True
+            if line[0:len(property_key)] == property_key:
-                    if clear:
+                property_key_found = True
-                        # clear existing property
+                if clear:
-                        line = ''
+                    # clear existing property
-                    else:
+                    line = ''
-                        # edit existing property
+                else:
-                        line = new_line
+                    # edit existing property
-                file_contents.append(line)
+                    line = new_line
-            if not property_key_found and not clear:
+            new_contents.append(line + '\n')
-                # add new property to file
+        if not property_key_found and not clear:
-                file_contents.append(new_line)
+            # add new property to file
+            new_contents.append(new_line)
 
-            property_file.seek(0)
+        write_data = ''.join(new_contents)
-            property_file.truncate()
+        sync_write_file(file_path, write_data)
 
-        with open(file_path, 'w') as property_file:
-            for line in file_contents:
-                property_file.write(line)
     except Exception as e:
         raise e
+
+
+def sync_read_file(path, mode='r'):
+    """synchronously read file
+
+    return file data
+    """
+    # lock is in /tmp to avoid permission issues
+    lpath = os.path.join(tempfile.gettempdir(), os.path.basename(path))
+    lock = LockFile(lpath)
+    try:
+        lock.acquire(True)
+        with open(path, mode) as data_file:
+            data = data_file.read()
+    except Exception as e:
+        raise e
+    finally:
+        lock.release()
+    return data
+
+
+def sync_write_file(path, data, mode='w'):
+    """synchronously write file"""
+    # lock is in /tmp to avoid permission issues
+    lpath = os.path.join(tempfile.gettempdir(), os.path.basename(path))
+    lock = LockFile(lpath)
+    try:
+        lock.acquire(True)
+        with open(path, mode) as data_file:
+            data_file.write(data)
+    except Exception as e:
+        raise e
+    finally:
+        lock.release()

tools/passwd_editor.py

@@ -21,15 +21,15 @@ from kollacli import utils
 def _print_pwd_keys(path):
     pwd_keys = ''
     prefix = ''
-    with open(path, 'r') as pwd_file:
+    pwd_data = utils.sync_read_file(path)
-        for line in pwd_file:
+    for line in pwd_data.split('\n'):
-            if line.startswith('#'):
+        if line.startswith('#'):
-                # skip commented lines
+            # skip commented lines
-                continue
+            continue
-            if ':' in line:
+        if ':' in line:
-                pwd_key = line.split(':')[0]
+            pwd_key = line.split(':')[0]
-                pwd_keys = pwd_keys + prefix + pwd_key
+            pwd_keys = ''.join([pwd_keys, prefix, pwd_key])
-                prefix = ','
+            prefix = ','
 
     print(pwd_keys)