Add support for copying files from a "zk://" source

Co-Authored-By: Michal Rostecki <mrostecki@mirantis.com>
Implements: blueprint zookeeper

Change-Id: I176f063d3802716846b921e210c1569d28bd90d8

docker/base/Dockerfile.j2

@@ -132,6 +132,9 @@ COPY versionlock.list /etc/yum/pluginconf.d/
 RUN yum install -y \
         sudo \
         which \
+        python \
+        python-jinja2 \
+        python-kazoo \
     && yum clean all
 
     {% endif %}
@@ -167,6 +170,8 @@ RUN apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com 199369E5404BD
     && apt-get dist-upgrade -y \
     && apt-get install -y --no-install-recommends \
         python \
+        python-jinja2 \
+        python-kazoo \
         curl \
     && apt-get clean \
     && sed -i "s|'purelib': '\$base/local/lib/python\$py_version_short/dist-packages',|'purelib': '\$base/lib/python\$py_version_short/dist-packages',|;s|'platlib': '\$platbase/local/lib/python\$py_version_short/dist-packages',|'platlib': '\$platbase/lib/python\$py_version_short/dist-packages',|;s|'headers': '\$base/local/include/python\$py_version_short/\$dist_name',|'headers': '\$base/include/python\$py_version_short/\$dist_name',|;s|'scripts': '\$base/local/bin',|'scripts': '\$base/bin',|;s|'data'   : '\$base/local',|'data'   : '\$base',|" /usr/lib/python2.7/distutils/command/install.py \

docker/base/set_configs.py

@@ -12,12 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import contextlib
 import json
 import logging
 import os
 from pwd import getpwnam
 import shutil
 import sys
+import urlparse
+
+from kazoo import client as kz_client
+from kazoo import exceptions as kz_exceptions
 
 
 # TODO(rhallisey): add docstring.
@@ -46,7 +51,13 @@ def validate_config(config):
 def validate_source(data):
     source = data.get('source')
 
-    if not os.path.exists(source):
+    if is_zk_transport(source):
+        with zk_connection(source) as zk:
+            exists = zk_path_exists(zk, source)
+    else:
+        exists = os.path.exists(source)
+
+    if not exists:
         if data.get('optional'):
             LOG.warn('{} does not exist, but is not required'.format(source))
             return False
@@ -57,6 +68,66 @@ def validate_source(data):
     return True
 
 
+def is_zk_transport(path):
+    if path.startswith('zk://'):
+        return True
+    if os.environ.get("KOLLA_ZK_HOSTS") is not None:
+        return True
+
+    return False
+
+
+@contextlib.contextmanager
+def zk_connection(url):
+    # support an environment and url
+    # if url, it should be like this:
+    # zk://<address>:<port>/<path>
+
+    zk_hosts = os.environ.get("KOLLA_ZK_HOSTS")
+    if zk_hosts is None:
+        components = urlparse.urlparse(url)
+        zk_hosts = components.netloc
+    zk = kz_client.KazooClient(hosts=zk_hosts)
+    zk.start()
+    try:
+        yield zk
+    finally:
+        zk.stop()
+
+
+def zk_path_exists(zk, path):
+    try:
+        components = urlparse.urlparse(path)
+        zk.get(components.path)
+        return True
+    except kz_exceptions.NoNodeError:
+        return False
+
+
+def zk_copy_tree(zk, src, dest):
+    """Recursively copy contents of url_source into dest."""
+    data, stat = zk.get(src)
+
+    if data:
+        dest_path = os.path.dirname(dest)
+        if not os.path.exists(dest_path):
+            LOG.info('Creating dest parent directory: {}'.format(
+                dest_path))
+            os.makedirs(dest_path)
+
+        LOG.info('Copying {} to {}'.format(src, dest))
+        with open(dest, 'w') as df:
+            df.write(data.decode("utf-8"))
+
+    try:
+        children = zk.get_children(src)
+    except kz_exceptions.NoNodeError:
+        return
+    for child in children:
+        zk_copy_tree(zk, os.path.join(src, child),
+                     os.path.join(dest, child))
+
+
 def copy_files(data):
     dest = data.get('dest')
     source = data.get('source')
@@ -68,6 +139,11 @@ def copy_files(data):
         else:
             os.remove(dest)
 
+    if is_zk_transport(source):
+        with zk_connection(source) as zk:
+            components = urlparse.urlparse(source)
+            return zk_copy_tree(zk, components.path, dest)
+
     if os.path.isdir(source):
         source_path = source
         dest_path = dest

docker/base/sudoers

@@ -13,6 +13,6 @@ root ALL=(ALL) ALL
 
 # anyone in the kolla group may run /usr/local/bin/kolla_set_configs as the
 # root user via sudo without password confirmation
-%kolla ALL=(root) NOPASSWD: /usr/local/bin/kolla_set_configs
+%kolla ALL=(root) NOPASSWD: /usr/local/bin/kolla_set_configs, /usr/bin/install
 
 #includedir /etc/sudoers.d

docker/openstack-base/Dockerfile.j2

@@ -81,6 +81,8 @@ RUN ln -s openstack-base-source/* /requirements \
     && pip install -U virtualenv \
     && virtualenv /var/lib/kolla/venv \
     && /var/lib/kolla/venv/bin/pip --no-cache-dir install -U -c requirements/upper-constraints.txt \
+        jinja2 \
+        kazoo \
         python-barbicanclient \
         python-ceilometerclient \
         python-congressclient \

test-requirements.txt

@@ -19,3 +19,4 @@ sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2
 testrepository>=0.0.18
 testscenarios>=0.4
 testtools>=1.4.0
+zake>=0.1.6 # Apache-2.0

tests/test_set_config.py

@@ -15,8 +15,11 @@ import json
 import mock
 import os.path
 import sys
+import tempfile
 
 from oslotest import base
+import testscenarios
+from zake import fake_client
 
 # nasty: to import set_config (not a part of the kolla package)
 this_dir = os.path.dirname(sys.modules[__name__].__file__)
@@ -62,3 +65,58 @@ class LoadFromEnv(base.BaseTestCase):
                                   mock.call().write(u'/bin/true'),
                                   mock.call().__exit__(None, None, None)],
                                  mo.mock_calls)
+
+
+class ZkCopyTest(testscenarios.WithScenarios, base.BaseTestCase):
+
+    scenarios = [
+        ('1', dict(in_paths=['a.conf'],
+                   in_subtree='/',
+                   expect_paths=[['a.conf']])),
+        ('2', dict(in_paths=['/a/b/c.x', '/a/b/foo.x', '/a/no.x'],
+                   in_subtree='/a/b',
+                   expect_paths=[['c.x'], ['foo.x']])),
+        ('3', dict(in_paths=['/a/b/c.x', '/a/z/foo.x'],
+                   in_subtree='/',
+                   expect_paths=[['a', 'b', 'c.x'], ['a', 'z', 'foo.x']])),
+    ]
+
+    def setUp(self):
+        super(ZkCopyTest, self).setUp()
+        self.client = fake_client.FakeClient()
+        self.client.start()
+        self.addCleanup(self.client.stop)
+        self.addCleanup(self.client.close)
+
+    def test_cp_tree(self):
+        # Note: oslotest.base cleans up all tempfiles as follows:
+        # self.useFixture(fixtures.NestedTempfile())
+        # so we don't have to.
+        temp_dir = tempfile.mkdtemp()
+
+        for path in self.in_paths:
+            self.client.create(path, 'one', makepath=True)
+        set_configs.zk_copy_tree(self.client, self.in_subtree, temp_dir)
+        for expect in self.expect_paths:
+            expect.insert(0, temp_dir)
+            expect_path = os.path.join(*expect)
+            self.assertTrue(os.path.exists(expect_path))
+
+
+class ZkExistsTest(base.BaseTestCase):
+    def setUp(self):
+        super(ZkExistsTest, self).setUp()
+        self.client = fake_client.FakeClient()
+        self.client.start()
+        self.addCleanup(self.client.stop)
+        self.addCleanup(self.client.close)
+
+    def test_path_exists_no(self):
+        self.client.create('/test/path/thing', 'one', makepath=True)
+        self.assertFalse(set_configs.zk_path_exists(self.client,
+                                                    '/test/missing/thing'))
+
+    def test_path_exists_yes(self):
+        self.client.create('/test/path/thing', 'one', makepath=True)
+        self.assertTrue(set_configs.zk_path_exists(self.client,
+                                                   '/test/path/thing'))