From 27c0ae06241e18f6639311aba65657b4ef83cf26 Mon Sep 17 00:00:00 2001 From: Angus Salkeld Date: Thu, 19 Nov 2015 15:35:22 +1000 Subject: [PATCH] Add support for copying files from a "zk://" source Co-Authored-By: Michal Rostecki Implements: blueprint zookeeper Change-Id: I176f063d3802716846b921e210c1569d28bd90d8 --- docker/base/Dockerfile.j2 | 5 ++ docker/base/set_configs.py | 78 ++++++++++++++++++++++++++++- docker/base/sudoers | 2 +- docker/openstack-base/Dockerfile.j2 | 2 + test-requirements.txt | 1 + tests/test_set_config.py | 58 +++++++++++++++++++++ 6 files changed, 144 insertions(+), 2 deletions(-) diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2 index 1c2d314335..149fdee13f 100644 --- a/docker/base/Dockerfile.j2 +++ b/docker/base/Dockerfile.j2 @@ -132,6 +132,9 @@ COPY versionlock.list /etc/yum/pluginconf.d/ RUN yum install -y \ sudo \ which \ + python \ + python-jinja2 \ + python-kazoo \ && yum clean all {% endif %} @@ -167,6 +170,8 @@ RUN apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com 199369E5404BD && apt-get dist-upgrade -y \ && apt-get install -y --no-install-recommends \ python \ + python-jinja2 \ + python-kazoo \ curl \ && apt-get clean \ && sed -i "s|'purelib': '\$base/local/lib/python\$py_version_short/dist-packages',|'purelib': '\$base/lib/python\$py_version_short/dist-packages',|;s|'platlib': '\$platbase/local/lib/python\$py_version_short/dist-packages',|'platlib': '\$platbase/lib/python\$py_version_short/dist-packages',|;s|'headers': '\$base/local/include/python\$py_version_short/\$dist_name',|'headers': '\$base/include/python\$py_version_short/\$dist_name',|;s|'scripts': '\$base/local/bin',|'scripts': '\$base/bin',|;s|'data' : '\$base/local',|'data' : '\$base',|" /usr/lib/python2.7/distutils/command/install.py \ diff --git a/docker/base/set_configs.py b/docker/base/set_configs.py index cd9ccd83f7..b567e385d9 100644 --- a/docker/base/set_configs.py +++ b/docker/base/set_configs.py @@ -12,12 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import contextlib import json import logging import os from pwd import getpwnam import shutil import sys +import urlparse + +from kazoo import client as kz_client +from kazoo import exceptions as kz_exceptions # TODO(rhallisey): add docstring. @@ -46,7 +51,13 @@ def validate_config(config): def validate_source(data): source = data.get('source') - if not os.path.exists(source): + if is_zk_transport(source): + with zk_connection(source) as zk: + exists = zk_path_exists(zk, source) + else: + exists = os.path.exists(source) + + if not exists: if data.get('optional'): LOG.warn('{} does not exist, but is not required'.format(source)) return False @@ -57,6 +68,66 @@ def validate_source(data): return True +def is_zk_transport(path): + if path.startswith('zk://'): + return True + if os.environ.get("KOLLA_ZK_HOSTS") is not None: + return True + + return False + + +@contextlib.contextmanager +def zk_connection(url): + # support an environment and url + # if url, it should be like this: + # zk://
:/ + + zk_hosts = os.environ.get("KOLLA_ZK_HOSTS") + if zk_hosts is None: + components = urlparse.urlparse(url) + zk_hosts = components.netloc + zk = kz_client.KazooClient(hosts=zk_hosts) + zk.start() + try: + yield zk + finally: + zk.stop() + + +def zk_path_exists(zk, path): + try: + components = urlparse.urlparse(path) + zk.get(components.path) + return True + except kz_exceptions.NoNodeError: + return False + + +def zk_copy_tree(zk, src, dest): + """Recursively copy contents of url_source into dest.""" + data, stat = zk.get(src) + + if data: + dest_path = os.path.dirname(dest) + if not os.path.exists(dest_path): + LOG.info('Creating dest parent directory: {}'.format( + dest_path)) + os.makedirs(dest_path) + + LOG.info('Copying {} to {}'.format(src, dest)) + with open(dest, 'w') as df: + df.write(data.decode("utf-8")) + + try: + children = zk.get_children(src) + except kz_exceptions.NoNodeError: + return + for child in children: + zk_copy_tree(zk, os.path.join(src, child), + os.path.join(dest, child)) + + def copy_files(data): dest = data.get('dest') source = data.get('source') @@ -68,6 +139,11 @@ def copy_files(data): else: os.remove(dest) + if is_zk_transport(source): + with zk_connection(source) as zk: + components = urlparse.urlparse(source) + return zk_copy_tree(zk, components.path, dest) + if os.path.isdir(source): source_path = source dest_path = dest diff --git a/docker/base/sudoers b/docker/base/sudoers index 76baefcb07..974f36a294 100644 --- a/docker/base/sudoers +++ b/docker/base/sudoers @@ -13,6 +13,6 @@ root ALL=(ALL) ALL # anyone in the kolla group may run /usr/local/bin/kolla_set_configs as the # root user via sudo without password confirmation -%kolla ALL=(root) NOPASSWD: /usr/local/bin/kolla_set_configs +%kolla ALL=(root) NOPASSWD: /usr/local/bin/kolla_set_configs, /usr/bin/install #includedir /etc/sudoers.d diff --git a/docker/openstack-base/Dockerfile.j2 b/docker/openstack-base/Dockerfile.j2 index 900fc3b4dd..79a6e2894c 100644 --- a/docker/openstack-base/Dockerfile.j2 +++ b/docker/openstack-base/Dockerfile.j2 @@ -81,6 +81,8 @@ RUN ln -s openstack-base-source/* /requirements \ && pip install -U virtualenv \ && virtualenv /var/lib/kolla/venv \ && /var/lib/kolla/venv/bin/pip --no-cache-dir install -U -c requirements/upper-constraints.txt \ + jinja2 \ + kazoo \ python-barbicanclient \ python-ceilometerclient \ python-congressclient \ diff --git a/test-requirements.txt b/test-requirements.txt index 89194d6fef..11f725250b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -19,3 +19,4 @@ sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 testrepository>=0.0.18 testscenarios>=0.4 testtools>=1.4.0 +zake>=0.1.6 # Apache-2.0 diff --git a/tests/test_set_config.py b/tests/test_set_config.py index 1e8ac9ce62..7092895c54 100644 --- a/tests/test_set_config.py +++ b/tests/test_set_config.py @@ -15,8 +15,11 @@ import json import mock import os.path import sys +import tempfile from oslotest import base +import testscenarios +from zake import fake_client # nasty: to import set_config (not a part of the kolla package) this_dir = os.path.dirname(sys.modules[__name__].__file__) @@ -62,3 +65,58 @@ class LoadFromEnv(base.BaseTestCase): mock.call().write(u'/bin/true'), mock.call().__exit__(None, None, None)], mo.mock_calls) + + +class ZkCopyTest(testscenarios.WithScenarios, base.BaseTestCase): + + scenarios = [ + ('1', dict(in_paths=['a.conf'], + in_subtree='/', + expect_paths=[['a.conf']])), + ('2', dict(in_paths=['/a/b/c.x', '/a/b/foo.x', '/a/no.x'], + in_subtree='/a/b', + expect_paths=[['c.x'], ['foo.x']])), + ('3', dict(in_paths=['/a/b/c.x', '/a/z/foo.x'], + in_subtree='/', + expect_paths=[['a', 'b', 'c.x'], ['a', 'z', 'foo.x']])), + ] + + def setUp(self): + super(ZkCopyTest, self).setUp() + self.client = fake_client.FakeClient() + self.client.start() + self.addCleanup(self.client.stop) + self.addCleanup(self.client.close) + + def test_cp_tree(self): + # Note: oslotest.base cleans up all tempfiles as follows: + # self.useFixture(fixtures.NestedTempfile()) + # so we don't have to. + temp_dir = tempfile.mkdtemp() + + for path in self.in_paths: + self.client.create(path, 'one', makepath=True) + set_configs.zk_copy_tree(self.client, self.in_subtree, temp_dir) + for expect in self.expect_paths: + expect.insert(0, temp_dir) + expect_path = os.path.join(*expect) + self.assertTrue(os.path.exists(expect_path)) + + +class ZkExistsTest(base.BaseTestCase): + def setUp(self): + super(ZkExistsTest, self).setUp() + self.client = fake_client.FakeClient() + self.client.start() + self.addCleanup(self.client.stop) + self.addCleanup(self.client.close) + + def test_path_exists_no(self): + self.client.create('/test/path/thing', 'one', makepath=True) + self.assertFalse(set_configs.zk_path_exists(self.client, + '/test/missing/thing')) + + def test_path_exists_yes(self): + self.client.create('/test/path/thing', 'one', makepath=True) + self.assertTrue(set_configs.zk_path_exists(self.client, + '/test/path/thing'))