Merge "Add support for copying files from a "zk://" source"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
939b464d12
@ -132,6 +132,9 @@ COPY versionlock.list /etc/yum/pluginconf.d/
|
||||
RUN yum install -y \
|
||||
sudo \
|
||||
which \
|
||||
python \
|
||||
python-jinja2 \
|
||||
python-kazoo \
|
||||
&& yum clean all
|
||||
|
||||
{% endif %}
|
||||
@ -167,6 +170,8 @@ RUN apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com 199369E5404BD
|
||||
&& apt-get dist-upgrade -y \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
python \
|
||||
python-jinja2 \
|
||||
python-kazoo \
|
||||
curl \
|
||||
&& apt-get clean \
|
||||
&& sed -i "s|'purelib': '\$base/local/lib/python\$py_version_short/dist-packages',|'purelib': '\$base/lib/python\$py_version_short/dist-packages',|;s|'platlib': '\$platbase/local/lib/python\$py_version_short/dist-packages',|'platlib': '\$platbase/lib/python\$py_version_short/dist-packages',|;s|'headers': '\$base/local/include/python\$py_version_short/\$dist_name',|'headers': '\$base/include/python\$py_version_short/\$dist_name',|;s|'scripts': '\$base/local/bin',|'scripts': '\$base/bin',|;s|'data' : '\$base/local',|'data' : '\$base',|" /usr/lib/python2.7/distutils/command/install.py \
|
||||
|
||||
@ -12,12 +12,17 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pwd import getpwnam
|
||||
import shutil
|
||||
import sys
|
||||
import urlparse
|
||||
|
||||
from kazoo import client as kz_client
|
||||
from kazoo import exceptions as kz_exceptions
|
||||
|
||||
|
||||
# TODO(rhallisey): add docstring.
|
||||
@ -46,7 +51,13 @@ def validate_config(config):
|
||||
def validate_source(data):
|
||||
source = data.get('source')
|
||||
|
||||
if not os.path.exists(source):
|
||||
if is_zk_transport(source):
|
||||
with zk_connection(source) as zk:
|
||||
exists = zk_path_exists(zk, source)
|
||||
else:
|
||||
exists = os.path.exists(source)
|
||||
|
||||
if not exists:
|
||||
if data.get('optional'):
|
||||
LOG.warn('{} does not exist, but is not required'.format(source))
|
||||
return False
|
||||
@ -57,6 +68,66 @@ def validate_source(data):
|
||||
return True
|
||||
|
||||
|
||||
def is_zk_transport(path):
|
||||
if path.startswith('zk://'):
|
||||
return True
|
||||
if os.environ.get("KOLLA_ZK_HOSTS") is not None:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def zk_connection(url):
|
||||
# support an environment and url
|
||||
# if url, it should be like this:
|
||||
# zk://<address>:<port>/<path>
|
||||
|
||||
zk_hosts = os.environ.get("KOLLA_ZK_HOSTS")
|
||||
if zk_hosts is None:
|
||||
components = urlparse.urlparse(url)
|
||||
zk_hosts = components.netloc
|
||||
zk = kz_client.KazooClient(hosts=zk_hosts)
|
||||
zk.start()
|
||||
try:
|
||||
yield zk
|
||||
finally:
|
||||
zk.stop()
|
||||
|
||||
|
||||
def zk_path_exists(zk, path):
|
||||
try:
|
||||
components = urlparse.urlparse(path)
|
||||
zk.get(components.path)
|
||||
return True
|
||||
except kz_exceptions.NoNodeError:
|
||||
return False
|
||||
|
||||
|
||||
def zk_copy_tree(zk, src, dest):
|
||||
"""Recursively copy contents of url_source into dest."""
|
||||
data, stat = zk.get(src)
|
||||
|
||||
if data:
|
||||
dest_path = os.path.dirname(dest)
|
||||
if not os.path.exists(dest_path):
|
||||
LOG.info('Creating dest parent directory: {}'.format(
|
||||
dest_path))
|
||||
os.makedirs(dest_path)
|
||||
|
||||
LOG.info('Copying {} to {}'.format(src, dest))
|
||||
with open(dest, 'w') as df:
|
||||
df.write(data.decode("utf-8"))
|
||||
|
||||
try:
|
||||
children = zk.get_children(src)
|
||||
except kz_exceptions.NoNodeError:
|
||||
return
|
||||
for child in children:
|
||||
zk_copy_tree(zk, os.path.join(src, child),
|
||||
os.path.join(dest, child))
|
||||
|
||||
|
||||
def copy_files(data):
|
||||
dest = data.get('dest')
|
||||
source = data.get('source')
|
||||
@ -68,6 +139,11 @@ def copy_files(data):
|
||||
else:
|
||||
os.remove(dest)
|
||||
|
||||
if is_zk_transport(source):
|
||||
with zk_connection(source) as zk:
|
||||
components = urlparse.urlparse(source)
|
||||
return zk_copy_tree(zk, components.path, dest)
|
||||
|
||||
if os.path.isdir(source):
|
||||
source_path = source
|
||||
dest_path = dest
|
||||
|
||||
@ -13,6 +13,6 @@ root ALL=(ALL) ALL
|
||||
|
||||
# anyone in the kolla group may run /usr/local/bin/kolla_set_configs as the
|
||||
# root user via sudo without password confirmation
|
||||
%kolla ALL=(root) NOPASSWD: /usr/local/bin/kolla_set_configs
|
||||
%kolla ALL=(root) NOPASSWD: /usr/local/bin/kolla_set_configs, /usr/bin/install
|
||||
|
||||
#includedir /etc/sudoers.d
|
||||
|
||||
@ -81,6 +81,8 @@ RUN ln -s openstack-base-source/* /requirements \
|
||||
&& pip install -U virtualenv \
|
||||
&& virtualenv /var/lib/kolla/venv \
|
||||
&& /var/lib/kolla/venv/bin/pip --no-cache-dir install -U -c requirements/upper-constraints.txt \
|
||||
jinja2 \
|
||||
kazoo \
|
||||
python-barbicanclient \
|
||||
python-ceilometerclient \
|
||||
python-congressclient \
|
||||
|
||||
@ -19,3 +19,4 @@ sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2
|
||||
testrepository>=0.0.18
|
||||
testscenarios>=0.4
|
||||
testtools>=1.4.0
|
||||
zake>=0.1.6 # Apache-2.0
|
||||
|
||||
@ -15,8 +15,11 @@ import json
|
||||
import mock
|
||||
import os.path
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
from oslotest import base
|
||||
import testscenarios
|
||||
from zake import fake_client
|
||||
|
||||
# nasty: to import set_config (not a part of the kolla package)
|
||||
this_dir = os.path.dirname(sys.modules[__name__].__file__)
|
||||
@ -62,3 +65,58 @@ class LoadFromEnv(base.BaseTestCase):
|
||||
mock.call().write(u'/bin/true'),
|
||||
mock.call().__exit__(None, None, None)],
|
||||
mo.mock_calls)
|
||||
|
||||
|
||||
class ZkCopyTest(testscenarios.WithScenarios, base.BaseTestCase):
|
||||
|
||||
scenarios = [
|
||||
('1', dict(in_paths=['a.conf'],
|
||||
in_subtree='/',
|
||||
expect_paths=[['a.conf']])),
|
||||
('2', dict(in_paths=['/a/b/c.x', '/a/b/foo.x', '/a/no.x'],
|
||||
in_subtree='/a/b',
|
||||
expect_paths=[['c.x'], ['foo.x']])),
|
||||
('3', dict(in_paths=['/a/b/c.x', '/a/z/foo.x'],
|
||||
in_subtree='/',
|
||||
expect_paths=[['a', 'b', 'c.x'], ['a', 'z', 'foo.x']])),
|
||||
]
|
||||
|
||||
def setUp(self):
|
||||
super(ZkCopyTest, self).setUp()
|
||||
self.client = fake_client.FakeClient()
|
||||
self.client.start()
|
||||
self.addCleanup(self.client.stop)
|
||||
self.addCleanup(self.client.close)
|
||||
|
||||
def test_cp_tree(self):
|
||||
# Note: oslotest.base cleans up all tempfiles as follows:
|
||||
# self.useFixture(fixtures.NestedTempfile())
|
||||
# so we don't have to.
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
|
||||
for path in self.in_paths:
|
||||
self.client.create(path, 'one', makepath=True)
|
||||
set_configs.zk_copy_tree(self.client, self.in_subtree, temp_dir)
|
||||
for expect in self.expect_paths:
|
||||
expect.insert(0, temp_dir)
|
||||
expect_path = os.path.join(*expect)
|
||||
self.assertTrue(os.path.exists(expect_path))
|
||||
|
||||
|
||||
class ZkExistsTest(base.BaseTestCase):
|
||||
def setUp(self):
|
||||
super(ZkExistsTest, self).setUp()
|
||||
self.client = fake_client.FakeClient()
|
||||
self.client.start()
|
||||
self.addCleanup(self.client.stop)
|
||||
self.addCleanup(self.client.close)
|
||||
|
||||
def test_path_exists_no(self):
|
||||
self.client.create('/test/path/thing', 'one', makepath=True)
|
||||
self.assertFalse(set_configs.zk_path_exists(self.client,
|
||||
'/test/missing/thing'))
|
||||
|
||||
def test_path_exists_yes(self):
|
||||
self.client.create('/test/path/thing', 'one', makepath=True)
|
||||
self.assertTrue(set_configs.zk_path_exists(self.client,
|
||||
'/test/path/thing'))
|
||||
|
||||
Loading…
Reference in New Issue
Block a user