Support for Configuration Language. File Downloading
Execution plan files can be tranfered as binaries as just an URL to be download. This code download the file when it is a URL Partially-Implements Blueprint: conf-language-support Change-Id: I102546a04a93dc74a4647cc8ad7a33d78ebca86b
This commit is contained in:
Henar Muñoz Frutos
parent
fdc8de63fe
commit
d7af39d3dd
|
|
@ -4,7 +4,7 @@
|
|||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
|
|
@ -14,11 +14,16 @@
|
|||
# limitations under the License.
|
||||
|
||||
import base64
|
||||
import git
|
||||
import os
|
||||
import requests
|
||||
import shutil
|
||||
import subprocess
|
||||
import urlparse
|
||||
|
||||
from muranoagent.common import config
|
||||
|
||||
|
||||
CONF = config.CONF
|
||||
|
||||
|
||||
|
|
@ -34,22 +39,33 @@ class FilesManager(object):
|
|||
os.makedirs(self._cache_folder)
|
||||
|
||||
def put_file(self, file_id, script):
|
||||
if type(file_id) is dict:
|
||||
file_name = file_id.keys()[0]
|
||||
file_def = file_id[file_name]
|
||||
else:
|
||||
file_name = file_id
|
||||
file_def = self._files[file_name]
|
||||
|
||||
if file_def.get('Type') == 'Downloadable':
|
||||
self._download_url_file(file_def)
|
||||
return self._cache_folder
|
||||
else:
|
||||
return self._make_symlink(file_id, file_name, script)
|
||||
|
||||
def _make_symlink(self, file_id, file_name, script):
|
||||
cache_path = self._fetch_file(file_id)
|
||||
|
||||
script_folder = os.path.join(self._cache_folder, script)
|
||||
if not os.path.exists(script_folder):
|
||||
if not os.path.isdir(script_folder):
|
||||
os.mkdir(script_folder)
|
||||
file_folder = os.path.join(script_folder,
|
||||
os.path.dirname(file_name))
|
||||
if not os.path.isdir(file_folder):
|
||||
os.makedirs(file_folder)
|
||||
|
||||
filedef = self._files[file_id]
|
||||
filename = filedef['Name']
|
||||
|
||||
file_folder = os.path.join(script_folder, os.path.dirname(filename))
|
||||
if not os.path.exists(file_folder):
|
||||
os.makedirs(file_folder)
|
||||
|
||||
script_path = os.path.join(script_folder, filename)
|
||||
|
||||
os.symlink(cache_path, script_path)
|
||||
if cache_path is not None:
|
||||
script_path = os.path.join(os.getcwd(), script_folder, file_name)
|
||||
if not os.path.lexists(script_path):
|
||||
os.symlink(cache_path, script_path)
|
||||
return script_path
|
||||
|
||||
def _fetch_file(self, file_id):
|
||||
|
|
@ -68,7 +84,59 @@ class FilesManager(object):
|
|||
self._fetched_files[file_id] = out_path
|
||||
return out_path
|
||||
|
||||
def _download_url_file(self, file_def):
|
||||
"""It download the file in the murano-agent. It can proceed
|
||||
from a git file or any other internal URL
|
||||
"""
|
||||
|
||||
if 'URL' not in file_def:
|
||||
raise ValueError("No valid URL in file {0}".
|
||||
format(file_def))
|
||||
url_file = file_def['URL']
|
||||
|
||||
if not self._url(url_file):
|
||||
raise ValueError("Provided URL is not valid {0}".
|
||||
format(url_file))
|
||||
|
||||
if not os.path.isdir(os.path.join(self._cache_folder, 'files')):
|
||||
os.makedirs(os.path.join(self._cache_folder, 'files'))
|
||||
|
||||
folder = self._get_file_folder(url_file)
|
||||
|
||||
if self._is_git_repository(url_file):
|
||||
if not os.path.isdir(folder):
|
||||
git.Git().clone(url_file, folder)
|
||||
else:
|
||||
self._download_file(url_file, folder)
|
||||
|
||||
def clear(self):
|
||||
os.chdir(os.path.dirname(self._cache_folder))
|
||||
shutil.rmtree(self._cache_folder, ignore_errors=True)
|
||||
shutil.rmtree(self._cache_folder, ignore_errors=True)
|
||||
|
||||
def _download_file(self, url, path):
|
||||
local_filename = url.split('/')[-1]
|
||||
r = requests.get(url, stream=True)
|
||||
with open(os.path.join(path, local_filename), 'wb') as f:
|
||||
for chunk in r.iter_content(chunk_size=1024):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
f.flush()
|
||||
return local_filename
|
||||
|
||||
def _url(self, file):
|
||||
return (urlparse.urlsplit(file).scheme or
|
||||
urlparse.urlsplit(file).netloc)
|
||||
|
||||
def _get_file_folder(self, file_url):
|
||||
if file_url.endswith('.git'):
|
||||
file_folder = file_url[file_url.rfind('/') + 1:
|
||||
file_url.find('.git')]
|
||||
else:
|
||||
file_folder = file_url[file_url.rfind('/') + 1:]
|
||||
|
||||
folder = os.path.join(self._cache_folder, 'files', file_folder)
|
||||
return folder
|
||||
|
||||
def _is_git_repository(self, url):
|
||||
return url.startswith(("git://",
|
||||
"git+http://", "git+https:/"))
|
||||
|
|
|
|||
|
|
@ -1,13 +0,0 @@
|
|||
# Copyright (c) 2013 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import fixtures
|
||||
from oslo.config import cfg
|
||||
import testtools
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
||||
|
||||
class MuranoAgentTestCase(testtools.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(MuranoAgentTestCase, self).setUp()
|
||||
self.useFixture(fixtures.FakeLogger('murano-agent'))
|
||||
|
||||
def override_config(self, name, override, group=None):
|
||||
CONF.set_override(name, override, group)
|
||||
self.addCleanup(CONF.clear_override, name, group)
|
||||
|
|
@ -0,0 +1,192 @@
|
|||
# Copyright (c) 2015 Telefonica I+D
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import bunch
|
||||
import git
|
||||
import mock
|
||||
|
||||
from muranoagent.common import config as cfg
|
||||
from muranoagent import files_manager
|
||||
from muranoagent.tests.unit import base
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
||||
|
||||
class TestFileManager(base.MuranoAgentTestCase):
|
||||
|
||||
@mock.patch('os.path.isdir')
|
||||
@mock.patch('os.mkdir')
|
||||
@mock.patch('os.makedirs')
|
||||
def setUp(self, mock_makedir, mock_mkdir, mock_path):
|
||||
mock_path.return_value = True
|
||||
mock_mkdir.return_value = None
|
||||
mock_makedir.return_value = None
|
||||
super(TestFileManager, self).setUp()
|
||||
CONF.set_override('storage', 'cache')
|
||||
|
||||
@mock.patch('os.makedirs')
|
||||
def test_get_folder_git(self, mock_path):
|
||||
"""It gets the folder where the URL is a git URL."""
|
||||
mock_path.return_value = None
|
||||
files = files_manager.FilesManager(self.get_template_downloable())
|
||||
folder = files._get_file_folder("http://tomcat.git")
|
||||
self.assertEqual(folder, "cache/files/ID/files/tomcat")
|
||||
|
||||
@mock.patch('os.makedirs')
|
||||
def test_get_folder_not_git(self, mock_path):
|
||||
"""It gets the folder from the URL."""
|
||||
mock_path.return_value = None
|
||||
files = files_manager.FilesManager(self.get_template_downloable())
|
||||
folder = files._get_file_folder("http://tomcat")
|
||||
self.assertEqual(folder, "cache/files/ID/files/tomcat")
|
||||
|
||||
@mock.patch("git.Git")
|
||||
@mock.patch('os.path.isdir')
|
||||
@mock.patch('os.makedirs')
|
||||
def test_execution_plan_type_downloable_git(self, mock_makedir, mock_path,
|
||||
mock_git):
|
||||
"""It tests an execution plan when there are files
|
||||
which should be downloable.
|
||||
"""
|
||||
mock_makedir.return_value = None
|
||||
mock_path.return_value = True
|
||||
mock_git.clone.return_value = None
|
||||
template = self.get_template_downloable_git()
|
||||
files = files_manager.FilesManager(self.get_template_downloable())
|
||||
files._download_url_file(template.Files['mycoockbook'])
|
||||
|
||||
@mock.patch('os.path.isdir')
|
||||
@mock.patch('os.mkdir')
|
||||
@mock.patch('os.makedirs')
|
||||
@mock.patch('__builtin__.open')
|
||||
@mock.patch('requests.get')
|
||||
def test_execution_plan_type_downloable(self, mock_requests, open_mock,
|
||||
mock_makedir,
|
||||
mock_mkdir, mock_path):
|
||||
"""It tests an execution plan when there are files
|
||||
which should be downloable.
|
||||
"""
|
||||
mock_path.return_value = True
|
||||
mock_mkdir.return_value = None
|
||||
mock_makedir.return_value = None
|
||||
mock_requests.return_value = None
|
||||
self._open_mock(open_mock)
|
||||
|
||||
template = self.get_template_downloable()
|
||||
files = files_manager.FilesManager(self.get_template_downloable())
|
||||
files._download_url_file(template.Files['file'])
|
||||
|
||||
@mock.patch('os.makedirs')
|
||||
def test_execution_plan_type_downloable_no_Url(self, mock_makedir):
|
||||
"""It validates the URL."""
|
||||
mock_makedir.return_value = None
|
||||
template = bunch.Bunch(
|
||||
ID='ID',
|
||||
Files={
|
||||
'mycoockbook': {
|
||||
'Name': 'mycoockbook.txt',
|
||||
'Type': 'Downloadable'
|
||||
}
|
||||
}
|
||||
)
|
||||
files = files_manager.FilesManager(template)
|
||||
self.assertRaises(ValueError, files._download_url_file,
|
||||
template.Files['mycoockbook'])
|
||||
|
||||
@mock.patch("git.Git")
|
||||
@mock.patch('os.path.isdir')
|
||||
@mock.patch('os.makedirs')
|
||||
def test_putfile_downloable(self, mock_makedir, mock_git, path):
|
||||
"""It tests the putfile method when the file is a git
|
||||
URL.
|
||||
"""
|
||||
path.return_value = True
|
||||
mock_git.clone.return_value = None
|
||||
mock_makedir.return_value = None
|
||||
template = self.get_template_downloable_git()
|
||||
files = files_manager.FilesManager(template)
|
||||
for file in template.get('Files'):
|
||||
files.put_file(file, 'deploy')
|
||||
|
||||
@mock.patch('__builtin__.open')
|
||||
@mock.patch('os.path.lexists')
|
||||
@mock.patch('os.path.isdir')
|
||||
@mock.patch('os.makedirs')
|
||||
def test_putfile_file(self, mock_makedir, mock_path,
|
||||
mock_exists, open_mock):
|
||||
"""It tests the putfile method."""
|
||||
mock_path.return_value = True
|
||||
mock_makedir.return_value = None
|
||||
mock_exists.return_value = True
|
||||
context_manager_mock = mock.Mock()
|
||||
open_mock.return_value = context_manager_mock
|
||||
file_mock = mock.Mock()
|
||||
file_mock.read.return_value = ''
|
||||
enter_mock = mock.Mock()
|
||||
enter_mock.return_value = file_mock
|
||||
exit_mock = mock.Mock()
|
||||
setattr(context_manager_mock, '__enter__', enter_mock)
|
||||
setattr(context_manager_mock, '__exit__', exit_mock)
|
||||
|
||||
template = self.get_template_file()
|
||||
files = files_manager.FilesManager(template)
|
||||
for file in template.get('Files'):
|
||||
files.put_file(file, 'deploy')
|
||||
|
||||
def get_template_downloable_git(self):
|
||||
return bunch.Bunch(
|
||||
ID='ID',
|
||||
Files={
|
||||
'mycoockbook': {
|
||||
'Name': 'mycoockbook.txt',
|
||||
'URL': 'git://github.com/tomcat.git',
|
||||
'Type': 'Downloadable'
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
def get_template_downloable(self):
|
||||
return bunch.Bunch(
|
||||
ID='ID',
|
||||
Files={
|
||||
'file': {
|
||||
'Name': 'myfile',
|
||||
'URL': 'https://www.apache.org/licenses/LICENSE-2.0',
|
||||
'Type': 'Downloadable'
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
def get_template_file(self):
|
||||
return bunch.Bunch(
|
||||
ID='ID',
|
||||
Files={
|
||||
'test': {
|
||||
'Body': 'dGV4dA==\n',
|
||||
'BodyType': 'Base64',
|
||||
'Name': 'installer'
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
def _open_mock(self, open_mock):
|
||||
context_manager_mock = mock.Mock()
|
||||
open_mock.return_value = context_manager_mock
|
||||
file_mock = mock.Mock()
|
||||
file_mock.read.return_value = ''
|
||||
enter_mock = mock.Mock()
|
||||
enter_mock.return_value = file_mock
|
||||
exit_mock = mock.Mock()
|
||||
setattr(context_manager_mock, '__enter__', enter_mock)
|
||||
setattr(context_manager_mock, '__exit__', exit_mock)
|
||||
|
|
@ -14,3 +14,4 @@ stevedore>=1.0.0 # Apache-2.0
|
|||
# not listed in global requirements
|
||||
semver!=2.0
|
||||
bunch
|
||||
requests
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
hacking>=0.8.0,<0.9
|
||||
|
||||
unittest2
|
||||
coverage>=3.6
|
||||
discover
|
||||
mock>=1.0
|
||||
gitpython
|
||||
sphinx>=1.1.2
|
||||
testtools>=0.9.32
|
||||
testrepository>=0.0.18
|
||||
|
|
|
|||
Loading…
Reference in New Issue