diff --git a/muranoagent/files_manager.py b/muranoagent/files_manager.py index d899d008..25c15bb8 100644 --- a/muranoagent/files_manager.py +++ b/muranoagent/files_manager.py @@ -4,7 +4,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -14,11 +14,16 @@ # limitations under the License. import base64 +import git import os +import requests import shutil +import subprocess +import urlparse from muranoagent.common import config + CONF = config.CONF @@ -34,22 +39,33 @@ class FilesManager(object): os.makedirs(self._cache_folder) def put_file(self, file_id, script): + if type(file_id) is dict: + file_name = file_id.keys()[0] + file_def = file_id[file_name] + else: + file_name = file_id + file_def = self._files[file_name] + + if file_def.get('Type') == 'Downloadable': + self._download_url_file(file_def) + return self._cache_folder + else: + return self._make_symlink(file_id, file_name, script) + + def _make_symlink(self, file_id, file_name, script): cache_path = self._fetch_file(file_id) - script_folder = os.path.join(self._cache_folder, script) - if not os.path.exists(script_folder): + if not os.path.isdir(script_folder): os.mkdir(script_folder) + file_folder = os.path.join(script_folder, + os.path.dirname(file_name)) + if not os.path.isdir(file_folder): + os.makedirs(file_folder) - filedef = self._files[file_id] - filename = filedef['Name'] - - file_folder = os.path.join(script_folder, os.path.dirname(filename)) - if not os.path.exists(file_folder): - os.makedirs(file_folder) - - script_path = os.path.join(script_folder, filename) - - os.symlink(cache_path, script_path) + if cache_path is not None: + script_path = os.path.join(os.getcwd(), script_folder, file_name) + if not os.path.lexists(script_path): + os.symlink(cache_path, script_path) return script_path def _fetch_file(self, file_id): @@ -68,7 +84,59 @@ class FilesManager(object): self._fetched_files[file_id] = out_path return out_path + def _download_url_file(self, file_def): + """It download the file in the murano-agent. It can proceed + from a git file or any other internal URL + """ + + if 'URL' not in file_def: + raise ValueError("No valid URL in file {0}". + format(file_def)) + url_file = file_def['URL'] + + if not self._url(url_file): + raise ValueError("Provided URL is not valid {0}". + format(url_file)) + + if not os.path.isdir(os.path.join(self._cache_folder, 'files')): + os.makedirs(os.path.join(self._cache_folder, 'files')) + + folder = self._get_file_folder(url_file) + + if self._is_git_repository(url_file): + if not os.path.isdir(folder): + git.Git().clone(url_file, folder) + else: + self._download_file(url_file, folder) + def clear(self): - os.chdir(os.path.dirname(self._cache_folder)) shutil.rmtree(self._cache_folder, ignore_errors=True) shutil.rmtree(self._cache_folder, ignore_errors=True) + + def _download_file(self, url, path): + local_filename = url.split('/')[-1] + r = requests.get(url, stream=True) + with open(os.path.join(path, local_filename), 'wb') as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: + f.write(chunk) + f.flush() + return local_filename + + def _url(self, file): + return (urlparse.urlsplit(file).scheme or + urlparse.urlsplit(file).netloc) + + def _get_file_folder(self, file_url): + if file_url.endswith('.git'): + file_folder = file_url[file_url.rfind('/') + 1: + file_url.find('.git')] + else: + file_folder = file_url[file_url.rfind('/') + 1:] + + folder = os.path.join(self._cache_folder, 'files', file_folder) + return folder + + def _is_git_repository(self, url): + return url.startswith(("git://", + "git+http://", "git+https:/")) diff --git a/muranoagent/tests/__init__.py b/muranoagent/tests/__init__.py index 7d93825c..e69de29b 100644 --- a/muranoagent/tests/__init__.py +++ b/muranoagent/tests/__init__.py @@ -1,13 +0,0 @@ -# Copyright (c) 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. diff --git a/muranoagent/tests/unit/__init__.py b/muranoagent/tests/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/muranoagent/tests/unit/base.py b/muranoagent/tests/unit/base.py new file mode 100644 index 00000000..ca0bf286 --- /dev/null +++ b/muranoagent/tests/unit/base.py @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fixtures +from oslo.config import cfg +import testtools + +CONF = cfg.CONF + + +class MuranoAgentTestCase(testtools.TestCase): + + def setUp(self): + super(MuranoAgentTestCase, self).setUp() + self.useFixture(fixtures.FakeLogger('murano-agent')) + + def override_config(self, name, override, group=None): + CONF.set_override(name, override, group) + self.addCleanup(CONF.clear_override, name, group) diff --git a/muranoagent/tests/test_agent.py b/muranoagent/tests/unit/test_agent.py similarity index 100% rename from muranoagent/tests/test_agent.py rename to muranoagent/tests/unit/test_agent.py diff --git a/muranoagent/tests/unit/test_files_manager.py b/muranoagent/tests/unit/test_files_manager.py new file mode 100644 index 00000000..c7be2f83 --- /dev/null +++ b/muranoagent/tests/unit/test_files_manager.py @@ -0,0 +1,192 @@ +# Copyright (c) 2015 Telefonica I+D +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import bunch +import git +import mock + +from muranoagent.common import config as cfg +from muranoagent import files_manager +from muranoagent.tests.unit import base + +CONF = cfg.CONF + + +class TestFileManager(base.MuranoAgentTestCase): + + @mock.patch('os.path.isdir') + @mock.patch('os.mkdir') + @mock.patch('os.makedirs') + def setUp(self, mock_makedir, mock_mkdir, mock_path): + mock_path.return_value = True + mock_mkdir.return_value = None + mock_makedir.return_value = None + super(TestFileManager, self).setUp() + CONF.set_override('storage', 'cache') + + @mock.patch('os.makedirs') + def test_get_folder_git(self, mock_path): + """It gets the folder where the URL is a git URL.""" + mock_path.return_value = None + files = files_manager.FilesManager(self.get_template_downloable()) + folder = files._get_file_folder("http://tomcat.git") + self.assertEqual(folder, "cache/files/ID/files/tomcat") + + @mock.patch('os.makedirs') + def test_get_folder_not_git(self, mock_path): + """It gets the folder from the URL.""" + mock_path.return_value = None + files = files_manager.FilesManager(self.get_template_downloable()) + folder = files._get_file_folder("http://tomcat") + self.assertEqual(folder, "cache/files/ID/files/tomcat") + + @mock.patch("git.Git") + @mock.patch('os.path.isdir') + @mock.patch('os.makedirs') + def test_execution_plan_type_downloable_git(self, mock_makedir, mock_path, + mock_git): + """It tests an execution plan when there are files + which should be downloable. + """ + mock_makedir.return_value = None + mock_path.return_value = True + mock_git.clone.return_value = None + template = self.get_template_downloable_git() + files = files_manager.FilesManager(self.get_template_downloable()) + files._download_url_file(template.Files['mycoockbook']) + + @mock.patch('os.path.isdir') + @mock.patch('os.mkdir') + @mock.patch('os.makedirs') + @mock.patch('__builtin__.open') + @mock.patch('requests.get') + def test_execution_plan_type_downloable(self, mock_requests, open_mock, + mock_makedir, + mock_mkdir, mock_path): + """It tests an execution plan when there are files + which should be downloable. + """ + mock_path.return_value = True + mock_mkdir.return_value = None + mock_makedir.return_value = None + mock_requests.return_value = None + self._open_mock(open_mock) + + template = self.get_template_downloable() + files = files_manager.FilesManager(self.get_template_downloable()) + files._download_url_file(template.Files['file']) + + @mock.patch('os.makedirs') + def test_execution_plan_type_downloable_no_Url(self, mock_makedir): + """It validates the URL.""" + mock_makedir.return_value = None + template = bunch.Bunch( + ID='ID', + Files={ + 'mycoockbook': { + 'Name': 'mycoockbook.txt', + 'Type': 'Downloadable' + } + } + ) + files = files_manager.FilesManager(template) + self.assertRaises(ValueError, files._download_url_file, + template.Files['mycoockbook']) + + @mock.patch("git.Git") + @mock.patch('os.path.isdir') + @mock.patch('os.makedirs') + def test_putfile_downloable(self, mock_makedir, mock_git, path): + """It tests the putfile method when the file is a git + URL. + """ + path.return_value = True + mock_git.clone.return_value = None + mock_makedir.return_value = None + template = self.get_template_downloable_git() + files = files_manager.FilesManager(template) + for file in template.get('Files'): + files.put_file(file, 'deploy') + + @mock.patch('__builtin__.open') + @mock.patch('os.path.lexists') + @mock.patch('os.path.isdir') + @mock.patch('os.makedirs') + def test_putfile_file(self, mock_makedir, mock_path, + mock_exists, open_mock): + """It tests the putfile method.""" + mock_path.return_value = True + mock_makedir.return_value = None + mock_exists.return_value = True + context_manager_mock = mock.Mock() + open_mock.return_value = context_manager_mock + file_mock = mock.Mock() + file_mock.read.return_value = '' + enter_mock = mock.Mock() + enter_mock.return_value = file_mock + exit_mock = mock.Mock() + setattr(context_manager_mock, '__enter__', enter_mock) + setattr(context_manager_mock, '__exit__', exit_mock) + + template = self.get_template_file() + files = files_manager.FilesManager(template) + for file in template.get('Files'): + files.put_file(file, 'deploy') + + def get_template_downloable_git(self): + return bunch.Bunch( + ID='ID', + Files={ + 'mycoockbook': { + 'Name': 'mycoockbook.txt', + 'URL': 'git://github.com/tomcat.git', + 'Type': 'Downloadable' + } + } + ) + + def get_template_downloable(self): + return bunch.Bunch( + ID='ID', + Files={ + 'file': { + 'Name': 'myfile', + 'URL': 'https://www.apache.org/licenses/LICENSE-2.0', + 'Type': 'Downloadable' + } + } + ) + + def get_template_file(self): + return bunch.Bunch( + ID='ID', + Files={ + 'test': { + 'Body': 'dGV4dA==\n', + 'BodyType': 'Base64', + 'Name': 'installer' + } + } + ) + + def _open_mock(self, open_mock): + context_manager_mock = mock.Mock() + open_mock.return_value = context_manager_mock + file_mock = mock.Mock() + file_mock.read.return_value = '' + enter_mock = mock.Mock() + enter_mock.return_value = file_mock + exit_mock = mock.Mock() + setattr(context_manager_mock, '__enter__', enter_mock) + setattr(context_manager_mock, '__exit__', exit_mock) diff --git a/requirements.txt b/requirements.txt index 28ee1058..a5aa48d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,4 @@ stevedore>=1.0.0 # Apache-2.0 # not listed in global requirements semver!=2.0 bunch +requests diff --git a/test-requirements.txt b/test-requirements.txt index 741cb107..8e6df42c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,8 +1,9 @@ hacking>=0.8.0,<0.9 - unittest2 coverage>=3.6 discover +mock>=1.0 +gitpython sphinx>=1.1.2 testtools>=0.9.32 testrepository>=0.0.18