163 lines
5.5 KiB
Python
163 lines
5.5 KiB
Python
# Copyright (c) 2013 Mirantis Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import base64
|
|
import git
|
|
import os
|
|
import requests
|
|
import shutil
|
|
import urlparse
|
|
|
|
from oslo_log import log as logging
|
|
|
|
from muranoagent.common import config
|
|
|
|
CONF = config.CONF
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
class FilesManager(object):
|
|
def __init__(self, execution_plan):
|
|
self._fetched_files = {}
|
|
self._files = execution_plan.get('Files') or {}
|
|
|
|
self._cache_folder = os.path.join(
|
|
CONF.storage, 'files', execution_plan.ID)
|
|
if os.path.exists(self._cache_folder):
|
|
self.clear()
|
|
os.makedirs(self._cache_folder)
|
|
|
|
def put_file(self, file_id, script):
|
|
if type(file_id) is dict:
|
|
file_name = file_id.keys()[0]
|
|
file_def = file_id[file_name]
|
|
else:
|
|
file_def = self._files[file_id]
|
|
file_name = file_def['Name']
|
|
|
|
if file_def.get('Type') == 'Downloadable':
|
|
self._download_url_file(file_def)
|
|
return self._cache_folder
|
|
else:
|
|
return self._make_symlink(file_id, file_name, script)
|
|
|
|
def _make_symlink(self, file_id, file_name, script):
|
|
cache_path = self._fetch_file(file_id)
|
|
script_folder = os.path.join(self._cache_folder, script)
|
|
if not os.path.isdir(script_folder):
|
|
os.mkdir(script_folder)
|
|
|
|
file_folder = os.path.join(script_folder,
|
|
os.path.dirname(file_name))
|
|
if not os.path.isdir(file_folder):
|
|
os.makedirs(file_folder)
|
|
|
|
if cache_path is not None:
|
|
script_path = os.path.join(script_folder, file_name)
|
|
if not os.path.lexists(script_path):
|
|
os.symlink(cache_path, script_path)
|
|
return script_path
|
|
|
|
def _fetch_file(self, file_id):
|
|
if file_id in self._fetched_files:
|
|
return self._fetched_files[file_id]
|
|
|
|
filedef = self._files[file_id]
|
|
out_path = os.path.join(self._cache_folder, file_id)
|
|
body_type = filedef.get('BodyType', 'Text')
|
|
with open(out_path, 'w') as out_file:
|
|
if body_type == 'Text':
|
|
out_file.write(filedef['Body'])
|
|
elif body_type == 'Base64':
|
|
out_file.write(base64.b64decode(filedef['Body']))
|
|
|
|
self._fetched_files[file_id] = out_path
|
|
return out_path
|
|
|
|
def _download_url_file(self, file_def):
|
|
"""It download the file in the murano-agent.
|
|
|
|
It can proceed from a git file or any other internal URL
|
|
|
|
"""
|
|
|
|
if 'URL' not in file_def:
|
|
raise ValueError("No valid URL in file {0}".
|
|
format(file_def))
|
|
url_file = file_def['URL']
|
|
|
|
if not self._url(url_file):
|
|
raise ValueError("Provided URL is not valid {0}".
|
|
format(url_file))
|
|
|
|
if not os.path.isdir(os.path.join(self._cache_folder, 'files')):
|
|
os.makedirs(os.path.join(self._cache_folder, 'files'))
|
|
|
|
folder = self._get_file_folder(url_file, file_def['Name'])
|
|
|
|
if not os.path.isdir(folder):
|
|
os.makedirs(folder)
|
|
|
|
try:
|
|
if self._is_git_repository(url_file):
|
|
git.Git().clone(url_file, folder)
|
|
else:
|
|
self._download_file(url_file, folder)
|
|
except Exception as e:
|
|
if self._is_git_repository(url_file):
|
|
mns = ("Error to clone the git repository {0}: {1}".
|
|
format(url_file, e.message))
|
|
else:
|
|
mns = ("Error to download the file {0}: {1}".
|
|
format(url_file, e.message))
|
|
LOG.warn(mns)
|
|
raise ValueError(mns)
|
|
|
|
def clear(self):
|
|
shutil.rmtree(self._cache_folder, ignore_errors=True)
|
|
shutil.rmtree(self._cache_folder, ignore_errors=True)
|
|
|
|
def _download_file(self, url, path):
|
|
local_filename = url.split('/')[-1]
|
|
r = requests.get(url, stream=True)
|
|
with open(os.path.join(path, local_filename), 'wb') as f:
|
|
for chunk in r.iter_content(chunk_size=1024):
|
|
if chunk:
|
|
f.write(chunk)
|
|
f.flush()
|
|
return local_filename
|
|
|
|
def _url(self, file):
|
|
return (urlparse.urlsplit(file).scheme or
|
|
urlparse.urlsplit(file).netloc)
|
|
|
|
def _get_file_folder(self, file_url, folder_name):
|
|
if folder_name is None:
|
|
if file_url.endswith('.git'):
|
|
file_folder = file_url[file_url.rfind('/') +
|
|
1: file_url.find('.git')]
|
|
else:
|
|
file_folder = file_url[file_url.rfind('/') + 1:]
|
|
|
|
folder = os.path.join(self._cache_folder, 'files', file_folder)
|
|
else:
|
|
folder = os.path.join(self._cache_folder, 'files', folder_name)
|
|
return folder
|
|
|
|
def _is_git_repository(self, url):
|
|
return (url.startswith(("git://",
|
|
"git+http://", "git+https:/"))
|
|
or url.endswith('.git'))
|