[Feat] Support remote tarball as chart source

-Add functionality to download a tarball from a URL, decompress/extract
 the archive, and use as a chart source
-Compartmentalized functionality to later support extracting, but not
 downloading, local tarballs
-Refactor specific git utils to general source utils
-Small exception handling bug fix
This commit is contained in:
Tim Heyer 2017-07-28 17:43:04 +00:00 committed by Alexis Rivera DeLa Torre
parent bbc88b49d3
commit 4554cac0d9
13 changed files with 352 additions and 170 deletions

View File

@ -23,15 +23,3 @@ class KnownReleasesException(ArmadaException):
'''Exception that occurs when no known releases are found'''
message = 'No known releases found'
class ChartSourceException(ArmadaException):
'''Exception for unknown chart source type.'''
def __init__(self, chart_name, source_type):
self._chart_name = chart_name
self._source_type = source_type
self._message = 'Unknown source type \"' + self._source_type + '\" for \
chart \"' + self._chart_name + '\"'
super(ChartSourceException, self).__init__(self._message)

View File

@ -23,7 +23,7 @@ class DependencyException(ChartBuilderException):
'''Exception that occurs when dependencies cannot be resolved.'''
def __init__(self, chart_name):
self._chart_name
self._chart_name = chart_name
self._message = 'Failed to resolve dependencies for ' + \
self._chart_name + '.'
@ -33,7 +33,7 @@ class HelmChartBuildException(ChartBuilderException):
'''Exception that occurs when Helm Chart fails to build.'''
def __init__(self, chart_name):
self._chart_name
self._chart_name = chart_name
self._message = 'Failed to build Helm chart for ' + \
self._chart_name + '.'

View File

@ -1,38 +0,0 @@
# Copyright 2017 The Armada Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base_exception
class GitException(base_exception.ArmadaBaseException):
'''Base class for Git exceptions and error handling.'''
message = 'An unknown error occured while cloning a Git repository.'
class GitLocationException(GitException):
'''Exception that occurs when an error occurs cloning a Git repository.'''
def __init__(self, location):
self._location = location
self._message = self._location + ' is not a valid git repository.'
super(GitLocationException, self).__init__(self._message)
class SourceCleanupException(GitException):
'''Exception that occurs for an invalid dir.'''
def __init__(self, target_dir):
self._target_dir = target_dir
self._message = self._target_dir + ' is not a valid directory.'
super(SourceCleanupException, self).__init__(self._message)

View File

@ -0,0 +1,80 @@
# Copyright 2017 The Armada Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base_exception
class SourceException(base_exception.ArmadaBaseException):
'''Base class for Git exceptions and error handling.'''
message = 'An unknown error occured while accessing a chart source'
class GitLocationException(SourceException):
'''Exception that occurs when an error occurs cloning a Git repository.'''
def __init__(self, location):
self._location = location
self._message = self._location + ' is not a valid git repository.'
super(GitLocationException, self).__init__(self._message)
class SourceCleanupException(SourceException):
'''Exception that occurs for an invalid dir.'''
def __init__(self, target_dir):
self._target_dir = target_dir
self._message = self._target_dir + ' is not a valid directory.'
super(SourceCleanupException, self).__init__(self._message)
class TarballDownloadException(SourceException):
'''Exception that occurs when the tarball cannot be downloaded
from the provided URL
'''
def __init__(self, tarball_url):
self._tarball_url = tarball_url
self._message = 'Unable to download from ' + self._tarball_url
super(TarballDownloadException, self).__init__(self._message)
class TarballExtractException(SourceException):
'''Exception that occurs when extracting the tarball fails'''
def __init__(self, tarball_dir):
self._tarball_dir = tarball_dir
self._message = 'Unable to extract ' + self._tarball_dir
super(TarballExtractException, self).__init__(self._message)
class InvalidPathException(SourceException):
'''Exception that occurs when a nonexistant path is accessed'''
def __init__(self, path):
self._path = path
self._message = 'Unable to access path ' + self._path
super(InvalidPathException, self).__init__(self._message)
class ChartSourceException(SourceException):
'''Exception for unknown chart source type.'''
def __init__(self, chart_name, source_type):
self._chart_name = chart_name
self._source_type = source_type
self._message = 'Unknown source type \"' + self._source_type + '\" for \
chart \"' + self._chart_name + '\"'
super(ChartSourceException, self).__init__(self._message)

View File

@ -24,12 +24,12 @@ from tiller import Tiller
from manifest import Manifest
from ..exceptions import armada_exceptions
from ..exceptions import git_exceptions
from ..exceptions import source_exceptions
from ..exceptions import lint_exceptions
from ..exceptions import tiller_exceptions
from ..utils.release import release_prefix
from ..utils import git
from ..utils import source
from ..utils import lint
from ..const import KEYWORD_ARMADA, KEYWORD_GROUPS, KEYWORD_CHARTS,\
KEYWORD_PREFIX, STATUS_FAILED
@ -142,20 +142,24 @@ class Armada(object):
if ct_type == 'local':
ch.get('chart')['source_dir'] = (location, subpath)
elif ct_type == 'tar':
LOG.info('Downloading tarball from: %s', location)
tarball_dir = source.get_tarball(location)
ch.get('chart')['source_dir'] = (tarball_dir, subpath)
elif ct_type == 'git':
if location not in repos.keys():
try:
LOG.info('Cloning repo: %s', location)
repo_dir = git.git_clone(location, reference)
repo_dir = source.git_clone(location, reference)
except Exception:
raise git_exceptions.GitLocationException(location)
raise source_exceptions.GitLocationException(location)
repos[location] = repo_dir
ch.get('chart')['source_dir'] = (repo_dir, subpath)
else:
ch.get('chart')['source_dir'] = (repos.get(location), subpath)
else:
chart_name = ch.get('chart').get('chart_name')
raise armada_exceptions.ChartSourceException(ct_type, chart_name)
raise source_exceptions.ChartSourceException(ct_type, chart_name)
def get_releases_by_status(self, status):
'''
@ -305,7 +309,7 @@ class Armada(object):
for group in self.config.get(KEYWORD_ARMADA).get(KEYWORD_GROUPS):
for ch in group.get(KEYWORD_CHARTS):
if ch.get('chart').get('source').get('type') == 'git':
git.source_cleanup(ch.get('chart').get('source_dir')[0])
source.source_cleanup(ch.get('chart').get('source_dir')[0])
def show_diff(self, chart, installed_chart, installed_values, target_chart,
target_values):

View File

@ -202,7 +202,7 @@ class ChartBuilder(object):
templates=self.get_templates(),
dependencies=dependencies,
values=self.get_values(),
files=self.get_files(), )
files=self.get_files())
except Exception:
chart_name = self.chart.chart_name
raise chartbuilder_exceptions.HelmChartBuildException(chart_name)

View File

@ -1,67 +0,0 @@
# Copyright 2017 The Armada Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
import unittest
from armada.exceptions import git_exceptions
from armada.utils import git
class GitTestCase(unittest.TestCase):
@mock.patch('armada.utils.git.tempfile')
@mock.patch('armada.utils.git.pygit2')
def test_git_clone_good_url(self, mock_pygit, mock_temp):
mock_temp.mkdtemp.return_value = '/tmp/armada'
mock_pygit.clone_repository.return_value = "Repository"
url = 'http://github.com/att-comdev/armada'
dir = git.git_clone(url)
self.assertIsNotNone(dir)
def test_git_clone_empty_url(self):
url = ''
with self.assertRaises(Exception):
self.assertFalse(git.git_clone(url))
def test_git_clone_bad_url(self):
url = 'http://github.com/dummy/armada'
with self.assertRaises(Exception):
git.git_clone(url)
@mock.patch('armada.utils.git.shutil')
@mock.patch('armada.utils.git.path')
def test_source_cleanup(self, mock_path, mock_shutil):
mock_path.exists.return_value = True
path = 'armada'
try:
git.source_cleanup(path)
except git_exceptions.SourceCleanupException:
pass
mock_shutil.rmtree.assert_called_with(path)
@unittest.skip('not handled correctly')
@mock.patch('armada.utils.git.shutil')
@mock.patch('armada.utils.git.path')
def test_source_cleanup_bad_path(self, mock_path, mock_shutil):
mock_path.exists.return_value = False
path = 'armada'
with self.assertRaises(Exception):
git.source_cleanup(path)
mock_shutil.rmtree.assert_not_called()

View File

@ -0,0 +1,117 @@
# Copyright 2017 The Armada Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
import unittest
from armada.exceptions import source_exceptions
from armada.utils import source
class GitTestCase(unittest.TestCase):
SOURCE_UTILS_LOCATION = 'armada.utils.source'
@mock.patch('armada.utils.source.tempfile')
@mock.patch('armada.utils.source.pygit2')
def test_git_clone_good_url(self, mock_pygit, mock_temp):
mock_temp.mkdtemp.return_value = '/tmp/armada'
mock_pygit.clone_repository.return_value = "Repository"
url = 'http://github.com/att-comdev/armada'
dir = source.git_clone(url)
self.assertIsNotNone(dir)
def test_git_clone_empty_url(self):
url = ''
with self.assertRaises(Exception):
self.assertFalse(source.git_clone(url))
def test_git_clone_bad_url(self):
url = 'http://github.com/dummy/armada'
with self.assertRaises(Exception):
source.git_clone(url)
@mock.patch('armada.utils.source.tempfile')
@mock.patch('armada.utils.source.requests')
def test_tarball_download(self, mock_requests, mock_temp):
url = 'http://localhost:8879/charts/mariadb-0.1.0.tgz'
mock_temp.mkstemp.return_value = (None, '/tmp/armada')
mock_response = mock.Mock()
mock_response.content = 'some string'
mock_requests.get.return_value = mock_response
mock_open = mock.mock_open()
with mock.patch('{}.open'.format(self.SOURCE_UTILS_LOCATION),
mock_open, create=True):
source.download_tarball(url)
mock_temp.mkstemp.assert_called_once()
mock_requests.get.assert_called_once_with(url)
mock_open.assert_called_once_with('/tmp/armada', 'wb')
mock_open().write.assert_called_once_with(
mock_requests.get(url).content)
@mock.patch('armada.utils.source.tempfile')
@mock.patch('armada.utils.source.path')
@mock.patch('armada.utils.source.tarfile')
def test_tarball_extract(self, mock_tarfile, mock_path, mock_temp):
mock_path.exists.return_value = True
mock_temp.mkdtemp.return_value = '/tmp/armada'
mock_opened_file = mock.Mock()
mock_tarfile.open.return_value = mock_opened_file
path = '/tmp/mariadb-0.1.0.tgz'
source.extract_tarball(path)
mock_path.exists.assert_called_once()
mock_temp.mkdtemp.assert_called_once()
mock_tarfile.open.assert_called_once_with(path)
mock_opened_file.extractall.assert_called_once_with('/tmp/armada')
@mock.patch('armada.utils.source.path')
@mock.patch('armada.utils.source.tarfile')
def test_tarball_extract_bad_path(self, mock_tarfile, mock_path):
mock_path.exists.return_value = False
path = '/tmp/armada'
with self.assertRaises(Exception):
source.extract_tarball(path)
mock_tarfile.open.assert_not_called()
mock_tarfile.extractall.assert_not_called()
@mock.patch('armada.utils.source.shutil')
@mock.patch('armada.utils.source.path')
def test_source_cleanup(self, mock_path, mock_shutil):
mock_path.exists.return_value = True
path = 'armada'
try:
source.source_cleanup(path)
except source_exceptions.SourceCleanupException:
pass
mock_shutil.rmtree.assert_called_with(path)
@unittest.skip('not handled correctly')
@mock.patch('armada.utils.source.shutil')
@mock.patch('armada.utils.source.path')
def test_source_cleanup_bad_path(self, mock_path, mock_shutil):
mock_path.exists.return_value = False
path = 'armada'
with self.assertRaises(Exception):
source.source_cleanup(path)
mock_shutil.rmtree.assert_not_called()

View File

@ -1,30 +0,0 @@
import pygit2
import tempfile
import shutil
from os import path
from ..exceptions import git_exceptions
def git_clone(repo_url, branch='master'):
'''
clones repo to a /tmp/ dir
'''
if repo_url == '':
raise git_exceptions.GitLocationException(repo_url)
_tmp_dir = tempfile.mkdtemp(prefix='armada', dir='/tmp')
try:
pygit2.clone_repository(repo_url, _tmp_dir, checkout_branch=branch)
except Exception:
raise git_exceptions.GitLocationException(repo_url)
return _tmp_dir
def source_cleanup(target_dir):
'''
Clean up source
'''
if path.exists(target_dir):
shutil.rmtree(target_dir)

65
armada/utils/source.py Normal file
View File

@ -0,0 +1,65 @@
import pygit2
import requests
import tarfile
import tempfile
import shutil
from os import path
from ..exceptions import source_exceptions
def git_clone(repo_url, branch='master'):
'''
clones repo to a /tmp/ dir
'''
if repo_url == '':
raise source_exceptions.GitLocationException(repo_url)
_tmp_dir = tempfile.mkdtemp(prefix='armada', dir='/tmp')
try:
pygit2.clone_repository(repo_url, _tmp_dir, checkout_branch=branch)
except Exception:
raise source_exceptions.GitLocationException(repo_url)
return _tmp_dir
def get_tarball(tarball_url):
tarball_path = download_tarball(tarball_url)
return extract_tarball(tarball_path)
def download_tarball(tarball_url):
'''
Downloads a tarball to /tmp and returns the path
'''
try:
tarball_filename = tempfile.mkstemp(prefix='armada', dir='/tmp')[1]
response = requests.get(tarball_url)
with open(tarball_filename, 'wb') as f:
f.write(response.content)
except Exception:
raise source_exceptions.TarballDownloadException(tarball_url)
return tarball_filename
def extract_tarball(tarball_path):
'''
Extracts a tarball to /tmp and returns the path
'''
if not path.exists(tarball_path):
raise source_exceptions.InvalidPathException(tarball_path)
_tmp_dir = tempfile.mkdtemp(prefix='armada', dir='/tmp')
try:
file = tarfile.open(tarball_path)
file.extractall(_tmp_dir)
except Exception:
raise source_exceptions.TarballExtractException(tarball_path)
return _tmp_dir
def source_cleanup(target_dir):
'''
Clean up source
'''
if path.exists(target_dir):
shutil.rmtree(target_dir)

View File

@ -151,7 +151,7 @@ Source
+-------------+----------+---------------------------------------------------------------+
| keyword | type | action |
+=============+==========+===============================================================+
| type | string | source to build the chart: ``git`` or ``local`` |
| type | string | source to build the chart: ``git``, ``local``, or ``tar`` |
+-------------+----------+---------------------------------------------------------------+
| location | string | ``url`` or ``path`` to the chart's parent directory |
+-------------+----------+---------------------------------------------------------------+
@ -247,9 +247,9 @@ Simple Example
namespace: default
values: {}
source:
type: git
location: https://github.com/namespace/repo
subpath: .
type: tar
location: http://localhost:8879/namespace/repo
subpath: blog-2
reference: master
dependencies: []
---

View File

@ -4,8 +4,6 @@ Armada Exceptions
+------------------------+----------------------------------------------------------+
| Exception | Error Description |
+========================+==========================================================+
| ChartSourceException | Occurs when an unknown chart source type is encountered. |
+------------------------+----------------------------------------------------------+
| KnownReleasesException | Occurs when no known releases are found. |
+------------------------+----------------------------------------------------------+
@ -53,16 +51,24 @@ Chartbuilder Exceptions
| UnknownChartSourceException | The chart source is unknown or invalid. |
+-----------------------------+-------------------------------------------------------------+
Git Exceptions
Source Exceptions
===============
+------------------------+---------------------------------------------+
| Exception | Error Description |
+========================+=============================================+
| GitLocationException | Repository location is not valid. |
+------------------------+---------------------------------------------+
| SourceCleanupException | The source dir of a chart no longer exists. |
+------------------------+---------------------------------------------+
+--------------------------+---------------------------------------------------------------------+
| Exception | Error Description |
+==========================+=====================================================================+
| GitLocationException | Repository location is not valid. |
+--------------------------+---------------------------------------------------------------------+
| SourceCleanupException | The source dir of a chart no longer exists. |
+--------------------------+---------------------------------------------------------------------+
| TarballDownloadException | Occurs when the tarball cannot be downloaded from the provided URL. |
+--------------------------+---------------------------------------------------------------------+
| TarballExtractException | Occurs when extracting a tarball fails. |
+--------------------------+---------------------------------------------------------------------+
| InvalidPathException | Occurs when a nonexistant path is accessed. |
+--------------------------+---------------------------------------------------------------------+
| ChartSourceException | Occurs when an unknown chart source type is encountered. |
+--------------------------+---------------------------------------------------------------------+
Lint Exceptions
===============

57
examples/tar_example.yaml Normal file
View File

@ -0,0 +1,57 @@
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: helm-toolkit
data:
chart_name: helm-toolkit
release: helm-toolkit
namespace: helm-tookit
values: {}
source:
type: git
location: git://github.com/openstack/openstack-helm
subpath: helm-toolkit
reference: master
dependencies: []
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: mariadb-tarball
data:
chart_name: mariadb-tarball
release: mariadb-tarball
namespace: tarball
timeout: 3600
install:
no_hooks: false
upgrade:
no_hooks: false
values: {}
source:
type: tar
location: http://localhost:8879/charts/mariadb-0.1.0.tgz
subpath: mariadb
reference: null
dependencies:
- helm-toolkit
---
schema: armada/ChartGroup/v1
metadata:
schema: metadata/Document/v1
name: tar-example
data:
description: "Deploying mariadb tarball URL"
sequenced: True
chart_group:
- mariadb-tarball
---
schema: armada/Manifest/v1
metadata:
schema: metadata/Document/v1
name: armada-manifest
data:
release_prefix: armada
chart_groups:
- tar-example