airship/armada
Code Issues Proposed changes

[Feat] Support remote tarball as chart source

Browse Source 
-Add functionality to download a tarball from a URL, decompress/extract
 the archive, and use as a chart source
-Compartmentalized functionality to later support extracting, but not
 downloading, local tarballs
-Refactor specific git utils to general source utils
-Small exception handling bug fix
This commit is contained in:
Tim Heyer 2017-07-28 17:43:04 +00:00 committed by Alexis Rivera DeLa Torre
parent bbc88b49d3
commit 4554cac0d9
13 changed files with 352 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
armada/exceptions/armada_exceptions.py
View File

@ -23,15 +23,3 @@ class KnownReleasesException(ArmadaException):
'''Exception that occurs when no known releases are found'''
message = 'No known releases found'
class ChartSourceException(ArmadaException):
'''Exception for unknown chart source type.'''
def __init__(self, chart_name, source_type):
self._chart_name = chart_name
self._source_type = source_type
self._message = 'Unknown source type \"' + self._source_type + '\" for \
chart \"' + self._chart_name + '\"'
super(ChartSourceException, self).__init__(self._message)

4
armada/exceptions/chartbuilder_exceptions.py
View File

@ -23,7 +23,7 @@ class DependencyException(ChartBuilderException):
'''Exception that occurs when dependencies cannot be resolved.'''
def __init__(self, chart_name):
self._chart_name
self._chart_name = chart_name
self._message = 'Failed to resolve dependencies for ' + \
self._chart_name + '.'
@ -33,7 +33,7 @@ class HelmChartBuildException(ChartBuilderException):
'''Exception that occurs when Helm Chart fails to build.'''
def __init__(self, chart_name):
self._chart_name
self._chart_name = chart_name
self._message = 'Failed to build Helm chart for ' + \
self._chart_name + '.'

38
armada/exceptions/git_exceptions.py
View File

@ -1,38 +0,0 @@
# Copyright 2017 The Armada Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base_exception
class GitException(base_exception.ArmadaBaseException):
'''Base class for Git exceptions and error handling.'''
message = 'An unknown error occured while cloning a Git repository.'
class GitLocationException(GitException):
'''Exception that occurs when an error occurs cloning a Git repository.'''
def __init__(self, location):
self._location = location
self._message = self._location + ' is not a valid git repository.'
super(GitLocationException, self).__init__(self._message)
class SourceCleanupException(GitException):
'''Exception that occurs for an invalid dir.'''
def __init__(self, target_dir):
self._target_dir = target_dir
self._message = self._target_dir + ' is not a valid directory.'
super(SourceCleanupException, self).__init__(self._message)

80
armada/exceptions/source_exceptions.py Normal file
View File

@ -0,0 +1,80 @@
# Copyright 2017 The Armada Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base_exception
class SourceException(base_exception.ArmadaBaseException):
'''Base class for Git exceptions and error handling.'''
message = 'An unknown error occured while accessing a chart source'
class GitLocationException(SourceException):
'''Exception that occurs when an error occurs cloning a Git repository.'''
def __init__(self, location):
self._location = location
self._message = self._location + ' is not a valid git repository.'
super(GitLocationException, self).__init__(self._message)
class SourceCleanupException(SourceException):
'''Exception that occurs for an invalid dir.'''
def __init__(self, target_dir):
self._target_dir = target_dir
self._message = self._target_dir + ' is not a valid directory.'
super(SourceCleanupException, self).__init__(self._message)
class TarballDownloadException(SourceException):
'''Exception that occurs when the tarball cannot be downloaded
from the provided URL
'''
def __init__(self, tarball_url):
self._tarball_url = tarball_url
self._message = 'Unable to download from ' + self._tarball_url
super(TarballDownloadException, self).__init__(self._message)
class TarballExtractException(SourceException):
'''Exception that occurs when extracting the tarball fails'''
def __init__(self, tarball_dir):
self._tarball_dir = tarball_dir
self._message = 'Unable to extract ' + self._tarball_dir
super(TarballExtractException, self).__init__(self._message)
class InvalidPathException(SourceException):
'''Exception that occurs when a nonexistant path is accessed'''
def __init__(self, path):
self._path = path
self._message = 'Unable to access path ' + self._path
super(InvalidPathException, self).__init__(self._message)
class ChartSourceException(SourceException):
'''Exception for unknown chart source type.'''
def __init__(self, chart_name, source_type):
self._chart_name = chart_name
self._source_type = source_type
self._message = 'Unknown source type \"' + self._source_type + '\" for \
chart \"' + self._chart_name + '\"'
super(ChartSourceException, self).__init__(self._message)

16
armada/handlers/armada.py
View File

@ -24,12 +24,12 @@ from tiller import Tiller
from manifest import Manifest
from ..exceptions import armada_exceptions
from ..exceptions import git_exceptions
from ..exceptions import source_exceptions
from ..exceptions import lint_exceptions
from ..exceptions import tiller_exceptions
from ..utils.release import release_prefix
from ..utils import git
from ..utils import source
from ..utils import lint
from ..const import KEYWORD_ARMADA, KEYWORD_GROUPS, KEYWORD_CHARTS,\
KEYWORD_PREFIX, STATUS_FAILED
@ -142,20 +142,24 @@ class Armada(object):
if ct_type == 'local':
ch.get('chart')['source_dir'] = (location, subpath)
elif ct_type == 'tar':
LOG.info('Downloading tarball from: %s', location)
tarball_dir = source.get_tarball(location)
ch.get('chart')['source_dir'] = (tarball_dir, subpath)
elif ct_type == 'git':
if location not in repos.keys():
try:
LOG.info('Cloning repo: %s', location)
repo_dir = git.git_clone(location, reference)
repo_dir = source.git_clone(location, reference)
except Exception:
raise git_exceptions.GitLocationException(location)
raise source_exceptions.GitLocationException(location)
repos[location] = repo_dir
ch.get('chart')['source_dir'] = (repo_dir, subpath)
else:
ch.get('chart')['source_dir'] = (repos.get(location), subpath)
else:
chart_name = ch.get('chart').get('chart_name')
raise armada_exceptions.ChartSourceException(ct_type, chart_name)
raise source_exceptions.ChartSourceException(ct_type, chart_name)
def get_releases_by_status(self, status):
'''
@ -305,7 +309,7 @@ class Armada(object):
for group in self.config.get(KEYWORD_ARMADA).get(KEYWORD_GROUPS):
for ch in group.get(KEYWORD_CHARTS):
if ch.get('chart').get('source').get('type') == 'git':
git.source_cleanup(ch.get('chart').get('source_dir')[0])
source.source_cleanup(ch.get('chart').get('source_dir')[0])
def show_diff(self, chart, installed_chart, installed_values, target_chart,
target_values):

2
armada/handlers/chartbuilder.py
View File

@ -202,7 +202,7 @@ class ChartBuilder(object):
templates=self.get_templates(),
dependencies=dependencies,
values=self.get_values(),
files=self.get_files(), )
files=self.get_files())
except Exception:
chart_name = self.chart.chart_name
raise chartbuilder_exceptions.HelmChartBuildException(chart_name)

67
armada/tests/unit/utils/test_git.py
View File

@ -1,67 +0,0 @@
# Copyright 2017 The Armada Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
import unittest
from armada.exceptions import git_exceptions
from armada.utils import git
class GitTestCase(unittest.TestCase):
@mock.patch('armada.utils.git.tempfile')
@mock.patch('armada.utils.git.pygit2')
def test_git_clone_good_url(self, mock_pygit, mock_temp):
mock_temp.mkdtemp.return_value = '/tmp/armada'
mock_pygit.clone_repository.return_value = "Repository"
url = 'http://github.com/att-comdev/armada'
dir = git.git_clone(url)
self.assertIsNotNone(dir)
def test_git_clone_empty_url(self):
url = ''
with self.assertRaises(Exception):
self.assertFalse(git.git_clone(url))
def test_git_clone_bad_url(self):
url = 'http://github.com/dummy/armada'
with self.assertRaises(Exception):
git.git_clone(url)
@mock.patch('armada.utils.git.shutil')
@mock.patch('armada.utils.git.path')
def test_source_cleanup(self, mock_path, mock_shutil):
mock_path.exists.return_value = True
path = 'armada'
try:
git.source_cleanup(path)
except git_exceptions.SourceCleanupException:
pass
mock_shutil.rmtree.assert_called_with(path)
@unittest.skip('not handled correctly')
@mock.patch('armada.utils.git.shutil')
@mock.patch('armada.utils.git.path')
def test_source_cleanup_bad_path(self, mock_path, mock_shutil):
mock_path.exists.return_value = False
path = 'armada'
with self.assertRaises(Exception):
git.source_cleanup(path)
mock_shutil.rmtree.assert_not_called()

117
armada/tests/unit/utils/test_source.py Normal file
View File

@ -0,0 +1,117 @@
# Copyright 2017 The Armada Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
import unittest
from armada.exceptions import source_exceptions
from armada.utils import source
class GitTestCase(unittest.TestCase):
SOURCE_UTILS_LOCATION = 'armada.utils.source'
@mock.patch('armada.utils.source.tempfile')
@mock.patch('armada.utils.source.pygit2')
def test_git_clone_good_url(self, mock_pygit, mock_temp):
mock_temp.mkdtemp.return_value = '/tmp/armada'
mock_pygit.clone_repository.return_value = "Repository"
url = 'http://github.com/att-comdev/armada'
dir = source.git_clone(url)
self.assertIsNotNone(dir)
def test_git_clone_empty_url(self):
url = ''
with self.assertRaises(Exception):
self.assertFalse(source.git_clone(url))
def test_git_clone_bad_url(self):
url = 'http://github.com/dummy/armada'
with self.assertRaises(Exception):
source.git_clone(url)
@mock.patch('armada.utils.source.tempfile')
@mock.patch('armada.utils.source.requests')
def test_tarball_download(self, mock_requests, mock_temp):
url = 'http://localhost:8879/charts/mariadb-0.1.0.tgz'
mock_temp.mkstemp.return_value = (None, '/tmp/armada')
mock_response = mock.Mock()
mock_response.content = 'some string'
mock_requests.get.return_value = mock_response
mock_open = mock.mock_open()
with mock.patch('{}.open'.format(self.SOURCE_UTILS_LOCATION),
mock_open, create=True):
source.download_tarball(url)
mock_temp.mkstemp.assert_called_once()
mock_requests.get.assert_called_once_with(url)
mock_open.assert_called_once_with('/tmp/armada', 'wb')
mock_open().write.assert_called_once_with(
mock_requests.get(url).content)
@mock.patch('armada.utils.source.tempfile')
@mock.patch('armada.utils.source.path')
@mock.patch('armada.utils.source.tarfile')
def test_tarball_extract(self, mock_tarfile, mock_path, mock_temp):
mock_path.exists.return_value = True
mock_temp.mkdtemp.return_value = '/tmp/armada'
mock_opened_file = mock.Mock()
mock_tarfile.open.return_value = mock_opened_file
path = '/tmp/mariadb-0.1.0.tgz'
source.extract_tarball(path)
mock_path.exists.assert_called_once()
mock_temp.mkdtemp.assert_called_once()
mock_tarfile.open.assert_called_once_with(path)
mock_opened_file.extractall.assert_called_once_with('/tmp/armada')
@mock.patch('armada.utils.source.path')
@mock.patch('armada.utils.source.tarfile')
def test_tarball_extract_bad_path(self, mock_tarfile, mock_path):
mock_path.exists.return_value = False
path = '/tmp/armada'
with self.assertRaises(Exception):
source.extract_tarball(path)
mock_tarfile.open.assert_not_called()
mock_tarfile.extractall.assert_not_called()
@mock.patch('armada.utils.source.shutil')
@mock.patch('armada.utils.source.path')
def test_source_cleanup(self, mock_path, mock_shutil):
mock_path.exists.return_value = True
path = 'armada'
try:
source.source_cleanup(path)
except source_exceptions.SourceCleanupException:
pass
mock_shutil.rmtree.assert_called_with(path)
@unittest.skip('not handled correctly')
@mock.patch('armada.utils.source.shutil')
@mock.patch('armada.utils.source.path')
def test_source_cleanup_bad_path(self, mock_path, mock_shutil):
mock_path.exists.return_value = False
path = 'armada'
with self.assertRaises(Exception):
source.source_cleanup(path)
mock_shutil.rmtree.assert_not_called()

30
armada/utils/git.py
View File

@ -1,30 +0,0 @@
import pygit2
import tempfile
import shutil
from os import path
from ..exceptions import git_exceptions
def git_clone(repo_url, branch='master'):
'''
clones repo to a /tmp/ dir
'''
if repo_url == '':
raise git_exceptions.GitLocationException(repo_url)
_tmp_dir = tempfile.mkdtemp(prefix='armada', dir='/tmp')
try:
pygit2.clone_repository(repo_url, _tmp_dir, checkout_branch=branch)
except Exception:
raise git_exceptions.GitLocationException(repo_url)
return _tmp_dir
def source_cleanup(target_dir):
'''
Clean up source
'''
if path.exists(target_dir):
shutil.rmtree(target_dir)

65
armada/utils/source.py Normal file
View File

@ -0,0 +1,65 @@
import pygit2
import requests
import tarfile
import tempfile
import shutil
from os import path
from ..exceptions import source_exceptions
def git_clone(repo_url, branch='master'):
'''
clones repo to a /tmp/ dir
'''
if repo_url == '':
raise source_exceptions.GitLocationException(repo_url)
_tmp_dir = tempfile.mkdtemp(prefix='armada', dir='/tmp')
try:
pygit2.clone_repository(repo_url, _tmp_dir, checkout_branch=branch)
except Exception:
raise source_exceptions.GitLocationException(repo_url)
return _tmp_dir
def get_tarball(tarball_url):
tarball_path = download_tarball(tarball_url)
return extract_tarball(tarball_path)
def download_tarball(tarball_url):
'''
Downloads a tarball to /tmp and returns the path
'''
try:
tarball_filename = tempfile.mkstemp(prefix='armada', dir='/tmp')[1]
response = requests.get(tarball_url)
with open(tarball_filename, 'wb') as f:
f.write(response.content)
except Exception:
raise source_exceptions.TarballDownloadException(tarball_url)
return tarball_filename
def extract_tarball(tarball_path):
'''
Extracts a tarball to /tmp and returns the path
'''
if not path.exists(tarball_path):
raise source_exceptions.InvalidPathException(tarball_path)
_tmp_dir = tempfile.mkdtemp(prefix='armada', dir='/tmp')
try:
file = tarfile.open(tarball_path)
file.extractall(_tmp_dir)
except Exception:
raise source_exceptions.TarballExtractException(tarball_path)
return _tmp_dir
def source_cleanup(target_dir):
'''
Clean up source
'''
if path.exists(target_dir):
shutil.rmtree(target_dir)

8
docs/source/operations/guide-build-armada-yaml.rst
View File

@ -151,7 +151,7 @@ Source
+-------------+----------+---------------------------------------------------------------+
| keyword | type | action |
+=============+==========+===============================================================+
| type | string | source to build the chart: ``git`` or ``local`` |
| type | string | source to build the chart: ``git``, ``local``, or ``tar`` |
+-------------+----------+---------------------------------------------------------------+
| location | string | ``url`` or ``path`` to the chart's parent directory |
+-------------+----------+---------------------------------------------------------------+
@ -247,9 +247,9 @@ Simple Example
namespace: default
values: {}
source:
type: git
location: https://github.com/namespace/repo
subpath: .
type: tar
location: http://localhost:8879/namespace/repo
subpath: blog-2
reference: master
dependencies: []
---

26
docs/source/operations/guide-exceptions.rst
View File

@ -4,8 +4,6 @@ Armada Exceptions
+------------------------+----------------------------------------------------------+
| Exception | Error Description |
+========================+==========================================================+
| ChartSourceException | Occurs when an unknown chart source type is encountered. |
+------------------------+----------------------------------------------------------+
| KnownReleasesException | Occurs when no known releases are found. |
+------------------------+----------------------------------------------------------+
@ -53,16 +51,24 @@ Chartbuilder Exceptions
| UnknownChartSourceException | The chart source is unknown or invalid. |
+-----------------------------+-------------------------------------------------------------+
Git Exceptions
Source Exceptions
===============
+------------------------+---------------------------------------------+
| Exception | Error Description |
+========================+=============================================+
| GitLocationException | Repository location is not valid. |
+------------------------+---------------------------------------------+
| SourceCleanupException | The source dir of a chart no longer exists. |
+------------------------+---------------------------------------------+
+--------------------------+---------------------------------------------------------------------+
| Exception | Error Description |
+==========================+=====================================================================+
| GitLocationException | Repository location is not valid. |
+--------------------------+---------------------------------------------------------------------+
| SourceCleanupException | The source dir of a chart no longer exists. |
+--------------------------+---------------------------------------------------------------------+
| TarballDownloadException | Occurs when the tarball cannot be downloaded from the provided URL. |
+--------------------------+---------------------------------------------------------------------+
| TarballExtractException | Occurs when extracting a tarball fails. |
+--------------------------+---------------------------------------------------------------------+
| InvalidPathException | Occurs when a nonexistant path is accessed. |
+--------------------------+---------------------------------------------------------------------+
| ChartSourceException | Occurs when an unknown chart source type is encountered. |
+--------------------------+---------------------------------------------------------------------+
Lint Exceptions
===============

57
examples/tar_example.yaml Normal file
View File

@ -0,0 +1,57 @@
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: helm-toolkit
data:
chart_name: helm-toolkit
release: helm-toolkit
namespace: helm-tookit
values: {}
source:
type: git
location: git://github.com/openstack/openstack-helm
subpath: helm-toolkit
reference: master
dependencies: []
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: mariadb-tarball
data:
chart_name: mariadb-tarball
release: mariadb-tarball
namespace: tarball
timeout: 3600
install:
no_hooks: false
upgrade:
no_hooks: false
values: {}
source:
type: tar
location: http://localhost:8879/charts/mariadb-0.1.0.tgz
subpath: mariadb
reference: null
dependencies:
- helm-toolkit
---
schema: armada/ChartGroup/v1
metadata:
schema: metadata/Document/v1
name: tar-example
data:
description: "Deploying mariadb tarball URL"
sequenced: True
chart_group:
- mariadb-tarball
---
schema: armada/Manifest/v1
metadata:
schema: metadata/Document/v1
name: armada-manifest
data:
release_prefix: armada
chart_groups:
- tar-example