Merge "Fix manifest url doesn't work"
Jenkins
8 years ago
committed by
Gerrit Code Review
commit
129a1bcc62
@ -0,0 +1,62 @@
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import pecan
|
||||
import wsme
|
||||
from wsme import types as wtypes
|
||||
|
||||
from magnum.api.controllers import base
|
||||
from magnum.api.controllers.v1 import types
|
||||
from magnum.common import exception
|
||||
from magnum.common import urlfetch
|
||||
from magnum import objects
|
||||
|
||||
|
||||
class K8sResourceBase(base.APIBase):
|
||||
|
||||
_bay_uuid = None
|
||||
|
||||
def _get_bay_uuid(self):
|
||||
return self._bay_uuid
|
||||
|
||||
def _set_bay_uuid(self, value):
|
||||
if value and self._bay_uuid != value:
|
||||
try:
|
||||
bay = objects.Bay.get(pecan.request.context, value)
|
||||
self._bay_uuid = bay.uuid
|
||||
except exception.BayNotFound as e:
|
||||
# Change error code because 404 (NotFound) is inappropriate
|
||||
# response for a POST request to create a Service
|
||||
e.code = 400 # BadRequest
|
||||
raise e
|
||||
elif value == wtypes.Unset:
|
||||
self._bay_uuid = wtypes.Unset
|
||||
|
||||
bay_uuid = wsme.wsproperty(types.uuid, _get_bay_uuid, _set_bay_uuid,
|
||||
mandatory=True)
|
||||
"""Unique UUID of the bay this runs on"""
|
||||
|
||||
manifest_url = wtypes.text
|
||||
"""URL for service file to create the service"""
|
||||
|
||||
manifest = wtypes.text
|
||||
"""Data for service to create the service"""
|
||||
|
||||
def _get_manifest(self):
|
||||
if self.manifest is not wsme.Unset and self.manifest is not None:
|
||||
return self.manifest
|
||||
if (self.manifest_url is not wsme.Unset
|
||||
and self.manifest_url is not None):
|
||||
self.manifest = urlfetch.get(self.manifest_url)
|
||||
return self.manifest
|
||||
@ -0,0 +1,84 @@
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Utility for fetching a resource (e.g. a manifest) from a URL."""
|
||||
|
||||
from oslo.config import cfg
|
||||
import requests
|
||||
from requests import exceptions
|
||||
from six.moves import urllib
|
||||
|
||||
from magnum.common import exception
|
||||
from magnum.openstack.common._i18n import _
|
||||
from magnum.openstack.common._i18n import _LI
|
||||
from magnum.openstack.common import log as logging
|
||||
|
||||
URLFETCH_OPTS = [
|
||||
cfg.IntOpt('max_manifest_size',
|
||||
default=524288,
|
||||
help=_('Maximum raw byte size of any manifest.'))
|
||||
]
|
||||
|
||||
cfg.CONF.register_opts(URLFETCH_OPTS)
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class URLFetchError(exception.Invalid, IOError):
|
||||
pass
|
||||
|
||||
|
||||
def get(url, allowed_schemes=('http', 'https')):
|
||||
"""Get the data at the specified URL.
|
||||
|
||||
The URL must use the http: or https: schemes.
|
||||
The file: scheme is also supported if you override
|
||||
the allowed_schemes argument.
|
||||
Raise an IOError if getting the data fails.
|
||||
"""
|
||||
LOG.info(_LI('Fetching data from %s'), url)
|
||||
|
||||
components = urllib.parse.urlparse(url)
|
||||
|
||||
if components.scheme not in allowed_schemes:
|
||||
raise URLFetchError(_('Invalid URL scheme %s') % components.scheme)
|
||||
|
||||
if components.scheme == 'file':
|
||||
try:
|
||||
return urllib.request.urlopen(url).read()
|
||||
except urllib.error.URLError as uex:
|
||||
raise URLFetchError(_('Failed to retrieve manifest: %s') % uex)
|
||||
|
||||
try:
|
||||
resp = requests.get(url, stream=True)
|
||||
resp.raise_for_status()
|
||||
|
||||
# We cannot use resp.text here because it would download the
|
||||
# entire file, and a large enough file would bring down the
|
||||
# engine. The 'Content-Length' header could be faked, so it's
|
||||
# necessary to download the content in chunks to until
|
||||
# max_manifest_size is reached. The chunk_size we use needs
|
||||
# to balance CPU-intensive string concatenation with accuracy
|
||||
# (eg. it's possible to fetch 1000 bytes greater than
|
||||
# max_manifest_size with a chunk_size of 1000).
|
||||
reader = resp.iter_content(chunk_size=1000)
|
||||
result = ""
|
||||
for chunk in reader:
|
||||
result += chunk
|
||||
if len(result) > cfg.CONF.max_manifest_size:
|
||||
raise URLFetchError("Manifest exceeds maximum allowed size (%s"
|
||||
" bytes)" % cfg.CONF.max_manifest_size)
|
||||
return result
|
||||
|
||||
except exceptions.RequestException as ex:
|
||||
raise URLFetchError(_('Failed to retrieve manifest: %s') % ex)
|
||||
@ -0,0 +1,45 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import datetime
|
||||
|
||||
from mock import patch
|
||||
|
||||
from magnum.api.controllers.v1 import base as api_base
|
||||
from magnum.tests import base
|
||||
|
||||
|
||||
class TestK8sResourceBase(base.BaseTestCase):
|
||||
def setUp(self):
|
||||
super(TestK8sResourceBase, self).setUp()
|
||||
self.resource_base = api_base.K8sResourceBase(
|
||||
uuid='fe78db47-9a37-4e9f-8572-804a10abc0aa',
|
||||
created_at=datetime.datetime.utcnow(),
|
||||
updated_at=datetime.datetime.utcnow())
|
||||
|
||||
def test_get_manifest_with_manifest(self):
|
||||
expected_manifest = 'expected_manifest'
|
||||
self.resource_base.manifest = expected_manifest
|
||||
self.resource_base.manifest_url = 'file:///tmp/rc.yaml'
|
||||
|
||||
self.assertEqual(expected_manifest,
|
||||
self.resource_base._get_manifest())
|
||||
|
||||
@patch('magnum.common.urlfetch.get')
|
||||
def test_get_manifest_with_manifest_url(self,
|
||||
mock_urlfetch_get):
|
||||
expected_manifest = 'expected_manifest_from_url'
|
||||
mock_urlfetch_get.return_value = expected_manifest
|
||||
|
||||
self.resource_base.manifest_url = 'file:///tmp/rc.yaml'
|
||||
|
||||
self.assertEqual(expected_manifest,
|
||||
self.resource_base._get_manifest())
|
||||
@ -0,0 +1,55 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import mock
|
||||
from mock import patch
|
||||
from oslo.config import cfg
|
||||
|
||||
from magnum.common import urlfetch
|
||||
from magnum.tests import base
|
||||
|
||||
|
||||
class TestUrlFetch(base.BaseTestCase):
|
||||
def setUp(self):
|
||||
super(TestUrlFetch, self).setUp()
|
||||
|
||||
def test_get_unsupported_scheme(self):
|
||||
self.assertRaises(urlfetch.URLFetchError,
|
||||
urlfetch.get,
|
||||
'https://example.com',
|
||||
('http'))
|
||||
|
||||
@patch('requests.get')
|
||||
def test_get(self,
|
||||
mock_request_get):
|
||||
mock_reader = mock.MagicMock()
|
||||
mock_reader.__iter__.return_value = ['a', 'b', 'c']
|
||||
mock_response = mock.MagicMock()
|
||||
mock_response.iter_content.return_value = mock_reader
|
||||
mock_request_get.return_value = mock_response
|
||||
|
||||
self.assertEqual(urlfetch.get('http://example.com'), 'abc')
|
||||
|
||||
@patch('requests.get')
|
||||
def test_get_exceed_manifest_size(self,
|
||||
mock_request_get):
|
||||
cfg.CONF.set_override("max_manifest_size", 1)
|
||||
|
||||
mock_reader = mock.MagicMock()
|
||||
mock_reader.__iter__.return_value = ['a', 'b']
|
||||
mock_response = mock.MagicMock()
|
||||
mock_response.iter_content.return_value = mock_reader
|
||||
mock_request_get.return_value = mock_response
|
||||
|
||||
self.assertRaises(urlfetch.URLFetchError,
|
||||
urlfetch.get,
|
||||
'http://example.com')
|
||||
Loading…
Reference in New Issue