openstack
/
magnum
Code Issues Proposed changes

Merge "Fix manifest url doesn't work"

This commit is contained in:
Jenkins 2015-01-27 07:59:18 +00:00 committed by Gerrit Code Review
parent 576e81c882 a8e6bbe3e9
commit 129a1bcc62
8 changed files with 271 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
magnum/api/controllers/v1/base.py Normal file
View File

@ -0,0 +1,62 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import pecan
import wsme
from wsme import types as wtypes
from magnum.api.controllers import base
from magnum.api.controllers.v1 import types
from magnum.common import exception
from magnum.common import urlfetch
from magnum import objects
class K8sResourceBase(base.APIBase):
_bay_uuid = None
def _get_bay_uuid(self):
return self._bay_uuid
def _set_bay_uuid(self, value):
if value and self._bay_uuid != value:
try:
bay = objects.Bay.get(pecan.request.context, value)
self._bay_uuid = bay.uuid
except exception.BayNotFound as e:
# Change error code because 404 (NotFound) is inappropriate
# response for a POST request to create a Service
e.code = 400 # BadRequest
raise e
elif value == wtypes.Unset:
self._bay_uuid = wtypes.Unset
bay_uuid = wsme.wsproperty(types.uuid, _get_bay_uuid, _set_bay_uuid,
mandatory=True)
"""Unique UUID of the bay this runs on"""
manifest_url = wtypes.text
"""URL for service file to create the service"""
manifest = wtypes.text
"""Data for service to create the service"""
def _get_manifest(self):
if self.manifest is not wsme.Unset and self.manifest is not None:
return self.manifest
if (self.manifest_url is not wsme.Unset
and self.manifest_url is not None):
self.manifest = urlfetch.get(self.manifest_url)
return self.manifest

37
magnum/api/controllers/v1/pod.py
View File

@ -19,8 +19,8 @@ import wsme
from wsme import types as wtypes
import wsmeext.pecan as wsme_pecan
from magnum.api.controllers import base
from magnum.api.controllers import link
from magnum.api.controllers.v1 import base as v1_base
from magnum.api.controllers.v1 import collection
from magnum.api.controllers.v1 import types
from magnum.api.controllers.v1 import utils as api_utils
@ -36,31 +36,13 @@ class PodPatchType(types.JsonPatchType):
return ['/bay_uuid']
class Pod(base.APIBase):
class Pod(v1_base.K8sResourceBase):
"""API representation of a pod.
This class enforces type checking and value constraints, and converts
between the internal object model and the API representation of a pod.
"""
_bay_uuid = None
def _get_bay_uuid(self):
return self._bay_uuid
def _set_bay_uuid(self, value):
if value and self._bay_uuid != value:
try:
bay = objects.Bay.get(pecan.request.context, value)
self._bay_uuid = bay.uuid
except exception.BayNotFound as e:
# Change error code because 404 (NotFound) is inappropriate
# response for a POST request to create a Pod
e.code = 400 # BadRequest
raise e
elif value == wtypes.Unset:
self._bay_uuid = wtypes.Unset
uuid = types.uuid
"""Unique UUID for this pod"""
@ -70,10 +52,6 @@ class Pod(base.APIBase):
desc = wtypes.text
"""Description of this pod"""
bay_uuid = wsme.wsproperty(types.uuid, _get_bay_uuid, _set_bay_uuid,
mandatory=True)
"""Unique UUID of the bay the pod runs on"""
images = [wtypes.text]
"""A list of images used by containers in this pod."""
@ -83,12 +61,6 @@ class Pod(base.APIBase):
status = wtypes.text
"""Staus of this pod """
manifest_url = wtypes.text
"""URL for pod file to create the pod"""
manifest = wtypes.text
"""Data for pod to create the pod"""
links = wsme.wsattr([link.Link], readonly=True)
"""A list containing a self link and associated pod links"""
@ -143,10 +115,7 @@ class Pod(base.APIBase):
return cls._convert_with_links(sample, 'http://localhost:9511', expand)
def parse_manifest(self):
# Set pod name from its manifest
# TODO(yuanying): retrive pod name from manifest_url
if hasattr(self, "manifest") and self.manifest is not None:
manifest = k8s_manifest.parse(self.manifest)
manifest = k8s_manifest.parse(self._get_manifest())
self.name = manifest["id"]
if "labels" in manifest:
self.labels = manifest["labels"]

39
magnum/api/controllers/v1/replicationcontroller.py
View File

@ -20,8 +20,8 @@ import wsme
from wsme import types as wtypes
import wsmeext.pecan as wsme_pecan
from magnum.api.controllers import base
from magnum.api.controllers import link
from magnum.api.controllers.v1 import base as v1_base
from magnum.api.controllers.v1 import collection
from magnum.api.controllers.v1 import types
from magnum.api.controllers.v1 import utils as api_utils
@ -37,7 +37,7 @@ class ReplicationControllerPatchType(types.JsonPatchType):
return ['/bay_uuid']
class ReplicationController(base.APIBase):
class ReplicationController(v1_base.K8sResourceBase):
"""API representation of a ReplicationController.
This class enforces type checking and value constraints, and converts
@ -45,24 +45,6 @@ class ReplicationController(base.APIBase):
ReplicationController.
"""
_bay_uuid = None
def _get_bay_uuid(self):
return self._bay_uuid
def _set_bay_uuid(self, value):
if value and self._bay_uuid != value:
try:
bay = objects.Bay.get(pecan.request.context, value)
self._bay_uuid = bay.uuid
except exception.BayNotFound as e:
# Change error code because 404 (NotFound) is inappropriate
# response for a POST request to create a rc
e.code = 400 # BadRequest
raise e
elif value == wtypes.Unset:
self._bay_uuid = wtypes.Unset
uuid = types.uuid
"""Unique UUID for this ReplicationController"""
@ -72,22 +54,12 @@ class ReplicationController(base.APIBase):
images = [wtypes.text]
"""A list of images used by containers in this ReplicationController."""
bay_uuid = wsme.wsproperty(types.uuid, _get_bay_uuid, _set_bay_uuid,
mandatory=True)
"""Unique UUID of the bay the ReplicationController runs on"""
labels = wsme.wsattr({wtypes.text: wtypes.text}, readonly=True)
"""Selector of this ReplicationController"""
replicas = wsme.wsattr(wtypes.IntegerType(), readonly=True)
"""Replicas of this ReplicationController"""
manifest_url = wtypes.text
"""URL for ReplicationController file to create the RC"""
replicationcontroller_data = wtypes.text
"""Data for service to create the ReplicationController"""
links = wsme.wsattr([link.Link], readonly=True)
"""A list containing a self link and associated rc links"""
@ -142,12 +114,7 @@ class ReplicationController(base.APIBase):
return cls._convert_with_links(sample, 'http://localhost:9511', expand)
def parse_manifest(self):
# Set replication controller name and labels from its manifest
# TODO(jay-lau-513): retrieve replication controller name from
# manifest_url
if (hasattr(self, "replicationcontroller_data")
and self.replicationcontroller_data is not None):
manifest = k8s_manifest.parse(self.replicationcontroller_data)
manifest = k8s_manifest.parse(self._get_manifest())
self.name = manifest["id"]
if "labels" in manifest:
self.labels = manifest["labels"]

36
magnum/api/controllers/v1/service.py
View File

@ -18,8 +18,8 @@ import wsme
from wsme import types as wtypes
import wsmeext.pecan as wsme_pecan
from magnum.api.controllers import base
from magnum.api.controllers import link
from magnum.api.controllers.v1 import base as v1_base
from magnum.api.controllers.v1 import collection
from magnum.api.controllers.v1 import types
from magnum.api.controllers.v1 import utils as api_utils
@ -39,24 +39,7 @@ class ServicePatchType(types.JsonPatchType):
return ['/bay_uuid']
class Service(base.APIBase):
_bay_uuid = None
def _get_bay_uuid(self):
return self._bay_uuid
def _set_bay_uuid(self, value):
if value and self._bay_uuid != value:
try:
bay = objects.Bay.get(pecan.request.context, value)
self._bay_uuid = bay.uuid
except exception.BayNotFound as e:
# Change error code because 404 (NotFound) is inappropriate
# response for a POST request to create a Service
e.code = 400 # BadRequest
raise e
elif value == wtypes.Unset:
self._bay_uuid = wtypes.Unset
class Service(v1_base.K8sResourceBase):
uuid = types.uuid
"""Unique UUID for this service"""
@ -64,10 +47,6 @@ class Service(base.APIBase):
name = wsme.wsattr(wtypes.text, readonly=True)
""" The name of the service."""
bay_uuid = wsme.wsproperty(types.uuid, _get_bay_uuid, _set_bay_uuid,
mandatory=True)
"""Unique UUID of the bay the service runs on"""
labels = wsme.wsattr({wtypes.text: wtypes.text}, readonly=True)
"""Labels of this service"""
@ -80,12 +59,6 @@ class Service(base.APIBase):
port = wtypes.IntegerType()
"""Port of this service"""
manifest_url = wtypes.text
"""URL for service file to create the service"""
manifest = wtypes.text
"""Data for service to create the service"""
links = wsme.wsattr([link.Link], readonly=True)
"""A list containing a self link and associated service links"""
@ -146,10 +119,7 @@ class Service(base.APIBase):
return cls._convert_with_links(sample, 'http://localhost:9511', expand)
def parse_manifest(self):
# Set service name and port from its manifest
# TODO(yuanying): retrive service name from definition_url
if hasattr(self, "manifest") and self.manifest is not None:
manifest = k8s_manifest.parse(self.manifest)
manifest = k8s_manifest.parse(self._get_manifest())
self.name = manifest["id"]
if "port" in manifest:
self.port = manifest["port"]

84
magnum/common/urlfetch.py Normal file
View File

@ -0,0 +1,84 @@
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Utility for fetching a resource (e.g. a manifest) from a URL."""
from oslo.config import cfg
import requests
from requests import exceptions
from six.moves import urllib
from magnum.common import exception
from magnum.openstack.common._i18n import _
from magnum.openstack.common._i18n import _LI
from magnum.openstack.common import log as logging
URLFETCH_OPTS = [
cfg.IntOpt('max_manifest_size',
default=524288,
help=_('Maximum raw byte size of any manifest.'))
]
cfg.CONF.register_opts(URLFETCH_OPTS)
LOG = logging.getLogger(__name__)
class URLFetchError(exception.Invalid, IOError):
pass
def get(url, allowed_schemes=('http', 'https')):
"""Get the data at the specified URL.
The URL must use the http: or https: schemes.
The file: scheme is also supported if you override
the allowed_schemes argument.
Raise an IOError if getting the data fails.
"""
LOG.info(_LI('Fetching data from %s'), url)
components = urllib.parse.urlparse(url)
if components.scheme not in allowed_schemes:
raise URLFetchError(_('Invalid URL scheme %s') % components.scheme)
if components.scheme == 'file':
try:
return urllib.request.urlopen(url).read()
except urllib.error.URLError as uex:
raise URLFetchError(_('Failed to retrieve manifest: %s') % uex)
try:
resp = requests.get(url, stream=True)
resp.raise_for_status()
# We cannot use resp.text here because it would download the
# entire file, and a large enough file would bring down the
# engine. The 'Content-Length' header could be faked, so it's
# necessary to download the content in chunks to until
# max_manifest_size is reached. The chunk_size we use needs
# to balance CPU-intensive string concatenation with accuracy
# (eg. it's possible to fetch 1000 bytes greater than
# max_manifest_size with a chunk_size of 1000).
reader = resp.iter_content(chunk_size=1000)
result = ""
for chunk in reader:
result += chunk
if len(result) > cfg.CONF.max_manifest_size:
raise URLFetchError("Manifest exceeds maximum allowed size (%s"
" bytes)" % cfg.CONF.max_manifest_size)
return result
except exceptions.RequestException as ex:
raise URLFetchError(_('Failed to retrieve manifest: %s') % ex)

45
magnum/tests/api/controllers/v1/test_base.py Normal file
View File

@ -0,0 +1,45 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import datetime
from mock import patch
from magnum.api.controllers.v1 import base as api_base
from magnum.tests import base
class TestK8sResourceBase(base.BaseTestCase):
def setUp(self):
super(TestK8sResourceBase, self).setUp()
self.resource_base = api_base.K8sResourceBase(
uuid='fe78db47-9a37-4e9f-8572-804a10abc0aa',
created_at=datetime.datetime.utcnow(),
updated_at=datetime.datetime.utcnow())
def test_get_manifest_with_manifest(self):
expected_manifest = 'expected_manifest'
self.resource_base.manifest = expected_manifest
self.resource_base.manifest_url = 'file:///tmp/rc.yaml'
self.assertEqual(expected_manifest,
self.resource_base._get_manifest())
@patch('magnum.common.urlfetch.get')
def test_get_manifest_with_manifest_url(self,
mock_urlfetch_get):
expected_manifest = 'expected_manifest_from_url'
mock_urlfetch_get.return_value = expected_manifest
self.resource_base.manifest_url = 'file:///tmp/rc.yaml'
self.assertEqual(expected_manifest,
self.resource_base._get_manifest())

2
magnum/tests/api/controllers/v1/test_replicationcontroller.py
View File

@ -36,7 +36,7 @@ class TestRCController(db_base.DbTestCase):
params = '''
{
"bay_uuid": "%s",
"replicationcontroller_data": "\
"manifest": "\
{\
\\"id\\": \\"name_of_rc\\", \
\\"replicas\\": 3, \

55
magnum/tests/common/test_urlfetch.py Normal file
View File

@ -0,0 +1,55 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from mock import patch
from oslo.config import cfg
from magnum.common import urlfetch
from magnum.tests import base
class TestUrlFetch(base.BaseTestCase):
def setUp(self):
super(TestUrlFetch, self).setUp()
def test_get_unsupported_scheme(self):
self.assertRaises(urlfetch.URLFetchError,
urlfetch.get,
'https://example.com',
('http'))
@patch('requests.get')
def test_get(self,
mock_request_get):
mock_reader = mock.MagicMock()
mock_reader.__iter__.return_value = ['a', 'b', 'c']
mock_response = mock.MagicMock()
mock_response.iter_content.return_value = mock_reader
mock_request_get.return_value = mock_response
self.assertEqual(urlfetch.get('http://example.com'), 'abc')
@patch('requests.get')
def test_get_exceed_manifest_size(self,
mock_request_get):
cfg.CONF.set_override("max_manifest_size", 1)
mock_reader = mock.MagicMock()
mock_reader.__iter__.return_value = ['a', 'b']
mock_response = mock.MagicMock()
mock_response.iter_content.return_value = mock_reader
mock_request_get.return_value = mock_response
self.assertRaises(urlfetch.URLFetchError,
urlfetch.get,
'http://example.com')