diff --git a/magnum/api/controllers/v1/base.py b/magnum/api/controllers/v1/base.py new file mode 100644 index 0000000000..3ad1e3efc6 --- /dev/null +++ b/magnum/api/controllers/v1/base.py @@ -0,0 +1,62 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pecan +import wsme +from wsme import types as wtypes + +from magnum.api.controllers import base +from magnum.api.controllers.v1 import types +from magnum.common import exception +from magnum.common import urlfetch +from magnum import objects + + +class K8sResourceBase(base.APIBase): + + _bay_uuid = None + + def _get_bay_uuid(self): + return self._bay_uuid + + def _set_bay_uuid(self, value): + if value and self._bay_uuid != value: + try: + bay = objects.Bay.get(pecan.request.context, value) + self._bay_uuid = bay.uuid + except exception.BayNotFound as e: + # Change error code because 404 (NotFound) is inappropriate + # response for a POST request to create a Service + e.code = 400 # BadRequest + raise e + elif value == wtypes.Unset: + self._bay_uuid = wtypes.Unset + + bay_uuid = wsme.wsproperty(types.uuid, _get_bay_uuid, _set_bay_uuid, + mandatory=True) + """Unique UUID of the bay this runs on""" + + manifest_url = wtypes.text + """URL for service file to create the service""" + + manifest = wtypes.text + """Data for service to create the service""" + + def _get_manifest(self): + if self.manifest is not wsme.Unset and self.manifest is not None: + return self.manifest + if (self.manifest_url is not wsme.Unset + and self.manifest_url is not None): + self.manifest = urlfetch.get(self.manifest_url) + return self.manifest diff --git a/magnum/api/controllers/v1/pod.py b/magnum/api/controllers/v1/pod.py index b8a886f4e4..3317ee3175 100644 --- a/magnum/api/controllers/v1/pod.py +++ b/magnum/api/controllers/v1/pod.py @@ -19,8 +19,8 @@ import wsme from wsme import types as wtypes import wsmeext.pecan as wsme_pecan -from magnum.api.controllers import base from magnum.api.controllers import link +from magnum.api.controllers.v1 import base as v1_base from magnum.api.controllers.v1 import collection from magnum.api.controllers.v1 import types from magnum.api.controllers.v1 import utils as api_utils @@ -36,31 +36,13 @@ class PodPatchType(types.JsonPatchType): return ['/bay_uuid'] -class Pod(base.APIBase): +class Pod(v1_base.K8sResourceBase): """API representation of a pod. This class enforces type checking and value constraints, and converts between the internal object model and the API representation of a pod. """ - _bay_uuid = None - - def _get_bay_uuid(self): - return self._bay_uuid - - def _set_bay_uuid(self, value): - if value and self._bay_uuid != value: - try: - bay = objects.Bay.get(pecan.request.context, value) - self._bay_uuid = bay.uuid - except exception.BayNotFound as e: - # Change error code because 404 (NotFound) is inappropriate - # response for a POST request to create a Pod - e.code = 400 # BadRequest - raise e - elif value == wtypes.Unset: - self._bay_uuid = wtypes.Unset - uuid = types.uuid """Unique UUID for this pod""" @@ -70,10 +52,6 @@ class Pod(base.APIBase): desc = wtypes.text """Description of this pod""" - bay_uuid = wsme.wsproperty(types.uuid, _get_bay_uuid, _set_bay_uuid, - mandatory=True) - """Unique UUID of the bay the pod runs on""" - images = [wtypes.text] """A list of images used by containers in this pod.""" @@ -83,12 +61,6 @@ class Pod(base.APIBase): status = wtypes.text """Staus of this pod """ - manifest_url = wtypes.text - """URL for pod file to create the pod""" - - manifest = wtypes.text - """Data for pod to create the pod""" - links = wsme.wsattr([link.Link], readonly=True) """A list containing a self link and associated pod links""" @@ -143,13 +115,10 @@ class Pod(base.APIBase): return cls._convert_with_links(sample, 'http://localhost:9511', expand) def parse_manifest(self): - # Set pod name from its manifest - # TODO(yuanying): retrive pod name from manifest_url - if hasattr(self, "manifest") and self.manifest is not None: - manifest = k8s_manifest.parse(self.manifest) - self.name = manifest["id"] - if "labels" in manifest: - self.labels = manifest["labels"] + manifest = k8s_manifest.parse(self._get_manifest()) + self.name = manifest["id"] + if "labels" in manifest: + self.labels = manifest["labels"] class PodCollection(collection.Collection): diff --git a/magnum/api/controllers/v1/replicationcontroller.py b/magnum/api/controllers/v1/replicationcontroller.py index d186424739..9632190f52 100644 --- a/magnum/api/controllers/v1/replicationcontroller.py +++ b/magnum/api/controllers/v1/replicationcontroller.py @@ -20,8 +20,8 @@ import wsme from wsme import types as wtypes import wsmeext.pecan as wsme_pecan -from magnum.api.controllers import base from magnum.api.controllers import link +from magnum.api.controllers.v1 import base as v1_base from magnum.api.controllers.v1 import collection from magnum.api.controllers.v1 import types from magnum.api.controllers.v1 import utils as api_utils @@ -37,7 +37,7 @@ class ReplicationControllerPatchType(types.JsonPatchType): return ['/bay_uuid'] -class ReplicationController(base.APIBase): +class ReplicationController(v1_base.K8sResourceBase): """API representation of a ReplicationController. This class enforces type checking and value constraints, and converts @@ -45,24 +45,6 @@ class ReplicationController(base.APIBase): ReplicationController. """ - _bay_uuid = None - - def _get_bay_uuid(self): - return self._bay_uuid - - def _set_bay_uuid(self, value): - if value and self._bay_uuid != value: - try: - bay = objects.Bay.get(pecan.request.context, value) - self._bay_uuid = bay.uuid - except exception.BayNotFound as e: - # Change error code because 404 (NotFound) is inappropriate - # response for a POST request to create a rc - e.code = 400 # BadRequest - raise e - elif value == wtypes.Unset: - self._bay_uuid = wtypes.Unset - uuid = types.uuid """Unique UUID for this ReplicationController""" @@ -72,22 +54,12 @@ class ReplicationController(base.APIBase): images = [wtypes.text] """A list of images used by containers in this ReplicationController.""" - bay_uuid = wsme.wsproperty(types.uuid, _get_bay_uuid, _set_bay_uuid, - mandatory=True) - """Unique UUID of the bay the ReplicationController runs on""" - labels = wsme.wsattr({wtypes.text: wtypes.text}, readonly=True) """Selector of this ReplicationController""" replicas = wsme.wsattr(wtypes.IntegerType(), readonly=True) """Replicas of this ReplicationController""" - manifest_url = wtypes.text - """URL for ReplicationController file to create the RC""" - - replicationcontroller_data = wtypes.text - """Data for service to create the ReplicationController""" - links = wsme.wsattr([link.Link], readonly=True) """A list containing a self link and associated rc links""" @@ -142,17 +114,12 @@ class ReplicationController(base.APIBase): return cls._convert_with_links(sample, 'http://localhost:9511', expand) def parse_manifest(self): - # Set replication controller name and labels from its manifest - # TODO(jay-lau-513): retrieve replication controller name from - # manifest_url - if (hasattr(self, "replicationcontroller_data") - and self.replicationcontroller_data is not None): - manifest = k8s_manifest.parse(self.replicationcontroller_data) - self.name = manifest["id"] - if "labels" in manifest: - self.labels = manifest["labels"] - if "replicas" in manifest: - self.replicas = manifest["replicas"] + manifest = k8s_manifest.parse(self._get_manifest()) + self.name = manifest["id"] + if "labels" in manifest: + self.labels = manifest["labels"] + if "replicas" in manifest: + self.replicas = manifest["replicas"] class ReplicationControllerCollection(collection.Collection): diff --git a/magnum/api/controllers/v1/service.py b/magnum/api/controllers/v1/service.py index 0185780dc4..6e4145907a 100644 --- a/magnum/api/controllers/v1/service.py +++ b/magnum/api/controllers/v1/service.py @@ -18,8 +18,8 @@ import wsme from wsme import types as wtypes import wsmeext.pecan as wsme_pecan -from magnum.api.controllers import base from magnum.api.controllers import link +from magnum.api.controllers.v1 import base as v1_base from magnum.api.controllers.v1 import collection from magnum.api.controllers.v1 import types from magnum.api.controllers.v1 import utils as api_utils @@ -39,24 +39,7 @@ class ServicePatchType(types.JsonPatchType): return ['/bay_uuid'] -class Service(base.APIBase): - _bay_uuid = None - - def _get_bay_uuid(self): - return self._bay_uuid - - def _set_bay_uuid(self, value): - if value and self._bay_uuid != value: - try: - bay = objects.Bay.get(pecan.request.context, value) - self._bay_uuid = bay.uuid - except exception.BayNotFound as e: - # Change error code because 404 (NotFound) is inappropriate - # response for a POST request to create a Service - e.code = 400 # BadRequest - raise e - elif value == wtypes.Unset: - self._bay_uuid = wtypes.Unset +class Service(v1_base.K8sResourceBase): uuid = types.uuid """Unique UUID for this service""" @@ -64,10 +47,6 @@ class Service(base.APIBase): name = wsme.wsattr(wtypes.text, readonly=True) """ The name of the service.""" - bay_uuid = wsme.wsproperty(types.uuid, _get_bay_uuid, _set_bay_uuid, - mandatory=True) - """Unique UUID of the bay the service runs on""" - labels = wsme.wsattr({wtypes.text: wtypes.text}, readonly=True) """Labels of this service""" @@ -80,12 +59,6 @@ class Service(base.APIBase): port = wtypes.IntegerType() """Port of this service""" - manifest_url = wtypes.text - """URL for service file to create the service""" - - manifest = wtypes.text - """Data for service to create the service""" - links = wsme.wsattr([link.Link], readonly=True) """A list containing a self link and associated service links""" @@ -146,17 +119,14 @@ class Service(base.APIBase): return cls._convert_with_links(sample, 'http://localhost:9511', expand) def parse_manifest(self): - # Set service name and port from its manifest - # TODO(yuanying): retrive service name from definition_url - if hasattr(self, "manifest") and self.manifest is not None: - manifest = k8s_manifest.parse(self.manifest) - self.name = manifest["id"] - if "port" in manifest: - self.port = manifest["port"] - if "selector" in manifest: - self.selector = manifest["selector"] - if "labels" in manifest: - self.labels = manifest["labels"] + manifest = k8s_manifest.parse(self._get_manifest()) + self.name = manifest["id"] + if "port" in manifest: + self.port = manifest["port"] + if "selector" in manifest: + self.selector = manifest["selector"] + if "labels" in manifest: + self.labels = manifest["labels"] class ServiceCollection(collection.Collection): diff --git a/magnum/common/urlfetch.py b/magnum/common/urlfetch.py new file mode 100644 index 0000000000..12fc99391d --- /dev/null +++ b/magnum/common/urlfetch.py @@ -0,0 +1,84 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Utility for fetching a resource (e.g. a manifest) from a URL.""" + +from oslo.config import cfg +import requests +from requests import exceptions +from six.moves import urllib + +from magnum.common import exception +from magnum.openstack.common._i18n import _ +from magnum.openstack.common._i18n import _LI +from magnum.openstack.common import log as logging + +URLFETCH_OPTS = [ + cfg.IntOpt('max_manifest_size', + default=524288, + help=_('Maximum raw byte size of any manifest.')) +] + +cfg.CONF.register_opts(URLFETCH_OPTS) + +LOG = logging.getLogger(__name__) + + +class URLFetchError(exception.Invalid, IOError): + pass + + +def get(url, allowed_schemes=('http', 'https')): + """Get the data at the specified URL. + + The URL must use the http: or https: schemes. + The file: scheme is also supported if you override + the allowed_schemes argument. + Raise an IOError if getting the data fails. + """ + LOG.info(_LI('Fetching data from %s'), url) + + components = urllib.parse.urlparse(url) + + if components.scheme not in allowed_schemes: + raise URLFetchError(_('Invalid URL scheme %s') % components.scheme) + + if components.scheme == 'file': + try: + return urllib.request.urlopen(url).read() + except urllib.error.URLError as uex: + raise URLFetchError(_('Failed to retrieve manifest: %s') % uex) + + try: + resp = requests.get(url, stream=True) + resp.raise_for_status() + + # We cannot use resp.text here because it would download the + # entire file, and a large enough file would bring down the + # engine. The 'Content-Length' header could be faked, so it's + # necessary to download the content in chunks to until + # max_manifest_size is reached. The chunk_size we use needs + # to balance CPU-intensive string concatenation with accuracy + # (eg. it's possible to fetch 1000 bytes greater than + # max_manifest_size with a chunk_size of 1000). + reader = resp.iter_content(chunk_size=1000) + result = "" + for chunk in reader: + result += chunk + if len(result) > cfg.CONF.max_manifest_size: + raise URLFetchError("Manifest exceeds maximum allowed size (%s" + " bytes)" % cfg.CONF.max_manifest_size) + return result + + except exceptions.RequestException as ex: + raise URLFetchError(_('Failed to retrieve manifest: %s') % ex) diff --git a/magnum/tests/api/controllers/v1/test_base.py b/magnum/tests/api/controllers/v1/test_base.py new file mode 100644 index 0000000000..e649e98271 --- /dev/null +++ b/magnum/tests/api/controllers/v1/test_base.py @@ -0,0 +1,45 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import datetime + +from mock import patch + +from magnum.api.controllers.v1 import base as api_base +from magnum.tests import base + + +class TestK8sResourceBase(base.BaseTestCase): + def setUp(self): + super(TestK8sResourceBase, self).setUp() + self.resource_base = api_base.K8sResourceBase( + uuid='fe78db47-9a37-4e9f-8572-804a10abc0aa', + created_at=datetime.datetime.utcnow(), + updated_at=datetime.datetime.utcnow()) + + def test_get_manifest_with_manifest(self): + expected_manifest = 'expected_manifest' + self.resource_base.manifest = expected_manifest + self.resource_base.manifest_url = 'file:///tmp/rc.yaml' + + self.assertEqual(expected_manifest, + self.resource_base._get_manifest()) + + @patch('magnum.common.urlfetch.get') + def test_get_manifest_with_manifest_url(self, + mock_urlfetch_get): + expected_manifest = 'expected_manifest_from_url' + mock_urlfetch_get.return_value = expected_manifest + + self.resource_base.manifest_url = 'file:///tmp/rc.yaml' + + self.assertEqual(expected_manifest, + self.resource_base._get_manifest()) diff --git a/magnum/tests/api/controllers/v1/test_replicationcontroller.py b/magnum/tests/api/controllers/v1/test_replicationcontroller.py index d2636efb82..02acb77135 100644 --- a/magnum/tests/api/controllers/v1/test_replicationcontroller.py +++ b/magnum/tests/api/controllers/v1/test_replicationcontroller.py @@ -36,7 +36,7 @@ class TestRCController(db_base.DbTestCase): params = ''' { "bay_uuid": "%s", - "replicationcontroller_data": "\ + "manifest": "\ {\ \\"id\\": \\"name_of_rc\\", \ \\"replicas\\": 3, \ diff --git a/magnum/tests/common/test_urlfetch.py b/magnum/tests/common/test_urlfetch.py new file mode 100644 index 0000000000..5f1db1edef --- /dev/null +++ b/magnum/tests/common/test_urlfetch.py @@ -0,0 +1,55 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +from mock import patch +from oslo.config import cfg + +from magnum.common import urlfetch +from magnum.tests import base + + +class TestUrlFetch(base.BaseTestCase): + def setUp(self): + super(TestUrlFetch, self).setUp() + + def test_get_unsupported_scheme(self): + self.assertRaises(urlfetch.URLFetchError, + urlfetch.get, + 'https://example.com', + ('http')) + + @patch('requests.get') + def test_get(self, + mock_request_get): + mock_reader = mock.MagicMock() + mock_reader.__iter__.return_value = ['a', 'b', 'c'] + mock_response = mock.MagicMock() + mock_response.iter_content.return_value = mock_reader + mock_request_get.return_value = mock_response + + self.assertEqual(urlfetch.get('http://example.com'), 'abc') + + @patch('requests.get') + def test_get_exceed_manifest_size(self, + mock_request_get): + cfg.CONF.set_override("max_manifest_size", 1) + + mock_reader = mock.MagicMock() + mock_reader.__iter__.return_value = ['a', 'b'] + mock_response = mock.MagicMock() + mock_response.iter_content.return_value = mock_reader + mock_request_get.return_value = mock_response + + self.assertRaises(urlfetch.URLFetchError, + urlfetch.get, + 'http://example.com') \ No newline at end of file