backport fixes to upstream

Fix dependencies
add altering with utf8 migrate_version table
change place of protocol files
add config option for defining regions
Use provided sshKeys in instance creation
Return requested url in discovery protocol
Fixes for cmd scripts
Catch nova exception and re-raise GCE exception
add two new empty projects for images list
fixes for getProject
- when cinderclient is old
- when neutron is absent
fix style and tests

Change-Id: Id11d9d7b8613a9f59e054d1fd39ea6dccbe7b4f0

gceapi/api/__init__.py

@@ -38,14 +38,20 @@ LOG = logging.getLogger(__name__)
 
 gce_opts = [
     cfg.StrOpt('network_api',
-               default="neutron",
+        default="neutron",
-               help='Name of network API. neutron(quantum) or nova'),
+        help='Name of network API. neutron(quantum) or nova'),
     cfg.StrOpt('keystone_gce_url',
-               default='http://127.0.0.1:5000/v2.0',
+        default='http://127.0.0.1:5000/v2.0',
-               help='Keystone URL'),
+        help='Keystone URL'),
     cfg.StrOpt('public_network',
-               default='public',
+        default='public',
-               help='name of public network'),
+        help='name of public network'),
+    cfg.StrOpt('protocol_dir',
+        default=None,
+        help='Place of protocol files'),
+    cfg.StrOpt('region_list',
+        default='RegionOne',
+        help='list of regions separated by commas'),
     ]
 
 CONF = cfg.CONF

gceapi/api/discovery.py

@@ -17,10 +17,13 @@ import os
 import threading
 import webob
 
+from oslo.config import cfg
+
 from gceapi.openstack.common import log as logging
 from gceapi import wsgi_ext as openstack_wsgi
 
 LOG = logging.getLogger(__name__)
+FLAGS = cfg.CONF
 
 
 class Controller(object):
@@ -31,25 +34,42 @@ class Controller(object):
     def discovery(self, req, version):
         """Returns appropriate json by its version."""
 
-        if version in self._files:
+        key = version + req.host_url
-            return self._files[version]
+        if key in self._files:
+            return self._files[key]
 
         self._lock.acquire()
         try:
-            if version in self._files:
+            if key in self._files:
-                return self._files[version]
+                return self._files[key]
 
             jfile = self._load_file(version)
             jfile = jfile.replace("{HOST_URL}", req.host_url)
-            self._files[version] = jfile
+            self._files[key] = jfile
             return jfile
         finally:
             self._lock.release()
 
     def _load_file(self, version):
+        file = version + ".json"
+
+        protocol_dir = FLAGS.get("protocol_dir")
+        if protocol_dir:
+            file_name = os.path.join(protocol_dir, file)
+            try:
+                f = open(file_name)
+                result = f.read()
+                f.close()
+                return result
+            except Exception as ex:
+                pass
+
+        # NOTE(apavlov): develop mode - try to find inside project
+        # ../../etc/gceapi/protocols/
         current_file = os.path.abspath(inspect.getsourcefile(lambda _: None))
         current_dir = os.path.dirname(current_file)
-        file_name = os.path.join(current_dir, "compute", version + ".json")
+        dir = os.path.join(current_dir, "../../etc/gceapi/protocols")
+        file_name = os.path.join(dir, file)
         try:
             f = open(file_name)
         except Exception as ex:

gceapi/api/instance_api.py

@@ -12,6 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import random
 import string
 
 from gceapi.api import base_api
@@ -256,18 +257,6 @@ class API(base_api.API):
         flavor_id = machine_type_api.API().get_item(
             context, flavor_name, scope)["id"]
 
-        try:
-            metadatas = body['metadata']['items']
-        except KeyError:
-            metadatas = []
-        instance_metadata = dict([(x['key'], x['value']) for x in metadatas])
-
-        ssh_keys = instance_metadata.pop('sshKeys', None)
-        if ssh_keys is not None:
-            key_name = ssh_keys.split('\n')[0].split(":")[0]
-        else:
-            key_name = project_api.API().get_gce_user_keypair_name(context)
-
         disks = body.get('disks', [])
         disks.sort(None, lambda x: x.get("boot", False), True)
         bdm = dict()
@@ -304,14 +293,34 @@ class API(base_api.API):
                 groups_names.add(sg["name"])
         groups_names = list(groups_names)
 
-        operation_util.start_operation(context, self._get_add_item_progress)
+        try:
-        instance = client.servers.create(name, None, flavor_id,
+            metadatas = body['metadata']['items']
-            meta=instance_metadata, min_count=1, max_count=1,
+        except KeyError:
-            security_groups=groups_names, key_name=key_name,
+            metadatas = []
-            availability_zone=scope.get_name(), block_device_mapping=bdm,
+        instance_metadata = dict([(x['key'], x['value']) for x in metadatas])
-            nics=nics)
+
-        if not acs:
+        ssh_keys = instance_metadata.pop('sshKeys', None)
-            operation_util.set_item_id(context, instance.id)
+        if ssh_keys is not None:
+            key = ssh_keys.split('\n')[0].split(":")
+            key_name = key[0] + "-" + str(random.randint(10000, 99999))
+            key_data = key[1]
+            client.keypairs.create(key_name, key_data)
+        else:
+            key_name = project_api.API().get_gce_user_keypair_name(context)
+
+        try:
+            operation_util.start_operation(
+                context, self._get_add_item_progress)
+            instance = client.servers.create(name, None, flavor_id,
+                meta=instance_metadata, min_count=1, max_count=1,
+                security_groups=groups_names, key_name=key_name,
+                availability_zone=scope.get_name(), block_device_mapping=bdm,
+                nics=nics)
+            if not acs:
+                operation_util.set_item_id(context, instance.id)
+        finally:
+            if ssh_keys is not None:
+                client.keypairs.delete(key_name)
 
         for disk in disks:
             instance_disk_api.API().register_item(context, name,
@@ -319,7 +328,7 @@ class API(base_api.API):
 
         instance = utils.to_dict(client.servers.get(instance.id))
         instance = self._prepare_instance(client, context, instance)
-        if "descripton" in body:
+        if "description" in body:
             instance["description"] = body["description"]
         instance = self._add_db_item(context, instance)
 

gceapi/api/network_nova_api.py

@@ -19,6 +19,7 @@ from gceapi.api import clients
 from gceapi.api import operation_util
 from gceapi.api import utils
 from gceapi import exception
+from gceapi.openstack.common.gettextutils import _
 
 
 class API(base_api.API):
@@ -35,7 +36,11 @@ class API(base_api.API):
 
     def get_item(self, context, name, scope=None):
         client = clients.nova(context)
-        network = client.networks.find(label=name)
+        try:
+            network = client.networks.find(label=name)
+        except clients.novaclient.exceptions.NotFound:
+            msg = _("Network resource '%s' could not be found.") % name
+            raise exception.NotFound(msg)
         gce_network = self._get_db_item_by_id(context, network.id)
         return self._prepare_network(utils.to_dict(network), gce_network)
 
@@ -72,7 +77,7 @@ class API(base_api.API):
         network = None
         try:
             network = self.get_item(context, name)
-        except clients.novaclient.exceptions.NotFound:
+        except exception.NotFound:
             pass
         if network is not None:
             raise exception.DuplicateVlan

gceapi/api/oauth.py

@@ -31,7 +31,8 @@ FLAGS = cfg.CONF
 LOG = logging.getLogger(__name__)
 
 
-INTERNAL_GCUTIL_PROJECTS = ["debian-cloud", "centos-cloud", "google"]
+INTERNAL_GCUTIL_PROJECTS = ["debian-cloud", "centos-cloud", "suse-cloud",
+                            "rhel-cloud", "google"]
 
 
 class OAuthFault(openstack_wsgi.Fault):

gceapi/api/project_api.py

@@ -12,12 +12,16 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from oslo.config import cfg
+
 from gceapi.api import base_api
 from gceapi.api import clients
 from gceapi.api import operation_util
 from gceapi.api import utils
 from gceapi import exception
 
+CONF = cfg.CONF
+
 
 class API(base_api.API):
     """GCE Projects API."""
@@ -43,18 +47,30 @@ class API(base_api.API):
                                      for l in nova_limits.absolute)
 
         cinder_client = clients.cinder(context)
-        result["cinder_quotas"] = utils.to_dict(
+        try:
-            cinder_client.quotas.get(project_id, usage=True))
+            result["cinder_quotas"] = utils.to_dict(
+                cinder_client.quotas.get(project_id, usage=True))
+        except TypeError:
+            # NOTE(apavlov): cinderclient of version 1.0.6 and below
+            # has no usage parameter
+            result["cinder_quotas"] = dict([("limit", x)
+                for x in utils.to_dict(cinder_client.quotas.get(project_id))])
+
+        net_api = CONF.get("network_api")
+        if net_api is None or ("quantum" in net_api
+                               or "neutron" in net_api):
+            neutron_client = clients.neutron(context)
+            result["neutron_quota"] = (
+                neutron_client.show_quota(project_id)["quota"])
+            result["neutron_quota"]["network_used"] = len(neutron_client
+                .list_networks(tenant_id=project_id)["networks"])
+            result["neutron_quota"]["floatingip_used"] = len(neutron_client
+                .list_floatingips(tenant_id=project_id)["floatingips"])
+            result["neutron_quota"]["security_group_used"] = len(neutron_client
+                .list_security_groups(tenant_id=project_id)["security_groups"])
+        else:
+            result["neutron_quota"] = {}
 
-        neutron_client = clients.neutron(context)
-        result["neutron_quota"] = (
-            neutron_client.show_quota(project_id)["quota"])
-        result["neutron_quota"]["network_used"] = len(neutron_client
-            .list_networks(tenant_id=project_id)["networks"])
-        result["neutron_quota"]["floatingip_used"] = len(neutron_client
-            .list_floatingips(tenant_id=project_id)["floatingips"])
-        result["neutron_quota"]["security_group_used"] = len(neutron_client
-            .list_security_groups(tenant_id=project_id)["security_groups"])
         return result
 
     def get_items(self, context, scope=None):

gceapi/api/region_api.py

@@ -12,10 +12,14 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from oslo.config import cfg
+
 from gceapi.api import base_api
 from gceapi.api import scopes
 from gceapi import exception
 
+CONF = cfg.CONF
+
 
 class API(base_api.API):
     """GCE Regions API
@@ -24,7 +28,12 @@ class API(base_api.API):
     """
 
     KIND = "region"
-    _REGIONS = ["nova"]
+    _REGIONS = []
+
+    def __init__(self, *args, **kwargs):
+        super(API, self).__init__(*args, **kwargs)
+        regions = CONF.get("region_list").split(",")
+        self._REGIONS = [r.strip() for r in regions]
 
     def _get_type(self):
         return self.KIND

gceapi/cmd/manage.py

@@ -61,8 +61,8 @@ command_opt = cfg.SubCommandOpt('command',
 def main():
     CONF.register_cli_opt(command_opt)
     try:
-        default_config_files = cfg.find_config_files('gceapi', 'gceapi-engine')
+        default_config_files = cfg.find_config_files('gceapi')
-        CONF(sys.argv[1:], project='gceapi', prog='gceapi-manage',
+        CONF(sys.argv[1:], project='gceapi', prog='gce-api-manage',
              version=version.version_info.version_string(),
              default_config_files=default_config_files)
         log.setup("gceapi")

gceapi/db/sqlalchemy/migrate_repo/versions/001_icehouse.py

@@ -33,6 +33,14 @@ def upgrade(migrate_engine):
     )
     items.create()
 
+    if migrate_engine.name == "mysql":
+        # In Folsom we explicitly converted migrate_version to UTF8.
+        sql = "ALTER TABLE migrate_version CONVERT TO CHARACTER SET utf8;"
+        # Set default DB charset to UTF8.
+        sql += "ALTER DATABASE %s DEFAULT CHARACTER SET utf8;" % \
+                migrate_engine.url.database
+        migrate_engine.execute(sql)
+
 
 def downgrade(migrate_engine):
     raise NotImplementedError("Downgrade from Icehouse is unsupported.")

gceapi/tests/api/common.py

@@ -60,12 +60,12 @@ COMMON_PENDING_OPERATION = {
 COMMON_PENDING_OPERATION.update(COMMON_OPERATION)
 
 REGION_OPERATION_SPECIFIC = {
-    u'id': u'1085621292163955072',
+    u'id': u'6294142421306477203',
     u'selfLink': u'http://localhost/compute/v1beta15/projects/'
-                 'fake_project/regions/nova/operations/'
+                 'fake_project/regions/RegionOne/operations/'
                  'operation-735d48a5-284e-4fb4-a10c-a465ac0b8888',
     u'region': u'http://localhost/compute/v1beta15/projects/'
-               'fake_project/regions/nova',
+               'fake_project/regions/RegionOne',
 }
 
 COMMON_REGION_FINISHED_OPERATION = copy.copy(COMMON_FINISHED_OPERATION)

gceapi/tests/api/test_addresses.py

@@ -17,16 +17,16 @@ from gceapi.tests.api import common
 
 EXPECTED_ADDRESSES = [{
     "kind": "compute#address",
-    "id": "2729532145628373701",
+    "id": "4065623605586261056",
     "creationTimestamp": "",
     "status": "IN USE",
     "region": "http://localhost/compute/v1beta15/projects/"
-        "fake_project/regions/nova",
+        "fake_project/regions/RegionOne",
     "name": "address-172-24-4-227",
     "description": "",
     "address": "172.24.4.227",
     "selfLink": "http://localhost/compute/v1beta15/projects/"
-        "fake_project/regions/nova/addresses/address-172-24-4-227",
+        "fake_project/regions/RegionOne/addresses/address-172-24-4-227",
     "users": ["http://localhost/compute/v1beta15/projects/"
         "fake_project/zones/nova/instances/i1"]
 }]
@@ -40,36 +40,37 @@ class AddressesTest(common.GCEControllerTest):
 
     def test_get_address_by_invalid_name(self):
         response = self.request_gce("/fake_project/regions/"
-                                    "nova/addresses/fake")
+                                    "RegionOne/addresses/fake")
         self.assertEqual(404, response.status_int)
 
     def test_get_address_by_name(self):
         response = self.request_gce("/fake_project/regions/"
-                                    "nova/addresses/address-172-24-4-227")
+                                    "RegionOne/addresses/address-172-24-4-227")
 
         self.assertEqual(200, response.status_int)
         self.assertEqual(response.json_body, EXPECTED_ADDRESSES[0])
 
     def test_get_address_list_filtered(self):
-        response = self.request_gce("/fake_project/regions/nova/addresses"
+        response = self.request_gce("/fake_project/regions/RegionOne/addresses"
                                     "?filter=name+eq+address-172-24-4-227")
         expected = {
                 "kind": "compute#addressList",
-                "id": "projects/fake_project/regions/nova/addresses",
+                "id": "projects/fake_project/regions/RegionOne/addresses",
                 "selfLink": "http://localhost/compute/v1beta15/projects"
-                    "/fake_project/regions/nova/addresses",
+                    "/fake_project/regions/RegionOne/addresses",
                 "items": [EXPECTED_ADDRESSES[0]]
                 }
 
         self.assertEqual(response.json_body, expected)
 
     def test_get_address_list(self):
-        response = self.request_gce("/fake_project/regions/nova/addresses")
+        response = self.request_gce("/fake_project/regions/RegionOne"
+                                    "/addresses")
         expected = {
                 "kind": "compute#addressList",
-                "id": "projects/fake_project/regions/nova/addresses",
+                "id": "projects/fake_project/regions/RegionOne/addresses",
                 "selfLink": "http://localhost/compute/v1beta15/projects"
-                    "/fake_project/regions/nova/addresses",
+                    "/fake_project/regions/RegionOne/addresses",
                 "items": EXPECTED_ADDRESSES
                 }
 
@@ -85,7 +86,7 @@ class AddressesTest(common.GCEControllerTest):
             "selfLink": "http://localhost/compute/v1beta15/projects"
                 "/fake_project/aggregated/addresses",
             "items": {
-                "regions/nova": {
+                "regions/RegionOne": {
                     "addresses": [EXPECTED_ADDRESSES[0]]
                 },
             }
@@ -102,7 +103,7 @@ class AddressesTest(common.GCEControllerTest):
             "selfLink": "http://localhost/compute/v1beta15/projects"
                 "/fake_project/aggregated/addresses",
             "items": {
-                "regions/nova": {
+                "regions/RegionOne": {
                     "addresses": EXPECTED_ADDRESSES
                 },
             }
@@ -111,19 +112,21 @@ class AddressesTest(common.GCEControllerTest):
         self.assertEqual(response.json_body, expected)
 
     def test_delete_address_with_invalid_name(self):
-        response = self.request_gce("/fake_project/regions/nova"
+        response = self.request_gce("/fake_project/regions/RegionOne"
             "/addresses/fake-address", method="DELETE")
         self.assertEqual(404, response.status_int)
 
     def test_delete_address(self):
         response = self.request_gce(
-                "/fake_project/regions/nova/addresses/address-172-24-4-227",
+                "/fake_project/regions/RegionOne/"
+                "addresses/address-172-24-4-227",
                 method="DELETE")
         expected = {
             "operationType": "delete",
-            "targetId": "2729532145628373701",
+            "targetId": "4065623605586261056",
             "targetLink": "http://localhost/compute/v1beta15/projects/"
-                "fake_project/regions/nova/addresses/address-172-24-4-227",
+                "fake_project/regions/RegionOne/"
+                "addresses/address-172-24-4-227",
         }
         expected.update(common.COMMON_REGION_FINISHED_OPERATION)
         self.assertEqual(200, response.status_int)
@@ -133,15 +136,16 @@ class AddressesTest(common.GCEControllerTest):
         request_body = {
             "name": "fake-address",
         }
-        response = self.request_gce("/fake_project/regions/nova/addresses",
+        response = self.request_gce("/fake_project/regions/RegionOne/"
+                                    "addresses",
                                     method="POST",
                                     body=request_body)
         self.assertEqual(200, response.status_int)
         expected = {
             "operationType": "insert",
-            "targetId": "5571612063911429008",
+            "targetId": "4570437344333712421",
             "targetLink": "http://localhost/compute/v1beta15/projects/"
-                "fake_project/regions/nova/addresses/fake-address",
+                "fake_project/regions/RegionOne/addresses/fake-address",
         }
         expected.update(common.COMMON_REGION_FINISHED_OPERATION)
         self.assertDictEqual(expected, response.json_body)

gceapi/tests/api/test_regions.py

@@ -18,11 +18,11 @@ from gceapi.tests.api import common
 
 EXPECTED_REGIONS = [
   {
-    "id": "6643843765891209621",
+    "id": "1905250285734383880",
     "kind": "compute#region",
     "selfLink": "http://localhost/compute/v1beta15/projects/fake_project"
-        "/regions/nova",
+        "/regions/RegionOne",
-    "name": "nova",
+    "name": "RegionOne",
     "status": "UP",
     "zones": [
       "http://localhost/compute/v1beta15/projects/fake_project"
@@ -31,14 +31,14 @@ EXPECTED_REGIONS = [
 ]
 
 
-class RegionsControllerTest(common.GCEControllerTest):
+class RegionsTest(common.GCEControllerTest):
     """
     Test of the GCE API /regions appliication.
     """
 
     def setUp(self):
         """Run before each test."""
-        super(RegionsControllerTest, self).setUp()
+        super(RegionsTest, self).setUp()
         self.controller = regions.Controller()
 
     def test_get_region_by_invalid_name(self):
@@ -46,14 +46,14 @@ class RegionsControllerTest(common.GCEControllerTest):
         self.assertEqual(404, response.status_int)
 
     def test_get_region(self):
-        response = self.request_gce('/fake_project/regions/nova')
+        response = self.request_gce('/fake_project/regions/RegionOne')
         expected = EXPECTED_REGIONS[0]
 
         self.assertEqual(response.json_body, expected)
 
     def test_get_region_list_filtered(self):
         response = self.request_gce("/fake_project/regions"
-                                    "?filter=name+eq+nova")
+                                    "?filter=name+eq+RegionOne")
         expected = {
             "kind": "compute#regionList",
             "id": "projects/fake_project/regions",

gceapi/tests/api/test_zones.py

@@ -23,7 +23,7 @@ EXPECTED_ZONES = [{
         "/zones/nova",
     "name": "nova",
     "status": "UP",
-    "region": "nova",
+    "region": "RegionOne",
 }]
 
 

requirements.txt

@@ -23,4 +23,3 @@ sqlalchemy-migrate>=0.8.2
 stevedore>=0.14
 suds>=0.4
 WebOb>=1.2.3
-websockify>=0.5.1,<0.6