GCE-API support service for OpenStack

Change-Id: I80f6b024cb40ba31ebaacf35d7364f66115da9c6
Implements: blueprint gce-api

.testr.conf

@@ -0,0 +1,4 @@
+[DEFAULT]
+test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 OS_LOG_CAPTURE=1 ${PYTHON:-python} -m subunit.run discover -t ./ gceapi/tests $LISTOPT $IDOPTION
+test_id_option=--load-list $IDFILE
+test_list_option=--list

HACKING.rst

@@ -0,0 +1,43 @@
+Gceapi Style Commandments
+=========================
+
+- Step 1: Read the OpenStack Style Commandments
+  https://github.com/openstack-dev/hacking/blob/master/doc/source/index.rst
+- Step 2: Read on
+
+Gceapi Specific Commandments
+----------------------------
+
+General
+-------
+- Do not use locals(). Example::
+
+    LOG.debug(_("volume %(vol_name)s: creating size %(vol_size)sG") %
+              locals()) # BAD
+
+    LOG.debug(_("volume %(vol_name)s: creating size %(vol_size)sG") %
+              {'vol_name': vol_name,
+               'vol_size': vol_size}) # OKAY
+
+- Use 'raise' instead of 'raise e' to preserve original traceback or exception being reraised::
+
+    except Exception as e:
+        ...
+        raise e  # BAD
+
+    except Exception:
+        ...
+        raise  # OKAY
+
+
+
+Creating Unit Tests
+-------------------
+For every new feature, unit tests should be created that both test and
+(implicitly) document the usage of said feature. If submitting a patch for a
+bug that had no unit test, a new passing unit test should be added. If a
+submitted bug fix does have a unit test, be sure to add a new one that fails
+without the patch and passes with the patch.
+
+For more information on creating unit tests and utilizing the testing
+infrastructure in OpenStack Gceapi, please read gceapi/testing/README.rst.

LICENSE

@@ -0,0 +1,176 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+

MANIFEST.in

@@ -0,0 +1,20 @@
+include run_tests.sh ChangeLog
+include README.rst builddeb.sh
+include MANIFEST.in pylintrc
+include AUTHORS
+include run_tests.py
+include HACKING.rst
+include LICENSE
+include ChangeLog
+include babel.cfg tox.ini
+include openstack-common.conf
+include gceapi/openstack/common/README
+include gceapi/db/sqlalchemy/migrate_repo/README
+include gceapi/db/sqlalchemy/migrate_repo/migrate.cfg
+include gceapi/db/sqlalchemy/migrate_repo/versions/*.sql
+graft doc
+graft etc
+graft gceapi/locale
+graft gceapi/tests
+graft tools
+global-exclude *.pyc

README.rst

@@ -0,0 +1,228 @@
+OpenStack Nova GCE API README
+-----------------------------
+
+Support of GCE-API for OpenStack. 
+
+Usage
+=====
+
+Download gcloud from Google and install it.
+There are two ways for using it:
+
+1. Run authorization command:
+	python google-cloud-sdk/platform/gcutil/gcutil --authorization_uri_base=http://localhost:8777 auth
+  and next any other commands: 
+	python google-cloud-sdk/platform/gcutil/gcutil --api_host=http://localhost:8777/ --authorization_uri_base=http://localhost:8777 --project demo listzones
+	 
+2. You have to have Google account
+  You can activate an already-authorized account with
+ 	gcloud config set account <account>
+  or authorize a new account with
+	gcloud auth login
+  Next you must authorize in Openstack by running authorization command:
+	gcutil --authorization_uri_base=http://localhost:8777 auth
+  and next you can run any other commands: 
+	gcutil --api_host=http://localhost:8777/ --authorization_uri_base=http://localhost:8777 --project demo listzones
+
+Make gcutil always use your GCE API endpoint using '--api_host' flag and your GCE API
+authorization endpoint using '--authorization_uri_base' flag. Also you can store this
+settings in "~/.gcutil.flags" file.
+ 
+If it doesn't work by some reason check that your PYTHONPATH is exported and set correctly to something like
+``/usr/lib/python2.7/dist-packages:/usr/local/lib/python2.7/dist-packages``.
+
+Limitations
+===========
+
+* Names are unique in GCE and are used for identification. Names are not unique in Nova. IDs are used instead.
+Solution: GCE-managed OpenStack installation should also maintain unique naming.
+
+* GCE IDs are ulong (8 bytes). Openstack IDs can be different (int, string) but mostly they are GUID (16 bytes).
+Solution: Since Openstack IDs are of different length and nature and because GCE API never uses ID as a parameter
+now, 8-byte hashes are generated and returned for any ID to report.
+
+* GCE allows per-user SSH key specification, but Nova supports only one key.
+Solution: Nova GCE API just uses first key.
+
+Authentication specifics
+========================
+
+GCE API uses OAuth2.0 for authentication. Simple sufficient implementation of this protocol
+was added into GCE API service in nova because of its absence in keystone.
+Current implementation allows operation with several OpenStack projects for
+one authenticated user as Google allows. For this initial token returned during
+authentication doesn't contain information about project required by keystone.
+Instead another authentication happens with each request when incoming project
+information is added to existing user info and new token is acquired in keystone.
+
+Supported Features
+==================
+
+Standard Query Params (except for fields and prettyPrint) are not supported.
+
+Supported resource types
+
+* Addresses
+* Disks
+* Firewalls
+* Images
+* Instances
+* MachineTypes
+* Networks
+* Projects
+* Regions
+* Zones
+
+Unsupported resource types
+
+* ForwardingRules
+* HttpHealthChecks
+* TargetPools
+
+In the lists below:
+"+" means supported
+"-" unsupported
+
++Addresses
+
++aggregatedList  GET  /project/aggregated/addresses
++delete  DELETE  /project/regions/region/addresses/address
++get  GET  /project/regions/region/addresses/address
++insert  POST  /project/regions/region/addresses
++list  GET  /project/regions/region/addresses
+
++Disks
+
++aggregatedList  GET  /project/aggregated/disks
++createSnapshot  POST  /project/zones/zone/disks/disk/createSnapshot
++delete  DELETE  /project/zones/zone/disks/disk
++get  GET  /project/zones/zone/disks/disk
++insert  POST  /project/zones/zone/disks
++list  GET  /project/zones/zone/disks
+
++Firewalls
+
++delete  DELETE  /project/global/firewalls/firewall
++get  GET  /project/global/firewalls/firewall
++insert  POST  /project/global/firewalls
++list  GET  /project/global/firewalls
+-patch  PATCH  /project/global/firewalls/firewall
+-update  PUT  /project/global/firewalls/firewall
+
+-ForwardingRules
+
+-aggregatedList  GET  /project/aggregated/forwardingRules
+-delete  DELETE  /project/regions/region/forwardingRules/forwardingRule
+-get  GET  /project/regions/region/forwardingRules/forwardingRule
+-insert  POST  /project/regions/region/forwardingRules
+-list  GET  /project/regions/region/forwardingRules
+-setTarget  POST  /project/regions/region/forwardingRules/forwardingRule/setTarget
+
++GlobalOperations
+
++aggregatedList  GET  /project/aggregated/operations
++delete  DELETE  /project/global/operations/operation
++get  GET  /project/global/operations/operation
++list  GET  /project/global/operations
+
+-HttpHealthChecks
+
+-delete  DELETE  /project/global/httpHealthChecks/httpHealthCheck
+-get  GET  /project/global/httpHealthChecks/httpHealthCheck
+-insert  POST  /project/global/httpHealthChecks
+-list  GET  /project/global/httpHealthChecks
+-patch  PATCH  /project/global/httpHealthChecks/httpHealthCheck
+-update  PUT  /project/global/httpHealthChecks/httpHealthCheck
+
++Images
+
++delete  DELETE  /project/global/images/image
+-deprecate  POST  /project/global/images/image/deprecate
++get  GET  /project/global/images/image
++insert  POST  /project/global/images
++list  GET  /project/global/images
+
++Instances
+
++addAccessConfig  POST  /project/zones/zone/instances/instance/addAccessConfig
++aggregatedList  GET  /project/aggregated/instances
++attachDisk  POST  /project/zones/zone/instances/instance/attachDisk
++delete  DELETE  /project/zones/zone/instances/instance
++deleteAccessConfig  POST /project/zones/zone/instances/instance/deleteAccessConfig
++detachDisk  POST  /project/zones/zone/instances/instance/detachDisk
++get  GET  /project/zones/zone/instances/instance
+-getSerialPortOutput  GET  /project/zones/zone/instances/instance/serialPort
++insert  POST  /project/zones/zone/instances
++list  GET  /project/zones/zone/instances
++reset  POST  /project/zones/zone/instances/instance/reset
+-setMetadata  POST  /project/zones/zone/instances/instance/setMetadata
+-setTags  POST  /project/zones/zone/instances/instance/setTags
+-setScheduling  POST  /project/zones/zone/instances/instance/setScheduling
+
++MachineTypes
+
++aggregatedList  GET  /project/aggregated/machineTypes
++get  GET  /project/zones/zone/machineTypes/machineType
++list  GET  /project/zones/zone/machineTypes
+
++Networks
+
++delete  DELETE  /project/global/networks/network
++get  GET  /project/global/networks/network
++insert  POST  /project/global/networks
++list  GET  /project/global/networks
+
++Projects
+
++get  GET  /project
++setCommonInstanceMetadata  POST  /project/setCommonInstanceMetadata
+
+-RegionOperations
+
++delete  DELETE  /project/regions/region/operations/operation
++get  GET  /project/regions/region/operations/operation
++list  GET  /project/regions/region/operations
+
++Regions
+
++get  GET  /project/regions/region
++list  GET  /project/regions
+
++Routes
+
++delete  DELETE  /project/global/routes/route
++get  GET  /project/global/routes/route
++insert  POST  /project/global/routes
++list  GET  /project/global/routes
+
++Snapshots
+
++delete  DELETE  /project/global/snapshots/snapshot
++get  GET  /project/global/snapshots/snapshot
++list  GET  /project/global/snapshots
+
+-TargetPools
+
+-addHealthCheck  POST /project/regions/region/targetPools/targetPool/addHealthCheck
+-addInstance  POST  /project/regions/region/targetPools/targetPool/addInstance
+-aggregatedList  GET  /project/aggregated/targetPools
+-delete  DELETE  /project/regions/region/targetPools/targetPool
+-get  GET  /project/regions/region/targetPools/targetPool
+-getHealth  POST  /project/regions/region/targetPools/targetPool/getHealth
+-insert  POST  /project/regions/region/targetPools
+-list  GET  /project/regions/region/targetPools
+-removeHealthCheck  POST /project/regions/region/targetPools/targetPool/removeHealthCheck
+-removeInstance  POST /project/regions/region/targetPools/targetPool/removeInstance
+-setBackup  POST  /project/regions/region/targetPools/targetPool/setBackup
+
++ZoneOperations
+
++delete  DELETE  /project/zones/zone/operations/operation
++get  GET  /project/zones/zone/operations/operation
++list  GET  /project/zones/zone/operations
+
++Zones
+
++get  GET  /project/zones/zone
++list  GET  /project/zones
+

babel.cfg

@@ -0,0 +1 @@
+[python: **.py]

bin/gceapi-db-setup

@@ -0,0 +1,291 @@
+#!/bin/bash
+#
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+#
+# Print --help output and exit.
+#
+usage() {
+
+cat << EOF
+Set up a local MySQL database for use with gceapi.
+This script will create a 'gceapi' database that is accessible
+only on localhost by user 'gceapi' with password 'gceapi'.
+
+Usage: gceapi-db-setup <rpm|deb> [options]
+Options:
+        select a distro type (rpm or debian)
+
+        --help | -h
+                Print usage information.
+        --password <pw> | -p <pw>
+                Specify the password for the 'gceapi' MySQL user that will
+                use to connect to the 'gceapi' MySQL database. By default,
+                the password 'gceapi' will be used.
+        --rootpw <pw> | -r <pw>
+                Specify the root MySQL password. If the script installs
+                the MySQL server, it will set the root password to this value
+                instead of prompting for a password. If the MySQL server is
+                already installed, this password will be used to connect to the
+                database instead of having to prompt for it.
+        --yes | -y
+                In cases where the script would normally ask for confirmation
+                before doing something, such as installing mysql-server,
+                just assume yes. This is useful if you want to run the script
+                non-interactively.
+EOF
+
+        exit 0
+}
+
+install_mysql_server() {
+        if [ -z "${ASSUME_YES}" ] ; then
+                $PACKAGE_INSTALL mysql-server
+        else
+                $PACKAGE_INSTALL -y mysql-server
+        fi
+}
+
+start_mysql_server() {
+        $SERVICE_START
+}
+
+MYSQL_GCEAPI_PW_DEFAULT="gceapi"
+MYSQL_GCEAPI_PW=${MYSQL_GCEAPI_PW_DEFAULT}
+GCEAPI_CONFIG="/etc/gceapi/gceapi.conf"
+ASSUME_YES=""
+ELEVATE=""
+
+# Check for root privileges
+if [[ $EUID -ne 0 ]] ; then
+        echo "This operation requires superuser privileges, using sudo:"
+        if sudo -l > /dev/null ; then
+                ELEVATE="sudo"
+        else
+                exit 1
+        fi
+fi
+
+case "$1" in
+        rpm)
+                echo "Installing on an RPM system."
+                PACKAGE_INSTALL="$ELEVATE yum install"
+                PACKAGE_STATUS="rpm -q"
+                SERVICE_MYSQLD="mysqld"
+                SERVICE_START="$ELEVATE service $SERVICE_MYSQLD start"
+                SERVICE_STATUS="service $SERVICE_MYSQLD status"
+                SERVICE_ENABLE="$ELEVATE chkconfig"
+                ;;
+        deb)
+                echo "Installing on a Debian system."
+                PACKAGE_INSTALL="$ELEVATE apt-get install"
+                PACKAGE_STATUS="dpkg-query -s"
+                SERVICE_MYSQLD="mysql"
+                SERVICE_START="$ELEVATE service $SERVICE_MYSQLD start"
+                SERVICE_STATUS="$ELEVATE service $SERVICE_MYSQLD status"
+                SERVICE_ENABLE=""
+                ;;
+        *)
+                usage
+                ;;
+esac
+
+while [ $# -gt 0 ]
+do
+        case "$1" in
+                -h|--help)
+                        usage
+                        ;;
+                -p|--password)
+                        shift
+                        MYSQL_GCEAPI_PW=${1}
+                        ;;
+                -r|--rootpw)
+                        shift
+                        MYSQL_ROOT_PW=${1}
+                        ;;
+                -y|--yes)
+                        ASSUME_YES="yes"
+                        ;;
+                *)
+                        # ignore
+                        ;;
+        esac
+        shift
+done
+
+
+# Make sure MySQL is installed.
+
+NEW_MYSQL_INSTALL=0
+if ! $PACKAGE_STATUS mysql-server && ! $PACKAGE_STATUS mariadb-server > /dev/null
+then
+        if [ -z "${ASSUME_YES}" ] ; then
+                printf "mysql-server is not installed. Would you like to install it now? (y/n): "
+                read response
+                case "$response" in
+                        y|Y)
+                                ;;
+                        n|N)
+                                echo "mysql-server must be installed. Please install it before proceeding."
+                                exit 0
+                                ;;
+                        *)
+                                echo "Invalid response."
+                                exit 1
+                esac
+        fi
+
+        NEW_MYSQL_INSTALL=1
+        install_mysql_server
+fi
+
+
+# Make sure mysqld is running.
+
+if ! $SERVICE_STATUS > /dev/null
+then
+        if [ -z "${ASSUME_YES}" ] ; then
+                printf "$SERVICE_MYSQLD is not running. Would you like to start it now? (y/n): "
+                read response
+                case "$response" in
+                        y|Y)
+                                ;;
+                        n|N)
+                                echo "$SERVICE_MYSQLD must be running. Please start it before proceeding."
+                                exit 0
+                                ;;
+                        *)
+                                echo "Invalid response."
+                                exit 1
+                esac
+        fi
+
+        start_mysql_server
+
+        # If we both installed and started, ensure it starts at boot
+        [ $NEW_MYSQL_INSTALL -eq 1 ] && $SERVICE_ENABLE $SERVICE_MYSQLD on
+fi
+
+
+# Get MySQL root access.
+
+if [ $NEW_MYSQL_INSTALL -eq 1 ]
+then
+        if [ ! "${MYSQL_ROOT_PW+defined}" ] ; then
+                echo "Since this is a fresh installation of MySQL, please set a password for the 'root' mysql user."
+
+                PW_MATCH=0
+                while [ $PW_MATCH -eq 0 ]
+                do
+                        printf "Enter new password for 'root' mysql user: "
+                        read -s MYSQL_ROOT_PW
+                        echo
+                        printf "Enter new password again: "
+                        read -s PW2
+                        echo
+                        if [ "${MYSQL_ROOT_PW}" = "${PW2}" ] ; then
+                                PW_MATCH=1
+                        else
+                                echo "Passwords did not match."
+                        fi
+                done
+        fi
+
+        echo "UPDATE mysql.user SET password = password('${MYSQL_ROOT_PW}') WHERE user = 'root'; DELETE FROM mysql.user WHERE user = ''; flush privileges;" | mysql -u root
+        if ! [ $? -eq 0 ] ; then
+                echo "Failed to set password for 'root' MySQL user."
+                exit 1
+        fi
+elif [ ! "${MYSQL_ROOT_PW+defined}" ] ; then
+        printf "Please enter the password for the 'root' MySQL user: "
+        read -s MYSQL_ROOT_PW
+        echo
+fi
+
+
+# Sanity check MySQL credentials.
+
+MYSQL_ROOT_PW_ARG=""
+if [ "${MYSQL_ROOT_PW+defined}" ]
+then
+        MYSQL_ROOT_PW_ARG="--password=${MYSQL_ROOT_PW}"
+fi
+echo "SELECT 1;" | mysql -u root ${MYSQL_ROOT_PW_ARG} > /dev/null
+if ! [ $? -eq 0 ]
+then
+        echo "Failed to connect to the MySQL server. Please check your root user credentials."
+        exit 1
+fi
+echo "Verified connectivity to MySQL."
+
+
+# Now create the db.
+
+echo "Creating 'gceapi' database."
+cat << EOF | mysql -u root ${MYSQL_ROOT_PW_ARG}
+CREATE DATABASE IF NOT EXISTS gceapi;
+GRANT ALL ON gceapi.* TO 'gceapi'@'localhost' IDENTIFIED BY '${MYSQL_GCEAPI_PW}';
+GRANT ALL ON gceapi.* TO 'gceapi'@'%' IDENTIFIED BY '${MYSQL_GCEAPI_PW}';
+flush privileges;
+EOF
+
+
+# Make sure gceapi configuration has the right MySQL password.
+
+if [ "${MYSQL_GCEAPI_PW}" != "${MYSQL_GCEAPI_PW_DEFAULT}" ] ; then
+        echo "Updating 'gceapi' database password in ${GCEAPI_CONFIG}"
+        sed -i -e "s/mysql:\/\/gceapi:\(.*\)@/mysql:\/\/gceapi:${MYSQL_GCEAPI_PW}@/" ${GCEAPI_CONFIG}
+fi
+
+# override the logging config in gceapi.conf
+log_conf=$(mktemp /tmp/gceapi-logging.XXXXXXXXXX.conf)
+cat <<EOF > $log_conf
+[loggers]
+keys=root
+
+[handlers]
+keys=consoleHandler
+
+[formatters]
+keys=simpleFormatter
+
+[logger_root]
+level=INFO
+handlers=consoleHandler
+
+[handler_consoleHandler]
+class=StreamHandler
+formatter=simpleFormatter
+args=(sys.stdout,)
+
+[formatter_simpleFormatter]
+format=%(name)s - %(levelname)s - %(message)s
+EOF
+
+gce-api-manage --log-config=$log_conf db_sync
+rm $log_conf
+
+# Do a final sanity check on the database.
+
+echo "SELECT * FROM migrate_version;" | mysql -u gceapi --password=${MYSQL_GCEAPI_PW} gceapi > /dev/null
+if ! [ $? -eq 0 ]
+then
+        echo "Final sanity check failed."
+        exit 1
+fi
+
+echo "Complete!"

etc/gceapi/api-paste.ini

@@ -0,0 +1,48 @@
+##########
+# Shared #
+##########
+
+[filter:keystonecontext]
+paste.filter_factory = gceapi.auth:GceapiKeystoneContext.factory
+
+[filter:authtoken]
+paste.filter_factory = keystoneclient.middleware.auth_token:filter_factory
+auth_port = 35357
+auth_protocol = http
+auth_version = v2.0
+
+#######
+# GCE #
+#######
+[composite:gce]
+use = egg:Paste#urlmap
+/: gceapi_oauth
+/compute/v1/projects/: gceapi
+/discovery/v1/apis/compute/: gceapi_discovery
+
+[composite:gceapi]
+use = call:gceapi.auth:pipeline_factory
+keystone = gceauthtoken authtoken keystonecontext gceexecutor
+
+[filter:gceauthtoken]
+paste.filter_factory = gceapi.api.oauth:filter_factory
+
+[app:gceexecutor]
+paste.app_factory = gceapi.api:APIRouter.factory
+
+[composite:gceapi_oauth]
+use = call:gceapi.auth:pipeline_factory
+noauth = gceexecutor_oauth
+keystone = gceexecutor_oauth
+
+[app:gceexecutor_oauth]
+paste.app_factory = gceapi.api:APIRouterOAuth.factory
+
+[composite:gceapi_discovery]
+use = call:gceapi.auth:pipeline_factory
+noauth = gceexecutor_discovery
+keystone = gceexecutor_discovery
+
+[app:gceexecutor_discovery]
+paste.app_factory = gceapi.api:APIRouterDiscovery.factory
+####

gceapi/__init__.py

@@ -0,0 +1,31 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+:mod:`gceapi` -- Cloud IaaS Platform
+===================================
+
+.. automodule:: gceapi
+   :platform: Unix
+   :synopsis: Infrastructure-as-a-Service Cloud platform.
+"""
+
+import gettext
+
+
+gettext.install('gceapi', unicode=1)

gceapi/api/__init__.py

@@ -0,0 +1,253 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo.config import cfg
+
+from gceapi.api import addresses
+from gceapi.api import discovery
+from gceapi.api import disks
+from gceapi.api import firewalls
+from gceapi.api import images
+from gceapi.api import instances
+from gceapi.api import machine_types
+from gceapi.api import networks
+from gceapi.api import oauth
+from gceapi.api import operations
+from gceapi.api import projects
+from gceapi.api import regions
+from gceapi.api import routes
+from gceapi.api import snapshots
+from gceapi.api import zones
+from gceapi.openstack.common import log as logging
+from gceapi import wsgi
+from gceapi import wsgi_ext as openstack_api
+
+gce_opts = [
+        cfg.StrOpt('keystone_gce_url',
+            default='http://127.0.0.1:5000/v2.0',
+            help='Keystone URL'),
+        cfg.IntOpt('gce_port',
+            default=8777,
+            help='the port of the gce api server'),
+        cfg.StrOpt('gce_scheme',
+            default='http',
+            help='the protocol to use when connecting to the gce api '
+                 'server (http, https)'),
+        cfg.StrOpt('gce_path',
+            default='/compute/v1beta15/projects',
+            help='the path prefix used to call the gce api server'),
+        cfg.StrOpt('public_network',
+            default='public',
+            help='name of public network'),
+        ]
+
+CONF = cfg.CONF
+CONF.register_opts(gce_opts)
+
+LOG = logging.getLogger(__name__)
+
+
+class APIRouter(wsgi.Router):
+    """
+    Routes requests on the GCE API to the appropriate controller
+    and method.
+    """
+
+    @classmethod
+    def factory(cls, global_config, **local_config):
+        """Simple paste factory, `gceapi.wsgi.Router` doesn't have one."""
+
+        return cls()
+
+    def __init__(self):
+        mapper = openstack_api.ProjectMapper()
+        self.resources = {}
+        self._setup_routes(mapper)
+        super(APIRouter, self).__init__(mapper)
+
+    def _setup_routes(self, mapper):
+        mapper.redirect("", "/")
+
+        self.resources['regions'] = regions.create_resource()
+        self.resources['firewalls'] = firewalls.create_resource()
+        self.resources['disks'] = disks.create_resource()
+        self.resources['machineTypes'] = machine_types.create_resource()
+        self.resources['instances'] = instances.create_resource()
+        self.resources['images'] = images.create_resource()
+        self.resources['instances'] = instances.create_resource()
+        self.resources['zones'] = zones.create_resource()
+        self.resources['networks'] = networks.create_resource()
+        self.resources['instances'] = instances.create_resource()
+        self.resources['projects'] = projects.create_resource()
+        self.resources['snapshots'] = snapshots.create_resource()
+        self.resources['addresses'] = addresses.create_resource()
+        self.resources['routes'] = routes.create_resource()
+        self.resources['operations'] = operations.create_resource()
+
+        mapper.resource("disks", "zones/{scope_id}/disks",
+                controller=self.resources['disks'])
+        mapper.connect("/{project_id}/aggregated/disks",
+                controller=self.resources['disks'],
+                action="aggregated_list",
+                conditions={"method": ["GET"]})
+        mapper.connect("/{project_id}/zones/{scope_id}/disks/{id}/"
+                       "createSnapshot",
+                controller=self.resources['disks'],
+                action="create_snapshot",
+                conditions={"method": ["POST"]})
+
+        mapper.resource("machineTypes", "zones/{scope_id}/machineTypes",
+                controller=self.resources['machineTypes'])
+        mapper.connect("/{project_id}/aggregated/machineTypes",
+                controller=self.resources['machineTypes'],
+                action="aggregated_list",
+                conditions={"method": ["GET"]})
+
+        mapper.resource("instances", "zones/{scope_id}/instances",
+                controller=self.resources['instances'])
+        mapper.connect("/{project_id}/aggregated/instances",
+                controller=self.resources['instances'],
+                action="aggregated_list",
+                conditions={"method": ["GET"]})
+        mapper.connect("/{project_id}/zones/{scope_id}/instances/{id}/"
+                       "addAccessConfig",
+                controller=self.resources['instances'],
+                action="add_access_config",
+                conditions={"method": ["POST"]})
+        mapper.connect("/{project_id}/zones/{scope_id}/instances/{id}/"
+                       "deleteAccessConfig",
+                controller=self.resources['instances'],
+                action="delete_access_config",
+                conditions={"method": ["POST"]})
+        mapper.connect("/{project_id}/zones/{scope_id}/instances/{id}/reset",
+                controller=self.resources['instances'],
+                action="reset_instance",
+                conditions={"method": ["POST"]})
+        mapper.connect("/{project_id}/zones/{scope_id}/instances/{id}/"
+                       "attachDisk",
+                controller=self.resources['instances'],
+                action="attach_disk",
+                conditions={"method": ["POST"]})
+        mapper.connect("/{project_id}/zones/{scope_id}/instances/{id}/"
+                       "detachDisk",
+                controller=self.resources['instances'],
+                action="detach_disk",
+                conditions={"method": ["POST"]})
+
+        mapper.resource("images", "global/images",
+                controller=self.resources['images'])
+        mapper.resource("regions", "regions",
+                controller=self.resources['regions'])
+        mapper.resource("zones", "zones",
+                controller=self.resources['zones'])
+        mapper.resource("networks", "global/networks",
+                controller=self.resources["networks"])
+        mapper.resource("firewalls", "global/firewalls",
+                controller=self.resources["firewalls"])
+        mapper.resource("routes", "global/routes",
+                controller=self.resources['routes'])
+
+        mapper.connect("/{project_id}", controller=self.resources['projects'],
+                action="show", conditions={"method": ["GET"]})
+        mapper.connect("/{project_id}/setCommonInstanceMetadata",
+                controller=self.resources['projects'],
+                action="set_common_instance_metadata",
+                conditions={"method": ["POST"]})
+
+        mapper.resource("addresses", "regions/{scope_id}/addresses",
+                controller=self.resources['addresses'])
+        mapper.connect("/{project_id}/aggregated/addresses",
+                controller=self.resources['addresses'],
+                action="aggregated_list",
+                conditions={"method": ["GET"]})
+
+        mapper.resource("snapshots", "global/snapshots",
+                controller=self.resources['snapshots'])
+
+        mapper.resource("operations", "global/operations",
+                controller=self.resources['operations'])
+        mapper.resource("operations", "regions/{scope_id}/operations",
+                controller=self.resources['operations'])
+        mapper.resource("operations", "zones/{scope_id}/operations",
+                controller=self.resources['operations'])
+        mapper.connect("/{project_id}/aggregated/operations",
+                controller=self.resources['operations'],
+                action="aggregated_list",
+                conditions={"method": ["GET"]})
+
+
+class APIRouterOAuth(wsgi.Router):
+    """
+    Routes requests on the OAuth2.0 to the appropriate controller
+    and method.
+    """
+
+    @classmethod
+    def factory(cls, global_config, **local_config):
+        """Simple paste factory, `gceapi.wsgi.Router` doesn't have one."""
+
+        return cls()
+
+    def __init__(self):
+        mapper = openstack_api.ProjectMapper()
+        self.resources = {}
+        self._setup_routes(mapper)
+        super(APIRouterOAuth, self).__init__(mapper)
+
+    def _setup_routes(self, mapper):
+        mapper.redirect("", "/")
+
+        self.resources['oauth'] = oauth.create_resource()
+
+        mapper.connect("/auth",
+            controller=self.resources['oauth'],
+            action="auth",
+            conditions={"method": ["GET"]})
+        mapper.connect("/approval",
+            controller=self.resources['oauth'],
+            action="approval",
+            conditions={"method": ["POST"]})
+        mapper.connect("/token",
+            controller=self.resources['oauth'],
+            action="token",
+            conditions={"method": ["POST"]})
+
+
+class APIRouterDiscovery(wsgi.Router):
+    """
+    Routes requests on the GCE discovery API to the appropriate controller
+    and method.
+    """
+
+    @classmethod
+    def factory(cls, global_config, **local_config):
+        """Simple paste factory, `gceapi.wsgi.Router` doesn't have one."""
+
+        return cls()
+
+    def __init__(self):
+        mapper = openstack_api.ProjectMapper()
+        self.resources = {}
+        self._setup_routes(mapper)
+        super(APIRouterDiscovery, self).__init__(mapper)
+
+    def _setup_routes(self, mapper):
+        mapper.redirect("", "/")
+
+        self.resources['discovery'] = discovery.create_resource()
+
+        mapper.connect("/{version}/rest",
+                controller=self.resources['discovery'],
+                action="discovery",
+                conditions={"method": ["GET"]})

gceapi/api/address_api.py

@@ -0,0 +1,26 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gceapi.api import address_neutron_api
+from gceapi.api import address_nova_api
+from gceapi.api import base_api
+
+
+class API(base_api.API):
+    """GCE Address API."""
+
+    NEUTRON_API_MODULE = address_neutron_api
+    NOVA_API_MODULE = address_nova_api
+
+    __metaclass__ = base_api.NetSingleton

gceapi/api/address_neutron_api.py

@@ -0,0 +1,120 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo.config import cfg
+
+from gceapi.api import base_api
+from gceapi.api import clients
+from gceapi.api import network_api
+from gceapi.api import operation_util
+from gceapi.api import region_api
+from gceapi.api import scopes
+from gceapi import exception
+from gceapi.openstack.common.gettextutils import _
+
+CONF = cfg.CONF
+
+
+class API(base_api.API):
+    """GCE Address API - neutron implementation."""
+
+    KIND = "address"
+    PERSISTENT_ATTRIBUTES = ["id", "creationTimestamp", "name", "description"]
+
+    def __init__(self, *args, **kwargs):
+        super(API, self).__init__(*args, **kwargs)
+        self._region_api = region_api.API()
+
+    def _get_type(self):
+        return self.KIND
+
+    def _get_persistent_attributes(self):
+        return self.PERSISTENT_ATTRIBUTES
+
+    def get_scopes(self, context, item):
+        region = item["scope"]
+        if region is not None:
+            return [scopes.RegionScope(region)]
+        return self._region_api.get_items_as_scopes(context)
+
+    def get_item(self, context, name, scope=None):
+        return self._get_floating_ips(context, scope, name)[0]
+
+    def get_items(self, context, scope=None):
+        return self._get_floating_ips(context, scope)
+
+    def delete_item(self, context, name, scope=None):
+        floating_ip = self._get_floating_ips(context, scope, name)[0]
+        operation_util.start_operation(context)
+        self._delete_db_item(context, floating_ip)
+        clients.neutron(context).delete_floatingip(floating_ip["id"])
+
+    def add_item(self, context, name, body, scope=None):
+        if any(x["name"] == name
+               for x in self._get_floating_ips(context, scope)):
+            raise exception.InvalidInput(
+                    _("The resource '%s' already exists.") % name)
+        public_network_id = network_api.API().get_public_network_id(context)
+        operation_util.start_operation(context)
+        floating_ip = clients.neutron(context).create_floatingip(
+            {"floatingip": {"floating_network_id": public_network_id}})
+        floating_ip = self._prepare_floating_ip(
+            clients.nova(context), floating_ip["floatingip"], scope)
+        floating_ip["name"] = body["name"]
+        if "description" in body:
+            floating_ip["description"] = body["description"]
+        floating_ip = self._add_db_item(context, floating_ip)
+        return floating_ip
+
+    def _get_floating_ips(self, context, scope, name=None):
+        results = clients.neutron(context).list_floatingips(
+                tenant_id=context.project_id)["floatingips"]
+        gce_floating_ips = self._get_db_items_dict(context)
+        nova_client = clients.nova(context)
+        results = [self._prepare_floating_ip(nova_client, x, scope,
+                                             gce_floating_ips.get(x["id"]))
+                   for x in results]
+        unnamed_ips = self._purge_db(context, results, gce_floating_ips)
+        self._add_nonnamed_items(context, unnamed_ips)
+        if name is None:
+            return results
+
+        for item in results:
+            if item["name"] == name:
+                return [item]
+
+        raise exception.NotFound
+
+    def _prepare_floating_ip(self, nova_client, floating_ip, scope,
+                             db_item=None):
+        floating_ip["scope"] = scope
+        fixed_ip_address = floating_ip.get("fixed_ip_address")
+        floating_ip["status"] = "IN USE" if fixed_ip_address else "RESERVED"
+
+        if fixed_ip_address is not None:
+            instances = nova_client.servers.list(
+                search_opts={"fixed_ip": fixed_ip_address})
+            if instances:
+                floating_ip["instance_name"] = instances[0].name
+                floating_ip["instance_zone"] = getattr(
+                    instances[0], "OS-EXT-AZ:availability_zone")
+
+        return self._prepare_item(floating_ip, db_item)
+
+    def _add_nonnamed_items(self, context, items):
+        for item in items:
+            item["name"] = ("address-" +
+                            item["floating_ip_address"].replace(".", "-"))
+            item["creationTimestamp"] = ""
+            self._add_db_item(context, item)

gceapi/api/address_nova_api.py

@@ -0,0 +1,124 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gceapi.api import base_api
+from gceapi.api import clients
+from gceapi.api import operation_util
+from gceapi.api import region_api
+from gceapi.api import scopes
+from gceapi.api import utils
+from gceapi import exception
+from gceapi.openstack.common.gettextutils import _
+
+
+class API(base_api.API):
+    """GCE Address API - nova-network implementation."""
+
+    KIND = "address"
+    PERSISTENT_ATTRIBUTES = ["id", "creationTimestamp", "name", "description"]
+
+    def __init__(self, *args, **kwargs):
+        super(API, self).__init__(*args, **kwargs)
+        self._region_api = region_api.API()
+
+    def _get_type(self):
+        return self.KIND
+
+    def _get_persistent_attributes(self):
+        return self.PERSISTENT_ATTRIBUTES
+
+    def get_scopes(self, context, item):
+        region = item["scope"]
+        if region is not None:
+            return [scopes.RegionScope(region)]
+        return self._region_api.get_items_as_scopes(context)
+
+    def get_item(self, context, name, scope=None):
+        client = clients.nova(context)
+        return self._get_floating_ips(client, context, scope, name)[0]
+
+    def get_items(self, context, scope=None):
+        client = clients.nova(context)
+        return self._get_floating_ips(client, context, scope)
+
+    def delete_item(self, context, name, scope=None):
+        client = clients.nova(context)
+        floating_ip = self._get_floating_ips(client, context, scope, name)[0]
+        operation_util.start_operation(context)
+        self._delete_db_item(context, floating_ip)
+        client.floating_ips.delete(floating_ip["id"])
+
+    def add_item(self, context, name, body, scope=None):
+        client = clients.nova(context)
+        if any(x["name"] == name
+               for x in self._get_floating_ips(client, context, scope)):
+            raise exception.InvalidInput(
+                    _("The resource '%s' already exists.") % name)
+        operation_util.start_operation(context)
+        result = client.floating_ips.create()
+        floating_ip = self._prepare_floating_ip(client, context, result, scope)
+        floating_ip["name"] = body["name"]
+        if "description" in body:
+            floating_ip["description"] = body["description"]
+        floating_ip = self._add_db_item(context, floating_ip)
+        return floating_ip
+
+    def _get_floating_ips(self, client, context, scope, name=None):
+        results = client.floating_ips.list()
+        gce_floating_ips = self._get_db_items_dict(context)
+        results = [self._prepare_floating_ip(
+                      client, context, x, scope,
+                      gce_floating_ips.get(str(x.id)))
+                   for x in results]
+        unnamed_ips = self._purge_db(context, results, gce_floating_ips)
+        self._add_nonnamed_items(context, unnamed_ips)
+
+        if name is None:
+            return results
+
+        for item in results:
+            if item["name"] == name:
+                return [item]
+
+        raise exception.NotFound
+
+    def _prepare_floating_ip(self, client, context, floating_ip, scope,
+                             db_item=None):
+        floating_ip = utils.to_dict(floating_ip)
+        fixed_ip = floating_ip.get("fixed_ip")
+        floating_ip = {
+            "fixed_ip_address": fixed_ip if fixed_ip else None,
+            "floating_ip_address": floating_ip["ip"],
+            "id": floating_ip["id"],
+            "port_id": None,
+            "tenant_id": context.project_id,
+            "scope": scope,
+            "status": "IN USE" if fixed_ip else "RESERVED",
+        }
+
+        instance_id = floating_ip.get("instance_id")
+        if instance_id is not None:
+            instance = client.servers.get(instance_id)
+            floating_ip["instance_name"] = instance.name
+            floating_ip["instance_zone"] = getattr(
+                instance, "OS-EXT-AZ:availability_zone")
+
+        return self._prepare_item(floating_ip, db_item)
+
+    def _add_nonnamed_items(self, context, items):
+        for item in items:
+            item["name"] = ("address-" +
+                            item["floating_ip_address"].replace(".", "-"))
+            item["creationTimestamp"] = ""
+            self._add_db_item(context, item)

gceapi/api/addresses.py

@@ -0,0 +1,49 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gceapi.api import address_api
+from gceapi.api import common as gce_common
+from gceapi.api import scopes
+from gceapi.api import wsgi as gce_wsgi
+
+
+class Controller(gce_common.Controller):
+    """GCE Address controller"""
+
+    def __init__(self, *args, **kwargs):
+        super(Controller, self).__init__(address_api.API(),
+                                         *args, **kwargs)
+
+    def format_item(self, request, floating_ip, scope):
+        result_dict = {
+            "creationTimestamp": floating_ip.get("creationTimestamp", ""),
+            "status": floating_ip["status"],
+            "name": floating_ip["name"],
+            "address": floating_ip["floating_ip_address"],
+        }
+        if "description" in floating_ip:
+            result_dict["description"] = floating_ip["description"]
+        else:
+            result_dict["description"] = ""
+
+        if "instance_name" in floating_ip:
+            result_dict["users"] = [self._qualify(
+                request, "instances", floating_ip["instance_name"],
+                scopes.ZoneScope(floating_ip["instance_zone"]))]
+
+        return self._format_item(request, result_dict, scope)
+
+
+def create_resource():
+    return gce_wsgi.GCEResource(Controller())

gceapi/api/base_api.py

@@ -0,0 +1,213 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Base classes of GCE API conversion layer.
+
+Classes in this layer aggregate functionality of OpenStack necessary
+and sufficient to handle supported GCE API requests
+"""
+
+from oslo.config import cfg
+
+from gceapi import db
+from gceapi import exception
+from gceapi.openstack.common import timeutils
+
+FLAGS = cfg.CONF
+
+
+class Singleton(type):
+    """Singleton metaclass.
+
+    KIND must be overriden in classes based on this type.
+    """
+    _instances = {}
+    KIND = ""
+
+    def __call__(self, *args, **kwargs):
+        if not self.KIND:
+            raise NotImplementedError
+        if self.KIND not in self._instances:
+            singleton = super(Singleton, self).__call__(*args, **kwargs)
+            self._instances[self.KIND] = singleton
+        return self._instances[self.KIND]
+
+    @classmethod
+    def get_instance(cls, kind):
+        """Get singleton by name."""
+
+        return cls._instances.get(kind)
+
+
+class NetSingleton(Singleton):
+    """Proxy loader for net depended API.
+
+    NEUTRON_API_MODULE and NOVA_API_MODULE must be overriden in classes
+    based on this type.
+    """
+
+    NEUTRON_API_MODULE = None
+    NOVA_API_MODULE = None
+
+    def __call__(self):
+        net_api = FLAGS.get("network_api")
+        # NOTE(Alex): Initializing proper network singleton
+        if net_api is None or ("quantum" in net_api
+                               or "neutron" in net_api):
+            return self.NEUTRON_API_MODULE.API()
+        else:
+            return self.NOVA_API_MODULE.API()
+
+
+class API(object):
+    """Base GCE API abstraction class
+
+    Inherited classes should implement one class of GCE API functionality.
+    There should be enough public methods implemented to cover necessary
+    methods of GCE API in the class. Other public methods can exist to be
+    invoked from other APIs of this layer.
+    Class in this layer should use each others functionality instead of
+    calling corresponding low-level routines.
+    Basic methods should be named including "item(s)" instead of specific
+    functional names.
+
+    Descendants are stateless singletons.
+    Supports callbacks for interaction of APIs in this layer
+    """
+    # TODO(Alex): Now action methods get body of parameters straight from GCE
+    # request while returning results in terms of Openstack to be converted
+    # to GCE terms in controller. In next version this layer should be revised
+    # to work symmetrically with incoming and outgoing data.
+
+    __metaclass__ = Singleton
+
+    def __init__(self, *args, **kwargs):
+        super(API, self).__init__(*args, **kwargs)
+        self._callbacks = []
+
+    def _get_type(self):
+        """GCE API object type method. Should be overriden."""
+
+        raise NotImplementedError
+
+    def _get_persistent_attributes(self):
+        """Iterable of name of columns stored in GCE API database.
+
+        Should be overriden.
+        """
+
+        raise NotImplementedError
+
+    def get_item(self, context, name, scope=None):
+        """Returns fully filled item for particular inherited API."""
+
+        raise exception.NotFound
+
+    def get_items(self, context, scope=None):
+        """Returns list of items."""
+
+        return []
+
+    def delete_item(self, context, name, scope=None):
+        """Deletes an item."""
+
+        raise exception.NotFound
+
+    def add_item(self, context, name, body, scope=None):
+        """Creates an item. It returns created item."""
+
+        raise exception.NotFound
+
+    def get_scopes(self, context, item):
+        """Returns which zones/regions the item belongs too."""
+
+        return []
+
+    def _process_callbacks(self, context, reason, item, **kwargs):
+        for cb_reason, cb_func in self._callbacks:
+            if cb_reason == reason:
+                cb_func(context, item, **kwargs)
+
+    def _register_callback(self, reason, func):
+        """Callbacks registration
+
+        Callbacks can be registered by one API to be called by another before
+        some action for checking possibility of the action or to process
+        pre-actions
+        """
+
+        self._callbacks.append((reason, func))
+
+    def _prepare_item(self, item, db_item):
+        if db_item is not None:
+            item.update(db_item)
+        return item
+
+    def _add_db_item(self, context, item):
+        db_item = dict((key, item.get(key))
+                   for key in self._get_persistent_attributes()
+                   if key in item)
+        if ("creationTimestamp" in self._get_persistent_attributes() and
+                "creationTimestamp" not in db_item):
+            # TODO(ft): Google not returns microseconds but returns
+            # server time zone: 2013-12-06T03:34:31.340-08:00
+            utcnow = timeutils.isotime(None, True)
+            db_item["creationTimestamp"] = utcnow
+            item["creationTimestamp"] = utcnow
+        db.add_item(context, self._get_type(), db_item)
+        return item
+
+    def _delete_db_item(self, context, item):
+        return db.delete_item(context, self._get_type(), item["id"])
+
+    def _update_db_item(self, context, item):
+        db_item = dict((key, item.get(key))
+                   for key in self._get_persistent_attributes()
+                   if key in item)
+        db.update_item(context, self._get_type(), db_item)
+
+    def _get_db_items(self, context):
+        return db.get_items(context, self._get_type())
+
+    def _get_db_items_dict(self, context):
+        return dict((item["id"], item) for item in self._get_db_items(context))
+
+    def _get_db_item_by_id(self, context, item_id):
+        return db.get_item_by_id(context, self._get_type(), item_id)
+
+    def _get_db_item_by_name(self, context, name):
+        return db.get_item_by_name(context, self._get_type(), name)
+
+    def _purge_db(self, context, os_items, db_items_dict):
+        only_os_items = []
+        existed_db_items = set()
+        for item in os_items:
+            db_item = db_items_dict.get(str(item["id"]))
+            if db_item is None:
+                only_os_items.append(item)
+            else:
+                existed_db_items.add(db_item["id"])
+        for item in db_items_dict.itervalues():
+            if item["id"] not in existed_db_items:
+                self._delete_db_item(context, item)
+        return only_os_items
+
+
+class _CallbackReasons(object):
+    check_delete = 1
+    pre_delete = 2
+    post_add = 3
+
+
+_callback_reasons = _CallbackReasons()

gceapi/api/clients.py

@@ -0,0 +1,141 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from keystoneclient.v2_0 import client as kc
+from novaclient import client as novaclient
+from novaclient import shell as novashell
+from oslo.config import cfg
+
+from gceapi.openstack.common.gettextutils import _
+from gceapi.openstack.common import log as logging
+
+logger = logging.getLogger(__name__)
+
+CONF = cfg.CONF
+
+
+try:
+    from neutronclient.v2_0 import client as neutronclient
+except ImportError:
+    neutronclient = None
+    logger.info(_('neutronclient not available'))
+try:
+    from cinderclient import client as cinderclient
+except ImportError:
+    cinderclient = None
+    logger.info(_('cinderclient not available'))
+try:
+    from glanceclient import client as glanceclient
+except ImportError:
+    glanceclient = None
+    logger.info(_('glanceclient not available'))
+
+
+def nova(context, service_type='compute'):
+    computeshell = novashell.OpenStackComputeShell()
+    extensions = computeshell._discover_extensions("1.1")
+
+    args = {
+        'project_id': context.project_id,
+        'auth_url': CONF.keystone_gce_url,
+        'service_type': service_type,
+        'username': None,
+        'api_key': None,
+        'extensions': extensions,
+    }
+
+    client = novaclient.Client(1.1, **args)
+
+    management_url = _url_for(context, service_type=service_type)
+    client.client.auth_token = context.auth_token
+    client.client.management_url = management_url
+
+    return client
+
+
+def neutron(context):
+    if neutronclient is None:
+        return None
+
+    args = {
+        'auth_url': CONF.keystone_gce_url,
+        'service_type': 'network',
+        'token': context.auth_token,
+        'endpoint_url': _url_for(context, service_type='network'),
+    }
+
+    return neutronclient.Client(**args)
+
+
+def glance(context):
+    if glanceclient is None:
+        return None
+
+    args = {
+        'auth_url': CONF.keystone_gce_url,
+        'service_type': 'image',
+        'token': context.auth_token,
+    }
+
+    return glanceclient.Client(
+        "1", endpoint=_url_for(context, service_type='image'), **args)
+
+
+def cinder(context):
+    if cinderclient is None:
+        return nova(context, 'volume')
+
+    args = {
+        'service_type': 'volume',
+        'auth_url': CONF.keystone_gce_url,
+        'username': None,
+        'api_key': None,
+    }
+
+    _cinder = cinderclient.Client('1', **args)
+    management_url = _url_for(context, service_type='volume')
+    _cinder.client.auth_token = context.auth_token
+    _cinder.client.management_url = management_url
+
+    return _cinder
+
+
+def keystone(context):
+    _keystone = kc.Client(
+        token=context.auth_token,
+        tenant_id=context.project_id,
+        auth_url=CONF.keystone_gce_url)
+
+    return _keystone
+
+
+def _url_for(context, **kwargs):
+    service_catalog = context.service_catalog
+    if not service_catalog:
+        catalog = keystone(context).service_catalog.catalog
+        service_catalog = catalog["serviceCatalog"]
+        context.service_catalog = service_catalog
+
+    service_type = kwargs["service_type"]
+    for service in service_catalog:
+        if service["type"] != service_type:
+            continue
+        for endpoint in service["endpoints"]:
+            if "publicURL" in endpoint:
+                return endpoint["publicURL"]
+        else:
+            return None
+
+    return None

gceapi/api/common.py

@@ -0,0 +1,389 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Base GCE API controller"""
+
+import os.path
+import re
+from webob import exc
+
+from gceapi.api import operation_api
+from gceapi.api import operation_util
+from gceapi.api import scopes
+from gceapi.api import utils
+from gceapi import exception
+from gceapi.openstack.common.gettextutils import _
+from gceapi.openstack.common import timeutils
+
+
+class Controller(object):
+    """Base controller
+
+    Implements base CRUD methods.
+    Individual GCE controllers should inherit this and:
+    - implement format_item() method,
+    - override _get_type() method,
+    - add necessary specific request handlers,
+    - use _api to hold instance of related GCE API (see base_api.py).
+    """
+
+    _api = None
+
+    # Initialization
+    def __init__(self, api):
+        """Base initialization.
+
+        Inherited classes should init _api and call super().
+        """
+
+        self._api = api
+        self._type_name = self._api._get_type()
+        self._collection_name = utils.get_collection_name(self._type_name)
+        self._type_kind = utils.get_type_kind(self._type_name)
+        self._list_kind = utils.get_list_kind(self._type_name)
+        self._aggregated_kind = utils.get_aggregated_kind(self._type_name)
+        self._operation_api = operation_api.API()
+
+    def process_result(self, request, action, action_result):
+        context = self._get_context(request)
+        operation = operation_util.save_operaton(context, action_result)
+        if operation is not None:
+            scope = self._operation_api.get_scopes(context, operation)[0]
+            action_result = self._format_operation(request, operation, scope)
+
+        if isinstance(action_result, Exception):
+            return self._format_error(action_result)
+        if action_result is None:
+            return None, 204
+        return self._format_output(request, action, action_result), 200
+
+    # Base methods, should be overriden
+
+    def format_item(self, request, image, scope):
+        """Main item resource conversion routine
+
+        Overriden in inherited classes should implement conversion of
+        OpenStack resource into GCE resource.
+        """
+
+        raise exc.HTTPNotImplemented
+
+    # Actions
+    def index(self, req, scope_id=None):
+        """GCE list requests, global or with zone/region specified."""
+
+        context = self._get_context(req)
+        scope = self._get_scope(req, scope_id)
+
+        items = self._api.get_items(context, scope)
+        items = [{
+            "scope": scope,
+            "item": self.format_item(req, i, scope)
+        } for i in items]
+        items = self._filter_items(req, items)
+        items, next_page_token = self._page_items(req, items)
+        items = [i["item"] for i in items]
+
+        return self._format_list(req, items, next_page_token, scope)
+
+    def show(self, req, id=None, scope_id=None):
+        """GCE get requests, global or zone/region specified."""
+
+        context = self._get_context(req)
+        scope = self._get_scope(req, scope_id)
+        try:
+            item = self._api.get_item(context, id, scope)
+            return self.format_item(req, item, scope)
+        except (exception.NotFound, KeyError, IndexError):
+            msg = _("Resource '%s' could not be found") % id
+            raise exc.HTTPNotFound(explanation=msg)
+
+    def aggregated_list(self, req):
+        """GCE aggregated list requests for all zones/regions."""
+
+        context = self._get_context(req)
+        items = list()
+        for item in self._api.get_items(context, None):
+            for scope in self._api.get_scopes(context, item):
+                items.append({
+                    "scope": scope,
+                    "item": self.format_item(req, item, scope)
+                })
+        items = self._filter_items(req, items)
+        items, next_page_token = self._page_items(req, items)
+
+        items_by_scopes = {}
+        for item in items:
+            scope_path = item["scope"].get_path()
+            items_by_scope = items_by_scopes.setdefault(scope_path,
+                {self._collection_name: []})[self._collection_name]
+            items_by_scope.append(item["item"])
+
+        return self._format_list(req, items_by_scopes, next_page_token,
+            scopes.AggregatedScope())
+
+    def delete(self, req, id, scope_id=None):
+        """GCE delete requests."""
+
+        scope = self._get_scope(req, scope_id)
+        context = self._get_context(req)
+        operation_util.init_operation(context, "delete",
+                                      self._type_name, id, scope)
+        try:
+            self._api.delete_item(context, id, scope)
+        except (exception.NotFound, KeyError, IndexError):
+            msg = _("Resource '%s' could not be found") % id
+            raise exc.HTTPNotFound(explanation=msg)
+
+    def create(self, req, body, scope_id=None):
+        """GCE add requests."""
+
+        scope = self._get_scope(req, scope_id)
+        context = self._get_context(req)
+        operation_util.init_operation(context, "insert",
+                                      self._type_name, body["name"], scope)
+        self._api.add_item(context, body['name'], body, scope)
+
+    # Filtering
+    def _filter_items(self, req, items):
+        """Filtering result list
+
+        Only one filter is supported(eg. by one field)
+        Only two comparison strings are supported: 'eq' and 'ne'
+        There are no logical expressions with fields
+        """
+        if not items:
+            return items
+        if "filter" not in req.params:
+            return items
+
+        filter_def = req.params["filter"].split()
+        if len(filter_def) != 3:
+            # TODO(apavlov): raise exception
+            return items
+        if filter_def[1] != "eq" and filter_def[1] != "ne":
+            # TODO(apavlov): raise exception
+            return items
+        if filter_def[0] not in items[0]["item"]:
+            # TODO(apavlov): raise exception
+            return items
+
+        filter_field = filter_def[0]
+        filter_cmp = filter_def[1] == "eq"
+        filter_pattern = filter_def[2]
+        if filter_pattern[0] == "'" and filter_pattern[-1] == "'":
+            filter_pattern = filter_pattern[1:-1]
+
+        result_list = list()
+        for item in items:
+            field = item["item"][filter_field]
+            result = re.match(filter_pattern, field)
+            if filter_cmp != (result is None):
+                result_list.append(item)
+
+        return result_list
+
+    # Paging
+    def _page_items(self, req, items):
+        if not items:
+            return items, None
+        if "maxResults" not in req.params:
+            return items, None
+
+        limit = int(req.params["maxResults"])
+        if limit >= len(items):
+            return items, None
+
+        page_index = int(req.params.get("pageToken", 0))
+        if page_index < 0 or page_index * limit > len(items):
+            # TODO(apavlov): raise exception
+            return [], None
+
+        items.sort(None, lambda x: x["item"].get("name"))
+        start = limit * page_index
+        if start + limit >= len(items):
+            return items[start:], None
+
+        return items[start:start + limit], str(page_index + 1)
+
+    # Utility
+    def _get_context(self, req):
+        return req.environ['gceapi.context']
+
+    def _get_scope(self, req, scope_id):
+        scope = scopes.construct_from_path(req.path_info, scope_id)
+        if scope is None:
+            return None
+        scope_api = scope.get_scope_api()
+        if scope_api is not None:
+            try:
+                context = self._get_context(req)
+                scope_api.get_item(context, scope.get_name(), None)
+            except ValueError as ex:
+                raise exc.HTTPNotFound(detail=ex)
+
+        return scope
+
+    # Result formatting
+    def _format_date(self, date_string):
+        """Returns standard format for given date."""
+        if date_string is None:
+            return None
+        if isinstance(date_string, basestring):
+            date_string = timeutils.parse_isotime(date_string)
+        return date_string.strftime('%Y-%m-%dT%H:%M:%SZ')
+
+    def _get_id(self, link):
+        hashed_link = hash(link)
+        if hashed_link < 0:
+            hashed_link = -hashed_link
+        return str(hashed_link)
+
+    def _qualify(self, request, controller, identifier, scope):
+        """Creates fully qualified selfLink for an item or collection
+
+        Specific formatting for projects and zones/regions,
+        'global' prefix For global resources,
+        'zones/zone_id' prefix for zone(similar for regions) resources.
+        """
+
+        result = os.path.join(
+            request.application_url, self._get_context(request).project_name)
+        if controller:
+            if scope:
+                result = os.path.join(result, scope.get_path())
+            result = os.path.join(result, controller)
+            if identifier:
+                result = os.path.join(result, identifier)
+        return result
+
+    def _format_item(self, request, result_dict, scope):
+        return self._add_item_header(request, result_dict, scope,
+                                     self._type_kind, self._collection_name)
+
+    def _format_operation(self, request, operation, scope):
+        result_dict = {
+            "name": operation["name"],
+            "operationType": operation["type"],
+            "insertTime": operation["insert_time"],
+            "startTime": operation["start_time"],
+            "status": operation["status"],
+            "progress": operation["progress"],
+            "user": operation["user"],
+        }
+        result_dict["targetLink"] = self._qualify(
+                request, utils.get_collection_name(operation["target_type"]),
+                operation["target_name"], scope)
+        result_dict["targetId"] = self._get_id(result_dict["targetLink"])
+        if "end_time" in operation:
+            result_dict["endTime"] = operation["end_time"]
+        if "error_code" in operation:
+            result_dict.update({
+                "httpErrorStatusCode": operation["error_code"],
+                "httpErrorMessage": operation["error_message"],
+                "error": {"errors": operation["errors"]},
+            })
+        type_name = self._operation_api._get_type()
+        return self._add_item_header(request, result_dict, scope,
+                                     utils.get_type_kind(type_name),
+                                     utils.get_collection_name(type_name))
+
+    def _add_item_header(self, request, result_dict, scope,
+                         _type_kind, _collection_name):
+        if scope is not None and scope.get_name() is not None:
+            result_dict[scope.get_type()] = self._qualify(
+                    request, scope.get_collection(), scope.get_name(), None)
+        result_dict["kind"] = _type_kind
+        result_dict["selfLink"] = self._qualify(
+                request, _collection_name, result_dict.get("name"), scope)
+        result_dict["id"] = self._get_id(result_dict["selfLink"])
+        return result_dict
+
+    def _format_list(self, request, result_list, next_page_token, scope):
+        result_dict = {}
+        result_dict["items"] = result_list
+        if next_page_token:
+            result_dict["nextPageToken"] = next_page_token
+        result_dict["kind"] = (self._aggregated_kind
+            if scope and isinstance(scope, scopes.AggregatedScope)
+            else self._list_kind)
+
+        context = self._get_context(request)
+        list_id = os.path.join("projects", context.project_name)
+        if scope:
+            list_id = os.path.join(list_id, scope.get_path())
+        list_id = os.path.join(list_id, self._collection_name)
+        result_dict["id"] = list_id
+
+        result_dict["selfLink"] = self._qualify(
+                request, self._collection_name, None, scope)
+        return result_dict
+
+    def _format_error(self, ex_value):
+        if isinstance(ex_value, exception.NotAuthorized):
+            msg = _('Unauthorized')
+            code = 401
+        elif isinstance(ex_value, exc.HTTPException):
+            msg = ex_value.explanation
+            code = ex_value.code
+        elif isinstance(ex_value, exception.GceapiException):
+            msg = ex_value.args[0]
+            code = ex_value.code
+        else:
+            msg = _('Internal server error')
+            code = 500
+
+        return {
+            'error': {'errors': [{'message': msg}]},
+            'code': code,
+            'message': msg
+            }, code
+
+    def _format_output(self, request, action, action_result):
+        # TODO(ft): this metod must be safe and ignore unknown fields
+        fields = request.params.get('fields', None)
+        # TODO(ft): GCE can also format results of other action
+        if action not in ('index', 'show') or fields is None:
+            return action_result
+
+        if action == 'show':
+            action_result = utils.apply_template(fields, action_result)
+            return action_result
+        sp = utils.split_by_comma(fields)
+        top_level = []
+        items = []
+        for string in sp:
+            if 'items' in string:
+                items.append(string)
+            else:
+                top_level.append(string)
+        res = {}
+        if len(items) > 0:
+            res['items'] = []
+        for string in top_level:
+            dct = utils.apply_template(string, action_result)
+            for key, val in dct.items():
+                res[key] = val
+        for string in items:
+            if '(' in string:
+                dct = utils.apply_template(string, action_result)
+                for key, val in dct.items():
+                    res[key] = val
+            elif string.startswith('items/'):
+                string = string[len('items/'):]
+                for element in action_result['items']:
+                    dct = utils.apply_template(string, element)
+                    res['items'].append(dct)
+
+        return res

gceapi/api/discovery.py

@@ -0,0 +1,63 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import inspect
+import os
+import threading
+import webob
+
+from gceapi.openstack.common import log as logging
+from gceapi import wsgi_ext as openstack_wsgi
+
+LOG = logging.getLogger(__name__)
+
+
+class Controller(object):
+
+    _lock = threading.RLock()
+    _files = {}
+
+    def discovery(self, req, version):
+        """Returns appropriate json by its version."""
+
+        if version in self._files:
+            return self._files[version]
+
+        self._lock.acquire()
+        try:
+            if version in self._files:
+                return self._files[version]
+
+            jfile = self._load_file(version)
+            jfile = jfile.replace("{HOST_URL}", req.host_url)
+            self._files[version] = jfile
+            return jfile
+        finally:
+            self._lock.release()
+
+    def _load_file(self, version):
+        current_file = os.path.abspath(inspect.getsourcefile(lambda _: None))
+        current_dir = os.path.dirname(current_file)
+        file_name = os.path.join(current_dir, "compute", version + ".json")
+        try:
+            f = open(file_name)
+        except Exception as ex:
+            raise webob.exc.HTTPNotFound(ex)
+        result = f.read()
+        f.close()
+        return result
+
+
+def create_resource():
+    return openstack_wsgi.Resource(Controller())

gceapi/api/disk_api.py

@@ -0,0 +1,161 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gceapi.api import base_api
+from gceapi.api import clients
+from gceapi.api import image_api
+from gceapi.api import operation_api
+from gceapi.api import operation_util
+from gceapi.api import scopes
+from gceapi.api import utils
+from gceapi import exception
+
+
+GB = 1024 ** 3
+
+
+class API(base_api.API):
+    """GCE Disk API."""
+
+    KIND = "disk"
+    _status_map = {
+            "creating": "CREATING",
+            "downloading": "CREATING",
+            "available": "READY",
+            "attaching": "READY",
+            "in-use": "READY",
+            # "deleting": "",
+            "error": "FAILED",
+            # "error_deleting": "",
+            "backing-up": "READY",
+            "restoring-backup": "READY",
+            # "error_restoring": ""
+    }
+
+    def __init__(self, *args, **kwargs):
+        super(API, self).__init__(*args, **kwargs)
+        operation_api.API().register_get_progress_method(
+                "disk-add",
+                self._get_add_item_progress)
+        operation_api.API().register_get_progress_method(
+                "disk-delete",
+                self._get_delete_item_progress)
+
+    def _get_type(self):
+        return self.KIND
+
+    def get_item(self, context, name, scope=None):
+        client = clients.cinder(context)
+        volumes = client.volumes.list(search_opts={"display_name": name})
+        volumes = self._filter_volumes_by_zone(volumes, scope)
+        volumes = [utils.to_dict(item) for item in volumes]
+        if not volumes or len(volumes) != 1:
+            raise exception.NotFound
+        return self._prepare_item(client, volumes[0])
+
+    def get_items(self, context, scope=None):
+        client = clients.cinder(context)
+        volumes = client.volumes.list()
+        volumes = self._filter_volumes_by_zone(volumes, scope)
+        volumes = [utils.to_dict(item) for item in volumes]
+        for volume in volumes:
+            self._prepare_item(client, volume)
+        return volumes
+
+    def get_scopes(self, context, item):
+        return [scopes.ZoneScope(item["availability_zone"])]
+
+    def _prepare_item(self, client, item):
+        snapshot = None
+        snapshot_id = item["snapshot_id"]
+        if snapshot_id:
+            snapshot = utils.to_dict(client.volume_snapshots.get(snapshot_id))
+        item["snapshot"] = snapshot
+        item["status"] = self._status_map.get(item["status"], item["status"])
+        item["name"] = item["display_name"]
+        image = item.get("volume_image_metadata")
+        if image:
+            item["image_name"] = image["image_name"]
+        return item
+
+    def _filter_volumes_by_zone(self, volumes, scope):
+        if scope is None:
+            return volumes
+        return filter(
+            lambda volume: volume.availability_zone == scope.get_name(),
+            volumes)
+
+    def delete_item(self, context, name, scope=None):
+        client = clients.cinder(context).volumes
+        volumes = client.list(search_opts={"display_name": name})
+        if not volumes or len(volumes) != 1:
+            raise exception.NotFound
+        operation_util.start_operation(context,
+                                       self._get_delete_item_progress,
+                                       volumes[0].id)
+        client.delete(volumes[0])
+
+    def add_item(self, context, name, body, scope=None):
+        sizeGb = int(body['sizeGb']) if 'sizeGb' in body else None
+
+        snapshot_uri = body.get("sourceSnapshot")
+        image_uri = body.get("sourceImage")
+        snapshot_id = None
+        image_id = None
+
+        client = clients.cinder(context)
+        if snapshot_uri:
+            snapshot_name = utils._extract_name_from_url(snapshot_uri)
+            snapshots = client.volume_snapshots.list(
+                search_opts={"display_name": snapshot_name})
+            if not snapshots or len(snapshots) != 1:
+                raise exception.NotFound
+            snapshot_id = snapshots[0].id
+        elif image_uri:
+            image_name = utils._extract_name_from_url(image_uri)
+            image = image_api.API().get_item(context, image_name, scope)
+            image_id = image['id']
+            # Cinder API doesn't get size from image, so we do this
+            image_size_in_gb = (int(image['size']) + GB - 1) / GB
+            if not sizeGb or sizeGb < image_size_in_gb:
+                sizeGb = image_size_in_gb
+
+        operation_util.start_operation(context, self._get_add_item_progress)
+        volume = client.volumes.create(
+            sizeGb, snapshot_id=snapshot_id,
+            display_name=body.get('name'),
+            display_description=body.get('description'),
+            imageRef=image_id,
+            availability_zone=scope.get_name())
+        operation_util.set_item_id(context, volume.id)
+
+        return self._prepare_item(client, utils.to_dict(volume))
+
+    def _get_add_item_progress(self, context, volume_id):
+        client = clients.cinder(context)
+        try:
+            volume = client.volumes.get(volume_id)
+        except clients.cinderclient.exceptions.NotFound:
+            return operation_api.gef_final_progress()
+        if (volume.status not in ["creating", "downloading"]):
+            return operation_api.gef_final_progress(volume.status == "error")
+
+    def _get_delete_item_progress(self, context, volume_id):
+        client = clients.cinder(context)
+        try:
+            volume = client.volumes.get(volume_id)
+        except clients.cinderclient.exceptions.NotFound:
+            return operation_api.gef_final_progress()
+        if volume.status not in ["deleting", "deleted"]:
+            return operation_api.gef_final_progress(True)

gceapi/api/disks.py

@@ -0,0 +1,69 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gceapi.api import common as gce_common
+from gceapi.api import disk_api
+from gceapi.api import operation_util
+from gceapi.api import scopes
+from gceapi.api import snapshot_api
+from gceapi.api import wsgi as gce_wsgi
+
+
+class Controller(gce_common.Controller):
+    """GCE Disk controller"""
+
+    def __init__(self, *args, **kwargs):
+        super(Controller, self).__init__(disk_api.API(),
+                                         *args, **kwargs)
+
+    def format_item(self, request, volume, scope):
+        result_dict = {
+                "creationTimestamp": self._format_date(volume["created_at"]),
+                "status": volume["status"],
+                "name": volume["display_name"],
+                "description": volume["display_description"],
+                "sizeGb": volume["size"],
+                }
+        snapshot = volume["snapshot"]
+        if snapshot:
+            result_dict["sourceSnapshot"] = self._qualify(request,
+                "snapshots", snapshot["display_name"],
+                scopes.GlobalScope())
+            result_dict["sourceSnapshotId"] = snapshot["id"]
+        image_name = volume.get("image_name")
+        if image_name:
+            result_dict["sourceImage"] = self._qualify(request,
+                "images", image_name, scopes.GlobalScope())
+            result_dict["sourceImageId"] = self._get_id(
+                result_dict["sourceImage"])
+
+        return self._format_item(request, result_dict, scope)
+
+    def create(self, req, body, scope_id):
+        source_image = req.params.get("sourceImage")
+        if source_image is not None:
+            body["sourceImage"] = source_image
+        return super(Controller, self).create(req, body, scope_id)
+
+    def create_snapshot(self, req, body, scope_id, id):
+        body["disk_name"] = id
+        scope = self._get_scope(req, scope_id)
+        context = self._get_context(req)
+        operation_util.init_operation(context, "createSnapshot",
+                                      self._type_name, id, scope)
+        snapshot_api.API().add_item(context, body, scope)
+
+
+def create_resource():
+    return gce_wsgi.GCEResource(Controller())

gceapi/api/firewall_api.py

@@ -0,0 +1,256 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import copy
+
+from gceapi.api import base_api
+from gceapi.api import clients
+from gceapi.api import network_api
+from gceapi.api import operation_util
+from gceapi.api import utils
+from gceapi import exception
+from gceapi.openstack.common.gettextutils import _
+from gceapi.openstack.common import log as logging
+
+
+PROTOCOL_MAP = {
+    '1': 'icmp',
+    '6': 'tcp',
+    '17': 'udp',
+}
+LOG = logging.getLogger(__name__)
+
+
+class API(base_api.API):
+    """GCE Firewall API."""
+
+    KIND = "firewall"
+    PERSISTENT_ATTRIBUTES = ["id", "creationTimestamp", "network_name"]
+
+    def __init__(self, *args, **kwargs):
+        super(API, self).__init__(*args, **kwargs)
+        network_api.API()._register_callback(
+            base_api._callback_reasons.pre_delete,
+            self.delete_network_firewalls)
+
+    def _get_type(self):
+        return self.KIND
+
+    def _get_persistent_attributes(self):
+        return self.PERSISTENT_ATTRIBUTES
+
+    def get_item(self, context, name, scope=None):
+        client = clients.nova(context)
+        try:
+            firewall = client.security_groups.find(name=name)
+        except (clients.novaclient.exceptions.NotFound,
+                clients.novaclient.exceptions.NoUniqueMatch):
+            raise exception.NotFound()
+        firewall = self._prepare_firewall(utils.to_dict(firewall))
+        db_firewall = self._get_db_item_by_id(context, firewall["id"])
+        self._prepare_item(firewall, db_firewall)
+        return firewall
+
+    def get_items(self, context, scope=None):
+        client = clients.nova(context)
+        firewalls = client.security_groups.list()
+        items = list()
+        gce_firewalls = self._get_db_items_dict(context)
+        for firewall in firewalls:
+            item = self._prepare_firewall(utils.to_dict(firewall))
+            self._prepare_item(item, gce_firewalls.get(item["id"]))
+            items.append(item)
+        self._purge_db(context, items, gce_firewalls)
+        return items
+
+    def add_item(self, context, name, body, scope=None):
+        network = self._get_network_by_url(context, body['network'])
+        self._check_rules(body)
+        group_description = body.get("description", "")
+        client = clients.nova(context)
+        operation_util.start_operation(context)
+        sg = client.security_groups.create(body['name'], group_description)
+        try:
+            rules = self._convert_to_secgroup_rules(body)
+            for rule in rules:
+                client.security_group_rules.create(
+                    sg.id, ip_protocol=rule["protocol"],
+                    from_port=rule["from_port"], to_port=rule["to_port"],
+                    cidr=rule["cidr"], )
+        except Exception:
+            client.security_groups.delete(sg)
+            raise
+        new_firewall = utils.to_dict(client.security_groups.get(sg.id))
+        new_firewall = self._prepare_firewall(new_firewall)
+        new_firewall["creationTimestamp"] = 1
+        new_firewall["network_name"] = network["name"]
+        new_firewall = self._add_db_item(context, new_firewall)
+        self._process_callbacks(
+            context, base_api._callback_reasons.post_add, new_firewall)
+        return new_firewall
+
+    def delete_item(self, context, name, scope=None):
+        firewall = self.get_item(context, name)
+        operation_util.start_operation(context)
+        self._process_callbacks(
+            context, base_api._callback_reasons.pre_delete, firewall)
+        client = clients.nova(context)
+        try:
+            client.security_groups.delete(firewall["id"])
+            self._delete_db_item(context, firewall)
+        except clients.novaclient.exceptions.ClientException as ex:
+            raise exception.GceapiException(message=ex.message, code=ex.code)
+
+    def _prepare_firewall(self, firewall):
+        # NOTE(ft): OpenStack security groups are more powerful than
+        # gce firewalls so when we cannot completely convert secgroup
+        # we add prefixes to firewall description
+        # [*] - cidr rules too complex to convert
+        # [+] - non-cidr rules presents
+
+        non_cidr_rule_exists = False
+        too_complex_for_gce = False
+
+        # NOTE(ft): group OpenStack rules by cidr and proto
+        # cidr group must be comparable object
+        def _ports_to_str(rule):
+            if rule['from_port'] == rule['to_port']:
+                return str(rule['from_port'])
+            else:
+                return "%s-%s" % (rule['from_port'], rule['to_port'])
+
+        grouped_rules = {}
+        for rule in firewall["rules"]:
+            if "cidr" not in rule["ip_range"] or not rule["ip_range"]["cidr"]:
+                non_cidr_rule_exists = True
+                continue
+            cidr = rule.get("ip_range", {}).get("cidr")
+            proto = rule["ip_protocol"]
+            cidr_group = grouped_rules.setdefault(cidr, {})
+            proto_ports = cidr_group.setdefault(proto, set())
+            proto_ports.add(_ports_to_str(rule))
+
+        # NOTE(ft): compare cidr grups to understand
+        # whether OpenStack rules are too complex or not
+        common_rules = None
+        for cidr in grouped_rules:
+            if common_rules:
+                if common_rules != grouped_rules[cidr]:
+                    too_complex_for_gce = True
+                    break
+            else:
+                common_rules = grouped_rules[cidr]
+
+        # NOTE(ft): check icmp rules:
+        # if per icmp type rule present then rules are too complex
+        if not too_complex_for_gce and common_rules and "icmp" in common_rules:
+            icmp_rules = common_rules["icmp"]
+            if len(icmp_rules) == 1:
+                icmp_rule = icmp_rules.pop()
+                if icmp_rule != "-1":
+                    too_complex_for_gce = True
+            else:
+                too_complex_for_gce = True
+
+        # NOTE(ft): build gce rules if possible
+        def _build_gce_port_rule(proto, rules):
+            gce_rule = {"IPProtocol": proto}
+            if proto != "icmp":
+                gce_rule["ports"] = rules
+            return gce_rule
+
+        sourceRanges = []
+        allowed = []
+        if not too_complex_for_gce:
+            sourceRanges = [cidr for cidr in grouped_rules] or ["0.0.0.0/0"]
+            if common_rules:
+                allowed = [_build_gce_port_rule(proto, common_rules[proto])
+                           for proto in common_rules]
+        firewall["sourceRanges"] = sourceRanges
+        firewall["allowed"] = allowed
+
+        # NOTE(ft): add prefixes to description
+        description = firewall.get("description")
+        prefixes = []
+        if too_complex_for_gce:
+            prefixes.append("[*]")
+        if non_cidr_rule_exists:
+            prefixes.append("[+]")
+        if prefixes:
+            if description is not None:
+                prefixes.append(description)
+            description = "".join(prefixes)
+            firewall["description"] = description
+
+        return firewall
+
+    def _get_network_by_url(self, context, url):
+        # NOTE(apavlov): Check existence of such network
+        network_name = utils._extract_name_from_url(url)
+        return network_api.API().get_item(context, network_name)
+
+    def _check_rules(self, firewall):
+        if not firewall.get('sourceRanges') or firewall.get('sourceTags'):
+            msg = _("Not 'sourceRange' neither 'sourceTags' is provided")
+            raise exception.InvalidRequest(msg)
+        for allowed in firewall.get('allowed', []):
+            proto = allowed.get('IPProtocol')
+            proto = PROTOCOL_MAP.get(proto, proto)
+            if not proto or proto not in PROTOCOL_MAP.values():
+                msg = _("Invlaid protocol")
+                raise exception.InvalidRequest(msg)
+            if proto == 'icmp' and allowed.get('ports'):
+                msg = _("Invalid options for icmp protocol")
+                raise exception.InvalidRequest(msg)
+
+    def _convert_to_secgroup_rules(self, firewall):
+        rules = []
+        for source_range in firewall['sourceRanges']:
+            for allowed in firewall.get('allowed', []):
+                proto = allowed['IPProtocol']
+                proto = PROTOCOL_MAP.get(proto, proto)
+                rule = {
+                    "protocol": proto,
+                    "cidr": source_range,
+                }
+                if proto == "icmp":
+                    rule["from_port"] = -1
+                    rule["to_port"] = -1
+                    rules.append(rule)
+                else:
+                    for port in allowed.get('ports', []):
+                        if "-" in port:
+                            from_port, to_port = port.split("-")
+                        else:
+                            from_port = to_port = port
+                        rule["from_port"] = from_port
+                        rule["to_port"] = to_port
+                        rules.append(copy.copy(rule))
+        return rules
+
+    def get_network_firewalls(self, context, network_name):
+        firewalls = self.get_items(context, None)
+        return [f for f in firewalls
+                if f.get("network_name", None) == network_name]
+
+    def delete_network_firewalls(self, context, network):
+        network_name = network["name"]
+        client = clients.nova(context)
+        for secgroup in self.get_network_firewalls(context, network_name):
+            try:
+                client.security_groups.delete(secgroup["id"])
+            except Exception:
+                LOG.exception(("Failed to delete security group (%s) while"
+                               "delete network (%s))"),
+                              secgroup["name"], network_name)

gceapi/api/firewalls.py

@@ -0,0 +1,45 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gceapi.api import common as gce_common
+from gceapi.api import firewall_api
+from gceapi.api import scopes
+from gceapi.api import wsgi as gce_wsgi
+
+
+class Controller(gce_common.Controller):
+    """GCE Firewall controller"""
+
+    def __init__(self, *args, **kwargs):
+        super(Controller, self).__init__(firewall_api.API(), *args, **kwargs)
+
+    def format_item(self, request, firewall, scope):
+        result_dict = {
+            "creationTimestamp": firewall.get("creationTimestamp", ""),
+            "name": firewall["name"],
+            "description": firewall["description"],
+            "sourceRanges": firewall["sourceRanges"],
+            "allowed": firewall["allowed"],
+        }
+
+        network = firewall.get("network_name")
+        if network:
+            result_dict["network"] = self._qualify(request,
+                "networks", network, scopes.GlobalScope())
+
+        return self._format_item(request, result_dict, scope)
+
+
+def create_resource():
+    return gce_wsgi.GCEResource(Controller())

gceapi/api/image_api.py

@@ -0,0 +1,142 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+try:
+    from glanceclient import exc as glanceclient_exc
+except ImportError:
+    glanceclient_exc = None
+
+from gceapi.api import base_api
+from gceapi.api import clients
+from gceapi.api import operation_api
+from gceapi.api import operation_util
+from gceapi.api import utils
+from gceapi import exception
+from gceapi.openstack.common.gettextutils import _
+
+
+class API(base_api.API):
+    """GCE Image API."""
+
+    KIND = "image"
+    PERSISTENT_ATTRIBUTES = ["id", "description", "image_ref"]
+
+    _status_map = {
+        "queued": "PENDING",
+        "saving": "PENDING",
+        "active": "READY",
+        "killed": "FAILED",
+        # "deleted": "",
+        # "pending_delete": ""
+    }
+
+    def __init__(self, *args, **kwargs):
+        super(API, self).__init__(*args, **kwargs)
+        operation_api.API().register_get_progress_method(
+                "image-add",
+                self._get_add_item_progress)
+        operation_api.API().register_get_progress_method(
+                "image-delete",
+                self._get_delete_item_progress)
+
+    def _get_type(self):
+        return self.KIND
+
+    def _get_persistent_attributes(self):
+        return self.PERSISTENT_ATTRIBUTES
+
+    def get_item(self, context, name, scope=None):
+        image_service = clients.glance(context).images
+        images = image_service.list(
+            filters={"name": name, "disk_format": "raw"})
+        result = None
+        for image in images:
+            if image.status == "deleted":
+                continue
+            if result:
+                msg = _("Image resource '%s' found more than once") % name
+                raise exception.NotFound(msg)
+            result = self._prepare_image(utils.to_dict(image))
+            db_image = self._get_db_item_by_id(context, result["id"])
+            self._prepare_item(result, db_image)
+        if not result:
+            msg = _("Image resource '%s' could not be found") % name
+            raise exception.NotFound(msg)
+        return result
+
+    def get_items(self, context, scope=None):
+        image_service = clients.glance(context).images
+        images = image_service.list(filters={"disk_format": "raw"})
+        items = list()
+        gce_images = self._get_db_items_dict(context)
+        for image in images:
+            result = self._prepare_image(utils.to_dict(image))
+            self._prepare_item(result, gce_images.get(result["id"]))
+            items.append(result)
+        self._purge_db(context, items, gce_images)
+        return items
+
+    def _prepare_image(self, item):
+        item["status"] = self._status_map.get(item["status"], item["status"])
+        return item
+
+    def delete_item(self, context, name, scope=None):
+        """Delete an image, if allowed."""
+        image = self.get_item(context, name, scope)
+        image_service = clients.glance(context).images
+        operation_util.start_operation(context,
+                                       self._get_delete_item_progress,
+                                       image["id"])
+        image_service.delete(image["id"])
+        self._delete_db_item(context, image)
+
+    def add_item(self, context, name, body, scope=None):
+        name = body['name']
+        image_ref = body['rawDisk']['source']
+        meta = {
+            'name': name,
+            'disk_format': 'raw',
+            'container_format': 'bare',
+            'min_disk': 0,
+            'min_ram': 0,
+            'copy_from': image_ref,
+        }
+        image_service = clients.glance(context).images
+        operation_util.start_operation(context, self._get_add_item_progress)
+        image = image_service.create(**meta)
+        operation_util.set_item_id(context, image.id)
+
+        new_image = self._prepare_image(utils.to_dict(image))
+        new_image["description"] = body.get("description", "")
+        new_image["image_ref"] = image_ref
+        new_image = self._add_db_item(context, new_image)
+        return new_image
+
+    def _get_add_item_progress(self, context, image_id):
+        image_service = clients.glance(context).images
+        try:
+            image = image_service.get(image_id)
+        except glanceclient_exc.HTTPNotFound:
+            return operation_api.gef_final_progress()
+        if image.status not in ["queued", "saving"]:
+            return operation_api.gef_final_progress(image.status == "killed")
+
+    def _get_delete_item_progress(self, context, image_id):
+        image_service = clients.glance(context).images
+        try:
+            image = image_service.get(image_id)
+        except glanceclient_exc.HTTPNotFound:
+            return operation_api.gef_final_progress()
+        if image.status not in ["pending_delete", "deleted"]:
+            return operation_api.gef_final_progress(True)

gceapi/api/images.py

@@ -0,0 +1,44 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gceapi.api import common as gce_common
+from gceapi.api import image_api
+from gceapi.api import wsgi as gce_wsgi
+
+
+class Controller(gce_common.Controller):
+    """GCE Image controller"""
+
+    def __init__(self, *args, **kwargs):
+        super(Controller, self).__init__(image_api.API(), *args, **kwargs)
+
+    def format_item(self, request, image, scope):
+        result_dict = {
+            "creationTimestamp": self._format_date(image["created_at"]),
+            "name": image["name"],
+            "sourceType": image["disk_format"].upper(),
+            "rawDisk": {
+                "containerType": "TAR",
+                "source": image.get("image_ref", ""),
+            },
+            "status": image["status"],
+            "archiveSizeBytes": image["size"],
+            "description": image.get("description", "")
+        }
+
+        return self._format_item(request, result_dict, scope)
+
+
+def create_resource():
+    return gce_wsgi.GCEResource(Controller())

gceapi/api/instance_address_api.py

@@ -0,0 +1,162 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gceapi.api import base_api
+from gceapi.api import clients
+from gceapi.api import operation_util
+from gceapi import exception
+from gceapi.openstack.common.gettextutils import _
+from gceapi.openstack.common import log as logging
+
+LOG = logging.getLogger(__name__)
+
+
+class API(base_api.API):
+    """GCE Access config API."""
+
+    KIND = "access_config"
+    PERSISTENT_ATTRIBUTES = ["id", "instance_name",
+                             "nic", "name", "type", "addr"]
+    DEFAULT_ACCESS_CONFIG_TYPE = "ONE_TO_ONE_NAT"
+    DEFAULT_ACCESS_CONFIG_NAME = "External NAT"
+
+    def __init__(self, *args, **kwargs):
+        super(API, self).__init__(*args, **kwargs)
+
+    def _get_type(self):
+        return self.KIND
+
+    def _get_persistent_attributes(self):
+        return self.PERSISTENT_ATTRIBUTES
+
+    def get_item(self, context, instance_name, name):
+        items = self._get_db_items(context)
+        items = [i for i in items
+                if i["instance_name"] == instance_name and i["name"] == name]
+        if len(items) != 1:
+            raise exception.NotFound
+        return items[0]
+
+    def get_items(self, context, instance_name):
+        items = self._get_db_items(context)
+        return [i for i in items if i["instance_name"] == instance_name]
+
+    def add_item(self, context, instance_name, nic, addr, addr_type, name):
+        if not nic:
+            msg = _("Network interface is invalid or empty")
+            raise exception.InvalidRequest(msg)
+
+        if addr_type is None:
+            addr_type = self.DEFAULT_ACCESS_CONFIG_TYPE
+        elif addr_type != self.DEFAULT_ACCESS_CONFIG_TYPE:
+            msg = _("Only '%s' type of access config currently supported.")\
+                    % self.DEFAULT_ACCESS_CONFIG_TYPE
+            raise exception.InvalidRequest(msg)
+
+        client = clients.nova(context)
+        instances = client.servers.list(search_opts={"name": instance_name})
+        if not instances or len(instances) != 1:
+            raise exception.NotFound
+        instance = instances[0]
+
+        fixed_ip = None
+        for network in instance.addresses:
+            if nic != network:
+                continue
+            for address in instance.addresses[network]:
+                atype = address["OS-EXT-IPS:type"]
+                if atype == "floating":
+                    msg = _('At most one access config currently supported.')
+                    raise exception.InvalidRequest(msg)
+                if atype == "fixed":
+                    fixed_ip = address["addr"]
+
+        if not fixed_ip:
+            msg = _('Network interface not found')
+            raise exception.InvalidRequest(msg)
+
+        floating_ips = client.floating_ips.list()
+        if addr is None:
+            # NOTE(apavlov): try to find unused
+            for floating_ip in floating_ips:
+                if floating_ip.instance_id is None:
+                    addr = floating_ip.ip
+                    break
+            else:
+                msg = _('There is no unused floating ips.')
+                raise exception.InvalidRequest(msg)
+        else:
+            for floating_ip in floating_ips:
+                if floating_ip.ip != addr:
+                    continue
+                if floating_ip.instance_id is None:
+                    break
+                msg = _("Floating ip '%s' is already associated") % floating_ip
+                raise exception.InvalidRequest(msg)
+            else:
+                msg = _("There is no such floating ip '%s'.") % addr
+                raise exception.InvalidRequest(msg)
+
+        operation_util.start_operation(context)
+        instance.add_floating_ip(addr, fixed_ip)
+
+        return self.register_item(context, instance_name,
+                                  nic, addr, addr_type, name)
+
+    def register_item(self, context, instance_name,
+                      nic, addr, addr_type, name):
+        if not nic:
+            msg = _("Network interface is invalid or empty")
+            raise exception.InvalidRequest(msg)
+
+        if addr_type is None:
+            addr_type = self.DEFAULT_ACCESS_CONFIG_TYPE
+        elif addr_type != self.DEFAULT_ACCESS_CONFIG_TYPE:
+            msg = _("Only '%s' type of access config currently supported.")\
+                % self.DEFAULT_ACCESS_CONFIG_TYPE
+            raise exception.InvalidRequest(msg)
+
+        if name is None:
+            name = self.DEFAULT_ACCESS_CONFIG_NAME
+        if not addr:
+            msg = _("There is no address to assign.")
+            raise exception.InvalidRequest(msg)
+
+        new_item = {
+            "id": instance_name + "-" + addr,
+            "instance_name": instance_name,
+            "nic": nic,
+            "name": name,
+            "type": addr_type,
+            "addr": addr
+        }
+        new_item = self._add_db_item(context, new_item)
+        return new_item
+
+    def delete_item(self, context, instance_name, name):
+        client = clients.nova(context)
+        instances = client.servers.list(search_opts={"name": instance_name})
+        if not instances or len(instances) != 1:
+            raise exception.NotFound
+        instance = instances[0]
+
+        item = self.get_item(context, instance_name, name)
+        floating_ip = item["addr"]
+        operation_util.start_operation(context)
+        instance.remove_floating_ip(floating_ip)
+        self._delete_db_item(context, item)
+
+    def unregister_item(self, context, instance_name, name):
+        item = self.get_item(context, instance_name, name)
+        self._delete_db_item(context, item)

gceapi/api/instance_api.py

@@ -0,0 +1,377 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import string
+
+from gceapi.api import base_api
+from gceapi.api import clients
+from gceapi.api import disk_api
+from gceapi.api import firewall_api
+from gceapi.api import instance_address_api
+from gceapi.api import instance_disk_api
+from gceapi.api import machine_type_api
+from gceapi.api import network_api
+from gceapi.api import operation_api
+from gceapi.api import operation_util
+from gceapi.api import project_api
+from gceapi.api import scopes
+from gceapi.api import utils
+from gceapi import exception
+from gceapi.openstack.common.gettextutils import _
+from gceapi.openstack.common import log as logging
+
+LOG = logging.getLogger(__name__)
+
+
+class API(base_api.API):
+    """GCE Instance API."""
+
+    KIND = "instance"
+    PERSISTENT_ATTRIBUTES = ["id", "description"]
+
+    _status_map = {
+        "UNKNOWN": "STOPPED",
+        "ACTIVE": "RUNNING",
+        "REBOOT": "RUNNING",
+        "HARD_REBOOT": "RUNNING",
+        "PASSWORD": "RUNNING",
+        "REBUILD": "RUNNING",
+        "MIGRATING": "RUNNING",
+        "RESIZE": "RUNNING",
+        "BUILD": "PROVISIONING",
+        "SHUTOFF": "STOPPED",
+        "VERIFY_RESIZE": "RUNNING",
+        "REVERT_RESIZE": "RUNNING",
+        "PAUSED": "STOPPED",
+        "SUSPENDED": "STOPPED",
+        "RESCUE": "RUNNING",
+        "ERROR": "STOPPED",
+        "DELETED": "TERMINATED",
+        "SOFT_DELETED": "TERMINATED",
+        "SHELVED": "STOPPED",
+        "SHELVED_OFFLOADED": "STOPPED",
+    }
+
+    def __init__(self, *args, **kwargs):
+        super(API, self).__init__(*args, **kwargs)
+        network_api.API()._register_callback(
+            base_api._callback_reasons.check_delete,
+            self._can_delete_network)
+        firewall_api.API()._register_callback(
+            base_api._callback_reasons.post_add,
+            self._add_secgroup_to_instances)
+        firewall_api.API()._register_callback(
+            base_api._callback_reasons.pre_delete,
+            self._remove_secgroup_from_instances)
+        operation_api.API().register_get_progress_method(
+                "instance-add",
+                self._get_add_item_progress)
+        operation_api.API().register_get_progress_method(
+                "instance-delete",
+                self._get_delete_item_progress)
+        operation_api.API().register_get_progress_method(
+                "instance-reset",
+                self._get_reset_instance_progress)
+
+    def _get_type(self):
+        return self.KIND
+
+    def _get_persistent_attributes(self):
+        return self.PERSISTENT_ATTRIBUTES
+
+    def get_item(self, context, name, scope=None):
+        return self.search_items(context, {"name": name}, scope)[0]
+
+    def get_items(self, context, scope=None):
+        return self.search_items(context, None, scope)
+
+    def get_scopes(self, context, item):
+        return [scopes.ZoneScope(item["OS-EXT-AZ:availability_zone"])]
+
+    def search_items(self, context, search_opts, scope):
+        client = clients.nova(context)
+        instances = client.servers.list(search_opts=search_opts)
+
+        filtered_instances = []
+        for instance in instances:
+            iscope = getattr(instance, "OS-EXT-AZ:availability_zone")
+            if scope is not None and scope.get_name() != iscope:
+                continue
+
+            instance = utils.to_dict(instance)
+            instance = self._prepare_instance(client, context, instance)
+            db_instance = self._get_db_item_by_id(context, instance["id"])
+            self._prepare_item(instance, db_instance)
+            filtered_instances.append(instance)
+
+        if len(filtered_instances) == len(instances) and not search_opts:
+            gce_instances = self._get_db_items_dict(context)
+            self._purge_db(context, filtered_instances, gce_instances)
+
+        return filtered_instances
+
+    def _prepare_instance(self, client, context, instance):
+        instance["statusMessage"] = instance["status"]
+        instance["status"] = self._status_map.get(
+            instance["status"], "STOPPED")
+        instance["flavor"]["name"] = machine_type_api.API().get_item_by_id(
+            context, instance["flavor"]["id"])["name"]
+
+        cinder_client = clients.cinder(context)
+        volumes = instance["os-extended-volumes:volumes_attached"]
+        instance["volumes"] = [utils.to_dict(
+            cinder_client.volumes.get(v["id"])) for v in volumes]
+        ads = instance_disk_api.API().get_items(context, instance["name"])
+        ads = dict((ad["volume_id"], ad) for ad in ads)
+        for volume in instance["volumes"]:
+            ad = ads.pop(volume["id"], None)
+            if not ad:
+                name = volume["display_name"]
+                ad = instance_disk_api.API().register_item(context,
+                    instance["name"], volume["id"], name)
+            volume["device_name"] = ad["name"]
+        # NOTE(apavlov): cleanup unused from db for this instance
+        for ad in ads:
+            ad = instance_disk_api.API().unregister_item(context,
+                instance["name"], ads[ad]["name"])
+
+        acs = instance_address_api.API().get_items(context, instance["name"])
+        acs = dict((ac["addr"], ac) for ac in acs)
+        for network in instance["addresses"]:
+            for address in instance["addresses"][network]:
+                if address["OS-EXT-IPS:type"] == "floating":
+                    ac = acs.pop(address["addr"], None)
+                    if not ac:
+                        ac = instance_address_api.API().register_item(context,
+                            instance["name"], network, address["addr"],
+                            None, None)
+                    address["name"] = ac["name"]
+                    address["type"] = ac["type"]
+        # NOTE(apavlov): cleanup unused from db for this instance
+        for ac in acs:
+            ac = instance_address_api.API().unregister_item(context,
+                instance["name"], acs[ac]["name"])
+
+        return instance
+
+    def _can_delete_network(self, context, network):
+        client = clients.nova(context)
+        instances = client.servers.list(search_opts=None)
+        for instance in instances:
+            if network["name"] in instance.networks:
+                raise exception.NetworkInUse(network_id=network["id"])
+
+    def _get_instances_with_network(self, context, network_name, scope):
+        affected_instances = []
+        client = clients.nova(context)
+        instances = client.servers.list(search_opts=None)
+        for instance in instances:
+            if network_name in instance.networks:
+                affected_instances.append(instance)
+        return affected_instances
+
+    def _add_secgroup_to_instances(self, context, secgroup, **kwargs):
+        network_name = secgroup.get("network_name")
+        if not network_name:
+            return
+        affected_instances = self._get_instances_with_network(
+                context, network_name, kwargs.get("scope"))
+        # TODO(ft): implement common safe method
+        # to run add/remove with exception logging
+        for instance in affected_instances:
+            try:
+                instance.add_security_group(secgroup["name"])
+            except Exception:
+                LOG.exception(("Failed to add instance "
+                               "(%s) to security group (%s)"),
+                              instance.id, secgroup["name"])
+
+    def _remove_secgroup_from_instances(self, context, secgroup, **kwargs):
+        network_name = secgroup.get("network_name")
+        if not network_name:
+            return
+        affected_instances = self._get_instances_with_network(
+                context, network_name, kwargs.get("scope"))
+        # TODO(ft): implement common safe method
+        # to run add/remove with exception logging
+        for instance in affected_instances:
+            try:
+                instance.remove_security_group(secgroup["name"])
+            except Exception:
+                LOG.exception(("Failed to remove securiy group (%s) "
+                               "from instance (%s)"),
+                              secgroup["name"], instance.id)
+
+    def reset_instance(self, context, scope, name):
+        client = clients.nova(context)
+        instances = client.servers.list(search_opts={"name": name})
+        if not instances or len(instances) != 1:
+            raise exception.NotFound
+        instance = instances[0]
+        operation_util.start_operation(context,
+                                       self._get_reset_instance_progress,
+                                       instance.id)
+        instance.reboot("HARD")
+
+    def delete_item(self, context, name, scope=None):
+        client = clients.nova(context)
+        instances = client.servers.list(search_opts={"name": name})
+        if not instances or len(instances) != 1:
+            raise exception.NotFound
+        instance = instances[0]
+        operation_util.start_operation(context,
+                                       self._get_delete_item_progress,
+                                       instance.id)
+        instance.delete()
+        instance = utils.to_dict(instance)
+        instance = self._prepare_instance(client, context, instance)
+        self._delete_db_item(context, instance)
+
+        ads = instance_disk_api.API().get_items(context, instance["name"])
+        for ad in ads:
+            ad = instance_disk_api.API().unregister_item(context,
+                instance["name"], ad["name"])
+
+        acs = instance_address_api.API().get_items(context, instance["name"])
+        for ac in acs:
+            ac = instance_address_api.API().unregister_item(context,
+                instance["name"], ac["name"])
+
+    def add_item(self, context, name, body, scope=None):
+        name = body['name']
+        client = clients.nova(context)
+
+        flavor_name = utils._extract_name_from_url(body['machineType'])
+        flavor_id = machine_type_api.API().get_item(
+            context, flavor_name, scope)["id"]
+
+        try:
+            metadatas = body['metadata']['items']
+        except KeyError:
+            metadatas = []
+        instance_metadata = dict([(x['key'], x['value']) for x in metadatas])
+
+        ssh_keys = instance_metadata.pop('sshKeys', None)
+        if ssh_keys is not None:
+            key_name = ssh_keys.split('\n')[0].split(":")[0]
+        else:
+            key_name = project_api.API().get_gce_user_keypair_name(context)
+
+        disks = body.get('disks', [])
+        disks.sort(None, lambda x: x.get("boot", False), True)
+        bdm = dict()
+        diskDevice = 0
+        for disk in disks:
+            device_name = "vd" + string.ascii_lowercase[diskDevice]
+            volume_name = utils._extract_name_from_url(disk["source"])
+            volume = disk_api.API().get_item(context, volume_name, scope)
+            disk["id"] = volume["id"]
+            bdm[device_name] = volume["id"]
+            diskDevice += 1
+
+        nics = []
+        #NOTE(ft) 'default' security group contains output rules
+        #but output rules doesn't configurable by GCE API
+        #all outgoing traffic permitted
+        #so we support this behaviour
+        groups_names = set(['default'])
+        acs = dict()
+        for net_iface in body['networkInterfaces']:
+            net_name = utils._extract_name_from_url(net_iface["network"])
+            ac = net_iface.get("accessConfigs")
+            if ac:
+                if len(ac) > 1:
+                    msg = _('At most one access config currently supported.')
+                    raise exception.InvalidRequest(msg)
+                else:
+                    acs[net_name] = ac[0]
+
+            network = network_api.API().get_item(context, net_name, None)
+            nics.append({"net-id": network["id"]})
+            for sg in firewall_api.API().get_network_firewalls(
+                    context, net_name):
+                groups_names.add(sg["name"])
+        groups_names = list(groups_names)
+
+        operation_util.start_operation(context, self._get_add_item_progress)
+        instance = client.servers.create(name, None, flavor_id,
+            meta=instance_metadata, min_count=1, max_count=1,
+            security_groups=groups_names, key_name=key_name,
+            availability_zone=scope.get_name(), block_device_mapping=bdm,
+            nics=nics)
+        if not acs:
+            operation_util.set_item_id(context, instance.id)
+
+        for disk in disks:
+            instance_disk_api.API().register_item(context, name,
+                disk["id"], disk["deviceName"])
+
+        instance = utils.to_dict(client.servers.get(instance.id))
+        instance = self._prepare_instance(client, context, instance)
+        if "descripton" in body:
+            instance["description"] = body["description"]
+        instance = self._add_db_item(context, instance)
+
+        if acs:
+            operation_util.continue_operation(
+                    context,
+                    lambda: self._add_access_config(context, instance,
+                                                    scope, acs))
+
+        return instance
+
+    def _add_access_config(self, context, instance, scope, acs):
+        progress = self._get_add_item_progress(context, instance["id"])
+        if progress is None or not operation_api.is_final_progress(progress):
+            return progress
+
+        client = clients.nova(context)
+        try:
+            instance = client.servers.get(instance["id"])
+        except clients.novaclient.exceptions.NotFound:
+            return operation_api.gef_final_progress()
+
+        for net in acs:
+            ac = acs[net]
+            instance_address_api.API().add_item(context, instance.name,
+                net, ac.get("natIP"), ac.get("type"), ac.get("name"))
+        return operation_api.gef_final_progress()
+
+    def _get_add_item_progress(self, context, instance_id):
+        client = clients.nova(context)
+        try:
+            instance = client.servers.get(instance_id)
+        except clients.novaclient.exceptions.NotFound:
+            return operation_api.gef_final_progress()
+        if instance.status != "BUILD":
+            return operation_api.gef_final_progress(instance.status == "ERROR")
+
+    def _get_delete_item_progress(self, context, instance_id):
+        client = clients.nova(context)
+        try:
+            instance = client.servers.get(instance_id)
+        except clients.novaclient.exceptions.NotFound:
+            return operation_api.gef_final_progress()
+        if getattr(instance, "OS-EXT-STS:task_state") != "deleting":
+            return operation_api.gef_final_progress(
+                    instance.status != "DELETED")
+
+    def _get_reset_instance_progress(self, context, instance_id):
+        client = clients.nova(context)
+        try:
+            instance = client.servers.get(instance_id)
+        except clients.novaclient.exceptions.NotFound:
+            return operation_api.gef_final_progress()
+        if instance.status != "HARD_REBOOT":
+            return operation_api.gef_final_progress()

gceapi/api/instance_disk_api.py

@@ -0,0 +1,146 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import string
+
+from gceapi.api import base_api
+from gceapi.api import clients
+from gceapi.api import disk_api
+from gceapi.api import operation_api
+from gceapi.api import operation_util
+from gceapi.api import utils
+from gceapi import exception
+from gceapi.openstack.common.gettextutils import _
+from gceapi.openstack.common import log as logging
+
+LOG = logging.getLogger(__name__)
+
+
+class API(base_api.API):
+    """GCE Attached disk API."""
+
+    KIND = "attached_disk"
+    PERSISTENT_ATTRIBUTES = ["id", "instance_name", "volume_id", "name"]
+
+    def __init__(self, *args, **kwargs):
+        super(API, self).__init__(*args, **kwargs)
+        operation_api.API().register_get_progress_method(
+                "attached_disk-add",
+                self._get_add_item_progress)
+        operation_api.API().register_get_progress_method(
+                "attached_disk-delete",
+                self._get_delete_item_progress)
+
+    def _get_type(self):
+        return self.KIND
+
+    def _get_persistent_attributes(self):
+        return self.PERSISTENT_ATTRIBUTES
+
+    def get_item(self, context, instance_name, name):
+        items = self._get_db_items(context)
+        items = [i for i in items
+                if i["instance_name"] == instance_name and i["name"] == name]
+        if len(items) != 1:
+            raise exception.NotFound
+        return items[0]
+
+    def get_items(self, context, instance_name):
+        items = self._get_db_items(context)
+        return [i for i in items if i["instance_name"] == instance_name]
+
+    def add_item(self, context, instance_name, source, name):
+        if not name:
+            msg = _("There is no name to assign.")
+            raise exception.InvalidRequest(msg)
+
+        volume_name = utils._extract_name_from_url(source)
+        if not volume_name:
+            msg = _("There is no volume to assign.")
+            raise exception.NotFound(msg)
+        volume = disk_api.API().get_item(context, volume_name, None)
+
+        nova_client = clients.nova(context)
+        instances = nova_client.servers.list(
+            search_opts={"name": instance_name})
+        if not instances or len(instances) != 1:
+            raise exception.NotFound
+        instance = instances[0]
+
+        devices = list()
+        volumes_client = nova_client.volumes
+        for server_volume in volumes_client.get_server_volumes(instance.id):
+            devices.append(server_volume.device)
+        device_name = None
+        for letter in string.ascii_lowercase[1:]:
+            device_name = "vd" + letter
+            for device in devices:
+                if device_name in device:
+                    break
+            else:
+                break
+        else:
+            raise exception.OverQuota
+
+        operation_util.start_operation(context, self._get_add_item_progress)
+        volumes_client.create_server_volume(
+            instance.id, volume["id"], "/dev/" + device_name)
+
+        item = self.register_item(context, instance_name, volume["id"], name)
+        operation_util.set_item_id(context, item["id"])
+
+    def register_item(self, context, instance_name, volume_id, name):
+        if not name:
+            msg = _("There is no name to assign.")
+            raise exception.InvalidRequest(msg)
+        if not volume_id:
+            msg = _("There is no volume_id to assign.")
+            raise exception.InvalidRequest(msg)
+
+        new_item = {
+            "id": instance_name + "-" + volume_id,
+            "instance_name": instance_name,
+            "volume_id": volume_id,
+            "name": name,
+        }
+        new_item = self._add_db_item(context, new_item)
+        return new_item
+
+    def delete_item(self, context, instance_name, name):
+        item = self.get_item(context, instance_name, name)
+        volume_id = item["volume_id"]
+
+        nova_client = clients.nova(context)
+        instances = nova_client.servers.list(
+            search_opts={"name": instance_name})
+        if not instances or len(instances) != 1:
+            raise exception.NotFound
+        instance = instances[0]
+
+        operation_util.start_operation(context,
+                                       self._get_delete_item_progress,
+                                       item["id"])
+        nova_client.volumes.delete_server_volume(instance.id, volume_id)
+
+        self._delete_db_item(context, item)
+
+    def unregister_item(self, context, instance_name, name):
+        item = self.get_item(context, instance_name, name)
+        self._delete_db_item(context, item)
+
+    def _get_add_item_progress(self, context, dummy_id):
+        return operation_api.gef_final_progress()
+
+    def _get_delete_item_progress(self, context, dummy_id):
+        return operation_api.gef_final_progress()

gceapi/api/instances.py

@@ -0,0 +1,157 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import webob
+
+from gceapi.api import common as gce_common
+from gceapi.api import instance_address_api
+from gceapi.api import instance_api
+from gceapi.api import instance_disk_api
+from gceapi.api import operation_util
+from gceapi.api import scopes
+from gceapi.api import wsgi as gce_wsgi
+from gceapi import exception
+from gceapi.openstack.common.gettextutils import _
+from gceapi.openstack.common import log as logging
+
+logger = logging.getLogger(__name__)
+
+
+class Controller(gce_common.Controller):
+    """GCE Instance controller"""
+
+    def __init__(self, *args, **kwargs):
+        super(Controller, self).__init__(instance_api.API(),
+                                         *args, **kwargs)
+        self._instance_address_api = instance_address_api.API()
+        self._instance_disk_api = instance_disk_api.API()
+
+    def format_item(self, request, instance, scope):
+        result_dict = {
+            "creationTimestamp": self._format_date(instance["created"]),
+            "status": instance["status"],
+            "statusMessage": instance["statusMessage"],
+            "name": instance["name"],
+            "machineType": self._qualify(request,
+                "machineTypes", instance["flavor"]["name"], scope),
+            "networkInterfaces": [],
+            "disks": [],
+            "metadata": {
+                "kind": "compute#metadata",
+                "items": list(),
+            },
+        }
+
+        description = instance.get("description", "")
+        if description:
+            result_dict["description"] = description
+
+        metadata = instance.get("metadata", {})
+        for i in metadata:
+            result_dict["metadata"]["items"].append(
+                {"key": i, "value": metadata[i]})
+
+        for network in instance["addresses"]:
+            ni = dict()
+            ni["network"] = self._qualify(request,
+                "networks", network,
+                scopes.GlobalScope())
+            # NOTE(apavlov): The name of the network interface, generated by
+            # the server. For network devices, these are eth0, eth1, etc.
+            # But we provide name of network here because Openstack doesn`t
+            # have device name.
+            ni["name"] = network
+            ni["accessConfigs"] = []
+            for address in instance["addresses"][network]:
+                atype = address["OS-EXT-IPS:type"]
+                if atype == "fixed" and "networkIP" not in ni:
+                    ni["networkIP"] = address["addr"]
+                    continue
+                if atype == "floating":
+                    ni["accessConfigs"].append({
+                        "kind": "compute#accessConfig",
+                        "name": address["name"],
+                        "type": address["type"],
+                        "natIP": address["addr"]
+                    })
+                    continue
+                logger.warn(_("Unexpected address for instance '%(i)' in "
+                    "network '%(n)") % {"i": instance["name"], "n": network})
+            result_dict["networkInterfaces"].append(ni)
+
+        disk_index = 0
+        for volume in instance["volumes"]:
+            readonly = volume.get("metadata", {}).get("readonly", "False")
+            google_disk = {
+                "kind": "compute#attachedDisk",
+                "index": disk_index,
+                "type": "PERSISTENT",
+                "mode": "READ_ONLY" if readonly == "True" else "READ_WRITE",
+                "source": self._qualify(request,
+                    "disks", volume["display_name"], scope),
+                "deviceName": volume["device_name"],
+                "boot": True if volume["bootable"] == "true" else False
+            }
+            result_dict["disks"].append(google_disk)
+            disk_index += 1
+
+        return self._format_item(request, result_dict, scope)
+
+    def reset_instance(self, req, scope_id, id):
+        context = self._get_context(req)
+        scope = self._get_scope(req, scope_id)
+        operation_util.init_operation(context, "reset",
+                                      self._type_name, id, scope)
+        try:
+            self._api.reset_instance(context, scope, id)
+        except (exception.NotFound, KeyError, IndexError):
+            msg = _("Instance %s could not be found") % id
+            raise webob.exc.HTTPNotFound(explanation=msg)
+
+    def add_access_config(self, req, body, scope_id, id):
+        context = self._get_context(req)
+        scope = self._get_scope(req, scope_id)
+        operation_util.init_operation(context, "addAccessConfig",
+                                      self._type_name, id, scope)
+        self._instance_address_api.add_item(context, id,
+            req.params.get('networkInterface'), body.get("natIP"),
+            body.get("type"), body.get("name"))
+
+    def delete_access_config(self, req, scope_id, id):
+        context = self._get_context(req)
+        scope = self._get_scope(req, scope_id)
+        operation_util.init_operation(context, "deleteAccessConfig",
+                                      self._type_name, id, scope)
+        self._instance_address_api.delete_item(context, id,
+           req.params.get('accessConfig'))
+
+    def attach_disk(self, req, body, scope_id, id):
+        context = self._get_context(req)
+        scope = self._get_scope(req, scope_id)
+        operation_util.init_operation(context, "attachDisk",
+                                      self._type_name, id, scope)
+        self._instance_disk_api.add_item(context, id,
+            body["source"], body.get("deviceName"))
+
+    def detach_disk(self, req, scope_id, id):
+        context = self._get_context(req)
+        scope = self._get_scope(req, scope_id)
+        operation_util.init_operation(context, "detachDisk",
+                                      self._type_name, id, scope)
+        self._instance_disk_api.delete_item(context, id,
+            req.params.get('deviceName'))
+
+
+def create_resource():
+    return gce_wsgi.GCEResource(Controller())

gceapi/api/machine_type_api.py

@@ -0,0 +1,68 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gceapi.api import base_api
+from gceapi.api import clients
+from gceapi.api import utils
+from gceapi.api import zone_api
+from gceapi import exception
+
+
+class API(base_api.API):
+    """GCE Machine types API."""
+
+    KIND = "machineType"
+
+    def __init__(self, *args, **kwargs):
+        super(API, self).__init__(*args, **kwargs)
+        self._zone_api = zone_api.API()
+
+    def _get_type(self):
+        return self.KIND
+
+    def get_item(self, context, name, scope=None):
+        client = clients.nova(context)
+        try:
+            item = client.flavors.find(name=self._from_gce(name))
+        except (clients.novaclient.exceptions.NotFound,
+                clients.novaclient.exceptions.NoUniqueMatch):
+            raise exception.NotFound
+        if not item:
+            raise exception.NotFound
+        return self._prepare_item(utils.to_dict(item))
+
+    def get_items(self, context, scope=None):
+        client = clients.nova(context)
+        items = client.flavors.list()
+        return [self._prepare_item(utils.to_dict(item))
+                for item in items]
+
+    def get_scopes(self, context, item):
+        # TODO(apavlov): too slow for all...
+        return self._zone_api.get_items_as_scopes(context)
+
+    def get_item_by_id(self, context, machine_type_id):
+        client = clients.nova(context)
+        item = client.flavors.get(machine_type_id)
+        return self._prepare_item(utils.to_dict(item))
+
+    def _prepare_item(self, item):
+        item["name"] = self._to_gce(item["name"])
+        return item
+
+    def _from_gce(self, name):
+        return name.replace("-", ".")
+
+    def _to_gce(self, name):
+        return name.replace(".", "-")

gceapi/api/machine_types.py

@@ -0,0 +1,49 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gceapi.api import common as gce_common
+from gceapi.api import machine_type_api
+from gceapi.api import wsgi as gce_wsgi
+
+
+class Controller(gce_common.Controller):
+    """GCE Machine types controller"""
+
+    def __init__(self, *args, **kwargs):
+        super(Controller, self).__init__(machine_type_api.API(),
+                                         *args, **kwargs)
+
+    def format_item(self, request, flavor, scope):
+        result_dict = {
+            "name": flavor["name"],
+            "description": "",
+            "guestCpus": flavor["vcpus"],
+            "memoryMb": flavor["ram"],
+            "imageSpaceGb": flavor["disk"],
+            # NOTE(Alex): Is not supported by Openstack
+            "maximumPersistentDisks": 0,
+            # NOTE(Alex): Is not supported by Openstack
+            "maximumPersistentDisksSizeGb": 0,
+            }
+
+        if "OS-FLV-EXT-DATA:ephemeral" in flavor:
+            size = flavor["OS-FLV-EXT-DATA:ephemeral"]
+            if size > 0:
+                result_dict["scratchDisks"] = [{"diskGb": size}]
+
+        return self._format_item(request, result_dict, scope)
+
+
+def create_resource():
+    return gce_wsgi.GCEResource(Controller())

gceapi/api/network_api.py

@@ -0,0 +1,26 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gceapi.api import base_api
+from gceapi.api import network_neutron_api
+from gceapi.api import network_nova_api
+
+
+class API(base_api.API):
+    """GCE Network API."""
+
+    NEUTRON_API_MODULE = network_neutron_api
+    NOVA_API_MODULE = network_nova_api
+
+    __metaclass__ = base_api.NetSingleton

gceapi/api/network_neutron_api.py

@@ -0,0 +1,143 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import netaddr
+from oslo.config import cfg
+
+from gceapi.api import base_api
+from gceapi.api import clients
+from gceapi.api import operation_util
+from gceapi import exception
+from gceapi.openstack.common.gettextutils import _
+from gceapi.openstack.common import log as logging
+
+
+CONF = cfg.CONF
+LOG = logging.getLogger(__name__)
+
+
+class API(base_api.API):
+    """GCE Network API - neutron implementation."""
+
+    KIND = "network"
+    PERSISTENT_ATTRIBUTES = ["id", "creationTimestamp", "description"]
+
+    def __init__(self, *args, **kwargs):
+        super(API, self).__init__(*args, **kwargs)
+        self._public_network_name = CONF.public_network
+
+    def _get_type(self):
+        return self.KIND
+
+    def _get_persistent_attributes(self):
+        return self.PERSISTENT_ATTRIBUTES
+
+    def get_item(self, context, name, scope=None):
+        client = clients.neutron(context)
+        networks = client.list_networks(
+            tenant_id=context.project_id, name=name)["networks"]
+        if not networks:
+            msg = _("Network resource '%s' could not be found.") % name
+            raise exception.NotFound(msg)
+        else:
+            # NOTE(Alex) There might be more than one network with this name.
+            # TODO(Alex) We have to decide if we should support IDs as
+            # parameters for names as well and return error if we have
+            # multi-results when addressed by name.
+            network = networks[0]
+            gce_network = self._get_db_item_by_id(context, network["id"])
+            return self._prepare_network(client, network, gce_network)
+
+    def get_items(self, context, scope=None):
+        client = clients.neutron(context)
+        networks = client.list_networks(tenant_id=context.project_id)
+        networks = networks["networks"]
+        gce_networks = self._get_db_items_dict(context)
+        result_networks = []
+        for network in networks:
+            network = self._prepare_network(client, network,
+                                            gce_networks.get(network["id"]))
+            result_networks.append(network)
+        self._purge_db(context, result_networks, gce_networks)
+        return result_networks
+
+    def delete_item(self, context, name, scope=None):
+        client = clients.neutron(context)
+        network = self.get_item(context, name)
+
+        self._process_callbacks(
+            context, base_api._callback_reasons.check_delete, network)
+        operation_util.start_operation(context)
+        self._delete_db_item(context, network)
+        self._process_callbacks(
+            context, base_api._callback_reasons.pre_delete, network)
+
+        client.delete_network(network["id"])
+
+    def add_item(self, context, name, body, scope=None):
+        ip_range = body['IPv4Range']
+        gateway = body.get('gatewayIPv4')
+        if gateway is None:
+            network_cidr = netaddr.IPNetwork(ip_range)
+            gateway_ip = netaddr.IPAddress(network_cidr.first + 1)
+            gateway = str(gateway_ip)
+        client = clients.neutron(context)
+        network = None
+        try:
+            network = self.get_item(context, name)
+        except exception.NotFound:
+            pass
+        if network is not None:
+            raise exception.DuplicateVlan
+        network_body = {}
+        network_body["network"] = {"name": name}
+        operation_util.start_operation(context)
+        network = client.create_network(network_body)
+        network = network["network"]
+        if ip_range:
+            subnet_body = {}
+            subnet_body["subnet"] = {
+                # NOTE(Alex) "name": name + ".default_subnet",
+                # Won't give it a name for now
+                "network_id": network["id"],
+                "ip_version": "4",
+                "cidr": ip_range,
+                "gateway_ip": gateway}
+            result_data = client.create_subnet(subnet_body)
+            subnet_id = result_data["subnet"]["id"]
+        network = self._prepare_network(client, network)
+        network["description"] = body.get("description")
+        network = self._add_db_item(context, network)
+        self._process_callbacks(
+            context, base_api._callback_reasons.post_add,
+            network, subnet_id=subnet_id)
+        return network
+
+    def _prepare_network(self, client, network, db_network=None):
+        subnets = network['subnets']
+        if subnets and len(subnets) > 0:
+            subnet = client.show_subnet(subnets[0])
+            subnet = subnet["subnet"]
+            network["subnet_id"] = subnet["id"]
+            network["IPv4Range"] = subnet.get("cidr", None)
+            network["gatewayIPv4"] = subnet.get("gateway_ip", None)
+        return self._prepare_item(network, db_network)
+
+    def get_public_network_id(self, context):
+        """Get id of public network appointed to GCE in config."""
+        client = clients.neutron(context)
+        search_opts = {"name": self._public_network_name,
+                       "router:external": True}
+        networks = client.list_networks(**search_opts)["networks"]
+        return networks[0]["id"]

gceapi/api/network_nova_api.py

@@ -0,0 +1,94 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import netaddr
+
+from gceapi.api import base_api
+from gceapi.api import clients
+from gceapi.api import operation_util
+from gceapi.api import utils
+from gceapi import exception
+
+
+class API(base_api.API):
+    """GCE Network API - nova-network implementation."""
+
+    KIND = "network"
+    PERSISTENT_ATTRIBUTES = ["id", "creationTimestamp", "description"]
+
+    def _get_type(self):
+        return self.KIND
+
+    def _get_persistent_attributes(self):
+        return self.PERSISTENT_ATTRIBUTES
+
+    def get_item(self, context, name, scope=None):
+        client = clients.nova(context)
+        network = client.networks.find(label=name)
+        gce_network = self._get_db_item_by_id(context, network.id)
+        return self._prepare_network(utils.to_dict(network), gce_network)
+
+    def get_items(self, context, scope=None):
+        client = clients.nova(context)
+        networks = client.networks.list()
+        gce_networks = self._get_db_items_dict(context)
+        result_networks = []
+        for network in networks:
+            result_networks.append(
+                    self._prepare_network(utils.to_dict(network),
+                                          gce_networks.get(network["id"])))
+        self._purge_db(context, result_networks, gce_networks)
+        return result_networks
+
+    def delete_item(self, context, name, scope=None):
+        network = self.get_item(context, name)
+        self._process_callbacks(
+            context, base_api._callback_reasons.check_delete, network)
+        operation_util.start_operation(context)
+        self._delete_db_item(context, network)
+        self._process_callbacks(
+            context, base_api._callback_reasons.pre_delete, network)
+        client = clients.nova(context)
+        client.networks.delete(network["id"])
+
+    def add_item(self, context, name, body, scope=None):
+        ip_range = body['IPv4Range']
+        gateway = body.get('gatewayIPv4')
+        if gateway is None:
+            network_cidr = netaddr.IPNetwork(ip_range)
+            gateway_ip = netaddr.IPAddress(network_cidr.first + 1)
+            gateway = str(gateway_ip)
+        network = None
+        try:
+            network = self.get_item(context, name)
+        except clients.novaclient.exceptions.NotFound:
+            pass
+        if network is not None:
+            raise exception.DuplicateVlan
+        kwargs = {'label': name, 'cidr': ip_range, 'gateway': gateway}
+        client = clients.nova(context)
+        operation_util.start_operation(context)
+        network = client.networks.create(**kwargs)
+        network = self._prepare_network(utils.to_dict(network))
+        if "description" in body:
+            network["description"] = body["description"]
+        return self._add_db_item(context, network)
+
+    def _prepare_network(self, network, db_data=None):
+        return self._prepare_item({
+                'name': network['label'],
+                'IPv4Range': network['cidr'],
+                'gatewayIPv4': network['gateway'],
+                'id': network['id']},
+            db_data)

gceapi/api/networks.py

@@ -0,0 +1,40 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gceapi.api import common as gce_common
+from gceapi.api import network_api
+from gceapi.api import wsgi as gce_wsgi
+
+
+class Controller(gce_common.Controller):
+    """GCE Network controller"""
+
+    def __init__(self, *args, **kwargs):
+        super(Controller, self).__init__(network_api.API(), *args, **kwargs)
+
+    def format_item(self, request, network, scope):
+        result_dict = {
+            "name": network["name"],
+            "IPv4Range": network.get("IPv4Range", ""),
+            "gatewayIPv4": network.get("gatewayIPv4", ""),
+            "creationTimestamp": network.get("creationTimestamp", ""),
+        }
+        if "description" in network:
+            result_dict["description"] = network["description"]
+
+        return self._format_item(request, result_dict, scope)
+
+
+def create_resource():
+    return gce_wsgi.GCEResource(Controller())

gceapi/api/oauth.py

@@ -0,0 +1,239 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import base64
+import json
+import time
+import uuid
+
+from keystoneclient import exceptions
+from keystoneclient.v2_0 import client as keystone_client
+from oslo.config import cfg
+import webob
+
+from gceapi.openstack.common.gettextutils import _
+from gceapi.openstack.common import log as logging
+from gceapi.openstack.common import timeutils
+from gceapi import wsgi_ext as openstack_wsgi
+
+FLAGS = cfg.CONF
+LOG = logging.getLogger(__name__)
+
+
+INTERNAL_GCUTIL_PROJECTS = ["debian-cloud", "centos-cloud", "google"]
+
+
+class OAuthFault(openstack_wsgi.Fault):
+    """Fault compliant with RFC
+
+    To prevent extra info added by openstack.wsgi.Fault class
+    to response which is not compliant RFC6749.
+    """
+    @webob.dec.wsgify(RequestClass=openstack_wsgi.Request)
+    def __call__(self, req):
+        return self.wrapped_exc
+
+
+class Controller(object):
+    """Simple OAuth2.0 Controller
+
+    If you need other apps to work with GCE API you should add it here
+    in VALID_CLIENTS.
+    Based on https://developers.google.com/accounts/docs/OAuth2InstalledApp
+    and on RFC 6749(paragraph 4.1).
+    """
+
+    AUTH_TIMEOUT = 300
+    VALID_CLIENTS = {
+        "32555940559.apps.googleusercontent.com": "ZmssLNjJy2998hD4CTg2ejr2"}
+
+    INTERNAL_REDIRECT_URI = "urn:ietf:wg:oauth:2.0:oob"
+    AUTH_PAGE_TEMPLATE =\
+        "<!DOCTYPE html>"\
+        "<html xmlns=\"http://www.w3.org/1999/xhtml\"><body>"\
+        "Enter Openstack username and password to access GCE API<br/>"\
+        "<br/>"\
+        "<form action=\"approval\" name=\"approval\" method=\"post\">"\
+        "<input type=\"hidden\" name=\"redirect_uri\" value=\""\
+        + "{redirect_uri}\"/>"\
+        "<input type=\"hidden\" name=\"code\" value=\"{code}\"/>"\
+        "<input type=\"text\" name=\"username\" value=\"\"/><br/>"\
+        "<input type=\"password\" name=\"password\" value=\"\"/><br/>"\
+        "<input type=\"submit\" value=\"Login\"/>"\
+        "</form>"\
+        "</body></html>"
+
+    class Client:
+        auth_start_time = 0
+        auth_token = None
+        expires_in = 1
+
+    # NOTE(apavlov): there is no cleaning of the dictionary
+    _clients = {}
+
+    def _check_redirect_uri(self, uri):
+        if uri is None:
+            msg = _("redirect_uri should be present")
+            raise webob.exc.HTTPBadRequest(explanation=msg)
+        if "localhost" not in uri and uri != self.INTERNAL_REDIRECT_URI:
+            msg = _("redirect_uri has invalid format."
+                    "it must confirms installed application uri of GCE")
+            json_body = {"error": "invalid_request",
+                         "error_description": msg}
+            raise OAuthFault(webob.exc.HTTPBadRequest(json_body=json_body))
+
+    def auth(self, req):
+        """OAuth protocol authorization endpoint handler
+
+        Returns login authorization webpage invoked for example by gcutil auth.
+        """
+        client_id = req.GET.get("client_id")
+        if client_id is None or client_id not in self.VALID_CLIENTS:
+            json_body = {"error": "unauthorized_client"}
+            raise OAuthFault(webob.exc.HTTPBadRequest(json_body=json_body))
+
+        if req.GET.get("response_type") != "code":
+            json_body = {"error": "unsupported_response_type"}
+            raise OAuthFault(webob.exc.HTTPBadRequest(json_body=json_body))
+        self._check_redirect_uri(req.GET.get("redirect_uri"))
+
+        code = base64.urlsafe_b64encode(uuid.uuid4().bytes).replace('=', '')
+        self._clients[code] = self.Client()
+        self._clients[code].auth_start_time = time.time()
+
+        html_page = self.AUTH_PAGE_TEMPLATE.format(
+            redirect_uri=req.GET.get("redirect_uri"),
+            code=code)
+        return html_page
+
+    def approval(self, req):
+        """OAuth protocol authorization endpoint handler second part
+
+        Returns webpage with verification code or redirects to provided
+        redirect_uri specified in auth request.
+        """
+        code = req.POST.get("code")
+        if code is None:
+            json_body = {"error": "invalid_request"}
+            raise OAuthFault(webob.exc.HTTPBadRequest(json_body=json_body))
+
+        client = self._clients.get(code)
+        if client is None:
+            json_body = {"error": "invalid_client"}
+            raise OAuthFault(webob.exc.HTTPBadRequest(json_body=json_body))
+
+        if time.time() - client.auth_start_time > self.AUTH_TIMEOUT:
+            raise webob.exc.HTTPRequestTimeout()
+
+        redirect_uri = req.POST.get("redirect_uri")
+        self._check_redirect_uri(redirect_uri)
+
+        username = req.POST.get("username")
+        password = req.POST.get("password")
+
+        try:
+            keystone = keystone_client.Client(
+                username=username,
+                password=password,
+                auth_url=FLAGS.keystone_gce_url)
+            token = keystone.auth_ref["token"]
+            client.auth_token = token["id"]
+            s = timeutils.parse_isotime(token["issued_at"])
+            e = timeutils.parse_isotime(token["expires"])
+            client.expires_in = (e - s).seconds
+        except Exception as ex:
+            return webob.exc.HTTPUnauthorized(ex)
+
+        if redirect_uri == self.INTERNAL_REDIRECT_URI:
+            return "<html><body>Verification code is: "\
+                + code + "</body></html>"
+
+        uri = redirect_uri + "?code=" + code
+        raise webob.exc.HTTPFound(location=uri)
+
+    def token(self, req):
+        """OAuth protocol authorization endpoint handler second part
+
+        Returns json with tokens(access_token and optionally refresh_token).
+        """
+        client_id = req.POST.get("client_id")
+        if client_id is None or client_id not in self.VALID_CLIENTS:
+            json_body = {"error": "unauthorized_client"}
+            raise OAuthFault(webob.exc.HTTPBadRequest(json_body=json_body))
+        valid_secret = self.VALID_CLIENTS[client_id]
+        client_secret = req.POST.get("client_secret")
+        if client_secret is None or client_secret != valid_secret:
+            json_body = {"error": "unauthorized_client"}
+            raise OAuthFault(webob.exc.HTTPBadRequest(json_body=json_body))
+
+        if req.POST.get("grant_type") != "authorization_code":
+            json_body = {"error": "unsupported_grant_type"}
+            raise OAuthFault(webob.exc.HTTPBadRequest(json_body=json_body))
+
+        code = req.POST.get("code")
+        client = self._clients.get(code)
+        if client is None:
+            json_body = {"error": "invalid_client"}
+            raise OAuthFault(webob.exc.HTTPBadRequest(json_body=json_body))
+
+        result = {"access_token": client.auth_token,
+                  "expires_in": client.expires_in,
+                  "token_type": "Bearer"}
+        return json.dumps(result)
+
+
+class AuthProtocol(object):
+    """Filter for translating oauth token to keystone token."""
+    def __init__(self, app):
+        self.app = app
+        self.keystone_url = FLAGS.keystone_gce_url
+
+    def __call__(self, env, start_response):
+        auth_token = env.get("HTTP_AUTHORIZATION")
+        if auth_token is None:
+            return self._reject_request(start_response)
+
+        project = env["PATH_INFO"].split("/")[1]
+        try:
+            keystone = keystone_client.Client(
+                token=auth_token.split()[1],
+                tenant_name=project,
+                force_new_token=True,
+                auth_url=self.keystone_url)
+            env["HTTP_X_AUTH_TOKEN"] = keystone.auth_ref["token"]["id"]
+            return self.app(env, start_response)
+        except exceptions.Unauthorized:
+            if project in INTERNAL_GCUTIL_PROJECTS:
+                # NOTE(apavlov): return empty if no such projects(by gcutil)
+                headers = [('Content-type', 'application/json;charset=UTF-8')]
+                start_response('200 Ok', headers)
+                return ["{}"]
+
+            return self._reject_request(start_response)
+
+    def _reject_request(self, start_response):
+        headers = [('Content-type', 'application/json;charset=UTF-8')]
+        start_response('401 Unauthorized', headers)
+        json_body = {"error": "access_denied"}
+        return [json.dumps(json_body)]
+
+
+def filter_factory(global_conf, **local_conf):
+    def auth_filter(app):
+        return AuthProtocol(app)
+    return auth_filter
+
+
+def create_resource():
+    return openstack_wsgi.Resource(Controller())

gceapi/api/operation_api.py

@@ -0,0 +1,168 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import uuid
+
+from gceapi.api import base_api
+from gceapi.api import scopes
+from gceapi import exception
+from gceapi.openstack.common.gettextutils import _
+from gceapi.openstack.common import timeutils
+
+
+class API(base_api.API):
+    """GCE operation API."""
+
+    KIND = "operation"
+    PERSISTENT_ATTRIBUTES = ["id", "insert_time", "start_time", "end_time",
+                             "name", "type", "user", "status", "progress",
+                             "scope_type", "scope_name",
+                             "target_type", "target_name",
+                             "method_key", "item_id",
+                             "error_code", "error_message", "errors"]
+
+    def __init__(self, *args, **kwargs):
+        super(API, self).__init__(*args, **kwargs)
+        self._method_keys = {}
+        self._get_progress_methods = {}
+
+    def _get_type(self):
+        return self.KIND
+
+    def _get_persistent_attributes(self):
+        return self.PERSISTENT_ATTRIBUTES
+
+    def register_get_progress_method(self, method_key, method):
+        if method_key in self._get_progress_methods:
+            raise exception.Invalid()
+        # TODO(ft): check 'method' formal arguments
+        self._method_keys[method] = method_key
+        self._get_progress_methods[method_key] = method
+
+    def get_scopes(self, context, item):
+        return [scopes.construct(item["scope_type"], item["scope_name"])]
+
+    def get_item(self, context, name, scope=None):
+        operation = self._get_db_item_by_name(context, name)
+        if (operation is None or
+                operation["scope_type"] != scope.get_type() or
+                operation["scope_name"] != scope.get_name()):
+            raise exception.NotFound
+        operation = self._update_operation_progress(context, operation)
+        return operation
+
+    def get_items(self, context, scope=None):
+        operations = self._get_db_items(context)
+        if scope is not None:
+            operations = [operation for operation in operations
+                          if (operation["scope_type"] == scope.get_type() and
+                              operation["scope_name"] == scope.get_name())]
+        for operation in operations:
+            operation = self._update_operation_progress(context, operation)
+        return operations
+
+    def delete_item(self, context, name, scope=None):
+        # NOTE(ft): Google deletes operation with no check it's scope
+        item = self._get_db_item_by_name(context, name)
+        if item is None:
+            raise exception.NotFound
+        self._delete_db_item(context, item)
+
+    def _update_operation_progress(self, context, operation):
+        if operation["status"] == "DONE" or not operation.get("item_id"):
+            return operation
+        method_key = operation["method_key"]
+        get_progress = self._get_progress_methods[method_key]
+        operation_progress = get_progress(context, operation["item_id"])
+        if operation_progress is None:
+            return operation
+        operation.update(operation_progress)
+        if operation["progress"] == 100:
+            operation["status"] = "DONE"
+            operation["end_time"] = timeutils.isotime(None, True)
+        self._update_db_item(context, operation)
+        return operation
+
+    def construct_operation(self, context, op_type, target_type, target_name,
+                            scope):
+        operation_id = str(uuid.uuid4())
+        operation = {
+            "id": operation_id,
+            "name": "operation-" + operation_id,
+            "insert_time": timeutils.isotime(context.timestamp, True),
+            "user": context.user_name,
+            "type": op_type,
+            "target_type": target_type,
+            "target_name": target_name,
+            "scope_type": scope.get_type(),
+            "scope_name": scope.get_name(),
+        }
+        return operation
+
+    def save_operation(self, context, operation, start_time,
+                       get_progress_method, item_id, operation_result):
+        if isinstance(operation_result, Exception):
+            operation.update(_error_from_exception(operation_result))
+        operation["start_time"] = start_time
+        method_key = self._method_keys.get(get_progress_method)
+        if method_key is None or "error_code" in operation:
+            operation["progress"] = 100
+            operation["status"] = "DONE"
+            operation["end_time"] = timeutils.isotime(None, True)
+        else:
+            operation["progress"] = 0
+            operation["status"] = "RUNNING"
+            operation["method_key"] = method_key
+            if item_id is not None:
+                operation["item_id"] = item_id
+        return self._add_db_item(context, operation)
+
+    def update_operation(self, context, operation_id, operation_result):
+        operation = self._get_db_item_by_id(context, operation_id)
+        if operation is None:
+            # NOTE(ft): it may lead to hungup not finished operation in DB
+            return
+        if isinstance(operation_result, Exception):
+            operation.update(_error_from_exception(operation_result))
+        else:
+            operation.update(operation_result)
+        if operation["progress"] == 100 or "error_code" in operation:
+            operation["status"] = "DONE"
+            operation["end_time"] = timeutils.isotime(None, True)
+        operation.update(operation)
+        self._update_db_item(context, operation)
+
+
+def gef_final_progress(with_error=False):
+    progress = {"progress": 100}
+    if with_error:
+        progress["error_code"] = 500
+        progress["error_message"] = _('Internal server error')
+        progress["errors"] = [{
+           "code": "UNKNOWN_OS_ERROR",
+           "message": _("Operation finished with unknown error. "
+                        "See OpenStack logs.")
+        }]
+    return progress
+
+
+def is_final_progress(progress):
+    return progress is not None and (progress.get("progress") == 100 or
+                                     progress.get("error_code") is not None)
+
+
+def _error_from_exception(ex):
+    return {"errors": [{"code": ex.__class__.__name__, "message": str(ex)}],
+            "error_code": 500,
+            "error_message": _('Internal server error')}

gceapi/api/operation_util.py

@@ -0,0 +1,73 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import threading
+
+from gceapi.api import operation_api
+from gceapi.openstack.common import timeutils
+
+
+def init_operation(context, op_type, target_type, target_name, scope):
+    if context.operation is not None:
+        return
+    operation = operation_api.API().construct_operation(
+            context, op_type, target_type, target_name, scope)
+    context.operation = operation
+    return operation
+
+
+def save_operaton(context, action_result):
+    if context.operation is None or context.operation_start_time is None:
+        return None
+    return operation_api.API().save_operation(
+            context,
+            context.operation,
+            context.operation_start_time,
+            context.operation_get_progress_method,
+            context.operation_item_id,
+            action_result)
+
+
+def start_operation(context, get_progress_method=None, item_id=None):
+    if context.operation is None or context.operation_start_time is not None:
+        return
+    context.operation_start_time = timeutils.isotime(None, True)
+    context.operation_get_progress_method = get_progress_method
+    context.operation_item_id = item_id
+    set_item_id(context, item_id)
+
+
+def set_item_id(context, item_id):
+    if context.operation is None or context.operation_start_time is None:
+        return
+    context.operation_item_id = item_id
+
+
+def continue_operation(context, func, timeout=5):
+    threading.Timer(timeout, _continue_operation, [context, func]).start()
+
+
+def _continue_operation(context, func):
+    operation = context.operation
+    try:
+        operation_result = func()
+    except Exception as ex:
+        operation_result = ex
+    if operation is None:
+        return
+    if operation_result is None:
+        continue_operation(context, func, timeout=2)
+    else:
+        operation_api.API().update_operation(context, operation["id"],
+                                             operation_result)

gceapi/api/operations.py

@@ -0,0 +1,31 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gceapi.api import common as gce_common
+from gceapi.api import operation_api
+from gceapi.api import wsgi as gce_wsgi
+
+
+class Controller(gce_common.Controller):
+    """GCE Route controller"""
+
+    def __init__(self, *args, **kwargs):
+        super(Controller, self).__init__(operation_api.API(), *args, **kwargs)
+
+    def format_item(self, request, operation, scope):
+        return self._format_operation(request, operation, scope)
+
+
+def create_resource():
+    return gce_wsgi.GCEResource(Controller())

gceapi/api/project_api.py

@@ -0,0 +1,103 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gceapi.api import base_api
+from gceapi.api import clients
+from gceapi.api import operation_util
+from gceapi.api import utils
+from gceapi import exception
+
+
+class API(base_api.API):
+    """GCE Projects API."""
+
+    KIND = "project"
+
+    def _get_type(self):
+        return self.KIND
+
+    def get_item(self, context, name, scope=None):
+        project_name = context.project_name
+
+        keystone = clients.keystone(context)
+        project = [t for t in keystone.tenants.list()
+                if t.name == project_name][0]
+
+        result = utils.to_dict(project)
+        result["keypair"] = self._get_gce_keypair(context)
+        project_id = project.id
+
+        nova_limits = clients.nova(context).limits.get(tenant_id=project_id)
+        result["nova_limits"] = dict((l.name, l.value)
+                                     for l in nova_limits.absolute)
+
+        cinder_client = clients.cinder(context)
+        result["cinder_quotas"] = utils.to_dict(
+            cinder_client.quotas.get(project_id, usage=True))
+
+        neutron_client = clients.neutron(context)
+        result["neutron_quota"] = (
+            neutron_client.show_quota(project_id)["quota"])
+        result["neutron_quota"]["network_used"] = len(neutron_client
+            .list_networks(tenant_id=project_id)["networks"])
+        result["neutron_quota"]["floatingip_used"] = len(neutron_client
+            .list_floatingips(tenant_id=project_id)["floatingips"])
+        result["neutron_quota"]["security_group_used"] = len(neutron_client
+            .list_security_groups(tenant_id=project_id)["security_groups"])
+        return result
+
+    def get_items(self, context, scope=None):
+        raise exception.NotFound
+
+    def set_common_instance_metadata(self, context, metadata_list):
+        instance_metadata = dict(
+            [(x['key'], x['value']) for x in metadata_list])
+        operation_util.start_operation(context)
+        ssh_keys = instance_metadata.pop('sshKeys', None)
+        if ssh_keys:
+            nova_client = clients.nova(context)
+            for key_data in ssh_keys.split('\n'):
+                user_name, ssh_key = key_data.split(":")
+                self._update_key(nova_client, user_name, ssh_key)
+
+    def get_gce_user_keypair_name(self, context):
+        client = clients.nova(context)
+        for keypair in client.keypairs.list():
+            if keypair.name == context.user_name:
+                return keypair.name
+
+        return None
+
+    def _get_gce_keypair(self, context):
+        client = clients.nova(context)
+        key_datas = []
+        for keypair in client.keypairs.list():
+            key_datas.append(keypair.name + ':' + keypair.public_key)
+
+        if not key_datas:
+            return None
+
+        return {'key': 'sshKeys', 'value': "\n".join(key_datas)}
+
+    def _update_key(self, nova_client, user_name, ssh_key):
+        try:
+            keypair = nova_client.keypairs.get(user_name)
+            if keypair.public_key == ssh_key:
+                return
+
+            keypair.delete()
+        except clients.novaclient.exceptions.NotFound:
+            pass
+
+        keypair = nova_client.keypairs.create(user_name, ssh_key)

gceapi/api/projects.py

@@ -0,0 +1,104 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import webob
+
+from gceapi import exception
+
+from gceapi.api import common as gce_common
+from gceapi.api import operation_util
+from gceapi.api import project_api
+from gceapi.api import scopes
+from gceapi.api import wsgi as gce_wsgi
+from gceapi.openstack.common.gettextutils import _
+
+
+class Controller(gce_common.Controller):
+    """GCE Projects controller"""
+
+    def __init__(self, *args, **kwargs):
+        super(Controller, self).__init__(project_api.API(), *args, **kwargs)
+
+    def format_item(self, request, project, scope):
+        desc = project["description"]
+        result_dict = {
+            "name": project["name"],
+            "description": desc if desc else "",
+            "commonInstanceMetadata": {
+                "kind": "compute#metadata",
+                "items": [project["keypair"]]
+            } if project["keypair"] else {
+                "kind": "compute#metadata",
+            },
+            "quotas": []
+        }
+
+        self._add_quota(result_dict["quotas"], "CPU",
+            project["nova_limits"].get("maxTotalCores", -1),
+            project["nova_limits"].get("totalCoresUsed", -1))
+        self._add_quota(result_dict["quotas"], "INSTANCES",
+            project["nova_limits"].get("maxTotalInstances", -1),
+            project["nova_limits"].get("totalInstancesUsed", -1))
+
+        quota = project["cinder_quotas"].get("gigabytes", {})
+        self._add_quota(result_dict["quotas"], "DISKS_TOTAL_GB",
+            quota.get("limit", -1), quota.get("in_use", -1))
+        quota = project["cinder_quotas"].get("snapshots", {})
+        self._add_quota(result_dict["quotas"], "SNAPSHOTS",
+            quota.get("limit", -1), quota.get("in_use", -1))
+        quota = project["cinder_quotas"].get("volumes", {})
+        self._add_quota(result_dict["quotas"], "DISKS",
+            quota.get("limit", -1), quota.get("in_use", -1))
+
+        self._add_quota(result_dict["quotas"], "FIREWALLS",
+            project["neutron_quota"].get("security_group", -1),
+            project["neutron_quota"].get("security_group_used", -1))
+        self._add_quota(result_dict["quotas"], "STATIC_ADDRESSES",
+            project["neutron_quota"].get("floatingip", -1),
+            project["neutron_quota"].get("floatingip_used", -1))
+        self._add_quota(result_dict["quotas"], "NETWORKS",
+            project["neutron_quota"].get("network", -1),
+            project["neutron_quota"].get("network_used", -1))
+
+        return self._format_item(request, result_dict, scope)
+
+    def set_common_instance_metadata(self, req, body):
+        context = self._get_context(req)
+        operation_util.init_operation(context, "setMetadata", self._type_name,
+                                      None, scopes.GlobalScope())
+        try:
+            self._api.set_common_instance_metadata(
+                context, body.get("items", []))
+        except exception.KeypairLimitExceeded:
+            msg = _("Quota exceeded, too many key pairs.")
+            raise webob.exc.HTTPRequestEntityTooLarge(
+                        explanation=msg,
+                        headers={'Retry-After': 0})
+        except exception.InvalidKeypair:
+            msg = _("Keypair data is invalid")
+            raise webob.exc.HTTPBadRequest(explanation=msg)
+        except exception.KeyPairExists:
+            msg = _("Key pair already exists.")
+            raise webob.exc.HTTPConflict(explanation=msg)
+
+    def _add_quota(self, quotas, metric, limit, usage):
+        quotas.append({
+            "metric": metric,
+            "limit": float(limit),
+            "usage": float(usage),
+        })
+
+
+def create_resource():
+    return gce_wsgi.GCEResource(Controller())

gceapi/api/region_api.py

@@ -0,0 +1,43 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gceapi.api import base_api
+from gceapi.api import scopes
+from gceapi import exception
+
+
+class API(base_api.API):
+    """GCE Regions API
+
+    Stubbed now for support only one predefined region nova
+    """
+
+    KIND = "region"
+    _REGIONS = ["nova"]
+
+    def _get_type(self):
+        return self.KIND
+
+    def get_item(self, context, name, scope=None):
+        regions = self.get_items(context)
+        for region in regions:
+            if region["name"] == name:
+                return region
+        raise exception.NotFound
+
+    def get_items(self, context, scope=None):
+        return [dict(("name", region) for region in self._REGIONS)]
+
+    def get_items_as_scopes(self, context):
+        return [scopes.RegionScope(region) for region in self._REGIONS]

gceapi/api/regions.py

@@ -0,0 +1,41 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gceapi.api import common as gce_common
+from gceapi.api import region_api
+from gceapi.api import wsgi as gce_wsgi
+from gceapi.api import zone_api
+
+
+class Controller(gce_common.Controller):
+    """GCE Regions controller."""
+
+    def __init__(self, *args, **kwargs):
+        super(Controller, self).__init__(region_api.API(), *args, **kwargs)
+        self._zone_api = zone_api.API()
+
+    def format_item(self, req, region, scope):
+        zones = self._zone_api.get_items(self._get_context(req), scope)
+        result_dict = {
+            "name": region["name"],
+            "status": "UP",
+            "zones": [self._qualify(req, "zones", zone["name"], None)
+                      for zone in zones]
+        }
+
+        return self._format_item(req, result_dict, scope)
+
+
+def create_resource():
+    return gce_wsgi.GCEResource(Controller())

gceapi/api/route_api.py

@@ -0,0 +1,26 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gceapi.api import base_api
+from gceapi.api import route_neutron_api
+from gceapi.api import route_nova_api
+
+
+class API(base_api.API):
+    """GCE Route API."""
+
+    NEUTRON_API_MODULE = route_neutron_api
+    NOVA_API_MODULE = route_nova_api
+
+    __metaclass__ = base_api.NetSingleton

gceapi/api/route_neutron_api.py

@@ -0,0 +1,409 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import netaddr
+import string
+
+from gceapi.api import base_api
+from gceapi.api import clients
+from gceapi.api import network_api
+from gceapi.api import operation_util
+from gceapi.api import utils
+from gceapi import exception
+from gceapi.openstack.common.gettextutils import _
+
+
+ALL_IP_CIDR = "0.0.0.0/0"
+
+
+class API(base_api.API):
+    """GCE Address API - neutron implementation."""
+
+    KIND = "route"
+    PERSISTENT_ATTRIBUTES = ["id", "creationTimestamp", "description",
+                             "is_default"]
+    TRANS_TABLE = string.maketrans("./", "--")
+
+    def __init__(self, *args, **kwargs):
+        super(API, self).__init__(*args, **kwargs)
+        network_api.API()._register_callback(
+            base_api._callback_reasons.post_add,
+            self._create_network_router)
+        network_api.API()._register_callback(
+            base_api._callback_reasons.check_delete,
+            self._check_delete_network)
+        network_api.API()._register_callback(
+            base_api._callback_reasons.pre_delete,
+            self._delete_network_router)
+
+    def _get_type(self):
+        return self.KIND
+
+    def _get_persistent_attributes(self):
+        return self.PERSISTENT_ATTRIBUTES
+
+    def get_item(self, context, name, scope=None):
+        routes, dummy = self._sync_routes(context)
+        return routes[name]
+
+    def get_items(self, context, scope=None):
+        routes, dummy = self._sync_routes(context)
+        return routes.values()
+
+    def delete_item(self, context, name, scope=None):
+        routes, aliased_routes = self._sync_routes(context)
+        route = routes[name]
+        if route.get("nexthop") is None:
+            raise exception.InvalidInput(
+                    _("The local route cannot be deleted."))
+        destination = route["destination"]
+        nexthop = route["nexthop"]
+        # NOTE(ft): delete OS route only if it doesn't have aliases
+        # at the moment
+        client = clients.neutron(context)
+        operation_util.start_operation(context)
+        if self._get_route_key(route) not in aliased_routes:
+            dummy, router = self._get_network_objects(client,
+                                                      route["network"])
+            if "external_gateway_info" in route:
+                client.remove_gateway_router(router["id"])
+            else:
+                routes = [r for r in router["routes"]
+                          if (destination != r["destination"] or
+                              nexthop != r["nexthop"])]
+                client.update_router(
+                        router["id"],
+                        {"router": {"routes": routes, }, })
+        self._delete_db_item(context, route)
+
+    def add_item(self, context, name, body, scope=None):
+        routes, dummy = self._sync_routes(context)
+        if name in routes:
+            raise exception.InvalidInput(
+                    _("The resource '%s' already exists.") % name)
+
+        # NOTE(ft): check network is plugged to router
+        network_name = utils._extract_name_from_url(body["network"])
+        network = network_api.API().get_item(context, network_name)
+
+        nexthop = body.get("nextHopGateway")
+        if (nexthop is not None and
+                (utils._extract_name_from_url(nexthop) ==
+                 "default-internet-gateway") and
+                # NOTE(ft): OS doesn't support IP mask for external gateway
+                body.get("destRange") == ALL_IP_CIDR):
+            operation_util.start_operation(context)
+            return self._create_internet_route(context, network, body)
+
+        nexthop = body.get("nextHopIp")
+        if nexthop is not None:
+            operation_util.start_operation(context)
+            return self._create_custom_route(context, network, body)
+
+        raise exception.InvalidInput(_("Unsupported route."))
+
+    def _create_internet_route(self, context, network, body):
+        client = clients.neutron(context)
+        port, router = self._get_network_objects(client, network)
+        public_network_id = network_api.API().get_public_network_id(context)
+        external_gateway_info = {"network_id": public_network_id}
+        router = client.add_gateway_router(
+                router["id"],
+                external_gateway_info)["router"]
+        gateway_port = client.list_ports(
+                device_id=router["id"],
+                device_owner="network:router_gateway")["ports"][0]
+        route = self._add_gce_route(context, network, port, body,
+                                   is_default=False,
+                                   destination=gateway_port["id"],
+                                   nexthop=ALL_IP_CIDR)
+        route["network"] = network
+        route["port"] = port
+        route["external_gateway_info"] = external_gateway_info
+        return route
+
+    def _create_custom_route(self, context, network, body):
+        client = clients.neutron(context)
+        port, router = self._get_network_objects(client, network)
+        destination = body.get("destRange")
+        nexthop = body.get("nextHopIp")
+        routes = router["routes"]
+        if all(r["destination"] != destination or r["nexthop"] != nexthop
+               for r in routes):
+            routes.append({
+                    "destination": destination,
+                    "nexthop": nexthop,
+                })
+            client.update_router(
+                    router["id"],
+                    {"router": {"routes": router["routes"], }, })
+        route = self._add_gce_route(context, network, port, body,
+                                   is_default=False, destination=destination,
+                                   nexthop=nexthop)
+        route["network"] = network
+        route["port"] = port
+        return route
+
+    def _sync_routes(self, context):
+        os_routes = self._get_os_routes(context)
+        gce_routes = self._get_gce_routes(context)
+        aliased_routes = {}
+        routes = {}
+        for (key, os_route) in os_routes.items():
+            gce_route_list = gce_routes.pop(key, None)
+            if gce_route_list is None:
+                continue
+            for gce_route in gce_route_list:
+                routes[gce_route["name"]] = dict(os_route, **dict(gce_route))
+            os_routes.pop(key)
+            if len(gce_route_list) > 1:
+                aliased_routes[key] = gce_route_list
+
+        # NOTE(ft): add new named routes
+        for os_route in os_routes.itervalues():
+            network = os_route["network"]
+            port = os_route["port"]
+            route = self._add_gce_route(context, network, port, os_route,
+                                       is_default=True,
+                                       creationTimestamp="")
+            os_route.update(route)
+            routes[os_route["name"]] = os_route
+
+        # NOTE(ft): delete obsolete named routes
+        for gce_route_list in gce_routes.itervalues():
+            for gce_route in gce_route_list:
+                self._delete_db_item(context, gce_route)
+        return (routes, aliased_routes)
+
+    def _get_gce_routes(self, context):
+        gce_routes = self._get_db_items(context)
+        gce_routes_dict = {}
+        for route in gce_routes:
+            route = self._unpack_route_from_db_format(route)
+            key = self._get_route_key(route)
+            val_array = gce_routes_dict.get(key)
+            if val_array is None:
+                gce_routes_dict[key] = [route]
+            else:
+                val_array.append(route)
+        return gce_routes_dict
+
+    def _get_route_key(self, route):
+        if route["port_id"] is None:
+            return route["network_id"]
+        else:
+            return (route["network_id"] + route["port_id"] +
+                    route["destination"] + route["nexthop"])
+
+    def _get_os_routes(self, context):
+        client = clients.neutron(context)
+        routers = client.list_routers(tenant_id=context.project_id)["routers"]
+        routers = dict((r["id"], r) for r in routers)
+        ports = client.list_ports(
+                tenant_id=context.project_id,
+                device_owner="network:router_interface")["ports"]
+        ports = dict((p["network_id"], p) for p in ports)
+        gateway_ports = client.list_ports(
+                device_owner="network:router_gateway")["ports"]
+        gateway_ports = dict((p["device_id"], p) for p in gateway_ports)
+        routes = {}
+        networks = network_api.API().get_items(context)
+        for network in networks:
+            # NOTE(ft): append local route
+            network_id = network["id"]
+            routes[network_id] = self._init_local_route(network)
+
+            port = ports.get(network_id)
+            if port is None:
+                continue
+            router = routers.get(port["device_id"])
+            if router is None:
+                continue
+            key_prefix = network_id + port["id"]
+
+            # NOTE(ft): append internet route
+            external_gateway_info = router.get("external_gateway_info")
+            gateway_port = gateway_ports.get(router["id"])
+            if (external_gateway_info is not None and
+                    gateway_port is not None):
+                key = key_prefix + ALL_IP_CIDR + gateway_port["id"]
+                routes[key] = self._init_internet_route(
+                        network, port, gateway_port["id"],
+                        external_gateway_info)
+
+            # NOTE(ft): append other routes
+            for route in router["routes"]:
+                destination = route["destination"]
+                nexthop = route["nexthop"]
+                key = key_prefix + destination + nexthop
+                routes[key] = self._init_custom_route(
+                        network, port, destination, nexthop)
+        return routes
+
+    def _get_network_objects(self, client, network):
+        subnet_id = network.get("subnet_id")
+        if subnet_id is None:
+            raise exception.PortNotFound(_("Network has no router."))
+        ports = client.list_ports(
+                network_id=network["id"],
+                device_owner="network:router_interface")["ports"]
+        port = next((p for p in ports
+                     if any(fip["subnet_id"] == subnet_id
+                            for fip in p["fixed_ips"])), None)
+        if port is None:
+            raise exception.PortNotFound(_("Network has no router."))
+        router = client.show_router(port["device_id"])["router"]
+        return (port, router)
+
+    def _create_network_router(self, context, network, subnet_id):
+        public_network_id = network_api.API().get_public_network_id(context)
+        client = clients.neutron(context)
+        router = client.create_router(body={"router": {
+            "name": network["name"],
+            "admin_state_up": True,
+            "external_gateway_info": {"network_id": public_network_id},
+        }})["router"]
+        client.add_interface_router(router["id"], {"subnet_id": subnet_id})
+
+    def _check_delete_network(self, context, network):
+        network_id = network["id"]
+        # NOTE(ft): check non default routes not longer exists
+        # must be done for internet routes
+        routes, dummy = self._sync_routes(context)
+        for route in routes.itervalues():
+            if (route["network_id"] == network_id and
+                    not route["is_default"]):
+                raise exception.InvalidInput(_("Network contains routes"))
+        # NOTE(ft): check invisible routes not longer exists
+        # must be done for routes on non default subnet and other non GCE stuff
+        client = clients.neutron(context)
+        checked_routers = set()
+        subnets = client.list_subnets(network_id=network_id)["subnets"]
+        cidrs = [netaddr.IPNetwork(subnet["cidr"]) for subnet in subnets]
+        ports = client.list_ports(
+                network_id=network["id"],
+                device_owner="network:router_interface")["ports"]
+        for port in ports:
+            if port["device_id"] in checked_routers:
+                continue
+            checked_routers.add(port["device_id"])
+            router = client.show_router(port["device_id"])["router"]
+            for route in router["routes"]:
+                nexthop = netaddr.IPAddress(route["nexthop"])
+                if any(nexthop in cidr for cidr in cidrs):
+                    raise exception.InvalidInput(_("Network contains routes"))
+        # TODO(ft): here is the good place to create default routes in DB
+        # now thew will be created on next 'route' request,
+        # but 'creationTimestamp' will be absent
+
+    def _delete_network_router(self, context, network):
+        client = clients.neutron(context)
+        ports = client.list_ports(
+                network_id=network["id"],
+                device_owner="network:router_interface")["ports"]
+        router_ids = set()
+        for port in ports:
+            if port["device_owner"] == "network:router_interface":
+                router_ids.add(port["device_id"])
+                client.remove_interface_router(port["device_id"],
+                                               {"port_id": port["id"]})
+        # NOTE(ft): leave routers if network is plugged to more than one route
+        # because it's look like some non GCE settings, so we don't want
+        # to decide whether we can delete router or not
+        if len(router_ids) != 1:
+            return
+        router = router_ids.pop()
+        # NOTE(ft): leave router if other subnets are plugged to it
+        ports = client.list_ports(
+                device_id=router,
+                device_owner="network:router_interface")["ports"]
+        if len(ports) == 0:
+            client.delete_router(router)
+        # TODO(ft): here is the good place to purge DB from routes
+
+    def _add_gce_route(self, context, network, port, route, **kwargs):
+        db_route = {}
+        for key in self.PERSISTENT_ATTRIBUTES:
+            value = route.get(key)
+            if value is None:
+                value = kwargs.get(key)
+            if value is not None or key in kwargs:
+                db_route[key] = value
+
+        def get_from_dicts(key, dict1, dict2, default=None):
+            value = dict1.get(key)
+            if value is None:
+                value = dict2.get(key)
+            return value if value is not None else default
+
+        route_id = "//".join([network["id"],
+                              port["id"] if port is not None else "",
+                              get_from_dicts("destination", route, kwargs),
+                              get_from_dicts("nexthop", route, kwargs, ""),
+                              get_from_dicts("name", route, kwargs)])
+        db_route["id"] = route_id
+        db_route = self._add_db_item(context, db_route)
+        return self._unpack_route_from_db_format(db_route)
+
+    def _unpack_route_from_db_format(self, route):
+        parts = route["id"].split("//")
+        route["network_id"] = parts[0]
+        route["port_id"] = parts[1] if parts[1] != "" else None
+        route["destination"] = parts[2]
+        route["nexthop"] = parts[3] if parts[3] != "" else None
+        route["name"] = parts[4]
+        return route
+
+    def _init_local_route(self, network):
+        return {
+            "id": None,
+            "name": "default-route-%s-local" % network["id"],
+            "description": "Default route to the virtual network.",
+            "network": network,
+            "port": None,
+            "destination": network.get("IPv4Range", ""),
+            "nexthop": None,
+            "is_default": True,
+        }
+
+    def _init_internet_route(self, network, port, nexthop, gateway_info):
+        return {
+            "id": None,
+            "name": "default-route-%s-internet" % network["id"],
+            "description": "Default route to the Internet.",
+            "network": network,
+            "port": port,
+            "destination": ALL_IP_CIDR,
+            "nexthop": nexthop,
+            "is_default": True,
+            "external_gateway_info": gateway_info,
+        }
+
+    def _init_custom_route(self, network, port, destination, nexthop):
+        name = ("custom-route-%(nw)s-dst-%(dst)s-gw-%(nh)s" %
+                {
+                    "nw": network["id"],
+                    "dst": destination,
+                    "nh": nexthop,
+                })
+        name = str(name).translate(self.TRANS_TABLE)
+        return {
+            "id": None,
+            "name": name,
+            "network": network,
+            "port": port,
+            "destination": destination,
+            "nexthop": nexthop,
+            "is_default": False,
+        }

gceapi/api/route_nova_api.py

@@ -0,0 +1,41 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gceapi.api import base_api
+from gceapi import exception
+from gceapi.openstack.common.gettextutils import _
+
+
+NOT_SUPPORTED_MESSAGE = _("Routes are not supported with nova network")
+
+
+class API(base_api.API):
+    """GCE Address API - nova-network implementation."""
+
+    KIND = "route"
+
+    def _get_type(self):
+        return self.KIND
+
+    def get_item(self, context, name, scope_id=None):
+        raise exception.InvalidInput(reason=NOT_SUPPORTED_MESSAGE)
+
+    def get_items(self, context, scope_id=None):
+        raise exception.InvalidInput(reason=NOT_SUPPORTED_MESSAGE)
+
+    def delete_item(self, context, name, scope_id=None):
+        raise exception.InvalidInput(reason=NOT_SUPPORTED_MESSAGE)
+
+    def add_item(self, context, name, body, scope_id=None):
+        raise exception.InvalidInput(reason=NOT_SUPPORTED_MESSAGE)

gceapi/api/routes.py

@@ -0,0 +1,52 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gceapi.api import common as gce_common
+from gceapi.api import route_api
+from gceapi.api import wsgi as gce_wsgi
+
+
+class Controller(gce_common.Controller):
+    """GCE Route controller"""
+
+    def __init__(self, *args, **kwargs):
+        super(Controller, self).__init__(route_api.API(), *args, **kwargs)
+
+    def format_item(self, request, route, scope):
+        network_name = self._qualify(
+                request, "networks", route["network"]["name"], None)
+        result_dict = {
+            "name": route["name"],
+            "network": network_name,
+            "destRange": route.get("destination"),
+            "creationTimestamp": route.get("creationTimestamp", ""),
+            "priority": 1000,
+        }
+        if "external_gateway_info" in route:
+            result_dict["nextHopGateway"] = self._qualify(
+                    request, "gateways", "default-internet-gateway", scope)
+        else:
+            nextHop = route.get("nexthop")
+            if nextHop is not None:
+                result_dict["nextHopIp"] = nextHop
+            else:
+                result_dict["nextHopNetwork"] = network_name
+        if "description" in route:
+            result_dict["description"] = route["description"]
+
+        return self._format_item(request, result_dict, scope)
+
+
+def create_resource():
+    return gce_wsgi.GCEResource(Controller())

gceapi/api/scopes.py

@@ -0,0 +1,120 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import abc
+
+from webob import exc
+
+from gceapi.api import base_api
+from gceapi.api import utils
+
+
+class Scope(object):
+    """Scope that contains resource.
+
+    The following scopes exists: global, aggregated, zones, regions.
+    """
+
+    _type = None
+    _collection = None
+    _name = None
+
+    @abc.abstractmethod
+    def __init__(self, scope_name):
+        self._name = scope_name
+
+    def get_type(self):
+        return self._type
+
+    def get_name(self):
+        return self._name
+
+    def get_collection(self):
+        return self._collection
+
+    def get_path(self):
+        if self._collection is not None and self._name is not None:
+            return "/".join([self._collection, self._name])
+        else:
+            return self._type
+
+    def get_scope_api(self):
+        base_api.Singleton.get_instance(self.get_type())
+
+
+class GlobalScope(Scope):
+
+    _type = "global"
+
+    def __init__(self):
+        super(GlobalScope, self).__init__(None)
+
+
+class AggregatedScope(Scope):
+
+    _type = "aggregated"
+
+    def __init__(self):
+        super(AggregatedScope, self).__init__(None)
+
+
+class ZoneScope(Scope):
+
+    _type = "zone"
+    _collection = utils.get_collection_name(_type)
+
+    def __init__(self, scope_name):
+        super(ZoneScope, self).__init__(scope_name)
+
+
+class RegionScope(Scope):
+
+    _type = "region"
+    _collection = utils.get_collection_name(_type)
+
+    def __init__(self, scope_name):
+        super(RegionScope, self).__init__(scope_name)
+
+
+def construct(scope_type, scope_id):
+    if scope_type == "zone":
+        return ZoneScope(scope_id)
+    elif scope_type == "region":
+        return RegionScope(scope_id)
+    elif scope_type == "global":
+        return GlobalScope()
+    elif scope_type == "aggregated":
+        return AggregatedScope()
+    return None
+
+
+def construct_from_path(path, scope_id):
+    path_info = [item for item in path.split("/") if item]
+    path_count = len(path_info)
+    if path_count == 0:
+        raise exc.HTTPBadRequest(comment="Bad path %s" % path)
+    if path_count < 3:
+        return None
+    collection_or_type = path_info[1]
+    if collection_or_type in ("zones", "regions") and scope_id is None:
+        return None
+    if collection_or_type == "zones":
+        return ZoneScope(scope_id)
+    elif collection_or_type == "regions":
+        return RegionScope(scope_id)
+    elif collection_or_type == "global":
+        return GlobalScope()
+    elif collection_or_type == "aggregated":
+        return AggregatedScope()
+    raise exc.HTTPBadRequest(comment="Bad path %s" % path)

gceapi/api/snapshot_api.py

@@ -0,0 +1,113 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gceapi.api import base_api
+from gceapi.api import clients
+from gceapi.api import operation_api
+from gceapi.api import operation_util
+from gceapi.api import utils
+from gceapi import exception
+
+
+class API(base_api.API):
+    """GCE Snapshot API."""
+
+    KIND = "snapshot"
+    _status_map = {
+        'creating': 'CREATING',
+        'available': 'READY',
+        'active': 'READY',
+        'deleting': 'DELETING',
+        'deleted': 'DELETING',
+        'error': 'FAILED'}
+
+    def __init__(self, *args, **kwargs):
+        super(API, self).__init__(*args, **kwargs)
+        operation_api.API().register_get_progress_method(
+                "snapshot-add",
+                self._get_add_item_progress)
+        operation_api.API().register_get_progress_method(
+                "snapshot-delete",
+                self._get_delete_item_progress)
+
+    def _get_type(self):
+        return self.KIND
+
+    def get_item(self, context, name, scope=None):
+        client = clients.cinder(context)
+        snapshots = client.volume_snapshots.list(
+            search_opts={"display_name": name})
+        if snapshots and len(snapshots) == 1:
+            return self._prepare_item(client, utils.to_dict(snapshots[0]))
+        raise exception.NotFound
+
+    def get_items(self, context, scope=None):
+        client = clients.cinder(context)
+        snapshots = [utils.to_dict(item)
+                     for item in client.volume_snapshots.list()]
+        for snapshot in snapshots:
+            self._prepare_item(client, snapshot)
+        return snapshots
+
+    def delete_item(self, context, name, scope=None):
+        client = clients.cinder(context).volume_snapshots
+        snapshots = client.list(search_opts={"display_name": name})
+        if not snapshots or len(snapshots) != 1:
+            raise exception.NotFound
+        operation_util.start_operation(context,
+                                       self._get_delete_item_progress,
+                                       snapshots[0].id)
+        client.delete(snapshots[0])
+
+    def add_item(self, context, body, scope=None):
+        name = body["name"]
+        disk_name = body["disk_name"]
+        client = clients.cinder(context)
+        volumes = client.volumes.list(search_opts={"display_name": disk_name})
+        if not volumes or len(volumes) != 1:
+            raise exception.NotFound
+
+        operation_util.start_operation(context, self._get_add_item_progress)
+        snapshot = client.volume_snapshots.create(
+            volumes[0].id, True, name, body["description"])
+        operation_util.set_item_id(context, snapshot.id)
+
+        return self._prepare_item(client, utils.to_dict(snapshot))
+
+    def _prepare_item(self, client, item):
+        item["name"] = item["display_name"]
+        try:
+            item["disk"] = utils.to_dict(client.volumes.get(item["volume_id"]))
+        except Exception:
+            pass
+        item["status"] = self._status_map.get(item["status"], item["status"])
+        return item
+
+    def _get_add_item_progress(self, context, snapshot_id):
+        client = clients.cinder(context)
+        try:
+            snapshot = client.volume_snapshots.get(snapshot_id)
+        except clients.cinderclient.exceptions.NotFound:
+            return operation_api.gef_final_progress()
+        if (snapshot.status != "creating"):
+            return operation_api.gef_final_progress(snapshot.status == "error")
+
+    def _get_delete_item_progress(self, context, snapshot_id):
+        client = clients.cinder(context)
+        try:
+            snapshot = client.volume_snapshots.get(snapshot_id)
+        except clients.cinderclient.exceptions.NotFound:
+            return operation_api.gef_final_progress()
+        if snapshot.status not in ["deleting", "deleted"]:
+            return operation_api.gef_final_progress(True)

gceapi/api/snapshots.py

@@ -0,0 +1,51 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gceapi.api import common as gce_common
+from gceapi.api import scopes
+from gceapi.api import snapshot_api
+from gceapi.api import wsgi as gce_wsgi
+from gceapi import exception
+
+
+class Controller(gce_common.Controller):
+    """GCE Snapshot controller"""
+
+    def __init__(self, *args, **kwargs):
+        super(Controller, self).__init__(snapshot_api.API(), *args, **kwargs)
+
+    def format_item(self, request, snapshot, scope):
+        result_dict = {
+            "creationTimestamp": self._format_date(snapshot["created_at"]),
+            "status": snapshot["status"],
+            "diskSizeGb": snapshot["size"],
+            "name": snapshot["name"],
+            "description": snapshot["display_description"],
+            }
+        disk = snapshot.get("disk")
+        if disk is not None:
+            result_dict["sourceDisk"] = self._qualify(
+                request, "disks", disk["display_name"],
+                scopes.ZoneScope(disk["availability_zone"]))
+            result_dict["sourceDiskId"] = self._get_id(
+                result_dict["sourceDisk"])
+
+        return self._format_item(request, result_dict, scope)
+
+    def create(self, req, body, scope):
+        raise exception.NotFound
+
+
+def create_resource():
+    return gce_wsgi.GCEResource(Controller())

gceapi/api/utils.py

@@ -0,0 +1,160 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Utilities and helper functions."""
+
+
+def _parse_slash(string):
+    res = ''
+    sp = string.split('/')
+    for element in reversed(sp):
+        res = {element: res}
+    return res
+
+
+def split_by_comma(string):
+    between = 0
+    last_split = 0
+    sp = []
+
+    i = 0
+    while i < len(string):
+        if string[i] == '(':
+            between += 1
+        elif string[i] == ')':
+            between -= 1
+        elif string[i] == ',' and not between:
+            sp.append(string[last_split:i])
+            last_split = i + 1
+        i += 1
+    sp.append(string[last_split:])
+    return sp
+
+
+def _parse_template(string):
+    sp = split_by_comma(string)
+
+    i = 0
+    while i < len(sp):
+        if '(' in sp[i]:
+            sp[i] = sp[i].replace('(', ' ').replace(')', ' ').split()
+        i += 1
+
+    json = {}
+    i = 0
+    while i < len(sp):
+        if isinstance(sp[i], list):
+            fields = sp[i][1].split(',')
+            json[sp[i][0]] = [{}]
+            for field in fields:
+                dct = _parse_slash(field)
+                key = dct.keys()[0]
+                json[sp[i][0]][0][key] = dct[key]
+        else:
+            field = _parse_slash(sp[i])
+            key = field.keys()[0]
+            json[key] = field[key]
+        i += 1
+
+    return json
+
+
+def apply_template(template_string, json):
+
+    def apply_recursive(template, json):
+        res = {}
+        if template == '':
+            return json
+        for key, val in template.items():
+            if key in json and val == '':
+                res[key] = json[key]
+            elif key in json and val == '*':
+                pass
+            elif key in json and isinstance(val, list):
+                if not isinstance(json[key], list):
+                    raise ValueError()
+                array = []
+                for element in json[key]:
+                    r = apply_recursive(val[0], element)
+                    array.append(r)
+                res[key] = array
+            elif key in json and isinstance(val, dict):
+                r = apply_recursive(val, json[key])
+                res[key] = r
+            elif key not in json and key == '*':
+                for k, v in json.items():
+                    try:
+                        r = apply_recursive(val, v)
+                    except ValueError:
+                        continue
+                    res[k] = r
+            elif key not in json:
+                raise ValueError()
+        return res
+
+    return apply_recursive(_parse_template(template_string), json)
+
+
+def to_dict(obj, recursive=False, classkey=None):
+    if hasattr(obj, "__dict__"):
+        data = dict()
+        for key in dir(obj):
+            try:
+                value = getattr(obj, key)
+                if not callable(value) and not key.startswith('_'):
+                    data[key] = (value if not recursive
+                        else to_dict(value, recursive, classkey))
+            except AttributeError:
+                pass
+        if classkey is not None and hasattr(obj, "__class__"):
+            data[classkey] = obj.__class__.__name__
+        return data
+
+    if not recursive:
+        return obj
+
+    if isinstance(obj, dict):
+        for k in obj.keys():
+            obj[k] = to_dict(obj[k], recursive, classkey)
+        return obj
+    elif hasattr(obj, "__iter__"):
+        return [to_dict(v, recursive, classkey) for v in obj]
+
+    return obj
+
+
+def _extract_name_from_url(url):
+    """Get object name from fully qualified link."""
+    return url.split('/')[-1]
+
+
+def get_collection_name(type_name):
+    if type_name == "project":
+        return None
+    elif type_name.endswith("s"):
+        return "%ses" % type_name
+    else:
+        return "%ss" % type_name
+
+
+def get_type_kind(type_name):
+    return "compute#%s" % type_name
+
+
+def get_list_kind(type_name):
+    return "compute#%sList" % type_name
+
+
+def get_aggregated_kind(type_name):
+    return "compute#%sAggregatedList" % type_name

gceapi/api/wsgi.py

@@ -0,0 +1,223 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import webob
+
+from gceapi import exception
+from gceapi.openstack.common.gettextutils import _
+from gceapi.openstack.common import jsonutils
+from gceapi.openstack.common import log as logging
+from gceapi import wsgi_ext as openstack_wsgi
+
+LOG = logging.getLogger(__name__)
+
+
+class JSONDictSerializer(openstack_wsgi.DictSerializer):
+    """JSON request body serialization."""
+
+    def serialize(self, data, request):
+        params = {'false': False, 'true': True}
+        pretty_print = request.params.get("prettyPrint", True)
+        if pretty_print in params:
+            pretty_print = params[pretty_print]
+        ident = None
+        if pretty_print:
+            ident = 4
+        ret = jsonutils.dumps(data,
+                default=jsonutils.to_primitive, indent=ident)
+        return ret
+
+
+class GCEResponse(openstack_wsgi.ResponseObject):
+    """GCE Response body serialization."""
+
+    def serialize(self, request, content_type, default_serializers=None):
+        if self.serializer:
+            serializer = self.serializer
+        else:
+            _mtype, _serializer = self.get_serializer(content_type,
+                                                      default_serializers)
+            serializer = _serializer()
+
+        response = webob.Response()
+        response.status_int = self.code
+        for hdr, value in self._headers.items():
+            response.headers[hdr] = value
+        response.headers['Content-Type'] = content_type
+        if self.obj is not None:
+            response.body = serializer.serialize(self.obj, request)
+
+        return response
+
+
+class GCEFault(webob.exc.HTTPException):
+    """Wrap webob.exc.HTTPException to provide API friendly response."""
+
+    def __init__(self, exception):
+        """
+        Create a Fault for the given webob.exc.exception or gceapi.exception.
+        """
+        self.wrapped_exc = exception
+        for key, value in self.wrapped_exc.headers.items():
+            self.wrapped_exc.headers[key] = str(value)
+
+
+class GCEResourceExceptionHandler(object):
+    """Context manager to handle Resource exceptions.
+
+    Used when processing exceptions generated by API implementation
+    methods (or their extensions).  Converts most exceptions to Fault
+    exceptions, with the appropriate logging.
+    """
+
+    def __enter__(self):
+        return None
+
+    def __exit__(self, ex_type, ex_value, ex_traceback):
+        if not ex_value:
+            return True
+
+        if isinstance(ex_value, exception.NotAuthorized):
+            msg = unicode(ex_value)
+            raise GCEFault(webob.exc.HTTPForbidden(explanation=msg))
+        elif isinstance(ex_value, exception.Invalid):
+            msg = unicode(ex_value)
+            raise GCEFault(exception.ConvertedException(
+                    code=ex_value.code, explanation=msg))
+
+        # Under python 2.6, TypeError's exception value is actually a string,
+        # so test # here via ex_type instead:
+        # http://bugs.python.org/issue7853
+        elif issubclass(ex_type, TypeError):
+            exc_info = (ex_type, ex_value, ex_traceback)
+            LOG.error(_('Exception handling resource: %s') % ex_value,
+                    exc_info=exc_info)
+            raise GCEFault(webob.exc.HTTPBadRequest())
+        elif isinstance(ex_value, GCEFault):
+            LOG.info(_("Fault thrown: %s"), unicode(ex_value))
+            raise ex_value
+        elif isinstance(ex_value, webob.exc.HTTPException):
+            LOG.info(_("HTTP exception thrown: %s"), unicode(ex_value))
+            raise GCEFault(ex_value)
+        elif isinstance(ex_value, exception.GceapiException):
+            LOG.info(_("Gceapi exception thrown: %s"), unicode(ex_value))
+            raise GCEFault(ex_value)
+        else:
+            msg = unicode(ex_value)
+            raise GCEFault(exception.ConvertedException(
+                    code=500, title=ex_type.__name__, explanation=msg))
+
+
+class GCEResource(openstack_wsgi.Resource):
+    """Common GCE resource response formatter"""
+
+    def __init__(self, *args, **kwargs):
+        super(GCEResource, self).__init__(*args, **kwargs)
+        self.default_serializers = dict(json=JSONDictSerializer)
+
+    def _check_requested_project(self, project_id, context):
+        if (not context or project_id is None
+        or (project_id not in [context.project_id, context.project_name])):
+            msg = _("Project '%s' could not be found") % project_id \
+                if project_id is not None \
+                else _("Project hasn`t been provided")
+
+            raise GCEFault(webob.exc.HTTPBadRequest(
+                explanation=msg))
+
+    def _process_stack(self, request, action, action_args,
+                       content_type, body, accept):
+        """Implement the processing stack."""
+        method = None
+        try:
+            # Get the implementing method
+            try:
+                method = self.get_method(request, action, content_type, body)
+            except (AttributeError, TypeError):
+                msg = _("There is no such action: %s") % action
+                raise GCEFault(webob.exc.HTTPNotFound(
+                    explanation=msg))
+            except KeyError as ex:
+                msg = _("There is no such action: %s") % ex.args[0]
+                raise GCEFault(webob.exc.HTTPBadRequest(
+                    explanation=msg))
+            except exception.MalformedRequestBody:
+                msg = _("Malformed request body")
+                raise GCEFault(webob.exc.HTTPBadRequest(
+                    explanation=msg))
+
+            # Now, deserialize the request body...
+            try:
+                if content_type:
+                    contents = self.deserialize(method, content_type, body)
+                else:
+                    contents = {}
+            except exception.InvalidContentType:
+                msg = _("Unsupported Content-Type")
+                raise GCEFault(webob.exc.HTTPBadRequest(
+                    explanation=msg))
+            except exception.MalformedRequestBody:
+                msg = _("Malformed request body")
+                raise GCEFault(webob.exc.HTTPBadRequest(
+                    explanation=msg))
+
+            # Update the action args
+            action_args.update(contents)
+
+            # Check project
+            project_id = action_args.pop("project_id", None)
+            context = request.environ.get('gceapi.context')
+            action_result = self._check_requested_project(project_id, context)
+
+            if action_result is None:
+                with GCEResourceExceptionHandler():
+                    action_result = self.dispatch(method, request, action_args)
+
+        except GCEFault as ex:
+            action_result = ex.wrapped_exc
+
+        response = None
+        resp_obj = None
+        if (action_result is None or type(action_result) is dict or
+                isinstance(action_result, Exception)):
+            action_result, result_code = self.controller.process_result(
+                    request, action, action_result)
+            resp_obj = GCEResponse(action_result, code=result_code)
+        elif isinstance(action_result, GCEResponse):
+            resp_obj = action_result
+        else:
+            response = action_result
+
+        # Serialize response object
+        if resp_obj:
+            if method is not None:
+                serializers = getattr(method, 'wsgi_serializers', {})
+            else:
+                serializers = {}
+            resp_obj._bind_method_serializers(serializers)
+            if method is not None and hasattr(method, 'wsgi_code'):
+                resp_obj._default_code = method.wsgi_code
+            resp_obj.preserialize(accept, self.default_serializers)
+            response = resp_obj.serialize(request, accept,
+                                          self.default_serializers)
+
+        try:
+            msg_dict = dict(url=request.url, status=response.status_int)
+            msg = _("%(url)s returned with HTTP %(status)d") % msg_dict
+        except AttributeError as e:
+            msg_dict = dict(url=request.url, e=e)
+            msg = _("%(url)s returned a fault: %(e)s") % msg_dict
+
+        LOG.info(msg)
+        return response

gceapi/api/zone_api.py

@@ -0,0 +1,68 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gceapi.api import base_api
+from gceapi.api import clients
+from gceapi.api import scopes
+from gceapi import exception
+
+
+class API(base_api.API):
+    """GCE Zones API."""
+
+    KIND = "zone"
+    COMPUTE_SERVICE = "nova-compute"
+
+    def _get_type(self):
+        return self.KIND
+
+    def get_item(self, context, name, scope=None):
+        zones = self.get_items(context)
+        for zone in zones:
+            if zone["name"] == name:
+                return zone
+        raise exception.NotFound
+
+    def get_items(self, context, scope=None):
+        client = clients.nova(context)
+        try:
+            nova_zones = client.availability_zones.list()
+        except clients.novaclient.exceptions.Forbidden as e:
+            try:
+                nova_zones = client.availability_zones.list(detailed=False)
+            except Exception:
+                raise e
+
+        filtered_zones = list()
+        for zone in nova_zones:
+            if not zone.hosts:
+                filtered_zones.append(zone)
+                continue
+            for host in zone.hosts:
+                if self.COMPUTE_SERVICE in zone.hosts[host]:
+                    filtered_zones.append(zone)
+                    break
+        zones = list()
+        for zone in filtered_zones:
+            zones.append({
+                "name": zone.zoneName,
+                "status": "UP" if zone.zoneState["available"] else "DOWN",
+                "hosts": [host for host in zone.hosts]
+                         if zone.hosts else list()
+            })
+        return zones
+
+    def get_items_as_scopes(self, context):
+        return [scopes.ZoneScope(zone["name"])
+                for zone in self.get_items(context)]

gceapi/api/zones.py

@@ -0,0 +1,38 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gceapi.api import common as gce_common
+from gceapi.api import region_api
+from gceapi.api import wsgi as gce_wsgi
+from gceapi.api import zone_api
+
+
+class Controller(gce_common.Controller):
+    """GCE Zones controller."""
+
+    def __init__(self, *args, **kwargs):
+        super(Controller, self).__init__(zone_api.API(), *args, **kwargs)
+
+    def format_item(self, request, zone, scope):
+        result_dict = {
+            "name": zone["name"],
+            "status": zone["status"],
+            "region": region_api.API().get_items(None)[0]["name"],
+        }
+
+        return self._format_item(request, result_dict, scope)
+
+
+def create_resource():
+    return gce_wsgi.GCEResource(Controller())

gceapi/auth.py

@@ -0,0 +1,142 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright (c) 2011 OpenStack Foundation
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""
+Common Auth Middleware.
+
+"""
+
+from oslo.config import cfg
+import webob.dec
+import webob.exc
+
+from gceapi import context
+from gceapi.openstack.common.gettextutils import _
+from gceapi.openstack.common import jsonutils
+from gceapi.openstack.common import log as logging
+from gceapi import wsgi
+
+
+auth_opts = [
+    cfg.BoolOpt('api_rate_limit',
+                default=False,
+                help='whether to use per-user rate limiting for the api.'),
+    cfg.StrOpt('auth_strategy',
+               default='noauth',
+               help='The strategy to use for auth: noauth or keystone.'),
+    cfg.BoolOpt('use_forwarded_for',
+                default=False,
+                help='Treat X-Forwarded-For as the canonical remote address. '
+                     'Only enable this if you have a sanitizing proxy.'),
+]
+
+CONF = cfg.CONF
+CONF.register_opts(auth_opts)
+
+LOG = logging.getLogger(__name__)
+
+
+def pipeline_factory(loader, global_conf, **local_conf):
+    """A paste pipeline replica that keys off of auth_strategy."""
+    pipeline = local_conf[CONF.auth_strategy]
+    if not CONF.api_rate_limit:
+        limit_name = CONF.auth_strategy + '_nolimit'
+        pipeline = local_conf.get(limit_name, pipeline)
+    pipeline = pipeline.split()
+    filters = [loader.get_filter(n) for n in pipeline[:-1]]
+    app = loader.get_app(pipeline[-1])
+    filters.reverse()
+    for filter_func in filters:
+        app = filter_func(app)
+    return app
+
+
+class InjectContext(wsgi.Middleware):
+    """Add a 'gceapi.context' to WSGI environ."""
+
+    def __init__(self, context, *args, **kwargs):
+        self.context = context
+        super(InjectContext, self).__init__(*args, **kwargs)
+
+    @webob.dec.wsgify(RequestClass=wsgi.Request)
+    def __call__(self, req):
+        req.environ['gceapi.context'] = self.context
+        return self.application
+
+
+class GceapiKeystoneContext(wsgi.Middleware):
+    """Make a request context from keystone headers."""
+
+    @webob.dec.wsgify(RequestClass=wsgi.Request)
+    def __call__(self, req):
+        user_id = req.headers.get('X_USER')
+        user_id = req.headers.get('X_USER_ID', user_id)
+        if user_id is None:
+            LOG.debug("Neither X_USER_ID nor X_USER found in request")
+            return webob.exc.HTTPUnauthorized()
+
+        roles = self._get_roles(req)
+
+        if 'X_TENANT_ID' in req.headers:
+            # This is the new header since Keystone went to ID/Name
+            project_id = req.headers['X_TENANT_ID']
+        else:
+            # This is for legacy compatibility
+            project_id = req.headers['X_TENANT']
+        project_name = req.headers.get('X_TENANT_NAME')
+        user_name = req.headers.get('X_USER_NAME')
+
+        # Get the auth token
+        auth_token = req.headers.get('X_AUTH_TOKEN',
+                                     req.headers.get('X_STORAGE_TOKEN'))
+
+        # Build a context, including the auth_token...
+        remote_address = req.remote_addr
+        if CONF.use_forwarded_for:
+            remote_address = req.headers.get('X-Forwarded-For', remote_address)
+
+        service_catalog = None
+        if req.headers.get('X_SERVICE_CATALOG') is not None:
+            try:
+                catalog_header = req.headers.get('X_SERVICE_CATALOG')
+                service_catalog = jsonutils.loads(catalog_header)
+            except ValueError:
+                raise webob.exc.HTTPInternalServerError(
+                          _('Invalid service catalog json.'))
+
+        ctx = context.RequestContext(user_id,
+                                     project_id,
+                                     user_name=user_name,
+                                     project_name=project_name,
+                                     roles=roles,
+                                     auth_token=auth_token,
+                                     remote_address=remote_address,
+                                     service_catalog=service_catalog)
+
+        req.environ['gceapi.context'] = ctx
+        return self.application
+
+    def _get_roles(self, req):
+        """Get the list of roles."""
+
+        if 'X_ROLES' in req.headers:
+            roles = req.headers.get('X_ROLES', '')
+        else:
+            # Fallback to deprecated role header:
+            roles = req.headers.get('X_ROLE', '')
+            if roles:
+                LOG.warn(_("Sourcing roles from deprecated X-Role HTTP "
+                           "header"))
+        return [r.strip() for r in roles.split(',')]

gceapi/cmd/__init__.py

@@ -0,0 +1,19 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from gceapi.openstack.common import gettextutils
+gettextutils.install('gceapi')

gceapi/cmd/api.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Gceapi API Server
+"""
+
+import eventlet
+import sys
+
+eventlet.patcher.monkey_patch(os=False)
+
+from oslo.config import cfg
+
+from gceapi import config
+from gceapi.openstack.common import log as logging
+from gceapi import service
+
+CONF = cfg.CONF
+CONF.import_opt('use_ssl', 'gceapi.service')
+
+
+def main():
+    config.parse_args(sys.argv)
+    logging.setup('gceapi')
+
+    server = service.WSGIService(
+        'gce', use_ssl=CONF.use_ssl, max_url_len=16384)
+    service.serve(server)
+    service.wait()
+
+
+if __name__ == '__main__':
+    main()

gceapi/cmd/manage.py

@@ -0,0 +1,75 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+"""
+  CLI interface for GCE API management.
+"""
+
+import sys
+
+from oslo.config import cfg
+
+from gceapi.db import migration
+from gceapi.openstack.common import log
+from gceapi import version
+
+
+CONF = cfg.CONF
+
+
+def do_db_version():
+    """Print database's current migration level."""
+    print(migration.db_version())
+
+
+def do_db_sync():
+    """
+    Place a database under migration control and upgrade,
+    creating first if necessary.
+    """
+    migration.db_sync(CONF.command.version)
+
+
+def add_command_parsers(subparsers):
+    parser = subparsers.add_parser('db_version')
+    parser.set_defaults(func=do_db_version)
+
+    parser = subparsers.add_parser('db_sync')
+    parser.set_defaults(func=do_db_sync)
+    parser.add_argument('version', nargs='?')
+    parser.add_argument('current_version', nargs='?')
+
+
+command_opt = cfg.SubCommandOpt('command',
+                                title='Commands',
+                                help='Available commands',
+                                handler=add_command_parsers)
+
+
+def main():
+    CONF.register_cli_opt(command_opt)
+    try:
+        default_config_files = cfg.find_config_files('gceapi', 'gceapi-engine')
+        CONF(sys.argv[1:], project='gceapi', prog='gceapi-manage',
+             version=version.version_info.version_string(),
+             default_config_files=default_config_files)
+        log.setup("gceapi")
+    except RuntimeError as e:
+        sys.exit("ERROR: %s" % e)
+
+    try:
+        CONF.command.func()
+    except Exception as e:
+        sys.exit("ERROR: %s" % e)

gceapi/config.py

@@ -0,0 +1,35 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+# Copyright 2012 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo.config import cfg
+
+from gceapi.openstack.common.db.sqlalchemy import session as db_session
+from gceapi import paths
+from gceapi import version
+
+_DEFAULT_SQL_CONNECTION = 'sqlite:///' + paths.state_path_def('$sqlite_db')
+
+
+def parse_args(argv, default_config_files=None):
+    db_session.set_defaults(sql_connection=_DEFAULT_SQL_CONNECTION,
+                            sqlite_db='gceapi.sqlite')
+    cfg.CONF(argv[1:],
+             project='gceapi',
+             version=version.version_string(),
+             default_config_files=default_config_files)

gceapi/context.py

@@ -0,0 +1,162 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack Foundation
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""RequestContext: context for requests that persist through all of gceapi."""
+
+import uuid
+
+from gceapi import exception
+from gceapi.openstack.common.gettextutils import _
+from gceapi.openstack.common import local
+from gceapi.openstack.common import log as logging
+from gceapi.openstack.common import timeutils
+
+
+LOG = logging.getLogger(__name__)
+
+
+def generate_request_id():
+    return 'req-' + str(uuid.uuid4())
+
+
+class RequestContext(object):
+    """Security context and request information.
+
+    Represents the user taking a given action within the system.
+
+    """
+
+    def __init__(self, user_id, project_id, is_admin=None, read_deleted="no",
+                 roles=None, remote_address=None, timestamp=None,
+                 request_id=None, auth_token=None, overwrite=True,
+                 user_name=None, project_name=None,
+                 service_catalog=None, **kwargs):
+        """
+        :param read_deleted: 'no' indicates deleted records are hidden, 'yes'
+            indicates deleted records are visible, 'only' indicates that
+            *only* deleted records are visible.
+
+        :param overwrite: Set to False to ensure that the greenthread local
+            copy of the index is not overwritten.
+
+        :param kwargs: Extra arguments that might be present, but we ignore
+            because they possibly came in from older rpc messages.
+        """
+        if kwargs:
+            LOG.warn(_('Arguments dropped when creating context: %s') %
+                    str(kwargs))
+
+        self.user_id = user_id
+        self.project_id = project_id
+        self.roles = roles or []
+        self.read_deleted = read_deleted
+        self.remote_address = remote_address
+        if not timestamp:
+            timestamp = timeutils.utcnow()
+        if isinstance(timestamp, basestring):
+            timestamp = timeutils.parse_strtime(timestamp)
+        self.timestamp = timestamp
+        if not request_id:
+            request_id = generate_request_id()
+        self.request_id = request_id
+        self.auth_token = auth_token
+
+        self.service_catalog = service_catalog
+
+        self.user_name = user_name
+        self.project_name = project_name
+        self.is_admin = is_admin
+        #if self.is_admin is None:
+        #    self.is_admin = policy.check_is_admin(self)
+        if overwrite or not hasattr(local.store, 'context'):
+            self.update_store()
+        self.operation = None
+        self.operation_start_time = None
+        self.operation_get_progress_method = None
+        self.operation_item_id = None
+
+    def _get_read_deleted(self):
+        return self._read_deleted
+
+    def _set_read_deleted(self, read_deleted):
+        if read_deleted not in ('no', 'yes', 'only'):
+            raise ValueError(_("read_deleted can only be one of 'no', "
+                               "'yes' or 'only', not %r") % read_deleted)
+        self._read_deleted = read_deleted
+
+    def _del_read_deleted(self):
+        del self._read_deleted
+
+    read_deleted = property(_get_read_deleted, _set_read_deleted,
+                            _del_read_deleted)
+
+    def update_store(self):
+        local.store.context = self
+
+    def to_dict(self):
+        return {'user_id': self.user_id,
+                'project_id': self.project_id,
+                'is_admin': self.is_admin,
+                'read_deleted': self.read_deleted,
+                'roles': self.roles,
+                'remote_address': self.remote_address,
+                'timestamp': timeutils.strtime(self.timestamp),
+                'request_id': self.request_id,
+                'auth_token': self.auth_token,
+                'user_name': self.user_name,
+                'service_catalog': self.service_catalog,
+                'project_name': self.project_name,
+                'tenant': self.tenant,
+                'user': self.user}
+
+    @classmethod
+    def from_dict(cls, values):
+        return cls(**values)
+
+    # NOTE(sirp): the openstack/common version of RequestContext uses
+    # tenant/user whereas the gceapi version uses project_id/user_id. We need
+    # this shim in order to use context-aware code from openstack/common, like
+    # logging, until we make the switch to using openstack/common's version of
+    # RequestContext.
+    @property
+    def tenant(self):
+        return self.project_id
+
+    @property
+    def user(self):
+        return self.user_id
+
+
+def is_user_context(context):
+    """Indicates if the request context is a normal user."""
+    if not context:
+        return False
+    if context.is_admin:
+        return False
+    if not context.user_id or not context.project_id:
+        return False
+    return True
+
+
+def require_context(ctxt):
+    """Raise exception.NotAuthorized() if context is not a user or an
+    admin context.
+    """
+    if not ctxt.is_admin and not is_user_context(ctxt):
+        raise exception.NotAuthorized()

gceapi/db/__init__.py

@@ -0,0 +1,19 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+DB abstraction for Gceapi
+"""
+
+from gceapi.db.api import *

gceapi/db/api.py

@@ -0,0 +1,58 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Defines interface for DB access.
+
+Functions in this module are imported into the gceapi.db namespace. Call these
+functions from gceapi.db namespace, not the gceapi.db.api namespace.
+
+**Related Flags**
+
+:dbackend:  string to lookup in the list of LazyPluggable backends.
+            `sqlalchemy` is the only supported backend right now.
+
+:connection:  string specifying the sqlalchemy connection to use, like:
+              `sqlite:///var/lib/gceapi/gceapi.sqlite`.
+
+"""
+
+from gceapi.openstack.common.db import api as db_api
+
+
+_BACKEND_MAPPING = {'sqlalchemy': 'gceapi.db.sqlalchemy.api'}
+IMPL = db_api.DBAPI(backend_mapping=_BACKEND_MAPPING)
+
+
+def add_item(context, kind, data):
+    IMPL.add_item(context, kind, data)
+
+
+def delete_item(context, kind, item_id):
+    IMPL.delete_item(context, kind, item_id)
+
+
+def update_item(context, kind, item):
+    IMPL.update_item(context, kind, item)
+
+
+def get_items(context, kind):
+    return IMPL.get_items(context, kind)
+
+
+def get_item_by_id(context, kind, item_id):
+    return IMPL.get_item_by_id(context, kind, item_id)
+
+
+def get_item_by_name(context, kind, name):
+    return IMPL.get_item_by_name(context, kind, name)

gceapi/db/migration.py

@@ -0,0 +1,73 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Database setup and migration commands."""
+
+from oslo.config import cfg
+
+from gceapi import exception
+from gceapi.openstack.common.gettextutils import _
+
+CONF = cfg.CONF
+
+
+class LazyPluggable(object):
+    """A pluggable backend loaded lazily based on some value."""
+
+    def __init__(self, pivot, config_group=None, **backends):
+        self.__backends = backends
+        self.__pivot = pivot
+        self.__backend = None
+        self.__config_group = config_group
+
+    def __get_backend(self):
+        if not self.__backend:
+            if self.__config_group is None:
+                backend_name = CONF[self.__pivot]
+            else:
+                backend_name = CONF[self.__config_group][self.__pivot]
+            if backend_name not in self.__backends:
+                msg = _('Invalid backend: %s') % backend_name
+                raise exception.GceapiException(msg)
+
+            backend = self.__backends[backend_name]
+            if isinstance(backend, tuple):
+                name = backend[0]
+                fromlist = backend[1]
+            else:
+                name = backend
+                fromlist = backend
+
+            self.__backend = __import__(name, None, None, fromlist)
+        return self.__backend
+
+    def __getattr__(self, key):
+        backend = self.__get_backend()
+        return getattr(backend, key)
+
+IMPL = LazyPluggable('backend',
+                     config_group='database',
+                     sqlalchemy='gceapi.db.sqlalchemy.migration')
+
+INIT_VERSION = 0
+
+
+def db_sync(version=None):
+    """Migrate the database to `version` or the most recent version."""
+    return IMPL.db_sync(INIT_VERSION, version=version)
+
+
+def db_version():
+    """Display the current database version."""
+    return IMPL.db_version(INIT_VERSION)

gceapi/db/sqlalchemy/__init__.py

@@ -0,0 +1,13 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.

gceapi/db/sqlalchemy/api.py

@@ -0,0 +1,133 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Implementation of SQLAlchemy backend."""
+
+import ast
+import functools
+import sys
+
+from oslo.config import cfg
+
+import gceapi.context
+from gceapi.db.sqlalchemy import models
+from gceapi.openstack.common.db.sqlalchemy import session as db_session
+
+CONF = cfg.CONF
+CONF.import_opt('connection',
+                'gceapi.openstack.common.db.sqlalchemy.session',
+                group='database')
+
+get_session = db_session.get_session
+
+
+def get_backend():
+    """The backend is this module itself."""
+    return sys.modules[__name__]
+
+
+def require_context(f):
+    """Decorator to require *any* user or admin context.
+
+    The first argument to the wrapped function must be the context.
+    """
+
+    @functools.wraps(f)
+    def wrapper(*args, **kwargs):
+        gceapi.context.require_context(args[0])
+        return f(*args, **kwargs)
+    return wrapper
+
+
+def model_query(context, model, *args, **kwargs):
+    """Query helper that accounts for context's `read_deleted` field.
+
+    :param context: context to query under
+    :param session: if present, the session to use
+    """
+    session = kwargs.get('session') or get_session()
+
+    return session.query(model, *args).\
+            filter_by(project_id=context.project_id)
+
+
+@require_context
+def add_item(context, kind, data):
+    item_ref = models.Item()
+    item_ref.update({
+        "project_id": context.project_id,
+        "kind": kind,
+    })
+    item_ref.update(_pack_item_data(data))
+    item_ref.save()
+
+
+@require_context
+def delete_item(context, kind, item_id):
+    model_query(context, models.Item).\
+            filter_by(kind=kind,
+                      id=item_id).\
+            delete()
+
+
+@require_context
+def update_item(context, kind, item):
+    item_ref = model_query(context, models.Item).\
+            filter_by(kind=kind,
+                      id=item["id"]).\
+            one()
+    item_ref.update(_pack_item_data(item))
+    item_ref.save()
+
+
+@require_context
+def get_items(context, kind):
+    return [_unpack_item_data(item)
+            for item in model_query(context, models.Item).
+                    filter_by(kind=kind).
+                    all()]
+
+
+@require_context
+def get_item_by_id(context, kind, item_id):
+    return _unpack_item_data(model_query(context, models.Item).
+            filter_by(kind=kind,
+                      id=item_id).
+            first())
+
+
+@require_context
+def get_item_by_name(context, kind, name):
+    return _unpack_item_data(model_query(context, models.Item).
+            filter_by(kind=kind,
+                      name=name).
+            first())
+
+
+def _pack_item_data(item_data):
+    return {
+        "id": item_data.pop("id"),
+        "name": item_data.pop("name", None),
+        "data": str(item_data),
+    }
+
+
+def _unpack_item_data(item_ref):
+    if item_ref is None:
+        return None
+    data = ast.literal_eval(item_ref.data)
+    data["id"] = item_ref.id
+    if item_ref.name is not None:
+        data["name"] = item_ref.name
+    return data

gceapi/db/sqlalchemy/migrate_repo/README

@@ -0,0 +1,4 @@
+This is a database migration repository.
+
+More information at
+http://code.google.com/p/sqlalchemy-migrate/

gceapi/db/sqlalchemy/migrate_repo/__init__.py

@@ -0,0 +1,13 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.

gceapi/db/sqlalchemy/migrate_repo/manage.py

@@ -0,0 +1,19 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from migrate.versioning.shell import main
+
+
+if __name__ == '__main__':
+    main(debug='False', repository='.')

gceapi/db/sqlalchemy/migrate_repo/migrate.cfg

@@ -0,0 +1,20 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=gceapi
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to 
+# change the table name in each database too. 
+version_table=migrate_version
+
+# When committing a change script, Migrate will attempt to generate the 
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the 
+# commit continues, perhaps ending successfully. 
+# Databases in this list MUST compile successfully during a commit, or the 
+# entire commit will fail. List the databases your application will actually 
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=[]

gceapi/db/sqlalchemy/migrate_repo/versions/001_icehouse.py

@@ -0,0 +1,38 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from sqlalchemy import Column, Index, MetaData, PrimaryKeyConstraint
+from sqlalchemy import String, Table, Text
+
+
+def upgrade(migrate_engine):
+    meta = MetaData()
+    meta.bind = migrate_engine
+
+    items = Table('items', meta,
+        Column("id", String(length=255)),
+        Column("project_id", String(length=255)),
+        Column("kind", String(length=50)),
+        Column("name", String(length=63)),
+        Column("data", Text()),
+        PrimaryKeyConstraint('kind', 'id'),
+        Index('items_project_kind_name_idx', 'project_id', 'kind', 'name'),
+        mysql_engine="InnoDB",
+        mysql_charset="utf8"
+    )
+    items.create()
+
+
+def downgrade(migrate_engine):
+    raise NotImplementedError("Downgrade from Icehouse is unsupported.")

gceapi/db/sqlalchemy/migrate_repo/versions/__init__.py

@@ -0,0 +1,13 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.

gceapi/db/sqlalchemy/migration.py

@@ -0,0 +1,30 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+
+from gceapi.openstack.common.db.sqlalchemy import migration
+
+
+def db_sync(init_version, version=None):
+    return migration.db_sync(_get_repo_path(), version, init_version)
+
+
+def db_version(init_version):
+    return migration.db_version(_get_repo_path(), init_version)
+
+
+def _get_repo_path():
+    return os.path.join(os.path.abspath(os.path.dirname(__file__)),
+                        'migrate_repo')

gceapi/db/sqlalchemy/models.py

@@ -0,0 +1,37 @@
+#    Copyright 2013 Cloudscaling Group, Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+SQLAlchemy models for gceapi data.
+"""
+
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import Column, Index, PrimaryKeyConstraint, String, Text
+
+from gceapi.openstack.common.db.sqlalchemy import models
+
+BASE = declarative_base()
+
+
+class Item(BASE, models.ModelBase):
+    __tablename__ = 'items'
+    __table_args__ = (
+        PrimaryKeyConstraint('kind', 'id'),
+        Index('items_project_kind_name_idx', 'project_id', 'kind', 'name'),
+    )
+    id = Column(String(length=255))
+    project_id = Column(String(length=255))
+    kind = Column(String(length=50))
+    name = Column(String(length=63))
+    data = Column(Text())

gceapi/openstack/common/db/api.py

@@ -0,0 +1,57 @@
+# Copyright (c) 2013 Rackspace Hosting
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Multiple DB API backend support.
+
+Supported configuration options:
+
+The following two parameters are in the 'database' group:
+`backend`: DB backend name or full module path to DB backend module.
+
+A DB backend module should implement a method named 'get_backend' which
+takes no arguments.  The method can return any object that implements DB
+API methods.
+"""
+
+from oslo.config import cfg
+
+from gceapi.openstack.common import importutils
+
+
+db_opts = [
+    cfg.StrOpt('backend',
+               default='sqlalchemy',
+               deprecated_name='db_backend',
+               deprecated_group='DEFAULT',
+               help='The backend to use for db'),
+]
+
+CONF = cfg.CONF
+CONF.register_opts(db_opts, 'database')
+
+
+class DBAPI(object):
+    def __init__(self, backend_mapping=None):
+        if backend_mapping is None:
+            backend_mapping = {}
+        backend_name = CONF.database.backend
+        # Import the untranslated name if we don't have a
+        # mapping.
+        backend_path = backend_mapping.get(backend_name, backend_name)
+        backend_mod = importutils.import_module(backend_path)
+        self.__backend = backend_mod.get_backend()
+
+    def __getattr__(self, key):
+        return getattr(self.__backend, key)

gceapi/openstack/common/db/exception.py

@@ -0,0 +1,54 @@
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""DB related custom exceptions."""
+
+from gceapi.openstack.common.gettextutils import _
+
+
+class DBError(Exception):
+    """Wraps an implementation specific exception."""
+    def __init__(self, inner_exception=None):
+        self.inner_exception = inner_exception
+        super(DBError, self).__init__(str(inner_exception))
+
+
+class DBDuplicateEntry(DBError):
+    """Wraps an implementation specific exception."""
+    def __init__(self, columns=[], inner_exception=None):
+        self.columns = columns
+        super(DBDuplicateEntry, self).__init__(inner_exception)
+
+
+class DBDeadlock(DBError):
+    def __init__(self, inner_exception=None):
+        super(DBDeadlock, self).__init__(inner_exception)
+
+
+class DBInvalidUnicodeParameter(Exception):
+    message = _("Invalid Parameter: "
+                "Unicode is not supported by the current database.")
+
+
+class DbMigrationError(DBError):
+    """Wraps migration specific exception."""
+    def __init__(self, message=None):
+        super(DbMigrationError, self).__init__(str(message))
+
+
+class DBConnectionError(DBError):
+    """Wraps connection specific exception."""
+    pass

gceapi/openstack/common/db/sqlalchemy/migration.py

@@ -0,0 +1,265 @@
+# coding: utf-8
+#
+# Copyright (c) 2013 OpenStack Foundation
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Base on code in migrate/changeset/databases/sqlite.py which is under
+# the following license:
+#
+# The MIT License
+#
+# Copyright (c) 2009 Evan Rosson, Jan Dittberner, Domen Kožar
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import os
+import re
+
+from migrate.changeset import ansisql
+from migrate.changeset.databases import sqlite
+from migrate import exceptions as versioning_exceptions
+from migrate.versioning import api as versioning_api
+from migrate.versioning.repository import Repository
+import sqlalchemy
+from sqlalchemy.schema import UniqueConstraint
+
+from gceapi.openstack.common.db import exception
+from gceapi.openstack.common.db.sqlalchemy import session as db_session
+from gceapi.openstack.common.gettextutils import _
+
+
+get_engine = db_session.get_engine
+
+
+def _get_unique_constraints(self, table):
+    """Retrieve information about existing unique constraints of the table
+
+    This feature is needed for _recreate_table() to work properly.
+    Unfortunately, it's not available in sqlalchemy 0.7.x/0.8.x.
+
+    """
+
+    data = table.metadata.bind.execute(
+        """SELECT sql
+           FROM sqlite_master
+           WHERE
+               type='table' AND
+               name=:table_name""",
+        table_name=table.name
+    ).fetchone()[0]
+
+    UNIQUE_PATTERN = "CONSTRAINT (\w+) UNIQUE \(([^\)]+)\)"
+    return [
+        UniqueConstraint(
+            *[getattr(table.columns, c.strip(' "')) for c in cols.split(",")],
+            name=name
+        )
+        for name, cols in re.findall(UNIQUE_PATTERN, data)
+    ]
+
+
+def _recreate_table(self, table, column=None, delta=None, omit_uniques=None):
+    """Recreate the table properly
+
+    Unlike the corresponding original method of sqlalchemy-migrate this one
+    doesn't drop existing unique constraints when creating a new one.
+
+    """
+
+    table_name = self.preparer.format_table(table)
+
+    # we remove all indexes so as not to have
+    # problems during copy and re-create
+    for index in table.indexes:
+        index.drop()
+
+    # reflect existing unique constraints
+    for uc in self._get_unique_constraints(table):
+        table.append_constraint(uc)
+    # omit given unique constraints when creating a new table if required
+    table.constraints = set([
+        cons for cons in table.constraints
+        if omit_uniques is None or cons.name not in omit_uniques
+    ])
+
+    self.append('ALTER TABLE %s RENAME TO migration_tmp' % table_name)
+    self.execute()
+
+    insertion_string = self._modify_table(table, column, delta)
+
+    table.create(bind=self.connection)
+    self.append(insertion_string % {'table_name': table_name})
+    self.execute()
+    self.append('DROP TABLE migration_tmp')
+    self.execute()
+
+
+def _visit_migrate_unique_constraint(self, *p, **k):
+    """Drop the given unique constraint
+
+    The corresponding original method of sqlalchemy-migrate just
+    raises NotImplemented error
+
+    """
+
+    self.recreate_table(p[0].table, omit_uniques=[p[0].name])
+
+
+def patch_migrate():
+    """A workaround for SQLite's inability to alter things
+
+    SQLite abilities to alter tables are very limited (please read
+    http://www.sqlite.org/lang_altertable.html for more details).
+    E. g. one can't drop a column or a constraint in SQLite. The
+    workaround for this is to recreate the original table omitting
+    the corresponding constraint (or column).
+
+    sqlalchemy-migrate library has recreate_table() method that
+    implements this workaround, but it does it wrong:
+
+        - information about unique constraints of a table
+          is not retrieved. So if you have a table with one
+          unique constraint and a migration adding another one
+          you will end up with a table that has only the
+          latter unique constraint, and the former will be lost
+
+        - dropping of unique constraints is not supported at all
+
+    The proper way to fix this is to provide a pull-request to
+    sqlalchemy-migrate, but the project seems to be dead. So we
+    can go on with monkey-patching of the lib at least for now.
+
+    """
+
+    # this patch is needed to ensure that recreate_table() doesn't drop
+    # existing unique constraints of the table when creating a new one
+    helper_cls = sqlite.SQLiteHelper
+    helper_cls.recreate_table = _recreate_table
+    helper_cls._get_unique_constraints = _get_unique_constraints
+
+    # this patch is needed to be able to drop existing unique constraints
+    constraint_cls = sqlite.SQLiteConstraintDropper
+    constraint_cls.visit_migrate_unique_constraint = \
+        _visit_migrate_unique_constraint
+    constraint_cls.__bases__ = (ansisql.ANSIColumnDropper,
+                                sqlite.SQLiteConstraintGenerator)
+
+
+def db_sync(abs_path, version=None, init_version=0):
+    """Upgrade or downgrade a database.
+
+    Function runs the upgrade() or downgrade() functions in change scripts.
+
+    :param abs_path:     Absolute path to migrate repository.
+    :param version:      Database will upgrade/downgrade until this version.
+                         If None - database will update to the latest
+                         available version.
+    :param init_version: Initial database version
+    """
+    if version is not None:
+        try:
+            version = int(version)
+        except ValueError:
+            raise exception.DbMigrationError(
+                message=_("version should be an integer"))
+
+    current_version = db_version(abs_path, init_version)
+    repository = _find_migrate_repo(abs_path)
+    _db_schema_sanity_check()
+    if version is None or version > current_version:
+        return versioning_api.upgrade(get_engine(), repository, version)
+    else:
+        return versioning_api.downgrade(get_engine(), repository,
+                                        version)
+
+
+def _db_schema_sanity_check():
+    engine = get_engine()
+    if engine.name == 'mysql':
+        onlyutf8_sql = ('SELECT TABLE_NAME,TABLE_COLLATION '
+                        'from information_schema.TABLES '
+                        'where TABLE_SCHEMA=%s and '
+                        'TABLE_COLLATION NOT LIKE "%%utf8%%"')
+
+        table_names = [res[0] for res in engine.execute(onlyutf8_sql,
+                                                        engine.url.database)]
+        if len(table_names) > 0:
+            raise ValueError(_('Tables "%s" have non utf8 collation, '
+                               'please make sure all tables are CHARSET=utf8'
+                               ) % ','.join(table_names))
+
+
+def db_version(abs_path, init_version):
+    """Show the current version of the repository.
+
+    :param abs_path: Absolute path to migrate repository
+    :param version:  Initial database version
+    """
+    repository = _find_migrate_repo(abs_path)
+    try:
+        return versioning_api.db_version(get_engine(), repository)
+    except versioning_exceptions.DatabaseNotControlledError:
+        meta = sqlalchemy.MetaData()
+        engine = get_engine()
+        meta.reflect(bind=engine)
+        tables = meta.tables
+        if len(tables) == 0 or 'alembic_version' in tables:
+            db_version_control(abs_path, init_version)
+            return versioning_api.db_version(get_engine(), repository)
+        else:
+            raise exception.DbMigrationError(
+                message=_(
+                    "The database is not under version control, but has "
+                    "tables. Please stamp the current version of the schema "
+                    "manually."))
+
+
+def db_version_control(abs_path, version=None):
+    """Mark a database as under this repository's version control.
+
+    Once a database is under version control, schema changes should
+    only be done via change scripts in this repository.
+
+    :param abs_path: Absolute path to migrate repository
+    :param version:  Initial database version
+    """
+    repository = _find_migrate_repo(abs_path)
+    versioning_api.version_control(get_engine(), repository, version)
+    return version
+
+
+def _find_migrate_repo(abs_path):
+    """Get the project's change script repository
+
+    :param abs_path: Absolute path to migrate repository
+    """
+    if not os.path.exists(abs_path):
+        raise exception.DbMigrationError("Path %s not found" % abs_path)
+    return Repository(abs_path)

gceapi/openstack/common/db/sqlalchemy/models.py

@@ -0,0 +1,117 @@
+# Copyright (c) 2011 X.commerce, a business unit of eBay Inc.
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# Copyright 2011 Piston Cloud Computing, Inc.
+# Copyright 2012 Cloudscaling Group, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""
+SQLAlchemy models.
+"""
+
+import six
+
+from sqlalchemy import Column, Integer
+from sqlalchemy import DateTime
+from sqlalchemy.orm import object_mapper
+
+from gceapi.openstack.common.db.sqlalchemy import session as sa
+from gceapi.openstack.common import timeutils
+
+
+class ModelBase(object):
+    """Base class for models."""
+    __table_initialized__ = False
+
+    def save(self, session=None):
+        """Save this object."""
+        if not session:
+            session = sa.get_session()
+        # NOTE(boris-42): This part of code should be look like:
+        #                       session.add(self)
+        #                       session.flush()
+        #                 But there is a bug in sqlalchemy and eventlet that
+        #                 raises NoneType exception if there is no running
+        #                 transaction and rollback is called. As long as
+        #                 sqlalchemy has this bug we have to create transaction
+        #                 explicitly.
+        with session.begin(subtransactions=True):
+            session.add(self)
+            session.flush()
+
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+    def __getitem__(self, key):
+        return getattr(self, key)
+
+    def get(self, key, default=None):
+        return getattr(self, key, default)
+
+    @property
+    def _extra_keys(self):
+        """Specifies custom fields
+
+        Subclasses can override this property to return a list
+        of custom fields that should be included in their dict
+        representation.
+
+        For reference check tests/db/sqlalchemy/test_models.py
+        """
+        return []
+
+    def __iter__(self):
+        columns = dict(object_mapper(self).columns).keys()
+        # NOTE(russellb): Allow models to specify other keys that can be looked
+        # up, beyond the actual db columns.  An example would be the 'name'
+        # property for an Instance.
+        columns.extend(self._extra_keys)
+        self._i = iter(columns)
+        return self
+
+    def next(self):
+        n = six.advance_iterator(self._i)
+        return n, getattr(self, n)
+
+    def update(self, values):
+        """Make the model object behave like a dict."""
+        for k, v in six.iteritems(values):
+            setattr(self, k, v)
+
+    def iteritems(self):
+        """Make the model object behave like a dict.
+
+        Includes attributes from joins.
+        """
+        local = dict(self)
+        joined = dict([(k, v) for k, v in six.iteritems(self.__dict__)
+                      if not k[0] == '_'])
+        local.update(joined)
+        return six.iteritems(local)
+
+
+class TimestampMixin(object):
+    created_at = Column(DateTime, default=lambda: timeutils.utcnow())
+    updated_at = Column(DateTime, onupdate=lambda: timeutils.utcnow())
+
+
+class SoftDeleteMixin(object):
+    deleted_at = Column(DateTime)
+    deleted = Column(Integer, default=0)
+
+    def soft_delete(self, session=None):
+        """Mark this object as deleted."""
+        self.deleted = self.id
+        self.deleted_at = timeutils.utcnow()
+        self.save(session=session)

gceapi/openstack/common/db/sqlalchemy/provision.py

@@ -0,0 +1,187 @@
+# Copyright 2013 Mirantis.inc
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Provision test environment for specific DB backends"""
+
+import argparse
+import os
+import random
+import string
+
+from six import moves
+import sqlalchemy
+
+from gceapi.openstack.common.db import exception as exc
+
+
+SQL_CONNECTION = os.getenv('OS_TEST_DBAPI_ADMIN_CONNECTION', 'sqlite://')
+
+
+def _gen_credentials(*names):
+    """Generate credentials."""
+    auth_dict = {}
+    for name in names:
+        val = ''.join(random.choice(string.ascii_lowercase)
+                      for i in moves.range(10))
+        auth_dict[name] = val
+    return auth_dict
+
+
+def _get_engine(uri=SQL_CONNECTION):
+    """Engine creation
+
+    By default the uri is SQL_CONNECTION which is admin credentials.
+    Call the function without arguments to get admin connection. Admin
+    connection required to create temporary user and database for each
+    particular test. Otherwise use existing connection to recreate connection
+    to the temporary database.
+    """
+    return sqlalchemy.create_engine(uri, poolclass=sqlalchemy.pool.NullPool)
+
+
+def _execute_sql(engine, sql, driver):
+    """Initialize connection, execute sql query and close it."""
+    try:
+        with engine.connect() as conn:
+            if driver == 'postgresql':
+                conn.connection.set_isolation_level(0)
+            for s in sql:
+                conn.execute(s)
+    except sqlalchemy.exc.OperationalError:
+        msg = ('%s does not match database admin '
+               'credentials or database does not exist.')
+        raise exc.DBConnectionError(msg % SQL_CONNECTION)
+
+
+def create_database(engine):
+    """Provide temporary user and database for each particular test."""
+    driver = engine.name
+
+    auth = _gen_credentials('database', 'user', 'passwd')
+
+    sqls = {
+        'mysql': [
+            "drop database if exists %(database)s;",
+            "grant all on %(database)s.* to '%(user)s'@'localhost'"
+            " identified by '%(passwd)s';",
+            "create database %(database)s;",
+        ],
+        'postgresql': [
+            "drop database if exists %(database)s;",
+            "drop user if exists %(user)s;",
+            "create user %(user)s with password '%(passwd)s';",
+            "create database %(database)s owner %(user)s;",
+        ]
+    }
+
+    if driver == 'sqlite':
+        return 'sqlite:////tmp/%s' % auth['database']
+
+    try:
+        sql_rows = sqls[driver]
+    except KeyError:
+        raise ValueError('Unsupported RDBMS %s' % driver)
+    sql_query = map(lambda x: x % auth, sql_rows)
+
+    _execute_sql(engine, sql_query, driver)
+
+    params = auth.copy()
+    params['backend'] = driver
+    return "%(backend)s://%(user)s:%(passwd)s@localhost/%(database)s" % params
+
+
+def drop_database(engine, current_uri):
+    """Drop temporary database and user after each particular test."""
+    engine = _get_engine(current_uri)
+    admin_engine = _get_engine()
+    driver = engine.name
+    auth = {'database': engine.url.database, 'user': engine.url.username}
+
+    if driver == 'sqlite':
+        try:
+            os.remove(auth['database'])
+        except OSError:
+            pass
+        return
+
+    sqls = {
+        'mysql': [
+            "drop database if exists %(database)s;",
+            "drop user '%(user)s'@'localhost';",
+        ],
+        'postgresql': [
+            "drop database if exists %(database)s;",
+            "drop user if exists %(user)s;",
+        ]
+    }
+
+    try:
+        sql_rows = sqls[driver]
+    except KeyError:
+        raise ValueError('Unsupported RDBMS %s' % driver)
+    sql_query = map(lambda x: x % auth, sql_rows)
+
+    _execute_sql(admin_engine, sql_query, driver)
+
+
+def main():
+    """Controller to handle commands
+
+    ::create: Create test user and database with random names.
+    ::drop: Drop user and database created by previous command.
+    """
+    parser = argparse.ArgumentParser(
+        description='Controller to handle database creation and dropping'
+        ' commands.',
+        epilog='Under normal circumstances is not used directly.'
+        ' Used in .testr.conf to automate test database creation'
+        ' and dropping processes.')
+    subparsers = parser.add_subparsers(
+        help='Subcommands to manipulate temporary test databases.')
+
+    create = subparsers.add_parser(
+        'create',
+        help='Create temporary test '
+        'databases and users.')
+    create.set_defaults(which='create')
+    create.add_argument(
+        'instances_count',
+        type=int,
+        help='Number of databases to create.')
+
+    drop = subparsers.add_parser(
+        'drop',
+        help='Drop temporary test databases and users.')
+    drop.set_defaults(which='drop')
+    drop.add_argument(
+        'instances',
+        nargs='+',
+        help='List of databases uri to be dropped.')
+
+    args = parser.parse_args()
+
+    engine = _get_engine()
+    which = args.which
+
+    if which == "create":
+        for i in range(int(args.instances_count)):
+            print(create_database(engine))
+    elif which == "drop":
+        for db in args.instances:
+            drop_database(engine, db)
+
+
+if __name__ == "__main__":
+    main()

gceapi/openstack/common/db/sqlalchemy/session.py

@@ -0,0 +1,867 @@
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Session Handling for SQLAlchemy backend.
+
+Initializing:
+
+* Call set_defaults with the minimal of the following kwargs:
+    sql_connection, sqlite_db
+
+  Example::
+
+    session.set_defaults(
+        sql_connection="sqlite:///var/lib/gceapi/sqlite.db",
+        sqlite_db="/var/lib/gceapi/sqlite.db")
+
+Recommended ways to use sessions within this framework:
+
+* Don't use them explicitly; this is like running with AUTOCOMMIT=1.
+  model_query() will implicitly use a session when called without one
+  supplied. This is the ideal situation because it will allow queries
+  to be automatically retried if the database connection is interrupted.
+
+    Note: Automatic retry will be enabled in a future patch.
+
+  It is generally fine to issue several queries in a row like this. Even though
+  they may be run in separate transactions and/or separate sessions, each one
+  will see the data from the prior calls. If needed, undo- or rollback-like
+  functionality should be handled at a logical level. For an example, look at
+  the code around quotas and reservation_rollback().
+
+  Examples::
+
+    def get_foo(context, foo):
+        return (model_query(context, models.Foo).
+                filter_by(foo=foo).
+                first())
+
+    def update_foo(context, id, newfoo):
+        (model_query(context, models.Foo).
+                filter_by(id=id).
+                update({'foo': newfoo}))
+
+    def create_foo(context, values):
+        foo_ref = models.Foo()
+        foo_ref.update(values)
+        foo_ref.save()
+        return foo_ref
+
+
+* Within the scope of a single method, keeping all the reads and writes within
+  the context managed by a single session. In this way, the session's __exit__
+  handler will take care of calling flush() and commit() for you.
+  If using this approach, you should not explicitly call flush() or commit().
+  Any error within the context of the session will cause the session to emit
+  a ROLLBACK. Database Errors like IntegrityError will be raised in
+  session's __exit__ handler, and any try/except within the context managed
+  by session will not be triggered. And catching other non-database errors in
+  the session will not trigger the ROLLBACK, so exception handlers should
+  always be outside the session, unless the developer wants to do a partial
+  commit on purpose. If the connection is dropped before this is possible,
+  the database will implicitly roll back the transaction.
+
+     Note: statements in the session scope will not be automatically retried.
+
+  If you create models within the session, they need to be added, but you
+  do not need to call model.save()
+
+  ::
+
+    def create_many_foo(context, foos):
+        session = get_session()
+        with session.begin():
+            for foo in foos:
+                foo_ref = models.Foo()
+                foo_ref.update(foo)
+                session.add(foo_ref)
+
+    def update_bar(context, foo_id, newbar):
+        session = get_session()
+        with session.begin():
+            foo_ref = (model_query(context, models.Foo, session).
+                        filter_by(id=foo_id).
+                        first())
+            (model_query(context, models.Bar, session).
+                        filter_by(id=foo_ref['bar_id']).
+                        update({'bar': newbar}))
+
+  Note: update_bar is a trivially simple example of using "with session.begin".
+  Whereas create_many_foo is a good example of when a transaction is needed,
+  it is always best to use as few queries as possible. The two queries in
+  update_bar can be better expressed using a single query which avoids
+  the need for an explicit transaction. It can be expressed like so::
+
+    def update_bar(context, foo_id, newbar):
+        subq = (model_query(context, models.Foo.id).
+                filter_by(id=foo_id).
+                limit(1).
+                subquery())
+        (model_query(context, models.Bar).
+                filter_by(id=subq.as_scalar()).
+                update({'bar': newbar}))
+
+  For reference, this emits approximately the following SQL statement::
+
+    UPDATE bar SET bar = ${newbar}
+        WHERE id=(SELECT bar_id FROM foo WHERE id = ${foo_id} LIMIT 1);
+
+  Note: create_duplicate_foo is a trivially simple example of catching an
+  exception while using "with session.begin". Here create two duplicate
+  instances with same primary key, must catch the exception out of context
+  managed by a single session:
+
+    def create_duplicate_foo(context):
+        foo1 = models.Foo()
+        foo2 = models.Foo()
+        foo1.id = foo2.id = 1
+        session = get_session()
+        try:
+            with session.begin():
+                session.add(foo1)
+                session.add(foo2)
+        except exception.DBDuplicateEntry as e:
+            handle_error(e)
+
+* Passing an active session between methods. Sessions should only be passed
+  to private methods. The private method must use a subtransaction; otherwise
+  SQLAlchemy will throw an error when you call session.begin() on an existing
+  transaction. Public methods should not accept a session parameter and should
+  not be involved in sessions within the caller's scope.
+
+  Note that this incurs more overhead in SQLAlchemy than the above means
+  due to nesting transactions, and it is not possible to implicitly retry
+  failed database operations when using this approach.
+
+  This also makes code somewhat more difficult to read and debug, because a
+  single database transaction spans more than one method. Error handling
+  becomes less clear in this situation. When this is needed for code clarity,
+  it should be clearly documented.
+
+  ::
+
+    def myfunc(foo):
+        session = get_session()
+        with session.begin():
+            # do some database things
+            bar = _private_func(foo, session)
+        return bar
+
+    def _private_func(foo, session=None):
+        if not session:
+            session = get_session()
+        with session.begin(subtransaction=True):
+            # do some other database things
+        return bar
+
+
+There are some things which it is best to avoid:
+
+* Don't keep a transaction open any longer than necessary.
+
+  This means that your "with session.begin()" block should be as short
+  as possible, while still containing all the related calls for that
+  transaction.
+
+* Avoid "with_lockmode('UPDATE')" when possible.
+
+  In MySQL/InnoDB, when a "SELECT ... FOR UPDATE" query does not match
+  any rows, it will take a gap-lock. This is a form of write-lock on the
+  "gap" where no rows exist, and prevents any other writes to that space.
+  This can effectively prevent any INSERT into a table by locking the gap
+  at the end of the index. Similar problems will occur if the SELECT FOR UPDATE
+  has an overly broad WHERE clause, or doesn't properly use an index.
+
+  One idea proposed at ODS Fall '12 was to use a normal SELECT to test the
+  number of rows matching a query, and if only one row is returned,
+  then issue the SELECT FOR UPDATE.
+
+  The better long-term solution is to use INSERT .. ON DUPLICATE KEY UPDATE.
+  However, this can not be done until the "deleted" columns are removed and
+  proper UNIQUE constraints are added to the tables.
+
+
+Enabling soft deletes:
+
+* To use/enable soft-deletes, the SoftDeleteMixin must be added
+  to your model class. For example::
+
+      class NovaBase(models.SoftDeleteMixin, models.ModelBase):
+          pass
+
+
+Efficient use of soft deletes:
+
+* There are two possible ways to mark a record as deleted::
+
+    model.soft_delete() and query.soft_delete().
+
+  model.soft_delete() method works with single already fetched entry.
+  query.soft_delete() makes only one db request for all entries that correspond
+  to query.
+
+* In almost all cases you should use query.soft_delete(). Some examples::
+
+        def soft_delete_bar():
+            count = model_query(BarModel).find(some_condition).soft_delete()
+            if count == 0:
+                raise Exception("0 entries were soft deleted")
+
+        def complex_soft_delete_with_synchronization_bar(session=None):
+            if session is None:
+                session = get_session()
+            with session.begin(subtransactions=True):
+                count = (model_query(BarModel).
+                            find(some_condition).
+                            soft_delete(synchronize_session=True))
+                            # Here synchronize_session is required, because we
+                            # don't know what is going on in outer session.
+                if count == 0:
+                    raise Exception("0 entries were soft deleted")
+
+* There is only one situation where model.soft_delete() is appropriate: when
+  you fetch a single record, work with it, and mark it as deleted in the same
+  transaction.
+
+  ::
+
+        def soft_delete_bar_model():
+            session = get_session()
+            with session.begin():
+                bar_ref = model_query(BarModel).find(some_condition).first()
+                # Work with bar_ref
+                bar_ref.soft_delete(session=session)
+
+  However, if you need to work with all entries that correspond to query and
+  then soft delete them you should use query.soft_delete() method::
+
+        def soft_delete_multi_models():
+            session = get_session()
+            with session.begin():
+                query = (model_query(BarModel, session=session).
+                            find(some_condition))
+                model_refs = query.all()
+                # Work with model_refs
+                query.soft_delete(synchronize_session=False)
+                # synchronize_session=False should be set if there is no outer
+                # session and these entries are not used after this.
+
+  When working with many rows, it is very important to use query.soft_delete,
+  which issues a single query. Using model.soft_delete(), as in the following
+  example, is very inefficient.
+
+  ::
+
+        for bar_ref in bar_refs:
+            bar_ref.soft_delete(session=session)
+        # This will produce count(bar_refs) db requests.
+"""
+
+import functools
+import os.path
+import re
+import time
+
+from oslo.config import cfg
+import six
+from sqlalchemy import exc as sqla_exc
+from sqlalchemy.interfaces import PoolListener
+import sqlalchemy.orm
+from sqlalchemy.pool import NullPool, StaticPool
+from sqlalchemy.sql.expression import literal_column
+
+from gceapi.openstack.common.db import exception
+from gceapi.openstack.common.gettextutils import _
+from gceapi.openstack.common import log as logging
+from gceapi.openstack.common import timeutils
+
+sqlite_db_opts = [
+    cfg.StrOpt('sqlite_db',
+               default='gceapi.sqlite',
+               help='The file name to use with SQLite'),
+    cfg.BoolOpt('sqlite_synchronous',
+                default=True,
+                help='If True, SQLite uses synchronous mode'),
+]
+
+database_opts = [
+    cfg.StrOpt('connection',
+               default='sqlite:///' +
+                       os.path.abspath(os.path.join(os.path.dirname(__file__),
+                       '../', '$sqlite_db')),
+               help='The SQLAlchemy connection string used to connect to the '
+                    'database',
+               secret=True,
+               deprecated_opts=[cfg.DeprecatedOpt('sql_connection',
+                                                  group='DEFAULT'),
+                                cfg.DeprecatedOpt('sql_connection',
+                                                  group='DATABASE'),
+                                cfg.DeprecatedOpt('connection',
+                                                  group='sql'), ]),
+    cfg.StrOpt('slave_connection',
+               default='',
+               secret=True,
+               help='The SQLAlchemy connection string used to connect to the '
+                    'slave database'),
+    cfg.IntOpt('idle_timeout',
+               default=3600,
+               deprecated_opts=[cfg.DeprecatedOpt('sql_idle_timeout',
+                                                  group='DEFAULT'),
+                                cfg.DeprecatedOpt('sql_idle_timeout',
+                                                  group='DATABASE'),
+                                cfg.DeprecatedOpt('idle_timeout',
+                                                  group='sql')],
+               help='Timeout before idle sql connections are reaped'),
+    cfg.IntOpt('min_pool_size',
+               default=1,
+               deprecated_opts=[cfg.DeprecatedOpt('sql_min_pool_size',
+                                                  group='DEFAULT'),
+                                cfg.DeprecatedOpt('sql_min_pool_size',
+                                                  group='DATABASE')],
+               help='Minimum number of SQL connections to keep open in a '
+                    'pool'),
+    cfg.IntOpt('max_pool_size',
+               default=None,
+               deprecated_opts=[cfg.DeprecatedOpt('sql_max_pool_size',
+                                                  group='DEFAULT'),
+                                cfg.DeprecatedOpt('sql_max_pool_size',
+                                                  group='DATABASE')],
+               help='Maximum number of SQL connections to keep open in a '
+                    'pool'),
+    cfg.IntOpt('max_retries',
+               default=10,
+               deprecated_opts=[cfg.DeprecatedOpt('sql_max_retries',
+                                                  group='DEFAULT'),
+                                cfg.DeprecatedOpt('sql_max_retries',
+                                                  group='DATABASE')],
+               help='Maximum db connection retries during startup. '
+                    '(setting -1 implies an infinite retry count)'),
+    cfg.IntOpt('retry_interval',
+               default=10,
+               deprecated_opts=[cfg.DeprecatedOpt('sql_retry_interval',
+                                                  group='DEFAULT'),
+                                cfg.DeprecatedOpt('reconnect_interval',
+                                                  group='DATABASE')],
+               help='Interval between retries of opening a sql connection'),
+    cfg.IntOpt('max_overflow',
+               default=None,
+               deprecated_opts=[cfg.DeprecatedOpt('sql_max_overflow',
+                                                  group='DEFAULT'),
+                                cfg.DeprecatedOpt('sqlalchemy_max_overflow',
+                                                  group='DATABASE')],
+               help='If set, use this value for max_overflow with sqlalchemy'),
+    cfg.IntOpt('connection_debug',
+               default=0,
+               deprecated_opts=[cfg.DeprecatedOpt('sql_connection_debug',
+                                                  group='DEFAULT')],
+               help='Verbosity of SQL debugging information. 0=None, '
+                    '100=Everything'),
+    cfg.BoolOpt('connection_trace',
+                default=False,
+                deprecated_opts=[cfg.DeprecatedOpt('sql_connection_trace',
+                                                   group='DEFAULT')],
+                help='Add python stack traces to SQL as comment strings'),
+    cfg.IntOpt('pool_timeout',
+               default=None,
+               deprecated_opts=[cfg.DeprecatedOpt('sqlalchemy_pool_timeout',
+                                                  group='DATABASE')],
+               help='If set, use this value for pool_timeout with sqlalchemy'),
+]
+
+CONF = cfg.CONF
+CONF.register_opts(sqlite_db_opts)
+CONF.register_opts(database_opts, 'database')
+
+LOG = logging.getLogger(__name__)
+
+_ENGINE = None
+_MAKER = None
+_SLAVE_ENGINE = None
+_SLAVE_MAKER = None
+
+
+def set_defaults(sql_connection, sqlite_db, max_pool_size=None,
+                 max_overflow=None, pool_timeout=None):
+    """Set defaults for configuration variables."""
+    cfg.set_defaults(database_opts,
+                     connection=sql_connection)
+    cfg.set_defaults(sqlite_db_opts,
+                     sqlite_db=sqlite_db)
+    # Update the QueuePool defaults
+    if max_pool_size is not None:
+        cfg.set_defaults(database_opts,
+                         max_pool_size=max_pool_size)
+    if max_overflow is not None:
+        cfg.set_defaults(database_opts,
+                         max_overflow=max_overflow)
+    if pool_timeout is not None:
+        cfg.set_defaults(database_opts,
+                         pool_timeout=pool_timeout)
+
+
+def cleanup():
+    global _ENGINE, _MAKER
+    global _SLAVE_ENGINE, _SLAVE_MAKER
+
+    if _MAKER:
+        _MAKER.close_all()
+        _MAKER = None
+    if _ENGINE:
+        _ENGINE.dispose()
+        _ENGINE = None
+    if _SLAVE_MAKER:
+        _SLAVE_MAKER.close_all()
+        _SLAVE_MAKER = None
+    if _SLAVE_ENGINE:
+        _SLAVE_ENGINE.dispose()
+        _SLAVE_ENGINE = None
+
+
+class SqliteForeignKeysListener(PoolListener):
+    """Ensures that the foreign key constraints are enforced in SQLite.
+
+    The foreign key constraints are disabled by default in SQLite,
+    so the foreign key constraints will be enabled here for every
+    database connection
+    """
+    def connect(self, dbapi_con, con_record):
+        dbapi_con.execute('pragma foreign_keys=ON')
+
+
+def get_session(autocommit=True, expire_on_commit=False, sqlite_fk=False,
+                slave_session=False, mysql_traditional_mode=False):
+    """Return a SQLAlchemy session."""
+    global _MAKER
+    global _SLAVE_MAKER
+    maker = _MAKER
+
+    if slave_session:
+        maker = _SLAVE_MAKER
+
+    if maker is None:
+        engine = get_engine(sqlite_fk=sqlite_fk, slave_engine=slave_session,
+                            mysql_traditional_mode=mysql_traditional_mode)
+        maker = get_maker(engine, autocommit, expire_on_commit)
+
+    if slave_session:
+        _SLAVE_MAKER = maker
+    else:
+        _MAKER = maker
+
+    session = maker()
+    return session
+
+
+# note(boris-42): In current versions of DB backends unique constraint
+# violation messages follow the structure:
+#
+# sqlite:
+# 1 column - (IntegrityError) column c1 is not unique
+# N columns - (IntegrityError) column c1, c2, ..., N are not unique
+#
+# sqlite since 3.7.16:
+# 1 column - (IntegrityError) UNIQUE constraint failed: k1
+#
+# N columns - (IntegrityError) UNIQUE constraint failed: k1, k2
+#
+# postgres:
+# 1 column - (IntegrityError) duplicate key value violates unique
+#               constraint "users_c1_key"
+# N columns - (IntegrityError) duplicate key value violates unique
+#               constraint "name_of_our_constraint"
+#
+# mysql:
+# 1 column - (IntegrityError) (1062, "Duplicate entry 'value_of_c1' for key
+#               'c1'")
+# N columns - (IntegrityError) (1062, "Duplicate entry 'values joined
+#               with -' for key 'name_of_our_constraint'")
+_DUP_KEY_RE_DB = {
+    "sqlite": (re.compile(r"^.*columns?([^)]+)(is|are)\s+not\s+unique$"),
+               re.compile(r"^.*UNIQUE\s+constraint\s+failed:\s+(.+)$")),
+    "postgresql": (re.compile(r"^.*duplicate\s+key.*\"([^\"]+)\"\s*\n.*$"),),
+    "mysql": (re.compile(r"^.*\(1062,.*'([^\']+)'\"\)$"),)
+}
+
+
+def _raise_if_duplicate_entry_error(integrity_error, engine_name):
+    """Raise exception if two entries are duplicated.
+
+    In this function will be raised DBDuplicateEntry exception if integrity
+    error wrap unique constraint violation.
+    """
+
+    def get_columns_from_uniq_cons_or_name(columns):
+        # note(vsergeyev): UniqueConstraint name convention: "uniq_t0c10c2"
+        #                  where `t` it is table name and columns `c1`, `c2`
+        #                  are in UniqueConstraint.
+        uniqbase = "uniq_"
+        if not columns.startswith(uniqbase):
+            if engine_name == "postgresql":
+                return [columns[columns.index("_") + 1:columns.rindex("_")]]
+            return [columns]
+        return columns[len(uniqbase):].split("0")[1:]
+
+    if engine_name not in ["mysql", "sqlite", "postgresql"]:
+        return
+
+    # FIXME(johannes): The usage of the .message attribute has been
+    # deprecated since Python 2.6. However, the exceptions raised by
+    # SQLAlchemy can differ when using unicode() and accessing .message.
+    # An audit across all three supported engines will be necessary to
+    # ensure there are no regressions.
+    for pattern in _DUP_KEY_RE_DB[engine_name]:
+        match = pattern.match(integrity_error.message)
+        if match:
+            break
+    else:
+        return
+
+    columns = match.group(1)
+
+    if engine_name == "sqlite":
+        columns = columns.strip().split(", ")
+    else:
+        columns = get_columns_from_uniq_cons_or_name(columns)
+    raise exception.DBDuplicateEntry(columns, integrity_error)
+
+
+# NOTE(comstud): In current versions of DB backends, Deadlock violation
+# messages follow the structure:
+#
+# mysql:
+# (OperationalError) (1213, 'Deadlock found when trying to get lock; try '
+#                     'restarting transaction') <query_str> <query_args>
+_DEADLOCK_RE_DB = {
+    "mysql": re.compile(r"^.*\(1213, 'Deadlock.*")
+}
+
+
+def _raise_if_deadlock_error(operational_error, engine_name):
+    """Raise exception on deadlock condition.
+
+    Raise DBDeadlock exception if OperationalError contains a Deadlock
+    condition.
+    """
+    re = _DEADLOCK_RE_DB.get(engine_name)
+    if re is None:
+        return
+    # FIXME(johannes): The usage of the .message attribute has been
+    # deprecated since Python 2.6. However, the exceptions raised by
+    # SQLAlchemy can differ when using unicode() and accessing .message.
+    # An audit across all three supported engines will be necessary to
+    # ensure there are no regressions.
+    m = re.match(operational_error.message)
+    if not m:
+        return
+    raise exception.DBDeadlock(operational_error)
+
+
+def _wrap_db_error(f):
+    @functools.wraps(f)
+    def _wrap(*args, **kwargs):
+        try:
+            return f(*args, **kwargs)
+        except UnicodeEncodeError:
+            raise exception.DBInvalidUnicodeParameter()
+        # note(boris-42): We should catch unique constraint violation and
+        # wrap it by our own DBDuplicateEntry exception. Unique constraint
+        # violation is wrapped by IntegrityError.
+        except sqla_exc.OperationalError as e:
+            _raise_if_deadlock_error(e, get_engine().name)
+            # NOTE(comstud): A lot of code is checking for OperationalError
+            # so let's not wrap it for now.
+            raise
+        except sqla_exc.IntegrityError as e:
+            # note(boris-42): SqlAlchemy doesn't unify errors from different
+            # DBs so we must do this. Also in some tables (for example
+            # instance_types) there are more than one unique constraint. This
+            # means we should get names of columns, which values violate
+            # unique constraint, from error message.
+            _raise_if_duplicate_entry_error(e, get_engine().name)
+            raise exception.DBError(e)
+        except Exception as e:
+            LOG.exception(_('DB exception wrapped.'))
+            raise exception.DBError(e)
+    return _wrap
+
+
+def get_engine(sqlite_fk=False, slave_engine=False,
+               mysql_traditional_mode=False):
+    """Return a SQLAlchemy engine."""
+    global _ENGINE
+    global _SLAVE_ENGINE
+    engine = _ENGINE
+    db_uri = CONF.database.connection
+
+    if slave_engine:
+        engine = _SLAVE_ENGINE
+        db_uri = CONF.database.slave_connection
+
+    if engine is None:
+        engine = create_engine(db_uri, sqlite_fk=sqlite_fk,
+                               mysql_traditional_mode=mysql_traditional_mode)
+    if slave_engine:
+        _SLAVE_ENGINE = engine
+    else:
+        _ENGINE = engine
+
+    return engine
+
+
+def _synchronous_switch_listener(dbapi_conn, connection_rec):
+    """Switch sqlite connections to non-synchronous mode."""
+    dbapi_conn.execute("PRAGMA synchronous = OFF")
+
+
+def _add_regexp_listener(dbapi_con, con_record):
+    """Add REGEXP function to sqlite connections."""
+
+    def regexp(expr, item):
+        reg = re.compile(expr)
+        return reg.search(six.text_type(item)) is not None
+    dbapi_con.create_function('regexp', 2, regexp)
+
+
+def _thread_yield(dbapi_con, con_record):
+    """Ensure other greenthreads get a chance to be executed.
+
+    If we use eventlet.monkey_patch(), eventlet.greenthread.sleep(0) will
+    execute instead of time.sleep(0).
+    Force a context switch. With common database backends (eg MySQLdb and
+    sqlite), there is no implicit yield caused by network I/O since they are
+    implemented by C libraries that eventlet cannot monkey patch.
+    """
+    time.sleep(0)
+
+
+def _ping_listener(engine, dbapi_conn, connection_rec, connection_proxy):
+    """Ensures that MySQL and DB2 connections are alive.
+
+    Borrowed from:
+    http://groups.google.com/group/sqlalchemy/msg/a4ce563d802c929f
+    """
+    cursor = dbapi_conn.cursor()
+    try:
+        ping_sql = 'select 1'
+        if engine.name == 'ibm_db_sa':
+            # DB2 requires a table expression
+            ping_sql = 'select 1 from (values (1)) AS t1'
+        cursor.execute(ping_sql)
+    except Exception as ex:
+        if engine.dialect.is_disconnect(ex, dbapi_conn, cursor):
+            msg = _('Database server has gone away: %s') % ex
+            LOG.warning(msg)
+            raise sqla_exc.DisconnectionError(msg)
+        else:
+            raise
+
+
+def _set_mode_traditional(dbapi_con, connection_rec, connection_proxy):
+    """Set engine mode to 'traditional'.
+
+    Required to prevent silent truncates at insert or update operations
+    under MySQL. By default MySQL truncates inserted string if it longer
+    than a declared field just with warning. That is fraught with data
+    corruption.
+    """
+    dbapi_con.cursor().execute("SET SESSION sql_mode = TRADITIONAL;")
+
+
+def _is_db_connection_error(args):
+    """Return True if error in connecting to db."""
+    # NOTE(adam_g): This is currently MySQL specific and needs to be extended
+    #               to support Postgres and others.
+    # For the db2, the error code is -30081 since the db2 is still not ready
+    conn_err_codes = ('2002', '2003', '2006', '-30081')
+    for err_code in conn_err_codes:
+        if args.find(err_code) != -1:
+            return True
+    return False
+
+
+def create_engine(sql_connection, sqlite_fk=False,
+                  mysql_traditional_mode=False):
+    """Return a new SQLAlchemy engine."""
+    # NOTE(geekinutah): At this point we could be connecting to the normal
+    #                   db handle or the slave db handle. Things like
+    #                   _wrap_db_error aren't going to work well if their
+    #                   backends don't match. Let's check.
+    _assert_matching_drivers()
+    connection_dict = sqlalchemy.engine.url.make_url(sql_connection)
+
+    engine_args = {
+        "pool_recycle": CONF.database.idle_timeout,
+        "echo": False,
+        'convert_unicode': True,
+    }
+
+    # Map our SQL debug level to SQLAlchemy's options
+    if CONF.database.connection_debug >= 100:
+        engine_args['echo'] = 'debug'
+    elif CONF.database.connection_debug >= 50:
+        engine_args['echo'] = True
+
+    if "sqlite" in connection_dict.drivername:
+        if sqlite_fk:
+            engine_args["listeners"] = [SqliteForeignKeysListener()]
+        engine_args["poolclass"] = NullPool
+
+        if CONF.database.connection == "sqlite://":
+            engine_args["poolclass"] = StaticPool
+            engine_args["connect_args"] = {'check_same_thread': False}
+    else:
+        if CONF.database.max_pool_size is not None:
+            engine_args['pool_size'] = CONF.database.max_pool_size
+        if CONF.database.max_overflow is not None:
+            engine_args['max_overflow'] = CONF.database.max_overflow
+        if CONF.database.pool_timeout is not None:
+            engine_args['pool_timeout'] = CONF.database.pool_timeout
+
+    engine = sqlalchemy.create_engine(sql_connection, **engine_args)
+
+    sqlalchemy.event.listen(engine, 'checkin', _thread_yield)
+
+    if engine.name in ['mysql', 'ibm_db_sa']:
+        callback = functools.partial(_ping_listener, engine)
+        sqlalchemy.event.listen(engine, 'checkout', callback)
+        if mysql_traditional_mode:
+            sqlalchemy.event.listen(engine, 'checkout', _set_mode_traditional)
+        else:
+            LOG.warning(_("This application has not enabled MySQL traditional"
+                          " mode, which means silent data corruption may"
+                          " occur. Please encourage the application"
+                          " developers to enable this mode."))
+    elif 'sqlite' in connection_dict.drivername:
+        if not CONF.sqlite_synchronous:
+            sqlalchemy.event.listen(engine, 'connect',
+                                    _synchronous_switch_listener)
+        sqlalchemy.event.listen(engine, 'connect', _add_regexp_listener)
+
+    if (CONF.database.connection_trace and
+            engine.dialect.dbapi.__name__ == 'MySQLdb'):
+        _patch_mysqldb_with_stacktrace_comments()
+
+    try:
+        engine.connect()
+    except sqla_exc.OperationalError as e:
+        if not _is_db_connection_error(e.args[0]):
+            raise
+
+        remaining = CONF.database.max_retries
+        if remaining == -1:
+            remaining = 'infinite'
+        while True:
+            msg = _('SQL connection failed. %s attempts left.')
+            LOG.warning(msg % remaining)
+            if remaining != 'infinite':
+                remaining -= 1
+            time.sleep(CONF.database.retry_interval)
+            try:
+                engine.connect()
+                break
+            except sqla_exc.OperationalError as e:
+                if (remaining != 'infinite' and remaining == 0) or \
+                        not _is_db_connection_error(e.args[0]):
+                    raise
+    return engine
+
+
+class Query(sqlalchemy.orm.query.Query):
+    """Subclass of sqlalchemy.query with soft_delete() method."""
+    def soft_delete(self, synchronize_session='evaluate'):
+        return self.update({'deleted': literal_column('id'),
+                            'updated_at': literal_column('updated_at'),
+                            'deleted_at': timeutils.utcnow()},
+                           synchronize_session=synchronize_session)
+
+
+class Session(sqlalchemy.orm.session.Session):
+    """Custom Session class to avoid SqlAlchemy Session monkey patching."""
+    @_wrap_db_error
+    def query(self, *args, **kwargs):
+        return super(Session, self).query(*args, **kwargs)
+
+    @_wrap_db_error
+    def flush(self, *args, **kwargs):
+        return super(Session, self).flush(*args, **kwargs)
+
+    @_wrap_db_error
+    def execute(self, *args, **kwargs):
+        return super(Session, self).execute(*args, **kwargs)
+
+
+def get_maker(engine, autocommit=True, expire_on_commit=False):
+    """Return a SQLAlchemy sessionmaker using the given engine."""
+    return sqlalchemy.orm.sessionmaker(bind=engine,
+                                       class_=Session,
+                                       autocommit=autocommit,
+                                       expire_on_commit=expire_on_commit,
+                                       query_cls=Query)
+
+
+def _patch_mysqldb_with_stacktrace_comments():
+    """Adds current stack trace as a comment in queries.
+
+    Patches MySQLdb.cursors.BaseCursor._do_query.
+    """
+    import MySQLdb.cursors
+    import traceback
+
+    old_mysql_do_query = MySQLdb.cursors.BaseCursor._do_query
+
+    def _do_query(self, q):
+        stack = ''
+        for filename, line, method, function in traceback.extract_stack():
+            # exclude various common things from trace
+            if filename.endswith('session.py') and method == '_do_query':
+                continue
+            if filename.endswith('api.py') and method == 'wrapper':
+                continue
+            if filename.endswith('utils.py') and method == '_inner':
+                continue
+            if filename.endswith('exception.py') and method == '_wrap':
+                continue
+            # db/api is just a wrapper around db/sqlalchemy/api
+            if filename.endswith('db/api.py'):
+                continue
+            # only trace inside gceapi
+            index = filename.rfind('gceapi')
+            if index == -1:
+                continue
+            stack += "File:%s:%s Method:%s() Line:%s | " \
+                     % (filename[index:], line, method, function)
+
+        # strip trailing " | " from stack
+        if stack:
+            stack = stack[:-3]
+            qq = "%s /* %s */" % (q, stack)
+        else:
+            qq = q
+        old_mysql_do_query(self, qq)
+
+    setattr(MySQLdb.cursors.BaseCursor, '_do_query', _do_query)
+
+
+def _assert_matching_drivers():
+    """Make sure slave handle and normal handle have the same driver."""
+    # NOTE(geekinutah): There's no use case for writing to one backend and
+    #                 reading from another. Who knows what the future holds?
+    if CONF.database.slave_connection == '':
+        return
+
+    normal = sqlalchemy.engine.url.make_url(CONF.database.connection)
+    slave = sqlalchemy.engine.url.make_url(CONF.database.slave_connection)
+    assert normal.drivername == slave.drivername

gceapi/openstack/common/db/sqlalchemy/test_migrations.py

@@ -0,0 +1,269 @@
+# Copyright 2010-2011 OpenStack Foundation
+# Copyright 2012-2013 IBM Corp.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import functools
+import os
+import subprocess
+
+import lockfile
+from six import moves
+import sqlalchemy
+import sqlalchemy.exc
+
+from gceapi.openstack.common.db.sqlalchemy import utils
+from gceapi.openstack.common.gettextutils import _
+from gceapi.openstack.common import log as logging
+from gceapi.openstack.common.py3kcompat import urlutils
+from gceapi.openstack.common import test
+
+LOG = logging.getLogger(__name__)
+
+
+def _have_mysql(user, passwd, database):
+    present = os.environ.get('TEST_MYSQL_PRESENT')
+    if present is None:
+        return utils.is_backend_avail(backend='mysql',
+                                      user=user,
+                                      passwd=passwd,
+                                      database=database)
+    return present.lower() in ('', 'true')
+
+
+def _have_postgresql(user, passwd, database):
+    present = os.environ.get('TEST_POSTGRESQL_PRESENT')
+    if present is None:
+        return utils.is_backend_avail(backend='postgres',
+                                      user=user,
+                                      passwd=passwd,
+                                      database=database)
+    return present.lower() in ('', 'true')
+
+
+def _set_db_lock(lock_path=None, lock_prefix=None):
+    def decorator(f):
+        @functools.wraps(f)
+        def wrapper(*args, **kwargs):
+            try:
+                path = lock_path or os.environ.get("GCEAPI_LOCK_PATH")
+                lock = lockfile.FileLock(os.path.join(path, lock_prefix))
+                with lock:
+                    LOG.debug(_('Got lock "%s"') % f.__name__)
+                    return f(*args, **kwargs)
+            finally:
+                LOG.debug(_('Lock released "%s"') % f.__name__)
+        return wrapper
+    return decorator
+
+
+class BaseMigrationTestCase(test.BaseTestCase):
+    """Base class fort testing of migration utils."""
+
+    def __init__(self, *args, **kwargs):
+        super(BaseMigrationTestCase, self).__init__(*args, **kwargs)
+
+        self.DEFAULT_CONFIG_FILE = os.path.join(os.path.dirname(__file__),
+                                                'test_migrations.conf')
+        # Test machines can set the TEST_MIGRATIONS_CONF variable
+        # to override the location of the config file for migration testing
+        self.CONFIG_FILE_PATH = os.environ.get('TEST_MIGRATIONS_CONF',
+                                               self.DEFAULT_CONFIG_FILE)
+        self.test_databases = {}
+        self.migration_api = None
+
+    def setUp(self):
+        super(BaseMigrationTestCase, self).setUp()
+
+        # Load test databases from the config file. Only do this
+        # once. No need to re-run this on each test...
+        LOG.debug('config_path is %s' % self.CONFIG_FILE_PATH)
+        if os.path.exists(self.CONFIG_FILE_PATH):
+            cp = moves.configparser.RawConfigParser()
+            try:
+                cp.read(self.CONFIG_FILE_PATH)
+                defaults = cp.defaults()
+                for key, value in defaults.items():
+                    self.test_databases[key] = value
+            except moves.configparser.ParsingError as e:
+                self.fail("Failed to read test_migrations.conf config "
+                          "file. Got error: %s" % e)
+        else:
+            self.fail("Failed to find test_migrations.conf config "
+                      "file.")
+
+        self.engines = {}
+        for key, value in self.test_databases.items():
+            self.engines[key] = sqlalchemy.create_engine(value)
+
+        # We start each test case with a completely blank slate.
+        self._reset_databases()
+
+    def tearDown(self):
+        # We destroy the test data store between each test case,
+        # and recreate it, which ensures that we have no side-effects
+        # from the tests
+        self._reset_databases()
+        super(BaseMigrationTestCase, self).tearDown()
+
+    def execute_cmd(self, cmd=None):
+        process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
+                                   stderr=subprocess.STDOUT)
+        output = process.communicate()[0]
+        LOG.debug(output)
+        self.assertEqual(0, process.returncode,
+                         "Failed to run: %s\n%s" % (cmd, output))
+
+    def _reset_pg(self, conn_pieces):
+        (user,
+         password,
+         database,
+         host) = utils.get_db_connection_info(conn_pieces)
+        os.environ['PGPASSWORD'] = password
+        os.environ['PGUSER'] = user
+        # note(boris-42): We must create and drop database, we can't
+        # drop database which we have connected to, so for such
+        # operations there is a special database template1.
+        sqlcmd = ("psql -w -U %(user)s -h %(host)s -c"
+                  " '%(sql)s' -d template1")
+
+        sql = ("drop database if exists %s;") % database
+        droptable = sqlcmd % {'user': user, 'host': host, 'sql': sql}
+        self.execute_cmd(droptable)
+
+        sql = ("create database %s;") % database
+        createtable = sqlcmd % {'user': user, 'host': host, 'sql': sql}
+        self.execute_cmd(createtable)
+
+        os.unsetenv('PGPASSWORD')
+        os.unsetenv('PGUSER')
+
+    @_set_db_lock(lock_prefix='migration_tests-')
+    def _reset_databases(self):
+        for key, engine in self.engines.items():
+            conn_string = self.test_databases[key]
+            conn_pieces = urlutils.urlparse(conn_string)
+            engine.dispose()
+            if conn_string.startswith('sqlite'):
+                # We can just delete the SQLite database, which is
+                # the easiest and cleanest solution
+                db_path = conn_pieces.path.strip('/')
+                if os.path.exists(db_path):
+                    os.unlink(db_path)
+                # No need to recreate the SQLite DB. SQLite will
+                # create it for us if it's not there...
+            elif conn_string.startswith('mysql'):
+                # We can execute the MySQL client to destroy and re-create
+                # the MYSQL database, which is easier and less error-prone
+                # than using SQLAlchemy to do this via MetaData...trust me.
+                (user, password, database, host) = \
+                    utils.get_db_connection_info(conn_pieces)
+                sql = ("drop database if exists %(db)s; "
+                       "create database %(db)s;") % {'db': database}
+                cmd = ("mysql -u \"%(user)s\" -p\"%(password)s\" -h %(host)s "
+                       "-e \"%(sql)s\"") % {'user': user, 'password': password,
+                                            'host': host, 'sql': sql}
+                self.execute_cmd(cmd)
+            elif conn_string.startswith('postgresql'):
+                self._reset_pg(conn_pieces)
+
+
+class WalkVersionsMixin(object):
+    def _walk_versions(self, engine=None, snake_walk=False, downgrade=True):
+        # Determine latest version script from the repo, then
+        # upgrade from 1 through to the latest, with no data
+        # in the databases. This just checks that the schema itself
+        # upgrades successfully.
+
+        # Place the database under version control
+        self.migration_api.version_control(engine, self.REPOSITORY,
+                                           self.INIT_VERSION)
+        self.assertEqual(self.INIT_VERSION,
+                         self.migration_api.db_version(engine,
+                                                       self.REPOSITORY))
+
+        LOG.debug('latest version is %s' % self.REPOSITORY.latest)
+        versions = range(self.INIT_VERSION + 1, self.REPOSITORY.latest + 1)
+
+        for version in versions:
+            # upgrade -> downgrade -> upgrade
+            self._migrate_up(engine, version, with_data=True)
+            if snake_walk:
+                downgraded = self._migrate_down(
+                    engine, version - 1, with_data=True)
+                if downgraded:
+                    self._migrate_up(engine, version)
+
+        if downgrade:
+            # Now walk it back down to 0 from the latest, testing
+            # the downgrade paths.
+            for version in reversed(versions):
+                # downgrade -> upgrade -> downgrade
+                downgraded = self._migrate_down(engine, version - 1)
+
+                if snake_walk and downgraded:
+                    self._migrate_up(engine, version)
+                    self._migrate_down(engine, version - 1)
+
+    def _migrate_down(self, engine, version, with_data=False):
+        try:
+            self.migration_api.downgrade(engine, self.REPOSITORY, version)
+        except NotImplementedError:
+            # NOTE(sirp): some migrations, namely release-level
+            # migrations, don't support a downgrade.
+            return False
+
+        self.assertEqual(
+            version, self.migration_api.db_version(engine, self.REPOSITORY))
+
+        # NOTE(sirp): `version` is what we're downgrading to (i.e. the 'target'
+        # version). So if we have any downgrade checks, they need to be run for
+        # the previous (higher numbered) migration.
+        if with_data:
+            post_downgrade = getattr(
+                self, "_post_downgrade_%03d" % (version + 1), None)
+            if post_downgrade:
+                post_downgrade(engine)
+
+        return True
+
+    def _migrate_up(self, engine, version, with_data=False):
+        """migrate up to a new version of the db.
+
+        We allow for data insertion and post checks at every
+        migration version with special _pre_upgrade_### and
+        _check_### functions in the main test.
+        """
+        # NOTE(sdague): try block is here because it's impossible to debug
+        # where a failed data migration happens otherwise
+        try:
+            if with_data:
+                data = None
+                pre_upgrade = getattr(
+                    self, "_pre_upgrade_%03d" % version, None)
+                if pre_upgrade:
+                    data = pre_upgrade(engine)
+
+            self.migration_api.upgrade(engine, self.REPOSITORY, version)
+            self.assertEqual(version,
+                             self.migration_api.db_version(engine,
+                                                           self.REPOSITORY))
+            if with_data:
+                check = getattr(self, "_check_%03d" % version, None)
+                if check:
+                    check(engine, data)
+        except Exception:
+            LOG.error("Failed to migrate to version %s on engine %s" %
+                      (version, engine))
+            raise

gceapi/openstack/common/db/sqlalchemy/utils.py

@@ -0,0 +1,548 @@
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# Copyright 2010-2011 OpenStack Foundation.
+# Copyright 2012 Justin Santa Barbara
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import re
+
+from migrate.changeset import UniqueConstraint
+import sqlalchemy
+from sqlalchemy import Boolean
+from sqlalchemy import CheckConstraint
+from sqlalchemy import Column
+from sqlalchemy.engine import reflection
+from sqlalchemy.ext.compiler import compiles
+from sqlalchemy import func
+from sqlalchemy import Index
+from sqlalchemy import Integer
+from sqlalchemy import MetaData
+from sqlalchemy.sql.expression import literal_column
+from sqlalchemy.sql.expression import UpdateBase
+from sqlalchemy.sql import select
+from sqlalchemy import String
+from sqlalchemy import Table
+from sqlalchemy.types import NullType
+
+from gceapi.openstack.common.gettextutils import _
+
+from gceapi.openstack.common import log as logging
+from gceapi.openstack.common import timeutils
+
+
+LOG = logging.getLogger(__name__)
+
+_DBURL_REGEX = re.compile(r"[^:]+://([^:]+):([^@]+)@.+")
+
+
+def sanitize_db_url(url):
+    match = _DBURL_REGEX.match(url)
+    if match:
+        return '%s****:****%s' % (url[:match.start(1)], url[match.end(2):])
+    return url
+
+
+class InvalidSortKey(Exception):
+    message = _("Sort key supplied was not valid.")
+
+
+# copy from glance/db/sqlalchemy/api.py
+def paginate_query(query, model, limit, sort_keys, marker=None,
+                   sort_dir=None, sort_dirs=None):
+    """Returns a query with sorting / pagination criteria added.
+
+    Pagination works by requiring a unique sort_key, specified by sort_keys.
+    (If sort_keys is not unique, then we risk looping through values.)
+    We use the last row in the previous page as the 'marker' for pagination.
+    So we must return values that follow the passed marker in the order.
+    With a single-valued sort_key, this would be easy: sort_key > X.
+    With a compound-values sort_key, (k1, k2, k3) we must do this to repeat
+    the lexicographical ordering:
+    (k1 > X1) or (k1 == X1 && k2 > X2) or (k1 == X1 && k2 == X2 && k3 > X3)
+
+    We also have to cope with different sort_directions.
+
+    Typically, the id of the last row is used as the client-facing pagination
+    marker, then the actual marker object must be fetched from the db and
+    passed in to us as marker.
+
+    :param query: the query object to which we should add paging/sorting
+    :param model: the ORM model class
+    :param limit: maximum number of items to return
+    :param sort_keys: array of attributes by which results should be sorted
+    :param marker: the last item of the previous page; we returns the next
+                    results after this value.
+    :param sort_dir: direction in which results should be sorted (asc, desc)
+    :param sort_dirs: per-column array of sort_dirs, corresponding to sort_keys
+
+    :rtype: sqlalchemy.orm.query.Query
+    :return: The query with sorting/pagination added.
+    """
+
+    if 'id' not in sort_keys:
+        # TODO(justinsb): If this ever gives a false-positive, check
+        # the actual primary key, rather than assuming its id
+        LOG.warning(_('Id not in sort_keys; is sort_keys unique?'))
+
+    assert(not (sort_dir and sort_dirs))
+
+    # Default the sort direction to ascending
+    if sort_dirs is None and sort_dir is None:
+        sort_dir = 'asc'
+
+    # Ensure a per-column sort direction
+    if sort_dirs is None:
+        sort_dirs = [sort_dir for _sort_key in sort_keys]
+
+    assert(len(sort_dirs) == len(sort_keys))
+
+    # Add sorting
+    for current_sort_key, current_sort_dir in zip(sort_keys, sort_dirs):
+        try:
+            sort_dir_func = {
+                'asc': sqlalchemy.asc,
+                'desc': sqlalchemy.desc,
+            }[current_sort_dir]
+        except KeyError:
+            raise ValueError(_("Unknown sort direction, "
+                               "must be 'desc' or 'asc'"))
+        try:
+            sort_key_attr = getattr(model, current_sort_key)
+        except AttributeError:
+            raise InvalidSortKey()
+        query = query.order_by(sort_dir_func(sort_key_attr))
+
+    # Add pagination
+    if marker is not None:
+        marker_values = []
+        for sort_key in sort_keys:
+            v = getattr(marker, sort_key)
+            marker_values.append(v)
+
+        # Build up an array of sort criteria as in the docstring
+        criteria_list = []
+        for i in range(len(sort_keys)):
+            crit_attrs = []
+            for j in range(i):
+                model_attr = getattr(model, sort_keys[j])
+                crit_attrs.append((model_attr == marker_values[j]))
+
+            model_attr = getattr(model, sort_keys[i])
+            if sort_dirs[i] == 'desc':
+                crit_attrs.append((model_attr < marker_values[i]))
+            else:
+                crit_attrs.append((model_attr > marker_values[i]))
+
+            criteria = sqlalchemy.sql.and_(*crit_attrs)
+            criteria_list.append(criteria)
+
+        f = sqlalchemy.sql.or_(*criteria_list)
+        query = query.filter(f)
+
+    if limit is not None:
+        query = query.limit(limit)
+
+    return query
+
+
+def get_table(engine, name):
+    """Returns an sqlalchemy table dynamically from db.
+
+    Needed because the models don't work for us in migrations
+    as models will be far out of sync with the current data.
+    """
+    metadata = MetaData()
+    metadata.bind = engine
+    return Table(name, metadata, autoload=True)
+
+
+class InsertFromSelect(UpdateBase):
+    """Form the base for `INSERT INTO table (SELECT ... )` statement."""
+    def __init__(self, table, select):
+        self.table = table
+        self.select = select
+
+
+@compiles(InsertFromSelect)
+def visit_insert_from_select(element, compiler, **kw):
+    """Form the `INSERT INTO table (SELECT ... )` statement."""
+    return "INSERT INTO %s %s" % (
+        compiler.process(element.table, asfrom=True),
+        compiler.process(element.select))
+
+
+class ColumnError(Exception):
+    """Error raised when no column or an invalid column is found."""
+
+
+def _get_not_supported_column(col_name_col_instance, column_name):
+    try:
+        column = col_name_col_instance[column_name]
+    except KeyError:
+        msg = _("Please specify column %s in col_name_col_instance "
+                "param. It is required because column has unsupported "
+                "type by sqlite).")
+        raise ColumnError(msg % column_name)
+
+    if not isinstance(column, Column):
+        msg = _("col_name_col_instance param has wrong type of "
+                "column instance for column %s It should be instance "
+                "of sqlalchemy.Column.")
+        raise ColumnError(msg % column_name)
+    return column
+
+
+def drop_unique_constraint(migrate_engine, table_name, uc_name, *columns,
+                           **col_name_col_instance):
+    """Drop unique constraint from table.
+
+    This method drops UC from table and works for mysql, postgresql and sqlite.
+    In mysql and postgresql we are able to use "alter table" construction.
+    Sqlalchemy doesn't support some sqlite column types and replaces their
+    type with NullType in metadata. We process these columns and replace
+    NullType with the correct column type.
+
+    :param migrate_engine: sqlalchemy engine
+    :param table_name:     name of table that contains uniq constraint.
+    :param uc_name:        name of uniq constraint that will be dropped.
+    :param columns:        columns that are in uniq constraint.
+    :param col_name_col_instance:   contains pair column_name=column_instance.
+                            column_instance is instance of Column. These params
+                            are required only for columns that have unsupported
+                            types by sqlite. For example BigInteger.
+    """
+
+    meta = MetaData()
+    meta.bind = migrate_engine
+    t = Table(table_name, meta, autoload=True)
+
+    if migrate_engine.name == "sqlite":
+        override_cols = [
+            _get_not_supported_column(col_name_col_instance, col.name)
+            for col in t.columns
+            if isinstance(col.type, NullType)
+        ]
+        for col in override_cols:
+            t.columns.replace(col)
+
+    uc = UniqueConstraint(*columns, table=t, name=uc_name)
+    uc.drop()
+
+
+def drop_old_duplicate_entries_from_table(migrate_engine, table_name,
+                                          use_soft_delete, *uc_column_names):
+    """Drop all old rows having the same values for columns in uc_columns.
+
+    This method drop (or mark ad `deleted` if use_soft_delete is True) old
+    duplicate rows form table with name `table_name`.
+
+    :param migrate_engine:  Sqlalchemy engine
+    :param table_name:      Table with duplicates
+    :param use_soft_delete: If True - values will be marked as `deleted`,
+                            if False - values will be removed from table
+    :param uc_column_names: Unique constraint columns
+    """
+    meta = MetaData()
+    meta.bind = migrate_engine
+
+    table = Table(table_name, meta, autoload=True)
+    columns_for_group_by = [table.c[name] for name in uc_column_names]
+
+    columns_for_select = [func.max(table.c.id)]
+    columns_for_select.extend(columns_for_group_by)
+
+    duplicated_rows_select = select(columns_for_select,
+                                    group_by=columns_for_group_by,
+                                    having=func.count(table.c.id) > 1)
+
+    for row in migrate_engine.execute(duplicated_rows_select):
+        # NOTE(boris-42): Do not remove row that has the biggest ID.
+        delete_condition = table.c.id != row[0]
+        is_none = None  # workaround for pyflakes
+        delete_condition &= table.c.deleted_at == is_none
+        for name in uc_column_names:
+            delete_condition &= table.c[name] == row[name]
+
+        rows_to_delete_select = select([table.c.id]).where(delete_condition)
+        for row in migrate_engine.execute(rows_to_delete_select).fetchall():
+            LOG.info(_("Deleting duplicated row with id: %(id)s from table: "
+                       "%(table)s") % dict(id=row[0], table=table_name))
+
+        if use_soft_delete:
+            delete_statement = table.update().\
+                where(delete_condition).\
+                values({
+                    'deleted': literal_column('id'),
+                    'updated_at': literal_column('updated_at'),
+                    'deleted_at': timeutils.utcnow()
+                })
+        else:
+            delete_statement = table.delete().where(delete_condition)
+        migrate_engine.execute(delete_statement)
+
+
+def _get_default_deleted_value(table):
+    if isinstance(table.c.id.type, Integer):
+        return 0
+    if isinstance(table.c.id.type, String):
+        return ""
+    raise ColumnError(_("Unsupported id columns type"))
+
+
+def _restore_indexes_on_deleted_columns(migrate_engine, table_name, indexes):
+    table = get_table(migrate_engine, table_name)
+
+    insp = reflection.Inspector.from_engine(migrate_engine)
+    real_indexes = insp.get_indexes(table_name)
+    existing_index_names = dict(
+        [(index['name'], index['column_names']) for index in real_indexes])
+
+    # NOTE(boris-42): Restore indexes on `deleted` column
+    for index in indexes:
+        if 'deleted' not in index['column_names']:
+            continue
+        name = index['name']
+        if name in existing_index_names:
+            column_names = [table.c[c] for c in existing_index_names[name]]
+            old_index = Index(name, *column_names, unique=index["unique"])
+            old_index.drop(migrate_engine)
+
+        column_names = [table.c[c] for c in index['column_names']]
+        new_index = Index(index["name"], *column_names, unique=index["unique"])
+        new_index.create(migrate_engine)
+
+
+def change_deleted_column_type_to_boolean(migrate_engine, table_name,
+                                          **col_name_col_instance):
+    if migrate_engine.name == "sqlite":
+        return _change_deleted_column_type_to_boolean_sqlite(
+            migrate_engine, table_name, **col_name_col_instance)
+    insp = reflection.Inspector.from_engine(migrate_engine)
+    indexes = insp.get_indexes(table_name)
+
+    table = get_table(migrate_engine, table_name)
+
+    old_deleted = Column('old_deleted', Boolean, default=False)
+    old_deleted.create(table, populate_default=False)
+
+    table.update().\
+        where(table.c.deleted == table.c.id).\
+        values(old_deleted=True).\
+        execute()
+
+    table.c.deleted.drop()
+    table.c.old_deleted.alter(name="deleted")
+
+    _restore_indexes_on_deleted_columns(migrate_engine, table_name, indexes)
+
+
+def _change_deleted_column_type_to_boolean_sqlite(migrate_engine, table_name,
+                                                  **col_name_col_instance):
+    insp = reflection.Inspector.from_engine(migrate_engine)
+    table = get_table(migrate_engine, table_name)
+
+    columns = []
+    for column in table.columns:
+        column_copy = None
+        if column.name != "deleted":
+            if isinstance(column.type, NullType):
+                column_copy = _get_not_supported_column(col_name_col_instance,
+                                                        column.name)
+            else:
+                column_copy = column.copy()
+        else:
+            column_copy = Column('deleted', Boolean, default=0)
+        columns.append(column_copy)
+
+    constraints = [constraint.copy() for constraint in table.constraints]
+
+    meta = table.metadata
+    new_table = Table(table_name + "__tmp__", meta,
+                      *(columns + constraints))
+    new_table.create()
+
+    indexes = []
+    for index in insp.get_indexes(table_name):
+        column_names = [new_table.c[c] for c in index['column_names']]
+        indexes.append(Index(index["name"], *column_names,
+                             unique=index["unique"]))
+
+    c_select = []
+    for c in table.c:
+        if c.name != "deleted":
+            c_select.append(c)
+        else:
+            c_select.append(table.c.deleted == table.c.id)
+
+    ins = InsertFromSelect(new_table, select(c_select))
+    migrate_engine.execute(ins)
+
+    table.drop()
+    [index.create(migrate_engine) for index in indexes]
+
+    new_table.rename(table_name)
+    new_table.update().\
+        where(new_table.c.deleted == new_table.c.id).\
+        values(deleted=True).\
+        execute()
+
+
+def change_deleted_column_type_to_id_type(migrate_engine, table_name,
+                                          **col_name_col_instance):
+    if migrate_engine.name == "sqlite":
+        return _change_deleted_column_type_to_id_type_sqlite(
+            migrate_engine, table_name, **col_name_col_instance)
+    insp = reflection.Inspector.from_engine(migrate_engine)
+    indexes = insp.get_indexes(table_name)
+
+    table = get_table(migrate_engine, table_name)
+
+    new_deleted = Column('new_deleted', table.c.id.type,
+                         default=_get_default_deleted_value(table))
+    new_deleted.create(table, populate_default=True)
+
+    deleted = True  # workaround for pyflakes
+    table.update().\
+        where(table.c.deleted == deleted).\
+        values(new_deleted=table.c.id).\
+        execute()
+    table.c.deleted.drop()
+    table.c.new_deleted.alter(name="deleted")
+
+    _restore_indexes_on_deleted_columns(migrate_engine, table_name, indexes)
+
+
+def _change_deleted_column_type_to_id_type_sqlite(migrate_engine, table_name,
+                                                  **col_name_col_instance):
+    # NOTE(boris-42): sqlaclhemy-migrate can't drop column with check
+    #                 constraints in sqlite DB and our `deleted` column has
+    #                 2 check constraints. So there is only one way to remove
+    #                 these constraints:
+    #                 1) Create new table with the same columns, constraints
+    #                 and indexes. (except deleted column).
+    #                 2) Copy all data from old to new table.
+    #                 3) Drop old table.
+    #                 4) Rename new table to old table name.
+    insp = reflection.Inspector.from_engine(migrate_engine)
+    meta = MetaData(bind=migrate_engine)
+    table = Table(table_name, meta, autoload=True)
+    default_deleted_value = _get_default_deleted_value(table)
+
+    columns = []
+    for column in table.columns:
+        column_copy = None
+        if column.name != "deleted":
+            if isinstance(column.type, NullType):
+                column_copy = _get_not_supported_column(col_name_col_instance,
+                                                        column.name)
+            else:
+                column_copy = column.copy()
+        else:
+            column_copy = Column('deleted', table.c.id.type,
+                                 default=default_deleted_value)
+        columns.append(column_copy)
+
+    def is_deleted_column_constraint(constraint):
+        # NOTE(boris-42): There is no other way to check is CheckConstraint
+        #                 associated with deleted column.
+        if not isinstance(constraint, CheckConstraint):
+            return False
+        sqltext = str(constraint.sqltext)
+        return (sqltext.endswith("deleted in (0, 1)") or
+                sqltext.endswith("deleted IN (:deleted_1, :deleted_2)"))
+
+    constraints = []
+    for constraint in table.constraints:
+        if not is_deleted_column_constraint(constraint):
+            constraints.append(constraint.copy())
+
+    new_table = Table(table_name + "__tmp__", meta,
+                      *(columns + constraints))
+    new_table.create()
+
+    indexes = []
+    for index in insp.get_indexes(table_name):
+        column_names = [new_table.c[c] for c in index['column_names']]
+        indexes.append(Index(index["name"], *column_names,
+                             unique=index["unique"]))
+
+    ins = InsertFromSelect(new_table, table.select())
+    migrate_engine.execute(ins)
+
+    table.drop()
+    [index.create(migrate_engine) for index in indexes]
+
+    new_table.rename(table_name)
+    deleted = True  # workaround for pyflakes
+    new_table.update().\
+        where(new_table.c.deleted == deleted).\
+        values(deleted=new_table.c.id).\
+        execute()
+
+    # NOTE(boris-42): Fix value of deleted column: False -> "" or 0.
+    deleted = False  # workaround for pyflakes
+    new_table.update().\
+        where(new_table.c.deleted == deleted).\
+        values(deleted=default_deleted_value).\
+        execute()
+
+
+def get_connect_string(backend, database, user=None, passwd=None):
+    """Get database connection
+
+    Try to get a connection with a very specific set of values, if we get
+    these then we'll run the tests, otherwise they are skipped
+    """
+    args = {'backend': backend,
+            'user': user,
+            'passwd': passwd,
+            'database': database}
+    if backend == 'sqlite':
+        template = '%(backend)s:///%(database)s'
+    else:
+        template = "%(backend)s://%(user)s:%(passwd)s@localhost/%(database)s"
+    return template % args
+
+
+def is_backend_avail(backend, database, user=None, passwd=None):
+    try:
+        connect_uri = get_connect_string(backend=backend,
+                                         database=database,
+                                         user=user,
+                                         passwd=passwd)
+        engine = sqlalchemy.create_engine(connect_uri)
+        connection = engine.connect()
+    except Exception:
+        # intentionally catch all to handle exceptions even if we don't
+        # have any backend code loaded.
+        return False
+    else:
+        connection.close()
+        engine.dispose()
+        return True
+
+
+def get_db_connection_info(conn_pieces):
+    database = conn_pieces.path.strip('/')
+    loc_pieces = conn_pieces.netloc.split('@')
+    host = loc_pieces[1]
+
+    auth_pieces = loc_pieces[0].split(':')
+    user = auth_pieces[0]
+    password = ""
+    if len(auth_pieces) > 1:
+        password = auth_pieces[1].strip()
+
+    return (user, password, database, host)

gceapi/openstack/common/eventlet_backdoor.py

@@ -0,0 +1,144 @@
+# Copyright (c) 2012 OpenStack Foundation.
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from __future__ import print_function
+
+import errno
+import gc
+import os
+import pprint
+import socket
+import sys
+import traceback
+
+import eventlet
+import eventlet.backdoor
+import greenlet
+from oslo.config import cfg
+
+from gceapi.openstack.common.gettextutils import _
+from gceapi.openstack.common import log as logging
+
+help_for_backdoor_port = (
+    "Acceptable values are 0, <port>, and <start>:<end>, where 0 results "
+    "in listening on a random tcp port number; <port> results in listening "
+    "on the specified port number (and not enabling backdoor if that port "
+    "is in use); and <start>:<end> results in listening on the smallest "
+    "unused port number within the specified range of port numbers.  The "
+    "chosen port is displayed in the service's log file.")
+eventlet_backdoor_opts = [
+    cfg.StrOpt('backdoor_port',
+               default=None,
+               help="Enable eventlet backdoor.  %s" % help_for_backdoor_port)
+]
+
+CONF = cfg.CONF
+CONF.register_opts(eventlet_backdoor_opts)
+LOG = logging.getLogger(__name__)
+
+
+class EventletBackdoorConfigValueError(Exception):
+    def __init__(self, port_range, help_msg, ex):
+        msg = ('Invalid backdoor_port configuration %(range)s: %(ex)s. '
+               '%(help)s' %
+               {'range': port_range, 'ex': ex, 'help': help_msg})
+        super(EventletBackdoorConfigValueError, self).__init__(msg)
+        self.port_range = port_range
+
+
+def _dont_use_this():
+    print("Don't use this, just disconnect instead")
+
+
+def _find_objects(t):
+    return [o for o in gc.get_objects() if isinstance(o, t)]
+
+
+def _print_greenthreads():
+    for i, gt in enumerate(_find_objects(greenlet.greenlet)):
+        print(i, gt)
+        traceback.print_stack(gt.gr_frame)
+        print()
+
+
+def _print_nativethreads():
+    for threadId, stack in sys._current_frames().items():
+        print(threadId)
+        traceback.print_stack(stack)
+        print()
+
+
+def _parse_port_range(port_range):
+    if ':' not in port_range:
+        start, end = port_range, port_range
+    else:
+        start, end = port_range.split(':', 1)
+    try:
+        start, end = int(start), int(end)
+        if end < start:
+            raise ValueError
+        return start, end
+    except ValueError as ex:
+        raise EventletBackdoorConfigValueError(port_range, ex,
+                                               help_for_backdoor_port)
+
+
+def _listen(host, start_port, end_port, listen_func):
+    try_port = start_port
+    while True:
+        try:
+            return listen_func((host, try_port))
+        except socket.error as exc:
+            if (exc.errno != errno.EADDRINUSE or
+               try_port >= end_port):
+                raise
+            try_port += 1
+
+
+def initialize_if_enabled():
+    backdoor_locals = {
+        'exit': _dont_use_this,      # So we don't exit the entire process
+        'quit': _dont_use_this,      # So we don't exit the entire process
+        'fo': _find_objects,
+        'pgt': _print_greenthreads,
+        'pnt': _print_nativethreads,
+    }
+
+    if CONF.backdoor_port is None:
+        return None
+
+    start_port, end_port = _parse_port_range(str(CONF.backdoor_port))
+
+    # NOTE(johannes): The standard sys.displayhook will print the value of
+    # the last expression and set it to __builtin__._, which overwrites
+    # the __builtin__._ that gettext sets. Let's switch to using pprint
+    # since it won't interact poorly with gettext, and it's easier to
+    # read the output too.
+    def displayhook(val):
+        if val is not None:
+            pprint.pprint(val)
+    sys.displayhook = displayhook
+
+    sock = _listen('localhost', start_port, end_port, eventlet.listen)
+
+    # In the case of backdoor port being zero, a port number is assigned by
+    # listen().  In any case, pull the port number out here.
+    port = sock.getsockname()[1]
+    LOG.info(_('Eventlet backdoor listening on %(port)s for process %(pid)d') %
+             {'port': port, 'pid': os.getpid()})
+    eventlet.spawn_n(eventlet.backdoor.backdoor_server, sock,
+                     locals=backdoor_locals)
+    return port

gceapi/openstack/common/excutils.py

@@ -0,0 +1,99 @@
+# Copyright 2011 OpenStack Foundation.
+# Copyright 2012, Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Exception related utilities.
+"""
+
+import logging
+import sys
+import time
+import traceback
+
+import six
+
+from gceapi.openstack.common.gettextutils import _
+
+
+class save_and_reraise_exception(object):
+    """Save current exception, run some code and then re-raise.
+
+    In some cases the exception context can be cleared, resulting in None
+    being attempted to be re-raised after an exception handler is run. This
+    can happen when eventlet switches greenthreads or when running an
+    exception handler, code raises and catches an exception. In both
+    cases the exception context will be cleared.
+
+    To work around this, we save the exception state, run handler code, and
+    then re-raise the original exception. If another exception occurs, the
+    saved exception is logged and the new exception is re-raised.
+
+    In some cases the caller may not want to re-raise the exception, and
+    for those circumstances this context provides a reraise flag that
+    can be used to suppress the exception.  For example::
+
+      except Exception:
+          with save_and_reraise_exception() as ctxt:
+              decide_if_need_reraise()
+              if not should_be_reraised:
+                  ctxt.reraise = False
+    """
+    def __init__(self):
+        self.reraise = True
+
+    def __enter__(self):
+        self.type_, self.value, self.tb, = sys.exc_info()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if exc_type is not None:
+            logging.error(_('Original exception being dropped: %s'),
+                          traceback.format_exception(self.type_,
+                                                     self.value,
+                                                     self.tb))
+            return False
+        if self.reraise:
+            six.reraise(self.type_, self.value, self.tb)
+
+
+def forever_retry_uncaught_exceptions(infunc):
+    def inner_func(*args, **kwargs):
+        last_log_time = 0
+        last_exc_message = None
+        exc_count = 0
+        while True:
+            try:
+                return infunc(*args, **kwargs)
+            except Exception as exc:
+                this_exc_message = six.u(str(exc))
+                if this_exc_message == last_exc_message:
+                    exc_count += 1
+                else:
+                    exc_count = 1
+                # Do not log any more frequently than once a minute unless
+                # the exception message changes
+                cur_time = int(time.time())
+                if (cur_time - last_log_time > 60 or
+                        this_exc_message != last_exc_message):
+                    logging.exception(
+                        _('Unexpected exception occurred %d time(s)... '
+                          'retrying.') % exc_count)
+                    last_log_time = cur_time
+                    last_exc_message = this_exc_message
+                    exc_count = 0
+                # This should be a very rare event. In case it isn't, do
+                # a sleep.
+                time.sleep(1)
+    return inner_func

gceapi/openstack/common/gettextutils.py

@@ -0,0 +1,440 @@
+# Copyright 2012 Red Hat, Inc.
+# Copyright 2013 IBM Corp.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+gettext for openstack-common modules.
+
+Usual usage in an openstack.common module:
+
+    from gceapi.openstack.common.gettextutils import _
+"""
+
+import copy
+import gettext
+import locale
+from logging import handlers
+import os
+import re
+
+from babel import localedata
+import six
+
+_localedir = os.environ.get('gceapi'.upper() + '_LOCALEDIR')
+_t = gettext.translation('gceapi', localedir=_localedir, fallback=True)
+
+_AVAILABLE_LANGUAGES = {}
+USE_LAZY = False
+
+
+def enable_lazy():
+    """Convenience function for configuring _() to use lazy gettext
+
+    Call this at the start of execution to enable the gettextutils._
+    function to use lazy gettext functionality. This is useful if
+    your project is importing _ directly instead of using the
+    gettextutils.install() way of importing the _ function.
+    """
+    global USE_LAZY
+    USE_LAZY = True
+
+
+def _(msg):
+    if USE_LAZY:
+        return Message(msg, domain='gceapi')
+    else:
+        if six.PY3:
+            return _t.gettext(msg)
+        return _t.ugettext(msg)
+
+
+def install(domain, lazy=False):
+    """Install a _() function using the given translation domain.
+
+    Given a translation domain, install a _() function using gettext's
+    install() function.
+
+    The main difference from gettext.install() is that we allow
+    overriding the default localedir (e.g. /usr/share/locale) using
+    a translation-domain-specific environment variable (e.g.
+    NOVA_LOCALEDIR).
+
+    :param domain: the translation domain
+    :param lazy: indicates whether or not to install the lazy _() function.
+                 The lazy _() introduces a way to do deferred translation
+                 of messages by installing a _ that builds Message objects,
+                 instead of strings, which can then be lazily translated into
+                 any available locale.
+    """
+    if lazy:
+        # NOTE(mrodden): Lazy gettext functionality.
+        #
+        # The following introduces a deferred way to do translations on
+        # messages in OpenStack. We override the standard _() function
+        # and % (format string) operation to build Message objects that can
+        # later be translated when we have more information.
+        def _lazy_gettext(msg):
+            """Create and return a Message object.
+
+            Lazy gettext function for a given domain, it is a factory method
+            for a project/module to get a lazy gettext function for its own
+            translation domain (i.e. nova, glance, cinder, etc.)
+
+            Message encapsulates a string so that we can translate
+            it later when needed.
+            """
+            return Message(msg, domain=domain)
+
+        from six import moves
+        moves.builtins.__dict__['_'] = _lazy_gettext
+    else:
+        localedir = '%s_LOCALEDIR' % domain.upper()
+        if six.PY3:
+            gettext.install(domain,
+                            localedir=os.environ.get(localedir))
+        else:
+            gettext.install(domain,
+                            localedir=os.environ.get(localedir),
+                            unicode=True)
+
+
+class Message(six.text_type):
+    """A Message object is a unicode object that can be translated.
+
+    Translation of Message is done explicitly using the translate() method.
+    For all non-translation intents and purposes, a Message is simply unicode,
+    and can be treated as such.
+    """
+
+    def __new__(cls, msgid, msgtext=None, params=None,
+                domain='gceapi', *args):
+        """Create a new Message object.
+
+        In order for translation to work gettext requires a message ID, this
+        msgid will be used as the base unicode text. It is also possible
+        for the msgid and the base unicode text to be different by passing
+        the msgtext parameter.
+        """
+        # If the base msgtext is not given, we use the default translation
+        # of the msgid (which is in English) just in case the system locale is
+        # not English, so that the base text will be in that locale by default.
+        if not msgtext:
+            msgtext = Message._translate_msgid(msgid, domain)
+        # We want to initialize the parent unicode with the actual object that
+        # would have been plain unicode if 'Message' was not enabled.
+        msg = super(Message, cls).__new__(cls, msgtext)
+        msg.msgid = msgid
+        msg.domain = domain
+        msg.params = params
+        return msg
+
+    def translate(self, desired_locale=None):
+        """Translate this message to the desired locale.
+
+        :param desired_locale: The desired locale to translate the message to,
+                               if no locale is provided the message will be
+                               translated to the system's default locale.
+
+        :returns: the translated message in unicode
+        """
+
+        translated_message = Message._translate_msgid(self.msgid,
+                                                      self.domain,
+                                                      desired_locale)
+        if self.params is None:
+            # No need for more translation
+            return translated_message
+
+        # This Message object may have been formatted with one or more
+        # Message objects as substitution arguments, given either as a single
+        # argument, part of a tuple, or as one or more values in a dictionary.
+        # When translating this Message we need to translate those Messages too
+        translated_params = _translate_args(self.params, desired_locale)
+
+        translated_message = translated_message % translated_params
+
+        return translated_message
+
+    @staticmethod
+    def _translate_msgid(msgid, domain, desired_locale=None):
+        if not desired_locale:
+            system_locale = locale.getdefaultlocale()
+            # If the system locale is not available to the runtime use English
+            if not system_locale[0]:
+                desired_locale = 'en_US'
+            else:
+                desired_locale = system_locale[0]
+
+        locale_dir = os.environ.get(domain.upper() + '_LOCALEDIR')
+        lang = gettext.translation(domain,
+                                   localedir=locale_dir,
+                                   languages=[desired_locale],
+                                   fallback=True)
+        if six.PY3:
+            translator = lang.gettext
+        else:
+            translator = lang.ugettext
+
+        translated_message = translator(msgid)
+        return translated_message
+
+    def __mod__(self, other):
+        # When we mod a Message we want the actual operation to be performed
+        # by the parent class (i.e. unicode()), the only thing  we do here is
+        # save the original msgid and the parameters in case of a translation
+        params = self._sanitize_mod_params(other)
+        unicode_mod = super(Message, self).__mod__(params)
+        modded = Message(self.msgid,
+                         msgtext=unicode_mod,
+                         params=params,
+                         domain=self.domain)
+        return modded
+
+    def _sanitize_mod_params(self, other):
+        """Sanitize the object being modded with this Message.
+
+        - Add support for modding 'None' so translation supports it
+        - Trim the modded object, which can be a large dictionary, to only
+        those keys that would actually be used in a translation
+        - Snapshot the object being modded, in case the message is
+        translated, it will be used as it was when the Message was created
+        """
+        if other is None:
+            params = (other,)
+        elif isinstance(other, dict):
+            params = self._trim_dictionary_parameters(other)
+        else:
+            params = self._copy_param(other)
+        return params
+
+    def _trim_dictionary_parameters(self, dict_param):
+        """Return a dict that only has matching entries in the msgid."""
+        # NOTE(luisg): Here we trim down the dictionary passed as parameters
+        # to avoid carrying a lot of unnecessary weight around in the message
+        # object, for example if someone passes in Message() % locals() but
+        # only some params are used, and additionally we prevent errors for
+        # non-deepcopyable objects by unicoding() them.
+
+        # Look for %(param) keys in msgid;
+        # Skip %% and deal with the case where % is first character on the line
+        keys = re.findall('(?:[^%]|^)?%\((\w*)\)[a-z]', self.msgid)
+
+        # If we don't find any %(param) keys but have a %s
+        if not keys and re.findall('(?:[^%]|^)%[a-z]', self.msgid):
+            # Apparently the full dictionary is the parameter
+            params = self._copy_param(dict_param)
+        else:
+            params = {}
+            # Save our existing parameters as defaults to protect
+            # ourselves from losing values if we are called through an
+            # (erroneous) chain that builds a valid Message with
+            # arguments, and then does something like "msg % kwds"
+            # where kwds is an empty dictionary.
+            src = {}
+            if isinstance(self.params, dict):
+                src.update(self.params)
+            src.update(dict_param)
+            for key in keys:
+                params[key] = self._copy_param(src[key])
+
+        return params
+
+    def _copy_param(self, param):
+        try:
+            return copy.deepcopy(param)
+        except TypeError:
+            # Fallback to casting to unicode this will handle the
+            # python code-like objects that can't be deep-copied
+            return six.text_type(param)
+
+    def __add__(self, other):
+        msg = _('Message objects do not support addition.')
+        raise TypeError(msg)
+
+    def __radd__(self, other):
+        return self.__add__(other)
+
+    def __str__(self):
+        # NOTE(luisg): Logging in python 2.6 tries to str() log records,
+        # and it expects specifically a UnicodeError in order to proceed.
+        msg = _('Message objects do not support str() because they may '
+                'contain non-ascii characters. '
+                'Please use unicode() or translate() instead.')
+        raise UnicodeError(msg)
+
+
+def get_available_languages(domain):
+    """Lists the available languages for the given translation domain.
+
+    :param domain: the domain to get languages for
+    """
+    if domain in _AVAILABLE_LANGUAGES:
+        return copy.copy(_AVAILABLE_LANGUAGES[domain])
+
+    localedir = '%s_LOCALEDIR' % domain.upper()
+    find = lambda x: gettext.find(domain,
+                                  localedir=os.environ.get(localedir),
+                                  languages=[x])
+
+    # NOTE(mrodden): en_US should always be available (and first in case
+    # order matters) since our in-line message strings are en_US
+    language_list = ['en_US']
+    # NOTE(luisg): Babel <1.0 used a function called list(), which was
+    # renamed to locale_identifiers() in >=1.0, the requirements master list
+    # requires >=0.9.6, uncapped, so defensively work with both. We can remove
+    # this check when the master list updates to >=1.0, and update all projects
+    list_identifiers = (getattr(localedata, 'list', None) or
+                        getattr(localedata, 'locale_identifiers'))
+    locale_identifiers = list_identifiers()
+
+    for i in locale_identifiers:
+        if find(i) is not None:
+            language_list.append(i)
+
+    # NOTE(luisg): Babel>=1.0,<1.3 has a bug where some OpenStack supported
+    # locales (e.g. 'zh_CN', and 'zh_TW') aren't supported even though they
+    # are perfectly legitimate locales:
+    #     https://github.com/mitsuhiko/babel/issues/37
+    # In Babel 1.3 they fixed the bug and they support these locales, but
+    # they are still not explicitly "listed" by locale_identifiers().
+    # That is  why we add the locales here explicitly if necessary so that
+    # they are listed as supported.
+    aliases = {'zh': 'zh_CN',
+               'zh_Hant_HK': 'zh_HK',
+               'zh_Hant': 'zh_TW',
+               'fil': 'tl_PH'}
+    for (locale, alias) in six.iteritems(aliases):
+        if locale in language_list and alias not in language_list:
+            language_list.append(alias)
+
+    _AVAILABLE_LANGUAGES[domain] = language_list
+    return copy.copy(language_list)
+
+
+def translate(obj, desired_locale=None):
+    """Gets the translated unicode representation of the given object.
+
+    If the object is not translatable it is returned as-is.
+    If the locale is None the object is translated to the system locale.
+
+    :param obj: the object to translate
+    :param desired_locale: the locale to translate the message to, if None the
+                           default system locale will be used
+    :returns: the translated object in unicode, or the original object if
+              it could not be translated
+    """
+    message = obj
+    if not isinstance(message, Message):
+        # If the object to translate is not already translatable,
+        # let's first get its unicode representation
+        message = six.text_type(obj)
+    if isinstance(message, Message):
+        # Even after unicoding() we still need to check if we are
+        # running with translatable unicode before translating
+        return message.translate(desired_locale)
+    return obj
+
+
+def _translate_args(args, desired_locale=None):
+    """Translates all the translatable elements of the given arguments object.
+
+    This method is used for translating the translatable values in method
+    arguments which include values of tuples or dictionaries.
+    If the object is not a tuple or a dictionary the object itself is
+    translated if it is translatable.
+
+    If the locale is None the object is translated to the system locale.
+
+    :param args: the args to translate
+    :param desired_locale: the locale to translate the args to, if None the
+                           default system locale will be used
+    :returns: a new args object with the translated contents of the original
+    """
+    if isinstance(args, tuple):
+        return tuple(translate(v, desired_locale) for v in args)
+    if isinstance(args, dict):
+        translated_dict = {}
+        for (k, v) in six.iteritems(args):
+            translated_v = translate(v, desired_locale)
+            translated_dict[k] = translated_v
+        return translated_dict
+    return translate(args, desired_locale)
+
+
+class TranslationHandler(handlers.MemoryHandler):
+    """Handler that translates records before logging them.
+
+    The TranslationHandler takes a locale and a target logging.Handler object
+    to forward LogRecord objects to after translating them. This handler
+    depends on Message objects being logged, instead of regular strings.
+
+    The handler can be configured declaratively in the logging.conf as follows:
+
+        [handlers]
+        keys = translatedlog, translator
+
+        [handler_translatedlog]
+        class = handlers.WatchedFileHandler
+        args = ('/var/log/api-localized.log',)
+        formatter = context
+
+        [handler_translator]
+        class = openstack.common.log.TranslationHandler
+        target = translatedlog
+        args = ('zh_CN',)
+
+    If the specified locale is not available in the system, the handler will
+    log in the default locale.
+    """
+
+    def __init__(self, locale=None, target=None):
+        """Initialize a TranslationHandler
+
+        :param locale: locale to use for translating messages
+        :param target: logging.Handler object to forward
+                       LogRecord objects to after translation
+        """
+        # NOTE(luisg): In order to allow this handler to be a wrapper for
+        # other handlers, such as a FileHandler, and still be able to
+        # configure it using logging.conf, this handler has to extend
+        # MemoryHandler because only the MemoryHandlers' logging.conf
+        # parsing is implemented such that it accepts a target handler.
+        handlers.MemoryHandler.__init__(self, capacity=0, target=target)
+        self.locale = locale
+
+    def setFormatter(self, fmt):
+        self.target.setFormatter(fmt)
+
+    def emit(self, record):
+        # We save the message from the original record to restore it
+        # after translation, so other handlers are not affected by this
+        original_msg = record.msg
+        original_args = record.args
+
+        try:
+            self._translate_and_log_record(record)
+        finally:
+            record.msg = original_msg
+            record.args = original_args
+
+    def _translate_and_log_record(self, record):
+        record.msg = translate(record.msg, self.locale)
+
+        # In addition to translating the message, we also need to translate
+        # arguments that were passed to the log method that were not part
+        # of the main message e.g., log.info(_('Some message %s'), this_one))
+        record.args = _translate_args(record.args, self.locale)
+
+        self.target.emit(record)

gceapi/openstack/common/importutils.py

@@ -0,0 +1,66 @@
+# Copyright 2011 OpenStack Foundation.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Import related utilities and helper functions.
+"""
+
+import sys
+import traceback
+
+
+def import_class(import_str):
+    """Returns a class from a string including module and class."""
+    mod_str, _sep, class_str = import_str.rpartition('.')
+    try:
+        __import__(mod_str)
+        return getattr(sys.modules[mod_str], class_str)
+    except (ValueError, AttributeError):
+        raise ImportError('Class %s cannot be found (%s)' %
+                          (class_str,
+                           traceback.format_exception(*sys.exc_info())))
+
+
+def import_object(import_str, *args, **kwargs):
+    """Import a class and return an instance of it."""
+    return import_class(import_str)(*args, **kwargs)
+
+
+def import_object_ns(name_space, import_str, *args, **kwargs):
+    """Tries to import object from default namespace.
+
+    Imports a class and return an instance of it, first by trying
+    to find the class in a default namespace, then failing back to
+    a full path if not found in the default namespace.
+    """
+    import_value = "%s.%s" % (name_space, import_str)
+    try:
+        return import_class(import_value)(*args, **kwargs)
+    except ImportError:
+        return import_class(import_str)(*args, **kwargs)
+
+
+def import_module(import_str):
+    """Import a module."""
+    __import__(import_str)
+    return sys.modules[import_str]
+
+
+def try_import(import_str, default=None):
+    """Try to import a module and if it fails return default."""
+    try:
+        return import_module(import_str)
+    except ImportError:
+        return default

gceapi/openstack/common/jsonutils.py

@@ -0,0 +1,182 @@
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# Copyright 2011 Justin Santa Barbara
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+'''
+JSON related utilities.
+
+This module provides a few things:
+
+    1) A handy function for getting an object down to something that can be
+    JSON serialized.  See to_primitive().
+
+    2) Wrappers around loads() and dumps().  The dumps() wrapper will
+    automatically use to_primitive() for you if needed.
+
+    3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson
+    is available.
+'''
+
+
+import datetime
+import functools
+import inspect
+import itertools
+import json
+try:
+    import xmlrpclib
+except ImportError:
+    # NOTE(jaypipes): xmlrpclib was renamed to xmlrpc.client in Python3
+    #                 however the function and object call signatures
+    #                 remained the same. This whole try/except block should
+    #                 be removed and replaced with a call to six.moves once
+    #                 six 1.4.2 is released. See http://bit.ly/1bqrVzu
+    import xmlrpc.client as xmlrpclib
+
+import six
+
+from gceapi.openstack.common import gettextutils
+from gceapi.openstack.common import importutils
+from gceapi.openstack.common import timeutils
+
+netaddr = importutils.try_import("netaddr")
+
+_nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod,
+                     inspect.isfunction, inspect.isgeneratorfunction,
+                     inspect.isgenerator, inspect.istraceback, inspect.isframe,
+                     inspect.iscode, inspect.isbuiltin, inspect.isroutine,
+                     inspect.isabstract]
+
+_simple_types = (six.string_types + six.integer_types
+                 + (type(None), bool, float))
+
+
+def to_primitive(value, convert_instances=False, convert_datetime=True,
+                 level=0, max_depth=3):
+    """Convert a complex object into primitives.
+
+    Handy for JSON serialization. We can optionally handle instances,
+    but since this is a recursive function, we could have cyclical
+    data structures.
+
+    To handle cyclical data structures we could track the actual objects
+    visited in a set, but not all objects are hashable. Instead we just
+    track the depth of the object inspections and don't go too deep.
+
+    Therefore, convert_instances=True is lossy ... be aware.
+
+    """
+    # handle obvious types first - order of basic types determined by running
+    # full tests on nova project, resulting in the following counts:
+    # 572754 <type 'NoneType'>
+    # 460353 <type 'int'>
+    # 379632 <type 'unicode'>
+    # 274610 <type 'str'>
+    # 199918 <type 'dict'>
+    # 114200 <type 'datetime.datetime'>
+    #  51817 <type 'bool'>
+    #  26164 <type 'list'>
+    #   6491 <type 'float'>
+    #    283 <type 'tuple'>
+    #     19 <type 'long'>
+    if isinstance(value, _simple_types):
+        return value
+
+    if isinstance(value, datetime.datetime):
+        if convert_datetime:
+            return timeutils.strtime(value)
+        else:
+            return value
+
+    # value of itertools.count doesn't get caught by nasty_type_tests
+    # and results in infinite loop when list(value) is called.
+    if type(value) == itertools.count:
+        return six.text_type(value)
+
+    # FIXME(vish): Workaround for LP bug 852095. Without this workaround,
+    #              tests that raise an exception in a mocked method that
+    #              has a @wrap_exception with a notifier will fail. If
+    #              we up the dependency to 0.5.4 (when it is released) we
+    #              can remove this workaround.
+    if getattr(value, '__module__', None) == 'mox':
+        return 'mock'
+
+    if level > max_depth:
+        return '?'
+
+    # The try block may not be necessary after the class check above,
+    # but just in case ...
+    try:
+        recursive = functools.partial(to_primitive,
+                                      convert_instances=convert_instances,
+                                      convert_datetime=convert_datetime,
+                                      level=level,
+                                      max_depth=max_depth)
+        if isinstance(value, dict):
+            return dict((k, recursive(v)) for k, v in six.iteritems(value))
+        elif isinstance(value, (list, tuple)):
+            return [recursive(lv) for lv in value]
+
+        # It's not clear why xmlrpclib created their own DateTime type, but
+        # for our purposes, make it a datetime type which is explicitly
+        # handled
+        if isinstance(value, xmlrpclib.DateTime):
+            value = datetime.datetime(*tuple(value.timetuple())[:6])
+
+        if convert_datetime and isinstance(value, datetime.datetime):
+            return timeutils.strtime(value)
+        elif isinstance(value, gettextutils.Message):
+            return value.data
+        elif hasattr(value, 'iteritems'):
+            return recursive(dict(value.iteritems()), level=level + 1)
+        elif hasattr(value, '__iter__'):
+            return recursive(list(value))
+        elif convert_instances and hasattr(value, '__dict__'):
+            # Likely an instance of something. Watch for cycles.
+            # Ignore class member vars.
+            return recursive(value.__dict__, level=level + 1)
+        elif netaddr and isinstance(value, netaddr.IPAddress):
+            return six.text_type(value)
+        else:
+            if any(test(value) for test in _nasty_type_tests):
+                return six.text_type(value)
+            return value
+    except TypeError:
+        # Class objects are tricky since they may define something like
+        # __iter__ defined but it isn't callable as list().
+        return six.text_type(value)
+
+
+def dumps(value, default=to_primitive, **kwargs):
+    return json.dumps(value, default=default, **kwargs)
+
+
+def loads(s):
+    return json.loads(s)
+
+
+def load(s):
+    return json.load(s)
+
+
+try:
+    import anyjson
+except ImportError:
+    pass
+else:
+    anyjson._modules.append((__name__, 'dumps', TypeError,
+                                       'loads', ValueError, 'load'))
+    anyjson.force_implementation(__name__)

gceapi/openstack/common/local.py

@@ -0,0 +1,45 @@
+# Copyright 2011 OpenStack Foundation.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Local storage of variables using weak references"""
+
+import threading
+import weakref
+
+
+class WeakLocal(threading.local):
+    def __getattribute__(self, attr):
+        rval = super(WeakLocal, self).__getattribute__(attr)
+        if rval:
+            # NOTE(mikal): this bit is confusing. What is stored is a weak
+            # reference, not the value itself. We therefore need to lookup
+            # the weak reference and return the inner value here.
+            rval = rval()
+        return rval
+
+    def __setattr__(self, attr, value):
+        value = weakref.ref(value)
+        return super(WeakLocal, self).__setattr__(attr, value)
+
+
+# NOTE(mikal): the name "store" should be deprecated in the future
+store = WeakLocal()
+
+# A "weak" store uses weak references and allows an object to fall out of scope
+# when it falls out of scope in the code that uses the thread local storage. A
+# "strong" store will hold a reference to the object so that it never falls out
+# of scope.
+weak_store = WeakLocal()
+strong_store = threading.local()

gceapi/openstack/common/log.py

@@ -0,0 +1,657 @@
+# Copyright 2011 OpenStack Foundation.
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Openstack logging handler.
+
+This module adds to logging functionality by adding the option to specify
+a context object when calling the various log methods.  If the context object
+is not specified, default formatting is used. Additionally, an instance uuid
+may be passed as part of the log message, which is intended to make it easier
+for admins to find messages related to a specific instance.
+
+It also allows setting of formatting information through conf.
+
+"""
+
+import inspect
+import itertools
+import logging
+import logging.config
+import logging.handlers
+import os
+import re
+import sys
+import traceback
+
+from oslo.config import cfg
+import six
+from six import moves
+
+from gceapi.openstack.common.gettextutils import _
+from gceapi.openstack.common import importutils
+from gceapi.openstack.common import jsonutils
+from gceapi.openstack.common import local
+
+
+_DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
+
+_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
+
+# NOTE(ldbragst): Let's build a list of regex objects using the list of
+# _SANITIZE_KEYS we already have. This way, we only have to add the new key
+# to the list of _SANITIZE_KEYS and we can generate regular expressions
+# for XML and JSON automatically.
+_SANITIZE_PATTERNS = []
+_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
+                    r'(<%(key)s>).*?(</%(key)s>)',
+                    r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
+                    r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])']
+
+for key in _SANITIZE_KEYS:
+    for pattern in _FORMAT_PATTERNS:
+        reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
+        _SANITIZE_PATTERNS.append(reg_ex)
+
+
+common_cli_opts = [
+    cfg.BoolOpt('debug',
+                short='d',
+                default=False,
+                help='Print debugging output (set logging level to '
+                     'DEBUG instead of default WARNING level).'),
+    cfg.BoolOpt('verbose',
+                short='v',
+                default=False,
+                help='Print more verbose output (set logging level to '
+                     'INFO instead of default WARNING level).'),
+]
+
+logging_cli_opts = [
+    cfg.StrOpt('log-config-append',
+               metavar='PATH',
+               deprecated_name='log-config',
+               help='The name of logging configuration file. It does not '
+                    'disable existing loggers, but just appends specified '
+                    'logging configuration to any other existing logging '
+                    'options. Please see the Python logging module '
+                    'documentation for details on logging configuration '
+                    'files.'),
+    cfg.StrOpt('log-format',
+               default=None,
+               metavar='FORMAT',
+               help='DEPRECATED. '
+                    'A logging.Formatter log message format string which may '
+                    'use any of the available logging.LogRecord attributes. '
+                    'This option is deprecated.  Please use '
+                    'logging_context_format_string and '
+                    'logging_default_format_string instead.'),
+    cfg.StrOpt('log-date-format',
+               default=_DEFAULT_LOG_DATE_FORMAT,
+               metavar='DATE_FORMAT',
+               help='Format string for %%(asctime)s in log records. '
+                    'Default: %(default)s'),
+    cfg.StrOpt('log-file',
+               metavar='PATH',
+               deprecated_name='logfile',
+               help='(Optional) Name of log file to output to. '
+                    'If no default is set, logging will go to stdout.'),
+    cfg.StrOpt('log-dir',
+               deprecated_name='logdir',
+               help='(Optional) The base directory used for relative '
+                    '--log-file paths'),
+    cfg.BoolOpt('use-syslog',
+                default=False,
+                help='Use syslog for logging. '
+                     'Existing syslog format is DEPRECATED during I, '
+                     'and then will be changed in J to honor RFC5424'),
+    cfg.BoolOpt('use-syslog-rfc-format',
+                # TODO(bogdando) remove or use True after existing
+                #    syslog format deprecation in J
+                default=False,
+                help='(Optional) Use syslog rfc5424 format for logging. '
+                     'If enabled, will add APP-NAME (RFC5424) before the '
+                     'MSG part of the syslog message.  The old format '
+                     'without APP-NAME is deprecated in I, '
+                     'and will be removed in J.'),
+    cfg.StrOpt('syslog-log-facility',
+               default='LOG_USER',
+               help='Syslog facility to receive log lines')
+]
+
+generic_log_opts = [
+    cfg.BoolOpt('use_stderr',
+                default=True,
+                help='Log output to standard error')
+]
+
+log_opts = [
+    cfg.StrOpt('logging_context_format_string',
+               default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
+                       '%(name)s [%(request_id)s %(user_identity)s] '
+                       '%(instance)s%(message)s',
+               help='Format string to use for log messages with context'),
+    cfg.StrOpt('logging_default_format_string',
+               default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
+                       '%(name)s [-] %(instance)s%(message)s',
+               help='Format string to use for log messages without context'),
+    cfg.StrOpt('logging_debug_format_suffix',
+               default='%(funcName)s %(pathname)s:%(lineno)d',
+               help='Data to append to log format when level is DEBUG'),
+    cfg.StrOpt('logging_exception_prefix',
+               default='%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s '
+               '%(instance)s',
+               help='Prefix each line of exception output with this format'),
+    cfg.ListOpt('default_log_levels',
+                default=[
+                    'amqp=WARN',
+                    'amqplib=WARN',
+                    'boto=WARN',
+                    'qpid=WARN',
+                    'sqlalchemy=WARN',
+                    'suds=INFO',
+                    'iso8601=WARN',
+                    'requests.packages.urllib3.connectionpool=WARN'
+                ],
+                help='List of logger=LEVEL pairs'),
+    cfg.BoolOpt('publish_errors',
+                default=False,
+                help='Publish error events'),
+    cfg.BoolOpt('fatal_deprecations',
+                default=False,
+                help='Make deprecations fatal'),
+
+    # NOTE(mikal): there are two options here because sometimes we are handed
+    # a full instance (and could include more information), and other times we
+    # are just handed a UUID for the instance.
+    cfg.StrOpt('instance_format',
+               default='[instance: %(uuid)s] ',
+               help='If an instance is passed with the log message, format '
+                    'it like this'),
+    cfg.StrOpt('instance_uuid_format',
+               default='[instance: %(uuid)s] ',
+               help='If an instance UUID is passed with the log message, '
+                    'format it like this'),
+]
+
+CONF = cfg.CONF
+CONF.register_cli_opts(common_cli_opts)
+CONF.register_cli_opts(logging_cli_opts)
+CONF.register_opts(generic_log_opts)
+CONF.register_opts(log_opts)
+
+# our new audit level
+# NOTE(jkoelker) Since we synthesized an audit level, make the logging
+#                module aware of it so it acts like other levels.
+logging.AUDIT = logging.INFO + 1
+logging.addLevelName(logging.AUDIT, 'AUDIT')
+
+
+try:
+    NullHandler = logging.NullHandler
+except AttributeError:  # NOTE(jkoelker) NullHandler added in Python 2.7
+    class NullHandler(logging.Handler):
+        def handle(self, record):
+            pass
+
+        def emit(self, record):
+            pass
+
+        def createLock(self):
+            self.lock = None
+
+
+def _dictify_context(context):
+    if context is None:
+        return None
+    if not isinstance(context, dict) and getattr(context, 'to_dict', None):
+        context = context.to_dict()
+    return context
+
+
+def _get_binary_name():
+    return os.path.basename(inspect.stack()[-1][1])
+
+
+def _get_log_file_path(binary=None):
+    logfile = CONF.log_file
+    logdir = CONF.log_dir
+
+    if logfile and not logdir:
+        return logfile
+
+    if logfile and logdir:
+        return os.path.join(logdir, logfile)
+
+    if logdir:
+        binary = binary or _get_binary_name()
+        return '%s.log' % (os.path.join(logdir, binary),)
+
+    return None
+
+
+def mask_password(message, secret="***"):
+    """Replace password with 'secret' in message.
+
+    :param message: The string which includes security information.
+    :param secret: value with which to replace passwords.
+    :returns: The unicode value of message with the password fields masked.
+
+    For example:
+
+    >>> mask_password("'adminPass' : 'aaaaa'")
+    "'adminPass' : '***'"
+    >>> mask_password("'admin_pass' : 'aaaaa'")
+    "'admin_pass' : '***'"
+    >>> mask_password('"password" : "aaaaa"')
+    '"password" : "***"'
+    >>> mask_password("'original_password' : 'aaaaa'")
+    "'original_password' : '***'"
+    >>> mask_password("u'original_password' :   u'aaaaa'")
+    "u'original_password' :   u'***'"
+    """
+    message = six.text_type(message)
+
+    # NOTE(ldbragst): Check to see if anything in message contains any key
+    # specified in _SANITIZE_KEYS, if not then just return the message since
+    # we don't have to mask any passwords.
+    if not any(key in message for key in _SANITIZE_KEYS):
+        return message
+
+    secret = r'\g<1>' + secret + r'\g<2>'
+    for pattern in _SANITIZE_PATTERNS:
+        message = re.sub(pattern, secret, message)
+    return message
+
+
+class BaseLoggerAdapter(logging.LoggerAdapter):
+
+    def audit(self, msg, *args, **kwargs):
+        self.log(logging.AUDIT, msg, *args, **kwargs)
+
+
+class LazyAdapter(BaseLoggerAdapter):
+    def __init__(self, name='unknown', version='unknown'):
+        self._logger = None
+        self.extra = {}
+        self.name = name
+        self.version = version
+
+    @property
+    def logger(self):
+        if not self._logger:
+            self._logger = getLogger(self.name, self.version)
+        return self._logger
+
+
+class ContextAdapter(BaseLoggerAdapter):
+    warn = logging.LoggerAdapter.warning
+
+    def __init__(self, logger, project_name, version_string):
+        self.logger = logger
+        self.project = project_name
+        self.version = version_string
+
+    @property
+    def handlers(self):
+        return self.logger.handlers
+
+    def deprecated(self, msg, *args, **kwargs):
+        stdmsg = _("Deprecated: %s") % msg
+        if CONF.fatal_deprecations:
+            self.critical(stdmsg, *args, **kwargs)
+            raise DeprecatedConfig(msg=stdmsg)
+        else:
+            self.warn(stdmsg, *args, **kwargs)
+
+    def process(self, msg, kwargs):
+        # NOTE(mrodden): catch any Message/other object and
+        #                coerce to unicode before they can get
+        #                to the python logging and possibly
+        #                cause string encoding trouble
+        if not isinstance(msg, six.string_types):
+            msg = six.text_type(msg)
+
+        if 'extra' not in kwargs:
+            kwargs['extra'] = {}
+        extra = kwargs['extra']
+
+        context = kwargs.pop('context', None)
+        if not context:
+            context = getattr(local.store, 'context', None)
+        if context:
+            extra.update(_dictify_context(context))
+
+        instance = kwargs.pop('instance', None)
+        instance_uuid = (extra.get('instance_uuid', None) or
+                         kwargs.pop('instance_uuid', None))
+        instance_extra = ''
+        if instance:
+            instance_extra = CONF.instance_format % instance
+        elif instance_uuid:
+            instance_extra = (CONF.instance_uuid_format
+                              % {'uuid': instance_uuid})
+        extra['instance'] = instance_extra
+
+        extra.setdefault('user_identity', kwargs.pop('user_identity', None))
+
+        extra['project'] = self.project
+        extra['version'] = self.version
+        extra['extra'] = extra.copy()
+        return msg, kwargs
+
+
+class JSONFormatter(logging.Formatter):
+    def __init__(self, fmt=None, datefmt=None):
+        # NOTE(jkoelker) we ignore the fmt argument, but its still there
+        #                since logging.config.fileConfig passes it.
+        self.datefmt = datefmt
+
+    def formatException(self, ei, strip_newlines=True):
+        lines = traceback.format_exception(*ei)
+        if strip_newlines:
+            lines = [moves.filter(
+                lambda x: x,
+                line.rstrip().splitlines()) for line in lines]
+            lines = list(itertools.chain(*lines))
+        return lines
+
+    def format(self, record):
+        message = {'message': record.getMessage(),
+                   'asctime': self.formatTime(record, self.datefmt),
+                   'name': record.name,
+                   'msg': record.msg,
+                   'args': record.args,
+                   'levelname': record.levelname,
+                   'levelno': record.levelno,
+                   'pathname': record.pathname,
+                   'filename': record.filename,
+                   'module': record.module,
+                   'lineno': record.lineno,
+                   'funcname': record.funcName,
+                   'created': record.created,
+                   'msecs': record.msecs,
+                   'relative_created': record.relativeCreated,
+                   'thread': record.thread,
+                   'thread_name': record.threadName,
+                   'process_name': record.processName,
+                   'process': record.process,
+                   'traceback': None}
+
+        if hasattr(record, 'extra'):
+            message['extra'] = record.extra
+
+        if record.exc_info:
+            message['traceback'] = self.formatException(record.exc_info)
+
+        return jsonutils.dumps(message)
+
+
+def _create_logging_excepthook(product_name):
+    def logging_excepthook(exc_type, value, tb):
+        extra = {}
+        if CONF.verbose or CONF.debug:
+            extra['exc_info'] = (exc_type, value, tb)
+        getLogger(product_name).critical(
+            "".join(traceback.format_exception_only(exc_type, value)),
+            **extra)
+    return logging_excepthook
+
+
+class LogConfigError(Exception):
+
+    message = _('Error loading logging config %(log_config)s: %(err_msg)s')
+
+    def __init__(self, log_config, err_msg):
+        self.log_config = log_config
+        self.err_msg = err_msg
+
+    def __str__(self):
+        return self.message % dict(log_config=self.log_config,
+                                   err_msg=self.err_msg)
+
+
+def _load_log_config(log_config_append):
+    try:
+        logging.config.fileConfig(log_config_append,
+                                  disable_existing_loggers=False)
+    except moves.configparser.Error as exc:
+        raise LogConfigError(log_config_append, str(exc))
+
+
+def setup(product_name):
+    """Setup logging."""
+    if CONF.log_config_append:
+        _load_log_config(CONF.log_config_append)
+    else:
+        _setup_logging_from_conf()
+    sys.excepthook = _create_logging_excepthook(product_name)
+
+
+def set_defaults(logging_context_format_string):
+    cfg.set_defaults(log_opts,
+                     logging_context_format_string=
+                     logging_context_format_string)
+
+
+def _find_facility_from_conf():
+    facility_names = logging.handlers.SysLogHandler.facility_names
+    facility = getattr(logging.handlers.SysLogHandler,
+                       CONF.syslog_log_facility,
+                       None)
+
+    if facility is None and CONF.syslog_log_facility in facility_names:
+        facility = facility_names.get(CONF.syslog_log_facility)
+
+    if facility is None:
+        valid_facilities = facility_names.keys()
+        consts = ['LOG_AUTH', 'LOG_AUTHPRIV', 'LOG_CRON', 'LOG_DAEMON',
+                  'LOG_FTP', 'LOG_KERN', 'LOG_LPR', 'LOG_MAIL', 'LOG_NEWS',
+                  'LOG_AUTH', 'LOG_SYSLOG', 'LOG_USER', 'LOG_UUCP',
+                  'LOG_LOCAL0', 'LOG_LOCAL1', 'LOG_LOCAL2', 'LOG_LOCAL3',
+                  'LOG_LOCAL4', 'LOG_LOCAL5', 'LOG_LOCAL6', 'LOG_LOCAL7']
+        valid_facilities.extend(consts)
+        raise TypeError(_('syslog facility must be one of: %s') %
+                        ', '.join("'%s'" % fac
+                                  for fac in valid_facilities))
+
+    return facility
+
+
+class RFCSysLogHandler(logging.handlers.SysLogHandler):
+    def __init__(self, *args, **kwargs):
+        self.binary_name = _get_binary_name()
+        super(RFCSysLogHandler, self).__init__(*args, **kwargs)
+
+    def format(self, record):
+        msg = super(RFCSysLogHandler, self).format(record)
+        msg = self.binary_name + ' ' + msg
+        return msg
+
+
+def _setup_logging_from_conf():
+    log_root = getLogger(None).logger
+    for handler in log_root.handlers:
+        log_root.removeHandler(handler)
+
+    if CONF.use_syslog:
+        facility = _find_facility_from_conf()
+        # TODO(bogdando) use the format provided by RFCSysLogHandler
+        #   after existing syslog format deprecation in J
+        if CONF.use_syslog_rfc_format:
+            syslog = RFCSysLogHandler(address='/dev/log',
+                                      facility=facility)
+        else:
+            syslog = logging.handlers.SysLogHandler(address='/dev/log',
+                                                    facility=facility)
+        log_root.addHandler(syslog)
+
+    logpath = _get_log_file_path()
+    if logpath:
+        filelog = logging.handlers.WatchedFileHandler(logpath)
+        log_root.addHandler(filelog)
+
+    if CONF.use_stderr:
+        streamlog = ColorHandler()
+        log_root.addHandler(streamlog)
+
+    elif not logpath:
+        # pass sys.stdout as a positional argument
+        # python2.6 calls the argument strm, in 2.7 it's stream
+        streamlog = logging.StreamHandler(sys.stdout)
+        log_root.addHandler(streamlog)
+
+    if CONF.publish_errors:
+        handler = importutils.import_object(
+            "gceapi.openstack.common.log_handler.PublishErrorsHandler",
+            logging.ERROR)
+        log_root.addHandler(handler)
+
+    datefmt = CONF.log_date_format
+    for handler in log_root.handlers:
+        # NOTE(alaski): CONF.log_format overrides everything currently.  This
+        # should be deprecated in favor of context aware formatting.
+        if CONF.log_format:
+            handler.setFormatter(logging.Formatter(fmt=CONF.log_format,
+                                                   datefmt=datefmt))
+            log_root.info('Deprecated: log_format is now deprecated and will '
+                          'be removed in the next release')
+        else:
+            handler.setFormatter(ContextFormatter(datefmt=datefmt))
+
+    if CONF.debug:
+        log_root.setLevel(logging.DEBUG)
+    elif CONF.verbose:
+        log_root.setLevel(logging.INFO)
+    else:
+        log_root.setLevel(logging.WARNING)
+
+    for pair in CONF.default_log_levels:
+        mod, _sep, level_name = pair.partition('=')
+        level = logging.getLevelName(level_name)
+        logger = logging.getLogger(mod)
+        logger.setLevel(level)
+
+_loggers = {}
+
+
+def getLogger(name='unknown', version='unknown'):
+    if name not in _loggers:
+        _loggers[name] = ContextAdapter(logging.getLogger(name),
+                                        name,
+                                        version)
+    return _loggers[name]
+
+
+def getLazyLogger(name='unknown', version='unknown'):
+    """Returns lazy logger.
+
+    Creates a pass-through logger that does not create the real logger
+    until it is really needed and delegates all calls to the real logger
+    once it is created.
+    """
+    return LazyAdapter(name, version)
+
+
+class WritableLogger(object):
+    """A thin wrapper that responds to `write` and logs."""
+
+    def __init__(self, logger, level=logging.INFO):
+        self.logger = logger
+        self.level = level
+
+    def write(self, msg):
+        self.logger.log(self.level, msg.rstrip())
+
+
+class ContextFormatter(logging.Formatter):
+    """A context.RequestContext aware formatter configured through flags.
+
+    The flags used to set format strings are: logging_context_format_string
+    and logging_default_format_string.  You can also specify
+    logging_debug_format_suffix to append extra formatting if the log level is
+    debug.
+
+    For information about what variables are available for the formatter see:
+    http://docs.python.org/library/logging.html#formatter
+
+    """
+
+    def format(self, record):
+        """Uses contextstring if request_id is set, otherwise default."""
+        # NOTE(sdague): default the fancier formatting params
+        # to an empty string so we don't throw an exception if
+        # they get used
+        for key in ('instance', 'color'):
+            if key not in record.__dict__:
+                record.__dict__[key] = ''
+
+        if record.__dict__.get('request_id', None):
+            self._fmt = CONF.logging_context_format_string
+        else:
+            self._fmt = CONF.logging_default_format_string
+
+        if (record.levelno == logging.DEBUG and
+                CONF.logging_debug_format_suffix):
+            self._fmt += " " + CONF.logging_debug_format_suffix
+
+        # Cache this on the record, Logger will respect our formatted copy
+        if record.exc_info:
+            record.exc_text = self.formatException(record.exc_info, record)
+        return logging.Formatter.format(self, record)
+
+    def formatException(self, exc_info, record=None):
+        """Format exception output with CONF.logging_exception_prefix."""
+        if not record:
+            return logging.Formatter.formatException(self, exc_info)
+
+        stringbuffer = moves.StringIO()
+        traceback.print_exception(exc_info[0], exc_info[1], exc_info[2],
+                                  None, stringbuffer)
+        lines = stringbuffer.getvalue().split('\n')
+        stringbuffer.close()
+
+        if CONF.logging_exception_prefix.find('%(asctime)') != -1:
+            record.asctime = self.formatTime(record, self.datefmt)
+
+        formatted_lines = []
+        for line in lines:
+            pl = CONF.logging_exception_prefix % record.__dict__
+            fl = '%s%s' % (pl, line)
+            formatted_lines.append(fl)
+        return '\n'.join(formatted_lines)
+
+
+class ColorHandler(logging.StreamHandler):
+    LEVEL_COLORS = {
+        logging.DEBUG: '\033[00;32m',  # GREEN
+        logging.INFO: '\033[00;36m',  # CYAN
+        logging.AUDIT: '\033[01;36m',  # BOLD CYAN
+        logging.WARN: '\033[01;33m',  # BOLD YELLOW
+        logging.ERROR: '\033[01;31m',  # BOLD RED
+        logging.CRITICAL: '\033[01;31m',  # BOLD RED
+    }
+
+    def format(self, record):
+        record.color = self.LEVEL_COLORS[record.levelno]
+        return logging.StreamHandler.format(self, record)
+
+
+class DeprecatedConfig(Exception):
+    message = _("Fatal call to deprecated config: %(msg)s")
+
+    def __init__(self, msg):
+        super(Exception, self).__init__(self.message % dict(msg=msg))

gceapi/openstack/common/py3kcompat/urlutils.py

@@ -0,0 +1,67 @@
+#
+# Copyright 2013 Canonical Ltd.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+"""
+Python2/Python3 compatibility layer for OpenStack
+"""
+
+import six
+
+if six.PY3:
+    # python3
+    import urllib.error
+    import urllib.parse
+    import urllib.request
+
+    urlencode = urllib.parse.urlencode
+    urljoin = urllib.parse.urljoin
+    quote = urllib.parse.quote
+    quote_plus = urllib.parse.quote_plus
+    parse_qsl = urllib.parse.parse_qsl
+    unquote = urllib.parse.unquote
+    unquote_plus = urllib.parse.unquote_plus
+    urlparse = urllib.parse.urlparse
+    urlsplit = urllib.parse.urlsplit
+    urlunsplit = urllib.parse.urlunsplit
+    SplitResult = urllib.parse.SplitResult
+
+    urlopen = urllib.request.urlopen
+    URLError = urllib.error.URLError
+    pathname2url = urllib.request.pathname2url
+else:
+    # python2
+    import urllib
+    import urllib2
+    import urlparse
+
+    urlencode = urllib.urlencode
+    quote = urllib.quote
+    quote_plus = urllib.quote_plus
+    unquote = urllib.unquote
+    unquote_plus = urllib.unquote_plus
+
+    parse = urlparse
+    parse_qsl = parse.parse_qsl
+    urljoin = parse.urljoin
+    urlparse = parse.urlparse
+    urlsplit = parse.urlsplit
+    urlunsplit = parse.urlunsplit
+    SplitResult = parse.SplitResult
+
+    urlopen = urllib2.urlopen
+    URLError = urllib2.URLError
+    pathname2url = urllib.pathname2url

gceapi/openstack/common/test.py

@@ -0,0 +1,88 @@
+# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Common utilities used in testing"""
+
+import logging
+import os
+import tempfile
+
+import fixtures
+import testtools
+
+_TRUE_VALUES = ('True', 'true', '1', 'yes')
+_LOG_FORMAT = "%(levelname)8s [%(name)s] %(message)s"
+
+
+class BaseTestCase(testtools.TestCase):
+
+    def setUp(self):
+        super(BaseTestCase, self).setUp()
+        self._set_timeout()
+        self._fake_output()
+        self._fake_logs()
+        self.useFixture(fixtures.NestedTempfile())
+        self.useFixture(fixtures.TempHomeDir())
+        self.tempdirs = []
+
+    def _set_timeout(self):
+        test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
+        try:
+            test_timeout = int(test_timeout)
+        except ValueError:
+            # If timeout value is invalid do not set a timeout.
+            test_timeout = 0
+        if test_timeout > 0:
+            self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
+
+    def _fake_output(self):
+        if os.environ.get('OS_STDOUT_CAPTURE') in _TRUE_VALUES:
+            stdout = self.useFixture(fixtures.StringStream('stdout')).stream
+            self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
+        if os.environ.get('OS_STDERR_CAPTURE') in _TRUE_VALUES:
+            stderr = self.useFixture(fixtures.StringStream('stderr')).stream
+            self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
+
+    def _fake_logs(self):
+        if os.environ.get('OS_DEBUG') in _TRUE_VALUES:
+            level = logging.DEBUG
+        else:
+            level = logging.INFO
+        capture_logs = os.environ.get('OS_LOG_CAPTURE') in _TRUE_VALUES
+        if capture_logs:
+            self.useFixture(
+                fixtures.FakeLogger(
+                    format=_LOG_FORMAT,
+                    level=level,
+                    nuke_handlers=capture_logs,
+                )
+            )
+        else:
+            logging.basicConfig(format=_LOG_FORMAT, level=level)
+
+    def create_tempfiles(self, files, ext='.conf'):
+        tempfiles = []
+        for (basename, contents) in files:
+            if not os.path.isabs(basename):
+                (fd, path) = tempfile.mkstemp(prefix=basename, suffix=ext)
+            else:
+                path = basename + ext
+                fd = os.open(path, os.O_CREAT | os.O_WRONLY)
+            tempfiles.append(path)
+            try:
+                os.write(fd, contents)
+            finally:
+                os.close(fd)
+        return tempfiles

gceapi/openstack/common/timeutils.py

@@ -0,0 +1,210 @@
+# Copyright 2011 OpenStack Foundation.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Time related utilities and helper functions.
+"""
+
+import calendar
+import datetime
+import time
+
+import iso8601
+import six
+
+
+# ISO 8601 extended time format with microseconds
+_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f'
+_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
+PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND
+
+
+def isotime(at=None, subsecond=False):
+    """Stringify time in ISO 8601 format."""
+    if not at:
+        at = utcnow()
+    st = at.strftime(_ISO8601_TIME_FORMAT
+                     if not subsecond
+                     else _ISO8601_TIME_FORMAT_SUBSECOND)
+    tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC'
+    st += ('Z' if tz == 'UTC' else tz)
+    return st
+
+
+def parse_isotime(timestr):
+    """Parse time from ISO 8601 format."""
+    try:
+        return iso8601.parse_date(timestr)
+    except iso8601.ParseError as e:
+        raise ValueError(six.text_type(e))
+    except TypeError as e:
+        raise ValueError(six.text_type(e))
+
+
+def strtime(at=None, fmt=PERFECT_TIME_FORMAT):
+    """Returns formatted utcnow."""
+    if not at:
+        at = utcnow()
+    return at.strftime(fmt)
+
+
+def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT):
+    """Turn a formatted time back into a datetime."""
+    return datetime.datetime.strptime(timestr, fmt)
+
+
+def normalize_time(timestamp):
+    """Normalize time in arbitrary timezone to UTC naive object."""
+    offset = timestamp.utcoffset()
+    if offset is None:
+        return timestamp
+    return timestamp.replace(tzinfo=None) - offset
+
+
+def is_older_than(before, seconds):
+    """Return True if before is older than seconds."""
+    if isinstance(before, six.string_types):
+        before = parse_strtime(before).replace(tzinfo=None)
+    else:
+        before = before.replace(tzinfo=None)
+
+    return utcnow() - before > datetime.timedelta(seconds=seconds)
+
+
+def is_newer_than(after, seconds):
+    """Return True if after is newer than seconds."""
+    if isinstance(after, six.string_types):
+        after = parse_strtime(after).replace(tzinfo=None)
+    else:
+        after = after.replace(tzinfo=None)
+
+    return after - utcnow() > datetime.timedelta(seconds=seconds)
+
+
+def utcnow_ts():
+    """Timestamp version of our utcnow function."""
+    if utcnow.override_time is None:
+        # NOTE(kgriffs): This is several times faster
+        # than going through calendar.timegm(...)
+        return int(time.time())
+
+    return calendar.timegm(utcnow().timetuple())
+
+
+def utcnow():
+    """Overridable version of utils.utcnow."""
+    if utcnow.override_time:
+        try:
+            return utcnow.override_time.pop(0)
+        except AttributeError:
+            return utcnow.override_time
+    return datetime.datetime.utcnow()
+
+
+def iso8601_from_timestamp(timestamp):
+    """Returns a iso8601 formatted date from timestamp."""
+    return isotime(datetime.datetime.utcfromtimestamp(timestamp))
+
+
+utcnow.override_time = None
+
+
+def set_time_override(override_time=None):
+    """Overrides utils.utcnow.
+
+    Make it return a constant time or a list thereof, one at a time.
+
+    :param override_time: datetime instance or list thereof. If not
+                          given, defaults to the current UTC time.
+    """
+    utcnow.override_time = override_time or datetime.datetime.utcnow()
+
+
+def advance_time_delta(timedelta):
+    """Advance overridden time using a datetime.timedelta."""
+    assert(not utcnow.override_time is None)
+    try:
+        for dt in utcnow.override_time:
+            dt += timedelta
+    except TypeError:
+        utcnow.override_time += timedelta
+
+
+def advance_time_seconds(seconds):
+    """Advance overridden time by seconds."""
+    advance_time_delta(datetime.timedelta(0, seconds))
+
+
+def clear_time_override():
+    """Remove the overridden time."""
+    utcnow.override_time = None
+
+
+def marshall_now(now=None):
+    """Make an rpc-safe datetime with microseconds.
+
+    Note: tzinfo is stripped, but not required for relative times.
+    """
+    if not now:
+        now = utcnow()
+    return dict(day=now.day, month=now.month, year=now.year, hour=now.hour,
+                minute=now.minute, second=now.second,
+                microsecond=now.microsecond)
+
+
+def unmarshall_time(tyme):
+    """Unmarshall a datetime dict."""
+    return datetime.datetime(day=tyme['day'],
+                             month=tyme['month'],
+                             year=tyme['year'],
+                             hour=tyme['hour'],
+                             minute=tyme['minute'],
+                             second=tyme['second'],
+                             microsecond=tyme['microsecond'])
+
+
+def delta_seconds(before, after):
+    """Return the difference between two timing objects.
+
+    Compute the difference in seconds between two date, time, or
+    datetime objects (as a float, to microsecond resolution).
+    """
+    delta = after - before
+    return total_seconds(delta)
+
+
+def total_seconds(delta):
+    """Return the total seconds of datetime.timedelta object.
+
+    Compute total seconds of datetime.timedelta, datetime.timedelta
+    doesn't have method total_seconds in Python2.6, calculate it manually.
+    """
+    try:
+        return delta.total_seconds()
+    except AttributeError:
+        return ((delta.days * 24 * 3600) + delta.seconds +
+                float(delta.microseconds) / (10 ** 6))
+
+
+def is_soon(dt, window):
+    """Determines if time is going to happen in the next window seconds.
+
+    :param dt: the time
+    :param window: minimum seconds to remain to consider the time not soon
+
+    :return: True if expiration is within the given duration
+    """
+    soon = (utcnow() + datetime.timedelta(seconds=window))
+    return normalize_time(dt) <= soon

gceapi/paths.py

@@ -0,0 +1,68 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+# Copyright 2012 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+
+from oslo.config import cfg
+
+path_opts = [
+    cfg.StrOpt('pybasedir',
+               default=os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                                    '../')),
+               help='Directory where the gceapi python module is installed'),
+    cfg.StrOpt('bindir',
+               default='$pybasedir/bin',
+               help='Directory where gceapi binaries are installed'),
+    cfg.StrOpt('state_path',
+               default='$pybasedir',
+               help="Top-level directory for maintaining gceapi's state"),
+]
+
+CONF = cfg.CONF
+CONF.register_opts(path_opts)
+
+
+def basedir_def(*args):
+    """Return an uninterpolated path relative to $pybasedir."""
+    return os.path.join('$pybasedir', *args)
+
+
+def bindir_def(*args):
+    """Return an uninterpolated path relative to $bindir."""
+    return os.path.join('$bindir', *args)
+
+
+def state_path_def(*args):
+    """Return an uninterpolated path relative to $state_path."""
+    return os.path.join('$state_path', *args)
+
+
+def basedir_rel(*args):
+    """Return a path relative to $pybasedir."""
+    return os.path.join(CONF.pybasedir, *args)
+
+
+def bindir_rel(*args):
+    """Return a path relative to $bindir."""
+    return os.path.join(CONF.bindir, *args)
+
+
+def state_path_rel(*args):
+    """Return a path relative to $state_path."""
+    return os.path.join(CONF.state_path, *args)

gceapi/service.py

@@ -0,0 +1,263 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# Copyright 2011 Justin Santa Barbara
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Generic Node base class for all workers that run on hosts."""
+
+import signal
+import sys
+
+import eventlet
+import greenlet
+from oslo.config import cfg
+
+from gceapi.openstack.common import eventlet_backdoor
+from gceapi.openstack.common.gettextutils import _
+from gceapi.openstack.common import importutils
+from gceapi.openstack.common import log as logging
+from gceapi import wsgi
+
+LOG = logging.getLogger(__name__)
+
+service_opts = [
+    cfg.BoolOpt('use_ssl',
+                default=False,
+                help='Enable ssl connections or not'),
+    cfg.IntOpt('service_down_time',
+               default=60,
+               help='maximum time since last check-in for up service'),
+    cfg.StrOpt('gce_listen',
+               default="0.0.0.0",
+               help='IP address for gce api to listen'),
+    cfg.IntOpt('gce_listen_port',
+               default=8777,
+               help='port for gce api to listen'),
+    cfg.StrOpt('network_api',
+               default="neutron",
+               help='Name of network API. neutron(quantum) or nova'),
+    ]
+
+CONF = cfg.CONF
+CONF.register_opts(service_opts)
+
+
+class SignalExit(SystemExit):
+    def __init__(self, signo, exccode=1):
+        super(SignalExit, self).__init__(exccode)
+        self.signo = signo
+
+
+class Launcher(object):
+    """Launch one or more services and wait for them to complete."""
+
+    def __init__(self):
+        """Initialize the service launcher.
+
+        :returns: None
+
+        """
+        self._services = []
+        self.backdoor_port = eventlet_backdoor.initialize_if_enabled()
+
+    @staticmethod
+    def run_server(server):
+        """Start and wait for a server to finish.
+
+        :param service: Server to run and wait for.
+        :returns: None
+
+        """
+        server.start()
+        server.wait()
+
+    def launch_server(self, server):
+        """Load and start the given server.
+
+        :param server: The server you would like to start.
+        :returns: None
+
+        """
+        if self.backdoor_port is not None:
+            server.backdoor_port = self.backdoor_port
+        gt = eventlet.spawn(self.run_server, server)
+        self._services.append(gt)
+
+    def stop(self):
+        """Stop all services which are currently running.
+
+        :returns: None
+
+        """
+        for service in self._services:
+            service.kill()
+
+    def wait(self):
+        """Waits until all services have been stopped, and then returns.
+
+        :returns: None
+
+        """
+        for service in self._services:
+            try:
+                service.wait()
+            except greenlet.GreenletExit:
+                pass
+
+
+class ServiceLauncher(Launcher):
+    def _handle_signal(self, signo, frame):
+        # Allow the process to be killed again and die from natural causes
+        signal.signal(signal.SIGTERM, signal.SIG_DFL)
+        signal.signal(signal.SIGINT, signal.SIG_DFL)
+
+        raise SignalExit(signo)
+
+    def wait(self):
+        signal.signal(signal.SIGTERM, self._handle_signal)
+        signal.signal(signal.SIGINT, self._handle_signal)
+
+        LOG.debug(_('Full set of CONF:'))
+        for flag in CONF:
+            flag_get = CONF.get(flag, None)
+            # hide flag contents from log if contains a password
+            # should use secret flag when switch over to openstack-common
+            if ("_password" in flag or "_key" in flag or
+                    (flag == "sql_connection" and "mysql:" in flag_get)):
+                LOG.debug(_('%(flag)s : FLAG SET ') % {'flag': flag})
+            else:
+                LOG.debug('%(flag)s : %(flag_get)s' % {'flag': flag,
+                                                       'flag_get': flag_get})
+
+        status = None
+        try:
+            super(ServiceLauncher, self).wait()
+        except SignalExit as exc:
+            signame = {signal.SIGTERM: 'SIGTERM',
+                       signal.SIGINT: 'SIGINT'}[exc.signo]
+            LOG.info(_('Caught %s, exiting'), signame)
+            status = exc.code
+        except SystemExit as exc:
+            status = exc.code
+        finally:
+            self.stop()
+
+        if status is not None:
+            sys.exit(status)
+
+
+class WSGIService(object):
+    """Provides ability to launch API from a 'paste' configuration."""
+
+    def __init__(self, name, loader=None, use_ssl=False, max_url_len=None):
+        """Initialize, but do not start the WSGI server.
+
+        :param name: The name of the WSGI server given to the loader.
+        :param loader: Loads the WSGI application using the given name.
+        :returns: None
+
+        """
+        self.name = name
+        self.manager = self._get_manager()
+        self.loader = loader or wsgi.Loader()
+        self.app = self.loader.load_app(name)
+        self.host = getattr(CONF, '%s_listen' % name, "0.0.0.0")
+        self.port = getattr(CONF, '%s_listen_port' % name, 0)
+        self.use_ssl = use_ssl
+        self.server = wsgi.Server(name,
+                                  self.app,
+                                  host=self.host,
+                                  port=self.port,
+                                  use_ssl=self.use_ssl,
+                                  max_url_len=max_url_len)
+        # Pull back actual port used
+        self.port = self.server.port
+        self.backdoor_port = None
+
+    def _get_manager(self):
+        """Initialize a Manager object appropriate for this service.
+
+        Use the service name to look up a Manager subclass from the
+        configuration and initialize an instance. If no class name
+        is configured, just return None.
+
+        :returns: a Manager instance, or None.
+
+        """
+        fl = '%s_manager' % self.name
+        if fl not in CONF:
+            return None
+
+        manager_class_name = CONF.get(fl, None)
+        if not manager_class_name:
+            return None
+
+        manager_class = importutils.import_class(manager_class_name)
+        return manager_class()
+
+    def start(self):
+        """Start serving this service using loaded configuration.
+
+        Also, retrieve updated port number in case '0' was passed in, which
+        indicates a random port should be used.
+
+        :returns: None
+
+        """
+        if self.manager:
+            self.manager.init_host()
+            self.manager.pre_start_hook()
+        if self.backdoor_port is not None:
+            self.manager.backdoor_port = self.backdoor_port
+        self.server.start()
+        if self.manager:
+            self.manager.post_start_hook()
+
+    def stop(self):
+        """Stop serving this API.
+
+        :returns: None
+
+        """
+        self.server.stop()
+
+    def wait(self):
+        """Wait for the service to stop serving this API.
+
+        :returns: None
+
+        """
+        self.server.wait()
+
+
+# NOTE(vish): the global launcher is to maintain the existing
+#             functionality of calling service.serve +
+#             service.wait
+_launcher = None
+
+
+def serve(server):
+    global _launcher
+    if _launcher:
+        raise RuntimeError(_('serve() can only be called once'))
+
+    _launcher = ServiceLauncher()
+    _launcher.launch_server(server)
+
+
+def wait():
+    _launcher.wait()

gceapi/test.py

@@ -0,0 +1,180 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Base classes for our unit tests.
+
+Allows overriding of flags for use of fakes, and some black magic for
+inline callbacks.
+
+"""
+
+import os
+import shutil
+
+import collections
+import eventlet
+import fixtures
+import mox
+from oslo.config import cfg
+import stubout
+import testtools
+
+from gceapi.openstack.common import log as logging
+from gceapi import paths
+
+
+test_opts = [
+    cfg.StrOpt('sqlite_clean_db',
+               default='clean.sqlite',
+               help='File name of clean sqlite db'),
+    cfg.StrOpt('network_api',
+               default="neutron",
+               help='Name of network API. neutron(quantum) or nova'),
+    ]
+
+CONF = cfg.CONF
+CONF.register_opts(test_opts)
+CONF.import_opt('connection',
+                'gceapi.openstack.common.db.sqlalchemy.session',
+                group='database')
+CONF.set_override('use_stderr', False)
+
+logging.setup('gceapi')
+LOG = logging.getLogger(__name__)
+
+eventlet.monkey_patch(os=False)
+
+_DB_CACHE = None
+
+
+class Database(fixtures.Fixture):
+
+    def __init__(self, db_session, db_migrate, sql_connection,
+                    sqlite_db, sqlite_clean_db):
+        self.sql_connection = sql_connection
+        self.sqlite_db = sqlite_db
+        self.sqlite_clean_db = sqlite_clean_db
+
+        self.engine = db_session.get_engine()
+        self.engine.dispose()
+        conn = self.engine.connect()
+        if sql_connection == "sqlite://":
+            if db_migrate.db_version() > db_migrate.INIT_VERSION:
+                return
+        else:
+            testdb = paths.state_path_rel(sqlite_db)
+            if os.path.exists(testdb):
+                return
+        db_migrate.db_sync()
+        self.post_migrations()
+        if sql_connection == "sqlite://":
+            conn = self.engine.connect()
+            self._DB = "".join(line for line in conn.connection.iterdump())
+            self.engine.dispose()
+        else:
+            cleandb = paths.state_path_rel(sqlite_clean_db)
+            shutil.copyfile(testdb, cleandb)
+
+    def setUp(self):
+        super(Database, self).setUp()
+
+        if self.sql_connection == "sqlite://":
+            conn = self.engine.connect()
+            conn.connection.executescript(self._DB)
+            self.addCleanup(self.engine.dispose)
+        else:
+            shutil.copyfile(paths.state_path_rel(self.sqlite_clean_db),
+                            paths.state_path_rel(self.sqlite_db))
+
+
+class MoxStubout(fixtures.Fixture):
+    """Deal with code around mox and stubout as a fixture."""
+
+    def setUp(self):
+        super(MoxStubout, self).setUp()
+        # emulate some of the mox stuff, we can't use the metaclass
+        # because it screws with our generators
+        self.mox = mox.Mox()
+        self.stubs = stubout.StubOutForTesting()
+        self.addCleanup(self.mox.UnsetStubs)
+        self.addCleanup(self.stubs.UnsetAll)
+        self.addCleanup(self.stubs.SmartUnsetAll)
+        self.addCleanup(self.mox.VerifyAll)
+
+
+class TestCase(testtools.TestCase):
+    """Test case base class for all unit tests."""
+
+    def setUp(self):
+        """Run before each test method to initialize test environment."""
+        super(TestCase, self).setUp()
+        test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
+        try:
+            test_timeout = int(test_timeout)
+        except ValueError:
+            # If timeout value is invalid do not set a timeout.
+            test_timeout = 0
+        if test_timeout > 0:
+            self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
+        self.useFixture(fixtures.NestedTempfile())
+        self.useFixture(fixtures.TempHomeDir())
+
+        if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or
+                os.environ.get('OS_STDOUT_CAPTURE') == '1'):
+            stdout = self.useFixture(fixtures.StringStream('stdout')).stream
+            self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
+        if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or
+                os.environ.get('OS_STDERR_CAPTURE') == '1'):
+            stderr = self.useFixture(fixtures.StringStream('stderr')).stream
+            self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
+
+        self.log_fixture = self.useFixture(fixtures.FakeLogger('gceapi'))
+
+        mox_fixture = self.useFixture(MoxStubout())
+        self.mox = mox_fixture.mox
+        self.stubs = mox_fixture.stubs
+        self.addCleanup(self._clear_attrs)
+        self.useFixture(fixtures.EnvironmentVariable('http_proxy'))
+        CONF.set_override('fatal_exception_format_errors', True)
+
+    def _clear_attrs(self):
+        # Delete attributes that don't start with _ so they don't pin
+        # memory around unnecessarily for the duration of the test
+        # suite
+        for key in [k for k in self.__dict__.keys() if k[0] != '_']:
+            del self.__dict__[key]
+
+    def flags(self, **kw):
+        """Override flag variables for a test."""
+        group = kw.pop('group', None)
+        for k, v in kw.iteritems():
+            CONF.set_override(k, v, group)
+
+    def assertDictEqual(self, d1, d2, msg=None):
+        for k, v1 in d1.iteritems():
+            self.assertIn(k, d2)
+            v2 = d2[k]
+            if(isinstance(v1, collections.Iterable) and
+               not isinstance(v1, basestring)):
+                self.assertItemsEqual(v1, v2, msg)
+            else:
+                self.assertEqual(v1, v2, msg)
+        return True
+
+    def assertItemsEqual(self, expected_seq, actual_seq, msg=None):
+        self.assertEqual(sorted(expected_seq), sorted(actual_seq), msg)