First version of Mors - Lease Manager

What works:
- All the add/delete/update APIs integrated with UI and tested end to
  end
- Basic unit tests that test the above metioned APIs.

What (may) not be working or in other words is not fully tested
- Actual deletes of the VM, it used to work, but code has gone through
  major changes so need to test again.
- Cases:
-- Making sure Override of the lease works.
-- Cases where VM changes tenants or is deleted before the lease expiry
-- Removal of the tenant (not tested at all)

Next steps:
- Better unit test cases - better verification
- Deployment scripts (Ansible playbooks)

Adding a manage script for managing database upgrade script

Adding manage.py

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

.reviewboardrc

@@ -0,0 +1,6 @@
+REVIEWBOARD_URL = 'https://rbcommons.com/s/platform9/'
+REPOSITORY = 'pf9-mors'
+GUESS_DESCRIPTION = True
+GUESS_SUMMARY = True
+TARGET_GROUPS = 'platform9'
+TRACKING_BRANCH = 'origin/atherton'

README.md

@@ -1,2 +1,7 @@
 # pf9-mors
 Mors is the Roman God of death. Mors helps us implement leases.
+
+The functionality is described here in details:
+https://platform9.atlassian.net/wiki/pages/viewpage.action?pageId=58490897
+
+

build.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -e
+source ../pf9-version/pf9-version.rc
+export ROOT_DIR=`pwd`
+cd $ROOT_DIR/support/
+make all

etc/init.d/pf9-mors

@@ -0,0 +1,100 @@
+#!/bin/sh
+#
+# chkconfig:   - 98 02
+# description: Platform9 Lease Manager (code -named Mors)
+
+### BEGIN INIT INFO
+# Provides: pf9-mors
+# Required-Start: $remote_fs $network $syslog
+# Required-Stop: $remote_fs $syslog
+# Default-Stop: 0 1 6
+# Short-Description: Mors Server
+# Description: Platform9 Lease Manager (code -named Mors)
+### END INIT INFO
+
+. /etc/rc.d/init.d/functions
+
+name=pf9-mors
+prog=pf9-mors
+bindir=/opt/pf9/$name/bin
+python=$bindir/python
+exec="$python $bindir/pf9_mors.py"
+pidfile="/var/run/$name.pid"
+log_stdout="/var/log/pf9/$name-out.log"
+
+[ -e /etc/sysconfig/$prog ] && . /etc/sysconfig/$prog
+
+lockfile=/var/lock/subsys/$prog
+
+start() {
+    [ -x $python ] || exit 5
+    echo -n $"Starting $prog: "
+    daemon --pidfile $pidfile "$exec >> $log_stdout 2>&1 & echo \$! > $pidfile"
+    retval=$?
+    echo
+    [ $retval -eq 0 ] && touch $lockfile
+    return $retval
+}
+
+stop() {
+    echo -n $"Stopping $prog: "
+    killproc -p $pidfile $prog
+    retval=$?
+    echo
+    [ $retval -eq 0 ] && rm -f $lockfile
+    return $retval
+}
+
+restart() {
+    stop
+    start
+}
+
+reload() {
+    restart
+}
+
+force_reload() {
+    restart
+}
+
+rh_status() {
+    status -p $pidfile $prog
+}
+
+rh_status_q() {
+    rh_status >/dev/null 2>&1
+}
+
+
+case "$1" in
+    start)
+        rh_status_q && exit 0
+        $1
+        ;;
+    stop)
+        rh_status_q || exit 0
+        $1
+        ;;
+    restart)
+        $1
+        ;;
+    reload)
+        rh_status_q || exit 7
+        $1
+        ;;
+    force-reload)
+        force_reload
+        ;;
+    status)
+        rh_status
+        ;;
+    condrestart|try-restart)
+        rh_status_q || exit 0
+        restart
+        ;;
+    *)
+        echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}"
+        exit 2
+esac
+exit $?

etc/nginx/conf.d/locations/mors.conf

@@ -0,0 +1,4 @@
+location /mors/ {
+    proxy_pass http://127.0.0.1:8989/;
+    include /etc/nginx/conf.d/pf9/cors.conf;
+}

etc/pf9/pf9-mors-api-paste.ini

@@ -0,0 +1,14 @@
+[app:myService]
+paste.app_factory = mors.mors_wsgi:app_factory
+
+[pipeline:main]
+pipeline = authtoken  myService
+
+[filter:authtoken]
+paste.filter_factory = keystonemiddleware.auth_token:filter_factory
+auth_host = 127.0.0.1
+auth_port = 35357
+auth_protocol = http
+admin_token = RzvUwrEgiOaQFTXV
+auth_uri = http://127.0.0.1:8080/keystone
+identity_uri = http://127.0.0.1:8080/keystone_admin

etc/pf9/pf9-mors.ini

@@ -0,0 +1,16 @@
+[DEFAULT]
+db_conn=
+context_factory=
+lease_handler=
+listen_port=8989
+sleep_seconds=300
+paste-ini=/etc/pf9/pf9-mors-api-paste.ini
+log_file=/var/log/pf9/pf9-mors.log
+repo=/opt/pf9/pf9-mors/lib/python2.7/site-packages/mors_repo
+[nova]
+user_name=
+password=
+version=2
+auth_url=
+region_name=
+

mors/context_util.py

@@ -0,0 +1,69 @@
+# Copyright (c) 2016 Platform9 Systems Inc.
+# All Rights reserved
+
+from flask import request, jsonify
+import functools, os
+
+
+def get_context():
+    return Context(request.headers['X-User-Id'],
+                   request.headers['X-User'],
+                   request.headers['X-Roles'],
+                   request.headers['X-Tenant-Id'])
+
+
+def error_handler(func):
+    from sqlalchemy.exc import IntegrityError
+    import traceback,sys
+    @functools.wraps(func)
+    def inner(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except ValueError as exc:
+            traceback.print_exc(file=sys.stdout)
+            return jsonify({'error': 'Invalid input'}), 422, {'ContentType': 'application/json'}
+
+        except IntegrityError as exc:
+            traceback.print_exc(file=sys.stdout)
+            return jsonify({'error': 'Already exists'}), 409, {'ContentType': 'application/json'}
+
+    return inner
+
+def enforce(required=[]):
+    """
+    Generates a decorator that checks permissions before calling the
+    contained pecan handler function.
+    :param list[str] required: Roles require to run function.
+    """
+
+    def _enforce(fun):
+
+        @functools.wraps(fun)
+        def newfun(self, *args, **kwargs):
+
+            if not (required):
+                return fun(*args, **kwargs)
+            else:
+                roles_hdr = request.headers('X-Roles')
+                if roles_hdr:
+                    roles = roles_hdr.split(',')
+                else:
+                    roles = []
+
+                if set(roles) & set(required):
+
+                    return fun( *args, **kwargs)
+                else:
+                    return jsonify({'error': 'Unauthorized'}), 403, {'ContentType': 'application/json'}
+
+        return newfun
+
+    return _enforce
+
+
+class Context:
+    def __init__(self, user_id, user_name, roles_str, tenant_id):
+        self.user_id = user_id
+        self.user_name = user_name
+        self.roles = roles_str.split(',')
+        self.tenant_id = tenant_id

mors/lease_manager.py

@@ -0,0 +1,186 @@
+# Copyright Platform9 Systems Inc. 2016
+
+from datetime import datetime, timedelta
+
+from leasehandler import get_lease_handler
+from persistence import DbPersistence
+from eventlet.greenthread import spawn_after
+import logging
+from leasehandler.constants import SUCCESS_OK, ERR_UNKNOWN, ERR_NOT_FOUND
+
+logger = logging.getLogger(__name__)
+
+
+def get_tenant_lease_data(data):
+    """
+    Simple function to transform tenant database proxy object into an externally
+    consumable dictionary.
+    :param data: database row object
+    """
+    return {'vm_lease_policy': {'tenant_uuid': data['tenant_uuid'],
+                                'expiry_days': data['expiry_days'],
+                                'created_at': data['created_at'],
+                                'created_by': data['created_by'],
+                                'updated_at': data['updated_at'],
+                                'updated_by': data['updated_by']}}
+
+
+def get_vm_lease_data(data):
+    """
+    Simple function to transform instance database proxy object into an externally
+    consumable dictionary.
+    :param data: database row object
+    """
+    return {'instance_uuid': data['instance_uuid'],
+            'tenant_uuid': data['tenant_uuid'],
+            'expiry': data['expiry'],
+            'created_at': data['created_at'],
+            'created_by': data['created_by'],
+            'updated_at': data['updated_at'],
+            'updated_by': data['updated_by']}
+
+
+class LeaseManager:
+    """
+    Lease Manager is the main class for mors dealing with CRUD operations for the REST API
+    as well as the actual deletion of the Instances. Instance deletion and discovery is achieved
+    through an object 'leasehandler'.
+    """
+    def __init__(self, conf):
+        self.domain_mgr = DbPersistence(conf.get("DEFAULT", "db_conn"))
+        self.lease_handler = get_lease_handler(conf)
+        self.sleep_seconds = conf.getint("DEFAULT", "sleep_seconds")
+
+    def add_tenant_lease(self, context, tenant_obj):
+        logger.info("Adding tenant lease %s", tenant_obj)
+        self.domain_mgr.add_tenant_lease(
+            tenant_obj['tenant_uuid'],
+            tenant_obj['expiry_days'],
+            context.user_id,
+            datetime.utcnow())
+
+    def update_tenant_lease(self, context, tenant_obj):
+        logger.info("Update tenant lease %s", tenant_obj)
+        self.domain_mgr.update_tenant_lease(
+            tenant_obj['tenant_uuid'],
+            tenant_obj['expiry_days'],
+            context.user_id,
+            datetime.utcnow())
+
+    def delete_tenant_lease(self, context, tenant_id):
+        logger.info("Delete tenant lease %s", tenant_id)
+        return self.domain_mgr.delete_tenant_lease(tenant_id)
+
+    def get_tenant_leases(self, context):
+        logger.debug("Getting all tenant lease")
+        all_tenants = self.domain_mgr.get_all_tenant_leases()
+        all_tenants = map(lambda x: get_tenant_lease_data(x), all_tenants)
+        logger.debug("Getting all tenant lease %s", all_tenants)
+        return all_tenants
+
+    def get_tenant_lease(self, context, tenant_id):
+        data = self.domain_mgr.get_tenant_lease(tenant_id)
+        logger.debug("Getting tenant lease %s", data)
+        if data:
+            return get_tenant_lease_data(data)
+        return {}
+
+    def get_tenant_and_associated_instance_leases(self, context, tenant_uuid):
+        logger.debug("Getting tenant and instances leases %s", tenant_uuid)
+        return {
+            'tenant_lease': self.get_tenant_lease(context, tenant_uuid),
+            'all_vms':
+                map(lambda x: get_vm_lease_data(x), self.domain_mgr.get_instance_leases_by_tenant(tenant_uuid))
+        }
+
+    # To Be Implemented
+    def check_instance_lease_violation(self, instance_lease, tenant_lease):
+        return True
+
+    def get_instance_lease(self, context, instance_id):
+        data = self.domain_mgr.get_instance_lease(instance_id)
+        if data:
+            data = get_vm_lease_data(data)
+        logger.debug("Get instance lease %s %s", instance_id, data)
+        return data
+
+    def add_instance_lease(self, context, tenant_uuid, instance_lease_obj):
+        logger.info("Add instance lease %s", instance_lease_obj)
+        tenant_lease = self.domain_mgr.get_tenant_lease(tenant_uuid)
+        self.check_instance_lease_violation(instance_lease_obj, tenant_lease)
+        self.domain_mgr.add_instance_lease(instance_lease_obj['instance_uuid'],
+                                           tenant_uuid,
+                                           instance_lease_obj['expiry'],
+                                           context.user_id,
+                                           datetime.utcnow())
+
+    def update_instance_lease(self, context, tenant_uuid, instance_lease_obj):
+        logger.info("Update instance lease %s", instance_lease_obj)
+        self.domain_mgr.update_instance_lease(instance_lease_obj['instance_uuid'],
+                                              tenant_uuid,
+                                              instance_lease_obj['expiry'],
+                                              context.user_id,
+                                              datetime.utcnow())
+
+    def delete_instance_lease(self, context, instance_uuid):
+        logger.info("Delete instance lease %s", instance_uuid)
+        self.domain_mgr.delete_instance_leases([instance_uuid])
+
+    def start(self):
+        spawn_after(self.sleep_seconds, self.run)
+
+    # Could have used a generator here, would save memory but wonder if it is a good idea given the error conditions
+    # This is a simple implementation which goes and deletes VMs one by one
+    def _get_vms_to_delete_for_tenant(self, tenant_uuid, expiry_days):
+        vms_to_delete = []
+        vm_ids_to_delete = set()
+        now = datetime.utcnow()
+        add_days = timedelta(days=expiry_days)
+        instance_leases = self.get_tenant_and_associated_instance_leases(None, tenant_uuid)['all_vms']
+        for i_lease in instance_leases:
+            if now > i_lease['expiry']:
+                logger.info("Explicit lease for %s queueing for deletion", i_lease['instance_uuid'])
+                vms_to_delete.append(i_lease)
+                vm_ids_to_delete.add(i_lease['instance_uuid'])
+            else:
+                logger.debug("Ignoring vm, vm not expired yet %s", i_lease['instance_uuid'])
+
+        tenant_vms = self.lease_handler.get_all_vms(tenant_uuid)
+        for vm in tenant_vms:
+            expiry_date = vm['created_at'] + add_days
+            if now > expiry_date and not (vm['instance_uuid'] in vm_ids_to_delete):
+                logger.info("Instance %s queued up for deletion creation date %s", vm['instance_uuid'],
+                            vm['created_at'])
+                vms_to_delete.append(vm)
+            else:
+                logger.debug("Ignoring vm, vm not expired yet or already deleted %s, %s", vm['instance_uuid'],
+                             vm['created_at'])
+
+        return vms_to_delete
+
+    def _delete_vms_for_tenant(self, t_lease):
+        tenant_vms_to_delete = self._get_vms_to_delete_for_tenant(t_lease['tenant_uuid'], t_lease['expiry_days'])
+
+        # Keep it simple and delete them serially
+        result = self.lease_handler.delete_vms(tenant_vms_to_delete)
+
+        remove_from_db = []
+
+        for vm_result in result.items():
+            # If either the VM has been successfully deleted or has already been deleted
+            # Remove from our database
+            if vm_result[1] == SUCCESS_OK or vm_result[1] == ERR_NOT_FOUND:
+                remove_from_db.append(vm_result[0])
+
+        if len(remove_from_db) > 0:
+            logger.info("Removing vms %s from db", remove_from_db)
+            self.domain_mgr.delete_instance_leases(remove_from_db)
+
+    def run(self):
+        # Delete the cleanup
+        tenant_leases = self.domain_mgr.get_all_tenant_leases()
+        for t_lease in tenant_leases:
+            self._delete_vms_for_tenant(t_lease)
+
+        # Sleep again for sleep_seconds
+        spawn_after(self.sleep_seconds, self.run)

mors/leasehandler/__init__.py

@@ -0,0 +1,12 @@
+# Copyright 2016 Platform9 Systems Inc.
+
+from nova_lease_handler import NovaLeaseHandler
+from fake_lease_handler import FakeLeaseHandler
+import constants
+
+
+def get_lease_handler(conf):
+    if conf.get("DEFAULT", "lease_handler") == "test":
+        return FakeLeaseHandler(conf)
+    else:
+        return NovaLeaseHandler(conf)

mors/leasehandler/constants.py

@@ -0,0 +1,5 @@
+# Copyright Platform9 Systems Inc. 2016
+
+SUCCESS_OK = 0
+ERR_NOT_FOUND = 1
+ERR_UNKNOWN = 2

mors/leasehandler/fake_lease_handler.py

@@ -0,0 +1,38 @@
+# Copyright Platform9 Systems Inc. 2016
+import constants
+import logging
+from datetime import datetime
+
+# @TODO: Need to move this to a test folder
+
+class FakeLeaseHandler:
+    # Singleton tenants
+    tenants = {}
+
+    def __init__(self,conf):
+        self.logger = logging.getLogger("test-lease-handler")
+        pass
+
+    def add_tenant_data(self, tenant_id, instances):
+        FakeLeaseHandler.tenants[tenant_id] = instances
+        print FakeLeaseHandler.tenants
+
+    def get_tenant_data(self, tenant_id):
+        return FakeLeaseHandler.tenants[tenant_id]
+
+    def get_all_vms(self, tenant_uuid):
+        return FakeLeaseHandler.tenants[tenant_uuid]
+
+    def delete_vm(self, tenant_uuid, vm_id):
+        vms = FakeLeaseHandler.tenants[tenant_uuid]
+        new_vm_data = filter(lambda x: x['instance_uuid'] != vm_id, vms)
+        FakeLeaseHandler.tenants[tenant_uuid] = new_vm_data
+
+
+    def delete_vms(self, vms):
+        result = {}
+        for vm in vms:
+            self.logger.info("Deleting VM  vm %s", vm)
+            self.delete_vm(vm['tenant_uuid'], vm['instance_uuid'])
+            result[vm['instance_uuid']] = constants.SUCCESS_OK
+        return result

mors/leasehandler/nova_lease_handler.py

@@ -0,0 +1,70 @@
+# Copyright 2016 Platform9 Systems Inc.
+from novaclient import client
+import logging
+import novaclient
+from datetime import datetime
+from constants import SUCCESS_OK, ERR_NOT_FOUND, ERR_UNKNOWN
+logger = logging.getLogger(__name__)
+
+DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
+
+def get_vm_data(data):
+    return {'instance_uuid': data.id,
+            'tenant_uuid': data.tenant_id,
+            'created_at': datetime.strptime(data.created)}
+
+class NovaLeaseHandler:
+    def __init__(self, conf):
+        self.conf = conf
+
+    def _get_nova_client(self):
+        return client.Client(self.conf.get("nova", "version"),
+                             username=self.conf.get("nova", "user_name"),
+                             region_name=self.conf.get("nova", "region_name"),
+                             tenant_id=self.conf.get("nova", "tenant_uuid"),
+                             api_key=self.conf.get("nova", "password"),
+                             auth_url=self.conf.get("nova", "auth_url"),
+                             connection_pool=False)
+
+    def get_all_vms(self, tenant_uuid):
+        """
+        Get all vms for a given tenant
+        :param tenant_uuid:
+        :return: an iteratble that returns a set of vms (each vm has a UUID and a created_at field)
+        """
+        try:
+            with self._get_nova_client() as nova:
+                vms = nova.servers.list(search_opts={'all_tenants':1, 'tenant_id':tenant_uuid})
+                return map(lambda x: get_vm_data(x), vms)
+        except Exception as e:
+            logger.exception("Error getting list of vms for tenant %s", tenant_uuid)
+
+        return []
+
+    def _delete_vm(self, nova, vm_uuid):
+        try:
+            logger.info("Deleting VM %s", vm_uuid)
+            nova.server.delete(vm_uuid)
+            return SUCCESS_OK
+        except novaclient.exceptions.NotFound:
+            return ERR_NOT_FOUND
+        except Exception as e:
+            logger.exception("Error deleting vm %s", vm_uuid)
+            return ERR_UNKNOWN
+
+    def delete_vms(self, vms):
+        """
+        Delete a VM on a given tenant
+        :param tenant_uuid:
+        :param vm_uuid:
+        :return: dictionary of vm_id to result
+        """
+        result = {}
+        try:
+            with self._get_nova_client() as nova:
+                for vm in vms:
+                    result[vm['instance_uuid']] = self._delete_vm(nova, vm['instance_uuid'])
+            return result
+        except Exception as e:
+            logger.exception("Error deleting vm %s", vms)
+        return result

mors/mors_wsgi.py

@@ -0,0 +1,130 @@
+# Copyright (c) 2016 Platform9 Systems Inc.
+# All Rights reserved
+
+from flask import Flask, request, jsonify
+from lease_manager import LeaseManager
+from context_util import enforce, get_context, error_handler
+from flask.json import JSONEncoder
+from datetime import datetime
+
+DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
+APP_NAME = "MORS"
+
+
+class CustomJSONEncoder(JSONEncoder):
+    def default(self, obj):
+        try:
+            if isinstance(obj, datetime):
+                return obj.strftime(DATE_FORMAT)
+            iterable = iter(obj)
+        except TypeError:
+            pass
+        else:
+            return list(iterable)
+        return JSONEncoder.default(self, obj)
+
+
+app = Flask(__name__)
+app.debug = True
+app.json_encoder = CustomJSONEncoder
+
+lease_manager = None
+
+@enforce(required=['admin'])
+@app.route("/v1/tenant", methods=['GET'])
+@app.route("/v1/tenant/", methods=['GET'])
+@error_handler
+def get_all_tenants():
+    all_tenants = lease_manager.get_tenant_leases(get_context())
+    if all_tenants:
+        return jsonify({"all_tenants":all_tenants})
+    else:
+        return jsonify({}), 200, {'ContentType': 'application/json'}
+
+@enforce(required=['_member_'])
+@app.route("/v1/tenant/<tenant_id>", methods=['GET'])
+@error_handler
+def get_tenant(tenant_id):
+    tenant_lease = lease_manager.get_tenant_lease(get_context(), tenant_id)
+    if not tenant_lease:
+        return jsonify({'success': False}), 404, {'ContentType': 'application/json'}
+    return jsonify(tenant_lease)
+
+
+@enforce(required=['admin'])
+@app.route("/v1/tenant/<tenant_id>", methods=['PUT', 'POST'])
+@error_handler
+def add_update_tenant(tenant_id):
+    tenant_lease = request.get_json()["vm_lease_policy"]
+    if request.method == "POST":
+        lease_manager.add_tenant_lease(get_context(), tenant_lease)
+    else:
+        lease_manager.update_tenant_lease(get_context(), tenant_lease)
+    return jsonify({'success': True}), 200, {'ContentType': 'application/json'}
+
+@enforce(required=['admin'])
+@app.route("/v1/tenant/<tenant_id>", methods=['DELETE'])
+@error_handler
+def delete_tenant_lease(tenant_id):
+    lease_manager.delete_tenant_lease(get_context(), tenant_id)
+    return jsonify({'success': True}), 200, {'ContentType': 'application/json'}
+
+#-- Instnace - tenant related
+
+@enforce(required=['_member_'])
+@app.route("/v1/tenant/<tenant_id>/instances/", methods=['GET'])
+@error_handler
+def get_tenant_and_instances(tenant_id):
+    instances = lease_manager.get_tenant_and_associated_instance_leases(get_context(), tenant_id)
+    if not instances:
+        return jsonify({'success': False}), 404, {'ContentType': 'application/json'}
+    return jsonify(instances)
+
+# --- Instance related ---
+@enforce(required=['_member_'])
+@app.route("/v1/tenant/<tenant_id>/instance/<instance_id>", methods=['GET'])
+@error_handler
+def get_vm_lease(tenant_id, instance_id):
+    lease_info = lease_manager.get_instance_lease(get_context(), instance_id)
+    if lease_info:
+        return jsonify(lease_info), 200, {'ContentType': 'application/json'}
+    else:
+        return jsonify({'error': 'Not found'}), 404, {'ContentType': 'application/json'}
+
+@enforce(required=['_member_'])
+@app.route("/v1/tenant/<tenant_id>/instance/<instance_id>", methods=['DELETE'])
+@error_handler
+def delete_vm_lease(tenant_id, instance_id):
+    lease_manager.delete_instance_lease(get_context(), instance_id)
+    return jsonify({'success': True}), 200, {'ContentType': 'application/json'}
+
+@enforce(required=['_member_'])
+@app.route("/v1/tenant/<tenant_id>/instance/<instance_id>", methods=['PUT', 'POST'])
+@error_handler
+def add_update_vm_lease(tenant_id, instance_id):
+    lease_obj = request.get_json()
+    # ds = '2012-03-01T10:00:00Z' # or any date sting of differing formats.
+    date = datetime.strptime(lease_obj['expiry'], DATE_FORMAT)
+    lease_obj['expiry'] = date
+    if request.method == "POST":
+        lease_manager.add_instance_lease(get_context(), tenant_id, lease_obj)
+    else:
+        lease_manager.update_tenant_lease(get_context(), tenant_id, lease_obj)
+    return jsonify({'success': True}), 200, {'ContentType': 'application/json'}
+
+
+def start_server(conf):
+    global lease_manager
+    lease_manager = LeaseManager(conf)
+    lease_manager.start()
+
+
+def shutdown_server():
+    func = request.environ.get('werkzeug.server.shutdown')
+    if func is None:
+        raise RuntimeError('Not running with the Werkzeug Server')
+    func()
+
+
+def app_factory(global_config, **local_conf):
+    return app

mors/persistence.py

@@ -0,0 +1,108 @@
+# Copyright Platform9 Systems Inc. 2016
+
+from sqlalchemy.pool import QueuePool
+from sqlalchemy import create_engine, text
+from sqlalchemy import Table, Column, Integer, String, MetaData, DateTime
+import logging, functools
+
+logger = logging.getLogger(__name__)
+
+
+def db_connect(transaction=False):
+    """
+    Generates a decorator that get connection from a pool and returns
+    it to the pool when the internal function is done
+    :param transaction bool: should this function create and end transaction.
+    """
+
+    def _db_connect(fun):
+        if hasattr(fun, '__name__'):
+            fun.__name__ = 'method_decorator(%s)' % fun.__name__
+        else:
+            fun.__name__ = 'method_decorator(%s)' % fun.__class__.__name__
+
+        @functools.wraps(fun)
+        def newfun(self, *args, **kwargs):
+            conn = self.engine.connect()
+            if transaction:
+                trans = conn.begin()
+            try:
+                ret = fun(self, conn, *args, **kwargs)
+                if transaction:
+                    trans.commit()
+                return ret
+            except Exception as e:
+                if transaction:
+                    trans.rollback()
+                logger.exception("Error during transaction ")
+                raise
+            finally:
+                conn.close()
+
+        return newfun
+
+    return _db_connect
+
+
+class DbPersistence:
+    def __init__(self, db_conn_string):
+        self.engine = create_engine(db_conn_string, poolclass=QueuePool)
+        self.metadata = MetaData(bind=self.engine)
+        self.tenant_lease = Table('tenant_lease', self.metadata, autoload=True)
+        self.instance_lease = Table('instance_lease', self.metadata, autoload=True)
+
+    @db_connect(transaction=False)
+    def get_all_tenant_leases(self, conn):
+        return conn.execute(self.tenant_lease.select()).fetchall()
+
+    @db_connect(transaction=False)
+    def get_tenant_lease(self, conn, tenant_uuid):
+        return conn.execute(self.tenant_lease.select(tenant_uuid == tenant_uuid)).first()
+
+    @db_connect(transaction=True)
+    def add_tenant_lease(self, conn, tenant_uuid, expiry_days, created_by, created_at):
+        logger.debug("Adding tenant lease %s %d %s %s", tenant_uuid, expiry_days, str(created_at), created_by)
+        conn.execute(self.tenant_lease.insert(), tenant_uuid=tenant_uuid, expiry_days=expiry_days,
+                     created_at=created_at, created_by=created_by)
+
+    @db_connect(transaction=True)
+    def update_tenant_lease(self, conn, tenant_uuid, expiry_days, updated_by, updated_at):
+        logger.debug("Updating tenant lease %s %d %s %s", tenant_uuid, expiry_days, str(updated_at), updated_by)
+        conn.execute(self.tenant_lease.update(tenant_uuid == tenant_uuid), expiry_days=expiry_days,
+                     updated_at=updated_at, updated_by=updated_by)
+
+    @db_connect(transaction=True)
+    def delete_tenant_lease(self, conn, tenant_uuid):
+        # Should we just soft delete ?
+        logger.debug("Deleting tenant lease %s", tenant_uuid)
+        conn.execute(self.tenant_lease.delete().where(tenant_uuid == tenant_uuid))
+        conn.execute(self.instance_lease.delete().where(tenant_uuid == tenant_uuid))
+
+    @db_connect(transaction=False)
+    def get_instance_leases_by_tenant(self, conn, tenant_uuid):
+        return conn.execute(self.instance_lease.select(tenant_uuid == tenant_uuid)).fetchall()
+
+    @db_connect(transaction=False)
+    def get_instance_lease(self, conn, instance_uuid):
+        return conn.execute(
+            self.instance_lease.select((instance_uuid == instance_uuid))).first()
+
+    @db_connect(transaction=True)
+    def add_instance_lease(self, conn, instance_uuid, tenant_uuid, expiry, created_by, created_at):
+        logger.debug("Adding instance lease %s %s %s %s", instance_uuid, tenant_uuid, expiry, created_by)
+        conn.execute(self.instance_lease.insert(), instance_uuid=instance_uuid, tenant_uuid=tenant_uuid,
+                     expiry=expiry,
+                     created_at=created_at, created_by=created_by)
+
+    @db_connect(transaction=True)
+    def update_instance_lease(self, conn, instance_uuid, tenant_uuid, expiry, updated_by, updated_at):
+        logger.debug("Updating instance lease %s %s %s %s", instance_uuid, tenant_uuid, expiry, updated_by)
+        conn.execute(self.instance_lease.update(instance_uuid == instance_uuid), tenant_uuid=tenant_uuid,
+                     expiry=expiry,
+                     updated_at=updated_at, updated_by=updated_by)
+
+    @db_connect(transaction=True)
+    def delete_instance_leases(self, conn, instance_uuids):
+        # Delete 10 at a time, should we soft delete
+        logger.debug("Deleting instance leases %s", str(instance_uuids))
+        conn.execute(self.instance_lease.delete().where(self.instance_lease.c.instance_uuid.in_(instance_uuids)))

mors_manage.py

@@ -0,0 +1,27 @@
+#!/opt/pf9/pf9-mors/bin/python
+# Copyright (c) 2016 Platform9 Systems Inc.
+# All Rights reserved
+import argparse, logging
+import ConfigParser
+from migrate.versioning.api import upgrade, create, version_control
+
+def _get_arg_parser():
+    parser = argparse.ArgumentParser(description="Lease Manager for VirtualMachines")
+    parser.add_argument('--config-file', dest='config_file', default='/etc/pf9/pf9-mors.ini')
+    parser.add_argument('--command', dest='command', default='db_sync')
+    return parser.parse_args()
+
+if __name__ == '__main__':
+    parser = _get_arg_parser()
+    conf = ConfigParser.ConfigParser()
+    conf.readfp(open(parser.config_file))
+    if 'db_sync' == parser.command:
+        version_control(conf.get("DEFAULT", "db_conn"), conf.get("DEFAULT", "repo"))
+        upgrade(conf.get("DEFAULT", "db_conn"), , conf.get("DEFAULT", "repo"))
+        exit(0)
+    else:
+        print 'Unknown command'
+        exit(1)
+
+
+

mors_repo/README

@@ -0,0 +1,4 @@
+This is a database migration repository.
+
+More information at
+http://code.google.com/p/sqlalchemy-migrate/

mors_repo/manage.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+from migrate.versioning.shell import main
+
+if __name__ == '__main__':
+    main(debug='False')

mors_repo/migrate.cfg

@@ -0,0 +1,25 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=Mors
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to
+# change the table name in each database too.
+version_table=migrate_version
+
+# When committing a change script, Migrate will attempt to generate the
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the
+# commit continues, perhaps ending successfully.
+# Databases in this list MUST compile successfully during a commit, or the
+# entire commit will fail. List the databases your application will actually
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=[]
+
+# When creating new change scripts, Migrate will stamp the new script with
+# a version number. By default this is latest_version + 1. You can set this
+# to 'true' to tell Migrate to use the UTC timestamp instead.
+use_timestamp_numbering=False

mors_repo/versions/001_Add_initial_tables.py

@@ -0,0 +1,36 @@
+# Copyright Platform9 Systems Inc. 2016
+from sqlalchemy import Table, Column, Integer, String, MetaData, DateTime
+
+meta = MetaData()
+
+tenant_lease = Table(
+    'tenant_lease', meta,
+    Column('tenant_uuid', String(40), primary_key=True),
+    Column('expiry_days', Integer),
+    Column('created_at', DateTime),
+    Column('updated_at', DateTime),
+    Column('created_by', String(40)),
+    Column('updated_by', String(40))
+)
+
+vm_lease = Table(
+    'instance_lease', meta,
+    Column('instance_uuid', String(40), primary_key=True),
+    Column('tenant_uuid', String(40)),
+    Column('expiry', DateTime),
+    Column('created_at', DateTime),
+    Column('updated_at', DateTime),
+    Column('created_by', String(40)),
+    Column('updated_by', String(40))
+)
+
+def upgrade(migrate_engine):
+    meta.bind = migrate_engine
+    tenant_lease.create()
+    vm_lease.create()
+
+
+def downgrade(migrate_engine):
+    meta.bind = migrate_engine
+    vm_lease.drop()
+    tenant_lease.drop()

pf9_mors.py

@@ -0,0 +1,48 @@
+#!/opt/pf9/pf9-mors/bin/python
+# Copyright (c) 2016 Platform9 Systems Inc.
+# All Rights reserved
+
+from paste.deploy import loadapp
+from eventlet import wsgi
+import eventlet
+import argparse, logging
+import logging.handlers
+import ConfigParser, os
+from mors import mors_wsgi
+
+eventlet.monkey_patch()
+def _get_arg_parser():
+    parser = argparse.ArgumentParser(description="Lease Manager for VirtualMachines")
+    parser.add_argument('--config-file', dest='config_file', default='/etc/pf9/pf9-mors.ini')
+    parser.add_argument('--paste-ini', dest='paste_file')
+    return parser.parse_args()
+
+def _configure_logging(conf):
+    log_filename = conf.get("DEFAULT", "log_file")
+    logging.basicConfig(filename=log_filename,
+                        level=logging.DEBUG,
+                        format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
+                        datefmt='%m-%d %H:%M')
+    handler = logging.handlers.RotatingFileHandler(
+        log_filename, maxBytes=1024 * 1024 * 5, backupCount=5)
+    logging.root.addHandler(handler)
+
+
+def start_server(conf, paste_ini):
+    _configure_logging(conf)
+    paste_file = None
+    if paste_ini:
+        paste_file = paste_ini
+    else:
+        paste_file = conf.get("DEFAULT", "paste-ini")
+
+    wsgi_app = loadapp('config:%s' % paste_file, 'main')
+    mors_wsgi.start_server(conf)
+    wsgi.server(eventlet.listen(('', conf.getint("DEFAULT", "listen_port"))), wsgi_app)
+
+
+if __name__ == '__main__':
+    parser = _get_arg_parser()
+    conf = ConfigParser.ConfigParser()
+    conf.readfp(open(parser.config_file))
+    start_server(conf, parser.pate_file)

setup.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+
+from setuptools import setup
+
+setup(name='pf9-mors',
+      version='0.1',
+      description='Platform9 Mors (lease manager)',
+      author='Roopak Parikh',
+      author_email='rparikh@platform9.net',
+      url='https://github.com/platform9/pf9-mors',
+      packages=['mors',
+                'mors/leasehandler',
+                'mors_repo',
+                'mors_repo/versions'],
+      install_requires=[
+          'pbr==0.11.0',
+          'pytz==2015.7',
+          'keystoneauth1==2.3.0',
+          'oslo.i18n==3.4.0',
+          'oslo.serialization==2.4.0',
+          'oslo.utils==3.7.0',
+          'keystonemiddleware==4.3.0',
+          'Paste==1.7.5.1',
+          'PasteDeploy==1.5.2',
+          'pip==1.5.2',
+          'python-novaclient==3.2.0',
+          'flask==0.10.0',
+          'SQLAlchemy==0.9.8',
+          'sqlalchemy-migrate==0.9.5',
+          'PyMySQL',
+          'eventlet==0.18.4',
+          'nose',
+          'proboscis'
+      ],
+      scripts=['pf9_mors.py', 'mors_manage.py']
+      )

support/Makefile

@@ -0,0 +1,65 @@
+#! vim noexpandtab
+# Copyright (C) 2016 Platform 9 Systems, Inc.
+
+TOP_DIR := $(abspath ../)
+SRC_DIR := $(TOP_DIR)
+BUILD_DIR := $(TOP_DIR)/build
+
+NPM := npm
+
+APP_NAME :=pf9-mors
+APP_DESC :="Platform9 mors (lease manager)"
+APP_BUILD_DIR := $(BUILD_DIR)
+
+PF9_VERSION ?=2.0.0
+BUILD_NUMBER ?= 0
+GIT_HASH := $(shell git rev-parse --short HEAD)
+FULL_VERSION := $(PF9_VERSION)-$(BUILD_NUMBER)
+
+APP_DESC :="Platform9 mors(lease manager) git hash $(GIT_HASH)"
+
+APP_RPM_DIR := $(APP_BUILD_DIR)/rpmbuild
+APP_RPM_STAGE_DIR := $(APP_BUILD_DIR)/stage
+APP_RPM_VENV := $(APP_RPM_STAGE_DIR)/opt/pf9/$(APP_NAME)
+APP_ARCHITECTURE := noarch
+APP_RPM := $(APP_RPM_DIR)/$(APP_NAME)-$(FULL_VERSION).noarch.rpm
+APP_SPEC_FILE := $(APP_BUILD_DIR)/$(APP_NAME)-rpm.spec
+
+############################################################
+
+${APP_RPM_DIR}:
+	mkdir -p $@
+
+${APP_RPM_STAGE_DIR}:
+	mkdir -p $@
+
+${APP_RPM_VENV}:
+	mkdir -p $@
+	virtualenv $@
+	$@/bin/pip install ${SRC_DIR}
+
+stage: $(APP_RPM_DIR) $(APP_RPM_STAGE_DIR) $(APP_RPM_VENV)
+	cp -r $(SRC_DIR)/etc/ $(APP_RPM_STAGE_DIR)/
+
+${APP_RPM}: stage
+	echo "RPM build "
+	fpm -t rpm \
+		-s dir \
+		-n $(APP_NAME) \
+		--description $(APP_DESC) \
+		--version $(PF9_VERSION) \
+		--iteration $(BUILD_NUMBER) \
+		--provides $(APP_NAME) \
+		--provides pf9app \
+		--license "Commercial" \
+		--architecture $(APP_ARCHITECTURE) \
+		--url "http://www.platform9.net" \
+		--vendor Platform9 \
+		-p $@ \
+		-C $(APP_RPM_STAGE_DIR) . && \
+	$(SRC_DIR)/support/sign_packages.sh ${APP_RPM}
+
+clean:
+	rm -rf $(BUILD_DIR)
+
+all: clean $(APP_RPM)

support/mors.expect

@@ -0,0 +1,9 @@
+#!/usr/bin/expect
+
+set timeout 15
+spawn bash -c "rpm --resign $argv"
+match_max 100000
+expect -exact "Enter pass phrase: "
+send -- "\r"
+expect "Pass phrase is good."
+expect eof

support/sign_packages.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+if [ "x${SIGN_PACKAGES}" = "x1" ]; then
+    expect $(dirname $0)/mors.expect $@
+fi

test.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Copyright (c) Platform9 systems. All rights reserved
+output_dir=./build
+log_filter=-paramiko.transport
+setup_venv() {
+    virtualenv ${output_dir}/venv
+    source ${output_dir}/venv/bin/activate
+    pip install -e .
+}
+
+run_tests() {
+    python ./test/run_tests.py --verbose --with-xunit --xunit-file=${output_dir}/test_output.xml \
+              --logging-clear-handlers ${exclude} ${nocapture} --logging-filter=${log_filter} ${module} \
+              --logging-format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s'
+}
+
+setup_venv
+run_tests
+

test/api-paste.ini

@@ -0,0 +1,14 @@
+[app:myService]
+paste.app_factory = mors.mors_wsgi:app_factory
+
+[pipeline:main]
+pipeline =  myService
+
+[filter:authtoken]
+paste.filter_factory = keystonemiddleware.auth_token:filter_factory
+auth_host = 127.0.0.1
+auth_port = 35357
+auth_protocol = http
+admin_token = RzvUwrEgiOaQFTXV
+auth_uri = http://127.0.0.1:8080/keystone
+identity_uri = http://127.0.0.1:8080/keystone_admin

test/pf9-mors.ini

@@ -0,0 +1,16 @@
+[DEFAULT]
+db_conn=sqlite+pysqlite:///test/test.db
+context_factory=test
+lease_handler=test
+listen_port=8989
+sleep_seconds=3
+paste-ini=test/api-paste.ini
+log_file=build/test.log
+
+[nova]
+user_name=rparikh@platform9.net
+password=asdsdsadf
+version=2
+auth_url=https://pf9.platform9.net/v2/keystone
+region_name=RegionOne
+

test/run_tests.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2016 Platform9 Systems Inc.
+# All Rights reserved
+
+def run_tests():
+    from proboscis import TestProgram
+    import test_api
+
+    # Run Proboscis and exit.
+    print "Starting tests ---"
+    TestProgram().run_and_exit()
+    print "Tests done ---"
+
+if __name__ == '__main__':
+    print "Run tests"
+    run_tests()

test/test_api.py

@@ -0,0 +1,236 @@
+# Copyright (c) 2016 Platform9 Systems Inc.
+# All Rights reserved
+
+from migrate.versioning.api import upgrade, create, version_control
+import ConfigParser, os
+import requests
+import eventlet
+from pf9_mors import start_server
+from mors.mors_wsgi import DATE_FORMAT
+import logging, sys
+from datetime import datetime, timedelta
+from proboscis.asserts import assert_equal
+from proboscis.asserts import assert_false
+from proboscis.asserts import assert_raises
+from proboscis.asserts import assert_true
+from proboscis import SkipTest
+from proboscis import test
+import shutil
+from mors.leasehandler.fake_lease_handler import FakeLeaseHandler
+
+try:
+    import http.client as http_client
+except ImportError:
+    # Python 2
+    import httplib as http_client
+
+http_client.HTTPConnection.debuglevel = 1
+
+root = logging.getLogger()
+root.setLevel(logging.DEBUG)
+
+ch = logging.StreamHandler(sys.stdout)
+ch.setLevel(logging.DEBUG)
+root.addHandler(ch)
+
+logger = logging.getLogger(__name__)
+eventlet.monkey_patch()
+conf = None
+
+headers = {
+    'X-User-Id': 'asdfsd-asdf-sdadf',
+    'X-User': 'roopak@pf9.com',
+    'X-roles': 'admin,_member_',
+    'X-Tenant-Id': 'poioio-oio-oioo'
+}
+tenant_id1 = "tenantid-1"
+tenant_id2 = "tenantid-2"
+instance_id1 = "instanceid-1-t-1"
+instance_id2 = "instanceid-2-t-1"
+instance_id3 = "instanceid-3-t-2"
+
+expiry_day1 = 4
+port = 8080
+
+
+def _setup_lease_handler():
+    fakeLeaseHandler = FakeLeaseHandler(conf)
+    now = datetime.now()
+    dt = timedelta(days=3)
+    creation_time = now - dt
+    t1_vms = [{'instance_uuid': 'instance-123-t1', 'tenant_uuid': tenant_id1, 'created_at': creation_time},
+              {'instance_uuid': 'instance-456-t1', 'tenant_uuid': tenant_id1, 'created_at': now},
+              {'instance_uuid': instance_id1, 'tenant_uuid': tenant_id1, 'created_at': now},
+              {'instance_uuid': instance_id2, 'tenant_uuid': tenant_id1, 'created_at': now}]
+    fakeLeaseHandler.add_tenant_data(tenant_id1, t1_vms)
+
+    t2_vms = [{'instance_uuid': 'instance-123-t2', 'tenant_uuid': tenant_id2, 'created_at': creation_time},
+              {'instance_uuid': 'instance-456-t2', 'tenant_uuid': tenant_id2, 'created_at': now},
+              {'instance_uuid': instance_id3, 'tenant_uuid': tenant_id2, 'created_at': now}]
+    fakeLeaseHandler.add_tenant_data(tenant_id2, t2_vms)
+
+
+@test
+def initialize():
+    global conf
+    global port
+    if os.path.exists("./sqlite+pysqlite:"):
+        shutil.rmtree("./sqlite+pysqlite:")
+    if os.path.exists("./test/test.db"):
+        os.remove("./test/test.db")
+    conf = ConfigParser.ConfigParser()
+    conf.readfp(open("test/pf9-mors.ini"))
+    #create(conf.get("DEFAULT", "db_conn"), "./mors_repo")
+    version_control(conf.get("DEFAULT", "db_conn"), "./mors_repo")
+    upgrade(conf.get("DEFAULT", "db_conn"), "./mors_repo")
+    port = conf.get("DEFAULT", "listen_port")
+
+    _setup_lease_handler()
+    api_paste_file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'api-paste.ini')
+    eventlet.greenthread.spawn(start_server, conf, api_paste_file)
+    eventlet.greenthread.sleep(5)
+
+
+@test(depends_on=[initialize])
+def test_create_tenant():
+    r = requests.post('http://127.0.0.1:' + port + '/v1/tenant/' + tenant_id1,
+                      json={"vm_lease_policy": {"tenant_uuid": tenant_id1, "expiry_days": expiry_day1}},
+                      headers=headers)
+    logger.debug(r.text)
+    assert_equal(r.status_code, 200)
+
+
+@test(depends_on=[test_create_tenant])
+def test_update_tenant():
+    r = requests.put('http://127.0.0.1:' + port + '/v1/tenant/' + tenant_id1,
+                     json={"vm_lease_policy": {"tenant_uuid": tenant_id1, "expiry_days": 3}}, headers=headers)
+    logger.debug(r.text)
+    assert_equal(r.status_code, 200)
+
+
+@test(depends_on=[test_update_tenant])
+def test_get_all_tenants():
+    r = requests.get('http://127.0.0.1:' + port + '/v1/tenant/',
+                     headers=headers)
+    logger.debug(r.text)
+    assert_equal(r.status_code, 200)
+
+
+@test(depends_on=[test_get_all_tenants])
+def test_get_tenant():
+    r = requests.get('http://127.0.0.1:' + port + '/v1/tenant/' + tenant_id1, headers=headers)
+    logger.debug(r.text)
+    assert_equal(r.status_code, 200)
+
+
+@test(depends_on=[test_get_tenant])
+def test_create_tenant_neg():
+    # Try creating again and it should result in error
+    r = requests.post('http://127.0.0.1:' + port + '/v1/tenant/' + tenant_id1,
+                      json={"vm_lease_policy": {"tenant_uuid": tenant_id1, "expiry_days": expiry_day1}},
+                      headers=headers)
+    logger.debug(r.text)
+    assert_equal(r.status_code, 409)
+
+
+@test(depends_on=[test_create_tenant_neg])
+def test_create_instance():
+    # Now test the instance manipulation
+    expiry = datetime.utcnow()
+    expiry_str = datetime.strftime(expiry, DATE_FORMAT)
+    r = requests.post('http://127.0.0.1:' + port + '/v1/tenant/' + tenant_id1 + '/instance/' + instance_id1,
+                      json={"instance_uuid": instance_id1, "expiry": expiry_str},
+                      headers=headers)
+    logger.debug(r.text)
+    assert_equal(r.status_code, 200)
+
+
+@test(depends_on=[test_create_instance])
+def test_get_instance():
+    r = requests.get('http://127.0.0.1:' + port + '/v1/tenant/' + tenant_id1 + '/instance/' + instance_id1,
+                     headers=headers)
+    logger.debug(r.text)
+    assert_equal(r.status_code, 200)
+
+
+@test(depends_on=[test_get_instance])
+def test_deleted_instance():
+    eventlet.greenthread.sleep(50)
+    # The instance lease should be deleted by now
+    r = requests.get('http://127.0.0.1:' + port + '/v1/tenant/' + tenant_id1 + '/instance/' + instance_id1,
+                     headers=headers)
+    logger.debug(r.text)
+    assert_equal(r.status_code, 404)
+
+
+@test(depends_on=[test_deleted_instance])
+def test_create_instance2():
+    # Now test the instance manipulation
+    expiry = datetime.utcnow()
+    expiry_str = datetime.strftime(expiry, DATE_FORMAT)
+    r = requests.post('http://127.0.0.1:' + port + '/v1/tenant/' + tenant_id1 + '/instance/' + instance_id2,
+                      json={"instance_uuid": instance_id2, "expiry": expiry_str},
+                      headers=headers)
+    logger.debug(r.text)
+    assert_equal(r.status_code, 200)
+
+
+@test(depends_on=[test_create_instance2])
+def test_delete_instance_lease():
+    r = requests.delete('http://127.0.0.1:' + port + '/v1/tenant/' + tenant_id1 + '/instance/' + instance_id2,
+                        json={"tenant_uuid": tenant_id1, "instance_uuid": instance_id2},
+                        headers=headers)
+    logger.debug(r.text)
+    assert_equal(r.status_code, 200)
+
+
+@test(depends_on=[test_deleted_instance])
+def test_create_tenant2():
+    r = requests.post('http://127.0.0.1:' + port + '/v1/tenant/' + tenant_id2,
+                      json={"vm_lease_policy": {"tenant_uuid": tenant_id2, "expiry_days": expiry_day1}},
+                      headers=headers)
+    logger.debug(r.text)
+    assert_equal(r.status_code, 200)
+
+
+@test(depends_on=[test_create_tenant2])
+def test_create_instance3():
+    # Now test the instance manipulation
+    expiry = datetime.utcnow()
+    expiry_str = datetime.strftime(expiry, DATE_FORMAT)
+    r = requests.post('http://127.0.0.1:' + port + '/v1/tenant/' + tenant_id2 + '/instance/' + instance_id3,
+                      json={"tenant_uuid": tenant_id2, "instance_uuid": instance_id3, "expiry": expiry_str},
+                      headers=headers)
+    logger.debug(r.text)
+    assert_equal(r.status_code, 200)
+
+
+@test(depends_on=[test_create_instance3])
+def test_delete_tenant2():
+    r = requests.delete('http://127.0.0.1:' + port + '/v1/tenant/' + tenant_id2,
+                        json={"tenant_uuid": tenant_id2}, headers=headers)
+    logger.debug(r.text)
+    assert_equal(r.status_code, 200)
+
+
+@test(depends_on=[test_delete_tenant2])
+def test_get_tenant2():
+    r = requests.get('http://127.0.0.1:' + port + '/v1/tenant/' + tenant_id2, headers=headers)
+    logger.debug(r.text)
+    assert_equal(r.status_code, 404)
+
+
+@test(depends_on=[test_delete_tenant2])
+def test_get_instance3():
+    r = requests.get('http://127.0.0.1:' + port + '/v1/tenant/' + tenant_id2 + '/instance/' + instance_id3,
+                     headers=headers)
+    logger.debug(r.text)
+    assert_equal(r.status_code, 404)
+
+
+@test(depends_on=[test_get_instance3])
+def test_get_all_instances_for_tenant2():
+    r = requests.get('http://127.0.0.1:' + port + '/v1/tenant/' + tenant_id2 + '/instances/',
+                     headers=headers)
+    logger.debug(r.text)
+    assert_equal(r.status_code, 200)

test/test_persistence.py

@@ -0,0 +1,56 @@
+from mors.persistence import DbPersistence
+import pytest
+from migrate.versioning.api import upgrade,create,version_control
+from datetime import datetime
+import os
+
+db_persistence = None
+TEST_DB="test/test_db11"
+
+def setup_module(mod):
+    global db_persistence
+    DB_URL = "sqlite:///"+TEST_DB
+    create(DB_URL, "./mors_repo")
+    version_control(DB_URL, "./mors_repo")
+    upgrade(DB_URL,"./mors_repo")
+    db_persistence = DbPersistence(DB_URL)
+    return db_persistence
+
+def teardown_module(mod):
+    os.unlink(TEST_DB)
+
+def test_apis():
+    tenant_id = "aasdsadfsadf"
+    tenant_user1 = "a@xyz.com"
+    expiry_day1 = 3
+    tenant_created_date = datetime.utcnow()
+
+    db_persistence.add_tenant_lease(tenant_id, expiry_day1, tenant_user1, tenant_created_date)
+    t_lease = db_persistence.get_tenant_lease(tenant_id)
+    assert (t_lease.tenant_uuid == tenant_id)
+    assert (t_lease.created_by == tenant_user1)
+    assert (t_lease.created_at == tenant_created_date)
+
+    # Now try update
+    tenant_user2 = "b@xyz.com"
+    tenant_updated_date = datetime.utcnow()
+    db_persistence.update_tenant_lease(tenant_id, expiry_day1, tenant_user2, tenant_updated_date)
+    t_lease = db_persistence.get_tenant_lease(tenant_id)
+    assert (t_lease.tenant_uuid == tenant_id)
+    assert (t_lease.created_by == tenant_user1)
+    assert (t_lease.updated_by == tenant_user2)
+    assert (t_lease.created_at == tenant_created_date)
+    assert (t_lease.updated_at == tenant_updated_date)
+
+
+    # Instance lease now
+    instance_uuid = "asdf2-2342-23423"
+    now = datetime.utcnow()
+    db_persistence.add_instance_lease(instance_uuid, tenant_id, now, tenant_user1, now)
+    i_lease = db_persistence.get_instance_lease(instance_uuid, tenant_id)
+    assert (i_lease.instance_uuid == instance_uuid)
+    assert (i_lease.tenant_uuid == tenant_id)
+    assert (i_lease.expiry == now)
+    assert (i_lease.created_by == tenant_user1)
+
+