Moved to python3.5

Now Iotronic requires python3 and python2 is not supported anymore

Change-Id: I2d2fdccb1788412d21d216052fb843ff24061040

etc/iotronic/iotronic.conf

@@ -25,7 +25,7 @@ wamp_realm = s4t
 
 
 [database]
-connection = mysql://<user>:<password>@<host>/iotronic
+connection = mysql+pymsql://<user>:<password>@<host>/iotronic
 
 [keystone_authtoken]
 auth_uri = http://<keystone_host>:5000

iotronic/conductor/endpoints.py

@@ -1,5 +1,6 @@
-# coding=utf-8
+# Copyright 2017 MDSLAB - University of Messina
-
+# All Rights Reserved.
+#
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
 #    a copy of the License at
@@ -12,7 +13,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import cPickle as cpickle
+import _pickle as cpickle
 from iotronic.common import exception
 from iotronic.common import states
 from iotronic.conductor.provisioner import Provisioner

iotronic/db/sqlalchemy/models.py

@@ -21,7 +21,6 @@ SQLAlchemy models for iot data.
 import json
 
 from oslo_config import cfg
-from oslo_db import options as db_options
 from oslo_db.sqlalchemy import models
 import six.moves.urllib.parse as urlparse
 from sqlalchemy import Boolean
@@ -43,7 +42,6 @@ _DEFAULT_SQL_CONNECTION = 'sqlite:///' + \
                           paths.state_path_def('iotronic.sqlite')
 
 cfg.CONF.register_opts(sql_opts, 'database')
-db_options.set_defaults(cfg.CONF, _DEFAULT_SQL_CONNECTION, 'iotronic.sqlite')
 
 
 def table_args():

iotronic/openstack/common/fileutils.py

@@ -16,11 +16,11 @@
 import contextlib
 import errno
 import os
+from oslo_log import log as logging
+from oslo_utils import excutils
 import stat
 import tempfile
 
-from oslo_utils import excutils
-from oslo_log import log as logging
 
 LOG = logging.getLogger(__name__)
 

iotronic/wamp/agent.py

@@ -13,10 +13,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from autobahn.twisted import wamp
+import asyncio
-from autobahn.twisted import websocket
+import txaio
-from autobahn.wamp import types
-from twisted.internet.defer import inlineCallbacks
 
 from iotronic.common import exception
 from iotronic.common.i18n import _LI
@@ -26,14 +24,15 @@ from oslo_config import cfg
 from oslo_log import log as logging
 import oslo_messaging
 from oslo_messaging.rpc import dispatcher
-import threading
+
 from threading import Thread
-from twisted.internet.protocol import ReconnectingClientFactory
+
-from twisted.internet import reactor
 
 import os
 import signal
 
+from autobahn.asyncio.component import Component
+
 LOG = logging.getLogger(__name__)
 
 wamp_opts = [
@@ -58,114 +57,38 @@ wamp_opts = [
 CONF = cfg.CONF
 CONF.register_opts(wamp_opts, 'wamp')
 
-shared_result = {}
+txaio.start_logging(level="info")
+
 wamp_session_caller = None
 AGENT_HOST = None
+LOOP = None
+connected = False
 
 
-def wamp_request(e, kwarg, session):
+async def wamp_request(kwarg):
-    id = threading.current_thread().ident
+    LOG.debug("calling: " + kwarg['wamp_rpc_call'])
-    shared_result[id] = {}
+    d = await wamp_session_caller.call(kwarg['wamp_rpc_call'], *kwarg['data'])
-    shared_result[id]['result'] = None
+    return d
-
-    def success(d):
-        shared_result[id]['result'] = d
-        LOG.debug("DEVICE sent: %s", str(d))
-        e.set()
-        return shared_result[id]['result']
-
-    def fail(failure):
-        shared_result[id]['result'] = failure
-        LOG.error("WAMP FAILURE: %s", str(failure))
-        e.set()
-        return shared_result[id]['result']
-
-    LOG.debug("Calling %s...", kwarg['wamp_rpc_call'])
-    d = session.wamp_session.call(wamp_session_caller,
-                                  kwarg['wamp_rpc_call'], *kwarg['data'])
-    d.addCallback(success)
-    d.addErrback(fail)
 
 
 # OSLO ENDPOINT
 class WampEndpoint(object):
-    def __init__(self, wamp_session, agent_uuid):
+    def __init__(self, agent_uuid):
-        self.wamp_session = wamp_session
         setattr(self, agent_uuid + '.s4t_invoke_wamp', self.s4t_invoke_wamp)
 
     def s4t_invoke_wamp(self, ctx, **kwarg):
-        e = threading.Event()
+        LOG.debug("CONDUCTOR sent me: " + kwarg['wamp_rpc_call'])
-        LOG.debug("CONDUCTOR sent me:", kwarg)
 
-        th = threading.Thread(target=wamp_request, args=(e, kwarg, self))
+        r = asyncio.run_coroutine_threadsafe(wamp_request(kwarg), LOOP)
-        th.start()
 
-        e.wait()
+        return r.result()
-        LOG.debug("result received from wamp call: %s",
-                  str(shared_result[th.ident]['result']))
-
-        result = shared_result[th.ident]['result']
-        del shared_result[th.ident]['result']
-        return result
-
-
-class WampFrontend(wamp.ApplicationSession):
-    @inlineCallbacks
-    def onJoin(self, details):
-        global wamp_session_caller, AGENT_HOST
-        wamp_session_caller = self
-
-        import iotronic.wamp.functions as fun
-
-        self.subscribe(fun.board_on_leave, 'wamp.session.on_leave')
-        self.subscribe(fun.board_on_join, 'wamp.session.on_join')
-
-        try:
-            if CONF.wamp.register_agent:
-                self.register(fun.registration, u'stack4things.register')
-                LOG.info("I have been set as registration agent")
-            self.register(fun.connection,
-                          AGENT_HOST + u'.stack4things.connection')
-            self.register(fun.echo,
-                          AGENT_HOST + u'.stack4things.echo')
-            LOG.info("procedure registered")
-        except Exception as e:
-            LOG.error("could not register procedure: {0}".format(e))
-
-        LOG.info("WAMP session ready.")
-
-        session_l = yield self.call(u'wamp.session.list')
-        session_l.remove(details.session)
-        fun.update_sessions(session_l)
-
-    def onDisconnect(self):
-        LOG.info("disconnected")
-
-
-class WampClientFactory(websocket.WampWebSocketClientFactory,
-                        ReconnectingClientFactory):
-    maxDelay = 30
-
-    def clientConnectionFailed(self, connector, reason):
-        # print "reason:", reason
-        LOG.warning("Wamp Connection Failed.")
-        ReconnectingClientFactory.clientConnectionFailed(self,
-                                                         connector, reason)
-
-    def clientConnectionLost(self, connector, reason):
-        # print "reason:", reason
-        LOG.warning("Wamp Connection Lost.")
-        ReconnectingClientFactory.clientConnectionLost(self,
-                                                       connector, reason)
 
 
 class RPCServer(Thread):
     def __init__(self):
-        global AGENT_HOST
-
         # AMQP CONFIG
         endpoints = [
-            WampEndpoint(WampFrontend, AGENT_HOST),
+            WampEndpoint(AGENT_HOST),
         ]
 
         Thread.__init__(self)
@@ -191,28 +114,84 @@ class RPCServer(Thread):
 
 class WampManager(object):
     def __init__(self):
-        component_config = types.ComponentConfig(
-            realm=unicode(CONF.wamp.wamp_realm))
-        session_factory = wamp.ApplicationSessionFactory(
-            config=component_config)
-        session_factory.session = WampFrontend
-        transport_factory = WampClientFactory(session_factory,
-                                              url=CONF.wamp.wamp_transport_url)
-
-        transport_factory.autoPingInterval = CONF.wamp.autoPingInterval
-        transport_factory.autoPingTimeout = CONF.wamp.autoPingTimeout
 
         LOG.debug("wamp url: %s wamp realm: %s",
                   CONF.wamp.wamp_transport_url, CONF.wamp.wamp_realm)
-        websocket.connectWS(transport_factory)
+
+        self.loop = asyncio.get_event_loop()
+        global LOOP
+        LOOP = self.loop
+
+        comp = Component(
+            transports=CONF.wamp.wamp_transport_url,
+            realm=CONF.wamp.wamp_realm
+        )
+
+        self.comp = comp
+
+        @comp.on_join
+        async def onJoin(session, details):
+
+            global connected
+            connected = True
+
+            global wamp_session_caller, AGENT_HOST
+            wamp_session_caller = session
+
+            import iotronic.wamp.functions as fun
+
+            session.subscribe(fun.board_on_leave,
+                              'wamp.session.on_leave')
+            session.subscribe(fun.board_on_join,
+                              'wamp.session.on_join')
+
+            try:
+                if CONF.wamp.register_agent:
+                    session.register(fun.registration,
+                                     u'stack4things.register')
+                    LOG.info("I have been set as registration agent")
+                session.register(fun.connection,
+                                 AGENT_HOST +
+                                 + u'.stack4things.connection')
+                session.register(fun.echo,
+                                 AGENT_HOST +
+                                 + u'.stack4things.echo')
+                LOG.debug("procedure registered")
+
+            except Exception as e:
+                LOG.error("could not register procedure: {0}".format(e))
+
+            LOG.info("WAMP session ready.")
+
+            session_l = await session.call(u'wamp.session.list')
+            session_l.remove(details.session)
+            fun.update_sessions(session_l)
+
+        @comp.on_leave
+        async def onLeave(session, details):
+            LOG.warning('WAMP Session Left: ' + str(details))
+
+        @comp.on_disconnect
+        async def onDisconnect(session, was_clean):
+            LOG.warning('WAMP Transport Left: ' + str(was_clean))
+
+            global connected
+            connected = False
+            if not connected:
+                comp.start(self.loop)
 
     def start(self):
         LOG.info("Starting WAMP server...")
-        reactor.run()
+        self.comp.start(self.loop)
+        self.loop.run_forever()
 
     def stop(self):
-        LOG.info("Stopping WAMP-agent server...")
+        LOG.info("Stopping WAMP server...")
-        reactor.stop()
+
+        # Canceling pending tasks and stopping the loop
+        asyncio.gather(*asyncio.Task.all_tasks()).cancel()
+        # Stopping the loop
+        self.loop.stop()
         LOG.info("WAMP server stopped.")
 
 
@@ -222,9 +201,13 @@ class WampAgent(object):
         signal.signal(signal.SIGINT, self.stop_handler)
 
         logging.register_options(CONF)
+
         CONF(project='iotronic')
         logging.setup(CONF, "iotronic-wamp-agent")
 
+        if CONF.debug:
+            txaio.start_logging(level="debug")
+
         # to be removed asap
         self.host = host
         self.dbapi = dbapi.get_instance()

iotronic/wamp/functions.py

@@ -87,7 +87,7 @@ def board_on_leave(session_id):
     board = objects.Board.get_by_uuid(ctxt, old_session.board_uuid)
     board.status = states.OFFLINE
     board.save()
-    LOG.debug('Board %s is now  %s', old_session.uuid, states.OFFLINE)
+    LOG.debug('Board %s is now  %s', board.uuid, states.OFFLINE)
 
 
 def connection(uuid, session):

requirements.txt

@@ -12,8 +12,9 @@ oslo.policy>=1.15.0  # Apache-2.0
 oslo.messaging>=5.2.0  # Apache-2.0 
 oslo.db!=4.13.1,!=4.13.2,>=4.11.0  # Apache-2.0    
 pecan!=1.0.2,!=1.0.3,!=1.0.4,!=1.2,>=1.0.0  # BSD
-#paramiko>=2.0  # LGPLv2.1+
+paramiko>=2.0.0  # LGPLv2.1+
+PyMySQL>=0.7.6  # MIT License
+SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=1.0.10 # MIT
 keystonemiddleware!=4.5.0,>=4.2.0 # Apache-2.0
 autobahn>=0.10.1  # MIT License
-#Twisted>=16.5.0  # MIT
 WSME>=0.8  # MIT

setup.cfg

@@ -14,8 +14,6 @@ classifier =
     License :: OSI Approved :: Apache Software License
     Operating System :: POSIX :: Linux
     Programming Language :: Python
-    Programming Language :: Python :: 2
-    Programming Language :: Python :: 2.7
     Programming Language :: Python :: 3
     Programming Language :: Python :: 3.5
 
@@ -24,6 +22,9 @@ console_scripts =
     iotronic-conductor = iotronic.cmd.conductor:main
     iotronic-wamp-agent = iotronic.cmd.wamp_agent:main
 
+[options]
+build_scripts =
+    executable= /usr/bin/env python
 
 [files]
 packages =

test-requirements.txt

@@ -12,6 +12,7 @@ oslotest>=1.10.0 # Apache-2.0
 testrepository>=0.0.18  # Apache-2.0/BSD
 testscenarios>=0.4  # Apache-2.0/BSD
 testtools>=1.4.0  # MIT
+trollius>=1.0  # Apache-2.0
 eventlet!=0.18.3,>=0.18.2  # MIT
 oslo.config!=3.18.0,>=3.14.0  # Apache-2.0    
 oslo.log>=3.11.0  # Apache-2.0
@@ -19,9 +20,11 @@ oslo.concurrency>=3.8.0  # Apache-2.0
 oslo.policy>=1.15.0  # Apache-2.0    
 oslo.messaging>=5.2.0  # Apache-2.0 
 oslo.db!=4.13.1,!=4.13.2,>=4.11.0  # Apache-2.0  
+paramiko>=2.0.0  # LGPLv2.1+
 pecan!=1.0.2,!=1.0.3,!=1.0.4,!=1.2,>=1.0.0  # BSD
-#paramiko>=2.0  # LGPLv2.1+
+PyMySQL>=0.7.6  # MIT License
+SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=1.0.10 # MIT
 keystonemiddleware!=4.5.0,>=4.2.0 # Apache-2.0
 autobahn>=0.10.1  # MIT License
-#Twisted>=16.5.0  # MIT
 WSME>=0.8  # MIT
+

tox.ini

@@ -1,43 +1,39 @@
 [tox]
-minversion = 2.0
+minversion = 2.3.1
-envlist = py35,py34,py27,pypy,pep8
+envlist = py35,pep8
 skipsdist = True
 
 [testenv]
-usedevelop = True
+setenv =
-# tox is silly... these need to be separated by a newline....
+    VIRTUAL_ENV={envdir}
+    PYTHONWARNINGS=default::DeprecationWarning
+    LANGUAGE=en_US
+    LC_ALL=en_US.utf-8
 whitelist_externals = bash
-                      find
+    find
-                      rm
+    rm
+usedevelop = True
 install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
-setenv = VIRTUAL_ENV={envdir}
-         LANGUAGE=en_US
-         LC_ALL=en_US.utf-8
 deps = -r{toxinidir}/test-requirements.txt
 commands =
-  find . -type f -name "*.pyc" -delete
+    find . -type f -name "*.pyc" -delete
-
-[testenv:py27]
-commands =
-    {[testenv]commands}
 
 [testenv:pep8]
+basepython = python2.7
 commands = flake8 {posargs}
 
-[testenv:venv]
+[testenv:py35]
-commands = {posargs}
+basepython = python3.5
 
-[testenv:cover]
-commands = python setup.py test --coverage --testr-args='{posargs}'
-
-[testenv:docs]
-commands = python setup.py build_sphinx
-
-[testenv:debug]
-commands = oslo_debug_helper {posargs}
 
 [flake8]
-
+# TODO(dmllr): Analyze or fix the warnings blacklisted below
+# E711 comparison to None should be 'if cond is not None:'
+# E712 comparison to True should be 'if cond is True:' or 'if cond:'
+# H404 multi line docstring should start with a summary
+# H405 multi line docstring summary not separated with an empty line
+# E123, E125 skipped as they are invalid PEP-8.
 show-source = True
 builtins = _
-exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build
+ignore = E711,E712,H404,H405,E123,E125,E901,H301
+exclude = .venv,.git,.tox,dist,doc,etc,*lib/python*,*egg,build