more tests, adding in missing files

openstack-common.conf

@@ -0,0 +1,18 @@
+[DEFAULT]
+
+# The list of modules to copy from openstack-common
+module=gettextutils
+module=importutils
+module=install_venv_common
+module=jsonutils
+module=local
+module=log
+module=notifier
+module=policy
+module=setup
+module=timeutils
+module=uuidutils
+module=version
+
+# The base module to hold the copy of openstack.common
+base=staccato

staccato/common/config.py

@@ -91,7 +91,6 @@ def get_config_object(args=None, usage=None, default_config_files=None):
 def get_protocol_policy(conf):
     protocol_conf_file = conf.protocol_policy
     if protocol_conf_file is None:
-        # TODO log a warning
         return {}
     proto_file = conf.find_file(protocol_conf_file)
     policy = json.load(open(proto_file, 'r'))

staccato/db/__init__.py

@@ -137,6 +137,9 @@ class StaccatoDB(object):
                 query = query.filter(sql_expression.and_(
                     models.XferRequest.owner == owner,
                     models.XferRequest.state == constants.States.STATE_RUNNING))
+            else:
+                query = query.filter(
+                    models.XferRequest.state == constants.States.STATE_RUNNING)
             if limit is not None:
                 query = query.limit(limit)
             xfer_requests = query.all()

staccato/db/migrate_repo/migrate.cfg

@@ -0,0 +1,20 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=Staccato Migrations
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to
+# change the table name in each database too.
+version_table=migrate_version
+
+# When committing a change script, Migrate will attempt to generate the
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the
+# commit continues, perhaps ending successfully.
+# Databases in this list MUST compile successfully during a commit, or the
+# entire commit will fail. List the databases your application will actually
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=[]

staccato/scheduler/interface.py

@@ -0,0 +1,5 @@
+import staccato.scheduler.simple_thread as simple_thread
+
+def get_scheduler(conf):
+    # todo: pull this from the configuration information
+    return simple_thread.SimpleCountSchedler(conf)

staccato/tests/functional/test_db.py

@@ -1,6 +1,8 @@
 import os
 from staccato.tests import utils
 from staccato.common import config
+import staccato.common.exceptions as exceptions
+import staccato.xfer.constants as constants
 
 
 class TestDB(utils.TempFileCleanupBaseTest):
@@ -45,6 +47,32 @@ class TestDB(utils.TempFileCleanupBaseTest):
         xfer2 = self.db.lookup_xfer_request_by_id(xfer1.id)
         self.assertEqual(xfer1.id, xfer2.id)
 
+    def test_db_xfer_lookup_with_owner(self):
+        src = "src://url"
+        dst = "dst://url"
+        sm = "src.module"
+        dm = "dst.module"
+
+        xfer1 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer2 = self.db.lookup_xfer_request_by_id(xfer1.id, owner=self.owner)
+        self.assertEqual(xfer1.id, xfer2.id)
+
+    def test_db_xfer_lookup_with_wrong_owner(self):
+        src = "src://url"
+        dst = "dst://url"
+        sm = "src.module"
+        dm = "dst.module"
+
+        xfer1 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        self.assertRaises(exceptions.StaccatoNotFoundInDBException,
+                          self.db.lookup_xfer_request_by_id,
+                          xfer1.id, **{'owner': 'someoneelse'})
+
+    def test_db_xfer_lookup_not_there(self):
+        self.assertRaises(exceptions.StaccatoNotFoundInDBException,
+                          self.db.lookup_xfer_request_by_id,
+                          "notthere")
+
     def test_db_xfer_update(self):
         src = "src://url"
         dst = "dst://url"
@@ -56,3 +84,294 @@ class TestDB(utils.TempFileCleanupBaseTest):
         self.db.save_db_obj(xfer1)
         xfer2 = self.db.lookup_xfer_request_by_id(xfer1.id)
         self.assertEqual(xfer2.next_ndx, 10)
+
+    def test_lookup_all_no_owner(self):
+        src = "src://url"
+        dst = "dst://url"
+        sm = "src.module"
+        dm = "dst.module"
+
+        xfer1 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer2 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+
+        xfer_list = self.db.lookup_xfer_request_all()
+        id_list = [x.id for x in xfer_list]
+
+        self.assertEqual(len(xfer_list), 2)
+        self.assertTrue(xfer1.id in id_list)
+        self.assertTrue(xfer2.id in id_list)
+
+    def test_lookup_all_owner(self):
+        src = "src://url"
+        dst = "dst://url"
+        sm = "src.module"
+        dm = "dst.module"
+
+        xfer1 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer2 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+
+        xfer_list = self.db.lookup_xfer_request_all(owner=self.owner)
+        id_list = [x.id for x in xfer_list]
+
+        self.assertEqual(len(xfer_list), 2)
+        self.assertTrue(xfer1.id in id_list)
+        self.assertTrue(xfer2.id in id_list)
+
+    def test_lookup_all_wrong_owner(self):
+        src = "src://url"
+        dst = "dst://url"
+        sm = "src.module"
+        dm = "dst.module"
+
+        xfer1 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer2 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+
+        xfer_list = self.db.lookup_xfer_request_all(owner='notme')
+        self.assertEqual(len(xfer_list), 0)
+
+    def test_lookup_all_many_owners(self):
+        src = "src://url"
+        dst = "dst://url"
+        sm = "src.module"
+        dm = "dst.module"
+
+        xfer1 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer2 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer3 = self.db.get_new_xfer('notme', src, dst, sm, dm)
+        xfer4 = self.db.get_new_xfer('notme', src, dst, sm, dm)
+        xfer5 = self.db.get_new_xfer('someoneelse', src, dst, sm, dm)
+
+        xfer_list = self.db.lookup_xfer_request_all(owner=self.owner)
+        id_list = [x.id for x in xfer_list]
+
+        self.assertEqual(len(xfer_list), 2)
+        self.assertTrue(xfer1.id in id_list)
+        self.assertTrue(xfer2.id in id_list)
+
+    def test_get_all_ready_new_no_owner(self):
+        src = "src://url"
+        dst = "dst://url"
+        sm = "src.module"
+        dm = "dst.module"
+
+        xfer1 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer2 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+
+        xfer_list = self.db.get_all_ready()
+        id_list = [x.id for x in xfer_list]
+
+        self.assertEqual(len(xfer_list), 2)
+        self.assertTrue(xfer1.id in id_list)
+        self.assertTrue(xfer2.id in id_list)
+
+    def test_get_all_ready_new_owner(self):
+        src = "src://url"
+        dst = "dst://url"
+        sm = "src.module"
+        dm = "dst.module"
+
+        xfer1 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer2 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+
+        xfer_list = self.db.get_all_ready(owner=self.owner)
+        id_list = [x.id for x in xfer_list]
+
+        self.assertEqual(len(xfer_list), 2)
+        self.assertTrue(xfer1.id in id_list)
+        self.assertTrue(xfer2.id in id_list)
+
+    def test_get_all_ready_wrong_owner(self):
+        src = "src://url"
+        dst = "dst://url"
+        sm = "src.module"
+        dm = "dst.module"
+
+        xfer1 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer2 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+
+        xfer_list = self.db.get_all_ready(owner='notme')
+        self.assertEqual(len(xfer_list), 0)
+
+    def test_get_all_ready_some_not(self):
+        src = "src://url"
+        dst = "dst://url"
+        sm = "src.module"
+        dm = "dst.module"
+
+        xfer1 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer2 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer3 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+
+        xfer3.state = constants.States.STATE_RUNNING
+        self.db.save_db_obj(xfer3)
+
+        xfer_list = self.db.get_all_ready()
+        id_list = [x.id for x in xfer_list]
+
+        self.assertEqual(len(xfer_list), 2)
+        self.assertTrue(xfer1.id in id_list)
+        self.assertTrue(xfer2.id in id_list)
+
+    def test_get_all_ready_some_error(self):
+        src = "src://url"
+        dst = "dst://url"
+        sm = "src.module"
+        dm = "dst.module"
+
+        xfer1 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer2 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+
+        xfer2.state = constants.States.STATE_ERROR
+        self.db.save_db_obj(xfer2)
+
+
+        xfer_list = self.db.get_all_ready()
+        id_list = [x.id for x in xfer_list]
+
+        self.assertEqual(len(xfer_list), 2)
+        self.assertTrue(xfer1.id in id_list)
+        self.assertTrue(xfer2.id in id_list)
+
+    def test_get_all_running(self):
+        src = "src://url"
+        dst = "dst://url"
+        sm = "src.module"
+        dm = "dst.module"
+
+        xfer1 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer2 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer3 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+
+        for x in [xfer1, xfer2, xfer3]:
+            x.state = constants.States.STATE_RUNNING
+            self.db.save_db_obj(x)
+
+        xfer_list = self.db.get_all_running()
+        id_list = [x.id for x in xfer_list]
+
+        self.assertEqual(len(xfer_list), 3)
+        self.assertTrue(xfer1.id in id_list)
+        self.assertTrue(xfer2.id in id_list)
+        self.assertTrue(xfer3.id in id_list)
+
+    def test_get_all_running_some_not(self):
+        src = "src://url"
+        dst = "dst://url"
+        sm = "src.module"
+        dm = "dst.module"
+
+        xfer1 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer2 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer3 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+
+        for x in [xfer1, xfer3]:
+            x.state = constants.States.STATE_RUNNING
+            self.db.save_db_obj(x)
+
+        xfer_list = self.db.get_all_running()
+        id_list = [x.id for x in xfer_list]
+
+        self.assertEqual(len(xfer_list), 2)
+        self.assertTrue(xfer1.id in id_list)
+        self.assertTrue(xfer3.id in id_list)
+
+    def test_delete_from_db(self):
+        src = "src://url"
+        dst = "dst://url"
+        sm = "src.module"
+        dm = "dst.module"
+
+        xfer1 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer2 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer3 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+
+        self.db.delete_db_obj(xfer2)
+
+        xfer_list = self.db.get_all_ready()
+        id_list = [x.id for x in xfer_list]
+
+        self.assertEqual(len(xfer_list), 2)
+        self.assertTrue(xfer1.id in id_list)
+        self.assertTrue(xfer3.id in id_list)
+
+    def test_get_many_requests_no_owner(self):
+        src = "src://url"
+        dst = "dst://url"
+        sm = "src.module"
+        dm = "dst.module"
+
+        xfer1 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer2 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer3 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+
+        id_list = [xfer1.id, xfer2.id, xfer3.id]
+        xfer_list = self.db.get_xfer_requests(ids=id_list)
+        self.assertEqual(len(xfer_list), 3)
+        self.assertTrue(xfer1.id in id_list)
+        self.assertTrue(xfer2.id in id_list)
+        self.assertTrue(xfer3.id in id_list)
+
+    def test_get_many_requests_owner(self):
+        src = "src://url"
+        dst = "dst://url"
+        sm = "src.module"
+        dm = "dst.module"
+
+        xfer1 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer2 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer3 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+
+        id_list = [xfer1.id, xfer2.id, xfer3.id]
+        xfer_list = self.db.get_xfer_requests(ids=id_list, owner=self.owner)
+        self.assertEqual(len(xfer_list), 3)
+        self.assertTrue(xfer1.id in id_list)
+        self.assertTrue(xfer2.id in id_list)
+        self.assertTrue(xfer3.id in id_list)
+
+    def test_get_many_requests_owner_subset(self):
+        src = "src://url"
+        dst = "dst://url"
+        sm = "src.module"
+        dm = "dst.module"
+
+        xfer1 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer2 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer3 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+
+        id_list = [xfer1.id, xfer3.id]
+        xfer_list = self.db.get_xfer_requests(ids=id_list, owner=self.owner)
+        self.assertEqual(len(xfer_list), 2)
+        self.assertTrue(xfer1.id in id_list)
+        self.assertTrue(xfer3.id in id_list)
+
+    def test_get_many_requests_some_wrong_owner_subset(self):
+        src = "src://url"
+        dst = "dst://url"
+        sm = "src.module"
+        dm = "dst.module"
+
+        xfer1 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer2 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer3 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer4 = self.db.get_new_xfer('notme', src, dst, sm, dm)
+
+        id_list = [xfer1.id, xfer3.id]
+        xfer_list = self.db.get_xfer_requests(ids=id_list, owner=self.owner)
+        self.assertEqual(len(xfer_list), 2)
+        self.assertTrue(xfer1.id in id_list)
+        self.assertTrue(xfer3.id in id_list)
+
+    def test_get_many_requests_get_invalid(self):
+        src = "src://url"
+        dst = "dst://url"
+        sm = "src.module"
+        dm = "dst.module"
+
+        xfer1 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+        xfer2 = self.db.get_new_xfer(self.owner, src, dst, sm, dm)
+
+        id_list = [xfer1.id, xfer2.id, 'nothereatall']
+        xfer_list = self.db.get_xfer_requests(ids=id_list, owner=self.owner)
+        self.assertEqual(len(xfer_list), 2)
+        self.assertTrue(xfer1.id in id_list)
+        self.assertTrue(xfer2.id in id_list)

staccato/tests/unit/test_config.py

@@ -1,10 +1,29 @@
+import json
 import testtools
 
 from staccato.common import config
+from staccato.tests import utils
 
 
-class TestConfig(testtools.TestCase):
+class TestConfig(utils.TempFileCleanupBaseTest):
 
     def test_db_connection_default(self):
         conf = config.get_config_object(args=[])
         self.assertEquals(conf.sql_connection, 'sqlite:///staccato.sqlite')
+
+    def test_protocol_policy_retrieve_none(self):
+        conf = config.get_config_object(args=[])
+        conf.protocol_policy = None
+        j = config.get_protocol_policy(conf)
+        self.assertEqual(j, {})
+
+    def test_protocol_policy_retrieve(self):
+        conf = config.get_config_object(args=[])
+        p_file = self.get_tempfile()
+        policy = {"hello": "world"}
+        fptr = open(p_file, 'w')
+        fptr.write(json.dumps(policy))
+        fptr.close()
+        conf.protocol_policy = p_file
+        j = config.get_protocol_policy(conf)
+        self.assertEqual(j, policy)

staccato/tests/unit/test_utils.py

@@ -1,7 +1,12 @@
 import testtools
 
-import staccato.xfer.utils as xfers_utils
+import staccato.tests.utils as tests_utils
+import staccato.common.utils as common_utils
 
+import staccato.xfer.utils as xfers_utils
+import staccato.protocols.interface as proto_iface
+import staccato.common.exceptions as exceptions
+import staccato.common.config as config
 
 class FakeXferRequest(object):
         next_ndx = 0
@@ -66,3 +71,37 @@ class TestXferCheckpointerSingleSync(testtools.TestCase):
         blocks = [(30, 40), (0, 10), (20, 30), (10, 25)]
         self._run_blocks(blocks)
         self.assertEqual(self.fake_xfer.next_ndx, 40)
+
+class TestBasicUtils(tests_utils.TempFileCleanupBaseTest):
+
+    def test_empty_interface(self):
+        blank_interface = proto_iface.BaseProtocolInterface()
+        kwargs = {
+            'url_parts': None,
+            'writer': None,
+            'monitor': None,
+            'source_opts': None,
+            'start': 0,
+            'end': None,
+        }
+
+        interface_funcs = [blank_interface.get_reader,
+                           blank_interface.get_writer,
+                           blank_interface.new_read,
+                           blank_interface.new_write]
+
+        for func in interface_funcs:
+            self.assertRaises(
+                exceptions.StaccatoNotImplementedException,
+                func,
+                **kwargs)
+
+    def test_load_paste_app_error(self):
+        conf = config.get_config_object(args=[])
+        p_file = self.get_tempfile()
+        self.assertRaises(
+            RuntimeError,
+            common_utils.load_paste_app,
+            'notthere', p_file, conf)
+
+    # TODO: test the loading of a good app

tools/pip-requires

@@ -0,0 +1,39 @@
+# The greenlet package must be compiled with gcc and needs
+# the Python.h headers. Make sure you install the python-dev
+# package to get the right headers...
+greenlet>=0.3.1
+
+# < 0.8.0/0.8 does not work, see https://bugs.launchpad.net/bugs/1153983
+SQLAlchemy>=0.7.8,<=0.7.99
+anyjson
+eventlet>=0.9.12
+PasteDeploy
+routes
+WebOb>=1.2
+wsgiref
+argparse
+boto
+sqlalchemy-migrate>=0.7
+httplib2
+kombu
+pycrypto>=2.1.0alpha1
+iso8601>=0.1.4
+oslo.config>=1.1.0
+
+
+# For Swift storage backend.
+python-swiftclient>=1.2,<2
+
+# Note you will need gcc buildtools installed and must
+# have installed libxml headers for lxml to be successfully
+# installed using pip, therefore you will need to install the
+# libxml2-dev and libxslt-dev Ubuntu packages.
+lxml
+
+# For paste.util.template used in keystone.common.template
+Paste
+
+passlib
+jsonschema
+python-keystoneclient>=0.2.0
+pyOpenSSL

tools/test-requires

@@ -0,0 +1,29 @@
+# Packages needed for dev testing
+distribute>=0.6.24
+
+# Install bounded pep8/pyflakes first, then let flake8 install
+pep8==1.4.5
+pyflakes==0.7.2
+flake8==2.0
+hacking>=0.5.3,<0.6
+
+# For translations processing
+Babel
+
+# Needed for testing
+coverage
+fixtures>=0.3.12
+mox
+nose
+nose-exclude
+openstack.nose_plugin>=0.7
+nosehtmloutput>=0.0.3
+sphinx>=1.1.2
+requests
+testtools>=0.9.22
+
+# Optional packages that should be installed when testing
+MySQL-python
+psycopg2
+pysendfile==2.0.0
+xattr>=0.6.0

tox.ini

@@ -0,0 +1,32 @@
+[tox]
+envlist = py26,py27,pep8
+
+[testenv]
+setenv = VIRTUAL_ENV={envdir}
+         NOSE_WITH_OPENSTACK=1
+         NOSE_OPENSTACK_COLOR=1
+         NOSE_OPENSTACK_RED=0.05
+         NOSE_OPENSTACK_YELLOW=0.025
+         NOSE_OPENSTACK_SHOW_ELAPSED=1
+         NOSE_OPENSTACK_STDOUT=1
+deps = -r{toxinidir}/tools/pip-requires
+       -r{toxinidir}/tools/test-requires
+commands = nosetests {posargs}
+
+[tox:jenkins]
+downloadcache = ~/cache/pip
+
+[testenv:pep8]
+commands =
+  flake8
+
+[testenv:cover]
+setenv = NOSE_WITH_COVERAGE=1
+
+[testenv:venv]
+commands = {posargs}
+
+[flake8]
+ignore = E125,E126,E711,E712,F,H
+builtins = _
+exclude = .venv,.git,.tox,dist,doc,etc,*glance/locale*,*openstack/common*,*lib/python*,*egg,build