Merge pull request #296 from asadoughi/rm9115_iter0

Iteration of security groups quark agent code

quark/agent/agent.py

@@ -18,6 +18,7 @@ import sys
 import time
 
 from neutron.common import config
+from neutron.common import utils as n_utils
 from neutron.openstack.common import log as logging
 from oslo.config import cfg
 
@@ -60,9 +61,9 @@ def run():
             _sleep()
             continue
 
-        new_instances = set(new_instances.values())
+        new_instances_set = set(new_instances.values())
-        added_instances = new_instances - instances
+        added_instances = new_instances_set - instances
-        removed_instances = instances - new_instances
+        removed_instances = instances - new_instances_set
         if added_instances or removed_instances:
             LOG.debug("instances: added %s removed %s",
                       added_instances, removed_instances)
@@ -77,8 +78,8 @@ def run():
             new_security_groups = redis_client.get_security_groups(
                 new_interfaces)
             added_sg, updated_sg, removed_sg = vc.diff(new_security_groups)
-            xapi.update_interfaces(new_instances,
+            xapi_client.update_interfaces(new_instances,
-                                   added_sg, updated_sg, removed_sg)
+                                          added_sg, updated_sg, removed_sg)
         except Exception:
             LOG.exception("Unable to get security groups from Redis and apply"
                           " them to xapi")
@@ -87,13 +88,15 @@ def run():
 
         vc.commit(new_security_groups)
 
-        instances = new_instances
+        instances = new_instances_set
         interfaces = new_interfaces
         _sleep()
 
 
 def main():
     config.init(sys.argv[1:])
+    config.setup_logging()
+    n_utils.log_opt_values(LOG)
     if not CONF.config_file:
         sys.exit(_("ERROR: Unable to find configuration file via the default"
                    " search paths (~/.neutron/, ~/, /etc/neutron/, /etc/) and"

quark/agent/tests/test_version_control.py

@@ -89,6 +89,18 @@ class TestVersionControl(test_base.TestBase):
             self.assertEqual(updated, [])
             self.assertEqual(removed, [])
 
+    def test_diff_nonempty_file_added_new_security_groups_null(self):
+        o_fn = "quark.agent.version_control._open_or_create_file_for_reading"
+        with mock.patch(o_fn) as m_open:
+            m_open.return_value = StringIO('{"3.4": null}')
+            added, updated, removed = self.version_control.diff({
+                VIF("3", "4"): "bar",
+            })
+            m_open.assert_called_once_with(self.FILEPATH)
+            self.assertEqual(added, [VIF("3", "4")])
+            self.assertEqual(updated, [])
+            self.assertEqual(removed, [])
+
     def test_diff_nonempty_file_updated_new_security_groups(self):
         o_fn = "quark.agent.version_control._open_or_create_file_for_reading"
         with mock.patch(o_fn) as m_open:
@@ -111,6 +123,18 @@ class TestVersionControl(test_base.TestBase):
             self.assertEqual(updated, [])
             self.assertEqual(removed, [VIF("3", "4")])
 
+    def test_diff_nonempty_file_removed_new_security_groups_null(self):
+        o_fn = "quark.agent.version_control._open_or_create_file_for_reading"
+        with mock.patch(o_fn) as m_open:
+            m_open.return_value = StringIO('{"3.4": "bar"}')
+            added, updated, removed = self.version_control.diff({
+                VIF("3", "4"): None,
+            })
+            m_open.assert_called_once_with(self.FILEPATH)
+            self.assertEqual(added, [])
+            self.assertEqual(updated, [])
+            self.assertEqual(removed, [VIF("3", "4")])
+
     def test_diff_nonempty_file_all_kinds_new_security_groups(self):
         o_fn = "quark.agent.version_control._open_or_create_file_for_reading"
         with mock.patch(o_fn) as m_open:

quark/agent/version_control.py

@@ -84,13 +84,14 @@ class VersionControl(object):
 
         vifs = set(old_security_groups.keys() + new_security_groups.keys())
         for vif in vifs:
-            old_version = old_security_groups.get(vif)
+            old_version_null = old_security_groups.get(vif) is None
-            new_version = new_security_groups.get(vif)
+            new_version_null = new_security_groups.get(vif) is None
-            if not new_version:
+            if not old_version_null and new_version_null:
                 removed_sg.append(vif)
-            elif not old_version:
+            if old_version_null and not new_version_null:
                 added_sg.append(vif)
-            elif new_version != old_version:
+            if (not old_version_null and not new_version_null and
+                    old_security_groups[vif] != new_security_groups[vif]):
                 updated_sg.append(vif)
 
         return added_sg, updated_sg, removed_sg

quark/security_groups/redis_client.py

@@ -257,12 +257,14 @@ class Client(object):
         Returns a dictionary of xapi.VIFs mapped to security group version
         UUIDs from a set of xapi.VIF.
         """
-
+        LOG.debug("Getting security groups from Redis for {0}".format(
+            new_interfaces))
         new_interfaces = tuple(new_interfaces)
 
         p = self._client.pipeline()
         for vif in new_interfaces:
-            p.get(str(vif))
+            key = self.rule_key(vif.device_id, vif.mac_address)
+            p.get(key)
         security_groups = p.execute()
 
         ret = {}

quark/tests/security_groups/test_redis_client.py

@@ -267,10 +267,10 @@ class TestRedisForAgent(test_base.TestBase):
         new_interfaces = ([VIF(1, 2), VIF(3, 4), VIF(5, 6), VIF(7, 8)])
         group_uuids = rc.get_security_groups(new_interfaces)
         mock_pipeline.get.assert_has_calls(
-            [mock.call("1.2"),
+            [mock.call("1.000000000002"),
-             mock.call("3.4"),
+             mock.call("3.000000000004"),
-             mock.call("5.6"),
+             mock.call("5.000000000006"),
-             mock.call("7.8")],
+             mock.call("7.000000000008")],
             any_order=True)
         mock_pipeline.execute.assert_called_once_with()
         self.assertEqual(group_uuids,