Make PATEntries and SRV Recordset creation idempotent.

files/tatu.conf

@@ -7,7 +7,7 @@ use_barbican_key_manager = True
 #num_pat_bastions_per_server = 2
 #pat_dns_zone_name = tatuPAT.com.
 #pat_dns_zone_email = tatu@nono.nono
-#sqlalchemy_engine = mysql+pymysql://root:pinot@127.0.0.1/neutron?charset=utf8
+sqlalchemy_engine = mysql+pymysql://root:pinot@127.0.0.1
 auth_url = http://localhost/identity/v3
 user_id = fab01a1f2a7749b78a53dffe441a1879
 password = pinot

scripts/get-user-cert

@@ -57,7 +57,7 @@ response = requests.post(
     data=json.dumps(user)
 )
 if response.status_code != 201:
-    print 'Failed: ' + response
+    print 'Failed: ' + str(response)
     exit()
 
 assert 'location' in response.headers

tatu/api/models.py

@@ -213,7 +213,7 @@ class HostCerts(object):
                 'key-cert.pub': host.cert,
                 'hostname': host.hostname,
             }
-            if CONF.tatu.use_pat_bastion:
+            if CONF.tatu.use_pat_bastions:
                 item['pat_bastions'] = ','.join(
                     '{}:{}'.format(t[1], t[0]) for t in
                     get_port_ip_tuples(host.host_id, 22))
@@ -290,7 +290,7 @@ class NovaVendorData(object):
 
         # TODO(pino): make the whole workflow fault-tolerant
         # TODO(pino): make this configurable per project or subnet
-        if CONF.tatu.use_pat_bastion:
+        if CONF.tatu.use_pat_bastions:
             port_ip_tuples = create_pat_entries(self.session,
                                                 req.body['instance-id'], 22)
             add_srv_records(req.body['hostname'], req.body['project-id'],

tatu/config.py

@@ -41,7 +41,7 @@ opts = [
                default='tatu@nono.nono',
                help='Email of admin for DNS zone for PAT bastions'),
     cfg.StrOpt('sqlalchemy_engine',
-               default='mysql+pymysql://root:pinot@127.0.0.1/neutron?charset=utf8',
+               default='mysql+pymysql://root:pinot@127.0.0.1',
                help='SQLAlchemy database URL'),
     cfg.StrOpt('auth_url',
                default='http://localhost/identity/v3',
@@ -64,12 +64,12 @@ logging.register_options(CONF)
 log_levels = logging.get_default_log_levels() + \
              ['tatu=DEBUG', '__main__=DEBUG']
 logging.set_defaults(default_log_levels=log_levels)
+
+
 try:
-    CONF(args=[],
+    CONF(args=[], default_config_files=['/etc/tatu/tatu.conf',
-         default_config_files=['/etc/tatu/tatu.conf',
+                                       'files/tatu.conf',
-                                        'files/tatu.conf'
+                                       '/etc/neutron/dragonflow.ini'])
-                               ]
-         )
 except Exception as e:
     LOG.error("Failed to load configuration file: {}".format(e))
  
@@ -91,7 +91,6 @@ NOVA = nova_client.Client('2', session=session)
 NEUTRON = neutron_client.Client(session=session)
 DESIGNATE = designate_client.Client(session=session)
 
-dragonflow_cfg.CONF(args=[], default_config_files=['/etc/neutron/dragonflow.ini'])
 dragonflow_cfg.CONF.set_override('enable_df_pub_sub', False, group='df')
 DRAGONFLOW = api_nb.NbApi.get_instance(False)
 

tatu/db/models.py

@@ -68,7 +68,7 @@ class UserCert(Base):
     __tablename__ = 'user_certs'
 
     user_id = sa.Column(sa.String(36), primary_key=True)
-    fingerprint = sa.Column(sa.String(36), primary_key=True)
+    fingerprint = sa.Column(sa.String(60), primary_key=True)
     auth_id = sa.Column(sa.String(36), sa.ForeignKey('authorities.auth_id'))
     cert = sa.Column(sa.Text)
 

tatu/db/persistence.py

@@ -11,16 +11,22 @@
 #    under the License.
 
 import os
+from oslo_log import log as logging
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker, scoped_session
 
 from tatu import config
 from tatu.db.models import Base
 
+LOG = logging.getLogger(__name__)
+
 class SQLAlchemySessionManager(object):
 
     def __init__(self):
+        LOG.info('Creating sqlalchemy engine {}'.format(config.CONF.tatu.sqlalchemy_engine))
         self.engine = create_engine(config.CONF.tatu.sqlalchemy_engine)
+        self.engine.execute("CREATE DATABASE IF NOT EXISTS tatu;")
+        self.engine.execute("USE tatu;")
         Base.metadata.create_all(self.engine)
         self.Session = scoped_session(sessionmaker(self.engine))
 

tatu/dns.py

@@ -57,11 +57,15 @@ def add_srv_records(hostname, project_id, port_ip_tuples):
     for port, ip in port_ip_tuples:
         bastion = bastion_name_from_ip(ip)
         # SRV record format is: priority weight port A-name
-        records.add(
+        records.append(
             '10 50 {} {}'.format(port, bastion))
 
-    DESIGNATE.recordsets.create(ZONE['id'], get_srv_url(hostname, project_id),
+    try:
-                                'SRV', records)
+        DESIGNATE.recordsets.create(ZONE['id'],
+                                    get_srv_url(hostname, project_id),
+                                    'SRV', records)
+    except Conflict:
+        pass
 
 
 _setup_zone()

tatu/pat.py

@@ -86,26 +86,32 @@ def _df_find_lrouter_by_lport(lport):
 
 def get_port_ip_tuples(instance_id, fixed_lport):
     port_ip_tuples = []
+    all_entries = DRAGONFLOW.get_all(PATEntry)
+    LOG.debug('Found {} PATEntries: {}'.format(len(all_entries), all_entries))
     server = NOVA.servers.get(instance_id)
     ifaces = server.interface_list()
     for iface in ifaces:
-        lport = DRAGONFLOW.get(LogicalPort(id=iface['port_id']))
+        lport = DRAGONFLOW.get(LogicalPort(id=iface.port_id))
         lrouter = _df_find_lrouter_by_lport(lport)
         if lrouter is None: continue
-        pat_entries = DRAGONFLOW.get(PATEntry(lport=lport))
+        for entry in all_entries:
-        for entry in pat_entries:
+            if entry.lport.id == lport.id and entry.fixed_l4_port == fixed_lport:
-            if entry.fixed_l4_port == fixed_lport:
+                pat =  DRAGONFLOW.get(PAT(id=entry.pat.id))
-                port_ip_tuples.append((entry.pat_l4_port, str(entry.pat.ip)))
+                port_ip_tuples.append((entry.pat_l4_port, str(pat.ip_address)))
+        if port_ip_tuples: break
     return port_ip_tuples
 
 
 def create_pat_entries(sql_session, instance_id, fixed_l4_port,
                        num=CONF.tatu.num_pat_bastions_per_server):
-    port_ip_tuples = []
+    port_ip_tuples = get_port_ip_tuples(instance_id, fixed_l4_port)
+    LOG.debug('Found {} tuples: {}'.format(len(port_ip_tuples), port_ip_tuples))
+    if port_ip_tuples: return port_ip_tuples
+    LOG.debug('Creating new tuples.')
     server = NOVA.servers.get(instance_id)
     ifaces = server.interface_list()
     for iface in ifaces:
-        lport = DRAGONFLOW.get(LogicalPort(id=iface['port_id']))
+        lport = DRAGONFLOW.get(LogicalPort(id=iface.port_id))
         # TODO(pino): no router? consider SNAT of source IP to 169.254.169.254
         lrouter = _df_find_lrouter_by_lport(lport)
         if lrouter is None: continue
@@ -114,7 +120,7 @@ def create_pat_entries(sql_session, instance_id, fixed_l4_port,
         if (num < len(PATS)):
             pats = random.sample(pats, num)
         for pat in pats:
-            pat_l4_port = tatu_db.reserve_l4_port(sql_session, str(pat.ip))
+            pat_l4_port = tatu_db.reserve_l4_port(sql_session, str(pat.ip_address))
             pat_entry = PATEntry(
                 id = '{}:{}'.format(pat.id, pat_l4_port),
                 topic = 'tatu',
@@ -123,10 +129,10 @@ def create_pat_entries(sql_session, instance_id, fixed_l4_port,
                 fixed_ip_address = _get_ip4_from_lport(lport),
                 fixed_l4_port = fixed_l4_port,
                 lport = lport,
-                lrouter = df_fields.ReferenceField(LogicalRouter),
+                lrouter = lrouter,
             )
             DRAGONFLOW.create(pat_entry)
-            port_ip_tuples.append((pat_l4_port, str(pat.ip)))
+            port_ip_tuples.append((pat_l4_port, str(pat.ip_address)))
         # if we got here, we now have the required pat_entries
         break
     return port_ip_tuples