Fixes to PAT and DNS support.

tatu/api/models.py

@@ -15,16 +15,14 @@ import json
 import logging
 import uuid
 from Crypto.PublicKey import RSA
-from oslo_config import cfg
 from oslo_log import log as logging
 
+from tatu.config import CONF
+from tatu.db import models as db
 from tatu.dns import add_srv_records
 from tatu.pat import create_pat_entries
 
 LOG = logging.getLogger(__name__)
-CONF = cfg.CONF
-
-from tatu.db import models as db
 
 
 def validate_uuid(map, key):
@@ -237,7 +235,7 @@ class NovaVendorData(object):
         # TODO(pino): make the whole workflow fault-tolerant
         # TODO(pino): make this configurable per project or subnet
         if CONF.tatu.use_pat_bastion:
-            pat_entries = create_pat_entries(req.body['instance-id'], 22,
+            port_ip_tuples = create_pat_entries(self.session,
-                                             num=CONF.tatu.bastion_redundancy)
+                                             req.body['instance-id'], 22)
-            add_srv_records(req.body['project-id'], req.body['hostname'],
+            add_srv_records(req.body['hostname'], req.body['project-id'],
-                            pat_entries)
+                            port_ip_tuples)

tatu/config.py

@@ -10,22 +10,52 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from designateclient.v2 import client as designate_client
+from dragonflow import conf as dragonflow_cfg
+from dragonflow.db import api_nb
+from keystoneauth1 import session as keystone_session
+from keystoneauth1.identity import v3
+from novaclient import client as nova_client
+from neutronclient.v2_0 import client as neutron_client
 from oslo_config import cfg
 from oslo_log import log as logging
 from castellan.options import set_defaults as set_castellan_defaults
-from tatu import castellano
-import sys
 
 LOG = logging.getLogger(__name__)
-# 3 steps: register options; read the config file; use the options
 
+# 1) register options; 2) read the config file; 3) use the options
 opts = [
     cfg.BoolOpt('use_barbican_key_manager', default=False,
-                help='Enable the usage of the OpenStack Key Management '
+                help='Use OpenStack Barbican to store sensitive data'),
-                     'service provided by barbican.'),
+    cfg.BoolOpt('use_pat_bastions', default=True,
+                help='Use PAT as a "poor man\'s" approach to bastions'),
+    cfg.IntOpt('num_total_pats', default=3,
+                help='Number of available PAT addresses for bastions'),
+    cfg.IntOpt('num_pat_bastions_per_server', default=2,
+                help='Number of PAT bastions per server for redundancy'),
+    cfg.StrOpt('pat_dns_zone_name',
+               default='tatuPAT.com.',
+               help='Name of DNS zone for A and SRV records for PAT bastions'),
+    cfg.StrOpt('pat_dns_zone_email',
+               default='tatu@nono.nono',
+               help='Email of admin for DNS zone for PAT bastions'),
+    cfg.StrOpt('sqlalchemy_engine',
+               default='mysql+pymysql://root:pinot@127.0.0.1/neutron?charset=utf8',
+               help='SQLAlchemy database URL'),
+    cfg.StrOpt('auth_url',
+               default='http://localhost/identity/v3',
+               help='OpenStack Keystone URL'),
+    cfg.StrOpt('user_id',
+               default='fab01a1f2a7749b78a53dffe441a1879',
+               help='OpenStack Keystone admin privileged user-id'),
+    cfg.StrOpt('password',
+               default='pinot',
+               help='OpenStack Keystone password'),
+    cfg.StrOpt('project_id',
+               default='2e6c998ad16f4045821304470a57d160',
+               help='OpenStack Keystone admin project UUID'),
 ]
 
-DOMAIN = "tatu"
 CONF = cfg.CONF
 CONF.register_opts(opts, group='tatu')
 
@@ -33,14 +63,18 @@ logging.register_options(CONF)
 log_levels = logging.get_default_log_levels() + \
              ['tatu=DEBUG', '__main__=DEBUG']
 logging.set_defaults(default_log_levels=log_levels)
-#CONF(default_config_files=cfg.find_config_files(project='tatu', prog='tatu'))
 
 try:
-    CONF(args=[], default_config_files=['files/tatu.conf'])
+    CONF(args=[],
+         default_config_files=['/etc/tatu/tatu.conf',
+                                        'tatu.conf',
+                                        'files/tatu.conf'
+                               ]
+         )
 except Exception as e:
     LOG.error("Failed to load configuration file: {}".format(e))
  
-logging.setup(CONF, DOMAIN)
+logging.setup(CONF, "tatu")
 if CONF.tatu.use_barbican_key_manager:
     LOG.debug("Using Barbican as key manager.")
     set_castellan_defaults(CONF)
@@ -49,3 +83,15 @@ else:
     set_castellan_defaults(CONF,
                            api_class='tatu.castellano.TatuKeyManager')
 
+auth = v3.Password(auth_url=CONF.tatu.auth_url,
+                   user_id=CONF.tatu.user_id,
+                   password=CONF.tatu.password,
+                   project_id=CONF.tatu.project_id)
+session = keystone_session.Session(auth=auth)
+NOVA = nova_client.Client('2', session=session)
+NEUTRON = neutron_client.Client(session=session)
+DESIGNATE = designate_client.Client(session=session)
+
+dragonflow_cfg.CONF(default_config_files=['/etc/neutron/dragonflow.ini'])
+dragonflow_cfg.CONF.set_override('enable_df_pub_sub', False, group='df')
+DRAGONFLOW = api_nb.NbApi.get_instance(False)

tatu/db/models.py

@@ -200,3 +200,17 @@ def createHostCert(session, token_id, host_id, pub):
     session.add(token)
     session.commit()
     return host
+
+
+class L4PortReservation(Base):
+    __tablename__ = 'port_reservation'
+    ip_address = sa.Column(sa.String(36), primary_key=True)
+    # For now, just auto-increment the l4 port. Later, we'll reuse them.
+    l4_port = sa.Column(sa.Integer, primary_key=True, autoincrement=True)
+
+
+def reserve_l4_port(session, ip):
+    rsv = L4PortReservation(ip_address=str(ip))
+    session.add(rsv)
+    session.commit()
+    return rsv.l4_port

tatu/db/persistence.py

@@ -14,18 +14,13 @@ import os
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker, scoped_session
 
+from tatu import config
 from tatu.db.models import Base
 
-
-def get_url():
-    return os.getenv("DATABASE_URL", "sqlite:///development.db")
-    # return os.getenv("DATABASE_URL", "sqlite:///:memory:")
-
-
 class SQLAlchemySessionManager(object):
 
     def __init__(self):
-        self.engine = create_engine(get_url())
+        self.engine = create_engine(config.CONF.tatu.sqlalchemy_engine)
         Base.metadata.create_all(self.engine)
         self.Session = scoped_session(sessionmaker(self.engine))
 

tatu/dns.py

@@ -12,56 +12,54 @@
 
 import os
 from designateclient.exceptions import Conflict
-from designateclient.v2 import client
-from keystoneclient import session
-from keystoneclient.auth.identity.generic.password import Password
-from oslo_config import cfg
 from oslo_log import log as logging
 
+from tatu.config import CONF, DESIGNATE
+
 LOG = logging.getLogger(__name__)
-CONF = cfg.CONF
+ZONE = None
-
-auth = Password(auth_url=os.getenv('OS_AUTH_URL'),
-                username=os.getenv('OS_USERNAME'),
-                password=os.getenv('OS_PASSWORD'),
-                project_name=os.getenv('OS_PROJECT_NAME'),
-                project_domain_id='default',
-                user_domain_id='default')
-
-s = session.Session(auth=auth)
-
-client = client.Client(session=s)
-zone = None
-bastions = {}
 
 
-def setup(bastions=[]):
+def _setup_zone():
-    # TODO: retrieve the zone name and email from configuration
     try:
-        global zone
+        global ZONE
-        zone = client.zones.create('julia.com.', email='pino@yahoo.com')
+        ZONE = DESIGNATE.zones.create(CONF.tatu.pat_dns_zone_name,
+                                      email=CONF.tatu.pat_dns_zone_email)
     except Conflict:
         pass
 
-        # TODO: fetch all existing bastions
+
+def bastion_name_from_ip(ip_address):
+    return "bastion-{}.{}".format(ip_address.replace('.', '-'),
+                                  ZONE['name'])
 
 
-def add_bastion(ip_address, project_id, project_name, num):
+def register_bastion(ip_address):
-    bastion_name = "{}-{}-{}.{}".format(str(project_id)[:8], project_name, num,
+    try:
-                                        zone['name'])
+        DESIGNATE.recordsets.create(ZONE['id'],
-    client.recordsets.create(zone['id'], bastion_name, 'A', [ip_address])
+                                    bastion_name_from_ip(ip_address),
-    bastions.add(ip_address, bastion_name)
+                                    'A', [ip_address])
-    return bastion_name
+    except Conflict:
+        pass
 
 
-def add_srv_records(project_id, hostname, pat_entries):
+def sync_bastions(ip_addresses):
+    for ip in ip_addresses:
+        register_bastion(ip)
+
+
+def add_srv_records(hostname, project_id, port_ip_tuples):
     records = []
-    for pat_entry in pat_entries:
+    for port, ip in port_ip_tuples:
-        b = bastions[pat_entries.pat.ip_address]
+        bastion = bastion_name_from_ip(ip)
         # SRV record format is: priority weight port A-name
         records.add(
-            '10 50 {} {}'.format(pat_entry.pat_l4_port, b))
+            '10 50 {} {}'.format(port, bastion))
 
-    client.recordsets.create(zone['id'],
+    DESIGNATE.recordsets.create(ZONE['id'],
-                             'ssh.{}.{}'.format(hostname, project_id[:8]),
+                                '_ssh._tcp.{}.{}'.format(hostname,
-                             'SRV', records)
+                                                         project_id[:8]),
+                                'SRV', records)
+
+
+_setup_zone()

tatu/pat.py

@@ -10,81 +10,105 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from dragonflow.db import api_nb
+from dragonflow.db.models.core import Chassis
-from dragonflow.db.models import l3
+from dragonflow.db.models.l2 import LogicalPort
-from oslo_log import log
+from dragonflow.db.models.l2 import LogicalRouter
-from neutronclient.v2_0 import client
+from dragonflow.db.models.l3 import PAT
-from novaclient import client
+from dragonflow.db.models.l3 import PATEntry
+from oslo_log import log as logging
 import random
-from tatu.db import models as db
+from tatu import dns
+from tatu.config import CONF, NEUTRON, NOVA, DRAGONFLOW
+from tatu.db import models as tatu_db
 
-# Need to load /etc/neutron/dragonflow.ini
+LOG = logging.getLogger(__name__)
-# config.init(sys.argv[1:])
+PATS = DRAGONFLOW.get_all(PAT)
-dragonflow = api_nb.NbApi.get_instance(False)
 
-def add_pat():
+def _sync_pats():
-    # First choose a host where the PAT will be bound.
+    # TODO(pino): re-bind PATs when hypervisors fail (here and on notification)
-    nova = client.Client(VERSION, USERNAME, PASSWORD, PROJECT_ID, AUTH_URL)
+    all_chassis = DRAGONFLOW.get_all(Chassis)
-    hosts = nova.servers.list()
+    # Filter the chassis that already have PATS assigned
-    host_id = random.sample(hosts, 1)[0].id
+    free_chassis = set(all_chassis).difference(p.chassis for p in PATS)
+    # Don't make more PATs than there are free chassis
+    num_to_make = min(CONF.tatu.num_total_pats - len(PATS),
+                      len(free_chassis))
+    if num_to_make <= 0:
+        return
+    assigned_chassis = random.sample(free_chassis, num_to_make)
+    for c in assigned_chassis:
+        _add_pat(c)
+    dns.sync_bastions(str(p.ip_address) for p in PATS)
 
-    # Now create the new port on the public network.
-    neutron = client.Client(username=USER,
-                            password=PASS,
-                            project_name=PROJECT_NAME,
-                            auth_url=KEYSTONE_URL)
 
-    # Find the public network and allocate 2 ports.
+def _add_pat(chassis):
-    networks = neutron.list_networks(name='public')
+    # Find the public network and allocate a new port.
+    networks = NEUTRON.list_networks(name='public')
     network_id = networks['networks'][0]['id']
-
+    body = {
-    body_value = {
         "port": {
             "admin_state_up": True,
-            "name": TatuPAT,
+            "name": 'TatuPAT', # TODO(pino): set device owner to Tatu?
             "network_id": network_id,
-            "binding: host_id": host_id
         }
     }
-    pat_lport = neutron.create_port()
+    neutron_port = NEUTRON.create_port(body)
-    # TODO: Bind the port to a specific host
+    lport = DRAGONFLOW.get(LogicalPort(id=neutron_port['port']['id']))
-
+    ip = get_ip4_from_lport(lport)
-    pat = l3.PAT(
+    pat = PAT(
-        topic = 'foo',
+        id = str(ip),
-        ip_address = pat_lport.ip,
+        topic = 'tatu', # TODO(pino): What topic? Admin project_id?
-        lport = pat_lport
+        ip_address = ip,
+        lport = lport
     )
-    dragonflow.create(pat)
+    # We only need to store the PAT in dragonflow's DB, not API/MySQL
-    db.add_pat(pat.lport)
+    DRAGONFLOW.create(pat)
-    return pat_lport.ip
+    PATS.append(pat)
 
 
-# At startup, we create 1 PAT if none exists
+def _get_ip4_from_lport(lport):
-if not db.get_pats():
+    for ip in lport.ips:
-    add_pat()
+        if ip.version is 4:
-
+            return ip
-# TODO(pino): need to re-bind PATs when hosts fail.
+    return None
 
 
-def create_pat_entries(instance_id, fixed_l4_port, num=2):
+def df_find_lrouter_by_lport(lport):
-    # TODO(pino): Use Neutron client to find a suitable lport on the instance
+    lrouters = DRAGONFLOW.get_all(LogicalRouter)
-    lport = None
+    for lr in lrouters:
-    lrouter = None
+        for lp in lr.ports:
-    # Reserve N assignments (i.e. IP:port pairs) on distinct IPs.
+            if lp.lswitch.id == lport.lswitch.id:
-    pats = db.get_pats()
+                return lr
-    pat_entries = set()
+    return None
-    if (num < len(pats)):
+
-        pats = random.sample(pats, num_assignments)
+def create_pat_entries(sql_session, instance_id, fixed_l4_port,
-    for pat in pats:
+                       num=CONF.tatu.num_pat_bastions_per_server):
-        pat_l4_port = db.reserve_l4_port(pat.ip, lport.id, lport.ip, fixed_l4_port)
+    port_ip_tuples = []
-        pat_entry = l3.PATEntry(
+    server = NOVA.servers.get(instance_id)
-            pat = pat,
+    ifaces = server.interface_list()
-            pat_l4_port = pat_l4_port,
+    for iface in ifaces:
-            fixed_ip_address = lport.ip,
+        lport = DRAGONFLOW.get(LogicalPort(id=iface['port_id']))
-            fixed_l4_port = fixed_l4_port,
+        # TODO(pino): no router? consider SNAT of source IP to 169.254.169.254
-            lport = lport,
+        lrouter = df_find_lrouter_by_lport(lport)
-            lrouter = df_fields.ReferenceField(LogicalRouter),
+        if lrouter is None: continue
-        )
+        # Reserve N l4 ports on distinct IPs.
-        dragonflow.create(pat_entry)
+        pats = PATS
-        pat_entries.add(pat_entry)
+        if (num < len(PATS)):
-    return pat_entries
+            pats = random.sample(pats, num)
+        for pat in pats:
+            pat_l4_port = tatu_db.reserve_l4_port(sql_session, str(pat.ip))
+            pat_entry = PATEntry(
+                pat = pat,
+                pat_l4_port = pat_l4_port,
+                fixed_ip_address = _get_ip4_from_lport(lport),
+                fixed_l4_port = fixed_l4_port,
+                lport = lport,
+                lrouter = df_fields.ReferenceField(LogicalRouter),
+            )
+            DRAGONFLOW.create(pat_entry)
+            port_ip_tuples.append((pat_l4_port, str(pat.ip)))
+        # if we got here, we now have the required pat_entries
+        break
+    return port_ip_tuples
+
+
+_sync_pats()

tatu/utils.py

@@ -13,8 +13,8 @@
 import os
 import shutil
 import subprocess
-from tempfile import mkdtemp
 import uuid
+from tempfile import mkdtemp
 
 
 def random_uuid():
@@ -24,7 +24,6 @@ def random_uuid():
 def generateCert(auth_key, entity_key, hostname=None, principals='root'):
     # Temporarily write the authority private key, entity public key to files
     prefix = uuid.uuid4().hex
-    # Todo: make the temporary directory configurable or secure it.
     temp_dir = mkdtemp()
     ca_file = '/'.join([temp_dir, 'ca_key'])
     pub_file = '/'.join([temp_dir, 'entity.pub'])