add option to use ipv6 for image update and node launching

add option to use ipv6 as ssh connect ip for building snapshot
image and launching jenkins slaves.

Conflicts:
	doc/source/configuration.rst
	nodepool/nodepool.py

Change-Id: I7e023e7581fc0b5ec1ee34d1e5a1eeaacd7d3bfd

doc/source/configuration.rst

@@ -243,6 +243,7 @@ provider, the Nodepool image types are also defined (see
       template-hostname: '{image.name}-{timestamp}.template.openstack.org'
       pool: 'public'
       image-type: qcow2
+      ipv6-preferred: False
       networks:
         - net-id: 'some-uuid'
         - net-label: 'some-network-name'
@@ -348,6 +349,12 @@ provider, the Nodepool image types are also defined (see
     queried via the Nova os-tenant-networks API extension (this requires that
     the cloud provider has deployed this extension).
 
+  ``ipv6_preferred``
+    If it is set to True, nodepool will try to find ipv6 in public net first
+    as the ip address for ssh connection to build snapshot images and create
+    jenkins slave definition. If ipv6 is not found or the key is not
+    specified or set to False, ipv4 address will be used.
+
   ``pool``
     Specify a floating ip pool in cases where the 'public' pool is unavailable
     or undesirable.

nodepool/fakeprovider.py

@@ -83,14 +83,29 @@ class FakeList(object):
 
     def create(self, **kw):
         should_fail = kw.get('SHOULD_FAIL', '').lower() == 'true'
+        nics = kw.get('nics', [])
+        addresses = None
+        # if keyword 'ipv6-uuid' is found in provider config,
+        # ipv6 address will be available in public addr dict.
+        for nic in nics:
+            if 'ipv6-uuid' not in nic['net-id']:
+                continue
+            addresses = dict(
+                public=[dict(version=4, addr='fake'),
+                        dict(version=6, addr='fake_v6')],
+                private=[dict(version=4, addr='fake')]
+            )
+            break
+        if not addresses:
+            addresses = dict(
+                public=[dict(version=4, addr='fake')],
+                private=[dict(version=4, addr='fake')]
+            )
         s = Dummy(id=uuid.uuid4().hex,
                   name=kw['name'],
                   status='BUILD',
                   adminPass='fake',
-                  addresses=dict(
-                      public=[dict(version=4, addr='fake')],
-                      private=[dict(version=4, addr='fake')]
-                  ),
+                  addresses=addresses,
                   metadata=kw.get('meta', {}),
                   manager=self,
                   should_fail=should_fail)

nodepool/nodepool.py

@@ -478,6 +478,13 @@ class NodeLauncher(threading.Thread):
                                          server['status']))
 
         ip = server.get('public_v4')
+        ip_v6 = server.get('public_v6')
+        if self.provider.ipv6_preferred:
+            if ip_v6:
+                ip = ip_v6
+            else:
+                self.log.warning('Preferred ipv6 not available, '
+                                 'falling back to ipv4.')
         if not ip and self.manager.hasExtension('os-floating-ips'):
             ip = self.manager.addPublicIP(server_id,
                                           pool=self.provider.pool)
@@ -486,8 +493,9 @@ class NodeLauncher(threading.Thread):
 
         self.node.ip_private = server.get('private_v4')
         self.node.ip = ip
-        self.log.debug("Node id: %s is running, ip: %s, ipv6: %s" %
-                       (self.node.id, ip, server.get('public_v6')))
+        self.log.debug("Node id: %s is running, ipv4: %s, ipv6: %s" %
+                       (self.node.id, server.get('public_v4'),
+                        server.get('public_v6')))
 
         self.log.debug("Node id: %s testing ssh at ip: %s" %
                        (self.node.id, ip))
@@ -761,6 +769,13 @@ class SubNodeLauncher(threading.Thread):
                                          self.node_id, server['status']))
 
         ip = server.get('public_v4')
+        ip_v6 = server.get('public_v6')
+        if self.provider.ipv6_preferred:
+            if ip_v6:
+                ip = ip_v6
+            else:
+                self.log.warning('Preferred ipv6 not available, '
+                                 'falling back to ipv4.')
         if not ip and self.manager.hasExtension('os-floating-ips'):
             ip = self.manager.addPublicIP(server_id,
                                           pool=self.provider.pool)
@@ -770,8 +785,8 @@ class SubNodeLauncher(threading.Thread):
         self.subnode.ip_private = server.get('private_v4')
         self.subnode.ip = ip
         self.log.debug("Subnode id: %s for node id: %s is running, "
-                       "ip: %s, ipv6: %s" %
-                       (self.subnode_id, self.node_id, ip,
+                       "ipv4: %s, ipv6: %s" %
+                       (self.subnode_id, self.node_id, server.get('public_v4'),
                         server.get('public_v6')))
 
         self.log.debug("Subnode id: %s for node id: %s testing ssh at ip: %s" %
@@ -1092,12 +1107,19 @@ class SnapshotImageUpdater(ImageUpdater):
                             (server_id, self.snap_image.id, server['status']))
 
         ip = server.get('public_v4')
+        ip_v6 = server.get('public_v6')
+        if self.provider.ipv6_preferred:
+            if ip_v6:
+                ip = ip_v6
+            else:
+                self.log.warning('Preferred ipv6 not available, '
+                                 'falling back to ipv4.')
         if not ip and self.manager.hasExtension('os-floating-ips'):
             ip = self.manager.addPublicIP(server_id,
                                           pool=self.provider.pool)
         if not ip:
             raise Exception("Unable to find public IP of server")
-        server['public_v4'] = ip
+        server['public_ip'] = ip
 
         self.bootstrapServer(server, key, use_password=use_password)
 
@@ -1145,7 +1167,7 @@ class SnapshotImageUpdater(ImageUpdater):
         else:
             ssh_kwargs['password'] = server['admin_pass']
 
-        host = utils.ssh_connect(server['public_v4'], 'root', ssh_kwargs,
+        host = utils.ssh_connect(server['public_ip'], 'root', ssh_kwargs,
                                  timeout=CONNECT_TIMEOUT)
 
         if not host:
@@ -1154,7 +1176,7 @@ class SnapshotImageUpdater(ImageUpdater):
             # didn't occur), we can connect with a very sort timeout.
             for username in ['ubuntu', 'fedora', 'cloud-user', 'centos']:
                 try:
-                    host = utils.ssh_connect(server['public_v4'], username,
+                    host = utils.ssh_connect(server['public_ip'], username,
                                              ssh_kwargs,
                                              timeout=10)
                     if host:
@@ -1341,6 +1363,7 @@ class NodePool(threading.Thread):
             p.launch_timeout = provider.get('launch-timeout', 3600)
             p.use_neutron = bool(provider.get('networks', ()))
             p.networks = provider.get('networks')
+            p.ipv6_preferred = provider.get('ipv6-preferred')
             p.azs = provider.get('availability-zones')
             p.template_hostname = provider.get(
                 'template-hostname',
@@ -1479,6 +1502,7 @@ class NodePool(threading.Thread):
             new_pm.launch_timeout != old_pm.provider.launch_timeout or
             new_pm.use_neutron != old_pm.provider.use_neutron or
             new_pm.networks != old_pm.provider.networks or
+            new_pm.ipv6_preferred != old_pm.provider.ipv6_preferred or
             new_pm.azs != old_pm.provider.azs):
             return False
         new_images = new_pm.images

nodepool/nodeutils.py

@@ -43,7 +43,7 @@ def iterate_timeout(max_seconds, purpose):
 
 
 def ssh_connect(ip, username, connect_kwargs={}, timeout=60):
-    if ip == 'fake':
+    if 'fake' in ip:
         return fakeprovider.FakeSSHClient()
     # HPcloud may return ECONNREFUSED or EHOSTUNREACH
     # for about 30 seconds after adding the IP

nodepool/tests/__init__.py

@@ -102,6 +102,7 @@ class BaseTestCase(testtools.TestCase, testresources.ResourcedTestCase):
                      'fake-provider',
                      'fake-provider1',
                      'fake-provider2',
+                     'fake-provider3',
                      'fake-dib-provider',
                      'fake-jenkins',
                      'fake-target',

nodepool/tests/fixtures/node_ipv6.yaml

@@ -0,0 +1,96 @@
+script-dir: .
+dburi: '{dburi}'
+
+cron:
+  check: '*/15 * * * *'
+  cleanup: '*/1 * * * *'
+  image-update: '14 2 * * *'
+
+zmq-publishers:
+  - tcp://localhost:8881
+
+#gearman-servers:
+#  - host: localhost
+
+labels:
+  - name: fake-label1
+    image: fake-image
+    min-ready: 1
+    providers:
+      - name: fake-provider1
+
+  - name: fake-label2
+    image: fake-image
+    min-ready: 1
+    providers:
+      - name: fake-provider2
+
+  - name: fake-label3
+    image: fake-image
+    min-ready: 1
+    providers:
+      - name: fake-provider3
+
+providers:
+  - name: fake-provider1
+    keypair: 'if-present-use-this-keypair'
+    username: 'fake'
+    password: 'fake'
+    auth-url: 'fake'
+    project-id: 'fake'
+    max-servers: 96
+    pool: 'fake'
+    networks:
+      - net-id: 'ipv6-uuid'
+    ipv6-preferred: True
+    rate: 0.0001
+    images:
+      - name: fake-image
+        base-image: 'Fake Precise'
+        min-ram: 8192
+        name-filter: 'Fake'
+        setup: prepare_node_devstack.sh
+
+  - name: fake-provider2
+    keypair: 'if-present-use-this-keypair'
+    username: 'fake'
+    password: 'fake'
+    auth-url: 'fake'
+    project-id: 'fake'
+    max-servers: 96
+    pool: 'fake'
+    networks:
+      - net-id: 'ipv6-uuid'
+    rate: 0.0001
+    images:
+      - name: fake-image
+        base-image: 'Fake Precise'
+        min-ram: 8192
+        name-filter: 'Fake'
+        setup: prepare_node_devstack.sh
+
+  - name: fake-provider3
+    keypair: 'if-present-use-this-keypair'
+    username: 'fake'
+    password: 'fake'
+    auth-url: 'fake'
+    project-id: 'fake'
+    max-servers: 96
+    pool: 'fake'
+    networks:
+      - net-id: 'some-uuid'
+    ipv6-preferred: True
+    rate: 0.0001
+    images:
+      - name: fake-image
+        base-image: 'Fake Precise'
+        min-ram: 8192
+        name-filter: 'Fake'
+        setup: prepare_node_devstack.sh
+
+targets:
+  - name: fake-target
+    jenkins:
+      url: https://jenkins.example.org/
+      user: fake
+      apikey: fake

nodepool/tests/test_nodepool.py

@@ -207,6 +207,39 @@ class TestNodepool(tests.DBTestCase):
             self.assertEqual(len(nodes), 1)
             self.assertEqual(nodes[0].az, 'az1')
 
+    def test_node_ipv6(self):
+        """Test that a node is created w/ or w/o ipv6 preferred flag"""
+        configfile = self.setup_config('node_ipv6.yaml')
+        pool = self.useNodepool(configfile, watermark_sleep=1)
+        pool.start()
+        self.waitForImage(pool, 'fake-provider1', 'fake-image')
+        self.waitForImage(pool, 'fake-provider2', 'fake-image')
+        self.waitForImage(pool, 'fake-provider3', 'fake-image')
+        self.waitForNodes(pool)
+
+        with pool.getDB().getSession() as session:
+            # ipv6 preferred set to true and ipv6 address available
+            nodes = session.getNodes(provider_name='fake-provider1',
+                                     label_name='fake-label1',
+                                     target_name='fake-target',
+                                     state=nodedb.READY)
+            self.assertEqual(len(nodes), 1)
+            self.assertEqual(nodes[0].ip, 'fake_v6')
+            # ipv6 preferred unspecified and ipv6 address available
+            nodes = session.getNodes(provider_name='fake-provider2',
+                                     label_name='fake-label2',
+                                     target_name='fake-target',
+                                     state=nodedb.READY)
+            self.assertEqual(len(nodes), 1)
+            self.assertEqual(nodes[0].ip, 'fake')
+            # ipv6 preferred set to true but ipv6 address unavailable
+            nodes = session.getNodes(provider_name='fake-provider3',
+                                     label_name='fake-label3',
+                                     target_name='fake-target',
+                                     state=nodedb.READY)
+            self.assertEqual(len(nodes), 1)
+            self.assertEqual(nodes[0].ip, 'fake')
+
     def test_node_delete_success(self):
         configfile = self.setup_config('node.yaml')
         pool = self.useNodepool(configfile, watermark_sleep=1)