Add option to assign nodes via Gearman

As port af the Zuulv2.5 effort, this interface is not designed
to be long-lived.

Change-Id: I6392a08a232cf99c48b0adbc91000d8b16053ae3

devstack/plugin.sh

@@ -162,6 +162,7 @@ zmq-publishers: []
 # this does not need to be a jenkins target.
 targets:
   - name: dummy
+    assign-via-gearman: True
 
 cron:
   cleanup: '*/1 * * * *'

nodepool/cmd/nodepoolcmd.py

@@ -155,8 +155,8 @@ class NodePoolCmd(object):
                                        '%(message)s')
 
     def list(self, node_id=None):
-        t = PrettyTable(["ID", "Provider", "AZ", "Label", "Target", "Hostname",
-                         "NodeName", "Server ID", "IP", "State",
+        t = PrettyTable(["ID", "Provider", "AZ", "Label", "Target", "Manager",
+                         "Hostname", "NodeName", "Server ID", "IP", "State",
                          "Age"])
         t.align = 'l'
         with self.pool.getDB().getSession() as session:
@@ -164,7 +164,8 @@ class NodePoolCmd(object):
                 if node_id and node.id != node_id:
                     continue
                 t.add_row([node.id, node.provider_name, node.az,
-                           node.label_name, node.target_name, node.hostname,
+                           node.label_name, node.target_name,
+                           node.manager_name, node.hostname,
                            node.nodename, node.external_id, node.ip,
                            nodedb.STATE_NAMES[node.state],
                            NodePoolCmd._age(node.state_time)])

nodepool/config.py

@@ -278,6 +278,8 @@ def loadConfig(config_path):
         t.jenkins_apikey = None
         t.jenkins_credentials_id = None
 
+        t.assign_via_gearman = target.get('assign-via-gearman', False)
+
         t.hostname = target.get(
             'hostname',
             '{label.name}-{provider.name}-{node_id}'

nodepool/jobs.py

@@ -175,3 +175,12 @@ class ImageDeleteJob(NodepoolJob):
         self.image_id = image_id
         job_name = 'image-delete:%s' % image_id
         super(ImageDeleteJob, self).__init__(job_name, '', nodepool)
+
+
+class NodeAssignmentJob(NodepoolJob):
+    log = logging.getLogger("jobs.NodeAssignmentJob")
+
+    def __init__(self, node_id, target_name, data, nodepool):
+        self.node_id = node_id
+        job_name = 'node-assign:%s' % target_name
+        super(NodeAssignmentJob, self).__init__(job_name, data, nodepool)

nodepool/nodedb.py

@@ -87,6 +87,7 @@ node_table = Table(
     Column('provider_name', String(255), index=True, nullable=False),
     Column('label_name', String(255), index=True, nullable=False),
     Column('target_name', String(255), index=True, nullable=False),
+    Column('manager_name', String(255)),
     # Machine name
     Column('hostname', String(255), index=True),
     # Eg, jenkins node name
@@ -183,10 +184,11 @@ class SnapshotImage(object):
 class Node(object):
     def __init__(self, provider_name, label_name, target_name, az,
                  hostname=None, external_id=None, ip=None, ip_private=None,
-                 state=BUILDING):
+                 manager_name=None, state=BUILDING):
         self.provider_name = provider_name
         self.label_name = label_name
         self.target_name = target_name
+        self.manager_name = manager_name
         self.external_id = external_id
         self.az = az
         self.ip = ip

nodepool/nodepool.py

@@ -561,6 +561,10 @@ class NodeLauncher(threading.Thread):
             self.createJenkinsNode()
             self.log.info("Node id: %s added to jenkins" % self.node.id)
 
+        if self.target.assign_via_gearman:
+            self.log.info("Node id: %s assigning via gearman" % self.node.id)
+            self.assignViaGearman()
+
         return dt
 
     def createJenkinsNode(self):
@@ -585,6 +589,24 @@ class NodeLauncher(threading.Thread):
             params = dict(NODE=self.node.nodename)
             jenkins.startBuild(self.target.jenkins_test_job, params)
 
+    def assignViaGearman(self):
+        args = dict(name=self.node.nodename,
+                    host=self.node.ip,
+                    description='Dynamic single use %s node' % self.label.name,
+                    labels=self.label.name,
+                    root=self.image.user_home)
+        job = jobs.NodeAssignmentJob(self.node.id, self.node.target_name,
+                                     args, self.nodepool)
+        self.nodepool.gearman_client.submitJob(job, timeout=300)
+        job.waitForCompletion()
+        self.log.info("Node id: %s received %s from assignment" % (
+            self.node.id, job.data))
+        if job.failure:
+            raise Exception("Node id: %s received job failure on assignment" %
+                            self.node.id)
+        data = json.loads(job.data[-1])
+        self.node.manager_name = data['manager']
+
     def writeNodepoolInfo(self, nodelist):
         key = paramiko.RSAKey.generate(2048)
         public_key = key.get_name() + ' ' + key.get_base64()

nodepool/tests/test_commands.py

@@ -45,7 +45,7 @@ class TestNodepoolCMD(tests.DBTestCase):
         self.assert_listed(configfile, ['image-list'], 7, status, image_cnt)
 
     def assert_nodes_listed(self, configfile, node_cnt, status="ready"):
-        self.assert_listed(configfile, ['list'], 9, status, node_cnt)
+        self.assert_listed(configfile, ['list'], 10, status, node_cnt)
 
     def test_snapshot_image_update(self):
         configfile = self.setup_config("node.yaml")
@@ -233,7 +233,7 @@ class TestNodepoolCMD(tests.DBTestCase):
         self.assert_listed(configfile, ['list'], 0, 1, 1)
         self.assert_nodes_listed(configfile, 1, 'ready')
         # Delete node 1
-        self.assert_listed(configfile, ['delete', '1'], 9, 'delete', 1)
+        self.assert_listed(configfile, ['delete', '1'], 10, 'delete', 1)
 
     def test_delete_now(self):
         configfile = self.setup_config('node.yaml')

tools/fake.yaml

@@ -49,4 +49,5 @@ providers:
         setup: prepare_node_devstack.sh
 
 targets:
-  - name: fake-jenkins
+  - name: zuul
+    assign-via-gearman: True