Import code working with Ironic nodes from os_cloud_config

This patch introduces tripleo_common.utils.{glance,nodes} modules.
The glance one is imported with only import fixes (including importing
a private exceptions module from the glance client).

The nodes.py is a rewritten and fixed version taken from
https://review.openstack.org/#/c/263309/. Main changes:
* Stop hardcoding flavor (e.g. agent, pxe) for each driver
* Support only generic properties with pm_* names. Driver-specific things
  should stay with their prefix. Existing pm_* driver-specific things
  were deprecated.
* Pass through everything that starts with driver-specific prefix to
  node's driver_info.
* Dropped handling Conflict exceptions - ironicclient is doing it for
  some time already (and does better job in it).
* Issue a specific exception for malformed instackenv.json.
* Optimize calls to ironic (use list with details instead of list+get)
* Use 'add' operation instead of 'replace', as it allows both adding
  and overwriting (despite its name).
* Fixed some small issues like adding a dict to a set.

Change-Id: I7efffc5c6627776a20fad4bf4cf266330c4b8b6b

requirements.txt

@@ -8,3 +8,6 @@ python-heatclient>=0.6.0 # Apache-2.0
 oslo.config>=3.7.0 # Apache-2.0
 oslo.log>=1.14.0 # Apache-2.0
 oslo.utils>=3.5.0 # Apache-2.0
+python-glanceclient>=2.0.0 # Apache-2.0
+python-ironicclient>=1.1.0 # Apache-2.0
+six>=1.9.0 # MIT

tripleo_common/exception.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+class InvalidNode(ValueError):
+    """Node data is invalid."""
+
+    def __init__(self, message, node=None):
+        message = 'Invalid node data: %s' % message
+        self.node = node
+        super(InvalidNode, self).__init__(message)

tripleo_common/tests/utils/test_glance.py

@@ -0,0 +1,100 @@
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import collections
+import tempfile
+
+from glanceclient import exc as exceptions
+import mock
+import testtools
+
+from tripleo_common.tests import base
+from tripleo_common.utils import glance
+
+
+class GlanceTest(base.TestCase):
+
+    def setUp(self):
+        super(GlanceTest, self).setUp()
+        self.image = collections.namedtuple('image', ['id'])
+
+    def test_return_existing_kernel_and_ramdisk(self):
+        client = mock.MagicMock()
+        expected = {'kernel': 'aaa', 'ramdisk': 'zzz'}
+        client.images.find.side_effect = (self.image('aaa'), self.image('zzz'))
+        ids = glance.create_or_find_kernel_and_ramdisk(client, 'bm-kernel',
+                                                       'bm-ramdisk')
+        client.images.create.assert_not_called()
+        self.assertEqual(expected, ids)
+
+    def test_raise_exception_kernel(self):
+        client = mock.MagicMock()
+        client.images.find.side_effect = exceptions.NotFound
+        message = "Kernel image not found in Glance, and no path specified."
+        with testtools.ExpectedException(ValueError, message):
+            glance.create_or_find_kernel_and_ramdisk(client, 'bm-kernel',
+                                                     None)
+
+    def test_raise_exception_ramdisk(self):
+        client = mock.MagicMock()
+        client.images.find.side_effect = (self.image('aaa'),
+                                          exceptions.NotFound)
+        message = "Ramdisk image not found in Glance, and no path specified."
+        with testtools.ExpectedException(ValueError, message):
+            glance.create_or_find_kernel_and_ramdisk(client, 'bm-kernel',
+                                                     'bm-ramdisk')
+
+    def test_skip_missing_no_kernel(self):
+        client = mock.MagicMock()
+        client.images.find.side_effect = (exceptions.NotFound,
+                                          self.image('bbb'))
+        expected = {'kernel': None, 'ramdisk': 'bbb'}
+        ids = glance.create_or_find_kernel_and_ramdisk(
+            client, 'bm-kernel', 'bm-ramdisk', skip_missing=True)
+        self.assertEqual(ids, expected)
+
+    def test_skip_missing_no_ramdisk(self):
+        client = mock.MagicMock()
+        client.images.find.side_effect = (self.image('aaa'),
+                                          exceptions.NotFound)
+        expected = {'kernel': 'aaa', 'ramdisk': None}
+        ids = glance.create_or_find_kernel_and_ramdisk(
+            client, 'bm-kernel', 'bm-ramdisk', skip_missing=True)
+        self.assertEqual(ids, expected)
+
+    def test_skip_missing_kernel_and_ramdisk(self):
+        client = mock.MagicMock()
+        client.images.find.side_effect = exceptions.NotFound
+        expected = {'kernel': None, 'ramdisk': None}
+        ids = glance.create_or_find_kernel_and_ramdisk(
+            client, 'bm-kernel', 'bm-ramdisk', skip_missing=True)
+        self.assertEqual(ids, expected)
+
+    def test_create_kernel_and_ramdisk(self):
+        client = mock.MagicMock()
+        client.images.find.side_effect = exceptions.NotFound
+        client.images.create.side_effect = (self.image('aaa'),
+                                            self.image('zzz'))
+        expected = {'kernel': 'aaa', 'ramdisk': 'zzz'}
+        with tempfile.NamedTemporaryFile() as imagefile:
+            ids = glance.create_or_find_kernel_and_ramdisk(
+                client, 'bm-kernel', 'bm-ramdisk', kernel_path=imagefile.name,
+                ramdisk_path=imagefile.name)
+        kernel_create = mock.call(name='bm-kernel', disk_format='aki',
+                                  is_public=True, data=mock.ANY)
+        ramdisk_create = mock.call(name='bm-ramdisk', disk_format='ari',
+                                   is_public=True, data=mock.ANY)
+        client.images.create.assert_has_calls([kernel_create, ramdisk_create])
+        self.assertEqual(expected, ids)

tripleo_common/tests/utils/test_nodes.py

@@ -0,0 +1,475 @@
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import collections
+
+import mock
+from testtools import matchers
+
+from tripleo_common import exception
+from tripleo_common.tests import base
+from tripleo_common.utils import nodes
+
+
+class DriverInfoTest(base.TestCase):
+    def setUp(self):
+        super(DriverInfoTest, self).setUp()
+        self.driver_info = nodes.DriverInfo(
+            'foo',
+            mapping={
+                'pm_1': 'foo_1',
+                'pm_2': 'foo_2'
+            },
+            deprecated_mapping={
+                'pm_3': 'foo_3'
+            })
+
+    def test_convert_key(self):
+        self.assertEqual('foo_1', self.driver_info.convert_key('pm_1'))
+        self.assertEqual('foo_42', self.driver_info.convert_key('foo_42'))
+        self.assertIsNone(self.driver_info.convert_key('bar_baz'))
+
+    @mock.patch.object(nodes.LOG, 'warning', autospec=True)
+    def test_convert_key_deprecated(self, mock_log):
+        self.assertEqual('foo_3', self.driver_info.convert_key('pm_3'))
+        self.assertTrue(mock_log.called)
+
+    @mock.patch.object(nodes.LOG, 'warning', autospec=True)
+    def test_convert_key_pm_unsupported(self, mock_log):
+        self.assertIsNone(self.driver_info.convert_key('pm_42'))
+        self.assertTrue(mock_log.called)
+
+    def test_convert(self):
+        result = self.driver_info.convert({'pm_1': 'val1',
+                                           'foo_42': 42,
+                                           'unknown': 'foo'})
+        self.assertEqual({'foo_1': 'val1', 'foo_42': 42}, result)
+
+
+class PrefixedDriverInfoTest(base.TestCase):
+    def setUp(self):
+        super(PrefixedDriverInfoTest, self).setUp()
+        self.driver_info = nodes.PrefixedDriverInfo(
+            'foo', deprecated_mapping={'pm_d': 'foo_d'})
+
+    def test_convert_key(self):
+        keys = {'pm_addr': 'foo_address',
+                'pm_user': 'foo_username',
+                'pm_password': 'foo_password',
+                'foo_something': 'foo_something',
+                'pm_d': 'foo_d'}
+        for key, expected in keys.items():
+            self.assertEqual(expected, self.driver_info.convert_key(key))
+
+        for key in ('unknown', 'pm_port'):
+            self.assertIsNone(self.driver_info.convert_key(key))
+
+    def test_unique_id_from_fields(self):
+        fields = {'pm_addr': 'localhost',
+                  'pm_user': 'user',
+                  'pm_password': '123456',
+                  'pm_port': 42}
+        self.assertEqual('localhost',
+                         self.driver_info.unique_id_from_fields(fields))
+
+    def test_unique_id_from_node(self):
+        node = mock.Mock(driver_info={'foo_address': 'localhost',
+                                      'foo_port': 42})
+        self.assertEqual('localhost',
+                         self.driver_info.unique_id_from_node(node))
+
+
+class PrefixedDriverInfoTestWithPort(base.TestCase):
+    def setUp(self):
+        super(PrefixedDriverInfoTestWithPort, self).setUp()
+        self.driver_info = nodes.PrefixedDriverInfo(
+            'foo', deprecated_mapping={'pm_d': 'foo_d'},
+            has_port=True)
+
+    def test_convert_key_with_port(self):
+        keys = {'pm_addr': 'foo_address',
+                'pm_user': 'foo_username',
+                'pm_password': 'foo_password',
+                'foo_something': 'foo_something',
+                'pm_d': 'foo_d',
+                'pm_port': 'foo_port'}
+        for key, expected in keys.items():
+            self.assertEqual(expected, self.driver_info.convert_key(key))
+
+        self.assertIsNone(self.driver_info.convert_key('unknown'))
+
+    def test_unique_id_from_fields(self):
+        fields = {'pm_addr': 'localhost',
+                  'pm_user': 'user',
+                  'pm_password': '123456',
+                  'pm_port': 42}
+        self.assertEqual('localhost:42',
+                         self.driver_info.unique_id_from_fields(fields))
+
+    def test_unique_id_from_node(self):
+        node = mock.Mock(driver_info={'foo_address': 'localhost',
+                                      'foo_port': 42})
+        self.assertEqual('localhost:42',
+                         self.driver_info.unique_id_from_node(node))
+
+
+class iBootDriverInfoTest(base.TestCase):
+    def setUp(self):
+        super(iBootDriverInfoTest, self).setUp()
+        self.driver_info = nodes.iBootDriverInfo()
+
+    def test_unique_id_from_fields(self):
+        fields = {'pm_addr': 'localhost',
+                  'pm_user': 'user',
+                  'pm_password': '123456',
+                  'pm_port': 42,
+                  'iboot_relay_id': 'r1'}
+        self.assertEqual('localhost:42#r1',
+                         self.driver_info.unique_id_from_fields(fields))
+
+    def test_unique_id_from_fields_no_relay(self):
+        fields = {'pm_addr': 'localhost',
+                  'pm_user': 'user',
+                  'pm_password': '123456',
+                  'pm_port': 42}
+        self.assertEqual('localhost:42',
+                         self.driver_info.unique_id_from_fields(fields))
+
+    def test_unique_id_from_node(self):
+        node = mock.Mock(driver_info={'iboot_address': 'localhost',
+                                      'iboot_port': 42,
+                                      'iboot_relay_id': 'r1'})
+        self.assertEqual('localhost:42#r1',
+                         self.driver_info.unique_id_from_node(node))
+
+    def test_unique_id_from_node_no_relay(self):
+        node = mock.Mock(driver_info={'iboot_address': 'localhost',
+                                      'iboot_port': 42})
+        self.assertEqual('localhost:42',
+                         self.driver_info.unique_id_from_node(node))
+
+
+class FindNodeHandlerTest(base.TestCase):
+    def test_found(self):
+        test = [('fake', 'fake'),
+                ('fake_pxe', 'fake'),
+                ('pxe_ssh', 'ssh'),
+                ('pxe_ipmitool', 'ipmi'),
+                ('pxe_ilo', 'ilo'),
+                ('agent_irmc', 'irmc')]
+        for driver, prefix in test:
+            handler = nodes._find_node_handler({'pm_type': driver})
+            self.assertEqual(prefix, handler._prefix)
+
+    def test_no_driver(self):
+        self.assertRaises(exception.InvalidNode,
+                          nodes._find_node_handler, {})
+
+    def test_unknown_driver(self):
+        self.assertRaises(exception.InvalidNode,
+                          nodes._find_node_handler, {'pm_type': 'foobar'})
+
+
+class NodesTest(base.TestCase):
+
+    def _get_node(self):
+        return {'cpu': '1', 'memory': '2048', 'disk': '30', 'arch': 'amd64',
+                'mac': ['aaa'], 'pm_addr': 'foo.bar', 'pm_user': 'test',
+                'pm_password': 'random', 'pm_type': 'pxe_ssh', 'name': 'node1',
+                'capabilities': 'num_nics:6'}
+
+    def test_register_all_nodes_ironic_no_hw_stats(self):
+        node_list = [self._get_node()]
+
+        # Remove the hardware stats from the node dictionary
+        node_list[0].pop("cpu")
+        node_list[0].pop("memory")
+        node_list[0].pop("disk")
+        node_list[0].pop("arch")
+
+        # Node properties should be created with empty string values for the
+        # hardware statistics
+        node_properties = {"capabilities": "num_nics:6"}
+
+        ironic = mock.MagicMock()
+        nodes.register_all_nodes('servicehost', node_list, client=ironic)
+        pxe_node_driver_info = {"ssh_address": "foo.bar",
+                                "ssh_username": "test",
+                                "ssh_key_contents": "random",
+                                "ssh_virt_type": "virsh"}
+        pxe_node = mock.call(driver="pxe_ssh",
+                             name='node1',
+                             driver_info=pxe_node_driver_info,
+                             properties=node_properties)
+        port_call = mock.call(node_uuid=ironic.node.create.return_value.uuid,
+                              address='aaa')
+        power_off_call = mock.call(ironic.node.create.return_value.uuid, 'off')
+        ironic.node.create.assert_has_calls([pxe_node, mock.ANY])
+        ironic.port.create.assert_has_calls([port_call])
+        ironic.node.set_power_state.assert_has_calls([power_off_call])
+
+    def test_register_all_nodes(self):
+        node_list = [self._get_node()]
+        node_properties = {"cpus": "1",
+                           "memory_mb": "2048",
+                           "local_gb": "30",
+                           "cpu_arch": "amd64",
+                           "capabilities": "num_nics:6"}
+        ironic = mock.MagicMock()
+        nodes.register_all_nodes('servicehost', node_list, client=ironic)
+        pxe_node_driver_info = {"ssh_address": "foo.bar",
+                                "ssh_username": "test",
+                                "ssh_key_contents": "random",
+                                "ssh_virt_type": "virsh"}
+        pxe_node = mock.call(driver="pxe_ssh",
+                             name='node1',
+                             driver_info=pxe_node_driver_info,
+                             properties=node_properties)
+        port_call = mock.call(node_uuid=ironic.node.create.return_value.uuid,
+                              address='aaa')
+        power_off_call = mock.call(ironic.node.create.return_value.uuid, 'off')
+        ironic.node.create.assert_has_calls([pxe_node, mock.ANY])
+        ironic.port.create.assert_has_calls([port_call])
+        ironic.node.set_power_state.assert_has_calls([power_off_call])
+
+    def test_register_all_nodes_kernel_ramdisk(self):
+        node_list = [self._get_node()]
+        node_properties = {"cpus": "1",
+                           "memory_mb": "2048",
+                           "local_gb": "30",
+                           "cpu_arch": "amd64",
+                           "capabilities": "num_nics:6"}
+        ironic = mock.MagicMock()
+        glance = mock.MagicMock()
+        image = collections.namedtuple('image', ['id'])
+        glance.images.find.side_effect = (image('kernel-123'),
+                                          image('ramdisk-999'))
+        nodes.register_all_nodes('servicehost', node_list, client=ironic,
+                                 glance_client=glance, kernel_name='bm-kernel',
+                                 ramdisk_name='bm-ramdisk')
+        pxe_node_driver_info = {"ssh_address": "foo.bar",
+                                "ssh_username": "test",
+                                "ssh_key_contents": "random",
+                                "ssh_virt_type": "virsh",
+                                "deploy_kernel": "kernel-123",
+                                "deploy_ramdisk": "ramdisk-999"}
+        pxe_node = mock.call(driver="pxe_ssh",
+                             name='node1',
+                             driver_info=pxe_node_driver_info,
+                             properties=node_properties)
+        port_call = mock.call(node_uuid=ironic.node.create.return_value.uuid,
+                              address='aaa')
+        power_off_call = mock.call(ironic.node.create.return_value.uuid, 'off')
+        ironic.node.create.assert_has_calls([pxe_node, mock.ANY])
+        ironic.port.create.assert_has_calls([port_call])
+        ironic.node.set_power_state.assert_has_calls([power_off_call])
+
+    def test_register_update(self):
+        node = self._get_node()
+        ironic = mock.MagicMock()
+        node_map = {'mac': {'aaa': 1}}
+
+        def side_effect(*args, **kwargs):
+            update_patch = [
+                {'path': '/name', 'value': 'node1'},
+                {'path': '/driver_info/ssh_key_contents', 'value': 'random'},
+                {'path': '/driver_info/ssh_address', 'value': 'foo.bar'},
+                {'path': '/properties/memory_mb', 'value': '2048'},
+                {'path': '/properties/local_gb', 'value': '30'},
+                {'path': '/properties/cpu_arch', 'value': 'amd64'},
+                {'path': '/properties/cpus', 'value': '1'},
+                {'path': '/properties/capabilities', 'value': 'num_nics:6'},
+                {'path': '/driver_info/ssh_username', 'value': 'test'},
+                {'path': '/driver_info/ssh_virt_type', 'value': 'virsh'}]
+            for key in update_patch:
+                key['op'] = 'add'
+            self.assertThat(update_patch,
+                            matchers.MatchesSetwise(*(map(matchers.Equals,
+                                                          args[1]))))
+            return mock.Mock(uuid='uuid1')
+
+        ironic.node.update.side_effect = side_effect
+        nodes._update_or_register_ironic_node(None, node, node_map,
+                                              client=ironic)
+        ironic.node.update.assert_called_once_with(1, mock.ANY)
+
+    def _update_by_type(self, pm_type):
+        ironic = mock.MagicMock()
+        node_map = {'mac': {}, 'pm_addr': {}}
+        node = self._get_node()
+        node['pm_type'] = pm_type
+        node_map['pm_addr']['foo.bar'] = ironic.node.get.return_value.uuid
+        nodes._update_or_register_ironic_node('servicehost', node,
+                                              node_map, client=ironic)
+        ironic.node.update.assert_called_once_with(
+            ironic.node.get.return_value.uuid, mock.ANY)
+
+    def test_update_node_ironic_pxe_ipmitool(self):
+        self._update_by_type('pxe_ipmitool')
+
+    def test_update_node_ironic_pxe_drac(self):
+        self._update_by_type('pxe_drac')
+
+    def test_update_node_ironic_pxe_ilo(self):
+        self._update_by_type('pxe_ilo')
+
+    def test_update_node_ironic_pxe_irmc(self):
+        self._update_by_type('pxe_irmc')
+
+    def test_register_node_update(self):
+        node = self._get_node()
+        node['mac'][0] = node['mac'][0].upper()
+        ironic = mock.MagicMock()
+        node_map = {'mac': {'aaa': 1}}
+
+        def side_effect(*args, **kwargs):
+            update_patch = [
+                {'path': '/name', 'value': 'node1'},
+                {'path': '/driver_info/ssh_key_contents', 'value': 'random'},
+                {'path': '/driver_info/ssh_address', 'value': 'foo.bar'},
+                {'path': '/properties/memory_mb', 'value': '2048'},
+                {'path': '/properties/local_gb', 'value': '30'},
+                {'path': '/properties/cpu_arch', 'value': 'amd64'},
+                {'path': '/properties/cpus', 'value': '1'},
+                {'path': '/properties/capabilities', 'value': 'num_nics:6'},
+                {'path': '/driver_info/ssh_username', 'value': 'test'},
+                {'path': '/driver_info/ssh_virt_type', 'value': 'virsh'}]
+            for key in update_patch:
+                key['op'] = 'add'
+            self.assertThat(update_patch,
+                            matchers.MatchesSetwise(*(map(matchers.Equals,
+                                                          args[1]))))
+            return mock.Mock(uuid='uuid1')
+
+        ironic.node.update.side_effect = side_effect
+        nodes._update_or_register_ironic_node(None, node, node_map,
+                                              client=ironic)
+        ironic.node.update.assert_called_once_with(1, mock.ANY)
+
+    def test_register_ironic_node_int_values(self):
+        node_properties = {"cpus": "1",
+                           "memory_mb": "2048",
+                           "local_gb": "30",
+                           "cpu_arch": "amd64",
+                           "capabilities": "num_nics:6"}
+        node = self._get_node()
+        node['cpu'] = 1
+        node['memory'] = 2048
+        node['disk'] = 30
+        client = mock.MagicMock()
+        nodes.register_ironic_node('service_host', node, client=client)
+        client.node.create.assert_called_once_with(driver=mock.ANY,
+                                                   name='node1',
+                                                   properties=node_properties,
+                                                   driver_info=mock.ANY)
+
+    def test_register_ironic_node_fake_pxe(self):
+        node_properties = {"cpus": "1",
+                           "memory_mb": "2048",
+                           "local_gb": "30",
+                           "cpu_arch": "amd64",
+                           "capabilities": "num_nics:6"}
+        node = self._get_node()
+        for v in ('pm_addr', 'pm_user', 'pm_password'):
+            del node[v]
+        node['pm_type'] = 'fake_pxe'
+        client = mock.MagicMock()
+        nodes.register_ironic_node('service_host', node, client=client)
+        client.node.create.assert_called_once_with(driver='fake_pxe',
+                                                   name='node1',
+                                                   properties=node_properties,
+                                                   driver_info={})
+
+    def test_register_ironic_node_update_int_values(self):
+        node = self._get_node()
+        ironic = mock.MagicMock()
+        node['cpu'] = 1
+        node['memory'] = 2048
+        node['disk'] = 30
+        node_map = {'mac': {'aaa': 1}}
+
+        def side_effect(*args, **kwargs):
+            update_patch = [
+                {'path': '/name', 'value': 'node1'},
+                {'path': '/driver_info/ssh_key_contents', 'value': 'random'},
+                {'path': '/driver_info/ssh_address', 'value': 'foo.bar'},
+                {'path': '/properties/memory_mb', 'value': '2048'},
+                {'path': '/properties/local_gb', 'value': '30'},
+                {'path': '/properties/cpu_arch', 'value': 'amd64'},
+                {'path': '/properties/cpus', 'value': '1'},
+                {'path': '/properties/capabilities', 'value': 'num_nics:6'},
+                {'path': '/driver_info/ssh_username', 'value': 'test'},
+                {'path': '/driver_info/ssh_virt_type', 'value': 'virsh'}]
+            for key in update_patch:
+                key['op'] = 'add'
+            self.assertThat(update_patch,
+                            matchers.MatchesSetwise(*(map(matchers.Equals,
+                                                      args[1]))))
+            return mock.Mock(uuid='uuid1')
+
+        ironic.node.update.side_effect = side_effect
+        nodes._update_or_register_ironic_node(None, node, node_map,
+                                              client=ironic)
+
+    def test_clean_up_extra_nodes_ironic(self):
+        node = collections.namedtuple('node', ['uuid'])
+        client = mock.MagicMock()
+        client.node.list.return_value = [node('foobar')]
+        nodes._clean_up_extra_nodes(set(('abcd',)), client, remove=True)
+        client.node.delete.assert_called_once_with('foobar')
+
+    def test__get_node_id_fake_pxe(self):
+        node = self._get_node()
+        node['pm_type'] = 'fake_pxe'
+        handler = nodes._find_driver_handler('fake_pxe')
+        node_map = {'mac': {'aaa': 'abcdef'}, 'pm_addr': {}}
+        self.assertEqual('abcdef', nodes._get_node_id(node, handler, node_map))
+
+    def test__get_node_id_conflict(self):
+        node = self._get_node()
+        handler = nodes._find_driver_handler('pxe_ipmitool')
+        node_map = {'mac': {'aaa': 'abcdef'},
+                    'pm_addr': {'foo.bar': 'defabc'}}
+        self.assertRaises(exception.InvalidNode,
+                          nodes._get_node_id,
+                          node, handler, node_map)
+
+
+class TestPopulateNodeMapping(base.TestCase):
+    def test_populate_node_mapping_ironic(self):
+        client = mock.MagicMock()
+        ironic_node = collections.namedtuple('node', ['uuid', 'driver',
+                                             'driver_info'])
+        ironic_port = collections.namedtuple('port', ['address'])
+        node1 = ironic_node('abcdef', 'pxe_ssh', None)
+        node2 = ironic_node('fedcba', 'pxe_ipmitool',
+                            {'ipmi_address': '10.0.1.2'})
+        client.node.list_ports.side_effect = ([ironic_port('aaa')],
+                                              [])
+        client.node.list.return_value = [node1, node2]
+        expected = {'mac': {'aaa': 'abcdef'},
+                    'pm_addr': {'10.0.1.2': 'fedcba'}}
+        self.assertEqual(expected, nodes._populate_node_mapping(client))
+
+    def test_populate_node_mapping_ironic_fake_pxe(self):
+        client = mock.MagicMock()
+        ironic_node = collections.namedtuple('node', ['uuid', 'driver',
+                                             'driver_info'])
+        ironic_port = collections.namedtuple('port', ['address'])
+        node = ironic_node('abcdef', 'fake_pxe', None)
+        client.node.list_ports.return_value = [ironic_port('aaa')]
+        client.node.list.return_value = [node]
+        expected = {'mac': {'aaa': 'abcdef'}, 'pm_addr': {}}
+        self.assertEqual(expected, nodes._populate_node_mapping(client))

tripleo_common/utils/glance.py

@@ -0,0 +1,65 @@
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import collections
+import logging
+
+from glanceclient import exc as exceptions
+
+LOG = logging.getLogger(__name__)
+
+
+def create_or_find_kernel_and_ramdisk(glanceclient, kernel_name, ramdisk_name,
+                                      kernel_path=None, ramdisk_path=None,
+                                      skip_missing=False):
+    """Find or create a given kernel and ramdisk in Glance.
+
+    If either kernel_path or ramdisk_path is None, they will not be created,
+    and an exception will be raised if it does not exist in Glance.
+
+    :param glanceclient: A client for Glance.
+    :param kernel_name: Name to search for or create for the kernel.
+    :param ramdisk_name: Name to search for or create for the ramdisk.
+    :param kernel_path: Path to the kernel on disk.
+    :param ramdisk_path: Path to the ramdisk on disk.
+    :param skip_missing: If `True', do not raise an exception if either the
+                         kernel or ramdisk image is not found.
+
+    :returns: A dictionary mapping kernel or ramdisk to the ID in Glance.
+    """
+    kernel_image = _upload_file(glanceclient, kernel_name, kernel_path,
+                                'aki', 'Kernel', skip_missing=skip_missing)
+    ramdisk_image = _upload_file(glanceclient, ramdisk_name, ramdisk_path,
+                                 'ari', 'Ramdisk', skip_missing=skip_missing)
+    return {'kernel': kernel_image.id, 'ramdisk': ramdisk_image.id}
+
+
+def _upload_file(glanceclient, name, path, disk_format, type_name,
+                 skip_missing=False):
+    image_tuple = collections.namedtuple('image', ['id'])
+    try:
+        image = glanceclient.images.find(name=name, disk_format=disk_format)
+    except exceptions.NotFound:
+        if path:
+            image = glanceclient.images.create(
+                name=name, disk_format=disk_format, is_public=True,
+                data=open(path, 'rb'))
+        else:
+            if skip_missing:
+                image = image_tuple(None)
+            else:
+                raise ValueError("%s image not found in Glance, and no path "
+                                 "specified." % type_name)
+    return image

tripleo_common/utils/nodes.py

@@ -0,0 +1,374 @@
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import re
+
+from ironicclient import exc as ironicexp
+import six
+
+from tripleo_common import exception
+from tripleo_common.utils import glance
+
+LOG = logging.getLogger(__name__)
+
+
+class DriverInfo(object):
+    """Class encapsulating field conversion logic."""
+    DEFAULTS = {}
+
+    def __init__(self, prefix, mapping, deprecated_mapping=None):
+        self._prefix = prefix
+        self._mapping = mapping
+        self._deprecated_mapping = deprecated_mapping or {}
+
+    def convert_key(self, key):
+        if key in self._mapping:
+            return self._mapping[key]
+        elif key in self._deprecated_mapping:
+            real = self._deprecated_mapping[key]
+            LOG.warning('Key %s is deprecated, please use %s',
+                        key, real)
+            return real
+        elif key.startswith(self._prefix):
+            return key
+        elif key != 'pm_type' and key.startswith('pm_'):
+            LOG.warning('Key %s is not supported and will not be passed',
+                        key)
+        else:
+            LOG.debug('Skipping key %s not starting with prefix %s',
+                      key, self._prefix)
+
+    def convert(self, fields):
+        """Convert fields from instackenv.json format to ironic names."""
+        result = self.DEFAULTS.copy()
+        for key, value in fields.items():
+            new_key = self.convert_key(key)
+            if new_key is not None:
+                result[new_key] = value
+        return result
+
+    def unique_id_from_fields(self, fields):
+        """Return a string uniquely identifying a node in instackenv."""
+
+    def unique_id_from_node(self, node):
+        """Return a string uniquely identifying a node in ironic db."""
+
+
+class PrefixedDriverInfo(DriverInfo):
+    def __init__(self, prefix, deprecated_mapping=None,
+                 has_port=False, address_field='address'):
+        mapping = {
+            'pm_addr': '%s_%s' % (prefix, address_field),
+            'pm_user': '%s_username' % prefix,
+            'pm_password': '%s_password' % prefix,
+        }
+        if has_port:
+            mapping['pm_port'] = '%s_port' % prefix
+        self._has_port = has_port
+
+        super(PrefixedDriverInfo, self).__init__(
+            prefix, mapping,
+            deprecated_mapping=deprecated_mapping
+        )
+
+    def unique_id_from_fields(self, fields):
+        result = fields['pm_addr']
+        if self._has_port and 'pm_port' in fields:
+            result = '%s:%s' % (result, fields['pm_port'])
+        return result
+
+    def unique_id_from_node(self, node):
+        new_key = self.convert_key('pm_addr')
+        assert new_key is not None
+        try:
+            result = node.driver_info[new_key]
+        except KeyError:
+            # Node cannot be identified
+            return
+
+        if self._has_port:
+            new_port = self.convert_key('pm_port')
+            assert new_port
+            try:
+                return '%s:%s' % (result, node.driver_info[new_port])
+            except KeyError:
+                pass
+
+        return result
+
+
+class SshDriverInfo(DriverInfo):
+    DEFAULTS = {'ssh_virt_type': 'virsh'}
+
+    def __init__(self):
+        super(SshDriverInfo, self).__init__(
+            'ssh',
+            {
+                'pm_addr': 'ssh_address',
+                'pm_user': 'ssh_username',
+                # TODO(dtantsur): support ssh_key_filename as well
+                'pm_password': 'ssh_key_contents',
+            },
+            deprecated_mapping={
+                'pm_virt_type': 'ssh_virt_type',
+            }
+        )
+
+
+class iBootDriverInfo(PrefixedDriverInfo):
+    def __init__(self):
+        super(iBootDriverInfo, self).__init__(
+            'iboot', has_port=True,
+            deprecated_mapping={
+                'pm_relay_id': 'iboot_relay_id',
+            }
+        )
+
+    def unique_id_from_fields(self, fields):
+        result = super(iBootDriverInfo, self).unique_id_from_fields(fields)
+        if 'iboot_relay_id' in fields:
+            result = '%s#%s' % (result, fields['iboot_relay_id'])
+        return result
+
+    def unique_id_from_node(self, node):
+        try:
+            result = super(iBootDriverInfo, self).unique_id_from_node(node)
+        except IndexError:
+            return
+
+        if node.driver_info.get('iboot_relay_id'):
+            result = '%s#%s' % (result, node.driver_info['iboot_relay_id'])
+
+        return result
+
+
+DRIVER_INFO = {
+    # production drivers
+    '.*_ipmi(tool|native)': PrefixedDriverInfo('ipmi'),
+    '.*_drac': PrefixedDriverInfo('drac', address_field='host'),
+    '.*_ilo': PrefixedDriverInfo('ilo'),
+    '.*_ucs': PrefixedDriverInfo(
+        'ucs',
+        address_field='hostname',
+        deprecated_mapping={
+            'pm_service_profile': 'ucs_service_profile'
+        }),
+    '.*_irmc': PrefixedDriverInfo(
+        'irmc', has_port=True,
+        deprecated_mapping={
+            'pm_auth_method': 'irmc_auth_method',
+            'pm_client_timeout': 'irmc_client_timeout',
+            'pm_sensor_method': 'irmc_sensor_method',
+            'pm_deploy_iso': 'irmc_deploy_iso',
+        }),
+    # test drivers
+    '.*_ssh': SshDriverInfo(),
+    '.*_iboot': iBootDriverInfo(),
+    '.*_wol': DriverInfo(
+        'wol',
+        mapping={
+            'pm_addr': 'wol_host',
+            'pm_port': 'wol_port',
+        }),
+    '.*_amt': PrefixedDriverInfo('amt'),
+    'fake(|_pxe|_agent)': DriverInfo('fake', mapping={}),
+}
+
+
+def _find_driver_handler(driver):
+    for driver_tpl, handler in DRIVER_INFO.items():
+        if re.match(driver_tpl, driver) is not None:
+            return handler
+
+    # FIXME(dtantsur): handle all drivers without hardcoding them
+    raise exception.InvalidNode('unknown pm_type (ironic driver to use): '
+                                '%s' % driver)
+
+
+def _find_node_handler(fields):
+    try:
+        driver = fields['pm_type']
+    except KeyError:
+        raise exception.InvalidNode('pm_type (ironic driver to use) is '
+                                    'required', node=fields)
+    return _find_driver_handler(driver)
+
+
+def register_ironic_node(service_host, node, client=None, blocking=None):
+    if blocking is not None:
+        LOG.warning('blocking argument to register_ironic_node is deprecated '
+                    'and does nothing')
+
+    driver_info = {}
+    handler = _find_node_handler(node)
+
+    if "kernel_id" in node:
+        driver_info["deploy_kernel"] = node["kernel_id"]
+    if "ramdisk_id" in node:
+        driver_info["deploy_ramdisk"] = node["ramdisk_id"]
+
+    driver_info.update(handler.convert(node))
+
+    mapping = {'cpus': 'cpu',
+               'memory_mb': 'memory',
+               'local_gb': 'disk',
+               'cpu_arch': 'arch'}
+    properties = {k: six.text_type(node.get(v))
+                  for k, v in mapping.items()
+                  if node.get(v) is not None}
+
+    if 'capabilities' in node:
+        properties.update({"capabilities":
+                          six.text_type(node.get('capabilities'))})
+
+    create_map = {"driver": node["pm_type"],
+                  "properties": properties,
+                  "driver_info": driver_info}
+
+    if 'name' in node:
+        create_map.update({"name": six.text_type(node.get('name'))})
+
+    node_id = handler.unique_id_from_fields(node)
+    LOG.debug('Registering node %s with ironic.', node_id)
+    ironic_node = client.node.create(**create_map)
+
+    for mac in node.get("mac", []):
+        client.port.create(address=mac, node_uuid=ironic_node.uuid)
+
+    validation = client.node.validate(ironic_node.uuid)
+    if not validation.power['result']:
+        LOG.warning('Node %s did not pass power credentials validation: %s',
+                    ironic_node.uuid, validation.power['reason'])
+
+    try:
+        client.node.set_power_state(ironic_node.uuid, 'off')
+    except ironicexp.Conflict:
+        # Conflict means the Ironic conductor does something with a node,
+        # ignore the exception.
+        pass
+
+    return ironic_node
+
+
+def _populate_node_mapping(client):
+    LOG.debug('Populating list of registered nodes.')
+    node_map = {'mac': {}, 'pm_addr': {}}
+    nodes = client.node.list(detail=True)
+    for node in nodes:
+        for port in client.node.list_ports(node.uuid):
+            node_map['mac'][port.address] = node.uuid
+
+        handler = _find_driver_handler(node.driver)
+        unique_id = handler.unique_id_from_node(node)
+        if unique_id:
+            node_map['pm_addr'][unique_id] = node.uuid
+
+    return node_map
+
+
+def _get_node_id(node, handler, node_map):
+    candidates = []
+    for mac in node.get('mac', []):
+        try:
+            candidates.append(node_map['mac'][mac.lower()])
+        except KeyError:
+            pass
+
+    unique_id = handler.unique_id_from_fields(node)
+    if unique_id:
+        try:
+            candidates.append(node_map['pm_addr'][unique_id])
+        except KeyError:
+            pass
+
+    if len(candidates) > 1:
+        raise exception.InvalidNode('Several candidates found for the same '
+                                    'node data: %s' % candidates,
+                                    node=node)
+    elif candidates:
+        return candidates[0]
+
+
+def _update_or_register_ironic_node(service_host, node, node_map, client=None):
+    handler = _find_node_handler(node)
+    node_uuid = _get_node_id(node, handler, node_map)
+
+    if node_uuid:
+        LOG.info('Node %s already registered, updating details.',
+                 node_uuid)
+
+        patched = {}
+        for field, path in [('cpu', '/properties/cpus'),
+                            ('memory', '/properties/memory_mb'),
+                            ('disk', '/properties/local_gb'),
+                            ('arch', '/properties/cpu_arch'),
+                            ('name', '/name'),
+                            ('capabilities', '/properties/capabilities')]:
+            if field in node:
+                patched[path] = node.pop(field)
+
+        driver_info = handler.convert(node)
+        for key, value in driver_info.items():
+            patched['/driver_info/%s' % key] = value
+
+        node_patch = []
+        for key, value in patched.items():
+            node_patch.append({'path': key,
+                               'value': six.text_type(value),
+                               'op': 'add'})
+        ironic_node = client.node.update(node_uuid, node_patch)
+    else:
+        ironic_node = register_ironic_node(service_host, node, client)
+
+    return ironic_node.uuid
+
+
+def _clean_up_extra_nodes(seen, client, remove=False):
+    all_nodes = set([n.uuid for n in client.node.list()])
+    remove_func = client.node.delete
+    extra_nodes = all_nodes - seen
+    for node in extra_nodes:
+        if remove:
+            LOG.debug('Removing extra registered node %s.' % node)
+            remove_func(node)
+        else:
+            LOG.debug('Extra registered node %s found.' % node)
+
+
+def register_all_nodes(service_host, nodes_list, client=None, remove=False,
+                       blocking=True, keystone_client=None, glance_client=None,
+                       kernel_name=None, ramdisk_name=None):
+    LOG.debug('Registering all nodes.')
+    node_map = _populate_node_mapping(client)
+
+    glance_ids = {'kernel': None, 'ramdisk': None}
+    if kernel_name and ramdisk_name:
+        glance_ids = glance.create_or_find_kernel_and_ramdisk(
+            glance_client, kernel_name, ramdisk_name)
+
+    seen = set()
+    for node in nodes_list:
+        if glance_ids['kernel'] and 'kernel_id' not in node:
+            node['kernel_id'] = glance_ids['kernel']
+        if glance_ids['ramdisk'] and 'ramdisk_id' not in node:
+            node['ramdisk_id'] = glance_ids['ramdisk']
+
+        uuid = _update_or_register_ironic_node(service_host,
+                                               node, node_map,
+                                               client=client)
+        seen.add(uuid)
+
+    _clean_up_extra_nodes(seen, client, remove=remove)