openstack
/
fuel-web
Code Issues Proposed changes

Use appropriate types to store IP and MAC addrs 

In PostgreSQL there are special types for storing IPv4, IPv6 address and
subnetworks and also MAC addresses. Storing data in these types is more
efficient than in strings because it requires less space. In addition,
it provides additional operations for columns of these types.

This patch changes types of some columns for models such as:

    IPAddr
        ip_addr: inet

    IPAddrRange
        first: inet
        last: inet

    Node
        mac: macaddr
        ip: inet

    NodeNICInterface
        mac: macaddr
        ip_addr: inet
        netmask: inet

    NodeNICInterface
        mac: macaddr

    NetworkGroup
        cidr: cidr
        gateway: inet

    NeutronConfig
        base_mac: macaddr
        internal_cidr: cidr
        internal_gateway: inet
        baremetal_gateway: inet

    NovaNetworkConfig
        fixed_networks_cidr: cidr

Co-Authored-By: Ilya Kharin <ikharin@mirantis.com>
Change-Id: Id8431eb7d2e3ca1a1cc29fc865d57359b29d0e35
Partial-Bug: #1484973
This commit is contained in:
smurashov 2015-10-16 14:23:21 +03:00 committed by Ilya Kharin
parent dd155ad347
commit 83bd490394
8 changed files with 93 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
nailgun/nailgun/db/migration/alembic_migrations/versions/fuel_8_0.py
View File

@ -108,9 +108,11 @@ def upgrade():
upgrade_with_components()
dashboard_entries_upgrade()
upgrade_master_settings()
upgrade_all_network_data_from_string_to_appropriate_data_type()
def downgrade():
downgrade_all_network_data_to_string()
downgrade_master_settings()
dashboard_entries_downgrade()
downgrade_with_components()
@ -528,3 +530,65 @@ def dashboard_entries_upgrade():
def dashboard_entries_downgrade():
op.drop_table('dashboard_entries')
def upgrade_all_network_data_from_string_to_appropriate_data_type():
convert_column_type('ip_addrs', 'ip_addr', 'inet')
convert_column_type('ip_addr_ranges', 'first', 'inet')
convert_column_type('ip_addr_ranges', 'last', 'inet')
convert_column_type('network_groups', 'cidr', 'cidr')
convert_column_type('network_groups', 'gateway', 'inet')
convert_column_type('neutron_config', 'base_mac', 'macaddr')
convert_column_type('neutron_config', 'internal_cidr', 'cidr')
convert_column_type('neutron_config', 'internal_gateway', 'inet')
convert_column_type('neutron_config', 'baremetal_gateway', 'inet')
convert_column_type('nova_network_config', 'fixed_networks_cidr',
'cidr')
convert_column_type('nodes', 'mac', 'macaddr')
convert_column_type('nodes', 'ip', 'inet')
convert_column_type('node_nic_interfaces', 'mac', 'macaddr')
convert_column_type('node_nic_interfaces', 'ip_addr', 'inet')
convert_column_type('node_nic_interfaces', 'netmask', 'inet')
convert_column_type('node_bond_interfaces', 'mac', 'macaddr')
def convert_column_type(table_name, column_name, psql_type):
op.execute('ALTER TABLE {0} ALTER COLUMN {1}'
' TYPE {2} USING cast({1} as {2})'.format(table_name,
column_name,
psql_type))
def downgrade_all_network_data_to_string():
ip_type_to_string('ip_addrs', 'ip_addr', 25)
ip_type_to_string('ip_addr_ranges', 'first', 25)
ip_type_to_string('ip_addr_ranges', 'last', 25)
op.alter_column('network_groups', 'cidr', type_=sa.String(length=25))
ip_type_to_string('network_groups', 'gateway', 25)
op.alter_column('neutron_config', 'base_mac',
type_=fields.LowercaseString(17))
op.alter_column('neutron_config', 'internal_cidr',
type_=sa.String(length=25))
ip_type_to_string('neutron_config', 'internal_gateway', 25)
ip_type_to_string('neutron_config', 'baremetal_gateway', 25)
op.alter_column('nova_network_config', 'fixed_networks_cidr',
type_=sa.String(length=25))
op.alter_column('nodes', 'mac', type_=fields.LowercaseString(17))
ip_type_to_string('nodes', 'ip', 15)
op.alter_column('node_nic_interfaces', 'mac',
type_=fields.LowercaseString(17))
ip_type_to_string('node_nic_interfaces', 'ip_addr', 25)
ip_type_to_string('node_nic_interfaces', 'netmask', 25)
op.alter_column('node_bond_interfaces', 'mac', type_=sa.String(length=50))
def ip_type_to_string(table_name, column_name, string_len):
# NOTE(akscram): The potential data loss is possible for IPv6 addresses
# in case of string_len < 39.
op.execute(
'ALTER TABLE {0} ALTER COLUMN {1} '
'TYPE varchar({2}) USING '
'split_part(cast({1} as varchar(43)), \'/\', 1)'.format(table_name,
column_name,
string_len)
)

11
nailgun/nailgun/db/sqlalchemy/models/network.py
View File

@ -15,6 +15,7 @@
# under the License.
from sqlalchemy import Column
from sqlalchemy.dialects import postgresql as psql
from sqlalchemy.ext.mutable import MutableDict
from sqlalchemy import ForeignKey
from sqlalchemy import Integer
@ -31,7 +32,7 @@ class IPAddr(Base):
network = Column(Integer, ForeignKey('network_groups.id',
ondelete="CASCADE"))
node = Column(Integer, ForeignKey('nodes.id', ondelete="CASCADE"))
ip_addr = Column(String(25), nullable=False)
ip_addr = Column(psql.INET, nullable=False)
vip_type = Column(String(25), nullable=True)
network_data = relationship("NetworkGroup")
@ -43,8 +44,8 @@ class IPAddrRange(Base):
id = Column(Integer, primary_key=True)
network_group_id = Column(Integer, ForeignKey('network_groups.id',
ondelete="CASCADE"))
first = Column(String(25), nullable=False)
last = Column(String(25), nullable=False)
first = Column(psql.INET, nullable=False)
last = Column(psql.INET, nullable=False)
class NetworkGroup(Base):
@ -57,8 +58,8 @@ class NetworkGroup(Base):
# can be nullable only for fuelweb admin net
group_id = Column(Integer, ForeignKey('nodegroups.id'), nullable=True)
vlan_start = Column(Integer)
cidr = Column(String(25))
gateway = Column(String(25))
cidr = Column(psql.CIDR)
gateway = Column(psql.INET)
ip_ranges = relationship(
"IPAddrRange",
backref="network_group",

12
nailgun/nailgun/db/sqlalchemy/models/network_config.py
View File

@ -15,6 +15,7 @@
# under the License.
from sqlalchemy import Column
from sqlalchemy.dialects import postgresql as psql
from sqlalchemy import Enum
from sqlalchemy import ForeignKey
from sqlalchemy import Integer
@ -23,7 +24,6 @@ from sqlalchemy import String
from nailgun import consts
from nailgun.db.sqlalchemy.models.base import Base
from nailgun.db.sqlalchemy.models.fields import JSON
from nailgun.db.sqlalchemy.models.fields import LowercaseString
class NetworkingConfig(Base):
@ -58,10 +58,10 @@ class NeutronConfig(NetworkingConfig):
vlan_range = Column(JSON, default=[])
gre_id_range = Column(JSON, default=[])
base_mac = Column(LowercaseString(17), nullable=False)
internal_cidr = Column(String(25))
internal_gateway = Column(String(25))
baremetal_gateway = Column(String(25))
base_mac = Column(psql.MACADDR, nullable=False)
internal_cidr = Column(psql.CIDR)
internal_gateway = Column(psql.INET)
baremetal_gateway = Column(psql.INET)
baremetal_range = Column(JSON, default=[])
# Neutron L3 names for default internal / floating networks
@ -90,7 +90,7 @@ class NovaNetworkConfig(NetworkingConfig):
id = Column(Integer, ForeignKey('networking_configs.id'), primary_key=True)
fixed_networks_cidr = Column(String(25))
fixed_networks_cidr = Column(psql.CIDR)
fixed_networks_vlan_start = Column(Integer)
fixed_network_size = Column(Integer, nullable=False, default=256)
fixed_networks_amount = Column(Integer, nullable=False, default=1)

13
nailgun/nailgun/db/sqlalchemy/models/node.py
View File

@ -32,7 +32,6 @@ from sqlalchemy.dialects import postgresql as psql
from nailgun import consts
from nailgun.db.sqlalchemy.models.base import Base
from nailgun.db.sqlalchemy.models.fields import JSON
from nailgun.db.sqlalchemy.models.fields import LowercaseString
from nailgun.db.sqlalchemy.models.mutable import MutableList
from nailgun.db.sqlalchemy.models.network import NetworkBondAssignment
from nailgun.db.sqlalchemy.models.network import NetworkNICAssignment
@ -74,8 +73,8 @@ class Node(Base):
default=consts.NODE_STATUSES.discover
)
meta = Column(JSON, default={})
mac = Column(LowercaseString(17), nullable=False, unique=True)
ip = Column(String(15))
mac = Column(psql.MACADDR, nullable=False, unique=True)
ip = Column(psql.INET)
hostname = Column(String(255), nullable=False,
default="", server_default="")
manufacturer = Column(Unicode(50))
@ -235,15 +234,15 @@ class NodeNICInterface(Base):
ForeignKey('nodes.id', ondelete="CASCADE"),
nullable=False)
name = Column(String(128), nullable=False)
mac = Column(LowercaseString(17), nullable=False)
mac = Column(psql.MACADDR, nullable=False)
max_speed = Column(Integer)
current_speed = Column(Integer)
assigned_networks_list = relationship(
"NetworkGroup",
secondary=NetworkNICAssignment.__table__,
order_by="NetworkGroup.id")
ip_addr = Column(String(25))
netmask = Column(String(25))
ip_addr = Column(psql.INET)
netmask = Column(psql.INET)
state = Column(String(25))
interface_properties = Column(JSON, default={}, nullable=False,
server_default='{}')
@ -298,7 +297,7 @@ class NodeBondInterface(Base):
ForeignKey('nodes.id', ondelete="CASCADE"),
nullable=False)
name = Column(String(32), nullable=False)
mac = Column(LowercaseString(17))
mac = Column(psql.MACADDR)
assigned_networks_list = relationship(
"NetworkGroup",
secondary=NetworkBondAssignment.__table__,

6
nailgun/nailgun/test/integration/test_db_refresh.py
View File

@ -48,7 +48,7 @@ class TestDBRefresh(TestCase):
def test_session_update(self):
node = Node()
node.mac = u"ASDFGHJKLMNOPR"
node.mac = "aa:bb:cc:dd:ff:11"
node.timestamp = datetime.now()
self.db.add(node)
self.db.commit()
@ -56,10 +56,10 @@ class TestDBRefresh(TestCase):
node2 = self.db2.query(Node).filter(
Node.id == node.id
).first()
node2.mac = u"12345678"
node2.mac = "aa:bb:cc:dd:ff:11"
self.db2.add(node2)
self.db2.commit()
self.db.query(Node).filter(
Node.id == node.id
).first()
self.assertEqual(node.mac, u"12345678")
self.assertEqual(node.mac, "aa:bb:cc:dd:ff:11")

3
nailgun/nailgun/test/integration/test_orchestrator_serializer.py
View File

@ -1887,7 +1887,8 @@ class TestNeutronOrchestratorSerializer(OrchestratorSerializerTestBase):
network[pkey] = pval
network['meta']['use_gateway'] = True
elif network['meta']['notation'] and not network['gateway']:
network['gateway'] = str(IPNetwork(network['cidr']).first)
cidr = IPNetwork(network['cidr'])
network['gateway'] = six.text_type(IPAddress(cidr.first))
network['meta']['use_gateway'] = True
netconfig['networking_parameters']['floating_ranges'] = \
[['199.10.0.77', '199.10.0.177']]

5
nailgun/nailgun/test/integration/test_orchestrator_serializer_70.py
View File

@ -1611,8 +1611,11 @@ class TestCustomNetGroupIpAllocation(BaseDeploymentSerializer):
'ip_range': ['172.16.122.2', '172.16.122.255']})
self.prepare_for_deployment(self.env.nodes)
ip_column_name = models.IPAddr.ip_addr.label('ip')
ip_addrs_count = db().query(models.IPAddr).filter(
models.IPAddr.ip_addr.like('172.16.122.%')).count()
"inet '172.16.122/24' >> {0}".format(ip_column_name)
).count()
self.assertEqual(ip_addrs_count, 2)

4
nailgun/nailgun/test/unit/test_network_group_handler.py
View File

@ -256,12 +256,12 @@ class TestHandlers(BaseIntegrationTest):
resp = self.env._create_network_group()
new_ng = resp.json_body
new_cidr = "10.3.0.1/20"
new_cidr = "10.3.0.0/20"
new_ng['cidr'] = new_cidr
new_ng['name'] = 'test'
new_ng['meta']['use_gateway'] = True
generated_range = IPNetwork("10.3.0.1/20")
generated_range = IPNetwork("10.3.0.0/20")
new_ip_range = [str(generated_range[2]), str(generated_range[-2])]
self.app.put(