deb-sahara/sahara/topology/topology_helper.py

# Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import hashlib

from oslo.config import cfg

from sahara import context
from sahara import exceptions as ex
from sahara.i18n import _
from sahara.i18n import _LI
from sahara.openstack.common import log
from sahara.utils.openstack import nova
from sahara.utils import xmlutils as x


TOPOLOGY_CONFIG = {
    "topology.node.switch.mapping.impl":
    "org.apache.hadoop.net.ScriptBasedMapping",
    "topology.script.file.name":
    "/etc/hadoop/topology.sh"
}

LOG = log.getLogger(__name__)

opts = [
    cfg.BoolOpt('enable_data_locality',
                default=False,
                help="""Enables data locality for hadoop cluster.
                 Also enables data locality for Swift used by hadoop.
                 If enabled, 'compute_topology' and 'swift_topology'
                 configuration parameters should point to OpenStack and Swift
                 topology correspondingly."""),
    cfg.BoolOpt('enable_hypervisor_awareness',
                default=True,
                help="""Enables four-level topology for data locality.
                Works only if corresponding plugin supports such mode."""),
    cfg.StrOpt('compute_topology_file',
               default='etc/sahara/compute.topology',
               help="""File with nova compute topology.
                It should contain mapping between nova computes and racks.
                File format:
                compute1 /rack1
                compute2 /rack2
                compute3 /rack2"""),
    cfg.StrOpt('swift_topology_file',
               default='etc/sahara/swift.topology',
               help="""File with Swift topology.
                It should contain mapping between Swift nodes and racks.
                File format:
                node1 /rack1
                node2 /rack2
                node3 /rack2""")
]

CONF = cfg.CONF
CONF.register_opts(opts)


def _read_swift_topology():
    LOG.debug("Reading Swift nodes topology from %s", CONF.swift_topology_file)
    topology = {}
    try:
        with open(CONF.swift_topology_file) as f:
            for line in f:
                line = line.strip()
                if not line:
                    continue
                (host, path) = line.split()
                topology[host] = path
    except IOError:
        LOG.debug("Unable to read Swift nodes topology from %s",
                  CONF.swift_topology_file)
        return {}

    return topology


def _read_compute_topology():
    LOG.debug("Reading compute nodes topology from %s",
              CONF.compute_topology_file)
    ctx = context.ctx()
    tenant_id = str(ctx.tenant_id)
    topology = {}
    try:
        with open(CONF.compute_topology_file) as f:
            for line in f:
                line = line.strip()
                if not line:
                    continue
                (host, path) = line.split()
                # Calulating host id based on tenant id and host
                # using the same algorithm as in nova
                # see nova/api/openstack/compute/views/servers.py
                # def _get_host_id(instance):
                sha_hash = hashlib.sha224(tenant_id + host)
                topology[sha_hash.hexdigest()] = path
    except IOError:
        raise ex.NotFoundException(
            CONF.compute_topology_file,
            _("Unable to find file %s with compute topology"))
    return topology


def generate_topology_map(cluster, is_node_awareness):
    mapping = _read_compute_topology()
    nova_client = nova.client()
    topology_mapping = {}
    for ng in cluster.node_groups:
        for i in ng.instances:
            # TODO(alazarev) get all servers info with one request
            ni = nova_client.servers.get(i.instance_id)
            hostId = ni.hostId
            if hostId not in mapping:
                raise ex.NotFoundException(
                    i.instance_id,
                    _("Was not able to find compute node topology for VM %s"))
            rack = mapping[hostId]
            if is_node_awareness:
                rack += "/" + hostId

            topology_mapping[i.instance_name] = rack
            topology_mapping[i.management_ip] = rack
            topology_mapping[i.internal_ip] = rack

    topology_mapping.update(_read_swift_topology())

    return topology_mapping


def vm_awareness_core_config():
    c = x.load_hadoop_xml_defaults('topology/resources/core-template.xml')
    result = [cfg for cfg in c if cfg['value']]

    if not CONF.enable_hypervisor_awareness:
        # not leveraging 4-layer approach so override template value
        param = next((prop for prop in result
                      if prop['name'] == 'net.topology.impl'), None)
        if param:
            param['value'] = 'org.apache.hadoop.net.NetworkTopology'

    LOG.info(_LI("Vm awareness will add following configs in core-site "
             "params: %s"), result)
    return result


def vm_awareness_mapred_config():
    c = x.load_hadoop_xml_defaults('topology/resources/mapred-template.xml')
    result = [cfg for cfg in c if cfg['value']]
    LOG.info(_LI("Vm awareness will add following configs in map-red "
             "params: %s"), result)
    return result


def vm_awareness_all_config():
    return vm_awareness_core_config() + vm_awareness_mapred_config()