root/build-tools/stx/utils.py

# Copyright (c) 2021 Wind River Systems, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Copyright (C) 2021 Wind River Systems,Inc

import logging
import os
import pathlib
import subprocess
import urllib.parse
import urllib.request

# The CENGNURL reference is retained for backward compatability 
# with pre-existing build environmnets.
CENGNURL = os.environ.get('CENGNURL')
if CENGNURL:
    CENGN_BASE = os.path.join(CENGNURL, "debian")
STX_MIRROR_URL = os.environ.get('STX_MIRROR_URL')
if STX_MIRROR_URL:
    STX_MIRROR_BASE = os.path.join(STX_MIRROR_URL, "debian")
if not STX_MIRROR_BASE:
    STX_MIRROR_BASE=CENGN_BASE

log_levels = {
    'debug': logging.DEBUG,
    'info': logging.INFO,
    'warning': logging.WARNING,
    'error': logging.ERROR,
    'crit': logging.CRITICAL
}

def set_logger(logger, log_level='debug'):
    logger.setLevel(log_levels[log_level])

    class ColorFormatter(logging.Formatter):
        FORMAT = ("%(asctime)s - $BOLD%(name)-s$RESET - %(levelname)s: %(message)s")

        BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = list(range(8))

        RESET_SEQ = "\033[0m"
        COLOR_SEQ = "\033[1;%dm"
        BOLD_SEQ = "\033[1m"

        COLORS = {
            'WARNING': YELLOW,
            'INFO': GREEN,
            'DEBUG': BLUE,
            'ERROR': RED
        }

        def formatter_msg(self, msg, use_color=True):
            if use_color:
                msg = msg.replace("$RESET", self.RESET_SEQ)
                msg = msg.replace("$BOLD", self.BOLD_SEQ)
            else:
                msg = msg.replace("$RESET", "").replace("$BOLD", "")
            return msg

        def __init__(self, use_color=True):
            msg = self.formatter_msg(self.FORMAT, use_color)
            logging.Formatter.__init__(self, msg)
            self.use_color = use_color

        def format(self, record):
            lname = record.levelname
            if self.use_color and lname in self.COLORS:
                fcolor = 30 + self.COLORS[lname]
                lncolor = self.COLOR_SEQ % fcolor + lname + self.RESET_SEQ
                record.levelname = lncolor
            return logging.Formatter.format(self, record)

    # create log and console handler and set level
    fh = logging.FileHandler('/localdisk/builder.log')
    fh.setLevel(log_levels[log_level])
    fh.setFormatter(ColorFormatter(use_color=False))
    logger.addHandler(fh)

    ch = logging.StreamHandler()
    ch.setLevel(log_levels[log_level])
    ch.setFormatter(ColorFormatter())
    logger.addHandler(ch)

    logger.propagate = 0


# Read file 'lst_file', sprip out blank lines and lines starting with '#'.
# Return the remaining lines as a list.  Optionally subject the lines
# to additional processing via the entry_handler prior to inclusion in
# the list
def bc_safe_fetch(lst_file, entry_handler=None, entry_handler_arg=None):
    entries = []
    try:
        with open(lst_file, 'r') as flist:
            lines = list(line for line in (p.strip() for p in flist) if line)
    except IOError as e:
        logger.error(str(e))
    except Exception as e:
        logger.error(str(e))
    else:
        for entry in lines:
            entry = entry.strip()
            if entry.startswith('#'):
                continue
            if entry == "":
                continue
            if entry_handler:
                if entry_handler_arg:
                    entries.extend(entry_handler(entry, entry_handler_arg))
                else:
                    entries.extend(entry_handler(entry))
            else:
                entries.append(entry)
    return entries


def limited_walk(dir, max_depth=1):
    dir = dir.rstrip(os.path.sep)
    assert os.path.isdir(dir)
    num_sep_dir = dir.count(os.path.sep)
    for root, dirs, files in os.walk(dir):
        yield root, dirs, files
        num_sep_root = root.count(os.path.sep)
        if num_sep_dir + max_depth <= num_sep_root:
            del dirs[:]


def run_shell_cmd_full(cmd, logger, error_level=logging.ERROR):
    if type(cmd) is str:
        shell = True
    elif type(cmd) in (tuple, list):
        shell = False
    else:
        raise Exception("Unrecognized 'cmd' type '%s'. Must be one of [str, list, tuple]." % (type(cmd)))

    logger.info(f'[ Run - "{cmd}" ]')
    try:
        process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                                   universal_newlines=True, shell=shell)
    except Exception as e:
        msg = f'[ Failed to execute command: "{cmd}" Exception: "{e}" ]'
        logger.log(error_level, msg)
        # Suppress the original exception when raising our own exception.
        # Syntax is acquired from: https://peps.python.org/pep-0409/#proposal
        raise Exception(msg) from None

    outs, errs = process.communicate()

    for log in outs.strip().split("\n"):
        log = log.strip()
        if log:
            logger.debug("stdout: %s", log)

    for log in errs.strip().split("\n"):
        log = log.strip()
        if log:
            logger.debug("stderr: %s", log)

    if process.returncode != 0:
        msg = f'[ Command failed with a non-zero return code: "{cmd}" return code: {process.returncode} ]'
        logger.log(error_level, msg)
        raise subprocess.CalledProcessError(
                returncode=process.returncode,
                cmd=cmd,
                output=outs,
                stderr=errs
        )

    return outs.strip(),errs.strip()

def run_shell_cmd(cmd, logger, error_level=logging.ERROR):
    return run_shell_cmd_full(cmd, logger, error_level)[0]

def url_to_stx_mirror(url):

    url_change = urllib.parse.urlparse(url)
    url_path = pathlib.Path(url_change.path)
    if url_change.netloc != '':
        path = pathlib.Path(url_change.netloc, url_path.relative_to("/"))
    else:
        path = url_path

    # FIXME: the ":" in a path is converted to "%25", after
    # uploading to STX_MIRROR, the "%25" in the path is converted
    # to "%2525".
    return os.path.join(STX_MIRROR_BASE, path).replace("%25", "%2525")


def get_download_url(url, strategy):

    alt_rt_url = None
    stx_mirror_url = url_to_stx_mirror(url)
    if strategy == "stx_mirror":
        rt_url = stx_mirror_url
    elif strategy == "upstream":
        rt_url = url
    elif strategy == "stx_mirror_first":
        try:
            urllib.request.urlopen(stx_mirror_url)
            rt_url = stx_mirror_url
            alt_rt_url = url
        except:
            rt_url = url
    elif strategy == "upstream_first":
        try:
            urllib.request.urlopen(url)
            rt_url = url
            alt_rt_url = stx_mirror_url
        except:
            rt_url = stx_mirror_url
    else:
        raise Exception(f'Invalid value "{strategy}" of STX_MIRROR_STRATEGY')

    return (rt_url, alt_rt_url)

def deb_file_name_to_dict(deb_file):
    ver_array = []
    arch = None
    pkg_epoch = None
    pkg_ver = None
    deb_array = deb_file.split("_")
    pkg_name = deb_array[0]
    if len(deb_array) >= 3:
        arch = deb_array[2].split(".")[0]
    if len(deb_array) >= 2:
        ver_array = deb_array[1].split(":")
    if len(ver_array) >= 2:
        pkg_ver = ver_array[-1]
        pkg_epoch = ver_array[0]
    elif len(ver_array) == 1:
        pkg_ver = ver_array[0]
        pkg_epoch = None
    pkg_dict = {'name':pkg_name, 'ver':pkg_ver, 'epoch':pkg_epoch, 'arch':arch, 'url':None}
    return pkg_dict
    
def deb_url_name_to_dict(deb_url):
    deb_file = os.path.basename(dub_url)
    pkg_dict = deb_file_name_to_dict(deb_file)
    pkg_dict['url'] = deb_url
    return pkg_dict