root/build-tools/stx/debdownloader

#!/usr/bin/python3

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Copyright (C) 2021 Wind River Systems,Inc

import apt
import logging
import os
import repo_manage
import sys
import utils

REPO_BUILD = 'deb-local-build'
REPO_BIN = 'deb-local-binary'
DL_BINARY_DIR = os.path.join(os.environ.get('STX_MIRROR'), 'binaries')

logger = logging.getLogger('debdownloader')
utils.set_logger(logger)


class DebDownloader():
    def __init__(self, _dl_dir, _logger):
        self.dl_dir = _dl_dir
        self.apt_cache = apt.cache.Cache()
        self.logger = _logger

    def download(self, _name, _version):
        package = self.apt_cache[_name]
        candidate = package.versions.get(_version)
        if not candidate:
            logger.error(' '.join(['Fail to download', _name,
                         'with wrong version', _version, '?']))
            logger.error('May need to update the package list file')
            return None

        package.candidate = candidate
        try:
            ret = package.candidate.fetch_binary(self.dl_dir)
            if ret:
                self.logger.debug("Fetch binray ok with result %s" % ret)
                return ret
        except apt.package.FetchError:
            self.logger.error("Fail to fetch binray %s-%s" % (_name, _version))
        return None

    def get_downloaded_pkgs(self):
        dl_pkgs_list = []
        if not os.path.exists(self.dl_dir):
            return None

        for root, dirs, files in os.walk(self.dl_dir):
            if root or dirs:
                pass
            for file in files:
                dl_pkgs_list.append(file)

        return dl_pkgs_list


if __name__ == "__main__":
    if not len(sys.argv) == 2:
        logger.info("Usage: debdownloader <path to binary packages list>")
        sys.exit(0)

    if not sys.argv[1] or not os.path.isfile(sys.argv[1]):
        logger.error("Binary packages list doesn't exist")
        sys.exit(1)

    if not os.path.exists(DL_BINARY_DIR):
        os.makedirs(DL_BINARY_DIR)

    dl = DebDownloader(DL_BINARY_DIR, logger)
    debs_downloaded = dl.get_downloaded_pkgs()
    debs_need_download = []
    debs_need_upload = []

    # Create local binary repo with repo manager
    repomgr = repo_manage.RepoMgr('aptly', os.environ.get('REPOMGR_URL'),
                                  '/tmp/', os.environ.get('REPOMGR_ORIGIN'),
                                  logger)
    repomgr.upload_pkg(REPO_BIN, None)

    with open(sys.argv[1], 'r') as flist:
        lines = list(line for line in (lpkg.strip() for lpkg in flist) if line)
        for pkg in lines:
            pkg = pkg.strip()
            if pkg.startswith('#'):
                continue
            pkgname_parts = pkg.split()
            name = pkgname_parts[0]
            if len(pkgname_parts) == 1:
                logger.error(' '.join(['The version of package',
                                      name, 'must be defined']))
                logger.error(' '.join(['Please updated the list file',
                             sys.argv[1]]))
                sys.exit(1)
            version = pkgname_parts[1]
            # strip epoch
            major_ver = version.split(":")[-1]
            pname_x86 = ''.join([name, '_', major_ver, '_amd64.deb'])
            pname_all = ''.join([name, '_', major_ver, '_all.deb'])
            if repomgr.search_pkg(REPO_BIN, name, version):
                logger.info(''.join([name, '-', version,
                            ' is already in binary repo, skip download']))
            else:
                if debs_downloaded and pname_x86 in debs_downloaded:
                    logger.info(''.join([name, '-', version,
                                ' already downloaded, skip download']))
                    debs_need_upload.append(pname_x86)
                else:
                    if debs_downloaded and pname_all in debs_downloaded:
                        logger.info(''.join([name, '-', version,
                                    ' already downloaded, skip download']))
                        debs_need_upload.append(pname_all)
                    else:
                        debs_need_download.append(name + '_' + version)

    for deb in debs_need_upload:
        if repomgr.upload_pkg(REPO_BIN, os.path.join(DL_BINARY_DIR, deb)):
            logger.info(' '.join([os.path.join(DL_BINARY_DIR, deb),
                                  'is uploaded to', REPO_BIN]))
        else:
            logger.info(' '.join([os.path.join(DL_BINARY_DIR, deb),
                                  'fail to uploaded to', REPO_BIN]))

    for deb in debs_need_download:
        logger.debug(' '.join(['package', deb, 'is need to be downloaded']))
        debname_parts = deb.split('_')
        dlret = dl.download(debname_parts[0], debname_parts[1])
        if dlret:
            logger.info(''.join([debname_parts[0], '-', debname_parts[1],
                        ' download ok']))
            if repomgr.upload_pkg(REPO_BIN, dlret):
                logger.info(''.join([debname_parts[0], '-', debname_parts[1],
                            ' is uploaded to ', REPO_BIN]))
            else:
                logger.info(''.join([debname_parts[0], '-', debname_parts[1],
                            ' fail to upload to ', REPO_BIN]))

    logger.info("debdownloaader feeds local binary repo done")