0401ee1408
In parallel building, some quick publishes been created base on repo
directly. When removing that very repo, those quick publishes should
be removed firstly, or the repo can not been removed.
Test Plan:
PASS: build-pkgs --parallel 30
repo_manage.py remove_repo -r deb-local-build
Closes-bug: 2000316
Depends-On: https://review.opendev.org/c/starlingx/root/+/868862
Signed-off-by: Zhang Xiao <xiao.zhang@windriver.com>
Change-Id: I099088ec302ce2e8ac46123bb07804935dabd382
875 lines
41 KiB
Python
Executable File
875 lines
41 KiB
Python
Executable File
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
# Copyright (C) 2021-2022 WindRiver Corporation
|
|
#
|
|
# Requires aptly-api-client:
|
|
# https://github.com/masselstine/aptly-api-client
|
|
#
|
|
# Old document of relate RESTAPI:
|
|
# https://www.aptly.info/doc/api/
|
|
#
|
|
# Realization of its real RESTAPI(go)
|
|
# https://github.com/molior-dbs/aptly
|
|
from aptly_api import Client
|
|
from debian import debian_support
|
|
import os
|
|
import time
|
|
from typing import Optional
|
|
|
|
PREFIX_LOCAL = 'deb-local-'
|
|
PREFIX_REMOTE = 'deb-remote-'
|
|
PREFIX_MERGE = 'deb-merge-'
|
|
SIGN_KEY = 'E82373F817C276756FA64756FAAD0555200D6582'
|
|
SIGN_PASSWD = 'starlingx'
|
|
DEFAULT_TIMEOUT_COUNT = 1
|
|
STX_DIST = os.environ.get('STX_DIST')
|
|
|
|
# Class used to manage aptly data base, it can:
|
|
# create_remote: Create a repository link to a remote mirror
|
|
# deploy_remote: Sync and deploy a remote mirror
|
|
# list_remotes: List all remote repositories
|
|
# remove_remote: Delete a remote repository
|
|
# create_local: Create a local repository
|
|
# upload_pkg_local: Upload a deb package into a local repository
|
|
# delete_pkg_local: Remove a deb package from a local repository
|
|
# pkg_exist: Search a package in a set of repos
|
|
# copy_pkgs: Copy packages from one repo to another
|
|
# deploy_local: Deploy a local repository
|
|
# list_local: List all local repositories
|
|
# remove_local: Delete a local repository
|
|
# clean_all: Clean all meta data including repo, public, distribution, package, task
|
|
|
|
|
|
class Deb_aptly():
|
|
def __init__(self, url, origin, logger):
|
|
'''The basic interface to manage aptly database. '''
|
|
self.logger = logger
|
|
self.url = url
|
|
self.aptly = Client(self.url)
|
|
self.logger.info('Aptly connected, version: %s', self.aptly.misc.version())
|
|
if origin:
|
|
self.origin = origin.strip() or None
|
|
else:
|
|
self.origin = None
|
|
|
|
# Create a remote mirror(make sure the name has specified prefix)
|
|
# Input
|
|
# name: the name of the remote repo : PREFIX_REMOTE-xxx
|
|
# url: the base url of the remote mirror: http://nginx.org/packages/debian
|
|
# distributions: the distribution: buster
|
|
# components: components=['nginx']
|
|
# architectures: architectures=['i386', 'arm64']
|
|
# with_sources: with_sources=True
|
|
# Output: None or Class 'aptly_api.parts.mirrors.Mirror'
|
|
def create_remote(self, name, url, distribution, **kwargs):
|
|
'''Base on a Debian repository, construct a repo as its mirror'''
|
|
if not name.startswith(PREFIX_REMOTE):
|
|
self.logger.error('%s is not started with %s, Failed.', name, PREFIX_REMOTE)
|
|
raise ValueError('remote repository create failed: prefix error')
|
|
|
|
remote_list = self.aptly.mirrors.list()
|
|
for remote in remote_list:
|
|
if remote.name == name:
|
|
self.logger.warning('mirror %s already exists.', remote.name)
|
|
return None
|
|
extra_param = {}
|
|
extra_param['distribution'] = distribution
|
|
extra_param['ignore_signatures'] = True
|
|
for key, value in kwargs.items():
|
|
if key == 'components':
|
|
extra_param['components'] = value
|
|
if key == 'with_sources':
|
|
extra_param['with_sources'] = True
|
|
if key == 'architectures':
|
|
extra_param['architectures'] = value
|
|
# Not find good/small repository with udebs, not verified
|
|
# if key == 'with_udebs':
|
|
# extra_param['with_udebs'] = True
|
|
remote = self.aptly.mirrors.create(name, url, **extra_param)
|
|
return remote
|
|
|
|
# update a mirror called "name". Mirror exist.
|
|
# Return False if failed
|
|
def __update_mirror(self, name):
|
|
'''Sync the mirror, may take minutes, depends on the size of the mirror ans the network. '''
|
|
mirror_list = self.aptly.mirrors.list()
|
|
# Add variable mirror_find just to avoid W0631
|
|
mirror_find = False
|
|
for mirror in mirror_list:
|
|
if mirror.name == name:
|
|
mirror_find = True
|
|
break
|
|
if not mirror_find:
|
|
self.logger.warning('Publish failed for mirror %s not find', name)
|
|
return False
|
|
# Please do NOT add any parameters here beside "ignore_signatures=True", that may
|
|
# overwrite previous settings and get strange results.
|
|
task = self.aptly.mirrors.update(name=name, ignore_signatures=True)
|
|
task_state = self.__wait_for_task(task, 15)
|
|
if task_state == 'SUCCEEDED':
|
|
return True
|
|
else:
|
|
self.logger.warning('Mirror %s update failed: %s', name, task_state)
|
|
return False
|
|
|
|
# Create a snapshot based on several others
|
|
# name : string, the name of new build snapshot
|
|
# source_snapshots: list of snapshots to be merge, order matters, snapshot at front of
|
|
# list has higher priority than snapshot later in the list.
|
|
# For each package, only the one with higher version can be selected:
|
|
# Return False on failure
|
|
def __merge_snapshot(self, name, source_snapshots):
|
|
'''Merge several snapshots into one, prepare for later deploy.'''
|
|
if not name.startswith(PREFIX_MERGE):
|
|
self.logger.error('%s did not start with %s, Failed.' % (name, PREFIX_MERGE))
|
|
return False
|
|
package_refs = []
|
|
# package_uniq_dict[pkgname_arch] = [package.key, snapshot]
|
|
package_uniq_dict = dict()
|
|
source_snapshots = [x.strip() for x in source_snapshots if x.strip() != '']
|
|
# remove duplicates (keep order)
|
|
source_snapshots = list(dict.fromkeys(source_snapshots))
|
|
snap_list = self.aptly.snapshots.list()
|
|
for snapshot in source_snapshots:
|
|
snap_exist = False
|
|
for snap in snap_list:
|
|
if snap.name == snapshot:
|
|
snap_exist = True
|
|
package_list = self.aptly.snapshots.list_packages(snap.name, with_deps=False, detailed=False)
|
|
# Debug only
|
|
# package_list.sort()
|
|
# self.logger.debug('%s packages in repo %s' % (len(package_list), snapshot))
|
|
for package in package_list:
|
|
key_list = package.key.split()
|
|
# 0: pkg_arch 1: pkg_name 2: pkg_version 3: pkg_key of aptly
|
|
pkgname_arch = '_'.join([key_list[1], key_list[0]])
|
|
# Source packages are useless for LAT, ignore them.
|
|
if "Psource" == key_list[0]:
|
|
continue
|
|
# Check and drop duplicate packages
|
|
if pkgname_arch in package_uniq_dict.keys():
|
|
need_replace = False
|
|
orig_version = package_uniq_dict[pkgname_arch][0].split()[2]
|
|
if STX_DIST in orig_version and STX_DIST not in key_list[2]:
|
|
self.logger.warn('STX package %s %s has been eclipsed by upstream version %s' %
|
|
(pkgname_arch, orig_version, key_list[2]))
|
|
if debian_support.version_compare(key_list[2], orig_version) > 0:
|
|
self.logger.warn('Drop duplicate package: %s.' %
|
|
' of '.join(package_uniq_dict[pkgname_arch]))
|
|
package_refs.remove(package_uniq_dict[pkgname_arch][0])
|
|
package_refs.append(package.key)
|
|
package_uniq_dict[pkgname_arch] = [package.key, snapshot]
|
|
else:
|
|
self.logger.warn('Drop duplicate package: %s of %s.' % (package.key, snapshot))
|
|
continue
|
|
package_uniq_dict[pkgname_arch] = [package.key, snapshot]
|
|
package_refs.append(package.key)
|
|
break
|
|
if not snap_exist:
|
|
self.logger.error('snapshot %s does not exist, merge failed.' % snapshot)
|
|
return False
|
|
|
|
# Remove a same name publish if exists
|
|
# For exist snapshot called NAME, we will:
|
|
# 1, rename it to backup-NAME
|
|
# 2, Create a new snapshot: NAME
|
|
# 3, delete snapshot backup-name
|
|
backup_name = None
|
|
publish_list = self.aptly.publish.list()
|
|
for publish in publish_list:
|
|
if publish.prefix == name:
|
|
task = self.aptly.publish.drop(prefix=name, distribution=publish.distribution, force_delete=True)
|
|
task_state = self.__wait_for_task(task)
|
|
if task_state != 'SUCCEEDED':
|
|
self.logger.warning('Drop publication failed %s : %s' % (name, task_state))
|
|
return False
|
|
# Remove the backup snapshot if it exists
|
|
snap_list = self.aptly.snapshots.list()
|
|
for snap in snap_list:
|
|
if snap.name == 'backup-' + name:
|
|
backup_name = 'backup-' + name
|
|
task = self.aptly.snapshots.delete(snapshotname=backup_name, force=True)
|
|
task_state = self.__wait_for_task(task)
|
|
if task_state != 'SUCCEEDED':
|
|
self.logger.warning('Drop snapshot failed %s : %s' % (backup_name, task_state))
|
|
return False
|
|
# Rename the snapshot if it exists
|
|
for snap in snap_list:
|
|
if snap.name == name:
|
|
backup_name = 'backup-' + name
|
|
self.__wait_for_task(self.aptly.snapshots.update(name, backup_name))
|
|
|
|
# crate a snapshot with package_refs. Duplicate package_refs is harmless.
|
|
# Note: The key is "package_refs" instead of "source_snapshots", for function
|
|
# "create_from_packages", parameter "source_snapshots" almost has no means.
|
|
task = None
|
|
task = self.aptly.snapshots.create_from_packages(name, source_snapshots=source_snapshots, package_refs=package_refs)
|
|
task_state = self.__wait_for_task(task)
|
|
if task_state != 'SUCCEEDED':
|
|
if backup_name:
|
|
self.__wait_for_task(self.aptly.snapshots.update(backup_name, name))
|
|
self.logger.warning('merge_snapshot: Snapshot for %s creation failed: %s. ' % (name, task_state))
|
|
return False
|
|
# Remove the backup snapshot if it is created above
|
|
if backup_name:
|
|
task = self.aptly.snapshots.delete(snapshotname=backup_name, force=True)
|
|
task_state = self.__wait_for_task(task)
|
|
if task_state != 'SUCCEEDED':
|
|
self.logger.warning('Drop snapshot failed %s : %s' % (backup_name, task_state))
|
|
return True
|
|
|
|
# Create a snapshot based on "name" with same name
|
|
# local: True ==> local_repo False ==> remote_mirror
|
|
# Return False if failed
|
|
def __create_snapshot(self, name, local):
|
|
'''For local-repo or remote-repo, create a snapshot for it, prepare for later deploy.'''
|
|
# Remove a same name publish if exists
|
|
# For exist snapshot called NAME, we will:
|
|
# 1, rename it to backup-NAME
|
|
# 2, Create a new snapshot: NAME
|
|
# 3, delete snapshot backup-name
|
|
backup_name = None
|
|
publish_list = self.aptly.publish.list()
|
|
for publish in publish_list:
|
|
if publish.prefix == name:
|
|
task = self.aptly.publish.drop(prefix=name, distribution=publish.distribution, force_delete=True)
|
|
task_state = self.__wait_for_task(task)
|
|
if task_state != 'SUCCEEDED':
|
|
self.logger.warning('Remove publication failed %s : %s' % (name, task_state))
|
|
# Rename the snapshot if exists
|
|
snap_list = self.aptly.snapshots.list()
|
|
|
|
exists = [snap for snap in snap_list if snap.name == name]
|
|
backup_exists = [snap for snap in snap_list if snap.name == 'backup-' + name]
|
|
if exists:
|
|
backup_name = 'backup-' + name
|
|
if backup_exists:
|
|
self.__wait_for_task(self.aptly.snapshots.delete(backup_name, force=True))
|
|
self.__wait_for_task(self.aptly.snapshots.update(name, backup_name))
|
|
|
|
# crate a snapshot
|
|
task = None
|
|
if local:
|
|
task = self.aptly.snapshots.create_from_repo(name, name)
|
|
else:
|
|
task = self.aptly.snapshots.create_from_mirror(name, name)
|
|
task_state = self.__wait_for_task(task)
|
|
if task_state != 'SUCCEEDED':
|
|
if backup_name:
|
|
self.__wait_for_task(self.aptly.snapshots.update(backup_name, name))
|
|
self.logger.warning('create_snapshot: Snapshot for %s creation failed: %s.' % (name, task_state))
|
|
return False
|
|
if backup_name:
|
|
task = self.aptly.snapshots.delete(snapshotname=backup_name, force=True)
|
|
task_state = self.__wait_for_task(task)
|
|
if task_state != 'SUCCEEDED':
|
|
self.logger.warning('Remove snapshot failed %s : %s' % (backup_name, task_state))
|
|
return True
|
|
|
|
# Wait for an aptly task up to a maximum of "count" minutes.
|
|
# By dafault, wait for DEFAULT_TIMEOUT_COUNT minute(s).
|
|
# Return: SUCCEEDED, FAILED, TIMEOUTED, EINVAL
|
|
def __wait_for_task(self, task, count=DEFAULT_TIMEOUT_COUNT):
|
|
'''Wait for an aptly task for one or more minutes'''
|
|
if count not in range(1, 30):
|
|
self.logger.error('Requested wait of % minutes is greater than 30 minutes max wait.', count)
|
|
return 'EINVAL'
|
|
while count > 0:
|
|
count -= 1
|
|
try:
|
|
# Function wait_for_task_by_id will return in 60 seconds, or timeout.
|
|
self.aptly.tasks.wait_for_task_by_id(task.id)
|
|
except Exception as e:
|
|
if count > 0:
|
|
self.logger.debug('Aptly task is still running...')
|
|
else:
|
|
self.logger.debug('%s' % e)
|
|
continue
|
|
else:
|
|
# return 'SUCCEEDED' or 'FAILED'
|
|
return self.aptly.tasks.show(task.id).state
|
|
return 'TIMEOUTED'
|
|
|
|
# Publish a local repository directly, without snapshot or signature
|
|
# If an old publish exists, drop it firstly and then create a new one.
|
|
# Do not use publish.update just for safety.
|
|
# (repo)repo_name ==> (publish)repo_name-suffix
|
|
def __quick_publish_repo(self, repo_name, suffix):
|
|
'''Create a publish based on a local repository directly, without snapshot.'''
|
|
# Caller already checked the repo_name, no need to check again
|
|
if not suffix:
|
|
self.logger.error('Quick publish needs suffix, none provided')
|
|
return
|
|
publish_name = '-'.join([repo_name, suffix])
|
|
publish_list = self.aptly.publish.list()
|
|
for publish in publish_list:
|
|
if publish.prefix == publish_name:
|
|
task = self.aptly.publish.drop(prefix=publish_name, distribution=publish.distribution, force_delete=True)
|
|
task_state = self.__wait_for_task(task)
|
|
if task_state != 'SUCCEEDED':
|
|
self.logger.warning('Drop failed publication %s : %s', publish_name, task_state)
|
|
return None
|
|
task = self.aptly.publish.publish(source_kind='local', sources=[{'Name': repo_name}],
|
|
architectures=['amd64', 'source'], prefix=publish_name,
|
|
distribution=None, sign_skip=True)
|
|
task_state = self.__wait_for_task(task, 10)
|
|
if task_state != 'SUCCEEDED':
|
|
self.logger.warning('Quick publish for %s create failed: %s', publish_name, task_state)
|
|
return None
|
|
return publish_name + ' ' + 'bullseye'
|
|
|
|
# Publish a snap called "name" with prefix as name, "bullseye" as the distribution
|
|
# Return None or prefix/distribution
|
|
def __publish_snap(self, name):
|
|
'''Deploy a snapshot.'''
|
|
# Remove a same name publish if exists
|
|
publish_list = self.aptly.publish.list()
|
|
for publish in publish_list:
|
|
if publish.prefix == name:
|
|
task = self.aptly.publish.drop(prefix=name, distribution=publish.distribution, force_delete=True)
|
|
task_state = self.__wait_for_task(task)
|
|
if task_state != 'SUCCEEDED':
|
|
self.logger.warning('Drop publish failed %s : %s', name, task_state)
|
|
return None
|
|
|
|
# is_remote: True => remote repo; False => local repo
|
|
is_remote = False
|
|
mirror = None
|
|
mirror_list = self.aptly.mirrors.list()
|
|
for mirror in mirror_list:
|
|
if mirror.name == name:
|
|
is_remote = True
|
|
break
|
|
|
|
# crate a publish
|
|
extra_param = {}
|
|
if is_remote:
|
|
# it is a remote repo: info storied in "mirror"
|
|
# Add 'source' to publish source packages, if no source packages, that is also harmless.
|
|
extra_param['architectures'] = mirror.architectures.append('source')
|
|
extra_param['distribution'] = mirror.distribution
|
|
extra_param['origin'] = None
|
|
else:
|
|
# Only support binary_amd64 and source packages
|
|
extra_param['architectures'] = ['amd64', 'source']
|
|
extra_param['distribution'] = None
|
|
extra_param['origin'] = self.origin
|
|
|
|
extra_param['source_kind'] = 'snapshot'
|
|
extra_param['sources'] = [{'Name': name}]
|
|
extra_param['sign_skip'] = True
|
|
extra_param['prefix'] = name
|
|
task = self.aptly.publish.publish(source_kind='snapshot', sources=extra_param['sources'],
|
|
architectures=extra_param['architectures'], prefix=extra_param['prefix'],
|
|
distribution=extra_param['distribution'],
|
|
sign_gpgkey=SIGN_KEY, sign_passphrase=SIGN_PASSWD,
|
|
origin=extra_param['origin'])
|
|
task_state = self.__wait_for_task(task, 10)
|
|
if task_state != 'SUCCEEDED':
|
|
self.logger.warning('Publication %s failed: %s' % (name, task_state))
|
|
return None
|
|
publish_list = self.aptly.publish.list()
|
|
for publish in publish_list:
|
|
if publish.prefix == name:
|
|
repo_str = publish.prefix + ' ' + publish.distribution
|
|
return repo_str
|
|
return None
|
|
|
|
# sync a remote mirror and deploy it
|
|
# Input: the name of the remote
|
|
# Output: bool
|
|
def deploy_remote(self, name):
|
|
'''Deploy a mirror, it will sync/update, snapshot and publish at last.
|
|
It may take minutes, depends on the size of the mirror and the bandwidth,
|
|
'''
|
|
if not name.startswith(PREFIX_REMOTE):
|
|
self.logger.warning('%s has no %s prefix, not a remote repository.', name, PREFIX_REMOTE)
|
|
return None
|
|
|
|
remote_list = self.aptly.mirrors.list()
|
|
remote = None
|
|
for remote in remote_list:
|
|
if remote.name == name:
|
|
break
|
|
|
|
if not remote:
|
|
self.logger.warning('mirror %s not find, please create it firstly.', name)
|
|
return None
|
|
|
|
if self.__update_mirror(name):
|
|
if self.__create_snapshot(name, False):
|
|
return self.__publish_snap(name)
|
|
return None
|
|
|
|
# info all remote repositories through logger
|
|
def list_remotes(self, quiet=False):
|
|
'''List all remote repositories/mirrors.'''
|
|
r_list = []
|
|
remote_list = self.aptly.mirrors.list()
|
|
if not len(remote_list):
|
|
if not quiet:
|
|
self.logger.info('No remote repo')
|
|
return r_list
|
|
if not quiet:
|
|
self.logger.info('%d remotes:', len(remote_list))
|
|
for remote in remote_list:
|
|
r_list.append(remote.name)
|
|
if not quiet:
|
|
self.logger.info('%s : %s : %s', remote.name, remote.archive_root, remote.distribution)
|
|
return r_list
|
|
|
|
# find and remove a remote
|
|
# Input: the name of the remote
|
|
# Output: Bool
|
|
def remove_remote(self, name):
|
|
'''Delete a remote repository/mirror and all related publish and snapshot.'''
|
|
if not name.startswith(PREFIX_REMOTE):
|
|
self.logger.warning('%s is not a correct remote name', name)
|
|
return False
|
|
|
|
# find and remove related publish
|
|
publish_list = self.aptly.publish.list()
|
|
for publish in publish_list:
|
|
if publish.prefix == name:
|
|
task = self.aptly.publish.drop(prefix=name, distribution=publish.distribution, force_delete=True)
|
|
self.__wait_for_task(task)
|
|
|
|
# find and remove related snapshot
|
|
snap_list = self.aptly.snapshots.list()
|
|
for snap in snap_list:
|
|
if snap.name == name:
|
|
task = self.aptly.snapshots.delete(snapshotname=name, force=True)
|
|
task_state = self.__wait_for_task(task)
|
|
if task_state != 'SUCCEEDED':
|
|
self.logger.warning('Drop snapshot failed %s : %s', name, task_state)
|
|
|
|
# find and remove the remote(mirror)
|
|
remote_list = self.aptly.mirrors.list()
|
|
for remote in remote_list:
|
|
if remote.name == name:
|
|
task = self.aptly.mirrors.drop(name=name, force=True)
|
|
task_state = self.__wait_for_task(task)
|
|
if task_state != 'SUCCEEDED':
|
|
self.logger.warning('Drop mirror failed %s : %s', name, task_state)
|
|
|
|
# Delete orphan files, wait up to 5 minutes for the cleanup to complete
|
|
task = self.aptly.db.cleanup()
|
|
self.__wait_for_task(task, 5)
|
|
|
|
return True
|
|
|
|
# info all local repositories through logger
|
|
def list_local(self, quiet=False):
|
|
'''List all local repository.'''
|
|
local_list = []
|
|
repo_list = self.aptly.repos.list()
|
|
if not len(repo_list):
|
|
self.logger.info('No local repo')
|
|
return local_list
|
|
if not quiet:
|
|
self.logger.info('%d local repos:', len(repo_list))
|
|
for repo in repo_list:
|
|
# rpo.name, repo.url, repo.distributions, repo.components
|
|
local_list.append(repo.name)
|
|
if not quiet:
|
|
self.logger.info('%s : %s : %s', repo.name, repo.default_distribution, repo.default_component)
|
|
return local_list
|
|
|
|
# Create a local repository
|
|
# Input:the name of the repo
|
|
# Output: None or repo
|
|
def create_local(self, local_name):
|
|
'''Create an empty local repository.'''
|
|
if not local_name.startswith(PREFIX_LOCAL):
|
|
self.logger.error('%s is not started with %s, Failed.', local_name, PREFIX_LOCAL)
|
|
raise ValueError('local repository create failed: prefix error.')
|
|
|
|
repo_list = self.aptly.repos.list()
|
|
for repo in repo_list:
|
|
if local_name == repo.name:
|
|
self.logger.warning('%s exists, please choose another name', local_name)
|
|
return None
|
|
|
|
# Static settings: bullseye main
|
|
repo = self.aptly.repos.create(local_name, default_distribution='bullseye', default_component='main')
|
|
return repo
|
|
|
|
# Upload a bundle of Debian package files into a local repository.
|
|
# For source package, all its package files need to be uploaded in one
|
|
# function call, or, uploaded files will not be inserted into repository
|
|
# but just deleted.
|
|
# Input:
|
|
# pkg_files: the path-name of the package files. If the file name
|
|
# contains "%3a", it will be replaced by ":".
|
|
# repo_name: the name of the local repository
|
|
# Output: Bool
|
|
def upload_pkg_local(self, pkg_files, repo_name):
|
|
'''Upload a bundle of package files into a local repository.'''
|
|
# sanity check: every package file is readable, local repository exists
|
|
if not pkg_files:
|
|
self.logger.warning('pkg_files should not be empty!')
|
|
return False
|
|
for pkg_file in set(pkg_files):
|
|
if not os.access(pkg_file, os.R_OK):
|
|
self.logger.warning('%s is NOT accessible to read.', pkg_file)
|
|
return False
|
|
if not repo_name.startswith(PREFIX_LOCAL):
|
|
self.logger.warning('%s is NOT a well formed name.', repo_name)
|
|
return False
|
|
|
|
repo_list = self.aptly.repos.list()
|
|
repo_found = False
|
|
for repo in repo_list:
|
|
if repo_name == repo.name:
|
|
self.logger.debug('repo %s was found and can be used', repo_name)
|
|
repo_found = True
|
|
break
|
|
|
|
if not repo_found:
|
|
self.logger.warning('repo %s does not exist, please create it first.', repo_name)
|
|
return False
|
|
|
|
# If the process was interrupted, leaving behind a file folder,
|
|
# clean it up by removing it before we start.
|
|
for file in self.aptly.files.list():
|
|
self.aptly.files.delete(file)
|
|
for pkg_file in set(pkg_files):
|
|
# For files with ":" in its filename, tools like 'apt' may replace it
|
|
# with '%3a' by mistake, this will cause error in aptly.
|
|
if pkg_file.find('%3a') >= 0:
|
|
rename_file = pkg_file.replace('%3a', ':')
|
|
try:
|
|
os.rename(pkg_file, rename_file)
|
|
except Exception as e:
|
|
self.logger.error('Error: %s' % e)
|
|
self.logger.error('Package file %s rename error.' % pkg_file)
|
|
raise Exception('Package file %s rename error, upload failed.' % pkg_file)
|
|
else:
|
|
# Upload package file into related file folder.
|
|
self.aptly.files.upload(repo_name, rename_file)
|
|
else:
|
|
self.aptly.files.upload(repo_name, pkg_file)
|
|
|
|
# Add uploaded file into local repository.
|
|
task = self.aptly.repos.add_uploaded_file(repo_name, repo_name, remove_processed_files=True)
|
|
task_state = self.__wait_for_task(task)
|
|
if task_state != 'SUCCEEDED':
|
|
self.logger.warning('add_upload_file failed %s : %s : %s', list(pkg_files)[0], repo_name, task_state)
|
|
return True
|
|
|
|
# Delete a Debian package from a local repository.
|
|
# Input:
|
|
# local_repo: the name of the local repository
|
|
# pkg_name: the path-name of the deb file
|
|
# pkg_type: 'binary' or 'source'
|
|
# pkg_version: version of the deb file
|
|
# Output: None
|
|
def delete_pkg_local(self, local_repo, pkg_name, pkg_type, pkg_version=None):
|
|
'''Delete a binary package from a local repository.'''
|
|
# self.logger.debug('delete_pkg_local not supported yet.')
|
|
if pkg_type not in {'binary', 'source'}:
|
|
self.logger.error('package type must be one of either "binary" or "source"')
|
|
return
|
|
if not pkg_version:
|
|
query = pkg_name
|
|
else:
|
|
query = pkg_name + ' (' + pkg_version + ')'
|
|
# If we want more detailed info, add "detailed=True, with_deps=True" for search_packages.
|
|
search_result = self.aptly.repos.search_packages(local_repo, query=query)
|
|
self.logger.debug('delete_pkg_local find %d packages.' % len(search_result))
|
|
for pkg in search_result:
|
|
if (pkg_type == 'source' and pkg.key.split()[0] == 'Psource') or \
|
|
(pkg_type != 'source' and pkg.key.split()[0] != 'Psource'):
|
|
task = self.aptly.repos.delete_packages_by_key(local_repo, pkg.key)
|
|
task_state = self.__wait_for_task(task)
|
|
if task_state != 'SUCCEEDED':
|
|
self.logger.warning('Delete package failed %s : %s' % (pkg_name, task_state))
|
|
|
|
def pkg_list(self, repo_list):
|
|
'''list packages available from any of the listed repos, local or remote.'''
|
|
pkg_list=[]
|
|
for repo_name in repo_list:
|
|
if repo_name.startswith(PREFIX_LOCAL):
|
|
query = 'Name'
|
|
pkgs_raw = self.aptly.repos.search_packages(repo_name, query=query)
|
|
pkgs_key = [pkg.key for pkg in pkgs_raw]
|
|
elif repo_name.startswith(PREFIX_REMOTE):
|
|
pkgs_key = self.aptly.mirrors.packages(repo_name)
|
|
for key in pkgs_key:
|
|
pkg_name = key.split()[1]
|
|
pkg_ver = key.split()[2]
|
|
pkg_arch = key.split()[0][1:]
|
|
if pkg_arch == 'source':
|
|
pkg_list.append("%s_%s.dsc" % (pkg_name, pkg_ver))
|
|
else:
|
|
pkg_list.append("%s_%s_%s.deb" % (pkg_name, pkg_ver, pkg_arch))
|
|
return pkg_list
|
|
|
|
|
|
# Search a package in a set of repos, return True if find, or False
|
|
# repolist: a list of repo names, including local repo and mirror
|
|
# pkg_name: package name
|
|
# architecture: Architecture of the package, now, only check 'source' or not
|
|
# pkg_version: the version of the package, None means version insensitive
|
|
def pkg_exist(self, repo_list, pkg_name, architecture, pkg_version=None):
|
|
'''Search a package in a bundle of repositories including local repo and remote one.'''
|
|
for repo_name in repo_list:
|
|
if repo_name.startswith(PREFIX_LOCAL):
|
|
if not pkg_version:
|
|
query = pkg_name
|
|
else:
|
|
query = pkg_name + ' (' + pkg_version + ')'
|
|
# If we want more detailed info, add "detailed=True, with_deps=True" for search_packages.
|
|
search_result = self.aptly.repos.search_packages(repo_name, query=query)
|
|
for pkg in search_result:
|
|
if architecture != 'source' and pkg.key.split()[0] != 'Psource':
|
|
self.logger.debug('pkg_exist find package %s in %s.', pkg_name, repo_name)
|
|
return True
|
|
if architecture == 'source' and pkg.key.split()[0] == 'Psource':
|
|
self.logger.debug('pkg_exist find package %s in %s.', pkg_name, repo_name)
|
|
return True
|
|
elif repo_name.startswith(PREFIX_REMOTE):
|
|
pkgs = self.aptly.mirrors.packages(repo_name)
|
|
for pkg in pkgs:
|
|
if pkg.split()[1] == pkg_name:
|
|
if architecture != 'source' and pkg.split()[0] != 'Psource' and (not pkg_version or pkg_version == pkg.split()[2]):
|
|
self.logger.debug('pkg_exist find package %s in %s.', pkg_name, repo_name)
|
|
return True
|
|
if architecture == 'source' and pkg.split()[0] == 'Psource' and (not pkg_version or pkg_version == pkg.split()[2]):
|
|
self.logger.debug('pkg_exist find package %s in %s.', pkg_name, repo_name)
|
|
return True
|
|
return False
|
|
|
|
# Copy a set of packages from one repository into another
|
|
# source: the repository name that packages been copied from
|
|
# dest: the repository name that packages been copied to
|
|
# pkg_list: list of package name to be copied
|
|
# pkg_type: binary or source. Default is binary
|
|
# overwrite: True or False. Overwrite existing packages or not
|
|
def copy_pkgs(self, source, dest, pkg_list, pkg_type='binary', overwrite=True):
|
|
'''Copy package from one repository to another local repository'''
|
|
dest_exist = False
|
|
source_exist = False
|
|
# package key list of destination and source repository
|
|
dest_pkg_keys = list()
|
|
src_pkg_keys = list()
|
|
if source == dest:
|
|
self.logger.error('%s and %s are the same repository.' % (source, dest))
|
|
return False
|
|
for repo in self.aptly.repos.list():
|
|
if dest == repo.name:
|
|
dest_exist = True
|
|
pkgs = self.aptly.repos.search_packages(dest, query='Name')
|
|
dest_pkg_keys = [pkg.key for pkg in pkgs]
|
|
if source == repo.name:
|
|
source_exist = True
|
|
pkgs = self.aptly.repos.search_packages(source, query='Name')
|
|
src_pkg_keys = [pkg.key for pkg in pkgs]
|
|
if not dest_exist:
|
|
self.logger.warning('Destination repository %s does not exist.', dest)
|
|
return False
|
|
if not source_exist:
|
|
for repo in self.aptly.mirrors.list():
|
|
if source == repo.name:
|
|
source_exist = True
|
|
src_pkg_keys = self.aptly.mirrors.packages(source)
|
|
break
|
|
if not source_exist:
|
|
self.logger.warning('Source repository %s dose not exist.', source)
|
|
return False
|
|
del_keys = list()
|
|
add_keys = list()
|
|
for key in src_pkg_keys:
|
|
package_name = key.split()[1]
|
|
package_type = key.split()[0]
|
|
if package_name not in pkg_list:
|
|
continue
|
|
if (pkg_type == 'source' and package_type != 'Psource') or (pkg_type == 'binary' and package_type == 'Psource'):
|
|
continue
|
|
# Find a package in source repository to be copied.
|
|
pkg_list.remove(package_name)
|
|
# Already exists in destination repository
|
|
if key in dest_pkg_keys:
|
|
continue
|
|
pkg_in_dest = False
|
|
for dest_key in dest_pkg_keys:
|
|
# [0] package type/arch: Psource, Pamd64, Pall. [1] package name
|
|
if package_type == dest_key.split()[0] and package_name == dest_key.split()[1]:
|
|
pkg_in_dest = True
|
|
if overwrite:
|
|
del_keys.append(dest_key)
|
|
add_keys.append(key)
|
|
break
|
|
if not pkg_in_dest:
|
|
add_keys.append(key)
|
|
if not pkg_list:
|
|
break
|
|
|
|
# check to see if any packages not find in source repository
|
|
if pkg_list:
|
|
self.logger.warning('Copy package error, %s package %s not exist in %s' % (pkg_type, ' '.join(pkg_list), source))
|
|
return False
|
|
# Remove duplicate packages from destination repository
|
|
if del_keys:
|
|
task = self.aptly.repos.delete_packages_by_key(dest, *del_keys)
|
|
task_state = self.__wait_for_task(task)
|
|
if task_state != 'SUCCEEDED':
|
|
self.logger.warning('Delete packages failed: %s\n%s' % (task_state, '\n'.join(del_keys)))
|
|
return False
|
|
# Insert packages into destination repository
|
|
if add_keys:
|
|
task = self.aptly.repos.add_packages_by_key(dest, *add_keys)
|
|
task_state = self.__wait_for_task(task)
|
|
if task_state != 'SUCCEEDED':
|
|
self.logger.warning('Copy packages failed: %s\n%s' % (task_state, '\n'.join(add_keys)))
|
|
return False
|
|
return True
|
|
|
|
# Merge several repositories into a new one(just snapshot and publish)
|
|
# name: the name of the new build snapshot/publish
|
|
# source_snapshots: list, snapshots to be merged
|
|
def merge_repos(self, name, source_snapshots):
|
|
'''Merge several repositories into a new publish.'''
|
|
if not name.startswith(PREFIX_MERGE):
|
|
self.logger.warning('The name should started with %s.', PREFIX_MERGE)
|
|
return None
|
|
|
|
if self.__merge_snapshot(name, source_snapshots):
|
|
ret = self.__publish_snap(name)
|
|
return ret
|
|
|
|
# deploy a local repository
|
|
# Input
|
|
# name: the name of the local repository
|
|
# suffix: suffix of the publish name
|
|
# Output: None or DebAptDistributionResponse
|
|
def deploy_local(self, name, suffix=''):
|
|
'''Deploy a local repository.'''
|
|
if not name.startswith(PREFIX_LOCAL):
|
|
self.logger.warning('%s is NOT a well formed name.', name)
|
|
return None
|
|
|
|
repo_list = self.aptly.repos.list()
|
|
repo_find = False
|
|
for repo in repo_list:
|
|
if name == repo.name:
|
|
self.logger.debug('%s find, can be used', name)
|
|
repo_find = True
|
|
break
|
|
if not repo_find:
|
|
self.logger.warning('local repo %s not found.', name)
|
|
return None
|
|
|
|
if suffix:
|
|
return self.__quick_publish_repo(name, suffix)
|
|
|
|
if self.__create_snapshot(name, True):
|
|
ret = self.__publish_snap(name)
|
|
# Delete orphan files, wait up to 5 minutes for the cleanup to complete
|
|
task = self.aptly.db.cleanup()
|
|
self.__wait_for_task(task, 5)
|
|
return ret
|
|
return None
|
|
|
|
# remove a local repository
|
|
# Input: the name of the local repository
|
|
# Output: None
|
|
def remove_local(self, name):
|
|
'''Delete a local repository, including related publish and snapshot.'''
|
|
if not name.startswith(PREFIX_LOCAL):
|
|
self.logger.warning('%s is not a correct name', name)
|
|
return None
|
|
|
|
# find and remove related publish
|
|
publish_list = self.aptly.publish.list()
|
|
for publish in publish_list:
|
|
# Remove all related publish including quick publish
|
|
if publish.prefix.startswith(name + '-') or publish.prefix == name:
|
|
task = self.aptly.publish.drop(prefix=publish.prefix, distribution='bullseye', force_delete=True)
|
|
task_state = self.__wait_for_task(task)
|
|
if task_state != 'SUCCEEDED':
|
|
self.logger.warning('Drop publish failed %s : %s', name, task_state)
|
|
|
|
# find and remove related snapshot
|
|
snap_list = self.aptly.snapshots.list()
|
|
for snap in snap_list:
|
|
if snap.name == name:
|
|
task = self.aptly.snapshots.delete(snapshotname=name, force=True)
|
|
task_state = self.__wait_for_task(task)
|
|
if task_state != 'SUCCEEDED':
|
|
self.logger.warning('Drop snapshot failed %s : %s', name, task_state)
|
|
|
|
# find and remove the remote(mirror)
|
|
repo_list = self.aptly.repos.list()
|
|
for repo in repo_list:
|
|
if repo.name == name:
|
|
task = self.aptly.repos.delete(reponame=name, force=True)
|
|
task_state = self.__wait_for_task(task)
|
|
if task_state != 'SUCCEEDED':
|
|
self.logger.warning('Drop repo failed %s : %s', name, task_state)
|
|
|
|
# Delete orphan files, wait up to 5 minutes for the cleanup to complete
|
|
task = self.aptly.db.cleanup()
|
|
self.__wait_for_task(task, 5)
|
|
|
|
return None
|
|
|
|
# clean all metadata including remote, repository, public, distribution, task and content
|
|
# In theory, with this operation, there should be nothing left in aptly_deb
|
|
# database. Please use it carefully.
|
|
def clean_all(self):
|
|
'''Clean all metadata including remote, repository, public, distribution, task and content.'''
|
|
# clean publishes
|
|
pub_list = self.aptly.publish.list()
|
|
self.logger.info('%d publish', len(pub_list))
|
|
for pub in pub_list:
|
|
task = self.aptly.publish.drop(prefix=pub.prefix, distribution=pub.distribution, force_delete=True)
|
|
task_state = self.__wait_for_task(task)
|
|
if task_state != 'SUCCEEDED':
|
|
self.logger.warning('Drop publish failed %s : %s', pub.frepix, task_state)
|
|
# clean snapshots
|
|
snap_list = self.aptly.snapshots.list()
|
|
self.logger.info('%d snapshot', len(snap_list))
|
|
for snap in snap_list:
|
|
task = self.aptly.snapshots.delete(snapshotname=snap.name, force=True)
|
|
task_state = self.__wait_for_task(task)
|
|
if task_state != 'SUCCEEDED':
|
|
self.logger.warning('Drop snapshot failed %s : %s', snap.name, task_state)
|
|
|
|
# clean mirrors
|
|
mirror_list = self.aptly.mirrors.list()
|
|
self.logger.info('%d mirror', len(mirror_list))
|
|
for mirror in mirror_list:
|
|
task = self.aptly.mirrors.drop(name=mirror.name, force=True)
|
|
task_state = self.__wait_for_task(task)
|
|
if task_state != 'SUCCEEDED':
|
|
self.logger.warning('Drop mirror failed %s : %s', mirror.name, task_state)
|
|
# clean local repos
|
|
repo_list = self.aptly.repos.list()
|
|
self.logger.info('%d repo', len(repo_list))
|
|
for repo in repo_list:
|
|
task = self.aptly.repos.delete(reponame=repo.name, force=True)
|
|
task_state = self.__wait_for_task(task)
|
|
if task_state != 'SUCCEEDED':
|
|
self.logger.warning('Drop repo failed %s : %s', repo.name, task_state)
|
|
# clean file folders
|
|
file_list = self.aptly.files.list()
|
|
self.logger.info('%d file folder', len(file_list))
|
|
for file in file_list:
|
|
self.aptly.files.delete(file)
|
|
# clean tasks
|
|
self.aptly.tasks.clear()
|
|
# Delete orphan files, up to 5 minutes
|
|
task = self.aptly.db.cleanup()
|
|
self.__wait_for_task(task, 5)
|