Allow our repositories to have multiple versions of a package

The original code would download all verions names in our lst files,
upload them one at a time to Aptly, and as part of each upload it
would delete other versions of the package.  The end result is that
the last uploaded version is the only one remaining in Aptly.

Fix is to delete old versions of a package from Aptly as a
separate step, with full knowledge of all versions we need to keep.

Testing:

1) Add multiple versions of a package to a lst file.
Run downloader.  All expected versions are found
in Aptly.

2) Remove the extra version of the package.
Run downloader.  The extra, no longer needed package
version are removed from Aptly leaving only the one we want.

Story: 2010797
Task: 48697

Change-Id: I6de33d8d1c2bb5161c905765e7757102ea9b8cac
Signed-off-by: Scott Little <scott.little@windriver.com>
This commit is contained in:
Scott Little 2023-09-11 11:33:40 -04:00
parent 3050ff7785
commit 9b0a139466
3 changed files with 213 additions and 84 deletions

@ -259,7 +259,7 @@ class DebDownloader(BaseDownloader):
logger.info("Successfully created repository %s", repo) logger.info("Successfully created repository %s", repo)
return True return True
def download(self, _name, _version, url=None): def download(self, _name, _version, url=None, retries=3):
if url != None: if url != None:
# The "+" in url is converted to "%2B", so convert # The "+" in url is converted to "%2B", so convert
# it back to "+" in save file. # it back to "+" in save file.
@ -268,13 +268,30 @@ class DebDownloader(BaseDownloader):
tmp_file = ".".join([ret, "tmp"]) tmp_file = ".".join([ret, "tmp"])
utils.run_shell_cmd(["rm", "-rf", tmp_file], logger) utils.run_shell_cmd(["rm", "-rf", tmp_file], logger)
(dl_url, alt_dl_url) = utils.get_download_url(url, CENGN_STRATEGY) (dl_url, alt_dl_url) = utils.get_download_url(url, CENGN_STRATEGY)
if alt_dl_url: for i in range(1,retries+1):
try: if alt_dl_url:
utils.run_shell_cmd(["curl", "-k", "-L", "-f", dl_url, "-o", tmp_file], logger) try:
except: utils.run_shell_cmd(["curl", "-k", "-L", "-f", dl_url, "-o", tmp_file], logger)
utils.run_shell_cmd(["curl", "-k", "-L", "-f", alt_dl_url, "-o", tmp_file], logger) except:
else: if i < retries:
utils.run_shell_cmd(["curl", "-k", "-L", "-f", dl_url, "-o", tmp_file], logger) try:
utils.run_shell_cmd(["curl", "-k", "-L", "-f", alt_dl_url, "-o", tmp_file], logger)
break
except Exception as e:
logger.error(str(e))
else:
utils.run_shell_cmd(["curl", "-k", "-L", "-f", alt_dl_url, "-o", tmp_file], logger)
break
else:
if i < retries:
try:
utils.run_shell_cmd(["curl", "-k", "-L", "-f", dl_url, "-o", tmp_file], logger)
break
except Exception as e:
logger.error(str(e))
else:
utils.run_shell_cmd(["curl", "-k", "-L", "-f", dl_url, "-o", tmp_file], logger)
break
utils.run_shell_cmd(["mv", tmp_file, ret], logger) utils.run_shell_cmd(["mv", tmp_file, ret], logger)
return ret return ret
@ -285,6 +302,8 @@ class DebDownloader(BaseDownloader):
logger.error(' '.join(['Fail to download', _name, logger.error(' '.join(['Fail to download', _name,
'with wrong version', _version, '?'])) 'with wrong version', _version, '?']))
logger.error('May need to update the package list file') logger.error('May need to update the package list file')
logger.error('package: %s', str(package))
logger.error('package.versions: %s', str(package.versions))
return None return None
package.candidate = candidate package.candidate = candidate
@ -302,6 +321,7 @@ class DebDownloader(BaseDownloader):
os.system('rm -f ' + os.path.join(self.dl_dir, deb_name + '*.deb')) os.system('rm -f ' + os.path.join(self.dl_dir, deb_name + '*.deb'))
return None return None
def reports(self): def reports(self):
for layer in self.layer_binaries: for layer in self.layer_binaries:
repo = self._get_layer_binaries_repository(layer) repo = self._get_layer_binaries_repository(layer)
@ -320,44 +340,98 @@ class DebDownloader(BaseDownloader):
logger.info("Show result for binary download:") logger.info("Show result for binary download:")
return super(DebDownloader, self).reports() return super(DebDownloader, self).reports()
def download_list(self, repo, list_file): def download_list_files(self, repo, list_files):
if not os.path.exists(list_file): pkg_data=[]
return if len(list_files):
for list_file in list_files:
self.downloaded = get_downloaded(self.dl_dir, 'binary') if not os.path.exists(list_file):
with open(list_file) as flist:
lines = list(line for line in (lpkg.strip() for lpkg in flist) if line)
for pkg in lines:
pkg = pkg.strip()
if pkg.startswith('#'):
continue continue
pkg_name_array = pkg.split() with open(list_file) as flist:
pkg_name = pkg_name_array[0] lines = list(line for line in (lpkg.strip() for lpkg in flist) if line)
if len(pkg_name_array) == 1: for pkg in lines:
logger.error("The package version of %s should be defined", pkg_name) pkg = pkg.strip()
logger.error("Please update the list file %s", list_file) if pkg.startswith('#'):
sys.exit(1) continue
# strip epoch pkg_name_array = pkg.split()
pkg_ver = pkg_name_array[1].split(":")[-1] pkg_name = pkg_name_array[0]
if len(pkg_name_array) == 3: if len(pkg_name_array) == 1:
url = pkg_name_array[2] logger.error("The package version of %s should be defined in file %s", pkg_name, list_file)
# Get arch from filename logger.error("Please update the list file %s", list_file)
arch = pathlib.Path(url).stem.split("_")[-1] sys.exit(1)
# strip epoch
ver_array = pkg_name_array[1].split(":")
if len(ver_array) == 1:
pkg_ver = ver_array[0]
pkg_epoch = None
else:
pkg_ver = ver_array[-1]
pkg_epoch = ver_array[0]
if len(pkg_name_array) == 3:
url = pkg_name_array[2]
url_dict = utils.deb_file_name_to_dict(os.path.basename(url).replace("%2B", "+"))
logger.debug("dkg_data: name=%s, ver=%s, url=%s, url_dict=%s, file=%s", pkg_name, pkg_ver, url,url, str(url_dict), list_file)
if url_dict['ver'] and url_dict['ver'] != pkg_ver:
logger.warning("Package version mismatch for package %s, $s vs %s, in file %s", pkg_name, pkg_ver, url_dict['ver'], list_file)
pkg_ver = url_dict['ver']
if url_dict['epoch'] and url_dict['epoch'] != pkg_epoch:
logger.warning("Package epoch mismatch for package %s, $s vs %s, in file %s", pkg_name, pkg_epoch, url_dict['epoch'], list_file)
pkg_epoch = url_dict['epoch']
# Get arch from filename
arch = pathlib.Path(url).stem.split("_")[-1]
else:
url = None
try:
package = self.apt_cache[pkg_name]
except Exception as e:
logger.error(str(e))
sys.exit(1)
arch = package.candidate.architecture
pkg_dict={'name':pkg_name, 'ver':pkg_ver, 'epoch':pkg_epoch, 'arch':arch, 'url':url, 'repo':repo}
pkg_data.append(pkg_dict)
self.download_list(repo, pkg_data)
def download_list(self, repo, pkg_data):
logger.info(' '.join(['pkg_data:', str(pkg_data)]))
# List of packages already downloaded
self.downloaded = get_downloaded(self.dl_dir, 'binary')
logger.info(' '.join(['previously downloaded:', str(self.downloaded)]))
# list of package already uploaded to ANY repo
previously_uploaded = self.repomgr.list_pkgs(repo)
logger.info(' '.join(['previously uploaded to repo', repo, ':', str(previously_uploaded)]))
pkg_data_map = {}
if pkg_data:
for pkg_dict in pkg_data:
pkg_name = pkg_dict['name']
pkg_ver = pkg_dict['ver']
pkg_epoch = pkg_dict['epoch']
arch = pkg_dict['arch']
url = pkg_dict['url']
repo = pkg_dict['repo']
if pkg_name not in pkg_data_map:
pkg_data_map[pkg_name] = []
pkg_data_map[pkg_name].append(pkg_dict)
pkg_name_ver = '_'.join([pkg_name, pkg_ver])
if pkg_epoch:
pkg_name_epoch_ver = '_'.join([pkg_name, ':'.join([pkg_epoch, pkg_ver])])
else: else:
url = None pkg_name_epoch_ver = pkg_name_ver
try:
package = self.apt_cache[pkg_name] pname_arch = '_'.join([pkg_name_ver, arch]) + '.deb'
except Exception as e: pname_epoch_arch = '_'.join([pkg_name_epoch_ver, arch]) + '.deb'
logger.error(str(e))
sys.exit(1) self.dl_need.append(pkg_name_ver)
arch = package.candidate.architecture
pname_arch = '_'.join([pkg_name, pkg_ver, arch]) + '.deb'
pname_epoch_arch = '_'.join([pkg_name, pkg_name_array[1], arch]) + '.deb'
self.dl_need.append(pkg_name + '_' + pkg_ver)
if self.downloaded and pname_arch in self.downloaded: if self.downloaded and pname_arch in self.downloaded:
logger.debug(''.join([pkg_name, '_', pkg_ver, logger.debug(''.join([pname_epoch_arch, ' has been downloaded, skip']))
' has been downloaded, skip']))
self.dl_success.append(pkg_name + '_' + pkg_ver) self.dl_success.append(pkg_name + '_' + pkg_ver)
self.need_upload.append([pname_arch, pname_epoch_arch]) self.need_upload.append([pname_arch, pname_epoch_arch])
else: else:
@ -365,22 +439,75 @@ class DebDownloader(BaseDownloader):
# fetch the package with 'apt' module, there is not 'epoch' # fetch the package with 'apt' module, there is not 'epoch'
# in the dowloaded package name. This also requires the 'epoch' # in the dowloaded package name. This also requires the 'epoch'
# should be defined in the package list file with ':' # should be defined in the package list file with ':'
self.need_download.append([pkg_name + '_' + pkg_name_array[1], url]) self.need_download.append([pname_arch, pkg_name_epoch_ver, url])
previously_uploaded = self.repomgr.list_pkgs(repo) # Download packages
logger.info(' '.join(['previously_uploaded', str(previously_uploaded)])) for debs in self.need_download:
pname_arch = debs[0]
pname_epoch_arch = debs[1]
url = debs[2]
logger.debug(' '.join(['package', pname_epoch_arch, 'needs to be downloaded']))
debnames = pname_epoch_arch.split('_')
deb_name = debnames[0]
ret = self.download(debnames[0], debnames[1], url)
if ret:
deb_ver = debnames[1].split(":")[-1]
deb_ver_epoch = '_'.join([debnames[0], debnames[1]])
logger.info(' '.join([deb_ver_epoch, ' download ok']))
# strip epoch
self.dl_success.append('_'.join([debnames[0], deb_ver]))
self.need_upload.append([pname_arch, pname_epoch_arch])
if previously_uploaded and deb_ver_epoch in previously_uploaded:
try:
del_ret = self.repomgr.delete_pkg(repo, deb_name, 'binary', deb_ver)
logger.debug("deleted the old %s from repo %s, ret %d", deb_name, repo, del_ret)
except Exception as e:
logger.error(str(e))
logger.error("Exception on deleting %s from %s", deb_name, repo)
else:
self.dl_failed.append(pname_epoch_arch)
self.need_download.clear()
logger.info(' '.join(['need_upload', str(self.need_upload)]))
# Delete previously uploaded packages that are no longer needed
for prev_upload in previously_uploaded:
prev_upload_dict = utils.deb_file_name_to_dict(prev_upload)
del_name = prev_upload_dict['name']
delete_me = True
# Verify the package is no londer needed
if pkg_data_map and del_name in pkg_data_map:
for needed_dict in pkg_data_map[del_name]:
if prev_upload_dict['ver'] == needed_dict['ver'] and \
prev_upload_dict['epoch'] == needed_dict['epoch'] and \
prev_upload_dict['arch'] == needed_dict['arch']:
# We still need this one
delete_me = False
continue
if delete_me:
del_ver = prev_upload_dict['ver']
logger.debug("Deleting pkg %s_%s freom %s", del_name, del_ver, repo)
try:
del_ret = self.repomgr.delete_pkg(repo, del_name, 'binary', del_ver)
except Exception as e:
logger.error(str(e))
logger.error("Exception on deleting %s from %s", '_'.join([del_name, del_ver]), repo)
# Upload needed packages
for debs in self.need_upload: for debs in self.need_upload:
deb = debs[0] deb_ver = debs[0]
deb_fver = debs[1] deb_ver_epoch = debs[1]
deb_path = os.path.join(stx_bin_mirror, deb) deb_path = os.path.join(stx_bin_mirror, deb_ver)
# Search the package with the "eopch" in aptly repo # Search the package with the "epoch" in aptly repo
if previously_uploaded and deb_fver in previously_uploaded: if previously_uploaded and deb_ver_epoch in previously_uploaded:
logger.info("%s has already been uploaded to %s, skip", deb_path, repo) logger.info("%s has already been uploaded to %s, skip", deb_path, repo)
continue continue
deb_needed_dict = utils.deb_file_name_to_dict(deb_ver)
logger.debug("Uploading pkg %s", deb_path)
try: try:
debnames = deb.split('_')
del_ret = self.repomgr.delete_pkg(repo, debnames[0], 'binary', None)
logger.debug("Only need uploading: Tried to delete the old %s, ret %d", debnames[0], del_ret)
upload_ret = self.repomgr.upload_pkg(repo, deb_path, deploy=False) upload_ret = self.repomgr.upload_pkg(repo, deb_path, deploy=False)
except Exception as e: except Exception as e:
logger.error(str(e)) logger.error(str(e))
@ -394,34 +521,7 @@ class DebDownloader(BaseDownloader):
break break
self.need_upload.clear() self.need_upload.clear()
for debs in self.need_download:
deb = debs[0]
url = debs[1]
logger.debug(' '.join(['package', deb, 'needs to be downloaded']))
debnames = deb.split('_')
ret = self.download(debnames[0], debnames[1], url)
if ret:
logger.info(''.join([debnames[0], '_', debnames[1], ' download ok']))
# strip epoch
deb_ver = debnames[1].split(":")[-1]
self.dl_success.append('_'.join([debnames[0], deb_ver]))
try:
del_ret = self.repomgr.delete_pkg(repo, debnames[0], 'binary', None)
logger.debug("Tried to delete the old %s, ret %d", debnames[0], del_ret)
upload_ret = self.repomgr.upload_pkg(repo, ret, deploy=False)
except Exception as e:
logger.error(str(e))
logger.error("Exception on uploading %s to %s", deb_path, repo)
sys.exit(1)
else:
if upload_ret:
logger.info(''.join([debnames[0], '_', debnames[1], ' is uploaded to ', repo]))
else:
logger.error(''.join([debnames[0], '_', debnames[1], ' fail to upload to ', repo]))
break
else:
self.dl_failed.append(deb)
self.need_download.clear()
def start(self): def start(self):
"""Here define: """Here define:
@ -433,16 +533,18 @@ class DebDownloader(BaseDownloader):
empty = True empty = True
for layer in self.layer_binaries: for layer in self.layer_binaries:
repo = self._get_layer_binaries_repository(layer)
if self.layer_binaries[layer]: if self.layer_binaries[layer]:
for bin_list in self.layer_binaries[layer]: for bin_list in self.layer_binaries[layer]:
empty = False empty = False
self.download_list(repo, bin_list)
if empty: if empty:
logger.error("There are no lists of binary packages found") logger.error("There are no lists of binary packages found")
sys.exit(1) sys.exit(1)
for layer in self.layer_binaries:
repo = self._get_layer_binaries_repository(layer)
if self.layer_binaries[layer]:
self.download_list_files(repo, self.layer_binaries[layer])
class SrcDownloader(BaseDownloader): class SrcDownloader(BaseDownloader):
def __init__(self, arch, _dl_dir, force): def __init__(self, arch, _dl_dir, force):

@ -653,6 +653,7 @@ class RepoMgr():
# Output: True if all works. # Output: True if all works.
def upload_pkg(self, repo_name, package, deploy=True): def upload_pkg(self, repo_name, package, deploy=True):
'''Upload a Debian package into a specified repository.''' '''Upload a Debian package into a specified repository.'''
self.logger.info("upload_pkg: %s to %s", package, repo_name)
local_list = self.repo.list_local(quiet=True) local_list = self.repo.list_local(quiet=True)
if repo_name not in local_list: if repo_name not in local_list:
self.logger.info('upload_pkg: repository %s does not exist, creating it.' % repo_name) self.logger.info('upload_pkg: repository %s does not exist, creating it.' % repo_name)

@ -206,3 +206,29 @@ def get_download_url(url, strategy):
raise Exception(f'Invalid value "{strategy}" of CENGN_STRATEGY') raise Exception(f'Invalid value "{strategy}" of CENGN_STRATEGY')
return (rt_url, alt_rt_url) return (rt_url, alt_rt_url)
def deb_file_name_to_dict(deb_file):
ver_array = []
arch = None
pkg_epoch = None
pkg_ver = None
deb_array = deb_file.split("_")
pkg_name = deb_array[0]
if len(deb_array) >= 3:
arch = deb_array[2].split(".")[0]
if len(deb_array) >= 2:
ver_array = deb_array[1].split(":")
if len(ver_array) >= 2:
pkg_ver = ver_array[-1]
pkg_epoch = ver_array[0]
elif len(ver_array) == 1:
pkg_ver = ver_array[0]
pkg_epoch = None
pkg_dict = {'name':pkg_name, 'ver':pkg_ver, 'epoch':pkg_epoch, 'arch':arch, 'url':None}
return pkg_dict
def deb_url_name_to_dict(deb_url):
deb_file = os.path.basename(dub_url)
pkg_dict = deb_file_name_to_dict(deb_file)
pkg_dict['url'] = deb_url
return pkg_dict