starlingx/root
Code Issues Proposed changes

Adopt OS and LAT specific mirrors

Browse Source 
Establish separate OS and LAT mirrors to allow the upstream use of
Debian repos with a StarlingX LAT mirror for the LAT SDK

Test Plan:
- PASS: stx-init-env --rebuild, downloader, build-pkgs -a, build-image

Prototype: Concurrent Builds in master

Change-Id: I596b02353d5c537ecf2d7d335cfa1b46835320c8
Depends-On: https://review.opendev.org/c/starlingx/tools/+/946805
Story: NNNNNNN
Task: NNNNN
Signed-off-by: Robert Church <robert.church@windriver.com>
Signed-off-by: Scott Little <scott.little@windriver.com>
This commit is contained in:
Robert Church
2025-03-30 14:35:07 -05:00
committed by Scott Little
parent 4e62ff45e0
commit 0b821ea00a
6 changed files with 29 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
build-tools/stx/build-pkgs
View File

@@ -1263,7 +1263,7 @@ class BuildController():
if not skip_create_dsc:
try:
src_mirror_dir = os.path.join(os.environ.get('STX_MIRROR'), 'sources')
src_mirror_dir = os.path.join(os.environ.get('OS_MIRROR'), 'sources')
dsc_recipes = self.kits['dsc_maker'][build_type].package(pkg_dir, src_mirror_dir)
except Exception as e:
logger.error(str(e))

2
build-tools/stx/debdownloader
View File

@@ -23,7 +23,7 @@ import utils
REPO_BUILD = 'deb-local-build'
REPO_BIN = 'deb-local-binary'
DL_BINARY_DIR = os.path.join(os.environ.get('STX_MIRROR'), 'binaries')
DL_BINARY_DIR = os.path.join(os.environ.get('OS_MIRROR'), 'binaries')
logger = logging.getLogger('debdownloader')
utils.set_logger(logger)

27
build-tools/stx/debrepack.py
View File

@@ -35,29 +35,14 @@ import yaml
RELEASENOTES = " ".join([os.environ.get('PROJECT'), os.environ.get('MY_RELEASE'), "distribution"])
DIST = os.environ.get('STX_DIST')
# The CENGN_STRATEGY and CENGNURL references is retained for backward
# compatability with pre-existing build environments.
CENGN_BASE = None
CENGNURL = os.environ.get('CENGNURL')
CENGN_STRATEGY = os.environ.get('CENGN_STRATEGY')
if CENGNURL is not None:
CENGN_BASE = os.path.join(CENGNURL, "debian")
STX_MIRROR_BASE = None
STX_MIRROR_URL = os.environ.get('STX_MIRROR_URL')
if STX_MIRROR_URL is not None:
STX_MIRROR_BASE = os.path.join(STX_MIRROR_URL, "debian")
if STX_MIRROR_BASE is None:
STX_MIRROR_BASE = CENGN_BASE
STX_MIRROR_BASE = os.path.join(os.environ.get('STX_MIRROR_URL'), "debian")
OS_MIRROR_BASE = None
OS_MIRROR_URL = os.environ.get('OS_MIRROR_URL')
if OS_MIRROR_URL is not None:
OS_MIRROR_DIST_PATH = os.environ.get('OS_MIRROR_DIST_PATH')
OS_MIRROR_BASE = os.path.join(OS_MIRROR_URL, OS_MIRROR_DIST_PATH)
STX_MIRROR_STRATEGY = os.environ.get('STX_MIRROR_STRATEGY')
if STX_MIRROR_BASE is None:
STX_MIRROR_BASE = CENGN_BASE
if STX_MIRROR_STRATEGY is None:
STX_MIRROR_STRATEGY = CENGN_STRATEGY
if STX_MIRROR_STRATEGY == "cengn":
STX_MIRROR_STRATEGY = "stx_mirror"
if STX_MIRROR_STRATEGY == "cengn_first":
STX_MIRROR_STRATEGY = "stx_mirror_first"
STX_MIRROR_STRATEGY = "stx_mirror_first"
BTYPE = "@KERNEL_TYPE@"

11
build-tools/stx/downloader
View File

@@ -37,7 +37,7 @@ subprocess.run(['ionice', '-c', '3', '-p', str(pid)])
DEFAULT_ARCH = 'amd64'
REPO_BIN = 'deb-local-binary'
mirror_root = os.environ.get('STX_MIRROR')
mirror_root = os.environ.get('OS_MIRROR')
stx_src_mirror = os.path.join(mirror_root, 'sources')
stx_bin_mirror = os.path.join(mirror_root, 'binaries')
all_binary_lists = ['base-bullseye.lst', 'os-std.lst', 'os-rt.lst']
@@ -52,15 +52,8 @@ ALL_DISTROS = discovery.get_all_distros()
ALL_LAYERS = discovery.get_all_layers(distro=STX_DEFAULT_DISTRO)
ALL_BUILD_TYPES = discovery.get_all_build_types(distro=STX_DEFAULT_DISTRO)
# The CENGN_STRATEGY reference is retained for backward compatability
# with pre-existing build environments.
CENGN_STRATEGY = os.environ.get('CENGN_STRATEGY')
STX_MIRROR_STRATEGY = os.environ.get('STX_MIRROR_STRATEGY')
if not STX_MIRROR_STRATEGY:
STX_MIRROR_STRATEGY = CENGN_STRATEGY
if STX_MIRROR_STRATEGY == "cengn":
STX_MIRROR_STRATEGY = "stx_mirror"
elif STX_MIRROR_STRATEGY == "cengn_first":
if STX_MIRROR_STRATEGY is None:
STX_MIRROR_STRATEGY = "stx_mirror_first"
def get_downloaded(dl_dir, dl_type):

32
build-tools/stx/utils.py
View File

@@ -21,16 +21,10 @@ import subprocess
import urllib.parse
import urllib.request
# The CENGNURL reference is retained for backward compatability
# with pre-existing build environmnets.
CENGNURL = os.environ.get('CENGNURL')
if CENGNURL:
CENGN_BASE = os.path.join(CENGNURL, "debian")
STX_MIRROR_URL = os.environ.get('STX_MIRROR_URL')
if STX_MIRROR_URL:
STX_MIRROR_BASE = os.path.join(STX_MIRROR_URL, "debian")
if not STX_MIRROR_BASE:
STX_MIRROR_BASE=CENGN_BASE
OS_MIRROR_URL = os.environ.get('OS_MIRROR_URL')
OS_MIRROR_DIST_PATH = os.environ.get('OS_MIRROR_DIST_PATH')
if OS_MIRROR_URL:
OS_MIRROR_BASE = os.path.join(OS_MIRROR_URL, OS_MIRROR_DIST_PATH)
log_levels = {
'debug': logging.DEBUG,
@@ -181,7 +175,7 @@ def run_shell_cmd_full(cmd, logger, error_level=logging.ERROR):
def run_shell_cmd(cmd, logger, error_level=logging.ERROR):
return run_shell_cmd_full(cmd, logger, error_level)[0]
def url_to_stx_mirror(url):
def url_to_os_mirror(url):
url_change = urllib.parse.urlparse(url)
url_path = pathlib.Path(url_change.path)
@@ -191,23 +185,23 @@ def url_to_stx_mirror(url):
path = url_path
# FIXME: the ":" in a path is converted to "%25", after
# uploading to STX_MIRROR, the "%25" in the path is converted
# uploading to OS_MIRROR, the "%25" in the path is converted
# to "%2525".
return os.path.join(STX_MIRROR_BASE, path).replace("%25", "%2525")
return os.path.join(OS_MIRROR_BASE, path).replace("%25", "%2525")
def get_download_url(url, strategy):
alt_rt_url = None
stx_mirror_url = url_to_stx_mirror(url)
os_mirror_url = url_to_os_mirror(url)
if strategy == "stx_mirror":
rt_url = stx_mirror_url
rt_url = os_mirror_url
elif strategy == "upstream":
rt_url = url
elif strategy == "stx_mirror_first":
try:
urllib.request.urlopen(stx_mirror_url)
rt_url = stx_mirror_url
urllib.request.urlopen(os_mirror_url)
rt_url = os_mirror_url
alt_rt_url = url
except:
rt_url = url
@@ -215,9 +209,9 @@ def get_download_url(url, strategy):
try:
urllib.request.urlopen(url)
rt_url = url
alt_rt_url = stx_mirror_url
alt_rt_url = os_mirror_url
except:
rt_url = stx_mirror_url
rt_url = os_mirror_url
else:
raise Exception(f'Invalid value "{strategy}" of STX_MIRROR_STRATEGY')

12
build-tools/url_utils.sh
View File

@@ -236,12 +236,12 @@ repo_url_to_sub_path () {
return 0
}
STX_MIRROR_PROTOCOL="${STX_MIRROR_PROTOCOL:-https}"
STX_MIRROR_HOST="${STX_MIRROR_HOST:-mirror.starlingx.windriver.com}"
STX_MIRROR_PORT="${STX_MIRROR_PORT:-443}"
STX_MIRROR_URL_ROOT="${STX_MIRROR_URL_ROOT:-mirror}"
OS_MIRROR_PROTOCOL="${OS_MIRROR_PROTOCOL:-https}"
OS_MIRROR_HOST="${OS_MIRROR_HOST:-mirror.starlingx.windriver.com}"
OS_MIRROR_PORT="${OS_MIRROR_PORT:-443}"
OS_MIRROR_URL_ROOT="${OS_MIRROR_URL_ROOT:-mirror}"
url_to_stx_mirror_url () {
url_to_os_mirror_url () {
local URL="$1"
local DISTRO="$2"
local URL_PATH=""
@@ -264,7 +264,7 @@ url_to_stx_mirror_url () {
return 1
fi
echo "$STX_MIRROR_PROTOCOL://$STX_MIRROR_HOST:$STX_MIRROR_PORT/$STX_MIRROR_URL_ROOT/$DISTRO/$URL_PATH"
echo "$OS_MIRROR_PROTOCOL://$OS_MIRROR_HOST:$OS_MIRROR_PORT/$OS_MIRROR_URL_ROOT/$DISTRO/$URL_PATH"
return 0
}