Use "apt-ostree" to manage software deploy start
Unpack and install Debian packages into a checked out Ostree branch then commit results back into the repository so that it can be deployable. The way that this works is that a "patch" is uploaded to the controller. The patch is a tarball that contains metadata information and Debian packages to be extracted. The process is the following: 1. Create the Debian package feed when the software-controller starts. 2. Upload the patch to the controller. 3. Extract the Debian package(s) fro the patch. 4. Upload the Debian packages to the Debian package feed. 5. Grab a list of the packages that have been included in the patch. 6. Run "apt-ostree compose install" to install or upgrade the Debian packages installed. 7. Results are commited back into the Ostree repository. Signed-off-by: Charles Short <charles.short@windriver.com> Test Plan PASS Build software debian package PASS BUILD ISO PASS Verify setting in /etc/sofware/software.conf Task: 49229 Story: 2010676 Change-Id: I7127f62043428693f85260fa1ee944e84f6b532d
This commit is contained in:
Charles Short
parent
ad7525f25d
commit
728cfdbff1
|
|
@ -24,9 +24,10 @@ NAME=$(basename $0)
|
|||
|
||||
REPO_ID=updates
|
||||
REPO_ROOT=/var/www/pages/${REPO_ID}
|
||||
REPO_DIR=${REPO_ROOT}/rel-${SW_VERSION}
|
||||
REPO_DIR=${REPO_ROOT}/debian/rel-${SW_VERSION}
|
||||
GROUPS_FILE=$REPO_DIR/comps.xml
|
||||
PATCHING_DIR=/opt/software
|
||||
RELEASE=bullseye
|
||||
|
||||
logfile=/var/log/software.log
|
||||
|
||||
|
|
@ -43,10 +44,17 @@ function do_setup {
|
|||
# Does the repo exist?
|
||||
if [ ! -d $REPO_DIR ]; then
|
||||
LOG "Creating repo."
|
||||
mkdir -p $REPO_DIR
|
||||
|
||||
# The original Centos code would create the groups and call createrepo
|
||||
# todo(jcasteli): determine if the ostree code needs a setup also
|
||||
# TODO(cshort) Remove this once gpg support is added.
|
||||
sed -i '$a gpg-verify=false' \
|
||||
/var/www/pages/feed/rel-${SW_VERSION}/ostree_repo/config
|
||||
sed -i '$a gpg-verify=false' \
|
||||
/sysroot/ostree/repo/config
|
||||
|
||||
apt-ostree repo init \
|
||||
--feed $REPO_DIR \
|
||||
--release $RELEASE \
|
||||
--origin $REPO_ID
|
||||
fi
|
||||
|
||||
if [ ! -d $PATCHING_DIR ]; then
|
||||
|
|
|
|||
|
|
@ -8,3 +8,5 @@ agent_port = 5495
|
|||
# alternate PostgreSQL server port for bringing up
|
||||
# db to run on to-release
|
||||
alt_postgresql_port = 6666
|
||||
# todo(cshort): This needs to be configured in puppet.
|
||||
package_feed="http://controller:8080/updates/debian/rel-23.09/ bullseye updates"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,66 @@
|
|||
"""
|
||||
Copyright (c) 2023 Wind River Systems, Inc.
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""
|
||||
import logging
|
||||
import subprocess
|
||||
|
||||
from software import constants
|
||||
import software.config as cfg
|
||||
from software.exceptions import APTOSTreeCommandFail
|
||||
|
||||
LOG = logging.getLogger('main_logger')
|
||||
|
||||
|
||||
def package_upload(feed_dir, package):
|
||||
"""
|
||||
Upload a Debian package to an apt repository.
|
||||
|
||||
:param feed_dir: apt package feed directory
|
||||
:param package: Debian package
|
||||
"""
|
||||
try:
|
||||
msg = "Uploading package: %s" % package
|
||||
LOG.info(msg)
|
||||
|
||||
subprocess.run(
|
||||
["apt-ostree", "repo", "add",
|
||||
"--feed", str(feed_dir),
|
||||
"--release", constants.DEBIAN_RELEASE,
|
||||
package],
|
||||
check=True,
|
||||
capture_output=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
msg = "Failed to upload package: %s" % package
|
||||
info_msg = "\"apt-ostree repo add\" error: return code %s , Output: %s" \
|
||||
% (e.returncode, e.stderr.decode("utf-8"))
|
||||
LOG.error(info_msg)
|
||||
raise APTOSTreeCommandFail(msg)
|
||||
|
||||
|
||||
def run_install(repo_dir, packages):
|
||||
"""
|
||||
Run Debian package upgrade.
|
||||
|
||||
:param repo_dir: the path to the ostree repo
|
||||
"""
|
||||
try:
|
||||
LOG.info("Running apt-ostree install")
|
||||
|
||||
packages = " ".join(packages)
|
||||
subprocess.run(
|
||||
["apt-ostree", "compose", "install",
|
||||
"--repo", repo_dir,
|
||||
"--branch", "starlingx",
|
||||
"--feed", cfg.package_feed,
|
||||
packages],
|
||||
check=True,
|
||||
capture_output=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
msg = "Failed to install packages."
|
||||
info_msg = "\"apt-ostree compose intstall\" error: return code %s , Output: %s" \
|
||||
% (e.returncode, e.stderr.decode("utf-8"))
|
||||
LOG.error(info_msg)
|
||||
raise APTOSTreeCommandFail(msg)
|
||||
|
|
@ -24,6 +24,7 @@ api_port = 0
|
|||
alt_postgresql_port = 0
|
||||
mgmt_if = None
|
||||
nodetype = None
|
||||
package_feed = None
|
||||
platform_conf_mtime = 0
|
||||
software_conf_mtime = 0
|
||||
software_conf = constants.SOFTWARE_CONFIG_FILE_LOCAL
|
||||
|
|
@ -59,6 +60,11 @@ pecan_opts = [
|
|||
'guess_content_type_from_ext',
|
||||
default=False
|
||||
),
|
||||
cfg.StrOpt(
|
||||
"package_feed",
|
||||
default="http://controller:8080/updates/debian/rel-%s/ bullseye updates"
|
||||
% constants.STARLINGX_RELEASE
|
||||
),
|
||||
]
|
||||
|
||||
# register the configuration for this component
|
||||
|
|
@ -80,6 +86,9 @@ def read_config():
|
|||
'controller_port': "5494",
|
||||
'agent_port': "5495",
|
||||
'alt_postgresql_port': "6666",
|
||||
"package_feed":
|
||||
"http://controller:8080/updates/debian/rel-%s/ bullseye updates"
|
||||
% constants.STARLINGX_RELEASE,
|
||||
}
|
||||
|
||||
global controller_mcast_group
|
||||
|
|
@ -88,6 +97,7 @@ def read_config():
|
|||
global controller_port
|
||||
global agent_port
|
||||
global alt_postgresql_port
|
||||
global package_feed
|
||||
|
||||
config = configparser.ConfigParser(defaults)
|
||||
|
||||
|
|
@ -102,6 +112,7 @@ def read_config():
|
|||
controller_port = config.getint('runtime', 'controller_port')
|
||||
agent_port = config.getint('runtime', 'agent_port')
|
||||
alt_postgresql_port = config.getint('runtime', 'alt_postgresql_port')
|
||||
package_feed = config.get("runtime", "package_feed")
|
||||
|
||||
# The platform.conf file has no section headers, which causes problems
|
||||
# for ConfigParser. So we'll fake it out.
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ try:
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
from tsconfig.tsconfig import SW_VERSION
|
||||
|
||||
ADDRESS_VERSION_IPV4 = 4
|
||||
ADDRESS_VERSION_IPV6 = 6
|
||||
CONTROLLER_FLOATING_HOSTNAME = "controller"
|
||||
|
|
@ -73,8 +75,11 @@ PATCH_AGENT_STATE_INSTALL_REJECTED = "install-rejected"
|
|||
|
||||
FEED_OSTREE_BASE_DIR = "/var/www/pages/feed"
|
||||
OSTREE_BASE_DEPLOYMENT_DIR = "/ostree/deploy/debian/deploy/"
|
||||
PACKAGE_FEED_DIR = "/var/www/pages/updates/debian"
|
||||
OSTREE_REF = "starlingx"
|
||||
OSTREE_REMOTE = "debian"
|
||||
DEBIAN_RELEASE = "bullseye"
|
||||
STARLINGX_RELEASE = SW_VERSION
|
||||
PATCH_SCRIPTS_STAGING_DIR = "/var/www/pages/updates/software-scripts"
|
||||
SYSROOT_OSTREE = "/sysroot/ostree/repo"
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,11 @@ class SoftwareError(Exception):
|
|||
return self.message or ""
|
||||
|
||||
|
||||
class APTOSTreeCommandFail(SoftwareError):
|
||||
"""Apt-ostree errror."""
|
||||
pass
|
||||
|
||||
|
||||
class MetadataFail(SoftwareError):
|
||||
"""Metadata error."""
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -26,10 +26,12 @@ from wsgiref import simple_server
|
|||
|
||||
from oslo_config import cfg as oslo_cfg
|
||||
|
||||
import software.apt_utils as apt_utils
|
||||
import software.ostree_utils as ostree_utils
|
||||
from software.api import app
|
||||
from software.authapi import app as auth_app
|
||||
from software.base import PatchService
|
||||
from software.exceptions import APTOSTreeCommandFail
|
||||
from software.exceptions import MetadataFail
|
||||
from software.exceptions import UpgradeNotSupported
|
||||
from software.exceptions import OSTreeCommandFail
|
||||
|
|
@ -2032,11 +2034,14 @@ class PatchController(PatchService):
|
|||
# todo(jcasteli) Remove once the logic to include major release version
|
||||
# in release list is implemented
|
||||
running_sw_version = "23.09.0"
|
||||
# Commit1 in release metadata.xml file represents the latest commit
|
||||
for release_id in sorted(list(self.release_data.metadata)):
|
||||
if self.latest_feed_commit == self.release_data.contents[release_id]["commit1"]["commit"]:
|
||||
running_sw_version = self.release_data.metadata[release_id]["sw_version"]
|
||||
LOG.info("Running software version: %s", running_sw_version)
|
||||
|
||||
# todo(chuck) Remove once to determine how we are associating a patch
|
||||
# with a release.
|
||||
# release in release metadata.xml file represents the latest commit
|
||||
# for release_id in sorted(list(self.release_data.metadata)):
|
||||
# if SW_VERSION == self.release_data.contents[release_id]["release"]:
|
||||
# running_sw_version = self.release_data.metadata[release_id]["sw_version"]
|
||||
# LOG.info("Running software version: %s", running_sw_version)
|
||||
|
||||
higher = utils.compare_release_version(self.release_data.metadata[deployment]["sw_version"],
|
||||
running_sw_version)
|
||||
|
|
@ -2090,6 +2095,12 @@ class PatchController(PatchService):
|
|||
LOG.info(msg)
|
||||
audit_log_info(msg)
|
||||
|
||||
packages = self.release_data.metadata[release].get("packages")
|
||||
if packages is None:
|
||||
msg = "Unable to determine pckages to install"
|
||||
LOG.error(msg)
|
||||
raise MetadataFail(msg)
|
||||
|
||||
if self.release_data.metadata[release]["state"] != constants.AVAILABLE \
|
||||
or self.release_data.metadata[release]["state"] == constants.COMMITTED:
|
||||
msg = "%s is already being deployed" % release
|
||||
|
|
@ -2103,50 +2114,49 @@ class PatchController(PatchService):
|
|||
latest_commit = ""
|
||||
try:
|
||||
latest_commit = ostree_utils.get_feed_latest_commit(release_sw_version)
|
||||
LOG.info("Latest commit: %s" % latest_commit)
|
||||
except OSTreeCommandFail:
|
||||
LOG.exception("Failure during commit consistency check for %s.", release)
|
||||
|
||||
if self.release_data.contents[release]["base"]["commit"] != latest_commit:
|
||||
msg = "The base commit %s for %s does not match the latest commit %s " \
|
||||
"on this system." \
|
||||
% (self.release_data.contents[release]["base"]["commit"],
|
||||
release,
|
||||
latest_commit)
|
||||
LOG.info(msg)
|
||||
msg_info += msg + "\n"
|
||||
continue
|
||||
|
||||
ostree_tar_filename = self.get_ostree_tar_filename(release_sw_version, release)
|
||||
package_repo_dir = "%s/rel-%s" % (constants.PACKAGE_FEED_DIR, release_sw_version)
|
||||
feed_ostree = "%s/rel-%s/ostree_repo" % (constants.FEED_OSTREE_BASE_DIR, release_sw_version)
|
||||
|
||||
# Create a temporary working directory
|
||||
tmpdir = tempfile.mkdtemp(prefix="deployment_")
|
||||
|
||||
# Save the current directory, so we can chdir back after
|
||||
orig_wd = os.getcwd()
|
||||
|
||||
# Change to the tmpdir
|
||||
os.chdir(tmpdir)
|
||||
|
||||
try:
|
||||
# Extract the software.tar
|
||||
tar = tarfile.open(ostree_tar_filename)
|
||||
tar.extractall()
|
||||
feed_ostree = "%s/rel-%s/ostree_repo" % (constants.FEED_OSTREE_BASE_DIR, release_sw_version)
|
||||
# Copy extracted folders of software.tar to the feed ostree repo
|
||||
shutil.copytree(tmpdir, feed_ostree, dirs_exist_ok=True)
|
||||
tar.extractall(path=tmpdir)
|
||||
except tarfile.TarError:
|
||||
msg = "Failed to extract the ostree tarball for %s" % release
|
||||
LOG.exception(msg)
|
||||
raise OSTreeTarFail(msg)
|
||||
except shutil.Error:
|
||||
msg = "Failed to copy the ostree tarball for %s" % release
|
||||
|
||||
# Upload the package to the package feed.
|
||||
try:
|
||||
deb_dir = os.scandir(tmpdir)
|
||||
for deb in deb_dir:
|
||||
apt_utils.package_upload(package_repo_dir,
|
||||
os.path.join(tmpdir, deb.name))
|
||||
except OSError as e:
|
||||
msg = "Failed to scan %s for Debian packages. Error: %s" \
|
||||
% (package_repo_dir, e.errno)
|
||||
LOG.exception(msg)
|
||||
raise OSTreeTarFail(msg)
|
||||
finally:
|
||||
# Change back to original working dir
|
||||
os.chdir(orig_wd)
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
|
||||
try:
|
||||
apt_utils.run_install(feed_ostree, packages)
|
||||
except APTOSTreeCommandFail:
|
||||
LOG.exception("Failed to intall Debian package.")
|
||||
raise APTOSTreeCommandFail(msg)
|
||||
|
||||
# Update the feed ostree summary
|
||||
ostree_utils.update_repo_summary_file(feed_ostree)
|
||||
|
||||
try:
|
||||
# Move the release metadata to deploying dir
|
||||
deploystate = self.release_data.metadata[release]["state"]
|
||||
|
|
@ -2167,9 +2177,8 @@ class PatchController(PatchService):
|
|||
else:
|
||||
self.release_data.metadata[release]["state"] = constants.UNKNOWN
|
||||
|
||||
# Commit1 in release metadata.xml file represents the latest commit
|
||||
# after this release has been applied to the feed repo
|
||||
self.latest_feed_commit = self.release_data.contents[release]["commit1"]["commit"]
|
||||
# Get the latest commit after performing "apt-ostree install".
|
||||
self.latest_feed_commit = ostree_utils.get_feed_latest_commit(SW_VERSION)
|
||||
|
||||
with self.hosts_lock:
|
||||
self.interim_state[release] = list(self.hosts)
|
||||
|
|
|
|||
|
|
@ -356,6 +356,12 @@ class ReleaseData(object):
|
|||
for req_release in req.findall("req_patch_id"):
|
||||
self.metadata[release_id]["requires"].append(req_release.text)
|
||||
|
||||
self.metadata[release_id]["packages"] = []
|
||||
for req in root.findall("packages"):
|
||||
for deb in req.findall("deb"):
|
||||
self.metadata[release_id]["packages"].append(
|
||||
deb.text.split("_")[0])
|
||||
|
||||
self.contents[release_id] = {}
|
||||
|
||||
for content in root.findall("contents/ostree"):
|
||||
|
|
|
|||
Loading…
Reference in New Issue