starlingx
/
update
Code Issues Proposed changes

Debian: In-service patch implementation 

This commit enables installation of in-service patches in a
Debian env by doing a hot update to make the pending
deployment live.

Test:
1) Verify that "sw-patch upload" stages the restart scripts
   (if any) at /etc/patching/patch-scripts
2) Verify that "sw-patch delete" deletes the restart scripts
   (if any) from /etc/patching/patch-scripts
3) Verify restart scripts are executed after a patch is applied
4) Apply and install in-service patch

Story: 2009969
Task: 45585
Depends-On: https://review.opendev.org/c/849134
Signed-off-by: Jessica Castelino <jessica.castelino@windriver.com>
Change-Id: I93a39c0b4de73c5043592acaad1a62e17144d186
This commit is contained in:
Jessica Castelino 2022-06-09 14:24:53 +00:00
parent bf1451ff01
commit 80134ff8e6
8 changed files with 150 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
sw-patch/cgcs-patch/cgcs_patch/constants.py
View File

@ -46,6 +46,8 @@ OSTREE_REF = "starlingx"
OSTREE_REMOTE = "debian"
FEED_OSTREE_BASE_DIR = "/var/www/pages/feed"
SYSROOT_OSTREE = "/sysroot/ostree/repo"
OSTREE_BASE_DEPLOYMENT_DIR = "/ostree/deploy/debian/deploy/"
PATCH_SCRIPTS_STAGING_DIR = "/run/patching/patch-scripts"
ENABLE_DEV_CERTIFICATE_PATCH_IDENTIFIER = 'ENABLE_DEV_CERTIFICATE'

50
sw-patch/cgcs-patch/cgcs_patch/ostree_utils.py
View File

@ -5,6 +5,8 @@ SPDX-License-Identifier: Apache-2.0
"""
import logging
import os.path
import sh
import subprocess
from cgcs_patch import constants
@ -210,3 +212,51 @@ def create_deployment():
% (e.returncode, e.stderr.decode("utf-8"))
LOG.info(info_msg)
raise OSTreeCommandFail(msg)
def fetch_pending_deployment():
"""
Fetch the deployment ID of the pending deployment
:return: The deployment ID of the pending deployment
"""
cmd = "ostree admin status |grep pending |awk '{printf $2}'"
try:
output = subprocess.run(cmd, shell=True, check=True, capture_output=True)
except subprocess.CalledProcessError as e:
msg = "Failed to fetch ostree admin status."
info_msg = "OSTree Admin Status Error: return code: %s , Output: %s" \
% (e.returncode, e.stderr.decode("utf-8"))
LOG.info(info_msg)
raise OSTreeCommandFail(msg)
# Store the output of the above command in a string
pending_deployment = output.stdout.decode('utf-8')
return pending_deployment
def mount_new_deployment(deployment_dir):
"""
Unmount /usr and /etc from the file system and remount it to directory
<depoyment_dir>/usr and <depoyment_dir>/etc respectively
:param deployment_dir: a path on the filesystem which points to the pending
deployment
example: /ostree/deploy/debian/deploy/<deployment_id>
"""
try:
if os.path.ismount("/usr"):
sh.umount("-l", "/usr")
if os.path.ismount("/etc"):
sh.umount("-l", "/etc")
new_usr_mount_dir = "%s/usr" % (deployment_dir)
new_etc_mount_dir = "%s/etc" % (deployment_dir)
sh.mount("--bind", "-o", "ro,noatime", new_usr_mount_dir, "/usr")
sh.mount("--bind", "-o", "ro,noatime", new_etc_mount_dir, "/etc")
except sh.ErrorReturnCode as e:
msg = "Failed to re-mount /usr and /etc."
info_msg = "OSTree Deployment Mount Error: Output: %s" \
% (e.stderr.decode("utf-8"))
LOG.info(info_msg)
raise OSTreeCommandFail(msg)

17
sw-patch/cgcs-patch/cgcs_patch/patch_agent.py
View File

@ -239,6 +239,9 @@ class PatchMessageAgentInstallReq(messages.PatchMessage):
global pa
resp = PatchMessageAgentInstallResp()
if not self.force:
setflag(node_is_patched_rr_file)
if not os.path.exists(node_is_locked_file):
if self.force:
LOG.info("Installing on unlocked node, with force option")
@ -406,11 +409,9 @@ class PatchAgent(PatchService):
try:
# Create insvc patch directories
if os.path.exists(insvc_patch_scripts):
shutil.rmtree(insvc_patch_scripts, ignore_errors=True)
if os.path.exists(insvc_patch_flags):
shutil.rmtree(insvc_patch_flags, ignore_errors=True)
if not os.path.exists(insvc_patch_scripts):
os.makedirs(insvc_patch_scripts, 0o700)
if not os.path.exists(insvc_patch_flags):
os.makedirs(insvc_patch_flags, 0o700)
except Exception:
LOG.exception("Failed to create in-service patch directories")
@ -459,15 +460,19 @@ class PatchAgent(PatchService):
LOG.info("Disallowing patch-scripts. Treating as reboot-required")
setflag(node_is_patched_rr_file)
else:
LOG.info("Running in-service patch-scripts")
LOG.info("Mounting the new deployment")
try:
pending_deployment = ostree_utils.fetch_pending_deployment()
deployment_dir = constants.OSTREE_BASE_DEPLOYMENT_DIR + pending_deployment
ostree_utils.mount_new_deployment(deployment_dir)
LOG.info("Running in-service patch-scripts")
subprocess.check_output(run_insvc_patch_scripts_cmd, stderr=subprocess.STDOUT)
# Clear the node_is_patched flag, since we've handled it in-service
clearflag(node_is_patched_file)
self.node_is_patched = False
except subprocess.CalledProcessError as e:
LOG.exception("In-Service patch scripts failed")
LOG.exception("In-Service patch installation failed")
LOG.error("Command output: %s", e.output)
success = False

56
sw-patch/cgcs-patch/cgcs_patch/patch_controller.py
View File

@ -32,6 +32,7 @@ from cgcs_patch.patch_functions import committed_dir
from cgcs_patch.patch_functions import PatchFile
from cgcs_patch.patch_functions import package_dir
from cgcs_patch.patch_functions import repo_dir
from cgcs_patch.patch_functions import root_scripts_dir
from cgcs_patch.patch_functions import semantics_dir
from cgcs_patch.patch_functions import SW_VERSION
from cgcs_patch.patch_functions import root_package_dir
@ -833,6 +834,23 @@ class PatchController(PatchService):
ostree_tar_filename = "%s/%s-software.tar" % (ostree_tar_dir, patch_id)
return ostree_tar_filename
def delete_restart_script(self, patch_id):
'''
Deletes the restart script (if any) associated with the patch
:param patch_id: The patch ID
'''
if not self.patch_data.metadata[patch_id]["restart_script"]:
return
restart_script_path = "%s/%s" % (root_scripts_dir, self.patch_data.metadata[patch_id]["restart_script"])
try:
# Delete the metadata
os.remove(restart_script_path)
except OSError:
msg = "Failed to remove restart script for %s" % patch_id
LOG.exception(msg)
raise PatchError(msg)
def get_repo_filename(self, patch_sw_version, contentname):
contentfile = self.get_store_filename(patch_sw_version, contentname)
if not os.path.isfile(contentfile):
@ -1387,6 +1405,7 @@ class PatchController(PatchService):
LOG.exception(msg)
raise MetadataFail(msg)
self.delete_restart_script(patch_id)
self.patch_data.delete_patch(patch_id)
msg = "%s has been deleted" % patch_id
LOG.info(msg)
@ -1941,6 +1960,42 @@ class PatchController(PatchService):
return rc
def copy_restart_scripts(self):
with self.patch_data_lock:
for patch_id in self.patch_data.metadata:
if (self.patch_data.metadata[patch_id]["patchstate"] in
[constants.PARTIAL_APPLY, constants.PARTIAL_REMOVE]) \
and self.patch_data.metadata[patch_id]["restart_script"]:
try:
restart_script_name = self.patch_data.metadata[patch_id]["restart_script"]
restart_script_path = "%s/%s" \
% (root_scripts_dir, restart_script_name)
dest_path = constants.PATCH_SCRIPTS_STAGING_DIR
dest_script_file = "%s/%s" \
% (constants.PATCH_SCRIPTS_STAGING_DIR, restart_script_name)
if not os.path.exists(dest_path):
os.makedirs(dest_path, 0o700)
shutil.copyfile(restart_script_path, dest_script_file)
os.chmod(dest_script_file, 0o700)
msg = "Creating restart script for %s" % patch_id
LOG.info(msg)
except shutil.Error:
msg = "Failed to copy the restart script for %s" % patch_id
LOG.exception(msg)
raise PatchError(msg)
elif self.patch_data.metadata[patch_id]["restart_script"]:
try:
restart_script_name = self.patch_data.metadata[patch_id]["restart_script"]
restart_script_path = "%s/%s" \
% (constants.PATCH_SCRIPTS_STAGING_DIR, restart_script_name)
if os.path.exists(restart_script_path):
os.remove(restart_script_path)
msg = "Removing restart script for %s" % patch_id
LOG.info(msg)
except shutil.Error:
msg = "Failed to delete the restart script for %s" % patch_id
LOG.exception(msg)
def patch_host_install(self, host_ip, force, async_req=False):
msg_info = ""
msg_warning = ""
@ -1971,6 +2026,7 @@ class PatchController(PatchService):
if self.allow_insvc_patching:
LOG.info("Allowing in-service patching")
force = True
self.copy_restart_scripts()
self.hosts[ip].install_pending = True
self.hosts[ip].install_status = False

33
sw-patch/cgcs-patch/cgcs_patch/patch_functions.py
View File

@ -48,6 +48,7 @@ repo_root_dir = "/var/www/pages/updates"
repo_dir = {SW_VERSION: "%s/rel-%s" % (repo_root_dir, SW_VERSION)}
root_package_dir = "%s/packages" % patch_dir
root_scripts_dir = "/etc/patching/patch-scripts"
package_dir = {SW_VERSION: "%s/%s" % (root_package_dir, SW_VERSION)}
logfile = "/var/log/patching.log"
@ -335,6 +336,7 @@ class PatchData(object):
"summary",
"description",
"install_instructions",
"restart_script",
"warnings",
"apply_active_release_only"]:
value = root.findtext(key)
@ -615,19 +617,21 @@ class PatchFile(object):
# Open the patch file and extract the contents to the current dir
tar = tarfile.open(path, "r:gz")
filelist = ["metadata.tar", "software.tar"]
if "semantics.tar" in [f.name for f in tar.getmembers()]:
filelist.append("semantics.tar")
filelist = []
for f in tar.getmembers():
filelist.append(f.name)
if detached_signature_file not in filelist:
msg = "Patch not signed"
LOG.warning(msg)
for f in filelist:
tar.extract(f)
tar.extract("signature")
try:
tar.extract(detached_signature_file)
except KeyError:
msg = "Patch has not been signed"
LOG.warning(msg)
# Filelist used for signature validation and verification
sig_filelist = ["metadata.tar", "software.tar"]
if "semantics.tar" in filelist:
sig_filelist.append("semantics.tar")
# Verify the data integrity signature first
sigfile = open("signature", "r")
@ -635,7 +639,7 @@ class PatchFile(object):
sigfile.close()
expected_sig = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
for f in filelist:
for f in sig_filelist:
sig ^= get_md5(f)
if sig != expected_sig:
@ -646,7 +650,7 @@ class PatchFile(object):
# Verify detached signature
if os.path.exists(detached_signature_file):
sig_valid = verify_files(
filelist,
sig_filelist,
detached_signature_file,
cert_type=cert_type)
if sig_valid is True:
@ -850,6 +854,13 @@ class PatchFile(object):
shutil.move("software.tar",
"%s/%s-software.tar" % (abs_ostree_tar_dir, patch_id))
if thispatch.metadata[patch_id]["restart_script"]:
if not os.path.exists(root_scripts_dir):
os.makedirs(root_scripts_dir)
restart_script_name = thispatch.metadata[patch_id]["restart_script"]
shutil.move(restart_script_name,
"%s/%s" % (root_scripts_dir, restart_script_name))
except PatchValidationFailure as e:
raise e
except PatchMismatchFailure as e:

2
sw-patch/cgcs-patch/pylint.rc
View File

@ -293,7 +293,7 @@ zope=no
# List of members which are set dynamically and missed by pylint inference
# system, and so shouldn't trigger E0201 when accessed. Python regular
# expressions are accepted.
generated-members=REQUEST,acl_users,aq_parent
generated-members=REQUEST,acl_users,aq_parent,sh.*
[VARIABLES]

1
sw-patch/cgcs-patch/requirements.txt
View File

@ -8,3 +8,4 @@ pecan
pycryptodomex
lxml
requests_toolbelt
sh

6
sw-patch/debian/deb_folder/control
View File

@ -13,7 +13,8 @@ Build-Depends-Indep: python3-keystonemiddleware,
python3-stestr,
python3-testtools,
python3-six,
tsconfig
tsconfig,
python3-sh
Standards-Version: 4.4.1
Package: cgcs-patch
@ -45,6 +46,7 @@ Depends: ${python3:Depends},
python3-lxml,
python3-requests-toolbelt,
python3-six,
tsconfig
tsconfig,
python3-sh
Description: Starlingx platfom patching (python3)
Starlingx platform patching system python libraries