StarlingX Installation/Update/Patching/Backup/Restore
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

357 lines
12 KiB

# Copyright (c) 2022 Wind River Systems, Inc.
# SPDX-License-Identifier: Apache-2.0
Create a test patch for Debian
- fake restart script
- fake dettached signature (signature.v2)
- Requires the ostree tool installed - apt-get install ostree
e.g: export STX_BUILD_HOME=/localdisk/designer/lsampaio/stx-debian
- pip3 install pycryptodomex
Setup Steps:
sudo chmod 644 $STX_BUILD_HOME/localdisk/deploy/ostree_repo/.lock
sudo chmod 644 $STX_BUILD_HOME/localdisk/lat/std/deploy/ostree_repo/.lock
python --prepare --repo ostree_repo --clone-repo ostree-clone
Patch Steps:
<make some ostree changes>
rm -Rf $STX_BUILD_HOME/localdisk/deploy/delta_dir
rm -Rf $STX_BUILD_HOME/localdisk/lat/std/deploy/delta_dir
python --create --repo ostree_repo --clone-repo ostree-clone
import argparse
import hashlib
import logging
import tarfile
import tempfile
import os
import shutil
import subprocess
import sys
import xml.etree.ElementTree as ET
from xml.dom import minidom
sys.path.insert(0, "../cgcs-patch")
from cgcs_patch.patch_signing import sign_files
# ostree_repo location
DEPLOY_DIR = os.path.join(os.environ['STX_BUILD_HOME'], 'localdisk/lat/std/deploy')
OSTREE_REPO = os.path.join(DEPLOY_DIR, 'ostree_repo')
# Delta dir used by rsync, hardcoded for now
DELTA_DIR = 'delta_dir'
detached_signature_file = 'signature.v2'
format='%(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
log = logging.getLogger('make_test_patch')
def prepare_env(name='ostree-clone'):
Generates a copy of the current ostree_repo which is used
to create the delta dir during patch creation
:param name: name of the cloned directory
''''Preparing ostree clone directory')
clone_dir = os.path.join(DEPLOY_DIR, name)
if os.path.isdir(clone_dir):
log.error('Clone directory exists {}'.format(name))
current_sha = open(os.path.join(OSTREE_REPO, 'refs/heads/starlingx'), 'r').read()'Current SHA: {}'.format(current_sha))'Cloning the directory...')['rsync', '-a', OSTREE_REPO + '/', clone_dir])'Prepared ostree repo clone at {}'.format(clone_dir))
def create_delta_dir(delta_dir='delta_dir', clone_dir='ostree-clone', clean_mode=False):
Creates the ostree delta directory
Contains the changes from the REPO (updated) and the cloned dir (pre update)
:param delta_dir: delta directory name
:param clone_dir: clone dir name
''''Creating ostree delta')
clone_dir = os.path.join(DEPLOY_DIR, clone_dir)
if os.path.isdir(delta_dir):
if clean_mode:'Delta dir exists {}, cleaning it'.format(delta_dir))
log.error('Delta dir exists {}, clean it up and try again'.format(delta_dir))
if not os.path.isdir(clone_dir):
log.error('Clone dir not found')
exit(1)['rsync', '-rpgo', '--compare-dest', clone_dir, OSTREE_REPO + '/', delta_dir + '/'])'Delta dir created')
def add_text_tag_to_xml(parent,
Utility function for adding a text tag to an XML object
:param parent: Parent element
:param name: Element name
:param text: Text value
:return:The created element
tag = ET.SubElement(parent, name)
tag.text = text
return tag
def gen_xml(patch_id, ostree_content, file_name="metadata.xml"):
Generate patch metadata XML file
:param file_name: Path to output file
top = ET.Element("patch")
add_text_tag_to_xml(top, 'id', patch_id)
add_text_tag_to_xml(top, 'sw_version', SOFTWARE_VERSION)
add_text_tag_to_xml(top, 'summary', 'Summary text')
add_text_tag_to_xml(top, 'description', 'Description text')
add_text_tag_to_xml(top, 'install_instructions', 'Install instructions text')
add_text_tag_to_xml(top, 'warnings', 'Warnings text')
add_text_tag_to_xml(top, 'status', 'DEV')
add_text_tag_to_xml(top, 'unremovable', 'N')
add_text_tag_to_xml(top, 'reboot_required', 'Y')
add_text_tag_to_xml(top, 'apply_active_release_only', '')
add_text_tag_to_xml(top, 'restart_script', '')
# Parse ostree_content
content = ET.SubElement(top, 'contents')
ostree = ET.SubElement(content, 'ostree')
add_text_tag_to_xml(ostree, 'number_of_commits', str(len(ostree_content['commits'])))
base_commit = ET.SubElement(ostree, 'base')
add_text_tag_to_xml(base_commit, 'commit', ostree_content['base']['commit'])
add_text_tag_to_xml(base_commit, 'checksum', ostree_content['base']['checksum'])
for i, c in enumerate(ostree_content['commits']):
commit = ET.SubElement(ostree, 'commit' + str(i + 1))
add_text_tag_to_xml(commit, 'commit', c['commit'])
add_text_tag_to_xml(commit, 'checksum', c['checksum'])
add_text_tag_to_xml(top, 'requires', '')
add_text_tag_to_xml(top, 'semantics', '')
# print
outfile = open(file_name, 'w')
tree = ET.tostring(top)
outfile.write(minidom.parseString(tree).toprettyxml(indent=" "))
def gen_restart_script(file_name):
Generate restart script
:param file_name: Path to script file
# print
outfile = open(file_name, 'w')
r = 'echo test restart script'
def get_md5(path):
Utility function for generating the md5sum of a file
:param path: Path to file
md5 = hashlib.md5()
block_size = 8192
with open(path, 'rb') as f:
for chunk in iter(lambda:, b''):
return int(md5.hexdigest(), 16)
def get_commit_checksum(commit_id, repo='ostree_repo'):
Get commit checksum from a commit id
:param commit_id
:param repo
# get all checksums
cmd = 'ostree --repo={} log starlingx | grep -i checksum | sed \'s/.* //\''.format(repo)
cksums = subprocess.check_output(cmd, shell=True).decode(sys.stdout.encoding).strip().split('\n')
def get_commits_from_base(base_sha, repo='ostree_repo'):
Get a list of commits from base sha
:param base_sha
:param repo
commits_from_base = []
cmd = 'ostree --repo={} log starlingx | grep commit | sed \'s/.* //\''.format(repo)
commits = subprocess.check_output(cmd, shell=True).decode(sys.stdout.encoding).strip().split('\n')
if commits[0] == base_sha:'base and top commit are the same')
return commits_from_base
# find base and add the commits to the list
for i, c in enumerate(commits):
if c == base_sha:
break'saving commit {}'.format(c))
# find commit checksum
cksum = get_commit_checksum(i, repo)
'commit': c,
'checksum': cksum
return commits_from_base
def create_patch(patch_id, patch_file, repo='ostree_repo', clone_dir='ostree-clone', clean_mode=False):
Creates a debian patch using ostree delta between 2 repos (rsync)
:param repo: main ostree_repo where build-image adds new commits
:param clone_dir: repo cloned before the changes
# read the base sha from the clone
base_sha = open(os.path.join(clone_dir, 'refs/heads/starlingx'), 'r').read().strip()'Generating delta dir')
create_delta_dir(delta_dir=DELTA_DIR, clone_dir=clone_dir, clean_mode=clean_mode)
# ostree --repo=ostree_repo show starlingx | grep -i checksum | sed 's/.* //'
cmd = 'ostree --repo={} show starlingx | grep -i checksum | sed \'s/.* //\''.format(clone_dir)
base_checksum = subprocess.check_output(cmd, shell=True).decode(sys.stdout.encoding).strip()
commits = get_commits_from_base(base_sha, repo)
if commits:
ostree_content = {
'base': {
'commit': base_sha,
'checksum': base_checksum
ostree_content['commits'] = commits
else:'No changes detected')
exit(0)'Generating patch file...')
# Create software.tar, metadata.tar and signatures
# Create a temporary working directory
tmpdir = tempfile.mkdtemp(prefix='patch_')
# Change to the tmpdir
tar ='software.tar', 'w')
tar.add(os.path.join(DEPLOY_DIR, DELTA_DIR), arcname='')
tar.close'Generating xml with ostree content {}'.format(commits))
gen_xml(patch_id, ostree_content)
tar ='metadata.tar', 'w')
tar.close()'Saving restart scripts (if any)')
# TODO: verify how to handle the restart script
filelist = ['metadata.tar', 'software.tar']
# Generate the signature file
for f in filelist:
sig ^= get_md5(f)
sigfile = open('signature', 'w')
sigfile.write('%x' % sig)
# this comes from patch_functions write_patch
# Generate the detached signature
# Note: if cert_type requests a formal signature, but the signing key
# is not found, we'll instead sign with the 'dev' key and
# need_resign_with_formal is set to True.
need_resign_with_formal = sign_files(
# Create the patch
tar =, patch_file), 'w:gz')
for f in filelist:
shutil.rmtree(tmpdir)'Patch file created {} at {}'.format(patch_file, DEPLOY_DIR))
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Debian make_test_patch helper")
parser.add_argument('-r', '--repo', type=str,
help='Ostree repo name',
default=None, required=True)
parser.add_argument('-p', '--prepare', action='store_true',
help='Prepare the ostree_repo clone directory, should be executed before making changes to the environment')
parser.add_argument('-cr', '--clone-repo', type=str,
help='Clone repo directory name',
default=None, required=True)
parser.add_argument('-c', '--create', action='store_true',
help='Create patch, should be executed after changes are done to the environment')
parser.add_argument('-i', '--id', type=str,
help='Patch ID', default='PATCH_0001')
parser.add_argument('-cl', '--clean-mode', action='store_true',
help='Whether to clean the delta directory automatically')
args = parser.parse_args()'STX_BUILD_HOME: {}'.format(os.environ['STX_BUILD_HOME']))'DEPLOY DIR: {}'.format(DEPLOY_DIR))'DELTA DIR: {}'.format(DELTA_DIR))
patch_id =
patch_file = patch_id + '.patch'
if args.prepare:'Calling prepare environment')
elif args.create:'Calling create patch')
create_patch(patch_id, patch_file, args.repo, args.clone_repo, args.clean_mode)