Merge "Fix: load-import not feed controller-1"
This commit is contained in:
commit
e2a0423bbd
@ -8,6 +8,7 @@ import configparser
|
|||||||
import gc
|
import gc
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import select
|
import select
|
||||||
import sh
|
import sh
|
||||||
import shutil
|
import shutil
|
||||||
@ -709,47 +710,91 @@ class PatchController(PatchService):
|
|||||||
LOG.error("Failed to rsync: %s", output)
|
LOG.error("Failed to rsync: %s", output)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
LOG.info("Starting feed sync")
|
||||||
|
# get the folders in .../feed remote controller
|
||||||
|
remote_feed_dirs = {}
|
||||||
try:
|
try:
|
||||||
for neighbour in list(self.hosts):
|
# The output is a string that lists the all folders and subfolders
|
||||||
if (self.hosts[neighbour].nodetype == "controller" and
|
# Example output:
|
||||||
self.hosts[neighbour].ip == host):
|
# b'. .. rel-22.12/ rel-22.12/subfolders rel-23.09/ rel-23.09/subfoldres'
|
||||||
LOG.info("Starting feed sync")
|
output = subprocess.check_output(["rsync",
|
||||||
# The output is a string that lists the directories
|
"-nav",
|
||||||
# Example output:
|
"--exclude='*'",
|
||||||
# >>> dir_names = sh.ls("/var/www/pages/feed/")
|
"rsync://%s/feed/" % host_url,
|
||||||
# >>> dir_names.stdout
|
"."], stderr=subprocess.STDOUT)
|
||||||
# b'rel-22.12 rel-22.5\n'
|
|
||||||
dir_names = sh.ls(constants.FEED_OSTREE_BASE_DIR)
|
|
||||||
|
|
||||||
# Convert the output above into a list that can be iterated
|
# Convert the output into a list filtered that can be iterated
|
||||||
# >>> list_of_dirs = dir_names.stdout.decode().rstrip().split()
|
# Example output:
|
||||||
# >>> print(list_of_dirs)
|
# ['rel-22.12', 'rel-23.09']
|
||||||
# ['rel-22.12', 'rel-22.5']
|
mate_folders = output.decode().splitlines()
|
||||||
|
pattern = r'^rel-\d{2}\.\d{2}/$'
|
||||||
|
remote_feed_dirs = [folder[:-1] for folder in mate_folders if re.match(pattern, folder)]
|
||||||
|
|
||||||
list_of_dirs = dir_names.stdout.decode("utf-8").rstrip().split()
|
LOG.info("Mate feed folders: %s", remote_feed_dirs)
|
||||||
|
|
||||||
for rel_dir in list_of_dirs:
|
|
||||||
feed_ostree = "%s/%s/ostree_repo/" % (constants.FEED_OSTREE_BASE_DIR, rel_dir)
|
|
||||||
if not os.path.isdir(feed_ostree):
|
|
||||||
LOG.info("Skipping feed dir %s", feed_ostree)
|
|
||||||
continue
|
|
||||||
LOG.info("Syncing %s", feed_ostree)
|
|
||||||
output = subprocess.check_output(["ostree",
|
|
||||||
"--repo=%s" % feed_ostree,
|
|
||||||
"pull",
|
|
||||||
"--depth=-1",
|
|
||||||
"--mirror",
|
|
||||||
"starlingx"],
|
|
||||||
stderr=subprocess.STDOUT)
|
|
||||||
output = subprocess.check_output(["ostree",
|
|
||||||
"summary",
|
|
||||||
"--update",
|
|
||||||
"--repo=%s" % feed_ostree],
|
|
||||||
stderr=subprocess.STDOUT)
|
|
||||||
LOG.info("Synced to mate feed via ostree pull: %s", output)
|
|
||||||
except subprocess.CalledProcessError:
|
except subprocess.CalledProcessError:
|
||||||
LOG.error("Failed to sync feed repo between controllers: %s", output)
|
LOG.error("Failed to get mate feed repositories: %s", output)
|
||||||
return False
|
return False
|
||||||
|
except Exception as e: # pylint: disable=broad-exception-caught
|
||||||
|
LOG.error("Failed to get feed subfolders: %s", e)
|
||||||
|
return False
|
||||||
|
|
||||||
|
# The output is a string that lists the directories
|
||||||
|
# Example output:
|
||||||
|
# >>> dir_names = sh.ls("/var/www/pages/feed/")
|
||||||
|
# >>> dir_names.stdout
|
||||||
|
# b'rel-22.12 rel-22.5\n'
|
||||||
|
dir_names = sh.ls(constants.FEED_OSTREE_BASE_DIR)
|
||||||
|
|
||||||
|
# Convert the output above into a list that can be iterated
|
||||||
|
# >>> list_of_dirs = dir_names.stdout.decode().rstrip().split()
|
||||||
|
# >>> print(list_of_dirs)
|
||||||
|
# ['rel-22.12', 'rel-22.5']
|
||||||
|
local_feed_dirs = dir_names.stdout.decode("utf-8").rstrip().split()
|
||||||
|
|
||||||
|
# get only the folders from remote not present in local and create them
|
||||||
|
missing_folders = [folder for folder in remote_feed_dirs if folder not in local_feed_dirs]
|
||||||
|
|
||||||
|
for folder in missing_folders:
|
||||||
|
LOG.info("Creating folder %s in %s", folder, constants.FEED_OSTREE_BASE_DIR)
|
||||||
|
try:
|
||||||
|
rel_dir_path = f"{constants.FEED_OSTREE_BASE_DIR}/{folder}/ostree_repo/"
|
||||||
|
cmd = ["mkdir", "-p", rel_dir_path]
|
||||||
|
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
|
||||||
|
cmd = ["ostree", f"--repo={rel_dir_path}", "init", "--mode=archive"]
|
||||||
|
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, cwd=rel_dir_path)
|
||||||
|
cmd = [
|
||||||
|
"ostree",
|
||||||
|
"remote",
|
||||||
|
"add",
|
||||||
|
"--no-gpg-verify", # TODO(lvieira) Remove this once gpg support is added.
|
||||||
|
"starlingx",
|
||||||
|
f"http://controller:8080/feed/{folder}/ostree_repo/"
|
||||||
|
]
|
||||||
|
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, cwd=rel_dir_path)
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
LOG.error("Failed create ostree directories: %s", output)
|
||||||
|
return False
|
||||||
|
|
||||||
|
LOG.info("Starting to sync from mate feed via ostree pull")
|
||||||
|
# This is syncing all folders, maybe it can be moved
|
||||||
|
# to above loop and sync only the new created folders
|
||||||
|
for rel_dir in remote_feed_dirs:
|
||||||
|
feed_ostree = "%s/%s/ostree_repo/" % (constants.FEED_OSTREE_BASE_DIR, rel_dir)
|
||||||
|
if not os.path.isdir(feed_ostree):
|
||||||
|
LOG.info("Skipping feed dir %s", feed_ostree)
|
||||||
|
continue
|
||||||
|
LOG.info("Syncing %s", feed_ostree)
|
||||||
|
output = subprocess.check_output(["ostree",
|
||||||
|
"--repo=%s" % feed_ostree,
|
||||||
|
"pull",
|
||||||
|
"--depth=-1",
|
||||||
|
"--mirror",
|
||||||
|
"starlingx"], stderr=subprocess.STDOUT)
|
||||||
|
output = subprocess.check_output(["ostree",
|
||||||
|
"summary",
|
||||||
|
"--update",
|
||||||
|
"--repo=%s" % feed_ostree], stderr=subprocess.STDOUT)
|
||||||
|
LOG.info("Synced to mate feed via ostree pull")
|
||||||
|
|
||||||
self.read_state_file()
|
self.read_state_file()
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user