diff --git a/software-client/software_client/v1/release.py b/software-client/software_client/v1/release.py index cc1bc153..e5199df4 100644 --- a/software-client/software_client/v1/release.py +++ b/software-client/software_client/v1/release.py @@ -258,12 +258,14 @@ class ReleaseManager(base.Manager): utils.print_software_op_result(req) return - def install_local(self): + def install_local(self, delete): # Ignore interrupts during this function signal.signal(signal.SIGINT, signal.SIG_IGN) + body = {} + body["delete"] = delete - path = "/v1/deploy/install_local" - return self._post(path) + path = "/v1/deploy_host/install_local" + return self._post(path, body=body) def release_delete(self, release_id): release_ids = "/".join(release_id) diff --git a/software-client/software_client/v1/release_shell.py b/software-client/software_client/v1/release_shell.py index 565cebcb..87ac55b4 100644 --- a/software-client/software_client/v1/release_shell.py +++ b/software-client/software_client/v1/release_shell.py @@ -80,12 +80,15 @@ def do_commit_patch(cc, args): return cc.release.commit_patch(args) +@utils.arg('--delete', + required=False, + action='store_true', + help='Delete patch install/remove on the localhost mode') def do_install_local(cc, args): - """Trigger patch install/remove on the local host. - This command can only be used for patch installation - prior to initial configuration. + """Set patch install/remove on the local host. + This command can only be used for patch installation. """ - resp, data = cc.release.install_local() + resp, data = cc.release.install_local(args.delete) if args.debug: utils.print_result_debug(resp, data) diff --git a/software/software/api/controllers/v1/deploy_host.py b/software/software/api/controllers/v1/deploy_host.py index bd0c4223..21d5e4ae 100644 --- a/software/software/api/controllers/v1/deploy_host.py +++ b/software/software/api/controllers/v1/deploy_host.py @@ -50,7 +50,7 @@ class DeployHostController(RestController): return result @expose(method='POST', template='json') - def install_local(self): + def install_local(self, delete): reload_release_data() - result = sc.software_install_local_api() + result = sc.software_install_local_api(delete) return result diff --git a/software/software/base.py b/software/software/base.py index f7fdd405..ad212d98 100644 --- a/software/software/base.py +++ b/software/software/base.py @@ -26,6 +26,7 @@ class PatchService(object): self.mcast_addr = None self.socket_lock = None self.pre_bootstrap = True + self.install_local = True def update_config(self): # Implemented in subclass diff --git a/software/software/constants.py b/software/software/constants.py index 5a66df8f..a462cbf7 100644 --- a/software/software/constants.py +++ b/software/software/constants.py @@ -172,3 +172,6 @@ BASE_TAG = "base" COMMIT_TAG = "commit" CHECKSUM_TAG = "checksum" COMMIT1_TAG = "commit1" + +# install local flag +INSTALL_LOCAL_FLAG = "/opt/software/.install_local" diff --git a/software/software/software_agent.py b/software/software/software_agent.py index 2fc25cc2..f467b8f5 100644 --- a/software/software/software_agent.py +++ b/software/software/software_agent.py @@ -51,7 +51,6 @@ run_install_software_scripts_cmd = "/usr/sbin/run-software-scripts" pa = None http_port_real = http_port -http_hostname = '127.0.0.1' def setflag(fname): @@ -70,18 +69,21 @@ def clearflag(fname): LOG.exception("Failed to clear %s flag", fname) -def pull_install_scripts_from_controller(): +def pull_install_scripts_from_controller(install_local=False): # If the rsync fails, it raises an exception to # the caller "handle_install()" and fails the # host-install request for this host. # The restart_scripts are optional, so if the files # are not present, it should not raise any exception + host = constants.CONTROLLER + if install_local: + host = '127.0.0.1' try: output = subprocess.check_output(["rsync", "-acv", "--delete", "--exclude", "tmp", - "rsync://%s/repo/software-scripts/" % http_hostname, + "rsync://%s/repo/software-scripts/" % host, "%s/" % insvc_software_scripts], stderr=subprocess.STDOUT) LOG.info("Synced restart scripts from controller: %s", output) @@ -94,7 +96,9 @@ def pull_install_scripts_from_controller(): def check_install_uuid(): - controller_install_uuid_url = "http://%s:%s/feed/rel-%s/install_uuid" % (http_hostname, http_port_real, SW_VERSION) + controller_install_uuid_url = "http://%s:%s/feed/rel-%s/install_uuid" % (constants.CONTROLLER, + http_port_real, + SW_VERSION) try: req = requests.get(controller_install_uuid_url) if req.status_code != 200: @@ -290,7 +294,8 @@ class PatchMessageAgentInstallReq(messages.PatchMessage): resp.reject_reason = 'Node must be locked.' resp.send(sock, addr) return - resp.status = pa.handle_install(major_release=self.major_release, commit_id=self.commit_id) + resp.status = pa.handle_install(major_release=self.major_release, + commit_id=self.commit_id) resp.send(sock, addr) def send(self, sock): # pylint: disable=unused-argument @@ -444,7 +449,7 @@ class PatchAgent(PatchService): def query(self, major_release=None): """Check current patch state """ - if not check_install_uuid(): + if not self.install_local and not check_install_uuid(): LOG.info("Failed install_uuid check. Skipping query") return False @@ -504,7 +509,7 @@ class PatchAgent(PatchService): # Check the INSTALL_UUID first. If it doesn't match the active # controller, we don't want to install patches. - if not check_install_uuid(): + if not self.install_local and not check_install_uuid(): LOG.error("Failed install_uuid check. Skipping install") self.patch_failed = True @@ -613,7 +618,7 @@ class PatchAgent(PatchService): os.path.exists(mount_pending_file): try: LOG.info("Running pre-install patch-scripts") - pull_install_scripts_from_controller() + pull_install_scripts_from_controller(install_local=self.install_local) subprocess.check_output([run_install_software_scripts_cmd, "preinstall"], stderr=subprocess.STDOUT) @@ -750,6 +755,7 @@ class PatchAgent(PatchService): def handle_bootstrap(self, connections): # If bootstrap is completed re-initialize sockets self.pre_bootstrap = False + self.install_local = False self.setup_socket() while self.sock_out is None: time.sleep(30) @@ -764,10 +770,13 @@ class PatchAgent(PatchService): def run(self): # Check if bootstrap stage is completed - global http_hostname if self.pre_bootstrap and cfg.get_mgmt_ip(): self.pre_bootstrap = False - http_hostname = constants.CONTROLLER_FLOATING_HOSTNAME + + if self.pre_bootstrap or os.path.isfile(constants.INSTALL_LOCAL_FLAG): + self.install_local = True + else: + self.install_local = False self.setup_socket() @@ -796,9 +805,13 @@ class PatchAgent(PatchService): while True: if self.pre_bootstrap and cfg.get_mgmt_ip(): self.handle_bootstrap(connections) - http_hostname = constants.CONTROLLER_FLOATING_HOSTNAME first_hello = True + if os.path.isfile(constants.INSTALL_LOCAL_FLAG) or self.pre_bootstrap: + self.install_local = True + else: + self.install_local = False + inputs = [self.sock_in, self.listener] + connections outputs = [] @@ -918,7 +931,13 @@ def main(): cfg.read_config() pa = PatchAgent() + if os.path.isfile(constants.INSTALL_LOCAL_FLAG): + pa.install_local = True + else: + pa.install_local = False + pa.query() + if os.path.exists(agent_running_after_reboot_flag): delete_older_deployments_flag = False else: @@ -928,7 +947,7 @@ def main(): if len(sys.argv) <= 1: pa.run() elif sys.argv[1] == "--install": - if not check_install_uuid(): + if not pa.install_local and not check_install_uuid(): # In certain cases, the lighttpd server could still be running using # its default port 80, as opposed to the port configured in platform.conf global http_port_real diff --git a/software/software/software_controller.py b/software/software/software_controller.py index c308bccf..07535cf9 100644 --- a/software/software/software_controller.py +++ b/software/software/software_controller.py @@ -31,9 +31,9 @@ from oslo_config import cfg as oslo_cfg import software.apt_utils as apt_utils import software.ostree_utils as ostree_utils -import software.software_functions as sf from software.api import app from software.authapi import app as auth_app +from software.constants import INSTALL_LOCAL_FLAG from software.states import DEPLOY_STATES from software.states import DEPLOY_HOST_STATES from software.base import PatchService @@ -96,7 +96,6 @@ import software.constants as constants from software import states from tsconfig.tsconfig import INITIAL_CONFIG_COMPLETE_FLAG -from tsconfig.tsconfig import INITIAL_CONTROLLER_CONFIG_COMPLETE from tsconfig.tsconfig import VOLATILE_CONTROLLER_CONFIG_COMPLETE import xml.etree.ElementTree as ET @@ -259,8 +258,6 @@ class PatchMessageHello(messages.PatchMessage): def handle(self, sock, addr): global sc host = addr[0] - if sc.pre_bootstrap: - return if host == cfg.get_mgmt_ip(): # Ignore messages from self return @@ -274,6 +271,8 @@ class PatchMessageHello(messages.PatchMessage): def send(self, sock): global sc + if sc.install_local: + return self.encode() message = json.dumps(self.message) sock.sendto(str.encode(message), (sc.controller_address, cfg.controller_port)) @@ -315,8 +314,6 @@ class PatchMessageSyncReq(messages.PatchMessage): def handle(self, sock, addr): global sc host = addr[0] - if sc.pre_bootstrap: - return if host == cfg.get_mgmt_ip(): # Ignore messages from self return @@ -383,7 +380,7 @@ class PatchMessageHelloAgent(messages.PatchMessage): self.encode() message = json.dumps(self.message) sock.sendto(str.encode(message), (sc.agent_address, cfg.agent_port)) - if not sc.pre_bootstrap: + if not sc.install_local: local_hostname = utils.ip_to_versioned_localhost(cfg.agent_mcast_group) sock.sendto(str.encode(message), (local_hostname, cfg.agent_port)) @@ -405,7 +402,7 @@ class PatchMessageSendLatestFeedCommit(messages.PatchMessage): self.encode() message = json.dumps(self.message) sock.sendto(str.encode(message), (sc.agent_address, cfg.agent_port)) - if not sc.pre_bootstrap: + if not sc.install_local: local_hostname = utils.ip_to_versioned_localhost(cfg.agent_mcast_group) sock.sendto(str.encode(message), (local_hostname, cfg.agent_port)) @@ -666,8 +663,6 @@ class PatchMessageDropHostReq(messages.PatchMessage): def handle(self, sock, addr): global sc host = addr[0] - if sc.pre_bootstrap: - return if host == cfg.get_mgmt_ip(): # Ignore messages from self return @@ -681,6 +676,8 @@ class PatchMessageDropHostReq(messages.PatchMessage): def send(self, sock): global sc + if sc.install_local: + return self.encode() message = json.dumps(self.message) sock.sendto(str.encode(message), (sc.controller_address, cfg.controller_port)) @@ -1051,9 +1048,6 @@ class PatchController(PatchService): self.sync_from_nbr(host) def sync_from_nbr(self, host): - if self.pre_bootstrap: - return True - # Sync the software repo host_url = utils.ip_to_url(host) try: @@ -1273,70 +1267,44 @@ class PatchController(PatchService): LOG.exception(msg) raise SoftwareFail(msg) - def software_install_local_api(self): + def software_install_local_api(self, delete): """ - Trigger patch installation prior to configuration + Enable patch installation to local controller :return: dict of info, warning and error messages """ msg_info = "" msg_warning = "" msg_error = "" - # Check to see if initial configuration has completed - if os.path.isfile(INITIAL_CONTROLLER_CONFIG_COMPLETE): - # Disallow the install - msg = "This command can only be used before initial system configuration." - LOG.exception(msg) - raise SoftwareServiceError(error=msg) + dbapi = get_instance() + deploy = dbapi.get_deploy_all() + if len(deploy) > 0: + msg_info += "Software Deploy operation is in progress.\n" + msg_info += "Please finish current deploy before modifying install local mode.\n" + return dict(info=msg_info, warning=msg_warning, error=msg_error) - update_hosts_file = False + if os.path.isfile(INSTALL_LOCAL_FLAG) and delete: + # Remove install local flag if enabled + if os.path.isfile(INSTALL_LOCAL_FLAG): + try: + os.remove(INSTALL_LOCAL_FLAG) + except Exception: + LOG.exception("Failed to clear %s flag", INSTALL_LOCAL_FLAG) + msg = "Software deployment in local installation mode is stopped" + msg_info += f"{msg}.\n" + LOG.info(msg) + return dict(info=msg_info, warning=msg_warning, error=msg_error) - # Check to see if the controller hostname is already known. - if not utils.gethostbyname(constants.CONTROLLER_FLOATING_HOSTNAME): - update_hosts_file = True - - # To allow software installation to occur before configuration, we need - # to alias controller to localhost - # There is a HOSTALIASES feature that would be preferred here, but it - # unfortunately requires dnsmasq to be running, which it is not at this point. - - if update_hosts_file: - # Make a backup of /etc/hosts - try: - shutil.copy2(ETC_HOSTS_FILE_PATH, ETC_HOSTS_BACKUP_FILE_PATH) - except Exception: - msg = f"Error occurred while copying {ETC_HOSTS_FILE_PATH}." - LOG.exception(msg) - raise SoftwareFail(msg) - - # Update /etc/hosts - with open(ETC_HOSTS_FILE_PATH, 'a') as f: - f.write("127.0.0.1 controller\n") - - # Run the software install - try: - # Use the restart option of the sw-patch init script, which will - # install patches but won't automatically reboot if the RR flag is set - subprocess.check_output(['/etc/init.d/sw-patch', 'restart']) - except subprocess.CalledProcessError: - msg = "Failed to install patches." - LOG.exception(msg) - raise SoftwareFail(msg) - - if update_hosts_file: - # Restore /etc/hosts - os.rename(ETC_HOSTS_BACKUP_FILE_PATH, ETC_HOSTS_FILE_PATH) - - for release in self.release_collection.iterate_releases(): - if release.state == states.DEPLOYING: - release.update_state(states.DEPLOYED) - elif release.state == states.REMOVING: - release.update_state(states.AVAILABLE) - - msg_info += "Software installation is complete.\n" - msg_info += "Please reboot before continuing with configuration." - - return dict(info=msg_info, warning=msg_warning, error=msg_error) + elif not delete and not os.path.isfile(INSTALL_LOCAL_FLAG): + open(INSTALL_LOCAL_FLAG, 'a').close() + msg = "Software deployment in local installation mode is started" + msg_info += f"{msg}.\n" + LOG.info(msg) + return dict(info=msg_info, warning=msg_warning, error=msg_error) + else: + mode = 'disabled' if delete else 'enabled' + msg_info += f"Software deployment in local installation mode is already {mode}.\n" + return dict(info=msg_info, warning=msg_warning, error=msg_error) def major_release_upload_check(self): """ @@ -1971,7 +1939,7 @@ class PatchController(PatchService): self.check_patch_states() - if self.sock_out is None: + if self.sock_out is None or self.install_local: return True # Send the sync requests @@ -1987,10 +1955,7 @@ class PatchController(PatchService): self.socket_lock.release() # Now we wait, up to two mins. future enhancement: Wait on a condition - if self.pre_bootstrap: - my_ip = utils.gethostbyname(constants.PREBOOTSTRAP_HOSTNAME) - else: - my_ip = cfg.get_mgmt_ip() + my_ip = cfg.get_mgmt_ip() sync_rc = False max_time = time.time() + 120 while time.time() < max_time: @@ -2610,6 +2575,16 @@ class PatchController(PatchService): feed_repo = "%s/rel-%s/ostree_repo" % (constants.FEED_OSTREE_BASE_DIR, deploy_sw_version) commit_id = deploy_release.commit_id + # Set hostname in case of local install + hostname = None + if self.pre_bootstrap: + hostname = constants.PREBOOTSTRAP_HOSTNAME + elif self.install_local: + hostname = socket.gethostname() + valid_hostnames = [constants.CONTROLLER_0_HOSTNAME, constants.CONTROLLER_1_HOSTNAME] + if hostname not in valid_hostnames: + LOG.warning("Using unknown hostname for local install: %s", hostname) + patch_release = True if utils.is_upgrade_deploy(SW_VERSION, deploy_release.sw_release): # TODO(bqian) remove default latest commit when a commit-id is built into GA metadata @@ -2678,8 +2653,8 @@ class PatchController(PatchService): # operation is "remove" with order [R4, R3] if operation == "apply": - collect_current_load_for_hosts(deploy_sw_version) - create_deploy_hosts() + collect_current_load_for_hosts(deploy_sw_version, hostname=hostname) + create_deploy_hosts(hostname=hostname) # reverse = True is used for apply operation deployment_list = self.release_apply_remove_order(deployment, running_release.sw_release, reverse=True) @@ -2734,7 +2709,7 @@ class PatchController(PatchService): # TODO(bqian) get the list of undeployed required release ids # i.e, when deploying 24.03.3, which requires 24.03.2 and 24.03.1, all # 3 release ids should be passed into to create new ReleaseState - collect_current_load_for_hosts(deploy_sw_version) + collect_current_load_for_hosts(deploy_sw_version, hostname=hostname) release_state = ReleaseState(release_ids=[release.id]) release_state.start_deploy() @@ -2773,8 +2748,8 @@ class PatchController(PatchService): latest_commit, feed_repo) elif operation == "remove": - collect_current_load_for_hosts(deploy_sw_version) - create_deploy_hosts() + collect_current_load_for_hosts(deploy_sw_version, hostname=hostname) + create_deploy_hosts(hostname=hostname) deployment_list = self.release_apply_remove_order(deployment, running_release.sw_version) msg = "Deploy start order for remove operation: %s" % ",".join(deployment_list) LOG.info(msg) @@ -2876,7 +2851,7 @@ class PatchController(PatchService): # TODO(bqian) get the list of undeployed required release ids # i.e, when deploying 24.03.3, which requires 24.03.2 and 24.03.1, all # 3 release ids should be passed into to create new ReleaseState - collect_current_load_for_hosts(deploy_sw_version) + collect_current_load_for_hosts(deploy_sw_version, hostname=hostname) release_state = ReleaseState(release_ids=[release.id]) release_state.start_remove() deploy_state = DeployState.get_instance() @@ -2980,6 +2955,17 @@ class PatchController(PatchService): release_state.available() + if os.path.isfile(INSTALL_LOCAL_FLAG): + # Remove install local flag if enabled + if os.path.isfile(INSTALL_LOCAL_FLAG): + try: + os.remove(INSTALL_LOCAL_FLAG) + except Exception: + msg_error = "Failed to clear install-local mode flag" + LOG.error(msg_error) + raise SoftwareServiceError(msg_error) + LOG.info("Software deployment in local installation mode is stopped") + if is_major_release: clean_up_deployment_data(major_release) msg_info += "Deploy deleted with success" @@ -3248,7 +3234,7 @@ class PatchController(PatchService): for release in self.release_collection.iterate_releases(): if to_release == release.sw_release: release_id = release.id - if not self.pre_bootstrap: + if not self.install_local: deploy_host_validations( hostname, is_major_release=self.release_collection.get_release_by_id(release_id).is_ga_release, @@ -3833,7 +3819,11 @@ class PatchControllerMainThread(threading.Thread): try: if sc.pre_bootstrap and cfg.get_mgmt_ip(): sc.pre_bootstrap = False - sf.pre_bootstrap_stage = False + + if sc.pre_bootstrap or os.path.isfile(INSTALL_LOCAL_FLAG): + sc.install_local = True + else: + sc.install_local = False # Update the out of sync alarm cache when the thread starts out_of_sync_alarm_fault = sc.get_out_of_sync_alarm() @@ -3885,7 +3875,6 @@ class PatchControllerMainThread(threading.Thread): # If bootstrap is completed re-initialize sockets if sc.pre_bootstrap and cfg.get_mgmt_ip(): sc.pre_bootstrap = False - sf.pre_bootstrap_stage = False sock_in = sc.setup_socket() while sock_in is None: @@ -3906,6 +3895,12 @@ class PatchControllerMainThread(threading.Thread): s.shutdown(socket.SHUT_RDWR) s.close() + local_mode = sc.pre_bootstrap or os.path.isfile(INSTALL_LOCAL_FLAG) + if local_mode and not sc.install_local: + sc.install_local = True + elif not local_mode and sc.install_local: + sc.install_local = False + inputs = [sc.sock_in] + agent_query_conns outputs = [] diff --git a/software/software/software_functions.py b/software/software/software_functions.py index 9c8a8e27..bff47062 100644 --- a/software/software/software_functions.py +++ b/software/software/software_functions.py @@ -68,7 +68,6 @@ auditLOG = logging.getLogger('audit_logger') audit_log_msg_prefix = 'User: sysadmin/admin Action: ' detached_signature_file = "signature.v2" -pre_bootstrap_stage = True def handle_exception(exc_type, exc_value, exc_traceback): @@ -1223,7 +1222,7 @@ def read_upgrade_support_versions(mounted_dir): return to_release, supported_from_releases -def create_deploy_hosts(): +def create_deploy_hosts(hostname=None): """ Create deploy-hosts entities based on hostnames from sysinv. @@ -1231,8 +1230,9 @@ def create_deploy_hosts(): db_api_instance = get_instance() db_api_instance.begin_update() try: - if pre_bootstrap_stage: - db_api_instance.create_deploy_host(constants.PREBOOTSTRAP_HOSTNAME) + # If hostname is passed (Eg. in case of install local) use that. + if hostname: + db_api_instance.create_deploy_host(hostname) else: for ihost in get_ihost_list(): db_api_instance.create_deploy_host(ihost.hostname) @@ -1244,14 +1244,15 @@ def create_deploy_hosts(): db_api_instance.end_update() -def collect_current_load_for_hosts(local_load): +def collect_current_load_for_hosts(local_load, hostname=None): load_data = { "current_loads": [] } try: - if pre_bootstrap_stage: + # If hostname is passed (Eg. in case of install local) use that. + if hostname: load_data["current_loads"].append({ - "hostname": constants.PREBOOTSTRAP_HOSTNAME, + "hostname": hostname, "running_release": local_load }) else: diff --git a/software/software/tests/test_software_controller_messages.py b/software/software/tests/test_software_controller_messages.py index a83375b6..a8e8a674 100644 --- a/software/software/tests/test_software_controller_messages.py +++ b/software/software/tests/test_software_controller_messages.py @@ -52,6 +52,7 @@ class FakeSoftwareController(object): self.base_pkgdata = mock.Mock() self.software_data = mock.Mock() self.pre_bootstrap = False + self.install_local = False def check_patch_states(self): pass