diff --git a/bin/refresh.sh b/bin/refresh.sh index 890fdd75..51134ea3 100755 --- a/bin/refresh.sh +++ b/bin/refresh.sh @@ -1,14 +1,9 @@ #!/bin/bash -/opt/compass/bin/manage_db.py checkdb -if [[ "$?" == "0" ]]; then -/opt/compass/bin/manage_db.py clean_clusters -fi /opt/compass/bin/manage_db.py createdb -/opt/compass/bin/manage_db.py sync_switch_configs -/opt/compass/bin/manage_db.py sync_from_installers service httpd restart service rsyslog restart service redis restart redis-cli flushall -service compassd restart +service compass-celeryd restart +service compass-progress-updated restart diff --git a/compass/actions/poll_switch.py b/compass/actions/poll_switch.py index 968155fc..e4094544 100644 --- a/compass/actions/poll_switch.py +++ b/compass/actions/poll_switch.py @@ -19,6 +19,7 @@ import netaddr from compass.actions import util from compass.db.api import database from compass.db.api import switch as switch_api +from compass.db.api import user as user_api from compass.hdsdiscovery.hdmanager import HDManager @@ -71,18 +72,19 @@ def _poll_switch(ip_addr, credentials, req_obj='mac', oper="SCAN"): {} ) + logging.info('poll switch result: %s' % str(results)) state = under_monitoring machine_dicts = {} for machine in results: mac = machine['mac'] port = machine['port'] - vlan = machine['vlan'] + vlan = int(machine['vlan']) if vlan: vlans = [vlan] else: vlans = [] if mac not in machine_dicts: - machine_dicts[mac] = {'port': port, 'vlans': vlans} + machine_dicts[mac] = {'mac': mac, 'port': port, 'vlans': vlans} else: machine_dicts[mac]['port'] = port machine_dicts[mac]['vlans'].extend(vlans) @@ -90,11 +92,14 @@ def _poll_switch(ip_addr, credentials, req_obj='mac', oper="SCAN"): logging.debug('update switch %s state to under monitoring', ip_addr) return ( {'vendor': vendor, 'state': state, 'err_msg': err_msg}, - machine_dicts + machine_dicts.values() ) -def poll_switch(ip_addr, credentials, req_obj='mac', oper="SCAN"): +def poll_switch( + poller_email, ip_addr, credentials, + req_obj='mac', oper="SCAN" +): """Query switch and update switch machines. .. note:: @@ -113,6 +118,8 @@ def poll_switch(ip_addr, credentials, req_obj='mac', oper="SCAN"): .. note:: The function should be called out of database session scope. """ + poller = user_api.get_user_object(poller_email) + ip_int = long(netaddr.IPAddress(ip_addr)) with util.lock('poll switch %s' % ip_addr) as lock: if not lock: raise Exception( @@ -120,21 +127,22 @@ def poll_switch(ip_addr, credentials, req_obj='mac', oper="SCAN"): ) logging.debug('poll switch: %s', ip_addr) - ip_int = long(netaddr.IPAddress(ip_addr)) switch_dict, machine_dicts = _poll_switch( ip_addr, credentials, req_obj=req_obj, oper=oper ) - with database.session() as session: - switch = switch_api.get_switch_internal( - session, False, ip_int=ip_int - ) - if not switch: - logging.error('no switch found for %s', ip_addr) - return + switches = switch_api.list_switches( + poller, ip_int=ip_int + ) + if not switches: + logging.error('no switch found for %s', ip_addr) + return - switch_api.update_switch_internal( - session, switch, **switch_dict - ) - switch_api.add_switch_machines_internal( - session, switch, machine_dicts, False + for switch in switches: + switch_api.update_switch( + poller, switch['id'], **switch_dict ) + for machine_dict in machine_dicts: + print 'add machine: %s' % machine_dict + switch_api.add_switch_machine( + poller, switch['id'], False, **machine_dict + ) diff --git a/compass/api/api.py b/compass/api/api.py index 0230fd3f..f70f3ca5 100644 --- a/compass/api/api.py +++ b/compass/api/api.py @@ -159,13 +159,13 @@ def _login(use_cookie): raise exception_handler.BadRequest( 'missing email or password in data' ) - if 'expires' not in data: + if 'expire_timestamp' not in data: expire_timestamp = ( datetime.datetime.now() + app.config['REMEMBER_COOKIE_DURATION'] ) else: expire_timestamp = util.parse_datetime( - data['expires'], exception_handler.BadRequest + data['expire_timestamp'], exception_handler.BadRequest ) data['expire_timestamp'] = expire_timestamp @@ -559,7 +559,6 @@ def update_switch(switch_id): def patch_switch(switch_id): """patch switch.""" data = _get_request_data() - _replace_data(data, {'credentials': 'patched_credentials'}) return utils.make_json_response( 200, switch_api.patch_switch(current_user, switch_id, **data) @@ -622,7 +621,6 @@ def update_switch_filters(switch_id): def patch_switch_filters(switch_id): """patch switch filters.""" data = _get_request_data() - _replace_data(data, {'filters': 'patched_filters'}) return utils.make_json_response( 200, switch_api.patch_switch_filter(current_user, switch_id, **data) @@ -790,15 +788,6 @@ def update_switch_machine(switch_id, machine_id): def patch_switch_machine(switch_id, machine_id): """patch switch machine.""" data = _get_request_data() - _replace_data( - data, - { - 'vlans': 'patched_vlans', - 'ipmi_credentials': 'patched_ipmi_credentials', - 'tag': 'patched_tag', - 'location': 'patched_location' - } - ) return utils.make_json_response( 200, switch_api.patch_switch_machine( @@ -858,7 +847,6 @@ def list_switchmachines(): """List switch machines.""" data = _get_request_args() _filter_ip(data) - _replace_data(data, {'ip_int': 'switch_ip_int'}) _filter_port(data) _filter_general(data, 'vlans') _filter_tag(data) @@ -878,7 +866,6 @@ def list_switchmachines_hosts(): """List switch machines or hosts.""" data = _get_request_args() _filter_ip(data) - _replace_data(data, {'ip_int': 'switch_ip_int'}) _filter_port(data) _filter_general(data, 'vlans') _filter_tag(data) @@ -933,15 +920,6 @@ def update_switchmachine(switch_machine_id): def patch_switchmachine(switch_machine_id): """patch switch machine.""" data = _get_request_data() - _replace_data( - data, - { - 'vlans': 'patched_vlans', - 'ipmi_credentials': 'patched_ipmi_credentials', - 'tag': 'patched_tag', - 'location': 'patched_location' - } - ) return utils.make_json_response( 200, switch_api.patch_switchmachine( @@ -1014,14 +992,6 @@ def update_machine(machine_id): def patch_machine(machine_id): """patch machine.""" data = _get_request_data() - _replace_data( - data, - { - 'ipmi_credentials': 'patched_ipmi_credentials', - 'tag': 'patched_tag', - 'location': 'patched_location' - } - ) return utils.make_json_response( 200, machine_api.patch_machine( @@ -1288,13 +1258,6 @@ def show_cluster_metadata(cluster_id): def update_cluster_config(cluster_id): """update cluster config.""" data = _get_request_data() - _replace_data( - data, - { - 'os_config': 'put_os_config', - 'package_config': 'put_os_config' - } - ) return utils.make_json_response( 200, cluster_api.update_cluster_config(current_user, cluster_id, **data) @@ -1307,13 +1270,6 @@ def update_cluster_config(cluster_id): def patch_cluster_config(cluster_id): """patch cluster config.""" data = _get_request_data() - _replace_data( - data, - { - 'os_config': 'patched_os_config', - 'package_config': 'patched_package_config' - } - ) return utils.make_json_response( 200, cluster_api.patch_cluster_config(current_user, cluster_id, **data) @@ -1524,12 +1480,6 @@ def show_clusterhost_config(clusterhost_id): def update_cluster_host_config(cluster_id, host_id): """update clusterhost config.""" data = _get_request_data() - _replace_data( - data, - { - 'package_config': 'put_os_config' - } - ) return utils.make_json_response( 200, cluster_api.update_cluster_host_config( @@ -1544,12 +1494,6 @@ def update_cluster_host_config(cluster_id, host_id): def update_clusterhost_config(clusterhost_id): """update clusterhost config.""" data = _get_request_data() - _replace_data( - data, - { - 'package_config': 'put_os_config' - } - ) return utils.make_json_response( 200, cluster_api.update_clusterhost_config( @@ -1567,12 +1511,6 @@ def update_clusterhost_config(clusterhost_id): def patch_cluster_host_config(cluster_id, host_id): """patch clusterhost config.""" data = _get_request_data() - _replace_data( - data, - { - 'package_config': 'patched_package_config' - } - ) return utils.make_json_response( 200, cluster_api.patch_cluster_host_config( @@ -1587,12 +1525,6 @@ def patch_cluster_host_config(cluster_id, host_id): def patch_clusterhost_config(clusterhost_id): """patch clusterhost config.""" data = _get_request_data() - _replace_data( - data, - { - 'package_config': 'patched_package_config' - } - ) return utils.make_json_response( 200, cluster_api.patch_clusterhost_config( @@ -1816,12 +1748,6 @@ def show_host_config(host_id): def update_host_config(host_id): """update host config.""" data = _get_request_data() - _replace_data( - data, - { - 'os_config': 'put_os_config', - } - ) return utils.make_json_response( 200, host_api.update_host_config(current_user, host_id, **data) @@ -1834,12 +1760,6 @@ def update_host_config(host_id): def patch_host_config(host_id): """patch host config.""" data = _get_request_data() - _replace_data( - data, - { - 'os_config': 'patched_os_config', - } - ) return utils.make_json_response( 200, host_api.patch_host_config(current_user, host_id, **data) @@ -1882,16 +1802,19 @@ def list_hostnetworks(): ) -@app.route("/hosts//networks/", methods=['GET']) +@app.route( + "/hosts//networks/", + methods=['GET'] +) @log_user_action @login_required -def show_host_network(host_id, subnet_id): +def show_host_network(host_id, host_network_id): """Get host network.""" data = _get_request_args() return utils.make_json_response( 200, host_api.get_host_network( - current_user, host_id, subnet_id, **data + current_user, host_id, host_network_id, **data ) ) @@ -1921,16 +1844,19 @@ def add_host_network(host_id): ) -@app.route("/hosts//networks/", methods=['PUT']) +@app.route( + "/hosts//networks/", + methods=['PUT'] +) @log_user_action @login_required -def update_host_network(host_id, subnet_id): +def update_host_network(host_id, host_network_id): """update host network.""" data = _get_request_data() return utils.make_json_response( 200, host_api.update_host_network( - current_user, host_id, subnet_id, **data + current_user, host_id, host_network_id, **data ) ) @@ -1950,18 +1876,18 @@ def update_hostnetwork(host_network_id): @app.route( - "/hosts//networks/", + "/hosts//networks/", methods=['DELETE'] ) @log_user_action @login_required -def delete_host_network(host_id, subnet_id): +def delete_host_network(host_id, host_network_id): """Delete host network.""" data = _get_request_data() return utils.make_json_response( 200, host_api.del_host_network( - current_user, host_id, subnet_id, **data + current_user, host_id, host_network_id, **data ) ) diff --git a/compass/db/api/adapter.py b/compass/db/api/adapter.py index a4db88ba..1dd792b5 100644 --- a/compass/db/api/adapter.py +++ b/compass/db/api/adapter.py @@ -73,14 +73,14 @@ def add_adapters_internal(session): if 'OS_INSTALLER' in config: os_installer = utils.get_db_object( session, models.OSInstaller, - name=config['OS_INSTALLER'] + instance_name=config['OS_INSTALLER'] ) else: os_installer = None if 'PACKAGE_INSTALLER' in config: package_installer = utils.get_db_object( session, models.PackageInstaller, - name=config['PACKAGE_INSTALLER'] + instance_name=config['PACKAGE_INSTALLER'] ) else: package_installer = None diff --git a/compass/db/api/adapter_holder.py b/compass/db/api/adapter_holder.py index 95024c54..38a14699 100644 --- a/compass/db/api/adapter_holder.py +++ b/compass/db/api/adapter_holder.py @@ -26,19 +26,17 @@ from compass.db import exception SUPPORTED_FIELDS = [ 'name', 'distributed_system_name', - 'os_installer_name', - 'package_installer_name', ] RESP_FIELDS = [ - 'id', 'name', 'roles', 'os_installer_name', - 'package_installer_name', 'distributed_system_name', + 'id', 'name', 'roles', + 'distributed_system_name', 'supported_oses', 'display_name' ] RESP_OS_FIELDS = [ 'id', 'os_id', 'name' ] RESP_ROLES_FIELDS = [ - 'id', 'name', 'description', 'optional' + 'id', 'name', 'display_name', 'description', 'optional' ] diff --git a/compass/db/api/cluster.py b/compass/db/api/cluster.py index 3f7161fb..8804c0dc 100644 --- a/compass/db/api/cluster.py +++ b/compass/db/api/cluster.py @@ -8,11 +8,13 @@ # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Cluster database operations.""" +import functools import logging from compass.db.api import database @@ -37,7 +39,8 @@ RESP_FIELDS = [ 'created_at', 'updated_at' ] RESP_CLUSTERHOST_FIELDS = [ - 'id', 'host_id', 'machine_id', 'name', 'cluster_id', + 'id', 'host_id', 'machine_id', 'name', 'hostname', + 'cluster_id', 'clustername', 'mac', 'os_installed', 'distributed_system_installed', 'os_name', 'distributed_system_name', 'reinstall_os', 'reinstall_distributed_system', @@ -52,6 +55,12 @@ RESP_CONFIG_FIELDS = [ 'created_at', 'updated_at' ] +RESP_DEPLOYED_CONFIG_FIELDS = [ + 'deployed_os_config', + 'deployed_package_config', + 'created_at', + 'updated_at' +] RESP_METADATA_FIELDS = [ 'os_config', 'package_config' @@ -65,12 +74,19 @@ RESP_CLUSTERHOST_CONFIG_FIELDS = [ 'created_at', 'updated_at' ] +RESP_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS = [ + 'deployed_os_config', + 'deployed_package_config', + 'created_at', + 'updated_at' +] RESP_STATE_FIELDS = [ - 'id', 'state', 'progress', 'message', + 'id', 'state', 'percentage', 'message', + 'status', 'created_at', 'updated_at' ] RESP_CLUSTERHOST_STATE_FIELDS = [ - 'id', 'state', 'progress', 'message', + 'id', 'state', 'percentage', 'message', 'created_at', 'updated_at' ] RESP_REVIEW_FIELDS = [ @@ -87,17 +103,29 @@ UPDATED_HOST_FIELDS = ['name', 'reinstall_os'] UPDATED_CONFIG_FIELDS = [ 'put_os_config', 'put_package_config', 'config_step' ] +UPDATED_DEPLOYED_CONFIG_FIELDS = [ + 'deployed_os_config', 'deployed_package_config' +] PATCHED_CONFIG_FIELDS = [ 'patched_os_config', 'patched_package_config', 'config_step' ] UPDATED_CLUSTERHOST_CONFIG_FIELDS = [ + 'put_os_config', 'put_package_config' ] PATCHED_CLUSTERHOST_CONFIG_FIELDS = [ + 'patched_os_config', 'patched_package_config' ] +UPDATED_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS = [ + 'deployed_os_config', + 'deployed_package_config' +] UPDATED_CLUSTERHOST_STATE_FIELDS = [ - 'state', 'progress', 'message' + 'state', 'percentage', 'message' +] +UPDATED_CLUSTER_STATE_FIELDS = [ + 'state' ] @@ -120,10 +148,13 @@ def list_clusters(session, lister, **filters): permission.PERMISSION_LIST_CLUSTERS ) @utils.wrap_to_dict(RESP_FIELDS) -def get_cluster(session, getter, cluster_id, **kwargs): +def get_cluster( + session, getter, cluster_id, + exception_when_missing=True, **kwargs +): """Get cluster info.""" return utils.get_db_object( - session, models.Cluster, id=cluster_id + session, models.Cluster, exception_when_missing, id=cluster_id ) @@ -143,16 +174,15 @@ def is_cluster_validated( raise exception.Forbidden( 'cluster %s is not validated' % cluster.name ) - for clusterhost in cluster.clusterhsots: - if not clusterhost.config_validated: - raise exception.Forbidden( - 'clusterhost %s is not validated' % clusterhost.name - ) - host = clusterhost.host - if not host.config_validated: - raise exception.Forbidden( - 'host %s is not validated' % host.name - ) + + +def is_clusterhost_validated( + session, clusterhost +): + if not clusterhost.config_validated: + raise exception.Forbidden( + 'clusterhost %s is not validated' % clusterhost.name + ) def is_cluster_editable( @@ -165,7 +195,10 @@ def is_cluster_editable( return _conditional_exception( cluster, exception_when_not_editable ) - elif not cluster.reinstall_distributed_system: + elif ( + cluster.distributed_system and + not cluster.reinstall_distributed_system + ): return _conditional_exception( cluster, exception_when_not_editable ) @@ -184,11 +217,15 @@ def is_cluster_editable( permission.PERMISSION_ADD_CLUSTER ) @utils.wrap_to_dict(RESP_FIELDS) -def add_cluster(session, creator, name, adapter_id, **kwargs): +def add_cluster( + session, creator, + exception_when_existing=True, + name=None, **kwargs +): """Create a cluster.""" return utils.add_db_object( - session, models.Cluster, True, - name, creator_id=creator.id, adapter_id=adapter_id, + session, models.Cluster, exception_when_existing, + name, creator_id=creator.id, **kwargs ) @@ -241,6 +278,19 @@ def get_cluster_config(session, getter, cluster_id, **kwargs): ) +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission_in_session( + permission.PERMISSION_LIST_CLUSTER_CONFIG +) +@utils.wrap_to_dict(RESP_DEPLOYED_CONFIG_FIELDS) +def get_cluster_deployed_config(session, getter, cluster_id, **kwargs): + """Get cluster deployed config.""" + return utils.get_db_object( + session, models.Cluster, id=cluster_id + ) + + @utils.supported_filters([]) @database.run_in_session() @user_api.check_user_permission_in_session( @@ -270,25 +320,40 @@ def get_cluster_metadata(session, getter, cluster_id, **kwargs): permission.PERMISSION_ADD_CLUSTER_CONFIG ) @utils.wrap_to_dict(RESP_CONFIG_FIELDS) -def update_cluster_config_internal(session, updater, cluster, **kwargs): +def _update_cluster_config(session, updater, cluster, **kwargs): """Update a cluster config.""" is_cluster_editable(session, cluster, updater) - utils.update_db_object( + return utils.update_db_object( session, cluster, config_validated=False, **kwargs ) - os_config = cluster.os_config - if os_config: - metadata_api.validate_os_config( - os_config, cluster.os_id - ) - package_config = cluster.package_config - if package_config: - metadata_api.validate_package_config( - package_config, cluster.adapter_id - ) - return cluster +@utils.supported_filters( + optional_support_keys=UPDATED_DEPLOYED_CONFIG_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission_in_session( + permission.PERMISSION_ADD_CLUSTER_CONFIG +) +@utils.wrap_to_dict(RESP_DEPLOYED_CONFIG_FIELDS) +def update_cluster_deployed_config( + session, updater, cluster_id, **kwargs +): + """Update cluster deployed config.""" + cluster = utils.get_db_object( + session, models.Cluster, id=cluster_id + ) + is_cluster_editable(session, cluster, updater) + is_cluster_validated(session, cluster) + return utils.update_db_object( + session, cluster, **kwargs + ) + + +@utils.replace_filters( + os_config='put_os_config', + package_config='put_package_config' +) @utils.supported_filters(optional_support_keys=UPDATED_CONFIG_FIELDS) @database.run_in_session() def update_cluster_config(session, updater, cluster_id, **kwargs): @@ -296,11 +361,31 @@ def update_cluster_config(session, updater, cluster_id, **kwargs): cluster = utils.get_db_object( session, models.Cluster, id=cluster_id ) - return update_cluster_config_internal( - session, updater, cluster, **kwargs + os_config_validates = functools.partial( + metadata_api.validate_os_config, os_id=cluster.os_id) + package_config_validates = functools.partial( + metadata_api.validate_package_config, adapter_id=cluster.adapter_id) + + @utils.input_validates( + put_os_config=os_config_validates, + put_package_config=package_config_validates + ) + def update_config_internal( + cluster, **in_kwargs + ): + return _update_cluster_config( + session, updater, cluster, **in_kwargs + ) + + return update_config_internal( + cluster, **kwargs ) +@utils.replace_filters( + os_config='patched_os_config', + package_config='patched_package_config' +) @utils.supported_filters(optional_support_keys=PATCHED_CONFIG_FIELDS) @database.run_in_session() def patch_cluster_config(session, updater, cluster_id, **kwargs): @@ -308,8 +393,23 @@ def patch_cluster_config(session, updater, cluster_id, **kwargs): cluster = utils.get_db_object( session, models.Cluster, id=cluster_id ) - return update_cluster_config_internal( - session, updater, cluster, **kwargs + + os_config_validates = functools.partial( + metadata_api.validate_os_config, os_id=cluster.os_id) + package_config_validates = functools.partial( + metadata_api.validate_package_config, adapter_id=cluster.adapter_id) + + @utils.output_validates( + os_config=os_config_validates, + package_config=package_config_validates + ) + def update_config_internal(cluster, **in_kwargs): + return _update_cluster_config( + session, updater, cluster, **in_kwargs + ) + + return update_config_internal( + cluster, **kwargs ) @@ -363,32 +463,32 @@ def add_clusterhost_internal( creator=cluster.creator, **kwargs ) - return utils.add_db_object( + utils.add_db_object( session, models.ClusterHost, exception_when_existing, cluster.id, machine_id ) -def _add_clusterhosts(session, cluster, machine_dicts): - for machine_dict in machine_dicts: +def _add_clusterhosts(session, cluster, machines): + for machine_dict in machines: add_clusterhost_internal( session, cluster, **machine_dict ) -def _remove_clusterhosts(session, cluster, host_ids): +def _remove_clusterhosts(session, cluster, hosts): utils.del_db_objects( session, models.ClusterHost, - cluster_id=cluster.id, host_id=host_ids + cluster_id=cluster.id, host_id=hosts ) -def _set_clusterhosts(session, cluster, machine_dicts): +def _set_clusterhosts(session, cluster, machines): utils.del_db_objects( session, models.ClusterHost, cluster_id=cluster.id ) - for machine_dict in machine_dicts: + for machine_dict in machines: add_clusterhost_internal( session, cluster, True, **machine_dict ) @@ -427,10 +527,14 @@ def list_clusterhosts(session, lister, **filters): permission.PERMISSION_LIST_CLUSTERHOSTS ) @utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS) -def get_cluster_host(session, getter, cluster_id, host_id, **kwargs): +def get_cluster_host( + session, getter, cluster_id, host_id, + exception_when_missing=True, **kwargs +): """Get clusterhost info.""" return utils.get_db_object( session, models.ClusterHost, + exception_when_missing, cluster_id=cluster_id, host_id=host_id ) @@ -441,10 +545,15 @@ def get_cluster_host(session, getter, cluster_id, host_id, **kwargs): permission.PERMISSION_LIST_CLUSTERHOSTS ) @utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS) -def get_clusterhost(session, getter, clusterhost_id, **kwargs): +def get_clusterhost( + session, getter, clusterhost_id, + exception_when_missing=True, **kwargs +): """Get clusterhost info.""" return utils.get_db_object( - session, models.ClusterHost, id=clusterhost_id + session, models.ClusterHost, + exception_when_missing, + id=clusterhost_id ) @@ -457,14 +566,17 @@ def get_clusterhost(session, getter, clusterhost_id, **kwargs): permission.PERMISSION_UPDATE_CLUSTER_HOSTS ) @utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS) -def add_cluster_host(session, creator, cluster_id, machine_id, **kwargs): +def add_cluster_host( + session, creator, cluster_id, + exception_when_existing=True, **kwargs +): """Add cluster host.""" cluster = utils.get_db_object( session, models.Cluster, id=cluster_id ) return add_clusterhost_internal( - session, cluster, True, - machine_id=machine_id, **kwargs + session, cluster, exception_when_existing, + **kwargs ) @@ -516,6 +628,22 @@ def get_cluster_host_config(session, getter, cluster_id, host_id, **kwargs): ) +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission_in_session( + permission.PERMISSION_LIST_CLUSTERHOST_CONFIG +) +@utils.wrap_to_dict(RESP_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS) +def get_cluster_host_deployed_config( + session, getter, cluster_id, host_id, **kwargs +): + """Get clusterhost deployed config.""" + return utils.get_db_object( + session, models.ClusterHost, + cluster_id=cluster_id, host_id=host_id + ) + + @utils.supported_filters([]) @database.run_in_session() @user_api.check_user_permission_in_session( @@ -529,28 +657,107 @@ def get_clusterhost_config(session, getter, clusterhost_id, **kwargs): ) +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission_in_session( + permission.PERMISSION_LIST_CLUSTERHOST_CONFIG +) +@utils.wrap_to_dict(RESP_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS) +def get_clusterhost_deployed_config(session, getter, clusterhost_id, **kwargs): + """Get clusterhost deployed config.""" + return utils.get_db_object( + session, models.ClusterHost, id=clusterhost_id + ) + + @user_api.check_user_permission_in_session( permission.PERMISSION_ADD_CLUSTERHOST_CONFIG ) @utils.wrap_to_dict(RESP_CLUSTERHOST_CONFIG_FIELDS) -def update_clusterhost_config_internal( +def _update_clusterhost_config(session, updater, clusterhost, **kwargs): + from compass.db.api import host as host_api + ignore_keys = [] + if host_api.is_host_editable( + session, clusterhost.host, updater, + exception_when_not_editable=False + ): + ignore_keys.append('put_os_config') + + def os_config_validates(os_config): + from compass.db.api import host as host_api + host = clusterhost.host + metadata_api.validate_os_config(os_config, host.os_id) + + def package_config_validates(package_config): + cluster = clusterhost.cluster + is_cluster_editable(session, cluster, updater) + metadata_api.validate_package_config( + package_config, cluster.adapter_id + ) + + @utils.supported_filters( + optional_support_keys=UPDATED_CLUSTERHOST_CONFIG_FIELDS, + ignore_support_keys=ignore_keys + ) + @utils.input_validates( + put_os_config=os_config_validates, + put_package_config=package_config_validates + ) + def update_config_internal(clusterihost, **in_kwargs): + return utils.update_db_object( + session, clusterhost, **in_kwargs + ) + + return update_config_internal( + clusterhost, **kwargs + ) + + +@user_api.check_user_permission_in_session( + permission.PERMISSION_ADD_CLUSTERHOST_CONFIG +) +@utils.wrap_to_dict(RESP_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS) +def _update_clusterhost_deployed_config( session, updater, clusterhost, **kwargs ): - """Update clusterhost config internal.""" - is_cluster_editable(session, clusterhost.cluster, updater) - utils.update_db_object( - session, clusterhost, config_validated=False, **kwargs + from compass.db.api import host as host_api + ignore_keys = [] + if host_api.is_host_editable( + session, clusterhost.host, updater, + exception_when_not_editable=False + ): + ignore_keys.append('deployed_os_config') + + def os_config_validates(os_config): + host = clusterhost.host + host_api.is_host_validated(session, host) + + def package_config_validates(package_config): + cluster = clusterhost.cluster + is_cluster_editable(session, cluster, updater) + is_clusterhost_validated(session, clusterhost) + + @utils.supported_filters( + optional_support_keys=UPDATED_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS, + ignore_support_keys=ignore_keys ) - package_config = clusterhost.package_config - if package_config: - metadata_api.validate_package_config( - package_config, clusterhost.cluster.adapter_id + @utils.input_validates( + deployed_os_config=os_config_validates, + deployed_package_config=package_config_validates + ) + def update_config_internal(clusterhost, **in_kwargs): + return utils.update_db_object( + session, clusterhost, **in_kwargs ) - return clusterhost + + return update_config_internal( + clusterhost, **kwargs + ) -@utils.supported_filters( - optional_support_keys=UPDATED_CLUSTERHOST_CONFIG_FIELDS +@utils.replace_filters( + os_config='put_os_config', + package_config='put_package_config' ) @database.run_in_session() def update_cluster_host_config( @@ -561,13 +768,32 @@ def update_cluster_host_config( session, models.ClusterHost, cluster_id=cluster_id, host_id=host_id ) - return update_clusterhost_config_internal( + return _update_clusterhost_config( session, updater, clusterhost, **kwargs ) -@utils.supported_filters( - optional_support_keys=UPDATED_CLUSTERHOST_CONFIG_FIELDS +@utils.replace_filters( + os_config='deployed_os_config', + package_config='deployed_package_config' +) +@database.run_in_session() +def update_cluster_host_depolyed_config( + session, updater, cluster_id, host_id, **kwargs +): + """Update clusterhost deployed config.""" + clusterhost = utils.get_db_object( + session, models.ClusterHost, + cluster_id=cluster_id, host_id=host_id + ) + return _update_clusterhost_deployed_config( + session, updater, clusterhost, **kwargs + ) + + +@utils.replace_filters( + os_config='put_os_config', + package_config='put_package_config' ) @database.run_in_session() def update_clusterhost_config( @@ -577,12 +803,74 @@ def update_clusterhost_config( clusterhost = utils.get_db_object( session, models.ClusterHost, id=clusterhost_id ) - return update_clusterhost_config_internal( + return _update_clusterhost_config( session, updater, clusterhost, **kwargs ) -@utils.supported_filters(PATCHED_CLUSTERHOST_CONFIG_FIELDS) +@utils.replace_filters( + os_config='deployed_os_config', + package_config='deployed_package_config' +) +@database.run_in_session() +def update_clusterhost_deployed_config( + session, updater, clusterhost_id, **kwargs +): + """Update clusterhost deployed config.""" + clusterhost = utils.get_db_object( + session, models.ClusterHost, id=clusterhost_id + ) + return _update_clusterhost_deployed_config( + session, updater, clusterhost, **kwargs + ) + + +@user_api.check_user_permission_in_session( + permission.PERMISSION_ADD_CLUSTERHOST_CONFIG +) +@utils.wrap_to_dict(RESP_CLUSTERHOST_CONFIG_FIELDS) +def _patch_clusterhost_config(session, updater, clusterhost, **kwargs): + from compass.db.api import host as host_api + ignore_keys = [] + if host_api.is_host_editable( + session, clusterhost.host, updater, + exception_when_not_editable=False + ): + ignore_keys.append('patched_os_config') + + def os_config_validates(os_config): + host = clusterhost.host + metadata_api.validate_os_config(os_config, host.os_id) + + def package_config_validates(package_config): + cluster = clusterhost.cluster + is_cluster_editable(session, cluster, updater) + metadata_api.validate_package_config( + package_config, cluster.adapter_id + ) + + @utils.supported_filters( + optional_support_keys=PATCHED_CLUSTERHOST_CONFIG_FIELDS, + ignore_support_keys=ignore_keys + ) + @utils.output_validates( + os_config=os_config_validates, + package_config=package_config_validates + ) + def update_config_internal(clusterhost, **in_kwargs): + return _update_cluster_config( + session, updater, clusterhost, **in_kwargs + ) + + return update_config_internal( + clusterhost, **kwargs + ) + + +@utils.replace_filters( + os_config='patched_os_config', + package_config='patched_package_config' +) @database.run_in_session() def patch_cluster_host_config( session, updater, cluster_id, host_id, **kwargs @@ -592,12 +880,15 @@ def patch_cluster_host_config( session, models.ClusterHost, cluster_id=cluster_id, host_id=host_id ) - return update_clusterhost_config_internal( + return _patch_clusterhost_config( session, updater, clusterhost, **kwargs ) -@utils.supported_filters(PATCHED_CLUSTERHOST_CONFIG_FIELDS) +@utils.replace_filters( + os_config='patched_os_config', + package_config='patched_package_config' +) @database.run_in_session() def patch_clusterhost_config( session, updater, clusterhost_id, **kwargs @@ -606,28 +897,59 @@ def patch_clusterhost_config( clusterhost = utils.get_db_object( session, models.ClusterHost, id=clusterhost_id ) - return update_clusterhost_config_internal( + return _patch_clusterhost_config( session, updater, clusterhost, **kwargs ) +@user_api.check_user_permission_in_session( + permission.PERMISSION_DEL_CLUSTERHOST_CONFIG +) +@utils.wrap_to_dict(RESP_CLUSTERHOST_CONFIG_FIELDS) +def _delete_clusterhost_config( + session, deleter, clusterhost +): + from compass.db.api import host as host_api + ignore_keys = [] + if host_api.is_host_editable( + session, clusterhost.host, deleter, + exception_when_not_editable=False + ): + ignore_keys.append('os_config') + + def package_config_validates(package_config): + is_cluster_editable(session, clusterhost.cluster, deleter) + + @utils.supported_filters( + optional_support_keys=['os_config', 'package_config'], + ignore_support_keys=ignore_keys + ) + @utils.output_validates( + package_config=package_config_validates + ) + def update_config_internal(clusterhost, **in_kwargs): + return utils.update_db_object( + session, clusterhost, **in_kwargs + ) + + return update_config_internal( + clusterhost, os_config={}, + package_config={} + ) + + @utils.supported_filters([]) @database.run_in_session() -@user_api.check_user_permission_in_session( - permission.PERMISSION_DEL_CLUSTERHOST_CONFIG -) -@utils.wrap_to_dict(RESP_CLUSTERHOST_CONFIG_FIELDS) def delete_cluster_host_config( session, deleter, cluster_id, host_id ): """Delete a clusterhost config.""" clusterhost = utils.get_db_object( session, models.ClusterHost, - cluster_id=cluster_id, hsot_id=host_id + cluster_id=cluster_id, host_id=host_id ) - is_cluster_editable(session, clusterhost.cluster, deleter) - return utils.update_db_object( - session, clusterhost, package_config={}, config_validated=False + return _delete_clusterhost_config( + session, deleter, clusterhost ) @@ -642,9 +964,8 @@ def delete_clusterhost_config(session, deleter, clusterhost_id): clusterhost = utils.get_db_object( session, models.ClusterHost, id=clusterhost_id ) - is_cluster_editable(session, clusterhost.cluster, deleter) - return utils.update_db_object( - session, clusterhost, package_config={}, config_validated=False + return _delete_clusterhost_config( + session, deleter, clusterhost ) @@ -655,10 +976,13 @@ def delete_clusterhost_config(session, deleter, clusterhost_id): @user_api.check_user_permission_in_session( permission.PERMISSION_UPDATE_CLUSTER_HOSTS ) -@utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS) +@utils.wrap_to_dict( + ['hosts'], + hosts=RESP_CLUSTERHOST_FIELDS +) def update_cluster_hosts( - session, updater, cluster_id, add_hosts=[], set_hosts=None, - remove_hosts=[] + session, updater, cluster_id, add_hosts={}, set_hosts=None, + remove_hosts={} ): """Update cluster hosts.""" cluster = utils.get_db_object( @@ -666,12 +990,16 @@ def update_cluster_hosts( ) is_cluster_editable(session, cluster, updater) if remove_hosts: - _remove_clusterhosts(session, cluster, remove_hosts) + _remove_clusterhosts(session, cluster, **remove_hosts) if add_hosts: - _add_clusterhosts(session, cluster, add_hosts) + _add_clusterhosts(session, cluster, **add_hosts) if set_hosts is not None: - _set_clusterhosts(session, cluster, set_hosts) - return cluster.clusterhosts + _set_clusterhosts(session, cluster, **set_hosts) + return { + 'hosts': [ + clusterhost.host for clusterhost in cluster.clusterhosts + ] + } @utils.supported_filters(optional_support_keys=['review']) @@ -691,12 +1019,21 @@ def review_cluster(session, reviewer, cluster_id, review={}, **kwargs): session, models.Cluster, id=cluster_id ) is_cluster_editable(session, cluster, reviewer) + clusterhost_ids = review.get('clusterhosts', []) + filters = { + 'cluster_id': cluster_id + } + if clusterhost_ids: + filters['id'] = clusterhost_ids + clusterhosts = utils.list_db_objects( + session, models.ClusterHost, **filters + ) os_config = cluster.os_config if os_config: metadata_api.validate_os_config( os_config, cluster.os_id, True ) - for clusterhost in cluster.clusterhosts: + for clusterhost in clusterhosts: host = clusterhost.host if not host_api.is_host_editable( session, host, reviewer, False @@ -714,23 +1051,21 @@ def review_cluster(session, reviewer, cluster_id, review={}, **kwargs): deployed_os_config, host.os_id, True ) host_api.validate_host(session, host) - host.deployed_os_config = deployed_os_config host.config_validated = True package_config = cluster.package_config if package_config: metadata_api.validate_package_config( package_config, cluster.adapter_id, True ) - for clusterhost in cluster.clusterhosts: + for clusterhost in clusterhosts: clusterhost_package_config = clusterhost.package_config - deployed_package_config = util.mrege_dict( + deployed_package_config = util.merge_dict( package_config, clusterhost_package_config ) metadata_api.validate_package_config( deployed_package_config, cluster.adapter_id, True ) - clusterhost.deployed_package_config = deployed_package_config clusterhost.config_validated = True cluster.config_validated = True return { @@ -753,14 +1088,47 @@ def deploy_cluster( session, deployer, cluster_id, deploy={}, **kwargs ): """deploy cluster.""" + from compass.db.api import host as host_api from compass.tasks import client as celery_client cluster = utils.get_db_object( session, models.Cluster, id=cluster_id ) + clusterhost_ids = deploy.get('clusterhosts', []) + filters = { + 'cluster_id': cluster_id + } + if clusterhost_ids: + filters['id'] = clusterhost_ids + clusterhosts = utils.list_db_objects( + session, models.ClusterHost, **filters + ) + is_cluster_editable(session, cluster, deployer) is_cluster_validated(session, cluster) + utils.update_db_object( + session, cluster.state, state='INITIALIZED' + ) + for clusterhost in clusterhosts: + if cluster.distributed_system: + is_clusterhost_validated(session, clusterhost) + utils.update_db_object( + session, clusterhost.state, + state='INITIALIZED' + ) + host = clusterhost.host + if host_api.is_host_editable( + session, host, deployer, + exception_when_not_editable=False + ): + host_api.is_host_validated( + session, host + ) + utils.update_db_object( + session, host.state, state='INITIALIZED' + ) + celery_client.celery.send_task( 'compass.tasks.deploy', - (cluster_id, deploy.get('clusterhosts', [])) + (deployer.email, cluster_id, deploy.get('clusterhosts', [])) ) return { 'status': 'deploy action sent', @@ -850,3 +1218,22 @@ def update_clusterhost_state( ) utils.update_db_object(session, clusterhost.state, **kwargs) return clusterhost.state_dict() + + +@utils.supported_filters( + optional_support_keys=UPDATED_CLUSTER_STATE_FIELDS +) +@database.run_in_session() +@user_api.check_user_permission_in_session( + permission.PERMISSION_UPDATE_CLUSTER_STATE +) +@utils.wrap_to_dict(RESP_STATE_FIELDS) +def update_cluster_state( + session, updater, cluster_id, **kwargs +): + """Update a cluster state.""" + cluster = utils.get_db_object( + session, models.Cluster, id=cluster_id + ) + utils.update_db_object(session, cluster.state, **kwargs) + return cluster.state_dict() diff --git a/compass/db/api/database.py b/compass/db/api/database.py index 389608d0..170cfd63 100644 --- a/compass/db/api/database.py +++ b/compass/db/api/database.py @@ -19,6 +19,7 @@ import netaddr from contextlib import contextmanager from sqlalchemy import create_engine +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import scoped_session from sqlalchemy.orm import sessionmaker from threading import local @@ -80,7 +81,11 @@ def session(): new_session.rollback() logging.error('failed to commit session') logging.exception(error) - if isinstance(error, exception.DatabaseException): + if isinstance(error, IntegrityError): + raise exception.NotAcceptable( + 'operation error in database' + ) + elif isinstance(error, exception.DatabaseException): raise error else: raise exception.DatabaseException(str(error)) @@ -122,8 +127,8 @@ def _setup_user_table(user_session): from compass.db.api import user user.add_user_internal( user_session, - setting.COMPASS_ADMIN_EMAIL, - setting.COMPASS_ADMIN_PASSWORD, + email=setting.COMPASS_ADMIN_EMAIL, + password=setting.COMPASS_ADMIN_PASSWORD, is_admin=True ) diff --git a/compass/db/api/host.py b/compass/db/api/host.py index d2ea77c4..fefd7304 100644 --- a/compass/db/api/host.py +++ b/compass/db/api/host.py @@ -13,6 +13,7 @@ # limitations under the License. """Host database operations.""" +import functools import logging from compass.db.api import database @@ -52,6 +53,9 @@ RESP_CONFIG_FIELDS = [ 'created_at', 'updated_at' ] +RESP_DEPLOYED_CONFIG_FIELDS = [ + 'deployed_os_config' +] RESP_DEPLOY_FIELDS = [ 'status', 'host' ] @@ -62,19 +66,25 @@ UPDATED_CONFIG_FIELDS = [ PATCHED_CONFIG_FIELDS = [ 'patched_os_config' ] +UPDATED_DEPLOYED_CONFIG_FIELDS = [ + 'deployed_os_config' +] ADDED_NETWORK_FIELDS = [ 'interface', 'ip', 'subnet_id' ] OPTIONAL_ADDED_NETWORK_FIELDS = ['is_mgmt', 'is_promiscuous'] UPDATED_NETWORK_FIELDS = [ - 'interface', 'ip', 'subnet_id', 'subnet', 'is_mgmt', + 'ip', 'subnet_id', 'subnet', 'is_mgmt', 'is_promiscuous' ] +IGNORED_NETWORK_FIELDS = [ + 'interface' +] RESP_STATE_FIELDS = [ - 'id', 'state', 'progress', 'message' + 'id', 'state', 'percentage', 'message' ] UPDATED_STATE_FIELDS = [ - 'id', 'state', 'progress', 'message' + 'id', 'state', 'percentage', 'message' ] @@ -126,10 +136,14 @@ def list_machines_or_hosts(session, lister, **filters): permission.PERMISSION_LIST_HOSTS ) @utils.wrap_to_dict(RESP_FIELDS) -def get_host(session, getter, host_id, **kwargs): +def get_host( + session, getter, host_id, + exception_when_missing=True, **kwargs +): """get host info.""" return utils.get_db_object( - session, models.Host, id=host_id + session, models.Host, + exception_when_missing, id=host_id ) @@ -139,11 +153,17 @@ def get_host(session, getter, host_id, **kwargs): permission.PERMISSION_LIST_HOSTS ) @utils.wrap_to_dict(RESP_FIELDS) -def get_machine_or_host(session, getter, host_id, **kwargs): +def get_machine_or_host( + session, getter, host_id, + exception_when_missing=True, **kwargs +): """get host info.""" machine = utils.get_db_object( - session, models.Machine, id=host_id + session, models.Machine, + exception_when_missing, id=host_id ) + if not machine: + return None host = machine.host if host: return host @@ -186,7 +206,7 @@ def is_host_editable( reinstall_os_set=False, exception_when_not_editable=True ): if reinstall_os_set: - if host.state.state == 'INSTALLING': + if host.state.state == 'DEPLOYING': return _conditional_exception( host, exception_when_not_editable ) @@ -222,7 +242,7 @@ def validate_host(session, host): ) -@utils.supported_filters(UPDATED_FIELDS) +@utils.supported_filters(optional_support_keys=UPDATED_FIELDS) @database.run_in_session() @user_api.check_user_permission_in_session( permission.PERMISSION_UPDATE_HOST @@ -268,35 +288,98 @@ def get_host_config(session, getter, host_id, **kwargs): ) +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission_in_session( + permission.PERMISSION_LIST_HOST_CONFIG +) +@utils.wrap_to_dict(RESP_DEPLOYED_CONFIG_FIELDS) +def get_host_deployed_config(session, getter, host_id, **kwargs): + """Get host deployed config.""" + return utils.get_db_object( + session, models.Host, id=host_id + ) + + +@utils.replace_filters( + os_config='deployed_os_config' +) +@utils.supported_filters(UPDATED_DEPLOYED_CONFIG_FIELDS) +@database.run_in_session() @user_api.check_user_permission_in_session( permission.PERMISSION_ADD_HOST_CONFIG ) @utils.wrap_to_dict(RESP_CONFIG_FIELDS) -def _update_host_config(session, updater, host_id, **kwargs): - """Update host config.""" +def update_host_deployed_config(session, updater, host_id, **kwargs): + """Update host deployed config.""" host = utils.get_db_object( session, models.Host, id=host_id ) is_host_editable(session, host, updater) - utils.update_db_object(session, host, config_validated=False, **kwargs) - os_config = host.os_config - if os_config: - metadata_api.validate_os_config( - os_config, host.adapter_id - ) - return host + is_host_validated(session, host) + return utils.update_db_object(session, host, **kwargs) +@user_api.check_user_permission_in_session( + permission.PERMISSION_ADD_HOST_CONFIG +) +@utils.wrap_to_dict(RESP_CONFIG_FIELDS) +def update_host_config_internal(session, updater, host, **kwargs): + """Update host config.""" + is_host_editable(session, host, updater) + return utils.update_db_object(session, host, **kwargs) + + +@utils.replace_filters( + os_config='put_os_config' +) @utils.supported_filters(UPDATED_CONFIG_FIELDS) @database.run_in_session() def update_host_config(session, updater, host_id, **kwargs): - return _update_host_config(session, updater, host_id, **kwargs) + host = utils.get_db_object( + session, models.Host, id=host_id + ) + + os_config_validates = functools.partial( + metadata_api.validate_os_config, os_id=host.os_id) + + @utils.input_validates( + put_os_config=os_config_validates, + ) + def update_config_internal(host, **in_kwargs): + return update_host_config_internal( + session, updater, host, **kwargs + ) + + return update_config_internal( + host, **kwargs + ) +@utils.replace_filters( + os_config='patched_os_config' +) @utils.supported_filters(PATCHED_CONFIG_FIELDS) @database.run_in_session() def patch_host_config(session, updater, host_id, **kwargs): - return _update_host_config(session, updater, host_id, **kwargs) + host = utils.get_db_object( + session, models.Host, id=host_id + ) + + os_config_validates = functools.partial( + metadata_api.validate_os_config, os_id=host.os_id) + + @utils.output_validates( + os_config=os_config_validates, + ) + def update_config_internal(host, **in_kwargs): + return update_host_config_internal( + session, updater, host, **in_kwargs + ) + + return update_config_internal( + session, updater, host, **kwargs + ) @utils.supported_filters([]) @@ -312,7 +395,7 @@ def del_host_config(session, deleter, host_id): ) is_host_editable(session, host, deleter) return utils.update_db_object( - session, host, os_config={}, config_validated=False + session, host, os_config={} ) @@ -353,12 +436,22 @@ def list_hostnetworks(session, lister, **filters): permission.PERMISSION_LIST_HOST_NETWORKS ) @utils.wrap_to_dict(RESP_NETWORK_FIELDS) -def get_host_network(session, getter, host_id, subnet_id, **kwargs): +def get_host_network( + session, getter, host_id, + host_network_id, **kwargs +): """Get host network.""" - return utils.get_db_object( + host_network = utils.get_db_object( session, models.HostNetwork, - host_id=host_id, subnet_id=subnet_id + id=host_network_id ) + if host_network.host_id != host_id: + raise exception.RecordNotExists( + 'host %s does not own host network %s' % ( + host_id, host_network_id + ) + ) + return host_network @utils.supported_filters([]) @@ -383,37 +476,54 @@ def get_hostnetwork(session, getter, host_network_id, **kwargs): permission.PERMISSION_ADD_HOST_NETWORK ) @utils.wrap_to_dict(RESP_NETWORK_FIELDS) -def add_host_network(session, creator, host_id, **kwargs): +def add_host_network( + session, creator, host_id, + exception_when_existing=True, + interface=None, **kwargs +): """Create a host network.""" host = utils.get_db_object( session, models.Host, id=host_id ) is_host_editable(session, host, creator) return utils.add_db_object( - session, models.HostNetwork, True, - host_id, **kwargs + session, models.HostNetwork, + exception_when_existing, + host_id, interface, **kwargs ) @utils.supported_filters( - optional_support_keys=UPDATED_NETWORK_FIELDS + optional_support_keys=UPDATED_NETWORK_FIELDS, + ignore_support_keys=IGNORED_NETWORK_FIELDS ) @database.run_in_session() @user_api.check_user_permission_in_session( permission.PERMISSION_ADD_HOST_NETWORK ) @utils.wrap_to_dict(RESP_NETWORK_FIELDS) -def update_host_network(session, updater, host_id, subnet_id, **kwargs): +def update_host_network( + session, updater, host_id, host_network_id, **kwargs +): """Update a host network.""" host_network = utils.get_db_object( session, models.HostNetwork, - host_id=host_id, subnet_id=subnet_id + id=host_network_id ) + if host_network.host_id != host_id: + raise exception.RecordNotExists( + 'host %s does not own host network %s' % ( + host_id, host_network_id + ) + ) is_host_editable(session, host_network.host, updater) return utils.update_db_object(session, host_network, **kwargs) -@utils.supported_filters(UPDATED_NETWORK_FIELDS) +@utils.supported_filters( + optional_support_keys=UPDATED_NETWORK_FIELDS, + ignore_support_keys=IGNORED_NETWORK_FIELDS +) @database.run_in_session() @user_api.check_user_permission_in_session( permission.PERMISSION_ADD_HOST_NETWORK @@ -434,12 +544,18 @@ def update_hostnetwork(session, updater, host_network_id, **kwargs): permission.PERMISSION_DEL_HOST_NETWORK ) @utils.wrap_to_dict(RESP_NETWORK_FIELDS) -def del_host_network(session, deleter, host_id, subnet_id, **kwargs): +def del_host_network(session, deleter, host_id, host_network_id, **kwargs): """Delete a host network.""" host_network = utils.get_db_object( session, models.HostNetwork, - host_id=host_id, subnet_id=subnet_id + id=host_network_id ) + if host_network.host_id != host_id: + raise exception.RecordNotExists( + 'host %s does not own host network %s' % ( + host_id, host_network_id + ) + ) is_host_editable(session, host_network.host, deleter) return utils.del_db_object(session, host_network) diff --git a/compass/db/api/installer.py b/compass/db/api/installer.py index ae1b435d..8f482999 100644 --- a/compass/db/api/installer.py +++ b/compass/db/api/installer.py @@ -30,9 +30,9 @@ def _add_installers(session, model, configs): for config in configs: installers.append(utils.add_db_object( session, model, - True, config['NAME'], - installer_type=config['TYPE'], - config=config['CONFIG'] + True, config['INSTANCE_NAME'], + name=config['NAME'], + settings=config.get('SETTINGS', {}) )) return installers diff --git a/compass/db/api/machine.py b/compass/db/api/machine.py index 87686183..563505ee 100644 --- a/compass/db/api/machine.py +++ b/compass/db/api/machine.py @@ -38,54 +38,21 @@ RESP_FIELDS = [ ] -def _check_ipmi_credentials_ip(ip): - utils.check_ip(ip) - - -def _check_ipmi_credentials(ipmi_credentials): - if not ipmi_credentials: - return - if not isinstance(ipmi_credentials, dict): - raise exception.InvalidParameter( - 'invalid ipmi credentials %s' % ipmi_credentials - - ) - for key in ipmi_credentials: - if key not in ['ip', 'username', 'password']: - raise exception.InvalidParameter( - 'unrecognized field %s in ipmi credentials %s' % ( - key, ipmi_credentials - ) - ) - for key in ['ip', 'username', 'password']: - if key not in ipmi_credentials: - raise exception.InvalidParameter( - 'no field %s in ipmi credentials %s' % ( - key, ipmi_credentials - ) - ) - check_ipmi_credential_field = '_check_ipmi_credentials_%s' % key - this_module = globals() - if check_ipmi_credential_field in this_module: - this_module[check_ipmi_credential_field]( - ipmi_credentials[key] - ) - else: - logging.debug( - 'function %s is not defined', check_ipmi_credential_field - ) - - @utils.supported_filters([]) @database.run_in_session() @user_api.check_user_permission_in_session( permission.PERMISSION_LIST_MACHINES ) @utils.wrap_to_dict(RESP_FIELDS) -def get_machine(session, getter, machine_id, **kwargs): +def get_machine( + session, getter, machine_id, + exception_when_missing=True, + **kwargs +): """get field dict of a machine.""" return utils.get_db_object( - session, models.Machine, True, id=machine_id + session, models.Machine, + exception_when_missing, id=machine_id ) @@ -119,7 +86,7 @@ def _update_machine(session, updater, machine_id, **kwargs): @utils.supported_filters(optional_support_keys=UPDATED_FIELDS) -@utils.input_validates(ipmi_credentials=_check_ipmi_credentials) +@utils.input_validates(ipmi_credentials=utils.check_ipmi_credentials) @database.run_in_session() def update_machine(session, updater, machine_id, **kwargs): return _update_machine( @@ -127,9 +94,14 @@ def update_machine(session, updater, machine_id, **kwargs): ) +@utils.replace_filters( + ipmi_credentials='patched_ipmi_credentials', + tag='patched_tag', + location='patched_location' +) @utils.supported_filters(optional_support_keys=PATCHED_FIELDS) @database.run_in_session() -@utils.output_validates(ipmi_credentials=_check_ipmi_credentials) +@utils.output_validates(ipmi_credentials=utils.check_ipmi_credentials) def patch_machine(session, updater, machine_id, **kwargs): return _update_machine( session, updater, machine_id, **kwargs diff --git a/compass/db/api/network.py b/compass/db/api/network.py index 7eac06a8..a8f94ffa 100644 --- a/compass/db/api/network.py +++ b/compass/db/api/network.py @@ -67,10 +67,13 @@ def list_subnets(session, lister, **filters): permission.PERMISSION_LIST_NETWORKS ) @utils.wrap_to_dict(RESP_FIELDS) -def get_subnet(session, getter, subnet_id, **kwargs): +def get_subnet( + session, getter, subnet_id, + exception_when_missing=True, **kwargs +): """Get subnet info.""" return utils.get_db_object( - session, models.Network, id=subnet_id + session, models.Network, exception_when_missing, id=subnet_id ) @@ -84,10 +87,14 @@ def get_subnet(session, getter, subnet_id, **kwargs): permission.PERMISSION_ADD_NETWORK ) @utils.wrap_to_dict(RESP_FIELDS) -def add_subnet(session, creator, subnet, **kwargs): +def add_subnet( + session, creator, subnet, + exception_when_existing=True, **kwargs +): """Create a subnet.""" return utils.add_db_object( - session, models.Network, True, subnet, **kwargs + session, models.Network, + exception_when_existing, subnet, **kwargs ) diff --git a/compass/db/api/permission.py b/compass/db/api/permission.py index 3e20f7ff..f77a77f0 100644 --- a/compass/db/api/permission.py +++ b/compass/db/api/permission.py @@ -136,6 +136,10 @@ PERMISSION_DEPLOY_HOST = PermissionWrapper( PERMISSION_GET_CLUSTER_STATE = PermissionWrapper( 'get_cluster_state', 'get cluster state', 'get cluster state' ) +PERMISSION_UPDATE_CLUSTER_STATE = PermissionWrapper( + 'update_cluster_state', 'update cluster state', + 'update cluster state' +) PERMISSION_LIST_HOSTS = PermissionWrapper( 'list_hosts', 'list hosts', 'list hosts' ) @@ -234,6 +238,7 @@ PERMISSIONS = [ PERMISSION_REVIEW_CLUSTER, PERMISSION_DEPLOY_CLUSTER, PERMISSION_GET_CLUSTER_STATE, + PERMISSION_UPDATE_CLUSTER_STATE, PERMISSION_LIST_HOSTS, PERMISSION_LIST_HOST_CLUSTERS, PERMISSION_UPDATE_HOST, @@ -276,10 +281,14 @@ def list_permissions(session, lister, **filters): @database.run_in_session() @user_api.check_user_permission_in_session(PERMISSION_LIST_PERMISSIONS) @utils.wrap_to_dict(RESP_FIELDS) -def get_permission(session, getter, permission_id, **kwargs): +def get_permission( + session, getter, permission_id, + exception_when_missing=True, **kwargs +): """get permissions.""" return utils.get_db_object( - session, models.Permission, id=permission_id + session, models.Permission, + exception_when_missing, id=permission_id ) diff --git a/compass/db/api/switch.py b/compass/db/api/switch.py index b8f3dd52..41b5d1dd 100644 --- a/compass/db/api/switch.py +++ b/compass/db/api/switch.py @@ -57,7 +57,7 @@ OPTIONAL_CHECK_FILTER_FIELDS = [ 'ports', 'port_prefix', 'port_suffix', 'port_start', 'port_end' ] -ALL_ADDED_MACHINES_FIELDS = ['port', 'vlans'] +ADDED_SWITCH_MACHINES_FIELDS = ['port', 'vlans'] UPDATED_MACHINES_FIELDS = [ 'port', 'vlans', 'ipmi_credentials', 'tag', 'location' @@ -88,52 +88,18 @@ RESP_MACHINES_HOSTS_FIELDS = [ 'ipmi_credentials', 'tag', 'location', 'name', 'os_name', 'clusters' ] +RESP_CLUSTER_FIELDS = [ + 'name', 'id' +] -def _check_credentials_version(version): - if version not in ['1', '2c', '3']: - raise exception.InvalidParameter( - 'unknown snmp version %s' % version - ) - - -def _check_credentials(credentials): - if not credentials: - return - if not isinstance(credentials, dict): - raise exception.InvalidParameter( - 'credentials %s is not dict' % credentials - ) - for key in credentials: - if key not in ['version', 'community']: +def _check_filters(switch_filters): + for switch_filter in switch_filters: + if not isinstance(switch_filter, dict): raise exception.InvalidParameter( - 'unrecognized key %s in credentials %s' % (key, credentials) + 'filter %s is not dict' % switch_filter ) - for key in ['version', 'community']: - if key not in credentials: - raise exception.InvalidParameter( - 'there is no %s field in credentials %s' % (key, credentials) - ) - - key_check_func_name = '_check_credentials_%s' % key - this_module = globals() - if key_check_func_name in this_module: - this_module[key_check_func_name]( - credentials[key] - ) - else: - logging.debug( - 'function %s is not defined in %s', - key_check_func_name, this_module - ) - - -def _check_filter(switch_filter): - if not isinstance(switch_filter, dict): - raise exception.InvalidParameter( - 'filter %s is not dict' % switch_filter - ) - _check_filter_internal(**switch_filter) + _check_filter_internal(**switch_filter) @utils.supported_filters( @@ -159,11 +125,12 @@ def _check_filter_internal( ) -def _check_vlan(vlan): - if not isinstance(vlan, int): - raise exception.InvalidParameter( - 'vlan %s is not int' % vlan - ) +def _check_vlans(vlans): + for vlan in vlans: + if not isinstance(vlan, int): + raise exception.InvalidParameter( + 'vlan %s is not int' % vlan + ) def add_switch_internal( @@ -193,10 +160,14 @@ def get_switch_internal( permission.PERMISSION_LIST_SWITCHES ) @utils.wrap_to_dict(RESP_FIELDS) -def get_switch(session, getter, switch_id, **kwargs): +def get_switch( + session, getter, switch_id, + exception_when_missing=True, **kwargs +): """get field dict of a switch.""" return utils.get_db_object( - session, models.Switch, id=switch_id + session, models.Switch, + exception_when_missing, id=switch_id ) @@ -208,9 +179,16 @@ def get_switch(session, getter, switch_id, **kwargs): @utils.wrap_to_dict(RESP_FIELDS) def list_switches(session, lister, **filters): """List switches.""" - return utils.list_db_objects( + switches = utils.list_db_objects( session, models.Switch, **filters ) + if 'ip_int' in filters: + return switches + else: + return [ + switch for switch in switches + if switch.ip != setting.DEFAULT_SWITCH_IP + ] @utils.supported_filters([]) @@ -231,18 +209,21 @@ def del_switch(session, deleter, switch_id, **kwargs): ) @utils.input_validates( ip=utils.check_ip, - credentials=_check_credentials + credentials=utils.check_switch_credentials ) @database.run_in_session() @user_api.check_user_permission_in_session( permission.PERMISSION_ADD_SWITCH ) @utils.wrap_to_dict(RESP_FIELDS) -def add_switch(session, creator, ip, **kwargs): +def add_switch( + session, creator, exception_when_existing=True, + ip=None, **kwargs +): """Create a switch.""" ip_int = long(netaddr.IPAddress(ip)) return add_switch_internal( - session, ip_int, **kwargs + session, ip_int, exception_when_existing, **kwargs ) @@ -267,15 +248,22 @@ def _update_switch(session, updater, switch_id, **kwargs): @utils.supported_filters(optional_support_keys=UPDATED_FIELDS) -@utils.input_validates(credentials=_check_credentials) +@utils.input_validates( + credentials=utils.check_switch_credentials +) @database.run_in_session() def update_switch(session, updater, switch_id, **kwargs): return _update_switch(session, updater, switch_id, **kwargs) +@utils.replace_filters( + credentials='patched_credentials' +) @utils.supported_filters(optional_support_keys=PATCHED_FIELDS) @database.run_in_session() -@utils.output_validates(credentials=_check_credentials) +@utils.output_validates( + credentials=utils.check_switch_credentials +) def patch_switch(session, updater, switch_id, **kwargs): return _update_switch(session, updater, switch_id, **kwargs) @@ -299,7 +287,9 @@ def list_switch_filters(session, lister, **filters): permission.PERMISSION_LIST_SWITCH_FILTERS ) @utils.wrap_to_dict(RESP_FILTERS_FIELDS) -def get_switch_filters(session, getter, switch_id, **kwargs): +def get_switch_filters( + session, getter, switch_id, **kwargs +): """get switch filter.""" return utils.get_db_object( session, models.Switch, id=switch_id @@ -307,7 +297,7 @@ def get_switch_filters(session, getter, switch_id, **kwargs): @utils.supported_filters(optional_support_keys=UPDATED_FILTERS_FIELDS) -@utils.input_validates(filters=_check_filter) +@utils.input_validates(filters=_check_filters) @database.run_in_session() @user_api.check_user_permission_in_session( permission.PERMISSION_UPDATE_SWITCH_FILTERS @@ -319,8 +309,11 @@ def update_switch_filters(session, updater, switch_id, **kwargs): return utils.update_db_object(session, switch, **kwargs) +@utils.replace_filters( + filters='patched_filters' +) @utils.supported_filters(optional_support_keys=PATCHED_FILTERS_FIELDS) -@utils.input_validates(patched_filters=_check_filter) +@utils.input_validates(patched_filters=_check_filters) @database.run_in_session() @user_api.check_user_permission_in_session( permission.PERMISSION_UPDATE_SWITCH_FILTERS @@ -433,7 +426,7 @@ def _filter_vlans(vlan_filter, obj): location=utils.general_filter_callback ) @utils.wrap_to_dict(RESP_MACHINES_FIELDS) -def _list_switch_machines(session, user, switch_machines, **filters): +def _filter_switch_machines(session, user, switch_machines, **filters): return [ switch_machine for switch_machine in switch_machines if filter_machine_internal( @@ -454,8 +447,11 @@ def _list_switch_machines(session, user, switch_machines, **filters): os_name=utils.general_filter_callback, os_id=utils.general_filter_callback ) -@utils.wrap_to_dict(RESP_MACHINES_HOSTS_FIELDS) -def _list_switch_machines_hosts(session, user, switch_machines, **filters): +@utils.wrap_to_dict( + RESP_MACHINES_HOSTS_FIELDS, + clusters=RESP_CLUSTER_FIELDS +) +def _filter_switch_machines_hosts(session, user, switch_machines, **filters): filtered_switch_machines = [ switch_machine for switch_machine in switch_machines if filter_machine_internal( @@ -492,12 +488,15 @@ def _list_switch_machines_hosts(session, user, switch_machines, **filters): @database.run_in_session() def list_switch_machines(session, getter, switch_id, **filters): """Get switch machines.""" - switch_machines, host_filters = get_switch_machines_internal( + switch_machines = get_switch_machines_internal( session, switch_id=switch_id, **filters ) - return _list_switch_machines(session, getter, switch_machines, **filters) + return _filter_switch_machines(session, getter, switch_machines, **filters) +@utils.replace_filters( + ip_int='switch_ip_int' +) @utils.supported_filters( optional_support_keys=SUPPORTED_SWITCH_MACHINES_FIELDS ) @@ -507,7 +506,16 @@ def list_switchmachines(session, lister, **filters): switch_machines = get_switch_machines_internal( session, **filters ) - return _list_switch_machines(session, lister, switch_machines, **filters) + if 'ip_int' in filters: + filtered_switch_machines = switch_machines + else: + filtered_switch_machines = [ + switch_machine for switch_machine in switch_machines + if switch_machine.switch_ip != setting.DEFAULT_SWITCH_IP + ] + return _filter_switch_machines( + session, lister, filtered_switch_machines, **filters + ) @utils.supported_filters( @@ -519,11 +527,14 @@ def list_switch_machines_hosts(session, getter, switch_id, **filters): switch_machines = get_switch_machines_internal( session, switch_id=switch_id, **filters ) - return _list_switch_machines_hosts( + return _filter_switch_machines_hosts( session, getter, switch_machines, **filters ) +@utils.replace_filters( + ip_int='switch_ip_int' +) @utils.supported_filters( optional_support_keys=SUPPORTED_SWITCH_MACHINES_HOSTS_FIELDS ) @@ -533,29 +544,46 @@ def list_switchmachines_hosts(session, lister, **filters): switch_machines = get_switch_machines_internal( session, **filters ) - return _list_switch_machines_hosts( - session, lister, switch_machines, **filters + if 'ip_int' in filters: + filtered_switch_machines = switch_machines + else: + filtered_switch_machines = [ + switch_machine for switch_machine in switch_machines + if switch_machine.switch_ip != setting.DEFAULT_SWITCH_IP + ] + return _filter_switch_machines_hosts( + session, lister, filtered_switch_machines, **filters ) -def add_switch_machines_internal( - session, switch, machine_dicts, - exception_when_switch_machine_existing=True +@utils.supported_filters( + ADDED_MACHINES_FIELDS, + optional_support_keys=OPTIONAL_ADDED_MACHINES_FIELDS +) +@utils.input_validates(mac=utils.check_mac, vlans=_check_vlans) +@database.run_in_session() +@user_api.check_user_permission_in_session( + permission.PERMISSION_ADD_SWITCH_MACHINE +) +@utils.wrap_to_dict(RESP_MACHINES_FIELDS) +def add_switch_machine( + session, creator, switch_id, + exception_when_existing=True, + mac=None, **kwargs ): - machine_id_switch_machine_dict = {} - for mac, all_dict in machine_dicts.items(): - switch_machine_dict = {} - machine_dict = {} - for key, value in all_dict.items(): - if key in ALL_ADDED_MACHINES_FIELDS: - switch_machine_dict[key] = value - else: - machine_dict[key] = value - #TODO(xiaodong): add ipmi field checks' - machine = utils.add_db_object( - session, models.Machine, False, - mac, **machine_dict) - machine_id_switch_machine_dict[machine.id] = switch_machine_dict + """Add switch machine.""" + switch = utils.get_db_object( + session, models.Switch, id=switch_id) + switch_machine_dict = {} + machine_dict = {} + for key, value in kwargs.items(): + if key in ADDED_SWITCH_MACHINES_FIELDS: + switch_machine_dict[key] = value + else: + machine_dict[key] = value + machine = utils.add_db_object( + session, models.Machine, False, + mac, **machine_dict) switches = [switch] if switch.ip != setting.DEFAULT_SWITCH_IP: @@ -566,35 +594,13 @@ def add_switch_machines_internal( switch_machines = [] for machine_switch in switches: - for machine_id, switch_machine_dict in ( - machine_id_switch_machine_dict.items() - ): - utils.add_db_object( - session, models.SwitchMachine, - exception_when_switch_machine_existing, - machine_switch.id, machine_id, **switch_machine_dict - ) - switch_machines.extend(machine_switch.switch_machines) + switch_machines.append(utils.add_db_object( + session, models.SwitchMachine, + exception_when_existing, + machine_switch.id, machine.id, + **switch_machine_dict + )) - return switch_machines - - -@utils.supported_filters( - ADDED_MACHINES_FIELDS, - optional_support_keys=OPTIONAL_ADDED_MACHINES_FIELDS -) -@utils.input_validates(mac=utils.check_mac, vlans=_check_vlan) -@database.run_in_session() -@user_api.check_user_permission_in_session( - permission.PERMISSION_ADD_SWITCH_MACHINE -) -@utils.wrap_to_dict(RESP_MACHINES_FIELDS) -def add_switch_machine(session, creator, switch_id, mac, **kwargs): - """Add switch machine.""" - switch = utils.get_db_object( - session, models.Switch, id=switch_id) - switch_machines = add_switch_machines_internal( - session, switch, {mac: kwargs}) return switch_machines[0] @@ -610,7 +616,7 @@ def poll_switch_machines(session, poller, switch_id, **kwargs): switch = utils.get_db_object(session, models.Switch, id=switch_id) celery_client.celery.send_task( 'compass.tasks.pollswitch', - (switch.ip, switch.credentials) + (poller.email, switch.ip, switch.credentials) ) return { 'status': 'action %s sent' % kwargs, @@ -625,10 +631,14 @@ def poll_switch_machines(session, poller, switch_id, **kwargs): permission.PERMISSION_LIST_SWITCH_MACHINES ) @utils.wrap_to_dict(RESP_MACHINES_FIELDS) -def get_switch_machine(session, getter, switch_id, machine_id, **kwargs): +def get_switch_machine( + session, getter, switch_id, machine_id, + exception_when_missing=True, **kwargs +): """get field dict of a switch machine.""" return utils.get_db_object( session, models.SwitchMachine, + exception_when_missing, switch_id=switch_id, machine_id=machine_id ) @@ -639,10 +649,15 @@ def get_switch_machine(session, getter, switch_id, machine_id, **kwargs): permission.PERMISSION_LIST_SWITCH_MACHINES ) @utils.wrap_to_dict(RESP_MACHINES_FIELDS) -def get_switchmachine(session, getter, switch_machine_id, **kwargs): +def get_switchmachine( + session, getter, switch_machine_id, + exception_when_missing=True, + **kwargs +): """get field dict of a switch machine.""" return utils.get_db_object( - session, models.SwitchMachine, id=switch_machine_id + session, models.SwitchMachine, + exception_when_missing, id=switch_machine_id ) @@ -667,7 +682,7 @@ def update_switch_machine_internal( @utils.supported_filters(optional_support_keys=UPDATED_MACHINES_FIELDS) -@utils.input_validates(vlans=_check_vlan) +@utils.input_validates(vlans=_check_vlans) @database.run_in_session() @user_api.check_user_permission_in_session( permission.PERMISSION_ADD_SWITCH_MACHINE @@ -686,7 +701,7 @@ def update_switch_machine(session, updater, switch_id, machine_id, **kwargs): @utils.supported_filters(optional_support_keys=UPDATED_MACHINES_FIELDS) -@utils.input_validates(vlans=_check_vlan) +@utils.input_validates(vlans=_check_vlans) @database.run_in_session() @user_api.check_user_permission_in_session( permission.PERMISSION_ADD_SWITCH_MACHINE @@ -704,8 +719,14 @@ def update_switchmachine(session, updater, switch_machine_id, **kwargs): ) +@utils.replace_filters( + vlans='patched_vlans', + ipmi_credentials='patched_ipmi_credentials', + tag='patched_tag', + location='patched_location' +) @utils.supported_filters(optional_support_keys=PATCHED_MACHINES_FIELDS) -@utils.input_validates(patched_vlans=_check_vlan) +@utils.input_validates(patched_vlans=_check_vlans) @database.run_in_session() @user_api.check_user_permission_in_session( permission.PERMISSION_ADD_SWITCH_MACHINE @@ -723,8 +744,14 @@ def patch_switch_machine(session, updater, switch_id, machine_id, **kwargs): ) +@utils.replace_filters( + vlans='patched_vlans', + ipmi_credentials='patched_ipmi_credentials', + tag='patched_tag', + location='patched_location' +) @utils.supported_filters(optional_support_keys=PATCHED_MACHINES_FIELDS) -@utils.input_validates(patched_vlans=_check_vlan) +@utils.input_validates(patched_vlans=_check_vlans) @database.run_in_session() @user_api.check_user_permission_in_session( permission.PERMISSION_ADD_SWITCH_MACHINE @@ -772,18 +799,21 @@ def del_switchmachine(session, deleter, switch_machine_id, **kwargs): return utils.del_db_object(session, switch_machine) -@utils.supported_filters(optional_support_keys=UPDATED_SWITCH_MACHINES_FIELDS) +@utils.supported_filters( + ['machine_id'], + optional_support_keys=UPDATED_SWITCH_MACHINES_FIELDS +) def _update_machine_internal(session, switch_id, machine_id, **kwargs): utils.add_db_object( - session, models.SwitchMachine, False, switch_id, machine_id, - **kwargs + session, models.SwitchMachine, False, + switch_id, machine_id, **kwargs ) def _add_machines(session, switch, machines): - for machine_id, switch_machine_attrs in machines.items(): + for machine in machines.items(): _update_machine_internal( - session, switch.id, machine_id, **switch_machine_attrs + session, switch.id, **machine ) @@ -799,9 +829,9 @@ def _set_machines(session, switch, machines): session, models.SwitchMachine, switch_id=switch.id ) - for machine_id, switch_machine_attrs in machines.items(): + for switch_machine in machines.items(): _update_machine_internal( - session, switch.id, machine_id, **switch_machine_attrs + session, switch.id, **switch_machine ) diff --git a/compass/db/api/user.py b/compass/db/api/user.py index 905bca71..391a8e75 100644 --- a/compass/db/api/user.py +++ b/compass/db/api/user.py @@ -65,13 +65,14 @@ def get_user_internal(session, exception_when_missing=True, **kwargs): def add_user_internal( - session, email, password, - exception_when_existing=True, **kwargs + session, exception_when_existing=True, + email=None, **kwargs ): """internal function used only by other db.api modules.""" - user = utils.add_db_object(session, models.User, - exception_when_existing, email, - password=password, **kwargs) + user = utils.add_db_object( + session, models.User, + exception_when_existing, email, + **kwargs) _add_user_permissions( session, user, name=setting.COMPASS_DEFAULT_PERMISSIONS @@ -180,14 +181,18 @@ def _set_user_permissions(session, user, **permission_filters): class UserWrapper(UserMixin): def __init__( self, id, email, crypted_password, - active, is_admin, expire_timestamp, token='', **kwargs + active=True, is_admin=False, + expire_timestamp=None, token='', **kwargs ): self.id = id self.email = email self.password = crypted_password self.active = active self.is_admin = is_admin - self.expire_timestamp = expire_timestamp + if expire_timestamp: + self.expire_timestamp = expire_timestamp + else: + self.expire_timestamp = datetime.datetime.now() if not token: self.token = self.get_auth_token() else: @@ -278,9 +283,14 @@ def clean_user_token(session, user, token): @check_user_admin_or_owner() @database.run_in_session() @utils.wrap_to_dict(RESP_FIELDS) -def get_user(session, getter, user_id, **kwargs): +def get_user( + session, getter, user_id, + exception_when_missing=True, **kwargs +): """get field dict of a user.""" - return utils.get_db_object(session, models.User, id=user_id) + return utils.get_db_object( + session, models.User, exception_when_missing, id=user_id + ) @utils.supported_filters( @@ -303,10 +313,14 @@ def list_users(session, lister, **filters): @check_user_admin() @database.run_in_session() @utils.wrap_to_dict(RESP_FIELDS) -def add_user(session, creator, email, password, **kwargs): +def add_user( + session, creator, + exception_when_existing=True, + **kwargs +): """Create a user and return created user object.""" return add_user_internal( - session, email, password, **kwargs + session, exception_when_existing, **kwargs ) @@ -349,7 +363,7 @@ def update_user(session, updater, user_id, **kwargs): @check_user_admin_or_owner() @database.run_in_session() @utils.wrap_to_dict(PERMISSION_RESP_FIELDS) -def get_permissions(session, getter, user_id, **kwargs): +def get_permissions(session, lister, user_id, **kwargs): """List permissions of a user.""" return utils.list_db_objects( session, models.UserPermission, user_id=user_id, **kwargs @@ -360,10 +374,14 @@ def get_permissions(session, getter, user_id, **kwargs): @check_user_admin_or_owner() @database.run_in_session() @utils.wrap_to_dict(PERMISSION_RESP_FIELDS) -def get_permission(session, getter, user_id, permission_id, **kwargs): +def get_permission( + session, getter, user_id, permission_id, + exception_when_missing, **kwargs +): """Get a specific user permission.""" return utils.get_db_object( session, models.UserPermission, + exception_when_missing, user_id=user_id, permission_id=permission_id, **kwargs ) @@ -387,10 +405,13 @@ def del_permission(session, deleter, user_id, permission_id, **kwargs): @check_user_admin() @database.run_in_session() @utils.wrap_to_dict(PERMISSION_RESP_FIELDS) -def add_permission(session, creator, user_id, permission_id): +def add_permission( + session, creator, user_id, + exception_when_missing=True, permission_id=None +): """Add an user permission.""" return utils.add_db_object( - session, models.UserPermission, True, + session, models.UserPermission, exception_when_missing, user_id, permission_id ) diff --git a/compass/db/api/utils.py b/compass/db/api/utils.py index c3857313..b8039675 100644 --- a/compass/db/api/utils.py +++ b/compass/db/api/utils.py @@ -155,6 +155,8 @@ def wrap_to_dict(support_keys=[], **filters): def decorator(func): @functools.wraps(func) def wrapper(*args, **kwargs): + logging.info('wrap to dict: args: %s', str(args)) + logging.info('wrap to dict: kwargs: %s', kwargs) return _wrapper_dict( func(*args, **kwargs), support_keys, **filters ) @@ -185,6 +187,21 @@ def _wrapper_dict(data, support_keys, **filters): return info +def replace_filters(**filter_mapping): + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **filters): + replaced_filters = {} + for key, value in filters.items(): + if key in filter_mapping: + replaced_filters[filter_mapping[key]] = value + else: + replaced_filters[key] = value + return func(*args, **replaced_filters) + return wrapper + return decorator + + def supported_filters( support_keys=[], optional_support_keys=[], @@ -311,18 +328,10 @@ def output_filters(missing_ok=False, **filter_callbacks): def _input_validates(args_validators, kwargs_validators, *args, **kwargs): for i, value in enumerate(args): if i < len(args_validators) and args_validators[i]: - if isinstance(value, list): - for sub_value in value: - args_validators[i](sub_value) - else: - args_validators[i](value) + args_validators[i](value) for key, value in kwargs.items(): if kwargs_validators.get(key): - if isinstance(value, list): - for sub_value in value: - kwargs_validators[key](sub_value) - else: - kwargs_validators[key](value) + kwargs_validators[key](value) def input_validates(*args_validators, **kwargs_validators): @@ -432,8 +441,8 @@ def add_db_object(session, table, exception_when_existing=True, if new_object: session.add(db_object) - db_object.initialize() session.flush() + db_object.initialize() db_object.validate() return db_object @@ -468,8 +477,8 @@ def update_db_object(session, db_object, **kwargs): db_object, kwargs) for key, value in kwargs.items(): setattr(db_object, key, value) - db_object.initialize() session.flush() + db_object.update() db_object.validate() return db_object @@ -500,3 +509,79 @@ def check_mac(mac): raise exception.InvalidParameter( 'invalid mac address %s' % mac ) + + +def _check_ipmi_credentials_ip(ip): + check_ip(ip) + + +def check_ipmi_credentials(ipmi_credentials): + if not ipmi_credentials: + return + if not isinstance(ipmi_credentials, dict): + raise exception.InvalidParameter( + 'invalid ipmi credentials %s' % ipmi_credentials + + ) + for key in ipmi_credentials: + if key not in ['ip', 'username', 'password']: + raise exception.InvalidParameter( + 'unrecognized field %s in ipmi credentials %s' % ( + key, ipmi_credentials + ) + ) + for key in ['ip', 'username', 'password']: + if key not in ipmi_credentials: + raise exception.InvalidParameter( + 'no field %s in ipmi credentials %s' % ( + key, ipmi_credentials + ) + ) + check_ipmi_credential_field = '_check_ipmi_credentials_%s' % key + this_module = globals() + if check_ipmi_credential_field in this_module: + this_module[check_ipmi_credential_field]( + ipmi_credentials[key] + ) + else: + logging.debug( + 'function %s is not defined', check_ipmi_credential_field + ) + + +def _check_switch_credentials_version(version): + if version not in ['1', '2c', '3']: + raise exception.InvalidParameter( + 'unknown snmp version %s' % version + ) + + +def check_switch_credentials(credentials): + if not credentials: + return + if not isinstance(credentials, dict): + raise exception.InvalidParameter( + 'credentials %s is not dict' % credentials + ) + for key in credentials: + if key not in ['version', 'community']: + raise exception.InvalidParameter( + 'unrecognized key %s in credentials %s' % (key, credentials) + ) + for key in ['version', 'community']: + if key not in credentials: + raise exception.InvalidParameter( + 'there is no %s field in credentials %s' % (key, credentials) + ) + + key_check_func_name = '_check_switch_credentials_%s' % key + this_module = globals() + if key_check_func_name in this_module: + this_module[key_check_func_name]( + credentials[key] + ) + else: + logging.debug( + 'function %s is not defined in %s', + key_check_func_name, this_module + ) diff --git a/compass/db/exception.py b/compass/db/exception.py index 26437fcc..263a7b18 100644 --- a/compass/db/exception.py +++ b/compass/db/exception.py @@ -61,6 +61,13 @@ class Forbidden(DatabaseException): self.status_code = 403 +class NotAcceptable(DatabaseException): + """The data is not acceptable.""" + def __init__(self, message): + super(NotAcceptable, self).__init__(message) + self.status_code = 406 + + class InvalidParameter(DatabaseException): """Define the exception that the request has invalid or missing parameters. """ diff --git a/compass/db/models.py b/compass/db/models.py index 702502e9..ae2e23ef 100644 --- a/compass/db/models.py +++ b/compass/db/models.py @@ -14,6 +14,7 @@ """Database model""" import datetime +import logging import netaddr import simplejson as json @@ -67,6 +68,9 @@ class TimestampMixin(object): class HelperMixin(object): def initialize(self): + self.update() + + def update(self): pass def validate(self): @@ -91,7 +95,7 @@ class MetadataMixin(HelperMixin): description = Column(Text) is_required = Column(Boolean, default=False) required_in_whole_config = Column(Boolean, default=False) - mapping_to = Column(JSONEncoded) + mapping_to = Column(String(80), default='') validator_data = Column('validator', Text) js_validator = Column(Text) default_value = Column(JSONEncoded) @@ -107,9 +111,6 @@ class MetadataMixin(HelperMixin): self.path = self.name super(MetadataMixin, self).initialize() - def validate(self): - super(MetadataMixin, self).validate() - @property def validator(self): if not self.validator_data: @@ -249,14 +250,14 @@ class FieldMixin(HelperMixin): class InstallerMixin(HelperMixin): - name = Column(String(80), unique=True) - installer_type = Column(String(80)) - config = Column(MutableDict.as_mutable(JSONEncoded), default={}) + name = Column(String(80)) + instance_name = Column(String(80), unique=True) + settings = Column(MutableDict.as_mutable(JSONEncoded), default={}) def validate(self): - if not self.installer_type: + if not self.name: raise exception.InvalidParameter( - 'installer_type is not set in installer %s' % self.name + 'name is not set in installer %s' % self.instance_name ) super(InstallerMixin, self).validate() @@ -264,24 +265,31 @@ class InstallerMixin(HelperMixin): class StateMixin(TimestampMixin, HelperMixin): state = Column( Enum( - 'INITIALIZED', 'INSTALLING', 'SUCCESSFUL', 'ERROR' + 'UNINITIALIZED', 'INITIALIZED', + 'INSTALLING', 'SUCCESSFUL', 'ERROR' ), - default='INITIALIZED' + default='UNINITIIALIZED' ) - progress = Column(Float, default=0.0) + percentage = Column(Float, default=0.0) message = Column(Text, default='') severity = Column( Enum('INFO', 'WARNING', 'ERROR'), default='INFO' ) - def initialize(self): + def update(self): + if self.state in ['UNINITIALIZED', 'INITIALIZED']: + self.percentage = 0.0 + self.severity = 'INFO' + self.message = '' if self.severity == 'ERROR': self.state = 'ERROR' - elif self.progress >= 1.0: + if self.state == 'SUCCESSFUL': + self.percentage = 1.0 + if self.percentage >= 1.0: self.state = 'SUCCESSFUL' - self.progress = 1.0 - super(StateMixin, self).initialize() + self.percentage = 1.0 + super(StateMixin, self).update() class HostNetwork(BASE, TimestampMixin, HelperMixin): @@ -307,8 +315,9 @@ class HostNetwork(BASE, TimestampMixin, HelperMixin): UniqueConstraint('host_id', 'interface', name='constraint'), ) - def __init__(self, host_id, **kwargs): + def __init__(self, host_id, interface, **kwargs): self.host_id = host_id + self.interface = interface super(HostNetwork, self).__init__(**kwargs) @property @@ -323,15 +332,15 @@ class HostNetwork(BASE, TimestampMixin, HelperMixin): def subnet(self): return self.network.subnet + @subnet.expression + def subnet(cls): + return cls.network.subnet + @property def netmask(self): return str(netaddr.IPNetwork(self.subnet).netmask) def validate(self): - if not self.interface: - raise exception.InvalidParameter( - 'interface is not set in host %s network' % self.host_id - ) if not self.network: raise exception.InvalidParameter( 'subnet is not set in %s interface %s' % ( @@ -367,6 +376,7 @@ class HostNetwork(BASE, TimestampMixin, HelperMixin): dict_info['ip'] = self.ip dict_info['interface'] = self.interface dict_info['netmask'] = self.netmask + dict_info['subnet'] = self.subnet return dict_info @@ -428,6 +438,7 @@ class ClusterHost(BASE, TimestampMixin, HelperMixin): @patched_package_config.setter def patched_package_config(self, value): self.package_config = util.merge_dict(dict(self.package_config), value) + self.config_validated = False @property def put_package_config(self): @@ -438,77 +449,93 @@ class ClusterHost(BASE, TimestampMixin, HelperMixin): package_config = dict(self.package_config) package_config.update(value) self.package_config = package_config + self.config_validated = False + + @property + def patched_os_config(self): + return self.host.os_config + + @patched_os_config.setter + def patched_os_config(self, value): + host = self.host + host.os_config = util.merge_dict(dict(host.os_config), value) + + @property + def put_os_config(self): + return self.host.os_config + + @put_os_config.setter + def put_os_config(self, value): + host = self.host + os_config = dict(host.os_config) + os_config.update(value) + host.os_config = os_config @hybrid_property def distributed_system_name(self): - cluster = self.cluster - if cluster: - return cluster.distributed_system_name - else: - return None + return self.cluster.distributed_system_name + + @distributed_system_name.expression + def distributed_system_name(cls): + return cls.cluster.distributed_system_name @hybrid_property def os_name(self): - host = self.host - if host: - return host.os_name - else: - return None + return self.host.os_name + + @os_name.expression + def os_name(cls): + return cls.host.os_name @hybrid_property def clustername(self): - cluster = self.cluster - if cluster: - return cluster.name - else: - return None + return self.cluster.name + + @clustername.expression + def clustername(cls): + return cls.cluster.name @hybrid_property def hostname(self): - host = self.host - if host: - return host.name - else: - return None + return self.host.name + + @hostname.expression + def hostname(cls): + return cls.host.name @property def distributed_system_installed(self): - state = self.state - if state: - return state.state == 'SUCCESSFUL' - else: - return False + return self.state.state == 'SUCCESSFUL' + + @property + def resintall_os(self): + return self.host.reinstall_os + + @property + def reinstall_distributed_system(self): + return self.cluster.reinstall_distributed_system @property def os_installed(self): - host = self.host - if host: - return host.os_installed - else: - return None + return self.host.os_installed - @property + @hybrid_property def owner(self): - cluster = self.cluster - if cluster: - return cluster.owner - else: - return None + return self.cluster.owner + + @owner.expression + def owner(cls): + return cls.cluster.owner def state_dict(self): - state = self.state - if state.progress <= 0.0: - host = self.host - if host: - dict_info = host.state_dict() - else: - dict_info = {} - cluster = self.cluster - if cluster and cluster.distributed_system: - dict_info['state'] = state.state - else: - dict_info = state.to_dict() - return dict_info + cluster = self.cluster + host = self.host + if ( + not cluster.distributed_system or + host.state.state != 'SUCCESSFUL' + ): + return host.state_dict() + return self.state.to_dict() def to_dict(self): dict_info = self.host.to_dict() @@ -516,10 +543,10 @@ class ClusterHost(BASE, TimestampMixin, HelperMixin): dict_info.update({ 'distributed_system_name': self.distributed_system_name, 'distributed_system_installed': self.distributed_system_installed, - 'reinstall_distributed_system': ( - self.cluster.reinstall_distributed_system - ), + 'reinstall_distributed_system': self.reinstall_distributed_system, 'owner': self.owner, + 'clustername': self.clustername, + 'hostname': self.hostname, 'name': self.name }) return dict_info @@ -535,10 +562,11 @@ class HostState(BASE, StateMixin): primary_key=True ) - def initialize(self): + def update(self): + host = self.host if self.state == 'INSTALLING': - self.host.reinstall_os = False - super(HostState, self).initialize() + host.reinstall_os = False + super(HostState, self).update() class Host(BASE, TimestampMixin, HelperMixin): @@ -595,6 +623,7 @@ class Host(BASE, TimestampMixin, HelperMixin): @patched_os_config.setter def patched_os_config(self, value): self.os_config = util.merge_dict(dict(self.os_config), value) + self.config_validated = False @property def put_os_config(self): @@ -605,18 +634,29 @@ class Host(BASE, TimestampMixin, HelperMixin): os_config = dict(self.os_config) os_config.update(value) self.os_config = os_config + self.config_validated = False def __init__(self, id, **kwargs): self.id = id + self.name = str(self.id) + self.state = HostState() super(Host, self).__init__(**kwargs) def initialize(self): if not self.name: self.name = str(self.id) - if not self.state or self.reinstall_os: - self.state = HostState() super(Host, self).initialize() + def update(self): + if self.reinstall_os: + self.state = HostState() + os = self.os + if os: + self.os_name = os.name + else: + self.os_name = None + super(Host, self).update() + def validate(self): os = self.os if not os: @@ -627,7 +667,6 @@ class Host(BASE, TimestampMixin, HelperMixin): raise exception.InvalidParameter( 'os %s is not deployable' % os.name ) - self.os_name = os.name creator = self.creator if not creator: raise exception.InvalidParameter( @@ -637,31 +676,24 @@ class Host(BASE, TimestampMixin, HelperMixin): @hybrid_property def owner(self): - creator = self.creator - if creator: - return creator.email - else: - return None + return self.creator.email + + @owner.expression + def owner(cls): + return cls.creator.email @property def os_installed(self): - state = self.state - if state: - return state.state == 'SUCCESSFUL' - else: - return False + return self.state.state == 'SUCCESSFUL' def state_dict(self): - state = self.state - if state: - return state.to_dict() - else: - return {} + return self.state.to_dict() def to_dict(self): dict_info = self.machine.to_dict() dict_info.update(super(Host, self).to_dict()) dict_info.update({ + 'machine_id': self.machine.id, 'owner': self.owner, 'os_installed': self.os_installed, 'networks': [ @@ -681,50 +713,77 @@ class ClusterState(BASE, StateMixin): ForeignKey('cluster.id', onupdate='CASCADE', ondelete='CASCADE'), primary_key=True ) + total_hosts = Column( + Integer, + default=0 + ) + installing_hosts = Column( + Integer, + default=0 + ) + completed_hosts = Column( + Integer, + default=0 + ) + failed_hosts = Column( + Integer, + default=0 + ) - def initialize(self): + def to_dict(self): + dict_info = super(ClusterState, self).to_dict() + dict_info['status'] = { + 'total_hosts': self.total_hosts, + 'installing_hosts': self.installing_hosts, + 'completed_hosts': self.completed_hosts, + 'failed_hosts': self.failed_hosts + } + return dict_info + + def update(self): cluster = self.cluster + clusterhosts = cluster.clusterhosts + self.total_hosts = len(clusterhosts) + if self.state in ['UNINITIALIZED', 'INITIALIZED']: + self.installing_hosts = 0 + self.failed_hosts = 0 + self.completed_hosts = 0 if self.state == 'INSTALLING': cluster.reinstall_distributed_system = False - clusterhosts = cluster.clusterhosts - total_clusterhosts = 0 - failed_clusterhosts = 0 - installing_clusterhosts = 0 - finished_clusterhosts = 0 - progress = 0 - if not cluster.distributed_system: - for clusterhost in clusterhosts: - host = clusterhost.host - host_state = host.state.state - total_clusterhosts += 1 - progress += host.state.progress - if host_state == 'SUCCESSFUL': - finished_clusterhosts += 1 - elif host_state == 'INSTALLING': - installing_clusterhosts += 1 - elif host_state == 'ERROR': - failed_clusterhosts += 1 - else: - for clusterhost in clusterhosts: - clusterhost_state = clusterhost.state.state - total_clusterhosts += 1 - progress += clusterhost.state.progress - if clusterhost_state == 'SUCCESSFUL': - finished_clusterhosts += 1 - elif clusterhost_state == 'INSTALLING': - installing_clusterhosts += 1 - elif clusterhost_state == 'ERROR': - failed_clusterhosts += 1 - self.progress = progress / total_clusterhosts - self.message = ( - 'toal %s, installing %s, finished %s, error $s' - ) % ( - total_clusterhosts, installing_clusterhosts, - finished_clusterhosts, failed_clusterhosts - ) - if failed_clusterhosts: - self.severity = 'ERROR' - super(ClusterState, self).initialize() + if not cluster.distributed_system: + for clusterhost in clusterhosts: + host = clusterhost.host + host_state = host.state.state + if host_state == 'INSTALLING': + self.intsalling_hosts += 1 + elif host_state == 'ERROR': + self.failed_hosts += 1 + elif host_state == 'SUCCESSFUL': + self.completed_hosts += 1 + else: + for clusterhost in clusterhosts: + clusterhost_state = clusterhost.state.state + if clusterhost_state == 'INSTALLING': + self.intsalling_hosts += 1 + elif clusterhost_state == 'ERROR': + self.failed_hosts += 1 + elif clusterhost_state == 'SUCCESSFUL': + self.completed_hosts += 1 + if self.total_hosts: + self.percentage = ( + float(self.completed_hosts) + / + float(self.total_hosts) + ) + self.message = ( + 'toal %s, installing %s, complted: %s, error $s' + ) % ( + self.total_hosts, self.completed_hosts, + self.intsalling_hosts, self.failed_hosts + ) + if self.failed_hosts: + self.severity = 'ERROR' + super(ClusterState, self).update() class Cluster(BASE, TimestampMixin, HelperMixin): @@ -746,6 +805,8 @@ class Cluster(BASE, TimestampMixin, HelperMixin): ) os_config = Column(JSONEncoded, default={}) package_config = Column(JSONEncoded, default={}) + deployed_os_config = Column(JSONEncoded, default={}) + deployed_package_config = Column(JSONEncoded, default={}) config_validated = Column(Boolean, default=False) adapter_id = Column(Integer, ForeignKey('adapter.id')) adapter_name = Column(String(80), nullable=True) @@ -766,12 +827,39 @@ class Cluster(BASE, TimestampMixin, HelperMixin): def __init__(self, name, **kwargs): self.name = name + self.state = ClusterState() super(Cluster, self).__init__(**kwargs) def initialize(self): - if not self.state or self.reinstall_distributed_system: + adapter = self.adapter + if adapter: + self.put_package_config = { + 'roles': [role.name for role in adapter.roles] + } + + def update(self): + if self.reinstall_distributed_system: self.state = ClusterState() - super(Cluster, self).initialize() + os = self.os + if os: + self.os_name = os.name + else: + self.os_name = None + self.os_config = {} + adapter = self.adapter + if adapter: + self.adapter_name = adapter.name + self.distributed_system = adapter.adapter_distributed_system + self.distributed_system_name = self.distributed_system.name + self.put_package_config = { + 'roles': [role.name for role in adapter.roles] + } + else: + self.adapter_name = None + self.distributed_system = None + self.distributed_system_name = None + self.package_config = {} + super(Cluster, self).update() def validate(self): creator = self.creator @@ -780,14 +868,10 @@ class Cluster(BASE, TimestampMixin, HelperMixin): 'creator is not set in cluster %s' % self.id ) os = self.os - if os: - if not os.deployable: - raise exception.InvalidParameter( - 'os %s is not deployable' % os.name - ) - self.os_name = os.name - else: - self.os_name = None + if os and not os.deployable: + raise exception.InvalidParameter( + 'os %s is not deployable' % os.name + ) adapter = self.adapter if adapter: if not adapter.deployable: @@ -801,25 +885,13 @@ class Cluster(BASE, TimestampMixin, HelperMixin): raise exception.InvalidParameter( 'os %s is not supported' % os.name ) - self.adapter_name = adapter.name - distributed_system = ( - adapter.adapter_distributed_system - ) - self.distributed_system = distributed_system - if distributed_system: - if not distributed_system.deployable: - raise exception.InvalidParamerter( - 'distributed system %s is not deployable' % ( - distributed_system.name - ) + distributed_system = self.distributed_system + if distributed_system and not distributed_system.deployable: + raise exception.InvalidParamerter( + 'distributed system %s is not deployable' % ( + distributed_system.name ) - self.distributed_system_name = ( - distributed_system.name ) - else: - self.adapter_name = None - self.distributed_system = None - self.distributed_system_name = None super(Cluster, self).validate() @property @@ -829,6 +901,7 @@ class Cluster(BASE, TimestampMixin, HelperMixin): @patched_os_config.setter def patched_os_config(self, value): self.os_config = util.merge_dict(dict(self.os_config), value) + self.config_validated = False @property def put_os_config(self): @@ -839,6 +912,7 @@ class Cluster(BASE, TimestampMixin, HelperMixin): os_config = dict(self.os_config) os_config.update(value) self.os_config = os_config + self.config_validated = False @property def patched_package_config(self): @@ -846,7 +920,9 @@ class Cluster(BASE, TimestampMixin, HelperMixin): @patched_package_config.setter def patched_package_config(self, value): - self.package_config = util.merge_dict(dict(self.package_config), value) + package_config = dict(self.package_config) + self.package_config = util.merge_dict(package_config, value) + self.config_validated = False @property def put_package_config(self): @@ -857,29 +933,22 @@ class Cluster(BASE, TimestampMixin, HelperMixin): package_config = dict(self.package_config) package_config.update(value) self.package_config = package_config + self.config_validated = False @hybrid_property def owner(self): - creator = self.creator - if creator: - return creator.email - else: - return None + return self.creator.email + + @owner.expression + def owner(cls): + return cls.creator.email @property def distributed_system_installed(self): - state = self.state - if state: - return self.state.state == 'SUCCESSFUL' - else: - return False + return self.state.state == 'SUCCESSFUL' def state_dict(self): - state = self.state - if state: - return self.state.to_dict() - else: - return {} + return self.state.to_dict() def to_dict(self): dict_info = super(Cluster, self).to_dict() @@ -1660,54 +1729,6 @@ class Adapter(BASE, HelperMixin): else: return None - @property - def package_installer_name(self): - installer = self.adapter_package_installer - if installer: - return installer.name - else: - return None - - @property - def os_installer_name(self): - installer = self.adapter_os_installer - if installer: - return installer.name - else: - return None - - @property - def package_installer_type(self): - installer = self.adapter_package_installer - if installer: - return installer.installer_type - else: - return None - - @property - def os_installer_type(self): - installer = self.adapter_os_installer - if installer: - return installer.installer_type - else: - return None - - @property - def package_installer_config(self): - installer = self.adapter_package_installer - if installer: - return installer.config - else: - return None - - @property - def os_installer_config(self): - installer = self.adapter_os_installer - if installer: - return installer.config - else: - return None - @property def adapter_distributed_system(self): distributed_system = self.distributed_system @@ -1719,14 +1740,6 @@ class Adapter(BASE, HelperMixin): else: return None - @property - def distributed_system_name(self): - distributed_system = self.adapter_distributed_system - if distributed_system: - return distributed_system.name - else: - return None - @property def adapter_supported_oses(self): supported_oses = self.supported_oses @@ -1751,21 +1764,24 @@ class Adapter(BASE, HelperMixin): def to_dict(self): dict_info = super(Adapter, self).to_dict() - adapter_roles = self.adapter_roles - supported_oses = self.adapter_supported_oses dict_info.update({ - 'roles': [role.to_dict() for role in adapter_roles], - 'supported_oses': [ - adapter_os.to_dict() for adapter_os in supported_oses + 'roles': [ + role.to_dict() for role in self.adapter_roles + ], + 'supported_oses': [ + adapter_os.to_dict() + for adapter_os in self.adapter_supported_oses ], - 'distributed_system_name': self.distributed_system_name, - 'os_installer_name': self.os_installer_name, - 'os_installer_type': self.os_installer_type, - 'os_installer_config': self.os_installer_config, - 'package_installer_name': self.package_installer_name, - 'package_installer_type': self.package_installer_type, - 'package_installer_config': self.package_installer_config }) + distributed_system = self.distributed_system + if distributed_system: + dict_info['distributed_system_name'] = distributed_system.name + os_installer = self.adapter_os_installer + if os_installer: + dict_info['os_installer'] = os_installer.to_dict() + package_installer = self.adapter_package_installer + if package_installer: + dict_info['package_installer'] = package_installer.to_dict() return dict_info @@ -1817,8 +1833,8 @@ class OSInstaller(BASE, InstallerMixin): backref=backref('os_installer') ) - def __init__(self, name, **kwargs): - self.name = name + def __init__(self, instance_name, **kwargs): + self.instance_name = instance_name super(OSInstaller, self).__init__(**kwargs) @@ -1833,8 +1849,8 @@ class PackageInstaller(BASE, InstallerMixin): backref=backref('package_installer') ) - def __init__(self, name, **kwargs): - self.name = name + def __init__(self, instance_name, **kwargs): + self.instance_name = instance_name super(PackageInstaller, self).__init__(**kwargs) @@ -1857,10 +1873,10 @@ class Network(BASE, TimestampMixin, HelperMixin): self.subnet = subnet super(Network, self).__init__(**kwargs) - def intialize(self): + def initialize(self): if not self.name: self.name = self.subnet - super(Network, self).intialize() + super(Network, self).initialize() def validate(self): try: diff --git a/compass/tasks/tasks.py b/compass/tasks/tasks.py index 19dcf05b..1a6b5d79 100644 --- a/compass/tasks/tasks.py +++ b/compass/tasks/tasks.py @@ -46,7 +46,10 @@ def global_celery_init(**_): @celery.task(name='compass.tasks.pollswitch') -def pollswitch(ip_addr, credentials, req_obj='mac', oper='SCAN'): +def pollswitch( + poller_email, ip_addr, credentials, + req_obj='mac', oper='SCAN' +): """Query switch and return expected result. :param ip_addr: switch ip address. @@ -60,14 +63,15 @@ def pollswitch(ip_addr, credentials, req_obj='mac', oper='SCAN'): """ try: poll_switch.poll_switch( - ip_addr, credentials, req_obj=req_obj, oper=oper + poller_email, ip_addr, credentials, + req_obj=req_obj, oper=oper ) except Exception as error: logging.exception(error) @celery.task(name='compass.tasks.deploy_cluster') -def deploy_cluster(cluster_id, clusterhost_ids): +def deploy_cluster(deployer_email, cluster_id, clusterhost_ids): """Deploy the given cluster. :param cluster_hosts: the cluster and hosts of each cluster to deploy. @@ -77,7 +81,7 @@ def deploy_cluster(cluster_id, clusterhost_ids): @celery.task(name='compass.tasks.reinstall_cluster') -def reinstall_cluster(cluster_id, clusterhost_ids): +def reinstall_cluster(installer_email, cluster_id, clusterhost_ids): """reinstall the given cluster. :param cluster_hosts: the cluster and hosts of each cluster to reinstall. diff --git a/compass/tests/db/api/test_utils.py b/compass/tests/db/api/test_utils.py index 1b058745..0b96b467 100644 --- a/compass/tests/db/api/test_utils.py +++ b/compass/tests/db/api/test_utils.py @@ -145,11 +145,9 @@ class TestModelFilter(unittest2.TestCase): expected, ret = self._filter_test_dict_util( 'gt', 'update_clusterhost_state', - 48, - id=47 + 49, + id=48 ) - print 'expected: %s' % expected - print 'ret: %s' % ret self.assertTrue( all(item in ret[0].items() for item in expected.items()) ) @@ -169,8 +167,8 @@ class TestModelFilter(unittest2.TestCase): expected, ret = self._filter_test_dict_util( 'ge', 'update_clusterhost_state', - 48, - id=48 + 49, + id=49 ) print 'expected: %s' % expected print 'ret: %s' % ret diff --git a/conf/os_field/general_list.conf b/conf/os_field/general_list.conf new file mode 100644 index 00000000..9b37cb99 --- /dev/null +++ b/conf/os_field/general_list.conf @@ -0,0 +1,2 @@ +NAME = 'general_list' +FIELD_TYPE = list diff --git a/conf/os_installer/cobbler.conf b/conf/os_installer/cobbler.conf index 564673b9..92b0f795 100644 --- a/conf/os_installer/cobbler.conf +++ b/conf/os_installer/cobbler.conf @@ -1,6 +1,6 @@ NAME = 'cobbler' -TYPE = 'cobbler' -CONFIG = { +INSTANCE_NAME = 'cobbler' +SETTINGS = { 'url': 'http://127.0.0.1/cobbler_api', 'token': ('cobbler', 'cobbler') } diff --git a/conf/os_metadata/general.conf b/conf/os_metadata/general.conf index 920b63f0..20cddb8e 100644 --- a/conf/os_metadata/general.conf +++ b/conf/os_metadata/general.conf @@ -8,14 +8,77 @@ METADATA = { '_self': { 'field': 'general', 'default_value': 'EN', - 'options': ['EN'], + 'options': ['EN', 'CN'], } }, 'timezone': { '_self': { 'field': 'general', - 'default_value': 'PDT', - 'options': ['PDT'], + 'default_value': 'GMT -8:00', + 'options': [ + 'GMT -12:00', 'GMT -11:00', 'GMT -10:00', 'GMT -9:00', + 'GMT -8:00', 'GMT -7:00', 'GMT -6:00', 'GMT -5:00', + 'GMT -4:00', 'GMT -3:00', 'GMT -2:00', 'GMT -1:00', + 'GMT 0:00', 'GMT +1:00', 'GMT +2:00', 'GMT +3:00', + 'GMT +4:00', 'GMT +5:00', 'GMT +6:00', 'GMT +7:00', + 'GMT +8:00', 'GMT +9:00', 'GMT +10:00', 'GMT +11:00', + 'GMT +12:00' + ], + } + }, + 'http_proxy': { + '_self': { + 'field': 'general', + 'default_value': 'http://10.145.88.211:3128', + 'options': [ + 'http://10.145.88.211:3128' + ], + } + }, + 'https_proxy': { + '_self': { + 'field': 'general', + 'default_value': 'http://10.145.88.211:3128', + 'options': [ + 'http://10.145.88.211:3128' + ], + } + }, + 'no_proxy': { + '_self': { + 'field': 'general_list', + 'default_value': [ + '127.0.0.1', + 'compass', + '10.145.88.211' + ], + 'options': [ + '127.0.0.1', + 'compass', + '10.145.88.211' + ] + } + }, + 'ntp_server': { + '_self': { + 'is_required': True, + 'field': 'general', + 'default_value': '10.145.88.211', + 'options': [ + '10.145.88.211' + ] + } + }, + 'dns_servers': { + '_self': { + 'is_required': True, + 'field': 'general_list', + 'default_value': [ + '10.145.88.211', + ], + 'options': [ + '10.145.88.211' + ] } }, 'domain': { @@ -26,6 +89,15 @@ METADATA = { 'options': ['ods.com'], } }, + 'search_path': { + '_self': { + 'field': 'general_list', + 'default_value': [ + 'ods.com' + ], + 'options': ['ods.com'] + } + }, 'default_gateway': { '_self': { 'is_required': True, diff --git a/conf/package_field/roles.conf b/conf/package_field/roles.conf new file mode 100644 index 00000000..a0319ed2 --- /dev/null +++ b/conf/package_field/roles.conf @@ -0,0 +1,3 @@ +NAME = 'roles' +FIELD_TYPE = list +DESCRIPTION = 'roles' diff --git a/conf/package_installer/chef-icehouse.conf b/conf/package_installer/chef-icehouse.conf index e67f3a57..c932f8dd 100644 --- a/conf/package_installer/chef-icehouse.conf +++ b/conf/package_installer/chef-icehouse.conf @@ -1,5 +1,5 @@ -NAME = 'chef(icehouse)' -TYPE = 'chef' -CONFIG = { - 'url': 'https://127.0.0.1', +NAME = 'chef' +INSTANCE_NAME = 'chef(icehouse)' +SETTINGS = { + 'url': 'https://127.0.0.1' } diff --git a/conf/package_metadata/openstack.conf b/conf/package_metadata/openstack.conf index 3598bd36..c27246e2 100644 --- a/conf/package_metadata/openstack.conf +++ b/conf/package_metadata/openstack.conf @@ -4,19 +4,28 @@ METADATA = { '_self': { 'required_in_whole_config': True, }, - '$credentials': { - 'username': { - '_self': { - 'is_required': True, - 'field': 'username', - } - }, - 'password': { - '_self': { - 'is_required': True, - 'field': 'password' + '$credential_type': { + '$credential': { + 'username': { + '_self': { + 'is_required': True, + 'field': 'username', + } + }, + 'password': { + '_self': { + 'is_required': True, + 'field': 'password' + } } } + } + }, + 'roles': { + '_self': { + 'required_in_whole_config': True, + 'field': 'roles', + 'options': [], }, }, 'network_mapping': { @@ -24,11 +33,9 @@ METADATA = { 'required_in_whole_config': True }, '$interface_type': { - 'interface': { - '_self': { - 'is_required': True, - 'field': 'general' - } + '_self': { + 'is_required': True, + 'field': 'general' } } }