Merge "update api code" into dev/experimental

bin/refresh.sh

@@ -1,14 +1,9 @@
 #!/bin/bash
-/opt/compass/bin/manage_db.py checkdb
-if [[ "$?" == "0" ]]; then
-/opt/compass/bin/manage_db.py clean_clusters
-fi
 /opt/compass/bin/manage_db.py createdb
-/opt/compass/bin/manage_db.py sync_switch_configs 
-/opt/compass/bin/manage_db.py sync_from_installers
 service httpd restart
 service rsyslog restart
 service redis restart
 redis-cli flushall
-service compassd restart
+service compass-celeryd restart
+service compass-progress-updated restart
 

compass/actions/poll_switch.py

@@ -19,6 +19,7 @@ import netaddr
 from compass.actions import util
 from compass.db.api import database
 from compass.db.api import switch as switch_api
+from compass.db.api import user as user_api
 from compass.hdsdiscovery.hdmanager import HDManager
 
 
@@ -71,18 +72,19 @@ def _poll_switch(ip_addr, credentials, req_obj='mac', oper="SCAN"):
             {}
         )
 
+    logging.info('poll switch result: %s' % str(results))
     state = under_monitoring
     machine_dicts = {}
     for machine in results:
         mac = machine['mac']
         port = machine['port']
-        vlan = machine['vlan']
+        vlan = int(machine['vlan'])
         if vlan:
             vlans = [vlan]
         else:
             vlans = []
         if mac not in machine_dicts:
-            machine_dicts[mac] = {'port': port, 'vlans': vlans}
+            machine_dicts[mac] = {'mac': mac, 'port': port, 'vlans': vlans}
         else:
             machine_dicts[mac]['port'] = port
             machine_dicts[mac]['vlans'].extend(vlans)
@@ -90,11 +92,14 @@ def _poll_switch(ip_addr, credentials, req_obj='mac', oper="SCAN"):
     logging.debug('update switch %s state to under monitoring', ip_addr)
     return (
         {'vendor': vendor, 'state': state, 'err_msg': err_msg},
-        machine_dicts
+        machine_dicts.values()
     )
 
 
-def poll_switch(ip_addr, credentials, req_obj='mac', oper="SCAN"):
+def poll_switch(
+    poller_email, ip_addr, credentials,
+    req_obj='mac', oper="SCAN"
+):
     """Query switch and update switch machines.
 
     .. note::
@@ -113,6 +118,8 @@ def poll_switch(ip_addr, credentials, req_obj='mac', oper="SCAN"):
     .. note::
        The function should be called out of database session scope.
     """
+    poller = user_api.get_user_object(poller_email)
+    ip_int = long(netaddr.IPAddress(ip_addr))
     with util.lock('poll switch %s' % ip_addr) as lock:
         if not lock:
             raise Exception(
@@ -120,21 +127,22 @@ def poll_switch(ip_addr, credentials, req_obj='mac', oper="SCAN"):
             )
 
         logging.debug('poll switch: %s', ip_addr)
-        ip_int = long(netaddr.IPAddress(ip_addr))
         switch_dict, machine_dicts = _poll_switch(
             ip_addr, credentials, req_obj=req_obj, oper=oper
         )
-        with database.session() as session:
+        switches = switch_api.list_switches(
-            switch = switch_api.get_switch_internal(
+            poller, ip_int=ip_int
-                session, False, ip_int=ip_int
         )
-            if not switch:
+        if not switches:
             logging.error('no switch found for %s', ip_addr)
             return
 
-            switch_api.update_switch_internal(
+        for switch in switches:
-                session, switch, **switch_dict
+            switch_api.update_switch(
+                poller, switch['id'], **switch_dict
             )
-            switch_api.add_switch_machines_internal(
+            for machine_dict in machine_dicts:
-                session, switch, machine_dicts, False
+                print 'add machine: %s' % machine_dict
+                switch_api.add_switch_machine(
+                    poller, switch['id'], False, **machine_dict
                 )

compass/api/api.py

@@ -159,13 +159,13 @@ def _login(use_cookie):
         raise exception_handler.BadRequest(
             'missing email or password in data'
         )
-    if 'expires' not in data:
+    if 'expire_timestamp' not in data:
         expire_timestamp = (
             datetime.datetime.now() + app.config['REMEMBER_COOKIE_DURATION']
         )
     else:
         expire_timestamp = util.parse_datetime(
-            data['expires'], exception_handler.BadRequest
+            data['expire_timestamp'], exception_handler.BadRequest
         )
 
     data['expire_timestamp'] = expire_timestamp
@@ -559,7 +559,6 @@ def update_switch(switch_id):
 def patch_switch(switch_id):
     """patch switch."""
     data = _get_request_data()
-    _replace_data(data, {'credentials': 'patched_credentials'})
     return utils.make_json_response(
         200,
         switch_api.patch_switch(current_user, switch_id, **data)
@@ -622,7 +621,6 @@ def update_switch_filters(switch_id):
 def patch_switch_filters(switch_id):
     """patch switch filters."""
     data = _get_request_data()
-    _replace_data(data, {'filters': 'patched_filters'})
     return utils.make_json_response(
         200,
         switch_api.patch_switch_filter(current_user, switch_id, **data)
@@ -790,15 +788,6 @@ def update_switch_machine(switch_id, machine_id):
 def patch_switch_machine(switch_id, machine_id):
     """patch switch machine."""
     data = _get_request_data()
-    _replace_data(
-        data,
-        {
-            'vlans': 'patched_vlans',
-            'ipmi_credentials': 'patched_ipmi_credentials',
-            'tag': 'patched_tag',
-            'location': 'patched_location'
-        }
-    )
     return utils.make_json_response(
         200,
         switch_api.patch_switch_machine(
@@ -858,7 +847,6 @@ def list_switchmachines():
     """List switch machines."""
     data = _get_request_args()
     _filter_ip(data)
-    _replace_data(data, {'ip_int': 'switch_ip_int'})
     _filter_port(data)
     _filter_general(data, 'vlans')
     _filter_tag(data)
@@ -878,7 +866,6 @@ def list_switchmachines_hosts():
     """List switch machines or hosts."""
     data = _get_request_args()
     _filter_ip(data)
-    _replace_data(data, {'ip_int': 'switch_ip_int'})
     _filter_port(data)
     _filter_general(data, 'vlans')
     _filter_tag(data)
@@ -933,15 +920,6 @@ def update_switchmachine(switch_machine_id):
 def patch_switchmachine(switch_machine_id):
     """patch switch machine."""
     data = _get_request_data()
-    _replace_data(
-        data,
-        {
-            'vlans': 'patched_vlans',
-            'ipmi_credentials': 'patched_ipmi_credentials',
-            'tag': 'patched_tag',
-            'location': 'patched_location'
-        }
-    )
     return utils.make_json_response(
         200,
         switch_api.patch_switchmachine(
@@ -1014,14 +992,6 @@ def update_machine(machine_id):
 def patch_machine(machine_id):
     """patch machine."""
     data = _get_request_data()
-    _replace_data(
-        data,
-        {
-            'ipmi_credentials': 'patched_ipmi_credentials',
-            'tag': 'patched_tag',
-            'location': 'patched_location'
-        }
-    )
     return utils.make_json_response(
         200,
         machine_api.patch_machine(
@@ -1288,13 +1258,6 @@ def show_cluster_metadata(cluster_id):
 def update_cluster_config(cluster_id):
     """update cluster config."""
     data = _get_request_data()
-    _replace_data(
-        data,
-        {
-            'os_config': 'put_os_config',
-            'package_config': 'put_os_config'
-        }
-    )
     return utils.make_json_response(
         200,
         cluster_api.update_cluster_config(current_user, cluster_id, **data)
@@ -1307,13 +1270,6 @@ def update_cluster_config(cluster_id):
 def patch_cluster_config(cluster_id):
     """patch cluster config."""
     data = _get_request_data()
-    _replace_data(
-        data,
-        {
-            'os_config': 'patched_os_config',
-            'package_config': 'patched_package_config'
-        }
-    )
     return utils.make_json_response(
         200,
         cluster_api.patch_cluster_config(current_user, cluster_id, **data)
@@ -1524,12 +1480,6 @@ def show_clusterhost_config(clusterhost_id):
 def update_cluster_host_config(cluster_id, host_id):
     """update clusterhost config."""
     data = _get_request_data()
-    _replace_data(
-        data,
-        {
-            'package_config': 'put_os_config'
-        }
-    )
     return utils.make_json_response(
         200,
         cluster_api.update_cluster_host_config(
@@ -1544,12 +1494,6 @@ def update_cluster_host_config(cluster_id, host_id):
 def update_clusterhost_config(clusterhost_id):
     """update clusterhost config."""
     data = _get_request_data()
-    _replace_data(
-        data,
-        {
-            'package_config': 'put_os_config'
-        }
-    )
     return utils.make_json_response(
         200,
         cluster_api.update_clusterhost_config(
@@ -1567,12 +1511,6 @@ def update_clusterhost_config(clusterhost_id):
 def patch_cluster_host_config(cluster_id, host_id):
     """patch clusterhost config."""
     data = _get_request_data()
-    _replace_data(
-        data,
-        {
-            'package_config': 'patched_package_config'
-        }
-    )
     return utils.make_json_response(
         200,
         cluster_api.patch_cluster_host_config(
@@ -1587,12 +1525,6 @@ def patch_cluster_host_config(cluster_id, host_id):
 def patch_clusterhost_config(clusterhost_id):
     """patch clusterhost config."""
     data = _get_request_data()
-    _replace_data(
-        data,
-        {
-            'package_config': 'patched_package_config'
-        }
-    )
     return utils.make_json_response(
         200,
         cluster_api.patch_clusterhost_config(
@@ -1816,12 +1748,6 @@ def show_host_config(host_id):
 def update_host_config(host_id):
     """update host config."""
     data = _get_request_data()
-    _replace_data(
-        data,
-        {
-            'os_config': 'put_os_config',
-        }
-    )
     return utils.make_json_response(
         200,
         host_api.update_host_config(current_user, host_id, **data)
@@ -1834,12 +1760,6 @@ def update_host_config(host_id):
 def patch_host_config(host_id):
     """patch host config."""
     data = _get_request_data()
-    _replace_data(
-        data,
-        {
-            'os_config': 'patched_os_config',
-        }
-    )
     return utils.make_json_response(
         200,
         host_api.patch_host_config(current_user, host_id, **data)
@@ -1882,16 +1802,19 @@ def list_hostnetworks():
     )
 
 
-@app.route("/hosts/<int:host_id>/networks/<int:subnet_id>", methods=['GET'])
+@app.route(
+    "/hosts/<int:host_id>/networks/<int:host_network_id>",
+    methods=['GET']
+)
 @log_user_action
 @login_required
-def show_host_network(host_id, subnet_id):
+def show_host_network(host_id, host_network_id):
     """Get host network."""
     data = _get_request_args()
     return utils.make_json_response(
         200,
         host_api.get_host_network(
-            current_user, host_id, subnet_id, **data
+            current_user, host_id, host_network_id, **data
         )
     )
 
@@ -1921,16 +1844,19 @@ def add_host_network(host_id):
     )
 
 
-@app.route("/hosts/<int:host_id>/networks/<int:subnet_id>", methods=['PUT'])
+@app.route(
+    "/hosts/<int:host_id>/networks/<int:host_network_id>",
+    methods=['PUT']
+)
 @log_user_action
 @login_required
-def update_host_network(host_id, subnet_id):
+def update_host_network(host_id, host_network_id):
     """update host network."""
     data = _get_request_data()
     return utils.make_json_response(
         200,
         host_api.update_host_network(
-            current_user, host_id, subnet_id, **data
+            current_user, host_id, host_network_id, **data
         )
     )
 
@@ -1950,18 +1876,18 @@ def update_hostnetwork(host_network_id):
 
 
 @app.route(
-    "/hosts/<int:host_id>/networks/<int:subnet_id>",
+    "/hosts/<int:host_id>/networks/<int:host_network_id>",
     methods=['DELETE']
 )
 @log_user_action
 @login_required
-def delete_host_network(host_id, subnet_id):
+def delete_host_network(host_id, host_network_id):
     """Delete host network."""
     data = _get_request_data()
     return utils.make_json_response(
         200,
         host_api.del_host_network(
-            current_user, host_id, subnet_id, **data
+            current_user, host_id, host_network_id, **data
         )
     )
 

compass/db/api/adapter.py

@@ -73,14 +73,14 @@ def add_adapters_internal(session):
         if 'OS_INSTALLER' in config:
             os_installer = utils.get_db_object(
                 session, models.OSInstaller,
-                name=config['OS_INSTALLER']
+                instance_name=config['OS_INSTALLER']
             )
         else:
             os_installer = None
         if 'PACKAGE_INSTALLER' in config:
             package_installer = utils.get_db_object(
                 session, models.PackageInstaller,
-                name=config['PACKAGE_INSTALLER']
+                instance_name=config['PACKAGE_INSTALLER']
             )
         else:
             package_installer = None

compass/db/api/adapter_holder.py

@@ -26,19 +26,17 @@ from compass.db import exception
 SUPPORTED_FIELDS = [
     'name',
     'distributed_system_name',
-    'os_installer_name',
-    'package_installer_name',
 ]
 RESP_FIELDS = [
-    'id', 'name', 'roles', 'os_installer_name',
+    'id', 'name', 'roles',
-    'package_installer_name', 'distributed_system_name',
+    'distributed_system_name',
     'supported_oses', 'display_name'
 ]
 RESP_OS_FIELDS = [
     'id', 'os_id', 'name'
 ]
 RESP_ROLES_FIELDS = [
-    'id', 'name', 'description', 'optional'
+    'id', 'name', 'display_name', 'description', 'optional'
 ]
 
 

compass/db/api/cluster.py

@@ -8,11 +8,13 @@
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
+
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 """Cluster database operations."""
+import functools
 import logging
 
 from compass.db.api import database
@@ -37,7 +39,8 @@ RESP_FIELDS = [
     'created_at', 'updated_at'
 ]
 RESP_CLUSTERHOST_FIELDS = [
-    'id', 'host_id', 'machine_id', 'name', 'cluster_id',
+    'id', 'host_id', 'machine_id', 'name', 'hostname',
+    'cluster_id', 'clustername',
     'mac', 'os_installed', 'distributed_system_installed',
     'os_name', 'distributed_system_name',
     'reinstall_os', 'reinstall_distributed_system',
@@ -52,6 +55,12 @@ RESP_CONFIG_FIELDS = [
     'created_at',
     'updated_at'
 ]
+RESP_DEPLOYED_CONFIG_FIELDS = [
+    'deployed_os_config',
+    'deployed_package_config',
+    'created_at',
+    'updated_at'
+]
 RESP_METADATA_FIELDS = [
     'os_config',
     'package_config'
@@ -65,12 +74,19 @@ RESP_CLUSTERHOST_CONFIG_FIELDS = [
     'created_at',
     'updated_at'
 ]
+RESP_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS = [
+    'deployed_os_config',
+    'deployed_package_config',
+    'created_at',
+    'updated_at'
+]
 RESP_STATE_FIELDS = [
-    'id', 'state', 'progress', 'message',
+    'id', 'state', 'percentage', 'message',
+    'status',
     'created_at', 'updated_at'
 ]
 RESP_CLUSTERHOST_STATE_FIELDS = [
-    'id', 'state', 'progress', 'message',
+    'id', 'state', 'percentage', 'message',
     'created_at', 'updated_at'
 ]
 RESP_REVIEW_FIELDS = [
@@ -87,17 +103,29 @@ UPDATED_HOST_FIELDS = ['name', 'reinstall_os']
 UPDATED_CONFIG_FIELDS = [
     'put_os_config', 'put_package_config', 'config_step'
 ]
+UPDATED_DEPLOYED_CONFIG_FIELDS = [
+    'deployed_os_config', 'deployed_package_config'
+]
 PATCHED_CONFIG_FIELDS = [
     'patched_os_config', 'patched_package_config', 'config_step'
 ]
 UPDATED_CLUSTERHOST_CONFIG_FIELDS = [
+    'put_os_config',
     'put_package_config'
 ]
 PATCHED_CLUSTERHOST_CONFIG_FIELDS = [
+    'patched_os_config',
     'patched_package_config'
 ]
+UPDATED_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS = [
+    'deployed_os_config',
+    'deployed_package_config'
+]
 UPDATED_CLUSTERHOST_STATE_FIELDS = [
-    'state', 'progress', 'message'
+    'state', 'percentage', 'message'
+]
+UPDATED_CLUSTER_STATE_FIELDS = [
+    'state'
 ]
 
 
@@ -120,10 +148,13 @@ def list_clusters(session, lister, **filters):
     permission.PERMISSION_LIST_CLUSTERS
 )
 @utils.wrap_to_dict(RESP_FIELDS)
-def get_cluster(session, getter, cluster_id, **kwargs):
+def get_cluster(
+    session, getter, cluster_id,
+    exception_when_missing=True, **kwargs
+):
     """Get cluster info."""
     return utils.get_db_object(
-        session, models.Cluster, id=cluster_id
+        session, models.Cluster, exception_when_missing, id=cluster_id
     )
 
 
@@ -143,16 +174,15 @@ def is_cluster_validated(
         raise exception.Forbidden(
             'cluster %s is not validated' % cluster.name
         )
-    for clusterhost in cluster.clusterhsots:
+
+
+def is_clusterhost_validated(
+    session, clusterhost
+):
     if not clusterhost.config_validated:
         raise exception.Forbidden(
             'clusterhost %s is not validated' % clusterhost.name
         )
-        host = clusterhost.host
-        if not host.config_validated:
-            raise exception.Forbidden(
-                'host %s is not validated' % host.name
-            )
 
 
 def is_cluster_editable(
@@ -165,7 +195,10 @@ def is_cluster_editable(
             return _conditional_exception(
                 cluster, exception_when_not_editable
             )
-    elif not cluster.reinstall_distributed_system:
+    elif (
+        cluster.distributed_system and
+        not cluster.reinstall_distributed_system
+    ):
         return _conditional_exception(
             cluster, exception_when_not_editable
         )
@@ -184,11 +217,15 @@ def is_cluster_editable(
     permission.PERMISSION_ADD_CLUSTER
 )
 @utils.wrap_to_dict(RESP_FIELDS)
-def add_cluster(session, creator, name, adapter_id, **kwargs):
+def add_cluster(
+    session, creator,
+    exception_when_existing=True,
+    name=None, **kwargs
+):
     """Create a cluster."""
     return utils.add_db_object(
-        session, models.Cluster, True,
+        session, models.Cluster, exception_when_existing,
-        name, creator_id=creator.id, adapter_id=adapter_id,
+        name, creator_id=creator.id,
         **kwargs
     )
 
@@ -241,6 +278,19 @@ def get_cluster_config(session, getter, cluster_id, **kwargs):
     )
 
 
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission_in_session(
+    permission.PERMISSION_LIST_CLUSTER_CONFIG
+)
+@utils.wrap_to_dict(RESP_DEPLOYED_CONFIG_FIELDS)
+def get_cluster_deployed_config(session, getter, cluster_id, **kwargs):
+    """Get cluster deployed config."""
+    return utils.get_db_object(
+        session, models.Cluster, id=cluster_id
+    )
+
+
 @utils.supported_filters([])
 @database.run_in_session()
 @user_api.check_user_permission_in_session(
@@ -270,25 +320,40 @@ def get_cluster_metadata(session, getter, cluster_id, **kwargs):
     permission.PERMISSION_ADD_CLUSTER_CONFIG
 )
 @utils.wrap_to_dict(RESP_CONFIG_FIELDS)
-def update_cluster_config_internal(session, updater, cluster, **kwargs):
+def _update_cluster_config(session, updater, cluster, **kwargs):
     """Update a cluster config."""
     is_cluster_editable(session, cluster, updater)
-    utils.update_db_object(
+    return utils.update_db_object(
         session, cluster, config_validated=False, **kwargs
     )
-    os_config = cluster.os_config
-    if os_config:
-        metadata_api.validate_os_config(
-            os_config, cluster.os_id
-        )
-    package_config = cluster.package_config
-    if package_config:
-        metadata_api.validate_package_config(
-            package_config, cluster.adapter_id
-        )
-    return cluster
 
 
+@utils.supported_filters(
+    optional_support_keys=UPDATED_DEPLOYED_CONFIG_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission_in_session(
+    permission.PERMISSION_ADD_CLUSTER_CONFIG
+)
+@utils.wrap_to_dict(RESP_DEPLOYED_CONFIG_FIELDS)
+def update_cluster_deployed_config(
+    session, updater, cluster_id, **kwargs
+):
+    """Update cluster deployed config."""
+    cluster = utils.get_db_object(
+        session, models.Cluster, id=cluster_id
+    )
+    is_cluster_editable(session, cluster, updater)
+    is_cluster_validated(session, cluster)
+    return utils.update_db_object(
+        session, cluster, **kwargs
+    )
+
+
+@utils.replace_filters(
+    os_config='put_os_config',
+    package_config='put_package_config'
+)
 @utils.supported_filters(optional_support_keys=UPDATED_CONFIG_FIELDS)
 @database.run_in_session()
 def update_cluster_config(session, updater, cluster_id, **kwargs):
@@ -296,11 +361,31 @@ def update_cluster_config(session, updater, cluster_id, **kwargs):
     cluster = utils.get_db_object(
         session, models.Cluster, id=cluster_id
     )
-    return update_cluster_config_internal(
+    os_config_validates = functools.partial(
-        session, updater, cluster, **kwargs
+        metadata_api.validate_os_config, os_id=cluster.os_id)
+    package_config_validates = functools.partial(
+        metadata_api.validate_package_config, adapter_id=cluster.adapter_id)
+
+    @utils.input_validates(
+        put_os_config=os_config_validates,
+        put_package_config=package_config_validates
+    )
+    def update_config_internal(
+        cluster, **in_kwargs
+    ):
+        return _update_cluster_config(
+            session, updater, cluster, **in_kwargs
+        )
+
+    return update_config_internal(
+        cluster, **kwargs
     )
 
 
+@utils.replace_filters(
+    os_config='patched_os_config',
+    package_config='patched_package_config'
+)
 @utils.supported_filters(optional_support_keys=PATCHED_CONFIG_FIELDS)
 @database.run_in_session()
 def patch_cluster_config(session, updater, cluster_id, **kwargs):
@@ -308,8 +393,23 @@ def patch_cluster_config(session, updater, cluster_id, **kwargs):
     cluster = utils.get_db_object(
         session, models.Cluster, id=cluster_id
     )
-    return update_cluster_config_internal(
+
-        session, updater, cluster, **kwargs
+    os_config_validates = functools.partial(
+        metadata_api.validate_os_config, os_id=cluster.os_id)
+    package_config_validates = functools.partial(
+        metadata_api.validate_package_config, adapter_id=cluster.adapter_id)
+
+    @utils.output_validates(
+        os_config=os_config_validates,
+        package_config=package_config_validates
+    )
+    def update_config_internal(cluster, **in_kwargs):
+        return _update_cluster_config(
+            session, updater, cluster, **in_kwargs
+        )
+
+    return update_config_internal(
+        cluster, **kwargs
     )
 
 
@@ -363,32 +463,32 @@ def add_clusterhost_internal(
             creator=cluster.creator,
             **kwargs
         )
-    return utils.add_db_object(
+    utils.add_db_object(
         session, models.ClusterHost, exception_when_existing,
         cluster.id, machine_id
     )
 
 
-def _add_clusterhosts(session, cluster, machine_dicts):
+def _add_clusterhosts(session, cluster, machines):
-    for machine_dict in machine_dicts:
+    for machine_dict in machines:
         add_clusterhost_internal(
             session, cluster, **machine_dict
         )
 
 
-def _remove_clusterhosts(session, cluster, host_ids):
+def _remove_clusterhosts(session, cluster, hosts):
     utils.del_db_objects(
         session, models.ClusterHost,
-        cluster_id=cluster.id, host_id=host_ids
+        cluster_id=cluster.id, host_id=hosts
     )
 
 
-def _set_clusterhosts(session, cluster, machine_dicts):
+def _set_clusterhosts(session, cluster, machines):
     utils.del_db_objects(
         session, models.ClusterHost,
         cluster_id=cluster.id
     )
-    for machine_dict in machine_dicts:
+    for machine_dict in machines:
         add_clusterhost_internal(
             session, cluster, True, **machine_dict
         )
@@ -427,10 +527,14 @@ def list_clusterhosts(session, lister, **filters):
     permission.PERMISSION_LIST_CLUSTERHOSTS
 )
 @utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS)
-def get_cluster_host(session, getter, cluster_id, host_id, **kwargs):
+def get_cluster_host(
+    session, getter, cluster_id, host_id,
+    exception_when_missing=True, **kwargs
+):
     """Get clusterhost info."""
     return utils.get_db_object(
         session, models.ClusterHost,
+        exception_when_missing,
         cluster_id=cluster_id, host_id=host_id
     )
 
@@ -441,10 +545,15 @@ def get_cluster_host(session, getter, cluster_id, host_id, **kwargs):
     permission.PERMISSION_LIST_CLUSTERHOSTS
 )
 @utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS)
-def get_clusterhost(session, getter, clusterhost_id, **kwargs):
+def get_clusterhost(
+    session, getter, clusterhost_id,
+    exception_when_missing=True, **kwargs
+):
     """Get clusterhost info."""
     return utils.get_db_object(
-        session, models.ClusterHost, id=clusterhost_id
+        session, models.ClusterHost,
+        exception_when_missing,
+        id=clusterhost_id
     )
 
 
@@ -457,14 +566,17 @@ def get_clusterhost(session, getter, clusterhost_id, **kwargs):
     permission.PERMISSION_UPDATE_CLUSTER_HOSTS
 )
 @utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS)
-def add_cluster_host(session, creator, cluster_id, machine_id, **kwargs):
+def add_cluster_host(
+    session, creator, cluster_id,
+    exception_when_existing=True, **kwargs
+):
     """Add cluster host."""
     cluster = utils.get_db_object(
         session, models.Cluster, id=cluster_id
     )
     return add_clusterhost_internal(
-        session, cluster, True,
+        session, cluster, exception_when_existing,
-        machine_id=machine_id, **kwargs
+        **kwargs
     )
 
 
@@ -516,6 +628,22 @@ def get_cluster_host_config(session, getter, cluster_id, host_id, **kwargs):
     )
 
 
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission_in_session(
+    permission.PERMISSION_LIST_CLUSTERHOST_CONFIG
+)
+@utils.wrap_to_dict(RESP_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS)
+def get_cluster_host_deployed_config(
+    session, getter, cluster_id, host_id, **kwargs
+):
+    """Get clusterhost deployed config."""
+    return utils.get_db_object(
+        session, models.ClusterHost,
+        cluster_id=cluster_id, host_id=host_id
+    )
+
+
 @utils.supported_filters([])
 @database.run_in_session()
 @user_api.check_user_permission_in_session(
@@ -529,28 +657,107 @@ def get_clusterhost_config(session, getter, clusterhost_id, **kwargs):
     )
 
 
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission_in_session(
+    permission.PERMISSION_LIST_CLUSTERHOST_CONFIG
+)
+@utils.wrap_to_dict(RESP_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS)
+def get_clusterhost_deployed_config(session, getter, clusterhost_id, **kwargs):
+    """Get clusterhost deployed config."""
+    return utils.get_db_object(
+        session, models.ClusterHost, id=clusterhost_id
+    )
+
+
 @user_api.check_user_permission_in_session(
     permission.PERMISSION_ADD_CLUSTERHOST_CONFIG
 )
 @utils.wrap_to_dict(RESP_CLUSTERHOST_CONFIG_FIELDS)
-def update_clusterhost_config_internal(
+def _update_clusterhost_config(session, updater, clusterhost, **kwargs):
+    from compass.db.api import host as host_api
+    ignore_keys = []
+    if host_api.is_host_editable(
+        session, clusterhost.host, updater,
+        exception_when_not_editable=False
+    ):
+        ignore_keys.append('put_os_config')
+
+    def os_config_validates(os_config):
+        from compass.db.api import host as host_api
+        host = clusterhost.host
+        metadata_api.validate_os_config(os_config, host.os_id)
+
+    def package_config_validates(package_config):
+        cluster = clusterhost.cluster
+        is_cluster_editable(session, cluster, updater)
+        metadata_api.validate_package_config(
+            package_config, cluster.adapter_id
+        )
+
+    @utils.supported_filters(
+        optional_support_keys=UPDATED_CLUSTERHOST_CONFIG_FIELDS,
+        ignore_support_keys=ignore_keys
+    )
+    @utils.input_validates(
+        put_os_config=os_config_validates,
+        put_package_config=package_config_validates
+    )
+    def update_config_internal(clusterihost, **in_kwargs):
+        return utils.update_db_object(
+            session, clusterhost, **in_kwargs
+        )
+
+    return update_config_internal(
+        clusterhost, **kwargs
+    )
+
+
+@user_api.check_user_permission_in_session(
+    permission.PERMISSION_ADD_CLUSTERHOST_CONFIG
+)
+@utils.wrap_to_dict(RESP_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS)
+def _update_clusterhost_deployed_config(
     session, updater, clusterhost, **kwargs
 ):
-    """Update clusterhost config internal."""
+    from compass.db.api import host as host_api
-    is_cluster_editable(session, clusterhost.cluster, updater)
+    ignore_keys = []
-    utils.update_db_object(
+    if host_api.is_host_editable(
-        session, clusterhost, config_validated=False, **kwargs
+        session, clusterhost.host, updater,
+        exception_when_not_editable=False
+    ):
+        ignore_keys.append('deployed_os_config')
+
+    def os_config_validates(os_config):
+        host = clusterhost.host
+        host_api.is_host_validated(session, host)
+
+    def package_config_validates(package_config):
+        cluster = clusterhost.cluster
+        is_cluster_editable(session, cluster, updater)
+        is_clusterhost_validated(session, clusterhost)
+
+    @utils.supported_filters(
+        optional_support_keys=UPDATED_CLUSTERHOST_DEPLOYED_CONFIG_FIELDS,
+        ignore_support_keys=ignore_keys
     )
-    package_config = clusterhost.package_config
+    @utils.input_validates(
-    if package_config:
+        deployed_os_config=os_config_validates,
-        metadata_api.validate_package_config(
+        deployed_package_config=package_config_validates
-            package_config, clusterhost.cluster.adapter_id
+    )
+    def update_config_internal(clusterhost, **in_kwargs):
+        return utils.update_db_object(
+            session, clusterhost, **in_kwargs
+        )
+
+    return update_config_internal(
+        clusterhost, **kwargs
     )
-    return clusterhost
 
 
-@utils.supported_filters(
+@utils.replace_filters(
-    optional_support_keys=UPDATED_CLUSTERHOST_CONFIG_FIELDS
+    os_config='put_os_config',
+    package_config='put_package_config'
 )
 @database.run_in_session()
 def update_cluster_host_config(
@@ -561,13 +768,32 @@ def update_cluster_host_config(
         session, models.ClusterHost,
         cluster_id=cluster_id, host_id=host_id
     )
-    return update_clusterhost_config_internal(
+    return _update_clusterhost_config(
         session, updater, clusterhost, **kwargs
     )
 
 
-@utils.supported_filters(
+@utils.replace_filters(
-    optional_support_keys=UPDATED_CLUSTERHOST_CONFIG_FIELDS
+    os_config='deployed_os_config',
+    package_config='deployed_package_config'
+)
+@database.run_in_session()
+def update_cluster_host_depolyed_config(
+    session, updater, cluster_id, host_id, **kwargs
+):
+    """Update clusterhost deployed config."""
+    clusterhost = utils.get_db_object(
+        session, models.ClusterHost,
+        cluster_id=cluster_id, host_id=host_id
+    )
+    return _update_clusterhost_deployed_config(
+        session, updater, clusterhost, **kwargs
+    )
+
+
+@utils.replace_filters(
+    os_config='put_os_config',
+    package_config='put_package_config'
 )
 @database.run_in_session()
 def update_clusterhost_config(
@@ -577,12 +803,74 @@ def update_clusterhost_config(
     clusterhost = utils.get_db_object(
         session, models.ClusterHost, id=clusterhost_id
     )
-    return update_clusterhost_config_internal(
+    return _update_clusterhost_config(
         session, updater, clusterhost, **kwargs
     )
 
 
-@utils.supported_filters(PATCHED_CLUSTERHOST_CONFIG_FIELDS)
+@utils.replace_filters(
+    os_config='deployed_os_config',
+    package_config='deployed_package_config'
+)
+@database.run_in_session()
+def update_clusterhost_deployed_config(
+    session, updater, clusterhost_id, **kwargs
+):
+    """Update clusterhost deployed config."""
+    clusterhost = utils.get_db_object(
+        session, models.ClusterHost, id=clusterhost_id
+    )
+    return _update_clusterhost_deployed_config(
+        session, updater, clusterhost, **kwargs
+    )
+
+
+@user_api.check_user_permission_in_session(
+    permission.PERMISSION_ADD_CLUSTERHOST_CONFIG
+)
+@utils.wrap_to_dict(RESP_CLUSTERHOST_CONFIG_FIELDS)
+def _patch_clusterhost_config(session, updater, clusterhost, **kwargs):
+    from compass.db.api import host as host_api
+    ignore_keys = []
+    if host_api.is_host_editable(
+        session, clusterhost.host, updater,
+        exception_when_not_editable=False
+    ):
+        ignore_keys.append('patched_os_config')
+
+    def os_config_validates(os_config):
+        host = clusterhost.host
+        metadata_api.validate_os_config(os_config, host.os_id)
+
+    def package_config_validates(package_config):
+        cluster = clusterhost.cluster
+        is_cluster_editable(session, cluster, updater)
+        metadata_api.validate_package_config(
+            package_config, cluster.adapter_id
+        )
+
+    @utils.supported_filters(
+        optional_support_keys=PATCHED_CLUSTERHOST_CONFIG_FIELDS,
+        ignore_support_keys=ignore_keys
+    )
+    @utils.output_validates(
+        os_config=os_config_validates,
+        package_config=package_config_validates
+    )
+    def update_config_internal(clusterhost, **in_kwargs):
+        return _update_cluster_config(
+            session, updater, clusterhost, **in_kwargs
+        )
+
+    return update_config_internal(
+        clusterhost, **kwargs
+    )
+
+
+@utils.replace_filters(
+    os_config='patched_os_config',
+    package_config='patched_package_config'
+)
 @database.run_in_session()
 def patch_cluster_host_config(
     session, updater, cluster_id, host_id, **kwargs
@@ -592,12 +880,15 @@ def patch_cluster_host_config(
         session, models.ClusterHost,
         cluster_id=cluster_id, host_id=host_id
     )
-    return update_clusterhost_config_internal(
+    return _patch_clusterhost_config(
         session, updater, clusterhost, **kwargs
     )
 
 
-@utils.supported_filters(PATCHED_CLUSTERHOST_CONFIG_FIELDS)
+@utils.replace_filters(
+    os_config='patched_os_config',
+    package_config='patched_package_config'
+)
 @database.run_in_session()
 def patch_clusterhost_config(
     session, updater, clusterhost_id, **kwargs
@@ -606,28 +897,59 @@ def patch_clusterhost_config(
     clusterhost = utils.get_db_object(
         session, models.ClusterHost, id=clusterhost_id
     )
-    return update_clusterhost_config_internal(
+    return _patch_clusterhost_config(
         session, updater, clusterhost, **kwargs
     )
 
 
+@user_api.check_user_permission_in_session(
+    permission.PERMISSION_DEL_CLUSTERHOST_CONFIG
+)
+@utils.wrap_to_dict(RESP_CLUSTERHOST_CONFIG_FIELDS)
+def _delete_clusterhost_config(
+    session, deleter, clusterhost
+):
+    from compass.db.api import host as host_api
+    ignore_keys = []
+    if host_api.is_host_editable(
+        session, clusterhost.host, deleter,
+        exception_when_not_editable=False
+    ):
+        ignore_keys.append('os_config')
+
+    def package_config_validates(package_config):
+        is_cluster_editable(session, clusterhost.cluster, deleter)
+
+    @utils.supported_filters(
+        optional_support_keys=['os_config', 'package_config'],
+        ignore_support_keys=ignore_keys
+    )
+    @utils.output_validates(
+        package_config=package_config_validates
+    )
+    def update_config_internal(clusterhost, **in_kwargs):
+        return utils.update_db_object(
+            session, clusterhost, **in_kwargs
+        )
+
+    return update_config_internal(
+        clusterhost, os_config={},
+        package_config={}
+    )
+
+
 @utils.supported_filters([])
 @database.run_in_session()
-@user_api.check_user_permission_in_session(
-    permission.PERMISSION_DEL_CLUSTERHOST_CONFIG
-)
-@utils.wrap_to_dict(RESP_CLUSTERHOST_CONFIG_FIELDS)
 def delete_cluster_host_config(
     session, deleter, cluster_id, host_id
 ):
     """Delete a clusterhost config."""
     clusterhost = utils.get_db_object(
         session, models.ClusterHost,
-        cluster_id=cluster_id, hsot_id=host_id
+        cluster_id=cluster_id, host_id=host_id
     )
-    is_cluster_editable(session, clusterhost.cluster, deleter)
+    return _delete_clusterhost_config(
-    return utils.update_db_object(
+        session, deleter, clusterhost
-        session, clusterhost, package_config={}, config_validated=False
     )
 
 
@@ -642,9 +964,8 @@ def delete_clusterhost_config(session, deleter, clusterhost_id):
     clusterhost = utils.get_db_object(
         session, models.ClusterHost, id=clusterhost_id
     )
-    is_cluster_editable(session, clusterhost.cluster, deleter)
+    return _delete_clusterhost_config(
-    return utils.update_db_object(
+        session, deleter, clusterhost
-        session, clusterhost, package_config={}, config_validated=False
     )
 
 
@@ -655,10 +976,13 @@ def delete_clusterhost_config(session, deleter, clusterhost_id):
 @user_api.check_user_permission_in_session(
     permission.PERMISSION_UPDATE_CLUSTER_HOSTS
 )
-@utils.wrap_to_dict(RESP_CLUSTERHOST_FIELDS)
+@utils.wrap_to_dict(
+    ['hosts'],
+    hosts=RESP_CLUSTERHOST_FIELDS
+)
 def update_cluster_hosts(
-    session, updater, cluster_id, add_hosts=[], set_hosts=None,
+    session, updater, cluster_id, add_hosts={}, set_hosts=None,
-    remove_hosts=[]
+    remove_hosts={}
 ):
     """Update cluster hosts."""
     cluster = utils.get_db_object(
@@ -666,12 +990,16 @@ def update_cluster_hosts(
     )
     is_cluster_editable(session, cluster, updater)
     if remove_hosts:
-        _remove_clusterhosts(session, cluster, remove_hosts)
+        _remove_clusterhosts(session, cluster, **remove_hosts)
     if add_hosts:
-        _add_clusterhosts(session, cluster, add_hosts)
+        _add_clusterhosts(session, cluster, **add_hosts)
     if set_hosts is not None:
-        _set_clusterhosts(session, cluster, set_hosts)
+        _set_clusterhosts(session, cluster, **set_hosts)
-    return cluster.clusterhosts
+    return {
+        'hosts': [
+            clusterhost.host for clusterhost in cluster.clusterhosts
+        ]
+    }
 
 
 @utils.supported_filters(optional_support_keys=['review'])
@@ -691,12 +1019,21 @@ def review_cluster(session, reviewer, cluster_id, review={}, **kwargs):
         session, models.Cluster, id=cluster_id
     )
     is_cluster_editable(session, cluster, reviewer)
+    clusterhost_ids = review.get('clusterhosts', [])
+    filters = {
+        'cluster_id': cluster_id
+    }
+    if clusterhost_ids:
+        filters['id'] = clusterhost_ids
+    clusterhosts = utils.list_db_objects(
+        session, models.ClusterHost, **filters
+    )
     os_config = cluster.os_config
     if os_config:
         metadata_api.validate_os_config(
             os_config, cluster.os_id, True
         )
-        for clusterhost in cluster.clusterhosts:
+        for clusterhost in clusterhosts:
             host = clusterhost.host
             if not host_api.is_host_editable(
                 session, host, reviewer, False
@@ -714,23 +1051,21 @@ def review_cluster(session, reviewer, cluster_id, review={}, **kwargs):
                 deployed_os_config, host.os_id, True
             )
             host_api.validate_host(session, host)
-            host.deployed_os_config = deployed_os_config
             host.config_validated = True
     package_config = cluster.package_config
     if package_config:
         metadata_api.validate_package_config(
             package_config, cluster.adapter_id, True
         )
-        for clusterhost in cluster.clusterhosts:
+        for clusterhost in clusterhosts:
             clusterhost_package_config = clusterhost.package_config
-            deployed_package_config = util.mrege_dict(
+            deployed_package_config = util.merge_dict(
                 package_config, clusterhost_package_config
             )
             metadata_api.validate_package_config(
                 deployed_package_config,
                 cluster.adapter_id, True
             )
-            clusterhost.deployed_package_config = deployed_package_config
             clusterhost.config_validated = True
     cluster.config_validated = True
     return {
@@ -753,14 +1088,47 @@ def deploy_cluster(
     session, deployer, cluster_id, deploy={}, **kwargs
 ):
     """deploy cluster."""
+    from compass.db.api import host as host_api
     from compass.tasks import client as celery_client
     cluster = utils.get_db_object(
         session, models.Cluster, id=cluster_id
     )
+    clusterhost_ids = deploy.get('clusterhosts', [])
+    filters = {
+        'cluster_id': cluster_id
+    }
+    if clusterhost_ids:
+        filters['id'] = clusterhost_ids
+    clusterhosts = utils.list_db_objects(
+        session, models.ClusterHost, **filters
+    )
+    is_cluster_editable(session, cluster, deployer)
     is_cluster_validated(session, cluster)
+    utils.update_db_object(
+        session, cluster.state, state='INITIALIZED'
+    )
+    for clusterhost in clusterhosts:
+        if cluster.distributed_system:
+            is_clusterhost_validated(session, clusterhost)
+            utils.update_db_object(
+                session, clusterhost.state,
+                state='INITIALIZED'
+            )
+        host = clusterhost.host
+        if host_api.is_host_editable(
+            session, host, deployer,
+            exception_when_not_editable=False
+        ):
+            host_api.is_host_validated(
+                session, host
+            )
+            utils.update_db_object(
+                session, host.state, state='INITIALIZED'
+            )
+
     celery_client.celery.send_task(
         'compass.tasks.deploy',
-        (cluster_id, deploy.get('clusterhosts', []))
+        (deployer.email, cluster_id, deploy.get('clusterhosts', []))
     )
     return {
         'status': 'deploy action sent',
@@ -850,3 +1218,22 @@ def update_clusterhost_state(
     )
     utils.update_db_object(session, clusterhost.state, **kwargs)
     return clusterhost.state_dict()
+
+
+@utils.supported_filters(
+    optional_support_keys=UPDATED_CLUSTER_STATE_FIELDS
+)
+@database.run_in_session()
+@user_api.check_user_permission_in_session(
+    permission.PERMISSION_UPDATE_CLUSTER_STATE
+)
+@utils.wrap_to_dict(RESP_STATE_FIELDS)
+def update_cluster_state(
+    session, updater, cluster_id, **kwargs
+):
+    """Update a cluster state."""
+    cluster = utils.get_db_object(
+        session, models.Cluster, id=cluster_id
+    )
+    utils.update_db_object(session, cluster.state, **kwargs)
+    return cluster.state_dict()

compass/db/api/database.py

@@ -19,6 +19,7 @@ import netaddr
 
 from contextlib import contextmanager
 from sqlalchemy import create_engine
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import scoped_session
 from sqlalchemy.orm import sessionmaker
 from threading import local
@@ -80,7 +81,11 @@ def session():
         new_session.rollback()
         logging.error('failed to commit session')
         logging.exception(error)
-        if isinstance(error, exception.DatabaseException):
+        if isinstance(error, IntegrityError):
+            raise exception.NotAcceptable(
+                'operation error in database'
+            )
+        elif isinstance(error, exception.DatabaseException):
             raise error
         else:
             raise exception.DatabaseException(str(error))
@@ -122,8 +127,8 @@ def _setup_user_table(user_session):
     from compass.db.api import user
     user.add_user_internal(
         user_session,
-        setting.COMPASS_ADMIN_EMAIL,
+        email=setting.COMPASS_ADMIN_EMAIL,
-        setting.COMPASS_ADMIN_PASSWORD,
+        password=setting.COMPASS_ADMIN_PASSWORD,
         is_admin=True
     )
 

compass/db/api/host.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 """Host database operations."""
+import functools
 import logging
 
 from compass.db.api import database
@@ -52,6 +53,9 @@ RESP_CONFIG_FIELDS = [
     'created_at',
     'updated_at'
 ]
+RESP_DEPLOYED_CONFIG_FIELDS = [
+    'deployed_os_config'
+]
 RESP_DEPLOY_FIELDS = [
     'status', 'host'
 ]
@@ -62,19 +66,25 @@ UPDATED_CONFIG_FIELDS = [
 PATCHED_CONFIG_FIELDS = [
     'patched_os_config'
 ]
+UPDATED_DEPLOYED_CONFIG_FIELDS = [
+    'deployed_os_config'
+]
 ADDED_NETWORK_FIELDS = [
     'interface', 'ip', 'subnet_id'
 ]
 OPTIONAL_ADDED_NETWORK_FIELDS = ['is_mgmt', 'is_promiscuous']
 UPDATED_NETWORK_FIELDS = [
-    'interface', 'ip', 'subnet_id', 'subnet', 'is_mgmt',
+    'ip', 'subnet_id', 'subnet', 'is_mgmt',
     'is_promiscuous'
 ]
+IGNORED_NETWORK_FIELDS = [
+    'interface'
+]
 RESP_STATE_FIELDS = [
-    'id', 'state', 'progress', 'message'
+    'id', 'state', 'percentage', 'message'
 ]
 UPDATED_STATE_FIELDS = [
-    'id', 'state', 'progress', 'message'
+    'id', 'state', 'percentage', 'message'
 ]
 
 
@@ -126,10 +136,14 @@ def list_machines_or_hosts(session, lister, **filters):
     permission.PERMISSION_LIST_HOSTS
 )
 @utils.wrap_to_dict(RESP_FIELDS)
-def get_host(session, getter, host_id, **kwargs):
+def get_host(
+    session, getter, host_id,
+    exception_when_missing=True, **kwargs
+):
     """get host info."""
     return utils.get_db_object(
-        session, models.Host, id=host_id
+        session, models.Host,
+        exception_when_missing, id=host_id
     )
 
 
@@ -139,11 +153,17 @@ def get_host(session, getter, host_id, **kwargs):
     permission.PERMISSION_LIST_HOSTS
 )
 @utils.wrap_to_dict(RESP_FIELDS)
-def get_machine_or_host(session, getter, host_id, **kwargs):
+def get_machine_or_host(
+    session, getter, host_id,
+    exception_when_missing=True, **kwargs
+):
     """get host info."""
     machine = utils.get_db_object(
-        session, models.Machine, id=host_id
+        session, models.Machine,
+        exception_when_missing, id=host_id
     )
+    if not machine:
+        return None
     host = machine.host
     if host:
         return host
@@ -186,7 +206,7 @@ def is_host_editable(
     reinstall_os_set=False, exception_when_not_editable=True
 ):
     if reinstall_os_set:
-        if host.state.state == 'INSTALLING':
+        if host.state.state == 'DEPLOYING':
             return _conditional_exception(
                 host, exception_when_not_editable
             )
@@ -222,7 +242,7 @@ def validate_host(session, host):
         )
 
 
-@utils.supported_filters(UPDATED_FIELDS)
+@utils.supported_filters(optional_support_keys=UPDATED_FIELDS)
 @database.run_in_session()
 @user_api.check_user_permission_in_session(
     permission.PERMISSION_UPDATE_HOST
@@ -268,35 +288,98 @@ def get_host_config(session, getter, host_id, **kwargs):
     )
 
 
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission_in_session(
+    permission.PERMISSION_LIST_HOST_CONFIG
+)
+@utils.wrap_to_dict(RESP_DEPLOYED_CONFIG_FIELDS)
+def get_host_deployed_config(session, getter, host_id, **kwargs):
+    """Get host deployed config."""
+    return utils.get_db_object(
+        session, models.Host, id=host_id
+    )
+
+
+@utils.replace_filters(
+    os_config='deployed_os_config'
+)
+@utils.supported_filters(UPDATED_DEPLOYED_CONFIG_FIELDS)
+@database.run_in_session()
 @user_api.check_user_permission_in_session(
     permission.PERMISSION_ADD_HOST_CONFIG
 )
 @utils.wrap_to_dict(RESP_CONFIG_FIELDS)
-def _update_host_config(session, updater, host_id, **kwargs):
+def update_host_deployed_config(session, updater, host_id, **kwargs):
-    """Update host config."""
+    """Update host deployed config."""
     host = utils.get_db_object(
         session, models.Host, id=host_id
     )
     is_host_editable(session, host, updater)
-    utils.update_db_object(session, host, config_validated=False, **kwargs)
+    is_host_validated(session, host)
-    os_config = host.os_config
+    return utils.update_db_object(session, host, **kwargs)
-    if os_config:
-        metadata_api.validate_os_config(
-            os_config, host.adapter_id
-        )
-    return host
 
 
+@user_api.check_user_permission_in_session(
+    permission.PERMISSION_ADD_HOST_CONFIG
+)
+@utils.wrap_to_dict(RESP_CONFIG_FIELDS)
+def update_host_config_internal(session, updater, host, **kwargs):
+    """Update host config."""
+    is_host_editable(session, host, updater)
+    return utils.update_db_object(session, host, **kwargs)
+
+
+@utils.replace_filters(
+    os_config='put_os_config'
+)
 @utils.supported_filters(UPDATED_CONFIG_FIELDS)
 @database.run_in_session()
 def update_host_config(session, updater, host_id, **kwargs):
-    return _update_host_config(session, updater, host_id, **kwargs)
+    host = utils.get_db_object(
+        session, models.Host, id=host_id
+    )
+
+    os_config_validates = functools.partial(
+        metadata_api.validate_os_config, os_id=host.os_id)
+
+    @utils.input_validates(
+        put_os_config=os_config_validates,
+    )
+    def update_config_internal(host, **in_kwargs):
+        return update_host_config_internal(
+            session, updater, host, **kwargs
+        )
+
+    return update_config_internal(
+        host, **kwargs
+    )
 
 
+@utils.replace_filters(
+    os_config='patched_os_config'
+)
 @utils.supported_filters(PATCHED_CONFIG_FIELDS)
 @database.run_in_session()
 def patch_host_config(session, updater, host_id, **kwargs):
-    return _update_host_config(session, updater, host_id, **kwargs)
+    host = utils.get_db_object(
+        session, models.Host, id=host_id
+    )
+
+    os_config_validates = functools.partial(
+        metadata_api.validate_os_config, os_id=host.os_id)
+
+    @utils.output_validates(
+        os_config=os_config_validates,
+    )
+    def update_config_internal(host, **in_kwargs):
+        return update_host_config_internal(
+            session, updater, host, **in_kwargs
+        )
+
+    return update_config_internal(
+        session, updater, host, **kwargs
+    )
 
 
 @utils.supported_filters([])
@@ -312,7 +395,7 @@ def del_host_config(session, deleter, host_id):
     )
     is_host_editable(session, host, deleter)
     return utils.update_db_object(
-        session, host, os_config={}, config_validated=False
+        session, host, os_config={}
     )
 
 
@@ -353,12 +436,22 @@ def list_hostnetworks(session, lister, **filters):
     permission.PERMISSION_LIST_HOST_NETWORKS
 )
 @utils.wrap_to_dict(RESP_NETWORK_FIELDS)
-def get_host_network(session, getter, host_id, subnet_id, **kwargs):
+def get_host_network(
+    session, getter, host_id,
+    host_network_id, **kwargs
+):
     """Get host network."""
-    return utils.get_db_object(
+    host_network = utils.get_db_object(
         session, models.HostNetwork,
-        host_id=host_id, subnet_id=subnet_id
+        id=host_network_id
     )
+    if host_network.host_id != host_id:
+        raise exception.RecordNotExists(
+            'host %s does not own host network %s' % (
+                host_id, host_network_id
+            )
+        )
+    return host_network
 
 
 @utils.supported_filters([])
@@ -383,37 +476,54 @@ def get_hostnetwork(session, getter, host_network_id, **kwargs):
     permission.PERMISSION_ADD_HOST_NETWORK
 )
 @utils.wrap_to_dict(RESP_NETWORK_FIELDS)
-def add_host_network(session, creator, host_id, **kwargs):
+def add_host_network(
+    session, creator, host_id,
+    exception_when_existing=True,
+    interface=None, **kwargs
+):
     """Create a host network."""
     host = utils.get_db_object(
         session, models.Host, id=host_id
     )
     is_host_editable(session, host, creator)
     return utils.add_db_object(
-        session, models.HostNetwork, True,
+        session, models.HostNetwork,
-        host_id, **kwargs
+        exception_when_existing,
+        host_id, interface, **kwargs
     )
 
 
 @utils.supported_filters(
-    optional_support_keys=UPDATED_NETWORK_FIELDS
+    optional_support_keys=UPDATED_NETWORK_FIELDS,
+    ignore_support_keys=IGNORED_NETWORK_FIELDS
 )
 @database.run_in_session()
 @user_api.check_user_permission_in_session(
     permission.PERMISSION_ADD_HOST_NETWORK
 )
 @utils.wrap_to_dict(RESP_NETWORK_FIELDS)
-def update_host_network(session, updater, host_id, subnet_id, **kwargs):
+def update_host_network(
+    session, updater, host_id, host_network_id, **kwargs
+):
     """Update a host network."""
     host_network = utils.get_db_object(
         session, models.HostNetwork,
-        host_id=host_id, subnet_id=subnet_id
+        id=host_network_id
+    )
+    if host_network.host_id != host_id:
+        raise exception.RecordNotExists(
+            'host %s does not own host network %s' % (
+                host_id, host_network_id
+            )
         )
     is_host_editable(session, host_network.host, updater)
     return utils.update_db_object(session, host_network, **kwargs)
 
 
-@utils.supported_filters(UPDATED_NETWORK_FIELDS)
+@utils.supported_filters(
+    optional_support_keys=UPDATED_NETWORK_FIELDS,
+    ignore_support_keys=IGNORED_NETWORK_FIELDS
+)
 @database.run_in_session()
 @user_api.check_user_permission_in_session(
     permission.PERMISSION_ADD_HOST_NETWORK
@@ -434,11 +544,17 @@ def update_hostnetwork(session, updater, host_network_id, **kwargs):
     permission.PERMISSION_DEL_HOST_NETWORK
 )
 @utils.wrap_to_dict(RESP_NETWORK_FIELDS)
-def del_host_network(session, deleter, host_id, subnet_id, **kwargs):
+def del_host_network(session, deleter, host_id, host_network_id, **kwargs):
     """Delete a host network."""
     host_network = utils.get_db_object(
         session, models.HostNetwork,
-        host_id=host_id, subnet_id=subnet_id
+        id=host_network_id
+    )
+    if host_network.host_id != host_id:
+        raise exception.RecordNotExists(
+            'host %s does not own host network %s' % (
+                host_id, host_network_id
+            )
         )
     is_host_editable(session, host_network.host, deleter)
     return utils.del_db_object(session, host_network)

compass/db/api/installer.py

@@ -30,9 +30,9 @@ def _add_installers(session, model, configs):
     for config in configs:
         installers.append(utils.add_db_object(
             session, model,
-            True, config['NAME'],
+            True, config['INSTANCE_NAME'],
-            installer_type=config['TYPE'],
+            name=config['NAME'],
-            config=config['CONFIG']
+            settings=config.get('SETTINGS', {})
         ))
     return installers
 

compass/db/api/machine.py

@@ -38,54 +38,21 @@ RESP_FIELDS = [
 ]
 
 
-def _check_ipmi_credentials_ip(ip):
-    utils.check_ip(ip)
-
-
-def _check_ipmi_credentials(ipmi_credentials):
-    if not ipmi_credentials:
-        return
-    if not isinstance(ipmi_credentials, dict):
-        raise exception.InvalidParameter(
-            'invalid ipmi credentials %s' % ipmi_credentials
-
-        )
-    for key in ipmi_credentials:
-        if key not in ['ip', 'username', 'password']:
-            raise exception.InvalidParameter(
-                'unrecognized field %s in ipmi credentials %s' % (
-                    key, ipmi_credentials
-                )
-            )
-    for key in ['ip', 'username', 'password']:
-        if key not in ipmi_credentials:
-            raise exception.InvalidParameter(
-                'no field %s in ipmi credentials %s' % (
-                    key, ipmi_credentials
-                )
-            )
-        check_ipmi_credential_field = '_check_ipmi_credentials_%s' % key
-        this_module = globals()
-        if check_ipmi_credential_field in this_module:
-            this_module[check_ipmi_credential_field](
-                ipmi_credentials[key]
-            )
-        else:
-            logging.debug(
-                'function %s is not defined', check_ipmi_credential_field
-            )
-
-
 @utils.supported_filters([])
 @database.run_in_session()
 @user_api.check_user_permission_in_session(
     permission.PERMISSION_LIST_MACHINES
 )
 @utils.wrap_to_dict(RESP_FIELDS)
-def get_machine(session, getter, machine_id, **kwargs):
+def get_machine(
+    session, getter, machine_id,
+    exception_when_missing=True,
+    **kwargs
+):
     """get field dict of a machine."""
     return utils.get_db_object(
-        session, models.Machine, True, id=machine_id
+        session, models.Machine,
+        exception_when_missing, id=machine_id
     )
 
 
@@ -119,7 +86,7 @@ def _update_machine(session, updater, machine_id, **kwargs):
 
 
 @utils.supported_filters(optional_support_keys=UPDATED_FIELDS)
-@utils.input_validates(ipmi_credentials=_check_ipmi_credentials)
+@utils.input_validates(ipmi_credentials=utils.check_ipmi_credentials)
 @database.run_in_session()
 def update_machine(session, updater, machine_id, **kwargs):
     return _update_machine(
@@ -127,9 +94,14 @@ def update_machine(session, updater, machine_id, **kwargs):
     )
 
 
+@utils.replace_filters(
+    ipmi_credentials='patched_ipmi_credentials',
+    tag='patched_tag',
+    location='patched_location'
+)
 @utils.supported_filters(optional_support_keys=PATCHED_FIELDS)
 @database.run_in_session()
-@utils.output_validates(ipmi_credentials=_check_ipmi_credentials)
+@utils.output_validates(ipmi_credentials=utils.check_ipmi_credentials)
 def patch_machine(session, updater, machine_id, **kwargs):
     return _update_machine(
         session, updater, machine_id, **kwargs

compass/db/api/network.py

@@ -67,10 +67,13 @@ def list_subnets(session, lister, **filters):
     permission.PERMISSION_LIST_NETWORKS
 )
 @utils.wrap_to_dict(RESP_FIELDS)
-def get_subnet(session, getter, subnet_id, **kwargs):
+def get_subnet(
+    session, getter, subnet_id,
+    exception_when_missing=True, **kwargs
+):
     """Get subnet info."""
     return utils.get_db_object(
-        session, models.Network, id=subnet_id
+        session, models.Network, exception_when_missing, id=subnet_id
     )
 
 
@@ -84,10 +87,14 @@ def get_subnet(session, getter, subnet_id, **kwargs):
     permission.PERMISSION_ADD_NETWORK
 )
 @utils.wrap_to_dict(RESP_FIELDS)
-def add_subnet(session, creator, subnet, **kwargs):
+def add_subnet(
+    session, creator, subnet,
+    exception_when_existing=True, **kwargs
+):
     """Create a subnet."""
     return utils.add_db_object(
-        session, models.Network, True, subnet, **kwargs
+        session, models.Network,
+        exception_when_existing, subnet, **kwargs
     )
 
 

compass/db/api/permission.py

@@ -136,6 +136,10 @@ PERMISSION_DEPLOY_HOST = PermissionWrapper(
 PERMISSION_GET_CLUSTER_STATE = PermissionWrapper(
     'get_cluster_state', 'get cluster state', 'get cluster state'
 )
+PERMISSION_UPDATE_CLUSTER_STATE = PermissionWrapper(
+    'update_cluster_state', 'update cluster state',
+    'update cluster state'
+)
 PERMISSION_LIST_HOSTS = PermissionWrapper(
     'list_hosts', 'list hosts', 'list hosts'
 )
@@ -234,6 +238,7 @@ PERMISSIONS = [
     PERMISSION_REVIEW_CLUSTER,
     PERMISSION_DEPLOY_CLUSTER,
     PERMISSION_GET_CLUSTER_STATE,
+    PERMISSION_UPDATE_CLUSTER_STATE,
     PERMISSION_LIST_HOSTS,
     PERMISSION_LIST_HOST_CLUSTERS,
     PERMISSION_UPDATE_HOST,
@@ -276,10 +281,14 @@ def list_permissions(session, lister, **filters):
 @database.run_in_session()
 @user_api.check_user_permission_in_session(PERMISSION_LIST_PERMISSIONS)
 @utils.wrap_to_dict(RESP_FIELDS)
-def get_permission(session, getter, permission_id, **kwargs):
+def get_permission(
+    session, getter, permission_id,
+    exception_when_missing=True, **kwargs
+):
     """get permissions."""
     return utils.get_db_object(
-        session, models.Permission, id=permission_id
+        session, models.Permission,
+        exception_when_missing, id=permission_id
     )
 
 

compass/db/api/switch.py

@@ -57,7 +57,7 @@ OPTIONAL_CHECK_FILTER_FIELDS = [
     'ports', 'port_prefix', 'port_suffix',
     'port_start', 'port_end'
 ]
-ALL_ADDED_MACHINES_FIELDS = ['port', 'vlans']
+ADDED_SWITCH_MACHINES_FIELDS = ['port', 'vlans']
 UPDATED_MACHINES_FIELDS = [
     'port', 'vlans', 'ipmi_credentials',
     'tag', 'location'
@@ -88,47 +88,13 @@ RESP_MACHINES_HOSTS_FIELDS = [
     'ipmi_credentials', 'tag', 'location',
     'name', 'os_name', 'clusters'
 ]
+RESP_CLUSTER_FIELDS = [
+    'name', 'id'
+]
 
 
-def _check_credentials_version(version):
+def _check_filters(switch_filters):
-    if version not in ['1', '2c', '3']:
+    for switch_filter in switch_filters:
-        raise exception.InvalidParameter(
-            'unknown snmp version %s' % version
-        )
-
-
-def _check_credentials(credentials):
-    if not credentials:
-        return
-    if not isinstance(credentials, dict):
-        raise exception.InvalidParameter(
-            'credentials %s is not dict' % credentials
-        )
-    for key in credentials:
-        if key not in ['version', 'community']:
-            raise exception.InvalidParameter(
-                'unrecognized key %s in credentials %s' % (key, credentials)
-            )
-    for key in ['version', 'community']:
-        if key not in credentials:
-            raise exception.InvalidParameter(
-                'there is no %s field in credentials %s' % (key, credentials)
-            )
-
-        key_check_func_name = '_check_credentials_%s' % key
-        this_module = globals()
-        if key_check_func_name in this_module:
-            this_module[key_check_func_name](
-                credentials[key]
-            )
-        else:
-            logging.debug(
-                'function %s is not defined in %s',
-                key_check_func_name, this_module
-            )
-
-
-def _check_filter(switch_filter):
         if not isinstance(switch_filter, dict):
             raise exception.InvalidParameter(
                 'filter %s is not dict' % switch_filter
@@ -159,7 +125,8 @@ def _check_filter_internal(
                 )
 
 
-def _check_vlan(vlan):
+def _check_vlans(vlans):
+    for vlan in vlans:
         if not isinstance(vlan, int):
             raise exception.InvalidParameter(
                 'vlan %s is not int' % vlan
@@ -193,10 +160,14 @@ def get_switch_internal(
     permission.PERMISSION_LIST_SWITCHES
 )
 @utils.wrap_to_dict(RESP_FIELDS)
-def get_switch(session, getter, switch_id, **kwargs):
+def get_switch(
+    session, getter, switch_id,
+    exception_when_missing=True, **kwargs
+):
     """get field dict of a switch."""
     return utils.get_db_object(
-        session, models.Switch, id=switch_id
+        session, models.Switch,
+        exception_when_missing, id=switch_id
     )
 
 
@@ -208,9 +179,16 @@ def get_switch(session, getter, switch_id, **kwargs):
 @utils.wrap_to_dict(RESP_FIELDS)
 def list_switches(session, lister, **filters):
     """List switches."""
-    return utils.list_db_objects(
+    switches = utils.list_db_objects(
         session, models.Switch, **filters
     )
+    if 'ip_int' in filters:
+        return switches
+    else:
+        return [
+            switch for switch in switches
+            if switch.ip != setting.DEFAULT_SWITCH_IP
+        ]
 
 
 @utils.supported_filters([])
@@ -231,18 +209,21 @@ def del_switch(session, deleter, switch_id, **kwargs):
 )
 @utils.input_validates(
     ip=utils.check_ip,
-    credentials=_check_credentials
+    credentials=utils.check_switch_credentials
 )
 @database.run_in_session()
 @user_api.check_user_permission_in_session(
     permission.PERMISSION_ADD_SWITCH
 )
 @utils.wrap_to_dict(RESP_FIELDS)
-def add_switch(session, creator, ip, **kwargs):
+def add_switch(
+    session, creator, exception_when_existing=True,
+    ip=None, **kwargs
+):
     """Create a switch."""
     ip_int = long(netaddr.IPAddress(ip))
     return add_switch_internal(
-        session, ip_int, **kwargs
+        session, ip_int, exception_when_existing, **kwargs
     )
 
 
@@ -267,15 +248,22 @@ def _update_switch(session, updater, switch_id, **kwargs):
 
 
 @utils.supported_filters(optional_support_keys=UPDATED_FIELDS)
-@utils.input_validates(credentials=_check_credentials)
+@utils.input_validates(
+    credentials=utils.check_switch_credentials
+)
 @database.run_in_session()
 def update_switch(session, updater, switch_id, **kwargs):
     return _update_switch(session, updater, switch_id, **kwargs)
 
 
+@utils.replace_filters(
+    credentials='patched_credentials'
+)
 @utils.supported_filters(optional_support_keys=PATCHED_FIELDS)
 @database.run_in_session()
-@utils.output_validates(credentials=_check_credentials)
+@utils.output_validates(
+    credentials=utils.check_switch_credentials
+)
 def patch_switch(session, updater, switch_id, **kwargs):
     return _update_switch(session, updater, switch_id, **kwargs)
 
@@ -299,7 +287,9 @@ def list_switch_filters(session, lister, **filters):
     permission.PERMISSION_LIST_SWITCH_FILTERS
 )
 @utils.wrap_to_dict(RESP_FILTERS_FIELDS)
-def get_switch_filters(session, getter, switch_id, **kwargs):
+def get_switch_filters(
+    session, getter, switch_id, **kwargs
+):
     """get switch filter."""
     return utils.get_db_object(
         session, models.Switch, id=switch_id
@@ -307,7 +297,7 @@ def get_switch_filters(session, getter, switch_id, **kwargs):
 
 
 @utils.supported_filters(optional_support_keys=UPDATED_FILTERS_FIELDS)
-@utils.input_validates(filters=_check_filter)
+@utils.input_validates(filters=_check_filters)
 @database.run_in_session()
 @user_api.check_user_permission_in_session(
     permission.PERMISSION_UPDATE_SWITCH_FILTERS
@@ -319,8 +309,11 @@ def update_switch_filters(session, updater, switch_id, **kwargs):
     return utils.update_db_object(session, switch, **kwargs)
 
 
+@utils.replace_filters(
+    filters='patched_filters'
+)
 @utils.supported_filters(optional_support_keys=PATCHED_FILTERS_FIELDS)
-@utils.input_validates(patched_filters=_check_filter)
+@utils.input_validates(patched_filters=_check_filters)
 @database.run_in_session()
 @user_api.check_user_permission_in_session(
     permission.PERMISSION_UPDATE_SWITCH_FILTERS
@@ -433,7 +426,7 @@ def _filter_vlans(vlan_filter, obj):
     location=utils.general_filter_callback
 )
 @utils.wrap_to_dict(RESP_MACHINES_FIELDS)
-def _list_switch_machines(session, user, switch_machines, **filters):
+def _filter_switch_machines(session, user, switch_machines, **filters):
     return [
         switch_machine for switch_machine in switch_machines
         if filter_machine_internal(
@@ -454,8 +447,11 @@ def _list_switch_machines(session, user, switch_machines, **filters):
     os_name=utils.general_filter_callback,
     os_id=utils.general_filter_callback
 )
-@utils.wrap_to_dict(RESP_MACHINES_HOSTS_FIELDS)
+@utils.wrap_to_dict(
-def _list_switch_machines_hosts(session, user, switch_machines, **filters):
+    RESP_MACHINES_HOSTS_FIELDS,
+    clusters=RESP_CLUSTER_FIELDS
+)
+def _filter_switch_machines_hosts(session, user, switch_machines, **filters):
     filtered_switch_machines = [
         switch_machine for switch_machine in switch_machines
         if filter_machine_internal(
@@ -492,12 +488,15 @@ def _list_switch_machines_hosts(session, user, switch_machines, **filters):
 @database.run_in_session()
 def list_switch_machines(session, getter, switch_id, **filters):
     """Get switch machines."""
-    switch_machines, host_filters = get_switch_machines_internal(
+    switch_machines = get_switch_machines_internal(
         session, switch_id=switch_id, **filters
     )
-    return _list_switch_machines(session, getter, switch_machines, **filters)
+    return _filter_switch_machines(session, getter, switch_machines, **filters)
 
 
+@utils.replace_filters(
+    ip_int='switch_ip_int'
+)
 @utils.supported_filters(
     optional_support_keys=SUPPORTED_SWITCH_MACHINES_FIELDS
 )
@@ -507,7 +506,16 @@ def list_switchmachines(session, lister, **filters):
     switch_machines = get_switch_machines_internal(
         session, **filters
     )
-    return _list_switch_machines(session, lister, switch_machines, **filters)
+    if 'ip_int' in filters:
+        filtered_switch_machines = switch_machines
+    else:
+        filtered_switch_machines = [
+            switch_machine for switch_machine in switch_machines
+            if switch_machine.switch_ip != setting.DEFAULT_SWITCH_IP
+        ]
+    return _filter_switch_machines(
+        session, lister, filtered_switch_machines, **filters
+    )
 
 
 @utils.supported_filters(
@@ -519,11 +527,14 @@ def list_switch_machines_hosts(session, getter, switch_id, **filters):
     switch_machines = get_switch_machines_internal(
         session, switch_id=switch_id, **filters
     )
-    return _list_switch_machines_hosts(
+    return _filter_switch_machines_hosts(
         session, getter, switch_machines, **filters
     )
 
 
+@utils.replace_filters(
+    ip_int='switch_ip_int'
+)
 @utils.supported_filters(
     optional_support_keys=SUPPORTED_SWITCH_MACHINES_HOSTS_FIELDS
 )
@@ -533,29 +544,46 @@ def list_switchmachines_hosts(session, lister, **filters):
     switch_machines = get_switch_machines_internal(
         session, **filters
     )
-    return _list_switch_machines_hosts(
+    if 'ip_int' in filters:
-        session, lister, switch_machines, **filters
+        filtered_switch_machines = switch_machines
+    else:
+        filtered_switch_machines = [
+            switch_machine for switch_machine in switch_machines
+            if switch_machine.switch_ip != setting.DEFAULT_SWITCH_IP
+        ]
+    return _filter_switch_machines_hosts(
+        session, lister, filtered_switch_machines, **filters
     )
 
 
-def add_switch_machines_internal(
+@utils.supported_filters(
-    session, switch, machine_dicts,
+    ADDED_MACHINES_FIELDS,
-    exception_when_switch_machine_existing=True
+    optional_support_keys=OPTIONAL_ADDED_MACHINES_FIELDS
+)
+@utils.input_validates(mac=utils.check_mac, vlans=_check_vlans)
+@database.run_in_session()
+@user_api.check_user_permission_in_session(
+    permission.PERMISSION_ADD_SWITCH_MACHINE
+)
+@utils.wrap_to_dict(RESP_MACHINES_FIELDS)
+def add_switch_machine(
+    session, creator, switch_id,
+    exception_when_existing=True,
+    mac=None, **kwargs
 ):
-    machine_id_switch_machine_dict = {}
+    """Add switch machine."""
-    for mac, all_dict in machine_dicts.items():
+    switch = utils.get_db_object(
+        session, models.Switch, id=switch_id)
     switch_machine_dict = {}
     machine_dict = {}
-        for key, value in all_dict.items():
+    for key, value in kwargs.items():
-            if key in ALL_ADDED_MACHINES_FIELDS:
+        if key in ADDED_SWITCH_MACHINES_FIELDS:
             switch_machine_dict[key] = value
         else:
             machine_dict[key] = value
-        #TODO(xiaodong): add ipmi field checks'
     machine = utils.add_db_object(
         session, models.Machine, False,
         mac, **machine_dict)
-        machine_id_switch_machine_dict[machine.id] = switch_machine_dict
 
     switches = [switch]
     if switch.ip != setting.DEFAULT_SWITCH_IP:
@@ -566,35 +594,13 @@ def add_switch_machines_internal(
 
     switch_machines = []
     for machine_switch in switches:
-        for machine_id, switch_machine_dict in (
+        switch_machines.append(utils.add_db_object(
-            machine_id_switch_machine_dict.items()
-        ):
-            utils.add_db_object(
             session, models.SwitchMachine,
-                exception_when_switch_machine_existing,
+            exception_when_existing,
-                machine_switch.id, machine_id, **switch_machine_dict
+            machine_switch.id, machine.id,
-            )
+            **switch_machine_dict
-        switch_machines.extend(machine_switch.switch_machines)
+        ))
 
-    return switch_machines
-
-
-@utils.supported_filters(
-    ADDED_MACHINES_FIELDS,
-    optional_support_keys=OPTIONAL_ADDED_MACHINES_FIELDS
-)
-@utils.input_validates(mac=utils.check_mac, vlans=_check_vlan)
-@database.run_in_session()
-@user_api.check_user_permission_in_session(
-    permission.PERMISSION_ADD_SWITCH_MACHINE
-)
-@utils.wrap_to_dict(RESP_MACHINES_FIELDS)
-def add_switch_machine(session, creator, switch_id, mac, **kwargs):
-    """Add switch machine."""
-    switch = utils.get_db_object(
-        session, models.Switch, id=switch_id)
-    switch_machines = add_switch_machines_internal(
-        session, switch, {mac: kwargs})
     return switch_machines[0]
 
 
@@ -610,7 +616,7 @@ def poll_switch_machines(session, poller, switch_id, **kwargs):
     switch = utils.get_db_object(session, models.Switch, id=switch_id)
     celery_client.celery.send_task(
         'compass.tasks.pollswitch',
-        (switch.ip, switch.credentials)
+        (poller.email, switch.ip, switch.credentials)
     )
     return {
         'status': 'action %s sent' % kwargs,
@@ -625,10 +631,14 @@ def poll_switch_machines(session, poller, switch_id, **kwargs):
     permission.PERMISSION_LIST_SWITCH_MACHINES
 )
 @utils.wrap_to_dict(RESP_MACHINES_FIELDS)
-def get_switch_machine(session, getter, switch_id, machine_id, **kwargs):
+def get_switch_machine(
+    session, getter, switch_id, machine_id,
+    exception_when_missing=True, **kwargs
+):
     """get field dict of a switch machine."""
     return utils.get_db_object(
         session, models.SwitchMachine,
+        exception_when_missing,
         switch_id=switch_id, machine_id=machine_id
     )
 
@@ -639,10 +649,15 @@ def get_switch_machine(session, getter, switch_id, machine_id, **kwargs):
     permission.PERMISSION_LIST_SWITCH_MACHINES
 )
 @utils.wrap_to_dict(RESP_MACHINES_FIELDS)
-def get_switchmachine(session, getter, switch_machine_id, **kwargs):
+def get_switchmachine(
+    session, getter, switch_machine_id,
+    exception_when_missing=True,
+    **kwargs
+):
     """get field dict of a switch machine."""
     return utils.get_db_object(
-        session, models.SwitchMachine, id=switch_machine_id
+        session, models.SwitchMachine,
+        exception_when_missing, id=switch_machine_id
     )
 
 
@@ -667,7 +682,7 @@ def update_switch_machine_internal(
 
 
 @utils.supported_filters(optional_support_keys=UPDATED_MACHINES_FIELDS)
-@utils.input_validates(vlans=_check_vlan)
+@utils.input_validates(vlans=_check_vlans)
 @database.run_in_session()
 @user_api.check_user_permission_in_session(
     permission.PERMISSION_ADD_SWITCH_MACHINE
@@ -686,7 +701,7 @@ def update_switch_machine(session, updater, switch_id, machine_id, **kwargs):
 
 
 @utils.supported_filters(optional_support_keys=UPDATED_MACHINES_FIELDS)
-@utils.input_validates(vlans=_check_vlan)
+@utils.input_validates(vlans=_check_vlans)
 @database.run_in_session()
 @user_api.check_user_permission_in_session(
     permission.PERMISSION_ADD_SWITCH_MACHINE
@@ -704,8 +719,14 @@ def update_switchmachine(session, updater, switch_machine_id, **kwargs):
     )
 
 
+@utils.replace_filters(
+    vlans='patched_vlans',
+    ipmi_credentials='patched_ipmi_credentials',
+    tag='patched_tag',
+    location='patched_location'
+)
 @utils.supported_filters(optional_support_keys=PATCHED_MACHINES_FIELDS)
-@utils.input_validates(patched_vlans=_check_vlan)
+@utils.input_validates(patched_vlans=_check_vlans)
 @database.run_in_session()
 @user_api.check_user_permission_in_session(
     permission.PERMISSION_ADD_SWITCH_MACHINE
@@ -723,8 +744,14 @@ def patch_switch_machine(session, updater, switch_id, machine_id, **kwargs):
     )
 
 
+@utils.replace_filters(
+    vlans='patched_vlans',
+    ipmi_credentials='patched_ipmi_credentials',
+    tag='patched_tag',
+    location='patched_location'
+)
 @utils.supported_filters(optional_support_keys=PATCHED_MACHINES_FIELDS)
-@utils.input_validates(patched_vlans=_check_vlan)
+@utils.input_validates(patched_vlans=_check_vlans)
 @database.run_in_session()
 @user_api.check_user_permission_in_session(
     permission.PERMISSION_ADD_SWITCH_MACHINE
@@ -772,18 +799,21 @@ def del_switchmachine(session, deleter, switch_machine_id, **kwargs):
     return utils.del_db_object(session, switch_machine)
 
 
-@utils.supported_filters(optional_support_keys=UPDATED_SWITCH_MACHINES_FIELDS)
+@utils.supported_filters(
+    ['machine_id'],
+    optional_support_keys=UPDATED_SWITCH_MACHINES_FIELDS
+)
 def _update_machine_internal(session, switch_id, machine_id, **kwargs):
     utils.add_db_object(
-        session, models.SwitchMachine, False, switch_id, machine_id,
+        session, models.SwitchMachine, False,
-        **kwargs
+        switch_id, machine_id, **kwargs
     )
 
 
 def _add_machines(session, switch, machines):
-    for machine_id, switch_machine_attrs in machines.items():
+    for machine in machines.items():
         _update_machine_internal(
-            session, switch.id, machine_id, **switch_machine_attrs
+            session, switch.id, **machine
         )
 
 
@@ -799,9 +829,9 @@ def _set_machines(session, switch, machines):
         session, models.SwitchMachine,
         switch_id=switch.id
     )
-    for machine_id, switch_machine_attrs in machines.items():
+    for switch_machine in machines.items():
         _update_machine_internal(
-            session, switch.id, machine_id, **switch_machine_attrs
+            session, switch.id, **switch_machine
         )
 
 

compass/db/api/user.py

@@ -65,13 +65,14 @@ def get_user_internal(session, exception_when_missing=True, **kwargs):
 
 
 def add_user_internal(
-    session, email, password,
+    session, exception_when_existing=True,
-    exception_when_existing=True, **kwargs
+    email=None, **kwargs
 ):
     """internal function used only by other db.api modules."""
-    user = utils.add_db_object(session, models.User,
+    user = utils.add_db_object(
+        session, models.User,
         exception_when_existing, email,
-                               password=password, **kwargs)
+        **kwargs)
     _add_user_permissions(
         session, user,
         name=setting.COMPASS_DEFAULT_PERMISSIONS
@@ -180,14 +181,18 @@ def _set_user_permissions(session, user, **permission_filters):
 class UserWrapper(UserMixin):
     def __init__(
         self, id, email, crypted_password,
-        active, is_admin, expire_timestamp, token='', **kwargs
+        active=True, is_admin=False,
+        expire_timestamp=None, token='', **kwargs
     ):
         self.id = id
         self.email = email
         self.password = crypted_password
         self.active = active
         self.is_admin = is_admin
+        if expire_timestamp:
             self.expire_timestamp = expire_timestamp
+        else:
+            self.expire_timestamp = datetime.datetime.now()
         if not token:
             self.token = self.get_auth_token()
         else:
@@ -278,9 +283,14 @@ def clean_user_token(session, user, token):
 @check_user_admin_or_owner()
 @database.run_in_session()
 @utils.wrap_to_dict(RESP_FIELDS)
-def get_user(session, getter, user_id, **kwargs):
+def get_user(
+    session, getter, user_id,
+    exception_when_missing=True, **kwargs
+):
     """get field dict of a user."""
-    return utils.get_db_object(session, models.User, id=user_id)
+    return utils.get_db_object(
+        session, models.User, exception_when_missing, id=user_id
+    )
 
 
 @utils.supported_filters(
@@ -303,10 +313,14 @@ def list_users(session, lister, **filters):
 @check_user_admin()
 @database.run_in_session()
 @utils.wrap_to_dict(RESP_FIELDS)
-def add_user(session, creator, email, password, **kwargs):
+def add_user(
+    session, creator,
+    exception_when_existing=True,
+    **kwargs
+):
     """Create a user and return created user object."""
     return add_user_internal(
-        session, email, password, **kwargs
+        session, exception_when_existing, **kwargs
     )
 
 
@@ -349,7 +363,7 @@ def update_user(session, updater, user_id, **kwargs):
 @check_user_admin_or_owner()
 @database.run_in_session()
 @utils.wrap_to_dict(PERMISSION_RESP_FIELDS)
-def get_permissions(session, getter, user_id, **kwargs):
+def get_permissions(session, lister, user_id, **kwargs):
     """List permissions of a user."""
     return utils.list_db_objects(
         session, models.UserPermission, user_id=user_id, **kwargs
@@ -360,10 +374,14 @@ def get_permissions(session, getter, user_id, **kwargs):
 @check_user_admin_or_owner()
 @database.run_in_session()
 @utils.wrap_to_dict(PERMISSION_RESP_FIELDS)
-def get_permission(session, getter, user_id, permission_id, **kwargs):
+def get_permission(
+    session, getter, user_id, permission_id,
+    exception_when_missing, **kwargs
+):
     """Get a specific user permission."""
     return utils.get_db_object(
         session, models.UserPermission,
+        exception_when_missing,
         user_id=user_id, permission_id=permission_id,
         **kwargs
     )
@@ -387,10 +405,13 @@ def del_permission(session, deleter, user_id, permission_id, **kwargs):
 @check_user_admin()
 @database.run_in_session()
 @utils.wrap_to_dict(PERMISSION_RESP_FIELDS)
-def add_permission(session, creator, user_id, permission_id):
+def add_permission(
+    session, creator, user_id,
+    exception_when_missing=True, permission_id=None
+):
     """Add an user permission."""
     return utils.add_db_object(
-        session, models.UserPermission, True,
+        session, models.UserPermission, exception_when_missing,
         user_id, permission_id
     )
 

compass/db/api/utils.py

@@ -155,6 +155,8 @@ def wrap_to_dict(support_keys=[], **filters):
     def decorator(func):
         @functools.wraps(func)
         def wrapper(*args, **kwargs):
+            logging.info('wrap to dict: args: %s', str(args))
+            logging.info('wrap to dict: kwargs: %s', kwargs)
             return _wrapper_dict(
                 func(*args, **kwargs), support_keys, **filters
             )
@@ -185,6 +187,21 @@ def _wrapper_dict(data, support_keys, **filters):
     return info
 
 
+def replace_filters(**filter_mapping):
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **filters):
+            replaced_filters = {}
+            for key, value in filters.items():
+                if key in filter_mapping:
+                    replaced_filters[filter_mapping[key]] = value
+                else:
+                    replaced_filters[key] = value
+            return func(*args, **replaced_filters)
+        return wrapper
+    return decorator
+
+
 def supported_filters(
     support_keys=[],
     optional_support_keys=[],
@@ -311,17 +328,9 @@ def output_filters(missing_ok=False, **filter_callbacks):
 def _input_validates(args_validators, kwargs_validators, *args, **kwargs):
     for i, value in enumerate(args):
         if i < len(args_validators) and args_validators[i]:
-            if isinstance(value, list):
-                for sub_value in value:
-                    args_validators[i](sub_value)
-            else:
             args_validators[i](value)
     for key, value in kwargs.items():
         if kwargs_validators.get(key):
-            if isinstance(value, list):
-                for sub_value in value:
-                    kwargs_validators[key](sub_value)
-            else:
             kwargs_validators[key](value)
 
 
@@ -432,8 +441,8 @@ def add_db_object(session, table, exception_when_existing=True,
 
         if new_object:
             session.add(db_object)
-        db_object.initialize()
         session.flush()
+        db_object.initialize()
         db_object.validate()
         return db_object
 
@@ -468,8 +477,8 @@ def update_db_object(session, db_object, **kwargs):
                       db_object, kwargs)
         for key, value in kwargs.items():
             setattr(db_object, key, value)
-        db_object.initialize()
         session.flush()
+        db_object.update()
         db_object.validate()
         return db_object
 
@@ -500,3 +509,79 @@ def check_mac(mac):
         raise exception.InvalidParameter(
             'invalid mac address %s' % mac
         )
+
+
+def _check_ipmi_credentials_ip(ip):
+    check_ip(ip)
+
+
+def check_ipmi_credentials(ipmi_credentials):
+    if not ipmi_credentials:
+        return
+    if not isinstance(ipmi_credentials, dict):
+        raise exception.InvalidParameter(
+            'invalid ipmi credentials %s' % ipmi_credentials
+
+        )
+    for key in ipmi_credentials:
+        if key not in ['ip', 'username', 'password']:
+            raise exception.InvalidParameter(
+                'unrecognized field %s in ipmi credentials %s' % (
+                    key, ipmi_credentials
+                )
+            )
+    for key in ['ip', 'username', 'password']:
+        if key not in ipmi_credentials:
+            raise exception.InvalidParameter(
+                'no field %s in ipmi credentials %s' % (
+                    key, ipmi_credentials
+                )
+            )
+        check_ipmi_credential_field = '_check_ipmi_credentials_%s' % key
+        this_module = globals()
+        if check_ipmi_credential_field in this_module:
+            this_module[check_ipmi_credential_field](
+                ipmi_credentials[key]
+            )
+        else:
+            logging.debug(
+                'function %s is not defined', check_ipmi_credential_field
+            )
+
+
+def _check_switch_credentials_version(version):
+    if version not in ['1', '2c', '3']:
+        raise exception.InvalidParameter(
+            'unknown snmp version %s' % version
+        )
+
+
+def check_switch_credentials(credentials):
+    if not credentials:
+        return
+    if not isinstance(credentials, dict):
+        raise exception.InvalidParameter(
+            'credentials %s is not dict' % credentials
+        )
+    for key in credentials:
+        if key not in ['version', 'community']:
+            raise exception.InvalidParameter(
+                'unrecognized key %s in credentials %s' % (key, credentials)
+            )
+    for key in ['version', 'community']:
+        if key not in credentials:
+            raise exception.InvalidParameter(
+                'there is no %s field in credentials %s' % (key, credentials)
+            )
+
+        key_check_func_name = '_check_switch_credentials_%s' % key
+        this_module = globals()
+        if key_check_func_name in this_module:
+            this_module[key_check_func_name](
+                credentials[key]
+            )
+        else:
+            logging.debug(
+                'function %s is not defined in %s',
+                key_check_func_name, this_module
+            )

compass/db/exception.py

@@ -61,6 +61,13 @@ class Forbidden(DatabaseException):
         self.status_code = 403
 
 
+class NotAcceptable(DatabaseException):
+    """The data is not acceptable."""
+    def __init__(self, message):
+        super(NotAcceptable, self).__init__(message)
+        self.status_code = 406
+
+
 class InvalidParameter(DatabaseException):
     """Define the exception that the request has invalid or missing parameters.
     """

compass/db/models.py

@@ -14,6 +14,7 @@
 
 """Database model"""
 import datetime
+import logging
 import netaddr
 import simplejson as json
 
@@ -67,6 +68,9 @@ class TimestampMixin(object):
 
 class HelperMixin(object):
     def initialize(self):
+        self.update()
+
+    def update(self):
         pass
 
     def validate(self):
@@ -91,7 +95,7 @@ class MetadataMixin(HelperMixin):
     description = Column(Text)
     is_required = Column(Boolean, default=False)
     required_in_whole_config = Column(Boolean, default=False)
-    mapping_to = Column(JSONEncoded)
+    mapping_to = Column(String(80), default='')
     validator_data = Column('validator', Text)
     js_validator = Column(Text)
     default_value = Column(JSONEncoded)
@@ -107,9 +111,6 @@ class MetadataMixin(HelperMixin):
             self.path = self.name
         super(MetadataMixin, self).initialize()
 
-    def validate(self):
-        super(MetadataMixin, self).validate()
-
     @property
     def validator(self):
         if not self.validator_data:
@@ -249,14 +250,14 @@ class FieldMixin(HelperMixin):
 
 
 class InstallerMixin(HelperMixin):
-    name = Column(String(80), unique=True)
+    name = Column(String(80))
-    installer_type = Column(String(80))
+    instance_name = Column(String(80), unique=True)
-    config = Column(MutableDict.as_mutable(JSONEncoded), default={})
+    settings = Column(MutableDict.as_mutable(JSONEncoded), default={})
 
     def validate(self):
-        if not self.installer_type:
+        if not self.name:
             raise exception.InvalidParameter(
-                'installer_type is not set in installer %s' % self.name
+                'name is not set in installer %s' % self.instance_name
             )
         super(InstallerMixin, self).validate()
 
@@ -264,24 +265,31 @@ class InstallerMixin(HelperMixin):
 class StateMixin(TimestampMixin, HelperMixin):
     state = Column(
         Enum(
-            'INITIALIZED', 'INSTALLING', 'SUCCESSFUL', 'ERROR'
+            'UNINITIALIZED', 'INITIALIZED',
+            'INSTALLING', 'SUCCESSFUL', 'ERROR'
         ),
-        default='INITIALIZED'
+        default='UNINITIIALIZED'
     )
-    progress = Column(Float, default=0.0)
+    percentage = Column(Float, default=0.0)
     message = Column(Text, default='')
     severity = Column(
         Enum('INFO', 'WARNING', 'ERROR'),
         default='INFO'
     )
 
-    def initialize(self):
+    def update(self):
+        if self.state in ['UNINITIALIZED', 'INITIALIZED']:
+            self.percentage = 0.0
+            self.severity = 'INFO'
+            self.message = ''
         if self.severity == 'ERROR':
             self.state = 'ERROR'
-        elif self.progress >= 1.0:
+        if self.state == 'SUCCESSFUL':
+            self.percentage = 1.0
+        if self.percentage >= 1.0:
             self.state = 'SUCCESSFUL'
-            self.progress = 1.0
+            self.percentage = 1.0
-        super(StateMixin, self).initialize()
+        super(StateMixin, self).update()
 
 
 class HostNetwork(BASE, TimestampMixin, HelperMixin):
@@ -307,8 +315,9 @@ class HostNetwork(BASE, TimestampMixin, HelperMixin):
         UniqueConstraint('host_id', 'interface', name='constraint'),
     )
 
-    def __init__(self, host_id, **kwargs):
+    def __init__(self, host_id, interface, **kwargs):
         self.host_id = host_id
+        self.interface = interface
         super(HostNetwork, self).__init__(**kwargs)
 
     @property
@@ -323,15 +332,15 @@ class HostNetwork(BASE, TimestampMixin, HelperMixin):
     def subnet(self):
         return self.network.subnet
 
+    @subnet.expression
+    def subnet(cls):
+        return cls.network.subnet
+
     @property
     def netmask(self):
         return str(netaddr.IPNetwork(self.subnet).netmask)
 
     def validate(self):
-        if not self.interface:
-            raise exception.InvalidParameter(
-                'interface is not set in host %s network' % self.host_id
-            )
         if not self.network:
             raise exception.InvalidParameter(
                 'subnet is not set in %s interface %s' % (
@@ -367,6 +376,7 @@ class HostNetwork(BASE, TimestampMixin, HelperMixin):
         dict_info['ip'] = self.ip
         dict_info['interface'] = self.interface
         dict_info['netmask'] = self.netmask
+        dict_info['subnet'] = self.subnet
         return dict_info
 
 
@@ -428,6 +438,7 @@ class ClusterHost(BASE, TimestampMixin, HelperMixin):
     @patched_package_config.setter
     def patched_package_config(self, value):
         self.package_config = util.merge_dict(dict(self.package_config), value)
+        self.config_validated = False
 
     @property
     def put_package_config(self):
@@ -438,77 +449,93 @@ class ClusterHost(BASE, TimestampMixin, HelperMixin):
         package_config = dict(self.package_config)
         package_config.update(value)
         self.package_config = package_config
+        self.config_validated = False
+
+    @property
+    def patched_os_config(self):
+        return self.host.os_config
+
+    @patched_os_config.setter
+    def patched_os_config(self, value):
+        host = self.host
+        host.os_config = util.merge_dict(dict(host.os_config), value)
+
+    @property
+    def put_os_config(self):
+        return self.host.os_config
+
+    @put_os_config.setter
+    def put_os_config(self, value):
+        host = self.host
+        os_config = dict(host.os_config)
+        os_config.update(value)
+        host.os_config = os_config
 
     @hybrid_property
     def distributed_system_name(self):
-        cluster = self.cluster
+        return self.cluster.distributed_system_name
-        if cluster:
+
-            return cluster.distributed_system_name
+    @distributed_system_name.expression
-        else:
+    def distributed_system_name(cls):
-            return None
+        return cls.cluster.distributed_system_name
 
     @hybrid_property
     def os_name(self):
-        host = self.host
+        return self.host.os_name
-        if host:
+
-            return host.os_name
+    @os_name.expression
-        else:
+    def os_name(cls):
-            return None
+        return cls.host.os_name
 
     @hybrid_property
     def clustername(self):
-        cluster = self.cluster
+        return self.cluster.name
-        if cluster:
+
-            return cluster.name
+    @clustername.expression
-        else:
+    def clustername(cls):
-            return None
+        return cls.cluster.name
 
     @hybrid_property
     def hostname(self):
-        host = self.host
+        return self.host.name
-        if host:
+
-            return host.name
+    @hostname.expression
-        else:
+    def hostname(cls):
-            return None
+        return cls.host.name
 
     @property
     def distributed_system_installed(self):
-        state = self.state
+        return self.state.state == 'SUCCESSFUL'
-        if state:
+
-            return state.state == 'SUCCESSFUL'
+    @property
-        else:
+    def resintall_os(self):
-            return False
+        return self.host.reinstall_os
+
+    @property
+    def reinstall_distributed_system(self):
+        return self.cluster.reinstall_distributed_system
 
     @property
     def os_installed(self):
-        host = self.host
+        return self.host.os_installed
-        if host:
-            return host.os_installed
-        else:
-            return None
 
-    @property
+    @hybrid_property
     def owner(self):
-        cluster = self.cluster
+        return self.cluster.owner
-        if cluster:
+
-            return cluster.owner
+    @owner.expression
-        else:
+    def owner(cls):
-            return None
+        return cls.cluster.owner
 
     def state_dict(self):
-        state = self.state
-        if state.progress <= 0.0:
-            host = self.host
-            if host:
-                dict_info = host.state_dict()
-            else:
-                dict_info = {}
         cluster = self.cluster
-            if cluster and cluster.distributed_system:
+        host = self.host
-                dict_info['state'] = state.state
+        if (
-        else:
+            not cluster.distributed_system or
-            dict_info = state.to_dict()
+            host.state.state != 'SUCCESSFUL'
-        return dict_info
+        ):
+            return host.state_dict()
+        return self.state.to_dict()
 
     def to_dict(self):
         dict_info = self.host.to_dict()
@@ -516,10 +543,10 @@ class ClusterHost(BASE, TimestampMixin, HelperMixin):
         dict_info.update({
             'distributed_system_name': self.distributed_system_name,
             'distributed_system_installed': self.distributed_system_installed,
-            'reinstall_distributed_system': (
+            'reinstall_distributed_system': self.reinstall_distributed_system,
-                self.cluster.reinstall_distributed_system
-            ),
             'owner': self.owner,
+            'clustername': self.clustername,
+            'hostname': self.hostname,
             'name': self.name
         })
         return dict_info
@@ -535,10 +562,11 @@ class HostState(BASE, StateMixin):
         primary_key=True
     )
 
-    def initialize(self):
+    def update(self):
+        host = self.host
         if self.state == 'INSTALLING':
-            self.host.reinstall_os = False
+            host.reinstall_os = False
-        super(HostState, self).initialize()
+        super(HostState, self).update()
 
 
 class Host(BASE, TimestampMixin, HelperMixin):
@@ -595,6 +623,7 @@ class Host(BASE, TimestampMixin, HelperMixin):
     @patched_os_config.setter
     def patched_os_config(self, value):
         self.os_config = util.merge_dict(dict(self.os_config), value)
+        self.config_validated = False
 
     @property
     def put_os_config(self):
@@ -605,18 +634,29 @@ class Host(BASE, TimestampMixin, HelperMixin):
         os_config = dict(self.os_config)
         os_config.update(value)
         self.os_config = os_config
+        self.config_validated = False
 
     def __init__(self, id, **kwargs):
         self.id = id
+        self.name = str(self.id)
+        self.state = HostState()
         super(Host, self).__init__(**kwargs)
 
     def initialize(self):
         if not self.name:
             self.name = str(self.id)
-        if not self.state or self.reinstall_os:
-            self.state = HostState()
         super(Host, self).initialize()
 
+    def update(self):
+        if self.reinstall_os:
+            self.state = HostState()
+        os = self.os
+        if os:
+            self.os_name = os.name
+        else:
+            self.os_name = None
+        super(Host, self).update()
+
     def validate(self):
         os = self.os
         if not os:
@@ -627,7 +667,6 @@ class Host(BASE, TimestampMixin, HelperMixin):
             raise exception.InvalidParameter(
                 'os %s is not deployable' % os.name
             )
-        self.os_name = os.name
         creator = self.creator
         if not creator:
             raise exception.InvalidParameter(
@@ -637,31 +676,24 @@ class Host(BASE, TimestampMixin, HelperMixin):
 
     @hybrid_property
     def owner(self):
-        creator = self.creator
+        return self.creator.email
-        if creator:
+
-            return creator.email
+    @owner.expression
-        else:
+    def owner(cls):
-            return None
+        return cls.creator.email
 
     @property
     def os_installed(self):
-        state = self.state
+        return self.state.state == 'SUCCESSFUL'
-        if state:
-            return state.state == 'SUCCESSFUL'
-        else:
-            return False
 
     def state_dict(self):
-        state = self.state
+        return self.state.to_dict()
-        if state:
-            return state.to_dict()
-        else:
-            return {}
 
     def to_dict(self):
         dict_info = self.machine.to_dict()
         dict_info.update(super(Host, self).to_dict())
         dict_info.update({
+            'machine_id': self.machine.id,
             'owner': self.owner,
             'os_installed': self.os_installed,
             'networks': [
@@ -681,50 +713,77 @@ class ClusterState(BASE, StateMixin):
         ForeignKey('cluster.id', onupdate='CASCADE', ondelete='CASCADE'),
         primary_key=True
     )
+    total_hosts = Column(
+        Integer,
+        default=0
+    )
+    installing_hosts = Column(
+        Integer,
+        default=0
+    )
+    completed_hosts = Column(
+        Integer,
+        default=0
+    )
+    failed_hosts = Column(
+        Integer,
+        default=0
+    )
 
-    def initialize(self):
+    def to_dict(self):
+        dict_info = super(ClusterState, self).to_dict()
+        dict_info['status'] = {
+            'total_hosts': self.total_hosts,
+            'installing_hosts': self.installing_hosts,
+            'completed_hosts': self.completed_hosts,
+            'failed_hosts': self.failed_hosts
+        }
+        return dict_info
+
+    def update(self):
         cluster = self.cluster
+        clusterhosts = cluster.clusterhosts
+        self.total_hosts = len(clusterhosts)
+        if self.state in ['UNINITIALIZED', 'INITIALIZED']:
+            self.installing_hosts = 0
+            self.failed_hosts = 0
+            self.completed_hosts = 0
         if self.state == 'INSTALLING':
             cluster.reinstall_distributed_system = False
-        clusterhosts = cluster.clusterhosts
-        total_clusterhosts = 0
-        failed_clusterhosts = 0
-        installing_clusterhosts = 0
-        finished_clusterhosts = 0
-        progress = 0
             if not cluster.distributed_system:
                 for clusterhost in clusterhosts:
                     host = clusterhost.host
                     host_state = host.state.state
-                total_clusterhosts += 1
+                    if host_state == 'INSTALLING':
-                progress += host.state.progress
+                        self.intsalling_hosts += 1
-                if host_state == 'SUCCESSFUL':
-                    finished_clusterhosts += 1
-                elif host_state == 'INSTALLING':
-                    installing_clusterhosts += 1
                     elif host_state == 'ERROR':
-                    failed_clusterhosts += 1
+                        self.failed_hosts += 1
+                    elif host_state == 'SUCCESSFUL':
+                        self.completed_hosts += 1
             else:
                 for clusterhost in clusterhosts:
                     clusterhost_state = clusterhost.state.state
-                total_clusterhosts += 1
+                    if clusterhost_state == 'INSTALLING':
-                progress += clusterhost.state.progress
+                        self.intsalling_hosts += 1
-                if clusterhost_state == 'SUCCESSFUL':
-                    finished_clusterhosts += 1
-                elif clusterhost_state == 'INSTALLING':
-                    installing_clusterhosts += 1
                     elif clusterhost_state == 'ERROR':
-                    failed_clusterhosts += 1
+                        self.failed_hosts += 1
-        self.progress = progress / total_clusterhosts
+                    elif clusterhost_state == 'SUCCESSFUL':
-        self.message = (
+                        self.completed_hosts += 1
-            'toal %s, installing %s, finished %s, error $s'
+            if self.total_hosts:
-        ) % (
+                self.percentage = (
-            total_clusterhosts, installing_clusterhosts,
+                    float(self.completed_hosts)
-            finished_clusterhosts, failed_clusterhosts
+                    /
+                    float(self.total_hosts)
                 )
-        if failed_clusterhosts:
+            self.message = (
+                'toal %s, installing %s, complted: %s, error $s'
+            ) % (
+                self.total_hosts, self.completed_hosts,
+                self.intsalling_hosts, self.failed_hosts
+            )
+            if self.failed_hosts:
                 self.severity = 'ERROR'
-        super(ClusterState, self).initialize()
+        super(ClusterState, self).update()
 
 
 class Cluster(BASE, TimestampMixin, HelperMixin):
@@ -746,6 +805,8 @@ class Cluster(BASE, TimestampMixin, HelperMixin):
     )
     os_config = Column(JSONEncoded, default={})
     package_config = Column(JSONEncoded, default={})
+    deployed_os_config = Column(JSONEncoded, default={})
+    deployed_package_config = Column(JSONEncoded, default={})
     config_validated = Column(Boolean, default=False)
     adapter_id = Column(Integer, ForeignKey('adapter.id'))
     adapter_name = Column(String(80), nullable=True)
@@ -766,12 +827,39 @@ class Cluster(BASE, TimestampMixin, HelperMixin):
 
     def __init__(self, name, **kwargs):
         self.name = name
+        self.state = ClusterState()
         super(Cluster, self).__init__(**kwargs)
 
     def initialize(self):
-        if not self.state or self.reinstall_distributed_system:
+        adapter = self.adapter
+        if adapter:
+            self.put_package_config = {
+                'roles': [role.name for role in adapter.roles]
+            }
+
+    def update(self):
+        if self.reinstall_distributed_system:
             self.state = ClusterState()
-        super(Cluster, self).initialize()
+        os = self.os
+        if os:
+            self.os_name = os.name
+        else:
+            self.os_name = None
+            self.os_config = {}
+        adapter = self.adapter
+        if adapter:
+            self.adapter_name = adapter.name
+            self.distributed_system = adapter.adapter_distributed_system
+            self.distributed_system_name = self.distributed_system.name
+            self.put_package_config = {
+                'roles': [role.name for role in adapter.roles]
+            }
+        else:
+            self.adapter_name = None
+            self.distributed_system = None
+            self.distributed_system_name = None
+            self.package_config = {}
+        super(Cluster, self).update()
 
     def validate(self):
         creator = self.creator
@@ -780,14 +868,10 @@ class Cluster(BASE, TimestampMixin, HelperMixin):
                 'creator is not set in cluster %s' % self.id
             )
         os = self.os
-        if os:
+        if os and not os.deployable:
-            if not os.deployable:
             raise exception.InvalidParameter(
                 'os %s is not deployable' % os.name
             )
-            self.os_name = os.name
-        else:
-            self.os_name = None
         adapter = self.adapter
         if adapter:
             if not adapter.deployable:
@@ -801,25 +885,13 @@ class Cluster(BASE, TimestampMixin, HelperMixin):
                 raise exception.InvalidParameter(
                     'os %s is not supported' % os.name
                 )
-            self.adapter_name = adapter.name
+            distributed_system = self.distributed_system
-            distributed_system = (
+            if distributed_system and not distributed_system.deployable:
-                adapter.adapter_distributed_system
-            )
-            self.distributed_system = distributed_system
-            if distributed_system:
-                if not distributed_system.deployable:
                 raise exception.InvalidParamerter(
                     'distributed system %s is not deployable' % (
                         distributed_system.name
                     )
                 )
-                self.distributed_system_name = (
-                    distributed_system.name
-                )
-        else:
-            self.adapter_name = None
-            self.distributed_system = None
-            self.distributed_system_name = None
         super(Cluster, self).validate()
 
     @property
@@ -829,6 +901,7 @@ class Cluster(BASE, TimestampMixin, HelperMixin):
     @patched_os_config.setter
     def patched_os_config(self, value):
         self.os_config = util.merge_dict(dict(self.os_config), value)
+        self.config_validated = False
 
     @property
     def put_os_config(self):
@@ -839,6 +912,7 @@ class Cluster(BASE, TimestampMixin, HelperMixin):
         os_config = dict(self.os_config)
         os_config.update(value)
         self.os_config = os_config
+        self.config_validated = False
 
     @property
     def patched_package_config(self):
@@ -846,7 +920,9 @@ class Cluster(BASE, TimestampMixin, HelperMixin):
 
     @patched_package_config.setter
     def patched_package_config(self, value):
-        self.package_config = util.merge_dict(dict(self.package_config), value)
+        package_config = dict(self.package_config)
+        self.package_config = util.merge_dict(package_config, value)
+        self.config_validated = False
 
     @property
     def put_package_config(self):
@@ -857,29 +933,22 @@ class Cluster(BASE, TimestampMixin, HelperMixin):
         package_config = dict(self.package_config)
         package_config.update(value)
         self.package_config = package_config
+        self.config_validated = False
 
     @hybrid_property
     def owner(self):
-        creator = self.creator
+        return self.creator.email
-        if creator:
+
-            return creator.email
+    @owner.expression
-        else:
+    def owner(cls):
-            return None
+        return cls.creator.email
 
     @property
     def distributed_system_installed(self):
-        state = self.state
-        if state:
         return self.state.state == 'SUCCESSFUL'
-        else:
-            return False
 
     def state_dict(self):
-        state = self.state
-        if state:
         return self.state.to_dict()
-        else:
-            return {}
 
     def to_dict(self):
         dict_info = super(Cluster, self).to_dict()
@@ -1660,54 +1729,6 @@ class Adapter(BASE, HelperMixin):
         else:
             return None
 
-    @property
-    def package_installer_name(self):
-        installer = self.adapter_package_installer
-        if installer:
-            return installer.name
-        else:
-            return None
-
-    @property
-    def os_installer_name(self):
-        installer = self.adapter_os_installer
-        if installer:
-            return installer.name
-        else:
-            return None
-
-    @property
-    def package_installer_type(self):
-        installer = self.adapter_package_installer
-        if installer:
-            return installer.installer_type
-        else:
-            return None
-
-    @property
-    def os_installer_type(self):
-        installer = self.adapter_os_installer
-        if installer:
-            return installer.installer_type
-        else:
-            return None
-
-    @property
-    def package_installer_config(self):
-        installer = self.adapter_package_installer
-        if installer:
-            return installer.config
-        else:
-            return None
-
-    @property
-    def os_installer_config(self):
-        installer = self.adapter_os_installer
-        if installer:
-            return installer.config
-        else:
-            return None
-
     @property
     def adapter_distributed_system(self):
         distributed_system = self.distributed_system
@@ -1719,14 +1740,6 @@ class Adapter(BASE, HelperMixin):
         else:
             return None
 
-    @property
-    def distributed_system_name(self):
-        distributed_system = self.adapter_distributed_system
-        if distributed_system:
-            return distributed_system.name
-        else:
-            return None
-
     @property
     def adapter_supported_oses(self):
         supported_oses = self.supported_oses
@@ -1751,21 +1764,24 @@ class Adapter(BASE, HelperMixin):
 
     def to_dict(self):
         dict_info = super(Adapter, self).to_dict()
-        adapter_roles = self.adapter_roles
-        supported_oses = self.adapter_supported_oses
         dict_info.update({
-            'roles': [role.to_dict() for role in adapter_roles],
+            'roles': [
-            'supported_oses': [
+                role.to_dict() for role in self.adapter_roles
-                adapter_os.to_dict() for adapter_os in supported_oses
+            ],
+            'supported_oses': [
+                adapter_os.to_dict()
+                for adapter_os in self.adapter_supported_oses
             ],
-            'distributed_system_name': self.distributed_system_name,
-            'os_installer_name': self.os_installer_name,
-            'os_installer_type': self.os_installer_type,
-            'os_installer_config': self.os_installer_config,
-            'package_installer_name': self.package_installer_name,
-            'package_installer_type': self.package_installer_type,
-            'package_installer_config': self.package_installer_config
         })
+        distributed_system = self.distributed_system
+        if distributed_system:
+            dict_info['distributed_system_name'] = distributed_system.name
+        os_installer = self.adapter_os_installer
+        if os_installer:
+            dict_info['os_installer'] = os_installer.to_dict()
+        package_installer = self.adapter_package_installer
+        if package_installer:
+            dict_info['package_installer'] = package_installer.to_dict()
         return dict_info
 
 
@@ -1817,8 +1833,8 @@ class OSInstaller(BASE, InstallerMixin):
         backref=backref('os_installer')
     )
 
-    def __init__(self, name, **kwargs):
+    def __init__(self, instance_name, **kwargs):
-        self.name = name
+        self.instance_name = instance_name
         super(OSInstaller, self).__init__(**kwargs)
 
 
@@ -1833,8 +1849,8 @@ class PackageInstaller(BASE, InstallerMixin):
         backref=backref('package_installer')
     )
 
-    def __init__(self, name, **kwargs):
+    def __init__(self, instance_name, **kwargs):
-        self.name = name
+        self.instance_name = instance_name
         super(PackageInstaller, self).__init__(**kwargs)
 
 
@@ -1857,10 +1873,10 @@ class Network(BASE, TimestampMixin, HelperMixin):
         self.subnet = subnet
         super(Network, self).__init__(**kwargs)
 
-    def intialize(self):
+    def initialize(self):
         if not self.name:
             self.name = self.subnet
-        super(Network, self).intialize()
+        super(Network, self).initialize()
 
     def validate(self):
         try:

compass/tasks/tasks.py

@@ -46,7 +46,10 @@ def global_celery_init(**_):
 
 
 @celery.task(name='compass.tasks.pollswitch')
-def pollswitch(ip_addr, credentials, req_obj='mac', oper='SCAN'):
+def pollswitch(
+    poller_email, ip_addr, credentials,
+    req_obj='mac', oper='SCAN'
+):
     """Query switch and return expected result.
 
     :param ip_addr: switch ip address.
@@ -60,14 +63,15 @@ def pollswitch(ip_addr, credentials, req_obj='mac', oper='SCAN'):
     """
     try:
         poll_switch.poll_switch(
-            ip_addr, credentials, req_obj=req_obj, oper=oper
+            poller_email, ip_addr, credentials,
+            req_obj=req_obj, oper=oper
         )
     except Exception as error:
         logging.exception(error)
 
 
 @celery.task(name='compass.tasks.deploy_cluster')
-def deploy_cluster(cluster_id, clusterhost_ids):
+def deploy_cluster(deployer_email, cluster_id, clusterhost_ids):
     """Deploy the given cluster.
 
     :param cluster_hosts: the cluster and hosts of each cluster to deploy.
@@ -77,7 +81,7 @@ def deploy_cluster(cluster_id, clusterhost_ids):
 
 
 @celery.task(name='compass.tasks.reinstall_cluster')
-def reinstall_cluster(cluster_id, clusterhost_ids):
+def reinstall_cluster(installer_email, cluster_id, clusterhost_ids):
     """reinstall the given cluster.
 
     :param cluster_hosts: the cluster and hosts of each cluster to reinstall.

compass/tests/db/api/test_utils.py

@@ -145,11 +145,9 @@ class TestModelFilter(unittest2.TestCase):
         expected, ret = self._filter_test_dict_util(
             'gt',
             'update_clusterhost_state',
-            48,
+            49,
-            id=47
+            id=48
         )
-        print 'expected: %s' % expected
-        print 'ret: %s' % ret
         self.assertTrue(
             all(item in ret[0].items() for item in expected.items())
         )
@@ -169,8 +167,8 @@ class TestModelFilter(unittest2.TestCase):
         expected, ret = self._filter_test_dict_util(
             'ge',
             'update_clusterhost_state',
-            48,
+            49,
-            id=48
+            id=49
         )
         print 'expected: %s' % expected
         print 'ret: %s' % ret

conf/os_field/general_list.conf

@@ -0,0 +1,2 @@
+NAME = 'general_list'
+FIELD_TYPE = list

conf/os_installer/cobbler.conf

@@ -1,6 +1,6 @@
 NAME = 'cobbler'
-TYPE = 'cobbler'
+INSTANCE_NAME = 'cobbler'
-CONFIG = {
+SETTINGS = {
     'url': 'http://127.0.0.1/cobbler_api',
     'token': ('cobbler', 'cobbler') 
 }

conf/os_metadata/general.conf

@@ -8,14 +8,77 @@ METADATA = {
             '_self': {
                 'field': 'general',
                 'default_value': 'EN',
-                'options': ['EN'],
+                'options': ['EN', 'CN'],
             }
         },
         'timezone': {
             '_self': {
                 'field': 'general',
-                'default_value': 'PDT',
+                'default_value': 'GMT -8:00',
-                'options': ['PDT'],
+                'options': [
+                    'GMT -12:00', 'GMT -11:00', 'GMT -10:00', 'GMT -9:00',
+                    'GMT -8:00', 'GMT -7:00', 'GMT -6:00', 'GMT -5:00',
+                    'GMT -4:00', 'GMT -3:00', 'GMT -2:00', 'GMT -1:00',
+                    'GMT 0:00', 'GMT +1:00', 'GMT +2:00', 'GMT +3:00',
+                    'GMT +4:00', 'GMT +5:00', 'GMT +6:00', 'GMT +7:00',
+                    'GMT +8:00', 'GMT +9:00', 'GMT +10:00', 'GMT +11:00',
+                    'GMT +12:00'
+                ],
+            }
+        },
+        'http_proxy': {
+            '_self': {
+                'field': 'general',
+                'default_value': 'http://10.145.88.211:3128',
+                'options': [
+                    'http://10.145.88.211:3128'
+                ],
+            }
+        },
+        'https_proxy': {
+            '_self': {
+                'field': 'general',
+                'default_value': 'http://10.145.88.211:3128',
+                'options': [
+                    'http://10.145.88.211:3128'
+                ],
+            }
+        },
+        'no_proxy': {
+            '_self': {
+                'field': 'general_list',
+                'default_value': [
+                    '127.0.0.1',
+                    'compass',
+                    '10.145.88.211'
+                ],
+                'options': [
+                    '127.0.0.1',
+                    'compass',
+                    '10.145.88.211'
+                ]
+            }
+        },
+        'ntp_server': {
+            '_self': {
+                'is_required': True,
+                'field': 'general',
+                'default_value': '10.145.88.211',
+                'options': [
+                    '10.145.88.211'
+                ]
+            }
+        },
+        'dns_servers': {
+            '_self': {
+                'is_required': True,
+                'field': 'general_list',
+                'default_value': [
+                    '10.145.88.211',
+                ],
+                'options': [
+                    '10.145.88.211'
+                ]
             }
         },
         'domain': {
@@ -26,6 +89,15 @@ METADATA = {
                 'options': ['ods.com'],
             }
         },
+        'search_path': {
+            '_self': {
+                'field': 'general_list',
+                'default_value': [
+                    'ods.com'
+                ],
+                'options': ['ods.com']
+            }
+        },
         'default_gateway': {
             '_self': {
                 'is_required': True,

conf/package_field/roles.conf

@@ -0,0 +1,3 @@
+NAME = 'roles'
+FIELD_TYPE = list
+DESCRIPTION = 'roles'

conf/package_installer/chef-icehouse.conf

@@ -1,5 +1,5 @@
-NAME = 'chef(icehouse)'
+NAME = 'chef'
-TYPE = 'chef'
+INSTANCE_NAME = 'chef(icehouse)'
-CONFIG = {
+SETTINGS = {
-    'url': 'https://127.0.0.1',
+    'url': 'https://127.0.0.1'
 }

conf/package_metadata/openstack.conf

@@ -4,7 +4,8 @@ METADATA = {
         '_self': {
             'required_in_whole_config': True,
         },
-        '$credentials': {
+        '$credential_type': {
+            '$credential': {
                 'username': {
                     '_self': {
                         'is_required': True,
@@ -17,6 +18,14 @@ METADATA = {
                         'field': 'password'
                     }
                 }
+            }
+        }
+    },
+    'roles': {
+        '_self': {
+            'required_in_whole_config': True,
+            'field': 'roles',
+            'options': [],
         },
     },
     'network_mapping': {
@@ -24,12 +33,10 @@ METADATA = {
             'required_in_whole_config': True
         },
         '$interface_type': {
-            'interface': {
             '_self': {
                 'is_required': True,
                 'field': 'general'
             }
         }
     }
-    }
 }