Separate progress into host and package.

Change-Id: I72ae548037f77ec761d10aa869d9954873fb47f5

compass/db/v1/model.py

@@ -0,0 +1,724 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""database model."""
+from datetime import datetime
+from hashlib import md5
+import logging
+import simplejson as json
+import uuid
+
+from sqlalchemy import Column, ColumnDefault, Integer, String
+from sqlalchemy import Float, Enum, DateTime, ForeignKey, Text, Boolean
+from sqlalchemy import UniqueConstraint
+from sqlalchemy.orm import relationship, backref
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.ext.hybrid import hybrid_property
+
+from compass.utils import util
+
+from flask.ext.login import UserMixin
+from itsdangerous import URLSafeTimedSerializer
+
+BASE = declarative_base()
+#TODO(grace) SECRET_KEY should be generated when installing compass
+#and save to a config file or DB
+SECRET_KEY = "abcd"
+
+#This is used for generating a token by user's ID and
+#decode the ID from this token
+login_serializer = URLSafeTimedSerializer(SECRET_KEY)
+
+
+class User(BASE, UserMixin):
+    """User table."""
+    __tablename__ = 'user'
+    id = Column(Integer, primary_key=True)
+    email = Column(String(80), unique=True)
+    password = Column(String(225), default='')
+    active = Column(Boolean, default=True)
+
+    def __init__(self, email, password, **kwargs):
+        self.email = email
+        self.password = self._set_password(password)
+
+    def __repr__(self):
+        return '<User name: %s>' % self.email
+
+    def _set_password(self, password):
+        return self._hash_password(password)
+
+    def get_password(self):
+        return self.password
+
+    def valid_password(self, password):
+        return self.password == self._hash_password(password)
+
+    def get_auth_token(self):
+        return login_serializer.dumps(self.id)
+
+    def is_active(self):
+        return self.active
+
+    def _hash_password(self, password):
+        return md5(password).hexdigest()
+
+
+class SwitchConfig(BASE):
+    """Swtich Config table.
+
+    :param id: The unique identifier of the switch config.
+    :param ip: The IP address of the switch.
+    :param filter_port: The port of the switch which need to be filtered.
+    """
+    __tablename__ = 'switch_config'
+    id = Column(Integer, primary_key=True)
+    ip = Column(String(80))
+    filter_port = Column(String(16))
+    __table_args__ = (UniqueConstraint('ip', 'filter_port', name='filter1'), )
+
+    def __init__(self, **kwargs):
+        super(SwitchConfig, self).__init__(**kwargs)
+
+
+class Switch(BASE):
+    """Switch table.
+
+    :param id: the unique identifier of the switch. int as primary key.
+    :param ip: the IP address of the switch.
+    :param vendor_info: the name of the vendor
+    :param credential_data: used for accessing and retrieving information
+                            from the switch. Store json format as string.
+    :param state: Enum.'initialized/repolling': polling switch not complete to
+                  learn all MAC addresses of devices connected to the switch;
+                  'unreachable': one of the final state, indicates that the
+                  switch is unreachable at this time, no MAC address could be
+                  retrieved from the switch.
+                  'notsupported': one of the final state, indicates that the
+                  vendor found is not supported yet, no MAC address will be
+                  retrieved from the switch.
+                  'error': one of the final state, indicates that something
+                           wrong happend.
+                  'under_monitoring': one of the final state, indicates that
+                  MAC addresses has been learned successfully from the switch.
+    :param err_msg: Error message when polling switch failed.
+    :param machines: refer to list of Machine connected to the switch.
+    """
+    __tablename__ = 'switch'
+
+    id = Column(Integer, primary_key=True)
+    ip = Column(String(80), unique=True)
+    credential_data = Column(Text)
+    vendor_info = Column(String(256), nullable=True)
+    state = Column(Enum('initialized', 'unreachable', 'notsupported',
+                        'repolling', 'error', 'under_monitoring',
+                        name='switch_state'),
+                   default='initialized')
+    err_msg = Column(Text)
+
+    def __init__(self, **kwargs):
+        super(Switch, self).__init__(**kwargs)
+
+    def __repr__(self):
+        return '<Switch ip: %r, credential: %r, vendor: %r, state: %s>'\
+            % (self.ip, self.credential, self.vendor, self.state)
+
+    @hybrid_property
+    def vendor(self):
+        """vendor property getter"""
+        return self.vendor_info
+
+    @vendor.setter
+    def vendor(self, value):
+        """vendor property setter"""
+        self.vendor_info = value
+
+    @property
+    def credential(self):
+        """credential data getter.
+
+        :returns: python primitive dictionary object.
+        """
+        if self.credential_data:
+            try:
+                credential = json.loads(self.credential_data)
+                return credential
+            except Exception as error:
+                logging.error('failed to load credential data %s: %s',
+                              self.id, self.credential_data)
+                logging.exception(error)
+                raise error
+        else:
+            return {}
+
+    @credential.setter
+    def credential(self, value):
+        """credential property setter
+
+        :param value: dict of configuration data needed to update.
+        """
+        if value:
+            try:
+                credential = {}
+                if self.credential_data:
+                    credential = json.loads(self.credential_data)
+
+                credential.update(value)
+                self.credential_data = json.dumps(credential)
+
+            except Exception as error:
+                logging.error('failed to dump credential data %s: %s',
+                              self.id, value)
+                logging.exception(error)
+                raise error
+
+        else:
+            self.credential_data = json.dumps({})
+
+        logging.debug('switch now is %s', self)
+
+
+class Machine(BASE):
+    """Machine table.
+
+       .. note::
+          currently, we are taking care of management plane.
+          Therefore, we assume one machine is connected to one switch.
+
+    :param id: int, identity as primary key
+    :param mac: string, the MAC address of the machine.
+    :param switch_id: switch id that this machine connected on to.
+    :param port: nth port of the switch that this machine connected.
+    :param vlan: vlan id that this machine connected on to.
+    :param update_timestamp: last time this entry got updated.
+    :param switch: refer to the Switch the machine connects to.
+    """
+    __tablename__ = 'machine'
+
+    id = Column(Integer, primary_key=True)
+    mac = Column(String(24), default='')
+    port = Column(String(16), default='')
+    vlan = Column(Integer, default=0)
+    update_timestamp = Column(DateTime, default=datetime.now,
+                              onupdate=datetime.now)
+    switch_id = Column(Integer, ForeignKey('switch.id',
+                                           onupdate='CASCADE',
+                                           ondelete='SET NULL'))
+    __table_args__ = (UniqueConstraint('mac', 'switch_id',
+                                       name='unique_machine'),)
+    switch = relationship('Switch', backref=backref('machines',
+                                                    lazy='dynamic'))
+
+    def __init__(self, **kwargs):
+        super(Machine, self).__init__(**kwargs)
+
+    def __repr__(self):
+        return '<Machine %r: port=%r vlan=%r switch=%r>' % (
+            self.mac, self.port, self.vlan, self.switch)
+
+
+class HostState(BASE):
+    """The state of the ClusterHost.
+
+    :param id: int, identity as primary key.
+    :param state: Enum. 'UNINITIALIZED': the host is ready to setup.
+                 'INSTALLING': the host is not installing.
+                 'READY': the host is setup.
+                 'ERROR': the host has error.
+    :param progress: float, the installing progress from 0 to 1.
+    :param message: the latest installing message.
+    :param severity: Enum, the installing message severity.
+                     ('INFO', 'WARNING', 'ERROR')
+    :param update_timestamp: the lastest timestamp the entry got updated.
+    :param host: refer to ClusterHost.
+    :param os_progress: float, the installing progress of OS from 0 to 1.
+    """
+    __tablename__ = "host_state"
+
+    id = Column(Integer, ForeignKey('cluster_host.id',
+                                    onupdate='CASCADE',
+                                    ondelete='CASCADE'),
+                primary_key=True)
+    state = Column(Enum('UNINITIALIZED', 'INSTALLING', 'READY', 'ERROR'),
+                   ColumnDefault('UNINITIALIZED'))
+    progress = Column(Float, ColumnDefault(0.0))
+    message = Column(Text)
+    severity = Column(Enum('INFO', 'WARNING', 'ERROR'), ColumnDefault('INFO'))
+    update_timestamp = Column(DateTime, default=datetime.now,
+                              onupdate=datetime.now)
+    host = relationship('ClusterHost', backref=backref('state',
+                                                       uselist=False))
+
+    os_progress = Column(Float, ColumnDefault(0.0))
+    os_message = Column(Text)
+    os_severity = Column(
+        Enum('INFO', 'WARNING', 'ERROR'),
+        ColumnDefault('INFO')
+    )
+    """
+    this is added by Lei for separating os and package progress purposes
+    os_state = Column(Enum('UNINITIALIZED', 'INSTALLING', 'OS_READY', 'ERROR'),
+                   ColumnDefault('UNINITIALIZED'))
+    """
+
+    def __init__(self, **kwargs):
+        super(HostState, self).__init__(**kwargs)
+
+    @hybrid_property
+    def hostname(self):
+        """hostname getter"""
+        return self.host.hostname
+
+    @hybrid_property
+    def fullname(self):
+        """fullname getter"""
+        return self.host.fullname
+
+    def __repr__(self):
+        return (
+            '<HostState %r: state=%r, progress=%s, '
+            'message=%s, severity=%s, os_progress=%s>'
+        ) % (
+            self.hostname, self.state, self.progress,
+            self.message, self.severity, self.os_progress
+        )
+
+
+class ClusterState(BASE):
+    """The state of the Cluster.
+
+    :param id: int, identity as primary key.
+    :param state: Enum, 'UNINITIALIZED': the cluster is ready to setup.
+                 'INSTALLING': the cluster is not installing.
+                 'READY': the cluster is setup.
+                 'ERROR': the cluster has error.
+    :param progress: float, the installing progress from 0 to 1.
+    :param message: the latest installing message.
+    :param severity: Enum, the installing message severity.
+                     ('INFO', 'WARNING', 'ERROR').
+    :param update_timestamp: the lastest timestamp the entry got updated.
+    :param cluster: refer to Cluster.
+    """
+    __tablename__ = 'cluster_state'
+    id = Column(Integer, ForeignKey('cluster.id',
+                                    onupdate='CASCADE',
+                                    ondelete='CASCADE'),
+                primary_key=True)
+    state = Column(Enum('UNINITIALIZED', 'INSTALLING', 'READY', 'ERROR'),
+                   ColumnDefault('UNINITIALIZED'))
+    progress = Column(Float, ColumnDefault(0.0))
+    message = Column(Text)
+    severity = Column(Enum('INFO', 'WARNING', 'ERROR'), ColumnDefault('INFO'))
+    update_timestamp = Column(DateTime, default=datetime.now,
+                              onupdate=datetime.now)
+    cluster = relationship('Cluster', backref=backref('state',
+                                                      uselist=False))
+
+    def __init__(self, **kwargs):
+        super(ClusterState, self).__init__(**kwargs)
+
+    @hybrid_property
+    def clustername(self):
+        """clustername getter"""
+        return self.cluster.name
+
+    def __repr__(self):
+        return (
+            '<ClusterState %r: state=%r, progress=%s, '
+            'message=%s, severity=%s>'
+        ) % (
+            self.clustername, self.state, self.progress,
+            self.message, self.severity
+        )
+
+
+class Cluster(BASE):
+    """Cluster configuration information.
+
+    :param id: int, identity as primary key.
+    :param name: str, cluster name.
+    :param mutable: bool, if the Cluster is mutable.
+    :param security_config: str stores json formatted security information.
+    :param networking_config: str stores json formatted networking information.
+    :param partition_config: string stores json formatted parition information.
+    :param adapter_id: the refer id in the Adapter table.
+    :param raw_config: str stores json formatted other cluster information.
+    :param adapter: refer to the Adapter.
+    :param state: refer to the ClusterState.
+    """
+    __tablename__ = 'cluster'
+
+    id = Column(Integer, primary_key=True)
+    name = Column(String(80), unique=True)
+    mutable = Column(Boolean, default=True)
+    security_config = Column(Text)
+    networking_config = Column(Text)
+    partition_config = Column(Text)
+    adapter_id = Column(Integer, ForeignKey('adapter.id',
+                                            onupdate='CASCADE',
+                                            ondelete='SET NULL'),
+                        nullable=True)
+    raw_config = Column(Text)
+    adapter = relationship("Adapter", backref=backref('clusters',
+                                                      lazy='dynamic'))
+
+    def __init__(self, **kwargs):
+        if 'name' not in kwargs or not kwargs['name']:
+            kwargs['name'] = str(uuid.uuid4())
+
+        super(Cluster, self).__init__(**kwargs)
+
+    def __repr__(self):
+        return '<Cluster %r: config=%r>' % (self.name, self.config)
+
+    @property
+    def partition(self):
+        """partition getter"""
+        if self.partition_config:
+            try:
+                return json.loads(self.partition_config)
+            except Exception as error:
+                logging.error('failed to load security config %s: %s',
+                              self.id, self.partition_config)
+                logging.exception(error)
+                raise error
+        else:
+            return {}
+
+    @partition.setter
+    def partition(self, value):
+        """partition setter"""
+        logging.debug('cluster %s set partition %s', self.id, value)
+        if value:
+            try:
+                self.partition_config = json.dumps(value)
+            except Exception as error:
+                logging.error('failed to dump partition config %s: %s',
+                              self.id, value)
+                logging.exception(error)
+                raise error
+        else:
+            self.partition_config = None
+
+    @property
+    def security(self):
+        """security getter"""
+        if self.security_config:
+            try:
+                return json.loads(self.security_config)
+            except Exception as error:
+                logging.error('failed to load security config %s: %s',
+                              self.id, self.security_config)
+                logging.exception(error)
+                raise error
+        else:
+            return {}
+
+    @security.setter
+    def security(self, value):
+        """security setter"""
+        logging.debug('cluster %s set security %s', self.id, value)
+        if value:
+            try:
+                self.security_config = json.dumps(value)
+            except Exception as error:
+                logging.error('failed to dump security config %s: %s',
+                              self.id, value)
+                logging.exception(error)
+                raise error
+        else:
+            self.security_config = None
+
+    @property
+    def networking(self):
+        """networking getter"""
+        if self.networking_config:
+            try:
+                return json.loads(self.networking_config)
+            except Exception as error:
+                logging.error('failed to load networking config %s: %s',
+                              self.id, self.networking_config)
+                logging.exception(error)
+                raise error
+        else:
+            return {}
+
+    @networking.setter
+    def networking(self, value):
+        """networking setter."""
+        logging.debug('cluster %s set networking %s', self.id, value)
+        if value:
+            try:
+                self.networking_config = json.dumps(value)
+            except Exception as error:
+                logging.error('failed to dump networking config %s: %s',
+                              self.id, value)
+                logging.exception(error)
+                raise error
+        else:
+            self.networking_config = None
+
+    @hybrid_property
+    def config(self):
+        """get config from security, networking, partition."""
+        config = {}
+        if self.raw_config:
+            try:
+                config = json.loads(self.raw_config)
+            except Exception as error:
+                logging.error('failed to load raw config %s: %s',
+                              self.id, self.raw_config)
+                logging.exception(error)
+                raise error
+
+        util.merge_dict(config, {'security': self.security})
+        util.merge_dict(config, {'networking': self.networking})
+        util.merge_dict(config, {'partition': self.partition})
+        util.merge_dict(config, {'clusterid': self.id,
+                                 'clustername': self.name})
+        return config
+
+    @config.setter
+    def config(self, value):
+        """set config to security, networking, partition."""
+        logging.debug('cluster %s set config %s', self.id, value)
+        if not value:
+            self.security = None
+            self.networking = None
+            self.partition = None
+            self.raw_config = None
+            return
+
+        self.security = value.get('security')
+        self.networking = value.get('networking')
+        self.partition = value.get('partition')
+
+        try:
+            self.raw_config = json.dumps(value)
+        except Exception as error:
+            logging.error('failed to dump raw config %s: %s',
+                          self.id, value)
+            logging.exception(error)
+            raise error
+
+
+class ClusterHost(BASE):
+    """ClusterHost information.
+
+    :param id: int, identity as primary key.
+    :param machine_id: int, the id of the Machine.
+    :param cluster_id: int, the id of the Cluster.
+    :param mutable: if the ClusterHost information is mutable.
+    :param hostname: str, host name.
+    :param config_data: string, json formatted config data.
+    :param cluster: refer to Cluster the host in.
+    :param machine: refer to the Machine the host on.
+    :param state: refer to HostState indicates the host state.
+    """
+    __tablename__ = 'cluster_host'
+
+    id = Column(Integer, primary_key=True)
+
+    machine_id = Column(Integer, ForeignKey('machine.id',
+                                            onupdate='CASCADE',
+                                            ondelete='CASCADE'),
+                        nullable=True, unique=True)
+
+    cluster_id = Column(Integer, ForeignKey('cluster.id',
+                                            onupdate='CASCADE',
+                                            ondelete='SET NULL'),
+                        nullable=True)
+
+    hostname = Column(String(80))
+    config_data = Column(Text)
+    mutable = Column(Boolean, default=True)
+    __table_args__ = (UniqueConstraint('cluster_id', 'hostname',
+                                       name='unique_host'),)
+
+    cluster = relationship("Cluster",
+                           backref=backref('hosts', lazy='dynamic'))
+    machine = relationship("Machine",
+                           backref=backref('host', uselist=False))
+
+    def __init__(self, **kwargs):
+        if 'hostname' not in kwargs or not kwargs['hostname']:
+            kwargs['hostname'] = str(uuid.uuid4())
+
+        super(ClusterHost, self).__init__(**kwargs)
+
+    def __repr__(self):
+        return '<ClusterHost %r: cluster=%r machine=%r>' % (
+            self.hostname, self.cluster, self.machine)
+
+    @hybrid_property
+    def fullname(self):
+        return '%s.%s' % (self.hostname, self.cluster.id)
+
+    @property
+    def config(self):
+        """config getter."""
+        config = {}
+        try:
+            if self.config_data:
+                config.update(json.loads(self.config_data))
+
+            config.update({
+                'hostid': self.id,
+                'hostname': self.hostname,
+            })
+            if self.cluster:
+                config.update({
+                    'clusterid': self.cluster.id,
+                    'clustername': self.cluster.name,
+                    'fullname': self.fullname,
+                })
+
+            if self.machine:
+                util.merge_dict(
+                    config, {
+                        'networking': {
+                            'interfaces': {
+                                'management': {
+                                    'mac': self.machine.mac
+                                }
+                            }
+                        },
+                        'switch_port': self.machine.port,
+                        'vlan': self.machine.vlan,
+                    })
+                if self.machine.switch:
+                    util.merge_dict(
+                        config, {'switch_ip': self.machine.switch.ip})
+
+        except Exception as error:
+            logging.error('failed to load config %s: %s',
+                          self.hostname, self.config_data)
+            logging.exception(error)
+            raise error
+
+        return config
+
+    @config.setter
+    def config(self, value):
+        """config setter"""
+        if not self.config_data:
+            config = {
+            }
+            self.config_data = json.dumps(config)
+
+        if value:
+            try:
+                config = json.loads(self.config_data)
+                util.merge_dict(config, value)
+
+                self.config_data = json.dumps(config)
+            except Exception as error:
+                logging.error('failed to dump config %s: %s',
+                              self.hostname, value)
+                logging.exception(error)
+                raise error
+
+
+class LogProgressingHistory(BASE):
+    """host installing log history for each file.
+
+    :param id: int, identity as primary key.
+    :param pathname: str, the full path of the installing log file. unique.
+    :param position: int, the position of the log file it has processed.
+    :param partial_line: str, partial line of the log.
+    :param progressing: float, indicate the installing progress between 0 to 1.
+    :param message: str, str, the installing message.
+    :param severity: Enum, the installing message severity.
+                     ('ERROR', 'WARNING', 'INFO')
+    :param line_matcher_name: str, the line matcher name of the log processor.
+    :param update_timestamp: datetime, the latest timestamp the entry updated.
+    """
+    __tablename__ = 'log_progressing_history'
+    id = Column(Integer, primary_key=True)
+    pathname = Column(String(80), unique=True)
+    position = Column(Integer, ColumnDefault(0))
+    partial_line = Column(Text)
+    progress = Column(Float, ColumnDefault(0.0))
+    message = Column(Text)
+    severity = Column(Enum('ERROR', 'WARNING', 'INFO'), ColumnDefault('INFO'))
+    line_matcher_name = Column(String(80), ColumnDefault('start'))
+    update_timestamp = Column(DateTime, default=datetime.now,
+                              onupdate=datetime.now)
+
+    def __init__(self, **kwargs):
+        super(LogProgressingHistory, self).__init__(**kwargs)
+
+    def __repr__(self):
+        return (
+            'LogProgressingHistory[%r: position %r,'
+            'partial_line %r,progress %r,message %r,'
+            'severity %r]'
+        ) % (
+            self.pathname, self.position,
+            self.partial_line,
+            self.progress,
+            self.message,
+            self.severity
+        )
+
+
+class Adapter(BASE):
+    """Table stores ClusterHost installing Adapter information.
+
+    :param id: int, identity as primary key.
+    :param name: string, adapter name, unique.
+    :param os: string, os name for installing the host.
+    :param target_system: string, target system to be installed on the host.
+    :param clusters: refer to the list of Cluster.
+    """
+    __tablename__ = 'adapter'
+    id = Column(Integer, primary_key=True)
+    name = Column(String(80), unique=True)
+    os = Column(String(80))
+    target_system = Column(String(80))
+    __table_args__ = (
+        UniqueConstraint('os', 'target_system', name='unique_adapter'),)
+
+    def __init__(self, **kwargs):
+        super(Adapter, self).__init__(**kwargs)
+
+    def __repr__(self):
+        return '<Adapter %r: os %r, target_system %r>' % (
+            self.name, self.os, self.target_system
+        )
+
+
+class Role(BASE):
+    """The Role table stores avaiable roles of one target system.
+
+       .. note::
+          the host can be deployed to one or several roles in the cluster.
+
+    :param id: int, identity as primary key.
+    :param name: role name.
+    :param target_system: str, the target_system.
+    :param description: str, the description of the role.
+    """
+    __tablename__ = 'role'
+    id = Column(Integer, primary_key=True)
+    name = Column(String(80), unique=True)
+    target_system = Column(String(80))
+    description = Column(Text)
+
+    def __init__(self, **kwargs):
+        super(Role, self).__init__(**kwargs)
+
+    def __repr__(self):
+        return '<Role %r : target_system %r, description:%r>' % (
+            self.name, self.target_system, self.description)

compass/log_analyzor/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

compass/log_analyzor/adapter_matcher.py

@@ -0,0 +1,467 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module to provider installing progress calculation for the adapter.
+
+   .. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
+"""
+import logging
+import re
+
+from compass.db import database
+from compass.db.model import Cluster
+from compass.db.model import ClusterHost
+from compass.log_analyzor.line_matcher import Progress
+
+
+class AdapterItemMatcher(object):
+    """Progress matcher for the os installing or package installing."""
+
+    def __init__(self, file_matchers):
+        self.file_matchers_ = file_matchers
+        self.min_progress_ = 0.0
+        self.max_progress_ = 1.0
+
+    def update_progress_range(self, min_progress, max_progress):
+        """update min_progress and max_progress."""
+        self.min_progress_ = min_progress
+        self.max_progress_ = max_progress
+        for file_matcher in self.file_matchers_:
+            file_matcher.update_absolute_progress_range(
+                self.min_progress_, self.max_progress_)
+
+    def __str__(self):
+        return '%s[file_matchers: %s, min_progress: %s, max_progress: %s]' % (
+            self.__class__.__name__, self.file_matchers_,
+            self.min_progress_, self.max_progress_)
+
+    def update_progress(self, fullname, progress):
+        """Update progress.
+
+        :param fullname: the fullname of the installing host.
+        :type fullname: str
+        :param progress: Progress instance to update.
+        """
+        for file_matcher in self.file_matchers_:
+            file_matcher.update_progress(fullname, progress)
+
+
+class OSMatcher(object):
+    """Progress matcher for os installer."""
+
+    def __init__(self, os_installer_name, os_pattern,
+                 item_matcher, min_progress, max_progress):
+        if not (0.0 <= min_progress <= max_progress <= 1.0):
+            raise IndexError('%s restriction not mat:'
+                             '0.0 <= min_progress(%s) '
+                             '<= max_progress(%s) <= 1.0' % (
+                                 self.__class__.__name__,
+                                 min_progress, max_progress))
+
+        self.name_ = os_installer_name
+        self.os_regex_ = re.compile(os_pattern)
+        self.matcher_ = item_matcher
+        self.matcher_.update_progress_range(min_progress, max_progress)
+
+    def __repr__(self):
+        return '%s[name:%s, os_pattern:%s, matcher:%s]' % (
+            self.__class__.__name__, self.name_,
+            self.os_regex_.pattern, self.matcher_)
+
+    def match(self, os_installer_name, os_name):
+        """Check if the os matcher is acceptable."""
+        return all([
+            self.name_ == os_installer_name,
+            self.os_regex_.match(os_name)])
+
+    def update_progress(self, fullname, progress):
+        """Update progress."""
+        logging.debug('selfname: %s', self.name_)
+        self.matcher_.update_progress(fullname, progress)
+
+
+class PackageMatcher(object):
+    """Progress matcher for package installer."""
+
+    def __init__(self, package_installer_name, target_system,
+                 item_matcher, min_progress, max_progress):
+        if not (0.0 <= min_progress <= max_progress <= 1.0):
+            raise IndexError('%s restriction not mat:'
+                             '0.0 <= min_progress(%s) '
+                             '<= max_progress(%s) <= 1.0' % (
+                                 self.__class__.__name__,
+                                 min_progress, max_progress))
+
+        self.name_ = package_installer_name
+        self.target_system_ = target_system
+        self.matcher_ = item_matcher
+        self.matcher_.update_progress_range(min_progress, max_progress)
+
+    def __repr__(self):
+        return '%s[name:%s, target_system:%s, matcher:%s]' % (
+            self.__class__.__name__, self.name_,
+            self.target_system_, self.matcher_)
+
+    def match(self, package_installer_name, target_system):
+        """Check if the package matcher is acceptable."""
+        return all([
+            self.name_ == package_installer_name,
+            self.target_system_ == target_system])
+
+    def update_progress(self, fullname, progress):
+        """Update progress."""
+        self.matcher_.update_progress(fullname, progress)
+
+
+class AdapterMatcher(object):
+    """Adapter matcher to update adapter installing progress."""
+
+    def __init__(self, os_matcher, package_matcher):
+        self.os_matcher_ = os_matcher
+        self.package_matcher_ = package_matcher
+
+    def match(self, os_installer_name, os_name,
+              package_installer_name, target_system):
+        """Check if the adapter matcher is acceptable.
+
+        :param os_installer_name: the os installer name.
+        :type os_installer_name: str
+        :param os_name: the os name.
+        :type os_name: str
+        :param package_installer_name: the package installer name.
+        :type package_installer_name: str
+        :param target_system: the target system to deploy
+        :type target_system: str
+
+        :returns: bool
+
+           .. note::
+              Return True if the AdapterMatcher can process the log files
+              generated from the os installation and package installation.
+        """
+        return all([
+            self.os_matcher_.match(os_installer_name, os_name),
+            self.package_matcher_.match(
+                package_installer_name, target_system)])
+
+    def __str__(self):
+        return '%s[os_matcher:%s, package_matcher:%s]' % (
+            self.__class__.__name__,
+            self.os_matcher_, self.package_matcher_)
+
+    @classmethod
+    def _get_host_progress(cls, hostid):
+        """Get Host Progress from database.
+
+        .. notes::
+           The function should be called in database session.
+        """
+        session = database.current_session()
+        host = session.query(
+            ClusterHost
+        ).filter_by(id=hostid).first()
+        if not host:
+            logging.error(
+                'there is no host for %s in ClusterHost', hostid)
+            return None, None, None
+
+        if not host.state:
+            logging.error('there is no related HostState for %s',
+                          hostid)
+            return host.fullname, None, None
+
+        """
+        return (
+            host.fullname,
+            host.state.state,
+            Progress(host.state.progress,
+                     host.state.message,
+                     host.state.severity))
+        """
+        return {
+            'os': (
+                host.fullname,
+                host.state.state,
+                Progress(host.state.os_progress,
+                         host.state.os_message,
+                         host.state.os_severity)),
+            'package': (
+                host.fullname,
+                host.state.state,
+                Progress(host.state.progress,
+                         host.state.message,
+                         host.state.severity))}
+
+    @classmethod
+    def _update_host_os_progress(cls, hostid, os_progress):
+        """Update host progress to database.
+
+        .. note::
+           The function should be called in database session.
+        """
+        session = database.current_session()
+        host = session.query(
+            ClusterHost).filter_by(id=hostid).first()
+        if not host:
+            logging.error(
+                'there is no host for %s in ClusterHost', hostid)
+            return
+
+        if not host.state:
+            logging.error(
+                'there is no related HostState for %s', hostid)
+            return
+
+        logging.debug('os progress: %s', os_progress.progress)
+
+        if host.state.os_progress > os_progress.progress:
+            logging.error(
+                'host %s os_progress is not increased '
+                'from %s to %s',
+                hostid, host.state, os_progress)
+            return
+        if (
+            host.state.os_progress == os_progress.progress and
+            host.state.os_message == os_progress.message
+        ):
+            logging.info(
+                'ignore update host %s progress %s to %s',
+                hostid, os_progress, host.state)
+            return
+
+        host.state.os_progress = os_progress.progress
+        """host.state.os_progress = progress.progress"""
+        host.state.os_message = os_progress.message
+        if os_progress.severity:
+            host.state.os_severity = os_progress.severity
+
+        if host.state.os_progress >= 1.0:
+            host.state.os_state = 'OS_READY'
+
+        if host.state.os_severity == 'ERROR':
+            host.state.os_state = 'ERROR'
+
+        if host.state.os_state != 'INSTALLING':
+            host.mutable = True
+
+        logging.debug(
+            'update host %s state %s',
+            hostid, host.state)
+
+    @classmethod
+    def _update_host_package_progress(cls, hostid, progress):
+        """Update host progress to database.
+
+        .. note::
+           The function should be called in database session.
+        """
+        session = database.current_session()
+        host = session.query(
+            ClusterHost).filter_by(id=hostid).first()
+
+        logging.debug('package progress: %s', progress.progress)
+        logging.debug('package ssssstate: %s', host.state.state)
+        if not host:
+            logging.error(
+                'there is no host for %s in ClusterHost', hostid)
+            return
+
+        if not host.state:
+            logging.error(
+                'there is no related HostState for %s', hostid)
+            return
+
+        if not host.state.state in ['OS_READY', 'INSTALLING']:
+            logging.error(
+                'host %s issssss not in INSTALLING state',
+                hostid)
+            return
+
+        if host.state.progress > progress.progress:
+            logging.error(
+                'host %s progress is not increased '
+                'from %s to %s',
+                hostid, host.state, progress)
+            return
+
+        if (
+            host.state.progress == progress.progress and
+            host.state.message == progress.message
+        ):
+            logging.info(
+                'ignore update host %s progress %s to %s',
+                hostid, progress, host.state)
+            return
+
+        host.state.progress = progress.progress
+        host.state.message = progress.message
+        if progress.severity:
+            host.state.severity = progress.severity
+
+        if host.state.progress >= 1.0:
+            host.state.state = 'READY'
+
+        if host.state.severity == 'ERROR':
+            host.state.state = 'ERROR'
+
+        if host.state.state != 'INSTALLING':
+            host.mutable = True
+
+        logging.debug(
+            'update host %s state %s',
+            hostid, host.state)
+
+    @classmethod
+    def _update_cluster_progress(cls, clusterid):
+        """Update cluster installing progress to database.
+
+        .. note::
+           The function should be called in the database session.
+        """
+        session = database.current_session()
+        cluster = session.query(
+            Cluster).filter_by(id=clusterid).first()
+        if not cluster:
+            logging.error(
+                'there is no cluster for %s in Cluster',
+                clusterid)
+            return
+
+        if not cluster.state:
+            logging.error(
+                'there is no ClusterState for %s',
+                clusterid)
+
+        if cluster.state.state != 'INSTALLING':
+            logging.error('cluster %s is not in INSTALLING state',
+                          clusterid)
+            return
+
+        cluster_progress = 0.0
+        cluster_messages = {}
+        cluster_severities = set([])
+        hostids = []
+        for host in cluster.hosts:
+            if host.state:
+                hostids.append(host.id)
+                cluster_progress += host.state.progress
+                if host.state.message:
+                    cluster_messages[host.hostname] = host.state.message
+
+                if host.state.severity:
+                    cluster_severities.add(host.state.severity)
+
+        cluster.state.progress = cluster_progress / len(hostids)
+        cluster.state.message = '\n'.join(
+            [
+                '%s: %s' % (hostname, message)
+                for hostname, message in cluster_messages.items()
+            ]
+        )
+        for severity in ['ERROR', 'WARNING', 'INFO']:
+            if severity in cluster_severities:
+                cluster.state.severity = severity
+                break
+
+        if cluster.state.progress >= 1.0:
+            cluster.state.state = 'READY'
+
+        if cluster.state.severity == 'ERROR':
+            cluster.state.state = 'ERROR'
+
+        if cluster.state.state != 'INSTALLING':
+            cluster.mutable = True
+
+        logging.debug(
+            'update cluster %s state %s',
+            clusterid, cluster.state)
+
+    def update_progress(self, clusterid, hostids):
+        """Update cluster progress and hosts progresses.
+
+        :param clusterid: the id of the cluster to update.
+        :type clusterid: int.
+        :param hostids: the ids of the hosts to update.
+        :type hostids: list of int.
+        """
+        logging.debug('printing os_matcher %s', self.__str__())
+        host_os_progresses = {}
+        host_package_progresses = {}
+        with database.session():
+            for hostid in hostids:
+                host_overall_state = (
+                    self._get_host_progress(hostid))
+                logging.debug('host overall state: %s', host_overall_state)
+                fullname, host_state, host_os_progress = (
+                    host_overall_state['os'])
+                _, _, host_package_progress = host_overall_state['package']
+                if (not fullname or
+                        not host_os_progress or
+                        not host_package_progress):
+                    logging.error(
+                        'nothing to update host %s',
+                        fullname)
+                    continue
+
+                logging.debug('got host %s state %s os_progress %s'
+                              'package_progress %s',
+                              fullname,
+                              host_state, host_os_progress,
+                              host_package_progress)
+
+                host_os_progresses[hostid] = (
+                    fullname, host_state, host_os_progress)
+                host_package_progresses[hostid] = (
+                    fullname, host_state, host_package_progress)
+
+        for hostid, host_value in host_os_progresses.items():
+            fullname, host_state, host_os_progress = host_value
+            if host_state == 'INSTALLING' and host_os_progress.progress < 1.0:
+                self.os_matcher_.update_progress(
+                    fullname, host_os_progress)
+            else:
+                logging.error(
+                    'there is no need to update host %s '
+                    'OS progress: state %s os_progress %s',
+                    fullname, host_state, host_os_progress)
+
+        for hostid, host_value in host_package_progresses.items():
+            fullname, host_state, host_package_progress = host_value
+            if (host_state == 'INSTALLING' and
+                    host_package_progress.progress < 1.0):
+                self.package_matcher_.update_progress(
+                    fullname, host_package_progress)
+            else:
+                logging.error(
+                    'there is no need to update host %s '
+                    'Package progress: state %s package_progress %s',
+                    fullname, host_state, host_package_progress)
+
+        with database.session():
+            for hostid in hostids:
+                if hostid not in host_os_progresses:
+                    continue
+
+                if hostid not in host_package_progresses:
+                    continue
+
+                _, _, host_os_progress = host_os_progresses[hostid]
+                _, _, host_package_progress = host_package_progresses[hostid]
+                self._update_host_os_progress(hostid, host_os_progress)
+                self._update_host_package_progress(
+                    hostid,
+                    host_package_progress
+                )
+
+            self._update_cluster_progress(clusterid)

compass/log_analyzor/file_matcher.py

@@ -0,0 +1,347 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module to update intalling progress by processing log file.
+
+   .. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
+"""
+import logging
+import os.path
+
+from compass.db import database
+from compass.db.model import LogProgressingHistory
+from compass.log_analyzor.line_matcher import Progress
+from compass.utils import setting_wrapper as setting
+
+
+class FileFilter(object):
+    """base class to filter log file."""
+    def __repr__(self):
+        return self.__class__.__name__
+
+    def filter(self, pathname):
+        """Filter log file.
+
+        :param pathname: the absolute path name to the log file.
+        """
+        raise NotImplementedError(str(self))
+
+
+class CompositeFileFilter(FileFilter):
+    """filter log file based on the list of filters."""
+    def __init__(self, filters):
+        self.filters_ = filters
+
+    def __str__(self):
+        return 'CompositeFileFilter[%s]' % self.filters_
+
+    def append_filter(self, file_filter):
+        """append filter."""
+        self.filters_.append(file_filter)
+
+    def filter(self, pathname):
+        """filter log file."""
+        for file_filter in self.filters_:
+            if not file_filter.filter(pathname):
+                return False
+
+        return True
+
+
+class FilterFileExist(FileFilter):
+    """filter log file if not exists."""
+    def filter(self, pathname):
+        """filter log file."""
+        file_exist = os.path.isfile(pathname)
+        if not file_exist:
+            logging.error("%s is not exist", pathname)
+
+        return file_exist
+
+
+def get_file_filter():
+    """get file filter"""
+    composite_filter = CompositeFileFilter([FilterFileExist()])
+    return composite_filter
+
+
+class FileReader(object):
+    """Class to read log file.
+
+    The class provide support to read log file from the position
+    it has read last time. and update the position when it finish
+    reading the log.
+    """
+    def __init__(self, pathname):
+        self.pathname_ = pathname
+        self.position_ = 0
+        self.partial_line_ = ''
+
+    def __repr__(self):
+        return (
+            '%s[pathname:%s, position:%s, partial_line:%s]' % (
+                self.__class__.__name__, self.pathname_, self.position_,
+                self.partial_line_
+            )
+        )
+
+    def get_history(self):
+        """Get log file read history from database.
+
+        :returns: (line_matcher_name progress)
+
+        .. note::
+           The function should be called out of database session.
+           It reads the log_progressing_history table to get the
+           position in the log file it has read in last run,
+           the partial line of the log, the line matcher name
+           in the last run, the progress, the message and the
+           severity it has got in the last run.
+        """
+        with database.session() as session:
+            history = session.query(
+                LogProgressingHistory
+            ).filter_by(
+                pathname=self.pathname_
+            ).first()
+            if history:
+                self.position_ = history.position
+                self.partial_line_ = history.partial_line
+                line_matcher_name = history.line_matcher_name
+                progress = Progress(history.progress,
+                                    history.message,
+                                    history.severity)
+            else:
+                line_matcher_name = 'start'
+                progress = Progress(0.0, '', None)
+
+            return line_matcher_name, progress
+
+    def update_history(self, line_matcher_name, progress):
+        """Update log_progressing_history table.
+
+        :param line_matcher_name: the line matcher name.
+        :param progress: Progress instance to record the installing progress.
+
+        .. note::
+           The function should be called out of database session.
+           It updates the log_processing_history table.
+        """
+        with database.session() as session:
+            history = session.query(LogProgressingHistory).filter_by(
+                pathname=self.pathname_).first()
+
+            if history:
+                if history.position >= self.position_:
+                    logging.error(
+                        '%s history position %s is ahead of current '
+                        'position %s',
+                        self.pathname_,
+                        history.position,
+                        self.position_)
+                    return
+
+                history.position = self.position_
+                history.partial_line = self.partial_line_
+                history.line_matcher_name = line_matcher_name
+                history.progress = progress.progress
+                history.message = progress.message
+                history.severity = progress.severity
+            else:
+                history = LogProgressingHistory(
+                    pathname=self.pathname_, position=self.position_,
+                    partial_line=self.partial_line_,
+                    line_matcher_name=line_matcher_name,
+                    progress=progress.progress,
+                    message=progress.message,
+                    severity=progress.severity)
+                session.merge(history)
+            logging.debug('update file %s to history %s',
+                          self.pathname_, history)
+
+    def readline(self):
+        """Generate each line of the log file."""
+        old_position = self.position_
+        try:
+            with open(self.pathname_) as logfile:
+                logfile.seek(self.position_)
+                while True:
+                    line = logfile.readline()
+                    self.partial_line_ += line
+                    position = logfile.tell()
+                    if position > self.position_:
+                        self.position_ = position
+
+                    if self.partial_line_.endswith('\n'):
+                        yield_line = self.partial_line_
+                        self.partial_line_ = ''
+                        yield yield_line
+                    else:
+                        break
+
+                if self.partial_line_:
+                    yield self.partial_line_
+
+        except Exception as error:
+            logging.error('failed to processing file %s', self.pathname_)
+            raise error
+
+        logging.debug(
+            'processing file %s log %s bytes to position %s',
+            self.pathname_, self.position_ - old_position,
+            self.position_)
+
+
+class FileReaderFactory(object):
+    """factory class to create FileReader instance."""
+
+    def __init__(self, logdir, filefilter):
+        self.logdir_ = logdir
+        self.filefilter_ = filefilter
+
+    def __str__(self):
+        return '%s[logdir: %s filefilter: %s]' % (
+            self.__class__.__name__, self.logdir_, self.filefilter_)
+
+    def get_file_reader(self, fullname, filename):
+        """Get FileReader instance.
+
+        :param fullname: fullname of installing host.
+        :param filename: the filename of the log file.
+
+        :returns: :class:`FileReader` instance if it is not filtered.
+        """
+        pathname = os.path.join(self.logdir_, fullname, filename)
+        logging.debug('get FileReader from %s', pathname)
+        if not self.filefilter_.filter(pathname):
+            logging.error('%s is filtered', pathname)
+            return None
+
+        return FileReader(pathname)
+
+
+FILE_READER_FACTORY = FileReaderFactory(
+    setting.INSTALLATION_LOGDIR, get_file_filter())
+
+
+class FileMatcher(object):
+    """File matcher to get the installing progress from the log file."""
+    def __init__(self, line_matchers, min_progress, max_progress, filename):
+        if not 0.0 <= min_progress <= max_progress <= 1.0:
+            raise IndexError(
+                '%s restriction is not mat: 0.0 <= min_progress'
+                '(%s) <= max_progress(%s) <= 1.0' % (
+                    self.__class__.__name__,
+                    min_progress,
+                    max_progress))
+
+        self.line_matchers_ = line_matchers
+        self.min_progress_ = min_progress
+        self.max_progress_ = max_progress
+        self.absolute_min_progress_ = 0.0
+        self.absolute_max_progress_ = 1.0
+        self.absolute_progress_diff_ = 1.0
+        self.filename_ = filename
+
+    def update_absolute_progress_range(self, min_progress, max_progress):
+        """update the min progress and max progress the log file indicates."""
+        progress_diff = max_progress - min_progress
+        self.absolute_min_progress_ = (
+            min_progress + self.min_progress_ * progress_diff)
+        self.absolute_max_progress_ = (
+            min_progress + self.max_progress_ * progress_diff)
+        self.absolute_progress_diff_ = (
+            self.absolute_max_progress_ - self.absolute_min_progress_)
+
+    def __str__(self):
+        return (
+            '%s[ filename: %s, progress range: [%s:%s], '
+            'line_matchers: %s]' % (
+                self.__class__.__name__, self.filename_,
+                self.absolute_min_progress_,
+                self.absolute_max_progress_, self.line_matchers_)
+        )
+
+    def update_total_progress(self, file_progress, total_progress):
+        """Get the total progress from file progress."""
+        if not file_progress.message:
+            logging.info(
+                'ignore update file %s progress %s to total progress',
+                self.filename_, file_progress)
+            return
+
+        total_progress_data = min(
+            (
+                self.absolute_min_progress_ + (
+                    file_progress.progress * self.absolute_progress_diff_
+                )
+            ),
+            self.absolute_max_progress_
+        )
+
+        # total progress should only be updated when the new calculated
+        # progress is greater than the recored total progress or the
+        # progress to update is the same but the message is different.
+        if (
+            total_progress.progress < total_progress_data or (
+                total_progress.progress == total_progress_data and
+                total_progress.message != file_progress.message
+            )
+        ):
+            total_progress.progress = total_progress_data
+            total_progress.message = file_progress.message
+            total_progress.severity = file_progress.severity
+            logging.debug('update file %s total progress %s',
+                          self.filename_, total_progress)
+        else:
+            logging.info(
+                'ignore update file %s progress %s to total progress %s',
+                self.filename_, file_progress, total_progress)
+
+    def update_progress(self, fullname, total_progress):
+        """update progress from file.
+
+        :param fullname: the fullname of the installing host.
+        :type fullname: str
+        :param total_progress: Progress instance to update.
+
+        the function update installing progress by reading the log file.
+        It contains a list of line matcher, when one log line matches
+        with current line matcher, the installing progress is updated.
+        and the current line matcher got updated.
+        Notes: some line may be processed multi times. The case is the
+        last line of log file is processed in one run, while in the other
+        run, it will be reprocessed at the beginning because there is
+        no line end indicator for the last line of the file.
+        """
+        file_reader = FILE_READER_FACTORY.get_file_reader(
+            fullname, self.filename_)
+        if not file_reader:
+            return
+
+        line_matcher_name, file_progress = file_reader.get_history()
+        for line in file_reader.readline():
+            if line_matcher_name not in self.line_matchers_:
+                logging.debug('early exit at\n%s\nbecause %s is not in %s',
+                              line, line_matcher_name, self.line_matchers_)
+                break
+
+            index = line_matcher_name
+            while index in self.line_matchers_:
+                line_matcher = self.line_matchers_[index]
+                index, line_matcher_name = line_matcher.update_progress(
+                    line, file_progress)
+
+        file_reader.update_history(line_matcher_name, file_progress)
+        self.update_total_progress(file_progress, total_progress)

compass/log_analyzor/line_matcher.py

@@ -0,0 +1,230 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module to get the progress when found match with a line of the log."""
+import logging
+import re
+
+from abc import ABCMeta
+
+from compass.utils import util
+
+
+class Progress(object):
+    """Progress object to store installing progress and message."""
+
+    def __init__(self, progress, message, severity):
+        """Constructor
+
+        :param progress: installing progress between 0 to 1.
+        :param message: installing message.
+        :param severity: installing message severity.
+        """
+        self.progress = progress
+        self.message = message
+        self.severity = severity
+
+    def __repr__(self):
+        return '%s[progress:%s, message:%s, severity:%s]' % (
+            self.__class__.__name__,
+            self.progress,
+            self.message,
+            self.severity)
+
+
+class ProgressCalculator(object):
+    """base class to generate progress."""
+
+    __metaclass__ = ABCMeta
+
+    @classmethod
+    def update_progress(
+        cls, progress_data, message,
+        severity, progress
+    ):
+        """Update progress with the given progress_data, message and severity.
+
+        :param progress_data: installing progress.
+        :type progress_data: float between 0 to 1.
+        :param message: installing progress message.
+        :param severity: installing message severity.
+        :param progress: :class:`Progress` instance to update
+        """
+        # the progress is only updated when the new progress
+        # is greater than the stored progress or the progress
+        # to update is the same but the message is different.
+        if (
+            progress_data > progress.progress or (
+                progress_data == progress.progress and
+                message != progress.message
+            )
+        ):
+            progress.progress = progress_data
+            if message:
+                progress.message = message
+
+            if severity:
+                progress.severity = severity
+
+            logging.debug('update progress to %s', progress)
+        else:
+            logging.info('ignore update progress %s to %s',
+                         progress_data, progress)
+
+    def update(self, message, severity, progress):
+        """vritual method to update progress by message and severity.
+
+        :param message: installing message.
+        :param severity: installing severity.
+        """
+        raise NotImplementedError(str(self))
+
+    def __repr__(self):
+        return self.__class__.__name__
+
+
+class IncrementalProgress(ProgressCalculator):
+    """Class to increment the progress."""
+
+    def __init__(self, min_progress,
+                 max_progress, incremental_ratio):
+        super(IncrementalProgress, self).__init__()
+        if not 0.0 <= min_progress <= max_progress <= 1.0:
+            raise IndexError(
+                '%s restriction is not mat: 0.0 <= min_progress(%s)'
+                ' <= max_progress(%s) <= 1.0' % (
+                    self.__class__.__name__, min_progress, max_progress))
+
+        if not 0.0 <= incremental_ratio <= 1.0:
+            raise IndexError(
+                '%s restriction is not mat: '
+                '0.0 <= incremental_ratio(%s) <=  1.0' % (
+                    self.__class__.__name__, incremental_ratio))
+
+        self.min_progress_ = min_progress
+        self.max_progress_ = max_progress
+        self.incremental_progress_ = (
+            incremental_ratio * (max_progress - min_progress))
+
+    def __str__(self):
+        return '%s[%s:%s:%s]' % (
+            self.__class__.__name__,
+            self.min_progress_,
+            self.max_progress_,
+            self.incremental_progress_
+        )
+
+    def update(self, message, severity, progress):
+        """update progress from message and severity."""
+        progress_data = max(
+            self.min_progress_,
+            min(
+                self.max_progress_,
+                progress.progress + self.incremental_progress_
+            )
+        )
+        self.update_progress(progress_data,
+                             message, severity, progress)
+
+
+class RelativeProgress(ProgressCalculator):
+    """class to update progress to the given relative progress."""
+
+    def __init__(self, progress):
+        super(RelativeProgress, self).__init__()
+        if not 0.0 <= progress <= 1.0:
+            raise IndexError(
+                '%s restriction is not mat: 0.0 <= progress(%s) <= 1.0' % (
+                    self.__class__.__name__, progress))
+
+        self.progress_ = progress
+
+    def __str__(self):
+        return '%s[%s]' % (self.__class__.__name__, self.progress_)
+
+    def update(self, message, severity, progress):
+        """update progress from message and severity."""
+        self.update_progress(
+            self.progress_, message, severity, progress)
+
+
+class SameProgress(ProgressCalculator):
+    """class to update message and severity for  progress."""
+
+    def update(self, message, severity, progress):
+        """update progress from the message and severity."""
+        self.update_progress(progress.progress, message,
+                             severity, progress)
+
+
+class LineMatcher(object):
+    """Progress matcher for each line."""
+
+    def __init__(self, pattern, progress=None,
+                 message_template='', severity=None,
+                 unmatch_sameline_next_matcher_name='',
+                 unmatch_nextline_next_matcher_name='',
+                 match_sameline_next_matcher_name='',
+                 match_nextline_next_matcher_name=''):
+        self.regex_ = re.compile(pattern)
+        if not progress:
+            self.progress_ = SameProgress()
+        elif isinstance(progress, ProgressCalculator):
+            self.progress_ = progress
+        elif util.is_instance(progress, [int, float]):
+            self.progress_ = RelativeProgress(progress)
+        else:
+            raise TypeError(
+                'progress unsupport type %s: %s' % (
+                    type(progress), progress))
+
+        self.message_template_ = message_template
+        self.severity_ = severity
+        self.unmatch_sameline_ = unmatch_sameline_next_matcher_name
+        self.unmatch_nextline_ = unmatch_nextline_next_matcher_name
+        self.match_sameline_ = match_sameline_next_matcher_name
+        self.match_nextline_ = match_nextline_next_matcher_name
+
+    def __str__(self):
+        return '%s[pattern:%r, message_template:%r, severity:%r]' % (
+            self.__class__.__name__, self.regex_.pattern,
+            self.message_template_, self.severity_)
+
+    def update_progress(self, line, progress):
+        """Update progress by the line.
+
+        :param line: one line in log file to indicate the installing progress.
+           .. note::
+              The line may be partial if the latest line of the log file is
+              not the whole line. But the whole line may be resent
+              in the next run.
+        :praam progress: the :class:`Progress` instance to update.
+        """
+        mat = self.regex_.search(line)
+        if not mat:
+            return (
+                self.unmatch_sameline_,
+                self.unmatch_nextline_)
+
+        try:
+            message = self.message_template_ % mat.groupdict()
+        except Exception as error:
+            logging.error('failed to get message %s %% %s in line matcher %s',
+                          self.message_template_, mat.groupdict(), self)
+            raise error
+
+        self.progress_.update(message, self.severity_, progress)
+        return (
+            self.match_sameline_,
+            self.match_nextline_)

compass/log_analyzor/progress_calculator.py

@@ -0,0 +1,469 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""module to provide updating installing process function.
+
+   .. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
+"""
+import logging
+
+from compass.log_analyzor.adapter_matcher import AdapterItemMatcher
+from compass.log_analyzor.adapter_matcher import AdapterMatcher
+from compass.log_analyzor.adapter_matcher import OSMatcher
+from compass.log_analyzor.adapter_matcher import PackageMatcher
+from compass.log_analyzor.file_matcher import FileMatcher
+from compass.log_analyzor.line_matcher import IncrementalProgress
+from compass.log_analyzor.line_matcher import LineMatcher
+
+
+# TODO(weidong): reconsider intialization method for the following.
+OS_INSTALLER_CONFIGURATIONS = {
+    'Ubuntu': AdapterItemMatcher(
+        file_matchers=[
+            FileMatcher(
+                filename='syslog',
+                min_progress=0.0,
+                max_progress=1.0,
+                line_matchers={
+                    'start': LineMatcher(
+                        pattern=r'.*',
+                        progress=.05,
+                        message_template='start installing',
+                        unmatch_nextline_next_matcher_name='start',
+                        match_nextline_next_matcher_name='ethdetect'
+                    ),
+                    'ethdetect': LineMatcher(
+                        pattern=r'Menu.*item.*\'ethdetect\'.*selected',
+                        progress=.1,
+                        message_template='ethdetect selected',
+                        unmatch_nextline_next_matcher_name='ethdetect',
+                        match_nextline_next_matcher_name='netcfg'
+                    ),
+                    'netcfg': LineMatcher(
+                        pattern=r'Menu.*item.*\'netcfg\'.*selected',
+                        progress=.12,
+                        message_template='netcfg selected',
+                        unmatch_nextline_next_matcher_name='netcfg',
+                        match_nextline_next_matcher_name='network-preseed'
+                    ),
+                    'network-preseed': LineMatcher(
+                        pattern=r'Menu.*item.*\'network-preseed\'.*selected',
+                        progress=.15,
+                        message_template='network-preseed selected',
+                        unmatch_nextline_next_matcher_name='network-preseed',
+                        match_nextline_next_matcher_name='localechooser'
+                    ),
+                    'localechoose': LineMatcher(
+                        pattern=r'Menu.*item.*\'localechooser\'.*selected',
+                        progress=.18,
+                        message_template='localechooser selected',
+                        unmatch_nextline_next_matcher_name='localechooser',
+                        match_nextline_next_matcher_name='download-installer'
+                    ),
+                    'download-installer': LineMatcher(
+                        pattern=(
+                            r'Menu.*item.*\'download-installer\'.*selected'
+                        ),
+                        progress=.2,
+                        message_template='download installer selected',
+                        unmatch_nextline_next_matcher_name=(
+                            'download-installer'),
+                        match_nextline_next_matcher_name='clock-setup'
+                    ),
+                    'clock-setup': LineMatcher(
+                        pattern=r'Menu.*item.*\'clock-setup\'.*selected',
+                        progress=.3,
+                        message_template='clock-setup selected',
+                        unmatch_nextline_next_matcher_name='clock-setup',
+                        match_nextline_next_matcher_name='disk-detect'
+                    ),
+                    'disk-detect': LineMatcher(
+                        pattern=r'Menu.*item.*\'disk-detect\'.*selected',
+                        progress=.32,
+                        message_template='disk-detect selected',
+                        unmatch_nextline_next_matcher_name='disk-detect',
+                        match_nextline_next_matcher_name='partman-base'
+                    ),
+                    'partman-base': LineMatcher(
+                        pattern=r'Menu.*item.*\'partman-base\'.*selected',
+                        progress=.35,
+                        message_template='partman-base selected',
+                        unmatch_nextline_next_matcher_name='partman-base',
+                        match_nextline_next_matcher_name='live-installer'
+                    ),
+                    'live-installer': LineMatcher(
+                        pattern=r'Menu.*item.*\'live-installer\'.*selected',
+                        progress=.45,
+                        message_template='live-installer selected',
+                        unmatch_nextline_next_matcher_name='live-installer',
+                        match_nextline_next_matcher_name='pkgsel'
+                    ),
+                    'pkgsel': LineMatcher(
+                        pattern=r'Menu.*item.*\'pkgsel\'.*selected',
+                        progress=.5,
+                        message_template='pkgsel selected',
+                        unmatch_nextline_next_matcher_name='pkgsel',
+                        match_nextline_next_matcher_name='grub-installer'
+                    ),
+                    'grub-installer': LineMatcher(
+                        pattern=r'Menu.*item.*\'grub-installer\'.*selected',
+                        progress=.9,
+                        message_template='grub-installer selected',
+                        unmatch_nextline_next_matcher_name='grub-installer',
+                        match_nextline_next_matcher_name='finish-install'
+                    ),
+                    'finish-install': LineMatcher(
+                        pattern=r'Menu.*item.*\'finish-install\'.*selected',
+                        progress=.95,
+                        message_template='finish-install selected',
+                        unmatch_nextline_next_matcher_name='finish-install',
+                        match_nextline_next_matcher_name='finish-install-done'
+                    ),
+                    'finish-install-done': LineMatcher(
+                        pattern=r'Running.*finish-install.d/.*save-logs',
+                        progress=1.0,
+                        message_template='finish-install is done',
+                        unmatch_nextline_next_matcher_name=(
+                            'finish-install-done'
+                        ),
+                        match_nextline_next_matcher_name='exit'
+                    ),
+                }
+            ),
+            FileMatcher(
+                filename='status',
+                min_progress=.2,
+                max_progress=.3,
+                line_matchers={
+                    'start': LineMatcher(
+                        pattern=r'Package: (?P<package>.*)',
+                        progress=IncrementalProgress(0.0, 0.99, 0.05),
+                        message_template='Installing udeb %(package)s',
+                        unmatch_nextline_next_matcher_name='start',
+                        match_nextline_next_matcher_name='start'
+                    )
+                }
+            ),
+            FileMatcher(
+                filename='initial-status',
+                min_progress=.5,
+                max_progress=.9,
+                line_matchers={
+                    'start': LineMatcher(
+                        pattern=r'Package: (?P<package>.*)',
+                        progress=IncrementalProgress(0.0, 0.99, 0.01),
+                        message_template='Installing deb %(package)s',
+                        unmatch_nextline_next_matcher_name='start',
+                        match_nextline_next_matcher_name='start'
+                    )
+                }
+            ),
+        ]
+    ),
+    'CentOS': AdapterItemMatcher(
+        file_matchers=[
+            FileMatcher(
+                filename='sys.log',
+                min_progress=0.0,
+                max_progress=0.1,
+                line_matchers={
+                    'start': LineMatcher(
+                        pattern=r'NOTICE (?P<message>.*)',
+                        progress=IncrementalProgress(.1, .9, .1),
+                        message_template='%(message)s',
+                        unmatch_nextline_next_matcher_name='start',
+                        match_nextline_next_matcher_name='exit'
+                    ),
+                }
+            ),
+            FileMatcher(
+                filename='anaconda.log',
+                min_progress=0.1,
+                max_progress=1.0,
+                line_matchers={
+                    'start': LineMatcher(
+                        pattern=r'setting.*up.*kickstart',
+                        progress=.1,
+                        message_template=(
+                            'Setting up kickstart configurations'),
+                        unmatch_nextline_next_matcher_name='start',
+                        match_nextline_next_matcher_name='STEP_STAGE2'
+                    ),
+                    'STEP_STAGE2': LineMatcher(
+                        pattern=r'starting.*STEP_STAGE2',
+                        progress=.15,
+                        message_template=(
+                            'Downloading installation '
+                            'images from server'),
+                        unmatch_nextline_next_matcher_name='STEP_STAGE2',
+                        match_nextline_next_matcher_name='start_anaconda'
+                    ),
+                    'start_anaconda': LineMatcher(
+                        pattern=r'Running.*anaconda.*script',
+                        progress=.2,
+                        unmatch_nextline_next_matcher_name=(
+                            'start_anaconda'),
+                        match_nextline_next_matcher_name=(
+                            'start_kickstart_pre')
+                    ),
+                    'start_kickstart_pre': LineMatcher(
+                        pattern=r'Running.*kickstart.*pre.*script',
+                        progress=.25,
+                        unmatch_nextline_next_matcher_name=(
+                            'start_kickstart_pre'),
+                        match_nextline_next_matcher_name=(
+                            'kickstart_pre_done')
+                    ),
+                    'kickstart_pre_done': LineMatcher(
+                        pattern=(
+                            r'All.*kickstart.*pre.*script.*have.*been.*run'),
+                        progress=.3,
+                        unmatch_nextline_next_matcher_name=(
+                            'kickstart_pre_done'),
+                        match_nextline_next_matcher_name=(
+                            'start_enablefilesystem')
+                    ),
+                    'start_enablefilesystem': LineMatcher(
+                        pattern=r'moving.*step.*enablefilesystems',
+                        progress=0.3,
+                        message_template=(
+                            'Performing hard-disk partitioning and '
+                            'enabling filesystems'),
+                        unmatch_nextline_next_matcher_name=(
+                            'start_enablefilesystem'),
+                        match_nextline_next_matcher_name=(
+                            'enablefilesystem_done')
+                    ),
+                    'enablefilesystem_done': LineMatcher(
+                        pattern=r'leaving.*step.*enablefilesystems',
+                        progress=.35,
+                        message_template='Filesystems are enabled',
+                        unmatch_nextline_next_matcher_name=(
+                            'enablefilesystem_done'),
+                        match_nextline_next_matcher_name=(
+                            'setup_repositories')
+                    ),
+                    'setup_repositories': LineMatcher(
+                        pattern=r'moving.*step.*reposetup',
+                        progress=0.35,
+                        message_template=(
+                            'Setting up Customized Repositories'),
+                        unmatch_nextline_next_matcher_name=(
+                            'setup_repositories'),
+                        match_nextline_next_matcher_name=(
+                            'repositories_ready')
+                    ),
+                    'repositories_ready': LineMatcher(
+                        pattern=r'leaving.*step.*reposetup',
+                        progress=0.4,
+                        message_template=(
+                            'Customized Repositories setting up are done'),
+                        unmatch_nextline_next_matcher_name=(
+                            'repositories_ready'),
+                        match_nextline_next_matcher_name='checking_dud'
+                    ),
+                    'checking_dud': LineMatcher(
+                        pattern=r'moving.*step.*postselection',
+                        progress=0.4,
+                        message_template='Checking DUD modules',
+                        unmatch_nextline_next_matcher_name='checking_dud',
+                        match_nextline_next_matcher_name='dud_checked'
+                    ),
+                    'dud_checked': LineMatcher(
+                        pattern=r'leaving.*step.*postselection',
+                        progress=0.5,
+                        message_template='Checking DUD modules are done',
+                        unmatch_nextline_next_matcher_name='dud_checked',
+                        match_nextline_next_matcher_name='installing_packages'
+                    ),
+                    'installing_packages': LineMatcher(
+                        pattern=r'moving.*step.*installpackages',
+                        progress=0.5,
+                        message_template='Installing packages',
+                        unmatch_nextline_next_matcher_name=(
+                            'installing_packages'),
+                        match_nextline_next_matcher_name=(
+                            'packages_installed')
+                    ),
+                    'packages_installed': LineMatcher(
+                        pattern=r'leaving.*step.*installpackages',
+                        progress=0.8,
+                        message_template='Packages are installed',
+                        unmatch_nextline_next_matcher_name=(
+                            'packages_installed'),
+                        match_nextline_next_matcher_name=(
+                            'installing_bootloader')
+                    ),
+                    'installing_bootloader': LineMatcher(
+                        pattern=r'moving.*step.*instbootloader',
+                        progress=0.9,
+                        message_template='Installing bootloaders',
+                        unmatch_nextline_next_matcher_name=(
+                            'installing_bootloader'),
+                        match_nextline_next_matcher_name=(
+                            'bootloader_installed'),
+                    ),
+                    'bootloader_installed': LineMatcher(
+                        pattern=r'leaving.*step.*instbootloader',
+                        progress=1.0,
+                        message_template='bootloaders is installed',
+                        unmatch_nextline_next_matcher_name=(
+                            'bootloader_installed'),
+                        match_nextline_next_matcher_name='exit'
+                    ),
+                }
+            ),
+            FileMatcher(
+                filename='install.log',
+                min_progress=0.56,
+                max_progress=0.80,
+                line_matchers={
+                    'start': LineMatcher(
+                        pattern=r'Installing (?P<package>.*)',
+                        progress=IncrementalProgress(0.0, 0.99, 0.005),
+                        message_template='Installing %(package)s',
+                        unmatch_sameline_next_matcher_name='package_complete',
+                        unmatch_nextline_next_matcher_name='start',
+                        match_nextline_next_matcher_name='start'
+                    ),
+                    'package_complete': LineMatcher(
+                        pattern='FINISHED.*INSTALLING.*PACKAGES',
+                        progress=1.0,
+                        message_template='installing packages finished',
+                        unmatch_nextline_next_matcher_name='start',
+                        match_nextline_next_matcher_name='exit'
+                    ),
+                }
+            ),
+        ]
+    ),
+}
+
+
+PACKAGE_INSTALLER_CONFIGURATIONS = {
+    'openstack': AdapterItemMatcher(
+        file_matchers=[
+            FileMatcher(
+                filename='chef-client.log',
+                min_progress=0.1,
+                max_progress=1.0,
+                line_matchers={
+                    'start': LineMatcher(
+                        pattern=(
+                            r'Processing\s*(?P<install_type>.*)'
+                            r'\[(?P<package>.*)\].*'),
+                        progress=IncrementalProgress(0.0, .90, 0.005),
+                        message_template=(
+                            'Processing %(install_type)s %(package)s'),
+                        unmatch_sameline_next_matcher_name=(
+                            'chef_complete'),
+                        unmatch_nextline_next_matcher_name='start',
+                        match_nextline_next_matcher_name='start'
+                    ),
+                    'chef_complete': LineMatcher(
+                        pattern=r'Chef.*Run.*complete',
+                        progress=1.0,
+                        message_template='Chef run complete',
+                        unmatch_nextline_next_matcher_name='start',
+                        match_nextline_next_matcher_name='exit'
+                    ),
+                }
+            ),
+        ]
+    ),
+}
+
+OS_ADAPTER_CONFIGURATIONS = [
+    OSMatcher(
+        os_installer_name='cobbler',
+        os_pattern='CentOS.*',
+        item_matcher=OS_INSTALLER_CONFIGURATIONS['CentOS'],
+        min_progress=0.0,
+        max_progress=1.0
+    ),
+    OSMatcher(
+        os_installer_name='cobbler',
+        os_pattern='Ubuntu.*',
+        item_matcher=OS_INSTALLER_CONFIGURATIONS['Ubuntu'],
+        min_progress=0.0,
+        max_progress=1.0
+    )
+]
+
+PACKAGE_ADAPTER_CONFIGURATIONS = [
+    PackageMatcher(
+        package_installer_name='chef',
+        target_system='openstack',
+        item_matcher=PACKAGE_INSTALLER_CONFIGURATIONS['openstack'],
+        min_progress=0.0,
+        max_progress=1.0
+    )
+]
+
+
+def _get_os_adapter_matcher(os_installer, os_name):
+    """Get OS adapter matcher by os name and installer name."""
+    for configuration in OS_ADAPTER_CONFIGURATIONS:
+        if configuration.match(os_installer, os_name):
+            return configuration
+        else:
+            logging.debug('configuration %s does not match %s and %s',
+                          configuration, os_name, os_installer)
+    logging.error('No configuration found for os installer %s os %s',
+                  os_installer, os_name)
+    return None
+
+
+def _get_package_adapter_matcher(package_installer, target_system):
+    """Get package adapter matcher by pacakge name and installer name."""
+    for configuration in PACKAGE_ADAPTER_CONFIGURATIONS:
+        if configuration.match(package_installer, target_system):
+            return configuration
+        else:
+            logging.debug('configuration %s does not match %s and %s',
+                          configuration, target_system, package_installer)
+    logging.error('No configuration found for os installer %s os %s',
+                  package_installer, target_system)
+    return None
+
+
+def update_progress(os_installer, os_names, package_installer, target_systems,
+                    cluster_hosts):
+    """Update adapter installing progress.
+
+    :param os_installer: os installer name
+    :param package_installer: package installer name.
+    :param cluster_hosts: clusters and hosts in each cluster to update.
+    :param cluster_hosts: dict of int to list of int.
+    """
+    for clusterid, hostids in cluster_hosts.items():
+        """
+        adapter = _get_adapter_matcher(os_installer, os_names[clusterid],
+                                       package_installer,
+                                       target_systems[clusterid])
+        if not adapter:
+            continue
+
+        adapter.update_progress(clusterid, hostids)
+        """
+        os_adapter = _get_os_adapter_matcher(os_installer, os_names[clusterid])
+        package_adapter = _get_package_adapter_matcher(
+            package_installer,
+            target_systems[clusterid]
+        )
+        if not (os_adapter or package_adapter):
+            continue
+
+        adapter = AdapterMatcher(os_adapter, package_adapter)
+        adapter.update_progress(clusterid, hostids)