openstack/storlets
Code Issues Proposed changes

moving syslog messages from container to host

Browse Source
This commit is contained in:
Doron Chen
2015-08-02 15:05:46 +03:00
parent 347a830350
commit c0809470d1
14 changed files with 55 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
Deploy/playbook/roles/docker_base_jre_image/files/rsyslog.conf
View File

@@ -1,60 +0,0 @@
# /etc/rsyslog.conf Configuration file for rsyslog.
#
# For more information see
# /usr/share/doc/rsyslog-doc/html/rsyslog_conf.html
#
# Default logging rules can be found in /etc/rsyslog.d/50-default.conf
#################
#### MODULES ####
#################
$ModLoad imuxsock # provides support for local system logging
$ModLoad imklog # provides kernel logging support
#$ModLoad immark # provides --MARK-- message capability
# provides UDP syslog reception
$ModLoad imudp
$UDPServerRun 514
# provides TCP syslog reception
#$ModLoad imtcp
#$InputTCPServerRun 514
# Enable non-kernel facility klog messages
$KLogPermitNonKernelFacility on
###########################
#### GLOBAL DIRECTIVES ####
###########################
#
# Use traditional timestamp format.
# To enable high precision timestamps, comment out the following line.
#
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
# Filter duplicated messages
$RepeatedMsgReduction on
#
# Set the default permissions for all log files.
#
$FileOwner syslog
$FileGroup adm
$FileCreateMode 0640
$DirCreateMode 0755
$Umask 0022
$PrivDropToUser syslog
$PrivDropToGroup syslog
#
# Where to place spool and state files
#
$WorkDirectory /var/spool/rsyslog
#
# Include all config files in /etc/rsyslog.d/
#
$IncludeConfig /etc/rsyslog.d/*.conf

1
Deploy/playbook/roles/docker_base_jre_image/tasks/ubuntu_14.04_jre8.yml
View File

@@ -67,7 +67,6 @@
dest: /data/registry/repositories/ubuntu_14.04_jre8
with_items:
- logback.xml
- rsyslog.conf
tags: ubuntu_14.04_jre8
- include: create_layer.yml dockerfile_prefix={{ base_os_image}}_jre8 layer_suffix=jre8 tags=jre8

4
Deploy/playbook/roles/docker_base_jre_image/templates/ubuntu_14.04_jre8_Dockerfile
View File

@@ -19,16 +19,12 @@ MAINTAINER {{ base_image_maintainer }}
RUN ["apt-get", "update"]
RUN ["apt-get", "install", "python", "-y"]
RUN ["apt-get", "install", "rsyslog", "-y"]
RUN ["apt-get", "install", "software-properties-common", "-y"]
RUN ["add-apt-repository", "ppa:webupd8team/java"]
RUN ["apt-get", "update"]
RUN echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | sudo debconf-set-selections
RUN ["apt-get", "install", "oracle-java8-installer", "-y"]
COPY rsyslog.conf /etc/rsyslog.conf
RUN ["chmod", "0644", "/etc/rsyslog.conf"]
COPY logback-classic-1.1.2.jar /opt/ibm/
RUN ["chmod", "0744", "/opt/ibm/logback-classic-1.1.2.jar"]

4
Deploy/playbook/roles/docker_storlet_engine_image/files/init_container.sh
View File

@@ -26,6 +26,6 @@
if [ $1 = "debug" ]; then
$2
else
service rsyslog start
python /usr/local/lib/python2.7/dist-packages/storlet_daemon_factory/daemon_factory.py $2 $3
#service rsyslog start
python /usr/local/lib/python2.7/dist-packages/storlet_daemon_factory/daemon_factory.py $2 $3 $HOSTNAME
fi

8
Engine/SBus/SBusJavaFacade/SBusJNI.c
View File

@@ -116,13 +116,17 @@ int init_java_accessors( JNIEnv* env )
JNIEXPORT void JNICALL
Java_com_ibm_storlet_sbus_SBusJNI_startLogger( JNIEnv* env,
jobject obj,
jstring jLevel )
jstring jLevel,
jstring jContId )
{
const char* pLogLevel = (*env)->GetStringUTFChars( env, jLevel, NULL );
if( NULL == pLogLevel )
return;
const char* pContId = (*env)->GetStringUTFChars( env, jContId, NULL );
if( NULL == pContId )
return;
sbus_start_logger( pLogLevel);
sbus_start_logger( pLogLevel, pContId);
(*env)->ReleaseStringUTFChars( env, jLevel, pLogLevel );
}

4
Engine/SBus/SBusJavaFacade/src/com/ibm/storlet/sbus/SBus.java
View File

@@ -39,10 +39,10 @@ public class SBus
*
* Instantiate the SBusBackend object. Start logging
* */
public SBus() throws IOException
public SBus( final String contId ) throws IOException
{
SBusBack_ = new SBusBackend();
SBusBack_.startLogger( eLogLevel.SBUS_LOG_DEBUG );
SBusBack_.startLogger( eLogLevel.SBUS_LOG_DEBUG, contId );
}
/*------------------------------------------------------------------------

4
Engine/SBus/SBusJavaFacade/src/com/ibm/storlet/sbus/SBusBackend.java
View File

@@ -53,7 +53,7 @@ public class SBusBackend
/*------------------------------------------------------------------------
* Initiate logging with the required detail level
* */
public void startLogger( eLogLevel eLogLevel )
public void startLogger( eLogLevel eLogLevel, String contId )
{
String strLogLevel = null;
switch( eLogLevel )
@@ -77,7 +77,7 @@ public class SBusBackend
strLogLevel = "WARNINIG";
break;
}
SBusJNIObj_.startLogger(strLogLevel);
SBusJNIObj_.startLogger(strLogLevel, contId);
}
/*------------------------------------------------------------------------

2
Engine/SBus/SBusJavaFacade/src/com/ibm/storlet/sbus/SBusJNI.java
View File

@@ -36,7 +36,7 @@ public class SBusJNI
System.loadLibrary("jsbus");
}
public native void startLogger( final String strLogLevel );
public native void startLogger( final String strLogLevel, final String contId );
public native void stopLogger();
public native int createSBus( final String strBusName );
public native int listenSBus( int nBus );

6
Engine/SBus/SBusPythonFacade/SBus.py
View File

@@ -75,7 +75,7 @@ class SBus(object):
'''--------------------------------------------------------------------'''
@staticmethod
def start_logger(str_log_level='DEBUG'):
def start_logger(str_log_level='DEBUG', container_id=None):
'''
@summary: Start logger.
@@ -88,8 +88,8 @@ class SBus(object):
# load the C-library
sbus_back_ = ctypes.CDLL(SBus.SBUS_SO_NAME)
sbus_back_.sbus_start_logger.argtypes = [c_char_p]
sbus_back_.sbus_start_logger(str_log_level)
sbus_back_.sbus_start_logger.argtypes = [c_char_p, c_char_p]
sbus_back_.sbus_start_logger(str_log_level, container_id)
'''--------------------------------------------------------------------'''
@staticmethod

11
Engine/SBus/SBusTransportLayer/sbus.c
View File

@@ -70,18 +70,25 @@ int sbus_translate_log_level( const char* str_log_level )
return level;
}
char str[80] = "CONT #";
/*----------------------------------------------------------------------------
* start_sbus_logger
*
* initiates sbus object' logger
*
*/
void sbus_start_logger( const char* str_log_level )
void sbus_start_logger( const char* str_log_level, const char* container_id )
{
int n_level = sbus_translate_log_level( str_log_level );
closelog();
openlog( SBUS_SYSLOG_PATH, LOG_PID, LOG_SYSLOG );
strcat(str, container_id);
strcat(str, ": ");
strcat(str, SBUS_SYSLOG_PATH);
openlog( str, LOG_PID, LOG_SYSLOG );
if( LOG_EMERG == n_level )
setlogmask( LOG_EMERG );
else

2
Engine/SBus/SBusTransportLayer/sbus.h
View File

@@ -29,7 +29,7 @@
/*----------------------------------------------------------------------------
* start/halt logging
*/
extern void sbus_start_logger( const char* str_log_level );
extern void sbus_start_logger( const char* str_log_level, const char* container_id);
extern void sbus_stop_logger( void );
/*----------------------------------------------------------------------------

3
Engine/SDaemon/src/com/ibm/storlet/daemon/SDaemon.java
View File

@@ -137,6 +137,7 @@ public class SDaemon
String strSBusPath = args[1];
String strLogLevel = args[2];
int nPoolSize = Integer.parseInt(args[3]);
String strContId = args[4];
if( initLog( strStorletName_, strLogLevel ) == false )
return;
@@ -147,7 +148,7 @@ public class SDaemon
storletTaskFactory_ = new STaskFactory( storlet, logger_ );
logger_.trace("Instanciating SBus");
sbus_ = new SBus();
sbus_ = new SBus(strContId);
try
{
logger_.trace("Initialising SBus");

4
Engine/SMScripts/restart_docker_container.c
View File

@@ -23,7 +23,7 @@
/*
* Stop and Run a docker container using:
* docker stop <container name>
* docker run --name <container name> -d -v <mount dir 1> -v <mount dir 2> -i -t <image> --net='none'
* docker run --name <container name> -d -v /dev/log:/dev/log -v <mount dir 1> -v <mount dir 2> -i -t <image> --net='none'
* <container name> - The name of the container to stop / to start
* <image name> - the name of the image from which to start the container
* <mount dir 1> - The directory where the named pipes are placed. Typically mounted to /mnt/channels in the container
@@ -52,7 +52,7 @@ int main(int argc, char **argv) {
ret = system(command);
sprintf(command,
"/usr/bin/docker run --net=none --name %s -d -v %s -v %s -i -t %s",
"/usr/bin/docker run --net=none --name %s -d -v /dev/log:/dev/log -v %s -v %s -i -t %s",
container_name,
mount_dir1,
mount_dir2,

40
Engine/storlet_daemon_factory/daemon_factory.py
View File

@@ -75,7 +75,8 @@ class daemon_factory():
storlet_name,
pool_size,
uds_path,
log_level):
log_level,
container_id):
'''
@summary: get_jvm_args
Check the input parameters, produce the list
@@ -95,6 +96,8 @@ class daemon_factory():
@type uds_path: String
@param log_level: Logger verbosity level
@type log_level: String
@param container_id: container id
@type container_id: String
@return: Error code, 0 if successful
@rtype: Integer
@@ -142,7 +145,8 @@ class daemon_factory():
str(storlet_name),
str(uds_path),
str(log_level),
str('%d' % pool_size)]
str('%d' % pool_size),
str(container_id)]
str_pargs = ' '.join(map(str, pargs))
self.logger.debug('START_DAEMON: pargs = %s' % str_pargs)
else:
@@ -245,7 +249,8 @@ class daemon_factory():
storlet_name,
pool_size,
uds_path,
log_level):
log_level,
container_id):
'''
@summary: process_start_daemon
Start storlet daemon process
@@ -270,7 +275,8 @@ class daemon_factory():
storlet_name,
pool_size,
uds_path,
log_level)
log_level,
container_id)
if 0 != n_error_id:
self.logger.debug('Problems with arguments for {0}'.
format(storlet_name))
@@ -468,13 +474,15 @@ class daemon_factory():
return b_status
'''--------------------------------------------------------------------'''
def dispatch_command(self, dtg):
def dispatch_command(self, dtg, container_id):
'''
@summary: dispatch_command
Parse datagram. React on the request.
@param dtg: Datagram to process
@type dtg: SBus python facade Datagram
@param container_id: container id
@type container_id: String
@return: Status
@rtype: Boolean
@@ -507,7 +515,8 @@ class daemon_factory():
prms['storlet_name'],
prms['pool_size'],
prms['uds_path'],
prms['log_level'])
prms['log_level'],
container_id)
elif command == SBUS_CMD_STOP_DAEMON:
self.logger.debug( 'Do SBUS_CMD_STOP_DAEMON' )
b_status, error_text = self.process_kill(\
@@ -537,7 +546,7 @@ class daemon_factory():
return b_status, error_text, b_iterate
'''--------------------------------------------------------------------'''
def main_loop(self):
def main_loop(self, container_id):
'''
@summary: main_loop
The 'internal' loop. Listen to SBus, receive datagram,
@@ -575,7 +584,7 @@ class daemon_factory():
else:
self.logger.debug("Received outfd %d" % outfd.fileno())
b_status, error_text, b_iterate = self.dispatch_command(dtg)
b_status, error_text, b_iterate = self.dispatch_command(dtg, container_id)
self.log_and_report(outfd, b_status, error_text)
outfd.close()
@@ -612,7 +621,7 @@ class daemon_factory():
'''======================= END OF daemon_factory CLASS ===================='''
'''------------------------------------------------------------------------'''
def start_logger(logger_name, log_level):
def start_logger(logger_name, log_level, container_id):
'''
@summary: start_logger
Initialize logging of this process.
@@ -640,7 +649,7 @@ def start_logger(logger_name, log_level):
level = logging.ERROR
logger = logging.getLogger(logger_name)
logger = logging.getLogger("CONT #" + container_id + ": " + logger_name)
if log_level == 'OFF':
logging.disable(logging.CRITICAL)
@@ -674,7 +683,7 @@ def usage():
@rtype: void
'''
print "daemon_factory <path> <log level>"
print "daemon_factory <path> <log level> <container_id>"
'''------------------------------------------------------------------------'''
def main(argv):
@@ -686,15 +695,16 @@ def main(argv):
- create an instance of daemon_factory,
- start the main loop.
'''
if (len(argv) != 2):
if (len(argv) != 3):
usage()
return
pipe_path = argv[0]
log_level = argv[1]
logger = start_logger("daemon_factory", log_level)
container_id = argv[2]
logger = start_logger("daemon_factory", log_level, container_id)
logger.debug("Daemon factory started")
SBus.start_logger("DEBUG")
SBus.start_logger("DEBUG", container_id=container_id)
# Impersonate the swift user
pw = pwd.getpwnam('swift')
@@ -703,7 +713,7 @@ def main(argv):
factory = daemon_factory(pipe_path, logger)
factory.main_loop()
factory.main_loop(container_id)
'''------------------------------------------------------------------------'''
if __name__ == "__main__":