Add run_process() to start services without screen

* USE_SCREEN defaults to True, set it to False to exec the services
  directly via bash.  SCREEN_DEV is still supported until the CI
  scripts get updated.
* The extra logging file descriptors are properly closed in the child process
  and stdout/stderr are redirected to the log files.
* The screen_rc() call is still present; this means that stack-screenrc will
  have a complete record of what was started and rejoin-stack.sh may be able
  to re-create the setup under screen.
* The python interpreter was unwilling to write to the log files without
  unbufering stdout by using PYTHONUNBUFFERED.  This feels hackish and should
  be investigated further.

Change-Id: I012ed049f2c8b185a2e6929d73edc29e167bc21f

functions

@@ -735,26 +735,69 @@ function restart_service() {
 }
 
 
+# _run_process() is designed to be backgrounded by run_process() to simulate a
+# fork.  It includes the dirty work of closing extra filehandles and preparing log
+# files to produce the same logs as screen_it().  The log filename is derived
+# from the service name and global-and-now-misnamed SCREEN_LOGDIR
+# _run_process service "command-line"
+function _run_process() {
+    local service=$1
+    local command="$2"
+
+    # Undo logging redirections and close the extra descriptors
+    exec 1>&3
+    exec 2>&3
+    exec 3>&-
+    exec 6>&-
+
+    if [[ -n ${SCREEN_LOGDIR} ]]; then
+        exec 1>&${SCREEN_LOGDIR}/screen-${1}.${CURRENT_LOG_TIME}.log 2>&1
+        ln -sf ${SCREEN_LOGDIR}/screen-${1}.${CURRENT_LOG_TIME}.log ${SCREEN_LOGDIR}/screen-${1}.log
+
+        # TODO(dtroyer): Hack to get stdout from the Python interpreter for the logs.
+        export PYTHONUNBUFFERED=1
+    fi
+
+    exec /bin/bash -c "$command"
+    die "$service exec failure: $command"
+}
+
+
+# run_process() launches a child process that closes all file descriptors and
+# then exec's the passed in command.  This is meant to duplicate the semantics
+# of screen_it() without screen.  PIDs are written to
+# $SERVICE_DIR/$SCREEN_NAME/$service.pid
+# run_process service "command-line"
+function run_process() {
+    local service=$1
+    local command="$2"
+
+    # Spawn the child process
+    _run_process "$service" "$command" &
+    echo $!
+}
+
+
 # Helper to launch a service in a named screen
 # screen_it service "command-line"
 function screen_it {
     SCREEN_NAME=${SCREEN_NAME:-stack}
     SERVICE_DIR=${SERVICE_DIR:-${DEST}/status}
-    SCREEN_DEV=`trueorfalse True $SCREEN_DEV`
+    USE_SCREEN=$(trueorfalse True $USE_SCREEN)
 
     if is_service_enabled $1; then
         # Append the service to the screen rc file
         screen_rc "$1" "$2"
 
-        screen -S $SCREEN_NAME -X screen -t $1
+        if [[ "$USE_SCREEN" = "True" ]]; then
+            screen -S $SCREEN_NAME -X screen -t $1
 
-        if [[ -n ${SCREEN_LOGDIR} ]]; then
-            screen -S $SCREEN_NAME -p $1 -X logfile ${SCREEN_LOGDIR}/screen-${1}.${CURRENT_LOG_TIME}.log
-            screen -S $SCREEN_NAME -p $1 -X log on
-            ln -sf ${SCREEN_LOGDIR}/screen-${1}.${CURRENT_LOG_TIME}.log ${SCREEN_LOGDIR}/screen-${1}.log
-        fi
+            if [[ -n ${SCREEN_LOGDIR} ]]; then
+                screen -S $SCREEN_NAME -p $1 -X logfile ${SCREEN_LOGDIR}/screen-${1}.${CURRENT_LOG_TIME}.log
+                screen -S $SCREEN_NAME -p $1 -X log on
+                ln -sf ${SCREEN_LOGDIR}/screen-${1}.${CURRENT_LOG_TIME}.log ${SCREEN_LOGDIR}/screen-${1}.log
+            fi
 
-        if [[ "$SCREEN_DEV" = "True" ]]; then
             # sleep to allow bash to be ready to be send the command - we are
             # creating a new window in screen and then sends characters, so if
             # bash isn't running by the time we send the command, nothing happens
@@ -763,7 +806,8 @@ function screen_it {
             NL=`echo -ne '\015'`
             screen -S $SCREEN_NAME -p $1 -X stuff "$2 || touch \"$SERVICE_DIR/$SCREEN_NAME/$1.failure\"$NL"
         else
-            screen -S $SCREEN_NAME -p $1 -X exec /bin/bash -c "$2 || touch \"$SERVICE_DIR/$SCREEN_NAME/$1.failure\""
+            # Spawn directly without screen
+            run_process "$1" "$2" >$SERVICE_DIR/$SCREEN_NAME/$service.pid
         fi
     fi
 }

stack.sh

@@ -824,8 +824,17 @@ fi
 # Configure screen
 # ----------------
 
-if [ -z "$SCREEN_HARDSTATUS" ]; then
-    SCREEN_HARDSTATUS='%{= .} %-Lw%{= .}%> %n%f %t*%{= .}%+Lw%< %-=%{g}(%{d}%H/%l%{g})'
+USE_SCREEN=$(trueorfalse True $USE_SCREEN)
+if [[ "$USE_SCREEN" == "True" ]]; then
+    # Create a new named screen to run processes in
+    screen -d -m -S $SCREEN_NAME -t shell -s /bin/bash
+    sleep 1
+
+    # Set a reasonable status bar
+    if [ -z "$SCREEN_HARDSTATUS" ]; then
+        SCREEN_HARDSTATUS='%{= .} %-Lw%{= .}%> %n%f %t*%{= .}%+Lw%< %-=%{g}(%{d}%H/%l%{g})'
+    fi
+    screen -r $SCREEN_NAME -X hardstatus alwayslastline "$SCREEN_HARDSTATUS"
 fi
 
 # Clear screen rc file
@@ -834,12 +843,6 @@ if [[ -e $SCREENRC ]]; then
     echo -n > $SCREENRC
 fi
 
-# Create a new named screen to run processes in
-screen -d -m -S $SCREEN_NAME -t shell -s /bin/bash
-sleep 1
-
-# Set a reasonable status bar
-screen -r $SCREEN_NAME -X hardstatus alwayslastline "$SCREEN_HARDSTATUS"
 
 # Initialize the directory for service status check
 init_service_check

stackrc

@@ -30,8 +30,8 @@ NOVA_ENABLED_APIS=ec2,osapi_compute,metadata
 # stuffing text into the screen windows so that a developer can use
 # ctrl-c, up-arrow, enter to restart the service. Starting services
 # this way is slightly unreliable, and a bit slower, so this can
-# be disabled for automated testing by setting this value to false.
-SCREEN_DEV=True
+# be disabled for automated testing by setting this value to False.
+USE_SCREEN=True
 
 # Repositories
 # ------------
@@ -198,3 +198,6 @@ VOLUME_BACKING_FILE_SIZE=${VOLUME_BACKING_FILE_SIZE:-5130M}
 
 PRIVATE_NETWORK_NAME=${PRIVATE_NETWORK_NAME:-"private"}
 PUBLIC_NETWORK_NAME=${PUBLIC_NETWORK_NAME:-"nova"}
+
+# Compatibility until it's eradicated from CI
+USE_SCREEN=${SCREEN_DEV:-$USE_SCREEN}