Print output from spark-daemon only when it fails to launch

2013-08-31 17:31:07 -07:00 · 2013-08-31 17:31:07 -07:00 · 2ee6a7e32a
--- a/bin/slaves.sh
+++ b/bin/slaves.sh
@ -42,7 +42,7 @@ bin=`cd "$bin"; pwd`
 . "$bin/spark-config.sh"

 # If the slaves file is specified in the command line,
-# then it takes precedence over the definition in 
+# then it takes precedence over the definition in
 # spark-env.sh. Save it here.
 HOSTLIST=$SPARK_SLAVES

@ -58,8 +58,6 @@ if [ "$HOSTLIST" = "" ]; then
  fi
 fi

-echo $"${@// /\\ }"
-
 # By default disable strict host key checking
 if [ "$SPARK_SSH_OPTS" = "" ]; then
  SPARK_SSH_OPTS="-o StrictHostKeyChecking=no"
--- a/bin/spark-daemon.sh
+++ b/bin/spark-daemon.sh
@ -75,6 +75,9 @@ if [ "$SPARK_IDENT_STRING" = "" ]; then
  export SPARK_IDENT_STRING="$USER"
 fi

+
+export SPARK_PRINT_LAUNCH_COMMAND="1"
+
 # get log directory
 if [ "$SPARK_LOG_DIR" = "" ]; then
  export SPARK_LOG_DIR="$SPARK_HOME/logs"
@ -122,12 +125,19 @@ case $startStop in
      rsync -a -e ssh --delete --exclude=.svn --exclude='logs/*' --exclude='contrib/hod/logs/*' $SPARK_MASTER/ "$SPARK_HOME"
    fi

-    spark_rotate_log $log
+    spark_rotate_log "$log"
    echo starting $command, logging to $log
    cd "$SPARK_PREFIX"
    nohup nice -n $SPARK_NICENESS "$SPARK_PREFIX"/spark-class $command "$@" >> "$log" 2>&1 < /dev/null &
-    echo $! > $pid
-    sleep 1; head "$log"
+    newpid=$!
+    echo $newpid > $pid
+    sleep 2
+    # Check if the process has died; in that case we'll tail the log so the user can see
+    if ! kill -0 $newpid >/dev/null 2>&1; then
+      echo "failed to launch $command:"
+      tail -2 "$log" | sed 's/^/  /'
+      echo "full log in $log"
+    fi
    ;;

  (stop)
--- a/bin/start-slaves.sh
+++ b/bin/start-slaves.sh
@ -35,8 +35,6 @@ if [ "$SPARK_MASTER_IP" = "" ]; then
  SPARK_MASTER_IP=`hostname`
 fi

-echo "Master IP: $SPARK_MASTER_IP"
-
 # Launch the slaves
 if [ "$SPARK_WORKER_INSTANCES" = "" ]; then
  exec "$bin/slaves.sh" cd "$SPARK_HOME" \; "$bin/start-slave.sh" 1 spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT
--- a/bin/stop-all.sh
+++ b/bin/stop-all.sh
@ -20,6 +20,7 @@
 # Start all spark daemons.
 # Run this on the master nde

+
 bin=`dirname "$0"`
 bin=`cd "$bin"; pwd`

--- a/core/src/main/scala/spark/deploy/worker/ui/WorkerWebUI.scala
+++ b/core/src/main/scala/spark/deploy/worker/ui/WorkerWebUI.scala
@ -143,7 +143,6 @@ class WorkerWebUI(val worker: Worker, val workDir: File, requestedPort: Option[I
      <html>
        <body>
          {linkToMaster}
-          <hr />
          <div>
            <div style="float:left;width:40%">{backButton}</div>
            <div style="float:left;">{range}</div>