Merge pull request #24081 from Microsoft/jjh/signaltofile

Windows: Stack dump to file

daemon/daemon.go

@@ -412,7 +412,7 @@ func NewDaemon(config *Config, registryService registry.Service, containerdRemot
 
 	// set up SIGUSR1 handler on Unix-like systems, or a Win32 global event
 	// on Windows to dump Go routine stacks
-	setupDumpStackTrap()
+	setupDumpStackTrap(config.Root)
 
 	uidMaps, gidMaps, err := setupRemappedRoot(config)
 	if err != nil {

daemon/debugtrap_unix.go

@@ -10,12 +10,12 @@ import (
 	psignal "github.com/docker/docker/pkg/signal"
 )
 
-func setupDumpStackTrap() {
+func setupDumpStackTrap(_ string) {
 	c := make(chan os.Signal, 1)
 	signal.Notify(c, syscall.SIGUSR1)
 	go func() {
 		for range c {
-			psignal.DumpStacks()
+			psignal.DumpStacks("")
 		}
 	}()
 }

daemon/debugtrap_unsupported.go

@@ -2,6 +2,6 @@
 
 package daemon
 
-func setupDumpStackTrap() {
+func setupDumpStackTrap(_ string) {
 	return
 }

daemon/debugtrap_windows.go

@@ -10,7 +10,7 @@ import (
 	"github.com/docker/docker/pkg/system"
 )
 
-func setupDumpStackTrap() {
+func setupDumpStackTrap(root string) {
 	// Windows does not support signals like *nix systems. So instead of
 	// trapping on SIGUSR1 to dump stacks, we wait on a Win32 event to be
 	// signaled.
@@ -23,7 +23,7 @@ func setupDumpStackTrap() {
 			logrus.Debugf("Stackdump - waiting signal at %s", ev)
 			for {
 				syscall.WaitForSingleObject(h, syscall.INFINITE)
-				signal.DumpStacks()
+				signal.DumpStacks(root)
 			}
 		}
 	}()

pkg/signal/trap.go

@@ -3,9 +3,11 @@ package signal
 import (
 	"os"
 	gosignal "os/signal"
+	"path/filepath"
 	"runtime"
 	"sync/atomic"
 	"syscall"
+	"time"
 
 	"github.com/Sirupsen/logrus"
 )
@@ -52,7 +54,7 @@ func Trap(cleanup func()) {
 						logrus.Info("Forcing docker daemon shutdown without cleanup; 3 interrupts received")
 					}
 				case syscall.SIGQUIT:
-					DumpStacks()
+					DumpStacks("")
 					logrus.Info("Forcing docker daemon shutdown without cleanup on SIGQUIT")
 				}
 				//for the SIGINT/TERM, and SIGQUIT non-clean shutdown case, exit with 128 + signal #
@@ -63,7 +65,7 @@ func Trap(cleanup func()) {
 }
 
 // DumpStacks dumps the runtime stack.
-func DumpStacks() {
+func DumpStacks(root string) {
 	var (
 		buf       []byte
 		stackSize int
@@ -77,5 +79,30 @@ func DumpStacks() {
 	buf = buf[:stackSize]
 	// Note that if the daemon is started with a less-verbose log-level than "info" (the default), the goroutine
 	// traces won't show up in the log.
+	if root == "" {
 		logrus.Infof("=== BEGIN goroutine stack dump ===\n%s\n=== END goroutine stack dump ===", buf)
+	} else {
+		// Dumps the stacks to a file in the root directory of the daemon
+		// On Windows, this overcomes two issues - one being that if the stack is too big, it doesn't
+		// get written to the event log when the Windows daemon is running as a service.
+		// Second, using logrus, the tabs and new-lines end up getting written as literal
+		// \t and \n's, meaning you need to use something like notepad++ to convert the
+		// output into something readable using 'type' from a command line or notepad/notepad++ etc.
+		path := filepath.Join(root, "goroutine-stacks.log")
+		f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
+		if err != nil {
+			logrus.Warnf("Could not open %s to write the goroutine stacks: %v", path, err)
+			return
+		}
+		defer f.Close()
+		f.WriteString("=== BEGIN goroutine stack dump ===\n")
+		f.WriteString(time.Now().String() + "\n")
+		if _, err := f.Write(buf); err != nil {
+			logrus.Warnf("Could not write goroutine stacks to %s: %v", path, err)
+			return
+		}
+		f.WriteString("=== END goroutine stack dump ===\n")
+		f.Sync()
+		logrus.Infof("goroutine stacks written to %s", path)
+	}
 }