Merge branch 'master' into 0.6.5-dm-plugin

CHANGELOG.md

@@ -1,5 +1,43 @@
 # Changelog
 
+## 0.6.7 (2013-11-21)
+
+#### Runtime
+
+* Improved stability, fixes some race conditons
+* Skip the volumes mounted when deleting the volumes of container.
+* Fix layer size computation: handle hard links correctly
+* Use the work Path for docker cp CONTAINER:PATH
+* Fix tmp dir never cleanup
+* Speedup docker ps
+* More informative error message on name collisions
+* Fix nameserver regex
+* Always return long id's
+* Fix container restart race condition
+* Keep published ports on docker stop;docker start
+* Fix container networking on Fedora
+* Correctly express "any address" to iptables
+* Fix network setup when reconnecting to ghost container
+* Prevent deletion if image is used by a running container
+* Lock around read operations in graph
+
+#### RemoteAPI
+
+* Return full ID on docker rmi
+
+#### Client
+
++ Add -tree option to images
++ Offline image transfer
+* Exit with status 2 on usage error and display usage on stderr
+* Do not forward SIGCHLD to container
+* Use string timestamp for docker events -since
+
+#### Other
+
+* Update to go 1.2rc5
++ Add /etc/default/docker support to upstart
+
 ## 0.6.6 (2013-11-06)
 
 #### Runtime
@@ -17,6 +55,7 @@
 + Prevent DNS server conflicts in CreateBridgeIface
 + Validate bind mounts on the server side
 + Use parent image config in docker build
+* Fix regression in /etc/hosts
 
 #### Client
 

VERSION

@@ -1 +1 @@
-0.6.6-dev
+0.6.7-dev

commands.go

@@ -1736,60 +1736,60 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig,
 }
 
 func parseRun(cmd *flag.FlagSet, args []string, capabilities *Capabilities) (*Config, *HostConfig, *flag.FlagSet, error) {
+	var (
+		// FIXME: use utils.ListOpts for attach and volumes?
+		flAttach  = NewAttachOpts()
+		flVolumes = NewPathOpts()
+
+		flPublish     utils.ListOpts
+		flExpose      utils.ListOpts
+		flEnv         utils.ListOpts
+		flDns         utils.ListOpts
+		flVolumesFrom utils.ListOpts
+		flLxcOpts     utils.ListOpts
+		flLinks       utils.ListOpts
+
+		flAutoRemove      = cmd.Bool("rm", false, "Automatically remove the container when it exits (incompatible with -d)")
+		flDetach          = cmd.Bool("d", false, "Detached mode: Run container in the background, print new container id")
+		flNetwork         = cmd.Bool("n", true, "Enable networking for this container")
+		flPrivileged      = cmd.Bool("privileged", false, "Give extended privileges to this container")
+		flPublishAll      = cmd.Bool("P", false, "Publish all exposed ports to the host interfaces")
+		flStdin           = cmd.Bool("i", false, "Keep stdin open even if not attached")
+		flTty             = cmd.Bool("t", false, "Allocate a pseudo-tty")
+		flContainerIDFile = cmd.String("cidfile", "", "Write the container ID to the file")
+		flEntrypoint      = cmd.String("entrypoint", "", "Overwrite the default entrypoint of the image")
+		flHostname        = cmd.String("h", "", "Container host name")
+		flMemoryString    = cmd.String("m", "", "Memory limit (format: <number><optional unit>, where unit = b, k, m or g)")
+		flUser            = cmd.String("u", "", "Username or UID")
+		flWorkingDir      = cmd.String("w", "", "Working directory inside the container")
+		flCpuShares       = cmd.Int64("c", 0, "CPU shares (relative weight)")
+
+		// For documentation purpose
+		_ = cmd.Bool("sig-proxy", true, "Proxify all received signal to the process (even in non-tty mode)")
+		_ = cmd.String("name", "", "Assign a name to the container")
+	)
 
-	flHostname := cmd.String("h", "", "Container host name")
-	flWorkingDir := cmd.String("w", "", "Working directory inside the container")
-	flUser := cmd.String("u", "", "Username or UID")
-	flDetach := cmd.Bool("d", false, "Detached mode: Run container in the background, print new container id")
-	flAttach := NewAttachOpts()
 	cmd.Var(flAttach, "a", "Attach to stdin, stdout or stderr.")
-	flStdin := cmd.Bool("i", false, "Keep stdin open even if not attached")
-	flTty := cmd.Bool("t", false, "Allocate a pseudo-tty")
-	flMemoryString := cmd.String("m", "", "Memory limit (format: <number><optional unit>, where unit = b, k, m or g)")
-	flContainerIDFile := cmd.String("cidfile", "", "Write the container ID to the file")
-	flNetwork := cmd.Bool("n", true, "Enable networking for this container")
-	flPrivileged := cmd.Bool("privileged", false, "Give extended privileges to this container")
-	flAutoRemove := cmd.Bool("rm", false, "Automatically remove the container when it exits (incompatible with -d)")
-	cmd.Bool("sig-proxy", true, "Proxify all received signal to the process (even in non-tty mode)")
-	cmd.String("name", "", "Assign a name to the container")
-	flPublishAll := cmd.Bool("P", false, "Publish all exposed ports to the host interfaces")
-
-	if capabilities != nil && *flMemoryString != "" && !capabilities.MemoryLimit {
-		//fmt.Fprintf(stdout, "WARNING: Your kernel does not support memory limit capabilities. Limitation discarded.\n")
-		*flMemoryString = ""
-	}
-
-	flCpuShares := cmd.Int64("c", 0, "CPU shares (relative weight)")
-
-	var flPublish utils.ListOpts
-	cmd.Var(&flPublish, "p", "Publish a container's port to the host (use 'docker port' to see the actual mapping)")
-
-	var flExpose utils.ListOpts
-	cmd.Var(&flExpose, "expose", "Expose a port from the container without publishing it to your host")
-
-	var flEnv utils.ListOpts
-	cmd.Var(&flEnv, "e", "Set environment variables")
-
-	var flDns utils.ListOpts
-	cmd.Var(&flDns, "dns", "Set custom dns servers")
-
-	flVolumes := NewPathOpts()
 	cmd.Var(flVolumes, "v", "Bind mount a volume (e.g. from the host: -v /host:/container, from docker: -v /container)")
 
-	var flVolumesFrom utils.ListOpts
+	cmd.Var(&flPublish, "p", fmt.Sprintf("Publish a container's port to the host (format: %s) (use 'docker port' to see the actual mapping)", PortSpecTemplateFormat))
+	cmd.Var(&flExpose, "expose", "Expose a port from the container without publishing it to your host")
+	cmd.Var(&flEnv, "e", "Set environment variables")
+	cmd.Var(&flDns, "dns", "Set custom dns servers")
 	cmd.Var(&flVolumesFrom, "volumes-from", "Mount volumes from the specified container(s)")
-
-	flEntrypoint := cmd.String("entrypoint", "", "Overwrite the default entrypoint of the image")
-
-	var flLxcOpts utils.ListOpts
 	cmd.Var(&flLxcOpts, "lxc-conf", "Add custom lxc options -lxc-conf=\"lxc.cgroup.cpuset.cpus = 0,1\"")
-
-	var flLinks utils.ListOpts
 	cmd.Var(&flLinks, "link", "Add link to another container (name:alias)")
 
 	if err := cmd.Parse(args); err != nil {
 		return nil, nil, cmd, err
 	}
+
+	// Check if the kernel supports memory limit cgroup.
+	if capabilities != nil && *flMemoryString != "" && !capabilities.MemoryLimit {
+		*flMemoryString = ""
+	}
+
+	// Validate input params
 	if *flDetach && len(flAttach) > 0 {
 		return nil, nil, cmd, ErrConflictAttachDetach
 	}
@@ -1811,8 +1811,7 @@ func parseRun(cmd *flag.FlagSet, args []string, capabilities *Capabilities) (*Co
 		}
 	}
 
-	envs := []string{}
-
+	var envs []string
 	for _, env := range flEnv {
 		arr := strings.Split(env, "=")
 		if len(arr) > 1 {
@@ -1824,19 +1823,15 @@ func parseRun(cmd *flag.FlagSet, args []string, capabilities *Capabilities) (*Co
 	}
 
 	var flMemory int64
-
 	if *flMemoryString != "" {
 		parsedMemory, err := utils.RAMInBytes(*flMemoryString)
-
 		if err != nil {
 			return nil, nil, cmd, err
 		}
-
 		flMemory = parsedMemory
 	}
 
 	var binds []string
-
 	// add any bind targets to the list of container volumes
 	for bind := range flVolumes {
 		arr := strings.Split(bind, ":")
@@ -1851,10 +1846,12 @@ func parseRun(cmd *flag.FlagSet, args []string, capabilities *Capabilities) (*Co
 		}
 	}
 
-	parsedArgs := cmd.Args()
-	runCmd := []string{}
-	entrypoint := []string{}
-	image := ""
+	var (
+		parsedArgs = cmd.Args()
+		runCmd     []string
+		entrypoint []string
+		image      string
+	)
 	if len(parsedArgs) >= 1 {
 		image = cmd.Arg(0)
 	}
@@ -1865,16 +1862,16 @@ func parseRun(cmd *flag.FlagSet, args []string, capabilities *Capabilities) (*Co
 		entrypoint = []string{*flEntrypoint}
 	}
 
-	var lxcConf []KeyValuePair
 	lxcConf, err := parseLxcConfOpts(flLxcOpts)
 	if err != nil {
 		return nil, nil, cmd, err
 	}
 
-	hostname := *flHostname
-	domainname := ""
-
-	parts := strings.SplitN(hostname, ".", 2)
+	var (
+		domainname string
+		hostname   = *flHostname
+		parts      = strings.SplitN(hostname, ".", 2)
+	)
 	if len(parts) > 1 {
 		hostname = parts[0]
 		domainname = parts[1]
@@ -1952,30 +1949,33 @@ func (cli *DockerCli) CmdRun(args ...string) error {
 		return nil
 	}
 
-	flRm := cmd.Lookup("rm")
-	autoRemove, _ := strconv.ParseBool(flRm.Value.String())
+	// Retrieve relevant client-side config
+	var (
+		flName        = cmd.Lookup("name")
+		flRm          = cmd.Lookup("rm")
+		flSigProxy    = cmd.Lookup("sig-proxy")
+		autoRemove, _ = strconv.ParseBool(flRm.Value.String())
+		sigProxy, _   = strconv.ParseBool(flSigProxy.Value.String())
+	)
 
-	flSigProxy := cmd.Lookup("sig-proxy")
-	sigProxy, _ := strconv.ParseBool(flSigProxy.Value.String())
-	flName := cmd.Lookup("name")
+	// Disable sigProxy in case on TTY
 	if config.Tty {
 		sigProxy = false
 	}
 
-	var containerIDFile *os.File
+	var containerIDFile io.WriteCloser
 	if len(hostConfig.ContainerIDFile) > 0 {
-		if _, err := ioutil.ReadFile(hostConfig.ContainerIDFile); err == nil {
+		if _, err := os.Stat(hostConfig.ContainerIDFile); err == nil {
 			return fmt.Errorf("cid file found, make sure the other container isn't running or delete %s", hostConfig.ContainerIDFile)
 		}
-		containerIDFile, err = os.Create(hostConfig.ContainerIDFile)
-		if err != nil {
+		if containerIDFile, err = os.Create(hostConfig.ContainerIDFile); err != nil {
 			return fmt.Errorf("failed to create the container ID file: %s", err)
 		}
 		defer containerIDFile.Close()
 	}
+
 	containerValues := url.Values{}
-	name := flName.Value.String()
-	if name != "" {
+	if name := flName.Value.String(); name != "" {
 		containerValues.Set("name", name)
 	}
 
@@ -1996,8 +1996,7 @@ func (cli *DockerCli) CmdRun(args ...string) error {
 		v.Set("tag", tag)
 
 		// Resolve the Repository name from fqn to endpoint + name
-		var endpoint string
-		endpoint, _, err = registry.ResolveRepositoryName(repos)
+		endpoint, _, err := registry.ResolveRepositoryName(repos)
 		if err != nil {
 			return err
 		}
@@ -2015,14 +2014,10 @@ func (cli *DockerCli) CmdRun(args ...string) error {
 		registryAuthHeader := []string{
 			base64.URLEncoding.EncodeToString(buf),
 		}
-		err = cli.stream("POST", "/images/create?"+v.Encode(), nil, cli.err, map[string][]string{
-			"X-Registry-Auth": registryAuthHeader,
-		})
-		if err != nil {
+		if err = cli.stream("POST", "/images/create?"+v.Encode(), nil, cli.err, map[string][]string{"X-Registry-Auth": registryAuthHeader}); err != nil {
 			return err
 		}
-		body, _, err = cli.call("POST", "/containers/create?"+containerValues.Encode(), config)
-		if err != nil {
+		if body, _, err = cli.call("POST", "/containers/create?"+containerValues.Encode(), config); err != nil {
 			return err
 		}
 	}
@@ -2030,17 +2025,17 @@ func (cli *DockerCli) CmdRun(args ...string) error {
 		return err
 	}
 
-	runResult := &APIRun{}
-	err = json.Unmarshal(body, runResult)
-	if err != nil {
+	var runResult APIRun
+	if err := json.Unmarshal(body, &runResult); err != nil {
 		return err
 	}
 
 	for _, warning := range runResult.Warnings {
 		fmt.Fprintf(cli.err, "WARNING: %s\n", warning)
 	}
+
 	if len(hostConfig.ContainerIDFile) > 0 {
-		if _, err = containerIDFile.WriteString(runResult.ID); err != nil {
+		if _, err = containerIDFile.Write([]byte(runResult.ID)); err != nil {
 			return fmt.Errorf("failed to write the container ID to the file: %s", err)
 		}
 	}
@@ -2051,27 +2046,29 @@ func (cli *DockerCli) CmdRun(args ...string) error {
 	}
 
 	var (
-		wait  chan struct{}
-		errCh chan error
+		waitDisplayId chan struct{}
+		errCh         chan error
 	)
 
 	if !config.AttachStdout && !config.AttachStderr {
 		// Make this asynchrone in order to let the client write to stdin before having to read the ID
-		wait = make(chan struct{})
+		waitDisplayId = make(chan struct{})
 		go func() {
-			defer close(wait)
+			defer close(waitDisplayId)
 			fmt.Fprintf(cli.out, "%s\n", runResult.ID)
 		}()
 	}
 
+	// We need to make the chan because the select needs to have a closing
+	// chan, it can't be uninitialized
 	hijacked := make(chan bool)
-
 	if config.AttachStdin || config.AttachStdout || config.AttachStderr {
-
-		v := url.Values{}
+		var (
+			out, stderr io.Writer
+			in          io.ReadCloser
+			v           = url.Values{}
+		)
 		v.Set("stream", "1")
-		var out, stderr io.Writer
-		var in io.ReadCloser
 
 		if config.AttachStdin {
 			v.Set("stdin", "1")
@@ -2125,31 +2122,37 @@ func (cli *DockerCli) CmdRun(args ...string) error {
 		}
 	}
 
+	// Detached mode: wait for the id to be displayed and return.
 	if !config.AttachStdout && !config.AttachStderr {
 		// Detached mode
-		<-wait
-	} else {
-		running, status, err := getExitCode(cli, runResult.ID)
-		if err != nil {
-			return err
-		}
-		if autoRemove {
-			if running {
-				return fmt.Errorf("Impossible to auto-remove a detached container")
-			}
-			// Wait for the process to
-			if _, _, err := cli.call("POST", "/containers/"+runResult.ID+"/wait", nil); err != nil {
-				return err
-			}
-			if _, _, err := cli.call("DELETE", "/containers/"+runResult.ID, nil); err != nil {
-				return err
-			}
-		}
-		if status != 0 {
-			return &utils.StatusError{Status: status}
-		}
+		<-waitDisplayId
+		return nil
 	}
 
+	var status int
+
+	// Attached mode
+	if autoRemove {
+		// Autoremove: wait for the container to finish, retrieve
+		// the exit code and remove the container
+		if _, _, err := cli.call("POST", "/containers/"+runResult.ID+"/wait", nil); err != nil {
+			return err
+		}
+		if _, status, err = getExitCode(cli, runResult.ID); err != nil {
+			return err
+		}
+		if _, _, err := cli.call("DELETE", "/containers/"+runResult.ID, nil); err != nil {
+			return err
+		}
+	} else {
+		// No Autoremove: Simply retrieve the exit code
+		if _, status, err = getExitCode(cli, runResult.ID); err != nil {
+			return err
+		}
+	}
+	if status != 0 {
+		return &utils.StatusError{Status: status}
+	}
 	return nil
 }
 
@@ -2334,7 +2337,7 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer, h
 	}
 
 	if matchesContentType(resp.Header.Get("Content-Type"), "application/json") {
-		return utils.DisplayJSONMessagesStream(resp.Body, out)
+		return utils.DisplayJSONMessagesStream(resp.Body, out, cli.isTerminal)
 	}
 	if _, err := io.Copy(out, resp.Body); err != nil {
 		return err

network.go

@@ -661,6 +661,9 @@ func (manager *NetworkManager) Allocate() (*NetworkInterface, error) {
 }
 
 func (manager *NetworkManager) Close() error {
+	if manager.disabled {
+		return nil
+	}
 	err1 := manager.tcpPortAllocator.Close()
 	err2 := manager.udpPortAllocator.Close()
 	err3 := manager.ipAllocator.Close()

server.go

@@ -63,7 +63,10 @@ func jobInitApi(job *engine.Job) string {
 	}()
 	job.Eng.Hack_SetGlobalVar("httpapi.server", srv)
 	job.Eng.Hack_SetGlobalVar("httpapi.runtime", srv.runtime)
-	job.Eng.Hack_SetGlobalVar("httpapi.bridgeIP", srv.runtime.networkManager.bridgeNetwork.IP)
+	// https://github.com/dotcloud/docker/issues/2768
+	if srv.runtime.networkManager.bridgeNetwork != nil {
+		job.Eng.Hack_SetGlobalVar("httpapi.bridgeIP", srv.runtime.networkManager.bridgeNetwork.IP)
+	}
 	if err := job.Eng.Register("create", srv.ContainerCreate); err != nil {
 		return err.Error()
 	}

utils.go

@@ -235,14 +235,23 @@ func parseLxcOpt(opt string) (string, string, error) {
 	return strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]), nil
 }
 
+// FIXME: network related stuff (including parsing) should be grouped in network file
+const (
+	PortSpecTemplate       = "ip:hostPort:containerPort"
+	PortSpecTemplateFormat = "ip:hostPort:containerPort | ip::containerPort | hostPort:containerPort"
+)
+
 // We will receive port specs in the format of ip:public:private/proto and these need to be
 // parsed in the internal types
 func parsePortSpecs(ports []string) (map[Port]struct{}, map[Port][]PortBinding, error) {
-	exposedPorts := make(map[Port]struct{}, len(ports))
-	bindings := make(map[Port][]PortBinding)
+	var (
+		exposedPorts = make(map[Port]struct{}, len(ports))
+		bindings     = make(map[Port][]PortBinding)
+	)
 
 	for _, rawPort := range ports {
 		proto := "tcp"
+
 		if i := strings.LastIndex(rawPort, "/"); i != -1 {
 			proto = rawPort[i+1:]
 			rawPort = rawPort[:i]
@@ -253,13 +262,16 @@ func parsePortSpecs(ports []string) (map[Port]struct{}, map[Port][]PortBinding,
 			rawPort = fmt.Sprintf(":%s", rawPort)
 		}
 
-		parts, err := utils.PartParser("ip:hostPort:containerPort", rawPort)
+		parts, err := utils.PartParser(PortSpecTemplate, rawPort)
 		if err != nil {
 			return nil, nil, err
 		}
-		containerPort := parts["containerPort"]
-		rawIp := parts["ip"]
-		hostPort := parts["hostPort"]
+
+		var (
+			containerPort = parts["containerPort"]
+			rawIp         = parts["ip"]
+			hostPort      = parts["hostPort"]
+		)
 
 		if containerPort == "" {
 			return nil, nil, fmt.Errorf("No port specified: %s<empty>", rawPort)

utils/utils.go

@@ -779,14 +779,19 @@ func NewHTTPRequestError(msg string, res *http.Response) error {
 	}
 }
 
-func (jm *JSONMessage) Display(out io.Writer) error {
+func (jm *JSONMessage) Display(out io.Writer, isTerminal bool) error {
 	if jm.Error != nil {
 		if jm.Error.Code == 401 {
 			return fmt.Errorf("Authentication is required.")
 		}
 		return jm.Error
 	}
-	fmt.Fprintf(out, "%c[2K\r", 27)
+	endl := ""
+	if isTerminal {
+		// <ESC>[2K = erase entire current line
+		fmt.Fprintf(out, "%c[2K\r", 27)
+		endl = "\r"
+	}
 	if jm.Time != 0 {
 		fmt.Fprintf(out, "[%s] ", time.Unix(jm.Time, 0))
 	}
@@ -797,14 +802,14 @@ func (jm *JSONMessage) Display(out io.Writer) error {
 		fmt.Fprintf(out, "(from %s) ", jm.From)
 	}
 	if jm.Progress != "" {
-		fmt.Fprintf(out, "%s %s\r", jm.Status, jm.Progress)
+		fmt.Fprintf(out, "%s %s%s", jm.Status, jm.Progress, endl)
 	} else {
-		fmt.Fprintf(out, "%s\r\n", jm.Status)
+		fmt.Fprintf(out, "%s%s\n", jm.Status, endl)
 	}
 	return nil
 }
 
-func DisplayJSONMessagesStream(in io.Reader, out io.Writer) error {
+func DisplayJSONMessagesStream(in io.Reader, out io.Writer, isTerminal bool) error {
 	dec := json.NewDecoder(in)
 	ids := make(map[string]int)
 	diff := 0
@@ -825,11 +830,17 @@ func DisplayJSONMessagesStream(in io.Reader, out io.Writer) error {
 			} else {
 				diff = len(ids) - line
 			}
-			fmt.Fprintf(out, "%c[%dA", 27, diff)
+			if isTerminal {
+				// <ESC>[{diff}A = move cursor up diff rows
+				fmt.Fprintf(out, "%c[%dA", 27, diff)
+			}
 		}
-		err := jm.Display(out)
+		err := jm.Display(out, isTerminal)
 		if jm.ID != "" {
-			fmt.Fprintf(out, "%c[%dB", 27, diff)
+			if isTerminal {
+				// <ESC>[{diff}B = move cursor down diff rows
+				fmt.Fprintf(out, "%c[%dB", 27, diff)
+			}
 		}
 		if err != nil {
 			return err
@@ -1226,12 +1237,14 @@ func IsClosedError(err error) bool {
 
 func PartParser(template, data string) (map[string]string, error) {
 	// ip:public:private
-	templateParts := strings.Split(template, ":")
-	parts := strings.Split(data, ":")
+	var (
+		templateParts = strings.Split(template, ":")
+		parts         = strings.Split(data, ":")
+		out           = make(map[string]string, len(templateParts))
+	)
 	if len(parts) != len(templateParts) {
 		return nil, fmt.Errorf("Invalid format to parse.  %s should match template %s", data, template)
 	}
-	out := make(map[string]string, len(templateParts))
 
 	for i, t := range templateParts {
 		value := ""