rebase master

FIXME

@@ -0,0 +1,18 @@
+
+## FIXME
+
+This file is a loose collection of things to improve in the codebase, for the internal
+use of the maintainers.
+
+They are not big enough to be in the roadmap, not user-facing enough to be github issues,
+and not important enough to be discussed in the mailing list.
+
+They are just like FIXME comments in the source code, except we're not sure where in the source
+to put them - so we put them here :)
+
+
+* Merge Runtime, Server and Builder into Runtime
+* Run linter on codebase
+* Unify build commands and regular commands
+* Move source code into src/ subdir for clarity
+* Clean up the Makefile, it's a mess

NOTICE

@@ -3,4 +3,9 @@ Copyright 2012-2013 dotCloud, inc.
 
 This product includes software developed at dotCloud, inc. (http://www.dotcloud.com).
 
-This product contains software (https://github.com/kr/pty) developed by Keith Rarick, licensed under the MIT License.
+This product contains software (https://github.com/kr/pty) developed by Keith Rarick, licensed under the MIT License.
+
+Transfers of Docker shall be in accordance with applicable export controls of any country and all other applicable
+legal requirements.  Docker shall not be distributed or downloaded to or in Cuba, Iran, North Korea, Sudan or Syria
+and shall not be distributed or downloaded to any person on the Denied Persons List administered by the U.S.
+Department of Commerce.

README.md

@@ -373,5 +373,8 @@ Standard Container Specification
 
 ### Legal
 
-Transfers Docker shall be in accordance with any applicable export control or other legal requirements.
+Transfers of Docker shall be in accordance with applicable export controls of any country and all other applicable
+legal requirements.  Docker shall not be distributed or downloaded to or in Cuba, Iran, North Korea, Sudan or Syria
+and shall not be distributed or downloaded to any person on the Denied Persons List administered by the U.S.
+Department of Commerce.
 

api.go

@@ -45,6 +45,8 @@ func httpError(w http.ResponseWriter, err error) {
 		http.Error(w, err.Error(), http.StatusNotFound)
 	} else if strings.HasPrefix(err.Error(), "Bad parameter") {
 		http.Error(w, err.Error(), http.StatusBadRequest)
+	} else if strings.HasPrefix(err.Error(), "Conflict") {
+		http.Error(w, err.Error(), http.StatusConflict)
 	} else if strings.HasPrefix(err.Error(), "Impossible") {
 		http.Error(w, err.Error(), http.StatusNotAcceptable)
 	} else {
@@ -500,14 +502,30 @@ func deleteContainers(srv *Server, version float64, w http.ResponseWriter, r *ht
 }
 
 func deleteImages(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
+	if err := parseForm(r); err != nil {
+		return err
+	}
 	if vars == nil {
 		return fmt.Errorf("Missing parameter")
 	}
 	name := vars["name"]
-	if err := srv.ImageDelete(name); err != nil {
+	imgs, err := srv.ImageDelete(name, version > 1.1)
+	if err != nil {
 		return err
 	}
-	w.WriteHeader(http.StatusNoContent)
+	if imgs != nil {
+		if len(*imgs) != 0 {
+			b, err := json.Marshal(imgs)
+			if err != nil {
+				return err
+			}
+			writeJSON(w, b)
+		} else {
+			return fmt.Errorf("Conflict, %s wasn't deleted", name)
+		}
+	} else {
+		w.WriteHeader(http.StatusNoContent)
+	}
 	return nil
 }
 

api_params.go

@@ -23,6 +23,11 @@ type APIInfo struct {
 	SwapLimit   bool `json:",omitempty"`
 }
 
+type APIRmi struct {
+	Deleted  string `json:",omitempty"`
+	Untagged string `json:",omitempty"`
+}
+
 type APIContainers struct {
 	ID      string `json:"Id"`
 	Image   string

api_test.go

@@ -1266,8 +1266,63 @@ func TestGetEnabledCors(t *testing.T) {
 }
 
 func TestDeleteImages(t *testing.T) {
-	//FIXME: Implement this test
-	t.Log("Test not implemented")
+	runtime, err := newTestRuntime()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer nuke(runtime)
+
+	srv := &Server{runtime: runtime}
+
+	if err := srv.runtime.repositories.Set("test", "test", unitTestImageName, true); err != nil {
+		t.Fatal(err)
+	}
+
+	images, err := srv.Images(false, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(images) != 2 {
+		t.Errorf("Excepted 2 images, %d found", len(images))
+	}
+
+	req, err := http.NewRequest("DELETE", "/images/test:test", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	r := httptest.NewRecorder()
+	if err := deleteImages(srv, APIVERSION, r, req, map[string]string{"name": "test:test"}); err != nil {
+		t.Fatal(err)
+	}
+	if r.Code != http.StatusOK {
+		t.Fatalf("%d OK expected, received %d\n", http.StatusOK, r.Code)
+	}
+
+	var outs []APIRmi
+	if err := json.Unmarshal(r.Body.Bytes(), &outs); err != nil {
+		t.Fatal(err)
+	}
+	if len(outs) != 1 {
+		t.Fatalf("Expected %d event (untagged), got %d", 1, len(outs))
+	}
+	images, err = srv.Images(false, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(images) != 1 {
+		t.Errorf("Excepted 1 image, %d found", len(images))
+	}
+
+	/*	if c := runtime.Get(container.Id); c != nil {
+			t.Fatalf("The container as not been deleted")
+		}
+
+		if _, err := os.Stat(path.Join(container.rwPath(), "test")); err == nil {
+			t.Fatalf("The test file has not been deleted")
+		} */
 }
 
 // Mocked types for tests

commands.go

@@ -581,11 +581,22 @@ func (cli *DockerCli) CmdRmi(args ...string) error {
 	}
 
 	for _, name := range cmd.Args() {
-		_, _, err := cli.call("DELETE", "/images/"+name, nil)
+		body, _, err := cli.call("DELETE", "/images/"+name, nil)
 		if err != nil {
-			fmt.Printf("%s", err)
+			fmt.Fprintf(os.Stderr, "%s", err)
 		} else {
-			fmt.Println(name)
+			var outs []APIRmi
+			err = json.Unmarshal(body, &outs)
+			if err != nil {
+				return err
+			}
+			for _, out := range outs {
+				if out.Deleted != "" {
+					fmt.Println("Deleted:", out.Deleted)
+				} else {
+					fmt.Println("Untagged:", out.Untagged)
+				}
+			}
 		}
 	}
 	return nil

container.go

@@ -355,6 +355,18 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s
 				errors <- err
 			}()
 		}
+	} else {
+		go func() {
+			if stdinCloser != nil {
+				defer stdinCloser.Close()
+			}
+
+			if cStdout, err := container.StdoutPipe(); err != nil {
+				utils.Debugf("Error stdout pipe")
+			} else {
+				io.Copy(&utils.NopWriter{}, cStdout)
+			}
+		}()
 	}
 	if stderr != nil {
 		nJobs += 1
@@ -381,7 +393,20 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s
 				errors <- err
 			}()
 		}
+	} else {
+		go func() {
+			if stdinCloser != nil {
+				defer stdinCloser.Close()
+			}
+
+			if cStderr, err := container.StderrPipe(); err != nil {
+				utils.Debugf("Error stdout pipe")
+			} else {
+				io.Copy(&utils.NopWriter{}, cStderr)
+			}
+		}()
 	}
+
 	return utils.Go(func() error {
 		if cStdout != nil {
 			defer cStdout.Close()

docs/sources/api/docker_remote_api.rst

@@ -35,6 +35,9 @@ The client should send it's authConfig as POST on each call of /images/(name)/pu
 .. http:get:: /auth is now deprecated
 .. http:post:: /auth only checks the configuration but doesn't store it on the server
 
+Deleting an image is now improved, will only untag the image if it has chidrens and remove all the untagged parents if has any.
+.. http:post:: /images/<name>/delete now returns a JSON with the list of images deleted/untagged
+
 
 :doc:`docker_remote_api_v1.1`
 *****************************
@@ -60,13 +63,15 @@ Uses json stream instead of HTML hijack, it looks like this:
 	   {"error":"Invalid..."}
 	   ...
 
-:doc:`docker_remote_api_v1.0`
-*****************************
 
 docker v0.3.4 8d73740_
 
+What's new
+----------
+
 Initial version
 
+
 .. _a8ae398: https://github.com/dotcloud/docker/commit/a8ae398bf52e97148ee7bd0d5868de2e15bd297f
 .. _8d73740: https://github.com/dotcloud/docker/commit/8d73740343778651c09160cde9661f5f387b36f4
 

docs/sources/api/docker_remote_api_v1.1.rst

@@ -1,3 +1,4 @@
+
 :title: Remote API v1.1
 :description: API Documentation for Docker
 :keywords: API, Docker, rcli, REST, documentation
@@ -744,6 +745,7 @@ Tag an image into a repository
 	:statuscode 200: no error
 	:statuscode 400: bad parameter
 	:statuscode 404: no such image
+	:statuscode 409: conflict
         :statuscode 500: server error
 
 

docs/sources/api/docker_remote_api_v1.2.rst

@@ -745,6 +745,7 @@ Tag an image into a repository
 	:statuscode 200: no error
 	:statuscode 400: bad parameter
 	:statuscode 404: no such image
+	:statuscode 409: conflict
         :statuscode 500: server error
 
 
@@ -765,10 +766,18 @@ Remove an image
 
         .. sourcecode:: http
 
-           HTTP/1.1 204 OK
+	   HTTP/1.1 200 OK
+	   Content-type: application/json
+
+	   [
+	    {"Untagged":"3e2f21a89f"},
+	    {"Deleted":"3e2f21a89f"},
+	    {"Deleted":"53b4f83ac9"}
+	   ]
 
 	:statuscode 204: no error
         :statuscode 404: no such image
+	:statuscode 409: conflict
         :statuscode 500: server error
 
 

docs/sources/commandline/command/build.rst

@@ -19,10 +19,15 @@ Examples
 
     docker build .
 
-This will take the local Dockerfile
+| This will read the Dockerfile from the current directory. It will also send any other files and directories found in the current directory to the docker daemon.
+| The contents of this directory would be used by ADD commands found within the Dockerfile.
+| This will send a lot of data to the docker daemon if the current directory contains a lot of data.
+| If the absolute path is provided instead of '.', only the files and directories required by the ADD commands from the Dockerfile will be added to the context and transferred to the docker daemon.
+|
 
 .. code-block:: bash
 
     docker build -
 
-This will read a Dockerfile form Stdin without context
+| This will read a Dockerfile from Stdin without context. Due to the lack of a context, no contents of any local directory will be sent to the docker daemon.
+| ADD doesn't work when running in this mode due to the absence of the context, thus having no source files to copy to the container.

hack/ROADMAP.md

@@ -86,3 +86,20 @@ Production-ready
 Docker is still alpha software, and not suited for production.
 We are working hard to get there, and we are confident that it will be possible within a few months.
 
+
+Advanced port redirections
+--------------------------
+
+Docker currently supports 2 flavors of port redirection: STATIC->STATIC (eg. "redirect public port 80 to private port 80")
+and RANDOM->STATIC (eg. "redirect any public port to private port 80").
+
+With these 2 flavors, docker can support the majority of backend programs out there. But some applications have more exotic
+requirements, generally to implement custom clustering techniques. These applications include Hadoop, MongoDB, Riak, RabbitMQ,
+Disco, and all programs relying on Erlang's OTP.
+
+To support these applications, Docker needs to support more advanced redirection flavors, including:
+
+* RANDOM->RANDOM
+* STATIC1->STATIC2
+
+These flavors should be implemented without breaking existing semantics, if at all possible.

hack/dockerbuilder/Dockerfile

@@ -13,7 +13,7 @@ run	apt-get update
 # Packages required to checkout, build and upload docker
 run	DEBIAN_FRONTEND=noninteractive apt-get install -y -q s3cmd
 run	DEBIAN_FRONTEND=noninteractive apt-get install -y -q curl
-run	curl -s -o /go.tar.gz https://go.googlecode.com/files/go1.1.linux-amd64.tar.gz
+run	curl -s -o /go.tar.gz https://go.googlecode.com/files/go1.1.1.linux-amd64.tar.gz
 run	tar -C /usr/local -xzf /go.tar.gz
 run	echo "export PATH=/usr/local/go/bin:$PATH" > /.bashrc
 run	echo "export PATH=/usr/local/go/bin:$PATH" > /.bash_profile

image.go

@@ -126,6 +126,8 @@ func MountAUFS(ro []string, rw string, target string) error {
 	}
 	branches := fmt.Sprintf("br:%v:%v", rwBranch, roBranches)
 
+	branches += ",xino=/dev/shm/aufs.xino"
+
 	//if error, try to load aufs kernel module
 	if err := mount("none", target, "aufs", 0, branches); err != nil {
 		log.Printf("Kernel does not support AUFS, trying to load the AUFS module with modprobe...")

runtime_test.go

@@ -17,7 +17,7 @@ import (
 )
 
 const unitTestImageName string = "docker-ut"
-
+const unitTestImageId string = "e9aa60c60128cad1"
 const unitTestStoreBase string = "/var/lib/docker/unit-tests"
 
 func nuke(runtime *Runtime) error {

server.go

@@ -1,6 +1,7 @@
 package docker
 
 import (
+	"errors"
 	"fmt"
 	"github.com/dotcloud/docker/auth"
 	"github.com/dotcloud/docker/registry"
@@ -717,17 +718,112 @@ func (srv *Server) ContainerDestroy(name string, removeVolume bool) error {
 	return nil
 }
 
-func (srv *Server) ImageDelete(name string) error {
-	img, err := srv.runtime.repositories.LookupImage(name)
-	if err != nil {
-		return fmt.Errorf("No such image: %s", name)
+var ErrImageReferenced = errors.New("Image referenced by a repository")
+
+func (srv *Server) deleteImageAndChildren(id string, imgs *[]APIRmi) error {
+	// If the image is referenced by a repo, do not delete
+	if len(srv.runtime.repositories.ByID()[id]) != 0 {
+		return ErrImageReferenced
 	}
-	if err := srv.runtime.graph.Delete(img.ID); err != nil {
-		return fmt.Errorf("Error deleting image %s: %s", name, err.Error())
+
+	// If the image is not referenced but has children, go recursive
+	referenced := false
+	byParents, err := srv.runtime.graph.ByParent()
+	if err != nil {
+		return err
+	}
+	for _, img := range byParents[id] {
+		if err := srv.deleteImageAndChildren(img.ID, imgs); err != nil {
+			if err != ErrImageReferenced {
+				return err
+			}
+			referenced = true
+		}
+	}
+	if referenced {
+		return ErrImageReferenced
+	}
+
+	// If the image is not referenced and has no children, remove it
+	byParents, err = srv.runtime.graph.ByParent()
+	if err != nil {
+		return err
+	}
+	if len(byParents[id]) == 0 {
+		if err := srv.runtime.repositories.DeleteAll(id); err != nil {
+			return err
+		}
+		err := srv.runtime.graph.Delete(id)
+		if err != nil {
+			return err
+		}
+		*imgs = append(*imgs, APIRmi{Deleted: utils.TruncateID(id)})
+		return nil
 	}
 	return nil
 }
 
+func (srv *Server) deleteImageParents(img *Image, imgs *[]APIRmi) error {
+	if img.Parent != "" {
+		parent, err := srv.runtime.graph.Get(img.Parent)
+		if err != nil {
+			return err
+		}
+		// Remove all children images
+		if err := srv.deleteImageAndChildren(img.Parent, imgs); err != nil {
+			return err
+		}
+		return srv.deleteImageParents(parent, imgs)
+	}
+	return nil
+}
+
+func (srv *Server) deleteImage(img *Image, repoName, tag string) (*[]APIRmi, error) {
+	//Untag the current image
+	var imgs []APIRmi
+	tagDeleted, err := srv.runtime.repositories.Delete(repoName, tag)
+	if err != nil {
+		return nil, err
+	}
+	if tagDeleted {
+		imgs = append(imgs, APIRmi{Untagged: img.ShortID()})
+	}
+	if len(srv.runtime.repositories.ByID()[img.ID]) == 0 {
+		if err := srv.deleteImageAndChildren(img.ID, &imgs); err != nil {
+			if err != ErrImageReferenced {
+				return &imgs, err
+			}
+		} else if err := srv.deleteImageParents(img, &imgs); err != nil {
+			if err != ErrImageReferenced {
+				return &imgs, err
+			}
+		}
+	}
+	return &imgs, nil
+}
+
+func (srv *Server) ImageDelete(name string, autoPrune bool) (*[]APIRmi, error) {
+	img, err := srv.runtime.repositories.LookupImage(name)
+	if err != nil {
+		return nil, fmt.Errorf("No such image: %s", name)
+	}
+	if !autoPrune {
+		if err := srv.runtime.graph.Delete(img.ID); err != nil {
+			return nil, fmt.Errorf("Error deleting image %s: %s", name, err.Error())
+		}
+		return nil, nil
+	}
+
+	var tag string
+	if strings.Contains(name, ":") {
+		nameParts := strings.Split(name, ":")
+		name = nameParts[0]
+		tag = nameParts[1]
+	}
+
+	return srv.deleteImage(img, name, tag)
+}
+
 func (srv *Server) ImageGetCached(imgId string, config *Config) (*Image, error) {
 
 	// Retrieve all images

server_test.go

@@ -4,6 +4,58 @@ import (
 	"testing"
 )
 
+func TestContainerTagImageDelete(t *testing.T) {
+	runtime, err := newTestRuntime()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer nuke(runtime)
+
+	srv := &Server{runtime: runtime}
+
+	if err := srv.runtime.repositories.Set("utest", "tag1", unitTestImageName, false); err != nil {
+		t.Fatal(err)
+	}
+	if err := srv.runtime.repositories.Set("utest/docker", "tag2", unitTestImageName, false); err != nil {
+		t.Fatal(err)
+	}
+
+	images, err := srv.Images(false, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(images) != 3 {
+		t.Errorf("Excepted 3 images, %d found", len(images))
+	}
+
+	if _, err := srv.ImageDelete("utest/docker:tag2", true); err != nil {
+		t.Fatal(err)
+	}
+
+	images, err = srv.Images(false, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(images) != 2 {
+		t.Errorf("Excepted 2 images, %d found", len(images))
+	}
+
+	if _, err := srv.ImageDelete("utest:tag1", true); err != nil {
+		t.Fatal(err)
+	}
+
+	images, err = srv.Images(false, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(images) != 1 {
+		t.Errorf("Excepted 1 image, %d found", len(images))
+	}
+}
+
 func TestCreateRm(t *testing.T) {
 	runtime, err := newTestRuntime()
 	if err != nil {

tags.go

@@ -110,6 +110,52 @@ func (store *TagStore) ImageName(id string) string {
 	return utils.TruncateID(id)
 }
 
+func (store *TagStore) DeleteAll(id string) error {
+	names, exists := store.ByID()[id]
+	if !exists || len(names) == 0 {
+		return nil
+	}
+	for _, name := range names {
+		if strings.Contains(name, ":") {
+			nameParts := strings.Split(name, ":")
+			if _, err := store.Delete(nameParts[0], nameParts[1]); err != nil {
+				return err
+			}
+		} else {
+			if _, err := store.Delete(name, ""); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (store *TagStore) Delete(repoName, tag string) (bool, error) {
+	deleted := false
+	if err := store.Reload(); err != nil {
+		return false, err
+	}
+	if r, exists := store.Repositories[repoName]; exists {
+		if tag != "" {
+			if _, exists2 := r[tag]; exists2 {
+				delete(r, tag)
+				if len(r) == 0 {
+					delete(store.Repositories, repoName)
+				}
+				deleted = true
+			} else {
+				return false, fmt.Errorf("No such tag: %s:%s", repoName, tag)
+			}
+		} else {
+			delete(store.Repositories, repoName)
+			deleted = true
+		}
+	} else {
+		fmt.Errorf("No such repository: %s", repoName)
+	}
+	return deleted, store.Save()
+}
+
 func (store *TagStore) Set(repoName, tag, imageName string, force bool) error {
 	img, err := store.LookupImage(imageName)
 	if err != nil {
@@ -133,7 +179,7 @@ func (store *TagStore) Set(repoName, tag, imageName string, force bool) error {
 	} else {
 		repo = make(map[string]string)
 		if old, exists := store.Repositories[repoName]; exists && !force {
-			return fmt.Errorf("Tag %s:%s is already set to %s", repoName, tag, old)
+			return fmt.Errorf("Conflict: Tag %s:%s is already set to %s", repoName, tag, old)
 		}
 		store.Repositories[repoName] = repo
 	}
@@ -151,14 +197,20 @@ func (store *TagStore) Get(repoName string) (Repository, error) {
 	return nil, nil
 }
 
-func (store *TagStore) GetImage(repoName, tag string) (*Image, error) {
+func (store *TagStore) GetImage(repoName, tagOrId string) (*Image, error) {
 	repo, err := store.Get(repoName)
 	if err != nil {
 		return nil, err
 	} else if repo == nil {
 		return nil, nil
 	}
-	if revision, exists := repo[tag]; exists {
+	//go through all the tags, to see if tag is in fact an ID
+	for _, revision := range repo {
+		if strings.HasPrefix(revision, tagOrId) {
+			return store.graph.Get(revision)
+		}
+	}
+	if revision, exists := repo[tagOrId]; exists {
 		return store.graph.Get(revision)
 	}
 	return nil, nil

tags_test.go

@@ -0,0 +1,49 @@
+package docker
+
+import (
+	"testing"
+)
+
+func TestLookupImage(t *testing.T) {
+	runtime, err := newTestRuntime()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer nuke(runtime)
+
+	if img, err := runtime.repositories.LookupImage(unitTestImageName); err != nil {
+		t.Fatal(err)
+	} else if img == nil {
+		t.Errorf("Expected 1 image, none found")
+	}
+
+	if img, err := runtime.repositories.LookupImage(unitTestImageName + ":" + DEFAULTTAG); err != nil {
+		t.Fatal(err)
+	} else if img == nil {
+		t.Errorf("Expected 1 image, none found")
+	}
+
+	if img, err := runtime.repositories.LookupImage(unitTestImageName + ":" + "fail"); err == nil {
+		t.Errorf("Expected error, none found")
+	} else if img != nil {
+		t.Errorf("Expected 0 image, 1 found")
+	}
+
+	if img, err := runtime.repositories.LookupImage("fail:fail"); err == nil {
+		t.Errorf("Expected error, none found")
+	} else if img != nil {
+		t.Errorf("Expected 0 image, 1 found")
+	}
+
+	if img, err := runtime.repositories.LookupImage(unitTestImageId); err != nil {
+		t.Fatal(err)
+	} else if img == nil {
+		t.Errorf("Expected 1 image, none found")
+	}
+
+	if img, err := runtime.repositories.LookupImage(unitTestImageName + ":" + unitTestImageId); err != nil {
+		t.Fatal(err)
+	} else if img == nil {
+		t.Errorf("Expected 1 image, none found")
+	}
+}