From 33f6a0aaf58734f39af564123ca2c0056a3f1bd4 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Mon, 18 Mar 2013 00:15:35 -0700 Subject: [PATCH 01/80] docker/graph: a store for filesystem images and the graph of their relationships --- {fs => graph}/archive.go | 4 +- {fs => graph}/archive_test.go | 2 +- graph/changes.go | 106 +++++++++++++++ graph/graph.go | 143 ++++++++++++++++++++ graph/graph_test.go | 143 ++++++++++++++++++++ graph/image.go | 245 ++++++++++++++++++++++++++++++++++ {fs => graph}/mount_darwin.go | 2 +- {fs => graph}/mount_linux.go | 2 +- 8 files changed, 643 insertions(+), 4 deletions(-) rename {fs => graph}/archive.go (97%) rename {fs => graph}/archive_test.go (99%) create mode 100644 graph/changes.go create mode 100644 graph/graph.go create mode 100644 graph/graph_test.go create mode 100644 graph/image.go rename {fs => graph}/mount_darwin.go (92%) rename {fs => graph}/mount_linux.go (92%) diff --git a/fs/archive.go b/graph/archive.go similarity index 97% rename from fs/archive.go rename to graph/archive.go index f43cc64b66..b6281bf4b5 100644 --- a/fs/archive.go +++ b/graph/archive.go @@ -1,4 +1,4 @@ -package fs +package graph import ( "errors" @@ -7,6 +7,8 @@ import ( "os/exec" ) +type Archive io.Reader + type Compression uint32 const ( diff --git a/fs/archive_test.go b/graph/archive_test.go similarity index 99% rename from fs/archive_test.go rename to graph/archive_test.go index b182a1563e..c158537a6e 100644 --- a/fs/archive_test.go +++ b/graph/archive_test.go @@ -1,4 +1,4 @@ -package fs +package graph import ( "io/ioutil" diff --git a/graph/changes.go b/graph/changes.go new file mode 100644 index 0000000000..eebf7657e7 --- /dev/null +++ b/graph/changes.go @@ -0,0 +1,106 @@ +package graph + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +type ChangeType int + +const ( + ChangeModify = iota + ChangeAdd + ChangeDelete +) + +type Change struct { + Path string + Kind ChangeType +} + +func (change *Change) String() string { + var kind string + switch change.Kind { + case ChangeModify: + kind = "C" + case ChangeAdd: + kind = "A" + case ChangeDelete: + kind = "D" + } + return fmt.Sprintf("%s %s", kind, change.Path) +} + +func Changes(layers []string, rw string) ([]Change, error) { + var changes []Change + err := filepath.Walk(rw, func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + + // Rebase path + path, err = filepath.Rel(rw, path) + if err != nil { + return err + } + path = filepath.Join("/", path) + + // Skip root + if path == "/" { + return nil + } + + // Skip AUFS metadata + if matched, err := filepath.Match("/.wh..wh.*", path); err != nil || matched { + return err + } + + change := Change{ + Path: path, + } + + // Find out what kind of modification happened + file := filepath.Base(path) + // If there is a whiteout, then the file was removed + if strings.HasPrefix(file, ".wh.") { + originalFile := strings.TrimLeft(file, ".wh.") + change.Path = filepath.Join(filepath.Dir(path), originalFile) + change.Kind = ChangeDelete + } else { + // Otherwise, the file was added + change.Kind = ChangeAdd + + // ...Unless it already existed in a top layer, in which case, it's a modification + for _, layer := range layers { + stat, err := os.Stat(filepath.Join(layer, path)) + if err != nil && !os.IsNotExist(err) { + return err + } + if err == nil { + // The file existed in the top layer, so that's a modification + + // However, if it's a directory, maybe it wasn't actually modified. + // If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar + if stat.IsDir() && f.IsDir() { + if f.Size() == stat.Size() && f.Mode() == stat.Mode() && f.ModTime() == stat.ModTime() { + // Both directories are the same, don't record the change + return nil + } + } + change.Kind = ChangeModify + break + } + } + } + + // Record change + changes = append(changes, change) + return nil + }) + if err != nil { + return nil, err + } + return changes, nil +} diff --git a/graph/graph.go b/graph/graph.go new file mode 100644 index 0000000000..7e5e884c8e --- /dev/null +++ b/graph/graph.go @@ -0,0 +1,143 @@ +package graph + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "time" +) + +type Graph struct { + Root string +} + +func New(root string) (*Graph, error) { + abspath, err := filepath.Abs(root) + if err != nil { + return nil, err + } + // Create the root directory if it doesn't exists + if err := os.Mkdir(root, 0700); err != nil && !os.IsExist(err) { + return nil, err + } + return &Graph{ + Root: abspath, + }, nil +} + +func (graph *Graph) Exists(id string) bool { + if _, err := graph.Get(id); err != nil { + return false + } + return true +} + +func (graph *Graph) Get(id string) (*Image, error) { + img, err := LoadImage(graph.imageRoot(id)) + if err != nil { + return nil, err + } + if img.Id != id { + return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.Id) + } + return img, nil +} + +func (graph *Graph) Create(layerData Archive, parent, comment string) (*Image, error) { + img := &Image{ + Id: GenerateId(), + Parent: parent, + Comment: comment, + Created: time.Now(), + } + if err := graph.Register(layerData, img); err != nil { + return nil, err + } + return img, nil +} + +func (graph *Graph) Register(layerData Archive, img *Image) error { + if err := ValidateId(img.Id); err != nil { + return err + } + // (This is a convenience to save time. Race conditions are taken care of by os.Rename) + if graph.Exists(img.Id) { + return fmt.Errorf("Image %s already exists", img.Id) + } + tmp, err := graph.Mktemp(img.Id) + defer os.RemoveAll(tmp) + if err != nil { + return fmt.Errorf("Mktemp failed: %s", err) + } + if err := StoreImage(img, layerData, tmp); err != nil { + return err + } + // Commit + if err := os.Rename(tmp, graph.imageRoot(img.Id)); err != nil { + return err + } + img.graph = graph + return nil +} + +func (graph *Graph) Mktemp(id string) (string, error) { + tmp, err := New(path.Join(graph.Root, ":tmp:")) + if err != nil { + return "", fmt.Errorf("Couldn't create temp: %s", err) + } + if tmp.Exists(id) { + return "", fmt.Errorf("Image %d already exists", id) + } + return tmp.imageRoot(id), nil +} + +func (graph *Graph) Garbage() (*Graph, error) { + return New(path.Join(graph.Root, ":garbage:")) +} + +func (graph *Graph) Delete(id string) error { + garbage, err := graph.Garbage() + if err != nil { + return err + } + return os.Rename(graph.imageRoot(id), garbage.imageRoot(id)) +} + +func (graph *Graph) Undelete(id string) error { + garbage, err := graph.Garbage() + if err != nil { + return err + } + return os.Rename(garbage.imageRoot(id), graph.imageRoot(id)) +} + +func (graph *Graph) GarbageCollect() error { + garbage, err := graph.Garbage() + if err != nil { + return err + } + return os.RemoveAll(garbage.Root) +} + +func (graph *Graph) All() ([]*Image, error) { + files, err := ioutil.ReadDir(graph.Root) + if err != nil { + return nil, err + } + var images []*Image + for _, st := range files { + if img, err := graph.Get(st.Name()); err != nil { + // Skip image + continue + } else { + images = append(images, img) + } + } + return images, nil +} + +func (graph *Graph) imageRoot(id string) string { + return path.Join(graph.Root, id) +} diff --git a/graph/graph_test.go b/graph/graph_test.go new file mode 100644 index 0000000000..8487aaf1df --- /dev/null +++ b/graph/graph_test.go @@ -0,0 +1,143 @@ +package graph + +import ( + "github.com/dotcloud/docker/fake" + "io/ioutil" + "os" + "path" + "testing" + "time" +) + +func TestInit(t *testing.T) { + graph := tempGraph(t) + defer os.RemoveAll(graph.Root) + // Root should exist + if _, err := os.Stat(graph.Root); err != nil { + t.Fatal(err) + } + // All() should be empty + if l, err := graph.All(); err != nil { + t.Fatal(err) + } else if len(l) != 0 { + t.Fatalf("List() should return %d, not %d", 0, len(l)) + } +} + +// FIXME: Do more extensive tests (ex: create multiple, delete, recreate; +// create multiple, check the amount of images and paths, etc..) +func TestCreate(t *testing.T) { + graph := tempGraph(t) + defer os.RemoveAll(graph.Root) + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + image, err := graph.Create(archive, "", "Testing") + if err != nil { + t.Fatal(err) + } + if err := ValidateId(image.Id); err != nil { + t.Fatal(err) + } + if image.Comment != "Testing" { + t.Fatalf("Wrong comment: should be '%s', not '%s'", "Testing", image.Comment) + } + if images, err := graph.All(); err != nil { + t.Fatal(err) + } else if l := len(images); l != 1 { + t.Fatalf("Wrong number of images. Should be %d, not %d", 1, l) + } +} + +func TestRegister(t *testing.T) { + graph := tempGraph(t) + defer os.RemoveAll(graph.Root) + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + image := &Image{ + Id: GenerateId(), + Comment: "testing", + Created: time.Now(), + } + err = graph.Register(archive, image) + if err != nil { + t.Fatal(err) + } + if images, err := graph.All(); err != nil { + t.Fatal(err) + } else if l := len(images); l != 1 { + t.Fatalf("Wrong number of images. Should be %d, not %d", 1, l) + } + if resultImg, err := graph.Get(image.Id); err != nil { + t.Fatal(err) + } else { + if resultImg.Id != image.Id { + t.Fatalf("Wrong image ID. Should be '%s', not '%s'", image.Id, resultImg.Id) + } + if resultImg.Comment != image.Comment { + t.Fatalf("Wrong image comment. Should be '%s', not '%s'", image.Comment, resultImg.Comment) + } + } +} + +func TestMount(t *testing.T) { + graph := tempGraph(t) + defer os.RemoveAll(graph.Root) + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + image, err := graph.Create(archive, "", "Testing") + if err != nil { + t.Fatal(err) + } + tmp, err := ioutil.TempDir("", "docker-test-graph-mount-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + rootfs := path.Join(tmp, "rootfs") + if err := os.MkdirAll(rootfs, 0700); err != nil { + t.Fatal(err) + } + rw := path.Join(tmp, "rw") + if err := os.MkdirAll(rw, 0700); err != nil { + t.Fatal(err) + } + if err := image.Mount(rootfs, rw); err != nil { + t.Fatal(err) + } + // FIXME: test for mount contents + defer func() { + if err := Unmount(rootfs); err != nil { + t.Error(err) + } + }() +} + +/* + * HELPER FUNCTIONS + */ + +func tempGraph(t *testing.T) *Graph { + tmp, err := ioutil.TempDir("", "docker-graph-") + if err != nil { + t.Fatal(err) + } + graph, err := New(tmp) + if err != nil { + t.Fatal(err) + } + return graph +} + +func testArchive(t *testing.T) Archive { + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + return archive +} diff --git a/graph/image.go b/graph/image.go new file mode 100644 index 0000000000..ca03461530 --- /dev/null +++ b/graph/image.go @@ -0,0 +1,245 @@ +package graph + +import ( + "encoding/json" + "fmt" + "github.com/dotcloud/docker/future" + "io/ioutil" + "os" + "path" + "strings" + "syscall" + "time" +) + +type Image struct { + Id string + Parent string + Comment string + Created time.Time + graph *Graph +} + +func LoadImage(root string) (*Image, error) { + // Load the json data + jsonData, err := ioutil.ReadFile(jsonPath(root)) + if err != nil { + return nil, err + } + var img Image + if err := json.Unmarshal(jsonData, &img); err != nil { + return nil, err + } + if err := ValidateId(img.Id); err != nil { + return nil, err + } + // Check that the filesystem layer exists + if stat, err := os.Stat(layerPath(root)); err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("Couldn't load image %s: no filesystem layer", img.Id) + } else { + return nil, err + } + } else if !stat.IsDir() { + return nil, fmt.Errorf("Couldn't load image %s: %s is not a directory", img.Id, layerPath(root)) + } + return &img, nil +} + +func StoreImage(img *Image, layerData Archive, root string) error { + // Check that root doesn't already exist + if _, err := os.Stat(root); err == nil { + return fmt.Errorf("Image %s already exists", img.Id) + } else if !os.IsNotExist(err) { + return err + } + // Store the layer + layer := layerPath(root) + if err := os.MkdirAll(layer, 0700); err != nil { + return err + } + if err := Untar(layerData, layer); err != nil { + return err + } + // Store the json ball + jsonData, err := json.Marshal(img) + if err != nil { + return err + } + if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil { + return err + } + return nil +} + +func layerPath(root string) string { + return path.Join(root, "layer") +} + +func jsonPath(root string) string { + return path.Join(root, "json") +} + +func MountAUFS(ro []string, rw string, target string) error { + // FIXME: Now mount the layers + rwBranch := fmt.Sprintf("%v=rw", rw) + roBranches := "" + for _, layer := range ro { + roBranches += fmt.Sprintf("%v=ro:", layer) + } + branches := fmt.Sprintf("br:%v:%v", rwBranch, roBranches) + return mount("none", target, "aufs", 0, branches) +} + +func Unmount(target string) error { + if err := syscall.Unmount(target, 0); err != nil { + return err + } + // Even though we just unmounted the filesystem, AUFS will prevent deleting the mntpoint + // for some time. We'll just keep retrying until it succeeds. + for retries := 0; retries < 1000; retries++ { + err := os.Remove(target) + if err == nil { + // rm mntpoint succeeded + return nil + } + if os.IsNotExist(err) { + // mntpoint doesn't exist anymore. Success. + return nil + } + // fmt.Printf("(%v) Remove %v returned: %v\n", retries, target, err) + time.Sleep(10 * time.Millisecond) + } + return fmt.Errorf("Umount: Failed to umount %v", target) +} + +func (image *Image) Mount(root, rw string) error { + layers, err := image.layers() + if err != nil { + return err + } + // FIXME: @creack shouldn't we do this after going over changes? + if err := MountAUFS(layers, rw, root); err != nil { + return err + } + // FIXME: Create tests for deletion + // FIXME: move this part to change.go + // Retrieve the changeset from the parent and apply it to the container + // - Retrieve the changes + changes, err := Changes(layers, layers[0]) + if err != nil { + return err + } + // Iterate on changes + for _, c := range changes { + // If there is a delete + if c.Kind == ChangeDelete { + // Make sure the directory exists + file_path, file_name := path.Dir(c.Path), path.Base(c.Path) + if err := os.MkdirAll(path.Join(rw, file_path), 0755); err != nil { + return err + } + // And create the whiteout (we just need to create empty file, discard the return) + if _, err := os.Create(path.Join(path.Join(rw, file_path), + ".wh."+path.Base(file_name))); err != nil { + return err + } + } + } + return nil +} + +func ValidateId(id string) error { + if id == "" { + return fmt.Errorf("Image id can't be empty") + } + if strings.Contains(id, ":") { + return fmt.Errorf("Invalid character in image id: ':'") + } + return nil +} + +func GenerateId() string { + future.Seed() + return future.RandomId() +} + +// Image includes convenience proxy functions to its graph +// These functions will return an error if the image is not registered +// (ie. if image.graph == nil) + +func (img *Image) History() ([]*Image, error) { + var parents []*Image + if err := img.WalkHistory( + func(img *Image) { + parents = append(parents, img) + }, + ); err != nil { + return nil, err + } + return parents, nil +} + +// layers returns all the filesystem layers needed to mount an image +func (img *Image) layers() ([]string, error) { + var list []string + var e error + if err := img.WalkHistory( + func(img *Image) { + if layer, err := img.layer(); err != nil { + e = err + } else if layer != "" { + list = append(list, layer) + } + }, + ); err != nil { + return nil, err + } else if e != nil { // Did an error occur inside the handler? + return nil, e + } + if len(list) == 0 { + return nil, fmt.Errorf("No layer found for image %s\n", img.Id) + } + return list, nil +} + +func (img *Image) WalkHistory(handler func(*Image)) error { + var err error + currentImg := img + for currentImg != nil { + if handler != nil { + handler(currentImg) + } + currentImg, err = currentImg.GetParent() + if err != nil { + return fmt.Errorf("Error while getting parent image: %v", err) + } + } + return nil +} + +func (img *Image) GetParent() (*Image, error) { + if img.Parent == "" { + return nil, nil + } + if img.graph == nil { + return nil, fmt.Errorf("Can't lookup parent of unregistered image") + } + return img.graph.Get(img.Parent) +} + +func (img *Image) root() (string, error) { + if img.graph == nil { + return "", fmt.Errorf("Can't lookup root of unregistered image") + } + return img.graph.imageRoot(img.Id), nil +} + +// Return the path of an image's layer +func (img *Image) layer() (string, error) { + root, err := img.root() + if err != nil { + return "", err + } + return layerPath(root), nil +} diff --git a/fs/mount_darwin.go b/graph/mount_darwin.go similarity index 92% rename from fs/mount_darwin.go rename to graph/mount_darwin.go index 540d6f7691..1fb24390b3 100644 --- a/fs/mount_darwin.go +++ b/graph/mount_darwin.go @@ -1,4 +1,4 @@ -package fs +package graph import "errors" diff --git a/fs/mount_linux.go b/graph/mount_linux.go similarity index 92% rename from fs/mount_linux.go rename to graph/mount_linux.go index b36888f75c..fd849821cf 100644 --- a/fs/mount_linux.go +++ b/graph/mount_linux.go @@ -1,4 +1,4 @@ -package fs +package graph import "syscall" From 3c1db4ca43ecd741c50e884c2ea3aac43e3191e1 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Mon, 18 Mar 2013 17:57:18 -0700 Subject: [PATCH 02/80] Ported test for image deletion --- graph/graph_test.go | 48 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/graph/graph_test.go b/graph/graph_test.go index 8487aaf1df..0fbebb8948 100644 --- a/graph/graph_test.go +++ b/graph/graph_test.go @@ -118,6 +118,54 @@ func TestMount(t *testing.T) { }() } +func TestDelete(t *testing.T) { + graph := tempGraph(t) + defer os.RemoveAll(graph.Root) + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + assertNImages(graph, t, 0) + img, err := graph.Create(archive, "", "Bla bla") + if err != nil { + t.Fatal(err) + } + assertNImages(graph, t, 1) + if err := graph.Delete(img.Id); err != nil { + t.Fatal(err) + } + assertNImages(graph, t, 0) + + // Test 2 create (same name) / 1 delete + img1, err := graph.Create(archive, "foo", "Testing") + if err != nil { + t.Fatal(err) + } + if _, err = graph.Create(archive, "foo", "Testing"); err != nil { + t.Fatal(err) + } + assertNImages(graph, t, 2) + if err := graph.Delete(img1.Id); err != nil { + t.Fatal(err) + } + assertNImages(graph, t, 1) + + // Test delete wrong name + if err := graph.Delete("Not_foo"); err == nil { + t.Fatalf("Deleting wrong ID should return an error") + } + assertNImages(graph, t, 1) + +} + +func assertNImages(graph *Graph, t *testing.T, n int) { + if images, err := graph.All(); err != nil { + t.Fatal(err) + } else if actualN := len(images); actualN != n { + t.Fatalf("Expected %d images, found %d", n, actualN) + } +} + /* * HELPER FUNCTIONS */ From 31296cc3f72cd98cbab8c33b2b1d3ea2d46be725 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 20 Mar 2013 20:21:59 -0700 Subject: [PATCH 03/80] Removed deprecated or useless commands (cp, layers, reset, mirror, debug, web) --- commands.go | 96 ----------------------------------------------------- 1 file changed, 96 deletions(-) diff --git a/commands.go b/commands.go index ef3f6ed148..0e0ea2c809 100644 --- a/commands.go +++ b/commands.go @@ -46,14 +46,12 @@ func (srv *Server) Help() string { {"info", "Display system-wide information"}, {"inspect", "Return low-level information on a container"}, {"kill", "Kill a running container"}, - {"layers", "(debug only) List filesystem layers"}, {"login", "Register or Login to the docker registry server"}, {"logs", "Fetch the logs of a container"}, {"ls", "List the contents of a container's directory"}, {"mirror", "(debug only) (No documentation available)"}, {"port", "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT"}, {"ps", "List containers"}, - {"reset", "Reset changes to a container's filesystem"}, {"restart", "Restart a running container"}, {"rm", "Remove a container"}, {"rmi", "Remove an image"}, @@ -64,7 +62,6 @@ func (srv *Server) Help() string { {"umount", "(debug only) Mount a container's filesystem"}, {"version", "Show the docker version information"}, {"wait", "Block until a container stops, then print its exit code"}, - {"web", "A web UI for docker"}, {"write", "Write the contents of standard input to a container's file"}, } { help += fmt.Sprintf(" %-10.10s%s\n", cmd...) @@ -623,40 +620,6 @@ func (srv *Server) CmdPs(stdin io.ReadCloser, stdout io.Writer, args ...string) return nil } -func (srv *Server) CmdLayers(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, - "layers", "[OPTIONS]", - "List filesystem layers (debug only)") - if err := cmd.Parse(args); err != nil { - return nil - } - for _, layer := range srv.images.Layers() { - fmt.Fprintln(stdout, layer) - } - return nil -} - -func (srv *Server) CmdCp(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, - "cp", "[OPTIONS] IMAGE NAME", - "Create a copy of IMAGE and call it NAME") - if err := cmd.Parse(args); err != nil { - return nil - } - if image, err := srv.images.Get(cmd.Arg(0)); err != nil { - return err - } else if image == nil { - return errors.New("Image " + cmd.Arg(0) + " does not exist") - } else { - if img, err := image.Copy(cmd.Arg(1)); err != nil { - return err - } else { - fmt.Fprintln(stdout, img.Id) - } - } - return nil -} - func (srv *Server) CmdCommit(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "commit", "[OPTIONS] CONTAINER [DEST]", @@ -745,26 +708,6 @@ func (srv *Server) CmdDiff(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } -func (srv *Server) CmdReset(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, - "reset", "CONTAINER [OPTIONS]", - "Reset changes to a container's filesystem") - if err := cmd.Parse(args); err != nil { - return nil - } - if cmd.NArg() < 1 { - return errors.New("Not enough arguments") - } - for _, name := range cmd.Args() { - if container := srv.containers.Get(name); container != nil { - if err := container.Mountpoint.Reset(); err != nil { - return errors.New("Reset " + container.Id + ": " + err.Error()) - } - } - } - return nil -} - func (srv *Server) CmdLogs(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "logs", "[OPTIONS] CONTAINER", "Fetch the logs of a container") if err := cmd.Parse(args); err != nil { @@ -1007,45 +950,6 @@ func NewServer() (*Server, error) { return srv, nil } -func (srv *Server) CmdMirror(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - _, err := io.Copy(stdout, stdin) - return err -} - -func (srv *Server) CmdDebug(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - for { - if line, err := bufio.NewReader(stdin).ReadString('\n'); err == nil { - fmt.Printf("--- %s", line) - } else if err == io.EOF { - if len(line) > 0 { - fmt.Printf("--- %s\n", line) - } - break - } else { - return err - } - } - return nil -} - -func (srv *Server) CmdWeb(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "web", "[OPTIONS]", "A web UI for docker") - showurl := cmd.Bool("u", false, "Return the URL of the web UI") - if err := cmd.Parse(args); err != nil { - return nil - } - if *showurl { - fmt.Fprintln(stdout, "http://localhost:4242/web") - } else { - if file, err := os.Open("dockerweb.html"); err != nil { - return err - } else if _, err := io.Copy(stdout, file); err != nil { - return err - } - } - return nil -} - type Server struct { containers *Docker images *fs.Store From c8db980add4cdc9546a7e38dba264f6862662f15 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 20 Mar 2013 22:12:38 -0700 Subject: [PATCH 04/80] Image.Changes(): list all changes between an image and a rw directory --- graph/image.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/graph/image.go b/graph/image.go index ca03461530..4b10aa05d4 100644 --- a/graph/image.go +++ b/graph/image.go @@ -149,6 +149,14 @@ func (image *Image) Mount(root, rw string) error { return nil } +func (image *Image) Changes(rw string) ([]Change, error) { + layers, err := image.layers() + if err != nil { + return nil, err + } + return Changes(layers, rw) +} + func ValidateId(id string) error { if id == "" { return fmt.Errorf("Image id can't be empty") From 9e8278134d73ffa0a4e72c2202aa1e5c2bcffc98 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 20 Mar 2013 22:13:28 -0700 Subject: [PATCH 05/80] Image.Mount(): create rw and rootfs directory if they don't exist --- graph/image.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/graph/image.go b/graph/image.go index 4b10aa05d4..a302b4ddfc 100644 --- a/graph/image.go +++ b/graph/image.go @@ -118,6 +118,13 @@ func (image *Image) Mount(root, rw string) error { if err != nil { return err } + // Create the target directories if they don't exist + if err := os.Mkdir(root, 0755); err != nil && !os.IsExist(err) { + return err + } + if err := os.Mkdir(rw, 0755); err != nil && !os.IsExist(err) { + return err + } // FIXME: @creack shouldn't we do this after going over changes? if err := MountAUFS(layers, rw, root); err != nil { return err From 240333277afb733af353e21dd3066c547da05b8a Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 20 Mar 2013 22:13:57 -0700 Subject: [PATCH 06/80] Removed Image.Unmount(). It belongs in container code --- graph/image.go | 27 +++++---------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/graph/image.go b/graph/image.go index a302b4ddfc..9cb72a9632 100644 --- a/graph/image.go +++ b/graph/image.go @@ -91,29 +91,12 @@ func MountAUFS(ro []string, rw string, target string) error { return mount("none", target, "aufs", 0, branches) } -func Unmount(target string) error { - if err := syscall.Unmount(target, 0); err != nil { - return err - } - // Even though we just unmounted the filesystem, AUFS will prevent deleting the mntpoint - // for some time. We'll just keep retrying until it succeeds. - for retries := 0; retries < 1000; retries++ { - err := os.Remove(target) - if err == nil { - // rm mntpoint succeeded - return nil - } - if os.IsNotExist(err) { - // mntpoint doesn't exist anymore. Success. - return nil - } - // fmt.Printf("(%v) Remove %v returned: %v\n", retries, target, err) - time.Sleep(10 * time.Millisecond) - } - return fmt.Errorf("Umount: Failed to umount %v", target) -} - func (image *Image) Mount(root, rw string) error { + if isMounted, err := IsMounted(root); err != nil { + return err + } else if isMounted { + return fmt.Errorf("%s is already mounted", root) + } layers, err := image.layers() if err != nil { return err From ea258c44926e8a8d7a49cedd90922d29185df573 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 20 Mar 2013 22:15:09 -0700 Subject: [PATCH 07/80] docker/fs is deprecated by docker/graph --- fs/changes.go | 127 ----------- fs/layers.go | 113 ---------- fs/layers_test.go | 96 --------- fs/remove_test.go | 222 -------------------- fs/store.go | 521 ---------------------------------------------- fs/store_test.go | 277 ------------------------ 6 files changed, 1356 deletions(-) delete mode 100644 fs/changes.go delete mode 100644 fs/layers.go delete mode 100644 fs/layers_test.go delete mode 100644 fs/remove_test.go delete mode 100644 fs/store.go delete mode 100644 fs/store_test.go diff --git a/fs/changes.go b/fs/changes.go deleted file mode 100644 index 659f688c45..0000000000 --- a/fs/changes.go +++ /dev/null @@ -1,127 +0,0 @@ -package fs - -import ( - "fmt" - "os" - "path/filepath" - "strings" -) - -type ChangeType int - -const ( - ChangeModify = iota - ChangeAdd - ChangeDelete -) - -type Change struct { - Path string - Kind ChangeType -} - -func (change *Change) String() string { - var kind string - switch change.Kind { - case ChangeModify: - kind = "C" - case ChangeAdd: - kind = "A" - case ChangeDelete: - kind = "D" - } - return fmt.Sprintf("%s %s", kind, change.Path) -} - -func (store *Store) Changes(mp *Mountpoint) ([]Change, error) { - var changes []Change - image, err := store.Get(mp.Image) - if err != nil { - return nil, err - } - layers, err := image.layers() - if err != nil { - return nil, err - } - - err = filepath.Walk(mp.Rw, func(path string, f os.FileInfo, err error) error { - if err != nil { - return err - } - - // Rebase path - path, err = filepath.Rel(mp.Rw, path) - if err != nil { - return err - } - path = filepath.Join("/", path) - - // Skip root - if path == "/" { - return nil - } - - // Skip AUFS metadata - if matched, err := filepath.Match("/.wh..wh.*", path); err != nil || matched { - return err - } - - change := Change{ - Path: path, - } - - // Find out what kind of modification happened - file := filepath.Base(path) - // If there is a whiteout, then the file was removed - if strings.HasPrefix(file, ".wh.") { - originalFile := strings.TrimLeft(file, ".wh.") - change.Path = filepath.Join(filepath.Dir(path), originalFile) - change.Kind = ChangeDelete - } else { - // Otherwise, the file was added - change.Kind = ChangeAdd - - // ...Unless it already existed in a top layer, in which case, it's a modification - for _, layer := range layers { - stat, err := os.Stat(filepath.Join(layer, path)) - if err != nil && !os.IsNotExist(err) { - return err - } - if err == nil { - // The file existed in the top layer, so that's a modification - - // However, if it's a directory, maybe it wasn't actually modified. - // If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar - if stat.IsDir() && f.IsDir() { - if f.Size() == stat.Size() && f.Mode() == stat.Mode() && f.ModTime() == stat.ModTime() { - // Both directories are the same, don't record the change - return nil - } - } - change.Kind = ChangeModify - break - } - } - } - - // Record change - changes = append(changes, change) - return nil - }) - if err != nil { - return nil, err - } - return changes, nil -} - -// Reset removes all changes to the filesystem, reverting it to its initial state. -func (mp *Mountpoint) Reset() error { - if err := os.RemoveAll(mp.Rw); err != nil { - return err - } - // We removed the RW directory itself along with its content: let's re-create an empty one. - if err := mp.createFolders(); err != nil { - return err - } - return nil -} diff --git a/fs/layers.go b/fs/layers.go deleted file mode 100644 index dc7e621f85..0000000000 --- a/fs/layers.go +++ /dev/null @@ -1,113 +0,0 @@ -package fs - -import ( - "errors" - "fmt" - "github.com/dotcloud/docker/future" - "io/ioutil" - "os" - "path" - "path/filepath" -) - -type LayerStore struct { - Root string -} - -func NewLayerStore(root string) (*LayerStore, error) { - abspath, err := filepath.Abs(root) - if err != nil { - return nil, err - } - // Create the root directory if it doesn't exists - if err := os.Mkdir(root, 0700); err != nil && !os.IsExist(err) { - return nil, err - } - return &LayerStore{ - Root: abspath, - }, nil -} - -func (store *LayerStore) List() []string { - files, err := ioutil.ReadDir(store.Root) - if err != nil { - return []string{} - } - var layers []string - for _, st := range files { - if st.IsDir() { - layers = append(layers, path.Join(store.Root, st.Name())) - } - } - return layers -} - -func (store *LayerStore) Get(id string) string { - if !store.Exists(id) { - return "" - } - return store.layerPath(id) -} - -func (store *LayerStore) rootExists() (bool, error) { - if stat, err := os.Stat(store.Root); err != nil { - if os.IsNotExist(err) { - return false, nil - } - return false, err - } else if !stat.IsDir() { - return false, errors.New("Not a directory: " + store.Root) - } - return true, nil -} - -func (store *LayerStore) Init() error { - if exists, err := store.rootExists(); err != nil { - return err - } else if exists { - return nil - } - return os.Mkdir(store.Root, 0700) -} - -func (store *LayerStore) Mktemp() (string, error) { - tmpName := future.RandomId() - tmpPath := path.Join(store.Root, "tmp-"+tmpName) - if err := os.Mkdir(tmpPath, 0700); err != nil { - return "", err - } - return tmpPath, nil -} - -func (store *LayerStore) layerPath(id string) string { - return path.Join(store.Root, id) -} - -func (store *LayerStore) AddLayer(id string, archive Archive) (string, error) { - if _, err := os.Stat(store.layerPath(id)); err == nil { - return "", fmt.Errorf("Layer already exists: %v", id) - } - tmp, err := store.Mktemp() - defer os.RemoveAll(tmp) - if err != nil { - return "", fmt.Errorf("Mktemp failed: %s", err) - } - if err := Untar(archive, tmp); err != nil { - return "", err - } - layer := store.layerPath(id) - if !store.Exists(id) { - if err := os.Rename(tmp, layer); err != nil { - return "", fmt.Errorf("Could not rename temp dir to layer %s: %s", layer, err) - } - } - return layer, nil -} - -func (store *LayerStore) Exists(id string) bool { - st, err := os.Stat(store.layerPath(id)) - if err != nil { - return false - } - return st.IsDir() -} diff --git a/fs/layers_test.go b/fs/layers_test.go deleted file mode 100644 index ee7e59fd23..0000000000 --- a/fs/layers_test.go +++ /dev/null @@ -1,96 +0,0 @@ -package fs - -import ( - "archive/tar" - "bytes" - "io" - "io/ioutil" - "os" - "testing" -) - -func fakeTar() (io.Reader, error) { - content := []byte("Hello world!\n") - buf := new(bytes.Buffer) - tw := tar.NewWriter(buf) - for _, name := range []string{"/etc/postgres/postgres.conf", "/etc/passwd", "/var/log/postgres/postgres.conf"} { - hdr := new(tar.Header) - hdr.Size = int64(len(content)) - hdr.Name = name - if err := tw.WriteHeader(hdr); err != nil { - return nil, err - } - tw.Write([]byte(content)) - } - tw.Close() - return buf, nil -} - -func TestLayersInit(t *testing.T) { - store := tempStore(t) - defer os.RemoveAll(store.Root) - // Root should exist - if _, err := os.Stat(store.Root); err != nil { - t.Fatal(err) - } - // List() should be empty - if l := store.List(); len(l) != 0 { - t.Fatalf("List() should return %d, not %d", 0, len(l)) - } -} - -func TestAddLayer(t *testing.T) { - store := tempStore(t) - defer os.RemoveAll(store.Root) - layer, err := store.AddLayer("foo", testArchive(t)) - if err != nil { - t.Fatal(err) - } - // Layer path should exist - if _, err := os.Stat(layer); err != nil { - t.Fatal(err) - } - // List() should return 1 layer - if l := store.List(); len(l) != 1 { - t.Fatalf("List() should return %d elements, not %d", 1, len(l)) - } - // Get("foo") should return the correct layer - if foo := store.Get("foo"); foo != layer { - t.Fatalf("get(\"foo\") should return '%d', not '%d'", layer, foo) - } -} - -func TestAddLayerDuplicate(t *testing.T) { - store := tempStore(t) - defer os.RemoveAll(store.Root) - if _, err := store.AddLayer("foobar123", testArchive(t)); err != nil { - t.Fatal(err) - } - if _, err := store.AddLayer("foobar123", testArchive(t)); err == nil { - t.Fatalf("Creating duplicate layer should fail") - } -} - -/* - * HELPER FUNCTIONS - */ - -func tempStore(t *testing.T) *LayerStore { - tmp, err := ioutil.TempDir("", "docker-fs-layerstore-") - if err != nil { - t.Fatal(err) - } - store, err := NewLayerStore(tmp) - if err != nil { - t.Fatal(err) - } - return store -} - -func testArchive(t *testing.T) Archive { - archive, err := fakeTar() - if err != nil { - t.Fatal(err) - } - return archive -} diff --git a/fs/remove_test.go b/fs/remove_test.go deleted file mode 100644 index 81a234decf..0000000000 --- a/fs/remove_test.go +++ /dev/null @@ -1,222 +0,0 @@ -package fs - -import ( - "fmt" - "testing" -) - -func countImages(store *Store) int { - paths, err := store.Images() - if err != nil { - panic(err) - } - return len(paths) -} - -func TestRemoveInPath(t *testing.T) { - store, err := TempStore("test-remove-in-path") - if err != nil { - t.Fatal(err) - } - defer nuke(store) - archive, err := fakeTar() - if err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 0 { - t.Fatalf("Expected 0 images, %d found", c) - } - - // Test 10 create / Delete all - for i := 0; i < 10; i++ { - if _, err := store.Create(archive, nil, "foo", "Testing"); err != nil { - t.Fatal(err) - } - } - if c := countImages(store); c != 10 { - t.Fatalf("Expected 10 images, %d found", c) - } - if err := store.RemoveInPath("foo"); err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 0 { - t.Fatalf("Expected 0 images, %d found", c) - } - - // Test 10 create / Delete 1 - for i := 0; i < 10; i++ { - if _, err := store.Create(archive, nil, fmt.Sprintf("foo-%d", i), "Testing"); err != nil { - t.Fatal(err) - } - } - if c := countImages(store); c != 10 { - t.Fatalf("Expected 10 images, %d found", c) - } - if err := store.RemoveInPath("foo-0"); err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 9 { - t.Fatalf("Expected 9 images, %d found", c) - } - - // Delete failure - if err := store.RemoveInPath("Not_Foo"); err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 9 { - t.Fatalf("Expected 9 images, %d found", c) - } -} - -func TestRemove(t *testing.T) { - store, err := TempStore("test-remove") - if err != nil { - t.Fatal(err) - } - defer nuke(store) - archive, err := fakeTar() - if err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 0 { - t.Fatalf("Expected 0 images, %d found", c) - } - - // Test 1 create / 1 delete - img, err := store.Create(archive, nil, "foo", "Testing") - if err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 1 { - t.Fatalf("Expected 1 images, %d found", c) - } - if err := store.Remove(img); err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 0 { - t.Fatalf("Expected 0 images, %d found", c) - } - - // Test 2 create (same name) / 1 delete - img1, err := store.Create(archive, nil, "foo", "Testing") - if err != nil { - t.Fatal(err) - } - img2, err := store.Create(archive, nil, "foo", "Testing") - if err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 2 { - t.Fatalf("Expected 2 images, %d found", c) - } - if err := store.Remove(img1); err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 1 { - t.Fatalf("Expected 1 images, %d found", c) - } - - // Test delete wrong name - // Note: If we change orm and Delete of non existing return error, we will need to change this test - if err := store.Remove(&Image{Id: "Not_foo", store: img2.store}); err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 1 { - t.Fatalf("Expected 1 images, %d found", c) - } - - // Test delete last one - if err := store.Remove(img2); err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 0 { - t.Fatalf("Expected 0 images, %d found", c) - } -} - -func TestRemoveRegexp(t *testing.T) { - store, err := TempStore("test-remove-regexp") - if err != nil { - t.Fatal(err) - } - defer nuke(store) - archive, err := fakeTar() - if err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 0 { - t.Fatalf("Expected 0 images, %d found", c) - } - - // Test 10 create with different names / Delete all good regexp - for i := 0; i < 10; i++ { - if _, err := store.Create(archive, nil, fmt.Sprintf("foo-%d", i), "Testing"); err != nil { - t.Fatal(err) - } - } - if c := countImages(store); c != 10 { - t.Fatalf("Expected 10 images, %d found", c) - } - if err := store.RemoveRegexp("foo"); err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 0 { - t.Fatalf("Expected 0 images, %d found", c) - } - - // Test 10 create with different names / Delete all good regexp globing - for i := 0; i < 10; i++ { - if _, err := store.Create(archive, nil, fmt.Sprintf("foo-%d", i), "Testing"); err != nil { - t.Fatal(err) - } - } - if c := countImages(store); c != 10 { - t.Fatalf("Expected 10 images, %d found", c) - } - if err := store.RemoveRegexp("foo-*"); err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 0 { - t.Fatalf("Expected 0 images, %d found", c) - } - - // Test 10 create with different names / Delete all bad regexp - for i := 0; i < 10; i++ { - if _, err := store.Create(archive, nil, fmt.Sprintf("foo-%d", i), "Testing"); err != nil { - t.Fatal(err) - } - } - if c := countImages(store); c != 10 { - t.Fatalf("Expected 10 images, %d found", c) - } - if err := store.RemoveRegexp("oo-*"); err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 0 { - t.Fatalf("Expected 0 images, %d found", c) - } - - // Test 10 create with different names / Delete none strict regexp - for i := 0; i < 10; i++ { - if _, err := store.Create(archive, nil, fmt.Sprintf("foo-%d", i), "Testing"); err != nil { - t.Fatal(err) - } - } - if c := countImages(store); c != 10 { - t.Fatalf("Expected 10 images, %d found", c) - } - if err := store.RemoveRegexp("^oo-"); err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 10 { - t.Fatalf("Expected 10 images, %d found", c) - } - - // Test delete 2 - if err := store.RemoveRegexp("^foo-[1,2]$"); err != nil { - t.Fatal(err) - } - if c := countImages(store); c != 8 { - t.Fatalf("Expected 8 images, %d found", c) - } -} diff --git a/fs/store.go b/fs/store.go deleted file mode 100644 index d7fcb35421..0000000000 --- a/fs/store.go +++ /dev/null @@ -1,521 +0,0 @@ -package fs - -import ( - "database/sql" - "fmt" - "github.com/dotcloud/docker/future" - _ "github.com/mattn/go-sqlite3" - "github.com/shykes/gorp" //Forked to implement CreateTablesOpts - "io" - "io/ioutil" - "os" - "path" - "path/filepath" - "regexp" - "strings" - "syscall" - "time" -) - -type Store struct { - Root string - db *sql.DB - orm *gorp.DbMap - layers *LayerStore -} - -type Archive io.Reader - -func New(root string) (*Store, error) { - isNewStore := true - - if err := os.Mkdir(root, 0700); err != nil && !os.IsExist(err) { - return nil, err - } - db, err := sql.Open("sqlite3", path.Join(root, "db")) - if err != nil { - return nil, err - } - orm := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}} - orm.AddTableWithName(Image{}, "images").SetKeys(false, "Id") - orm.AddTableWithName(Path{}, "paths").SetKeys(false, "Path", "Image") - orm.AddTableWithName(Mountpoint{}, "mountpoints").SetKeys(false, "Root") - orm.AddTableWithName(Tag{}, "tags").SetKeys(false, "TagName") - if isNewStore { - if err := orm.CreateTablesOpts(true); err != nil { - return nil, err - } - } - - layers, err := NewLayerStore(path.Join(root, "layers")) - if err != nil { - return nil, err - } - return &Store{ - Root: root, - db: db, - orm: orm, - layers: layers, - }, nil -} - -func (store *Store) imageList(src []interface{}) []*Image { - var images []*Image - for _, i := range src { - img := i.(*Image) - img.store = store - images = append(images, img) - } - return images -} - -func (store *Store) Images() ([]*Image, error) { - images, err := store.orm.Select(Image{}, "select * from images") - if err != nil { - return nil, err - } - return store.imageList(images), nil -} - -func (store *Store) Paths() ([]string, error) { - var paths []string - rows, err := store.db.Query("select distinct Path from paths order by Path") - if err != nil { - return nil, err - } - for rows.Next() { - var path string - if err := rows.Scan(&path); err != nil { - return nil, err - } - paths = append(paths, path) - } - return paths, nil -} - -func (store *Store) RemoveInPath(pth string) error { - images, err := store.List(pth) - if err != nil { - return err - } - for _, img := range images { - if err = store.Remove(img); err != nil { - return err - } - } - return nil -} - -// DeleteMatch deletes all images whose name matches `pattern` -func (store *Store) RemoveRegexp(pattern string) error { - // Retrieve all the paths - paths, err := store.Paths() - if err != nil { - return err - } - // Check the pattern on each elements - for _, pth := range paths { - if match, err := regexp.MatchString(pattern, pth); err != nil { - return err - } else if match { - // If there is a match, remove it - if err := store.RemoveInPath(pth); err != nil { - return nil - } - } - } - return nil -} - -func (store *Store) Remove(img *Image) error { - _, err := store.orm.Delete(img) - return err -} - -func (store *Store) List(pth string) ([]*Image, error) { - pth = path.Clean(pth) - images, err := store.orm.Select(Image{}, "select images.* from images, paths where Path=? and paths.Image=images.Id order by images.Created desc", pth) - if err != nil { - return nil, err - } - return store.imageList(images), nil -} - -func (store *Store) Find(pth string) (*Image, error) { - pth = path.Clean(pth) - img, err := store.Get(pth) - if err != nil { - return nil, err - } else if img != nil { - return img, nil - } - - var q string - var args []interface{} - // FIXME: this breaks if the path contains a ':' - // If format is path:rev - if parts := strings.SplitN(pth, ":", 2); len(parts) == 2 { - q = "select Images.* from images, paths where Path=? and images.Id=? and paths.Image=images.Id" - args = []interface{}{parts[0], parts[1]} - // If format is path:rev - } else { - q = "select images.* from images, paths where Path=? and paths.Image=images.Id order by images.Created desc limit 1" - args = []interface{}{parts[0]} - } - images, err := store.orm.Select(Image{}, q, args...) - if err != nil { - return nil, err - } else if len(images) < 1 { - return nil, nil - } - img = images[0].(*Image) - img.store = store - return img, nil -} - -func (store *Store) Get(id string) (*Image, error) { - img, err := store.orm.Get(Image{}, id) - if img == nil { - return nil, err - } - res := img.(*Image) - res.store = store - return res, err -} - -func (store *Store) Create(layerData Archive, parent *Image, pth, comment string) (*Image, error) { - // FIXME: actually do something with the layer... - img := &Image{ - Id: future.RandomId(), - Comment: comment, - Created: time.Now().Unix(), - store: store, - } - if parent != nil { - img.Parent = parent.Id - } - // FIXME: Archive should contain compression info. For now we only support uncompressed. - err := store.Register(layerData, img, pth) - return img, err -} - -func (store *Store) Register(layerData Archive, img *Image, pth string) error { - img.store = store - _, err := store.layers.AddLayer(img.Id, layerData) - if err != nil { - return fmt.Errorf("Could not add layer: %s", err) - } - pathObj := &Path{ - Path: path.Clean(pth), - Image: img.Id, - } - trans, err := store.orm.Begin() - if err != nil { - return fmt.Errorf("Could not begin transaction: %s", err) - } - if err := trans.Insert(img); err != nil { - return fmt.Errorf("Could not insert image info: %s", err) - } - if err := trans.Insert(pathObj); err != nil { - return fmt.Errorf("Could not insert path info: %s", err) - } - if err := trans.Commit(); err != nil { - return fmt.Errorf("Could not commit transaction: %s", err) - } - return nil -} - -func (store *Store) Layers() []string { - return store.layers.List() -} - -type Image struct { - Id string - Parent string - Comment string - Created int64 - store *Store `db:"-"` -} - -func (image *Image) Copy(pth string) (*Image, error) { - if err := image.store.orm.Insert(&Path{Path: pth, Image: image.Id}); err != nil { - return nil, err - } - return image, nil -} - -type Mountpoint struct { - Image string - Root string - Rw string - Store *Store `db:"-"` -} - -func (image *Image) Mountpoint(root, rw string) (*Mountpoint, error) { - mountpoint := &Mountpoint{ - Root: path.Clean(root), - Rw: path.Clean(rw), - Image: image.Id, - Store: image.store, - } - if err := image.store.orm.Insert(mountpoint); err != nil { - return nil, err - } - return mountpoint, nil -} - -func (image *Image) layers() ([]string, error) { - var list []string - var err error - currentImg := image - for currentImg != nil { - if layer := image.store.layers.Get(currentImg.Id); layer != "" { - list = append(list, layer) - } else { - return list, fmt.Errorf("Layer not found for image %s", image.Id) - } - currentImg, err = currentImg.store.Get(currentImg.Parent) - if err != nil { - return list, fmt.Errorf("Error while getting parent image: %v", err) - } - } - if len(list) == 0 { - return nil, fmt.Errorf("No layer found for image %s\n", image.Id) - } - return list, nil -} - -func (image *Image) Mountpoints() ([]*Mountpoint, error) { - var mountpoints []*Mountpoint - res, err := image.store.orm.Select(Mountpoint{}, "select * from mountpoints where Image=?", image.Id) - if err != nil { - return nil, err - } - for _, mp := range res { - mountpoints = append(mountpoints, mp.(*Mountpoint)) - } - return mountpoints, nil -} - -func (image *Image) Mount(root, rw string) (*Mountpoint, error) { - var mountpoint *Mountpoint - if mp, err := image.store.FetchMountpoint(root, rw); err != nil { - return nil, err - } else if mp == nil { - mountpoint, err = image.Mountpoint(root, rw) - if err != nil { - return nil, fmt.Errorf("Could not create mountpoint: %s", err) - } else if mountpoint == nil { - return nil, fmt.Errorf("No mountpoint created") - } - } else { - mountpoint = mp - } - - if err := mountpoint.createFolders(); err != nil { - return nil, err - } - - // FIXME: Now mount the layers - rwBranch := fmt.Sprintf("%v=rw", mountpoint.Rw) - roBranches := "" - layers, err := image.layers() - if err != nil { - return nil, err - } - for _, layer := range layers { - roBranches += fmt.Sprintf("%v=ro:", layer) - } - branches := fmt.Sprintf("br:%v:%v", rwBranch, roBranches) - if err := mount("none", mountpoint.Root, "aufs", 0, branches); err != nil { - return mountpoint, err - } - if !mountpoint.Mounted() { - return mountpoint, fmt.Errorf("Mount failed") - } - - // FIXME: Create tests for deletion - // FIXME: move this part to change.go, maybe refactor - // fs.Change() to avoid the fake mountpoint - // Retrieve the changeset from the parent and apply it to the container - // - Retrieve the changes - changes, err := image.store.Changes(&Mountpoint{ - Image: image.Id, - Root: layers[0], - Rw: layers[0], - Store: image.store}) - if err != nil { - return nil, err - } - // Iterate on changes - for _, c := range changes { - // If there is a delete - if c.Kind == ChangeDelete { - // Make sure the directory exists - file_path, file_name := path.Dir(c.Path), path.Base(c.Path) - if err := os.MkdirAll(path.Join(mountpoint.Rw, file_path), 0755); err != nil { - return nil, err - } - // And create the whiteout (we just need to create empty file, discard the return) - if _, err := os.Create(path.Join(path.Join(mountpoint.Rw, file_path), - ".wh."+path.Base(file_name))); err != nil { - return nil, err - } - } - } - return mountpoint, nil -} - -func (mp *Mountpoint) EnsureMounted() error { - if mp.Mounted() { - return nil - } - img, err := mp.Store.Get(mp.Image) - if err != nil { - return err - } - - _, err = img.Mount(mp.Root, mp.Rw) - return err -} - -func (mp *Mountpoint) createFolders() error { - if err := os.Mkdir(mp.Root, 0755); err != nil && !os.IsExist(err) { - return err - } - if err := os.Mkdir(mp.Rw, 0755); err != nil && !os.IsExist(err) { - return err - } - return nil -} - -func (mp *Mountpoint) Mounted() bool { - root, err := os.Stat(mp.Root) - if err != nil { - if os.IsNotExist(err) { - return false - } - panic(err) - } - parent, err := os.Stat(filepath.Join(mp.Root, "..")) - if err != nil { - panic(err) - } - - rootSt := root.Sys().(*syscall.Stat_t) - parentSt := parent.Sys().(*syscall.Stat_t) - return rootSt.Dev != parentSt.Dev -} - -func (mp *Mountpoint) Umount() error { - if !mp.Mounted() { - return fmt.Errorf("Mountpoint doesn't seem to be mounted") - } - if err := syscall.Unmount(mp.Root, 0); err != nil { - return fmt.Errorf("Unmount syscall failed: %v", err) - } - if mp.Mounted() { - return fmt.Errorf("Umount: Filesystem still mounted after calling umount(%v)", mp.Root) - } - // Even though we just unmounted the filesystem, AUFS will prevent deleting the mntpoint - // for some time. We'll just keep retrying until it succeeds. - for retries := 0; retries < 1000; retries++ { - err := os.Remove(mp.Root) - if err == nil { - // rm mntpoint succeeded - return nil - } - if os.IsNotExist(err) { - // mntpoint doesn't exist anymore. Success. - return nil - } - // fmt.Printf("(%v) Remove %v returned: %v\n", retries, mp.Root, err) - time.Sleep(10 * time.Millisecond) - } - return fmt.Errorf("Umount: Failed to umount %v", mp.Root) - -} - -func (mp *Mountpoint) Deregister() error { - if mp.Mounted() { - return fmt.Errorf("Mountpoint is currently mounted, can't deregister") - } - - _, err := mp.Store.orm.Delete(mp) - return err -} - -func (store *Store) FetchMountpoint(root, rw string) (*Mountpoint, error) { - res, err := store.orm.Select(Mountpoint{}, "select * from mountpoints where Root=? and Rw=?", root, rw) - if err != nil { - return nil, err - } else if len(res) < 1 || res[0] == nil { - return nil, nil - } - - mp := res[0].(*Mountpoint) - mp.Store = store - return mp, nil -} - -// OpenFile opens the named file for reading. -func (mp *Mountpoint) OpenFile(path string, flag int, perm os.FileMode) (*os.File, error) { - if err := mp.EnsureMounted(); err != nil { - return nil, err - } - return os.OpenFile(filepath.Join(mp.Root, path), flag, perm) -} - -// ReadDir reads the directory named by dirname, relative to the Mountpoint's root, -// and returns a list of sorted directory entries -func (mp *Mountpoint) ReadDir(dirname string) ([]os.FileInfo, error) { - if err := mp.EnsureMounted(); err != nil { - return nil, err - } - return ioutil.ReadDir(filepath.Join(mp.Root, dirname)) -} - -func (store *Store) AddTag(imageId, tagName string) error { - if image, err := store.Get(imageId); err != nil { - return err - } else if image == nil { - return fmt.Errorf("No image with ID %s", imageId) - } - - err2 := store.orm.Insert(&Tag{ - TagName: tagName, - Image: imageId, - }) - - return err2 -} - -func (store *Store) GetByTag(tagName string) (*Image, error) { - res, err := store.orm.Get(Tag{}, tagName) - if err != nil { - return nil, err - } else if res == nil { - return nil, fmt.Errorf("No image associated to tag \"%s\"", tagName) - } - - tag := res.(*Tag) - - img, err2 := store.Get(tag.Image) - if err2 != nil { - return nil, err2 - } else if img == nil { - return nil, fmt.Errorf("Tag was found but image seems to be inexistent.") - } - - return img, nil -} - -type Path struct { - Path string - Image string -} - -type Tag struct { - TagName string - Image string -} diff --git a/fs/store_test.go b/fs/store_test.go deleted file mode 100644 index 8413ea0812..0000000000 --- a/fs/store_test.go +++ /dev/null @@ -1,277 +0,0 @@ -package fs - -import ( - "fmt" - "github.com/dotcloud/docker/future" - "io/ioutil" - "os" - "testing" - "time" -) - -func TestInit(t *testing.T) { - store, err := TempStore("testinit") - if err != nil { - t.Fatal(err) - } - defer nuke(store) - paths, err := store.Paths() - if err != nil { - t.Fatal(err) - } - if l := len(paths); l != 0 { - t.Fatal("Fresh store should be empty after init (len=%d)", l) - } -} - -// FIXME: Do more extensive tests (ex: create multiple, delete, recreate; -// create multiple, check the amount of images and paths, etc..) -func TestCreate(t *testing.T) { - store, err := TempStore("testcreate") - if err != nil { - t.Fatal(err) - } - defer nuke(store) - archive, err := fakeTar() - if err != nil { - t.Fatal(err) - } - image, err := store.Create(archive, nil, "foo", "Testing") - if err != nil { - t.Fatal(err) - } - if images, err := store.Images(); err != nil { - t.Fatal(err) - } else if l := len(images); l != 1 { - t.Fatalf("Wrong number of images. Should be %d, not %d", 1, l) - } - if images, err := store.List("foo"); err != nil { - t.Fatal(err) - } else if l := len(images); l != 1 { - t.Fatalf("Path foo has wrong number of images (should be %d, not %d)", 1, l) - } else if images[0].Id != image.Id { - t.Fatalf("Imported image should be listed at path foo (%s != %s)", images[0], image) - } -} - -func TestRegister(t *testing.T) { - store, err := TempStore("testregister") - if err != nil { - t.Fatal(err) - } - defer nuke(store) - archive, err := fakeTar() - if err != nil { - t.Fatal(err) - } - image := &Image{ - Id: future.RandomId(), - Comment: "testing", - Created: time.Now().Unix(), - store: store, - } - err = store.Register(archive, image, "foo") - if err != nil { - t.Fatal(err) - } - if images, err := store.Images(); err != nil { - t.Fatal(err) - } else if l := len(images); l != 1 { - t.Fatalf("Wrong number of images. Should be %d, not %d", 1, l) - } - if images, err := store.List("foo"); err != nil { - t.Fatal(err) - } else if l := len(images); l != 1 { - t.Fatalf("Path foo has wrong number of images (should be %d, not %d)", 1, l) - } else if images[0].Id != image.Id { - t.Fatalf("Imported image should be listed at path foo (%s != %s)", images[0], image) - } -} - -func TestTag(t *testing.T) { - store, err := TempStore("testtag") - if err != nil { - t.Fatal(err) - } - defer nuke(store) - archive, err := fakeTar() - if err != nil { - t.Fatal(err) - } - image, err := store.Create(archive, nil, "foo", "Testing") - if err != nil { - t.Fatal(err) - } - if images, err := store.Images(); err != nil { - t.Fatal(err) - } else if l := len(images); l != 1 { - t.Fatalf("Wrong number of images. Should be %d, not %d", 1, l) - } - - if err := store.AddTag(image.Id, "baz"); err != nil { - t.Fatalf("Error while adding a tag to created image: %s", err) - } - - if taggedImage, err := store.GetByTag("baz"); err != nil { - t.Fatalf("Error while trying to retrieve image for tag 'baz': %s", err) - } else if taggedImage.Id != image.Id { - t.Fatalf("Expected to retrieve image %s but found %s instead", image.Id, taggedImage.Id) - } -} - -// Copy an image to a new path -func TestCopyNewPath(t *testing.T) { - store, err := TempStore("testcopynewpath") - if err != nil { - t.Fatal(err) - } - defer nuke(store) - archive, err := fakeTar() - if err != nil { - t.Fatal(err) - } - src, err := store.Create(archive, nil, "foo", "Testing") - if err != nil { - t.Fatal(err) - } - dst, err := src.Copy("bar") - if err != nil { - t.Fatal(err) - } - // ID should be the same - if src.Id != dst.Id { - t.Fatal("Different IDs") - } - // Check number of images at source path - if images, err := store.List("foo"); err != nil { - t.Fatal(err) - } else if l := len(images); l != 1 { - t.Fatal("Wrong number of images at source path (should be %d, not %d)", 1, l) - } - // Check number of images at destination path - if images, err := store.List("bar"); err != nil { - t.Fatal(err) - } else if l := len(images); l != 1 { - t.Fatal("Wrong number of images at destination path (should be %d, not %d)", 1, l) - } - if err := healthCheck(store); err != nil { - t.Fatal(err) - } -} - -// Copying an image to the same path twice should fail -func TestCopySameName(t *testing.T) { - store, err := TempStore("testcopysamename") - if err != nil { - t.Fatal(err) - } - defer nuke(store) - archive, err := fakeTar() - if err != nil { - t.Fatal(err) - } - src, err := store.Create(archive, nil, "foo", "Testing") - if err != nil { - t.Fatal(err) - } - _, err = src.Copy("foo") - if err == nil { - t.Fatal("Copying an image to the same patch twice should fail.") - } -} - -func TestMountPoint(t *testing.T) { - store, err := TempStore("test-mountpoint") - if err != nil { - t.Fatal(err) - } - defer nuke(store) - archive, err := fakeTar() - if err != nil { - t.Fatal(err) - } - image, err := store.Create(archive, nil, "foo", "Testing") - if err != nil { - t.Fatal(err) - } - mountpoint, err := image.Mountpoint("/tmp/a", "/tmp/b") - if err != nil { - t.Fatal(err) - } - if mountpoint.Root != "/tmp/a" { - t.Fatal("Wrong mountpoint root (should be %s, not %s)", "/tmp/a", mountpoint.Root) - } - if mountpoint.Rw != "/tmp/b" { - t.Fatal("Wrong mountpoint root (should be %s, not %s)", "/tmp/b", mountpoint.Rw) - } -} - -func TestMountpointDuplicateRoot(t *testing.T) { - store, err := TempStore("test-mountpoint") - if err != nil { - t.Fatal(err) - } - defer nuke(store) - archive, err := fakeTar() - if err != nil { - t.Fatal(err) - } - image, err := store.Create(archive, nil, "foo", "Testing") - if err != nil { - t.Fatal(err) - } - _, err = image.Mountpoint("/tmp/a", "/tmp/b") - if err != nil { - t.Fatal(err) - } - if _, err = image.Mountpoint("/tmp/a", "/tmp/foobar"); err == nil { - t.Fatal("Duplicate mountpoint root should fail") - } -} - -func TempStore(prefix string) (*Store, error) { - dir, err := ioutil.TempDir("", "docker-fs-test-"+prefix) - if err != nil { - return nil, err - } - return New(dir) -} - -func nuke(store *Store) error { - return os.RemoveAll(store.Root) -} - -// Look for inconsistencies in a store. -func healthCheck(store *Store) error { - parents := make(map[string]bool) - paths, err := store.Paths() - if err != nil { - return err - } - for _, path := range paths { - images, err := store.List(path) - if err != nil { - return err - } - IDs := make(map[string]bool) // All IDs for this path - for _, img := range images { - // Check for duplicate IDs per path - if _, exists := IDs[img.Id]; exists { - return fmt.Errorf("Duplicate ID: %s", img.Id) - } else { - IDs[img.Id] = true - } - // Store parent for 2nd pass - if parent := img.Parent; parent != "" { - parents[parent] = true - } - } - } - // Check non-existing parents - for parent := range parents { - if _, exists := parents[parent]; !exists { - return fmt.Errorf("Reference to non-registered parent: %s", parent) - } - } - return nil -} From 6f6eaca861fec53a6d21d420b813041dc93e8b93 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 20 Mar 2013 22:16:02 -0700 Subject: [PATCH 08/80] Removed mount code from container. It belongs in graph --- mount_darwin.go | 7 ------- mount_linux.go | 7 ------- 2 files changed, 14 deletions(-) delete mode 100644 mount_darwin.go delete mode 100644 mount_linux.go diff --git a/mount_darwin.go b/mount_darwin.go deleted file mode 100644 index aeac78cda5..0000000000 --- a/mount_darwin.go +++ /dev/null @@ -1,7 +0,0 @@ -package docker - -import "errors" - -func mount(source string, target string, fstype string, flags uintptr, data string) (err error) { - return errors.New("mount is not implemented on darwin") -} diff --git a/mount_linux.go b/mount_linux.go deleted file mode 100644 index 0efb253003..0000000000 --- a/mount_linux.go +++ /dev/null @@ -1,7 +0,0 @@ -package docker - -import "syscall" - -func mount(source string, target string, fstype string, flags uintptr, data string) (err error) { - return syscall.Mount(source, target, fstype, flags, data) -} From 75c866d6a3aa92d5d2a4e6948fff00f90b36e872 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 20 Mar 2013 22:41:03 -0700 Subject: [PATCH 09/80] Unmount() and Mounted(): utility functions to unmount a mountpoint and check if it's mounted, respectively --- graph/image.go | 4 ++-- graph/mount.go | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 graph/mount.go diff --git a/graph/image.go b/graph/image.go index 9cb72a9632..37ccc41a8f 100644 --- a/graph/image.go +++ b/graph/image.go @@ -92,9 +92,9 @@ func MountAUFS(ro []string, rw string, target string) error { } func (image *Image) Mount(root, rw string) error { - if isMounted, err := IsMounted(root); err != nil { + if mounted, err := Mounted(root); err != nil { return err - } else if isMounted { + } else if mounted { return fmt.Errorf("%s is already mounted", root) } layers, err := image.layers() diff --git a/graph/mount.go b/graph/mount.go new file mode 100644 index 0000000000..ebf8485285 --- /dev/null +++ b/graph/mount.go @@ -0,0 +1,48 @@ +package graph + +import ( + "fmt" + "os" + "path/filepath" + "syscall" + "time" +) + +func Unmount(target string) error { + if err := syscall.Unmount(target, 0); err != nil { + return err + } + // Even though we just unmounted the filesystem, AUFS will prevent deleting the mntpoint + // for some time. We'll just keep retrying until it succeeds. + for retries := 0; retries < 1000; retries++ { + err := os.Remove(target) + if err == nil { + // rm mntpoint succeeded + return nil + } + if os.IsNotExist(err) { + // mntpoint doesn't exist anymore. Success. + return nil + } + // fmt.Printf("(%v) Remove %v returned: %v\n", retries, target, err) + time.Sleep(10 * time.Millisecond) + } + return fmt.Errorf("Umount: Failed to umount %v", target) +} + +func Mounted(mountpoint string) (bool, error) { + mntpoint, err := os.Stat(mountpoint) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + parent, err := os.Stat(filepath.Join(mountpoint, "..")) + if err != nil { + return false, err + } + mntpointSt := mntpoint.Sys().(*syscall.Stat_t) + parentSt := parent.Sys().(*syscall.Stat_t) + return mntpointSt.Dev != parentSt.Dev, nil +} From 4d9c324495babc175db278fdbe39d291eaf25726 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 20 Mar 2013 22:41:31 -0700 Subject: [PATCH 10/80] Removed extra import --- graph/image.go | 1 - 1 file changed, 1 deletion(-) diff --git a/graph/image.go b/graph/image.go index 37ccc41a8f..98bdc8874b 100644 --- a/graph/image.go +++ b/graph/image.go @@ -8,7 +8,6 @@ import ( "os" "path" "strings" - "syscall" "time" ) From 3eff62394bf11f579552114ce6157675acbd6d25 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 20 Mar 2013 22:42:08 -0700 Subject: [PATCH 11/80] Removed dependency on the fake package in graph unit tests --- graph/graph_test.go | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/graph/graph_test.go b/graph/graph_test.go index 0fbebb8948..d634ea85d5 100644 --- a/graph/graph_test.go +++ b/graph/graph_test.go @@ -1,7 +1,9 @@ package graph import ( - "github.com/dotcloud/docker/fake" + "archive/tar" + "bytes" + "io" "io/ioutil" "os" "path" @@ -29,7 +31,7 @@ func TestInit(t *testing.T) { func TestCreate(t *testing.T) { graph := tempGraph(t) defer os.RemoveAll(graph.Root) - archive, err := fake.FakeTar() + archive, err := fakeTar() if err != nil { t.Fatal(err) } @@ -53,7 +55,7 @@ func TestCreate(t *testing.T) { func TestRegister(t *testing.T) { graph := tempGraph(t) defer os.RemoveAll(graph.Root) - archive, err := fake.FakeTar() + archive, err := fakeTar() if err != nil { t.Fatal(err) } @@ -86,7 +88,7 @@ func TestRegister(t *testing.T) { func TestMount(t *testing.T) { graph := tempGraph(t) defer os.RemoveAll(graph.Root) - archive, err := fake.FakeTar() + archive, err := fakeTar() if err != nil { t.Fatal(err) } @@ -121,7 +123,7 @@ func TestMount(t *testing.T) { func TestDelete(t *testing.T) { graph := tempGraph(t) defer os.RemoveAll(graph.Root) - archive, err := fake.FakeTar() + archive, err := fakeTar() if err != nil { t.Fatal(err) } @@ -183,9 +185,26 @@ func tempGraph(t *testing.T) *Graph { } func testArchive(t *testing.T) Archive { - archive, err := fake.FakeTar() + archive, err := fakeTar() if err != nil { t.Fatal(err) } return archive } + +func fakeTar() (io.Reader, error) { + content := []byte("Hello world!\n") + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + for _, name := range []string{"/etc/postgres/postgres.conf", "/etc/passwd", "/var/log/postgres/postgres.conf"} { + hdr := new(tar.Header) + hdr.Size = int64(len(content)) + hdr.Name = name + if err := tw.WriteHeader(hdr); err != nil { + return nil, err + } + tw.Write([]byte(content)) + } + tw.Close() + return buf, nil +} From 9d82bab041370cd26e78fe85ee5435c8b2429b59 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 20 Mar 2013 22:42:50 -0700 Subject: [PATCH 12/80] Removed anal warning from 'go vet' --- commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands.go b/commands.go index 0e0ea2c809..51b814b7af 100644 --- a/commands.go +++ b/commands.go @@ -64,7 +64,7 @@ func (srv *Server) Help() string { {"wait", "Block until a container stops, then print its exit code"}, {"write", "Write the contents of standard input to a container's file"}, } { - help += fmt.Sprintf(" %-10.10s%s\n", cmd...) + help += fmt.Sprintf(" %-10.10s%s\n", cmd[0], cmd[1]) } return help } From 34023558f5a37e441e39f003cdc4ad6736f38c5a Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 20 Mar 2013 22:48:52 -0700 Subject: [PATCH 13/80] Pruned more semi-useless commands: 'docker cat', 'docker cp', 'docker ls', 'docker write'. Removed outdated commands from help message --- commands.go | 94 ----------------------------------------------------- 1 file changed, 94 deletions(-) diff --git a/commands.go b/commands.go index 51b814b7af..e558c5f6b8 100644 --- a/commands.go +++ b/commands.go @@ -37,10 +37,7 @@ func (srv *Server) Help() string { {"ps", "Display a list of containers"}, {"import", "Create a new filesystem image from the contents of a tarball"}, {"attach", "Attach to a running container"}, - {"cat", "Write the contents of a container's file to standard output"}, {"commit", "Create a new image from a container's changes"}, - {"cp", "Create a copy of IMAGE and call it NAME"}, - {"debug", "(debug only) (No documentation available)"}, {"diff", "Inspect changes on a container's filesystem"}, {"images", "List images"}, {"info", "Display system-wide information"}, @@ -48,8 +45,6 @@ func (srv *Server) Help() string { {"kill", "Kill a running container"}, {"login", "Register or Login to the docker registry server"}, {"logs", "Fetch the logs of a container"}, - {"ls", "List the contents of a container's directory"}, - {"mirror", "(debug only) (No documentation available)"}, {"port", "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT"}, {"ps", "List containers"}, {"restart", "Restart a running container"}, @@ -59,10 +54,8 @@ func (srv *Server) Help() string { {"start", "Start a stopped container"}, {"stop", "Stop a running container"}, {"tar", "Stream the contents of a container as a tar archive"}, - {"umount", "(debug only) Mount a container's filesystem"}, {"version", "Show the docker version information"}, {"wait", "Block until a container stops, then print its exit code"}, - {"write", "Write the contents of standard input to a container's file"}, } { help += fmt.Sprintf(" %-10.10s%s\n", cmd[0], cmd[1]) } @@ -232,28 +225,6 @@ func (srv *Server) CmdStart(stdin io.ReadCloser, stdout io.Writer, args ...strin return nil } -func (srv *Server) CmdUmount(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "umount", "[OPTIONS] NAME", "umount a container's filesystem (debug only)") - if err := cmd.Parse(args); err != nil { - return nil - } - if cmd.NArg() < 1 { - cmd.Usage() - return nil - } - for _, name := range cmd.Args() { - if container := srv.containers.Get(name); container != nil { - if err := container.Mountpoint.Umount(); err != nil { - return err - } - fmt.Fprintln(stdout, container.Id) - } else { - return errors.New("No such container: " + name) - } - } - return nil -} - func (srv *Server) CmdMount(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "umount", "[OPTIONS] NAME", "mount a container's filesystem (debug only)") if err := cmd.Parse(args); err != nil { @@ -276,71 +247,6 @@ func (srv *Server) CmdMount(stdin io.ReadCloser, stdout io.Writer, args ...strin return nil } -func (srv *Server) CmdCat(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "cat", "[OPTIONS] CONTAINER PATH", "write the contents of a container's file to standard output") - if err := cmd.Parse(args); err != nil { - return nil - } - if cmd.NArg() < 2 { - cmd.Usage() - return nil - } - name, path := cmd.Arg(0), cmd.Arg(1) - if container := srv.containers.Get(name); container != nil { - if f, err := container.Mountpoint.OpenFile(path, os.O_RDONLY, 0); err != nil { - return err - } else if _, err := io.Copy(stdout, f); err != nil { - return err - } - return nil - } - return errors.New("No such container: " + name) -} - -func (srv *Server) CmdWrite(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "write", "[OPTIONS] CONTAINER PATH", "write the contents of standard input to a container's file") - if err := cmd.Parse(args); err != nil { - return nil - } - if cmd.NArg() < 2 { - cmd.Usage() - return nil - } - name, path := cmd.Arg(0), cmd.Arg(1) - if container := srv.containers.Get(name); container != nil { - if f, err := container.Mountpoint.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0600); err != nil { - return err - } else if _, err := io.Copy(f, stdin); err != nil { - return err - } - return nil - } - return errors.New("No such container: " + name) -} - -func (srv *Server) CmdLs(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "ls", "[OPTIONS] CONTAINER PATH", "List the contents of a container's directory") - if err := cmd.Parse(args); err != nil { - return nil - } - if cmd.NArg() < 2 { - cmd.Usage() - return nil - } - name, path := cmd.Arg(0), cmd.Arg(1) - if container := srv.containers.Get(name); container != nil { - if files, err := container.Mountpoint.ReadDir(path); err != nil { - return err - } else { - for _, f := range files { - fmt.Fprintln(stdout, f.Name()) - } - } - return nil - } - return errors.New("No such container: " + name) -} - func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "inspect", "[OPTIONS] CONTAINER", "Return low-level information on a container") if err := cmd.Parse(args); err != nil { From 89a140fb75b328b3c2869464a5d9880082966b55 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 00:21:03 -0700 Subject: [PATCH 14/80] Removed redundant mount_test.go (graph_test.go already tests the mount ability) --- mount_test.go | 134 -------------------------------------------------- 1 file changed, 134 deletions(-) delete mode 100644 mount_test.go diff --git a/mount_test.go b/mount_test.go deleted file mode 100644 index 56ac7eb417..0000000000 --- a/mount_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package docker - -import ( - "archive/tar" - "bytes" - "fmt" - "github.com/dotcloud/docker/fs" - "io" - "io/ioutil" - "os" - "testing" -) - -func fakeTar() (io.Reader, error) { - content := []byte("Hello world!\n") - buf := new(bytes.Buffer) - tw := tar.NewWriter(buf) - for _, name := range []string{"/etc/postgres/postgres.conf", "/etc/passwd", "/var/log/postgres/postgres.conf"} { - hdr := new(tar.Header) - hdr.Size = int64(len(content)) - hdr.Name = name - if err := tw.WriteHeader(hdr); err != nil { - return nil, err - } - tw.Write([]byte(content)) - } - tw.Close() - return buf, nil -} - -// Look for inconsistencies in a store. -func healthCheck(store *fs.Store) error { - parents := make(map[string]bool) - paths, err := store.Paths() - if err != nil { - return err - } - for _, path := range paths { - images, err := store.List(path) - if err != nil { - return err - } - IDs := make(map[string]bool) // All IDs for this path - for _, img := range images { - // Check for duplicate IDs per path - if _, exists := IDs[img.Id]; exists { - return fmt.Errorf("Duplicate ID: %s", img.Id) - } else { - IDs[img.Id] = true - } - // Store parent for 2nd pass - if parent := img.Parent; parent != "" { - parents[parent] = true - } - } - } - // Check non-existing parents - for parent := range parents { - if _, exists := parents[parent]; !exists { - return fmt.Errorf("Reference to non-registered parent: %s", parent) - } - } - return nil -} - -// Note: This test is in the docker package because he needs to be run as root -func TestMount(t *testing.T) { - dir, err := ioutil.TempDir("", "docker-fs-test-mount") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - store, err := fs.New(dir) - if err != nil { - t.Fatal(err) - } - - archive, err := fakeTar() - if err != nil { - t.Fatal(err) - } - - image, err := store.Create(archive, nil, "foo", "Testing") - if err != nil { - t.Fatal(err) - } - - // Create mount targets - root, err := ioutil.TempDir("", "docker-fs-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(root) - - rw, err := ioutil.TempDir("", "docker-fs-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(rw) - - mountpoint, err := image.Mount(root, rw) - if err != nil { - t.Fatal(err) - } - defer mountpoint.Umount() - // Mountpoint should be marked as mounted - if !mountpoint.Mounted() { - t.Fatal("Mountpoint not mounted") - } - // There should be one mountpoint registered - if mps, err := image.Mountpoints(); err != nil { - t.Fatal(err) - } else if len(mps) != 1 { - t.Fatal("Wrong number of mountpoints registered (should be %d, not %d)", 1, len(mps)) - } - // Unmounting should work - if err := mountpoint.Umount(); err != nil { - t.Fatal(err) - } - // De-registering should work - if err := mountpoint.Deregister(); err != nil { - t.Fatal(err) - } - if mps, err := image.Mountpoints(); err != nil { - t.Fatal(err) - } else if len(mps) != 0 { - t.Fatal("Wrong number of mountpoints registered (should be %d, not %d)", 0, len(mps)) - } - // General health check - if err := healthCheck(store); err != nil { - t.Fatal(err) - } -} From 84e8c4aa1df8f6f53d1c688a0e2bfc5b4d46adef Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 00:21:32 -0700 Subject: [PATCH 15/80] Fixed a bug in graph.Graph.Get() --- graph/graph.go | 1 + 1 file changed, 1 insertion(+) diff --git a/graph/graph.go b/graph/graph.go index 7e5e884c8e..7e0bc810cd 100644 --- a/graph/graph.go +++ b/graph/graph.go @@ -42,6 +42,7 @@ func (graph *Graph) Get(id string) (*Image, error) { if img.Id != id { return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.Id) } + img.graph = graph return img, nil } From 7c57a4cfc086b4af68ed9ec7c1b03b934db61f0c Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 00:25:00 -0700 Subject: [PATCH 16/80] Simplified the core container API, ported it to the new graph. Some features are missing eg. image 'paths' and tags --- commands.go | 218 +++++++++++++--------------------- container.go | 296 ++++++++++++++++++---------------------------- container_test.go | 84 +++++-------- docker.go | 134 +++++++++++++++++---- docker_test.go | 51 ++++---- lxc_template.go | 2 +- state.go | 8 -- 7 files changed, 363 insertions(+), 430 deletions(-) diff --git a/commands.go b/commands.go index e558c5f6b8..79619ca44d 100644 --- a/commands.go +++ b/commands.go @@ -1,20 +1,16 @@ package docker import ( - "bufio" "bytes" "encoding/json" "errors" "fmt" "github.com/dotcloud/docker/auth" - "github.com/dotcloud/docker/fs" "github.com/dotcloud/docker/future" "github.com/dotcloud/docker/rcli" "io" - "io/ioutil" "net/http" "net/url" - "os" "path" "strconv" "strings" @@ -137,7 +133,7 @@ func (srv *Server) CmdVersion(stdin io.ReadCloser, stdout io.Writer, args ...str // 'docker info': display system-wide information. func (srv *Server) CmdInfo(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - images, _ := srv.images.Images() + images, _ := srv.containers.graph.All() var imgcount int if images == nil { imgcount = 0 @@ -236,7 +232,7 @@ func (srv *Server) CmdMount(stdin io.ReadCloser, stdout io.Writer, args ...strin } for _, name := range cmd.Args() { if container := srv.containers.Get(name); container != nil { - if err := container.Mountpoint.EnsureMounted(); err != nil { + if err := container.EnsureMounted(); err != nil { return err } fmt.Fprintln(stdout, container.Id) @@ -260,7 +256,7 @@ func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...str var obj interface{} if container := srv.containers.Get(name); container != nil { obj = container - } else if image, err := srv.images.Find(name); err != nil { + } else if image, err := srv.containers.graph.Get(name); err != nil { return err } else if image != nil { obj = image @@ -310,27 +306,12 @@ func (srv *Server) CmdPort(stdin io.ReadCloser, stdout io.Writer, args ...string // 'docker rmi NAME' removes all images with the name NAME func (srv *Server) CmdRmi(stdin io.ReadCloser, stdout io.Writer, args ...string) (err error) { cmd := rcli.Subcmd(stdout, "rmimage", "[OPTIONS] IMAGE", "Remove an image") - fl_all := cmd.Bool("a", false, "Use IMAGE as a path and remove ALL images in this path") - fl_regexp := cmd.Bool("r", false, "Use IMAGE as a regular expression instead of an exact name") if cmd.Parse(args) != nil || cmd.NArg() < 1 { cmd.Usage() return nil } for _, name := range cmd.Args() { - if *fl_regexp { - err = srv.images.RemoveRegexp(name) - } else if *fl_all { - err = srv.images.RemoveInPath(name) - } else { - if image, err1 := srv.images.Find(name); err1 != nil { - err = err1 - } else if err1 == nil && image == nil { - err = fmt.Errorf("No such image: %s", name) - } else { - err = srv.images.Remove(image) - } - } - if err != nil { + if err := srv.containers.graph.Delete(name); err != nil { return err } } @@ -409,7 +390,7 @@ func (srv *Server) CmdImport(stdin io.ReadCloser, stdout io.Writer, args ...stri archive = future.ProgressReader(resp.Body, int(resp.ContentLength), stdout) } fmt.Fprintf(stdout, "Unpacking to %s\n", name) - img, err := srv.images.Create(archive, nil, name, "") + img, err := srv.containers.graph.Create(archive, "", "") if err != nil { return err } @@ -419,7 +400,7 @@ func (srv *Server) CmdImport(stdin io.ReadCloser, stdout io.Writer, args ...stri func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "images", "[OPTIONS] [NAME]", "List images") - limit := cmd.Int("l", 0, "Only show the N most recent versions of each image") + //limit := cmd.Int("l", 0, "Only show the N most recent versions of each image") quiet := cmd.Bool("q", false, "only show numeric IDs") if err := cmd.Parse(args); err != nil { return nil @@ -428,51 +409,65 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri cmd.Usage() return nil } - var nameFilter string - if cmd.NArg() == 1 { - nameFilter = cmd.Arg(0) - } + /* + var nameFilter string + if cmd.NArg() == 1 { + nameFilter = cmd.Arg(0) + } + */ w := tabwriter.NewWriter(stdout, 20, 1, 3, ' ', 0) if !*quiet { fmt.Fprintf(w, "NAME\tID\tCREATED\tPARENT\n") } - paths, err := srv.images.Paths() - if err != nil { - return err - } - for _, name := range paths { - if nameFilter != "" && nameFilter != name { - continue - } - ids, err := srv.images.List(name) + if *quiet { + images, err := srv.containers.graph.All() if err != nil { return err } - for idx, img := range ids { - if *limit > 0 && idx >= *limit { - break - } - if !*quiet { - for idx, field := range []string{ - /* NAME */ name, - /* ID */ img.Id, - /* CREATED */ future.HumanDuration(time.Now().Sub(time.Unix(img.Created, 0))) + " ago", - /* PARENT */ img.Parent, - } { - if idx == 0 { - w.Write([]byte(field)) - } else { - w.Write([]byte("\t" + field)) - } - } - w.Write([]byte{'\n'}) - } else { - stdout.Write([]byte(img.Id + "\n")) - } + for _, image := range images { + fmt.Fprintln(stdout, image.Id) } - } - if !*quiet { - w.Flush() + } else { + // FIXME: + // paths, err := srv.images.Paths() + // if err != nil { + // return err + // } + // for _, name := range paths { + // if nameFilter != "" && nameFilter != name { + // continue + // } + // ids, err := srv.images.List(name) + // if err != nil { + // return err + // } + // for idx, img := range ids { + // if *limit > 0 && idx >= *limit { + // break + // } + // if !*quiet { + // for idx, field := range []string{ + // /* NAME */ name, + // /* ID */ img.Id, + // /* CREATED */ future.HumanDuration(time.Now().Sub(time.Unix(img.Created, 0))) + " ago", + // /* PARENT */ img.Parent, + // } { + // if idx == 0 { + // w.Write([]byte(field)) + // } else { + // w.Write([]byte("\t" + field)) + // } + // } + // w.Write([]byte{'\n'}) + // } else { + // stdout.Write([]byte(img.Id + "\n")) + // } + // } + // } + // if !*quiet { + // w.Flush() + // } + // } return nil @@ -492,7 +487,6 @@ func (srv *Server) CmdPs(stdin io.ReadCloser, stdout io.Writer, args ...string) fmt.Fprintf(w, "ID\tIMAGE\tCOMMAND\tCREATED\tSTATUS\tCOMMENT\n") } for _, container := range srv.containers.List() { - comment := container.GetUserData("comment") if !container.State.Running && !*fl_all { continue } @@ -503,11 +497,11 @@ func (srv *Server) CmdPs(stdin io.ReadCloser, stdout io.Writer, args ...string) } for idx, field := range []string{ /* ID */ container.Id, - /* IMAGE */ container.GetUserData("image"), + /* IMAGE */ container.Image, /* COMMAND */ command, /* CREATED */ future.HumanDuration(time.Now().Sub(container.Created)) + " ago", /* STATUS */ container.State.String(), - /* COMMENT */ comment, + /* COMMENT */ "", } { if idx == 0 { w.Write([]byte(field)) @@ -540,17 +534,13 @@ func (srv *Server) CmdCommit(stdin io.ReadCloser, stdout io.Writer, args ...stri } if container := srv.containers.Get(containerName); container != nil { // FIXME: freeze the container before copying it to avoid data corruption? - rwTar, err := fs.Tar(container.Mountpoint.Rw, fs.Uncompressed) + // FIXME: this shouldn't be in commands. + rwTar, err := container.ExportRw() if err != nil { return err } // Create a new image from the container's base layers + a new layer from container changes - parentImg, err := srv.images.Get(container.Image) - if err != nil { - return err - } - - img, err := srv.images.Create(rwTar, parentImg, imgName, "") + img, err := srv.containers.graph.Create(rwTar, container.Image, "") if err != nil { return err } @@ -574,10 +564,10 @@ func (srv *Server) CmdTar(stdin io.ReadCloser, stdout io.Writer, args ...string) } name := cmd.Arg(0) if container := srv.containers.Get(name); container != nil { - if err := container.Mountpoint.EnsureMounted(); err != nil { + if err := container.EnsureMounted(); err != nil { return err } - data, err := fs.Tar(container.Mountpoint.Root, fs.Uncompressed) + data, err := container.Export() if err != nil { return err } @@ -603,7 +593,7 @@ func (srv *Server) CmdDiff(stdin io.ReadCloser, stdout io.Writer, args ...string if container := srv.containers.Get(cmd.Arg(0)); container == nil { return errors.New("No such container") } else { - changes, err := srv.images.Changes(container.Mountpoint) + changes, err := container.Changes() if err != nil { return err } @@ -625,10 +615,20 @@ func (srv *Server) CmdLogs(stdin io.ReadCloser, stdout io.Writer, args ...string } name := cmd.Arg(0) if container := srv.containers.Get(name); container != nil { - if _, err := io.Copy(stdout, container.StdoutLog()); err != nil { + log_stdout, err := container.ReadLog("stdout") + if err != nil { return err } - if _, err := io.Copy(stdout, container.StderrLog()); err != nil { + log_stderr, err := container.ReadLog("stderr") + if err != nil { + return err + } + // FIXME: Interpolate stdout and stderr instead of concatenating them + // FIXME: Differentiate stdout and stderr in the remote protocol + if _, err := io.Copy(stdout, log_stdout); err != nil { + return err + } + if _, err := io.Copy(stdout, log_stderr); err != nil { return err } return nil @@ -636,31 +636,6 @@ func (srv *Server) CmdLogs(stdin io.ReadCloser, stdout io.Writer, args ...string return errors.New("No such container: " + cmd.Arg(0)) } -func (srv *Server) CreateContainer(img *fs.Image, ports []int, user string, tty bool, openStdin bool, memory int64, comment string, cmd string, args ...string) (*Container, error) { - id := future.RandomId()[:8] - container, err := srv.containers.Create(id, cmd, args, img, - &Config{ - Hostname: id, - Ports: ports, - User: user, - Tty: tty, - OpenStdin: openStdin, - Memory: memory, - }) - if err != nil { - return nil, err - } - if err := container.SetUserData("image", img.Id); err != nil { - srv.containers.Destroy(container) - return nil, errors.New("Error setting container userdata: " + err.Error()) - } - if err := container.SetUserData("comment", comment); err != nil { - srv.containers.Destroy(container) - return nil, errors.New("Error setting container userdata: " + err.Error()) - } - return container, nil -} - func (srv *Server) CmdAttach(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "attach", "[OPTIONS]", "Attach to a running container") fl_i := cmd.Bool("i", false, "Attach to stdin") @@ -729,7 +704,6 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) fl_attach := cmd.Bool("a", false, "Attach stdin and stdout") fl_stdin := cmd.Bool("i", false, "Keep stdin open even if not attached") fl_tty := cmd.Bool("t", false, "Allocate a pseudo-tty") - fl_comment := cmd.String("c", "", "Comment") fl_memory := cmd.Int64("m", 0, "Memory limit (in bytes)") var fl_ports ports @@ -738,8 +712,6 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) return nil } name := cmd.Arg(0) - var img_name string - //var img_version string // Only here for reference var cmdline []string if len(cmd.Args()) >= 2 { @@ -758,33 +730,15 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) cmdline = []string{"/bin/bash", "-i"} } - // Find the image - img, err := srv.images.Find(name) - if err != nil { - return err - } else if img == nil { - // Separate the name:version tag - if strings.Contains(name, ":") { - parts := strings.SplitN(name, ":", 2) - img_name = parts[0] - //img_version = parts[1] // Only here for reference - } else { - img_name = name - } - - stdin_noclose := ioutil.NopCloser(stdin) - if err := srv.CmdImport(stdin_noclose, stdout, img_name); err != nil { - return err - } - img, err = srv.images.Find(name) - if err != nil || img == nil { - return errors.New("Could not find image after downloading: " + name) - } - } - // Create new container - container, err := srv.CreateContainer(img, fl_ports, *fl_user, *fl_tty, - *fl_stdin, *fl_memory, *fl_comment, cmdline[0], cmdline[1:]...) + container, err := srv.containers.Create(cmdline[0], cmdline[1:], name, + &Config{ + Ports: fl_ports, + User: *fl_user, + Tty: *fl_tty, + OpenStdin: *fl_stdin, + Memory: *fl_memory, + }) if err != nil { return errors.New("Error creating container: " + err.Error()) } @@ -850,7 +804,6 @@ func NewServer() (*Server, error) { return nil, err } srv := &Server{ - images: containers.Store, containers: containers, } return srv, nil @@ -858,5 +811,4 @@ func NewServer() (*Server, error) { type Server struct { containers *Docker - images *fs.Store } diff --git a/container.go b/container.go index c80129e2a8..26e18cc1de 100644 --- a/container.go +++ b/container.go @@ -3,7 +3,9 @@ package docker import ( "encoding/json" "errors" - "github.com/dotcloud/docker/fs" + "fmt" + "github.com/dotcloud/docker/future" + "github.com/dotcloud/docker/graph" "github.com/kr/pty" "io" "io/ioutil" @@ -16,40 +18,34 @@ import ( "time" ) -var sysInitPath string - -func init() { - sysInitPath = SelfPath() -} - type Container struct { - Id string - Root string + root string + + Id string Created time.Time Path string Args []string - Config *Config - Mountpoint *fs.Mountpoint - State *State - Image string + Config *Config + State State + Image string network *NetworkInterface networkManager *NetworkManager NetworkSettings *NetworkSettings - SysInitPath string - lxcConfigPath string - cmd *exec.Cmd - stdout *writeBroadcaster - stderr *writeBroadcaster - stdin io.ReadCloser - stdinPipe io.WriteCloser + SysInitPath string + cmd *exec.Cmd + stdout *writeBroadcaster + stderr *writeBroadcaster + stdin io.ReadCloser + stdinPipe io.WriteCloser stdoutLog *os.File stderrLog *os.File + runtime *Docker // FIXME: rename Docker to Runtime for clarity } type Config struct { @@ -69,104 +65,9 @@ type NetworkSettings struct { PortMapping map[string]string } -func createContainer(id string, root string, command string, args []string, image *fs.Image, config *Config, netManager *NetworkManager) (*Container, error) { - mountpoint, err := image.Mountpoint(path.Join(root, "rootfs"), path.Join(root, "rw")) - if err != nil { - return nil, err - } - container := &Container{ - Id: id, - Root: root, - Created: time.Now(), - Path: command, - Args: args, - Config: config, - Image: image.Id, - Mountpoint: mountpoint, - State: newState(), - networkManager: netManager, - NetworkSettings: &NetworkSettings{}, - SysInitPath: sysInitPath, - lxcConfigPath: path.Join(root, "config.lxc"), - stdout: newWriteBroadcaster(), - stderr: newWriteBroadcaster(), - } - if err := os.Mkdir(root, 0700); err != nil { - return nil, err - } - // Setup logging of stdout and stderr to disk - if stdoutLog, err := os.OpenFile(path.Join(container.Root, id+"-stdout.log"), os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600); err != nil { - return nil, err - } else { - container.stdoutLog = stdoutLog - } - if stderrLog, err := os.OpenFile(path.Join(container.Root, id+"-stderr.log"), os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600); err != nil { - return nil, err - } else { - container.stderrLog = stderrLog - } - if container.Config.OpenStdin { - container.stdin, container.stdinPipe = io.Pipe() - } else { - container.stdinPipe = NopWriteCloser(ioutil.Discard) // Silently drop stdin - } - container.stdout.AddWriter(NopWriteCloser(container.stdoutLog)) - container.stderr.AddWriter(NopWriteCloser(container.stderrLog)) - - if err := container.save(); err != nil { - return nil, err - } - return container, nil -} - -func loadContainer(store *fs.Store, containerPath string, netManager *NetworkManager) (*Container, error) { - data, err := ioutil.ReadFile(path.Join(containerPath, "config.json")) - if err != nil { - return nil, err - } - mountpoint, err := store.FetchMountpoint( - path.Join(containerPath, "rootfs"), - path.Join(containerPath, "rw"), - ) - if err != nil { - return nil, err - } else if mountpoint == nil { - return nil, errors.New("Couldn't load container: unregistered mountpoint.") - } - container := &Container{ - stdout: newWriteBroadcaster(), - stderr: newWriteBroadcaster(), - lxcConfigPath: path.Join(containerPath, "config.lxc"), - networkManager: netManager, - NetworkSettings: &NetworkSettings{}, - Mountpoint: mountpoint, - } - // Load container settings - if err := json.Unmarshal(data, container); err != nil { - return nil, err - } - - // Setup logging of stdout and stderr to disk - if stdoutLog, err := os.OpenFile(path.Join(container.Root, container.Id+"-stdout.log"), os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600); err != nil { - return nil, err - } else { - container.stdoutLog = stdoutLog - } - if stderrLog, err := os.OpenFile(path.Join(container.Root, container.Id+"-stderr.log"), os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600); err != nil { - return nil, err - } else { - container.stderrLog = stderrLog - } - container.stdout.AddWriter(NopWriteCloser(container.stdoutLog)) - container.stderr.AddWriter(NopWriteCloser(container.stderrLog)) - - if container.Config.OpenStdin { - container.stdin, container.stdinPipe = io.Pipe() - } else { - container.stdinPipe = NopWriteCloser(ioutil.Discard) // Silently drop stdin - } - container.State = newState() - return container, nil +func GenerateId() string { + future.Seed() + return future.RandomId() } func (container *Container) Cmd() *exec.Cmd { @@ -177,64 +78,32 @@ func (container *Container) When() time.Time { return container.Created } -func (container *Container) loadUserData() (map[string]string, error) { - jsonData, err := ioutil.ReadFile(path.Join(container.Root, "userdata.json")) - if err != nil { - if os.IsNotExist(err) { - return make(map[string]string), nil - } - return nil, err - } - data := make(map[string]string) - if err := json.Unmarshal(jsonData, &data); err != nil { - return nil, err - } - return data, nil -} - -func (container *Container) saveUserData(data map[string]string) error { - jsonData, err := json.Marshal(data) +func (container *Container) FromDisk() error { + data, err := ioutil.ReadFile(container.jsonPath()) if err != nil { return err } - return ioutil.WriteFile(path.Join(container.Root, "userdata.json"), jsonData, 0700) -} - -func (container *Container) SetUserData(key, value string) error { - data, err := container.loadUserData() - if err != nil { + // Load container settings + if err := json.Unmarshal(data, container); err != nil { return err } - data[key] = value - return container.saveUserData(data) + return nil } -func (container *Container) GetUserData(key string) string { - data, err := container.loadUserData() - if err != nil { - return "" - } - if value, exists := data[key]; exists { - return value - } - return "" -} - -func (container *Container) save() (err error) { +func (container *Container) ToDisk() (err error) { data, err := json.Marshal(container) if err != nil { return } - return ioutil.WriteFile(path.Join(container.Root, "config.json"), data, 0666) + return ioutil.WriteFile(container.jsonPath(), data, 0666) } func (container *Container) generateLXCConfig() error { - fo, err := os.Create(container.lxcConfigPath) + fo, err := os.Create(container.lxcConfigPath()) if err != nil { return err } defer fo.Close() - if err := LxcTemplateCompiled.Execute(fo, container); err != nil { return err } @@ -309,7 +178,7 @@ func (container *Container) start() error { } func (container *Container) Start() error { - if err := container.Mountpoint.EnsureMounted(); err != nil { + if err := container.EnsureMounted(); err != nil { return err } if err := container.allocateNetwork(); err != nil { @@ -320,7 +189,7 @@ func (container *Container) Start() error { } params := []string{ "-n", container.Id, - "-f", container.lxcConfigPath, + "-f", container.lxcConfigPath(), "--", "/sbin/init", } @@ -348,8 +217,10 @@ func (container *Container) Start() error { if err != nil { return err } + // FIXME: save state on disk *first*, then converge + // this way disk state is used as a journal, eg. we can restore after crash etc. container.State.setRunning(container.cmd.Process.Pid) - container.save() + container.ToDisk() go container.monitor() return nil } @@ -389,28 +260,12 @@ func (container *Container) StdoutPipe() (io.ReadCloser, error) { return newBufReader(reader), nil } -func (container *Container) StdoutLog() io.Reader { - r, err := os.Open(container.stdoutLog.Name()) - if err != nil { - return nil - } - return r -} - func (container *Container) StderrPipe() (io.ReadCloser, error) { reader, writer := io.Pipe() container.stderr.AddWriter(writer) return newBufReader(reader), nil } -func (container *Container) StderrLog() io.Reader { - r, err := os.Open(container.stderrLog.Name()) - if err != nil { - return nil - } - return r -} - func (container *Container) allocateNetwork() error { iface, err := container.networkManager.Allocate() if err != nil { @@ -450,7 +305,7 @@ func (container *Container) monitor() { } container.stdout.Close() container.stderr.Close() - if err := container.Mountpoint.Umount(); err != nil { + if err := container.Unmount(); err != nil { log.Printf("%v: Failed to umount filesystem: %v", container.Id, err) } @@ -461,7 +316,7 @@ func (container *Container) monitor() { // Report status back container.State.setStopped(exitCode) - container.save() + container.ToDisk() } func (container *Container) kill() error { @@ -523,6 +378,17 @@ func (container *Container) Wait() int { return container.State.ExitCode } +func (container *Container) ExportRw() (graph.Archive, error) { + return graph.Tar(container.rwPath(), graph.Uncompressed) +} + +func (container *Container) Export() (graph.Archive, error) { + if err := container.EnsureMounted(); err != nil { + return nil, err + } + return graph.Tar(container.RootfsPath(), graph.Uncompressed) +} + func (container *Container) WaitTimeout(timeout time.Duration) error { done := make(chan bool) go func() { @@ -538,3 +404,75 @@ func (container *Container) WaitTimeout(timeout time.Duration) error { } return nil } + +func (container *Container) EnsureMounted() error { + if mounted, err := container.Mounted(); err != nil { + return err + } else if mounted { + return nil + } + return container.Mount() +} + +func (container *Container) Mount() error { + image, err := container.GetImage() + if err != nil { + return err + } + return image.Mount(container.RootfsPath(), container.rwPath()) +} + +func (container *Container) Changes() ([]graph.Change, error) { + image, err := container.GetImage() + if err != nil { + return nil, err + } + return image.Changes(container.rwPath()) +} + +func (container *Container) GetImage() (*graph.Image, error) { + if container.runtime == nil { + return nil, fmt.Errorf("Can't get image of unregistered container") + } + return container.runtime.graph.Get(container.Image) +} + +func (container *Container) Mounted() (bool, error) { + return graph.Mounted(container.RootfsPath()) +} + +func (container *Container) Unmount() error { + return graph.Unmount(container.RootfsPath()) +} + +func (container *Container) logPath(name string) string { + return path.Join(container.root, fmt.Sprintf("%s-%s.log", container.Id, name)) +} + +func (container *Container) ReadLog(name string) (io.Reader, error) { + return os.Open(container.logPath(name)) +} + +func (container *Container) jsonPath() string { + return path.Join(container.root, "config.json") +} + +func (container *Container) lxcConfigPath() string { + return path.Join(container.root, "config.lxc") +} + +// This method must be exported to be used from the lxc template +func (container *Container) RootfsPath() string { + return path.Join(container.root, "rootfs") +} + +func (container *Container) rwPath() string { + return path.Join(container.root, "rw") +} + +func validateId(id string) error { + if id == "" { + return fmt.Errorf("Invalid empty id") + } + return nil +} diff --git a/container_test.go b/container_test.go index 18beeba253..1658e5832c 100644 --- a/container_test.go +++ b/container_test.go @@ -3,7 +3,6 @@ package docker import ( "bufio" "fmt" - "github.com/dotcloud/docker/fs" "io" "io/ioutil" "math/rand" @@ -21,10 +20,9 @@ func TestCommitRun(t *testing.T) { } defer nuke(docker) container1, err := docker.Create( - "precommit_test", "/bin/sh", []string{"-c", "echo hello > /world"}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{ Memory: 33554432, }, @@ -44,18 +42,11 @@ func TestCommitRun(t *testing.T) { t.Errorf("Container shouldn't be running") } - // FIXME: freeze the container before copying it to avoid data corruption? - rwTar, err := fs.Tar(container1.Mountpoint.Rw, fs.Uncompressed) + rwTar, err := container1.ExportRw() if err != nil { t.Error(err) } - // Create a new image from the container's base layers + a new layer from container changes - parentImg, err := docker.Store.Get(container1.Image) - if err != nil { - t.Error(err) - } - - img, err := docker.Store.Create(rwTar, parentImg, "test_commitrun", "unit test commited image") + img, err := docker.graph.Create(rwTar, container1.Image, "unit test commited image") if err != nil { t.Error(err) } @@ -63,10 +54,9 @@ func TestCommitRun(t *testing.T) { // FIXME: Make a TestCommit that stops here and check docker.root/layers/img.id/world container2, err := docker.Create( - "postcommit_test", "cat", []string{"/world"}, - img, + img.Id, &Config{ Memory: 33554432, }, @@ -98,10 +88,9 @@ func TestRun(t *testing.T) { } defer nuke(docker) container, err := docker.Create( - "run_test", "ls", []string{"-al"}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{ Memory: 33554432, }, @@ -129,10 +118,9 @@ func TestOutput(t *testing.T) { } defer nuke(docker) container, err := docker.Create( - "output_test", "echo", []string{"-n", "foobar"}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{}, ) if err != nil { @@ -155,10 +143,9 @@ func TestKill(t *testing.T) { } defer nuke(docker) container, err := docker.Create( - "stop_test", "cat", []string{"/dev/zero"}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{}, ) if err != nil { @@ -199,10 +186,9 @@ func TestExitCode(t *testing.T) { defer nuke(docker) trueContainer, err := docker.Create( - "exit_test_1", "/bin/true", []string{""}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{}, ) if err != nil { @@ -214,10 +200,9 @@ func TestExitCode(t *testing.T) { } falseContainer, err := docker.Create( - "exit_test_2", "/bin/false", []string{""}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{}, ) if err != nil { @@ -244,10 +229,9 @@ func TestRestart(t *testing.T) { } defer nuke(docker) container, err := docker.Create( - "restart_test", "echo", []string{"-n", "foobar"}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{}, ) if err != nil { @@ -279,10 +263,9 @@ func TestRestartStdin(t *testing.T) { } defer nuke(docker) container, err := docker.Create( - "restart_stdin_test", "cat", []string{}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{ OpenStdin: true, }, @@ -331,10 +314,9 @@ func TestUser(t *testing.T) { // Default user must be root container, err := docker.Create( - "user_default", "id", []string{}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{}, ) if err != nil { @@ -351,10 +333,9 @@ func TestUser(t *testing.T) { // Set a username container, err = docker.Create( - "user_root", "id", []string{}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{ User: "root", }, @@ -373,10 +354,9 @@ func TestUser(t *testing.T) { // Set a UID container, err = docker.Create( - "user_uid0", "id", []string{}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{ User: "0", }, @@ -395,10 +375,9 @@ func TestUser(t *testing.T) { // Set a different user by uid container, err = docker.Create( - "user_uid1", "id", []string{}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{ User: "1", }, @@ -419,10 +398,9 @@ func TestUser(t *testing.T) { // Set a different user by username container, err = docker.Create( - "user_daemon", "id", []string{}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{ User: "daemon", }, @@ -448,10 +426,9 @@ func TestMultipleContainers(t *testing.T) { defer nuke(docker) container1, err := docker.Create( - "container1", "cat", []string{"/dev/zero"}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{}, ) if err != nil { @@ -460,10 +437,9 @@ func TestMultipleContainers(t *testing.T) { defer docker.Destroy(container1) container2, err := docker.Create( - "container2", "cat", []string{"/dev/zero"}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{}, ) if err != nil { @@ -504,10 +480,9 @@ func TestStdin(t *testing.T) { } defer nuke(docker) container, err := docker.Create( - "stdin_test", "cat", []string{}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{ OpenStdin: true, }, @@ -540,10 +515,9 @@ func TestTty(t *testing.T) { } defer nuke(docker) container, err := docker.Create( - "tty_test", "cat", []string{}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{ OpenStdin: true, }, @@ -576,10 +550,9 @@ func TestEnv(t *testing.T) { } defer nuke(docker) container, err := docker.Create( - "env_test", "/usr/bin/env", []string{}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{}, ) if err != nil { @@ -651,10 +624,9 @@ func TestLXCConfig(t *testing.T) { memMax := 536870912 mem := memMin + rand.Intn(memMax-memMin) container, err := docker.Create( - "config_test", "/bin/true", []string{}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{ Hostname: "foobar", Memory: int64(mem), @@ -665,10 +637,10 @@ func TestLXCConfig(t *testing.T) { } defer docker.Destroy(container) container.generateLXCConfig() - grepFile(t, container.lxcConfigPath, "lxc.utsname = foobar") - grepFile(t, container.lxcConfigPath, + grepFile(t, container.lxcConfigPath(), "lxc.utsname = foobar") + grepFile(t, container.lxcConfigPath(), fmt.Sprintf("lxc.cgroup.memory.limit_in_bytes = %d", mem)) - grepFile(t, container.lxcConfigPath, + grepFile(t, container.lxcConfigPath(), fmt.Sprintf("lxc.cgroup.memory.memsw.limit_in_bytes = %d", mem*2)) } @@ -680,10 +652,9 @@ func BenchmarkRunSequencial(b *testing.B) { defer nuke(docker) for i := 0; i < b.N; i++ { container, err := docker.Create( - fmt.Sprintf("bench_%v", i), "echo", []string{"-n", "foo"}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{}, ) if err != nil { @@ -717,10 +688,9 @@ func BenchmarkRunParallel(b *testing.B) { tasks = append(tasks, complete) go func(i int, complete chan error) { container, err := docker.Create( - fmt.Sprintf("bench_%v", i), "echo", []string{"-n", "foo"}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{}, ) if err != nil { diff --git a/docker.go b/docker.go index 626c7452ea..65654ba752 100644 --- a/docker.go +++ b/docker.go @@ -3,12 +3,15 @@ package docker import ( "container/list" "fmt" - "github.com/dotcloud/docker/fs" + "github.com/dotcloud/docker/graph" + "io" "io/ioutil" "log" "os" "path" "sort" + "sync" + "time" ) type Docker struct { @@ -16,7 +19,13 @@ type Docker struct { repository string containers *list.List networkManager *NetworkManager - Store *fs.Store + graph *graph.Graph +} + +var sysInitPath string + +func init() { + sysInitPath = SelfPath() } func (docker *Docker) List() []*Container { @@ -49,20 +58,98 @@ func (docker *Docker) Exists(id string) bool { return docker.Get(id) != nil } -func (docker *Docker) Create(id string, command string, args []string, image *fs.Image, config *Config) (*Container, error) { - if docker.Exists(id) { - return nil, fmt.Errorf("Container %v already exists", id) - } - root := path.Join(docker.repository, id) +func (docker *Docker) containerRoot(id string) string { + return path.Join(docker.repository, id) +} - container, err := createContainer(id, root, command, args, image, config, docker.networkManager) - if err != nil { +func (docker *Docker) Create(command string, args []string, image string, config *Config) (*Container, error) { + container := &Container{ + // FIXME: we should generate the ID here instead of receiving it as an argument + Id: GenerateId(), + Created: time.Now(), + Path: command, + Args: args, + Config: config, + Image: image, + NetworkSettings: &NetworkSettings{}, + // FIXME: do we need to store this in the container? + SysInitPath: sysInitPath, + } + container.root = docker.containerRoot(container.Id) + // Step 1: create the container directory. + // This doubles as a barrier to avoid race conditions. + if err := os.Mkdir(container.root, 0700); err != nil { + return nil, err + } + // Step 2: save the container json + if err := container.ToDisk(); err != nil { + return nil, err + } + // Step 3: register the container + if err := docker.Register(container); err != nil { return nil, err } - docker.containers.PushBack(container) return container, nil } +func (docker *Docker) Load(id string) (*Container, error) { + container := &Container{root: docker.containerRoot(id)} + if err := container.FromDisk(); err != nil { + return nil, err + } + if container.Id != id { + return container, fmt.Errorf("Container %s is stored at %s", container.Id, id) + } + if err := docker.Register(container); err != nil { + return nil, err + } + return container, nil +} + +// Register makes a container object usable by the runtime as +func (docker *Docker) Register(container *Container) error { + if container.runtime != nil || docker.Exists(container.Id) { + return fmt.Errorf("Container is already loaded") + } + if err := validateId(container.Id); err != nil { + return err + } + container.runtime = docker + container.networkManager = docker.networkManager // FIXME: infer from docker.runtime + // Setup state lock (formerly in newState() + lock := new(sync.Mutex) + container.State.stateChangeLock = lock + container.State.stateChangeCond = sync.NewCond(lock) + // Attach to stdout and stderr + container.stderr = newWriteBroadcaster() + container.stdout = newWriteBroadcaster() + // Attach to stdin + if container.Config.OpenStdin { + container.stdin, container.stdinPipe = io.Pipe() + } else { + container.stdinPipe = NopWriteCloser(ioutil.Discard) // Silently drop stdin + } + // Setup logging of stdout and stderr to disk + if err := docker.LogToDisk(container.stdout, container.logPath("stdout")); err != nil { + return err + } + if err := docker.LogToDisk(container.stderr, container.logPath("stderr")); err != nil { + return err + } + // done + docker.containers.PushBack(container) + return nil +} + +func (docker *Docker) LogToDisk(src *writeBroadcaster, dst string) error { + log, err := os.OpenFile(dst, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600) + if err != nil { + return err + } + src.AddWriter(NopWriteCloser(log)) + return nil +} + func (docker *Docker) Destroy(container *Container) error { element := docker.getContainerElement(container.Id) if element == nil { @@ -72,18 +159,18 @@ func (docker *Docker) Destroy(container *Container) error { if err := container.Stop(); err != nil { return err } - if container.Mountpoint.Mounted() { - if err := container.Mountpoint.Umount(); err != nil { - return fmt.Errorf("Unable to umount container %v: %v", container.Id, err) + if mounted, err := container.Mounted(); err != nil { + return err + } else if mounted { + if err := container.Unmount(); err != nil { + return fmt.Errorf("Unable to unmount container %v: %v", container.Id, err) } } - if err := container.Mountpoint.Deregister(); err != nil { - return fmt.Errorf("Unable to deregiser -- ? mountpoint %v: %v", container.Mountpoint.Root, err) - } - if err := os.RemoveAll(container.Root); err != nil { + // Deregister the container before removing its directory, to avoid race conditions + docker.containers.Remove(element) + if err := os.RemoveAll(container.root); err != nil { return fmt.Errorf("Unable to remove filesystem for %v: %v", container.Id, err) } - docker.containers.Remove(element) return nil } @@ -93,12 +180,13 @@ func (docker *Docker) restore() error { return err } for _, v := range dir { - container, err := loadContainer(docker.Store, path.Join(docker.repository, v.Name()), docker.networkManager) + id := v.Name() + container, err := docker.Load(id) if err != nil { - log.Printf("Failed to load container %v: %v", v.Name(), err) + log.Printf("Failed to load container %v: %v", id, err) continue } - docker.containers.PushBack(container) + log.Printf("Loaded container %v", container.Id) } return nil } @@ -114,7 +202,7 @@ func NewFromDirectory(root string) (*Docker, error) { return nil, err } - store, err := fs.New(path.Join(root, "images")) + graph, err := graph.New(path.Join(root, "graph")) if err != nil { return nil, err } @@ -127,8 +215,8 @@ func NewFromDirectory(root string) (*Docker, error) { root: root, repository: docker_repo, containers: list.New(), - Store: store, networkManager: netManager, + graph: graph, } if err := docker.restore(); err != nil { diff --git a/docker_test.go b/docker_test.go index 735e15baf7..a8329e10dd 100644 --- a/docker_test.go +++ b/docker_test.go @@ -1,7 +1,7 @@ package docker import ( - "github.com/dotcloud/docker/fs" + "github.com/dotcloud/docker/graph" "io" "io/ioutil" "os" @@ -63,7 +63,6 @@ func init() { } // Create the "Server" srv := &Server{ - images: docker.Store, containers: docker, } // Retrieve the Image @@ -93,8 +92,8 @@ func newTestDocker() (*Docker, error) { return docker, nil } -func GetTestImage(docker *Docker) *fs.Image { - imgs, err := docker.Store.Images() +func GetTestImage(docker *Docker) *graph.Image { + imgs, err := docker.graph.All() if err != nil { panic(err) } else if len(imgs) < 1 { @@ -115,10 +114,9 @@ func TestCreate(t *testing.T) { t.Errorf("Expected 0 containers, %v found", len(docker.List())) } container, err := docker.Create( - "test_create", "ls", []string{"-al"}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{}, ) if err != nil { @@ -137,22 +135,22 @@ func TestCreate(t *testing.T) { } // Make sure the container List() returns is the right one - if docker.List()[0].Id != "test_create" { + if docker.List()[0].Id != container.Id { t.Errorf("Unexpected container %v returned by List", docker.List()[0]) } // Make sure we can get the container with Get() - if docker.Get("test_create") == nil { + if docker.Get(container.Id) == nil { t.Errorf("Unable to get newly created container") } // Make sure it is the right container - if docker.Get("test_create") != container { + if docker.Get(container.Id) != container { t.Errorf("Get() returned the wrong container") } // Make sure Exists returns it as existing - if !docker.Exists("test_create") { + if !docker.Exists(container.Id) { t.Errorf("Exists() returned false for a newly created container") } } @@ -164,10 +162,9 @@ func TestDestroy(t *testing.T) { } defer nuke(docker) container, err := docker.Create( - "test_destroy", "ls", []string{"-al"}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{}, ) if err != nil { @@ -189,12 +186,12 @@ func TestDestroy(t *testing.T) { } // Make sure docker.Get() refuses to return the unexisting container - if docker.Get("test_destroy") != nil { + if docker.Get(container.Id) != nil { t.Errorf("Unable to get newly created container") } // Make sure the container root directory does not exist anymore - _, err = os.Stat(container.Root) + _, err = os.Stat(container.root) if err == nil || !os.IsNotExist(err) { t.Errorf("Container root directory still exists after destroy") } @@ -213,10 +210,9 @@ func TestGet(t *testing.T) { } defer nuke(docker) container1, err := docker.Create( - "test1", "ls", []string{"-al"}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{}, ) if err != nil { @@ -225,10 +221,9 @@ func TestGet(t *testing.T) { defer docker.Destroy(container1) container2, err := docker.Create( - "test2", "ls", []string{"-al"}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{}, ) if err != nil { @@ -237,10 +232,9 @@ func TestGet(t *testing.T) { defer docker.Destroy(container2) container3, err := docker.Create( - "test3", "ls", []string{"-al"}, - GetTestImage(docker), + GetTestImage(docker).Id, &Config{}, ) if err != nil { @@ -248,16 +242,16 @@ func TestGet(t *testing.T) { } defer docker.Destroy(container3) - if docker.Get("test1") != container1 { - t.Errorf("Get(test1) returned %v while expecting %v", docker.Get("test1"), container1) + if docker.Get(container1.Id) != container1 { + t.Errorf("Get(test1) returned %v while expecting %v", docker.Get(container1.Id), container1) } - if docker.Get("test2") != container2 { - t.Errorf("Get(test2) returned %v while expecting %v", docker.Get("test2"), container2) + if docker.Get(container2.Id) != container2 { + t.Errorf("Get(test2) returned %v while expecting %v", docker.Get(container2.Id), container2) } - if docker.Get("test3") != container3 { - t.Errorf("Get(test3) returned %v while expecting %v", docker.Get("test3"), container3) + if docker.Get(container3.Id) != container3 { + t.Errorf("Get(test3) returned %v while expecting %v", docker.Get(container3.Id), container3) } } @@ -282,10 +276,9 @@ func TestRestore(t *testing.T) { // Create a container with one instance of docker container1, err := docker1.Create( - "restore_test", "ls", []string{"-al"}, - GetTestImage(docker1), + GetTestImage(docker1).Id, &Config{}, ) if err != nil { @@ -309,7 +302,7 @@ func TestRestore(t *testing.T) { if len(docker2.List()) != 1 { t.Errorf("Expected 1 container, %v found", len(docker2.List())) } - container2 := docker2.Get("restore_test") + container2 := docker2.Get(container1.Id) if container2 == nil { t.Fatal("Unable to Get container") } diff --git a/lxc_template.go b/lxc_template.go index b86beb6bbc..e3beb037f9 100755 --- a/lxc_template.go +++ b/lxc_template.go @@ -22,7 +22,7 @@ lxc.network.mtu = 1500 lxc.network.ipv4 = {{.NetworkSettings.IpAddress}}/{{.NetworkSettings.IpPrefixLen}} # root filesystem -{{$ROOTFS := .Mountpoint.Root}} +{{$ROOTFS := .RootfsPath}} lxc.rootfs = {{$ROOTFS}} # use a dedicated pts for the container (and limit the number of pseudo terminal diff --git a/state.go b/state.go index e864f304f9..d2c53b83ce 100644 --- a/state.go +++ b/state.go @@ -17,14 +17,6 @@ type State struct { stateChangeCond *sync.Cond } -func newState() *State { - lock := new(sync.Mutex) - return &State{ - stateChangeLock: lock, - stateChangeCond: sync.NewCond(lock), - } -} - // String returns a human-readable description of the state func (s *State) String() string { if s.Running { From b8547f31e4ba5d4381c572e7efa4bea3ccb80515 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 00:41:15 -0700 Subject: [PATCH 17/80] Renamed Docker{} to Runtime{} for clarity --- commands.go | 54 ++++++------- container.go | 2 +- container_test.go | 196 +++++++++++++++++++++++----------------------- docker.go | 78 +++++++++--------- docker_test.go | 128 +++++++++++++++--------------- 5 files changed, 229 insertions(+), 229 deletions(-) diff --git a/commands.go b/commands.go index 79619ca44d..480e74530e 100644 --- a/commands.go +++ b/commands.go @@ -116,7 +116,7 @@ func (srv *Server) CmdWait(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } for _, name := range cmd.Args() { - if container := srv.containers.Get(name); container != nil { + if container := srv.runtime.Get(name); container != nil { fmt.Fprintln(stdout, container.Wait()) } else { return errors.New("No such container: " + name) @@ -133,7 +133,7 @@ func (srv *Server) CmdVersion(stdin io.ReadCloser, stdout io.Writer, args ...str // 'docker info': display system-wide information. func (srv *Server) CmdInfo(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - images, _ := srv.containers.graph.All() + images, _ := srv.runtime.graph.All() var imgcount int if images == nil { imgcount = 0 @@ -149,7 +149,7 @@ func (srv *Server) CmdInfo(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } fmt.Fprintf(stdout, "containers: %d\nversion: %s\nimages: %d\n", - len(srv.containers.List()), + len(srv.runtime.List()), VERSION, imgcount) return nil @@ -165,7 +165,7 @@ func (srv *Server) CmdStop(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } for _, name := range cmd.Args() { - if container := srv.containers.Get(name); container != nil { + if container := srv.runtime.Get(name); container != nil { if err := container.Stop(); err != nil { return err } @@ -187,7 +187,7 @@ func (srv *Server) CmdRestart(stdin io.ReadCloser, stdout io.Writer, args ...str return nil } for _, name := range cmd.Args() { - if container := srv.containers.Get(name); container != nil { + if container := srv.runtime.Get(name); container != nil { if err := container.Restart(); err != nil { return err } @@ -209,7 +209,7 @@ func (srv *Server) CmdStart(stdin io.ReadCloser, stdout io.Writer, args ...strin return nil } for _, name := range cmd.Args() { - if container := srv.containers.Get(name); container != nil { + if container := srv.runtime.Get(name); container != nil { if err := container.Start(); err != nil { return err } @@ -231,7 +231,7 @@ func (srv *Server) CmdMount(stdin io.ReadCloser, stdout io.Writer, args ...strin return nil } for _, name := range cmd.Args() { - if container := srv.containers.Get(name); container != nil { + if container := srv.runtime.Get(name); container != nil { if err := container.EnsureMounted(); err != nil { return err } @@ -254,9 +254,9 @@ func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...str } name := cmd.Arg(0) var obj interface{} - if container := srv.containers.Get(name); container != nil { + if container := srv.runtime.Get(name); container != nil { obj = container - } else if image, err := srv.containers.graph.Get(name); err != nil { + } else if image, err := srv.runtime.graph.Get(name); err != nil { return err } else if image != nil { obj = image @@ -291,7 +291,7 @@ func (srv *Server) CmdPort(stdin io.ReadCloser, stdout io.Writer, args ...string } name := cmd.Arg(0) privatePort := cmd.Arg(1) - if container := srv.containers.Get(name); container == nil { + if container := srv.runtime.Get(name); container == nil { return errors.New("No such container: " + name) } else { if frontend, exists := container.NetworkSettings.PortMapping[privatePort]; !exists { @@ -311,7 +311,7 @@ func (srv *Server) CmdRmi(stdin io.ReadCloser, stdout io.Writer, args ...string) return nil } for _, name := range cmd.Args() { - if err := srv.containers.graph.Delete(name); err != nil { + if err := srv.runtime.graph.Delete(name); err != nil { return err } } @@ -324,11 +324,11 @@ func (srv *Server) CmdRm(stdin io.ReadCloser, stdout io.Writer, args ...string) return nil } for _, name := range cmd.Args() { - container := srv.containers.Get(name) + container := srv.runtime.Get(name) if container == nil { return errors.New("No such container: " + name) } - if err := srv.containers.Destroy(container); err != nil { + if err := srv.runtime.Destroy(container); err != nil { fmt.Fprintln(stdout, "Error destroying container "+name+": "+err.Error()) } } @@ -342,7 +342,7 @@ func (srv *Server) CmdKill(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } for _, name := range cmd.Args() { - container := srv.containers.Get(name) + container := srv.runtime.Get(name) if container == nil { return errors.New("No such container: " + name) } @@ -390,7 +390,7 @@ func (srv *Server) CmdImport(stdin io.ReadCloser, stdout io.Writer, args ...stri archive = future.ProgressReader(resp.Body, int(resp.ContentLength), stdout) } fmt.Fprintf(stdout, "Unpacking to %s\n", name) - img, err := srv.containers.graph.Create(archive, "", "") + img, err := srv.runtime.graph.Create(archive, "", "") if err != nil { return err } @@ -420,7 +420,7 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri fmt.Fprintf(w, "NAME\tID\tCREATED\tPARENT\n") } if *quiet { - images, err := srv.containers.graph.All() + images, err := srv.runtime.graph.All() if err != nil { return err } @@ -486,7 +486,7 @@ func (srv *Server) CmdPs(stdin io.ReadCloser, stdout io.Writer, args ...string) if !*quiet { fmt.Fprintf(w, "ID\tIMAGE\tCOMMAND\tCREATED\tSTATUS\tCOMMENT\n") } - for _, container := range srv.containers.List() { + for _, container := range srv.runtime.List() { if !container.State.Running && !*fl_all { continue } @@ -532,7 +532,7 @@ func (srv *Server) CmdCommit(stdin io.ReadCloser, stdout io.Writer, args ...stri cmd.Usage() return nil } - if container := srv.containers.Get(containerName); container != nil { + if container := srv.runtime.Get(containerName); container != nil { // FIXME: freeze the container before copying it to avoid data corruption? // FIXME: this shouldn't be in commands. rwTar, err := container.ExportRw() @@ -540,7 +540,7 @@ func (srv *Server) CmdCommit(stdin io.ReadCloser, stdout io.Writer, args ...stri return err } // Create a new image from the container's base layers + a new layer from container changes - img, err := srv.containers.graph.Create(rwTar, container.Image, "") + img, err := srv.runtime.graph.Create(rwTar, container.Image, "") if err != nil { return err } @@ -563,7 +563,7 @@ func (srv *Server) CmdTar(stdin io.ReadCloser, stdout io.Writer, args ...string) return errors.New("Sparse mode not yet implemented") // FIXME } name := cmd.Arg(0) - if container := srv.containers.Get(name); container != nil { + if container := srv.runtime.Get(name); container != nil { if err := container.EnsureMounted(); err != nil { return err } @@ -590,7 +590,7 @@ func (srv *Server) CmdDiff(stdin io.ReadCloser, stdout io.Writer, args ...string if cmd.NArg() < 1 { return errors.New("Not enough arguments") } - if container := srv.containers.Get(cmd.Arg(0)); container == nil { + if container := srv.runtime.Get(cmd.Arg(0)); container == nil { return errors.New("No such container") } else { changes, err := container.Changes() @@ -614,7 +614,7 @@ func (srv *Server) CmdLogs(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } name := cmd.Arg(0) - if container := srv.containers.Get(name); container != nil { + if container := srv.runtime.Get(name); container != nil { log_stdout, err := container.ReadLog("stdout") if err != nil { return err @@ -649,7 +649,7 @@ func (srv *Server) CmdAttach(stdin io.ReadCloser, stdout io.Writer, args ...stri return nil } name := cmd.Arg(0) - container := srv.containers.Get(name) + container := srv.runtime.Get(name) if container == nil { return errors.New("No such container: " + name) } @@ -731,7 +731,7 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) } // Create new container - container, err := srv.containers.Create(cmdline[0], cmdline[1:], name, + container, err := srv.runtime.Create(cmdline[0], cmdline[1:], name, &Config{ Ports: fl_ports, User: *fl_user, @@ -799,16 +799,16 @@ func NewServer() (*Server, error) { // if err != nil { // return nil, err // } - containers, err := New() + runtime, err := New() if err != nil { return nil, err } srv := &Server{ - containers: containers, + runtime: runtime, } return srv, nil } type Server struct { - containers *Docker + runtime *Runtime } diff --git a/container.go b/container.go index 26e18cc1de..80109ee1af 100644 --- a/container.go +++ b/container.go @@ -45,7 +45,7 @@ type Container struct { stdoutLog *os.File stderrLog *os.File - runtime *Docker // FIXME: rename Docker to Runtime for clarity + runtime *Runtime } type Config struct { diff --git a/container_test.go b/container_test.go index 1658e5832c..64558871ac 100644 --- a/container_test.go +++ b/container_test.go @@ -14,15 +14,15 @@ import ( ) func TestCommitRun(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) - container1, err := docker.Create( + defer nuke(runtime) + container1, err := runtime.Create( "/bin/sh", []string{"-c", "echo hello > /world"}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{ Memory: 33554432, }, @@ -30,7 +30,7 @@ func TestCommitRun(t *testing.T) { if err != nil { t.Fatal(err) } - defer docker.Destroy(container1) + defer runtime.Destroy(container1) if container1.State.Running { t.Errorf("Container shouldn't be running") @@ -46,14 +46,14 @@ func TestCommitRun(t *testing.T) { if err != nil { t.Error(err) } - img, err := docker.graph.Create(rwTar, container1.Image, "unit test commited image") + img, err := runtime.graph.Create(rwTar, container1.Image, "unit test commited image") if err != nil { t.Error(err) } // FIXME: Make a TestCommit that stops here and check docker.root/layers/img.id/world - container2, err := docker.Create( + container2, err := runtime.Create( "cat", []string{"/world"}, img.Id, @@ -64,7 +64,7 @@ func TestCommitRun(t *testing.T) { if err != nil { t.Fatal(err) } - defer docker.Destroy(container2) + defer runtime.Destroy(container2) stdout, err := container2.StdoutPipe() stderr, err := container2.StderrPipe() @@ -82,15 +82,15 @@ func TestCommitRun(t *testing.T) { } func TestRun(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) - container, err := docker.Create( + defer nuke(runtime) + container, err := runtime.Create( "ls", []string{"-al"}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{ Memory: 33554432, }, @@ -98,7 +98,7 @@ func TestRun(t *testing.T) { if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) if container.State.Running { t.Errorf("Container shouldn't be running") @@ -112,21 +112,21 @@ func TestRun(t *testing.T) { } func TestOutput(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) - container, err := docker.Create( + defer nuke(runtime) + container, err := runtime.Create( "echo", []string{"-n", "foobar"}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{}, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) output, err := container.Output() if err != nil { t.Fatal(err) @@ -137,21 +137,21 @@ func TestOutput(t *testing.T) { } func TestKill(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) - container, err := docker.Create( + defer nuke(runtime) + container, err := runtime.Create( "cat", []string{"/dev/zero"}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{}, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) if container.State.Running { t.Errorf("Container shouldn't be running") @@ -179,36 +179,36 @@ func TestKill(t *testing.T) { } func TestExitCode(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) + defer nuke(runtime) - trueContainer, err := docker.Create( + trueContainer, err := runtime.Create( "/bin/true", []string{""}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{}, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(trueContainer) + defer runtime.Destroy(trueContainer) if err := trueContainer.Run(); err != nil { t.Fatal(err) } - falseContainer, err := docker.Create( + falseContainer, err := runtime.Create( "/bin/false", []string{""}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{}, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(falseContainer) + defer runtime.Destroy(falseContainer) if err := falseContainer.Run(); err != nil { t.Fatal(err) } @@ -223,21 +223,21 @@ func TestExitCode(t *testing.T) { } func TestRestart(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) - container, err := docker.Create( + defer nuke(runtime) + container, err := runtime.Create( "echo", []string{"-n", "foobar"}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{}, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) output, err := container.Output() if err != nil { t.Fatal(err) @@ -257,15 +257,15 @@ func TestRestart(t *testing.T) { } func TestRestartStdin(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) - container, err := docker.Create( + defer nuke(runtime) + container, err := runtime.Create( "cat", []string{}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{ OpenStdin: true, }, @@ -273,7 +273,7 @@ func TestRestartStdin(t *testing.T) { if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) stdin, err := container.StdinPipe() stdout, err := container.StdoutPipe() @@ -306,23 +306,23 @@ func TestRestartStdin(t *testing.T) { } func TestUser(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) + defer nuke(runtime) // Default user must be root - container, err := docker.Create( + container, err := runtime.Create( "id", []string{}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{}, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) output, err := container.Output() if err != nil { t.Fatal(err) @@ -332,10 +332,10 @@ func TestUser(t *testing.T) { } // Set a username - container, err = docker.Create( + container, err = runtime.Create( "id", []string{}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{ User: "root", }, @@ -343,7 +343,7 @@ func TestUser(t *testing.T) { if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) output, err = container.Output() if err != nil || container.State.ExitCode != 0 { t.Fatal(err) @@ -353,10 +353,10 @@ func TestUser(t *testing.T) { } // Set a UID - container, err = docker.Create( + container, err = runtime.Create( "id", []string{}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{ User: "0", }, @@ -364,7 +364,7 @@ func TestUser(t *testing.T) { if err != nil || container.State.ExitCode != 0 { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) output, err = container.Output() if err != nil || container.State.ExitCode != 0 { t.Fatal(err) @@ -374,10 +374,10 @@ func TestUser(t *testing.T) { } // Set a different user by uid - container, err = docker.Create( + container, err = runtime.Create( "id", []string{}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{ User: "1", }, @@ -385,7 +385,7 @@ func TestUser(t *testing.T) { if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) output, err = container.Output() if err != nil { t.Fatal(err) @@ -397,10 +397,10 @@ func TestUser(t *testing.T) { } // Set a different user by username - container, err = docker.Create( + container, err = runtime.Create( "id", []string{}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{ User: "daemon", }, @@ -408,7 +408,7 @@ func TestUser(t *testing.T) { if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) output, err = container.Output() if err != nil || container.State.ExitCode != 0 { t.Fatal(err) @@ -419,33 +419,33 @@ func TestUser(t *testing.T) { } func TestMultipleContainers(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) + defer nuke(runtime) - container1, err := docker.Create( + container1, err := runtime.Create( "cat", []string{"/dev/zero"}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{}, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container1) + defer runtime.Destroy(container1) - container2, err := docker.Create( + container2, err := runtime.Create( "cat", []string{"/dev/zero"}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{}, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container2) + defer runtime.Destroy(container2) // Start both containers if err := container1.Start(); err != nil { @@ -474,15 +474,15 @@ func TestMultipleContainers(t *testing.T) { } func TestStdin(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) - container, err := docker.Create( + defer nuke(runtime) + container, err := runtime.Create( "cat", []string{}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{ OpenStdin: true, }, @@ -490,7 +490,7 @@ func TestStdin(t *testing.T) { if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) stdin, err := container.StdinPipe() stdout, err := container.StdoutPipe() @@ -509,15 +509,15 @@ func TestStdin(t *testing.T) { } func TestTty(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) - container, err := docker.Create( + defer nuke(runtime) + container, err := runtime.Create( "cat", []string{}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{ OpenStdin: true, }, @@ -525,7 +525,7 @@ func TestTty(t *testing.T) { if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) stdin, err := container.StdinPipe() stdout, err := container.StdoutPipe() @@ -544,21 +544,21 @@ func TestTty(t *testing.T) { } func TestEnv(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) - container, err := docker.Create( + defer nuke(runtime) + container, err := runtime.Create( "/usr/bin/env", []string{}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{}, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) stdout, err := container.StdoutPipe() if err != nil { t.Fatal(err) @@ -613,20 +613,20 @@ func grepFile(t *testing.T, path string, pattern string) { } func TestLXCConfig(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) + defer nuke(runtime) // Memory is allocated randomly for testing rand.Seed(time.Now().UTC().UnixNano()) memMin := 33554432 memMax := 536870912 mem := memMin + rand.Intn(memMax-memMin) - container, err := docker.Create( + container, err := runtime.Create( "/bin/true", []string{}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{ Hostname: "foobar", Memory: int64(mem), @@ -635,7 +635,7 @@ func TestLXCConfig(t *testing.T) { if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) container.generateLXCConfig() grepFile(t, container.lxcConfigPath(), "lxc.utsname = foobar") grepFile(t, container.lxcConfigPath(), @@ -645,22 +645,22 @@ func TestLXCConfig(t *testing.T) { } func BenchmarkRunSequencial(b *testing.B) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { b.Fatal(err) } - defer nuke(docker) + defer nuke(runtime) for i := 0; i < b.N; i++ { - container, err := docker.Create( + container, err := runtime.Create( "echo", []string{"-n", "foo"}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{}, ) if err != nil { b.Fatal(err) } - defer docker.Destroy(container) + defer runtime.Destroy(container) output, err := container.Output() if err != nil { b.Fatal(err) @@ -668,18 +668,18 @@ func BenchmarkRunSequencial(b *testing.B) { if string(output) != "foo" { b.Fatalf("Unexecpted output: %v", string(output)) } - if err := docker.Destroy(container); err != nil { + if err := runtime.Destroy(container); err != nil { b.Fatal(err) } } } func BenchmarkRunParallel(b *testing.B) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { b.Fatal(err) } - defer nuke(docker) + defer nuke(runtime) var tasks []chan error @@ -687,17 +687,17 @@ func BenchmarkRunParallel(b *testing.B) { complete := make(chan error) tasks = append(tasks, complete) go func(i int, complete chan error) { - container, err := docker.Create( + container, err := runtime.Create( "echo", []string{"-n", "foo"}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{}, ) if err != nil { complete <- err return } - defer docker.Destroy(container) + defer runtime.Destroy(container) if err := container.Start(); err != nil { complete <- err return @@ -709,7 +709,7 @@ func BenchmarkRunParallel(b *testing.B) { // if string(output) != "foo" { // complete <- fmt.Errorf("Unexecpted output: %v", string(output)) // } - if err := docker.Destroy(container); err != nil { + if err := runtime.Destroy(container); err != nil { complete <- err return } diff --git a/docker.go b/docker.go index 65654ba752..e362cdafe8 100644 --- a/docker.go +++ b/docker.go @@ -14,7 +14,7 @@ import ( "time" ) -type Docker struct { +type Runtime struct { root string repository string containers *list.List @@ -28,16 +28,16 @@ func init() { sysInitPath = SelfPath() } -func (docker *Docker) List() []*Container { +func (runtime *Runtime) List() []*Container { containers := new(History) - for e := docker.containers.Front(); e != nil; e = e.Next() { + for e := runtime.containers.Front(); e != nil; e = e.Next() { containers.Add(e.Value.(*Container)) } return *containers } -func (docker *Docker) getContainerElement(id string) *list.Element { - for e := docker.containers.Front(); e != nil; e = e.Next() { +func (runtime *Runtime) getContainerElement(id string) *list.Element { + for e := runtime.containers.Front(); e != nil; e = e.Next() { container := e.Value.(*Container) if container.Id == id { return e @@ -46,23 +46,23 @@ func (docker *Docker) getContainerElement(id string) *list.Element { return nil } -func (docker *Docker) Get(id string) *Container { - e := docker.getContainerElement(id) +func (runtime *Runtime) Get(id string) *Container { + e := runtime.getContainerElement(id) if e == nil { return nil } return e.Value.(*Container) } -func (docker *Docker) Exists(id string) bool { - return docker.Get(id) != nil +func (runtime *Runtime) Exists(id string) bool { + return runtime.Get(id) != nil } -func (docker *Docker) containerRoot(id string) string { - return path.Join(docker.repository, id) +func (runtime *Runtime) containerRoot(id string) string { + return path.Join(runtime.repository, id) } -func (docker *Docker) Create(command string, args []string, image string, config *Config) (*Container, error) { +func (runtime *Runtime) Create(command string, args []string, image string, config *Config) (*Container, error) { container := &Container{ // FIXME: we should generate the ID here instead of receiving it as an argument Id: GenerateId(), @@ -75,7 +75,7 @@ func (docker *Docker) Create(command string, args []string, image string, config // FIXME: do we need to store this in the container? SysInitPath: sysInitPath, } - container.root = docker.containerRoot(container.Id) + container.root = runtime.containerRoot(container.Id) // Step 1: create the container directory. // This doubles as a barrier to avoid race conditions. if err := os.Mkdir(container.root, 0700); err != nil { @@ -86,36 +86,36 @@ func (docker *Docker) Create(command string, args []string, image string, config return nil, err } // Step 3: register the container - if err := docker.Register(container); err != nil { + if err := runtime.Register(container); err != nil { return nil, err } return container, nil } -func (docker *Docker) Load(id string) (*Container, error) { - container := &Container{root: docker.containerRoot(id)} +func (runtime *Runtime) Load(id string) (*Container, error) { + container := &Container{root: runtime.containerRoot(id)} if err := container.FromDisk(); err != nil { return nil, err } if container.Id != id { return container, fmt.Errorf("Container %s is stored at %s", container.Id, id) } - if err := docker.Register(container); err != nil { + if err := runtime.Register(container); err != nil { return nil, err } return container, nil } // Register makes a container object usable by the runtime as -func (docker *Docker) Register(container *Container) error { - if container.runtime != nil || docker.Exists(container.Id) { +func (runtime *Runtime) Register(container *Container) error { + if container.runtime != nil || runtime.Exists(container.Id) { return fmt.Errorf("Container is already loaded") } if err := validateId(container.Id); err != nil { return err } - container.runtime = docker - container.networkManager = docker.networkManager // FIXME: infer from docker.runtime + container.runtime = runtime + container.networkManager = runtime.networkManager // FIXME: infer from docker.runtime // Setup state lock (formerly in newState() lock := new(sync.Mutex) container.State.stateChangeLock = lock @@ -130,18 +130,18 @@ func (docker *Docker) Register(container *Container) error { container.stdinPipe = NopWriteCloser(ioutil.Discard) // Silently drop stdin } // Setup logging of stdout and stderr to disk - if err := docker.LogToDisk(container.stdout, container.logPath("stdout")); err != nil { + if err := runtime.LogToDisk(container.stdout, container.logPath("stdout")); err != nil { return err } - if err := docker.LogToDisk(container.stderr, container.logPath("stderr")); err != nil { + if err := runtime.LogToDisk(container.stderr, container.logPath("stderr")); err != nil { return err } // done - docker.containers.PushBack(container) + runtime.containers.PushBack(container) return nil } -func (docker *Docker) LogToDisk(src *writeBroadcaster, dst string) error { +func (runtime *Runtime) LogToDisk(src *writeBroadcaster, dst string) error { log, err := os.OpenFile(dst, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600) if err != nil { return err @@ -150,8 +150,8 @@ func (docker *Docker) LogToDisk(src *writeBroadcaster, dst string) error { return nil } -func (docker *Docker) Destroy(container *Container) error { - element := docker.getContainerElement(container.Id) +func (runtime *Runtime) Destroy(container *Container) error { + element := runtime.getContainerElement(container.Id) if element == nil { return fmt.Errorf("Container %v not found - maybe it was already destroyed?", container.Id) } @@ -167,21 +167,21 @@ func (docker *Docker) Destroy(container *Container) error { } } // Deregister the container before removing its directory, to avoid race conditions - docker.containers.Remove(element) + runtime.containers.Remove(element) if err := os.RemoveAll(container.root); err != nil { return fmt.Errorf("Unable to remove filesystem for %v: %v", container.Id, err) } return nil } -func (docker *Docker) restore() error { - dir, err := ioutil.ReadDir(docker.repository) +func (runtime *Runtime) restore() error { + dir, err := ioutil.ReadDir(runtime.repository) if err != nil { return err } for _, v := range dir { id := v.Name() - container, err := docker.Load(id) + container, err := runtime.Load(id) if err != nil { log.Printf("Failed to load container %v: %v", id, err) continue @@ -191,14 +191,14 @@ func (docker *Docker) restore() error { return nil } -func New() (*Docker, error) { +func New() (*Runtime, error) { return NewFromDirectory("/var/lib/docker") } -func NewFromDirectory(root string) (*Docker, error) { - docker_repo := path.Join(root, "containers") +func NewFromDirectory(root string) (*Runtime, error) { + runtime_repo := path.Join(root, "containers") - if err := os.MkdirAll(docker_repo, 0700); err != nil && !os.IsExist(err) { + if err := os.MkdirAll(runtime_repo, 0700); err != nil && !os.IsExist(err) { return nil, err } @@ -211,18 +211,18 @@ func NewFromDirectory(root string) (*Docker, error) { return nil, err } - docker := &Docker{ + runtime := &Runtime{ root: root, - repository: docker_repo, + repository: runtime_repo, containers: list.New(), networkManager: netManager, graph: graph, } - if err := docker.restore(); err != nil { + if err := runtime.restore(); err != nil { return nil, err } - return docker, nil + return runtime, nil } type History []*Container diff --git a/docker_test.go b/docker_test.go index a8329e10dd..093dae67fd 100644 --- a/docker_test.go +++ b/docker_test.go @@ -16,8 +16,8 @@ const unitTestImageName string = "busybox" var unitTestStoreBase string var srv *Server -func nuke(docker *Docker) error { - return os.RemoveAll(docker.root) +func nuke(runtime *Runtime) error { + return os.RemoveAll(runtime.root) } func CopyDirectory(source, dest string) error { @@ -57,13 +57,13 @@ func init() { unitTestStoreBase = root // Make it our Store root - docker, err := NewFromDirectory(root) + runtime, err := NewFromDirectory(root) if err != nil { panic(err) } // Create the "Server" srv := &Server{ - containers: docker, + runtime: runtime, } // Retrieve the Image if err := srv.CmdImport(os.Stdin, os.Stdout, unitTestImageName); err != nil { @@ -71,7 +71,7 @@ func init() { } } -func newTestDocker() (*Docker, error) { +func newTestRuntime() (*Runtime, error) { root, err := ioutil.TempDir("", "docker-test") if err != nil { return nil, err @@ -84,16 +84,16 @@ func newTestDocker() (*Docker, error) { return nil, err } - docker, err := NewFromDirectory(root) + runtime, err := NewFromDirectory(root) if err != nil { return nil, err } - return docker, nil + return runtime, nil } -func GetTestImage(docker *Docker) *graph.Image { - imgs, err := docker.graph.All() +func GetTestImage(runtime *Runtime) *graph.Image { + imgs, err := runtime.graph.All() if err != nil { panic(err) } else if len(imgs) < 1 { @@ -103,20 +103,20 @@ func GetTestImage(docker *Docker) *graph.Image { } func TestCreate(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) + defer nuke(runtime) // Make sure we start we 0 containers - if len(docker.List()) != 0 { - t.Errorf("Expected 0 containers, %v found", len(docker.List())) + if len(runtime.List()) != 0 { + t.Errorf("Expected 0 containers, %v found", len(runtime.List())) } - container, err := docker.Create( + container, err := runtime.Create( "ls", []string{"-al"}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{}, ) if err != nil { @@ -124,69 +124,69 @@ func TestCreate(t *testing.T) { } defer func() { - if err := docker.Destroy(container); err != nil { + if err := runtime.Destroy(container); err != nil { t.Error(err) } }() // Make sure we can find the newly created container with List() - if len(docker.List()) != 1 { - t.Errorf("Expected 1 container, %v found", len(docker.List())) + if len(runtime.List()) != 1 { + t.Errorf("Expected 1 container, %v found", len(runtime.List())) } // Make sure the container List() returns is the right one - if docker.List()[0].Id != container.Id { - t.Errorf("Unexpected container %v returned by List", docker.List()[0]) + if runtime.List()[0].Id != container.Id { + t.Errorf("Unexpected container %v returned by List", runtime.List()[0]) } // Make sure we can get the container with Get() - if docker.Get(container.Id) == nil { + if runtime.Get(container.Id) == nil { t.Errorf("Unable to get newly created container") } // Make sure it is the right container - if docker.Get(container.Id) != container { + if runtime.Get(container.Id) != container { t.Errorf("Get() returned the wrong container") } // Make sure Exists returns it as existing - if !docker.Exists(container.Id) { + if !runtime.Exists(container.Id) { t.Errorf("Exists() returned false for a newly created container") } } func TestDestroy(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) - container, err := docker.Create( + defer nuke(runtime) + container, err := runtime.Create( "ls", []string{"-al"}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{}, ) if err != nil { t.Fatal(err) } // Destroy - if err := docker.Destroy(container); err != nil { + if err := runtime.Destroy(container); err != nil { t.Error(err) } - // Make sure docker.Exists() behaves correctly - if docker.Exists("test_destroy") { + // Make sure runtime.Exists() behaves correctly + if runtime.Exists("test_destroy") { t.Errorf("Exists() returned true") } - // Make sure docker.List() doesn't list the destroyed container - if len(docker.List()) != 0 { - t.Errorf("Expected 0 container, %v found", len(docker.List())) + // Make sure runtime.List() doesn't list the destroyed container + if len(runtime.List()) != 0 { + t.Errorf("Expected 0 container, %v found", len(runtime.List())) } - // Make sure docker.Get() refuses to return the unexisting container - if docker.Get(container.Id) != nil { + // Make sure runtime.Get() refuses to return the unexisting container + if runtime.Get(container.Id) != nil { t.Errorf("Unable to get newly created container") } @@ -197,61 +197,61 @@ func TestDestroy(t *testing.T) { } // Test double destroy - if err := docker.Destroy(container); err == nil { + if err := runtime.Destroy(container); err == nil { // It should have failed t.Errorf("Double destroy did not fail") } } func TestGet(t *testing.T) { - docker, err := newTestDocker() + runtime, err := newTestRuntime() if err != nil { t.Fatal(err) } - defer nuke(docker) - container1, err := docker.Create( + defer nuke(runtime) + container1, err := runtime.Create( "ls", []string{"-al"}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{}, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container1) + defer runtime.Destroy(container1) - container2, err := docker.Create( + container2, err := runtime.Create( "ls", []string{"-al"}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{}, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container2) + defer runtime.Destroy(container2) - container3, err := docker.Create( + container3, err := runtime.Create( "ls", []string{"-al"}, - GetTestImage(docker).Id, + GetTestImage(runtime).Id, &Config{}, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container3) + defer runtime.Destroy(container3) - if docker.Get(container1.Id) != container1 { - t.Errorf("Get(test1) returned %v while expecting %v", docker.Get(container1.Id), container1) + if runtime.Get(container1.Id) != container1 { + t.Errorf("Get(test1) returned %v while expecting %v", runtime.Get(container1.Id), container1) } - if docker.Get(container2.Id) != container2 { - t.Errorf("Get(test2) returned %v while expecting %v", docker.Get(container2.Id), container2) + if runtime.Get(container2.Id) != container2 { + t.Errorf("Get(test2) returned %v while expecting %v", runtime.Get(container2.Id), container2) } - if docker.Get(container3.Id) != container3 { - t.Errorf("Get(test3) returned %v while expecting %v", docker.Get(container3.Id), container3) + if runtime.Get(container3.Id) != container3 { + t.Errorf("Get(test3) returned %v while expecting %v", runtime.Get(container3.Id), container3) } } @@ -269,24 +269,24 @@ func TestRestore(t *testing.T) { t.Fatal(err) } - docker1, err := NewFromDirectory(root) + runtime1, err := NewFromDirectory(root) if err != nil { t.Fatal(err) } // Create a container with one instance of docker - container1, err := docker1.Create( + container1, err := runtime1.Create( "ls", []string{"-al"}, - GetTestImage(docker1).Id, + GetTestImage(runtime1).Id, &Config{}, ) if err != nil { t.Fatal(err) } - defer docker1.Destroy(container1) - if len(docker1.List()) != 1 { - t.Errorf("Expected 1 container, %v found", len(docker1.List())) + defer runtime1.Destroy(container1) + if len(runtime1.List()) != 1 { + t.Errorf("Expected 1 container, %v found", len(runtime1.List())) } if err := container1.Run(); err != nil { t.Fatal(err) @@ -294,15 +294,15 @@ func TestRestore(t *testing.T) { // Here are are simulating a docker restart - that is, reloading all containers // from scratch - docker2, err := NewFromDirectory(root) + runtime2, err := NewFromDirectory(root) if err != nil { t.Fatal(err) } - defer nuke(docker2) - if len(docker2.List()) != 1 { - t.Errorf("Expected 1 container, %v found", len(docker2.List())) + defer nuke(runtime2) + if len(runtime2.List()) != 1 { + t.Errorf("Expected 1 container, %v found", len(runtime2.List())) } - container2 := docker2.Get(container1.Id) + container2 := runtime2.Get(container1.Id) if container2 == nil { t.Fatal("Unable to Get container") } From 299d0b2720ad5efee89c11b47d6de51094714164 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 00:52:43 -0700 Subject: [PATCH 18/80] Moved HumanDuration() to the main package --- commands.go | 4 ++-- future/future.go | 23 ----------------------- state.go | 3 +-- utils.go | 27 +++++++++++++++++++++++++++ 4 files changed, 30 insertions(+), 27 deletions(-) diff --git a/commands.go b/commands.go index 480e74530e..d24607d7a2 100644 --- a/commands.go +++ b/commands.go @@ -449,7 +449,7 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri // for idx, field := range []string{ // /* NAME */ name, // /* ID */ img.Id, - // /* CREATED */ future.HumanDuration(time.Now().Sub(time.Unix(img.Created, 0))) + " ago", + // /* CREATED */ HumanDuration(time.Now().Sub(time.Unix(img.Created, 0))) + " ago", // /* PARENT */ img.Parent, // } { // if idx == 0 { @@ -499,7 +499,7 @@ func (srv *Server) CmdPs(stdin io.ReadCloser, stdout io.Writer, args ...string) /* ID */ container.Id, /* IMAGE */ container.Image, /* COMMAND */ command, - /* CREATED */ future.HumanDuration(time.Now().Sub(container.Created)) + " ago", + /* CREATED */ HumanDuration(time.Now().Sub(container.Created)) + " ago", /* STATUS */ container.State.String(), /* COMMENT */ "", } { diff --git a/future/future.go b/future/future.go index 21b0eee385..bd62b353e0 100644 --- a/future/future.go +++ b/future/future.go @@ -23,29 +23,6 @@ func ComputeId(content io.Reader) (string, error) { return fmt.Sprintf("%x", h.Sum(nil)[:8]), nil } -func HumanDuration(d time.Duration) string { - if seconds := int(d.Seconds()); seconds < 1 { - return "Less than a second" - } else if seconds < 60 { - return fmt.Sprintf("%d seconds", seconds) - } else if minutes := int(d.Minutes()); minutes == 1 { - return "About a minute" - } else if minutes < 60 { - return fmt.Sprintf("%d minutes", minutes) - } else if hours := int(d.Hours()); hours == 1 { - return "About an hour" - } else if hours < 48 { - return fmt.Sprintf("%d hours", hours) - } else if hours < 24*7*2 { - return fmt.Sprintf("%d days", hours/24) - } else if hours < 24*30*3 { - return fmt.Sprintf("%d weeks", hours/24/7) - } else if hours < 24*365*2 { - return fmt.Sprintf("%d months", hours/24/30) - } - return fmt.Sprintf("%d years", d.Hours()/24/365) -} - func randomBytes() io.Reader { return bytes.NewBuffer([]byte(fmt.Sprintf("%x", rand.Int()))) } diff --git a/state.go b/state.go index d2c53b83ce..f438ff8727 100644 --- a/state.go +++ b/state.go @@ -2,7 +2,6 @@ package docker import ( "fmt" - "github.com/dotcloud/docker/future" "sync" "time" ) @@ -20,7 +19,7 @@ type State struct { // String returns a human-readable description of the state func (s *State) String() string { if s.Running { - return fmt.Sprintf("Up %s", future.HumanDuration(time.Now().Sub(s.StartedAt))) + return fmt.Sprintf("Up %s", HumanDuration(time.Now().Sub(s.StartedAt))) } return fmt.Sprintf("Exit %d", s.ExitCode) } diff --git a/utils.go b/utils.go index 520073e3ab..9a409f6b59 100644 --- a/utils.go +++ b/utils.go @@ -3,13 +3,40 @@ package docker import ( "bytes" "container/list" + "fmt" "io" "os" "os/exec" "path/filepath" "sync" + "time" ) +// HumanDuration returns a human-readable approximation of a duration +// (eg. "About a minute", "4 hours ago", etc.) +func HumanDuration(d time.Duration) string { + if seconds := int(d.Seconds()); seconds < 1 { + return "Less than a second" + } else if seconds < 60 { + return fmt.Sprintf("%d seconds", seconds) + } else if minutes := int(d.Minutes()); minutes == 1 { + return "About a minute" + } else if minutes < 60 { + return fmt.Sprintf("%d minutes", minutes) + } else if hours := int(d.Hours()); hours == 1 { + return "About an hour" + } else if hours < 48 { + return fmt.Sprintf("%d hours", hours) + } else if hours < 24*7*2 { + return fmt.Sprintf("%d days", hours/24) + } else if hours < 24*30*3 { + return fmt.Sprintf("%d weeks", hours/24/7) + } else if hours < 24*365*2 { + return fmt.Sprintf("%d months", hours/24/30) + } + return fmt.Sprintf("%d years", d.Hours()/24/365) +} + func Trunc(s string, maxlen int) string { if len(s) <= maxlen { return s From d7c5d060c4b3c4a38a843e92f832bbf97483d16b Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 00:54:54 -0700 Subject: [PATCH 19/80] Moved Download() and progressReader{} to the main package --- commands.go | 4 ++-- future/future.go | 51 ------------------------------------------------ utils.go | 51 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 53 deletions(-) diff --git a/commands.go b/commands.go index d24607d7a2..a4c5a4ef43 100644 --- a/commands.go +++ b/commands.go @@ -383,11 +383,11 @@ func (srv *Server) CmdImport(stdin io.ReadCloser, stdout io.Writer, args ...stri fmt.Fprintf(stdout, "Downloading from %s\n", u.String()) // Download with curl (pretty progress bar) // If curl is not available, fallback to http.Get() - resp, err = future.Download(u.String(), stdout) + resp, err = Download(u.String(), stdout) if err != nil { return err } - archive = future.ProgressReader(resp.Body, int(resp.ContentLength), stdout) + archive = ProgressReader(resp.Body, int(resp.ContentLength), stdout) } fmt.Fprintf(stdout, "Unpacking to %s\n", name) img, err := srv.runtime.graph.Create(archive, "", "") diff --git a/future/future.go b/future/future.go index bd62b353e0..dc85115719 100644 --- a/future/future.go +++ b/future/future.go @@ -3,11 +3,9 @@ package future import ( "bytes" "crypto/sha256" - "errors" "fmt" "io" "math/rand" - "net/http" "time" ) @@ -62,52 +60,3 @@ func Pv(src io.Reader, info io.Writer) io.Reader { }() return r } - -// Request a given URL and return an io.Reader -func Download(url string, stderr io.Writer) (*http.Response, error) { - var resp *http.Response - var err error = nil - if resp, err = http.Get(url); err != nil { - return nil, err - } - if resp.StatusCode >= 400 { - return nil, errors.New("Got HTTP status code >= 400: " + resp.Status) - } - return resp, nil -} - -// Reader with progress bar -type progressReader struct { - reader io.ReadCloser // Stream to read from - output io.Writer // Where to send progress bar to - read_total int // Expected stream length (bytes) - read_progress int // How much has been read so far (bytes) - last_update int // How many bytes read at least update -} - -func (r *progressReader) Read(p []byte) (n int, err error) { - read, err := io.ReadCloser(r.reader).Read(p) - r.read_progress += read - - // Only update progress for every 1% read - update_every := int(0.01 * float64(r.read_total)) - if r.read_progress-r.last_update > update_every || r.read_progress == r.read_total { - fmt.Fprintf(r.output, "%d/%d (%.0f%%)\r", - r.read_progress, - r.read_total, - float64(r.read_progress)/float64(r.read_total)*100) - r.last_update = r.read_progress - } - // Send newline when complete - if err == io.EOF { - fmt.Fprintf(r.output, "\n") - } - - return read, err -} -func (r *progressReader) Close() error { - return io.ReadCloser(r.reader).Close() -} -func ProgressReader(r io.ReadCloser, size int, output io.Writer) *progressReader { - return &progressReader{r, output, size, 0, 0} -} diff --git a/utils.go b/utils.go index 9a409f6b59..e8fefafb2b 100644 --- a/utils.go +++ b/utils.go @@ -3,8 +3,10 @@ package docker import ( "bytes" "container/list" + "errors" "fmt" "io" + "net/http" "os" "os/exec" "path/filepath" @@ -12,6 +14,55 @@ import ( "time" ) +// Request a given URL and return an io.Reader +func Download(url string, stderr io.Writer) (*http.Response, error) { + var resp *http.Response + var err error = nil + if resp, err = http.Get(url); err != nil { + return nil, err + } + if resp.StatusCode >= 400 { + return nil, errors.New("Got HTTP status code >= 400: " + resp.Status) + } + return resp, nil +} + +// Reader with progress bar +type progressReader struct { + reader io.ReadCloser // Stream to read from + output io.Writer // Where to send progress bar to + read_total int // Expected stream length (bytes) + read_progress int // How much has been read so far (bytes) + last_update int // How many bytes read at least update +} + +func (r *progressReader) Read(p []byte) (n int, err error) { + read, err := io.ReadCloser(r.reader).Read(p) + r.read_progress += read + + // Only update progress for every 1% read + update_every := int(0.01 * float64(r.read_total)) + if r.read_progress-r.last_update > update_every || r.read_progress == r.read_total { + fmt.Fprintf(r.output, "%d/%d (%.0f%%)\r", + r.read_progress, + r.read_total, + float64(r.read_progress)/float64(r.read_total)*100) + r.last_update = r.read_progress + } + // Send newline when complete + if err == io.EOF { + fmt.Fprintf(r.output, "\n") + } + + return read, err +} +func (r *progressReader) Close() error { + return io.ReadCloser(r.reader).Close() +} +func ProgressReader(r io.ReadCloser, size int, output io.Writer) *progressReader { + return &progressReader{r, output, size, 0, 0} +} + // HumanDuration returns a human-readable approximation of a duration // (eg. "About a minute", "4 hours ago", etc.) func HumanDuration(d time.Duration) string { From 0208b6accddd4f8c8ccfb380dd281ac208fe19d1 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 01:07:07 -0700 Subject: [PATCH 20/80] moved GenerateId() to the graph package --- commands.go | 3 ++- container.go | 5 ++--- future/future.go | 25 ------------------------- graph/image.go | 21 ++++++++++++++++++--- 4 files changed, 22 insertions(+), 32 deletions(-) diff --git a/commands.go b/commands.go index a4c5a4ef43..d56f77aab2 100644 --- a/commands.go +++ b/commands.go @@ -9,6 +9,7 @@ import ( "github.com/dotcloud/docker/future" "github.com/dotcloud/docker/rcli" "io" + "math/rand" "net/http" "net/url" "path" @@ -795,7 +796,7 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) } func NewServer() (*Server, error) { - future.Seed() + rand.Seed(time.Now().UTC().UnixNano()) // if err != nil { // return nil, err // } diff --git a/container.go b/container.go index 80109ee1af..04bb1bf0b0 100644 --- a/container.go +++ b/container.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/dotcloud/docker/future" "github.com/dotcloud/docker/graph" "github.com/kr/pty" "io" @@ -66,8 +65,8 @@ type NetworkSettings struct { } func GenerateId() string { - future.Seed() - return future.RandomId() + return graph.GenerateId() // Re-use the same code to generate container and image IDs + // (this might change when image Ids become content-based) } func (container *Container) Cmd() *exec.Cmd { diff --git a/future/future.go b/future/future.go index dc85115719..c4b30fba41 100644 --- a/future/future.go +++ b/future/future.go @@ -1,35 +1,10 @@ package future import ( - "bytes" - "crypto/sha256" "fmt" "io" - "math/rand" - "time" ) -func Seed() { - rand.Seed(time.Now().UTC().UnixNano()) -} - -func ComputeId(content io.Reader) (string, error) { - h := sha256.New() - if _, err := io.Copy(h, content); err != nil { - return "", err - } - return fmt.Sprintf("%x", h.Sum(nil)[:8]), nil -} - -func randomBytes() io.Reader { - return bytes.NewBuffer([]byte(fmt.Sprintf("%x", rand.Int()))) -} - -func RandomId() string { - id, _ := ComputeId(randomBytes()) // can't fail - return id -} - func Go(f func() error) chan error { ch := make(chan error) go func() { diff --git a/graph/image.go b/graph/image.go index 98bdc8874b..210ba7cedb 100644 --- a/graph/image.go +++ b/graph/image.go @@ -1,10 +1,13 @@ package graph import ( + "bytes" + "crypto/sha256" "encoding/json" "fmt" - "github.com/dotcloud/docker/future" + "io" "io/ioutil" + "math/rand" "os" "path" "strings" @@ -157,8 +160,20 @@ func ValidateId(id string) error { } func GenerateId() string { - future.Seed() - return future.RandomId() + // FIXME: don't seed every time + rand.Seed(time.Now().UTC().UnixNano()) + randomBytes := bytes.NewBuffer([]byte(fmt.Sprintf("%x", rand.Int()))) + id, _ := ComputeId(randomBytes) // can't fail + return id +} + +// ComputeId reads from `content` until EOF, then returns a SHA of what it read, as a string. +func ComputeId(content io.Reader) (string, error) { + h := sha256.New() + if _, err := io.Copy(h, content); err != nil { + return "", err + } + return fmt.Sprintf("%x", h.Sum(nil)[:8]), nil } // Image includes convenience proxy functions to its graph From deb603aaf41d6885e4b51c7ca06acfa940ad2a41 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 01:10:44 -0700 Subject: [PATCH 21/80] Removed unused utility future.Pv() --- future/future.go | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/future/future.go b/future/future.go index c4b30fba41..fa9d348fcc 100644 --- a/future/future.go +++ b/future/future.go @@ -1,9 +1,6 @@ package future -import ( - "fmt" - "io" -) +import () func Go(f func() error) chan error { ch := make(chan error) @@ -12,26 +9,3 @@ func Go(f func() error) chan error { }() return ch } - -// Pv wraps an io.Reader such that it is passed through unchanged, -// but logs the number of bytes copied (comparable to the unix command pv) -func Pv(src io.Reader, info io.Writer) io.Reader { - var totalBytes int - data := make([]byte, 2048) - r, w := io.Pipe() - go func() { - for { - if n, err := src.Read(data); err != nil { - w.CloseWithError(err) - return - } else { - totalBytes += n - fmt.Fprintf(info, "--> %d bytes\n", totalBytes) - if _, err = w.Write(data[:n]); err != nil { - return - } - } - } - }() - return r -} From 623e91e2e3f3d0b408a43606df8d3c037a6b03be Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 01:13:55 -0700 Subject: [PATCH 22/80] Moved Go() to the main package... And got rid of the useless docker/future package --- commands.go | 7 +++---- docker/docker.go | 5 ++--- future/future.go | 11 ----------- utils.go | 10 ++++++++++ 4 files changed, 15 insertions(+), 18 deletions(-) delete mode 100644 future/future.go diff --git a/commands.go b/commands.go index d56f77aab2..6f7b12be41 100644 --- a/commands.go +++ b/commands.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "github.com/dotcloud/docker/auth" - "github.com/dotcloud/docker/future" "github.com/dotcloud/docker/rcli" "io" "math/rand" @@ -749,7 +748,7 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) return err } if *fl_attach { - future.Go(func() error { + Go(func() error { _, err := io.Copy(cmd_stdin, stdin) cmd_stdin.Close() return err @@ -769,11 +768,11 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) if err := container.Start(); err != nil { return err } - sending_stdout := future.Go(func() error { + sending_stdout := Go(func() error { _, err := io.Copy(stdout, cmd_stdout) return err }) - sending_stderr := future.Go(func() error { + sending_stderr := Go(func() error { _, err := io.Copy(stdout, cmd_stderr) return err }) diff --git a/docker/docker.go b/docker/docker.go index 79fe582e97..5fc70f064f 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -3,7 +3,6 @@ package main import ( "flag" "github.com/dotcloud/docker" - "github.com/dotcloud/docker/future" "github.com/dotcloud/docker/rcli" "github.com/dotcloud/docker/term" "io" @@ -57,11 +56,11 @@ func runCommand(args []string) error { // closing the connection. // See http://code.google.com/p/go/issues/detail?id=3345 if conn, err := rcli.Call("tcp", "127.0.0.1:4242", args...); err == nil { - receive_stdout := future.Go(func() error { + receive_stdout := docker.Go(func() error { _, err := io.Copy(os.Stdout, conn) return err }) - send_stdin := future.Go(func() error { + send_stdin := docker.Go(func() error { _, err := io.Copy(conn, os.Stdin) if err := conn.CloseWrite(); err != nil { log.Printf("Couldn't send EOF: " + err.Error()) diff --git a/future/future.go b/future/future.go deleted file mode 100644 index fa9d348fcc..0000000000 --- a/future/future.go +++ /dev/null @@ -1,11 +0,0 @@ -package future - -import () - -func Go(f func() error) chan error { - ch := make(chan error) - go func() { - ch <- f() - }() - return ch -} diff --git a/utils.go b/utils.go index e8fefafb2b..29f62f7534 100644 --- a/utils.go +++ b/utils.go @@ -14,6 +14,16 @@ import ( "time" ) +// Go is a basic promise implementation: it wraps calls a function in a goroutine, +// and returns a channel which will later return the function's return value. +func Go(f func() error) chan error { + ch := make(chan error) + go func() { + ch <- f() + }() + return ch +} + // Request a given URL and return an io.Reader func Download(url string, stderr io.Writer) (*http.Response, error) { var resp *http.Response From e627a0da1ee3013d9f4b74e96636699401e5d720 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 01:23:00 -0700 Subject: [PATCH 23/80] Renamed 'docker tar' to 'docker export' for symmetry with 'docker import' --- commands.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/commands.go b/commands.go index 6f7b12be41..18062ac1ea 100644 --- a/commands.go +++ b/commands.go @@ -49,7 +49,7 @@ func (srv *Server) Help() string { {"run", "Run a command in a new container"}, {"start", "Start a stopped container"}, {"stop", "Stop a running container"}, - {"tar", "Stream the contents of a container as a tar archive"}, + {"export", "Stream the contents of a container as a tar archive"}, {"version", "Show the docker version information"}, {"wait", "Block until a container stops, then print its exit code"}, } { @@ -551,10 +551,10 @@ func (srv *Server) CmdCommit(stdin io.ReadCloser, stdout io.Writer, args ...stri return errors.New("No such container: " + containerName) } -func (srv *Server) CmdTar(stdin io.ReadCloser, stdout io.Writer, args ...string) error { +func (srv *Server) CmdExport(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, - "tar", "CONTAINER", - "Stream the contents of a container as a tar archive") + "export", "CONTAINER", + "Export the contents of a filesystem as a tar archive") fl_sparse := cmd.Bool("s", false, "Generate a sparse tar stream (top layer + reference to bottom layers)") if err := cmd.Parse(args); err != nil { return nil @@ -564,9 +564,6 @@ func (srv *Server) CmdTar(stdin io.ReadCloser, stdout io.Writer, args ...string) } name := cmd.Arg(0) if container := srv.runtime.Get(name); container != nil { - if err := container.EnsureMounted(); err != nil { - return err - } data, err := container.Export() if err != nil { return err From 2f781f21288d9e579cf18fcf08a0750ce7b0ac38 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 01:24:12 -0700 Subject: [PATCH 24/80] Removed 'sparse export' mode, it is deprecated by the new diff-based transfer protocol --- commands.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/commands.go b/commands.go index 18062ac1ea..0dca488cf0 100644 --- a/commands.go +++ b/commands.go @@ -555,13 +555,9 @@ func (srv *Server) CmdExport(stdin io.ReadCloser, stdout io.Writer, args ...stri cmd := rcli.Subcmd(stdout, "export", "CONTAINER", "Export the contents of a filesystem as a tar archive") - fl_sparse := cmd.Bool("s", false, "Generate a sparse tar stream (top layer + reference to bottom layers)") if err := cmd.Parse(args); err != nil { return nil } - if *fl_sparse { - return errors.New("Sparse mode not yet implemented") // FIXME - } name := cmd.Arg(0) if container := srv.runtime.Get(name); container != nil { data, err := container.Export() From 377cebe36f4e4fc1cf927313df4a9c1cb6dc49cf Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 01:24:54 -0700 Subject: [PATCH 25/80] Renamed docker*.go to runtime*.go --- docker.go => runtime.go | 0 docker_test.go => runtime_test.go | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename docker.go => runtime.go (100%) rename docker_test.go => runtime_test.go (100%) diff --git a/docker.go b/runtime.go similarity index 100% rename from docker.go rename to runtime.go diff --git a/docker_test.go b/runtime_test.go similarity index 100% rename from docker_test.go rename to runtime_test.go From d65983f386e4a355c404e8b68a85ab301c37a59c Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 01:43:03 -0700 Subject: [PATCH 26/80] No need for a Container.networkManager field: it can be accessed via Container.runtime --- container.go | 3 +-- runtime.go | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/container.go b/container.go index 04bb1bf0b0..2dcc3d0e28 100644 --- a/container.go +++ b/container.go @@ -32,7 +32,6 @@ type Container struct { Image string network *NetworkInterface - networkManager *NetworkManager NetworkSettings *NetworkSettings SysInitPath string @@ -266,7 +265,7 @@ func (container *Container) StderrPipe() (io.ReadCloser, error) { } func (container *Container) allocateNetwork() error { - iface, err := container.networkManager.Allocate() + iface, err := container.runtime.networkManager.Allocate() if err != nil { return err } diff --git a/runtime.go b/runtime.go index e362cdafe8..8d017df690 100644 --- a/runtime.go +++ b/runtime.go @@ -115,7 +115,6 @@ func (runtime *Runtime) Register(container *Container) error { return err } container.runtime = runtime - container.networkManager = runtime.networkManager // FIXME: infer from docker.runtime // Setup state lock (formerly in newState() lock := new(sync.Mutex) container.State.stateChangeLock = lock From 3e8d1dfb69e74414b3466fee3195432f17445bbc Mon Sep 17 00:00:00 2001 From: creack Date: Thu, 21 Mar 2013 03:52:58 -0700 Subject: [PATCH 27/80] Enforce model for the json image format --- graph/image.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/graph/image.go b/graph/image.go index 210ba7cedb..af6cbd0190 100644 --- a/graph/image.go +++ b/graph/image.go @@ -15,10 +15,10 @@ import ( ) type Image struct { - Id string - Parent string - Comment string - Created time.Time + Id string `json:"id"` + Parent string `json:"parent,omitempty"` + Comment string `json:"comment,omitempty"` + Created time.Time `json:"created"` graph *Graph } From edcfd687efb6c00aefc39c07019307b917bd1ec8 Mon Sep 17 00:00:00 2001 From: creack Date: Thu, 21 Mar 2013 03:53:27 -0700 Subject: [PATCH 28/80] POC of push/pull for images, pull works, push do push but without the layer --- commands.go | 169 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) diff --git a/commands.go b/commands.go index b127106ea9..04011207a5 100644 --- a/commands.go +++ b/commands.go @@ -6,8 +6,10 @@ import ( "errors" "fmt" "github.com/dotcloud/docker/auth" + "github.com/dotcloud/docker/graph" "github.com/dotcloud/docker/rcli" "io" + "io/ioutil" "log" "math/rand" "net/http" @@ -400,6 +402,173 @@ func (srv *Server) CmdImport(stdin io.ReadCloser, stdout io.Writer, args ...stri return nil } +func (srv *Server) CmdPush(stdin io.ReadCloser, stdout io.Writer, args ...string) error { + cmd := rcli.Subcmd(stdout, "push", "[OPTIONS] IMAGE", "Push an image to the registry") + if err := cmd.Parse(args); err != nil { + return nil + } + if cmd.NArg() == 0 { + cmd.Usage() + return nil + } + + client := &http.Client{} + if img, err := srv.runtime.graph.Get(cmd.Arg(0)); err != nil { + return nil + } else { + img.WalkHistory(func(img *graph.Image) { + fmt.Fprintf(stdout, "Pushing %s\n", img.Id) + + data := strings.NewReader("{\"id\": \"ddd\", \"created\": \"2013-03-21T00:18:18-07:00\"}") + + req, err := http.NewRequest("PUT", "http://192.168.56.1:5000/v1/images/"+img.Id+"/json", data) + res, err := client.Do(req) + if err != nil || res.StatusCode != 200 { + switch res.StatusCode { + case 400: + fmt.Fprintf(stdout, "Error: Invalid Json\n") + return + default: + fmt.Fprintf(stdout, "Error: Internal server error\n") + return + } + fmt.Fprintf(stdout, "Error trying to push image {%s} (json): %s\n", img.Id, err) + return + } + + req2, err := http.NewRequest("PUT", "http://192.168.56.1:5000/v1/images/"+img.Id+"/layer", nil) + res2, err := client.Do(req2) + if err != nil || res2.StatusCode != 307 { + fmt.Fprintf(stdout, "Error trying to push image {%s} (layer 1): %s\n", img.Id, err) + return + } + url, err := res2.Location() + if err != nil || url == nil { + fmt.Fprintf(stdout, "Fail to retrieve layer storage URL for image {%s}: %s\n", img.Id, err) + return + } + + req3, err := http.NewRequest("PUT", url.String(), data) + res3, err := client.Do(req3) + if err != nil { + fmt.Fprintf(stdout, "Error trying to push image {%s} (layer 2): %s\n", img.Id, err) + return + } + fmt.Fprintf(stdout, "Status code storage: %d\n", res3.StatusCode) + }) + } + return nil +} + +func newImgJson(src []byte) (*graph.Image, error) { + ret := &graph.Image{} + + fmt.Printf("Json string: {%s}\n", src) + // FIXME: Is there a cleaner way to "puryfy" the input json? + src = []byte(strings.Replace(string(src), "null", "\"\"", -1)) + + if err := json.Unmarshal(src, ret); err != nil { + return nil, err + } + return ret, nil +} + +func newMultipleImgJson(src []byte) (map[*graph.Image]graph.Archive, error) { + ret := map[*graph.Image]graph.Archive{} + + fmt.Printf("Json string2: {%s}\n", src) + dec := json.NewDecoder(strings.NewReader(strings.Replace(string(src), "null", "\"\"", -1))) + for { + m := &graph.Image{} + if err := dec.Decode(m); err == io.EOF { + break + } else if err != nil { + return nil, err + } + ret[m] = nil + } + return ret, nil +} + +func getHistory(base_uri, id string) (map[*graph.Image]graph.Archive, error) { + res, err := http.Get(base_uri + id + "/history") + if err != nil { + return nil, fmt.Errorf("Error while getting from the server: %s\n", err) + } + defer res.Body.Close() + + jsonString, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("Error while reading the http response: %s\n", err) + } + + history, err := newMultipleImgJson(jsonString) + if err != nil { + return nil, fmt.Errorf("Error while parsing the json: %s\n", err) + } + return history, nil +} + +func getRemoteImage(base_uri, id string) (*graph.Image, graph.Archive, error) { + // Get the Json + res, err := http.Get(base_uri + id + "/json") + if err != nil { + return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err) + } + defer res.Body.Close() + + jsonString, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, nil, fmt.Errorf("Error while reading the http response: %s\n", err) + } + + img, err := newImgJson(jsonString) + if err != nil { + return nil, nil, fmt.Errorf("Error while parsing the json: %s\n", err) + } + img.Id = id + + // Get the layer + res, err = http.Get(base_uri + id + "/layer") + if err != nil { + return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err) + } + return img, res.Body, nil +} + +func (srv *Server) CmdPulli(stdin io.ReadCloser, stdout io.Writer, args ...string) error { + cmd := rcli.Subcmd(stdout, "pulli", "[OPTIONS] IMAGE", "Pull an image from the registry") + if err := cmd.Parse(args); err != nil { + return nil + } + if cmd.NArg() == 0 { + cmd.Usage() + return nil + } + + // First, retrieve the history + base_uri := "http://192.168.56.1:5000/v1/images/" + + // Now we have the history, remove the images we already have + history, err := getHistory(base_uri, cmd.Arg(0)) + if err != nil { + return err + } + for j := range history { + if !srv.runtime.graph.Exists(j.Id) { + img, layer, err := getRemoteImage(base_uri, j.Id) + if err != nil { + // FIXME: Keep goging in case of error? + return err + } + if err = srv.runtime.graph.Register(layer, img); err != nil { + return err + } + } + } + return nil +} + func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "images", "[OPTIONS] [NAME]", "List images") //limit := cmd.Int("l", 0, "Only show the N most recent versions of each image") From 42cf74d56b0c989474ef00bf542bc4d2103001e6 Mon Sep 17 00:00:00 2001 From: creack Date: Thu, 21 Mar 2013 06:33:29 -0700 Subject: [PATCH 29/80] POC: push/pull are (kinda) working --- commands.go | 87 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 62 insertions(+), 25 deletions(-) diff --git a/commands.go b/commands.go index 14e1500648..531828d122 100644 --- a/commands.go +++ b/commands.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "github.com/dotcloud/docker/auth" - "github.com/dotcloud/docker/graph" "github.com/dotcloud/docker/rcli" "io" "io/ioutil" @@ -14,6 +13,7 @@ import ( "math/rand" "net/http" "net/url" + "path" "runtime" "strconv" "strings" @@ -23,6 +23,7 @@ import ( ) const VERSION = "0.0.3" +const REGISTRY_ENDPOINT = "http://registry-creack.dotcloud.com/v1" func (srv *Server) Name() string { return "docker" @@ -415,52 +416,88 @@ func (srv *Server) CmdPush(stdin io.ReadCloser, stdout io.Writer, args ...string if img, err := srv.runtime.graph.Get(cmd.Arg(0)); err != nil { return nil } else { - img.WalkHistory(func(img *graph.Image) { + img.WalkHistory(func(img *Image) { fmt.Fprintf(stdout, "Pushing %s\n", img.Id) - data := strings.NewReader("{\"id\": \"ddd\", \"created\": \"2013-03-21T00:18:18-07:00\"}") - - req, err := http.NewRequest("PUT", "http://192.168.56.1:5000/v1/images/"+img.Id+"/json", data) + jsonRaw, err := ioutil.ReadFile(path.Join(srv.runtime.graph.Root, img.Id, "json")) + if err != nil { + fmt.Fprintf(stdout, "Error while retreiving the path for {%s}: %s\n", img.Id, err) + return + } + jsonData := strings.NewReader(string(jsonRaw)) + req, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/images/"+img.Id+"/json", jsonData) res, err := client.Do(req) if err != nil || res.StatusCode != 200 { + if res == nil { + fmt.Fprintf(stdout, + "Error: Internal server error trying to push image {%s} (json): %s\n", + img.Id, err) + return + } switch res.StatusCode { + case 204: + fmt.Fprintf(stdout, "Image already on the repository\n") + return case 400: fmt.Fprintf(stdout, "Error: Invalid Json\n") return default: - fmt.Fprintf(stdout, "Error: Internal server error\n") + fmt.Fprintf(stdout, + "Error: Internal server error trying to push image {%s} (json): %s (%d)\n", + img.Id, err, res.StatusCode) return } - fmt.Fprintf(stdout, "Error trying to push image {%s} (json): %s\n", img.Id, err) - return } - req2, err := http.NewRequest("PUT", "http://192.168.56.1:5000/v1/images/"+img.Id+"/layer", nil) + req2, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/images/"+img.Id+"/layer", nil) res2, err := client.Do(req2) if err != nil || res2.StatusCode != 307 { - fmt.Fprintf(stdout, "Error trying to push image {%s} (layer 1): %s\n", img.Id, err) + fmt.Fprintf(stdout, + "Error trying to push image {%s} (layer 1): %s\n", + img.Id, err) return } url, err := res2.Location() if err != nil || url == nil { - fmt.Fprintf(stdout, "Fail to retrieve layer storage URL for image {%s}: %s\n", img.Id, err) + fmt.Fprintf(stdout, + "Fail to retrieve layer storage URL for image {%s}: %s\n", + img.Id, err) return } - - req3, err := http.NewRequest("PUT", url.String(), data) - res3, err := client.Do(req3) + // FIXME: Don't do this :D. Check the S3 requierement and implement chunks of 5MB + layerData2, err := Tar(path.Join(srv.runtime.graph.Root, img.Id, "layer"), Gzip) + layerData, err := Tar(path.Join(srv.runtime.graph.Root, img.Id, "layer"), Gzip) if err != nil { - fmt.Fprintf(stdout, "Error trying to push image {%s} (layer 2): %s\n", img.Id, err) + fmt.Fprintf(stdout, + "Error while retrieving layer for {%s}: %s\n", + img.Id, err) + return + } + req3, err := http.NewRequest("PUT", url.String(), layerData) + tmp, _ := ioutil.ReadAll(layerData2) + req3.ContentLength = int64(len(tmp)) + + req3.TransferEncoding = []string{"none"} + res3, err := client.Do(req3) + if err != nil || res3.StatusCode != 200 { + if res3 == nil { + fmt.Fprintf(stdout, + "Error trying to push image {%s} (layer 2): %s\n", + img.Id, err) + } else { + fmt.Fprintf(stdout, + "Error trying to push image {%s} (layer 2): %s (%d)\n", + img.Id, err, res3.StatusCode) + } return } - fmt.Fprintf(stdout, "Status code storage: %d\n", res3.StatusCode) }) } return nil } -func newImgJson(src []byte) (*graph.Image, error) { - ret := &graph.Image{} +func newImgJson(src []byte) (*Image, error) { + ret := &Image{} fmt.Printf("Json string: {%s}\n", src) // FIXME: Is there a cleaner way to "puryfy" the input json? @@ -472,13 +509,13 @@ func newImgJson(src []byte) (*graph.Image, error) { return ret, nil } -func newMultipleImgJson(src []byte) (map[*graph.Image]graph.Archive, error) { - ret := map[*graph.Image]graph.Archive{} +func newMultipleImgJson(src []byte) (map[*Image]Archive, error) { + ret := map[*Image]Archive{} fmt.Printf("Json string2: {%s}\n", src) dec := json.NewDecoder(strings.NewReader(strings.Replace(string(src), "null", "\"\"", -1))) for { - m := &graph.Image{} + m := &Image{} if err := dec.Decode(m); err == io.EOF { break } else if err != nil { @@ -489,7 +526,7 @@ func newMultipleImgJson(src []byte) (map[*graph.Image]graph.Archive, error) { return ret, nil } -func getHistory(base_uri, id string) (map[*graph.Image]graph.Archive, error) { +func getHistory(base_uri, id string) (map[*Image]Archive, error) { res, err := http.Get(base_uri + id + "/history") if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) @@ -508,7 +545,7 @@ func getHistory(base_uri, id string) (map[*graph.Image]graph.Archive, error) { return history, nil } -func getRemoteImage(base_uri, id string) (*graph.Image, graph.Archive, error) { +func getRemoteImage(base_uri, id string) (*Image, Archive, error) { // Get the Json res, err := http.Get(base_uri + id + "/json") if err != nil { @@ -546,7 +583,7 @@ func (srv *Server) CmdPulli(stdin io.ReadCloser, stdout io.Writer, args ...strin } // First, retrieve the history - base_uri := "http://192.168.56.1:5000/v1/images/" + base_uri := REGISTRY_ENDPOINT + "/images/" // Now we have the history, remove the images we already have history, err := getHistory(base_uri, cmd.Arg(0)) @@ -622,7 +659,7 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri } } } - // Display images which aren't part of a + // Display images which aren't part of a if nameFilter == "" { for id, image := range allImages { if !*quiet { From 680f40c37e0d7f2fc924a51bc40c0bfc37f60b9e Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 12:18:47 -0700 Subject: [PATCH 30/80] graph.RepoStore: first draft of a Repository/Tag on top of the graph --- graph/tags.go | 100 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 graph/tags.go diff --git a/graph/tags.go b/graph/tags.go new file mode 100644 index 0000000000..337dc67e14 --- /dev/null +++ b/graph/tags.go @@ -0,0 +1,100 @@ +package graph + +import ( + "encoding/json" + "io/ioutil" + "path/filepath" +) + +type RepoStore struct { + path string + graph *Graph + Repositories map[string]*Repository +} + +type Repository struct { + Tags map[string]string +} + +func NewRepoStore(path string, graph *Graph) (*RepoStore, error) { + abspath, err := filepath.Abs(path) + if err != nil { + return nil, err + } + store := &RepoStore{ + path: abspath, + graph: graph, + Repositories: make(map[string]*Repository), + } + if err := store.Reload(); err != nil { + return nil, err + } + return store, nil +} + +func (store *RepoStore) Save() error { + // Store the json ball + jsonData, err := json.Marshal(store) + if err != nil { + return err + } + if err := ioutil.WriteFile(store.path, jsonData, 0600); err != nil { + return err + } + return nil +} + +func (store *RepoStore) Reload() error { + jsonData, err := ioutil.ReadFile(store.path) + if err != nil { + return err + } + if err := json.Unmarshal(jsonData, store); err != nil { + return err + } + return nil +} + +func (store *RepoStore) SetTag(repoName, tag, revision string) error { + if err := store.Reload(); err != nil { + return err + } + var repo *Repository + if r, exists := store.Repositories[repoName]; exists { + repo = r + } else { + repo = NewRepository() + store.Repositories[repoName] = repo + } + repo.Tags[tag] = revision + return store.Save() +} + +func (store *RepoStore) Get(repoName string) (*Repository, error) { + if err := store.Reload(); err != nil { + return nil, err + } + if r, exists := store.Repositories[repoName]; exists { + return r, nil + } + return nil, nil +} + +func (store *RepoStore) GetImage(repoName, tag string) (*Image, error) { + repo, err := store.Get(repoName) + if err != nil { + return nil, err + } else if repo == nil { + return nil, nil + } + if revision, exists := repo.Tags[tag]; exists { + return store.graph.Get(revision) + } + return nil, nil +} + +func NewRepository() *Repository { + return &Repository{ + Tags: make(map[string]string), + } +} From 44faa07b6c472b888bed65b56126cde56658e91a Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 17:35:49 -0700 Subject: [PATCH 31/80] First integration of runtime with repositories & tags --- commands.go | 144 +++++++++++++++++++++++++----------------------- graph/graph.go | 13 +++++ graph/tags.go | 40 ++++++-------- runtime.go | 10 +++- runtime_test.go | 2 +- 5 files changed, 113 insertions(+), 96 deletions(-) diff --git a/commands.go b/commands.go index b127106ea9..e0d4fd0390 100644 --- a/commands.go +++ b/commands.go @@ -12,7 +12,6 @@ import ( "math/rand" "net/http" "net/url" - "path" "runtime" "strconv" "strings" @@ -356,32 +355,26 @@ func (srv *Server) CmdKill(stdin io.ReadCloser, stdout io.Writer, args ...string } func (srv *Server) CmdImport(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "import", "[OPTIONS] NAME", "Create a new filesystem image from the contents of a tarball") - fl_stdin := cmd.Bool("stdin", false, "Read tarball from stdin") + cmd := rcli.Subcmd(stdout, "import", "[OPTIONS] URL|- [REPOSITORY [TAG]]", "Create a new filesystem image from the contents of a tarball") var archive io.Reader var resp *http.Response if err := cmd.Parse(args); err != nil { return nil } - name := cmd.Arg(0) - if name == "" { + src := cmd.Arg(0) + if src == "" { return errors.New("Not enough arguments") - } - if *fl_stdin { + } else if src == "-" { archive = stdin } else { - u, err := url.Parse(name) + u, err := url.Parse(src) if err != nil { return err } if u.Scheme == "" { u.Scheme = "http" } - if u.Host == "" { - u.Host = "get.docker.io" - u.Path = path.Join("/images", u.Path) - } fmt.Fprintf(stdout, "Downloading from %s\n", u.String()) // Download with curl (pretty progress bar) // If curl is not available, fallback to http.Get() @@ -391,11 +384,17 @@ func (srv *Server) CmdImport(stdin io.ReadCloser, stdout io.Writer, args ...stri } archive = ProgressReader(resp.Body, int(resp.ContentLength), stdout) } - fmt.Fprintf(stdout, "Unpacking to %s\n", name) - img, err := srv.runtime.graph.Create(archive, "", "") + img, err := srv.runtime.graph.Create(archive, "", "Imported from "+src) if err != nil { return err } + // Optionally register the image at REPO/TAG + if repository := cmd.Arg(1); repository != "" { + tag := cmd.Arg(2) // Repository will handle an empty tag properly + if err := srv.runtime.repositories.Set(repository, tag, img.Id); err != nil { + return err + } + } fmt.Fprintln(stdout, img.Id) return nil } @@ -411,68 +410,75 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri cmd.Usage() return nil } - /* - var nameFilter string - if cmd.NArg() == 1 { - nameFilter = cmd.Arg(0) - } - */ + var nameFilter string + if cmd.NArg() == 1 { + nameFilter = cmd.Arg(0) + } w := tabwriter.NewWriter(stdout, 20, 1, 3, ' ', 0) if !*quiet { - fmt.Fprintf(w, "NAME\tID\tCREATED\tPARENT\n") + fmt.Fprintf(w, "REPOSITORY\tTAG\tID\tCREATED\tPARENT\n") } - if *quiet { - images, err := srv.runtime.graph.All() - if err != nil { - return err + allImages, err := srv.runtime.graph.Map() + if err != nil { + return err + } + for name, repository := range srv.runtime.repositories.Repositories { + if nameFilter != "" && name != nameFilter { + continue } - for _, image := range images { - fmt.Fprintln(stdout, image.Id) + for tag, id := range repository { + image, err := srv.runtime.graph.Get(id) + if err != nil { + log.Printf("Warning: couldn't load %s from %s/%s: %s", id, name, tag, err) + continue + } + delete(allImages, id) + if !*quiet { + for idx, field := range []string{ + /* REPOSITORY */ name, + /* TAG */ tag, + /* ID */ id, + /* CREATED */ HumanDuration(time.Now().Sub(image.Created)) + " ago", + /* PARENT */ image.Parent, + } { + if idx == 0 { + w.Write([]byte(field)) + } else { + w.Write([]byte("\t" + field)) + } + } + w.Write([]byte{'\n'}) + } else { + stdout.Write([]byte(image.Id + "\n")) + } } - } else { - // FIXME: - // paths, err := srv.images.Paths() - // if err != nil { - // return err - // } - // for _, name := range paths { - // if nameFilter != "" && nameFilter != name { - // continue - // } - // ids, err := srv.images.List(name) - // if err != nil { - // return err - // } - // for idx, img := range ids { - // if *limit > 0 && idx >= *limit { - // break - // } - // if !*quiet { - // for idx, field := range []string{ - // /* NAME */ name, - // /* ID */ img.Id, - // /* CREATED */ HumanDuration(time.Now().Sub(time.Unix(img.Created, 0))) + " ago", - // /* PARENT */ img.Parent, - // } { - // if idx == 0 { - // w.Write([]byte(field)) - // } else { - // w.Write([]byte("\t" + field)) - // } - // } - // w.Write([]byte{'\n'}) - // } else { - // stdout.Write([]byte(img.Id + "\n")) - // } - // } - // } - // if !*quiet { - // w.Flush() - // } - // + } + // Display images which aren't part of a + if nameFilter != "" { + for id, image := range allImages { + if !*quiet { + for idx, field := range []string{ + /* REPOSITORY */ "", + /* TAG */ "", + /* ID */ id, + /* CREATED */ HumanDuration(time.Now().Sub(image.Created)) + " ago", + /* PARENT */ image.Parent, + } { + if idx == 0 { + w.Write([]byte(field)) + } else { + w.Write([]byte("\t" + field)) + } + } + } else { + stdout.Write([]byte(image.Id + "\n")) + } + } + } + if !*quiet { + w.Flush() } return nil - } func (srv *Server) CmdPs(stdin io.ReadCloser, stdout io.Writer, args ...string) error { diff --git a/graph/graph.go b/graph/graph.go index 7e0bc810cd..6470e17aea 100644 --- a/graph/graph.go +++ b/graph/graph.go @@ -122,6 +122,19 @@ func (graph *Graph) GarbageCollect() error { return os.RemoveAll(garbage.Root) } +func (graph *Graph) Map() (map[string]*Image, error) { + // FIXME: this should replace All() + all, err := graph.All() + if err != nil { + return nil, err + } + images := make(map[string]*Image, len(all)) + for _, image := range all { + images[image.Id] = image + } + return images, nil +} + func (graph *Graph) All() ([]*Image, error) { files, err := ioutil.ReadDir(graph.Root) if err != nil { diff --git a/graph/tags.go b/graph/tags.go index 337dc67e14..f3d702fcf7 100644 --- a/graph/tags.go +++ b/graph/tags.go @@ -6,33 +6,31 @@ import ( "path/filepath" ) -type RepoStore struct { +type TagStore struct { path string graph *Graph - Repositories map[string]*Repository + Repositories map[string]Repository } -type Repository struct { - Tags map[string]string -} +type Repository map[string]string -func NewRepoStore(path string, graph *Graph) (*RepoStore, error) { +func NewTagStore(path string, graph *Graph) (*TagStore, error) { abspath, err := filepath.Abs(path) if err != nil { return nil, err } - store := &RepoStore{ + store := &TagStore{ path: abspath, graph: graph, - Repositories: make(map[string]*Repository), + Repositories: make(map[string]Repository), } - if err := store.Reload(); err != nil { + if err := store.Save(); err != nil { return nil, err } return store, nil } -func (store *RepoStore) Save() error { +func (store *TagStore) Save() error { // Store the json ball jsonData, err := json.Marshal(store) if err != nil { @@ -44,7 +42,7 @@ func (store *RepoStore) Save() error { return nil } -func (store *RepoStore) Reload() error { +func (store *TagStore) Reload() error { jsonData, err := ioutil.ReadFile(store.path) if err != nil { return err @@ -55,22 +53,22 @@ func (store *RepoStore) Reload() error { return nil } -func (store *RepoStore) SetTag(repoName, tag, revision string) error { +func (store *TagStore) Set(repoName, tag, revision string) error { if err := store.Reload(); err != nil { return err } - var repo *Repository + var repo Repository if r, exists := store.Repositories[repoName]; exists { repo = r } else { - repo = NewRepository() + repo = make(map[string]string) store.Repositories[repoName] = repo } - repo.Tags[tag] = revision + repo[tag] = revision return store.Save() } -func (store *RepoStore) Get(repoName string) (*Repository, error) { +func (store *TagStore) Get(repoName string) (Repository, error) { if err := store.Reload(); err != nil { return nil, err } @@ -80,21 +78,15 @@ func (store *RepoStore) Get(repoName string) (*Repository, error) { return nil, nil } -func (store *RepoStore) GetImage(repoName, tag string) (*Image, error) { +func (store *TagStore) GetImage(repoName, tag string) (*Image, error) { repo, err := store.Get(repoName) if err != nil { return nil, err } else if repo == nil { return nil, nil } - if revision, exists := repo.Tags[tag]; exists { + if revision, exists := repo[tag]; exists { return store.graph.Get(revision) } return nil, nil } - -func NewRepository() *Repository { - return &Repository{ - Tags: make(map[string]string), - } -} diff --git a/runtime.go b/runtime.go index 8d017df690..ad543f0ed8 100644 --- a/runtime.go +++ b/runtime.go @@ -20,6 +20,7 @@ type Runtime struct { containers *list.List networkManager *NetworkManager graph *graph.Graph + repositories *graph.TagStore } var sysInitPath string @@ -201,10 +202,14 @@ func NewFromDirectory(root string) (*Runtime, error) { return nil, err } - graph, err := graph.New(path.Join(root, "graph")) + g, err := graph.New(path.Join(root, "graph")) if err != nil { return nil, err } + repositories, err := graph.NewTagStore(path.Join(root, "repositories"), g) + if err != nil { + return nil, fmt.Errorf("Couldn't create Tag store: %s", err) + } netManager, err := newNetworkManager(networkBridgeIface) if err != nil { return nil, err @@ -215,7 +220,8 @@ func NewFromDirectory(root string) (*Runtime, error) { repository: runtime_repo, containers: list.New(), networkManager: netManager, - graph: graph, + graph: g, + repositories: repositories, } if err := runtime.restore(); err != nil { diff --git a/runtime_test.go b/runtime_test.go index 093dae67fd..f97d049813 100644 --- a/runtime_test.go +++ b/runtime_test.go @@ -11,7 +11,7 @@ import ( ) const testLayerPath string = "/var/lib/docker/docker-ut.tar" -const unitTestImageName string = "busybox" +const unitTestImageName string = "http://get.docker.io/images/busybox" var unitTestStoreBase string var srv *Server From ef711962d5d70b4332908f0ca6460929e289c131 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 17:47:23 -0700 Subject: [PATCH 32/80] Folded graph/ back into main package --- graph/archive.go => archive.go | 2 +- graph/archive_test.go => archive_test.go | 2 +- graph/changes.go => changes.go | 2 +- commands.go | 2 +- container.go | 22 ++++++++-------------- graph/graph.go => graph.go | 8 ++++---- graph/graph_test.go => graph_test.go | 6 +++--- graph/image.go => image.go | 2 +- graph/mount.go => mount.go | 2 +- graph/mount_darwin.go => mount_darwin.go | 2 +- graph/mount_linux.go => mount_linux.go | 2 +- runtime.go | 15 +++++++-------- runtime_test.go | 13 ++++++------- graph/tags.go => tags.go | 2 +- 14 files changed, 37 insertions(+), 45 deletions(-) rename graph/archive.go => archive.go (99%) rename graph/archive_test.go => archive_test.go (98%) rename graph/changes.go => changes.go (99%) rename graph/graph.go => graph.go (95%) rename graph/graph_test.go => graph_test.go (98%) rename graph/image.go => image.go (99%) rename graph/mount.go => mount.go (98%) rename graph/mount_darwin.go => mount_darwin.go (92%) rename graph/mount_linux.go => mount_linux.go (92%) rename graph/tags.go => tags.go (99%) diff --git a/graph/archive.go b/archive.go similarity index 99% rename from graph/archive.go rename to archive.go index b6281bf4b5..78d4dfca0b 100644 --- a/graph/archive.go +++ b/archive.go @@ -1,4 +1,4 @@ -package graph +package docker import ( "errors" diff --git a/graph/archive_test.go b/archive_test.go similarity index 98% rename from graph/archive_test.go rename to archive_test.go index c158537a6e..9f00aeccd7 100644 --- a/graph/archive_test.go +++ b/archive_test.go @@ -1,4 +1,4 @@ -package graph +package docker import ( "io/ioutil" diff --git a/graph/changes.go b/changes.go similarity index 99% rename from graph/changes.go rename to changes.go index eebf7657e7..4c79918887 100644 --- a/graph/changes.go +++ b/changes.go @@ -1,4 +1,4 @@ -package graph +package docker import ( "fmt" diff --git a/commands.go b/commands.go index e0d4fd0390..78d24ad0a0 100644 --- a/commands.go +++ b/commands.go @@ -803,7 +803,7 @@ func NewServer() (*Server, error) { // if err != nil { // return nil, err // } - runtime, err := New() + runtime, err := NewRuntime() if err != nil { return nil, err } diff --git a/container.go b/container.go index 2dcc3d0e28..ea091b8fa5 100644 --- a/container.go +++ b/container.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/dotcloud/docker/graph" "github.com/kr/pty" "io" "io/ioutil" @@ -63,11 +62,6 @@ type NetworkSettings struct { PortMapping map[string]string } -func GenerateId() string { - return graph.GenerateId() // Re-use the same code to generate container and image IDs - // (this might change when image Ids become content-based) -} - func (container *Container) Cmd() *exec.Cmd { return container.cmd } @@ -376,15 +370,15 @@ func (container *Container) Wait() int { return container.State.ExitCode } -func (container *Container) ExportRw() (graph.Archive, error) { - return graph.Tar(container.rwPath(), graph.Uncompressed) +func (container *Container) ExportRw() (Archive, error) { + return Tar(container.rwPath(), Uncompressed) } -func (container *Container) Export() (graph.Archive, error) { +func (container *Container) Export() (Archive, error) { if err := container.EnsureMounted(); err != nil { return nil, err } - return graph.Tar(container.RootfsPath(), graph.Uncompressed) + return Tar(container.RootfsPath(), Uncompressed) } func (container *Container) WaitTimeout(timeout time.Duration) error { @@ -420,7 +414,7 @@ func (container *Container) Mount() error { return image.Mount(container.RootfsPath(), container.rwPath()) } -func (container *Container) Changes() ([]graph.Change, error) { +func (container *Container) Changes() ([]Change, error) { image, err := container.GetImage() if err != nil { return nil, err @@ -428,7 +422,7 @@ func (container *Container) Changes() ([]graph.Change, error) { return image.Changes(container.rwPath()) } -func (container *Container) GetImage() (*graph.Image, error) { +func (container *Container) GetImage() (*Image, error) { if container.runtime == nil { return nil, fmt.Errorf("Can't get image of unregistered container") } @@ -436,11 +430,11 @@ func (container *Container) GetImage() (*graph.Image, error) { } func (container *Container) Mounted() (bool, error) { - return graph.Mounted(container.RootfsPath()) + return Mounted(container.RootfsPath()) } func (container *Container) Unmount() error { - return graph.Unmount(container.RootfsPath()) + return Unmount(container.RootfsPath()) } func (container *Container) logPath(name string) string { diff --git a/graph/graph.go b/graph.go similarity index 95% rename from graph/graph.go rename to graph.go index 6470e17aea..5001852c32 100644 --- a/graph/graph.go +++ b/graph.go @@ -1,4 +1,4 @@ -package graph +package docker import ( "fmt" @@ -13,7 +13,7 @@ type Graph struct { Root string } -func New(root string) (*Graph, error) { +func NewGraph(root string) (*Graph, error) { abspath, err := filepath.Abs(root) if err != nil { return nil, err @@ -84,7 +84,7 @@ func (graph *Graph) Register(layerData Archive, img *Image) error { } func (graph *Graph) Mktemp(id string) (string, error) { - tmp, err := New(path.Join(graph.Root, ":tmp:")) + tmp, err := NewGraph(path.Join(graph.Root, ":tmp:")) if err != nil { return "", fmt.Errorf("Couldn't create temp: %s", err) } @@ -95,7 +95,7 @@ func (graph *Graph) Mktemp(id string) (string, error) { } func (graph *Graph) Garbage() (*Graph, error) { - return New(path.Join(graph.Root, ":garbage:")) + return NewGraph(path.Join(graph.Root, ":garbage:")) } func (graph *Graph) Delete(id string) error { diff --git a/graph/graph_test.go b/graph_test.go similarity index 98% rename from graph/graph_test.go rename to graph_test.go index d634ea85d5..20e879e52c 100644 --- a/graph/graph_test.go +++ b/graph_test.go @@ -1,4 +1,4 @@ -package graph +package docker import ( "archive/tar" @@ -28,7 +28,7 @@ func TestInit(t *testing.T) { // FIXME: Do more extensive tests (ex: create multiple, delete, recreate; // create multiple, check the amount of images and paths, etc..) -func TestCreate(t *testing.T) { +func TestGraphCreate(t *testing.T) { graph := tempGraph(t) defer os.RemoveAll(graph.Root) archive, err := fakeTar() @@ -177,7 +177,7 @@ func tempGraph(t *testing.T) *Graph { if err != nil { t.Fatal(err) } - graph, err := New(tmp) + graph, err := NewGraph(tmp) if err != nil { t.Fatal(err) } diff --git a/graph/image.go b/image.go similarity index 99% rename from graph/image.go rename to image.go index 210ba7cedb..e508a164ea 100644 --- a/graph/image.go +++ b/image.go @@ -1,4 +1,4 @@ -package graph +package docker import ( "bytes" diff --git a/graph/mount.go b/mount.go similarity index 98% rename from graph/mount.go rename to mount.go index ebf8485285..bb1a40eddb 100644 --- a/graph/mount.go +++ b/mount.go @@ -1,4 +1,4 @@ -package graph +package docker import ( "fmt" diff --git a/graph/mount_darwin.go b/mount_darwin.go similarity index 92% rename from graph/mount_darwin.go rename to mount_darwin.go index 1fb24390b3..aeac78cda5 100644 --- a/graph/mount_darwin.go +++ b/mount_darwin.go @@ -1,4 +1,4 @@ -package graph +package docker import "errors" diff --git a/graph/mount_linux.go b/mount_linux.go similarity index 92% rename from graph/mount_linux.go rename to mount_linux.go index fd849821cf..0efb253003 100644 --- a/graph/mount_linux.go +++ b/mount_linux.go @@ -1,4 +1,4 @@ -package graph +package docker import "syscall" diff --git a/runtime.go b/runtime.go index ad543f0ed8..2c15c35937 100644 --- a/runtime.go +++ b/runtime.go @@ -3,7 +3,6 @@ package docker import ( "container/list" "fmt" - "github.com/dotcloud/docker/graph" "io" "io/ioutil" "log" @@ -19,8 +18,8 @@ type Runtime struct { repository string containers *list.List networkManager *NetworkManager - graph *graph.Graph - repositories *graph.TagStore + graph *Graph + repositories *TagStore } var sysInitPath string @@ -191,22 +190,22 @@ func (runtime *Runtime) restore() error { return nil } -func New() (*Runtime, error) { - return NewFromDirectory("/var/lib/docker") +func NewRuntime() (*Runtime, error) { + return NewRuntimeFromDirectory("/var/lib/docker") } -func NewFromDirectory(root string) (*Runtime, error) { +func NewRuntimeFromDirectory(root string) (*Runtime, error) { runtime_repo := path.Join(root, "containers") if err := os.MkdirAll(runtime_repo, 0700); err != nil && !os.IsExist(err) { return nil, err } - g, err := graph.New(path.Join(root, "graph")) + g, err := NewGraph(path.Join(root, "graph")) if err != nil { return nil, err } - repositories, err := graph.NewTagStore(path.Join(root, "repositories"), g) + repositories, err := NewTagStore(path.Join(root, "repositories"), g) if err != nil { return nil, fmt.Errorf("Couldn't create Tag store: %s", err) } diff --git a/runtime_test.go b/runtime_test.go index f97d049813..fe3ab4810f 100644 --- a/runtime_test.go +++ b/runtime_test.go @@ -1,7 +1,6 @@ package docker import ( - "github.com/dotcloud/docker/graph" "io" "io/ioutil" "os" @@ -57,7 +56,7 @@ func init() { unitTestStoreBase = root // Make it our Store root - runtime, err := NewFromDirectory(root) + runtime, err := NewRuntimeFromDirectory(root) if err != nil { panic(err) } @@ -84,7 +83,7 @@ func newTestRuntime() (*Runtime, error) { return nil, err } - runtime, err := NewFromDirectory(root) + runtime, err := NewRuntimeFromDirectory(root) if err != nil { return nil, err } @@ -92,7 +91,7 @@ func newTestRuntime() (*Runtime, error) { return runtime, nil } -func GetTestImage(runtime *Runtime) *graph.Image { +func GetTestImage(runtime *Runtime) *Image { imgs, err := runtime.graph.All() if err != nil { panic(err) @@ -102,7 +101,7 @@ func GetTestImage(runtime *Runtime) *graph.Image { return imgs[0] } -func TestCreate(t *testing.T) { +func TestRuntimeCreate(t *testing.T) { runtime, err := newTestRuntime() if err != nil { t.Fatal(err) @@ -269,7 +268,7 @@ func TestRestore(t *testing.T) { t.Fatal(err) } - runtime1, err := NewFromDirectory(root) + runtime1, err := NewRuntimeFromDirectory(root) if err != nil { t.Fatal(err) } @@ -294,7 +293,7 @@ func TestRestore(t *testing.T) { // Here are are simulating a docker restart - that is, reloading all containers // from scratch - runtime2, err := NewFromDirectory(root) + runtime2, err := NewRuntimeFromDirectory(root) if err != nil { t.Fatal(err) } diff --git a/graph/tags.go b/tags.go similarity index 99% rename from graph/tags.go rename to tags.go index f3d702fcf7..908691a8f9 100644 --- a/graph/tags.go +++ b/tags.go @@ -1,4 +1,4 @@ -package graph +package docker import ( "encoding/json" From 4af8b711c0b86b38a851ec19b2a00e1317496924 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 18:59:12 -0700 Subject: [PATCH 33/80] Fixed output quirks in 'docker images' --- commands.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/commands.go b/commands.go index 78d24ad0a0..4b23204854 100644 --- a/commands.go +++ b/commands.go @@ -454,7 +454,7 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri } } // Display images which aren't part of a - if nameFilter != "" { + if nameFilter == "" { for id, image := range allImages { if !*quiet { for idx, field := range []string{ @@ -470,6 +470,7 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri w.Write([]byte("\t" + field)) } } + w.Write([]byte{'\n'}) } else { stdout.Write([]byte(image.Id + "\n")) } From d0c776528b50f884c970cf0563ef6b40d90efc44 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 19:00:43 -0700 Subject: [PATCH 34/80] Fix a bug which caused repository metadata to be cleared at startup --- tags.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tags.go b/tags.go index 908691a8f9..55a5135860 100644 --- a/tags.go +++ b/tags.go @@ -3,6 +3,7 @@ package docker import ( "encoding/json" "io/ioutil" + "os" "path/filepath" ) @@ -24,7 +25,12 @@ func NewTagStore(path string, graph *Graph) (*TagStore, error) { graph: graph, Repositories: make(map[string]Repository), } - if err := store.Save(); err != nil { + // Load the json file if it exists, otherwise create it. + if err := store.Reload(); os.IsNotExist(err) { + if err := store.Save(); err != nil { + return nil, err + } + } else if err != nil { return nil, err } return store, nil From 379d449c4417e505a3ea95a121680b318b757c46 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 19:01:55 -0700 Subject: [PATCH 35/80] 'docker run' can reference an image by REPOSITORY:TAG --- graph.go | 1 + runtime.go | 39 +++++++++++++++++++++++++++++++++------ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/graph.go b/graph.go index 5001852c32..b0440518ad 100644 --- a/graph.go +++ b/graph.go @@ -35,6 +35,7 @@ func (graph *Graph) Exists(id string) bool { } func (graph *Graph) Get(id string) (*Image, error) { + // FIXME: return nil when the image doesn't exist, instead of an error img, err := LoadImage(graph.imageRoot(id)) if err != nil { return nil, err diff --git a/runtime.go b/runtime.go index 2c15c35937..9aa213f7e2 100644 --- a/runtime.go +++ b/runtime.go @@ -9,6 +9,7 @@ import ( "os" "path" "sort" + "strings" "sync" "time" ) @@ -62,15 +63,41 @@ func (runtime *Runtime) containerRoot(id string) string { return path.Join(runtime.repository, id) } +func (runtime *Runtime) LookupImage(name string) (*Image, error) { + img, err := runtime.graph.Get(name) + if err != nil { + // FIXME: standardize on returning nil when the image doesn't exist, and err for everything else + // (so we can pass all errors here) + repoAndTag := strings.SplitN(name, ":", 2) + if len(repoAndTag) == 1 { + repoAndTag = append(repoAndTag, "") + } + if i, err := runtime.repositories.GetImage(repoAndTag[0], repoAndTag[1]); err != nil { + return nil, err + } else if i == nil { + return nil, fmt.Errorf("No such image: %s", name) + } else { + img = i + } + } + return img, nil +} + func (runtime *Runtime) Create(command string, args []string, image string, config *Config) (*Container, error) { + // Lookup image + img, err := runtime.LookupImage(image) + if err != nil { + return nil, err + } container := &Container{ // FIXME: we should generate the ID here instead of receiving it as an argument - Id: GenerateId(), - Created: time.Now(), - Path: command, - Args: args, - Config: config, - Image: image, + Id: GenerateId(), + Created: time.Now(), + Path: command, + Args: args, + Config: config, + Image: img.Id, // Always use the resolved image id + //FIXME: store the name under which the image was given, for reference NetworkSettings: &NetworkSettings{}, // FIXME: do we need to store this in the container? SysInitPath: sysInitPath, From 49a78929c61cbab0d25b49fdee43beac62fc9f66 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 20:06:20 -0700 Subject: [PATCH 36/80] Repositories and tags can't have ':' in their name (to allow parsing the REPO:TAG notation) --- tags.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tags.go b/tags.go index 55a5135860..a26a2f196d 100644 --- a/tags.go +++ b/tags.go @@ -2,9 +2,11 @@ package docker import ( "encoding/json" + "fmt" "io/ioutil" "os" "path/filepath" + "strings" ) type TagStore struct { @@ -60,6 +62,12 @@ func (store *TagStore) Reload() error { } func (store *TagStore) Set(repoName, tag, revision string) error { + if strings.Contains(repoName, ":") { + return fmt.Errorf("Illegal repository name: %s", repoName) + } + if strings.Contains(repoName, ":") { + return fmt.Errorf("Illegal tag name: %s", tag) + } if err := store.Reload(); err != nil { return err } From 8396798ebae99791ce3f2f43c276af8e6b96d782 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 20:07:37 -0700 Subject: [PATCH 37/80] 'docker commit' can optionally tag the new image into a repository --- commands.go | 27 ++++++++------------------- runtime.go | 27 +++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/commands.go b/commands.go index 4b23204854..7a6e4b332c 100644 --- a/commands.go +++ b/commands.go @@ -531,33 +531,22 @@ func (srv *Server) CmdPs(stdin io.ReadCloser, stdout io.Writer, args ...string) func (srv *Server) CmdCommit(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, - "commit", "[OPTIONS] CONTAINER [DEST]", + "commit", "[OPTIONS] CONTAINER [REPOSITORY [TAG]]", "Create a new image from a container's changes") if err := cmd.Parse(args); err != nil { return nil } - containerName, imgName := cmd.Arg(0), cmd.Arg(1) - if containerName == "" || imgName == "" { + containerName, repository, tag := cmd.Arg(0), cmd.Arg(1), cmd.Arg(2) + if containerName == "" { cmd.Usage() return nil } - if container := srv.runtime.Get(containerName); container != nil { - // FIXME: freeze the container before copying it to avoid data corruption? - // FIXME: this shouldn't be in commands. - rwTar, err := container.ExportRw() - if err != nil { - return err - } - // Create a new image from the container's base layers + a new layer from container changes - img, err := srv.runtime.graph.Create(rwTar, container.Image, "") - if err != nil { - return err - } - - fmt.Fprintln(stdout, img.Id) - return nil + img, err := srv.runtime.Commit(containerName, repository, tag) + if err != nil { + return err } - return errors.New("No such container: " + containerName) + fmt.Fprintln(stdout, img.Id) + return nil } func (srv *Server) CmdExport(stdin io.ReadCloser, stdout io.Writer, args ...string) error { diff --git a/runtime.go b/runtime.go index 9aa213f7e2..5df3b7b5b8 100644 --- a/runtime.go +++ b/runtime.go @@ -200,6 +200,33 @@ func (runtime *Runtime) Destroy(container *Container) error { return nil } +// Commit creates a new filesystem image from the current state of a container. +// The image can optionally be tagged into a repository +func (runtime *Runtime) Commit(id, repository, tag string) (*Image, error) { + container := runtime.Get(id) + if container == nil { + return nil, fmt.Errorf("No such container: %s", id) + } + // FIXME: freeze the container before copying it to avoid data corruption? + // FIXME: this shouldn't be in commands. + rwTar, err := container.ExportRw() + if err != nil { + return nil, err + } + // Create a new image from the container's base layers + a new layer from container changes + img, err := runtime.graph.Create(rwTar, container.Image, "") + if err != nil { + return nil, err + } + // Register the image if needed + if repository != "" { + if err := runtime.repositories.Set(repository, tag, img.Id); err != nil { + return img, err + } + } + return img, nil +} + func (runtime *Runtime) restore() error { dir, err := ioutil.ReadDir(runtime.repository) if err != nil { From 05ae69a6eb72e012ed35422a6842038a01a24b61 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 21:13:27 -0700 Subject: [PATCH 38/80] 'docker commit' records parent container id and command, in addition to parent image --- commands.go | 2 +- container_test.go | 2 +- graph.go | 8 ++++++-- graph_test.go | 10 +++++----- image.go | 12 +++++++----- runtime.go | 2 +- 6 files changed, 21 insertions(+), 15 deletions(-) diff --git a/commands.go b/commands.go index 7a6e4b332c..4325201f20 100644 --- a/commands.go +++ b/commands.go @@ -384,7 +384,7 @@ func (srv *Server) CmdImport(stdin io.ReadCloser, stdout io.Writer, args ...stri } archive = ProgressReader(resp.Body, int(resp.ContentLength), stdout) } - img, err := srv.runtime.graph.Create(archive, "", "Imported from "+src) + img, err := srv.runtime.graph.Create(archive, nil, "Imported from "+src) if err != nil { return err } diff --git a/container_test.go b/container_test.go index 64558871ac..94db11408d 100644 --- a/container_test.go +++ b/container_test.go @@ -46,7 +46,7 @@ func TestCommitRun(t *testing.T) { if err != nil { t.Error(err) } - img, err := runtime.graph.Create(rwTar, container1.Image, "unit test commited image") + img, err := runtime.graph.Create(rwTar, container1, "unit test commited image") if err != nil { t.Error(err) } diff --git a/graph.go b/graph.go index b0440518ad..c6bb30353a 100644 --- a/graph.go +++ b/graph.go @@ -47,13 +47,17 @@ func (graph *Graph) Get(id string) (*Image, error) { return img, nil } -func (graph *Graph) Create(layerData Archive, parent, comment string) (*Image, error) { +func (graph *Graph) Create(layerData Archive, container *Container, comment string) (*Image, error) { img := &Image{ Id: GenerateId(), - Parent: parent, Comment: comment, Created: time.Now(), } + if container != nil { + img.Parent = container.Image + img.ParentContainer = container.Id + img.ParentCommand = append([]string{container.Path}, container.Args...) + } if err := graph.Register(layerData, img); err != nil { return nil, err } diff --git a/graph_test.go b/graph_test.go index 20e879e52c..b9bdc5140f 100644 --- a/graph_test.go +++ b/graph_test.go @@ -35,7 +35,7 @@ func TestGraphCreate(t *testing.T) { if err != nil { t.Fatal(err) } - image, err := graph.Create(archive, "", "Testing") + image, err := graph.Create(archive, nil, "Testing") if err != nil { t.Fatal(err) } @@ -92,7 +92,7 @@ func TestMount(t *testing.T) { if err != nil { t.Fatal(err) } - image, err := graph.Create(archive, "", "Testing") + image, err := graph.Create(archive, nil, "Testing") if err != nil { t.Fatal(err) } @@ -128,7 +128,7 @@ func TestDelete(t *testing.T) { t.Fatal(err) } assertNImages(graph, t, 0) - img, err := graph.Create(archive, "", "Bla bla") + img, err := graph.Create(archive, nil, "Bla bla") if err != nil { t.Fatal(err) } @@ -139,11 +139,11 @@ func TestDelete(t *testing.T) { assertNImages(graph, t, 0) // Test 2 create (same name) / 1 delete - img1, err := graph.Create(archive, "foo", "Testing") + img1, err := graph.Create(archive, nil, "Testing") if err != nil { t.Fatal(err) } - if _, err = graph.Create(archive, "foo", "Testing"); err != nil { + if _, err = graph.Create(archive, nil, "Testing"); err != nil { t.Fatal(err) } assertNImages(graph, t, 2) diff --git a/image.go b/image.go index e508a164ea..84935e3996 100644 --- a/image.go +++ b/image.go @@ -15,11 +15,13 @@ import ( ) type Image struct { - Id string - Parent string - Comment string - Created time.Time - graph *Graph + Id string + Parent string + Comment string + Created time.Time + ParentContainer string + ParentCommand []string + graph *Graph } func LoadImage(root string) (*Image, error) { diff --git a/runtime.go b/runtime.go index 5df3b7b5b8..dcaaec6366 100644 --- a/runtime.go +++ b/runtime.go @@ -214,7 +214,7 @@ func (runtime *Runtime) Commit(id, repository, tag string) (*Image, error) { return nil, err } // Create a new image from the container's base layers + a new layer from container changes - img, err := runtime.graph.Create(rwTar, container.Image, "") + img, err := runtime.graph.Create(rwTar, container, "") if err != nil { return nil, err } From 1ad69ad415728cc462dd3e296295247449538b0b Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 21:42:18 -0700 Subject: [PATCH 39/80] 'docker history': show the history of an image --- commands.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/commands.go b/commands.go index 4325201f20..b7f92b3a78 100644 --- a/commands.go +++ b/commands.go @@ -35,6 +35,7 @@ func (srv *Server) Help() string { {"import", "Create a new filesystem image from the contents of a tarball"}, {"attach", "Attach to a running container"}, {"commit", "Create a new image from a container's changes"}, + {"history", "Show the history of an image"}, {"diff", "Inspect changes on a container's filesystem"}, {"images", "List images"}, {"info", "Display system-wide information"}, @@ -319,6 +320,27 @@ func (srv *Server) CmdRmi(stdin io.ReadCloser, stdout io.Writer, args ...string) return nil } +func (srv *Server) CmdHistory(stdin io.ReadCloser, stdout io.Writer, args ...string) error { + cmd := rcli.Subcmd(stdout, "history", "[OPTIONS] IMAGE", "Show the history of an image") + if cmd.Parse(args) != nil || cmd.NArg() != 1 { + cmd.Usage() + return nil + } + image, err := srv.runtime.LookupImage(cmd.Arg(0)) + if err != nil { + return err + } + var child *Image + return image.WalkHistory(func(img *Image) { + if child == nil { + fmt.Fprintf(stdout, " %s\n", img.Id) + } else { + fmt.Fprintf(stdout, " = %s + %s\n", img.Id, strings.Join(child.ParentCommand, " ")) + } + child = img + }) +} + func (srv *Server) CmdRm(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "rm", "[OPTIONS] CONTAINER", "Remove a container") if err := cmd.Parse(args); err != nil { From f50dcbe404fd64dd0c69194bab3ea4aecb1d9f0d Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 21 Mar 2013 22:45:22 -0700 Subject: [PATCH 40/80] Image.ParentCommand and Image.ParentConatiner should be stored --- image.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/image.go b/image.go index 546237c983..2f3e6f1b8f 100644 --- a/image.go +++ b/image.go @@ -19,8 +19,8 @@ type Image struct { Parent string `json:"parent,omitempty"` Comment string `json:"comment,omitempty"` Created time.Time `json:"created"` - ParentContainer string `json:-` - ParentCommand []string `json:-` + ParentContainer string `json:"parent_container,omitempty"` + ParentCommand []string `json:"parent_command,omitempty"` graph *Graph } From da266e6c7b66eda9ec6211bbf48fa71c3732d705 Mon Sep 17 00:00:00 2001 From: creack Date: Thu, 21 Mar 2013 10:10:14 -0700 Subject: [PATCH 41/80] Factorize the pull/push commands and create a registry.go --- commands.go | 193 +++---------------------------------------------- image.go | 16 +++-- registry.go | 201 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 220 insertions(+), 190 deletions(-) create mode 100644 registry.go diff --git a/commands.go b/commands.go index c5e321e229..60bfadfffe 100644 --- a/commands.go +++ b/commands.go @@ -8,12 +8,10 @@ import ( "github.com/dotcloud/docker/auth" "github.com/dotcloud/docker/rcli" "io" - "io/ioutil" "log" "math/rand" "net/http" "net/url" - "path" "runtime" "strconv" "strings" @@ -23,7 +21,6 @@ import ( ) const VERSION = "0.0.3" -const REGISTRY_ENDPOINT = "http://registry-creack.dotcloud.com/v1" func (srv *Server) Name() string { return "docker" @@ -434,168 +431,16 @@ func (srv *Server) CmdPush(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } - client := &http.Client{} - if img, err := srv.runtime.graph.Get(cmd.Arg(0)); err != nil { - return nil - } else { - img.WalkHistory(func(img *Image) { - fmt.Fprintf(stdout, "Pushing %s\n", img.Id) - - jsonRaw, err := ioutil.ReadFile(path.Join(srv.runtime.graph.Root, img.Id, "json")) - if err != nil { - fmt.Fprintf(stdout, "Error while retreiving the path for {%s}: %s\n", img.Id, err) - return - } - jsonData := strings.NewReader(string(jsonRaw)) - req, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/images/"+img.Id+"/json", jsonData) - res, err := client.Do(req) - if err != nil || res.StatusCode != 200 { - if res == nil { - fmt.Fprintf(stdout, - "Error: Internal server error trying to push image {%s} (json): %s\n", - img.Id, err) - return - } - switch res.StatusCode { - case 204: - fmt.Fprintf(stdout, "Image already on the repository\n") - return - case 400: - fmt.Fprintf(stdout, "Error: Invalid Json\n") - return - default: - fmt.Fprintf(stdout, - "Error: Internal server error trying to push image {%s} (json): %s (%d)\n", - img.Id, err, res.StatusCode) - return - } - } - - req2, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/images/"+img.Id+"/layer", nil) - res2, err := client.Do(req2) - if err != nil || res2.StatusCode != 307 { - fmt.Fprintf(stdout, - "Error trying to push image {%s} (layer 1): %s\n", - img.Id, err) - return - } - url, err := res2.Location() - if err != nil || url == nil { - fmt.Fprintf(stdout, - "Fail to retrieve layer storage URL for image {%s}: %s\n", - img.Id, err) - return - } - // FIXME: Don't do this :D. Check the S3 requierement and implement chunks of 5MB - layerData2, err := Tar(path.Join(srv.runtime.graph.Root, img.Id, "layer"), Gzip) - layerData, err := Tar(path.Join(srv.runtime.graph.Root, img.Id, "layer"), Gzip) - if err != nil { - fmt.Fprintf(stdout, - "Error while retrieving layer for {%s}: %s\n", - img.Id, err) - return - } - req3, err := http.NewRequest("PUT", url.String(), layerData) - tmp, _ := ioutil.ReadAll(layerData2) - req3.ContentLength = int64(len(tmp)) - - req3.TransferEncoding = []string{"none"} - res3, err := client.Do(req3) - if err != nil || res3.StatusCode != 200 { - if res3 == nil { - fmt.Fprintf(stdout, - "Error trying to push image {%s} (layer 2): %s\n", - img.Id, err) - } else { - fmt.Fprintf(stdout, - "Error trying to push image {%s} (layer 2): %s (%d)\n", - img.Id, err, res3.StatusCode) - } - return - } - }) + img, err := srv.runtime.graph.Get(cmd.Arg(0)) + if err != nil { + return err } - return nil + // FIXME: Handle repositories, etc. Not jist images + return srv.runtime.graph.PushImage(img) } -func newImgJson(src []byte) (*Image, error) { - ret := &Image{} - - fmt.Printf("Json string: {%s}\n", src) - // FIXME: Is there a cleaner way to "puryfy" the input json? - src = []byte(strings.Replace(string(src), "null", "\"\"", -1)) - - if err := json.Unmarshal(src, ret); err != nil { - return nil, err - } - return ret, nil -} - -func newMultipleImgJson(src []byte) (map[*Image]Archive, error) { - ret := map[*Image]Archive{} - - fmt.Printf("Json string2: {%s}\n", src) - dec := json.NewDecoder(strings.NewReader(strings.Replace(string(src), "null", "\"\"", -1))) - for { - m := &Image{} - if err := dec.Decode(m); err == io.EOF { - break - } else if err != nil { - return nil, err - } - ret[m] = nil - } - return ret, nil -} - -func getHistory(base_uri, id string) (map[*Image]Archive, error) { - res, err := http.Get(base_uri + id + "/history") - if err != nil { - return nil, fmt.Errorf("Error while getting from the server: %s\n", err) - } - defer res.Body.Close() - - jsonString, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, fmt.Errorf("Error while reading the http response: %s\n", err) - } - - history, err := newMultipleImgJson(jsonString) - if err != nil { - return nil, fmt.Errorf("Error while parsing the json: %s\n", err) - } - return history, nil -} - -func getRemoteImage(base_uri, id string) (*Image, Archive, error) { - // Get the Json - res, err := http.Get(base_uri + id + "/json") - if err != nil { - return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err) - } - defer res.Body.Close() - - jsonString, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, nil, fmt.Errorf("Error while reading the http response: %s\n", err) - } - - img, err := newImgJson(jsonString) - if err != nil { - return nil, nil, fmt.Errorf("Error while parsing the json: %s\n", err) - } - img.Id = id - - // Get the layer - res, err = http.Get(base_uri + id + "/layer") - if err != nil { - return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err) - } - return img, res.Body, nil -} - -func (srv *Server) CmdPulli(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "pulli", "[OPTIONS] IMAGE", "Pull an image from the registry") +func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string) error { + cmd := rcli.Subcmd(stdout, "pull", "[OPTIONS] IMAGE", "Pull an image from the registry") if err := cmd.Parse(args); err != nil { return nil } @@ -603,28 +448,8 @@ func (srv *Server) CmdPulli(stdin io.ReadCloser, stdout io.Writer, args ...strin cmd.Usage() return nil } - - // First, retrieve the history - base_uri := REGISTRY_ENDPOINT + "/images/" - - // Now we have the history, remove the images we already have - history, err := getHistory(base_uri, cmd.Arg(0)) - if err != nil { - return err - } - for j := range history { - if !srv.runtime.graph.Exists(j.Id) { - img, layer, err := getRemoteImage(base_uri, j.Id) - if err != nil { - // FIXME: Keep goging in case of error? - return err - } - if err = srv.runtime.graph.Register(layer, img); err != nil { - return err - } - } - } - return nil + // FIXME: Handle repositories, etc. Not jist images + return srv.runtime.graph.PullImage(cmd.Arg(0)) } func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...string) error { diff --git a/image.go b/image.go index 2f3e6f1b8f..36bd839f5c 100644 --- a/image.go +++ b/image.go @@ -181,12 +181,12 @@ func ComputeId(content io.Reader) (string, error) { // Image includes convenience proxy functions to its graph // These functions will return an error if the image is not registered // (ie. if image.graph == nil) - func (img *Image) History() ([]*Image, error) { var parents []*Image if err := img.WalkHistory( - func(img *Image) { + func(img *Image) error { parents = append(parents, img) + return nil }, ); err != nil { return nil, err @@ -195,16 +195,19 @@ func (img *Image) History() ([]*Image, error) { } // layers returns all the filesystem layers needed to mount an image +// FIXME: @shykes refactor this function with the new error handling +// (I'll do it if I have time tonight, I focus on the rest) func (img *Image) layers() ([]string, error) { var list []string var e error if err := img.WalkHistory( - func(img *Image) { + func(img *Image) (err error) { if layer, err := img.layer(); err != nil { e = err } else if layer != "" { list = append(list, layer) } + return err }, ); err != nil { return nil, err @@ -217,12 +220,13 @@ func (img *Image) layers() ([]string, error) { return list, nil } -func (img *Image) WalkHistory(handler func(*Image)) error { - var err error +func (img *Image) WalkHistory(handler func(*Image) error) (err error) { currentImg := img for currentImg != nil { if handler != nil { - handler(currentImg) + if err := handler(currentImg); err != nil { + return err + } } currentImg, err = currentImg.GetParent() if err != nil { diff --git a/registry.go b/registry.go new file mode 100644 index 0000000000..4ce3178fbd --- /dev/null +++ b/registry.go @@ -0,0 +1,201 @@ +package docker + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "path" + "strings" +) + +//FIXME: Set the endpoint in a conf file or via commandline +//const REGISTRY_ENDPOINT = "http://registry-creack.dotcloud.com/v1" +const REGISTRY_ENDPOINT = "http://192.168.56.1:5000/v1" + +// Build an Image object from raw json data +func NewImgJson(src []byte) (*Image, error) { + ret := &Image{} + + fmt.Printf("Json string: {%s}\n", src) + // FIXME: Is there a cleaner way to "puryfy" the input json? + src = []byte(strings.Replace(string(src), "null", "\"\"", -1)) + + if err := json.Unmarshal(src, ret); err != nil { + return nil, err + } + return ret, nil +} + +// Build an Image object list from a raw json data +// FIXME: Do this in "stream" mode +func NewMultipleImgJson(src []byte) ([]*Image, error) { + ret := []*Image{} + + dec := json.NewDecoder(strings.NewReader(strings.Replace(string(src), "null", "\"\"", -1))) + for { + m := &Image{} + if err := dec.Decode(m); err == io.EOF { + break + } else if err != nil { + return nil, err + } + ret = append(ret, m) + } + return ret, nil +} + +// Retrieve the history of a given image from the Registry. +// Return a list of the parent's json (requested image included) +func (graph *Graph) getRemoteHistory(imgId string) ([]*Image, error) { + res, err := http.Get(REGISTRY_ENDPOINT + "/images/" + imgId + "/history") + if err != nil { + return nil, fmt.Errorf("Error while getting from the server: %s\n", err) + } + defer res.Body.Close() + + jsonString, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("Error while reading the http response: %s\n", err) + } + + history, err := NewMultipleImgJson(jsonString) + if err != nil { + return nil, fmt.Errorf("Error while parsing the json: %s\n", err) + } + return history, nil +} + +// Retrieve an image from the Registry. +// Returns the Image object as well as the layer as an Archive (io.Reader) +func (graph *Graph) getRemoteImage(imgId string) (*Image, Archive, error) { + // Get the Json + res, err := http.Get(REGISTRY_ENDPOINT + "/images/" + imgId + "/json") + if err != nil { + return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err) + } + defer res.Body.Close() + + jsonString, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, nil, fmt.Errorf("Error while reading the http response: %s\n", err) + } + + img, err := NewImgJson(jsonString) + if err != nil { + return nil, nil, fmt.Errorf("Error while parsing the json: %s\n", err) + } + img.Id = imgId + + // Get the layer + res, err = http.Get(REGISTRY_ENDPOINT + "/images/" + imgId + "/layer") + if err != nil { + return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err) + } + return img, res.Body, nil +} + +func (graph *Graph) PullImage(imgId string) error { + history, err := graph.getRemoteHistory(imgId) + if err != nil { + return err + } + // FIXME: Try to stream the images? + // FIXME: Lunch the getRemoteImage() in goroutines + for _, j := range history { + if !graph.Exists(j.Id) { + img, layer, err := graph.getRemoteImage(j.Id) + if err != nil { + // FIXME: Keep goging in case of error? + return err + } + if err = graph.Register(layer, img); err != nil { + return err + } + } + } + return nil +} + +// Push a local image to the registry with its history if needed +func (graph *Graph) PushImage(imgOrig *Image) error { + client := &http.Client{} + + // FIXME: Factorize the code + // FIXME: Do the puts in goroutines + if err := imgOrig.WalkHistory(func(img *Image) error { + + jsonRaw, err := ioutil.ReadFile(path.Join(graph.Root, img.Id, "json")) + if err != nil { + return fmt.Errorf("Error while retreiving the path for {%s}: %s", img.Id, err) + } + // FIXME: try json with URF8 + jsonData := strings.NewReader(string(jsonRaw)) + req, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/images/"+img.Id+"/json", jsonData) + res, err := client.Do(req) + if err != nil || res.StatusCode != 200 { + if res == nil { + return fmt.Errorf( + "Error: Internal server error trying to push image {%s} (json): %s", + img.Id, err) + } + switch res.StatusCode { + case 204: + // Case where the image is already on the Registry + // FIXME: Do not be silent? + return nil + case 400: + return fmt.Errorf("Error: Invalid Json") + default: + return fmt.Errorf( + "Error: Internal server error trying to push image {%s} (json): %s (%d)\n", + img.Id, err, res.StatusCode) + } + } + + req2, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/images/"+img.Id+"/layer", nil) + res2, err := client.Do(req2) + if err != nil || res2.StatusCode != 307 { + return fmt.Errorf( + "Error trying to push image {%s} (layer 1): %s\n", + img.Id, err) + } + url, err := res2.Location() + if err != nil || url == nil { + return fmt.Errorf( + "Fail to retrieve layer storage URL for image {%s}: %s\n", + img.Id, err) + } + // FIXME: Don't do this :D. Check the S3 requierement and implement chunks of 5MB + // FIXME2: I won't stress it enough, DON'T DO THIS! very high priority + layerData2, err := Tar(path.Join(graph.Root, img.Id, "layer"), Gzip) + layerData, err := Tar(path.Join(graph.Root, img.Id, "layer"), Gzip) + if err != nil { + return fmt.Errorf( + "Error while retrieving layer for {%s}: %s\n", + img.Id, err) + } + req3, err := http.NewRequest("PUT", url.String(), layerData) + tmp, _ := ioutil.ReadAll(layerData2) + req3.ContentLength = int64(len(tmp)) + + req3.TransferEncoding = []string{"none"} + res3, err := client.Do(req3) + if err != nil || res3.StatusCode != 200 { + if res3 == nil { + return fmt.Errorf( + "Error trying to push image {%s} (layer 2): %s\n", + img.Id, err) + } else { + return fmt.Errorf( + "Error trying to push image {%s} (layer 2): %s (%d)\n", + img.Id, err, res3.StatusCode) + } + } + return nil + }); err != nil { + return err + } + return nil +} From f246cc9cdd9091baa5f956f7c8119a49e8379f91 Mon Sep 17 00:00:00 2001 From: creack Date: Thu, 21 Mar 2013 10:12:05 -0700 Subject: [PATCH 42/80] Apply the new WalkHistory prototype to merge --- commands.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/commands.go b/commands.go index 60bfadfffe..d5fcacb9f7 100644 --- a/commands.go +++ b/commands.go @@ -331,13 +331,14 @@ func (srv *Server) CmdHistory(stdin io.ReadCloser, stdout io.Writer, args ...str return err } var child *Image - return image.WalkHistory(func(img *Image) { + return image.WalkHistory(func(img *Image) error { if child == nil { fmt.Fprintf(stdout, " %s\n", img.Id) } else { fmt.Fprintf(stdout, " = %s + %s\n", img.Id, strings.Join(child.ParentCommand, " ")) } child = img + return nil }) } From d8fa52b7b5b6101538c15735500b3a01327ca96f Mon Sep 17 00:00:00 2001 From: creack Date: Thu, 21 Mar 2013 10:31:02 -0700 Subject: [PATCH 43/80] Comply the tests with golang TIP --- .gitignore | 1 + network_test.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 94f63b9f26..a9b5777c6d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ a.out build_src command-line-arguments.test .flymake* +docker.test diff --git a/network_test.go b/network_test.go index c456b54838..251ec158fd 100644 --- a/network_test.go +++ b/network_test.go @@ -102,7 +102,7 @@ func TestConversion(t *testing.T) { func TestIPAllocator(t *testing.T) { gwIP, n, _ := net.ParseCIDR("127.0.0.1/29") - alloc, err := newIPAllocator(&net.IPNet{gwIP, n.Mask}) + alloc, err := newIPAllocator(&net.IPNet{IP: gwIP, Mask: n.Mask}) if err != nil { t.Fatal(err) } From 11c4294846b55914ec5c5b475012a2234be9a9ff Mon Sep 17 00:00:00 2001 From: creack Date: Fri, 22 Mar 2013 01:25:27 -0700 Subject: [PATCH 44/80] Handle push/pull of repositories --- auth/auth.go | 3 +- commands.go | 35 +++++++++++++++---- registry.go | 99 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 128 insertions(+), 9 deletions(-) diff --git a/auth/auth.go b/auth/auth.go index 498b7d8413..2a3d1a439f 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -15,7 +15,8 @@ import ( const CONFIGFILE = "/var/lib/docker/.dockercfg" // the registry server we want to login against -const REGISTRY_SERVER = "https://registry.docker.io" +//const REGISTRY_SERVER = "https://registry.docker.io" +const REGISTRY_SERVER = "http://192.168.56.1:5000" type AuthConfig struct { Username string `json:"username"` diff --git a/commands.go b/commands.go index d5fcacb9f7..2ed36311e9 100644 --- a/commands.go +++ b/commands.go @@ -423,7 +423,8 @@ func (srv *Server) CmdImport(stdin io.ReadCloser, stdout io.Writer, args ...stri } func (srv *Server) CmdPush(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "push", "[OPTIONS] IMAGE", "Push an image to the registry") + cmd := rcli.Subcmd(stdout, "push", "[OPTIONS] IMAGE", "Push an image or a repository to the registry") + user := cmd.String("u", "", "specify the user for the repository") if err := cmd.Parse(args); err != nil { return nil } @@ -432,16 +433,31 @@ func (srv *Server) CmdPush(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } + // Try to get the image + // FIXME: Handle lookup + // FIXME: Also push the tags in case of ./docker push myrepo:mytag + // img, err := srv.runtime.LookupImage(cmd.Arg(0)) img, err := srv.runtime.graph.Get(cmd.Arg(0)) if err != nil { - return err + if *user == "" { + return fmt.Errorf("Not logged in and no user specified\n") + } + // If it fails, try to get the repository + if repo, exists := srv.runtime.repositories.Repositories[cmd.Arg(0)]; exists { + if err := srv.runtime.graph.PushRepository(*user, cmd.Arg(0), repo); err != nil { + return err + } + } else { + return err + } + return nil } - // FIXME: Handle repositories, etc. Not jist images return srv.runtime.graph.PushImage(img) } func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "pull", "[OPTIONS] IMAGE", "Pull an image from the registry") + cmd := rcli.Subcmd(stdout, "pull", "[OPTIONS] IMAGE", "Pull an image or a repository from the registry") + user := cmd.String("u", "", "specify the user for the repository") if err := cmd.Parse(args); err != nil { return nil } @@ -449,8 +465,15 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string cmd.Usage() return nil } - // FIXME: Handle repositories, etc. Not jist images - return srv.runtime.graph.PullImage(cmd.Arg(0)) + + if srv.runtime.graph.LookupRemoteImage(cmd.Arg(0)) { + return srv.runtime.graph.PullImage(cmd.Arg(0)) + } + if *user == "" { + return fmt.Errorf("Not loggin and no user specified\n") + } + // FIXME: Allow pull repo:tag + return srv.runtime.graph.PullRepository(*user, cmd.Arg(0), "", srv.runtime.repositories) } func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...string) error { diff --git a/registry.go b/registry.go index 4ce3178fbd..5a80c8eb78 100644 --- a/registry.go +++ b/registry.go @@ -3,6 +3,7 @@ package docker import ( "encoding/json" "fmt" + "github.com/dotcloud/docker/auth" "io" "io/ioutil" "net/http" @@ -12,7 +13,7 @@ import ( //FIXME: Set the endpoint in a conf file or via commandline //const REGISTRY_ENDPOINT = "http://registry-creack.dotcloud.com/v1" -const REGISTRY_ENDPOINT = "http://192.168.56.1:5000/v1" +const REGISTRY_ENDPOINT = auth.REGISTRY_SERVER + "/v1" // Build an Image object from raw json data func NewImgJson(src []byte) (*Image, error) { @@ -67,6 +68,15 @@ func (graph *Graph) getRemoteHistory(imgId string) ([]*Image, error) { return history, nil } +// Check if an image exists in the Registry +func (graph *Graph) LookupRemoteImage(imgId string) bool { + res, err := http.Get(REGISTRY_ENDPOINT + "/images/" + imgId + "/json") + if err != nil { + return false + } + return res.StatusCode == 307 +} + // Retrieve an image from the Registry. // Returns the Image object as well as the layer as an Archive (io.Reader) func (graph *Graph) getRemoteImage(imgId string) (*Image, Archive, error) { @@ -118,6 +128,46 @@ func (graph *Graph) PullImage(imgId string) error { return nil } +// FIXME: Handle the askedTag parameter +func (graph *Graph) PullRepository(user, repoName, askedTag string, repositories *TagStore) error { + client := &http.Client{} + + req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/users/"+user+"/"+repoName, nil) + if err != nil { + return err + } + authStruct, err := auth.LoadConfig() + if err != nil { + return err + } + + req.SetBasicAuth(authStruct.Username, authStruct.Password) + res, err := client.Do(req) + if err != nil { + return err + } + rawJson, err := ioutil.ReadAll(res.Body) + if err != nil { + return err + } + t := map[string]string{} + if err = json.Unmarshal(rawJson, &t); err != nil { + return err + } + for tag, rev := range t { + if err = graph.PullImage(rev); err != nil { + return err + } + if err = repositories.Set(repoName, tag, rev); err != nil { + return err + } + } + if err = repositories.Save(); err != nil { + return err + } + return nil +} + // Push a local image to the registry with its history if needed func (graph *Graph) PushImage(imgOrig *Image) error { client := &http.Client{} @@ -130,9 +180,12 @@ func (graph *Graph) PushImage(imgOrig *Image) error { if err != nil { return fmt.Errorf("Error while retreiving the path for {%s}: %s", img.Id, err) } - // FIXME: try json with URF8 + // FIXME: try json with UTF8 jsonData := strings.NewReader(string(jsonRaw)) req, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/images/"+img.Id+"/json", jsonData) + if err != nil { + return err + } res, err := client.Do(req) if err != nil || res.StatusCode != 200 { if res == nil { @@ -199,3 +252,45 @@ func (graph *Graph) PushImage(imgOrig *Image) error { } return nil } + +func (graph *Graph) pushTag(user, repo, revision, tag string) error { + + if tag == "" { + tag = "lastest" + } + + revision = "\"" + revision + "\"" + + client := &http.Client{} + req, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/users/"+user+"/"+repo+"/"+tag, strings.NewReader(revision)) + req.Header.Add("Content-type", "application/json") + res, err := client.Do(req) + if err != nil { + return err + } + fmt.Printf("Result of push tag: %d\n", res.StatusCode) + switch res.StatusCode { + default: + return fmt.Errorf("Error %d\n", res.StatusCode) + case 200: + case 201: + } + return nil +} + +func (graph *Graph) PushRepository(user, repoName string, repo Repository) error { + for tag, imgId := range repo { + fmt.Printf("tag: %s, imgId: %s\n", tag, imgId) + img, err := graph.Get(imgId) + if err != nil { + return err + } + if err = graph.PushImage(img); err != nil { + return err + } + if err = graph.pushTag(user, repoName, imgId, tag); err != nil { + return err + } + } + return nil +} From 640026ec59d26172d5beddf97f3894d4374ddd4d Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 22 Mar 2013 16:07:13 -0700 Subject: [PATCH 45/80] Looking up a tag by repository name will default to REPOSITORY:latest. The empty tag '' is no longer allowed. --- commands.go | 2 +- runtime.go | 23 +-------------------- tags.go | 57 ++++++++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 54 insertions(+), 28 deletions(-) diff --git a/commands.go b/commands.go index d5fcacb9f7..26ed1db138 100644 --- a/commands.go +++ b/commands.go @@ -326,7 +326,7 @@ func (srv *Server) CmdHistory(stdin io.ReadCloser, stdout io.Writer, args ...str cmd.Usage() return nil } - image, err := srv.runtime.LookupImage(cmd.Arg(0)) + image, err := srv.runtime.repositories.LookupImage(cmd.Arg(0)) if err != nil { return err } diff --git a/runtime.go b/runtime.go index dcaaec6366..3b16479f3c 100644 --- a/runtime.go +++ b/runtime.go @@ -9,7 +9,6 @@ import ( "os" "path" "sort" - "strings" "sync" "time" ) @@ -63,29 +62,9 @@ func (runtime *Runtime) containerRoot(id string) string { return path.Join(runtime.repository, id) } -func (runtime *Runtime) LookupImage(name string) (*Image, error) { - img, err := runtime.graph.Get(name) - if err != nil { - // FIXME: standardize on returning nil when the image doesn't exist, and err for everything else - // (so we can pass all errors here) - repoAndTag := strings.SplitN(name, ":", 2) - if len(repoAndTag) == 1 { - repoAndTag = append(repoAndTag, "") - } - if i, err := runtime.repositories.GetImage(repoAndTag[0], repoAndTag[1]); err != nil { - return nil, err - } else if i == nil { - return nil, fmt.Errorf("No such image: %s", name) - } else { - img = i - } - } - return img, nil -} - func (runtime *Runtime) Create(command string, args []string, image string, config *Config) (*Container, error) { // Lookup image - img, err := runtime.LookupImage(image) + img, err := runtime.repositories.LookupImage(image) if err != nil { return nil, err } diff --git a/tags.go b/tags.go index a26a2f196d..ae8c1030cd 100644 --- a/tags.go +++ b/tags.go @@ -9,6 +9,8 @@ import ( "strings" ) +const DEFAULT_TAG = "latest" + type TagStore struct { path string graph *Graph @@ -61,12 +63,35 @@ func (store *TagStore) Reload() error { return nil } -func (store *TagStore) Set(repoName, tag, revision string) error { - if strings.Contains(repoName, ":") { - return fmt.Errorf("Illegal repository name: %s", repoName) +func (store *TagStore) LookupImage(name string) (*Image, error) { + img, err := store.graph.Get(name) + if err != nil { + // FIXME: standardize on returning nil when the image doesn't exist, and err for everything else + // (so we can pass all errors here) + repoAndTag := strings.SplitN(name, ":", 2) + if len(repoAndTag) == 1 { + repoAndTag = append(repoAndTag, DEFAULT_TAG) + } + if i, err := store.GetImage(repoAndTag[0], repoAndTag[1]); err != nil { + return nil, err + } else if i == nil { + return nil, fmt.Errorf("No such image: %s", name) + } else { + img = i + } } - if strings.Contains(repoName, ":") { - return fmt.Errorf("Illegal tag name: %s", tag) + return img, nil +} + +func (store *TagStore) Set(repoName, tag, revision string) error { + if tag == "" { + tag = DEFAULT_TAG + } + if err := validateRepoName(repoName); err != nil { + return err + } + if err := validateTagName(tag); err != nil { + return err } if err := store.Reload(); err != nil { return err @@ -104,3 +129,25 @@ func (store *TagStore) GetImage(repoName, tag string) (*Image, error) { } return nil, nil } + +// Validate the name of a repository +func validateRepoName(name string) error { + if name == "" { + return fmt.Errorf("Repository name can't be empty") + } + if strings.Contains(name, ":") { + return fmt.Errorf("Illegal repository name: %s", name) + } + return nil +} + +// Validate the name of a tag +func validateTagName(name string) error { + if name == "" { + return fmt.Errorf("Tag name can't be empty") + } + if strings.Contains(name, "/") || strings.Contains(name, ":") { + return fmt.Errorf("Illegal tag name: %s", name) + } + return nil +} From c72ff318d3fb6ab887af21cf4410e1bc8104663a Mon Sep 17 00:00:00 2001 From: creack Date: Fri, 22 Mar 2013 02:19:39 -0700 Subject: [PATCH 46/80] Integrate Auth in runtime and make the config file relative to runtime root --- .gitignore | 1 + auth/auth.go | 62 ++++++++++++++++++++++++++--------------------- auth/auth_test.go | 2 +- commands.go | 25 +++++++------------ registry.go | 8 ++---- runtime.go | 8 ++++++ 6 files changed, 55 insertions(+), 51 deletions(-) diff --git a/.gitignore b/.gitignore index a9b5777c6d..eba48d297e 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ build_src command-line-arguments.test .flymake* docker.test +auth/auth.test diff --git a/auth/auth.go b/auth/auth.go index 2a3d1a439f..2779ae04c4 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -8,11 +8,12 @@ import ( "io/ioutil" "net/http" "os" + "path" "strings" ) // Where we store the config file -const CONFIGFILE = "/var/lib/docker/.dockercfg" +const CONFIGFILE = ".dockercfg" // the registry server we want to login against //const REGISTRY_SERVER = "https://registry.docker.io" @@ -22,10 +23,11 @@ type AuthConfig struct { Username string `json:"username"` Password string `json:"password"` Email string `json:"email"` + rootPath string `json:-` } // create a base64 encoded auth string to store in config -func EncodeAuth(authConfig AuthConfig) string { +func EncodeAuth(authConfig *AuthConfig) string { authStr := authConfig.Username + ":" + authConfig.Password msg := []byte(authStr) encoded := make([]byte, base64.StdEncoding.EncodedLen(len(msg))) @@ -34,50 +36,54 @@ func EncodeAuth(authConfig AuthConfig) string { } // decode the auth string -func DecodeAuth(authStr string) (AuthConfig, error) { +func DecodeAuth(authStr string) (*AuthConfig, error) { decLen := base64.StdEncoding.DecodedLen(len(authStr)) decoded := make([]byte, decLen) authByte := []byte(authStr) n, err := base64.StdEncoding.Decode(decoded, authByte) if err != nil { - return AuthConfig{}, err + return nil, err } if n > decLen { - return AuthConfig{}, errors.New("something went wrong decoding auth config") + return nil, fmt.Errorf("Something went wrong decoding auth config") } arr := strings.Split(string(decoded), ":") + if len(arr) != 2 { + return nil, fmt.Errorf("Invalid auth configuration file") + } password := strings.Trim(arr[1], "\x00") - return AuthConfig{Username: arr[0], Password: password}, nil + return &AuthConfig{Username: arr[0], Password: password}, nil } // load up the auth config information and return values -func LoadConfig() (AuthConfig, error) { - if _, err := os.Stat(CONFIGFILE); err == nil { - b, err := ioutil.ReadFile(CONFIGFILE) - if err != nil { - return AuthConfig{}, err - } - arr := strings.Split(string(b), "\n") - orig_auth := strings.Split(arr[0], " = ") - orig_email := strings.Split(arr[1], " = ") - authConfig, err := DecodeAuth(orig_auth[1]) - if err != nil { - return AuthConfig{}, err - } - authConfig.Email = orig_email[1] - return authConfig, nil - } else { - return AuthConfig{}, nil +// FIXME: use the internal golang config parser +func LoadConfig(rootPath string) (*AuthConfig, error) { + confFile := path.Join(rootPath, CONFIGFILE) + if _, err := os.Stat(confFile); err != nil { + return &AuthConfig{}, fmt.Errorf("The Auth config file is missing") } - return AuthConfig{}, nil + b, err := ioutil.ReadFile(confFile) + if err != nil { + return nil, err + } + arr := strings.Split(string(b), "\n") + orig_auth := strings.Split(arr[0], " = ") + orig_email := strings.Split(arr[1], " = ") + authConfig, err := DecodeAuth(orig_auth[1]) + if err != nil { + return nil, err + } + authConfig.Email = orig_email[1] + authConfig.rootPath = rootPath + return authConfig, nil } // save the auth config -func saveConfig(authStr string, email string) error { +func saveConfig(rootPath, authStr string, email string) error { lines := "auth = " + authStr + "\n" + "email = " + email + "\n" b := []byte(lines) - err := ioutil.WriteFile(CONFIGFILE, b, 0600) + err := ioutil.WriteFile(path.Join(rootPath, CONFIGFILE), b, 0600) if err != nil { return err } @@ -85,7 +91,7 @@ func saveConfig(authStr string, email string) error { } // try to register/login to the registry server -func Login(authConfig AuthConfig) (string, error) { +func Login(authConfig *AuthConfig) (string, error) { storeConfig := false reqStatusCode := 0 var status string @@ -146,7 +152,7 @@ func Login(authConfig AuthConfig) (string, error) { } if storeConfig { authStr := EncodeAuth(authConfig) - saveConfig(authStr, authConfig.Email) + saveConfig(authConfig.rootPath, authStr, authConfig.Email) } return status, nil } diff --git a/auth/auth_test.go b/auth/auth_test.go index d1650668e7..ca584f9314 100644 --- a/auth/auth_test.go +++ b/auth/auth_test.go @@ -5,7 +5,7 @@ import ( ) func TestEncodeAuth(t *testing.T) { - newAuthConfig := AuthConfig{Username: "ken", Password: "test", Email: "test@example.com"} + newAuthConfig := &AuthConfig{Username: "ken", Password: "test", Email: "test@example.com"} authStr := EncodeAuth(newAuthConfig) decAuthConfig, err := DecodeAuth(authStr) if err != nil { diff --git a/commands.go b/commands.go index 9716c6d3da..1a588b2586 100644 --- a/commands.go +++ b/commands.go @@ -69,17 +69,13 @@ func (srv *Server) CmdLogin(stdin io.ReadCloser, stdout io.Writer, args ...strin var username string var password string var email string - authConfig, err := auth.LoadConfig() - if err != nil { - fmt.Fprintf(stdout, "Error : %s\n", err) - } - fmt.Fprint(stdout, "Username (", authConfig.Username, "): ") + fmt.Fprint(stdout, "Username (", srv.runtime.authConfig.Username, "): ") fmt.Fscanf(stdin, "%s", &username) if username == "" { - username = authConfig.Username + username = srv.runtime.authConfig.Username } - if username != authConfig.Username { + if username != srv.runtime.authConfig.Username { fmt.Fprint(stdout, "Password: ") fmt.Fscanf(stdin, "%s", &password) @@ -87,16 +83,16 @@ func (srv *Server) CmdLogin(stdin io.ReadCloser, stdout io.Writer, args ...strin return errors.New("Error : Password Required\n") } - fmt.Fprint(stdout, "Email (", authConfig.Email, "): ") + fmt.Fprint(stdout, "Email (", srv.runtime.authConfig.Email, "): ") fmt.Fscanf(stdin, "%s", &email) if email == "" { - email = authConfig.Email + email = srv.runtime.authConfig.Email } } else { - password = authConfig.Password - email = authConfig.Email + password = srv.runtime.authConfig.Password + email = srv.runtime.authConfig.Email } - newAuthConfig := auth.AuthConfig{Username: username, Password: password, Email: email} + newAuthConfig := &auth.AuthConfig{Username: username, Password: password, Email: email} status, err := auth.Login(newAuthConfig) if err != nil { fmt.Fprintf(stdout, "Error : %s\n", err) @@ -473,7 +469,7 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string return fmt.Errorf("Not loggin and no user specified\n") } // FIXME: Allow pull repo:tag - return srv.runtime.graph.PullRepository(*user, cmd.Arg(0), "", srv.runtime.repositories) + return srv.runtime.graph.PullRepository(*user, cmd.Arg(0), "", srv.runtime.repositories, srv.runtime.authConfig) } func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...string) error { @@ -867,9 +863,6 @@ func NewServer() (*Server, error) { if runtime.GOARCH != "amd64" { log.Fatalf("The docker runtime currently only supports amd64 (not %s). This will change in the future. Aborting.", runtime.GOARCH) } - // if err != nil { - // return nil, err - // } runtime, err := NewRuntime() if err != nil { return nil, err diff --git a/registry.go b/registry.go index 5a80c8eb78..faef6f4e4c 100644 --- a/registry.go +++ b/registry.go @@ -129,19 +129,15 @@ func (graph *Graph) PullImage(imgId string) error { } // FIXME: Handle the askedTag parameter -func (graph *Graph) PullRepository(user, repoName, askedTag string, repositories *TagStore) error { +func (graph *Graph) PullRepository(user, repoName, askedTag string, repositories *TagStore, authConfig *auth.AuthConfig) error { client := &http.Client{} req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/users/"+user+"/"+repoName, nil) if err != nil { return err } - authStruct, err := auth.LoadConfig() - if err != nil { - return err - } - req.SetBasicAuth(authStruct.Username, authStruct.Password) + req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := client.Do(req) if err != nil { return err diff --git a/runtime.go b/runtime.go index 3b16479f3c..bb59c7787a 100644 --- a/runtime.go +++ b/runtime.go @@ -3,6 +3,7 @@ package docker import ( "container/list" "fmt" + "github.com/dotcloud/docker/auth" "io" "io/ioutil" "log" @@ -20,6 +21,7 @@ type Runtime struct { networkManager *NetworkManager graph *Graph repositories *TagStore + authConfig *auth.AuthConfig } var sysInitPath string @@ -246,6 +248,11 @@ func NewRuntimeFromDirectory(root string) (*Runtime, error) { if err != nil { return nil, err } + authConfig, err := auth.LoadConfig(root) + if err != nil && authConfig == nil { + // If the auth file does not exist, keep going + return nil, err + } runtime := &Runtime{ root: root, @@ -254,6 +261,7 @@ func NewRuntimeFromDirectory(root string) (*Runtime, error) { networkManager: netManager, graph: g, repositories: repositories, + authConfig: authConfig, } if err := runtime.restore(); err != nil { From 4307b7dd8ea8d2cdfdf749383f514e656ae8a408 Mon Sep 17 00:00:00 2001 From: creack Date: Fri, 22 Mar 2013 02:57:28 -0700 Subject: [PATCH 47/80] Add authentification to all registry call --- commands.go | 8 +++---- registry.go | 66 ++++++++++++++++++++++++++++++++++++++--------------- 2 files changed, 52 insertions(+), 22 deletions(-) diff --git a/commands.go b/commands.go index 1a588b2586..3e5fddf18c 100644 --- a/commands.go +++ b/commands.go @@ -440,7 +440,7 @@ func (srv *Server) CmdPush(stdin io.ReadCloser, stdout io.Writer, args ...string } // If it fails, try to get the repository if repo, exists := srv.runtime.repositories.Repositories[cmd.Arg(0)]; exists { - if err := srv.runtime.graph.PushRepository(*user, cmd.Arg(0), repo); err != nil { + if err := srv.runtime.graph.PushRepository(*user, cmd.Arg(0), repo, srv.runtime.authConfig); err != nil { return err } } else { @@ -448,7 +448,7 @@ func (srv *Server) CmdPush(stdin io.ReadCloser, stdout io.Writer, args ...string } return nil } - return srv.runtime.graph.PushImage(img) + return srv.runtime.graph.PushImage(img, srv.runtime.authConfig) } func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string) error { @@ -462,8 +462,8 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } - if srv.runtime.graph.LookupRemoteImage(cmd.Arg(0)) { - return srv.runtime.graph.PullImage(cmd.Arg(0)) + if srv.runtime.graph.LookupRemoteImage(cmd.Arg(0), srv.runtime.authConfig) { + return srv.runtime.graph.PullImage(cmd.Arg(0), srv.runtime.authConfig) } if *user == "" { return fmt.Errorf("Not loggin and no user specified\n") diff --git a/registry.go b/registry.go index faef6f4e4c..da1186ee54 100644 --- a/registry.go +++ b/registry.go @@ -49,10 +49,17 @@ func NewMultipleImgJson(src []byte) ([]*Image, error) { // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) -func (graph *Graph) getRemoteHistory(imgId string) ([]*Image, error) { - res, err := http.Get(REGISTRY_ENDPOINT + "/images/" + imgId + "/history") +func (graph *Graph) getRemoteHistory(imgId string, authConfig *auth.AuthConfig) ([]*Image, error) { + client := &http.Client{} + + req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/images/"+imgId+"/history", nil) if err != nil { - return nil, fmt.Errorf("Error while getting from the server: %s\n", err) + return nil, err + } + req.SetBasicAuth(authConfig.Username, authConfig.Password) + res, err := client.Do(req) + if err != nil { + return nil, err } defer res.Body.Close() @@ -69,8 +76,15 @@ func (graph *Graph) getRemoteHistory(imgId string) ([]*Image, error) { } // Check if an image exists in the Registry -func (graph *Graph) LookupRemoteImage(imgId string) bool { - res, err := http.Get(REGISTRY_ENDPOINT + "/images/" + imgId + "/json") +func (graph *Graph) LookupRemoteImage(imgId string, authConfig *auth.AuthConfig) bool { + client := &http.Client{} + + req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/images/"+imgId+"/json", nil) + if err != nil { + return false + } + req.SetBasicAuth(authConfig.Username, authConfig.Password) + res, err := client.Do(req) if err != nil { return false } @@ -79,12 +93,19 @@ func (graph *Graph) LookupRemoteImage(imgId string) bool { // Retrieve an image from the Registry. // Returns the Image object as well as the layer as an Archive (io.Reader) -func (graph *Graph) getRemoteImage(imgId string) (*Image, Archive, error) { +func (graph *Graph) getRemoteImage(imgId string, authConfig *auth.AuthConfig) (*Image, Archive, error) { + client := &http.Client{} + // Get the Json - res, err := http.Get(REGISTRY_ENDPOINT + "/images/" + imgId + "/json") + req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/images/"+imgId+"/json", nil) if err != nil { return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err) } + req.SetBasicAuth(authConfig.Username, authConfig.Password) + res, err := client.Do(req) + if err != nil { + return nil, nil, err + } defer res.Body.Close() jsonString, err := ioutil.ReadAll(res.Body) @@ -106,8 +127,8 @@ func (graph *Graph) getRemoteImage(imgId string) (*Image, Archive, error) { return img, res.Body, nil } -func (graph *Graph) PullImage(imgId string) error { - history, err := graph.getRemoteHistory(imgId) +func (graph *Graph) PullImage(imgId string, authConfig *auth.AuthConfig) error { + history, err := graph.getRemoteHistory(imgId, authConfig) if err != nil { return err } @@ -115,7 +136,7 @@ func (graph *Graph) PullImage(imgId string) error { // FIXME: Lunch the getRemoteImage() in goroutines for _, j := range history { if !graph.Exists(j.Id) { - img, layer, err := graph.getRemoteImage(j.Id) + img, layer, err := graph.getRemoteImage(j.Id, authConfig) if err != nil { // FIXME: Keep goging in case of error? return err @@ -136,12 +157,12 @@ func (graph *Graph) PullRepository(user, repoName, askedTag string, repositories if err != nil { return err } - req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := client.Do(req) if err != nil { return err } + defer res.Body.Close() rawJson, err := ioutil.ReadAll(res.Body) if err != nil { return err @@ -151,7 +172,7 @@ func (graph *Graph) PullRepository(user, repoName, askedTag string, repositories return err } for tag, rev := range t { - if err = graph.PullImage(rev); err != nil { + if err = graph.PullImage(rev, authConfig); err != nil { return err } if err = repositories.Set(repoName, tag, rev); err != nil { @@ -165,7 +186,7 @@ func (graph *Graph) PullRepository(user, repoName, askedTag string, repositories } // Push a local image to the registry with its history if needed -func (graph *Graph) PushImage(imgOrig *Image) error { +func (graph *Graph) PushImage(imgOrig *Image, authConfig *auth.AuthConfig) error { client := &http.Client{} // FIXME: Factorize the code @@ -182,6 +203,7 @@ func (graph *Graph) PushImage(imgOrig *Image) error { if err != nil { return err } + req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := client.Do(req) if err != nil || res.StatusCode != 200 { if res == nil { @@ -204,6 +226,7 @@ func (graph *Graph) PushImage(imgOrig *Image) error { } req2, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/images/"+img.Id+"/layer", nil) + req2.SetBasicAuth(authConfig.Username, authConfig.Password) res2, err := client.Do(req2) if err != nil || res2.StatusCode != 307 { return fmt.Errorf( @@ -226,7 +249,13 @@ func (graph *Graph) PushImage(imgOrig *Image) error { img.Id, err) } req3, err := http.NewRequest("PUT", url.String(), layerData) - tmp, _ := ioutil.ReadAll(layerData2) + if err != nil { + return err + } + tmp, err := ioutil.ReadAll(layerData2) + if err != nil { + return err + } req3.ContentLength = int64(len(tmp)) req3.TransferEncoding = []string{"none"} @@ -249,7 +278,7 @@ func (graph *Graph) PushImage(imgOrig *Image) error { return nil } -func (graph *Graph) pushTag(user, repo, revision, tag string) error { +func (graph *Graph) pushTag(user, repo, revision, tag string, authConfig *auth.AuthConfig) error { if tag == "" { tag = "lastest" @@ -260,6 +289,7 @@ func (graph *Graph) pushTag(user, repo, revision, tag string) error { client := &http.Client{} req, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/users/"+user+"/"+repo+"/"+tag, strings.NewReader(revision)) req.Header.Add("Content-type", "application/json") + req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := client.Do(req) if err != nil { return err @@ -274,17 +304,17 @@ func (graph *Graph) pushTag(user, repo, revision, tag string) error { return nil } -func (graph *Graph) PushRepository(user, repoName string, repo Repository) error { +func (graph *Graph) PushRepository(user, repoName string, repo Repository, authConfig *auth.AuthConfig) error { for tag, imgId := range repo { fmt.Printf("tag: %s, imgId: %s\n", tag, imgId) img, err := graph.Get(imgId) if err != nil { return err } - if err = graph.PushImage(img); err != nil { + if err = graph.PushImage(img, authConfig); err != nil { return err } - if err = graph.pushTag(user, repoName, imgId, tag); err != nil { + if err = graph.pushTag(user, repoName, imgId, tag, authConfig); err != nil { return err } } From 0eed4b43866a559c94097e4ffbfb49e674bda121 Mon Sep 17 00:00:00 2001 From: creack Date: Fri, 22 Mar 2013 03:10:09 -0700 Subject: [PATCH 48/80] Add some verbosity to the push/pull features --- commands.go | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/commands.go b/commands.go index 3e5fddf18c..2e999fac12 100644 --- a/commands.go +++ b/commands.go @@ -440,15 +440,24 @@ func (srv *Server) CmdPush(stdin io.ReadCloser, stdout io.Writer, args ...string } // If it fails, try to get the repository if repo, exists := srv.runtime.repositories.Repositories[cmd.Arg(0)]; exists { + fmt.Fprintf(stdout, "Pushing %s (%d images) on %s...\n", cmd.Arg(0), len(repo), *user+"/"+cmd.Arg(0)) if err := srv.runtime.graph.PushRepository(*user, cmd.Arg(0), repo, srv.runtime.authConfig); err != nil { return err } + fmt.Fprintf(stdout, "Push completed\n") + return nil } else { return err } return nil } - return srv.runtime.graph.PushImage(img, srv.runtime.authConfig) + fmt.Fprintf(stdout, "Pushing image %s..\n", img.Id) + err = srv.runtime.graph.PushImage(img, srv.runtime.authConfig) + if err != nil { + return err + } + fmt.Fprintf(stdout, "Push completed\n") + return nil } func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string) error { @@ -463,13 +472,23 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string } if srv.runtime.graph.LookupRemoteImage(cmd.Arg(0), srv.runtime.authConfig) { - return srv.runtime.graph.PullImage(cmd.Arg(0), srv.runtime.authConfig) + fmt.Fprintf(stdout, "Pulling %s...\n", cmd.Arg(0)) + if err := srv.runtime.graph.PullImage(cmd.Arg(0), srv.runtime.authConfig); err != nil { + return err + } + fmt.Fprintf(stdout, "Pulled\n") + return nil } if *user == "" { return fmt.Errorf("Not loggin and no user specified\n") } // FIXME: Allow pull repo:tag - return srv.runtime.graph.PullRepository(*user, cmd.Arg(0), "", srv.runtime.repositories, srv.runtime.authConfig) + fmt.Fprintf(stdout, "Pulling %s from %s...\n", cmd.Arg(0), *user+"/"+cmd.Arg(0)) + if err := srv.runtime.graph.PullRepository(*user, cmd.Arg(0), "", srv.runtime.repositories, srv.runtime.authConfig); err != nil { + return err + } + fmt.Fprintf(stdout, "Pull completed\n") + return nil } func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...string) error { From 3870ebee6db450efcebac81b829694a047583442 Mon Sep 17 00:00:00 2001 From: creack Date: Fri, 22 Mar 2013 03:22:36 -0700 Subject: [PATCH 49/80] Add content type to Push --- registry.go | 1 + 1 file changed, 1 insertion(+) diff --git a/registry.go b/registry.go index da1186ee54..7d6add42d4 100644 --- a/registry.go +++ b/registry.go @@ -203,6 +203,7 @@ func (graph *Graph) PushImage(imgOrig *Image, authConfig *auth.AuthConfig) error if err != nil { return err } + req.Header.Add("Content-type", "application/json") req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := client.Do(req) if err != nil || res.StatusCode != 200 { From e4f9a0dca0a3eb9fd762ef502d40a089ed16acd4 Mon Sep 17 00:00:00 2001 From: creack Date: Fri, 22 Mar 2013 03:24:37 -0700 Subject: [PATCH 50/80] Update the help with push/pull --- commands.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/commands.go b/commands.go index 1d782bf96c..8c2875370f 100644 --- a/commands.go +++ b/commands.go @@ -45,6 +45,8 @@ func (srv *Server) Help() string { {"logs", "Fetch the logs of a container"}, {"port", "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT"}, {"ps", "List containers"}, + {"pull", "Pull an image or a repository to the docker registry server"}, + {"push", "Push an image or a repository to the docker registry server"}, {"restart", "Restart a running container"}, {"rm", "Remove a container"}, {"rmi", "Remove an image"}, From e02f7912bcc07dbddd47bcf0106ca881fd768559 Mon Sep 17 00:00:00 2001 From: creack Date: Fri, 22 Mar 2013 03:43:57 -0700 Subject: [PATCH 51/80] Enforce login for push/pull --- commands.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/commands.go b/commands.go index 8c2875370f..3a1bf7f1d5 100644 --- a/commands.go +++ b/commands.go @@ -405,20 +405,21 @@ func (srv *Server) CmdPush(stdin io.ReadCloser, stdout io.Writer, args ...string if err := cmd.Parse(args); err != nil { return nil } - if cmd.NArg() == 0 { + if cmd.NArg() == 0 || *user == "" { cmd.Usage() return nil } + if srv.runtime.authConfig == nil { + return fmt.Errorf("Please login prior to push. ('docker login')") + } + // Try to get the image // FIXME: Handle lookup // FIXME: Also push the tags in case of ./docker push myrepo:mytag // img, err := srv.runtime.LookupImage(cmd.Arg(0)) img, err := srv.runtime.graph.Get(cmd.Arg(0)) if err != nil { - if *user == "" { - return fmt.Errorf("Not logged in and no user specified\n") - } // If it fails, try to get the repository if repo, exists := srv.runtime.repositories.Repositories[cmd.Arg(0)]; exists { fmt.Fprintf(stdout, "Pushing %s (%d images) on %s...\n", cmd.Arg(0), len(repo), *user+"/"+cmd.Arg(0)) @@ -447,11 +448,15 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string if err := cmd.Parse(args); err != nil { return nil } - if cmd.NArg() == 0 { + if cmd.NArg() == 0 || *user == "" { cmd.Usage() return nil } + if srv.runtime.authConfig == nil { + return fmt.Errorf("Please login prior to push. ('docker login')") + } + if srv.runtime.graph.LookupRemoteImage(cmd.Arg(0), srv.runtime.authConfig) { fmt.Fprintf(stdout, "Pulling %s...\n", cmd.Arg(0)) if err := srv.runtime.graph.PullImage(cmd.Arg(0), srv.runtime.authConfig); err != nil { @@ -460,9 +465,6 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string fmt.Fprintf(stdout, "Pulled\n") return nil } - if *user == "" { - return fmt.Errorf("Not loggin and no user specified\n") - } // FIXME: Allow pull repo:tag fmt.Fprintf(stdout, "Pulling %s from %s...\n", cmd.Arg(0), *user+"/"+cmd.Arg(0)) if err := srv.runtime.graph.PullRepository(*user, cmd.Arg(0), "", srv.runtime.repositories, srv.runtime.authConfig); err != nil { From 08cb430281bb1aa003f21a983029dd934bd99a10 Mon Sep 17 00:00:00 2001 From: creack Date: Fri, 22 Mar 2013 04:34:46 -0700 Subject: [PATCH 52/80] Move the debian makefile to avoid confusions --- .gitignore | 1 + Makefile => deb/Makefile.deb | 0 2 files changed, 1 insertion(+) rename Makefile => deb/Makefile.deb (100%) diff --git a/.gitignore b/.gitignore index eba48d297e..cabd399067 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .vagrant +bin docker/docker .*.swp a.out diff --git a/Makefile b/deb/Makefile.deb similarity index 100% rename from Makefile rename to deb/Makefile.deb From fc0eac37e47f220e83eacde51519de092b17e426 Mon Sep 17 00:00:00 2001 From: creack Date: Fri, 22 Mar 2013 04:37:18 -0700 Subject: [PATCH 53/80] Put back the "official" repo --- auth/auth.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/auth/auth.go b/auth/auth.go index 2779ae04c4..6eb9af22eb 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -16,8 +16,7 @@ import ( const CONFIGFILE = ".dockercfg" // the registry server we want to login against -//const REGISTRY_SERVER = "https://registry.docker.io" -const REGISTRY_SERVER = "http://192.168.56.1:5000" +const REGISTRY_SERVER = "https://registry.docker.io" type AuthConfig struct { Username string `json:"username"` From 77549ad4f6238fbc203eca8c328131680ea56039 Mon Sep 17 00:00:00 2001 From: creack Date: Fri, 22 Mar 2013 04:44:07 -0700 Subject: [PATCH 54/80] Improve the error management with the registry communication --- registry.go | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/registry.go b/registry.go index cc9f2f6f2b..6cd64ff779 100644 --- a/registry.go +++ b/registry.go @@ -58,7 +58,10 @@ func (graph *Graph) getRemoteHistory(imgId string, authConfig *auth.AuthConfig) } req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := client.Do(req) - if err != nil { + if err != nil || res.StatusCode != 200 { + if res != nil { + return nil, fmt.Errorf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgId) + } return nil, err } defer res.Body.Close() @@ -85,7 +88,7 @@ func (graph *Graph) LookupRemoteImage(imgId string, authConfig *auth.AuthConfig) } req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := client.Do(req) - if err != nil { + if err != nil || res.StatusCode != 307 { return false } return res.StatusCode == 307 @@ -103,7 +106,10 @@ func (graph *Graph) getRemoteImage(imgId string, authConfig *auth.AuthConfig) (* } req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := client.Do(req) - if err != nil { + if err != nil || res.StatusCode != 307 { + if res != nil { + return nil, nil, fmt.Errorf("Internal server error: %d trying to get image %s", res.StatusCode, imgId) + } return nil, nil, err } defer res.Body.Close() @@ -159,7 +165,10 @@ func (graph *Graph) PullRepository(user, repoName, askedTag string, repositories } req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := client.Do(req) - if err != nil { + if err != nil || res.StatusCode != 200 { + if res != nil { + return fmt.Errorf("Internal server error: %d trying to pull %s", res.StatusCode, repoName) + } return err } defer res.Body.Close() @@ -221,8 +230,8 @@ func (graph *Graph) PushImage(imgOrig *Image, authConfig *auth.AuthConfig) error return fmt.Errorf("Error: Invalid Json") default: return fmt.Errorf( - "Error: Internal server error trying to push image {%s} (json): %s (%d)\n", - img.Id, err, res.StatusCode) + "Error: Internal server error: %d trying to push image {%s} (json): %s\n", + res.StatusCode, img.Id, err) } } @@ -231,7 +240,7 @@ func (graph *Graph) PushImage(imgOrig *Image, authConfig *auth.AuthConfig) error res2, err := client.Do(req2) if err != nil || res2.StatusCode != 307 { return fmt.Errorf( - "Error trying to push image {%s} (layer 1): %s\n", + "Internal server error trying to push image {%s} (layer 1): %s\n", img.Id, err) } url, err := res2.Location() @@ -266,11 +275,10 @@ func (graph *Graph) PushImage(imgOrig *Image, authConfig *auth.AuthConfig) error return fmt.Errorf( "Error trying to push image {%s} (layer 2): %s\n", img.Id, err) - } else { - return fmt.Errorf( - "Error trying to push image {%s} (layer 2): %s (%d)\n", - img.Id, err, res3.StatusCode) } + return fmt.Errorf( + "Error trying to push image {%s} (layer 2): %s (%d)\n", + img.Id, err, res3.StatusCode) } return nil }); err != nil { @@ -292,7 +300,10 @@ func (graph *Graph) pushTag(user, repo, revision, tag string, authConfig *auth.A req.Header.Add("Content-type", "application/json") req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := client.Do(req) - if err != nil { + if err != nil || (res.StatusCode != 200 && res.StatusCode != 201) { + if res != nil { + return fmt.Errorf("Internal server error: %d trying to push tag %s on %s/%s", res.StatusCode, tag, user, repo) + } return err } fmt.Printf("Result of push tag: %d\n", res.StatusCode) From 09b27f9e8d762713ae64711cbdfb76aa52ec1bcf Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 22 Mar 2013 17:22:32 -0700 Subject: [PATCH 55/80] Fancier output for 'docker history' --- commands.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/commands.go b/commands.go index 9716c6d3da..18ffe91403 100644 --- a/commands.go +++ b/commands.go @@ -330,14 +330,15 @@ func (srv *Server) CmdHistory(stdin io.ReadCloser, stdout io.Writer, args ...str if err != nil { return err } - var child *Image + w := tabwriter.NewWriter(stdout, 20, 1, 3, ' ', 0) + defer w.Flush() + fmt.Fprintf(w, "ID\tCREATED\tCREATED BY\n") return image.WalkHistory(func(img *Image) error { - if child == nil { - fmt.Fprintf(stdout, " %s\n", img.Id) - } else { - fmt.Fprintf(stdout, " = %s + %s\n", img.Id, strings.Join(child.ParentCommand, " ")) - } - child = img + fmt.Fprintf(w, "%s\t%s\t%s\n", + img.Id, + HumanDuration(time.Now().Sub(img.Created))+" ago", + strings.Join(img.ParentCommand, " "), + ) return nil }) } From f8ebeaae104e742f3fc1a1fc2f72526b675d939c Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 22 Mar 2013 17:44:12 -0700 Subject: [PATCH 56/80] Removed debug command 'docker mount' --- commands.go | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/commands.go b/commands.go index f42fbe10ef..0521433b6d 100644 --- a/commands.go +++ b/commands.go @@ -219,28 +219,6 @@ func (srv *Server) CmdStart(stdin io.ReadCloser, stdout io.Writer, args ...strin return nil } -func (srv *Server) CmdMount(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "umount", "[OPTIONS] NAME", "mount a container's filesystem (debug only)") - if err := cmd.Parse(args); err != nil { - return nil - } - if cmd.NArg() < 1 { - cmd.Usage() - return nil - } - for _, name := range cmd.Args() { - if container := srv.runtime.Get(name); container != nil { - if err := container.EnsureMounted(); err != nil { - return err - } - fmt.Fprintln(stdout, container.Id) - } else { - return errors.New("No such container: " + name) - } - } - return nil -} - func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "inspect", "[OPTIONS] CONTAINER", "Return low-level information on a container") if err := cmd.Parse(args); err != nil { From 542c66997fce7cea590ca647592cc53f1cec6fca Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 22 Mar 2013 17:52:19 -0700 Subject: [PATCH 57/80] 'docker inspect' can lookup image by repository and tag --- commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands.go b/commands.go index 0521433b6d..519990f227 100644 --- a/commands.go +++ b/commands.go @@ -232,7 +232,7 @@ func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...str var obj interface{} if container := srv.runtime.Get(name); container != nil { obj = container - } else if image, err := srv.runtime.graph.Get(name); err != nil { + } else if image, err := srv.runtime.repositories.LookupImage(name); err != nil { return err } else if image != nil { obj = image From bf7602bc098888994eb28a2a6c9acd949a227448 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 22 Mar 2013 18:27:18 -0700 Subject: [PATCH 58/80] 'docker tag': assign a repository+tag to an image --- commands.go | 15 ++++++++++++++- registry.go | 2 +- runtime.go | 2 +- tags.go | 11 +++++++++-- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/commands.go b/commands.go index 1d782bf96c..4e512d8dda 100644 --- a/commands.go +++ b/commands.go @@ -389,7 +389,7 @@ func (srv *Server) CmdImport(stdin io.ReadCloser, stdout io.Writer, args ...stri // Optionally register the image at REPO/TAG if repository := cmd.Arg(1); repository != "" { tag := cmd.Arg(2) // Repository will handle an empty tag properly - if err := srv.runtime.repositories.Set(repository, tag, img.Id); err != nil { + if err := srv.runtime.repositories.Set(repository, tag, img.Id, true); err != nil { return err } } @@ -760,6 +760,19 @@ func (p *ports) Set(value string) error { return nil } +func (srv *Server) CmdTag(stdin io.ReadCloser, stdout io.Writer, args ...string) error { + cmd := rcli.Subcmd(stdout, "tag", "[OPTIONS] IMAGE REPOSITORY [TAG]", "Tag an image into a repository") + force := cmd.Bool("f", false, "Force") + if err := cmd.Parse(args); err != nil { + return nil + } + if cmd.NArg() < 2 { + cmd.Usage() + return nil + } + return srv.runtime.repositories.Set(cmd.Arg(1), cmd.Arg(2), cmd.Arg(0), *force) +} + func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "run", "[OPTIONS] IMAGE COMMAND [ARG...]", "Run a command in a new container") fl_user := cmd.String("u", "", "Username or UID") diff --git a/registry.go b/registry.go index da1186ee54..96c32ea2ef 100644 --- a/registry.go +++ b/registry.go @@ -175,7 +175,7 @@ func (graph *Graph) PullRepository(user, repoName, askedTag string, repositories if err = graph.PullImage(rev, authConfig); err != nil { return err } - if err = repositories.Set(repoName, tag, rev); err != nil { + if err = repositories.Set(repoName, tag, rev, true); err != nil { return err } } diff --git a/runtime.go b/runtime.go index bb59c7787a..21e1466f51 100644 --- a/runtime.go +++ b/runtime.go @@ -201,7 +201,7 @@ func (runtime *Runtime) Commit(id, repository, tag string) (*Image, error) { } // Register the image if needed if repository != "" { - if err := runtime.repositories.Set(repository, tag, img.Id); err != nil { + if err := runtime.repositories.Set(repository, tag, img.Id, true); err != nil { return img, err } } diff --git a/tags.go b/tags.go index ae8c1030cd..9f0a215b46 100644 --- a/tags.go +++ b/tags.go @@ -83,7 +83,11 @@ func (store *TagStore) LookupImage(name string) (*Image, error) { return img, nil } -func (store *TagStore) Set(repoName, tag, revision string) error { +func (store *TagStore) Set(repoName, tag, imageName string, force bool) error { + img, err := store.LookupImage(imageName) + if err != nil { + return err + } if tag == "" { tag = DEFAULT_TAG } @@ -101,9 +105,12 @@ func (store *TagStore) Set(repoName, tag, revision string) error { repo = r } else { repo = make(map[string]string) + if old, exists := store.Repositories[repoName]; exists && !force { + return fmt.Errorf("Tag %s:%s is already set to %s", repoName, tag, old) + } store.Repositories[repoName] = repo } - repo[tag] = revision + repo[tag] = img.Id return store.Save() } From 12049f956aedc7366492c82002c843677af4e15d Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 22 Mar 2013 19:22:06 -0700 Subject: [PATCH 59/80] 'docker {history,ps,images}': show human-friendly image names when applicable --- commands.go | 8 ++++---- tags.go | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/commands.go b/commands.go index 50eba9b7e1..d93eadfc1b 100644 --- a/commands.go +++ b/commands.go @@ -311,7 +311,7 @@ func (srv *Server) CmdHistory(stdin io.ReadCloser, stdout io.Writer, args ...str fmt.Fprintf(w, "ID\tCREATED\tCREATED BY\n") return image.WalkHistory(func(img *Image) error { fmt.Fprintf(w, "%s\t%s\t%s\n", - img.Id, + srv.runtime.repositories.ImageName(img.Id), HumanDuration(time.Now().Sub(img.Created))+" ago", strings.Join(img.ParentCommand, " "), ) @@ -514,7 +514,7 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri /* TAG */ tag, /* ID */ id, /* CREATED */ HumanDuration(time.Now().Sub(image.Created)) + " ago", - /* PARENT */ image.Parent, + /* PARENT */ srv.runtime.repositories.ImageName(image.Parent), } { if idx == 0 { w.Write([]byte(field)) @@ -537,7 +537,7 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri /* TAG */ "", /* ID */ id, /* CREATED */ HumanDuration(time.Now().Sub(image.Created)) + " ago", - /* PARENT */ image.Parent, + /* PARENT */ srv.runtime.repositories.ImageName(image.Parent), } { if idx == 0 { w.Write([]byte(field)) @@ -581,7 +581,7 @@ func (srv *Server) CmdPs(stdin io.ReadCloser, stdout io.Writer, args ...string) } for idx, field := range []string{ /* ID */ container.Id, - /* IMAGE */ container.Image, + /* IMAGE */ srv.runtime.repositories.ImageName(container.Image), /* COMMAND */ command, /* CREATED */ HumanDuration(time.Now().Sub(container.Created)) + " ago", /* STATUS */ container.State.String(), diff --git a/tags.go b/tags.go index 9f0a215b46..1df7c6e463 100644 --- a/tags.go +++ b/tags.go @@ -83,6 +83,30 @@ func (store *TagStore) LookupImage(name string) (*Image, error) { return img, nil } +// Return a reverse-lookup table of all the names which refer to each image +// Eg. {"43b5f19b10584": {"base:latest", "base:v1"}} +func (store *TagStore) ById() map[string][]string { + byId := make(map[string][]string) + for repoName, repository := range store.Repositories { + for tag, id := range repository { + name := repoName + ":" + tag + if _, exists := byId[id]; !exists { + byId[id] = []string{name} + } else { + byId[id] = append(byId[id], name) + } + } + } + return byId +} + +func (store *TagStore) ImageName(id string) string { + if names, exists := store.ById()[id]; exists && len(names) > 0 { + return names[0] + } + return id +} + func (store *TagStore) Set(repoName, tag, imageName string, force bool) error { img, err := store.LookupImage(imageName) if err != nil { From e726bdcce2606acef136a4d5ba47e367d9e461df Mon Sep 17 00:00:00 2001 From: creack Date: Fri, 22 Mar 2013 05:52:13 -0700 Subject: [PATCH 60/80] Fix the rootPath for auth --- auth/auth.go | 9 +++++++++ commands.go | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/auth/auth.go b/auth/auth.go index 6eb9af22eb..bbc9e32640 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -25,6 +25,15 @@ type AuthConfig struct { rootPath string `json:-` } +func NewAuthConfig(username, password, email, rootPath string) *AuthConfig { + return &AuthConfig{ + Username: username, + Password: password, + Email: email, + rootPath: rootPath, + } +} + // create a base64 encoded auth string to store in config func EncodeAuth(authConfig *AuthConfig) string { authStr := authConfig.Username + ":" + authConfig.Password diff --git a/commands.go b/commands.go index 6e8342ea37..a8247855fa 100644 --- a/commands.go +++ b/commands.go @@ -94,7 +94,7 @@ func (srv *Server) CmdLogin(stdin io.ReadCloser, stdout io.Writer, args ...strin password = srv.runtime.authConfig.Password email = srv.runtime.authConfig.Email } - newAuthConfig := &auth.AuthConfig{Username: username, Password: password, Email: email} + newAuthConfig := auth.NewAuthConfig(username, password, email, srv.runtime.root) status, err := auth.Login(newAuthConfig) if err != nil { fmt.Fprintf(stdout, "Error : %s\n", err) From 1ed78ee1608fb34a5ab6d136c626170eb6e2ec6a Mon Sep 17 00:00:00 2001 From: creack Date: Fri, 22 Mar 2013 06:38:54 -0700 Subject: [PATCH 61/80] Improve (drastically) the push --- commands.go | 31 ++++++++++++++++++++++++------- registry.go | 18 ++++++++++++------ 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/commands.go b/commands.go index a8247855fa..443d192bd2 100644 --- a/commands.go +++ b/commands.go @@ -402,30 +402,47 @@ func (srv *Server) CmdImport(stdin io.ReadCloser, stdout io.Writer, args ...stri } func (srv *Server) CmdPush(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "push", "[OPTIONS] IMAGE", "Push an image or a repository to the registry") - user := cmd.String("u", "", "specify the user for the repository") + cmd := rcli.Subcmd(stdout, "push", "LOCAL [REMOTE]", "Push an image or a repository to the registry") if err := cmd.Parse(args); err != nil { return nil } - if cmd.NArg() == 0 || *user == "" { + local := cmd.Arg(0) + remote := cmd.Arg(1) + + if local == "" { cmd.Usage() return nil } + // If the login failed, abort if srv.runtime.authConfig == nil { return fmt.Errorf("Please login prior to push. ('docker login')") } + if remote == "" { + tmp := strings.SplitN(local, "/", 2) + if len(tmp) == 1 { + remote = srv.runtime.authConfig.Username + "/" + local + } else { + remote = local + } + } else { + tmp := strings.SplitN(remote, "/", 2) + if len(tmp) == 1 { + return fmt.Errorf("The remote repository needs to be in the / format") + } + } + // Try to get the image // FIXME: Handle lookup // FIXME: Also push the tags in case of ./docker push myrepo:mytag // img, err := srv.runtime.LookupImage(cmd.Arg(0)) - img, err := srv.runtime.graph.Get(cmd.Arg(0)) + img, err := srv.runtime.graph.Get(local) if err != nil { // If it fails, try to get the repository - if repo, exists := srv.runtime.repositories.Repositories[cmd.Arg(0)]; exists { - fmt.Fprintf(stdout, "Pushing %s (%d images) on %s...\n", cmd.Arg(0), len(repo), *user+"/"+cmd.Arg(0)) - if err := srv.runtime.graph.PushRepository(*user, cmd.Arg(0), repo, srv.runtime.authConfig); err != nil { + if localRepo, exists := srv.runtime.repositories.Repositories[local]; exists { + fmt.Fprintf(stdout, "Pushing %s (%d images) on %s...\n", local, len(localRepo), remote) + if err := srv.runtime.graph.PushRepository(remote, localRepo, srv.runtime.authConfig); err != nil { return err } fmt.Fprintf(stdout, "Push completed\n") diff --git a/registry.go b/registry.go index 6cd64ff779..f39aa66552 100644 --- a/registry.go +++ b/registry.go @@ -287,22 +287,26 @@ func (graph *Graph) PushImage(imgOrig *Image, authConfig *auth.AuthConfig) error return nil } -func (graph *Graph) pushTag(user, repo, revision, tag string, authConfig *auth.AuthConfig) error { +// push a tag on the registry. +// Remote has the format '/ +func (graph *Graph) pushTag(remote, revision, tag string, authConfig *auth.AuthConfig) error { + // Keep this for backward compatibility if tag == "" { tag = "lastest" } + // "jsonify" the string revision = "\"" + revision + "\"" client := &http.Client{} - req, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/users/"+user+"/"+repo+"/"+tag, strings.NewReader(revision)) + req, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/users/"+remote+"/"+tag, strings.NewReader(revision)) req.Header.Add("Content-type", "application/json") req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := client.Do(req) if err != nil || (res.StatusCode != 200 && res.StatusCode != 201) { if res != nil { - return fmt.Errorf("Internal server error: %d trying to push tag %s on %s/%s", res.StatusCode, tag, user, repo) + return fmt.Errorf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote) } return err } @@ -316,8 +320,10 @@ func (graph *Graph) pushTag(user, repo, revision, tag string, authConfig *auth.A return nil } -func (graph *Graph) PushRepository(user, repoName string, repo Repository, authConfig *auth.AuthConfig) error { - for tag, imgId := range repo { +// Push a repository to the registry. +// Remote has the format '/ +func (graph *Graph) PushRepository(remote string, localRepo Repository, authConfig *auth.AuthConfig) error { + for tag, imgId := range localRepo { fmt.Printf("tag: %s, imgId: %s\n", tag, imgId) img, err := graph.Get(imgId) if err != nil { @@ -326,7 +332,7 @@ func (graph *Graph) PushRepository(user, repoName string, repo Repository, authC if err = graph.PushImage(img, authConfig); err != nil { return err } - if err = graph.pushTag(user, repoName, imgId, tag, authConfig); err != nil { + if err = graph.pushTag(remote, imgId, tag, authConfig); err != nil { return err } } From 9b5f0fac81367b9a1b4d09b87b9a8bd0fc92635a Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 22 Mar 2013 19:47:32 -0700 Subject: [PATCH 62/80] Fix 'docker import' to accept urls without explicit http:// scheme --- commands.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/commands.go b/commands.go index d93eadfc1b..6e8342ea37 100644 --- a/commands.go +++ b/commands.go @@ -374,6 +374,8 @@ func (srv *Server) CmdImport(stdin io.ReadCloser, stdout io.Writer, args ...stri } if u.Scheme == "" { u.Scheme = "http" + u.Host = src + u.Path = "" } fmt.Fprintf(stdout, "Downloading from %s\n", u.String()) // Download with curl (pretty progress bar) From 34fbaa5f6d0d8bd31504f43db56090be6dfa9af4 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 22 Mar 2013 20:36:34 -0700 Subject: [PATCH 63/80] 'docker run -e': set environment variables in a container --- commands.go | 15 +++++++++++++++ container.go | 10 ++++++++++ sysinit.go | 8 -------- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/commands.go b/commands.go index 6e8342ea37..bab2b7df71 100644 --- a/commands.go +++ b/commands.go @@ -766,6 +766,18 @@ func (p *ports) Set(value string) error { return nil } +// ListOpts type +type ListOpts []string + +func (opts *ListOpts) String() string { + return fmt.Sprint(*opts) +} + +func (opts *ListOpts) Set(value string) error { + *opts = append(*opts, value) + return nil +} + func (srv *Server) CmdTag(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "tag", "[OPTIONS] IMAGE REPOSITORY [TAG]", "Tag an image into a repository") force := cmd.Bool("f", false, "Force") @@ -789,6 +801,8 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) var fl_ports ports cmd.Var(&fl_ports, "p", "Map a network port to the container") + var fl_env ListOpts + cmd.Var(&fl_env, "e", "Set environment variables") if err := cmd.Parse(args); err != nil { return nil } @@ -819,6 +833,7 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) Tty: *fl_tty, OpenStdin: *fl_stdin, Memory: *fl_memory, + Env: fl_env, }) if err != nil { return errors.New("Error creating container: " + err.Error()) diff --git a/container.go b/container.go index ea091b8fa5..2b94fa7e00 100644 --- a/container.go +++ b/container.go @@ -53,6 +53,7 @@ type Config struct { Ports []int Tty bool // Attach standard streams to a tty, including stdin if it is not closed. OpenStdin bool // Open stdin + Env []string } type NetworkSettings struct { @@ -200,6 +201,15 @@ func (container *Container) Start() error { container.cmd = exec.Command("/usr/bin/lxc-start", params...) + // Setup environment + container.cmd.Env = append( + []string{ + "HOME=/", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + }, + container.Config.Env..., + ) + var err error if container.Config.Tty { err = container.startPty() diff --git a/sysinit.go b/sysinit.go index c475b3365d..f701417978 100644 --- a/sysinit.go +++ b/sysinit.go @@ -52,13 +52,6 @@ func changeUser(u string) { } } -// Set the environment to a known, repeatable state -func setupEnv() { - os.Clearenv() - os.Setenv("HOME", "/") - os.Setenv("PATH", "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin") -} - func executeProgram(name string, args []string) { path, err := exec.LookPath(name) if err != nil { @@ -86,6 +79,5 @@ func SysInit() { setupNetworking(*gw) changeUser(*u) - setupEnv() executeProgram(flag.Arg(0), flag.Args()) } From 841c7ac0f98c43c2d55fdac4148650e4d0118187 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 22 Mar 2013 20:46:14 -0700 Subject: [PATCH 64/80] Deprecated 'docker run -a'. Containers are run in the foreground by default. '-d' enables detached mode --- README.md | 8 ++++---- commands.go | 7 +++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 0a9b69c6de..7e8af5757c 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ Installing on Ubuntu 12.04 and 12.10 ```bash cd docker-master - sudo ./docker run -a -i -t base /bin/bash + sudo ./docker run -i -t base /bin/bash ``` Consider adding docker to your `PATH` for simplicity. @@ -136,7 +136,7 @@ docker import base # Run an interactive shell in the base image, # allocate a tty, attach stdin and stdout -docker run -a -i -t base /bin/bash +docker run -i -t base /bin/bash ``` @@ -148,7 +148,7 @@ Starting a long-running worker process (docker -d || echo "Docker daemon already running") & # Start a very useful long-running process -JOB=$(docker run base /bin/sh -c "while true; do echo Hello world; sleep 1; done") +JOB=$(docker run -d base /bin/sh -c "while true; do echo Hello world; sleep 1; done") # Collect the output of the job so far docker logs $JOB @@ -171,7 +171,7 @@ Expose a service on a TCP port ```bash # Expose port 4444 of this container, and tell netcat to listen on it -JOB=$(docker run -p 4444 base /bin/nc -l -p 4444) +JOB=$(docker run -d -p 4444 base /bin/nc -l -p 4444) # Which public port is NATed to my container? PORT=$(docker port $JOB 4444) diff --git a/commands.go b/commands.go index bab2b7df71..67a782f595 100644 --- a/commands.go +++ b/commands.go @@ -794,7 +794,7 @@ func (srv *Server) CmdTag(stdin io.ReadCloser, stdout io.Writer, args ...string) func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "run", "[OPTIONS] IMAGE COMMAND [ARG...]", "Run a command in a new container") fl_user := cmd.String("u", "", "Username or UID") - fl_attach := cmd.Bool("a", false, "Attach stdin and stdout") + fl_detach := cmd.Bool("d", false, "Detached mode: leave the container running in the background") fl_stdin := cmd.Bool("i", false, "Keep stdin open even if not attached") fl_tty := cmd.Bool("t", false, "Allocate a pseudo-tty") fl_memory := cmd.Int64("m", 0, "Memory limit (in bytes)") @@ -821,7 +821,6 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) if len(cmdline) == 0 { *fl_stdin = true *fl_tty = true - *fl_attach = true cmdline = []string{"/bin/bash", "-i"} } @@ -843,7 +842,7 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) if err != nil { return err } - if *fl_attach { + if !*fl_detach { Go(func() error { _, err := io.Copy(cmd_stdin, stdin) cmd_stdin.Close() @@ -852,7 +851,7 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) } } // Run the container - if *fl_attach { + if !*fl_detach { cmd_stderr, err := container.StderrPipe() if err != nil { return err From 829eeb07f8ecd9d53c667a6fdfe513f5a7df54ac Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 22 Mar 2013 20:55:17 -0700 Subject: [PATCH 65/80] 'docker run' with no argument no longer hardcodes a default image and command --- commands.go | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/commands.go b/commands.go index 67a782f595..aa60ca3346 100644 --- a/commands.go +++ b/commands.go @@ -806,24 +806,12 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) if err := cmd.Parse(args); err != nil { return nil } + if cmd.NArg() < 2 { + cmd.Usage() + return nil + } name := cmd.Arg(0) - var cmdline []string - - if len(cmd.Args()) >= 2 { - cmdline = cmd.Args()[1:] - } - // Choose a default image if needed - if name == "" { - name = "base" - } - - // Choose a default command if needed - if len(cmdline) == 0 { - *fl_stdin = true - *fl_tty = true - cmdline = []string{"/bin/bash", "-i"} - } - + cmdline := cmd.Args()[1:] // Create new container container, err := srv.runtime.Create(cmdline[0], cmdline[1:], name, &Config{ From c000a5ed75f8b238fd7d908118eea9d9e740f66d Mon Sep 17 00:00:00 2001 From: creack Date: Fri, 22 Mar 2013 07:10:52 -0700 Subject: [PATCH 66/80] Remove the possibility to choose the remote name on push --- commands.go | 20 +++++++------------- registry.go | 1 + 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/commands.go b/commands.go index 9e2bdbc70f..67877bc3b3 100644 --- a/commands.go +++ b/commands.go @@ -402,12 +402,11 @@ func (srv *Server) CmdImport(stdin io.ReadCloser, stdout io.Writer, args ...stri } func (srv *Server) CmdPush(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "push", "LOCAL [REMOTE]", "Push an image or a repository to the registry") + cmd := rcli.Subcmd(stdout, "push", "LOCAL", "Push an image or a repository to the registry") if err := cmd.Parse(args); err != nil { return nil } local := cmd.Arg(0) - remote := cmd.Arg(1) if local == "" { cmd.Usage() @@ -419,18 +418,13 @@ func (srv *Server) CmdPush(stdin io.ReadCloser, stdout io.Writer, args ...string return fmt.Errorf("Please login prior to push. ('docker login')") } - if remote == "" { - tmp := strings.SplitN(local, "/", 2) - if len(tmp) == 1 { - remote = srv.runtime.authConfig.Username + "/" + local - } else { - remote = local - } + var remote string + + tmp := strings.SplitN(local, "/", 2) + if len(tmp) == 1 { + remote = srv.runtime.authConfig.Username + "/" + local } else { - tmp := strings.SplitN(remote, "/", 2) - if len(tmp) == 1 { - return fmt.Errorf("The remote repository needs to be in the / format") - } + remote = local } // Try to get the image diff --git a/registry.go b/registry.go index f39aa66552..40037c15cc 100644 --- a/registry.go +++ b/registry.go @@ -221,6 +221,7 @@ func (graph *Graph) PushImage(imgOrig *Image, authConfig *auth.AuthConfig) error "Error: Internal server error trying to push image {%s} (json): %s", img.Id, err) } + fmt.Printf("Pushing return status: %d\n", res.StatusCode) switch res.StatusCode { case 204: // Case where the image is already on the Registry From d1c8eabc63a02c1791b18b7129a76c8710402666 Mon Sep 17 00:00:00 2001 From: creack Date: Fri, 22 Mar 2013 07:56:44 -0700 Subject: [PATCH 67/80] Fix/Improve push/pull registry --- commands.go | 16 ++++++++-------- registry.go | 19 +++++++++++++------ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/commands.go b/commands.go index 67877bc3b3..6659252bef 100644 --- a/commands.go +++ b/commands.go @@ -456,12 +456,12 @@ func (srv *Server) CmdPush(stdin io.ReadCloser, stdout io.Writer, args ...string } func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "pull", "[OPTIONS] IMAGE", "Pull an image or a repository from the registry") - user := cmd.String("u", "", "specify the user for the repository") + cmd := rcli.Subcmd(stdout, "pull", "IMAGE", "Pull an image or a repository from the registry") if err := cmd.Parse(args); err != nil { return nil } - if cmd.NArg() == 0 || *user == "" { + remote := cmd.Arg(0) + if remote == "" { cmd.Usage() return nil } @@ -470,17 +470,17 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string return fmt.Errorf("Please login prior to push. ('docker login')") } - if srv.runtime.graph.LookupRemoteImage(cmd.Arg(0), srv.runtime.authConfig) { - fmt.Fprintf(stdout, "Pulling %s...\n", cmd.Arg(0)) - if err := srv.runtime.graph.PullImage(cmd.Arg(0), srv.runtime.authConfig); err != nil { + if srv.runtime.graph.LookupRemoteImage(remote, srv.runtime.authConfig) { + fmt.Fprintf(stdout, "Pulling %s...\n", remote) + if err := srv.runtime.graph.PullImage(remote, srv.runtime.authConfig); err != nil { return err } fmt.Fprintf(stdout, "Pulled\n") return nil } // FIXME: Allow pull repo:tag - fmt.Fprintf(stdout, "Pulling %s from %s...\n", cmd.Arg(0), *user+"/"+cmd.Arg(0)) - if err := srv.runtime.graph.PullRepository(*user, cmd.Arg(0), "", srv.runtime.repositories, srv.runtime.authConfig); err != nil { + fmt.Fprintf(stdout, "Pulling %s...\n", remote) + if err := srv.runtime.graph.PullRepository(remote, "", srv.runtime.repositories, srv.runtime.authConfig); err != nil { return err } fmt.Fprintf(stdout, "Pull completed\n") diff --git a/registry.go b/registry.go index 40037c15cc..318a65a0e7 100644 --- a/registry.go +++ b/registry.go @@ -106,7 +106,7 @@ func (graph *Graph) getRemoteImage(imgId string, authConfig *auth.AuthConfig) (* } req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := client.Do(req) - if err != nil || res.StatusCode != 307 { + if err != nil || res.StatusCode != 200 { if res != nil { return nil, nil, fmt.Errorf("Internal server error: %d trying to get image %s", res.StatusCode, imgId) } @@ -126,10 +126,15 @@ func (graph *Graph) getRemoteImage(imgId string, authConfig *auth.AuthConfig) (* img.Id = imgId // Get the layer - res, err = http.Get(REGISTRY_ENDPOINT + "/images/" + imgId + "/layer") + req, err = http.NewRequest("GET", REGISTRY_ENDPOINT+"/images/"+imgId+"/layer", nil) if err != nil { return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err) } + req.SetBasicAuth(authConfig.Username, authConfig.Password) + res, err = client.Do(req) + if err != nil { + return nil, nil, err + } return img, res.Body, nil } @@ -156,10 +161,12 @@ func (graph *Graph) PullImage(imgId string, authConfig *auth.AuthConfig) error { } // FIXME: Handle the askedTag parameter -func (graph *Graph) PullRepository(user, repoName, askedTag string, repositories *TagStore, authConfig *auth.AuthConfig) error { +func (graph *Graph) PullRepository(remote, askedTag string, repositories *TagStore, authConfig *auth.AuthConfig) error { client := &http.Client{} - req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/users/"+user+"/"+repoName, nil) + fmt.Printf("Pulling repo: %s\n", REGISTRY_ENDPOINT+"/users/"+remote) + + req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/users/"+remote, nil) if err != nil { return err } @@ -167,7 +174,7 @@ func (graph *Graph) PullRepository(user, repoName, askedTag string, repositories res, err := client.Do(req) if err != nil || res.StatusCode != 200 { if res != nil { - return fmt.Errorf("Internal server error: %d trying to pull %s", res.StatusCode, repoName) + return fmt.Errorf("Internal server error: %d trying to pull %s", res.StatusCode, remote) } return err } @@ -184,7 +191,7 @@ func (graph *Graph) PullRepository(user, repoName, askedTag string, repositories if err = graph.PullImage(rev, authConfig); err != nil { return err } - if err = repositories.Set(repoName, tag, rev, true); err != nil { + if err = repositories.Set(remote, tag, rev, true); err != nil { return err } } From 0c4b639083e2660cc246a60b3f13f7fceca413ed Mon Sep 17 00:00:00 2001 From: creack Date: Fri, 22 Mar 2013 08:04:53 -0700 Subject: [PATCH 68/80] Update makefile gotest --- deb/Makefile.deb | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/deb/Makefile.deb b/deb/Makefile.deb index 2178ebcb93..c954b0f5b5 100644 --- a/deb/Makefile.deb +++ b/deb/Makefile.deb @@ -8,7 +8,7 @@ GITHUB_PATH=src/github.com/dotcloud/docker INSDIR=usr/bin SOURCE_PACKAGE=$(PKG_NAME)_$(PKG_VERSION).orig.tar.gz DEB_PACKAGE=$(PKG_NAME)_$(PKG_VERSION)_$(PKG_ARCH).deb -EXTRA_GO_PKG=fs auth +EXTRA_GO_PKG=./auth TMPDIR=$(shell mktemp -d -t XXXXXX) @@ -63,20 +63,10 @@ build_local: gotest: @echo "\033[36m[Testing]\033[00m docker..." - @sudo -E GOPATH=$(ROOT_PATH)/$(BUILD_SRC) go test -v && \ + @sudo -E GOPATH=$(ROOT_PATH)/$(BUILD_SRC) go test -v . $(EXTRA_GO_PKG) && \ echo -n "\033[32m[OK]\033[00m" || \ echo -n "\033[31m[FAIL]\033[00m"; \ echo " docker" - @echo "Testing extra repos {$(EXTRA_GO_PKG)}" - @for package in $(EXTRA_GO_PKG); do \ - echo "\033[36m[Testing]\033[00m docker/$$package..." && \ - cd $$package ; \ - sudo -E GOPATH=$(ROOT_PATH)/$(BUILD_SRC) go test -v && \ - echo -n "\033[32m[OK]\033[00m" || \ - echo -n "\033[31m[FAIL]\033[00m" ; \ - echo " docker/$$package" ; \ - cd .. ;\ - done @sudo rm -rf /tmp/docker-* clean: From 5e6355d1828441fe6d5ff4cd814825e05213b2c9 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 22 Mar 2013 10:22:22 -0700 Subject: [PATCH 69/80] Fix the lookup method --- registry.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/registry.go b/registry.go index 318a65a0e7..a458817bb2 100644 --- a/registry.go +++ b/registry.go @@ -80,14 +80,14 @@ func (graph *Graph) getRemoteHistory(imgId string, authConfig *auth.AuthConfig) // Check if an image exists in the Registry func (graph *Graph) LookupRemoteImage(imgId string, authConfig *auth.AuthConfig) bool { - client := &http.Client{} + rt := &http.Transport{Proxy: http.ProxyFromEnvironment} req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/images/"+imgId+"/json", nil) if err != nil { return false } req.SetBasicAuth(authConfig.Username, authConfig.Password) - res, err := client.Do(req) + res, err := rt.RoundTrip(req) if err != nil || res.StatusCode != 307 { return false } From a2e5333a93f85ffb6beb8ad88c88e115f235886c Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 22 Mar 2013 10:42:13 -0700 Subject: [PATCH 70/80] Make sure the remote repository exists prior to push --- registry.go | 49 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/registry.go b/registry.go index a458817bb2..1af117997c 100644 --- a/registry.go +++ b/registry.go @@ -328,19 +328,50 @@ func (graph *Graph) pushTag(remote, revision, tag string, authConfig *auth.AuthC return nil } +func (graph *Graph) LookupRemoteRepository(remote string, authConfig *auth.AuthConfig) bool { + rt := &http.Transport{Proxy: http.ProxyFromEnvironment} + + req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/users/"+remote, nil) + if err != nil { + return false + } + req.SetBasicAuth(authConfig.Username, authConfig.Password) + res, err := rt.RoundTrip(req) + if err != nil || res.StatusCode != 200 { + return false + } + return true +} + +func (graph *Graph) pushPrimitive(remote, tag, imgId string, authConfig *auth.AuthConfig) error { + // CHeck if the local impage exists + img, err := graph.Get(imgId) + if err != nil { + return err + } + // Push the image + if err = graph.PushImage(img, authConfig); err != nil { + return err + } + // And then the tag + if err = graph.pushTag(remote, imgId, tag, authConfig); err != nil { + return err + } + return nil +} + // Push a repository to the registry. // Remote has the format '/ func (graph *Graph) PushRepository(remote string, localRepo Repository, authConfig *auth.AuthConfig) error { + // Check if the remote repository exists + if !graph.LookupRemoteRepository(remote, authConfig) { + return fmt.Errorf("The remote repository %s does not exist\n", remote) + } + + // For each image within the repo, push them for tag, imgId := range localRepo { - fmt.Printf("tag: %s, imgId: %s\n", tag, imgId) - img, err := graph.Get(imgId) - if err != nil { - return err - } - if err = graph.PushImage(img, authConfig); err != nil { - return err - } - if err = graph.pushTag(remote, imgId, tag, authConfig); err != nil { + if err := graph.pushPrimitive(remote, tag, imgId, authConfig); err != nil { + // FIXME: Continue on error? return err } } From 6e507b9460b81e830ef2f5d10035d08e3948fe4d Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 22 Mar 2013 11:44:12 -0700 Subject: [PATCH 71/80] Add a Debugf() helper and a -D (debug) flag to docker --- docker/docker.go | 3 +++ rcli/tcp.go | 8 ++++++++ registry.go | 6 ++++++ runtime.go | 5 ++--- utils.go | 13 +++++++++++++ 5 files changed, 32 insertions(+), 3 deletions(-) diff --git a/docker/docker.go b/docker/docker.go index 5fc70f064f..686cd2181a 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -16,8 +16,11 @@ func main() { docker.SysInit() return } + // FIXME: Switch d and D ? (to be more sshd like) fl_daemon := flag.Bool("d", false, "Daemon mode") + fl_debug := flag.Bool("D", false, "Debug mode") flag.Parse() + rcli.DEBUG_FLAG = *fl_debug if *fl_daemon { if flag.NArg() != 0 { flag.Usage() diff --git a/rcli/tcp.go b/rcli/tcp.go index 869a3bcdb6..a1fa669023 100644 --- a/rcli/tcp.go +++ b/rcli/tcp.go @@ -10,6 +10,11 @@ import ( "net" ) +// Note: the globals are here to avoid import cycle +// FIXME: Handle debug levels mode? +var DEBUG_FLAG bool = false +var CLIENT_SOCKET io.Writer = nil + // Connect to a remote endpoint using protocol `proto` and address `addr`, // issue a single call, and return the result. // `proto` may be "tcp", "unix", etc. See the `net` package for available protocols. @@ -42,6 +47,9 @@ func ListenAndServe(proto, addr string, service Service) error { return err } else { go func() { + if DEBUG_FLAG { + CLIENT_SOCKET = conn + } if err := Serve(conn, service); err != nil { log.Printf("Error: " + err.Error() + "\n") fmt.Fprintf(conn, "Error: "+err.Error()+"\n") diff --git a/registry.go b/registry.go index 1af117997c..3d4c0e904f 100644 --- a/registry.go +++ b/registry.go @@ -213,6 +213,9 @@ func (graph *Graph) PushImage(imgOrig *Image, authConfig *auth.AuthConfig) error if err != nil { return fmt.Errorf("Error while retreiving the path for {%s}: %s", img.Id, err) } + + Debugf("Pushing image [%s] on {%s}\n", img.Id, REGISTRY_ENDPOINT+"/images/"+img.Id+"/json") + // FIXME: try json with UTF8 jsonData := strings.NewReader(string(jsonRaw)) req, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/images/"+img.Id+"/json", jsonData) @@ -257,6 +260,7 @@ func (graph *Graph) PushImage(imgOrig *Image, authConfig *auth.AuthConfig) error "Fail to retrieve layer storage URL for image {%s}: %s\n", img.Id, err) } + // FIXME: Don't do this :D. Check the S3 requierement and implement chunks of 5MB // FIXME2: I won't stress it enough, DON'T DO THIS! very high priority layerData2, err := Tar(path.Join(graph.Root, img.Id, "layer"), Gzip) @@ -307,6 +311,8 @@ func (graph *Graph) pushTag(remote, revision, tag string, authConfig *auth.AuthC // "jsonify" the string revision = "\"" + revision + "\"" + Debugf("Pushing tags for rev [%s] on {%s}\n", revision, REGISTRY_ENDPOINT+"/users/"+remote+"/"+tag) + client := &http.Client{} req, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/users/"+remote+"/"+tag, strings.NewReader(revision)) req.Header.Add("Content-type", "application/json") diff --git a/runtime.go b/runtime.go index da95dfe365..b7f2582b46 100644 --- a/runtime.go +++ b/runtime.go @@ -6,7 +6,6 @@ import ( "github.com/dotcloud/docker/auth" "io" "io/ioutil" - "log" "os" "path" "sort" @@ -216,10 +215,10 @@ func (runtime *Runtime) restore() error { id := v.Name() container, err := runtime.Load(id) if err != nil { - log.Printf("Failed to load container %v: %v", id, err) + Debugf("Failed to load container %v: %v", id, err) continue } - log.Printf("Loaded container %v", container.Id) + Debugf("Loaded container %v", container.Id) } return nil } diff --git a/utils.go b/utils.go index 29f62f7534..ce722f264f 100644 --- a/utils.go +++ b/utils.go @@ -5,7 +5,9 @@ import ( "container/list" "errors" "fmt" + "github.com/dotcloud/docker/rcli" "io" + "log" "net/http" "os" "os/exec" @@ -37,6 +39,17 @@ func Download(url string, stderr io.Writer) (*http.Response, error) { return resp, nil } +// Debug function, if the debug flag is set, then display. Do nothing otherwise +// If Docker is in damon mode, also send the debug info on the socket +func Debugf(format string, a ...interface{}) { + if rcli.DEBUG_FLAG { + log.Printf(format, a...) + if rcli.CLIENT_SOCKET != nil { + fmt.Fprintf(rcli.CLIENT_SOCKET, log.Prefix()+format, a...) + } + } +} + // Reader with progress bar type progressReader struct { reader io.ReadCloser // Stream to read from From 89763bc8af81dcda3d9989cf6687c75101bbd9bc Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 22 Mar 2013 13:10:17 -0700 Subject: [PATCH 72/80] Remove the lookup before pushing --- commands.go | 6 +++++- registry.go | 7 ++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/commands.go b/commands.go index 16f97f12dd..2b54578bf7 100644 --- a/commands.go +++ b/commands.go @@ -425,15 +425,19 @@ func (srv *Server) CmdPush(stdin io.ReadCloser, stdout io.Writer, args ...string remote = local } + Debugf("Pushing [%s] to [%s]\n", local, remote) + // Try to get the image // FIXME: Handle lookup // FIXME: Also push the tags in case of ./docker push myrepo:mytag // img, err := srv.runtime.LookupImage(cmd.Arg(0)) img, err := srv.runtime.graph.Get(local) if err != nil { + Debugf("The push refers to a repository [%s] (len: %d)\n", local, len(srv.runtime.repositories.Repositories[local])) + // If it fails, try to get the repository if localRepo, exists := srv.runtime.repositories.Repositories[local]; exists { - fmt.Fprintf(stdout, "Pushing %s (%d images) on %s...\n", local, len(localRepo), remote) + fmt.Fprintf(stdout, "Pushing %s (%d tags) on %s...\n", local, len(localRepo), remote) if err := srv.runtime.graph.PushRepository(remote, localRepo, srv.runtime.authConfig); err != nil { return err } diff --git a/registry.go b/registry.go index 3d4c0e904f..4eb1e07cf0 100644 --- a/registry.go +++ b/registry.go @@ -370,9 +370,10 @@ func (graph *Graph) pushPrimitive(remote, tag, imgId string, authConfig *auth.Au // Remote has the format '/ func (graph *Graph) PushRepository(remote string, localRepo Repository, authConfig *auth.AuthConfig) error { // Check if the remote repository exists - if !graph.LookupRemoteRepository(remote, authConfig) { - return fmt.Errorf("The remote repository %s does not exist\n", remote) - } + // FIXME: @lopter How to handle this? + // if !graph.LookupRemoteRepository(remote, authConfig) { + // return fmt.Errorf("The remote repository %s does not exist\n", remote) + // } // For each image within the repo, push them for tag, imgId := range localRepo { From 031f91df1a158d37f7f068ca1e17c25e1f112f29 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 23 Mar 2013 12:16:58 -0700 Subject: [PATCH 73/80] runtime.Create receives an image name + Config. The Config includes all required runtime information: command, environment, ports etc. --- commands.go | 42 +++------- container.go | 33 ++++++++ container_test.go | 193 ++++++++++++++++++---------------------------- runtime.go | 15 ++-- runtime_test.go | 48 +++++------- 5 files changed, 147 insertions(+), 184 deletions(-) diff --git a/commands.go b/commands.go index 6659252bef..5dade37a96 100644 --- a/commands.go +++ b/commands.go @@ -803,45 +803,27 @@ func (srv *Server) CmdTag(stdin io.ReadCloser, stdout io.Writer, args ...string) } func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "run", "[OPTIONS] IMAGE COMMAND [ARG...]", "Run a command in a new container") - fl_user := cmd.String("u", "", "Username or UID") - fl_detach := cmd.Bool("d", false, "Detached mode: leave the container running in the background") - fl_stdin := cmd.Bool("i", false, "Keep stdin open even if not attached") - fl_tty := cmd.Bool("t", false, "Allocate a pseudo-tty") - fl_memory := cmd.Int64("m", 0, "Memory limit (in bytes)") - var fl_ports ports - - cmd.Var(&fl_ports, "p", "Map a network port to the container") - var fl_env ListOpts - cmd.Var(&fl_env, "e", "Set environment variables") - if err := cmd.Parse(args); err != nil { - return nil + image, config, err := ParseRun(args) + if err != nil { + return err } - if cmd.NArg() < 2 { - cmd.Usage() - return nil + if image == "" { + return fmt.Errorf("Image not specified") + } + if len(config.Cmd) == 0 { + return fmt.Errorf("Command not specified") } - name := cmd.Arg(0) - cmdline := cmd.Args()[1:] // Create new container - container, err := srv.runtime.Create(cmdline[0], cmdline[1:], name, - &Config{ - Ports: fl_ports, - User: *fl_user, - Tty: *fl_tty, - OpenStdin: *fl_stdin, - Memory: *fl_memory, - Env: fl_env, - }) + container, err := srv.runtime.Create(image, config) if err != nil { return errors.New("Error creating container: " + err.Error()) } - if *fl_stdin { + if config.OpenStdin { cmd_stdin, err := container.StdinPipe() if err != nil { return err } - if !*fl_detach { + if !config.Detach { Go(func() error { _, err := io.Copy(cmd_stdin, stdin) cmd_stdin.Close() @@ -850,7 +832,7 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) } } // Run the container - if !*fl_detach { + if !config.Detach { cmd_stderr, err := container.StderrPipe() if err != nil { return err diff --git a/container.go b/container.go index 2b94fa7e00..afbf14e18f 100644 --- a/container.go +++ b/container.go @@ -3,6 +3,7 @@ package docker import ( "encoding/json" "errors" + "flag" "fmt" "github.com/kr/pty" "io" @@ -50,10 +51,42 @@ type Config struct { User string Memory int64 // Memory limit (in bytes) MemorySwap int64 // Total memory usage (memory + swap); set `-1' to disable swap + Detach bool Ports []int Tty bool // Attach standard streams to a tty, including stdin if it is not closed. OpenStdin bool // Open stdin Env []string + Cmd []string +} + +func ParseRun(args []string) (string, *Config, error) { + cmd := flag.NewFlagSet("", flag.ContinueOnError) + cmd.SetOutput(ioutil.Discard) + fl_user := cmd.String("u", "", "Username or UID") + fl_detach := cmd.Bool("d", false, "Detached mode: leave the container running in the background") + fl_stdin := cmd.Bool("i", false, "Keep stdin open even if not attached") + fl_tty := cmd.Bool("t", false, "Allocate a pseudo-tty") + fl_memory := cmd.Int64("m", 0, "Memory limit (in bytes)") + var fl_ports ports + + cmd.Var(&fl_ports, "p", "Map a network port to the container") + var fl_env ListOpts + cmd.Var(&fl_env, "e", "Set environment variables") + if err := cmd.Parse(args); err != nil { + return "", nil, err + } + image := cmd.Arg(0) + config := &Config{ + Ports: fl_ports, + User: *fl_user, + Tty: *fl_tty, + OpenStdin: *fl_stdin, + Memory: *fl_memory, + Detach: *fl_detach, + Env: fl_env, + Cmd: cmd.Args()[1:], + } + return image, config, nil } type NetworkSettings struct { diff --git a/container_test.go b/container_test.go index 94db11408d..e9e6992b7b 100644 --- a/container_test.go +++ b/container_test.go @@ -20,10 +20,9 @@ func TestCommitRun(t *testing.T) { } defer nuke(runtime) container1, err := runtime.Create( - "/bin/sh", - []string{"-c", "echo hello > /world"}, GetTestImage(runtime).Id, &Config{ + Cmd: []string{"/bin/sh", "-c", "echo hello > /world"}, Memory: 33554432, }, ) @@ -54,11 +53,10 @@ func TestCommitRun(t *testing.T) { // FIXME: Make a TestCommit that stops here and check docker.root/layers/img.id/world container2, err := runtime.Create( - "cat", - []string{"/world"}, img.Id, &Config{ Memory: 33554432, + Cmd: []string{"cat", "/world"}, }, ) if err != nil { @@ -88,11 +86,10 @@ func TestRun(t *testing.T) { } defer nuke(runtime) container, err := runtime.Create( - "ls", - []string{"-al"}, GetTestImage(runtime).Id, &Config{ Memory: 33554432, + Cmd: []string{"ls", "-al"}, }, ) if err != nil { @@ -118,10 +115,10 @@ func TestOutput(t *testing.T) { } defer nuke(runtime) container, err := runtime.Create( - "echo", - []string{"-n", "foobar"}, GetTestImage(runtime).Id, - &Config{}, + &Config{ + Cmd: []string{"echo", "-n", "foobar"}, + }, ) if err != nil { t.Fatal(err) @@ -142,11 +139,9 @@ func TestKill(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create( - "cat", - []string{"/dev/zero"}, - GetTestImage(runtime).Id, - &Config{}, + container, err := runtime.Create(GetTestImage(runtime).Id, &Config{ + Cmd: []string{"cat", "/dev/zero"}, + }, ) if err != nil { t.Fatal(err) @@ -185,11 +180,9 @@ func TestExitCode(t *testing.T) { } defer nuke(runtime) - trueContainer, err := runtime.Create( - "/bin/true", - []string{""}, - GetTestImage(runtime).Id, - &Config{}, + trueContainer, err := runtime.Create(GetTestImage(runtime).Id, &Config{ + Cmd: []string{"/bin/true", ""}, + }, ) if err != nil { t.Fatal(err) @@ -199,11 +192,9 @@ func TestExitCode(t *testing.T) { t.Fatal(err) } - falseContainer, err := runtime.Create( - "/bin/false", - []string{""}, - GetTestImage(runtime).Id, - &Config{}, + falseContainer, err := runtime.Create(GetTestImage(runtime).Id, &Config{ + Cmd: []string{"/bin/false", ""}, + }, ) if err != nil { t.Fatal(err) @@ -228,11 +219,9 @@ func TestRestart(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create( - "echo", - []string{"-n", "foobar"}, - GetTestImage(runtime).Id, - &Config{}, + container, err := runtime.Create(GetTestImage(runtime).Id, &Config{ + Cmd: []string{"echo", "-n", "foobar"}, + }, ) if err != nil { t.Fatal(err) @@ -262,13 +251,11 @@ func TestRestartStdin(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create( - "cat", - []string{}, - GetTestImage(runtime).Id, - &Config{ - OpenStdin: true, - }, + container, err := runtime.Create(GetTestImage(runtime).Id, &Config{ + Cmd: []string{"cat"}, + + OpenStdin: true, + }, ) if err != nil { t.Fatal(err) @@ -313,11 +300,9 @@ func TestUser(t *testing.T) { defer nuke(runtime) // Default user must be root - container, err := runtime.Create( - "id", - []string{}, - GetTestImage(runtime).Id, - &Config{}, + container, err := runtime.Create(GetTestImage(runtime).Id, &Config{ + Cmd: []string{"id"}, + }, ) if err != nil { t.Fatal(err) @@ -332,13 +317,11 @@ func TestUser(t *testing.T) { } // Set a username - container, err = runtime.Create( - "id", - []string{}, - GetTestImage(runtime).Id, - &Config{ - User: "root", - }, + container, err = runtime.Create(GetTestImage(runtime).Id, &Config{ + Cmd: []string{"id"}, + + User: "root", + }, ) if err != nil { t.Fatal(err) @@ -353,13 +336,11 @@ func TestUser(t *testing.T) { } // Set a UID - container, err = runtime.Create( - "id", - []string{}, - GetTestImage(runtime).Id, - &Config{ - User: "0", - }, + container, err = runtime.Create(GetTestImage(runtime).Id, &Config{ + Cmd: []string{"id"}, + + User: "0", + }, ) if err != nil || container.State.ExitCode != 0 { t.Fatal(err) @@ -374,13 +355,11 @@ func TestUser(t *testing.T) { } // Set a different user by uid - container, err = runtime.Create( - "id", - []string{}, - GetTestImage(runtime).Id, - &Config{ - User: "1", - }, + container, err = runtime.Create(GetTestImage(runtime).Id, &Config{ + Cmd: []string{"id"}, + + User: "1", + }, ) if err != nil { t.Fatal(err) @@ -397,13 +376,11 @@ func TestUser(t *testing.T) { } // Set a different user by username - container, err = runtime.Create( - "id", - []string{}, - GetTestImage(runtime).Id, - &Config{ - User: "daemon", - }, + container, err = runtime.Create(GetTestImage(runtime).Id, &Config{ + Cmd: []string{"id"}, + + User: "daemon", + }, ) if err != nil { t.Fatal(err) @@ -425,22 +402,18 @@ func TestMultipleContainers(t *testing.T) { } defer nuke(runtime) - container1, err := runtime.Create( - "cat", - []string{"/dev/zero"}, - GetTestImage(runtime).Id, - &Config{}, + container1, err := runtime.Create(GetTestImage(runtime).Id, &Config{ + Cmd: []string{"cat", "/dev/zero"}, + }, ) if err != nil { t.Fatal(err) } defer runtime.Destroy(container1) - container2, err := runtime.Create( - "cat", - []string{"/dev/zero"}, - GetTestImage(runtime).Id, - &Config{}, + container2, err := runtime.Create(GetTestImage(runtime).Id, &Config{ + Cmd: []string{"cat", "/dev/zero"}, + }, ) if err != nil { t.Fatal(err) @@ -479,13 +452,11 @@ func TestStdin(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create( - "cat", - []string{}, - GetTestImage(runtime).Id, - &Config{ - OpenStdin: true, - }, + container, err := runtime.Create(GetTestImage(runtime).Id, &Config{ + Cmd: []string{"cat"}, + + OpenStdin: true, + }, ) if err != nil { t.Fatal(err) @@ -514,13 +485,11 @@ func TestTty(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create( - "cat", - []string{}, - GetTestImage(runtime).Id, - &Config{ - OpenStdin: true, - }, + container, err := runtime.Create(GetTestImage(runtime).Id, &Config{ + Cmd: []string{"cat"}, + + OpenStdin: true, + }, ) if err != nil { t.Fatal(err) @@ -549,11 +518,9 @@ func TestEnv(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create( - "/usr/bin/env", - []string{}, - GetTestImage(runtime).Id, - &Config{}, + container, err := runtime.Create(GetTestImage(runtime).Id, &Config{ + Cmd: []string{"/usr/bin/env"}, + }, ) if err != nil { t.Fatal(err) @@ -623,14 +590,12 @@ func TestLXCConfig(t *testing.T) { memMin := 33554432 memMax := 536870912 mem := memMin + rand.Intn(memMax-memMin) - container, err := runtime.Create( - "/bin/true", - []string{}, - GetTestImage(runtime).Id, - &Config{ - Hostname: "foobar", - Memory: int64(mem), - }, + container, err := runtime.Create(GetTestImage(runtime).Id, &Config{ + Cmd: []string{"/bin/true"}, + + Hostname: "foobar", + Memory: int64(mem), + }, ) if err != nil { t.Fatal(err) @@ -651,11 +616,9 @@ func BenchmarkRunSequencial(b *testing.B) { } defer nuke(runtime) for i := 0; i < b.N; i++ { - container, err := runtime.Create( - "echo", - []string{"-n", "foo"}, - GetTestImage(runtime).Id, - &Config{}, + container, err := runtime.Create(GetTestImage(runtime).Id, &Config{ + Cmd: []string{"echo", "-n", "foo"}, + }, ) if err != nil { b.Fatal(err) @@ -687,11 +650,9 @@ func BenchmarkRunParallel(b *testing.B) { complete := make(chan error) tasks = append(tasks, complete) go func(i int, complete chan error) { - container, err := runtime.Create( - "echo", - []string{"-n", "foo"}, - GetTestImage(runtime).Id, - &Config{}, + container, err := runtime.Create(GetTestImage(runtime).Id, &Config{ + Cmd: []string{"echo", "-n", "foo"}, + }, ) if err != nil { complete <- err diff --git a/runtime.go b/runtime.go index 21e1466f51..a9efe446d7 100644 --- a/runtime.go +++ b/runtime.go @@ -64,7 +64,7 @@ func (runtime *Runtime) containerRoot(id string) string { return path.Join(runtime.repository, id) } -func (runtime *Runtime) Create(command string, args []string, image string, config *Config) (*Container, error) { +func (runtime *Runtime) Create(image string, config *Config) (*Container, error) { // Lookup image img, err := runtime.repositories.LookupImage(image) if err != nil { @@ -72,13 +72,12 @@ func (runtime *Runtime) Create(command string, args []string, image string, conf } container := &Container{ // FIXME: we should generate the ID here instead of receiving it as an argument - Id: GenerateId(), - Created: time.Now(), - Path: command, - Args: args, - Config: config, - Image: img.Id, // Always use the resolved image id - //FIXME: store the name under which the image was given, for reference + Id: GenerateId(), + Created: time.Now(), + Path: config.Cmd[0], + Args: config.Cmd[1:], //FIXME: de-duplicate from config + Config: config, + Image: img.Id, // Always use the resolved image id NetworkSettings: &NetworkSettings{}, // FIXME: do we need to store this in the container? SysInitPath: sysInitPath, diff --git a/runtime_test.go b/runtime_test.go index fe3ab4810f..a2d1e68a63 100644 --- a/runtime_test.go +++ b/runtime_test.go @@ -112,11 +112,9 @@ func TestRuntimeCreate(t *testing.T) { if len(runtime.List()) != 0 { t.Errorf("Expected 0 containers, %v found", len(runtime.List())) } - container, err := runtime.Create( - "ls", - []string{"-al"}, - GetTestImage(runtime).Id, - &Config{}, + container, err := runtime.Create(GetTestImage(runtime).Id, &Config{ + Cmd: []string{"ls", "-al"}, + }, ) if err != nil { t.Fatal(err) @@ -160,11 +158,9 @@ func TestDestroy(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create( - "ls", - []string{"-al"}, - GetTestImage(runtime).Id, - &Config{}, + container, err := runtime.Create(GetTestImage(runtime).Id, &Config{ + Cmd: []string{"ls", "-al"}, + }, ) if err != nil { t.Fatal(err) @@ -208,33 +204,27 @@ func TestGet(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container1, err := runtime.Create( - "ls", - []string{"-al"}, - GetTestImage(runtime).Id, - &Config{}, + container1, err := runtime.Create(GetTestImage(runtime).Id, &Config{ + Cmd: []string{"ls", "-al"}, + }, ) if err != nil { t.Fatal(err) } defer runtime.Destroy(container1) - container2, err := runtime.Create( - "ls", - []string{"-al"}, - GetTestImage(runtime).Id, - &Config{}, + container2, err := runtime.Create(GetTestImage(runtime).Id, &Config{ + Cmd: []string{"ls", "-al"}, + }, ) if err != nil { t.Fatal(err) } defer runtime.Destroy(container2) - container3, err := runtime.Create( - "ls", - []string{"-al"}, - GetTestImage(runtime).Id, - &Config{}, + container3, err := runtime.Create(GetTestImage(runtime).Id, &Config{ + Cmd: []string{"ls", "-al"}, + }, ) if err != nil { t.Fatal(err) @@ -274,11 +264,9 @@ func TestRestore(t *testing.T) { } // Create a container with one instance of docker - container1, err := runtime1.Create( - "ls", - []string{"-al"}, - GetTestImage(runtime1).Id, - &Config{}, + container1, err := runtime1.Create(GetTestImage(runtime1).Id, &Config{ + Cmd: []string{"ls", "-al"}, + }, ) if err != nil { t.Fatal(err) From 6ce64e8458b31aff150f2cae197d42e334018afa Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 23 Mar 2013 12:39:09 -0700 Subject: [PATCH 74/80] Moved image name into config. runtime.Create() now receives a single Config parameter --- commands.go | 6 +-- container.go | 9 +++-- container_test.go | 101 +++++++++++++++++++++++++++------------------- runtime.go | 4 +- runtime_test.go | 30 ++++++++------ 5 files changed, 88 insertions(+), 62 deletions(-) diff --git a/commands.go b/commands.go index 5dade37a96..0933171448 100644 --- a/commands.go +++ b/commands.go @@ -803,18 +803,18 @@ func (srv *Server) CmdTag(stdin io.ReadCloser, stdout io.Writer, args ...string) } func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - image, config, err := ParseRun(args) + config, err := ParseRun(args) if err != nil { return err } - if image == "" { + if config.Image == "" { return fmt.Errorf("Image not specified") } if len(config.Cmd) == 0 { return fmt.Errorf("Command not specified") } // Create new container - container, err := srv.runtime.Create(image, config) + container, err := srv.runtime.Create(config) if err != nil { return errors.New("Error creating container: " + err.Error()) } diff --git a/container.go b/container.go index afbf14e18f..f900599d00 100644 --- a/container.go +++ b/container.go @@ -57,9 +57,10 @@ type Config struct { OpenStdin bool // Open stdin Env []string Cmd []string + Image string // Name of the image as it was passed by the operator (eg. could be symbolic) } -func ParseRun(args []string) (string, *Config, error) { +func ParseRun(args []string) (*Config, error) { cmd := flag.NewFlagSet("", flag.ContinueOnError) cmd.SetOutput(ioutil.Discard) fl_user := cmd.String("u", "", "Username or UID") @@ -73,9 +74,8 @@ func ParseRun(args []string) (string, *Config, error) { var fl_env ListOpts cmd.Var(&fl_env, "e", "Set environment variables") if err := cmd.Parse(args); err != nil { - return "", nil, err + return nil, err } - image := cmd.Arg(0) config := &Config{ Ports: fl_ports, User: *fl_user, @@ -85,8 +85,9 @@ func ParseRun(args []string) (string, *Config, error) { Detach: *fl_detach, Env: fl_env, Cmd: cmd.Args()[1:], + Image: cmd.Arg(0), } - return image, config, nil + return config, nil } type NetworkSettings struct { diff --git a/container_test.go b/container_test.go index e9e6992b7b..3f9b02ab58 100644 --- a/container_test.go +++ b/container_test.go @@ -20,8 +20,8 @@ func TestCommitRun(t *testing.T) { } defer nuke(runtime) container1, err := runtime.Create( - GetTestImage(runtime).Id, &Config{ + Image: GetTestImage(runtime).Id, Cmd: []string{"/bin/sh", "-c", "echo hello > /world"}, Memory: 33554432, }, @@ -53,8 +53,8 @@ func TestCommitRun(t *testing.T) { // FIXME: Make a TestCommit that stops here and check docker.root/layers/img.id/world container2, err := runtime.Create( - img.Id, &Config{ + Image: img.Id, Memory: 33554432, Cmd: []string{"cat", "/world"}, }, @@ -86,8 +86,8 @@ func TestRun(t *testing.T) { } defer nuke(runtime) container, err := runtime.Create( - GetTestImage(runtime).Id, &Config{ + Image: GetTestImage(runtime).Id, Memory: 33554432, Cmd: []string{"ls", "-al"}, }, @@ -115,9 +115,9 @@ func TestOutput(t *testing.T) { } defer nuke(runtime) container, err := runtime.Create( - GetTestImage(runtime).Id, &Config{ - Cmd: []string{"echo", "-n", "foobar"}, + Image: GetTestImage(runtime).Id, + Cmd: []string{"echo", "-n", "foobar"}, }, ) if err != nil { @@ -139,8 +139,9 @@ func TestKill(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create(GetTestImage(runtime).Id, &Config{ - Cmd: []string{"cat", "/dev/zero"}, + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"cat", "/dev/zero"}, }, ) if err != nil { @@ -180,8 +181,10 @@ func TestExitCode(t *testing.T) { } defer nuke(runtime) - trueContainer, err := runtime.Create(GetTestImage(runtime).Id, &Config{ - Cmd: []string{"/bin/true", ""}, + trueContainer, err := runtime.Create(&Config{ + + Image: GetTestImage(runtime).Id, + Cmd: []string{"/bin/true", ""}, }, ) if err != nil { @@ -192,8 +195,9 @@ func TestExitCode(t *testing.T) { t.Fatal(err) } - falseContainer, err := runtime.Create(GetTestImage(runtime).Id, &Config{ - Cmd: []string{"/bin/false", ""}, + falseContainer, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"/bin/false", ""}, }, ) if err != nil { @@ -219,8 +223,9 @@ func TestRestart(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create(GetTestImage(runtime).Id, &Config{ - Cmd: []string{"echo", "-n", "foobar"}, + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"echo", "-n", "foobar"}, }, ) if err != nil { @@ -251,8 +256,9 @@ func TestRestartStdin(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create(GetTestImage(runtime).Id, &Config{ - Cmd: []string{"cat"}, + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"cat"}, OpenStdin: true, }, @@ -300,8 +306,9 @@ func TestUser(t *testing.T) { defer nuke(runtime) // Default user must be root - container, err := runtime.Create(GetTestImage(runtime).Id, &Config{ - Cmd: []string{"id"}, + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"id"}, }, ) if err != nil { @@ -317,8 +324,9 @@ func TestUser(t *testing.T) { } // Set a username - container, err = runtime.Create(GetTestImage(runtime).Id, &Config{ - Cmd: []string{"id"}, + container, err = runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"id"}, User: "root", }, @@ -336,8 +344,9 @@ func TestUser(t *testing.T) { } // Set a UID - container, err = runtime.Create(GetTestImage(runtime).Id, &Config{ - Cmd: []string{"id"}, + container, err = runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"id"}, User: "0", }, @@ -355,8 +364,9 @@ func TestUser(t *testing.T) { } // Set a different user by uid - container, err = runtime.Create(GetTestImage(runtime).Id, &Config{ - Cmd: []string{"id"}, + container, err = runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"id"}, User: "1", }, @@ -376,8 +386,9 @@ func TestUser(t *testing.T) { } // Set a different user by username - container, err = runtime.Create(GetTestImage(runtime).Id, &Config{ - Cmd: []string{"id"}, + container, err = runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"id"}, User: "daemon", }, @@ -402,8 +413,9 @@ func TestMultipleContainers(t *testing.T) { } defer nuke(runtime) - container1, err := runtime.Create(GetTestImage(runtime).Id, &Config{ - Cmd: []string{"cat", "/dev/zero"}, + container1, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"cat", "/dev/zero"}, }, ) if err != nil { @@ -411,8 +423,9 @@ func TestMultipleContainers(t *testing.T) { } defer runtime.Destroy(container1) - container2, err := runtime.Create(GetTestImage(runtime).Id, &Config{ - Cmd: []string{"cat", "/dev/zero"}, + container2, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"cat", "/dev/zero"}, }, ) if err != nil { @@ -452,8 +465,9 @@ func TestStdin(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create(GetTestImage(runtime).Id, &Config{ - Cmd: []string{"cat"}, + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"cat"}, OpenStdin: true, }, @@ -485,8 +499,9 @@ func TestTty(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create(GetTestImage(runtime).Id, &Config{ - Cmd: []string{"cat"}, + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"cat"}, OpenStdin: true, }, @@ -518,8 +533,9 @@ func TestEnv(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create(GetTestImage(runtime).Id, &Config{ - Cmd: []string{"/usr/bin/env"}, + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"/usr/bin/env"}, }, ) if err != nil { @@ -590,8 +606,9 @@ func TestLXCConfig(t *testing.T) { memMin := 33554432 memMax := 536870912 mem := memMin + rand.Intn(memMax-memMin) - container, err := runtime.Create(GetTestImage(runtime).Id, &Config{ - Cmd: []string{"/bin/true"}, + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"/bin/true"}, Hostname: "foobar", Memory: int64(mem), @@ -616,8 +633,9 @@ func BenchmarkRunSequencial(b *testing.B) { } defer nuke(runtime) for i := 0; i < b.N; i++ { - container, err := runtime.Create(GetTestImage(runtime).Id, &Config{ - Cmd: []string{"echo", "-n", "foo"}, + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"echo", "-n", "foo"}, }, ) if err != nil { @@ -650,8 +668,9 @@ func BenchmarkRunParallel(b *testing.B) { complete := make(chan error) tasks = append(tasks, complete) go func(i int, complete chan error) { - container, err := runtime.Create(GetTestImage(runtime).Id, &Config{ - Cmd: []string{"echo", "-n", "foo"}, + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"echo", "-n", "foo"}, }, ) if err != nil { diff --git a/runtime.go b/runtime.go index a9efe446d7..da95dfe365 100644 --- a/runtime.go +++ b/runtime.go @@ -64,9 +64,9 @@ func (runtime *Runtime) containerRoot(id string) string { return path.Join(runtime.repository, id) } -func (runtime *Runtime) Create(image string, config *Config) (*Container, error) { +func (runtime *Runtime) Create(config *Config) (*Container, error) { // Lookup image - img, err := runtime.repositories.LookupImage(image) + img, err := runtime.repositories.LookupImage(config.Image) if err != nil { return nil, err } diff --git a/runtime_test.go b/runtime_test.go index a2d1e68a63..76882f95a9 100644 --- a/runtime_test.go +++ b/runtime_test.go @@ -112,8 +112,9 @@ func TestRuntimeCreate(t *testing.T) { if len(runtime.List()) != 0 { t.Errorf("Expected 0 containers, %v found", len(runtime.List())) } - container, err := runtime.Create(GetTestImage(runtime).Id, &Config{ - Cmd: []string{"ls", "-al"}, + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"ls", "-al"}, }, ) if err != nil { @@ -158,8 +159,9 @@ func TestDestroy(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create(GetTestImage(runtime).Id, &Config{ - Cmd: []string{"ls", "-al"}, + container, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"ls", "-al"}, }, ) if err != nil { @@ -204,8 +206,9 @@ func TestGet(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container1, err := runtime.Create(GetTestImage(runtime).Id, &Config{ - Cmd: []string{"ls", "-al"}, + container1, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"ls", "-al"}, }, ) if err != nil { @@ -213,8 +216,9 @@ func TestGet(t *testing.T) { } defer runtime.Destroy(container1) - container2, err := runtime.Create(GetTestImage(runtime).Id, &Config{ - Cmd: []string{"ls", "-al"}, + container2, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"ls", "-al"}, }, ) if err != nil { @@ -222,8 +226,9 @@ func TestGet(t *testing.T) { } defer runtime.Destroy(container2) - container3, err := runtime.Create(GetTestImage(runtime).Id, &Config{ - Cmd: []string{"ls", "-al"}, + container3, err := runtime.Create(&Config{ + Image: GetTestImage(runtime).Id, + Cmd: []string{"ls", "-al"}, }, ) if err != nil { @@ -264,8 +269,9 @@ func TestRestore(t *testing.T) { } // Create a container with one instance of docker - container1, err := runtime1.Create(GetTestImage(runtime1).Id, &Config{ - Cmd: []string{"ls", "-al"}, + container1, err := runtime1.Create(&Config{ + Image: GetTestImage(runtime1).Id, + Cmd: []string{"ls", "-al"}, }, ) if err != nil { From f37c432bd5fc40ee45b5d31ea750c59c544f2df5 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 23 Mar 2013 14:18:35 -0700 Subject: [PATCH 75/80] Fixed 'docker inspect' to exit silently when an image doesn't exist --- commands.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/commands.go b/commands.go index 0933171448..16f97f12dd 100644 --- a/commands.go +++ b/commands.go @@ -234,9 +234,7 @@ func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...str var obj interface{} if container := srv.runtime.Get(name); container != nil { obj = container - } else if image, err := srv.runtime.repositories.LookupImage(name); err != nil { - return err - } else if image != nil { + } else if image, err := srv.runtime.repositories.LookupImage(name); err == nil && image != nil { obj = image } else { // No output means the object does not exist From 0146c80c40fa5b793fdda17ed2c96d49bd90e2ed Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 23 Mar 2013 14:48:16 -0700 Subject: [PATCH 76/80] An image embeds the configuration of its parent container ('ContainerConfig') --- commands.go | 2 +- graph.go | 4 ++-- image.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/commands.go b/commands.go index 16f97f12dd..30e0a82546 100644 --- a/commands.go +++ b/commands.go @@ -311,7 +311,7 @@ func (srv *Server) CmdHistory(stdin io.ReadCloser, stdout io.Writer, args ...str fmt.Fprintf(w, "%s\t%s\t%s\n", srv.runtime.repositories.ImageName(img.Id), HumanDuration(time.Now().Sub(img.Created))+" ago", - strings.Join(img.ParentCommand, " "), + strings.Join(img.ContainerConfig.Cmd, " "), ) return nil }) diff --git a/graph.go b/graph.go index c6bb30353a..fed19175ff 100644 --- a/graph.go +++ b/graph.go @@ -55,8 +55,8 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment stri } if container != nil { img.Parent = container.Image - img.ParentContainer = container.Id - img.ParentCommand = append([]string{container.Path}, container.Args...) + img.Container = container.Id + img.ContainerConfig = *container.Config } if err := graph.Register(layerData, img); err != nil { return nil, err diff --git a/image.go b/image.go index 36bd839f5c..1a417a357b 100644 --- a/image.go +++ b/image.go @@ -19,8 +19,8 @@ type Image struct { Parent string `json:"parent,omitempty"` Comment string `json:"comment,omitempty"` Created time.Time `json:"created"` - ParentContainer string `json:"parent_container,omitempty"` - ParentCommand []string `json:"parent_command,omitempty"` + Container string `json:"container,omitempty"` + ContainerConfig Config `json:"container_config,omitempty"` graph *Graph } From 966cddf26bc5e86c47faab0ddc69b42e9bd8a9f2 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 22 Mar 2013 13:21:44 -0700 Subject: [PATCH 77/80] Add some verbosity to the push/pull --- commands.go | 6 +++--- registry.go | 23 ++++++++++++----------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/commands.go b/commands.go index a5cd227268..117523a0c2 100644 --- a/commands.go +++ b/commands.go @@ -438,7 +438,7 @@ func (srv *Server) CmdPush(stdin io.ReadCloser, stdout io.Writer, args ...string // If it fails, try to get the repository if localRepo, exists := srv.runtime.repositories.Repositories[local]; exists { fmt.Fprintf(stdout, "Pushing %s (%d tags) on %s...\n", local, len(localRepo), remote) - if err := srv.runtime.graph.PushRepository(remote, localRepo, srv.runtime.authConfig); err != nil { + if err := srv.runtime.graph.PushRepository(stdout, remote, localRepo, srv.runtime.authConfig); err != nil { return err } fmt.Fprintf(stdout, "Push completed\n") @@ -449,7 +449,7 @@ func (srv *Server) CmdPush(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } fmt.Fprintf(stdout, "Pushing image %s..\n", img.Id) - err = srv.runtime.graph.PushImage(img, srv.runtime.authConfig) + err = srv.runtime.graph.PushImage(stdout, img, srv.runtime.authConfig) if err != nil { return err } @@ -482,7 +482,7 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string } // FIXME: Allow pull repo:tag fmt.Fprintf(stdout, "Pulling %s...\n", remote) - if err := srv.runtime.graph.PullRepository(remote, "", srv.runtime.repositories, srv.runtime.authConfig); err != nil { + if err := srv.runtime.graph.PullRepository(stdout, remote, "", srv.runtime.repositories, srv.runtime.authConfig); err != nil { return err } fmt.Fprintf(stdout, "Pull completed\n") diff --git a/registry.go b/registry.go index 4eb1e07cf0..309fe3c52c 100644 --- a/registry.go +++ b/registry.go @@ -19,7 +19,7 @@ const REGISTRY_ENDPOINT = auth.REGISTRY_SERVER + "/v1" func NewImgJson(src []byte) (*Image, error) { ret := &Image{} - fmt.Printf("Json string: {%s}\n", src) + Debugf("Json string: {%s}\n", src) // FIXME: Is there a cleaner way to "puryfy" the input json? src = []byte(strings.Replace(string(src), "null", "\"\"", -1)) @@ -161,10 +161,10 @@ func (graph *Graph) PullImage(imgId string, authConfig *auth.AuthConfig) error { } // FIXME: Handle the askedTag parameter -func (graph *Graph) PullRepository(remote, askedTag string, repositories *TagStore, authConfig *auth.AuthConfig) error { +func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, repositories *TagStore, authConfig *auth.AuthConfig) error { client := &http.Client{} - fmt.Printf("Pulling repo: %s\n", REGISTRY_ENDPOINT+"/users/"+remote) + fmt.Fprintf(stdout, "Pulling repo: %s\n", REGISTRY_ENDPOINT+"/users/"+remote) req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/users/"+remote, nil) if err != nil { @@ -202,7 +202,7 @@ func (graph *Graph) PullRepository(remote, askedTag string, repositories *TagSto } // Push a local image to the registry with its history if needed -func (graph *Graph) PushImage(imgOrig *Image, authConfig *auth.AuthConfig) error { +func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, authConfig *auth.AuthConfig) error { client := &http.Client{} // FIXME: Factorize the code @@ -214,7 +214,7 @@ func (graph *Graph) PushImage(imgOrig *Image, authConfig *auth.AuthConfig) error return fmt.Errorf("Error while retreiving the path for {%s}: %s", img.Id, err) } - Debugf("Pushing image [%s] on {%s}\n", img.Id, REGISTRY_ENDPOINT+"/images/"+img.Id+"/json") + fmt.Fprintf(stdout, "Pushing image [%s] on {%s}\n", img.Id, REGISTRY_ENDPOINT+"/images/"+img.Id+"/json") // FIXME: try json with UTF8 jsonData := strings.NewReader(string(jsonRaw)) @@ -231,11 +231,12 @@ func (graph *Graph) PushImage(imgOrig *Image, authConfig *auth.AuthConfig) error "Error: Internal server error trying to push image {%s} (json): %s", img.Id, err) } - fmt.Printf("Pushing return status: %d\n", res.StatusCode) + Debugf("Pushing return status: %d\n", res.StatusCode) switch res.StatusCode { case 204: // Case where the image is already on the Registry // FIXME: Do not be silent? + fmt.Fprintf(stdout, "The image %s is already up to date on the registry.\n", img.Id) return nil case 400: return fmt.Errorf("Error: Invalid Json") @@ -324,7 +325,7 @@ func (graph *Graph) pushTag(remote, revision, tag string, authConfig *auth.AuthC } return err } - fmt.Printf("Result of push tag: %d\n", res.StatusCode) + Debugf("Result of push tag: %d\n", res.StatusCode) switch res.StatusCode { default: return fmt.Errorf("Error %d\n", res.StatusCode) @@ -349,14 +350,14 @@ func (graph *Graph) LookupRemoteRepository(remote string, authConfig *auth.AuthC return true } -func (graph *Graph) pushPrimitive(remote, tag, imgId string, authConfig *auth.AuthConfig) error { +func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId string, authConfig *auth.AuthConfig) error { // CHeck if the local impage exists img, err := graph.Get(imgId) if err != nil { return err } // Push the image - if err = graph.PushImage(img, authConfig); err != nil { + if err = graph.PushImage(stdout, img, authConfig); err != nil { return err } // And then the tag @@ -368,7 +369,7 @@ func (graph *Graph) pushPrimitive(remote, tag, imgId string, authConfig *auth.Au // Push a repository to the registry. // Remote has the format '/ -func (graph *Graph) PushRepository(remote string, localRepo Repository, authConfig *auth.AuthConfig) error { +func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Repository, authConfig *auth.AuthConfig) error { // Check if the remote repository exists // FIXME: @lopter How to handle this? // if !graph.LookupRemoteRepository(remote, authConfig) { @@ -377,7 +378,7 @@ func (graph *Graph) PushRepository(remote string, localRepo Repository, authConf // For each image within the repo, push them for tag, imgId := range localRepo { - if err := graph.pushPrimitive(remote, tag, imgId, authConfig); err != nil { + if err := graph.pushPrimitive(stdout, remote, tag, imgId, authConfig); err != nil { // FIXME: Continue on error? return err } From dc2d930520cd528129330098e868c7fffe8349b9 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 22 Mar 2013 14:27:10 -0700 Subject: [PATCH 78/80] Remove the json alterations when decoding --- registry.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/registry.go b/registry.go index 309fe3c52c..62394d63fc 100644 --- a/registry.go +++ b/registry.go @@ -21,8 +21,6 @@ func NewImgJson(src []byte) (*Image, error) { Debugf("Json string: {%s}\n", src) // FIXME: Is there a cleaner way to "puryfy" the input json? - src = []byte(strings.Replace(string(src), "null", "\"\"", -1)) - if err := json.Unmarshal(src, ret); err != nil { return nil, err } From f43fbda2a464ba2ef26c88c25cdb933c144657f4 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 23 Mar 2013 16:17:01 -0700 Subject: [PATCH 79/80] No more dependency on sqlite --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7e8af5757c..851ddb4fce 100644 --- a/README.md +++ b/README.md @@ -227,7 +227,7 @@ Setting up a dev environment Instructions that have been verified to work on Ubuntu 12.10, ```bash -sudo apt-get -y install lxc wget bsdtar curl libsqlite3-dev golang git pkg-config +sudo apt-get -y install lxc wget bsdtar curl golang git export GOPATH=~/go/ export PATH=$GOPATH/bin:$PATH From d301c7b98c46fa0d1bba66be852a7453d108f028 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 23 Mar 2013 17:03:30 -0700 Subject: [PATCH 80/80] 'docker images' doesn't show all anonymous images by default - only anonymous heads --- commands.go | 9 ++++++++- graph.go | 49 ++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/commands.go b/commands.go index 117523a0c2..92232accc2 100644 --- a/commands.go +++ b/commands.go @@ -493,6 +493,7 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri cmd := rcli.Subcmd(stdout, "images", "[OPTIONS] [NAME]", "List images") //limit := cmd.Int("l", 0, "Only show the N most recent versions of each image") quiet := cmd.Bool("q", false, "only show numeric IDs") + fl_a := cmd.Bool("a", false, "show all images") if err := cmd.Parse(args); err != nil { return nil } @@ -508,7 +509,13 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri if !*quiet { fmt.Fprintf(w, "REPOSITORY\tTAG\tID\tCREATED\tPARENT\n") } - allImages, err := srv.runtime.graph.Map() + var allImages map[string]*Image + var err error + if *fl_a { + allImages, err = srv.runtime.graph.Map() + } else { + allImages, err = srv.runtime.graph.Heads() + } if err != nil { return err } diff --git a/graph.go b/graph.go index fed19175ff..35f092703f 100644 --- a/graph.go +++ b/graph.go @@ -141,20 +141,59 @@ func (graph *Graph) Map() (map[string]*Image, error) { } func (graph *Graph) All() ([]*Image, error) { + var images []*Image + err := graph.WalkAll(func(image *Image) { + images = append(images, image) + }) + return images, err +} + +func (graph *Graph) WalkAll(handler func(*Image)) error { files, err := ioutil.ReadDir(graph.Root) if err != nil { - return nil, err + return err } - var images []*Image for _, st := range files { if img, err := graph.Get(st.Name()); err != nil { // Skip image continue - } else { - images = append(images, img) + } else if handler != nil { + handler(img) } } - return images, nil + return nil +} + +func (graph *Graph) ByParent() (map[string][]*Image, error) { + byParent := make(map[string][]*Image) + err := graph.WalkAll(func(image *Image) { + image, err := graph.Get(image.Parent) + if err != nil { + return + } + if children, exists := byParent[image.Parent]; exists { + byParent[image.Parent] = []*Image{image} + } else { + byParent[image.Parent] = append(children, image) + } + }) + return byParent, err +} + +func (graph *Graph) Heads() (map[string]*Image, error) { + heads := make(map[string]*Image) + byParent, err := graph.ByParent() + if err != nil { + return nil, err + } + err = graph.WalkAll(func(image *Image) { + // If it's not in the byParent lookup table, then + // it's not a parent -> so it's a head! + if _, exists := byParent[image.Id]; !exists { + heads[image.Id] = image + } + }) + return heads, err } func (graph *Graph) imageRoot(id string) string {