Merge pull request #12769 from calavera/git_build_fragment

Use git url fragment to specify build reference and dir context.

docs/sources/reference/commandline/cli.md

@@ -652,13 +652,36 @@ an [*ADD*](/reference/builder/#add) instruction to reference a file in the
 context.
 
 The `URL` parameter can specify the location of a Git repository;
-the repository acts as the build context.  The system recursively clones the repository
+the repository acts as the build context. The system recursively clones the repository
 and its submodules using a `git clone --depth 1 --recursive` command.
 This command runs in a temporary directory on your local host.
 After the command succeeds, the directory is sent to the Docker daemon as the context.
 Local clones give you the ability to access private repositories using
 local user credentials, VPN's, and so forth.
 
+Git URLs accept context configuration in their fragment section, separated by a colon `:`.
+The first part represents the reference that Git will check out, this can be either
+a branch, a tag, or a commit SHA. The second part represents a subdirectory
+inside the repository that will be used as a build context.
+
+For example, run this command to use a directory called `docker` in the branch `container`:
+
+      $ docker build https://github.com/docker/rootfs.git#container:docker
+
+The following table represents all the valid suffixes with their build contexts:
+
+Build Syntax Suffix | Commit Used | Build Context Used
+--------------------|-------------|-------------------
+`myrepo.git` | `refs/heads/master` | `/`
+`myrepo.git#mytag` | `refs/tags/mytag` | `/`
+`myrepo.git#mybranch` | `refs/heads/mybranch` | `/`
+`myrepo.git#abcdef` | `sha1 = abcdef` | `/`
+`myrepo.git#:myfolder` | `refs/heads/master` | `/myfolder`
+`myrepo.git#master:myfolder` | `refs/heads/master` | `/myfolder`
+`myrepo.git#mytag:myfolder` | `refs/tags/mytag` | `/myfolder`
+`myrepo.git#mybranch:myfolder` | `refs/heads/mybranch` | `/myfolder`
+`myrepo.git#abcdef:myfolder` | `sha1 = abcdef` | `/myfolder`
+
 Instead of specifying a context, you can pass a single Dockerfile in the
 `URL` or pipe the file in via `STDIN`.  To pipe a Dockerfile from `STDIN`:
 

integration-cli/docker_cli_build_test.go

@@ -4118,6 +4118,35 @@ func (s *DockerSuite) TestBuildFromGIT(c *check.C) {
 	}
 }
 
+func (s *DockerSuite) TestBuildFromGITWithContext(c *check.C) {
+	name := "testbuildfromgit"
+	defer deleteImages(name)
+	git, err := fakeGIT("repo", map[string]string{
+		"docker/Dockerfile": `FROM busybox
+					ADD first /first
+					RUN [ -f /first ]
+					MAINTAINER docker`,
+		"docker/first": "test git data",
+	}, true)
+	if err != nil {
+		c.Fatal(err)
+	}
+	defer git.Close()
+
+	u := fmt.Sprintf("%s#master:docker", git.RepoURL)
+	_, err = buildImageFromPath(name, u, true)
+	if err != nil {
+		c.Fatal(err)
+	}
+	res, err := inspectField(name, "Author")
+	if err != nil {
+		c.Fatal(err)
+	}
+	if res != "docker" {
+		c.Fatalf("Maintainer should be docker, got %s", res)
+	}
+}
+
 func (s *DockerSuite) TestBuildCleanupCmdOnEntrypoint(c *check.C) {
 	name := "testbuildcmdcleanuponentrypoint"
 	if _, err := buildImage(name,

pkg/urlutil/git.go

@@ -1,6 +1,9 @@
 package urlutil
 
-import "strings"
+import (
+	"regexp"
+	"strings"
+)
 
 var (
 	validPrefixes = []string{
@@ -8,11 +11,13 @@ var (
 		"github.com/",
 		"git@",
 	}
+
+	urlPathWithFragmentSuffix = regexp.MustCompile(".git(?:#.+)?$")
 )
 
 // IsGitURL returns true if the provided str is a git repository URL.
 func IsGitURL(str string) bool {
-	if IsURL(str) && strings.HasSuffix(str, ".git") {
+	if IsURL(str) && urlPathWithFragmentSuffix.MatchString(str) {
 		return true
 	}
 	for _, prefix := range validPrefixes {

pkg/urlutil/git_test.go

@@ -9,10 +9,15 @@ var (
 		"git@bitbucket.org:atlassianlabs/atlassian-docker.git",
 		"https://github.com/docker/docker.git",
 		"http://github.com/docker/docker.git",
+		"http://github.com/docker/docker.git#branch",
+		"http://github.com/docker/docker.git#:dir",
 	}
 	incompleteGitUrls = []string{
 		"github.com/docker/docker",
 	}
+	invalidGitUrls = []string{
+		"http://github.com/docker/docker.git:#branch",
+	}
 )
 
 func TestValidGitTransport(t *testing.T) {
@@ -35,9 +40,16 @@ func TestIsGIT(t *testing.T) {
 			t.Fatalf("%q should be detected as valid Git url", url)
 		}
 	}
+
 	for _, url := range incompleteGitUrls {
 		if IsGitURL(url) == false {
 			t.Fatalf("%q should be detected as valid Git url", url)
 		}
 	}
+
+	for _, url := range invalidGitUrls {
+		if IsGitURL(url) == true {
+			t.Fatalf("%q should not be detected as valid Git prefix", url)
+		}
+	}
 }

utils/git.go

@@ -4,7 +4,10 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"net/url"
+	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 
 	"github.com/docker/docker/pkg/urlutil"
@@ -19,20 +22,26 @@ func GitClone(remoteURL string) (string, error) {
 		return "", err
 	}
 
-	clone := cloneArgs(remoteURL, root)
+	u, err := url.Parse(remoteURL)
+	if err != nil {
+		return "", err
+	}
 
-	if output, err := exec.Command("git", clone...).CombinedOutput(); err != nil {
+	fragment := u.Fragment
+	clone := cloneArgs(u, root)
+
+	if output, err := git(clone...); err != nil {
 		return "", fmt.Errorf("Error trying to use git: %s (%s)", err, output)
 	}
 
-	return root, nil
+	return checkoutGit(fragment, root)
 }
 
-func cloneArgs(remoteURL, root string) []string {
+func cloneArgs(remoteURL *url.URL, root string) []string {
 	args := []string{"clone", "--recursive"}
-	shallow := true
+	shallow := len(remoteURL.Fragment) == 0
 
-	if strings.HasPrefix(remoteURL, "http") {
+	if shallow && strings.HasPrefix(remoteURL.Scheme, "http") {
 		res, err := http.Head(fmt.Sprintf("%s/info/refs?service=git-upload-pack", remoteURL))
 		if err != nil || res.Header.Get("Content-Type") != "application/x-git-upload-pack-advertisement" {
 			shallow = false
@@ -43,5 +52,42 @@ func cloneArgs(remoteURL, root string) []string {
 		args = append(args, "--depth", "1")
 	}
 
-	return append(args, remoteURL, root)
+	if remoteURL.Fragment != "" {
+		remoteURL.Fragment = ""
+	}
+
+	return append(args, remoteURL.String(), root)
+}
+
+func checkoutGit(fragment, root string) (string, error) {
+	refAndDir := strings.SplitN(fragment, ":", 2)
+
+	if len(refAndDir[0]) != 0 {
+		if output, err := gitWithinDir(root, "checkout", refAndDir[0]); err != nil {
+			return "", fmt.Errorf("Error trying to use git: %s (%s)", err, output)
+		}
+	}
+
+	if len(refAndDir) > 1 && len(refAndDir[1]) != 0 {
+		newCtx := filepath.Join(root, refAndDir[1])
+		fi, err := os.Stat(newCtx)
+		if err != nil {
+			return "", err
+		}
+		if !fi.IsDir() {
+			return "", fmt.Errorf("Error setting git context, not a directory: %s", newCtx)
+		}
+		root = newCtx
+	}
+
+	return root, nil
+}
+
+func gitWithinDir(dir string, args ...string) ([]byte, error) {
+	a := []string{"--work-tree", dir, "--git-dir", filepath.Join(dir, ".git")}
+	return git(append(a, args...)...)
+}
+
+func git(args ...string) ([]byte, error) {
+	return exec.Command("git", args...).CombinedOutput()
 }

utils/git_test.go

@@ -2,9 +2,12 @@ package utils
 
 import (
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"os"
+	"path/filepath"
 	"reflect"
 	"testing"
 )
@@ -22,7 +25,7 @@ func TestCloneArgsSmartHttp(t *testing.T) {
 		w.Header().Set("Content-Type", fmt.Sprintf("application/x-%s-advertisement", q))
 	})
 
-	args := cloneArgs(gitURL, "/tmp")
+	args := cloneArgs(serverURL, "/tmp")
 	exp := []string{"clone", "--recursive", "--depth", "1", gitURL, "/tmp"}
 	if !reflect.DeepEqual(args, exp) {
 		t.Fatalf("Expected %v, got %v", exp, args)
@@ -41,16 +44,132 @@ func TestCloneArgsDumbHttp(t *testing.T) {
 		w.Header().Set("Content-Type", "text/plain")
 	})
 
-	args := cloneArgs(gitURL, "/tmp")
+	args := cloneArgs(serverURL, "/tmp")
 	exp := []string{"clone", "--recursive", gitURL, "/tmp"}
 	if !reflect.DeepEqual(args, exp) {
 		t.Fatalf("Expected %v, got %v", exp, args)
 	}
 }
+
 func TestCloneArgsGit(t *testing.T) {
-	args := cloneArgs("git://github.com/docker/docker", "/tmp")
+	u, _ := url.Parse("git://github.com/docker/docker")
+	args := cloneArgs(u, "/tmp")
 	exp := []string{"clone", "--recursive", "--depth", "1", "git://github.com/docker/docker", "/tmp"}
 	if !reflect.DeepEqual(args, exp) {
 		t.Fatalf("Expected %v, got %v", exp, args)
 	}
 }
+
+func TestCloneArgsStripFragment(t *testing.T) {
+	u, _ := url.Parse("git://github.com/docker/docker#test")
+	args := cloneArgs(u, "/tmp")
+	exp := []string{"clone", "--recursive", "git://github.com/docker/docker", "/tmp"}
+	if !reflect.DeepEqual(args, exp) {
+		t.Fatalf("Expected %v, got %v", exp, args)
+	}
+}
+
+func TestCheckoutGit(t *testing.T) {
+	root, err := ioutil.TempDir("", "docker-build-git-checkout")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(root)
+
+	gitDir := filepath.Join(root, "repo")
+	_, err = git("init", gitDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err = gitWithinDir(gitDir, "config", "user.email", "test@docker.com"); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err = gitWithinDir(gitDir, "config", "user.name", "Docker test"); err != nil {
+		t.Fatal(err)
+	}
+
+	if err = ioutil.WriteFile(filepath.Join(gitDir, "Dockerfile"), []byte("FROM scratch"), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	subDir := filepath.Join(gitDir, "subdir")
+	if err = os.Mkdir(subDir, 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	if err = ioutil.WriteFile(filepath.Join(subDir, "Dockerfile"), []byte("FROM scratch\nEXPOSE 5000"), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err = gitWithinDir(gitDir, "add", "-A"); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err = gitWithinDir(gitDir, "commit", "-am", "First commit"); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err = gitWithinDir(gitDir, "checkout", "-b", "test"); err != nil {
+		t.Fatal(err)
+	}
+
+	if err = ioutil.WriteFile(filepath.Join(gitDir, "Dockerfile"), []byte("FROM scratch\nEXPOSE 3000"), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	if err = ioutil.WriteFile(filepath.Join(subDir, "Dockerfile"), []byte("FROM busybox\nEXPOSE 5000"), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err = gitWithinDir(gitDir, "add", "-A"); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err = gitWithinDir(gitDir, "commit", "-am", "Branch commit"); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err = gitWithinDir(gitDir, "checkout", "master"); err != nil {
+		t.Fatal(err)
+	}
+
+	cases := []struct {
+		frag string
+		exp  string
+		fail bool
+	}{
+		{"", "FROM scratch", false},
+		{"master", "FROM scratch", false},
+		{":subdir", "FROM scratch\nEXPOSE 5000", false},
+		{":nosubdir", "", true},   // missing directory error
+		{":Dockerfile", "", true}, // not a directory error
+		{"master:nosubdir", "", true},
+		{"master:subdir", "FROM scratch\nEXPOSE 5000", false},
+		{"test", "FROM scratch\nEXPOSE 3000", false},
+		{"test:", "FROM scratch\nEXPOSE 3000", false},
+		{"test:subdir", "FROM busybox\nEXPOSE 5000", false},
+	}
+
+	for _, c := range cases {
+		r, err := checkoutGit(c.frag, gitDir)
+
+		fail := err != nil
+		if fail != c.fail {
+			t.Fatalf("Expected %v failure, error was %v\n", c.fail, err)
+		}
+		if c.fail {
+			continue
+		}
+
+		b, err := ioutil.ReadFile(filepath.Join(r, "Dockerfile"))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if string(b) != c.exp {
+			t.Fatalf("Expected %v, was %v\n", c.exp, string(b))
+		}
+	}
+}