Implement package redirects.

Add feature to redirect page requests for a non-canonical package path
to the canonical package path. This feature covers the following
scenarios:

- Non-canonical case used in GitHub user and repository names.
- Package import comments as described at http://goo.gl/MBL1n6.
- New Go sub-repository names as described at http://goo.gl/wPoqC5.

Packages at non-canonical paths are deleted from the database the next
time they are crawled.

Specific changes to the code are:

- Code in gddd-server/main.go used pdoc == nil to indicate that a
  package was not found. Update the code to use gosrc.NotFoundError to
  indicate that the package was not found.
- Add Redirect field to gosrc.NotFoundError. This field specifies the
  canonical path for a package if known.
- Update the code to set the gosrc.NotFoundError Redirect field as
  appropriate.
- Update the document page and refresh to redirect as appropriate. The
  API does not redirect.
- Update the sub-repo page (http://godoc.org/-/subrepo) and supporting
  indexing code for the new Go sub-repository names.

The package import comment feature is commented out because the feature
depends on Go 1.4.

database/index.go

@@ -57,7 +57,7 @@ func documentTerms(pdoc *doc.Package, score float64) []string {
 	projectRoot := normalizeProjectRoot(pdoc.ProjectRoot)
 	terms["project:"+projectRoot] = true
 
-	if strings.HasPrefix(pdoc.ImportPath, "code.google.com/p/go.") {
+	if strings.HasPrefix(pdoc.ImportPath, "golang.org/x/") {
 		terms["project:subrepo"] = true
 	}
 

doc/builder.go

@@ -509,7 +509,9 @@ func newPackage(dir *gosrc.Directory) (*Package, error) {
 	for _, env := range goEnvs {
 		ctxt.GOOS = env.GOOS
 		ctxt.GOARCH = env.GOARCH
-		bpkg, err = dir.Import(&ctxt, 0)
+		// TODO(garyburd): Change second argument to build.ImportComment when
+		// gddo is upgraded to Go 1.4.
+		bpkg, err = dir.Import(&ctxt, 0 /* build.ImportComment */)
 		if _, ok := err.(*build.NoGoError); !ok {
 			break
 		}
@@ -521,6 +523,20 @@ func newPackage(dir *gosrc.Directory) (*Package, error) {
 		return pkg, nil
 	}
 
+	/*
+	        TODO(garyburd): This block of code uses the import comment feature
+	        added in Go 1.4. Uncomment this block when gddo upgraded to Go 1.4.
+	        Also, change the second argument to dir.Import above from 0 to
+	        build.ImportComment.
+
+			if bpkg.ImportComment != "" && bpkg.ImportComment != dir.ImportPath {
+				return nil, gosrc.NotFoundError{
+					Message:  "not at canonical import path",
+					Redirect: bpkg.ImportComment,
+				}
+			}
+	*/
+
 	// Parse the Go files
 
 	files := make(map[string]*ast.File)

gddo-server/assets/templates/subrepo.html

@@ -19,4 +19,4 @@
   {{template "Pkgs" .pkgs}}
 {{end}}
 
-{{define "subrepo"}}<li><a href="https://code.google.com/p/go/source/browse/?repo={{.name}}">code.google.com/p/go.{{.name}}</a> — {{.desc}}{{end}}
+{{define "subrepo"}}<li><a href="https://code.google.com/p/go/source/browse/?repo={{.name}}">golang.org/x/{{.name}}</a> — {{.desc}}{{end}}

gddo-server/crawl.go

@@ -57,19 +57,23 @@ func crawlDoc(source string, importPath string, pdoc *doc.Package, hasSubdirs bo
 		// Go Frontend source tree mirror.
 		pdoc = nil
 		err = gosrc.NotFoundError{Message: "Go Frontend source tree mirror."}
+	} else if strings.HasPrefix(importPath, "code.google.com/p/go.") {
+		// Old import path for Go sub-repository.
+		pdoc = nil
+		err = gosrc.NotFoundError{Message: "old Go sub-repo", Redirect: "golang.org/x/" + importPath[len("code.google.com/p/go."):]}
 	} else if m := nestedProjectPat.FindStringIndex(importPath); m != nil && exists(importPath[m[0]+1:]) {
 		pdoc = nil
-		err = gosrc.NotFoundError{Message: "Copy of other project."}
+		err = gosrc.NotFoundError{Message: "copy of other project."}
 	} else if blocked, e := db.IsBlocked(importPath); blocked && e == nil {
 		pdoc = nil
-		err = gosrc.NotFoundError{Message: "Blocked."}
+		err = gosrc.NotFoundError{Message: "blocked."}
 	} else {
 		var pdocNew *doc.Package
 		pdocNew, err = doc.Get(httpClient, importPath, etag)
 		message = append(message, "fetch:", int64(time.Since(start)/time.Millisecond))
 		if err == nil && pdocNew.Name == "" && !hasSubdirs {
 			pdoc = nil
-			err = gosrc.NotFoundError{Message: "No Go files or subdirs"}
+			err = gosrc.NotFoundError{Message: "no Go files or subdirs"}
 		} else if err != gosrc.ErrNotModified {
 			pdoc = pdocNew
 		}
@@ -90,20 +94,21 @@ func crawlDoc(source string, importPath string, pdoc *doc.Package, hasSubdirs bo
 		if err := db.Put(pdoc, nextCrawl, false); err != nil {
 			log.Printf("ERROR db.Put(%q): %v", importPath, err)
 		}
+		return pdoc, nil
 	case err == gosrc.ErrNotModified:
 		message = append(message, "touch")
 		if err := db.SetNextCrawlEtag(pdoc.ProjectRoot, pdoc.Etag, nextCrawl); err != nil {
 			log.Printf("ERROR db.SetNextCrawl(%q): %v", importPath, err)
 		}
+		return pdoc, nil
 	case gosrc.IsNotFound(err):
 		message = append(message, "notfound:", err)
 		if err := db.Delete(importPath); err != nil {
 			log.Printf("ERROR db.Delete(%q): %v", importPath, err)
 		}
+		return nil, err
 	default:
 		message = append(message, "ERROR:", err)
 		return nil, err
 	}
-
-	return pdoc, nil
 }

gddo-server/main.go

@@ -76,7 +76,7 @@ func getDoc(path string, requestType int) (*doc.Package, []database.Package, err
 		// A hack in the database package uses the path "-" to represent the
 		// next document to crawl. Block "-" here so that requests to /- always
 		// return not found.
-		return nil, nil, nil
+		return nil, nil, &httpError{status: http.StatusNotFound}
 	}
 
 	pdoc, pkgs, nextCrawl, err := db.Get(path)
@@ -94,38 +94,45 @@ func getDoc(path string, requestType int) (*doc.Package, []database.Package, err
 		needsCrawl = nextCrawl.IsZero() && len(pkgs) > 0
 	}
 
-	if needsCrawl {
-		c := make(chan crawlResult, 1)
-		go func() {
-			pdoc, err := crawlDoc("web  ", path, pdoc, len(pkgs) > 0, nextCrawl)
-			c <- crawlResult{pdoc, err}
-		}()
-		var err error
-		timeout := *getTimeout
-		if pdoc == nil {
-			timeout = *firstGetTimeout
-		}
-		select {
-		case rr := <-c:
-			if rr.err == nil {
-				pdoc = rr.pdoc
-			}
-			err = rr.err
-		case <-time.After(timeout):
-			err = errUpdateTimeout
-		}
-		if err != nil {
-			if pdoc != nil {
-				log.Printf("Serving %q from database after error: %v", path, err)
-				err = nil
-			} else if err == errUpdateTimeout {
-				// Handle timeout on packages never seeen before as not found.
-				log.Printf("Serving %q as not found after timeout", path)
-				err = &httpError{status: http.StatusNotFound}
-			}
-		}
+	if !needsCrawl {
+		return pdoc, pkgs, nil
+	}
+
+	c := make(chan crawlResult, 1)
+	go func() {
+		pdoc, err := crawlDoc("web  ", path, pdoc, len(pkgs) > 0, nextCrawl)
+		c <- crawlResult{pdoc, err}
+	}()
+
+	timeout := *getTimeout
+	if pdoc == nil {
+		timeout = *firstGetTimeout
+	}
+
+	select {
+	case cr := <-c:
+		err = cr.err
+		if err == nil {
+			pdoc = cr.pdoc
+		}
+	case <-time.After(timeout):
+		err = errUpdateTimeout
+	}
+
+	switch {
+	case err == nil:
+		return pdoc, pkgs, nil
+	case gosrc.IsNotFound(err):
+		return nil, nil, err
+	case pdoc != nil:
+		log.Printf("Serving %q from database after error getting doc: %v", path, err)
+		return pdoc, pkgs, nil
+	case err == errUpdateTimeout:
+		log.Printf("Serving %q as not found after timeout getting doc", path)
+		return nil, nil, &httpError{status: http.StatusNotFound}
+	default:
+		return nil, nil, err
 	}
-	return pdoc, pkgs, err
 }
 
 func templateExt(req *http.Request) string {
@@ -198,7 +205,7 @@ func servePackage(resp http.ResponseWriter, req *http.Request) error {
 		p = p[len("/pkg"):]
 	}
 	if p != req.URL.Path {
-		http.Redirect(resp, req, p, 301)
+		http.Redirect(resp, req, p, http.StatusMovedPermanently)
 		return nil
 	}
 
@@ -219,6 +226,20 @@ func servePackage(resp http.ResponseWriter, req *http.Request) error {
 
 	importPath := strings.TrimPrefix(req.URL.Path, "/")
 	pdoc, pkgs, err := getDoc(importPath, requestType)
+
+	if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
+		// To prevent dumb clients from following redirect loops, respond with
+		// status 404 if the target document is not found.
+		if _, _, err := getDoc(e.Redirect, requestType); gosrc.IsNotFound(err) {
+			return &httpError{status: http.StatusNotFound}
+		}
+		u := "/" + e.Redirect
+		if req.URL.RawQuery != "" {
+			u += "?" + req.URL.RawQuery
+		}
+		http.Redirect(resp, req, u, http.StatusMovedPermanently)
+		return nil
+	}
 	if err != nil {
 		return err
 	}
@@ -332,7 +353,7 @@ func servePackage(resp http.ResponseWriter, req *http.Request) error {
 		if fname == "" {
 			break
 		}
-		http.Redirect(resp, req, fmt.Sprintf("?file=%s#%s", fname, id), 301)
+		http.Redirect(resp, req, fmt.Sprintf("?file=%s#%s", fname, id), http.StatusMovedPermanently)
 		return nil
 	case isView(req, "file"):
 		if srcFiles == nil {
@@ -412,7 +433,7 @@ func servePackage(resp http.ResponseWriter, req *http.Request) error {
 		if err != nil {
 			return err
 		}
-		http.Redirect(resp, req, u, 301)
+		http.Redirect(resp, req, u, http.StatusMovedPermanently)
 		return nil
 	case req.Form.Get("view") != "":
 		// Redirect deprecated view= queries.
@@ -430,7 +451,7 @@ func servePackage(resp http.ResponseWriter, req *http.Request) error {
 		if q != "" {
 			u := *req.URL
 			u.RawQuery = q
-			http.Redirect(resp, req, u.String(), 301)
+			http.Redirect(resp, req, u.String(), http.StatusMovedPermanently)
 			return nil
 		}
 	}
@@ -453,10 +474,14 @@ func serveRefresh(resp http.ResponseWriter, req *http.Request) error {
 	case <-time.After(*getTimeout):
 		err = errUpdateTimeout
 	}
+	if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
+		http.Redirect(resp, req, "/"+e.Redirect, http.StatusFound)
+		return nil
+	}
 	if err != nil {
 		return err
 	}
-	http.Redirect(resp, req, "/"+path, 302)
+	http.Redirect(resp, req, "/"+path, http.StatusFound)
 	return nil
 }
 
@@ -578,8 +603,12 @@ func serveHome(resp http.ResponseWriter, req *http.Request) error {
 
 	if gosrc.IsValidRemotePath(q) || (strings.Contains(q, "/") && gosrc.IsGoRepoPath(q)) {
 		pdoc, pkgs, err := getDoc(q, queryRequest)
+		if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
+			http.Redirect(resp, req, "/"+e.Redirect, http.StatusFound)
+			return nil
+		}
 		if err == nil && (pdoc != nil || len(pkgs) > 0) {
-			http.Redirect(resp, req, "/"+q, 302)
+			http.Redirect(resp, req, "/"+q, http.StatusFound)
 			return nil
 		}
 	}
@@ -729,6 +758,8 @@ func runHandler(resp http.ResponseWriter, req *http.Request,
 			logError(req, err, nil)
 		}
 		errfn(resp, req, e.status, e.err)
+	} else if gosrc.IsNotFound(err) {
+		errfn(resp, req, http.StatusNotFound, nil)
 	} else {
 		logError(req, err, nil)
 		errfn(resp, req, http.StatusInternalServerError, err)
@@ -918,14 +949,14 @@ func main() {
 	mux.Handle("/-/subrepo", handler(serveGoSubrepoIndex))
 	mux.Handle("/-/index", handler(serveIndex))
 	mux.Handle("/-/refresh", handler(serveRefresh))
-	mux.Handle("/a/index", http.RedirectHandler("/-/index", 301))
-	mux.Handle("/about", http.RedirectHandler("/-/about", 301))
+	mux.Handle("/a/index", http.RedirectHandler("/-/index", http.StatusMovedPermanently))
+	mux.Handle("/about", http.RedirectHandler("/-/about", http.StatusMovedPermanently))
 	mux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico"))
 	mux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html"))
 	mux.Handle("/humans.txt", staticServer.FileHandler("humans.txt"))
 	mux.Handle("/robots.txt", staticServer.FileHandler("robots.txt"))
 	mux.Handle("/BingSiteAuth.xml", staticServer.FileHandler("BingSiteAuth.xml"))
-	mux.Handle("/C", http.RedirectHandler("http://golang.org/doc/articles/c_go_cgo.html", 301))
+	mux.Handle("/C", http.RedirectHandler("http://golang.org/doc/articles/c_go_cgo.html", http.StatusMovedPermanently))
 	mux.Handle("/ajax.googleapis.com/", http.NotFoundHandler())
 	mux.Handle("/", handler(serveHome))
 

gosrc/client.go

@@ -22,7 +22,7 @@ type httpClient struct {
 
 func (c *httpClient) err(resp *http.Response) error {
 	if resp.StatusCode == 404 {
-		return NotFoundError{"Resource not found: " + resp.Request.URL.String()}
+		return NotFoundError{Message: "Resource not found: " + resp.Request.URL.String()}
 	}
 	if c.errFn != nil {
 		return c.errFn(resp)
@@ -82,7 +82,7 @@ func (c *httpClient) getJSON(url string, v interface{}) (*http.Response, error)
 	}
 	err = json.NewDecoder(resp.Body).Decode(v)
 	if _, ok := err.(*json.SyntaxError); ok {
-		err = NotFoundError{"JSON syntax error at " + url}
+		err = NotFoundError{Message: "JSON syntax error at " + url}
 	}
 	return resp, err
 }
@@ -115,7 +115,7 @@ func (c *httpClient) getFiles(urls []string, files []*File) error {
 			ch <- nil
 		}(i)
 	}
-	for _ = range files {
+	for range files {
 		if err := <-ch; err != nil {
 			return err
 		}

gosrc/github.go

@@ -35,6 +35,7 @@ func init() {
 var (
 	gitHubRawHeader     = http.Header{"Accept": {"application/vnd.github-blob.raw"}}
 	gitHubPreviewHeader = http.Header{"Accept": {"application/vnd.github.preview"}}
+	ownerRepoPat        = regexp.MustCompile(`^https://api.github.com/repos/([^/]+)/([^/]+)/`)
 )
 
 func gitHubError(resp *http.Response) error {
@@ -112,13 +113,18 @@ func getGitHubDir(client *http.Client, match map[string]string, savedEtag string
 	}
 
 	if len(contents) == 0 {
-		return nil, NotFoundError{"No files in directory."}
+		return nil, NotFoundError{Message: "No files in directory."}
 	}
 
-	// Because Github API URLs are case-insensitive, we check that the owner
-	// and repo returned from Github matches the one that we are requesting.
-	if !strings.HasPrefix(contents[0].GitURL, expand("https://api.github.com/repos/{owner}/{repo}/", match)) {
-		return nil, NotFoundError{"Github import path has incorrect case."}
+	// GitHub owner and repo names are case-insensitive. Redirect if requested
+	// names do not match the canonical names in API response.
+	if m := ownerRepoPat.FindStringSubmatch(contents[0].GitURL); m != nil && (m[1] != match["owner"] || m[2] != match["repo"]) {
+		match["owner"] = m[1]
+		match["repo"] = m[2]
+		return nil, NotFoundError{
+			Message:  "Github import path has incorrect case.",
+			Redirect: expand("github.com/{owner}/{repo}{dir}", match),
+		}
 	}
 
 	var files []*File
@@ -289,7 +295,7 @@ func getGistDir(client *http.Client, match map[string]string, savedEtag string)
 	}
 
 	if len(gist.History) == 0 {
-		return nil, NotFoundError{"History not found."}
+		return nil, NotFoundError{Message: "History not found."}
 	}
 	commit := gist.History[0].Version
 

gosrc/google.go

@@ -114,7 +114,7 @@ func getGoogleVCS(c *httpClient, match map[string]string) error {
 	}
 	m := googleRepoRe.FindSubmatch(p)
 	if m == nil {
-		return NotFoundError{"Could not VCS on Google Code project page."}
+		return NotFoundError{Message: "Could not find VCS on Google Code project page."}
 	}
 	match["vcs"] = string(m[1])
 	return nil

gosrc/gosrc.go

@@ -78,6 +78,9 @@ type Project struct {
 type NotFoundError struct {
 	// Diagnostic message describing why the directory was not found.
 	Message string
+
+	// Redirect specifies the path where package can be found.
+	Redirect string
 }
 
 func (e NotFoundError) Error() string {
@@ -130,7 +133,7 @@ func (s *service) match(importPath string) (map[string]string, error) {
 	m := s.pattern.FindStringSubmatch(importPath)
 	if m == nil {
 		if s.prefix != "" {
-			return nil, NotFoundError{"Import path prefix matches known service, but regexp does not."}
+			return nil, NotFoundError{Message: "Import path prefix matches known service, but regexp does not."}
 		}
 		return nil, nil
 	}
@@ -208,7 +211,7 @@ metaScan:
 				continue metaScan
 			}
 			if match != nil {
-				return nil, NotFoundError{"More than one <meta> found at " + scheme + "://" + importPath}
+				return nil, NotFoundError{Message: "More than one <meta> found at " + scheme + "://" + importPath}
 			}
 
 			projectRoot, vcs, repo := f[0], f[1], f[2]
@@ -216,7 +219,7 @@ metaScan:
 			repo = strings.TrimSuffix(repo, "."+vcs)
 			i := strings.Index(repo, "://")
 			if i < 0 {
-				return nil, NotFoundError{"Bad repo URL in <meta>."}
+				return nil, NotFoundError{Message: "Bad repo URL in <meta>."}
 			}
 			proto := repo[:i]
 			repo = repo[i+len("://"):]
@@ -239,7 +242,7 @@ metaScan:
 		}
 	}
 	if match == nil {
-		return nil, NotFoundError{"<meta> not found."}
+		return nil, NotFoundError{Message: "<meta> not found."}
 	}
 	return match, nil
 }
@@ -261,7 +264,7 @@ func getDynamic(client *http.Client, importPath, etag string) (*Directory, error
 			return nil, err
 		}
 		if rootMatch["projectRoot"] != match["projectRoot"] {
-			return nil, NotFoundError{"Project root mismatch."}
+			return nil, NotFoundError{Message: "Project root mismatch."}
 		}
 	}
 
@@ -325,7 +328,7 @@ func Get(client *http.Client, importPath string, etag string) (dir *Directory, e
 	}
 
 	if err == errNoMatch {
-		err = NotFoundError{"Import path not valid:"}
+		err = NotFoundError{Message: "Import path not valid:"}
 	}
 
 	return dir, err
@@ -335,7 +338,7 @@ func Get(client *http.Client, importPath string, etag string) (dir *Directory, e
 func GetPresentation(client *http.Client, importPath string) (*Presentation, error) {
 	ext := path.Ext(importPath)
 	if ext != ".slide" && ext != ".article" {
-		return nil, NotFoundError{"unknown file extension."}
+		return nil, NotFoundError{Message: "unknown file extension."}
 	}
 
 	importPath, file := path.Split(importPath)
@@ -353,7 +356,7 @@ func GetPresentation(client *http.Client, importPath string) (*Presentation, err
 			return s.getPresentation(client, match)
 		}
 	}
-	return nil, NotFoundError{"path does not match registered service"}
+	return nil, NotFoundError{Message: "path does not match registered service"}
 }
 
 // GetProject gets information about a repository.
@@ -370,5 +373,5 @@ func GetProject(client *http.Client, importPath string) (*Project, error) {
 			return s.getProject(client, match)
 		}
 	}
-	return nil, NotFoundError{"path does not match registered service"}
+	return nil, NotFoundError{Message: "path does not match registered service"}
 }

gosrc/launchpad.go

@@ -110,7 +110,7 @@ func getLaunchpadDir(client *http.Client, match map[string]string, savedEtag str
 	}
 
 	if !inTree {
-		return nil, NotFoundError{"Directory tree does not contain Go files."}
+		return nil, NotFoundError{Message: "Directory tree does not contain Go files."}
 	}
 
 	sort.Sort(byHash(hash))

gosrc/util.go

@@ -21,7 +21,7 @@ func bestTag(tags map[string]string, defaultTag string) (string, string, error)
 	if commit, ok := tags[defaultTag]; ok {
 		return defaultTag, commit, nil
 	}
-	return "", "", NotFoundError{"Tag or branch not found."}
+	return "", "", NotFoundError{Message: "Tag or branch not found."}
 }
 
 // expand replaces {k} in template with match[k] or subs[atoi(k)] if k is not in match.

gosrc/vcs.go

@@ -119,7 +119,7 @@ func downloadGit(schemes []string, repo, savedEtag string) (string, string, erro
 	}
 
 	if scheme == "" {
-		return "", "", NotFoundError{"VCS not found"}
+		return "", "", NotFoundError{Message: "VCS not found"}
 	}
 
 	tags := make(map[string]string)
@@ -173,7 +173,7 @@ func downloadGit(schemes []string, repo, savedEtag string) (string, string, erro
 func getVCSDir(client *http.Client, match map[string]string, etagSaved string) (*Directory, error) {
 	cmd := vcsCmds[match["vcs"]]
 	if cmd == nil {
-		return nil, NotFoundError{expand("VCS not supported: {vcs}", match)}
+		return nil, NotFoundError{Message: expand("VCS not supported: {vcs}", match)}
 	}
 
 	scheme := match["scheme"]
@@ -211,7 +211,7 @@ func getVCSDir(client *http.Client, match map[string]string, etagSaved string) (
 	f, err := os.Open(d)
 	if err != nil {
 		if os.IsNotExist(err) {
-			err = NotFoundError{err.Error()}
+			err = NotFoundError{Message: err.Error()}
 		}
 		return nil, err
 	}