Various improvements and fixes.

database/index.go

@@ -96,7 +96,11 @@ func documentTerms(pdoc *doc.Package, score float64) []string {
 }
 
 func documentScore(pdoc *doc.Package) float64 {
-	if pdoc.Name == "" || pdoc.IsCmd || len(pdoc.Errors) > 0 || strings.HasSuffix(pdoc.ImportPath, ".go") {
+	if pdoc.Name == "" ||
+		pdoc.IsCmd ||
+		len(pdoc.Errors) > 0 ||
+		strings.HasSuffix(pdoc.ImportPath, ".go") ||
+		strings.HasPrefix(pdoc.ImportPath, "gist.github.com/") {
 		return 0
 	}
 

doc/builder.go

@@ -139,6 +139,20 @@ func addReferences(references map[string]bool, s []byte) {
 	}
 }
 
+type byFuncName []*doc.Func
+
+func (s byFuncName) Len() int           { return len(s) }
+func (s byFuncName) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+func (s byFuncName) Less(i, j int) bool { return s[i].Name < s[j].Name }
+
+func removeAssociations(dpkg *doc.Package) {
+	for _, t := range dpkg.Types {
+		dpkg.Funcs = append(dpkg.Funcs, t.Funcs...)
+		t.Funcs = nil
+	}
+	sort.Sort(byFuncName(dpkg.Funcs))
+}
+
 // builder holds the state used when building the documentation.
 type builder struct {
 	pdoc *Package
@@ -582,6 +596,10 @@ func (b *builder) build(srcs []*source) (*Package, error) {
 
 	dpkg := doc.New(apkg, b.pdoc.ImportPath, mode)
 
+	if b.pdoc.ImportPath == "builtin" {
+		removeAssociations(dpkg)
+	}
+
 	b.pdoc.Name = dpkg.Name
 	b.pdoc.Doc = strings.TrimRight(dpkg.Doc, " \t\n\r")
 	b.pdoc.Synopsis = synopsis(b.pdoc.Doc)

doc/get.go

@@ -258,7 +258,7 @@ func Get(client *http.Client, importPath string, etag string) (pdoc *Package, er
 	if pdoc != nil {
 		pdoc.Etag = versionPrefix + pdoc.Etag
 		if pdoc.ImportPath != importPath {
-			return nil, fmt.Errorf("Get: pdoc.ImportPath = %q, want %q", pdoc.ImportPath, importPath)
+			return nil, fmt.Errorf("get: pdoc.ImportPath = %q, want %q", pdoc.ImportPath, importPath)
 		}
 	}
 

doc/github.go

@@ -41,10 +41,10 @@ func getGitHubDoc(client *http.Client, match map[string]string, savedEtag string
 		Object struct {
 			Type string
 			Sha  string
-			Url  string
+			URL  string
 		}
 		Ref string
-		Url string
+		URL string
 	}
 
 	err := httpGetJSON(client, expand("https://api.github.com/repos/{owner}/{repo}/git/refs?{cred}", match), nil, &refs)
@@ -73,10 +73,10 @@ func getGitHubDoc(client *http.Client, match map[string]string, savedEtag string
 	}
 
 	var contents []*struct {
-		Type     string
-		Name     string
-		Git_URL  string
-		HTML_URL string
+		Type    string
+		Name    string
+		GitURL  string `json:"git_url"`
+		HTMLURL string `json:"html_url"`
 	}
 
 	err = httpGetJSON(client, expand("https://api.github.com/repos/{owner}/{repo}/contents{dir}?ref={tag}&{cred}", match), nil, &contents)
@@ -90,7 +90,7 @@ func getGitHubDoc(client *http.Client, match map[string]string, savedEtag string
 
 	// Because Github API URLs are case-insensitive, we check that the owner
 	// and repo returned from Github matches the one that we are requesting.
-	if !strings.HasPrefix(contents[0].Git_URL, expand("https://api.github.com/repos/{owner}/{repo}/", match)) {
+	if !strings.HasPrefix(contents[0].GitURL, expand("https://api.github.com/repos/{owner}/{repo}/", match)) {
 		return nil, NotFoundError{"Github import path has incorrect case."}
 	}
 
@@ -106,58 +106,12 @@ func getGitHubDoc(client *http.Client, match map[string]string, savedEtag string
 		case isDocFile(item.Name):
 			files = append(files, &source{
 				name:      item.Name,
-				browseURL: item.HTML_URL,
-				rawURL:    item.Git_URL + "?" + gitHubCred,
+				browseURL: item.HTMLURL,
+				rawURL:    item.GitURL + "?" + gitHubCred,
 			})
 		}
 	}
 
-	/*
-		var tree struct {
-			Tree []struct {
-				Url  string
-				Path string
-				Type string
-			}
-			Url string
-		}
-
-		err = httpGetJSON(client, expand("https://api.github.com/repos/{owner}/{repo}/git/trees/{tag}?recursive=1&{cred}", match), nil, &tree)
-		if err != nil {
-			return nil, err
-		}
-
-		// Because Github API URLs are case-insensitive, we need to check that the
-		// userRepo returned from Github matches the one that we are requesting.
-		if !strings.HasPrefix(tree.Url, expand("https://api.github.com/repos/{owner}/{repo}/", match)) {
-			return nil, NotFoundError{"Github import path has incorrect case."}
-		}
-
-		inTree := false
-		dirPrefix := match["dir"]
-		if dirPrefix != "" {
-			dirPrefix = dirPrefix[1:] + "/"
-		}
-		var files []*source
-		for _, node := range tree.Tree {
-			if node.Type != "blob" || !strings.HasPrefix(node.Path, dirPrefix) {
-				continue
-			}
-			inTree = true
-			if d, f := path.Split(node.Path); d == dirPrefix && isDocFile(f) {
-				files = append(files, &source{
-					name:      f,
-					browseURL: expand("https://github.com/{owner}/{repo}/blob/{tag}/{0}", match, node.Path),
-					rawURL:    node.Url + "?" + gitHubCred,
-				})
-			}
-		}
-
-		if !inTree {
-			return nil, NotFoundError{"Directory tree does not contain Go files."}
-		}
-	*/
-
 	if err := fetchFiles(client, files, gitHubRawHeader); err != nil {
 		return nil, err
 	}

doc/google.go

@@ -27,7 +27,7 @@ var (
 	googleRevisionRe = regexp.MustCompile(`<h2>(?:[^ ]+ - )?Revision *([^:]+):`)
 	googleEtagRe     = regexp.MustCompile(`^(hg|git|svn)-`)
 	googleFileRe     = regexp.MustCompile(`<li><a href="([^"]+)"`)
-	googlePattern    = regexp.MustCompile(`^code\.google\.com/p/(?P<repo>[a-z0-9\-]+)(:?\.(?P<subrepo>[a-z0-9\-]+))?(?P<dir>/[a-z0-9A-Z_.\-/]+)?$`)
+	googlePattern    = regexp.MustCompile(`^code\.google\.com/(?P<pr>[pr])/(?P<repo>[a-z0-9\-]+)(:?\.(?P<subrepo>[a-z0-9\-]+))?(?P<dir>/[a-z0-9A-Z_.\-/]+)?$`)
 )
 
 func getGoogleDoc(client *http.Client, match map[string]string, savedEtag string) (*Package, error) {
@@ -45,13 +45,13 @@ func getGoogleDoc(client *http.Client, match map[string]string, savedEtag string
 	}
 
 	var etag string
-	if m := googleRevisionRe.FindSubmatch(p); m == nil {
+	m := googleRevisionRe.FindSubmatch(p)
+	if m == nil {
 		return nil, errors.New("Could not find revision for " + match["importPath"])
-	} else {
-		etag = expand("{vcs}-{0}", match, string(m[1]))
-		if etag == savedEtag {
-			return nil, ErrNotModified
-		}
+	}
+	etag = expand("{vcs}-{0}", match, string(m[1]))
+	if etag == savedEtag {
+		return nil, ErrNotModified
 	}
 
 	var subdirs []string
@@ -67,7 +67,7 @@ func getGoogleDoc(client *http.Client, match map[string]string, savedEtag string
 		case isDocFile(fname):
 			files = append(files, &source{
 				name:      fname,
-				browseURL: expand("http://code.google.com/p/{repo}/source/browse{dir}/{0}{query}", match, fname),
+				browseURL: expand("http://code.google.com/{pr}/{repo}/source/browse{dir}/{0}{query}", match, fname),
 				rawURL:    expand("http://{subrepo}{dot}{repo}.googlecode.com/{vcs}{dir}/{0}", match, fname),
 			})
 		}
@@ -79,19 +79,19 @@ func getGoogleDoc(client *http.Client, match map[string]string, savedEtag string
 
 	var projectURL string
 	if match["subrepo"] == "" {
-		projectURL = expand("https://code.google.com/p/{repo}/", match)
+		projectURL = expand("https://code.google.com/{pr}/{repo}/", match)
 	} else {
-		projectURL = expand("https://code.google.com/p/{repo}/source/browse?repo={subrepo}", match)
+		projectURL = expand("https://code.google.com/{pr}/{repo}/source/browse?repo={subrepo}", match)
 	}
 
 	b := &builder{
 		pdoc: &Package{
 			LineFmt:     "%s#%d",
 			ImportPath:  match["originalImportPath"],
-			ProjectRoot: expand("code.google.com/p/{repo}{dot}{subrepo}", match),
+			ProjectRoot: expand("code.google.com/{pr}/{repo}{dot}{subrepo}", match),
 			ProjectName: expand("{repo}{dot}{subrepo}", match),
 			ProjectURL:  projectURL,
-			BrowseURL:   expand("http://code.google.com/p/{repo}/source/browse{dir}/{query}", match),
+			BrowseURL:   expand("http://code.google.com/{pr}/{repo}/source/browse{dir}/{query}", match),
 			Etag:        etag,
 			VCS:         match["vcs"],
 		},
@@ -112,7 +112,7 @@ func setupGoogleMatch(match map[string]string) {
 
 func getGoogleVCS(client *http.Client, match map[string]string) error {
 	// Scrape the HTML project page to find the VCS.
-	p, err := httpGetBytes(client, expand("http://code.google.com/p/{repo}/source/checkout", match), nil)
+	p, err := httpGetBytes(client, expand("http://code.google.com/{pr}/{repo}/source/checkout", match), nil)
 	if err != nil {
 		return err
 	}
@@ -132,13 +132,13 @@ func getStandardDoc(client *http.Client, importPath string, savedEtag string) (*
 	}
 
 	var etag string
-	if m := googleRevisionRe.FindSubmatch(p); m == nil {
+	m := googleRevisionRe.FindSubmatch(p)
+	if m == nil {
 		return nil, errors.New("Could not find revision for " + importPath)
-	} else {
-		etag = string(m[1])
-		if etag == savedEtag {
-			return nil, ErrNotModified
-		}
+	}
+	etag = string(m[1])
+	if etag == savedEtag {
+		return nil, ErrNotModified
 	}
 
 	var files []*source

gddo-server/assets/templates/importers_robot.html

@@ -0,0 +1,10 @@
+{{define "Head"}}<title>{{.pdoc.PageName}} importers - GoDoc</title><meta name="robots" content="NOINDEX, NOFOLLOW">{{end}}
+
+{{define "Body"}}
+  {{template "ProjectNav" $}}
+  <h3>Packages that import {{$.pdoc.Name|html}}</h3>
+  <table class="table table-condensed">
+    <thead><tr><th>Path</th><th>Synopsis</th></tr></thead>
+    <tbody>{{range .pkgs}}<tr><td>{{.Path|importPath}}</td><td>{{.Synopsis|importPath}}</td></tr>{{end}}</tbody>
+  </table>
+{{end}}

gddo-server/main_test.go

@@ -0,0 +1,41 @@
+// Copyright 2013 Gary Burd
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"github.com/garyburd/indigo/web"
+	"testing"
+)
+
+var robotTests = []string{
+	"Mozilla/5.0 (compatible; TweetedTimes Bot/1.0; +http://tweetedtimes.com)",
+	"Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)",
+	"Mozilla/5.0 (compatible; MJ12bot/v1.4.3; http://www.majestic12.co.uk/bot.php?+)",
+	"Go 1.1 package http",
+	"Java/1.7.0_25	0.003	0.003",
+	"Python-urllib/2.6",
+	"Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)",
+	"Mozilla/5.0 (compatible; Ezooms/1.0; ezooms.bot@gmail.com)",
+	"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+}
+
+func TestRobots(t *testing.T) {
+	for _, tt := range robotTests {
+		req := web.Request{Header: web.Header{web.HeaderUserAgent: {tt}}}
+		if !isRobot(&req) {
+			t.Errorf("%s not a robot", tt)
+		}
+	}
+}

gddo-server/play.go

@@ -63,7 +63,7 @@ func findExample(pdoc *doc.Package, export, method, name string) *doc.Example {
 	return nil
 }
 
-var exampleIdPat = regexp.MustCompile(`([^-]+)(?:-([^-]+)(?:-(.*))?)?`)
+var exampleIdPat = regexp.MustCompile(`([^-]+)(?:-([^-]*)(?:-(.*))?)?`)
 
 func playURL(pdoc *doc.Package, id string) (string, error) {
 	if m := exampleIdPat.FindStringSubmatch(id); m != nil {